1289177Speter/* revprops.c --- everything needed to handle revprops in FSFS 2289177Speter * 3289177Speter * ==================================================================== 4289177Speter * Licensed to the Apache Software Foundation (ASF) under one 5289177Speter * or more contributor license agreements. See the NOTICE file 6289177Speter * distributed with this work for additional information 7289177Speter * regarding copyright ownership. The ASF licenses this file 8289177Speter * to you under the Apache License, Version 2.0 (the 9289177Speter * "License"); you may not use this file except in compliance 10289177Speter * with the License. You may obtain a copy of the License at 11289177Speter * 12289177Speter * http://www.apache.org/licenses/LICENSE-2.0 13289177Speter * 14289177Speter * Unless required by applicable law or agreed to in writing, 15289177Speter * software distributed under the License is distributed on an 16289177Speter * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17289177Speter * KIND, either express or implied. See the License for the 18289177Speter * specific language governing permissions and limitations 19289177Speter * under the License. 20289177Speter * ==================================================================== 21289177Speter */ 22289177Speter 23289177Speter#include <assert.h> 24289177Speter 25289177Speter#include "svn_pools.h" 26289177Speter#include "svn_hash.h" 27289177Speter#include "svn_dirent_uri.h" 28289177Speter 29289177Speter#include "fs_fs.h" 30289177Speter#include "revprops.h" 31289177Speter#include "util.h" 32289177Speter 33289177Speter#include "private/svn_subr_private.h" 34289177Speter#include "private/svn_string_private.h" 35289177Speter#include "../libsvn_fs/fs-loader.h" 36289177Speter 37289177Speter#include "svn_private_config.h" 38289177Speter 39289177Speter/* Give writing processes 10 seconds to replace an existing revprop 40289177Speter file with a new one. After that time, we assume that the writing 41289177Speter process got aborted and that we have re-read revprops. */ 42289177Speter#define REVPROP_CHANGE_TIMEOUT (10 * 1000000) 43289177Speter 44289177Spetersvn_error_t * 45289177Spetersvn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs, 46289177Speter svn_fs_upgrade_notify_t notify_func, 47289177Speter void *notify_baton, 48289177Speter svn_cancel_func_t cancel_func, 49289177Speter void *cancel_baton, 50289177Speter apr_pool_t *scratch_pool) 51289177Speter{ 52289177Speter fs_fs_data_t *ffd = fs->fsap_data; 53289177Speter const char *revprops_shard_path; 54289177Speter const char *revprops_pack_file_dir; 55289177Speter apr_int64_t shard; 56289177Speter apr_int64_t first_unpacked_shard 57289177Speter = ffd->min_unpacked_rev / ffd->max_files_per_dir; 58289177Speter 59289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 60289177Speter const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, 61289177Speter scratch_pool); 62289177Speter int compression_level = ffd->compress_packed_revprops 63289177Speter ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 64289177Speter : SVN_DELTA_COMPRESSION_LEVEL_NONE; 65289177Speter 66289177Speter /* first, pack all revprops shards to match the packed revision shards */ 67289177Speter for (shard = 0; shard < first_unpacked_shard; ++shard) 68289177Speter { 69289177Speter svn_pool_clear(iterpool); 70289177Speter 71289177Speter revprops_pack_file_dir = svn_dirent_join(revsprops_dir, 72289177Speter apr_psprintf(iterpool, 73289177Speter "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, 74289177Speter shard), 75289177Speter iterpool); 76289177Speter revprops_shard_path = svn_dirent_join(revsprops_dir, 77289177Speter apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), 78289177Speter iterpool); 79289177Speter 80289177Speter SVN_ERR(svn_fs_fs__pack_revprops_shard(revprops_pack_file_dir, 81289177Speter revprops_shard_path, 82289177Speter shard, ffd->max_files_per_dir, 83289177Speter (int)(0.9 * ffd->revprop_pack_size), 84289177Speter compression_level, 85289177Speter cancel_func, cancel_baton, 86289177Speter iterpool)); 87289177Speter if (notify_func) 88289177Speter SVN_ERR(notify_func(notify_baton, shard, 89289177Speter svn_fs_upgrade_pack_revprops, iterpool)); 90289177Speter } 91289177Speter 92289177Speter svn_pool_destroy(iterpool); 93289177Speter 94289177Speter return SVN_NO_ERROR; 95289177Speter} 96289177Speter 97289177Spetersvn_error_t * 98289177Spetersvn_fs_fs__upgrade_cleanup_pack_revprops(svn_fs_t *fs, 99289177Speter svn_fs_upgrade_notify_t notify_func, 100289177Speter void *notify_baton, 101289177Speter svn_cancel_func_t cancel_func, 102289177Speter void *cancel_baton, 103289177Speter apr_pool_t *scratch_pool) 104289177Speter{ 105289177Speter fs_fs_data_t *ffd = fs->fsap_data; 106289177Speter const char *revprops_shard_path; 107289177Speter apr_int64_t shard; 108289177Speter apr_int64_t first_unpacked_shard 109289177Speter = ffd->min_unpacked_rev / ffd->max_files_per_dir; 110289177Speter 111289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 112289177Speter const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, 113289177Speter scratch_pool); 114289177Speter 115289177Speter /* delete the non-packed revprops shards afterwards */ 116289177Speter for (shard = 0; shard < first_unpacked_shard; ++shard) 117289177Speter { 118289177Speter svn_pool_clear(iterpool); 119289177Speter 120289177Speter revprops_shard_path = svn_dirent_join(revsprops_dir, 121289177Speter apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), 122289177Speter iterpool); 123289177Speter SVN_ERR(svn_fs_fs__delete_revprops_shard(revprops_shard_path, 124289177Speter shard, 125289177Speter ffd->max_files_per_dir, 126289177Speter cancel_func, cancel_baton, 127289177Speter iterpool)); 128289177Speter if (notify_func) 129289177Speter SVN_ERR(notify_func(notify_baton, shard, 130289177Speter svn_fs_upgrade_cleanup_revprops, iterpool)); 131289177Speter } 132289177Speter 133289177Speter svn_pool_destroy(iterpool); 134289177Speter 135289177Speter return SVN_NO_ERROR; 136289177Speter} 137289177Speter 138289177Speter/* Container for all data required to access the packed revprop file 139289177Speter * for a given REVISION. This structure will be filled incrementally 140289177Speter * by read_pack_revprops() its sub-routines. 141289177Speter */ 142289177Spetertypedef struct packed_revprops_t 143289177Speter{ 144289177Speter /* revision number to read (not necessarily the first in the pack) */ 145289177Speter svn_revnum_t revision; 146289177Speter 147289177Speter /* current revprop generation. Used when populating the revprop cache */ 148289177Speter apr_int64_t generation; 149289177Speter 150289177Speter /* the actual revision properties */ 151289177Speter apr_hash_t *properties; 152289177Speter 153289177Speter /* their size when serialized to a single string 154289177Speter * (as found in PACKED_REVPROPS) */ 155289177Speter apr_size_t serialized_size; 156289177Speter 157289177Speter 158289177Speter /* name of the pack file (without folder path) */ 159289177Speter const char *filename; 160289177Speter 161289177Speter /* packed shard folder path */ 162289177Speter const char *folder; 163289177Speter 164289177Speter /* sum of values in SIZES */ 165289177Speter apr_size_t total_size; 166289177Speter 167289177Speter /* first revision in the pack (>= MANIFEST_START) */ 168289177Speter svn_revnum_t start_revision; 169289177Speter 170289177Speter /* size of the revprops in PACKED_REVPROPS */ 171289177Speter apr_array_header_t *sizes; 172289177Speter 173289177Speter /* offset of the revprops in PACKED_REVPROPS */ 174289177Speter apr_array_header_t *offsets; 175289177Speter 176289177Speter 177289177Speter /* concatenation of the serialized representation of all revprops 178289177Speter * in the pack, i.e. the pack content without header and compression */ 179289177Speter svn_stringbuf_t *packed_revprops; 180289177Speter 181289177Speter /* First revision covered by MANIFEST. 182289177Speter * Will equal the shard start revision or 1, for the 1st shard. */ 183289177Speter svn_revnum_t manifest_start; 184289177Speter 185289177Speter /* content of the manifest. 186289177Speter * Maps long(rev - MANIFEST_START) to const char* pack file name */ 187289177Speter apr_array_header_t *manifest; 188289177Speter} packed_revprops_t; 189289177Speter 190289177Speter/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. 191289177Speter * Also, put them into the revprop cache, if activated, for future use. 192289177Speter * Three more parameters are being used to update the revprop cache: FS is 193289177Speter * our file system, the revprops belong to REVISION and the global revprop 194289177Speter * GENERATION is used as well. 195289177Speter * 196289177Speter * The returned hash will be allocated in POOL, SCRATCH_POOL is being used 197289177Speter * for temporary allocations. 198289177Speter */ 199289177Speterstatic svn_error_t * 200289177Speterparse_revprop(apr_hash_t **properties, 201289177Speter svn_fs_t *fs, 202289177Speter svn_revnum_t revision, 203289177Speter apr_int64_t generation, 204289177Speter svn_string_t *content, 205289177Speter apr_pool_t *pool, 206289177Speter apr_pool_t *scratch_pool) 207289177Speter{ 208289177Speter svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); 209289177Speter *properties = apr_hash_make(pool); 210289177Speter 211289177Speter SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool), 212289177Speter apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.", 213289177Speter revision)); 214289177Speter 215289177Speter return SVN_NO_ERROR; 216289177Speter} 217289177Speter 218289177Speter/* Read the non-packed revprops for revision REV in FS, put them into the 219289177Speter * revprop cache if activated and return them in *PROPERTIES. GENERATION 220289177Speter * is the current revprop generation. 221289177Speter * 222289177Speter * If the data could not be read due to an otherwise recoverable error, 223289177Speter * leave *PROPERTIES unchanged. No error will be returned in that case. 224289177Speter * 225289177Speter * Allocations will be done in POOL. 226289177Speter */ 227289177Speterstatic svn_error_t * 228289177Speterread_non_packed_revprop(apr_hash_t **properties, 229289177Speter svn_fs_t *fs, 230289177Speter svn_revnum_t rev, 231289177Speter apr_int64_t generation, 232289177Speter apr_pool_t *pool) 233289177Speter{ 234289177Speter svn_stringbuf_t *content = NULL; 235289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 236289177Speter svn_boolean_t missing = FALSE; 237289177Speter int i; 238289177Speter 239289177Speter for (i = 0; 240289177Speter i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !missing && !content; 241289177Speter ++i) 242289177Speter { 243289177Speter svn_pool_clear(iterpool); 244289177Speter SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&content, 245289177Speter &missing, 246289177Speter svn_fs_fs__path_revprops(fs, rev, iterpool), 247289177Speter i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT , 248289177Speter iterpool)); 249289177Speter } 250289177Speter 251289177Speter if (content) 252289177Speter SVN_ERR(parse_revprop(properties, fs, rev, generation, 253289177Speter svn_stringbuf__morph_into_string(content), 254289177Speter pool, iterpool)); 255289177Speter 256289177Speter svn_pool_clear(iterpool); 257289177Speter 258289177Speter return SVN_NO_ERROR; 259289177Speter} 260289177Speter 261289177Speter/* Return the minimum length of any packed revprop file name in REVPROPS. */ 262289177Speterstatic apr_size_t 263289177Speterget_min_filename_len(packed_revprops_t *revprops) 264289177Speter{ 265289177Speter char number_buffer[SVN_INT64_BUFFER_SIZE]; 266289177Speter 267289177Speter /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being 268289177Speter * at least the first rev in the shard and <COUNT> having at least one 269289177Speter * digit. Thus, the minimum is 2 + #decimal places in the start rev. 270289177Speter */ 271289177Speter return svn__i64toa(number_buffer, revprops->manifest_start) + 2; 272289177Speter} 273289177Speter 274289177Speter/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST 275289177Speter * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. 276289177Speter */ 277289177Speterstatic svn_error_t * 278289177Speterget_revprop_packname(svn_fs_t *fs, 279289177Speter packed_revprops_t *revprops, 280289177Speter apr_pool_t *pool, 281289177Speter apr_pool_t *scratch_pool) 282289177Speter{ 283289177Speter fs_fs_data_t *ffd = fs->fsap_data; 284289177Speter svn_stringbuf_t *content = NULL; 285289177Speter const char *manifest_file_path; 286289177Speter int idx, rev_count; 287289177Speter char *buffer, *buffer_end; 288289177Speter const char **filenames, **filenames_end; 289289177Speter apr_size_t min_filename_len; 290289177Speter 291289177Speter /* Determine the dimensions. Rev 0 is excluded from the first shard. */ 292289177Speter rev_count = ffd->max_files_per_dir; 293289177Speter revprops->manifest_start 294289177Speter = revprops->revision - (revprops->revision % rev_count); 295289177Speter if (revprops->manifest_start == 0) 296289177Speter { 297289177Speter ++revprops->manifest_start; 298289177Speter --rev_count; 299289177Speter } 300289177Speter 301289177Speter revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*)); 302289177Speter 303289177Speter /* No line in the file can be less than this number of chars long. */ 304289177Speter min_filename_len = get_min_filename_len(revprops); 305289177Speter 306289177Speter /* Read the content of the manifest file */ 307289177Speter revprops->folder 308289177Speter = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, pool); 309289177Speter manifest_file_path 310289177Speter = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); 311289177Speter 312289177Speter SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, pool)); 313289177Speter 314289177Speter /* There CONTENT must have a certain minimal size and there no 315289177Speter * unterminated lines at the end of the file. Both guarantees also 316289177Speter * simplify the parser loop below. 317289177Speter */ 318289177Speter if ( content->len < rev_count * (min_filename_len + 1) 319289177Speter || content->data[content->len - 1] != '\n') 320289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 321289177Speter _("Packed revprop manifest for r%ld not " 322289177Speter "properly terminated"), revprops->revision); 323289177Speter 324289177Speter /* Chop (parse) the manifest CONTENT into filenames, one per line. 325289177Speter * We only have to replace all newlines with NUL and add all line 326289177Speter * starts to REVPROPS->MANIFEST. 327289177Speter * 328289177Speter * There must be exactly REV_COUNT lines and that is the number of 329289177Speter * lines we parse from BUFFER to FILENAMES. Set the end pointer for 330289177Speter * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid 331289177Speter * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN. 332289177Speter * 333289177Speter * Please note that this loop is performance critical for e.g. 'svn log'. 334289177Speter * It is run 1000x per revprop access, i.e. per revision and about 335289177Speter * 50 million times per sec (and CPU core). 336289177Speter */ 337289177Speter for (filenames = (const char **)revprops->manifest->elts, 338289177Speter filenames_end = filenames + rev_count, 339289177Speter buffer = content->data, 340289177Speter buffer_end = buffer + content->len - min_filename_len; 341289177Speter (filenames < filenames_end) && (buffer < buffer_end); 342289177Speter ++filenames) 343289177Speter { 344289177Speter /* BUFFER always points to the start of the next line / filename. */ 345289177Speter *filenames = buffer; 346289177Speter 347289177Speter /* Find the next EOL. This is guaranteed to stay within the CONTENT 348289177Speter * buffer because we left enough room after BUFFER_END and we know 349289177Speter * we will always see a newline as the last non-NUL char. */ 350289177Speter buffer += min_filename_len; 351289177Speter while (*buffer != '\n') 352289177Speter ++buffer; 353289177Speter 354289177Speter /* Found EOL. Turn it into the filename terminator and move BUFFER 355289177Speter * to the start of the next line or CONTENT buffer end. */ 356289177Speter *buffer = '\0'; 357289177Speter ++buffer; 358289177Speter } 359289177Speter 360289177Speter /* We must have reached the end of both buffers. */ 361289177Speter if (buffer < content->data + content->len) 362289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 363289177Speter _("Packed revprop manifest for r%ld " 364289177Speter "has too many entries"), revprops->revision); 365289177Speter 366289177Speter if (filenames < filenames_end) 367289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 368289177Speter _("Packed revprop manifest for r%ld " 369289177Speter "has too few entries"), revprops->revision); 370289177Speter 371289177Speter /* The target array has now exactly one entry per revision. */ 372289177Speter revprops->manifest->nelts = rev_count; 373289177Speter 374289177Speter /* Now get the file name */ 375289177Speter idx = (int)(revprops->revision - revprops->manifest_start); 376289177Speter revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); 377289177Speter 378289177Speter return SVN_NO_ERROR; 379289177Speter} 380289177Speter 381289177Speter/* Return TRUE, if revision R1 and R2 refer to the same shard in FS. 382289177Speter */ 383289177Speterstatic svn_boolean_t 384289177Spetersame_shard(svn_fs_t *fs, 385289177Speter svn_revnum_t r1, 386289177Speter svn_revnum_t r2) 387289177Speter{ 388289177Speter fs_fs_data_t *ffd = fs->fsap_data; 389289177Speter return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir); 390289177Speter} 391289177Speter 392289177Speter/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, 393289177Speter * fill the START_REVISION member, and make PACKED_REVPROPS point to the 394289177Speter * first serialized revprop. If READ_ALL is set, initialize the SIZES 395289177Speter * and OFFSETS members as well. 396289177Speter * 397289177Speter * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as 398289177Speter * well as the SERIALIZED_SIZE member. If revprop caching has been 399289177Speter * enabled, parse all revprops in the pack and cache them. 400289177Speter */ 401289177Speterstatic svn_error_t * 402289177Speterparse_packed_revprops(svn_fs_t *fs, 403289177Speter packed_revprops_t *revprops, 404289177Speter svn_boolean_t read_all, 405289177Speter apr_pool_t *pool, 406289177Speter apr_pool_t *scratch_pool) 407289177Speter{ 408289177Speter svn_stream_t *stream; 409289177Speter apr_int64_t first_rev, count, i; 410289177Speter apr_off_t offset; 411289177Speter const char *header_end; 412289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 413289177Speter 414289177Speter /* decompress (even if the data is only "stored", there is still a 415289177Speter * length header to remove) */ 416289177Speter svn_stringbuf_t *compressed = revprops->packed_revprops; 417289177Speter svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool); 418289177Speter SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX)); 419289177Speter 420289177Speter /* read first revision number and number of revisions in the pack */ 421289177Speter stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); 422289177Speter SVN_ERR(svn_fs_fs__read_number_from_stream(&first_rev, NULL, stream, 423289177Speter iterpool)); 424289177Speter SVN_ERR(svn_fs_fs__read_number_from_stream(&count, NULL, stream, 425289177Speter iterpool)); 426289177Speter 427289177Speter /* Check revision range for validity. */ 428289177Speter if ( !same_shard(fs, revprops->revision, first_rev) 429289177Speter || !same_shard(fs, revprops->revision, first_rev + count - 1) 430289177Speter || count < 1) 431289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 432289177Speter _("Revprop pack for revision r%ld" 433289177Speter " contains revprops for r%ld .. r%ld"), 434289177Speter revprops->revision, 435289177Speter (svn_revnum_t)first_rev, 436289177Speter (svn_revnum_t)(first_rev + count -1)); 437289177Speter 438289177Speter /* Since start & end are in the same shard, it is enough to just test 439289177Speter * the FIRST_REV for being actually packed. That will also cover the 440289177Speter * special case of rev 0 never being packed. */ 441289177Speter if (!svn_fs_fs__is_packed_revprop(fs, first_rev)) 442289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 443289177Speter _("Revprop pack for revision r%ld" 444289177Speter " starts at non-packed revisions r%ld"), 445289177Speter revprops->revision, (svn_revnum_t)first_rev); 446289177Speter 447289177Speter /* make PACKED_REVPROPS point to the first char after the header. 448289177Speter * This is where the serialized revprops are. */ 449289177Speter header_end = strstr(uncompressed->data, "\n\n"); 450289177Speter if (header_end == NULL) 451289177Speter return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 452289177Speter _("Header end not found")); 453289177Speter 454289177Speter offset = header_end - uncompressed->data + 2; 455289177Speter 456289177Speter revprops->packed_revprops = svn_stringbuf_create_empty(pool); 457289177Speter revprops->packed_revprops->data = uncompressed->data + offset; 458289177Speter revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); 459289177Speter revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); 460289177Speter 461289177Speter /* STREAM still points to the first entry in the sizes list. */ 462289177Speter revprops->start_revision = (svn_revnum_t)first_rev; 463289177Speter if (read_all) 464289177Speter { 465289177Speter /* Init / construct REVPROPS members. */ 466289177Speter revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); 467289177Speter revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); 468289177Speter } 469289177Speter 470289177Speter /* Now parse, revision by revision, the size and content of each 471289177Speter * revisions' revprops. */ 472289177Speter for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i) 473289177Speter { 474289177Speter apr_int64_t size; 475289177Speter svn_string_t serialized; 476289177Speter svn_revnum_t revision = (svn_revnum_t)(first_rev + i); 477289177Speter svn_pool_clear(iterpool); 478289177Speter 479289177Speter /* read & check the serialized size */ 480289177Speter SVN_ERR(svn_fs_fs__read_number_from_stream(&size, NULL, stream, 481289177Speter iterpool)); 482289177Speter if (size + offset > (apr_int64_t)revprops->packed_revprops->len) 483289177Speter return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 484289177Speter _("Packed revprop size exceeds pack file size")); 485289177Speter 486289177Speter /* Parse this revprops list, if necessary */ 487289177Speter serialized.data = revprops->packed_revprops->data + offset; 488289177Speter serialized.len = (apr_size_t)size; 489289177Speter 490289177Speter if (revision == revprops->revision) 491289177Speter { 492289177Speter SVN_ERR(parse_revprop(&revprops->properties, fs, revision, 493289177Speter revprops->generation, &serialized, 494289177Speter pool, iterpool)); 495289177Speter revprops->serialized_size = serialized.len; 496289177Speter 497289177Speter /* If we only wanted the revprops for REVISION then we are done. */ 498289177Speter if (!read_all) 499289177Speter break; 500289177Speter } 501289177Speter 502289177Speter if (read_all) 503289177Speter { 504289177Speter /* fill REVPROPS data structures */ 505289177Speter APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; 506289177Speter APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; 507289177Speter } 508289177Speter revprops->total_size += serialized.len; 509289177Speter 510289177Speter offset += serialized.len; 511289177Speter } 512289177Speter 513289177Speter return SVN_NO_ERROR; 514289177Speter} 515289177Speter 516289177Speter/* In filesystem FS, read the packed revprops for revision REV into 517289177Speter * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. 518289177Speter * If you want to modify revprop contents / update REVPROPS, READ_ALL 519289177Speter * must be set. Otherwise, only the properties of REV are being provided. 520289177Speter * Allocate data in POOL. 521289177Speter */ 522289177Speterstatic svn_error_t * 523289177Speterread_pack_revprop(packed_revprops_t **revprops, 524289177Speter svn_fs_t *fs, 525289177Speter svn_revnum_t rev, 526289177Speter apr_int64_t generation, 527289177Speter svn_boolean_t read_all, 528289177Speter apr_pool_t *pool) 529289177Speter{ 530289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 531289177Speter svn_boolean_t missing = FALSE; 532289177Speter svn_error_t *err; 533289177Speter packed_revprops_t *result; 534289177Speter int i; 535289177Speter 536289177Speter /* someone insisted that REV is packed. Double-check if necessary */ 537289177Speter if (!svn_fs_fs__is_packed_revprop(fs, rev)) 538289177Speter SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, iterpool)); 539289177Speter 540289177Speter if (!svn_fs_fs__is_packed_revprop(fs, rev)) 541289177Speter return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 542289177Speter _("No such packed revision %ld"), rev); 543289177Speter 544289177Speter /* initialize the result data structure */ 545289177Speter result = apr_pcalloc(pool, sizeof(*result)); 546289177Speter result->revision = rev; 547289177Speter result->generation = generation; 548289177Speter 549289177Speter /* try to read the packed revprops. This may require retries if we have 550289177Speter * concurrent writers. */ 551289177Speter for (i = 0; 552289177Speter i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !result->packed_revprops; 553289177Speter ++i) 554289177Speter { 555289177Speter const char *file_path; 556289177Speter svn_pool_clear(iterpool); 557289177Speter 558289177Speter /* there might have been concurrent writes. 559289177Speter * Re-read the manifest and the pack file. 560289177Speter */ 561289177Speter SVN_ERR(get_revprop_packname(fs, result, pool, iterpool)); 562289177Speter file_path = svn_dirent_join(result->folder, 563289177Speter result->filename, 564289177Speter iterpool); 565289177Speter SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&result->packed_revprops, 566289177Speter &missing, 567289177Speter file_path, 568289177Speter i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT, 569289177Speter pool)); 570289177Speter } 571289177Speter 572289177Speter /* the file content should be available now */ 573289177Speter if (!result->packed_revprops) 574289177Speter return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, 575289177Speter _("Failed to read revprop pack file for r%ld"), rev); 576289177Speter 577289177Speter /* parse it. RESULT will be complete afterwards. */ 578289177Speter err = parse_packed_revprops(fs, result, read_all, pool, iterpool); 579289177Speter svn_pool_destroy(iterpool); 580289177Speter if (err) 581289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, err, 582289177Speter _("Revprop pack file for r%ld is corrupt"), rev); 583289177Speter 584289177Speter *revprops = result; 585289177Speter 586289177Speter return SVN_NO_ERROR; 587289177Speter} 588289177Speter 589289177Speter/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. 590289177Speter * 591289177Speter * Allocations will be done in POOL. 592289177Speter */ 593289177Spetersvn_error_t * 594289177Spetersvn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p, 595289177Speter svn_fs_t *fs, 596289177Speter svn_revnum_t rev, 597289177Speter apr_pool_t *pool) 598289177Speter{ 599289177Speter fs_fs_data_t *ffd = fs->fsap_data; 600289177Speter apr_int64_t generation = 0; 601289177Speter 602289177Speter /* not found, yet */ 603289177Speter *proplist_p = NULL; 604289177Speter 605289177Speter /* should they be available at all? */ 606289177Speter SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool)); 607289177Speter 608289177Speter /* if REV had not been packed when we began, try reading it from the 609289177Speter * non-packed shard. If that fails, we will fall through to packed 610289177Speter * shard reads. */ 611289177Speter if (!svn_fs_fs__is_packed_revprop(fs, rev)) 612289177Speter { 613289177Speter svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, 614289177Speter generation, pool); 615289177Speter if (err) 616289177Speter { 617289177Speter if (!APR_STATUS_IS_ENOENT(err->apr_err) 618289177Speter || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 619289177Speter return svn_error_trace(err); 620289177Speter 621289177Speter svn_error_clear(err); 622289177Speter *proplist_p = NULL; /* in case read_non_packed_revprop changed it */ 623289177Speter } 624289177Speter } 625289177Speter 626289177Speter /* if revprop packing is available and we have not read the revprops, yet, 627289177Speter * try reading them from a packed shard. If that fails, REV is most 628289177Speter * likely invalid (or its revprops highly contested). */ 629289177Speter if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p) 630289177Speter { 631289177Speter packed_revprops_t *revprops; 632289177Speter SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool)); 633289177Speter *proplist_p = revprops->properties; 634289177Speter } 635289177Speter 636289177Speter /* The revprops should have been there. Did we get them? */ 637289177Speter if (!*proplist_p) 638289177Speter return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 639289177Speter _("Could not read revprops for revision %ld"), 640289177Speter rev); 641289177Speter 642289177Speter return SVN_NO_ERROR; 643289177Speter} 644289177Speter 645289177Speter/* Serialize the revision property list PROPLIST of revision REV in 646289177Speter * filesystem FS to a non-packed file. Return the name of that temporary 647289177Speter * file in *TMP_PATH and the file path that it must be moved to in 648289177Speter * *FINAL_PATH. 649289177Speter * 650289177Speter * Use POOL for allocations. 651289177Speter */ 652289177Speterstatic svn_error_t * 653289177Speterwrite_non_packed_revprop(const char **final_path, 654289177Speter const char **tmp_path, 655289177Speter svn_fs_t *fs, 656289177Speter svn_revnum_t rev, 657289177Speter apr_hash_t *proplist, 658289177Speter apr_pool_t *pool) 659289177Speter{ 660289177Speter apr_file_t *file; 661289177Speter svn_stream_t *stream; 662289177Speter *final_path = svn_fs_fs__path_revprops(fs, rev, pool); 663289177Speter 664289177Speter /* ### do we have a directory sitting around already? we really shouldn't 665289177Speter ### have to get the dirname here. */ 666289177Speter SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, 667289177Speter svn_dirent_dirname(*final_path, pool), 668289177Speter svn_io_file_del_none, pool, pool)); 669289177Speter stream = svn_stream_from_aprfile2(file, TRUE, pool); 670289177Speter SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 671289177Speter SVN_ERR(svn_stream_close(stream)); 672289177Speter 673289177Speter /* Flush temporary file to disk and close it. */ 674289177Speter SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 675289177Speter SVN_ERR(svn_io_file_close(file, pool)); 676289177Speter 677289177Speter return SVN_NO_ERROR; 678289177Speter} 679289177Speter 680289177Speter/* After writing the new revprop file(s), call this function to move the 681289177Speter * file at TMP_PATH to FINAL_PATH and give it the permissions from 682289177Speter * PERMS_REFERENCE. 683289177Speter * 684289177Speter * Finally, delete all the temporary files given in FILES_TO_DELETE. 685289177Speter * The latter may be NULL. 686289177Speter * 687289177Speter * Use POOL for temporary allocations. 688289177Speter */ 689289177Speterstatic svn_error_t * 690289177Speterswitch_to_new_revprop(svn_fs_t *fs, 691289177Speter const char *final_path, 692289177Speter const char *tmp_path, 693289177Speter const char *perms_reference, 694289177Speter apr_array_header_t *files_to_delete, 695289177Speter apr_pool_t *pool) 696289177Speter{ 697289177Speter SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference, 698289177Speter pool)); 699289177Speter 700289177Speter /* Clean up temporary files, if necessary. */ 701289177Speter if (files_to_delete) 702289177Speter { 703289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 704289177Speter int i; 705289177Speter 706289177Speter for (i = 0; i < files_to_delete->nelts; ++i) 707289177Speter { 708289177Speter const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*); 709289177Speter 710289177Speter svn_pool_clear(iterpool); 711289177Speter SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 712289177Speter } 713289177Speter 714289177Speter svn_pool_destroy(iterpool); 715289177Speter } 716289177Speter return SVN_NO_ERROR; 717289177Speter} 718289177Speter 719289177Speter/* Write a pack file header to STREAM that starts at revision START_REVISION 720289177Speter * and contains the indexes [START,END) of SIZES. 721289177Speter */ 722289177Speterstatic svn_error_t * 723289177Speterserialize_revprops_header(svn_stream_t *stream, 724289177Speter svn_revnum_t start_revision, 725289177Speter apr_array_header_t *sizes, 726289177Speter int start, 727289177Speter int end, 728289177Speter apr_pool_t *pool) 729289177Speter{ 730289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 731289177Speter int i; 732289177Speter 733289177Speter SVN_ERR_ASSERT(start < end); 734289177Speter 735289177Speter /* start revision and entry count */ 736289177Speter SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision)); 737289177Speter SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start)); 738289177Speter 739289177Speter /* the sizes array */ 740289177Speter for (i = start; i < end; ++i) 741289177Speter { 742289177Speter /* Non-standard pool usage. 743289177Speter * 744289177Speter * We only allocate a few bytes each iteration -- even with a 745289177Speter * million iterations we would still be in good shape memory-wise. 746289177Speter */ 747289177Speter apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t); 748289177Speter SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n", 749289177Speter size)); 750289177Speter } 751289177Speter 752289177Speter /* the double newline char indicates the end of the header */ 753289177Speter SVN_ERR(svn_stream_printf(stream, iterpool, "\n")); 754289177Speter 755289177Speter svn_pool_destroy(iterpool); 756289177Speter return SVN_NO_ERROR; 757289177Speter} 758289177Speter 759289177Speter/* Writes the a pack file to FILE. It copies the serialized data 760289177Speter * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX. 761289177Speter * 762289177Speter * The data for the latter is taken from NEW_SERIALIZED. Note, that 763289177Speter * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is 764289177Speter * taken in that case but only a subset of the old data will be copied. 765289177Speter * 766289177Speter * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. 767289177Speter * POOL is used for temporary allocations. 768289177Speter */ 769289177Speterstatic svn_error_t * 770289177Speterrepack_revprops(svn_fs_t *fs, 771289177Speter packed_revprops_t *revprops, 772289177Speter int start, 773289177Speter int end, 774289177Speter int changed_index, 775289177Speter svn_stringbuf_t *new_serialized, 776289177Speter apr_off_t new_total_size, 777289177Speter apr_file_t *file, 778289177Speter apr_pool_t *pool) 779289177Speter{ 780289177Speter fs_fs_data_t *ffd = fs->fsap_data; 781289177Speter svn_stream_t *stream; 782289177Speter int i; 783289177Speter 784289177Speter /* create data empty buffers and the stream object */ 785289177Speter svn_stringbuf_t *uncompressed 786289177Speter = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool); 787289177Speter svn_stringbuf_t *compressed 788289177Speter = svn_stringbuf_create_empty(pool); 789289177Speter stream = svn_stream_from_stringbuf(uncompressed, pool); 790289177Speter 791289177Speter /* write the header*/ 792289177Speter SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start, 793289177Speter revprops->sizes, start, end, pool)); 794289177Speter 795289177Speter /* append the serialized revprops */ 796289177Speter for (i = start; i < end; ++i) 797289177Speter if (i == changed_index) 798289177Speter { 799289177Speter SVN_ERR(svn_stream_write(stream, 800289177Speter new_serialized->data, 801289177Speter &new_serialized->len)); 802289177Speter } 803289177Speter else 804289177Speter { 805289177Speter apr_size_t size 806289177Speter = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t); 807289177Speter apr_size_t offset 808289177Speter = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t); 809289177Speter 810289177Speter SVN_ERR(svn_stream_write(stream, 811289177Speter revprops->packed_revprops->data + offset, 812289177Speter &size)); 813289177Speter } 814289177Speter 815289177Speter /* flush the stream buffer (if any) to our underlying data buffer */ 816289177Speter SVN_ERR(svn_stream_close(stream)); 817289177Speter 818289177Speter /* compress / store the data */ 819289177Speter SVN_ERR(svn__compress(uncompressed, 820289177Speter compressed, 821289177Speter ffd->compress_packed_revprops 822289177Speter ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 823289177Speter : SVN_DELTA_COMPRESSION_LEVEL_NONE)); 824289177Speter 825289177Speter /* finally, write the content to the target file, flush and close it */ 826289177Speter SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len, 827289177Speter NULL, pool)); 828289177Speter SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 829289177Speter SVN_ERR(svn_io_file_close(file, pool)); 830289177Speter 831289177Speter return SVN_NO_ERROR; 832289177Speter} 833289177Speter 834289177Speter/* Allocate a new pack file name for revisions 835289177Speter * [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1] 836289177Speter * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, 837289177Speter * auto-create that array if necessary. Return an open file *FILE that is 838289177Speter * allocated in POOL. 839289177Speter */ 840289177Speterstatic svn_error_t * 841289177Speterrepack_file_open(apr_file_t **file, 842289177Speter svn_fs_t *fs, 843289177Speter packed_revprops_t *revprops, 844289177Speter int start, 845289177Speter int end, 846289177Speter apr_array_header_t **files_to_delete, 847289177Speter apr_pool_t *pool) 848289177Speter{ 849289177Speter apr_int64_t tag; 850289177Speter const char *tag_string; 851289177Speter svn_string_t *new_filename; 852289177Speter int i; 853289177Speter int manifest_offset 854289177Speter = (int)(revprops->start_revision - revprops->manifest_start); 855289177Speter 856289177Speter /* get the old (= current) file name and enlist it for later deletion */ 857289177Speter const char *old_filename = APR_ARRAY_IDX(revprops->manifest, 858289177Speter start + manifest_offset, 859289177Speter const char*); 860289177Speter 861289177Speter if (*files_to_delete == NULL) 862289177Speter *files_to_delete = apr_array_make(pool, 3, sizeof(const char*)); 863289177Speter 864289177Speter APR_ARRAY_PUSH(*files_to_delete, const char*) 865289177Speter = svn_dirent_join(revprops->folder, old_filename, pool); 866289177Speter 867289177Speter /* increase the tag part, i.e. the counter after the dot */ 868289177Speter tag_string = strchr(old_filename, '.'); 869289177Speter if (tag_string == NULL) 870289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 871289177Speter _("Packed file '%s' misses a tag"), 872289177Speter old_filename); 873289177Speter 874289177Speter SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); 875289177Speter new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT, 876289177Speter revprops->start_revision + start, 877289177Speter ++tag); 878289177Speter 879289177Speter /* update the manifest to point to the new file */ 880289177Speter for (i = start; i < end; ++i) 881289177Speter APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*) 882289177Speter = new_filename->data; 883289177Speter 884289177Speter /* open the file */ 885289177Speter SVN_ERR(svn_io_file_open(file, svn_dirent_join(revprops->folder, 886289177Speter new_filename->data, 887289177Speter pool), 888289177Speter APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); 889289177Speter 890289177Speter return SVN_NO_ERROR; 891289177Speter} 892289177Speter 893289177Speter/* For revision REV in filesystem FS, set the revision properties to 894289177Speter * PROPLIST. Return a new file in *TMP_PATH that the caller shall move 895289177Speter * to *FINAL_PATH to make the change visible. Files to be deleted will 896289177Speter * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. 897289177Speter * Use POOL for allocations. 898289177Speter */ 899289177Speterstatic svn_error_t * 900289177Speterwrite_packed_revprop(const char **final_path, 901289177Speter const char **tmp_path, 902289177Speter apr_array_header_t **files_to_delete, 903289177Speter svn_fs_t *fs, 904289177Speter svn_revnum_t rev, 905289177Speter apr_hash_t *proplist, 906289177Speter apr_pool_t *pool) 907289177Speter{ 908289177Speter fs_fs_data_t *ffd = fs->fsap_data; 909289177Speter packed_revprops_t *revprops; 910289177Speter apr_int64_t generation = 0; 911289177Speter svn_stream_t *stream; 912289177Speter apr_file_t *file; 913289177Speter svn_stringbuf_t *serialized; 914289177Speter apr_off_t new_total_size; 915289177Speter int changed_index; 916289177Speter 917289177Speter /* read contents of the current pack file */ 918289177Speter SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool)); 919289177Speter 920289177Speter /* serialize the new revprops */ 921289177Speter serialized = svn_stringbuf_create_empty(pool); 922289177Speter stream = svn_stream_from_stringbuf(serialized, pool); 923289177Speter SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 924289177Speter SVN_ERR(svn_stream_close(stream)); 925289177Speter 926289177Speter /* calculate the size of the new data */ 927289177Speter changed_index = (int)(rev - revprops->start_revision); 928289177Speter new_total_size = revprops->total_size - revprops->serialized_size 929289177Speter + serialized->len 930289177Speter + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; 931289177Speter 932289177Speter APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len; 933289177Speter 934289177Speter /* can we put the new data into the same pack as the before? */ 935289177Speter if ( new_total_size < ffd->revprop_pack_size 936289177Speter || revprops->sizes->nelts == 1) 937289177Speter { 938289177Speter /* simply replace the old pack file with new content as we do it 939289177Speter * in the non-packed case */ 940289177Speter 941289177Speter *final_path = svn_dirent_join(revprops->folder, revprops->filename, 942289177Speter pool); 943289177Speter SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder, 944289177Speter svn_io_file_del_none, pool, pool)); 945289177Speter SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts, 946289177Speter changed_index, serialized, new_total_size, 947289177Speter file, pool)); 948289177Speter } 949289177Speter else 950289177Speter { 951289177Speter /* split the pack file into two of roughly equal size */ 952289177Speter int right_count, left_count, i; 953289177Speter 954289177Speter int left = 0; 955289177Speter int right = revprops->sizes->nelts - 1; 956289177Speter apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE; 957289177Speter apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE; 958289177Speter 959289177Speter /* let left and right side grow such that their size difference 960289177Speter * is minimal after each step. */ 961289177Speter while (left <= right) 962289177Speter if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) 963289177Speter < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)) 964289177Speter { 965289177Speter left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) 966289177Speter + SVN_INT64_BUFFER_SIZE; 967289177Speter ++left; 968289177Speter } 969289177Speter else 970289177Speter { 971289177Speter right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t) 972289177Speter + SVN_INT64_BUFFER_SIZE; 973289177Speter --right; 974289177Speter } 975289177Speter 976289177Speter /* since the items need much less than SVN_INT64_BUFFER_SIZE 977289177Speter * bytes to represent their length, the split may not be optimal */ 978289177Speter left_count = left; 979289177Speter right_count = revprops->sizes->nelts - left; 980289177Speter 981289177Speter /* if new_size is large, one side may exceed the pack size limit. 982289177Speter * In that case, split before and after the modified revprop.*/ 983289177Speter if ( left_size > ffd->revprop_pack_size 984289177Speter || right_size > ffd->revprop_pack_size) 985289177Speter { 986289177Speter left_count = changed_index; 987289177Speter right_count = revprops->sizes->nelts - left_count - 1; 988289177Speter } 989289177Speter 990289177Speter /* write the new, split files */ 991289177Speter if (left_count) 992289177Speter { 993289177Speter SVN_ERR(repack_file_open(&file, fs, revprops, 0, 994289177Speter left_count, files_to_delete, pool)); 995289177Speter SVN_ERR(repack_revprops(fs, revprops, 0, left_count, 996289177Speter changed_index, serialized, new_total_size, 997289177Speter file, pool)); 998289177Speter } 999289177Speter 1000289177Speter if (left_count + right_count < revprops->sizes->nelts) 1001289177Speter { 1002289177Speter SVN_ERR(repack_file_open(&file, fs, revprops, changed_index, 1003289177Speter changed_index + 1, files_to_delete, 1004289177Speter pool)); 1005289177Speter SVN_ERR(repack_revprops(fs, revprops, changed_index, 1006289177Speter changed_index + 1, 1007289177Speter changed_index, serialized, new_total_size, 1008289177Speter file, pool)); 1009289177Speter } 1010289177Speter 1011289177Speter if (right_count) 1012289177Speter { 1013289177Speter SVN_ERR(repack_file_open(&file, fs, revprops, 1014289177Speter revprops->sizes->nelts - right_count, 1015289177Speter revprops->sizes->nelts, 1016289177Speter files_to_delete, pool)); 1017289177Speter SVN_ERR(repack_revprops(fs, revprops, 1018289177Speter revprops->sizes->nelts - right_count, 1019289177Speter revprops->sizes->nelts, changed_index, 1020289177Speter serialized, new_total_size, file, 1021289177Speter pool)); 1022289177Speter } 1023289177Speter 1024289177Speter /* write the new manifest */ 1025289177Speter *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); 1026289177Speter SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder, 1027289177Speter svn_io_file_del_none, pool, pool)); 1028289177Speter 1029289177Speter for (i = 0; i < revprops->manifest->nelts; ++i) 1030289177Speter { 1031289177Speter const char *filename = APR_ARRAY_IDX(revprops->manifest, i, 1032289177Speter const char*); 1033289177Speter SVN_ERR(svn_io_file_write_full(file, filename, strlen(filename), 1034289177Speter NULL, pool)); 1035289177Speter SVN_ERR(svn_io_file_putc('\n', file, pool)); 1036289177Speter } 1037289177Speter 1038289177Speter SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 1039289177Speter SVN_ERR(svn_io_file_close(file, pool)); 1040289177Speter } 1041289177Speter 1042289177Speter return SVN_NO_ERROR; 1043289177Speter} 1044289177Speter 1045289177Speter/* Set the revision property list of revision REV in filesystem FS to 1046289177Speter PROPLIST. Use POOL for temporary allocations. */ 1047289177Spetersvn_error_t * 1048289177Spetersvn_fs_fs__set_revision_proplist(svn_fs_t *fs, 1049289177Speter svn_revnum_t rev, 1050289177Speter apr_hash_t *proplist, 1051289177Speter apr_pool_t *pool) 1052289177Speter{ 1053289177Speter svn_boolean_t is_packed; 1054289177Speter const char *final_path; 1055289177Speter const char *tmp_path; 1056289177Speter const char *perms_reference; 1057289177Speter apr_array_header_t *files_to_delete = NULL; 1058289177Speter 1059289177Speter SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool)); 1060289177Speter 1061289177Speter /* this info will not change while we hold the global FS write lock */ 1062289177Speter is_packed = svn_fs_fs__is_packed_revprop(fs, rev); 1063289177Speter 1064289177Speter /* Serialize the new revprop data */ 1065289177Speter if (is_packed) 1066289177Speter SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, 1067289177Speter fs, rev, proplist, pool)); 1068289177Speter else 1069289177Speter SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, 1070289177Speter fs, rev, proplist, pool)); 1071289177Speter 1072289177Speter /* We use the rev file of this revision as the perms reference, 1073289177Speter * because when setting revprops for the first time, the revprop 1074289177Speter * file won't exist and therefore can't serve as its own reference. 1075289177Speter * (Whereas the rev file should already exist at this point.) 1076289177Speter */ 1077289177Speter perms_reference = svn_fs_fs__path_rev_absolute(fs, rev, pool); 1078289177Speter 1079289177Speter /* Now, switch to the new revprop data. */ 1080289177Speter SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, 1081289177Speter files_to_delete, pool)); 1082289177Speter 1083289177Speter return SVN_NO_ERROR; 1084289177Speter} 1085289177Speter 1086289177Speter/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. 1087289177Speter * Use POOL for temporary allocations. 1088289177Speter * Set *MISSING, if the reason is a missing manifest or pack file. 1089289177Speter */ 1090289177Spetersvn_boolean_t 1091289177Spetersvn_fs_fs__packed_revprop_available(svn_boolean_t *missing, 1092289177Speter svn_fs_t *fs, 1093289177Speter svn_revnum_t revision, 1094289177Speter apr_pool_t *pool) 1095289177Speter{ 1096289177Speter fs_fs_data_t *ffd = fs->fsap_data; 1097289177Speter svn_stringbuf_t *content = NULL; 1098289177Speter 1099289177Speter /* try to read the manifest file */ 1100289177Speter const char *folder 1101289177Speter = svn_fs_fs__path_revprops_pack_shard(fs, revision, pool); 1102289177Speter const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool); 1103289177Speter 1104289177Speter svn_error_t *err = svn_fs_fs__try_stringbuf_from_file(&content, 1105289177Speter missing, 1106289177Speter manifest_path, 1107289177Speter FALSE, 1108289177Speter pool); 1109289177Speter 1110289177Speter /* if the manifest cannot be read, consider the pack files inaccessible 1111289177Speter * even if the file itself exists. */ 1112289177Speter if (err) 1113289177Speter { 1114289177Speter svn_error_clear(err); 1115289177Speter return FALSE; 1116289177Speter } 1117289177Speter 1118289177Speter if (*missing) 1119289177Speter return FALSE; 1120289177Speter 1121289177Speter /* parse manifest content until we find the entry for REVISION. 1122289177Speter * Revision 0 is never packed. */ 1123289177Speter revision = revision < ffd->max_files_per_dir 1124289177Speter ? revision - 1 1125289177Speter : revision % ffd->max_files_per_dir; 1126289177Speter while (content->data) 1127289177Speter { 1128289177Speter char *next = strchr(content->data, '\n'); 1129289177Speter if (next) 1130289177Speter { 1131289177Speter *next = 0; 1132289177Speter ++next; 1133289177Speter } 1134289177Speter 1135289177Speter if (revision-- == 0) 1136289177Speter { 1137289177Speter /* the respective pack file must exist (and be a file) */ 1138289177Speter svn_node_kind_t kind; 1139289177Speter err = svn_io_check_path(svn_dirent_join(folder, content->data, 1140289177Speter pool), 1141289177Speter &kind, pool); 1142289177Speter if (err) 1143289177Speter { 1144289177Speter svn_error_clear(err); 1145289177Speter return FALSE; 1146289177Speter } 1147289177Speter 1148289177Speter *missing = kind == svn_node_none; 1149289177Speter return kind == svn_node_file; 1150289177Speter } 1151289177Speter 1152289177Speter content->data = next; 1153289177Speter } 1154289177Speter 1155289177Speter return FALSE; 1156289177Speter} 1157289177Speter 1158289177Speter 1159289177Speter/****** Packing FSFS shards *********/ 1160289177Speter 1161289177Spetersvn_error_t * 1162289177Spetersvn_fs_fs__copy_revprops(const char *pack_file_dir, 1163289177Speter const char *pack_filename, 1164289177Speter const char *shard_path, 1165289177Speter svn_revnum_t start_rev, 1166289177Speter svn_revnum_t end_rev, 1167289177Speter apr_array_header_t *sizes, 1168289177Speter apr_size_t total_size, 1169289177Speter int compression_level, 1170289177Speter svn_cancel_func_t cancel_func, 1171289177Speter void *cancel_baton, 1172289177Speter apr_pool_t *scratch_pool) 1173289177Speter{ 1174289177Speter svn_stream_t *pack_stream; 1175289177Speter apr_file_t *pack_file; 1176289177Speter svn_revnum_t rev; 1177289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1178289177Speter 1179289177Speter /* create empty data buffer and a write stream on top of it */ 1180289177Speter svn_stringbuf_t *uncompressed 1181289177Speter = svn_stringbuf_create_ensure(total_size, scratch_pool); 1182289177Speter svn_stringbuf_t *compressed 1183289177Speter = svn_stringbuf_create_empty(scratch_pool); 1184289177Speter pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); 1185289177Speter 1186289177Speter /* write the pack file header */ 1187289177Speter SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0, 1188289177Speter sizes->nelts, iterpool)); 1189289177Speter 1190289177Speter /* Some useful paths. */ 1191289177Speter SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir, 1192289177Speter pack_filename, 1193289177Speter scratch_pool), 1194289177Speter APR_WRITE | APR_CREATE, APR_OS_DEFAULT, 1195289177Speter scratch_pool)); 1196289177Speter 1197289177Speter /* Iterate over the revisions in this shard, squashing them together. */ 1198289177Speter for (rev = start_rev; rev <= end_rev; rev++) 1199289177Speter { 1200289177Speter const char *path; 1201289177Speter svn_stream_t *stream; 1202289177Speter 1203289177Speter svn_pool_clear(iterpool); 1204289177Speter 1205289177Speter /* Construct the file name. */ 1206289177Speter path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 1207289177Speter iterpool); 1208289177Speter 1209289177Speter /* Copy all the bits from the non-packed revprop file to the end of 1210289177Speter * the pack file. */ 1211289177Speter SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool)); 1212289177Speter SVN_ERR(svn_stream_copy3(stream, pack_stream, 1213289177Speter cancel_func, cancel_baton, iterpool)); 1214289177Speter } 1215289177Speter 1216289177Speter /* flush stream buffers to content buffer */ 1217289177Speter SVN_ERR(svn_stream_close(pack_stream)); 1218289177Speter 1219289177Speter /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ 1220289177Speter SVN_ERR(svn__compress(uncompressed, compressed, compression_level)); 1221289177Speter 1222289177Speter /* write the pack file content to disk */ 1223289177Speter SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len, 1224289177Speter NULL, scratch_pool)); 1225289177Speter SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool)); 1226289177Speter SVN_ERR(svn_io_file_close(pack_file, scratch_pool)); 1227289177Speter 1228289177Speter svn_pool_destroy(iterpool); 1229289177Speter 1230289177Speter return SVN_NO_ERROR; 1231289177Speter} 1232289177Speter 1233289177Spetersvn_error_t * 1234289177Spetersvn_fs_fs__pack_revprops_shard(const char *pack_file_dir, 1235289177Speter const char *shard_path, 1236289177Speter apr_int64_t shard, 1237289177Speter int max_files_per_dir, 1238289177Speter apr_off_t max_pack_size, 1239289177Speter int compression_level, 1240289177Speter svn_cancel_func_t cancel_func, 1241289177Speter void *cancel_baton, 1242289177Speter apr_pool_t *scratch_pool) 1243289177Speter{ 1244289177Speter const char *manifest_file_path, *pack_filename = NULL; 1245289177Speter apr_file_t *manifest_file; 1246289177Speter svn_stream_t *manifest_stream; 1247289177Speter svn_revnum_t start_rev, end_rev, rev; 1248289177Speter apr_off_t total_size; 1249289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1250289177Speter apr_array_header_t *sizes; 1251289177Speter 1252289177Speter /* Some useful paths. */ 1253289177Speter manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, 1254289177Speter scratch_pool); 1255289177Speter 1256289177Speter /* Remove any existing pack file for this shard, since it is incomplete. */ 1257289177Speter SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, 1258289177Speter scratch_pool)); 1259289177Speter 1260289177Speter /* Create the new directory and manifest file stream. */ 1261289177Speter SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); 1262289177Speter 1263289177Speter SVN_ERR(svn_io_file_open(&manifest_file, manifest_file_path, 1264289177Speter APR_WRITE | APR_BUFFERED | APR_CREATE | APR_EXCL, 1265289177Speter APR_OS_DEFAULT, scratch_pool)); 1266289177Speter manifest_stream = svn_stream_from_aprfile2(manifest_file, TRUE, 1267289177Speter scratch_pool); 1268289177Speter 1269289177Speter /* revisions to handle. Special case: revision 0 */ 1270289177Speter start_rev = (svn_revnum_t) (shard * max_files_per_dir); 1271289177Speter end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); 1272289177Speter if (start_rev == 0) 1273289177Speter ++start_rev; 1274289177Speter /* Special special case: if max_files_per_dir is 1, then at this point 1275289177Speter start_rev == 1 and end_rev == 0 (!). Fortunately, everything just 1276289177Speter works. */ 1277289177Speter 1278289177Speter /* initialize the revprop size info */ 1279289177Speter sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t)); 1280289177Speter total_size = 2 * SVN_INT64_BUFFER_SIZE; 1281289177Speter 1282289177Speter /* Iterate over the revisions in this shard, determine their size and 1283289177Speter * squashing them together into pack files. */ 1284289177Speter for (rev = start_rev; rev <= end_rev; rev++) 1285289177Speter { 1286289177Speter apr_finfo_t finfo; 1287289177Speter const char *path; 1288289177Speter 1289289177Speter svn_pool_clear(iterpool); 1290289177Speter 1291289177Speter /* Get the size of the file. */ 1292289177Speter path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 1293289177Speter iterpool); 1294289177Speter SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); 1295289177Speter 1296289177Speter /* if we already have started a pack file and this revprop cannot be 1297289177Speter * appended to it, write the previous pack file. */ 1298289177Speter if (sizes->nelts != 0 && 1299289177Speter total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size) 1300289177Speter { 1301289177Speter SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename, 1302289177Speter shard_path, start_rev, rev-1, 1303289177Speter sizes, (apr_size_t)total_size, 1304289177Speter compression_level, cancel_func, 1305289177Speter cancel_baton, iterpool)); 1306289177Speter 1307289177Speter /* next pack file starts empty again */ 1308289177Speter apr_array_clear(sizes); 1309289177Speter total_size = 2 * SVN_INT64_BUFFER_SIZE; 1310289177Speter start_rev = rev; 1311289177Speter } 1312289177Speter 1313289177Speter /* Update the manifest. Allocate a file name for the current pack 1314289177Speter * file if it is a new one */ 1315289177Speter if (sizes->nelts == 0) 1316289177Speter pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); 1317289177Speter 1318289177Speter SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n", 1319289177Speter pack_filename)); 1320289177Speter 1321289177Speter /* add to list of files to put into the current pack file */ 1322289177Speter APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size; 1323289177Speter total_size += SVN_INT64_BUFFER_SIZE + finfo.size; 1324289177Speter } 1325289177Speter 1326289177Speter /* write the last pack file */ 1327289177Speter if (sizes->nelts != 0) 1328289177Speter SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename, 1329289177Speter shard_path, start_rev, rev-1, 1330289177Speter sizes, (apr_size_t)total_size, 1331289177Speter compression_level, cancel_func, 1332289177Speter cancel_baton, iterpool)); 1333289177Speter 1334289177Speter /* flush the manifest file to disk and update permissions */ 1335289177Speter SVN_ERR(svn_stream_close(manifest_stream)); 1336289177Speter SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool)); 1337289177Speter SVN_ERR(svn_io_file_close(manifest_file, iterpool)); 1338289177Speter SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); 1339289177Speter 1340289177Speter svn_pool_destroy(iterpool); 1341289177Speter 1342289177Speter return SVN_NO_ERROR; 1343289177Speter} 1344289177Speter 1345289177Spetersvn_error_t * 1346289177Spetersvn_fs_fs__delete_revprops_shard(const char *shard_path, 1347289177Speter apr_int64_t shard, 1348289177Speter int max_files_per_dir, 1349289177Speter svn_cancel_func_t cancel_func, 1350289177Speter void *cancel_baton, 1351289177Speter apr_pool_t *scratch_pool) 1352289177Speter{ 1353289177Speter if (shard == 0) 1354289177Speter { 1355289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1356289177Speter int i; 1357289177Speter 1358289177Speter /* delete all files except the one for revision 0 */ 1359289177Speter for (i = 1; i < max_files_per_dir; ++i) 1360289177Speter { 1361289177Speter const char *path; 1362289177Speter svn_pool_clear(iterpool); 1363289177Speter 1364289177Speter path = svn_dirent_join(shard_path, 1365289177Speter apr_psprintf(iterpool, "%d", i), 1366289177Speter iterpool); 1367289177Speter if (cancel_func) 1368289177Speter SVN_ERR((*cancel_func)(cancel_baton)); 1369289177Speter 1370289177Speter SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 1371289177Speter } 1372289177Speter 1373289177Speter svn_pool_destroy(iterpool); 1374289177Speter } 1375289177Speter else 1376289177Speter SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, 1377289177Speter cancel_func, cancel_baton, scratch_pool)); 1378289177Speter 1379289177Speter return SVN_NO_ERROR; 1380289177Speter} 1381289177Speter 1382