1289177Speter/* revprops.c --- everything needed to handle revprops in FSX 2289177Speter * 3289177Speter * ==================================================================== 4289177Speter * Licensed to the Apache Software Foundation (ASF) under one 5289177Speter * or more contributor license agreements. See the NOTICE file 6289177Speter * distributed with this work for additional information 7289177Speter * regarding copyright ownership. The ASF licenses this file 8289177Speter * to you under the Apache License, Version 2.0 (the 9289177Speter * "License"); you may not use this file except in compliance 10289177Speter * with the License. You may obtain a copy of the License at 11289177Speter * 12289177Speter * http://www.apache.org/licenses/LICENSE-2.0 13289177Speter * 14289177Speter * Unless required by applicable law or agreed to in writing, 15289177Speter * software distributed under the License is distributed on an 16289177Speter * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17289177Speter * KIND, either express or implied. See the License for the 18289177Speter * specific language governing permissions and limitations 19289177Speter * under the License. 20289177Speter * ==================================================================== 21289177Speter */ 22289177Speter 23289177Speter#include <assert.h> 24289177Speter#include <apr_md5.h> 25289177Speter 26289177Speter#include "svn_pools.h" 27289177Speter#include "svn_hash.h" 28289177Speter#include "svn_dirent_uri.h" 29362181Sdim#include "svn_sorts.h" 30289177Speter 31289177Speter#include "fs_x.h" 32362181Sdim#include "low_level.h" 33289177Speter#include "revprops.h" 34289177Speter#include "util.h" 35289177Speter#include "transaction.h" 36289177Speter 37362181Sdim#include "private/svn_packed_data.h" 38362181Sdim#include "private/svn_sorts_private.h" 39289177Speter#include "private/svn_subr_private.h" 40289177Speter#include "private/svn_string_private.h" 41289177Speter#include "../libsvn_fs/fs-loader.h" 42289177Speter 43289177Speter#include "svn_private_config.h" 44289177Speter 45289177Speter/* Give writing processes 10 seconds to replace an existing revprop 46289177Speter file with a new one. After that time, we assume that the writing 47289177Speter process got aborted and that we have re-read revprops. */ 48289177Speter#define REVPROP_CHANGE_TIMEOUT (10 * 1000000) 49289177Speter 50289177Speter/* In case of an inconsistent read, close the generation file, yield, 51289177Speter re-open and re-read. This is the number of times we try this before 52289177Speter giving up. */ 53289177Speter#define GENERATION_READ_RETRY_COUNT 100 54289177Speter 55289177Speter 56289177Speter/* Revprop caching management. 57289177Speter * 58289177Speter * Mechanism: 59289177Speter * ---------- 60289177Speter * 61289177Speter * Revprop caching needs to be activated and will be deactivated for the 62289177Speter * respective FS instance if the necessary infrastructure could not be 63289177Speter * initialized. As long as no revprops are being read or changed, revprop 64289177Speter * caching imposes no overhead. 65289177Speter * 66289177Speter * When activated, we cache revprops using (revision, generation) pairs 67289177Speter * as keys with the generation being incremented upon every revprop change. 68289177Speter * Since the cache is process-local, the generation needs to be tracked 69289177Speter * for at least as long as the process lives but may be reset afterwards. 70362181Sdim * We track the revprop generation in a file that. 71289177Speter * 72289177Speter * A race condition exists between switching to the modified revprop data 73289177Speter * and bumping the generation number. In particular, the process may crash 74289177Speter * just after switching to the new revprop data and before bumping the 75289177Speter * generation. To be able to detect this scenario, we bump the generation 76289177Speter * twice per revprop change: once immediately before (creating an odd number) 77289177Speter * and once after the atomic switch (even generation). 78289177Speter * 79289177Speter * A writer holding the write lock can immediately assume a crashed writer 80289177Speter * in case of an odd generation or they would not have been able to acquire 81289177Speter * the lock. A reader detecting an odd generation will use that number and 82289177Speter * be forced to re-read any revprop data - usually getting the new revprops 83289177Speter * already. If the generation file modification timestamp is too old, the 84289177Speter * reader will assume a crashed writer, acquire the write lock and bump 85289177Speter * the generation if it is still odd. So, for about REVPROP_CHANGE_TIMEOUT 86289177Speter * after the crash, reader caches may be stale. 87289177Speter */ 88289177Speter 89289177Speter/* Read revprop generation as stored on disk for repository FS. The result is 90289177Speter * returned in *CURRENT. Call only for repos that support revprop caching. 91289177Speter */ 92289177Speterstatic svn_error_t * 93289177Speterread_revprop_generation_file(apr_int64_t *current, 94289177Speter svn_fs_t *fs, 95289177Speter apr_pool_t *scratch_pool) 96289177Speter{ 97289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 98289177Speter int i; 99289177Speter svn_error_t *err = SVN_NO_ERROR; 100362181Sdim const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool); 101289177Speter 102289177Speter /* Retry in case of incomplete file buffer updates. */ 103289177Speter for (i = 0; i < GENERATION_READ_RETRY_COUNT; ++i) 104289177Speter { 105362181Sdim svn_stringbuf_t *buf; 106362181Sdim 107289177Speter svn_error_clear(err); 108289177Speter svn_pool_clear(iterpool); 109289177Speter 110362181Sdim /* Read the generation file. */ 111362181Sdim err = svn_stringbuf_from_file2(&buf, path, iterpool); 112289177Speter 113362181Sdim /* If we could read the file, it should be complete due to our atomic 114362181Sdim * file replacement scheme. */ 115289177Speter if (!err) 116362181Sdim { 117362181Sdim svn_stringbuf_strip_whitespace(buf); 118362181Sdim SVN_ERR(svn_cstring_atoi64(current, buf->data)); 119362181Sdim break; 120362181Sdim } 121289177Speter 122362181Sdim /* Got unlucky the file was not available. Retry. */ 123289177Speter#if APR_HAS_THREADS 124289177Speter apr_thread_yield(); 125289177Speter#else 126289177Speter apr_sleep(0); 127289177Speter#endif 128289177Speter } 129289177Speter 130289177Speter svn_pool_destroy(iterpool); 131289177Speter 132289177Speter /* If we had to give up, propagate the error. */ 133289177Speter return svn_error_trace(err); 134289177Speter} 135289177Speter 136289177Speter/* Write the CURRENT revprop generation to disk for repository FS. 137289177Speter * Call only for repos that support revprop caching. 138289177Speter */ 139289177Speterstatic svn_error_t * 140289177Speterwrite_revprop_generation_file(svn_fs_t *fs, 141289177Speter apr_int64_t current, 142289177Speter apr_pool_t *scratch_pool) 143289177Speter{ 144289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 145289177Speter svn_stringbuf_t *buffer; 146362181Sdim const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool); 147289177Speter 148362181Sdim /* Invalidate our cached revprop generation in case the file operations 149362181Sdim * below fail. */ 150362181Sdim ffd->revprop_generation = -1; 151289177Speter 152362181Sdim /* Write the new number. */ 153362181Sdim buffer = svn_stringbuf_createf(scratch_pool, "%" APR_INT64_T_FMT "\n", 154362181Sdim current); 155362181Sdim SVN_ERR(svn_io_write_atomic2(path, buffer->data, buffer->len, 156362181Sdim path /* copy_perms */, FALSE, 157362181Sdim scratch_pool)); 158289177Speter 159362181Sdim /* Remember it to spare us the re-read. */ 160362181Sdim ffd->revprop_generation = current; 161362181Sdim 162289177Speter return SVN_NO_ERROR; 163289177Speter} 164289177Speter 165289177Spetersvn_error_t * 166289177Spetersvn_fs_x__reset_revprop_generation_file(svn_fs_t *fs, 167289177Speter apr_pool_t *scratch_pool) 168289177Speter{ 169362181Sdim /* Write the initial revprop generation file contents. */ 170362181Sdim SVN_ERR(write_revprop_generation_file(fs, 0, scratch_pool)); 171289177Speter 172289177Speter return SVN_NO_ERROR; 173289177Speter} 174289177Speter 175289177Speter/* Test whether revprop cache and necessary infrastructure are 176289177Speter available in FS. */ 177289177Speterstatic svn_boolean_t 178289177Speterhas_revprop_cache(svn_fs_t *fs, 179289177Speter apr_pool_t *scratch_pool) 180289177Speter{ 181289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 182289177Speter 183362181Sdim /* is the cache enabled? */ 184362181Sdim return ffd->revprop_cache != NULL; 185289177Speter} 186289177Speter 187289177Speter/* Baton structure for revprop_generation_fixup. */ 188289177Spetertypedef struct revprop_generation_fixup_t 189289177Speter{ 190289177Speter /* revprop generation to read */ 191289177Speter apr_int64_t *generation; 192289177Speter 193289177Speter /* file system context */ 194289177Speter svn_fs_t *fs; 195289177Speter} revprop_generation_upgrade_t; 196289177Speter 197289177Speter/* If the revprop generation has an odd value, it means the original writer 198289177Speter of the revprop got killed. We don't know whether that process as able 199289177Speter to change the revprop data but we assume that it was. Therefore, we 200289177Speter increase the generation in that case to basically invalidate everyone's 201289177Speter cache content. 202289177Speter Execute this only while holding the write lock to the repo in baton->FFD. 203289177Speter */ 204289177Speterstatic svn_error_t * 205289177Speterrevprop_generation_fixup(void *void_baton, 206289177Speter apr_pool_t *scratch_pool) 207289177Speter{ 208289177Speter revprop_generation_upgrade_t *baton = void_baton; 209289177Speter svn_fs_x__data_t *ffd = baton->fs->fsap_data; 210289177Speter assert(ffd->has_write_lock); 211289177Speter 212289177Speter /* Maybe, either the original revprop writer or some other reader has 213289177Speter already corrected / bumped the revprop generation. Thus, we need 214289177Speter to read it again. However, we will now be the only ones changing 215289177Speter the file contents due to us holding the write lock. */ 216289177Speter SVN_ERR(read_revprop_generation_file(baton->generation, baton->fs, 217289177Speter scratch_pool)); 218289177Speter 219289177Speter /* Cause everyone to re-read revprops upon their next access, if the 220289177Speter last revprop write did not complete properly. */ 221289177Speter if (*baton->generation % 2) 222289177Speter { 223289177Speter ++*baton->generation; 224289177Speter SVN_ERR(write_revprop_generation_file(baton->fs, 225289177Speter *baton->generation, 226289177Speter scratch_pool)); 227289177Speter } 228289177Speter 229289177Speter return SVN_NO_ERROR; 230289177Speter} 231289177Speter 232362181Sdim/* Read the current revprop generation of FS and its value in FS->FSAP_DATA. 233362181Sdim Also, detect aborted / crashed writers and recover from that. */ 234289177Speterstatic svn_error_t * 235362181Sdimread_revprop_generation(svn_fs_t *fs, 236289177Speter apr_pool_t *scratch_pool) 237289177Speter{ 238289177Speter apr_int64_t current = 0; 239289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 240289177Speter 241289177Speter /* read the current revprop generation number */ 242289177Speter SVN_ERR(read_revprop_generation_file(¤t, fs, scratch_pool)); 243289177Speter 244289177Speter /* is an unfinished revprop write under the way? */ 245289177Speter if (current % 2) 246289177Speter { 247289177Speter svn_boolean_t timeout = FALSE; 248289177Speter 249289177Speter /* Has the writer process been aborted? 250289177Speter * Either by timeout or by us being the writer now. 251289177Speter */ 252289177Speter if (!ffd->has_write_lock) 253289177Speter { 254289177Speter apr_time_t mtime; 255289177Speter SVN_ERR(svn_io_file_affected_time(&mtime, 256289177Speter svn_fs_x__path_revprop_generation(fs, scratch_pool), 257289177Speter scratch_pool)); 258289177Speter timeout = apr_time_now() > mtime + REVPROP_CHANGE_TIMEOUT; 259289177Speter } 260289177Speter 261289177Speter if (ffd->has_write_lock || timeout) 262289177Speter { 263289177Speter revprop_generation_upgrade_t baton; 264289177Speter baton.generation = ¤t; 265289177Speter baton.fs = fs; 266289177Speter 267289177Speter /* Ensure that the original writer process no longer exists by 268289177Speter * acquiring the write lock to this repository. Then, fix up 269289177Speter * the revprop generation. 270289177Speter */ 271289177Speter if (ffd->has_write_lock) 272289177Speter SVN_ERR(revprop_generation_fixup(&baton, scratch_pool)); 273289177Speter else 274289177Speter SVN_ERR(svn_fs_x__with_write_lock(fs, revprop_generation_fixup, 275289177Speter &baton, scratch_pool)); 276289177Speter } 277289177Speter } 278289177Speter 279289177Speter /* return the value we just got */ 280362181Sdim ffd->revprop_generation = current; 281289177Speter return SVN_NO_ERROR; 282289177Speter} 283289177Speter 284362181Sdimvoid 285362181Sdimsvn_fs_x__invalidate_revprop_generation(svn_fs_t *fs) 286362181Sdim{ 287362181Sdim svn_fs_x__data_t *ffd = fs->fsap_data; 288362181Sdim ffd->revprop_generation = -1; 289362181Sdim} 290362181Sdim 291362181Sdim/* Return TRUE if the revprop generation value in FS->FSAP_DATA is valid. */ 292362181Sdimstatic svn_boolean_t 293362181Sdimis_generation_valid(svn_fs_t *fs) 294362181Sdim{ 295362181Sdim svn_fs_x__data_t *ffd = fs->fsap_data; 296362181Sdim return ffd->revprop_generation >= 0; 297362181Sdim} 298362181Sdim 299289177Speter/* Set the revprop generation in FS to the next odd number to indicate 300362181Sdim that there is a revprop write process under way. Update the value 301362181Sdim in FS->FSAP_DATA accordingly. If the change times out, readers shall 302362181Sdim recover from that state & re-read revprops. 303289177Speter This is a no-op for repo formats that don't support revprop caching. */ 304289177Speterstatic svn_error_t * 305362181Sdimbegin_revprop_change(svn_fs_t *fs, 306289177Speter apr_pool_t *scratch_pool) 307289177Speter{ 308289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 309289177Speter SVN_ERR_ASSERT(ffd->has_write_lock); 310289177Speter 311289177Speter /* Set the revprop generation to an odd value to indicate 312289177Speter * that a write is in progress. 313289177Speter */ 314362181Sdim SVN_ERR(read_revprop_generation(fs, scratch_pool)); 315362181Sdim ++ffd->revprop_generation; 316362181Sdim SVN_ERR_ASSERT(ffd->revprop_generation % 2); 317362181Sdim SVN_ERR(write_revprop_generation_file(fs, ffd->revprop_generation, 318362181Sdim scratch_pool)); 319289177Speter 320289177Speter return SVN_NO_ERROR; 321289177Speter} 322289177Speter 323289177Speter/* Set the revprop generation in FS to the next even generation after 324362181Sdim the odd value in FS->FSAP_DATA to indicate that 325289177Speter a) readers shall re-read revprops, and 326289177Speter b) the write process has been completed (no recovery required). 327289177Speter This is a no-op for repo formats that don't support revprop caching. */ 328289177Speterstatic svn_error_t * 329289177Speterend_revprop_change(svn_fs_t *fs, 330289177Speter apr_pool_t *scratch_pool) 331289177Speter{ 332289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 333289177Speter SVN_ERR_ASSERT(ffd->has_write_lock); 334362181Sdim SVN_ERR_ASSERT(ffd->revprop_generation % 2); 335289177Speter 336289177Speter /* Set the revprop generation to an even value to indicate 337289177Speter * that a write has been completed. Since we held the write 338289177Speter * lock, nobody else could have updated the file contents. 339289177Speter */ 340362181Sdim SVN_ERR(write_revprop_generation_file(fs, ffd->revprop_generation + 1, 341362181Sdim scratch_pool)); 342289177Speter 343289177Speter return SVN_NO_ERROR; 344289177Speter} 345289177Speter 346362181Sdim/* Represents an entry in the packed revprop manifest. 347362181Sdim * There is one such entry per pack file. */ 348362181Sdimtypedef struct manifest_entry_t 349362181Sdim{ 350362181Sdim /* First revision in the pack file. */ 351362181Sdim svn_revnum_t start_rev; 352362181Sdim 353362181Sdim /* Tag (a counter) appended to the file name to distinguish it from 354362181Sdim outdated ones. */ 355362181Sdim apr_uint64_t tag; 356362181Sdim} manifest_entry_t; 357362181Sdim 358289177Speter/* Container for all data required to access the packed revprop file 359289177Speter * for a given REVISION. This structure will be filled incrementally 360289177Speter * by read_pack_revprops() its sub-routines. 361289177Speter */ 362289177Spetertypedef struct packed_revprops_t 363289177Speter{ 364289177Speter /* revision number to read (not necessarily the first in the pack) */ 365289177Speter svn_revnum_t revision; 366289177Speter 367289177Speter /* the actual revision properties */ 368289177Speter apr_hash_t *properties; 369289177Speter 370289177Speter /* their size when serialized to a single string 371289177Speter * (as found in PACKED_REVPROPS) */ 372289177Speter apr_size_t serialized_size; 373289177Speter 374289177Speter 375362181Sdim /* manifest entry describing the pack file */ 376362181Sdim manifest_entry_t entry; 377289177Speter 378289177Speter /* packed shard folder path */ 379289177Speter const char *folder; 380289177Speter 381289177Speter /* sum of values in SIZES */ 382289177Speter apr_size_t total_size; 383289177Speter 384362181Sdim /* Array of svn_string_t, containing the serialized revprops for 385362181Sdim * REVISION * I. */ 386362181Sdim apr_array_header_t *revprops; 387289177Speter 388289177Speter /* content of the manifest. 389362181Sdim * Sorted list of manifest_entry_t. */ 390289177Speter apr_array_header_t *manifest; 391289177Speter} packed_revprops_t; 392289177Speter 393289177Speter/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. 394289177Speter * Also, put them into the revprop cache, if activated, for future use. 395289177Speter * Three more parameters are being used to update the revprop cache: FS is 396362181Sdim * our file system, the revprops belong to REVISION. 397289177Speter * 398289177Speter * The returned hash will be allocated in RESULT_POOL, SCRATCH_POOL is 399289177Speter * being used for temporary allocations. 400289177Speter */ 401289177Speterstatic svn_error_t * 402289177Speterparse_revprop(apr_hash_t **properties, 403289177Speter svn_fs_t *fs, 404289177Speter svn_revnum_t revision, 405362181Sdim const svn_string_t *content, 406289177Speter apr_pool_t *result_pool, 407289177Speter apr_pool_t *scratch_pool) 408289177Speter{ 409362181Sdim SVN_ERR_W(svn_fs_x__parse_properties(properties, content, result_pool), 410362181Sdim apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.", 411362181Sdim revision)); 412289177Speter 413289177Speter if (has_revprop_cache(fs, scratch_pool)) 414289177Speter { 415289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 416289177Speter svn_fs_x__pair_cache_key_t key = { 0 }; 417289177Speter 418362181Sdim SVN_ERR_ASSERT(is_generation_valid(fs)); 419362181Sdim 420289177Speter key.revision = revision; 421362181Sdim key.second = ffd->revprop_generation; 422289177Speter SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, *properties, 423289177Speter scratch_pool)); 424289177Speter } 425289177Speter 426289177Speter return SVN_NO_ERROR; 427289177Speter} 428289177Speter 429362181Sdim/* Verify the checksum attached to CONTENT and remove it. 430362181Sdim * Use SCRATCH_POOL for temporary allocations. 431362181Sdim */ 432362181Sdimstatic svn_error_t * 433362181Sdimverify_checksum(svn_stringbuf_t *content, 434362181Sdim apr_pool_t *scratch_pool) 435362181Sdim{ 436362181Sdim const apr_byte_t *digest; 437362181Sdim svn_checksum_t *actual, *expected; 438362181Sdim 439362181Sdim /* Verify the checksum. */ 440362181Sdim if (content->len < sizeof(apr_uint32_t)) 441362181Sdim return svn_error_create(SVN_ERR_CORRUPT_PACKED_DATA, NULL, 442362181Sdim "File too short"); 443362181Sdim 444362181Sdim content->len -= sizeof(apr_uint32_t); 445362181Sdim digest = (apr_byte_t *)content->data + content->len; 446362181Sdim 447362181Sdim expected = svn_checksum__from_digest_fnv1a_32x4(digest, scratch_pool); 448362181Sdim SVN_ERR(svn_checksum(&actual, svn_checksum_fnv1a_32x4, content->data, 449362181Sdim content->len, scratch_pool)); 450362181Sdim 451362181Sdim if (!svn_checksum_match(actual, expected)) 452362181Sdim SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, 453362181Sdim "checksum mismatch")); 454362181Sdim 455362181Sdim return SVN_NO_ERROR; 456362181Sdim} 457362181Sdim 458289177Speter/* Read the non-packed revprops for revision REV in FS, put them into the 459362181Sdim * revprop cache if activated and return them in *PROPERTIES. 460289177Speter * 461289177Speter * If the data could not be read due to an otherwise recoverable error, 462289177Speter * leave *PROPERTIES unchanged. No error will be returned in that case. 463289177Speter * 464289177Speter * Allocate *PROPERTIES in RESULT_POOL and temporaries in SCRATCH_POOL. 465289177Speter */ 466289177Speterstatic svn_error_t * 467289177Speterread_non_packed_revprop(apr_hash_t **properties, 468289177Speter svn_fs_t *fs, 469289177Speter svn_revnum_t rev, 470289177Speter apr_pool_t *result_pool, 471289177Speter apr_pool_t *scratch_pool) 472289177Speter{ 473289177Speter svn_stringbuf_t *content = NULL; 474289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 475289177Speter svn_boolean_t missing = FALSE; 476289177Speter int i; 477289177Speter 478289177Speter for (i = 0; 479289177Speter i < SVN_FS_X__RECOVERABLE_RETRY_COUNT && !missing && !content; 480289177Speter ++i) 481289177Speter { 482289177Speter svn_pool_clear(iterpool); 483289177Speter SVN_ERR(svn_fs_x__try_stringbuf_from_file(&content, 484289177Speter &missing, 485289177Speter svn_fs_x__path_revprops(fs, rev, iterpool), 486289177Speter i + 1 < SVN_FS_X__RECOVERABLE_RETRY_COUNT, 487289177Speter iterpool)); 488289177Speter } 489289177Speter 490289177Speter if (content) 491362181Sdim { 492362181Sdim svn_string_t *as_string; 493289177Speter 494362181Sdim /* Consistency check. */ 495362181Sdim SVN_ERR_W(verify_checksum(content, scratch_pool), 496362181Sdim apr_psprintf(scratch_pool, 497362181Sdim "Revprop file for r%ld is corrupt", 498362181Sdim rev)); 499362181Sdim 500362181Sdim /* The contents string becomes part of the *PROPERTIES structure, i.e. 501362181Sdim * we must make sure it lives at least as long as the latter. */ 502362181Sdim as_string = svn_string_create_from_buf(content, result_pool); 503362181Sdim SVN_ERR(parse_revprop(properties, fs, rev, as_string, 504362181Sdim result_pool, iterpool)); 505362181Sdim } 506362181Sdim 507289177Speter svn_pool_clear(iterpool); 508289177Speter 509289177Speter return SVN_NO_ERROR; 510289177Speter} 511289177Speter 512362181Sdim/* Serialize ROOT into FILE and append a checksum to it. 513362181Sdim * Use SCRATCH_POOL for temporary allocations. 514362181Sdim */ 515362181Sdimstatic svn_error_t * 516362181Sdimwrite_packed_data_checksummed(svn_packed__data_root_t *root, 517362181Sdim apr_file_t *file, 518362181Sdim apr_pool_t *scratch_pool) 519289177Speter{ 520362181Sdim svn_checksum_t *checksum; 521362181Sdim svn_stream_t *stream; 522289177Speter 523362181Sdim stream = svn_stream_from_aprfile2(file, TRUE, scratch_pool); 524362181Sdim stream = svn_checksum__wrap_write_stream(&checksum, stream, 525362181Sdim svn_checksum_fnv1a_32x4, 526362181Sdim scratch_pool); 527362181Sdim SVN_ERR(svn_packed__data_write(stream, root, scratch_pool)); 528362181Sdim SVN_ERR(svn_stream_close(stream)); 529362181Sdim 530362181Sdim /* Append the checksum */ 531362181Sdim SVN_ERR(svn_io_file_write_full(file, checksum->digest, 532362181Sdim svn_checksum_size(checksum), NULL, 533362181Sdim scratch_pool)); 534362181Sdim 535362181Sdim return SVN_NO_ERROR; 536289177Speter} 537289177Speter 538362181Sdim/* Serialize the packed revprops MANIFEST into FILE. 539362181Sdim * Use SCRATCH_POOL for temporary allocations. 540362181Sdim */ 541362181Sdimstatic svn_error_t * 542362181Sdimwrite_manifest(apr_file_t *file, 543362181Sdim const apr_array_header_t *manifest, 544362181Sdim apr_pool_t *scratch_pool) 545362181Sdim{ 546362181Sdim int i; 547362181Sdim svn_packed__data_root_t *root = svn_packed__data_create_root(scratch_pool); 548362181Sdim 549362181Sdim /* one top-level stream per struct element */ 550362181Sdim svn_packed__int_stream_t *start_rev_stream 551362181Sdim = svn_packed__create_int_stream(root, TRUE, FALSE); 552362181Sdim svn_packed__int_stream_t *tag_stream 553362181Sdim = svn_packed__create_int_stream(root, FALSE, FALSE); 554362181Sdim 555362181Sdim /* serialize ENTRIES */ 556362181Sdim for (i = 0; i < manifest->nelts; ++i) 557362181Sdim { 558362181Sdim manifest_entry_t *entry = &APR_ARRAY_IDX(manifest, i, manifest_entry_t); 559362181Sdim svn_packed__add_uint(start_rev_stream, entry->start_rev); 560362181Sdim svn_packed__add_uint(tag_stream, entry->tag); 561362181Sdim } 562362181Sdim 563362181Sdim /* Write to file and calculate the checksum. */ 564362181Sdim SVN_ERR(write_packed_data_checksummed(root, file, scratch_pool)); 565362181Sdim 566362181Sdim return SVN_NO_ERROR; 567362181Sdim} 568362181Sdim 569362181Sdim/* Read *ROOT from CONTENT and verify its checksum. Allocate *ROOT in 570362181Sdim * RESULT_POOL and use SCRATCH_POOL for temporary allocations. 571362181Sdim */ 572362181Sdimstatic svn_error_t * 573362181Sdimread_packed_data_checksummed(svn_packed__data_root_t **root, 574362181Sdim svn_stringbuf_t *content, 575362181Sdim apr_pool_t *result_pool, 576362181Sdim apr_pool_t *scratch_pool) 577362181Sdim{ 578362181Sdim svn_stream_t *stream; 579362181Sdim 580362181Sdim SVN_ERR(verify_checksum(content, scratch_pool)); 581362181Sdim 582362181Sdim stream = svn_stream_from_stringbuf(content, scratch_pool); 583362181Sdim SVN_ERR(svn_packed__data_read(root, stream, result_pool, scratch_pool)); 584362181Sdim 585362181Sdim return SVN_NO_ERROR; 586362181Sdim} 587362181Sdim 588362181Sdim/* Read the packed revprops manifest from the CONTENT buffer and return it 589362181Sdim * in *MANIFEST, allocated in RESULT_POOL. REVISION is the revision number 590362181Sdim * to put into error messages. Use SCRATCH_POOL for temporary allocations. 591362181Sdim */ 592362181Sdimstatic svn_error_t * 593362181Sdimread_manifest(apr_array_header_t **manifest, 594362181Sdim svn_stringbuf_t *content, 595362181Sdim svn_revnum_t revision, 596362181Sdim apr_pool_t *result_pool, 597362181Sdim apr_pool_t *scratch_pool) 598362181Sdim{ 599362181Sdim apr_size_t i; 600362181Sdim apr_size_t count; 601362181Sdim 602362181Sdim svn_packed__data_root_t *root; 603362181Sdim svn_packed__int_stream_t *start_rev_stream; 604362181Sdim svn_packed__int_stream_t *tag_stream; 605362181Sdim 606362181Sdim /* Verify the checksum and decode packed data. */ 607362181Sdim SVN_ERR_W(read_packed_data_checksummed(&root, content, result_pool, 608362181Sdim scratch_pool), 609362181Sdim apr_psprintf(scratch_pool, 610362181Sdim "Revprop manifest file for r%ld is corrupt", 611362181Sdim revision)); 612362181Sdim 613362181Sdim /* get streams */ 614362181Sdim start_rev_stream = svn_packed__first_int_stream(root); 615362181Sdim tag_stream = svn_packed__next_int_stream(start_rev_stream); 616362181Sdim 617362181Sdim /* read ids array */ 618362181Sdim count = svn_packed__int_count(start_rev_stream); 619362181Sdim *manifest = apr_array_make(result_pool, (int)count, 620362181Sdim sizeof(manifest_entry_t)); 621362181Sdim 622362181Sdim for (i = 0; i < count; ++i) 623362181Sdim { 624362181Sdim manifest_entry_t *entry = apr_array_push(*manifest); 625362181Sdim entry->start_rev = (svn_revnum_t)svn_packed__get_int(start_rev_stream); 626362181Sdim entry->tag = svn_packed__get_uint(tag_stream); 627362181Sdim } 628362181Sdim 629362181Sdim return SVN_NO_ERROR; 630362181Sdim} 631362181Sdim 632362181Sdim/* Implements the standard comparison function signature comparing the 633362181Sdim * manifest_entry_t(lhs).start_rev to svn_revnum_t(rhs). */ 634362181Sdimstatic int 635362181Sdimcompare_entry_revision(const void *lhs, 636362181Sdim const void *rhs) 637362181Sdim{ 638362181Sdim const manifest_entry_t *entry = lhs; 639362181Sdim const svn_revnum_t *revision = rhs; 640362181Sdim 641362181Sdim if (entry->start_rev < *revision) 642362181Sdim return -1; 643362181Sdim 644362181Sdim return entry->start_rev == *revision ? 0 : 1; 645362181Sdim} 646362181Sdim 647362181Sdim/* Return the index in MANIFEST that has the info for the pack file 648362181Sdim * containing REVISION. */ 649362181Sdimstatic int 650362181Sdimget_entry(apr_array_header_t *manifest, 651362181Sdim svn_revnum_t revision) 652362181Sdim{ 653362181Sdim manifest_entry_t *entry; 654362181Sdim int idx = svn_sort__bsearch_lower_bound(manifest, &revision, 655362181Sdim compare_entry_revision); 656362181Sdim 657362181Sdim assert(manifest->nelts > 0); 658362181Sdim if (idx >= manifest->nelts) 659362181Sdim return idx - 1; 660362181Sdim 661362181Sdim entry = &APR_ARRAY_IDX(manifest, idx, manifest_entry_t); 662362181Sdim if (entry->start_rev > revision && idx > 0) 663362181Sdim return idx - 1; 664362181Sdim 665362181Sdim return idx; 666362181Sdim} 667362181Sdim 668362181Sdim/* Return the full path of the revprop pack file given by ENTRY within 669362181Sdim * REVPROPS. Allocate the result in RESULT_POOL. */ 670362181Sdimstatic const char * 671362181Sdimget_revprop_pack_filepath(packed_revprops_t *revprops, 672362181Sdim manifest_entry_t *entry, 673362181Sdim apr_pool_t *result_pool) 674362181Sdim{ 675362181Sdim const char *filename = apr_psprintf(result_pool, "%ld.%" APR_UINT64_T_FMT, 676362181Sdim entry->start_rev, entry->tag); 677362181Sdim return svn_dirent_join(revprops->folder, filename, result_pool); 678362181Sdim} 679362181Sdim 680289177Speter/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST 681289177Speter * members. Use RESULT_POOL for allocating results and SCRATCH_POOL for 682289177Speter * temporaries. 683289177Speter */ 684289177Speterstatic svn_error_t * 685289177Speterget_revprop_packname(svn_fs_t *fs, 686289177Speter packed_revprops_t *revprops, 687289177Speter apr_pool_t *result_pool, 688289177Speter apr_pool_t *scratch_pool) 689289177Speter{ 690289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 691289177Speter svn_stringbuf_t *content = NULL; 692289177Speter const char *manifest_file_path; 693362181Sdim int idx; 694362181Sdim svn_revnum_t previous_start_rev; 695362181Sdim int i; 696289177Speter 697289177Speter /* Determine the dimensions. Rev 0 is excluded from the first shard. */ 698362181Sdim int rev_count = ffd->max_files_per_dir; 699362181Sdim svn_revnum_t manifest_start 700289177Speter = revprops->revision - (revprops->revision % rev_count); 701362181Sdim if (manifest_start == 0) 702289177Speter { 703362181Sdim ++manifest_start; 704289177Speter --rev_count; 705289177Speter } 706289177Speter 707289177Speter /* Read the content of the manifest file */ 708362181Sdim revprops->folder = svn_fs_x__path_pack_shard(fs, revprops->revision, 709362181Sdim result_pool); 710289177Speter manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, 711289177Speter result_pool); 712289177Speter SVN_ERR(svn_fs_x__read_content(&content, manifest_file_path, result_pool)); 713362181Sdim SVN_ERR(read_manifest(&revprops->manifest, content, revprops->revision, 714362181Sdim result_pool, scratch_pool)); 715289177Speter 716362181Sdim /* Verify the manifest data. */ 717362181Sdim if (revprops->manifest->nelts == 0) 718362181Sdim return svn_error_createf(SVN_ERR_FS_CORRUPT_REVPROP_MANIFEST, NULL, 719362181Sdim "Revprop manifest for r%ld is empty", 720362181Sdim revprops->revision); 721289177Speter 722362181Sdim previous_start_rev = 0; 723362181Sdim for (i = 0; i < revprops->manifest->nelts; ++i) 724289177Speter { 725362181Sdim svn_revnum_t start_rev = APR_ARRAY_IDX(revprops->manifest, i, 726362181Sdim manifest_entry_t).start_rev; 727362181Sdim if ( start_rev < manifest_start 728362181Sdim || start_rev >= manifest_start + rev_count) 729362181Sdim return svn_error_createf(SVN_ERR_FS_CORRUPT_REVPROP_MANIFEST, NULL, 730362181Sdim "Revprop manifest for r%ld contains " 731362181Sdim "out-of-range revision r%ld", 732362181Sdim revprops->revision, start_rev); 733289177Speter 734362181Sdim if (start_rev < previous_start_rev) 735362181Sdim return svn_error_createf(SVN_ERR_FS_CORRUPT_REVPROP_MANIFEST, NULL, 736362181Sdim "Entries in revprop manifest for r%ld " 737362181Sdim "are not ordered", revprops->revision); 738289177Speter 739362181Sdim previous_start_rev = start_rev; 740289177Speter } 741289177Speter 742362181Sdim /* Now get the pack file description */ 743362181Sdim idx = get_entry(revprops->manifest, revprops->revision); 744362181Sdim revprops->entry = APR_ARRAY_IDX(revprops->manifest, idx, 745362181Sdim manifest_entry_t); 746289177Speter 747289177Speter return SVN_NO_ERROR; 748289177Speter} 749289177Speter 750289177Speter/* Return TRUE, if revision R1 and R2 refer to the same shard in FS. 751289177Speter */ 752289177Speterstatic svn_boolean_t 753289177Spetersame_shard(svn_fs_t *fs, 754289177Speter svn_revnum_t r1, 755289177Speter svn_revnum_t r2) 756289177Speter{ 757289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 758289177Speter return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir); 759289177Speter} 760289177Speter 761362181Sdim/* Given FS and the full packed file content in CONTENT and make 762362181Sdim * PACKED_REVPROPS point to the first serialized revprop. If READ_ALL 763362181Sdim * is set, initialize the SIZES and OFFSETS members as well. 764289177Speter * 765289177Speter * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as 766289177Speter * well as the SERIALIZED_SIZE member. If revprop caching has been 767289177Speter * enabled, parse all revprops in the pack and cache them. 768289177Speter */ 769289177Speterstatic svn_error_t * 770289177Speterparse_packed_revprops(svn_fs_t *fs, 771289177Speter packed_revprops_t *revprops, 772362181Sdim svn_stringbuf_t *content, 773289177Speter svn_boolean_t read_all, 774289177Speter apr_pool_t *result_pool, 775289177Speter apr_pool_t *scratch_pool) 776289177Speter{ 777362181Sdim apr_size_t count, i; 778289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 779289177Speter svn_boolean_t cache_all = has_revprop_cache(fs, scratch_pool); 780362181Sdim svn_packed__data_root_t *root; 781362181Sdim svn_packed__byte_stream_t *revprops_stream; 782362181Sdim svn_revnum_t first_rev = revprops->entry.start_rev; 783289177Speter 784362181Sdim /* Verify the checksum and decode packed data. */ 785362181Sdim SVN_ERR_W(read_packed_data_checksummed(&root, content, result_pool, 786362181Sdim scratch_pool), 787362181Sdim apr_psprintf(scratch_pool, 788362181Sdim "Revprop pack file for r%ld is corrupt", 789362181Sdim first_rev)); 790289177Speter 791362181Sdim /* get streams */ 792362181Sdim revprops_stream = svn_packed__first_byte_stream(root); 793362181Sdim count = svn_packed__byte_block_count(revprops_stream); 794289177Speter 795289177Speter /* Check revision range for validity. */ 796362181Sdim if (!same_shard(fs, first_rev, first_rev + count - 1) || count < 1) 797289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 798289177Speter _("Revprop pack for revision r%ld" 799289177Speter " contains revprops for r%ld .. r%ld"), 800289177Speter revprops->revision, 801289177Speter (svn_revnum_t)first_rev, 802289177Speter (svn_revnum_t)(first_rev + count -1)); 803289177Speter 804289177Speter /* Since start & end are in the same shard, it is enough to just test 805289177Speter * the FIRST_REV for being actually packed. That will also cover the 806289177Speter * special case of rev 0 never being packed. */ 807289177Speter if (!svn_fs_x__is_packed_revprop(fs, first_rev)) 808289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 809289177Speter _("Revprop pack for revision r%ld" 810289177Speter " starts at non-packed revisions r%ld"), 811289177Speter revprops->revision, (svn_revnum_t)first_rev); 812289177Speter 813362181Sdim /* Request all data (just references to data already expanded in ROOT) */ 814362181Sdim revprops->revprops = apr_array_make(result_pool, (int)count, 815362181Sdim sizeof(svn_string_t)); 816362181Sdim for (i = 0, revprops->total_size = 0; i < count; ++i) 817362181Sdim { 818362181Sdim svn_string_t *props = apr_array_push(revprops->revprops); 819362181Sdim props->data = svn_packed__get_bytes(revprops_stream, &props->len); 820289177Speter 821362181Sdim revprops->total_size += props->len; 822362181Sdim } 823289177Speter 824362181Sdim /* Now parse the serialized revprops. */ 825362181Sdim for (i = 0; i < count; ++i) 826289177Speter { 827362181Sdim const svn_string_t *serialized; 828362181Sdim svn_revnum_t revision; 829289177Speter 830289177Speter svn_pool_clear(iterpool); 831289177Speter 832362181Sdim serialized = &APR_ARRAY_IDX(revprops->revprops, (int)i, svn_string_t); 833362181Sdim revision = first_rev + (long)i; 834289177Speter 835289177Speter /* Parse this revprops list, if necessary */ 836289177Speter if (revision == revprops->revision) 837289177Speter { 838289177Speter /* Parse (and possibly cache) the one revprop list we care about. */ 839289177Speter SVN_ERR(parse_revprop(&revprops->properties, fs, revision, 840362181Sdim serialized, result_pool, iterpool)); 841362181Sdim revprops->serialized_size = serialized->len; 842289177Speter 843289177Speter /* If we only wanted the revprops for REVISION then we are done. */ 844289177Speter if (!read_all && !cache_all) 845289177Speter break; 846289177Speter } 847289177Speter else if (cache_all) 848289177Speter { 849289177Speter /* Parse and cache all other revprop lists. */ 850289177Speter apr_hash_t *properties; 851362181Sdim SVN_ERR(parse_revprop(&properties, fs, revision, serialized, 852289177Speter iterpool, iterpool)); 853289177Speter } 854362181Sdim } 855289177Speter 856362181Sdim svn_pool_destroy(iterpool); 857289177Speter 858289177Speter return SVN_NO_ERROR; 859289177Speter} 860289177Speter 861289177Speter/* In filesystem FS, read the packed revprops for revision REV into 862362181Sdim * *REVPROPS. Populate the revprop cache, if enabled. If you want to 863362181Sdim * modify revprop contents / update REVPROPS, READ_ALL must be set. 864362181Sdim * Otherwise, only the properties of REV are being provided. 865289177Speter * 866289177Speter * Allocate *PROPERTIES in RESULT_POOL and temporaries in SCRATCH_POOL. 867289177Speter */ 868289177Speterstatic svn_error_t * 869289177Speterread_pack_revprop(packed_revprops_t **revprops, 870289177Speter svn_fs_t *fs, 871289177Speter svn_revnum_t rev, 872289177Speter svn_boolean_t read_all, 873289177Speter apr_pool_t *result_pool, 874289177Speter apr_pool_t *scratch_pool) 875289177Speter{ 876289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 877289177Speter svn_boolean_t missing = FALSE; 878289177Speter packed_revprops_t *result; 879289177Speter int i; 880289177Speter 881289177Speter /* someone insisted that REV is packed. Double-check if necessary */ 882289177Speter if (!svn_fs_x__is_packed_revprop(fs, rev)) 883289177Speter SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, iterpool)); 884289177Speter 885289177Speter if (!svn_fs_x__is_packed_revprop(fs, rev)) 886289177Speter return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 887289177Speter _("No such packed revision %ld"), rev); 888289177Speter 889289177Speter /* initialize the result data structure */ 890289177Speter result = apr_pcalloc(result_pool, sizeof(*result)); 891289177Speter result->revision = rev; 892289177Speter 893289177Speter /* try to read the packed revprops. This may require retries if we have 894289177Speter * concurrent writers. */ 895362181Sdim for (i = 0; i < SVN_FS_X__RECOVERABLE_RETRY_COUNT; ++i) 896289177Speter { 897289177Speter const char *file_path; 898362181Sdim svn_stringbuf_t *contents = NULL; 899362181Sdim 900289177Speter svn_pool_clear(iterpool); 901289177Speter 902289177Speter /* there might have been concurrent writes. 903289177Speter * Re-read the manifest and the pack file. 904289177Speter */ 905289177Speter SVN_ERR(get_revprop_packname(fs, result, result_pool, iterpool)); 906362181Sdim file_path = get_revprop_pack_filepath(result, &result->entry, 907362181Sdim iterpool); 908362181Sdim SVN_ERR(svn_fs_x__try_stringbuf_from_file(&contents, 909289177Speter &missing, 910289177Speter file_path, 911289177Speter i + 1 < SVN_FS_X__RECOVERABLE_RETRY_COUNT, 912362181Sdim iterpool)); 913289177Speter 914362181Sdim if (contents) 915362181Sdim { 916362181Sdim SVN_ERR_W(parse_packed_revprops(fs, result, contents, read_all, 917362181Sdim result_pool, iterpool), 918362181Sdim apr_psprintf(iterpool, 919362181Sdim "Revprop pack file for r%ld is corrupt", 920362181Sdim rev)); 921362181Sdim break; 922362181Sdim } 923362181Sdim 924289177Speter /* If we could not find the file, there was a write. 925289177Speter * So, we should refresh our revprop generation info as well such 926289177Speter * that others may find data we will put into the cache. They would 927289177Speter * consider it outdated, otherwise. 928289177Speter */ 929289177Speter if (missing && has_revprop_cache(fs, iterpool)) 930362181Sdim SVN_ERR(read_revprop_generation(fs, iterpool)); 931289177Speter } 932289177Speter 933289177Speter /* the file content should be available now */ 934362181Sdim if (!result->revprops) 935289177Speter return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, 936289177Speter _("Failed to read revprop pack file for r%ld"), rev); 937289177Speter 938289177Speter *revprops = result; 939289177Speter 940289177Speter return SVN_NO_ERROR; 941289177Speter} 942289177Speter 943289177Spetersvn_error_t * 944289177Spetersvn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, 945289177Speter svn_fs_t *fs, 946289177Speter svn_revnum_t rev, 947289177Speter svn_boolean_t bypass_cache, 948362181Sdim svn_boolean_t refresh, 949289177Speter apr_pool_t *result_pool, 950289177Speter apr_pool_t *scratch_pool) 951289177Speter{ 952289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 953289177Speter 954289177Speter /* not found, yet */ 955289177Speter *proplist_p = NULL; 956289177Speter 957289177Speter /* should they be available at all? */ 958289177Speter SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool)); 959289177Speter 960362181Sdim /* Ensure that the revprop generation info is valid. */ 961362181Sdim if (refresh || !is_generation_valid(fs)) 962362181Sdim SVN_ERR(read_revprop_generation(fs, scratch_pool)); 963362181Sdim 964289177Speter /* Try cache lookup first. */ 965289177Speter if (!bypass_cache && has_revprop_cache(fs, scratch_pool)) 966289177Speter { 967289177Speter svn_boolean_t is_cached; 968289177Speter svn_fs_x__pair_cache_key_t key = { 0 }; 969289177Speter 970289177Speter key.revision = rev; 971362181Sdim key.second = ffd->revprop_generation; 972289177Speter SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, 973289177Speter ffd->revprop_cache, &key, result_pool)); 974289177Speter if (is_cached) 975289177Speter return SVN_NO_ERROR; 976289177Speter } 977289177Speter 978289177Speter /* if REV had not been packed when we began, try reading it from the 979289177Speter * non-packed shard. If that fails, we will fall through to packed 980289177Speter * shard reads. */ 981289177Speter if (!svn_fs_x__is_packed_revprop(fs, rev)) 982289177Speter { 983289177Speter svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, 984362181Sdim result_pool, scratch_pool); 985289177Speter if (err) 986289177Speter { 987289177Speter if (!APR_STATUS_IS_ENOENT(err->apr_err)) 988289177Speter return svn_error_trace(err); 989289177Speter 990289177Speter svn_error_clear(err); 991289177Speter *proplist_p = NULL; /* in case read_non_packed_revprop changed it */ 992289177Speter } 993289177Speter } 994289177Speter 995289177Speter /* if revprop packing is available and we have not read the revprops, yet, 996289177Speter * try reading them from a packed shard. If that fails, REV is most 997289177Speter * likely invalid (or its revprops highly contested). */ 998289177Speter if (!*proplist_p) 999289177Speter { 1000289177Speter packed_revprops_t *revprops; 1001362181Sdim SVN_ERR(read_pack_revprop(&revprops, fs, rev, FALSE, 1002289177Speter result_pool, scratch_pool)); 1003289177Speter *proplist_p = revprops->properties; 1004289177Speter } 1005289177Speter 1006289177Speter /* The revprops should have been there. Did we get them? */ 1007289177Speter if (!*proplist_p) 1008289177Speter return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 1009289177Speter _("Could not read revprops for revision %ld"), 1010289177Speter rev); 1011289177Speter 1012289177Speter return SVN_NO_ERROR; 1013289177Speter} 1014289177Speter 1015362181Sdimsvn_error_t * 1016362181Sdimsvn_fs_x__write_non_packed_revprops(apr_file_t *file, 1017362181Sdim apr_hash_t *proplist, 1018362181Sdim apr_pool_t *scratch_pool) 1019362181Sdim{ 1020362181Sdim svn_stream_t *stream; 1021362181Sdim svn_checksum_t *checksum; 1022362181Sdim 1023362181Sdim stream = svn_stream_from_aprfile2(file, TRUE, scratch_pool); 1024362181Sdim stream = svn_checksum__wrap_write_stream(&checksum, stream, 1025362181Sdim svn_checksum_fnv1a_32x4, 1026362181Sdim scratch_pool); 1027362181Sdim SVN_ERR(svn_fs_x__write_properties(stream, proplist, scratch_pool)); 1028362181Sdim SVN_ERR(svn_stream_close(stream)); 1029362181Sdim 1030362181Sdim /* Append the checksum */ 1031362181Sdim SVN_ERR(svn_io_file_write_full(file, checksum->digest, 1032362181Sdim svn_checksum_size(checksum), NULL, 1033362181Sdim scratch_pool)); 1034362181Sdim 1035362181Sdim return SVN_NO_ERROR; 1036362181Sdim} 1037362181Sdim 1038289177Speter/* Serialize the revision property list PROPLIST of revision REV in 1039289177Speter * filesystem FS to a non-packed file. Return the name of that temporary 1040289177Speter * file in *TMP_PATH and the file path that it must be moved to in 1041362181Sdim * *FINAL_PATH. Schedule necessary fsync calls in BATCH. 1042289177Speter * 1043289177Speter * Allocate *FINAL_PATH and *TMP_PATH in RESULT_POOL. Use SCRATCH_POOL 1044289177Speter * for temporary allocations. 1045289177Speter */ 1046289177Speterstatic svn_error_t * 1047289177Speterwrite_non_packed_revprop(const char **final_path, 1048289177Speter const char **tmp_path, 1049289177Speter svn_fs_t *fs, 1050289177Speter svn_revnum_t rev, 1051289177Speter apr_hash_t *proplist, 1052362181Sdim svn_fs_x__batch_fsync_t *batch, 1053289177Speter apr_pool_t *result_pool, 1054289177Speter apr_pool_t *scratch_pool) 1055289177Speter{ 1056362181Sdim apr_file_t *file; 1057289177Speter *final_path = svn_fs_x__path_revprops(fs, rev, result_pool); 1058289177Speter 1059362181Sdim *tmp_path = apr_pstrcat(result_pool, *final_path, ".tmp", SVN_VA_NULL); 1060362181Sdim SVN_ERR(svn_fs_x__batch_fsync_open_file(&file, batch, *tmp_path, 1061362181Sdim scratch_pool)); 1062289177Speter 1063362181Sdim SVN_ERR(svn_fs_x__write_non_packed_revprops(file, proplist, scratch_pool)); 1064362181Sdim 1065289177Speter return SVN_NO_ERROR; 1066289177Speter} 1067289177Speter 1068289177Speter/* After writing the new revprop file(s), call this function to move the 1069289177Speter * file at TMP_PATH to FINAL_PATH and give it the permissions from 1070362181Sdim * PERMS_REFERENCE. Schedule necessary fsync calls in BATCH. 1071289177Speter * 1072289177Speter * If indicated in BUMP_GENERATION, increase FS' revprop generation. 1073289177Speter * Finally, delete all the temporary files given in FILES_TO_DELETE. 1074289177Speter * The latter may be NULL. 1075289177Speter * 1076289177Speter * Use SCRATCH_POOL for temporary allocations. 1077289177Speter */ 1078289177Speterstatic svn_error_t * 1079289177Speterswitch_to_new_revprop(svn_fs_t *fs, 1080289177Speter const char *final_path, 1081289177Speter const char *tmp_path, 1082289177Speter const char *perms_reference, 1083289177Speter apr_array_header_t *files_to_delete, 1084289177Speter svn_boolean_t bump_generation, 1085362181Sdim svn_fs_x__batch_fsync_t *batch, 1086289177Speter apr_pool_t *scratch_pool) 1087289177Speter{ 1088289177Speter /* Now, we may actually be replacing revprops. Make sure that all other 1089289177Speter threads and processes will know about this. */ 1090289177Speter if (bump_generation) 1091362181Sdim SVN_ERR(begin_revprop_change(fs, scratch_pool)); 1092289177Speter 1093362181Sdim /* Ensure the new file contents makes it to disk before switching over to 1094362181Sdim * it. */ 1095362181Sdim SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool)); 1096362181Sdim 1097362181Sdim /* Make the revision visible to all processes and threads. */ 1098289177Speter SVN_ERR(svn_fs_x__move_into_place(tmp_path, final_path, perms_reference, 1099362181Sdim batch, scratch_pool)); 1100362181Sdim SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool)); 1101289177Speter 1102289177Speter /* Indicate that the update (if relevant) has been completed. */ 1103289177Speter if (bump_generation) 1104362181Sdim SVN_ERR(end_revprop_change(fs, scratch_pool)); 1105289177Speter 1106289177Speter /* Clean up temporary files, if necessary. */ 1107289177Speter if (files_to_delete) 1108289177Speter { 1109289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1110289177Speter int i; 1111289177Speter 1112289177Speter for (i = 0; i < files_to_delete->nelts; ++i) 1113289177Speter { 1114289177Speter const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*); 1115289177Speter 1116289177Speter svn_pool_clear(iterpool); 1117289177Speter SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 1118289177Speter } 1119289177Speter 1120289177Speter svn_pool_destroy(iterpool); 1121289177Speter } 1122289177Speter return SVN_NO_ERROR; 1123289177Speter} 1124289177Speter 1125362181Sdim/* Writes the a pack file to FILE. It copies the serialized data 1126362181Sdim * from REVPROPS for the indexes [START,END). 1127289177Speter * 1128289177Speter * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. 1129289177Speter * SCRATCH_POOL is used for temporary allocations. 1130289177Speter */ 1131289177Speterstatic svn_error_t * 1132289177Speterrepack_revprops(svn_fs_t *fs, 1133289177Speter packed_revprops_t *revprops, 1134289177Speter int start, 1135289177Speter int end, 1136362181Sdim apr_size_t new_total_size, 1137362181Sdim apr_file_t *file, 1138289177Speter apr_pool_t *scratch_pool) 1139289177Speter{ 1140289177Speter int i; 1141289177Speter 1142362181Sdim svn_packed__data_root_t *root = svn_packed__data_create_root(scratch_pool); 1143362181Sdim svn_packed__byte_stream_t *revprops_stream 1144362181Sdim = svn_packed__create_bytes_stream(root); 1145289177Speter 1146289177Speter /* append the serialized revprops */ 1147289177Speter for (i = start; i < end; ++i) 1148362181Sdim { 1149362181Sdim const svn_string_t *props 1150362181Sdim = &APR_ARRAY_IDX(revprops->revprops, i, svn_string_t); 1151289177Speter 1152362181Sdim svn_packed__add_bytes(revprops_stream, props->data, props->len); 1153362181Sdim } 1154289177Speter 1155362181Sdim /* Write to file. */ 1156362181Sdim SVN_ERR(write_packed_data_checksummed(root, file, scratch_pool)); 1157289177Speter 1158289177Speter return SVN_NO_ERROR; 1159289177Speter} 1160289177Speter 1161362181Sdim/* Allocate a new pack file name for revisions starting at START_REV in 1162362181Sdim * REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, 1163362181Sdim * auto-create that array if necessary. Return an open file *FILE that is 1164362181Sdim * allocated in RESULT_POOL. Allocate the paths in *FILES_TO_DELETE from 1165362181Sdim * the same pool that contains the array itself. Schedule necessary fsync 1166362181Sdim * calls in BATCH. 1167289177Speter * 1168289177Speter * Use SCRATCH_POOL for temporary allocations. 1169289177Speter */ 1170289177Speterstatic svn_error_t * 1171362181Sdimrepack_file_open(apr_file_t **file, 1172362181Sdim svn_fs_t *fs, 1173362181Sdim packed_revprops_t *revprops, 1174362181Sdim svn_revnum_t start_rev, 1175362181Sdim apr_array_header_t **files_to_delete, 1176362181Sdim svn_fs_x__batch_fsync_t *batch, 1177362181Sdim apr_pool_t *result_pool, 1178362181Sdim apr_pool_t *scratch_pool) 1179289177Speter{ 1180362181Sdim manifest_entry_t new_entry; 1181362181Sdim const char *new_path; 1182362181Sdim int idx; 1183289177Speter 1184362181Sdim /* We always replace whole pack files - possibly by more than one new file. 1185362181Sdim * When we create the file for the first part of the pack, enlist the old 1186362181Sdim * one for later deletion */ 1187362181Sdim SVN_ERR_ASSERT(start_rev >= revprops->entry.start_rev); 1188289177Speter 1189289177Speter if (*files_to_delete == NULL) 1190289177Speter *files_to_delete = apr_array_make(result_pool, 3, sizeof(const char*)); 1191289177Speter 1192362181Sdim if (revprops->entry.start_rev == start_rev) 1193362181Sdim APR_ARRAY_PUSH(*files_to_delete, const char*) 1194362181Sdim = get_revprop_pack_filepath(revprops, &revprops->entry, 1195362181Sdim (*files_to_delete)->pool); 1196289177Speter 1197362181Sdim /* Initialize the new manifest entry. Bump the tag part. */ 1198362181Sdim new_entry.start_rev = start_rev; 1199362181Sdim new_entry.tag = revprops->entry.tag + 1; 1200289177Speter 1201289177Speter /* update the manifest to point to the new file */ 1202362181Sdim idx = get_entry(revprops->manifest, start_rev); 1203362181Sdim if (revprops->entry.start_rev == start_rev) 1204362181Sdim APR_ARRAY_IDX(revprops->manifest, idx, manifest_entry_t) = new_entry; 1205362181Sdim else 1206362181Sdim SVN_ERR(svn_sort__array_insert2(revprops->manifest, &new_path, idx + 1)); 1207289177Speter 1208362181Sdim /* open the file */ 1209362181Sdim new_path = get_revprop_pack_filepath(revprops, &new_entry, scratch_pool); 1210362181Sdim SVN_ERR(svn_fs_x__batch_fsync_open_file(file, batch, new_path, 1211362181Sdim scratch_pool)); 1212289177Speter 1213289177Speter return SVN_NO_ERROR; 1214289177Speter} 1215289177Speter 1216362181Sdim/* Return the length of the serialized reprop list of index I in REVPROPS. */ 1217362181Sdimstatic apr_size_t 1218362181Sdimprops_len(packed_revprops_t *revprops, 1219362181Sdim int i) 1220362181Sdim{ 1221362181Sdim return APR_ARRAY_IDX(revprops->revprops, i, svn_string_t).len; 1222362181Sdim} 1223362181Sdim 1224289177Speter/* For revision REV in filesystem FS, set the revision properties to 1225289177Speter * PROPLIST. Return a new file in *TMP_PATH that the caller shall move 1226289177Speter * to *FINAL_PATH to make the change visible. Files to be deleted will 1227289177Speter * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. 1228362181Sdim * Schedule necessary fsync calls in BATCH. 1229289177Speter * 1230289177Speter * Allocate output values in RESULT_POOL and temporaries from SCRATCH_POOL. 1231289177Speter */ 1232289177Speterstatic svn_error_t * 1233289177Speterwrite_packed_revprop(const char **final_path, 1234289177Speter const char **tmp_path, 1235289177Speter apr_array_header_t **files_to_delete, 1236289177Speter svn_fs_t *fs, 1237289177Speter svn_revnum_t rev, 1238289177Speter apr_hash_t *proplist, 1239362181Sdim svn_fs_x__batch_fsync_t *batch, 1240289177Speter apr_pool_t *result_pool, 1241289177Speter apr_pool_t *scratch_pool) 1242289177Speter{ 1243289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 1244289177Speter packed_revprops_t *revprops; 1245289177Speter svn_stream_t *stream; 1246362181Sdim apr_file_t *file; 1247289177Speter svn_stringbuf_t *serialized; 1248362181Sdim apr_size_t new_total_size; 1249289177Speter int changed_index; 1250362181Sdim int count; 1251289177Speter 1252289177Speter /* read the current revprop generation. This value will not change 1253289177Speter * while we hold the global write lock to this FS. */ 1254289177Speter if (has_revprop_cache(fs, scratch_pool)) 1255362181Sdim SVN_ERR(read_revprop_generation(fs, scratch_pool)); 1256289177Speter 1257289177Speter /* read contents of the current pack file */ 1258362181Sdim SVN_ERR(read_pack_revprop(&revprops, fs, rev, TRUE, 1259289177Speter scratch_pool, scratch_pool)); 1260289177Speter 1261289177Speter /* serialize the new revprops */ 1262289177Speter serialized = svn_stringbuf_create_empty(scratch_pool); 1263289177Speter stream = svn_stream_from_stringbuf(serialized, scratch_pool); 1264362181Sdim SVN_ERR(svn_fs_x__write_properties(stream, proplist, scratch_pool)); 1265289177Speter SVN_ERR(svn_stream_close(stream)); 1266289177Speter 1267362181Sdim /* estimate the size of the new data */ 1268362181Sdim count = revprops->revprops->nelts; 1269362181Sdim changed_index = (int)(rev - revprops->entry.start_rev); 1270289177Speter new_total_size = revprops->total_size - revprops->serialized_size 1271289177Speter + serialized->len 1272362181Sdim + (count + 2) * SVN_INT64_BUFFER_SIZE; 1273289177Speter 1274362181Sdim APR_ARRAY_IDX(revprops->revprops, changed_index, svn_string_t) 1275362181Sdim = *svn_stringbuf__morph_into_string(serialized); 1276289177Speter 1277289177Speter /* can we put the new data into the same pack as the before? */ 1278362181Sdim if (new_total_size < ffd->revprop_pack_size || count == 1) 1279289177Speter { 1280289177Speter /* simply replace the old pack file with new content as we do it 1281289177Speter * in the non-packed case */ 1282289177Speter 1283362181Sdim *final_path = get_revprop_pack_filepath(revprops, &revprops->entry, 1284362181Sdim result_pool); 1285362181Sdim *tmp_path = apr_pstrcat(result_pool, *final_path, ".tmp", SVN_VA_NULL); 1286362181Sdim SVN_ERR(svn_fs_x__batch_fsync_open_file(&file, batch, *tmp_path, 1287362181Sdim scratch_pool)); 1288362181Sdim SVN_ERR(repack_revprops(fs, revprops, 0, count, 1289362181Sdim new_total_size, file, scratch_pool)); 1290289177Speter } 1291289177Speter else 1292289177Speter { 1293289177Speter /* split the pack file into two of roughly equal size */ 1294362181Sdim int right_count, left_count; 1295289177Speter 1296289177Speter int left = 0; 1297362181Sdim int right = count - 1; 1298362181Sdim apr_size_t left_size = 2 * SVN_INT64_BUFFER_SIZE; 1299362181Sdim apr_size_t right_size = 2 * SVN_INT64_BUFFER_SIZE; 1300289177Speter 1301289177Speter /* let left and right side grow such that their size difference 1302289177Speter * is minimal after each step. */ 1303289177Speter while (left <= right) 1304362181Sdim if ( left_size + props_len(revprops, left) 1305362181Sdim < right_size + props_len(revprops, right)) 1306289177Speter { 1307362181Sdim left_size += props_len(revprops, left) + SVN_INT64_BUFFER_SIZE; 1308289177Speter ++left; 1309289177Speter } 1310289177Speter else 1311289177Speter { 1312362181Sdim right_size += props_len(revprops, right) + SVN_INT64_BUFFER_SIZE; 1313289177Speter --right; 1314289177Speter } 1315289177Speter 1316289177Speter /* since the items need much less than SVN_INT64_BUFFER_SIZE 1317289177Speter * bytes to represent their length, the split may not be optimal */ 1318289177Speter left_count = left; 1319362181Sdim right_count = count - left; 1320289177Speter 1321289177Speter /* if new_size is large, one side may exceed the pack size limit. 1322289177Speter * In that case, split before and after the modified revprop.*/ 1323289177Speter if ( left_size > ffd->revprop_pack_size 1324289177Speter || right_size > ffd->revprop_pack_size) 1325289177Speter { 1326289177Speter left_count = changed_index; 1327362181Sdim right_count = count - left_count - 1; 1328289177Speter } 1329289177Speter 1330289177Speter /* Allocate this here such that we can call the repack functions with 1331289177Speter * the scratch pool alone. */ 1332289177Speter if (*files_to_delete == NULL) 1333289177Speter *files_to_delete = apr_array_make(result_pool, 3, 1334289177Speter sizeof(const char*)); 1335289177Speter 1336289177Speter /* write the new, split files */ 1337289177Speter if (left_count) 1338289177Speter { 1339362181Sdim SVN_ERR(repack_file_open(&file, fs, revprops, 1340362181Sdim revprops->entry.start_rev, 1341362181Sdim files_to_delete, batch, 1342362181Sdim scratch_pool, scratch_pool)); 1343289177Speter SVN_ERR(repack_revprops(fs, revprops, 0, left_count, 1344362181Sdim new_total_size, file, scratch_pool)); 1345289177Speter } 1346289177Speter 1347362181Sdim if (left_count + right_count < count) 1348289177Speter { 1349362181Sdim SVN_ERR(repack_file_open(&file, fs, revprops, rev, 1350362181Sdim files_to_delete, batch, 1351362181Sdim scratch_pool, scratch_pool)); 1352289177Speter SVN_ERR(repack_revprops(fs, revprops, changed_index, 1353289177Speter changed_index + 1, 1354362181Sdim new_total_size, file, scratch_pool)); 1355289177Speter } 1356289177Speter 1357289177Speter if (right_count) 1358289177Speter { 1359362181Sdim SVN_ERR(repack_file_open(&file, fs, revprops, rev + 1, 1360362181Sdim files_to_delete, batch, 1361362181Sdim scratch_pool, scratch_pool)); 1362362181Sdim SVN_ERR(repack_revprops(fs, revprops, count - right_count, count, 1363362181Sdim new_total_size, file, scratch_pool)); 1364289177Speter } 1365289177Speter 1366289177Speter /* write the new manifest */ 1367289177Speter *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, 1368289177Speter result_pool); 1369362181Sdim *tmp_path = apr_pstrcat(result_pool, *final_path, ".tmp", SVN_VA_NULL); 1370362181Sdim SVN_ERR(svn_fs_x__batch_fsync_open_file(&file, batch, *tmp_path, 1371362181Sdim scratch_pool)); 1372362181Sdim SVN_ERR(write_manifest(file, revprops->manifest, scratch_pool)); 1373289177Speter } 1374289177Speter 1375289177Speter return SVN_NO_ERROR; 1376289177Speter} 1377289177Speter 1378289177Speter/* Set the revision property list of revision REV in filesystem FS to 1379289177Speter PROPLIST. Use SCRATCH_POOL for temporary allocations. */ 1380289177Spetersvn_error_t * 1381289177Spetersvn_fs_x__set_revision_proplist(svn_fs_t *fs, 1382289177Speter svn_revnum_t rev, 1383289177Speter apr_hash_t *proplist, 1384289177Speter apr_pool_t *scratch_pool) 1385289177Speter{ 1386289177Speter svn_boolean_t is_packed; 1387289177Speter svn_boolean_t bump_generation = FALSE; 1388289177Speter const char *final_path; 1389289177Speter const char *tmp_path; 1390289177Speter const char *perms_reference; 1391289177Speter apr_array_header_t *files_to_delete = NULL; 1392362181Sdim svn_fs_x__batch_fsync_t *batch; 1393362181Sdim svn_fs_x__data_t *ffd = fs->fsap_data; 1394289177Speter 1395289177Speter SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool)); 1396289177Speter 1397362181Sdim /* Perform all fsyncs through this instance. */ 1398362181Sdim SVN_ERR(svn_fs_x__batch_fsync_create(&batch, ffd->flush_to_disk, 1399362181Sdim scratch_pool)); 1400362181Sdim 1401289177Speter /* this info will not change while we hold the global FS write lock */ 1402289177Speter is_packed = svn_fs_x__is_packed_revprop(fs, rev); 1403289177Speter 1404289177Speter /* Test whether revprops already exist for this revision. 1405289177Speter * Only then will we need to bump the revprop generation. 1406289177Speter * The fact that they did not yet exist is never cached. */ 1407289177Speter if (is_packed) 1408289177Speter { 1409289177Speter bump_generation = TRUE; 1410289177Speter } 1411289177Speter else 1412289177Speter { 1413289177Speter svn_node_kind_t kind; 1414289177Speter SVN_ERR(svn_io_check_path(svn_fs_x__path_revprops(fs, rev, 1415289177Speter scratch_pool), 1416289177Speter &kind, scratch_pool)); 1417289177Speter bump_generation = kind != svn_node_none; 1418289177Speter } 1419289177Speter 1420289177Speter /* Serialize the new revprop data */ 1421289177Speter if (is_packed) 1422289177Speter SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, 1423362181Sdim fs, rev, proplist, batch, scratch_pool, 1424289177Speter scratch_pool)); 1425289177Speter else 1426289177Speter SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, 1427362181Sdim fs, rev, proplist, batch, 1428362181Sdim scratch_pool, scratch_pool)); 1429289177Speter 1430289177Speter /* We use the rev file of this revision as the perms reference, 1431289177Speter * because when setting revprops for the first time, the revprop 1432289177Speter * file won't exist and therefore can't serve as its own reference. 1433289177Speter * (Whereas the rev file should already exist at this point.) 1434289177Speter */ 1435289177Speter perms_reference = svn_fs_x__path_rev_absolute(fs, rev, scratch_pool); 1436289177Speter 1437289177Speter /* Now, switch to the new revprop data. */ 1438289177Speter SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, 1439362181Sdim files_to_delete, bump_generation, batch, 1440289177Speter scratch_pool)); 1441289177Speter 1442289177Speter return SVN_NO_ERROR; 1443289177Speter} 1444289177Speter 1445289177Speter/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. 1446289177Speter * Use SCRATCH_POOL for temporary allocations. 1447289177Speter * Set *MISSING, if the reason is a missing manifest or pack file. 1448289177Speter */ 1449289177Spetersvn_boolean_t 1450289177Spetersvn_fs_x__packed_revprop_available(svn_boolean_t *missing, 1451289177Speter svn_fs_t *fs, 1452289177Speter svn_revnum_t revision, 1453289177Speter apr_pool_t *scratch_pool) 1454289177Speter{ 1455362181Sdim svn_node_kind_t kind; 1456362181Sdim packed_revprops_t *revprops; 1457362181Sdim svn_error_t *err; 1458289177Speter 1459289177Speter /* try to read the manifest file */ 1460362181Sdim revprops = apr_pcalloc(scratch_pool, sizeof(*revprops)); 1461362181Sdim revprops->revision = revision; 1462362181Sdim err = get_revprop_packname(fs, revprops, scratch_pool, scratch_pool); 1463289177Speter 1464289177Speter /* if the manifest cannot be read, consider the pack files inaccessible 1465289177Speter * even if the file itself exists. */ 1466289177Speter if (err) 1467289177Speter { 1468289177Speter svn_error_clear(err); 1469289177Speter return FALSE; 1470289177Speter } 1471289177Speter 1472362181Sdim /* the respective pack file must exist (and be a file) */ 1473362181Sdim err = svn_io_check_path(get_revprop_pack_filepath(revprops, 1474362181Sdim &revprops->entry, 1475362181Sdim scratch_pool), 1476362181Sdim &kind, scratch_pool); 1477362181Sdim if (err) 1478289177Speter { 1479362181Sdim svn_error_clear(err); 1480362181Sdim return FALSE; 1481289177Speter } 1482289177Speter 1483362181Sdim *missing = kind == svn_node_none; 1484362181Sdim return kind == svn_node_file; 1485289177Speter} 1486289177Speter 1487289177Speter 1488289177Speter/****** Packing FSX shards *********/ 1489289177Speter 1490362181Sdim/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH 1491362181Sdim * in filesystem FS to the pack file at PACK_FILE_NAME in PACK_FILE_DIR. 1492362181Sdim * 1493362181Sdim * The file sizes have already been determined and written to SIZES. 1494362181Sdim * Please note that this function will be executed while the filesystem 1495362181Sdim * has been locked and that revprops files will therefore not be modified 1496362181Sdim * while the pack is in progress. 1497362181Sdim * 1498362181Sdim * COMPRESSION_LEVEL defines how well the resulting pack file shall be 1499362181Sdim * compressed or whether is shall be compressed at all. TOTAL_SIZE is 1500362181Sdim * a hint on which initial buffer size we should use to hold the pack file 1501362181Sdim * content. Schedule necessary fsync calls in BATCH. 1502362181Sdim * 1503362181Sdim * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations 1504362181Sdim * are done in SCRATCH_POOL. 1505362181Sdim */ 1506362181Sdimstatic svn_error_t * 1507362181Sdimcopy_revprops(svn_fs_t *fs, 1508362181Sdim const char *pack_file_dir, 1509362181Sdim const char *pack_filename, 1510362181Sdim const char *shard_path, 1511362181Sdim svn_revnum_t start_rev, 1512362181Sdim svn_revnum_t end_rev, 1513362181Sdim apr_array_header_t *sizes, 1514362181Sdim apr_size_t total_size, 1515362181Sdim int compression_level, 1516362181Sdim svn_fs_x__batch_fsync_t *batch, 1517362181Sdim svn_cancel_func_t cancel_func, 1518362181Sdim void *cancel_baton, 1519362181Sdim apr_pool_t *scratch_pool) 1520289177Speter{ 1521289177Speter apr_file_t *pack_file; 1522289177Speter svn_revnum_t rev; 1523289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1524289177Speter 1525362181Sdim svn_packed__data_root_t *root = svn_packed__data_create_root(scratch_pool); 1526362181Sdim svn_packed__byte_stream_t *stream 1527362181Sdim = svn_packed__create_bytes_stream(root); 1528289177Speter 1529289177Speter /* Iterate over the revisions in this shard, squashing them together. */ 1530289177Speter for (rev = start_rev; rev <= end_rev; rev++) 1531289177Speter { 1532289177Speter const char *path; 1533362181Sdim svn_stringbuf_t *props; 1534289177Speter 1535289177Speter svn_pool_clear(iterpool); 1536289177Speter 1537289177Speter /* Construct the file name. */ 1538362181Sdim path = svn_fs_x__path_revprops(fs, rev, iterpool); 1539289177Speter 1540289177Speter /* Copy all the bits from the non-packed revprop file to the end of 1541289177Speter * the pack file. */ 1542362181Sdim SVN_ERR(svn_stringbuf_from_file2(&props, path, iterpool)); 1543362181Sdim SVN_ERR_W(verify_checksum(props, iterpool), 1544362181Sdim apr_psprintf(iterpool, "Failed to read revprops for r%ld.", 1545362181Sdim rev)); 1546362181Sdim 1547362181Sdim svn_packed__add_bytes(stream, props->data, props->len); 1548289177Speter } 1549289177Speter 1550362181Sdim /* Create the auto-fsync'ing pack file. */ 1551362181Sdim SVN_ERR(svn_fs_x__batch_fsync_open_file(&pack_file, batch, 1552362181Sdim svn_dirent_join(pack_file_dir, 1553362181Sdim pack_filename, 1554362181Sdim scratch_pool), 1555362181Sdim scratch_pool)); 1556289177Speter 1557362181Sdim /* write all to disk */ 1558362181Sdim SVN_ERR(write_packed_data_checksummed(root, pack_file, scratch_pool)); 1559289177Speter 1560289177Speter svn_pool_destroy(iterpool); 1561289177Speter 1562289177Speter return SVN_NO_ERROR; 1563289177Speter} 1564289177Speter 1565289177Spetersvn_error_t * 1566362181Sdimsvn_fs_x__pack_revprops_shard(svn_fs_t *fs, 1567362181Sdim const char *pack_file_dir, 1568289177Speter const char *shard_path, 1569289177Speter apr_int64_t shard, 1570289177Speter int max_files_per_dir, 1571362181Sdim apr_int64_t max_pack_size, 1572289177Speter int compression_level, 1573362181Sdim svn_fs_x__batch_fsync_t *batch, 1574289177Speter svn_cancel_func_t cancel_func, 1575289177Speter void *cancel_baton, 1576289177Speter apr_pool_t *scratch_pool) 1577289177Speter{ 1578289177Speter const char *manifest_file_path, *pack_filename = NULL; 1579362181Sdim apr_file_t *manifest_file; 1580289177Speter svn_revnum_t start_rev, end_rev, rev; 1581362181Sdim apr_size_t total_size; 1582289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1583289177Speter apr_array_header_t *sizes; 1584362181Sdim apr_array_header_t *manifest; 1585289177Speter 1586362181Sdim /* Sanitize config file values. */ 1587362181Sdim apr_size_t max_size = (apr_size_t)MIN(MAX(max_pack_size, 1), 1588362181Sdim SVN_MAX_OBJECT_SIZE); 1589362181Sdim 1590289177Speter /* Some useful paths. */ 1591289177Speter manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, 1592289177Speter scratch_pool); 1593289177Speter 1594362181Sdim /* Create the manifest file. */ 1595362181Sdim SVN_ERR(svn_fs_x__batch_fsync_open_file(&manifest_file, batch, 1596362181Sdim manifest_file_path, scratch_pool)); 1597289177Speter 1598289177Speter /* revisions to handle. Special case: revision 0 */ 1599289177Speter start_rev = (svn_revnum_t) (shard * max_files_per_dir); 1600289177Speter end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); 1601289177Speter if (start_rev == 0) 1602362181Sdim { 1603362181Sdim /* Never pack revprops for r0, just copy it. */ 1604362181Sdim SVN_ERR(svn_io_copy_file(svn_fs_x__path_revprops(fs, 0, iterpool), 1605362181Sdim svn_dirent_join(pack_file_dir, "p0", 1606362181Sdim scratch_pool), 1607362181Sdim TRUE, 1608362181Sdim iterpool)); 1609289177Speter 1610362181Sdim ++start_rev; 1611362181Sdim /* Special special case: if max_files_per_dir is 1, then at this point 1612362181Sdim start_rev == 1 and end_rev == 0 (!). Fortunately, everything just 1613362181Sdim works. */ 1614362181Sdim } 1615362181Sdim 1616289177Speter /* initialize the revprop size info */ 1617362181Sdim sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_size_t)); 1618289177Speter total_size = 2 * SVN_INT64_BUFFER_SIZE; 1619289177Speter 1620362181Sdim manifest = apr_array_make(scratch_pool, 4, sizeof(manifest_entry_t)); 1621362181Sdim 1622289177Speter /* Iterate over the revisions in this shard, determine their size and 1623289177Speter * squashing them together into pack files. */ 1624289177Speter for (rev = start_rev; rev <= end_rev; rev++) 1625289177Speter { 1626289177Speter apr_finfo_t finfo; 1627289177Speter const char *path; 1628289177Speter 1629289177Speter svn_pool_clear(iterpool); 1630289177Speter 1631289177Speter /* Get the size of the file. */ 1632362181Sdim path = svn_fs_x__path_revprops(fs, rev, iterpool); 1633289177Speter SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); 1634289177Speter 1635362181Sdim /* If we already have started a pack file and this revprop cannot be 1636362181Sdim * appended to it, write the previous pack file. Note this overflow 1637362181Sdim * check works because we enforced MAX_SIZE <= SVN_MAX_OBJECT_SIZE. */ 1638362181Sdim if (sizes->nelts != 0 1639362181Sdim && ( finfo.size > max_size 1640362181Sdim || total_size > max_size 1641362181Sdim || SVN_INT64_BUFFER_SIZE + finfo.size > max_size - total_size)) 1642289177Speter { 1643362181Sdim SVN_ERR(copy_revprops(fs, pack_file_dir, pack_filename, 1644362181Sdim shard_path, start_rev, rev-1, 1645362181Sdim sizes, (apr_size_t)total_size, 1646362181Sdim compression_level, batch, cancel_func, 1647362181Sdim cancel_baton, iterpool)); 1648289177Speter 1649289177Speter /* next pack file starts empty again */ 1650289177Speter apr_array_clear(sizes); 1651289177Speter total_size = 2 * SVN_INT64_BUFFER_SIZE; 1652289177Speter start_rev = rev; 1653289177Speter } 1654289177Speter 1655289177Speter /* Update the manifest. Allocate a file name for the current pack 1656289177Speter * file if it is a new one */ 1657289177Speter if (sizes->nelts == 0) 1658362181Sdim { 1659362181Sdim manifest_entry_t *entry = apr_array_push(manifest); 1660362181Sdim entry->start_rev = rev; 1661362181Sdim entry->tag = 0; 1662289177Speter 1663362181Sdim pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); 1664362181Sdim } 1665289177Speter 1666289177Speter /* add to list of files to put into the current pack file */ 1667362181Sdim APR_ARRAY_PUSH(sizes, apr_size_t) = finfo.size; 1668289177Speter total_size += SVN_INT64_BUFFER_SIZE + finfo.size; 1669289177Speter } 1670289177Speter 1671289177Speter /* write the last pack file */ 1672289177Speter if (sizes->nelts != 0) 1673362181Sdim SVN_ERR(copy_revprops(fs, pack_file_dir, pack_filename, shard_path, 1674362181Sdim start_rev, rev-1, sizes, 1675362181Sdim (apr_size_t)total_size, compression_level, 1676362181Sdim batch, cancel_func, cancel_baton, iterpool)); 1677289177Speter 1678362181Sdim SVN_ERR(write_manifest(manifest_file, manifest, iterpool)); 1679362181Sdim 1680362181Sdim /* flush all data to disk and update permissions */ 1681289177Speter SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); 1682289177Speter svn_pool_destroy(iterpool); 1683289177Speter 1684289177Speter return SVN_NO_ERROR; 1685289177Speter} 1686