1/* fs.h : interface to Subversion filesystem, private to libsvn_fs 2 * 3 * ==================================================================== 4 * Licensed to the Apache Software Foundation (ASF) under one 5 * or more contributor license agreements. See the NOTICE file 6 * distributed with this work for additional information 7 * regarding copyright ownership. The ASF licenses this file 8 * to you under the Apache License, Version 2.0 (the 9 * "License"); you may not use this file except in compliance 10 * with the License. You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, 15 * software distributed under the License is distributed on an 16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 * KIND, either express or implied. See the License for the 18 * specific language governing permissions and limitations 19 * under the License. 20 * ==================================================================== 21 */ 22 23#ifndef SVN_LIBSVN_FS_X_H 24#define SVN_LIBSVN_FS_X_H 25 26#include <apr_pools.h> 27#include <apr_hash.h> 28#include <apr_network_io.h> 29#include <apr_md5.h> 30#include <apr_sha1.h> 31 32#include "svn_fs.h" 33#include "svn_config.h" 34#include "private/svn_atomic.h" 35#include "private/svn_cache.h" 36#include "private/svn_fs_private.h" 37#include "private/svn_sqlite.h" 38#include "private/svn_mutex.h" 39 40#include "id.h" 41 42#ifdef __cplusplus 43extern "C" { 44#endif /* __cplusplus */ 45 46 47/*** The filesystem structure. ***/ 48 49/* Following are defines that specify the textual elements of the 50 native filesystem directories and revision files. */ 51 52/* Names of special files in the fs_x filesystem. */ 53#define PATH_FORMAT "format" /* Contains format number */ 54#define PATH_UUID "uuid" /* Contains UUID */ 55#define PATH_CURRENT "current" /* Youngest revision */ 56#define PATH_LOCK_FILE "write-lock" /* Revision lock file */ 57#define PATH_PACK_LOCK_FILE "pack-lock" /* Pack lock file */ 58#define PATH_REVS_DIR "revs" /* Directory of revisions */ 59#define PATH_REVPROPS_DIR "revprops" /* Directory of revprops */ 60#define PATH_TXNS_DIR "transactions" /* Directory of transactions */ 61#define PATH_NODE_ORIGINS_DIR "node-origins" /* Lazy node-origin cache */ 62#define PATH_TXN_PROTOS_DIR "txn-protorevs" /* Directory of proto-revs */ 63#define PATH_TXN_CURRENT "txn-current" /* File with next txn key */ 64#define PATH_TXN_CURRENT_LOCK "txn-current-lock" /* Lock for txn-current */ 65#define PATH_LOCKS_DIR "locks" /* Directory of locks */ 66#define PATH_MIN_UNPACKED_REV "min-unpacked-rev" /* Oldest revision which 67 has not been packed. */ 68#define PATH_REVPROP_GENERATION "revprop-generation" 69 /* Current revprop generation*/ 70#define PATH_MANIFEST "manifest" /* Manifest file name */ 71#define PATH_PACKED "pack" /* Packed revision data file */ 72#define PATH_EXT_PACKED_SHARD ".pack" /* Extension for packed 73 shards */ 74#define PATH_EXT_L2P_INDEX ".l2p" /* extension of the log- 75 to-phys index */ 76#define PATH_EXT_P2L_INDEX ".p2l" /* extension of the phys- 77 to-log index */ 78/* If you change this, look at tests/svn_test_fs.c(maybe_install_fsx_conf) */ 79#define PATH_CONFIG "fsx.conf" /* Configuration */ 80 81/* Names of special files and file extensions for transactions */ 82#define PATH_CHANGES "changes" /* Records changes made so far */ 83#define PATH_TXN_PROPS "props" /* Transaction properties */ 84#define PATH_TXN_PROPS_FINAL "props-final" /* Final transaction properties 85 before moving to revprops */ 86#define PATH_NEXT_IDS "next-ids" /* Next temporary ID assignments */ 87#define PATH_PREFIX_NODE "node." /* Prefix for node filename */ 88#define PATH_EXT_TXN ".txn" /* Extension of txn dir */ 89#define PATH_EXT_CHILDREN ".children" /* Extension for dir contents */ 90#define PATH_EXT_PROPS ".props" /* Extension for node props */ 91#define PATH_EXT_REV ".rev" /* Extension of protorev file */ 92#define PATH_EXT_REV_LOCK ".rev-lock" /* Extension of protorev lock file */ 93#define PATH_TXN_ITEM_INDEX "itemidx" /* File containing the current item 94 index number */ 95#define PATH_INDEX "index" /* name of index files w/o ext */ 96 97/* Names of files in legacy FS formats */ 98#define PATH_REV "rev" /* Proto rev file */ 99#define PATH_REV_LOCK "rev-lock" /* Proto rev (write) lock file */ 100 101/* Names of sections and options in fsx.conf. */ 102#define CONFIG_SECTION_CACHES "caches" 103#define CONFIG_OPTION_FAIL_STOP "fail-stop" 104#define CONFIG_SECTION_REP_SHARING "rep-sharing" 105#define CONFIG_OPTION_ENABLE_REP_SHARING "enable-rep-sharing" 106#define CONFIG_SECTION_DELTIFICATION "deltification" 107#define CONFIG_OPTION_MAX_DELTIFICATION_WALK "max-deltification-walk" 108#define CONFIG_OPTION_MAX_LINEAR_DELTIFICATION "max-linear-deltification" 109#define CONFIG_OPTION_COMPRESSION_LEVEL "compression-level" 110#define CONFIG_SECTION_PACKED_REVPROPS "packed-revprops" 111#define CONFIG_OPTION_REVPROP_PACK_SIZE "revprop-pack-size" 112#define CONFIG_OPTION_COMPRESS_PACKED_REVPROPS "compress-packed-revprops" 113#define CONFIG_SECTION_IO "io" 114#define CONFIG_OPTION_BLOCK_SIZE "block-size" 115#define CONFIG_OPTION_L2P_PAGE_SIZE "l2p-page-size" 116#define CONFIG_OPTION_P2L_PAGE_SIZE "p2l-page-size" 117#define CONFIG_SECTION_DEBUG "debug" 118#define CONFIG_OPTION_PACK_AFTER_COMMIT "pack-after-commit" 119 120/* The format number of this filesystem. 121 This is independent of the repository format number, and 122 independent of any other FS back ends. 123 124 Note: If you bump this, please update the switch statement in 125 svn_fs_x__create() as well. 126 */ 127#define SVN_FS_X__FORMAT_NUMBER 1 128 129/* On most operating systems apr implements file locks per process, not 130 per file. On Windows apr implements the locking as per file handle 131 locks, so we don't have to add our own mutex for just in-process 132 synchronization. */ 133#if APR_HAS_THREADS && !defined(WIN32) 134#define SVN_FS_X__USE_LOCK_MUTEX 1 135#else 136#define SVN_FS_X__USE_LOCK_MUTEX 0 137#endif 138 139/* Private FSX-specific data shared between all svn_txn_t objects that 140 relate to a particular transaction in a filesystem (as identified 141 by transaction id and filesystem UUID). Objects of this type are 142 allocated in their own subpool of the common pool. */ 143typedef struct svn_fs_x__shared_txn_data_t 144{ 145 /* The next transaction in the list, or NULL if there is no following 146 transaction. */ 147 struct svn_fs_x__shared_txn_data_t *next; 148 149 /* ID of this transaction. */ 150 svn_fs_x__txn_id_t txn_id; 151 152 /* Whether the transaction's prototype revision file is locked for 153 writing by any thread in this process (including the current 154 thread; recursive locks are not permitted). This is effectively 155 a non-recursive mutex. */ 156 svn_boolean_t being_written; 157 158 /* The pool in which this object has been allocated; a subpool of the 159 common pool. */ 160 apr_pool_t *pool; 161} svn_fs_x__shared_txn_data_t; 162 163/* Private FSX-specific data shared between all svn_fs_t objects that 164 relate to a particular filesystem, as identified by filesystem UUID. 165 Objects of this type are allocated in the common pool. */ 166typedef struct svn_fs_x__shared_data_t 167{ 168 /* A list of shared transaction objects for each transaction that is 169 currently active, or NULL if none are. All access to this list, 170 including the contents of the objects stored in it, is synchronised 171 under TXN_LIST_LOCK. */ 172 svn_fs_x__shared_txn_data_t *txns; 173 174 /* A free transaction object, or NULL if there is no free object. 175 Access to this object is synchronised under TXN_LIST_LOCK. */ 176 svn_fs_x__shared_txn_data_t *free_txn; 177 178 /* The following lock must be taken out in reverse order of their 179 declaration here. Any subset may be acquired and held at any given 180 time but their relative acquisition order must not change. 181 182 (lock 'txn-current' before 'pack' before 'write' before 'txn-list') */ 183 184 /* A lock for intra-process synchronization when accessing the TXNS list. */ 185 svn_mutex__t *txn_list_lock; 186 187 /* A lock for intra-process synchronization when grabbing the 188 repository write lock. */ 189 svn_mutex__t *fs_write_lock; 190 191 /* A lock for intra-process synchronization when grabbing the 192 repository pack operation lock. */ 193 svn_mutex__t *fs_pack_lock; 194 195 /* A lock for intra-process synchronization when locking the 196 txn-current file. */ 197 svn_mutex__t *txn_current_lock; 198 199 /* The common pool, under which this object is allocated, subpools 200 of which are used to allocate the transaction objects. */ 201 apr_pool_t *common_pool; 202} svn_fs_x__shared_data_t; 203 204/* Data structure for the 1st level DAG node cache. */ 205typedef struct svn_fs_x__dag_cache_t svn_fs_x__dag_cache_t; 206 207/* Key type for all caches that use revision + offset / counter as key. 208 209 Note: Cache keys should be 16 bytes for best performance and there 210 should be no padding. */ 211typedef struct svn_fs_x__pair_cache_key_t 212{ 213 /* The object's revision. Use the 64 data type to prevent padding. */ 214 apr_int64_t revision; 215 216 /* Sub-address: item index, revprop generation, packed flag, etc. */ 217 apr_int64_t second; 218} svn_fs_x__pair_cache_key_t; 219 220/* Key type that identifies a representation / rep header. 221 222 Note: Cache keys should require no padding. */ 223typedef struct svn_fs_x__representation_cache_key_t 224{ 225 /* Revision that contains the representation */ 226 apr_int64_t revision; 227 228 /* Packed or non-packed representation (boolean)? */ 229 apr_int64_t is_packed; 230 231 /* Item index of the representation */ 232 apr_uint64_t item_index; 233} svn_fs_x__representation_cache_key_t; 234 235/* Key type that identifies a txdelta window. 236 237 Note: Cache keys should require no padding. */ 238typedef struct svn_fs_x__window_cache_key_t 239{ 240 /* The object's revision. Use the 64 data type to prevent padding. */ 241 apr_int64_t revision; 242 243 /* Window number within that representation. */ 244 apr_int64_t chunk_index; 245 246 /* Item index of the representation */ 247 apr_uint64_t item_index; 248} svn_fs_x__window_cache_key_t; 249 250/* Private (non-shared) FSX-specific data for each svn_fs_t object. 251 Any caches in here may be NULL. */ 252typedef struct svn_fs_x__data_t 253{ 254 /* The format number of this FS. */ 255 int format; 256 257 /* The maximum number of files to store per directory. */ 258 int max_files_per_dir; 259 260 /* Rev / pack file read granularity in bytes. */ 261 apr_int64_t block_size; 262 263 /* Rev / pack file granularity (in bytes) covered by a single phys-to-log 264 * index page. */ 265 /* Capacity in entries of log-to-phys index pages */ 266 apr_int64_t l2p_page_size; 267 268 /* Rev / pack file granularity covered by phys-to-log index pages */ 269 apr_int64_t p2l_page_size; 270 271 /* The revision that was youngest, last time we checked. */ 272 svn_revnum_t youngest_rev_cache; 273 274 /* Caches of immutable data. (Note that these may be shared between 275 multiple svn_fs_t's for the same filesystem.) */ 276 277 /* Access to the configured memcached instances. May be NULL. */ 278 svn_memcache_t *memcache; 279 280 /* If TRUE, don't ignore any cache-related errors. If FALSE, errors from 281 e.g. memcached may be ignored as caching is an optional feature. */ 282 svn_boolean_t fail_stop; 283 284 /* Caches native dag_node_t* instances and acts as a 1st level cache */ 285 svn_fs_x__dag_cache_t *dag_node_cache; 286 287 /* DAG node cache for immutable nodes. Maps (revision, fspath) 288 to (dag_node_t *). This is the 2nd level cache for DAG nodes. */ 289 svn_cache__t *rev_node_cache; 290 291 /* A cache of the contents of immutable directories; maps from 292 unparsed FS ID to a apr_hash_t * mapping (const char *) dirent 293 names to (svn_fs_x__dirent_t *). */ 294 svn_cache__t *dir_cache; 295 296 /* Fulltext cache; currently only used with memcached. Maps from 297 rep key (revision/offset) to svn_stringbuf_t. */ 298 svn_cache__t *fulltext_cache; 299 300 /* Access object to the revprop "generation". Will be NULL until 301 the first access. May be also get closed and set to NULL again. */ 302 apr_file_t *revprop_generation_file; 303 304 /* Revision property cache. Maps from (rev,generation) to apr_hash_t. */ 305 svn_cache__t *revprop_cache; 306 307 /* Node properties cache. Maps from rep key to apr_hash_t. */ 308 svn_cache__t *properties_cache; 309 310 /* Pack manifest cache; a cache mapping (svn_revnum_t) shard number to 311 a manifest; and a manifest is a mapping from (svn_revnum_t) revision 312 number offset within a shard to (apr_off_t) byte-offset in the 313 respective pack file. */ 314 svn_cache__t *packed_offset_cache; 315 316 /* Cache for txdelta_window_t objects; 317 * the key is svn_fs_x__window_cache_key_t */ 318 svn_cache__t *txdelta_window_cache; 319 320 /* Cache for combined windows as svn_stringbuf_t objects; 321 the key is svn_fs_x__window_cache_key_t */ 322 svn_cache__t *combined_window_cache; 323 324 /* Cache for svn_fs_x__rep_header_t objects; 325 * the key is (revision, item index) */ 326 svn_cache__t *node_revision_cache; 327 328 /* Cache for noderevs_t containers; 329 the key is a (pack file revision, file offset) pair */ 330 svn_cache__t *noderevs_container_cache; 331 332 /* Cache for change lists as APR arrays of svn_fs_x__change_t * objects; 333 the key is the revision */ 334 svn_cache__t *changes_cache; 335 336 /* Cache for change_list_t containers; 337 the key is a (pack file revision, file offset) pair */ 338 svn_cache__t *changes_container_cache; 339 340 /* Cache for star-delta / representation containers; 341 the key is a (pack file revision, file offset) pair */ 342 svn_cache__t *reps_container_cache; 343 344 /* Cache for svn_fs_x__rep_header_t objects; the key is a 345 (revision, item index) pair */ 346 svn_cache__t *rep_header_cache; 347 348 /* Cache for svn_mergeinfo_t objects; the key is a combination of 349 revision, inheritance flags and path. */ 350 svn_cache__t *mergeinfo_cache; 351 352 /* Cache for presence of svn_mergeinfo_t on a noderev; the key is a 353 combination of revision, inheritance flags and path; value is "1" 354 if the node has mergeinfo, "0" if it doesn't. */ 355 svn_cache__t *mergeinfo_existence_cache; 356 357 /* Cache for l2p_header_t objects; the key is (revision, is-packed). 358 Will be NULL for pre-format7 repos */ 359 svn_cache__t *l2p_header_cache; 360 361 /* Cache for l2p_page_t objects; the key is svn_fs_x__page_cache_key_t. 362 Will be NULL for pre-format7 repos */ 363 svn_cache__t *l2p_page_cache; 364 365 /* Cache for p2l_header_t objects; the key is (revision, is-packed). 366 Will be NULL for pre-format7 repos */ 367 svn_cache__t *p2l_header_cache; 368 369 /* Cache for apr_array_header_t objects containing svn_fs_x__p2l_entry_t 370 elements; the key is svn_fs_x__page_cache_key_t. 371 Will be NULL for pre-format7 repos */ 372 svn_cache__t *p2l_page_cache; 373 374 /* TRUE while the we hold a lock on the write lock file. */ 375 svn_boolean_t has_write_lock; 376 377 /* Data shared between all svn_fs_t objects for a given filesystem. */ 378 svn_fs_x__shared_data_t *shared; 379 380 /* The sqlite database used for rep caching. */ 381 svn_sqlite__db_t *rep_cache_db; 382 383 /* Thread-safe boolean */ 384 svn_atomic_t rep_cache_db_opened; 385 386 /* The oldest revision not in a pack file. It also applies to revprops 387 * if revprop packing has been enabled by the FSX format version. */ 388 svn_revnum_t min_unpacked_rev; 389 390 /* Whether rep-sharing is supported by the filesystem 391 * and allowed by the configuration. */ 392 svn_boolean_t rep_sharing_allowed; 393 394 /* File size limit in bytes up to which multiple revprops shall be packed 395 * into a single file. */ 396 apr_int64_t revprop_pack_size; 397 398 /* Whether packed revprop files shall be compressed. */ 399 svn_boolean_t compress_packed_revprops; 400 401 /* Restart deltification histories after each multiple of this value */ 402 apr_int64_t max_deltification_walk; 403 404 /* Maximum number of length of the linear part at the top of the 405 * deltification history after which skip deltas will be used. */ 406 apr_int64_t max_linear_deltification; 407 408 /* Compression level to use with txdelta storage format in new revs. */ 409 int delta_compression_level; 410 411 /* Pack after every commit. */ 412 svn_boolean_t pack_after_commit; 413 414 /* Per-instance filesystem ID, which provides an additional level of 415 uniqueness for filesystems that share the same UUID, but should 416 still be distinguishable (e.g. backups produced by svn_fs_hotcopy() 417 or dump / load cycles). */ 418 const char *instance_id; 419 420 /* Pointer to svn_fs_open. */ 421 svn_error_t *(*svn_fs_open_)(svn_fs_t **, const char *, apr_hash_t *, 422 apr_pool_t *, apr_pool_t *); 423} svn_fs_x__data_t; 424 425 426/*** Filesystem Transaction ***/ 427typedef struct svn_fs_x__transaction_t 428{ 429 /* property list (const char * name, svn_string_t * value). 430 may be NULL if there are no properties. */ 431 apr_hash_t *proplist; 432 433 /* revision upon which this txn is base. (unfinished only) */ 434 svn_revnum_t base_rev; 435 436 /* copies list (const char * copy_ids), or NULL if there have been 437 no copies in this transaction. */ 438 apr_array_header_t *copies; 439 440} svn_fs_x__transaction_t; 441 442 443/*** Representation ***/ 444/* If you add fields to this, check to see if you need to change 445 * svn_fs_x__rep_copy. */ 446typedef struct svn_fs_x__representation_t 447{ 448 /* Checksums digests for the contents produced by this representation. 449 This checksum is for the contents the rep shows to consumers, 450 regardless of how the rep stores the data under the hood. It is 451 independent of the storage (fulltext, delta, whatever). 452 453 If has_sha1 is FALSE, then for compatibility behave as though this 454 checksum matches the expected checksum. 455 456 The md5 checksum is always filled, unless this is rep which was 457 retrieved from the rep-cache. The sha1 checksum is only computed on 458 a write, for use with rep-sharing. */ 459 svn_boolean_t has_sha1; 460 unsigned char sha1_digest[APR_SHA1_DIGESTSIZE]; 461 unsigned char md5_digest[APR_MD5_DIGESTSIZE]; 462 463 /* Change set and item number where this representation is located. */ 464 svn_fs_x__id_t id; 465 466 /* The size of the representation in bytes as seen in the revision 467 file. */ 468 svn_filesize_t size; 469 470 /* The size of the fulltext of the representation. */ 471 svn_filesize_t expanded_size; 472 473} svn_fs_x__representation_t; 474 475 476/*** Node-Revision ***/ 477/* If you add fields to this, check to see if you need to change 478 * copy_node_revision in dag.c. */ 479typedef struct svn_fs_x__noderev_t 480{ 481 /* Predecessor node revision id. Will be "unused" if there is no 482 predecessor for this node revision. */ 483 svn_fs_x__id_t predecessor_id; 484 485 /* The ID of this noderev */ 486 svn_fs_x__id_t noderev_id; 487 488 /* Identifier of the node that this noderev belongs to. */ 489 svn_fs_x__id_t node_id; 490 491 /* Copy identifier of this line of history. */ 492 svn_fs_x__id_t copy_id; 493 494 /* If this node-rev is a copy, where was it copied from? */ 495 const char *copyfrom_path; 496 svn_revnum_t copyfrom_rev; 497 498 /* Helper for history tracing, root of the parent tree from whence 499 this node-rev was copied. */ 500 svn_revnum_t copyroot_rev; 501 const char *copyroot_path; 502 503 /* node kind */ 504 svn_node_kind_t kind; 505 506 /* number of predecessors this node revision has (recursively). */ 507 int predecessor_count; 508 509 /* representation key for this node's properties. may be NULL if 510 there are no properties. */ 511 svn_fs_x__representation_t *prop_rep; 512 513 /* representation for this node's data. may be NULL if there is 514 no data. */ 515 svn_fs_x__representation_t *data_rep; 516 517 /* path at which this node first came into existence. */ 518 const char *created_path; 519 520 /* Does this node itself have svn:mergeinfo? */ 521 svn_boolean_t has_mergeinfo; 522 523 /* Number of nodes with svn:mergeinfo properties that are 524 descendants of this node (including it itself) */ 525 apr_int64_t mergeinfo_count; 526 527} svn_fs_x__noderev_t; 528 529 530/** The type of a directory entry. */ 531typedef struct svn_fs_x__dirent_t 532{ 533 534 /** The name of this directory entry. */ 535 const char *name; 536 537 /** The node revision ID it names. */ 538 svn_fs_x__id_t id; 539 540 /** The node kind. */ 541 svn_node_kind_t kind; 542} svn_fs_x__dirent_t; 543 544 545/*** Change ***/ 546typedef struct svn_fs_x__change_t 547{ 548 /* Path of the change. */ 549 svn_string_t path; 550 551 /* node revision id of changed path */ 552 svn_fs_x__id_t noderev_id; 553 554 /* See svn_fs_path_change2_t for a description for the remaining elements. 555 */ 556 svn_fs_path_change_kind_t change_kind; 557 558 svn_boolean_t text_mod; 559 svn_boolean_t prop_mod; 560 svn_node_kind_t node_kind; 561 562 svn_boolean_t copyfrom_known; 563 svn_revnum_t copyfrom_rev; 564 const char *copyfrom_path; 565 566 svn_tristate_t mergeinfo_mod; 567} svn_fs_x__change_t; 568 569 570#ifdef __cplusplus 571} 572#endif /* __cplusplus */ 573 574#endif /* SVN_LIBSVN_FS_X_H */ 575