1/* verify.c --- verification of FSX filesystems 2 * 3 * ==================================================================== 4 * Licensed to the Apache Software Foundation (ASF) under one 5 * or more contributor license agreements. See the NOTICE file 6 * distributed with this work for additional information 7 * regarding copyright ownership. The ASF licenses this file 8 * to you under the Apache License, Version 2.0 (the 9 * "License"); you may not use this file except in compliance 10 * with the License. You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, 15 * software distributed under the License is distributed on an 16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 * KIND, either express or implied. See the License for the 18 * specific language governing permissions and limitations 19 * under the License. 20 * ==================================================================== 21 */ 22 23#include "verify.h" 24#include "fs_x.h" 25#include "svn_time.h" 26#include "private/svn_subr_private.h" 27 28#include "cached_data.h" 29#include "rep-cache.h" 30#include "util.h" 31#include "index.h" 32 33#include "../libsvn_fs/fs-loader.h" 34 35#include "svn_private_config.h" 36 37 38/** Verifying. **/ 39 40/* Baton type expected by verify_walker(). The purpose is to limit the 41 * number of notifications sent. 42 */ 43typedef struct verify_walker_baton_t 44{ 45 /* number of calls to verify_walker() since the last clean */ 46 int iteration_count; 47 48 /* progress notification callback to invoke periodically (may be NULL) */ 49 svn_fs_progress_notify_func_t notify_func; 50 51 /* baton to use with NOTIFY_FUNC */ 52 void *notify_baton; 53 54 /* remember the last revision for which we called notify_func */ 55 svn_revnum_t last_notified_revision; 56} verify_walker_baton_t; 57 58/* Used by svn_fs_x__verify(). 59 Implements svn_fs_x__walk_rep_reference().walker. */ 60static svn_error_t * 61verify_walker(svn_fs_x__representation_t *rep, 62 void *baton, 63 svn_fs_t *fs, 64 apr_pool_t *scratch_pool) 65{ 66 verify_walker_baton_t *walker_baton = baton; 67 68 /* notify and free resources periodically */ 69 if (walker_baton->iteration_count > 1000) 70 { 71 svn_revnum_t revision = svn_fs_x__get_revnum(rep->id.change_set); 72 if ( walker_baton->notify_func 73 && revision != walker_baton->last_notified_revision) 74 { 75 walker_baton->notify_func(revision, 76 walker_baton->notify_baton, 77 scratch_pool); 78 walker_baton->last_notified_revision = revision; 79 } 80 81 walker_baton->iteration_count = 0; 82 } 83 84 /* access the repo data */ 85 SVN_ERR(svn_fs_x__check_rep(rep, fs, scratch_pool)); 86 87 /* update resource usage counters */ 88 walker_baton->iteration_count++; 89 90 return SVN_NO_ERROR; 91} 92 93/* Verify the rep cache DB's consistency with our rev / pack data. 94 * The function signature is similar to svn_fs_x__verify. 95 * The values of START and END have already been auto-selected and 96 * verified. 97 */ 98static svn_error_t * 99verify_rep_cache(svn_fs_t *fs, 100 svn_revnum_t start, 101 svn_revnum_t end, 102 svn_fs_progress_notify_func_t notify_func, 103 void *notify_baton, 104 svn_cancel_func_t cancel_func, 105 void *cancel_baton, 106 apr_pool_t *scratch_pool) 107{ 108 svn_boolean_t exists; 109 110 /* rep-cache verification. */ 111 SVN_ERR(svn_fs_x__exists_rep_cache(&exists, fs, scratch_pool)); 112 if (exists) 113 { 114 /* provide a baton to allow the reuse of open file handles between 115 iterations (saves 2/3 of OS level file operations). */ 116 verify_walker_baton_t *baton 117 = apr_pcalloc(scratch_pool, sizeof(*baton)); 118 119 baton->last_notified_revision = SVN_INVALID_REVNUM; 120 baton->notify_func = notify_func; 121 baton->notify_baton = notify_baton; 122 123 /* tell the user that we are now ready to do *something* */ 124 if (notify_func) 125 notify_func(SVN_INVALID_REVNUM, notify_baton, scratch_pool); 126 127 /* Do not attempt to walk the rep-cache database if its file does 128 not exist, since doing so would create it --- which may confuse 129 the administrator. Don't take any lock. */ 130 SVN_ERR(svn_fs_x__walk_rep_reference(fs, start, end, 131 verify_walker, baton, 132 cancel_func, cancel_baton, 133 scratch_pool)); 134 } 135 136 return SVN_NO_ERROR; 137} 138 139/* Verify that the MD5 checksum of the data between offsets START and END 140 * in FILE matches the EXPECTED checksum. If there is a mismatch use the 141 * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC 142 * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ 143static svn_error_t * 144verify_index_checksum(apr_file_t *file, 145 const char *name, 146 apr_off_t start, 147 apr_off_t end, 148 svn_checksum_t *expected, 149 svn_cancel_func_t cancel_func, 150 void *cancel_baton, 151 apr_pool_t *scratch_pool) 152{ 153 unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; 154 apr_off_t size = end - start; 155 svn_checksum_t *actual; 156 svn_checksum_ctx_t *context 157 = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); 158 159 /* Calculate the index checksum. */ 160 SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool)); 161 while (size > 0) 162 { 163 apr_size_t to_read = size > sizeof(buffer) 164 ? sizeof(buffer) 165 : (apr_size_t)size; 166 SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, 167 scratch_pool)); 168 SVN_ERR(svn_checksum_update(context, buffer, to_read)); 169 size -= to_read; 170 171 if (cancel_func) 172 SVN_ERR(cancel_func(cancel_baton)); 173 } 174 175 SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); 176 177 /* Verify that it matches the expected checksum. */ 178 if (!svn_checksum_match(expected, actual)) 179 { 180 const char *file_name; 181 182 SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 183 SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, 184 _("%s checksum mismatch in file %s"), 185 name, file_name)); 186 } 187 188 return SVN_NO_ERROR; 189} 190 191/* Verify the MD5 checksums of the index data in the rev / pack file 192 * containing revision START in FS. If given, invoke CANCEL_FUNC with 193 * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary 194 * allocations. 195 */ 196static svn_error_t * 197verify_index_checksums(svn_fs_t *fs, 198 svn_revnum_t start, 199 svn_cancel_func_t cancel_func, 200 void *cancel_baton, 201 apr_pool_t *scratch_pool) 202{ 203 svn_fs_x__revision_file_t *rev_file; 204 205 /* Open the rev / pack file and read the footer */ 206 SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, 207 scratch_pool, scratch_pool)); 208 SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); 209 210 /* Verify the index contents against the checksum from the footer. */ 211 SVN_ERR(verify_index_checksum(rev_file->file, "L2P index", 212 rev_file->l2p_offset, rev_file->p2l_offset, 213 rev_file->l2p_checksum, 214 cancel_func, cancel_baton, scratch_pool)); 215 SVN_ERR(verify_index_checksum(rev_file->file, "P2L index", 216 rev_file->p2l_offset, rev_file->footer_offset, 217 rev_file->p2l_checksum, 218 cancel_func, cancel_baton, scratch_pool)); 219 220 /* Done. */ 221 SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 222 223 return SVN_NO_ERROR; 224} 225 226/* Verify that for all log-to-phys index entries for revisions START to 227 * START + COUNT-1 in FS there is a consistent entry in the phys-to-log 228 * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 229 * intervals. Use SCRATCH_POOL for temporary allocations. 230 */ 231static svn_error_t * 232compare_l2p_to_p2l_index(svn_fs_t *fs, 233 svn_revnum_t start, 234 svn_revnum_t count, 235 svn_cancel_func_t cancel_func, 236 void *cancel_baton, 237 apr_pool_t *scratch_pool) 238{ 239 svn_revnum_t i; 240 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 241 apr_array_header_t *max_ids; 242 243 /* common file access structure */ 244 svn_fs_x__revision_file_t *rev_file; 245 SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, 246 iterpool)); 247 248 /* determine the range of items to check for each revision */ 249 SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, start, count, scratch_pool, 250 iterpool)); 251 252 /* check all items in all revisions if the given range */ 253 for (i = 0; i < max_ids->nelts; ++i) 254 { 255 apr_uint64_t k; 256 apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t); 257 svn_revnum_t revision = start + i; 258 259 for (k = 0; k < max_id; ++k) 260 { 261 apr_off_t offset; 262 apr_uint32_t sub_item; 263 svn_fs_x__id_t l2p_item; 264 svn_fs_x__id_t *p2l_item; 265 266 l2p_item.change_set = svn_fs_x__change_set_by_rev(revision); 267 l2p_item.number = k; 268 269 /* get L2P entry. Ignore unused entries. */ 270 SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev_file, 271 &l2p_item, iterpool)); 272 if (offset == -1) 273 continue; 274 275 /* find the corresponding P2L entry */ 276 SVN_ERR(svn_fs_x__p2l_item_lookup(&p2l_item, fs, rev_file, 277 revision, offset, sub_item, 278 iterpool, iterpool)); 279 280 if (p2l_item == NULL) 281 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 282 NULL, 283 _("p2l index entry not found for " 284 "PHYS o%s:s%ld returned by " 285 "l2p index for LOG r%ld:i%ld"), 286 apr_off_t_toa(scratch_pool, offset), 287 (long)sub_item, revision, (long)k); 288 289 if (!svn_fs_x__id_eq(&l2p_item, p2l_item)) 290 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 291 NULL, 292 _("p2l index info LOG r%ld:i%ld" 293 " does not match " 294 "l2p index for LOG r%ld:i%ld"), 295 svn_fs_x__get_revnum(p2l_item->change_set), 296 (long)p2l_item->number, revision, 297 (long)k); 298 299 svn_pool_clear(iterpool); 300 } 301 302 if (cancel_func) 303 SVN_ERR(cancel_func(cancel_baton)); 304 } 305 306 svn_pool_destroy(iterpool); 307 308 SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 309 310 return SVN_NO_ERROR; 311} 312 313/* Verify that for all phys-to-log index entries for revisions START to 314 * START + COUNT-1 in FS there is a consistent entry in the log-to-phys 315 * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 316 * intervals. Use SCRATCH_POOL for temporary allocations. 317 * 318 * Please note that we can only check on pack / rev file granularity and 319 * must only be called for a single rev / pack file. 320 */ 321static svn_error_t * 322compare_p2l_to_l2p_index(svn_fs_t *fs, 323 svn_revnum_t start, 324 svn_revnum_t count, 325 svn_cancel_func_t cancel_func, 326 void *cancel_baton, 327 apr_pool_t *scratch_pool) 328{ 329 svn_fs_x__data_t *ffd = fs->fsap_data; 330 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 331 apr_pool_t *iterpool2 = svn_pool_create(scratch_pool); 332 apr_off_t max_offset; 333 apr_off_t offset = 0; 334 335 /* common file access structure */ 336 svn_fs_x__revision_file_t *rev_file; 337 SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, 338 iterpool)); 339 340 /* get the size of the rev / pack file as covered by the P2L index */ 341 SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, 342 scratch_pool)); 343 344 /* for all offsets in the file, get the P2L index entries and check 345 them against the L2P index */ 346 for (offset = 0; offset < max_offset; ) 347 { 348 apr_array_header_t *entries; 349 svn_fs_x__p2l_entry_t *last_entry; 350 int i; 351 352 svn_pool_clear(iterpool); 353 354 /* get all entries for the current block */ 355 SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, 356 offset, ffd->p2l_page_size, 357 iterpool, iterpool)); 358 if (entries->nelts == 0) 359 return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, 360 NULL, 361 _("p2l does not cover offset %s" 362 " for revision %ld"), 363 apr_off_t_toa(scratch_pool, offset), start); 364 365 /* process all entries (and later continue with the next block) */ 366 last_entry 367 = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_x__p2l_entry_t); 368 offset = last_entry->offset + last_entry->size; 369 370 for (i = 0; i < entries->nelts; ++i) 371 { 372 apr_uint32_t k; 373 svn_fs_x__p2l_entry_t *entry 374 = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); 375 376 /* check all sub-items for consist entries in the L2P index */ 377 for (k = 0; k < entry->item_count; ++k) 378 { 379 apr_off_t l2p_offset; 380 apr_uint32_t sub_item; 381 svn_fs_x__id_t *p2l_item = &entry->items[k]; 382 svn_revnum_t revision 383 = svn_fs_x__get_revnum(p2l_item->change_set); 384 385 svn_pool_clear(iterpool2); 386 SVN_ERR(svn_fs_x__item_offset(&l2p_offset, &sub_item, fs, 387 rev_file, p2l_item, iterpool2)); 388 389 if (sub_item != k || l2p_offset != entry->offset) 390 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 391 NULL, 392 _("l2p index entry PHYS o%s:s%ld " 393 "does not match p2l index value " 394 "LOG r%ld:i%ld for PHYS o%s:s%ld"), 395 apr_off_t_toa(scratch_pool, 396 l2p_offset), 397 (long)sub_item, 398 revision, 399 (long)p2l_item->number, 400 apr_off_t_toa(scratch_pool, 401 entry->offset), 402 (long)k); 403 } 404 } 405 406 if (cancel_func) 407 SVN_ERR(cancel_func(cancel_baton)); 408 } 409 410 svn_pool_destroy(iterpool2); 411 svn_pool_destroy(iterpool); 412 413 SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 414 415 return SVN_NO_ERROR; 416} 417 418/* Items smaller than this can be read at once into a buffer and directly 419 * be checksummed. Larger items require stream processing. 420 * Must be a multiple of 8. */ 421#define STREAM_THRESHOLD 4096 422 423/* Verify that the next SIZE bytes read from FILE are NUL. SIZE must not 424 * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. 425 */ 426static svn_error_t * 427expect_buffer_nul(apr_file_t *file, 428 apr_off_t size, 429 apr_pool_t *scratch_pool) 430{ 431 union 432 { 433 unsigned char buffer[STREAM_THRESHOLD]; 434 apr_uint64_t chunks[STREAM_THRESHOLD / sizeof(apr_uint64_t)]; 435 } data; 436 437 apr_size_t i; 438 SVN_ERR_ASSERT(size <= STREAM_THRESHOLD); 439 440 /* read the whole data block; error out on failure */ 441 data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0; 442 SVN_ERR(svn_io_file_read_full2(file, data.buffer, size, NULL, NULL, 443 scratch_pool)); 444 445 /* chunky check */ 446 for (i = 0; i < size / sizeof(apr_uint64_t); ++i) 447 if (data.chunks[i] != 0) 448 break; 449 450 /* byte-wise check upon mismatch or at the end of the block */ 451 for (i *= sizeof(apr_uint64_t); i < size; ++i) 452 if (data.buffer[i] != 0) 453 { 454 const char *file_name; 455 apr_off_t offset; 456 457 SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 458 SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); 459 offset -= size - i; 460 461 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 462 _("Empty section in file %s contains " 463 "non-NUL data at offset %s"), 464 file_name, 465 apr_off_t_toa(scratch_pool, offset)); 466 } 467 468 return SVN_NO_ERROR; 469} 470 471/* Verify that the next SIZE bytes read from FILE are NUL. 472 * Use SCRATCH_POOL for temporary allocations. 473 */ 474static svn_error_t * 475read_all_nul(apr_file_t *file, 476 apr_off_t size, 477 apr_pool_t *scratch_pool) 478{ 479 for (; size >= STREAM_THRESHOLD; size -= STREAM_THRESHOLD) 480 SVN_ERR(expect_buffer_nul(file, STREAM_THRESHOLD, scratch_pool)); 481 482 if (size) 483 SVN_ERR(expect_buffer_nul(file, size, scratch_pool)); 484 485 return SVN_NO_ERROR; 486} 487 488/* Compare the ACTUAL checksum with the one expected by ENTRY. 489 * Return an error in case of mismatch. Use the name of FILE 490 * in error message. Allocate temporary data in SCRATCH_POOL. 491 */ 492static svn_error_t * 493expected_checksum(apr_file_t *file, 494 svn_fs_x__p2l_entry_t *entry, 495 apr_uint32_t actual, 496 apr_pool_t *scratch_pool) 497{ 498 if (actual != entry->fnv1_checksum) 499 { 500 const char *file_name; 501 502 SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 503 SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 504 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 505 _("Checksum mismatch in item at offset %s of " 506 "length %s bytes in file %s"), 507 apr_off_t_toa(scratch_pool, entry->offset), 508 apr_off_t_toa(scratch_pool, entry->size), 509 file_name); 510 } 511 512 return SVN_NO_ERROR; 513} 514 515/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read 516 * from FILE will match ENTRY's expected checksum. SIZE must not 517 * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. 518 */ 519static svn_error_t * 520expected_buffered_checksum(apr_file_t *file, 521 svn_fs_x__p2l_entry_t *entry, 522 apr_pool_t *scratch_pool) 523{ 524 unsigned char buffer[STREAM_THRESHOLD]; 525 SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD); 526 527 SVN_ERR(svn_io_file_read_full2(file, buffer, (apr_size_t)entry->size, 528 NULL, NULL, scratch_pool)); 529 SVN_ERR(expected_checksum(file, entry, 530 svn__fnv1a_32x4(buffer, (apr_size_t)entry->size), 531 scratch_pool)); 532 533 return SVN_NO_ERROR; 534} 535 536/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read from 537 * FILE will match ENTRY's expected checksum. 538 * Use SCRATCH_POOL for temporary allocations. 539 */ 540static svn_error_t * 541expected_streamed_checksum(apr_file_t *file, 542 svn_fs_x__p2l_entry_t *entry, 543 apr_pool_t *scratch_pool) 544{ 545 unsigned char buffer[STREAM_THRESHOLD]; 546 svn_checksum_t *checksum; 547 svn_checksum_ctx_t *context 548 = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, scratch_pool); 549 apr_off_t size = entry->size; 550 551 while (size > 0) 552 { 553 apr_size_t to_read = size > sizeof(buffer) 554 ? sizeof(buffer) 555 : (apr_size_t)size; 556 SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, 557 scratch_pool)); 558 SVN_ERR(svn_checksum_update(context, buffer, to_read)); 559 size -= to_read; 560 } 561 562 SVN_ERR(svn_checksum_final(&checksum, context, scratch_pool)); 563 SVN_ERR(expected_checksum(file, entry, 564 ntohl(*(const apr_uint32_t *)checksum->digest), 565 scratch_pool)); 566 567 return SVN_NO_ERROR; 568} 569 570/* Verify that for all phys-to-log index entries for revisions START to 571 * START + COUNT-1 in FS match the actual pack / rev file contents. 572 * If given, invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 573 * Use SCRATCH_POOL for temporary allocations. 574 * 575 * Please note that we can only check on pack / rev file granularity and 576 * must only be called for a single rev / pack file. 577 */ 578static svn_error_t * 579compare_p2l_to_rev(svn_fs_t *fs, 580 svn_revnum_t start, 581 svn_revnum_t count, 582 svn_cancel_func_t cancel_func, 583 void *cancel_baton, 584 apr_pool_t *scratch_pool) 585{ 586 svn_fs_x__data_t *ffd = fs->fsap_data; 587 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 588 apr_off_t max_offset; 589 apr_off_t offset = 0; 590 svn_fs_x__revision_file_t *rev_file; 591 592 /* open the pack / rev file that is covered by the p2l index */ 593 SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, 594 iterpool)); 595 596 /* check file size vs. range covered by index */ 597 SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); 598 SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, 599 scratch_pool)); 600 601 if (rev_file->l2p_offset != max_offset) 602 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, 603 _("File size of %s for revision r%ld does " 604 "not match p2l index size of %s"), 605 apr_off_t_toa(scratch_pool, 606 rev_file->l2p_offset), 607 start, 608 apr_off_t_toa(scratch_pool, 609 max_offset)); 610 611 SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, NULL, 0, 612 scratch_pool)); 613 614 /* for all offsets in the file, get the P2L index entries and check 615 them against the L2P index */ 616 for (offset = 0; offset < max_offset; ) 617 { 618 apr_array_header_t *entries; 619 int i; 620 621 svn_pool_clear(iterpool); 622 623 /* get all entries for the current block */ 624 SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, 625 offset, ffd->p2l_page_size, 626 iterpool, iterpool)); 627 628 /* The above might have moved the file pointer. 629 * Ensure we actually start reading at OFFSET. */ 630 SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, 631 NULL, offset, iterpool)); 632 633 /* process all entries (and later continue with the next block) */ 634 for (i = 0; i < entries->nelts; ++i) 635 { 636 svn_fs_x__p2l_entry_t *entry 637 = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); 638 639 /* skip bits we previously checked */ 640 if (i == 0 && entry->offset < offset) 641 continue; 642 643 /* skip zero-sized entries */ 644 if (entry->size == 0) 645 continue; 646 647 /* p2l index must cover all rev / pack file offsets exactly once */ 648 if (entry->offset != offset) 649 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 650 NULL, 651 _("p2l index entry for revision r%ld" 652 " is non-contiguous between offsets " 653 " %s and %s"), 654 start, 655 apr_off_t_toa(scratch_pool, offset), 656 apr_off_t_toa(scratch_pool, 657 entry->offset)); 658 659 /* empty sections must contain NUL bytes only */ 660 if (entry->type == SVN_FS_X__ITEM_TYPE_UNUSED) 661 { 662 /* skip filler entry at the end of the p2l index */ 663 if (entry->offset != max_offset) 664 SVN_ERR(read_all_nul(rev_file->file, entry->size, iterpool)); 665 } 666 else 667 { 668 if (entry->size < STREAM_THRESHOLD) 669 SVN_ERR(expected_buffered_checksum(rev_file->file, entry, 670 iterpool)); 671 else 672 SVN_ERR(expected_streamed_checksum(rev_file->file, entry, 673 iterpool)); 674 } 675 676 /* advance offset */ 677 offset += entry->size; 678 } 679 680 if (cancel_func) 681 SVN_ERR(cancel_func(cancel_baton)); 682 } 683 684 svn_pool_destroy(iterpool); 685 686 return SVN_NO_ERROR; 687} 688 689/* Verify that the revprops of the revisions START to END in FS can be 690 * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 691 * 692 * The values of START and END have already been auto-selected and 693 * verified. 694 */ 695static svn_error_t * 696verify_revprops(svn_fs_t *fs, 697 svn_revnum_t start, 698 svn_revnum_t end, 699 svn_cancel_func_t cancel_func, 700 void *cancel_baton, 701 apr_pool_t *scratch_pool) 702{ 703 svn_revnum_t revision; 704 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 705 706 for (revision = start; revision < end; ++revision) 707 { 708 svn_string_t *date; 709 apr_time_t timetemp; 710 711 svn_pool_clear(iterpool); 712 713 /* Access the svn:date revprop. 714 * This implies parsing all revprops for that revision. */ 715 SVN_ERR(svn_fs_x__revision_prop(&date, fs, revision, 716 SVN_PROP_REVISION_DATE, 717 iterpool, iterpool)); 718 719 /* The time stamp is the only revprop that, if given, needs to 720 * have a valid content. */ 721 if (date) 722 SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool)); 723 724 if (cancel_func) 725 SVN_ERR(cancel_func(cancel_baton)); 726 } 727 728 svn_pool_destroy(iterpool); 729 730 return SVN_NO_ERROR; 731} 732 733/* Verify that on-disk representation has not been tempered with (in a way 734 * that leaves the repository in a corrupted state). This compares log-to- 735 * phys with phys-to-log indexes, verifies the low-level checksums and 736 * checks that all revprops are available. The function signature is 737 * similar to svn_fs_x__verify. 738 * 739 * The values of START and END have already been auto-selected and 740 * verified. 741 */ 742static svn_error_t * 743verify_metadata_consistency(svn_fs_t *fs, 744 svn_revnum_t start, 745 svn_revnum_t end, 746 svn_fs_progress_notify_func_t notify_func, 747 void *notify_baton, 748 svn_cancel_func_t cancel_func, 749 void *cancel_baton, 750 apr_pool_t *scratch_pool) 751{ 752 svn_error_t *err; 753 svn_fs_x__data_t *ffd = fs->fsap_data; 754 svn_revnum_t revision, next_revision; 755 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 756 757 for (revision = start; revision <= end; revision = next_revision) 758 { 759 svn_revnum_t count = svn_fs_x__packed_base_rev(fs, revision); 760 svn_revnum_t pack_start = count; 761 svn_revnum_t pack_end = pack_start + svn_fs_x__pack_size(fs, revision); 762 763 svn_pool_clear(iterpool); 764 765 if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) 766 notify_func(pack_start, notify_baton, iterpool); 767 768 /* Check for external corruption to the indexes. */ 769 err = verify_index_checksums(fs, pack_start, cancel_func, 770 cancel_baton, iterpool); 771 772 /* two-way index check */ 773 if (!err) 774 err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, 775 cancel_func, cancel_baton, iterpool); 776 if (!err) 777 err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, 778 cancel_func, cancel_baton, iterpool); 779 780 /* verify in-index checksums and types vs. actual rev / pack files */ 781 if (!err) 782 err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, 783 cancel_func, cancel_baton, iterpool); 784 785 /* ensure that revprops are available and accessible */ 786 if (!err) 787 err = verify_revprops(fs, pack_start, pack_end, 788 cancel_func, cancel_baton, iterpool); 789 790 /* concurrent packing is one of the reasons why verification may fail. 791 Make sure, we operate on up-to-date information. */ 792 if (err) 793 SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, 794 fs, scratch_pool)); 795 796 /* retry the whole shard if it got packed in the meantime */ 797 if (err && count != svn_fs_x__pack_size(fs, revision)) 798 { 799 svn_error_clear(err); 800 801 /* We could simply assign revision here but the code below is 802 more intuitive to maintainers. */ 803 next_revision = svn_fs_x__packed_base_rev(fs, revision); 804 } 805 else 806 { 807 SVN_ERR(err); 808 next_revision = pack_end; 809 } 810 } 811 812 svn_pool_destroy(iterpool); 813 814 return SVN_NO_ERROR; 815} 816 817svn_error_t * 818svn_fs_x__verify(svn_fs_t *fs, 819 svn_revnum_t start, 820 svn_revnum_t end, 821 svn_fs_progress_notify_func_t notify_func, 822 void *notify_baton, 823 svn_cancel_func_t cancel_func, 824 void *cancel_baton, 825 apr_pool_t *scratch_pool) 826{ 827 svn_fs_x__data_t *ffd = fs->fsap_data; 828 svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */ 829 830 /* Input validation. */ 831 if (! SVN_IS_VALID_REVNUM(start)) 832 start = 0; 833 if (! SVN_IS_VALID_REVNUM(end)) 834 end = youngest; 835 SVN_ERR(svn_fs_x__ensure_revision_exists(start, fs, scratch_pool)); 836 SVN_ERR(svn_fs_x__ensure_revision_exists(end, fs, scratch_pool)); 837 838 /* log/phys index consistency. We need to check them first to make 839 sure we can access the rev / pack files in format7. */ 840 SVN_ERR(verify_metadata_consistency(fs, start, end, 841 notify_func, notify_baton, 842 cancel_func, cancel_baton, 843 scratch_pool)); 844 845 /* rep cache consistency */ 846 SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton, 847 cancel_func, cancel_baton, scratch_pool)); 848 849 return SVN_NO_ERROR; 850} 851