verify.c revision 362181
1/* verify.c --- verification of FSX filesystems 2 * 3 * ==================================================================== 4 * Licensed to the Apache Software Foundation (ASF) under one 5 * or more contributor license agreements. See the NOTICE file 6 * distributed with this work for additional information 7 * regarding copyright ownership. The ASF licenses this file 8 * to you under the Apache License, Version 2.0 (the 9 * "License"); you may not use this file except in compliance 10 * with the License. You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, 15 * software distributed under the License is distributed on an 16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 * KIND, either express or implied. See the License for the 18 * specific language governing permissions and limitations 19 * under the License. 20 * ==================================================================== 21 */ 22 23#include "verify.h" 24#include "fs_x.h" 25#include "svn_time.h" 26#include "private/svn_subr_private.h" 27 28#include "cached_data.h" 29#include "rep-cache.h" 30#include "revprops.h" 31#include "util.h" 32#include "index.h" 33 34#include "../libsvn_fs/fs-loader.h" 35 36#include "svn_private_config.h" 37 38 39/** Verifying. **/ 40 41/* Baton type expected by verify_walker(). The purpose is to limit the 42 * number of notifications sent. 43 */ 44typedef struct verify_walker_baton_t 45{ 46 /* number of calls to verify_walker() since the last clean */ 47 int iteration_count; 48 49 /* progress notification callback to invoke periodically (may be NULL) */ 50 svn_fs_progress_notify_func_t notify_func; 51 52 /* baton to use with NOTIFY_FUNC */ 53 void *notify_baton; 54 55 /* remember the last revision for which we called notify_func */ 56 svn_revnum_t last_notified_revision; 57} verify_walker_baton_t; 58 59/* Used by svn_fs_x__verify(). 60 Implements svn_fs_x__walk_rep_reference().walker. */ 61static svn_error_t * 62verify_walker(svn_fs_x__representation_t *rep, 63 void *baton, 64 svn_fs_t *fs, 65 apr_pool_t *scratch_pool) 66{ 67 verify_walker_baton_t *walker_baton = baton; 68 69 /* notify and free resources periodically */ 70 if (walker_baton->iteration_count > 1000) 71 { 72 svn_revnum_t revision = svn_fs_x__get_revnum(rep->id.change_set); 73 if ( walker_baton->notify_func 74 && revision != walker_baton->last_notified_revision) 75 { 76 walker_baton->notify_func(revision, 77 walker_baton->notify_baton, 78 scratch_pool); 79 walker_baton->last_notified_revision = revision; 80 } 81 82 walker_baton->iteration_count = 0; 83 } 84 85 /* access the repo data */ 86 SVN_ERR(svn_fs_x__check_rep(rep, fs, scratch_pool)); 87 88 /* update resource usage counters */ 89 walker_baton->iteration_count++; 90 91 return SVN_NO_ERROR; 92} 93 94/* Verify the rep cache DB's consistency with our rev / pack data. 95 * The function signature is similar to svn_fs_x__verify. 96 * The values of START and END have already been auto-selected and 97 * verified. 98 */ 99static svn_error_t * 100verify_rep_cache(svn_fs_t *fs, 101 svn_revnum_t start, 102 svn_revnum_t end, 103 svn_fs_progress_notify_func_t notify_func, 104 void *notify_baton, 105 svn_cancel_func_t cancel_func, 106 void *cancel_baton, 107 apr_pool_t *scratch_pool) 108{ 109 svn_boolean_t exists; 110 111 /* rep-cache verification. */ 112 SVN_ERR(svn_fs_x__exists_rep_cache(&exists, fs, scratch_pool)); 113 if (exists) 114 { 115 /* provide a baton to allow the reuse of open file handles between 116 iterations (saves 2/3 of OS level file operations). */ 117 verify_walker_baton_t *baton 118 = apr_pcalloc(scratch_pool, sizeof(*baton)); 119 120 baton->last_notified_revision = SVN_INVALID_REVNUM; 121 baton->notify_func = notify_func; 122 baton->notify_baton = notify_baton; 123 124 /* tell the user that we are now ready to do *something* */ 125 if (notify_func) 126 notify_func(SVN_INVALID_REVNUM, notify_baton, scratch_pool); 127 128 /* Do not attempt to walk the rep-cache database if its file does 129 not exist, since doing so would create it --- which may confuse 130 the administrator. Don't take any lock. */ 131 SVN_ERR(svn_fs_x__walk_rep_reference(fs, start, end, 132 verify_walker, baton, 133 cancel_func, cancel_baton, 134 scratch_pool)); 135 } 136 137 return SVN_NO_ERROR; 138} 139 140/* Verify that the MD5 checksum of the data between offsets START and END 141 * in FILE matches the EXPECTED checksum. If there is a mismatch use the 142 * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC 143 * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ 144static svn_error_t * 145verify_index_checksum(svn_fs_x__revision_file_t *file, 146 const char *name, 147 svn_fs_x__index_info_t *index_info, 148 svn_cancel_func_t cancel_func, 149 void *cancel_baton, 150 apr_pool_t *scratch_pool) 151{ 152 unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; 153 apr_off_t size = index_info->end - index_info->start; 154 svn_checksum_t *actual; 155 svn_checksum_ctx_t *context 156 = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); 157 158 /* Calculate the index checksum. */ 159 SVN_ERR(svn_fs_x__rev_file_seek(file, NULL, index_info->start)); 160 while (size > 0) 161 { 162 apr_size_t to_read = size > sizeof(buffer) 163 ? sizeof(buffer) 164 : (apr_size_t)size; 165 SVN_ERR(svn_fs_x__rev_file_read(file, buffer, to_read)); 166 SVN_ERR(svn_checksum_update(context, buffer, to_read)); 167 size -= to_read; 168 169 if (cancel_func) 170 SVN_ERR(cancel_func(cancel_baton)); 171 } 172 173 SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); 174 175 /* Verify that it matches the expected checksum. */ 176 if (!svn_checksum_match(index_info->checksum, actual)) 177 { 178 const char *file_name; 179 180 SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); 181 SVN_ERR(svn_checksum_mismatch_err(index_info->checksum, actual, 182 scratch_pool, 183 _("%s checksum mismatch in file %s"), 184 name, file_name)); 185 } 186 187 return SVN_NO_ERROR; 188} 189 190/* Verify the MD5 checksums of the index data in the rev / pack file 191 * containing revision START in FS. If given, invoke CANCEL_FUNC with 192 * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary 193 * allocations. 194 */ 195static svn_error_t * 196verify_index_checksums(svn_fs_t *fs, 197 svn_revnum_t start, 198 svn_cancel_func_t cancel_func, 199 void *cancel_baton, 200 apr_pool_t *scratch_pool) 201{ 202 svn_fs_x__revision_file_t *rev_file; 203 svn_fs_x__index_info_t l2p_index_info; 204 svn_fs_x__index_info_t p2l_index_info; 205 206 /* Open the rev / pack file and read the footer */ 207 SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); 208 SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file)); 209 SVN_ERR(svn_fs_x__rev_file_p2l_info(&p2l_index_info, rev_file)); 210 211 /* Verify the index contents against the checksum from the footer. */ 212 SVN_ERR(verify_index_checksum(rev_file, "L2P index", &l2p_index_info, 213 cancel_func, cancel_baton, scratch_pool)); 214 SVN_ERR(verify_index_checksum(rev_file, "P2L index", &p2l_index_info, 215 cancel_func, cancel_baton, scratch_pool)); 216 217 /* Done. */ 218 SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 219 220 return SVN_NO_ERROR; 221} 222 223/* Verify that for all log-to-phys index entries for revisions START to 224 * START + COUNT-1 in FS there is a consistent entry in the phys-to-log 225 * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 226 * intervals. Use SCRATCH_POOL for temporary allocations. 227 */ 228static svn_error_t * 229compare_l2p_to_p2l_index(svn_fs_t *fs, 230 svn_revnum_t start, 231 svn_revnum_t count, 232 svn_cancel_func_t cancel_func, 233 void *cancel_baton, 234 apr_pool_t *scratch_pool) 235{ 236 svn_revnum_t i; 237 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 238 apr_array_header_t *max_ids; 239 240 /* common file access structure */ 241 svn_fs_x__revision_file_t *rev_file; 242 SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); 243 244 /* determine the range of items to check for each revision */ 245 SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, start, count, scratch_pool, 246 iterpool)); 247 248 /* check all items in all revisions if the given range */ 249 for (i = 0; i < max_ids->nelts; ++i) 250 { 251 apr_uint64_t k; 252 apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t); 253 svn_revnum_t revision = start + i; 254 255 for (k = 0; k < max_id; ++k) 256 { 257 apr_off_t offset; 258 apr_uint32_t sub_item; 259 svn_fs_x__id_t l2p_item; 260 svn_fs_x__id_t *p2l_item; 261 262 l2p_item.change_set = svn_fs_x__change_set_by_rev(revision); 263 l2p_item.number = k; 264 265 /* get L2P entry. Ignore unused entries. */ 266 SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev_file, 267 &l2p_item, iterpool)); 268 if (offset == -1) 269 continue; 270 271 /* find the corresponding P2L entry */ 272 SVN_ERR(svn_fs_x__p2l_item_lookup(&p2l_item, fs, rev_file, 273 revision, offset, sub_item, 274 iterpool, iterpool)); 275 276 if (p2l_item == NULL) 277 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 278 NULL, 279 _("p2l index entry not found for " 280 "PHYS o%s:s%ld returned by " 281 "l2p index for LOG r%ld:i%ld"), 282 apr_off_t_toa(scratch_pool, offset), 283 (long)sub_item, revision, (long)k); 284 285 if (!svn_fs_x__id_eq(&l2p_item, p2l_item)) 286 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 287 NULL, 288 _("p2l index info LOG r%ld:i%ld" 289 " does not match " 290 "l2p index for LOG r%ld:i%ld"), 291 svn_fs_x__get_revnum(p2l_item->change_set), 292 (long)p2l_item->number, revision, 293 (long)k); 294 295 svn_pool_clear(iterpool); 296 } 297 298 if (cancel_func) 299 SVN_ERR(cancel_func(cancel_baton)); 300 } 301 302 svn_pool_destroy(iterpool); 303 304 SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 305 306 return SVN_NO_ERROR; 307} 308 309/* Verify that for all phys-to-log index entries for revisions START to 310 * START + COUNT-1 in FS there is a consistent entry in the log-to-phys 311 * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 312 * intervals. Use SCRATCH_POOL for temporary allocations. 313 * 314 * Please note that we can only check on pack / rev file granularity and 315 * must only be called for a single rev / pack file. 316 */ 317static svn_error_t * 318compare_p2l_to_l2p_index(svn_fs_t *fs, 319 svn_revnum_t start, 320 svn_revnum_t count, 321 svn_cancel_func_t cancel_func, 322 void *cancel_baton, 323 apr_pool_t *scratch_pool) 324{ 325 svn_fs_x__data_t *ffd = fs->fsap_data; 326 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 327 apr_pool_t *iterpool2 = svn_pool_create(scratch_pool); 328 apr_off_t max_offset; 329 apr_off_t offset = 0; 330 331 /* common file access structure */ 332 svn_fs_x__revision_file_t *rev_file; 333 SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); 334 335 /* get the size of the rev / pack file as covered by the P2L index */ 336 SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, 337 scratch_pool)); 338 339 /* for all offsets in the file, get the P2L index entries and check 340 them against the L2P index */ 341 for (offset = 0; offset < max_offset; ) 342 { 343 apr_array_header_t *entries; 344 svn_fs_x__p2l_entry_t *last_entry; 345 int i; 346 347 svn_pool_clear(iterpool); 348 349 /* get all entries for the current block */ 350 SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, 351 offset, ffd->p2l_page_size, 352 iterpool, iterpool)); 353 if (entries->nelts == 0) 354 return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, 355 NULL, 356 _("p2l does not cover offset %s" 357 " for revision %ld"), 358 apr_off_t_toa(scratch_pool, offset), start); 359 360 /* process all entries (and later continue with the next block) */ 361 last_entry 362 = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_x__p2l_entry_t); 363 offset = last_entry->offset + last_entry->size; 364 365 for (i = 0; i < entries->nelts; ++i) 366 { 367 apr_uint32_t k; 368 svn_fs_x__p2l_entry_t *entry 369 = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); 370 371 /* check all sub-items for consist entries in the L2P index */ 372 for (k = 0; k < entry->item_count; ++k) 373 { 374 apr_off_t l2p_offset; 375 apr_uint32_t sub_item; 376 svn_fs_x__id_t *p2l_item = &entry->items[k]; 377 svn_revnum_t revision 378 = svn_fs_x__get_revnum(p2l_item->change_set); 379 380 svn_pool_clear(iterpool2); 381 SVN_ERR(svn_fs_x__item_offset(&l2p_offset, &sub_item, fs, 382 rev_file, p2l_item, iterpool2)); 383 384 if (sub_item != k || l2p_offset != entry->offset) 385 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 386 NULL, 387 _("l2p index entry PHYS o%s:s%ld " 388 "does not match p2l index value " 389 "LOG r%ld:i%ld for PHYS o%s:s%ld"), 390 apr_off_t_toa(scratch_pool, 391 l2p_offset), 392 (long)sub_item, 393 revision, 394 (long)p2l_item->number, 395 apr_off_t_toa(scratch_pool, 396 entry->offset), 397 (long)k); 398 } 399 } 400 401 if (cancel_func) 402 SVN_ERR(cancel_func(cancel_baton)); 403 } 404 405 svn_pool_destroy(iterpool2); 406 svn_pool_destroy(iterpool); 407 408 SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 409 410 return SVN_NO_ERROR; 411} 412 413/* Items smaller than this can be read at once into a buffer and directly 414 * be checksummed. Larger items require stream processing. 415 * Must be a multiple of 8. */ 416#define STREAM_THRESHOLD 4096 417 418/* Verify that the next SIZE bytes read from FILE are NUL. SIZE must not 419 * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. 420 */ 421static svn_error_t * 422expect_buffer_nul(svn_fs_x__revision_file_t *file, 423 apr_off_t size, 424 apr_pool_t *scratch_pool) 425{ 426 union 427 { 428 unsigned char buffer[STREAM_THRESHOLD]; 429 apr_uint64_t chunks[STREAM_THRESHOLD / sizeof(apr_uint64_t)]; 430 } data; 431 432 apr_size_t i; 433 SVN_ERR_ASSERT(size <= STREAM_THRESHOLD); 434 435 /* read the whole data block; error out on failure */ 436 data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0; 437 SVN_ERR(svn_fs_x__rev_file_read(file, data.buffer, size)); 438 439 /* chunky check */ 440 for (i = 0; i < size / sizeof(apr_uint64_t); ++i) 441 if (data.chunks[i] != 0) 442 break; 443 444 /* byte-wise check upon mismatch or at the end of the block */ 445 for (i *= sizeof(apr_uint64_t); i < size; ++i) 446 if (data.buffer[i] != 0) 447 { 448 const char *file_name; 449 apr_off_t offset; 450 451 SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); 452 SVN_ERR(svn_fs_x__rev_file_offset(&offset, file)); 453 offset -= size - i; 454 455 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 456 _("Empty section in file %s contains " 457 "non-NUL data at offset %s"), 458 file_name, 459 apr_off_t_toa(scratch_pool, offset)); 460 } 461 462 return SVN_NO_ERROR; 463} 464 465/* Verify that the next SIZE bytes read from FILE are NUL. 466 * Use SCRATCH_POOL for temporary allocations. 467 */ 468static svn_error_t * 469read_all_nul(svn_fs_x__revision_file_t *file, 470 apr_off_t size, 471 apr_pool_t *scratch_pool) 472{ 473 for (; size >= STREAM_THRESHOLD; size -= STREAM_THRESHOLD) 474 SVN_ERR(expect_buffer_nul(file, STREAM_THRESHOLD, scratch_pool)); 475 476 if (size) 477 SVN_ERR(expect_buffer_nul(file, size, scratch_pool)); 478 479 return SVN_NO_ERROR; 480} 481 482/* Compare the ACTUAL checksum with the one expected by ENTRY. 483 * Return an error in case of mismatch. Use the name of FILE 484 * in error message. Allocate temporary data in SCRATCH_POOL. 485 */ 486static svn_error_t * 487expected_checksum(svn_fs_x__revision_file_t *file, 488 svn_fs_x__p2l_entry_t *entry, 489 apr_uint32_t actual, 490 apr_pool_t *scratch_pool) 491{ 492 if (actual != entry->fnv1_checksum) 493 { 494 const char *file_name; 495 496 SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); 497 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 498 _("Checksum mismatch in item at offset %s of " 499 "length %s bytes in file %s"), 500 apr_off_t_toa(scratch_pool, entry->offset), 501 apr_off_t_toa(scratch_pool, entry->size), 502 file_name); 503 } 504 505 return SVN_NO_ERROR; 506} 507 508/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read 509 * from FILE will match ENTRY's expected checksum. SIZE must not 510 * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. 511 */ 512static svn_error_t * 513expected_buffered_checksum(svn_fs_x__revision_file_t *file, 514 svn_fs_x__p2l_entry_t *entry, 515 apr_pool_t *scratch_pool) 516{ 517 unsigned char buffer[STREAM_THRESHOLD]; 518 SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD); 519 520 SVN_ERR(svn_fs_x__rev_file_read(file, buffer, (apr_size_t)entry->size)); 521 SVN_ERR(expected_checksum(file, entry, 522 svn__fnv1a_32x4(buffer, (apr_size_t)entry->size), 523 scratch_pool)); 524 525 return SVN_NO_ERROR; 526} 527 528/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read from 529 * FILE will match ENTRY's expected checksum. 530 * Use SCRATCH_POOL for temporary allocations. 531 */ 532static svn_error_t * 533expected_streamed_checksum(svn_fs_x__revision_file_t *file, 534 svn_fs_x__p2l_entry_t *entry, 535 apr_pool_t *scratch_pool) 536{ 537 unsigned char buffer[STREAM_THRESHOLD]; 538 svn_checksum_t *checksum; 539 svn_checksum_ctx_t *context 540 = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, scratch_pool); 541 apr_off_t size = entry->size; 542 543 while (size > 0) 544 { 545 apr_size_t to_read = size > sizeof(buffer) 546 ? sizeof(buffer) 547 : (apr_size_t)size; 548 SVN_ERR(svn_fs_x__rev_file_read(file, buffer, to_read)); 549 SVN_ERR(svn_checksum_update(context, buffer, to_read)); 550 size -= to_read; 551 } 552 553 SVN_ERR(svn_checksum_final(&checksum, context, scratch_pool)); 554 SVN_ERR(expected_checksum(file, entry, 555 ntohl(*(const apr_uint32_t *)checksum->digest), 556 scratch_pool)); 557 558 return SVN_NO_ERROR; 559} 560 561/* Verify that for all phys-to-log index entries for revisions START to 562 * START + COUNT-1 in FS match the actual pack / rev file contents. 563 * If given, invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 564 * Use SCRATCH_POOL for temporary allocations. 565 * 566 * Please note that we can only check on pack / rev file granularity and 567 * must only be called for a single rev / pack file. 568 */ 569static svn_error_t * 570compare_p2l_to_rev(svn_fs_t *fs, 571 svn_revnum_t start, 572 svn_revnum_t count, 573 svn_cancel_func_t cancel_func, 574 void *cancel_baton, 575 apr_pool_t *scratch_pool) 576{ 577 svn_fs_x__data_t *ffd = fs->fsap_data; 578 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 579 apr_off_t max_offset; 580 apr_off_t offset = 0; 581 svn_fs_x__revision_file_t *rev_file; 582 svn_fs_x__index_info_t l2p_index_info; 583 584 /* open the pack / rev file that is covered by the p2l index */ 585 SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); 586 587 /* check file size vs. range covered by index */ 588 SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file)); 589 SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, 590 scratch_pool)); 591 592 if (l2p_index_info.start != max_offset) 593 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, 594 _("File size of %s for revision r%ld does " 595 "not match p2l index size of %s"), 596 apr_off_t_toa(scratch_pool, 597 l2p_index_info.start), 598 start, 599 apr_off_t_toa(scratch_pool, 600 max_offset)); 601 602 SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, 0)); 603 604 /* for all offsets in the file, get the P2L index entries and check 605 them against the L2P index */ 606 for (offset = 0; offset < max_offset; ) 607 { 608 apr_array_header_t *entries; 609 int i; 610 611 svn_pool_clear(iterpool); 612 613 /* get all entries for the current block */ 614 SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, 615 offset, ffd->p2l_page_size, 616 iterpool, iterpool)); 617 618 /* The above might have moved the file pointer. 619 * Ensure we actually start reading at OFFSET. */ 620 SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, offset)); 621 622 /* process all entries (and later continue with the next block) */ 623 for (i = 0; i < entries->nelts; ++i) 624 { 625 svn_fs_x__p2l_entry_t *entry 626 = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); 627 628 /* skip bits we previously checked */ 629 if (i == 0 && entry->offset < offset) 630 continue; 631 632 /* skip zero-sized entries */ 633 if (entry->size == 0) 634 continue; 635 636 /* p2l index must cover all rev / pack file offsets exactly once */ 637 if (entry->offset != offset) 638 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 639 NULL, 640 _("p2l index entry for revision r%ld" 641 " is non-contiguous between offsets " 642 " %s and %s"), 643 start, 644 apr_off_t_toa(scratch_pool, offset), 645 apr_off_t_toa(scratch_pool, 646 entry->offset)); 647 648 /* empty sections must contain NUL bytes only */ 649 if (entry->type == SVN_FS_X__ITEM_TYPE_UNUSED) 650 { 651 /* skip filler entry at the end of the p2l index */ 652 if (entry->offset != max_offset) 653 SVN_ERR(read_all_nul(rev_file, entry->size, iterpool)); 654 } 655 else 656 { 657 if (entry->size < STREAM_THRESHOLD) 658 SVN_ERR(expected_buffered_checksum(rev_file, entry, 659 iterpool)); 660 else 661 SVN_ERR(expected_streamed_checksum(rev_file, entry, 662 iterpool)); 663 } 664 665 /* advance offset */ 666 offset += entry->size; 667 } 668 669 if (cancel_func) 670 SVN_ERR(cancel_func(cancel_baton)); 671 } 672 673 svn_pool_destroy(iterpool); 674 675 return SVN_NO_ERROR; 676} 677 678/* Verify that the revprops of the revisions START to END in FS can be 679 * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 680 * 681 * The values of START and END have already been auto-selected and 682 * verified. 683 */ 684static svn_error_t * 685verify_revprops(svn_fs_t *fs, 686 svn_revnum_t start, 687 svn_revnum_t end, 688 svn_cancel_func_t cancel_func, 689 void *cancel_baton, 690 apr_pool_t *scratch_pool) 691{ 692 svn_revnum_t revision; 693 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 694 695 /* Invalidate the revprop generation once. 696 * Use the cache inside the loop to speed up packed revprop access. */ 697 svn_fs_x__invalidate_revprop_generation(fs); 698 699 for (revision = start; revision < end; ++revision) 700 { 701 svn_string_t *date; 702 apr_time_t timetemp; 703 704 svn_pool_clear(iterpool); 705 706 /* Access the svn:date revprop. 707 * This implies parsing all revprops for that revision. */ 708 SVN_ERR(svn_fs_x__revision_prop(&date, fs, revision, 709 SVN_PROP_REVISION_DATE, FALSE, 710 iterpool, iterpool)); 711 712 /* The time stamp is the only revprop that, if given, needs to 713 * have a valid content. */ 714 if (date) 715 SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool)); 716 717 if (cancel_func) 718 SVN_ERR(cancel_func(cancel_baton)); 719 } 720 721 svn_pool_destroy(iterpool); 722 723 return SVN_NO_ERROR; 724} 725 726/* Verify that on-disk representation has not been tempered with (in a way 727 * that leaves the repository in a corrupted state). This compares log-to- 728 * phys with phys-to-log indexes, verifies the low-level checksums and 729 * checks that all revprops are available. The function signature is 730 * similar to svn_fs_x__verify. 731 * 732 * The values of START and END have already been auto-selected and 733 * verified. 734 */ 735static svn_error_t * 736verify_metadata_consistency(svn_fs_t *fs, 737 svn_revnum_t start, 738 svn_revnum_t end, 739 svn_fs_progress_notify_func_t notify_func, 740 void *notify_baton, 741 svn_cancel_func_t cancel_func, 742 void *cancel_baton, 743 apr_pool_t *scratch_pool) 744{ 745 svn_error_t *err; 746 svn_fs_x__data_t *ffd = fs->fsap_data; 747 svn_revnum_t revision, next_revision; 748 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 749 750 for (revision = start; revision <= end; revision = next_revision) 751 { 752 svn_revnum_t count = svn_fs_x__packed_base_rev(fs, revision); 753 svn_revnum_t pack_start = count; 754 svn_revnum_t pack_end = pack_start + svn_fs_x__pack_size(fs, revision); 755 756 svn_pool_clear(iterpool); 757 758 if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) 759 notify_func(pack_start, notify_baton, iterpool); 760 761 /* Check for external corruption to the indexes. */ 762 err = verify_index_checksums(fs, pack_start, cancel_func, 763 cancel_baton, iterpool); 764 765 /* two-way index check */ 766 if (!err) 767 err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, 768 cancel_func, cancel_baton, iterpool); 769 if (!err) 770 err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, 771 cancel_func, cancel_baton, iterpool); 772 773 /* verify in-index checksums and types vs. actual rev / pack files */ 774 if (!err) 775 err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, 776 cancel_func, cancel_baton, iterpool); 777 778 /* ensure that revprops are available and accessible */ 779 if (!err) 780 err = verify_revprops(fs, pack_start, pack_end, 781 cancel_func, cancel_baton, iterpool); 782 783 /* concurrent packing is one of the reasons why verification may fail. 784 Make sure, we operate on up-to-date information. */ 785 if (err) 786 { 787 svn_error_t *err2 788 = svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, 789 fs, scratch_pool); 790 791 /* Be careful to not leak ERR. */ 792 if (err2) 793 return svn_error_trace(svn_error_compose_create(err, err2)); 794 } 795 796 /* retry the whole shard if it got packed in the meantime */ 797 if (err && count != svn_fs_x__pack_size(fs, revision)) 798 { 799 svn_error_clear(err); 800 801 /* We could simply assign revision here but the code below is 802 more intuitive to maintainers. */ 803 next_revision = svn_fs_x__packed_base_rev(fs, revision); 804 } 805 else 806 { 807 SVN_ERR(err); 808 next_revision = pack_end; 809 } 810 } 811 812 svn_pool_destroy(iterpool); 813 814 return SVN_NO_ERROR; 815} 816 817svn_error_t * 818svn_fs_x__verify(svn_fs_t *fs, 819 svn_revnum_t start, 820 svn_revnum_t end, 821 svn_fs_progress_notify_func_t notify_func, 822 void *notify_baton, 823 svn_cancel_func_t cancel_func, 824 void *cancel_baton, 825 apr_pool_t *scratch_pool) 826{ 827 /* Input validation. */ 828 if (! SVN_IS_VALID_REVNUM(start)) 829 start = 0; 830 if (! SVN_IS_VALID_REVNUM(end)) 831 { 832 SVN_ERR(svn_fs_x__youngest_rev(&end, fs, scratch_pool)); 833 } 834 835 SVN_ERR(svn_fs_x__ensure_revision_exists(start, fs, scratch_pool)); 836 SVN_ERR(svn_fs_x__ensure_revision_exists(end, fs, scratch_pool)); 837 838 /* log/phys index consistency. We need to check them first to make 839 sure we can access the rev / pack files in format7. */ 840 SVN_ERR(verify_metadata_consistency(fs, start, end, 841 notify_func, notify_baton, 842 cancel_func, cancel_baton, 843 scratch_pool)); 844 845 /* rep cache consistency */ 846 SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton, 847 cancel_func, cancel_baton, scratch_pool)); 848 849 return SVN_NO_ERROR; 850} 851