1/* tools.c - tools for slap tools */ 2/* $OpenLDAP$ */ 3/* This work is part of OpenLDAP Software <http://www.openldap.org/>. 4 * 5 * Copyright 2000-2011 The OpenLDAP Foundation. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted only as authorized by the OpenLDAP 10 * Public License. 11 * 12 * A copy of this license is available in the file LICENSE in the 13 * top-level directory of the distribution or, alternatively, at 14 * <http://www.OpenLDAP.org/license.html>. 15 */ 16 17#include "portable.h" 18 19#include <stdio.h> 20#include <ac/string.h> 21#include <ac/errno.h> 22 23#define AVL_INTERNAL 24#include "back-bdb.h" 25#include "idl.h" 26 27static DBC *cursor = NULL; 28static DBT key, data; 29static EntryHeader eh; 30static ID nid, previd = NOID; 31static char ehbuf[16]; 32 33typedef struct dn_id { 34 ID id; 35 struct berval dn; 36} dn_id; 37 38#define HOLE_SIZE 4096 39static dn_id hbuf[HOLE_SIZE], *holes = hbuf; 40static unsigned nhmax = HOLE_SIZE; 41static unsigned nholes; 42 43static int index_nattrs; 44 45static struct berval *tool_base; 46static int tool_scope; 47static Filter *tool_filter; 48static Entry *tool_next_entry; 49 50#ifdef BDB_TOOL_IDL_CACHING 51#define bdb_tool_idl_cmp BDB_SYMBOL(tool_idl_cmp) 52#define bdb_tool_idl_flush_one BDB_SYMBOL(tool_idl_flush_one) 53#define bdb_tool_idl_flush BDB_SYMBOL(tool_idl_flush) 54 55static int bdb_tool_idl_flush( BackendDB *be ); 56 57#define IDBLOCK 1024 58 59typedef struct bdb_tool_idl_cache_entry { 60 struct bdb_tool_idl_cache_entry *next; 61 ID ids[IDBLOCK]; 62} bdb_tool_idl_cache_entry; 63 64typedef struct bdb_tool_idl_cache { 65 struct berval kstr; 66 bdb_tool_idl_cache_entry *head, *tail; 67 ID first, last; 68 int count; 69} bdb_tool_idl_cache; 70 71static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list; 72#endif /* BDB_TOOL_IDL_CACHING */ 73 74static ID bdb_tool_ix_id; 75static Operation *bdb_tool_ix_op; 76static int *bdb_tool_index_threads, bdb_tool_index_tcount; 77static void *bdb_tool_index_rec; 78static struct bdb_info *bdb_tool_info; 79static ldap_pvt_thread_mutex_t bdb_tool_index_mutex; 80static ldap_pvt_thread_cond_t bdb_tool_index_cond_main; 81static ldap_pvt_thread_cond_t bdb_tool_index_cond_work; 82 83#if DB_VERSION_FULL >= 0x04060000 84#define USE_TRICKLE 1 85#else 86/* Seems to slow things down too much in BDB 4.5 */ 87#undef USE_TRICKLE 88#endif 89 90#ifdef USE_TRICKLE 91static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex; 92static ldap_pvt_thread_cond_t bdb_tool_trickle_cond; 93static ldap_pvt_thread_cond_t bdb_tool_trickle_cond_end; 94 95static void * bdb_tool_trickle_task( void *ctx, void *ptr ); 96static int bdb_tool_trickle_active; 97#endif 98 99static void * bdb_tool_index_task( void *ctx, void *ptr ); 100 101static int 102bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ); 103 104static int bdb_tool_threads; 105 106int bdb_tool_entry_open( 107 BackendDB *be, int mode ) 108{ 109 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 110 111 /* initialize key and data thangs */ 112 DBTzero( &key ); 113 DBTzero( &data ); 114 key.flags = DB_DBT_USERMEM; 115 key.data = &nid; 116 key.size = key.ulen = sizeof( nid ); 117 data.flags = DB_DBT_USERMEM; 118 119 if (cursor == NULL) { 120 int rc = bdb->bi_id2entry->bdi_db->cursor( 121 bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor, 122 bdb->bi_db_opflags ); 123 if( rc != 0 ) { 124 return -1; 125 } 126 } 127 128 /* Set up for threaded slapindex */ 129 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) { 130 if ( !bdb_tool_info ) { 131#ifdef USE_TRICKLE 132 ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex ); 133 ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond ); 134 ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond_end ); 135 ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv ); 136#endif 137 138 ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex ); 139 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main ); 140 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work ); 141 if ( bdb->bi_nattrs ) { 142 int i; 143 bdb_tool_threads = slap_tool_thread_max - 1; 144 if ( bdb_tool_threads > 1 ) { 145 bdb_tool_index_threads = ch_malloc( bdb_tool_threads * sizeof( int )); 146 bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec )); 147 bdb_tool_index_tcount = bdb_tool_threads - 1; 148 for (i=1; i<bdb_tool_threads; i++) { 149 int *ptr = ch_malloc( sizeof( int )); 150 *ptr = i; 151 ldap_pvt_thread_pool_submit( &connection_pool, 152 bdb_tool_index_task, ptr ); 153 } 154 } 155 } 156 bdb_tool_info = bdb; 157 } 158 } 159 160 return 0; 161} 162 163int bdb_tool_entry_close( 164 BackendDB *be ) 165{ 166 if ( bdb_tool_info ) { 167 slapd_shutdown = 1; 168#ifdef USE_TRICKLE 169 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); 170 171 /* trickle thread may not have started yet */ 172 while ( !bdb_tool_trickle_active ) 173 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end, 174 &bdb_tool_trickle_mutex ); 175 176 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond ); 177 while ( bdb_tool_trickle_active ) 178 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end, 179 &bdb_tool_trickle_mutex ); 180 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); 181#endif 182 if ( bdb_tool_threads > 1 ) { 183 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); 184 185 /* There might still be some threads starting */ 186 while ( bdb_tool_index_tcount > 0 ) { 187 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 188 &bdb_tool_index_mutex ); 189 } 190 191 bdb_tool_index_tcount = bdb_tool_threads - 1; 192 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work ); 193 194 /* Make sure all threads are stopped */ 195 while ( bdb_tool_index_tcount > 0 ) { 196 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 197 &bdb_tool_index_mutex ); 198 } 199 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 200 201 ch_free( bdb_tool_index_threads ); 202 ch_free( bdb_tool_index_rec ); 203 bdb_tool_index_tcount = bdb_tool_threads - 1; 204 } 205 bdb_tool_info = NULL; 206 slapd_shutdown = 0; 207 } 208 209 if( eh.bv.bv_val ) { 210 ch_free( eh.bv.bv_val ); 211 eh.bv.bv_val = NULL; 212 } 213 214 if( cursor ) { 215 cursor->c_close( cursor ); 216 cursor = NULL; 217 } 218 219#ifdef BDB_TOOL_IDL_CACHING 220 bdb_tool_idl_flush( be ); 221#endif 222 223 if( nholes ) { 224 unsigned i; 225 fprintf( stderr, "Error, entries missing!\n"); 226 for (i=0; i<nholes; i++) { 227 fprintf(stderr, " entry %ld: %s\n", 228 holes[i].id, holes[i].dn.bv_val); 229 } 230 return -1; 231 } 232 233 return 0; 234} 235 236ID 237bdb_tool_entry_first_x( 238 BackendDB *be, 239 struct berval *base, 240 int scope, 241 Filter *f ) 242{ 243 tool_base = base; 244 tool_scope = scope; 245 tool_filter = f; 246 247 return bdb_tool_entry_next( be ); 248} 249 250ID bdb_tool_entry_next( 251 BackendDB *be ) 252{ 253 int rc; 254 ID id; 255 struct bdb_info *bdb; 256 257 assert( be != NULL ); 258 assert( slapMode & SLAP_TOOL_MODE ); 259 260 bdb = (struct bdb_info *) be->be_private; 261 assert( bdb != NULL ); 262 263next:; 264 /* Get the header */ 265 data.ulen = data.dlen = sizeof( ehbuf ); 266 data.data = ehbuf; 267 data.flags |= DB_DBT_PARTIAL; 268 rc = cursor->c_get( cursor, &key, &data, DB_NEXT ); 269 270 if( rc ) { 271 /* If we're doing linear indexing and there are more attrs to 272 * index, and we're at the end of the database, start over. 273 */ 274 if ( index_nattrs && rc == DB_NOTFOUND ) { 275 /* optional - do a checkpoint here? */ 276 bdb_attr_info_free( bdb->bi_attrs[0] ); 277 bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs]; 278 index_nattrs--; 279 rc = cursor->c_get( cursor, &key, &data, DB_FIRST ); 280 if ( rc ) { 281 return NOID; 282 } 283 } else { 284 return NOID; 285 } 286 } 287 288 BDB_DISK2ID( key.data, &id ); 289 previd = id; 290 291 if ( tool_filter || tool_base ) { 292 static Operation op = {0}; 293 static Opheader ohdr = {0}; 294 295 op.o_hdr = &ohdr; 296 op.o_bd = be; 297 op.o_tmpmemctx = NULL; 298 op.o_tmpmfuncs = &ch_mfuncs; 299 300 if ( tool_next_entry ) { 301 bdb_entry_release( &op, tool_next_entry, 0 ); 302 tool_next_entry = NULL; 303 } 304 305 rc = bdb_tool_entry_get_int( be, id, &tool_next_entry ); 306 if ( rc == LDAP_NO_SUCH_OBJECT ) { 307 goto next; 308 } 309 310 assert( tool_next_entry != NULL ); 311 312#ifdef BDB_HIER 313 /* TODO: needed until BDB_HIER is handled accordingly 314 * in bdb_tool_entry_get_int() */ 315 if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) ) 316 { 317 bdb_entry_release( &op, tool_next_entry, 0 ); 318 tool_next_entry = NULL; 319 goto next; 320 } 321#endif 322 323 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE ) 324 { 325 bdb_entry_release( &op, tool_next_entry, 0 ); 326 tool_next_entry = NULL; 327 goto next; 328 } 329 } 330 331 return id; 332} 333 334ID bdb_tool_dn2id_get( 335 Backend *be, 336 struct berval *dn 337) 338{ 339 Operation op = {0}; 340 Opheader ohdr = {0}; 341 EntryInfo *ei = NULL; 342 int rc; 343 344 if ( BER_BVISEMPTY(dn) ) 345 return 0; 346 347 op.o_hdr = &ohdr; 348 op.o_bd = be; 349 op.o_tmpmemctx = NULL; 350 op.o_tmpmfuncs = &ch_mfuncs; 351 352 rc = bdb_cache_find_ndn( &op, 0, dn, &ei ); 353 if ( ei ) bdb_cache_entryinfo_unlock( ei ); 354 if ( rc == DB_NOTFOUND ) 355 return NOID; 356 357 return ei->bei_id; 358} 359 360static int 361bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ) 362{ 363 Entry *e = NULL; 364 char *dptr; 365 int rc, eoff; 366 367 assert( be != NULL ); 368 assert( slapMode & SLAP_TOOL_MODE ); 369 370 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) { 371 *ep = tool_next_entry; 372 tool_next_entry = NULL; 373 return LDAP_SUCCESS; 374 } 375 376 if ( id != previd ) { 377 data.ulen = data.dlen = sizeof( ehbuf ); 378 data.data = ehbuf; 379 data.flags |= DB_DBT_PARTIAL; 380 381 BDB_ID2DISK( id, &nid ); 382 rc = cursor->c_get( cursor, &key, &data, DB_SET ); 383 if ( rc ) { 384 rc = LDAP_OTHER; 385 goto done; 386 } 387 } 388 389 /* Get the header */ 390 dptr = eh.bv.bv_val; 391 eh.bv.bv_val = ehbuf; 392 eh.bv.bv_len = data.size; 393 rc = entry_header( &eh ); 394 eoff = eh.data - eh.bv.bv_val; 395 eh.bv.bv_val = dptr; 396 if ( rc ) { 397 rc = LDAP_OTHER; 398 goto done; 399 } 400 401 /* Get the size */ 402 data.flags &= ~DB_DBT_PARTIAL; 403 data.ulen = 0; 404 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT ); 405 if ( rc != DB_BUFFER_SMALL ) { 406 rc = LDAP_OTHER; 407 goto done; 408 } 409 410 /* Allocate a block and retrieve the data */ 411 eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size; 412 eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len ); 413 eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval ); 414 data.data = eh.data; 415 data.ulen = data.size; 416 417 /* Skip past already parsed nattr/nvals */ 418 eh.data += eoff; 419 420 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT ); 421 if ( rc ) { 422 rc = LDAP_OTHER; 423 goto done; 424 } 425 426#ifndef BDB_HIER 427 /* TODO: handle BDB_HIER accordingly */ 428 if ( tool_base != NULL ) { 429 struct berval ndn; 430 entry_decode_dn( &eh, NULL, &ndn ); 431 432 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) { 433 return LDAP_NO_SUCH_OBJECT; 434 } 435 } 436#endif 437 438#ifdef SLAP_ZONE_ALLOC 439 /* FIXME: will add ctx later */ 440 rc = entry_decode( &eh, &e, NULL ); 441#else 442 rc = entry_decode( &eh, &e ); 443#endif 444 445 if( rc == LDAP_SUCCESS ) { 446 e->e_id = id; 447#ifdef BDB_HIER 448 if ( slapMode & SLAP_TOOL_READONLY ) { 449 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 450 EntryInfo *ei = NULL; 451 Operation op = {0}; 452 Opheader ohdr = {0}; 453 454 op.o_hdr = &ohdr; 455 op.o_bd = be; 456 op.o_tmpmemctx = NULL; 457 op.o_tmpmfuncs = &ch_mfuncs; 458 459 rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei ); 460 if ( rc == LDAP_SUCCESS ) { 461 bdb_cache_entryinfo_unlock( ei ); 462 e->e_private = ei; 463 ei->bei_e = e; 464 bdb_fix_dn( e, 0 ); 465 ei->bei_e = NULL; 466 e->e_private = NULL; 467 } 468 } 469#endif 470 } 471done: 472 if ( e != NULL ) { 473 *ep = e; 474 } 475 476 return rc; 477} 478 479Entry* 480bdb_tool_entry_get( BackendDB *be, ID id ) 481{ 482 Entry *e = NULL; 483 484 (void)bdb_tool_entry_get_int( be, id, &e ); 485 return e; 486} 487 488static int bdb_tool_next_id( 489 Operation *op, 490 DB_TXN *tid, 491 Entry *e, 492 struct berval *text, 493 int hole ) 494{ 495 struct berval dn = e->e_name; 496 struct berval ndn = e->e_nname; 497 struct berval pdn, npdn; 498 EntryInfo *ei = NULL, eidummy; 499 int rc; 500 501 if (ndn.bv_len == 0) { 502 e->e_id = 0; 503 return 0; 504 } 505 506 rc = bdb_cache_find_ndn( op, tid, &ndn, &ei ); 507 if ( ei ) bdb_cache_entryinfo_unlock( ei ); 508 if ( rc == DB_NOTFOUND ) { 509 if ( !be_issuffix( op->o_bd, &ndn ) ) { 510 ID eid = e->e_id; 511 dnParent( &dn, &pdn ); 512 dnParent( &ndn, &npdn ); 513 e->e_name = pdn; 514 e->e_nname = npdn; 515 rc = bdb_tool_next_id( op, tid, e, text, 1 ); 516 e->e_name = dn; 517 e->e_nname = ndn; 518 if ( rc ) { 519 return rc; 520 } 521 /* If parent didn't exist, it was created just now 522 * and its ID is now in e->e_id. Make sure the current 523 * entry gets added under the new parent ID. 524 */ 525 if ( eid != e->e_id ) { 526 eidummy.bei_id = e->e_id; 527 ei = &eidummy; 528 } 529 } 530 rc = bdb_next_id( op->o_bd, &e->e_id ); 531 if ( rc ) { 532 snprintf( text->bv_val, text->bv_len, 533 "next_id failed: %s (%d)", 534 db_strerror(rc), rc ); 535 Debug( LDAP_DEBUG_ANY, 536 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 537 return rc; 538 } 539 rc = bdb_dn2id_add( op, tid, ei, e ); 540 if ( rc ) { 541 snprintf( text->bv_val, text->bv_len, 542 "dn2id_add failed: %s (%d)", 543 db_strerror(rc), rc ); 544 Debug( LDAP_DEBUG_ANY, 545 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 546 } else if ( hole ) { 547 if ( nholes == nhmax - 1 ) { 548 if ( holes == hbuf ) { 549 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 ); 550 AC_MEMCPY( holes, hbuf, sizeof(hbuf) ); 551 } else { 552 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 ); 553 } 554 nhmax *= 2; 555 } 556 ber_dupbv( &holes[nholes].dn, &ndn ); 557 holes[nholes++].id = e->e_id; 558 } 559 } else if ( !hole ) { 560 unsigned i, j; 561 562 e->e_id = ei->bei_id; 563 564 for ( i=0; i<nholes; i++) { 565 if ( holes[i].id == e->e_id ) { 566 free(holes[i].dn.bv_val); 567 for (j=i;j<nholes;j++) holes[j] = holes[j+1]; 568 holes[j].id = 0; 569 nholes--; 570 break; 571 } else if ( holes[i].id > e->e_id ) { 572 break; 573 } 574 } 575 } 576 return rc; 577} 578 579static int 580bdb_tool_index_add( 581 Operation *op, 582 DB_TXN *txn, 583 Entry *e ) 584{ 585 struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private; 586 587 if ( !bdb->bi_nattrs ) 588 return 0; 589 590 if ( bdb_tool_threads > 1 ) { 591 IndexRec *ir; 592 int i, rc; 593 Attribute *a; 594 595 ir = bdb_tool_index_rec; 596 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec )); 597 598 for ( a = e->e_attrs; a != NULL; a = a->a_next ) { 599 rc = bdb_index_recset( bdb, a, a->a_desc->ad_type, 600 &a->a_desc->ad_tags, ir ); 601 if ( rc ) 602 return rc; 603 } 604 bdb_tool_ix_id = e->e_id; 605 bdb_tool_ix_op = op; 606 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); 607 /* Wait for all threads to be ready */ 608 while ( bdb_tool_index_tcount > 0 ) { 609 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 610 &bdb_tool_index_mutex ); 611 } 612 for ( i=1; i<bdb_tool_threads; i++ ) 613 bdb_tool_index_threads[i] = LDAP_BUSY; 614 bdb_tool_index_tcount = bdb_tool_threads - 1; 615 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work ); 616 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 617 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 ); 618 if ( rc ) 619 return rc; 620 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); 621 for ( i=1; i<bdb_tool_threads; i++ ) { 622 if ( bdb_tool_index_threads[i] == LDAP_BUSY ) { 623 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 624 &bdb_tool_index_mutex ); 625 i--; 626 continue; 627 } 628 if ( bdb_tool_index_threads[i] ) { 629 rc = bdb_tool_index_threads[i]; 630 break; 631 } 632 } 633 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 634 return rc; 635 } else { 636 return bdb_index_entry_add( op, txn, e ); 637 } 638} 639 640ID bdb_tool_entry_put( 641 BackendDB *be, 642 Entry *e, 643 struct berval *text ) 644{ 645 int rc; 646 struct bdb_info *bdb; 647 DB_TXN *tid = NULL; 648 Operation op = {0}; 649 Opheader ohdr = {0}; 650 651 assert( be != NULL ); 652 assert( slapMode & SLAP_TOOL_MODE ); 653 654 assert( text != NULL ); 655 assert( text->bv_val != NULL ); 656 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 657 658 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put) 659 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 ); 660 661 bdb = (struct bdb_info *) be->be_private; 662 663 if (! (slapMode & SLAP_TOOL_QUICK)) { 664 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 665 bdb->bi_db_opflags ); 666 if( rc != 0 ) { 667 snprintf( text->bv_val, text->bv_len, 668 "txn_begin failed: %s (%d)", 669 db_strerror(rc), rc ); 670 Debug( LDAP_DEBUG_ANY, 671 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 672 text->bv_val, 0, 0 ); 673 return NOID; 674 } 675 } 676 677 op.o_hdr = &ohdr; 678 op.o_bd = be; 679 op.o_tmpmemctx = NULL; 680 op.o_tmpmfuncs = &ch_mfuncs; 681 682 /* add dn2id indices */ 683 rc = bdb_tool_next_id( &op, tid, e, text, 0 ); 684 if( rc != 0 ) { 685 goto done; 686 } 687 688#ifdef USE_TRICKLE 689 if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) { 690 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond ); 691 } 692#endif 693 694 if ( !bdb->bi_linear_index ) 695 rc = bdb_tool_index_add( &op, tid, e ); 696 if( rc != 0 ) { 697 snprintf( text->bv_val, text->bv_len, 698 "index_entry_add failed: %s (%d)", 699 rc == LDAP_OTHER ? "Internal error" : 700 db_strerror(rc), rc ); 701 Debug( LDAP_DEBUG_ANY, 702 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 703 text->bv_val, 0, 0 ); 704 goto done; 705 } 706 707 /* id2entry index */ 708 rc = bdb_id2entry_add( be, tid, e ); 709 if( rc != 0 ) { 710 snprintf( text->bv_val, text->bv_len, 711 "id2entry_add failed: %s (%d)", 712 db_strerror(rc), rc ); 713 Debug( LDAP_DEBUG_ANY, 714 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 715 text->bv_val, 0, 0 ); 716 goto done; 717 } 718 719done: 720 if( rc == 0 ) { 721 if ( !( slapMode & SLAP_TOOL_QUICK )) { 722 rc = TXN_COMMIT( tid, 0 ); 723 if( rc != 0 ) { 724 snprintf( text->bv_val, text->bv_len, 725 "txn_commit failed: %s (%d)", 726 db_strerror(rc), rc ); 727 Debug( LDAP_DEBUG_ANY, 728 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 729 text->bv_val, 0, 0 ); 730 e->e_id = NOID; 731 } 732 } 733 734 } else { 735 if ( !( slapMode & SLAP_TOOL_QUICK )) { 736 TXN_ABORT( tid ); 737 snprintf( text->bv_val, text->bv_len, 738 "txn_aborted! %s (%d)", 739 rc == LDAP_OTHER ? "Internal error" : 740 db_strerror(rc), rc ); 741 Debug( LDAP_DEBUG_ANY, 742 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 743 text->bv_val, 0, 0 ); 744 } 745 e->e_id = NOID; 746 } 747 748 return e->e_id; 749} 750 751int bdb_tool_entry_reindex( 752 BackendDB *be, 753 ID id, 754 AttributeDescription **adv ) 755{ 756 struct bdb_info *bi = (struct bdb_info *) be->be_private; 757 int rc; 758 Entry *e; 759 DB_TXN *tid = NULL; 760 Operation op = {0}; 761 Opheader ohdr = {0}; 762 763 Debug( LDAP_DEBUG_ARGS, 764 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n", 765 (long) id, 0, 0 ); 766 assert( tool_base == NULL ); 767 assert( tool_filter == NULL ); 768 769 /* No indexes configured, nothing to do. Could return an 770 * error here to shortcut things. 771 */ 772 if (!bi->bi_attrs) { 773 return 0; 774 } 775 776 /* Check for explicit list of attrs to index */ 777 if ( adv ) { 778 int i, j, n; 779 780 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) { 781 /* count */ 782 for ( n = 0; adv[n]; n++ ) ; 783 784 /* insertion sort */ 785 for ( i = 0; i < n; i++ ) { 786 AttributeDescription *ad = adv[i]; 787 for ( j = i-1; j>=0; j--) { 788 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break; 789 adv[j+1] = adv[j]; 790 } 791 adv[j+1] = ad; 792 } 793 } 794 795 for ( i = 0; adv[i]; i++ ) { 796 if ( bi->bi_attrs[i]->ai_desc != adv[i] ) { 797 for ( j = i+1; j < bi->bi_nattrs; j++ ) { 798 if ( bi->bi_attrs[j]->ai_desc == adv[i] ) { 799 AttrInfo *ai = bi->bi_attrs[i]; 800 bi->bi_attrs[i] = bi->bi_attrs[j]; 801 bi->bi_attrs[j] = ai; 802 break; 803 } 804 } 805 if ( j == bi->bi_nattrs ) { 806 Debug( LDAP_DEBUG_ANY, 807 LDAP_XSTRING(bdb_tool_entry_reindex) 808 ": no index configured for %s\n", 809 adv[i]->ad_cname.bv_val, 0, 0 ); 810 return -1; 811 } 812 } 813 } 814 bi->bi_nattrs = i; 815 } 816 817 /* Get the first attribute to index */ 818 if (bi->bi_linear_index && !index_nattrs) { 819 index_nattrs = bi->bi_nattrs - 1; 820 bi->bi_nattrs = 1; 821 } 822 823 e = bdb_tool_entry_get( be, id ); 824 825 if( e == NULL ) { 826 Debug( LDAP_DEBUG_ANY, 827 LDAP_XSTRING(bdb_tool_entry_reindex) 828 ": could not locate id=%ld\n", 829 (long) id, 0, 0 ); 830 return -1; 831 } 832 833 if (! (slapMode & SLAP_TOOL_QUICK)) { 834 rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags ); 835 if( rc != 0 ) { 836 Debug( LDAP_DEBUG_ANY, 837 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": " 838 "txn_begin failed: %s (%d)\n", 839 db_strerror(rc), rc, 0 ); 840 goto done; 841 } 842 } 843 844 /* 845 * just (re)add them for now 846 * assume that some other routine (not yet implemented) 847 * will zap index databases 848 * 849 */ 850 851 Debug( LDAP_DEBUG_TRACE, 852 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n", 853 (long) id, e->e_dn, 0 ); 854 855 op.o_hdr = &ohdr; 856 op.o_bd = be; 857 op.o_tmpmemctx = NULL; 858 op.o_tmpmfuncs = &ch_mfuncs; 859 860 rc = bdb_tool_index_add( &op, tid, e ); 861 862done: 863 if( rc == 0 ) { 864 if (! (slapMode & SLAP_TOOL_QUICK)) { 865 rc = TXN_COMMIT( tid, 0 ); 866 if( rc != 0 ) { 867 Debug( LDAP_DEBUG_ANY, 868 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) 869 ": txn_commit failed: %s (%d)\n", 870 db_strerror(rc), rc, 0 ); 871 e->e_id = NOID; 872 } 873 } 874 875 } else { 876 if (! (slapMode & SLAP_TOOL_QUICK)) { 877 TXN_ABORT( tid ); 878 Debug( LDAP_DEBUG_ANY, 879 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) 880 ": txn_aborted! %s (%d)\n", 881 db_strerror(rc), rc, 0 ); 882 } 883 e->e_id = NOID; 884 } 885 bdb_entry_release( &op, e, 0 ); 886 887 return rc; 888} 889 890ID bdb_tool_entry_modify( 891 BackendDB *be, 892 Entry *e, 893 struct berval *text ) 894{ 895 int rc; 896 struct bdb_info *bdb; 897 DB_TXN *tid = NULL; 898 Operation op = {0}; 899 Opheader ohdr = {0}; 900 901 assert( be != NULL ); 902 assert( slapMode & SLAP_TOOL_MODE ); 903 904 assert( text != NULL ); 905 assert( text->bv_val != NULL ); 906 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 907 908 assert ( e->e_id != NOID ); 909 910 Debug( LDAP_DEBUG_TRACE, 911 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n", 912 (long) e->e_id, e->e_dn, 0 ); 913 914 bdb = (struct bdb_info *) be->be_private; 915 916 if (! (slapMode & SLAP_TOOL_QUICK)) { 917 if( cursor ) { 918 cursor->c_close( cursor ); 919 cursor = NULL; 920 } 921 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 922 bdb->bi_db_opflags ); 923 if( rc != 0 ) { 924 snprintf( text->bv_val, text->bv_len, 925 "txn_begin failed: %s (%d)", 926 db_strerror(rc), rc ); 927 Debug( LDAP_DEBUG_ANY, 928 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", 929 text->bv_val, 0, 0 ); 930 return NOID; 931 } 932 } 933 934 op.o_hdr = &ohdr; 935 op.o_bd = be; 936 op.o_tmpmemctx = NULL; 937 op.o_tmpmfuncs = &ch_mfuncs; 938 939 /* id2entry index */ 940 rc = bdb_id2entry_update( be, tid, e ); 941 if( rc != 0 ) { 942 snprintf( text->bv_val, text->bv_len, 943 "id2entry_add failed: %s (%d)", 944 db_strerror(rc), rc ); 945 Debug( LDAP_DEBUG_ANY, 946 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", 947 text->bv_val, 0, 0 ); 948 goto done; 949 } 950 951done: 952 if( rc == 0 ) { 953 if (! (slapMode & SLAP_TOOL_QUICK)) { 954 rc = TXN_COMMIT( tid, 0 ); 955 if( rc != 0 ) { 956 snprintf( text->bv_val, text->bv_len, 957 "txn_commit failed: %s (%d)", 958 db_strerror(rc), rc ); 959 Debug( LDAP_DEBUG_ANY, 960 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": " 961 "%s\n", text->bv_val, 0, 0 ); 962 e->e_id = NOID; 963 } 964 } 965 966 } else { 967 if (! (slapMode & SLAP_TOOL_QUICK)) { 968 TXN_ABORT( tid ); 969 snprintf( text->bv_val, text->bv_len, 970 "txn_aborted! %s (%d)", 971 db_strerror(rc), rc ); 972 Debug( LDAP_DEBUG_ANY, 973 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", 974 text->bv_val, 0, 0 ); 975 } 976 e->e_id = NOID; 977 } 978 979 return e->e_id; 980} 981 982#ifdef BDB_TOOL_IDL_CACHING 983static int 984bdb_tool_idl_cmp( const void *v1, const void *v2 ) 985{ 986 const bdb_tool_idl_cache *c1 = v1, *c2 = v2; 987 int rc; 988 989 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc; 990 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len ); 991} 992 993static int 994bdb_tool_idl_flush_one( void *v1, void *arg ) 995{ 996 bdb_tool_idl_cache *ic = v1; 997 DB *db = arg; 998 struct bdb_info *bdb = bdb_tool_info; 999 bdb_tool_idl_cache_entry *ice; 1000 DBC *curs; 1001 DBT key, data; 1002 int i, rc; 1003 ID id, nid; 1004 1005 /* Freshly allocated, ignore it */ 1006 if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) { 1007 return 0; 1008 } 1009 1010 rc = db->cursor( db, NULL, &curs, 0 ); 1011 if ( rc ) 1012 return -1; 1013 1014 DBTzero( &key ); 1015 DBTzero( &data ); 1016 1017 bv2DBT( &ic->kstr, &key ); 1018 1019 data.size = data.ulen = sizeof( ID ); 1020 data.flags = DB_DBT_USERMEM; 1021 data.data = &nid; 1022 1023 rc = curs->c_get( curs, &key, &data, DB_SET ); 1024 /* If key already exists and we're writing a range... */ 1025 if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) { 1026 /* If it's not currently a range, must delete old info */ 1027 if ( nid ) { 1028 /* Skip lo */ 1029 while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 ) 1030 curs->c_del( curs, 0 ); 1031 1032 nid = 0; 1033 /* Store range marker */ 1034 curs->c_put( curs, &key, &data, DB_KEYFIRST ); 1035 } else { 1036 1037 /* Skip lo */ 1038 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP ); 1039 1040 /* Get hi */ 1041 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP ); 1042 1043 /* Delete hi */ 1044 curs->c_del( curs, 0 ); 1045 } 1046 BDB_ID2DISK( ic->last, &nid ); 1047 curs->c_put( curs, &key, &data, DB_KEYLAST ); 1048 rc = 0; 1049 } else if ( rc && rc != DB_NOTFOUND ) { 1050 rc = -1; 1051 } else if ( ic->count > BDB_IDL_DB_SIZE ) { 1052 /* range, didn't exist before */ 1053 nid = 0; 1054 rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); 1055 if ( rc == 0 ) { 1056 BDB_ID2DISK( ic->first, &nid ); 1057 rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); 1058 if ( rc == 0 ) { 1059 BDB_ID2DISK( ic->last, &nid ); 1060 rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); 1061 } 1062 } 1063 if ( rc ) { 1064 rc = -1; 1065 } 1066 } else { 1067 int n; 1068 1069 /* Just a normal write */ 1070 rc = 0; 1071 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) { 1072 int end; 1073 if ( ice->next ) { 1074 end = IDBLOCK; 1075 } else { 1076 end = ic->count & (IDBLOCK-1); 1077 if ( !end ) 1078 end = IDBLOCK; 1079 } 1080 for ( i=0; i<end; i++ ) { 1081 if ( !ice->ids[i] ) continue; 1082 BDB_ID2DISK( ice->ids[i], &nid ); 1083 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA ); 1084 if ( rc ) { 1085 if ( rc == DB_KEYEXIST ) { 1086 rc = 0; 1087 continue; 1088 } 1089 rc = -1; 1090 break; 1091 } 1092 } 1093 if ( rc ) { 1094 rc = -1; 1095 break; 1096 } 1097 } 1098 if ( ic->head ) { 1099 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); 1100 ic->tail->next = bdb_tool_idl_free_list; 1101 bdb_tool_idl_free_list = ic->head; 1102 bdb->bi_idl_cache_size -= n; 1103 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); 1104 } 1105 } 1106 if ( ic != db->app_private ) { 1107 ch_free( ic ); 1108 } else { 1109 ic->head = ic->tail = NULL; 1110 } 1111 curs->c_close( curs ); 1112 return rc; 1113} 1114 1115static int 1116bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic ) 1117{ 1118 Avlnode *root = db->app_private; 1119 int rc; 1120 1121 db->app_private = ic; 1122 rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER ); 1123 avl_free( root, NULL ); 1124 db->app_private = NULL; 1125 if ( rc != -1 ) 1126 rc = 0; 1127 return rc; 1128} 1129 1130static int 1131bdb_tool_idl_flush( BackendDB *be ) 1132{ 1133 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 1134 DB *db; 1135 Avlnode *root; 1136 int i, rc = 0; 1137 1138 for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) { 1139 db = bdb->bi_databases[i]->bdi_db; 1140 if ( !db->app_private ) continue; 1141 rc = bdb_tool_idl_flush_db( db, NULL ); 1142 if ( rc ) 1143 break; 1144 } 1145 if ( !rc ) { 1146 bdb->bi_idl_cache_size = 0; 1147 } 1148 return rc; 1149} 1150 1151int bdb_tool_idl_add( 1152 BackendDB *be, 1153 DB *db, 1154 DB_TXN *txn, 1155 DBT *key, 1156 ID id ) 1157{ 1158 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 1159 bdb_tool_idl_cache *ic, itmp; 1160 bdb_tool_idl_cache_entry *ice; 1161 int rc; 1162 1163 if ( !bdb->bi_idl_cache_max_size ) 1164 return bdb_idl_insert_key( be, db, txn, key, id ); 1165 1166 DBT2bv( key, &itmp.kstr ); 1167 1168 ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp ); 1169 1170 /* No entry yet, create one */ 1171 if ( !ic ) { 1172 DBC *curs; 1173 DBT data; 1174 ID nid; 1175 int rc; 1176 1177 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len ); 1178 ic->kstr.bv_len = itmp.kstr.bv_len; 1179 ic->kstr.bv_val = (char *)(ic+1); 1180 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len ); 1181 ic->head = ic->tail = NULL; 1182 ic->last = 0; 1183 ic->count = 0; 1184 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp, 1185 avl_dup_error ); 1186 1187 /* load existing key count here */ 1188 rc = db->cursor( db, NULL, &curs, 0 ); 1189 if ( rc ) return rc; 1190 1191 data.ulen = sizeof( ID ); 1192 data.flags = DB_DBT_USERMEM; 1193 data.data = &nid; 1194 rc = curs->c_get( curs, key, &data, DB_SET ); 1195 if ( rc == 0 ) { 1196 if ( nid == 0 ) { 1197 ic->count = BDB_IDL_DB_SIZE+1; 1198 } else { 1199 db_recno_t count; 1200 1201 curs->c_count( curs, &count, 0 ); 1202 ic->count = count; 1203 BDB_DISK2ID( &nid, &ic->first ); 1204 } 1205 } 1206 curs->c_close( curs ); 1207 } 1208 /* are we a range already? */ 1209 if ( ic->count > BDB_IDL_DB_SIZE ) { 1210 ic->last = id; 1211 return 0; 1212 /* Are we at the limit, and converting to a range? */ 1213 } else if ( ic->count == BDB_IDL_DB_SIZE ) { 1214 int n; 1215 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) 1216 /* counting */ ; 1217 if ( n ) { 1218 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); 1219 ic->tail->next = bdb_tool_idl_free_list; 1220 bdb_tool_idl_free_list = ic->head; 1221 bdb->bi_idl_cache_size -= n; 1222 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); 1223 } 1224 ic->head = ic->tail = NULL; 1225 ic->last = id; 1226 ic->count++; 1227 return 0; 1228 } 1229 /* No free block, create that too */ 1230 if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) { 1231 ice = NULL; 1232 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); 1233 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) { 1234 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); 1235 rc = bdb_tool_idl_flush_db( db, ic ); 1236 if ( rc ) 1237 return rc; 1238 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp, 1239 avl_dup_error ); 1240 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); 1241 } 1242 bdb->bi_idl_cache_size++; 1243 if ( bdb_tool_idl_free_list ) { 1244 ice = bdb_tool_idl_free_list; 1245 bdb_tool_idl_free_list = ice->next; 1246 } 1247 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); 1248 if ( !ice ) { 1249 ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry )); 1250 } 1251 memset( ice, 0, sizeof( *ice )); 1252 if ( !ic->head ) { 1253 ic->head = ice; 1254 } else { 1255 ic->tail->next = ice; 1256 } 1257 ic->tail = ice; 1258 if ( !ic->count ) 1259 ic->first = id; 1260 } 1261 ice = ic->tail; 1262 ice->ids[ ic->count & (IDBLOCK-1) ] = id; 1263 ic->count++; 1264 1265 return 0; 1266} 1267#endif 1268 1269#ifdef USE_TRICKLE 1270static void * 1271bdb_tool_trickle_task( void *ctx, void *ptr ) 1272{ 1273 DB_ENV *env = ptr; 1274 int wrote; 1275 1276 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); 1277 bdb_tool_trickle_active = 1; 1278 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end ); 1279 while ( 1 ) { 1280 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond, 1281 &bdb_tool_trickle_mutex ); 1282 if ( slapd_shutdown ) 1283 break; 1284 env->memp_trickle( env, 30, &wrote ); 1285 } 1286 bdb_tool_trickle_active = 0; 1287 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end ); 1288 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); 1289 1290 return NULL; 1291} 1292#endif 1293 1294static void * 1295bdb_tool_index_task( void *ctx, void *ptr ) 1296{ 1297 int base = *(int *)ptr; 1298 1299 free( ptr ); 1300 while ( 1 ) { 1301 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); 1302 bdb_tool_index_tcount--; 1303 if ( !bdb_tool_index_tcount ) 1304 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main ); 1305 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work, 1306 &bdb_tool_index_mutex ); 1307 if ( slapd_shutdown ) { 1308 bdb_tool_index_tcount--; 1309 if ( !bdb_tool_index_tcount ) 1310 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main ); 1311 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 1312 break; 1313 } 1314 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 1315 1316 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op, 1317 bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base ); 1318 } 1319 1320 return NULL; 1321} 1322