1/* $NetBSD: tools.c,v 1.1.1.3 2010/12/12 15:23:01 adam Exp $ */ 2 3/* tools.c - tools for slap tools */ 4/* OpenLDAP: pkg/ldap/servers/slapd/back-bdb/tools.c,v 1.105.2.19 2010/04/14 23:54:26 quanah Exp */ 5/* This work is part of OpenLDAP Software <http://www.openldap.org/>. 6 * 7 * Copyright 2000-2010 The OpenLDAP Foundation. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted only as authorized by the OpenLDAP 12 * Public License. 13 * 14 * A copy of this license is available in the file LICENSE in the 15 * top-level directory of the distribution or, alternatively, at 16 * <http://www.OpenLDAP.org/license.html>. 17 */ 18 19#include "portable.h" 20 21#include <stdio.h> 22#include <ac/string.h> 23#include <ac/errno.h> 24 25#define AVL_INTERNAL 26#include "back-bdb.h" 27#include "idl.h" 28 29static DBC *cursor = NULL; 30static DBT key, data; 31static EntryHeader eh; 32static ID nid, previd = NOID; 33static char ehbuf[16]; 34 35typedef struct dn_id { 36 ID id; 37 struct berval dn; 38} dn_id; 39 40#define HOLE_SIZE 4096 41static dn_id hbuf[HOLE_SIZE], *holes = hbuf; 42static unsigned nhmax = HOLE_SIZE; 43static unsigned nholes; 44 45static int index_nattrs; 46 47static struct berval *tool_base; 48static int tool_scope; 49static Filter *tool_filter; 50static Entry *tool_next_entry; 51 52#ifdef BDB_TOOL_IDL_CACHING 53#define bdb_tool_idl_cmp BDB_SYMBOL(tool_idl_cmp) 54#define bdb_tool_idl_flush_one BDB_SYMBOL(tool_idl_flush_one) 55#define bdb_tool_idl_flush BDB_SYMBOL(tool_idl_flush) 56 57static int bdb_tool_idl_flush( BackendDB *be ); 58 59#define IDBLOCK 1024 60 61typedef struct bdb_tool_idl_cache_entry { 62 struct bdb_tool_idl_cache_entry *next; 63 ID ids[IDBLOCK]; 64} bdb_tool_idl_cache_entry; 65 66typedef struct bdb_tool_idl_cache { 67 struct berval kstr; 68 bdb_tool_idl_cache_entry *head, *tail; 69 ID first, last; 70 int count; 71} bdb_tool_idl_cache; 72 73static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list; 74#endif /* BDB_TOOL_IDL_CACHING */ 75 76static ID bdb_tool_ix_id; 77static Operation *bdb_tool_ix_op; 78static int *bdb_tool_index_threads, bdb_tool_index_tcount; 79static void *bdb_tool_index_rec; 80static struct bdb_info *bdb_tool_info; 81static ldap_pvt_thread_mutex_t bdb_tool_index_mutex; 82static ldap_pvt_thread_cond_t bdb_tool_index_cond_main; 83static ldap_pvt_thread_cond_t bdb_tool_index_cond_work; 84 85#if DB_VERSION_FULL >= 0x04060000 86#define USE_TRICKLE 1 87#else 88/* Seems to slow things down too much in BDB 4.5 */ 89#undef USE_TRICKLE 90#endif 91 92#ifdef USE_TRICKLE 93static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex; 94static ldap_pvt_thread_cond_t bdb_tool_trickle_cond; 95 96static void * bdb_tool_trickle_task( void *ctx, void *ptr ); 97#endif 98 99static void * bdb_tool_index_task( void *ctx, void *ptr ); 100 101static int 102bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ); 103 104int bdb_tool_entry_open( 105 BackendDB *be, int mode ) 106{ 107 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 108 109 /* initialize key and data thangs */ 110 DBTzero( &key ); 111 DBTzero( &data ); 112 key.flags = DB_DBT_USERMEM; 113 key.data = &nid; 114 key.size = key.ulen = sizeof( nid ); 115 data.flags = DB_DBT_USERMEM; 116 117 if (cursor == NULL) { 118 int rc = bdb->bi_id2entry->bdi_db->cursor( 119 bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor, 120 bdb->bi_db_opflags ); 121 if( rc != 0 ) { 122 return -1; 123 } 124 } 125 126 /* Set up for threaded slapindex */ 127 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) { 128 if ( !bdb_tool_info ) { 129#ifdef USE_TRICKLE 130 ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex ); 131 ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond ); 132 ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv ); 133#endif 134 135 ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex ); 136 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main ); 137 ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work ); 138 if ( bdb->bi_nattrs ) { 139 int i; 140 bdb_tool_index_threads = ch_malloc( slap_tool_thread_max * sizeof( int )); 141 bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec )); 142 bdb_tool_index_tcount = slap_tool_thread_max - 1; 143 for (i=1; i<slap_tool_thread_max; i++) { 144 int *ptr = ch_malloc( sizeof( int )); 145 *ptr = i; 146 ldap_pvt_thread_pool_submit( &connection_pool, 147 bdb_tool_index_task, ptr ); 148 } 149 } 150 bdb_tool_info = bdb; 151 } 152 } 153 154 return 0; 155} 156 157int bdb_tool_entry_close( 158 BackendDB *be ) 159{ 160 if ( bdb_tool_info ) { 161 slapd_shutdown = 1; 162#ifdef USE_TRICKLE 163 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); 164 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond ); 165 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); 166#endif 167 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); 168 bdb_tool_index_tcount = slap_tool_thread_max - 1; 169 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work ); 170 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 171 } 172 173 if( eh.bv.bv_val ) { 174 ch_free( eh.bv.bv_val ); 175 eh.bv.bv_val = NULL; 176 } 177 178 if( cursor ) { 179 cursor->c_close( cursor ); 180 cursor = NULL; 181 } 182 183#ifdef BDB_TOOL_IDL_CACHING 184 bdb_tool_idl_flush( be ); 185#endif 186 187 if( nholes ) { 188 unsigned i; 189 fprintf( stderr, "Error, entries missing!\n"); 190 for (i=0; i<nholes; i++) { 191 fprintf(stderr, " entry %ld: %s\n", 192 holes[i].id, holes[i].dn.bv_val); 193 } 194 return -1; 195 } 196 197 return 0; 198} 199 200ID 201bdb_tool_entry_first_x( 202 BackendDB *be, 203 struct berval *base, 204 int scope, 205 Filter *f ) 206{ 207 tool_base = base; 208 tool_scope = scope; 209 tool_filter = f; 210 211 return bdb_tool_entry_next( be ); 212} 213 214ID bdb_tool_entry_next( 215 BackendDB *be ) 216{ 217 int rc; 218 ID id; 219 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 220 221 assert( be != NULL ); 222 assert( slapMode & SLAP_TOOL_MODE ); 223 assert( bdb != NULL ); 224 225next:; 226 /* Get the header */ 227 data.ulen = data.dlen = sizeof( ehbuf ); 228 data.data = ehbuf; 229 data.flags |= DB_DBT_PARTIAL; 230 rc = cursor->c_get( cursor, &key, &data, DB_NEXT ); 231 232 if( rc ) { 233 /* If we're doing linear indexing and there are more attrs to 234 * index, and we're at the end of the database, start over. 235 */ 236 if ( index_nattrs && rc == DB_NOTFOUND ) { 237 /* optional - do a checkpoint here? */ 238 bdb_attr_info_free( bdb->bi_attrs[0] ); 239 bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs]; 240 index_nattrs--; 241 rc = cursor->c_get( cursor, &key, &data, DB_FIRST ); 242 if ( rc ) { 243 return NOID; 244 } 245 } else { 246 return NOID; 247 } 248 } 249 250 BDB_DISK2ID( key.data, &id ); 251 previd = id; 252 253 if ( tool_filter || tool_base ) { 254 static Operation op = {0}; 255 static Opheader ohdr = {0}; 256 257 op.o_hdr = &ohdr; 258 op.o_bd = be; 259 op.o_tmpmemctx = NULL; 260 op.o_tmpmfuncs = &ch_mfuncs; 261 262 if ( tool_next_entry ) { 263 bdb_entry_release( &op, tool_next_entry, 0 ); 264 tool_next_entry = NULL; 265 } 266 267 rc = bdb_tool_entry_get_int( be, id, &tool_next_entry ); 268 if ( rc == LDAP_NO_SUCH_OBJECT ) { 269 goto next; 270 } 271 272 assert( tool_next_entry != NULL ); 273 274#ifdef BDB_HIER 275 /* TODO: needed until BDB_HIER is handled accordingly 276 * in bdb_tool_entry_get_int() */ 277 if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) ) 278 { 279 bdb_entry_release( &op, tool_next_entry, 0 ); 280 tool_next_entry = NULL; 281 goto next; 282 } 283#endif 284 285 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE ) 286 { 287 bdb_entry_release( &op, tool_next_entry, 0 ); 288 tool_next_entry = NULL; 289 goto next; 290 } 291 } 292 293 return id; 294} 295 296ID bdb_tool_dn2id_get( 297 Backend *be, 298 struct berval *dn 299) 300{ 301 Operation op = {0}; 302 Opheader ohdr = {0}; 303 EntryInfo *ei = NULL; 304 int rc; 305 306 if ( BER_BVISEMPTY(dn) ) 307 return 0; 308 309 op.o_hdr = &ohdr; 310 op.o_bd = be; 311 op.o_tmpmemctx = NULL; 312 op.o_tmpmfuncs = &ch_mfuncs; 313 314 rc = bdb_cache_find_ndn( &op, 0, dn, &ei ); 315 if ( ei ) bdb_cache_entryinfo_unlock( ei ); 316 if ( rc == DB_NOTFOUND ) 317 return NOID; 318 319 return ei->bei_id; 320} 321 322static int 323bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ) 324{ 325 Entry *e = NULL; 326 char *dptr; 327 int rc, eoff; 328 329 assert( be != NULL ); 330 assert( slapMode & SLAP_TOOL_MODE ); 331 332 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) { 333 *ep = tool_next_entry; 334 tool_next_entry = NULL; 335 return LDAP_SUCCESS; 336 } 337 338 if ( id != previd ) { 339 data.ulen = data.dlen = sizeof( ehbuf ); 340 data.data = ehbuf; 341 data.flags |= DB_DBT_PARTIAL; 342 343 BDB_ID2DISK( id, &nid ); 344 rc = cursor->c_get( cursor, &key, &data, DB_SET ); 345 if ( rc ) { 346 rc = LDAP_OTHER; 347 goto done; 348 } 349 } 350 351 /* Get the header */ 352 dptr = eh.bv.bv_val; 353 eh.bv.bv_val = ehbuf; 354 eh.bv.bv_len = data.size; 355 rc = entry_header( &eh ); 356 eoff = eh.data - eh.bv.bv_val; 357 eh.bv.bv_val = dptr; 358 if ( rc ) { 359 rc = LDAP_OTHER; 360 goto done; 361 } 362 363 /* Get the size */ 364 data.flags &= ~DB_DBT_PARTIAL; 365 data.ulen = 0; 366 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT ); 367 if ( rc != DB_BUFFER_SMALL ) { 368 rc = LDAP_OTHER; 369 goto done; 370 } 371 372 /* Allocate a block and retrieve the data */ 373 eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size; 374 eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len ); 375 eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval ); 376 data.data = eh.data; 377 data.ulen = data.size; 378 379 /* Skip past already parsed nattr/nvals */ 380 eh.data += eoff; 381 382 rc = cursor->c_get( cursor, &key, &data, DB_CURRENT ); 383 if ( rc ) { 384 rc = LDAP_OTHER; 385 goto done; 386 } 387 388#ifndef BDB_HIER 389 /* TODO: handle BDB_HIER accordingly */ 390 if ( tool_base != NULL ) { 391 struct berval ndn; 392 entry_decode_dn( &eh, NULL, &ndn ); 393 394 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) { 395 return LDAP_NO_SUCH_OBJECT; 396 } 397 } 398#endif 399 400#ifdef SLAP_ZONE_ALLOC 401 /* FIXME: will add ctx later */ 402 rc = entry_decode( &eh, &e, NULL ); 403#else 404 rc = entry_decode( &eh, &e ); 405#endif 406 407 if( rc == LDAP_SUCCESS ) { 408 e->e_id = id; 409#ifdef BDB_HIER 410 if ( slapMode & SLAP_TOOL_READONLY ) { 411 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 412 EntryInfo *ei = NULL; 413 Operation op = {0}; 414 Opheader ohdr = {0}; 415 416 op.o_hdr = &ohdr; 417 op.o_bd = be; 418 op.o_tmpmemctx = NULL; 419 op.o_tmpmfuncs = &ch_mfuncs; 420 421 rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei ); 422 if ( rc == LDAP_SUCCESS ) { 423 bdb_cache_entryinfo_unlock( ei ); 424 e->e_private = ei; 425 ei->bei_e = e; 426 bdb_fix_dn( e, 0 ); 427 ei->bei_e = NULL; 428 e->e_private = NULL; 429 } 430 } 431#endif 432 } 433done: 434 if ( e != NULL ) { 435 *ep = e; 436 } 437 438 return rc; 439} 440 441Entry* 442bdb_tool_entry_get( BackendDB *be, ID id ) 443{ 444 Entry *e = NULL; 445 446 (void)bdb_tool_entry_get_int( be, id, &e ); 447 return e; 448} 449 450static int bdb_tool_next_id( 451 Operation *op, 452 DB_TXN *tid, 453 Entry *e, 454 struct berval *text, 455 int hole ) 456{ 457 struct berval dn = e->e_name; 458 struct berval ndn = e->e_nname; 459 struct berval pdn, npdn; 460 EntryInfo *ei = NULL, eidummy; 461 int rc; 462 463 if (ndn.bv_len == 0) { 464 e->e_id = 0; 465 return 0; 466 } 467 468 rc = bdb_cache_find_ndn( op, tid, &ndn, &ei ); 469 if ( ei ) bdb_cache_entryinfo_unlock( ei ); 470 if ( rc == DB_NOTFOUND ) { 471 if ( !be_issuffix( op->o_bd, &ndn ) ) { 472 ID eid = e->e_id; 473 dnParent( &dn, &pdn ); 474 dnParent( &ndn, &npdn ); 475 e->e_name = pdn; 476 e->e_nname = npdn; 477 rc = bdb_tool_next_id( op, tid, e, text, 1 ); 478 e->e_name = dn; 479 e->e_nname = ndn; 480 if ( rc ) { 481 return rc; 482 } 483 /* If parent didn't exist, it was created just now 484 * and its ID is now in e->e_id. Make sure the current 485 * entry gets added under the new parent ID. 486 */ 487 if ( eid != e->e_id ) { 488 eidummy.bei_id = e->e_id; 489 ei = &eidummy; 490 } 491 } 492 rc = bdb_next_id( op->o_bd, &e->e_id ); 493 if ( rc ) { 494 snprintf( text->bv_val, text->bv_len, 495 "next_id failed: %s (%d)", 496 db_strerror(rc), rc ); 497 Debug( LDAP_DEBUG_ANY, 498 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 499 return rc; 500 } 501 rc = bdb_dn2id_add( op, tid, ei, e ); 502 if ( rc ) { 503 snprintf( text->bv_val, text->bv_len, 504 "dn2id_add failed: %s (%d)", 505 db_strerror(rc), rc ); 506 Debug( LDAP_DEBUG_ANY, 507 "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); 508 } else if ( hole ) { 509 if ( nholes == nhmax - 1 ) { 510 if ( holes == hbuf ) { 511 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 ); 512 AC_MEMCPY( holes, hbuf, sizeof(hbuf) ); 513 } else { 514 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 ); 515 } 516 nhmax *= 2; 517 } 518 ber_dupbv( &holes[nholes].dn, &ndn ); 519 holes[nholes++].id = e->e_id; 520 } 521 } else if ( !hole ) { 522 unsigned i, j; 523 524 e->e_id = ei->bei_id; 525 526 for ( i=0; i<nholes; i++) { 527 if ( holes[i].id == e->e_id ) { 528 free(holes[i].dn.bv_val); 529 for (j=i;j<nholes;j++) holes[j] = holes[j+1]; 530 holes[j].id = 0; 531 nholes--; 532 break; 533 } else if ( holes[i].id > e->e_id ) { 534 break; 535 } 536 } 537 } 538 return rc; 539} 540 541static int 542bdb_tool_index_add( 543 Operation *op, 544 DB_TXN *txn, 545 Entry *e ) 546{ 547 struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private; 548 549 if ( !bdb->bi_nattrs ) 550 return 0; 551 552 if ( slapMode & SLAP_TOOL_QUICK ) { 553 IndexRec *ir; 554 int i, rc; 555 Attribute *a; 556 557 ir = bdb_tool_index_rec; 558 memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec )); 559 560 for ( a = e->e_attrs; a != NULL; a = a->a_next ) { 561 rc = bdb_index_recset( bdb, a, a->a_desc->ad_type, 562 &a->a_desc->ad_tags, ir ); 563 if ( rc ) 564 return rc; 565 } 566 bdb_tool_ix_id = e->e_id; 567 bdb_tool_ix_op = op; 568 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); 569 /* Wait for all threads to be ready */ 570 while ( bdb_tool_index_tcount ) { 571 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 572 &bdb_tool_index_mutex ); 573 } 574 for ( i=1; i<slap_tool_thread_max; i++ ) 575 bdb_tool_index_threads[i] = LDAP_BUSY; 576 bdb_tool_index_tcount = slap_tool_thread_max - 1; 577 ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work ); 578 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 579 rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 ); 580 if ( rc ) 581 return rc; 582 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); 583 for ( i=1; i<slap_tool_thread_max; i++ ) { 584 if ( bdb_tool_index_threads[i] == LDAP_BUSY ) { 585 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, 586 &bdb_tool_index_mutex ); 587 i--; 588 continue; 589 } 590 if ( bdb_tool_index_threads[i] ) { 591 rc = bdb_tool_index_threads[i]; 592 break; 593 } 594 } 595 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 596 return rc; 597 } else { 598 return bdb_index_entry_add( op, txn, e ); 599 } 600} 601 602ID bdb_tool_entry_put( 603 BackendDB *be, 604 Entry *e, 605 struct berval *text ) 606{ 607 int rc; 608 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 609 DB_TXN *tid = NULL; 610 Operation op = {0}; 611 Opheader ohdr = {0}; 612 613 assert( be != NULL ); 614 assert( slapMode & SLAP_TOOL_MODE ); 615 616 assert( text != NULL ); 617 assert( text->bv_val != NULL ); 618 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 619 620 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put) 621 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 ); 622 623 if (! (slapMode & SLAP_TOOL_QUICK)) { 624 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 625 bdb->bi_db_opflags ); 626 if( rc != 0 ) { 627 snprintf( text->bv_val, text->bv_len, 628 "txn_begin failed: %s (%d)", 629 db_strerror(rc), rc ); 630 Debug( LDAP_DEBUG_ANY, 631 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 632 text->bv_val, 0, 0 ); 633 return NOID; 634 } 635 } 636 637 op.o_hdr = &ohdr; 638 op.o_bd = be; 639 op.o_tmpmemctx = NULL; 640 op.o_tmpmfuncs = &ch_mfuncs; 641 642 /* add dn2id indices */ 643 rc = bdb_tool_next_id( &op, tid, e, text, 0 ); 644 if( rc != 0 ) { 645 goto done; 646 } 647 648#ifdef USE_TRICKLE 649 if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) { 650 ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond ); 651 } 652#endif 653 654 if ( !bdb->bi_linear_index ) 655 rc = bdb_tool_index_add( &op, tid, e ); 656 if( rc != 0 ) { 657 snprintf( text->bv_val, text->bv_len, 658 "index_entry_add failed: %s (%d)", 659 rc == LDAP_OTHER ? "Internal error" : 660 db_strerror(rc), rc ); 661 Debug( LDAP_DEBUG_ANY, 662 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 663 text->bv_val, 0, 0 ); 664 goto done; 665 } 666 667 /* id2entry index */ 668 rc = bdb_id2entry_add( be, tid, e ); 669 if( rc != 0 ) { 670 snprintf( text->bv_val, text->bv_len, 671 "id2entry_add failed: %s (%d)", 672 db_strerror(rc), rc ); 673 Debug( LDAP_DEBUG_ANY, 674 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 675 text->bv_val, 0, 0 ); 676 goto done; 677 } 678 679done: 680 if( rc == 0 ) { 681 if ( !( slapMode & SLAP_TOOL_QUICK )) { 682 rc = TXN_COMMIT( tid, 0 ); 683 if( rc != 0 ) { 684 snprintf( text->bv_val, text->bv_len, 685 "txn_commit failed: %s (%d)", 686 db_strerror(rc), rc ); 687 Debug( LDAP_DEBUG_ANY, 688 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 689 text->bv_val, 0, 0 ); 690 e->e_id = NOID; 691 } 692 } 693 694 } else { 695 if ( !( slapMode & SLAP_TOOL_QUICK )) { 696 TXN_ABORT( tid ); 697 snprintf( text->bv_val, text->bv_len, 698 "txn_aborted! %s (%d)", 699 rc == LDAP_OTHER ? "Internal error" : 700 db_strerror(rc), rc ); 701 Debug( LDAP_DEBUG_ANY, 702 "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", 703 text->bv_val, 0, 0 ); 704 } 705 e->e_id = NOID; 706 } 707 708 return e->e_id; 709} 710 711int bdb_tool_entry_reindex( 712 BackendDB *be, 713 ID id, 714 AttributeDescription **adv ) 715{ 716 struct bdb_info *bi = (struct bdb_info *) be->be_private; 717 int rc; 718 Entry *e; 719 DB_TXN *tid = NULL; 720 Operation op = {0}; 721 Opheader ohdr = {0}; 722 723 assert( tool_base == NULL ); 724 assert( tool_filter == NULL ); 725 726 Debug( LDAP_DEBUG_ARGS, 727 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n", 728 (long) id, 0, 0 ); 729 730 /* No indexes configured, nothing to do. Could return an 731 * error here to shortcut things. 732 */ 733 if (!bi->bi_attrs) { 734 return 0; 735 } 736 737 /* Check for explicit list of attrs to index */ 738 if ( adv ) { 739 int i, j, n; 740 741 if ( bi->bi_attrs[0]->ai_desc != adv[0] ) { 742 /* count */ 743 for ( n = 0; adv[n]; n++ ) ; 744 745 /* insertion sort */ 746 for ( i = 0; i < n; i++ ) { 747 AttributeDescription *ad = adv[i]; 748 for ( j = i-1; j>=0; j--) { 749 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break; 750 adv[j+1] = adv[j]; 751 } 752 adv[j+1] = ad; 753 } 754 } 755 756 for ( i = 0; adv[i]; i++ ) { 757 if ( bi->bi_attrs[i]->ai_desc != adv[i] ) { 758 for ( j = i+1; j < bi->bi_nattrs; j++ ) { 759 if ( bi->bi_attrs[j]->ai_desc == adv[i] ) { 760 AttrInfo *ai = bi->bi_attrs[i]; 761 bi->bi_attrs[i] = bi->bi_attrs[j]; 762 bi->bi_attrs[j] = ai; 763 break; 764 } 765 } 766 if ( j == bi->bi_nattrs ) { 767 Debug( LDAP_DEBUG_ANY, 768 LDAP_XSTRING(bdb_tool_entry_reindex) 769 ": no index configured for %s\n", 770 adv[i]->ad_cname.bv_val, 0, 0 ); 771 return -1; 772 } 773 } 774 } 775 bi->bi_nattrs = i; 776 } 777 778 /* Get the first attribute to index */ 779 if (bi->bi_linear_index && !index_nattrs) { 780 index_nattrs = bi->bi_nattrs - 1; 781 bi->bi_nattrs = 1; 782 } 783 784 e = bdb_tool_entry_get( be, id ); 785 786 if( e == NULL ) { 787 Debug( LDAP_DEBUG_ANY, 788 LDAP_XSTRING(bdb_tool_entry_reindex) 789 ": could not locate id=%ld\n", 790 (long) id, 0, 0 ); 791 return -1; 792 } 793 794 if (! (slapMode & SLAP_TOOL_QUICK)) { 795 rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags ); 796 if( rc != 0 ) { 797 Debug( LDAP_DEBUG_ANY, 798 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": " 799 "txn_begin failed: %s (%d)\n", 800 db_strerror(rc), rc, 0 ); 801 goto done; 802 } 803 } 804 805 /* 806 * just (re)add them for now 807 * assume that some other routine (not yet implemented) 808 * will zap index databases 809 * 810 */ 811 812 Debug( LDAP_DEBUG_TRACE, 813 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n", 814 (long) id, e->e_dn, 0 ); 815 816 op.o_hdr = &ohdr; 817 op.o_bd = be; 818 op.o_tmpmemctx = NULL; 819 op.o_tmpmfuncs = &ch_mfuncs; 820 821 rc = bdb_tool_index_add( &op, tid, e ); 822 823done: 824 if( rc == 0 ) { 825 if (! (slapMode & SLAP_TOOL_QUICK)) { 826 rc = TXN_COMMIT( tid, 0 ); 827 if( rc != 0 ) { 828 Debug( LDAP_DEBUG_ANY, 829 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) 830 ": txn_commit failed: %s (%d)\n", 831 db_strerror(rc), rc, 0 ); 832 e->e_id = NOID; 833 } 834 } 835 836 } else { 837 if (! (slapMode & SLAP_TOOL_QUICK)) { 838 TXN_ABORT( tid ); 839 Debug( LDAP_DEBUG_ANY, 840 "=> " LDAP_XSTRING(bdb_tool_entry_reindex) 841 ": txn_aborted! %s (%d)\n", 842 db_strerror(rc), rc, 0 ); 843 } 844 e->e_id = NOID; 845 } 846 bdb_entry_release( &op, e, 0 ); 847 848 return rc; 849} 850 851ID bdb_tool_entry_modify( 852 BackendDB *be, 853 Entry *e, 854 struct berval *text ) 855{ 856 int rc; 857 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 858 DB_TXN *tid = NULL; 859 Operation op = {0}; 860 Opheader ohdr = {0}; 861 862 assert( be != NULL ); 863 assert( slapMode & SLAP_TOOL_MODE ); 864 865 assert( text != NULL ); 866 assert( text->bv_val != NULL ); 867 assert( text->bv_val[0] == '\0' ); /* overconservative? */ 868 869 assert ( e->e_id != NOID ); 870 871 Debug( LDAP_DEBUG_TRACE, 872 "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n", 873 (long) e->e_id, e->e_dn, 0 ); 874 875 if (! (slapMode & SLAP_TOOL_QUICK)) { 876 if( cursor ) { 877 cursor->c_close( cursor ); 878 cursor = NULL; 879 } 880 rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, 881 bdb->bi_db_opflags ); 882 if( rc != 0 ) { 883 snprintf( text->bv_val, text->bv_len, 884 "txn_begin failed: %s (%d)", 885 db_strerror(rc), rc ); 886 Debug( LDAP_DEBUG_ANY, 887 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", 888 text->bv_val, 0, 0 ); 889 return NOID; 890 } 891 } 892 893 op.o_hdr = &ohdr; 894 op.o_bd = be; 895 op.o_tmpmemctx = NULL; 896 op.o_tmpmfuncs = &ch_mfuncs; 897 898 /* id2entry index */ 899 rc = bdb_id2entry_update( be, tid, e ); 900 if( rc != 0 ) { 901 snprintf( text->bv_val, text->bv_len, 902 "id2entry_add failed: %s (%d)", 903 db_strerror(rc), rc ); 904 Debug( LDAP_DEBUG_ANY, 905 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", 906 text->bv_val, 0, 0 ); 907 goto done; 908 } 909 910done: 911 if( rc == 0 ) { 912 if (! (slapMode & SLAP_TOOL_QUICK)) { 913 rc = TXN_COMMIT( tid, 0 ); 914 if( rc != 0 ) { 915 snprintf( text->bv_val, text->bv_len, 916 "txn_commit failed: %s (%d)", 917 db_strerror(rc), rc ); 918 Debug( LDAP_DEBUG_ANY, 919 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": " 920 "%s\n", text->bv_val, 0, 0 ); 921 e->e_id = NOID; 922 } 923 } 924 925 } else { 926 if (! (slapMode & SLAP_TOOL_QUICK)) { 927 TXN_ABORT( tid ); 928 snprintf( text->bv_val, text->bv_len, 929 "txn_aborted! %s (%d)", 930 db_strerror(rc), rc ); 931 Debug( LDAP_DEBUG_ANY, 932 "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", 933 text->bv_val, 0, 0 ); 934 } 935 e->e_id = NOID; 936 } 937 938 return e->e_id; 939} 940 941#ifdef BDB_TOOL_IDL_CACHING 942static int 943bdb_tool_idl_cmp( const void *v1, const void *v2 ) 944{ 945 const bdb_tool_idl_cache *c1 = v1, *c2 = v2; 946 int rc; 947 948 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc; 949 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len ); 950} 951 952static int 953bdb_tool_idl_flush_one( void *v1, void *arg ) 954{ 955 bdb_tool_idl_cache *ic = v1; 956 DB *db = arg; 957 struct bdb_info *bdb = bdb_tool_info; 958 bdb_tool_idl_cache_entry *ice; 959 DBC *curs; 960 DBT key, data; 961 int i, rc; 962 ID id, nid; 963 964 /* Freshly allocated, ignore it */ 965 if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) { 966 return 0; 967 } 968 969 rc = db->cursor( db, NULL, &curs, 0 ); 970 if ( rc ) 971 return -1; 972 973 DBTzero( &key ); 974 DBTzero( &data ); 975 976 bv2DBT( &ic->kstr, &key ); 977 978 data.size = data.ulen = sizeof( ID ); 979 data.flags = DB_DBT_USERMEM; 980 data.data = &nid; 981 982 rc = curs->c_get( curs, &key, &data, DB_SET ); 983 /* If key already exists and we're writing a range... */ 984 if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) { 985 /* If it's not currently a range, must delete old info */ 986 if ( nid ) { 987 /* Skip lo */ 988 while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 ) 989 curs->c_del( curs, 0 ); 990 991 nid = 0; 992 /* Store range marker */ 993 curs->c_put( curs, &key, &data, DB_KEYFIRST ); 994 } else { 995 996 /* Skip lo */ 997 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP ); 998 999 /* Get hi */ 1000 rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP ); 1001 1002 /* Delete hi */ 1003 curs->c_del( curs, 0 ); 1004 } 1005 BDB_ID2DISK( ic->last, &nid ); 1006 curs->c_put( curs, &key, &data, DB_KEYLAST ); 1007 rc = 0; 1008 } else if ( rc && rc != DB_NOTFOUND ) { 1009 rc = -1; 1010 } else if ( ic->count > BDB_IDL_DB_SIZE ) { 1011 /* range, didn't exist before */ 1012 nid = 0; 1013 rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); 1014 if ( rc == 0 ) { 1015 BDB_ID2DISK( ic->first, &nid ); 1016 rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); 1017 if ( rc == 0 ) { 1018 BDB_ID2DISK( ic->last, &nid ); 1019 rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); 1020 } 1021 } 1022 if ( rc ) { 1023 rc = -1; 1024 } 1025 } else { 1026 int n; 1027 1028 /* Just a normal write */ 1029 rc = 0; 1030 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) { 1031 int end; 1032 if ( ice->next ) { 1033 end = IDBLOCK; 1034 } else { 1035 end = ic->count & (IDBLOCK-1); 1036 if ( !end ) 1037 end = IDBLOCK; 1038 } 1039 for ( i=0; i<end; i++ ) { 1040 if ( !ice->ids[i] ) continue; 1041 BDB_ID2DISK( ice->ids[i], &nid ); 1042 rc = curs->c_put( curs, &key, &data, DB_NODUPDATA ); 1043 if ( rc ) { 1044 if ( rc == DB_KEYEXIST ) { 1045 rc = 0; 1046 continue; 1047 } 1048 rc = -1; 1049 break; 1050 } 1051 } 1052 if ( rc ) { 1053 rc = -1; 1054 break; 1055 } 1056 } 1057 if ( ic->head ) { 1058 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); 1059 ic->tail->next = bdb_tool_idl_free_list; 1060 bdb_tool_idl_free_list = ic->head; 1061 bdb->bi_idl_cache_size -= n; 1062 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); 1063 } 1064 } 1065 if ( ic != db->app_private ) { 1066 ch_free( ic ); 1067 } else { 1068 ic->head = ic->tail = NULL; 1069 } 1070 curs->c_close( curs ); 1071 return rc; 1072} 1073 1074static int 1075bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic ) 1076{ 1077 Avlnode *root = db->app_private; 1078 int rc; 1079 1080 db->app_private = ic; 1081 rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER ); 1082 avl_free( root, NULL ); 1083 db->app_private = NULL; 1084 if ( rc != -1 ) 1085 rc = 0; 1086 return rc; 1087} 1088 1089static int 1090bdb_tool_idl_flush( BackendDB *be ) 1091{ 1092 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 1093 DB *db; 1094 Avlnode *root; 1095 int i, rc = 0; 1096 1097 for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) { 1098 db = bdb->bi_databases[i]->bdi_db; 1099 if ( !db->app_private ) continue; 1100 rc = bdb_tool_idl_flush_db( db, NULL ); 1101 if ( rc ) 1102 break; 1103 } 1104 if ( !rc ) { 1105 bdb->bi_idl_cache_size = 0; 1106 } 1107 return rc; 1108} 1109 1110int bdb_tool_idl_add( 1111 BackendDB *be, 1112 DB *db, 1113 DB_TXN *txn, 1114 DBT *key, 1115 ID id ) 1116{ 1117 struct bdb_info *bdb = (struct bdb_info *) be->be_private; 1118 bdb_tool_idl_cache *ic, itmp; 1119 bdb_tool_idl_cache_entry *ice; 1120 int rc; 1121 1122 if ( !bdb->bi_idl_cache_max_size ) 1123 return bdb_idl_insert_key( be, db, txn, key, id ); 1124 1125 DBT2bv( key, &itmp.kstr ); 1126 1127 ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp ); 1128 1129 /* No entry yet, create one */ 1130 if ( !ic ) { 1131 DBC *curs; 1132 DBT data; 1133 ID nid; 1134 int rc; 1135 1136 ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len ); 1137 ic->kstr.bv_len = itmp.kstr.bv_len; 1138 ic->kstr.bv_val = (char *)(ic+1); 1139 AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len ); 1140 ic->head = ic->tail = NULL; 1141 ic->last = 0; 1142 ic->count = 0; 1143 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp, 1144 avl_dup_error ); 1145 1146 /* load existing key count here */ 1147 rc = db->cursor( db, NULL, &curs, 0 ); 1148 if ( rc ) return rc; 1149 1150 data.ulen = sizeof( ID ); 1151 data.flags = DB_DBT_USERMEM; 1152 data.data = &nid; 1153 rc = curs->c_get( curs, key, &data, DB_SET ); 1154 if ( rc == 0 ) { 1155 if ( nid == 0 ) { 1156 ic->count = BDB_IDL_DB_SIZE+1; 1157 } else { 1158 db_recno_t count; 1159 1160 curs->c_count( curs, &count, 0 ); 1161 ic->count = count; 1162 BDB_DISK2ID( &nid, &ic->first ); 1163 } 1164 } 1165 curs->c_close( curs ); 1166 } 1167 /* are we a range already? */ 1168 if ( ic->count > BDB_IDL_DB_SIZE ) { 1169 ic->last = id; 1170 return 0; 1171 /* Are we at the limit, and converting to a range? */ 1172 } else if ( ic->count == BDB_IDL_DB_SIZE ) { 1173 int n; 1174 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) 1175 /* counting */ ; 1176 if ( n ) { 1177 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); 1178 ic->tail->next = bdb_tool_idl_free_list; 1179 bdb_tool_idl_free_list = ic->head; 1180 bdb->bi_idl_cache_size -= n; 1181 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); 1182 } 1183 ic->head = ic->tail = NULL; 1184 ic->last = id; 1185 ic->count++; 1186 return 0; 1187 } 1188 /* No free block, create that too */ 1189 if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) { 1190 ice = NULL; 1191 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); 1192 if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) { 1193 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); 1194 rc = bdb_tool_idl_flush_db( db, ic ); 1195 if ( rc ) 1196 return rc; 1197 avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp, 1198 avl_dup_error ); 1199 ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); 1200 } 1201 bdb->bi_idl_cache_size++; 1202 if ( bdb_tool_idl_free_list ) { 1203 ice = bdb_tool_idl_free_list; 1204 bdb_tool_idl_free_list = ice->next; 1205 } 1206 ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); 1207 if ( !ice ) { 1208 ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry )); 1209 } 1210 memset( ice, 0, sizeof( *ice )); 1211 if ( !ic->head ) { 1212 ic->head = ice; 1213 } else { 1214 ic->tail->next = ice; 1215 } 1216 ic->tail = ice; 1217 if ( !ic->count ) 1218 ic->first = id; 1219 } 1220 ice = ic->tail; 1221 ice->ids[ ic->count & (IDBLOCK-1) ] = id; 1222 ic->count++; 1223 1224 return 0; 1225} 1226#endif 1227 1228#ifdef USE_TRICKLE 1229static void * 1230bdb_tool_trickle_task( void *ctx, void *ptr ) 1231{ 1232 DB_ENV *env = ptr; 1233 int wrote; 1234 1235 ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); 1236 while ( 1 ) { 1237 ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond, 1238 &bdb_tool_trickle_mutex ); 1239 if ( slapd_shutdown ) 1240 break; 1241 env->memp_trickle( env, 30, &wrote ); 1242 } 1243 ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); 1244 1245 return NULL; 1246} 1247#endif 1248 1249static void * 1250bdb_tool_index_task( void *ctx, void *ptr ) 1251{ 1252 int base = *(int *)ptr; 1253 1254 free( ptr ); 1255 while ( 1 ) { 1256 ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); 1257 bdb_tool_index_tcount--; 1258 if ( !bdb_tool_index_tcount ) 1259 ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main ); 1260 ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work, 1261 &bdb_tool_index_mutex ); 1262 ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); 1263 if ( slapd_shutdown ) 1264 break; 1265 1266 bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op, 1267 bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base ); 1268 } 1269 1270 return NULL; 1271} 1272