dict_open.c revision 1.3
1/* $NetBSD: dict_open.c,v 1.3 2020/03/18 19:05:21 christos Exp $ */ 2 3/*++ 4/* NAME 5/* dict_open 3 6/* SUMMARY 7/* low-level dictionary interface 8/* SYNOPSIS 9/* #include <dict.h> 10/* 11/* DICT *dict_open(dict_spec, open_flags, dict_flags) 12/* const char *dict_spec; 13/* int open_flags; 14/* int dict_flags; 15/* 16/* DICT *dict_open3(dict_type, dict_name, open_flags, dict_flags) 17/* const char *dict_type; 18/* const char *dict_name; 19/* int open_flags; 20/* int dict_flags; 21/* 22/* int dict_put(dict, key, value) 23/* DICT *dict; 24/* const char *key; 25/* const char *value; 26/* 27/* const char *dict_get(dict, key) 28/* DICT *dict; 29/* const char *key; 30/* 31/* int dict_del(dict, key) 32/* DICT *dict; 33/* const char *key; 34/* 35/* int dict_seq(dict, func, key, value) 36/* DICT *dict; 37/* int func; 38/* const char **key; 39/* const char **value; 40/* 41/* void dict_close(dict) 42/* DICT *dict; 43/* 44/* typedef DICT *(*DICT_OPEN_FN) (const char *, int, int); 45/* 46/* dict_open_register(type, open) 47/* const char *type; 48/* DICT_OPEN_FN open; 49/* 50/* typedef DICT_OPEN_FN (*DICT_OPEN_EXTEND_FN)(const char *type); 51/* 52/* DICT_OPEN_EXTEND_FN dict_open_extend(call_back) 53/* DICT_OPEN_EXTEND_FN call_back; 54/* 55/* ARGV *dict_mapnames() 56/* 57/* typedef ARGV *(*DICT_MAPNAMES_EXTEND_FN)(ARGV *names); 58/* 59/* DICT_MAPNAMES_EXTEND_FN dict_mapnames_extend(call_back) 60/* DICT_MAPNAMES_EXTEND_FN call_back; 61/* 62/* int dict_isjmp(dict) 63/* DICT *dict; 64/* 65/* int dict_setjmp(dict) 66/* DICT *dict; 67/* 68/* int dict_longjmp(dict, val) 69/* DICT *dict; 70/* int val; 71/* 72/* void dict_type_override(dict, type) 73/* DICT *dict; 74/* const char *type; 75/* DESCRIPTION 76/* This module implements a low-level interface to multiple 77/* physical dictionary types. 78/* 79/* dict_open() takes a type:name pair that specifies a dictionary type 80/* and dictionary name, opens the dictionary, and returns a dictionary 81/* handle. The \fIopen_flags\fR arguments are as in open(2). The 82/* \fIdict_flags\fR are the bit-wise OR of zero or more of the following: 83/* .IP DICT_FLAG_DUP_WARN 84/* Warn about duplicate keys, if the underlying database does not 85/* support duplicate keys. The default is to terminate with a fatal 86/* error. 87/* .IP DICT_FLAG_DUP_IGNORE 88/* Ignore duplicate keys if the underlying database does not 89/* support duplicate keys. The default is to terminate with a fatal 90/* error. 91/* .IP DICT_FLAG_DUP_REPLACE 92/* Replace duplicate keys if the underlying database supports such 93/* an operation. The default is to terminate with a fatal error. 94/* .IP DICT_FLAG_TRY0NULL 95/* With maps where this is appropriate, append no null byte to 96/* keys and values. 97/* When neither DICT_FLAG_TRY0NULL nor DICT_FLAG_TRY1NULL are 98/* specified, the software guesses what format to use for reading; 99/* and in the absence of definite information, a system-dependent 100/* default is chosen for writing. 101/* .IP DICT_FLAG_TRY1NULL 102/* With maps where this is appropriate, append one null byte to 103/* keys and values. 104/* When neither DICT_FLAG_TRY0NULL nor DICT_FLAG_TRY1NULL are 105/* specified, the software guesses what format to use for reading; 106/* and in the absence of definite information, a system-dependent 107/* default is chosen for writing. 108/* .IP DICT_FLAG_LOCK 109/* With maps where this is appropriate, acquire an exclusive lock 110/* before writing, and acquire a shared lock before reading. 111/* Release the lock when the operation completes. 112/* .IP DICT_FLAG_OPEN_LOCK 113/* The behavior of this flag depends on whether a database 114/* sets the DICT_FLAG_MULTI_WRITER flag to indicate that it 115/* is multi-writer safe. 116/* 117/* With databases that are not multi-writer safe, dict_open() 118/* acquires a persistent exclusive lock, or it terminates with 119/* a fatal run-time error. 120/* 121/* With databases that are multi-writer safe, dict_open() 122/* downgrades the DICT_FLAG_OPEN_LOCK flag (persistent lock) 123/* to DICT_FLAG_LOCK (temporary lock). 124/* .IP DICT_FLAG_FOLD_FIX 125/* With databases whose lookup fields are fixed-case strings, 126/* fold the search string to lower case before accessing the 127/* database. This includes hash:, cdb:, dbm:. nis:, ldap:, 128/* *sql. WARNING: case folding is supported only for ASCII or 129/* valid UTF-8. 130/* .IP DICT_FLAG_FOLD_MUL 131/* With databases where one lookup field can match both upper 132/* and lower case, fold the search key to lower case before 133/* accessing the database. This includes regexp: and pcre:. 134/* WARNING: case folding is supported only for ASCII or valid 135/* UTF-8. 136/* .IP DICT_FLAG_FOLD_ANY 137/* Short-hand for (DICT_FLAG_FOLD_FIX | DICT_FLAG_FOLD_MUL). 138/* .IP DICT_FLAG_SYNC_UPDATE 139/* With file-based maps, flush I/O buffers to file after each update. 140/* Thus feature is not supported with some file-based dictionaries. 141/* .IP DICT_FLAG_NO_REGSUB 142/* Disallow regular expression substitution from the lookup string 143/* into the lookup result, to block data injection attacks. 144/* .IP DICT_FLAG_NO_PROXY 145/* Disallow access through the unprivileged \fBproxymap\fR 146/* service, to block privilege escalation attacks. 147/* .IP DICT_FLAG_NO_UNAUTH 148/* Disallow lookup mechanisms that lack any form of authentication, 149/* to block privilege escalation attacks (example: tcp_table; 150/* even NIS can be secured to some extent by requiring that 151/* the server binds to a privileged port). 152/* .IP DICT_FLAG_PARANOID 153/* A combination of all the paranoia flags: DICT_FLAG_NO_REGSUB, 154/* DICT_FLAG_NO_PROXY and DICT_FLAG_NO_UNAUTH. 155/* .IP DICT_FLAG_BULK_UPDATE 156/* Enable preliminary code for bulk-mode database updates. 157/* The caller must create an exception handler with dict_jmp_alloc() 158/* and must trap exceptions from the database client with dict_setjmp(). 159/* .IP DICT_FLAG_DEBUG 160/* Enable additional logging. 161/* .IP DICT_FLAG_UTF8_REQUEST 162/* With util_utf8_enable != 0, require that lookup/update/delete 163/* keys and values are valid UTF-8. Skip a lookup/update/delete 164/* request with a non-UTF-8 key, skip an update request with 165/* a non-UTF-8 value, and fail a lookup request with a non-UTF-8 166/* value. 167/* .IP DICT_FLAG_SRC_RHS_IS_FILE 168/* With dictionaries that are created from source text, each 169/* value in the source of a dictionary specifies a list of 170/* file names separated by comma and/or whitespace. The file 171/* contents are concatenated with a newline inserted between 172/* files, and the base64-encoded result is stored under the 173/* key. 174/* .sp 175/* NOTE 1: it is up to the application to decode lookup results 176/* with dict_file_lookup() or equivalent (this requires that 177/* the dictionary is opened with DICT_FLAG_SRC_RHS_IS_FILE). 178/* Decoding is not built into the normal dictionary lookup 179/* method, because that would complicate dictionary nesting, 180/* pipelining, and proxying. 181/* .sp 182/* NOTE 2: it is up to the application to convert file names 183/* into base64-encoded file content before calling the dictionary 184/* update method (see dict_file(3) for support). Automatic 185/* file content encoding is available only when a dictionary 186/* is created from source text. 187/* .PP 188/* Specify DICT_FLAG_NONE for no special processing. 189/* 190/* The dictionary types are as follows: 191/* .IP environ 192/* The process environment array. The \fIdict_name\fR argument is ignored. 193/* .IP dbm 194/* DBM file. 195/* .IP hash 196/* Berkeley DB file in hash format. 197/* .IP btree 198/* Berkeley DB file in btree format. 199/* .IP nis 200/* NIS map. Only read access is supported. 201/* .IP nisplus 202/* NIS+ map. Only read access is supported. 203/* .IP netinfo 204/* NetInfo table. Only read access is supported. 205/* .IP ldap 206/* LDAP ("light-weight" directory access protocol) database access. 207/* .IP pcre 208/* PERL-compatible regular expressions. 209/* .IP regexp 210/* POSIX-compatible regular expressions. 211/* .IP texthash 212/* Flat text in postmap(1) input format. 213/* .PP 214/* dict_open3() takes separate arguments for dictionary type and 215/* name, but otherwise performs the same functions as dict_open(). 216/* 217/* The dict_get(), dict_put(), dict_del(), and dict_seq() 218/* macros evaluate their first argument multiple times. 219/* These names should have been in uppercase. 220/* 221/* dict_get() retrieves the value stored in the named dictionary 222/* under the given key. A null pointer means the value was not found. 223/* As with dict_lookup(), the result is owned by the lookup table 224/* implementation. Make a copy if the result is to be modified, 225/* or if the result is to survive multiple table lookups. 226/* 227/* dict_put() stores the specified key and value into the named 228/* dictionary. A zero (DICT_STAT_SUCCESS) result means the 229/* update was made. 230/* 231/* dict_del() removes a dictionary entry, and returns 232/* DICT_STAT_SUCCESS in case of success. 233/* 234/* dict_seq() iterates over all members in the named dictionary. 235/* func is define DICT_SEQ_FUN_FIRST (select first member) or 236/* DICT_SEQ_FUN_NEXT (select next member). A zero (DICT_STAT_SUCCESS) 237/* result means that an entry was found. 238/* 239/* dict_close() closes the specified dictionary and cleans up the 240/* associated data structures. 241/* 242/* dict_open_register() adds support for a new dictionary type. 243/* 244/* dict_open_extend() registers a call-back function that looks 245/* up the dictionary open() function for a type that is not 246/* registered, or null in case of error. The result value is 247/* the last previously-registered call-back or null. 248/* 249/* dict_mapnames() returns a sorted list with the names of all available 250/* dictionary types. 251/* 252/* dict_mapnames_extend() registers a call-back function that 253/* enumerates additional dictionary type names. The result 254/* will be sorted by dict_mapnames(). The result value 255/* is the last previously-registered call-back or null. 256/* 257/* dict_setjmp() saves processing context and makes that context 258/* available for use with dict_longjmp(). Normally, dict_setjmp() 259/* returns zero. A non-zero result means that dict_setjmp() 260/* returned through a dict_longjmp() call; the result is the 261/* \fIval\fR argument given to dict_longjmp(). dict_isjmp() 262/* returns non-zero when dict_setjmp() and dict_longjmp() 263/* are enabled for a given dictionary. 264/* 265/* NB: non-local jumps such as dict_longjmp() are not safe for 266/* jumping out of any routine that manipulates DICT data. 267/* longjmp() like calls are best avoided in signal handlers. 268/* 269/* dict_type_override() changes the symbolic dictionary type. 270/* This is used by dictionaries whose internals are based on 271/* some other dictionary type. 272/* DIAGNOSTICS 273/* Fatal error: open error, unsupported dictionary type, attempt to 274/* update non-writable dictionary. 275/* 276/* The lookup routine returns non-null when the request is 277/* satisfied. The update, delete and sequence routines return 278/* zero (DICT_STAT_SUCCESS) when the request is satisfied. 279/* The dict->errno value is non-zero only when the last operation 280/* was not satisfied due to a dictionary access error. This 281/* can have the following values: 282/* .IP DICT_ERR_NONE(zero) 283/* There was no dictionary access error. For example, the 284/* request was satisfied, the requested information did not 285/* exist in the dictionary, or the information already existed 286/* when it should not exist (collision). 287/* .IP DICT_ERR_RETRY(<0) 288/* The dictionary was temporarily unavailable. This can happen 289/* with network-based services. 290/* .IP DICT_ERR_CONFIG(<0) 291/* The dictionary was unavailable due to a configuration error. 292/* .PP 293/* Generally, a program is expected to test the function result 294/* value for "success" first. If the operation was not successful, 295/* a program is expected to test for a non-zero dict->error 296/* status to distinguish between a data notfound/collision 297/* condition or a dictionary access error. 298/* LICENSE 299/* .ad 300/* .fi 301/* The Secure Mailer license must be distributed with this software. 302/* AUTHOR(S) 303/* Wietse Venema 304/* IBM T.J. Watson Research 305/* P.O. Box 704 306/* Yorktown Heights, NY 10598, USA 307/* 308/* Wietse Venema 309/* Google, Inc. 310/* 111 8th Avenue 311/* New York, NY 10011, USA 312/*--*/ 313 314/* System library. */ 315 316#include <sys_defs.h> 317#include <string.h> 318#include <stdlib.h> 319 320/* Utility library. */ 321 322#include <argv.h> 323#include <mymalloc.h> 324#include <msg.h> 325#include <dict.h> 326#include <dict_cdb.h> 327#include <dict_env.h> 328#include <dict_unix.h> 329#include <dict_tcp.h> 330#include <dict_sdbm.h> 331#include <dict_dbm.h> 332#include <dict_db.h> 333#include <dict_lmdb.h> 334#include <dict_nis.h> 335#include <dict_nisplus.h> 336#include <dict_ni.h> 337#include <dict_pcre.h> 338#include <dict_regexp.h> 339#include <dict_static.h> 340#include <dict_cidr.h> 341#include <dict_ht.h> 342#include <dict_thash.h> 343#include <dict_sockmap.h> 344#include <dict_fail.h> 345#include <dict_pipe.h> 346#include <dict_random.h> 347#include <dict_union.h> 348#include <dict_inline.h> 349#include <stringops.h> 350#include <split_at.h> 351#include <htable.h> 352#include <myflock.h> 353 354 /* 355 * lookup table for available map types. 356 */ 357typedef struct { 358 char *type; 359 DICT_OPEN_FN open; 360} DICT_OPEN_INFO; 361 362static const DICT_OPEN_INFO dict_open_info[] = { 363 DICT_TYPE_ENVIRON, dict_env_open, 364 DICT_TYPE_HT, dict_ht_open, 365 DICT_TYPE_UNIX, dict_unix_open, 366 DICT_TYPE_TCP, dict_tcp_open, 367#ifdef HAS_DBM 368 DICT_TYPE_DBM, dict_dbm_open, 369#endif 370#ifdef HAS_DB 371 DICT_TYPE_HASH, dict_hash_open, 372 DICT_TYPE_BTREE, dict_btree_open, 373#endif 374#ifdef HAS_NIS 375 DICT_TYPE_NIS, dict_nis_open, 376#endif 377#ifdef HAS_NISPLUS 378 DICT_TYPE_NISPLUS, dict_nisplus_open, 379#endif 380#ifdef HAS_NETINFO 381 DICT_TYPE_NETINFO, dict_ni_open, 382#endif 383#ifdef HAS_POSIX_REGEXP 384 DICT_TYPE_REGEXP, dict_regexp_open, 385#endif 386 DICT_TYPE_STATIC, dict_static_open, 387 DICT_TYPE_CIDR, dict_cidr_open, 388 DICT_TYPE_THASH, dict_thash_open, 389 DICT_TYPE_SOCKMAP, dict_sockmap_open, 390 DICT_TYPE_FAIL, dict_fail_open, 391 DICT_TYPE_PIPE, dict_pipe_open, 392 DICT_TYPE_RANDOM, dict_random_open, 393 DICT_TYPE_UNION, dict_union_open, 394 DICT_TYPE_INLINE, dict_inline_open, 395#ifndef USE_DYNAMIC_MAPS 396#ifdef HAS_PCRE 397 DICT_TYPE_PCRE, dict_pcre_open, 398#endif 399#ifdef HAS_CDB 400 DICT_TYPE_CDB, dict_cdb_open, 401#endif 402#ifdef HAS_SDBM 403 DICT_TYPE_SDBM, dict_sdbm_open, 404#endif 405#ifdef HAS_LMDB 406 DICT_TYPE_LMDB, dict_lmdb_open, 407#endif 408#endif /* !USE_DYNAMIC_MAPS */ 409 0, 410}; 411 412static HTABLE *dict_open_hash; 413 414 /* 415 * Extension hooks. 416 */ 417static DICT_OPEN_EXTEND_FN dict_open_extend_hook; 418static DICT_MAPNAMES_EXTEND_FN dict_mapnames_extend_hook; 419 420 /* 421 * Workaround. 422 */ 423DEFINE_DICT_LMDB_MAP_SIZE; 424DEFINE_DICT_DB_CACHE_SIZE; 425 426/* dict_open_init - one-off initialization */ 427 428static void dict_open_init(void) 429{ 430 const char *myname = "dict_open_init"; 431 const DICT_OPEN_INFO *dp; 432 433 if (dict_open_hash != 0) 434 msg_panic("%s: multiple initialization", myname); 435 dict_open_hash = htable_create(10); 436 437 for (dp = dict_open_info; dp->type; dp++) 438 htable_enter(dict_open_hash, dp->type, (void *) dp); 439} 440 441/* dict_open - open dictionary */ 442 443DICT *dict_open(const char *dict_spec, int open_flags, int dict_flags) 444{ 445 char *saved_dict_spec = mystrdup(dict_spec); 446 char *dict_name; 447 DICT *dict; 448 449 if ((dict_name = split_at(saved_dict_spec, ':')) == 0) 450 msg_fatal("open dictionary: expecting \"type:name\" form instead of \"%s\"", 451 dict_spec); 452 453 dict = dict_open3(saved_dict_spec, dict_name, open_flags, dict_flags); 454 myfree(saved_dict_spec); 455 return (dict); 456} 457 458 459/* dict_open3 - open dictionary */ 460 461DICT *dict_open3(const char *dict_type, const char *dict_name, 462 int open_flags, int dict_flags) 463{ 464 const char *myname = "dict_open"; 465 DICT_OPEN_INFO *dp; 466 DICT_OPEN_FN open_fn; 467 DICT *dict; 468 469 if (*dict_type == 0 || *dict_name == 0) 470 msg_fatal("open dictionary: expecting \"type:name\" form instead of \"%s:%s\"", 471 dict_type, dict_name); 472 if (dict_open_hash == 0) 473 dict_open_init(); 474 if ((dp = (DICT_OPEN_INFO *) htable_find(dict_open_hash, dict_type)) == 0) { 475 if (dict_open_extend_hook != 0 476 && (open_fn = dict_open_extend_hook(dict_type)) != 0) { 477 dict_open_register(dict_type, open_fn); 478 dp = (DICT_OPEN_INFO *) htable_find(dict_open_hash, dict_type); 479 } 480 if (dp == 0) 481 return (dict_surrogate(dict_type, dict_name, open_flags, dict_flags, 482 "unsupported dictionary type: %s", dict_type)); 483 } 484 if ((dict = dp->open(dict_name, open_flags, dict_flags)) == 0) 485 return (dict_surrogate(dict_type, dict_name, open_flags, dict_flags, 486 "cannot open %s:%s: %m", dict_type, dict_name)); 487 if (msg_verbose) 488 msg_info("%s: %s:%s", myname, dict_type, dict_name); 489 /* XXX The choice between wait-for-lock or no-wait is hard-coded. */ 490 if (dict->flags & DICT_FLAG_OPEN_LOCK) { 491 if (dict->flags & DICT_FLAG_LOCK) 492 msg_panic("%s: attempt to open %s:%s with both \"open\" lock and \"access\" lock", 493 myname, dict_type, dict_name); 494 /* Multi-writer safe map: downgrade persistent lock to temporary. */ 495 if (dict->flags & DICT_FLAG_MULTI_WRITER) { 496 dict->flags &= ~DICT_FLAG_OPEN_LOCK; 497 dict->flags |= DICT_FLAG_LOCK; 498 } 499 /* Multi-writer unsafe map: acquire exclusive lock or bust. */ 500 else if (dict->lock(dict, MYFLOCK_OP_EXCLUSIVE | MYFLOCK_OP_NOWAIT) < 0) 501 msg_fatal("%s:%s: unable to get exclusive lock: %m", 502 dict_type, dict_name); 503 } 504 /* Last step: insert proxy for UTF-8 syntax checks and casefolding. */ 505 if ((dict->flags & DICT_FLAG_UTF8_ACTIVE) == 0 506 && DICT_NEED_UTF8_ACTIVATION(util_utf8_enable, dict_flags)) 507 dict = dict_utf8_activate(dict); 508 return (dict); 509} 510 511/* dict_open_register - register dictionary type */ 512 513void dict_open_register(const char *type, DICT_OPEN_FN open) 514{ 515 const char *myname = "dict_open_register"; 516 DICT_OPEN_INFO *dp; 517 HTABLE_INFO *ht; 518 519 if (dict_open_hash == 0) 520 dict_open_init(); 521 if (htable_find(dict_open_hash, type)) 522 msg_panic("%s: dictionary type exists: %s", myname, type); 523 dp = (DICT_OPEN_INFO *) mymalloc(sizeof(*dp)); 524 dp->open = open; 525 ht = htable_enter(dict_open_hash, type, (void *) dp); 526 dp->type = ht->key; 527} 528 529/* dict_open_extend - register alternate dictionary search routine */ 530 531DICT_OPEN_EXTEND_FN dict_open_extend(DICT_OPEN_EXTEND_FN new_cb) 532{ 533 DICT_OPEN_EXTEND_FN old_cb; 534 535 old_cb = dict_open_extend_hook; 536 dict_open_extend_hook = new_cb; 537 return (old_cb); 538} 539 540/* dict_sort_alpha_cpp - qsort() callback */ 541 542static int dict_sort_alpha_cpp(const void *a, const void *b) 543{ 544 return (strcmp(((char **) a)[0], ((char **) b)[0])); 545} 546 547/* dict_mapnames - return an ARGV of available map_names */ 548 549ARGV *dict_mapnames() 550{ 551 HTABLE_INFO **ht_info; 552 HTABLE_INFO **ht; 553 DICT_OPEN_INFO *dp; 554 ARGV *mapnames; 555 556 if (dict_open_hash == 0) 557 dict_open_init(); 558 mapnames = argv_alloc(dict_open_hash->used + 1); 559 for (ht_info = ht = htable_list(dict_open_hash); *ht; ht++) { 560 dp = (DICT_OPEN_INFO *) ht[0]->value; 561 argv_add(mapnames, dp->type, ARGV_END); 562 } 563 if (dict_mapnames_extend_hook != 0) 564 (void) dict_mapnames_extend_hook(mapnames); 565 qsort((void *) mapnames->argv, mapnames->argc, sizeof(mapnames->argv[0]), 566 dict_sort_alpha_cpp); 567 myfree((void *) ht_info); 568 argv_terminate(mapnames); 569 return mapnames; 570} 571 572/* dict_mapnames_extend - register alternate dictionary type list routine */ 573 574DICT_MAPNAMES_EXTEND_FN dict_mapnames_extend(DICT_MAPNAMES_EXTEND_FN new_cb) 575{ 576 DICT_MAPNAMES_EXTEND_FN old_cb; 577 578 old_cb = dict_mapnames_extend_hook; 579 dict_mapnames_extend_hook = new_cb; 580 return (old_cb); 581} 582 583/* dict_type_override - disguise a dictionary type */ 584 585void dict_type_override(DICT *dict, const char *type) 586{ 587 myfree(dict->type); 588 dict->type = mystrdup(type); 589} 590 591#ifdef TEST 592 593 /* 594 * Proof-of-concept test program. 595 */ 596int main(int argc, char **argv) 597{ 598 dict_test(argc, argv); 599 return (0); 600} 601 602#endif 603