makemandb.c revision 1.33 
1/*	$NetBSD: makemandb.c,v 1.33 2016/03/31 20:17:58 christos Exp $	*/
2/*
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/cdefs.h>
20__RCSID("$NetBSD: makemandb.c,v 1.33 2016/03/31 20:17:58 christos Exp $");
21
22#include <sys/stat.h>
23#include <sys/types.h>
24
25#include <assert.h>
26#include <dirent.h>
27#include <err.h>
28#include <archive.h>
29#include <libgen.h>
30#include <md5.h>
31#include <stdio.h>
32#include <stdlib.h>
33#include <string.h>
34#include <unistd.h>
35#include <util.h>
36
37#include "apropos-utils.h"
38#include "dist/man.h"
39#include "dist/mandoc.h"
40#include "dist/mdoc.h"
41
42#define BUFLEN 1024
43#define MDOC 0	//If the page is of mdoc(7) type
44#define MAN 1	//If the page  is of man(7) type
45
46/*
47 * A data structure for holding section specific data.
48 */
49typedef struct secbuff {
50	char *data;
51	size_t buflen;	//Total length of buffer allocated initially
52	size_t offset;	// Current offset in the buffer.
53} secbuff;
54
55typedef struct makemandb_flags {
56	int optimize;
57	int limit;	// limit the indexing to only NAME section
58	int recreate;	// Database was created from scratch
59	int verbosity;	// 0: quiet, 1: default, 2: verbose
60} makemandb_flags;
61
62typedef struct mandb_rec {
63	/* Fields for mandb table */
64	char *name;	// for storing the name of the man page
65	char *name_desc; // for storing the one line description (.Nd)
66	secbuff desc; // for storing the DESCRIPTION section
67	secbuff lib; // for the LIBRARY section
68	secbuff return_vals; // RETURN VALUES
69	secbuff env; // ENVIRONMENT
70	secbuff files; // FILES
71	secbuff exit_status; // EXIT STATUS
72	secbuff diagnostics; // DIAGNOSTICS
73	secbuff errors; // ERRORS
74	char section[2];
75
76	int xr_found; // To track whether a .Xr was seen when parsing a section
77
78	/* Fields for mandb_meta table */
79	char *md5_hash;
80	dev_t device;
81	ino_t inode;
82	time_t mtime;
83
84	/* Fields for mandb_links table */
85	char *machine;
86	char *links; //all the links to a page in a space separated form
87	char *file_path;
88
89	/* Non-db fields */
90	int page_type; //Indicates the type of page: mdoc or man
91} mandb_rec;
92
93static void append(secbuff *sbuff, const char *src);
94static void init_secbuffs(mandb_rec *);
95static void free_secbuffs(mandb_rec *);
96static int check_md5(const char *, sqlite3 *, const char *, char **, void *, size_t);
97static void cleanup(mandb_rec *);
98static void set_section(const struct mdoc *, const struct man *, mandb_rec *);
99static void set_machine(const struct mdoc *, mandb_rec *);
100static int insert_into_db(sqlite3 *, mandb_rec *);
101static	void begin_parse(const char *, struct mparse *, mandb_rec *,
102			 const void *, size_t len);
103static void pmdoc_node(const struct mdoc_node *, mandb_rec *);
104static void pmdoc_Nm(const struct mdoc_node *, mandb_rec *);
105static void pmdoc_Nd(const struct mdoc_node *, mandb_rec *);
106static void pmdoc_Sh(const struct mdoc_node *, mandb_rec *);
107static void pmdoc_Xr(const struct mdoc_node *, mandb_rec *);
108static void pmdoc_Pp(const struct mdoc_node *, mandb_rec *);
109static void pmdoc_macro_handler(const struct mdoc_node *, mandb_rec *,
110				enum mdoct);
111static void pman_node(const struct man_node *n, mandb_rec *);
112static void pman_parse_node(const struct man_node *, secbuff *);
113static void pman_parse_name(const struct man_node *, mandb_rec *);
114static void pman_sh(const struct man_node *, mandb_rec *);
115static void pman_block(const struct man_node *, mandb_rec *);
116static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
117static void mdoc_parse_section(enum mdoc_sec, const char *, mandb_rec *);
118static void man_parse_section(enum man_sec, const struct man_node *, mandb_rec *);
119static void build_file_cache(sqlite3 *, const char *, const char *,
120			     struct stat *);
121static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
122__dead static void usage(void);
123static void optimize(sqlite3 *);
124static char *parse_escape(const char *);
125static void replace_hyph(char *);
126static makemandb_flags mflags = { .verbosity = 1 };
127
128typedef	void (*pman_nf)(const struct man_node *n, mandb_rec *);
129typedef	void (*pmdoc_nf)(const struct mdoc_node *n, mandb_rec *);
130static	const pmdoc_nf mdocs[MDOC_MAX + 1] = {
131	NULL, /* Ap */
132	NULL, /* Dd */
133	NULL, /* Dt */
134	NULL, /* Os */
135
136	pmdoc_Sh, /* Sh */
137	NULL, /* Ss */
138	pmdoc_Pp, /* Pp */
139	NULL, /* D1 */
140
141	NULL, /* Dl */
142	NULL, /* Bd */
143	NULL, /* Ed */
144	NULL, /* Bl */
145
146	NULL, /* El */
147	NULL, /* It */
148	NULL, /* Ad */
149	NULL, /* An */
150
151	NULL, /* Ar */
152	NULL, /* Cd */
153	NULL, /* Cm */
154	NULL, /* Dv */
155
156	NULL, /* Er */
157	NULL, /* Ev */
158	NULL, /* Ex */
159	NULL, /* Fa */
160
161	NULL, /* Fd */
162	NULL, /* Fl */
163	NULL, /* Fn */
164	NULL, /* Ft */
165
166	NULL, /* Ic */
167	NULL, /* In */
168	NULL, /* Li */
169	pmdoc_Nd, /* Nd */
170
171	pmdoc_Nm, /* Nm */
172	NULL, /* Op */
173	NULL, /* Ot */
174	NULL, /* Pa */
175
176	NULL, /* Rv */
177	NULL, /* St */
178	NULL, /* Va */
179	NULL, /* Vt */
180
181	pmdoc_Xr, /* Xr */
182	NULL, /* %A */
183	NULL, /* %B */
184	NULL, /* %D */
185
186	NULL, /* %I */
187	NULL, /* %J */
188	NULL, /* %N */
189	NULL, /* %O */
190
191	NULL, /* %P */
192	NULL, /* %R */
193	NULL, /* %T */
194	NULL, /* %V */
195
196	NULL, /* Ac */
197	NULL, /* Ao */
198	NULL, /* Aq */
199	NULL, /* At */
200
201	NULL, /* Bc */
202	NULL, /* Bf */
203	NULL, /* Bo */
204	NULL, /* Bq */
205
206	NULL, /* Bsx */
207	NULL, /* Bx */
208	NULL, /* Db */
209	NULL, /* Dc */
210
211	NULL, /* Do */
212	NULL, /* Dq */
213	NULL, /* Ec */
214	NULL, /* Ef */
215
216	NULL, /* Em */
217	NULL, /* Eo */
218	NULL, /* Fx */
219	NULL, /* Ms */
220
221	NULL, /* No */
222	NULL, /* Ns */
223	NULL, /* Nx */
224	NULL, /* Ox */
225
226	NULL, /* Pc */
227	NULL, /* Pf */
228	NULL, /* Po */
229	NULL, /* Pq */
230
231	NULL, /* Qc */
232	NULL, /* Ql */
233	NULL, /* Qo */
234	NULL, /* Qq */
235
236	NULL, /* Re */
237	NULL, /* Rs */
238	NULL, /* Sc */
239	NULL, /* So */
240
241	NULL, /* Sq */
242	NULL, /* Sm */
243	NULL, /* Sx */
244	NULL, /* Sy */
245
246	NULL, /* Tn */
247	NULL, /* Ux */
248	NULL, /* Xc */
249	NULL, /* Xo */
250
251	NULL, /* Fo */
252	NULL, /* Fc */
253	NULL, /* Oo */
254	NULL, /* Oc */
255
256	NULL, /* Bk */
257	NULL, /* Ek */
258	NULL, /* Bt */
259	NULL, /* Hf */
260
261	NULL, /* Fr */
262	NULL, /* Ud */
263	NULL, /* Lb */
264	NULL, /* Lp */
265
266	NULL, /* Lk */
267	NULL, /* Mt */
268	NULL, /* Brq */
269	NULL, /* Bro */
270
271	NULL, /* Brc */
272	NULL, /* %C */
273	NULL, /* Es */
274	NULL, /* En */
275
276	NULL, /* Dx */
277	NULL, /* %Q */
278	NULL, /* br */
279	NULL, /* sp */
280
281	NULL, /* %U */
282	NULL, /* Ta */
283	NULL, /* ll */
284	NULL, /* text */
285};
286
287static	const pman_nf mans[MAN_MAX] = {
288	NULL,	//br
289	NULL,	//TH
290	pman_sh, //SH
291	NULL,	//SS
292	NULL,	//TP
293	NULL,	//LP
294	NULL,	//PP
295	NULL,	//P
296	NULL,	//IP
297	NULL,	//HP
298	NULL,	//SM
299	NULL,	//SB
300	NULL,	//BI
301	NULL,	//IB
302	NULL,	//BR
303	NULL,	//RB
304	NULL,	//R
305	pman_block,	//B
306	NULL,	//I
307	NULL,	//IR
308	NULL,	//RI
309	NULL,	//sp
310	NULL,	//nf
311	NULL,	//fi
312	NULL,	//RE
313	NULL,	//RS
314	NULL,	//DT
315	NULL,	//UC
316	NULL,	//PD
317	NULL,	//AT
318	NULL,	//in
319	NULL,	//ft
320	NULL,	//OP
321	NULL,	//EX
322	NULL,	//EE
323	NULL,	//UR
324	NULL,	//UE
325	NULL,	//ll
326};
327
328
329int
330main(int argc, char *argv[])
331{
332	FILE *file;
333	struct mchars *mchars;
334	const char *sqlstr, *manconf = NULL;
335	char *line, *command, *parent;
336	char *errmsg;
337	int ch;
338	struct mparse *mp;
339	sqlite3 *db;
340	ssize_t len;
341	size_t linesize;
342	struct mandb_rec rec;
343
344	while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
345		switch (ch) {
346		case 'C':
347			manconf = optarg;
348			break;
349		case 'f':
350			mflags.recreate = 1;
351			break;
352		case 'l':
353			mflags.limit = 1;
354			break;
355		case 'o':
356			mflags.optimize = 1;
357			break;
358		case 'Q':
359			mflags.verbosity = 0;
360			break;
361		case 'q':
362			mflags.verbosity = 1;
363			break;
364		case 'v':
365			mflags.verbosity = 2;
366			break;
367		default:
368			usage();
369		}
370	}
371
372	memset(&rec, 0, sizeof(rec));
373
374	init_secbuffs(&rec);
375	mchars = mchars_alloc();
376	if (mchars == NULL)
377		errx(EXIT_FAILURE, "Can't allocate mchars");
378	mp = mparse_alloc(0, MANDOCLEVEL_BADARG, NULL, mchars, NULL);
379
380	if (manconf) {
381		char *arg;
382		size_t command_len = shquote(manconf, NULL, 0) + 1;
383		arg = emalloc(command_len);
384		shquote(manconf, arg, command_len);
385		easprintf(&command, "man -p -C %s", arg);
386		free(arg);
387	} else {
388		command = estrdup("man -p");
389		manconf = MANCONF;
390	}
391
392	if (mflags.recreate) {
393		char *dbp = get_dbpath(manconf);
394		/* No error here, it will fail in init_db in the same call */
395		if (dbp != NULL)
396			remove(dbp);
397	}
398
399	if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
400		exit(EXIT_FAILURE);
401
402	sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, 	&errmsg);
403	if (errmsg != NULL) {
404		warnx("%s", errmsg);
405		free(errmsg);
406		close_db(db);
407		exit(EXIT_FAILURE);
408	}
409
410	sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
411	    &errmsg);
412	if (errmsg != NULL) {
413		warnx("%s", errmsg);
414		free(errmsg);
415		close_db(db);
416		exit(EXIT_FAILURE);
417	}
418
419
420	/* Call man -p to get the list of man page dirs */
421	if ((file = popen(command, "r")) == NULL) {
422		close_db(db);
423		err(EXIT_FAILURE, "fopen failed");
424	}
425	free(command);
426
427	/* Begin the transaction for indexing the pages	*/
428	sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
429	if (errmsg != NULL) {
430		warnx("%s", errmsg);
431		free(errmsg);
432		exit(EXIT_FAILURE);
433	}
434
435	sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
436		 " file PRIMARY KEY);"
437		 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
438		 " ON file_cache (device, inode)";
439
440	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
441	if (errmsg != NULL) {
442		warnx("%s", errmsg);
443		free(errmsg);
444		close_db(db);
445		exit(EXIT_FAILURE);
446	}
447
448	if (mflags.verbosity)
449		printf("Building temporary file cache\n");
450	line = NULL;
451	linesize = 0;
452	while ((len = getline(&line, &linesize, file)) != -1) {
453		/* Replace the new line character at the end of string with '\0' */
454		line[len - 1] = '\0';
455		parent = estrdup(line);
456		char *pdir = estrdup(dirname(parent));
457		free(parent);
458		/* Traverse the man page directories and parse the pages */
459		traversedir(pdir, line, db, mp);
460		free(pdir);
461	}
462	free(line);
463
464	if (pclose(file) == -1) {
465		close_db(db);
466		cleanup(&rec);
467		free_secbuffs(&rec);
468		err(EXIT_FAILURE, "pclose error");
469	}
470
471	if (mflags.verbosity)
472		printf("Performing index update\n");
473	update_db(db, mp, &rec);
474	mparse_free(mp);
475	mchars_free(mchars);
476	free_secbuffs(&rec);
477
478	/* Commit the transaction */
479	sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
480	if (errmsg != NULL) {
481		warnx("%s", errmsg);
482		free(errmsg);
483		close_db(db);
484		exit(EXIT_FAILURE);
485	}
486
487	if (mflags.optimize)
488		optimize(db);
489
490	close_db(db);
491	return 0;
492}
493
494/*
495 * traversedir --
496 *  Traverses the given directory recursively and passes all the man page files
497 *  in the way to build_file_cache()
498 */
499static void
500traversedir(const char *parent, const char *file, sqlite3 *db,
501            struct mparse *mp)
502{
503	struct stat sb;
504	struct dirent *dirp;
505	DIR *dp;
506	char *buf;
507
508	if (stat(file, &sb) < 0) {
509		if (mflags.verbosity)
510			warn("stat failed: %s", file);
511		return;
512	}
513
514	/* If it is a directory, traverse it recursively */
515	if (S_ISDIR(sb.st_mode)) {
516		if ((dp = opendir(file)) == NULL) {
517			if (mflags.verbosity)
518				warn("opendir error: %s", file);
519			return;
520		}
521
522		while ((dirp = readdir(dp)) != NULL) {
523			/* Avoid . and .. entries in a directory */
524			if (strncmp(dirp->d_name, ".", 1)) {
525				easprintf(&buf, "%s/%s", file, dirp->d_name);
526				traversedir(parent, buf, db, mp);
527				free(buf);
528			}
529		}
530		closedir(dp);
531	}
532
533	if (!S_ISREG(sb.st_mode) && !S_ISLNK(sb.st_mode))
534		return;
535
536	if (sb.st_size == 0) {
537		if (mflags.verbosity)
538			warnx("Empty file: %s", file);
539		return;
540	}
541	build_file_cache(db, parent, file, &sb);
542}
543
544/* build_file_cache --
545 *   This function generates an md5 hash of the file passed as its 2nd parameter
546 *   and stores it in a temporary table file_cache along with the full file path.
547 *   This is done to support incremental updation of the database.
548 *   The temporary table file_cache is dropped thereafter in the function
549 *   update_db(), once the database has been updated.
550 */
551static void
552build_file_cache(sqlite3 *db, const char *parent, const char *file,
553		 struct stat *sb)
554{
555	const char *sqlstr;
556	sqlite3_stmt *stmt = NULL;
557	int rc, idx;
558	assert(file != NULL);
559	dev_t device_cache = sb->st_dev;
560	ino_t inode_cache = sb->st_ino;
561	time_t mtime_cache = sb->st_mtime;
562
563	sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
564		 " :mtime, :parent, :file)";
565	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
566	if (rc != SQLITE_OK) {
567		if (mflags.verbosity)
568			warnx("%s", sqlite3_errmsg(db));
569		return;
570	}
571
572	idx = sqlite3_bind_parameter_index(stmt, ":device");
573	rc = sqlite3_bind_int64(stmt, idx, device_cache);
574	if (rc != SQLITE_OK) {
575		if (mflags.verbosity)
576			warnx("%s", sqlite3_errmsg(db));
577		sqlite3_finalize(stmt);
578		return;
579	}
580
581	idx = sqlite3_bind_parameter_index(stmt, ":inode");
582	rc = sqlite3_bind_int64(stmt, idx, inode_cache);
583	if (rc != SQLITE_OK) {
584		if (mflags.verbosity)
585			warnx("%s", sqlite3_errmsg(db));
586		sqlite3_finalize(stmt);
587		return;
588	}
589
590	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
591	rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
592	if (rc != SQLITE_OK) {
593		if (mflags.verbosity)
594			warnx("%s", sqlite3_errmsg(db));
595		sqlite3_finalize(stmt);
596		return;
597	}
598
599	idx = sqlite3_bind_parameter_index(stmt, ":parent");
600	rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
601	if (rc != SQLITE_OK) {
602		if (mflags.verbosity)
603			warnx("%s", sqlite3_errmsg(db));
604		sqlite3_finalize(stmt);
605		return;
606	}
607
608	idx = sqlite3_bind_parameter_index(stmt, ":file");
609	rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
610	if (rc != SQLITE_OK) {
611		if (mflags.verbosity)
612			warnx("%s", sqlite3_errmsg(db));
613		sqlite3_finalize(stmt);
614		return;
615	}
616
617	sqlite3_step(stmt);
618	sqlite3_finalize(stmt);
619}
620
621static void
622update_existing_entry(sqlite3 *db, const char *file, const char *hash,
623    mandb_rec *rec, int *new_count, int *link_count, int *err_count)
624{
625	int update_count, rc, idx;
626	const char *inner_sqlstr;
627	sqlite3_stmt *inner_stmt;
628
629	update_count = sqlite3_total_changes(db);
630	inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
631		       " inode = :inode, mtime = :mtime WHERE"
632		       " md5_hash = :md5 AND file = :file AND"
633		       " (device <> :device2 OR inode <> "
634		       "  :inode2 OR mtime <> :mtime2)";
635	rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
636	if (rc != SQLITE_OK) {
637		if (mflags.verbosity)
638			warnx("%s", sqlite3_errmsg(db));
639		return;
640	}
641	idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
642	sqlite3_bind_int64(inner_stmt, idx, rec->device);
643	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
644	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
645	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
646	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
647	idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
648	sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
649	idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
650	sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
651	idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
652	sqlite3_bind_int64(inner_stmt, idx, rec->device);
653	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
654	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
655	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
656	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
657
658	rc = sqlite3_step(inner_stmt);
659	if (rc == SQLITE_DONE) {
660		/* Check if an update has been performed. */
661		if (update_count != sqlite3_total_changes(db)) {
662			if (mflags.verbosity == 2)
663				printf("Updated %s\n", file);
664			(*new_count)++;
665		} else {
666			/* Otherwise it was a hardlink. */
667			(*link_count)++;
668		}
669	} else {
670		if (mflags.verbosity == 2)
671			warnx("Could not update the meta data for %s", file);
672		(*err_count)++;
673	}
674	sqlite3_finalize(inner_stmt);
675}
676
677/* read_and_decompress --
678 *	Reads the given file into memory. If it is compressed, decompress
679 *	it before returning to the caller.
680 */
681static int
682read_and_decompress(const char *file, void **bufp, size_t *len)
683{
684	size_t off;
685	ssize_t r;
686	struct archive *a;
687	struct archive_entry *ae;
688	char *buf;
689
690	if ((a = archive_read_new()) == NULL)
691		errx(EXIT_FAILURE, "memory allocation failed");
692
693	*bufp = NULL;
694	if (archive_read_support_compression_all(a) != ARCHIVE_OK ||
695	    archive_read_support_format_raw(a) != ARCHIVE_OK ||
696	    archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
697	    archive_read_next_header(a, &ae) != ARCHIVE_OK)
698		goto archive_error;
699	*len = 65536;
700	buf = emalloc(*len);
701	off = 0;
702	for (;;) {
703		r = archive_read_data(a, buf + off, *len - off);
704		if (r == ARCHIVE_OK) {
705			archive_read_close(a);
706			*bufp = buf;
707			*len = off;
708			return 0;
709		}
710		if (r <= 0) {
711			free(buf);
712			break;
713		}
714		off += r;
715		if (off == *len) {
716			*len *= 2;
717			if (*len < off) {
718				if (mflags.verbosity)
719					warnx("File too large: %s", file);
720				free(buf);
721				archive_read_close(a);
722				return -1;
723			}
724			buf = erealloc(buf, *len);
725		}
726	}
727
728archive_error:
729	warnx("Error while reading `%s': %s", file, archive_error_string(a));
730	archive_read_close(a);
731	return -1;
732}
733
734/* update_db --
735 *	Does an incremental updation of the database by checking the file_cache.
736 *	It parses and adds the pages which are present in file_cache,
737 *	but not in the database.
738 *	It also removes the pages which are present in the databse,
739 *	but not in the file_cache.
740 */
741static void
742update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
743{
744	const char *sqlstr;
745	sqlite3_stmt *stmt = NULL;
746	char *file;
747	char *parent;
748	char *errmsg = NULL;
749	char *md5sum;
750	void *buf;
751	size_t buflen;
752	struct sql_row {
753		struct sql_row *next;
754		dev_t device;
755		ino_t inode;
756		time_t mtime;
757		char *parent;
758		char *file;
759	} *rows, *row;
760	int new_count = 0;	/* Counter for newly indexed/updated pages */
761	int total_count = 0;	/* Counter for total number of pages */
762	int err_count = 0;	/* Counter for number of failed pages */
763	int link_count = 0;	/* Counter for number of hard/sym links */
764	int md5_status;
765	int rc;
766
767	sqlstr = "SELECT device, inode, mtime, parent, file"
768	         " FROM metadb.file_cache fc"
769	         " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
770	         "  device = fc.device AND inode = fc.inode AND "
771	         "  mtime = fc.mtime AND file = fc.file)";
772
773	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
774	if (rc != SQLITE_OK) {
775		if (mflags.verbosity)
776		warnx("%s", sqlite3_errmsg(db));
777		close_db(db);
778		errx(EXIT_FAILURE, "Could not query file cache");
779	}
780
781	buf = NULL;
782	rows = NULL;
783	while (sqlite3_step(stmt) == SQLITE_ROW) {
784		row = emalloc(sizeof(struct sql_row));
785		row->device = sqlite3_column_int64(stmt, 0);
786		row->inode = sqlite3_column_int64(stmt, 1);
787		row->mtime = sqlite3_column_int64(stmt, 2);
788		row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
789		row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
790		row->next = rows;
791		rows = row;
792		total_count++;
793	}
794	sqlite3_finalize(stmt);
795
796	for ( ; rows != NULL; free(parent), free(file), free(buf)) {
797		row = rows;
798		rows = rows->next;
799
800		rec->device = row->device;
801		rec->inode = row->inode;
802		rec->mtime = row->mtime;
803		parent = row->parent;
804		file = row->file;
805		free(row);
806
807		if (read_and_decompress(file, &buf, &buflen)) {
808			err_count++;
809			continue;
810		}
811		md5_status = check_md5(file, db, "mandb_meta", &md5sum, buf, buflen);
812		assert(md5sum != NULL);
813		if (md5_status == -1) {
814			if (mflags.verbosity)
815				warnx("An error occurred in checking md5 value"
816			      " for file %s", file);
817			err_count++;
818			continue;
819		}
820
821		if (md5_status == 0) {
822			/*
823			 * The MD5 hash is already present in the database,
824			 * so simply update the metadata, ignoring symlinks.
825			 */
826			struct stat sb;
827			stat(file, &sb);
828			if (S_ISLNK(sb.st_mode)) {
829				free(md5sum);
830				link_count++;
831				continue;
832			}
833			update_existing_entry(db, file, md5sum, rec,
834			    &new_count, &link_count, &err_count);
835			free(md5sum);
836			continue;
837		}
838
839		if (md5_status == 1) {
840			/*
841			 * The MD5 hash was not present in the database.
842			 * This means is either a new file or an updated file.
843			 * We should go ahead with parsing.
844			 */
845			if (mflags.verbosity == 2)
846				printf("Parsing: %s\n", file);
847			rec->md5_hash = md5sum;
848			rec->file_path = estrdup(file);
849			// file_path is freed by insert_into_db itself.
850			chdir(parent);
851			begin_parse(file, mp, rec, buf, buflen);
852			if (insert_into_db(db, rec) < 0) {
853				if (mflags.verbosity)
854					warnx("Error in indexing %s", file);
855				err_count++;
856			} else {
857				new_count++;
858			}
859		}
860	}
861
862	if (mflags.verbosity == 2) {
863		printf("Total Number of new or updated pages encountered = %d\n"
864			"Total number of (hard or symbolic) links found = %d\n"
865			"Total number of pages that were successfully"
866			" indexed/updated = %d\n"
867			"Total number of pages that could not be indexed"
868			" due to errors = %d\n",
869			total_count - link_count, link_count, new_count, err_count);
870	}
871
872	if (mflags.recreate)
873		return;
874
875	if (mflags.verbosity == 2)
876		printf("Deleting stale index entries\n");
877
878	sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
879		 " (SELECT file FROM metadb.file_cache);"
880		 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
881		 " (SELECT md5_hash from mandb_meta);"
882		 "DROP TABLE metadb.file_cache;"
883		 "DELETE FROM mandb WHERE rowid NOT IN"
884		 " (SELECT id FROM mandb_meta);";
885
886	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
887	if (errmsg != NULL) {
888		warnx("Removing old entries failed: %s", errmsg);
889		warnx("Please rebuild database from scratch with -f.");
890		free(errmsg);
891		return;
892	}
893}
894
895/*
896 * begin_parse --
897 *  parses the man page using libmandoc
898 */
899static void
900begin_parse(const char *file, struct mparse *mp, mandb_rec *rec,
901    const void *buf, size_t len)
902{
903	struct mdoc *mdoc;
904	struct man *man;
905	mparse_reset(mp);
906
907	rec->xr_found = 0;
908
909	if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_BADARG) {
910		/* Printing this warning at verbosity level 2
911		 * because some packages from pkgsrc might trigger several
912		 * of such warnings.
913		 */
914		if (mflags.verbosity == 2)
915			warnx("%s: Parse failure", file);
916		return;
917	}
918
919	mparse_result(mp, &mdoc, &man, NULL);
920	if (mdoc == NULL && man == NULL) {
921		if (mflags.verbosity == 2)
922			warnx("Not a man(7) or mdoc(7) page");
923		return;
924	}
925
926	set_machine(mdoc, rec);
927	set_section(mdoc, man, rec);
928	if (mdoc) {
929		rec->page_type = MDOC;
930		pmdoc_node(mdoc_node(mdoc), rec);
931	} else {
932		rec->page_type = MAN;
933		pman_node(man_node(man), rec);
934	}
935}
936
937/*
938 * set_section --
939 *  Extracts the section number and normalizes it to only the numeric part
940 *  (Which should be the first character of the string).
941 */
942static void
943set_section(const struct mdoc *md, const struct man *m, mandb_rec *rec)
944{
945	if (md) {
946		const struct mdoc_meta *md_meta = mdoc_meta(md);
947		if (md_meta->msec == NULL) {
948			rec->section[0] = '?';
949		} else
950			rec->section[0] = md_meta->msec[0];
951	} else if (m) {
952		const struct man_meta *m_meta = man_meta(m);
953		if (m_meta->msec == NULL)
954			rec->section[0] = '?';
955		else
956			rec->section[0] = m_meta->msec[0];
957	} else
958		return;
959
960	if (rec->section[0] == '?' && mflags.verbosity == 2)
961		warnx("%s: Missing section number", rec->file_path);
962}
963
964/*
965 * get_machine --
966 *  Extracts the machine architecture information if available.
967 */
968static void
969set_machine(const struct mdoc *md, mandb_rec *rec)
970{
971	if (md == NULL)
972		return;
973	const struct mdoc_meta *md_meta = mdoc_meta(md);
974	if (md_meta->arch)
975		rec->machine = estrdup(md_meta->arch);
976}
977
978static void
979pmdoc_node(const struct mdoc_node *n, mandb_rec *rec)
980{
981
982	if (n == NULL)
983		return;
984
985	switch (n->type) {
986	case (MDOC_BODY):
987		/* FALLTHROUGH */
988	case (MDOC_TAIL):
989		/* FALLTHROUGH */
990	case (MDOC_ELEM):
991		if (mdocs[n->tok] == NULL)
992			break;
993		(*mdocs[n->tok])(n, rec);
994		break;
995	default:
996		break;
997	}
998
999	pmdoc_node(n->child, rec);
1000	pmdoc_node(n->next, rec);
1001}
1002
1003/*
1004 * pmdoc_Nm --
1005 *  Extracts the Name of the manual page from the .Nm macro
1006 */
1007static void
1008pmdoc_Nm(const struct mdoc_node *n, mandb_rec *rec)
1009{
1010	if (n->sec != SEC_NAME)
1011		return;
1012
1013	for (n = n->child; n; n = n->next) {
1014		if (n->type == MDOC_TEXT) {
1015			char *escaped_name = parse_escape(n->string);
1016			concat(&rec->name, escaped_name);
1017			free(escaped_name);
1018		}
1019	}
1020}
1021
1022/*
1023 * pmdoc_Nd --
1024 *  Extracts the one line description of the man page from the .Nd macro
1025 */
1026static void
1027pmdoc_Nd(const struct mdoc_node *n, mandb_rec *rec)
1028{
1029	char *buf = NULL;
1030	char *name;
1031	char *nd_text;
1032
1033	if (n == NULL || (n->type != MDOC_TEXT && n->tok == MDOC_MAX))
1034		return;
1035
1036	if (n->type == MDOC_TEXT) {
1037		if (rec->xr_found && n->next) {
1038			/*
1039			 * An Xr macro was seen previously, so parse this
1040			 * and the next node, as "Name(Section)".
1041			 */
1042			name = n->string;
1043			n = n->next;
1044			assert(n->type == MDOC_TEXT);
1045			easprintf(&buf, "%s(%s)", name, n->string);
1046			concat(&rec->name_desc, buf);
1047			free(buf);
1048		} else {
1049			nd_text = parse_escape(n->string);
1050			concat(&rec->name_desc, nd_text);
1051			free(nd_text);
1052		}
1053		rec->xr_found = 0;
1054	} else if (mdocs[n->tok] == pmdoc_Xr) {
1055		/* Remember that we have encountered an Xr macro */
1056		rec->xr_found = 1;
1057	}
1058
1059	if (n->child)
1060		pmdoc_Nd(n->child, rec);
1061
1062	if(n->next)
1063		pmdoc_Nd(n->next, rec);
1064}
1065
1066/*
1067 * pmdoc_macro_handler--
1068 *  This function is a single point of handling all the special macros that we
1069 *  want to handle especially. For example the .Xr macro for properly parsing
1070 *  the referenced page name along with the section number, or the .Pp macro
1071 *  for adding a new line whenever we encounter it.
1072 */
1073static void
1074pmdoc_macro_handler(const struct mdoc_node *n, mandb_rec *rec, enum mdoct doct)
1075{
1076	const struct mdoc_node *sn;
1077	assert(n);
1078
1079	switch (doct) {
1080	/*  Parse the man page references.
1081	 * Basically the .Xr macros are used like:
1082	 *  .Xr ls 1
1083 	 *  and formatted like this:
1084	 *  ls(1)
1085	 *  Prepare a buffer to format the data like the above example and call
1086	 *  pmdoc_parse_section to append it.
1087	 */
1088	case MDOC_Xr:
1089		n = n->child;
1090		while (n->type != MDOC_TEXT && n->next)
1091			n = n->next;
1092
1093		if (n && n->type != MDOC_TEXT)
1094			return;
1095		sn = n;
1096		if (n->next)
1097			n = n->next;
1098
1099		while (n->type != MDOC_TEXT && n->next)
1100			n = n->next;
1101
1102		if (n && n->type == MDOC_TEXT) {
1103			char *buf;
1104			easprintf(&buf, "%s(%s)", sn->string, n->string);
1105			mdoc_parse_section(n->sec, buf, rec);
1106			free(buf);
1107		}
1108
1109		break;
1110
1111	/* Parse the .Pp macro to add a new line */
1112	case MDOC_Pp:
1113		if (n->type == MDOC_TEXT)
1114			mdoc_parse_section(n->sec, "\n", rec);
1115		break;
1116	default:
1117		break;
1118	}
1119
1120}
1121
1122/*
1123 * pmdoc_Xr, pmdoc_Pp--
1124 *  Empty stubs.
1125 *  The parser calls these functions each time it encounters
1126 *  a .Xr or .Pp macro. We are parsing all the data from
1127 *  the pmdoc_Sh function, so don't do anything here.
1128 *  (See if else blocks in pmdoc_Sh.)
1129 */
1130static void
1131pmdoc_Xr(const struct mdoc_node *n, mandb_rec *rec)
1132{
1133}
1134
1135static void
1136pmdoc_Pp(const struct mdoc_node *n, mandb_rec *rec)
1137{
1138}
1139
1140/*
1141 * pmdoc_Sh --
1142 *  Called when a .Sh macro is encountered and loops through its body, calling
1143 *  mdoc_parse_section to append the data to the section specific buffer.
1144 *  Two special macros which may occur inside the body of Sh are .Nm and .Xr and
1145 *  they need special handling, thus the separate if branches for them.
1146 */
1147static void
1148pmdoc_Sh(const struct mdoc_node *n, mandb_rec *rec)
1149{
1150	if (n == NULL || (n->type != MDOC_TEXT && n->tok == MDOC_MAX))
1151		return;
1152	int xr_found = 0;
1153
1154	if (n->type == MDOC_TEXT) {
1155		mdoc_parse_section(n->sec, n->string, rec);
1156	} else if (mdocs[n->tok] == pmdoc_Nm && rec->name != NULL) {
1157		/*
1158		 * When encountering a .Nm macro, substitute it
1159		 * with its previously cached value of the argument.
1160		 */
1161		mdoc_parse_section(n->sec, rec->name, rec);
1162	} else if (mdocs[n->tok] == pmdoc_Xr) {
1163		/*
1164		 * When encountering other inline macros,
1165		 * call pmdoc_macro_handler.
1166		 */
1167		pmdoc_macro_handler(n, rec, MDOC_Xr);
1168		xr_found = 1;
1169	} else if (mdocs[n->tok] == pmdoc_Pp) {
1170		pmdoc_macro_handler(n, rec, MDOC_Pp);
1171	}
1172
1173	/*
1174	 * If an Xr macro was encountered then the child node has
1175	 * already been explored by pmdoc_macro_handler.
1176	 */
1177	if (xr_found == 0)
1178		pmdoc_Sh(n->child, rec);
1179	pmdoc_Sh(n->next, rec);
1180}
1181
1182/*
1183 * mdoc_parse_section--
1184 *  Utility function for parsing sections of the mdoc type pages.
1185 *  Takes two params:
1186 *   1. sec is an enum which indicates the section in which we are present
1187 *   2. string is the string which we need to append to the secbuff for this
1188 *      particular section.
1189 *  The function appends string to the global section buffer and returns.
1190 */
1191static void
1192mdoc_parse_section(enum mdoc_sec sec, const char *string, mandb_rec *rec)
1193{
1194	/*
1195	 * If the user specified the 'l' flag, then parse and store only the
1196	 * NAME section. Ignore the rest.
1197	 */
1198	if (mflags.limit)
1199		return;
1200
1201	switch (sec) {
1202	case SEC_LIBRARY:
1203		append(&rec->lib, string);
1204		break;
1205	case SEC_RETURN_VALUES:
1206		append(&rec->return_vals, string);
1207		break;
1208	case SEC_ENVIRONMENT:
1209		append(&rec->env, string);
1210		break;
1211	case SEC_FILES:
1212		append(&rec->files, string);
1213		break;
1214	case SEC_EXIT_STATUS:
1215		append(&rec->exit_status, string);
1216		break;
1217	case SEC_DIAGNOSTICS:
1218		append(&rec->diagnostics, string);
1219		break;
1220	case SEC_ERRORS:
1221		append(&rec->errors, string);
1222		break;
1223	case SEC_NAME:
1224	case SEC_SYNOPSIS:
1225	case SEC_EXAMPLES:
1226	case SEC_STANDARDS:
1227	case SEC_HISTORY:
1228	case SEC_AUTHORS:
1229	case SEC_BUGS:
1230		break;
1231	default:
1232		append(&rec->desc, string);
1233		break;
1234	}
1235}
1236
1237static void
1238pman_node(const struct man_node *n, mandb_rec *rec)
1239{
1240	if (n == NULL)
1241		return;
1242
1243	switch (n->type) {
1244	case (MAN_BODY):
1245		/* FALLTHROUGH */
1246	case (MAN_BLOCK):
1247		/* FALLTHROUGH */
1248	case (MAN_ELEM):
1249		if (mans[n->tok] != NULL)
1250			(*mans[n->tok])(n, rec);
1251		break;
1252	default:
1253		break;
1254	}
1255
1256	pman_node(n->child, rec);
1257	pman_node(n->next, rec);
1258}
1259
1260/*
1261 * pman_parse_name --
1262 *  Parses the NAME section and puts the complete content in the name_desc
1263 *  variable.
1264 */
1265static void
1266pman_parse_name(const struct man_node *n, mandb_rec *rec)
1267{
1268	if (n == NULL)
1269		return;
1270
1271	if (n->type == MAN_TEXT) {
1272		char *tmp = parse_escape(n->string);
1273		concat(&rec->name_desc, tmp);
1274		free(tmp);
1275	}
1276
1277	if (n->child)
1278		pman_parse_name(n->child, rec);
1279
1280	if(n->next)
1281		pman_parse_name(n->next, rec);
1282}
1283
1284/*
1285 * A stub function to be able to parse the macros like .B embedded inside
1286 * a section.
1287 */
1288static void
1289pman_block(const struct man_node *n, mandb_rec *rec)
1290{
1291}
1292
1293/*
1294 * pman_sh --
1295 * This function does one of the two things:
1296 *  1. If the present section is NAME, then it will:
1297 *    (a) Extract the name of the page (in case of multiple comma separated
1298 *        names, it will pick up the first one).
1299 *    (b) Build a space spearated list of all the symlinks/hardlinks to
1300 *        this page and store in the buffer 'links'. These are extracted from
1301 *        the comma separated list of names in the NAME section as well.
1302 *    (c) Move on to the one line description section, which is after the list
1303 *        of names in the NAME section.
1304 *  2. Otherwise, it will check the section name and call the man_parse_section
1305 *     function, passing the enum corresponding that section.
1306 */
1307static void
1308pman_sh(const struct man_node *n, mandb_rec *rec)
1309{
1310	static const struct {
1311		enum man_sec section;
1312		const char *header;
1313	} mapping[] = {
1314	    { MANSEC_DESCRIPTION, "DESCRIPTION" },
1315	    { MANSEC_SYNOPSIS, "SYNOPSIS" },
1316	    { MANSEC_LIBRARY, "LIBRARY" },
1317	    { MANSEC_ERRORS, "ERRORS" },
1318	    { MANSEC_FILES, "FILES" },
1319	    { MANSEC_RETURN_VALUES, "RETURN VALUE" },
1320	    { MANSEC_RETURN_VALUES, "RETURN VALUES" },
1321	    { MANSEC_EXIT_STATUS, "EXIT STATUS" },
1322	    { MANSEC_EXAMPLES, "EXAMPLES" },
1323	    { MANSEC_EXAMPLES, "EXAMPLE" },
1324	    { MANSEC_STANDARDS, "STANDARDS" },
1325	    { MANSEC_HISTORY, "HISTORY" },
1326	    { MANSEC_BUGS, "BUGS" },
1327	    { MANSEC_AUTHORS, "AUTHORS" },
1328	    { MANSEC_COPYRIGHT, "COPYRIGHT" },
1329	};
1330	const struct man_node *head;
1331	char *name_desc;
1332	size_t sz;
1333	size_t i;
1334
1335	if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
1336	    head->type != MAN_TEXT)
1337		return;
1338
1339	/*
1340	 * Check if this section should be extracted and
1341	 * where it should be stored. Handled the trival cases first.
1342	 */
1343	for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
1344		if (strcmp(head->string, mapping[i].header) == 0) {
1345			man_parse_section(mapping[i].section, n, rec);
1346			return;
1347		}
1348	}
1349
1350	if (strcmp(head->string, "NAME") == 0) {
1351		/*
1352		 * We are in the NAME section.
1353		 * pman_parse_name will put the complete content in name_desc.
1354		 */
1355		pman_parse_name(n, rec);
1356
1357		name_desc = rec->name_desc;
1358		if (name_desc == NULL)
1359			return;
1360
1361		/* Remove any leading spaces. */
1362		while (name_desc[0] == ' ')
1363			name_desc++;
1364
1365		/* If the line begins with a "\&", avoid those */
1366		if (name_desc[0] == '\\' && name_desc[1] == '&')
1367			name_desc += 2;
1368
1369		/* Now name_desc should be left with a comma-space
1370		 * separated list of names and the one line description
1371		 * of the page:
1372		 *     "a, b, c \- sample description"
1373		 * Take out the first name, before the first comma
1374		 * (or space) and store it in rec->name.
1375		 * If the page has aliases then they should be
1376		 * in the form of a comma separated list.
1377		 * Keep looping while there is a comma in name_desc,
1378		 * extract the alias name and store in rec->links.
1379		 * When there are no more commas left, break out.
1380		 */
1381		int has_alias = 0;	// Any more aliases left?
1382		while (*name_desc) {
1383			/* Remove any leading spaces or hyphens. */
1384			if (name_desc[0] == ' ' || name_desc[0] =='-') {
1385				name_desc++;
1386				continue;
1387			}
1388			sz = strcspn(name_desc, ", ");
1389
1390			/* Extract the first term and store it in rec->name. */
1391			if (rec->name == NULL) {
1392				if (name_desc[sz] == ',')
1393					has_alias = 1;
1394				name_desc[sz] = 0;
1395				rec->name = emalloc(sz + 1);
1396				memcpy(rec->name, name_desc, sz + 1);
1397				name_desc += sz + 1;
1398				continue;
1399			}
1400
1401			/*
1402			 * Once rec->name is set, rest of the names
1403			 * are to be treated as links or aliases.
1404			 */
1405			if (rec->name && has_alias) {
1406				if (name_desc[sz] != ',') {
1407					/* No more commas left -->
1408					 * no more aliases to take out
1409					 */
1410					has_alias = 0;
1411				}
1412				name_desc[sz] = 0;
1413				concat2(&rec->links, name_desc, sz);
1414				name_desc += sz + 1;
1415				continue;
1416			}
1417			break;
1418		}
1419
1420		/* Parse any escape sequences that might be there */
1421		char *temp = parse_escape(name_desc);
1422		free(rec->name_desc);
1423		rec->name_desc = temp;
1424		temp = parse_escape(rec->name);
1425		free(rec->name);
1426		rec->name = temp;
1427		return;
1428	}
1429
1430	/* The RETURN VALUE section might be specified in multiple ways */
1431	if (strcmp(head->string, "RETURN") == 0 &&
1432	    head->next != NULL && head->next->type == MAN_TEXT &&
1433	    (strcmp(head->next->string, "VALUE") == 0 ||
1434	    strcmp(head->next->string, "VALUES") == 0)) {
1435		man_parse_section(MANSEC_RETURN_VALUES, n, rec);
1436		return;
1437	}
1438
1439	/*
1440	 * EXIT STATUS section can also be specified all on one line or on two
1441	 * separate lines.
1442	 */
1443	if (strcmp(head->string, "EXIT") == 0 &&
1444	    head->next != NULL && head->next->type == MAN_TEXT &&
1445	    strcmp(head->next->string, "STATUS") == 0) {
1446		man_parse_section(MANSEC_EXIT_STATUS, n, rec);
1447		return;
1448	}
1449
1450	/* Store the rest of the content in desc. */
1451	man_parse_section(MANSEC_NONE, n, rec);
1452}
1453
1454/*
1455 * pman_parse_node --
1456 *  Generic function to iterate through a node. Usually called from
1457 *  man_parse_section to parse a particular section of the man page.
1458 */
1459static void
1460pman_parse_node(const struct man_node *n, secbuff *s)
1461{
1462	if (n == NULL)
1463		return;
1464
1465	if (n->type == MAN_TEXT)
1466		append(s, n->string);
1467
1468	pman_parse_node(n->child, s);
1469	pman_parse_node(n->next, s);
1470}
1471
1472/*
1473 * man_parse_section --
1474 *  Takes two parameters:
1475 *   sec: Tells which section we are present in
1476 *   n: Is the present node of the AST.
1477 * Depending on the section, we call pman_parse_node to parse that section and
1478 * concatenate the content from that section into the buffer for that section.
1479 */
1480static void
1481man_parse_section(enum man_sec sec, const struct man_node *n, mandb_rec *rec)
1482{
1483	/*
1484	 * If the user sepecified the 'l' flag then just parse
1485	 * the NAME section, ignore the rest.
1486	 */
1487	if (mflags.limit)
1488		return;
1489
1490	switch (sec) {
1491	case MANSEC_LIBRARY:
1492		pman_parse_node(n, &rec->lib);
1493		break;
1494	case MANSEC_RETURN_VALUES:
1495		pman_parse_node(n, &rec->return_vals);
1496		break;
1497	case MANSEC_ENVIRONMENT:
1498		pman_parse_node(n, &rec->env);
1499		break;
1500	case MANSEC_FILES:
1501		pman_parse_node(n, &rec->files);
1502		break;
1503	case MANSEC_EXIT_STATUS:
1504		pman_parse_node(n, &rec->exit_status);
1505		break;
1506	case MANSEC_DIAGNOSTICS:
1507		pman_parse_node(n, &rec->diagnostics);
1508		break;
1509	case MANSEC_ERRORS:
1510		pman_parse_node(n, &rec->errors);
1511		break;
1512	case MANSEC_NAME:
1513	case MANSEC_SYNOPSIS:
1514	case MANSEC_EXAMPLES:
1515	case MANSEC_STANDARDS:
1516	case MANSEC_HISTORY:
1517	case MANSEC_BUGS:
1518	case MANSEC_AUTHORS:
1519	case MANSEC_COPYRIGHT:
1520		break;
1521	default:
1522		pman_parse_node(n, &rec->desc);
1523		break;
1524	}
1525
1526}
1527
1528/*
1529 * insert_into_db --
1530 *  Inserts the parsed data of the man page in the Sqlite databse.
1531 *  If any of the values is NULL, then we cleanup and return -1 indicating
1532 *  an error.
1533 *  Otherwise, store the data in the database and return 0.
1534 */
1535static int
1536insert_into_db(sqlite3 *db, mandb_rec *rec)
1537{
1538	int rc = 0;
1539	int idx = -1;
1540	const char *sqlstr = NULL;
1541	sqlite3_stmt *stmt = NULL;
1542	char *ln = NULL;
1543	char *errmsg = NULL;
1544	long int mandb_rowid;
1545
1546	/*
1547	 * At the very minimum we want to make sure that we store
1548	 * the following data:
1549	 *   Name, one line description, and the MD5 hash
1550	 */
1551	if (rec->name == NULL || rec->name_desc == NULL ||
1552	    rec->md5_hash == NULL) {
1553		cleanup(rec);
1554		return -1;
1555	}
1556
1557	/* Write null byte at the end of all the sec_buffs */
1558	rec->desc.data[rec->desc.offset] = 0;
1559	rec->lib.data[rec->lib.offset] = 0;
1560	rec->env.data[rec->env.offset] = 0;
1561	rec->return_vals.data[rec->return_vals.offset] = 0;
1562	rec->exit_status.data[rec->exit_status.offset] = 0;
1563	rec->files.data[rec->files.offset] = 0;
1564	rec->diagnostics.data[rec->diagnostics.offset] = 0;
1565	rec->errors.data[rec->errors.offset] = 0;
1566
1567	/*
1568	 * In case of a mdoc page: (sorry, no better place to put this code)
1569	 * parse the comma separated list of names of man pages,
1570	 * the first name will be stored in the mandb table, rest will be
1571	 * treated as links and put in the mandb_links table.
1572	 */
1573	if (rec->page_type == MDOC) {
1574		char *tmp;
1575		rec->links = estrdup(rec->name);
1576		free(rec->name);
1577		int sz = strcspn(rec->links, " \0");
1578		rec->name = emalloc(sz + 1);
1579		memcpy(rec->name, rec->links, sz);
1580		if(rec->name[sz - 1] == ',')
1581			rec->name[sz - 1] = 0;
1582		else
1583			rec->name[sz] = 0;
1584		while (rec->links[sz] == ' ')
1585			++sz;
1586		tmp = estrdup(rec->links + sz);
1587		free(rec->links);
1588		rec->links = tmp;
1589	}
1590
1591/*------------------------ Populate the mandb table---------------------------*/
1592	sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
1593		 " :lib, :return_vals, :env, :files, :exit_status,"
1594		 " :diagnostics, :errors, :md5_hash, :machine)";
1595
1596	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1597	if (rc != SQLITE_OK)
1598		goto Out;
1599
1600	idx = sqlite3_bind_parameter_index(stmt, ":name");
1601	rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
1602	if (rc != SQLITE_OK) {
1603		sqlite3_finalize(stmt);
1604		goto Out;
1605	}
1606
1607	idx = sqlite3_bind_parameter_index(stmt, ":section");
1608	rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
1609	if (rc != SQLITE_OK) {
1610		sqlite3_finalize(stmt);
1611		goto Out;
1612	}
1613
1614	idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
1615	rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
1616	if (rc != SQLITE_OK) {
1617		sqlite3_finalize(stmt);
1618		goto Out;
1619	}
1620
1621	idx = sqlite3_bind_parameter_index(stmt, ":desc");
1622	rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
1623	                       rec->desc.offset + 1, NULL);
1624	if (rc != SQLITE_OK) {
1625		sqlite3_finalize(stmt);
1626		goto Out;
1627	}
1628
1629	idx = sqlite3_bind_parameter_index(stmt, ":lib");
1630	rc = sqlite3_bind_text(stmt, idx, rec->lib.data, rec->lib.offset + 1, NULL);
1631	if (rc != SQLITE_OK) {
1632		sqlite3_finalize(stmt);
1633		goto Out;
1634	}
1635
1636	idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
1637	rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
1638	                      rec->return_vals.offset + 1, NULL);
1639	if (rc != SQLITE_OK) {
1640		sqlite3_finalize(stmt);
1641		goto Out;
1642	}
1643
1644	idx = sqlite3_bind_parameter_index(stmt, ":env");
1645	rc = sqlite3_bind_text(stmt, idx, rec->env.data, rec->env.offset + 1, NULL);
1646	if (rc != SQLITE_OK) {
1647		sqlite3_finalize(stmt);
1648		goto Out;
1649	}
1650
1651	idx = sqlite3_bind_parameter_index(stmt, ":files");
1652	rc = sqlite3_bind_text(stmt, idx, rec->files.data,
1653	                       rec->files.offset + 1, NULL);
1654	if (rc != SQLITE_OK) {
1655		sqlite3_finalize(stmt);
1656		goto Out;
1657	}
1658
1659	idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
1660	rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
1661	                       rec->exit_status.offset + 1, NULL);
1662	if (rc != SQLITE_OK) {
1663		sqlite3_finalize(stmt);
1664		goto Out;
1665	}
1666
1667	idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
1668	rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
1669	                       rec->diagnostics.offset + 1, NULL);
1670	if (rc != SQLITE_OK) {
1671		sqlite3_finalize(stmt);
1672		goto Out;
1673	}
1674
1675	idx = sqlite3_bind_parameter_index(stmt, ":errors");
1676	rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
1677	                       rec->errors.offset + 1, NULL);
1678	if (rc != SQLITE_OK) {
1679		sqlite3_finalize(stmt);
1680		goto Out;
1681	}
1682
1683	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1684	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1685	if (rc != SQLITE_OK) {
1686		sqlite3_finalize(stmt);
1687		goto Out;
1688	}
1689
1690	idx = sqlite3_bind_parameter_index(stmt, ":machine");
1691	if (rec->machine)
1692		rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
1693	else
1694		rc = sqlite3_bind_null(stmt, idx);
1695	if (rc != SQLITE_OK) {
1696		sqlite3_finalize(stmt);
1697		goto Out;
1698	}
1699
1700	rc = sqlite3_step(stmt);
1701	if (rc != SQLITE_DONE) {
1702		sqlite3_finalize(stmt);
1703		goto Out;
1704	}
1705
1706	sqlite3_finalize(stmt);
1707
1708	/* Get the row id of the last inserted row */
1709	mandb_rowid = sqlite3_last_insert_rowid(db);
1710
1711/*------------------------Populate the mandb_meta table-----------------------*/
1712	sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
1713		 " :file, :md5_hash, :id)";
1714	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1715	if (rc != SQLITE_OK)
1716		goto Out;
1717
1718	idx = sqlite3_bind_parameter_index(stmt, ":device");
1719	rc = sqlite3_bind_int64(stmt, idx, rec->device);
1720	if (rc != SQLITE_OK) {
1721		sqlite3_finalize(stmt);
1722		goto Out;
1723	}
1724
1725	idx = sqlite3_bind_parameter_index(stmt, ":inode");
1726	rc = sqlite3_bind_int64(stmt, idx, rec->inode);
1727	if (rc != SQLITE_OK) {
1728		sqlite3_finalize(stmt);
1729		goto Out;
1730	}
1731
1732	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1733	rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
1734	if (rc != SQLITE_OK) {
1735		sqlite3_finalize(stmt);
1736		goto Out;
1737	}
1738
1739	idx = sqlite3_bind_parameter_index(stmt, ":file");
1740	rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1741	if (rc != SQLITE_OK) {
1742		sqlite3_finalize(stmt);
1743		goto Out;
1744	}
1745
1746	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1747	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1748	if (rc != SQLITE_OK) {
1749		sqlite3_finalize(stmt);
1750		goto Out;
1751	}
1752
1753	idx = sqlite3_bind_parameter_index(stmt, ":id");
1754	rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
1755	if (rc != SQLITE_OK) {
1756		sqlite3_finalize(stmt);
1757		goto Out;
1758	}
1759
1760	rc = sqlite3_step(stmt);
1761	sqlite3_finalize(stmt);
1762	if (rc == SQLITE_CONSTRAINT_UNIQUE) {
1763		/* The *most* probable reason for reaching here is that
1764		 * the UNIQUE contraint on the file column of the mandb_meta
1765		 * table was violated.
1766		 * This can happen when a file was updated/modified.
1767		 * To fix this we need to do two things:
1768		 * 1. Delete the row for the older version of this file
1769		 *    from mandb table.
1770		 * 2. Run an UPDATE query to update the row for this file
1771		 *    in the mandb_meta table.
1772		 */
1773		warnx("Trying to update index for %s", rec->file_path);
1774		char *sql = sqlite3_mprintf("DELETE FROM mandb "
1775					    "WHERE rowid = (SELECT id"
1776					    "  FROM mandb_meta"
1777					    "  WHERE file = %Q)",
1778					    rec->file_path);
1779		sqlite3_exec(db, sql, NULL, NULL, &errmsg);
1780		sqlite3_free(sql);
1781		if (errmsg != NULL) {
1782			if (mflags.verbosity)
1783				warnx("%s", errmsg);
1784			free(errmsg);
1785		}
1786		sqlstr = "UPDATE mandb_meta SET device = :device,"
1787			 " inode = :inode, mtime = :mtime, id = :id,"
1788			 " md5_hash = :md5 WHERE file = :file";
1789		rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1790		if (rc != SQLITE_OK) {
1791			if (mflags.verbosity)
1792				warnx("Update failed with error: %s",
1793			    sqlite3_errmsg(db));
1794			close_db(db);
1795			cleanup(rec);
1796			errx(EXIT_FAILURE,
1797			    "Consider running makemandb with -f option");
1798		}
1799
1800		idx = sqlite3_bind_parameter_index(stmt, ":device");
1801		sqlite3_bind_int64(stmt, idx, rec->device);
1802		idx = sqlite3_bind_parameter_index(stmt, ":inode");
1803		sqlite3_bind_int64(stmt, idx, rec->inode);
1804		idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1805		sqlite3_bind_int64(stmt, idx, rec->mtime);
1806		idx = sqlite3_bind_parameter_index(stmt, ":id");
1807		sqlite3_bind_int64(stmt, idx, mandb_rowid);
1808		idx = sqlite3_bind_parameter_index(stmt, ":md5");
1809		sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1810		idx = sqlite3_bind_parameter_index(stmt, ":file");
1811		sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1812		rc = sqlite3_step(stmt);
1813		sqlite3_finalize(stmt);
1814
1815		if (rc != SQLITE_DONE) {
1816			if (mflags.verbosity)
1817				warnx("%s", sqlite3_errmsg(db));
1818			close_db(db);
1819			cleanup(rec);
1820			errx(EXIT_FAILURE,
1821			    "Consider running makemandb with -f option");
1822		}
1823	} else if (rc != SQLITE_DONE) {
1824		/* Otherwise make this error fatal */
1825		warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
1826		cleanup(rec);
1827		close_db(db);
1828		exit(EXIT_FAILURE);
1829	}
1830
1831/*------------------------ Populate the mandb_links table---------------------*/
1832	char *str = NULL;
1833	char *links;
1834	if (rec->links && strlen(rec->links)) {
1835		links = rec->links;
1836		for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
1837			if (ln[0] == ',')
1838				ln++;
1839			if(ln[strlen(ln) - 1] == ',')
1840				ln[strlen(ln) - 1] = 0;
1841
1842			str = sqlite3_mprintf("INSERT INTO mandb_links"
1843					      " VALUES (%Q, %Q, %Q, %Q, %Q)",
1844					      ln, rec->name, rec->section,
1845					      rec->machine, rec->md5_hash);
1846			sqlite3_exec(db, str, NULL, NULL, &errmsg);
1847			sqlite3_free(str);
1848			if (errmsg != NULL) {
1849				warnx("%s", errmsg);
1850				cleanup(rec);
1851				free(errmsg);
1852				return -1;
1853			}
1854		}
1855	}
1856
1857	cleanup(rec);
1858	return 0;
1859
1860  Out:
1861	if (mflags.verbosity)
1862		warnx("%s", sqlite3_errmsg(db));
1863	cleanup(rec);
1864	return -1;
1865}
1866
1867/*
1868 * check_md5--
1869 *  Generates the md5 hash of the file and checks if it already doesn't exist
1870 *  in the table (passed as the 3rd parameter).
1871 *  This function is being used to avoid hardlinks.
1872 *  On successful completion it will also set the value of the fourth parameter
1873 *  to the md5 hash of the file (computed previously). It is the responsibility
1874 *  of the caller to free this buffer.
1875 *  Return values:
1876 *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
1877 *  0: If the md5 hash does not exist in the table.
1878 *  1: If the hash exists in the database.
1879 */
1880static int
1881check_md5(const char *file, sqlite3 *db, const char *table, char **md5sum,
1882    void *buf, size_t buflen)
1883{
1884	int rc = 0;
1885	int idx = -1;
1886	char *sqlstr = NULL;
1887	sqlite3_stmt *stmt = NULL;
1888
1889	assert(file != NULL);
1890	*md5sum = MD5Data(buf, buflen, NULL);
1891	if (*md5sum == NULL) {
1892		if (mflags.verbosity)
1893			warn("md5 failed: %s", file);
1894		return -1;
1895	}
1896
1897	easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash",
1898	    table);
1899	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1900	if (rc != SQLITE_OK) {
1901		free(sqlstr);
1902		free(*md5sum);
1903		*md5sum = NULL;
1904		return -1;
1905	}
1906
1907	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1908	rc = sqlite3_bind_text(stmt, idx, *md5sum, -1, NULL);
1909	if (rc != SQLITE_OK) {
1910		if (mflags.verbosity)
1911			warnx("%s", sqlite3_errmsg(db));
1912		sqlite3_finalize(stmt);
1913		free(sqlstr);
1914		free(*md5sum);
1915		*md5sum = NULL;
1916		return -1;
1917	}
1918
1919	if (sqlite3_step(stmt) == SQLITE_ROW) {
1920		sqlite3_finalize(stmt);
1921		free(sqlstr);
1922		return 0;
1923	}
1924
1925	sqlite3_finalize(stmt);
1926	free(sqlstr);
1927	return 1;
1928}
1929
1930/* Optimize the index for faster search */
1931static void
1932optimize(sqlite3 *db)
1933{
1934	const char *sqlstr;
1935	char *errmsg = NULL;
1936
1937	if (mflags.verbosity == 2)
1938		printf("Optimizing the database index\n");
1939	sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
1940		 "VACUUM";
1941	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
1942	if (errmsg != NULL) {
1943		if (mflags.verbosity)
1944			warnx("%s", errmsg);
1945		free(errmsg);
1946		return;
1947	}
1948}
1949
1950/*
1951 * cleanup --
1952 *  cleans up the global buffers
1953 */
1954static void
1955cleanup(mandb_rec *rec)
1956{
1957	rec->desc.offset = 0;
1958	rec->lib.offset = 0;
1959	rec->return_vals.offset = 0;
1960	rec->env.offset = 0;
1961	rec->exit_status.offset = 0;
1962	rec->diagnostics.offset = 0;
1963	rec->errors.offset = 0;
1964	rec->files.offset = 0;
1965
1966	free(rec->machine);
1967	rec->machine = NULL;
1968
1969	free(rec->links);
1970	rec->links = NULL;
1971
1972	free(rec->file_path);
1973	rec->file_path = NULL;
1974
1975	free(rec->name);
1976	rec->name = NULL;
1977
1978	free(rec->name_desc);
1979	rec->name_desc = NULL;
1980
1981	free(rec->md5_hash);
1982	rec->md5_hash = NULL;
1983}
1984
1985/*
1986 * init_secbuffs--
1987 *  Sets the value of buflen for all the sec_buff field of rec. And then
1988 *  allocate memory to each sec_buff member of rec.
1989 */
1990static void
1991init_secbuffs(mandb_rec *rec)
1992{
1993	/*
1994	 * Some sec_buff might need more memory, for example desc,
1995	 * which stores the data of the DESCRIPTION section,
1996	 * while some might need very small amount of memory.
1997	 * Therefore explicitly setting the value of buflen field for
1998	 * each sec_buff.
1999	 */
2000	rec->desc.buflen = 10 * BUFLEN;
2001	rec->desc.data = emalloc(rec->desc.buflen);
2002	rec->desc.offset = 0;
2003
2004	rec->lib.buflen = BUFLEN / 2;
2005	rec->lib.data = emalloc(rec->lib.buflen);
2006	rec->lib.offset = 0;
2007
2008	rec->return_vals.buflen = BUFLEN;
2009	rec->return_vals.data = emalloc(rec->return_vals.buflen);
2010	rec->return_vals.offset = 0;
2011
2012	rec->exit_status.buflen = BUFLEN;
2013	rec->exit_status.data = emalloc(rec->exit_status.buflen);
2014	rec->exit_status.offset = 0;
2015
2016	rec->env.buflen = BUFLEN;
2017	rec->env.data = emalloc(rec->env.buflen);
2018	rec->env.offset = 0;
2019
2020	rec->files.buflen = BUFLEN;
2021	rec->files.data = emalloc(rec->files.buflen);
2022	rec->files.offset = 0;
2023
2024	rec->diagnostics.buflen = BUFLEN;
2025	rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
2026	rec->diagnostics.offset = 0;
2027
2028	rec->errors.buflen = BUFLEN;
2029	rec->errors.data = emalloc(rec->errors.buflen);
2030	rec->errors.offset = 0;
2031}
2032
2033/*
2034 * free_secbuffs--
2035 *  This function should be called at the end, when all the pages have been
2036 *  parsed.
2037 *  It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
2038 */
2039static void
2040free_secbuffs(mandb_rec *rec)
2041{
2042	free(rec->desc.data);
2043	free(rec->lib.data);
2044	free(rec->return_vals.data);
2045	free(rec->exit_status.data);
2046	free(rec->env.data);
2047	free(rec->files.data);
2048	free(rec->diagnostics.data);
2049	free(rec->errors.data);
2050}
2051
2052static void
2053replace_hyph(char *str)
2054{
2055	char *iter = str;
2056	while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
2057		*iter = '-';
2058
2059	iter = str;
2060	while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
2061		*iter = '-';
2062}
2063
2064static char *
2065parse_escape(const char *str)
2066{
2067	const char *backslash, *last_backslash;
2068	char *result, *iter;
2069	size_t len;
2070
2071	assert(str);
2072
2073	last_backslash = str;
2074	backslash = strchr(str, '\\');
2075	if (backslash == NULL) {
2076		result = estrdup(str);
2077		replace_hyph(result);
2078		return result;
2079	}
2080
2081	result = emalloc(strlen(str) + 1);
2082	iter = result;
2083
2084	do {
2085		len = backslash - last_backslash;
2086		memcpy(iter, last_backslash, len);
2087		iter += len;
2088		if (backslash[1] == '-' || backslash[1] == ' ') {
2089			*iter++ = backslash[1];
2090			last_backslash = backslash + 2;
2091			backslash = strchr(backslash + 2, '\\');
2092		} else {
2093			++backslash;
2094			mandoc_escape(&backslash, NULL, NULL);
2095			last_backslash = backslash;
2096			if (backslash == NULL)
2097				break;
2098			backslash = strchr(last_backslash, '\\');
2099		}
2100	} while (backslash != NULL);
2101	if (last_backslash != NULL)
2102		strcpy(iter, last_backslash);
2103
2104	replace_hyph(result);
2105	return result;
2106}
2107
2108/*
2109 * append--
2110 *  Concatenates a space and src at the end of sbuff->data (much like concat in
2111 *  apropos-utils.c).
2112 *  Rather than reallocating space for writing data, it uses the value of the
2113 *  offset field of sec_buff to write new data at the free space left in the
2114 *  buffer.
2115 *  In case the size of the data to be appended exceeds the number of bytes left
2116 *  in the buffer, it reallocates buflen number of bytes and then continues.
2117 *  Value of offset field should be adjusted as new data is written.
2118 *
2119 *  NOTE: This function does not write the null byte at the end of the buffers,
2120 *  write a null byte at the position pointed to by offset before inserting data
2121 *  in the db.
2122 */
2123static void
2124append(secbuff *sbuff, const char *src)
2125{
2126	short flag = 0;
2127	size_t srclen, newlen;
2128	char *temp;
2129
2130	assert(src != NULL);
2131	temp = parse_escape(src);
2132	srclen = strlen(temp);
2133
2134	if (sbuff->data == NULL) {
2135		sbuff->data = emalloc(sbuff->buflen);
2136		sbuff->offset = 0;
2137	}
2138
2139	newlen = sbuff->offset + srclen + 2;
2140	if (newlen >= sbuff->buflen) {
2141		while (sbuff->buflen < newlen)
2142			sbuff->buflen += sbuff->buflen;
2143		sbuff->data = erealloc(sbuff->data, sbuff->buflen);
2144		flag = 1;
2145	}
2146
2147	/* Append a space at the end of the buffer. */
2148	if (sbuff->offset || flag)
2149		sbuff->data[sbuff->offset++] = ' ';
2150	/* Now, copy src at the end of the buffer. */
2151	memcpy(sbuff->data + sbuff->offset, temp, srclen);
2152	sbuff->offset += srclen;
2153	free(temp);
2154}
2155
2156static void
2157usage(void)
2158{
2159	fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
2160	exit(1);
2161}
2162