makemandb.c revision 1.44 
1/*	$NetBSD: makemandb.c,v 1.44 2016/10/03 16:11:11 abhinav Exp $	*/
2/*
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/cdefs.h>
20__RCSID("$NetBSD: makemandb.c,v 1.44 2016/10/03 16:11:11 abhinav Exp $");
21
22#include <sys/stat.h>
23#include <sys/types.h>
24
25#include <assert.h>
26#include <dirent.h>
27#include <err.h>
28#include <archive.h>
29#include <libgen.h>
30#include <md5.h>
31#include <stdio.h>
32#include <stdlib.h>
33#include <string.h>
34#include <unistd.h>
35#include <util.h>
36
37#include "apropos-utils.h"
38#include "dist/man.h"
39#include "dist/mandoc.h"
40#include "dist/mdoc.h"
41#include "dist/roff.h"
42
43#define BUFLEN 1024
44#define MDOC 0	//If the page is of mdoc(7) type
45#define MAN 1	//If the page  is of man(7) type
46
47/*
48 * A data structure for holding section specific data.
49 */
50typedef struct secbuff {
51	char *data;
52	size_t buflen;	//Total length of buffer allocated initially
53	size_t offset;	// Current offset in the buffer.
54} secbuff;
55
56typedef struct makemandb_flags {
57	int optimize;
58	int limit;	// limit the indexing to only NAME section
59	int recreate;	// Database was created from scratch
60	int verbosity;	// 0: quiet, 1: default, 2: verbose
61} makemandb_flags;
62
63typedef struct roff_mandb_rec {
64	/* Fields for mandb table */
65	char *name;	// for storing the name of the man page
66	char *name_desc; // for storing the one line description (.Nd)
67	secbuff desc; // for storing the DESCRIPTION section
68	secbuff lib; // for the LIBRARY section
69	secbuff return_vals; // RETURN VALUES
70	secbuff env; // ENVIRONMENT
71	secbuff files; // FILES
72	secbuff exit_status; // EXIT STATUS
73	secbuff diagnostics; // DIAGNOSTICS
74	secbuff errors; // ERRORS
75	char *section;
76
77	int xr_found; // To track whether a .Xr was seen when parsing a section
78
79	/* Fields for mandb_meta table */
80	char *md5_hash;
81	dev_t device;
82	ino_t inode;
83	time_t mtime;
84
85	/* Fields for mandb_links table */
86	char *machine;
87	char *links; //all the links to a page in a space separated form
88	char *file_path;
89
90	/* Non-db fields */
91	int page_type; //Indicates the type of page: mdoc or man
92} mandb_rec;
93
94typedef	void (*proff_nf)(const struct roff_node *n, mandb_rec *);
95
96static void append(secbuff *sbuff, const char *src);
97static void init_secbuffs(mandb_rec *);
98static void free_secbuffs(mandb_rec *);
99static int check_md5(const char *, sqlite3 *, const char *, char **, void *, size_t);
100static void cleanup(mandb_rec *);
101static void set_section(const struct roff_man *, mandb_rec *);
102static void set_machine(const struct roff_man *, mandb_rec *);
103static int insert_into_db(sqlite3 *, mandb_rec *);
104static	void begin_parse(const char *, struct mparse *, mandb_rec *,
105			 const void *, size_t len);
106static void proff_node(const struct roff_node *, mandb_rec *, const proff_nf *);
107static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
108static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
109static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
110static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
111static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
112static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
113static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
114static void pman_parse_node(const struct roff_node *, secbuff *);
115static void pman_parse_name(const struct roff_node *, mandb_rec *);
116static void pman_sh(const struct roff_node *, mandb_rec *);
117static void pman_block(const struct roff_node *, mandb_rec *);
118static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
119static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
120static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
121static void build_file_cache(sqlite3 *, const char *, const char *,
122			     struct stat *);
123static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
124__dead static void usage(void);
125static void optimize(sqlite3 *);
126static char *parse_escape(const char *);
127static void replace_hyph(char *);
128static makemandb_flags mflags = { .verbosity = 1 };
129
130static	const proff_nf mdocs[MDOC_MAX + 1] = {
131	NULL, /* Ap */
132	NULL, /* Dd */
133	NULL, /* Dt */
134	NULL, /* Os */
135
136	pmdoc_Sh, /* Sh */
137	NULL, /* Ss */
138	pmdoc_Pp, /* Pp */
139	NULL, /* D1 */
140
141	NULL, /* Dl */
142	NULL, /* Bd */
143	NULL, /* Ed */
144	NULL, /* Bl */
145
146	NULL, /* El */
147	NULL, /* It */
148	NULL, /* Ad */
149	NULL, /* An */
150
151	NULL, /* Ar */
152	NULL, /* Cd */
153	NULL, /* Cm */
154	NULL, /* Dv */
155
156	NULL, /* Er */
157	NULL, /* Ev */
158	NULL, /* Ex */
159	NULL, /* Fa */
160
161	NULL, /* Fd */
162	NULL, /* Fl */
163	NULL, /* Fn */
164	NULL, /* Ft */
165
166	NULL, /* Ic */
167	NULL, /* In */
168	NULL, /* Li */
169	pmdoc_Nd, /* Nd */
170
171	pmdoc_Nm, /* Nm */
172	NULL, /* Op */
173	NULL, /* Ot */
174	NULL, /* Pa */
175
176	NULL, /* Rv */
177	NULL, /* St */
178	NULL, /* Va */
179	NULL, /* Vt */
180
181	pmdoc_Xr, /* Xr */
182	NULL, /* %A */
183	NULL, /* %B */
184	NULL, /* %D */
185
186	NULL, /* %I */
187	NULL, /* %J */
188	NULL, /* %N */
189	NULL, /* %O */
190
191	NULL, /* %P */
192	NULL, /* %R */
193	NULL, /* %T */
194	NULL, /* %V */
195
196	NULL, /* Ac */
197	NULL, /* Ao */
198	NULL, /* Aq */
199	NULL, /* At */
200
201	NULL, /* Bc */
202	NULL, /* Bf */
203	NULL, /* Bo */
204	NULL, /* Bq */
205
206	NULL, /* Bsx */
207	NULL, /* Bx */
208	NULL, /* Db */
209	NULL, /* Dc */
210
211	NULL, /* Do */
212	NULL, /* Dq */
213	NULL, /* Ec */
214	NULL, /* Ef */
215
216	NULL, /* Em */
217	NULL, /* Eo */
218	NULL, /* Fx */
219	NULL, /* Ms */
220
221	NULL, /* No */
222	NULL, /* Ns */
223	NULL, /* Nx */
224	NULL, /* Ox */
225
226	NULL, /* Pc */
227	NULL, /* Pf */
228	NULL, /* Po */
229	NULL, /* Pq */
230
231	NULL, /* Qc */
232	NULL, /* Ql */
233	NULL, /* Qo */
234	NULL, /* Qq */
235
236	NULL, /* Re */
237	NULL, /* Rs */
238	NULL, /* Sc */
239	NULL, /* So */
240
241	NULL, /* Sq */
242	NULL, /* Sm */
243	NULL, /* Sx */
244	NULL, /* Sy */
245
246	NULL, /* Tn */
247	NULL, /* Ux */
248	NULL, /* Xc */
249	NULL, /* Xo */
250
251	NULL, /* Fo */
252	NULL, /* Fc */
253	NULL, /* Oo */
254	NULL, /* Oc */
255
256	NULL, /* Bk */
257	NULL, /* Ek */
258	NULL, /* Bt */
259	NULL, /* Hf */
260
261	NULL, /* Fr */
262	NULL, /* Ud */
263	NULL, /* Lb */
264	NULL, /* Lp */
265
266	NULL, /* Lk */
267	NULL, /* Mt */
268	NULL, /* Brq */
269	NULL, /* Bro */
270
271	NULL, /* Brc */
272	NULL, /* %C */
273	NULL, /* Es */
274	NULL, /* En */
275
276	NULL, /* Dx */
277	NULL, /* %Q */
278	NULL, /* br */
279	NULL, /* sp */
280
281	NULL, /* %U */
282	NULL, /* Ta */
283	NULL, /* ll */
284	NULL, /* text */
285};
286
287static	const proff_nf mans[MAN_MAX] = {
288	NULL,	//br
289	NULL,	//TH
290	pman_sh, //SH
291	NULL,	//SS
292	NULL,	//TP
293	NULL,	//LP
294	NULL,	//PP
295	NULL,	//P
296	NULL,	//IP
297	NULL,	//HP
298	NULL,	//SM
299	NULL,	//SB
300	NULL,	//BI
301	NULL,	//IB
302	NULL,	//BR
303	NULL,	//RB
304	NULL,	//R
305	pman_block,	//B
306	NULL,	//I
307	NULL,	//IR
308	NULL,	//RI
309	NULL,	//sp
310	NULL,	//nf
311	NULL,	//fi
312	NULL,	//RE
313	NULL,	//RS
314	NULL,	//DT
315	NULL,	//UC
316	NULL,	//PD
317	NULL,	//AT
318	NULL,	//in
319	NULL,	//ft
320	NULL,	//OP
321	NULL,	//EX
322	NULL,	//EE
323	NULL,	//UR
324	NULL,	//UE
325	NULL,	//ll
326};
327
328
329int
330main(int argc, char *argv[])
331{
332	FILE *file;
333	const char *sqlstr, *manconf = NULL;
334	char *line, *command, *parent;
335	char *errmsg;
336	int ch;
337	struct mparse *mp;
338	sqlite3 *db;
339	ssize_t len;
340	size_t linesize;
341	struct roff_mandb_rec rec;
342
343	while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
344		switch (ch) {
345		case 'C':
346			manconf = optarg;
347			break;
348		case 'f':
349			mflags.recreate = 1;
350			break;
351		case 'l':
352			mflags.limit = 1;
353			break;
354		case 'o':
355			mflags.optimize = 1;
356			break;
357		case 'Q':
358			mflags.verbosity = 0;
359			break;
360		case 'q':
361			mflags.verbosity = 1;
362			break;
363		case 'v':
364			mflags.verbosity = 2;
365			break;
366		default:
367			usage();
368		}
369	}
370
371	memset(&rec, 0, sizeof(rec));
372
373	init_secbuffs(&rec);
374	mchars_alloc();
375	mp = mparse_alloc(0, MANDOCLEVEL_BADARG, NULL, NULL);
376
377	if (manconf) {
378		char *arg;
379		size_t command_len = shquote(manconf, NULL, 0) + 1;
380		arg = emalloc(command_len);
381		shquote(manconf, arg, command_len);
382		easprintf(&command, "man -p -C %s", arg);
383		free(arg);
384	} else {
385		command = estrdup("man -p");
386		manconf = MANCONF;
387	}
388
389	if (mflags.recreate) {
390		char *dbp = get_dbpath(manconf);
391		/* No error here, it will fail in init_db in the same call */
392		if (dbp != NULL)
393			remove(dbp);
394	}
395
396	if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
397		exit(EXIT_FAILURE);
398
399	sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, 	&errmsg);
400	if (errmsg != NULL) {
401		warnx("%s", errmsg);
402		free(errmsg);
403		close_db(db);
404		exit(EXIT_FAILURE);
405	}
406
407	sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
408	    &errmsg);
409	if (errmsg != NULL) {
410		warnx("%s", errmsg);
411		free(errmsg);
412		close_db(db);
413		exit(EXIT_FAILURE);
414	}
415
416
417	/* Call man -p to get the list of man page dirs */
418	if ((file = popen(command, "r")) == NULL) {
419		close_db(db);
420		err(EXIT_FAILURE, "fopen failed");
421	}
422	free(command);
423
424	/* Begin the transaction for indexing the pages	*/
425	sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
426	if (errmsg != NULL) {
427		warnx("%s", errmsg);
428		free(errmsg);
429		close_db(db);
430		exit(EXIT_FAILURE);
431	}
432
433	sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
434		 " file PRIMARY KEY);"
435		 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
436		 " ON file_cache (device, inode)";
437
438	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
439	if (errmsg != NULL) {
440		warnx("%s", errmsg);
441		free(errmsg);
442		close_db(db);
443		exit(EXIT_FAILURE);
444	}
445
446	if (mflags.verbosity)
447		printf("Building temporary file cache\n");
448	line = NULL;
449	linesize = 0;
450	while ((len = getline(&line, &linesize, file)) != -1) {
451		/* Replace the new line character at the end of string with '\0' */
452		line[len - 1] = '\0';
453		parent = estrdup(line);
454		char *pdir = estrdup(dirname(parent));
455		free(parent);
456		/* Traverse the man page directories and parse the pages */
457		traversedir(pdir, line, db, mp);
458		free(pdir);
459	}
460	free(line);
461
462	if (pclose(file) == -1) {
463		close_db(db);
464		cleanup(&rec);
465		free_secbuffs(&rec);
466		err(EXIT_FAILURE, "pclose error");
467	}
468
469	if (mflags.verbosity)
470		printf("Performing index update\n");
471	update_db(db, mp, &rec);
472	mparse_free(mp);
473	mchars_free();
474	free_secbuffs(&rec);
475
476	/* Commit the transaction */
477	sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
478	if (errmsg != NULL) {
479		warnx("%s", errmsg);
480		free(errmsg);
481		close_db(db);
482		exit(EXIT_FAILURE);
483	}
484
485	if (mflags.optimize)
486		optimize(db);
487
488	close_db(db);
489	return 0;
490}
491
492/*
493 * traversedir --
494 *  Traverses the given directory recursively and passes all the man page files
495 *  in the way to build_file_cache()
496 */
497static void
498traversedir(const char *parent, const char *file, sqlite3 *db,
499            struct mparse *mp)
500{
501	struct stat sb;
502	struct dirent *dirp;
503	DIR *dp;
504	char *buf;
505
506	if (stat(file, &sb) < 0) {
507		if (mflags.verbosity)
508			warn("stat failed: %s", file);
509		return;
510	}
511
512	/* If it is a directory, traverse it recursively */
513	if (S_ISDIR(sb.st_mode)) {
514		if ((dp = opendir(file)) == NULL) {
515			if (mflags.verbosity)
516				warn("opendir error: %s", file);
517			return;
518		}
519
520		while ((dirp = readdir(dp)) != NULL) {
521			/* Avoid . and .. entries in a directory */
522			if (strncmp(dirp->d_name, ".", 1)) {
523				easprintf(&buf, "%s/%s", file, dirp->d_name);
524				traversedir(parent, buf, db, mp);
525				free(buf);
526			}
527		}
528		closedir(dp);
529	}
530
531	if (!S_ISREG(sb.st_mode) && !S_ISLNK(sb.st_mode))
532		return;
533
534	if (sb.st_size == 0) {
535		if (mflags.verbosity)
536			warnx("Empty file: %s", file);
537		return;
538	}
539	build_file_cache(db, parent, file, &sb);
540}
541
542/* build_file_cache --
543 *   This function generates an md5 hash of the file passed as its 2nd parameter
544 *   and stores it in a temporary table file_cache along with the full file path.
545 *   This is done to support incremental updation of the database.
546 *   The temporary table file_cache is dropped thereafter in the function
547 *   update_db(), once the database has been updated.
548 */
549static void
550build_file_cache(sqlite3 *db, const char *parent, const char *file,
551		 struct stat *sb)
552{
553	const char *sqlstr;
554	sqlite3_stmt *stmt = NULL;
555	int rc, idx;
556	assert(file != NULL);
557	dev_t device_cache = sb->st_dev;
558	ino_t inode_cache = sb->st_ino;
559	time_t mtime_cache = sb->st_mtime;
560
561	sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
562		 " :mtime, :parent, :file)";
563	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
564	if (rc != SQLITE_OK) {
565		if (mflags.verbosity)
566			warnx("%s", sqlite3_errmsg(db));
567		return;
568	}
569
570	idx = sqlite3_bind_parameter_index(stmt, ":device");
571	rc = sqlite3_bind_int64(stmt, idx, device_cache);
572	if (rc != SQLITE_OK) {
573		if (mflags.verbosity)
574			warnx("%s", sqlite3_errmsg(db));
575		sqlite3_finalize(stmt);
576		return;
577	}
578
579	idx = sqlite3_bind_parameter_index(stmt, ":inode");
580	rc = sqlite3_bind_int64(stmt, idx, inode_cache);
581	if (rc != SQLITE_OK) {
582		if (mflags.verbosity)
583			warnx("%s", sqlite3_errmsg(db));
584		sqlite3_finalize(stmt);
585		return;
586	}
587
588	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
589	rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
590	if (rc != SQLITE_OK) {
591		if (mflags.verbosity)
592			warnx("%s", sqlite3_errmsg(db));
593		sqlite3_finalize(stmt);
594		return;
595	}
596
597	idx = sqlite3_bind_parameter_index(stmt, ":parent");
598	rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
599	if (rc != SQLITE_OK) {
600		if (mflags.verbosity)
601			warnx("%s", sqlite3_errmsg(db));
602		sqlite3_finalize(stmt);
603		return;
604	}
605
606	idx = sqlite3_bind_parameter_index(stmt, ":file");
607	rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
608	if (rc != SQLITE_OK) {
609		if (mflags.verbosity)
610			warnx("%s", sqlite3_errmsg(db));
611		sqlite3_finalize(stmt);
612		return;
613	}
614
615	sqlite3_step(stmt);
616	sqlite3_finalize(stmt);
617}
618
619static void
620update_existing_entry(sqlite3 *db, const char *file, const char *hash,
621    mandb_rec *rec, int *new_count, int *link_count, int *err_count)
622{
623	int update_count, rc, idx;
624	const char *inner_sqlstr;
625	sqlite3_stmt *inner_stmt;
626
627	update_count = sqlite3_total_changes(db);
628	inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
629		       " inode = :inode, mtime = :mtime WHERE"
630		       " md5_hash = :md5 AND file = :file AND"
631		       " (device <> :device2 OR inode <> "
632		       "  :inode2 OR mtime <> :mtime2)";
633	rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
634	if (rc != SQLITE_OK) {
635		if (mflags.verbosity)
636			warnx("%s", sqlite3_errmsg(db));
637		return;
638	}
639	idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
640	sqlite3_bind_int64(inner_stmt, idx, rec->device);
641	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
642	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
643	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
644	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
645	idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
646	sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
647	idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
648	sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
649	idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
650	sqlite3_bind_int64(inner_stmt, idx, rec->device);
651	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
652	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
653	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
654	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
655
656	rc = sqlite3_step(inner_stmt);
657	if (rc == SQLITE_DONE) {
658		/* Check if an update has been performed. */
659		if (update_count != sqlite3_total_changes(db)) {
660			if (mflags.verbosity == 2)
661				printf("Updated %s\n", file);
662			(*new_count)++;
663		} else {
664			/* Otherwise it was a hardlink. */
665			(*link_count)++;
666		}
667	} else {
668		if (mflags.verbosity == 2)
669			warnx("Could not update the meta data for %s", file);
670		(*err_count)++;
671	}
672	sqlite3_finalize(inner_stmt);
673}
674
675/* read_and_decompress --
676 *	Reads the given file into memory. If it is compressed, decompress
677 *	it before returning to the caller.
678 */
679static int
680read_and_decompress(const char *file, void **bufp, size_t *len)
681{
682	size_t off;
683	ssize_t r;
684	struct archive *a;
685	struct archive_entry *ae;
686	char *buf;
687
688	if ((a = archive_read_new()) == NULL)
689		errx(EXIT_FAILURE, "memory allocation failed");
690
691	*bufp = NULL;
692	if (archive_read_support_compression_all(a) != ARCHIVE_OK ||
693	    archive_read_support_format_raw(a) != ARCHIVE_OK ||
694	    archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
695	    archive_read_next_header(a, &ae) != ARCHIVE_OK)
696		goto archive_error;
697	*len = 65536;
698	buf = emalloc(*len);
699	off = 0;
700	for (;;) {
701		r = archive_read_data(a, buf + off, *len - off);
702		if (r == ARCHIVE_OK) {
703			archive_read_finish(a);
704			*bufp = buf;
705			*len = off;
706			return 0;
707		}
708		if (r <= 0) {
709			free(buf);
710			break;
711		}
712		off += r;
713		if (off == *len) {
714			*len *= 2;
715			if (*len < off) {
716				if (mflags.verbosity)
717					warnx("File too large: %s", file);
718				free(buf);
719				archive_read_finish(a);
720				return -1;
721			}
722			buf = erealloc(buf, *len);
723		}
724	}
725
726archive_error:
727	warnx("Error while reading `%s': %s", file, archive_error_string(a));
728	archive_read_finish(a);
729	return -1;
730}
731
732/* update_db --
733 *	Does an incremental updation of the database by checking the file_cache.
734 *	It parses and adds the pages which are present in file_cache,
735 *	but not in the database.
736 *	It also removes the pages which are present in the databse,
737 *	but not in the file_cache.
738 */
739static void
740update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
741{
742	const char *sqlstr;
743	sqlite3_stmt *stmt = NULL;
744	char *file;
745	char *parent;
746	char *errmsg = NULL;
747	char *md5sum;
748	void *buf;
749	size_t buflen;
750	struct sql_row {
751		struct sql_row *next;
752		dev_t device;
753		ino_t inode;
754		time_t mtime;
755		char *parent;
756		char *file;
757	} *rows, *row;
758	int new_count = 0;	/* Counter for newly indexed/updated pages */
759	int total_count = 0;	/* Counter for total number of pages */
760	int err_count = 0;	/* Counter for number of failed pages */
761	int link_count = 0;	/* Counter for number of hard/sym links */
762	int md5_status;
763	int rc;
764
765	sqlstr = "SELECT device, inode, mtime, parent, file"
766	         " FROM metadb.file_cache fc"
767	         " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
768	         "  device = fc.device AND inode = fc.inode AND "
769	         "  mtime = fc.mtime AND file = fc.file)";
770
771	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
772	if (rc != SQLITE_OK) {
773		if (mflags.verbosity)
774			warnx("%s", sqlite3_errmsg(db));
775		close_db(db);
776		errx(EXIT_FAILURE, "Could not query file cache");
777	}
778
779	buf = NULL;
780	rows = NULL;
781	while (sqlite3_step(stmt) == SQLITE_ROW) {
782		row = emalloc(sizeof(struct sql_row));
783		row->device = sqlite3_column_int64(stmt, 0);
784		row->inode = sqlite3_column_int64(stmt, 1);
785		row->mtime = sqlite3_column_int64(stmt, 2);
786		row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
787		row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
788		row->next = rows;
789		rows = row;
790		total_count++;
791	}
792	sqlite3_finalize(stmt);
793
794	for ( ; rows != NULL; free(parent), free(file), free(buf)) {
795		row = rows;
796		rows = rows->next;
797
798		rec->device = row->device;
799		rec->inode = row->inode;
800		rec->mtime = row->mtime;
801		parent = row->parent;
802		file = row->file;
803		free(row);
804
805		if (read_and_decompress(file, &buf, &buflen)) {
806			err_count++;
807			continue;
808		}
809		md5_status = check_md5(file, db, "mandb_meta", &md5sum, buf,
810		    buflen);
811		assert(md5sum != NULL);
812		if (md5_status == -1) {
813			if (mflags.verbosity)
814				warnx("An error occurred in checking md5 value"
815			      " for file %s", file);
816			err_count++;
817			continue;
818		}
819
820		if (md5_status == 0) {
821			/*
822			 * The MD5 hash is already present in the database,
823			 * so simply update the metadata, ignoring symlinks.
824			 */
825			struct stat sb;
826			stat(file, &sb);
827			if (S_ISLNK(sb.st_mode)) {
828				free(md5sum);
829				link_count++;
830				continue;
831			}
832			update_existing_entry(db, file, md5sum, rec,
833			    &new_count, &link_count, &err_count);
834			free(md5sum);
835			continue;
836		}
837
838		if (md5_status == 1) {
839			/*
840			 * The MD5 hash was not present in the database.
841			 * This means is either a new file or an updated file.
842			 * We should go ahead with parsing.
843			 */
844			if (chdir(parent) == -1) {
845				if (mflags.verbosity)
846					warn("chdir failed for `%s', could "
847					    "not index `%s'", parent, file);
848				err_count++;
849				free(md5sum);
850				continue;
851			}
852
853			if (mflags.verbosity == 2)
854				printf("Parsing: %s\n", file);
855			rec->md5_hash = md5sum;
856			rec->file_path = estrdup(file);
857			// file_path is freed by insert_into_db itself.
858			begin_parse(file, mp, rec, buf, buflen);
859			if (insert_into_db(db, rec) < 0) {
860				if (mflags.verbosity)
861					warnx("Error in indexing `%s'", file);
862				err_count++;
863			} else {
864				new_count++;
865			}
866		}
867	}
868
869	if (mflags.verbosity == 2) {
870		printf("Total Number of new or updated pages encountered = %d\n"
871		    "Total number of (hard or symbolic) links found = %d\n"
872		    "Total number of pages that were successfully"
873		    " indexed/updated = %d\n"
874		    "Total number of pages that could not be indexed"
875		    " due to errors = %d\n",
876		    total_count - link_count, link_count, new_count, err_count);
877	}
878
879	if (mflags.recreate)
880		return;
881
882	if (mflags.verbosity == 2)
883		printf("Deleting stale index entries\n");
884
885	sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
886		 " (SELECT file FROM metadb.file_cache);"
887		 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
888		 " (SELECT md5_hash from mandb_meta);"
889		 "DROP TABLE metadb.file_cache;"
890		 "DELETE FROM mandb WHERE rowid NOT IN"
891		 " (SELECT id FROM mandb_meta);";
892
893	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
894	if (errmsg != NULL) {
895		warnx("Removing old entries failed: %s", errmsg);
896		warnx("Please rebuild database from scratch with -f.");
897		free(errmsg);
898		return;
899	}
900}
901
902/*
903 * begin_parse --
904 *  parses the man page using libmandoc
905 */
906static void
907begin_parse(const char *file, struct mparse *mp, mandb_rec *rec,
908    const void *buf, size_t len)
909{
910	struct roff_man *roff;
911	mparse_reset(mp);
912
913	rec->xr_found = 0;
914
915	if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_BADARG) {
916		/* Printing this warning at verbosity level 2
917		 * because some packages from pkgsrc might trigger several
918		 * of such warnings.
919		 */
920		if (mflags.verbosity == 2)
921			warnx("%s: Parse failure", file);
922		return;
923	}
924
925	mparse_result(mp, &roff, NULL);
926	if (roff == NULL) {
927		if (mflags.verbosity == 2)
928			warnx("Not a roff(7) page");
929		return;
930	}
931
932	if (roff->macroset == MACROSET_MDOC) {
933		mdoc_validate(roff);
934		rec->page_type = MDOC;
935		proff_node(roff->first->child, rec, mdocs);
936	} else if (roff->macroset == MACROSET_MAN) {
937		man_validate(roff);
938		rec->page_type = MAN;
939		proff_node(roff->first->child, rec, mans);
940	} else
941		warnx("Unknown macroset %d", roff->macroset);
942	set_machine(roff, rec);
943	set_section(roff, rec);
944}
945
946/*
947 * set_section --
948 *  Extracts the section number and normalizes it to only the numeric part
949 *  (Which should be the first character of the string).
950 */
951static void
952set_section(const struct roff_man *rm, mandb_rec *rec)
953{
954	if (!rm)
955		return;
956	const struct roff_meta *rm_meta = &rm->meta;
957	const char *s = rm_meta->msec == NULL ? "?" : rm_meta->msec;
958	easprintf(&rec->section, "%s", s);
959	if (rec->section[0] == '?' && mflags.verbosity == 2)
960		warnx("%s: Missing section number", rec->file_path);
961}
962
963/*
964 * get_machine --
965 *  Extracts the machine architecture information if available.
966 */
967static void
968set_machine(const struct roff_man *rm, mandb_rec *rec)
969{
970	if (rm == NULL)
971		return;
972	const struct roff_meta *rm_meta = &rm->meta;
973	if (rm_meta->arch)
974		rec->machine = estrdup(rm_meta->arch);
975}
976
977/*
978 * pmdoc_Nm --
979 *  Extracts the Name of the manual page from the .Nm macro
980 */
981static void
982pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
983{
984	if (n->sec != SEC_NAME)
985		return;
986
987	for (n = n->child; n; n = n->next) {
988		if (n->type == ROFFT_TEXT) {
989			char *escaped_name = parse_escape(n->string);
990			concat(&rec->name, escaped_name);
991			free(escaped_name);
992		}
993	}
994}
995
996/*
997 * pmdoc_Nd --
998 *  Extracts the one line description of the man page from the .Nd macro
999 */
1000static void
1001pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
1002{
1003	if (n->type == ROFFT_BODY)
1004		deroff(&rec->name_desc, n);
1005
1006}
1007
1008/*
1009 * pmdoc_macro_handler--
1010 *  This function is a single point of handling all the special macros that we
1011 *  want to handle especially. For example the .Xr macro for properly parsing
1012 *  the referenced page name along with the section number, or the .Pp macro
1013 *  for adding a new line whenever we encounter it.
1014 */
1015static void
1016pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
1017{
1018	const struct roff_node *sn;
1019	assert(n);
1020
1021	switch (doct) {
1022	/*  Parse the man page references.
1023	 * Basically the .Xr macros are used like:
1024	 *  .Xr ls 1
1025 	 *  and formatted like this:
1026	 *  ls(1)
1027	 *  Prepare a buffer to format the data like the above example and call
1028	 *  pmdoc_parse_section to append it.
1029	 */
1030	case MDOC_Xr:
1031		n = n->child;
1032		while (n->type != ROFFT_TEXT && n->next)
1033			n = n->next;
1034
1035		if (n && n->type != ROFFT_TEXT)
1036			return;
1037		sn = n;
1038		if (n->next)
1039			n = n->next;
1040
1041		while (n->type != ROFFT_TEXT && n->next)
1042			n = n->next;
1043
1044		if (n && n->type == ROFFT_TEXT) {
1045			char *buf;
1046			easprintf(&buf, "%s(%s)", sn->string, n->string);
1047			mdoc_parse_section(n->sec, buf, rec);
1048			free(buf);
1049		}
1050
1051		break;
1052
1053	/* Parse the .Pp macro to add a new line */
1054	case MDOC_Pp:
1055		if (n->type == ROFFT_TEXT)
1056			mdoc_parse_section(n->sec, "\n", rec);
1057		break;
1058	default:
1059		break;
1060	}
1061
1062}
1063
1064/*
1065 * pmdoc_Xr, pmdoc_Pp--
1066 *  Empty stubs.
1067 *  The parser calls these functions each time it encounters
1068 *  a .Xr or .Pp macro. We are parsing all the data from
1069 *  the pmdoc_Sh function, so don't do anything here.
1070 *  (See if else blocks in pmdoc_Sh.)
1071 */
1072static void
1073pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
1074{
1075}
1076
1077static void
1078pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
1079{
1080}
1081
1082/*
1083 * pmdoc_Sh --
1084 *  Called when a .Sh macro is encountered and tries to parse its body
1085 */
1086static void
1087pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
1088{
1089	if (n == NULL)
1090		return;
1091
1092	switch (n->sec) {
1093	case SEC_NAME:
1094	case SEC_SYNOPSIS:
1095	case SEC_EXAMPLES:
1096	case SEC_STANDARDS:
1097	case SEC_HISTORY:
1098	case SEC_AUTHORS:
1099	case SEC_BUGS:
1100		/*
1101		 * We don't care about text from these sections
1102		 */
1103		return;
1104	default:
1105		break;
1106	}
1107
1108	if (n->type == ROFFT_BLOCK)
1109		mdoc_parse_Sh(n->body, rec);
1110}
1111
1112/*
1113 *  Called from pmdoc_Sh to parse body of a .Sh macro. It calls
1114 *  mdoc_parse_section to append the data to the section specific buffer.
1115 *  Two special macros which may occur inside the body of Sh are .Nm and .Xr and
1116 *  they need special handling, thus the separate if branches for them.
1117 */
1118static void
1119mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
1120{
1121	if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
1122		return;
1123	int xr_found = 0;
1124
1125	if (n->type == ROFFT_TEXT) {
1126		mdoc_parse_section(n->sec, n->string, rec);
1127	} else if (mdocs[n->tok] == pmdoc_Nm && rec->name != NULL) {
1128		/*
1129		 * When encountering a .Nm macro, substitute it
1130		 * with its previously cached value of the argument.
1131		 */
1132		mdoc_parse_section(n->sec, rec->name, rec);
1133	} else if (mdocs[n->tok] == pmdoc_Xr) {
1134		/*
1135		 * When encountering other inline macros,
1136		 * call pmdoc_macro_handler.
1137		 */
1138		pmdoc_macro_handler(n, rec, MDOC_Xr);
1139		xr_found = 1;
1140	} else if (mdocs[n->tok] == pmdoc_Pp) {
1141		pmdoc_macro_handler(n, rec, MDOC_Pp);
1142	}
1143
1144	/*
1145	 * If an Xr macro was encountered then the child node has
1146	 * already been explored by pmdoc_macro_handler.
1147	 */
1148	if (xr_found == 0)
1149		mdoc_parse_Sh(n->child, rec);
1150	mdoc_parse_Sh(n->next, rec);
1151}
1152
1153/*
1154 * mdoc_parse_section--
1155 *  Utility function for parsing sections of the mdoc type pages.
1156 *  Takes two params:
1157 *   1. sec is an enum which indicates the section in which we are present
1158 *   2. string is the string which we need to append to the secbuff for this
1159 *      particular section.
1160 *  The function appends string to the global section buffer and returns.
1161 */
1162static void
1163mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
1164{
1165	/*
1166	 * If the user specified the 'l' flag, then parse and store only the
1167	 * NAME section. Ignore the rest.
1168	 */
1169	if (mflags.limit)
1170		return;
1171
1172	switch (sec) {
1173	case SEC_LIBRARY:
1174		append(&rec->lib, string);
1175		break;
1176	case SEC_RETURN_VALUES:
1177		append(&rec->return_vals, string);
1178		break;
1179	case SEC_ENVIRONMENT:
1180		append(&rec->env, string);
1181		break;
1182	case SEC_FILES:
1183		append(&rec->files, string);
1184		break;
1185	case SEC_EXIT_STATUS:
1186		append(&rec->exit_status, string);
1187		break;
1188	case SEC_DIAGNOSTICS:
1189		append(&rec->diagnostics, string);
1190		break;
1191	case SEC_ERRORS:
1192		append(&rec->errors, string);
1193		break;
1194	default:
1195		append(&rec->desc, string);
1196		break;
1197	}
1198}
1199
1200static void
1201proff_node(const struct roff_node *n, mandb_rec *rec, const proff_nf *func)
1202{
1203	if (n == NULL)
1204		return;
1205
1206	switch (n->type) {
1207	case (ROFFT_BODY):
1208		/* FALLTHROUGH */
1209	case (ROFFT_BLOCK):
1210		/* FALLTHROUGH */
1211	case (ROFFT_ELEM):
1212		if (func[n->tok] != NULL)
1213			(*func[n->tok])(n, rec);
1214		break;
1215	default:
1216		break;
1217	}
1218
1219	proff_node(n->child, rec, func);
1220	proff_node(n->next, rec, func);
1221}
1222
1223/*
1224 * pman_parse_name --
1225 *  Parses the NAME section and puts the complete content in the name_desc
1226 *  variable.
1227 */
1228static void
1229pman_parse_name(const struct roff_node *n, mandb_rec *rec)
1230{
1231	if (n == NULL)
1232		return;
1233
1234	if (n->type == ROFFT_TEXT) {
1235		char *tmp = parse_escape(n->string);
1236		concat(&rec->name_desc, tmp);
1237		free(tmp);
1238	}
1239
1240	if (n->child)
1241		pman_parse_name(n->child, rec);
1242
1243	if(n->next)
1244		pman_parse_name(n->next, rec);
1245}
1246
1247/*
1248 * A stub function to be able to parse the macros like .B embedded inside
1249 * a section.
1250 */
1251static void
1252pman_block(const struct roff_node *n, mandb_rec *rec)
1253{
1254}
1255
1256/*
1257 * pman_sh --
1258 * This function does one of the two things:
1259 *  1. If the present section is NAME, then it will:
1260 *    (a) Extract the name of the page (in case of multiple comma separated
1261 *        names, it will pick up the first one).
1262 *    (b) Build a space spearated list of all the symlinks/hardlinks to
1263 *        this page and store in the buffer 'links'. These are extracted from
1264 *        the comma separated list of names in the NAME section as well.
1265 *    (c) Move on to the one line description section, which is after the list
1266 *        of names in the NAME section.
1267 *  2. Otherwise, it will check the section name and call the man_parse_section
1268 *     function, passing the enum corresponding to that section.
1269 */
1270static void
1271pman_sh(const struct roff_node *n, mandb_rec *rec)
1272{
1273	static const struct {
1274		enum man_sec section;
1275		const char *header;
1276	} mapping[] = {
1277	    { MANSEC_DESCRIPTION, "DESCRIPTION" },
1278	    { MANSEC_SYNOPSIS, "SYNOPSIS" },
1279	    { MANSEC_LIBRARY, "LIBRARY" },
1280	    { MANSEC_ERRORS, "ERRORS" },
1281	    { MANSEC_FILES, "FILES" },
1282	    { MANSEC_RETURN_VALUES, "RETURN VALUE" },
1283	    { MANSEC_RETURN_VALUES, "RETURN VALUES" },
1284	    { MANSEC_EXIT_STATUS, "EXIT STATUS" },
1285	    { MANSEC_EXAMPLES, "EXAMPLES" },
1286	    { MANSEC_EXAMPLES, "EXAMPLE" },
1287	    { MANSEC_STANDARDS, "STANDARDS" },
1288	    { MANSEC_HISTORY, "HISTORY" },
1289	    { MANSEC_BUGS, "BUGS" },
1290	    { MANSEC_AUTHORS, "AUTHORS" },
1291	    { MANSEC_COPYRIGHT, "COPYRIGHT" },
1292	};
1293	const struct roff_node *head;
1294	char *name_desc;
1295	size_t sz;
1296	size_t i;
1297
1298	if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
1299	    head->type != ROFFT_TEXT)
1300		return;
1301
1302	/*
1303	 * Check if this section should be extracted and
1304	 * where it should be stored. Handled the trival cases first.
1305	 */
1306	for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
1307		if (strcmp(head->string, mapping[i].header) == 0) {
1308			man_parse_section(mapping[i].section, n, rec);
1309			return;
1310		}
1311	}
1312
1313	if (strcmp(head->string, "NAME") == 0) {
1314		/*
1315		 * We are in the NAME section.
1316		 * pman_parse_name will put the complete content in name_desc.
1317		 */
1318		pman_parse_name(n, rec);
1319
1320		name_desc = rec->name_desc;
1321		if (name_desc == NULL)
1322			return;
1323
1324		/* Remove any leading spaces. */
1325		while (name_desc[0] == ' ')
1326			name_desc++;
1327
1328		/* If the line begins with a "\&", avoid those */
1329		if (name_desc[0] == '\\' && name_desc[1] == '&')
1330			name_desc += 2;
1331
1332		/* Now name_desc should be left with a comma-space
1333		 * separated list of names and the one line description
1334		 * of the page:
1335		 *     "a, b, c \- sample description"
1336		 * Take out the first name, before the first comma
1337		 * (or space) and store it in rec->name.
1338		 * If the page has aliases then they should be
1339		 * in the form of a comma separated list.
1340		 * Keep looping while there is a comma in name_desc,
1341		 * extract the alias name and store in rec->links.
1342		 * When there are no more commas left, break out.
1343		 */
1344		int has_alias = 0;	// Any more aliases left?
1345		while (*name_desc) {
1346			/* Remove any leading spaces or hyphens. */
1347			if (name_desc[0] == ' ' || name_desc[0] == '-') {
1348				name_desc++;
1349				continue;
1350			}
1351			sz = strcspn(name_desc, ", ");
1352
1353			/* Extract the first term and store it in rec->name. */
1354			if (rec->name == NULL) {
1355				if (name_desc[sz] == ',')
1356					has_alias = 1;
1357				rec->name = estrndup(name_desc, sz);
1358				/* XXX This would only happen with a poorly
1359				 * written man page, maybe warn? */
1360				if (name_desc[sz] == '\0')
1361					break;
1362				name_desc += sz + 1;
1363				continue;
1364			}
1365
1366			/*
1367			 * Once rec->name is set, rest of the names
1368			 * are to be treated as links or aliases.
1369			 */
1370			if (rec->name && has_alias) {
1371				if (name_desc[sz] != ',') {
1372					/* No more commas left --> no more
1373					 * aliases to take out */
1374					has_alias = 0;
1375				}
1376				concat2(&rec->links, name_desc, sz);
1377				/* XXX This would only happen with a poorly
1378				 * written man page, maybe warn? */
1379				if (name_desc[sz] == '\0')
1380					break;
1381				name_desc += sz + 1;
1382				continue;
1383			}
1384			break;
1385		}
1386
1387		/* Parse any escape sequences that might be there */
1388		char *temp = parse_escape(name_desc);
1389		free(rec->name_desc);
1390		rec->name_desc = temp;
1391		temp = parse_escape(rec->name);
1392		free(rec->name);
1393		rec->name = temp;
1394		return;
1395	}
1396
1397	/* The RETURN VALUE section might be specified in multiple ways */
1398	if (strcmp(head->string, "RETURN") == 0 &&
1399	    head->next != NULL && head->next->type == ROFFT_TEXT &&
1400	    (strcmp(head->next->string, "VALUE") == 0 ||
1401	    strcmp(head->next->string, "VALUES") == 0)) {
1402		man_parse_section(MANSEC_RETURN_VALUES, n, rec);
1403		return;
1404	}
1405
1406	/*
1407	 * EXIT STATUS section can also be specified all on one line or on two
1408	 * separate lines.
1409	 */
1410	if (strcmp(head->string, "EXIT") == 0 &&
1411	    head->next != NULL && head->next->type == ROFFT_TEXT &&
1412	    strcmp(head->next->string, "STATUS") == 0) {
1413		man_parse_section(MANSEC_EXIT_STATUS, n, rec);
1414		return;
1415	}
1416
1417	/* Store the rest of the content in desc. */
1418	man_parse_section(MANSEC_NONE, n, rec);
1419}
1420
1421/*
1422 * pman_parse_node --
1423 *  Generic function to iterate through a node. Usually called from
1424 *  man_parse_section to parse a particular section of the man page.
1425 */
1426static void
1427pman_parse_node(const struct roff_node *n, secbuff *s)
1428{
1429	if (n == NULL)
1430		return;
1431
1432	if (n->type == ROFFT_TEXT)
1433		append(s, n->string);
1434
1435	pman_parse_node(n->child, s);
1436	pman_parse_node(n->next, s);
1437}
1438
1439/*
1440 * man_parse_section --
1441 *  Takes two parameters:
1442 *   sec: Tells which section we are present in
1443 *   n: Is the present node of the AST.
1444 * Depending on the section, we call pman_parse_node to parse that section and
1445 * concatenate the content from that section into the buffer for that section.
1446 */
1447static void
1448man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
1449{
1450	/*
1451	 * If the user sepecified the 'l' flag then just parse
1452	 * the NAME section, ignore the rest.
1453	 */
1454	if (mflags.limit)
1455		return;
1456
1457	switch (sec) {
1458	case MANSEC_LIBRARY:
1459		pman_parse_node(n, &rec->lib);
1460		break;
1461	case MANSEC_RETURN_VALUES:
1462		pman_parse_node(n, &rec->return_vals);
1463		break;
1464	case MANSEC_ENVIRONMENT:
1465		pman_parse_node(n, &rec->env);
1466		break;
1467	case MANSEC_FILES:
1468		pman_parse_node(n, &rec->files);
1469		break;
1470	case MANSEC_EXIT_STATUS:
1471		pman_parse_node(n, &rec->exit_status);
1472		break;
1473	case MANSEC_DIAGNOSTICS:
1474		pman_parse_node(n, &rec->diagnostics);
1475		break;
1476	case MANSEC_ERRORS:
1477		pman_parse_node(n, &rec->errors);
1478		break;
1479	case MANSEC_NAME:
1480	case MANSEC_SYNOPSIS:
1481	case MANSEC_EXAMPLES:
1482	case MANSEC_STANDARDS:
1483	case MANSEC_HISTORY:
1484	case MANSEC_BUGS:
1485	case MANSEC_AUTHORS:
1486	case MANSEC_COPYRIGHT:
1487		break;
1488	default:
1489		pman_parse_node(n, &rec->desc);
1490		break;
1491	}
1492
1493}
1494
1495/*
1496 * insert_into_db --
1497 *  Inserts the parsed data of the man page in the Sqlite databse.
1498 *  If any of the values is NULL, then we cleanup and return -1 indicating
1499 *  an error.
1500 *  Otherwise, store the data in the database and return 0.
1501 */
1502static int
1503insert_into_db(sqlite3 *db, mandb_rec *rec)
1504{
1505	int rc = 0;
1506	int idx = -1;
1507	const char *sqlstr = NULL;
1508	sqlite3_stmt *stmt = NULL;
1509	char *ln = NULL;
1510	char *errmsg = NULL;
1511	long int mandb_rowid;
1512
1513	/*
1514	 * At the very minimum we want to make sure that we store
1515	 * the following data:
1516	 *   Name, one line description, and the MD5 hash
1517	 */
1518	if (rec->name == NULL || rec->name_desc == NULL ||
1519	    rec->md5_hash == NULL) {
1520		cleanup(rec);
1521		return -1;
1522	}
1523
1524	/* Write null byte at the end of all the sec_buffs */
1525	rec->desc.data[rec->desc.offset] = 0;
1526	rec->lib.data[rec->lib.offset] = 0;
1527	rec->env.data[rec->env.offset] = 0;
1528	rec->return_vals.data[rec->return_vals.offset] = 0;
1529	rec->exit_status.data[rec->exit_status.offset] = 0;
1530	rec->files.data[rec->files.offset] = 0;
1531	rec->diagnostics.data[rec->diagnostics.offset] = 0;
1532	rec->errors.data[rec->errors.offset] = 0;
1533
1534	/*
1535	 * In case of a mdoc page: (sorry, no better place to put this code)
1536	 * parse the comma separated list of names of man pages,
1537	 * the first name will be stored in the mandb table, rest will be
1538	 * treated as links and put in the mandb_links table.
1539	 */
1540	if (rec->page_type == MDOC) {
1541		char *tmp;
1542		rec->links = estrdup(rec->name);
1543		free(rec->name);
1544		size_t sz = strcspn(rec->links, " \0");
1545		rec->name = emalloc(sz + 1);
1546		memcpy(rec->name, rec->links, sz);
1547		if(rec->name[sz - 1] == ',')
1548			rec->name[sz - 1] = 0;
1549		else
1550			rec->name[sz] = 0;
1551		while (rec->links[sz] == ' ')
1552			++sz;
1553		tmp = estrdup(rec->links + sz);
1554		free(rec->links);
1555		rec->links = tmp;
1556	}
1557
1558/*------------------------ Populate the mandb table---------------------------*/
1559	sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
1560		 " :lib, :return_vals, :env, :files, :exit_status,"
1561		 " :diagnostics, :errors, :md5_hash, :machine)";
1562
1563	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1564	if (rc != SQLITE_OK)
1565		goto Out;
1566
1567	idx = sqlite3_bind_parameter_index(stmt, ":name");
1568	rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
1569	if (rc != SQLITE_OK) {
1570		sqlite3_finalize(stmt);
1571		goto Out;
1572	}
1573
1574	idx = sqlite3_bind_parameter_index(stmt, ":section");
1575	rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
1576	if (rc != SQLITE_OK) {
1577		sqlite3_finalize(stmt);
1578		goto Out;
1579	}
1580
1581	idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
1582	rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
1583	if (rc != SQLITE_OK) {
1584		sqlite3_finalize(stmt);
1585		goto Out;
1586	}
1587
1588	idx = sqlite3_bind_parameter_index(stmt, ":desc");
1589	rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
1590	                       rec->desc.offset + 1, NULL);
1591	if (rc != SQLITE_OK) {
1592		sqlite3_finalize(stmt);
1593		goto Out;
1594	}
1595
1596	idx = sqlite3_bind_parameter_index(stmt, ":lib");
1597	rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
1598	    rec->lib.offset + 1, NULL);
1599	if (rc != SQLITE_OK) {
1600		sqlite3_finalize(stmt);
1601		goto Out;
1602	}
1603
1604	idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
1605	rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
1606	                      rec->return_vals.offset + 1, NULL);
1607	if (rc != SQLITE_OK) {
1608		sqlite3_finalize(stmt);
1609		goto Out;
1610	}
1611
1612	idx = sqlite3_bind_parameter_index(stmt, ":env");
1613	rc = sqlite3_bind_text(stmt, idx, rec->env.data,
1614	    rec->env.offset + 1, NULL);
1615	if (rc != SQLITE_OK) {
1616		sqlite3_finalize(stmt);
1617		goto Out;
1618	}
1619
1620	idx = sqlite3_bind_parameter_index(stmt, ":files");
1621	rc = sqlite3_bind_text(stmt, idx, rec->files.data,
1622	                       rec->files.offset + 1, NULL);
1623	if (rc != SQLITE_OK) {
1624		sqlite3_finalize(stmt);
1625		goto Out;
1626	}
1627
1628	idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
1629	rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
1630	                       rec->exit_status.offset + 1, NULL);
1631	if (rc != SQLITE_OK) {
1632		sqlite3_finalize(stmt);
1633		goto Out;
1634	}
1635
1636	idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
1637	rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
1638	                       rec->diagnostics.offset + 1, NULL);
1639	if (rc != SQLITE_OK) {
1640		sqlite3_finalize(stmt);
1641		goto Out;
1642	}
1643
1644	idx = sqlite3_bind_parameter_index(stmt, ":errors");
1645	rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
1646	                       rec->errors.offset + 1, NULL);
1647	if (rc != SQLITE_OK) {
1648		sqlite3_finalize(stmt);
1649		goto Out;
1650	}
1651
1652	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1653	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1654	if (rc != SQLITE_OK) {
1655		sqlite3_finalize(stmt);
1656		goto Out;
1657	}
1658
1659	idx = sqlite3_bind_parameter_index(stmt, ":machine");
1660	if (rec->machine)
1661		rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
1662	else
1663		rc = sqlite3_bind_null(stmt, idx);
1664	if (rc != SQLITE_OK) {
1665		sqlite3_finalize(stmt);
1666		goto Out;
1667	}
1668
1669	rc = sqlite3_step(stmt);
1670	if (rc != SQLITE_DONE) {
1671		sqlite3_finalize(stmt);
1672		goto Out;
1673	}
1674
1675	sqlite3_finalize(stmt);
1676
1677	/* Get the row id of the last inserted row */
1678	mandb_rowid = sqlite3_last_insert_rowid(db);
1679
1680/*------------------------Populate the mandb_meta table-----------------------*/
1681	sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
1682		 " :file, :md5_hash, :id)";
1683	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1684	if (rc != SQLITE_OK)
1685		goto Out;
1686
1687	idx = sqlite3_bind_parameter_index(stmt, ":device");
1688	rc = sqlite3_bind_int64(stmt, idx, rec->device);
1689	if (rc != SQLITE_OK) {
1690		sqlite3_finalize(stmt);
1691		goto Out;
1692	}
1693
1694	idx = sqlite3_bind_parameter_index(stmt, ":inode");
1695	rc = sqlite3_bind_int64(stmt, idx, rec->inode);
1696	if (rc != SQLITE_OK) {
1697		sqlite3_finalize(stmt);
1698		goto Out;
1699	}
1700
1701	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1702	rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
1703	if (rc != SQLITE_OK) {
1704		sqlite3_finalize(stmt);
1705		goto Out;
1706	}
1707
1708	idx = sqlite3_bind_parameter_index(stmt, ":file");
1709	rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1710	if (rc != SQLITE_OK) {
1711		sqlite3_finalize(stmt);
1712		goto Out;
1713	}
1714
1715	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1716	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1717	if (rc != SQLITE_OK) {
1718		sqlite3_finalize(stmt);
1719		goto Out;
1720	}
1721
1722	idx = sqlite3_bind_parameter_index(stmt, ":id");
1723	rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
1724	if (rc != SQLITE_OK) {
1725		sqlite3_finalize(stmt);
1726		goto Out;
1727	}
1728
1729	rc = sqlite3_step(stmt);
1730	sqlite3_finalize(stmt);
1731	if (rc == SQLITE_CONSTRAINT_UNIQUE) {
1732		/* The *most* probable reason for reaching here is that
1733		 * the UNIQUE contraint on the file column of the mandb_meta
1734		 * table was violated.
1735		 * This can happen when a file was updated/modified.
1736		 * To fix this we need to do two things:
1737		 * 1. Delete the row for the older version of this file
1738		 *    from mandb table.
1739		 * 2. Run an UPDATE query to update the row for this file
1740		 *    in the mandb_meta table.
1741		 */
1742		warnx("Trying to update index for %s", rec->file_path);
1743		char *sql = sqlite3_mprintf("DELETE FROM mandb "
1744					    "WHERE rowid = (SELECT id"
1745					    "  FROM mandb_meta"
1746					    "  WHERE file = %Q)",
1747					    rec->file_path);
1748		sqlite3_exec(db, sql, NULL, NULL, &errmsg);
1749		sqlite3_free(sql);
1750		if (errmsg != NULL) {
1751			if (mflags.verbosity)
1752				warnx("%s", errmsg);
1753			free(errmsg);
1754		}
1755		sqlstr = "UPDATE mandb_meta SET device = :device,"
1756			 " inode = :inode, mtime = :mtime, id = :id,"
1757			 " md5_hash = :md5 WHERE file = :file";
1758		rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1759		if (rc != SQLITE_OK) {
1760			if (mflags.verbosity)
1761				warnx("Update failed with error: %s",
1762			    sqlite3_errmsg(db));
1763			close_db(db);
1764			cleanup(rec);
1765			errx(EXIT_FAILURE,
1766			    "Consider running makemandb with -f option");
1767		}
1768
1769		idx = sqlite3_bind_parameter_index(stmt, ":device");
1770		sqlite3_bind_int64(stmt, idx, rec->device);
1771		idx = sqlite3_bind_parameter_index(stmt, ":inode");
1772		sqlite3_bind_int64(stmt, idx, rec->inode);
1773		idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1774		sqlite3_bind_int64(stmt, idx, rec->mtime);
1775		idx = sqlite3_bind_parameter_index(stmt, ":id");
1776		sqlite3_bind_int64(stmt, idx, mandb_rowid);
1777		idx = sqlite3_bind_parameter_index(stmt, ":md5");
1778		sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1779		idx = sqlite3_bind_parameter_index(stmt, ":file");
1780		sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1781		rc = sqlite3_step(stmt);
1782		sqlite3_finalize(stmt);
1783
1784		if (rc != SQLITE_DONE) {
1785			if (mflags.verbosity)
1786				warnx("%s", sqlite3_errmsg(db));
1787			close_db(db);
1788			cleanup(rec);
1789			errx(EXIT_FAILURE,
1790			    "Consider running makemandb with -f option");
1791		}
1792	} else if (rc != SQLITE_DONE) {
1793		/* Otherwise make this error fatal */
1794		warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
1795		cleanup(rec);
1796		close_db(db);
1797		exit(EXIT_FAILURE);
1798	}
1799
1800/*------------------------ Populate the mandb_links table---------------------*/
1801	char *str = NULL;
1802	char *links;
1803	if (rec->links && strlen(rec->links)) {
1804		links = rec->links;
1805		for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
1806			if (ln[0] == ',')
1807				ln++;
1808			if(ln[strlen(ln) - 1] == ',')
1809				ln[strlen(ln) - 1] = 0;
1810
1811			str = sqlite3_mprintf("INSERT INTO mandb_links"
1812					      " VALUES (%Q, %Q, %Q, %Q, %Q)",
1813					      ln, rec->name, rec->section,
1814					      rec->machine, rec->md5_hash);
1815			sqlite3_exec(db, str, NULL, NULL, &errmsg);
1816			sqlite3_free(str);
1817			if (errmsg != NULL) {
1818				warnx("%s", errmsg);
1819				cleanup(rec);
1820				free(errmsg);
1821				return -1;
1822			}
1823		}
1824	}
1825
1826	cleanup(rec);
1827	return 0;
1828
1829  Out:
1830	if (mflags.verbosity)
1831		warnx("%s", sqlite3_errmsg(db));
1832	cleanup(rec);
1833	return -1;
1834}
1835
1836/*
1837 * check_md5--
1838 *  Generates the md5 hash of the file and checks if it already doesn't exist
1839 *  in the table (passed as the 3rd parameter).
1840 *  This function is being used to avoid hardlinks.
1841 *  On successful completion it will also set the value of the fourth parameter
1842 *  to the md5 hash of the file (computed previously). It is the responsibility
1843 *  of the caller to free this buffer.
1844 *  Return values:
1845 *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
1846 *  0: If the md5 hash does not exist in the table.
1847 *  1: If the hash exists in the database.
1848 */
1849static int
1850check_md5(const char *file, sqlite3 *db, const char *table, char **md5sum,
1851    void *buf, size_t buflen)
1852{
1853	int rc = 0;
1854	int idx = -1;
1855	char *sqlstr = NULL;
1856	sqlite3_stmt *stmt = NULL;
1857
1858	assert(file != NULL);
1859	*md5sum = MD5Data(buf, buflen, NULL);
1860	if (*md5sum == NULL) {
1861		if (mflags.verbosity)
1862			warn("md5 failed: %s", file);
1863		return -1;
1864	}
1865
1866	easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash",
1867	    table);
1868	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1869	if (rc != SQLITE_OK) {
1870		free(sqlstr);
1871		free(*md5sum);
1872		*md5sum = NULL;
1873		return -1;
1874	}
1875
1876	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1877	rc = sqlite3_bind_text(stmt, idx, *md5sum, -1, NULL);
1878	if (rc != SQLITE_OK) {
1879		if (mflags.verbosity)
1880			warnx("%s", sqlite3_errmsg(db));
1881		sqlite3_finalize(stmt);
1882		free(sqlstr);
1883		free(*md5sum);
1884		*md5sum = NULL;
1885		return -1;
1886	}
1887
1888	if (sqlite3_step(stmt) == SQLITE_ROW) {
1889		sqlite3_finalize(stmt);
1890		free(sqlstr);
1891		return 0;
1892	}
1893
1894	sqlite3_finalize(stmt);
1895	free(sqlstr);
1896	return 1;
1897}
1898
1899/* Optimize the index for faster search */
1900static void
1901optimize(sqlite3 *db)
1902{
1903	const char *sqlstr;
1904	char *errmsg = NULL;
1905
1906	if (mflags.verbosity == 2)
1907		printf("Optimizing the database index\n");
1908	sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
1909		 "VACUUM";
1910	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
1911	if (errmsg != NULL) {
1912		if (mflags.verbosity)
1913			warnx("%s", errmsg);
1914		free(errmsg);
1915		return;
1916	}
1917}
1918
1919/*
1920 * cleanup --
1921 *  cleans up the global buffers
1922 */
1923static void
1924cleanup(mandb_rec *rec)
1925{
1926	rec->desc.offset = 0;
1927	rec->lib.offset = 0;
1928	rec->return_vals.offset = 0;
1929	rec->env.offset = 0;
1930	rec->exit_status.offset = 0;
1931	rec->diagnostics.offset = 0;
1932	rec->errors.offset = 0;
1933	rec->files.offset = 0;
1934
1935	free(rec->machine);
1936	rec->machine = NULL;
1937
1938	free(rec->links);
1939	rec->links = NULL;
1940
1941	free(rec->file_path);
1942	rec->file_path = NULL;
1943
1944	free(rec->name);
1945	rec->name = NULL;
1946
1947	free(rec->name_desc);
1948	rec->name_desc = NULL;
1949
1950	free(rec->md5_hash);
1951	rec->md5_hash = NULL;
1952
1953	free(rec->section);
1954	rec->section = NULL;
1955}
1956
1957/*
1958 * init_secbuffs--
1959 *  Sets the value of buflen for all the sec_buff field of rec. And then
1960 *  allocate memory to each sec_buff member of rec.
1961 */
1962static void
1963init_secbuffs(mandb_rec *rec)
1964{
1965	/*
1966	 * Some sec_buff might need more memory, for example desc,
1967	 * which stores the data of the DESCRIPTION section,
1968	 * while some might need very small amount of memory.
1969	 * Therefore explicitly setting the value of buflen field for
1970	 * each sec_buff.
1971	 */
1972	rec->desc.buflen = 10 * BUFLEN;
1973	rec->desc.data = emalloc(rec->desc.buflen);
1974	rec->desc.offset = 0;
1975
1976	rec->lib.buflen = BUFLEN / 2;
1977	rec->lib.data = emalloc(rec->lib.buflen);
1978	rec->lib.offset = 0;
1979
1980	rec->return_vals.buflen = BUFLEN;
1981	rec->return_vals.data = emalloc(rec->return_vals.buflen);
1982	rec->return_vals.offset = 0;
1983
1984	rec->exit_status.buflen = BUFLEN;
1985	rec->exit_status.data = emalloc(rec->exit_status.buflen);
1986	rec->exit_status.offset = 0;
1987
1988	rec->env.buflen = BUFLEN;
1989	rec->env.data = emalloc(rec->env.buflen);
1990	rec->env.offset = 0;
1991
1992	rec->files.buflen = BUFLEN;
1993	rec->files.data = emalloc(rec->files.buflen);
1994	rec->files.offset = 0;
1995
1996	rec->diagnostics.buflen = BUFLEN;
1997	rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
1998	rec->diagnostics.offset = 0;
1999
2000	rec->errors.buflen = BUFLEN;
2001	rec->errors.data = emalloc(rec->errors.buflen);
2002	rec->errors.offset = 0;
2003}
2004
2005/*
2006 * free_secbuffs--
2007 *  This function should be called at the end, when all the pages have been
2008 *  parsed.
2009 *  It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
2010 */
2011static void
2012free_secbuffs(mandb_rec *rec)
2013{
2014	free(rec->desc.data);
2015	free(rec->lib.data);
2016	free(rec->return_vals.data);
2017	free(rec->exit_status.data);
2018	free(rec->env.data);
2019	free(rec->files.data);
2020	free(rec->diagnostics.data);
2021	free(rec->errors.data);
2022}
2023
2024static void
2025replace_hyph(char *str)
2026{
2027	char *iter = str;
2028	while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
2029		*iter = '-';
2030
2031	iter = str;
2032	while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
2033		*iter = '-';
2034}
2035
2036static char *
2037parse_escape(const char *str)
2038{
2039	const char *backslash, *last_backslash;
2040	char *result, *iter;
2041	size_t len;
2042
2043	assert(str);
2044
2045	last_backslash = str;
2046	backslash = strchr(str, '\\');
2047	if (backslash == NULL) {
2048		result = estrdup(str);
2049		replace_hyph(result);
2050		return result;
2051	}
2052
2053	result = emalloc(strlen(str) + 1);
2054	iter = result;
2055
2056	do {
2057		len = backslash - last_backslash;
2058		memcpy(iter, last_backslash, len);
2059		iter += len;
2060		if (backslash[1] == '-' || backslash[1] == ' ') {
2061			*iter++ = backslash[1];
2062			last_backslash = backslash + 2;
2063			backslash = strchr(last_backslash, '\\');
2064		} else {
2065			++backslash;
2066			mandoc_escape(&backslash, NULL, NULL);
2067			last_backslash = backslash;
2068			if (backslash == NULL)
2069				break;
2070			backslash = strchr(last_backslash, '\\');
2071		}
2072	} while (backslash != NULL);
2073	if (last_backslash != NULL)
2074		strcpy(iter, last_backslash);
2075
2076	replace_hyph(result);
2077	return result;
2078}
2079
2080/*
2081 * append--
2082 *  Concatenates a space and src at the end of sbuff->data (much like concat in
2083 *  apropos-utils.c).
2084 *  Rather than reallocating space for writing data, it uses the value of the
2085 *  offset field of sec_buff to write new data at the free space left in the
2086 *  buffer.
2087 *  In case the size of the data to be appended exceeds the number of bytes left
2088 *  in the buffer, it reallocates buflen number of bytes and then continues.
2089 *  Value of offset field should be adjusted as new data is written.
2090 *
2091 *  NOTE: This function does not write the null byte at the end of the buffers,
2092 *  write a null byte at the position pointed to by offset before inserting data
2093 *  in the db.
2094 */
2095static void
2096append(secbuff *sbuff, const char *src)
2097{
2098	short flag = 0;
2099	size_t srclen, newlen;
2100	char *temp;
2101
2102	assert(src != NULL);
2103	temp = parse_escape(src);
2104	srclen = strlen(temp);
2105
2106	if (sbuff->data == NULL) {
2107		sbuff->data = emalloc(sbuff->buflen);
2108		sbuff->offset = 0;
2109	}
2110
2111	newlen = sbuff->offset + srclen + 2;
2112	if (newlen >= sbuff->buflen) {
2113		while (sbuff->buflen < newlen)
2114			sbuff->buflen += sbuff->buflen;
2115		sbuff->data = erealloc(sbuff->data, sbuff->buflen);
2116		flag = 1;
2117	}
2118
2119	/* Append a space at the end of the buffer. */
2120	if (sbuff->offset || flag)
2121		sbuff->data[sbuff->offset++] = ' ';
2122	/* Now, copy src at the end of the buffer. */
2123	memcpy(sbuff->data + sbuff->offset, temp, srclen);
2124	sbuff->offset += srclen;
2125	free(temp);
2126}
2127
2128static void
2129usage(void)
2130{
2131	fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
2132	exit(1);
2133}
2134