makemandb.c revision 1.52 
1/*	$NetBSD: makemandb.c,v 1.52 2017/05/01 06:43:56 abhinav Exp $	*/
2/*
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/cdefs.h>
20__RCSID("$NetBSD: makemandb.c,v 1.52 2017/05/01 06:43:56 abhinav Exp $");
21
22#include <sys/stat.h>
23#include <sys/types.h>
24
25#include <assert.h>
26#include <dirent.h>
27#include <err.h>
28#include <archive.h>
29#include <libgen.h>
30#include <md5.h>
31#include <stdio.h>
32#include <stdlib.h>
33#include <string.h>
34#include <unistd.h>
35#include <util.h>
36
37#include "apropos-utils.h"
38#include "dist/man.h"
39#include "dist/mandoc.h"
40#include "dist/mdoc.h"
41#include "dist/roff.h"
42
43#define BUFLEN 1024
44#define MDOC 0	//If the page is of mdoc(7) type
45#define MAN 1	//If the page  is of man(7) type
46
47/*
48 * A data structure for holding section specific data.
49 */
50typedef struct secbuff {
51	char *data;
52	size_t buflen;	//Total length of buffer allocated initially
53	size_t offset;	// Current offset in the buffer.
54} secbuff;
55
56typedef struct makemandb_flags {
57	int optimize;
58	int limit;	// limit the indexing to only NAME section
59	int recreate;	// Database was created from scratch
60	int verbosity;	// 0: quiet, 1: default, 2: verbose
61} makemandb_flags;
62
63typedef struct roff_mandb_rec {
64	/* Fields for mandb table */
65	char *name;	// for storing the name of the man page
66	char *name_desc; // for storing the one line description (.Nd)
67	secbuff desc; // for storing the DESCRIPTION section
68	secbuff lib; // for the LIBRARY section
69	secbuff return_vals; // RETURN VALUES
70	secbuff env; // ENVIRONMENT
71	secbuff files; // FILES
72	secbuff exit_status; // EXIT STATUS
73	secbuff diagnostics; // DIAGNOSTICS
74	secbuff errors; // ERRORS
75	char *section;
76
77	int xr_found; // To track whether a .Xr was seen when parsing a section
78
79	/* Fields for mandb_meta table */
80	char *md5_hash;
81	dev_t device;
82	ino_t inode;
83	time_t mtime;
84
85	/* Fields for mandb_links table */
86	char *machine;
87	char *links; //all the links to a page in a space separated form
88	char *file_path;
89
90	/* Non-db fields */
91	int page_type; //Indicates the type of page: mdoc or man
92} mandb_rec;
93
94typedef	void (*proff_nf)(const struct roff_node *n, mandb_rec *);
95
96static void append(secbuff *sbuff, const char *src);
97static void init_secbuffs(mandb_rec *);
98static void free_secbuffs(mandb_rec *);
99static int check_md5(const char *, sqlite3 *, char **, void *, size_t);
100static void cleanup(mandb_rec *);
101static void set_section(const struct roff_man *, mandb_rec *);
102static void set_machine(const struct roff_man *, mandb_rec *);
103static int insert_into_db(sqlite3 *, mandb_rec *);
104static	void begin_parse(const char *, struct mparse *, mandb_rec *,
105			 const void *, size_t len);
106static void proff_node(const struct roff_node *, mandb_rec *, const proff_nf *);
107static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
108static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
109static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
110static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
111static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
112static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
113static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
114static void pman_parse_node(const struct roff_node *, secbuff *);
115static void pman_parse_name(const struct roff_node *, mandb_rec *);
116static void pman_sh(const struct roff_node *, mandb_rec *);
117static void pman_block(const struct roff_node *, mandb_rec *);
118static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
119static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
120static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
121static void build_file_cache(sqlite3 *, const char *, const char *,
122			     struct stat *);
123static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
124__dead static void usage(void);
125static void optimize(sqlite3 *);
126static char *parse_escape(const char *);
127static void replace_hyph(char *);
128static makemandb_flags mflags = { .verbosity = 1 };
129
130static	const proff_nf mdocs[MDOC_MAX + 1] = {
131	NULL, /* Ap */
132	NULL, /* Dd */
133	NULL, /* Dt */
134	NULL, /* Os */
135
136	pmdoc_Sh, /* Sh */
137	NULL, /* Ss */
138	pmdoc_Pp, /* Pp */
139	NULL, /* D1 */
140
141	NULL, /* Dl */
142	NULL, /* Bd */
143	NULL, /* Ed */
144	NULL, /* Bl */
145
146	NULL, /* El */
147	NULL, /* It */
148	NULL, /* Ad */
149	NULL, /* An */
150
151	NULL, /* Ar */
152	NULL, /* Cd */
153	NULL, /* Cm */
154	NULL, /* Dv */
155
156	NULL, /* Er */
157	NULL, /* Ev */
158	NULL, /* Ex */
159	NULL, /* Fa */
160
161	NULL, /* Fd */
162	NULL, /* Fl */
163	NULL, /* Fn */
164	NULL, /* Ft */
165
166	NULL, /* Ic */
167	NULL, /* In */
168	NULL, /* Li */
169	pmdoc_Nd, /* Nd */
170
171	pmdoc_Nm, /* Nm */
172	NULL, /* Op */
173	NULL, /* Ot */
174	NULL, /* Pa */
175
176	NULL, /* Rv */
177	NULL, /* St */
178	NULL, /* Va */
179	NULL, /* Vt */
180
181	pmdoc_Xr, /* Xr */
182	NULL, /* %A */
183	NULL, /* %B */
184	NULL, /* %D */
185
186	NULL, /* %I */
187	NULL, /* %J */
188	NULL, /* %N */
189	NULL, /* %O */
190
191	NULL, /* %P */
192	NULL, /* %R */
193	NULL, /* %T */
194	NULL, /* %V */
195
196	NULL, /* Ac */
197	NULL, /* Ao */
198	NULL, /* Aq */
199	NULL, /* At */
200
201	NULL, /* Bc */
202	NULL, /* Bf */
203	NULL, /* Bo */
204	NULL, /* Bq */
205
206	NULL, /* Bsx */
207	NULL, /* Bx */
208	NULL, /* Db */
209	NULL, /* Dc */
210
211	NULL, /* Do */
212	NULL, /* Dq */
213	NULL, /* Ec */
214	NULL, /* Ef */
215
216	NULL, /* Em */
217	NULL, /* Eo */
218	NULL, /* Fx */
219	NULL, /* Ms */
220
221	NULL, /* No */
222	NULL, /* Ns */
223	NULL, /* Nx */
224	NULL, /* Ox */
225
226	NULL, /* Pc */
227	NULL, /* Pf */
228	NULL, /* Po */
229	NULL, /* Pq */
230
231	NULL, /* Qc */
232	NULL, /* Ql */
233	NULL, /* Qo */
234	NULL, /* Qq */
235
236	NULL, /* Re */
237	NULL, /* Rs */
238	NULL, /* Sc */
239	NULL, /* So */
240
241	NULL, /* Sq */
242	NULL, /* Sm */
243	NULL, /* Sx */
244	NULL, /* Sy */
245
246	NULL, /* Tn */
247	NULL, /* Ux */
248	NULL, /* Xc */
249	NULL, /* Xo */
250
251	NULL, /* Fo */
252	NULL, /* Fc */
253	NULL, /* Oo */
254	NULL, /* Oc */
255
256	NULL, /* Bk */
257	NULL, /* Ek */
258	NULL, /* Bt */
259	NULL, /* Hf */
260
261	NULL, /* Fr */
262	NULL, /* Ud */
263	NULL, /* Lb */
264	NULL, /* Lp */
265
266	NULL, /* Lk */
267	NULL, /* Mt */
268	NULL, /* Brq */
269	NULL, /* Bro */
270
271	NULL, /* Brc */
272	NULL, /* %C */
273	NULL, /* Es */
274	NULL, /* En */
275
276	NULL, /* Dx */
277	NULL, /* %Q */
278	NULL, /* br */
279	NULL, /* sp */
280
281	NULL, /* %U */
282	NULL, /* Ta */
283	NULL, /* ll */
284	NULL, /* text */
285};
286
287static	const proff_nf mans[MAN_MAX] = {
288	NULL,	//br
289	NULL,	//TH
290	pman_sh, //SH
291	NULL,	//SS
292	NULL,	//TP
293	NULL,	//LP
294	NULL,	//PP
295	NULL,	//P
296	NULL,	//IP
297	NULL,	//HP
298	NULL,	//SM
299	NULL,	//SB
300	NULL,	//BI
301	NULL,	//IB
302	NULL,	//BR
303	NULL,	//RB
304	NULL,	//R
305	pman_block,	//B
306	NULL,	//I
307	NULL,	//IR
308	NULL,	//RI
309	NULL,	//sp
310	NULL,	//nf
311	NULL,	//fi
312	NULL,	//RE
313	NULL,	//RS
314	NULL,	//DT
315	NULL,	//UC
316	NULL,	//PD
317	NULL,	//AT
318	NULL,	//in
319	NULL,	//ft
320	NULL,	//OP
321	NULL,	//EX
322	NULL,	//EE
323	NULL,	//UR
324	NULL,	//UE
325	NULL,	//ll
326};
327
328
329int
330main(int argc, char *argv[])
331{
332	FILE *file;
333	const char *sqlstr, *manconf = NULL;
334	char *line, *command;
335	char *errmsg;
336	int ch;
337	struct mparse *mp;
338	sqlite3 *db;
339	ssize_t len;
340	size_t linesize;
341	struct roff_mandb_rec rec;
342
343	while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
344		switch (ch) {
345		case 'C':
346			manconf = optarg;
347			break;
348		case 'f':
349			mflags.recreate = 1;
350			break;
351		case 'l':
352			mflags.limit = 1;
353			break;
354		case 'o':
355			mflags.optimize = 1;
356			break;
357		case 'Q':
358			mflags.verbosity = 0;
359			break;
360		case 'q':
361			mflags.verbosity = 1;
362			break;
363		case 'v':
364			mflags.verbosity = 2;
365			break;
366		default:
367			usage();
368		}
369	}
370
371	memset(&rec, 0, sizeof(rec));
372
373	init_secbuffs(&rec);
374	mchars_alloc();
375	mp = mparse_alloc(0, MANDOCLEVEL_BADARG, NULL, NULL);
376
377	if (manconf) {
378		char *arg;
379		size_t command_len = shquote(manconf, NULL, 0) + 1;
380		arg = emalloc(command_len);
381		shquote(manconf, arg, command_len);
382		easprintf(&command, "man -p -C %s", arg);
383		free(arg);
384	} else {
385		command = estrdup("man -p");
386		manconf = MANCONF;
387	}
388
389	if (mflags.recreate) {
390		char *dbp = get_dbpath(manconf);
391		/* No error here, it will fail in init_db in the same call */
392		if (dbp != NULL)
393			remove(dbp);
394	}
395
396	if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
397		exit(EXIT_FAILURE);
398
399	sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, 	&errmsg);
400	if (errmsg != NULL) {
401		warnx("%s", errmsg);
402		free(errmsg);
403		close_db(db);
404		exit(EXIT_FAILURE);
405	}
406
407	sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
408	    &errmsg);
409	if (errmsg != NULL) {
410		warnx("%s", errmsg);
411		free(errmsg);
412		close_db(db);
413		exit(EXIT_FAILURE);
414	}
415
416
417	/* Call man -p to get the list of man page dirs */
418	if ((file = popen(command, "r")) == NULL) {
419		close_db(db);
420		err(EXIT_FAILURE, "fopen failed");
421	}
422	free(command);
423
424	/* Begin the transaction for indexing the pages	*/
425	sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
426	if (errmsg != NULL) {
427		warnx("%s", errmsg);
428		free(errmsg);
429		close_db(db);
430		exit(EXIT_FAILURE);
431	}
432
433	sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
434		 " file PRIMARY KEY);"
435		 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
436		 " ON file_cache (device, inode)";
437
438	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
439	if (errmsg != NULL) {
440		warnx("%s", errmsg);
441		free(errmsg);
442		close_db(db);
443		exit(EXIT_FAILURE);
444	}
445
446	if (mflags.verbosity)
447		printf("Building temporary file cache\n");
448	line = NULL;
449	linesize = 0;
450	while ((len = getline(&line, &linesize, file)) != -1) {
451		/* Replace the new line character at the end of string with '\0' */
452		line[len - 1] = '\0';
453		char *pdir = dirname(line);
454		/* Traverse the man page directories and parse the pages */
455		traversedir(pdir, line, db, mp);
456	}
457	free(line);
458
459	if (pclose(file) == -1) {
460		close_db(db);
461		cleanup(&rec);
462		free_secbuffs(&rec);
463		err(EXIT_FAILURE, "pclose error");
464	}
465
466	if (mflags.verbosity)
467		printf("Performing index update\n");
468	update_db(db, mp, &rec);
469	mparse_free(mp);
470	mchars_free();
471	free_secbuffs(&rec);
472
473	/* Commit the transaction */
474	sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
475	if (errmsg != NULL) {
476		warnx("%s", errmsg);
477		free(errmsg);
478		close_db(db);
479		exit(EXIT_FAILURE);
480	}
481
482	if (mflags.optimize)
483		optimize(db);
484
485	close_db(db);
486	return 0;
487}
488
489/*
490 * traversedir --
491 *  Traverses the given directory recursively and passes all the man page files
492 *  in the way to build_file_cache()
493 */
494static void
495traversedir(const char *parent, const char *file, sqlite3 *db,
496            struct mparse *mp)
497{
498	struct stat sb;
499	struct dirent *dirp;
500	DIR *dp;
501	char *buf;
502
503	if (stat(file, &sb) < 0) {
504		if (mflags.verbosity)
505			warn("stat failed: %s", file);
506		return;
507	}
508
509	/* If it is a directory, traverse it recursively */
510	if (S_ISDIR(sb.st_mode)) {
511		if ((dp = opendir(file)) == NULL) {
512			if (mflags.verbosity)
513				warn("opendir error: %s", file);
514			return;
515		}
516
517		while ((dirp = readdir(dp)) != NULL) {
518			/* Avoid . and .. entries in a directory */
519			if (dirp->d_name[0] != '.') {
520				easprintf(&buf, "%s/%s", file, dirp->d_name);
521				traversedir(parent, buf, db, mp);
522				free(buf);
523			}
524		}
525		closedir(dp);
526		return;
527	}
528
529	if (!S_ISREG(sb.st_mode))
530		return;
531
532	if (sb.st_size == 0) {
533		if (mflags.verbosity)
534			warnx("Empty file: %s", file);
535		return;
536	}
537	build_file_cache(db, parent, file, &sb);
538}
539
540/* build_file_cache --
541 *   This function generates an md5 hash of the file passed as its 2nd parameter
542 *   and stores it in a temporary table file_cache along with the full file path.
543 *   This is done to support incremental updation of the database.
544 *   The temporary table file_cache is dropped thereafter in the function
545 *   update_db(), once the database has been updated.
546 */
547static void
548build_file_cache(sqlite3 *db, const char *parent, const char *file,
549		 struct stat *sb)
550{
551	const char *sqlstr;
552	sqlite3_stmt *stmt = NULL;
553	int rc, idx;
554	assert(file != NULL);
555	dev_t device_cache = sb->st_dev;
556	ino_t inode_cache = sb->st_ino;
557	time_t mtime_cache = sb->st_mtime;
558
559	sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
560		 " :mtime, :parent, :file)";
561	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
562	if (rc != SQLITE_OK) {
563		if (mflags.verbosity)
564			warnx("%s", sqlite3_errmsg(db));
565		return;
566	}
567
568	idx = sqlite3_bind_parameter_index(stmt, ":device");
569	rc = sqlite3_bind_int64(stmt, idx, device_cache);
570	if (rc != SQLITE_OK) {
571		if (mflags.verbosity)
572			warnx("%s", sqlite3_errmsg(db));
573		sqlite3_finalize(stmt);
574		return;
575	}
576
577	idx = sqlite3_bind_parameter_index(stmt, ":inode");
578	rc = sqlite3_bind_int64(stmt, idx, inode_cache);
579	if (rc != SQLITE_OK) {
580		if (mflags.verbosity)
581			warnx("%s", sqlite3_errmsg(db));
582		sqlite3_finalize(stmt);
583		return;
584	}
585
586	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
587	rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
588	if (rc != SQLITE_OK) {
589		if (mflags.verbosity)
590			warnx("%s", sqlite3_errmsg(db));
591		sqlite3_finalize(stmt);
592		return;
593	}
594
595	idx = sqlite3_bind_parameter_index(stmt, ":parent");
596	rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
597	if (rc != SQLITE_OK) {
598		if (mflags.verbosity)
599			warnx("%s", sqlite3_errmsg(db));
600		sqlite3_finalize(stmt);
601		return;
602	}
603
604	idx = sqlite3_bind_parameter_index(stmt, ":file");
605	rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
606	if (rc != SQLITE_OK) {
607		if (mflags.verbosity)
608			warnx("%s", sqlite3_errmsg(db));
609		sqlite3_finalize(stmt);
610		return;
611	}
612
613	sqlite3_step(stmt);
614	sqlite3_finalize(stmt);
615}
616
617static void
618update_existing_entry(sqlite3 *db, const char *file, const char *hash,
619    mandb_rec *rec, int *new_count, int *link_count, int *err_count)
620{
621	int update_count, rc, idx;
622	const char *inner_sqlstr;
623	sqlite3_stmt *inner_stmt;
624
625	update_count = sqlite3_total_changes(db);
626	inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
627		       " inode = :inode, mtime = :mtime WHERE"
628		       " md5_hash = :md5 AND file = :file AND"
629		       " (device <> :device2 OR inode <> "
630		       "  :inode2 OR mtime <> :mtime2)";
631	rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
632	if (rc != SQLITE_OK) {
633		if (mflags.verbosity)
634			warnx("%s", sqlite3_errmsg(db));
635		return;
636	}
637	idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
638	sqlite3_bind_int64(inner_stmt, idx, rec->device);
639	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
640	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
641	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
642	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
643	idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
644	sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
645	idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
646	sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
647	idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
648	sqlite3_bind_int64(inner_stmt, idx, rec->device);
649	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
650	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
651	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
652	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
653
654	rc = sqlite3_step(inner_stmt);
655	if (rc == SQLITE_DONE) {
656		/* Check if an update has been performed. */
657		if (update_count != sqlite3_total_changes(db)) {
658			if (mflags.verbosity == 2)
659				printf("Updated %s\n", file);
660			(*new_count)++;
661		} else {
662			/* Otherwise it was a hardlink. */
663			(*link_count)++;
664		}
665	} else {
666		if (mflags.verbosity == 2)
667			warnx("Could not update the meta data for %s", file);
668		(*err_count)++;
669	}
670	sqlite3_finalize(inner_stmt);
671}
672
673/* read_and_decompress --
674 *	Reads the given file into memory. If it is compressed, decompress
675 *	it before returning to the caller.
676 */
677static int
678read_and_decompress(const char *file, void **bufp, size_t *len)
679{
680	size_t off;
681	ssize_t r;
682	struct archive *a;
683	struct archive_entry *ae;
684	char *buf;
685
686	if ((a = archive_read_new()) == NULL)
687		errx(EXIT_FAILURE, "memory allocation failed");
688
689	*bufp = NULL;
690	if (archive_read_support_filter_all(a) != ARCHIVE_OK ||
691	    archive_read_support_format_raw(a) != ARCHIVE_OK ||
692	    archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
693	    archive_read_next_header(a, &ae) != ARCHIVE_OK)
694		goto archive_error;
695	*len = 65536;
696	buf = emalloc(*len);
697	off = 0;
698	for (;;) {
699		r = archive_read_data(a, buf + off, *len - off);
700		if (r == ARCHIVE_OK) {
701			archive_read_free(a);
702			*bufp = buf;
703			*len = off;
704			return 0;
705		}
706		if (r <= 0) {
707			free(buf);
708			break;
709		}
710		off += r;
711		if (off == *len) {
712			*len *= 2;
713			if (*len < off) {
714				if (mflags.verbosity)
715					warnx("File too large: %s", file);
716				free(buf);
717				archive_read_free(a);
718				return -1;
719			}
720			buf = erealloc(buf, *len);
721		}
722	}
723
724archive_error:
725	warnx("Error while reading `%s': %s", file, archive_error_string(a));
726	archive_read_free(a);
727	return -1;
728}
729
730/* update_db --
731 *	Does an incremental updation of the database by checking the file_cache.
732 *	It parses and adds the pages which are present in file_cache,
733 *	but not in the database.
734 *	It also removes the pages which are present in the databse,
735 *	but not in the file_cache.
736 */
737static void
738update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
739{
740	const char *sqlstr;
741	sqlite3_stmt *stmt = NULL;
742	char *file;
743	char *parent;
744	char *errmsg = NULL;
745	char *md5sum;
746	void *buf;
747	size_t buflen;
748	struct sql_row {
749		struct sql_row *next;
750		dev_t device;
751		ino_t inode;
752		time_t mtime;
753		char *parent;
754		char *file;
755	} *rows, *row;
756	int new_count = 0;	/* Counter for newly indexed/updated pages */
757	int total_count = 0;	/* Counter for total number of pages */
758	int err_count = 0;	/* Counter for number of failed pages */
759	int link_count = 0;	/* Counter for number of hard/sym links */
760	int md5_status;
761	int rc;
762
763	sqlstr = "SELECT device, inode, mtime, parent, file"
764	         " FROM metadb.file_cache fc"
765	         " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
766	         "  device = fc.device AND inode = fc.inode AND "
767	         "  mtime = fc.mtime AND file = fc.file)";
768
769	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
770	if (rc != SQLITE_OK) {
771		if (mflags.verbosity)
772			warnx("%s", sqlite3_errmsg(db));
773		close_db(db);
774		errx(EXIT_FAILURE, "Could not query file cache");
775	}
776
777	buf = NULL;
778	rows = NULL;
779	while (sqlite3_step(stmt) == SQLITE_ROW) {
780		row = emalloc(sizeof(struct sql_row));
781		row->device = sqlite3_column_int64(stmt, 0);
782		row->inode = sqlite3_column_int64(stmt, 1);
783		row->mtime = sqlite3_column_int64(stmt, 2);
784		row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
785		row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
786		row->next = rows;
787		rows = row;
788		total_count++;
789	}
790	sqlite3_finalize(stmt);
791
792	for ( ; rows != NULL; free(parent), free(file), free(buf)) {
793		row = rows;
794		rows = rows->next;
795
796		rec->device = row->device;
797		rec->inode = row->inode;
798		rec->mtime = row->mtime;
799		parent = row->parent;
800		file = row->file;
801		free(row);
802
803		if (read_and_decompress(file, &buf, &buflen)) {
804			err_count++;
805			continue;
806		}
807		md5_status = check_md5(file, db, &md5sum, buf, buflen);
808		assert(md5sum != NULL);
809		if (md5_status == -1) {
810			if (mflags.verbosity)
811				warnx("An error occurred in checking md5 value"
812			      " for file %s", file);
813			err_count++;
814			continue;
815		}
816
817		if (md5_status == 0) {
818			/*
819			 * The MD5 hash is already present in the database,
820			 * so simply update the metadata.
821			 */
822			update_existing_entry(db, file, md5sum, rec,
823			    &new_count, &link_count, &err_count);
824			free(md5sum);
825			continue;
826		}
827
828		if (md5_status == 1) {
829			/*
830			 * The MD5 hash was not present in the database.
831			 * This means is either a new file or an updated file.
832			 * We should go ahead with parsing.
833			 */
834			if (chdir(parent) == -1) {
835				if (mflags.verbosity)
836					warn("chdir failed for `%s', could "
837					    "not index `%s'", parent, file);
838				err_count++;
839				free(md5sum);
840				continue;
841			}
842
843			if (mflags.verbosity == 2)
844				printf("Parsing: %s\n", file);
845			rec->md5_hash = md5sum;
846			rec->file_path = estrdup(file);
847			// file_path is freed by insert_into_db itself.
848			begin_parse(file, mp, rec, buf, buflen);
849			if (insert_into_db(db, rec) < 0) {
850				if (mflags.verbosity)
851					warnx("Error in indexing `%s'", file);
852				err_count++;
853			} else {
854				new_count++;
855			}
856		}
857	}
858
859	if (mflags.verbosity == 2) {
860		printf("Number of new or updated pages encountered: %d\n"
861		    "Number of hard links found: %d\n"
862		    "Number of pages that were successfully"
863		    " indexed or updated: %d\n"
864		    "Number of pages that could not be indexed"
865		    " due to errors: %d\n",
866		    total_count - link_count, link_count, new_count, err_count);
867	}
868
869	if (mflags.recreate)
870		return;
871
872	if (mflags.verbosity == 2)
873		printf("Deleting stale index entries\n");
874
875	sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
876		 " (SELECT file FROM metadb.file_cache);"
877		 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
878		 " (SELECT md5_hash from mandb_meta);"
879		 "DROP TABLE metadb.file_cache;"
880		 "DELETE FROM mandb WHERE rowid NOT IN"
881		 " (SELECT id FROM mandb_meta);";
882
883	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
884	if (errmsg != NULL) {
885		warnx("Removing old entries failed: %s", errmsg);
886		warnx("Please rebuild database from scratch with -f.");
887		free(errmsg);
888		return;
889	}
890}
891
892/*
893 * begin_parse --
894 *  parses the man page using libmandoc
895 */
896static void
897begin_parse(const char *file, struct mparse *mp, mandb_rec *rec,
898    const void *buf, size_t len)
899{
900	struct roff_man *roff;
901	mparse_reset(mp);
902
903	rec->xr_found = 0;
904
905	if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_BADARG) {
906		/* Printing this warning at verbosity level 2
907		 * because some packages from pkgsrc might trigger several
908		 * of such warnings.
909		 */
910		if (mflags.verbosity == 2)
911			warnx("%s: Parse failure", file);
912		return;
913	}
914
915	mparse_result(mp, &roff, NULL);
916	if (roff == NULL) {
917		if (mflags.verbosity == 2)
918			warnx("Not a roff(7) page");
919		return;
920	}
921
922	if (roff->macroset == MACROSET_MDOC) {
923		mdoc_validate(roff);
924		rec->page_type = MDOC;
925		proff_node(roff->first->child, rec, mdocs);
926	} else if (roff->macroset == MACROSET_MAN) {
927		man_validate(roff);
928		rec->page_type = MAN;
929		proff_node(roff->first->child, rec, mans);
930	} else
931		warnx("Unknown macroset %d", roff->macroset);
932	set_machine(roff, rec);
933	set_section(roff, rec);
934}
935
936/*
937 * set_section --
938 *  Extracts the section number and normalizes it to only the numeric part
939 *  (Which should be the first character of the string).
940 */
941static void
942set_section(const struct roff_man *rm, mandb_rec *rec)
943{
944	if (!rm)
945		return;
946	const struct roff_meta *rm_meta = &rm->meta;
947	const char *s = rm_meta->msec == NULL ? "?" : rm_meta->msec;
948	easprintf(&rec->section, "%s", s);
949	if (rec->section[0] == '?' && mflags.verbosity == 2)
950		warnx("%s: Missing section number", rec->file_path);
951}
952
953/*
954 * get_machine --
955 *  Extracts the machine architecture information if available.
956 */
957static void
958set_machine(const struct roff_man *rm, mandb_rec *rec)
959{
960	if (rm == NULL)
961		return;
962	const struct roff_meta *rm_meta = &rm->meta;
963	if (rm_meta->arch)
964		rec->machine = estrdup(rm_meta->arch);
965}
966
967/*
968 * pmdoc_Nm --
969 *  Extracts the Name of the manual page from the .Nm macro
970 */
971static void
972pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
973{
974	if (n->sec != SEC_NAME)
975		return;
976
977	for (n = n->child; n; n = n->next) {
978		if (n->type == ROFFT_TEXT) {
979			char *escaped_name = parse_escape(n->string);
980			concat(&rec->name, escaped_name);
981			free(escaped_name);
982		}
983	}
984}
985
986/*
987 * pmdoc_Nd --
988 *  Extracts the one line description of the man page from the .Nd macro
989 */
990static void
991pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
992{
993	if (n->type == ROFFT_BODY)
994		deroff(&rec->name_desc, n);
995	if (rec->name_desc)
996		replace_hyph(rec->name_desc);
997
998}
999
1000/*
1001 * pmdoc_macro_handler--
1002 *  This function is a single point of handling all the special macros that we
1003 *  want to handle especially. For example the .Xr macro for properly parsing
1004 *  the referenced page name along with the section number, or the .Pp macro
1005 *  for adding a new line whenever we encounter it.
1006 */
1007static void
1008pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
1009{
1010	const struct roff_node *sn;
1011	assert(n);
1012
1013	switch (doct) {
1014	/*  Parse the man page references.
1015	 * Basically the .Xr macros are used like:
1016	 *  .Xr ls 1
1017 	 *  and formatted like this:
1018	 *  ls(1)
1019	 *  Prepare a buffer to format the data like the above example and call
1020	 *  pmdoc_parse_section to append it.
1021	 */
1022	case MDOC_Xr:
1023		n = n->child;
1024		while (n->type != ROFFT_TEXT && n->next)
1025			n = n->next;
1026
1027		if (n && n->type != ROFFT_TEXT)
1028			return;
1029		sn = n;
1030		if (n->next)
1031			n = n->next;
1032
1033		while (n->type != ROFFT_TEXT && n->next)
1034			n = n->next;
1035
1036		if (n && n->type == ROFFT_TEXT) {
1037			char *buf;
1038			easprintf(&buf, "%s(%s)", sn->string, n->string);
1039			mdoc_parse_section(n->sec, buf, rec);
1040			free(buf);
1041		}
1042
1043		break;
1044
1045	/* Parse the .Pp macro to add a new line */
1046	case MDOC_Pp:
1047		if (n->type == ROFFT_TEXT)
1048			mdoc_parse_section(n->sec, "\n", rec);
1049		break;
1050	default:
1051		break;
1052	}
1053
1054}
1055
1056/*
1057 * pmdoc_Xr, pmdoc_Pp--
1058 *  Empty stubs.
1059 *  The parser calls these functions each time it encounters
1060 *  a .Xr or .Pp macro. We are parsing all the data from
1061 *  the pmdoc_Sh function, so don't do anything here.
1062 *  (See if else blocks in pmdoc_Sh.)
1063 */
1064static void
1065pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
1066{
1067}
1068
1069static void
1070pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
1071{
1072}
1073
1074/*
1075 * pmdoc_Sh --
1076 *  Called when a .Sh macro is encountered and tries to parse its body
1077 */
1078static void
1079pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
1080{
1081	if (n == NULL)
1082		return;
1083
1084	switch (n->sec) {
1085	case SEC_NAME:
1086	case SEC_SYNOPSIS:
1087	case SEC_EXAMPLES:
1088	case SEC_STANDARDS:
1089	case SEC_HISTORY:
1090	case SEC_AUTHORS:
1091	case SEC_BUGS:
1092		/*
1093		 * We don't care about text from these sections
1094		 */
1095		return;
1096	default:
1097		break;
1098	}
1099
1100	if (n->type == ROFFT_BLOCK)
1101		mdoc_parse_Sh(n->body, rec);
1102}
1103
1104/*
1105 *  Called from pmdoc_Sh to parse body of a .Sh macro. It calls
1106 *  mdoc_parse_section to append the data to the section specific buffer.
1107 *  The .Xr macro needs special handling, thus the separate if branch for it.
1108 */
1109static void
1110mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
1111{
1112	if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
1113		return;
1114	int xr_found = 0;
1115
1116	if (n->type == ROFFT_TEXT) {
1117		mdoc_parse_section(n->sec, n->string, rec);
1118	} else if (mdocs[n->tok] == pmdoc_Xr) {
1119		/*
1120		 * When encountering other inline macros,
1121		 * call pmdoc_macro_handler.
1122		 */
1123		pmdoc_macro_handler(n, rec, MDOC_Xr);
1124		xr_found = 1;
1125	} else if (mdocs[n->tok] == pmdoc_Pp) {
1126		pmdoc_macro_handler(n, rec, MDOC_Pp);
1127	}
1128
1129	/*
1130	 * If an Xr macro was encountered then the child node has
1131	 * already been explored by pmdoc_macro_handler.
1132	 */
1133	if (xr_found == 0)
1134		mdoc_parse_Sh(n->child, rec);
1135	mdoc_parse_Sh(n->next, rec);
1136}
1137
1138/*
1139 * mdoc_parse_section--
1140 *  Utility function for parsing sections of the mdoc type pages.
1141 *  Takes two params:
1142 *   1. sec is an enum which indicates the section in which we are present
1143 *   2. string is the string which we need to append to the secbuff for this
1144 *      particular section.
1145 *  The function appends string to the global section buffer and returns.
1146 */
1147static void
1148mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
1149{
1150	/*
1151	 * If the user specified the 'l' flag, then parse and store only the
1152	 * NAME section. Ignore the rest.
1153	 */
1154	if (mflags.limit)
1155		return;
1156
1157	switch (sec) {
1158	case SEC_LIBRARY:
1159		append(&rec->lib, string);
1160		break;
1161	case SEC_RETURN_VALUES:
1162		append(&rec->return_vals, string);
1163		break;
1164	case SEC_ENVIRONMENT:
1165		append(&rec->env, string);
1166		break;
1167	case SEC_FILES:
1168		append(&rec->files, string);
1169		break;
1170	case SEC_EXIT_STATUS:
1171		append(&rec->exit_status, string);
1172		break;
1173	case SEC_DIAGNOSTICS:
1174		append(&rec->diagnostics, string);
1175		break;
1176	case SEC_ERRORS:
1177		append(&rec->errors, string);
1178		break;
1179	default:
1180		append(&rec->desc, string);
1181		break;
1182	}
1183}
1184
1185static void
1186proff_node(const struct roff_node *n, mandb_rec *rec, const proff_nf *func)
1187{
1188	if (n == NULL)
1189		return;
1190
1191	switch (n->type) {
1192	case (ROFFT_BODY):
1193		/* FALLTHROUGH */
1194	case (ROFFT_BLOCK):
1195		/* FALLTHROUGH */
1196	case (ROFFT_ELEM):
1197		if (func[n->tok] != NULL)
1198			(*func[n->tok])(n, rec);
1199		break;
1200	default:
1201		break;
1202	}
1203
1204	proff_node(n->child, rec, func);
1205	proff_node(n->next, rec, func);
1206}
1207
1208/*
1209 * pman_parse_name --
1210 *  Parses the NAME section and puts the complete content in the name_desc
1211 *  variable.
1212 */
1213static void
1214pman_parse_name(const struct roff_node *n, mandb_rec *rec)
1215{
1216	if (n == NULL)
1217		return;
1218
1219	if (n->type == ROFFT_TEXT) {
1220		char *tmp = parse_escape(n->string);
1221		concat(&rec->name_desc, tmp);
1222		free(tmp);
1223	}
1224
1225	if (n->child)
1226		pman_parse_name(n->child, rec);
1227
1228	if(n->next)
1229		pman_parse_name(n->next, rec);
1230}
1231
1232/*
1233 * A stub function to be able to parse the macros like .B embedded inside
1234 * a section.
1235 */
1236static void
1237pman_block(const struct roff_node *n, mandb_rec *rec)
1238{
1239}
1240
1241/*
1242 * pman_sh --
1243 * This function does one of the two things:
1244 *  1. If the present section is NAME, then it will:
1245 *    (a) Extract the name of the page (in case of multiple comma separated
1246 *        names, it will pick up the first one).
1247 *    (b) Build a space spearated list of all the symlinks/hardlinks to
1248 *        this page and store in the buffer 'links'. These are extracted from
1249 *        the comma separated list of names in the NAME section as well.
1250 *    (c) Move on to the one line description section, which is after the list
1251 *        of names in the NAME section.
1252 *  2. Otherwise, it will check the section name and call the man_parse_section
1253 *     function, passing the enum corresponding to that section.
1254 */
1255static void
1256pman_sh(const struct roff_node *n, mandb_rec *rec)
1257{
1258	static const struct {
1259		enum man_sec section;
1260		const char *header;
1261	} mapping[] = {
1262	    { MANSEC_DESCRIPTION, "DESCRIPTION" },
1263	    { MANSEC_SYNOPSIS, "SYNOPSIS" },
1264	    { MANSEC_LIBRARY, "LIBRARY" },
1265	    { MANSEC_ERRORS, "ERRORS" },
1266	    { MANSEC_FILES, "FILES" },
1267	    { MANSEC_RETURN_VALUES, "RETURN VALUE" },
1268	    { MANSEC_RETURN_VALUES, "RETURN VALUES" },
1269	    { MANSEC_EXIT_STATUS, "EXIT STATUS" },
1270	    { MANSEC_EXAMPLES, "EXAMPLES" },
1271	    { MANSEC_EXAMPLES, "EXAMPLE" },
1272	    { MANSEC_STANDARDS, "STANDARDS" },
1273	    { MANSEC_HISTORY, "HISTORY" },
1274	    { MANSEC_BUGS, "BUGS" },
1275	    { MANSEC_AUTHORS, "AUTHORS" },
1276	    { MANSEC_COPYRIGHT, "COPYRIGHT" },
1277	};
1278	const struct roff_node *head;
1279	char *name_desc;
1280	size_t sz;
1281	size_t i;
1282
1283	if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
1284	    head->type != ROFFT_TEXT)
1285		return;
1286
1287	/*
1288	 * Check if this section should be extracted and
1289	 * where it should be stored. Handled the trival cases first.
1290	 */
1291	for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
1292		if (strcmp(head->string, mapping[i].header) == 0) {
1293			man_parse_section(mapping[i].section, n, rec);
1294			return;
1295		}
1296	}
1297
1298	if (strcmp(head->string, "NAME") == 0) {
1299		/*
1300		 * We are in the NAME section.
1301		 * pman_parse_name will put the complete content in name_desc.
1302		 */
1303		pman_parse_name(n, rec);
1304
1305		name_desc = rec->name_desc;
1306		if (name_desc == NULL)
1307			return;
1308
1309		/* Remove any leading spaces. */
1310		while (name_desc[0] == ' ')
1311			name_desc++;
1312
1313		/* If the line begins with a "\&", avoid those */
1314		if (name_desc[0] == '\\' && name_desc[1] == '&')
1315			name_desc += 2;
1316
1317		/* Now name_desc should be left with a comma-space
1318		 * separated list of names and the one line description
1319		 * of the page:
1320		 *     "a, b, c \- sample description"
1321		 * Take out the first name, before the first comma
1322		 * (or space) and store it in rec->name.
1323		 * If the page has aliases then they should be
1324		 * in the form of a comma separated list.
1325		 * Keep looping while there is a comma in name_desc,
1326		 * extract the alias name and store in rec->links.
1327		 * When there are no more commas left, break out.
1328		 */
1329		int has_alias = 0;	// Any more aliases left?
1330		while (*name_desc) {
1331			/* Remove any leading spaces or hyphens. */
1332			if (name_desc[0] == ' ' || name_desc[0] == '-') {
1333				name_desc++;
1334				continue;
1335			}
1336			sz = strcspn(name_desc, ", ");
1337
1338			/* Extract the first term and store it in rec->name. */
1339			if (rec->name == NULL) {
1340				if (name_desc[sz] == ',')
1341					has_alias = 1;
1342				rec->name = estrndup(name_desc, sz);
1343				/* XXX This would only happen with a poorly
1344				 * written man page, maybe warn? */
1345				if (name_desc[sz] == '\0')
1346					break;
1347				name_desc += sz + 1;
1348				continue;
1349			}
1350
1351			/*
1352			 * Once rec->name is set, rest of the names
1353			 * are to be treated as links or aliases.
1354			 */
1355			if (rec->name && has_alias) {
1356				if (name_desc[sz] != ',') {
1357					/* No more commas left --> no more
1358					 * aliases to take out */
1359					has_alias = 0;
1360				}
1361				concat2(&rec->links, name_desc, sz);
1362				/* XXX This would only happen with a poorly
1363				 * written man page, maybe warn? */
1364				if (name_desc[sz] == '\0')
1365					break;
1366				name_desc += sz + 1;
1367				continue;
1368			}
1369			break;
1370		}
1371
1372		/* Parse any escape sequences that might be there */
1373		char *temp = parse_escape(name_desc);
1374		free(rec->name_desc);
1375		rec->name_desc = temp;
1376		temp = parse_escape(rec->name);
1377		free(rec->name);
1378		rec->name = temp;
1379		return;
1380	}
1381
1382	/* The RETURN VALUE section might be specified in multiple ways */
1383	if (strcmp(head->string, "RETURN") == 0 &&
1384	    head->next != NULL && head->next->type == ROFFT_TEXT &&
1385	    (strcmp(head->next->string, "VALUE") == 0 ||
1386	    strcmp(head->next->string, "VALUES") == 0)) {
1387		man_parse_section(MANSEC_RETURN_VALUES, n, rec);
1388		return;
1389	}
1390
1391	/*
1392	 * EXIT STATUS section can also be specified all on one line or on two
1393	 * separate lines.
1394	 */
1395	if (strcmp(head->string, "EXIT") == 0 &&
1396	    head->next != NULL && head->next->type == ROFFT_TEXT &&
1397	    strcmp(head->next->string, "STATUS") == 0) {
1398		man_parse_section(MANSEC_EXIT_STATUS, n, rec);
1399		return;
1400	}
1401
1402	/* Store the rest of the content in desc. */
1403	man_parse_section(MANSEC_NONE, n, rec);
1404}
1405
1406/*
1407 * pman_parse_node --
1408 *  Generic function to iterate through a node. Usually called from
1409 *  man_parse_section to parse a particular section of the man page.
1410 */
1411static void
1412pman_parse_node(const struct roff_node *n, secbuff *s)
1413{
1414	if (n == NULL)
1415		return;
1416
1417	if (n->type == ROFFT_TEXT)
1418		append(s, n->string);
1419
1420	pman_parse_node(n->child, s);
1421	pman_parse_node(n->next, s);
1422}
1423
1424/*
1425 * man_parse_section --
1426 *  Takes two parameters:
1427 *   sec: Tells which section we are present in
1428 *   n: Is the present node of the AST.
1429 * Depending on the section, we call pman_parse_node to parse that section and
1430 * concatenate the content from that section into the buffer for that section.
1431 */
1432static void
1433man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
1434{
1435	/*
1436	 * If the user sepecified the 'l' flag then just parse
1437	 * the NAME section, ignore the rest.
1438	 */
1439	if (mflags.limit)
1440		return;
1441
1442	switch (sec) {
1443	case MANSEC_LIBRARY:
1444		pman_parse_node(n, &rec->lib);
1445		break;
1446	case MANSEC_RETURN_VALUES:
1447		pman_parse_node(n, &rec->return_vals);
1448		break;
1449	case MANSEC_ENVIRONMENT:
1450		pman_parse_node(n, &rec->env);
1451		break;
1452	case MANSEC_FILES:
1453		pman_parse_node(n, &rec->files);
1454		break;
1455	case MANSEC_EXIT_STATUS:
1456		pman_parse_node(n, &rec->exit_status);
1457		break;
1458	case MANSEC_DIAGNOSTICS:
1459		pman_parse_node(n, &rec->diagnostics);
1460		break;
1461	case MANSEC_ERRORS:
1462		pman_parse_node(n, &rec->errors);
1463		break;
1464	case MANSEC_NAME:
1465	case MANSEC_SYNOPSIS:
1466	case MANSEC_EXAMPLES:
1467	case MANSEC_STANDARDS:
1468	case MANSEC_HISTORY:
1469	case MANSEC_BUGS:
1470	case MANSEC_AUTHORS:
1471	case MANSEC_COPYRIGHT:
1472		break;
1473	default:
1474		pman_parse_node(n, &rec->desc);
1475		break;
1476	}
1477
1478}
1479
1480/*
1481 * insert_into_db --
1482 *  Inserts the parsed data of the man page in the Sqlite databse.
1483 *  If any of the values is NULL, then we cleanup and return -1 indicating
1484 *  an error.
1485 *  Otherwise, store the data in the database and return 0.
1486 */
1487static int
1488insert_into_db(sqlite3 *db, mandb_rec *rec)
1489{
1490	int rc = 0;
1491	int idx = -1;
1492	const char *sqlstr = NULL;
1493	sqlite3_stmt *stmt = NULL;
1494	char *ln = NULL;
1495	char *errmsg = NULL;
1496	long int mandb_rowid;
1497
1498	/*
1499	 * At the very minimum we want to make sure that we store
1500	 * the following data:
1501	 *   Name, one line description, and the MD5 hash
1502	 */
1503	if (rec->name == NULL || rec->name_desc == NULL ||
1504	    rec->md5_hash == NULL) {
1505		cleanup(rec);
1506		return -1;
1507	}
1508
1509	/* Write null byte at the end of all the sec_buffs */
1510	rec->desc.data[rec->desc.offset] = 0;
1511	rec->lib.data[rec->lib.offset] = 0;
1512	rec->env.data[rec->env.offset] = 0;
1513	rec->return_vals.data[rec->return_vals.offset] = 0;
1514	rec->exit_status.data[rec->exit_status.offset] = 0;
1515	rec->files.data[rec->files.offset] = 0;
1516	rec->diagnostics.data[rec->diagnostics.offset] = 0;
1517	rec->errors.data[rec->errors.offset] = 0;
1518
1519	/*
1520	 * In case of a mdoc page: (sorry, no better place to put this code)
1521	 * parse the comma separated list of names of man pages,
1522	 * the first name will be stored in the mandb table, rest will be
1523	 * treated as links and put in the mandb_links table.
1524	 */
1525	if (rec->page_type == MDOC) {
1526		char *tmp;
1527		rec->links = estrdup(rec->name);
1528		free(rec->name);
1529		size_t sz = strcspn(rec->links, " \0");
1530		rec->name = emalloc(sz + 1);
1531		memcpy(rec->name, rec->links, sz);
1532		if(rec->name[sz - 1] == ',')
1533			rec->name[sz - 1] = 0;
1534		else
1535			rec->name[sz] = 0;
1536		while (rec->links[sz] == ' ')
1537			++sz;
1538		tmp = estrdup(rec->links + sz);
1539		free(rec->links);
1540		rec->links = tmp;
1541	}
1542
1543/*------------------------ Populate the mandb table---------------------------*/
1544	sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
1545		 " :lib, :return_vals, :env, :files, :exit_status,"
1546		 " :diagnostics, :errors, :md5_hash, :machine)";
1547
1548	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1549	if (rc != SQLITE_OK)
1550		goto Out;
1551
1552	idx = sqlite3_bind_parameter_index(stmt, ":name");
1553	rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
1554	if (rc != SQLITE_OK) {
1555		sqlite3_finalize(stmt);
1556		goto Out;
1557	}
1558
1559	idx = sqlite3_bind_parameter_index(stmt, ":section");
1560	rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
1561	if (rc != SQLITE_OK) {
1562		sqlite3_finalize(stmt);
1563		goto Out;
1564	}
1565
1566	idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
1567	rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
1568	if (rc != SQLITE_OK) {
1569		sqlite3_finalize(stmt);
1570		goto Out;
1571	}
1572
1573	idx = sqlite3_bind_parameter_index(stmt, ":desc");
1574	rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
1575	                       rec->desc.offset + 1, NULL);
1576	if (rc != SQLITE_OK) {
1577		sqlite3_finalize(stmt);
1578		goto Out;
1579	}
1580
1581	idx = sqlite3_bind_parameter_index(stmt, ":lib");
1582	rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
1583	    rec->lib.offset + 1, NULL);
1584	if (rc != SQLITE_OK) {
1585		sqlite3_finalize(stmt);
1586		goto Out;
1587	}
1588
1589	idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
1590	rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
1591	                      rec->return_vals.offset + 1, NULL);
1592	if (rc != SQLITE_OK) {
1593		sqlite3_finalize(stmt);
1594		goto Out;
1595	}
1596
1597	idx = sqlite3_bind_parameter_index(stmt, ":env");
1598	rc = sqlite3_bind_text(stmt, idx, rec->env.data,
1599	    rec->env.offset + 1, NULL);
1600	if (rc != SQLITE_OK) {
1601		sqlite3_finalize(stmt);
1602		goto Out;
1603	}
1604
1605	idx = sqlite3_bind_parameter_index(stmt, ":files");
1606	rc = sqlite3_bind_text(stmt, idx, rec->files.data,
1607	                       rec->files.offset + 1, NULL);
1608	if (rc != SQLITE_OK) {
1609		sqlite3_finalize(stmt);
1610		goto Out;
1611	}
1612
1613	idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
1614	rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
1615	                       rec->exit_status.offset + 1, NULL);
1616	if (rc != SQLITE_OK) {
1617		sqlite3_finalize(stmt);
1618		goto Out;
1619	}
1620
1621	idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
1622	rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
1623	                       rec->diagnostics.offset + 1, NULL);
1624	if (rc != SQLITE_OK) {
1625		sqlite3_finalize(stmt);
1626		goto Out;
1627	}
1628
1629	idx = sqlite3_bind_parameter_index(stmt, ":errors");
1630	rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
1631	                       rec->errors.offset + 1, NULL);
1632	if (rc != SQLITE_OK) {
1633		sqlite3_finalize(stmt);
1634		goto Out;
1635	}
1636
1637	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1638	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1639	if (rc != SQLITE_OK) {
1640		sqlite3_finalize(stmt);
1641		goto Out;
1642	}
1643
1644	idx = sqlite3_bind_parameter_index(stmt, ":machine");
1645	if (rec->machine)
1646		rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
1647	else
1648		rc = sqlite3_bind_null(stmt, idx);
1649	if (rc != SQLITE_OK) {
1650		sqlite3_finalize(stmt);
1651		goto Out;
1652	}
1653
1654	rc = sqlite3_step(stmt);
1655	if (rc != SQLITE_DONE) {
1656		sqlite3_finalize(stmt);
1657		goto Out;
1658	}
1659
1660	sqlite3_finalize(stmt);
1661
1662	/* Get the row id of the last inserted row */
1663	mandb_rowid = sqlite3_last_insert_rowid(db);
1664
1665/*------------------------Populate the mandb_meta table-----------------------*/
1666	sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
1667		 " :file, :md5_hash, :id)";
1668	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1669	if (rc != SQLITE_OK)
1670		goto Out;
1671
1672	idx = sqlite3_bind_parameter_index(stmt, ":device");
1673	rc = sqlite3_bind_int64(stmt, idx, rec->device);
1674	if (rc != SQLITE_OK) {
1675		sqlite3_finalize(stmt);
1676		goto Out;
1677	}
1678
1679	idx = sqlite3_bind_parameter_index(stmt, ":inode");
1680	rc = sqlite3_bind_int64(stmt, idx, rec->inode);
1681	if (rc != SQLITE_OK) {
1682		sqlite3_finalize(stmt);
1683		goto Out;
1684	}
1685
1686	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1687	rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
1688	if (rc != SQLITE_OK) {
1689		sqlite3_finalize(stmt);
1690		goto Out;
1691	}
1692
1693	idx = sqlite3_bind_parameter_index(stmt, ":file");
1694	rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1695	if (rc != SQLITE_OK) {
1696		sqlite3_finalize(stmt);
1697		goto Out;
1698	}
1699
1700	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1701	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1702	if (rc != SQLITE_OK) {
1703		sqlite3_finalize(stmt);
1704		goto Out;
1705	}
1706
1707	idx = sqlite3_bind_parameter_index(stmt, ":id");
1708	rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
1709	if (rc != SQLITE_OK) {
1710		sqlite3_finalize(stmt);
1711		goto Out;
1712	}
1713
1714	rc = sqlite3_step(stmt);
1715	sqlite3_finalize(stmt);
1716	if (rc == SQLITE_CONSTRAINT_UNIQUE) {
1717		/* The *most* probable reason for reaching here is that
1718		 * the UNIQUE contraint on the file column of the mandb_meta
1719		 * table was violated.
1720		 * This can happen when a file was updated/modified.
1721		 * To fix this we need to do two things:
1722		 * 1. Delete the row for the older version of this file
1723		 *    from mandb table.
1724		 * 2. Run an UPDATE query to update the row for this file
1725		 *    in the mandb_meta table.
1726		 */
1727		warnx("Trying to update index for %s", rec->file_path);
1728		char *sql = sqlite3_mprintf("DELETE FROM mandb "
1729					    "WHERE rowid = (SELECT id"
1730					    "  FROM mandb_meta"
1731					    "  WHERE file = %Q)",
1732					    rec->file_path);
1733		sqlite3_exec(db, sql, NULL, NULL, &errmsg);
1734		sqlite3_free(sql);
1735		if (errmsg != NULL) {
1736			if (mflags.verbosity)
1737				warnx("%s", errmsg);
1738			free(errmsg);
1739		}
1740		sqlstr = "UPDATE mandb_meta SET device = :device,"
1741			 " inode = :inode, mtime = :mtime, id = :id,"
1742			 " md5_hash = :md5 WHERE file = :file";
1743		rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1744		if (rc != SQLITE_OK) {
1745			if (mflags.verbosity)
1746				warnx("Update failed with error: %s",
1747			    sqlite3_errmsg(db));
1748			close_db(db);
1749			cleanup(rec);
1750			errx(EXIT_FAILURE,
1751			    "Consider running makemandb with -f option");
1752		}
1753
1754		idx = sqlite3_bind_parameter_index(stmt, ":device");
1755		sqlite3_bind_int64(stmt, idx, rec->device);
1756		idx = sqlite3_bind_parameter_index(stmt, ":inode");
1757		sqlite3_bind_int64(stmt, idx, rec->inode);
1758		idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1759		sqlite3_bind_int64(stmt, idx, rec->mtime);
1760		idx = sqlite3_bind_parameter_index(stmt, ":id");
1761		sqlite3_bind_int64(stmt, idx, mandb_rowid);
1762		idx = sqlite3_bind_parameter_index(stmt, ":md5");
1763		sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1764		idx = sqlite3_bind_parameter_index(stmt, ":file");
1765		sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1766		rc = sqlite3_step(stmt);
1767		sqlite3_finalize(stmt);
1768
1769		if (rc != SQLITE_DONE) {
1770			if (mflags.verbosity)
1771				warnx("%s", sqlite3_errmsg(db));
1772			close_db(db);
1773			cleanup(rec);
1774			errx(EXIT_FAILURE,
1775			    "Consider running makemandb with -f option");
1776		}
1777	} else if (rc != SQLITE_DONE) {
1778		/* Otherwise make this error fatal */
1779		warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
1780		cleanup(rec);
1781		close_db(db);
1782		exit(EXIT_FAILURE);
1783	}
1784
1785/*------------------------ Populate the mandb_links table---------------------*/
1786	char *str = NULL;
1787	char *links;
1788	if (rec->links && strlen(rec->links)) {
1789		links = rec->links;
1790		for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
1791			if (ln[0] == ',')
1792				ln++;
1793			if(ln[strlen(ln) - 1] == ',')
1794				ln[strlen(ln) - 1] = 0;
1795
1796			str = sqlite3_mprintf("INSERT INTO mandb_links"
1797					      " VALUES (%Q, %Q, %Q, %Q, %Q)",
1798					      ln, rec->name, rec->section,
1799					      rec->machine, rec->md5_hash);
1800			sqlite3_exec(db, str, NULL, NULL, &errmsg);
1801			sqlite3_free(str);
1802			if (errmsg != NULL) {
1803				warnx("%s", errmsg);
1804				cleanup(rec);
1805				free(errmsg);
1806				return -1;
1807			}
1808		}
1809	}
1810
1811	cleanup(rec);
1812	return 0;
1813
1814  Out:
1815	if (mflags.verbosity)
1816		warnx("%s", sqlite3_errmsg(db));
1817	cleanup(rec);
1818	return -1;
1819}
1820
1821/*
1822 * check_md5--
1823 *  Generates the md5 hash of the file and checks if it already doesn't exist
1824 *  in the table.
1825 *  This function is being used to avoid hardlinks.
1826 *  On successful completion it will also set the value of the fourth parameter
1827 *  to the md5 hash of the file (computed previously). It is the responsibility
1828 *  of the caller to free this buffer.
1829 *  Return values:
1830 *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
1831 *  0: If the md5 hash does not exist in the table.
1832 *  1: If the hash exists in the database.
1833 */
1834static int
1835check_md5(const char *file, sqlite3 *db, char **md5sum, void *buf, size_t buflen)
1836{
1837	int rc = 0;
1838	int idx = -1;
1839	char *sqlstr = NULL;
1840	sqlite3_stmt *stmt = NULL;
1841
1842	assert(file != NULL);
1843	*md5sum = MD5Data(buf, buflen, NULL);
1844	if (*md5sum == NULL) {
1845		if (mflags.verbosity)
1846			warn("md5 failed: %s", file);
1847		return -1;
1848	}
1849
1850	easprintf(&sqlstr, "SELECT * FROM mandb_meta WHERE md5_hash = :md5_hash");
1851	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1852	if (rc != SQLITE_OK) {
1853		free(sqlstr);
1854		free(*md5sum);
1855		*md5sum = NULL;
1856		return -1;
1857	}
1858
1859	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1860	rc = sqlite3_bind_text(stmt, idx, *md5sum, -1, NULL);
1861	if (rc != SQLITE_OK) {
1862		if (mflags.verbosity)
1863			warnx("%s", sqlite3_errmsg(db));
1864		sqlite3_finalize(stmt);
1865		free(sqlstr);
1866		free(*md5sum);
1867		*md5sum = NULL;
1868		return -1;
1869	}
1870
1871	if (sqlite3_step(stmt) == SQLITE_ROW) {
1872		sqlite3_finalize(stmt);
1873		free(sqlstr);
1874		return 0;
1875	}
1876
1877	sqlite3_finalize(stmt);
1878	free(sqlstr);
1879	return 1;
1880}
1881
1882/* Optimize the index for faster search */
1883static void
1884optimize(sqlite3 *db)
1885{
1886	const char *sqlstr;
1887	char *errmsg = NULL;
1888
1889	if (mflags.verbosity == 2)
1890		printf("Optimizing the database index\n");
1891	sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
1892		 "VACUUM";
1893	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
1894	if (errmsg != NULL) {
1895		if (mflags.verbosity)
1896			warnx("%s", errmsg);
1897		free(errmsg);
1898		return;
1899	}
1900}
1901
1902/*
1903 * cleanup --
1904 *  cleans up the global buffers
1905 */
1906static void
1907cleanup(mandb_rec *rec)
1908{
1909	rec->desc.offset = 0;
1910	rec->lib.offset = 0;
1911	rec->return_vals.offset = 0;
1912	rec->env.offset = 0;
1913	rec->exit_status.offset = 0;
1914	rec->diagnostics.offset = 0;
1915	rec->errors.offset = 0;
1916	rec->files.offset = 0;
1917
1918	free(rec->machine);
1919	rec->machine = NULL;
1920
1921	free(rec->links);
1922	rec->links = NULL;
1923
1924	free(rec->file_path);
1925	rec->file_path = NULL;
1926
1927	free(rec->name);
1928	rec->name = NULL;
1929
1930	free(rec->name_desc);
1931	rec->name_desc = NULL;
1932
1933	free(rec->md5_hash);
1934	rec->md5_hash = NULL;
1935
1936	free(rec->section);
1937	rec->section = NULL;
1938}
1939
1940/*
1941 * init_secbuffs--
1942 *  Sets the value of buflen for all the sec_buff field of rec. And then
1943 *  allocate memory to each sec_buff member of rec.
1944 */
1945static void
1946init_secbuffs(mandb_rec *rec)
1947{
1948	/*
1949	 * Some sec_buff might need more memory, for example desc,
1950	 * which stores the data of the DESCRIPTION section,
1951	 * while some might need very small amount of memory.
1952	 * Therefore explicitly setting the value of buflen field for
1953	 * each sec_buff.
1954	 */
1955	rec->desc.buflen = 10 * BUFLEN;
1956	rec->desc.data = emalloc(rec->desc.buflen);
1957	rec->desc.offset = 0;
1958
1959	rec->lib.buflen = BUFLEN / 2;
1960	rec->lib.data = emalloc(rec->lib.buflen);
1961	rec->lib.offset = 0;
1962
1963	rec->return_vals.buflen = BUFLEN;
1964	rec->return_vals.data = emalloc(rec->return_vals.buflen);
1965	rec->return_vals.offset = 0;
1966
1967	rec->exit_status.buflen = BUFLEN;
1968	rec->exit_status.data = emalloc(rec->exit_status.buflen);
1969	rec->exit_status.offset = 0;
1970
1971	rec->env.buflen = BUFLEN;
1972	rec->env.data = emalloc(rec->env.buflen);
1973	rec->env.offset = 0;
1974
1975	rec->files.buflen = BUFLEN;
1976	rec->files.data = emalloc(rec->files.buflen);
1977	rec->files.offset = 0;
1978
1979	rec->diagnostics.buflen = BUFLEN;
1980	rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
1981	rec->diagnostics.offset = 0;
1982
1983	rec->errors.buflen = BUFLEN;
1984	rec->errors.data = emalloc(rec->errors.buflen);
1985	rec->errors.offset = 0;
1986}
1987
1988/*
1989 * free_secbuffs--
1990 *  This function should be called at the end, when all the pages have been
1991 *  parsed.
1992 *  It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
1993 */
1994static void
1995free_secbuffs(mandb_rec *rec)
1996{
1997	free(rec->desc.data);
1998	free(rec->lib.data);
1999	free(rec->return_vals.data);
2000	free(rec->exit_status.data);
2001	free(rec->env.data);
2002	free(rec->files.data);
2003	free(rec->diagnostics.data);
2004	free(rec->errors.data);
2005}
2006
2007static void
2008replace_hyph(char *str)
2009{
2010	char *iter = str;
2011	while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
2012		*iter = '-';
2013
2014	iter = str;
2015	while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
2016		*iter = '-';
2017}
2018
2019static char *
2020parse_escape(const char *str)
2021{
2022	const char *backslash, *last_backslash;
2023	char *result, *iter;
2024	size_t len;
2025
2026	assert(str);
2027
2028	last_backslash = str;
2029	backslash = strchr(str, '\\');
2030	if (backslash == NULL) {
2031		result = estrdup(str);
2032		replace_hyph(result);
2033		return result;
2034	}
2035
2036	result = emalloc(strlen(str) + 1);
2037	iter = result;
2038
2039	do {
2040		len = backslash - last_backslash;
2041		memcpy(iter, last_backslash, len);
2042		iter += len;
2043		if (backslash[1] == '-' || backslash[1] == ' ') {
2044			*iter++ = backslash[1];
2045			last_backslash = backslash + 2;
2046			backslash = strchr(last_backslash, '\\');
2047		} else {
2048			++backslash;
2049			mandoc_escape(&backslash, NULL, NULL);
2050			last_backslash = backslash;
2051			if (backslash == NULL)
2052				break;
2053			backslash = strchr(last_backslash, '\\');
2054		}
2055	} while (backslash != NULL);
2056	if (last_backslash != NULL)
2057		strcpy(iter, last_backslash);
2058
2059	replace_hyph(result);
2060	return result;
2061}
2062
2063/*
2064 * append--
2065 *  Concatenates a space and src at the end of sbuff->data (much like concat in
2066 *  apropos-utils.c).
2067 *  Rather than reallocating space for writing data, it uses the value of the
2068 *  offset field of sec_buff to write new data at the free space left in the
2069 *  buffer.
2070 *  In case the size of the data to be appended exceeds the number of bytes left
2071 *  in the buffer, it reallocates buflen number of bytes and then continues.
2072 *  Value of offset field should be adjusted as new data is written.
2073 *
2074 *  NOTE: This function does not write the null byte at the end of the buffers,
2075 *  write a null byte at the position pointed to by offset before inserting data
2076 *  in the db.
2077 */
2078static void
2079append(secbuff *sbuff, const char *src)
2080{
2081	short flag = 0;
2082	size_t srclen, newlen;
2083	char *temp;
2084
2085	assert(src != NULL);
2086	temp = parse_escape(src);
2087	srclen = strlen(temp);
2088
2089	if (sbuff->data == NULL) {
2090		sbuff->data = emalloc(sbuff->buflen);
2091		sbuff->offset = 0;
2092	}
2093
2094	newlen = sbuff->offset + srclen + 2;
2095	if (newlen >= sbuff->buflen) {
2096		while (sbuff->buflen < newlen)
2097			sbuff->buflen += sbuff->buflen;
2098		sbuff->data = erealloc(sbuff->data, sbuff->buflen);
2099		flag = 1;
2100	}
2101
2102	/* Append a space at the end of the buffer. */
2103	if (sbuff->offset || flag)
2104		sbuff->data[sbuff->offset++] = ' ';
2105	/* Now, copy src at the end of the buffer. */
2106	memcpy(sbuff->data + sbuff->offset, temp, srclen);
2107	sbuff->offset += srclen;
2108	free(temp);
2109}
2110
2111static void
2112usage(void)
2113{
2114	fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
2115	exit(1);
2116}
2117