makemandb.c revision 1.50 
1/*	$NetBSD: makemandb.c,v 1.50 2017/04/30 08:41:18 abhinav Exp $	*/
2/*
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/cdefs.h>
20__RCSID("$NetBSD: makemandb.c,v 1.50 2017/04/30 08:41:18 abhinav Exp $");
21
22#include <sys/stat.h>
23#include <sys/types.h>
24
25#include <assert.h>
26#include <dirent.h>
27#include <err.h>
28#include <archive.h>
29#include <libgen.h>
30#include <md5.h>
31#include <stdio.h>
32#include <stdlib.h>
33#include <string.h>
34#include <unistd.h>
35#include <util.h>
36
37#include "apropos-utils.h"
38#include "dist/man.h"
39#include "dist/mandoc.h"
40#include "dist/mdoc.h"
41#include "dist/roff.h"
42
43#define BUFLEN 1024
44#define MDOC 0	//If the page is of mdoc(7) type
45#define MAN 1	//If the page  is of man(7) type
46
47/*
48 * A data structure for holding section specific data.
49 */
50typedef struct secbuff {
51	char *data;
52	size_t buflen;	//Total length of buffer allocated initially
53	size_t offset;	// Current offset in the buffer.
54} secbuff;
55
56typedef struct makemandb_flags {
57	int optimize;
58	int limit;	// limit the indexing to only NAME section
59	int recreate;	// Database was created from scratch
60	int verbosity;	// 0: quiet, 1: default, 2: verbose
61} makemandb_flags;
62
63typedef struct roff_mandb_rec {
64	/* Fields for mandb table */
65	char *name;	// for storing the name of the man page
66	char *name_desc; // for storing the one line description (.Nd)
67	secbuff desc; // for storing the DESCRIPTION section
68	secbuff lib; // for the LIBRARY section
69	secbuff return_vals; // RETURN VALUES
70	secbuff env; // ENVIRONMENT
71	secbuff files; // FILES
72	secbuff exit_status; // EXIT STATUS
73	secbuff diagnostics; // DIAGNOSTICS
74	secbuff errors; // ERRORS
75	char *section;
76
77	int xr_found; // To track whether a .Xr was seen when parsing a section
78
79	/* Fields for mandb_meta table */
80	char *md5_hash;
81	dev_t device;
82	ino_t inode;
83	time_t mtime;
84
85	/* Fields for mandb_links table */
86	char *machine;
87	char *links; //all the links to a page in a space separated form
88	char *file_path;
89
90	/* Non-db fields */
91	int page_type; //Indicates the type of page: mdoc or man
92} mandb_rec;
93
94typedef	void (*proff_nf)(const struct roff_node *n, mandb_rec *);
95
96static void append(secbuff *sbuff, const char *src);
97static void init_secbuffs(mandb_rec *);
98static void free_secbuffs(mandb_rec *);
99static int check_md5(const char *, sqlite3 *, const char *, char **, void *, size_t);
100static void cleanup(mandb_rec *);
101static void set_section(const struct roff_man *, mandb_rec *);
102static void set_machine(const struct roff_man *, mandb_rec *);
103static int insert_into_db(sqlite3 *, mandb_rec *);
104static	void begin_parse(const char *, struct mparse *, mandb_rec *,
105			 const void *, size_t len);
106static void proff_node(const struct roff_node *, mandb_rec *, const proff_nf *);
107static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
108static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
109static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
110static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
111static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
112static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
113static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
114static void pman_parse_node(const struct roff_node *, secbuff *);
115static void pman_parse_name(const struct roff_node *, mandb_rec *);
116static void pman_sh(const struct roff_node *, mandb_rec *);
117static void pman_block(const struct roff_node *, mandb_rec *);
118static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
119static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
120static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
121static void build_file_cache(sqlite3 *, const char *, const char *,
122			     struct stat *);
123static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
124__dead static void usage(void);
125static void optimize(sqlite3 *);
126static char *parse_escape(const char *);
127static void replace_hyph(char *);
128static makemandb_flags mflags = { .verbosity = 1 };
129
130static	const proff_nf mdocs[MDOC_MAX + 1] = {
131	NULL, /* Ap */
132	NULL, /* Dd */
133	NULL, /* Dt */
134	NULL, /* Os */
135
136	pmdoc_Sh, /* Sh */
137	NULL, /* Ss */
138	pmdoc_Pp, /* Pp */
139	NULL, /* D1 */
140
141	NULL, /* Dl */
142	NULL, /* Bd */
143	NULL, /* Ed */
144	NULL, /* Bl */
145
146	NULL, /* El */
147	NULL, /* It */
148	NULL, /* Ad */
149	NULL, /* An */
150
151	NULL, /* Ar */
152	NULL, /* Cd */
153	NULL, /* Cm */
154	NULL, /* Dv */
155
156	NULL, /* Er */
157	NULL, /* Ev */
158	NULL, /* Ex */
159	NULL, /* Fa */
160
161	NULL, /* Fd */
162	NULL, /* Fl */
163	NULL, /* Fn */
164	NULL, /* Ft */
165
166	NULL, /* Ic */
167	NULL, /* In */
168	NULL, /* Li */
169	pmdoc_Nd, /* Nd */
170
171	pmdoc_Nm, /* Nm */
172	NULL, /* Op */
173	NULL, /* Ot */
174	NULL, /* Pa */
175
176	NULL, /* Rv */
177	NULL, /* St */
178	NULL, /* Va */
179	NULL, /* Vt */
180
181	pmdoc_Xr, /* Xr */
182	NULL, /* %A */
183	NULL, /* %B */
184	NULL, /* %D */
185
186	NULL, /* %I */
187	NULL, /* %J */
188	NULL, /* %N */
189	NULL, /* %O */
190
191	NULL, /* %P */
192	NULL, /* %R */
193	NULL, /* %T */
194	NULL, /* %V */
195
196	NULL, /* Ac */
197	NULL, /* Ao */
198	NULL, /* Aq */
199	NULL, /* At */
200
201	NULL, /* Bc */
202	NULL, /* Bf */
203	NULL, /* Bo */
204	NULL, /* Bq */
205
206	NULL, /* Bsx */
207	NULL, /* Bx */
208	NULL, /* Db */
209	NULL, /* Dc */
210
211	NULL, /* Do */
212	NULL, /* Dq */
213	NULL, /* Ec */
214	NULL, /* Ef */
215
216	NULL, /* Em */
217	NULL, /* Eo */
218	NULL, /* Fx */
219	NULL, /* Ms */
220
221	NULL, /* No */
222	NULL, /* Ns */
223	NULL, /* Nx */
224	NULL, /* Ox */
225
226	NULL, /* Pc */
227	NULL, /* Pf */
228	NULL, /* Po */
229	NULL, /* Pq */
230
231	NULL, /* Qc */
232	NULL, /* Ql */
233	NULL, /* Qo */
234	NULL, /* Qq */
235
236	NULL, /* Re */
237	NULL, /* Rs */
238	NULL, /* Sc */
239	NULL, /* So */
240
241	NULL, /* Sq */
242	NULL, /* Sm */
243	NULL, /* Sx */
244	NULL, /* Sy */
245
246	NULL, /* Tn */
247	NULL, /* Ux */
248	NULL, /* Xc */
249	NULL, /* Xo */
250
251	NULL, /* Fo */
252	NULL, /* Fc */
253	NULL, /* Oo */
254	NULL, /* Oc */
255
256	NULL, /* Bk */
257	NULL, /* Ek */
258	NULL, /* Bt */
259	NULL, /* Hf */
260
261	NULL, /* Fr */
262	NULL, /* Ud */
263	NULL, /* Lb */
264	NULL, /* Lp */
265
266	NULL, /* Lk */
267	NULL, /* Mt */
268	NULL, /* Brq */
269	NULL, /* Bro */
270
271	NULL, /* Brc */
272	NULL, /* %C */
273	NULL, /* Es */
274	NULL, /* En */
275
276	NULL, /* Dx */
277	NULL, /* %Q */
278	NULL, /* br */
279	NULL, /* sp */
280
281	NULL, /* %U */
282	NULL, /* Ta */
283	NULL, /* ll */
284	NULL, /* text */
285};
286
287static	const proff_nf mans[MAN_MAX] = {
288	NULL,	//br
289	NULL,	//TH
290	pman_sh, //SH
291	NULL,	//SS
292	NULL,	//TP
293	NULL,	//LP
294	NULL,	//PP
295	NULL,	//P
296	NULL,	//IP
297	NULL,	//HP
298	NULL,	//SM
299	NULL,	//SB
300	NULL,	//BI
301	NULL,	//IB
302	NULL,	//BR
303	NULL,	//RB
304	NULL,	//R
305	pman_block,	//B
306	NULL,	//I
307	NULL,	//IR
308	NULL,	//RI
309	NULL,	//sp
310	NULL,	//nf
311	NULL,	//fi
312	NULL,	//RE
313	NULL,	//RS
314	NULL,	//DT
315	NULL,	//UC
316	NULL,	//PD
317	NULL,	//AT
318	NULL,	//in
319	NULL,	//ft
320	NULL,	//OP
321	NULL,	//EX
322	NULL,	//EE
323	NULL,	//UR
324	NULL,	//UE
325	NULL,	//ll
326};
327
328
329int
330main(int argc, char *argv[])
331{
332	FILE *file;
333	const char *sqlstr, *manconf = NULL;
334	char *line, *command, *parent;
335	char *errmsg;
336	int ch;
337	struct mparse *mp;
338	sqlite3 *db;
339	ssize_t len;
340	size_t linesize;
341	struct roff_mandb_rec rec;
342
343	while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
344		switch (ch) {
345		case 'C':
346			manconf = optarg;
347			break;
348		case 'f':
349			mflags.recreate = 1;
350			break;
351		case 'l':
352			mflags.limit = 1;
353			break;
354		case 'o':
355			mflags.optimize = 1;
356			break;
357		case 'Q':
358			mflags.verbosity = 0;
359			break;
360		case 'q':
361			mflags.verbosity = 1;
362			break;
363		case 'v':
364			mflags.verbosity = 2;
365			break;
366		default:
367			usage();
368		}
369	}
370
371	memset(&rec, 0, sizeof(rec));
372
373	init_secbuffs(&rec);
374	mchars_alloc();
375	mp = mparse_alloc(0, MANDOCLEVEL_BADARG, NULL, NULL);
376
377	if (manconf) {
378		char *arg;
379		size_t command_len = shquote(manconf, NULL, 0) + 1;
380		arg = emalloc(command_len);
381		shquote(manconf, arg, command_len);
382		easprintf(&command, "man -p -C %s", arg);
383		free(arg);
384	} else {
385		command = estrdup("man -p");
386		manconf = MANCONF;
387	}
388
389	if (mflags.recreate) {
390		char *dbp = get_dbpath(manconf);
391		/* No error here, it will fail in init_db in the same call */
392		if (dbp != NULL)
393			remove(dbp);
394	}
395
396	if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
397		exit(EXIT_FAILURE);
398
399	sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, 	&errmsg);
400	if (errmsg != NULL) {
401		warnx("%s", errmsg);
402		free(errmsg);
403		close_db(db);
404		exit(EXIT_FAILURE);
405	}
406
407	sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
408	    &errmsg);
409	if (errmsg != NULL) {
410		warnx("%s", errmsg);
411		free(errmsg);
412		close_db(db);
413		exit(EXIT_FAILURE);
414	}
415
416
417	/* Call man -p to get the list of man page dirs */
418	if ((file = popen(command, "r")) == NULL) {
419		close_db(db);
420		err(EXIT_FAILURE, "fopen failed");
421	}
422	free(command);
423
424	/* Begin the transaction for indexing the pages	*/
425	sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
426	if (errmsg != NULL) {
427		warnx("%s", errmsg);
428		free(errmsg);
429		close_db(db);
430		exit(EXIT_FAILURE);
431	}
432
433	sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
434		 " file PRIMARY KEY);"
435		 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
436		 " ON file_cache (device, inode)";
437
438	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
439	if (errmsg != NULL) {
440		warnx("%s", errmsg);
441		free(errmsg);
442		close_db(db);
443		exit(EXIT_FAILURE);
444	}
445
446	if (mflags.verbosity)
447		printf("Building temporary file cache\n");
448	line = NULL;
449	linesize = 0;
450	while ((len = getline(&line, &linesize, file)) != -1) {
451		/* Replace the new line character at the end of string with '\0' */
452		line[len - 1] = '\0';
453		parent = estrdup(line);
454		char *pdir = estrdup(dirname(parent));
455		free(parent);
456		/* Traverse the man page directories and parse the pages */
457		traversedir(pdir, line, db, mp);
458		free(pdir);
459	}
460	free(line);
461
462	if (pclose(file) == -1) {
463		close_db(db);
464		cleanup(&rec);
465		free_secbuffs(&rec);
466		err(EXIT_FAILURE, "pclose error");
467	}
468
469	if (mflags.verbosity)
470		printf("Performing index update\n");
471	update_db(db, mp, &rec);
472	mparse_free(mp);
473	mchars_free();
474	free_secbuffs(&rec);
475
476	/* Commit the transaction */
477	sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
478	if (errmsg != NULL) {
479		warnx("%s", errmsg);
480		free(errmsg);
481		close_db(db);
482		exit(EXIT_FAILURE);
483	}
484
485	if (mflags.optimize)
486		optimize(db);
487
488	close_db(db);
489	return 0;
490}
491
492/*
493 * traversedir --
494 *  Traverses the given directory recursively and passes all the man page files
495 *  in the way to build_file_cache()
496 */
497static void
498traversedir(const char *parent, const char *file, sqlite3 *db,
499            struct mparse *mp)
500{
501	struct stat sb;
502	struct dirent *dirp;
503	DIR *dp;
504	char *buf;
505
506	if (stat(file, &sb) < 0) {
507		if (mflags.verbosity)
508			warn("stat failed: %s", file);
509		return;
510	}
511
512	/* If it is a directory, traverse it recursively */
513	if (S_ISDIR(sb.st_mode)) {
514		if ((dp = opendir(file)) == NULL) {
515			if (mflags.verbosity)
516				warn("opendir error: %s", file);
517			return;
518		}
519
520		while ((dirp = readdir(dp)) != NULL) {
521			/* Avoid . and .. entries in a directory */
522			if (dirp->d_name[0] != '.') {
523				easprintf(&buf, "%s/%s", file, dirp->d_name);
524				traversedir(parent, buf, db, mp);
525				free(buf);
526			}
527		}
528		closedir(dp);
529		return;
530	}
531
532	if (!S_ISREG(sb.st_mode))
533		return;
534
535	if (sb.st_size == 0) {
536		if (mflags.verbosity)
537			warnx("Empty file: %s", file);
538		return;
539	}
540	build_file_cache(db, parent, file, &sb);
541}
542
543/* build_file_cache --
544 *   This function generates an md5 hash of the file passed as its 2nd parameter
545 *   and stores it in a temporary table file_cache along with the full file path.
546 *   This is done to support incremental updation of the database.
547 *   The temporary table file_cache is dropped thereafter in the function
548 *   update_db(), once the database has been updated.
549 */
550static void
551build_file_cache(sqlite3 *db, const char *parent, const char *file,
552		 struct stat *sb)
553{
554	const char *sqlstr;
555	sqlite3_stmt *stmt = NULL;
556	int rc, idx;
557	assert(file != NULL);
558	dev_t device_cache = sb->st_dev;
559	ino_t inode_cache = sb->st_ino;
560	time_t mtime_cache = sb->st_mtime;
561
562	sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
563		 " :mtime, :parent, :file)";
564	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
565	if (rc != SQLITE_OK) {
566		if (mflags.verbosity)
567			warnx("%s", sqlite3_errmsg(db));
568		return;
569	}
570
571	idx = sqlite3_bind_parameter_index(stmt, ":device");
572	rc = sqlite3_bind_int64(stmt, idx, device_cache);
573	if (rc != SQLITE_OK) {
574		if (mflags.verbosity)
575			warnx("%s", sqlite3_errmsg(db));
576		sqlite3_finalize(stmt);
577		return;
578	}
579
580	idx = sqlite3_bind_parameter_index(stmt, ":inode");
581	rc = sqlite3_bind_int64(stmt, idx, inode_cache);
582	if (rc != SQLITE_OK) {
583		if (mflags.verbosity)
584			warnx("%s", sqlite3_errmsg(db));
585		sqlite3_finalize(stmt);
586		return;
587	}
588
589	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
590	rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
591	if (rc != SQLITE_OK) {
592		if (mflags.verbosity)
593			warnx("%s", sqlite3_errmsg(db));
594		sqlite3_finalize(stmt);
595		return;
596	}
597
598	idx = sqlite3_bind_parameter_index(stmt, ":parent");
599	rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
600	if (rc != SQLITE_OK) {
601		if (mflags.verbosity)
602			warnx("%s", sqlite3_errmsg(db));
603		sqlite3_finalize(stmt);
604		return;
605	}
606
607	idx = sqlite3_bind_parameter_index(stmt, ":file");
608	rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
609	if (rc != SQLITE_OK) {
610		if (mflags.verbosity)
611			warnx("%s", sqlite3_errmsg(db));
612		sqlite3_finalize(stmt);
613		return;
614	}
615
616	sqlite3_step(stmt);
617	sqlite3_finalize(stmt);
618}
619
620static void
621update_existing_entry(sqlite3 *db, const char *file, const char *hash,
622    mandb_rec *rec, int *new_count, int *link_count, int *err_count)
623{
624	int update_count, rc, idx;
625	const char *inner_sqlstr;
626	sqlite3_stmt *inner_stmt;
627
628	update_count = sqlite3_total_changes(db);
629	inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
630		       " inode = :inode, mtime = :mtime WHERE"
631		       " md5_hash = :md5 AND file = :file AND"
632		       " (device <> :device2 OR inode <> "
633		       "  :inode2 OR mtime <> :mtime2)";
634	rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
635	if (rc != SQLITE_OK) {
636		if (mflags.verbosity)
637			warnx("%s", sqlite3_errmsg(db));
638		return;
639	}
640	idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
641	sqlite3_bind_int64(inner_stmt, idx, rec->device);
642	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
643	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
644	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
645	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
646	idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
647	sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
648	idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
649	sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
650	idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
651	sqlite3_bind_int64(inner_stmt, idx, rec->device);
652	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
653	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
654	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
655	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
656
657	rc = sqlite3_step(inner_stmt);
658	if (rc == SQLITE_DONE) {
659		/* Check if an update has been performed. */
660		if (update_count != sqlite3_total_changes(db)) {
661			if (mflags.verbosity == 2)
662				printf("Updated %s\n", file);
663			(*new_count)++;
664		} else {
665			/* Otherwise it was a hardlink. */
666			(*link_count)++;
667		}
668	} else {
669		if (mflags.verbosity == 2)
670			warnx("Could not update the meta data for %s", file);
671		(*err_count)++;
672	}
673	sqlite3_finalize(inner_stmt);
674}
675
676/* read_and_decompress --
677 *	Reads the given file into memory. If it is compressed, decompress
678 *	it before returning to the caller.
679 */
680static int
681read_and_decompress(const char *file, void **bufp, size_t *len)
682{
683	size_t off;
684	ssize_t r;
685	struct archive *a;
686	struct archive_entry *ae;
687	char *buf;
688
689	if ((a = archive_read_new()) == NULL)
690		errx(EXIT_FAILURE, "memory allocation failed");
691
692	*bufp = NULL;
693	if (archive_read_support_filter_all(a) != ARCHIVE_OK ||
694	    archive_read_support_format_raw(a) != ARCHIVE_OK ||
695	    archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
696	    archive_read_next_header(a, &ae) != ARCHIVE_OK)
697		goto archive_error;
698	*len = 65536;
699	buf = emalloc(*len);
700	off = 0;
701	for (;;) {
702		r = archive_read_data(a, buf + off, *len - off);
703		if (r == ARCHIVE_OK) {
704			archive_read_free(a);
705			*bufp = buf;
706			*len = off;
707			return 0;
708		}
709		if (r <= 0) {
710			free(buf);
711			break;
712		}
713		off += r;
714		if (off == *len) {
715			*len *= 2;
716			if (*len < off) {
717				if (mflags.verbosity)
718					warnx("File too large: %s", file);
719				free(buf);
720				archive_read_free(a);
721				return -1;
722			}
723			buf = erealloc(buf, *len);
724		}
725	}
726
727archive_error:
728	warnx("Error while reading `%s': %s", file, archive_error_string(a));
729	archive_read_free(a);
730	return -1;
731}
732
733/* update_db --
734 *	Does an incremental updation of the database by checking the file_cache.
735 *	It parses and adds the pages which are present in file_cache,
736 *	but not in the database.
737 *	It also removes the pages which are present in the databse,
738 *	but not in the file_cache.
739 */
740static void
741update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
742{
743	const char *sqlstr;
744	sqlite3_stmt *stmt = NULL;
745	char *file;
746	char *parent;
747	char *errmsg = NULL;
748	char *md5sum;
749	void *buf;
750	size_t buflen;
751	struct sql_row {
752		struct sql_row *next;
753		dev_t device;
754		ino_t inode;
755		time_t mtime;
756		char *parent;
757		char *file;
758	} *rows, *row;
759	int new_count = 0;	/* Counter for newly indexed/updated pages */
760	int total_count = 0;	/* Counter for total number of pages */
761	int err_count = 0;	/* Counter for number of failed pages */
762	int link_count = 0;	/* Counter for number of hard/sym links */
763	int md5_status;
764	int rc;
765
766	sqlstr = "SELECT device, inode, mtime, parent, file"
767	         " FROM metadb.file_cache fc"
768	         " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
769	         "  device = fc.device AND inode = fc.inode AND "
770	         "  mtime = fc.mtime AND file = fc.file)";
771
772	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
773	if (rc != SQLITE_OK) {
774		if (mflags.verbosity)
775			warnx("%s", sqlite3_errmsg(db));
776		close_db(db);
777		errx(EXIT_FAILURE, "Could not query file cache");
778	}
779
780	buf = NULL;
781	rows = NULL;
782	while (sqlite3_step(stmt) == SQLITE_ROW) {
783		row = emalloc(sizeof(struct sql_row));
784		row->device = sqlite3_column_int64(stmt, 0);
785		row->inode = sqlite3_column_int64(stmt, 1);
786		row->mtime = sqlite3_column_int64(stmt, 2);
787		row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
788		row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
789		row->next = rows;
790		rows = row;
791		total_count++;
792	}
793	sqlite3_finalize(stmt);
794
795	for ( ; rows != NULL; free(parent), free(file), free(buf)) {
796		row = rows;
797		rows = rows->next;
798
799		rec->device = row->device;
800		rec->inode = row->inode;
801		rec->mtime = row->mtime;
802		parent = row->parent;
803		file = row->file;
804		free(row);
805
806		if (read_and_decompress(file, &buf, &buflen)) {
807			err_count++;
808			continue;
809		}
810		md5_status = check_md5(file, db, "mandb_meta", &md5sum, buf,
811		    buflen);
812		assert(md5sum != NULL);
813		if (md5_status == -1) {
814			if (mflags.verbosity)
815				warnx("An error occurred in checking md5 value"
816			      " for file %s", file);
817			err_count++;
818			continue;
819		}
820
821		if (md5_status == 0) {
822			/*
823			 * The MD5 hash is already present in the database,
824			 * so simply update the metadata.
825			 */
826			update_existing_entry(db, file, md5sum, rec,
827			    &new_count, &link_count, &err_count);
828			free(md5sum);
829			continue;
830		}
831
832		if (md5_status == 1) {
833			/*
834			 * The MD5 hash was not present in the database.
835			 * This means is either a new file or an updated file.
836			 * We should go ahead with parsing.
837			 */
838			if (chdir(parent) == -1) {
839				if (mflags.verbosity)
840					warn("chdir failed for `%s', could "
841					    "not index `%s'", parent, file);
842				err_count++;
843				free(md5sum);
844				continue;
845			}
846
847			if (mflags.verbosity == 2)
848				printf("Parsing: %s\n", file);
849			rec->md5_hash = md5sum;
850			rec->file_path = estrdup(file);
851			// file_path is freed by insert_into_db itself.
852			begin_parse(file, mp, rec, buf, buflen);
853			if (insert_into_db(db, rec) < 0) {
854				if (mflags.verbosity)
855					warnx("Error in indexing `%s'", file);
856				err_count++;
857			} else {
858				new_count++;
859			}
860		}
861	}
862
863	if (mflags.verbosity == 2) {
864		printf("Number of new or updated pages encountered: %d\n"
865		    "Number of hard links found: %d\n"
866		    "Number of pages that were successfully"
867		    " indexed or updated: %d\n"
868		    "Number of pages that could not be indexed"
869		    " due to errors: %d\n",
870		    total_count - link_count, link_count, new_count, err_count);
871	}
872
873	if (mflags.recreate)
874		return;
875
876	if (mflags.verbosity == 2)
877		printf("Deleting stale index entries\n");
878
879	sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
880		 " (SELECT file FROM metadb.file_cache);"
881		 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
882		 " (SELECT md5_hash from mandb_meta);"
883		 "DROP TABLE metadb.file_cache;"
884		 "DELETE FROM mandb WHERE rowid NOT IN"
885		 " (SELECT id FROM mandb_meta);";
886
887	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
888	if (errmsg != NULL) {
889		warnx("Removing old entries failed: %s", errmsg);
890		warnx("Please rebuild database from scratch with -f.");
891		free(errmsg);
892		return;
893	}
894}
895
896/*
897 * begin_parse --
898 *  parses the man page using libmandoc
899 */
900static void
901begin_parse(const char *file, struct mparse *mp, mandb_rec *rec,
902    const void *buf, size_t len)
903{
904	struct roff_man *roff;
905	mparse_reset(mp);
906
907	rec->xr_found = 0;
908
909	if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_BADARG) {
910		/* Printing this warning at verbosity level 2
911		 * because some packages from pkgsrc might trigger several
912		 * of such warnings.
913		 */
914		if (mflags.verbosity == 2)
915			warnx("%s: Parse failure", file);
916		return;
917	}
918
919	mparse_result(mp, &roff, NULL);
920	if (roff == NULL) {
921		if (mflags.verbosity == 2)
922			warnx("Not a roff(7) page");
923		return;
924	}
925
926	if (roff->macroset == MACROSET_MDOC) {
927		mdoc_validate(roff);
928		rec->page_type = MDOC;
929		proff_node(roff->first->child, rec, mdocs);
930	} else if (roff->macroset == MACROSET_MAN) {
931		man_validate(roff);
932		rec->page_type = MAN;
933		proff_node(roff->first->child, rec, mans);
934	} else
935		warnx("Unknown macroset %d", roff->macroset);
936	set_machine(roff, rec);
937	set_section(roff, rec);
938}
939
940/*
941 * set_section --
942 *  Extracts the section number and normalizes it to only the numeric part
943 *  (Which should be the first character of the string).
944 */
945static void
946set_section(const struct roff_man *rm, mandb_rec *rec)
947{
948	if (!rm)
949		return;
950	const struct roff_meta *rm_meta = &rm->meta;
951	const char *s = rm_meta->msec == NULL ? "?" : rm_meta->msec;
952	easprintf(&rec->section, "%s", s);
953	if (rec->section[0] == '?' && mflags.verbosity == 2)
954		warnx("%s: Missing section number", rec->file_path);
955}
956
957/*
958 * get_machine --
959 *  Extracts the machine architecture information if available.
960 */
961static void
962set_machine(const struct roff_man *rm, mandb_rec *rec)
963{
964	if (rm == NULL)
965		return;
966	const struct roff_meta *rm_meta = &rm->meta;
967	if (rm_meta->arch)
968		rec->machine = estrdup(rm_meta->arch);
969}
970
971/*
972 * pmdoc_Nm --
973 *  Extracts the Name of the manual page from the .Nm macro
974 */
975static void
976pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
977{
978	if (n->sec != SEC_NAME)
979		return;
980
981	for (n = n->child; n; n = n->next) {
982		if (n->type == ROFFT_TEXT) {
983			char *escaped_name = parse_escape(n->string);
984			concat(&rec->name, escaped_name);
985			free(escaped_name);
986		}
987	}
988}
989
990/*
991 * pmdoc_Nd --
992 *  Extracts the one line description of the man page from the .Nd macro
993 */
994static void
995pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
996{
997	if (n->type == ROFFT_BODY)
998		deroff(&rec->name_desc, n);
999	if (rec->name_desc)
1000		replace_hyph(rec->name_desc);
1001
1002}
1003
1004/*
1005 * pmdoc_macro_handler--
1006 *  This function is a single point of handling all the special macros that we
1007 *  want to handle especially. For example the .Xr macro for properly parsing
1008 *  the referenced page name along with the section number, or the .Pp macro
1009 *  for adding a new line whenever we encounter it.
1010 */
1011static void
1012pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
1013{
1014	const struct roff_node *sn;
1015	assert(n);
1016
1017	switch (doct) {
1018	/*  Parse the man page references.
1019	 * Basically the .Xr macros are used like:
1020	 *  .Xr ls 1
1021 	 *  and formatted like this:
1022	 *  ls(1)
1023	 *  Prepare a buffer to format the data like the above example and call
1024	 *  pmdoc_parse_section to append it.
1025	 */
1026	case MDOC_Xr:
1027		n = n->child;
1028		while (n->type != ROFFT_TEXT && n->next)
1029			n = n->next;
1030
1031		if (n && n->type != ROFFT_TEXT)
1032			return;
1033		sn = n;
1034		if (n->next)
1035			n = n->next;
1036
1037		while (n->type != ROFFT_TEXT && n->next)
1038			n = n->next;
1039
1040		if (n && n->type == ROFFT_TEXT) {
1041			char *buf;
1042			easprintf(&buf, "%s(%s)", sn->string, n->string);
1043			mdoc_parse_section(n->sec, buf, rec);
1044			free(buf);
1045		}
1046
1047		break;
1048
1049	/* Parse the .Pp macro to add a new line */
1050	case MDOC_Pp:
1051		if (n->type == ROFFT_TEXT)
1052			mdoc_parse_section(n->sec, "\n", rec);
1053		break;
1054	default:
1055		break;
1056	}
1057
1058}
1059
1060/*
1061 * pmdoc_Xr, pmdoc_Pp--
1062 *  Empty stubs.
1063 *  The parser calls these functions each time it encounters
1064 *  a .Xr or .Pp macro. We are parsing all the data from
1065 *  the pmdoc_Sh function, so don't do anything here.
1066 *  (See if else blocks in pmdoc_Sh.)
1067 */
1068static void
1069pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
1070{
1071}
1072
1073static void
1074pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
1075{
1076}
1077
1078/*
1079 * pmdoc_Sh --
1080 *  Called when a .Sh macro is encountered and tries to parse its body
1081 */
1082static void
1083pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
1084{
1085	if (n == NULL)
1086		return;
1087
1088	switch (n->sec) {
1089	case SEC_NAME:
1090	case SEC_SYNOPSIS:
1091	case SEC_EXAMPLES:
1092	case SEC_STANDARDS:
1093	case SEC_HISTORY:
1094	case SEC_AUTHORS:
1095	case SEC_BUGS:
1096		/*
1097		 * We don't care about text from these sections
1098		 */
1099		return;
1100	default:
1101		break;
1102	}
1103
1104	if (n->type == ROFFT_BLOCK)
1105		mdoc_parse_Sh(n->body, rec);
1106}
1107
1108/*
1109 *  Called from pmdoc_Sh to parse body of a .Sh macro. It calls
1110 *  mdoc_parse_section to append the data to the section specific buffer.
1111 *  The .Xr macro needs special handling, thus the separate if branch for it.
1112 */
1113static void
1114mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
1115{
1116	if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
1117		return;
1118	int xr_found = 0;
1119
1120	if (n->type == ROFFT_TEXT) {
1121		mdoc_parse_section(n->sec, n->string, rec);
1122	} else if (mdocs[n->tok] == pmdoc_Xr) {
1123		/*
1124		 * When encountering other inline macros,
1125		 * call pmdoc_macro_handler.
1126		 */
1127		pmdoc_macro_handler(n, rec, MDOC_Xr);
1128		xr_found = 1;
1129	} else if (mdocs[n->tok] == pmdoc_Pp) {
1130		pmdoc_macro_handler(n, rec, MDOC_Pp);
1131	}
1132
1133	/*
1134	 * If an Xr macro was encountered then the child node has
1135	 * already been explored by pmdoc_macro_handler.
1136	 */
1137	if (xr_found == 0)
1138		mdoc_parse_Sh(n->child, rec);
1139	mdoc_parse_Sh(n->next, rec);
1140}
1141
1142/*
1143 * mdoc_parse_section--
1144 *  Utility function for parsing sections of the mdoc type pages.
1145 *  Takes two params:
1146 *   1. sec is an enum which indicates the section in which we are present
1147 *   2. string is the string which we need to append to the secbuff for this
1148 *      particular section.
1149 *  The function appends string to the global section buffer and returns.
1150 */
1151static void
1152mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
1153{
1154	/*
1155	 * If the user specified the 'l' flag, then parse and store only the
1156	 * NAME section. Ignore the rest.
1157	 */
1158	if (mflags.limit)
1159		return;
1160
1161	switch (sec) {
1162	case SEC_LIBRARY:
1163		append(&rec->lib, string);
1164		break;
1165	case SEC_RETURN_VALUES:
1166		append(&rec->return_vals, string);
1167		break;
1168	case SEC_ENVIRONMENT:
1169		append(&rec->env, string);
1170		break;
1171	case SEC_FILES:
1172		append(&rec->files, string);
1173		break;
1174	case SEC_EXIT_STATUS:
1175		append(&rec->exit_status, string);
1176		break;
1177	case SEC_DIAGNOSTICS:
1178		append(&rec->diagnostics, string);
1179		break;
1180	case SEC_ERRORS:
1181		append(&rec->errors, string);
1182		break;
1183	default:
1184		append(&rec->desc, string);
1185		break;
1186	}
1187}
1188
1189static void
1190proff_node(const struct roff_node *n, mandb_rec *rec, const proff_nf *func)
1191{
1192	if (n == NULL)
1193		return;
1194
1195	switch (n->type) {
1196	case (ROFFT_BODY):
1197		/* FALLTHROUGH */
1198	case (ROFFT_BLOCK):
1199		/* FALLTHROUGH */
1200	case (ROFFT_ELEM):
1201		if (func[n->tok] != NULL)
1202			(*func[n->tok])(n, rec);
1203		break;
1204	default:
1205		break;
1206	}
1207
1208	proff_node(n->child, rec, func);
1209	proff_node(n->next, rec, func);
1210}
1211
1212/*
1213 * pman_parse_name --
1214 *  Parses the NAME section and puts the complete content in the name_desc
1215 *  variable.
1216 */
1217static void
1218pman_parse_name(const struct roff_node *n, mandb_rec *rec)
1219{
1220	if (n == NULL)
1221		return;
1222
1223	if (n->type == ROFFT_TEXT) {
1224		char *tmp = parse_escape(n->string);
1225		concat(&rec->name_desc, tmp);
1226		free(tmp);
1227	}
1228
1229	if (n->child)
1230		pman_parse_name(n->child, rec);
1231
1232	if(n->next)
1233		pman_parse_name(n->next, rec);
1234}
1235
1236/*
1237 * A stub function to be able to parse the macros like .B embedded inside
1238 * a section.
1239 */
1240static void
1241pman_block(const struct roff_node *n, mandb_rec *rec)
1242{
1243}
1244
1245/*
1246 * pman_sh --
1247 * This function does one of the two things:
1248 *  1. If the present section is NAME, then it will:
1249 *    (a) Extract the name of the page (in case of multiple comma separated
1250 *        names, it will pick up the first one).
1251 *    (b) Build a space spearated list of all the symlinks/hardlinks to
1252 *        this page and store in the buffer 'links'. These are extracted from
1253 *        the comma separated list of names in the NAME section as well.
1254 *    (c) Move on to the one line description section, which is after the list
1255 *        of names in the NAME section.
1256 *  2. Otherwise, it will check the section name and call the man_parse_section
1257 *     function, passing the enum corresponding to that section.
1258 */
1259static void
1260pman_sh(const struct roff_node *n, mandb_rec *rec)
1261{
1262	static const struct {
1263		enum man_sec section;
1264		const char *header;
1265	} mapping[] = {
1266	    { MANSEC_DESCRIPTION, "DESCRIPTION" },
1267	    { MANSEC_SYNOPSIS, "SYNOPSIS" },
1268	    { MANSEC_LIBRARY, "LIBRARY" },
1269	    { MANSEC_ERRORS, "ERRORS" },
1270	    { MANSEC_FILES, "FILES" },
1271	    { MANSEC_RETURN_VALUES, "RETURN VALUE" },
1272	    { MANSEC_RETURN_VALUES, "RETURN VALUES" },
1273	    { MANSEC_EXIT_STATUS, "EXIT STATUS" },
1274	    { MANSEC_EXAMPLES, "EXAMPLES" },
1275	    { MANSEC_EXAMPLES, "EXAMPLE" },
1276	    { MANSEC_STANDARDS, "STANDARDS" },
1277	    { MANSEC_HISTORY, "HISTORY" },
1278	    { MANSEC_BUGS, "BUGS" },
1279	    { MANSEC_AUTHORS, "AUTHORS" },
1280	    { MANSEC_COPYRIGHT, "COPYRIGHT" },
1281	};
1282	const struct roff_node *head;
1283	char *name_desc;
1284	size_t sz;
1285	size_t i;
1286
1287	if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
1288	    head->type != ROFFT_TEXT)
1289		return;
1290
1291	/*
1292	 * Check if this section should be extracted and
1293	 * where it should be stored. Handled the trival cases first.
1294	 */
1295	for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
1296		if (strcmp(head->string, mapping[i].header) == 0) {
1297			man_parse_section(mapping[i].section, n, rec);
1298			return;
1299		}
1300	}
1301
1302	if (strcmp(head->string, "NAME") == 0) {
1303		/*
1304		 * We are in the NAME section.
1305		 * pman_parse_name will put the complete content in name_desc.
1306		 */
1307		pman_parse_name(n, rec);
1308
1309		name_desc = rec->name_desc;
1310		if (name_desc == NULL)
1311			return;
1312
1313		/* Remove any leading spaces. */
1314		while (name_desc[0] == ' ')
1315			name_desc++;
1316
1317		/* If the line begins with a "\&", avoid those */
1318		if (name_desc[0] == '\\' && name_desc[1] == '&')
1319			name_desc += 2;
1320
1321		/* Now name_desc should be left with a comma-space
1322		 * separated list of names and the one line description
1323		 * of the page:
1324		 *     "a, b, c \- sample description"
1325		 * Take out the first name, before the first comma
1326		 * (or space) and store it in rec->name.
1327		 * If the page has aliases then they should be
1328		 * in the form of a comma separated list.
1329		 * Keep looping while there is a comma in name_desc,
1330		 * extract the alias name and store in rec->links.
1331		 * When there are no more commas left, break out.
1332		 */
1333		int has_alias = 0;	// Any more aliases left?
1334		while (*name_desc) {
1335			/* Remove any leading spaces or hyphens. */
1336			if (name_desc[0] == ' ' || name_desc[0] == '-') {
1337				name_desc++;
1338				continue;
1339			}
1340			sz = strcspn(name_desc, ", ");
1341
1342			/* Extract the first term and store it in rec->name. */
1343			if (rec->name == NULL) {
1344				if (name_desc[sz] == ',')
1345					has_alias = 1;
1346				rec->name = estrndup(name_desc, sz);
1347				/* XXX This would only happen with a poorly
1348				 * written man page, maybe warn? */
1349				if (name_desc[sz] == '\0')
1350					break;
1351				name_desc += sz + 1;
1352				continue;
1353			}
1354
1355			/*
1356			 * Once rec->name is set, rest of the names
1357			 * are to be treated as links or aliases.
1358			 */
1359			if (rec->name && has_alias) {
1360				if (name_desc[sz] != ',') {
1361					/* No more commas left --> no more
1362					 * aliases to take out */
1363					has_alias = 0;
1364				}
1365				concat2(&rec->links, name_desc, sz);
1366				/* XXX This would only happen with a poorly
1367				 * written man page, maybe warn? */
1368				if (name_desc[sz] == '\0')
1369					break;
1370				name_desc += sz + 1;
1371				continue;
1372			}
1373			break;
1374		}
1375
1376		/* Parse any escape sequences that might be there */
1377		char *temp = parse_escape(name_desc);
1378		free(rec->name_desc);
1379		rec->name_desc = temp;
1380		temp = parse_escape(rec->name);
1381		free(rec->name);
1382		rec->name = temp;
1383		return;
1384	}
1385
1386	/* The RETURN VALUE section might be specified in multiple ways */
1387	if (strcmp(head->string, "RETURN") == 0 &&
1388	    head->next != NULL && head->next->type == ROFFT_TEXT &&
1389	    (strcmp(head->next->string, "VALUE") == 0 ||
1390	    strcmp(head->next->string, "VALUES") == 0)) {
1391		man_parse_section(MANSEC_RETURN_VALUES, n, rec);
1392		return;
1393	}
1394
1395	/*
1396	 * EXIT STATUS section can also be specified all on one line or on two
1397	 * separate lines.
1398	 */
1399	if (strcmp(head->string, "EXIT") == 0 &&
1400	    head->next != NULL && head->next->type == ROFFT_TEXT &&
1401	    strcmp(head->next->string, "STATUS") == 0) {
1402		man_parse_section(MANSEC_EXIT_STATUS, n, rec);
1403		return;
1404	}
1405
1406	/* Store the rest of the content in desc. */
1407	man_parse_section(MANSEC_NONE, n, rec);
1408}
1409
1410/*
1411 * pman_parse_node --
1412 *  Generic function to iterate through a node. Usually called from
1413 *  man_parse_section to parse a particular section of the man page.
1414 */
1415static void
1416pman_parse_node(const struct roff_node *n, secbuff *s)
1417{
1418	if (n == NULL)
1419		return;
1420
1421	if (n->type == ROFFT_TEXT)
1422		append(s, n->string);
1423
1424	pman_parse_node(n->child, s);
1425	pman_parse_node(n->next, s);
1426}
1427
1428/*
1429 * man_parse_section --
1430 *  Takes two parameters:
1431 *   sec: Tells which section we are present in
1432 *   n: Is the present node of the AST.
1433 * Depending on the section, we call pman_parse_node to parse that section and
1434 * concatenate the content from that section into the buffer for that section.
1435 */
1436static void
1437man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
1438{
1439	/*
1440	 * If the user sepecified the 'l' flag then just parse
1441	 * the NAME section, ignore the rest.
1442	 */
1443	if (mflags.limit)
1444		return;
1445
1446	switch (sec) {
1447	case MANSEC_LIBRARY:
1448		pman_parse_node(n, &rec->lib);
1449		break;
1450	case MANSEC_RETURN_VALUES:
1451		pman_parse_node(n, &rec->return_vals);
1452		break;
1453	case MANSEC_ENVIRONMENT:
1454		pman_parse_node(n, &rec->env);
1455		break;
1456	case MANSEC_FILES:
1457		pman_parse_node(n, &rec->files);
1458		break;
1459	case MANSEC_EXIT_STATUS:
1460		pman_parse_node(n, &rec->exit_status);
1461		break;
1462	case MANSEC_DIAGNOSTICS:
1463		pman_parse_node(n, &rec->diagnostics);
1464		break;
1465	case MANSEC_ERRORS:
1466		pman_parse_node(n, &rec->errors);
1467		break;
1468	case MANSEC_NAME:
1469	case MANSEC_SYNOPSIS:
1470	case MANSEC_EXAMPLES:
1471	case MANSEC_STANDARDS:
1472	case MANSEC_HISTORY:
1473	case MANSEC_BUGS:
1474	case MANSEC_AUTHORS:
1475	case MANSEC_COPYRIGHT:
1476		break;
1477	default:
1478		pman_parse_node(n, &rec->desc);
1479		break;
1480	}
1481
1482}
1483
1484/*
1485 * insert_into_db --
1486 *  Inserts the parsed data of the man page in the Sqlite databse.
1487 *  If any of the values is NULL, then we cleanup and return -1 indicating
1488 *  an error.
1489 *  Otherwise, store the data in the database and return 0.
1490 */
1491static int
1492insert_into_db(sqlite3 *db, mandb_rec *rec)
1493{
1494	int rc = 0;
1495	int idx = -1;
1496	const char *sqlstr = NULL;
1497	sqlite3_stmt *stmt = NULL;
1498	char *ln = NULL;
1499	char *errmsg = NULL;
1500	long int mandb_rowid;
1501
1502	/*
1503	 * At the very minimum we want to make sure that we store
1504	 * the following data:
1505	 *   Name, one line description, and the MD5 hash
1506	 */
1507	if (rec->name == NULL || rec->name_desc == NULL ||
1508	    rec->md5_hash == NULL) {
1509		cleanup(rec);
1510		return -1;
1511	}
1512
1513	/* Write null byte at the end of all the sec_buffs */
1514	rec->desc.data[rec->desc.offset] = 0;
1515	rec->lib.data[rec->lib.offset] = 0;
1516	rec->env.data[rec->env.offset] = 0;
1517	rec->return_vals.data[rec->return_vals.offset] = 0;
1518	rec->exit_status.data[rec->exit_status.offset] = 0;
1519	rec->files.data[rec->files.offset] = 0;
1520	rec->diagnostics.data[rec->diagnostics.offset] = 0;
1521	rec->errors.data[rec->errors.offset] = 0;
1522
1523	/*
1524	 * In case of a mdoc page: (sorry, no better place to put this code)
1525	 * parse the comma separated list of names of man pages,
1526	 * the first name will be stored in the mandb table, rest will be
1527	 * treated as links and put in the mandb_links table.
1528	 */
1529	if (rec->page_type == MDOC) {
1530		char *tmp;
1531		rec->links = estrdup(rec->name);
1532		free(rec->name);
1533		size_t sz = strcspn(rec->links, " \0");
1534		rec->name = emalloc(sz + 1);
1535		memcpy(rec->name, rec->links, sz);
1536		if(rec->name[sz - 1] == ',')
1537			rec->name[sz - 1] = 0;
1538		else
1539			rec->name[sz] = 0;
1540		while (rec->links[sz] == ' ')
1541			++sz;
1542		tmp = estrdup(rec->links + sz);
1543		free(rec->links);
1544		rec->links = tmp;
1545	}
1546
1547/*------------------------ Populate the mandb table---------------------------*/
1548	sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
1549		 " :lib, :return_vals, :env, :files, :exit_status,"
1550		 " :diagnostics, :errors, :md5_hash, :machine)";
1551
1552	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1553	if (rc != SQLITE_OK)
1554		goto Out;
1555
1556	idx = sqlite3_bind_parameter_index(stmt, ":name");
1557	rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
1558	if (rc != SQLITE_OK) {
1559		sqlite3_finalize(stmt);
1560		goto Out;
1561	}
1562
1563	idx = sqlite3_bind_parameter_index(stmt, ":section");
1564	rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
1565	if (rc != SQLITE_OK) {
1566		sqlite3_finalize(stmt);
1567		goto Out;
1568	}
1569
1570	idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
1571	rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
1572	if (rc != SQLITE_OK) {
1573		sqlite3_finalize(stmt);
1574		goto Out;
1575	}
1576
1577	idx = sqlite3_bind_parameter_index(stmt, ":desc");
1578	rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
1579	                       rec->desc.offset + 1, NULL);
1580	if (rc != SQLITE_OK) {
1581		sqlite3_finalize(stmt);
1582		goto Out;
1583	}
1584
1585	idx = sqlite3_bind_parameter_index(stmt, ":lib");
1586	rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
1587	    rec->lib.offset + 1, NULL);
1588	if (rc != SQLITE_OK) {
1589		sqlite3_finalize(stmt);
1590		goto Out;
1591	}
1592
1593	idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
1594	rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
1595	                      rec->return_vals.offset + 1, NULL);
1596	if (rc != SQLITE_OK) {
1597		sqlite3_finalize(stmt);
1598		goto Out;
1599	}
1600
1601	idx = sqlite3_bind_parameter_index(stmt, ":env");
1602	rc = sqlite3_bind_text(stmt, idx, rec->env.data,
1603	    rec->env.offset + 1, NULL);
1604	if (rc != SQLITE_OK) {
1605		sqlite3_finalize(stmt);
1606		goto Out;
1607	}
1608
1609	idx = sqlite3_bind_parameter_index(stmt, ":files");
1610	rc = sqlite3_bind_text(stmt, idx, rec->files.data,
1611	                       rec->files.offset + 1, NULL);
1612	if (rc != SQLITE_OK) {
1613		sqlite3_finalize(stmt);
1614		goto Out;
1615	}
1616
1617	idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
1618	rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
1619	                       rec->exit_status.offset + 1, NULL);
1620	if (rc != SQLITE_OK) {
1621		sqlite3_finalize(stmt);
1622		goto Out;
1623	}
1624
1625	idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
1626	rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
1627	                       rec->diagnostics.offset + 1, NULL);
1628	if (rc != SQLITE_OK) {
1629		sqlite3_finalize(stmt);
1630		goto Out;
1631	}
1632
1633	idx = sqlite3_bind_parameter_index(stmt, ":errors");
1634	rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
1635	                       rec->errors.offset + 1, NULL);
1636	if (rc != SQLITE_OK) {
1637		sqlite3_finalize(stmt);
1638		goto Out;
1639	}
1640
1641	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1642	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1643	if (rc != SQLITE_OK) {
1644		sqlite3_finalize(stmt);
1645		goto Out;
1646	}
1647
1648	idx = sqlite3_bind_parameter_index(stmt, ":machine");
1649	if (rec->machine)
1650		rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
1651	else
1652		rc = sqlite3_bind_null(stmt, idx);
1653	if (rc != SQLITE_OK) {
1654		sqlite3_finalize(stmt);
1655		goto Out;
1656	}
1657
1658	rc = sqlite3_step(stmt);
1659	if (rc != SQLITE_DONE) {
1660		sqlite3_finalize(stmt);
1661		goto Out;
1662	}
1663
1664	sqlite3_finalize(stmt);
1665
1666	/* Get the row id of the last inserted row */
1667	mandb_rowid = sqlite3_last_insert_rowid(db);
1668
1669/*------------------------Populate the mandb_meta table-----------------------*/
1670	sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
1671		 " :file, :md5_hash, :id)";
1672	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1673	if (rc != SQLITE_OK)
1674		goto Out;
1675
1676	idx = sqlite3_bind_parameter_index(stmt, ":device");
1677	rc = sqlite3_bind_int64(stmt, idx, rec->device);
1678	if (rc != SQLITE_OK) {
1679		sqlite3_finalize(stmt);
1680		goto Out;
1681	}
1682
1683	idx = sqlite3_bind_parameter_index(stmt, ":inode");
1684	rc = sqlite3_bind_int64(stmt, idx, rec->inode);
1685	if (rc != SQLITE_OK) {
1686		sqlite3_finalize(stmt);
1687		goto Out;
1688	}
1689
1690	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1691	rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
1692	if (rc != SQLITE_OK) {
1693		sqlite3_finalize(stmt);
1694		goto Out;
1695	}
1696
1697	idx = sqlite3_bind_parameter_index(stmt, ":file");
1698	rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1699	if (rc != SQLITE_OK) {
1700		sqlite3_finalize(stmt);
1701		goto Out;
1702	}
1703
1704	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1705	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1706	if (rc != SQLITE_OK) {
1707		sqlite3_finalize(stmt);
1708		goto Out;
1709	}
1710
1711	idx = sqlite3_bind_parameter_index(stmt, ":id");
1712	rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
1713	if (rc != SQLITE_OK) {
1714		sqlite3_finalize(stmt);
1715		goto Out;
1716	}
1717
1718	rc = sqlite3_step(stmt);
1719	sqlite3_finalize(stmt);
1720	if (rc == SQLITE_CONSTRAINT_UNIQUE) {
1721		/* The *most* probable reason for reaching here is that
1722		 * the UNIQUE contraint on the file column of the mandb_meta
1723		 * table was violated.
1724		 * This can happen when a file was updated/modified.
1725		 * To fix this we need to do two things:
1726		 * 1. Delete the row for the older version of this file
1727		 *    from mandb table.
1728		 * 2. Run an UPDATE query to update the row for this file
1729		 *    in the mandb_meta table.
1730		 */
1731		warnx("Trying to update index for %s", rec->file_path);
1732		char *sql = sqlite3_mprintf("DELETE FROM mandb "
1733					    "WHERE rowid = (SELECT id"
1734					    "  FROM mandb_meta"
1735					    "  WHERE file = %Q)",
1736					    rec->file_path);
1737		sqlite3_exec(db, sql, NULL, NULL, &errmsg);
1738		sqlite3_free(sql);
1739		if (errmsg != NULL) {
1740			if (mflags.verbosity)
1741				warnx("%s", errmsg);
1742			free(errmsg);
1743		}
1744		sqlstr = "UPDATE mandb_meta SET device = :device,"
1745			 " inode = :inode, mtime = :mtime, id = :id,"
1746			 " md5_hash = :md5 WHERE file = :file";
1747		rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1748		if (rc != SQLITE_OK) {
1749			if (mflags.verbosity)
1750				warnx("Update failed with error: %s",
1751			    sqlite3_errmsg(db));
1752			close_db(db);
1753			cleanup(rec);
1754			errx(EXIT_FAILURE,
1755			    "Consider running makemandb with -f option");
1756		}
1757
1758		idx = sqlite3_bind_parameter_index(stmt, ":device");
1759		sqlite3_bind_int64(stmt, idx, rec->device);
1760		idx = sqlite3_bind_parameter_index(stmt, ":inode");
1761		sqlite3_bind_int64(stmt, idx, rec->inode);
1762		idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1763		sqlite3_bind_int64(stmt, idx, rec->mtime);
1764		idx = sqlite3_bind_parameter_index(stmt, ":id");
1765		sqlite3_bind_int64(stmt, idx, mandb_rowid);
1766		idx = sqlite3_bind_parameter_index(stmt, ":md5");
1767		sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1768		idx = sqlite3_bind_parameter_index(stmt, ":file");
1769		sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1770		rc = sqlite3_step(stmt);
1771		sqlite3_finalize(stmt);
1772
1773		if (rc != SQLITE_DONE) {
1774			if (mflags.verbosity)
1775				warnx("%s", sqlite3_errmsg(db));
1776			close_db(db);
1777			cleanup(rec);
1778			errx(EXIT_FAILURE,
1779			    "Consider running makemandb with -f option");
1780		}
1781	} else if (rc != SQLITE_DONE) {
1782		/* Otherwise make this error fatal */
1783		warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
1784		cleanup(rec);
1785		close_db(db);
1786		exit(EXIT_FAILURE);
1787	}
1788
1789/*------------------------ Populate the mandb_links table---------------------*/
1790	char *str = NULL;
1791	char *links;
1792	if (rec->links && strlen(rec->links)) {
1793		links = rec->links;
1794		for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
1795			if (ln[0] == ',')
1796				ln++;
1797			if(ln[strlen(ln) - 1] == ',')
1798				ln[strlen(ln) - 1] = 0;
1799
1800			str = sqlite3_mprintf("INSERT INTO mandb_links"
1801					      " VALUES (%Q, %Q, %Q, %Q, %Q)",
1802					      ln, rec->name, rec->section,
1803					      rec->machine, rec->md5_hash);
1804			sqlite3_exec(db, str, NULL, NULL, &errmsg);
1805			sqlite3_free(str);
1806			if (errmsg != NULL) {
1807				warnx("%s", errmsg);
1808				cleanup(rec);
1809				free(errmsg);
1810				return -1;
1811			}
1812		}
1813	}
1814
1815	cleanup(rec);
1816	return 0;
1817
1818  Out:
1819	if (mflags.verbosity)
1820		warnx("%s", sqlite3_errmsg(db));
1821	cleanup(rec);
1822	return -1;
1823}
1824
1825/*
1826 * check_md5--
1827 *  Generates the md5 hash of the file and checks if it already doesn't exist
1828 *  in the table (passed as the 3rd parameter).
1829 *  This function is being used to avoid hardlinks.
1830 *  On successful completion it will also set the value of the fourth parameter
1831 *  to the md5 hash of the file (computed previously). It is the responsibility
1832 *  of the caller to free this buffer.
1833 *  Return values:
1834 *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
1835 *  0: If the md5 hash does not exist in the table.
1836 *  1: If the hash exists in the database.
1837 */
1838static int
1839check_md5(const char *file, sqlite3 *db, const char *table, char **md5sum,
1840    void *buf, size_t buflen)
1841{
1842	int rc = 0;
1843	int idx = -1;
1844	char *sqlstr = NULL;
1845	sqlite3_stmt *stmt = NULL;
1846
1847	assert(file != NULL);
1848	*md5sum = MD5Data(buf, buflen, NULL);
1849	if (*md5sum == NULL) {
1850		if (mflags.verbosity)
1851			warn("md5 failed: %s", file);
1852		return -1;
1853	}
1854
1855	easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash",
1856	    table);
1857	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1858	if (rc != SQLITE_OK) {
1859		free(sqlstr);
1860		free(*md5sum);
1861		*md5sum = NULL;
1862		return -1;
1863	}
1864
1865	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1866	rc = sqlite3_bind_text(stmt, idx, *md5sum, -1, NULL);
1867	if (rc != SQLITE_OK) {
1868		if (mflags.verbosity)
1869			warnx("%s", sqlite3_errmsg(db));
1870		sqlite3_finalize(stmt);
1871		free(sqlstr);
1872		free(*md5sum);
1873		*md5sum = NULL;
1874		return -1;
1875	}
1876
1877	if (sqlite3_step(stmt) == SQLITE_ROW) {
1878		sqlite3_finalize(stmt);
1879		free(sqlstr);
1880		return 0;
1881	}
1882
1883	sqlite3_finalize(stmt);
1884	free(sqlstr);
1885	return 1;
1886}
1887
1888/* Optimize the index for faster search */
1889static void
1890optimize(sqlite3 *db)
1891{
1892	const char *sqlstr;
1893	char *errmsg = NULL;
1894
1895	if (mflags.verbosity == 2)
1896		printf("Optimizing the database index\n");
1897	sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
1898		 "VACUUM";
1899	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
1900	if (errmsg != NULL) {
1901		if (mflags.verbosity)
1902			warnx("%s", errmsg);
1903		free(errmsg);
1904		return;
1905	}
1906}
1907
1908/*
1909 * cleanup --
1910 *  cleans up the global buffers
1911 */
1912static void
1913cleanup(mandb_rec *rec)
1914{
1915	rec->desc.offset = 0;
1916	rec->lib.offset = 0;
1917	rec->return_vals.offset = 0;
1918	rec->env.offset = 0;
1919	rec->exit_status.offset = 0;
1920	rec->diagnostics.offset = 0;
1921	rec->errors.offset = 0;
1922	rec->files.offset = 0;
1923
1924	free(rec->machine);
1925	rec->machine = NULL;
1926
1927	free(rec->links);
1928	rec->links = NULL;
1929
1930	free(rec->file_path);
1931	rec->file_path = NULL;
1932
1933	free(rec->name);
1934	rec->name = NULL;
1935
1936	free(rec->name_desc);
1937	rec->name_desc = NULL;
1938
1939	free(rec->md5_hash);
1940	rec->md5_hash = NULL;
1941
1942	free(rec->section);
1943	rec->section = NULL;
1944}
1945
1946/*
1947 * init_secbuffs--
1948 *  Sets the value of buflen for all the sec_buff field of rec. And then
1949 *  allocate memory to each sec_buff member of rec.
1950 */
1951static void
1952init_secbuffs(mandb_rec *rec)
1953{
1954	/*
1955	 * Some sec_buff might need more memory, for example desc,
1956	 * which stores the data of the DESCRIPTION section,
1957	 * while some might need very small amount of memory.
1958	 * Therefore explicitly setting the value of buflen field for
1959	 * each sec_buff.
1960	 */
1961	rec->desc.buflen = 10 * BUFLEN;
1962	rec->desc.data = emalloc(rec->desc.buflen);
1963	rec->desc.offset = 0;
1964
1965	rec->lib.buflen = BUFLEN / 2;
1966	rec->lib.data = emalloc(rec->lib.buflen);
1967	rec->lib.offset = 0;
1968
1969	rec->return_vals.buflen = BUFLEN;
1970	rec->return_vals.data = emalloc(rec->return_vals.buflen);
1971	rec->return_vals.offset = 0;
1972
1973	rec->exit_status.buflen = BUFLEN;
1974	rec->exit_status.data = emalloc(rec->exit_status.buflen);
1975	rec->exit_status.offset = 0;
1976
1977	rec->env.buflen = BUFLEN;
1978	rec->env.data = emalloc(rec->env.buflen);
1979	rec->env.offset = 0;
1980
1981	rec->files.buflen = BUFLEN;
1982	rec->files.data = emalloc(rec->files.buflen);
1983	rec->files.offset = 0;
1984
1985	rec->diagnostics.buflen = BUFLEN;
1986	rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
1987	rec->diagnostics.offset = 0;
1988
1989	rec->errors.buflen = BUFLEN;
1990	rec->errors.data = emalloc(rec->errors.buflen);
1991	rec->errors.offset = 0;
1992}
1993
1994/*
1995 * free_secbuffs--
1996 *  This function should be called at the end, when all the pages have been
1997 *  parsed.
1998 *  It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
1999 */
2000static void
2001free_secbuffs(mandb_rec *rec)
2002{
2003	free(rec->desc.data);
2004	free(rec->lib.data);
2005	free(rec->return_vals.data);
2006	free(rec->exit_status.data);
2007	free(rec->env.data);
2008	free(rec->files.data);
2009	free(rec->diagnostics.data);
2010	free(rec->errors.data);
2011}
2012
2013static void
2014replace_hyph(char *str)
2015{
2016	char *iter = str;
2017	while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
2018		*iter = '-';
2019
2020	iter = str;
2021	while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
2022		*iter = '-';
2023}
2024
2025static char *
2026parse_escape(const char *str)
2027{
2028	const char *backslash, *last_backslash;
2029	char *result, *iter;
2030	size_t len;
2031
2032	assert(str);
2033
2034	last_backslash = str;
2035	backslash = strchr(str, '\\');
2036	if (backslash == NULL) {
2037		result = estrdup(str);
2038		replace_hyph(result);
2039		return result;
2040	}
2041
2042	result = emalloc(strlen(str) + 1);
2043	iter = result;
2044
2045	do {
2046		len = backslash - last_backslash;
2047		memcpy(iter, last_backslash, len);
2048		iter += len;
2049		if (backslash[1] == '-' || backslash[1] == ' ') {
2050			*iter++ = backslash[1];
2051			last_backslash = backslash + 2;
2052			backslash = strchr(last_backslash, '\\');
2053		} else {
2054			++backslash;
2055			mandoc_escape(&backslash, NULL, NULL);
2056			last_backslash = backslash;
2057			if (backslash == NULL)
2058				break;
2059			backslash = strchr(last_backslash, '\\');
2060		}
2061	} while (backslash != NULL);
2062	if (last_backslash != NULL)
2063		strcpy(iter, last_backslash);
2064
2065	replace_hyph(result);
2066	return result;
2067}
2068
2069/*
2070 * append--
2071 *  Concatenates a space and src at the end of sbuff->data (much like concat in
2072 *  apropos-utils.c).
2073 *  Rather than reallocating space for writing data, it uses the value of the
2074 *  offset field of sec_buff to write new data at the free space left in the
2075 *  buffer.
2076 *  In case the size of the data to be appended exceeds the number of bytes left
2077 *  in the buffer, it reallocates buflen number of bytes and then continues.
2078 *  Value of offset field should be adjusted as new data is written.
2079 *
2080 *  NOTE: This function does not write the null byte at the end of the buffers,
2081 *  write a null byte at the position pointed to by offset before inserting data
2082 *  in the db.
2083 */
2084static void
2085append(secbuff *sbuff, const char *src)
2086{
2087	short flag = 0;
2088	size_t srclen, newlen;
2089	char *temp;
2090
2091	assert(src != NULL);
2092	temp = parse_escape(src);
2093	srclen = strlen(temp);
2094
2095	if (sbuff->data == NULL) {
2096		sbuff->data = emalloc(sbuff->buflen);
2097		sbuff->offset = 0;
2098	}
2099
2100	newlen = sbuff->offset + srclen + 2;
2101	if (newlen >= sbuff->buflen) {
2102		while (sbuff->buflen < newlen)
2103			sbuff->buflen += sbuff->buflen;
2104		sbuff->data = erealloc(sbuff->data, sbuff->buflen);
2105		flag = 1;
2106	}
2107
2108	/* Append a space at the end of the buffer. */
2109	if (sbuff->offset || flag)
2110		sbuff->data[sbuff->offset++] = ' ';
2111	/* Now, copy src at the end of the buffer. */
2112	memcpy(sbuff->data + sbuff->offset, temp, srclen);
2113	sbuff->offset += srclen;
2114	free(temp);
2115}
2116
2117static void
2118usage(void)
2119{
2120	fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
2121	exit(1);
2122}
2123