dwatch revision 330559
1#!/bin/sh
2#-
3# Copyright (c) 2014-2018 Devin Teske
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions
8# are met:
9# 1. Redistributions of source code must retain the above copyright
10#    notice, this list of conditions and the following disclaimer.
11# 2. Redistributions in binary form must reproduce the above copyright
12#    notice, this list of conditions and the following disclaimer in the
13#    documentation and/or other materials provided with the distribution.
14#
15# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25# SUCH DAMAGE.
26#
27############################################################ IDENT(1)
28#
29# $Title: Watch processes as they trigger a particular DTrace probe $
30# $FreeBSD: head/cddl/usr.sbin/dwatch/dwatch 330559 2018-03-06 23:44:19Z dteske $
31#
32############################################################ CONFIGURATION
33
34#
35# DTrace pragma settings
36#
37DTRACE_PRAGMA="
38	option quiet
39	option dynvarsize=16m
40	option switchrate=10hz
41" # END-QUOTE
42
43#
44# Profiles
45#
46: ${DWATCH_PROFILES_PATH="/usr/libexec/dwatch:/usr/local/libexec/dwatch"}
47
48############################################################ GLOBALS
49
50VERSION='$Version: 1.0-beta-91 $' # -V
51
52pgm="${0##*/}" # Program basename
53
54#
55# Command-line arguments
56#
57PROBE_ARG=
58
59#
60# Command-line options
61#
62CONSOLE=		# -y
63CONSOLE_FORCE=		# -y
64[ -t 1 ] && CONSOLE=1	# -y
65COUNT=0			# -N count
66CUSTOM_DETAILS=		# -E code
67CUSTOM_TEST=		# -t test
68DEBUG=			# -d
69DESTRUCTIVE_ACTIONS=	# -w
70EXECNAME=		# -k name
71EXECREGEX=		# -z regex
72EXIT_AFTER_COMPILE=	# -e
73FILTER=			# -r regex
74PROBE_COALESCE=		# -F
75GROUP=			# -g group
76JID=			# -j jail
77LIST=			# -l
78LIST_PROFILES=		# -Q
79MAX_ARGS=64		# -B num
80MAX_DEPTH=64		# -K num
81ONELINE=		# -1
82OUTPUT=			# -o file
83OUTPUT_CMD=		# -O cmd
84PID=			# -p pid
85PROBE_TYPE=		# -f -m -n -P
86PROFILE=		# -X profile
87PSTREE=			# -R
88QUIET=			# -q
89TIMEOUT=		# -T time
90TRACE=			# -x
91USER=			# -u user
92USE_PROFILE=		# -X profile
93VERBOSE=		# -v
94
95#
96# Global exit status
97#
98SUCCESS=0
99FAILURE=1
100
101#
102# Miscellaneous
103#
104ACTIONS=
105EVENT_DETAILS=
106EVENT_TAG='printf("%d.%d %s[%d]: ",
107		this->uid0, this->gid0, execname, this->pid0);'
108EVENT_TEST=
109FILE=
110ID=3
111MODULE_CHECKED=
112PROBE=
113PSARGS=1
114RGID=
115RUID=
116SUDO=
117export SUDO_PROMPT="[sudo] Password:"
118TITLE=\$Title:
119
120############################################################ FUNCTIONS
121
122ansi() { local fmt="$2 $4"; [ "$CONSOLE" ] && fmt="\\033[$1m$2\\033[$3m $4";
123	shift 4; printf "$fmt\n" "$@"; }
124die() { exec >&2; [ "$*" ] && echo "$pgm:" "$@"; exit $FAILURE; }
125info() { [ "$QUIET" ] || ansi 35 "INFO" 39 "$@" >&2; }
126
127usage()
128{
129	local optfmt="\t%-10s %s\n"
130	exec >&2
131	[ "$*" ] && printf "%s: %s\n" "$pgm" "$*"
132	printf "Usage: %s [-1defFmnPqRvVwxy] [%s] [%s] [%s] [%s]\n" "$pgm" \
133		"-B num" "-E code" "-g group" "-j jail"
134	printf "\t      [%s] [%s] [%s] [%s] [%s] [%s]\n" \
135		"-k name" "-K num" "-N count" "-o file" "-O cmd" "-p pid"
136	printf "\t      [%s] [%s] [%s] [%s] [%s] [%s]\n" \
137		"-r regex" "-t test" "-T time" "-u user" "-X profile" \
138		"-z regex"
139	printf "\t      probe[,...] [args ...]\n"
140	printf "       %s -l [-fmnPqy] [-r regex] [probe ...]\n" "$pgm"
141	printf "       %s -Q [-1qy] [-r regex]\n" "$pgm"
142	printf "\n"
143	printf "$optfmt" "-1" \
144		"Print one line per process/profile (Default; disables \`-R')."
145	printf "$optfmt" "-B num" \
146		"Maximum process arguments to display (Default $MAX_ARGS)."
147	printf "$optfmt" "-d" \
148		"Debug. Send dtrace(1) script to stdout instead of executing."
149	printf "$optfmt" "-e" \
150		"Exit after compiling request but prior to enabling probes."
151	printf "$optfmt" "-E code" \
152		"DTrace code for event details. If \`-', read from stdin."
153	printf "$optfmt" "-f" \
154		"Enable probe matching the specified function name."
155	printf "$optfmt" "-F" \
156		"Coalesce trace output by function."
157	printf "$optfmt" "-g group" \
158		"Group filter. Only show processes matching group name/gid."
159	printf "$optfmt" "-j jail" \
160		"Jail filter. Only show processes matching jail name/jid."
161	printf "$optfmt" "-k name" \
162		"Only show processes matching name."
163	printf "$optfmt" "-K num" \
164		"Maximum directory depth to display (Default $MAX_DEPTH)."
165	printf "$optfmt" "-l" \
166		"List available probes on standard output and exit."
167	printf "$optfmt" "-m" \
168		"Enable probe matching the specified module name."
169	printf "$optfmt" "-n" \
170		"Enable probe matching the specified probe name."
171	printf "$optfmt" "-N count" \
172		"Exit after count matching entries (Default 0 for disabled)."
173	printf "$optfmt" "-o file" \
174		"Set output file. If \`-', the path \`/dev/stdout' is used."
175	printf "$optfmt" "-O cmd" \
176		"Execute cmd for each event."
177	printf "$optfmt" "-p pid" \
178		"Process id filter. Only show processes with matching pid."
179	printf "$optfmt" "-P" \
180		"Enable probe matching the specified provider name."
181	printf "$optfmt" "-q" \
182		"Quiet. Hide informational messages and all dtrace(1) errors."
183	printf "$optfmt" "-Q" \
184		"List available profiles in DWATCH_PROFILES_PATH and exit."
185	printf "$optfmt" "-r regex" \
186		"Filter. Only show blocks matching awk(1) regular expression."
187	printf "$optfmt" "-R" \
188		"Show parent, grandparent, and ancestor of process."
189	printf "$optfmt" "-t test" \
190		"Test clause (predicate) to limit events (Default none)."
191	printf "$optfmt" "-T time" \
192		"Timeout. Format is \`\#[smhd]' or simply \`\#' for seconds."
193	printf "$optfmt" "-u user" \
194		"User filter. Only show processes matching user name/uid."
195	printf "$optfmt" "-v" \
196		"Verbose. Show all errors from dtrace(1)."
197	printf "$optfmt" "-V" \
198		"Report dwatch version on standard output and exit."
199	printf "$optfmt" "-w" \
200		"Permit destructive actions (copyout*, stop, panic, etc.)."
201	printf "$optfmt" "-x" \
202		"Trace. Print \`<probe-id>' when a probe is triggered."
203	printf "$optfmt" "-X profile" \
204		"Load profile name from DWATCH_PROFILES_PATH."
205	printf "$optfmt" "-y" \
206		"Always treat stdout as console (enable colors/columns/etc.)."
207	printf "$optfmt" "-z regex" \
208		"Only show processes matching awk(1) regular expression."
209	die
210}
211
212dtrace_cmd()
213{
214	local status stdout
215	local timeout=
216
217	if [ "$1" = "-t" ]; then
218		shift
219		[ "$TIMEOUT" ] && timeout=1
220	fi
221
222	exec 3>&1
223	stdout=3
224
225	#
226	# Filter dtrace(1) stderr while preserving exit status
227	#
228	status=$(
229		exec 4>&1
230		to_status=4
231		( trap 'echo $? >&$to_status' EXIT
232			eval $SUDO ${timeout:+timeout \"\$TIMEOUT\"} dtrace \
233				\"\$@\" 2>&1 ${QUIET:+2> /dev/null} >&$stdout
234		) | dtrace_stderr_filter >&2
235	)
236
237	return $status
238}
239
240dtrace_stderr_filter()
241{
242	if [ "$VERBOSE" ]; then
243		cat
244		return
245		# NOTREACHED
246	fi
247
248	awk ' # Start awk(1) stderr-filter
249	/[[:digit:]]+ drops? on CPU [[:digit:]]+/ { next }
250	/failed to write to <stdout>: No such file or directory/ { next }
251	/failed to write to <stdout>: Broken pipe/ { next }
252	/processing aborted: Broken pipe/ { next }
253	/invalid address \(0x[[:xdigit:]]+\) in action #[[:digit:]]+/ { next }
254	/out of scratch space in action #[[:digit:]]+/ { next }
255	/^Bus error$/ { next }
256	{ print; fflush() }
257	' # END-QUOTE
258}
259
260expand_probe()
261{
262	local OPTIND=1 OPTARG flag
263	local type=
264
265	while getopts t: flag; do
266		case "$flag" in
267		t) type="$OPTARG" ;;
268		esac
269	done
270	shift $(( $OPTIND - 1 ))
271
272	local probe="$1"
273	case "$probe" in
274	*:*)
275		echo "$probe"
276		return $SUCCESS
277		;;
278	esac
279
280	dtrace_cmd -l | awk -v probe="$probe" -v type="$type" '
281	# Start awk(1) processor
282	#################################################### BEGIN
283	BEGIN { getline dtrace_header }
284	#################################################### FUNCTIONS
285	function dump(unused1,unused2) {
286		if (n) {
287			if (NcF[n] == 1) f = N2F[n]
288			if (NcM[n] == 1) m = N2M[n]
289			if (NcP[n] == 1) p = N2P[n]
290		} else if (f) {
291			if (FcM[f] == 1) m = F2M[f]
292			if (FcP[f] == 1) p = F2P[f]
293			if (FcN[f] == 0 && found) n = "entry"
294		} else if (m) {
295			if (McP[m] == 1) p = M2P[m]
296		}
297		printf "%s:%s:%s:%s\n", p, m, f, n
298		exit !found
299	}
300	function inFMP() { return probe in F || probe in M || probe in P }
301	function inNMP() { return probe in N || probe in M || probe in P }
302	function inNFP() { return probe in N || probe in F || probe in P }
303	function inNFM() { return probe in N || probe in F || probe in M }
304	function diva(value, peerA, peerB, peerC) {
305		return value >= peerA && value >= peerB && value >= peerC
306	}
307	#################################################### MAIN
308	type == "name" && $NF != probe { next }
309	type == "function" && NF >=4 && $(NF-1) != probe { next }
310	type == "module" && NF == 5 && $(NF-2) != probe { next }
311	type == "provider" && $2 != probe { next }
312	type || $2 == probe || $3 == probe || $4 == probe || $5 == probe {
313		P[_p = $2]++
314		M[_m = (NF >= 5 ? $(NF-2) : "")]++
315		F[_f = (NF >= 4 ? $(NF-1) : "")]++
316		N[_n = $NF]++
317		if (N2F[_n] != _f) NcF[_n]++; N2F[_n] = _f
318		if (N2M[_n] != _m) NcM[_n]++; N2M[_n] = _m
319		if (N2P[_n] != _p) NcP[_n]++; N2P[_n] = _p
320		if (_n !~ /entry|return/) {
321			if (F2N[_f] != _n) FcN[_f]++
322			F2N[_f] = _n
323		}
324		if (F2M[_f] != _m) FcM[_f]++; F2M[_f] = _m
325		if (F2P[_f] != _p) FcP[_f]++; F2P[_f] = _p
326		if (M2P[_m] != _p) McP[_m]++; M2P[_m] = _p
327	}
328	#################################################### END
329	END {
330		if (type == "name")     dump(n = probe, found = probe in N)
331		if (type == "function") dump(f = probe, found = probe in F)
332		if (type == "module")   dump(m = probe, found = probe in M)
333		if (type == "provider") dump(p = probe, found = probe in P)
334		if (probe in N) {
335			found = 1
336			if (!inFMP()) dump(n = probe)
337			if (diva(F[probe], N[probe], M[probe], P[probe]))
338				dump(f = probe)
339			if (diva(M[probe], N[probe], F[probe], P[probe]))
340				dump(m = probe)
341			if (diva(P[probe], N[probe], F[probe], M[probe]))
342				dump(p = probe)
343			dump(n = probe) # N is the diva
344		} else if (probe in F) {
345			found = 1
346			if (!inNMP()) dump(f = probe)
347			if (diva(N[probe], F[probe], M[probe], P[probe]))
348				dump(n = probe)
349			if (diva(M[probe], F[probe], N[probe], P[probe]))
350				dump(m = probe)
351			if (diva(P[probe], F[probe], N[probe], M[probe]))
352				dump(p = probe)
353			dump(f = probe) # F is the diva
354		} else if (probe in M) {
355			found = 1
356			if (!inNFP()) dump(m = probe)
357			if (diva(N[probe], M[probe], F[probe], P[probe]))
358				dump(n = probe)
359			if (diva(F[probe], M[probe], N[probe], P[probe]))
360				dump(f = probe)
361			if (diva(P[probe], M[probe], N[probe], F[probe]))
362				dump(p = probe)
363			dump(m = probe) # M is the diva
364		} else if (probe in P) {
365			found = 1
366			if (!inNFM()) dump(p = probe)
367			if (diva(N[probe], P[probe], F[probe], M[probe]))
368				dump(n = probe)
369			if (diva(F[probe], P[probe], N[probe], M[probe]))
370				dump(f = probe)
371			if (diva(M[probe], P[probe], N[probe], F[probe]))
372				dump(m = probe)
373			dump(p = probe) # P is the diva
374		}
375		if (!found) print probe
376		exit !found
377	}
378	' # END-QUOTE
379}
380
381list_probes()
382{
383	local OPTIND=1 OPTARG flag
384	local column=0 header="PROVIDER:MODULE:FUNCTION:NAME"
385	local filter= quiet= type=
386
387	while getopts f:qt: flag; do
388		case "$flag" in
389		f) filter="$OPTARG" ;;
390		q) quiet=1 ;;
391		t) type="$OPTARG" ;;
392		esac
393	done
394	shift $(( $OPTIND - 1 ))
395
396	if [ $# -eq 0 ]; then
397		case "$type" in
398		provider) column=1 header="PROVIDER" ;;
399		module)   column=2 header="MODULE" ;;
400		function) column=3 header="FUNCTION" ;;
401		name)     column=4 header="NAME" ;;
402		esac
403	fi
404
405	[ "$quiet" ] || echo "$header"
406
407	local arg probe=
408	for arg in "$@"; do
409		arg=$( expand_probe -t "$type" -- "$arg" )
410		probe="$probe${probe:+, }$arg"
411	done
412
413	dtrace_cmd -l${probe:+n "$probe"} | awk -v pattern="$(
414		# Prevent backslashes from being lost
415		echo "$filter" | awk 'gsub(/\\/,"&&")||1'
416	)" -v want="$column" -v console="$CONSOLE" '
417		BEGIN { getline dtrace_header }
418		function ans(seq) { return console ? "\033[" seq "m" : "" }
419		NF > 3 && $(NF-1) ~ /^#/ { next }
420		!_[$0 = column[0] = sprintf("%s:%s:%s:%s",
421			column[1] = $2,
422			column[2] = (NF >= 5 ? $(NF-2) : ""),
423			column[3] = (NF >= 4 ? $(NF-1) : ""),
424			column[4] = $NF)]++ &&
425			!__[$0 = column[want]]++ &&
426			gsub(pattern, ans("31;1") "&" ans("39;22")) {
427				print | "sort"
428			}
429		END { close("sort") }
430	' # END-QUOTE
431
432	exit $SUCCESS
433}
434
435list_profiles()
436{
437	local OPTIND=1 OPTARG flag
438	local filter= oneline= quiet=
439
440	while getopts 1f:q flag; do
441		case "$flag" in
442		1) oneline=1 ;;
443		f) filter="$OPTARG" ;;
444		q) quiet=1 ;;
445		esac
446	done
447	shift $(( $OPTIND - 1 ))
448
449	# Prevent backslashes from being lost
450	filter=$( echo "$filter" | awk 'gsub(/\\/,"&&")||1' )
451
452	# Build a list of profiles available
453	local profiles
454	profiles=$( { IFS=:
455		for dir in $DWATCH_PROFILES_PATH; do
456			[ -d "$dir" ] || continue
457			for path in $dir/*; do
458				[ -f "$path" ] || continue
459				name="${path##*/}"
460				[ "$name" = "${name%%[!0-9A-Za-z_-]*}" ] ||
461					continue
462				echo $name
463			done
464		done
465	} | sort -u )
466
467	# Get the longest profile name
468	local longest_profile_name
469	longest_profile_name=$( echo "$profiles" |
470		awk -v N=0 '(L = length($0)) > N { N = L } END { print N }' )
471
472	# Get the width of the terminal
473	local max_size="$( stty size 2> /dev/null )"
474	: ${max_size:=24 80}
475	local max_width="${max_size#*[$IFS]}"
476
477	# Determine how many columns we can display
478	local x=$longest_profile_name ncols=1
479	[ "$QUIET" ] || x=$(( $x + 8 )) # Accommodate leading tab character
480	x=$(( $x + 3 + $longest_profile_name )) # Preload end of next column
481	while [ $x -lt $max_width ]; do
482		ncols=$(( $ncols + 1 ))
483		x=$(( $x + 3 + $longest_profile_name ))
484	done
485
486	# Output single lines if sent to a pipe
487	if [ "$oneline" ]; then
488		echo "$profiles" | awk -v filter="$filter" -v cons="$CONSOLE" '
489			function ans(s) { return cons ? "\033[" s "m" : "" }
490			gsub(filter, ans("31;1") "&" ans("39;22"))
491		' # END-QUOTE
492		return $SUCCESS
493		# NOTREACHED
494	fi
495
496	[ "$quiet" ] || echo PROFILES:
497	echo "$profiles" | awk \
498		-v colsize=$longest_profile_name \
499		-v console="$CONSOLE" \
500		-v ncols=$ncols \
501		-v quiet="$quiet" \
502		-v filter="$filter" \
503	' # Begin awk(1) processor
504		function ans(seq) { return console ? "\033[" seq "m" : "" }
505		BEGIN {
506			row_item[1] = ""
507			replace = ans("31;1") "&" ans("39;22")
508			ansi_offset = length(replace) - 1
509		}
510		function print_row()
511		{
512			cs = colsize + ansi_offset * \
513				gsub(filter, replace, row_item[1])
514			printf "%s%-*s", quiet ? "" : "\t", cs, row_item[1]
515			for (i = 2; i <= cur_col; i++) {
516				cs = colsize + ansi_offset * \
517					gsub(filter, replace, row_item[i])
518				printf "   %-*s", cs, row_item[i]
519			}
520			printf "\n"
521		}
522		$0 ~ filter {
523			n++
524			cur_col = ((n - 1) % ncols) + 1
525			row_item[cur_col] = $0
526			if (cur_col == ncols) print_row()
527		}
528		END { if (cur_col < ncols) print_row() }
529	' # END-QUOTE
530
531	exit $SUCCESS
532}
533
534load_profile()
535{
536	local profile="$1"
537
538	[ "$profile" ] ||
539		die "missing profile argument (\`$pgm -Q' to list profiles)"
540
541	local oldIFS="$IFS"
542	local dir found=
543
544	IFS=:
545	for dir in $DWATCH_PROFILES_PATH; do
546		[ -d "$dir" ] || continue
547		[ -f "$dir/$profile" ] || continue
548		PROFILE="$profile" found=1
549		info "Sourcing $profile profile [found in %s]" "$dir"
550		. "$dir/$profile"
551		break
552	done
553	IFS="$oldIFS"
554
555	[ "$found" ] ||
556		die "no module named \`$profile' (\`$pgm -Q' to list profiles)"
557}
558
559pproc()
560{
561	local OPTIND=1 OPTARG flag
562	local P= N=0
563
564	while getopts P: flag; do
565		case "$flag" in
566		P) P="$OPTARG" ;;
567		esac
568	done
569	shift $(( OPTIND - 1 ))
570
571	local proc=$1
572	if [ ! "$proc" ]; then
573		if [ "$P" = "0" ]; then
574			proc="curthread->td_proc"
575		else
576			proc="this->proc ? this->proc->p_pptr : NULL"
577		fi
578	fi
579
580	awk 'NR > 1 && $0 { $0 = "\t" $0 }
581		gsub(/\\\t/, "\t") || 1
582	' <<-EOFPREAMBLE
583	this->proc = $proc;
584	this->uid$P = this->proc ? this->proc->p_ucred->cr_uid : -1;
585	this->gid$P = this->proc ? this->proc->p_ucred->cr_rgid : -1;
586	this->pid$P = this->proc ? this->proc->p_pid : -1;
587	this->jid$P = this->proc ? this->proc->p_ucred->cr_prison->pr_id : -1;
588
589	this->p_args = this->proc ? this->proc->p_args : 0;
590	this->ar_length = this->p_args ? this->p_args->ar_length : 0;
591	this->ar_args = (char *)(this->p_args ? this->p_args->ar_args : 0);
592
593	this->args$P = this->arg${P}_$N = this->ar_length > 0 ?
594	\	this->ar_args : stringof(this->proc->p_comm);
595	this->len = this->ar_length > 0 ? strlen(this->ar_args) + 1 : 0;
596	this->ar_args += this->len;
597	this->ar_length -= this->len;
598
599	EOFPREAMBLE
600
601	awk -v P=$P -v MAX_ARGS=$MAX_ARGS '
602		$0 { $0 = "\t" $0 }
603		buf = buf $0 "\n" { }
604		END {
605			while (++N <= MAX_ARGS) {
606				$0 = buf
607				gsub(/P/, P)
608				gsub(/N/, N)
609				gsub(/\\\t/, "\t")
610				sub(/\n$/, "")
611				print
612			}
613		}
614	' <<-EOFARGS
615	this->argP_N = this->ar_length > 0 ? this->ar_args : "";
616	this->argsP = strjoin(this->argsP,
617	\	strjoin(this->argP_N != "" ? " " : "", this->argP_N));
618	this->len = this->ar_length > 0 ? strlen(this->ar_args) + 1 : 0;
619	this->ar_args += this->len;
620	this->ar_length -= this->len;
621
622	EOFARGS
623
624	N=$(( $MAX_ARGS + 1 ))
625	awk 'sub(/^\\\t/, "\t") || 1, $0 = "\t" $0' <<-EOFPROC
626	this->arg${P}_$N = this->ar_length > 0 ? "..." : "";
627	this->args$P = strjoin(this->args$P,
628	\	strjoin(this->arg${P}_$N != "" ? " " : "", this->arg${P}_$N));
629	EOFPROC
630}
631
632pproc_dump()
633{
634	local OPTIND=1 OPTARG flag
635	local verbose=
636
637	while getopts v flag; do
638		case "$flag" in
639		v) verbose=1 ;;
640		esac
641	done
642	shift $(( $OPTIND - 1 ))
643
644	local P=$1
645	if [ "$verbose" ]; then
646		awk -v P=$P '
647			BEGIN { printf "\t" }
648			NR > 1 && $0 { $0 = "\t" $0 }
649			buf = buf $0 "\n" { }
650			END {
651				$0 = buf
652				if (P < 3) S = sprintf("%" 7-2*(P+1) "s", "")
653				gsub(/S/, S)
654				gsub(/B/, P < 3 ? "\\" : "")
655				gsub(/\\\t/, "\t")
656				sub(/\n$/, "")
657				print
658			}
659		' <<-EOFPREAMBLE
660		printf(" SB-+= %05d %d.%d %s\n",
661		\	this->pid$P, this->uid$P, this->gid$P, this->args$P);
662		EOFPREAMBLE
663	else
664		cat <<-EOFPREAMBLE
665		printf("%s", this->args$P);
666		EOFPREAMBLE
667	fi
668}
669
670############################################################ MAIN
671
672# If we're running as root, no need for sudo(8)
673[ "$( id -u )" != 0 ] && type sudo > /dev/null 2>&1 && SUDO=sudo
674
675#
676# Process command-line options
677#
678while getopts 1B:deE:fFg:j:k:K:lmnN:o:O:p:PqQr:Rt:T:u:vVwxX:yz: flag; do
679	case "$flag" in
680	1) ONELINE=1 PSTREE= ;;
681	B) MAX_ARGS="$OPTARG" ;;
682	d) DEBUG=1 ;;
683	e) EXIT_AFTER_COMPILE=1 ;;
684	E) CUSTOM_DETAILS=1
685	   EVENT_DETAILS="${EVENT_DETAILS%;}"
686	   [ "$EVENT_DETAILS" ] && EVENT_DETAILS="$EVENT_DETAILS;
687		printf(\" \");
688		" # END-QUOTE
689	   # Read event code from stdin if `-' is argument
690	   [ "$OPTARG" = "-" ] && OPTARG=$( cat )
691	   EVENT_DETAILS="$EVENT_DETAILS$OPTARG" ;;
692	f) PROBE_TYPE=function ;;
693	F) PROBE_COALESCE=1 ;;
694	g) GROUP="$OPTARG" ;;
695	j) JID="$OPTARG" ;;
696	k) EXECNAME="$EXECNAME${EXECNAME:+ }$OPTARG"
697	   case "$OPTARG" in
698	   \**\*) name="${OPTARG%\*}"
699		predicate="strstr(execname, \"${name#\*}\") != NULL" ;;
700	   \**) name="${OPTARG#\*}"
701		predicate="strstr(execname, \"$name\") == (execname +"
702		predicate="$predicate strlen(execname) - ${#name})" ;;
703	   *\*) predicate="strstr(execname, \"${OPTARG%\*}\") == execname" ;;
704	   *) predicate="execname == \"$OPTARG\""
705	   esac
706	   EVENT_TEST="$predicate${EVENT_TEST:+ ||
707		($EVENT_TEST)}" ;;
708	K) MAX_DEPTH="$OPTARG" ;;
709	l) LIST=1 ;;
710	m) PROBE_TYPE=module ;;
711	n) PROBE_TYPE=name ;;
712	N) COUNT="$OPTARG" ;;
713	o) OUTPUT="$OPTARG" ;;
714	O) OUTPUT_CMD="$OPTARG" ;;
715	p) PID="$OPTARG" ;;
716	P) PROBE_TYPE=provider ;;
717	q) QUIET=1 ;;
718	Q) LIST_PROFILES=1 ;;
719	r) FILTER="$OPTARG" ;;
720	R) PSTREE=1 ;;
721	t) CUSTOM_TEST="${CUSTOM_TEST:+($CUSTOM_TEST) && }$OPTARG" ;;
722	T) TIMEOUT="$OPTARG" ;;
723	u) USER="$OPTARG" ;;
724	v) VERBOSE=1 ;;
725	V) vers="${VERSION#\$*[:\$]}"
726	   vers="${vers% \$}"
727	   printf "%s: %s\n" "$pgm" "${vers# }"
728	   exit ;;
729	w) DESTRUCTIVE_ACTIONS=1 ;;
730	x) TRACE=1 ;;
731	X) USE_PROFILE=1 PROFILE="$OPTARG" ;;
732	y) CONSOLE=1 CONSOLE_FORCE=1 ;;
733	z) EXECREGEX="$OPTARG" ;;
734	*) usage
735	   # NOTREACHED
736	esac
737done
738shift $(( $OPTIND - 1 ))
739
740#
741# List probes if `-l' was given
742#
743[ "$LIST" ] &&
744	list_probes -f "$FILTER" ${QUIET:+-q} -t "$PROBE_TYPE" -- "$@"
745	# NOTREACHED
746
747#
748# List profiles if `-Q' was given
749#
750[ "$LIST_PROFILES" ] &&
751	list_profiles ${ONELINE:+-1} -f "$FILTER" ${QUIET:+-q}
752	# NOTREACHED
753
754#
755# Validate number of arguments
756#
757if [ ! "$PROFILE" ]; then
758	# If not given `-X profile' then a probe argument is required
759	[ $# -gt 0 ] || usage # NOTREACHED
760fi
761
762#
763# Validate `-N count' option argument
764#
765case "$COUNT" in
766"") usage "-N option requires a number argument" ;; # NOTREACHED
767*[!0-9]*) usage "-N argument must be a number" ;; # NOTREACHED
768esac
769
770#
771# Validate `-B num' option argument
772#
773case "$MAX_ARGS" in
774"") usage "-B option requires a number argument" ;; # NOTREACHED
775*[!0-9]*) usage "-B argument must be a number" ;; # NOTREACHED
776esac
777
778#
779# Validate `-K num' option argument
780#
781case "$MAX_DEPTH" in
782"") usage "-K option requires a number argument" ;; # NOTREACHED
783*[!0-9]*) usage "-K argument must be a number" ;; # NOTREACHED
784esac
785
786#
787# Validate `-j jail' option argument
788#
789case "$JID" in
790"") : fall through ;;
791*[!0-9]*) JID=$( jls -j "$JID" jid ) || exit ;;
792esac
793
794#
795# Validate `-u user' option argument
796#
797case "$USER" in
798"") : fall through ;;
799*[![:alnum:]_-]*) RUID="$USER" ;;
800*[!0-9]*) RUID=$( id -u "$USER" 2> /dev/null ) || die "No such user: $USER" ;;
801*) RUID=$USER
802esac
803
804#
805# Validate `-g group' option argument
806#
807case "$GROUP" in
808"") : fall-through ;;
809*[![:alnum:]_-]*) RGID="$GROUP" ;;
810*[!0-9]*)
811	RGID=$( getent group | awk -F: -v group="$GROUP" '
812		$1 == group { print $3; exit found=1 }
813		END { exit !found }
814	' ) || die "No such group: $GROUP" ;;
815*) RGID=$GROUP
816esac
817
818#
819# Expand probe argument into probe(s)
820#
821case "$1" in
822-*) : Assume dtrace options such as "-c cmd" or "-p pid" ;; # No probe(s) given
823*)
824	PROBE_ARG="$1"
825	shift
826esac
827if [ "$PROBE_ARG" ]; then
828	oldIFS="$IFS"
829	IFS="$IFS,"
830	for arg in $PROBE_ARG; do
831		arg=$( expand_probe -t "$PROBE_TYPE" -- "$arg" )
832		PROBE="$PROBE${PROBE:+, }$arg"
833	done
834	IFS="$oldIFS"
835fi
836
837#
838# Set default event details if `-E code' was not given
839#
840[ "$CUSTOM_DETAILS" ] || EVENT_DETAILS=$( pproc_dump 0 )
841
842#
843# Load profile if given `-X profile'
844#
845[ "$USE_PROFILE" ] && load_profile "$PROFILE"
846[ "$PROBE" ] || die "PROBE not defined by profile and none given as argument"
847
848#
849# Show the user what's being watched
850#
851[ "$DEBUG$QUIET$EXIT_AFTER_COMPILE" ] || info "Watching '$PROBE' ..."
852
853#
854# Header for watched probe entry
855#
856case "$PROBE" in
857*,*) : fall-through ;;
858*:execve:entry|execve:entry)
859	ACTIONS=$( awk 'gsub(/\\\t/, "\t") || 1' <<-EOF
860		$PROBE /* probe ID $ID */
861		{${TRACE:+
862		\	printf("<$ID>");}
863		\	this->caller_execname = execname;
864		}
865		EOF
866	)
867	PROBE="${PROBE%entry}return"
868	ID=$(( $ID + 1 ))
869	EVENT_TEST="execname != this->caller_execname${EVENT_TEST:+ &&
870		($EVENT_TEST)}"
871	EVENT_TAG='printf("%d.%d %s[%d]: ",
872		this->uid1, this->gid1, this->caller_execname, this->pid1);'
873	;;
874esac
875
876#
877# Jail clause/predicate
878#
879if [ "$JID" ]; then
880	prison_id="curthread->td_proc->p_ucred->cr_prison->pr_id"
881	EVENT_TEST="$prison_id == $JID${EVENT_TEST:+ &&
882		($EVENT_TEST)}"
883fi
884
885#
886# Custom test clause/predicate
887#
888if [ "$CUSTOM_TEST" ]; then
889	case "$EVENT_TEST" in
890	"") EVENT_TEST="$CUSTOM_TEST" ;;
891	 *) EVENT_TEST="$EVENT_TEST &&
892		($CUSTOM_TEST)"
893	esac
894fi
895
896#
897# Make sure dynamic code has trailing semi-colons if non-NULL
898#
899EVENT_TAG="${EVENT_TAG%;}${EVENT_TAG:+;}"
900EVENT_DETAILS="${EVENT_DETAILS%;}${EVENT_DETAILS:+;}"
901
902#
903# DTrace script
904#
905# If `-d' is given, script is sent to stdout for debugging
906# If `-c count", `-g group', `-r regex', or `-u user' is given, run script with
907# dtrace and send output to awk(1) post-processor (making sure to preserve the
908# exit code returned by dtrace invocation). Otherwise, simply run script with
909# dtrace and then exit.
910#
911exec 9<<EOF
912$PROBE /* probe ID 2 */
913{${TRACE:+
914	printf("<2>");
915}
916	/*
917	 * Examine process, parent process, and grandparent process details
918	 */
919
920	/******************* CURPROC *******************/
921
922	$( pproc -P0 )
923
924	/******************* PPARENT *******************/
925
926	$( if [ "$PSTREE" ]; then pproc -P1; else echo -n \
927	"this->proc = this->proc ? this->proc->p_pptr : NULL;
928	this->pid1 = this->proc ? this->proc->p_pid : -1;
929	this->uid1 = this->proc ? this->proc->p_ucred->cr_uid : -1;
930	this->gid1 = this->proc ? this->proc->p_ucred->cr_rgid : -1;
931	this->jid1 = this->proc ? this->proc->p_ucred->cr_prison->pr_id : -1;"
932	fi )
933
934	/******************* GPARENT *******************/
935
936	$( [ "$PSTREE" ] && pproc -P2 )
937
938	/******************* APARENT *******************/
939
940	$( [ "$PSTREE" ] && pproc -P3 )
941}
942EOF
943PSARGS_ACTION=$( cat <&9 )
944[ "$OUTPUT" -a ! "$CONSOLE_FORCE" ] && CONSOLE=
945{
946	if [ "$DEBUG" ]; then
947		# Send script to stdout
948		cat
949		exit
950	fi
951
952	if [ "$CUSTOM_TEST$EXECNAME$JID$OUTPUT$TIMEOUT$TRACE$VERBOSE" -a \
953	    ! "$QUIET" ]
954	then
955		msg=Setting
956		[ "$CUSTOM_TEST" ] && msg="$msg test: $CUSTOM_TEST"
957		[ "$EXECNAME" ] && msg="$msg execname: $EXECNAME"
958		[ "$JID" ] && msg="$msg jid: $JID"
959		[ "$OUTPUT" ] && msg="$msg output: $OUTPUT"
960		[ "$TIMEOUT" ] && msg="$msg timeout: $TIMEOUT"
961		[ "$TRACE" ] && msg="$msg trace: $TRACE"
962		[ "$VERBOSE" ] && msg="$msg verbose: $VERBOSE"
963		info "$msg"
964	fi
965
966	exec 3>&1
967	console_stdout=3
968
969	if [ $COUNT -eq 0 -a ! "$EXECREGEX$FILTER$GROUP$OUTPUT_CMD$PID$USER" ]
970	then
971		case "$OUTPUT" in
972		-) output_path=/dev/stdout ;;
973		*) output_path="$OUTPUT"
974		esac
975
976		# Run script without pipe to awk post-processor
977		dtrace_cmd -t \
978			${DESTRUCTIVE_ACTIONS:+-w} \
979			${EXIT_AFTER_COMPILE:+-e} \
980			${OUTPUT:+-o "$output_path"} \
981			-s /dev/stdin \
982			"$@"
983		exit
984	fi
985
986	# Prevent backslashes from being lost
987	FILTER=$( echo "$FILTER" | awk 'gsub(/\\/,"&&")||1' )
988	EXECREGEX=$( echo "$EXECREGEX" | awk 'gsub(/\\/,"&&")||1' )
989
990	if [ ! "$QUIET" ]; then
991		msg=Filtering
992		[ "$EXECREGEX" ] && msg="$msg execregex: $EXECREGEX"
993		[ "$FILTER" ] && msg="$msg filter: $FILTER"
994		[ "$GROUP" ] && msg="$msg group: $GROUP"
995		[ "$OUTPUT_CMD" ] && msg="$msg cmd: $OUTPUT_CMD"
996		[ "$PID" ] && msg="$msg pid: $PID"
997		[ "$USER" ] && msg="$msg user: $USER"
998		[ $COUNT -gt 0 ] && msg="$msg count: $COUNT"
999		info "$msg"
1000	fi
1001
1002	#
1003	# Send script output to post-processor for filtering
1004	#
1005	status=$(
1006		exec 4>&1
1007		to_status=4
1008		( exec 5>&1; to_dtrace_stderr_filter=5; (
1009			trap 'echo $? >&$to_status' EXIT
1010			eval $SUDO ${TIMEOUT:+timeout \"\$TIMEOUT\"} dtrace \
1011				${EXIT_AFTER_COMPILE:+-e} \
1012				${DESTRUCTIVE_ACTIONS:+-w} \
1013				-s /dev/stdin \
1014				\"\$@\" \
1015				2>&$to_dtrace_stderr_filter \
1016				${QUIET:+2> /dev/null}
1017		) | $SUDO awk \
1018			-v cmd="$OUTPUT_CMD" \
1019			-v console="$CONSOLE" \
1020			-v count=$COUNT \
1021			-v execregex="$EXECREGEX" \
1022			-v filter="$FILTER" \
1023			-v gid="$RGID" \
1024			-v output="$OUTPUT" \
1025			-v pid="$PID" \
1026			-v pstree=$PSTREE \
1027			-v quiet=$QUIET \
1028			-v tty=$( ps -o tty= -p $$ ) \
1029			-v uid="$RUID" \
1030		' # Start awk(1) post-processor
1031		############################################ BEGIN
1032		BEGIN {
1033			true = 1
1034			ansi = "(\\033\\[[[:digit:];]+m)?"
1035			num = year = day = "[[:digit:]]+"
1036			month = "[[:alpha:]]+"
1037			date = year " " month " +" day
1038			time = "[012][0-9]:[0-5][0-9]:[0-5][0-9]"
1039			date_time = ansi date " +" time ansi
1040			name1 = "[^\\[]*"
1041			name2 = "[^\\n]*"
1042			if (output == "-")
1043				output = "/dev/stdout"
1044
1045			#
1046			# Field definitions
1047			#
1048			nexecmatches = 2
1049			execstart[1] = sprintf( \
1050				"^(%s) (%s)\\.(%s) (%s)\\[(%s)\\]: ",
1051				date_time, num, num, name1, num)
1052			execstart[2] = sprintf( \
1053				"\\n +\\\\?-\\+= (%s) (%s)\\.(%s) ",
1054				num, num, num)
1055			npidmatches = 2
1056			pidstart[1] = sprintf("^(%s) (%s)\\.(%s) (%s)\\[",
1057				date_time, num, num, name1)
1058			pidstart[2] = "\\n +\\\\?-\\+= "
1059			pidpreen[2] = "^0*"
1060			piddeflt[2] = "0"
1061			ngidmatches = 2
1062			gidstart[1] = sprintf("^(%s) (%s)\\.", date_time, num)
1063			gidstart[2] = sprintf("\\n +\\\\?-\\+= (%s) (%s)\\.",
1064				ansi num ansi, num)
1065			nuidmatches = 2
1066			uidstart[1] = sprintf("^(%s) ", date_time)
1067			uidstart[2] = sprintf("\\n +\\\\?-\\+= (%s) ",
1068				ansi num ansi)
1069		}
1070		############################################ FUNCTIONS
1071		function strip(s) { gsub(/\033\[[0-9;]*m/, "", s); return s }
1072		function esc(str) { gsub(/'\''/, "&\\\\&&", str); return str }
1073		function arg(str) { return "'\''" esc(str) "'\''" }
1074		function env(var, str) { return var "=" arg(str) " " }
1075		function ans(seq) { return console ? "\033[" seq "m" : "" }
1076		function runcmd() {
1077			return system(sprintf("%s/bin/sh -c %s",
1078				env("TAG", strip(tag)) \
1079					env("DETAILS", strip(details)),
1080				arg(cmd)))
1081		}
1082		function filter_block() {
1083			if (length(lines) < 1) return 0
1084			block_match = 0
1085			newstr = ""
1086			start = 1
1087			if (match(lines, "^(" date_time ") ")) {
1088				newstr = newstr substr(lines, 1,
1089					RSTART + RLENGTH - 1)
1090				start = RSTART + RLENGTH
1091			}
1092			replace = ans("31;1") "&" ans("39;22")
1093			workstr = substr(lines, start)
1094			if (gsub(filter, replace, workstr)) block_match = 1
1095			lines = newstr workstr
1096			return block_match
1097		}
1098		function filter_field(startre, fieldre, matchre, isword,
1099			preenre, defaultstr)
1100		{
1101			if (length(lines) < 1) return 0
1102			field_match = 0
1103			newstr = ""
1104			start = 1
1105			while ((workstr = substr(lines, start)) &&
1106				(workstr ~ (startre fieldre)))
1107			{
1108				match(workstr, startre)
1109				start += end = RSTART + RLENGTH - 1
1110				newstr = newstr substr(workstr, 1, end)
1111				workstr = substr(workstr, end + 1)
1112				match(workstr, fieldre)
1113				start += end = RSTART + RLENGTH - 1
1114				field = matchstr = substr(workstr, 1, end)
1115				sub(preenre, "", matchstr)
1116				if (!matchstr) matchstr = defaultstr
1117				if (isword) {
1118					if (match(matchstr, matchre) &&
1119						RSTART == 1 &&
1120						RLENGTH == length(matchstr)) {
1121						field_match = 1
1122						field = ans(7) field ans(27)
1123					}
1124				} else {
1125					replace = ans(7) "&" ans(27)
1126					if (gsub(matchre, replace, matchstr)) {
1127						field_match = 1
1128						field = matchstr
1129					}
1130				}
1131				newstr = newstr field
1132			}
1133			lines = newstr workstr
1134			return field_match
1135		}
1136		function dump() {
1137			lines = block
1138			block = ""
1139			found = 0
1140			if (execregex != "") {
1141				for (n = 1; n <= nexecmatches; n++)
1142					if (filter_field(execstart[n], name2,
1143						execregex)) found = 1
1144				if (!found) return
1145			}
1146			if (pid != "") {
1147				for (n = 1; n <= npidmatches; n++)
1148					if (filter_field(pidstart[n], num, pid,
1149						true, pidpreen[n],
1150						piddeflt[n])) found = 1
1151				if (!found) return
1152			}
1153			if (gid != "") {
1154				for (n = 1; n <= ngidmatches; n++)
1155					if (filter_field(gidstart[n], num,
1156						gid, true)) found = 1
1157				if (!found) return
1158			}
1159			if (uid != "") {
1160				for (n = 1; n <= nuidmatches; n++)
1161					if (filter_field(uidstart[n], num,
1162						uid, true)) found = 1
1163				if (!found) return
1164			}
1165			if (filter != "" && !filter_block()) return
1166			if (lines) {
1167				stdout = 1
1168				if (output) {
1169					stdout = 0
1170					if (!console) lines = strip(lines)
1171					print lines > output
1172				} else if (cmd) {
1173					if (!quiet) print lines
1174					tag = details = lines
1175					sub(/: .*/, "", tag)
1176					sub(/.*: /, "", details)
1177					if (!console) tag = strip(tag)
1178					runcmd()
1179				} else print lines
1180			}
1181			fflush()
1182			++matches
1183		}
1184		############################################ MAIN
1185		{ block = (block ? block "\n" : block) $0 }
1186		!pstree { dump() }
1187		$0 ~ sprintf("^%6s\\\\-\\+= %s ", "", num) { dump() }
1188		count && matches >= count { exit }
1189		############################################ END
1190		END {
1191			dump()
1192			system(sprintf("pkill -t %s dtrace %s", tty,
1193				quiet ? "2> /dev/null" : ""))
1194		}
1195		' >&$console_stdout ) | dtrace_stderr_filter >&2
1196	) # status
1197	exit $status
1198
1199} <<EOF
1200#!/usr/sbin/dtrace -s
1201/* -
1202 * Copyright (c) 2014-2018 Devin Teske <dteske@FreeBSD.org>
1203 * All rights reserved.
1204 * Redistribution and use in source and binary forms, with or without
1205 * modification, are permitted provided that the following conditions
1206 * are met:
1207 * 1. Redistributions of source code must retain the above copyright
1208 *    notice, this list of conditions and the following disclaimer.
1209 * 2. Redistributions in binary form must reproduce the above copyright
1210 *    notice, this list of conditions and the following disclaimer in the
1211 *    documentation and/or other materials provided with the distribution.
1212 *
1213 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS \`\`AS IS'' AND
1214 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1215 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1216 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
1217 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
1218 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
1219 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
1220 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
1221 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
1222 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
1223 * SUCH DAMAGE.
1224 *
1225 * $TITLE dtrace(1) script to log process(es) triggering $PROBE $
1226 * \$FreeBSD: head/cddl/usr.sbin/dwatch/dwatch 330559 2018-03-06 23:44:19Z dteske $
1227 */
1228
1229$( echo "$DTRACE_PRAGMA" | awk '
1230	!/^[[:space:]]*(#|$)/, sub(/^[[:space:]]*/, "#pragma D ")||1
1231' )
1232
1233int console;
1234
1235dtrace:::BEGIN { console = ${CONSOLE:-0} } /* probe ID 1 */
1236
1237/*********************************************************/
1238
1239${PSARGS:+$PSARGS_ACTION}
1240${ACTIONS:+
1241/*********************************************************/
1242
1243$ACTIONS
1244}
1245/*********************************************************/
1246
1247$PROBE${EVENT_TEST:+ /$EVENT_TEST/} /* probe ID $ID */
1248{${TRACE:+
1249	printf("<$ID>");
1250}
1251	/***********************************************/
1252
1253	printf("%s%Y%s ",
1254		console ? "\033[32m" : "",
1255		walltimestamp,
1256		console ? "\033[39m" : "");
1257
1258	/****************** EVENT_TAG ******************/
1259
1260	${EVENT_TAG#[[:space:]]}
1261${PROBE_COALESCE:+
1262	/**************** PROBE_COALESCE ***************/
1263
1264	printf("%s%s:%s:%s:%s ", probename == "entry" ? "-> " :
1265			probename == "return" ? "<- " :
1266			probename == "start" ? "-> " :
1267			probename == "done" ? "<- " : " | ",
1268		probeprov, probemod, probefunc, probename);
1269}
1270	/**************** EVENT_DETAILS ****************/
1271
1272	${EVENT_DETAILS#[[:space:]]}
1273
1274	/***********************************************/
1275
1276	printf("\\n");
1277${PSTREE:+
1278	/*
1279	 * Print process, parent, grandparent, and ancestor details
1280	 */
1281$(	pproc_dump -v 3
1282	pproc_dump -v 2
1283	pproc_dump -v 1
1284	pproc_dump -v 0
1285)}
1286}
1287EOF
1288
1289################################################################################
1290# END
1291################################################################################
1292