1.fp 5 CW
2.de Af
3.ds ;G \\*(;G\\f\\$1\\$3\\f\\$2
4.if !\\$4 .Af \\$2 \\$1 "\\$4" "\\$5" "\\$6" "\\$7" "\\$8" "\\$9"
5..
6.de aF
7.ie \\$3 .ft \\$1
8.el \{\
9.ds ;G \&
10.nr ;G \\n(.f
11.Af "\\$1" "\\$2" "\\$3" "\\$4" "\\$5" "\\$6" "\\$7" "\\$8" "\\$9"
12\\*(;G
13.ft \\n(;G \}
14..
15.de L
16.aF 5 \\n(.f "\\$1" "\\$2" "\\$3" "\\$4" "\\$5" "\\$6" "\\$7"
17..
18.de LR
19.aF 5 1 "\\$1" "\\$2" "\\$3" "\\$4" "\\$5" "\\$6" "\\$7"
20..
21.de RL
22.aF 1 5 "\\$1" "\\$2" "\\$3" "\\$4" "\\$5" "\\$6" "\\$7"
23..
24.de EX		\" start example
25.ta 1i 2i 3i 4i 5i 6i
26.PP
27.RS 
28.PD 0
29.ft 5
30.nf
31..
32.de EE		\" end example
33.fi
34.ft
35.PD
36.RE
37.PP
38..
39.TH TOK 3
40.SH NAME
41tok \- space separated token stream routines
42.SH SYNOPSIS
43.L "#include <ast.h>"
44.sp
45.L "void* tokopen(char* string)"
46.L "char* tokread(void* tok)"
47.L "void tokclose(void* tok)"
48.sp
49.L "int tokscan(char* string, char** next, const char* format, ...);"
50.sp
51.L "Sfio_t* tokline(const char* input, int flags, int* line);"
52.SH DESCRIPTION
53.L tokopen
54returns a pointer to a space separated token stream on the 0 terminated
55string
56.LR string .
57.L tokread
58returns a pointer to the next
59space separated token in the token stream
60.L tok
61as returned by
62.LR tokopen .
630 is returned when no tokens remain.
64.L tokread
65temporarily modifies
66.L string
67by inserting 0's to terminate each token.
68.L tokclose
69closes the token stream and restores
70.L string
71to its original state.
72.PP
73.L tokscan
74scans the string
75.L string
76for tokens specified in
77.LR format .
78It is a more forgiving
79.IR sscanf (3).
80If
81.L "next != 0"
82then it will point to the next unread character in
83.L string
84on return.
85The number of scanned tokens is returned.
86.L \-1
87is returned if 
88.L string
89was not empty and
90.L format
91failed to match and tokens.
92.PP
93.I space
94in
95.L format
96matches 0 or more
97.I space
98or
99.I tab
100characters.
101.I newline
102in format eats the remainder of the current line in
103.LR string .
104"...", '...' and \e\fIcharacter\fP quotes are interpreted.
105A quoted
106.I carriage-return
107is converted to
108.IR newline .
109.I newline
110in
111.L string
112is equivalent to end of string except when quoted.
113.I \enewline
114is a line splice.
115.PP
116.L %
117in
118.L format
119prefixes format conversion characters; each conversion character 
120corresponds to a
121.L tokscan
122argument following the
123.L format
124argument.
125The format conversions are:
126.TP
127.L %c
128A single 
129.LR char .
130.TP
131.L "%hd %d %ld"
132[short, int, long] base 10 integer.
133.TP
134.L "%hn %n %ln"
135[short, int, long] C-style base integer.
136.TP
137.L "%ho %o %lo"
138[short, int, long] base 8 integer.
139.TP
140.L %s
141String.
142.TP
143.L "%hu %u %lu"
144[short, int, long] C-style base unsigned integer.
145.TP
146.L %v
147The next two arguments are a pointer to a
148.L char**
149argument vector and the maximum number of elements in the vector.
150.TP
151.L "%hx %x %lx"
152[short, int, long] base 16 integer.
153.PP
154.L %s
155and
156.L %v
157data may also be counted length strings of the form
158\f5(\fIcount\fP:\fIdata\fP)\fR
159where
160.I count
161is the number of characters in
162.I data
163and the terminating
164.L )
165may also be a
166.IR tab ,
167or the data may be
168.L (null)
169which represents the
170.L NULL
171string.
172.PP
173.L tokline
174returns an
175.IR sfio (3)
176stream to a file or string that splices
177.I \enewline
178into single lines,
179allows "..." and '...' to quotes to span
180.I newlines
181(done by translating quoted
182.I newline
183to
184.IR carriage-return ;
185.L tokscan
186above converts quoted
187.I carriage-return
188back to 
189.IR newline ),
190and deletes
191.I "# ... newline"
192comments.
193This is done by pushing an
194.I sfio
195discipline onto a string or file stream.
196Seeks are disabled on the resulting stream.
197If
198.L "flags == SF_READ"
199then
200.L input
201is a file name;
202If
203.L "flags == SF_STRING"
204then
205.L input
206is a 0 terminated string;
207otherwise
208.L input
209is an open 
210.L Sfio_t*
211stream.
212If
213.L "line != 0"
214then it points to a line count that is initialized to 0
215and is incremented for each input line.
216.SH "SEE ALSO"
217sfio(3)
218