Lines Matching refs:token
322 /* A token consists of a sequence of characters. */
323 struct token
327 char *chars; /* the token's constituents */
330 /* Initialize a 'struct token'. */
332 init_token (struct token *tp)
339 /* Free the memory pointed to by a 'struct token'. */
341 free_token (struct token *tp)
346 /* Ensure there is enough room in the token for one more character. */
348 grow_token (struct token *tp)
428 struct token *token; /* for t_string */
438 free_token (wp->token);
439 free (wp->token);
443 /* Convert a t_string token to a char*. */
452 n = wp->token->charcount;
454 memcpy (str, wp->token->chars, n);
578 Return the first character past the token. */
669 grow_token (wp->token);
670 wp->token->chars[wp->token->charcount++] = '$';
695 grow_token (wp->token);
696 wp->token->chars[wp->token->charcount++] = utf8buf[i];
703 grow_token (wp->token);
704 wp->token->chars[wp->token->charcount++] = (unsigned char) c;
739 precede it, with no non-whitespace token on a line between both. */
781 wp->token = (struct token *) xmalloc (sizeof (struct token));
782 init_token (wp->token);
805 free_token (wp->token);
806 free (wp->token);