Lines Matching refs:tokens
272 my ($tokens, $block);
288 $tokens = $self->split_text($text)
294 $block = $self->_parse($tokens, $info);
327 my @tokens = ();
330 return \@tokens ## RETURN ##
405 push(@tokens, $interp
428 # [ $dirtext, $line_no(s), \@tokens ]
429 push(@tokens,
444 push(@tokens, $interp
449 return \@tokens; ## RETURN ##
463 my @tokens = ();
485 push(@tokens, 'TEXT', $pre);
490 push(@tokens, [ $dir, $line, $self->tokenise_directive($var) ]);
495 push(@tokens, 'TEXT', $dir);
499 return \@tokens;
511 # The method splits the directive into individual tokens as recognised
513 # constructs a list of tokens each represented by 2 elements, as per
533 my @tokens = ( );
593 push(@tokens, ('"') x 2,
618 # set, unless (we've got some preceding tokens and) the previous
622 ($anycase && (! @tokens || $tokens[-2] ne 'DOT'))
640 push(@tokens, $type, $token);
646 # print STDERR "tokenise directive() returning:\n [ @tokens ]\n"
649 return \@tokens; ## RETURN ##
734 # _parse(\@tokens, \@info)
736 # Parses the list of input tokens passed by reference and returns a
749 my ($self, $tokens, $info) = @_;
788 while (! defined $token && @$tokens) {
789 $token = shift(@$tokens);
796 # frightened. We're pushing parse tokens onto the
802 unshift(@$tokens,
819 unshift(@$tokens, @$token, (';') x 2);
830 unshift(@$tokens,
840 $value = shift(@$tokens);