1##
2# Collection of methods for writing parsers against RDoc::RubyLex and
3# RDoc::RubyToken
4
5module RDoc::Parser::RubyTools
6
7  include RDoc::RubyToken
8
9  ##
10  # Adds a token listener +obj+, but you should probably use token_listener
11
12  def add_token_listener(obj)
13    @token_listeners ||= []
14    @token_listeners << obj
15  end
16
17  ##
18  # Fetches the next token from the scanner
19
20  def get_tk
21    tk = nil
22
23    if @tokens.empty? then
24      tk = @scanner.token
25      @read.push @scanner.get_readed
26      puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
27    else
28      @read.push @unget_read.shift
29      tk = @tokens.shift
30      puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
31    end
32
33    tk = nil if TkEND_OF_SCRIPT === tk
34
35    if TkSYMBEG === tk then
36      set_token_position tk.line_no, tk.char_no
37
38      case tk1 = get_tk
39      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
40        if tk1.respond_to?(:name) then
41          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
42        else
43          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
44        end
45
46        # remove the identifier we just read to replace it with a symbol
47        @token_listeners.each do |obj|
48          obj.pop_token
49        end if @token_listeners
50      else
51        tk = tk1
52      end
53    end
54
55    # inform any listeners of our shiny new token
56    @token_listeners.each do |obj|
57      obj.add_token(tk)
58    end if @token_listeners
59
60    tk
61  end
62
63  ##
64  # Reads and returns all tokens up to one of +tokens+.  Leaves the matched
65  # token in the token list.
66
67  def get_tk_until(*tokens)
68    read = []
69
70    loop do
71      tk = get_tk
72
73      case tk
74      when *tokens then
75        unget_tk tk
76        break
77      end
78
79      read << tk
80    end
81
82    read
83  end
84
85  ##
86  # Retrieves a String representation of the read tokens
87
88  def get_tkread
89    read = @read.join("")
90    @read = []
91    read
92  end
93
94  ##
95  # Peek equivalent for get_tkread
96
97  def peek_read
98    @read.join('')
99  end
100
101  ##
102  # Peek at the next token, but don't remove it from the stream
103
104  def peek_tk
105    unget_tk(tk = get_tk)
106    tk
107  end
108
109  ##
110  # Removes the token listener +obj+
111
112  def remove_token_listener(obj)
113    @token_listeners.delete(obj)
114  end
115
116  ##
117  # Resets the tools
118
119  def reset
120    @read       = []
121    @tokens     = []
122    @unget_read = []
123    @nest = 0
124  end
125
126  ##
127  # Skips whitespace tokens including newlines if +skip_nl+ is true
128
129  def skip_tkspace(skip_nl = true) # HACK dup
130    tokens = []
131
132    while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
133      tokens.push tk
134    end
135
136    unget_tk tk
137    tokens
138  end
139
140  ##
141  # Has +obj+ listen to tokens
142
143  def token_listener(obj)
144    add_token_listener obj
145    yield
146  ensure
147    remove_token_listener obj
148  end
149
150  ##
151  # Returns +tk+ to the scanner
152
153  def unget_tk(tk)
154    @tokens.unshift tk
155    @unget_read.unshift @read.pop
156
157    # Remove this token from any listeners
158    @token_listeners.each do |obj|
159      obj.pop_token
160    end if @token_listeners
161
162    nil
163  end
164
165end
166
167
168