1#--
2#   irb/ruby-token.rb - ruby tokens
3#   	$Release Version: 0.9.5$
4#   	$Revision: 11708 $
5#   	$Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
6#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
7#++
8# Definitions of all tokens involved in the lexical analysis.
9#
10# This class is not documented because it is so deep in the internals.
11
12module RDoc::RubyToken
13  # :stopdoc:
14
15  EXPR_BEG = :EXPR_BEG
16  EXPR_MID = :EXPR_MID
17  EXPR_END = :EXPR_END
18  EXPR_ARG = :EXPR_ARG
19  EXPR_FNAME = :EXPR_FNAME
20  EXPR_DOT = :EXPR_DOT
21  EXPR_CLASS = :EXPR_CLASS
22
23  # for ruby 1.4X
24  if !defined?(Symbol)
25    Symbol = Integer
26  end
27
28  def set_token_position(line, char)
29    @prev_line_no = line
30    @prev_char_no = char
31  end
32
33  class Token
34    def initialize(seek, line_no, char_no, text = nil)
35      @seek = seek
36      @line_no = line_no
37      @char_no = char_no
38      @text = text
39    end
40
41    attr :seek
42    attr :line_no
43    attr :char_no
44
45    attr_accessor :text
46
47    def ==(other)
48      self.class == other.class and
49      other.line_no == @line_no and
50      other.char_no == @char_no and
51      other.text == @text
52    end
53
54    ##
55    # Because we're used in contexts that expect to return a token, we set the
56    # text string and then return ourselves
57
58    def set_text(text)
59      @text = text
60      self
61    end
62
63    def inspect # :nodoc:
64      klass = self.class.name.split('::').last
65      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @text]
66    end
67
68  end
69
70  class TkNode < Token
71    def initialize(seek, line_no, char_no, node = nil)
72      super seek, line_no, char_no
73      @node = node
74    end
75
76    attr :node
77
78    def ==(other)
79      self.class == other.class and
80      other.line_no == @line_no and
81      other.char_no == @char_no and
82      other.node == @node
83    end
84
85    def set_text text
86      @node = text
87      self
88    end
89
90    alias text node
91
92    def inspect # :nodoc:
93      klass = self.class.name.split('::').last
94      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @node]
95    end
96
97  end
98
99  class TkId < Token
100    def initialize(seek, line_no, char_no, name)
101      super(seek, line_no, char_no)
102      @name = name
103    end
104    attr :name
105
106    def ==(other)
107      self.class == other.class and
108      other.line_no == @line_no and
109      other.char_no == @char_no and
110      other.name == @name
111    end
112
113    def set_text text
114      @name = text
115      self
116    end
117
118    alias text name
119
120    def inspect # :nodoc:
121      klass = self.class.name.split('::').last
122      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
123    end
124
125  end
126
127  class TkKW < TkId
128  end
129
130  class TkVal < Token
131    def initialize(seek, line_no, char_no, value = nil)
132      super(seek, line_no, char_no)
133      @value = value
134    end
135    attr_accessor :value
136
137    def ==(other)
138      self.class == other.class and
139      other.line_no == @line_no and
140      other.char_no == @char_no and
141      other.value == @value
142    end
143
144    def set_text text
145      @value = text
146      self
147    end
148
149    alias text value
150
151    def inspect # :nodoc:
152      klass = self.class.name.split('::').last
153      "{%s %s, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @value]
154    end
155
156  end
157
158  class TkOp < Token
159    def initialize(seek, line_no, char_no, name = nil)
160      super seek, line_no, char_no
161      @name = name
162    end
163
164    attr_accessor :name
165
166    def ==(other)
167      self.class == other.class and
168      other.line_no == @line_no and
169      other.char_no == @char_no and
170      other.name == @name
171    end
172
173    def set_text text
174      @name = text
175      self
176    end
177
178    alias text name
179
180    def inspect # :nodoc:
181      klass = self.class.name.split('::').last
182      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
183    end
184
185  end
186
187  class TkOPASGN < TkOp
188    def initialize(seek, line_no, char_no, op)
189      super(seek, line_no, char_no)
190      op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
191      @op = op
192      @text = nil
193    end
194
195    attr :op
196
197    def ==(other)
198      self.class == other.class and
199      other.line_no == @line_no and
200      other.char_no == @char_no and
201      other.op == @op
202    end
203
204    def text
205      @text ||= "#{TkToken2Reading[op]}="
206    end
207
208    def inspect # :nodoc:
209      klass = self.class.name.split('::').last
210      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @op]
211    end
212
213  end
214
215  class TkUnknownChar < Token
216    def initialize(seek, line_no, char_no, name)
217      super(seek, line_no, char_no)
218      @name = name
219    end
220    attr :name
221
222    def ==(other)
223      self.class == other.class and
224      other.line_no == @line_no and
225      other.char_no == @char_no and
226      other.name == @name
227    end
228
229    def set_text text
230      @name = text
231      self
232    end
233
234    alias text name
235
236    def inspect # :nodoc:
237      klass = self.class.name.split('::').last
238      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
239    end
240
241  end
242
243  class TkError < Token
244  end
245
246  def Token(token, value = nil)
247    value ||= TkToken2Reading[token]
248
249    case token
250    when String
251      if (tk = TkReading2Token[token]).nil?
252        IRB.fail TkReading2TokenNoKey, token
253      end
254
255      tk = Token(tk[0], value)
256
257      if tk.kind_of?(TkOp) then
258        tk.name = token
259      end
260    when Symbol
261      if (tk = TkSymbol2Token[token]).nil?
262        IRB.fail TkSymbol2TokenNoKey, token
263      end
264
265      tk = Token(tk[0], value)
266    else
267      if token.instance_method(:initialize).arity == 3 then
268        tk = token.new(@prev_seek, @prev_line_no, @prev_char_no)
269        tk.set_text value
270      else
271        tk = token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
272      end
273    end
274
275    tk
276  end
277
278  TokenDefinitions = [
279    [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
280    [:TkMODULE,     TkKW,  "module", EXPR_BEG],
281    [:TkDEF,        TkKW,  "def",    EXPR_FNAME],
282    [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
283    [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
284    [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
285    [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
286    [:TkEND,        TkKW,  "end",    EXPR_END],
287    [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
288    [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
289    [:TkTHEN,       TkKW,  "then",   EXPR_BEG],
290    [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
291    [:TkELSE,       TkKW,  "else",   EXPR_BEG],
292    [:TkCASE,       TkKW,  "case",   EXPR_BEG],
293    [:TkWHEN,       TkKW,  "when",   EXPR_BEG],
294    [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
295    [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
296    [:TkFOR,        TkKW,  "for",    EXPR_BEG],
297    [:TkBREAK,      TkKW,  "break",  EXPR_MID],
298    [:TkNEXT,       TkKW,  "next",   EXPR_END],
299    [:TkREDO,       TkKW,  "redo",   EXPR_END],
300    [:TkRETRY,      TkKW,  "retry",  EXPR_END],
301    [:TkIN,         TkKW,  "in",     EXPR_BEG],
302    [:TkDO,         TkKW,  "do",     EXPR_BEG],
303    [:TkRETURN,     TkKW,  "return", EXPR_MID],
304    [:TkYIELD,      TkKW,  "yield",  EXPR_END],
305    [:TkSUPER,      TkKW,  "super",  EXPR_END],
306    [:TkSELF,       TkKW,  "self",   EXPR_END],
307    [:TkNIL,        TkKW,  "nil",    EXPR_END],
308    [:TkTRUE,       TkKW,  "true",   EXPR_END],
309    [:TkFALSE,      TkKW,  "false",  EXPR_END],
310    [:TkAND,        TkKW,  "and",    EXPR_BEG],
311    [:TkOR,         TkKW,  "or",     EXPR_BEG],
312    [:TkNOT,        TkKW,  "not",    EXPR_BEG],
313    [:TkIF_MOD,     TkKW],
314    [:TkUNLESS_MOD, TkKW],
315    [:TkWHILE_MOD,  TkKW],
316    [:TkUNTIL_MOD,  TkKW],
317    [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
318    [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
319    [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
320    [:TklEND,       TkKW,  "END",      EXPR_END],
321    [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
322    [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],
323
324    [:TkIDENTIFIER, TkId],
325    [:TkFID,        TkId],
326    [:TkGVAR,       TkId],
327    [:TkCVAR,       TkId],
328    [:TkIVAR,       TkId],
329    [:TkCONSTANT,   TkId],
330
331    [:TkINTEGER,    TkVal],
332    [:TkFLOAT,      TkVal],
333    [:TkSTRING,     TkVal],
334    [:TkHEREDOC,    TkVal],
335    [:TkXSTRING,    TkVal],
336    [:TkREGEXP,     TkVal],
337    [:TkSYMBOL,     TkVal],
338
339    [:TkDSTRING,    TkNode],
340    [:TkDXSTRING,   TkNode],
341    [:TkDREGEXP,    TkNode],
342    [:TkNTH_REF,    TkNode],
343    [:TkBACK_REF,   TkNode],
344
345    [:TkUPLUS,      TkOp,   "+@"],
346    [:TkUMINUS,     TkOp,   "-@"],
347    [:TkPOW,        TkOp,   "**"],
348    [:TkCMP,        TkOp,   "<=>"],
349    [:TkEQ,         TkOp,   "=="],
350    [:TkEQQ,        TkOp,   "==="],
351    [:TkNEQ,        TkOp,   "!="],
352    [:TkGEQ,        TkOp,   ">="],
353    [:TkLEQ,        TkOp,   "<="],
354    [:TkANDOP,      TkOp,   "&&"],
355    [:TkOROP,       TkOp,   "||"],
356    [:TkMATCH,      TkOp,   "=~"],
357    [:TkNMATCH,     TkOp,   "!~"],
358    [:TkDOT2,       TkOp,   ".."],
359    [:TkDOT3,       TkOp,   "..."],
360    [:TkAREF,       TkOp,   "[]"],
361    [:TkASET,       TkOp,   "[]="],
362    [:TkLSHFT,      TkOp,   "<<"],
363    [:TkRSHFT,      TkOp,   ">>"],
364    [:TkCOLON2,     TkOp,   '::'],
365    [:TkCOLON3,     TkOp,   '::'],
366    #[:OPASGN,       TkOp],               # +=, -=  etc. #
367    [:TkASSOC,      TkOp,   "=>"],
368    [:TkQUESTION,   TkOp,   "?"], #?
369    [:TkCOLON,      TkOp,   ":"],        #:
370
371    [:TkfLPAREN,    Token,  "("], # func( #
372    [:TkfLBRACK,    Token,  "["], # func[ #
373    [:TkfLBRACE,    Token,  "{"], # func{ #
374    [:TkSYMBEG,     Token,  ":"], # :SYMBOL
375
376    [:TkAMPER,      TkOp,   "&"],
377    [:TkGT,         TkOp,   ">"],
378    [:TkLT,         TkOp,   "<"],
379    [:TkPLUS,       TkOp,   "+"],
380    [:TkSTAR,       TkOp,   "*"],
381    [:TkMINUS,      TkOp,   "-"],
382    [:TkMULT,       TkOp,   "*"],
383    [:TkDIV,        TkOp,   "/"],
384    [:TkMOD,        TkOp,   "%"],
385    [:TkBITOR,      TkOp,   "|"],
386    [:TkBITXOR,     TkOp,   "^"],
387    [:TkBITAND,     TkOp,   "&"],
388    [:TkBITNOT,     TkOp,   "~"],
389    [:TkNOTOP,      TkOp,   "!"],
390
391    [:TkBACKQUOTE,  TkOp,   "`"],
392
393    [:TkASSIGN,     Token,  "="],
394    [:TkDOT,        Token,  "."],
395    [:TkLPAREN,     Token,  "("],  #(exp)
396    [:TkLBRACK,     Token,  "["],  #[arry]
397    [:TkLBRACE,     Token,  "{"],  #{hash}
398    [:TkRPAREN,     Token,  ")"],
399    [:TkRBRACK,     Token,  "]"],
400    [:TkRBRACE,     Token,  "}"],
401    [:TkCOMMA,      Token,  ","],
402    [:TkSEMICOLON,  Token,  ";"],
403
404    [:TkCOMMENT,    TkVal],
405    [:TkSPACE,      Token,  " "],
406    [:TkNL,         Token,  "\n"],
407    [:TkEND_OF_SCRIPT],
408
409    [:TkBACKSLASH,  TkUnknownChar,  "\\"],
410    [:TkAT,         TkUnknownChar,  "@"],
411    [:TkDOLLAR,     TkUnknownChar,  "$"],
412  ]
413
414  # {reading => token_class}
415  # {reading => [token_class, *opt]}
416  TkReading2Token = {}
417  TkToken2Reading = {}
418  TkSymbol2Token = {}
419
420  def self.def_token(token_n, super_token = Token, reading = nil, *opts)
421    token_n = token_n.id2name if token_n.kind_of?(Symbol)
422    if const_defined?(token_n)
423      IRB.fail AlreadyDefinedToken, token_n
424    end
425    token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
426
427    if reading
428      TkToken2Reading[token_c] = reading
429
430      return if TkReading2Token[reading]
431
432      if opts.empty?
433        TkReading2Token[reading] = [token_c]
434      else
435        TkReading2Token[reading] = [token_c].concat(opts)
436      end
437    end
438    TkSymbol2Token[token_n.intern] = token_c
439  end
440
441  for defs in TokenDefinitions
442    def_token(*defs)
443  end
444
445  def_token :TkRD_COMMENT, TkCOMMENT
446
447  NEWLINE_TOKEN = TkNL.new 0, 0, 0, "\n"
448
449  class TkSYMBOL
450
451    def to_sym
452      @sym ||= text[1..-1].intern
453    end
454
455  end
456
457  # :startdoc:
458end
459
460