1#
2#   irb/ruby-token.rb - ruby tokens
3#   	$Release Version: 0.9.6$
4#   	$Revision: 38358 $
5#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
6#
7# --
8#
9#
10#
11# :stopdoc:
12module RubyToken
13  EXPR_BEG = :EXPR_BEG
14  EXPR_MID = :EXPR_MID
15  EXPR_END = :EXPR_END
16  EXPR_ARG = :EXPR_ARG
17  EXPR_FNAME = :EXPR_FNAME
18  EXPR_DOT = :EXPR_DOT
19  EXPR_CLASS = :EXPR_CLASS
20
21  class Token
22    def initialize(seek, line_no, char_no)
23      @seek = seek
24      @line_no = line_no
25      @char_no = char_no
26    end
27    attr :seek, :line_no, :char_no
28  end
29
30  class TkNode < Token
31    def initialize(seek, line_no, char_no)
32      super
33    end
34    attr :node
35  end
36
37  class TkId < Token
38    def initialize(seek, line_no, char_no, name)
39      super(seek, line_no, char_no)
40      @name = name
41    end
42    attr :name
43  end
44
45  class TkVal < Token
46    def initialize(seek, line_no, char_no, value = nil)
47      super(seek, line_no, char_no)
48      @value = value
49    end
50    attr :value
51  end
52
53  class TkOp < Token
54    attr_accessor :name
55  end
56
57  class TkOPASGN < TkOp
58    def initialize(seek, line_no, char_no, op)
59      super(seek, line_no, char_no)
60      op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
61      @op = op
62    end
63    attr :op
64  end
65
66  class TkUnknownChar < Token
67    def initialize(seek, line_no, char_no, id)
68      super(seek, line_no, char_no)
69      @name = name
70    end
71    attr :name
72  end
73
74  class TkError < Token
75  end
76
77  def Token(token, value = nil)
78    case token
79    when String
80      if (tk = TkReading2Token[token]).nil?
81	IRB.fail TkReading2TokenNoKey, token
82      end
83      tk = Token(tk[0], value)
84      if tk.kind_of?(TkOp)
85	tk.name = token
86      end
87      return tk
88    when Symbol
89      if (tk = TkSymbol2Token[token]).nil?
90	IRB.fail TkSymbol2TokenNoKey, token
91      end
92      return Token(tk[0], value)
93    else
94      if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
95	token.new(@prev_seek, @prev_line_no, @prev_char_no)
96      else
97	token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
98      end
99    end
100  end
101
102  TokenDefinitions = [
103    [:TkCLASS,      TkId,  "class",  EXPR_CLASS],
104    [:TkMODULE,     TkId,  "module", EXPR_BEG],
105    [:TkDEF,	    TkId,  "def",    EXPR_FNAME],
106    [:TkUNDEF,      TkId,  "undef",  EXPR_FNAME],
107    [:TkBEGIN,      TkId,  "begin",  EXPR_BEG],
108    [:TkRESCUE,     TkId,  "rescue", EXPR_MID],
109    [:TkENSURE,     TkId,  "ensure", EXPR_BEG],
110    [:TkEND,	    TkId,  "end",    EXPR_END],
111    [:TkIF,         TkId,  "if",     EXPR_BEG, :TkIF_MOD],
112    [:TkUNLESS,     TkId,  "unless", EXPR_BEG, :TkUNLESS_MOD],
113    [:TkTHEN,	    TkId,  "then",   EXPR_BEG],
114    [:TkELSIF,      TkId,  "elsif",  EXPR_BEG],
115    [:TkELSE,	    TkId,  "else",   EXPR_BEG],
116    [:TkCASE,	    TkId,  "case",   EXPR_BEG],
117    [:TkWHEN,	    TkId,  "when",   EXPR_BEG],
118    [:TkWHILE,      TkId,  "while",  EXPR_BEG, :TkWHILE_MOD],
119    [:TkUNTIL,      TkId,  "until",  EXPR_BEG, :TkUNTIL_MOD],
120    [:TkFOR,	    TkId,  "for",    EXPR_BEG],
121    [:TkBREAK,      TkId,  "break",  EXPR_END],
122    [:TkNEXT,	    TkId,  "next",   EXPR_END],
123    [:TkREDO,	    TkId,  "redo",   EXPR_END],
124    [:TkRETRY,      TkId,  "retry",  EXPR_END],
125    [:TkIN,	    TkId,  "in",     EXPR_BEG],
126    [:TkDO,	    TkId,  "do",     EXPR_BEG],
127    [:TkRETURN,     TkId,  "return", EXPR_MID],
128    [:TkYIELD,      TkId,  "yield",  EXPR_END],
129    [:TkSUPER,      TkId,  "super",  EXPR_END],
130    [:TkSELF,	    TkId,  "self",   EXPR_END],
131    [:TkNIL, 	    TkId,  "nil",    EXPR_END],
132    [:TkTRUE,	    TkId,  "true",   EXPR_END],
133    [:TkFALSE,      TkId,  "false",  EXPR_END],
134    [:TkAND,	    TkId,  "and",    EXPR_BEG],
135    [:TkOR, 	    TkId,  "or",     EXPR_BEG],
136    [:TkNOT,	    TkId,  "not",    EXPR_BEG],
137    [:TkIF_MOD,     TkId],
138    [:TkUNLESS_MOD, TkId],
139    [:TkWHILE_MOD,  TkId],
140    [:TkUNTIL_MOD,  TkId],
141    [:TkALIAS,      TkId,  "alias",    EXPR_FNAME],
142    [:TkDEFINED,    TkId,  "defined?", EXPR_END],
143    [:TklBEGIN,     TkId,  "BEGIN",    EXPR_END],
144    [:TklEND,	    TkId,  "END",      EXPR_END],
145    [:Tk__LINE__,   TkId,  "__LINE__", EXPR_END],
146    [:Tk__FILE__,   TkId,  "__FILE__", EXPR_END],
147
148    [:TkIDENTIFIER, TkId],
149    [:TkFID,	    TkId],
150    [:TkGVAR,	    TkId],
151    [:TkCVAR,	    TkId],
152    [:TkIVAR,	    TkId],
153    [:TkCONSTANT,   TkId],
154
155    [:TkINTEGER,    TkVal],
156    [:TkFLOAT,      TkVal],
157    [:TkSTRING,     TkVal],
158    [:TkXSTRING,    TkVal],
159    [:TkREGEXP,     TkVal],
160    [:TkSYMBOL,     TkVal],
161
162    [:TkDSTRING,    TkNode],
163    [:TkDXSTRING,   TkNode],
164    [:TkDREGEXP,    TkNode],
165    [:TkNTH_REF,    TkNode],
166    [:TkBACK_REF,   TkNode],
167
168    [:TkUPLUS,      TkOp,   "+@"],
169    [:TkUMINUS,     TkOp,   "-@"],
170    [:TkPOW,	    TkOp,   "**"],
171    [:TkCMP,	    TkOp,   "<=>"],
172    [:TkEQ,	    TkOp,   "=="],
173    [:TkEQQ,	    TkOp,   "==="],
174    [:TkNEQ,	    TkOp,   "!="],
175    [:TkGEQ,	    TkOp,   ">="],
176    [:TkLEQ,	    TkOp,   "<="],
177    [:TkANDOP,      TkOp,   "&&"],
178    [:TkOROP,	    TkOp,   "||"],
179    [:TkMATCH,      TkOp,   "=~"],
180    [:TkNMATCH,     TkOp,   "!~"],
181    [:TkDOT2,	    TkOp,   ".."],
182    [:TkDOT3,	    TkOp,   "..."],
183    [:TkAREF,	    TkOp,   "[]"],
184    [:TkASET,	    TkOp,   "[]="],
185    [:TkLSHFT,      TkOp,   "<<"],
186    [:TkRSHFT,      TkOp,   ">>"],
187    [:TkCOLON2,     TkOp],
188    [:TkCOLON3,     TkOp],
189#   [:OPASGN,	    TkOp],               # +=, -=  etc. #
190    [:TkASSOC,      TkOp,   "=>"],
191    [:TkQUESTION,   TkOp,   "?"],	 #?
192    [:TkCOLON,      TkOp,   ":"],        #:
193
194    [:TkfLPAREN],         # func( #
195    [:TkfLBRACK],         # func[ #
196    [:TkfLBRACE],         # func{ #
197    [:TkSTAR],            # *arg
198    [:TkAMPER],           # &arg #
199    [:TkSYMBEG],          # :SYMBOL
200
201    [:TkGT,	    TkOp,   ">"],
202    [:TkLT,	    TkOp,   "<"],
203    [:TkPLUS,	    TkOp,   "+"],
204    [:TkMINUS,      TkOp,   "-"],
205    [:TkMULT,	    TkOp,   "*"],
206    [:TkDIV,	    TkOp,   "/"],
207    [:TkMOD,	    TkOp,   "%"],
208    [:TkBITOR,      TkOp,   "|"],
209    [:TkBITXOR,     TkOp,   "^"],
210    [:TkBITAND,     TkOp,   "&"],
211    [:TkBITNOT,     TkOp,   "~"],
212    [:TkNOTOP,      TkOp,   "!"],
213
214    [:TkBACKQUOTE,  TkOp,   "`"],
215
216    [:TkASSIGN,     Token,  "="],
217    [:TkDOT,	    Token,  "."],
218    [:TkLPAREN,     Token,  "("],  #(exp)
219    [:TkLBRACK,     Token,  "["],  #[arry]
220    [:TkLBRACE,     Token,  "{"],  #{hash}
221    [:TkRPAREN,     Token,  ")"],
222    [:TkRBRACK,     Token,  "]"],
223    [:TkRBRACE,     Token,  "}"],
224    [:TkCOMMA,      Token,  ","],
225    [:TkSEMICOLON,  Token,  ";"],
226
227    [:TkCOMMENT],
228    [:TkRD_COMMENT],
229    [:TkSPACE],
230    [:TkNL],
231    [:TkEND_OF_SCRIPT],
232
233    [:TkBACKSLASH,  TkUnknownChar,  "\\"],
234    [:TkAT,	    TkUnknownChar,  "@"],
235    [:TkDOLLAR,     TkUnknownChar,  "$"],
236  ]
237
238  # {reading => token_class}
239  # {reading => [token_class, *opt]}
240  TkReading2Token = {}
241  TkSymbol2Token = {}
242
243  def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
244    token_n = token_n.id2name if token_n.kind_of?(Symbol)
245    if RubyToken.const_defined?(token_n)
246      IRB.fail AlreadyDefinedToken, token_n
247    end
248    token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
249
250    if reading
251      if TkReading2Token[reading]
252	IRB.fail TkReading2TokenDuplicateError, token_n, reading
253      end
254      if opts.empty?
255	TkReading2Token[reading] = [token_c]
256      else
257	TkReading2Token[reading] = [token_c].concat(opts)
258      end
259    end
260    TkSymbol2Token[token_n.intern] = token_c
261  end
262
263  for defs in TokenDefinitions
264    def_token(*defs)
265  end
266end
267# :startdoc:
268