1#-- 2# irb/ruby-token.rb - ruby tokens 3# $Release Version: 0.9.5$ 4# $Revision: 11708 $ 5# $Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $ 6# by Keiju ISHITSUKA(keiju@ruby-lang.org) 7#++ 8# Definitions of all tokens involved in the lexical analysis. 9# 10# This class is not documented because it is so deep in the internals. 11 12module RDoc::RubyToken 13 # :stopdoc: 14 15 EXPR_BEG = :EXPR_BEG 16 EXPR_MID = :EXPR_MID 17 EXPR_END = :EXPR_END 18 EXPR_ARG = :EXPR_ARG 19 EXPR_FNAME = :EXPR_FNAME 20 EXPR_DOT = :EXPR_DOT 21 EXPR_CLASS = :EXPR_CLASS 22 23 # for ruby 1.4X 24 if !defined?(Symbol) 25 Symbol = Integer 26 end 27 28 def set_token_position(line, char) 29 @prev_line_no = line 30 @prev_char_no = char 31 end 32 33 class Token 34 def initialize(seek, line_no, char_no, text = nil) 35 @seek = seek 36 @line_no = line_no 37 @char_no = char_no 38 @text = text 39 end 40 41 attr :seek 42 attr :line_no 43 attr :char_no 44 45 attr_accessor :text 46 47 def ==(other) 48 self.class == other.class and 49 other.line_no == @line_no and 50 other.char_no == @char_no and 51 other.text == @text 52 end 53 54 ## 55 # Because we're used in contexts that expect to return a token, we set the 56 # text string and then return ourselves 57 58 def set_text(text) 59 @text = text 60 self 61 end 62 63 def inspect # :nodoc: 64 klass = self.class.name.split('::').last 65 "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @text] 66 end 67 68 end 69 70 class TkNode < Token 71 def initialize(seek, line_no, char_no, node = nil) 72 super seek, line_no, char_no 73 @node = node 74 end 75 76 attr :node 77 78 def ==(other) 79 self.class == other.class and 80 other.line_no == @line_no and 81 other.char_no == @char_no and 82 other.node == @node 83 end 84 85 def set_text text 86 @node = text 87 self 88 end 89 90 alias text node 91 92 def inspect # :nodoc: 93 klass = self.class.name.split('::').last 94 "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @node] 95 end 96 97 end 98 99 class TkId < Token 100 def initialize(seek, line_no, char_no, name) 101 super(seek, line_no, char_no) 102 @name = name 103 end 104 attr :name 105 106 def ==(other) 107 self.class == other.class and 108 other.line_no == @line_no and 109 other.char_no == @char_no and 110 other.name == @name 111 end 112 113 def set_text text 114 @name = text 115 self 116 end 117 118 alias text name 119 120 def inspect # :nodoc: 121 klass = self.class.name.split('::').last 122 "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name] 123 end 124 125 end 126 127 class TkKW < TkId 128 end 129 130 class TkVal < Token 131 def initialize(seek, line_no, char_no, value = nil) 132 super(seek, line_no, char_no) 133 @value = value 134 end 135 attr_accessor :value 136 137 def ==(other) 138 self.class == other.class and 139 other.line_no == @line_no and 140 other.char_no == @char_no and 141 other.value == @value 142 end 143 144 def set_text text 145 @value = text 146 self 147 end 148 149 alias text value 150 151 def inspect # :nodoc: 152 klass = self.class.name.split('::').last 153 "{%s %s, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @value] 154 end 155 156 end 157 158 class TkOp < Token 159 def initialize(seek, line_no, char_no, name = nil) 160 super seek, line_no, char_no 161 @name = name 162 end 163 164 attr_accessor :name 165 166 def ==(other) 167 self.class == other.class and 168 other.line_no == @line_no and 169 other.char_no == @char_no and 170 other.name == @name 171 end 172 173 def set_text text 174 @name = text 175 self 176 end 177 178 alias text name 179 180 def inspect # :nodoc: 181 klass = self.class.name.split('::').last 182 "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name] 183 end 184 185 end 186 187 class TkOPASGN < TkOp 188 def initialize(seek, line_no, char_no, op) 189 super(seek, line_no, char_no) 190 op = TkReading2Token[op][0] unless op.kind_of?(Symbol) 191 @op = op 192 @text = nil 193 end 194 195 attr :op 196 197 def ==(other) 198 self.class == other.class and 199 other.line_no == @line_no and 200 other.char_no == @char_no and 201 other.op == @op 202 end 203 204 def text 205 @text ||= "#{TkToken2Reading[op]}=" 206 end 207 208 def inspect # :nodoc: 209 klass = self.class.name.split('::').last 210 "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @op] 211 end 212 213 end 214 215 class TkUnknownChar < Token 216 def initialize(seek, line_no, char_no, name) 217 super(seek, line_no, char_no) 218 @name = name 219 end 220 attr :name 221 222 def ==(other) 223 self.class == other.class and 224 other.line_no == @line_no and 225 other.char_no == @char_no and 226 other.name == @name 227 end 228 229 def set_text text 230 @name = text 231 self 232 end 233 234 alias text name 235 236 def inspect # :nodoc: 237 klass = self.class.name.split('::').last 238 "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name] 239 end 240 241 end 242 243 class TkError < Token 244 end 245 246 def Token(token, value = nil) 247 value ||= TkToken2Reading[token] 248 249 case token 250 when String 251 if (tk = TkReading2Token[token]).nil? 252 IRB.fail TkReading2TokenNoKey, token 253 end 254 255 tk = Token(tk[0], value) 256 257 if tk.kind_of?(TkOp) then 258 tk.name = token 259 end 260 when Symbol 261 if (tk = TkSymbol2Token[token]).nil? 262 IRB.fail TkSymbol2TokenNoKey, token 263 end 264 265 tk = Token(tk[0], value) 266 else 267 if token.instance_method(:initialize).arity == 3 then 268 tk = token.new(@prev_seek, @prev_line_no, @prev_char_no) 269 tk.set_text value 270 else 271 tk = token.new(@prev_seek, @prev_line_no, @prev_char_no, value) 272 end 273 end 274 275 tk 276 end 277 278 TokenDefinitions = [ 279 [:TkCLASS, TkKW, "class", EXPR_CLASS], 280 [:TkMODULE, TkKW, "module", EXPR_BEG], 281 [:TkDEF, TkKW, "def", EXPR_FNAME], 282 [:TkUNDEF, TkKW, "undef", EXPR_FNAME], 283 [:TkBEGIN, TkKW, "begin", EXPR_BEG], 284 [:TkRESCUE, TkKW, "rescue", EXPR_MID], 285 [:TkENSURE, TkKW, "ensure", EXPR_BEG], 286 [:TkEND, TkKW, "end", EXPR_END], 287 [:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD], 288 [:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD], 289 [:TkTHEN, TkKW, "then", EXPR_BEG], 290 [:TkELSIF, TkKW, "elsif", EXPR_BEG], 291 [:TkELSE, TkKW, "else", EXPR_BEG], 292 [:TkCASE, TkKW, "case", EXPR_BEG], 293 [:TkWHEN, TkKW, "when", EXPR_BEG], 294 [:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD], 295 [:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD], 296 [:TkFOR, TkKW, "for", EXPR_BEG], 297 [:TkBREAK, TkKW, "break", EXPR_MID], 298 [:TkNEXT, TkKW, "next", EXPR_END], 299 [:TkREDO, TkKW, "redo", EXPR_END], 300 [:TkRETRY, TkKW, "retry", EXPR_END], 301 [:TkIN, TkKW, "in", EXPR_BEG], 302 [:TkDO, TkKW, "do", EXPR_BEG], 303 [:TkRETURN, TkKW, "return", EXPR_MID], 304 [:TkYIELD, TkKW, "yield", EXPR_END], 305 [:TkSUPER, TkKW, "super", EXPR_END], 306 [:TkSELF, TkKW, "self", EXPR_END], 307 [:TkNIL, TkKW, "nil", EXPR_END], 308 [:TkTRUE, TkKW, "true", EXPR_END], 309 [:TkFALSE, TkKW, "false", EXPR_END], 310 [:TkAND, TkKW, "and", EXPR_BEG], 311 [:TkOR, TkKW, "or", EXPR_BEG], 312 [:TkNOT, TkKW, "not", EXPR_BEG], 313 [:TkIF_MOD, TkKW], 314 [:TkUNLESS_MOD, TkKW], 315 [:TkWHILE_MOD, TkKW], 316 [:TkUNTIL_MOD, TkKW], 317 [:TkALIAS, TkKW, "alias", EXPR_FNAME], 318 [:TkDEFINED, TkKW, "defined?", EXPR_END], 319 [:TklBEGIN, TkKW, "BEGIN", EXPR_END], 320 [:TklEND, TkKW, "END", EXPR_END], 321 [:Tk__LINE__, TkKW, "__LINE__", EXPR_END], 322 [:Tk__FILE__, TkKW, "__FILE__", EXPR_END], 323 324 [:TkIDENTIFIER, TkId], 325 [:TkFID, TkId], 326 [:TkGVAR, TkId], 327 [:TkCVAR, TkId], 328 [:TkIVAR, TkId], 329 [:TkCONSTANT, TkId], 330 331 [:TkINTEGER, TkVal], 332 [:TkFLOAT, TkVal], 333 [:TkSTRING, TkVal], 334 [:TkHEREDOC, TkVal], 335 [:TkXSTRING, TkVal], 336 [:TkREGEXP, TkVal], 337 [:TkSYMBOL, TkVal], 338 339 [:TkDSTRING, TkNode], 340 [:TkDXSTRING, TkNode], 341 [:TkDREGEXP, TkNode], 342 [:TkNTH_REF, TkNode], 343 [:TkBACK_REF, TkNode], 344 345 [:TkUPLUS, TkOp, "+@"], 346 [:TkUMINUS, TkOp, "-@"], 347 [:TkPOW, TkOp, "**"], 348 [:TkCMP, TkOp, "<=>"], 349 [:TkEQ, TkOp, "=="], 350 [:TkEQQ, TkOp, "==="], 351 [:TkNEQ, TkOp, "!="], 352 [:TkGEQ, TkOp, ">="], 353 [:TkLEQ, TkOp, "<="], 354 [:TkANDOP, TkOp, "&&"], 355 [:TkOROP, TkOp, "||"], 356 [:TkMATCH, TkOp, "=~"], 357 [:TkNMATCH, TkOp, "!~"], 358 [:TkDOT2, TkOp, ".."], 359 [:TkDOT3, TkOp, "..."], 360 [:TkAREF, TkOp, "[]"], 361 [:TkASET, TkOp, "[]="], 362 [:TkLSHFT, TkOp, "<<"], 363 [:TkRSHFT, TkOp, ">>"], 364 [:TkCOLON2, TkOp, '::'], 365 [:TkCOLON3, TkOp, '::'], 366 #[:OPASGN, TkOp], # +=, -= etc. # 367 [:TkASSOC, TkOp, "=>"], 368 [:TkQUESTION, TkOp, "?"], #? 369 [:TkCOLON, TkOp, ":"], #: 370 371 [:TkfLPAREN, Token, "("], # func( # 372 [:TkfLBRACK, Token, "["], # func[ # 373 [:TkfLBRACE, Token, "{"], # func{ # 374 [:TkSYMBEG, Token, ":"], # :SYMBOL 375 376 [:TkAMPER, TkOp, "&"], 377 [:TkGT, TkOp, ">"], 378 [:TkLT, TkOp, "<"], 379 [:TkPLUS, TkOp, "+"], 380 [:TkSTAR, TkOp, "*"], 381 [:TkMINUS, TkOp, "-"], 382 [:TkMULT, TkOp, "*"], 383 [:TkDIV, TkOp, "/"], 384 [:TkMOD, TkOp, "%"], 385 [:TkBITOR, TkOp, "|"], 386 [:TkBITXOR, TkOp, "^"], 387 [:TkBITAND, TkOp, "&"], 388 [:TkBITNOT, TkOp, "~"], 389 [:TkNOTOP, TkOp, "!"], 390 391 [:TkBACKQUOTE, TkOp, "`"], 392 393 [:TkASSIGN, Token, "="], 394 [:TkDOT, Token, "."], 395 [:TkLPAREN, Token, "("], #(exp) 396 [:TkLBRACK, Token, "["], #[arry] 397 [:TkLBRACE, Token, "{"], #{hash} 398 [:TkRPAREN, Token, ")"], 399 [:TkRBRACK, Token, "]"], 400 [:TkRBRACE, Token, "}"], 401 [:TkCOMMA, Token, ","], 402 [:TkSEMICOLON, Token, ";"], 403 404 [:TkCOMMENT, TkVal], 405 [:TkSPACE, Token, " "], 406 [:TkNL, Token, "\n"], 407 [:TkEND_OF_SCRIPT], 408 409 [:TkBACKSLASH, TkUnknownChar, "\\"], 410 [:TkAT, TkUnknownChar, "@"], 411 [:TkDOLLAR, TkUnknownChar, "$"], 412 ] 413 414 # {reading => token_class} 415 # {reading => [token_class, *opt]} 416 TkReading2Token = {} 417 TkToken2Reading = {} 418 TkSymbol2Token = {} 419 420 def self.def_token(token_n, super_token = Token, reading = nil, *opts) 421 token_n = token_n.id2name if token_n.kind_of?(Symbol) 422 if const_defined?(token_n) 423 IRB.fail AlreadyDefinedToken, token_n 424 end 425 token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}") 426 427 if reading 428 TkToken2Reading[token_c] = reading 429 430 return if TkReading2Token[reading] 431 432 if opts.empty? 433 TkReading2Token[reading] = [token_c] 434 else 435 TkReading2Token[reading] = [token_c].concat(opts) 436 end 437 end 438 TkSymbol2Token[token_n.intern] = token_c 439 end 440 441 for defs in TokenDefinitions 442 def_token(*defs) 443 end 444 445 def_token :TkRD_COMMENT, TkCOMMENT 446 447 NEWLINE_TOKEN = TkNL.new 0, 0, 0, "\n" 448 449 class TkSYMBOL 450 451 def to_sym 452 @sym ||= text[1..-1].intern 453 end 454 455 end 456 457 # :startdoc: 458end 459 460