1# 2# irb/ruby-token.rb - ruby tokens 3# $Release Version: 0.9.6$ 4# $Revision: 38358 $ 5# by Keiju ISHITSUKA(keiju@ruby-lang.org) 6# 7# -- 8# 9# 10# 11# :stopdoc: 12module RubyToken 13 EXPR_BEG = :EXPR_BEG 14 EXPR_MID = :EXPR_MID 15 EXPR_END = :EXPR_END 16 EXPR_ARG = :EXPR_ARG 17 EXPR_FNAME = :EXPR_FNAME 18 EXPR_DOT = :EXPR_DOT 19 EXPR_CLASS = :EXPR_CLASS 20 21 class Token 22 def initialize(seek, line_no, char_no) 23 @seek = seek 24 @line_no = line_no 25 @char_no = char_no 26 end 27 attr :seek, :line_no, :char_no 28 end 29 30 class TkNode < Token 31 def initialize(seek, line_no, char_no) 32 super 33 end 34 attr :node 35 end 36 37 class TkId < Token 38 def initialize(seek, line_no, char_no, name) 39 super(seek, line_no, char_no) 40 @name = name 41 end 42 attr :name 43 end 44 45 class TkVal < Token 46 def initialize(seek, line_no, char_no, value = nil) 47 super(seek, line_no, char_no) 48 @value = value 49 end 50 attr :value 51 end 52 53 class TkOp < Token 54 attr_accessor :name 55 end 56 57 class TkOPASGN < TkOp 58 def initialize(seek, line_no, char_no, op) 59 super(seek, line_no, char_no) 60 op = TkReading2Token[op][0] unless op.kind_of?(Symbol) 61 @op = op 62 end 63 attr :op 64 end 65 66 class TkUnknownChar < Token 67 def initialize(seek, line_no, char_no, id) 68 super(seek, line_no, char_no) 69 @name = name 70 end 71 attr :name 72 end 73 74 class TkError < Token 75 end 76 77 def Token(token, value = nil) 78 case token 79 when String 80 if (tk = TkReading2Token[token]).nil? 81 IRB.fail TkReading2TokenNoKey, token 82 end 83 tk = Token(tk[0], value) 84 if tk.kind_of?(TkOp) 85 tk.name = token 86 end 87 return tk 88 when Symbol 89 if (tk = TkSymbol2Token[token]).nil? 90 IRB.fail TkSymbol2TokenNoKey, token 91 end 92 return Token(tk[0], value) 93 else 94 if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty? 95 token.new(@prev_seek, @prev_line_no, @prev_char_no) 96 else 97 token.new(@prev_seek, @prev_line_no, @prev_char_no, value) 98 end 99 end 100 end 101 102 TokenDefinitions = [ 103 [:TkCLASS, TkId, "class", EXPR_CLASS], 104 [:TkMODULE, TkId, "module", EXPR_BEG], 105 [:TkDEF, TkId, "def", EXPR_FNAME], 106 [:TkUNDEF, TkId, "undef", EXPR_FNAME], 107 [:TkBEGIN, TkId, "begin", EXPR_BEG], 108 [:TkRESCUE, TkId, "rescue", EXPR_MID], 109 [:TkENSURE, TkId, "ensure", EXPR_BEG], 110 [:TkEND, TkId, "end", EXPR_END], 111 [:TkIF, TkId, "if", EXPR_BEG, :TkIF_MOD], 112 [:TkUNLESS, TkId, "unless", EXPR_BEG, :TkUNLESS_MOD], 113 [:TkTHEN, TkId, "then", EXPR_BEG], 114 [:TkELSIF, TkId, "elsif", EXPR_BEG], 115 [:TkELSE, TkId, "else", EXPR_BEG], 116 [:TkCASE, TkId, "case", EXPR_BEG], 117 [:TkWHEN, TkId, "when", EXPR_BEG], 118 [:TkWHILE, TkId, "while", EXPR_BEG, :TkWHILE_MOD], 119 [:TkUNTIL, TkId, "until", EXPR_BEG, :TkUNTIL_MOD], 120 [:TkFOR, TkId, "for", EXPR_BEG], 121 [:TkBREAK, TkId, "break", EXPR_END], 122 [:TkNEXT, TkId, "next", EXPR_END], 123 [:TkREDO, TkId, "redo", EXPR_END], 124 [:TkRETRY, TkId, "retry", EXPR_END], 125 [:TkIN, TkId, "in", EXPR_BEG], 126 [:TkDO, TkId, "do", EXPR_BEG], 127 [:TkRETURN, TkId, "return", EXPR_MID], 128 [:TkYIELD, TkId, "yield", EXPR_END], 129 [:TkSUPER, TkId, "super", EXPR_END], 130 [:TkSELF, TkId, "self", EXPR_END], 131 [:TkNIL, TkId, "nil", EXPR_END], 132 [:TkTRUE, TkId, "true", EXPR_END], 133 [:TkFALSE, TkId, "false", EXPR_END], 134 [:TkAND, TkId, "and", EXPR_BEG], 135 [:TkOR, TkId, "or", EXPR_BEG], 136 [:TkNOT, TkId, "not", EXPR_BEG], 137 [:TkIF_MOD, TkId], 138 [:TkUNLESS_MOD, TkId], 139 [:TkWHILE_MOD, TkId], 140 [:TkUNTIL_MOD, TkId], 141 [:TkALIAS, TkId, "alias", EXPR_FNAME], 142 [:TkDEFINED, TkId, "defined?", EXPR_END], 143 [:TklBEGIN, TkId, "BEGIN", EXPR_END], 144 [:TklEND, TkId, "END", EXPR_END], 145 [:Tk__LINE__, TkId, "__LINE__", EXPR_END], 146 [:Tk__FILE__, TkId, "__FILE__", EXPR_END], 147 148 [:TkIDENTIFIER, TkId], 149 [:TkFID, TkId], 150 [:TkGVAR, TkId], 151 [:TkCVAR, TkId], 152 [:TkIVAR, TkId], 153 [:TkCONSTANT, TkId], 154 155 [:TkINTEGER, TkVal], 156 [:TkFLOAT, TkVal], 157 [:TkSTRING, TkVal], 158 [:TkXSTRING, TkVal], 159 [:TkREGEXP, TkVal], 160 [:TkSYMBOL, TkVal], 161 162 [:TkDSTRING, TkNode], 163 [:TkDXSTRING, TkNode], 164 [:TkDREGEXP, TkNode], 165 [:TkNTH_REF, TkNode], 166 [:TkBACK_REF, TkNode], 167 168 [:TkUPLUS, TkOp, "+@"], 169 [:TkUMINUS, TkOp, "-@"], 170 [:TkPOW, TkOp, "**"], 171 [:TkCMP, TkOp, "<=>"], 172 [:TkEQ, TkOp, "=="], 173 [:TkEQQ, TkOp, "==="], 174 [:TkNEQ, TkOp, "!="], 175 [:TkGEQ, TkOp, ">="], 176 [:TkLEQ, TkOp, "<="], 177 [:TkANDOP, TkOp, "&&"], 178 [:TkOROP, TkOp, "||"], 179 [:TkMATCH, TkOp, "=~"], 180 [:TkNMATCH, TkOp, "!~"], 181 [:TkDOT2, TkOp, ".."], 182 [:TkDOT3, TkOp, "..."], 183 [:TkAREF, TkOp, "[]"], 184 [:TkASET, TkOp, "[]="], 185 [:TkLSHFT, TkOp, "<<"], 186 [:TkRSHFT, TkOp, ">>"], 187 [:TkCOLON2, TkOp], 188 [:TkCOLON3, TkOp], 189# [:OPASGN, TkOp], # +=, -= etc. # 190 [:TkASSOC, TkOp, "=>"], 191 [:TkQUESTION, TkOp, "?"], #? 192 [:TkCOLON, TkOp, ":"], #: 193 194 [:TkfLPAREN], # func( # 195 [:TkfLBRACK], # func[ # 196 [:TkfLBRACE], # func{ # 197 [:TkSTAR], # *arg 198 [:TkAMPER], # &arg # 199 [:TkSYMBEG], # :SYMBOL 200 201 [:TkGT, TkOp, ">"], 202 [:TkLT, TkOp, "<"], 203 [:TkPLUS, TkOp, "+"], 204 [:TkMINUS, TkOp, "-"], 205 [:TkMULT, TkOp, "*"], 206 [:TkDIV, TkOp, "/"], 207 [:TkMOD, TkOp, "%"], 208 [:TkBITOR, TkOp, "|"], 209 [:TkBITXOR, TkOp, "^"], 210 [:TkBITAND, TkOp, "&"], 211 [:TkBITNOT, TkOp, "~"], 212 [:TkNOTOP, TkOp, "!"], 213 214 [:TkBACKQUOTE, TkOp, "`"], 215 216 [:TkASSIGN, Token, "="], 217 [:TkDOT, Token, "."], 218 [:TkLPAREN, Token, "("], #(exp) 219 [:TkLBRACK, Token, "["], #[arry] 220 [:TkLBRACE, Token, "{"], #{hash} 221 [:TkRPAREN, Token, ")"], 222 [:TkRBRACK, Token, "]"], 223 [:TkRBRACE, Token, "}"], 224 [:TkCOMMA, Token, ","], 225 [:TkSEMICOLON, Token, ";"], 226 227 [:TkCOMMENT], 228 [:TkRD_COMMENT], 229 [:TkSPACE], 230 [:TkNL], 231 [:TkEND_OF_SCRIPT], 232 233 [:TkBACKSLASH, TkUnknownChar, "\\"], 234 [:TkAT, TkUnknownChar, "@"], 235 [:TkDOLLAR, TkUnknownChar, "$"], 236 ] 237 238 # {reading => token_class} 239 # {reading => [token_class, *opt]} 240 TkReading2Token = {} 241 TkSymbol2Token = {} 242 243 def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts) 244 token_n = token_n.id2name if token_n.kind_of?(Symbol) 245 if RubyToken.const_defined?(token_n) 246 IRB.fail AlreadyDefinedToken, token_n 247 end 248 token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}") 249 250 if reading 251 if TkReading2Token[reading] 252 IRB.fail TkReading2TokenDuplicateError, token_n, reading 253 end 254 if opts.empty? 255 TkReading2Token[reading] = [token_c] 256 else 257 TkReading2Token[reading] = [token_c].concat(opts) 258 end 259 end 260 TkSymbol2Token[token_n.intern] = token_c 261 end 262 263 for defs in TokenDefinitions 264 def_token(*defs) 265 end 266end 267# :startdoc: 268