1## 2# Collection of methods for writing parsers against RDoc::RubyLex and 3# RDoc::RubyToken 4 5module RDoc::Parser::RubyTools 6 7 include RDoc::RubyToken 8 9 ## 10 # Adds a token listener +obj+, but you should probably use token_listener 11 12 def add_token_listener(obj) 13 @token_listeners ||= [] 14 @token_listeners << obj 15 end 16 17 ## 18 # Fetches the next token from the scanner 19 20 def get_tk 21 tk = nil 22 23 if @tokens.empty? then 24 tk = @scanner.token 25 @read.push @scanner.get_readed 26 puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG 27 else 28 @read.push @unget_read.shift 29 tk = @tokens.shift 30 puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG 31 end 32 33 tk = nil if TkEND_OF_SCRIPT === tk 34 35 if TkSYMBEG === tk then 36 set_token_position tk.line_no, tk.char_no 37 38 case tk1 = get_tk 39 when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then 40 if tk1.respond_to?(:name) then 41 tk = Token(TkSYMBOL).set_text(":" + tk1.name) 42 else 43 tk = Token(TkSYMBOL).set_text(":" + tk1.text) 44 end 45 46 # remove the identifier we just read to replace it with a symbol 47 @token_listeners.each do |obj| 48 obj.pop_token 49 end if @token_listeners 50 else 51 tk = tk1 52 end 53 end 54 55 # inform any listeners of our shiny new token 56 @token_listeners.each do |obj| 57 obj.add_token(tk) 58 end if @token_listeners 59 60 tk 61 end 62 63 ## 64 # Reads and returns all tokens up to one of +tokens+. Leaves the matched 65 # token in the token list. 66 67 def get_tk_until(*tokens) 68 read = [] 69 70 loop do 71 tk = get_tk 72 73 case tk 74 when *tokens then 75 unget_tk tk 76 break 77 end 78 79 read << tk 80 end 81 82 read 83 end 84 85 ## 86 # Retrieves a String representation of the read tokens 87 88 def get_tkread 89 read = @read.join("") 90 @read = [] 91 read 92 end 93 94 ## 95 # Peek equivalent for get_tkread 96 97 def peek_read 98 @read.join('') 99 end 100 101 ## 102 # Peek at the next token, but don't remove it from the stream 103 104 def peek_tk 105 unget_tk(tk = get_tk) 106 tk 107 end 108 109 ## 110 # Removes the token listener +obj+ 111 112 def remove_token_listener(obj) 113 @token_listeners.delete(obj) 114 end 115 116 ## 117 # Resets the tools 118 119 def reset 120 @read = [] 121 @tokens = [] 122 @unget_read = [] 123 @nest = 0 124 end 125 126 ## 127 # Skips whitespace tokens including newlines if +skip_nl+ is true 128 129 def skip_tkspace(skip_nl = true) # HACK dup 130 tokens = [] 131 132 while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do 133 tokens.push tk 134 end 135 136 unget_tk tk 137 tokens 138 end 139 140 ## 141 # Has +obj+ listen to tokens 142 143 def token_listener(obj) 144 add_token_listener obj 145 yield 146 ensure 147 remove_token_listener obj 148 end 149 150 ## 151 # Returns +tk+ to the scanner 152 153 def unget_tk(tk) 154 @tokens.unshift tk 155 @unget_read.unshift @read.pop 156 157 # Remove this token from any listeners 158 @token_listeners.each do |obj| 159 obj.pop_token 160 end if @token_listeners 161 162 nil 163 end 164 165end 166 167 168