1# coding: UTF-8 2 3require 'rdoc/test_case' 4 5class TestRDocRubyLex < RDoc::TestCase 6 7 def setup 8 @TK = RDoc::RubyToken 9 end 10 11 def mu_pp obj 12 s = '' 13 s = PP.pp obj, s 14 s = s.force_encoding(Encoding.default_external) if defined? Encoding 15 s.chomp 16 end 17 18 def test_class_tokenize 19 tokens = RDoc::RubyLex.tokenize "def x() end", nil 20 21 expected = [ 22 @TK::TkDEF .new( 0, 1, 0, "def"), 23 @TK::TkSPACE .new( 3, 1, 3, " "), 24 @TK::TkIDENTIFIER.new( 4, 1, 4, "x"), 25 @TK::TkLPAREN .new( 5, 1, 5, "("), 26 @TK::TkRPAREN .new( 6, 1, 6, ")"), 27 @TK::TkSPACE .new( 7, 1, 7, " "), 28 @TK::TkEND .new( 8, 1, 8, "end"), 29 @TK::TkNL .new(11, 1, 11, "\n"), 30 ] 31 32 assert_equal expected, tokens 33 end 34 35 def test_class_tokenize___END__ 36 tokens = RDoc::RubyLex.tokenize '__END__', nil 37 38 expected = [ 39 @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'), 40 @TK::TkNL .new(7, 1, 7, "\n"), 41 ] 42 43 assert_equal expected, tokens 44 end 45 46 def test_class_tokenize_character_literal 47 tokens = RDoc::RubyLex.tokenize "?\\", nil 48 49 expected = [ 50 @TK::TkSTRING.new( 0, 1, 0, "\\"), 51 @TK::TkNL .new( 2, 1, 2, "\n"), 52 ] 53 54 assert_equal expected, tokens 55 end 56 57 def test_class_tokenize_def_heredoc 58 tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil 59def x 60 <<E 61Line 1 62Line 2 63E 64end 65 RUBY 66 67 expected = [ 68 @TK::TkDEF .new( 0, 1, 0, 'def'), 69 @TK::TkSPACE .new( 3, 1, 3, ' '), 70 @TK::TkIDENTIFIER.new( 4, 1, 4, 'x'), 71 @TK::TkNL .new( 5, 1, 5, "\n"), 72 @TK::TkSPACE .new( 6, 2, 0, ' '), 73 @TK::TkHEREDOC .new( 8, 2, 2, 74 %Q{<<E\nLine 1\nLine 2\nE}), 75 @TK::TkNL .new(27, 5, 28, "\n"), 76 @TK::TkEND .new(28, 6, 0, 'end'), 77 @TK::TkNL .new(31, 6, 28, "\n"), 78 ] 79 80 assert_equal expected, tokens 81 end 82 83 def test_class_tokenize_hash_symbol 84 tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil 85 86 expected = [ 87 @TK::TkLBRACE .new( 0, 1, 0, '{'), 88 @TK::TkSPACE .new( 1, 1, 1, ' '), 89 @TK::TkIDENTIFIER.new( 2, 1, 2, 'class'), 90 @TK::TkSYMBEG .new( 7, 1, 7, ':'), 91 @TK::TkSTRING .new( 8, 1, 8, '"foo"'), 92 @TK::TkSPACE .new(13, 1, 13, ' '), 93 @TK::TkRBRACE .new(14, 1, 14, '}'), 94 @TK::TkNL .new(15, 1, 15, "\n"), 95 ] 96 97 assert_equal expected, tokens 98 end 99 100 def test_class_tokenize_heredoc_CR_NL 101 tokens = RDoc::RubyLex.tokenize <<-RUBY, nil 102string = <<-STRING\r 103Line 1\r 104Line 2\r 105 STRING\r 106 RUBY 107 108 expected = [ 109 @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), 110 @TK::TkSPACE .new( 6, 1, 6, ' '), 111 @TK::TkASSIGN .new( 7, 1, 7, '='), 112 @TK::TkSPACE .new( 8, 1, 8, ' '), 113 @TK::TkHEREDOC .new( 9, 1, 9, 114 %Q{<<-STRING\nLine 1\nLine 2\n STRING}), 115 @TK::TkSPACE .new(44, 4, 45, "\r"), 116 @TK::TkNL .new(45, 4, 46, "\n"), 117 ] 118 119 assert_equal expected, tokens 120 end 121 122 def test_class_tokenize_heredoc_call 123 tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil 124string = <<-STRING.chomp 125Line 1 126Line 2 127 STRING 128 RUBY 129 130 expected = [ 131 @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), 132 @TK::TkSPACE .new( 6, 1, 6, ' '), 133 @TK::TkASSIGN .new( 7, 1, 7, '='), 134 @TK::TkSPACE .new( 8, 1, 8, ' '), 135 @TK::TkSTRING .new( 9, 1, 9, %Q{"Line 1\nLine 2\n"}), 136 @TK::TkDOT .new(41, 4, 42, '.'), 137 @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'), 138 @TK::TkNL .new(47, 4, 48, "\n"), 139 ] 140 141 assert_equal expected, tokens 142 end 143 144 def test_class_tokenize_heredoc_indent 145 tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil 146string = <<-STRING 147Line 1 148Line 2 149 STRING 150 RUBY 151 152 expected = [ 153 @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), 154 @TK::TkSPACE .new( 6, 1, 6, ' '), 155 @TK::TkASSIGN .new( 7, 1, 7, '='), 156 @TK::TkSPACE .new( 8, 1, 8, ' '), 157 @TK::TkHEREDOC .new( 9, 1, 9, 158 %Q{<<-STRING\nLine 1\nLine 2\n STRING}), 159 @TK::TkNL .new(41, 4, 42, "\n"), 160 ] 161 162 assert_equal expected, tokens 163 end 164 165 def test_class_tokenize_heredoc_percent_N 166 tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil 167a b <<-U 168%N 169U 170 RUBY 171 172 expected = [ 173 @TK::TkIDENTIFIER.new( 0, 1, 0, 'a'), 174 @TK::TkSPACE .new( 1, 1, 1, ' '), 175 @TK::TkIDENTIFIER.new( 2, 1, 2, 'b'), 176 @TK::TkSPACE .new( 3, 1, 3, ' '), 177 @TK::TkHEREDOC .new( 4, 1, 4, %Q{<<-U\n%N\nU}), 178 @TK::TkNL .new(13, 3, 14, "\n"), 179 ] 180 181 assert_equal expected, tokens 182 end 183 184 def test_class_tokenize_identifier_high_unicode 185 tokens = RDoc::RubyLex.tokenize '����', nil 186 187 expected = @TK::TkIDENTIFIER.new(0, 1, 0, '����') 188 189 assert_equal expected, tokens.first 190 end 191 192 def test_class_tokenize_percent_1 193 tokens = RDoc::RubyLex.tokenize 'v%10==10', nil 194 195 expected = [ 196 @TK::TkIDENTIFIER.new(0, 1, 0, 'v'), 197 @TK::TkMOD.new( 1, 1, 1, '%'), 198 @TK::TkINTEGER.new( 2, 1, 2, '10'), 199 @TK::TkEQ.new( 4, 1, 4, '=='), 200 @TK::TkINTEGER.new( 6, 1, 6, '10'), 201 @TK::TkNL.new( 8, 1, 8, "\n"), 202 ] 203 204 assert_equal expected, tokens 205 end 206 207 def test_class_tokenize_percent_r 208 tokens = RDoc::RubyLex.tokenize '%r[hi]', nil 209 210 expected = [ 211 @TK::TkREGEXP.new( 0, 1, 0, '%r[hi]'), 212 @TK::TkNL .new( 6, 1, 6, "\n"), 213 ] 214 215 assert_equal expected, tokens 216 end 217 218 def test_class_tokenize_percent_w 219 tokens = RDoc::RubyLex.tokenize '%w[hi]', nil 220 221 expected = [ 222 @TK::TkDSTRING.new( 0, 1, 0, '%w[hi]'), 223 @TK::TkNL .new( 6, 1, 6, "\n"), 224 ] 225 226 assert_equal expected, tokens 227 end 228 229 def test_class_tokenize_regexp 230 tokens = RDoc::RubyLex.tokenize "/hay/", nil 231 232 expected = [ 233 @TK::TkREGEXP.new( 0, 1, 0, "/hay/"), 234 @TK::TkNL .new( 5, 1, 5, "\n"), 235 ] 236 237 assert_equal expected, tokens 238 end 239 240 def test_class_tokenize_regexp_backref 241 tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil 242 243 expected = [ 244 @TK::TkREGEXP.new( 0, 1, 0, "/[csh](..) [csh]\\1 in/"), 245 @TK::TkNL .new(22, 1, 22, "\n"), 246 ] 247 248 assert_equal expected, tokens 249 end 250 251 def test_class_tokenize_regexp_escape 252 tokens = RDoc::RubyLex.tokenize "/\\//", nil 253 254 expected = [ 255 @TK::TkREGEXP.new( 0, 1, 0, "/\\//"), 256 @TK::TkNL .new( 4, 1, 4, "\n"), 257 ] 258 259 assert_equal expected, tokens 260 end 261 262 def test_class_tokenize_string 263 tokens = RDoc::RubyLex.tokenize "'hi'", nil 264 265 expected = [ 266 @TK::TkSTRING.new( 0, 1, 0, "'hi'"), 267 @TK::TkNL .new( 4, 1, 4, "\n"), 268 ] 269 270 assert_equal expected, tokens 271 end 272 273 def test_unary_minus 274 ruby_lex = RDoc::RubyLex.new("-1", nil) 275 assert_equal("-1", ruby_lex.token.value) 276 277 ruby_lex = RDoc::RubyLex.new("a[-2]", nil) 278 2.times { ruby_lex.token } # skip "a" and "[" 279 assert_equal("-2", ruby_lex.token.value) 280 281 ruby_lex = RDoc::RubyLex.new("a[0..-12]", nil) 282 4.times { ruby_lex.token } # skip "a", "[", "0", and ".." 283 assert_equal("-12", ruby_lex.token.value) 284 285 ruby_lex = RDoc::RubyLex.new("0+-0.1", nil) 286 2.times { ruby_lex.token } # skip "0" and "+" 287 assert_equal("-0.1", ruby_lex.token.value) 288 end 289 290end 291 292