1# 2# test_scanner_events.rb 3# 4begin 5 require 'ripper' 6 require 'test/unit' 7 ripper_test = true 8 module TestRipper; end 9rescue LoadError 10end 11 12class TestRipper::ScannerEvents < Test::Unit::TestCase 13 14 def test_event_coverage 15 dispatched = Ripper::SCANNER_EVENTS.map {|event,_| event } 16 dispatched.each do |e| 17 assert_equal true, respond_to?("test_#{e}", true), "event not tested: #{e}" 18 end 19 end 20 21 def scan(target, str) 22 sym = "on_#{target}".intern 23 Ripper.lex(str).select {|_1,type,_2| type == sym }.map {|_1,_2,tok| tok } 24 end 25 26 def test_tokenize 27 assert_equal [], 28 Ripper.tokenize('') 29 assert_equal ['a'], 30 Ripper.tokenize('a') 31 assert_equal ['1'], 32 Ripper.tokenize('1') 33 assert_equal ['1', ';', 'def', ' ', 'm', '(', 'arg', ')', 'end'], 34 Ripper.tokenize("1;def m(arg)end") 35 assert_equal ['print', '(', '<<''EOS', ')', "\n", "heredoc\n", "EOS\n"], 36 Ripper.tokenize("print(<<""EOS)\nheredoc\nEOS\n") 37 assert_equal ['print', '(', ' ', '<<''EOS', ')', "\n", "heredoc\n", "EOS\n"], 38 Ripper.tokenize("print( <<""EOS)\nheredoc\nEOS\n") 39 assert_equal ["\#\n", "\n", "\#\n", "\n", "nil", "\n"], 40 Ripper.tokenize("\#\n\n\#\n\nnil\n") 41 end 42 43 def test_lex 44 assert_equal [], 45 Ripper.lex('') 46 assert_equal [[[1,0], :on_ident, "a"]], 47 Ripper.lex('a') 48 assert_equal [[[1, 0], :on_kw, "nil"]], 49 Ripper.lex("nil") 50 assert_equal [[[1, 0], :on_kw, "def"], 51 [[1, 3], :on_sp, " "], 52 [[1, 4], :on_ident, "m"], 53 [[1, 5], :on_lparen, "("], 54 [[1, 6], :on_ident, "a"], 55 [[1, 7], :on_rparen, ")"], 56 [[1, 8], :on_kw, "end"]], 57 Ripper.lex("def m(a)end") 58 assert_equal [[[1, 0], :on_int, "1"], 59 [[1, 1], :on_nl, "\n"], 60 [[2, 0], :on_int, "2"], 61 [[2, 1], :on_nl, "\n"], 62 [[3, 0], :on_int, "3"]], 63 Ripper.lex("1\n2\n3") 64 assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"], 65 [[1, 5], :on_nl, "\n"], 66 [[2, 0], :on_tstring_content, "heredoc\n"], 67 [[3, 0], :on_heredoc_end, "EOS"]], 68 Ripper.lex("<<""EOS\nheredoc\nEOS") 69 assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"], 70 [[1, 5], :on_nl, "\n"], 71 [[2, 0], :on_heredoc_end, "EOS"]], 72 Ripper.lex("<<""EOS\nEOS"), 73 "bug#4543" 74 assert_equal [[[1, 0], :on_regexp_beg, "/"], 75 [[1, 1], :on_tstring_content, "foo\nbar"], 76 [[2, 3], :on_regexp_end, "/"]], 77 Ripper.lex("/foo\nbar/") 78 assert_equal [[[1, 0], :on_regexp_beg, "/"], 79 [[1, 1], :on_tstring_content, "foo\n\u3020"], 80 [[2, 3], :on_regexp_end, "/"]], 81 Ripper.lex("/foo\n\u3020/") 82 assert_equal [[[1, 0], :on_tstring_beg, "'"], 83 [[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0"], 84 [[2, 3], :on_tstring_end, "'"]], 85 Ripper.lex("'foo\n\xe3\x80\xa0'") 86 assert_equal [[[1, 0], :on_tstring_beg, "'"], 87 [[1, 1], :on_tstring_content, "\u3042\n\u3044"], 88 [[2, 3], :on_tstring_end, "'"]], 89 Ripper.lex("'\u3042\n\u3044'") 90 end 91 92 def test_location 93 assert_location "" 94 assert_location " " 95 assert_location ":" 96 assert_location "\n" 97 assert_location "\r\n" 98 assert_location "\n\n\n\n\n\r\n\n\n" 99 assert_location "\n;\n;\n;\n;\n" 100 assert_location "nil" 101 assert_location "@ivar" 102 assert_location "1;2;3" 103 assert_location "1\n2\n3" 104 assert_location "1\n2\n3\n" 105 assert_location "def m(a) nil end" 106 assert_location "if true then false else nil end" 107 assert_location "BEGIN{print nil}" 108 assert_location "%w(a b\nc\r\nd \ne )" 109 assert_location %Q["a\nb\r\nc"] 110 assert_location "print(<<""EOS)\nheredoc\nEOS\n" 111 assert_location "print(<<-\"EOS\")\nheredoc\n EOS\n" 112 end 113 114 def assert_location(src) 115 buf = '' 116 Ripper.lex(src).each do |pos, type, tok| 117 line, col = *pos 118 assert_equal buf.count("\n") + 1, line, 119 "wrong lineno: #{tok.inspect} (#{type}) [#{line}:#{col}]" 120 assert_equal buf.sub(/\A.*\n/m, '').size, col, 121 "wrong column: #{tok.inspect} (#{type}) [#{line}:#{col}]" 122 buf << tok 123 end 124 assert_equal src, buf 125 end 126 127 def test_backref 128 assert_equal ["$`", "$&", "$'", '$1', '$2', '$3'], 129 scan('backref', %q[m($~, $`, $&, $', $1, $2, $3)]) 130 end 131 132 def test_backtick 133 assert_equal ["`"], 134 scan('backtick', %q[p `make all`]) 135 end 136 137 def test_comma 138 assert_equal [','] * 6, 139 scan('comma', %q[ m(0,1,2,3,4,5,6) ]) 140 assert_equal [], 141 scan('comma', %q[".,.,.,.,.,.,.."]) 142 assert_equal [], 143 scan('comma', "<<""EOS\n,,,,,,,,,,\nEOS") 144 end 145 146 def test_period 147 assert_equal [], 148 scan('period', '') 149 assert_equal ['.'], 150 scan('period', 'a.b') 151 assert_equal ['.'], 152 scan('period', 'Object.new') 153 assert_equal [], 154 scan('period', '"."') 155 assert_equal [], 156 scan('period', '1..2') 157 assert_equal [], 158 scan('period', '1...3') 159 end 160 161 def test_const 162 assert_equal ['CONST'], 163 scan('const', 'CONST') 164 assert_equal ['C'], 165 scan('const', 'C') 166 assert_equal ['CONST_A'], 167 scan('const', 'CONST_A') 168 assert_equal ['Const', 'Const2', 'Const3'], 169 scan('const', 'Const; Const2; Const3') 170 assert_equal ['Const'], 171 scan('const', 'Const(a)') 172 assert_equal ['M', 'A', 'A2'], 173 scan('const', 'M(A,A2)') 174 assert_equal [], 175 scan('const', '') 176 assert_equal [], 177 scan('const', 'm(lvar, @ivar, @@cvar, $gvar)') 178 end 179 180 def test_cvar 181 assert_equal [], 182 scan('cvar', '') 183 assert_equal ['@@cvar'], 184 scan('cvar', '@@cvar') 185 assert_equal ['@@__cvar__'], 186 scan('cvar', '@@__cvar__') 187 assert_equal ['@@CVAR'], 188 scan('cvar', '@@CVAR') 189 assert_equal ['@@cvar'], 190 scan('cvar', ' @@cvar#comment') 191 assert_equal ['@@cvar'], 192 scan('cvar', ':@@cvar') 193 assert_equal ['@@cvar'], 194 scan('cvar', 'm(lvar, @ivar, @@cvar, $gvar)') 195 assert_equal [], 196 scan('cvar', '"@@cvar"') 197 end 198 199 def test_embexpr_beg 200 assert_equal [], 201 scan('embexpr_beg', '') 202 assert_equal ['#{'], 203 scan('embexpr_beg', '"#{expr}"') 204 assert_equal [], 205 scan('embexpr_beg', '%q[#{expr}]') 206 assert_equal ['#{'], 207 scan('embexpr_beg', '%Q[#{expr}]') 208 assert_equal ['#{'], 209 scan('embexpr_beg', "m(<<""EOS)\n\#{expr}\nEOS") 210 end 211 212 def test_embexpr_end 213 assert_equal [], 214 scan('embexpr_end', '') 215 assert_equal ['}'], 216 scan('embexpr_end', '"#{expr}"') 217 assert_equal [], 218 scan('embexpr_end', '%q[#{expr}]') 219 assert_equal ['}'], 220 scan('embexpr_end', '%Q[#{expr}]') 221 assert_equal ['}'], 222 scan('embexpr_end', "m(<<""EOS)\n\#{expr}\nEOS") 223 end 224 225 def test_embvar 226 assert_equal [], 227 scan('embvar', '') 228 assert_equal ['#'], 229 scan('embvar', '"#$gvar"') 230 assert_equal ['#'], 231 scan('embvar', '"#@ivar"') 232 assert_equal ['#'], 233 scan('embvar', '"#@@cvar"') 234 assert_equal [], 235 scan('embvar', '"#lvar"') 236 assert_equal [], 237 scan('embvar', '"#"') 238 assert_equal [], 239 scan('embvar', '"\#$gvar"') 240 assert_equal [], 241 scan('embvar', '"\#@ivar"') 242 assert_equal [], 243 scan('embvar', '%q[#@ivar]') 244 assert_equal ['#'], 245 scan('embvar', '%Q[#@ivar]') 246 end 247 248 def test_float 249 assert_equal [], 250 scan('float', '') 251 assert_equal ['1.000'], 252 scan('float', '1.000') 253 assert_equal ['123.456'], 254 scan('float', '123.456') 255 assert_equal ['1.2345678901234567890123456789'], 256 scan('float', '1.2345678901234567890123456789') 257 assert_equal ['1.000'], 258 scan('float', ' 1.000# comment') 259 assert_equal ['1.234e5'], 260 scan('float', '1.234e5') 261 assert_equal ['1.234e1234567890'], 262 scan('float', '1.234e1234567890') 263 assert_equal ['1.0'], 264 scan('float', 'm(a,b,1.0,c,d)') 265 end 266 267 def test_gvar 268 assert_equal [], 269 scan('gvar', '') 270 assert_equal ['$a'], 271 scan('gvar', '$a') 272 assert_equal ['$A'], 273 scan('gvar', '$A') 274 assert_equal ['$gvar'], 275 scan('gvar', 'm(lvar, @ivar, @@cvar, $gvar)') 276 assert_equal %w($_ $~ $* $$ $? $! $@ $/ $\\ $; $, $. $= $: $< $> $"), 277 scan('gvar', 'm($_, $~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $")') 278 end 279 280 def test_ident 281 assert_equal [], 282 scan('ident', '') 283 assert_equal ['lvar'], 284 scan('ident', 'lvar') 285 assert_equal ['m', 'lvar'], 286 scan('ident', 'm(lvar, @ivar, @@cvar, $gvar)') 287 end 288 289 def test_int 290 assert_equal [], 291 scan('int', '') 292 assert_equal ['1', '10', '100000000000000'], 293 scan('int', 'm(1,10,100000000000000)') 294 end 295 296 def test_ivar 297 assert_equal [], 298 scan('ivar', '') 299 assert_equal ['@ivar'], 300 scan('ivar', '@ivar') 301 assert_equal ['@__ivar__'], 302 scan('ivar', '@__ivar__') 303 assert_equal ['@IVAR'], 304 scan('ivar', '@IVAR') 305 assert_equal ['@ivar'], 306 scan('ivar', 'm(lvar, @ivar, @@cvar, $gvar)') 307 end 308 309 def test_kw 310 assert_equal [], 311 scan('kw', '') 312 assert_equal %w(not), 313 scan('kw', 'not 1') 314 assert_equal %w(and), 315 scan('kw', '1 and 2') 316 assert_equal %w(or), 317 scan('kw', '1 or 2') 318 assert_equal %w(if then else end), 319 scan('kw', 'if 1 then 2 else 3 end') 320 assert_equal %w(if then elsif else end), 321 scan('kw', 'if 1 then 2 elsif 3 else 4 end') 322 assert_equal %w(unless then end), 323 scan('kw', 'unless 1 then end') 324 assert_equal %w(if true), 325 scan('kw', '1 if true') 326 assert_equal %w(unless false), 327 scan('kw', '2 unless false') 328 assert_equal %w(case when when else end), 329 scan('kw', 'case n; when 1; when 2; else 3 end') 330 assert_equal %w(while do nil end), 331 scan('kw', 'while 1 do nil end') 332 assert_equal %w(until do nil end), 333 scan('kw', 'until 1 do nil end') 334 assert_equal %w(while), 335 scan('kw', '1 while 2') 336 assert_equal %w(until), 337 scan('kw', '1 until 2') 338 assert_equal %w(while break next retry end), 339 scan('kw', 'while 1; break; next; retry end') 340 assert_equal %w(for in next break end), 341 scan('kw', 'for x in obj; next 1; break 2 end') 342 assert_equal %w(begin rescue retry end), 343 scan('kw', 'begin 1; rescue; retry; end') 344 assert_equal %w(rescue), 345 scan('kw', '1 rescue 2') 346 assert_equal %w(def redo return end), 347 scan('kw', 'def m() redo; return end') 348 assert_equal %w(def yield yield end), 349 scan('kw', 'def m() yield; yield 1 end') 350 assert_equal %w(def super super super end), 351 scan('kw', 'def m() super; super(); super(1) end') 352 assert_equal %w(alias), 353 scan('kw', 'alias a b') 354 assert_equal %w(undef), 355 scan('kw', 'undef public') 356 assert_equal %w(class end), 357 scan('kw', 'class A < Object; end') 358 assert_equal %w(module end), 359 scan('kw', 'module M; end') 360 assert_equal %w(class end), 361 scan('kw', 'class << obj; end') 362 assert_equal %w(BEGIN), 363 scan('kw', 'BEGIN { }') 364 assert_equal %w(END), 365 scan('kw', 'END { }') 366 assert_equal %w(self), 367 scan('kw', 'self.class') 368 assert_equal %w(nil true false), 369 scan('kw', 'p(nil, true, false)') 370 assert_equal %w(__FILE__ __LINE__), 371 scan('kw', 'p __FILE__, __LINE__') 372 assert_equal %w(defined?), 373 scan('kw', 'defined?(Object)') 374 end 375 376 def test_lbrace 377 assert_equal [], 378 scan('lbrace', '') 379 assert_equal ['{'], 380 scan('lbrace', '3.times{ }') 381 assert_equal ['{'], 382 scan('lbrace', '3.times { }') 383 assert_equal ['{'], 384 scan('lbrace', '3.times{}') 385 assert_equal [], 386 scan('lbrace', '"{}"') 387 assert_equal ['{'], 388 scan('lbrace', '{1=>2}') 389 end 390 391 def test_rbrace 392 assert_equal [], 393 scan('rbrace', '') 394 assert_equal ['}'], 395 scan('rbrace', '3.times{ }') 396 assert_equal ['}'], 397 scan('rbrace', '3.times { }') 398 assert_equal ['}'], 399 scan('rbrace', '3.times{}') 400 assert_equal [], 401 scan('rbrace', '"{}"') 402 assert_equal ['}'], 403 scan('rbrace', '{1=>2}') 404 end 405 406 def test_lbracket 407 assert_equal [], 408 scan('lbracket', '') 409 assert_equal ['['], 410 scan('lbracket', '[]') 411 assert_equal ['['], 412 scan('lbracket', 'a[1]') 413 assert_equal [], 414 scan('lbracket', 'm(%q[])') 415 end 416 417 def test_rbracket 418 assert_equal [], 419 scan('rbracket', '') 420 assert_equal [']'], 421 scan('rbracket', '[]') 422 assert_equal [']'], 423 scan('rbracket', 'a[1]') 424 assert_equal [], 425 scan('rbracket', 'm(%q[])') 426 end 427 428 def test_lparen 429 assert_equal [], 430 scan('lparen', '') 431 assert_equal ['('], 432 scan('lparen', '()') 433 assert_equal ['('], 434 scan('lparen', 'm()') 435 assert_equal ['('], 436 scan('lparen', 'm (a)') 437 assert_equal [], 438 scan('lparen', '"()"') 439 assert_equal [], 440 scan('lparen', '"%w()"') 441 end 442 443 def test_rparen 444 assert_equal [], 445 scan('rparen', '') 446 assert_equal [')'], 447 scan('rparen', '()') 448 assert_equal [')'], 449 scan('rparen', 'm()') 450 assert_equal [')'], 451 scan('rparen', 'm (a)') 452 assert_equal [], 453 scan('rparen', '"()"') 454 assert_equal [], 455 scan('rparen', '"%w()"') 456 end 457 458 def test_op 459 assert_equal [], 460 scan('op', '') 461 assert_equal ['|'], 462 scan('op', '1 | 1') 463 assert_equal ['^'], 464 scan('op', '1 ^ 1') 465 assert_equal ['&'], 466 scan('op', '1 & 1') 467 assert_equal ['<=>'], 468 scan('op', '1 <=> 1') 469 assert_equal ['=='], 470 scan('op', '1 == 1') 471 assert_equal ['==='], 472 scan('op', '1 === 1') 473 assert_equal ['=~'], 474 scan('op', '1 =~ 1') 475 assert_equal ['>'], 476 scan('op', '1 > 1') 477 assert_equal ['>='], 478 scan('op', '1 >= 1') 479 assert_equal ['<'], 480 scan('op', '1 < 1') 481 assert_equal ['<='], 482 scan('op', '1 <= 1') 483 assert_equal ['<''<'], 484 scan('op', '1 <''< 1') 485 assert_equal ['>>'], 486 scan('op', '1 >> 1') 487 assert_equal ['+'], 488 scan('op', '1 + 1') 489 assert_equal ['-'], 490 scan('op', '1 - 1') 491 assert_equal ['*'], 492 scan('op', '1 * 1') 493 assert_equal ['/'], 494 scan('op', '1 / 1') 495 assert_equal ['%'], 496 scan('op', '1 % 1') 497 assert_equal ['**'], 498 scan('op', '1 ** 1') 499 assert_equal ['~'], 500 scan('op', '~1') 501 assert_equal ['-'], 502 scan('op', '-a') 503 assert_equal ['+'], 504 scan('op', '+a') 505 assert_equal ['[]'], 506 scan('op', ':[]') 507 assert_equal ['[]='], 508 scan('op', ':[]=') 509 assert_equal [], 510 scan('op', %q[`make all`]) 511 end 512 513 def test_symbeg 514 assert_equal [], 515 scan('symbeg', '') 516 assert_equal [':'], 517 scan('symbeg', ':sym') 518 assert_equal [':'], 519 scan('symbeg', '[1,2,3,:sym]') 520 assert_equal [], 521 scan('symbeg', '":sym"') 522 assert_equal [], 523 scan('symbeg', 'a ? b : c') 524 end 525 526 def test_tstring_beg 527 assert_equal [], 528 scan('tstring_beg', '') 529 assert_equal ['"'], 530 scan('tstring_beg', '"abcdef"') 531 assert_equal ['%q['], 532 scan('tstring_beg', '%q[abcdef]') 533 assert_equal ['%Q['], 534 scan('tstring_beg', '%Q[abcdef]') 535 end 536 537 def test_tstring_content 538 assert_equal [], 539 scan('tstring_content', '') 540 assert_equal ['abcdef'], 541 scan('tstring_content', '"abcdef"') 542 assert_equal ['abcdef'], 543 scan('tstring_content', '%q[abcdef]') 544 assert_equal ['abcdef'], 545 scan('tstring_content', '%Q[abcdef]') 546 assert_equal ['abc', 'def'], 547 scan('tstring_content', '"abc#{1}def"') 548 assert_equal ['sym'], 549 scan('tstring_content', ':"sym"') 550 assert_equal ['a b c'], 551 scan('tstring_content', ':"a b c"'), 552 "bug#4544" 553 assert_equal ["a\nb\nc"], 554 scan('tstring_content', ":'a\nb\nc'"), 555 "bug#4544" 556 end 557 558 def test_tstring_end 559 assert_equal [], 560 scan('tstring_end', '') 561 assert_equal ['"'], 562 scan('tstring_end', '"abcdef"') 563 assert_equal [']'], 564 scan('tstring_end', '%q[abcdef]') 565 assert_equal [']'], 566 scan('tstring_end', '%Q[abcdef]') 567 end 568 569 def test_regexp_beg 570 assert_equal [], 571 scan('regexp_beg', '') 572 assert_equal ['/'], 573 scan('regexp_beg', '/re/') 574 assert_equal ['%r<'], 575 scan('regexp_beg', '%r<re>') 576 assert_equal [], 577 scan('regexp_beg', '5 / 5') 578 end 579 580 def test_regexp_end 581 assert_equal [], 582 scan('regexp_end', '') 583 assert_equal ['/'], 584 scan('regexp_end', '/re/') 585 assert_equal ['>'], 586 scan('regexp_end', '%r<re>') 587 end 588 589 def test_words_beg 590 assert_equal [], 591 scan('words_beg', '') 592 assert_equal ['%W('], 593 scan('words_beg', '%W()') 594 assert_equal ['%W('], 595 scan('words_beg', '%W(w w w)') 596 assert_equal ['%W( '], 597 scan('words_beg', '%W( w w w )') 598 end 599 600 def test_qwords_beg 601 assert_equal [], 602 scan('qwords_beg', '') 603 assert_equal ['%w('], 604 scan('qwords_beg', '%w()') 605 assert_equal ['%w('], 606 scan('qwords_beg', '%w(w w w)') 607 assert_equal ['%w( '], 608 scan('qwords_beg', '%w( w w w )') 609 end 610 611 def test_qsymbols_beg 612 assert_equal [], 613 scan('qsymbols_beg', '') 614 assert_equal ['%i('], 615 scan('qsymbols_beg', '%i()') 616 assert_equal ['%i('], 617 scan('qsymbols_beg', '%i(w w w)') 618 assert_equal ['%i( '], 619 scan('qsymbols_beg', '%i( w w w )') 620 end 621 622 def test_symbols_beg 623 assert_equal [], 624 scan('symbols_beg', '') 625 assert_equal ['%I('], 626 scan('symbols_beg', '%I()') 627 assert_equal ['%I('], 628 scan('symbols_beg', '%I(w w w)') 629 assert_equal ['%I( '], 630 scan('symbols_beg', '%I( w w w )') 631 end 632 633 # FIXME: Close paren must not present (`words_end' scanner event?). 634 def test_words_sep 635 assert_equal [], 636 scan('words_sep', '') 637 assert_equal [')'], 638 scan('words_sep', '%w()') 639 assert_equal [' ', ' ', ')'], 640 scan('words_sep', '%w(w w w)') 641 assert_equal [' ', ' ', ' )'], 642 scan('words_sep', '%w( w w w )') 643 assert_equal ["\n", ' ', ' )'], 644 scan('words_sep', "%w( w\nw w )") 645 end 646 647 def test_heredoc_beg 648 assert_equal [], 649 scan('heredoc_beg', '') 650 assert_equal ['<<''EOS'], 651 scan('heredoc_beg', "<<""EOS\nheredoc\nEOS") 652 assert_equal ['<<''EOS'], 653 scan('heredoc_beg', "<<""EOS\nheredoc\nEOS\n") 654 assert_equal ['<<''EOS'], 655 scan('heredoc_beg', "<<""EOS\nheredoc\nEOS \n") 656 assert_equal ['<<''-EOS'], 657 scan('heredoc_beg', "<<""-EOS\nheredoc\n\tEOS \n") 658 assert_equal ['<<''"EOS"'], 659 scan('heredoc_beg', '<<''"EOS"'"\nheredoc\nEOS") 660 assert_equal ["<<""'EOS'"], 661 scan('heredoc_beg', "<<""'EOS'\nheredoc\nEOS") 662 assert_equal ['<<''`EOS`'], 663 scan('heredoc_beg', "<<""`EOS`\nheredoc\nEOS") 664 assert_equal ['<<''" "'], 665 scan('heredoc_beg', '<<''" "'"\nheredoc\nEOS") 666 end 667 668 def test_tstring_content_HEREDOC 669 assert_equal [], 670 scan('tstring_content', '') 671 assert_equal ["heredoc\n"], 672 scan('tstring_content', "<<""EOS\nheredoc\nEOS") 673 assert_equal ["heredoc\n"], 674 scan('tstring_content', "<<""EOS\nheredoc\nEOS\n") 675 assert_equal ["here\ndoc \nEOS \n"], 676 scan('tstring_content', "<<""EOS\nhere\ndoc \nEOS \n") 677 assert_equal ["heredoc\n\tEOS \n"], 678 scan('tstring_content', "<<""-EOS\nheredoc\n\tEOS \n") 679 bug7255 = '[ruby-core:48703]' 680 assert_equal ["there\n""heredoc", "\n"], 681 scan('tstring_content', "<<""EOS\n""there\n""heredoc\#{foo}\nEOS"), 682 bug7255 683 assert_equal ["there\n""heredoc", "\n"], 684 scan('tstring_content', "<<""EOS\n""there\n""heredoc\#@foo\nEOS"), 685 bug7255 686 end 687 688 def test_heredoc_end 689 assert_equal [], 690 scan('heredoc_end', '') 691 assert_equal ["EOS"], 692 scan('heredoc_end', "<<""EOS\nEOS"), 693 "bug#4543" 694 assert_equal ["EOS"], 695 scan('heredoc_end', "<<""EOS\nheredoc\nEOS") 696 assert_equal ["EOS\n"], 697 scan('heredoc_end', "<<""EOS\nheredoc\nEOS\n") 698 assert_equal [], 699 scan('heredoc_end', "<<""EOS\nheredoc\nEOS \n") 700 assert_equal [], 701 scan('heredoc_end', "<<""-EOS\nheredoc\n\tEOS \n") 702 end 703 704 def test_semicolon 705 assert_equal [], 706 scan('semicolon', '') 707 assert_equal %w(;), 708 scan('semicolon', ';') 709 assert_equal %w(; ;), 710 scan('semicolon', '; ;') 711 assert_equal %w(; ; ;), 712 scan('semicolon', 'nil;nil;nil;') 713 assert_equal %w(; ; ;), 714 scan('semicolon', 'nil;nil;nil;nil') 715 assert_equal [], 716 scan('semicolon', '";"') 717 assert_equal [], 718 scan('semicolon', '%w(;)') 719 assert_equal [], 720 scan('semicolon', '/;/') 721 end 722 723 def test_comment 724 assert_equal [], 725 scan('comment', '') 726 assert_equal ['# comment'], 727 scan('comment', '# comment') 728 assert_equal ["# comment\n"], 729 scan('comment', "# comment\n") 730 assert_equal ["# comment\n"], 731 scan('comment', "# comment\n1 + 1") 732 assert_equal ["# comment\n"], 733 scan('comment', "1 + 1 + 1# comment\n1 + 1") 734 end 735 736 def test_embdoc_beg 737 assert_equal [], 738 scan('embdoc_beg', '') 739 assert_equal ["=begin\n"], 740 scan('embdoc_beg', "=begin\ndoc\n=end") 741 assert_equal ["=begin \n"], 742 scan('embdoc_beg', "=begin \ndoc\n=end\n") 743 assert_equal ["=begin comment\n"], 744 scan('embdoc_beg', "=begin comment\ndoc\n=end\n") 745 end 746 747 def test_embdoc 748 assert_equal [], 749 scan('embdoc', '') 750 assert_equal ["doc\n"], 751 scan('embdoc', "=begin\ndoc\n=end") 752 assert_equal ["doc\n"], 753 scan('embdoc', "=begin\ndoc\n=end\n") 754 end 755 756 def test_embdoc_end 757 assert_equal [], 758 scan('embdoc_end', '') 759 assert_equal ["=end"], 760 scan('embdoc_end', "=begin\ndoc\n=end") 761 assert_equal ["=end\n"], 762 scan('embdoc_end', "=begin\ndoc\n=end\n") 763 end 764 765 def test_sp 766 assert_equal [], 767 scan('sp', '') 768 assert_equal [' '], 769 scan('sp', ' ') 770 assert_equal [' '], 771 scan('sp', ' 1') 772 assert_equal [], 773 scan('sp', "\n") 774 assert_equal [' '], 775 scan('sp', " \n") 776 assert_equal [' ', ' '], 777 scan('sp', "1 + 1") 778 assert_equal [], 779 scan('sp', "' '") 780 assert_equal [], 781 scan('sp', "%w( )") 782 assert_equal [], 783 scan('sp', "%w( w )") 784 assert_equal [], 785 scan('sp', "p(/ /)") 786 end 787 788 # `nl' event always means End-Of-Statement. 789 def test_nl 790 assert_equal [], 791 scan('nl', '') 792 assert_equal [], 793 scan('nl', "\n") 794 assert_equal ["\n"], 795 scan('nl', "1 + 1\n") 796 assert_equal ["\n", "\n"], 797 scan('nl', "1 + 1\n2 + 2\n") 798 assert_equal [], 799 scan('nl', "1 +\n1") 800 assert_equal [], 801 scan('nl', "1;\n") 802 assert_equal ["\r\n"], 803 scan('nl', "1 + 1\r\n") 804 assert_equal [], 805 scan('nl', "1;\r\n") 806 end 807 808 def test_ignored_nl 809 assert_equal [], 810 scan('ignored_nl', '') 811 assert_equal ["\n"], 812 scan('ignored_nl', "\n") 813 assert_equal [], 814 scan('ignored_nl', "1 + 1\n") 815 assert_equal [], 816 scan('ignored_nl', "1 + 1\n2 + 2\n") 817 assert_equal ["\n"], 818 scan('ignored_nl', "1 +\n1") 819 assert_equal ["\n"], 820 scan('ignored_nl', "1;\n") 821 assert_equal [], 822 scan('ignored_nl', "1 + 1\r\n") 823 assert_equal ["\r\n"], 824 scan('ignored_nl', "1;\r\n") 825 end 826 827 def test___end__ 828 assert_equal [], 829 scan('__end__', "") 830 assert_equal ["__END__"], 831 scan('__end__', "__END__") 832 assert_equal ["__END__\n"], 833 scan('__end__', "__END__\n") 834 assert_equal ["__END__\n"], 835 Ripper.tokenize("__END__\njunk junk junk") 836 assert_equal ["__END__"], 837 scan('__end__', "1\n__END__") 838 assert_equal [], 839 scan('__end__', "print('__END__')") 840 end 841 842 def test_CHAR 843 assert_equal [], 844 scan('CHAR', "") 845 assert_equal ["?a"], 846 scan('CHAR', "?a") 847 assert_equal [], 848 scan('CHAR', "@ivar") 849 end 850 851 def test_label 852 end 853 854 def test_tlambda 855 end 856 857 def test_tlambeg 858 end 859 860 def test_tlambda_arg 861 end 862 863end if ripper_test 864