1# coding: UTF-8
2
3require 'rdoc/test_case'
4
5class TestRDocRubyLex < RDoc::TestCase
6
7  def setup
8    @TK = RDoc::RubyToken
9  end
10
11  def mu_pp obj
12    s = ''
13    s = PP.pp obj, s
14    s = s.force_encoding(Encoding.default_external) if defined? Encoding
15    s.chomp
16  end
17
18  def test_class_tokenize
19    tokens = RDoc::RubyLex.tokenize "def x() end", nil
20
21    expected = [
22      @TK::TkDEF       .new( 0, 1,  0, "def"),
23      @TK::TkSPACE     .new( 3, 1,  3, " "),
24      @TK::TkIDENTIFIER.new( 4, 1,  4, "x"),
25      @TK::TkLPAREN    .new( 5, 1,  5, "("),
26      @TK::TkRPAREN    .new( 6, 1,  6, ")"),
27      @TK::TkSPACE     .new( 7, 1,  7, " "),
28      @TK::TkEND       .new( 8, 1,  8, "end"),
29      @TK::TkNL        .new(11, 1, 11, "\n"),
30    ]
31
32    assert_equal expected, tokens
33  end
34
35  def test_class_tokenize___END__
36    tokens = RDoc::RubyLex.tokenize '__END__', nil
37
38    expected = [
39      @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'),
40      @TK::TkNL           .new(7, 1, 7, "\n"),
41    ]
42
43    assert_equal expected, tokens
44  end
45
46  def test_class_tokenize_character_literal
47    tokens = RDoc::RubyLex.tokenize "?\\", nil
48
49    expected = [
50      @TK::TkSTRING.new( 0, 1,  0, "\\"),
51      @TK::TkNL    .new( 2, 1,  2, "\n"),
52    ]
53
54    assert_equal expected, tokens
55  end
56
57  def test_class_tokenize_def_heredoc
58    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
59def x
60  <<E
61Line 1
62Line 2
63E
64end
65    RUBY
66
67    expected = [
68      @TK::TkDEF       .new( 0, 1,  0, 'def'),
69      @TK::TkSPACE     .new( 3, 1,  3, ' '),
70      @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
71      @TK::TkNL        .new( 5, 1,  5, "\n"),
72      @TK::TkSPACE     .new( 6, 2,  0, '  '),
73      @TK::TkHEREDOC   .new( 8, 2,  2,
74                            %Q{<<E\nLine 1\nLine 2\nE}),
75      @TK::TkNL        .new(27, 5, 28, "\n"),
76      @TK::TkEND       .new(28, 6,  0, 'end'),
77      @TK::TkNL        .new(31, 6, 28, "\n"),
78    ]
79
80    assert_equal expected, tokens
81  end
82
83  def test_class_tokenize_hash_symbol
84    tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
85
86    expected = [
87      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
88      @TK::TkSPACE     .new( 1, 1,  1, ' '),
89      @TK::TkIDENTIFIER.new( 2, 1,  2, 'class'),
90      @TK::TkSYMBEG    .new( 7, 1,  7, ':'),
91      @TK::TkSTRING    .new( 8, 1,  8, '"foo"'),
92      @TK::TkSPACE     .new(13, 1, 13, ' '),
93      @TK::TkRBRACE    .new(14, 1, 14, '}'),
94      @TK::TkNL        .new(15, 1, 15, "\n"),
95    ]
96
97    assert_equal expected, tokens
98  end
99
100  def test_class_tokenize_heredoc_CR_NL
101    tokens = RDoc::RubyLex.tokenize <<-RUBY, nil
102string = <<-STRING\r
103Line 1\r
104Line 2\r
105  STRING\r
106    RUBY
107
108    expected = [
109      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
110      @TK::TkSPACE     .new( 6, 1,  6, ' '),
111      @TK::TkASSIGN    .new( 7, 1,  7, '='),
112      @TK::TkSPACE     .new( 8, 1,  8, ' '),
113      @TK::TkHEREDOC   .new( 9, 1,  9,
114                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
115      @TK::TkSPACE     .new(44, 4, 45, "\r"),
116      @TK::TkNL        .new(45, 4, 46, "\n"),
117    ]
118
119    assert_equal expected, tokens
120  end
121
122  def test_class_tokenize_heredoc_call
123    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
124string = <<-STRING.chomp
125Line 1
126Line 2
127  STRING
128    RUBY
129
130    expected = [
131      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
132      @TK::TkSPACE     .new( 6, 1,  6, ' '),
133      @TK::TkASSIGN    .new( 7, 1,  7, '='),
134      @TK::TkSPACE     .new( 8, 1,  8, ' '),
135      @TK::TkSTRING    .new( 9, 1,  9, %Q{"Line 1\nLine 2\n"}),
136      @TK::TkDOT       .new(41, 4, 42, '.'),
137      @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'),
138      @TK::TkNL        .new(47, 4, 48, "\n"),
139    ]
140
141    assert_equal expected, tokens
142  end
143
144  def test_class_tokenize_heredoc_indent
145    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
146string = <<-STRING
147Line 1
148Line 2
149  STRING
150    RUBY
151
152    expected = [
153      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
154      @TK::TkSPACE     .new( 6, 1,  6, ' '),
155      @TK::TkASSIGN    .new( 7, 1,  7, '='),
156      @TK::TkSPACE     .new( 8, 1,  8, ' '),
157      @TK::TkHEREDOC   .new( 9, 1,  9,
158                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
159      @TK::TkNL        .new(41, 4, 42, "\n"),
160    ]
161
162    assert_equal expected, tokens
163  end
164
165  def test_class_tokenize_heredoc_percent_N
166    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
167a b <<-U
168%N
169U
170    RUBY
171
172    expected = [
173      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
174      @TK::TkSPACE     .new( 1, 1,  1, ' '),
175      @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
176      @TK::TkSPACE     .new( 3, 1,  3, ' '),
177      @TK::TkHEREDOC   .new( 4, 1,  4, %Q{<<-U\n%N\nU}),
178      @TK::TkNL        .new(13, 3, 14, "\n"),
179    ]
180
181    assert_equal expected, tokens
182  end
183
184  def test_class_tokenize_identifier_high_unicode
185    tokens = RDoc::RubyLex.tokenize '����', nil
186
187    expected = @TK::TkIDENTIFIER.new(0, 1, 0, '����')
188
189    assert_equal expected, tokens.first
190  end
191
192  def test_class_tokenize_percent_1
193    tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
194
195    expected = [
196      @TK::TkIDENTIFIER.new(0, 1, 0, 'v'),
197      @TK::TkMOD.new(       1, 1, 1, '%'),
198      @TK::TkINTEGER.new(   2, 1, 2, '10'),
199      @TK::TkEQ.new(        4, 1, 4, '=='),
200      @TK::TkINTEGER.new(   6, 1, 6, '10'),
201      @TK::TkNL.new(        8, 1, 8, "\n"),
202    ]
203
204    assert_equal expected, tokens
205  end
206
207  def test_class_tokenize_percent_r
208    tokens = RDoc::RubyLex.tokenize '%r[hi]', nil
209
210    expected = [
211      @TK::TkREGEXP.new( 0, 1,  0, '%r[hi]'),
212      @TK::TkNL    .new( 6, 1, 6, "\n"),
213    ]
214
215    assert_equal expected, tokens
216  end
217
218  def test_class_tokenize_percent_w
219    tokens = RDoc::RubyLex.tokenize '%w[hi]', nil
220
221    expected = [
222      @TK::TkDSTRING.new( 0, 1,  0, '%w[hi]'),
223      @TK::TkNL     .new( 6, 1, 6, "\n"),
224    ]
225
226    assert_equal expected, tokens
227  end
228
229  def test_class_tokenize_regexp
230    tokens = RDoc::RubyLex.tokenize "/hay/", nil
231
232    expected = [
233      @TK::TkREGEXP.new( 0, 1,  0, "/hay/"),
234      @TK::TkNL    .new( 5, 1,  5, "\n"),
235    ]
236
237    assert_equal expected, tokens
238  end
239
240  def test_class_tokenize_regexp_backref
241    tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil
242
243    expected = [
244      @TK::TkREGEXP.new( 0, 1,  0, "/[csh](..) [csh]\\1 in/"),
245      @TK::TkNL    .new(22, 1, 22, "\n"),
246    ]
247
248    assert_equal expected, tokens
249  end
250
251  def test_class_tokenize_regexp_escape
252    tokens = RDoc::RubyLex.tokenize "/\\//", nil
253
254    expected = [
255      @TK::TkREGEXP.new( 0, 1,  0, "/\\//"),
256      @TK::TkNL    .new( 4, 1,  4, "\n"),
257    ]
258
259    assert_equal expected, tokens
260  end
261
262  def test_class_tokenize_string
263    tokens = RDoc::RubyLex.tokenize "'hi'", nil
264
265    expected = [
266      @TK::TkSTRING.new( 0, 1,  0, "'hi'"),
267      @TK::TkNL    .new( 4, 1,  4, "\n"),
268    ]
269
270    assert_equal expected, tokens
271  end
272
273  def test_unary_minus
274    ruby_lex = RDoc::RubyLex.new("-1", nil)
275    assert_equal("-1", ruby_lex.token.value)
276
277    ruby_lex = RDoc::RubyLex.new("a[-2]", nil)
278    2.times { ruby_lex.token } # skip "a" and "["
279    assert_equal("-2", ruby_lex.token.value)
280
281    ruby_lex = RDoc::RubyLex.new("a[0..-12]", nil)
282    4.times { ruby_lex.token } # skip "a", "[", "0", and ".."
283    assert_equal("-12", ruby_lex.token.value)
284
285    ruby_lex = RDoc::RubyLex.new("0+-0.1", nil)
286    2.times { ruby_lex.token } # skip "0" and "+"
287    assert_equal("-0.1", ruby_lex.token.value)
288  end
289
290end
291
292