1# coding: utf-8
2
3require 'rdoc/test_case'
4
5class TestRDocMarkupParser < RDoc::TestCase
6
7  def setup
8    super
9
10    @have_byteslice = ''.respond_to? :byteslice
11
12    @RMP = @RM::Parser
13  end
14
15  def mu_pp(obj)
16    s = ''
17    s = PP.pp obj, s
18    s = s.force_encoding(Encoding.default_external) if @have_encoding
19    s.chomp
20  end
21
22  def test_build_heading
23    parser = @RMP.new
24
25    parser.tokens.replace [
26      [:TEXT,    'heading three',  4, 0],
27      [:NEWLINE, "\n",            17, 0],
28    ]
29
30    assert_equal @RM::Heading.new(3, 'heading three'), parser.build_heading(3)
31  end
32
33  def test_char_pos
34    parser = @RMP.new
35    s = parser.setup_scanner 'cät'
36
37    s.scan(/\S+/)
38
39    if @have_byteslice or @have_encoding then
40      assert_equal 3, parser.char_pos(s.pos)
41    else
42      assert_equal 4, parser.char_pos(s.pos)
43    end
44  end
45
46  def test_get
47    parser = util_parser
48
49    assert_equal [:HEADER, 1, 0, 0], parser.get
50
51    assert_equal 7, parser.tokens.length
52  end
53
54  def test_parse_bullet
55    str = <<-STR
56* l1
57* l2
58    STR
59
60    expected = [
61      @RM::List.new(:BULLET, *[
62        @RM::ListItem.new(nil,
63          @RM::Paragraph.new('l1')),
64        @RM::ListItem.new(nil,
65          @RM::Paragraph.new('l2'))])]
66
67    assert_equal expected, @RMP.parse(str).parts
68  end
69
70  def test_parse_bullet_utf_8
71    str = <<-STR
72* ���������������
73    STR
74
75    expected = [
76      @RM::List.new(:BULLET, *[
77        @RM::ListItem.new(nil,
78          @RM::Paragraph.new('���������������'))])]
79
80    assert_equal expected, @RMP.parse(str).parts
81  end
82
83  def test_parse_bullet_verbatim_heading
84    str = <<-STR
85* l1
86    v
87
88= H
89    STR
90
91    expected = [
92      @RM::List.new(:BULLET, *[
93        @RM::ListItem.new(nil,
94          @RM::Paragraph.new('l1'),
95          @RM::Verbatim.new("v\n"))]),
96      @RM::Heading.new(1, 'H')]
97
98    assert_equal expected, @RMP.parse(str).parts
99  end
100
101  def test_parse_bullet_heading
102    str = <<-STR
103* = l1
104    STR
105
106    expected = [
107      @RM::List.new(:BULLET, *[
108        @RM::ListItem.new(nil,
109          @RM::Heading.new(1, 'l1'))])]
110
111    assert_equal expected, @RMP.parse(str).parts
112  end
113
114  def test_parse_bullet_indent
115    str = <<-STR
116* l1
117  * l1.1
118* l2
119    STR
120
121    expected = [
122      @RM::List.new(:BULLET, *[
123        @RM::ListItem.new(nil,
124          @RM::Paragraph.new('l1'),
125          @RM::List.new(:BULLET, *[
126            @RM::ListItem.new(nil,
127              @RM::Paragraph.new('l1.1'))])),
128        @RM::ListItem.new(nil,
129          @RM::Paragraph.new('l2'))])]
130
131    assert_equal expected, @RMP.parse(str).parts
132  end
133
134  def test_parse_bullet_paragraph
135    str = <<-STR
136now is
137* l1
138* l2
139the time
140    STR
141
142    expected = [
143      @RM::Paragraph.new('now is'),
144      @RM::List.new(:BULLET, *[
145        @RM::ListItem.new(nil,
146          @RM::Paragraph.new('l1')),
147        @RM::ListItem.new(nil,
148          @RM::Paragraph.new('l2')),
149      ]),
150      @RM::Paragraph.new('the time'),
151    ]
152
153    assert_equal expected, @RMP.parse(str).parts
154  end
155
156  def test_parse_bullet_multiline
157    str = <<-STR
158* l1
159  l1+
160* l2
161    STR
162
163    expected = [
164      list(:BULLET,
165        item(nil,
166          para('l1 ', 'l1+')),
167        item(nil,
168          para('l2')))]
169
170    assert_equal expected, @RMP.parse(str).parts
171  end
172
173  def test_parse_bullet_multiparagraph
174    str = <<-STR
175* l1
176
177  l1+
178    STR
179
180    expected = [
181      @RM::List.new(:BULLET, *[
182        @RM::ListItem.new(nil,
183          @RM::Paragraph.new('l1'),
184          @RM::BlankLine.new,
185          @RM::Paragraph.new('l1+')),
186      ]),
187    ]
188
189    assert_equal expected, @RMP.parse(str).parts
190  end
191
192  def test_parse_bullet_indent_verbatim
193    str = <<-STR
194* l1
195  * l1.1
196    text
197      code
198        code
199
200    text
201* l2
202    STR
203
204    expected = [
205      list(:BULLET,
206        item(nil,
207          para('l1'),
208          list(:BULLET,
209            item(nil,
210              para('l1.1 ', 'text'),
211              verb("code\n", "  code\n"),
212              para('text')))),
213        item(nil,
214          para('l2')))]
215
216    assert_equal expected, @RMP.parse(str).parts
217  end
218
219  def test_parse_dash
220    str = <<-STR
221- one
222- two
223    STR
224
225    expected = [
226      @RM::List.new(:BULLET, *[
227        @RM::ListItem.new(nil,
228          @RM::Paragraph.new('one')),
229        @RM::ListItem.new(nil,
230          @RM::Paragraph.new('two'))])]
231
232    assert_equal expected, @RMP.parse(str).parts
233  end
234
235  def test_parse_heading
236    str = '= heading one'
237
238    expected = [
239      @RM::Heading.new(1, 'heading one')]
240
241    assert_equal expected, @RMP.parse(str).parts
242  end
243
244  def test_parse_heading_three
245    str = '=== heading three'
246
247    expected = [
248      @RM::Heading.new(3, 'heading three')]
249
250    assert_equal expected, @RMP.parse(str).parts
251  end
252
253  def test_parse_heading_bullet
254    str = '= * heading one'
255
256    expected = [
257      @RM::Heading.new(1, '* heading one')]
258
259    assert_equal expected, @RMP.parse(str).parts
260  end
261
262  def test_parse_heading_empty
263    str = <<-STR
264===
265* bullet
266    STR
267
268    expected = [
269      @RM::Heading.new(3, ''),
270      @RM::BlankLine.new,
271      @RM::List.new(:BULLET, *[
272        @RM::ListItem.new(nil,
273          @RM::Paragraph.new('bullet'))]),
274    ]
275
276    assert_equal expected, @RMP.parse(str).parts
277  end
278
279  def test_parse_heading_heading
280    str = '= ='
281
282    expected = [
283      @RM::Heading.new(1, '=')]
284
285    assert_equal expected, @RMP.parse(str).parts
286  end
287
288  def test_parse_heading_lalpha
289    str = '= b. heading one'
290
291    expected = [
292      @RM::Heading.new(1, 'b. heading one')]
293
294    assert_equal expected, @RMP.parse(str).parts
295  end
296
297  def test_parse_heading_label
298    str = '= [heading one]'
299
300    expected = [
301      @RM::Heading.new(1, '[heading one]')]
302
303    assert_equal expected, @RMP.parse(str).parts
304  end
305
306  def test_parse_heading_note
307    str = '= heading one::'
308
309    expected = [
310      @RM::Heading.new(1, 'heading one::')]
311
312    assert_equal expected, @RMP.parse(str).parts
313  end
314
315  def test_parse_heading_number
316    str = '= 5. heading one'
317
318    expected = [
319      @RM::Heading.new(1, '5. heading one')]
320
321    assert_equal expected, @RMP.parse(str).parts
322  end
323
324  def test_parse_heading_ualpha
325    str = '= B. heading one'
326
327    expected = [
328      @RM::Heading.new(1, 'B. heading one')]
329
330    assert_equal expected, @RMP.parse(str).parts
331  end
332
333  def test_parse_label
334    str = <<-STR
335[one] item one
336[two] item two
337    STR
338
339    expected = [
340      list(:LABEL,
341        item(%w[one],
342          para('item one')),
343        item(%w[two],
344          para('item two')))]
345
346    assert_equal expected, @RMP.parse(str).parts
347  end
348
349  def test_parse_label_bullet
350    str = <<-STR
351[cat] l1
352      * l1.1
353[dog] l2
354    STR
355
356    expected = [
357      list(:LABEL,
358        item(%w[cat],
359          para('l1'),
360          list(:BULLET,
361            item(nil,
362              para('l1.1')))),
363        item(%w[dog],
364          para('l2')))]
365
366    assert_equal expected, @RMP.parse(str).parts
367  end
368
369  def test_parse_label_multi_label
370    str = <<-STR
371[one]
372[two] some description
373    STR
374
375    expected = [
376      list(:LABEL,
377        item(%w[one two],
378          para('some description')))]
379
380    assert_equal expected, @RMP.parse(str).parts
381  end
382
383  def test_parse_label_multi_line
384    str = <<-STR
385[cat] l1
386      continuation
387[dog] l2
388    STR
389
390    expected = [
391      list(:LABEL,
392        item(%w[cat],
393          para('l1 ', 'continuation')),
394        item(%w[dog],
395          para('l2')))]
396
397    assert_equal expected, @RMP.parse(str).parts
398  end
399
400  def test_parse_label_newline
401    str = <<-STR
402[one]
403  item one
404[two]
405  item two
406    STR
407
408    expected = [
409      list(:LABEL,
410        item(%w[one],
411          para('item one')),
412        item(%w[two],
413          para('item two')))]
414
415    assert_equal expected, @RMP.parse(str).parts
416  end
417
418  def test_parse_lalpha
419    str = <<-STR
420a. l1
421b. l2
422    STR
423
424    expected = [
425      @RM::List.new(:LALPHA, *[
426        @RM::ListItem.new(nil,
427          @RM::Paragraph.new('l1')),
428        @RM::ListItem.new(nil,
429          @RM::Paragraph.new('l2'))])]
430
431    assert_equal expected, @RMP.parse(str).parts
432  end
433
434  def test_parse_lalpha_ualpha
435    str = <<-STR
436a. l1
437b. l2
438A. l3
439A. l4
440    STR
441
442    expected = [
443      @RM::List.new(:LALPHA, *[
444        @RM::ListItem.new(nil,
445          @RM::Paragraph.new('l1')),
446        @RM::ListItem.new(nil,
447          @RM::Paragraph.new('l2'))]),
448      @RM::List.new(:UALPHA, *[
449        @RM::ListItem.new(nil,
450          @RM::Paragraph.new('l3')),
451        @RM::ListItem.new(nil,
452          @RM::Paragraph.new('l4'))])]
453
454    assert_equal expected, @RMP.parse(str).parts
455  end
456
457  def test_parse_lalpha_utf_8
458    str = <<-STR
459a. ���������������
460    STR
461
462    expected = [
463      @RM::List.new(:LALPHA, *[
464        @RM::ListItem.new(nil,
465          @RM::Paragraph.new('���������������'))])]
466
467    assert_equal expected, @RMP.parse(str).parts
468  end
469
470  def test_parse_line_break
471    str = "now is\nthe time  \nfor all"
472
473    expected = [
474      para('now is ', 'the time'),
475      blank_line,
476      para('for all')]
477
478    assert_equal expected, @RMP.parse(str).parts
479  end
480
481  def test_parse_list_list_1
482    str = <<-STR
48310. para 1
484
485    [label 1]
486      para 1.1
487
488        code
489
490      para 1.2
491    STR
492
493    expected = [
494      list(:NUMBER,
495        item(nil,
496          para('para 1'),
497          blank_line,
498          list(:LABEL,
499            item(%w[label\ 1],
500              para('para 1.1'),
501              blank_line,
502              verb("code\n"),
503              para('para 1.2')))))]
504
505    assert_equal expected, @RMP.parse(str).parts
506  end
507
508  def test_parse_list_list_2
509    str = <<-STR
5106. para
511
512   label 1::  text 1
513   label 2::  text 2
514    STR
515
516    expected = [
517      list(:NUMBER,
518        item(nil,
519          para('para'),
520          blank_line,
521          list(:NOTE,
522            item(%w[label\ 1],
523              para('text 1')),
524            item(%w[label\ 2],
525              para('text 2')))))]
526
527    assert_equal expected, @RMP.parse(str).parts
528  end
529
530  def test_parse_list_verbatim
531    str = <<-STR
532* one
533    verb1
534    verb2
535* two
536    STR
537
538    expected = [
539      @RM::List.new(:BULLET, *[
540        @RM::ListItem.new(nil,
541          @RM::Paragraph.new('one'),
542          @RM::Verbatim.new("verb1\n", "verb2\n")),
543        @RM::ListItem.new(nil,
544          @RM::Paragraph.new('two'))])]
545
546    assert_equal expected, @RMP.parse(str).parts
547  end
548
549  def test_parse_lists
550    str = <<-STR
551now is
552* l1
5531. n1
5542. n2
555* l2
556the time
557    STR
558
559    expected = [
560      @RM::Paragraph.new('now is'),
561      @RM::List.new(:BULLET, *[
562        @RM::ListItem.new(nil,
563          @RM::Paragraph.new('l1'))]),
564      @RM::List.new(:NUMBER, *[
565        @RM::ListItem.new(nil,
566          @RM::Paragraph.new('n1')),
567        @RM::ListItem.new(nil,
568          @RM::Paragraph.new('n2'))]),
569      @RM::List.new(:BULLET, *[
570        @RM::ListItem.new(nil,
571          @RM::Paragraph.new('l2'))]),
572      @RM::Paragraph.new('the time')]
573
574    assert_equal expected, @RMP.parse(str).parts
575  end
576
577  def test_parse_note
578    str = <<-STR
579one:: item one
580two:: item two
581    STR
582
583    expected = [
584      list(:NOTE,
585        item(%w[one],
586          para('item one')),
587        item(%w[two],
588          para('item two')))]
589
590    assert_equal expected, @RMP.parse(str).parts
591  end
592
593  def test_parse_note_empty
594    str = <<-STR
595one::
596two::
597    STR
598
599    expected = [
600      list(:NOTE,
601        item(%w[one two],
602          blank_line))]
603
604    assert_equal expected, @RMP.parse(str).parts
605  end
606
607  def test_parse_note_note
608    str = <<-STR
609one:: two::
610    STR
611
612    expected = [
613      list(:NOTE,
614        item(%w[one],
615          list(:NOTE,
616            item(%w[two],
617              blank_line))))]
618
619    assert_equal expected, @RMP.parse(str).parts
620  end
621
622  def test_parse_number_bullet
623    str = <<-STR
6241. l1
625   * l1.1
6262. l2
627    STR
628
629    expected = [
630      list(:NUMBER,
631        item(nil,
632          para('l1'),
633          list(:BULLET,
634            item(nil,
635              para('l1.1')))),
636        item(nil,
637          para('l2')))]
638
639    assert_equal expected, @RMP.parse(str).parts
640  end
641
642  def test_parse_paragraph
643    str = <<-STR
644now is the time
645
646for all good men
647    STR
648
649    expected = [
650      @RM::Paragraph.new('now is the time'),
651      @RM::BlankLine.new,
652      @RM::Paragraph.new('for all good men')]
653    assert_equal expected, @RMP.parse(str).parts
654  end
655
656  def test_parse_paragraph_multiline
657    str = "now is the time\nfor all good men"
658
659    expected = @RM::Paragraph.new 'now is the time ', 'for all good men'
660    assert_equal [expected], @RMP.parse(str).parts
661  end
662
663  def test_parse_paragraph_verbatim
664    str = <<-STR
665now is the time
666  code _line_ here
667for all good men
668    STR
669
670    expected = [
671      @RM::Paragraph.new('now is the time'),
672      @RM::Verbatim.new("code _line_ here\n"),
673      @RM::Paragraph.new('for all good men'),
674    ]
675    assert_equal expected, @RMP.parse(str).parts
676  end
677
678  def test_parse_rule
679    str = <<-STR
680now is the time
681
682---
683
684for all good men
685    STR
686
687    expected = [
688      @RM::Paragraph.new('now is the time'),
689      @RM::BlankLine.new,
690      @RM::Rule.new(1),
691      @RM::BlankLine.new,
692      @RM::Paragraph.new('for all good men')]
693
694    assert_equal expected, @RMP.parse(str).parts
695  end
696
697  def test_parse_ualpha
698    str = <<-STR
699A. l1
700B. l2
701    STR
702
703    expected = [
704      @RM::List.new(:UALPHA, *[
705        @RM::ListItem.new(nil,
706          @RM::Paragraph.new('l1')),
707        @RM::ListItem.new(nil,
708          @RM::Paragraph.new('l2'))])]
709
710    assert_equal expected, @RMP.parse(str).parts
711  end
712
713  def test_parse_trailing_cr
714    expected = [ @RM::Paragraph.new('Text') ]
715    # FIXME hangs the parser:
716    assert_equal expected, @RMP.parse("Text\r").parts
717  end
718
719  def test_parse_verbatim
720    str = <<-STR
721now is
722   code
723the time
724    STR
725
726    expected = [
727      @RM::Paragraph.new('now is'),
728      @RM::Verbatim.new("code\n"),
729      @RM::Paragraph.new('the time'),
730    ]
731
732    assert_equal expected, @RMP.parse(str).parts
733  end
734
735  def test_parse_verbatim_bullet
736    str = <<-STR
737  * blah
738    STR
739
740    expected = [
741      @RM::Verbatim.new("* blah\n")]
742
743    assert_equal expected, @RMP.parse(str).parts
744  end
745
746  def test_parse_verbatim_dash
747    str = <<-STR
748  - blah
749    STR
750
751    expected = [
752      @RM::Verbatim.new("- blah\n")]
753
754    assert_equal expected, @RMP.parse(str).parts
755  end
756
757  def test_parse_verbatim_fold
758    str = <<-STR
759now is
760   code
761
762
763   code1
764
765the time
766    STR
767
768    expected = [
769      @RM::Paragraph.new('now is'),
770      @RM::Verbatim.new("code\n", "\n", "code1\n"),
771      @RM::Paragraph.new('the time'),
772    ]
773
774    assert_equal expected, @RMP.parse(str).parts
775  end
776
777  def test_parse_verbatim_heading
778    str = <<-STR
779text
780   ===   heading three
781    STR
782
783    expected = [
784      @RM::Paragraph.new('text'),
785      @RM::Verbatim.new("===   heading three\n")]
786
787    assert_equal expected, @RMP.parse(str).parts
788  end
789
790  def test_parse_verbatim_heading2
791    str = "text\n   code\n=== heading three"
792
793    expected = [
794      @RM::Paragraph.new('text'),
795      @RM::Verbatim.new("code\n"),
796      @RM::Heading.new(3, 'heading three')]
797
798    assert_equal expected, @RMP.parse(str).parts
799  end
800
801  def test_parse_verbatim_label
802    str = <<-STR
803  [blah] blah
804    STR
805
806    expected = [
807      @RM::Verbatim.new("[blah] blah\n")]
808
809    assert_equal expected, @RMP.parse(str).parts
810  end
811
812  def test_parse_verbatim_lalpha
813    str = <<-STR
814  b. blah
815    STR
816
817    expected = [
818      @RM::Verbatim.new("b. blah\n")]
819
820    assert_equal expected, @RMP.parse(str).parts
821  end
822
823  def test_parse_verbatim_markup_example
824    str = <<-STR
825text
826   code
827   === heading three
828    STR
829
830    expected = [
831      @RM::Paragraph.new('text'),
832      @RM::Verbatim.new("code\n", "=== heading three\n")]
833
834    assert_equal expected, @RMP.parse(str).parts
835  end
836
837  def test_parse_verbatim_merge
838    str = <<-STR
839now is
840   code
841
842   code1
843the time
844    STR
845
846    expected = [
847      @RM::Paragraph.new('now is'),
848      @RM::Verbatim.new("code\n", "\n", "code1\n"),
849      @RM::Paragraph.new('the time'),
850    ]
851
852    assert_equal expected, @RMP.parse(str).parts
853  end
854
855  def test_parse_verbatim_merge2
856    str = <<-STR
857now is
858   code
859
860   code1
861
862   code2
863the time
864    STR
865
866    expected = [
867      @RM::Paragraph.new('now is'),
868      @RM::Verbatim.new("code\n", "\n", "code1\n", "\n", "code2\n"),
869      @RM::Paragraph.new('the time'),
870    ]
871
872    assert_equal expected, @RMP.parse(str).parts
873  end
874
875  def test_parse_verbatim_multiline
876    str = <<-STR
877now is
878   code
879   code1
880the time
881    STR
882
883    expected = [
884      @RM::Paragraph.new('now is'),
885      @RM::Verbatim.new("code\n", "code1\n"),
886      @RM::Paragraph.new('the time'),
887    ]
888
889    assert_equal expected, @RMP.parse(str).parts
890  end
891
892  def test_parse_verbatim_multilevel
893    str = <<-STR
894now is the time
895  code
896 more code
897for all good men
898    STR
899
900    expected = [
901      @RM::Paragraph.new('now is the time'),
902      @RM::Verbatim.new(" code\n",
903                        "more code\n"),
904      @RM::Paragraph.new('for all good men'),
905    ]
906
907    assert_equal expected, @RMP.parse(str).parts
908  end
909
910  def test_parse_verbatim_note
911    str = <<-STR
912  blah:: blah
913    STR
914
915    expected = [
916      @RM::Verbatim.new("blah:: blah\n")]
917
918    assert_equal expected, @RMP.parse(str).parts
919  end
920
921  def test_parse_verbatim_number
922    str = <<-STR
923  2. blah
924    STR
925
926    expected = [
927      @RM::Verbatim.new("2. blah\n")]
928
929    assert_equal expected, @RMP.parse(str).parts
930  end
931
932  def test_parse_verbatim_rule
933    str = <<-STR
934text
935
936  --- lib/blah.rb.orig
937  +++ lib/blah.rb
938    STR
939
940    expected = [
941      @RM::Paragraph.new('text'),
942      @RM::BlankLine.new,
943      @RM::Verbatim.new("--- lib/blah.rb.orig\n",
944                        "+++ lib/blah.rb\n")]
945
946    assert_equal expected, @RMP.parse(str).parts
947  end
948
949  def test_parse_verbatim_rule2
950    str = <<-STR.chomp
951text
952
953  ---
954    STR
955
956    expected = [
957      @RM::Paragraph.new('text'),
958      @RM::BlankLine.new,
959      @RM::Verbatim.new("---")]
960
961    assert_equal expected, @RMP.parse(str).parts
962  end
963
964  def test_parse_verbatim_trim
965    str = <<-STR
966now is
967   code
968
969   code1
970
971the time
972    STR
973
974    expected = [
975      @RM::Paragraph.new('now is'),
976      @RM::Verbatim.new("code\n",
977                        "\n",
978                        "code1\n"),
979      @RM::Paragraph.new('the time'),
980    ]
981
982    assert_equal expected, @RMP.parse(str).parts
983  end
984
985  def test_parse_verbatim_ualpha
986    str = <<-STR
987  B. blah
988    STR
989
990    expected = [
991      @RM::Verbatim.new("B. blah\n")]
992
993    assert_equal expected, @RMP.parse(str).parts
994  end
995
996  def test_parse_whitespace
997    expected = [
998      @RM::Paragraph.new('hello'),
999    ]
1000
1001    assert_equal expected, @RMP.parse('hello').parts
1002
1003    expected = [
1004      @RM::Verbatim.new('hello '),
1005    ]
1006
1007    assert_equal expected, @RMP.parse('  hello ').parts
1008
1009    expected = [
1010      @RM::Verbatim.new('hello          '),
1011    ]
1012
1013    assert_equal expected, @RMP.parse('                 hello          ').parts
1014
1015    expected = [
1016      @RM::Paragraph.new('1'),
1017      @RM::Verbatim.new("2\n", ' 3'),
1018    ]
1019
1020    assert_equal expected, @RMP.parse("1\n 2\n  3").parts
1021
1022    expected = [
1023      @RM::Verbatim.new("1\n",
1024                        " 2\n",
1025                        "  3"),
1026    ]
1027
1028    assert_equal expected, @RMP.parse("  1\n   2\n    3").parts
1029
1030    expected = [
1031      @RM::Paragraph.new('1'),
1032      @RM::Verbatim.new("2\n",
1033                        " 3\n"),
1034      @RM::Paragraph.new('1'),
1035      @RM::Verbatim.new('2'),
1036    ]
1037
1038    assert_equal expected, @RMP.parse("1\n 2\n  3\n1\n 2").parts
1039
1040    expected = [
1041      @RM::Verbatim.new("1\n",
1042                        " 2\n",
1043                        "  3\n",
1044                        "1\n",
1045                        ' 2'),
1046    ]
1047
1048    assert_equal expected, @RMP.parse("  1\n   2\n    3\n  1\n   2").parts
1049
1050    expected = [
1051      @RM::Verbatim.new("1\n",
1052                        " 2\n",
1053                        "\n",
1054                        '  3'),
1055    ]
1056
1057    assert_equal expected, @RMP.parse("  1\n   2\n\n    3").parts
1058  end
1059
1060  def test_peek_token
1061    parser = util_parser
1062
1063    assert_equal [:HEADER, 1, 0, 0], parser.peek_token
1064
1065    assert_equal 8, parser.tokens.length
1066  end
1067
1068  def test_skip
1069    parser = util_parser
1070
1071    assert_equal [:HEADER, 1, 0, 0], parser.skip(:HEADER)
1072
1073    assert_equal [:TEXT, 'Heading', 2, 0], parser.get
1074
1075    assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token
1076
1077    assert_raises RDoc::Markup::Parser::ParseError do
1078      parser.skip :NONE
1079    end
1080
1081    assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token
1082
1083    assert_equal nil, parser.skip(:NONE, false)
1084
1085    assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token
1086  end
1087
1088  def test_tokenize_bullet
1089    str = <<-STR
1090* l1
1091    STR
1092
1093    expected = [
1094      [:BULLET,  '*',     0, 0],
1095      [:TEXT,    'l1',    2, 0],
1096      [:NEWLINE, "\n",    4, 0],
1097    ]
1098
1099    assert_equal expected, @RMP.tokenize(str)
1100  end
1101
1102  def test_tokenize_bullet_indent
1103    str = <<-STR
1104* l1
1105  * l1.1
1106    STR
1107
1108    expected = [
1109      [:BULLET,  '*',     0, 0],
1110      [:TEXT,    'l1',    2, 0],
1111      [:NEWLINE, "\n",    4, 0],
1112      [:BULLET,  '*',     2, 1],
1113      [:TEXT,    'l1.1',  4, 1],
1114      [:NEWLINE, "\n",    8, 1],
1115    ]
1116
1117    assert_equal expected, @RMP.tokenize(str)
1118  end
1119
1120  def test_tokenize_heading
1121    str = <<-STR
1122= Heading
1123== Heading 2
1124    STR
1125
1126    expected = [
1127      [:HEADER,  1,            0, 0],
1128      [:TEXT,    'Heading',    2, 0],
1129      [:NEWLINE, "\n",         9, 0],
1130      [:HEADER,  2,            0, 1],
1131      [:TEXT,    'Heading 2',  3, 1],
1132      [:NEWLINE, "\n",        12, 1],
1133    ]
1134
1135    assert_equal expected, @RMP.tokenize(str)
1136  end
1137
1138  def test_tokenize_heading_empty
1139    str = <<-STR
1140===
1141* bullet
1142    STR
1143
1144    expected = [
1145      [:HEADER,  3,        0, 0],
1146      [:NEWLINE, "\n",     3, 0],
1147      [:BULLET,  "*",      0, 1],
1148      [:TEXT,    "bullet", 2, 1],
1149      [:NEWLINE, "\n",     8, 1],
1150    ]
1151
1152    assert_equal expected, @RMP.tokenize(str)
1153  end
1154
1155  def test_tokenize_heading_heading
1156    str = <<-STR
1157= =
1158    STR
1159
1160    expected = [
1161      [:HEADER,  1,    0, 0],
1162      [:TEXT,    '=',  2, 0],
1163      [:NEWLINE, "\n", 3, 0],
1164    ]
1165
1166    assert_equal expected, @RMP.tokenize(str)
1167  end
1168
1169  def test_tokenize_heading_no_space
1170    str = <<-STR
1171=Heading
1172==Heading 2
1173    STR
1174
1175    expected = [
1176      [:HEADER,  1,            0, 0],
1177      [:TEXT,    'Heading',    1, 0],
1178      [:NEWLINE, "\n",         8, 0],
1179      [:HEADER,  2,            0, 1],
1180      [:TEXT,    'Heading 2',  2, 1],
1181      [:NEWLINE, "\n",        11, 1],
1182    ]
1183
1184    assert_equal expected, @RMP.tokenize(str)
1185  end
1186
1187  def test_tokenize_label
1188    str = <<-STR
1189[cat] l1
1190[dog] l1.1
1191    STR
1192
1193    expected = [
1194      [:LABEL,   'cat',   0, 0],
1195      [:TEXT,    'l1',    6, 0],
1196      [:NEWLINE, "\n",    8, 0],
1197      [:LABEL,   'dog',   0, 1],
1198      [:TEXT,    'l1.1',  6, 1],
1199      [:NEWLINE, "\n",   10, 1],
1200    ]
1201
1202    assert_equal expected, @RMP.tokenize(str)
1203  end
1204
1205  def test_tokenize_label_note
1206    str = <<-STR
1207[label]
1208  note::
1209    STR
1210
1211    expected = [
1212      [:LABEL,   'label', 0, 0],
1213      [:NEWLINE, "\n",    7, 0],
1214      [:NOTE,    'note',  2, 1],
1215      [:NEWLINE, "\n",    8, 1],
1216    ]
1217
1218    assert_equal expected, @RMP.tokenize(str)
1219  end
1220
1221  def test_tokenize_label_newline
1222    str = <<-STR
1223[cat]
1224  l1
1225    STR
1226
1227    expected = [
1228      [:LABEL,   'cat',   0, 0],
1229      [:NEWLINE, "\n",    5, 0],
1230      [:TEXT,    'l1',    2, 1],
1231      [:NEWLINE, "\n",    4, 1],
1232    ]
1233
1234    assert_equal expected, @RMP.tokenize(str)
1235  end
1236
1237  def test_tokenize_label_newline_windows
1238    str = <<-STR
1239[cat]\r
1240  l1\r
1241    STR
1242
1243    expected = [
1244      [:LABEL,   'cat',   0, 0],
1245      [:NEWLINE, "\n",    6, 0],
1246      [:TEXT,    'l1',    2, 1],
1247      [:NEWLINE, "\n",    5, 1],
1248    ]
1249
1250    assert_equal expected, @RMP.tokenize(str)
1251  end
1252
1253  def test_tokenize_lalpha
1254    str = <<-STR
1255a. l1
1256b. l1.1
1257    STR
1258
1259    expected = [
1260      [:LALPHA,  'a',     0, 0],
1261      [:TEXT,    'l1',    3, 0],
1262      [:NEWLINE, "\n",    5, 0],
1263      [:LALPHA,  'b',     0, 1],
1264      [:TEXT,    'l1.1',  3, 1],
1265      [:NEWLINE, "\n",    7, 1],
1266    ]
1267
1268    assert_equal expected, @RMP.tokenize(str)
1269  end
1270
1271  def test_tokenize_line_break
1272    str = "now is\nthe time  \nfor all\n"
1273
1274    expected = [
1275      [:TEXT,    'now is',    0, 0],
1276      [:NEWLINE, "\n",        6, 0],
1277      [:TEXT,    'the time',  0, 1],
1278      [:BREAK,   "  ",        8, 1],
1279      [:NEWLINE, "\n",       10, 1],
1280      [:TEXT,    'for all',   0, 2],
1281      [:NEWLINE, "\n",        7, 2],
1282    ]
1283
1284    assert_equal expected, @RMP.tokenize(str)
1285  end
1286
1287  def test_tokenize_line_break_long
1288    str = "now is\nthe time   \nfor all\n"
1289
1290    expected = [
1291      [:TEXT,    'now is',     0, 0],
1292      [:NEWLINE, "\n",         6, 0],
1293      [:TEXT,    'the time ',  0, 1],
1294      [:BREAK,   '  ',         9, 1],
1295      [:NEWLINE, "\n",        11, 1],
1296      [:TEXT,    'for all',    0, 2],
1297      [:NEWLINE, "\n",         7, 2],
1298    ]
1299
1300    assert_equal expected, @RMP.tokenize(str)
1301  end
1302
1303  def test_tokenize_line_break_no_short
1304    str = "now is\nthe time \nfor all\n"
1305
1306    expected = [
1307      [:TEXT,    'now is',    0, 0],
1308      [:NEWLINE, "\n",        6, 0],
1309      [:TEXT,    'the time ', 0, 1],
1310      [:NEWLINE, "\n",        9, 1],
1311      [:TEXT,    'for all',   0, 2],
1312      [:NEWLINE, "\n",        7, 2],
1313    ]
1314
1315    assert_equal expected, @RMP.tokenize(str)
1316  end
1317
1318  def test_tokenize_note
1319    str = <<-STR
1320cat:: l1
1321dog:: l1.1
1322    STR
1323
1324    expected = [
1325      [:NOTE,    'cat',   0, 0],
1326      [:TEXT,    'l1',    6, 0],
1327      [:NEWLINE, "\n",    8, 0],
1328      [:NOTE,    'dog',   0, 1],
1329      [:TEXT,    'l1.1',  6, 1],
1330      [:NEWLINE, "\n",   10, 1],
1331    ]
1332
1333    assert_equal expected, @RMP.tokenize(str)
1334  end
1335
1336  def test_tokenize_note_empty
1337    str = <<-STR
1338cat::
1339dog::
1340    STR
1341
1342    expected = [
1343      [:NOTE,    'cat', 0, 0],
1344      [:NEWLINE, "\n",  5, 0],
1345      [:NOTE,    'dog', 0, 1],
1346      [:NEWLINE, "\n",  5, 1],
1347    ]
1348
1349    assert_equal expected, @RMP.tokenize(str)
1350  end
1351
1352  def test_tokenize_note_newline
1353    str = <<-STR
1354cat::
1355  l1
1356    STR
1357
1358    expected = [
1359      [:NOTE,    'cat',   0, 0],
1360      [:NEWLINE, "\n",    5, 0],
1361      [:TEXT,    'l1',    2, 1],
1362      [:NEWLINE, "\n",    4, 1],
1363    ]
1364
1365    assert_equal expected, @RMP.tokenize(str)
1366  end
1367
1368  def test_tokenize_note_utf_8
1369    skip 'Encoding not implemented' unless @have_encoding
1370
1371    str = <<-STR
1372c��t:: l1a
1373      l1b
1374d��g:: l2a
1375      l2b
1376    STR
1377
1378    expected = [
1379      [:NOTE,    'c��t',   0, 0],
1380      [:TEXT,    'l1a',   6, 0],
1381      [:NEWLINE, "\n",    9, 0],
1382      [:TEXT,    'l1b',   6, 1],
1383      [:NEWLINE, "\n",    9, 1],
1384      [:NOTE,    'd��g',   0, 2],
1385      [:TEXT,    'l2a',   6, 2],
1386      [:NEWLINE, "\n",    9, 2],
1387      [:TEXT,    'l2b',   6, 3],
1388      [:NEWLINE, "\n",    9, 3],
1389    ]
1390
1391    assert_equal expected, @RMP.tokenize(str)
1392  end
1393
1394  def test_tokenize_note_newline_windows
1395    str = <<-STR
1396cat::\r
1397  l1\r
1398    STR
1399
1400    expected = [
1401      [:NOTE,    'cat',   0, 0],
1402      [:NEWLINE, "\n",    6, 0],
1403      [:TEXT,    'l1',    2, 1],
1404      [:NEWLINE, "\n",    5, 1],
1405    ]
1406
1407    assert_equal expected, @RMP.tokenize(str)
1408  end
1409
1410  def test_tokenize_note_not
1411    str = <<-STR
1412Cat::Dog
1413    STR
1414
1415    expected = [
1416      [:TEXT,    'Cat::Dog', 0, 0],
1417      [:NEWLINE, "\n",       8, 0],
1418    ]
1419
1420    assert_equal expected, @RMP.tokenize(str)
1421  end
1422
1423  def test_tokenize_number
1424    str = <<-STR
14251. l1
14262. l1.1
1427    STR
1428
1429    expected = [
1430      [:NUMBER,  '1',     0, 0],
1431      [:TEXT,    'l1',    3, 0],
1432      [:NEWLINE, "\n",    5, 0],
1433      [:NUMBER,  '2',     0, 1],
1434      [:TEXT,    'l1.1',  3, 1],
1435      [:NEWLINE, "\n",    7, 1],
1436    ]
1437
1438    assert_equal expected, @RMP.tokenize(str)
1439  end
1440
1441  def test_tokenize_number_period
1442    str = <<-STR
14431. blah blah blah
1444   l.
14452. blah blah blah blah
1446   d.
1447    STR
1448
1449    expected = [
1450      [:NUMBER,  "1",                    0, 0],
1451      [:TEXT,    "blah blah blah",       3, 0],
1452      [:NEWLINE, "\n",                  17, 0],
1453
1454      [:TEXT,    "l.",                   3, 1],
1455      [:NEWLINE, "\n",                   5, 1],
1456
1457      [:NUMBER,  "2",                    0, 2],
1458      [:TEXT,    "blah blah blah blah",  3, 2],
1459      [:NEWLINE, "\n",                  22, 2],
1460
1461      [:TEXT,    "d.",                   3, 3],
1462      [:NEWLINE, "\n",                   5, 3]
1463    ]
1464
1465    assert_equal expected, @RMP.tokenize(str)
1466  end
1467
1468  def test_tokenize_number_period_continue
1469    str = <<-STR
14701. blah blah blah
1471   l.  more stuff
14722. blah blah blah blah
1473   d. other stuff
1474    STR
1475
1476    expected = [
1477      [:NUMBER,  "1",                    0, 0],
1478      [:TEXT,    "blah blah blah",       3, 0],
1479      [:NEWLINE, "\n",                  17, 0],
1480
1481      [:LALPHA,  "l",                    3, 1],
1482      [:TEXT,    "more stuff",           7, 1],
1483      [:NEWLINE, "\n",                  17, 1],
1484
1485      [:NUMBER,  "2",                    0, 2],
1486      [:TEXT,    "blah blah blah blah",  3, 2],
1487      [:NEWLINE, "\n",                  22, 2],
1488
1489      [:LALPHA,  "d",                    3, 3],
1490      [:TEXT,    "other stuff",          6, 3],
1491      [:NEWLINE, "\n",                  17, 3]
1492    ]
1493
1494    assert_equal expected, @RMP.tokenize(str)
1495  end
1496
1497  def test_tokenize_paragraphs
1498    str = <<-STR
1499now is
1500the time
1501
1502for all
1503    STR
1504
1505    expected = [
1506      [:TEXT,    'now is',   0, 0],
1507      [:NEWLINE, "\n",       6, 0],
1508      [:TEXT,    'the time', 0, 1],
1509      [:NEWLINE, "\n",       8, 1],
1510      [:NEWLINE, "\n",       0, 2],
1511      [:TEXT,    'for all',  0, 3],
1512      [:NEWLINE, "\n",       7, 3],
1513    ]
1514
1515    assert_equal expected, @RMP.tokenize(str)
1516  end
1517
1518  def test_tokenize_rule
1519    str = <<-STR
1520---
1521
1522--- blah ---
1523    STR
1524
1525    expected = [
1526      [:RULE,    1,            0, 0],
1527      [:NEWLINE, "\n",         3, 0],
1528      [:NEWLINE, "\n",         0, 1],
1529      [:TEXT, "--- blah ---",  0, 2],
1530      [:NEWLINE, "\n",        12, 2],
1531    ]
1532
1533    assert_equal expected, @RMP.tokenize(str)
1534  end
1535
1536  def test_tokenize_rule_windows
1537    str = <<-STR
1538---\r
1539
1540--- blah ---\r
1541    STR
1542
1543    expected = [
1544      [:RULE,    1,            0, 0],
1545      [:NEWLINE, "\n",         4, 0],
1546      [:NEWLINE, "\n",         0, 1],
1547      [:TEXT, "--- blah ---",  0, 2],
1548      [:NEWLINE, "\n",        13, 2],
1549    ]
1550
1551    assert_equal expected, @RMP.tokenize(str)
1552  end
1553
1554  def test_tokenize_ualpha
1555    str = <<-STR
1556A. l1
1557B. l1.1
1558    STR
1559
1560    expected = [
1561      [:UALPHA,  'A',     0, 0],
1562      [:TEXT,    'l1',    3, 0],
1563      [:NEWLINE, "\n",    5, 0],
1564      [:UALPHA,  'B',     0, 1],
1565      [:TEXT,    'l1.1',  3, 1],
1566      [:NEWLINE, "\n",    7, 1],
1567    ]
1568
1569    assert_equal expected, @RMP.tokenize(str)
1570  end
1571
1572  def test_tokenize_verbatim_heading
1573    str = <<-STR
1574Example heading:
1575
1576   === heading three
1577    STR
1578
1579    expected = [
1580      [:TEXT,    'Example heading:',  0, 0],
1581      [:NEWLINE, "\n",               16, 0],
1582      [:NEWLINE, "\n",                0, 1],
1583      [:HEADER,  3,                   3, 2],
1584      [:TEXT,    'heading three',     7, 2],
1585      [:NEWLINE, "\n",               20, 2],
1586    ]
1587
1588    assert_equal expected, @RMP.tokenize(str)
1589  end
1590
1591  def test_tokenize_verbatim_rule
1592    str = <<-STR
1593  Verbatim section here that is double-underlined
1594  ===============================================
1595    STR
1596
1597    expected = [
1598      [:TEXT,    'Verbatim section here that is double-underlined',  2, 0],
1599      [:NEWLINE, "\n",                                              49, 0],
1600      [:HEADER,  47,                                                 2, 1],
1601      [:NEWLINE, "\n",                                              49, 1],
1602    ]
1603
1604    assert_equal expected, @RMP.tokenize(str)
1605  end
1606
1607  def test_tokenize_verbatim_rule_fancy
1608    str = <<-STR
1609  A
1610    b
1611  ===============================================
1612    c
1613    STR
1614
1615    expected = [
1616      [:TEXT,    'A',   2, 0],
1617      [:NEWLINE, "\n",  3, 0],
1618      [:TEXT,    'b',   4, 1],
1619      [:NEWLINE, "\n",  5, 1],
1620      [:HEADER,  47,    2, 2],
1621      [:NEWLINE, "\n", 49, 2],
1622      [:TEXT,    'c',   4, 3],
1623      [:NEWLINE, "\n",  5, 3],
1624    ]
1625
1626    assert_equal expected, @RMP.tokenize(str)
1627  end
1628
1629  def test_token_pos
1630    parser = @RMP.new
1631    s = parser.setup_scanner 'c��t'
1632
1633    s.scan(/\S+/)
1634
1635    if @have_encoding or @have_byteslice then
1636      assert_equal [3, 0], parser.token_pos(s.pos)
1637    else
1638      assert_equal [4, 0], parser.token_pos(s.pos)
1639    end
1640  end
1641
1642  # HACK move to Verbatim test case
1643  def test_verbatim_normalize
1644    v = @RM::Verbatim.new "foo\n", "\n", "\n", "bar\n"
1645
1646    v.normalize
1647
1648    assert_equal ["foo\n", "\n", "bar\n"], v.parts
1649
1650    v = @RM::Verbatim.new "foo\n", "\n"
1651
1652    v.normalize
1653
1654    assert_equal ["foo\n"], v.parts
1655  end
1656
1657  def test_unget
1658    parser = util_parser
1659
1660    parser.get
1661
1662    parser.unget
1663
1664    assert_equal [:HEADER, 1, 0, 0], parser.peek_token
1665
1666    assert_raises @RMP::Error do
1667      parser.unget
1668    end
1669
1670    assert_equal 8, parser.tokens.length
1671  end
1672
1673  def util_parser
1674    str = <<-STR
1675= Heading
1676
1677Some text here
1678some more text over here
1679    STR
1680
1681    @parser = @RMP.new
1682    @parser.tokenize str
1683    @parser
1684  end
1685
1686end
1687
1688