CommentLexer.cpp revision 263508 
1#include "clang/AST/CommentLexer.h"
2#include "clang/AST/CommentCommandTraits.h"
3#include "clang/AST/CommentDiagnostic.h"
4#include "clang/Basic/CharInfo.h"
5#include "llvm/ADT/StringExtras.h"
6#include "llvm/ADT/StringSwitch.h"
7#include "llvm/Support/ConvertUTF.h"
8#include "llvm/Support/ErrorHandling.h"
9
10namespace clang {
11namespace comments {
12
13void Token::dump(const Lexer &L, const SourceManager &SM) const {
14  llvm::errs() << "comments::Token Kind=" << Kind << " ";
15  Loc.dump(SM);
16  llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
17}
18
19static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
20  return isLetter(C);
21}
22
23static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
24  return isDigit(C);
25}
26
27static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
28  return isHexDigit(C);
29}
30
31static inline StringRef convertCodePointToUTF8(
32                                      llvm::BumpPtrAllocator &Allocator,
33                                      unsigned CodePoint) {
34  char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
35  char *ResolvedPtr = Resolved;
36  if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
37    return StringRef(Resolved, ResolvedPtr - Resolved);
38  else
39    return StringRef();
40}
41
42namespace {
43
44#include "clang/AST/CommentHTMLTags.inc"
45#include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
46
47} // unnamed namespace
48
49StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
50  // Fast path, first check a few most widely used named character references.
51  return llvm::StringSwitch<StringRef>(Name)
52      .Case("amp", "&")
53      .Case("lt", "<")
54      .Case("gt", ">")
55      .Case("quot", "\"")
56      .Case("apos", "\'")
57      // Slow path.
58      .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
59}
60
61StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
62  unsigned CodePoint = 0;
63  for (unsigned i = 0, e = Name.size(); i != e; ++i) {
64    assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
65    CodePoint *= 10;
66    CodePoint += Name[i] - '0';
67  }
68  return convertCodePointToUTF8(Allocator, CodePoint);
69}
70
71StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
72  unsigned CodePoint = 0;
73  for (unsigned i = 0, e = Name.size(); i != e; ++i) {
74    CodePoint *= 16;
75    const char C = Name[i];
76    assert(isHTMLHexCharacterReferenceCharacter(C));
77    CodePoint += llvm::hexDigitValue(C);
78  }
79  return convertCodePointToUTF8(Allocator, CodePoint);
80}
81
82void Lexer::skipLineStartingDecorations() {
83  // This function should be called only for C comments
84  assert(CommentState == LCS_InsideCComment);
85
86  if (BufferPtr == CommentEnd)
87    return;
88
89  switch (*BufferPtr) {
90  case ' ':
91  case '\t':
92  case '\f':
93  case '\v': {
94    const char *NewBufferPtr = BufferPtr;
95    NewBufferPtr++;
96    if (NewBufferPtr == CommentEnd)
97      return;
98
99    char C = *NewBufferPtr;
100    while (isHorizontalWhitespace(C)) {
101      NewBufferPtr++;
102      if (NewBufferPtr == CommentEnd)
103        return;
104      C = *NewBufferPtr;
105    }
106    if (C == '*')
107      BufferPtr = NewBufferPtr + 1;
108    break;
109  }
110  case '*':
111    BufferPtr++;
112    break;
113  }
114}
115
116namespace {
117/// Returns pointer to the first newline character in the string.
118const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
119  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
120    if (isVerticalWhitespace(*BufferPtr))
121      return BufferPtr;
122  }
123  return BufferEnd;
124}
125
126const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
127  if (BufferPtr == BufferEnd)
128    return BufferPtr;
129
130  if (*BufferPtr == '\n')
131    BufferPtr++;
132  else {
133    assert(*BufferPtr == '\r');
134    BufferPtr++;
135    if (BufferPtr != BufferEnd && *BufferPtr == '\n')
136      BufferPtr++;
137  }
138  return BufferPtr;
139}
140
141const char *skipNamedCharacterReference(const char *BufferPtr,
142                                        const char *BufferEnd) {
143  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
144    if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
145      return BufferPtr;
146  }
147  return BufferEnd;
148}
149
150const char *skipDecimalCharacterReference(const char *BufferPtr,
151                                          const char *BufferEnd) {
152  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
153    if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
154      return BufferPtr;
155  }
156  return BufferEnd;
157}
158
159const char *skipHexCharacterReference(const char *BufferPtr,
160                                      const char *BufferEnd) {
161  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
162    if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
163      return BufferPtr;
164  }
165  return BufferEnd;
166}
167
168bool isHTMLIdentifierStartingCharacter(char C) {
169  return isLetter(C);
170}
171
172bool isHTMLIdentifierCharacter(char C) {
173  return isAlphanumeric(C);
174}
175
176const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
177  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
178    if (!isHTMLIdentifierCharacter(*BufferPtr))
179      return BufferPtr;
180  }
181  return BufferEnd;
182}
183
184/// Skip HTML string quoted in single or double quotes.  Escaping quotes inside
185/// string allowed.
186///
187/// Returns pointer to closing quote.
188const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
189{
190  const char Quote = *BufferPtr;
191  assert(Quote == '\"' || Quote == '\'');
192
193  BufferPtr++;
194  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
195    const char C = *BufferPtr;
196    if (C == Quote && BufferPtr[-1] != '\\')
197      return BufferPtr;
198  }
199  return BufferEnd;
200}
201
202const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
203  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
204    if (!isWhitespace(*BufferPtr))
205      return BufferPtr;
206  }
207  return BufferEnd;
208}
209
210bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
211  return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
212}
213
214bool isCommandNameStartCharacter(char C) {
215  return isLetter(C);
216}
217
218bool isCommandNameCharacter(char C) {
219  return isAlphanumeric(C);
220}
221
222const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
223  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
224    if (!isCommandNameCharacter(*BufferPtr))
225      return BufferPtr;
226  }
227  return BufferEnd;
228}
229
230/// Return the one past end pointer for BCPL comments.
231/// Handles newlines escaped with backslash or trigraph for backslahs.
232const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
233  const char *CurPtr = BufferPtr;
234  while (CurPtr != BufferEnd) {
235    while (!isVerticalWhitespace(*CurPtr)) {
236      CurPtr++;
237      if (CurPtr == BufferEnd)
238        return BufferEnd;
239    }
240    // We found a newline, check if it is escaped.
241    const char *EscapePtr = CurPtr - 1;
242    while(isHorizontalWhitespace(*EscapePtr))
243      EscapePtr--;
244
245    if (*EscapePtr == '\\' ||
246        (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
247         EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
248      // We found an escaped newline.
249      CurPtr = skipNewline(CurPtr, BufferEnd);
250    } else
251      return CurPtr; // Not an escaped newline.
252  }
253  return BufferEnd;
254}
255
256/// Return the one past end pointer for C comments.
257/// Very dumb, does not handle escaped newlines or trigraphs.
258const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
259  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
260    if (*BufferPtr == '*') {
261      assert(BufferPtr + 1 != BufferEnd);
262      if (*(BufferPtr + 1) == '/')
263        return BufferPtr;
264    }
265  }
266  llvm_unreachable("buffer end hit before '*/' was seen");
267}
268
269} // unnamed namespace
270
271void Lexer::lexCommentText(Token &T) {
272  assert(CommentState == LCS_InsideBCPLComment ||
273         CommentState == LCS_InsideCComment);
274
275  switch (State) {
276  case LS_Normal:
277    break;
278  case LS_VerbatimBlockFirstLine:
279    lexVerbatimBlockFirstLine(T);
280    return;
281  case LS_VerbatimBlockBody:
282    lexVerbatimBlockBody(T);
283    return;
284  case LS_VerbatimLineText:
285    lexVerbatimLineText(T);
286    return;
287  case LS_HTMLStartTag:
288    lexHTMLStartTag(T);
289    return;
290  case LS_HTMLEndTag:
291    lexHTMLEndTag(T);
292    return;
293  }
294
295  assert(State == LS_Normal);
296
297  const char *TokenPtr = BufferPtr;
298  assert(TokenPtr < CommentEnd);
299  while (TokenPtr != CommentEnd) {
300    switch(*TokenPtr) {
301      case '\\':
302      case '@': {
303        // Commands that start with a backslash and commands that start with
304        // 'at' have equivalent semantics.  But we keep information about the
305        // exact syntax in AST for comments.
306        tok::TokenKind CommandKind =
307            (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
308        TokenPtr++;
309        if (TokenPtr == CommentEnd) {
310          formTextToken(T, TokenPtr);
311          return;
312        }
313        char C = *TokenPtr;
314        switch (C) {
315        default:
316          break;
317
318        case '\\': case '@': case '&': case '$':
319        case '#':  case '<': case '>': case '%':
320        case '\"': case '.': case ':':
321          // This is one of \\ \@ \& \$ etc escape sequences.
322          TokenPtr++;
323          if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
324            // This is the \:: escape sequence.
325            TokenPtr++;
326          }
327          StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
328          formTokenWithChars(T, TokenPtr, tok::text);
329          T.setText(UnescapedText);
330          return;
331        }
332
333        // Don't make zero-length commands.
334        if (!isCommandNameStartCharacter(*TokenPtr)) {
335          formTextToken(T, TokenPtr);
336          return;
337        }
338
339        TokenPtr = skipCommandName(TokenPtr, CommentEnd);
340        unsigned Length = TokenPtr - (BufferPtr + 1);
341
342        // Hardcoded support for lexing LaTeX formula commands
343        // \f$ \f[ \f] \f{ \f} as a single command.
344        if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
345          C = *TokenPtr;
346          if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
347            TokenPtr++;
348            Length++;
349          }
350        }
351
352        const StringRef CommandName(BufferPtr + 1, Length);
353
354        const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
355        if (!Info) {
356          if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
357            StringRef CorrectedName = Info->Name;
358            SourceLocation Loc = getSourceLocation(BufferPtr);
359            SourceRange CommandRange(Loc.getLocWithOffset(1),
360                                     getSourceLocation(TokenPtr));
361            Diag(Loc, diag::warn_correct_comment_command_name)
362              << CommandName << CorrectedName
363              << FixItHint::CreateReplacement(CommandRange, CorrectedName);
364          } else {
365            formTokenWithChars(T, TokenPtr, tok::unknown_command);
366            T.setUnknownCommandName(CommandName);
367            Diag(T.getLocation(), diag::warn_unknown_comment_command_name);
368            return;
369          }
370        }
371        if (Info->IsVerbatimBlockCommand) {
372          setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
373          return;
374        }
375        if (Info->IsVerbatimLineCommand) {
376          setupAndLexVerbatimLine(T, TokenPtr, Info);
377          return;
378        }
379        formTokenWithChars(T, TokenPtr, CommandKind);
380        T.setCommandID(Info->getID());
381        return;
382      }
383
384      case '&':
385        lexHTMLCharacterReference(T);
386        return;
387
388      case '<': {
389        TokenPtr++;
390        if (TokenPtr == CommentEnd) {
391          formTextToken(T, TokenPtr);
392          return;
393        }
394        const char C = *TokenPtr;
395        if (isHTMLIdentifierStartingCharacter(C))
396          setupAndLexHTMLStartTag(T);
397        else if (C == '/')
398          setupAndLexHTMLEndTag(T);
399        else
400          formTextToken(T, TokenPtr);
401
402        return;
403      }
404
405      case '\n':
406      case '\r':
407        TokenPtr = skipNewline(TokenPtr, CommentEnd);
408        formTokenWithChars(T, TokenPtr, tok::newline);
409
410        if (CommentState == LCS_InsideCComment)
411          skipLineStartingDecorations();
412        return;
413
414      default: {
415        size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr).
416                         find_first_of("\n\r\\@&<");
417        if (End != StringRef::npos)
418          TokenPtr += End;
419        else
420          TokenPtr = CommentEnd;
421        formTextToken(T, TokenPtr);
422        return;
423      }
424    }
425  }
426}
427
428void Lexer::setupAndLexVerbatimBlock(Token &T,
429                                     const char *TextBegin,
430                                     char Marker, const CommandInfo *Info) {
431  assert(Info->IsVerbatimBlockCommand);
432
433  VerbatimBlockEndCommandName.clear();
434  VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
435  VerbatimBlockEndCommandName.append(Info->EndCommandName);
436
437  formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
438  T.setVerbatimBlockID(Info->getID());
439
440  // If there is a newline following the verbatim opening command, skip the
441  // newline so that we don't create an tok::verbatim_block_line with empty
442  // text content.
443  if (BufferPtr != CommentEnd &&
444      isVerticalWhitespace(*BufferPtr)) {
445    BufferPtr = skipNewline(BufferPtr, CommentEnd);
446    State = LS_VerbatimBlockBody;
447    return;
448  }
449
450  State = LS_VerbatimBlockFirstLine;
451}
452
453void Lexer::lexVerbatimBlockFirstLine(Token &T) {
454again:
455  assert(BufferPtr < CommentEnd);
456
457  // FIXME: It would be better to scan the text once, finding either the block
458  // end command or newline.
459  //
460  // Extract current line.
461  const char *Newline = findNewline(BufferPtr, CommentEnd);
462  StringRef Line(BufferPtr, Newline - BufferPtr);
463
464  // Look for end command in current line.
465  size_t Pos = Line.find(VerbatimBlockEndCommandName);
466  const char *TextEnd;
467  const char *NextLine;
468  if (Pos == StringRef::npos) {
469    // Current line is completely verbatim.
470    TextEnd = Newline;
471    NextLine = skipNewline(Newline, CommentEnd);
472  } else if (Pos == 0) {
473    // Current line contains just an end command.
474    const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
475    StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
476    formTokenWithChars(T, End, tok::verbatim_block_end);
477    T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
478    State = LS_Normal;
479    return;
480  } else {
481    // There is some text, followed by end command.  Extract text first.
482    TextEnd = BufferPtr + Pos;
483    NextLine = TextEnd;
484    // If there is only whitespace before end command, skip whitespace.
485    if (isWhitespace(BufferPtr, TextEnd)) {
486      BufferPtr = TextEnd;
487      goto again;
488    }
489  }
490
491  StringRef Text(BufferPtr, TextEnd - BufferPtr);
492  formTokenWithChars(T, NextLine, tok::verbatim_block_line);
493  T.setVerbatimBlockText(Text);
494
495  State = LS_VerbatimBlockBody;
496}
497
498void Lexer::lexVerbatimBlockBody(Token &T) {
499  assert(State == LS_VerbatimBlockBody);
500
501  if (CommentState == LCS_InsideCComment)
502    skipLineStartingDecorations();
503
504  lexVerbatimBlockFirstLine(T);
505}
506
507void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
508                                    const CommandInfo *Info) {
509  assert(Info->IsVerbatimLineCommand);
510  formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
511  T.setVerbatimLineID(Info->getID());
512
513  State = LS_VerbatimLineText;
514}
515
516void Lexer::lexVerbatimLineText(Token &T) {
517  assert(State == LS_VerbatimLineText);
518
519  // Extract current line.
520  const char *Newline = findNewline(BufferPtr, CommentEnd);
521  const StringRef Text(BufferPtr, Newline - BufferPtr);
522  formTokenWithChars(T, Newline, tok::verbatim_line_text);
523  T.setVerbatimLineText(Text);
524
525  State = LS_Normal;
526}
527
528void Lexer::lexHTMLCharacterReference(Token &T) {
529  const char *TokenPtr = BufferPtr;
530  assert(*TokenPtr == '&');
531  TokenPtr++;
532  if (TokenPtr == CommentEnd) {
533    formTextToken(T, TokenPtr);
534    return;
535  }
536  const char *NamePtr;
537  bool isNamed = false;
538  bool isDecimal = false;
539  char C = *TokenPtr;
540  if (isHTMLNamedCharacterReferenceCharacter(C)) {
541    NamePtr = TokenPtr;
542    TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
543    isNamed = true;
544  } else if (C == '#') {
545    TokenPtr++;
546    if (TokenPtr == CommentEnd) {
547      formTextToken(T, TokenPtr);
548      return;
549    }
550    C = *TokenPtr;
551    if (isHTMLDecimalCharacterReferenceCharacter(C)) {
552      NamePtr = TokenPtr;
553      TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
554      isDecimal = true;
555    } else if (C == 'x' || C == 'X') {
556      TokenPtr++;
557      NamePtr = TokenPtr;
558      TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
559    } else {
560      formTextToken(T, TokenPtr);
561      return;
562    }
563  } else {
564    formTextToken(T, TokenPtr);
565    return;
566  }
567  if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
568      *TokenPtr != ';') {
569    formTextToken(T, TokenPtr);
570    return;
571  }
572  StringRef Name(NamePtr, TokenPtr - NamePtr);
573  TokenPtr++; // Skip semicolon.
574  StringRef Resolved;
575  if (isNamed)
576    Resolved = resolveHTMLNamedCharacterReference(Name);
577  else if (isDecimal)
578    Resolved = resolveHTMLDecimalCharacterReference(Name);
579  else
580    Resolved = resolveHTMLHexCharacterReference(Name);
581
582  if (Resolved.empty()) {
583    formTextToken(T, TokenPtr);
584    return;
585  }
586  formTokenWithChars(T, TokenPtr, tok::text);
587  T.setText(Resolved);
588  return;
589}
590
591void Lexer::setupAndLexHTMLStartTag(Token &T) {
592  assert(BufferPtr[0] == '<' &&
593         isHTMLIdentifierStartingCharacter(BufferPtr[1]));
594  const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
595  StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
596  if (!isHTMLTagName(Name)) {
597    formTextToken(T, TagNameEnd);
598    return;
599  }
600
601  formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
602  T.setHTMLTagStartName(Name);
603
604  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
605
606  const char C = *BufferPtr;
607  if (BufferPtr != CommentEnd &&
608      (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
609    State = LS_HTMLStartTag;
610}
611
612void Lexer::lexHTMLStartTag(Token &T) {
613  assert(State == LS_HTMLStartTag);
614
615  const char *TokenPtr = BufferPtr;
616  char C = *TokenPtr;
617  if (isHTMLIdentifierCharacter(C)) {
618    TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
619    StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
620    formTokenWithChars(T, TokenPtr, tok::html_ident);
621    T.setHTMLIdent(Ident);
622  } else {
623    switch (C) {
624    case '=':
625      TokenPtr++;
626      formTokenWithChars(T, TokenPtr, tok::html_equals);
627      break;
628    case '\"':
629    case '\'': {
630      const char *OpenQuote = TokenPtr;
631      TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
632      const char *ClosingQuote = TokenPtr;
633      if (TokenPtr != CommentEnd) // Skip closing quote.
634        TokenPtr++;
635      formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
636      T.setHTMLQuotedString(StringRef(OpenQuote + 1,
637                                      ClosingQuote - (OpenQuote + 1)));
638      break;
639    }
640    case '>':
641      TokenPtr++;
642      formTokenWithChars(T, TokenPtr, tok::html_greater);
643      State = LS_Normal;
644      return;
645    case '/':
646      TokenPtr++;
647      if (TokenPtr != CommentEnd && *TokenPtr == '>') {
648        TokenPtr++;
649        formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
650      } else
651        formTextToken(T, TokenPtr);
652
653      State = LS_Normal;
654      return;
655    }
656  }
657
658  // Now look ahead and return to normal state if we don't see any HTML tokens
659  // ahead.
660  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
661  if (BufferPtr == CommentEnd) {
662    State = LS_Normal;
663    return;
664  }
665
666  C = *BufferPtr;
667  if (!isHTMLIdentifierStartingCharacter(C) &&
668      C != '=' && C != '\"' && C != '\'' && C != '>') {
669    State = LS_Normal;
670    return;
671  }
672}
673
674void Lexer::setupAndLexHTMLEndTag(Token &T) {
675  assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
676
677  const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
678  const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
679  StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
680  if (!isHTMLTagName(Name)) {
681    formTextToken(T, TagNameEnd);
682    return;
683  }
684
685  const char *End = skipWhitespace(TagNameEnd, CommentEnd);
686
687  formTokenWithChars(T, End, tok::html_end_tag);
688  T.setHTMLTagEndName(Name);
689
690  if (BufferPtr != CommentEnd && *BufferPtr == '>')
691    State = LS_HTMLEndTag;
692}
693
694void Lexer::lexHTMLEndTag(Token &T) {
695  assert(BufferPtr != CommentEnd && *BufferPtr == '>');
696
697  formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
698  State = LS_Normal;
699}
700
701Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
702             const CommandTraits &Traits,
703             SourceLocation FileLoc,
704             const char *BufferStart, const char *BufferEnd):
705    Allocator(Allocator), Diags(Diags), Traits(Traits),
706    BufferStart(BufferStart), BufferEnd(BufferEnd),
707    FileLoc(FileLoc), BufferPtr(BufferStart),
708    CommentState(LCS_BeforeComment), State(LS_Normal) {
709}
710
711void Lexer::lex(Token &T) {
712again:
713  switch (CommentState) {
714  case LCS_BeforeComment:
715    if (BufferPtr == BufferEnd) {
716      formTokenWithChars(T, BufferPtr, tok::eof);
717      return;
718    }
719
720    assert(*BufferPtr == '/');
721    BufferPtr++; // Skip first slash.
722    switch(*BufferPtr) {
723    case '/': { // BCPL comment.
724      BufferPtr++; // Skip second slash.
725
726      if (BufferPtr != BufferEnd) {
727        // Skip Doxygen magic marker, if it is present.
728        // It might be missing because of a typo //< or /*<, or because we
729        // merged this non-Doxygen comment into a bunch of Doxygen comments
730        // around it: /** ... */ /* ... */ /** ... */
731        const char C = *BufferPtr;
732        if (C == '/' || C == '!')
733          BufferPtr++;
734      }
735
736      // Skip less-than symbol that marks trailing comments.
737      // Skip it even if the comment is not a Doxygen one, because //< and /*<
738      // are frequent typos.
739      if (BufferPtr != BufferEnd && *BufferPtr == '<')
740        BufferPtr++;
741
742      CommentState = LCS_InsideBCPLComment;
743      if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
744        State = LS_Normal;
745      CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
746      goto again;
747    }
748    case '*': { // C comment.
749      BufferPtr++; // Skip star.
750
751      // Skip Doxygen magic marker.
752      const char C = *BufferPtr;
753      if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
754        BufferPtr++;
755
756      // Skip less-than symbol that marks trailing comments.
757      if (BufferPtr != BufferEnd && *BufferPtr == '<')
758        BufferPtr++;
759
760      CommentState = LCS_InsideCComment;
761      State = LS_Normal;
762      CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
763      goto again;
764    }
765    default:
766      llvm_unreachable("second character of comment should be '/' or '*'");
767    }
768
769  case LCS_BetweenComments: {
770    // Consecutive comments are extracted only if there is only whitespace
771    // between them.  So we can search for the start of the next comment.
772    const char *EndWhitespace = BufferPtr;
773    while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
774      EndWhitespace++;
775
776    // Turn any whitespace between comments (and there is only whitespace
777    // between them -- guaranteed by comment extraction) into a newline.  We
778    // have two newlines between C comments in total (first one was synthesized
779    // after a comment).
780    formTokenWithChars(T, EndWhitespace, tok::newline);
781
782    CommentState = LCS_BeforeComment;
783    break;
784  }
785
786  case LCS_InsideBCPLComment:
787  case LCS_InsideCComment:
788    if (BufferPtr != CommentEnd) {
789      lexCommentText(T);
790      break;
791    } else {
792      // Skip C comment closing sequence.
793      if (CommentState == LCS_InsideCComment) {
794        assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
795        BufferPtr += 2;
796        assert(BufferPtr <= BufferEnd);
797
798        // Synthenize newline just after the C comment, regardless if there is
799        // actually a newline.
800        formTokenWithChars(T, BufferPtr, tok::newline);
801
802        CommentState = LCS_BetweenComments;
803        break;
804      } else {
805        // Don't synthesized a newline after BCPL comment.
806        CommentState = LCS_BetweenComments;
807        goto again;
808      }
809    }
810  }
811}
812
813StringRef Lexer::getSpelling(const Token &Tok,
814                             const SourceManager &SourceMgr,
815                             bool *Invalid) const {
816  SourceLocation Loc = Tok.getLocation();
817  std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
818
819  bool InvalidTemp = false;
820  StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
821  if (InvalidTemp) {
822    *Invalid = true;
823    return StringRef();
824  }
825
826  const char *Begin = File.data() + LocInfo.second;
827  return StringRef(Begin, Tok.getLength());
828}
829
830} // end namespace comments
831} // end namespace clang
832
833