RawCommentList.cpp revision 363496 
1//===--- RawCommentList.cpp - Processing raw comments -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/AST/RawCommentList.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Comment.h"
12#include "clang/AST/CommentBriefParser.h"
13#include "clang/AST/CommentCommandTraits.h"
14#include "clang/AST/CommentLexer.h"
15#include "clang/AST/CommentParser.h"
16#include "clang/AST/CommentSema.h"
17#include "clang/Basic/CharInfo.h"
18#include "llvm/ADT/STLExtras.h"
19
20using namespace clang;
21
22namespace {
23/// Get comment kind and bool describing if it is a trailing comment.
24std::pair<RawComment::CommentKind, bool> getCommentKind(StringRef Comment,
25                                                        bool ParseAllComments) {
26  const size_t MinCommentLength = ParseAllComments ? 2 : 3;
27  if ((Comment.size() < MinCommentLength) || Comment[0] != '/')
28    return std::make_pair(RawComment::RCK_Invalid, false);
29
30  RawComment::CommentKind K;
31  if (Comment[1] == '/') {
32    if (Comment.size() < 3)
33      return std::make_pair(RawComment::RCK_OrdinaryBCPL, false);
34
35    if (Comment[2] == '/')
36      K = RawComment::RCK_BCPLSlash;
37    else if (Comment[2] == '!')
38      K = RawComment::RCK_BCPLExcl;
39    else
40      return std::make_pair(RawComment::RCK_OrdinaryBCPL, false);
41  } else {
42    assert(Comment.size() >= 4);
43
44    // Comment lexer does not understand escapes in comment markers, so pretend
45    // that this is not a comment.
46    if (Comment[1] != '*' ||
47        Comment[Comment.size() - 2] != '*' ||
48        Comment[Comment.size() - 1] != '/')
49      return std::make_pair(RawComment::RCK_Invalid, false);
50
51    if (Comment[2] == '*')
52      K = RawComment::RCK_JavaDoc;
53    else if (Comment[2] == '!')
54      K = RawComment::RCK_Qt;
55    else
56      return std::make_pair(RawComment::RCK_OrdinaryC, false);
57  }
58  const bool TrailingComment = (Comment.size() > 3) && (Comment[3] == '<');
59  return std::make_pair(K, TrailingComment);
60}
61
62bool mergedCommentIsTrailingComment(StringRef Comment) {
63  return (Comment.size() > 3) && (Comment[3] == '<');
64}
65
66/// Returns true if R1 and R2 both have valid locations that start on the same
67/// column.
68bool commentsStartOnSameColumn(const SourceManager &SM, const RawComment &R1,
69                               const RawComment &R2) {
70  SourceLocation L1 = R1.getBeginLoc();
71  SourceLocation L2 = R2.getBeginLoc();
72  bool Invalid = false;
73  unsigned C1 = SM.getPresumedColumnNumber(L1, &Invalid);
74  if (!Invalid) {
75    unsigned C2 = SM.getPresumedColumnNumber(L2, &Invalid);
76    return !Invalid && (C1 == C2);
77  }
78  return false;
79}
80} // unnamed namespace
81
82/// Determines whether there is only whitespace in `Buffer` between `P`
83/// and the previous line.
84/// \param Buffer The buffer to search in.
85/// \param P The offset from the beginning of `Buffer` to start from.
86/// \return true if all of the characters in `Buffer` ranging from the closest
87/// line-ending character before `P` (or the beginning of `Buffer`) to `P - 1`
88/// are whitespace.
89static bool onlyWhitespaceOnLineBefore(const char *Buffer, unsigned P) {
90  // Search backwards until we see linefeed or carriage return.
91  for (unsigned I = P; I != 0; --I) {
92    char C = Buffer[I - 1];
93    if (isVerticalWhitespace(C))
94      return true;
95    if (!isHorizontalWhitespace(C))
96      return false;
97  }
98  // We hit the beginning of the buffer.
99  return true;
100}
101
102/// Returns whether `K` is an ordinary comment kind.
103static bool isOrdinaryKind(RawComment::CommentKind K) {
104  return (K == RawComment::RCK_OrdinaryBCPL) ||
105         (K == RawComment::RCK_OrdinaryC);
106}
107
108RawComment::RawComment(const SourceManager &SourceMgr, SourceRange SR,
109                       const CommentOptions &CommentOpts, bool Merged) :
110    Range(SR), RawTextValid(false), BriefTextValid(false),
111    IsAttached(false), IsTrailingComment(false),
112    IsAlmostTrailingComment(false) {
113  // Extract raw comment text, if possible.
114  if (SR.getBegin() == SR.getEnd() || getRawText(SourceMgr).empty()) {
115    Kind = RCK_Invalid;
116    return;
117  }
118
119  // Guess comment kind.
120  std::pair<CommentKind, bool> K =
121      getCommentKind(RawText, CommentOpts.ParseAllComments);
122
123  // Guess whether an ordinary comment is trailing.
124  if (CommentOpts.ParseAllComments && isOrdinaryKind(K.first)) {
125    FileID BeginFileID;
126    unsigned BeginOffset;
127    std::tie(BeginFileID, BeginOffset) =
128        SourceMgr.getDecomposedLoc(Range.getBegin());
129    if (BeginOffset != 0) {
130      bool Invalid = false;
131      const char *Buffer =
132          SourceMgr.getBufferData(BeginFileID, &Invalid).data();
133      IsTrailingComment |=
134          (!Invalid && !onlyWhitespaceOnLineBefore(Buffer, BeginOffset));
135    }
136  }
137
138  if (!Merged) {
139    Kind = K.first;
140    IsTrailingComment |= K.second;
141
142    IsAlmostTrailingComment = RawText.startswith("//<") ||
143                                 RawText.startswith("/*<");
144  } else {
145    Kind = RCK_Merged;
146    IsTrailingComment =
147        IsTrailingComment || mergedCommentIsTrailingComment(RawText);
148  }
149}
150
151StringRef RawComment::getRawTextSlow(const SourceManager &SourceMgr) const {
152  FileID BeginFileID;
153  FileID EndFileID;
154  unsigned BeginOffset;
155  unsigned EndOffset;
156
157  std::tie(BeginFileID, BeginOffset) =
158      SourceMgr.getDecomposedLoc(Range.getBegin());
159  std::tie(EndFileID, EndOffset) = SourceMgr.getDecomposedLoc(Range.getEnd());
160
161  const unsigned Length = EndOffset - BeginOffset;
162  if (Length < 2)
163    return StringRef();
164
165  // The comment can't begin in one file and end in another.
166  assert(BeginFileID == EndFileID);
167
168  bool Invalid = false;
169  const char *BufferStart = SourceMgr.getBufferData(BeginFileID,
170                                                    &Invalid).data();
171  if (Invalid)
172    return StringRef();
173
174  return StringRef(BufferStart + BeginOffset, Length);
175}
176
177const char *RawComment::extractBriefText(const ASTContext &Context) const {
178  // Lazily initialize RawText using the accessor before using it.
179  (void)getRawText(Context.getSourceManager());
180
181  // Since we will be copying the resulting text, all allocations made during
182  // parsing are garbage after resulting string is formed.  Thus we can use
183  // a separate allocator for all temporary stuff.
184  llvm::BumpPtrAllocator Allocator;
185
186  comments::Lexer L(Allocator, Context.getDiagnostics(),
187                    Context.getCommentCommandTraits(),
188                    Range.getBegin(),
189                    RawText.begin(), RawText.end());
190  comments::BriefParser P(L, Context.getCommentCommandTraits());
191
192  const std::string Result = P.Parse();
193  const unsigned BriefTextLength = Result.size();
194  char *BriefTextPtr = new (Context) char[BriefTextLength + 1];
195  memcpy(BriefTextPtr, Result.c_str(), BriefTextLength + 1);
196  BriefText = BriefTextPtr;
197  BriefTextValid = true;
198
199  return BriefTextPtr;
200}
201
202comments::FullComment *RawComment::parse(const ASTContext &Context,
203                                         const Preprocessor *PP,
204                                         const Decl *D) const {
205  // Lazily initialize RawText using the accessor before using it.
206  (void)getRawText(Context.getSourceManager());
207
208  comments::Lexer L(Context.getAllocator(), Context.getDiagnostics(),
209                    Context.getCommentCommandTraits(),
210                    getSourceRange().getBegin(),
211                    RawText.begin(), RawText.end());
212  comments::Sema S(Context.getAllocator(), Context.getSourceManager(),
213                   Context.getDiagnostics(),
214                   Context.getCommentCommandTraits(),
215                   PP);
216  S.setDecl(D);
217  comments::Parser P(L, S, Context.getAllocator(), Context.getSourceManager(),
218                     Context.getDiagnostics(),
219                     Context.getCommentCommandTraits());
220
221  return P.parseFullComment();
222}
223
224static bool onlyWhitespaceBetween(SourceManager &SM,
225                                  SourceLocation Loc1, SourceLocation Loc2,
226                                  unsigned MaxNewlinesAllowed) {
227  std::pair<FileID, unsigned> Loc1Info = SM.getDecomposedLoc(Loc1);
228  std::pair<FileID, unsigned> Loc2Info = SM.getDecomposedLoc(Loc2);
229
230  // Question does not make sense if locations are in different files.
231  if (Loc1Info.first != Loc2Info.first)
232    return false;
233
234  bool Invalid = false;
235  const char *Buffer = SM.getBufferData(Loc1Info.first, &Invalid).data();
236  if (Invalid)
237    return false;
238
239  unsigned NumNewlines = 0;
240  assert(Loc1Info.second <= Loc2Info.second && "Loc1 after Loc2!");
241  // Look for non-whitespace characters and remember any newlines seen.
242  for (unsigned I = Loc1Info.second; I != Loc2Info.second; ++I) {
243    switch (Buffer[I]) {
244    default:
245      return false;
246    case ' ':
247    case '\t':
248    case '\f':
249    case '\v':
250      break;
251    case '\r':
252    case '\n':
253      ++NumNewlines;
254
255      // Check if we have found more than the maximum allowed number of
256      // newlines.
257      if (NumNewlines > MaxNewlinesAllowed)
258        return false;
259
260      // Collapse \r\n and \n\r into a single newline.
261      if (I + 1 != Loc2Info.second &&
262          (Buffer[I + 1] == '\n' || Buffer[I + 1] == '\r') &&
263          Buffer[I] != Buffer[I + 1])
264        ++I;
265      break;
266    }
267  }
268
269  return true;
270}
271
272void RawCommentList::addComment(const RawComment &RC,
273                                const CommentOptions &CommentOpts,
274                                llvm::BumpPtrAllocator &Allocator) {
275  if (RC.isInvalid())
276    return;
277
278  // Ordinary comments are not interesting for us.
279  if (RC.isOrdinary() && !CommentOpts.ParseAllComments)
280    return;
281
282  std::pair<FileID, unsigned> Loc =
283      SourceMgr.getDecomposedLoc(RC.getBeginLoc());
284
285  const FileID CommentFile = Loc.first;
286  const unsigned CommentOffset = Loc.second;
287
288  // If this is the first Doxygen comment, save it (because there isn't
289  // anything to merge it with).
290  if (OrderedComments[CommentFile].empty()) {
291    OrderedComments[CommentFile][CommentOffset] =
292        new (Allocator) RawComment(RC);
293    return;
294  }
295
296  const RawComment &C1 = *OrderedComments[CommentFile].rbegin()->second;
297  const RawComment &C2 = RC;
298
299  // Merge comments only if there is only whitespace between them.
300  // Can't merge trailing and non-trailing comments unless the second is
301  // non-trailing ordinary in the same column, as in the case:
302  //   int x; // documents x
303  //          // more text
304  // versus:
305  //   int x; // documents x
306  //   int y; // documents y
307  // or:
308  //   int x; // documents x
309  //   // documents y
310  //   int y;
311  // Merge comments if they are on same or consecutive lines.
312  if ((C1.isTrailingComment() == C2.isTrailingComment() ||
313       (C1.isTrailingComment() && !C2.isTrailingComment() &&
314        isOrdinaryKind(C2.getKind()) &&
315        commentsStartOnSameColumn(SourceMgr, C1, C2))) &&
316      onlyWhitespaceBetween(SourceMgr, C1.getEndLoc(), C2.getBeginLoc(),
317                            /*MaxNewlinesAllowed=*/1)) {
318    SourceRange MergedRange(C1.getBeginLoc(), C2.getEndLoc());
319    *OrderedComments[CommentFile].rbegin()->second =
320        RawComment(SourceMgr, MergedRange, CommentOpts, true);
321  } else {
322    OrderedComments[CommentFile][CommentOffset] =
323        new (Allocator) RawComment(RC);
324  }
325}
326
327const std::map<unsigned, RawComment *> *
328RawCommentList::getCommentsInFile(FileID File) const {
329  auto CommentsInFile = OrderedComments.find(File);
330  if (CommentsInFile == OrderedComments.end())
331    return nullptr;
332
333  return &CommentsInFile->second;
334}
335
336bool RawCommentList::empty() const { return OrderedComments.empty(); }
337
338unsigned RawCommentList::getCommentBeginLine(RawComment *C, FileID File,
339                                             unsigned Offset) const {
340  auto Cached = CommentBeginLine.find(C);
341  if (Cached != CommentBeginLine.end())
342    return Cached->second;
343  const unsigned Line = SourceMgr.getLineNumber(File, Offset);
344  CommentBeginLine[C] = Line;
345  return Line;
346}
347
348unsigned RawCommentList::getCommentEndOffset(RawComment *C) const {
349  auto Cached = CommentEndOffset.find(C);
350  if (Cached != CommentEndOffset.end())
351    return Cached->second;
352  const unsigned Offset =
353      SourceMgr.getDecomposedLoc(C->getSourceRange().getEnd()).second;
354  CommentEndOffset[C] = Offset;
355  return Offset;
356}
357
358std::string RawComment::getFormattedText(const SourceManager &SourceMgr,
359                                         DiagnosticsEngine &Diags) const {
360  llvm::StringRef CommentText = getRawText(SourceMgr);
361  if (CommentText.empty())
362    return "";
363
364  llvm::BumpPtrAllocator Allocator;
365  // We do not parse any commands, so CommentOptions are ignored by
366  // comments::Lexer. Therefore, we just use default-constructed options.
367  CommentOptions DefOpts;
368  comments::CommandTraits EmptyTraits(Allocator, DefOpts);
369  comments::Lexer L(Allocator, Diags, EmptyTraits, getSourceRange().getBegin(),
370                    CommentText.begin(), CommentText.end(),
371                    /*ParseCommands=*/false);
372
373  std::string Result;
374  // A column number of the first non-whitespace token in the comment text.
375  // We skip whitespace up to this column, but keep the whitespace after this
376  // column. IndentColumn is calculated when lexing the first line and reused
377  // for the rest of lines.
378  unsigned IndentColumn = 0;
379
380  // Processes one line of the comment and adds it to the result.
381  // Handles skipping the indent at the start of the line.
382  // Returns false when eof is reached and true otherwise.
383  auto LexLine = [&](bool IsFirstLine) -> bool {
384    comments::Token Tok;
385    // Lex the first token on the line. We handle it separately, because we to
386    // fix up its indentation.
387    L.lex(Tok);
388    if (Tok.is(comments::tok::eof))
389      return false;
390    if (Tok.is(comments::tok::newline)) {
391      Result += "\n";
392      return true;
393    }
394    llvm::StringRef TokText = L.getSpelling(Tok, SourceMgr);
395    bool LocInvalid = false;
396    unsigned TokColumn =
397        SourceMgr.getSpellingColumnNumber(Tok.getLocation(), &LocInvalid);
398    assert(!LocInvalid && "getFormattedText for invalid location");
399
400    // Amount of leading whitespace in TokText.
401    size_t WhitespaceLen = TokText.find_first_not_of(" \t");
402    if (WhitespaceLen == StringRef::npos)
403      WhitespaceLen = TokText.size();
404    // Remember the amount of whitespace we skipped in the first line to remove
405    // indent up to that column in the following lines.
406    if (IsFirstLine)
407      IndentColumn = TokColumn + WhitespaceLen;
408
409    // Amount of leading whitespace we actually want to skip.
410    // For the first line we skip all the whitespace.
411    // For the rest of the lines, we skip whitespace up to IndentColumn.
412    unsigned SkipLen =
413        IsFirstLine
414            ? WhitespaceLen
415            : std::min<size_t>(
416                  WhitespaceLen,
417                  std::max<int>(static_cast<int>(IndentColumn) - TokColumn, 0));
418    llvm::StringRef Trimmed = TokText.drop_front(SkipLen);
419    Result += Trimmed;
420    // Lex all tokens in the rest of the line.
421    for (L.lex(Tok); Tok.isNot(comments::tok::eof); L.lex(Tok)) {
422      if (Tok.is(comments::tok::newline)) {
423        Result += "\n";
424        return true;
425      }
426      Result += L.getSpelling(Tok, SourceMgr);
427    }
428    // We've reached the end of file token.
429    return false;
430  };
431
432  auto DropTrailingNewLines = [](std::string &Str) {
433    while (!Str.empty() && Str.back() == '\n')
434      Str.pop_back();
435  };
436
437  // Process first line separately to remember indent for the following lines.
438  if (!LexLine(/*IsFirstLine=*/true)) {
439    DropTrailingNewLines(Result);
440    return Result;
441  }
442  // Process the rest of the lines.
443  while (LexLine(/*IsFirstLine=*/false))
444    ;
445  DropTrailingNewLines(Result);
446  return Result;
447}
448