UnwrappedLineParser.cpp revision 249261 
1//===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// \file
11/// \brief This file contains the implementation of the UnwrappedLineParser,
12/// which turns a stream of tokens into UnwrappedLines.
13///
14//===----------------------------------------------------------------------===//
15
16#define DEBUG_TYPE "format-parser"
17
18#include "UnwrappedLineParser.h"
19#include "clang/Basic/Diagnostic.h"
20#include "llvm/Support/Debug.h"
21
22namespace clang {
23namespace format {
24
25class ScopedDeclarationState {
26public:
27  ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
28                         bool MustBeDeclaration)
29      : Line(Line), Stack(Stack) {
30    Line.MustBeDeclaration = MustBeDeclaration;
31    Stack.push_back(MustBeDeclaration);
32  }
33  ~ScopedDeclarationState() {
34    Stack.pop_back();
35    if (!Stack.empty())
36      Line.MustBeDeclaration = Stack.back();
37    else
38      Line.MustBeDeclaration = true;
39  }
40private:
41  UnwrappedLine &Line;
42  std::vector<bool> &Stack;
43};
44
45class ScopedMacroState : public FormatTokenSource {
46public:
47  ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
48                   FormatToken &ResetToken)
49      : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
50        PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource) {
51    TokenSource = this;
52    Line.Level = 0;
53    Line.InPPDirective = true;
54  }
55
56  ~ScopedMacroState() {
57    TokenSource = PreviousTokenSource;
58    ResetToken = Token;
59    Line.InPPDirective = false;
60    Line.Level = PreviousLineLevel;
61  }
62
63  virtual FormatToken getNextToken() {
64    // The \c UnwrappedLineParser guards against this by never calling
65    // \c getNextToken() after it has encountered the first eof token.
66    assert(!eof());
67    Token = PreviousTokenSource->getNextToken();
68    if (eof())
69      return createEOF();
70    return Token;
71  }
72
73private:
74  bool eof() { return Token.NewlinesBefore > 0 && Token.HasUnescapedNewline; }
75
76  FormatToken createEOF() {
77    FormatToken FormatTok;
78    FormatTok.Tok.startToken();
79    FormatTok.Tok.setKind(tok::eof);
80    return FormatTok;
81  }
82
83  UnwrappedLine &Line;
84  FormatTokenSource *&TokenSource;
85  FormatToken &ResetToken;
86  unsigned PreviousLineLevel;
87  FormatTokenSource *PreviousTokenSource;
88
89  FormatToken Token;
90};
91
92class ScopedLineState {
93public:
94  ScopedLineState(UnwrappedLineParser &Parser,
95                  bool SwitchToPreprocessorLines = false)
96      : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
97    if (SwitchToPreprocessorLines)
98      Parser.CurrentLines = &Parser.PreprocessorDirectives;
99    PreBlockLine = Parser.Line.take();
100    Parser.Line.reset(new UnwrappedLine());
101    Parser.Line->Level = PreBlockLine->Level;
102    Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
103  }
104
105  ~ScopedLineState() {
106    if (!Parser.Line->Tokens.empty()) {
107      Parser.addUnwrappedLine();
108    }
109    assert(Parser.Line->Tokens.empty());
110    Parser.Line.reset(PreBlockLine);
111    Parser.MustBreakBeforeNextToken = true;
112    if (SwitchToPreprocessorLines)
113      Parser.CurrentLines = &Parser.Lines;
114  }
115
116private:
117  UnwrappedLineParser &Parser;
118  const bool SwitchToPreprocessorLines;
119
120  UnwrappedLine *PreBlockLine;
121};
122
123UnwrappedLineParser::UnwrappedLineParser(
124    clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
125    FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
126    : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
127      CurrentLines(&Lines), Diag(Diag), Style(Style), Tokens(&Tokens),
128      Callback(Callback) {}
129
130bool UnwrappedLineParser::parse() {
131  DEBUG(llvm::dbgs() << "----\n");
132  readToken();
133  bool Error = parseFile();
134  for (std::vector<UnwrappedLine>::iterator I = Lines.begin(), E = Lines.end();
135       I != E; ++I) {
136    Callback.consumeUnwrappedLine(*I);
137  }
138
139  // Create line with eof token.
140  pushToken(FormatTok);
141  Callback.consumeUnwrappedLine(*Line);
142
143  return Error;
144}
145
146bool UnwrappedLineParser::parseFile() {
147  ScopedDeclarationState DeclarationState(
148      *Line, DeclarationScopeStack,
149      /*MustBeDeclaration=*/ !Line->InPPDirective);
150  bool Error = parseLevel(/*HasOpeningBrace=*/ false);
151  // Make sure to format the remaining tokens.
152  flushComments(true);
153  addUnwrappedLine();
154  return Error;
155}
156
157bool UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
158  bool Error = false;
159  do {
160    switch (FormatTok.Tok.getKind()) {
161    case tok::comment:
162      nextToken();
163      addUnwrappedLine();
164      break;
165    case tok::l_brace:
166      // FIXME: Add parameter whether this can happen - if this happens, we must
167      // be in a non-declaration context.
168      Error |= parseBlock(/*MustBeDeclaration=*/ false);
169      addUnwrappedLine();
170      break;
171    case tok::r_brace:
172      if (HasOpeningBrace) {
173        return false;
174      } else {
175        Diag.Report(FormatTok.Tok.getLocation(),
176                    Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
177                                         "unexpected '}'"));
178        Error = true;
179        nextToken();
180        addUnwrappedLine();
181      }
182      break;
183    default:
184      parseStructuralElement();
185      break;
186    }
187  } while (!eof());
188  return Error;
189}
190
191bool UnwrappedLineParser::parseBlock(bool MustBeDeclaration,
192                                     unsigned AddLevels) {
193  assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
194  nextToken();
195
196  addUnwrappedLine();
197
198  ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
199                                          MustBeDeclaration);
200  Line->Level += AddLevels;
201  parseLevel(/*HasOpeningBrace=*/ true);
202
203  if (!FormatTok.Tok.is(tok::r_brace)) {
204    Line->Level -= AddLevels;
205    return true;
206  }
207
208  nextToken(); // Munch the closing brace.
209  Line->Level -= AddLevels;
210  return false;
211}
212
213void UnwrappedLineParser::parsePPDirective() {
214  assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
215  ScopedMacroState MacroState(*Line, Tokens, FormatTok);
216  nextToken();
217
218  if (FormatTok.Tok.getIdentifierInfo() == NULL) {
219    parsePPUnknown();
220    return;
221  }
222
223  switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
224  case tok::pp_define:
225    parsePPDefine();
226    break;
227  default:
228    parsePPUnknown();
229    break;
230  }
231}
232
233void UnwrappedLineParser::parsePPDefine() {
234  nextToken();
235
236  if (FormatTok.Tok.getKind() != tok::identifier) {
237    parsePPUnknown();
238    return;
239  }
240  nextToken();
241  if (FormatTok.Tok.getKind() == tok::l_paren &&
242      FormatTok.WhiteSpaceLength == 0) {
243    parseParens();
244  }
245  addUnwrappedLine();
246  Line->Level = 1;
247
248  // Errors during a preprocessor directive can only affect the layout of the
249  // preprocessor directive, and thus we ignore them. An alternative approach
250  // would be to use the same approach we use on the file level (no
251  // re-indentation if there was a structural error) within the macro
252  // definition.
253  parseFile();
254}
255
256void UnwrappedLineParser::parsePPUnknown() {
257  do {
258    nextToken();
259  } while (!eof());
260  addUnwrappedLine();
261}
262
263void UnwrappedLineParser::parseStructuralElement() {
264  assert(!FormatTok.Tok.is(tok::l_brace));
265  int TokenNumber = 0;
266  switch (FormatTok.Tok.getKind()) {
267  case tok::at:
268    nextToken();
269    if (FormatTok.Tok.is(tok::l_brace)) {
270      parseBracedList();
271      break;
272    }
273    switch (FormatTok.Tok.getObjCKeywordID()) {
274    case tok::objc_public:
275    case tok::objc_protected:
276    case tok::objc_package:
277    case tok::objc_private:
278      return parseAccessSpecifier();
279    case tok::objc_interface:
280    case tok::objc_implementation:
281      return parseObjCInterfaceOrImplementation();
282    case tok::objc_protocol:
283      return parseObjCProtocol();
284    case tok::objc_end:
285      return; // Handled by the caller.
286    case tok::objc_optional:
287    case tok::objc_required:
288      nextToken();
289      addUnwrappedLine();
290      return;
291    default:
292      break;
293    }
294    break;
295  case tok::kw_namespace:
296    parseNamespace();
297    return;
298  case tok::kw_inline:
299    nextToken();
300    TokenNumber++;
301    if (FormatTok.Tok.is(tok::kw_namespace)) {
302      parseNamespace();
303      return;
304    }
305    break;
306  case tok::kw_public:
307  case tok::kw_protected:
308  case tok::kw_private:
309    parseAccessSpecifier();
310    return;
311  case tok::kw_if:
312    parseIfThenElse();
313    return;
314  case tok::kw_for:
315  case tok::kw_while:
316    parseForOrWhileLoop();
317    return;
318  case tok::kw_do:
319    parseDoWhile();
320    return;
321  case tok::kw_switch:
322    parseSwitch();
323    return;
324  case tok::kw_default:
325    nextToken();
326    parseLabel();
327    return;
328  case tok::kw_case:
329    parseCaseLabel();
330    return;
331  case tok::kw_return:
332    parseReturn();
333    return;
334  case tok::kw_extern:
335    nextToken();
336    if (FormatTok.Tok.is(tok::string_literal)) {
337      nextToken();
338      if (FormatTok.Tok.is(tok::l_brace)) {
339        parseBlock(/*MustBeDeclaration=*/ true, 0);
340        addUnwrappedLine();
341        return;
342      }
343    }
344    // In all other cases, parse the declaration.
345    break;
346  default:
347    break;
348  }
349  do {
350    ++TokenNumber;
351    switch (FormatTok.Tok.getKind()) {
352    case tok::at:
353      nextToken();
354      if (FormatTok.Tok.is(tok::l_brace))
355        parseBracedList();
356      break;
357    case tok::kw_enum:
358      parseEnum();
359      break;
360    case tok::kw_struct:
361    case tok::kw_union:
362    case tok::kw_class:
363      parseRecord();
364      // A record declaration or definition is always the start of a structural
365      // element.
366      break;
367    case tok::semi:
368      nextToken();
369      addUnwrappedLine();
370      return;
371    case tok::r_brace:
372      addUnwrappedLine();
373      return;
374    case tok::l_paren:
375      parseParens();
376      break;
377    case tok::l_brace:
378      // A block outside of parentheses must be the last part of a
379      // structural element.
380      // FIXME: Figure out cases where this is not true, and add projections for
381      // them (the one we know is missing are lambdas).
382      parseBlock(/*MustBeDeclaration=*/ false);
383      addUnwrappedLine();
384      return;
385    case tok::identifier:
386      nextToken();
387      if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
388        parseLabel();
389        return;
390      }
391      break;
392    case tok::equal:
393      nextToken();
394      if (FormatTok.Tok.is(tok::l_brace)) {
395        parseBracedList();
396      }
397      break;
398    default:
399      nextToken();
400      break;
401    }
402  } while (!eof());
403}
404
405void UnwrappedLineParser::parseBracedList() {
406  nextToken();
407
408  do {
409    switch (FormatTok.Tok.getKind()) {
410    case tok::l_brace:
411      parseBracedList();
412      break;
413    case tok::r_brace:
414      nextToken();
415      return;
416    default:
417      nextToken();
418      break;
419    }
420  } while (!eof());
421}
422
423void UnwrappedLineParser::parseReturn() {
424  nextToken();
425
426  do {
427    switch (FormatTok.Tok.getKind()) {
428    case tok::l_brace:
429      parseBracedList();
430      break;
431    case tok::l_paren:
432      parseParens();
433      break;
434    case tok::r_brace:
435      // Assume missing ';'.
436      addUnwrappedLine();
437      return;
438    case tok::semi:
439      nextToken();
440      addUnwrappedLine();
441      return;
442    default:
443      nextToken();
444      break;
445    }
446  } while (!eof());
447}
448
449void UnwrappedLineParser::parseParens() {
450  assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
451  nextToken();
452  do {
453    switch (FormatTok.Tok.getKind()) {
454    case tok::l_paren:
455      parseParens();
456      break;
457    case tok::r_paren:
458      nextToken();
459      return;
460    case tok::l_brace: {
461      nextToken();
462      ScopedLineState LineState(*this);
463      ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
464                                              /*MustBeDeclaration=*/ false);
465      Line->Level += 1;
466      parseLevel(/*HasOpeningBrace=*/ true);
467      Line->Level -= 1;
468      break;
469    }
470    case tok::at:
471      nextToken();
472      if (FormatTok.Tok.is(tok::l_brace))
473        parseBracedList();
474      break;
475    default:
476      nextToken();
477      break;
478    }
479  } while (!eof());
480}
481
482void UnwrappedLineParser::parseIfThenElse() {
483  assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
484  nextToken();
485  if (FormatTok.Tok.is(tok::l_paren))
486    parseParens();
487  bool NeedsUnwrappedLine = false;
488  if (FormatTok.Tok.is(tok::l_brace)) {
489    parseBlock(/*MustBeDeclaration=*/ false);
490    NeedsUnwrappedLine = true;
491  } else {
492    addUnwrappedLine();
493    ++Line->Level;
494    parseStructuralElement();
495    --Line->Level;
496  }
497  if (FormatTok.Tok.is(tok::kw_else)) {
498    nextToken();
499    if (FormatTok.Tok.is(tok::l_brace)) {
500      parseBlock(/*MustBeDeclaration=*/ false);
501      addUnwrappedLine();
502    } else if (FormatTok.Tok.is(tok::kw_if)) {
503      parseIfThenElse();
504    } else {
505      addUnwrappedLine();
506      ++Line->Level;
507      parseStructuralElement();
508      --Line->Level;
509    }
510  } else if (NeedsUnwrappedLine) {
511    addUnwrappedLine();
512  }
513}
514
515void UnwrappedLineParser::parseNamespace() {
516  assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
517  nextToken();
518  if (FormatTok.Tok.is(tok::identifier))
519    nextToken();
520  if (FormatTok.Tok.is(tok::l_brace)) {
521    parseBlock(/*MustBeDeclaration=*/ true, 0);
522    // Munch the semicolon after a namespace. This is more common than one would
523    // think. Puttin the semicolon into its own line is very ugly.
524    if (FormatTok.Tok.is(tok::semi))
525      nextToken();
526    addUnwrappedLine();
527  }
528  // FIXME: Add error handling.
529}
530
531void UnwrappedLineParser::parseForOrWhileLoop() {
532  assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
533         "'for' or 'while' expected");
534  nextToken();
535  if (FormatTok.Tok.is(tok::l_paren))
536    parseParens();
537  if (FormatTok.Tok.is(tok::l_brace)) {
538    parseBlock(/*MustBeDeclaration=*/ false);
539    addUnwrappedLine();
540  } else {
541    addUnwrappedLine();
542    ++Line->Level;
543    parseStructuralElement();
544    --Line->Level;
545  }
546}
547
548void UnwrappedLineParser::parseDoWhile() {
549  assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
550  nextToken();
551  if (FormatTok.Tok.is(tok::l_brace)) {
552    parseBlock(/*MustBeDeclaration=*/ false);
553  } else {
554    addUnwrappedLine();
555    ++Line->Level;
556    parseStructuralElement();
557    --Line->Level;
558  }
559
560  // FIXME: Add error handling.
561  if (!FormatTok.Tok.is(tok::kw_while)) {
562    addUnwrappedLine();
563    return;
564  }
565
566  nextToken();
567  parseStructuralElement();
568}
569
570void UnwrappedLineParser::parseLabel() {
571  if (FormatTok.Tok.isNot(tok::colon))
572    return;
573  nextToken();
574  unsigned OldLineLevel = Line->Level;
575  if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
576    --Line->Level;
577  if (CommentsBeforeNextToken.empty() && FormatTok.Tok.is(tok::l_brace)) {
578    parseBlock(/*MustBeDeclaration=*/ false);
579    if (FormatTok.Tok.is(tok::kw_break))
580      parseStructuralElement(); // "break;" after "}" goes on the same line.
581  }
582  addUnwrappedLine();
583  Line->Level = OldLineLevel;
584}
585
586void UnwrappedLineParser::parseCaseLabel() {
587  assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
588  // FIXME: fix handling of complex expressions here.
589  do {
590    nextToken();
591  } while (!eof() && !FormatTok.Tok.is(tok::colon));
592  parseLabel();
593}
594
595void UnwrappedLineParser::parseSwitch() {
596  assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
597  nextToken();
598  if (FormatTok.Tok.is(tok::l_paren))
599    parseParens();
600  if (FormatTok.Tok.is(tok::l_brace)) {
601    parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
602    addUnwrappedLine();
603  } else {
604    addUnwrappedLine();
605    Line->Level += (Style.IndentCaseLabels ? 2 : 1);
606    parseStructuralElement();
607    Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
608  }
609}
610
611void UnwrappedLineParser::parseAccessSpecifier() {
612  nextToken();
613  // Otherwise, we don't know what it is, and we'd better keep the next token.
614  if (FormatTok.Tok.is(tok::colon))
615    nextToken();
616  addUnwrappedLine();
617}
618
619void UnwrappedLineParser::parseEnum() {
620  nextToken();
621  if (FormatTok.Tok.is(tok::identifier) ||
622      FormatTok.Tok.is(tok::kw___attribute) ||
623      FormatTok.Tok.is(tok::kw___declspec)) {
624    nextToken();
625    // We can have macros or attributes in between 'enum' and the enum name.
626    if (FormatTok.Tok.is(tok::l_paren)) {
627      parseParens();
628    }
629    if (FormatTok.Tok.is(tok::identifier))
630      nextToken();
631  }
632  if (FormatTok.Tok.is(tok::l_brace)) {
633    nextToken();
634    addUnwrappedLine();
635    ++Line->Level;
636    do {
637      switch (FormatTok.Tok.getKind()) {
638      case tok::l_paren:
639        parseParens();
640        break;
641      case tok::r_brace:
642        addUnwrappedLine();
643        nextToken();
644        --Line->Level;
645        return;
646      case tok::comma:
647        nextToken();
648        addUnwrappedLine();
649        break;
650      default:
651        nextToken();
652        break;
653      }
654    } while (!eof());
655  }
656  // We fall through to parsing a structural element afterwards, so that in
657  // enum A {} n, m;
658  // "} n, m;" will end up in one unwrapped line.
659}
660
661void UnwrappedLineParser::parseRecord() {
662  nextToken();
663  if (FormatTok.Tok.is(tok::identifier) ||
664      FormatTok.Tok.is(tok::kw___attribute) ||
665      FormatTok.Tok.is(tok::kw___declspec)) {
666    nextToken();
667    // We can have macros or attributes in between 'class' and the class name.
668    if (FormatTok.Tok.is(tok::l_paren)) {
669      parseParens();
670    }
671    // The actual identifier can be a nested name specifier, and in macros
672    // it is often token-pasted.
673    while (FormatTok.Tok.is(tok::identifier) ||
674           FormatTok.Tok.is(tok::coloncolon) || FormatTok.Tok.is(tok::hashhash))
675      nextToken();
676
677    // Note that parsing away template declarations here leads to incorrectly
678    // accepting function declarations as record declarations.
679    // In general, we cannot solve this problem. Consider:
680    // class A<int> B() {}
681    // which can be a function definition or a class definition when B() is a
682    // macro. If we find enough real-world cases where this is a problem, we
683    // can parse for the 'template' keyword in the beginning of the statement,
684    // and thus rule out the record production in case there is no template
685    // (this would still leave us with an ambiguity between template function
686    // and class declarations).
687    if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
688      while (!eof() && FormatTok.Tok.isNot(tok::l_brace)) {
689        if (FormatTok.Tok.is(tok::semi))
690          return;
691        nextToken();
692      }
693    }
694  }
695  if (FormatTok.Tok.is(tok::l_brace))
696    parseBlock(/*MustBeDeclaration=*/ true);
697  // We fall through to parsing a structural element afterwards, so
698  // class A {} n, m;
699  // will end up in one unwrapped line.
700}
701
702void UnwrappedLineParser::parseObjCProtocolList() {
703  assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
704  do
705    nextToken();
706  while (!eof() && FormatTok.Tok.isNot(tok::greater));
707  nextToken(); // Skip '>'.
708}
709
710void UnwrappedLineParser::parseObjCUntilAtEnd() {
711  do {
712    if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
713      nextToken();
714      addUnwrappedLine();
715      break;
716    }
717    parseStructuralElement();
718  } while (!eof());
719}
720
721void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
722  nextToken();
723  nextToken(); // interface name
724
725  // @interface can be followed by either a base class, or a category.
726  if (FormatTok.Tok.is(tok::colon)) {
727    nextToken();
728    nextToken(); // base class name
729  } else if (FormatTok.Tok.is(tok::l_paren))
730    // Skip category, if present.
731    parseParens();
732
733  if (FormatTok.Tok.is(tok::less))
734    parseObjCProtocolList();
735
736  // If instance variables are present, keep the '{' on the first line too.
737  if (FormatTok.Tok.is(tok::l_brace))
738    parseBlock(/*MustBeDeclaration=*/ true);
739
740  // With instance variables, this puts '}' on its own line.  Without instance
741  // variables, this ends the @interface line.
742  addUnwrappedLine();
743
744  parseObjCUntilAtEnd();
745}
746
747void UnwrappedLineParser::parseObjCProtocol() {
748  nextToken();
749  nextToken(); // protocol name
750
751  if (FormatTok.Tok.is(tok::less))
752    parseObjCProtocolList();
753
754  // Check for protocol declaration.
755  if (FormatTok.Tok.is(tok::semi)) {
756    nextToken();
757    return addUnwrappedLine();
758  }
759
760  addUnwrappedLine();
761  parseObjCUntilAtEnd();
762}
763
764void UnwrappedLineParser::addUnwrappedLine() {
765  if (Line->Tokens.empty())
766    return;
767  DEBUG({
768    llvm::dbgs() << "Line(" << Line->Level << ")"
769                 << (Line->InPPDirective ? " MACRO" : "") << ": ";
770    for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
771                                          E = Line->Tokens.end();
772         I != E; ++I) {
773      llvm::dbgs() << I->Tok.getName() << " ";
774
775    }
776    llvm::dbgs() << "\n";
777  });
778  CurrentLines->push_back(*Line);
779  Line->Tokens.clear();
780  if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
781    for (std::vector<UnwrappedLine>::iterator
782             I = PreprocessorDirectives.begin(),
783             E = PreprocessorDirectives.end();
784         I != E; ++I) {
785      CurrentLines->push_back(*I);
786    }
787    PreprocessorDirectives.clear();
788  }
789}
790
791bool UnwrappedLineParser::eof() const { return FormatTok.Tok.is(tok::eof); }
792
793void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
794  bool JustComments = Line->Tokens.empty();
795  for (SmallVectorImpl<FormatToken>::const_iterator
796           I = CommentsBeforeNextToken.begin(),
797           E = CommentsBeforeNextToken.end();
798       I != E; ++I) {
799    if (I->NewlinesBefore && JustComments) {
800      addUnwrappedLine();
801    }
802    pushToken(*I);
803  }
804  if (NewlineBeforeNext && JustComments) {
805    addUnwrappedLine();
806  }
807  CommentsBeforeNextToken.clear();
808}
809
810void UnwrappedLineParser::nextToken() {
811  if (eof())
812    return;
813  flushComments(FormatTok.NewlinesBefore > 0);
814  pushToken(FormatTok);
815  readToken();
816}
817
818void UnwrappedLineParser::readToken() {
819  bool CommentsInCurrentLine = true;
820  do {
821    FormatTok = Tokens->getNextToken();
822    while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
823           ((FormatTok.NewlinesBefore > 0 && FormatTok.HasUnescapedNewline) ||
824            FormatTok.IsFirst)) {
825      // If there is an unfinished unwrapped line, we flush the preprocessor
826      // directives only after that unwrapped line was finished later.
827      bool SwitchToPreprocessorLines =
828          !Line->Tokens.empty() && CurrentLines == &Lines;
829      ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
830      // Comments stored before the preprocessor directive need to be output
831      // before the preprocessor directive, at the same level as the
832      // preprocessor directive, as we consider them to apply to the directive.
833      flushComments(FormatTok.NewlinesBefore > 0);
834      parsePPDirective();
835    }
836    if (!FormatTok.Tok.is(tok::comment))
837      return;
838    if (FormatTok.NewlinesBefore > 0 || FormatTok.IsFirst) {
839      CommentsInCurrentLine = false;
840    }
841    if (CommentsInCurrentLine) {
842      pushToken(FormatTok);
843    } else {
844      CommentsBeforeNextToken.push_back(FormatTok);
845    }
846  } while (!eof());
847}
848
849void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
850  Line->Tokens.push_back(Tok);
851  if (MustBreakBeforeNextToken) {
852    Line->Tokens.back().MustBreakBefore = true;
853    MustBreakBeforeNextToken = false;
854  }
855}
856
857} // end namespace format
858} // end namespace clang
859