lib/Lex/TokenLexer.cpp

25 #include "clang/Lex/Token.h"
38 void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
94 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
138     SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
179     ResultToks.back().setFlag(Token::CommaAfterElided);
188     SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
192   Token *const VAOPTTokens =
195   SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
205       Token &LHS = VAOPTTokens[CurTokenIdx - 1];
227   Token StringifiedVAOPT = MacroArgs::StringifyArgument(
232     StringifiedVAOPT.setFlag(Token::LeadingSpace);
234   StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
243   SmallVector<Token, 128> ResultToks;
255     const Token &CurTok = Tokens[I];
372              "Token following # is not an argument or __VA_OPT__!");
387       const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo);
388       Token Res = MacroArgs::StringifyArgument(
390       Res.setFlag(Token::StringifiedInMacro);
395         Res.setFlag(Token::LeadingSpace);
423         ResultToks.back().setFlag(Token::LeadingSpace);
426         ResultToks.back().clearFlag(Token::LeadingSpace);
449       const Token *ResultArgToks;
453       const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
471           ResultToks.back().setFlag(Token::IgnoredComma);
475         for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
490         ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
492         ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
512     const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
533       for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
549         ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
552             Token::LeadingSpace, NextTokGetsSpace);
615 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
616                                          const Token &SecondTok) {
623 bool TokenLexer::Lex(Token &Tok) {
631     Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
632     Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
634       Tok.setFlag(Token::LeadingEmptyMacro);
647     Tok.setFlag(Token::IsReinjected);
692     Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
693     Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
697     if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
698     if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
725 bool TokenLexer::pasteTokens(Token &Tok) {
733 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
739              "Token at this Index must be ## or part of the MSVC 'L "
747     LHSTok.clearFlag(Token::LeadingSpace);
766     const Token &RHS = TokenStream[CurIdx];
794     Token ResultTokTmp;
805     Token Result;
885     Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine());
886     Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace());
946 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
954   assert(Macro && "Token streams can't paste comments");
990                                             Token *&begin_tokens,
991                                             Token * end_tokens) {
1007   Token *NextTok = begin_tokens + 1;
1011       break; // Token from different kind of FileID.
1015       break; // Token from different local/loaded location.
1022       break; // Token from a different macro.
1029   Token &LastConsecutiveTok = *(NextTok-1);
1042     Token &Tok = *begin_tokens;
1055                                             Token *begin_tokens,
1056                                             Token *end_tokens) {
1065       Token &Tok = *begin_tokens;
1076 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {