Lexer.cpp revision 263508
1//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the Lexer and Token interfaces. 11// 12//===----------------------------------------------------------------------===// 13// 14// TODO: GCC Diagnostics emitted by the lexer: 15// PEDWARN: (form feed|vertical tab) in preprocessing directive 16// 17// Universal characters, unicode, char mapping: 18// WARNING: `%.*s' is not in NFKC 19// WARNING: `%.*s' is not in NFC 20// 21// Other: 22// TODO: Options to support: 23// -fexec-charset,-fwide-exec-charset 24// 25//===----------------------------------------------------------------------===// 26 27#include "clang/Lex/Lexer.h" 28#include "clang/Basic/CharInfo.h" 29#include "clang/Basic/SourceManager.h" 30#include "clang/Lex/CodeCompletionHandler.h" 31#include "clang/Lex/LexDiagnostic.h" 32#include "clang/Lex/LiteralSupport.h" 33#include "clang/Lex/Preprocessor.h" 34#include "llvm/ADT/STLExtras.h" 35#include "llvm/ADT/StringExtras.h" 36#include "llvm/ADT/StringSwitch.h" 37#include "llvm/Support/Compiler.h" 38#include "llvm/Support/ConvertUTF.h" 39#include "llvm/Support/MemoryBuffer.h" 40#include "UnicodeCharSets.h" 41#include <cstring> 42using namespace clang; 43 44//===----------------------------------------------------------------------===// 45// Token Class Implementation 46//===----------------------------------------------------------------------===// 47 48/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 49bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 50 if (IdentifierInfo *II = getIdentifierInfo()) 51 return II->getObjCKeywordID() == objcKey; 52 return false; 53} 54 55/// getObjCKeywordID - Return the ObjC keyword kind. 56tok::ObjCKeywordKind Token::getObjCKeywordID() const { 57 IdentifierInfo *specId = getIdentifierInfo(); 58 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 59} 60 61 62//===----------------------------------------------------------------------===// 63// Lexer Class Implementation 64//===----------------------------------------------------------------------===// 65 66void Lexer::anchor() { } 67 68void Lexer::InitLexer(const char *BufStart, const char *BufPtr, 69 const char *BufEnd) { 70 BufferStart = BufStart; 71 BufferPtr = BufPtr; 72 BufferEnd = BufEnd; 73 74 assert(BufEnd[0] == 0 && 75 "We assume that the input buffer has a null character at the end" 76 " to simplify lexing!"); 77 78 // Check whether we have a BOM in the beginning of the buffer. If yes - act 79 // accordingly. Right now we support only UTF-8 with and without BOM, so, just 80 // skip the UTF-8 BOM if it's present. 81 if (BufferStart == BufferPtr) { 82 // Determine the size of the BOM. 83 StringRef Buf(BufferStart, BufferEnd - BufferStart); 84 size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 85 .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 86 .Default(0); 87 88 // Skip the BOM. 89 BufferPtr += BOMLength; 90 } 91 92 Is_PragmaLexer = false; 93 CurrentConflictMarkerState = CMK_None; 94 95 // Start of the file is a start of line. 96 IsAtStartOfLine = true; 97 IsAtPhysicalStartOfLine = true; 98 99 HasLeadingSpace = false; 100 HasLeadingEmptyMacro = false; 101 102 // We are not after parsing a #. 103 ParsingPreprocessorDirective = false; 104 105 // We are not after parsing #include. 106 ParsingFilename = false; 107 108 // We are not in raw mode. Raw mode disables diagnostics and interpretation 109 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 110 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 111 // or otherwise skipping over tokens. 112 LexingRawMode = false; 113 114 // Default to not keeping comments. 115 ExtendedTokenMode = 0; 116} 117 118/// Lexer constructor - Create a new lexer object for the specified buffer 119/// with the specified preprocessor managing the lexing process. This lexer 120/// assumes that the associated file buffer and Preprocessor objects will 121/// outlive it, so it doesn't take ownership of either of them. 122Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 123 : PreprocessorLexer(&PP, FID), 124 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 125 LangOpts(PP.getLangOpts()) { 126 127 InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 128 InputFile->getBufferEnd()); 129 130 resetExtendedTokenMode(); 131} 132 133void Lexer::resetExtendedTokenMode() { 134 assert(PP && "Cannot reset token mode without a preprocessor"); 135 if (LangOpts.TraditionalCPP) 136 SetKeepWhitespaceMode(true); 137 else 138 SetCommentRetentionState(PP->getCommentRetentionState()); 139} 140 141/// Lexer constructor - Create a new raw lexer object. This object is only 142/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 143/// range will outlive it, so it doesn't take ownership of it. 144Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, 145 const char *BufStart, const char *BufPtr, const char *BufEnd) 146 : FileLoc(fileloc), LangOpts(langOpts) { 147 148 InitLexer(BufStart, BufPtr, BufEnd); 149 150 // We *are* in raw mode. 151 LexingRawMode = true; 152} 153 154/// Lexer constructor - Create a new raw lexer object. This object is only 155/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 156/// range will outlive it, so it doesn't take ownership of it. 157Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 158 const SourceManager &SM, const LangOptions &langOpts) 159 : FileLoc(SM.getLocForStartOfFile(FID)), LangOpts(langOpts) { 160 161 InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 162 FromFile->getBufferEnd()); 163 164 // We *are* in raw mode. 165 LexingRawMode = true; 166} 167 168/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 169/// _Pragma expansion. This has a variety of magic semantics that this method 170/// sets up. It returns a new'd Lexer that must be delete'd when done. 171/// 172/// On entrance to this routine, TokStartLoc is a macro location which has a 173/// spelling loc that indicates the bytes to be lexed for the token and an 174/// expansion location that indicates where all lexed tokens should be 175/// "expanded from". 176/// 177/// FIXME: It would really be nice to make _Pragma just be a wrapper around a 178/// normal lexer that remaps tokens as they fly by. This would require making 179/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 180/// interface that could handle this stuff. This would pull GetMappedTokenLoc 181/// out of the critical path of the lexer! 182/// 183Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 184 SourceLocation ExpansionLocStart, 185 SourceLocation ExpansionLocEnd, 186 unsigned TokLen, Preprocessor &PP) { 187 SourceManager &SM = PP.getSourceManager(); 188 189 // Create the lexer as if we were going to lex the file normally. 190 FileID SpellingFID = SM.getFileID(SpellingLoc); 191 const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 192 Lexer *L = new Lexer(SpellingFID, InputFile, PP); 193 194 // Now that the lexer is created, change the start/end locations so that we 195 // just lex the subsection of the file that we want. This is lexing from a 196 // scratch buffer. 197 const char *StrData = SM.getCharacterData(SpellingLoc); 198 199 L->BufferPtr = StrData; 200 L->BufferEnd = StrData+TokLen; 201 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 202 203 // Set the SourceLocation with the remapping information. This ensures that 204 // GetMappedTokenLoc will remap the tokens as they are lexed. 205 L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 206 ExpansionLocStart, 207 ExpansionLocEnd, TokLen); 208 209 // Ensure that the lexer thinks it is inside a directive, so that end \n will 210 // return an EOD token. 211 L->ParsingPreprocessorDirective = true; 212 213 // This lexer really is for _Pragma. 214 L->Is_PragmaLexer = true; 215 return L; 216} 217 218 219/// Stringify - Convert the specified string into a C string, with surrounding 220/// ""'s, and with escaped \ and " characters. 221std::string Lexer::Stringify(const std::string &Str, bool Charify) { 222 std::string Result = Str; 223 char Quote = Charify ? '\'' : '"'; 224 for (unsigned i = 0, e = Result.size(); i != e; ++i) { 225 if (Result[i] == '\\' || Result[i] == Quote) { 226 Result.insert(Result.begin()+i, '\\'); 227 ++i; ++e; 228 } 229 } 230 return Result; 231} 232 233/// Stringify - Convert the specified string into a C string by escaping '\' 234/// and " characters. This does not add surrounding ""'s to the string. 235void Lexer::Stringify(SmallVectorImpl<char> &Str) { 236 for (unsigned i = 0, e = Str.size(); i != e; ++i) { 237 if (Str[i] == '\\' || Str[i] == '"') { 238 Str.insert(Str.begin()+i, '\\'); 239 ++i; ++e; 240 } 241 } 242} 243 244//===----------------------------------------------------------------------===// 245// Token Spelling 246//===----------------------------------------------------------------------===// 247 248/// \brief Slow case of getSpelling. Extract the characters comprising the 249/// spelling of this token from the provided input buffer. 250static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, 251 const LangOptions &LangOpts, char *Spelling) { 252 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); 253 254 size_t Length = 0; 255 const char *BufEnd = BufPtr + Tok.getLength(); 256 257 if (Tok.is(tok::string_literal)) { 258 // Munch the encoding-prefix and opening double-quote. 259 while (BufPtr < BufEnd) { 260 unsigned Size; 261 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 262 BufPtr += Size; 263 264 if (Spelling[Length - 1] == '"') 265 break; 266 } 267 268 // Raw string literals need special handling; trigraph expansion and line 269 // splicing do not occur within their d-char-sequence nor within their 270 // r-char-sequence. 271 if (Length >= 2 && 272 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { 273 // Search backwards from the end of the token to find the matching closing 274 // quote. 275 const char *RawEnd = BufEnd; 276 do --RawEnd; while (*RawEnd != '"'); 277 size_t RawLength = RawEnd - BufPtr + 1; 278 279 // Everything between the quotes is included verbatim in the spelling. 280 memcpy(Spelling + Length, BufPtr, RawLength); 281 Length += RawLength; 282 BufPtr += RawLength; 283 284 // The rest of the token is lexed normally. 285 } 286 } 287 288 while (BufPtr < BufEnd) { 289 unsigned Size; 290 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 291 BufPtr += Size; 292 } 293 294 assert(Length < Tok.getLength() && 295 "NeedsCleaning flag set on token that didn't need cleaning!"); 296 return Length; 297} 298 299/// getSpelling() - Return the 'spelling' of this token. The spelling of a 300/// token are the characters used to represent the token in the source file 301/// after trigraph expansion and escaped-newline folding. In particular, this 302/// wants to get the true, uncanonicalized, spelling of things like digraphs 303/// UCNs, etc. 304StringRef Lexer::getSpelling(SourceLocation loc, 305 SmallVectorImpl<char> &buffer, 306 const SourceManager &SM, 307 const LangOptions &options, 308 bool *invalid) { 309 // Break down the source location. 310 std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 311 312 // Try to the load the file buffer. 313 bool invalidTemp = false; 314 StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 315 if (invalidTemp) { 316 if (invalid) *invalid = true; 317 return StringRef(); 318 } 319 320 const char *tokenBegin = file.data() + locInfo.second; 321 322 // Lex from the start of the given location. 323 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 324 file.begin(), tokenBegin, file.end()); 325 Token token; 326 lexer.LexFromRawLexer(token); 327 328 unsigned length = token.getLength(); 329 330 // Common case: no need for cleaning. 331 if (!token.needsCleaning()) 332 return StringRef(tokenBegin, length); 333 334 // Hard case, we need to relex the characters into the string. 335 buffer.resize(length); 336 buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); 337 return StringRef(buffer.data(), buffer.size()); 338} 339 340/// getSpelling() - Return the 'spelling' of this token. The spelling of a 341/// token are the characters used to represent the token in the source file 342/// after trigraph expansion and escaped-newline folding. In particular, this 343/// wants to get the true, uncanonicalized, spelling of things like digraphs 344/// UCNs, etc. 345std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 346 const LangOptions &LangOpts, bool *Invalid) { 347 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 348 349 bool CharDataInvalid = false; 350 const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 351 &CharDataInvalid); 352 if (Invalid) 353 *Invalid = CharDataInvalid; 354 if (CharDataInvalid) 355 return std::string(); 356 357 // If this token contains nothing interesting, return it directly. 358 if (!Tok.needsCleaning()) 359 return std::string(TokStart, TokStart + Tok.getLength()); 360 361 std::string Result; 362 Result.resize(Tok.getLength()); 363 Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); 364 return Result; 365} 366 367/// getSpelling - This method is used to get the spelling of a token into a 368/// preallocated buffer, instead of as an std::string. The caller is required 369/// to allocate enough space for the token, which is guaranteed to be at least 370/// Tok.getLength() bytes long. The actual length of the token is returned. 371/// 372/// Note that this method may do two possible things: it may either fill in 373/// the buffer specified with characters, or it may *change the input pointer* 374/// to point to a constant buffer with the data already in it (avoiding a 375/// copy). The caller is not allowed to modify the returned buffer pointer 376/// if an internal buffer is returned. 377unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 378 const SourceManager &SourceMgr, 379 const LangOptions &LangOpts, bool *Invalid) { 380 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 381 382 const char *TokStart = 0; 383 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 384 if (Tok.is(tok::raw_identifier)) 385 TokStart = Tok.getRawIdentifierData(); 386 else if (!Tok.hasUCN()) { 387 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 388 // Just return the string from the identifier table, which is very quick. 389 Buffer = II->getNameStart(); 390 return II->getLength(); 391 } 392 } 393 394 // NOTE: this can be checked even after testing for an IdentifierInfo. 395 if (Tok.isLiteral()) 396 TokStart = Tok.getLiteralData(); 397 398 if (TokStart == 0) { 399 // Compute the start of the token in the input lexer buffer. 400 bool CharDataInvalid = false; 401 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 402 if (Invalid) 403 *Invalid = CharDataInvalid; 404 if (CharDataInvalid) { 405 Buffer = ""; 406 return 0; 407 } 408 } 409 410 // If this token contains nothing interesting, return it directly. 411 if (!Tok.needsCleaning()) { 412 Buffer = TokStart; 413 return Tok.getLength(); 414 } 415 416 // Otherwise, hard case, relex the characters into the string. 417 return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); 418} 419 420 421/// MeasureTokenLength - Relex the token at the specified location and return 422/// its length in bytes in the input file. If the token needs cleaning (e.g. 423/// includes a trigraph or an escaped newline) then this count includes bytes 424/// that are part of that. 425unsigned Lexer::MeasureTokenLength(SourceLocation Loc, 426 const SourceManager &SM, 427 const LangOptions &LangOpts) { 428 Token TheTok; 429 if (getRawToken(Loc, TheTok, SM, LangOpts)) 430 return 0; 431 return TheTok.getLength(); 432} 433 434/// \brief Relex the token at the specified location. 435/// \returns true if there was a failure, false on success. 436bool Lexer::getRawToken(SourceLocation Loc, Token &Result, 437 const SourceManager &SM, 438 const LangOptions &LangOpts, 439 bool IgnoreWhiteSpace) { 440 // TODO: this could be special cased for common tokens like identifiers, ')', 441 // etc to make this faster, if it mattered. Just look at StrData[0] to handle 442 // all obviously single-char tokens. This could use 443 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 444 // something. 445 446 // If this comes from a macro expansion, we really do want the macro name, not 447 // the token this macro expanded to. 448 Loc = SM.getExpansionLoc(Loc); 449 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 450 bool Invalid = false; 451 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 452 if (Invalid) 453 return true; 454 455 const char *StrData = Buffer.data()+LocInfo.second; 456 457 if (!IgnoreWhiteSpace && isWhitespace(StrData[0])) 458 return true; 459 460 // Create a lexer starting at the beginning of this token. 461 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 462 Buffer.begin(), StrData, Buffer.end()); 463 TheLexer.SetCommentRetentionState(true); 464 TheLexer.LexFromRawLexer(Result); 465 return false; 466} 467 468static SourceLocation getBeginningOfFileToken(SourceLocation Loc, 469 const SourceManager &SM, 470 const LangOptions &LangOpts) { 471 assert(Loc.isFileID()); 472 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 473 if (LocInfo.first.isInvalid()) 474 return Loc; 475 476 bool Invalid = false; 477 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 478 if (Invalid) 479 return Loc; 480 481 // Back up from the current location until we hit the beginning of a line 482 // (or the buffer). We'll relex from that point. 483 const char *BufStart = Buffer.data(); 484 if (LocInfo.second >= Buffer.size()) 485 return Loc; 486 487 const char *StrData = BufStart+LocInfo.second; 488 if (StrData[0] == '\n' || StrData[0] == '\r') 489 return Loc; 490 491 const char *LexStart = StrData; 492 while (LexStart != BufStart) { 493 if (LexStart[0] == '\n' || LexStart[0] == '\r') { 494 ++LexStart; 495 break; 496 } 497 498 --LexStart; 499 } 500 501 // Create a lexer starting at the beginning of this token. 502 SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 503 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 504 TheLexer.SetCommentRetentionState(true); 505 506 // Lex tokens until we find the token that contains the source location. 507 Token TheTok; 508 do { 509 TheLexer.LexFromRawLexer(TheTok); 510 511 if (TheLexer.getBufferLocation() > StrData) { 512 // Lexing this token has taken the lexer past the source location we're 513 // looking for. If the current token encompasses our source location, 514 // return the beginning of that token. 515 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 516 return TheTok.getLocation(); 517 518 // We ended up skipping over the source location entirely, which means 519 // that it points into whitespace. We're done here. 520 break; 521 } 522 } while (TheTok.getKind() != tok::eof); 523 524 // We've passed our source location; just return the original source location. 525 return Loc; 526} 527 528SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 529 const SourceManager &SM, 530 const LangOptions &LangOpts) { 531 if (Loc.isFileID()) 532 return getBeginningOfFileToken(Loc, SM, LangOpts); 533 534 if (!SM.isMacroArgExpansion(Loc)) 535 return Loc; 536 537 SourceLocation FileLoc = SM.getSpellingLoc(Loc); 538 SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 539 std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 540 std::pair<FileID, unsigned> BeginFileLocInfo 541 = SM.getDecomposedLoc(BeginFileLoc); 542 assert(FileLocInfo.first == BeginFileLocInfo.first && 543 FileLocInfo.second >= BeginFileLocInfo.second); 544 return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); 545} 546 547namespace { 548 enum PreambleDirectiveKind { 549 PDK_Skipped, 550 PDK_StartIf, 551 PDK_EndIf, 552 PDK_Unknown 553 }; 554} 555 556std::pair<unsigned, bool> 557Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, 558 const LangOptions &LangOpts, unsigned MaxLines) { 559 // Create a lexer starting at the beginning of the file. Note that we use a 560 // "fake" file source location at offset 1 so that the lexer will track our 561 // position within the file. 562 const unsigned StartOffset = 1; 563 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); 564 Lexer TheLexer(FileLoc, LangOpts, Buffer->getBufferStart(), 565 Buffer->getBufferStart(), Buffer->getBufferEnd()); 566 TheLexer.SetCommentRetentionState(true); 567 568 // StartLoc will differ from FileLoc if there is a BOM that was skipped. 569 SourceLocation StartLoc = TheLexer.getSourceLocation(); 570 571 bool InPreprocessorDirective = false; 572 Token TheTok; 573 Token IfStartTok; 574 unsigned IfCount = 0; 575 SourceLocation ActiveCommentLoc; 576 577 unsigned MaxLineOffset = 0; 578 if (MaxLines) { 579 const char *CurPtr = Buffer->getBufferStart(); 580 unsigned CurLine = 0; 581 while (CurPtr != Buffer->getBufferEnd()) { 582 char ch = *CurPtr++; 583 if (ch == '\n') { 584 ++CurLine; 585 if (CurLine == MaxLines) 586 break; 587 } 588 } 589 if (CurPtr != Buffer->getBufferEnd()) 590 MaxLineOffset = CurPtr - Buffer->getBufferStart(); 591 } 592 593 do { 594 TheLexer.LexFromRawLexer(TheTok); 595 596 if (InPreprocessorDirective) { 597 // If we've hit the end of the file, we're done. 598 if (TheTok.getKind() == tok::eof) { 599 break; 600 } 601 602 // If we haven't hit the end of the preprocessor directive, skip this 603 // token. 604 if (!TheTok.isAtStartOfLine()) 605 continue; 606 607 // We've passed the end of the preprocessor directive, and will look 608 // at this token again below. 609 InPreprocessorDirective = false; 610 } 611 612 // Keep track of the # of lines in the preamble. 613 if (TheTok.isAtStartOfLine()) { 614 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 615 616 // If we were asked to limit the number of lines in the preamble, 617 // and we're about to exceed that limit, we're done. 618 if (MaxLineOffset && TokOffset >= MaxLineOffset) 619 break; 620 } 621 622 // Comments are okay; skip over them. 623 if (TheTok.getKind() == tok::comment) { 624 if (ActiveCommentLoc.isInvalid()) 625 ActiveCommentLoc = TheTok.getLocation(); 626 continue; 627 } 628 629 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 630 // This is the start of a preprocessor directive. 631 Token HashTok = TheTok; 632 InPreprocessorDirective = true; 633 ActiveCommentLoc = SourceLocation(); 634 635 // Figure out which directive this is. Since we're lexing raw tokens, 636 // we don't have an identifier table available. Instead, just look at 637 // the raw identifier to recognize and categorize preprocessor directives. 638 TheLexer.LexFromRawLexer(TheTok); 639 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 640 StringRef Keyword(TheTok.getRawIdentifierData(), 641 TheTok.getLength()); 642 PreambleDirectiveKind PDK 643 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 644 .Case("include", PDK_Skipped) 645 .Case("__include_macros", PDK_Skipped) 646 .Case("define", PDK_Skipped) 647 .Case("undef", PDK_Skipped) 648 .Case("line", PDK_Skipped) 649 .Case("error", PDK_Skipped) 650 .Case("pragma", PDK_Skipped) 651 .Case("import", PDK_Skipped) 652 .Case("include_next", PDK_Skipped) 653 .Case("warning", PDK_Skipped) 654 .Case("ident", PDK_Skipped) 655 .Case("sccs", PDK_Skipped) 656 .Case("assert", PDK_Skipped) 657 .Case("unassert", PDK_Skipped) 658 .Case("if", PDK_StartIf) 659 .Case("ifdef", PDK_StartIf) 660 .Case("ifndef", PDK_StartIf) 661 .Case("elif", PDK_Skipped) 662 .Case("else", PDK_Skipped) 663 .Case("endif", PDK_EndIf) 664 .Default(PDK_Unknown); 665 666 switch (PDK) { 667 case PDK_Skipped: 668 continue; 669 670 case PDK_StartIf: 671 if (IfCount == 0) 672 IfStartTok = HashTok; 673 674 ++IfCount; 675 continue; 676 677 case PDK_EndIf: 678 // Mismatched #endif. The preamble ends here. 679 if (IfCount == 0) 680 break; 681 682 --IfCount; 683 continue; 684 685 case PDK_Unknown: 686 // We don't know what this directive is; stop at the '#'. 687 break; 688 } 689 } 690 691 // We only end up here if we didn't recognize the preprocessor 692 // directive or it was one that can't occur in the preamble at this 693 // point. Roll back the current token to the location of the '#'. 694 InPreprocessorDirective = false; 695 TheTok = HashTok; 696 } 697 698 // We hit a token that we don't recognize as being in the 699 // "preprocessing only" part of the file, so we're no longer in 700 // the preamble. 701 break; 702 } while (true); 703 704 SourceLocation End; 705 if (IfCount) 706 End = IfStartTok.getLocation(); 707 else if (ActiveCommentLoc.isValid()) 708 End = ActiveCommentLoc; // don't truncate a decl comment. 709 else 710 End = TheTok.getLocation(); 711 712 return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 713 IfCount? IfStartTok.isAtStartOfLine() 714 : TheTok.isAtStartOfLine()); 715} 716 717 718/// AdvanceToTokenCharacter - Given a location that specifies the start of a 719/// token, return a new location that specifies a character within the token. 720SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 721 unsigned CharNo, 722 const SourceManager &SM, 723 const LangOptions &LangOpts) { 724 // Figure out how many physical characters away the specified expansion 725 // character is. This needs to take into consideration newlines and 726 // trigraphs. 727 bool Invalid = false; 728 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 729 730 // If they request the first char of the token, we're trivially done. 731 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 732 return TokStart; 733 734 unsigned PhysOffset = 0; 735 736 // The usual case is that tokens don't contain anything interesting. Skip 737 // over the uninteresting characters. If a token only consists of simple 738 // chars, this method is extremely fast. 739 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 740 if (CharNo == 0) 741 return TokStart.getLocWithOffset(PhysOffset); 742 ++TokPtr, --CharNo, ++PhysOffset; 743 } 744 745 // If we have a character that may be a trigraph or escaped newline, use a 746 // lexer to parse it correctly. 747 for (; CharNo; --CharNo) { 748 unsigned Size; 749 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); 750 TokPtr += Size; 751 PhysOffset += Size; 752 } 753 754 // Final detail: if we end up on an escaped newline, we want to return the 755 // location of the actual byte of the token. For example foo\<newline>bar 756 // advanced by 3 should return the location of b, not of \\. One compounding 757 // detail of this is that the escape may be made by a trigraph. 758 if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 759 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 760 761 return TokStart.getLocWithOffset(PhysOffset); 762} 763 764/// \brief Computes the source location just past the end of the 765/// token at this source location. 766/// 767/// This routine can be used to produce a source location that 768/// points just past the end of the token referenced by \p Loc, and 769/// is generally used when a diagnostic needs to point just after a 770/// token where it expected something different that it received. If 771/// the returned source location would not be meaningful (e.g., if 772/// it points into a macro), this routine returns an invalid 773/// source location. 774/// 775/// \param Offset an offset from the end of the token, where the source 776/// location should refer to. The default offset (0) produces a source 777/// location pointing just past the end of the token; an offset of 1 produces 778/// a source location pointing to the last character in the token, etc. 779SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 780 const SourceManager &SM, 781 const LangOptions &LangOpts) { 782 if (Loc.isInvalid()) 783 return SourceLocation(); 784 785 if (Loc.isMacroID()) { 786 if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 787 return SourceLocation(); // Points inside the macro expansion. 788 } 789 790 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 791 if (Len > Offset) 792 Len = Len - Offset; 793 else 794 return Loc; 795 796 return Loc.getLocWithOffset(Len); 797} 798 799/// \brief Returns true if the given MacroID location points at the first 800/// token of the macro expansion. 801bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 802 const SourceManager &SM, 803 const LangOptions &LangOpts, 804 SourceLocation *MacroBegin) { 805 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 806 807 SourceLocation expansionLoc; 808 if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc)) 809 return false; 810 811 if (expansionLoc.isFileID()) { 812 // No other macro expansions, this is the first. 813 if (MacroBegin) 814 *MacroBegin = expansionLoc; 815 return true; 816 } 817 818 return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); 819} 820 821/// \brief Returns true if the given MacroID location points at the last 822/// token of the macro expansion. 823bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 824 const SourceManager &SM, 825 const LangOptions &LangOpts, 826 SourceLocation *MacroEnd) { 827 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 828 829 SourceLocation spellLoc = SM.getSpellingLoc(loc); 830 unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 831 if (tokLen == 0) 832 return false; 833 834 SourceLocation afterLoc = loc.getLocWithOffset(tokLen); 835 SourceLocation expansionLoc; 836 if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc)) 837 return false; 838 839 if (expansionLoc.isFileID()) { 840 // No other macro expansions. 841 if (MacroEnd) 842 *MacroEnd = expansionLoc; 843 return true; 844 } 845 846 return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); 847} 848 849static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, 850 const SourceManager &SM, 851 const LangOptions &LangOpts) { 852 SourceLocation Begin = Range.getBegin(); 853 SourceLocation End = Range.getEnd(); 854 assert(Begin.isFileID() && End.isFileID()); 855 if (Range.isTokenRange()) { 856 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); 857 if (End.isInvalid()) 858 return CharSourceRange(); 859 } 860 861 // Break down the source locations. 862 FileID FID; 863 unsigned BeginOffs; 864 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 865 if (FID.isInvalid()) 866 return CharSourceRange(); 867 868 unsigned EndOffs; 869 if (!SM.isInFileID(End, FID, &EndOffs) || 870 BeginOffs > EndOffs) 871 return CharSourceRange(); 872 873 return CharSourceRange::getCharRange(Begin, End); 874} 875 876CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, 877 const SourceManager &SM, 878 const LangOptions &LangOpts) { 879 SourceLocation Begin = Range.getBegin(); 880 SourceLocation End = Range.getEnd(); 881 if (Begin.isInvalid() || End.isInvalid()) 882 return CharSourceRange(); 883 884 if (Begin.isFileID() && End.isFileID()) 885 return makeRangeFromFileLocs(Range, SM, LangOpts); 886 887 if (Begin.isMacroID() && End.isFileID()) { 888 if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) 889 return CharSourceRange(); 890 Range.setBegin(Begin); 891 return makeRangeFromFileLocs(Range, SM, LangOpts); 892 } 893 894 if (Begin.isFileID() && End.isMacroID()) { 895 if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, 896 &End)) || 897 (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, 898 &End))) 899 return CharSourceRange(); 900 Range.setEnd(End); 901 return makeRangeFromFileLocs(Range, SM, LangOpts); 902 } 903 904 assert(Begin.isMacroID() && End.isMacroID()); 905 SourceLocation MacroBegin, MacroEnd; 906 if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && 907 ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, 908 &MacroEnd)) || 909 (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, 910 &MacroEnd)))) { 911 Range.setBegin(MacroBegin); 912 Range.setEnd(MacroEnd); 913 return makeRangeFromFileLocs(Range, SM, LangOpts); 914 } 915 916 bool Invalid = false; 917 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin), 918 &Invalid); 919 if (Invalid) 920 return CharSourceRange(); 921 922 if (BeginEntry.getExpansion().isMacroArgExpansion()) { 923 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End), 924 &Invalid); 925 if (Invalid) 926 return CharSourceRange(); 927 928 if (EndEntry.getExpansion().isMacroArgExpansion() && 929 BeginEntry.getExpansion().getExpansionLocStart() == 930 EndEntry.getExpansion().getExpansionLocStart()) { 931 Range.setBegin(SM.getImmediateSpellingLoc(Begin)); 932 Range.setEnd(SM.getImmediateSpellingLoc(End)); 933 return makeFileCharRange(Range, SM, LangOpts); 934 } 935 } 936 937 return CharSourceRange(); 938} 939 940StringRef Lexer::getSourceText(CharSourceRange Range, 941 const SourceManager &SM, 942 const LangOptions &LangOpts, 943 bool *Invalid) { 944 Range = makeFileCharRange(Range, SM, LangOpts); 945 if (Range.isInvalid()) { 946 if (Invalid) *Invalid = true; 947 return StringRef(); 948 } 949 950 // Break down the source location. 951 std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); 952 if (beginInfo.first.isInvalid()) { 953 if (Invalid) *Invalid = true; 954 return StringRef(); 955 } 956 957 unsigned EndOffs; 958 if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || 959 beginInfo.second > EndOffs) { 960 if (Invalid) *Invalid = true; 961 return StringRef(); 962 } 963 964 // Try to the load the file buffer. 965 bool invalidTemp = false; 966 StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); 967 if (invalidTemp) { 968 if (Invalid) *Invalid = true; 969 return StringRef(); 970 } 971 972 if (Invalid) *Invalid = false; 973 return file.substr(beginInfo.second, EndOffs - beginInfo.second); 974} 975 976StringRef Lexer::getImmediateMacroName(SourceLocation Loc, 977 const SourceManager &SM, 978 const LangOptions &LangOpts) { 979 assert(Loc.isMacroID() && "Only reasonble to call this on macros"); 980 981 // Find the location of the immediate macro expansion. 982 while (1) { 983 FileID FID = SM.getFileID(Loc); 984 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 985 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 986 Loc = Expansion.getExpansionLocStart(); 987 if (!Expansion.isMacroArgExpansion()) 988 break; 989 990 // For macro arguments we need to check that the argument did not come 991 // from an inner macro, e.g: "MAC1( MAC2(foo) )" 992 993 // Loc points to the argument id of the macro definition, move to the 994 // macro expansion. 995 Loc = SM.getImmediateExpansionRange(Loc).first; 996 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 997 if (SpellLoc.isFileID()) 998 break; // No inner macro. 999 1000 // If spelling location resides in the same FileID as macro expansion 1001 // location, it means there is no inner macro. 1002 FileID MacroFID = SM.getFileID(Loc); 1003 if (SM.isInFileID(SpellLoc, MacroFID)) 1004 break; 1005 1006 // Argument came from inner macro. 1007 Loc = SpellLoc; 1008 } 1009 1010 // Find the spelling location of the start of the non-argument expansion 1011 // range. This is where the macro name was spelled in order to begin 1012 // expanding this macro. 1013 Loc = SM.getSpellingLoc(Loc); 1014 1015 // Dig out the buffer where the macro name was spelled and the extents of the 1016 // name so that we can render it into the expansion note. 1017 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 1018 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 1019 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1020 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1021} 1022 1023bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { 1024 return isIdentifierBody(c, LangOpts.DollarIdents); 1025} 1026 1027 1028//===----------------------------------------------------------------------===// 1029// Diagnostics forwarding code. 1030//===----------------------------------------------------------------------===// 1031 1032/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 1033/// lexer buffer was all expanded at a single point, perform the mapping. 1034/// This is currently only used for _Pragma implementation, so it is the slow 1035/// path of the hot getSourceLocation method. Do not allow it to be inlined. 1036static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 1037 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 1038static SourceLocation GetMappedTokenLoc(Preprocessor &PP, 1039 SourceLocation FileLoc, 1040 unsigned CharNo, unsigned TokLen) { 1041 assert(FileLoc.isMacroID() && "Must be a macro expansion"); 1042 1043 // Otherwise, we're lexing "mapped tokens". This is used for things like 1044 // _Pragma handling. Combine the expansion location of FileLoc with the 1045 // spelling location. 1046 SourceManager &SM = PP.getSourceManager(); 1047 1048 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 1049 // characters come from spelling(FileLoc)+Offset. 1050 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 1051 SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 1052 1053 // Figure out the expansion loc range, which is the range covered by the 1054 // original _Pragma(...) sequence. 1055 std::pair<SourceLocation,SourceLocation> II = 1056 SM.getImmediateExpansionRange(FileLoc); 1057 1058 return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 1059} 1060 1061/// getSourceLocation - Return a source location identifier for the specified 1062/// offset in the current file. 1063SourceLocation Lexer::getSourceLocation(const char *Loc, 1064 unsigned TokLen) const { 1065 assert(Loc >= BufferStart && Loc <= BufferEnd && 1066 "Location out of range for this buffer!"); 1067 1068 // In the normal case, we're just lexing from a simple file buffer, return 1069 // the file id from FileLoc with the offset specified. 1070 unsigned CharNo = Loc-BufferStart; 1071 if (FileLoc.isFileID()) 1072 return FileLoc.getLocWithOffset(CharNo); 1073 1074 // Otherwise, this is the _Pragma lexer case, which pretends that all of the 1075 // tokens are lexed from where the _Pragma was defined. 1076 assert(PP && "This doesn't work on raw lexers"); 1077 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 1078} 1079 1080/// Diag - Forwarding function for diagnostics. This translate a source 1081/// position in the current buffer into a SourceLocation object for rendering. 1082DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 1083 return PP->Diag(getSourceLocation(Loc), DiagID); 1084} 1085 1086//===----------------------------------------------------------------------===// 1087// Trigraph and Escaped Newline Handling Code. 1088//===----------------------------------------------------------------------===// 1089 1090/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 1091/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 1092static char GetTrigraphCharForLetter(char Letter) { 1093 switch (Letter) { 1094 default: return 0; 1095 case '=': return '#'; 1096 case ')': return ']'; 1097 case '(': return '['; 1098 case '!': return '|'; 1099 case '\'': return '^'; 1100 case '>': return '}'; 1101 case '/': return '\\'; 1102 case '<': return '{'; 1103 case '-': return '~'; 1104 } 1105} 1106 1107/// DecodeTrigraphChar - If the specified character is a legal trigraph when 1108/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 1109/// return the result character. Finally, emit a warning about trigraph use 1110/// whether trigraphs are enabled or not. 1111static char DecodeTrigraphChar(const char *CP, Lexer *L) { 1112 char Res = GetTrigraphCharForLetter(*CP); 1113 if (!Res || !L) return Res; 1114 1115 if (!L->getLangOpts().Trigraphs) { 1116 if (!L->isLexingRawMode()) 1117 L->Diag(CP-2, diag::trigraph_ignored); 1118 return 0; 1119 } 1120 1121 if (!L->isLexingRawMode()) 1122 L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 1123 return Res; 1124} 1125 1126/// getEscapedNewLineSize - Return the size of the specified escaped newline, 1127/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 1128/// trigraph equivalent on entry to this function. 1129unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 1130 unsigned Size = 0; 1131 while (isWhitespace(Ptr[Size])) { 1132 ++Size; 1133 1134 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 1135 continue; 1136 1137 // If this is a \r\n or \n\r, skip the other half. 1138 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 1139 Ptr[Size-1] != Ptr[Size]) 1140 ++Size; 1141 1142 return Size; 1143 } 1144 1145 // Not an escaped newline, must be a \t or something else. 1146 return 0; 1147} 1148 1149/// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1150/// them), skip over them and return the first non-escaped-newline found, 1151/// otherwise return P. 1152const char *Lexer::SkipEscapedNewLines(const char *P) { 1153 while (1) { 1154 const char *AfterEscape; 1155 if (*P == '\\') { 1156 AfterEscape = P+1; 1157 } else if (*P == '?') { 1158 // If not a trigraph for escape, bail out. 1159 if (P[1] != '?' || P[2] != '/') 1160 return P; 1161 AfterEscape = P+3; 1162 } else { 1163 return P; 1164 } 1165 1166 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1167 if (NewLineSize == 0) return P; 1168 P = AfterEscape+NewLineSize; 1169 } 1170} 1171 1172/// \brief Checks that the given token is the first token that occurs after the 1173/// given location (this excludes comments and whitespace). Returns the location 1174/// immediately after the specified token. If the token is not found or the 1175/// location is inside a macro, the returned source location will be invalid. 1176SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1177 tok::TokenKind TKind, 1178 const SourceManager &SM, 1179 const LangOptions &LangOpts, 1180 bool SkipTrailingWhitespaceAndNewLine) { 1181 if (Loc.isMacroID()) { 1182 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 1183 return SourceLocation(); 1184 } 1185 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1186 1187 // Break down the source location. 1188 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1189 1190 // Try to load the file buffer. 1191 bool InvalidTemp = false; 1192 StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1193 if (InvalidTemp) 1194 return SourceLocation(); 1195 1196 const char *TokenBegin = File.data() + LocInfo.second; 1197 1198 // Lex from the start of the given location. 1199 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1200 TokenBegin, File.end()); 1201 // Find the token. 1202 Token Tok; 1203 lexer.LexFromRawLexer(Tok); 1204 if (Tok.isNot(TKind)) 1205 return SourceLocation(); 1206 SourceLocation TokenLoc = Tok.getLocation(); 1207 1208 // Calculate how much whitespace needs to be skipped if any. 1209 unsigned NumWhitespaceChars = 0; 1210 if (SkipTrailingWhitespaceAndNewLine) { 1211 const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1212 Tok.getLength(); 1213 unsigned char C = *TokenEnd; 1214 while (isHorizontalWhitespace(C)) { 1215 C = *(++TokenEnd); 1216 NumWhitespaceChars++; 1217 } 1218 1219 // Skip \r, \n, \r\n, or \n\r 1220 if (C == '\n' || C == '\r') { 1221 char PrevC = C; 1222 C = *(++TokenEnd); 1223 NumWhitespaceChars++; 1224 if ((C == '\n' || C == '\r') && C != PrevC) 1225 NumWhitespaceChars++; 1226 } 1227 } 1228 1229 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1230} 1231 1232/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 1233/// get its size, and return it. This is tricky in several cases: 1234/// 1. If currently at the start of a trigraph, we warn about the trigraph, 1235/// then either return the trigraph (skipping 3 chars) or the '?', 1236/// depending on whether trigraphs are enabled or not. 1237/// 2. If this is an escaped newline (potentially with whitespace between 1238/// the backslash and newline), implicitly skip the newline and return 1239/// the char after it. 1240/// 1241/// This handles the slow/uncommon case of the getCharAndSize method. Here we 1242/// know that we can accumulate into Size, and that we have already incremented 1243/// Ptr by Size bytes. 1244/// 1245/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 1246/// be updated to match. 1247/// 1248char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1249 Token *Tok) { 1250 // If we have a slash, look for an escaped newline. 1251 if (Ptr[0] == '\\') { 1252 ++Size; 1253 ++Ptr; 1254Slash: 1255 // Common case, backslash-char where the char is not whitespace. 1256 if (!isWhitespace(Ptr[0])) return '\\'; 1257 1258 // See if we have optional whitespace characters between the slash and 1259 // newline. 1260 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1261 // Remember that this token needs to be cleaned. 1262 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1263 1264 // Warn if there was whitespace between the backslash and newline. 1265 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 1266 Diag(Ptr, diag::backslash_newline_space); 1267 1268 // Found backslash<whitespace><newline>. Parse the char after it. 1269 Size += EscapedNewLineSize; 1270 Ptr += EscapedNewLineSize; 1271 1272 // If the char that we finally got was a \n, then we must have had 1273 // something like \<newline><newline>. We don't want to consume the 1274 // second newline. 1275 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1276 return ' '; 1277 1278 // Use slow version to accumulate a correct size field. 1279 return getCharAndSizeSlow(Ptr, Size, Tok); 1280 } 1281 1282 // Otherwise, this is not an escaped newline, just return the slash. 1283 return '\\'; 1284 } 1285 1286 // If this is a trigraph, process it. 1287 if (Ptr[0] == '?' && Ptr[1] == '?') { 1288 // If this is actually a legal trigraph (not something like "??x"), emit 1289 // a trigraph warning. If so, and if trigraphs are enabled, return it. 1290 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { 1291 // Remember that this token needs to be cleaned. 1292 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1293 1294 Ptr += 3; 1295 Size += 3; 1296 if (C == '\\') goto Slash; 1297 return C; 1298 } 1299 } 1300 1301 // If this is neither, return a single character. 1302 ++Size; 1303 return *Ptr; 1304} 1305 1306 1307/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 1308/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 1309/// and that we have already incremented Ptr by Size bytes. 1310/// 1311/// NOTE: When this method is updated, getCharAndSizeSlow (above) should 1312/// be updated to match. 1313char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 1314 const LangOptions &LangOpts) { 1315 // If we have a slash, look for an escaped newline. 1316 if (Ptr[0] == '\\') { 1317 ++Size; 1318 ++Ptr; 1319Slash: 1320 // Common case, backslash-char where the char is not whitespace. 1321 if (!isWhitespace(Ptr[0])) return '\\'; 1322 1323 // See if we have optional whitespace characters followed by a newline. 1324 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1325 // Found backslash<whitespace><newline>. Parse the char after it. 1326 Size += EscapedNewLineSize; 1327 Ptr += EscapedNewLineSize; 1328 1329 // If the char that we finally got was a \n, then we must have had 1330 // something like \<newline><newline>. We don't want to consume the 1331 // second newline. 1332 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1333 return ' '; 1334 1335 // Use slow version to accumulate a correct size field. 1336 return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); 1337 } 1338 1339 // Otherwise, this is not an escaped newline, just return the slash. 1340 return '\\'; 1341 } 1342 1343 // If this is a trigraph, process it. 1344 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 1345 // If this is actually a legal trigraph (not something like "??x"), return 1346 // it. 1347 if (char C = GetTrigraphCharForLetter(Ptr[2])) { 1348 Ptr += 3; 1349 Size += 3; 1350 if (C == '\\') goto Slash; 1351 return C; 1352 } 1353 } 1354 1355 // If this is neither, return a single character. 1356 ++Size; 1357 return *Ptr; 1358} 1359 1360//===----------------------------------------------------------------------===// 1361// Helper methods for lexing. 1362//===----------------------------------------------------------------------===// 1363 1364/// \brief Routine that indiscriminately skips bytes in the source file. 1365void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1366 BufferPtr += Bytes; 1367 if (BufferPtr > BufferEnd) 1368 BufferPtr = BufferEnd; 1369 // FIXME: What exactly does the StartOfLine bit mean? There are two 1370 // possible meanings for the "start" of the line: the first token on the 1371 // unexpanded line, or the first token on the expanded line. 1372 IsAtStartOfLine = StartOfLine; 1373 IsAtPhysicalStartOfLine = StartOfLine; 1374} 1375 1376static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { 1377 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 1378 static const llvm::sys::UnicodeCharSet C11AllowedIDChars( 1379 C11AllowedIDCharRanges); 1380 return C11AllowedIDChars.contains(C); 1381 } else if (LangOpts.CPlusPlus) { 1382 static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( 1383 CXX03AllowedIDCharRanges); 1384 return CXX03AllowedIDChars.contains(C); 1385 } else { 1386 static const llvm::sys::UnicodeCharSet C99AllowedIDChars( 1387 C99AllowedIDCharRanges); 1388 return C99AllowedIDChars.contains(C); 1389 } 1390} 1391 1392static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { 1393 assert(isAllowedIDChar(C, LangOpts)); 1394 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 1395 static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars( 1396 C11DisallowedInitialIDCharRanges); 1397 return !C11DisallowedInitialIDChars.contains(C); 1398 } else if (LangOpts.CPlusPlus) { 1399 return true; 1400 } else { 1401 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( 1402 C99DisallowedInitialIDCharRanges); 1403 return !C99DisallowedInitialIDChars.contains(C); 1404 } 1405} 1406 1407static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, 1408 const char *End) { 1409 return CharSourceRange::getCharRange(L.getSourceLocation(Begin), 1410 L.getSourceLocation(End)); 1411} 1412 1413static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, 1414 CharSourceRange Range, bool IsFirst) { 1415 // Check C99 compatibility. 1416 if (Diags.getDiagnosticLevel(diag::warn_c99_compat_unicode_id, 1417 Range.getBegin()) > DiagnosticsEngine::Ignored) { 1418 enum { 1419 CannotAppearInIdentifier = 0, 1420 CannotStartIdentifier 1421 }; 1422 1423 static const llvm::sys::UnicodeCharSet C99AllowedIDChars( 1424 C99AllowedIDCharRanges); 1425 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( 1426 C99DisallowedInitialIDCharRanges); 1427 if (!C99AllowedIDChars.contains(C)) { 1428 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1429 << Range 1430 << CannotAppearInIdentifier; 1431 } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) { 1432 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1433 << Range 1434 << CannotStartIdentifier; 1435 } 1436 } 1437 1438 // Check C++98 compatibility. 1439 if (Diags.getDiagnosticLevel(diag::warn_cxx98_compat_unicode_id, 1440 Range.getBegin()) > DiagnosticsEngine::Ignored) { 1441 static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( 1442 CXX03AllowedIDCharRanges); 1443 if (!CXX03AllowedIDChars.contains(C)) { 1444 Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) 1445 << Range; 1446 } 1447 } 1448 } 1449 1450bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 1451 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 1452 unsigned Size; 1453 unsigned char C = *CurPtr++; 1454 while (isIdentifierBody(C)) 1455 C = *CurPtr++; 1456 1457 --CurPtr; // Back up over the skipped character. 1458 1459 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 1460 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 1461 // 1462 // TODO: Could merge these checks into an InfoTable flag to make the 1463 // comparison cheaper 1464 if (isASCII(C) && C != '\\' && C != '?' && 1465 (C != '$' || !LangOpts.DollarIdents)) { 1466FinishIdentifier: 1467 const char *IdStart = BufferPtr; 1468 FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1469 Result.setRawIdentifierData(IdStart); 1470 1471 // If we are in raw mode, return this identifier raw. There is no need to 1472 // look up identifier information or attempt to macro expand it. 1473 if (LexingRawMode) 1474 return true; 1475 1476 // Fill in Result.IdentifierInfo and update the token kind, 1477 // looking up the identifier in the identifier table. 1478 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 1479 1480 // Finally, now that we know we have an identifier, pass this off to the 1481 // preprocessor, which may macro expand it or something. 1482 if (II->isHandleIdentifierCase()) 1483 return PP->HandleIdentifier(Result); 1484 1485 return true; 1486 } 1487 1488 // Otherwise, $,\,? in identifier found. Enter slower path. 1489 1490 C = getCharAndSize(CurPtr, Size); 1491 while (1) { 1492 if (C == '$') { 1493 // If we hit a $ and they are not supported in identifiers, we are done. 1494 if (!LangOpts.DollarIdents) goto FinishIdentifier; 1495 1496 // Otherwise, emit a diagnostic and continue. 1497 if (!isLexingRawMode()) 1498 Diag(CurPtr, diag::ext_dollar_in_identifier); 1499 CurPtr = ConsumeChar(CurPtr, Size, Result); 1500 C = getCharAndSize(CurPtr, Size); 1501 continue; 1502 1503 } else if (C == '\\') { 1504 const char *UCNPtr = CurPtr + Size; 1505 uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/0); 1506 if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) 1507 goto FinishIdentifier; 1508 1509 if (!isLexingRawMode()) { 1510 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1511 makeCharRange(*this, CurPtr, UCNPtr), 1512 /*IsFirst=*/false); 1513 } 1514 1515 Result.setFlag(Token::HasUCN); 1516 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || 1517 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) 1518 CurPtr = UCNPtr; 1519 else 1520 while (CurPtr != UCNPtr) 1521 (void)getAndAdvanceChar(CurPtr, Result); 1522 1523 C = getCharAndSize(CurPtr, Size); 1524 continue; 1525 } else if (!isASCII(C)) { 1526 const char *UnicodePtr = CurPtr; 1527 UTF32 CodePoint; 1528 ConversionResult Result = 1529 llvm::convertUTF8Sequence((const UTF8 **)&UnicodePtr, 1530 (const UTF8 *)BufferEnd, 1531 &CodePoint, 1532 strictConversion); 1533 if (Result != conversionOK || 1534 !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) 1535 goto FinishIdentifier; 1536 1537 if (!isLexingRawMode()) { 1538 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1539 makeCharRange(*this, CurPtr, UnicodePtr), 1540 /*IsFirst=*/false); 1541 } 1542 1543 CurPtr = UnicodePtr; 1544 C = getCharAndSize(CurPtr, Size); 1545 continue; 1546 } else if (!isIdentifierBody(C)) { 1547 goto FinishIdentifier; 1548 } 1549 1550 // Otherwise, this character is good, consume it. 1551 CurPtr = ConsumeChar(CurPtr, Size, Result); 1552 1553 C = getCharAndSize(CurPtr, Size); 1554 while (isIdentifierBody(C)) { 1555 CurPtr = ConsumeChar(CurPtr, Size, Result); 1556 C = getCharAndSize(CurPtr, Size); 1557 } 1558 } 1559} 1560 1561/// isHexaLiteral - Return true if Start points to a hex constant. 1562/// in microsoft mode (where this is supposed to be several different tokens). 1563bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { 1564 unsigned Size; 1565 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); 1566 if (C1 != '0') 1567 return false; 1568 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); 1569 return (C2 == 'x' || C2 == 'X'); 1570} 1571 1572/// LexNumericConstant - Lex the remainder of a integer or floating point 1573/// constant. From[-1] is the first character lexed. Return the end of the 1574/// constant. 1575bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 1576 unsigned Size; 1577 char C = getCharAndSize(CurPtr, Size); 1578 char PrevCh = 0; 1579 while (isPreprocessingNumberBody(C)) { // FIXME: UCNs in ud-suffix. 1580 CurPtr = ConsumeChar(CurPtr, Size, Result); 1581 PrevCh = C; 1582 C = getCharAndSize(CurPtr, Size); 1583 } 1584 1585 // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1586 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1587 // If we are in Microsoft mode, don't continue if the constant is hex. 1588 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 1589 if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) 1590 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1591 } 1592 1593 // If we have a hex FP constant, continue. 1594 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { 1595 // Outside C99, we accept hexadecimal floating point numbers as a 1596 // not-quite-conforming extension. Only do so if this looks like it's 1597 // actually meant to be a hexfloat, and not if it has a ud-suffix. 1598 bool IsHexFloat = true; 1599 if (!LangOpts.C99) { 1600 if (!isHexaLiteral(BufferPtr, LangOpts)) 1601 IsHexFloat = false; 1602 else if (std::find(BufferPtr, CurPtr, '_') != CurPtr) 1603 IsHexFloat = false; 1604 } 1605 if (IsHexFloat) 1606 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1607 } 1608 1609 // If we have a digit separator, continue. 1610 if (C == '\'' && getLangOpts().CPlusPlus1y) { 1611 unsigned NextSize; 1612 char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts()); 1613 if (isIdentifierBody(Next)) { 1614 if (!isLexingRawMode()) 1615 Diag(CurPtr, diag::warn_cxx11_compat_digit_separator); 1616 CurPtr = ConsumeChar(CurPtr, Size, Result); 1617 return LexNumericConstant(Result, CurPtr); 1618 } 1619 } 1620 1621 // Update the location of token as well as BufferPtr. 1622 const char *TokStart = BufferPtr; 1623 FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 1624 Result.setLiteralData(TokStart); 1625 return true; 1626} 1627 1628/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes 1629/// in C++11, or warn on a ud-suffix in C++98. 1630const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr, 1631 bool IsStringLiteral) { 1632 assert(getLangOpts().CPlusPlus); 1633 1634 // Maximally munch an identifier. FIXME: UCNs. 1635 unsigned Size; 1636 char C = getCharAndSize(CurPtr, Size); 1637 if (isIdentifierHead(C)) { 1638 if (!getLangOpts().CPlusPlus11) { 1639 if (!isLexingRawMode()) 1640 Diag(CurPtr, 1641 C == '_' ? diag::warn_cxx11_compat_user_defined_literal 1642 : diag::warn_cxx11_compat_reserved_user_defined_literal) 1643 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1644 return CurPtr; 1645 } 1646 1647 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix 1648 // that does not start with an underscore is ill-formed. As a conforming 1649 // extension, we treat all such suffixes as if they had whitespace before 1650 // them. 1651 bool IsUDSuffix = false; 1652 if (C == '_') 1653 IsUDSuffix = true; 1654 else if (IsStringLiteral && getLangOpts().CPlusPlus1y) { 1655 // In C++1y, we need to look ahead a few characters to see if this is a 1656 // valid suffix for a string literal or a numeric literal (this could be 1657 // the 'operator""if' defining a numeric literal operator). 1658 const unsigned MaxStandardSuffixLength = 3; 1659 char Buffer[MaxStandardSuffixLength] = { C }; 1660 unsigned Consumed = Size; 1661 unsigned Chars = 1; 1662 while (true) { 1663 unsigned NextSize; 1664 char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize, 1665 getLangOpts()); 1666 if (!isIdentifierBody(Next)) { 1667 // End of suffix. Check whether this is on the whitelist. 1668 IsUDSuffix = (Chars == 1 && Buffer[0] == 's') || 1669 NumericLiteralParser::isValidUDSuffix( 1670 getLangOpts(), StringRef(Buffer, Chars)); 1671 break; 1672 } 1673 1674 if (Chars == MaxStandardSuffixLength) 1675 // Too long: can't be a standard suffix. 1676 break; 1677 1678 Buffer[Chars++] = Next; 1679 Consumed += NextSize; 1680 } 1681 } 1682 1683 if (!IsUDSuffix) { 1684 if (!isLexingRawMode()) 1685 Diag(CurPtr, getLangOpts().MicrosoftMode ? 1686 diag::ext_ms_reserved_user_defined_literal : 1687 diag::ext_reserved_user_defined_literal) 1688 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1689 return CurPtr; 1690 } 1691 1692 Result.setFlag(Token::HasUDSuffix); 1693 do { 1694 CurPtr = ConsumeChar(CurPtr, Size, Result); 1695 C = getCharAndSize(CurPtr, Size); 1696 } while (isIdentifierBody(C)); 1697 } 1698 return CurPtr; 1699} 1700 1701/// LexStringLiteral - Lex the remainder of a string literal, after having lexed 1702/// either " or L" or u8" or u" or U". 1703bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 1704 tok::TokenKind Kind) { 1705 const char *NulCharacter = 0; // Does this string contain the \0 character? 1706 1707 if (!isLexingRawMode() && 1708 (Kind == tok::utf8_string_literal || 1709 Kind == tok::utf16_string_literal || 1710 Kind == tok::utf32_string_literal)) 1711 Diag(BufferPtr, getLangOpts().CPlusPlus 1712 ? diag::warn_cxx98_compat_unicode_literal 1713 : diag::warn_c99_compat_unicode_literal); 1714 1715 char C = getAndAdvanceChar(CurPtr, Result); 1716 while (C != '"') { 1717 // Skip escaped characters. Escaped newlines will already be processed by 1718 // getAndAdvanceChar. 1719 if (C == '\\') 1720 C = getAndAdvanceChar(CurPtr, Result); 1721 1722 if (C == '\n' || C == '\r' || // Newline. 1723 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1724 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1725 Diag(BufferPtr, diag::ext_unterminated_string); 1726 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1727 return true; 1728 } 1729 1730 if (C == 0) { 1731 if (isCodeCompletionPoint(CurPtr-1)) { 1732 PP->CodeCompleteNaturalLanguage(); 1733 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1734 cutOffLexing(); 1735 return true; 1736 } 1737 1738 NulCharacter = CurPtr-1; 1739 } 1740 C = getAndAdvanceChar(CurPtr, Result); 1741 } 1742 1743 // If we are in C++11, lex the optional ud-suffix. 1744 if (getLangOpts().CPlusPlus) 1745 CurPtr = LexUDSuffix(Result, CurPtr, true); 1746 1747 // If a nul character existed in the string, warn about it. 1748 if (NulCharacter && !isLexingRawMode()) 1749 Diag(NulCharacter, diag::null_in_string); 1750 1751 // Update the location of the token as well as the BufferPtr instance var. 1752 const char *TokStart = BufferPtr; 1753 FormTokenWithChars(Result, CurPtr, Kind); 1754 Result.setLiteralData(TokStart); 1755 return true; 1756} 1757 1758/// LexRawStringLiteral - Lex the remainder of a raw string literal, after 1759/// having lexed R", LR", u8R", uR", or UR". 1760bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 1761 tok::TokenKind Kind) { 1762 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 1763 // Between the initial and final double quote characters of the raw string, 1764 // any transformations performed in phases 1 and 2 (trigraphs, 1765 // universal-character-names, and line splicing) are reverted. 1766 1767 if (!isLexingRawMode()) 1768 Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); 1769 1770 unsigned PrefixLen = 0; 1771 1772 while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 1773 ++PrefixLen; 1774 1775 // If the last character was not a '(', then we didn't lex a valid delimiter. 1776 if (CurPtr[PrefixLen] != '(') { 1777 if (!isLexingRawMode()) { 1778 const char *PrefixEnd = &CurPtr[PrefixLen]; 1779 if (PrefixLen == 16) { 1780 Diag(PrefixEnd, diag::err_raw_delim_too_long); 1781 } else { 1782 Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 1783 << StringRef(PrefixEnd, 1); 1784 } 1785 } 1786 1787 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 1788 // it's possible the '"' was intended to be part of the raw string, but 1789 // there's not much we can do about that. 1790 while (1) { 1791 char C = *CurPtr++; 1792 1793 if (C == '"') 1794 break; 1795 if (C == 0 && CurPtr-1 == BufferEnd) { 1796 --CurPtr; 1797 break; 1798 } 1799 } 1800 1801 FormTokenWithChars(Result, CurPtr, tok::unknown); 1802 return true; 1803 } 1804 1805 // Save prefix and move CurPtr past it 1806 const char *Prefix = CurPtr; 1807 CurPtr += PrefixLen + 1; // skip over prefix and '(' 1808 1809 while (1) { 1810 char C = *CurPtr++; 1811 1812 if (C == ')') { 1813 // Check for prefix match and closing quote. 1814 if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 1815 CurPtr += PrefixLen + 1; // skip over prefix and '"' 1816 break; 1817 } 1818 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 1819 if (!isLexingRawMode()) 1820 Diag(BufferPtr, diag::err_unterminated_raw_string) 1821 << StringRef(Prefix, PrefixLen); 1822 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1823 return true; 1824 } 1825 } 1826 1827 // If we are in C++11, lex the optional ud-suffix. 1828 if (getLangOpts().CPlusPlus) 1829 CurPtr = LexUDSuffix(Result, CurPtr, true); 1830 1831 // Update the location of token as well as BufferPtr. 1832 const char *TokStart = BufferPtr; 1833 FormTokenWithChars(Result, CurPtr, Kind); 1834 Result.setLiteralData(TokStart); 1835 return true; 1836} 1837 1838/// LexAngledStringLiteral - Lex the remainder of an angled string literal, 1839/// after having lexed the '<' character. This is used for #include filenames. 1840bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 1841 const char *NulCharacter = 0; // Does this string contain the \0 character? 1842 const char *AfterLessPos = CurPtr; 1843 char C = getAndAdvanceChar(CurPtr, Result); 1844 while (C != '>') { 1845 // Skip escaped characters. 1846 if (C == '\\') { 1847 // Skip the escaped character. 1848 getAndAdvanceChar(CurPtr, Result); 1849 } else if (C == '\n' || C == '\r' || // Newline. 1850 (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 1851 isCodeCompletionPoint(CurPtr-1)))) { 1852 // If the filename is unterminated, then it must just be a lone < 1853 // character. Return this as such. 1854 FormTokenWithChars(Result, AfterLessPos, tok::less); 1855 return true; 1856 } else if (C == 0) { 1857 NulCharacter = CurPtr-1; 1858 } 1859 C = getAndAdvanceChar(CurPtr, Result); 1860 } 1861 1862 // If a nul character existed in the string, warn about it. 1863 if (NulCharacter && !isLexingRawMode()) 1864 Diag(NulCharacter, diag::null_in_string); 1865 1866 // Update the location of token as well as BufferPtr. 1867 const char *TokStart = BufferPtr; 1868 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 1869 Result.setLiteralData(TokStart); 1870 return true; 1871} 1872 1873 1874/// LexCharConstant - Lex the remainder of a character constant, after having 1875/// lexed either ' or L' or u' or U'. 1876bool Lexer::LexCharConstant(Token &Result, const char *CurPtr, 1877 tok::TokenKind Kind) { 1878 const char *NulCharacter = 0; // Does this character contain the \0 character? 1879 1880 if (!isLexingRawMode() && 1881 (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)) 1882 Diag(BufferPtr, getLangOpts().CPlusPlus 1883 ? diag::warn_cxx98_compat_unicode_literal 1884 : diag::warn_c99_compat_unicode_literal); 1885 1886 char C = getAndAdvanceChar(CurPtr, Result); 1887 if (C == '\'') { 1888 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1889 Diag(BufferPtr, diag::ext_empty_character); 1890 FormTokenWithChars(Result, CurPtr, tok::unknown); 1891 return true; 1892 } 1893 1894 while (C != '\'') { 1895 // Skip escaped characters. 1896 if (C == '\\') 1897 C = getAndAdvanceChar(CurPtr, Result); 1898 1899 if (C == '\n' || C == '\r' || // Newline. 1900 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1901 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1902 Diag(BufferPtr, diag::ext_unterminated_char); 1903 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1904 return true; 1905 } 1906 1907 if (C == 0) { 1908 if (isCodeCompletionPoint(CurPtr-1)) { 1909 PP->CodeCompleteNaturalLanguage(); 1910 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1911 cutOffLexing(); 1912 return true; 1913 } 1914 1915 NulCharacter = CurPtr-1; 1916 } 1917 C = getAndAdvanceChar(CurPtr, Result); 1918 } 1919 1920 // If we are in C++11, lex the optional ud-suffix. 1921 if (getLangOpts().CPlusPlus) 1922 CurPtr = LexUDSuffix(Result, CurPtr, false); 1923 1924 // If a nul character existed in the character, warn about it. 1925 if (NulCharacter && !isLexingRawMode()) 1926 Diag(NulCharacter, diag::null_in_char); 1927 1928 // Update the location of token as well as BufferPtr. 1929 const char *TokStart = BufferPtr; 1930 FormTokenWithChars(Result, CurPtr, Kind); 1931 Result.setLiteralData(TokStart); 1932 return true; 1933} 1934 1935/// SkipWhitespace - Efficiently skip over a series of whitespace characters. 1936/// Update BufferPtr to point to the next non-whitespace character and return. 1937/// 1938/// This method forms a token and returns true if KeepWhitespaceMode is enabled. 1939/// 1940bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr, 1941 bool &TokAtPhysicalStartOfLine) { 1942 // Whitespace - Skip it, then return the token after the whitespace. 1943 bool SawNewline = isVerticalWhitespace(CurPtr[-1]); 1944 1945 unsigned char Char = *CurPtr; 1946 1947 // Skip consecutive spaces efficiently. 1948 while (1) { 1949 // Skip horizontal whitespace very aggressively. 1950 while (isHorizontalWhitespace(Char)) 1951 Char = *++CurPtr; 1952 1953 // Otherwise if we have something other than whitespace, we're done. 1954 if (!isVerticalWhitespace(Char)) 1955 break; 1956 1957 if (ParsingPreprocessorDirective) { 1958 // End of preprocessor directive line, let LexTokenInternal handle this. 1959 BufferPtr = CurPtr; 1960 return false; 1961 } 1962 1963 // OK, but handle newline. 1964 SawNewline = true; 1965 Char = *++CurPtr; 1966 } 1967 1968 // If the client wants us to return whitespace, return it now. 1969 if (isKeepWhitespaceMode()) { 1970 FormTokenWithChars(Result, CurPtr, tok::unknown); 1971 if (SawNewline) { 1972 IsAtStartOfLine = true; 1973 IsAtPhysicalStartOfLine = true; 1974 } 1975 // FIXME: The next token will not have LeadingSpace set. 1976 return true; 1977 } 1978 1979 // If this isn't immediately after a newline, there is leading space. 1980 char PrevChar = CurPtr[-1]; 1981 bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); 1982 1983 Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 1984 if (SawNewline) { 1985 Result.setFlag(Token::StartOfLine); 1986 TokAtPhysicalStartOfLine = true; 1987 } 1988 1989 BufferPtr = CurPtr; 1990 return false; 1991} 1992 1993/// We have just read the // characters from input. Skip until we find the 1994/// newline character thats terminate the comment. Then update BufferPtr and 1995/// return. 1996/// 1997/// If we're in KeepCommentMode or any CommentHandler has inserted 1998/// some tokens, this will store the first token and return true. 1999bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, 2000 bool &TokAtPhysicalStartOfLine) { 2001 // If Line comments aren't explicitly enabled for this language, emit an 2002 // extension warning. 2003 if (!LangOpts.LineComment && !isLexingRawMode()) { 2004 Diag(BufferPtr, diag::ext_line_comment); 2005 2006 // Mark them enabled so we only emit one warning for this translation 2007 // unit. 2008 LangOpts.LineComment = true; 2009 } 2010 2011 // Scan over the body of the comment. The common case, when scanning, is that 2012 // the comment contains normal ascii characters with nothing interesting in 2013 // them. As such, optimize for this case with the inner loop. 2014 char C; 2015 do { 2016 C = *CurPtr; 2017 // Skip over characters in the fast loop. 2018 while (C != 0 && // Potentially EOF. 2019 C != '\n' && C != '\r') // Newline or DOS-style newline. 2020 C = *++CurPtr; 2021 2022 const char *NextLine = CurPtr; 2023 if (C != 0) { 2024 // We found a newline, see if it's escaped. 2025 const char *EscapePtr = CurPtr-1; 2026 while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace. 2027 --EscapePtr; 2028 2029 if (*EscapePtr == '\\') // Escaped newline. 2030 CurPtr = EscapePtr; 2031 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 2032 EscapePtr[-2] == '?') // Trigraph-escaped newline. 2033 CurPtr = EscapePtr-2; 2034 else 2035 break; // This is a newline, we're done. 2036 } 2037 2038 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 2039 // properly decode the character. Read it in raw mode to avoid emitting 2040 // diagnostics about things like trigraphs. If we see an escaped newline, 2041 // we'll handle it below. 2042 const char *OldPtr = CurPtr; 2043 bool OldRawMode = isLexingRawMode(); 2044 LexingRawMode = true; 2045 C = getAndAdvanceChar(CurPtr, Result); 2046 LexingRawMode = OldRawMode; 2047 2048 // If we only read only one character, then no special handling is needed. 2049 // We're done and can skip forward to the newline. 2050 if (C != 0 && CurPtr == OldPtr+1) { 2051 CurPtr = NextLine; 2052 break; 2053 } 2054 2055 // If we read multiple characters, and one of those characters was a \r or 2056 // \n, then we had an escaped newline within the comment. Emit diagnostic 2057 // unless the next line is also a // comment. 2058 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 2059 for (; OldPtr != CurPtr; ++OldPtr) 2060 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 2061 // Okay, we found a // comment that ends in a newline, if the next 2062 // line is also a // comment, but has spaces, don't emit a diagnostic. 2063 if (isWhitespace(C)) { 2064 const char *ForwardPtr = CurPtr; 2065 while (isWhitespace(*ForwardPtr)) // Skip whitespace. 2066 ++ForwardPtr; 2067 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 2068 break; 2069 } 2070 2071 if (!isLexingRawMode()) 2072 Diag(OldPtr-1, diag::ext_multi_line_line_comment); 2073 break; 2074 } 2075 } 2076 2077 if (CurPtr == BufferEnd+1) { 2078 --CurPtr; 2079 break; 2080 } 2081 2082 if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2083 PP->CodeCompleteNaturalLanguage(); 2084 cutOffLexing(); 2085 return false; 2086 } 2087 2088 } while (C != '\n' && C != '\r'); 2089 2090 // Found but did not consume the newline. Notify comment handlers about the 2091 // comment unless we're in a #if 0 block. 2092 if (PP && !isLexingRawMode() && 2093 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2094 getSourceLocation(CurPtr)))) { 2095 BufferPtr = CurPtr; 2096 return true; // A token has to be returned. 2097 } 2098 2099 // If we are returning comments as tokens, return this comment as a token. 2100 if (inKeepCommentMode()) 2101 return SaveLineComment(Result, CurPtr); 2102 2103 // If we are inside a preprocessor directive and we see the end of line, 2104 // return immediately, so that the lexer can return this as an EOD token. 2105 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 2106 BufferPtr = CurPtr; 2107 return false; 2108 } 2109 2110 // Otherwise, eat the \n character. We don't care if this is a \n\r or 2111 // \r\n sequence. This is an efficiency hack (because we know the \n can't 2112 // contribute to another token), it isn't needed for correctness. Note that 2113 // this is ok even in KeepWhitespaceMode, because we would have returned the 2114 /// comment above in that mode. 2115 ++CurPtr; 2116 2117 // The next returned token is at the start of the line. 2118 Result.setFlag(Token::StartOfLine); 2119 TokAtPhysicalStartOfLine = true; 2120 // No leading whitespace seen so far. 2121 Result.clearFlag(Token::LeadingSpace); 2122 BufferPtr = CurPtr; 2123 return false; 2124} 2125 2126/// If in save-comment mode, package up this Line comment in an appropriate 2127/// way and return it. 2128bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { 2129 // If we're not in a preprocessor directive, just return the // comment 2130 // directly. 2131 FormTokenWithChars(Result, CurPtr, tok::comment); 2132 2133 if (!ParsingPreprocessorDirective || LexingRawMode) 2134 return true; 2135 2136 // If this Line-style comment is in a macro definition, transmogrify it into 2137 // a C-style block comment. 2138 bool Invalid = false; 2139 std::string Spelling = PP->getSpelling(Result, &Invalid); 2140 if (Invalid) 2141 return true; 2142 2143 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); 2144 Spelling[1] = '*'; // Change prefix to "/*". 2145 Spelling += "*/"; // add suffix. 2146 2147 Result.setKind(tok::comment); 2148 PP->CreateString(Spelling, Result, 2149 Result.getLocation(), Result.getLocation()); 2150 return true; 2151} 2152 2153/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 2154/// character (either \\n or \\r) is part of an escaped newline sequence. Issue 2155/// a diagnostic if so. We know that the newline is inside of a block comment. 2156static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 2157 Lexer *L) { 2158 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 2159 2160 // Back up off the newline. 2161 --CurPtr; 2162 2163 // If this is a two-character newline sequence, skip the other character. 2164 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 2165 // \n\n or \r\r -> not escaped newline. 2166 if (CurPtr[0] == CurPtr[1]) 2167 return false; 2168 // \n\r or \r\n -> skip the newline. 2169 --CurPtr; 2170 } 2171 2172 // If we have horizontal whitespace, skip over it. We allow whitespace 2173 // between the slash and newline. 2174 bool HasSpace = false; 2175 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 2176 --CurPtr; 2177 HasSpace = true; 2178 } 2179 2180 // If we have a slash, we know this is an escaped newline. 2181 if (*CurPtr == '\\') { 2182 if (CurPtr[-1] != '*') return false; 2183 } else { 2184 // It isn't a slash, is it the ?? / trigraph? 2185 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 2186 CurPtr[-3] != '*') 2187 return false; 2188 2189 // This is the trigraph ending the comment. Emit a stern warning! 2190 CurPtr -= 2; 2191 2192 // If no trigraphs are enabled, warn that we ignored this trigraph and 2193 // ignore this * character. 2194 if (!L->getLangOpts().Trigraphs) { 2195 if (!L->isLexingRawMode()) 2196 L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 2197 return false; 2198 } 2199 if (!L->isLexingRawMode()) 2200 L->Diag(CurPtr, diag::trigraph_ends_block_comment); 2201 } 2202 2203 // Warn about having an escaped newline between the */ characters. 2204 if (!L->isLexingRawMode()) 2205 L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 2206 2207 // If there was space between the backslash and newline, warn about it. 2208 if (HasSpace && !L->isLexingRawMode()) 2209 L->Diag(CurPtr, diag::backslash_newline_space); 2210 2211 return true; 2212} 2213 2214#ifdef __SSE2__ 2215#include <emmintrin.h> 2216#elif __ALTIVEC__ 2217#include <altivec.h> 2218#undef bool 2219#endif 2220 2221/// We have just read from input the / and * characters that started a comment. 2222/// Read until we find the * and / characters that terminate the comment. 2223/// Note that we don't bother decoding trigraphs or escaped newlines in block 2224/// comments, because they cannot cause the comment to end. The only thing 2225/// that can happen is the comment could end with an escaped newline between 2226/// the terminating * and /. 2227/// 2228/// If we're in KeepCommentMode or any CommentHandler has inserted 2229/// some tokens, this will store the first token and return true. 2230bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr, 2231 bool &TokAtPhysicalStartOfLine) { 2232 // Scan one character past where we should, looking for a '/' character. Once 2233 // we find it, check to see if it was preceded by a *. This common 2234 // optimization helps people who like to put a lot of * characters in their 2235 // comments. 2236 2237 // The first character we get with newlines and trigraphs skipped to handle 2238 // the degenerate /*/ case below correctly if the * has an escaped newline 2239 // after it. 2240 unsigned CharSize; 2241 unsigned char C = getCharAndSize(CurPtr, CharSize); 2242 CurPtr += CharSize; 2243 if (C == 0 && CurPtr == BufferEnd+1) { 2244 if (!isLexingRawMode()) 2245 Diag(BufferPtr, diag::err_unterminated_block_comment); 2246 --CurPtr; 2247 2248 // KeepWhitespaceMode should return this broken comment as a token. Since 2249 // it isn't a well formed comment, just return it as an 'unknown' token. 2250 if (isKeepWhitespaceMode()) { 2251 FormTokenWithChars(Result, CurPtr, tok::unknown); 2252 return true; 2253 } 2254 2255 BufferPtr = CurPtr; 2256 return false; 2257 } 2258 2259 // Check to see if the first character after the '/*' is another /. If so, 2260 // then this slash does not end the block comment, it is part of it. 2261 if (C == '/') 2262 C = *CurPtr++; 2263 2264 while (1) { 2265 // Skip over all non-interesting characters until we find end of buffer or a 2266 // (probably ending) '/' character. 2267 if (CurPtr + 24 < BufferEnd && 2268 // If there is a code-completion point avoid the fast scan because it 2269 // doesn't check for '\0'. 2270 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 2271 // While not aligned to a 16-byte boundary. 2272 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 2273 C = *CurPtr++; 2274 2275 if (C == '/') goto FoundSlash; 2276 2277#ifdef __SSE2__ 2278 __m128i Slashes = _mm_set1_epi8('/'); 2279 while (CurPtr+16 <= BufferEnd) { 2280 int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, 2281 Slashes)); 2282 if (cmp != 0) { 2283 // Adjust the pointer to point directly after the first slash. It's 2284 // not necessary to set C here, it will be overwritten at the end of 2285 // the outer loop. 2286 CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1; 2287 goto FoundSlash; 2288 } 2289 CurPtr += 16; 2290 } 2291#elif __ALTIVEC__ 2292 __vector unsigned char Slashes = { 2293 '/', '/', '/', '/', '/', '/', '/', '/', 2294 '/', '/', '/', '/', '/', '/', '/', '/' 2295 }; 2296 while (CurPtr+16 <= BufferEnd && 2297 !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes)) 2298 CurPtr += 16; 2299#else 2300 // Scan for '/' quickly. Many block comments are very large. 2301 while (CurPtr[0] != '/' && 2302 CurPtr[1] != '/' && 2303 CurPtr[2] != '/' && 2304 CurPtr[3] != '/' && 2305 CurPtr+4 < BufferEnd) { 2306 CurPtr += 4; 2307 } 2308#endif 2309 2310 // It has to be one of the bytes scanned, increment to it and read one. 2311 C = *CurPtr++; 2312 } 2313 2314 // Loop to scan the remainder. 2315 while (C != '/' && C != '\0') 2316 C = *CurPtr++; 2317 2318 if (C == '/') { 2319 FoundSlash: 2320 if (CurPtr[-2] == '*') // We found the final */. We're done! 2321 break; 2322 2323 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 2324 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 2325 // We found the final */, though it had an escaped newline between the 2326 // * and /. We're done! 2327 break; 2328 } 2329 } 2330 if (CurPtr[0] == '*' && CurPtr[1] != '/') { 2331 // If this is a /* inside of the comment, emit a warning. Don't do this 2332 // if this is a /*/, which will end the comment. This misses cases with 2333 // embedded escaped newlines, but oh well. 2334 if (!isLexingRawMode()) 2335 Diag(CurPtr-1, diag::warn_nested_block_comment); 2336 } 2337 } else if (C == 0 && CurPtr == BufferEnd+1) { 2338 if (!isLexingRawMode()) 2339 Diag(BufferPtr, diag::err_unterminated_block_comment); 2340 // Note: the user probably forgot a */. We could continue immediately 2341 // after the /*, but this would involve lexing a lot of what really is the 2342 // comment, which surely would confuse the parser. 2343 --CurPtr; 2344 2345 // KeepWhitespaceMode should return this broken comment as a token. Since 2346 // it isn't a well formed comment, just return it as an 'unknown' token. 2347 if (isKeepWhitespaceMode()) { 2348 FormTokenWithChars(Result, CurPtr, tok::unknown); 2349 return true; 2350 } 2351 2352 BufferPtr = CurPtr; 2353 return false; 2354 } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2355 PP->CodeCompleteNaturalLanguage(); 2356 cutOffLexing(); 2357 return false; 2358 } 2359 2360 C = *CurPtr++; 2361 } 2362 2363 // Notify comment handlers about the comment unless we're in a #if 0 block. 2364 if (PP && !isLexingRawMode() && 2365 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2366 getSourceLocation(CurPtr)))) { 2367 BufferPtr = CurPtr; 2368 return true; // A token has to be returned. 2369 } 2370 2371 // If we are returning comments as tokens, return this comment as a token. 2372 if (inKeepCommentMode()) { 2373 FormTokenWithChars(Result, CurPtr, tok::comment); 2374 return true; 2375 } 2376 2377 // It is common for the tokens immediately after a /**/ comment to be 2378 // whitespace. Instead of going through the big switch, handle it 2379 // efficiently now. This is safe even in KeepWhitespaceMode because we would 2380 // have already returned above with the comment as a token. 2381 if (isHorizontalWhitespace(*CurPtr)) { 2382 SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine); 2383 return false; 2384 } 2385 2386 // Otherwise, just return so that the next character will be lexed as a token. 2387 BufferPtr = CurPtr; 2388 Result.setFlag(Token::LeadingSpace); 2389 return false; 2390} 2391 2392//===----------------------------------------------------------------------===// 2393// Primary Lexing Entry Points 2394//===----------------------------------------------------------------------===// 2395 2396/// ReadToEndOfLine - Read the rest of the current preprocessor line as an 2397/// uninterpreted string. This switches the lexer out of directive mode. 2398void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { 2399 assert(ParsingPreprocessorDirective && ParsingFilename == false && 2400 "Must be in a preprocessing directive!"); 2401 Token Tmp; 2402 2403 // CurPtr - Cache BufferPtr in an automatic variable. 2404 const char *CurPtr = BufferPtr; 2405 while (1) { 2406 char Char = getAndAdvanceChar(CurPtr, Tmp); 2407 switch (Char) { 2408 default: 2409 if (Result) 2410 Result->push_back(Char); 2411 break; 2412 case 0: // Null. 2413 // Found end of file? 2414 if (CurPtr-1 != BufferEnd) { 2415 if (isCodeCompletionPoint(CurPtr-1)) { 2416 PP->CodeCompleteNaturalLanguage(); 2417 cutOffLexing(); 2418 return; 2419 } 2420 2421 // Nope, normal character, continue. 2422 if (Result) 2423 Result->push_back(Char); 2424 break; 2425 } 2426 // FALL THROUGH. 2427 case '\r': 2428 case '\n': 2429 // Okay, we found the end of the line. First, back up past the \0, \r, \n. 2430 assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 2431 BufferPtr = CurPtr-1; 2432 2433 // Next, lex the character, which should handle the EOD transition. 2434 Lex(Tmp); 2435 if (Tmp.is(tok::code_completion)) { 2436 if (PP) 2437 PP->CodeCompleteNaturalLanguage(); 2438 Lex(Tmp); 2439 } 2440 assert(Tmp.is(tok::eod) && "Unexpected token!"); 2441 2442 // Finally, we're done; 2443 return; 2444 } 2445 } 2446} 2447 2448/// LexEndOfFile - CurPtr points to the end of this file. Handle this 2449/// condition, reporting diagnostics and handling other edge cases as required. 2450/// This returns true if Result contains a token, false if PP.Lex should be 2451/// called again. 2452bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 2453 // If we hit the end of the file while parsing a preprocessor directive, 2454 // end the preprocessor directive first. The next token returned will 2455 // then be the end of file. 2456 if (ParsingPreprocessorDirective) { 2457 // Done parsing the "line". 2458 ParsingPreprocessorDirective = false; 2459 // Update the location of token as well as BufferPtr. 2460 FormTokenWithChars(Result, CurPtr, tok::eod); 2461 2462 // Restore comment saving mode, in case it was disabled for directive. 2463 resetExtendedTokenMode(); 2464 return true; // Have a token. 2465 } 2466 2467 // If we are in raw mode, return this event as an EOF token. Let the caller 2468 // that put us in raw mode handle the event. 2469 if (isLexingRawMode()) { 2470 Result.startToken(); 2471 BufferPtr = BufferEnd; 2472 FormTokenWithChars(Result, BufferEnd, tok::eof); 2473 return true; 2474 } 2475 2476 // Issue diagnostics for unterminated #if and missing newline. 2477 2478 // If we are in a #if directive, emit an error. 2479 while (!ConditionalStack.empty()) { 2480 if (PP->getCodeCompletionFileLoc() != FileLoc) 2481 PP->Diag(ConditionalStack.back().IfLoc, 2482 diag::err_pp_unterminated_conditional); 2483 ConditionalStack.pop_back(); 2484 } 2485 2486 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2487 // a pedwarn. 2488 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) { 2489 DiagnosticsEngine &Diags = PP->getDiagnostics(); 2490 SourceLocation EndLoc = getSourceLocation(BufferEnd); 2491 unsigned DiagID; 2492 2493 if (LangOpts.CPlusPlus11) { 2494 // C++11 [lex.phases] 2.2 p2 2495 // Prefer the C++98 pedantic compatibility warning over the generic, 2496 // non-extension, user-requested "missing newline at EOF" warning. 2497 if (Diags.getDiagnosticLevel(diag::warn_cxx98_compat_no_newline_eof, 2498 EndLoc) != DiagnosticsEngine::Ignored) { 2499 DiagID = diag::warn_cxx98_compat_no_newline_eof; 2500 } else { 2501 DiagID = diag::warn_no_newline_eof; 2502 } 2503 } else { 2504 DiagID = diag::ext_no_newline_eof; 2505 } 2506 2507 Diag(BufferEnd, DiagID) 2508 << FixItHint::CreateInsertion(EndLoc, "\n"); 2509 } 2510 2511 BufferPtr = CurPtr; 2512 2513 // Finally, let the preprocessor handle this. 2514 return PP->HandleEndOfFile(Result, isPragmaLexer()); 2515} 2516 2517/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 2518/// the specified lexer will return a tok::l_paren token, 0 if it is something 2519/// else and 2 if there are no more tokens in the buffer controlled by the 2520/// lexer. 2521unsigned Lexer::isNextPPTokenLParen() { 2522 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 2523 2524 // Switch to 'skipping' mode. This will ensure that we can lex a token 2525 // without emitting diagnostics, disables macro expansion, and will cause EOF 2526 // to return an EOF token instead of popping the include stack. 2527 LexingRawMode = true; 2528 2529 // Save state that can be changed while lexing so that we can restore it. 2530 const char *TmpBufferPtr = BufferPtr; 2531 bool inPPDirectiveMode = ParsingPreprocessorDirective; 2532 bool atStartOfLine = IsAtStartOfLine; 2533 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; 2534 bool leadingSpace = HasLeadingSpace; 2535 2536 Token Tok; 2537 Lex(Tok); 2538 2539 // Restore state that may have changed. 2540 BufferPtr = TmpBufferPtr; 2541 ParsingPreprocessorDirective = inPPDirectiveMode; 2542 HasLeadingSpace = leadingSpace; 2543 IsAtStartOfLine = atStartOfLine; 2544 IsAtPhysicalStartOfLine = atPhysicalStartOfLine; 2545 2546 // Restore the lexer back to non-skipping mode. 2547 LexingRawMode = false; 2548 2549 if (Tok.is(tok::eof)) 2550 return 2; 2551 return Tok.is(tok::l_paren); 2552} 2553 2554/// \brief Find the end of a version control conflict marker. 2555static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, 2556 ConflictMarkerKind CMK) { 2557 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; 2558 size_t TermLen = CMK == CMK_Perforce ? 5 : 7; 2559 StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen); 2560 size_t Pos = RestOfBuffer.find(Terminator); 2561 while (Pos != StringRef::npos) { 2562 // Must occur at start of line. 2563 if (RestOfBuffer[Pos-1] != '\r' && 2564 RestOfBuffer[Pos-1] != '\n') { 2565 RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); 2566 Pos = RestOfBuffer.find(Terminator); 2567 continue; 2568 } 2569 return RestOfBuffer.data()+Pos; 2570 } 2571 return 0; 2572} 2573 2574/// IsStartOfConflictMarker - If the specified pointer is the start of a version 2575/// control conflict marker like '<<<<<<<', recognize it as such, emit an error 2576/// and recover nicely. This returns true if it is a conflict marker and false 2577/// if not. 2578bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 2579 // Only a conflict marker if it starts at the beginning of a line. 2580 if (CurPtr != BufferStart && 2581 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2582 return false; 2583 2584 // Check to see if we have <<<<<<< or >>>>. 2585 if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") && 2586 (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> ")) 2587 return false; 2588 2589 // If we have a situation where we don't care about conflict markers, ignore 2590 // it. 2591 if (CurrentConflictMarkerState || isLexingRawMode()) 2592 return false; 2593 2594 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; 2595 2596 // Check to see if there is an ending marker somewhere in the buffer at the 2597 // start of a line to terminate this conflict marker. 2598 if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { 2599 // We found a match. We are really in a conflict marker. 2600 // Diagnose this, and ignore to the end of line. 2601 Diag(CurPtr, diag::err_conflict_marker); 2602 CurrentConflictMarkerState = Kind; 2603 2604 // Skip ahead to the end of line. We know this exists because the 2605 // end-of-conflict marker starts with \r or \n. 2606 while (*CurPtr != '\r' && *CurPtr != '\n') { 2607 assert(CurPtr != BufferEnd && "Didn't find end of line"); 2608 ++CurPtr; 2609 } 2610 BufferPtr = CurPtr; 2611 return true; 2612 } 2613 2614 // No end of conflict marker found. 2615 return false; 2616} 2617 2618 2619/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if 2620/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it 2621/// is the end of a conflict marker. Handle it by ignoring up until the end of 2622/// the line. This returns true if it is a conflict marker and false if not. 2623bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 2624 // Only a conflict marker if it starts at the beginning of a line. 2625 if (CurPtr != BufferStart && 2626 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2627 return false; 2628 2629 // If we have a situation where we don't care about conflict markers, ignore 2630 // it. 2631 if (!CurrentConflictMarkerState || isLexingRawMode()) 2632 return false; 2633 2634 // Check to see if we have the marker (4 characters in a row). 2635 for (unsigned i = 1; i != 4; ++i) 2636 if (CurPtr[i] != CurPtr[0]) 2637 return false; 2638 2639 // If we do have it, search for the end of the conflict marker. This could 2640 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 2641 // be the end of conflict marker. 2642 if (const char *End = FindConflictEnd(CurPtr, BufferEnd, 2643 CurrentConflictMarkerState)) { 2644 CurPtr = End; 2645 2646 // Skip ahead to the end of line. 2647 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 2648 ++CurPtr; 2649 2650 BufferPtr = CurPtr; 2651 2652 // No longer in the conflict marker. 2653 CurrentConflictMarkerState = CMK_None; 2654 return true; 2655 } 2656 2657 return false; 2658} 2659 2660bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 2661 if (PP && PP->isCodeCompletionEnabled()) { 2662 SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 2663 return Loc == PP->getCodeCompletionLoc(); 2664 } 2665 2666 return false; 2667} 2668 2669uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, 2670 Token *Result) { 2671 unsigned CharSize; 2672 char Kind = getCharAndSize(StartPtr, CharSize); 2673 2674 unsigned NumHexDigits; 2675 if (Kind == 'u') 2676 NumHexDigits = 4; 2677 else if (Kind == 'U') 2678 NumHexDigits = 8; 2679 else 2680 return 0; 2681 2682 if (!LangOpts.CPlusPlus && !LangOpts.C99) { 2683 if (Result && !isLexingRawMode()) 2684 Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); 2685 return 0; 2686 } 2687 2688 const char *CurPtr = StartPtr + CharSize; 2689 const char *KindLoc = &CurPtr[-1]; 2690 2691 uint32_t CodePoint = 0; 2692 for (unsigned i = 0; i < NumHexDigits; ++i) { 2693 char C = getCharAndSize(CurPtr, CharSize); 2694 2695 unsigned Value = llvm::hexDigitValue(C); 2696 if (Value == -1U) { 2697 if (Result && !isLexingRawMode()) { 2698 if (i == 0) { 2699 Diag(BufferPtr, diag::warn_ucn_escape_no_digits) 2700 << StringRef(KindLoc, 1); 2701 } else { 2702 Diag(BufferPtr, diag::warn_ucn_escape_incomplete); 2703 2704 // If the user wrote \U1234, suggest a fixit to \u. 2705 if (i == 4 && NumHexDigits == 8) { 2706 CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); 2707 Diag(KindLoc, diag::note_ucn_four_not_eight) 2708 << FixItHint::CreateReplacement(URange, "u"); 2709 } 2710 } 2711 } 2712 2713 return 0; 2714 } 2715 2716 CodePoint <<= 4; 2717 CodePoint += Value; 2718 2719 CurPtr += CharSize; 2720 } 2721 2722 if (Result) { 2723 Result->setFlag(Token::HasUCN); 2724 if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) 2725 StartPtr = CurPtr; 2726 else 2727 while (StartPtr != CurPtr) 2728 (void)getAndAdvanceChar(StartPtr, *Result); 2729 } else { 2730 StartPtr = CurPtr; 2731 } 2732 2733 // Don't apply C family restrictions to UCNs in assembly mode 2734 if (LangOpts.AsmPreprocessor) 2735 return CodePoint; 2736 2737 // C99 6.4.3p2: A universal character name shall not specify a character whose 2738 // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or 2739 // 0060 (`), nor one in the range D800 through DFFF inclusive.) 2740 // C++11 [lex.charset]p2: If the hexadecimal value for a 2741 // universal-character-name corresponds to a surrogate code point (in the 2742 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, 2743 // if the hexadecimal value for a universal-character-name outside the 2744 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or 2745 // string literal corresponds to a control character (in either of the 2746 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the 2747 // basic source character set, the program is ill-formed. 2748 if (CodePoint < 0xA0) { 2749 if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) 2750 return CodePoint; 2751 2752 // We don't use isLexingRawMode() here because we need to warn about bad 2753 // UCNs even when skipping preprocessing tokens in a #if block. 2754 if (Result && PP) { 2755 if (CodePoint < 0x20 || CodePoint >= 0x7F) 2756 Diag(BufferPtr, diag::err_ucn_control_character); 2757 else { 2758 char C = static_cast<char>(CodePoint); 2759 Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); 2760 } 2761 } 2762 2763 return 0; 2764 2765 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { 2766 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. 2767 // We don't use isLexingRawMode() here because we need to diagnose bad 2768 // UCNs even when skipping preprocessing tokens in a #if block. 2769 if (Result && PP) { 2770 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) 2771 Diag(BufferPtr, diag::warn_ucn_escape_surrogate); 2772 else 2773 Diag(BufferPtr, diag::err_ucn_escape_invalid); 2774 } 2775 return 0; 2776 } 2777 2778 return CodePoint; 2779} 2780 2781bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C, 2782 const char *CurPtr) { 2783 static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars( 2784 UnicodeWhitespaceCharRanges); 2785 if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 2786 UnicodeWhitespaceChars.contains(C)) { 2787 Diag(BufferPtr, diag::ext_unicode_whitespace) 2788 << makeCharRange(*this, BufferPtr, CurPtr); 2789 2790 Result.setFlag(Token::LeadingSpace); 2791 return true; 2792 } 2793 return false; 2794} 2795 2796bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { 2797 if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { 2798 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2799 !PP->isPreprocessedOutput()) { 2800 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, 2801 makeCharRange(*this, BufferPtr, CurPtr), 2802 /*IsFirst=*/true); 2803 } 2804 2805 MIOpt.ReadToken(); 2806 return LexIdentifier(Result, CurPtr); 2807 } 2808 2809 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2810 !PP->isPreprocessedOutput() && 2811 !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { 2812 // Non-ASCII characters tend to creep into source code unintentionally. 2813 // Instead of letting the parser complain about the unknown token, 2814 // just drop the character. 2815 // Note that we can /only/ do this when the non-ASCII character is actually 2816 // spelled as Unicode, not written as a UCN. The standard requires that 2817 // we not throw away any possible preprocessor tokens, but there's a 2818 // loophole in the mapping of Unicode characters to basic character set 2819 // characters that allows us to map these particular characters to, say, 2820 // whitespace. 2821 Diag(BufferPtr, diag::err_non_ascii) 2822 << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); 2823 2824 BufferPtr = CurPtr; 2825 return false; 2826 } 2827 2828 // Otherwise, we have an explicit UCN or a character that's unlikely to show 2829 // up by accident. 2830 MIOpt.ReadToken(); 2831 FormTokenWithChars(Result, CurPtr, tok::unknown); 2832 return true; 2833} 2834 2835void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { 2836 IsAtStartOfLine = Result.isAtStartOfLine(); 2837 HasLeadingSpace = Result.hasLeadingSpace(); 2838 HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro(); 2839 // Note that this doesn't affect IsAtPhysicalStartOfLine. 2840} 2841 2842bool Lexer::Lex(Token &Result) { 2843 // Start a new token. 2844 Result.startToken(); 2845 2846 // Set up misc whitespace flags for LexTokenInternal. 2847 if (IsAtStartOfLine) { 2848 Result.setFlag(Token::StartOfLine); 2849 IsAtStartOfLine = false; 2850 } 2851 2852 if (HasLeadingSpace) { 2853 Result.setFlag(Token::LeadingSpace); 2854 HasLeadingSpace = false; 2855 } 2856 2857 if (HasLeadingEmptyMacro) { 2858 Result.setFlag(Token::LeadingEmptyMacro); 2859 HasLeadingEmptyMacro = false; 2860 } 2861 2862 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; 2863 IsAtPhysicalStartOfLine = false; 2864 bool isRawLex = isLexingRawMode(); 2865 (void) isRawLex; 2866 bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine); 2867 // (After the LexTokenInternal call, the lexer might be destroyed.) 2868 assert((returnedToken || !isRawLex) && "Raw lex must succeed"); 2869 return returnedToken; 2870} 2871 2872/// LexTokenInternal - This implements a simple C family lexer. It is an 2873/// extremely performance critical piece of code. This assumes that the buffer 2874/// has a null character at the end of the file. This returns a preprocessing 2875/// token, not a normal token, as such, it is an internal interface. It assumes 2876/// that the Flags of result have been cleared before calling this. 2877bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) { 2878LexNextToken: 2879 // New token, can't need cleaning yet. 2880 Result.clearFlag(Token::NeedsCleaning); 2881 Result.setIdentifierInfo(0); 2882 2883 // CurPtr - Cache BufferPtr in an automatic variable. 2884 const char *CurPtr = BufferPtr; 2885 2886 // Small amounts of horizontal whitespace is very common between tokens. 2887 if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 2888 ++CurPtr; 2889 while ((*CurPtr == ' ') || (*CurPtr == '\t')) 2890 ++CurPtr; 2891 2892 // If we are keeping whitespace and other tokens, just return what we just 2893 // skipped. The next lexer invocation will return the token after the 2894 // whitespace. 2895 if (isKeepWhitespaceMode()) { 2896 FormTokenWithChars(Result, CurPtr, tok::unknown); 2897 // FIXME: The next token will not have LeadingSpace set. 2898 return true; 2899 } 2900 2901 BufferPtr = CurPtr; 2902 Result.setFlag(Token::LeadingSpace); 2903 } 2904 2905 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 2906 2907 // Read a character, advancing over it. 2908 char Char = getAndAdvanceChar(CurPtr, Result); 2909 tok::TokenKind Kind; 2910 2911 switch (Char) { 2912 case 0: // Null. 2913 // Found end of file? 2914 if (CurPtr-1 == BufferEnd) 2915 return LexEndOfFile(Result, CurPtr-1); 2916 2917 // Check if we are performing code completion. 2918 if (isCodeCompletionPoint(CurPtr-1)) { 2919 // Return the code-completion token. 2920 Result.startToken(); 2921 FormTokenWithChars(Result, CurPtr, tok::code_completion); 2922 return true; 2923 } 2924 2925 if (!isLexingRawMode()) 2926 Diag(CurPtr-1, diag::null_in_file); 2927 Result.setFlag(Token::LeadingSpace); 2928 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 2929 return true; // KeepWhitespaceMode 2930 2931 // We know the lexer hasn't changed, so just try again with this lexer. 2932 // (We manually eliminate the tail call to avoid recursion.) 2933 goto LexNextToken; 2934 2935 case 26: // DOS & CP/M EOF: "^Z". 2936 // If we're in Microsoft extensions mode, treat this as end of file. 2937 if (LangOpts.MicrosoftExt) 2938 return LexEndOfFile(Result, CurPtr-1); 2939 2940 // If Microsoft extensions are disabled, this is just random garbage. 2941 Kind = tok::unknown; 2942 break; 2943 2944 case '\n': 2945 case '\r': 2946 // If we are inside a preprocessor directive and we see the end of line, 2947 // we know we are done with the directive, so return an EOD token. 2948 if (ParsingPreprocessorDirective) { 2949 // Done parsing the "line". 2950 ParsingPreprocessorDirective = false; 2951 2952 // Restore comment saving mode, in case it was disabled for directive. 2953 if (PP) 2954 resetExtendedTokenMode(); 2955 2956 // Since we consumed a newline, we are back at the start of a line. 2957 IsAtStartOfLine = true; 2958 IsAtPhysicalStartOfLine = true; 2959 2960 Kind = tok::eod; 2961 break; 2962 } 2963 2964 // No leading whitespace seen so far. 2965 Result.clearFlag(Token::LeadingSpace); 2966 2967 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 2968 return true; // KeepWhitespaceMode 2969 2970 // We only saw whitespace, so just try again with this lexer. 2971 // (We manually eliminate the tail call to avoid recursion.) 2972 goto LexNextToken; 2973 case ' ': 2974 case '\t': 2975 case '\f': 2976 case '\v': 2977 SkipHorizontalWhitespace: 2978 Result.setFlag(Token::LeadingSpace); 2979 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 2980 return true; // KeepWhitespaceMode 2981 2982 SkipIgnoredUnits: 2983 CurPtr = BufferPtr; 2984 2985 // If the next token is obviously a // or /* */ comment, skip it efficiently 2986 // too (without going through the big switch stmt). 2987 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 2988 LangOpts.LineComment && 2989 (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) { 2990 if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) 2991 return true; // There is a token to return. 2992 goto SkipIgnoredUnits; 2993 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 2994 if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) 2995 return true; // There is a token to return. 2996 goto SkipIgnoredUnits; 2997 } else if (isHorizontalWhitespace(*CurPtr)) { 2998 goto SkipHorizontalWhitespace; 2999 } 3000 // We only saw whitespace, so just try again with this lexer. 3001 // (We manually eliminate the tail call to avoid recursion.) 3002 goto LexNextToken; 3003 3004 // C99 6.4.4.1: Integer Constants. 3005 // C99 6.4.4.2: Floating Constants. 3006 case '0': case '1': case '2': case '3': case '4': 3007 case '5': case '6': case '7': case '8': case '9': 3008 // Notify MIOpt that we read a non-whitespace/non-comment token. 3009 MIOpt.ReadToken(); 3010 return LexNumericConstant(Result, CurPtr); 3011 3012 case 'u': // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal 3013 // Notify MIOpt that we read a non-whitespace/non-comment token. 3014 MIOpt.ReadToken(); 3015 3016 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 3017 Char = getCharAndSize(CurPtr, SizeTmp); 3018 3019 // UTF-16 string literal 3020 if (Char == '"') 3021 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3022 tok::utf16_string_literal); 3023 3024 // UTF-16 character constant 3025 if (Char == '\'') 3026 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3027 tok::utf16_char_constant); 3028 3029 // UTF-16 raw string literal 3030 if (Char == 'R' && LangOpts.CPlusPlus11 && 3031 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3032 return LexRawStringLiteral(Result, 3033 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3034 SizeTmp2, Result), 3035 tok::utf16_string_literal); 3036 3037 if (Char == '8') { 3038 char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 3039 3040 // UTF-8 string literal 3041 if (Char2 == '"') 3042 return LexStringLiteral(Result, 3043 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3044 SizeTmp2, Result), 3045 tok::utf8_string_literal); 3046 3047 if (Char2 == 'R' && LangOpts.CPlusPlus11) { 3048 unsigned SizeTmp3; 3049 char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3050 // UTF-8 raw string literal 3051 if (Char3 == '"') { 3052 return LexRawStringLiteral(Result, 3053 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3054 SizeTmp2, Result), 3055 SizeTmp3, Result), 3056 tok::utf8_string_literal); 3057 } 3058 } 3059 } 3060 } 3061 3062 // treat u like the start of an identifier. 3063 return LexIdentifier(Result, CurPtr); 3064 3065 case 'U': // Identifier (Uber) or C11/C++11 UTF-32 string literal 3066 // Notify MIOpt that we read a non-whitespace/non-comment token. 3067 MIOpt.ReadToken(); 3068 3069 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 3070 Char = getCharAndSize(CurPtr, SizeTmp); 3071 3072 // UTF-32 string literal 3073 if (Char == '"') 3074 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3075 tok::utf32_string_literal); 3076 3077 // UTF-32 character constant 3078 if (Char == '\'') 3079 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3080 tok::utf32_char_constant); 3081 3082 // UTF-32 raw string literal 3083 if (Char == 'R' && LangOpts.CPlusPlus11 && 3084 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3085 return LexRawStringLiteral(Result, 3086 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3087 SizeTmp2, Result), 3088 tok::utf32_string_literal); 3089 } 3090 3091 // treat U like the start of an identifier. 3092 return LexIdentifier(Result, CurPtr); 3093 3094 case 'R': // Identifier or C++0x raw string literal 3095 // Notify MIOpt that we read a non-whitespace/non-comment token. 3096 MIOpt.ReadToken(); 3097 3098 if (LangOpts.CPlusPlus11) { 3099 Char = getCharAndSize(CurPtr, SizeTmp); 3100 3101 if (Char == '"') 3102 return LexRawStringLiteral(Result, 3103 ConsumeChar(CurPtr, SizeTmp, Result), 3104 tok::string_literal); 3105 } 3106 3107 // treat R like the start of an identifier. 3108 return LexIdentifier(Result, CurPtr); 3109 3110 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 3111 // Notify MIOpt that we read a non-whitespace/non-comment token. 3112 MIOpt.ReadToken(); 3113 Char = getCharAndSize(CurPtr, SizeTmp); 3114 3115 // Wide string literal. 3116 if (Char == '"') 3117 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3118 tok::wide_string_literal); 3119 3120 // Wide raw string literal. 3121 if (LangOpts.CPlusPlus11 && Char == 'R' && 3122 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3123 return LexRawStringLiteral(Result, 3124 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3125 SizeTmp2, Result), 3126 tok::wide_string_literal); 3127 3128 // Wide character constant. 3129 if (Char == '\'') 3130 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3131 tok::wide_char_constant); 3132 // FALL THROUGH, treating L like the start of an identifier. 3133 3134 // C99 6.4.2: Identifiers. 3135 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 3136 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 3137 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 3138 case 'V': case 'W': case 'X': case 'Y': case 'Z': 3139 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 3140 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 3141 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 3142 case 'v': case 'w': case 'x': case 'y': case 'z': 3143 case '_': 3144 // Notify MIOpt that we read a non-whitespace/non-comment token. 3145 MIOpt.ReadToken(); 3146 return LexIdentifier(Result, CurPtr); 3147 3148 case '$': // $ in identifiers. 3149 if (LangOpts.DollarIdents) { 3150 if (!isLexingRawMode()) 3151 Diag(CurPtr-1, diag::ext_dollar_in_identifier); 3152 // Notify MIOpt that we read a non-whitespace/non-comment token. 3153 MIOpt.ReadToken(); 3154 return LexIdentifier(Result, CurPtr); 3155 } 3156 3157 Kind = tok::unknown; 3158 break; 3159 3160 // C99 6.4.4: Character Constants. 3161 case '\'': 3162 // Notify MIOpt that we read a non-whitespace/non-comment token. 3163 MIOpt.ReadToken(); 3164 return LexCharConstant(Result, CurPtr, tok::char_constant); 3165 3166 // C99 6.4.5: String Literals. 3167 case '"': 3168 // Notify MIOpt that we read a non-whitespace/non-comment token. 3169 MIOpt.ReadToken(); 3170 return LexStringLiteral(Result, CurPtr, tok::string_literal); 3171 3172 // C99 6.4.6: Punctuators. 3173 case '?': 3174 Kind = tok::question; 3175 break; 3176 case '[': 3177 Kind = tok::l_square; 3178 break; 3179 case ']': 3180 Kind = tok::r_square; 3181 break; 3182 case '(': 3183 Kind = tok::l_paren; 3184 break; 3185 case ')': 3186 Kind = tok::r_paren; 3187 break; 3188 case '{': 3189 Kind = tok::l_brace; 3190 break; 3191 case '}': 3192 Kind = tok::r_brace; 3193 break; 3194 case '.': 3195 Char = getCharAndSize(CurPtr, SizeTmp); 3196 if (Char >= '0' && Char <= '9') { 3197 // Notify MIOpt that we read a non-whitespace/non-comment token. 3198 MIOpt.ReadToken(); 3199 3200 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 3201 } else if (LangOpts.CPlusPlus && Char == '*') { 3202 Kind = tok::periodstar; 3203 CurPtr += SizeTmp; 3204 } else if (Char == '.' && 3205 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 3206 Kind = tok::ellipsis; 3207 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3208 SizeTmp2, Result); 3209 } else { 3210 Kind = tok::period; 3211 } 3212 break; 3213 case '&': 3214 Char = getCharAndSize(CurPtr, SizeTmp); 3215 if (Char == '&') { 3216 Kind = tok::ampamp; 3217 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3218 } else if (Char == '=') { 3219 Kind = tok::ampequal; 3220 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3221 } else { 3222 Kind = tok::amp; 3223 } 3224 break; 3225 case '*': 3226 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3227 Kind = tok::starequal; 3228 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3229 } else { 3230 Kind = tok::star; 3231 } 3232 break; 3233 case '+': 3234 Char = getCharAndSize(CurPtr, SizeTmp); 3235 if (Char == '+') { 3236 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3237 Kind = tok::plusplus; 3238 } else if (Char == '=') { 3239 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3240 Kind = tok::plusequal; 3241 } else { 3242 Kind = tok::plus; 3243 } 3244 break; 3245 case '-': 3246 Char = getCharAndSize(CurPtr, SizeTmp); 3247 if (Char == '-') { // -- 3248 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3249 Kind = tok::minusminus; 3250 } else if (Char == '>' && LangOpts.CPlusPlus && 3251 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 3252 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3253 SizeTmp2, Result); 3254 Kind = tok::arrowstar; 3255 } else if (Char == '>') { // -> 3256 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3257 Kind = tok::arrow; 3258 } else if (Char == '=') { // -= 3259 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3260 Kind = tok::minusequal; 3261 } else { 3262 Kind = tok::minus; 3263 } 3264 break; 3265 case '~': 3266 Kind = tok::tilde; 3267 break; 3268 case '!': 3269 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3270 Kind = tok::exclaimequal; 3271 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3272 } else { 3273 Kind = tok::exclaim; 3274 } 3275 break; 3276 case '/': 3277 // 6.4.9: Comments 3278 Char = getCharAndSize(CurPtr, SizeTmp); 3279 if (Char == '/') { // Line comment. 3280 // Even if Line comments are disabled (e.g. in C89 mode), we generally 3281 // want to lex this as a comment. There is one problem with this though, 3282 // that in one particular corner case, this can change the behavior of the 3283 // resultant program. For example, In "foo //**/ bar", C89 would lex 3284 // this as "foo / bar" and langauges with Line comments would lex it as 3285 // "foo". Check to see if the character after the second slash is a '*'. 3286 // If so, we will lex that as a "/" instead of the start of a comment. 3287 // However, we never do this if we are just preprocessing. 3288 bool TreatAsComment = LangOpts.LineComment && 3289 (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP); 3290 if (!TreatAsComment) 3291 if (!(PP && PP->isPreprocessedOutput())) 3292 TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; 3293 3294 if (TreatAsComment) { 3295 if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3296 TokAtPhysicalStartOfLine)) 3297 return true; // There is a token to return. 3298 3299 // It is common for the tokens immediately after a // comment to be 3300 // whitespace (indentation for the next line). Instead of going through 3301 // the big switch, handle it efficiently now. 3302 goto SkipIgnoredUnits; 3303 } 3304 } 3305 3306 if (Char == '*') { // /**/ comment. 3307 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3308 TokAtPhysicalStartOfLine)) 3309 return true; // There is a token to return. 3310 3311 // We only saw whitespace, so just try again with this lexer. 3312 // (We manually eliminate the tail call to avoid recursion.) 3313 goto LexNextToken; 3314 } 3315 3316 if (Char == '=') { 3317 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3318 Kind = tok::slashequal; 3319 } else { 3320 Kind = tok::slash; 3321 } 3322 break; 3323 case '%': 3324 Char = getCharAndSize(CurPtr, SizeTmp); 3325 if (Char == '=') { 3326 Kind = tok::percentequal; 3327 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3328 } else if (LangOpts.Digraphs && Char == '>') { 3329 Kind = tok::r_brace; // '%>' -> '}' 3330 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3331 } else if (LangOpts.Digraphs && Char == ':') { 3332 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3333 Char = getCharAndSize(CurPtr, SizeTmp); 3334 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 3335 Kind = tok::hashhash; // '%:%:' -> '##' 3336 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3337 SizeTmp2, Result); 3338 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize 3339 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3340 if (!isLexingRawMode()) 3341 Diag(BufferPtr, diag::ext_charize_microsoft); 3342 Kind = tok::hashat; 3343 } else { // '%:' -> '#' 3344 // We parsed a # character. If this occurs at the start of the line, 3345 // it's actually the start of a preprocessing directive. Callback to 3346 // the preprocessor to handle it. 3347 // FIXME: -fpreprocessed mode?? 3348 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) 3349 goto HandleDirective; 3350 3351 Kind = tok::hash; 3352 } 3353 } else { 3354 Kind = tok::percent; 3355 } 3356 break; 3357 case '<': 3358 Char = getCharAndSize(CurPtr, SizeTmp); 3359 if (ParsingFilename) { 3360 return LexAngledStringLiteral(Result, CurPtr); 3361 } else if (Char == '<') { 3362 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3363 if (After == '=') { 3364 Kind = tok::lesslessequal; 3365 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3366 SizeTmp2, Result); 3367 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 3368 // If this is actually a '<<<<<<<' version control conflict marker, 3369 // recognize it as such and recover nicely. 3370 goto LexNextToken; 3371 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { 3372 // If this is '<<<<' and we're in a Perforce-style conflict marker, 3373 // ignore it. 3374 goto LexNextToken; 3375 } else if (LangOpts.CUDA && After == '<') { 3376 Kind = tok::lesslessless; 3377 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3378 SizeTmp2, Result); 3379 } else { 3380 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3381 Kind = tok::lessless; 3382 } 3383 } else if (Char == '=') { 3384 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3385 Kind = tok::lessequal; 3386 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' 3387 if (LangOpts.CPlusPlus11 && 3388 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 3389 // C++0x [lex.pptoken]p3: 3390 // Otherwise, if the next three characters are <:: and the subsequent 3391 // character is neither : nor >, the < is treated as a preprocessor 3392 // token by itself and not as the first character of the alternative 3393 // token <:. 3394 unsigned SizeTmp3; 3395 char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3396 if (After != ':' && After != '>') { 3397 Kind = tok::less; 3398 if (!isLexingRawMode()) 3399 Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); 3400 break; 3401 } 3402 } 3403 3404 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3405 Kind = tok::l_square; 3406 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' 3407 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3408 Kind = tok::l_brace; 3409 } else { 3410 Kind = tok::less; 3411 } 3412 break; 3413 case '>': 3414 Char = getCharAndSize(CurPtr, SizeTmp); 3415 if (Char == '=') { 3416 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3417 Kind = tok::greaterequal; 3418 } else if (Char == '>') { 3419 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3420 if (After == '=') { 3421 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3422 SizeTmp2, Result); 3423 Kind = tok::greatergreaterequal; 3424 } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { 3425 // If this is actually a '>>>>' conflict marker, recognize it as such 3426 // and recover nicely. 3427 goto LexNextToken; 3428 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 3429 // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 3430 goto LexNextToken; 3431 } else if (LangOpts.CUDA && After == '>') { 3432 Kind = tok::greatergreatergreater; 3433 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3434 SizeTmp2, Result); 3435 } else { 3436 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3437 Kind = tok::greatergreater; 3438 } 3439 3440 } else { 3441 Kind = tok::greater; 3442 } 3443 break; 3444 case '^': 3445 Char = getCharAndSize(CurPtr, SizeTmp); 3446 if (Char == '=') { 3447 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3448 Kind = tok::caretequal; 3449 } else { 3450 Kind = tok::caret; 3451 } 3452 break; 3453 case '|': 3454 Char = getCharAndSize(CurPtr, SizeTmp); 3455 if (Char == '=') { 3456 Kind = tok::pipeequal; 3457 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3458 } else if (Char == '|') { 3459 // If this is '|||||||' and we're in a conflict marker, ignore it. 3460 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 3461 goto LexNextToken; 3462 Kind = tok::pipepipe; 3463 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3464 } else { 3465 Kind = tok::pipe; 3466 } 3467 break; 3468 case ':': 3469 Char = getCharAndSize(CurPtr, SizeTmp); 3470 if (LangOpts.Digraphs && Char == '>') { 3471 Kind = tok::r_square; // ':>' -> ']' 3472 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3473 } else if (LangOpts.CPlusPlus && Char == ':') { 3474 Kind = tok::coloncolon; 3475 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3476 } else { 3477 Kind = tok::colon; 3478 } 3479 break; 3480 case ';': 3481 Kind = tok::semi; 3482 break; 3483 case '=': 3484 Char = getCharAndSize(CurPtr, SizeTmp); 3485 if (Char == '=') { 3486 // If this is '====' and we're in a conflict marker, ignore it. 3487 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 3488 goto LexNextToken; 3489 3490 Kind = tok::equalequal; 3491 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3492 } else { 3493 Kind = tok::equal; 3494 } 3495 break; 3496 case ',': 3497 Kind = tok::comma; 3498 break; 3499 case '#': 3500 Char = getCharAndSize(CurPtr, SizeTmp); 3501 if (Char == '#') { 3502 Kind = tok::hashhash; 3503 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3504 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize 3505 Kind = tok::hashat; 3506 if (!isLexingRawMode()) 3507 Diag(BufferPtr, diag::ext_charize_microsoft); 3508 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3509 } else { 3510 // We parsed a # character. If this occurs at the start of the line, 3511 // it's actually the start of a preprocessing directive. Callback to 3512 // the preprocessor to handle it. 3513 // FIXME: -fpreprocessed mode?? 3514 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) 3515 goto HandleDirective; 3516 3517 Kind = tok::hash; 3518 } 3519 break; 3520 3521 case '@': 3522 // Objective C support. 3523 if (CurPtr[-1] == '@' && LangOpts.ObjC1) 3524 Kind = tok::at; 3525 else 3526 Kind = tok::unknown; 3527 break; 3528 3529 // UCNs (C99 6.4.3, C++11 [lex.charset]p2) 3530 case '\\': 3531 if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) { 3532 if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { 3533 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3534 return true; // KeepWhitespaceMode 3535 3536 // We only saw whitespace, so just try again with this lexer. 3537 // (We manually eliminate the tail call to avoid recursion.) 3538 goto LexNextToken; 3539 } 3540 3541 return LexUnicode(Result, CodePoint, CurPtr); 3542 } 3543 3544 Kind = tok::unknown; 3545 break; 3546 3547 default: { 3548 if (isASCII(Char)) { 3549 Kind = tok::unknown; 3550 break; 3551 } 3552 3553 UTF32 CodePoint; 3554 3555 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to 3556 // an escaped newline. 3557 --CurPtr; 3558 ConversionResult Status = 3559 llvm::convertUTF8Sequence((const UTF8 **)&CurPtr, 3560 (const UTF8 *)BufferEnd, 3561 &CodePoint, 3562 strictConversion); 3563 if (Status == conversionOK) { 3564 if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { 3565 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3566 return true; // KeepWhitespaceMode 3567 3568 // We only saw whitespace, so just try again with this lexer. 3569 // (We manually eliminate the tail call to avoid recursion.) 3570 goto LexNextToken; 3571 } 3572 return LexUnicode(Result, CodePoint, CurPtr); 3573 } 3574 3575 if (isLexingRawMode() || ParsingPreprocessorDirective || 3576 PP->isPreprocessedOutput()) { 3577 ++CurPtr; 3578 Kind = tok::unknown; 3579 break; 3580 } 3581 3582 // Non-ASCII characters tend to creep into source code unintentionally. 3583 // Instead of letting the parser complain about the unknown token, 3584 // just diagnose the invalid UTF-8, then drop the character. 3585 Diag(CurPtr, diag::err_invalid_utf8); 3586 3587 BufferPtr = CurPtr+1; 3588 // We're pretending the character didn't exist, so just try again with 3589 // this lexer. 3590 // (We manually eliminate the tail call to avoid recursion.) 3591 goto LexNextToken; 3592 } 3593 } 3594 3595 // Notify MIOpt that we read a non-whitespace/non-comment token. 3596 MIOpt.ReadToken(); 3597 3598 // Update the location of token as well as BufferPtr. 3599 FormTokenWithChars(Result, CurPtr, Kind); 3600 return true; 3601 3602HandleDirective: 3603 // We parsed a # character and it's the start of a preprocessing directive. 3604 3605 FormTokenWithChars(Result, CurPtr, tok::hash); 3606 PP->HandleDirective(Result); 3607 3608 if (PP->hadModuleLoaderFatalFailure()) { 3609 // With a fatal failure in the module loader, we abort parsing. 3610 assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof"); 3611 return true; 3612 } 3613 3614 // We parsed the directive; lex a token with the new state. 3615 return false; 3616} 3617