1193326Sed//===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===// 2193326Sed// 3193326Sed// The LLVM Compiler Infrastructure 4193326Sed// 5193326Sed// This file is distributed under the University of Illinois Open Source 6193326Sed// License. See LICENSE.TXT for details. 7193326Sed// 8193326Sed//===----------------------------------------------------------------------===// 9193326Sed// 10193326Sed// This provides a possible implementation of PTH support for Clang that is 11193326Sed// based on caching lexed tokens and identifiers. 12193326Sed// 13193326Sed//===----------------------------------------------------------------------===// 14193326Sed 15193326Sed#include "clang/Frontend/Utils.h" 16218893Sdim#include "clang/Basic/Diagnostic.h" 17193326Sed#include "clang/Basic/FileManager.h" 18218893Sdim#include "clang/Basic/FileSystemStatCache.h" 19193326Sed#include "clang/Basic/IdentifierTable.h" 20193326Sed#include "clang/Basic/OnDiskHashTable.h" 21218893Sdim#include "clang/Basic/SourceManager.h" 22193326Sed#include "clang/Lex/Lexer.h" 23193326Sed#include "clang/Lex/Preprocessor.h" 24198398Srdivacky#include "llvm/ADT/StringExtras.h" 25198398Srdivacky#include "llvm/ADT/StringMap.h" 26218893Sdim#include "llvm/Support/FileSystem.h" 27193326Sed#include "llvm/Support/MemoryBuffer.h" 28252723Sdim#include "llvm/Support/Path.h" 29193326Sed#include "llvm/Support/raw_ostream.h" 30193326Sed 31193326Sed// FIXME: put this somewhere else? 32193326Sed#ifndef S_ISDIR 33193326Sed#define S_ISDIR(x) (((x)&_S_IFDIR)!=0) 34193326Sed#endif 35193326Sed 36193326Sedusing namespace clang; 37193326Sedusing namespace clang::io; 38193326Sed 39193326Sed//===----------------------------------------------------------------------===// 40193326Sed// PTH-specific stuff. 41193326Sed//===----------------------------------------------------------------------===// 42193326Sed 43193326Sednamespace { 44199990Srdivackyclass PTHEntry { 45198092Srdivacky Offset TokenData, PPCondData; 46193326Sed 47198092Srdivackypublic: 48193326Sed PTHEntry() {} 49193326Sed 50193326Sed PTHEntry(Offset td, Offset ppcd) 51193326Sed : TokenData(td), PPCondData(ppcd) {} 52198092Srdivacky 53198092Srdivacky Offset getTokenOffset() const { return TokenData; } 54193326Sed Offset getPPCondTableOffset() const { return PPCondData; } 55193326Sed}; 56198092Srdivacky 57198092Srdivacky 58199990Srdivackyclass PTHEntryKeyVariant { 59193326Sed union { const FileEntry* FE; const char* Path; }; 60193326Sed enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind; 61263509Sdim FileData *Data; 62263509Sdim 63193326Sedpublic: 64263509Sdim PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE), Data(0) {} 65193326Sed 66263509Sdim PTHEntryKeyVariant(FileData *Data, const char *path) 67263509Sdim : Path(path), Kind(IsDE), Data(new FileData(*Data)) {} 68193326Sed 69263509Sdim explicit PTHEntryKeyVariant(const char *path) 70263509Sdim : Path(path), Kind(IsNoExist), Data(0) {} 71198092Srdivacky 72193326Sed bool isFile() const { return Kind == IsFE; } 73198092Srdivacky 74226890Sdim StringRef getString() const { 75193326Sed return Kind == IsFE ? FE->getName() : Path; 76193326Sed } 77198092Srdivacky 78193326Sed unsigned getKind() const { return (unsigned) Kind; } 79198092Srdivacky 80226890Sdim void EmitData(raw_ostream& Out) { 81193326Sed switch (Kind) { 82263509Sdim case IsFE: { 83202879Srdivacky // Emit stat information. 84263509Sdim llvm::sys::fs::UniqueID UID = FE->getUniqueID(); 85263509Sdim ::Emit64(Out, UID.getFile()); 86263509Sdim ::Emit64(Out, UID.getDevice()); 87202879Srdivacky ::Emit64(Out, FE->getModificationTime()); 88202879Srdivacky ::Emit64(Out, FE->getSize()); 89263509Sdim } break; 90202879Srdivacky case IsDE: 91202879Srdivacky // Emit stat information. 92263509Sdim ::Emit64(Out, Data->UniqueID.getFile()); 93263509Sdim ::Emit64(Out, Data->UniqueID.getDevice()); 94263509Sdim ::Emit64(Out, Data->ModTime); 95263509Sdim ::Emit64(Out, Data->Size); 96263509Sdim delete Data; 97202879Srdivacky break; 98202879Srdivacky default: 99202879Srdivacky break; 100193326Sed } 101193326Sed } 102198092Srdivacky 103193326Sed unsigned getRepresentationLength() const { 104193326Sed return Kind == IsNoExist ? 0 : 4 + 4 + 2 + 8 + 8; 105193326Sed } 106193326Sed}; 107198092Srdivacky 108199990Srdivackyclass FileEntryPTHEntryInfo { 109193326Sedpublic: 110193326Sed typedef PTHEntryKeyVariant key_type; 111193326Sed typedef key_type key_type_ref; 112198092Srdivacky 113193326Sed typedef PTHEntry data_type; 114193326Sed typedef const PTHEntry& data_type_ref; 115198092Srdivacky 116193326Sed static unsigned ComputeHash(PTHEntryKeyVariant V) { 117198398Srdivacky return llvm::HashString(V.getString()); 118193326Sed } 119198092Srdivacky 120198092Srdivacky static std::pair<unsigned,unsigned> 121226890Sdim EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V, 122193326Sed const PTHEntry& E) { 123193326Sed 124198398Srdivacky unsigned n = V.getString().size() + 1 + 1; 125193326Sed ::Emit16(Out, n); 126198092Srdivacky 127193326Sed unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0); 128193326Sed ::Emit8(Out, m); 129193326Sed 130193326Sed return std::make_pair(n, m); 131193326Sed } 132198092Srdivacky 133226890Sdim static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){ 134193326Sed // Emit the entry kind. 135193326Sed ::Emit8(Out, (unsigned) V.getKind()); 136193326Sed // Emit the string. 137198398Srdivacky Out.write(V.getString().data(), n - 1); 138193326Sed } 139198092Srdivacky 140226890Sdim static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V, 141193326Sed const PTHEntry& E, unsigned) { 142193326Sed 143193326Sed 144193326Sed // For file entries emit the offsets into the PTH file for token data 145193326Sed // and the preprocessor blocks table. 146193326Sed if (V.isFile()) { 147193326Sed ::Emit32(Out, E.getTokenOffset()); 148193326Sed ::Emit32(Out, E.getPPCondTableOffset()); 149193326Sed } 150198092Srdivacky 151193326Sed // Emit any other data associated with the key (i.e., stat information). 152193326Sed V.EmitData(Out); 153198092Srdivacky } 154193326Sed}; 155198092Srdivacky 156193326Sedclass OffsetOpt { 157193326Sed bool valid; 158193326Sed Offset off; 159193326Sedpublic: 160193326Sed OffsetOpt() : valid(false) {} 161193326Sed bool hasOffset() const { return valid; } 162193326Sed Offset getOffset() const { assert(valid); return off; } 163193326Sed void setOffset(Offset o) { off = o; valid = true; } 164193326Sed}; 165193326Sed} // end anonymous namespace 166193326Sed 167193326Sedtypedef OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo> PTHMap; 168193326Sed 169193326Sednamespace { 170199990Srdivackyclass PTHWriter { 171205408Srdivacky typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap; 172205408Srdivacky typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy; 173205408Srdivacky 174193326Sed IDMap IM; 175193326Sed llvm::raw_fd_ostream& Out; 176193326Sed Preprocessor& PP; 177193326Sed uint32_t idcount; 178193326Sed PTHMap PM; 179193326Sed CachedStrsTy CachedStrs; 180193326Sed Offset CurStrOffset; 181193326Sed std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries; 182193326Sed 183193326Sed //// Get the persistent id for the given IdentifierInfo*. 184193326Sed uint32_t ResolveID(const IdentifierInfo* II); 185198092Srdivacky 186193326Sed /// Emit a token to the PTH file. 187193326Sed void EmitToken(const Token& T); 188193326Sed 189204793Srdivacky void Emit8(uint32_t V) { ::Emit8(Out, V); } 190198092Srdivacky 191193326Sed void Emit16(uint32_t V) { ::Emit16(Out, V); } 192198092Srdivacky 193193326Sed void Emit32(uint32_t V) { ::Emit32(Out, V); } 194193326Sed 195193326Sed void EmitBuf(const char *Ptr, unsigned NumBytes) { 196193326Sed Out.write(Ptr, NumBytes); 197193326Sed } 198198092Srdivacky 199226890Sdim void EmitString(StringRef V) { 200205219Srdivacky ::Emit16(Out, V.size()); 201205219Srdivacky EmitBuf(V.data(), V.size()); 202205219Srdivacky } 203205219Srdivacky 204193326Sed /// EmitIdentifierTable - Emits two tables to the PTH file. The first is 205193326Sed /// a hashtable mapping from identifier strings to persistent IDs. 206193326Sed /// The second is a straight table mapping from persistent IDs to string data 207193326Sed /// (the keys of the first table). 208193326Sed std::pair<Offset, Offset> EmitIdentifierTable(); 209198092Srdivacky 210193326Sed /// EmitFileTable - Emit a table mapping from file name strings to PTH 211193326Sed /// token data. 212193326Sed Offset EmitFileTable() { return PM.Emit(Out); } 213193326Sed 214193326Sed PTHEntry LexTokens(Lexer& L); 215193326Sed Offset EmitCachedSpellings(); 216193326Sed 217193326Sedpublic: 218198092Srdivacky PTHWriter(llvm::raw_fd_ostream& out, Preprocessor& pp) 219193326Sed : Out(out), PP(pp), idcount(0), CurStrOffset(0) {} 220198092Srdivacky 221193326Sed PTHMap &getPM() { return PM; } 222205219Srdivacky void GeneratePTH(const std::string &MainFile); 223193326Sed}; 224193326Sed} // end anonymous namespace 225198092Srdivacky 226198092Srdivackyuint32_t PTHWriter::ResolveID(const IdentifierInfo* II) { 227193326Sed // Null IdentifierInfo's map to the persistent ID 0. 228193326Sed if (!II) 229193326Sed return 0; 230198092Srdivacky 231193326Sed IDMap::iterator I = IM.find(II); 232193326Sed if (I != IM.end()) 233193326Sed return I->second; // We've already added 1. 234198092Srdivacky 235193326Sed IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL. 236193326Sed return idcount; 237193326Sed} 238193326Sed 239193326Sedvoid PTHWriter::EmitToken(const Token& T) { 240193326Sed // Emit the token kind, flags, and length. 241193326Sed Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)| 242193326Sed (((uint32_t) T.getLength()) << 16)); 243198092Srdivacky 244193326Sed if (!T.isLiteral()) { 245193326Sed Emit32(ResolveID(T.getIdentifierInfo())); 246193326Sed } else { 247193326Sed // We cache *un-cleaned* spellings. This gives us 100% fidelity with the 248193326Sed // source code. 249226890Sdim StringRef s(T.getLiteralData(), T.getLength()); 250193326Sed 251193326Sed // Get the string entry. 252224145Sdim llvm::StringMapEntry<OffsetOpt> *E = &CachedStrs.GetOrCreateValue(s); 253198092Srdivacky 254193326Sed // If this is a new string entry, bump the PTH offset. 255193326Sed if (!E->getValue().hasOffset()) { 256193326Sed E->getValue().setOffset(CurStrOffset); 257193326Sed StrEntries.push_back(E); 258224145Sdim CurStrOffset += s.size() + 1; 259193326Sed } 260198092Srdivacky 261193326Sed // Emit the relative offset into the PTH file for the spelling string. 262193326Sed Emit32(E->getValue().getOffset()); 263193326Sed } 264198092Srdivacky 265193326Sed // Emit the offset into the original source file of this token so that we 266193326Sed // can reconstruct its SourceLocation. 267193326Sed Emit32(PP.getSourceManager().getFileOffset(T.getLocation())); 268193326Sed} 269193326Sed 270193326SedPTHEntry PTHWriter::LexTokens(Lexer& L) { 271193326Sed // Pad 0's so that we emit tokens to a 4-byte alignment. 272193326Sed // This speed up reading them back in. 273193326Sed Pad(Out, 4); 274205408Srdivacky Offset TokenOff = (Offset) Out.tell(); 275198092Srdivacky 276193326Sed // Keep track of matching '#if' ... '#endif'. 277193326Sed typedef std::vector<std::pair<Offset, unsigned> > PPCondTable; 278193326Sed PPCondTable PPCond; 279193326Sed std::vector<unsigned> PPStartCond; 280193326Sed bool ParsingPreprocessorDirective = false; 281193326Sed Token Tok; 282198092Srdivacky 283193326Sed do { 284193326Sed L.LexFromRawLexer(Tok); 285193326Sed NextToken: 286193326Sed 287193326Sed if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) && 288193326Sed ParsingPreprocessorDirective) { 289221345Sdim // Insert an eod token into the token cache. It has the same 290193326Sed // position as the next token that is not on the same line as the 291193326Sed // preprocessor directive. Observe that we continue processing 292193326Sed // 'Tok' when we exit this branch. 293193326Sed Token Tmp = Tok; 294221345Sdim Tmp.setKind(tok::eod); 295193326Sed Tmp.clearFlag(Token::StartOfLine); 296193326Sed Tmp.setIdentifierInfo(0); 297193326Sed EmitToken(Tmp); 298193326Sed ParsingPreprocessorDirective = false; 299193326Sed } 300198092Srdivacky 301218893Sdim if (Tok.is(tok::raw_identifier)) { 302205219Srdivacky PP.LookUpIdentifierInfo(Tok); 303193326Sed EmitToken(Tok); 304193326Sed continue; 305193326Sed } 306193326Sed 307193326Sed if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) { 308193326Sed // Special processing for #include. Store the '#' token and lex 309193326Sed // the next token. 310193326Sed assert(!ParsingPreprocessorDirective); 311193326Sed Offset HashOff = (Offset) Out.tell(); 312193326Sed 313193326Sed // Get the next token. 314212904Sdim Token NextTok; 315212904Sdim L.LexFromRawLexer(NextTok); 316193326Sed 317212904Sdim // If we see the start of line, then we had a null directive "#". In 318212904Sdim // this case, discard both tokens. 319212904Sdim if (NextTok.isAtStartOfLine()) 320193326Sed goto NextToken; 321218893Sdim 322212904Sdim // The token is the start of a directive. Emit it. 323212904Sdim EmitToken(Tok); 324212904Sdim Tok = NextTok; 325198092Srdivacky 326193326Sed // Did we see 'include'/'import'/'include_next'? 327218893Sdim if (Tok.isNot(tok::raw_identifier)) { 328193326Sed EmitToken(Tok); 329193326Sed continue; 330193326Sed } 331198092Srdivacky 332193326Sed IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok); 333193326Sed tok::PPKeywordKind K = II->getPPKeywordID(); 334198092Srdivacky 335193326Sed ParsingPreprocessorDirective = true; 336198092Srdivacky 337193326Sed switch (K) { 338193326Sed case tok::pp_not_keyword: 339193326Sed // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass 340193326Sed // them through. 341193326Sed default: 342193326Sed break; 343198092Srdivacky 344193326Sed case tok::pp_include: 345193326Sed case tok::pp_import: 346198092Srdivacky case tok::pp_include_next: { 347193326Sed // Save the 'include' token. 348193326Sed EmitToken(Tok); 349193326Sed // Lex the next token as an include string. 350193326Sed L.setParsingPreprocessorDirective(true); 351198092Srdivacky L.LexIncludeFilename(Tok); 352193326Sed L.setParsingPreprocessorDirective(false); 353193326Sed assert(!Tok.isAtStartOfLine()); 354218893Sdim if (Tok.is(tok::raw_identifier)) 355205219Srdivacky PP.LookUpIdentifierInfo(Tok); 356198092Srdivacky 357193326Sed break; 358193326Sed } 359193326Sed case tok::pp_if: 360193326Sed case tok::pp_ifdef: 361193326Sed case tok::pp_ifndef: { 362193326Sed // Add an entry for '#if' and friends. We initially set the target 363193326Sed // index to 0. This will get backpatched when we hit #endif. 364193326Sed PPStartCond.push_back(PPCond.size()); 365193326Sed PPCond.push_back(std::make_pair(HashOff, 0U)); 366193326Sed break; 367193326Sed } 368193326Sed case tok::pp_endif: { 369193326Sed // Add an entry for '#endif'. We set the target table index to itself. 370193326Sed // This will later be set to zero when emitting to the PTH file. We 371193326Sed // use 0 for uninitialized indices because that is easier to debug. 372193326Sed unsigned index = PPCond.size(); 373193326Sed // Backpatch the opening '#if' entry. 374193326Sed assert(!PPStartCond.empty()); 375193326Sed assert(PPCond.size() > PPStartCond.back()); 376193326Sed assert(PPCond[PPStartCond.back()].second == 0); 377193326Sed PPCond[PPStartCond.back()].second = index; 378198092Srdivacky PPStartCond.pop_back(); 379198092Srdivacky // Add the new entry to PPCond. 380193326Sed PPCond.push_back(std::make_pair(HashOff, index)); 381193326Sed EmitToken(Tok); 382198092Srdivacky 383193326Sed // Some files have gibberish on the same line as '#endif'. 384193326Sed // Discard these tokens. 385193326Sed do 386193326Sed L.LexFromRawLexer(Tok); 387193326Sed while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine()); 388193326Sed // We have the next token in hand. 389193326Sed // Don't immediately lex the next one. 390198092Srdivacky goto NextToken; 391193326Sed } 392193326Sed case tok::pp_elif: 393193326Sed case tok::pp_else: { 394193326Sed // Add an entry for #elif or #else. 395193326Sed // This serves as both a closing and opening of a conditional block. 396193326Sed // This means that its entry will get backpatched later. 397193326Sed unsigned index = PPCond.size(); 398193326Sed // Backpatch the previous '#if' entry. 399193326Sed assert(!PPStartCond.empty()); 400193326Sed assert(PPCond.size() > PPStartCond.back()); 401193326Sed assert(PPCond[PPStartCond.back()].second == 0); 402193326Sed PPCond[PPStartCond.back()].second = index; 403193326Sed PPStartCond.pop_back(); 404193326Sed // Now add '#elif' as a new block opening. 405193326Sed PPCond.push_back(std::make_pair(HashOff, 0U)); 406193326Sed PPStartCond.push_back(index); 407193326Sed break; 408193326Sed } 409193326Sed } 410193326Sed } 411198092Srdivacky 412193326Sed EmitToken(Tok); 413193326Sed } 414193326Sed while (Tok.isNot(tok::eof)); 415193326Sed 416193326Sed assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals."); 417193326Sed 418193326Sed // Next write out PPCond. 419193326Sed Offset PPCondOff = (Offset) Out.tell(); 420193326Sed 421193326Sed // Write out the size of PPCond so that clients can identifer empty tables. 422193326Sed Emit32(PPCond.size()); 423193326Sed 424193326Sed for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) { 425205408Srdivacky Emit32(PPCond[i].first - TokenOff); 426193326Sed uint32_t x = PPCond[i].second; 427193326Sed assert(x != 0 && "PPCond entry not backpatched."); 428193326Sed // Emit zero for #endifs. This allows us to do checking when 429193326Sed // we read the PTH file back in. 430193326Sed Emit32(x == i ? 0 : x); 431193326Sed } 432193326Sed 433205408Srdivacky return PTHEntry(TokenOff, PPCondOff); 434193326Sed} 435193326Sed 436193326SedOffset PTHWriter::EmitCachedSpellings() { 437193326Sed // Write each cached strings to the PTH file. 438193326Sed Offset SpellingsOff = Out.tell(); 439198092Srdivacky 440193326Sed for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator 441193326Sed I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I) 442193326Sed EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/); 443198092Srdivacky 444193326Sed return SpellingsOff; 445193326Sed} 446193326Sed 447205219Srdivackyvoid PTHWriter::GeneratePTH(const std::string &MainFile) { 448193326Sed // Generate the prologue. 449245431Sdim Out << "cfe-pth" << '\0'; 450193326Sed Emit32(PTHManager::Version); 451198092Srdivacky 452193326Sed // Leave 4 words for the prologue. 453193326Sed Offset PrologueOffset = Out.tell(); 454193326Sed for (unsigned i = 0; i < 4; ++i) 455193326Sed Emit32(0); 456198092Srdivacky 457193326Sed // Write the name of the MainFile. 458205219Srdivacky if (!MainFile.empty()) { 459205408Srdivacky EmitString(MainFile); 460193326Sed } else { 461193326Sed // String with 0 bytes. 462193326Sed Emit16(0); 463193326Sed } 464193326Sed Emit8(0); 465198092Srdivacky 466193326Sed // Iterate over all the files in SourceManager. Create a lexer 467193326Sed // for each file and cache the tokens. 468193326Sed SourceManager &SM = PP.getSourceManager(); 469235633Sdim const LangOptions &LOpts = PP.getLangOpts(); 470198092Srdivacky 471193326Sed for (SourceManager::fileinfo_iterator I = SM.fileinfo_begin(), 472193326Sed E = SM.fileinfo_end(); I != E; ++I) { 473193326Sed const SrcMgr::ContentCache &C = *I->second; 474221345Sdim const FileEntry *FE = C.OrigEntry; 475198092Srdivacky 476193326Sed // FIXME: Handle files with non-absolute paths. 477218893Sdim if (llvm::sys::path::is_relative(FE->getName())) 478193326Sed continue; 479193326Sed 480207619Srdivacky const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(), SM); 481193326Sed if (!B) continue; 482193326Sed 483193326Sed FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User); 484199990Srdivacky const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); 485199990Srdivacky Lexer L(FID, FromFile, SM, LOpts); 486193326Sed PM.insert(FE, LexTokens(L)); 487193326Sed } 488193326Sed 489193326Sed // Write out the identifier table. 490193326Sed const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable(); 491198092Srdivacky 492193326Sed // Write out the cached strings table. 493193326Sed Offset SpellingOff = EmitCachedSpellings(); 494198092Srdivacky 495193326Sed // Write out the file table. 496198092Srdivacky Offset FileTableOff = EmitFileTable(); 497198092Srdivacky 498193326Sed // Finally, write the prologue. 499193326Sed Out.seek(PrologueOffset); 500193326Sed Emit32(IdTableOff.first); 501193326Sed Emit32(IdTableOff.second); 502193326Sed Emit32(FileTableOff); 503193326Sed Emit32(SpellingOff); 504193326Sed} 505193326Sed 506193326Sednamespace { 507193326Sed/// StatListener - A simple "interpose" object used to monitor stat calls 508193326Sed/// invoked by FileManager while processing the original sources used 509193326Sed/// as input to PTH generation. StatListener populates the PTHWriter's 510193326Sed/// file map with stat information for directories as well as negative stats. 511193326Sed/// Stat information for files are populated elsewhere. 512218893Sdimclass StatListener : public FileSystemStatCache { 513193326Sed PTHMap &PM; 514193326Sedpublic: 515193326Sed StatListener(PTHMap &pm) : PM(pm) {} 516193326Sed ~StatListener() {} 517198092Srdivacky 518263509Sdim LookupResult getStat(const char *Path, FileData &Data, bool isFile, 519263509Sdim int *FileDescriptor) { 520263509Sdim LookupResult Result = statChained(Path, Data, isFile, FileDescriptor); 521198092Srdivacky 522218893Sdim if (Result == CacheMissing) // Failed 'stat'. 523218893Sdim PM.insert(PTHEntryKeyVariant(Path), PTHEntry()); 524263509Sdim else if (Data.IsDirectory) { 525193326Sed // Only cache directories with absolute paths. 526218893Sdim if (llvm::sys::path::is_relative(Path)) 527218893Sdim return Result; 528198092Srdivacky 529263509Sdim PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry()); 530193326Sed } 531198092Srdivacky 532218893Sdim return Result; 533193326Sed } 534193326Sed}; 535193326Sed} // end anonymous namespace 536193326Sed 537193326Sed 538193326Sedvoid clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) { 539193326Sed // Get the name of the main file. 540193326Sed const SourceManager &SrcMgr = PP.getSourceManager(); 541193326Sed const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID()); 542235633Sdim SmallString<128> MainFilePath(MainFile->getName()); 543198092Srdivacky 544218893Sdim llvm::sys::fs::make_absolute(MainFilePath); 545193326Sed 546193326Sed // Create the PTHWriter. 547193326Sed PTHWriter PW(*OS, PP); 548198092Srdivacky 549193326Sed // Install the 'stat' system call listener in the FileManager. 550198398Srdivacky StatListener *StatCache = new StatListener(PW.getPM()); 551198398Srdivacky PP.getFileManager().addStatCache(StatCache, /*AtBeginning=*/true); 552198092Srdivacky 553193326Sed // Lex through the entire file. This will populate SourceManager with 554193326Sed // all of the header information. 555193326Sed Token Tok; 556207619Srdivacky PP.EnterMainSourceFile(); 557193326Sed do { PP.Lex(Tok); } while (Tok.isNot(tok::eof)); 558198092Srdivacky 559193326Sed // Generate the PTH file. 560198398Srdivacky PP.getFileManager().removeStatCache(StatCache); 561205219Srdivacky PW.GeneratePTH(MainFilePath.str()); 562193326Sed} 563193326Sed 564193326Sed//===----------------------------------------------------------------------===// 565193326Sed 566205219Srdivackynamespace { 567193326Sedclass PTHIdKey { 568193326Sedpublic: 569193326Sed const IdentifierInfo* II; 570193326Sed uint32_t FileOffset; 571193326Sed}; 572193326Sed 573199990Srdivackyclass PTHIdentifierTableTrait { 574193326Sedpublic: 575193326Sed typedef PTHIdKey* key_type; 576193326Sed typedef key_type key_type_ref; 577198092Srdivacky 578193326Sed typedef uint32_t data_type; 579193326Sed typedef data_type data_type_ref; 580198092Srdivacky 581193326Sed static unsigned ComputeHash(PTHIdKey* key) { 582198398Srdivacky return llvm::HashString(key->II->getName()); 583193326Sed } 584198092Srdivacky 585198092Srdivacky static std::pair<unsigned,unsigned> 586226890Sdim EmitKeyDataLength(raw_ostream& Out, const PTHIdKey* key, uint32_t) { 587198398Srdivacky unsigned n = key->II->getLength() + 1; 588193326Sed ::Emit16(Out, n); 589193326Sed return std::make_pair(n, sizeof(uint32_t)); 590193326Sed } 591198092Srdivacky 592226890Sdim static void EmitKey(raw_ostream& Out, PTHIdKey* key, unsigned n) { 593193326Sed // Record the location of the key data. This is used when generating 594193326Sed // the mapping from persistent IDs to strings. 595193326Sed key->FileOffset = Out.tell(); 596198398Srdivacky Out.write(key->II->getNameStart(), n); 597193326Sed } 598198092Srdivacky 599226890Sdim static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID, 600193326Sed unsigned) { 601193326Sed ::Emit32(Out, pID); 602198092Srdivacky } 603193326Sed}; 604193326Sed} // end anonymous namespace 605193326Sed 606193326Sed/// EmitIdentifierTable - Emits two tables to the PTH file. The first is 607193326Sed/// a hashtable mapping from identifier strings to persistent IDs. The second 608193326Sed/// is a straight table mapping from persistent IDs to string data (the 609193326Sed/// keys of the first table). 610193326Sed/// 611193326Sedstd::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() { 612193326Sed // Build two maps: 613193326Sed // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset) 614193326Sed // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs 615193326Sed 616193326Sed // Note that we use 'calloc', so all the bytes are 0. 617193326Sed PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount, sizeof(PTHIdKey)); 618193326Sed 619193326Sed // Create the hashtable. 620193326Sed OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap; 621198092Srdivacky 622193326Sed // Generate mapping from persistent IDs -> IdentifierInfo*. 623193326Sed for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) { 624193326Sed // Decrement by 1 because we are using a vector for the lookup and 625193326Sed // 0 is reserved for NULL. 626193326Sed assert(I->second > 0); 627193326Sed assert(I->second-1 < idcount); 628193326Sed unsigned idx = I->second-1; 629198092Srdivacky 630193326Sed // Store the mapping from persistent ID to IdentifierInfo* 631193326Sed IIDMap[idx].II = I->first; 632198092Srdivacky 633193326Sed // Store the reverse mapping in a hashtable. 634193326Sed IIOffMap.insert(&IIDMap[idx], I->second); 635193326Sed } 636198092Srdivacky 637193326Sed // Write out the inverse map first. This causes the PCIDKey entries to 638193326Sed // record PTH file offsets for the string data. This is used to write 639193326Sed // the second table. 640193326Sed Offset StringTableOffset = IIOffMap.Emit(Out); 641198092Srdivacky 642198092Srdivacky // Now emit the table mapping from persistent IDs to PTH file offsets. 643193326Sed Offset IDOff = Out.tell(); 644193326Sed Emit32(idcount); // Emit the number of identifiers. 645193326Sed for (unsigned i = 0 ; i < idcount; ++i) 646193326Sed Emit32(IIDMap[i].FileOffset); 647198092Srdivacky 648193326Sed // Finally, release the inverse map. 649193326Sed free(IIDMap); 650198092Srdivacky 651193326Sed return std::make_pair(IDOff, StringTableOffset); 652193326Sed} 653