1249261Sdim//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===// 2249261Sdim// 3249261Sdim// The LLVM Compiler Infrastructure 4249261Sdim// 5249261Sdim// This file is distributed under the University of Illinois Open Source 6249261Sdim// License. See LICENSE.TXT for details. 7249261Sdim// 8249261Sdim//===----------------------------------------------------------------------===// 9249261Sdim// 10249261Sdim// This file implements the GlobalModuleIndex class. 11249261Sdim// 12249261Sdim//===----------------------------------------------------------------------===// 13249261Sdim 14249261Sdim#include "ASTReaderInternals.h" 15249261Sdim#include "clang/Basic/FileManager.h" 16249261Sdim#include "clang/Basic/OnDiskHashTable.h" 17249261Sdim#include "clang/Serialization/ASTBitCodes.h" 18249261Sdim#include "clang/Serialization/GlobalModuleIndex.h" 19249261Sdim#include "clang/Serialization/Module.h" 20249261Sdim#include "llvm/ADT/DenseMap.h" 21249261Sdim#include "llvm/ADT/MapVector.h" 22249261Sdim#include "llvm/ADT/SmallString.h" 23249261Sdim#include "llvm/ADT/StringExtras.h" 24249261Sdim#include "llvm/Bitcode/BitstreamReader.h" 25249261Sdim#include "llvm/Bitcode/BitstreamWriter.h" 26249261Sdim#include "llvm/Support/FileSystem.h" 27249261Sdim#include "llvm/Support/LockFileManager.h" 28249261Sdim#include "llvm/Support/MemoryBuffer.h" 29263509Sdim#include "llvm/Support/Path.h" 30249261Sdim#include <cstdio> 31249261Sdimusing namespace clang; 32249261Sdimusing namespace serialization; 33249261Sdim 34249261Sdim//----------------------------------------------------------------------------// 35249261Sdim// Shared constants 36249261Sdim//----------------------------------------------------------------------------// 37249261Sdimnamespace { 38249261Sdim enum { 39249261Sdim /// \brief The block containing the index. 40249261Sdim GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID 41249261Sdim }; 42249261Sdim 43249261Sdim /// \brief Describes the record types in the index. 44249261Sdim enum IndexRecordTypes { 45249261Sdim /// \brief Contains version information and potentially other metadata, 46249261Sdim /// used to determine if we can read this global index file. 47249261Sdim INDEX_METADATA, 48249261Sdim /// \brief Describes a module, including its file name and dependencies. 49249261Sdim MODULE, 50249261Sdim /// \brief The index for identifiers. 51249261Sdim IDENTIFIER_INDEX 52249261Sdim }; 53249261Sdim} 54249261Sdim 55249261Sdim/// \brief The name of the global index file. 56249261Sdimstatic const char * const IndexFileName = "modules.idx"; 57249261Sdim 58249261Sdim/// \brief The global index file version. 59249261Sdimstatic const unsigned CurrentVersion = 1; 60249261Sdim 61249261Sdim//----------------------------------------------------------------------------// 62249261Sdim// Global module index reader. 63249261Sdim//----------------------------------------------------------------------------// 64249261Sdim 65249261Sdimnamespace { 66249261Sdim 67249261Sdim/// \brief Trait used to read the identifier index from the on-disk hash 68249261Sdim/// table. 69249261Sdimclass IdentifierIndexReaderTrait { 70249261Sdimpublic: 71249261Sdim typedef StringRef external_key_type; 72249261Sdim typedef StringRef internal_key_type; 73249261Sdim typedef SmallVector<unsigned, 2> data_type; 74249261Sdim 75249261Sdim static bool EqualKey(const internal_key_type& a, const internal_key_type& b) { 76249261Sdim return a == b; 77249261Sdim } 78249261Sdim 79249261Sdim static unsigned ComputeHash(const internal_key_type& a) { 80249261Sdim return llvm::HashString(a); 81249261Sdim } 82249261Sdim 83249261Sdim static std::pair<unsigned, unsigned> 84249261Sdim ReadKeyDataLength(const unsigned char*& d) { 85249261Sdim using namespace clang::io; 86249261Sdim unsigned KeyLen = ReadUnalignedLE16(d); 87249261Sdim unsigned DataLen = ReadUnalignedLE16(d); 88249261Sdim return std::make_pair(KeyLen, DataLen); 89249261Sdim } 90249261Sdim 91249261Sdim static const internal_key_type& 92249261Sdim GetInternalKey(const external_key_type& x) { return x; } 93249261Sdim 94249261Sdim static const external_key_type& 95249261Sdim GetExternalKey(const internal_key_type& x) { return x; } 96249261Sdim 97249261Sdim static internal_key_type ReadKey(const unsigned char* d, unsigned n) { 98249261Sdim return StringRef((const char *)d, n); 99249261Sdim } 100249261Sdim 101249261Sdim static data_type ReadData(const internal_key_type& k, 102249261Sdim const unsigned char* d, 103249261Sdim unsigned DataLen) { 104249261Sdim using namespace clang::io; 105249261Sdim 106249261Sdim data_type Result; 107249261Sdim while (DataLen > 0) { 108249261Sdim unsigned ID = ReadUnalignedLE32(d); 109249261Sdim Result.push_back(ID); 110249261Sdim DataLen -= 4; 111249261Sdim } 112249261Sdim 113249261Sdim return Result; 114249261Sdim } 115249261Sdim}; 116249261Sdim 117249261Sdimtypedef OnDiskChainedHashTable<IdentifierIndexReaderTrait> IdentifierIndexTable; 118249261Sdim 119249261Sdim} 120249261Sdim 121249261SdimGlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer, 122249261Sdim llvm::BitstreamCursor Cursor) 123249261Sdim : Buffer(Buffer), IdentifierIndex(), 124249261Sdim NumIdentifierLookups(), NumIdentifierLookupHits() 125249261Sdim{ 126249261Sdim // Read the global index. 127249261Sdim bool InGlobalIndexBlock = false; 128249261Sdim bool Done = false; 129249261Sdim while (!Done) { 130249261Sdim llvm::BitstreamEntry Entry = Cursor.advance(); 131249261Sdim 132249261Sdim switch (Entry.Kind) { 133249261Sdim case llvm::BitstreamEntry::Error: 134249261Sdim return; 135249261Sdim 136249261Sdim case llvm::BitstreamEntry::EndBlock: 137249261Sdim if (InGlobalIndexBlock) { 138249261Sdim InGlobalIndexBlock = false; 139249261Sdim Done = true; 140249261Sdim continue; 141249261Sdim } 142249261Sdim return; 143249261Sdim 144249261Sdim 145249261Sdim case llvm::BitstreamEntry::Record: 146249261Sdim // Entries in the global index block are handled below. 147249261Sdim if (InGlobalIndexBlock) 148249261Sdim break; 149249261Sdim 150249261Sdim return; 151249261Sdim 152249261Sdim case llvm::BitstreamEntry::SubBlock: 153249261Sdim if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) { 154249261Sdim if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID)) 155249261Sdim return; 156249261Sdim 157249261Sdim InGlobalIndexBlock = true; 158249261Sdim } else if (Cursor.SkipBlock()) { 159249261Sdim return; 160249261Sdim } 161249261Sdim continue; 162249261Sdim } 163249261Sdim 164249261Sdim SmallVector<uint64_t, 64> Record; 165249261Sdim StringRef Blob; 166249261Sdim switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) { 167249261Sdim case INDEX_METADATA: 168249261Sdim // Make sure that the version matches. 169249261Sdim if (Record.size() < 1 || Record[0] != CurrentVersion) 170249261Sdim return; 171249261Sdim break; 172249261Sdim 173249261Sdim case MODULE: { 174249261Sdim unsigned Idx = 0; 175249261Sdim unsigned ID = Record[Idx++]; 176249261Sdim 177249261Sdim // Make room for this module's information. 178249261Sdim if (ID == Modules.size()) 179249261Sdim Modules.push_back(ModuleInfo()); 180249261Sdim else 181249261Sdim Modules.resize(ID + 1); 182249261Sdim 183249261Sdim // Size/modification time for this module file at the time the 184249261Sdim // global index was built. 185249261Sdim Modules[ID].Size = Record[Idx++]; 186249261Sdim Modules[ID].ModTime = Record[Idx++]; 187249261Sdim 188249261Sdim // File name. 189249261Sdim unsigned NameLen = Record[Idx++]; 190249261Sdim Modules[ID].FileName.assign(Record.begin() + Idx, 191249261Sdim Record.begin() + Idx + NameLen); 192249261Sdim Idx += NameLen; 193249261Sdim 194249261Sdim // Dependencies 195249261Sdim unsigned NumDeps = Record[Idx++]; 196249261Sdim Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(), 197249261Sdim Record.begin() + Idx, 198249261Sdim Record.begin() + Idx + NumDeps); 199249261Sdim Idx += NumDeps; 200249261Sdim 201249261Sdim // Make sure we're at the end of the record. 202249261Sdim assert(Idx == Record.size() && "More module info?"); 203249261Sdim 204249261Sdim // Record this module as an unresolved module. 205249261Sdim UnresolvedModules[llvm::sys::path::stem(Modules[ID].FileName)] = ID; 206249261Sdim break; 207249261Sdim } 208249261Sdim 209249261Sdim case IDENTIFIER_INDEX: 210249261Sdim // Wire up the identifier index. 211249261Sdim if (Record[0]) { 212249261Sdim IdentifierIndex = IdentifierIndexTable::Create( 213249261Sdim (const unsigned char *)Blob.data() + Record[0], 214249261Sdim (const unsigned char *)Blob.data(), 215249261Sdim IdentifierIndexReaderTrait()); 216249261Sdim } 217249261Sdim break; 218249261Sdim } 219249261Sdim } 220249261Sdim} 221249261Sdim 222249261SdimGlobalModuleIndex::~GlobalModuleIndex() { } 223249261Sdim 224249261Sdimstd::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode> 225249261SdimGlobalModuleIndex::readIndex(StringRef Path) { 226249261Sdim // Load the index file, if it's there. 227249261Sdim llvm::SmallString<128> IndexPath; 228249261Sdim IndexPath += Path; 229249261Sdim llvm::sys::path::append(IndexPath, IndexFileName); 230249261Sdim 231249261Sdim llvm::OwningPtr<llvm::MemoryBuffer> Buffer; 232263509Sdim if (llvm::MemoryBuffer::getFile(IndexPath.c_str(), Buffer) != 233263509Sdim llvm::errc::success) 234249261Sdim return std::make_pair((GlobalModuleIndex *)0, EC_NotFound); 235249261Sdim 236249261Sdim /// \brief The bitstream reader from which we'll read the AST file. 237249261Sdim llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(), 238249261Sdim (const unsigned char *)Buffer->getBufferEnd()); 239249261Sdim 240249261Sdim /// \brief The main bitstream cursor for the main block. 241249261Sdim llvm::BitstreamCursor Cursor(Reader); 242249261Sdim 243249261Sdim // Sniff for the signature. 244249261Sdim if (Cursor.Read(8) != 'B' || 245249261Sdim Cursor.Read(8) != 'C' || 246249261Sdim Cursor.Read(8) != 'G' || 247249261Sdim Cursor.Read(8) != 'I') { 248249261Sdim return std::make_pair((GlobalModuleIndex *)0, EC_IOError); 249249261Sdim } 250249261Sdim 251249261Sdim return std::make_pair(new GlobalModuleIndex(Buffer.take(), Cursor), EC_None); 252249261Sdim} 253249261Sdim 254249261Sdimvoid 255249261SdimGlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) { 256249261Sdim ModuleFiles.clear(); 257249261Sdim for (unsigned I = 0, N = Modules.size(); I != N; ++I) { 258249261Sdim if (ModuleFile *MF = Modules[I].File) 259249261Sdim ModuleFiles.push_back(MF); 260249261Sdim } 261249261Sdim} 262249261Sdim 263249261Sdimvoid GlobalModuleIndex::getModuleDependencies( 264249261Sdim ModuleFile *File, 265249261Sdim SmallVectorImpl<ModuleFile *> &Dependencies) { 266249261Sdim // Look for information about this module file. 267249261Sdim llvm::DenseMap<ModuleFile *, unsigned>::iterator Known 268249261Sdim = ModulesByFile.find(File); 269249261Sdim if (Known == ModulesByFile.end()) 270249261Sdim return; 271249261Sdim 272249261Sdim // Record dependencies. 273249261Sdim Dependencies.clear(); 274249261Sdim ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies; 275249261Sdim for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) { 276249261Sdim if (ModuleFile *MF = Modules[I].File) 277249261Sdim Dependencies.push_back(MF); 278249261Sdim } 279249261Sdim} 280249261Sdim 281249261Sdimbool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) { 282249261Sdim Hits.clear(); 283249261Sdim 284249261Sdim // If there's no identifier index, there is nothing we can do. 285249261Sdim if (!IdentifierIndex) 286249261Sdim return false; 287249261Sdim 288249261Sdim // Look into the identifier index. 289249261Sdim ++NumIdentifierLookups; 290249261Sdim IdentifierIndexTable &Table 291249261Sdim = *static_cast<IdentifierIndexTable *>(IdentifierIndex); 292249261Sdim IdentifierIndexTable::iterator Known = Table.find(Name); 293249261Sdim if (Known == Table.end()) { 294249261Sdim return true; 295249261Sdim } 296249261Sdim 297249261Sdim SmallVector<unsigned, 2> ModuleIDs = *Known; 298249261Sdim for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) { 299249261Sdim if (ModuleFile *MF = Modules[ModuleIDs[I]].File) 300249261Sdim Hits.insert(MF); 301249261Sdim } 302249261Sdim 303249261Sdim ++NumIdentifierLookupHits; 304249261Sdim return true; 305249261Sdim} 306249261Sdim 307249261Sdimbool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) { 308249261Sdim // Look for the module in the global module index based on the module name. 309249261Sdim StringRef Name = llvm::sys::path::stem(File->FileName); 310249261Sdim llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name); 311249261Sdim if (Known == UnresolvedModules.end()) { 312249261Sdim return true; 313249261Sdim } 314249261Sdim 315249261Sdim // Rectify this module with the global module index. 316249261Sdim ModuleInfo &Info = Modules[Known->second]; 317249261Sdim 318249261Sdim // If the size and modification time match what we expected, record this 319249261Sdim // module file. 320249261Sdim bool Failed = true; 321249261Sdim if (File->File->getSize() == Info.Size && 322249261Sdim File->File->getModificationTime() == Info.ModTime) { 323249261Sdim Info.File = File; 324249261Sdim ModulesByFile[File] = Known->second; 325249261Sdim 326249261Sdim Failed = false; 327249261Sdim } 328249261Sdim 329249261Sdim // One way or another, we have resolved this module file. 330249261Sdim UnresolvedModules.erase(Known); 331249261Sdim return Failed; 332249261Sdim} 333249261Sdim 334249261Sdimvoid GlobalModuleIndex::printStats() { 335249261Sdim std::fprintf(stderr, "*** Global Module Index Statistics:\n"); 336249261Sdim if (NumIdentifierLookups) { 337249261Sdim fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n", 338249261Sdim NumIdentifierLookupHits, NumIdentifierLookups, 339249261Sdim (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups); 340249261Sdim } 341249261Sdim std::fprintf(stderr, "\n"); 342249261Sdim} 343249261Sdim 344249261Sdim//----------------------------------------------------------------------------// 345249261Sdim// Global module index writer. 346249261Sdim//----------------------------------------------------------------------------// 347249261Sdim 348249261Sdimnamespace { 349249261Sdim /// \brief Provides information about a specific module file. 350249261Sdim struct ModuleFileInfo { 351249261Sdim /// \brief The numberic ID for this module file. 352249261Sdim unsigned ID; 353249261Sdim 354249261Sdim /// \brief The set of modules on which this module depends. Each entry is 355249261Sdim /// a module ID. 356249261Sdim SmallVector<unsigned, 4> Dependencies; 357249261Sdim }; 358249261Sdim 359249261Sdim /// \brief Builder that generates the global module index file. 360249261Sdim class GlobalModuleIndexBuilder { 361249261Sdim FileManager &FileMgr; 362249261Sdim 363249261Sdim /// \brief Mapping from files to module file information. 364249261Sdim typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap; 365249261Sdim 366249261Sdim /// \brief Information about each of the known module files. 367249261Sdim ModuleFilesMap ModuleFiles; 368249261Sdim 369249261Sdim /// \brief Mapping from identifiers to the list of module file IDs that 370249261Sdim /// consider this identifier to be interesting. 371249261Sdim typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap; 372249261Sdim 373249261Sdim /// \brief A mapping from all interesting identifiers to the set of module 374249261Sdim /// files in which those identifiers are considered interesting. 375249261Sdim InterestingIdentifierMap InterestingIdentifiers; 376249261Sdim 377249261Sdim /// \brief Write the block-info block for the global module index file. 378249261Sdim void emitBlockInfoBlock(llvm::BitstreamWriter &Stream); 379249261Sdim 380249261Sdim /// \brief Retrieve the module file information for the given file. 381249261Sdim ModuleFileInfo &getModuleFileInfo(const FileEntry *File) { 382249261Sdim llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known 383249261Sdim = ModuleFiles.find(File); 384249261Sdim if (Known != ModuleFiles.end()) 385249261Sdim return Known->second; 386249261Sdim 387249261Sdim unsigned NewID = ModuleFiles.size(); 388249261Sdim ModuleFileInfo &Info = ModuleFiles[File]; 389249261Sdim Info.ID = NewID; 390249261Sdim return Info; 391249261Sdim } 392249261Sdim 393249261Sdim public: 394249261Sdim explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){} 395249261Sdim 396249261Sdim /// \brief Load the contents of the given module file into the builder. 397249261Sdim /// 398249261Sdim /// \returns true if an error occurred, false otherwise. 399249261Sdim bool loadModuleFile(const FileEntry *File); 400249261Sdim 401249261Sdim /// \brief Write the index to the given bitstream. 402249261Sdim void writeIndex(llvm::BitstreamWriter &Stream); 403249261Sdim }; 404249261Sdim} 405249261Sdim 406249261Sdimstatic void emitBlockID(unsigned ID, const char *Name, 407249261Sdim llvm::BitstreamWriter &Stream, 408249261Sdim SmallVectorImpl<uint64_t> &Record) { 409249261Sdim Record.clear(); 410249261Sdim Record.push_back(ID); 411249261Sdim Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record); 412249261Sdim 413249261Sdim // Emit the block name if present. 414249261Sdim if (Name == 0 || Name[0] == 0) return; 415249261Sdim Record.clear(); 416249261Sdim while (*Name) 417249261Sdim Record.push_back(*Name++); 418249261Sdim Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record); 419249261Sdim} 420249261Sdim 421249261Sdimstatic void emitRecordID(unsigned ID, const char *Name, 422249261Sdim llvm::BitstreamWriter &Stream, 423249261Sdim SmallVectorImpl<uint64_t> &Record) { 424249261Sdim Record.clear(); 425249261Sdim Record.push_back(ID); 426249261Sdim while (*Name) 427249261Sdim Record.push_back(*Name++); 428249261Sdim Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record); 429249261Sdim} 430249261Sdim 431249261Sdimvoid 432249261SdimGlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) { 433249261Sdim SmallVector<uint64_t, 64> Record; 434249261Sdim Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3); 435249261Sdim 436249261Sdim#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) 437249261Sdim#define RECORD(X) emitRecordID(X, #X, Stream, Record) 438249261Sdim BLOCK(GLOBAL_INDEX_BLOCK); 439249261Sdim RECORD(INDEX_METADATA); 440249261Sdim RECORD(MODULE); 441249261Sdim RECORD(IDENTIFIER_INDEX); 442249261Sdim#undef RECORD 443249261Sdim#undef BLOCK 444249261Sdim 445249261Sdim Stream.ExitBlock(); 446249261Sdim} 447249261Sdim 448249261Sdimnamespace { 449249261Sdim class InterestingASTIdentifierLookupTrait 450249261Sdim : public serialization::reader::ASTIdentifierLookupTraitBase { 451249261Sdim 452249261Sdim public: 453249261Sdim /// \brief The identifier and whether it is "interesting". 454249261Sdim typedef std::pair<StringRef, bool> data_type; 455249261Sdim 456249261Sdim data_type ReadData(const internal_key_type& k, 457249261Sdim const unsigned char* d, 458249261Sdim unsigned DataLen) { 459249261Sdim // The first bit indicates whether this identifier is interesting. 460249261Sdim // That's all we care about. 461249261Sdim using namespace clang::io; 462249261Sdim unsigned RawID = ReadUnalignedLE32(d); 463249261Sdim bool IsInteresting = RawID & 0x01; 464249261Sdim return std::make_pair(k, IsInteresting); 465249261Sdim } 466249261Sdim }; 467249261Sdim} 468249261Sdim 469249261Sdimbool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) { 470249261Sdim // Open the module file. 471249261Sdim OwningPtr<llvm::MemoryBuffer> Buffer; 472249261Sdim std::string ErrorStr; 473249261Sdim Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true)); 474249261Sdim if (!Buffer) { 475249261Sdim return true; 476249261Sdim } 477249261Sdim 478249261Sdim // Initialize the input stream 479249261Sdim llvm::BitstreamReader InStreamFile; 480249261Sdim llvm::BitstreamCursor InStream; 481249261Sdim InStreamFile.init((const unsigned char *)Buffer->getBufferStart(), 482249261Sdim (const unsigned char *)Buffer->getBufferEnd()); 483249261Sdim InStream.init(InStreamFile); 484249261Sdim 485249261Sdim // Sniff for the signature. 486249261Sdim if (InStream.Read(8) != 'C' || 487249261Sdim InStream.Read(8) != 'P' || 488249261Sdim InStream.Read(8) != 'C' || 489249261Sdim InStream.Read(8) != 'H') { 490249261Sdim return true; 491249261Sdim } 492249261Sdim 493249261Sdim // Record this module file and assign it a unique ID (if it doesn't have 494249261Sdim // one already). 495249261Sdim unsigned ID = getModuleFileInfo(File).ID; 496249261Sdim 497249261Sdim // Search for the blocks and records we care about. 498249261Sdim enum { Other, ControlBlock, ASTBlock } State = Other; 499249261Sdim bool Done = false; 500249261Sdim while (!Done) { 501249261Sdim llvm::BitstreamEntry Entry = InStream.advance(); 502249261Sdim switch (Entry.Kind) { 503249261Sdim case llvm::BitstreamEntry::Error: 504249261Sdim Done = true; 505249261Sdim continue; 506249261Sdim 507249261Sdim case llvm::BitstreamEntry::Record: 508249261Sdim // In the 'other' state, just skip the record. We don't care. 509249261Sdim if (State == Other) { 510249261Sdim InStream.skipRecord(Entry.ID); 511249261Sdim continue; 512249261Sdim } 513249261Sdim 514249261Sdim // Handle potentially-interesting records below. 515249261Sdim break; 516249261Sdim 517249261Sdim case llvm::BitstreamEntry::SubBlock: 518249261Sdim if (Entry.ID == CONTROL_BLOCK_ID) { 519249261Sdim if (InStream.EnterSubBlock(CONTROL_BLOCK_ID)) 520249261Sdim return true; 521249261Sdim 522249261Sdim // Found the control block. 523249261Sdim State = ControlBlock; 524249261Sdim continue; 525249261Sdim } 526249261Sdim 527249261Sdim if (Entry.ID == AST_BLOCK_ID) { 528249261Sdim if (InStream.EnterSubBlock(AST_BLOCK_ID)) 529249261Sdim return true; 530249261Sdim 531249261Sdim // Found the AST block. 532249261Sdim State = ASTBlock; 533249261Sdim continue; 534249261Sdim } 535249261Sdim 536249261Sdim if (InStream.SkipBlock()) 537249261Sdim return true; 538249261Sdim 539249261Sdim continue; 540249261Sdim 541249261Sdim case llvm::BitstreamEntry::EndBlock: 542249261Sdim State = Other; 543249261Sdim continue; 544249261Sdim } 545249261Sdim 546249261Sdim // Read the given record. 547249261Sdim SmallVector<uint64_t, 64> Record; 548249261Sdim StringRef Blob; 549249261Sdim unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob); 550249261Sdim 551249261Sdim // Handle module dependencies. 552249261Sdim if (State == ControlBlock && Code == IMPORTS) { 553249261Sdim // Load each of the imported PCH files. 554249261Sdim unsigned Idx = 0, N = Record.size(); 555249261Sdim while (Idx < N) { 556249261Sdim // Read information about the AST file. 557249261Sdim 558249261Sdim // Skip the imported kind 559249261Sdim ++Idx; 560249261Sdim 561249261Sdim // Skip the import location 562249261Sdim ++Idx; 563249261Sdim 564249261Sdim // Load stored size/modification time. 565249261Sdim off_t StoredSize = (off_t)Record[Idx++]; 566249261Sdim time_t StoredModTime = (time_t)Record[Idx++]; 567249261Sdim 568249261Sdim // Retrieve the imported file name. 569249261Sdim unsigned Length = Record[Idx++]; 570249261Sdim SmallString<128> ImportedFile(Record.begin() + Idx, 571249261Sdim Record.begin() + Idx + Length); 572249261Sdim Idx += Length; 573249261Sdim 574249261Sdim // Find the imported module file. 575249261Sdim const FileEntry *DependsOnFile 576249261Sdim = FileMgr.getFile(ImportedFile, /*openFile=*/false, 577249261Sdim /*cacheFailure=*/false); 578249261Sdim if (!DependsOnFile || 579249261Sdim (StoredSize != DependsOnFile->getSize()) || 580249261Sdim (StoredModTime != DependsOnFile->getModificationTime())) 581249261Sdim return true; 582249261Sdim 583249261Sdim // Record the dependency. 584249261Sdim unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID; 585249261Sdim getModuleFileInfo(File).Dependencies.push_back(DependsOnID); 586249261Sdim } 587249261Sdim 588249261Sdim continue; 589249261Sdim } 590249261Sdim 591249261Sdim // Handle the identifier table 592249261Sdim if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) { 593249261Sdim typedef OnDiskChainedHashTable<InterestingASTIdentifierLookupTrait> 594249261Sdim InterestingIdentifierTable; 595249261Sdim llvm::OwningPtr<InterestingIdentifierTable> 596249261Sdim Table(InterestingIdentifierTable::Create( 597249261Sdim (const unsigned char *)Blob.data() + Record[0], 598249261Sdim (const unsigned char *)Blob.data())); 599249261Sdim for (InterestingIdentifierTable::data_iterator D = Table->data_begin(), 600249261Sdim DEnd = Table->data_end(); 601249261Sdim D != DEnd; ++D) { 602249261Sdim std::pair<StringRef, bool> Ident = *D; 603249261Sdim if (Ident.second) 604249261Sdim InterestingIdentifiers[Ident.first].push_back(ID); 605249261Sdim else 606249261Sdim (void)InterestingIdentifiers[Ident.first]; 607249261Sdim } 608249261Sdim } 609249261Sdim 610249261Sdim // We don't care about this record. 611249261Sdim } 612249261Sdim 613249261Sdim return false; 614249261Sdim} 615249261Sdim 616249261Sdimnamespace { 617249261Sdim 618249261Sdim/// \brief Trait used to generate the identifier index as an on-disk hash 619249261Sdim/// table. 620249261Sdimclass IdentifierIndexWriterTrait { 621249261Sdimpublic: 622249261Sdim typedef StringRef key_type; 623249261Sdim typedef StringRef key_type_ref; 624249261Sdim typedef SmallVector<unsigned, 2> data_type; 625249261Sdim typedef const SmallVector<unsigned, 2> &data_type_ref; 626249261Sdim 627249261Sdim static unsigned ComputeHash(key_type_ref Key) { 628249261Sdim return llvm::HashString(Key); 629249261Sdim } 630249261Sdim 631249261Sdim std::pair<unsigned,unsigned> 632249261Sdim EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) { 633249261Sdim unsigned KeyLen = Key.size(); 634249261Sdim unsigned DataLen = Data.size() * 4; 635249261Sdim clang::io::Emit16(Out, KeyLen); 636249261Sdim clang::io::Emit16(Out, DataLen); 637249261Sdim return std::make_pair(KeyLen, DataLen); 638249261Sdim } 639249261Sdim 640249261Sdim void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) { 641249261Sdim Out.write(Key.data(), KeyLen); 642249261Sdim } 643249261Sdim 644249261Sdim void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data, 645249261Sdim unsigned DataLen) { 646249261Sdim for (unsigned I = 0, N = Data.size(); I != N; ++I) 647249261Sdim clang::io::Emit32(Out, Data[I]); 648249261Sdim } 649249261Sdim}; 650249261Sdim 651249261Sdim} 652249261Sdim 653249261Sdimvoid GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { 654249261Sdim using namespace llvm; 655249261Sdim 656249261Sdim // Emit the file header. 657249261Sdim Stream.Emit((unsigned)'B', 8); 658249261Sdim Stream.Emit((unsigned)'C', 8); 659249261Sdim Stream.Emit((unsigned)'G', 8); 660249261Sdim Stream.Emit((unsigned)'I', 8); 661249261Sdim 662249261Sdim // Write the block-info block, which describes the records in this bitcode 663249261Sdim // file. 664249261Sdim emitBlockInfoBlock(Stream); 665249261Sdim 666249261Sdim Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3); 667249261Sdim 668249261Sdim // Write the metadata. 669249261Sdim SmallVector<uint64_t, 2> Record; 670249261Sdim Record.push_back(CurrentVersion); 671249261Sdim Stream.EmitRecord(INDEX_METADATA, Record); 672249261Sdim 673249261Sdim // Write the set of known module files. 674249261Sdim for (ModuleFilesMap::iterator M = ModuleFiles.begin(), 675249261Sdim MEnd = ModuleFiles.end(); 676249261Sdim M != MEnd; ++M) { 677249261Sdim Record.clear(); 678249261Sdim Record.push_back(M->second.ID); 679249261Sdim Record.push_back(M->first->getSize()); 680249261Sdim Record.push_back(M->first->getModificationTime()); 681249261Sdim 682249261Sdim // File name 683249261Sdim StringRef Name(M->first->getName()); 684249261Sdim Record.push_back(Name.size()); 685249261Sdim Record.append(Name.begin(), Name.end()); 686249261Sdim 687249261Sdim // Dependencies 688249261Sdim Record.push_back(M->second.Dependencies.size()); 689249261Sdim Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end()); 690249261Sdim Stream.EmitRecord(MODULE, Record); 691249261Sdim } 692249261Sdim 693249261Sdim // Write the identifier -> module file mapping. 694249261Sdim { 695249261Sdim OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator; 696249261Sdim IdentifierIndexWriterTrait Trait; 697249261Sdim 698249261Sdim // Populate the hash table. 699249261Sdim for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(), 700249261Sdim IEnd = InterestingIdentifiers.end(); 701249261Sdim I != IEnd; ++I) { 702249261Sdim Generator.insert(I->first(), I->second, Trait); 703249261Sdim } 704249261Sdim 705249261Sdim // Create the on-disk hash table in a buffer. 706249261Sdim SmallString<4096> IdentifierTable; 707249261Sdim uint32_t BucketOffset; 708249261Sdim { 709249261Sdim llvm::raw_svector_ostream Out(IdentifierTable); 710249261Sdim // Make sure that no bucket is at offset 0 711249261Sdim clang::io::Emit32(Out, 0); 712249261Sdim BucketOffset = Generator.Emit(Out, Trait); 713249261Sdim } 714249261Sdim 715249261Sdim // Create a blob abbreviation 716249261Sdim BitCodeAbbrev *Abbrev = new BitCodeAbbrev(); 717249261Sdim Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX)); 718249261Sdim Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); 719249261Sdim Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); 720249261Sdim unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev); 721249261Sdim 722249261Sdim // Write the identifier table 723249261Sdim Record.clear(); 724249261Sdim Record.push_back(IDENTIFIER_INDEX); 725249261Sdim Record.push_back(BucketOffset); 726249261Sdim Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str()); 727249261Sdim } 728249261Sdim 729249261Sdim Stream.ExitBlock(); 730249261Sdim} 731249261Sdim 732249261SdimGlobalModuleIndex::ErrorCode 733249261SdimGlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) { 734249261Sdim llvm::SmallString<128> IndexPath; 735249261Sdim IndexPath += Path; 736249261Sdim llvm::sys::path::append(IndexPath, IndexFileName); 737249261Sdim 738249261Sdim // Coordinate building the global index file with other processes that might 739249261Sdim // try to do the same. 740249261Sdim llvm::LockFileManager Locked(IndexPath); 741249261Sdim switch (Locked) { 742249261Sdim case llvm::LockFileManager::LFS_Error: 743249261Sdim return EC_IOError; 744249261Sdim 745249261Sdim case llvm::LockFileManager::LFS_Owned: 746249261Sdim // We're responsible for building the index ourselves. Do so below. 747249261Sdim break; 748249261Sdim 749249261Sdim case llvm::LockFileManager::LFS_Shared: 750249261Sdim // Someone else is responsible for building the index. We don't care 751249261Sdim // when they finish, so we're done. 752249261Sdim return EC_Building; 753249261Sdim } 754249261Sdim 755249261Sdim // The module index builder. 756249261Sdim GlobalModuleIndexBuilder Builder(FileMgr); 757249261Sdim 758249261Sdim // Load each of the module files. 759249261Sdim llvm::error_code EC; 760249261Sdim for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd; 761249261Sdim D != DEnd && !EC; 762249261Sdim D.increment(EC)) { 763249261Sdim // If this isn't a module file, we don't care. 764249261Sdim if (llvm::sys::path::extension(D->path()) != ".pcm") { 765249261Sdim // ... unless it's a .pcm.lock file, which indicates that someone is 766249261Sdim // in the process of rebuilding a module. They'll rebuild the index 767249261Sdim // at the end of that translation unit, so we don't have to. 768249261Sdim if (llvm::sys::path::extension(D->path()) == ".pcm.lock") 769249261Sdim return EC_Building; 770249261Sdim 771249261Sdim continue; 772249261Sdim } 773249261Sdim 774249261Sdim // If we can't find the module file, skip it. 775249261Sdim const FileEntry *ModuleFile = FileMgr.getFile(D->path()); 776249261Sdim if (!ModuleFile) 777249261Sdim continue; 778249261Sdim 779249261Sdim // Load this module file. 780249261Sdim if (Builder.loadModuleFile(ModuleFile)) 781249261Sdim return EC_IOError; 782249261Sdim } 783249261Sdim 784249261Sdim // The output buffer, into which the global index will be written. 785249261Sdim SmallVector<char, 16> OutputBuffer; 786249261Sdim { 787249261Sdim llvm::BitstreamWriter OutputStream(OutputBuffer); 788249261Sdim Builder.writeIndex(OutputStream); 789249261Sdim } 790249261Sdim 791249261Sdim // Write the global index file to a temporary file. 792249261Sdim llvm::SmallString<128> IndexTmpPath; 793249261Sdim int TmpFD; 794263509Sdim if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD, 795263509Sdim IndexTmpPath)) 796249261Sdim return EC_IOError; 797249261Sdim 798249261Sdim // Open the temporary global index file for output. 799249261Sdim llvm::raw_fd_ostream Out(TmpFD, true); 800249261Sdim if (Out.has_error()) 801249261Sdim return EC_IOError; 802249261Sdim 803249261Sdim // Write the index. 804249261Sdim Out.write(OutputBuffer.data(), OutputBuffer.size()); 805249261Sdim Out.close(); 806249261Sdim if (Out.has_error()) 807249261Sdim return EC_IOError; 808249261Sdim 809249261Sdim // Remove the old index file. It isn't relevant any more. 810249261Sdim bool OldIndexExisted; 811249261Sdim llvm::sys::fs::remove(IndexPath.str(), OldIndexExisted); 812249261Sdim 813249261Sdim // Rename the newly-written index file to the proper name. 814249261Sdim if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) { 815249261Sdim // Rename failed; just remove the 816249261Sdim llvm::sys::fs::remove(IndexTmpPath.str(), OldIndexExisted); 817249261Sdim return EC_IOError; 818249261Sdim } 819249261Sdim 820249261Sdim // We're done. 821249261Sdim return EC_None; 822249261Sdim} 823252723Sdim 824252723Sdimnamespace { 825252723Sdim class GlobalIndexIdentifierIterator : public IdentifierIterator { 826252723Sdim /// \brief The current position within the identifier lookup table. 827252723Sdim IdentifierIndexTable::key_iterator Current; 828252723Sdim 829252723Sdim /// \brief The end position within the identifier lookup table. 830252723Sdim IdentifierIndexTable::key_iterator End; 831252723Sdim 832252723Sdim public: 833252723Sdim explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) { 834252723Sdim Current = Idx.key_begin(); 835252723Sdim End = Idx.key_end(); 836252723Sdim } 837252723Sdim 838252723Sdim virtual StringRef Next() { 839252723Sdim if (Current == End) 840252723Sdim return StringRef(); 841252723Sdim 842252723Sdim StringRef Result = *Current; 843252723Sdim ++Current; 844252723Sdim return Result; 845252723Sdim } 846252723Sdim }; 847252723Sdim} 848252723Sdim 849252723SdimIdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const { 850252723Sdim IdentifierIndexTable &Table = 851252723Sdim *static_cast<IdentifierIndexTable *>(IdentifierIndex); 852252723Sdim return new GlobalIndexIdentifierIterator(Table); 853252723Sdim} 854