1//===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9//  This file implements the CrossTranslationUnit interface.
10//
11//===----------------------------------------------------------------------===//
12#include "clang/CrossTU/CrossTranslationUnit.h"
13#include "clang/AST/ASTImporter.h"
14#include "clang/AST/Decl.h"
15#include "clang/AST/ParentMapContext.h"
16#include "clang/Basic/TargetInfo.h"
17#include "clang/CrossTU/CrossTUDiagnostic.h"
18#include "clang/Frontend/ASTUnit.h"
19#include "clang/Frontend/CompilerInstance.h"
20#include "clang/Frontend/TextDiagnosticPrinter.h"
21#include "clang/Index/USRGeneration.h"
22#include "llvm/ADT/Optional.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/ADT/Triple.h"
25#include "llvm/Option/ArgList.h"
26#include "llvm/Support/ErrorHandling.h"
27#include "llvm/Support/ManagedStatic.h"
28#include "llvm/Support/Path.h"
29#include "llvm/Support/YAMLParser.h"
30#include "llvm/Support/raw_ostream.h"
31#include <algorithm>
32#include <fstream>
33#include <sstream>
34#include <tuple>
35
36namespace clang {
37namespace cross_tu {
38
39namespace {
40
41#define DEBUG_TYPE "CrossTranslationUnit"
42STATISTIC(NumGetCTUCalled, "The # of getCTUDefinition function called");
43STATISTIC(
44    NumNotInOtherTU,
45    "The # of getCTUDefinition called but the function is not in any other TU");
46STATISTIC(NumGetCTUSuccess,
47          "The # of getCTUDefinition successfully returned the "
48          "requested function's body");
49STATISTIC(NumUnsupportedNodeFound, "The # of imports when the ASTImporter "
50                                   "encountered an unsupported AST Node");
51STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter "
52                            "encountered an ODR error");
53STATISTIC(NumTripleMismatch, "The # of triple mismatches");
54STATISTIC(NumLangMismatch, "The # of language mismatches");
55STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
56STATISTIC(NumASTLoadThresholdReached,
57          "The # of ASTs not loaded because of threshold");
58
59// Same as Triple's equality operator, but we check a field only if that is
60// known in both instances.
61bool hasEqualKnownFields(const llvm::Triple &Lhs, const llvm::Triple &Rhs) {
62  using llvm::Triple;
63  if (Lhs.getArch() != Triple::UnknownArch &&
64      Rhs.getArch() != Triple::UnknownArch && Lhs.getArch() != Rhs.getArch())
65    return false;
66  if (Lhs.getSubArch() != Triple::NoSubArch &&
67      Rhs.getSubArch() != Triple::NoSubArch &&
68      Lhs.getSubArch() != Rhs.getSubArch())
69    return false;
70  if (Lhs.getVendor() != Triple::UnknownVendor &&
71      Rhs.getVendor() != Triple::UnknownVendor &&
72      Lhs.getVendor() != Rhs.getVendor())
73    return false;
74  if (!Lhs.isOSUnknown() && !Rhs.isOSUnknown() &&
75      Lhs.getOS() != Rhs.getOS())
76    return false;
77  if (Lhs.getEnvironment() != Triple::UnknownEnvironment &&
78      Rhs.getEnvironment() != Triple::UnknownEnvironment &&
79      Lhs.getEnvironment() != Rhs.getEnvironment())
80    return false;
81  if (Lhs.getObjectFormat() != Triple::UnknownObjectFormat &&
82      Rhs.getObjectFormat() != Triple::UnknownObjectFormat &&
83      Lhs.getObjectFormat() != Rhs.getObjectFormat())
84    return false;
85  return true;
86}
87
88// FIXME: This class is will be removed after the transition to llvm::Error.
89class IndexErrorCategory : public std::error_category {
90public:
91  const char *name() const noexcept override { return "clang.index"; }
92
93  std::string message(int Condition) const override {
94    switch (static_cast<index_error_code>(Condition)) {
95    case index_error_code::unspecified:
96      return "An unknown error has occurred.";
97    case index_error_code::missing_index_file:
98      return "The index file is missing.";
99    case index_error_code::invalid_index_format:
100      return "Invalid index file format.";
101    case index_error_code::multiple_definitions:
102      return "Multiple definitions in the index file.";
103    case index_error_code::missing_definition:
104      return "Missing definition from the index file.";
105    case index_error_code::failed_import:
106      return "Failed to import the definition.";
107    case index_error_code::failed_to_get_external_ast:
108      return "Failed to load external AST source.";
109    case index_error_code::failed_to_generate_usr:
110      return "Failed to generate USR.";
111    case index_error_code::triple_mismatch:
112      return "Triple mismatch";
113    case index_error_code::lang_mismatch:
114      return "Language mismatch";
115    case index_error_code::lang_dialect_mismatch:
116      return "Language dialect mismatch";
117    case index_error_code::load_threshold_reached:
118      return "Load threshold reached";
119    case index_error_code::invocation_list_ambiguous:
120      return "Invocation list file contains multiple references to the same "
121             "source file.";
122    case index_error_code::invocation_list_file_not_found:
123      return "Invocation list file is not found.";
124    case index_error_code::invocation_list_empty:
125      return "Invocation list file is empty.";
126    case index_error_code::invocation_list_wrong_format:
127      return "Invocation list file is in wrong format.";
128    case index_error_code::invocation_list_lookup_unsuccessful:
129      return "Invocation list file does not contain the requested source file.";
130    }
131    llvm_unreachable("Unrecognized index_error_code.");
132  }
133};
134
135static llvm::ManagedStatic<IndexErrorCategory> Category;
136} // end anonymous namespace
137
138char IndexError::ID;
139
140void IndexError::log(raw_ostream &OS) const {
141  OS << Category->message(static_cast<int>(Code)) << '\n';
142}
143
144std::error_code IndexError::convertToErrorCode() const {
145  return std::error_code(static_cast<int>(Code), *Category);
146}
147
148llvm::Expected<llvm::StringMap<std::string>>
149parseCrossTUIndex(StringRef IndexPath) {
150  std::ifstream ExternalMapFile{std::string(IndexPath)};
151  if (!ExternalMapFile)
152    return llvm::make_error<IndexError>(index_error_code::missing_index_file,
153                                        IndexPath.str());
154
155  llvm::StringMap<std::string> Result;
156  std::string Line;
157  unsigned LineNo = 1;
158  while (std::getline(ExternalMapFile, Line)) {
159    StringRef LineRef{Line};
160    const size_t Delimiter = LineRef.find(" ");
161    if (Delimiter > 0 && Delimiter != std::string::npos) {
162      StringRef LookupName = LineRef.substr(0, Delimiter);
163
164      // Store paths with posix-style directory separator.
165      SmallVector<char, 32> FilePath;
166      llvm::Twine{LineRef.substr(Delimiter + 1)}.toVector(FilePath);
167      llvm::sys::path::native(FilePath, llvm::sys::path::Style::posix);
168
169      bool InsertionOccured;
170      std::tie(std::ignore, InsertionOccured) =
171          Result.try_emplace(LookupName, FilePath.begin(), FilePath.end());
172      if (!InsertionOccured)
173        return llvm::make_error<IndexError>(
174            index_error_code::multiple_definitions, IndexPath.str(), LineNo);
175    } else
176      return llvm::make_error<IndexError>(
177          index_error_code::invalid_index_format, IndexPath.str(), LineNo);
178    ++LineNo;
179  }
180  return Result;
181}
182
183std::string
184createCrossTUIndexString(const llvm::StringMap<std::string> &Index) {
185  std::ostringstream Result;
186  for (const auto &E : Index)
187    Result << E.getKey().str() << " " << E.getValue() << '\n';
188  return Result.str();
189}
190
191bool containsConst(const VarDecl *VD, const ASTContext &ACtx) {
192  CanQualType CT = ACtx.getCanonicalType(VD->getType());
193  if (!CT.isConstQualified()) {
194    const RecordType *RTy = CT->getAs<RecordType>();
195    if (!RTy || !RTy->hasConstFields())
196      return false;
197  }
198  return true;
199}
200
201static bool hasBodyOrInit(const FunctionDecl *D, const FunctionDecl *&DefD) {
202  return D->hasBody(DefD);
203}
204static bool hasBodyOrInit(const VarDecl *D, const VarDecl *&DefD) {
205  return D->getAnyInitializer(DefD);
206}
207template <typename T> static bool hasBodyOrInit(const T *D) {
208  const T *Unused;
209  return hasBodyOrInit(D, Unused);
210}
211
212CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
213    : Context(CI.getASTContext()), ASTStorage(CI) {}
214
215CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
216
217llvm::Optional<std::string>
218CrossTranslationUnitContext::getLookupName(const NamedDecl *ND) {
219  SmallString<128> DeclUSR;
220  bool Ret = index::generateUSRForDecl(ND, DeclUSR);
221  if (Ret)
222    return {};
223  return std::string(DeclUSR.str());
224}
225
226/// Recursively visits the decls of a DeclContext, and returns one with the
227/// given USR.
228template <typename T>
229const T *
230CrossTranslationUnitContext::findDefInDeclContext(const DeclContext *DC,
231                                                  StringRef LookupName) {
232  assert(DC && "Declaration Context must not be null");
233  for (const Decl *D : DC->decls()) {
234    const auto *SubDC = dyn_cast<DeclContext>(D);
235    if (SubDC)
236      if (const auto *ND = findDefInDeclContext<T>(SubDC, LookupName))
237        return ND;
238
239    const auto *ND = dyn_cast<T>(D);
240    const T *ResultDecl;
241    if (!ND || !hasBodyOrInit(ND, ResultDecl))
242      continue;
243    llvm::Optional<std::string> ResultLookupName = getLookupName(ResultDecl);
244    if (!ResultLookupName || *ResultLookupName != LookupName)
245      continue;
246    return ResultDecl;
247  }
248  return nullptr;
249}
250
251template <typename T>
252llvm::Expected<const T *> CrossTranslationUnitContext::getCrossTUDefinitionImpl(
253    const T *D, StringRef CrossTUDir, StringRef IndexName,
254    bool DisplayCTUProgress) {
255  assert(D && "D is missing, bad call to this function!");
256  assert(!hasBodyOrInit(D) &&
257         "D has a body or init in current translation unit!");
258  ++NumGetCTUCalled;
259  const llvm::Optional<std::string> LookupName = getLookupName(D);
260  if (!LookupName)
261    return llvm::make_error<IndexError>(
262        index_error_code::failed_to_generate_usr);
263  llvm::Expected<ASTUnit *> ASTUnitOrError =
264      loadExternalAST(*LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
265  if (!ASTUnitOrError)
266    return ASTUnitOrError.takeError();
267  ASTUnit *Unit = *ASTUnitOrError;
268  assert(&Unit->getFileManager() ==
269         &Unit->getASTContext().getSourceManager().getFileManager());
270
271  const llvm::Triple &TripleTo = Context.getTargetInfo().getTriple();
272  const llvm::Triple &TripleFrom =
273      Unit->getASTContext().getTargetInfo().getTriple();
274  // The imported AST had been generated for a different target.
275  // Some parts of the triple in the loaded ASTContext can be unknown while the
276  // very same parts in the target ASTContext are known. Thus we check for the
277  // known parts only.
278  if (!hasEqualKnownFields(TripleTo, TripleFrom)) {
279    // TODO: Pass the SourceLocation of the CallExpression for more precise
280    // diagnostics.
281    ++NumTripleMismatch;
282    return llvm::make_error<IndexError>(index_error_code::triple_mismatch,
283                                        std::string(Unit->getMainFileName()),
284                                        TripleTo.str(), TripleFrom.str());
285  }
286
287  const auto &LangTo = Context.getLangOpts();
288  const auto &LangFrom = Unit->getASTContext().getLangOpts();
289
290  // FIXME: Currenty we do not support CTU across C++ and C and across
291  // different dialects of C++.
292  if (LangTo.CPlusPlus != LangFrom.CPlusPlus) {
293    ++NumLangMismatch;
294    return llvm::make_error<IndexError>(index_error_code::lang_mismatch);
295  }
296
297  // If CPP dialects are different then return with error.
298  //
299  // Consider this STL code:
300  //   template<typename _Alloc>
301  //     struct __alloc_traits
302  //   #if __cplusplus >= 201103L
303  //     : std::allocator_traits<_Alloc>
304  //   #endif
305  //     { // ...
306  //     };
307  // This class template would create ODR errors during merging the two units,
308  // since in one translation unit the class template has a base class, however
309  // in the other unit it has none.
310  if (LangTo.CPlusPlus11 != LangFrom.CPlusPlus11 ||
311      LangTo.CPlusPlus14 != LangFrom.CPlusPlus14 ||
312      LangTo.CPlusPlus17 != LangFrom.CPlusPlus17 ||
313      LangTo.CPlusPlus20 != LangFrom.CPlusPlus20) {
314    ++NumLangDialectMismatch;
315    return llvm::make_error<IndexError>(
316        index_error_code::lang_dialect_mismatch);
317  }
318
319  TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
320  if (const T *ResultDecl = findDefInDeclContext<T>(TU, *LookupName))
321    return importDefinition(ResultDecl, Unit);
322  return llvm::make_error<IndexError>(index_error_code::failed_import);
323}
324
325llvm::Expected<const FunctionDecl *>
326CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
327                                                  StringRef CrossTUDir,
328                                                  StringRef IndexName,
329                                                  bool DisplayCTUProgress) {
330  return getCrossTUDefinitionImpl(FD, CrossTUDir, IndexName,
331                                  DisplayCTUProgress);
332}
333
334llvm::Expected<const VarDecl *>
335CrossTranslationUnitContext::getCrossTUDefinition(const VarDecl *VD,
336                                                  StringRef CrossTUDir,
337                                                  StringRef IndexName,
338                                                  bool DisplayCTUProgress) {
339  return getCrossTUDefinitionImpl(VD, CrossTUDir, IndexName,
340                                  DisplayCTUProgress);
341}
342
343void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE) {
344  switch (IE.getCode()) {
345  case index_error_code::missing_index_file:
346    Context.getDiagnostics().Report(diag::err_ctu_error_opening)
347        << IE.getFileName();
348    break;
349  case index_error_code::invalid_index_format:
350    Context.getDiagnostics().Report(diag::err_extdefmap_parsing)
351        << IE.getFileName() << IE.getLineNum();
352    break;
353  case index_error_code::multiple_definitions:
354    Context.getDiagnostics().Report(diag::err_multiple_def_index)
355        << IE.getLineNum();
356    break;
357  case index_error_code::triple_mismatch:
358    Context.getDiagnostics().Report(diag::warn_ctu_incompat_triple)
359        << IE.getFileName() << IE.getTripleToName() << IE.getTripleFromName();
360    break;
361  default:
362    break;
363  }
364}
365
366CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
367    CompilerInstance &CI)
368    : Loader(CI, CI.getAnalyzerOpts()->CTUDir,
369             CI.getAnalyzerOpts()->CTUInvocationList),
370      LoadGuard(CI.getASTContext().getLangOpts().CPlusPlus
371                    ? CI.getAnalyzerOpts()->CTUImportCppThreshold
372                    : CI.getAnalyzerOpts()->CTUImportThreshold) {}
373
374llvm::Expected<ASTUnit *>
375CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
376    StringRef FileName, bool DisplayCTUProgress) {
377  // Try the cache first.
378  auto ASTCacheEntry = FileASTUnitMap.find(FileName);
379  if (ASTCacheEntry == FileASTUnitMap.end()) {
380
381    // Do not load if the limit is reached.
382    if (!LoadGuard) {
383      ++NumASTLoadThresholdReached;
384      return llvm::make_error<IndexError>(
385          index_error_code::load_threshold_reached);
386    }
387
388    auto LoadAttempt = Loader.load(FileName);
389
390    if (!LoadAttempt)
391      return LoadAttempt.takeError();
392
393    std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
394
395    // Need the raw pointer and the unique_ptr as well.
396    ASTUnit *Unit = LoadedUnit.get();
397
398    // Update the cache.
399    FileASTUnitMap[FileName] = std::move(LoadedUnit);
400
401    LoadGuard.indicateLoadSuccess();
402
403    if (DisplayCTUProgress)
404      llvm::errs() << "CTU loaded AST file: " << FileName << "\n";
405
406    return Unit;
407
408  } else {
409    // Found in the cache.
410    return ASTCacheEntry->second.get();
411  }
412}
413
414llvm::Expected<ASTUnit *>
415CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
416    StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName,
417    bool DisplayCTUProgress) {
418  // Try the cache first.
419  auto ASTCacheEntry = NameASTUnitMap.find(FunctionName);
420  if (ASTCacheEntry == NameASTUnitMap.end()) {
421    // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
422
423    // Ensure that the Index is loaded, as we need to search in it.
424    if (llvm::Error IndexLoadError =
425            ensureCTUIndexLoaded(CrossTUDir, IndexName))
426      return std::move(IndexLoadError);
427
428    // Check if there is and entry in the index for the function.
429    if (!NameFileMap.count(FunctionName)) {
430      ++NumNotInOtherTU;
431      return llvm::make_error<IndexError>(index_error_code::missing_definition);
432    }
433
434    // Search in the index for the filename where the definition of FuncitonName
435    // resides.
436    if (llvm::Expected<ASTUnit *> FoundForFile =
437            getASTUnitForFile(NameFileMap[FunctionName], DisplayCTUProgress)) {
438
439      // Update the cache.
440      NameASTUnitMap[FunctionName] = *FoundForFile;
441      return *FoundForFile;
442
443    } else {
444      return FoundForFile.takeError();
445    }
446  } else {
447    // Found in the cache.
448    return ASTCacheEntry->second;
449  }
450}
451
452llvm::Expected<std::string>
453CrossTranslationUnitContext::ASTUnitStorage::getFileForFunction(
454    StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName) {
455  if (llvm::Error IndexLoadError = ensureCTUIndexLoaded(CrossTUDir, IndexName))
456    return std::move(IndexLoadError);
457  return NameFileMap[FunctionName];
458}
459
460llvm::Error CrossTranslationUnitContext::ASTUnitStorage::ensureCTUIndexLoaded(
461    StringRef CrossTUDir, StringRef IndexName) {
462  // Dont initialize if the map is filled.
463  if (!NameFileMap.empty())
464    return llvm::Error::success();
465
466  // Get the absolute path to the index file.
467  SmallString<256> IndexFile = CrossTUDir;
468  if (llvm::sys::path::is_absolute(IndexName))
469    IndexFile = IndexName;
470  else
471    llvm::sys::path::append(IndexFile, IndexName);
472
473  if (auto IndexMapping = parseCrossTUIndex(IndexFile)) {
474    // Initialize member map.
475    NameFileMap = *IndexMapping;
476    return llvm::Error::success();
477  } else {
478    // Error while parsing CrossTU index file.
479    return IndexMapping.takeError();
480  };
481}
482
483llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
484    StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
485    bool DisplayCTUProgress) {
486  // FIXME: The current implementation only supports loading decls with
487  //        a lookup name from a single translation unit. If multiple
488  //        translation units contains decls with the same lookup name an
489  //        error will be returned.
490
491  // Try to get the value from the heavily cached storage.
492  llvm::Expected<ASTUnit *> Unit = ASTStorage.getASTUnitForFunction(
493      LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
494
495  if (!Unit)
496    return Unit.takeError();
497
498  // Check whether the backing pointer of the Expected is a nullptr.
499  if (!*Unit)
500    return llvm::make_error<IndexError>(
501        index_error_code::failed_to_get_external_ast);
502
503  return Unit;
504}
505
506CrossTranslationUnitContext::ASTLoader::ASTLoader(
507    CompilerInstance &CI, StringRef CTUDir, StringRef InvocationListFilePath)
508    : CI(CI), CTUDir(CTUDir), InvocationListFilePath(InvocationListFilePath) {}
509
510CrossTranslationUnitContext::LoadResultTy
511CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) {
512  llvm::SmallString<256> Path;
513  if (llvm::sys::path::is_absolute(Identifier, PathStyle)) {
514    Path = Identifier;
515  } else {
516    Path = CTUDir;
517    llvm::sys::path::append(Path, PathStyle, Identifier);
518  }
519
520  // The path is stored in the InvocationList member in posix style. To
521  // successfully lookup an entry based on filepath, it must be converted.
522  llvm::sys::path::native(Path, PathStyle);
523
524  // Normalize by removing relative path components.
525  llvm::sys::path::remove_dots(Path, /*remove_dot_dot*/ true, PathStyle);
526
527  if (Path.endswith(".ast"))
528    return loadFromDump(Path);
529  else
530    return loadFromSource(Path);
531}
532
533CrossTranslationUnitContext::LoadResultTy
534CrossTranslationUnitContext::ASTLoader::loadFromDump(StringRef ASTDumpPath) {
535  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
536  TextDiagnosticPrinter *DiagClient =
537      new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
538  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
539  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
540      new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
541  return ASTUnit::LoadFromASTFile(
542      std::string(ASTDumpPath.str()),
543      CI.getPCHContainerOperations()->getRawReader(), ASTUnit::LoadEverything,
544      Diags, CI.getFileSystemOpts());
545}
546
547/// Load the AST from a source-file, which is supposed to be located inside the
548/// YAML formatted invocation list file under the filesystem path specified by
549/// \p InvocationList. The invocation list should contain absolute paths.
550/// \p SourceFilePath is the absolute path of the source file that contains the
551/// function definition the analysis is looking for. The Index is built by the
552/// \p clang-extdef-mapping tool, which is also supposed to be generating
553/// absolute paths.
554///
555/// Proper diagnostic emission requires absolute paths, so even if a future
556/// change introduces the handling of relative paths, this must be taken into
557/// consideration.
558CrossTranslationUnitContext::LoadResultTy
559CrossTranslationUnitContext::ASTLoader::loadFromSource(
560    StringRef SourceFilePath) {
561
562  if (llvm::Error InitError = lazyInitInvocationList())
563    return std::move(InitError);
564  assert(InvocationList);
565
566  auto Invocation = InvocationList->find(SourceFilePath);
567  if (Invocation == InvocationList->end())
568    return llvm::make_error<IndexError>(
569        index_error_code::invocation_list_lookup_unsuccessful);
570
571  const InvocationListTy::mapped_type &InvocationCommand = Invocation->second;
572
573  SmallVector<const char *, 32> CommandLineArgs(InvocationCommand.size());
574  std::transform(InvocationCommand.begin(), InvocationCommand.end(),
575                 CommandLineArgs.begin(),
576                 [](auto &&CmdPart) { return CmdPart.c_str(); });
577
578  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts{&CI.getDiagnosticOpts()};
579  auto *DiagClient = new ForwardingDiagnosticConsumer{CI.getDiagnosticClient()};
580  IntrusiveRefCntPtr<DiagnosticIDs> DiagID{
581      CI.getDiagnostics().getDiagnosticIDs()};
582  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
583      new DiagnosticsEngine{DiagID, &*DiagOpts, DiagClient});
584
585  return std::unique_ptr<ASTUnit>(ASTUnit::LoadFromCommandLine(
586      CommandLineArgs.begin(), (CommandLineArgs.end()),
587      CI.getPCHContainerOperations(), Diags,
588      CI.getHeaderSearchOpts().ResourceDir));
589}
590
591llvm::Expected<InvocationListTy>
592parseInvocationList(StringRef FileContent, llvm::sys::path::Style PathStyle) {
593  InvocationListTy InvocationList;
594
595  /// LLVM YAML parser is used to extract information from invocation list file.
596  llvm::SourceMgr SM;
597  llvm::yaml::Stream InvocationFile(FileContent, SM);
598
599  /// Only the first document is processed.
600  llvm::yaml::document_iterator FirstInvocationFile = InvocationFile.begin();
601
602  /// There has to be at least one document available.
603  if (FirstInvocationFile == InvocationFile.end())
604    return llvm::make_error<IndexError>(
605        index_error_code::invocation_list_empty);
606
607  llvm::yaml::Node *DocumentRoot = FirstInvocationFile->getRoot();
608  if (!DocumentRoot)
609    return llvm::make_error<IndexError>(
610        index_error_code::invocation_list_wrong_format);
611
612  /// According to the format specified the document must be a mapping, where
613  /// the keys are paths to source files, and values are sequences of invocation
614  /// parts.
615  auto *Mappings = dyn_cast<llvm::yaml::MappingNode>(DocumentRoot);
616  if (!Mappings)
617    return llvm::make_error<IndexError>(
618        index_error_code::invocation_list_wrong_format);
619
620  for (auto &NextMapping : *Mappings) {
621    /// The keys should be strings, which represent a source-file path.
622    auto *Key = dyn_cast<llvm::yaml::ScalarNode>(NextMapping.getKey());
623    if (!Key)
624      return llvm::make_error<IndexError>(
625          index_error_code::invocation_list_wrong_format);
626
627    SmallVector<char, 32> ValueStorage;
628    StringRef SourcePath = Key->getValue(ValueStorage);
629
630    // Store paths with PathStyle directory separator.
631    SmallVector<char, 32> NativeSourcePath;
632    llvm::Twine{SourcePath}.toVector(NativeSourcePath);
633    llvm::sys::path::native(NativeSourcePath, PathStyle);
634
635    StringRef InvocationKey{NativeSourcePath.begin(), NativeSourcePath.size()};
636
637    if (InvocationList.find(InvocationKey) != InvocationList.end())
638      return llvm::make_error<IndexError>(
639          index_error_code::invocation_list_ambiguous);
640
641    /// The values should be sequences of strings, each representing a part of
642    /// the invocation.
643    auto *Args = dyn_cast<llvm::yaml::SequenceNode>(NextMapping.getValue());
644    if (!Args)
645      return llvm::make_error<IndexError>(
646          index_error_code::invocation_list_wrong_format);
647
648    for (auto &Arg : *Args) {
649      auto *CmdString = dyn_cast<llvm::yaml::ScalarNode>(&Arg);
650      if (!CmdString)
651        return llvm::make_error<IndexError>(
652            index_error_code::invocation_list_wrong_format);
653      /// Every conversion starts with an empty working storage, as it is not
654      /// clear if this is a requirement of the YAML parser.
655      ValueStorage.clear();
656      InvocationList[InvocationKey].emplace_back(
657          CmdString->getValue(ValueStorage));
658    }
659
660    if (InvocationList[InvocationKey].empty())
661      return llvm::make_error<IndexError>(
662          index_error_code::invocation_list_wrong_format);
663  }
664
665  return InvocationList;
666}
667
668llvm::Error CrossTranslationUnitContext::ASTLoader::lazyInitInvocationList() {
669  /// Lazily initialize the invocation list member used for on-demand parsing.
670  if (InvocationList)
671    return llvm::Error::success();
672
673  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileContent =
674      llvm::MemoryBuffer::getFile(InvocationListFilePath);
675  if (!FileContent)
676    return llvm::make_error<IndexError>(
677        index_error_code::invocation_list_file_not_found);
678  std::unique_ptr<llvm::MemoryBuffer> ContentBuffer = std::move(*FileContent);
679  assert(ContentBuffer && "If no error was produced after loading, the pointer "
680                          "should not be nullptr.");
681
682  llvm::Expected<InvocationListTy> ExpectedInvocationList =
683      parseInvocationList(ContentBuffer->getBuffer(), PathStyle);
684
685  if (!ExpectedInvocationList)
686    return ExpectedInvocationList.takeError();
687
688  InvocationList = *ExpectedInvocationList;
689
690  return llvm::Error::success();
691}
692
693template <typename T>
694llvm::Expected<const T *>
695CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
696  assert(hasBodyOrInit(D) && "Decls to be imported should have body or init.");
697
698  assert(&D->getASTContext() == &Unit->getASTContext() &&
699         "ASTContext of Decl and the unit should match.");
700  ASTImporter &Importer = getOrCreateASTImporter(Unit);
701
702  auto ToDeclOrError = Importer.Import(D);
703  if (!ToDeclOrError) {
704    handleAllErrors(ToDeclOrError.takeError(),
705                    [&](const ImportError &IE) {
706                      switch (IE.Error) {
707                      case ImportError::NameConflict:
708                        ++NumNameConflicts;
709                         break;
710                      case ImportError::UnsupportedConstruct:
711                        ++NumUnsupportedNodeFound;
712                        break;
713                      case ImportError::Unknown:
714                        llvm_unreachable("Unknown import error happened.");
715                        break;
716                      }
717                    });
718    return llvm::make_error<IndexError>(index_error_code::failed_import);
719  }
720  auto *ToDecl = cast<T>(*ToDeclOrError);
721  assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
722  ++NumGetCTUSuccess;
723
724  // Parent map is invalidated after changing the AST.
725  ToDecl->getASTContext().getParentMapContext().clear();
726
727  return ToDecl;
728}
729
730llvm::Expected<const FunctionDecl *>
731CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
732                                              ASTUnit *Unit) {
733  return importDefinitionImpl(FD, Unit);
734}
735
736llvm::Expected<const VarDecl *>
737CrossTranslationUnitContext::importDefinition(const VarDecl *VD,
738                                              ASTUnit *Unit) {
739  return importDefinitionImpl(VD, Unit);
740}
741
742void CrossTranslationUnitContext::lazyInitImporterSharedSt(
743    TranslationUnitDecl *ToTU) {
744  if (!ImporterSharedSt)
745    ImporterSharedSt = std::make_shared<ASTImporterSharedState>(*ToTU);
746}
747
748ASTImporter &
749CrossTranslationUnitContext::getOrCreateASTImporter(ASTUnit *Unit) {
750  ASTContext &From = Unit->getASTContext();
751
752  auto I = ASTUnitImporterMap.find(From.getTranslationUnitDecl());
753  if (I != ASTUnitImporterMap.end())
754    return *I->second;
755  lazyInitImporterSharedSt(Context.getTranslationUnitDecl());
756  ASTImporter *NewImporter = new ASTImporter(
757      Context, Context.getSourceManager().getFileManager(), From,
758      From.getSourceManager().getFileManager(), false, ImporterSharedSt);
759  NewImporter->setFileIDImportHandler([this, Unit](FileID ToID, FileID FromID) {
760    assert(ImportedFileIDs.find(ToID) == ImportedFileIDs.end() &&
761           "FileID already imported, should not happen.");
762    ImportedFileIDs[ToID] = std::make_pair(FromID, Unit);
763  });
764  ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(NewImporter);
765  return *NewImporter;
766}
767
768llvm::Optional<std::pair<SourceLocation, ASTUnit *>>
769CrossTranslationUnitContext::getImportedFromSourceLocation(
770    const clang::SourceLocation &ToLoc) const {
771  const SourceManager &SM = Context.getSourceManager();
772  auto DecToLoc = SM.getDecomposedLoc(ToLoc);
773
774  auto I = ImportedFileIDs.find(DecToLoc.first);
775  if (I == ImportedFileIDs.end())
776    return {};
777
778  FileID FromID = I->second.first;
779  clang::ASTUnit *Unit = I->second.second;
780  SourceLocation FromLoc =
781      Unit->getSourceManager().getComposedLoc(FromID, DecToLoc.second);
782
783  return std::make_pair(FromLoc, Unit);
784}
785
786} // namespace cross_tu
787} // namespace clang
788