1//===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class which performs incremental code compilation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "IncrementalParser.h"
14
15#include "clang/AST/DeclContextInternals.h"
16#include "clang/CodeGen/BackendUtil.h"
17#include "clang/CodeGen/CodeGenAction.h"
18#include "clang/CodeGen/ModuleBuilder.h"
19#include "clang/Frontend/CompilerInstance.h"
20#include "clang/Frontend/FrontendAction.h"
21#include "clang/FrontendTool/Utils.h"
22#include "clang/Interpreter/Interpreter.h"
23#include "clang/Parse/Parser.h"
24#include "clang/Sema/Sema.h"
25#include "llvm/Option/ArgList.h"
26#include "llvm/Support/CrashRecoveryContext.h"
27#include "llvm/Support/Error.h"
28#include "llvm/Support/Timer.h"
29
30#include <sstream>
31
32namespace clang {
33
34class IncrementalASTConsumer final : public ASTConsumer {
35  Interpreter &Interp;
36  std::unique_ptr<ASTConsumer> Consumer;
37
38public:
39  IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
40      : Interp(InterpRef), Consumer(std::move(C)) {}
41
42  bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
43    if (DGR.isNull())
44      return true;
45    if (!Consumer)
46      return true;
47
48    for (Decl *D : DGR)
49      if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);
50          TSD && TSD->isSemiMissing())
51        TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));
52
53    return Consumer->HandleTopLevelDecl(DGR);
54  }
55  void HandleTranslationUnit(ASTContext &Ctx) override final {
56    Consumer->HandleTranslationUnit(Ctx);
57  }
58  void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
59    Consumer->HandleInlineFunctionDefinition(D);
60  }
61  void HandleInterestingDecl(DeclGroupRef D) override final {
62    Consumer->HandleInterestingDecl(D);
63  }
64  void HandleTagDeclDefinition(TagDecl *D) override final {
65    Consumer->HandleTagDeclDefinition(D);
66  }
67  void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
68    Consumer->HandleTagDeclRequiredDefinition(D);
69  }
70  void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
71    Consumer->HandleCXXImplicitFunctionInstantiation(D);
72  }
73  void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
74    Consumer->HandleTopLevelDeclInObjCContainer(D);
75  }
76  void HandleImplicitImportDecl(ImportDecl *D) override final {
77    Consumer->HandleImplicitImportDecl(D);
78  }
79  void CompleteTentativeDefinition(VarDecl *D) override final {
80    Consumer->CompleteTentativeDefinition(D);
81  }
82  void CompleteExternalDeclaration(VarDecl *D) override final {
83    Consumer->CompleteExternalDeclaration(D);
84  }
85  void AssignInheritanceModel(CXXRecordDecl *RD) override final {
86    Consumer->AssignInheritanceModel(RD);
87  }
88  void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
89    Consumer->HandleCXXStaticMemberVarInstantiation(D);
90  }
91  void HandleVTable(CXXRecordDecl *RD) override final {
92    Consumer->HandleVTable(RD);
93  }
94  ASTMutationListener *GetASTMutationListener() override final {
95    return Consumer->GetASTMutationListener();
96  }
97  ASTDeserializationListener *GetASTDeserializationListener() override final {
98    return Consumer->GetASTDeserializationListener();
99  }
100  void PrintStats() override final { Consumer->PrintStats(); }
101  bool shouldSkipFunctionBody(Decl *D) override final {
102    return Consumer->shouldSkipFunctionBody(D);
103  }
104  static bool classof(const clang::ASTConsumer *) { return true; }
105};
106
107/// A custom action enabling the incremental processing functionality.
108///
109/// The usual \p FrontendAction expects one call to ExecuteAction and once it
110/// sees a call to \p EndSourceFile it deletes some of the important objects
111/// such as \p Preprocessor and \p Sema assuming no further input will come.
112///
113/// \p IncrementalAction ensures it keep its underlying action's objects alive
114/// as long as the \p IncrementalParser needs them.
115///
116class IncrementalAction : public WrapperFrontendAction {
117private:
118  bool IsTerminating = false;
119
120public:
121  IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
122                    llvm::Error &Err)
123      : WrapperFrontendAction([&]() {
124          llvm::ErrorAsOutParameter EAO(&Err);
125          std::unique_ptr<FrontendAction> Act;
126          switch (CI.getFrontendOpts().ProgramAction) {
127          default:
128            Err = llvm::createStringError(
129                std::errc::state_not_recoverable,
130                "Driver initialization failed. "
131                "Incremental mode for action %d is not supported",
132                CI.getFrontendOpts().ProgramAction);
133            return Act;
134          case frontend::ASTDump:
135            [[fallthrough]];
136          case frontend::ASTPrint:
137            [[fallthrough]];
138          case frontend::ParseSyntaxOnly:
139            Act = CreateFrontendAction(CI);
140            break;
141          case frontend::PluginAction:
142            [[fallthrough]];
143          case frontend::EmitAssembly:
144            [[fallthrough]];
145          case frontend::EmitBC:
146            [[fallthrough]];
147          case frontend::EmitObj:
148            [[fallthrough]];
149          case frontend::PrintPreprocessedInput:
150            [[fallthrough]];
151          case frontend::EmitLLVMOnly:
152            Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
153            break;
154          }
155          return Act;
156        }()) {}
157  FrontendAction *getWrapped() const { return WrappedAction.get(); }
158  TranslationUnitKind getTranslationUnitKind() override {
159    return TU_Incremental;
160  }
161
162  void ExecuteAction() override {
163    CompilerInstance &CI = getCompilerInstance();
164    assert(CI.hasPreprocessor() && "No PP!");
165
166    // Use a code completion consumer?
167    CodeCompleteConsumer *CompletionConsumer = nullptr;
168    if (CI.hasCodeCompletionConsumer())
169      CompletionConsumer = &CI.getCodeCompletionConsumer();
170
171    Preprocessor &PP = CI.getPreprocessor();
172    PP.EnterMainSourceFile();
173
174    if (!CI.hasSema())
175      CI.createSema(getTranslationUnitKind(), CompletionConsumer);
176  }
177
178  // Do not terminate after processing the input. This allows us to keep various
179  // clang objects alive and to incrementally grow the current TU.
180  void EndSourceFile() override {
181    // The WrappedAction can be nullptr if we issued an error in the ctor.
182    if (IsTerminating && getWrapped())
183      WrapperFrontendAction::EndSourceFile();
184  }
185
186  void FinalizeAction() {
187    assert(!IsTerminating && "Already finalized!");
188    IsTerminating = true;
189    EndSourceFile();
190  }
191};
192
193CodeGenerator *IncrementalParser::getCodeGen() const {
194  FrontendAction *WrappedAct = Act->getWrapped();
195  if (!WrappedAct->hasIRSupport())
196    return nullptr;
197  return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
198}
199
200IncrementalParser::IncrementalParser() {}
201
202IncrementalParser::IncrementalParser(Interpreter &Interp,
203                                     std::unique_ptr<CompilerInstance> Instance,
204                                     llvm::LLVMContext &LLVMCtx,
205                                     llvm::Error &Err)
206    : CI(std::move(Instance)) {
207  llvm::ErrorAsOutParameter EAO(&Err);
208  Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
209  if (Err)
210    return;
211  CI->ExecuteAction(*Act);
212
213  if (getCodeGen())
214    CachedInCodeGenModule = GenModule();
215
216  std::unique_ptr<ASTConsumer> IncrConsumer =
217      std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
218  CI->setASTConsumer(std::move(IncrConsumer));
219  Consumer = &CI->getASTConsumer();
220  P.reset(
221      new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
222  P->Initialize();
223
224  // An initial PTU is needed as CUDA includes some headers automatically
225  auto PTU = ParseOrWrapTopLevelDecl();
226  if (auto E = PTU.takeError()) {
227    consumeError(std::move(E)); // FIXME
228    return;                     // PTU.takeError();
229  }
230
231  if (getCodeGen()) {
232    PTU->TheModule = GenModule();
233    assert(PTU->TheModule && "Failed to create initial PTU");
234  }
235}
236
237IncrementalParser::~IncrementalParser() {
238  P.reset();
239  Act->FinalizeAction();
240}
241
242llvm::Expected<PartialTranslationUnit &>
243IncrementalParser::ParseOrWrapTopLevelDecl() {
244  // Recover resources if we crash before exiting this method.
245  Sema &S = CI->getSema();
246  llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
247  Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
248  Sema::LocalEagerInstantiationScope LocalInstantiations(S);
249
250  PTUs.emplace_back(PartialTranslationUnit());
251  PartialTranslationUnit &LastPTU = PTUs.back();
252  // Add a new PTU.
253  ASTContext &C = S.getASTContext();
254  C.addTranslationUnitDecl();
255  LastPTU.TUPart = C.getTranslationUnitDecl();
256
257  // Skip previous eof due to last incremental input.
258  if (P->getCurToken().is(tok::annot_repl_input_end)) {
259    P->ConsumeAnyToken();
260    // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
261    // might want to do that around HandleEndOfTranslationUnit.
262    P->ExitScope();
263    S.CurContext = nullptr;
264    // Start a new PTU.
265    P->EnterScope(Scope::DeclScope);
266    S.ActOnTranslationUnitScope(P->getCurScope());
267  }
268
269  Parser::DeclGroupPtrTy ADecl;
270  Sema::ModuleImportState ImportState;
271  for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
272       AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
273    if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
274      return llvm::make_error<llvm::StringError>("Parsing failed. "
275                                                 "The consumer rejected a decl",
276                                                 std::error_code());
277  }
278
279  DiagnosticsEngine &Diags = getCI()->getDiagnostics();
280  if (Diags.hasErrorOccurred()) {
281    PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
282                                            nullptr};
283    CleanUpPTU(MostRecentPTU);
284
285    Diags.Reset(/*soft=*/true);
286    Diags.getClient()->clear();
287    return llvm::make_error<llvm::StringError>("Parsing failed.",
288                                               std::error_code());
289  }
290
291  // Process any TopLevelDecls generated by #pragma weak.
292  for (Decl *D : S.WeakTopLevelDecls()) {
293    DeclGroupRef DGR(D);
294    Consumer->HandleTopLevelDecl(DGR);
295  }
296
297  LocalInstantiations.perform();
298  GlobalInstantiations.perform();
299
300  Consumer->HandleTranslationUnit(C);
301
302  return LastPTU;
303}
304
305llvm::Expected<PartialTranslationUnit &>
306IncrementalParser::Parse(llvm::StringRef input) {
307  Preprocessor &PP = CI->getPreprocessor();
308  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
309
310  std::ostringstream SourceName;
311  SourceName << "input_line_" << InputCount++;
312
313  // Create an uninitialized memory buffer, copy code in and append "\n"
314  size_t InputSize = input.size(); // don't include trailing 0
315  // MemBuffer size should *not* include terminating zero
316  std::unique_ptr<llvm::MemoryBuffer> MB(
317      llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
318                                                        SourceName.str()));
319  char *MBStart = const_cast<char *>(MB->getBufferStart());
320  memcpy(MBStart, input.data(), InputSize);
321  MBStart[InputSize] = '\n';
322
323  SourceManager &SM = CI->getSourceManager();
324
325  // FIXME: Create SourceLocation, which will allow clang to order the overload
326  // candidates for example
327  SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
328
329  // Create FileID for the current buffer.
330  FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
331                               /*LoadedOffset=*/0, NewLoc);
332
333  // NewLoc only used for diags.
334  if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
335    return llvm::make_error<llvm::StringError>("Parsing failed. "
336                                               "Cannot enter source file.",
337                                               std::error_code());
338
339  auto PTU = ParseOrWrapTopLevelDecl();
340  if (!PTU)
341    return PTU.takeError();
342
343  if (PP.getLangOpts().DelayedTemplateParsing) {
344    // Microsoft-specific:
345    // Late parsed templates can leave unswallowed "macro"-like tokens.
346    // They will seriously confuse the Parser when entering the next
347    // source file. So lex until we are EOF.
348    Token Tok;
349    do {
350      PP.Lex(Tok);
351    } while (Tok.isNot(tok::annot_repl_input_end));
352  } else {
353    Token AssertTok;
354    PP.Lex(AssertTok);
355    assert(AssertTok.is(tok::annot_repl_input_end) &&
356           "Lexer must be EOF when starting incremental parse!");
357  }
358
359  if (std::unique_ptr<llvm::Module> M = GenModule())
360    PTU->TheModule = std::move(M);
361
362  return PTU;
363}
364
365std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
366  static unsigned ID = 0;
367  if (CodeGenerator *CG = getCodeGen()) {
368    // Clang's CodeGen is designed to work with a single llvm::Module. In many
369    // cases for convenience various CodeGen parts have a reference to the
370    // llvm::Module (TheModule or Module) which does not change when a new
371    // module is pushed. However, the execution engine wants to take ownership
372    // of the module which does not map well to CodeGen's design. To work this
373    // around we created an empty module to make CodeGen happy. We should make
374    // sure it always stays empty.
375    assert((!CachedInCodeGenModule ||
376            (CachedInCodeGenModule->empty() &&
377             CachedInCodeGenModule->global_empty() &&
378             CachedInCodeGenModule->alias_empty() &&
379             CachedInCodeGenModule->ifunc_empty())) &&
380           "CodeGen wrote to a readonly module");
381    std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
382    CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
383    return M;
384  }
385  return nullptr;
386}
387
388void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
389  TranslationUnitDecl *MostRecentTU = PTU.TUPart;
390  TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
391  if (StoredDeclsMap *Map = FirstTU->getPrimaryContext()->getLookupPtr()) {
392    for (auto I = Map->begin(); I != Map->end(); ++I) {
393      StoredDeclsList &List = I->second;
394      DeclContextLookupResult R = List.getLookupResult();
395      for (NamedDecl *D : R) {
396        if (D->getTranslationUnitDecl() == MostRecentTU) {
397          List.remove(D);
398        }
399      }
400      if (List.isNull())
401        Map->erase(I);
402    }
403  }
404}
405
406llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
407  CodeGenerator *CG = getCodeGen();
408  assert(CG);
409  return CG->GetMangledName(GD);
410}
411} // end namespace clang
412