1//===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Orc based MCJIT replacement.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H
14#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H
15
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ExecutionEngine/ExecutionEngine.h"
20#include "llvm/ExecutionEngine/GenericValue.h"
21#include "llvm/ExecutionEngine/JITSymbol.h"
22#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
23#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
24#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
25#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h"
26#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
27#include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
28#include "llvm/ExecutionEngine/RuntimeDyld.h"
29#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Mangler.h"
32#include "llvm/IR/Module.h"
33#include "llvm/Object/Archive.h"
34#include "llvm/Object/Binary.h"
35#include "llvm/Object/ObjectFile.h"
36#include "llvm/Support/Error.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Target/TargetMachine.h"
40#include <algorithm>
41#include <cassert>
42#include <cstddef>
43#include <cstdint>
44#include <map>
45#include <memory>
46#include <set>
47#include <string>
48#include <vector>
49
50namespace llvm {
51
52class ObjectCache;
53
54namespace orc {
55
56class OrcMCJITReplacement : public ExecutionEngine {
57
58  // OrcMCJITReplacement needs to do a little extra book-keeping to ensure that
59  // Orc's automatic finalization doesn't kick in earlier than MCJIT clients are
60  // expecting - see finalizeMemory.
61  class MCJITReplacementMemMgr : public MCJITMemoryManager {
62  public:
63    MCJITReplacementMemMgr(OrcMCJITReplacement &M,
64                           std::shared_ptr<MCJITMemoryManager> ClientMM)
65      : M(M), ClientMM(std::move(ClientMM)) {}
66
67    uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
68                                 unsigned SectionID,
69                                 StringRef SectionName) override {
70      uint8_t *Addr =
71          ClientMM->allocateCodeSection(Size, Alignment, SectionID,
72                                        SectionName);
73      M.SectionsAllocatedSinceLastLoad.insert(Addr);
74      return Addr;
75    }
76
77    uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
78                                 unsigned SectionID, StringRef SectionName,
79                                 bool IsReadOnly) override {
80      uint8_t *Addr = ClientMM->allocateDataSection(Size, Alignment, SectionID,
81                                                    SectionName, IsReadOnly);
82      M.SectionsAllocatedSinceLastLoad.insert(Addr);
83      return Addr;
84    }
85
86    void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign,
87                                uintptr_t RODataSize, uint32_t RODataAlign,
88                                uintptr_t RWDataSize,
89                                uint32_t RWDataAlign) override {
90      return ClientMM->reserveAllocationSpace(CodeSize, CodeAlign,
91                                              RODataSize, RODataAlign,
92                                              RWDataSize, RWDataAlign);
93    }
94
95    bool needsToReserveAllocationSpace() override {
96      return ClientMM->needsToReserveAllocationSpace();
97    }
98
99    void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr,
100                          size_t Size) override {
101      return ClientMM->registerEHFrames(Addr, LoadAddr, Size);
102    }
103
104    void deregisterEHFrames() override {
105      return ClientMM->deregisterEHFrames();
106    }
107
108    void notifyObjectLoaded(RuntimeDyld &RTDyld,
109                            const object::ObjectFile &O) override {
110      return ClientMM->notifyObjectLoaded(RTDyld, O);
111    }
112
113    void notifyObjectLoaded(ExecutionEngine *EE,
114                            const object::ObjectFile &O) override {
115      return ClientMM->notifyObjectLoaded(EE, O);
116    }
117
118    bool finalizeMemory(std::string *ErrMsg = nullptr) override {
119      // Each set of objects loaded will be finalized exactly once, but since
120      // symbol lookup during relocation may recursively trigger the
121      // loading/relocation of other modules, and since we're forwarding all
122      // finalizeMemory calls to a single underlying memory manager, we need to
123      // defer forwarding the call on until all necessary objects have been
124      // loaded. Otherwise, during the relocation of a leaf object, we will end
125      // up finalizing memory, causing a crash further up the stack when we
126      // attempt to apply relocations to finalized memory.
127      // To avoid finalizing too early, look at how many objects have been
128      // loaded but not yet finalized. This is a bit of a hack that relies on
129      // the fact that we're lazily emitting object files: The only way you can
130      // get more than one set of objects loaded but not yet finalized is if
131      // they were loaded during relocation of another set.
132      if (M.UnfinalizedSections.size() == 1)
133        return ClientMM->finalizeMemory(ErrMsg);
134      return false;
135    }
136
137  private:
138    OrcMCJITReplacement &M;
139    std::shared_ptr<MCJITMemoryManager> ClientMM;
140  };
141
142  class LinkingORCResolver : public orc::SymbolResolver {
143  public:
144    LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {}
145
146    SymbolNameSet getResponsibilitySet(const SymbolNameSet &Symbols) override {
147      SymbolNameSet Result;
148
149      for (auto &S : Symbols) {
150        if (auto Sym = M.findMangledSymbol(*S)) {
151          if (!Sym.getFlags().isStrong())
152            Result.insert(S);
153        } else if (auto Err = Sym.takeError()) {
154          M.reportError(std::move(Err));
155          return SymbolNameSet();
156        } else {
157          if (auto Sym2 = M.ClientResolver->findSymbolInLogicalDylib(*S)) {
158            if (!Sym2.getFlags().isStrong())
159              Result.insert(S);
160          } else if (auto Err = Sym2.takeError()) {
161            M.reportError(std::move(Err));
162            return SymbolNameSet();
163          } else
164            Result.insert(S);
165        }
166      }
167
168      return Result;
169    }
170
171    SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query,
172                         SymbolNameSet Symbols) override {
173      SymbolNameSet UnresolvedSymbols;
174      bool NewSymbolsResolved = false;
175
176      for (auto &S : Symbols) {
177        if (auto Sym = M.findMangledSymbol(*S)) {
178          if (auto Addr = Sym.getAddress()) {
179            Query->notifySymbolMetRequiredState(
180                S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
181            NewSymbolsResolved = true;
182          } else {
183            M.ES.legacyFailQuery(*Query, Addr.takeError());
184            return SymbolNameSet();
185          }
186        } else if (auto Err = Sym.takeError()) {
187          M.ES.legacyFailQuery(*Query, std::move(Err));
188          return SymbolNameSet();
189        } else {
190          if (auto Sym2 = M.ClientResolver->findSymbol(*S)) {
191            if (auto Addr = Sym2.getAddress()) {
192              Query->notifySymbolMetRequiredState(
193                  S, JITEvaluatedSymbol(*Addr, Sym2.getFlags()));
194              NewSymbolsResolved = true;
195            } else {
196              M.ES.legacyFailQuery(*Query, Addr.takeError());
197              return SymbolNameSet();
198            }
199          } else if (auto Err = Sym2.takeError()) {
200            M.ES.legacyFailQuery(*Query, std::move(Err));
201            return SymbolNameSet();
202          } else
203            UnresolvedSymbols.insert(S);
204        }
205      }
206
207      if (NewSymbolsResolved && Query->isComplete())
208        Query->handleComplete();
209
210      return UnresolvedSymbols;
211    }
212
213  private:
214    OrcMCJITReplacement &M;
215  };
216
217private:
218  static ExecutionEngine *
219  createOrcMCJITReplacement(std::string *ErrorMsg,
220                            std::shared_ptr<MCJITMemoryManager> MemMgr,
221                            std::shared_ptr<LegacyJITSymbolResolver> Resolver,
222                            std::unique_ptr<TargetMachine> TM) {
223    return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver),
224                                   std::move(TM));
225  }
226
227  void reportError(Error Err) {
228    logAllUnhandledErrors(std::move(Err), errs(), "MCJIT error: ");
229  }
230
231public:
232  OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr,
233                      std::shared_ptr<LegacyJITSymbolResolver> ClientResolver,
234                      std::unique_ptr<TargetMachine> TM)
235      : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)),
236        MemMgr(
237            std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))),
238        Resolver(std::make_shared<LinkingORCResolver>(*this)),
239        ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this),
240        NotifyFinalized(*this),
241        ObjectLayer(
242            AcknowledgeORCv1Deprecation, ES,
243            [this](VModuleKey K) {
244              return ObjectLayerT::Resources{this->MemMgr, this->Resolver};
245            },
246            NotifyObjectLoaded, NotifyFinalized),
247        CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer,
248                     SimpleCompiler(*this->TM),
249                     [this](VModuleKey K, std::unique_ptr<Module> M) {
250                       Modules.push_back(std::move(M));
251                     }),
252        LazyEmitLayer(AcknowledgeORCv1Deprecation, CompileLayer) {}
253
254  static void Register() {
255    OrcMCJITReplacementCtor = createOrcMCJITReplacement;
256  }
257
258  void addModule(std::unique_ptr<Module> M) override {
259    // If this module doesn't have a DataLayout attached then attach the
260    // default.
261    if (M->getDataLayout().isDefault()) {
262      M->setDataLayout(getDataLayout());
263    } else {
264      assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch");
265    }
266
267    // Rename, bump linkage and record static constructors and destructors.
268    // We have to do this before we hand over ownership of the module to the
269    // JIT.
270    std::vector<std::string> CtorNames, DtorNames;
271    {
272      unsigned CtorId = 0, DtorId = 0;
273      for (auto Ctor : orc::getConstructors(*M)) {
274        std::string NewCtorName = ("__ORCstatic_ctor." + Twine(CtorId++)).str();
275        Ctor.Func->setName(NewCtorName);
276        Ctor.Func->setLinkage(GlobalValue::ExternalLinkage);
277        Ctor.Func->setVisibility(GlobalValue::HiddenVisibility);
278        CtorNames.push_back(mangle(NewCtorName));
279      }
280      for (auto Dtor : orc::getDestructors(*M)) {
281        std::string NewDtorName = ("__ORCstatic_dtor." + Twine(DtorId++)).str();
282        dbgs() << "Found dtor: " << NewDtorName << "\n";
283        Dtor.Func->setName(NewDtorName);
284        Dtor.Func->setLinkage(GlobalValue::ExternalLinkage);
285        Dtor.Func->setVisibility(GlobalValue::HiddenVisibility);
286        DtorNames.push_back(mangle(NewDtorName));
287      }
288    }
289
290    auto K = ES.allocateVModule();
291
292    UnexecutedConstructors[K] = std::move(CtorNames);
293    UnexecutedDestructors[K] = std::move(DtorNames);
294
295    cantFail(LazyEmitLayer.addModule(K, std::move(M)));
296  }
297
298  void addObjectFile(std::unique_ptr<object::ObjectFile> O) override {
299    cantFail(ObjectLayer.addObject(
300        ES.allocateVModule(), MemoryBuffer::getMemBufferCopy(O->getData())));
301  }
302
303  void addObjectFile(object::OwningBinary<object::ObjectFile> O) override {
304    std::unique_ptr<object::ObjectFile> Obj;
305    std::unique_ptr<MemoryBuffer> ObjBuffer;
306    std::tie(Obj, ObjBuffer) = O.takeBinary();
307    cantFail(ObjectLayer.addObject(ES.allocateVModule(), std::move(ObjBuffer)));
308  }
309
310  void addArchive(object::OwningBinary<object::Archive> A) override {
311    Archives.push_back(std::move(A));
312  }
313
314  bool removeModule(Module *M) override {
315    auto I = Modules.begin();
316    for (auto E = Modules.end(); I != E; ++I)
317      if (I->get() == M)
318        break;
319    if (I == Modules.end())
320      return false;
321    Modules.erase(I);
322    return true;
323  }
324
325  uint64_t getSymbolAddress(StringRef Name) {
326    return cantFail(findSymbol(Name).getAddress());
327  }
328
329  JITSymbol findSymbol(StringRef Name) {
330    return findMangledSymbol(mangle(Name));
331  }
332
333  void finalizeObject() override {
334    // This is deprecated - Aim to remove in ExecutionEngine.
335    // REMOVE IF POSSIBLE - Doesn't make sense for New JIT.
336  }
337
338  void mapSectionAddress(const void *LocalAddress,
339                         uint64_t TargetAddress) override {
340    for (auto &P : UnfinalizedSections)
341      if (P.second.count(LocalAddress))
342        ObjectLayer.mapSectionAddress(P.first, LocalAddress, TargetAddress);
343  }
344
345  uint64_t getGlobalValueAddress(const std::string &Name) override {
346    return getSymbolAddress(Name);
347  }
348
349  uint64_t getFunctionAddress(const std::string &Name) override {
350    return getSymbolAddress(Name);
351  }
352
353  void *getPointerToFunction(Function *F) override {
354    uint64_t FAddr = getSymbolAddress(F->getName());
355    return reinterpret_cast<void *>(static_cast<uintptr_t>(FAddr));
356  }
357
358  void *getPointerToNamedFunction(StringRef Name,
359                                  bool AbortOnFailure = true) override {
360    uint64_t Addr = getSymbolAddress(Name);
361    if (!Addr && AbortOnFailure)
362      llvm_unreachable("Missing symbol!");
363    return reinterpret_cast<void *>(static_cast<uintptr_t>(Addr));
364  }
365
366  GenericValue runFunction(Function *F,
367                           ArrayRef<GenericValue> ArgValues) override;
368
369  void setObjectCache(ObjectCache *NewCache) override {
370    CompileLayer.getCompiler().setObjectCache(NewCache);
371  }
372
373  void setProcessAllSections(bool ProcessAllSections) override {
374    ObjectLayer.setProcessAllSections(ProcessAllSections);
375  }
376
377  void runStaticConstructorsDestructors(bool isDtors) override;
378
379private:
380  JITSymbol findMangledSymbol(StringRef Name) {
381    if (auto Sym = LazyEmitLayer.findSymbol(Name, false))
382      return Sym;
383    if (auto Sym = ClientResolver->findSymbol(Name))
384      return Sym;
385    if (auto Sym = scanArchives(Name))
386      return Sym;
387
388    return nullptr;
389  }
390
391  JITSymbol scanArchives(StringRef Name) {
392    for (object::OwningBinary<object::Archive> &OB : Archives) {
393      object::Archive *A = OB.getBinary();
394      // Look for our symbols in each Archive
395      auto OptionalChildOrErr = A->findSym(Name);
396      if (!OptionalChildOrErr)
397        report_fatal_error(OptionalChildOrErr.takeError());
398      auto &OptionalChild = *OptionalChildOrErr;
399      if (OptionalChild) {
400        // FIXME: Support nested archives?
401        Expected<std::unique_ptr<object::Binary>> ChildBinOrErr =
402            OptionalChild->getAsBinary();
403        if (!ChildBinOrErr) {
404          // TODO: Actually report errors helpfully.
405          consumeError(ChildBinOrErr.takeError());
406          continue;
407        }
408        std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get();
409        if (ChildBin->isObject()) {
410          cantFail(ObjectLayer.addObject(
411              ES.allocateVModule(),
412              MemoryBuffer::getMemBufferCopy(ChildBin->getData())));
413          if (auto Sym = ObjectLayer.findSymbol(Name, true))
414            return Sym;
415        }
416      }
417    }
418    return nullptr;
419  }
420
421  class NotifyObjectLoadedT {
422  public:
423    using LoadedObjInfoListT =
424        std::vector<std::unique_ptr<RuntimeDyld::LoadedObjectInfo>>;
425
426    NotifyObjectLoadedT(OrcMCJITReplacement &M) : M(M) {}
427
428    void operator()(VModuleKey K, const object::ObjectFile &Obj,
429                    const RuntimeDyld::LoadedObjectInfo &Info) const {
430      M.UnfinalizedSections[K] = std::move(M.SectionsAllocatedSinceLastLoad);
431      M.SectionsAllocatedSinceLastLoad = SectionAddrSet();
432      M.MemMgr->notifyObjectLoaded(&M, Obj);
433    }
434  private:
435    OrcMCJITReplacement &M;
436  };
437
438  class NotifyFinalizedT {
439  public:
440    NotifyFinalizedT(OrcMCJITReplacement &M) : M(M) {}
441
442    void operator()(VModuleKey K, const object::ObjectFile &Obj,
443                    const RuntimeDyld::LoadedObjectInfo &Info) {
444      M.UnfinalizedSections.erase(K);
445    }
446
447  private:
448    OrcMCJITReplacement &M;
449  };
450
451  std::string mangle(StringRef Name) {
452    std::string MangledName;
453    {
454      raw_string_ostream MangledNameStream(MangledName);
455      Mang.getNameWithPrefix(MangledNameStream, Name, getDataLayout());
456    }
457    return MangledName;
458  }
459
460  using ObjectLayerT = LegacyRTDyldObjectLinkingLayer;
461  using CompileLayerT = LegacyIRCompileLayer<ObjectLayerT, orc::SimpleCompiler>;
462  using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>;
463
464  ExecutionSession ES;
465
466  std::unique_ptr<TargetMachine> TM;
467  std::shared_ptr<MCJITReplacementMemMgr> MemMgr;
468  std::shared_ptr<LinkingORCResolver> Resolver;
469  std::shared_ptr<LegacyJITSymbolResolver> ClientResolver;
470  Mangler Mang;
471
472  // IMPORTANT: ShouldDelete *must* come before LocalModules: The shared_ptr
473  // delete blocks in LocalModules refer to the ShouldDelete map, so
474  // LocalModules needs to be destructed before ShouldDelete.
475  std::map<Module*, bool> ShouldDelete;
476
477  NotifyObjectLoadedT NotifyObjectLoaded;
478  NotifyFinalizedT NotifyFinalized;
479
480  ObjectLayerT ObjectLayer;
481  CompileLayerT CompileLayer;
482  LazyEmitLayerT LazyEmitLayer;
483
484  std::map<VModuleKey, std::vector<std::string>> UnexecutedConstructors;
485  std::map<VModuleKey, std::vector<std::string>> UnexecutedDestructors;
486
487  // We need to store ObjLayerT::ObjSetHandles for each of the object sets
488  // that have been emitted but not yet finalized so that we can forward the
489  // mapSectionAddress calls appropriately.
490  using SectionAddrSet = std::set<const void *>;
491  SectionAddrSet SectionsAllocatedSinceLastLoad;
492  std::map<VModuleKey, SectionAddrSet> UnfinalizedSections;
493
494  std::vector<object::OwningBinary<object::Archive>> Archives;
495};
496
497} // end namespace orc
498
499} // end namespace llvm
500
501#endif // LLVM_LIB_EXECUTIONENGINE_ORC_MCJITREPLACEMENT_H
502