1//===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Orc based MCJIT replacement.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H
14#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H
15
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ExecutionEngine/ExecutionEngine.h"
20#include "llvm/ExecutionEngine/GenericValue.h"
21#include "llvm/ExecutionEngine/JITSymbol.h"
22#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
23#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
24#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
25#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h"
26#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
27#include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
28#include "llvm/ExecutionEngine/RuntimeDyld.h"
29#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Mangler.h"
32#include "llvm/IR/Module.h"
33#include "llvm/Object/Archive.h"
34#include "llvm/Object/Binary.h"
35#include "llvm/Object/ObjectFile.h"
36#include "llvm/Support/Error.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Target/TargetMachine.h"
40#include <algorithm>
41#include <cassert>
42#include <cstddef>
43#include <cstdint>
44#include <map>
45#include <memory>
46#include <set>
47#include <string>
48#include <vector>
49
50namespace llvm {
51
52class ObjectCache;
53
54namespace orc {
55
56class OrcMCJITReplacement : public ExecutionEngine {
57
58  // OrcMCJITReplacement needs to do a little extra book-keeping to ensure that
59  // Orc's automatic finalization doesn't kick in earlier than MCJIT clients are
60  // expecting - see finalizeMemory.
61  class MCJITReplacementMemMgr : public MCJITMemoryManager {
62  public:
63    MCJITReplacementMemMgr(OrcMCJITReplacement &M,
64                           std::shared_ptr<MCJITMemoryManager> ClientMM)
65      : M(M), ClientMM(std::move(ClientMM)) {}
66
67    uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
68                                 unsigned SectionID,
69                                 StringRef SectionName) override {
70      uint8_t *Addr =
71          ClientMM->allocateCodeSection(Size, Alignment, SectionID,
72                                        SectionName);
73      M.SectionsAllocatedSinceLastLoad.insert(Addr);
74      return Addr;
75    }
76
77    uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
78                                 unsigned SectionID, StringRef SectionName,
79                                 bool IsReadOnly) override {
80      uint8_t *Addr = ClientMM->allocateDataSection(Size, Alignment, SectionID,
81                                                    SectionName, IsReadOnly);
82      M.SectionsAllocatedSinceLastLoad.insert(Addr);
83      return Addr;
84    }
85
86    void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign,
87                                uintptr_t RODataSize, uint32_t RODataAlign,
88                                uintptr_t RWDataSize,
89                                uint32_t RWDataAlign) override {
90      return ClientMM->reserveAllocationSpace(CodeSize, CodeAlign,
91                                              RODataSize, RODataAlign,
92                                              RWDataSize, RWDataAlign);
93    }
94
95    bool needsToReserveAllocationSpace() override {
96      return ClientMM->needsToReserveAllocationSpace();
97    }
98
99    void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr,
100                          size_t Size) override {
101      return ClientMM->registerEHFrames(Addr, LoadAddr, Size);
102    }
103
104    void deregisterEHFrames() override {
105      return ClientMM->deregisterEHFrames();
106    }
107
108    void notifyObjectLoaded(RuntimeDyld &RTDyld,
109                            const object::ObjectFile &O) override {
110      return ClientMM->notifyObjectLoaded(RTDyld, O);
111    }
112
113    void notifyObjectLoaded(ExecutionEngine *EE,
114                            const object::ObjectFile &O) override {
115      return ClientMM->notifyObjectLoaded(EE, O);
116    }
117
118    bool finalizeMemory(std::string *ErrMsg = nullptr) override {
119      // Each set of objects loaded will be finalized exactly once, but since
120      // symbol lookup during relocation may recursively trigger the
121      // loading/relocation of other modules, and since we're forwarding all
122      // finalizeMemory calls to a single underlying memory manager, we need to
123      // defer forwarding the call on until all necessary objects have been
124      // loaded. Otherwise, during the relocation of a leaf object, we will end
125      // up finalizing memory, causing a crash further up the stack when we
126      // attempt to apply relocations to finalized memory.
127      // To avoid finalizing too early, look at how many objects have been
128      // loaded but not yet finalized. This is a bit of a hack that relies on
129      // the fact that we're lazily emitting object files: The only way you can
130      // get more than one set of objects loaded but not yet finalized is if
131      // they were loaded during relocation of another set.
132      if (M.UnfinalizedSections.size() == 1)
133        return ClientMM->finalizeMemory(ErrMsg);
134      return false;
135    }
136
137  private:
138    OrcMCJITReplacement &M;
139    std::shared_ptr<MCJITMemoryManager> ClientMM;
140  };
141
142  class LinkingORCResolver : public orc::SymbolResolver {
143  public:
144    LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {}
145
146    SymbolNameSet getResponsibilitySet(const SymbolNameSet &Symbols) override {
147      SymbolNameSet Result;
148
149      for (auto &S : Symbols) {
150        if (auto Sym = M.findMangledSymbol(*S)) {
151          if (!Sym.getFlags().isStrong())
152            Result.insert(S);
153        } else if (auto Err = Sym.takeError()) {
154          M.reportError(std::move(Err));
155          return SymbolNameSet();
156        } else {
157          if (auto Sym2 =
158                  M.ClientResolver->findSymbolInLogicalDylib(std::string(*S))) {
159            if (!Sym2.getFlags().isStrong())
160              Result.insert(S);
161          } else if (auto Err = Sym2.takeError()) {
162            M.reportError(std::move(Err));
163            return SymbolNameSet();
164          } else
165            Result.insert(S);
166        }
167      }
168
169      return Result;
170    }
171
172    SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query,
173                         SymbolNameSet Symbols) override {
174      SymbolNameSet UnresolvedSymbols;
175      bool NewSymbolsResolved = false;
176
177      for (auto &S : Symbols) {
178        if (auto Sym = M.findMangledSymbol(*S)) {
179          if (auto Addr = Sym.getAddress()) {
180            Query->notifySymbolMetRequiredState(
181                S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
182            NewSymbolsResolved = true;
183          } else {
184            M.ES.legacyFailQuery(*Query, Addr.takeError());
185            return SymbolNameSet();
186          }
187        } else if (auto Err = Sym.takeError()) {
188          M.ES.legacyFailQuery(*Query, std::move(Err));
189          return SymbolNameSet();
190        } else {
191          if (auto Sym2 = M.ClientResolver->findSymbol(std::string(*S))) {
192            if (auto Addr = Sym2.getAddress()) {
193              Query->notifySymbolMetRequiredState(
194                  S, JITEvaluatedSymbol(*Addr, Sym2.getFlags()));
195              NewSymbolsResolved = true;
196            } else {
197              M.ES.legacyFailQuery(*Query, Addr.takeError());
198              return SymbolNameSet();
199            }
200          } else if (auto Err = Sym2.takeError()) {
201            M.ES.legacyFailQuery(*Query, std::move(Err));
202            return SymbolNameSet();
203          } else
204            UnresolvedSymbols.insert(S);
205        }
206      }
207
208      if (NewSymbolsResolved && Query->isComplete())
209        Query->handleComplete();
210
211      return UnresolvedSymbols;
212    }
213
214  private:
215    OrcMCJITReplacement &M;
216  };
217
218private:
219  static ExecutionEngine *
220  createOrcMCJITReplacement(std::string *ErrorMsg,
221                            std::shared_ptr<MCJITMemoryManager> MemMgr,
222                            std::shared_ptr<LegacyJITSymbolResolver> Resolver,
223                            std::unique_ptr<TargetMachine> TM) {
224    return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver),
225                                   std::move(TM));
226  }
227
228  void reportError(Error Err) {
229    logAllUnhandledErrors(std::move(Err), errs(), "MCJIT error: ");
230  }
231
232public:
233  OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr,
234                      std::shared_ptr<LegacyJITSymbolResolver> ClientResolver,
235                      std::unique_ptr<TargetMachine> TM)
236      : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)),
237        MemMgr(
238            std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))),
239        Resolver(std::make_shared<LinkingORCResolver>(*this)),
240        ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this),
241        NotifyFinalized(*this),
242        ObjectLayer(
243            AcknowledgeORCv1Deprecation, ES,
244            [this](VModuleKey K) {
245              return ObjectLayerT::Resources{this->MemMgr, this->Resolver};
246            },
247            NotifyObjectLoaded, NotifyFinalized),
248        CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer,
249                     SimpleCompiler(*this->TM),
250                     [this](VModuleKey K, std::unique_ptr<Module> M) {
251                       Modules.push_back(std::move(M));
252                     }),
253        LazyEmitLayer(AcknowledgeORCv1Deprecation, CompileLayer) {}
254
255  static void Register() {
256    OrcMCJITReplacementCtor = createOrcMCJITReplacement;
257  }
258
259  void addModule(std::unique_ptr<Module> M) override {
260    // If this module doesn't have a DataLayout attached then attach the
261    // default.
262    if (M->getDataLayout().isDefault()) {
263      M->setDataLayout(getDataLayout());
264    } else {
265      assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch");
266    }
267
268    // Rename, bump linkage and record static constructors and destructors.
269    // We have to do this before we hand over ownership of the module to the
270    // JIT.
271    std::vector<std::string> CtorNames, DtorNames;
272    {
273      unsigned CtorId = 0, DtorId = 0;
274      for (auto Ctor : orc::getConstructors(*M)) {
275        std::string NewCtorName = ("__ORCstatic_ctor." + Twine(CtorId++)).str();
276        Ctor.Func->setName(NewCtorName);
277        Ctor.Func->setLinkage(GlobalValue::ExternalLinkage);
278        Ctor.Func->setVisibility(GlobalValue::HiddenVisibility);
279        CtorNames.push_back(mangle(NewCtorName));
280      }
281      for (auto Dtor : orc::getDestructors(*M)) {
282        std::string NewDtorName = ("__ORCstatic_dtor." + Twine(DtorId++)).str();
283        dbgs() << "Found dtor: " << NewDtorName << "\n";
284        Dtor.Func->setName(NewDtorName);
285        Dtor.Func->setLinkage(GlobalValue::ExternalLinkage);
286        Dtor.Func->setVisibility(GlobalValue::HiddenVisibility);
287        DtorNames.push_back(mangle(NewDtorName));
288      }
289    }
290
291    auto K = ES.allocateVModule();
292
293    UnexecutedConstructors[K] = std::move(CtorNames);
294    UnexecutedDestructors[K] = std::move(DtorNames);
295
296    cantFail(LazyEmitLayer.addModule(K, std::move(M)));
297  }
298
299  void addObjectFile(std::unique_ptr<object::ObjectFile> O) override {
300    cantFail(ObjectLayer.addObject(
301        ES.allocateVModule(), MemoryBuffer::getMemBufferCopy(O->getData())));
302  }
303
304  void addObjectFile(object::OwningBinary<object::ObjectFile> O) override {
305    std::unique_ptr<object::ObjectFile> Obj;
306    std::unique_ptr<MemoryBuffer> ObjBuffer;
307    std::tie(Obj, ObjBuffer) = O.takeBinary();
308    cantFail(ObjectLayer.addObject(ES.allocateVModule(), std::move(ObjBuffer)));
309  }
310
311  void addArchive(object::OwningBinary<object::Archive> A) override {
312    Archives.push_back(std::move(A));
313  }
314
315  bool removeModule(Module *M) override {
316    auto I = Modules.begin();
317    for (auto E = Modules.end(); I != E; ++I)
318      if (I->get() == M)
319        break;
320    if (I == Modules.end())
321      return false;
322    Modules.erase(I);
323    return true;
324  }
325
326  uint64_t getSymbolAddress(StringRef Name) {
327    return cantFail(findSymbol(Name).getAddress());
328  }
329
330  JITSymbol findSymbol(StringRef Name) {
331    return findMangledSymbol(mangle(Name));
332  }
333
334  void finalizeObject() override {
335    // This is deprecated - Aim to remove in ExecutionEngine.
336    // REMOVE IF POSSIBLE - Doesn't make sense for New JIT.
337  }
338
339  void mapSectionAddress(const void *LocalAddress,
340                         uint64_t TargetAddress) override {
341    for (auto &P : UnfinalizedSections)
342      if (P.second.count(LocalAddress))
343        ObjectLayer.mapSectionAddress(P.first, LocalAddress, TargetAddress);
344  }
345
346  uint64_t getGlobalValueAddress(const std::string &Name) override {
347    return getSymbolAddress(Name);
348  }
349
350  uint64_t getFunctionAddress(const std::string &Name) override {
351    return getSymbolAddress(Name);
352  }
353
354  void *getPointerToFunction(Function *F) override {
355    uint64_t FAddr = getSymbolAddress(F->getName());
356    return reinterpret_cast<void *>(static_cast<uintptr_t>(FAddr));
357  }
358
359  void *getPointerToNamedFunction(StringRef Name,
360                                  bool AbortOnFailure = true) override {
361    uint64_t Addr = getSymbolAddress(Name);
362    if (!Addr && AbortOnFailure)
363      llvm_unreachable("Missing symbol!");
364    return reinterpret_cast<void *>(static_cast<uintptr_t>(Addr));
365  }
366
367  GenericValue runFunction(Function *F,
368                           ArrayRef<GenericValue> ArgValues) override;
369
370  void setObjectCache(ObjectCache *NewCache) override {
371    CompileLayer.getCompiler().setObjectCache(NewCache);
372  }
373
374  void setProcessAllSections(bool ProcessAllSections) override {
375    ObjectLayer.setProcessAllSections(ProcessAllSections);
376  }
377
378  void runStaticConstructorsDestructors(bool isDtors) override;
379
380private:
381  JITSymbol findMangledSymbol(StringRef Name) {
382    if (auto Sym = LazyEmitLayer.findSymbol(std::string(Name), false))
383      return Sym;
384    if (auto Sym = ClientResolver->findSymbol(std::string(Name)))
385      return Sym;
386    if (auto Sym = scanArchives(Name))
387      return Sym;
388
389    return nullptr;
390  }
391
392  JITSymbol scanArchives(StringRef Name) {
393    for (object::OwningBinary<object::Archive> &OB : Archives) {
394      object::Archive *A = OB.getBinary();
395      // Look for our symbols in each Archive
396      auto OptionalChildOrErr = A->findSym(Name);
397      if (!OptionalChildOrErr)
398        report_fatal_error(OptionalChildOrErr.takeError());
399      auto &OptionalChild = *OptionalChildOrErr;
400      if (OptionalChild) {
401        // FIXME: Support nested archives?
402        Expected<std::unique_ptr<object::Binary>> ChildBinOrErr =
403            OptionalChild->getAsBinary();
404        if (!ChildBinOrErr) {
405          // TODO: Actually report errors helpfully.
406          consumeError(ChildBinOrErr.takeError());
407          continue;
408        }
409        std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get();
410        if (ChildBin->isObject()) {
411          cantFail(ObjectLayer.addObject(
412              ES.allocateVModule(),
413              MemoryBuffer::getMemBufferCopy(ChildBin->getData())));
414          if (auto Sym = ObjectLayer.findSymbol(Name, true))
415            return Sym;
416        }
417      }
418    }
419    return nullptr;
420  }
421
422  class NotifyObjectLoadedT {
423  public:
424    using LoadedObjInfoListT =
425        std::vector<std::unique_ptr<RuntimeDyld::LoadedObjectInfo>>;
426
427    NotifyObjectLoadedT(OrcMCJITReplacement &M) : M(M) {}
428
429    void operator()(VModuleKey K, const object::ObjectFile &Obj,
430                    const RuntimeDyld::LoadedObjectInfo &Info) const {
431      M.UnfinalizedSections[K] = std::move(M.SectionsAllocatedSinceLastLoad);
432      M.SectionsAllocatedSinceLastLoad = SectionAddrSet();
433      M.MemMgr->notifyObjectLoaded(&M, Obj);
434    }
435  private:
436    OrcMCJITReplacement &M;
437  };
438
439  class NotifyFinalizedT {
440  public:
441    NotifyFinalizedT(OrcMCJITReplacement &M) : M(M) {}
442
443    void operator()(VModuleKey K, const object::ObjectFile &Obj,
444                    const RuntimeDyld::LoadedObjectInfo &Info) {
445      M.UnfinalizedSections.erase(K);
446    }
447
448  private:
449    OrcMCJITReplacement &M;
450  };
451
452  std::string mangle(StringRef Name) {
453    std::string MangledName;
454    {
455      raw_string_ostream MangledNameStream(MangledName);
456      Mang.getNameWithPrefix(MangledNameStream, Name, getDataLayout());
457    }
458    return MangledName;
459  }
460
461  using ObjectLayerT = LegacyRTDyldObjectLinkingLayer;
462  using CompileLayerT = LegacyIRCompileLayer<ObjectLayerT, orc::SimpleCompiler>;
463  using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>;
464
465  ExecutionSession ES;
466
467  std::unique_ptr<TargetMachine> TM;
468  std::shared_ptr<MCJITReplacementMemMgr> MemMgr;
469  std::shared_ptr<LinkingORCResolver> Resolver;
470  std::shared_ptr<LegacyJITSymbolResolver> ClientResolver;
471  Mangler Mang;
472
473  // IMPORTANT: ShouldDelete *must* come before LocalModules: The shared_ptr
474  // delete blocks in LocalModules refer to the ShouldDelete map, so
475  // LocalModules needs to be destructed before ShouldDelete.
476  std::map<Module*, bool> ShouldDelete;
477
478  NotifyObjectLoadedT NotifyObjectLoaded;
479  NotifyFinalizedT NotifyFinalized;
480
481  ObjectLayerT ObjectLayer;
482  CompileLayerT CompileLayer;
483  LazyEmitLayerT LazyEmitLayer;
484
485  std::map<VModuleKey, std::vector<std::string>> UnexecutedConstructors;
486  std::map<VModuleKey, std::vector<std::string>> UnexecutedDestructors;
487
488  // We need to store ObjLayerT::ObjSetHandles for each of the object sets
489  // that have been emitted but not yet finalized so that we can forward the
490  // mapSectionAddress calls appropriately.
491  using SectionAddrSet = std::set<const void *>;
492  SectionAddrSet SectionsAllocatedSinceLastLoad;
493  std::map<VModuleKey, SectionAddrSet> UnfinalizedSections;
494
495  std::vector<object::OwningBinary<object::Archive>> Archives;
496};
497
498} // end namespace orc
499
500} // end namespace llvm
501
502#endif // LLVM_LIB_EXECUTIONENGINE_ORC_MCJITREPLACEMENT_H
503