1//===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Contains utilities for adding indirections and breaking up modules.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
14#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15
16#include "llvm/ADT/StringMap.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/ExecutionEngine/JITSymbol.h"
20#include "llvm/ExecutionEngine/Orc/Core.h"
21#include "llvm/Support/Error.h"
22#include "llvm/Support/Memory.h"
23#include "llvm/Support/Process.h"
24#include "llvm/Transforms/Utils/ValueMapper.h"
25#include <algorithm>
26#include <cassert>
27#include <cstdint>
28#include <functional>
29#include <map>
30#include <memory>
31#include <system_error>
32#include <utility>
33#include <vector>
34
35namespace llvm {
36
37class Constant;
38class Function;
39class FunctionType;
40class GlobalAlias;
41class GlobalVariable;
42class Module;
43class PointerType;
44class Triple;
45class Value;
46
47namespace orc {
48
49/// Base class for pools of compiler re-entry trampolines.
50/// These trampolines are callable addresses that save all register state
51/// before calling a supplied function to return the trampoline landing
52/// address, then restore all state before jumping to that address. They
53/// are used by various ORC APIs to support lazy compilation
54class TrampolinePool {
55public:
56  virtual ~TrampolinePool() {}
57
58  /// Get an available trampoline address.
59  /// Returns an error if no trampoline can be created.
60  virtual Expected<JITTargetAddress> getTrampoline() = 0;
61
62private:
63  virtual void anchor();
64};
65
66/// A trampoline pool for trampolines within the current process.
67template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
68public:
69  using GetTrampolineLandingFunction =
70      std::function<JITTargetAddress(JITTargetAddress TrampolineAddr)>;
71
72  /// Creates a LocalTrampolinePool with the given RunCallback function.
73  /// Returns an error if this function is unable to correctly allocate, write
74  /// and protect the resolver code block.
75  static Expected<std::unique_ptr<LocalTrampolinePool>>
76  Create(GetTrampolineLandingFunction GetTrampolineLanding) {
77    Error Err = Error::success();
78
79    auto LTP = std::unique_ptr<LocalTrampolinePool>(
80        new LocalTrampolinePool(std::move(GetTrampolineLanding), Err));
81
82    if (Err)
83      return std::move(Err);
84    return std::move(LTP);
85  }
86
87  /// Get a free trampoline. Returns an error if one can not be provided (e.g.
88  /// because the pool is empty and can not be grown).
89  Expected<JITTargetAddress> getTrampoline() override {
90    std::lock_guard<std::mutex> Lock(LTPMutex);
91    if (AvailableTrampolines.empty()) {
92      if (auto Err = grow())
93        return std::move(Err);
94    }
95    assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
96    auto TrampolineAddr = AvailableTrampolines.back();
97    AvailableTrampolines.pop_back();
98    return TrampolineAddr;
99  }
100
101  /// Returns the given trampoline to the pool for re-use.
102  void releaseTrampoline(JITTargetAddress TrampolineAddr) {
103    std::lock_guard<std::mutex> Lock(LTPMutex);
104    AvailableTrampolines.push_back(TrampolineAddr);
105  }
106
107private:
108  static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
109    LocalTrampolinePool<ORCABI> *TrampolinePool =
110        static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
111    return TrampolinePool->GetTrampolineLanding(static_cast<JITTargetAddress>(
112        reinterpret_cast<uintptr_t>(TrampolineId)));
113  }
114
115  LocalTrampolinePool(GetTrampolineLandingFunction GetTrampolineLanding,
116                      Error &Err)
117      : GetTrampolineLanding(std::move(GetTrampolineLanding)) {
118
119    ErrorAsOutParameter _(&Err);
120
121    /// Try to set up the resolver block.
122    std::error_code EC;
123    ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
124        ORCABI::ResolverCodeSize, nullptr,
125        sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
126    if (EC) {
127      Err = errorCodeToError(EC);
128      return;
129    }
130
131    ORCABI::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()),
132                              &reenter, this);
133
134    EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
135                                          sys::Memory::MF_READ |
136                                              sys::Memory::MF_EXEC);
137    if (EC) {
138      Err = errorCodeToError(EC);
139      return;
140    }
141  }
142
143  Error grow() {
144    assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
145
146    std::error_code EC;
147    auto TrampolineBlock =
148        sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
149            sys::Process::getPageSizeEstimate(), nullptr,
150            sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
151    if (EC)
152      return errorCodeToError(EC);
153
154    unsigned NumTrampolines =
155        (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) /
156        ORCABI::TrampolineSize;
157
158    uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base());
159    ORCABI::writeTrampolines(TrampolineMem, ResolverBlock.base(),
160                             NumTrampolines);
161
162    for (unsigned I = 0; I < NumTrampolines; ++I)
163      this->AvailableTrampolines.push_back(
164          static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(
165              TrampolineMem + (I * ORCABI::TrampolineSize))));
166
167    if (auto EC = sys::Memory::protectMappedMemory(
168                    TrampolineBlock.getMemoryBlock(),
169                    sys::Memory::MF_READ | sys::Memory::MF_EXEC))
170      return errorCodeToError(EC);
171
172    TrampolineBlocks.push_back(std::move(TrampolineBlock));
173    return Error::success();
174  }
175
176  GetTrampolineLandingFunction GetTrampolineLanding;
177
178  std::mutex LTPMutex;
179  sys::OwningMemoryBlock ResolverBlock;
180  std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
181  std::vector<JITTargetAddress> AvailableTrampolines;
182};
183
184/// Target-independent base class for compile callback management.
185class JITCompileCallbackManager {
186public:
187  using CompileFunction = std::function<JITTargetAddress()>;
188
189  virtual ~JITCompileCallbackManager() = default;
190
191  /// Reserve a compile callback.
192  Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile);
193
194  /// Execute the callback for the given trampoline id. Called by the JIT
195  ///        to compile functions on demand.
196  JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr);
197
198protected:
199  /// Construct a JITCompileCallbackManager.
200  JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,
201                            ExecutionSession &ES,
202                            JITTargetAddress ErrorHandlerAddress)
203      : TP(std::move(TP)), ES(ES),
204        CallbacksJD(ES.createJITDylib("<Callbacks>")),
205        ErrorHandlerAddress(ErrorHandlerAddress) {}
206
207  void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
208    this->TP = std::move(TP);
209  }
210
211private:
212  std::mutex CCMgrMutex;
213  std::unique_ptr<TrampolinePool> TP;
214  ExecutionSession &ES;
215  JITDylib &CallbacksJD;
216  JITTargetAddress ErrorHandlerAddress;
217  std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;
218  size_t NextCallbackId = 0;
219};
220
221/// Manage compile callbacks for in-process JITs.
222template <typename ORCABI>
223class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
224public:
225  /// Create a new LocalJITCompileCallbackManager.
226  static Expected<std::unique_ptr<LocalJITCompileCallbackManager>>
227  Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) {
228    Error Err = Error::success();
229    auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
230        new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));
231    if (Err)
232      return std::move(Err);
233    return std::move(CCMgr);
234  }
235
236private:
237  /// Construct a InProcessJITCompileCallbackManager.
238  /// @param ErrorHandlerAddress The address of an error handler in the target
239  ///                            process to be used if a compile callback fails.
240  LocalJITCompileCallbackManager(ExecutionSession &ES,
241                                 JITTargetAddress ErrorHandlerAddress,
242                                 Error &Err)
243      : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
244    ErrorAsOutParameter _(&Err);
245    auto TP = LocalTrampolinePool<ORCABI>::Create(
246        [this](JITTargetAddress TrampolineAddr) {
247          return executeCompileCallback(TrampolineAddr);
248        });
249
250    if (!TP) {
251      Err = TP.takeError();
252      return;
253    }
254
255    setTrampolinePool(std::move(*TP));
256  }
257};
258
259/// Base class for managing collections of named indirect stubs.
260class IndirectStubsManager {
261public:
262  /// Map type for initializing the manager. See init.
263  using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>;
264
265  virtual ~IndirectStubsManager() = default;
266
267  /// Create a single stub with the given name, target address and flags.
268  virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,
269                           JITSymbolFlags StubFlags) = 0;
270
271  /// Create StubInits.size() stubs with the given names, target
272  ///        addresses, and flags.
273  virtual Error createStubs(const StubInitsMap &StubInits) = 0;
274
275  /// Find the stub with the given name. If ExportedStubsOnly is true,
276  ///        this will only return a result if the stub's flags indicate that it
277  ///        is exported.
278  virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
279
280  /// Find the implementation-pointer for the stub.
281  virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0;
282
283  /// Change the value of the implementation pointer for the stub.
284  virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;
285
286private:
287  virtual void anchor();
288};
289
290/// IndirectStubsManager implementation for the host architecture, e.g.
291///        OrcX86_64. (See OrcArchitectureSupport.h).
292template <typename TargetT>
293class LocalIndirectStubsManager : public IndirectStubsManager {
294public:
295  Error createStub(StringRef StubName, JITTargetAddress StubAddr,
296                   JITSymbolFlags StubFlags) override {
297    std::lock_guard<std::mutex> Lock(StubsMutex);
298    if (auto Err = reserveStubs(1))
299      return Err;
300
301    createStubInternal(StubName, StubAddr, StubFlags);
302
303    return Error::success();
304  }
305
306  Error createStubs(const StubInitsMap &StubInits) override {
307    std::lock_guard<std::mutex> Lock(StubsMutex);
308    if (auto Err = reserveStubs(StubInits.size()))
309      return Err;
310
311    for (auto &Entry : StubInits)
312      createStubInternal(Entry.first(), Entry.second.first,
313                         Entry.second.second);
314
315    return Error::success();
316  }
317
318  JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
319    std::lock_guard<std::mutex> Lock(StubsMutex);
320    auto I = StubIndexes.find(Name);
321    if (I == StubIndexes.end())
322      return nullptr;
323    auto Key = I->second.first;
324    void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
325    assert(StubAddr && "Missing stub address");
326    auto StubTargetAddr =
327        static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
328    auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second);
329    if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
330      return nullptr;
331    return StubSymbol;
332  }
333
334  JITEvaluatedSymbol findPointer(StringRef Name) override {
335    std::lock_guard<std::mutex> Lock(StubsMutex);
336    auto I = StubIndexes.find(Name);
337    if (I == StubIndexes.end())
338      return nullptr;
339    auto Key = I->second.first;
340    void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
341    assert(PtrAddr && "Missing pointer address");
342    auto PtrTargetAddr =
343        static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
344    return JITEvaluatedSymbol(PtrTargetAddr, I->second.second);
345  }
346
347  Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override {
348    using AtomicIntPtr = std::atomic<uintptr_t>;
349
350    std::lock_guard<std::mutex> Lock(StubsMutex);
351    auto I = StubIndexes.find(Name);
352    assert(I != StubIndexes.end() && "No stub pointer for symbol");
353    auto Key = I->second.first;
354    AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(
355        IndirectStubsInfos[Key.first].getPtr(Key.second));
356    *AtomicStubPtr = static_cast<uintptr_t>(NewAddr);
357    return Error::success();
358  }
359
360private:
361  Error reserveStubs(unsigned NumStubs) {
362    if (NumStubs <= FreeStubs.size())
363      return Error::success();
364
365    unsigned NewStubsRequired = NumStubs - FreeStubs.size();
366    unsigned NewBlockId = IndirectStubsInfos.size();
367    typename TargetT::IndirectStubsInfo ISI;
368    if (auto Err =
369            TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, nullptr))
370      return Err;
371    for (unsigned I = 0; I < ISI.getNumStubs(); ++I)
372      FreeStubs.push_back(std::make_pair(NewBlockId, I));
373    IndirectStubsInfos.push_back(std::move(ISI));
374    return Error::success();
375  }
376
377  void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,
378                          JITSymbolFlags StubFlags) {
379    auto Key = FreeStubs.back();
380    FreeStubs.pop_back();
381    *IndirectStubsInfos[Key.first].getPtr(Key.second) =
382        reinterpret_cast<void *>(static_cast<uintptr_t>(InitAddr));
383    StubIndexes[StubName] = std::make_pair(Key, StubFlags);
384  }
385
386  std::mutex StubsMutex;
387  std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos;
388  using StubKey = std::pair<uint16_t, uint16_t>;
389  std::vector<StubKey> FreeStubs;
390  StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
391};
392
393/// Create a local compile callback manager.
394///
395/// The given target triple will determine the ABI, and the given
396/// ErrorHandlerAddress will be used by the resulting compile callback
397/// manager if a compile callback fails.
398Expected<std::unique_ptr<JITCompileCallbackManager>>
399createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
400                                  JITTargetAddress ErrorHandlerAddress);
401
402/// Create a local indriect stubs manager builder.
403///
404/// The given target triple will determine the ABI.
405std::function<std::unique_ptr<IndirectStubsManager>()>
406createLocalIndirectStubsManagerBuilder(const Triple &T);
407
408/// Build a function pointer of FunctionType with the given constant
409///        address.
410///
411///   Usage example: Turn a trampoline address into a function pointer constant
412/// for use in a stub.
413Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr);
414
415/// Create a function pointer with the given type, name, and initializer
416///        in the given Module.
417GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,
418                                  Constant *Initializer);
419
420/// Turn a function declaration into a stub function that makes an
421///        indirect call using the given function pointer.
422void makeStub(Function &F, Value &ImplPointer);
423
424/// Promotes private symbols to global hidden, and renames to prevent clashes
425/// with other promoted symbols. The same SymbolPromoter instance should be
426/// used for all symbols to be added to a single JITDylib.
427class SymbolLinkagePromoter {
428public:
429  /// Promote symbols in the given module. Returns the set of global values
430  /// that have been renamed/promoted.
431  std::vector<GlobalValue *> operator()(Module &M);
432
433private:
434  unsigned NextId = 0;
435};
436
437/// Clone a function declaration into a new module.
438///
439///   This function can be used as the first step towards creating a callback
440/// stub (see makeStub), or moving a function body (see moveFunctionBody).
441///
442///   If the VMap argument is non-null, a mapping will be added between F and
443/// the new declaration, and between each of F's arguments and the new
444/// declaration's arguments. This map can then be passed in to moveFunction to
445/// move the function body if required. Note: When moving functions between
446/// modules with these utilities, all decls should be cloned (and added to a
447/// single VMap) before any bodies are moved. This will ensure that references
448/// between functions all refer to the versions in the new module.
449Function *cloneFunctionDecl(Module &Dst, const Function &F,
450                            ValueToValueMapTy *VMap = nullptr);
451
452/// Move the body of function 'F' to a cloned function declaration in a
453///        different module (See related cloneFunctionDecl).
454///
455///   If the target function declaration is not supplied via the NewF parameter
456/// then it will be looked up via the VMap.
457///
458///   This will delete the body of function 'F' from its original parent module,
459/// but leave its declaration.
460void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
461                      ValueMaterializer *Materializer = nullptr,
462                      Function *NewF = nullptr);
463
464/// Clone a global variable declaration into a new module.
465GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
466                                        ValueToValueMapTy *VMap = nullptr);
467
468/// Move global variable GV from its parent module to cloned global
469///        declaration in a different module.
470///
471///   If the target global declaration is not supplied via the NewGV parameter
472/// then it will be looked up via the VMap.
473///
474///   This will delete the initializer of GV from its original parent module,
475/// but leave its declaration.
476void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
477                                   ValueToValueMapTy &VMap,
478                                   ValueMaterializer *Materializer = nullptr,
479                                   GlobalVariable *NewGV = nullptr);
480
481/// Clone a global alias declaration into a new module.
482GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
483                                  ValueToValueMapTy &VMap);
484
485/// Clone module flags metadata into the destination module.
486void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
487                              ValueToValueMapTy &VMap);
488
489} // end namespace orc
490
491} // end namespace llvm
492
493#endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
494