1//===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10#include "llvm/ADT/STLExtras.h"
11#include "llvm/ExecutionEngine/JITLink/x86_64.h"
12#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
13#include "llvm/IR/IRBuilder.h"
14#include "llvm/MC/MCDisassembler/MCDisassembler.h"
15#include "llvm/MC/MCInstrAnalysis.h"
16#include "llvm/Support/Format.h"
17#include "llvm/TargetParser/Triple.h"
18#include "llvm/Transforms/Utils/Cloning.h"
19#include <sstream>
20
21#define DEBUG_TYPE "orc"
22
23using namespace llvm;
24using namespace llvm::orc;
25
26namespace {
27
28class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
29public:
30  using CompileFunction = JITCompileCallbackManager::CompileFunction;
31
32  CompileCallbackMaterializationUnit(SymbolStringPtr Name,
33                                     CompileFunction Compile)
34      : MaterializationUnit(Interface(
35            SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
36        Name(std::move(Name)), Compile(std::move(Compile)) {}
37
38  StringRef getName() const override { return "<Compile Callbacks>"; }
39
40private:
41  void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
42    SymbolMap Result;
43    Result[Name] = {Compile(), JITSymbolFlags::Exported};
44    // No dependencies, so these calls cannot fail.
45    cantFail(R->notifyResolved(Result));
46    cantFail(R->notifyEmitted());
47  }
48
49  void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
50    llvm_unreachable("Discard should never occur on a LMU?");
51  }
52
53  SymbolStringPtr Name;
54  CompileFunction Compile;
55};
56
57} // namespace
58
59namespace llvm {
60namespace orc {
61
62TrampolinePool::~TrampolinePool() = default;
63void IndirectStubsManager::anchor() {}
64
65Expected<ExecutorAddr>
66JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
67  if (auto TrampolineAddr = TP->getTrampoline()) {
68    auto CallbackName =
69        ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
70
71    std::lock_guard<std::mutex> Lock(CCMgrMutex);
72    AddrToSymbol[*TrampolineAddr] = CallbackName;
73    cantFail(
74        CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
75            std::move(CallbackName), std::move(Compile))));
76    return *TrampolineAddr;
77  } else
78    return TrampolineAddr.takeError();
79}
80
81ExecutorAddr
82JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) {
83  SymbolStringPtr Name;
84
85  {
86    std::unique_lock<std::mutex> Lock(CCMgrMutex);
87    auto I = AddrToSymbol.find(TrampolineAddr);
88
89    // If this address is not associated with a compile callback then report an
90    // error to the execution session and return ErrorHandlerAddress to the
91    // callee.
92    if (I == AddrToSymbol.end()) {
93      Lock.unlock();
94      ES.reportError(
95          make_error<StringError>("No compile callback for trampoline at " +
96                                      formatv("{0:x}", TrampolineAddr),
97                                  inconvertibleErrorCode()));
98      return ErrorHandlerAddress;
99    } else
100      Name = I->second;
101  }
102
103  if (auto Sym =
104          ES.lookup(makeJITDylibSearchOrder(
105                        &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
106                    Name))
107    return Sym->getAddress();
108  else {
109    llvm::dbgs() << "Didn't find callback.\n";
110    // If anything goes wrong materializing Sym then report it to the session
111    // and return the ErrorHandlerAddress;
112    ES.reportError(Sym.takeError());
113    return ErrorHandlerAddress;
114  }
115}
116
117Expected<std::unique_ptr<JITCompileCallbackManager>>
118createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
119                                  ExecutorAddr ErrorHandlerAddress) {
120  switch (T.getArch()) {
121  default:
122    return make_error<StringError>(
123        std::string("No callback manager available for ") + T.str(),
124        inconvertibleErrorCode());
125  case Triple::aarch64:
126  case Triple::aarch64_32: {
127    typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
128    return CCMgrT::Create(ES, ErrorHandlerAddress);
129    }
130
131    case Triple::x86: {
132      typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
133      return CCMgrT::Create(ES, ErrorHandlerAddress);
134    }
135
136    case Triple::loongarch64: {
137      typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;
138      return CCMgrT::Create(ES, ErrorHandlerAddress);
139    }
140
141    case Triple::mips: {
142      typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
143      return CCMgrT::Create(ES, ErrorHandlerAddress);
144    }
145    case Triple::mipsel: {
146      typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
147      return CCMgrT::Create(ES, ErrorHandlerAddress);
148    }
149
150    case Triple::mips64:
151    case Triple::mips64el: {
152      typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
153      return CCMgrT::Create(ES, ErrorHandlerAddress);
154    }
155
156    case Triple::riscv64: {
157      typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
158      return CCMgrT::Create(ES, ErrorHandlerAddress);
159    }
160
161    case Triple::x86_64: {
162      if (T.getOS() == Triple::OSType::Win32) {
163        typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
164        return CCMgrT::Create(ES, ErrorHandlerAddress);
165      } else {
166        typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
167        return CCMgrT::Create(ES, ErrorHandlerAddress);
168      }
169    }
170
171  }
172}
173
174std::function<std::unique_ptr<IndirectStubsManager>()>
175createLocalIndirectStubsManagerBuilder(const Triple &T) {
176  switch (T.getArch()) {
177    default:
178      return [](){
179        return std::make_unique<
180                       orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
181      };
182
183    case Triple::aarch64:
184    case Triple::aarch64_32:
185      return [](){
186        return std::make_unique<
187                       orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
188      };
189
190    case Triple::x86:
191      return [](){
192        return std::make_unique<
193                       orc::LocalIndirectStubsManager<orc::OrcI386>>();
194      };
195
196    case Triple::loongarch64:
197      return []() {
198        return std::make_unique<
199            orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();
200      };
201
202    case Triple::mips:
203      return [](){
204          return std::make_unique<
205                      orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
206      };
207
208    case Triple::mipsel:
209      return [](){
210          return std::make_unique<
211                      orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
212      };
213
214    case Triple::mips64:
215    case Triple::mips64el:
216      return [](){
217          return std::make_unique<
218                      orc::LocalIndirectStubsManager<orc::OrcMips64>>();
219      };
220
221    case Triple::riscv64:
222      return []() {
223        return std::make_unique<
224            orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
225      };
226
227    case Triple::x86_64:
228      if (T.getOS() == Triple::OSType::Win32) {
229        return [](){
230          return std::make_unique<
231                     orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
232        };
233      } else {
234        return [](){
235          return std::make_unique<
236                     orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
237        };
238      }
239
240  }
241}
242
243Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) {
244  Constant *AddrIntVal =
245    ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue());
246  Constant *AddrPtrVal =
247    ConstantExpr::getIntToPtr(AddrIntVal, PointerType::get(&FT, 0));
248  return AddrPtrVal;
249}
250
251GlobalVariable* createImplPointer(PointerType &PT, Module &M,
252                                  const Twine &Name, Constant *Initializer) {
253  auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
254                               Initializer, Name, nullptr,
255                               GlobalValue::NotThreadLocal, 0, true);
256  IP->setVisibility(GlobalValue::HiddenVisibility);
257  return IP;
258}
259
260void makeStub(Function &F, Value &ImplPointer) {
261  assert(F.isDeclaration() && "Can't turn a definition into a stub.");
262  assert(F.getParent() && "Function isn't in a module.");
263  Module &M = *F.getParent();
264  BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
265  IRBuilder<> Builder(EntryBlock);
266  LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
267  std::vector<Value*> CallArgs;
268  for (auto &A : F.args())
269    CallArgs.push_back(&A);
270  CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
271  Call->setTailCall();
272  Call->setAttributes(F.getAttributes());
273  if (F.getReturnType()->isVoidTy())
274    Builder.CreateRetVoid();
275  else
276    Builder.CreateRet(Call);
277}
278
279std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
280  std::vector<GlobalValue *> PromotedGlobals;
281
282  for (auto &GV : M.global_values()) {
283    bool Promoted = true;
284
285    // Rename if necessary.
286    if (!GV.hasName())
287      GV.setName("__orc_anon." + Twine(NextId++));
288    else if (GV.getName().starts_with("\01L"))
289      GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
290    else if (GV.hasLocalLinkage())
291      GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
292    else
293      Promoted = false;
294
295    if (GV.hasLocalLinkage()) {
296      GV.setLinkage(GlobalValue::ExternalLinkage);
297      GV.setVisibility(GlobalValue::HiddenVisibility);
298      Promoted = true;
299    }
300    GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
301
302    if (Promoted)
303      PromotedGlobals.push_back(&GV);
304  }
305
306  return PromotedGlobals;
307}
308
309Function* cloneFunctionDecl(Module &Dst, const Function &F,
310                            ValueToValueMapTy *VMap) {
311  Function *NewF =
312    Function::Create(cast<FunctionType>(F.getValueType()),
313                     F.getLinkage(), F.getName(), &Dst);
314  NewF->copyAttributesFrom(&F);
315
316  if (VMap) {
317    (*VMap)[&F] = NewF;
318    auto NewArgI = NewF->arg_begin();
319    for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
320         ++ArgI, ++NewArgI)
321      (*VMap)[&*ArgI] = &*NewArgI;
322  }
323
324  return NewF;
325}
326
327GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
328                                        ValueToValueMapTy *VMap) {
329  GlobalVariable *NewGV = new GlobalVariable(
330      Dst, GV.getValueType(), GV.isConstant(),
331      GV.getLinkage(), nullptr, GV.getName(), nullptr,
332      GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
333  NewGV->copyAttributesFrom(&GV);
334  if (VMap)
335    (*VMap)[&GV] = NewGV;
336  return NewGV;
337}
338
339GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
340                                  ValueToValueMapTy &VMap) {
341  assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
342  auto *NewA = GlobalAlias::create(OrigA.getValueType(),
343                                   OrigA.getType()->getPointerAddressSpace(),
344                                   OrigA.getLinkage(), OrigA.getName(), &Dst);
345  NewA->copyAttributesFrom(&OrigA);
346  VMap[&OrigA] = NewA;
347  return NewA;
348}
349
350Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
351                                                   jitlink::LinkGraph &G,
352                                                   MCDisassembler &Disassembler,
353                                                   MCInstrAnalysis &MIA) {
354  // AArch64 appears to already come with the necessary relocations. Among other
355  // architectures, only x86_64 is currently implemented here.
356  if (G.getTargetTriple().getArch() != Triple::x86_64)
357    return Error::success();
358
359  raw_null_ostream CommentStream;
360  auto &STI = Disassembler.getSubtargetInfo();
361
362  // Determine the function bounds
363  auto &B = Sym.getBlock();
364  assert(!B.isZeroFill() && "expected content block");
365  auto SymAddress = Sym.getAddress();
366  auto SymStartInBlock =
367      (const uint8_t *)B.getContent().data() + Sym.getOffset();
368  auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
369  auto Content = ArrayRef(SymStartInBlock, SymSize);
370
371  LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
372
373  SmallDenseSet<uintptr_t, 8> ExistingRelocations;
374  for (auto &E : B.edges()) {
375    if (E.isRelocation())
376      ExistingRelocations.insert(E.getOffset());
377  }
378
379  size_t I = 0;
380  while (I < Content.size()) {
381    MCInst Instr;
382    uint64_t InstrSize = 0;
383    uint64_t InstrStart = SymAddress.getValue() + I;
384    auto DecodeStatus = Disassembler.getInstruction(
385        Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
386    if (DecodeStatus != MCDisassembler::Success) {
387      LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
388                        << InstrStart);
389      return make_error<StringError>(
390          formatv("failed to disassemble at address {0:x16}", InstrStart),
391          inconvertibleErrorCode());
392    }
393    // Advance to the next instruction.
394    I += InstrSize;
395
396    // Check for a PC-relative address equal to the symbol itself.
397    auto PCRelAddr =
398        MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
399    if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
400      continue;
401
402    auto RelocOffInInstr =
403        MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
404    if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
405      LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
406                        << InstrStart);
407      continue;
408    }
409
410    auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
411                           SymAddress + Sym.getOffset();
412    if (ExistingRelocations.contains(RelocOffInBlock))
413      continue;
414
415    LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
416    B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
417  }
418  return Error::success();
419}
420
421} // End namespace orc.
422} // End namespace llvm.
423