1//===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10
11#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12#include "llvm/Support/MathExtras.h"
13
14#include <future>
15
16using namespace llvm;
17using namespace llvm::orc;
18
19namespace llvm {
20namespace orc {
21
22class EPCIndirectionUtilsAccess {
23public:
24  using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
25  using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
26
27  static Expected<IndirectStubInfoVector>
28  getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
29    return EPCIU.getIndirectStubs(NumStubs);
30  };
31};
32
33} // end namespace orc
34} // end namespace llvm
35
36namespace {
37
38class EPCTrampolinePool : public TrampolinePool {
39public:
40  EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
41  Error deallocatePool();
42
43protected:
44  Error grow() override;
45
46  using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
47
48  EPCIndirectionUtils &EPCIU;
49  unsigned TrampolineSize = 0;
50  unsigned TrampolinesPerPage = 0;
51  std::vector<FinalizedAlloc> TrampolineBlocks;
52};
53
54class EPCIndirectStubsManager : public IndirectStubsManager,
55                                private EPCIndirectionUtilsAccess {
56public:
57  EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
58
59  Error deallocateStubs();
60
61  Error createStub(StringRef StubName, ExecutorAddr StubAddr,
62                   JITSymbolFlags StubFlags) override;
63
64  Error createStubs(const StubInitsMap &StubInits) override;
65
66  ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override;
67
68  ExecutorSymbolDef findPointer(StringRef Name) override;
69
70  Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override;
71
72private:
73  using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74
75  std::mutex ISMMutex;
76  EPCIndirectionUtils &EPCIU;
77  StringMap<StubInfo> StubInfos;
78};
79
80EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
81    : EPCIU(EPCIU) {
82  auto &EPC = EPCIU.getExecutorProcessControl();
83  auto &ABI = EPCIU.getABISupport();
84
85  TrampolineSize = ABI.getTrampolineSize();
86  TrampolinesPerPage =
87      (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88}
89
90Error EPCTrampolinePool::deallocatePool() {
91  std::promise<MSVCPError> DeallocResultP;
92  auto DeallocResultF = DeallocResultP.get_future();
93
94  EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
95      std::move(TrampolineBlocks),
96      [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
97
98  return DeallocResultF.get();
99}
100
101Error EPCTrampolinePool::grow() {
102  using namespace jitlink;
103
104  assert(AvailableTrampolines.empty() &&
105         "Grow called with trampolines still available");
106
107  auto ResolverAddress = EPCIU.getResolverBlockAddress();
108  assert(ResolverAddress && "Resolver address can not be null");
109
110  auto &EPC = EPCIU.getExecutorProcessControl();
111  auto PageSize = EPC.getPageSize();
112  auto Alloc = SimpleSegmentAlloc::Create(
113      EPC.getMemMgr(), nullptr,
114      {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
115  if (!Alloc)
116    return Alloc.takeError();
117
118  unsigned NumTrampolines = TrampolinesPerPage;
119
120  auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
121  EPCIU.getABISupport().writeTrampolines(
122      SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines);
123  for (unsigned I = 0; I < NumTrampolines; ++I)
124    AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize));
125
126  auto FA = Alloc->finalize();
127  if (!FA)
128    return FA.takeError();
129
130  TrampolineBlocks.push_back(std::move(*FA));
131
132  return Error::success();
133}
134
135Error EPCIndirectStubsManager::createStub(StringRef StubName,
136                                          ExecutorAddr StubAddr,
137                                          JITSymbolFlags StubFlags) {
138  StubInitsMap SIM;
139  SIM[StubName] = std::make_pair(StubAddr, StubFlags);
140  return createStubs(SIM);
141}
142
143Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
144  auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
145  if (!AvailableStubInfos)
146    return AvailableStubInfos.takeError();
147
148  {
149    std::lock_guard<std::mutex> Lock(ISMMutex);
150    unsigned ASIdx = 0;
151    for (auto &SI : StubInits) {
152      auto &A = (*AvailableStubInfos)[ASIdx++];
153      StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
154    }
155  }
156
157  auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
158  switch (EPCIU.getABISupport().getPointerSize()) {
159  case 4: {
160    unsigned ASIdx = 0;
161    std::vector<tpctypes::UInt32Write> PtrUpdates;
162    for (auto &SI : StubInits)
163      PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
164                            static_cast<uint32_t>(SI.second.first.getValue())});
165    return MemAccess.writeUInt32s(PtrUpdates);
166  }
167  case 8: {
168    unsigned ASIdx = 0;
169    std::vector<tpctypes::UInt64Write> PtrUpdates;
170    for (auto &SI : StubInits)
171      PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
172                            static_cast<uint64_t>(SI.second.first.getValue())});
173    return MemAccess.writeUInt64s(PtrUpdates);
174  }
175  default:
176    return make_error<StringError>("Unsupported pointer size",
177                                   inconvertibleErrorCode());
178  }
179}
180
181ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name,
182                                                    bool ExportedStubsOnly) {
183  std::lock_guard<std::mutex> Lock(ISMMutex);
184  auto I = StubInfos.find(Name);
185  if (I == StubInfos.end())
186    return ExecutorSymbolDef();
187  return {I->second.first.StubAddress, I->second.second};
188}
189
190ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) {
191  std::lock_guard<std::mutex> Lock(ISMMutex);
192  auto I = StubInfos.find(Name);
193  if (I == StubInfos.end())
194    return ExecutorSymbolDef();
195  return {I->second.first.PointerAddress, I->second.second};
196}
197
198Error EPCIndirectStubsManager::updatePointer(StringRef Name,
199                                             ExecutorAddr NewAddr) {
200
201  ExecutorAddr PtrAddr;
202  {
203    std::lock_guard<std::mutex> Lock(ISMMutex);
204    auto I = StubInfos.find(Name);
205    if (I == StubInfos.end())
206      return make_error<StringError>("Unknown stub name",
207                                     inconvertibleErrorCode());
208    PtrAddr = I->second.first.PointerAddress;
209  }
210
211  auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
212  switch (EPCIU.getABISupport().getPointerSize()) {
213  case 4: {
214    tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue());
215    return MemAccess.writeUInt32s(PUpdate);
216  }
217  case 8: {
218    tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue());
219    return MemAccess.writeUInt64s(PUpdate);
220  }
221  default:
222    return make_error<StringError>("Unsupported pointer size",
223                                   inconvertibleErrorCode());
224  }
225}
226
227} // end anonymous namespace.
228
229namespace llvm {
230namespace orc {
231
232EPCIndirectionUtils::ABISupport::~ABISupport() = default;
233
234Expected<std::unique_ptr<EPCIndirectionUtils>>
235EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
236  const auto &TT = EPC.getTargetTriple();
237  switch (TT.getArch()) {
238  default:
239    return make_error<StringError>(
240        std::string("No EPCIndirectionUtils available for ") + TT.str(),
241        inconvertibleErrorCode());
242  case Triple::aarch64:
243  case Triple::aarch64_32:
244    return CreateWithABI<OrcAArch64>(EPC);
245
246  case Triple::x86:
247    return CreateWithABI<OrcI386>(EPC);
248
249  case Triple::loongarch64:
250    return CreateWithABI<OrcLoongArch64>(EPC);
251
252  case Triple::mips:
253    return CreateWithABI<OrcMips32Be>(EPC);
254
255  case Triple::mipsel:
256    return CreateWithABI<OrcMips32Le>(EPC);
257
258  case Triple::mips64:
259  case Triple::mips64el:
260    return CreateWithABI<OrcMips64>(EPC);
261
262  case Triple::riscv64:
263    return CreateWithABI<OrcRiscv64>(EPC);
264
265  case Triple::x86_64:
266    if (TT.getOS() == Triple::OSType::Win32)
267      return CreateWithABI<OrcX86_64_Win32>(EPC);
268    else
269      return CreateWithABI<OrcX86_64_SysV>(EPC);
270  }
271}
272
273Error EPCIndirectionUtils::cleanup() {
274
275  auto &MemMgr = EPC.getMemMgr();
276  auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
277
278  if (TP)
279    Err = joinErrors(std::move(Err),
280                     static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
281
282  if (ResolverBlock)
283    Err =
284        joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
285
286  return Err;
287}
288
289Expected<ExecutorAddr>
290EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr,
291                                        ExecutorAddr ReentryCtxAddr) {
292  using namespace jitlink;
293
294  assert(ABI && "ABI can not be null");
295  auto ResolverSize = ABI->getResolverCodeSize();
296
297  auto Alloc =
298      SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
299                                 {{MemProt::Read | MemProt::Exec,
300                                   {ResolverSize, Align(EPC.getPageSize())}}});
301
302  if (!Alloc)
303    return Alloc.takeError();
304
305  auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
306  ResolverBlockAddr = SegInfo.Addr;
307  ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,
308                         ReentryFnAddr, ReentryCtxAddr);
309
310  auto FA = Alloc->finalize();
311  if (!FA)
312    return FA.takeError();
313
314  ResolverBlock = std::move(*FA);
315  return ResolverBlockAddr;
316}
317
318std::unique_ptr<IndirectStubsManager>
319EPCIndirectionUtils::createIndirectStubsManager() {
320  return std::make_unique<EPCIndirectStubsManager>(*this);
321}
322
323TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
324  if (!TP)
325    TP = std::make_unique<EPCTrampolinePool>(*this);
326  return *TP;
327}
328
329LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
330    ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) {
331  assert(!LCTM &&
332         "createLazyCallThroughManager can not have been called before");
333  LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
334                                                  &getTrampolinePool());
335  return *LCTM;
336}
337
338EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
339                                         std::unique_ptr<ABISupport> ABI)
340    : EPC(EPC), ABI(std::move(ABI)) {
341  assert(this->ABI && "ABI can not be null");
342
343  assert(EPC.getPageSize() > getABISupport().getStubSize() &&
344         "Stubs larger than one page are not supported");
345}
346
347Expected<EPCIndirectionUtils::IndirectStubInfoVector>
348EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
349  using namespace jitlink;
350
351  std::lock_guard<std::mutex> Lock(EPCUIMutex);
352
353  // If there aren't enough stubs available then allocate some more.
354  if (NumStubs > AvailableIndirectStubs.size()) {
355    auto NumStubsToAllocate = NumStubs;
356    auto PageSize = EPC.getPageSize();
357    auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
358    NumStubsToAllocate = StubBytes / ABI->getStubSize();
359    auto PtrBytes =
360        alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
361
362    auto StubProt = MemProt::Read | MemProt::Exec;
363    auto PtrProt = MemProt::Read | MemProt::Write;
364
365    auto Alloc = SimpleSegmentAlloc::Create(
366        EPC.getMemMgr(), nullptr,
367        {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
368         {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
369
370    if (!Alloc)
371      return Alloc.takeError();
372
373    auto StubSeg = Alloc->getSegInfo(StubProt);
374    auto PtrSeg = Alloc->getSegInfo(PtrProt);
375
376    ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr,
377                                 PtrSeg.Addr, NumStubsToAllocate);
378
379    auto FA = Alloc->finalize();
380    if (!FA)
381      return FA.takeError();
382
383    IndirectStubAllocs.push_back(std::move(*FA));
384
385    auto StubExecutorAddr = StubSeg.Addr;
386    auto PtrExecutorAddr = PtrSeg.Addr;
387    for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
388      AvailableIndirectStubs.push_back(
389          IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr));
390      StubExecutorAddr += ABI->getStubSize();
391      PtrExecutorAddr += ABI->getPointerSize();
392    }
393  }
394
395  assert(NumStubs <= AvailableIndirectStubs.size() &&
396         "Sufficient stubs should have been allocated above");
397
398  IndirectStubInfoVector Result;
399  while (NumStubs--) {
400    Result.push_back(AvailableIndirectStubs.back());
401    AvailableIndirectStubs.pop_back();
402  }
403
404  return std::move(Result);
405}
406
407static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
408                                JITTargetAddress TrampolineAddr) {
409  auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
410  std::promise<ExecutorAddr> LandingAddrP;
411  auto LandingAddrF = LandingAddrP.get_future();
412  LCTM.resolveTrampolineLandingAddress(
413      ExecutorAddr(TrampolineAddr),
414      [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); });
415  return LandingAddrF.get().getValue();
416}
417
418Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
419  auto &LCTM = EPCIU.getLazyCallThroughManager();
420  return EPCIU
421      .writeResolverBlock(ExecutorAddr::fromPtr(&reentry),
422                          ExecutorAddr::fromPtr(&LCTM))
423      .takeError();
424}
425
426} // end namespace orc
427} // end namespace llvm
428