1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
11#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
12#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
13#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
14
15#include "llvm/Object/COFF.h"
16
17#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
18
19#include "llvm/ExecutionEngine/JITLink/x86_64.h"
20
21#define DEBUG_TYPE "orc"
22
23using namespace llvm;
24using namespace llvm::orc;
25using namespace llvm::orc::shared;
26
27namespace llvm {
28namespace orc {
29namespace shared {
30
31using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
32using SPSCOFFJITDylibDepInfoMap =
33    SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
34using SPSCOFFObjectSectionsMap =
35    SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
36using SPSCOFFRegisterObjectSectionsArgs =
37    SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
38using SPSCOFFDeregisterObjectSectionsArgs =
39    SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
40
41} // namespace shared
42} // namespace orc
43} // namespace llvm
44namespace {
45
46class COFFHeaderMaterializationUnit : public MaterializationUnit {
47public:
48  COFFHeaderMaterializationUnit(COFFPlatform &CP,
49                                const SymbolStringPtr &HeaderStartSymbol)
50      : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)),
51        CP(CP) {}
52
53  StringRef getName() const override { return "COFFHeaderMU"; }
54
55  void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
56    unsigned PointerSize;
57    llvm::endianness Endianness;
58    const auto &TT = CP.getExecutionSession().getTargetTriple();
59
60    switch (TT.getArch()) {
61    case Triple::x86_64:
62      PointerSize = 8;
63      Endianness = llvm::endianness::little;
64      break;
65    default:
66      llvm_unreachable("Unrecognized architecture");
67    }
68
69    auto G = std::make_unique<jitlink::LinkGraph>(
70        "<COFFHeaderMU>", TT, PointerSize, Endianness,
71        jitlink::getGenericEdgeKindName);
72    auto &HeaderSection = G->createSection("__header", MemProt::Read);
73    auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
74
75    // Init symbol is __ImageBase symbol.
76    auto &ImageBaseSymbol = G->addDefinedSymbol(
77        HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(),
78        jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
79
80    addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol);
81
82    CP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
83  }
84
85  void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
86
87private:
88  struct HeaderSymbol {
89    const char *Name;
90    uint64_t Offset;
91  };
92
93  struct NTHeader {
94    support::ulittle32_t PEMagic;
95    object::coff_file_header FileHeader;
96    struct PEHeader {
97      object::pe32plus_header Header;
98      object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
99    } OptionalHeader;
100  };
101
102  struct HeaderBlockContent {
103    object::dos_header DOSHeader;
104    COFFHeaderMaterializationUnit::NTHeader NTHeader;
105  };
106
107  static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
108                                           jitlink::Section &HeaderSection) {
109    HeaderBlockContent Hdr = {};
110
111    // Set up magic
112    Hdr.DOSHeader.Magic[0] = 'M';
113    Hdr.DOSHeader.Magic[1] = 'Z';
114    Hdr.DOSHeader.AddressOfNewExeHeader =
115        offsetof(HeaderBlockContent, NTHeader);
116    uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
117    Hdr.NTHeader.PEMagic = PEMagic;
118    Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
119
120    switch (G.getTargetTriple().getArch()) {
121    case Triple::x86_64:
122      Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
123      break;
124    default:
125      llvm_unreachable("Unrecognized architecture");
126    }
127
128    auto HeaderContent = G.allocateContent(
129        ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
130
131    return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
132                                0);
133  }
134
135  static void addImageBaseRelocationEdge(jitlink::Block &B,
136                                         jitlink::Symbol &ImageBase) {
137    auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
138                           offsetof(NTHeader, OptionalHeader) +
139                           offsetof(object::pe32plus_header, ImageBase);
140    B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0);
141  }
142
143  static MaterializationUnit::Interface
144  createHeaderInterface(COFFPlatform &MOP,
145                        const SymbolStringPtr &HeaderStartSymbol) {
146    SymbolFlagsMap HeaderSymbolFlags;
147
148    HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
149
150    return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
151                                          HeaderStartSymbol);
152  }
153
154  COFFPlatform &CP;
155};
156
157} // end anonymous namespace
158
159namespace llvm {
160namespace orc {
161
162Expected<std::unique_ptr<COFFPlatform>> COFFPlatform::Create(
163    ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
164    JITDylib &PlatformJD, std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
165    LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
166    const char *VCRuntimePath, std::optional<SymbolAliasMap> RuntimeAliases) {
167
168  // If the target is not supported then bail out immediately.
169  if (!supportedTarget(ES.getTargetTriple()))
170    return make_error<StringError>("Unsupported COFFPlatform triple: " +
171                                       ES.getTargetTriple().str(),
172                                   inconvertibleErrorCode());
173
174  auto &EPC = ES.getExecutorProcessControl();
175
176  auto GeneratorArchive =
177      object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef());
178  if (!GeneratorArchive)
179    return GeneratorArchive.takeError();
180
181  auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
182      ObjLinkingLayer, nullptr, std::move(*GeneratorArchive));
183  if (!OrcRuntimeArchiveGenerator)
184    return OrcRuntimeArchiveGenerator.takeError();
185
186  // We need a second instance of the archive (for now) for the Platform. We
187  // can `cantFail` this call, since if it were going to fail it would have
188  // failed above.
189  auto RuntimeArchive = cantFail(
190      object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()));
191
192  // Create default aliases if the caller didn't supply any.
193  if (!RuntimeAliases)
194    RuntimeAliases = standardPlatformAliases(ES);
195
196  // Define the aliases.
197  if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
198    return std::move(Err);
199
200  auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>");
201
202  // Add JIT-dispatch function support symbols.
203  if (auto Err = HostFuncJD.define(
204          absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
205                            {EPC.getJITDispatchInfo().JITDispatchFunction,
206                             JITSymbolFlags::Exported}},
207                           {ES.intern("__orc_rt_jit_dispatch_ctx"),
208                            {EPC.getJITDispatchInfo().JITDispatchContext,
209                             JITSymbolFlags::Exported}}})))
210    return std::move(Err);
211
212  PlatformJD.addToLinkOrder(HostFuncJD);
213
214  // Create the instance.
215  Error Err = Error::success();
216  auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
217      ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
218      std::move(OrcRuntimeArchiveBuffer), std::move(RuntimeArchive),
219      std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err));
220  if (Err)
221    return std::move(Err);
222  return std::move(P);
223}
224
225Expected<std::unique_ptr<COFFPlatform>>
226COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
227                     JITDylib &PlatformJD, const char *OrcRuntimePath,
228                     LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
229                     const char *VCRuntimePath,
230                     std::optional<SymbolAliasMap> RuntimeAliases) {
231
232  auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath);
233  if (!ArchiveBuffer)
234    return createFileError(OrcRuntimePath, ArchiveBuffer.getError());
235
236  return Create(ES, ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer),
237                std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
238                std::move(RuntimeAliases));
239}
240
241Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
242  auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker");
243  if (!PerJDObj)
244    return PerJDObj.takeError();
245
246  if (!*PerJDObj)
247    return make_error<StringError>("Could not find per jd object file",
248                                   inconvertibleErrorCode());
249
250  auto Buffer = (*PerJDObj)->getAsBinary();
251  if (!Buffer)
252    return Buffer.takeError();
253
254  return (*Buffer)->getMemoryBufferRef();
255}
256
257static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
258                       ArrayRef<std::pair<const char *, const char *>> AL) {
259  for (auto &KV : AL) {
260    auto AliasName = ES.intern(KV.first);
261    assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
262    Aliases[std::move(AliasName)] = {ES.intern(KV.second),
263                                     JITSymbolFlags::Exported};
264  }
265}
266
267Error COFFPlatform::setupJITDylib(JITDylib &JD) {
268  if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>(
269          *this, COFFHeaderStartSymbol)))
270    return Err;
271
272  if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError())
273    return Err;
274
275  // Define the CXX aliases.
276  SymbolAliasMap CXXAliases;
277  addAliases(ES, CXXAliases, requiredCXXAliases());
278  if (auto Err = JD.define(symbolAliases(std::move(CXXAliases))))
279    return Err;
280
281  auto PerJDObj = getPerJDObjectFile();
282  if (!PerJDObj)
283    return PerJDObj.takeError();
284
285  auto I = getObjectFileInterface(ES, *PerJDObj);
286  if (!I)
287    return I.takeError();
288
289  if (auto Err = ObjLinkingLayer.add(
290          JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I)))
291    return Err;
292
293  if (!Bootstrapping) {
294    auto ImportedLibs = StaticVCRuntime
295                            ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
296                            : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
297    if (!ImportedLibs)
298      return ImportedLibs.takeError();
299    for (auto &Lib : *ImportedLibs)
300      if (auto Err = LoadDynLibrary(JD, Lib))
301        return Err;
302    if (StaticVCRuntime)
303      if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
304        return Err;
305  }
306
307  JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer));
308  return Error::success();
309}
310
311Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
312  std::lock_guard<std::mutex> Lock(PlatformMutex);
313  auto I = JITDylibToHeaderAddr.find(&JD);
314  if (I != JITDylibToHeaderAddr.end()) {
315    assert(HeaderAddrToJITDylib.count(I->second) &&
316           "HeaderAddrToJITDylib missing entry");
317    HeaderAddrToJITDylib.erase(I->second);
318    JITDylibToHeaderAddr.erase(I);
319  }
320  return Error::success();
321}
322
323Error COFFPlatform::notifyAdding(ResourceTracker &RT,
324                                 const MaterializationUnit &MU) {
325  auto &JD = RT.getJITDylib();
326  const auto &InitSym = MU.getInitializerSymbol();
327  if (!InitSym)
328    return Error::success();
329
330  RegisteredInitSymbols[&JD].add(InitSym,
331                                 SymbolLookupFlags::WeaklyReferencedSymbol);
332
333  LLVM_DEBUG({
334    dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
335           << MU.getName() << "\n";
336  });
337  return Error::success();
338}
339
340Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
341  llvm_unreachable("Not supported yet");
342}
343
344SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
345  SymbolAliasMap Aliases;
346  addAliases(ES, Aliases, standardRuntimeUtilityAliases());
347  return Aliases;
348}
349
350ArrayRef<std::pair<const char *, const char *>>
351COFFPlatform::requiredCXXAliases() {
352  static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
353      {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
354      {"_onexit", "__orc_rt_coff_onexit_per_jd"},
355      {"atexit", "__orc_rt_coff_atexit_per_jd"}};
356
357  return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
358}
359
360ArrayRef<std::pair<const char *, const char *>>
361COFFPlatform::standardRuntimeUtilityAliases() {
362  static const std::pair<const char *, const char *>
363      StandardRuntimeUtilityAliases[] = {
364          {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
365          {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
366          {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
367          {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
368          {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
369          {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
370
371  return ArrayRef<std::pair<const char *, const char *>>(
372      StandardRuntimeUtilityAliases);
373}
374
375bool COFFPlatform::supportedTarget(const Triple &TT) {
376  switch (TT.getArch()) {
377  case Triple::x86_64:
378    return true;
379  default:
380    return false;
381  }
382}
383
384COFFPlatform::COFFPlatform(
385    ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
386    JITDylib &PlatformJD,
387    std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
388    std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
389    std::unique_ptr<object::Archive> OrcRuntimeArchive,
390    LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
391    const char *VCRuntimePath, Error &Err)
392    : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
393      LoadDynLibrary(std::move(LoadDynLibrary)),
394      OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
395      OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
396      StaticVCRuntime(StaticVCRuntime),
397      COFFHeaderStartSymbol(ES.intern("__ImageBase")) {
398  ErrorAsOutParameter _(&Err);
399
400  Bootstrapping.store(true);
401  ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));
402
403  // Load vc runtime
404  auto VCRT =
405      COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath);
406  if (!VCRT) {
407    Err = VCRT.takeError();
408    return;
409  }
410  VCRuntimeBootstrap = std::move(*VCRT);
411
412  for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries())
413    DylibsToPreload.insert(Lib);
414
415  auto ImportedLibs =
416      StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD)
417                      : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD);
418  if (!ImportedLibs) {
419    Err = ImportedLibs.takeError();
420    return;
421  }
422
423  for (auto &Lib : *ImportedLibs)
424    DylibsToPreload.insert(Lib);
425
426  PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
427
428  // PlatformJD hasn't been set up by the platform yet (since we're creating
429  // the platform now), so set it up.
430  if (auto E2 = setupJITDylib(PlatformJD)) {
431    Err = std::move(E2);
432    return;
433  }
434
435  for (auto& Lib : DylibsToPreload)
436    if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
437      Err = std::move(E2);
438      return;
439    }
440
441  if (StaticVCRuntime)
442      if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) {
443          Err = std::move(E2);
444          return;
445      }
446
447  // Associate wrapper function tags with JIT-side function implementations.
448  if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
449      Err = std::move(E2);
450      return;
451  }
452
453  // Lookup addresses of runtime functions callable by the platform,
454  // call the platform bootstrap function to initialize the platform-state
455  // object in the executor.
456  if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
457      Err = std::move(E2);
458      return;
459  }
460
461  Bootstrapping.store(false);
462  JDBootstrapStates.clear();
463}
464
465Expected<COFFPlatform::JITDylibDepMap>
466COFFPlatform::buildJDDepMap(JITDylib &JD) {
467  return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> {
468    JITDylibDepMap JDDepMap;
469
470    SmallVector<JITDylib *, 16> Worklist({&JD});
471    while (!Worklist.empty()) {
472      auto CurJD = Worklist.back();
473      Worklist.pop_back();
474
475      auto &DM = JDDepMap[CurJD];
476      CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
477        DM.reserve(O.size());
478        for (auto &KV : O) {
479          if (KV.first == CurJD)
480            continue;
481          {
482            // Bare jitdylibs not known to the platform
483            std::lock_guard<std::mutex> Lock(PlatformMutex);
484            if (!JITDylibToHeaderAddr.count(KV.first)) {
485              LLVM_DEBUG({
486                dbgs() << "JITDylib unregistered to COFFPlatform detected in "
487                          "LinkOrder: "
488                       << CurJD->getName() << "\n";
489              });
490              continue;
491            }
492          }
493          DM.push_back(KV.first);
494          // Push unvisited entry.
495          if (!JDDepMap.count(KV.first)) {
496            Worklist.push_back(KV.first);
497            JDDepMap[KV.first] = {};
498          }
499        }
500      });
501    }
502    return std::move(JDDepMap);
503  });
504}
505
506void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
507                                        JITDylibSP JD,
508                                        JITDylibDepMap &JDDepMap) {
509  SmallVector<JITDylib *, 16> Worklist({JD.get()});
510  DenseSet<JITDylib *> Visited({JD.get()});
511  DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
512  ES.runSessionLocked([&]() {
513    while (!Worklist.empty()) {
514      auto CurJD = Worklist.back();
515      Worklist.pop_back();
516
517      auto RISItr = RegisteredInitSymbols.find(CurJD);
518      if (RISItr != RegisteredInitSymbols.end()) {
519        NewInitSymbols[CurJD] = std::move(RISItr->second);
520        RegisteredInitSymbols.erase(RISItr);
521      }
522
523      for (auto *DepJD : JDDepMap[CurJD])
524        if (!Visited.count(DepJD)) {
525          Worklist.push_back(DepJD);
526          Visited.insert(DepJD);
527        }
528    }
529  });
530
531  // If there are no further init symbols to look up then send the link order
532  // (as a list of header addresses) to the caller.
533  if (NewInitSymbols.empty()) {
534    // Build the dep info map to return.
535    COFFJITDylibDepInfoMap DIM;
536    DIM.reserve(JDDepMap.size());
537    for (auto &KV : JDDepMap) {
538      std::lock_guard<std::mutex> Lock(PlatformMutex);
539      COFFJITDylibDepInfo DepInfo;
540      DepInfo.reserve(KV.second.size());
541      for (auto &Dep : KV.second) {
542        DepInfo.push_back(JITDylibToHeaderAddr[Dep]);
543      }
544      auto H = JITDylibToHeaderAddr[KV.first];
545      DIM.push_back(std::make_pair(H, std::move(DepInfo)));
546    }
547    SendResult(DIM);
548    return;
549  }
550
551  // Otherwise issue a lookup and re-run this phase when it completes.
552  lookupInitSymbolsAsync(
553      [this, SendResult = std::move(SendResult), &JD,
554       JDDepMap = std::move(JDDepMap)](Error Err) mutable {
555        if (Err)
556          SendResult(std::move(Err));
557        else
558          pushInitializersLoop(std::move(SendResult), JD, JDDepMap);
559      },
560      ES, std::move(NewInitSymbols));
561}
562
563void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
564                                       ExecutorAddr JDHeaderAddr) {
565  JITDylibSP JD;
566  {
567    std::lock_guard<std::mutex> Lock(PlatformMutex);
568    auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
569    if (I != HeaderAddrToJITDylib.end())
570      JD = I->second;
571  }
572
573  LLVM_DEBUG({
574    dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
575    if (JD)
576      dbgs() << "pushing initializers for " << JD->getName() << "\n";
577    else
578      dbgs() << "No JITDylib for header address.\n";
579  });
580
581  if (!JD) {
582    SendResult(make_error<StringError>("No JITDylib with header addr " +
583                                           formatv("{0:x}", JDHeaderAddr),
584                                       inconvertibleErrorCode()));
585    return;
586  }
587
588  auto JDDepMap = buildJDDepMap(*JD);
589  if (!JDDepMap) {
590    SendResult(JDDepMap.takeError());
591    return;
592  }
593
594  pushInitializersLoop(std::move(SendResult), JD, *JDDepMap);
595}
596
597void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
598                                   ExecutorAddr Handle, StringRef SymbolName) {
599  LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
600
601  JITDylib *JD = nullptr;
602
603  {
604    std::lock_guard<std::mutex> Lock(PlatformMutex);
605    auto I = HeaderAddrToJITDylib.find(Handle);
606    if (I != HeaderAddrToJITDylib.end())
607      JD = I->second;
608  }
609
610  if (!JD) {
611    LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
612    SendResult(make_error<StringError>("No JITDylib associated with handle " +
613                                           formatv("{0:x}", Handle),
614                                       inconvertibleErrorCode()));
615    return;
616  }
617
618  // Use functor class to work around XL build compiler issue on AIX.
619  class RtLookupNotifyComplete {
620  public:
621    RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
622        : SendResult(std::move(SendResult)) {}
623    void operator()(Expected<SymbolMap> Result) {
624      if (Result) {
625        assert(Result->size() == 1 && "Unexpected result map count");
626        SendResult(Result->begin()->second.getAddress());
627      } else {
628        SendResult(Result.takeError());
629      }
630    }
631
632  private:
633    SendSymbolAddressFn SendResult;
634  };
635
636  ES.lookup(
637      LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
638      SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
639      RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
640}
641
642Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
643  ExecutionSession::JITDispatchHandlerAssociationMap WFs;
644
645  using LookupSymbolSPSSig =
646      SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
647  WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] =
648      ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
649                                              &COFFPlatform::rt_lookupSymbol);
650  using PushInitializersSPSSig =
651      SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
652  WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] =
653      ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
654          this, &COFFPlatform::rt_pushInitializers);
655
656  return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
657}
658
659Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
660  llvm::sort(BState.Initializers);
661  if (auto Err =
662          runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ"))
663    return Err;
664
665  if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init"))
666    return Err;
667
668  if (auto Err =
669          runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ"))
670    return Err;
671  return Error::success();
672}
673
674Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
675                                                       StringRef Start,
676                                                       StringRef End) {
677  for (auto &Initializer : BState.Initializers)
678    if (Initializer.first >= Start && Initializer.first <= End &&
679        Initializer.second) {
680      auto Res =
681          ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second);
682      if (!Res)
683        return Res.takeError();
684    }
685  return Error::success();
686}
687
688Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
689  // Lookup of runtime symbols causes the collection of initializers if
690  // it's static linking setting.
691  if (auto Err = lookupAndRecordAddrs(
692          ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
693          {
694              {ES.intern("__orc_rt_coff_platform_bootstrap"),
695               &orc_rt_coff_platform_bootstrap},
696              {ES.intern("__orc_rt_coff_platform_shutdown"),
697               &orc_rt_coff_platform_shutdown},
698              {ES.intern("__orc_rt_coff_register_jitdylib"),
699               &orc_rt_coff_register_jitdylib},
700              {ES.intern("__orc_rt_coff_deregister_jitdylib"),
701               &orc_rt_coff_deregister_jitdylib},
702              {ES.intern("__orc_rt_coff_register_object_sections"),
703               &orc_rt_coff_register_object_sections},
704              {ES.intern("__orc_rt_coff_deregister_object_sections"),
705               &orc_rt_coff_deregister_object_sections},
706          }))
707    return Err;
708
709  // Call bootstrap functions
710  if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap))
711    return Err;
712
713  // Do the pending jitdylib registration actions that we couldn't do
714  // because orc runtime was not linked fully.
715  for (auto KV : JDBootstrapStates) {
716    auto &JDBState = KV.second;
717    if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
718            orc_rt_coff_register_jitdylib, JDBState.JDName,
719            JDBState.HeaderAddr))
720      return Err;
721
722    for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
723      if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
724                                            SPSCOFFObjectSectionsMap, bool)>(
725              orc_rt_coff_register_object_sections, JDBState.HeaderAddr,
726              ObjSectionMap, false))
727        return Err;
728  }
729
730  // Run static initializers collected in bootstrap stage.
731  for (auto KV : JDBootstrapStates) {
732    auto &JDBState = KV.second;
733    if (auto Err = runBootstrapInitializers(JDBState))
734      return Err;
735  }
736
737  return Error::success();
738}
739
740Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
741                                      StringRef SymbolName) {
742  ExecutorAddr jit_function;
743  auto AfterCLookupErr = lookupAndRecordAddrs(
744      ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
745      {{ES.intern(SymbolName), &jit_function}});
746  if (!AfterCLookupErr) {
747    auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function);
748    if (!Res)
749      return Res.takeError();
750    return Error::success();
751  }
752  if (!AfterCLookupErr.isA<SymbolsNotFound>())
753    return AfterCLookupErr;
754  consumeError(std::move(AfterCLookupErr));
755  return Error::success();
756}
757
758void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
759    MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
760    jitlink::PassConfiguration &Config) {
761
762  bool IsBootstrapping = CP.Bootstrapping.load();
763
764  if (auto InitSymbol = MR.getInitializerSymbol()) {
765    if (InitSymbol == CP.COFFHeaderStartSymbol) {
766      Config.PostAllocationPasses.push_back(
767          [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
768            return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping);
769          });
770      return;
771    }
772    Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
773      return preserveInitializerSections(G, MR);
774    });
775  }
776
777  if (!IsBootstrapping)
778    Config.PostFixupPasses.push_back(
779        [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
780          return registerObjectPlatformSections(G, JD);
781        });
782  else
783    Config.PostFixupPasses.push_back(
784        [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
785          return registerObjectPlatformSectionsInBootstrap(G, JD);
786        });
787}
788
789ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
790COFFPlatform::COFFPlatformPlugin::getSyntheticSymbolDependencies(
791    MaterializationResponsibility &MR) {
792  std::lock_guard<std::mutex> Lock(PluginMutex);
793  auto I = InitSymbolDeps.find(&MR);
794  if (I != InitSymbolDeps.end()) {
795    SyntheticSymbolDependenciesMap Result;
796    Result[MR.getInitializerSymbol()] = std::move(I->second);
797    InitSymbolDeps.erase(&MR);
798    return Result;
799  }
800  return SyntheticSymbolDependenciesMap();
801}
802
803Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
804    jitlink::LinkGraph &G, MaterializationResponsibility &MR,
805    bool IsBootstraping) {
806  auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
807    return Sym->getName() == *CP.COFFHeaderStartSymbol;
808  });
809  assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
810
811  auto &JD = MR.getTargetJITDylib();
812  std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
813  auto HeaderAddr = (*I)->getAddress();
814  CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
815  CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
816  if (!IsBootstraping) {
817    G.allocActions().push_back(
818        {cantFail(WrapperFunctionCall::Create<
819                  SPSArgList<SPSString, SPSExecutorAddr>>(
820             CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)),
821         cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
822             CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
823  } else {
824    G.allocActions().push_back(
825        {{},
826         cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
827             CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
828    JDBootstrapState BState;
829    BState.JD = &JD;
830    BState.JDName = JD.getName();
831    BState.HeaderAddr = HeaderAddr;
832    CP.JDBootstrapStates.emplace(&JD, BState);
833  }
834
835  return Error::success();
836}
837
838Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
839    jitlink::LinkGraph &G, JITDylib &JD) {
840  COFFObjectSectionsMap ObjSecs;
841  auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
842  assert(HeaderAddr && "Must be registered jitdylib");
843  for (auto &S : G.sections()) {
844    jitlink::SectionRange Range(S);
845    if (Range.getSize())
846      ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
847  }
848
849  G.allocActions().push_back(
850      {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
851           CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)),
852       cantFail(
853           WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
854               CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
855               ObjSecs))});
856
857  return Error::success();
858}
859
860Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
861    jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
862  JITLinkSymbolSet InitSectionSymbols;
863  for (auto &Sec : G.sections())
864    if (isCOFFInitializerSection(Sec.getName()))
865      for (auto *B : Sec.blocks())
866        if (!B->edges_empty())
867          InitSectionSymbols.insert(
868              &G.addAnonymousSymbol(*B, 0, 0, false, true));
869
870  std::lock_guard<std::mutex> Lock(PluginMutex);
871  InitSymbolDeps[&MR] = InitSectionSymbols;
872  return Error::success();
873}
874
875Error COFFPlatform::COFFPlatformPlugin::
876    registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
877                                              JITDylib &JD) {
878  std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
879  auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
880  COFFObjectSectionsMap ObjSecs;
881  for (auto &S : G.sections()) {
882    jitlink::SectionRange Range(S);
883    if (Range.getSize())
884      ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
885  }
886
887  G.allocActions().push_back(
888      {{},
889       cantFail(
890           WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
891               CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
892               ObjSecs))});
893
894  auto &BState = CP.JDBootstrapStates[&JD];
895  BState.ObjectSectionsMaps.push_back(std::move(ObjSecs));
896
897  // Collect static initializers
898  for (auto &S : G.sections())
899    if (isCOFFInitializerSection(S.getName()))
900      for (auto *B : S.blocks()) {
901        if (B->edges_empty())
902          continue;
903        for (auto &E : B->edges())
904          BState.Initializers.push_back(std::make_pair(
905              S.getName().str(), E.getTarget().getAddress() + E.getAddend()));
906      }
907
908  return Error::success();
909}
910
911} // End namespace orc.
912} // End namespace llvm.
913