1//===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
10
11#include "llvm/BinaryFormat/MachO.h"
12#include "llvm/ExecutionEngine/JITLink/MachO.h"
13#include "llvm/ExecutionEngine/JITLink/aarch64.h"
14#include "llvm/ExecutionEngine/JITLink/x86_64.h"
15#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
16#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
17#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
18#include "llvm/ExecutionEngine/Orc/MachOBuilder.h"
19#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
20#include "llvm/Support/BinaryByteStream.h"
21#include "llvm/Support/Debug.h"
22#include <optional>
23
24#define DEBUG_TYPE "orc"
25
26using namespace llvm;
27using namespace llvm::orc;
28using namespace llvm::orc::shared;
29
30namespace llvm {
31namespace orc {
32namespace shared {
33
34using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
35using SPSMachOJITDylibDepInfoMap =
36    SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
37
38class SPSMachOExecutorSymbolFlags;
39
40template <>
41class SPSSerializationTraits<SPSMachOJITDylibDepInfo,
42                             MachOPlatform::MachOJITDylibDepInfo> {
43public:
44  static size_t size(const MachOPlatform::MachOJITDylibDepInfo &DDI) {
45    return SPSMachOJITDylibDepInfo::AsArgList::size(DDI.Sealed, DDI.DepHeaders);
46  }
47
48  static bool serialize(SPSOutputBuffer &OB,
49                        const MachOPlatform::MachOJITDylibDepInfo &DDI) {
50    return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, DDI.Sealed,
51                                                         DDI.DepHeaders);
52  }
53
54  static bool deserialize(SPSInputBuffer &IB,
55                          MachOPlatform::MachOJITDylibDepInfo &DDI) {
56    return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, DDI.Sealed,
57                                                           DDI.DepHeaders);
58  }
59};
60
61template <>
62class SPSSerializationTraits<SPSMachOExecutorSymbolFlags,
63                             MachOPlatform::MachOExecutorSymbolFlags> {
64private:
65  using UT = std::underlying_type_t<MachOPlatform::MachOExecutorSymbolFlags>;
66
67public:
68  static size_t size(const MachOPlatform::MachOExecutorSymbolFlags &SF) {
69    return sizeof(UT);
70  }
71
72  static bool serialize(SPSOutputBuffer &OB,
73                        const MachOPlatform::MachOExecutorSymbolFlags &SF) {
74    return SPSArgList<UT>::serialize(OB, static_cast<UT>(SF));
75  }
76
77  static bool deserialize(SPSInputBuffer &IB,
78                          MachOPlatform::MachOExecutorSymbolFlags &SF) {
79    UT Tmp;
80    if (!SPSArgList<UT>::deserialize(IB, Tmp))
81      return false;
82    SF = static_cast<MachOPlatform::MachOExecutorSymbolFlags>(Tmp);
83    return true;
84  }
85};
86
87} // namespace shared
88} // namespace orc
89} // namespace llvm
90
91namespace {
92
93using SPSRegisterSymbolsArgs =
94    SPSArgList<SPSExecutorAddr,
95               SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
96                                    SPSMachOExecutorSymbolFlags>>>;
97
98std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP,
99                                                        std::string Name) {
100  unsigned PointerSize;
101  llvm::endianness Endianness;
102  const auto &TT = MOP.getExecutionSession().getTargetTriple();
103
104  switch (TT.getArch()) {
105  case Triple::aarch64:
106  case Triple::x86_64:
107    PointerSize = 8;
108    Endianness = llvm::endianness::little;
109    break;
110  default:
111    llvm_unreachable("Unrecognized architecture");
112  }
113
114  return std::make_unique<jitlink::LinkGraph>(std::move(Name), TT, PointerSize,
115                                              Endianness,
116                                              jitlink::getGenericEdgeKindName);
117}
118
119// Creates a Bootstrap-Complete LinkGraph to run deferred actions.
120class MachOPlatformCompleteBootstrapMaterializationUnit
121    : public MaterializationUnit {
122public:
123  using SymbolTableVector =
124      SmallVector<std::tuple<ExecutorAddr, ExecutorAddr,
125                             MachOPlatform::MachOExecutorSymbolFlags>>;
126
127  MachOPlatformCompleteBootstrapMaterializationUnit(
128      MachOPlatform &MOP, StringRef PlatformJDName,
129      SymbolStringPtr CompleteBootstrapSymbol, SymbolTableVector SymTab,
130      shared::AllocActions DeferredAAs, ExecutorAddr MachOHeaderAddr,
131      ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown,
132      ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib,
133      ExecutorAddr RegisterObjectSymbolTable,
134      ExecutorAddr DeregisterObjectSymbolTable)
135      : MaterializationUnit(
136            {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
137        MOP(MOP), PlatformJDName(PlatformJDName),
138        CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
139        SymTab(std::move(SymTab)), DeferredAAs(std::move(DeferredAAs)),
140        MachOHeaderAddr(MachOHeaderAddr), PlatformBootstrap(PlatformBootstrap),
141        PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
142        DeregisterJITDylib(DeregisterJITDylib),
143        RegisterObjectSymbolTable(RegisterObjectSymbolTable),
144        DeregisterObjectSymbolTable(DeregisterObjectSymbolTable) {}
145
146  StringRef getName() const override {
147    return "MachOPlatformCompleteBootstrap";
148  }
149
150  void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
151    using namespace jitlink;
152    auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");
153    auto &PlaceholderSection =
154        G->createSection("__orc_rt_cplt_bs", MemProt::Read);
155    auto &PlaceholderBlock =
156        G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);
157    G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,
158                        Linkage::Strong, Scope::Hidden, false, true);
159
160    // Reserve space for the stolen actions, plus two extras.
161    G->allocActions().reserve(DeferredAAs.size() + 3);
162
163    // 1. Bootstrap the platform support code.
164    G->allocActions().push_back(
165        {cantFail(WrapperFunctionCall::Create<SPSArgList<>>(PlatformBootstrap)),
166         cantFail(
167             WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});
168
169    // 2. Register the platform JITDylib.
170    G->allocActions().push_back(
171        {cantFail(WrapperFunctionCall::Create<
172                  SPSArgList<SPSString, SPSExecutorAddr>>(
173             RegisterJITDylib, PlatformJDName, MachOHeaderAddr)),
174         cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
175             DeregisterJITDylib, MachOHeaderAddr))});
176
177    // 3. Register deferred symbols.
178    G->allocActions().push_back(
179        {cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>(
180             RegisterObjectSymbolTable, MachOHeaderAddr, SymTab)),
181         cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>(
182             DeregisterObjectSymbolTable, MachOHeaderAddr, SymTab))});
183
184    // 4. Add the deferred actions to the graph.
185    std::move(DeferredAAs.begin(), DeferredAAs.end(),
186              std::back_inserter(G->allocActions()));
187
188    MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
189  }
190
191  void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
192
193private:
194  MachOPlatform &MOP;
195  StringRef PlatformJDName;
196  SymbolStringPtr CompleteBootstrapSymbol;
197  SymbolTableVector SymTab;
198  shared::AllocActions DeferredAAs;
199  ExecutorAddr MachOHeaderAddr;
200  ExecutorAddr PlatformBootstrap;
201  ExecutorAddr PlatformShutdown;
202  ExecutorAddr RegisterJITDylib;
203  ExecutorAddr DeregisterJITDylib;
204  ExecutorAddr RegisterObjectSymbolTable;
205  ExecutorAddr DeregisterObjectSymbolTable;
206};
207
208static StringRef ObjCRuntimeObjectSectionsData[] = {
209    MachOObjCCatListSectionName,   MachOObjCClassListSectionName,
210    MachOObjCClassRefsSectionName, MachOObjCConstSectionName,
211    MachOObjCDataSectionName,      MachOObjCSelRefsSectionName};
212
213static StringRef ObjCRuntimeObjectSectionsText[] = {
214    MachOObjCClassNameSectionName, MachOObjCMethNameSectionName,
215    MachOObjCMethTypeSectionName,  MachOSwift5TypesSectionName,
216    MachOSwift5TypeRefSectionName, MachOSwift5FieldMetadataSectionName,
217    MachOSwift5EntrySectionName,   MachOSwift5ProtoSectionName,
218    MachOSwift5ProtosSectionName};
219
220static StringRef ObjCRuntimeObjectSectionName =
221    "__llvm_jitlink_ObjCRuntimeRegistrationObject";
222
223static StringRef ObjCImageInfoSymbolName =
224    "__llvm_jitlink_macho_objc_imageinfo";
225
226struct ObjCImageInfoFlags {
227  uint16_t SwiftABIVersion;
228  uint16_t SwiftVersion;
229  bool HasCategoryClassProperties;
230  bool HasSignedObjCClassROs;
231
232  static constexpr uint32_t SIGNED_CLASS_RO = (1 << 4);
233  static constexpr uint32_t HAS_CATEGORY_CLASS_PROPERTIES = (1 << 6);
234
235  explicit ObjCImageInfoFlags(uint32_t RawFlags) {
236    HasSignedObjCClassROs = RawFlags & SIGNED_CLASS_RO;
237    HasCategoryClassProperties = RawFlags & HAS_CATEGORY_CLASS_PROPERTIES;
238    SwiftABIVersion = (RawFlags >> 8) & 0xFF;
239    SwiftVersion = (RawFlags >> 16) & 0xFFFF;
240  }
241
242  uint32_t rawFlags() const {
243    uint32_t Result = 0;
244    if (HasCategoryClassProperties)
245      Result |= HAS_CATEGORY_CLASS_PROPERTIES;
246    if (HasSignedObjCClassROs)
247      Result |= SIGNED_CLASS_RO;
248    Result |= (SwiftABIVersion << 8);
249    Result |= (SwiftVersion << 16);
250    return Result;
251  }
252};
253} // end anonymous namespace
254
255namespace llvm {
256namespace orc {
257
258Expected<std::unique_ptr<MachOPlatform>> MachOPlatform::Create(
259    ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
260    JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime,
261    HeaderOptions PlatformJDOpts, MachOHeaderMUBuilder BuildMachOHeaderMU,
262    std::optional<SymbolAliasMap> RuntimeAliases) {
263
264  // If the target is not supported then bail out immediately.
265  if (!supportedTarget(ES.getTargetTriple()))
266    return make_error<StringError>("Unsupported MachOPlatform triple: " +
267                                       ES.getTargetTriple().str(),
268                                   inconvertibleErrorCode());
269
270  auto &EPC = ES.getExecutorProcessControl();
271
272  // Create default aliases if the caller didn't supply any.
273  if (!RuntimeAliases)
274    RuntimeAliases = standardPlatformAliases(ES);
275
276  // Define the aliases.
277  if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
278    return std::move(Err);
279
280  // Add JIT-dispatch function support symbols.
281  if (auto Err = PlatformJD.define(
282          absoluteSymbols({{ES.intern("___orc_rt_jit_dispatch"),
283                            {EPC.getJITDispatchInfo().JITDispatchFunction,
284                             JITSymbolFlags::Exported}},
285                           {ES.intern("___orc_rt_jit_dispatch_ctx"),
286                            {EPC.getJITDispatchInfo().JITDispatchContext,
287                             JITSymbolFlags::Exported}}})))
288    return std::move(Err);
289
290  // Create the instance.
291  Error Err = Error::success();
292  auto P = std::unique_ptr<MachOPlatform>(new MachOPlatform(
293      ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime),
294      std::move(PlatformJDOpts), std::move(BuildMachOHeaderMU), Err));
295  if (Err)
296    return std::move(Err);
297  return std::move(P);
298}
299
300Expected<std::unique_ptr<MachOPlatform>>
301MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
302                      JITDylib &PlatformJD, const char *OrcRuntimePath,
303                      HeaderOptions PlatformJDOpts,
304                      MachOHeaderMUBuilder BuildMachOHeaderMU,
305                      std::optional<SymbolAliasMap> RuntimeAliases) {
306
307  // Create a generator for the ORC runtime archive.
308  auto OrcRuntimeArchiveGenerator =
309      StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
310  if (!OrcRuntimeArchiveGenerator)
311    return OrcRuntimeArchiveGenerator.takeError();
312
313  return Create(ES, ObjLinkingLayer, PlatformJD,
314                std::move(*OrcRuntimeArchiveGenerator),
315                std::move(PlatformJDOpts), std::move(BuildMachOHeaderMU),
316                std::move(RuntimeAliases));
317}
318
319Error MachOPlatform::setupJITDylib(JITDylib &JD) {
320  return setupJITDylib(JD, /*Opts=*/{});
321}
322
323Error MachOPlatform::setupJITDylib(JITDylib &JD, HeaderOptions Opts) {
324  if (auto Err = JD.define(BuildMachOHeaderMU(*this, std::move(Opts))))
325    return Err;
326
327  return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError();
328}
329
330Error MachOPlatform::teardownJITDylib(JITDylib &JD) {
331  std::lock_guard<std::mutex> Lock(PlatformMutex);
332  auto I = JITDylibToHeaderAddr.find(&JD);
333  if (I != JITDylibToHeaderAddr.end()) {
334    assert(HeaderAddrToJITDylib.count(I->second) &&
335           "HeaderAddrToJITDylib missing entry");
336    HeaderAddrToJITDylib.erase(I->second);
337    JITDylibToHeaderAddr.erase(I);
338  }
339  JITDylibToPThreadKey.erase(&JD);
340  return Error::success();
341}
342
343Error MachOPlatform::notifyAdding(ResourceTracker &RT,
344                                  const MaterializationUnit &MU) {
345  auto &JD = RT.getJITDylib();
346  const auto &InitSym = MU.getInitializerSymbol();
347  if (!InitSym)
348    return Error::success();
349
350  RegisteredInitSymbols[&JD].add(InitSym,
351                                 SymbolLookupFlags::WeaklyReferencedSymbol);
352  LLVM_DEBUG({
353    dbgs() << "MachOPlatform: Registered init symbol " << *InitSym << " for MU "
354           << MU.getName() << "\n";
355  });
356  return Error::success();
357}
358
359Error MachOPlatform::notifyRemoving(ResourceTracker &RT) {
360  llvm_unreachable("Not supported yet");
361}
362
363static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
364                       ArrayRef<std::pair<const char *, const char *>> AL) {
365  for (auto &KV : AL) {
366    auto AliasName = ES.intern(KV.first);
367    assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
368    Aliases[std::move(AliasName)] = {ES.intern(KV.second),
369                                     JITSymbolFlags::Exported};
370  }
371}
372
373SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) {
374  SymbolAliasMap Aliases;
375  addAliases(ES, Aliases, requiredCXXAliases());
376  addAliases(ES, Aliases, standardRuntimeUtilityAliases());
377  return Aliases;
378}
379
380ArrayRef<std::pair<const char *, const char *>>
381MachOPlatform::requiredCXXAliases() {
382  static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
383      {"___cxa_atexit", "___orc_rt_macho_cxa_atexit"}};
384
385  return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
386}
387
388ArrayRef<std::pair<const char *, const char *>>
389MachOPlatform::standardRuntimeUtilityAliases() {
390  static const std::pair<const char *, const char *>
391      StandardRuntimeUtilityAliases[] = {
392          {"___orc_rt_run_program", "___orc_rt_macho_run_program"},
393          {"___orc_rt_jit_dlerror", "___orc_rt_macho_jit_dlerror"},
394          {"___orc_rt_jit_dlopen", "___orc_rt_macho_jit_dlopen"},
395          {"___orc_rt_jit_dlclose", "___orc_rt_macho_jit_dlclose"},
396          {"___orc_rt_jit_dlsym", "___orc_rt_macho_jit_dlsym"},
397          {"___orc_rt_log_error", "___orc_rt_log_error_to_stderr"}};
398
399  return ArrayRef<std::pair<const char *, const char *>>(
400      StandardRuntimeUtilityAliases);
401}
402
403bool MachOPlatform::supportedTarget(const Triple &TT) {
404  switch (TT.getArch()) {
405  case Triple::aarch64:
406  case Triple::x86_64:
407    return true;
408  default:
409    return false;
410  }
411}
412
413jitlink::Edge::Kind MachOPlatform::getPointerEdgeKind(jitlink::LinkGraph &G) {
414  switch (G.getTargetTriple().getArch()) {
415  case Triple::aarch64:
416    return jitlink::aarch64::Pointer64;
417  case Triple::x86_64:
418    return jitlink::x86_64::Pointer64;
419  default:
420    llvm_unreachable("Unsupported architecture");
421  }
422}
423
424MachOPlatform::MachOExecutorSymbolFlags
425MachOPlatform::flagsForSymbol(jitlink::Symbol &Sym) {
426  MachOPlatform::MachOExecutorSymbolFlags Flags{};
427  if (Sym.getLinkage() == jitlink::Linkage::Weak)
428    Flags |= MachOExecutorSymbolFlags::Weak;
429
430  if (Sym.isCallable())
431    Flags |= MachOExecutorSymbolFlags::Callable;
432
433  return Flags;
434}
435
436MachOPlatform::MachOPlatform(
437    ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
438    JITDylib &PlatformJD,
439    std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,
440    HeaderOptions PlatformJDOpts, MachOHeaderMUBuilder BuildMachOHeaderMU,
441    Error &Err)
442    : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer),
443      BuildMachOHeaderMU(std::move(BuildMachOHeaderMU)) {
444  ErrorAsOutParameter _(&Err);
445  ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this));
446  PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
447
448  BootstrapInfo BI;
449  Bootstrap = &BI;
450
451  // Bootstrap process -- here be phase-ordering dragons.
452  //
453  // The MachOPlatform class uses allocation actions to register metadata
454  // sections with the ORC runtime, however the runtime contains metadata
455  // registration functions that have their own metadata that they need to
456  // register (e.g. the frame-info registration functions have frame-info).
457  // We can't use an ordinary lookup to find these registration functions
458  // because their address is needed during the link of the containing graph
459  // itself (to build the allocation actions that will call the registration
460  // functions). Further complicating the situation (a) the graph containing
461  // the registration functions is allowed to depend on other graphs (e.g. the
462  // graph containing the ORC runtime RTTI support) so we need to handle an
463  // unknown set of dependencies during bootstrap, and (b) these graphs may
464  // be linked concurrently if the user has installed a concurrent dispatcher.
465  //
466  // We satisfy these constraints by implementing a bootstrap phase during which
467  // allocation actions generated by MachOPlatform are appended to a list of
468  // deferred allocation actions, rather than to the graphs themselves. At the
469  // end of the bootstrap process the deferred actions are attached to a final
470  // "complete-bootstrap" graph that causes them to be run.
471  //
472  // The bootstrap steps are as follows:
473  //
474  // 1. Request the graph containing the mach header. This graph is guaranteed
475  //    not to have any metadata so the fact that the registration functions
476  //    are not available yet is not a problem.
477  //
478  // 2. Look up the registration functions and discard the results. This will
479  //    trigger linking of the graph containing these functions, and
480  //    consequently any graphs that it depends on. We do not use the lookup
481  //    result to find the addresses of the functions requested (as described
482  //    above the lookup will return too late for that), instead we capture the
483  //    addresses in a post-allocation pass injected by the platform runtime
484  //    during bootstrap only.
485  //
486  // 3. During bootstrap the MachOPlatformPlugin keeps a count of the number of
487  //    graphs being linked (potentially concurrently), and we block until all
488  //    of these graphs have completed linking. This is to avoid a race on the
489  //    deferred-actions vector: the lookup for the runtime registration
490  //    functions may return while some functions (those that are being
491  //    incidentally linked in, but aren't reachable via the runtime functions)
492  //    are still being linked, and we need to capture any allocation actions
493  //    for this incidental code before we proceed.
494  //
495  // 4. Once all active links are complete we transfer the deferred actions to
496  //    a newly added CompleteBootstrap graph and then request a symbol from
497  //    the CompleteBootstrap graph to trigger materialization. This will cause
498  //    all deferred actions to be run, and once this lookup returns we can
499  //    proceed.
500  //
501  // 5. Finally, we associate runtime support methods in MachOPlatform with
502  //    the corresponding jit-dispatch tag variables in the ORC runtime to make
503  //    the support methods callable. The bootstrap is now complete.
504
505  // Step (1) Add header materialization unit and request.
506  if ((Err = PlatformJD.define(
507           this->BuildMachOHeaderMU(*this, std::move(PlatformJDOpts)))))
508    return;
509  if ((Err = ES.lookup(&PlatformJD, MachOHeaderStartSymbol).takeError()))
510    return;
511
512  // Step (2) Request runtime registration functions to trigger
513  // materialization..
514  if ((Err = ES.lookup(makeJITDylibSearchOrder(&PlatformJD),
515                       SymbolLookupSet(
516                           {PlatformBootstrap.Name, PlatformShutdown.Name,
517                            RegisterJITDylib.Name, DeregisterJITDylib.Name,
518                            RegisterObjectSymbolTable.Name,
519                            DeregisterObjectSymbolTable.Name,
520                            RegisterObjectPlatformSections.Name,
521                            DeregisterObjectPlatformSections.Name,
522                            CreatePThreadKey.Name}))
523                 .takeError()))
524    return;
525
526  // Step (3) Wait for any incidental linker work to complete.
527  {
528    std::unique_lock<std::mutex> Lock(BI.Mutex);
529    BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });
530    Bootstrap = nullptr;
531  }
532
533  // Step (4) Add complete-bootstrap materialization unit and request.
534  auto BootstrapCompleteSymbol = ES.intern("__orc_rt_macho_complete_bootstrap");
535  if ((Err = PlatformJD.define(
536           std::make_unique<MachOPlatformCompleteBootstrapMaterializationUnit>(
537               *this, PlatformJD.getName(), BootstrapCompleteSymbol,
538               std::move(BI.SymTab), std::move(BI.DeferredAAs),
539               BI.MachOHeaderAddr, PlatformBootstrap.Addr,
540               PlatformShutdown.Addr, RegisterJITDylib.Addr,
541               DeregisterJITDylib.Addr, RegisterObjectSymbolTable.Addr,
542               DeregisterObjectSymbolTable.Addr))))
543    return;
544  if ((Err = ES.lookup(makeJITDylibSearchOrder(
545                           &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),
546                       std::move(BootstrapCompleteSymbol))
547                 .takeError()))
548    return;
549
550  // (5) Associate runtime support functions.
551  if ((Err = associateRuntimeSupportFunctions()))
552    return;
553}
554
555Error MachOPlatform::associateRuntimeSupportFunctions() {
556  ExecutionSession::JITDispatchHandlerAssociationMap WFs;
557
558  using PushInitializersSPSSig =
559      SPSExpected<SPSMachOJITDylibDepInfoMap>(SPSExecutorAddr);
560  WFs[ES.intern("___orc_rt_macho_push_initializers_tag")] =
561      ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
562          this, &MachOPlatform::rt_pushInitializers);
563
564  using PushSymbolsSPSSig =
565      SPSError(SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>);
566  WFs[ES.intern("___orc_rt_macho_push_symbols_tag")] =
567      ES.wrapAsyncWithSPS<PushSymbolsSPSSig>(this,
568                                             &MachOPlatform::rt_pushSymbols);
569
570  return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
571}
572
573void MachOPlatform::pushInitializersLoop(
574    PushInitializersSendResultFn SendResult, JITDylibSP JD) {
575  DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
576  DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
577  SmallVector<JITDylib *, 16> Worklist({JD.get()});
578
579  ES.runSessionLocked([&]() {
580    while (!Worklist.empty()) {
581      // FIXME: Check for defunct dylibs.
582
583      auto DepJD = Worklist.back();
584      Worklist.pop_back();
585
586      // If we've already visited this JITDylib on this iteration then continue.
587      if (JDDepMap.count(DepJD))
588        continue;
589
590      // Add dep info.
591      auto &DM = JDDepMap[DepJD];
592      DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
593        for (auto &KV : O) {
594          if (KV.first == DepJD)
595            continue;
596          DM.push_back(KV.first);
597          Worklist.push_back(KV.first);
598        }
599      });
600
601      // Add any registered init symbols.
602      auto RISItr = RegisteredInitSymbols.find(DepJD);
603      if (RISItr != RegisteredInitSymbols.end()) {
604        NewInitSymbols[DepJD] = std::move(RISItr->second);
605        RegisteredInitSymbols.erase(RISItr);
606      }
607    }
608  });
609
610  // If there are no further init symbols to look up then send the link order
611  // (as a list of header addresses) to the caller.
612  if (NewInitSymbols.empty()) {
613
614    // To make the list intelligible to the runtime we need to convert all
615    // JITDylib pointers to their header addresses. Only include JITDylibs
616    // that appear in the JITDylibToHeaderAddr map (i.e. those that have been
617    // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
618    DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
619    HeaderAddrs.reserve(JDDepMap.size());
620    {
621      std::lock_guard<std::mutex> Lock(PlatformMutex);
622      for (auto &KV : JDDepMap) {
623        auto I = JITDylibToHeaderAddr.find(KV.first);
624        if (I != JITDylibToHeaderAddr.end())
625          HeaderAddrs[KV.first] = I->second;
626      }
627    }
628
629    // Build the dep info map to return.
630    MachOJITDylibDepInfoMap DIM;
631    DIM.reserve(JDDepMap.size());
632    for (auto &KV : JDDepMap) {
633      auto HI = HeaderAddrs.find(KV.first);
634      // Skip unmanaged JITDylibs.
635      if (HI == HeaderAddrs.end())
636        continue;
637      auto H = HI->second;
638      MachOJITDylibDepInfo DepInfo;
639      for (auto &Dep : KV.second) {
640        auto HJ = HeaderAddrs.find(Dep);
641        if (HJ != HeaderAddrs.end())
642          DepInfo.DepHeaders.push_back(HJ->second);
643      }
644      DIM.push_back(std::make_pair(H, std::move(DepInfo)));
645    }
646    SendResult(DIM);
647    return;
648  }
649
650  // Otherwise issue a lookup and re-run this phase when it completes.
651  lookupInitSymbolsAsync(
652      [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
653        if (Err)
654          SendResult(std::move(Err));
655        else
656          pushInitializersLoop(std::move(SendResult), JD);
657      },
658      ES, std::move(NewInitSymbols));
659}
660
661void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
662                                        ExecutorAddr JDHeaderAddr) {
663  JITDylibSP JD;
664  {
665    std::lock_guard<std::mutex> Lock(PlatformMutex);
666    auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
667    if (I != HeaderAddrToJITDylib.end())
668      JD = I->second;
669  }
670
671  LLVM_DEBUG({
672    dbgs() << "MachOPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
673    if (JD)
674      dbgs() << "pushing initializers for " << JD->getName() << "\n";
675    else
676      dbgs() << "No JITDylib for header address.\n";
677  });
678
679  if (!JD) {
680    SendResult(make_error<StringError>("No JITDylib with header addr " +
681                                           formatv("{0:x}", JDHeaderAddr),
682                                       inconvertibleErrorCode()));
683    return;
684  }
685
686  pushInitializersLoop(std::move(SendResult), JD);
687}
688
689void MachOPlatform::rt_pushSymbols(
690    PushSymbolsInSendResultFn SendResult, ExecutorAddr Handle,
691    const std::vector<std::pair<StringRef, bool>> &SymbolNames) {
692
693  JITDylib *JD = nullptr;
694
695  {
696    std::lock_guard<std::mutex> Lock(PlatformMutex);
697    auto I = HeaderAddrToJITDylib.find(Handle);
698    if (I != HeaderAddrToJITDylib.end())
699      JD = I->second;
700  }
701  LLVM_DEBUG({
702    dbgs() << "MachOPlatform::rt_pushSymbols(";
703    if (JD)
704      dbgs() << "\"" << JD->getName() << "\", [ ";
705    else
706      dbgs() << "<invalid handle " << Handle << ">, [ ";
707    for (auto &Name : SymbolNames)
708      dbgs() << "\"" << Name.first << "\" ";
709    dbgs() << "])\n";
710  });
711
712  if (!JD) {
713    SendResult(make_error<StringError>("No JITDylib associated with handle " +
714                                           formatv("{0:x}", Handle),
715                                       inconvertibleErrorCode()));
716    return;
717  }
718
719  SymbolLookupSet LS;
720  for (auto &[Name, Required] : SymbolNames)
721    LS.add(ES.intern(Name), Required
722                                ? SymbolLookupFlags::RequiredSymbol
723                                : SymbolLookupFlags::WeaklyReferencedSymbol);
724
725  ES.lookup(
726      LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
727      std::move(LS), SymbolState::Ready,
728      [SendResult = std::move(SendResult)](Expected<SymbolMap> Result) mutable {
729        SendResult(Result.takeError());
730      },
731      NoDependenciesToRegister);
732}
733
734Expected<uint64_t> MachOPlatform::createPThreadKey() {
735  if (!CreatePThreadKey.Addr)
736    return make_error<StringError>(
737        "Attempting to create pthread key in target, but runtime support has "
738        "not been loaded yet",
739        inconvertibleErrorCode());
740
741  Expected<uint64_t> Result(0);
742  if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
743          CreatePThreadKey.Addr, Result))
744    return std::move(Err);
745  return Result;
746}
747
748void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
749    MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
750    jitlink::PassConfiguration &Config) {
751
752  using namespace jitlink;
753
754  bool InBootstrapPhase =
755      &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
756
757  // If we're in the bootstrap phase then increment the active graphs.
758  if (InBootstrapPhase) {
759    Config.PrePrunePasses.push_back(
760        [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
761    Config.PostAllocationPasses.push_back([this](LinkGraph &G) {
762      return bootstrapPipelineRecordRuntimeFunctions(G);
763    });
764  }
765
766  // --- Handle Initializers ---
767  if (auto InitSymbol = MR.getInitializerSymbol()) {
768
769    // If the initializer symbol is the MachOHeader start symbol then just
770    // register it and then bail out -- the header materialization unit
771    // definitely doesn't need any other passes.
772    if (InitSymbol == MP.MachOHeaderStartSymbol && !InBootstrapPhase) {
773      Config.PostAllocationPasses.push_back([this, &MR](LinkGraph &G) {
774        return associateJITDylibHeaderSymbol(G, MR);
775      });
776      return;
777    }
778
779    // If the object contains an init symbol other than the header start symbol
780    // then add passes to preserve, process and register the init
781    // sections/symbols.
782    Config.PrePrunePasses.push_back([this, &MR](LinkGraph &G) {
783      if (auto Err = preserveImportantSections(G, MR))
784        return Err;
785      return processObjCImageInfo(G, MR);
786    });
787    Config.PostPrunePasses.push_back(
788        [this](LinkGraph &G) { return createObjCRuntimeObject(G); });
789    Config.PostAllocationPasses.push_back(
790        [this, &MR](LinkGraph &G) { return populateObjCRuntimeObject(G, MR); });
791  }
792
793  // Insert TLV lowering at the start of the PostPrunePasses, since we want
794  // it to run before GOT/PLT lowering.
795  Config.PostPrunePasses.insert(
796      Config.PostPrunePasses.begin(),
797      [this, &JD = MR.getTargetJITDylib()](LinkGraph &G) {
798        return fixTLVSectionsAndEdges(G, JD);
799      });
800
801  // Add symbol table prepare and register passes: These will add strings for
802  // all symbols to the c-strings section, and build a symbol table registration
803  // call.
804  auto JITSymTabInfo = std::make_shared<JITSymTabVector>();
805  Config.PostPrunePasses.push_back([this, JITSymTabInfo](LinkGraph &G) {
806    return prepareSymbolTableRegistration(G, *JITSymTabInfo);
807  });
808  Config.PostFixupPasses.push_back([this, &MR, JITSymTabInfo,
809                                    InBootstrapPhase](LinkGraph &G) {
810    return addSymbolTableRegistration(G, MR, *JITSymTabInfo, InBootstrapPhase);
811  });
812
813  // Add a pass to register the final addresses of any special sections in the
814  // object with the runtime.
815  Config.PostAllocationPasses.push_back(
816      [this, &JD = MR.getTargetJITDylib(), InBootstrapPhase](LinkGraph &G) {
817        return registerObjectPlatformSections(G, JD, InBootstrapPhase);
818      });
819
820  // If we're in the bootstrap phase then steal allocation actions and then
821  // decrement the active graphs.
822  if (InBootstrapPhase)
823    Config.PostFixupPasses.push_back(
824        [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
825}
826
827ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
828MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies(
829    MaterializationResponsibility &MR) {
830  std::lock_guard<std::mutex> Lock(PluginMutex);
831  auto I = InitSymbolDeps.find(&MR);
832  if (I != InitSymbolDeps.end()) {
833    SyntheticSymbolDependenciesMap Result;
834    Result[MR.getInitializerSymbol()] = std::move(I->second);
835    InitSymbolDeps.erase(&MR);
836    return Result;
837  }
838  return SyntheticSymbolDependenciesMap();
839}
840
841Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineStart(
842    jitlink::LinkGraph &G) {
843  // Increment the active graphs count in BootstrapInfo.
844  std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
845  ++MP.Bootstrap.load()->ActiveGraphs;
846  return Error::success();
847}
848
849Error MachOPlatform::MachOPlatformPlugin::
850    bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
851  // Record bootstrap function names.
852  std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
853      {*MP.MachOHeaderStartSymbol, &MP.Bootstrap.load()->MachOHeaderAddr},
854      {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
855      {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
856      {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
857      {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
858      {*MP.RegisterObjectSymbolTable.Name, &MP.RegisterObjectSymbolTable.Addr},
859      {*MP.DeregisterObjectSymbolTable.Name,
860       &MP.DeregisterObjectSymbolTable.Addr},
861      {*MP.RegisterObjectPlatformSections.Name,
862       &MP.RegisterObjectPlatformSections.Addr},
863      {*MP.DeregisterObjectPlatformSections.Name,
864       &MP.DeregisterObjectPlatformSections.Addr},
865      {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr},
866      {*MP.RegisterObjCRuntimeObject.Name, &MP.RegisterObjCRuntimeObject.Addr},
867      {*MP.DeregisterObjCRuntimeObject.Name,
868       &MP.DeregisterObjCRuntimeObject.Addr}};
869
870  bool RegisterMachOHeader = false;
871
872  for (auto *Sym : G.defined_symbols()) {
873    for (auto &RTSym : RuntimeSymbols) {
874      if (Sym->hasName() && Sym->getName() == RTSym.first) {
875        if (*RTSym.second)
876          return make_error<StringError>(
877              "Duplicate " + RTSym.first +
878                  " detected during MachOPlatform bootstrap",
879              inconvertibleErrorCode());
880
881        if (Sym->getName() == *MP.MachOHeaderStartSymbol)
882          RegisterMachOHeader = true;
883
884        *RTSym.second = Sym->getAddress();
885      }
886    }
887  }
888
889  if (RegisterMachOHeader) {
890    // If this graph defines the macho header symbol then create the internal
891    // mapping between it and PlatformJD.
892    std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
893    MP.JITDylibToHeaderAddr[&MP.PlatformJD] =
894        MP.Bootstrap.load()->MachOHeaderAddr;
895    MP.HeaderAddrToJITDylib[MP.Bootstrap.load()->MachOHeaderAddr] =
896        &MP.PlatformJD;
897  }
898
899  return Error::success();
900}
901
902Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineEnd(
903    jitlink::LinkGraph &G) {
904  std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
905  assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
906  --MP.Bootstrap.load()->ActiveGraphs;
907  // Notify Bootstrap->CV while holding the mutex because the mutex is
908  // also keeping Bootstrap->CV alive.
909  if (MP.Bootstrap.load()->ActiveGraphs == 0)
910    MP.Bootstrap.load()->CV.notify_all();
911  return Error::success();
912}
913
914Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol(
915    jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
916  auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
917    return Sym->getName() == *MP.MachOHeaderStartSymbol;
918  });
919  assert(I != G.defined_symbols().end() && "Missing MachO header start symbol");
920
921  auto &JD = MR.getTargetJITDylib();
922  std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
923  auto HeaderAddr = (*I)->getAddress();
924  MP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
925  MP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
926  // We can unconditionally add these actions to the Graph because this pass
927  // isn't used during bootstrap.
928  G.allocActions().push_back(
929      {cantFail(
930           WrapperFunctionCall::Create<SPSArgList<SPSString, SPSExecutorAddr>>(
931               MP.RegisterJITDylib.Addr, JD.getName(), HeaderAddr)),
932       cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
933           MP.DeregisterJITDylib.Addr, HeaderAddr))});
934  return Error::success();
935}
936
937Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections(
938    jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
939  // __objc_imageinfo is "important": we want to preserve it and record its
940  // address in the first graph that it appears in, then verify and discard it
941  // in all subsequent graphs. In this pass we preserve unconditionally -- we'll
942  // manually throw it away in the processObjCImageInfo pass.
943  if (auto *ObjCImageInfoSec =
944          G.findSectionByName(MachOObjCImageInfoSectionName)) {
945    if (ObjCImageInfoSec->blocks_size() != 1)
946      return make_error<StringError>(
947          "In " + G.getName() +
948              "__DATA,__objc_imageinfo contains multiple blocks",
949          inconvertibleErrorCode());
950    G.addAnonymousSymbol(**ObjCImageInfoSec->blocks().begin(), 0, 0, false,
951                         true);
952
953    for (auto *B : ObjCImageInfoSec->blocks())
954      if (!B->edges_empty())
955        return make_error<StringError>("In " + G.getName() + ", " +
956                                           MachOObjCImageInfoSectionName +
957                                           " contains references to symbols",
958                                       inconvertibleErrorCode());
959  }
960
961  // Init sections are important: We need to preserve them and so that their
962  // addresses can be captured and reported to the ORC runtime in
963  // registerObjectPlatformSections.
964  JITLinkSymbolSet InitSectionSymbols;
965  for (auto &InitSectionName : MachOInitSectionNames) {
966    // Skip ObjCImageInfo -- this shouldn't have any dependencies, and we may
967    // remove it later.
968    if (InitSectionName == MachOObjCImageInfoSectionName)
969      continue;
970
971    // Skip non-init sections.
972    auto *InitSection = G.findSectionByName(InitSectionName);
973    if (!InitSection)
974      continue;
975
976    // Make a pass over live symbols in the section: those blocks are already
977    // preserved.
978    DenseSet<jitlink::Block *> AlreadyLiveBlocks;
979    for (auto &Sym : InitSection->symbols()) {
980      auto &B = Sym->getBlock();
981      if (Sym->isLive() && Sym->getOffset() == 0 &&
982          Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
983        InitSectionSymbols.insert(Sym);
984        AlreadyLiveBlocks.insert(&B);
985      }
986    }
987
988    // Add anonymous symbols to preserve any not-already-preserved blocks.
989    for (auto *B : InitSection->blocks())
990      if (!AlreadyLiveBlocks.count(B))
991        InitSectionSymbols.insert(
992            &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
993  }
994
995  if (!InitSectionSymbols.empty()) {
996    std::lock_guard<std::mutex> Lock(PluginMutex);
997    InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
998  }
999
1000  return Error::success();
1001}
1002
1003Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo(
1004    jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
1005
1006  // If there's an ObjC imagine info then either
1007  //   (1) It's the first __objc_imageinfo we've seen in this JITDylib. In
1008  //       this case we name and record it.
1009  // OR
1010  //   (2) We already have a recorded __objc_imageinfo for this JITDylib,
1011  //       in which case we just verify it.
1012  auto *ObjCImageInfo = G.findSectionByName(MachOObjCImageInfoSectionName);
1013  if (!ObjCImageInfo)
1014    return Error::success();
1015
1016  auto ObjCImageInfoBlocks = ObjCImageInfo->blocks();
1017
1018  // Check that the section is not empty if present.
1019  if (ObjCImageInfoBlocks.empty())
1020    return make_error<StringError>("Empty " + MachOObjCImageInfoSectionName +
1021                                       " section in " + G.getName(),
1022                                   inconvertibleErrorCode());
1023
1024  // Check that there's only one block in the section.
1025  if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end())
1026    return make_error<StringError>("Multiple blocks in " +
1027                                       MachOObjCImageInfoSectionName +
1028                                       " section in " + G.getName(),
1029                                   inconvertibleErrorCode());
1030
1031  // Check that the __objc_imageinfo section is unreferenced.
1032  // FIXME: We could optimize this check if Symbols had a ref-count.
1033  for (auto &Sec : G.sections()) {
1034    if (&Sec != ObjCImageInfo)
1035      for (auto *B : Sec.blocks())
1036        for (auto &E : B->edges())
1037          if (E.getTarget().isDefined() &&
1038              &E.getTarget().getBlock().getSection() == ObjCImageInfo)
1039            return make_error<StringError>(MachOObjCImageInfoSectionName +
1040                                               " is referenced within file " +
1041                                               G.getName(),
1042                                           inconvertibleErrorCode());
1043  }
1044
1045  auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin();
1046  auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data();
1047  auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness());
1048  auto Flags =
1049      support::endian::read32(ObjCImageInfoData + 4, G.getEndianness());
1050
1051  // Lock the mutex while we verify / update the ObjCImageInfos map.
1052  std::lock_guard<std::mutex> Lock(PluginMutex);
1053
1054  auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib());
1055  if (ObjCImageInfoItr != ObjCImageInfos.end()) {
1056    // We've already registered an __objc_imageinfo section. Verify the
1057    // content of this new section matches, then delete it.
1058    if (ObjCImageInfoItr->second.Version != Version)
1059      return make_error<StringError>(
1060          "ObjC version in " + G.getName() +
1061              " does not match first registered version",
1062          inconvertibleErrorCode());
1063    if (ObjCImageInfoItr->second.Flags != Flags)
1064      if (Error E = mergeImageInfoFlags(G, MR, ObjCImageInfoItr->second, Flags))
1065        return E;
1066
1067    // __objc_imageinfo is valid. Delete the block.
1068    for (auto *S : ObjCImageInfo->symbols())
1069      G.removeDefinedSymbol(*S);
1070    G.removeBlock(ObjCImageInfoBlock);
1071  } else {
1072    LLVM_DEBUG({
1073      dbgs() << "MachOPlatform: Registered __objc_imageinfo for "
1074             << MR.getTargetJITDylib().getName() << " in " << G.getName()
1075             << "; flags = " << formatv("{0:x4}", Flags) << "\n";
1076    });
1077    // We haven't registered an __objc_imageinfo section yet. Register and
1078    // move on. The section should already be marked no-dead-strip.
1079    G.addDefinedSymbol(ObjCImageInfoBlock, 0, ObjCImageInfoSymbolName,
1080                       ObjCImageInfoBlock.getSize(), jitlink::Linkage::Strong,
1081                       jitlink::Scope::Hidden, false, true);
1082    if (auto Err = MR.defineMaterializing(
1083            {{MR.getExecutionSession().intern(ObjCImageInfoSymbolName),
1084              JITSymbolFlags()}}))
1085      return Err;
1086    ObjCImageInfos[&MR.getTargetJITDylib()] = {Version, Flags, false};
1087  }
1088
1089  return Error::success();
1090}
1091
1092Error MachOPlatform::MachOPlatformPlugin::mergeImageInfoFlags(
1093    jitlink::LinkGraph &G, MaterializationResponsibility &MR,
1094    ObjCImageInfo &Info, uint32_t NewFlags) {
1095  if (Info.Flags == NewFlags)
1096    return Error::success();
1097
1098  ObjCImageInfoFlags Old(Info.Flags);
1099  ObjCImageInfoFlags New(NewFlags);
1100
1101  // Check for incompatible flags.
1102  if (Old.SwiftABIVersion && New.SwiftABIVersion &&
1103      Old.SwiftABIVersion != New.SwiftABIVersion)
1104    return make_error<StringError>("Swift ABI version in " + G.getName() +
1105                                       " does not match first registered flags",
1106                                   inconvertibleErrorCode());
1107
1108  if (Old.HasCategoryClassProperties != New.HasCategoryClassProperties)
1109    return make_error<StringError>("ObjC category class property support in " +
1110                                       G.getName() +
1111                                       " does not match first registered flags",
1112                                   inconvertibleErrorCode());
1113  if (Old.HasSignedObjCClassROs != New.HasSignedObjCClassROs)
1114    return make_error<StringError>("ObjC class_ro_t pointer signing in " +
1115                                       G.getName() +
1116                                       " does not match first registered flags",
1117                                   inconvertibleErrorCode());
1118
1119  // If we cannot change the flags, ignore any remaining differences. Adding
1120  // Swift or changing its version are unlikely to cause problems in practice.
1121  if (Info.Finalized)
1122    return Error::success();
1123
1124  // Use the minimum Swift version.
1125  if (Old.SwiftVersion && New.SwiftVersion)
1126    New.SwiftVersion = std::min(Old.SwiftVersion, New.SwiftVersion);
1127  else if (Old.SwiftVersion)
1128    New.SwiftVersion = Old.SwiftVersion;
1129  // Add a Swift ABI version if it was pure objc before.
1130  if (!New.SwiftABIVersion)
1131    New.SwiftABIVersion = Old.SwiftABIVersion;
1132
1133  LLVM_DEBUG({
1134    dbgs() << "MachOPlatform: Merging __objc_imageinfo flags for "
1135           << MR.getTargetJITDylib().getName() << " (was "
1136           << formatv("{0:x4}", Old.rawFlags()) << ")"
1137           << " with " << G.getName() << " (" << formatv("{0:x4}", NewFlags)
1138           << ")"
1139           << " -> " << formatv("{0:x4}", New.rawFlags()) << "\n";
1140  });
1141
1142  Info.Flags = New.rawFlags();
1143  return Error::success();
1144}
1145
1146Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
1147    jitlink::LinkGraph &G, JITDylib &JD) {
1148
1149  // Rename external references to __tlv_bootstrap to ___orc_rt_tlv_get_addr.
1150  for (auto *Sym : G.external_symbols())
1151    if (Sym->getName() == "__tlv_bootstrap") {
1152      Sym->setName("___orc_rt_macho_tlv_get_addr");
1153      break;
1154    }
1155
1156  // Store key in __thread_vars struct fields.
1157  if (auto *ThreadDataSec = G.findSectionByName(MachOThreadVarsSectionName)) {
1158    std::optional<uint64_t> Key;
1159    {
1160      std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1161      auto I = MP.JITDylibToPThreadKey.find(&JD);
1162      if (I != MP.JITDylibToPThreadKey.end())
1163        Key = I->second;
1164    }
1165
1166    if (!Key) {
1167      if (auto KeyOrErr = MP.createPThreadKey())
1168        Key = *KeyOrErr;
1169      else
1170        return KeyOrErr.takeError();
1171    }
1172
1173    uint64_t PlatformKeyBits =
1174        support::endian::byte_swap(*Key, G.getEndianness());
1175
1176    for (auto *B : ThreadDataSec->blocks()) {
1177      if (B->getSize() != 3 * G.getPointerSize())
1178        return make_error<StringError>("__thread_vars block at " +
1179                                           formatv("{0:x}", B->getAddress()) +
1180                                           " has unexpected size",
1181                                       inconvertibleErrorCode());
1182
1183      auto NewBlockContent = G.allocateBuffer(B->getSize());
1184      llvm::copy(B->getContent(), NewBlockContent.data());
1185      memcpy(NewBlockContent.data() + G.getPointerSize(), &PlatformKeyBits,
1186             G.getPointerSize());
1187      B->setContent(NewBlockContent);
1188    }
1189  }
1190
1191  // Transform any TLV edges into GOT edges.
1192  for (auto *B : G.blocks())
1193    for (auto &E : B->edges())
1194      if (E.getKind() ==
1195          jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable)
1196        E.setKind(jitlink::x86_64::
1197                      RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable);
1198
1199  return Error::success();
1200}
1201
1202std::optional<MachOPlatform::MachOPlatformPlugin::UnwindSections>
1203MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo(
1204    jitlink::LinkGraph &G) {
1205  using namespace jitlink;
1206
1207  UnwindSections US;
1208
1209  // ScanSection records a section range and adds any executable blocks that
1210  // that section points to to the CodeBlocks vector.
1211  SmallVector<Block *> CodeBlocks;
1212  auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange) {
1213    if (Sec.blocks().empty())
1214      return;
1215    SecRange = (*Sec.blocks().begin())->getRange();
1216    for (auto *B : Sec.blocks()) {
1217      auto R = B->getRange();
1218      SecRange.Start = std::min(SecRange.Start, R.Start);
1219      SecRange.End = std::max(SecRange.End, R.End);
1220      for (auto &E : B->edges()) {
1221        if (!E.getTarget().isDefined())
1222          continue;
1223        auto &TargetBlock = E.getTarget().getBlock();
1224        auto &TargetSection = TargetBlock.getSection();
1225        if ((TargetSection.getMemProt() & MemProt::Exec) == MemProt::Exec)
1226          CodeBlocks.push_back(&TargetBlock);
1227      }
1228    }
1229  };
1230
1231  if (Section *EHFrameSec = G.findSectionByName(MachOEHFrameSectionName))
1232    ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection);
1233
1234  if (Section *CUInfoSec =
1235          G.findSectionByName(MachOCompactUnwindInfoSectionName))
1236    ScanUnwindInfoSection(*CUInfoSec, US.CompactUnwindSection);
1237
1238  // If we didn't find any pointed-to code-blocks then there's no need to
1239  // register any info.
1240  if (CodeBlocks.empty())
1241    return std::nullopt;
1242
1243  // We have info to register. Sort the code blocks into address order and
1244  // build a list of contiguous address ranges covering them all.
1245  llvm::sort(CodeBlocks, [](const Block *LHS, const Block *RHS) {
1246    return LHS->getAddress() < RHS->getAddress();
1247  });
1248  for (auto *B : CodeBlocks) {
1249    if (US.CodeRanges.empty() || US.CodeRanges.back().End != B->getAddress())
1250      US.CodeRanges.push_back(B->getRange());
1251    else
1252      US.CodeRanges.back().End = B->getRange().End;
1253  }
1254
1255  LLVM_DEBUG({
1256    dbgs() << "MachOPlatform identified unwind info in " << G.getName() << ":\n"
1257           << "  DWARF: ";
1258    if (US.DwarfSection.Start)
1259      dbgs() << US.DwarfSection << "\n";
1260    else
1261      dbgs() << "none\n";
1262    dbgs() << "  Compact-unwind: ";
1263    if (US.CompactUnwindSection.Start)
1264      dbgs() << US.CompactUnwindSection << "\n";
1265    else
1266      dbgs() << "none\n"
1267             << "for code ranges:\n";
1268    for (auto &CR : US.CodeRanges)
1269      dbgs() << "  " << CR << "\n";
1270    if (US.CodeRanges.size() >= G.sections_size())
1271      dbgs() << "WARNING: High number of discontiguous code ranges! "
1272                "Padding may be interfering with coalescing.\n";
1273  });
1274
1275  return US;
1276}
1277
1278Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
1279    jitlink::LinkGraph &G, JITDylib &JD, bool InBootstrapPhase) {
1280
1281  // Get a pointer to the thread data section if there is one. It will be used
1282  // below.
1283  jitlink::Section *ThreadDataSection =
1284      G.findSectionByName(MachOThreadDataSectionName);
1285
1286  // Handle thread BSS section if there is one.
1287  if (auto *ThreadBSSSection = G.findSectionByName(MachOThreadBSSSectionName)) {
1288    // If there's already a thread data section in this graph then merge the
1289    // thread BSS section content into it, otherwise just treat the thread
1290    // BSS section as the thread data section.
1291    if (ThreadDataSection)
1292      G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
1293    else
1294      ThreadDataSection = ThreadBSSSection;
1295  }
1296
1297  SmallVector<std::pair<StringRef, ExecutorAddrRange>, 8> MachOPlatformSecs;
1298
1299  // Collect data sections to register.
1300  StringRef DataSections[] = {MachODataDataSectionName,
1301                              MachODataCommonSectionName,
1302                              MachOEHFrameSectionName};
1303  for (auto &SecName : DataSections) {
1304    if (auto *Sec = G.findSectionByName(SecName)) {
1305      jitlink::SectionRange R(*Sec);
1306      if (!R.empty())
1307        MachOPlatformSecs.push_back({SecName, R.getRange()});
1308    }
1309  }
1310
1311  // Having merged thread BSS (if present) and thread data (if present),
1312  // record the resulting section range.
1313  if (ThreadDataSection) {
1314    jitlink::SectionRange R(*ThreadDataSection);
1315    if (!R.empty())
1316      MachOPlatformSecs.push_back({MachOThreadDataSectionName, R.getRange()});
1317  }
1318
1319  // If any platform sections were found then add an allocation action to call
1320  // the registration function.
1321  StringRef PlatformSections[] = {MachOModInitFuncSectionName,
1322                                  ObjCRuntimeObjectSectionName};
1323
1324  for (auto &SecName : PlatformSections) {
1325    auto *Sec = G.findSectionByName(SecName);
1326    if (!Sec)
1327      continue;
1328    jitlink::SectionRange R(*Sec);
1329    if (R.empty())
1330      continue;
1331
1332    MachOPlatformSecs.push_back({SecName, R.getRange()});
1333  }
1334
1335  std::optional<std::tuple<SmallVector<ExecutorAddrRange>, ExecutorAddrRange,
1336                           ExecutorAddrRange>>
1337      UnwindInfo;
1338  if (auto UI = findUnwindSectionInfo(G))
1339    UnwindInfo = std::make_tuple(std::move(UI->CodeRanges), UI->DwarfSection,
1340                                 UI->CompactUnwindSection);
1341
1342  if (!MachOPlatformSecs.empty() || UnwindInfo) {
1343    // Dump the scraped inits.
1344    LLVM_DEBUG({
1345      dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n";
1346      for (auto &KV : MachOPlatformSecs)
1347        dbgs() << "  " << KV.first << ": " << KV.second << "\n";
1348    });
1349
1350    using SPSRegisterObjectPlatformSectionsArgs = SPSArgList<
1351        SPSExecutorAddr,
1352        SPSOptional<SPSTuple<SPSSequence<SPSExecutorAddrRange>,
1353                             SPSExecutorAddrRange, SPSExecutorAddrRange>>,
1354        SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>>;
1355
1356    shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase)
1357                                             ? G.allocActions()
1358                                             : MP.Bootstrap.load()->DeferredAAs;
1359
1360    ExecutorAddr HeaderAddr;
1361    {
1362      std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1363      auto I = MP.JITDylibToHeaderAddr.find(&JD);
1364      assert(I != MP.JITDylibToHeaderAddr.end() &&
1365             "No header registered for JD");
1366      assert(I->second && "Null header registered for JD");
1367      HeaderAddr = I->second;
1368    }
1369    allocActions.push_back(
1370        {cantFail(
1371             WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
1372                 MP.RegisterObjectPlatformSections.Addr, HeaderAddr, UnwindInfo,
1373                 MachOPlatformSecs)),
1374         cantFail(
1375             WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
1376                 MP.DeregisterObjectPlatformSections.Addr, HeaderAddr,
1377                 UnwindInfo, MachOPlatformSecs))});
1378  }
1379
1380  return Error::success();
1381}
1382
1383Error MachOPlatform::MachOPlatformPlugin::createObjCRuntimeObject(
1384    jitlink::LinkGraph &G) {
1385
1386  bool NeedTextSegment = false;
1387  size_t NumRuntimeSections = 0;
1388
1389  for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData)
1390    if (G.findSectionByName(ObjCRuntimeSectionName))
1391      ++NumRuntimeSections;
1392
1393  for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) {
1394    if (G.findSectionByName(ObjCRuntimeSectionName)) {
1395      ++NumRuntimeSections;
1396      NeedTextSegment = true;
1397    }
1398  }
1399
1400  // Early out for no runtime sections.
1401  if (NumRuntimeSections == 0)
1402    return Error::success();
1403
1404  // If there were any runtime sections then we need to add an __objc_imageinfo
1405  // section.
1406  ++NumRuntimeSections;
1407
1408  size_t MachOSize = sizeof(MachO::mach_header_64) +
1409                     (NeedTextSegment + 1) * sizeof(MachO::segment_command_64) +
1410                     NumRuntimeSections * sizeof(MachO::section_64);
1411
1412  auto &Sec = G.createSection(ObjCRuntimeObjectSectionName,
1413                              MemProt::Read | MemProt::Write);
1414  G.createMutableContentBlock(Sec, MachOSize, ExecutorAddr(), 16, 0, true);
1415
1416  return Error::success();
1417}
1418
1419Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject(
1420    jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
1421
1422  auto *ObjCRuntimeObjectSec =
1423      G.findSectionByName(ObjCRuntimeObjectSectionName);
1424
1425  if (!ObjCRuntimeObjectSec)
1426    return Error::success();
1427
1428  switch (G.getTargetTriple().getArch()) {
1429  case Triple::aarch64:
1430  case Triple::x86_64:
1431    // Supported.
1432    break;
1433  default:
1434    return make_error<StringError>("Unrecognized MachO arch in triple " +
1435                                       G.getTargetTriple().str(),
1436                                   inconvertibleErrorCode());
1437  }
1438
1439  auto &SecBlock = **ObjCRuntimeObjectSec->blocks().begin();
1440
1441  struct SecDesc {
1442    MachO::section_64 Sec;
1443    unique_function<void(size_t RecordOffset)> AddFixups;
1444  };
1445
1446  std::vector<SecDesc> TextSections, DataSections;
1447  auto AddSection = [&](SecDesc &SD, jitlink::Section &GraphSec) {
1448    jitlink::SectionRange SR(GraphSec);
1449    StringRef FQName = GraphSec.getName();
1450    memset(&SD.Sec, 0, sizeof(MachO::section_64));
1451    memcpy(SD.Sec.sectname, FQName.drop_front(7).data(), FQName.size() - 7);
1452    memcpy(SD.Sec.segname, FQName.data(), 6);
1453    SD.Sec.addr = SR.getStart() - SecBlock.getAddress();
1454    SD.Sec.size = SR.getSize();
1455    SD.Sec.flags = MachO::S_REGULAR;
1456  };
1457
1458  // Add the __objc_imageinfo section.
1459  {
1460    DataSections.push_back({});
1461    auto &SD = DataSections.back();
1462    memset(&SD.Sec, 0, sizeof(SD.Sec));
1463    memcpy(SD.Sec.sectname, "__objc_imageinfo", 16);
1464    strcpy(SD.Sec.segname, "__DATA");
1465    SD.Sec.size = 8;
1466    SD.AddFixups = [&](size_t RecordOffset) {
1467      auto PointerEdge = getPointerEdgeKind(G);
1468
1469      // Look for an existing __objc_imageinfo symbol.
1470      jitlink::Symbol *ObjCImageInfoSym = nullptr;
1471      for (auto *Sym : G.external_symbols())
1472        if (Sym->getName() == ObjCImageInfoSymbolName) {
1473          ObjCImageInfoSym = Sym;
1474          break;
1475        }
1476      if (!ObjCImageInfoSym)
1477        for (auto *Sym : G.absolute_symbols())
1478          if (Sym->getName() == ObjCImageInfoSymbolName) {
1479            ObjCImageInfoSym = Sym;
1480            break;
1481          }
1482      if (!ObjCImageInfoSym)
1483        for (auto *Sym : G.defined_symbols())
1484          if (Sym->hasName() && Sym->getName() == ObjCImageInfoSymbolName) {
1485            ObjCImageInfoSym = Sym;
1486            std::optional<uint32_t> Flags;
1487            {
1488              std::lock_guard<std::mutex> Lock(PluginMutex);
1489              auto It = ObjCImageInfos.find(&MR.getTargetJITDylib());
1490              if (It != ObjCImageInfos.end()) {
1491                It->second.Finalized = true;
1492                Flags = It->second.Flags;
1493              }
1494            }
1495
1496            if (Flags) {
1497              // We own the definition of __objc_image_info; write the final
1498              // merged flags value.
1499              auto Content = Sym->getBlock().getMutableContent(G);
1500              assert(Content.size() == 8 &&
1501                  "__objc_image_info size should have been verified already");
1502              support::endian::write32(&Content[4], *Flags, G.getEndianness());
1503            }
1504            break;
1505          }
1506      if (!ObjCImageInfoSym)
1507        ObjCImageInfoSym =
1508            &G.addExternalSymbol(ObjCImageInfoSymbolName, 8, false);
1509
1510      SecBlock.addEdge(PointerEdge,
1511                       RecordOffset + ((char *)&SD.Sec.addr - (char *)&SD.Sec),
1512                       *ObjCImageInfoSym, -SecBlock.getAddress().getValue());
1513    };
1514  }
1515
1516  for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData) {
1517    if (auto *GraphSec = G.findSectionByName(ObjCRuntimeSectionName)) {
1518      DataSections.push_back({});
1519      AddSection(DataSections.back(), *GraphSec);
1520    }
1521  }
1522
1523  for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) {
1524    if (auto *GraphSec = G.findSectionByName(ObjCRuntimeSectionName)) {
1525      TextSections.push_back({});
1526      AddSection(TextSections.back(), *GraphSec);
1527    }
1528  }
1529
1530  assert(ObjCRuntimeObjectSec->blocks_size() == 1 &&
1531         "Unexpected number of blocks in runtime sections object");
1532
1533  // Build the header struct up-front. This also gives us a chance to check
1534  // that the triple is supported, which we'll assume below.
1535  MachO::mach_header_64 Hdr;
1536  Hdr.magic = MachO::MH_MAGIC_64;
1537  switch (G.getTargetTriple().getArch()) {
1538  case Triple::aarch64:
1539    Hdr.cputype = MachO::CPU_TYPE_ARM64;
1540    Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL;
1541    break;
1542  case Triple::x86_64:
1543    Hdr.cputype = MachO::CPU_TYPE_X86_64;
1544    Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL;
1545    break;
1546  default:
1547    llvm_unreachable("Unsupported architecture");
1548  }
1549
1550  Hdr.filetype = MachO::MH_DYLIB;
1551  Hdr.ncmds = 1 + !TextSections.empty();
1552  Hdr.sizeofcmds =
1553      Hdr.ncmds * sizeof(MachO::segment_command_64) +
1554      (TextSections.size() + DataSections.size()) * sizeof(MachO::section_64);
1555  Hdr.flags = 0;
1556  Hdr.reserved = 0;
1557
1558  auto SecContent = SecBlock.getAlreadyMutableContent();
1559  char *P = SecContent.data();
1560  auto WriteMachOStruct = [&](auto S) {
1561    if (G.getEndianness() != llvm::endianness::native)
1562      MachO::swapStruct(S);
1563    memcpy(P, &S, sizeof(S));
1564    P += sizeof(S);
1565  };
1566
1567  auto WriteSegment = [&](StringRef Name, std::vector<SecDesc> &Secs) {
1568    MachO::segment_command_64 SegLC;
1569    memset(&SegLC, 0, sizeof(SegLC));
1570    memcpy(SegLC.segname, Name.data(), Name.size());
1571    SegLC.cmd = MachO::LC_SEGMENT_64;
1572    SegLC.cmdsize = sizeof(MachO::segment_command_64) +
1573                    Secs.size() * sizeof(MachO::section_64);
1574    SegLC.nsects = Secs.size();
1575    WriteMachOStruct(SegLC);
1576    for (auto &SD : Secs) {
1577      if (SD.AddFixups)
1578        SD.AddFixups(P - SecContent.data());
1579      WriteMachOStruct(SD.Sec);
1580    }
1581  };
1582
1583  WriteMachOStruct(Hdr);
1584  if (!TextSections.empty())
1585    WriteSegment("__TEXT", TextSections);
1586  if (!DataSections.empty())
1587    WriteSegment("__DATA", DataSections);
1588
1589  assert(P == SecContent.end() && "Underflow writing ObjC runtime object");
1590  return Error::success();
1591}
1592
1593Error MachOPlatform::MachOPlatformPlugin::prepareSymbolTableRegistration(
1594    jitlink::LinkGraph &G, JITSymTabVector &JITSymTabInfo) {
1595
1596  auto *CStringSec = G.findSectionByName(MachOCStringSectionName);
1597  if (!CStringSec)
1598    CStringSec = &G.createSection(MachOCStringSectionName,
1599                                  MemProt::Read | MemProt::Exec);
1600
1601  // Make a map of existing strings so that we can re-use them:
1602  DenseMap<StringRef, jitlink::Symbol *> ExistingStrings;
1603  for (auto *Sym : CStringSec->symbols()) {
1604
1605    // The LinkGraph builder should have created single strings blocks, and all
1606    // plugins should have maintained this invariant.
1607    auto Content = Sym->getBlock().getContent();
1608    ExistingStrings.insert(
1609        std::make_pair(StringRef(Content.data(), Content.size()), Sym));
1610  }
1611
1612  // Add all symbol names to the string section, and record the symbols for
1613  // those names.
1614  {
1615    SmallVector<jitlink::Symbol *> SymsToProcess;
1616    for (auto *Sym : G.defined_symbols())
1617      SymsToProcess.push_back(Sym);
1618    for (auto *Sym : G.absolute_symbols())
1619      SymsToProcess.push_back(Sym);
1620
1621    for (auto *Sym : SymsToProcess) {
1622      if (!Sym->hasName())
1623        continue;
1624
1625      auto I = ExistingStrings.find(Sym->getName());
1626      if (I == ExistingStrings.end()) {
1627        auto &NameBlock = G.createMutableContentBlock(
1628            *CStringSec, G.allocateCString(Sym->getName()), orc::ExecutorAddr(),
1629            1, 0);
1630        auto &SymbolNameSym = G.addAnonymousSymbol(
1631            NameBlock, 0, NameBlock.getSize(), false, true);
1632        JITSymTabInfo.push_back({Sym, &SymbolNameSym});
1633      } else
1634        JITSymTabInfo.push_back({Sym, I->second});
1635    }
1636  }
1637
1638  return Error::success();
1639}
1640
1641Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration(
1642    jitlink::LinkGraph &G, MaterializationResponsibility &MR,
1643    JITSymTabVector &JITSymTabInfo, bool InBootstrapPhase) {
1644
1645  ExecutorAddr HeaderAddr;
1646  {
1647    std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1648    auto I = MP.JITDylibToHeaderAddr.find(&MR.getTargetJITDylib());
1649    assert(I != MP.JITDylibToHeaderAddr.end() && "No header registered for JD");
1650    assert(I->second && "Null header registered for JD");
1651    HeaderAddr = I->second;
1652  }
1653
1654  SymbolTableVector LocalSymTab;
1655  auto &SymTab = LLVM_LIKELY(!InBootstrapPhase) ? LocalSymTab
1656                                                : MP.Bootstrap.load()->SymTab;
1657  for (auto &[OriginalSymbol, NameSym] : JITSymTabInfo)
1658    SymTab.push_back({NameSym->getAddress(), OriginalSymbol->getAddress(),
1659                      flagsForSymbol(*OriginalSymbol)});
1660
1661  // Bail out if we're in the bootstrap phase -- registration of thees symbols
1662  // will be attached to the bootstrap graph.
1663  if (LLVM_UNLIKELY(InBootstrapPhase))
1664    return Error::success();
1665
1666  shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase)
1667                                           ? G.allocActions()
1668                                           : MP.Bootstrap.load()->DeferredAAs;
1669  allocActions.push_back(
1670      {cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>(
1671           MP.RegisterObjectSymbolTable.Addr, HeaderAddr, SymTab)),
1672       cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>(
1673           MP.DeregisterObjectSymbolTable.Addr, HeaderAddr, SymTab))});
1674
1675  return Error::success();
1676}
1677
1678template <typename MachOTraits>
1679jitlink::Block &createHeaderBlock(MachOPlatform &MOP,
1680                                  const MachOPlatform::HeaderOptions &Opts,
1681                                  JITDylib &JD, jitlink::LinkGraph &G,
1682                                  jitlink::Section &HeaderSection) {
1683  auto HdrInfo =
1684      getMachOHeaderInfoFromTriple(MOP.getExecutionSession().getTargetTriple());
1685  MachOBuilder<MachOTraits> B(HdrInfo.PageSize);
1686
1687  B.Header.filetype = MachO::MH_DYLIB;
1688  B.Header.cputype = HdrInfo.CPUType;
1689  B.Header.cpusubtype = HdrInfo.CPUSubType;
1690
1691  if (Opts.IDDylib)
1692    B.template addLoadCommand<MachO::LC_ID_DYLIB>(
1693        Opts.IDDylib->Name, Opts.IDDylib->Timestamp,
1694        Opts.IDDylib->CurrentVersion, Opts.IDDylib->CompatibilityVersion);
1695  else
1696    B.template addLoadCommand<MachO::LC_ID_DYLIB>(JD.getName(), 0, 0, 0);
1697
1698  for (auto &D : Opts.LoadDylibs)
1699    B.template addLoadCommand<MachO::LC_LOAD_DYLIB>(
1700        D.Name, D.Timestamp, D.CurrentVersion, D.CompatibilityVersion);
1701  for (auto &P : Opts.RPaths)
1702    B.template addLoadCommand<MachO::LC_RPATH>(P);
1703
1704  auto HeaderContent = G.allocateBuffer(B.layout());
1705  B.write(HeaderContent);
1706
1707  return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
1708                              0);
1709}
1710
1711SimpleMachOHeaderMU::SimpleMachOHeaderMU(MachOPlatform &MOP,
1712                                         SymbolStringPtr HeaderStartSymbol,
1713                                         MachOPlatform::HeaderOptions Opts)
1714    : MaterializationUnit(
1715          createHeaderInterface(MOP, std::move(HeaderStartSymbol))),
1716      MOP(MOP), Opts(std::move(Opts)) {}
1717
1718void SimpleMachOHeaderMU::materialize(
1719    std::unique_ptr<MaterializationResponsibility> R) {
1720  auto G = createPlatformGraph(MOP, "<MachOHeaderMU>");
1721  addMachOHeader(R->getTargetJITDylib(), *G, R->getInitializerSymbol());
1722  MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
1723}
1724
1725void SimpleMachOHeaderMU::discard(const JITDylib &JD,
1726                                  const SymbolStringPtr &Sym) {}
1727
1728void SimpleMachOHeaderMU::addMachOHeader(
1729    JITDylib &JD, jitlink::LinkGraph &G,
1730    const SymbolStringPtr &InitializerSymbol) {
1731  auto &HeaderSection = G.createSection("__header", MemProt::Read);
1732  auto &HeaderBlock = createHeaderBlock(JD, G, HeaderSection);
1733
1734  // Init symbol is header-start symbol.
1735  G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, HeaderBlock.getSize(),
1736                     jitlink::Linkage::Strong, jitlink::Scope::Default, false,
1737                     true);
1738  for (auto &HS : AdditionalHeaderSymbols)
1739    G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(),
1740                       jitlink::Linkage::Strong, jitlink::Scope::Default, false,
1741                       true);
1742}
1743
1744jitlink::Block &
1745SimpleMachOHeaderMU::createHeaderBlock(JITDylib &JD, jitlink::LinkGraph &G,
1746                                       jitlink::Section &HeaderSection) {
1747  switch (MOP.getExecutionSession().getTargetTriple().getArch()) {
1748  case Triple::aarch64:
1749  case Triple::x86_64:
1750    return ::createHeaderBlock<MachO64LE>(MOP, Opts, JD, G, HeaderSection);
1751  default:
1752    llvm_unreachable("Unsupported architecture");
1753  }
1754}
1755
1756MaterializationUnit::Interface SimpleMachOHeaderMU::createHeaderInterface(
1757    MachOPlatform &MOP, const SymbolStringPtr &HeaderStartSymbol) {
1758  SymbolFlagsMap HeaderSymbolFlags;
1759
1760  HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
1761  for (auto &HS : AdditionalHeaderSymbols)
1762    HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] =
1763        JITSymbolFlags::Exported;
1764
1765  return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
1766                                        HeaderStartSymbol);
1767}
1768
1769MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT) {
1770  switch (TT.getArch()) {
1771  case Triple::aarch64:
1772    return {/* PageSize   = */ 16 * 1024,
1773            /* CPUType    = */ MachO::CPU_TYPE_ARM64,
1774            /* CPUSubType = */ MachO::CPU_SUBTYPE_ARM64_ALL};
1775  case Triple::x86_64:
1776    return {/* PageSize   = */ 4 * 1024,
1777            /* CPUType    = */ MachO::CPU_TYPE_X86_64,
1778            /* CPUSubType = */ MachO::CPU_SUBTYPE_X86_64_ALL};
1779  default:
1780    llvm_unreachable("Unrecognized architecture");
1781  }
1782}
1783
1784} // End namespace orc.
1785} // End namespace llvm.
1786