1//===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// ELF/x86-64 jit-link implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
15#include "llvm/ExecutionEngine/JITLink/JITLink.h"
16#include "llvm/ExecutionEngine/JITLink/TableManager.h"
17#include "llvm/ExecutionEngine/JITLink/x86_64.h"
18#include "llvm/Object/ELFObjectFile.h"
19
20#include "DefineExternalSectionStartAndEndSymbols.h"
21#include "EHFrameSupportImpl.h"
22#include "ELFLinkGraphBuilder.h"
23#include "JITLinkGeneric.h"
24
25#define DEBUG_TYPE "jitlink"
26
27using namespace llvm;
28using namespace llvm::jitlink;
29
30namespace {
31
32constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
33constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
34
35class TLSInfoTableManager_ELF_x86_64
36    : public TableManager<TLSInfoTableManager_ELF_x86_64> {
37public:
38  static const uint8_t TLSInfoEntryContent[16];
39
40  static StringRef getSectionName() { return ELFTLSInfoSectionName; }
41
42  bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
43    if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
44      LLVM_DEBUG({
45        dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
46               << formatv("{0:x}", B->getFixupAddress(E)) << " ("
47               << formatv("{0:x}", B->getAddress()) << " + "
48               << formatv("{0:x}", E.getOffset()) << ")\n";
49      });
50      E.setKind(x86_64::Delta32);
51      E.setTarget(getEntryForTarget(G, E.getTarget()));
52      return true;
53    }
54    return false;
55  }
56
57  Symbol &createEntry(LinkGraph &G, Symbol &Target) {
58    // the TLS Info entry's key value will be written by the fixTLVSectionByName
59    // pass, so create mutable content.
60    auto &TLSInfoEntry = G.createMutableContentBlock(
61        getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),
62        orc::ExecutorAddr(), 8, 0);
63    TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
64    return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
65  }
66
67private:
68  Section &getTLSInfoSection(LinkGraph &G) {
69    if (!TLSInfoTable)
70      TLSInfoTable =
71          &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read);
72    return *TLSInfoTable;
73  }
74
75  ArrayRef<char> getTLSInfoEntryContent() const {
76    return {reinterpret_cast<const char *>(TLSInfoEntryContent),
77            sizeof(TLSInfoEntryContent)};
78  }
79
80  Section *TLSInfoTable = nullptr;
81};
82
83const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {
84    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
85    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
86};
87
88Error buildTables_ELF_x86_64(LinkGraph &G) {
89  LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
90
91  x86_64::GOTTableManager GOT;
92  x86_64::PLTTableManager PLT(GOT);
93  TLSInfoTableManager_ELF_x86_64 TLSInfo;
94  visitExistingEdges(G, GOT, PLT, TLSInfo);
95  return Error::success();
96}
97} // namespace
98
99namespace llvm {
100namespace jitlink {
101
102class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
103private:
104  using ELFT = object::ELF64LE;
105
106  Error addRelocations() override {
107    LLVM_DEBUG(dbgs() << "Processing relocations:\n");
108
109    using Base = ELFLinkGraphBuilder<ELFT>;
110    using Self = ELFLinkGraphBuilder_x86_64;
111    for (const auto &RelSect : Base::Sections) {
112      // Validate the section to read relocation entries from.
113      if (RelSect.sh_type == ELF::SHT_REL)
114        return make_error<StringError>(
115            "No SHT_REL in valid x64 ELF object files",
116            inconvertibleErrorCode());
117
118      if (Error Err = Base::forEachRelaRelocation(RelSect, this,
119                                                  &Self::addSingleRelocation))
120        return Err;
121    }
122
123    return Error::success();
124  }
125
126  Error addSingleRelocation(const typename ELFT::Rela &Rel,
127                            const typename ELFT::Shdr &FixupSection,
128                            Block &BlockToFix) {
129    using Base = ELFLinkGraphBuilder<ELFT>;
130
131    auto ELFReloc = Rel.getType(false);
132
133    // R_X86_64_NONE is a no-op.
134    if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE))
135      return Error::success();
136
137    uint32_t SymbolIndex = Rel.getSymbol(false);
138    auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
139    if (!ObjSymbol)
140      return ObjSymbol.takeError();
141
142    Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
143    if (!GraphSymbol)
144      return make_error<StringError>(
145          formatv("Could not find symbol at given index, did you add it to "
146                  "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
147                  SymbolIndex, (*ObjSymbol)->st_shndx,
148                  Base::GraphSymbols.size()),
149          inconvertibleErrorCode());
150
151    // Validate the relocation kind.
152    int64_t Addend = Rel.r_addend;
153    Edge::Kind Kind = Edge::Invalid;
154
155    switch (ELFReloc) {
156    case ELF::R_X86_64_PC32:
157    case ELF::R_X86_64_GOTPC32:
158      Kind = x86_64::Delta32;
159      break;
160    case ELF::R_X86_64_PC64:
161    case ELF::R_X86_64_GOTPC64:
162      Kind = x86_64::Delta64;
163      break;
164    case ELF::R_X86_64_32:
165      Kind = x86_64::Pointer32;
166      break;
167    case ELF::R_X86_64_16:
168      Kind = x86_64::Pointer16;
169      break;
170    case ELF::R_X86_64_8:
171      Kind = x86_64::Pointer8;
172      break;
173    case ELF::R_X86_64_32S:
174      Kind = x86_64::Pointer32Signed;
175      break;
176    case ELF::R_X86_64_64:
177      Kind = x86_64::Pointer64;
178      break;
179    case ELF::R_X86_64_GOTPCREL:
180      Kind = x86_64::RequestGOTAndTransformToDelta32;
181      break;
182    case ELF::R_X86_64_REX_GOTPCRELX:
183      Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
184      Addend = 0;
185      break;
186    case ELF::R_X86_64_TLSGD:
187      Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
188      break;
189    case ELF::R_X86_64_GOTPCRELX:
190      Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
191      Addend = 0;
192      break;
193    case ELF::R_X86_64_GOTPCREL64:
194      Kind = x86_64::RequestGOTAndTransformToDelta64;
195      break;
196    case ELF::R_X86_64_GOT64:
197      Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
198      break;
199    case ELF::R_X86_64_GOTOFF64:
200      Kind = x86_64::Delta64FromGOT;
201      break;
202    case ELF::R_X86_64_PLT32:
203      Kind = x86_64::BranchPCRel32;
204      // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to
205      // adjust the addend by '+4' to compensate.
206      Addend += 4;
207      break;
208    default:
209      return make_error<JITLinkError>(
210          "In " + G->getName() + ": Unsupported x86-64 relocation type " +
211          object::getELFRelocationTypeName(ELF::EM_X86_64, ELFReloc));
212    }
213
214    auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset;
215    Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
216    Edge GE(Kind, Offset, *GraphSymbol, Addend);
217    LLVM_DEBUG({
218      dbgs() << "    ";
219      printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind));
220      dbgs() << "\n";
221    });
222
223    BlockToFix.addEdge(std::move(GE));
224    return Error::success();
225  }
226
227public:
228  ELFLinkGraphBuilder_x86_64(StringRef FileName,
229                             const object::ELFFile<object::ELF64LE> &Obj,
230                             SubtargetFeatures Features)
231      : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"),
232                            std::move(Features), FileName,
233                            x86_64::getEdgeKindName) {}
234};
235
236class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
237  friend class JITLinker<ELFJITLinker_x86_64>;
238
239public:
240  ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
241                      std::unique_ptr<LinkGraph> G,
242                      PassConfiguration PassConfig)
243      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
244
245    if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple()))
246      getPassConfig().PostAllocationPasses.push_back(
247          [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
248  }
249
250private:
251  Symbol *GOTSymbol = nullptr;
252
253  Error getOrCreateGOTSymbol(LinkGraph &G) {
254    auto DefineExternalGOTSymbolIfPresent =
255        createDefineExternalSectionStartAndEndSymbolsPass(
256            [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
257              if (Sym.getName() == ELFGOTSymbolName)
258                if (auto *GOTSection = G.findSectionByName(
259                        x86_64::GOTTableManager::getSectionName())) {
260                  GOTSymbol = &Sym;
261                  return {*GOTSection, true};
262                }
263              return {};
264            });
265
266    // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
267    // external.
268    if (auto Err = DefineExternalGOTSymbolIfPresent(G))
269      return Err;
270
271    // If we succeeded then we're done.
272    if (GOTSymbol)
273      return Error::success();
274
275    // Otherwise look for a GOT section: If it already has a start symbol we'll
276    // record it, otherwise we'll create our own.
277    // If there's a GOT section but we didn't find an external GOT symbol...
278    if (auto *GOTSection =
279            G.findSectionByName(x86_64::GOTTableManager::getSectionName())) {
280
281      // Check for an existing defined symbol.
282      for (auto *Sym : GOTSection->symbols())
283        if (Sym->getName() == ELFGOTSymbolName) {
284          GOTSymbol = Sym;
285          return Error::success();
286        }
287
288      // If there's no defined symbol then create one.
289      SectionRange SR(*GOTSection);
290      if (SR.empty())
291        GOTSymbol =
292            &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
293                                 Linkage::Strong, Scope::Local, true);
294      else
295        GOTSymbol =
296            &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
297                                Linkage::Strong, Scope::Local, false, true);
298    }
299
300    // If we still haven't found a GOT symbol then double check the externals.
301    // We may have a GOT-relative reference but no GOT section, in which case
302    // we just need to point the GOT symbol at some address in this graph.
303    if (!GOTSymbol) {
304      for (auto *Sym : G.external_symbols()) {
305        if (Sym->getName() == ELFGOTSymbolName) {
306          auto Blocks = G.blocks();
307          if (!Blocks.empty()) {
308            G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());
309            GOTSymbol = Sym;
310            break;
311          }
312        }
313      }
314    }
315
316    return Error::success();
317  }
318
319  Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
320    return x86_64::applyFixup(G, B, E, GOTSymbol);
321  }
322};
323
324Expected<std::unique_ptr<LinkGraph>>
325createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
326  LLVM_DEBUG({
327    dbgs() << "Building jitlink graph for new input "
328           << ObjectBuffer.getBufferIdentifier() << "...\n";
329  });
330
331  auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
332  if (!ELFObj)
333    return ELFObj.takeError();
334
335  auto Features = (*ELFObj)->getFeatures();
336  if (!Features)
337    return Features.takeError();
338
339  auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
340  return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
341                                    ELFObjFile.getELFFile(),
342                                    std::move(*Features))
343      .buildGraph();
344}
345
346static SectionRangeSymbolDesc
347identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
348  constexpr StringRef StartSymbolPrefix = "__start";
349  constexpr StringRef EndSymbolPrefix = "__end";
350
351  auto SymName = Sym.getName();
352  if (SymName.starts_with(StartSymbolPrefix)) {
353    if (auto *Sec =
354            G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
355      return {*Sec, true};
356  } else if (SymName.starts_with(EndSymbolPrefix)) {
357    if (auto *Sec =
358            G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
359      return {*Sec, false};
360  }
361  return {};
362}
363
364void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
365                     std::unique_ptr<JITLinkContext> Ctx) {
366  PassConfiguration Config;
367
368  if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
369
370    Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
371    Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
372        ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64,
373        x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32));
374    Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
375
376    // Construct a JITLinker and run the link function.
377    // Add a mark-live pass.
378    if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
379      Config.PrePrunePasses.push_back(std::move(MarkLive));
380    else
381      Config.PrePrunePasses.push_back(markAllSymbolsLive);
382
383    // Add an in-place GOT/Stubs/TLSInfoEntry build pass.
384    Config.PostPrunePasses.push_back(buildTables_ELF_x86_64);
385
386    // Resolve any external section start / end symbols.
387    Config.PostAllocationPasses.push_back(
388        createDefineExternalSectionStartAndEndSymbolsPass(
389            identifyELFSectionStartAndEndSymbols));
390
391    // Add GOT/Stubs optimizer pass.
392    Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses);
393  }
394
395  if (auto Err = Ctx->modifyPassConfig(*G, Config))
396    return Ctx->notifyFailed(std::move(Err));
397
398  ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
399}
400} // end namespace jitlink
401} // end namespace llvm
402