1//===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// Orc based MCJIT replacement. 10// 11//===----------------------------------------------------------------------===// 12 13#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H 14#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H 15 16#include "llvm/ADT/ArrayRef.h" 17#include "llvm/ADT/STLExtras.h" 18#include "llvm/ADT/StringRef.h" 19#include "llvm/ExecutionEngine/ExecutionEngine.h" 20#include "llvm/ExecutionEngine/GenericValue.h" 21#include "llvm/ExecutionEngine/JITSymbol.h" 22#include "llvm/ExecutionEngine/Orc/CompileUtils.h" 23#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" 24#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" 25#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" 26#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" 27#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" 28#include "llvm/ExecutionEngine/RuntimeDyld.h" 29#include "llvm/IR/DataLayout.h" 30#include "llvm/IR/Function.h" 31#include "llvm/IR/Mangler.h" 32#include "llvm/IR/Module.h" 33#include "llvm/Object/Archive.h" 34#include "llvm/Object/Binary.h" 35#include "llvm/Object/ObjectFile.h" 36#include "llvm/Support/Error.h" 37#include "llvm/Support/ErrorHandling.h" 38#include "llvm/Support/raw_ostream.h" 39#include "llvm/Target/TargetMachine.h" 40#include <algorithm> 41#include <cassert> 42#include <cstddef> 43#include <cstdint> 44#include <map> 45#include <memory> 46#include <set> 47#include <string> 48#include <vector> 49 50namespace llvm { 51 52class ObjectCache; 53 54namespace orc { 55 56class OrcMCJITReplacement : public ExecutionEngine { 57 58 // OrcMCJITReplacement needs to do a little extra book-keeping to ensure that 59 // Orc's automatic finalization doesn't kick in earlier than MCJIT clients are 60 // expecting - see finalizeMemory. 61 class MCJITReplacementMemMgr : public MCJITMemoryManager { 62 public: 63 MCJITReplacementMemMgr(OrcMCJITReplacement &M, 64 std::shared_ptr<MCJITMemoryManager> ClientMM) 65 : M(M), ClientMM(std::move(ClientMM)) {} 66 67 uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, 68 unsigned SectionID, 69 StringRef SectionName) override { 70 uint8_t *Addr = 71 ClientMM->allocateCodeSection(Size, Alignment, SectionID, 72 SectionName); 73 M.SectionsAllocatedSinceLastLoad.insert(Addr); 74 return Addr; 75 } 76 77 uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, 78 unsigned SectionID, StringRef SectionName, 79 bool IsReadOnly) override { 80 uint8_t *Addr = ClientMM->allocateDataSection(Size, Alignment, SectionID, 81 SectionName, IsReadOnly); 82 M.SectionsAllocatedSinceLastLoad.insert(Addr); 83 return Addr; 84 } 85 86 void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, 87 uintptr_t RODataSize, uint32_t RODataAlign, 88 uintptr_t RWDataSize, 89 uint32_t RWDataAlign) override { 90 return ClientMM->reserveAllocationSpace(CodeSize, CodeAlign, 91 RODataSize, RODataAlign, 92 RWDataSize, RWDataAlign); 93 } 94 95 bool needsToReserveAllocationSpace() override { 96 return ClientMM->needsToReserveAllocationSpace(); 97 } 98 99 void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, 100 size_t Size) override { 101 return ClientMM->registerEHFrames(Addr, LoadAddr, Size); 102 } 103 104 void deregisterEHFrames() override { 105 return ClientMM->deregisterEHFrames(); 106 } 107 108 void notifyObjectLoaded(RuntimeDyld &RTDyld, 109 const object::ObjectFile &O) override { 110 return ClientMM->notifyObjectLoaded(RTDyld, O); 111 } 112 113 void notifyObjectLoaded(ExecutionEngine *EE, 114 const object::ObjectFile &O) override { 115 return ClientMM->notifyObjectLoaded(EE, O); 116 } 117 118 bool finalizeMemory(std::string *ErrMsg = nullptr) override { 119 // Each set of objects loaded will be finalized exactly once, but since 120 // symbol lookup during relocation may recursively trigger the 121 // loading/relocation of other modules, and since we're forwarding all 122 // finalizeMemory calls to a single underlying memory manager, we need to 123 // defer forwarding the call on until all necessary objects have been 124 // loaded. Otherwise, during the relocation of a leaf object, we will end 125 // up finalizing memory, causing a crash further up the stack when we 126 // attempt to apply relocations to finalized memory. 127 // To avoid finalizing too early, look at how many objects have been 128 // loaded but not yet finalized. This is a bit of a hack that relies on 129 // the fact that we're lazily emitting object files: The only way you can 130 // get more than one set of objects loaded but not yet finalized is if 131 // they were loaded during relocation of another set. 132 if (M.UnfinalizedSections.size() == 1) 133 return ClientMM->finalizeMemory(ErrMsg); 134 return false; 135 } 136 137 private: 138 OrcMCJITReplacement &M; 139 std::shared_ptr<MCJITMemoryManager> ClientMM; 140 }; 141 142 class LinkingORCResolver : public orc::SymbolResolver { 143 public: 144 LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {} 145 146 SymbolNameSet getResponsibilitySet(const SymbolNameSet &Symbols) override { 147 SymbolNameSet Result; 148 149 for (auto &S : Symbols) { 150 if (auto Sym = M.findMangledSymbol(*S)) { 151 if (!Sym.getFlags().isStrong()) 152 Result.insert(S); 153 } else if (auto Err = Sym.takeError()) { 154 M.reportError(std::move(Err)); 155 return SymbolNameSet(); 156 } else { 157 if (auto Sym2 = 158 M.ClientResolver->findSymbolInLogicalDylib(std::string(*S))) { 159 if (!Sym2.getFlags().isStrong()) 160 Result.insert(S); 161 } else if (auto Err = Sym2.takeError()) { 162 M.reportError(std::move(Err)); 163 return SymbolNameSet(); 164 } else 165 Result.insert(S); 166 } 167 } 168 169 return Result; 170 } 171 172 SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, 173 SymbolNameSet Symbols) override { 174 SymbolNameSet UnresolvedSymbols; 175 bool NewSymbolsResolved = false; 176 177 for (auto &S : Symbols) { 178 if (auto Sym = M.findMangledSymbol(*S)) { 179 if (auto Addr = Sym.getAddress()) { 180 Query->notifySymbolMetRequiredState( 181 S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); 182 NewSymbolsResolved = true; 183 } else { 184 M.ES.legacyFailQuery(*Query, Addr.takeError()); 185 return SymbolNameSet(); 186 } 187 } else if (auto Err = Sym.takeError()) { 188 M.ES.legacyFailQuery(*Query, std::move(Err)); 189 return SymbolNameSet(); 190 } else { 191 if (auto Sym2 = M.ClientResolver->findSymbol(std::string(*S))) { 192 if (auto Addr = Sym2.getAddress()) { 193 Query->notifySymbolMetRequiredState( 194 S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); 195 NewSymbolsResolved = true; 196 } else { 197 M.ES.legacyFailQuery(*Query, Addr.takeError()); 198 return SymbolNameSet(); 199 } 200 } else if (auto Err = Sym2.takeError()) { 201 M.ES.legacyFailQuery(*Query, std::move(Err)); 202 return SymbolNameSet(); 203 } else 204 UnresolvedSymbols.insert(S); 205 } 206 } 207 208 if (NewSymbolsResolved && Query->isComplete()) 209 Query->handleComplete(); 210 211 return UnresolvedSymbols; 212 } 213 214 private: 215 OrcMCJITReplacement &M; 216 }; 217 218private: 219 static ExecutionEngine * 220 createOrcMCJITReplacement(std::string *ErrorMsg, 221 std::shared_ptr<MCJITMemoryManager> MemMgr, 222 std::shared_ptr<LegacyJITSymbolResolver> Resolver, 223 std::unique_ptr<TargetMachine> TM) { 224 return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver), 225 std::move(TM)); 226 } 227 228 void reportError(Error Err) { 229 logAllUnhandledErrors(std::move(Err), errs(), "MCJIT error: "); 230 } 231 232public: 233 OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr, 234 std::shared_ptr<LegacyJITSymbolResolver> ClientResolver, 235 std::unique_ptr<TargetMachine> TM) 236 : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), 237 MemMgr( 238 std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))), 239 Resolver(std::make_shared<LinkingORCResolver>(*this)), 240 ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), 241 NotifyFinalized(*this), 242 ObjectLayer( 243 AcknowledgeORCv1Deprecation, ES, 244 [this](VModuleKey K) { 245 return ObjectLayerT::Resources{this->MemMgr, this->Resolver}; 246 }, 247 NotifyObjectLoaded, NotifyFinalized), 248 CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, 249 SimpleCompiler(*this->TM), 250 [this](VModuleKey K, std::unique_ptr<Module> M) { 251 Modules.push_back(std::move(M)); 252 }), 253 LazyEmitLayer(AcknowledgeORCv1Deprecation, CompileLayer) {} 254 255 static void Register() { 256 OrcMCJITReplacementCtor = createOrcMCJITReplacement; 257 } 258 259 void addModule(std::unique_ptr<Module> M) override { 260 // If this module doesn't have a DataLayout attached then attach the 261 // default. 262 if (M->getDataLayout().isDefault()) { 263 M->setDataLayout(getDataLayout()); 264 } else { 265 assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch"); 266 } 267 268 // Rename, bump linkage and record static constructors and destructors. 269 // We have to do this before we hand over ownership of the module to the 270 // JIT. 271 std::vector<std::string> CtorNames, DtorNames; 272 { 273 unsigned CtorId = 0, DtorId = 0; 274 for (auto Ctor : orc::getConstructors(*M)) { 275 std::string NewCtorName = ("__ORCstatic_ctor." + Twine(CtorId++)).str(); 276 Ctor.Func->setName(NewCtorName); 277 Ctor.Func->setLinkage(GlobalValue::ExternalLinkage); 278 Ctor.Func->setVisibility(GlobalValue::HiddenVisibility); 279 CtorNames.push_back(mangle(NewCtorName)); 280 } 281 for (auto Dtor : orc::getDestructors(*M)) { 282 std::string NewDtorName = ("__ORCstatic_dtor." + Twine(DtorId++)).str(); 283 dbgs() << "Found dtor: " << NewDtorName << "\n"; 284 Dtor.Func->setName(NewDtorName); 285 Dtor.Func->setLinkage(GlobalValue::ExternalLinkage); 286 Dtor.Func->setVisibility(GlobalValue::HiddenVisibility); 287 DtorNames.push_back(mangle(NewDtorName)); 288 } 289 } 290 291 auto K = ES.allocateVModule(); 292 293 UnexecutedConstructors[K] = std::move(CtorNames); 294 UnexecutedDestructors[K] = std::move(DtorNames); 295 296 cantFail(LazyEmitLayer.addModule(K, std::move(M))); 297 } 298 299 void addObjectFile(std::unique_ptr<object::ObjectFile> O) override { 300 cantFail(ObjectLayer.addObject( 301 ES.allocateVModule(), MemoryBuffer::getMemBufferCopy(O->getData()))); 302 } 303 304 void addObjectFile(object::OwningBinary<object::ObjectFile> O) override { 305 std::unique_ptr<object::ObjectFile> Obj; 306 std::unique_ptr<MemoryBuffer> ObjBuffer; 307 std::tie(Obj, ObjBuffer) = O.takeBinary(); 308 cantFail(ObjectLayer.addObject(ES.allocateVModule(), std::move(ObjBuffer))); 309 } 310 311 void addArchive(object::OwningBinary<object::Archive> A) override { 312 Archives.push_back(std::move(A)); 313 } 314 315 bool removeModule(Module *M) override { 316 auto I = Modules.begin(); 317 for (auto E = Modules.end(); I != E; ++I) 318 if (I->get() == M) 319 break; 320 if (I == Modules.end()) 321 return false; 322 Modules.erase(I); 323 return true; 324 } 325 326 uint64_t getSymbolAddress(StringRef Name) { 327 return cantFail(findSymbol(Name).getAddress()); 328 } 329 330 JITSymbol findSymbol(StringRef Name) { 331 return findMangledSymbol(mangle(Name)); 332 } 333 334 void finalizeObject() override { 335 // This is deprecated - Aim to remove in ExecutionEngine. 336 // REMOVE IF POSSIBLE - Doesn't make sense for New JIT. 337 } 338 339 void mapSectionAddress(const void *LocalAddress, 340 uint64_t TargetAddress) override { 341 for (auto &P : UnfinalizedSections) 342 if (P.second.count(LocalAddress)) 343 ObjectLayer.mapSectionAddress(P.first, LocalAddress, TargetAddress); 344 } 345 346 uint64_t getGlobalValueAddress(const std::string &Name) override { 347 return getSymbolAddress(Name); 348 } 349 350 uint64_t getFunctionAddress(const std::string &Name) override { 351 return getSymbolAddress(Name); 352 } 353 354 void *getPointerToFunction(Function *F) override { 355 uint64_t FAddr = getSymbolAddress(F->getName()); 356 return reinterpret_cast<void *>(static_cast<uintptr_t>(FAddr)); 357 } 358 359 void *getPointerToNamedFunction(StringRef Name, 360 bool AbortOnFailure = true) override { 361 uint64_t Addr = getSymbolAddress(Name); 362 if (!Addr && AbortOnFailure) 363 llvm_unreachable("Missing symbol!"); 364 return reinterpret_cast<void *>(static_cast<uintptr_t>(Addr)); 365 } 366 367 GenericValue runFunction(Function *F, 368 ArrayRef<GenericValue> ArgValues) override; 369 370 void setObjectCache(ObjectCache *NewCache) override { 371 CompileLayer.getCompiler().setObjectCache(NewCache); 372 } 373 374 void setProcessAllSections(bool ProcessAllSections) override { 375 ObjectLayer.setProcessAllSections(ProcessAllSections); 376 } 377 378 void runStaticConstructorsDestructors(bool isDtors) override; 379 380private: 381 JITSymbol findMangledSymbol(StringRef Name) { 382 if (auto Sym = LazyEmitLayer.findSymbol(std::string(Name), false)) 383 return Sym; 384 if (auto Sym = ClientResolver->findSymbol(std::string(Name))) 385 return Sym; 386 if (auto Sym = scanArchives(Name)) 387 return Sym; 388 389 return nullptr; 390 } 391 392 JITSymbol scanArchives(StringRef Name) { 393 for (object::OwningBinary<object::Archive> &OB : Archives) { 394 object::Archive *A = OB.getBinary(); 395 // Look for our symbols in each Archive 396 auto OptionalChildOrErr = A->findSym(Name); 397 if (!OptionalChildOrErr) 398 report_fatal_error(OptionalChildOrErr.takeError()); 399 auto &OptionalChild = *OptionalChildOrErr; 400 if (OptionalChild) { 401 // FIXME: Support nested archives? 402 Expected<std::unique_ptr<object::Binary>> ChildBinOrErr = 403 OptionalChild->getAsBinary(); 404 if (!ChildBinOrErr) { 405 // TODO: Actually report errors helpfully. 406 consumeError(ChildBinOrErr.takeError()); 407 continue; 408 } 409 std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); 410 if (ChildBin->isObject()) { 411 cantFail(ObjectLayer.addObject( 412 ES.allocateVModule(), 413 MemoryBuffer::getMemBufferCopy(ChildBin->getData()))); 414 if (auto Sym = ObjectLayer.findSymbol(Name, true)) 415 return Sym; 416 } 417 } 418 } 419 return nullptr; 420 } 421 422 class NotifyObjectLoadedT { 423 public: 424 using LoadedObjInfoListT = 425 std::vector<std::unique_ptr<RuntimeDyld::LoadedObjectInfo>>; 426 427 NotifyObjectLoadedT(OrcMCJITReplacement &M) : M(M) {} 428 429 void operator()(VModuleKey K, const object::ObjectFile &Obj, 430 const RuntimeDyld::LoadedObjectInfo &Info) const { 431 M.UnfinalizedSections[K] = std::move(M.SectionsAllocatedSinceLastLoad); 432 M.SectionsAllocatedSinceLastLoad = SectionAddrSet(); 433 M.MemMgr->notifyObjectLoaded(&M, Obj); 434 } 435 private: 436 OrcMCJITReplacement &M; 437 }; 438 439 class NotifyFinalizedT { 440 public: 441 NotifyFinalizedT(OrcMCJITReplacement &M) : M(M) {} 442 443 void operator()(VModuleKey K, const object::ObjectFile &Obj, 444 const RuntimeDyld::LoadedObjectInfo &Info) { 445 M.UnfinalizedSections.erase(K); 446 } 447 448 private: 449 OrcMCJITReplacement &M; 450 }; 451 452 std::string mangle(StringRef Name) { 453 std::string MangledName; 454 { 455 raw_string_ostream MangledNameStream(MangledName); 456 Mang.getNameWithPrefix(MangledNameStream, Name, getDataLayout()); 457 } 458 return MangledName; 459 } 460 461 using ObjectLayerT = LegacyRTDyldObjectLinkingLayer; 462 using CompileLayerT = LegacyIRCompileLayer<ObjectLayerT, orc::SimpleCompiler>; 463 using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>; 464 465 ExecutionSession ES; 466 467 std::unique_ptr<TargetMachine> TM; 468 std::shared_ptr<MCJITReplacementMemMgr> MemMgr; 469 std::shared_ptr<LinkingORCResolver> Resolver; 470 std::shared_ptr<LegacyJITSymbolResolver> ClientResolver; 471 Mangler Mang; 472 473 // IMPORTANT: ShouldDelete *must* come before LocalModules: The shared_ptr 474 // delete blocks in LocalModules refer to the ShouldDelete map, so 475 // LocalModules needs to be destructed before ShouldDelete. 476 std::map<Module*, bool> ShouldDelete; 477 478 NotifyObjectLoadedT NotifyObjectLoaded; 479 NotifyFinalizedT NotifyFinalized; 480 481 ObjectLayerT ObjectLayer; 482 CompileLayerT CompileLayer; 483 LazyEmitLayerT LazyEmitLayer; 484 485 std::map<VModuleKey, std::vector<std::string>> UnexecutedConstructors; 486 std::map<VModuleKey, std::vector<std::string>> UnexecutedDestructors; 487 488 // We need to store ObjLayerT::ObjSetHandles for each of the object sets 489 // that have been emitted but not yet finalized so that we can forward the 490 // mapSectionAddress calls appropriately. 491 using SectionAddrSet = std::set<const void *>; 492 SectionAddrSet SectionsAllocatedSinceLastLoad; 493 std::map<VModuleKey, SectionAddrSet> UnfinalizedSections; 494 495 std::vector<object::OwningBinary<object::Archive>> Archives; 496}; 497 498} // end namespace orc 499 500} // end namespace llvm 501 502#endif // LLVM_LIB_EXECUTIONENGINE_ORC_MCJITREPLACEMENT_H 503