//===- coff_platform.cpp --------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains code required to load the rest of the COFF runtime. // //===----------------------------------------------------------------------===// #define NOMINMAX #include #include "coff_platform.h" #include "debug.h" #include "error.h" #include "wrapper_function_utils.h" #include #include #include #include #include #include #include #define DEBUG_TYPE "coff_platform" using namespace __orc_rt; namespace __orc_rt { using COFFJITDylibDepInfo = std::vector; using COFFJITDylibDepInfoMap = std::unordered_map; using SPSCOFFObjectSectionsMap = SPSSequence>; using SPSCOFFJITDylibDepInfo = SPSSequence; using SPSCOFFJITDylibDepInfoMap = SPSSequence>; } // namespace __orc_rt ORC_RT_JIT_DISPATCH_TAG(__orc_rt_coff_symbol_lookup_tag) ORC_RT_JIT_DISPATCH_TAG(__orc_rt_coff_push_initializers_tag) namespace { class COFFPlatformRuntimeState { private: // Ctor/dtor section. // Manage lists of *tor functions sorted by the last character of subsection // name. struct XtorSection { void Register(char SubsectionChar, span Xtors) { Subsections[SubsectionChar - 'A'].push_back(Xtors); SubsectionsNew[SubsectionChar - 'A'].push_back(Xtors); } void RegisterNoRun(char SubsectionChar, span Xtors) { Subsections[SubsectionChar - 'A'].push_back(Xtors); } void Reset() { SubsectionsNew = Subsections; } void RunAllNewAndFlush(); private: std::array>, 26> Subsections; std::array>, 26> SubsectionsNew; }; struct JITDylibState { std::string Name; void *Header = nullptr; size_t LinkedAgainstRefCount = 0; size_t DlRefCount = 0; std::vector Deps; std::vector AtExits; XtorSection CInitSection; // XIA~XIZ XtorSection CXXInitSection; // XCA~XCZ XtorSection CPreTermSection; // XPA~XPZ XtorSection CTermSection; // XTA~XTZ bool referenced() const { return LinkedAgainstRefCount != 0 || DlRefCount != 0; } }; public: static void initialize(); static COFFPlatformRuntimeState &get(); static bool isInitialized() { return CPS; } static void destroy(); COFFPlatformRuntimeState() = default; // Delete copy and move constructors. COFFPlatformRuntimeState(const COFFPlatformRuntimeState &) = delete; COFFPlatformRuntimeState & operator=(const COFFPlatformRuntimeState &) = delete; COFFPlatformRuntimeState(COFFPlatformRuntimeState &&) = delete; COFFPlatformRuntimeState &operator=(COFFPlatformRuntimeState &&) = delete; const char *dlerror(); void *dlopen(std::string_view Name, int Mode); int dlclose(void *Header); void *dlsym(void *Header, std::string_view Symbol); Error registerJITDylib(std::string Name, void *Header); Error deregisterJITDylib(void *Header); Error registerAtExit(ExecutorAddr HeaderAddr, void (*AtExit)(void)); Error registerObjectSections( ExecutorAddr HeaderAddr, std::vector> Secs, bool RunInitializers); Error deregisterObjectSections( ExecutorAddr HeaderAddr, std::vector> Secs); void *findJITDylibBaseByPC(uint64_t PC); private: Error registerBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range); Error deregisterBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range); Error registerSEHFrames(ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange); Error deregisterSEHFrames(ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange); Expected dlopenImpl(std::string_view Path, int Mode); Error dlopenFull(JITDylibState &JDS); Error dlopenInitialize(JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo); Error dlcloseImpl(void *DSOHandle); Error dlcloseDeinitialize(JITDylibState &JDS); JITDylibState *getJITDylibStateByHeader(void *DSOHandle); JITDylibState *getJITDylibStateByName(std::string_view Path); Expected lookupSymbolInJITDylib(void *DSOHandle, std::string_view Symbol); static COFFPlatformRuntimeState *CPS; std::recursive_mutex JDStatesMutex; std::map JDStates; struct BlockRange { void *Header; size_t Size; }; std::map BlockRanges; std::unordered_map JDNameToHeader; std::string DLFcnError; }; } // namespace COFFPlatformRuntimeState *COFFPlatformRuntimeState::CPS = nullptr; COFFPlatformRuntimeState::JITDylibState * COFFPlatformRuntimeState::getJITDylibStateByHeader(void *Header) { auto I = JDStates.find(Header); if (I == JDStates.end()) return nullptr; return &I->second; } COFFPlatformRuntimeState::JITDylibState * COFFPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) { // FIXME: Avoid creating string copy here. auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); if (I == JDNameToHeader.end()) return nullptr; void *H = I->second; auto J = JDStates.find(H); assert(J != JDStates.end() && "JITDylib has name map entry but no header map entry"); return &J->second; } Error COFFPlatformRuntimeState::registerJITDylib(std::string Name, void *Header) { ORC_RT_DEBUG({ printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header); }); std::lock_guard Lock(JDStatesMutex); if (JDStates.count(Header)) { std::ostringstream ErrStream; ErrStream << "Duplicate JITDylib registration for header " << Header << " (name = " << Name << ")"; return make_error(ErrStream.str()); } if (JDNameToHeader.count(Name)) { std::ostringstream ErrStream; ErrStream << "Duplicate JITDylib registration for header " << Header << " (header = " << Header << ")"; return make_error(ErrStream.str()); } auto &JDS = JDStates[Header]; JDS.Name = std::move(Name); JDS.Header = Header; JDNameToHeader[JDS.Name] = Header; return Error::success(); } Error COFFPlatformRuntimeState::deregisterJITDylib(void *Header) { std::lock_guard Lock(JDStatesMutex); auto I = JDStates.find(Header); if (I == JDStates.end()) { std::ostringstream ErrStream; ErrStream << "Attempted to deregister unrecognized header " << Header; return make_error(ErrStream.str()); } // Remove std::string construction once we can use C++20. auto J = JDNameToHeader.find( std::string(I->second.Name.data(), I->second.Name.size())); assert(J != JDNameToHeader.end() && "Missing JDNameToHeader entry for JITDylib"); ORC_RT_DEBUG({ printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(), Header); }); JDNameToHeader.erase(J); JDStates.erase(I); return Error::success(); } void COFFPlatformRuntimeState::XtorSection::RunAllNewAndFlush() { for (auto &Subsection : SubsectionsNew) { for (auto &XtorGroup : Subsection) for (auto &Xtor : XtorGroup) if (Xtor) Xtor(); Subsection.clear(); } } const char *COFFPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } void *COFFPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { ORC_RT_DEBUG({ std::string S(Path.data(), Path.size()); printdbg("COFFPlatform::dlopen(\"%s\")\n", S.c_str()); }); std::lock_guard Lock(JDStatesMutex); if (auto H = dlopenImpl(Path, Mode)) return *H; else { // FIXME: Make dlerror thread safe. DLFcnError = toString(H.takeError()); return nullptr; } } int COFFPlatformRuntimeState::dlclose(void *DSOHandle) { ORC_RT_DEBUG({ auto *JDS = getJITDylibStateByHeader(DSOHandle); std::string DylibName; if (JDS) { std::string S; printdbg("COFFPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str()); } else printdbg("COFFPlatform::dlclose(%p) (%s)\n", DSOHandle, "invalid handle"); }); std::lock_guard Lock(JDStatesMutex); if (auto Err = dlcloseImpl(DSOHandle)) { // FIXME: Make dlerror thread safe. DLFcnError = toString(std::move(Err)); return -1; } return 0; } void *COFFPlatformRuntimeState::dlsym(void *Header, std::string_view Symbol) { auto Addr = lookupSymbolInJITDylib(Header, Symbol); if (!Addr) { return 0; } return Addr->toPtr(); } Expected COFFPlatformRuntimeState::dlopenImpl(std::string_view Path, int Mode) { // Try to find JITDylib state by name. auto *JDS = getJITDylibStateByName(Path); if (!JDS) return make_error("No registered JTIDylib for path " + std::string(Path.data(), Path.size())); if (auto Err = dlopenFull(*JDS)) return std::move(Err); // Bump the ref-count on this dylib. ++JDS->DlRefCount; // Return the header address. return JDS->Header; } Error COFFPlatformRuntimeState::dlopenFull(JITDylibState &JDS) { // Call back to the JIT to push the initializers. Expected DepInfoMap((COFFJITDylibDepInfoMap())); if (auto Err = WrapperFunction( SPSExecutorAddr)>::call(&__orc_rt_coff_push_initializers_tag, DepInfoMap, ExecutorAddr::fromPtr(JDS.Header))) return Err; if (!DepInfoMap) return DepInfoMap.takeError(); if (auto Err = dlopenInitialize(JDS, *DepInfoMap)) return Err; if (!DepInfoMap->empty()) { ORC_RT_DEBUG({ printdbg("Unrecognized dep-info key headers in dlopen of %s\n", JDS.Name.c_str()); }); std::ostringstream ErrStream; ErrStream << "Encountered unrecognized dep-info key headers " "while processing dlopen of " << JDS.Name; return make_error(ErrStream.str()); } return Error::success(); } Error COFFPlatformRuntimeState::dlopenInitialize( JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo) { ORC_RT_DEBUG({ printdbg("COFFPlatformRuntimeState::dlopenInitialize(\"%s\")\n", JDS.Name.c_str()); }); // Skip visited dependency. auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header)); if (I == DepInfo.end()) return Error::success(); auto DI = std::move(I->second); DepInfo.erase(I); // Run initializers of dependencies in proper order by depth-first traversal // of dependency graph. std::vector OldDeps; std::swap(JDS.Deps, OldDeps); JDS.Deps.reserve(DI.size()); for (auto DepHeaderAddr : DI) { auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr()); if (!DepJDS) { std::ostringstream ErrStream; ErrStream << "Encountered unrecognized dep header " << DepHeaderAddr.toPtr() << " while initializing " << JDS.Name; return make_error(ErrStream.str()); } ++DepJDS->LinkedAgainstRefCount; if (auto Err = dlopenInitialize(*DepJDS, DepInfo)) return Err; } // Run static initializers. JDS.CInitSection.RunAllNewAndFlush(); JDS.CXXInitSection.RunAllNewAndFlush(); // Decrement old deps. for (auto *DepJDS : OldDeps) { --DepJDS->LinkedAgainstRefCount; if (!DepJDS->referenced()) if (auto Err = dlcloseDeinitialize(*DepJDS)) return Err; } return Error::success(); } Error COFFPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { // Try to find JITDylib state by header. auto *JDS = getJITDylibStateByHeader(DSOHandle); if (!JDS) { std::ostringstream ErrStream; ErrStream << "No registered JITDylib for " << DSOHandle; return make_error(ErrStream.str()); } // Bump the ref-count. --JDS->DlRefCount; if (!JDS->referenced()) return dlcloseDeinitialize(*JDS); return Error::success(); } Error COFFPlatformRuntimeState::dlcloseDeinitialize(JITDylibState &JDS) { ORC_RT_DEBUG({ printdbg("COFFPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n", JDS.Name.c_str()); }); // Run atexits for (auto AtExit : JDS.AtExits) AtExit(); JDS.AtExits.clear(); // Run static terminators. JDS.CPreTermSection.RunAllNewAndFlush(); JDS.CTermSection.RunAllNewAndFlush(); // Queue all xtors as new again. JDS.CInitSection.Reset(); JDS.CXXInitSection.Reset(); JDS.CPreTermSection.Reset(); JDS.CTermSection.Reset(); // Deinitialize any dependencies. for (auto *DepJDS : JDS.Deps) { --DepJDS->LinkedAgainstRefCount; if (!DepJDS->referenced()) if (auto Err = dlcloseDeinitialize(*DepJDS)) return Err; } return Error::success(); } Expected COFFPlatformRuntimeState::lookupSymbolInJITDylib(void *header, std::string_view Sym) { Expected Result((ExecutorAddr())); if (auto Err = WrapperFunction( SPSExecutorAddr, SPSString)>::call(&__orc_rt_coff_symbol_lookup_tag, Result, ExecutorAddr::fromPtr(header), Sym)) return std::move(Err); return Result; } Error COFFPlatformRuntimeState::registerObjectSections( ExecutorAddr HeaderAddr, std::vector> Secs, bool RunInitializers) { std::lock_guard Lock(JDStatesMutex); auto I = JDStates.find(HeaderAddr.toPtr()); if (I == JDStates.end()) { std::ostringstream ErrStream; ErrStream << "Unrecognized header " << HeaderAddr.getValue(); return make_error(ErrStream.str()); } auto &JDState = I->second; for (auto &KV : Secs) { if (auto Err = registerBlockRange(HeaderAddr, KV.second)) return Err; if (KV.first.empty()) continue; char LastChar = KV.first.data()[KV.first.size() - 1]; if (KV.first == ".pdata") { if (auto Err = registerSEHFrames(HeaderAddr, KV.second)) return Err; } else if (KV.first >= ".CRT$XIA" && KV.first <= ".CRT$XIZ") { if (RunInitializers) JDState.CInitSection.Register(LastChar, KV.second.toSpan()); else JDState.CInitSection.RegisterNoRun(LastChar, KV.second.toSpan()); } else if (KV.first >= ".CRT$XCA" && KV.first <= ".CRT$XCZ") { if (RunInitializers) JDState.CXXInitSection.Register(LastChar, KV.second.toSpan()); else JDState.CXXInitSection.RegisterNoRun( LastChar, KV.second.toSpan()); } else if (KV.first >= ".CRT$XPA" && KV.first <= ".CRT$XPZ") JDState.CPreTermSection.Register(LastChar, KV.second.toSpan()); else if (KV.first >= ".CRT$XTA" && KV.first <= ".CRT$XTZ") JDState.CTermSection.Register(LastChar, KV.second.toSpan()); } return Error::success(); } Error COFFPlatformRuntimeState::deregisterObjectSections( ExecutorAddr HeaderAddr, std::vector> Secs) { std::lock_guard Lock(JDStatesMutex); auto I = JDStates.find(HeaderAddr.toPtr()); if (I == JDStates.end()) { std::ostringstream ErrStream; ErrStream << "Attempted to deregister unrecognized header " << HeaderAddr.getValue(); return make_error(ErrStream.str()); } auto &JDState = I->second; for (auto &KV : Secs) { if (auto Err = deregisterBlockRange(HeaderAddr, KV.second)) return Err; if (KV.first == ".pdata") if (auto Err = deregisterSEHFrames(HeaderAddr, KV.second)) return Err; } return Error::success(); } Error COFFPlatformRuntimeState::registerSEHFrames( ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange) { int N = (SEHFrameRange.End.getValue() - SEHFrameRange.Start.getValue()) / sizeof(RUNTIME_FUNCTION); auto Func = SEHFrameRange.Start.toPtr(); if (!RtlAddFunctionTable(Func, N, static_cast(HeaderAddr.getValue()))) return make_error("Failed to register SEH frames"); return Error::success(); } Error COFFPlatformRuntimeState::deregisterSEHFrames( ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange) { if (!RtlDeleteFunctionTable(SEHFrameRange.Start.toPtr())) return make_error("Failed to deregister SEH frames"); return Error::success(); } Error COFFPlatformRuntimeState::registerBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range) { assert(!BlockRanges.count(Range.Start.toPtr()) && "Block range address already registered"); BlockRange B = {HeaderAddr.toPtr(), Range.size()}; BlockRanges.emplace(Range.Start.toPtr(), B); return Error::success(); } Error COFFPlatformRuntimeState::deregisterBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range) { assert(BlockRanges.count(Range.Start.toPtr()) && "Block range address not registered"); BlockRanges.erase(Range.Start.toPtr()); return Error::success(); } Error COFFPlatformRuntimeState::registerAtExit(ExecutorAddr HeaderAddr, void (*AtExit)(void)) { std::lock_guard Lock(JDStatesMutex); auto I = JDStates.find(HeaderAddr.toPtr()); if (I == JDStates.end()) { std::ostringstream ErrStream; ErrStream << "Unrecognized header " << HeaderAddr.getValue(); return make_error(ErrStream.str()); } I->second.AtExits.push_back(AtExit); return Error::success(); } void COFFPlatformRuntimeState::initialize() { assert(!CPS && "COFFPlatformRuntimeState should be null"); CPS = new COFFPlatformRuntimeState(); } COFFPlatformRuntimeState &COFFPlatformRuntimeState::get() { assert(CPS && "COFFPlatformRuntimeState not initialized"); return *CPS; } void COFFPlatformRuntimeState::destroy() { assert(CPS && "COFFPlatformRuntimeState not initialized"); delete CPS; } void *COFFPlatformRuntimeState::findJITDylibBaseByPC(uint64_t PC) { std::lock_guard Lock(JDStatesMutex); auto It = BlockRanges.upper_bound(reinterpret_cast(PC)); if (It == BlockRanges.begin()) return nullptr; --It; auto &Range = It->second; if (PC >= reinterpret_cast(It->first) + Range.Size) return nullptr; return Range.Header; } ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_coff_platform_bootstrap(char *ArgData, size_t ArgSize) { COFFPlatformRuntimeState::initialize(); return WrapperFunctionResult().release(); } ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_coff_platform_shutdown(char *ArgData, size_t ArgSize) { COFFPlatformRuntimeState::destroy(); return WrapperFunctionResult().release(); } ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_coff_register_jitdylib(char *ArgData, size_t ArgSize) { return WrapperFunction::handle( ArgData, ArgSize, [](std::string &Name, ExecutorAddr HeaderAddr) { return COFFPlatformRuntimeState::get().registerJITDylib( std::move(Name), HeaderAddr.toPtr()); }) .release(); } ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_coff_deregister_jitdylib(char *ArgData, size_t ArgSize) { return WrapperFunction::handle( ArgData, ArgSize, [](ExecutorAddr HeaderAddr) { return COFFPlatformRuntimeState::get().deregisterJITDylib( HeaderAddr.toPtr()); }) .release(); } ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_coff_register_object_sections(char *ArgData, size_t ArgSize) { return WrapperFunction:: handle(ArgData, ArgSize, [](ExecutorAddr HeaderAddr, std::vector> &Secs, bool RunInitializers) { return COFFPlatformRuntimeState::get().registerObjectSections( HeaderAddr, std::move(Secs), RunInitializers); }) .release(); } ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_coff_deregister_object_sections(char *ArgData, size_t ArgSize) { return WrapperFunction:: handle(ArgData, ArgSize, [](ExecutorAddr HeaderAddr, std::vector> &Secs) { return COFFPlatformRuntimeState::get().deregisterObjectSections( HeaderAddr, std::move(Secs)); }) .release(); } //------------------------------------------------------------------------------ // JIT'd dlfcn alternatives. //------------------------------------------------------------------------------ const char *__orc_rt_coff_jit_dlerror() { return COFFPlatformRuntimeState::get().dlerror(); } void *__orc_rt_coff_jit_dlopen(const char *path, int mode) { return COFFPlatformRuntimeState::get().dlopen(path, mode); } int __orc_rt_coff_jit_dlclose(void *header) { return COFFPlatformRuntimeState::get().dlclose(header); } void *__orc_rt_coff_jit_dlsym(void *header, const char *symbol) { return COFFPlatformRuntimeState::get().dlsym(header, symbol); } //------------------------------------------------------------------------------ // COFF SEH exception support //------------------------------------------------------------------------------ struct ThrowInfo { uint32_t attributes; void *data; }; ORC_RT_INTERFACE void __stdcall __orc_rt_coff_cxx_throw_exception( void *pExceptionObject, ThrowInfo *pThrowInfo) { constexpr uint32_t EH_EXCEPTION_NUMBER = 'msc' | 0xE0000000; constexpr uint32_t EH_MAGIC_NUMBER1 = 0x19930520; auto BaseAddr = COFFPlatformRuntimeState::get().findJITDylibBaseByPC( reinterpret_cast(pThrowInfo)); if (!BaseAddr) { // This is not from JIT'd region. // FIXME: Use the default implementation like below when alias api is // capable. _CxxThrowException(pExceptionObject, pThrowInfo); fprintf(stderr, "Throwing exception from compiled callback into JIT'd " "exception handler not supported yet.\n"); abort(); return; } const ULONG_PTR parameters[] = { EH_MAGIC_NUMBER1, reinterpret_cast(pExceptionObject), reinterpret_cast(pThrowInfo), reinterpret_cast(BaseAddr), }; RaiseException(EH_EXCEPTION_NUMBER, EXCEPTION_NONCONTINUABLE, _countof(parameters), parameters); } //------------------------------------------------------------------------------ // COFF atexits //------------------------------------------------------------------------------ typedef int (*OnExitFunction)(void); typedef void (*AtExitFunction)(void); ORC_RT_INTERFACE OnExitFunction __orc_rt_coff_onexit(void *Header, OnExitFunction Func) { if (auto Err = COFFPlatformRuntimeState::get().registerAtExit( ExecutorAddr::fromPtr(Header), (void (*)(void))Func)) { consumeError(std::move(Err)); return nullptr; } return Func; } ORC_RT_INTERFACE int __orc_rt_coff_atexit(void *Header, AtExitFunction Func) { if (auto Err = COFFPlatformRuntimeState::get().registerAtExit( ExecutorAddr::fromPtr(Header), (void (*)(void))Func)) { consumeError(std::move(Err)); return -1; } return 0; } //------------------------------------------------------------------------------ // COFF Run Program //------------------------------------------------------------------------------ ORC_RT_INTERFACE int64_t __orc_rt_coff_run_program(const char *JITDylibName, const char *EntrySymbolName, int argc, char *argv[]) { using MainTy = int (*)(int, char *[]); void *H = __orc_rt_coff_jit_dlopen(JITDylibName, __orc_rt::coff::ORC_RT_RTLD_LAZY); if (!H) { __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); return -1; } auto *Main = reinterpret_cast(__orc_rt_coff_jit_dlsym(H, EntrySymbolName)); if (!Main) { __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); return -1; } int Result = Main(argc, argv); if (__orc_rt_coff_jit_dlclose(H) == -1) __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); return Result; }