1218885Sdim//===- Unix/Memory.cpp - Generic UNIX System Configuration ------*- C++ -*-===// 2218885Sdim// 3218885Sdim// The LLVM Compiler Infrastructure 4218885Sdim// 5218885Sdim// This file is distributed under the University of Illinois Open Source 6218885Sdim// License. See LICENSE.TXT for details. 7218885Sdim// 8218885Sdim//===----------------------------------------------------------------------===// 9218885Sdim// 10218885Sdim// This file defines some functions for various memory management utilities. 11218885Sdim// 12218885Sdim//===----------------------------------------------------------------------===// 13218885Sdim 14218885Sdim#include "Unix.h" 15218885Sdim#include "llvm/Support/DataTypes.h" 16245431Sdim#include "llvm/Support/ErrorHandling.h" 17218885Sdim#include "llvm/Support/Process.h" 18218885Sdim 19218885Sdim#ifdef HAVE_SYS_MMAN_H 20218885Sdim#include <sys/mman.h> 21218885Sdim#endif 22218885Sdim 23218885Sdim#ifdef __APPLE__ 24218885Sdim#include <mach/mach.h> 25218885Sdim#endif 26218885Sdim 27245431Sdim#if defined(__mips__) 28245431Sdim# if defined(__OpenBSD__) 29245431Sdim# include <mips64/sysarch.h> 30245431Sdim# else 31245431Sdim# include <sys/cachectl.h> 32245431Sdim# endif 33245431Sdim#endif 34245431Sdim 35263509Sdim#ifdef __APPLE__ 36245431Sdimextern "C" void sys_icache_invalidate(const void *Addr, size_t len); 37263509Sdim#else 38263509Sdimextern "C" void __clear_cache(void *, void*); 39263509Sdim#endif 40245431Sdim 41245431Sdimnamespace { 42245431Sdim 43245431Sdimint getPosixProtectionFlags(unsigned Flags) { 44245431Sdim switch (Flags) { 45245431Sdim case llvm::sys::Memory::MF_READ: 46245431Sdim return PROT_READ; 47245431Sdim case llvm::sys::Memory::MF_WRITE: 48245431Sdim return PROT_WRITE; 49245431Sdim case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_WRITE: 50245431Sdim return PROT_READ | PROT_WRITE; 51245431Sdim case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_EXEC: 52245431Sdim return PROT_READ | PROT_EXEC; 53245431Sdim case llvm::sys::Memory::MF_READ | 54245431Sdim llvm::sys::Memory::MF_WRITE | 55245431Sdim llvm::sys::Memory::MF_EXEC: 56245431Sdim return PROT_READ | PROT_WRITE | PROT_EXEC; 57245431Sdim case llvm::sys::Memory::MF_EXEC: 58252723Sdim#if defined(__FreeBSD__) 59252723Sdim // On PowerPC, having an executable page that has no read permission 60252723Sdim // can have unintended consequences. The function InvalidateInstruction- 61252723Sdim // Cache uses instructions dcbf and icbi, both of which are treated by 62252723Sdim // the processor as loads. If the page has no read permissions, 63252723Sdim // executing these instructions will result in a segmentation fault. 64252723Sdim // Somehow, this problem is not present on Linux, but it does happen 65252723Sdim // on FreeBSD. 66252723Sdim return PROT_READ | PROT_EXEC; 67252723Sdim#else 68245431Sdim return PROT_EXEC; 69252723Sdim#endif 70245431Sdim default: 71245431Sdim llvm_unreachable("Illegal memory protection flag specified!"); 72245431Sdim } 73245431Sdim // Provide a default return value as required by some compilers. 74245431Sdim return PROT_NONE; 75245431Sdim} 76245431Sdim 77245431Sdim} // namespace 78245431Sdim 79245431Sdimnamespace llvm { 80245431Sdimnamespace sys { 81245431Sdim 82245431SdimMemoryBlock 83245431SdimMemory::allocateMappedMemory(size_t NumBytes, 84245431Sdim const MemoryBlock *const NearBlock, 85245431Sdim unsigned PFlags, 86245431Sdim error_code &EC) { 87245431Sdim EC = error_code::success(); 88245431Sdim if (NumBytes == 0) 89245431Sdim return MemoryBlock(); 90245431Sdim 91252723Sdim static const size_t PageSize = process::get_self()->page_size(); 92245431Sdim const size_t NumPages = (NumBytes+PageSize-1)/PageSize; 93245431Sdim 94245431Sdim int fd = -1; 95245431Sdim#ifdef NEED_DEV_ZERO_FOR_MMAP 96245431Sdim static int zero_fd = open("/dev/zero", O_RDWR); 97245431Sdim if (zero_fd == -1) { 98245431Sdim EC = error_code(errno, system_category()); 99245431Sdim return MemoryBlock(); 100245431Sdim } 101245431Sdim fd = zero_fd; 102245431Sdim#endif 103245431Sdim 104245431Sdim int MMFlags = MAP_PRIVATE | 105245431Sdim#ifdef HAVE_MMAP_ANONYMOUS 106245431Sdim MAP_ANONYMOUS 107245431Sdim#else 108245431Sdim MAP_ANON 109245431Sdim#endif 110245431Sdim ; // Ends statement above 111245431Sdim 112245431Sdim int Protect = getPosixProtectionFlags(PFlags); 113245431Sdim 114245431Sdim // Use any near hint and the page size to set a page-aligned starting address 115245431Sdim uintptr_t Start = NearBlock ? reinterpret_cast<uintptr_t>(NearBlock->base()) + 116245431Sdim NearBlock->size() : 0; 117245431Sdim if (Start && Start % PageSize) 118245431Sdim Start += PageSize - Start % PageSize; 119245431Sdim 120245431Sdim void *Addr = ::mmap(reinterpret_cast<void*>(Start), PageSize*NumPages, 121245431Sdim Protect, MMFlags, fd, 0); 122245431Sdim if (Addr == MAP_FAILED) { 123245431Sdim if (NearBlock) //Try again without a near hint 124245431Sdim return allocateMappedMemory(NumBytes, 0, PFlags, EC); 125245431Sdim 126245431Sdim EC = error_code(errno, system_category()); 127245431Sdim return MemoryBlock(); 128245431Sdim } 129245431Sdim 130245431Sdim MemoryBlock Result; 131245431Sdim Result.Address = Addr; 132245431Sdim Result.Size = NumPages*PageSize; 133245431Sdim 134245431Sdim if (PFlags & MF_EXEC) 135245431Sdim Memory::InvalidateInstructionCache(Result.Address, Result.Size); 136245431Sdim 137245431Sdim return Result; 138245431Sdim} 139245431Sdim 140245431Sdimerror_code 141245431SdimMemory::releaseMappedMemory(MemoryBlock &M) { 142245431Sdim if (M.Address == 0 || M.Size == 0) 143245431Sdim return error_code::success(); 144245431Sdim 145245431Sdim if (0 != ::munmap(M.Address, M.Size)) 146245431Sdim return error_code(errno, system_category()); 147245431Sdim 148245431Sdim M.Address = 0; 149245431Sdim M.Size = 0; 150245431Sdim 151245431Sdim return error_code::success(); 152245431Sdim} 153245431Sdim 154245431Sdimerror_code 155245431SdimMemory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { 156245431Sdim if (M.Address == 0 || M.Size == 0) 157245431Sdim return error_code::success(); 158245431Sdim 159245431Sdim if (!Flags) 160245431Sdim return error_code(EINVAL, generic_category()); 161245431Sdim 162245431Sdim int Protect = getPosixProtectionFlags(Flags); 163245431Sdim 164245431Sdim int Result = ::mprotect(M.Address, M.Size, Protect); 165245431Sdim if (Result != 0) 166245431Sdim return error_code(errno, system_category()); 167245431Sdim 168245431Sdim if (Flags & MF_EXEC) 169245431Sdim Memory::InvalidateInstructionCache(M.Address, M.Size); 170245431Sdim 171245431Sdim return error_code::success(); 172245431Sdim} 173245431Sdim 174218885Sdim/// AllocateRWX - Allocate a slab of memory with read/write/execute 175218885Sdim/// permissions. This is typically used for JIT applications where we want 176218885Sdim/// to emit code to the memory then jump to it. Getting this type of memory 177218885Sdim/// is very OS specific. 178218885Sdim/// 179245431SdimMemoryBlock 180245431SdimMemory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock, 181245431Sdim std::string *ErrMsg) { 182218885Sdim if (NumBytes == 0) return MemoryBlock(); 183218885Sdim 184252723Sdim size_t PageSize = process::get_self()->page_size(); 185252723Sdim size_t NumPages = (NumBytes+PageSize-1)/PageSize; 186218885Sdim 187218885Sdim int fd = -1; 188218885Sdim#ifdef NEED_DEV_ZERO_FOR_MMAP 189218885Sdim static int zero_fd = open("/dev/zero", O_RDWR); 190218885Sdim if (zero_fd == -1) { 191218885Sdim MakeErrMsg(ErrMsg, "Can't open /dev/zero device"); 192218885Sdim return MemoryBlock(); 193218885Sdim } 194218885Sdim fd = zero_fd; 195218885Sdim#endif 196218885Sdim 197218885Sdim int flags = MAP_PRIVATE | 198218885Sdim#ifdef HAVE_MMAP_ANONYMOUS 199218885Sdim MAP_ANONYMOUS 200218885Sdim#else 201218885Sdim MAP_ANON 202218885Sdim#endif 203218885Sdim ; 204218885Sdim 205218885Sdim void* start = NearBlock ? (unsigned char*)NearBlock->base() + 206218885Sdim NearBlock->size() : 0; 207218885Sdim 208218885Sdim#if defined(__APPLE__) && defined(__arm__) 209252723Sdim void *pa = ::mmap(start, PageSize*NumPages, PROT_READ|PROT_EXEC, 210218885Sdim flags, fd, 0); 211218885Sdim#else 212252723Sdim void *pa = ::mmap(start, PageSize*NumPages, PROT_READ|PROT_WRITE|PROT_EXEC, 213218885Sdim flags, fd, 0); 214218885Sdim#endif 215218885Sdim if (pa == MAP_FAILED) { 216218885Sdim if (NearBlock) //Try again without a near hint 217218885Sdim return AllocateRWX(NumBytes, 0); 218218885Sdim 219218885Sdim MakeErrMsg(ErrMsg, "Can't allocate RWX Memory"); 220218885Sdim return MemoryBlock(); 221218885Sdim } 222218885Sdim 223218885Sdim#if defined(__APPLE__) && defined(__arm__) 224218885Sdim kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)pa, 225252723Sdim (vm_size_t)(PageSize*NumPages), 0, 226218885Sdim VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY); 227218885Sdim if (KERN_SUCCESS != kr) { 228218885Sdim MakeErrMsg(ErrMsg, "vm_protect max RX failed"); 229245431Sdim return MemoryBlock(); 230218885Sdim } 231218885Sdim 232218885Sdim kr = vm_protect(mach_task_self(), (vm_address_t)pa, 233252723Sdim (vm_size_t)(PageSize*NumPages), 0, 234218885Sdim VM_PROT_READ | VM_PROT_WRITE); 235218885Sdim if (KERN_SUCCESS != kr) { 236218885Sdim MakeErrMsg(ErrMsg, "vm_protect RW failed"); 237245431Sdim return MemoryBlock(); 238218885Sdim } 239218885Sdim#endif 240218885Sdim 241218885Sdim MemoryBlock result; 242218885Sdim result.Address = pa; 243252723Sdim result.Size = NumPages*PageSize; 244218885Sdim 245218885Sdim return result; 246218885Sdim} 247218885Sdim 248245431Sdimbool Memory::ReleaseRWX(MemoryBlock &M, std::string *ErrMsg) { 249218885Sdim if (M.Address == 0 || M.Size == 0) return false; 250218885Sdim if (0 != ::munmap(M.Address, M.Size)) 251218885Sdim return MakeErrMsg(ErrMsg, "Can't release RWX Memory"); 252218885Sdim return false; 253218885Sdim} 254218885Sdim 255245431Sdimbool Memory::setWritable (MemoryBlock &M, std::string *ErrMsg) { 256218885Sdim#if defined(__APPLE__) && defined(__arm__) 257218885Sdim if (M.Address == 0 || M.Size == 0) return false; 258245431Sdim Memory::InvalidateInstructionCache(M.Address, M.Size); 259218885Sdim kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)M.Address, 260218885Sdim (vm_size_t)M.Size, 0, VM_PROT_READ | VM_PROT_WRITE); 261218885Sdim return KERN_SUCCESS == kr; 262218885Sdim#else 263218885Sdim return true; 264218885Sdim#endif 265218885Sdim} 266218885Sdim 267245431Sdimbool Memory::setExecutable (MemoryBlock &M, std::string *ErrMsg) { 268218885Sdim#if defined(__APPLE__) && defined(__arm__) 269218885Sdim if (M.Address == 0 || M.Size == 0) return false; 270245431Sdim Memory::InvalidateInstructionCache(M.Address, M.Size); 271218885Sdim kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)M.Address, 272218885Sdim (vm_size_t)M.Size, 0, VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY); 273218885Sdim return KERN_SUCCESS == kr; 274263509Sdim#elif defined(__arm__) || defined(__aarch64__) 275263509Sdim Memory::InvalidateInstructionCache(M.Address, M.Size); 276263509Sdim return true; 277218885Sdim#else 278221345Sdim return true; 279218885Sdim#endif 280218885Sdim} 281218885Sdim 282245431Sdimbool Memory::setRangeWritable(const void *Addr, size_t Size) { 283218885Sdim#if defined(__APPLE__) && defined(__arm__) 284218885Sdim kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)Addr, 285218885Sdim (vm_size_t)Size, 0, 286218885Sdim VM_PROT_READ | VM_PROT_WRITE); 287218885Sdim return KERN_SUCCESS == kr; 288218885Sdim#else 289218885Sdim return true; 290218885Sdim#endif 291218885Sdim} 292218885Sdim 293245431Sdimbool Memory::setRangeExecutable(const void *Addr, size_t Size) { 294218885Sdim#if defined(__APPLE__) && defined(__arm__) 295218885Sdim kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)Addr, 296218885Sdim (vm_size_t)Size, 0, 297218885Sdim VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY); 298218885Sdim return KERN_SUCCESS == kr; 299218885Sdim#else 300218885Sdim return true; 301218885Sdim#endif 302218885Sdim} 303245431Sdim 304245431Sdim/// InvalidateInstructionCache - Before the JIT can run a block of code 305245431Sdim/// that has been emitted it must invalidate the instruction cache on some 306245431Sdim/// platforms. 307245431Sdimvoid Memory::InvalidateInstructionCache(const void *Addr, 308245431Sdim size_t Len) { 309245431Sdim 310245431Sdim// icache invalidation for PPC and ARM. 311245431Sdim#if defined(__APPLE__) 312245431Sdim 313245431Sdim# if (defined(__POWERPC__) || defined (__ppc__) || \ 314245431Sdim defined(_POWER) || defined(_ARCH_PPC)) || defined(__arm__) 315245431Sdim sys_icache_invalidate(const_cast<void *>(Addr), Len); 316245431Sdim# endif 317245431Sdim 318245431Sdim#else 319245431Sdim 320245431Sdim# if (defined(__POWERPC__) || defined (__ppc__) || \ 321245431Sdim defined(_POWER) || defined(_ARCH_PPC)) && defined(__GNUC__) 322245431Sdim const size_t LineSize = 32; 323245431Sdim 324245431Sdim const intptr_t Mask = ~(LineSize - 1); 325245431Sdim const intptr_t StartLine = ((intptr_t) Addr) & Mask; 326245431Sdim const intptr_t EndLine = ((intptr_t) Addr + Len + LineSize - 1) & Mask; 327245431Sdim 328245431Sdim for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize) 329245431Sdim asm volatile("dcbf 0, %0" : : "r"(Line)); 330245431Sdim asm volatile("sync"); 331245431Sdim 332245431Sdim for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize) 333245431Sdim asm volatile("icbi 0, %0" : : "r"(Line)); 334245431Sdim asm volatile("isync"); 335252723Sdim# elif (defined(__arm__) || defined(__aarch64__)) && defined(__GNUC__) && !defined(__FreeBSD__) 336245431Sdim // FIXME: Can we safely always call this for __GNUC__ everywhere? 337245431Sdim const char *Start = static_cast<const char *>(Addr); 338245431Sdim const char *End = Start + Len; 339245431Sdim __clear_cache(const_cast<char *>(Start), const_cast<char *>(End)); 340245431Sdim# elif defined(__mips__) 341245431Sdim const char *Start = static_cast<const char *>(Addr); 342252723Sdim# if defined(ANDROID) 343252723Sdim // The declaration of "cacheflush" in Android bionic: 344252723Sdim // extern int cacheflush(long start, long end, long flags); 345252723Sdim const char *End = Start + Len; 346252723Sdim long LStart = reinterpret_cast<long>(const_cast<char *>(Start)); 347252723Sdim long LEnd = reinterpret_cast<long>(const_cast<char *>(End)); 348252723Sdim cacheflush(LStart, LEnd, BCACHE); 349252723Sdim# else 350245431Sdim cacheflush(const_cast<char *>(Start), Len, BCACHE); 351252723Sdim# endif 352245431Sdim# endif 353245431Sdim 354245431Sdim#endif // end apple 355245431Sdim 356245431Sdim ValgrindDiscardTranslations(Addr, Len); 357245431Sdim} 358245431Sdim 359245431Sdim} // namespace sys 360245431Sdim} // namespace llvm 361