1238901Sandrew//===-- tsan_platform.h -----------------------------------------*- C++ -*-===// 2238901Sandrew// 3238901Sandrew// The LLVM Compiler Infrastructure 4238901Sandrew// 5238901Sandrew// This file is distributed under the University of Illinois Open Source 6238901Sandrew// License. See LICENSE.TXT for details. 7238901Sandrew// 8238901Sandrew//===----------------------------------------------------------------------===// 9238901Sandrew// 10238901Sandrew// This file is a part of ThreadSanitizer (TSan), a race detector. 11238901Sandrew// 12238901Sandrew// Platform-specific code. 13238901Sandrew//===----------------------------------------------------------------------===// 14238901Sandrew 15276789Sdim#ifndef TSAN_PLATFORM_H 16276789Sdim#define TSAN_PLATFORM_H 17276789Sdim 18276789Sdim#if !defined(__LP64__) && !defined(_WIN64) 19276789Sdim# error "Only 64-bit is supported" 20276789Sdim#endif 21276789Sdim 22276789Sdim#include "tsan_defs.h" 23276789Sdim#include "tsan_trace.h" 24276789Sdim 25276789Sdimnamespace __tsan { 26276789Sdim 27276789Sdim#if !defined(SANITIZER_GO) 28276789Sdim 29280031Sdim#if defined(__x86_64__) 30245614Sandrew/* 31280031SdimC/C++ on linux/x86_64 and freebsd/x86_64 32276789Sdim0000 0000 1000 - 0100 0000 0000: main binary and/or MAP_32BIT mappings 33276789Sdim0100 0000 0000 - 0200 0000 0000: - 34276789Sdim0200 0000 0000 - 1000 0000 0000: shadow 35276789Sdim1000 0000 0000 - 3000 0000 0000: - 36276789Sdim3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) 37276789Sdim4000 0000 0000 - 6000 0000 0000: - 38245614Sandrew6000 0000 0000 - 6200 0000 0000: traces 39245614Sandrew6200 0000 0000 - 7d00 0000 0000: - 40245614Sandrew7d00 0000 0000 - 7e00 0000 0000: heap 41276789Sdim7e00 0000 0000 - 7e80 0000 0000: - 42276789Sdim7e80 0000 0000 - 8000 0000 0000: modules and main thread stack 43276789Sdim*/ 44296417Sdimstruct Mapping { 45296417Sdim static const uptr kMetaShadowBeg = 0x300000000000ull; 46296417Sdim static const uptr kMetaShadowEnd = 0x400000000000ull; 47296417Sdim static const uptr kTraceMemBeg = 0x600000000000ull; 48296417Sdim static const uptr kTraceMemEnd = 0x620000000000ull; 49296417Sdim static const uptr kShadowBeg = 0x020000000000ull; 50296417Sdim static const uptr kShadowEnd = 0x100000000000ull; 51296417Sdim static const uptr kHeapMemBeg = 0x7d0000000000ull; 52296417Sdim static const uptr kHeapMemEnd = 0x7e0000000000ull; 53296417Sdim static const uptr kLoAppMemBeg = 0x000000001000ull; 54296417Sdim static const uptr kLoAppMemEnd = 0x010000000000ull; 55296417Sdim static const uptr kHiAppMemBeg = 0x7e8000000000ull; 56296417Sdim static const uptr kHiAppMemEnd = 0x800000000000ull; 57296417Sdim static const uptr kAppMemMsk = 0x7c0000000000ull; 58296417Sdim static const uptr kAppMemXor = 0x020000000000ull; 59296417Sdim static const uptr kVdsoBeg = 0xf000000000000000ull; 60296417Sdim}; 61280031Sdim#elif defined(__mips64) 62280031Sdim/* 63280031SdimC/C++ on linux/mips64 64280031Sdim0100 0000 00 - 0200 0000 00: main binary 65280031Sdim0200 0000 00 - 1400 0000 00: - 66280031Sdim1400 0000 00 - 2400 0000 00: shadow 67280031Sdim2400 0000 00 - 3000 0000 00: - 68280031Sdim3000 0000 00 - 4000 0000 00: metainfo (memory blocks and sync objects) 69280031Sdim4000 0000 00 - 6000 0000 00: - 70280031Sdim6000 0000 00 - 6200 0000 00: traces 71280031Sdim6200 0000 00 - fe00 0000 00: - 72280031Sdimfe00 0000 00 - ff00 0000 00: heap 73280031Sdimff00 0000 00 - ff80 0000 00: - 74280031Sdimff80 0000 00 - ffff ffff ff: modules and main thread stack 75280031Sdim*/ 76296417Sdimstruct Mapping { 77296417Sdim static const uptr kMetaShadowBeg = 0x3000000000ull; 78296417Sdim static const uptr kMetaShadowEnd = 0x4000000000ull; 79296417Sdim static const uptr kTraceMemBeg = 0x6000000000ull; 80296417Sdim static const uptr kTraceMemEnd = 0x6200000000ull; 81296417Sdim static const uptr kShadowBeg = 0x1400000000ull; 82296417Sdim static const uptr kShadowEnd = 0x2400000000ull; 83296417Sdim static const uptr kHeapMemBeg = 0xfe00000000ull; 84296417Sdim static const uptr kHeapMemEnd = 0xff00000000ull; 85296417Sdim static const uptr kLoAppMemBeg = 0x0100000000ull; 86296417Sdim static const uptr kLoAppMemEnd = 0x0200000000ull; 87296417Sdim static const uptr kHiAppMemBeg = 0xff80000000ull; 88296417Sdim static const uptr kHiAppMemEnd = 0xffffffffffull; 89296417Sdim static const uptr kAppMemMsk = 0xfc00000000ull; 90296417Sdim static const uptr kAppMemXor = 0x0400000000ull; 91296417Sdim static const uptr kVdsoBeg = 0xfffff00000ull; 92296417Sdim}; 93296417Sdim#elif defined(__aarch64__) 94296417Sdim// AArch64 supports multiple VMA which leads to multiple address transformation 95296417Sdim// functions. To support these multiple VMAS transformations and mappings TSAN 96296417Sdim// runtime for AArch64 uses an external memory read (vmaSize) to select which 97296417Sdim// mapping to use. Although slower, it make a same instrumented binary run on 98296417Sdim// multiple kernels. 99245614Sandrew 100296417Sdim/* 101296417SdimC/C++ on linux/aarch64 (39-bit VMA) 102296417Sdim0000 0010 00 - 0100 0000 00: main binary 103296417Sdim0100 0000 00 - 0800 0000 00: - 104296417Sdim0800 0000 00 - 2000 0000 00: shadow memory 105296417Sdim2000 0000 00 - 3100 0000 00: - 106296417Sdim3100 0000 00 - 3400 0000 00: metainfo 107296417Sdim3400 0000 00 - 5500 0000 00: - 108296417Sdim5500 0000 00 - 5600 0000 00: main binary (PIE) 109296417Sdim5600 0000 00 - 6000 0000 00: - 110296417Sdim6000 0000 00 - 6200 0000 00: traces 111296417Sdim6200 0000 00 - 7d00 0000 00: - 112296417Sdim7c00 0000 00 - 7d00 0000 00: heap 113296417Sdim7d00 0000 00 - 7fff ffff ff: modules and main thread stack 114296417Sdim*/ 115296417Sdimstruct Mapping39 { 116296417Sdim static const uptr kLoAppMemBeg = 0x0000001000ull; 117296417Sdim static const uptr kLoAppMemEnd = 0x0100000000ull; 118296417Sdim static const uptr kShadowBeg = 0x0800000000ull; 119296417Sdim static const uptr kShadowEnd = 0x2000000000ull; 120296417Sdim static const uptr kMetaShadowBeg = 0x3100000000ull; 121296417Sdim static const uptr kMetaShadowEnd = 0x3400000000ull; 122296417Sdim static const uptr kMidAppMemBeg = 0x5500000000ull; 123296417Sdim static const uptr kMidAppMemEnd = 0x5600000000ull; 124296417Sdim static const uptr kMidShadowOff = 0x5000000000ull; 125296417Sdim static const uptr kTraceMemBeg = 0x6000000000ull; 126296417Sdim static const uptr kTraceMemEnd = 0x6200000000ull; 127296417Sdim static const uptr kHeapMemBeg = 0x7c00000000ull; 128296417Sdim static const uptr kHeapMemEnd = 0x7d00000000ull; 129296417Sdim static const uptr kHiAppMemBeg = 0x7e00000000ull; 130296417Sdim static const uptr kHiAppMemEnd = 0x7fffffffffull; 131296417Sdim static const uptr kAppMemMsk = 0x7800000000ull; 132296417Sdim static const uptr kAppMemXor = 0x0200000000ull; 133296417Sdim static const uptr kVdsoBeg = 0x7f00000000ull; 134296417Sdim}; 135276789Sdim 136296417Sdim/* 137296417SdimC/C++ on linux/aarch64 (42-bit VMA) 138296417Sdim00000 0010 00 - 01000 0000 00: main binary 139296417Sdim01000 0000 00 - 10000 0000 00: - 140296417Sdim10000 0000 00 - 20000 0000 00: shadow memory 141296417Sdim20000 0000 00 - 26000 0000 00: - 142296417Sdim26000 0000 00 - 28000 0000 00: metainfo 143296417Sdim28000 0000 00 - 2aa00 0000 00: - 144296417Sdim2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) 145296417Sdim2ab00 0000 00 - 36200 0000 00: - 146296417Sdim36200 0000 00 - 36240 0000 00: traces 147296417Sdim36240 0000 00 - 3e000 0000 00: - 148296417Sdim3e000 0000 00 - 3f000 0000 00: heap 149296417Sdim3f000 0000 00 - 3ffff ffff ff: modules and main thread stack 150296417Sdim*/ 151296417Sdimstruct Mapping42 { 152296417Sdim static const uptr kLoAppMemBeg = 0x00000001000ull; 153296417Sdim static const uptr kLoAppMemEnd = 0x01000000000ull; 154296417Sdim static const uptr kShadowBeg = 0x10000000000ull; 155296417Sdim static const uptr kShadowEnd = 0x20000000000ull; 156296417Sdim static const uptr kMetaShadowBeg = 0x26000000000ull; 157296417Sdim static const uptr kMetaShadowEnd = 0x28000000000ull; 158296417Sdim static const uptr kMidAppMemBeg = 0x2aa00000000ull; 159296417Sdim static const uptr kMidAppMemEnd = 0x2ab00000000ull; 160296417Sdim static const uptr kMidShadowOff = 0x28000000000ull; 161296417Sdim static const uptr kTraceMemBeg = 0x36200000000ull; 162296417Sdim static const uptr kTraceMemEnd = 0x36400000000ull; 163296417Sdim static const uptr kHeapMemBeg = 0x3e000000000ull; 164296417Sdim static const uptr kHeapMemEnd = 0x3f000000000ull; 165296417Sdim static const uptr kHiAppMemBeg = 0x3f000000000ull; 166296417Sdim static const uptr kHiAppMemEnd = 0x3ffffffffffull; 167296417Sdim static const uptr kAppMemMsk = 0x3c000000000ull; 168296417Sdim static const uptr kAppMemXor = 0x04000000000ull; 169296417Sdim static const uptr kVdsoBeg = 0x37f00000000ull; 170296417Sdim}; 171276789Sdim 172296417Sdim// Indicates the runtime will define the memory regions at runtime. 173296417Sdim#define TSAN_RUNTIME_VMA 1 174296417Sdim// Indicates that mapping defines a mid range memory segment. 175296417Sdim#define TSAN_MID_APP_RANGE 1 176296417Sdim#elif defined(__powerpc64__) 177296417Sdim// PPC64 supports multiple VMA which leads to multiple address transformation 178296417Sdim// functions. To support these multiple VMAS transformations and mappings TSAN 179296417Sdim// runtime for PPC64 uses an external memory read (vmaSize) to select which 180296417Sdim// mapping to use. Although slower, it make a same instrumented binary run on 181296417Sdim// multiple kernels. 182276789Sdim 183296417Sdim/* 184296417SdimC/C++ on linux/powerpc64 (44-bit VMA) 185296417Sdim0000 0000 0100 - 0001 0000 0000: main binary 186296417Sdim0001 0000 0000 - 0001 0000 0000: - 187296417Sdim0001 0000 0000 - 0b00 0000 0000: shadow 188296417Sdim0b00 0000 0000 - 0b00 0000 0000: - 189296417Sdim0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects) 190296417Sdim0d00 0000 0000 - 0d00 0000 0000: - 191296417Sdim0d00 0000 0000 - 0f00 0000 0000: traces 192296417Sdim0f00 0000 0000 - 0f00 0000 0000: - 193296417Sdim0f00 0000 0000 - 0f50 0000 0000: heap 194296417Sdim0f50 0000 0000 - 0f60 0000 0000: - 195296417Sdim0f60 0000 0000 - 1000 0000 0000: modules and main thread stack 196296417Sdim*/ 197296417Sdimstruct Mapping44 { 198296417Sdim static const uptr kMetaShadowBeg = 0x0b0000000000ull; 199296417Sdim static const uptr kMetaShadowEnd = 0x0d0000000000ull; 200296417Sdim static const uptr kTraceMemBeg = 0x0d0000000000ull; 201296417Sdim static const uptr kTraceMemEnd = 0x0f0000000000ull; 202296417Sdim static const uptr kShadowBeg = 0x000100000000ull; 203296417Sdim static const uptr kShadowEnd = 0x0b0000000000ull; 204296417Sdim static const uptr kLoAppMemBeg = 0x000000000100ull; 205296417Sdim static const uptr kLoAppMemEnd = 0x000100000000ull; 206296417Sdim static const uptr kHeapMemBeg = 0x0f0000000000ull; 207296417Sdim static const uptr kHeapMemEnd = 0x0f5000000000ull; 208296417Sdim static const uptr kHiAppMemBeg = 0x0f6000000000ull; 209296417Sdim static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits 210296417Sdim static const uptr kAppMemMsk = 0x0f0000000000ull; 211296417Sdim static const uptr kAppMemXor = 0x002100000000ull; 212296417Sdim static const uptr kVdsoBeg = 0x3c0000000000000ull; 213296417Sdim}; 214276789Sdim 215296417Sdim/* 216296417SdimC/C++ on linux/powerpc64 (46-bit VMA) 217296417Sdim0000 0000 1000 - 0100 0000 0000: main binary 218296417Sdim0100 0000 0000 - 0200 0000 0000: - 219296417Sdim0100 0000 0000 - 1000 0000 0000: shadow 220296417Sdim1000 0000 0000 - 1000 0000 0000: - 221296417Sdim1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) 222296417Sdim2000 0000 0000 - 2000 0000 0000: - 223296417Sdim2000 0000 0000 - 2200 0000 0000: traces 224296417Sdim2200 0000 0000 - 3d00 0000 0000: - 225296417Sdim3d00 0000 0000 - 3e00 0000 0000: heap 226296417Sdim3e00 0000 0000 - 3e80 0000 0000: - 227296417Sdim3e80 0000 0000 - 4000 0000 0000: modules and main thread stack 228296417Sdim*/ 229296417Sdimstruct Mapping46 { 230296417Sdim static const uptr kMetaShadowBeg = 0x100000000000ull; 231296417Sdim static const uptr kMetaShadowEnd = 0x200000000000ull; 232296417Sdim static const uptr kTraceMemBeg = 0x200000000000ull; 233296417Sdim static const uptr kTraceMemEnd = 0x220000000000ull; 234296417Sdim static const uptr kShadowBeg = 0x010000000000ull; 235296417Sdim static const uptr kShadowEnd = 0x100000000000ull; 236296417Sdim static const uptr kHeapMemBeg = 0x3d0000000000ull; 237296417Sdim static const uptr kHeapMemEnd = 0x3e0000000000ull; 238296417Sdim static const uptr kLoAppMemBeg = 0x000000001000ull; 239296417Sdim static const uptr kLoAppMemEnd = 0x010000000000ull; 240296417Sdim static const uptr kHiAppMemBeg = 0x3e8000000000ull; 241296417Sdim static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits 242296417Sdim static const uptr kAppMemMsk = 0x3c0000000000ull; 243296417Sdim static const uptr kAppMemXor = 0x020000000000ull; 244296417Sdim static const uptr kVdsoBeg = 0x7800000000000000ull; 245296417Sdim}; 246276789Sdim 247296417Sdim// Indicates the runtime will define the memory regions at runtime. 248296417Sdim#define TSAN_RUNTIME_VMA 1 249296417Sdim#endif 250276789Sdim 251276789Sdim#elif defined(SANITIZER_GO) && !SANITIZER_WINDOWS 252276789Sdim 253276789Sdim/* Go on linux, darwin and freebsd 254276789Sdim0000 0000 1000 - 0000 1000 0000: executable 255276789Sdim0000 1000 0000 - 00c0 0000 0000: - 256251034Sed00c0 0000 0000 - 00e0 0000 0000: heap 257276789Sdim00e0 0000 0000 - 2000 0000 0000: - 258276789Sdim2000 0000 0000 - 2380 0000 0000: shadow 259276789Sdim2380 0000 0000 - 3000 0000 0000: - 260276789Sdim3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) 261276789Sdim4000 0000 0000 - 6000 0000 0000: - 262245614Sandrew6000 0000 0000 - 6200 0000 0000: traces 263276789Sdim6200 0000 0000 - 8000 0000 0000: - 264276789Sdim*/ 265245614Sandrew 266296417Sdimstruct Mapping { 267296417Sdim static const uptr kMetaShadowBeg = 0x300000000000ull; 268296417Sdim static const uptr kMetaShadowEnd = 0x400000000000ull; 269296417Sdim static const uptr kTraceMemBeg = 0x600000000000ull; 270296417Sdim static const uptr kTraceMemEnd = 0x620000000000ull; 271296417Sdim static const uptr kShadowBeg = 0x200000000000ull; 272296417Sdim static const uptr kShadowEnd = 0x238000000000ull; 273296417Sdim static const uptr kAppMemBeg = 0x000000001000ull; 274296417Sdim static const uptr kAppMemEnd = 0x00e000000000ull; 275296417Sdim}; 276276789Sdim 277296417Sdim#elif defined(SANITIZER_GO) && SANITIZER_WINDOWS 278296417Sdim 279296417Sdim/* Go on windows 280296417Sdim0000 0000 1000 - 0000 1000 0000: executable 281296417Sdim0000 1000 0000 - 00f8 0000 0000: - 282296417Sdim00c0 0000 0000 - 00e0 0000 0000: heap 283296417Sdim00e0 0000 0000 - 0100 0000 0000: - 284296417Sdim0100 0000 0000 - 0500 0000 0000: shadow 285296417Sdim0500 0000 0000 - 0560 0000 0000: - 286296417Sdim0560 0000 0000 - 0760 0000 0000: traces 287296417Sdim0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects) 288296417Sdim07d0 0000 0000 - 8000 0000 0000: - 289296417Sdim*/ 290296417Sdim 291296417Sdimstruct Mapping { 292296417Sdim static const uptr kMetaShadowBeg = 0x076000000000ull; 293296417Sdim static const uptr kMetaShadowEnd = 0x07d000000000ull; 294296417Sdim static const uptr kTraceMemBeg = 0x056000000000ull; 295296417Sdim static const uptr kTraceMemEnd = 0x076000000000ull; 296296417Sdim static const uptr kShadowBeg = 0x010000000000ull; 297296417Sdim static const uptr kShadowEnd = 0x050000000000ull; 298296417Sdim static const uptr kAppMemBeg = 0x000000001000ull; 299296417Sdim static const uptr kAppMemEnd = 0x00e000000000ull; 300296417Sdim} 301296417Sdim 302296417Sdim#else 303296417Sdim# error "Unknown platform" 304296417Sdim#endif 305296417Sdim 306296417Sdim 307296417Sdim#ifdef TSAN_RUNTIME_VMA 308296417Sdimextern uptr vmaSize; 309296417Sdim#endif 310296417Sdim 311296417Sdim 312296417Sdimenum MappingType { 313296417Sdim MAPPING_LO_APP_BEG, 314296417Sdim MAPPING_LO_APP_END, 315296417Sdim MAPPING_HI_APP_BEG, 316296417Sdim MAPPING_HI_APP_END, 317296417Sdim#ifdef TSAN_MID_APP_RANGE 318296417Sdim MAPPING_MID_APP_BEG, 319296417Sdim MAPPING_MID_APP_END, 320296417Sdim#endif 321296417Sdim MAPPING_HEAP_BEG, 322296417Sdim MAPPING_HEAP_END, 323296417Sdim MAPPING_APP_BEG, 324296417Sdim MAPPING_APP_END, 325296417Sdim MAPPING_SHADOW_BEG, 326296417Sdim MAPPING_SHADOW_END, 327296417Sdim MAPPING_META_SHADOW_BEG, 328296417Sdim MAPPING_META_SHADOW_END, 329296417Sdim MAPPING_TRACE_BEG, 330296417Sdim MAPPING_TRACE_END, 331296417Sdim MAPPING_VDSO_BEG, 332296417Sdim}; 333296417Sdim 334296417Sdimtemplate<typename Mapping, int Type> 335296417Sdimuptr MappingImpl(void) { 336296417Sdim switch (Type) { 337296417Sdim#ifndef SANITIZER_GO 338296417Sdim case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg; 339296417Sdim case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd; 340296417Sdim# ifdef TSAN_MID_APP_RANGE 341296417Sdim case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg; 342296417Sdim case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd; 343296417Sdim# endif 344296417Sdim case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg; 345296417Sdim case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd; 346296417Sdim case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg; 347296417Sdim case MAPPING_HEAP_END: return Mapping::kHeapMemEnd; 348296417Sdim case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg; 349296417Sdim#else 350296417Sdim case MAPPING_APP_BEG: return Mapping::kAppMemBeg; 351296417Sdim case MAPPING_APP_END: return Mapping::kAppMemEnd; 352296417Sdim#endif 353296417Sdim case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg; 354296417Sdim case MAPPING_SHADOW_END: return Mapping::kShadowEnd; 355296417Sdim case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg; 356296417Sdim case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd; 357296417Sdim case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg; 358296417Sdim case MAPPING_TRACE_END: return Mapping::kTraceMemEnd; 359296417Sdim } 360296417Sdim} 361296417Sdim 362296417Sdimtemplate<int Type> 363296417Sdimuptr MappingArchImpl(void) { 364296417Sdim#ifdef __aarch64__ 365296417Sdim if (vmaSize == 39) 366296417Sdim return MappingImpl<Mapping39, Type>(); 367296417Sdim else 368296417Sdim return MappingImpl<Mapping42, Type>(); 369296417Sdim DCHECK(0); 370296417Sdim#elif defined(__powerpc64__) 371296417Sdim if (vmaSize == 44) 372296417Sdim return MappingImpl<Mapping44, Type>(); 373296417Sdim else 374296417Sdim return MappingImpl<Mapping46, Type>(); 375296417Sdim DCHECK(0); 376296417Sdim#else 377296417Sdim return MappingImpl<Mapping, Type>(); 378296417Sdim#endif 379296417Sdim} 380296417Sdim 381296417Sdim#ifndef SANITIZER_GO 382276789SdimALWAYS_INLINE 383296417Sdimuptr LoAppMemBeg(void) { 384296417Sdim return MappingArchImpl<MAPPING_LO_APP_BEG>(); 385276789Sdim} 386296417SdimALWAYS_INLINE 387296417Sdimuptr LoAppMemEnd(void) { 388296417Sdim return MappingArchImpl<MAPPING_LO_APP_END>(); 389296417Sdim} 390276789Sdim 391296417Sdim#ifdef TSAN_MID_APP_RANGE 392276789SdimALWAYS_INLINE 393296417Sdimuptr MidAppMemBeg(void) { 394296417Sdim return MappingArchImpl<MAPPING_MID_APP_BEG>(); 395276789Sdim} 396296417SdimALWAYS_INLINE 397296417Sdimuptr MidAppMemEnd(void) { 398296417Sdim return MappingArchImpl<MAPPING_MID_APP_END>(); 399296417Sdim} 400296417Sdim#endif 401276789Sdim 402276789SdimALWAYS_INLINE 403296417Sdimuptr HeapMemBeg(void) { 404296417Sdim return MappingArchImpl<MAPPING_HEAP_BEG>(); 405276789Sdim} 406296417SdimALWAYS_INLINE 407296417Sdimuptr HeapMemEnd(void) { 408296417Sdim return MappingArchImpl<MAPPING_HEAP_END>(); 409296417Sdim} 410276789Sdim 411276789SdimALWAYS_INLINE 412296417Sdimuptr HiAppMemBeg(void) { 413296417Sdim return MappingArchImpl<MAPPING_HI_APP_BEG>(); 414276789Sdim} 415296417SdimALWAYS_INLINE 416296417Sdimuptr HiAppMemEnd(void) { 417296417Sdim return MappingArchImpl<MAPPING_HI_APP_END>(); 418296417Sdim} 419276789Sdim 420276789SdimALWAYS_INLINE 421296417Sdimuptr VdsoBeg(void) { 422296417Sdim return MappingArchImpl<MAPPING_VDSO_BEG>(); 423276789Sdim} 424276789Sdim 425296417Sdim#else 426296417Sdim 427276789SdimALWAYS_INLINE 428296417Sdimuptr AppMemBeg(void) { 429296417Sdim return MappingArchImpl<MAPPING_APP_BEG>(); 430276789Sdim} 431296417SdimALWAYS_INLINE 432296417Sdimuptr AppMemEnd(void) { 433296417Sdim return MappingArchImpl<MAPPING_APP_END>(); 434296417Sdim} 435276789Sdim 436296417Sdim#endif 437276789Sdim 438296417Sdimstatic inline 439296417Sdimbool GetUserRegion(int i, uptr *start, uptr *end) { 440296417Sdim switch (i) { 441296417Sdim default: 442296417Sdim return false; 443296417Sdim#ifndef SANITIZER_GO 444296417Sdim case 0: 445296417Sdim *start = LoAppMemBeg(); 446296417Sdim *end = LoAppMemEnd(); 447296417Sdim return true; 448296417Sdim case 1: 449296417Sdim *start = HiAppMemBeg(); 450296417Sdim *end = HiAppMemEnd(); 451296417Sdim return true; 452296417Sdim case 2: 453296417Sdim *start = HeapMemBeg(); 454296417Sdim *end = HeapMemEnd(); 455296417Sdim return true; 456296417Sdim# ifdef TSAN_MID_APP_RANGE 457296417Sdim case 3: 458296417Sdim *start = MidAppMemBeg(); 459296417Sdim *end = MidAppMemEnd(); 460296417Sdim return true; 461296417Sdim# endif 462296417Sdim#else 463296417Sdim case 0: 464296417Sdim *start = AppMemBeg(); 465296417Sdim *end = AppMemEnd(); 466296417Sdim return true; 467296417Sdim#endif 468296417Sdim } 469296417Sdim} 470276789Sdim 471296417SdimALWAYS_INLINE 472296417Sdimuptr ShadowBeg(void) { 473296417Sdim return MappingArchImpl<MAPPING_SHADOW_BEG>(); 474296417Sdim} 475296417SdimALWAYS_INLINE 476296417Sdimuptr ShadowEnd(void) { 477296417Sdim return MappingArchImpl<MAPPING_SHADOW_END>(); 478296417Sdim} 479245614Sandrew 480296417SdimALWAYS_INLINE 481296417Sdimuptr MetaShadowBeg(void) { 482296417Sdim return MappingArchImpl<MAPPING_META_SHADOW_BEG>(); 483296417Sdim} 484296417SdimALWAYS_INLINE 485296417Sdimuptr MetaShadowEnd(void) { 486296417Sdim return MappingArchImpl<MAPPING_META_SHADOW_END>(); 487296417Sdim} 488238901Sandrew 489276789SdimALWAYS_INLINE 490296417Sdimuptr TraceMemBeg(void) { 491296417Sdim return MappingArchImpl<MAPPING_TRACE_BEG>(); 492296417Sdim} 493296417SdimALWAYS_INLINE 494296417Sdimuptr TraceMemEnd(void) { 495296417Sdim return MappingArchImpl<MAPPING_TRACE_END>(); 496296417Sdim} 497296417Sdim 498296417Sdim 499296417Sdimtemplate<typename Mapping> 500296417Sdimbool IsAppMemImpl(uptr mem) { 501296417Sdim#ifndef SANITIZER_GO 502296417Sdim return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || 503296417Sdim# ifdef TSAN_MID_APP_RANGE 504296417Sdim (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || 505296417Sdim# endif 506296417Sdim (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || 507296417Sdim (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); 508296417Sdim#else 509296417Sdim return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd; 510296417Sdim#endif 511296417Sdim} 512296417Sdim 513296417SdimALWAYS_INLINE 514276789Sdimbool IsAppMem(uptr mem) { 515296417Sdim#ifdef __aarch64__ 516296417Sdim if (vmaSize == 39) 517296417Sdim return IsAppMemImpl<Mapping39>(mem); 518296417Sdim else 519296417Sdim return IsAppMemImpl<Mapping42>(mem); 520296417Sdim DCHECK(0); 521296417Sdim#elif defined(__powerpc64__) 522296417Sdim if (vmaSize == 44) 523296417Sdim return IsAppMemImpl<Mapping44>(mem); 524296417Sdim else 525296417Sdim return IsAppMemImpl<Mapping46>(mem); 526296417Sdim DCHECK(0); 527296417Sdim#else 528296417Sdim return IsAppMemImpl<Mapping>(mem); 529296417Sdim#endif 530276789Sdim} 531238901Sandrew 532296417Sdim 533296417Sdimtemplate<typename Mapping> 534296417Sdimbool IsShadowMemImpl(uptr mem) { 535296417Sdim return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd; 536296417Sdim} 537296417Sdim 538276789SdimALWAYS_INLINE 539276789Sdimbool IsShadowMem(uptr mem) { 540296417Sdim#ifdef __aarch64__ 541296417Sdim if (vmaSize == 39) 542296417Sdim return IsShadowMemImpl<Mapping39>(mem); 543296417Sdim else 544296417Sdim return IsShadowMemImpl<Mapping42>(mem); 545296417Sdim DCHECK(0); 546296417Sdim#elif defined(__powerpc64__) 547296417Sdim if (vmaSize == 44) 548296417Sdim return IsShadowMemImpl<Mapping44>(mem); 549296417Sdim else 550296417Sdim return IsShadowMemImpl<Mapping46>(mem); 551296417Sdim DCHECK(0); 552296417Sdim#else 553296417Sdim return IsShadowMemImpl<Mapping>(mem); 554296417Sdim#endif 555276789Sdim} 556238901Sandrew 557296417Sdim 558296417Sdimtemplate<typename Mapping> 559296417Sdimbool IsMetaMemImpl(uptr mem) { 560296417Sdim return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd; 561296417Sdim} 562296417Sdim 563276789SdimALWAYS_INLINE 564276789Sdimbool IsMetaMem(uptr mem) { 565296417Sdim#ifdef __aarch64__ 566296417Sdim if (vmaSize == 39) 567296417Sdim return IsMetaMemImpl<Mapping39>(mem); 568296417Sdim else 569296417Sdim return IsMetaMemImpl<Mapping42>(mem); 570296417Sdim DCHECK(0); 571296417Sdim#elif defined(__powerpc64__) 572296417Sdim if (vmaSize == 44) 573296417Sdim return IsMetaMemImpl<Mapping44>(mem); 574296417Sdim else 575296417Sdim return IsMetaMemImpl<Mapping46>(mem); 576296417Sdim DCHECK(0); 577296417Sdim#else 578296417Sdim return IsMetaMemImpl<Mapping>(mem); 579296417Sdim#endif 580276789Sdim} 581238901Sandrew 582296417Sdim 583296417Sdimtemplate<typename Mapping> 584296417Sdimuptr MemToShadowImpl(uptr x) { 585296417Sdim DCHECK(IsAppMem(x)); 586296417Sdim#ifndef SANITIZER_GO 587296417Sdim return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1))) 588296417Sdim ^ Mapping::kAppMemXor) * kShadowCnt; 589296417Sdim#else 590296417Sdim return ((x & ~(kShadowCell - 1)) * kShadowCnt) | Mapping::kShadowBeg; 591296417Sdim#endif 592296417Sdim} 593296417Sdim 594276789SdimALWAYS_INLINE 595276789Sdimuptr MemToShadow(uptr x) { 596296417Sdim#ifdef __aarch64__ 597296417Sdim if (vmaSize == 39) 598296417Sdim return MemToShadowImpl<Mapping39>(x); 599296417Sdim else 600296417Sdim return MemToShadowImpl<Mapping42>(x); 601296417Sdim DCHECK(0); 602296417Sdim#elif defined(__powerpc64__) 603296417Sdim if (vmaSize == 44) 604296417Sdim return MemToShadowImpl<Mapping44>(x); 605296417Sdim else 606296417Sdim return MemToShadowImpl<Mapping46>(x); 607296417Sdim DCHECK(0); 608296417Sdim#else 609296417Sdim return MemToShadowImpl<Mapping>(x); 610296417Sdim#endif 611238901Sandrew} 612238901Sandrew 613296417Sdim 614296417Sdimtemplate<typename Mapping> 615296417Sdimu32 *MemToMetaImpl(uptr x) { 616276789Sdim DCHECK(IsAppMem(x)); 617296417Sdim#ifndef SANITIZER_GO 618296417Sdim return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1))) 619296417Sdim ^ Mapping::kAppMemXor) / kMetaShadowCell * kMetaShadowSize) 620296417Sdim | Mapping::kMetaShadowBeg); 621296417Sdim#else 622276789Sdim return (u32*)(((x & ~(kMetaShadowCell - 1)) / \ 623296417Sdim kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg); 624296417Sdim#endif 625238901Sandrew} 626238901Sandrew 627276789SdimALWAYS_INLINE 628296417Sdimu32 *MemToMeta(uptr x) { 629296417Sdim#ifdef __aarch64__ 630296417Sdim if (vmaSize == 39) 631296417Sdim return MemToMetaImpl<Mapping39>(x); 632296417Sdim else 633296417Sdim return MemToMetaImpl<Mapping42>(x); 634296417Sdim DCHECK(0); 635296417Sdim#elif defined(__powerpc64__) 636296417Sdim if (vmaSize == 44) 637296417Sdim return MemToMetaImpl<Mapping44>(x); 638296417Sdim else 639296417Sdim return MemToMetaImpl<Mapping46>(x); 640296417Sdim DCHECK(0); 641296417Sdim#else 642296417Sdim return MemToMetaImpl<Mapping>(x); 643296417Sdim#endif 644238901Sandrew} 645238901Sandrew 646276789Sdim 647296417Sdimtemplate<typename Mapping> 648296417Sdimuptr ShadowToMemImpl(uptr s) { 649296417Sdim DCHECK(IsShadowMem(s)); 650296417Sdim#ifndef SANITIZER_GO 651296417Sdim if (s >= MemToShadow(Mapping::kLoAppMemBeg) 652296417Sdim && s <= MemToShadow(Mapping::kLoAppMemEnd - 1)) 653296417Sdim return (s / kShadowCnt) ^ Mapping::kAppMemXor; 654296417Sdim# ifdef TSAN_MID_APP_RANGE 655296417Sdim if (s >= MemToShadow(Mapping::kMidAppMemBeg) 656296417Sdim && s <= MemToShadow(Mapping::kMidAppMemEnd - 1)) 657296417Sdim return ((s / kShadowCnt) ^ Mapping::kAppMemXor) + Mapping::kMidShadowOff; 658296417Sdim# endif 659296417Sdim else 660296417Sdim return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk; 661238901Sandrew#else 662296417Sdim# ifndef SANITIZER_WINDOWS 663296417Sdim return (s & ~Mapping::kShadowBeg) / kShadowCnt; 664296417Sdim# else 665296417Sdim // FIXME(dvyukov): this is most likely wrong as the mapping is not bijection. 666296417Sdim return (s - Mapping::kShadowBeg) / kShadowCnt; 667296417Sdim# endif // SANITIZER_WINDOWS 668238901Sandrew#endif 669296417Sdim} 670238901Sandrew 671296417SdimALWAYS_INLINE 672296417Sdimuptr ShadowToMem(uptr s) { 673296417Sdim#ifdef __aarch64__ 674296417Sdim if (vmaSize == 39) 675296417Sdim return ShadowToMemImpl<Mapping39>(s); 676296417Sdim else 677296417Sdim return ShadowToMemImpl<Mapping42>(s); 678296417Sdim DCHECK(0); 679296417Sdim#elif defined(__powerpc64__) 680296417Sdim if (vmaSize == 44) 681296417Sdim return ShadowToMemImpl<Mapping44>(s); 682296417Sdim else 683296417Sdim return ShadowToMemImpl<Mapping46>(s); 684296417Sdim DCHECK(0); 685296417Sdim#else 686296417Sdim return ShadowToMemImpl<Mapping>(s); 687296417Sdim#endif 688296417Sdim} 689296417Sdim 690296417Sdim 691296417Sdim 692274201Sdim// The additional page is to catch shadow stack overflow as paging fault. 693276789Sdim// Windows wants 64K alignment for mmaps. 694276789Sdimconst uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) 695276789Sdim + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1); 696274201Sdim 697296417Sdimtemplate<typename Mapping> 698296417Sdimuptr GetThreadTraceImpl(int tid) { 699296417Sdim uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize; 700296417Sdim DCHECK_LT(p, Mapping::kTraceMemEnd); 701245614Sandrew return p; 702245614Sandrew} 703238901Sandrew 704296417SdimALWAYS_INLINE 705296417Sdimuptr GetThreadTrace(int tid) { 706296417Sdim#ifdef __aarch64__ 707296417Sdim if (vmaSize == 39) 708296417Sdim return GetThreadTraceImpl<Mapping39>(tid); 709296417Sdim else 710296417Sdim return GetThreadTraceImpl<Mapping42>(tid); 711296417Sdim DCHECK(0); 712296417Sdim#elif defined(__powerpc64__) 713296417Sdim if (vmaSize == 44) 714296417Sdim return GetThreadTraceImpl<Mapping44>(tid); 715296417Sdim else 716296417Sdim return GetThreadTraceImpl<Mapping46>(tid); 717296417Sdim DCHECK(0); 718296417Sdim#else 719296417Sdim return GetThreadTraceImpl<Mapping>(tid); 720296417Sdim#endif 721296417Sdim} 722296417Sdim 723296417Sdim 724296417Sdimtemplate<typename Mapping> 725296417Sdimuptr GetThreadTraceHeaderImpl(int tid) { 726296417Sdim uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize 727274201Sdim + kTraceSize * sizeof(Event); 728296417Sdim DCHECK_LT(p, Mapping::kTraceMemEnd); 729251034Sed return p; 730251034Sed} 731251034Sed 732296417SdimALWAYS_INLINE 733296417Sdimuptr GetThreadTraceHeader(int tid) { 734296417Sdim#ifdef __aarch64__ 735296417Sdim if (vmaSize == 39) 736296417Sdim return GetThreadTraceHeaderImpl<Mapping39>(tid); 737296417Sdim else 738296417Sdim return GetThreadTraceHeaderImpl<Mapping42>(tid); 739296417Sdim DCHECK(0); 740296417Sdim#elif defined(__powerpc64__) 741296417Sdim if (vmaSize == 44) 742296417Sdim return GetThreadTraceHeaderImpl<Mapping44>(tid); 743296417Sdim else 744296417Sdim return GetThreadTraceHeaderImpl<Mapping46>(tid); 745296417Sdim DCHECK(0); 746296417Sdim#else 747296417Sdim return GetThreadTraceHeaderImpl<Mapping>(tid); 748296417Sdim#endif 749296417Sdim} 750296417Sdim 751276789Sdimvoid InitializePlatform(); 752296417Sdimvoid InitializePlatformEarly(); 753296417Sdimvoid CheckAndProtect(); 754296417Sdimvoid InitializeShadowMemoryPlatform(); 755276789Sdimvoid FlushShadowMemory(); 756276789Sdimvoid WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive); 757238901Sandrew 758245614Sandrew// Says whether the addr relates to a global var. 759245614Sandrew// Guesses with high probability, may yield both false positives and negatives. 760245614Sandrewbool IsGlobalVar(uptr addr); 761251034Sedint ExtractResolvFDs(void *state, int *fds, int nfd); 762276789Sdimint ExtractRecvmsgFDs(void *msg, int *fds, int nfd); 763238901Sandrew 764276789Sdimint call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, 765276789Sdim void *abstime), void *c, void *m, void *abstime, 766276789Sdim void(*cleanup)(void *arg), void *arg); 767276789Sdim 768296417Sdimvoid DestroyThreadState(); 769296417Sdim 770238901Sandrew} // namespace __tsan 771238901Sandrew 772238901Sandrew#endif // TSAN_PLATFORM_H 773