ThreadSanitizer.cpp revision 288943
1//===-- ThreadSanitizer.cpp - race detector -------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file is a part of ThreadSanitizer, a race detector. 11// 12// The tool is under development, for the details about previous versions see 13// http://code.google.com/p/data-race-test 14// 15// The instrumentation phase is quite simple: 16// - Insert calls to run-time library before every memory access. 17// - Optimizations may apply to avoid instrumenting some of the accesses. 18// - Insert calls at function entry/exit. 19// The rest is handled by the run-time library. 20//===----------------------------------------------------------------------===// 21 22#include "llvm/Transforms/Instrumentation.h" 23#include "llvm/ADT/SmallSet.h" 24#include "llvm/ADT/SmallString.h" 25#include "llvm/ADT/SmallVector.h" 26#include "llvm/ADT/Statistic.h" 27#include "llvm/ADT/StringExtras.h" 28#include "llvm/Analysis/CaptureTracking.h" 29#include "llvm/Analysis/ValueTracking.h" 30#include "llvm/IR/DataLayout.h" 31#include "llvm/IR/Function.h" 32#include "llvm/IR/IRBuilder.h" 33#include "llvm/IR/IntrinsicInst.h" 34#include "llvm/IR/Intrinsics.h" 35#include "llvm/IR/LLVMContext.h" 36#include "llvm/IR/Metadata.h" 37#include "llvm/IR/Module.h" 38#include "llvm/IR/Type.h" 39#include "llvm/Support/CommandLine.h" 40#include "llvm/Support/Debug.h" 41#include "llvm/Support/MathExtras.h" 42#include "llvm/Support/raw_ostream.h" 43#include "llvm/Transforms/Utils/BasicBlockUtils.h" 44#include "llvm/Transforms/Utils/ModuleUtils.h" 45 46using namespace llvm; 47 48#define DEBUG_TYPE "tsan" 49 50static cl::opt<bool> ClInstrumentMemoryAccesses( 51 "tsan-instrument-memory-accesses", cl::init(true), 52 cl::desc("Instrument memory accesses"), cl::Hidden); 53static cl::opt<bool> ClInstrumentFuncEntryExit( 54 "tsan-instrument-func-entry-exit", cl::init(true), 55 cl::desc("Instrument function entry and exit"), cl::Hidden); 56static cl::opt<bool> ClInstrumentAtomics( 57 "tsan-instrument-atomics", cl::init(true), 58 cl::desc("Instrument atomics"), cl::Hidden); 59static cl::opt<bool> ClInstrumentMemIntrinsics( 60 "tsan-instrument-memintrinsics", cl::init(true), 61 cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); 62 63STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 64STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 65STATISTIC(NumOmittedReadsBeforeWrite, 66 "Number of reads ignored due to following writes"); 67STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 68STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 69STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); 70STATISTIC(NumOmittedReadsFromConstantGlobals, 71 "Number of reads from constant globals"); 72STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 73STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); 74 75static const char *const kTsanModuleCtorName = "tsan.module_ctor"; 76static const char *const kTsanInitName = "__tsan_init"; 77 78namespace { 79 80/// ThreadSanitizer: instrument the code in module to find races. 81struct ThreadSanitizer : public FunctionPass { 82 ThreadSanitizer() : FunctionPass(ID) {} 83 const char *getPassName() const override; 84 bool runOnFunction(Function &F) override; 85 bool doInitialization(Module &M) override; 86 static char ID; // Pass identification, replacement for typeid. 87 88 private: 89 void initializeCallbacks(Module &M); 90 bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); 91 bool instrumentAtomic(Instruction *I, const DataLayout &DL); 92 bool instrumentMemIntrinsic(Instruction *I); 93 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local, 94 SmallVectorImpl<Instruction *> &All, 95 const DataLayout &DL); 96 bool addrPointsToConstantData(Value *Addr); 97 int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); 98 99 Type *IntptrTy; 100 IntegerType *OrdTy; 101 // Callbacks to run-time library are computed in doInitialization. 102 Function *TsanFuncEntry; 103 Function *TsanFuncExit; 104 // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 105 static const size_t kNumberOfAccessSizes = 5; 106 Function *TsanRead[kNumberOfAccessSizes]; 107 Function *TsanWrite[kNumberOfAccessSizes]; 108 Function *TsanUnalignedRead[kNumberOfAccessSizes]; 109 Function *TsanUnalignedWrite[kNumberOfAccessSizes]; 110 Function *TsanAtomicLoad[kNumberOfAccessSizes]; 111 Function *TsanAtomicStore[kNumberOfAccessSizes]; 112 Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes]; 113 Function *TsanAtomicCAS[kNumberOfAccessSizes]; 114 Function *TsanAtomicThreadFence; 115 Function *TsanAtomicSignalFence; 116 Function *TsanVptrUpdate; 117 Function *TsanVptrLoad; 118 Function *MemmoveFn, *MemcpyFn, *MemsetFn; 119 Function *TsanCtorFunction; 120}; 121} // namespace 122 123char ThreadSanitizer::ID = 0; 124INITIALIZE_PASS(ThreadSanitizer, "tsan", 125 "ThreadSanitizer: detects data races.", 126 false, false) 127 128const char *ThreadSanitizer::getPassName() const { 129 return "ThreadSanitizer"; 130} 131 132FunctionPass *llvm::createThreadSanitizerPass() { 133 return new ThreadSanitizer(); 134} 135 136void ThreadSanitizer::initializeCallbacks(Module &M) { 137 IRBuilder<> IRB(M.getContext()); 138 // Initialize the callbacks. 139 TsanFuncEntry = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 140 "__tsan_func_entry", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 141 TsanFuncExit = checkSanitizerInterfaceFunction( 142 M.getOrInsertFunction("__tsan_func_exit", IRB.getVoidTy(), nullptr)); 143 OrdTy = IRB.getInt32Ty(); 144 for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 145 const size_t ByteSize = 1 << i; 146 const size_t BitSize = ByteSize * 8; 147 SmallString<32> ReadName("__tsan_read" + itostr(ByteSize)); 148 TsanRead[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 149 ReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 150 151 SmallString<32> WriteName("__tsan_write" + itostr(ByteSize)); 152 TsanWrite[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 153 WriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 154 155 SmallString<64> UnalignedReadName("__tsan_unaligned_read" + 156 itostr(ByteSize)); 157 TsanUnalignedRead[i] = 158 checkSanitizerInterfaceFunction(M.getOrInsertFunction( 159 UnalignedReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 160 161 SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + 162 itostr(ByteSize)); 163 TsanUnalignedWrite[i] = 164 checkSanitizerInterfaceFunction(M.getOrInsertFunction( 165 UnalignedWriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 166 167 Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 168 Type *PtrTy = Ty->getPointerTo(); 169 SmallString<32> AtomicLoadName("__tsan_atomic" + itostr(BitSize) + 170 "_load"); 171 TsanAtomicLoad[i] = checkSanitizerInterfaceFunction( 172 M.getOrInsertFunction(AtomicLoadName, Ty, PtrTy, OrdTy, nullptr)); 173 174 SmallString<32> AtomicStoreName("__tsan_atomic" + itostr(BitSize) + 175 "_store"); 176 TsanAtomicStore[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 177 AtomicStoreName, IRB.getVoidTy(), PtrTy, Ty, OrdTy, nullptr)); 178 179 for (int op = AtomicRMWInst::FIRST_BINOP; 180 op <= AtomicRMWInst::LAST_BINOP; ++op) { 181 TsanAtomicRMW[op][i] = nullptr; 182 const char *NamePart = nullptr; 183 if (op == AtomicRMWInst::Xchg) 184 NamePart = "_exchange"; 185 else if (op == AtomicRMWInst::Add) 186 NamePart = "_fetch_add"; 187 else if (op == AtomicRMWInst::Sub) 188 NamePart = "_fetch_sub"; 189 else if (op == AtomicRMWInst::And) 190 NamePart = "_fetch_and"; 191 else if (op == AtomicRMWInst::Or) 192 NamePart = "_fetch_or"; 193 else if (op == AtomicRMWInst::Xor) 194 NamePart = "_fetch_xor"; 195 else if (op == AtomicRMWInst::Nand) 196 NamePart = "_fetch_nand"; 197 else 198 continue; 199 SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 200 TsanAtomicRMW[op][i] = checkSanitizerInterfaceFunction( 201 M.getOrInsertFunction(RMWName, Ty, PtrTy, Ty, OrdTy, nullptr)); 202 } 203 204 SmallString<32> AtomicCASName("__tsan_atomic" + itostr(BitSize) + 205 "_compare_exchange_val"); 206 TsanAtomicCAS[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 207 AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, nullptr)); 208 } 209 TsanVptrUpdate = checkSanitizerInterfaceFunction( 210 M.getOrInsertFunction("__tsan_vptr_update", IRB.getVoidTy(), 211 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), nullptr)); 212 TsanVptrLoad = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 213 "__tsan_vptr_read", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 214 TsanAtomicThreadFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 215 "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, nullptr)); 216 TsanAtomicSignalFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 217 "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, nullptr)); 218 219 MemmoveFn = checkSanitizerInterfaceFunction( 220 M.getOrInsertFunction("memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 221 IRB.getInt8PtrTy(), IntptrTy, nullptr)); 222 MemcpyFn = checkSanitizerInterfaceFunction( 223 M.getOrInsertFunction("memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 224 IRB.getInt8PtrTy(), IntptrTy, nullptr)); 225 MemsetFn = checkSanitizerInterfaceFunction( 226 M.getOrInsertFunction("memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 227 IRB.getInt32Ty(), IntptrTy, nullptr)); 228} 229 230bool ThreadSanitizer::doInitialization(Module &M) { 231 const DataLayout &DL = M.getDataLayout(); 232 IntptrTy = DL.getIntPtrType(M.getContext()); 233 std::tie(TsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions( 234 M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{}, 235 /*InitArgs=*/{}); 236 237 appendToGlobalCtors(M, TsanCtorFunction, 0); 238 239 return true; 240} 241 242static bool isVtableAccess(Instruction *I) { 243 if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) 244 return Tag->isTBAAVtableAccess(); 245 return false; 246} 247 248bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 249 // If this is a GEP, just analyze its pointer operand. 250 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 251 Addr = GEP->getPointerOperand(); 252 253 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 254 if (GV->isConstant()) { 255 // Reads from constant globals can not race with any writes. 256 NumOmittedReadsFromConstantGlobals++; 257 return true; 258 } 259 } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 260 if (isVtableAccess(L)) { 261 // Reads from a vtable pointer can not race with any writes. 262 NumOmittedReadsFromVtable++; 263 return true; 264 } 265 } 266 return false; 267} 268 269// Instrumenting some of the accesses may be proven redundant. 270// Currently handled: 271// - read-before-write (within same BB, no calls between) 272// - not captured variables 273// 274// We do not handle some of the patterns that should not survive 275// after the classic compiler optimizations. 276// E.g. two reads from the same temp should be eliminated by CSE, 277// two writes should be eliminated by DSE, etc. 278// 279// 'Local' is a vector of insns within the same BB (no calls between). 280// 'All' is a vector of insns that will be instrumented. 281void ThreadSanitizer::chooseInstructionsToInstrument( 282 SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All, 283 const DataLayout &DL) { 284 SmallSet<Value*, 8> WriteTargets; 285 // Iterate from the end. 286 for (SmallVectorImpl<Instruction*>::reverse_iterator It = Local.rbegin(), 287 E = Local.rend(); It != E; ++It) { 288 Instruction *I = *It; 289 if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 290 WriteTargets.insert(Store->getPointerOperand()); 291 } else { 292 LoadInst *Load = cast<LoadInst>(I); 293 Value *Addr = Load->getPointerOperand(); 294 if (WriteTargets.count(Addr)) { 295 // We will write to this temp, so no reason to analyze the read. 296 NumOmittedReadsBeforeWrite++; 297 continue; 298 } 299 if (addrPointsToConstantData(Addr)) { 300 // Addr points to some constant data -- it can not race with any writes. 301 continue; 302 } 303 } 304 Value *Addr = isa<StoreInst>(*I) 305 ? cast<StoreInst>(I)->getPointerOperand() 306 : cast<LoadInst>(I)->getPointerOperand(); 307 if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) && 308 !PointerMayBeCaptured(Addr, true, true)) { 309 // The variable is addressable but not captured, so it cannot be 310 // referenced from a different thread and participate in a data race 311 // (see llvm/Analysis/CaptureTracking.h for details). 312 NumOmittedNonCaptured++; 313 continue; 314 } 315 All.push_back(I); 316 } 317 Local.clear(); 318} 319 320static bool isAtomic(Instruction *I) { 321 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 322 return LI->isAtomic() && LI->getSynchScope() == CrossThread; 323 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 324 return SI->isAtomic() && SI->getSynchScope() == CrossThread; 325 if (isa<AtomicRMWInst>(I)) 326 return true; 327 if (isa<AtomicCmpXchgInst>(I)) 328 return true; 329 if (isa<FenceInst>(I)) 330 return true; 331 return false; 332} 333 334bool ThreadSanitizer::runOnFunction(Function &F) { 335 // This is required to prevent instrumenting call to __tsan_init from within 336 // the module constructor. 337 if (&F == TsanCtorFunction) 338 return false; 339 initializeCallbacks(*F.getParent()); 340 SmallVector<Instruction*, 8> RetVec; 341 SmallVector<Instruction*, 8> AllLoadsAndStores; 342 SmallVector<Instruction*, 8> LocalLoadsAndStores; 343 SmallVector<Instruction*, 8> AtomicAccesses; 344 SmallVector<Instruction*, 8> MemIntrinCalls; 345 bool Res = false; 346 bool HasCalls = false; 347 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); 348 const DataLayout &DL = F.getParent()->getDataLayout(); 349 350 // Traverse all instructions, collect loads/stores/returns, check for calls. 351 for (auto &BB : F) { 352 for (auto &Inst : BB) { 353 if (isAtomic(&Inst)) 354 AtomicAccesses.push_back(&Inst); 355 else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) 356 LocalLoadsAndStores.push_back(&Inst); 357 else if (isa<ReturnInst>(Inst)) 358 RetVec.push_back(&Inst); 359 else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { 360 if (isa<MemIntrinsic>(Inst)) 361 MemIntrinCalls.push_back(&Inst); 362 HasCalls = true; 363 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, 364 DL); 365 } 366 } 367 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); 368 } 369 370 // We have collected all loads and stores. 371 // FIXME: many of these accesses do not need to be checked for races 372 // (e.g. variables that do not escape, etc). 373 374 // Instrument memory accesses only if we want to report bugs in the function. 375 if (ClInstrumentMemoryAccesses && SanitizeFunction) 376 for (auto Inst : AllLoadsAndStores) { 377 Res |= instrumentLoadOrStore(Inst, DL); 378 } 379 380 // Instrument atomic memory accesses in any case (they can be used to 381 // implement synchronization). 382 if (ClInstrumentAtomics) 383 for (auto Inst : AtomicAccesses) { 384 Res |= instrumentAtomic(Inst, DL); 385 } 386 387 if (ClInstrumentMemIntrinsics && SanitizeFunction) 388 for (auto Inst : MemIntrinCalls) { 389 Res |= instrumentMemIntrinsic(Inst); 390 } 391 392 // Instrument function entry/exit points if there were instrumented accesses. 393 if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 394 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 395 Value *ReturnAddress = IRB.CreateCall( 396 Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 397 IRB.getInt32(0)); 398 IRB.CreateCall(TsanFuncEntry, ReturnAddress); 399 for (auto RetInst : RetVec) { 400 IRBuilder<> IRBRet(RetInst); 401 IRBRet.CreateCall(TsanFuncExit, {}); 402 } 403 Res = true; 404 } 405 return Res; 406} 407 408bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, 409 const DataLayout &DL) { 410 IRBuilder<> IRB(I); 411 bool IsWrite = isa<StoreInst>(*I); 412 Value *Addr = IsWrite 413 ? cast<StoreInst>(I)->getPointerOperand() 414 : cast<LoadInst>(I)->getPointerOperand(); 415 int Idx = getMemoryAccessFuncIndex(Addr, DL); 416 if (Idx < 0) 417 return false; 418 if (IsWrite && isVtableAccess(I)) { 419 DEBUG(dbgs() << " VPTR : " << *I << "\n"); 420 Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 421 // StoredValue may be a vector type if we are storing several vptrs at once. 422 // In this case, just take the first element of the vector since this is 423 // enough to find vptr races. 424 if (isa<VectorType>(StoredValue->getType())) 425 StoredValue = IRB.CreateExtractElement( 426 StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); 427 if (StoredValue->getType()->isIntegerTy()) 428 StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 429 // Call TsanVptrUpdate. 430 IRB.CreateCall(TsanVptrUpdate, 431 {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 432 IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())}); 433 NumInstrumentedVtableWrites++; 434 return true; 435 } 436 if (!IsWrite && isVtableAccess(I)) { 437 IRB.CreateCall(TsanVptrLoad, 438 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 439 NumInstrumentedVtableReads++; 440 return true; 441 } 442 const unsigned Alignment = IsWrite 443 ? cast<StoreInst>(I)->getAlignment() 444 : cast<LoadInst>(I)->getAlignment(); 445 Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 446 const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 447 Value *OnAccessFunc = nullptr; 448 if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) 449 OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 450 else 451 OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; 452 IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 453 if (IsWrite) NumInstrumentedWrites++; 454 else NumInstrumentedReads++; 455 return true; 456} 457 458static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 459 uint32_t v = 0; 460 switch (ord) { 461 case NotAtomic: llvm_unreachable("unexpected atomic ordering!"); 462 case Unordered: // Fall-through. 463 case Monotonic: v = 0; break; 464 // case Consume: v = 1; break; // Not specified yet. 465 case Acquire: v = 2; break; 466 case Release: v = 3; break; 467 case AcquireRelease: v = 4; break; 468 case SequentiallyConsistent: v = 5; break; 469 } 470 return IRB->getInt32(v); 471} 472 473// If a memset intrinsic gets inlined by the code gen, we will miss races on it. 474// So, we either need to ensure the intrinsic is not inlined, or instrument it. 475// We do not instrument memset/memmove/memcpy intrinsics (too complicated), 476// instead we simply replace them with regular function calls, which are then 477// intercepted by the run-time. 478// Since tsan is running after everyone else, the calls should not be 479// replaced back with intrinsics. If that becomes wrong at some point, 480// we will need to call e.g. __tsan_memset to avoid the intrinsics. 481bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { 482 IRBuilder<> IRB(I); 483 if (MemSetInst *M = dyn_cast<MemSetInst>(I)) { 484 IRB.CreateCall( 485 MemsetFn, 486 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 487 IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false), 488 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 489 I->eraseFromParent(); 490 } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) { 491 IRB.CreateCall( 492 isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn, 493 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 494 IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()), 495 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 496 I->eraseFromParent(); 497 } 498 return false; 499} 500 501// Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 502// standards. For background see C++11 standard. A slightly older, publicly 503// available draft of the standard (not entirely up-to-date, but close enough 504// for casual browsing) is available here: 505// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 506// The following page contains more background information: 507// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 508 509bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { 510 IRBuilder<> IRB(I); 511 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 512 Value *Addr = LI->getPointerOperand(); 513 int Idx = getMemoryAccessFuncIndex(Addr, DL); 514 if (Idx < 0) 515 return false; 516 const size_t ByteSize = 1 << Idx; 517 const size_t BitSize = ByteSize * 8; 518 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 519 Type *PtrTy = Ty->getPointerTo(); 520 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 521 createOrdering(&IRB, LI->getOrdering())}; 522 CallInst *C = CallInst::Create(TsanAtomicLoad[Idx], Args); 523 ReplaceInstWithInst(I, C); 524 525 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 526 Value *Addr = SI->getPointerOperand(); 527 int Idx = getMemoryAccessFuncIndex(Addr, DL); 528 if (Idx < 0) 529 return false; 530 const size_t ByteSize = 1 << Idx; 531 const size_t BitSize = ByteSize * 8; 532 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 533 Type *PtrTy = Ty->getPointerTo(); 534 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 535 IRB.CreateIntCast(SI->getValueOperand(), Ty, false), 536 createOrdering(&IRB, SI->getOrdering())}; 537 CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args); 538 ReplaceInstWithInst(I, C); 539 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 540 Value *Addr = RMWI->getPointerOperand(); 541 int Idx = getMemoryAccessFuncIndex(Addr, DL); 542 if (Idx < 0) 543 return false; 544 Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 545 if (!F) 546 return false; 547 const size_t ByteSize = 1 << Idx; 548 const size_t BitSize = ByteSize * 8; 549 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 550 Type *PtrTy = Ty->getPointerTo(); 551 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 552 IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 553 createOrdering(&IRB, RMWI->getOrdering())}; 554 CallInst *C = CallInst::Create(F, Args); 555 ReplaceInstWithInst(I, C); 556 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 557 Value *Addr = CASI->getPointerOperand(); 558 int Idx = getMemoryAccessFuncIndex(Addr, DL); 559 if (Idx < 0) 560 return false; 561 const size_t ByteSize = 1 << Idx; 562 const size_t BitSize = ByteSize * 8; 563 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 564 Type *PtrTy = Ty->getPointerTo(); 565 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 566 IRB.CreateIntCast(CASI->getCompareOperand(), Ty, false), 567 IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false), 568 createOrdering(&IRB, CASI->getSuccessOrdering()), 569 createOrdering(&IRB, CASI->getFailureOrdering())}; 570 CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); 571 Value *Success = IRB.CreateICmpEQ(C, CASI->getCompareOperand()); 572 573 Value *Res = IRB.CreateInsertValue(UndefValue::get(CASI->getType()), C, 0); 574 Res = IRB.CreateInsertValue(Res, Success, 1); 575 576 I->replaceAllUsesWith(Res); 577 I->eraseFromParent(); 578 } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 579 Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 580 Function *F = FI->getSynchScope() == SingleThread ? 581 TsanAtomicSignalFence : TsanAtomicThreadFence; 582 CallInst *C = CallInst::Create(F, Args); 583 ReplaceInstWithInst(I, C); 584 } 585 return true; 586} 587 588int ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr, 589 const DataLayout &DL) { 590 Type *OrigPtrTy = Addr->getType(); 591 Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 592 assert(OrigTy->isSized()); 593 uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 594 if (TypeSize != 8 && TypeSize != 16 && 595 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 596 NumAccessesWithBadSize++; 597 // Ignore all unusual sizes. 598 return -1; 599 } 600 size_t Idx = countTrailingZeros(TypeSize / 8); 601 assert(Idx < kNumberOfAccessSizes); 602 return Idx; 603} 604