1//===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of ThreadSanitizer (TSan), a race detector. 10// 11// Body of the hottest inner loop. 12// If we wrap this body into a function, compilers (both gcc and clang) 13// produce sligtly less efficient code. 14//===----------------------------------------------------------------------===// 15do { 16 StatInc(thr, StatShadowProcessed); 17 const unsigned kAccessSize = 1 << kAccessSizeLog; 18 u64 *sp = &shadow_mem[idx]; 19 old = LoadShadow(sp); 20 if (LIKELY(old.IsZero())) { 21 StatInc(thr, StatShadowZero); 22 if (!stored) { 23 StoreIfNotYetStored(sp, &store_word); 24 stored = true; 25 } 26 break; 27 } 28 // is the memory access equal to the previous? 29 if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) { 30 StatInc(thr, StatShadowSameSize); 31 // same thread? 32 if (LIKELY(Shadow::TidsAreEqual(old, cur))) { 33 StatInc(thr, StatShadowSameThread); 34 if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) { 35 StoreIfNotYetStored(sp, &store_word); 36 stored = true; 37 } 38 break; 39 } 40 StatInc(thr, StatShadowAnotherThread); 41 if (HappensBefore(old, thr)) { 42 if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) { 43 StoreIfNotYetStored(sp, &store_word); 44 stored = true; 45 } 46 break; 47 } 48 if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))) 49 break; 50 goto RACE; 51 } 52 // Do the memory access intersect? 53 if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) { 54 StatInc(thr, StatShadowIntersect); 55 if (Shadow::TidsAreEqual(old, cur)) { 56 StatInc(thr, StatShadowSameThread); 57 break; 58 } 59 StatInc(thr, StatShadowAnotherThread); 60 if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) 61 break; 62 if (LIKELY(HappensBefore(old, thr))) 63 break; 64 goto RACE; 65 } 66 // The accesses do not intersect. 67 StatInc(thr, StatShadowNotIntersect); 68 break; 69} while (0); 70