1//===-- tsan_vector_clock.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11//===----------------------------------------------------------------------===//
12#include "tsan_vector_clock.h"
13
14#include "sanitizer_common/sanitizer_placement_new.h"
15#include "tsan_mman.h"
16
17namespace __tsan {
18
19#if TSAN_VECTORIZE
20const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128);
21#endif
22
23VectorClock::VectorClock() { Reset(); }
24
25void VectorClock::Reset() {
26#if !TSAN_VECTORIZE
27  for (uptr i = 0; i < kThreadSlotCount; i++)
28    clk_[i] = kEpochZero;
29#else
30  m128 z = _mm_setzero_si128();
31  m128* vclk = reinterpret_cast<m128*>(clk_);
32  for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z);
33#endif
34}
35
36void VectorClock::Acquire(const VectorClock* src) {
37  if (!src)
38    return;
39#if !TSAN_VECTORIZE
40  for (uptr i = 0; i < kThreadSlotCount; i++)
41    clk_[i] = max(clk_[i], src->clk_[i]);
42#else
43  m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
44  m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_);
45  for (uptr i = 0; i < kVectorClockSize; i++) {
46    m128 s = _mm_load_si128(&vsrc[i]);
47    m128 d = _mm_load_si128(&vdst[i]);
48    m128 m = _mm_max_epu16(s, d);
49    _mm_store_si128(&vdst[i], m);
50  }
51#endif
52}
53
54static VectorClock* AllocClock(VectorClock** dstp) {
55  if (UNLIKELY(!*dstp))
56    *dstp = New<VectorClock>();
57  return *dstp;
58}
59
60void VectorClock::Release(VectorClock** dstp) const {
61  VectorClock* dst = AllocClock(dstp);
62  dst->Acquire(this);
63}
64
65void VectorClock::ReleaseStore(VectorClock** dstp) const {
66  VectorClock* dst = AllocClock(dstp);
67  *dst = *this;
68}
69
70VectorClock& VectorClock::operator=(const VectorClock& other) {
71#if !TSAN_VECTORIZE
72  for (uptr i = 0; i < kThreadSlotCount; i++)
73    clk_[i] = other.clk_[i];
74#else
75  m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
76  m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_);
77  for (uptr i = 0; i < kVectorClockSize; i++) {
78    m128 s = _mm_load_si128(&vsrc[i]);
79    _mm_store_si128(&vdst[i], s);
80  }
81#endif
82  return *this;
83}
84
85void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) {
86  VectorClock* dst = AllocClock(dstp);
87#if !TSAN_VECTORIZE
88  for (uptr i = 0; i < kThreadSlotCount; i++) {
89    Epoch tmp = dst->clk_[i];
90    dst->clk_[i] = clk_[i];
91    clk_[i] = max(clk_[i], tmp);
92  }
93#else
94  m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
95  m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
96  for (uptr i = 0; i < kVectorClockSize; i++) {
97    m128 t = _mm_load_si128(&vdst[i]);
98    m128 c = _mm_load_si128(&vclk[i]);
99    m128 m = _mm_max_epu16(c, t);
100    _mm_store_si128(&vdst[i], c);
101    _mm_store_si128(&vclk[i], m);
102  }
103#endif
104}
105
106void VectorClock::ReleaseAcquire(VectorClock** dstp) {
107  VectorClock* dst = AllocClock(dstp);
108#if !TSAN_VECTORIZE
109  for (uptr i = 0; i < kThreadSlotCount; i++) {
110    dst->clk_[i] = max(dst->clk_[i], clk_[i]);
111    clk_[i] = dst->clk_[i];
112  }
113#else
114  m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
115  m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
116  for (uptr i = 0; i < kVectorClockSize; i++) {
117    m128 c = _mm_load_si128(&vclk[i]);
118    m128 d = _mm_load_si128(&vdst[i]);
119    m128 m = _mm_max_epu16(c, d);
120    _mm_store_si128(&vdst[i], m);
121    _mm_store_si128(&vclk[i], m);
122  }
123#endif
124}
125
126}  // namespace __tsan
127