1//===-- sanitizer_deadlock_detector2.cpp ----------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// Deadlock detector implementation based on adjacency lists. 10// 11//===----------------------------------------------------------------------===// 12 13#include "sanitizer_deadlock_detector_interface.h" 14#include "sanitizer_common.h" 15#include "sanitizer_allocator_internal.h" 16#include "sanitizer_placement_new.h" 17#include "sanitizer_mutex.h" 18 19#if SANITIZER_DEADLOCK_DETECTOR_VERSION == 2 20 21namespace __sanitizer { 22 23const int kMaxNesting = 64; 24const u32 kNoId = -1; 25const u32 kEndId = -2; 26const int kMaxLink = 8; 27const int kL1Size = 1024; 28const int kL2Size = 1024; 29const int kMaxMutex = kL1Size * kL2Size; 30 31struct Id { 32 u32 id; 33 u32 seq; 34 35 explicit Id(u32 id = 0, u32 seq = 0) 36 : id(id) 37 , seq(seq) { 38 } 39}; 40 41struct Link { 42 u32 id; 43 u32 seq; 44 u32 tid; 45 u32 stk0; 46 u32 stk1; 47 48 explicit Link(u32 id = 0, u32 seq = 0, u32 tid = 0, u32 s0 = 0, u32 s1 = 0) 49 : id(id) 50 , seq(seq) 51 , tid(tid) 52 , stk0(s0) 53 , stk1(s1) { 54 } 55}; 56 57struct DDPhysicalThread { 58 DDReport rep; 59 bool report_pending; 60 bool visited[kMaxMutex]; 61 Link pending[kMaxMutex]; 62 Link path[kMaxMutex]; 63}; 64 65struct ThreadMutex { 66 u32 id; 67 u32 stk; 68}; 69 70struct DDLogicalThread { 71 u64 ctx; 72 ThreadMutex locked[kMaxNesting]; 73 int nlocked; 74}; 75 76struct MutexState { 77 StaticSpinMutex mtx; 78 u32 seq; 79 int nlink; 80 Link link[kMaxLink]; 81}; 82 83struct DD final : public DDetector { 84 explicit DD(const DDFlags *flags); 85 86 DDPhysicalThread* CreatePhysicalThread(); 87 void DestroyPhysicalThread(DDPhysicalThread *pt); 88 89 DDLogicalThread* CreateLogicalThread(u64 ctx); 90 void DestroyLogicalThread(DDLogicalThread *lt); 91 92 void MutexInit(DDCallback *cb, DDMutex *m); 93 void MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock); 94 void MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock, 95 bool trylock); 96 void MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock); 97 void MutexDestroy(DDCallback *cb, DDMutex *m); 98 99 DDReport *GetReport(DDCallback *cb); 100 101 void CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, DDMutex *mtx); 102 void Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath); 103 u32 allocateId(DDCallback *cb); 104 MutexState *getMutex(u32 id); 105 u32 getMutexId(MutexState *m); 106 107 DDFlags flags; 108 109 MutexState *mutex[kL1Size]; 110 111 SpinMutex mtx; 112 InternalMmapVector<u32> free_id; 113 int id_gen = 0; 114}; 115 116DDetector *DDetector::Create(const DDFlags *flags) { 117 (void)flags; 118 void *mem = MmapOrDie(sizeof(DD), "deadlock detector"); 119 return new(mem) DD(flags); 120} 121 122DD::DD(const DDFlags *flags) : flags(*flags) { free_id.reserve(1024); } 123 124DDPhysicalThread* DD::CreatePhysicalThread() { 125 DDPhysicalThread *pt = (DDPhysicalThread*)MmapOrDie(sizeof(DDPhysicalThread), 126 "deadlock detector (physical thread)"); 127 return pt; 128} 129 130void DD::DestroyPhysicalThread(DDPhysicalThread *pt) { 131 pt->~DDPhysicalThread(); 132 UnmapOrDie(pt, sizeof(DDPhysicalThread)); 133} 134 135DDLogicalThread* DD::CreateLogicalThread(u64 ctx) { 136 DDLogicalThread *lt = (DDLogicalThread*)InternalAlloc( 137 sizeof(DDLogicalThread)); 138 lt->ctx = ctx; 139 lt->nlocked = 0; 140 return lt; 141} 142 143void DD::DestroyLogicalThread(DDLogicalThread *lt) { 144 lt->~DDLogicalThread(); 145 InternalFree(lt); 146} 147 148void DD::MutexInit(DDCallback *cb, DDMutex *m) { 149 VPrintf(2, "#%llu: DD::MutexInit(%p)\n", cb->lt->ctx, m); 150 m->id = kNoId; 151 m->recursion = 0; 152 atomic_store(&m->owner, 0, memory_order_relaxed); 153} 154 155MutexState *DD::getMutex(u32 id) { return &mutex[id / kL2Size][id % kL2Size]; } 156 157u32 DD::getMutexId(MutexState *m) { 158 for (int i = 0; i < kL1Size; i++) { 159 MutexState *tab = mutex[i]; 160 if (tab == 0) 161 break; 162 if (m >= tab && m < tab + kL2Size) 163 return i * kL2Size + (m - tab); 164 } 165 return -1; 166} 167 168u32 DD::allocateId(DDCallback *cb) { 169 u32 id = -1; 170 SpinMutexLock l(&mtx); 171 if (free_id.size() > 0) { 172 id = free_id.back(); 173 free_id.pop_back(); 174 } else { 175 CHECK_LT(id_gen, kMaxMutex); 176 if ((id_gen % kL2Size) == 0) { 177 mutex[id_gen / kL2Size] = (MutexState *)MmapOrDie( 178 kL2Size * sizeof(MutexState), "deadlock detector (mutex table)"); 179 } 180 id = id_gen++; 181 } 182 CHECK_LE(id, kMaxMutex); 183 VPrintf(3, "#%llu: DD::allocateId assign id %d\n", cb->lt->ctx, id); 184 return id; 185} 186 187void DD::MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock) { 188 VPrintf(2, "#%llu: DD::MutexBeforeLock(%p, wlock=%d) nlocked=%d\n", 189 cb->lt->ctx, m, wlock, cb->lt->nlocked); 190 DDPhysicalThread *pt = cb->pt; 191 DDLogicalThread *lt = cb->lt; 192 193 uptr owner = atomic_load(&m->owner, memory_order_relaxed); 194 if (owner == (uptr)cb->lt) { 195 VPrintf(3, "#%llu: DD::MutexBeforeLock recursive\n", 196 cb->lt->ctx); 197 return; 198 } 199 200 CHECK_LE(lt->nlocked, kMaxNesting); 201 202 // FIXME(dvyukov): don't allocate id if lt->nlocked == 0? 203 if (m->id == kNoId) 204 m->id = allocateId(cb); 205 206 ThreadMutex *tm = <->locked[lt->nlocked++]; 207 tm->id = m->id; 208 if (flags.second_deadlock_stack) 209 tm->stk = cb->Unwind(); 210 if (lt->nlocked == 1) { 211 VPrintf(3, "#%llu: DD::MutexBeforeLock first mutex\n", 212 cb->lt->ctx); 213 return; 214 } 215 216 bool added = false; 217 MutexState *mtx = getMutex(m->id); 218 for (int i = 0; i < lt->nlocked - 1; i++) { 219 u32 id1 = lt->locked[i].id; 220 u32 stk1 = lt->locked[i].stk; 221 MutexState *mtx1 = getMutex(id1); 222 SpinMutexLock l(&mtx1->mtx); 223 if (mtx1->nlink == kMaxLink) { 224 // FIXME(dvyukov): check stale links 225 continue; 226 } 227 int li = 0; 228 for (; li < mtx1->nlink; li++) { 229 Link *link = &mtx1->link[li]; 230 if (link->id == m->id) { 231 if (link->seq != mtx->seq) { 232 link->seq = mtx->seq; 233 link->tid = lt->ctx; 234 link->stk0 = stk1; 235 link->stk1 = cb->Unwind(); 236 added = true; 237 VPrintf(3, "#%llu: DD::MutexBeforeLock added %d->%d link\n", 238 cb->lt->ctx, getMutexId(mtx1), m->id); 239 } 240 break; 241 } 242 } 243 if (li == mtx1->nlink) { 244 // FIXME(dvyukov): check stale links 245 Link *link = &mtx1->link[mtx1->nlink++]; 246 link->id = m->id; 247 link->seq = mtx->seq; 248 link->tid = lt->ctx; 249 link->stk0 = stk1; 250 link->stk1 = cb->Unwind(); 251 added = true; 252 VPrintf(3, "#%llu: DD::MutexBeforeLock added %d->%d link\n", 253 cb->lt->ctx, getMutexId(mtx1), m->id); 254 } 255 } 256 257 if (!added || mtx->nlink == 0) { 258 VPrintf(3, "#%llu: DD::MutexBeforeLock don't check\n", 259 cb->lt->ctx); 260 return; 261 } 262 263 CycleCheck(pt, lt, m); 264} 265 266void DD::MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock, 267 bool trylock) { 268 VPrintf(2, "#%llu: DD::MutexAfterLock(%p, wlock=%d, try=%d) nlocked=%d\n", 269 cb->lt->ctx, m, wlock, trylock, cb->lt->nlocked); 270 DDLogicalThread *lt = cb->lt; 271 272 uptr owner = atomic_load(&m->owner, memory_order_relaxed); 273 if (owner == (uptr)cb->lt) { 274 VPrintf(3, "#%llu: DD::MutexAfterLock recursive\n", cb->lt->ctx); 275 CHECK(wlock); 276 m->recursion++; 277 return; 278 } 279 CHECK_EQ(owner, 0); 280 if (wlock) { 281 VPrintf(3, "#%llu: DD::MutexAfterLock set owner\n", cb->lt->ctx); 282 CHECK_EQ(m->recursion, 0); 283 m->recursion = 1; 284 atomic_store(&m->owner, (uptr)cb->lt, memory_order_relaxed); 285 } 286 287 if (!trylock) 288 return; 289 290 CHECK_LE(lt->nlocked, kMaxNesting); 291 if (m->id == kNoId) 292 m->id = allocateId(cb); 293 ThreadMutex *tm = <->locked[lt->nlocked++]; 294 tm->id = m->id; 295 if (flags.second_deadlock_stack) 296 tm->stk = cb->Unwind(); 297} 298 299void DD::MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock) { 300 VPrintf(2, "#%llu: DD::MutexBeforeUnlock(%p, wlock=%d) nlocked=%d\n", 301 cb->lt->ctx, m, wlock, cb->lt->nlocked); 302 DDLogicalThread *lt = cb->lt; 303 304 uptr owner = atomic_load(&m->owner, memory_order_relaxed); 305 if (owner == (uptr)cb->lt) { 306 VPrintf(3, "#%llu: DD::MutexBeforeUnlock recursive\n", cb->lt->ctx); 307 if (--m->recursion > 0) 308 return; 309 VPrintf(3, "#%llu: DD::MutexBeforeUnlock reset owner\n", cb->lt->ctx); 310 atomic_store(&m->owner, 0, memory_order_relaxed); 311 } 312 CHECK_NE(m->id, kNoId); 313 int last = lt->nlocked - 1; 314 for (int i = last; i >= 0; i--) { 315 if (cb->lt->locked[i].id == m->id) { 316 lt->locked[i] = lt->locked[last]; 317 lt->nlocked--; 318 break; 319 } 320 } 321} 322 323void DD::MutexDestroy(DDCallback *cb, DDMutex *m) { 324 VPrintf(2, "#%llu: DD::MutexDestroy(%p)\n", 325 cb->lt->ctx, m); 326 DDLogicalThread *lt = cb->lt; 327 328 if (m->id == kNoId) 329 return; 330 331 // Remove the mutex from lt->locked if there. 332 int last = lt->nlocked - 1; 333 for (int i = last; i >= 0; i--) { 334 if (lt->locked[i].id == m->id) { 335 lt->locked[i] = lt->locked[last]; 336 lt->nlocked--; 337 break; 338 } 339 } 340 341 // Clear and invalidate the mutex descriptor. 342 { 343 MutexState *mtx = getMutex(m->id); 344 SpinMutexLock l(&mtx->mtx); 345 mtx->seq++; 346 mtx->nlink = 0; 347 } 348 349 // Return id to cache. 350 { 351 SpinMutexLock l(&mtx); 352 free_id.push_back(m->id); 353 } 354} 355 356void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, 357 DDMutex *m) { 358 internal_memset(pt->visited, 0, sizeof(pt->visited)); 359 int npath = 0; 360 int npending = 0; 361 { 362 MutexState *mtx = getMutex(m->id); 363 SpinMutexLock l(&mtx->mtx); 364 for (int li = 0; li < mtx->nlink; li++) 365 pt->pending[npending++] = mtx->link[li]; 366 } 367 while (npending > 0) { 368 Link link = pt->pending[--npending]; 369 if (link.id == kEndId) { 370 npath--; 371 continue; 372 } 373 if (pt->visited[link.id]) 374 continue; 375 MutexState *mtx1 = getMutex(link.id); 376 SpinMutexLock l(&mtx1->mtx); 377 if (mtx1->seq != link.seq) 378 continue; 379 pt->visited[link.id] = true; 380 if (mtx1->nlink == 0) 381 continue; 382 pt->path[npath++] = link; 383 pt->pending[npending++] = Link(kEndId); 384 if (link.id == m->id) 385 return Report(pt, lt, npath); // Bingo! 386 for (int li = 0; li < mtx1->nlink; li++) { 387 Link *link1 = &mtx1->link[li]; 388 // MutexState *mtx2 = getMutex(link->id); 389 // FIXME(dvyukov): fast seq check 390 // FIXME(dvyukov): fast nlink != 0 check 391 // FIXME(dvyukov): fast pending check? 392 // FIXME(dvyukov): npending can be larger than kMaxMutex 393 pt->pending[npending++] = *link1; 394 } 395 } 396} 397 398void DD::Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath) { 399 DDReport *rep = &pt->rep; 400 rep->n = npath; 401 for (int i = 0; i < npath; i++) { 402 Link *link = &pt->path[i]; 403 Link *link0 = &pt->path[i ? i - 1 : npath - 1]; 404 rep->loop[i].thr_ctx = link->tid; 405 rep->loop[i].mtx_ctx0 = link0->id; 406 rep->loop[i].mtx_ctx1 = link->id; 407 rep->loop[i].stk[0] = flags.second_deadlock_stack ? link->stk0 : 0; 408 rep->loop[i].stk[1] = link->stk1; 409 } 410 pt->report_pending = true; 411} 412 413DDReport *DD::GetReport(DDCallback *cb) { 414 if (!cb->pt->report_pending) 415 return 0; 416 cb->pt->report_pending = false; 417 return &cb->pt->rep; 418} 419 420} // namespace __sanitizer 421#endif // #if SANITIZER_DEADLOCK_DETECTOR_VERSION == 2 422