1//===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
11/// FreeBSD-specific code.
12///
13//===----------------------------------------------------------------------===//
14
15#include "sanitizer_common/sanitizer_platform.h"
16#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
17
18#  include <dlfcn.h>
19#  include <elf.h>
20#  include <errno.h>
21#  include <link.h>
22#  include <pthread.h>
23#  include <signal.h>
24#  include <stdio.h>
25#  include <stdlib.h>
26#  include <sys/prctl.h>
27#  include <sys/resource.h>
28#  include <sys/time.h>
29#  include <unistd.h>
30#  include <unwind.h>
31
32#  include "hwasan.h"
33#  include "hwasan_dynamic_shadow.h"
34#  include "hwasan_interface_internal.h"
35#  include "hwasan_mapping.h"
36#  include "hwasan_report.h"
37#  include "hwasan_thread.h"
38#  include "hwasan_thread_list.h"
39#  include "sanitizer_common/sanitizer_common.h"
40#  include "sanitizer_common/sanitizer_procmaps.h"
41#  include "sanitizer_common/sanitizer_stackdepot.h"
42
43// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID.
44//
45// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF
46//   Not currently tested.
47// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON
48//   Integration tests downstream exist.
49// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF
50//    Tested with check-hwasan on x86_64-linux.
51// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON
52//    Tested with check-hwasan on aarch64-linux-android.
53#  if !SANITIZER_ANDROID
54SANITIZER_INTERFACE_ATTRIBUTE
55THREADLOCAL uptr __hwasan_tls;
56#  endif
57
58namespace __hwasan {
59
60// With the zero shadow base we can not actually map pages starting from 0.
61// This constant is somewhat arbitrary.
62constexpr uptr kZeroBaseShadowStart = 0;
63constexpr uptr kZeroBaseMaxShadowStart = 1 << 18;
64
65static void ProtectGap(uptr addr, uptr size) {
66  __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart,
67                          kZeroBaseMaxShadowStart);
68}
69
70uptr kLowMemStart;
71uptr kLowMemEnd;
72uptr kHighMemStart;
73uptr kHighMemEnd;
74
75static void PrintRange(uptr start, uptr end, const char *name) {
76  Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
77}
78
79static void PrintAddressSpaceLayout() {
80  PrintRange(kHighMemStart, kHighMemEnd, "HighMem");
81  if (kHighShadowEnd + 1 < kHighMemStart)
82    PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap");
83  else
84    CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
85  PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow");
86  if (kLowShadowEnd + 1 < kHighShadowStart)
87    PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap");
88  else
89    CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
90  PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
91  if (kLowMemEnd + 1 < kLowShadowStart)
92    PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap");
93  else
94    CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
95  PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
96  CHECK_EQ(0, kLowMemStart);
97}
98
99static uptr GetHighMemEnd() {
100  // HighMem covers the upper part of the address space.
101  uptr max_address = GetMaxUserVirtualAddress();
102  // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
103  // properly aligned:
104  max_address |= (GetMmapGranularity() << kShadowScale) - 1;
105  return max_address;
106}
107
108static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
109  __hwasan_shadow_memory_dynamic_address =
110      FindDynamicShadowStart(shadow_size_bytes);
111}
112
113void InitializeOsSupport() {
114#  define PR_SET_TAGGED_ADDR_CTRL 55
115#  define PR_GET_TAGGED_ADDR_CTRL 56
116#  define PR_TAGGED_ADDR_ENABLE (1UL << 0)
117  // Check we're running on a kernel that can use the tagged address ABI.
118  int local_errno = 0;
119  if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
120                       &local_errno) &&
121      local_errno == EINVAL) {
122#  if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
123    // Some older Android kernels have the tagged pointer ABI on
124    // unconditionally, and hence don't have the tagged-addr prctl while still
125    // allow the ABI.
126    // If targeting Android and the prctl is not around we assume this is the
127    // case.
128    return;
129#  else
130    if (flags()->fail_without_syscall_abi) {
131      Printf(
132          "FATAL: "
133          "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
134      Die();
135    }
136#  endif
137  }
138
139  // Turn on the tagged address ABI.
140  if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
141                                       PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) ||
142       !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) {
143#  if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
144    // Try the new prctl API for Intel LAM.  The API is based on a currently
145    // unsubmitted patch to the Linux kernel (as of May 2021) and is thus
146    // subject to change.  Patch is here:
147    // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/
148    int tag_bits = kTagBits;
149    int tag_shift = kAddressTagShift;
150    if (!internal_iserror(
151            internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE,
152                           reinterpret_cast<unsigned long>(&tag_bits),
153                           reinterpret_cast<unsigned long>(&tag_shift), 0))) {
154      CHECK_EQ(tag_bits, kTagBits);
155      CHECK_EQ(tag_shift, kAddressTagShift);
156      return;
157    }
158#  endif  // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
159    if (flags()->fail_without_syscall_abi) {
160      Printf(
161          "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
162          "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
163          "configuration.\n");
164      Die();
165    }
166  }
167#  undef PR_SET_TAGGED_ADDR_CTRL
168#  undef PR_GET_TAGGED_ADDR_CTRL
169#  undef PR_TAGGED_ADDR_ENABLE
170}
171
172bool InitShadow() {
173  // Define the entire memory range.
174  kHighMemEnd = GetHighMemEnd();
175
176  // Determine shadow memory base offset.
177  InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd));
178
179  // Place the low memory first.
180  kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1;
181  kLowMemStart = 0;
182
183  // Define the low shadow based on the already placed low memory.
184  kLowShadowEnd = MemToShadow(kLowMemEnd);
185  kLowShadowStart = __hwasan_shadow_memory_dynamic_address;
186
187  // High shadow takes whatever memory is left up there (making sure it is not
188  // interfering with low memory in the fixed case).
189  kHighShadowEnd = MemToShadow(kHighMemEnd);
190  kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1;
191
192  // High memory starts where allocated shadow allows.
193  kHighMemStart = ShadowToMem(kHighShadowStart);
194
195  // Check the sanity of the defined memory ranges (there might be gaps).
196  CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
197  CHECK_GT(kHighMemStart, kHighShadowEnd);
198  CHECK_GT(kHighShadowEnd, kHighShadowStart);
199  CHECK_GT(kHighShadowStart, kLowMemEnd);
200  CHECK_GT(kLowMemEnd, kLowMemStart);
201  CHECK_GT(kLowShadowEnd, kLowShadowStart);
202  CHECK_GT(kLowShadowStart, kLowMemEnd);
203
204  if (Verbosity())
205    PrintAddressSpaceLayout();
206
207  // Reserve shadow memory.
208  ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow");
209  ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow");
210
211  // Protect all the gaps.
212  ProtectGap(0, Min(kLowMemStart, kLowShadowStart));
213  if (kLowMemEnd + 1 < kLowShadowStart)
214    ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1);
215  if (kLowShadowEnd + 1 < kHighShadowStart)
216    ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1);
217  if (kHighShadowEnd + 1 < kHighMemStart)
218    ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1);
219
220  return true;
221}
222
223void InitThreads() {
224  CHECK(__hwasan_shadow_memory_dynamic_address);
225  uptr guard_page_size = GetMmapGranularity();
226  uptr thread_space_start =
227      __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment);
228  uptr thread_space_end =
229      __hwasan_shadow_memory_dynamic_address - guard_page_size;
230  ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
231                           "hwasan threads", /*madvise_shadow*/ false);
232  ProtectGap(thread_space_end,
233             __hwasan_shadow_memory_dynamic_address - thread_space_end);
234  InitThreadList(thread_space_start, thread_space_end - thread_space_start);
235  hwasanThreadList().CreateCurrentThread();
236}
237
238bool MemIsApp(uptr p) {
239// Memory outside the alias range has non-zero tags.
240#  if !defined(HWASAN_ALIASING_MODE)
241  CHECK(GetTagFromPointer(p) == 0);
242#  endif
243
244  return (p >= kHighMemStart && p <= kHighMemEnd) ||
245         (p >= kLowMemStart && p <= kLowMemEnd);
246}
247
248void InstallAtExitHandler() { atexit(HwasanAtExit); }
249
250// ---------------------- TSD ---------------- {{{1
251
252extern "C" void __hwasan_thread_enter() {
253  hwasanThreadList().CreateCurrentThread()->EnsureRandomStateInited();
254}
255
256extern "C" void __hwasan_thread_exit() {
257  Thread *t = GetCurrentThread();
258  // Make sure that signal handler can not see a stale current thread pointer.
259  atomic_signal_fence(memory_order_seq_cst);
260  if (t)
261    hwasanThreadList().ReleaseThread(t);
262}
263
264#  if HWASAN_WITH_INTERCEPTORS
265static pthread_key_t tsd_key;
266static bool tsd_key_inited = false;
267
268void HwasanTSDThreadInit() {
269  if (tsd_key_inited)
270    CHECK_EQ(0, pthread_setspecific(tsd_key,
271                                    (void *)GetPthreadDestructorIterations()));
272}
273
274void HwasanTSDDtor(void *tsd) {
275  uptr iterations = (uptr)tsd;
276  if (iterations > 1) {
277    CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1)));
278    return;
279  }
280  __hwasan_thread_exit();
281}
282
283void HwasanTSDInit() {
284  CHECK(!tsd_key_inited);
285  tsd_key_inited = true;
286  CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
287}
288#  else
289void HwasanTSDInit() {}
290void HwasanTSDThreadInit() {}
291#  endif
292
293#  if SANITIZER_ANDROID
294uptr *GetCurrentThreadLongPtr() { return (uptr *)get_android_tls_ptr(); }
295#  else
296uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; }
297#  endif
298
299#  if SANITIZER_ANDROID
300void AndroidTestTlsSlot() {
301  uptr kMagicValue = 0x010203040A0B0C0D;
302  uptr *tls_ptr = GetCurrentThreadLongPtr();
303  uptr old_value = *tls_ptr;
304  *tls_ptr = kMagicValue;
305  dlerror();
306  if (*(uptr *)get_android_tls_ptr() != kMagicValue) {
307    Printf(
308        "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used "
309        "for dlerror().\n");
310    Die();
311  }
312  *tls_ptr = old_value;
313}
314#  else
315void AndroidTestTlsSlot() {}
316#  endif
317
318static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
319  // Access type is passed in a platform dependent way (see below) and encoded
320  // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
321  // recoverable. Valid values of Y are 0 to 4, which are interpreted as
322  // log2(access_size), and 0xF, which means that access size is passed via
323  // platform dependent register (see below).
324#  if defined(__aarch64__)
325  // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
326  // access size is stored in X1 register. Access address is always in X0
327  // register.
328  uptr pc = (uptr)info->si_addr;
329  const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
330  if ((code & 0xff00) != 0x900)
331    return AccessInfo{};  // Not ours.
332
333  const bool is_store = code & 0x10;
334  const bool recover = code & 0x20;
335  const uptr addr = uc->uc_mcontext.regs[0];
336  const unsigned size_log = code & 0xf;
337  if (size_log > 4 && size_log != 0xf)
338    return AccessInfo{};  // Not ours.
339  const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
340
341#  elif defined(__x86_64__)
342  // Access type is encoded in the instruction following INT3 as
343  // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
344  // RSI register. Access address is always in RDI register.
345  uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
346  uint8_t *nop = (uint8_t *)pc;
347  if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
348      *(nop + 3) < 0x40)
349    return AccessInfo{};  // Not ours.
350  const unsigned code = *(nop + 3);
351
352  const bool is_store = code & 0x10;
353  const bool recover = code & 0x20;
354  const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
355  const unsigned size_log = code & 0xf;
356  if (size_log > 4 && size_log != 0xf)
357    return AccessInfo{};  // Not ours.
358  const uptr size =
359      size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
360
361#  else
362#    error Unsupported architecture
363#  endif
364
365  return AccessInfo{addr, size, is_store, !is_store, recover};
366}
367
368static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
369  AccessInfo ai = GetAccessInfo(info, uc);
370  if (!ai.is_store && !ai.is_load)
371    return false;
372
373  SignalContext sig{info, uc};
374  HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc);
375
376#  if defined(__aarch64__)
377  uc->uc_mcontext.pc += 4;
378#  elif defined(__x86_64__)
379#  else
380#    error Unsupported architecture
381#  endif
382  return true;
383}
384
385static void OnStackUnwind(const SignalContext &sig, const void *,
386                          BufferedStackTrace *stack) {
387  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
388                common_flags()->fast_unwind_on_fatal);
389}
390
391void HwasanOnDeadlySignal(int signo, void *info, void *context) {
392  // Probably a tag mismatch.
393  if (signo == SIGTRAP)
394    if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t *)context))
395      return;
396
397  HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
398}
399
400void Thread::InitStackAndTls(const InitState *) {
401  uptr tls_size;
402  uptr stack_size;
403  GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
404                       &tls_size);
405  stack_top_ = stack_bottom_ + stack_size;
406  tls_end_ = tls_begin_ + tls_size;
407}
408
409uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
410  CHECK(IsAligned(p, kShadowAlignment));
411  CHECK(IsAligned(size, kShadowAlignment));
412  uptr shadow_start = MemToShadow(p);
413  uptr shadow_size = MemToShadowSize(size);
414
415  uptr page_size = GetPageSizeCached();
416  uptr page_start = RoundUpTo(shadow_start, page_size);
417  uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
418  uptr threshold = common_flags()->clear_shadow_mmap_threshold;
419  if (SANITIZER_LINUX &&
420      UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
421    internal_memset((void *)shadow_start, tag, page_start - shadow_start);
422    internal_memset((void *)page_end, tag,
423                    shadow_start + shadow_size - page_end);
424    // For an anonymous private mapping MADV_DONTNEED will return a zero page on
425    // Linux.
426    ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
427  } else {
428    internal_memset((void *)shadow_start, tag, shadow_size);
429  }
430  return AddTagToPointer(p, tag);
431}
432
433void HwasanInstallAtForkHandler() {
434  auto before = []() {
435    HwasanAllocatorLock();
436    StackDepotLockAll();
437  };
438  auto after = []() {
439    StackDepotUnlockAll();
440    HwasanAllocatorUnlock();
441  };
442  pthread_atfork(before, after, after);
443}
444
445}  // namespace __hwasan
446
447#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
448