1//===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
11/// FreeBSD-specific code.
12///
13//===----------------------------------------------------------------------===//
14
15#include "sanitizer_common/sanitizer_platform.h"
16#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
17
18#include "hwasan.h"
19#include "hwasan_dynamic_shadow.h"
20#include "hwasan_interface_internal.h"
21#include "hwasan_mapping.h"
22#include "hwasan_report.h"
23#include "hwasan_thread.h"
24#include "hwasan_thread_list.h"
25
26#include <dlfcn.h>
27#include <elf.h>
28#include <link.h>
29#include <pthread.h>
30#include <signal.h>
31#include <stdio.h>
32#include <stdlib.h>
33#include <sys/resource.h>
34#include <sys/time.h>
35#include <unistd.h>
36#include <unwind.h>
37#include <sys/prctl.h>
38#include <errno.h>
39
40#include "sanitizer_common/sanitizer_common.h"
41#include "sanitizer_common/sanitizer_procmaps.h"
42
43// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID.
44//
45// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF
46//   Not currently tested.
47// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON
48//   Integration tests downstream exist.
49// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF
50//    Tested with check-hwasan on x86_64-linux.
51// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON
52//    Tested with check-hwasan on aarch64-linux-android.
53#if !SANITIZER_ANDROID
54SANITIZER_INTERFACE_ATTRIBUTE
55THREADLOCAL uptr __hwasan_tls;
56#endif
57
58namespace __hwasan {
59
60static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
61  CHECK_EQ((beg % GetMmapGranularity()), 0);
62  CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
63  uptr size = end - beg + 1;
64  DecreaseTotalMmap(size);  // Don't count the shadow against mmap_limit_mb.
65  if (!MmapFixedNoReserve(beg, size, name)) {
66    Report(
67        "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
68        "Perhaps you're using ulimit -v\n",
69        size);
70    Abort();
71  }
72}
73
74static void ProtectGap(uptr addr, uptr size) {
75  if (!size)
76    return;
77  void *res = MmapFixedNoAccess(addr, size, "shadow gap");
78  if (addr == (uptr)res)
79    return;
80  // A few pages at the start of the address space can not be protected.
81  // But we really want to protect as much as possible, to prevent this memory
82  // being returned as a result of a non-FIXED mmap().
83  if (addr == 0) {
84    uptr step = GetMmapGranularity();
85    while (size > step) {
86      addr += step;
87      size -= step;
88      void *res = MmapFixedNoAccess(addr, size, "shadow gap");
89      if (addr == (uptr)res)
90        return;
91    }
92  }
93
94  Report(
95      "ERROR: Failed to protect shadow gap [%p, %p]. "
96      "HWASan cannot proceed correctly. ABORTING.\n", (void *)addr,
97      (void *)(addr + size));
98  DumpProcessMap();
99  Die();
100}
101
102static uptr kLowMemStart;
103static uptr kLowMemEnd;
104static uptr kLowShadowEnd;
105static uptr kLowShadowStart;
106static uptr kHighShadowStart;
107static uptr kHighShadowEnd;
108static uptr kHighMemStart;
109static uptr kHighMemEnd;
110
111static void PrintRange(uptr start, uptr end, const char *name) {
112  Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
113}
114
115static void PrintAddressSpaceLayout() {
116  PrintRange(kHighMemStart, kHighMemEnd, "HighMem");
117  if (kHighShadowEnd + 1 < kHighMemStart)
118    PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap");
119  else
120    CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
121  PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow");
122  if (kLowShadowEnd + 1 < kHighShadowStart)
123    PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap");
124  else
125    CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
126  PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
127  if (kLowMemEnd + 1 < kLowShadowStart)
128    PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap");
129  else
130    CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
131  PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
132  CHECK_EQ(0, kLowMemStart);
133}
134
135static uptr GetHighMemEnd() {
136  // HighMem covers the upper part of the address space.
137  uptr max_address = GetMaxUserVirtualAddress();
138  // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
139  // properly aligned:
140  max_address |= (GetMmapGranularity() << kShadowScale) - 1;
141  return max_address;
142}
143
144static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
145  __hwasan_shadow_memory_dynamic_address =
146      FindDynamicShadowStart(shadow_size_bytes);
147}
148
149void InitPrctl() {
150#define PR_SET_TAGGED_ADDR_CTRL 55
151#define PR_GET_TAGGED_ADDR_CTRL 56
152#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
153  // Check we're running on a kernel that can use the tagged address ABI.
154  if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) == (uptr)-1 &&
155      errno == EINVAL) {
156#if SANITIZER_ANDROID
157    // Some older Android kernels have the tagged pointer ABI on
158    // unconditionally, and hence don't have the tagged-addr prctl while still
159    // allow the ABI.
160    // If targeting Android and the prctl is not around we assume this is the
161    // case.
162    return;
163#else
164    Printf(
165        "FATAL: "
166        "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
167    Die();
168#endif
169  }
170
171  // Turn on the tagged address ABI.
172  if (internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0) ==
173          (uptr)-1 ||
174      !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) {
175    Printf(
176        "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
177        "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
178        "configuration.\n");
179    Die();
180  }
181#undef PR_SET_TAGGED_ADDR_CTRL
182#undef PR_GET_TAGGED_ADDR_CTRL
183#undef PR_TAGGED_ADDR_ENABLE
184}
185
186bool InitShadow() {
187  // Define the entire memory range.
188  kHighMemEnd = GetHighMemEnd();
189
190  // Determine shadow memory base offset.
191  InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd));
192
193  // Place the low memory first.
194  kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1;
195  kLowMemStart = 0;
196
197  // Define the low shadow based on the already placed low memory.
198  kLowShadowEnd = MemToShadow(kLowMemEnd);
199  kLowShadowStart = __hwasan_shadow_memory_dynamic_address;
200
201  // High shadow takes whatever memory is left up there (making sure it is not
202  // interfering with low memory in the fixed case).
203  kHighShadowEnd = MemToShadow(kHighMemEnd);
204  kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1;
205
206  // High memory starts where allocated shadow allows.
207  kHighMemStart = ShadowToMem(kHighShadowStart);
208
209  // Check the sanity of the defined memory ranges (there might be gaps).
210  CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
211  CHECK_GT(kHighMemStart, kHighShadowEnd);
212  CHECK_GT(kHighShadowEnd, kHighShadowStart);
213  CHECK_GT(kHighShadowStart, kLowMemEnd);
214  CHECK_GT(kLowMemEnd, kLowMemStart);
215  CHECK_GT(kLowShadowEnd, kLowShadowStart);
216  CHECK_GT(kLowShadowStart, kLowMemEnd);
217
218  if (Verbosity())
219    PrintAddressSpaceLayout();
220
221  // Reserve shadow memory.
222  ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow");
223  ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow");
224
225  // Protect all the gaps.
226  ProtectGap(0, Min(kLowMemStart, kLowShadowStart));
227  if (kLowMemEnd + 1 < kLowShadowStart)
228    ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1);
229  if (kLowShadowEnd + 1 < kHighShadowStart)
230    ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1);
231  if (kHighShadowEnd + 1 < kHighMemStart)
232    ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1);
233
234  return true;
235}
236
237void InitThreads() {
238  CHECK(__hwasan_shadow_memory_dynamic_address);
239  uptr guard_page_size = GetMmapGranularity();
240  uptr thread_space_start =
241      __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment);
242  uptr thread_space_end =
243      __hwasan_shadow_memory_dynamic_address - guard_page_size;
244  ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
245                           "hwasan threads");
246  ProtectGap(thread_space_end,
247             __hwasan_shadow_memory_dynamic_address - thread_space_end);
248  InitThreadList(thread_space_start, thread_space_end - thread_space_start);
249}
250
251static void MadviseShadowRegion(uptr beg, uptr end) {
252  uptr size = end - beg + 1;
253  SetShadowRegionHugePageMode(beg, size);
254  if (common_flags()->use_madv_dontdump)
255    DontDumpShadowMemory(beg, size);
256}
257
258void MadviseShadow() {
259  MadviseShadowRegion(kLowShadowStart, kLowShadowEnd);
260  MadviseShadowRegion(kHighShadowStart, kHighShadowEnd);
261}
262
263bool MemIsApp(uptr p) {
264  CHECK(GetTagFromPointer(p) == 0);
265  return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
266}
267
268static void HwasanAtExit(void) {
269  if (common_flags()->print_module_map)
270    DumpProcessMap();
271  if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
272    ReportStats();
273  if (hwasan_report_count > 0) {
274    // ReportAtExitStatistics();
275    if (common_flags()->exitcode)
276      internal__exit(common_flags()->exitcode);
277  }
278}
279
280void InstallAtExitHandler() {
281  atexit(HwasanAtExit);
282}
283
284// ---------------------- TSD ---------------- {{{1
285
286extern "C" void __hwasan_thread_enter() {
287  hwasanThreadList().CreateCurrentThread()->InitRandomState();
288}
289
290extern "C" void __hwasan_thread_exit() {
291  Thread *t = GetCurrentThread();
292  // Make sure that signal handler can not see a stale current thread pointer.
293  atomic_signal_fence(memory_order_seq_cst);
294  if (t)
295    hwasanThreadList().ReleaseThread(t);
296}
297
298#if HWASAN_WITH_INTERCEPTORS
299static pthread_key_t tsd_key;
300static bool tsd_key_inited = false;
301
302void HwasanTSDThreadInit() {
303  if (tsd_key_inited)
304    CHECK_EQ(0, pthread_setspecific(tsd_key,
305                                    (void *)GetPthreadDestructorIterations()));
306}
307
308void HwasanTSDDtor(void *tsd) {
309  uptr iterations = (uptr)tsd;
310  if (iterations > 1) {
311    CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1)));
312    return;
313  }
314  __hwasan_thread_exit();
315}
316
317void HwasanTSDInit() {
318  CHECK(!tsd_key_inited);
319  tsd_key_inited = true;
320  CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
321}
322#else
323void HwasanTSDInit() {}
324void HwasanTSDThreadInit() {}
325#endif
326
327#if SANITIZER_ANDROID
328uptr *GetCurrentThreadLongPtr() {
329  return (uptr *)get_android_tls_ptr();
330}
331#else
332uptr *GetCurrentThreadLongPtr() {
333  return &__hwasan_tls;
334}
335#endif
336
337#if SANITIZER_ANDROID
338void AndroidTestTlsSlot() {
339  uptr kMagicValue = 0x010203040A0B0C0D;
340  uptr *tls_ptr = GetCurrentThreadLongPtr();
341  uptr old_value = *tls_ptr;
342  *tls_ptr = kMagicValue;
343  dlerror();
344  if (*(uptr *)get_android_tls_ptr() != kMagicValue) {
345    Printf(
346        "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used "
347        "for dlerror().\n");
348    Die();
349  }
350  *tls_ptr = old_value;
351}
352#else
353void AndroidTestTlsSlot() {}
354#endif
355
356Thread *GetCurrentThread() {
357  auto *R = (StackAllocationsRingBuffer *)GetCurrentThreadLongPtr();
358  return hwasanThreadList().GetThreadByBufferAddress((uptr)(R->Next()));
359}
360
361struct AccessInfo {
362  uptr addr;
363  uptr size;
364  bool is_store;
365  bool is_load;
366  bool recover;
367};
368
369static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
370  // Access type is passed in a platform dependent way (see below) and encoded
371  // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
372  // recoverable. Valid values of Y are 0 to 4, which are interpreted as
373  // log2(access_size), and 0xF, which means that access size is passed via
374  // platform dependent register (see below).
375#if defined(__aarch64__)
376  // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
377  // access size is stored in X1 register. Access address is always in X0
378  // register.
379  uptr pc = (uptr)info->si_addr;
380  const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
381  if ((code & 0xff00) != 0x900)
382    return AccessInfo{}; // Not ours.
383
384  const bool is_store = code & 0x10;
385  const bool recover = code & 0x20;
386  const uptr addr = uc->uc_mcontext.regs[0];
387  const unsigned size_log = code & 0xf;
388  if (size_log > 4 && size_log != 0xf)
389    return AccessInfo{}; // Not ours.
390  const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
391
392#elif defined(__x86_64__)
393  // Access type is encoded in the instruction following INT3 as
394  // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
395  // RSI register. Access address is always in RDI register.
396  uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
397  uint8_t *nop = (uint8_t*)pc;
398  if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40  ||
399      *(nop + 3) < 0x40)
400    return AccessInfo{}; // Not ours.
401  const unsigned code = *(nop + 3);
402
403  const bool is_store = code & 0x10;
404  const bool recover = code & 0x20;
405  const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
406  const unsigned size_log = code & 0xf;
407  if (size_log > 4 && size_log != 0xf)
408    return AccessInfo{}; // Not ours.
409  const uptr size =
410      size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
411
412#else
413# error Unsupported architecture
414#endif
415
416  return AccessInfo{addr, size, is_store, !is_store, recover};
417}
418
419static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame,
420                              ucontext_t *uc, uptr *registers_frame = nullptr) {
421  InternalMmapVector<BufferedStackTrace> stack_buffer(1);
422  BufferedStackTrace *stack = stack_buffer.data();
423  stack->Reset();
424  stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
425
426  // The second stack frame contains the failure __hwasan_check function, as
427  // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
428  // we wish to ignore. This (currently) only occurs on AArch64, as x64
429  // implementations use SIGTRAP to implement the failure, and thus do not go
430  // through the stack saver.
431  if (registers_frame && stack->trace && stack->size > 0) {
432    stack->trace++;
433    stack->size--;
434  }
435
436  bool fatal = flags()->halt_on_error || !ai.recover;
437  ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
438                    registers_frame);
439}
440
441static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
442  AccessInfo ai = GetAccessInfo(info, uc);
443  if (!ai.is_store && !ai.is_load)
444    return false;
445
446  SignalContext sig{info, uc};
447  HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc);
448
449#if defined(__aarch64__)
450  uc->uc_mcontext.pc += 4;
451#elif defined(__x86_64__)
452#else
453# error Unsupported architecture
454#endif
455  return true;
456}
457
458static void OnStackUnwind(const SignalContext &sig, const void *,
459                          BufferedStackTrace *stack) {
460  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
461                common_flags()->fast_unwind_on_fatal);
462}
463
464void HwasanOnDeadlySignal(int signo, void *info, void *context) {
465  // Probably a tag mismatch.
466  if (signo == SIGTRAP)
467    if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context))
468      return;
469
470  HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
471}
472
473
474} // namespace __hwasan
475
476// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
477// rest of the mismatch handling code (C++).
478void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
479                            size_t outsize) {
480  __hwasan::AccessInfo ai;
481  ai.is_store = access_info & 0x10;
482  ai.is_load = !ai.is_store;
483  ai.recover = access_info & 0x20;
484  ai.addr = addr;
485  if ((access_info & 0xf) == 0xf)
486    ai.size = outsize;
487  else
488    ai.size = 1 << (access_info & 0xf);
489
490  __hwasan::HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
491                              (uptr)__builtin_frame_address(0), nullptr,
492                              registers_frame);
493  __builtin_unreachable();
494}
495
496#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
497