1//===-- hwasan.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of HWAddressSanitizer.
10//
11// HWAddressSanitizer runtime.
12//===----------------------------------------------------------------------===//
13
14#include "hwasan.h"
15
16#include "hwasan_checks.h"
17#include "hwasan_dynamic_shadow.h"
18#include "hwasan_globals.h"
19#include "hwasan_poisoning.h"
20#include "hwasan_report.h"
21#include "hwasan_thread.h"
22#include "hwasan_thread_list.h"
23#include "sanitizer_common/sanitizer_atomic.h"
24#include "sanitizer_common/sanitizer_common.h"
25#include "sanitizer_common/sanitizer_flag_parser.h"
26#include "sanitizer_common/sanitizer_flags.h"
27#include "sanitizer_common/sanitizer_libc.h"
28#include "sanitizer_common/sanitizer_procmaps.h"
29#include "sanitizer_common/sanitizer_stackdepot.h"
30#include "sanitizer_common/sanitizer_stacktrace.h"
31#include "sanitizer_common/sanitizer_symbolizer.h"
32#include "ubsan/ubsan_flags.h"
33#include "ubsan/ubsan_init.h"
34
35// ACHTUNG! No system header includes in this file.
36
37using namespace __sanitizer;
38
39namespace __hwasan {
40
41static Flags hwasan_flags;
42
43Flags *flags() {
44  return &hwasan_flags;
45}
46
47int hwasan_inited = 0;
48int hwasan_instrumentation_inited = 0;
49bool hwasan_init_is_running;
50
51int hwasan_report_count = 0;
52
53void Flags::SetDefaults() {
54#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
55#include "hwasan_flags.inc"
56#undef HWASAN_FLAG
57}
58
59static void RegisterHwasanFlags(FlagParser *parser, Flags *f) {
60#define HWASAN_FLAG(Type, Name, DefaultValue, Description) \
61  RegisterFlag(parser, #Name, Description, &f->Name);
62#include "hwasan_flags.inc"
63#undef HWASAN_FLAG
64}
65
66static void InitializeFlags() {
67  SetCommonFlagsDefaults();
68  {
69    CommonFlags cf;
70    cf.CopyFrom(*common_flags());
71    cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH");
72    cf.malloc_context_size = 20;
73    cf.handle_ioctl = true;
74    // FIXME: test and enable.
75    cf.check_printf = false;
76    cf.intercept_tls_get_addr = true;
77    cf.exitcode = 99;
78    // 8 shadow pages ~512kB, small enough to cover common stack sizes.
79    cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8);
80    // Sigtrap is used in error reporting.
81    cf.handle_sigtrap = kHandleSignalExclusive;
82
83#if SANITIZER_ANDROID
84    // Let platform handle other signals. It is better at reporting them then we
85    // are.
86    cf.handle_segv = kHandleSignalNo;
87    cf.handle_sigbus = kHandleSignalNo;
88    cf.handle_abort = kHandleSignalNo;
89    cf.handle_sigill = kHandleSignalNo;
90    cf.handle_sigfpe = kHandleSignalNo;
91#endif
92    OverrideCommonFlags(cf);
93  }
94
95  Flags *f = flags();
96  f->SetDefaults();
97
98  FlagParser parser;
99  RegisterHwasanFlags(&parser, f);
100  RegisterCommonFlags(&parser);
101
102#if HWASAN_CONTAINS_UBSAN
103  __ubsan::Flags *uf = __ubsan::flags();
104  uf->SetDefaults();
105
106  FlagParser ubsan_parser;
107  __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
108  RegisterCommonFlags(&ubsan_parser);
109#endif
110
111  // Override from user-specified string.
112  if (__hwasan_default_options)
113    parser.ParseString(__hwasan_default_options());
114#if HWASAN_CONTAINS_UBSAN
115  const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
116  ubsan_parser.ParseString(ubsan_default_options);
117#endif
118
119  parser.ParseStringFromEnv("HWASAN_OPTIONS");
120#if HWASAN_CONTAINS_UBSAN
121  ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
122#endif
123
124  InitializeCommonFlags();
125
126  if (Verbosity()) ReportUnrecognizedFlags();
127
128  if (common_flags()->help) parser.PrintFlagDescriptions();
129}
130
131static void HWAsanCheckFailed(const char *file, int line, const char *cond,
132                              u64 v1, u64 v2) {
133  Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
134         line, cond, (uptr)v1, (uptr)v2);
135  PRINT_CURRENT_STACK_CHECK();
136  Die();
137}
138
139static constexpr uptr kMemoryUsageBufferSize = 4096;
140
141static void HwasanFormatMemoryUsage(InternalScopedString &s) {
142  HwasanThreadList &thread_list = hwasanThreadList();
143  auto thread_stats = thread_list.GetThreadStats();
144  auto *sds = StackDepotGetStats();
145  AllocatorStatCounters asc;
146  GetAllocatorStats(asc);
147  s.append(
148      "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd"
149      " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd"
150      " heap: %zd",
151      internal_getpid(), GetRSS(), thread_stats.n_live_threads,
152      thread_stats.total_stack_size,
153      thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(),
154      sds->allocated, sds->n_uniq_ids, asc[AllocatorStatMapped]);
155}
156
157#if SANITIZER_ANDROID
158static char *memory_usage_buffer = nullptr;
159
160static void InitMemoryUsage() {
161  memory_usage_buffer =
162      (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string");
163  CHECK(memory_usage_buffer);
164  memory_usage_buffer[0] = '\0';
165  DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize,
166                  memory_usage_buffer);
167}
168
169void UpdateMemoryUsage() {
170  if (!flags()->export_memory_stats)
171    return;
172  if (!memory_usage_buffer)
173    InitMemoryUsage();
174  InternalScopedString s(kMemoryUsageBufferSize);
175  HwasanFormatMemoryUsage(s);
176  internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1);
177  memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0';
178}
179#else
180void UpdateMemoryUsage() {}
181#endif
182
183} // namespace __hwasan
184
185using namespace __hwasan;
186
187void __sanitizer::BufferedStackTrace::UnwindImpl(
188    uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
189  Thread *t = GetCurrentThread();
190  if (!t) {
191    // The thread is still being created, or has already been destroyed.
192    size = 0;
193    return;
194  }
195  Unwind(max_depth, pc, bp, context, t->stack_top(), t->stack_bottom(),
196         request_fast);
197}
198
199static bool InitializeSingleGlobal(const hwasan_global &global) {
200  uptr full_granule_size = RoundDownTo(global.size(), 16);
201  TagMemoryAligned(global.addr(), full_granule_size, global.tag());
202  if (global.size() % 16)
203    TagMemoryAligned(global.addr() + full_granule_size, 16, global.size() % 16);
204  return false;
205}
206
207static void InitLoadedGlobals() {
208  dl_iterate_phdr(
209      [](dl_phdr_info *info, size_t /* size */, void * /* data */) -> int {
210        for (const hwasan_global &global : HwasanGlobalsFor(
211                 info->dlpi_addr, info->dlpi_phdr, info->dlpi_phnum))
212          InitializeSingleGlobal(global);
213        return 0;
214      },
215      nullptr);
216}
217
218// Prepare to run instrumented code on the main thread.
219static void InitInstrumentation() {
220  if (hwasan_instrumentation_inited) return;
221
222  InitPrctl();
223
224  if (!InitShadow()) {
225    Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
226    DumpProcessMap();
227    Die();
228  }
229
230  InitThreads();
231  hwasanThreadList().CreateCurrentThread();
232
233  hwasan_instrumentation_inited = 1;
234}
235
236// Interface.
237
238uptr __hwasan_shadow_memory_dynamic_address;  // Global interface symbol.
239
240// This function was used by the old frame descriptor mechanism. We keep it
241// around to avoid breaking ABI.
242void __hwasan_init_frames(uptr beg, uptr end) {}
243
244void __hwasan_init_static() {
245  InitShadowGOT();
246  InitInstrumentation();
247
248  // In the non-static code path we call dl_iterate_phdr here. But at this point
249  // libc might not have been initialized enough for dl_iterate_phdr to work.
250  // Fortunately, since this is a statically linked executable we can use the
251  // linker-defined symbol __ehdr_start to find the only relevant set of phdrs.
252  extern ElfW(Ehdr) __ehdr_start;
253  for (const hwasan_global &global : HwasanGlobalsFor(
254           /* base */ 0,
255           reinterpret_cast<const ElfW(Phdr) *>(
256               reinterpret_cast<const char *>(&__ehdr_start) +
257               __ehdr_start.e_phoff),
258           __ehdr_start.e_phnum))
259    InitializeSingleGlobal(global);
260}
261
262void __hwasan_init() {
263  CHECK(!hwasan_init_is_running);
264  if (hwasan_inited) return;
265  hwasan_init_is_running = 1;
266  SanitizerToolName = "HWAddressSanitizer";
267
268  InitTlsSize();
269
270  CacheBinaryName();
271  InitializeFlags();
272
273  // Install tool-specific callbacks in sanitizer_common.
274  SetCheckFailedCallback(HWAsanCheckFailed);
275
276  __sanitizer_set_report_path(common_flags()->log_path);
277
278  AndroidTestTlsSlot();
279
280  DisableCoreDumperIfNecessary();
281
282  InitInstrumentation();
283  InitLoadedGlobals();
284
285  // Needs to be called here because flags()->random_tags might not have been
286  // initialized when InitInstrumentation() was called.
287  GetCurrentThread()->InitRandomState();
288
289  MadviseShadow();
290
291  SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
292  // This may call libc -> needs initialized shadow.
293  AndroidLogInit();
294
295  InitializeInterceptors();
296  InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
297  InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
298
299  InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
300
301  HwasanTSDInit();
302  HwasanTSDThreadInit();
303
304  HwasanAllocatorInit();
305
306#if HWASAN_CONTAINS_UBSAN
307  __ubsan::InitAsPlugin();
308#endif
309
310  VPrintf(1, "HWAddressSanitizer init done\n");
311
312  hwasan_init_is_running = 0;
313  hwasan_inited = 1;
314}
315
316void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
317                             ElfW(Half) phnum) {
318  for (const hwasan_global &global : HwasanGlobalsFor(base, phdr, phnum))
319    InitializeSingleGlobal(global);
320}
321
322void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
323                               ElfW(Half) phnum) {
324  for (; phnum != 0; ++phdr, --phnum)
325    if (phdr->p_type == PT_LOAD)
326      TagMemory(base + phdr->p_vaddr, phdr->p_memsz, 0);
327}
328
329void __hwasan_print_shadow(const void *p, uptr sz) {
330  uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
331  uptr shadow_first = MemToShadow(ptr_raw);
332  uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
333  Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw,
334         ptr_raw + sz, GetTagFromPointer((uptr)p));
335  for (uptr s = shadow_first; s <= shadow_last; ++s)
336    Printf("  %zx: %x\n", ShadowToMem(s), *(tag_t *)s);
337}
338
339sptr __hwasan_test_shadow(const void *p, uptr sz) {
340  if (sz == 0)
341    return -1;
342  tag_t ptr_tag = GetTagFromPointer((uptr)p);
343  uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
344  uptr shadow_first = MemToShadow(ptr_raw);
345  uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
346  for (uptr s = shadow_first; s <= shadow_last; ++s)
347    if (*(tag_t *)s != ptr_tag) {
348      sptr offset = ShadowToMem(s) - ptr_raw;
349      return offset < 0 ? 0 : offset;
350    }
351  return -1;
352}
353
354u16 __sanitizer_unaligned_load16(const uu16 *p) {
355  return *p;
356}
357u32 __sanitizer_unaligned_load32(const uu32 *p) {
358  return *p;
359}
360u64 __sanitizer_unaligned_load64(const uu64 *p) {
361  return *p;
362}
363void __sanitizer_unaligned_store16(uu16 *p, u16 x) {
364  *p = x;
365}
366void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
367  *p = x;
368}
369void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
370  *p = x;
371}
372
373void __hwasan_loadN(uptr p, uptr sz) {
374  CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
375}
376void __hwasan_load1(uptr p) {
377  CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p);
378}
379void __hwasan_load2(uptr p) {
380  CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p);
381}
382void __hwasan_load4(uptr p) {
383  CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p);
384}
385void __hwasan_load8(uptr p) {
386  CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p);
387}
388void __hwasan_load16(uptr p) {
389  CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p);
390}
391
392void __hwasan_loadN_noabort(uptr p, uptr sz) {
393  CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz);
394}
395void __hwasan_load1_noabort(uptr p) {
396  CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p);
397}
398void __hwasan_load2_noabort(uptr p) {
399  CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p);
400}
401void __hwasan_load4_noabort(uptr p) {
402  CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p);
403}
404void __hwasan_load8_noabort(uptr p) {
405  CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p);
406}
407void __hwasan_load16_noabort(uptr p) {
408  CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
409}
410
411void __hwasan_storeN(uptr p, uptr sz) {
412  CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
413}
414void __hwasan_store1(uptr p) {
415  CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p);
416}
417void __hwasan_store2(uptr p) {
418  CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p);
419}
420void __hwasan_store4(uptr p) {
421  CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p);
422}
423void __hwasan_store8(uptr p) {
424  CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p);
425}
426void __hwasan_store16(uptr p) {
427  CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p);
428}
429
430void __hwasan_storeN_noabort(uptr p, uptr sz) {
431  CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz);
432}
433void __hwasan_store1_noabort(uptr p) {
434  CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p);
435}
436void __hwasan_store2_noabort(uptr p) {
437  CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p);
438}
439void __hwasan_store4_noabort(uptr p) {
440  CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p);
441}
442void __hwasan_store8_noabort(uptr p) {
443  CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p);
444}
445void __hwasan_store16_noabort(uptr p) {
446  CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
447}
448
449void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) {
450  TagMemoryAligned(p, sz, tag);
451}
452
453uptr __hwasan_tag_pointer(uptr p, u8 tag) {
454  return AddTagToPointer(p, tag);
455}
456
457void __hwasan_handle_longjmp(const void *sp_dst) {
458  uptr dst = (uptr)sp_dst;
459  // HWASan does not support tagged SP.
460  CHECK(GetTagFromPointer(dst) == 0);
461
462  uptr sp = (uptr)__builtin_frame_address(0);
463  static const uptr kMaxExpectedCleanupSize = 64 << 20;  // 64M
464  if (dst < sp || dst - sp > kMaxExpectedCleanupSize) {
465    Report(
466        "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: "
467        "stack top: %p; target %p; distance: %p (%zd)\n"
468        "False positive error reports may follow\n",
469        (void *)sp, (void *)dst, dst - sp);
470    return;
471  }
472  TagMemory(sp, dst - sp, 0);
473}
474
475void __hwasan_handle_vfork(const void *sp_dst) {
476  uptr sp = (uptr)sp_dst;
477  Thread *t = GetCurrentThread();
478  CHECK(t);
479  uptr top = t->stack_top();
480  uptr bottom = t->stack_bottom();
481  if (top == 0 || bottom == 0 || sp < bottom || sp >= top) {
482    Report(
483        "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: "
484        "stack top: %zx; current %zx; bottom: %zx \n"
485        "False positive error reports may follow\n",
486        top, sp, bottom);
487    return;
488  }
489  TagMemory(bottom, sp - bottom, 0);
490}
491
492extern "C" void *__hwasan_extra_spill_area() {
493  Thread *t = GetCurrentThread();
494  return &t->vfork_spill();
495}
496
497void __hwasan_print_memory_usage() {
498  InternalScopedString s(kMemoryUsageBufferSize);
499  HwasanFormatMemoryUsage(s);
500  Printf("%s\n", s.data());
501}
502
503static const u8 kFallbackTag = 0xBB;
504
505u8 __hwasan_generate_tag() {
506  Thread *t = GetCurrentThread();
507  if (!t) return kFallbackTag;
508  return t->GenerateRandomTag();
509}
510
511#if !SANITIZER_SUPPORTS_WEAK_HOOKS
512extern "C" {
513SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
514const char* __hwasan_default_options() { return ""; }
515}  // extern "C"
516#endif
517
518extern "C" {
519SANITIZER_INTERFACE_ATTRIBUTE
520void __sanitizer_print_stack_trace() {
521  GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
522  stack.Print();
523}
524} // extern "C"
525