1//===-------- cfi.cpp -----------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the runtime support for the cross-DSO CFI.
10//
11//===----------------------------------------------------------------------===//
12
13#include <assert.h>
14#include <elf.h>
15
16#include "sanitizer_common/sanitizer_common.h"
17#if SANITIZER_FREEBSD
18#include <sys/link_elf.h>
19#endif
20#include <link.h>
21#include <string.h>
22#include <stdlib.h>
23#include <sys/mman.h>
24
25#if SANITIZER_LINUX
26typedef ElfW(Phdr) Elf_Phdr;
27typedef ElfW(Ehdr) Elf_Ehdr;
28typedef ElfW(Addr) Elf_Addr;
29typedef ElfW(Sym) Elf_Sym;
30typedef ElfW(Dyn) Elf_Dyn;
31#elif SANITIZER_FREEBSD
32#if SANITIZER_WORDSIZE == 64
33#define ElfW64_Dyn Elf_Dyn
34#define ElfW64_Sym Elf_Sym
35#else
36#define ElfW32_Dyn Elf_Dyn
37#define ElfW32_Sym Elf_Sym
38#endif
39#endif
40
41#include "interception/interception.h"
42#include "sanitizer_common/sanitizer_flag_parser.h"
43#include "ubsan/ubsan_init.h"
44#include "ubsan/ubsan_flags.h"
45
46#ifdef CFI_ENABLE_DIAG
47#include "ubsan/ubsan_handlers.h"
48#endif
49
50using namespace __sanitizer;
51
52namespace __cfi {
53
54#define kCfiShadowLimitsStorageSize 4096 // 1 page
55// Lets hope that the data segment is mapped with 4K pages.
56// The pointer to the cfi shadow region is stored at the start of this page.
57// The rest of the page is unused and re-mapped read-only.
58static union {
59  char space[kCfiShadowLimitsStorageSize];
60  struct {
61    uptr start;
62    uptr size;
63  } limits;
64} cfi_shadow_limits_storage
65    __attribute__((aligned(kCfiShadowLimitsStorageSize)));
66static constexpr uptr kShadowGranularity = 12;
67static constexpr uptr kShadowAlign = 1UL << kShadowGranularity; // 4096
68
69static constexpr uint16_t kInvalidShadow = 0;
70static constexpr uint16_t kUncheckedShadow = 0xFFFFU;
71
72// Get the start address of the CFI shadow region.
73uptr GetShadow() {
74  return cfi_shadow_limits_storage.limits.start;
75}
76
77uptr GetShadowSize() {
78  return cfi_shadow_limits_storage.limits.size;
79}
80
81// This will only work while the shadow is not allocated.
82void SetShadowSize(uptr size) {
83  cfi_shadow_limits_storage.limits.size = size;
84}
85
86uptr MemToShadowOffset(uptr x) {
87  return (x >> kShadowGranularity) << 1;
88}
89
90uint16_t *MemToShadow(uptr x, uptr shadow_base) {
91  return (uint16_t *)(shadow_base + MemToShadowOffset(x));
92}
93
94typedef int (*CFICheckFn)(u64, void *, void *);
95
96// This class reads and decodes the shadow contents.
97class ShadowValue {
98  uptr addr;
99  uint16_t v;
100  explicit ShadowValue(uptr addr, uint16_t v) : addr(addr), v(v) {}
101
102public:
103  bool is_invalid() const { return v == kInvalidShadow; }
104
105  bool is_unchecked() const { return v == kUncheckedShadow; }
106
107  CFICheckFn get_cfi_check() const {
108    assert(!is_invalid() && !is_unchecked());
109    uptr aligned_addr = addr & ~(kShadowAlign - 1);
110    uptr p = aligned_addr - (((uptr)v - 1) << kShadowGranularity);
111    return reinterpret_cast<CFICheckFn>(p);
112  }
113
114  // Load a shadow value for the given application memory address.
115  static const ShadowValue load(uptr addr) {
116    uptr shadow_base = GetShadow();
117    uptr shadow_offset = MemToShadowOffset(addr);
118    if (shadow_offset > GetShadowSize())
119      return ShadowValue(addr, kInvalidShadow);
120    else
121      return ShadowValue(
122          addr, *reinterpret_cast<uint16_t *>(shadow_base + shadow_offset));
123  }
124};
125
126class ShadowBuilder {
127  uptr shadow_;
128
129public:
130  // Allocate a new empty shadow (for the entire address space) on the side.
131  void Start();
132  // Mark the given address range as unchecked.
133  // This is used for uninstrumented libraries like libc.
134  // Any CFI check with a target in that range will pass.
135  void AddUnchecked(uptr begin, uptr end);
136  // Mark the given address range as belonging to a library with the given
137  // cfi_check function.
138  void Add(uptr begin, uptr end, uptr cfi_check);
139  // Finish shadow construction. Atomically switch the current active shadow
140  // region with the newly constructed one and deallocate the former.
141  void Install();
142};
143
144void ShadowBuilder::Start() {
145  shadow_ = (uptr)MmapNoReserveOrDie(GetShadowSize(), "CFI shadow");
146  VReport(1, "CFI: shadow at %zx .. %zx\n", shadow_, shadow_ + GetShadowSize());
147}
148
149void ShadowBuilder::AddUnchecked(uptr begin, uptr end) {
150  uint16_t *shadow_begin = MemToShadow(begin, shadow_);
151  uint16_t *shadow_end = MemToShadow(end - 1, shadow_) + 1;
152  // memset takes a byte, so our unchecked shadow value requires both bytes to
153  // be the same. Make sure we're ok during compilation.
154  static_assert((kUncheckedShadow & 0xff) == ((kUncheckedShadow >> 8) & 0xff),
155                "Both bytes of the 16-bit value must be the same!");
156  memset(shadow_begin, kUncheckedShadow & 0xff,
157         (shadow_end - shadow_begin) * sizeof(*shadow_begin));
158}
159
160void ShadowBuilder::Add(uptr begin, uptr end, uptr cfi_check) {
161  assert((cfi_check & (kShadowAlign - 1)) == 0);
162
163  // Don't fill anything below cfi_check. We can not represent those addresses
164  // in the shadow, and must make sure at codegen to place all valid call
165  // targets above cfi_check.
166  begin = Max(begin, cfi_check);
167  uint16_t *s = MemToShadow(begin, shadow_);
168  uint16_t *s_end = MemToShadow(end - 1, shadow_) + 1;
169  uint16_t sv = ((begin - cfi_check) >> kShadowGranularity) + 1;
170  for (; s < s_end; s++, sv++)
171    *s = sv;
172}
173
174#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
175void ShadowBuilder::Install() {
176  MprotectReadOnly(shadow_, GetShadowSize());
177  uptr main_shadow = GetShadow();
178  if (main_shadow) {
179    // Update.
180#if SANITIZER_LINUX
181    void *res = mremap((void *)shadow_, GetShadowSize(), GetShadowSize(),
182                       MREMAP_MAYMOVE | MREMAP_FIXED, (void *)main_shadow);
183    CHECK(res != MAP_FAILED);
184#elif SANITIZER_NETBSD
185    void *res = mremap((void *)shadow_, GetShadowSize(), (void *)main_shadow,
186                       GetShadowSize(), MAP_FIXED);
187    CHECK(res != MAP_FAILED);
188#else
189    void *res = MmapFixedOrDie(shadow_, GetShadowSize(), "cfi shadow");
190    CHECK(res != MAP_FAILED);
191    ::memcpy(&shadow_, &main_shadow, GetShadowSize());
192#endif
193  } else {
194    // Initial setup.
195    CHECK_EQ(kCfiShadowLimitsStorageSize, GetPageSizeCached());
196    CHECK_EQ(0, GetShadow());
197    cfi_shadow_limits_storage.limits.start = shadow_;
198    MprotectReadOnly((uptr)&cfi_shadow_limits_storage,
199                     sizeof(cfi_shadow_limits_storage));
200    CHECK_EQ(shadow_, GetShadow());
201  }
202}
203#else
204#error not implemented
205#endif
206
207// This is a workaround for a glibc bug:
208// https://sourceware.org/bugzilla/show_bug.cgi?id=15199
209// Other platforms can, hopefully, just do
210//    dlopen(RTLD_NOLOAD | RTLD_LAZY)
211//    dlsym("__cfi_check").
212uptr find_cfi_check_in_dso(dl_phdr_info *info) {
213  const Elf_Dyn *dynamic = nullptr;
214  for (int i = 0; i < info->dlpi_phnum; ++i) {
215    if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) {
216      dynamic =
217          (const Elf_Dyn *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);
218      break;
219    }
220  }
221  if (!dynamic) return 0;
222  uptr strtab = 0, symtab = 0, strsz = 0;
223  for (const Elf_Dyn *p = dynamic; p->d_tag != PT_NULL; ++p) {
224    if (p->d_tag == DT_SYMTAB)
225      symtab = p->d_un.d_ptr;
226    else if (p->d_tag == DT_STRTAB)
227      strtab = p->d_un.d_ptr;
228    else if (p->d_tag == DT_STRSZ)
229      strsz = p->d_un.d_ptr;
230  }
231
232  if (symtab > strtab) {
233    VReport(1, "Can not handle: symtab > strtab (%zx > %zx)\n", symtab, strtab);
234    return 0;
235  }
236
237  // Verify that strtab and symtab are inside of the same LOAD segment.
238  // This excludes VDSO, which has (very high) bogus strtab and symtab pointers.
239  int phdr_idx;
240  for (phdr_idx = 0; phdr_idx < info->dlpi_phnum; phdr_idx++) {
241    const Elf_Phdr *phdr = &info->dlpi_phdr[phdr_idx];
242    if (phdr->p_type == PT_LOAD) {
243      uptr beg = info->dlpi_addr + phdr->p_vaddr;
244      uptr end = beg + phdr->p_memsz;
245      if (strtab >= beg && strtab + strsz < end && symtab >= beg &&
246          symtab < end)
247        break;
248    }
249  }
250  if (phdr_idx == info->dlpi_phnum) {
251    // Nope, either different segments or just bogus pointers.
252    // Can not handle this.
253    VReport(1, "Can not handle: symtab %zx, strtab %zx\n", symtab, strtab);
254    return 0;
255  }
256
257  for (const Elf_Sym *p = (const Elf_Sym *)symtab; (Elf_Addr)p < strtab;
258       ++p) {
259    // There is no reliable way to find the end of the symbol table. In
260    // lld-produces files, there are other sections between symtab and strtab.
261    // Stop looking when the symbol name is not inside strtab.
262    if (p->st_name >= strsz) break;
263    char *name = (char*)(strtab + p->st_name);
264    if (strcmp(name, "__cfi_check") == 0) {
265      assert(p->st_info == ELF32_ST_INFO(STB_GLOBAL, STT_FUNC) ||
266             p->st_info == ELF32_ST_INFO(STB_WEAK, STT_FUNC));
267      uptr addr = info->dlpi_addr + p->st_value;
268      return addr;
269    }
270  }
271  return 0;
272}
273
274int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *data) {
275  uptr cfi_check = find_cfi_check_in_dso(info);
276  if (cfi_check)
277    VReport(1, "Module '%s' __cfi_check %zx\n", info->dlpi_name, cfi_check);
278
279  ShadowBuilder *b = reinterpret_cast<ShadowBuilder *>(data);
280
281  for (int i = 0; i < info->dlpi_phnum; i++) {
282    const Elf_Phdr *phdr = &info->dlpi_phdr[i];
283    if (phdr->p_type == PT_LOAD) {
284      // Jump tables are in the executable segment.
285      // VTables are in the non-executable one.
286      // Need to fill shadow for both.
287      // FIXME: reject writable if vtables are in the r/o segment. Depend on
288      // PT_RELRO?
289      uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
290      uptr cur_end = cur_beg + phdr->p_memsz;
291      if (cfi_check) {
292        VReport(1, "   %zx .. %zx\n", cur_beg, cur_end);
293        b->Add(cur_beg, cur_end, cfi_check);
294      } else {
295        b->AddUnchecked(cur_beg, cur_end);
296      }
297    }
298  }
299  return 0;
300}
301
302// Init or update shadow for the current set of loaded libraries.
303void UpdateShadow() {
304  ShadowBuilder b;
305  b.Start();
306  dl_iterate_phdr(dl_iterate_phdr_cb, &b);
307  b.Install();
308}
309
310void InitShadow() {
311  CHECK_EQ(0, GetShadow());
312  CHECK_EQ(0, GetShadowSize());
313
314  uptr vma = GetMaxUserVirtualAddress();
315  // Shadow is 2 -> 2**kShadowGranularity.
316  SetShadowSize((vma >> (kShadowGranularity - 1)) + 1);
317  VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, GetShadowSize());
318
319  UpdateShadow();
320}
321
322THREADLOCAL int in_loader;
323Mutex shadow_update_lock;
324
325void EnterLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
326  if (in_loader == 0) {
327    shadow_update_lock.Lock();
328  }
329  ++in_loader;
330}
331
332void ExitLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
333  CHECK(in_loader > 0);
334  --in_loader;
335  UpdateShadow();
336  if (in_loader == 0) {
337    shadow_update_lock.Unlock();
338  }
339}
340
341ALWAYS_INLINE void CfiSlowPathCommon(u64 CallSiteTypeId, void *Ptr,
342                                     void *DiagData) {
343  uptr Addr = (uptr)Ptr;
344  VReport(3, "__cfi_slowpath: %llx, %p\n", CallSiteTypeId, Ptr);
345  ShadowValue sv = ShadowValue::load(Addr);
346  if (sv.is_invalid()) {
347    VReport(1, "CFI: invalid memory region for a check target: %p\n", Ptr);
348#ifdef CFI_ENABLE_DIAG
349    if (DiagData) {
350      __ubsan_handle_cfi_check_fail(
351          reinterpret_cast<__ubsan::CFICheckFailData *>(DiagData), Addr, false);
352      return;
353    }
354#endif
355    Trap();
356  }
357  if (sv.is_unchecked()) {
358    VReport(2, "CFI: unchecked call (shadow=FFFF): %p\n", Ptr);
359    return;
360  }
361  CFICheckFn cfi_check = sv.get_cfi_check();
362  VReport(2, "__cfi_check at %p\n", (void *)cfi_check);
363  cfi_check(CallSiteTypeId, Ptr, DiagData);
364}
365
366void InitializeFlags() {
367  SetCommonFlagsDefaults();
368#ifdef CFI_ENABLE_DIAG
369  __ubsan::Flags *uf = __ubsan::flags();
370  uf->SetDefaults();
371#endif
372
373  FlagParser cfi_parser;
374  RegisterCommonFlags(&cfi_parser);
375  cfi_parser.ParseStringFromEnv("CFI_OPTIONS");
376
377#ifdef CFI_ENABLE_DIAG
378  FlagParser ubsan_parser;
379  __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
380  RegisterCommonFlags(&ubsan_parser);
381
382  const char *ubsan_default_options = __ubsan_default_options();
383  ubsan_parser.ParseString(ubsan_default_options);
384  ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
385#endif
386
387  InitializeCommonFlags();
388
389  if (Verbosity())
390    ReportUnrecognizedFlags();
391
392  if (common_flags()->help) {
393    cfi_parser.PrintFlagDescriptions();
394  }
395}
396
397} // namespace __cfi
398
399using namespace __cfi;
400
401extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
402__cfi_slowpath(u64 CallSiteTypeId, void *Ptr) {
403  CfiSlowPathCommon(CallSiteTypeId, Ptr, nullptr);
404}
405
406#ifdef CFI_ENABLE_DIAG
407extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
408__cfi_slowpath_diag(u64 CallSiteTypeId, void *Ptr, void *DiagData) {
409  CfiSlowPathCommon(CallSiteTypeId, Ptr, DiagData);
410}
411#endif
412
413static void EnsureInterceptorsInitialized();
414
415// Setup shadow for dlopen()ed libraries.
416// The actual shadow setup happens after dlopen() returns, which means that
417// a library can not be a target of any CFI checks while its constructors are
418// running. It's unclear how to fix this without some extra help from libc.
419// In glibc, mmap inside dlopen is not interceptable.
420// Maybe a seccomp-bpf filter?
421// We could insert a high-priority constructor into the library, but that would
422// not help with the uninstrumented libraries.
423INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
424  EnsureInterceptorsInitialized();
425  EnterLoader();
426  void *handle = REAL(dlopen)(filename, flag);
427  ExitLoader();
428  return handle;
429}
430
431INTERCEPTOR(int, dlclose, void *handle) {
432  EnsureInterceptorsInitialized();
433  EnterLoader();
434  int res = REAL(dlclose)(handle);
435  ExitLoader();
436  return res;
437}
438
439static Mutex interceptor_init_lock;
440static bool interceptors_inited = false;
441
442static void EnsureInterceptorsInitialized() {
443  Lock lock(&interceptor_init_lock);
444  if (interceptors_inited)
445    return;
446
447  INTERCEPT_FUNCTION(dlopen);
448  INTERCEPT_FUNCTION(dlclose);
449
450  interceptors_inited = true;
451}
452
453extern "C" SANITIZER_INTERFACE_ATTRIBUTE
454#if !SANITIZER_CAN_USE_PREINIT_ARRAY
455// On ELF platforms, the constructor is invoked using .preinit_array (see below)
456__attribute__((constructor(0)))
457#endif
458void __cfi_init() {
459  SanitizerToolName = "CFI";
460  InitializeFlags();
461  InitShadow();
462
463#ifdef CFI_ENABLE_DIAG
464  __ubsan::InitAsPlugin();
465#endif
466}
467
468#if SANITIZER_CAN_USE_PREINIT_ARRAY
469// On ELF platforms, run cfi initialization before any other constructors.
470// On other platforms we use the constructor attribute to arrange to run our
471// initialization early.
472extern "C" {
473__attribute__((section(".preinit_array"),
474               used)) void (*__cfi_preinit)(void) = __cfi_init;
475}
476#endif
477