1//===-- asan_malloc_linux.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of AddressSanitizer, an address sanity checker.
10//
11// Linux-specific malloc interception.
12// We simply define functions like malloc, free, realloc, etc.
13// They will replace the corresponding libc functions automagically.
14//===----------------------------------------------------------------------===//
15
16#include "sanitizer_common/sanitizer_platform.h"
17#if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \
18    SANITIZER_NETBSD || SANITIZER_RTEMS || SANITIZER_SOLARIS
19
20#include "sanitizer_common/sanitizer_allocator_checks.h"
21#include "sanitizer_common/sanitizer_errno.h"
22#include "sanitizer_common/sanitizer_tls_get_addr.h"
23#include "asan_allocator.h"
24#include "asan_interceptors.h"
25#include "asan_internal.h"
26#include "asan_malloc_local.h"
27#include "asan_stack.h"
28
29// ---------------------- Replacement functions ---------------- {{{1
30using namespace __asan;
31
32static uptr allocated_for_dlsym;
33static uptr last_dlsym_alloc_size_in_words;
34static const uptr kDlsymAllocPoolSize = SANITIZER_RTEMS ? 4096 : 1024;
35static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
36
37static INLINE bool IsInDlsymAllocPool(const void *ptr) {
38  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
39  return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
40}
41
42static void *AllocateFromLocalPool(uptr size_in_bytes) {
43  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
44  void *mem = (void*)&alloc_memory_for_dlsym[allocated_for_dlsym];
45  last_dlsym_alloc_size_in_words = size_in_words;
46  allocated_for_dlsym += size_in_words;
47  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
48  return mem;
49}
50
51static void DeallocateFromLocalPool(const void *ptr) {
52  // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
53  // error messages and instead uses malloc followed by free. To avoid pool
54  // exhaustion due to long object filenames, handle that special case here.
55  uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
56  void *prev_mem = (void*)&alloc_memory_for_dlsym[prev_offset];
57  if (prev_mem == ptr) {
58    REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
59    allocated_for_dlsym = prev_offset;
60    last_dlsym_alloc_size_in_words = 0;
61  }
62}
63
64static int PosixMemalignFromLocalPool(void **memptr, uptr alignment,
65                                      uptr size_in_bytes) {
66  if (UNLIKELY(!CheckPosixMemalignAlignment(alignment)))
67    return errno_EINVAL;
68
69  CHECK(alignment >= kWordSize);
70
71  uptr addr = (uptr)&alloc_memory_for_dlsym[allocated_for_dlsym];
72  uptr aligned_addr = RoundUpTo(addr, alignment);
73  uptr aligned_size = RoundUpTo(size_in_bytes, kWordSize);
74
75  uptr *end_mem = (uptr*)(aligned_addr + aligned_size);
76  uptr allocated = end_mem - alloc_memory_for_dlsym;
77  if (allocated >= kDlsymAllocPoolSize)
78    return errno_ENOMEM;
79
80  allocated_for_dlsym = allocated;
81  *memptr = (void*)aligned_addr;
82  return 0;
83}
84
85#if SANITIZER_RTEMS
86void* MemalignFromLocalPool(uptr alignment, uptr size) {
87  void *ptr = nullptr;
88  alignment = Max(alignment, kWordSize);
89  PosixMemalignFromLocalPool(&ptr, alignment, size);
90  return ptr;
91}
92
93bool IsFromLocalPool(const void *ptr) {
94  return IsInDlsymAllocPool(ptr);
95}
96#endif
97
98static INLINE bool MaybeInDlsym() {
99  // Fuchsia doesn't use dlsym-based interceptors.
100  return !SANITIZER_FUCHSIA && asan_init_is_running;
101}
102
103static INLINE bool UseLocalPool() {
104  return EarlyMalloc() || MaybeInDlsym();
105}
106
107static void *ReallocFromLocalPool(void *ptr, uptr size) {
108  const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
109  const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
110  void *new_ptr;
111  if (UNLIKELY(UseLocalPool())) {
112    new_ptr = AllocateFromLocalPool(size);
113  } else {
114    ENSURE_ASAN_INITED();
115    GET_STACK_TRACE_MALLOC;
116    new_ptr = asan_malloc(size, &stack);
117  }
118  internal_memcpy(new_ptr, ptr, copy_size);
119  return new_ptr;
120}
121
122INTERCEPTOR(void, free, void *ptr) {
123  GET_STACK_TRACE_FREE;
124  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
125    DeallocateFromLocalPool(ptr);
126    return;
127  }
128  asan_free(ptr, &stack, FROM_MALLOC);
129}
130
131#if SANITIZER_INTERCEPT_CFREE
132INTERCEPTOR(void, cfree, void *ptr) {
133  GET_STACK_TRACE_FREE;
134  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
135    return;
136  asan_free(ptr, &stack, FROM_MALLOC);
137}
138#endif // SANITIZER_INTERCEPT_CFREE
139
140INTERCEPTOR(void*, malloc, uptr size) {
141  if (UNLIKELY(UseLocalPool()))
142    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
143    return AllocateFromLocalPool(size);
144  ENSURE_ASAN_INITED();
145  GET_STACK_TRACE_MALLOC;
146  return asan_malloc(size, &stack);
147}
148
149INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
150  if (UNLIKELY(UseLocalPool()))
151    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
152    return AllocateFromLocalPool(nmemb * size);
153  ENSURE_ASAN_INITED();
154  GET_STACK_TRACE_MALLOC;
155  return asan_calloc(nmemb, size, &stack);
156}
157
158INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
159  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
160    return ReallocFromLocalPool(ptr, size);
161  if (UNLIKELY(UseLocalPool()))
162    return AllocateFromLocalPool(size);
163  ENSURE_ASAN_INITED();
164  GET_STACK_TRACE_MALLOC;
165  return asan_realloc(ptr, size, &stack);
166}
167
168#if SANITIZER_INTERCEPT_REALLOCARRAY
169INTERCEPTOR(void*, reallocarray, void *ptr, uptr nmemb, uptr size) {
170  ENSURE_ASAN_INITED();
171  GET_STACK_TRACE_MALLOC;
172  return asan_reallocarray(ptr, nmemb, size, &stack);
173}
174#endif  // SANITIZER_INTERCEPT_REALLOCARRAY
175
176#if SANITIZER_INTERCEPT_MEMALIGN
177INTERCEPTOR(void*, memalign, uptr boundary, uptr size) {
178  GET_STACK_TRACE_MALLOC;
179  return asan_memalign(boundary, size, &stack, FROM_MALLOC);
180}
181
182INTERCEPTOR(void*, __libc_memalign, uptr boundary, uptr size) {
183  GET_STACK_TRACE_MALLOC;
184  void *res = asan_memalign(boundary, size, &stack, FROM_MALLOC);
185  DTLS_on_libc_memalign(res, size);
186  return res;
187}
188#endif // SANITIZER_INTERCEPT_MEMALIGN
189
190#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
191INTERCEPTOR(void*, aligned_alloc, uptr boundary, uptr size) {
192  GET_STACK_TRACE_MALLOC;
193  return asan_aligned_alloc(boundary, size, &stack);
194}
195#endif // SANITIZER_INTERCEPT_ALIGNED_ALLOC
196
197INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
198  GET_CURRENT_PC_BP_SP;
199  (void)sp;
200  return asan_malloc_usable_size(ptr, pc, bp);
201}
202
203#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
204// We avoid including malloc.h for portability reasons.
205// man mallinfo says the fields are "long", but the implementation uses int.
206// It doesn't matter much -- we just need to make sure that the libc's mallinfo
207// is not called.
208struct fake_mallinfo {
209  int x[10];
210};
211
212INTERCEPTOR(struct fake_mallinfo, mallinfo, void) {
213  struct fake_mallinfo res;
214  REAL(memset)(&res, 0, sizeof(res));
215  return res;
216}
217
218INTERCEPTOR(int, mallopt, int cmd, int value) {
219  return 0;
220}
221#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
222
223INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
224  if (UNLIKELY(UseLocalPool()))
225    return PosixMemalignFromLocalPool(memptr, alignment, size);
226  GET_STACK_TRACE_MALLOC;
227  return asan_posix_memalign(memptr, alignment, size, &stack);
228}
229
230INTERCEPTOR(void*, valloc, uptr size) {
231  GET_STACK_TRACE_MALLOC;
232  return asan_valloc(size, &stack);
233}
234
235#if SANITIZER_INTERCEPT_PVALLOC
236INTERCEPTOR(void*, pvalloc, uptr size) {
237  GET_STACK_TRACE_MALLOC;
238  return asan_pvalloc(size, &stack);
239}
240#endif // SANITIZER_INTERCEPT_PVALLOC
241
242INTERCEPTOR(void, malloc_stats, void) {
243  __asan_print_accumulated_stats();
244}
245
246#if SANITIZER_ANDROID
247// Format of __libc_malloc_dispatch has changed in Android L.
248// While we are moving towards a solution that does not depend on bionic
249// internals, here is something to support both K* and L releases.
250struct MallocDebugK {
251  void *(*malloc)(uptr bytes);
252  void (*free)(void *mem);
253  void *(*calloc)(uptr n_elements, uptr elem_size);
254  void *(*realloc)(void *oldMem, uptr bytes);
255  void *(*memalign)(uptr alignment, uptr bytes);
256  uptr (*malloc_usable_size)(void *mem);
257};
258
259struct MallocDebugL {
260  void *(*calloc)(uptr n_elements, uptr elem_size);
261  void (*free)(void *mem);
262  fake_mallinfo (*mallinfo)(void);
263  void *(*malloc)(uptr bytes);
264  uptr (*malloc_usable_size)(void *mem);
265  void *(*memalign)(uptr alignment, uptr bytes);
266  int (*posix_memalign)(void **memptr, uptr alignment, uptr size);
267  void* (*pvalloc)(uptr size);
268  void *(*realloc)(void *oldMem, uptr bytes);
269  void* (*valloc)(uptr size);
270};
271
272ALIGNED(32) const MallocDebugK asan_malloc_dispatch_k = {
273    WRAP(malloc),  WRAP(free),     WRAP(calloc),
274    WRAP(realloc), WRAP(memalign), WRAP(malloc_usable_size)};
275
276ALIGNED(32) const MallocDebugL asan_malloc_dispatch_l = {
277    WRAP(calloc),         WRAP(free),               WRAP(mallinfo),
278    WRAP(malloc),         WRAP(malloc_usable_size), WRAP(memalign),
279    WRAP(posix_memalign), WRAP(pvalloc),            WRAP(realloc),
280    WRAP(valloc)};
281
282namespace __asan {
283void ReplaceSystemMalloc() {
284  void **__libc_malloc_dispatch_p =
285      (void **)AsanDlSymNext("__libc_malloc_dispatch");
286  if (__libc_malloc_dispatch_p) {
287    // Decide on K vs L dispatch format by the presence of
288    // __libc_malloc_default_dispatch export in libc.
289    void *default_dispatch_p = AsanDlSymNext("__libc_malloc_default_dispatch");
290    if (default_dispatch_p)
291      *__libc_malloc_dispatch_p = (void *)&asan_malloc_dispatch_k;
292    else
293      *__libc_malloc_dispatch_p = (void *)&asan_malloc_dispatch_l;
294  }
295}
296}  // namespace __asan
297
298#else  // SANITIZER_ANDROID
299
300namespace __asan {
301void ReplaceSystemMalloc() {
302}
303}  // namespace __asan
304#endif  // SANITIZER_ANDROID
305
306#endif  // SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX ||
307        // SANITIZER_NETBSD || SANITIZER_SOLARIS
308