1//===-- msan_poisoning.cpp --------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemorySanitizer.
10//
11//===----------------------------------------------------------------------===//
12
13#include "msan_poisoning.h"
14
15#include "interception/interception.h"
16#include "msan_origin.h"
17#include "sanitizer_common/sanitizer_common.h"
18
19DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
20DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
21DECLARE_REAL(void *, memmove, void *dest, const void *src, uptr n)
22
23namespace __msan {
24
25u32 GetOriginIfPoisoned(uptr addr, uptr size) {
26  unsigned char *s = (unsigned char *)MEM_TO_SHADOW(addr);
27  for (uptr i = 0; i < size; ++i)
28    if (s[i]) return *(u32 *)SHADOW_TO_ORIGIN(((uptr)s + i) & ~3UL);
29  return 0;
30}
31
32void SetOriginIfPoisoned(uptr addr, uptr src_shadow, uptr size,
33                         u32 src_origin) {
34  uptr dst_s = MEM_TO_SHADOW(addr);
35  uptr src_s = src_shadow;
36  uptr src_s_end = src_s + size;
37
38  for (; src_s < src_s_end; ++dst_s, ++src_s)
39    if (*(u8 *)src_s) *(u32 *)SHADOW_TO_ORIGIN(dst_s & ~3UL) = src_origin;
40}
41
42void CopyOrigin(const void *dst, const void *src, uptr size,
43                StackTrace *stack) {
44  if (!MEM_IS_APP(dst) || !MEM_IS_APP(src)) return;
45
46  uptr d = (uptr)dst;
47  uptr beg = d & ~3UL;
48  // Copy left unaligned origin if that memory is poisoned.
49  if (beg < d) {
50    u32 o = GetOriginIfPoisoned((uptr)src, d - beg);
51    if (o) {
52      if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack);
53      *(u32 *)MEM_TO_ORIGIN(beg) = o;
54    }
55    beg += 4;
56  }
57
58  uptr end = (d + size) & ~3UL;
59  // If both ends fall into the same 4-byte slot, we are done.
60  if (end < beg) return;
61
62  // Copy right unaligned origin if that memory is poisoned.
63  if (end < d + size) {
64    u32 o = GetOriginIfPoisoned((uptr)src + (end - d), (d + size) - end);
65    if (o) {
66      if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack);
67      *(u32 *)MEM_TO_ORIGIN(end) = o;
68    }
69  }
70
71  if (beg < end) {
72    // Align src up.
73    uptr s = ((uptr)src + 3) & ~3UL;
74    // FIXME: factor out to msan_copy_origin_aligned
75    if (__msan_get_track_origins() > 1) {
76      u32 *src = (u32 *)MEM_TO_ORIGIN(s);
77      u32 *src_s = (u32 *)MEM_TO_SHADOW(s);
78      u32 *src_end = (u32 *)MEM_TO_ORIGIN(s + (end - beg));
79      u32 *dst = (u32 *)MEM_TO_ORIGIN(beg);
80      u32 src_o = 0;
81      u32 dst_o = 0;
82      for (; src < src_end; ++src, ++src_s, ++dst) {
83        if (!*src_s) continue;
84        if (*src != src_o) {
85          src_o = *src;
86          dst_o = ChainOrigin(src_o, stack);
87        }
88        *dst = dst_o;
89      }
90    } else {
91      REAL(memcpy)((void *)MEM_TO_ORIGIN(beg), (void *)MEM_TO_ORIGIN(s),
92                   end - beg);
93    }
94  }
95}
96
97void MoveShadowAndOrigin(const void *dst, const void *src, uptr size,
98                         StackTrace *stack) {
99  if (!MEM_IS_APP(dst)) return;
100  if (!MEM_IS_APP(src)) return;
101  if (src == dst) return;
102  REAL(memmove)((void *)MEM_TO_SHADOW((uptr)dst),
103                (void *)MEM_TO_SHADOW((uptr)src), size);
104  if (__msan_get_track_origins()) CopyOrigin(dst, src, size, stack);
105}
106
107void CopyShadowAndOrigin(const void *dst, const void *src, uptr size,
108                         StackTrace *stack) {
109  if (!MEM_IS_APP(dst)) return;
110  if (!MEM_IS_APP(src)) return;
111  REAL(memcpy)((void *)MEM_TO_SHADOW((uptr)dst),
112               (void *)MEM_TO_SHADOW((uptr)src), size);
113  if (__msan_get_track_origins()) CopyOrigin(dst, src, size, stack);
114}
115
116void CopyMemory(void *dst, const void *src, uptr size, StackTrace *stack) {
117  REAL(memcpy)(dst, src, size);
118  CopyShadowAndOrigin(dst, src, size, stack);
119}
120
121void SetShadow(const void *ptr, uptr size, u8 value) {
122  uptr PageSize = GetPageSizeCached();
123  uptr shadow_beg = MEM_TO_SHADOW(ptr);
124  uptr shadow_end = shadow_beg + size;
125  if (value ||
126      shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
127    REAL(memset)((void *)shadow_beg, value, shadow_end - shadow_beg);
128  } else {
129    uptr page_beg = RoundUpTo(shadow_beg, PageSize);
130    uptr page_end = RoundDownTo(shadow_end, PageSize);
131
132    if (page_beg >= page_end) {
133      REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
134    } else {
135      if (page_beg != shadow_beg) {
136        REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg);
137      }
138      if (page_end != shadow_end) {
139        REAL(memset)((void *)page_end, 0, shadow_end - page_end);
140      }
141      if (!MmapFixedNoReserve(page_beg, page_end - page_beg))
142        Die();
143    }
144  }
145}
146
147void SetOrigin(const void *dst, uptr size, u32 origin) {
148  // Origin mapping is 4 bytes per 4 bytes of application memory.
149  // Here we extend the range such that its left and right bounds are both
150  // 4 byte aligned.
151  uptr x = MEM_TO_ORIGIN((uptr)dst);
152  uptr beg = x & ~3UL;               // align down.
153  uptr end = (x + size + 3) & ~3UL;  // align up.
154  u64 origin64 = ((u64)origin << 32) | origin;
155  // This is like memset, but the value is 32-bit. We unroll by 2 to write
156  // 64 bits at once. May want to unroll further to get 128-bit stores.
157  if (beg & 7ULL) {
158    *(u32 *)beg = origin;
159    beg += 4;
160  }
161  for (uptr addr = beg; addr < (end & ~7UL); addr += 8) *(u64 *)addr = origin64;
162  if (end & 7ULL) *(u32 *)(end - 4) = origin;
163}
164
165void PoisonMemory(const void *dst, uptr size, StackTrace *stack) {
166  SetShadow(dst, size, (u8)-1);
167
168  if (__msan_get_track_origins()) {
169    Origin o = Origin::CreateHeapOrigin(stack);
170    SetOrigin(dst, size, o.raw_id());
171  }
172}
173
174}  // namespace __msan
175