1// SPDX-License-Identifier: GPL-2.0 2 3/* 4 * Copyright (c) 2021, Google LLC. 5 * Pasha Tatashin <pasha.tatashin@soleen.com> 6 */ 7#include <linux/kstrtox.h> 8#include <linux/mm.h> 9#include <linux/page_table_check.h> 10 11#undef pr_fmt 12#define pr_fmt(fmt) "page_table_check: " fmt 13 14struct page_table_check { 15 atomic_t anon_map_count; 16 atomic_t file_map_count; 17}; 18 19static bool __page_table_check_enabled __initdata = 20 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); 21 22DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); 23EXPORT_SYMBOL(page_table_check_disabled); 24 25static int __init early_page_table_check_param(char *buf) 26{ 27 return kstrtobool(buf, &__page_table_check_enabled); 28} 29 30early_param("page_table_check", early_page_table_check_param); 31 32static bool __init need_page_table_check(void) 33{ 34 return __page_table_check_enabled; 35} 36 37static void __init init_page_table_check(void) 38{ 39 if (!__page_table_check_enabled) 40 return; 41 static_branch_disable(&page_table_check_disabled); 42} 43 44struct page_ext_operations page_table_check_ops = { 45 .size = sizeof(struct page_table_check), 46 .need = need_page_table_check, 47 .init = init_page_table_check, 48 .need_shared_flags = false, 49}; 50 51static struct page_table_check *get_page_table_check(struct page_ext *page_ext) 52{ 53 BUG_ON(!page_ext); 54 return page_ext_data(page_ext, &page_table_check_ops); 55} 56 57/* 58 * An entry is removed from the page table, decrement the counters for that page 59 * verify that it is of correct type and counters do not become negative. 60 */ 61static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) 62{ 63 struct page_ext *page_ext; 64 struct page *page; 65 unsigned long i; 66 bool anon; 67 68 if (!pfn_valid(pfn)) 69 return; 70 71 page = pfn_to_page(pfn); 72 page_ext = page_ext_get(page); 73 74 BUG_ON(PageSlab(page)); 75 anon = PageAnon(page); 76 77 for (i = 0; i < pgcnt; i++) { 78 struct page_table_check *ptc = get_page_table_check(page_ext); 79 80 if (anon) { 81 BUG_ON(atomic_read(&ptc->file_map_count)); 82 BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); 83 } else { 84 BUG_ON(atomic_read(&ptc->anon_map_count)); 85 BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); 86 } 87 page_ext = page_ext_next(page_ext); 88 } 89 page_ext_put(page_ext); 90} 91 92/* 93 * A new entry is added to the page table, increment the counters for that page 94 * verify that it is of correct type and is not being mapped with a different 95 * type to a different process. 96 */ 97static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, 98 bool rw) 99{ 100 struct page_ext *page_ext; 101 struct page *page; 102 unsigned long i; 103 bool anon; 104 105 if (!pfn_valid(pfn)) 106 return; 107 108 page = pfn_to_page(pfn); 109 page_ext = page_ext_get(page); 110 111 BUG_ON(PageSlab(page)); 112 anon = PageAnon(page); 113 114 for (i = 0; i < pgcnt; i++) { 115 struct page_table_check *ptc = get_page_table_check(page_ext); 116 117 if (anon) { 118 BUG_ON(atomic_read(&ptc->file_map_count)); 119 BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); 120 } else { 121 BUG_ON(atomic_read(&ptc->anon_map_count)); 122 BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); 123 } 124 page_ext = page_ext_next(page_ext); 125 } 126 page_ext_put(page_ext); 127} 128 129/* 130 * page is on free list, or is being allocated, verify that counters are zeroes 131 * crash if they are not. 132 */ 133void __page_table_check_zero(struct page *page, unsigned int order) 134{ 135 struct page_ext *page_ext; 136 unsigned long i; 137 138 BUG_ON(PageSlab(page)); 139 140 page_ext = page_ext_get(page); 141 BUG_ON(!page_ext); 142 for (i = 0; i < (1ul << order); i++) { 143 struct page_table_check *ptc = get_page_table_check(page_ext); 144 145 BUG_ON(atomic_read(&ptc->anon_map_count)); 146 BUG_ON(atomic_read(&ptc->file_map_count)); 147 page_ext = page_ext_next(page_ext); 148 } 149 page_ext_put(page_ext); 150} 151 152void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte) 153{ 154 if (&init_mm == mm) 155 return; 156 157 if (pte_user_accessible_page(pte)) { 158 page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); 159 } 160} 161EXPORT_SYMBOL(__page_table_check_pte_clear); 162 163void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd) 164{ 165 if (&init_mm == mm) 166 return; 167 168 if (pmd_user_accessible_page(pmd)) { 169 page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT); 170 } 171} 172EXPORT_SYMBOL(__page_table_check_pmd_clear); 173 174void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud) 175{ 176 if (&init_mm == mm) 177 return; 178 179 if (pud_user_accessible_page(pud)) { 180 page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT); 181 } 182} 183EXPORT_SYMBOL(__page_table_check_pud_clear); 184 185void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte, 186 unsigned int nr) 187{ 188 unsigned int i; 189 190 if (&init_mm == mm) 191 return; 192 193 for (i = 0; i < nr; i++) 194 __page_table_check_pte_clear(mm, ptep_get(ptep + i)); 195 if (pte_user_accessible_page(pte)) 196 page_table_check_set(pte_pfn(pte), nr, pte_write(pte)); 197} 198EXPORT_SYMBOL(__page_table_check_ptes_set); 199 200void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd) 201{ 202 if (&init_mm == mm) 203 return; 204 205 __page_table_check_pmd_clear(mm, *pmdp); 206 if (pmd_user_accessible_page(pmd)) { 207 page_table_check_set(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT, 208 pmd_write(pmd)); 209 } 210} 211EXPORT_SYMBOL(__page_table_check_pmd_set); 212 213void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud) 214{ 215 if (&init_mm == mm) 216 return; 217 218 __page_table_check_pud_clear(mm, *pudp); 219 if (pud_user_accessible_page(pud)) { 220 page_table_check_set(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT, 221 pud_write(pud)); 222 } 223} 224EXPORT_SYMBOL(__page_table_check_pud_set); 225 226void __page_table_check_pte_clear_range(struct mm_struct *mm, 227 unsigned long addr, 228 pmd_t pmd) 229{ 230 if (&init_mm == mm) 231 return; 232 233 if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { 234 pte_t *ptep = pte_offset_map(&pmd, addr); 235 unsigned long i; 236 237 if (WARN_ON(!ptep)) 238 return; 239 for (i = 0; i < PTRS_PER_PTE; i++) { 240 __page_table_check_pte_clear(mm, ptep_get(ptep)); 241 addr += PAGE_SIZE; 242 ptep++; 243 } 244 pte_unmap(ptep - PTRS_PER_PTE); 245 } 246} 247