1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6#ifndef _ASM_RISCV_PGTABLE_64_H 7#define _ASM_RISCV_PGTABLE_64_H 8 9#include <linux/bits.h> 10#include <linux/const.h> 11#include <asm/errata_list.h> 12 13extern bool pgtable_l4_enabled; 14extern bool pgtable_l5_enabled; 15 16#define PGDIR_SHIFT_L3 30 17#define PGDIR_SHIFT_L4 39 18#define PGDIR_SHIFT_L5 48 19#define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) 20 21#define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ 22 (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) 23/* Size of region mapped by a page global directory */ 24#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) 25#define PGDIR_MASK (~(PGDIR_SIZE - 1)) 26 27/* p4d is folded into pgd in case of 4-level page table */ 28#define P4D_SHIFT_L3 30 29#define P4D_SHIFT_L4 39 30#define P4D_SHIFT_L5 39 31#define P4D_SHIFT (pgtable_l5_enabled ? P4D_SHIFT_L5 : \ 32 (pgtable_l4_enabled ? P4D_SHIFT_L4 : P4D_SHIFT_L3)) 33#define P4D_SIZE (_AC(1, UL) << P4D_SHIFT) 34#define P4D_MASK (~(P4D_SIZE - 1)) 35 36/* pud is folded into pgd in case of 3-level page table */ 37#define PUD_SHIFT 30 38#define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) 39#define PUD_MASK (~(PUD_SIZE - 1)) 40 41#define PMD_SHIFT 21 42/* Size of region mapped by a page middle directory */ 43#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) 44#define PMD_MASK (~(PMD_SIZE - 1)) 45 46/* Page 4th Directory entry */ 47typedef struct { 48 unsigned long p4d; 49} p4d_t; 50 51#define p4d_val(x) ((x).p4d) 52#define __p4d(x) ((p4d_t) { (x) }) 53#define PTRS_PER_P4D (PAGE_SIZE / sizeof(p4d_t)) 54 55/* Page Upper Directory entry */ 56typedef struct { 57 unsigned long pud; 58} pud_t; 59 60#define pud_val(x) ((x).pud) 61#define __pud(x) ((pud_t) { (x) }) 62#define PTRS_PER_PUD (PAGE_SIZE / sizeof(pud_t)) 63 64/* Page Middle Directory entry */ 65typedef struct { 66 unsigned long pmd; 67} pmd_t; 68 69#define pmd_val(x) ((x).pmd) 70#define __pmd(x) ((pmd_t) { (x) }) 71 72#define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) 73 74/* 75 * rv64 PTE format: 76 * | 63 | 62 61 | 60 54 | 53 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 77 * N MT RSV PFN reserved for SW D A G U X W R V 78 */ 79#define _PAGE_PFN_MASK GENMASK(53, 10) 80 81/* 82 * [63] Svnapot definitions: 83 * 0 Svnapot disabled 84 * 1 Svnapot enabled 85 */ 86#define _PAGE_NAPOT_SHIFT 63 87#define _PAGE_NAPOT BIT(_PAGE_NAPOT_SHIFT) 88/* 89 * Only 64KB (order 4) napot ptes supported. 90 */ 91#define NAPOT_CONT_ORDER_BASE 4 92enum napot_cont_order { 93 NAPOT_CONT64KB_ORDER = NAPOT_CONT_ORDER_BASE, 94 NAPOT_ORDER_MAX, 95}; 96 97#define for_each_napot_order(order) \ 98 for (order = NAPOT_CONT_ORDER_BASE; order < NAPOT_ORDER_MAX; order++) 99#define for_each_napot_order_rev(order) \ 100 for (order = NAPOT_ORDER_MAX - 1; \ 101 order >= NAPOT_CONT_ORDER_BASE; order--) 102#define napot_cont_order(val) (__builtin_ctzl((val.pte >> _PAGE_PFN_SHIFT) << 1)) 103 104#define napot_cont_shift(order) ((order) + PAGE_SHIFT) 105#define napot_cont_size(order) BIT(napot_cont_shift(order)) 106#define napot_cont_mask(order) (~(napot_cont_size(order) - 1UL)) 107#define napot_pte_num(order) BIT(order) 108 109#ifdef CONFIG_RISCV_ISA_SVNAPOT 110#define HUGE_MAX_HSTATE (2 + (NAPOT_ORDER_MAX - NAPOT_CONT_ORDER_BASE)) 111#else 112#define HUGE_MAX_HSTATE 2 113#endif 114 115/* 116 * [62:61] Svpbmt Memory Type definitions: 117 * 118 * 00 - PMA Normal Cacheable, No change to implied PMA memory type 119 * 01 - NC Non-cacheable, idempotent, weakly-ordered Main Memory 120 * 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory 121 * 11 - Rsvd Reserved for future standard use 122 */ 123#define _PAGE_NOCACHE_SVPBMT (1UL << 61) 124#define _PAGE_IO_SVPBMT (1UL << 62) 125#define _PAGE_MTMASK_SVPBMT (_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT) 126 127/* 128 * [63:59] T-Head Memory Type definitions: 129 * bit[63] SO - Strong Order 130 * bit[62] C - Cacheable 131 * bit[61] B - Bufferable 132 * bit[60] SH - Shareable 133 * bit[59] Sec - Trustable 134 * 00110 - NC Weakly-ordered, Non-cacheable, Bufferable, Shareable, Non-trustable 135 * 01110 - PMA Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable 136 * 10010 - IO Strongly-ordered, Non-cacheable, Non-bufferable, Shareable, Non-trustable 137 */ 138#define _PAGE_PMA_THEAD ((1UL << 62) | (1UL << 61) | (1UL << 60)) 139#define _PAGE_NOCACHE_THEAD ((1UL << 61) | (1UL << 60)) 140#define _PAGE_IO_THEAD ((1UL << 63) | (1UL << 60)) 141#define _PAGE_MTMASK_THEAD (_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59)) 142 143static inline u64 riscv_page_mtmask(void) 144{ 145 u64 val; 146 147 ALT_SVPBMT(val, _PAGE_MTMASK); 148 return val; 149} 150 151static inline u64 riscv_page_nocache(void) 152{ 153 u64 val; 154 155 ALT_SVPBMT(val, _PAGE_NOCACHE); 156 return val; 157} 158 159static inline u64 riscv_page_io(void) 160{ 161 u64 val; 162 163 ALT_SVPBMT(val, _PAGE_IO); 164 return val; 165} 166 167#define _PAGE_NOCACHE riscv_page_nocache() 168#define _PAGE_IO riscv_page_io() 169#define _PAGE_MTMASK riscv_page_mtmask() 170 171/* Set of bits to preserve across pte_modify() */ 172#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ 173 _PAGE_WRITE | _PAGE_EXEC | \ 174 _PAGE_USER | _PAGE_GLOBAL | \ 175 _PAGE_MTMASK)) 176 177static inline int pud_present(pud_t pud) 178{ 179 return (pud_val(pud) & _PAGE_PRESENT); 180} 181 182static inline int pud_none(pud_t pud) 183{ 184 return (pud_val(pud) == 0); 185} 186 187static inline int pud_bad(pud_t pud) 188{ 189 return !pud_present(pud); 190} 191 192#define pud_leaf pud_leaf 193static inline bool pud_leaf(pud_t pud) 194{ 195 return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF); 196} 197 198static inline int pud_user(pud_t pud) 199{ 200 return pud_val(pud) & _PAGE_USER; 201} 202 203static inline void set_pud(pud_t *pudp, pud_t pud) 204{ 205 WRITE_ONCE(*pudp, pud); 206} 207 208static inline void pud_clear(pud_t *pudp) 209{ 210 set_pud(pudp, __pud(0)); 211} 212 213static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot) 214{ 215 return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 216} 217 218static inline unsigned long _pud_pfn(pud_t pud) 219{ 220 return __page_val_to_pfn(pud_val(pud)); 221} 222 223static inline pmd_t *pud_pgtable(pud_t pud) 224{ 225 return (pmd_t *)pfn_to_virt(__page_val_to_pfn(pud_val(pud))); 226} 227 228static inline struct page *pud_page(pud_t pud) 229{ 230 return pfn_to_page(__page_val_to_pfn(pud_val(pud))); 231} 232 233#define mm_p4d_folded mm_p4d_folded 234static inline bool mm_p4d_folded(struct mm_struct *mm) 235{ 236 if (pgtable_l5_enabled) 237 return false; 238 239 return true; 240} 241 242#define mm_pud_folded mm_pud_folded 243static inline bool mm_pud_folded(struct mm_struct *mm) 244{ 245 if (pgtable_l4_enabled) 246 return false; 247 248 return true; 249} 250 251#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) 252 253static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) 254{ 255 unsigned long prot_val = pgprot_val(prot); 256 257 ALT_THEAD_PMA(prot_val); 258 259 return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val); 260} 261 262static inline unsigned long _pmd_pfn(pmd_t pmd) 263{ 264 return __page_val_to_pfn(pmd_val(pmd)); 265} 266 267#define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) 268 269#define pmd_ERROR(e) \ 270 pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) 271 272#define pud_ERROR(e) \ 273 pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) 274 275#define p4d_ERROR(e) \ 276 pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) 277 278static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) 279{ 280 if (pgtable_l4_enabled) 281 WRITE_ONCE(*p4dp, p4d); 282 else 283 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); 284} 285 286static inline int p4d_none(p4d_t p4d) 287{ 288 if (pgtable_l4_enabled) 289 return (p4d_val(p4d) == 0); 290 291 return 0; 292} 293 294static inline int p4d_present(p4d_t p4d) 295{ 296 if (pgtable_l4_enabled) 297 return (p4d_val(p4d) & _PAGE_PRESENT); 298 299 return 1; 300} 301 302static inline int p4d_bad(p4d_t p4d) 303{ 304 if (pgtable_l4_enabled) 305 return !p4d_present(p4d); 306 307 return 0; 308} 309 310static inline void p4d_clear(p4d_t *p4d) 311{ 312 if (pgtable_l4_enabled) 313 set_p4d(p4d, __p4d(0)); 314} 315 316static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot) 317{ 318 return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 319} 320 321static inline unsigned long _p4d_pfn(p4d_t p4d) 322{ 323 return __page_val_to_pfn(p4d_val(p4d)); 324} 325 326static inline pud_t *p4d_pgtable(p4d_t p4d) 327{ 328 if (pgtable_l4_enabled) 329 return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d))); 330 331 return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); 332} 333#define p4d_page_vaddr(p4d) ((unsigned long)p4d_pgtable(p4d)) 334 335static inline struct page *p4d_page(p4d_t p4d) 336{ 337 return pfn_to_page(__page_val_to_pfn(p4d_val(p4d))); 338} 339 340#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) 341 342#define pud_offset pud_offset 343pud_t *pud_offset(p4d_t *p4d, unsigned long address); 344 345static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 346{ 347 if (pgtable_l5_enabled) 348 WRITE_ONCE(*pgdp, pgd); 349 else 350 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); 351} 352 353static inline int pgd_none(pgd_t pgd) 354{ 355 if (pgtable_l5_enabled) 356 return (pgd_val(pgd) == 0); 357 358 return 0; 359} 360 361static inline int pgd_present(pgd_t pgd) 362{ 363 if (pgtable_l5_enabled) 364 return (pgd_val(pgd) & _PAGE_PRESENT); 365 366 return 1; 367} 368 369static inline int pgd_bad(pgd_t pgd) 370{ 371 if (pgtable_l5_enabled) 372 return !pgd_present(pgd); 373 374 return 0; 375} 376 377static inline void pgd_clear(pgd_t *pgd) 378{ 379 if (pgtable_l5_enabled) 380 set_pgd(pgd, __pgd(0)); 381} 382 383static inline p4d_t *pgd_pgtable(pgd_t pgd) 384{ 385 if (pgtable_l5_enabled) 386 return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd))); 387 388 return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) }); 389} 390#define pgd_page_vaddr(pgd) ((unsigned long)pgd_pgtable(pgd)) 391 392static inline struct page *pgd_page(pgd_t pgd) 393{ 394 return pfn_to_page(__page_val_to_pfn(pgd_val(pgd))); 395} 396#define pgd_page(pgd) pgd_page(pgd) 397 398#define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) 399 400#define p4d_offset p4d_offset 401p4d_t *p4d_offset(pgd_t *pgd, unsigned long address); 402 403#endif /* _ASM_RISCV_PGTABLE_64_H */ 404