1/*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Fran��ois Tigeot 7 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31#ifndef _LINUXKPI_LINUX_MM_H_ 32#define _LINUXKPI_LINUX_MM_H_ 33 34#include <linux/spinlock.h> 35#include <linux/gfp.h> 36#include <linux/kernel.h> 37#include <linux/mm_types.h> 38#include <linux/pfn.h> 39#include <linux/list.h> 40#include <linux/mmap_lock.h> 41#include <linux/overflow.h> 42#include <linux/shrinker.h> 43#include <linux/page.h> 44 45#include <asm/pgtable.h> 46 47#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE) 48 49/* 50 * Make sure our LinuxKPI defined virtual memory flags don't conflict 51 * with the ones defined by FreeBSD: 52 */ 53CTASSERT((VM_PROT_ALL & -(1 << 8)) == 0); 54 55#define VM_READ VM_PROT_READ 56#define VM_WRITE VM_PROT_WRITE 57#define VM_EXEC VM_PROT_EXECUTE 58 59#define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 60 61#define VM_PFNINTERNAL (1 << 8) /* FreeBSD private flag to vm_insert_pfn() */ 62#define VM_MIXEDMAP (1 << 9) 63#define VM_NORESERVE (1 << 10) 64#define VM_PFNMAP (1 << 11) 65#define VM_IO (1 << 12) 66#define VM_MAYWRITE (1 << 13) 67#define VM_DONTCOPY (1 << 14) 68#define VM_DONTEXPAND (1 << 15) 69#define VM_DONTDUMP (1 << 16) 70#define VM_SHARED (1 << 17) 71 72#define VMA_MAX_PREFAULT_RECORD 1 73 74#define FOLL_WRITE (1 << 0) 75#define FOLL_FORCE (1 << 1) 76 77#define VM_FAULT_OOM (1 << 0) 78#define VM_FAULT_SIGBUS (1 << 1) 79#define VM_FAULT_MAJOR (1 << 2) 80#define VM_FAULT_WRITE (1 << 3) 81#define VM_FAULT_HWPOISON (1 << 4) 82#define VM_FAULT_HWPOISON_LARGE (1 << 5) 83#define VM_FAULT_SIGSEGV (1 << 6) 84#define VM_FAULT_NOPAGE (1 << 7) 85#define VM_FAULT_LOCKED (1 << 8) 86#define VM_FAULT_RETRY (1 << 9) 87#define VM_FAULT_FALLBACK (1 << 10) 88 89#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \ 90 VM_FAULT_HWPOISON |VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK) 91 92#define FAULT_FLAG_WRITE (1 << 0) 93#define FAULT_FLAG_MKWRITE (1 << 1) 94#define FAULT_FLAG_ALLOW_RETRY (1 << 2) 95#define FAULT_FLAG_RETRY_NOWAIT (1 << 3) 96#define FAULT_FLAG_KILLABLE (1 << 4) 97#define FAULT_FLAG_TRIED (1 << 5) 98#define FAULT_FLAG_USER (1 << 6) 99#define FAULT_FLAG_REMOTE (1 << 7) 100#define FAULT_FLAG_INSTRUCTION (1 << 8) 101 102#define fault_flag_allow_retry_first(flags) \ 103 (((flags) & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_TRIED)) == FAULT_FLAG_ALLOW_RETRY) 104 105typedef int (*pte_fn_t)(linux_pte_t *, unsigned long addr, void *data); 106 107struct vm_area_struct { 108 vm_offset_t vm_start; 109 vm_offset_t vm_end; 110 vm_offset_t vm_pgoff; 111 pgprot_t vm_page_prot; 112 unsigned long vm_flags; 113 struct mm_struct *vm_mm; 114 void *vm_private_data; 115 const struct vm_operations_struct *vm_ops; 116 struct linux_file *vm_file; 117 118 /* internal operation */ 119 vm_paddr_t vm_pfn; /* PFN for memory map */ 120 vm_size_t vm_len; /* length for memory map */ 121 vm_pindex_t vm_pfn_first; 122 int vm_pfn_count; 123 int *vm_pfn_pcount; 124 vm_object_t vm_obj; 125 vm_map_t vm_cached_map; 126 TAILQ_ENTRY(vm_area_struct) vm_entry; 127}; 128 129struct vm_fault { 130 unsigned int flags; 131 pgoff_t pgoff; 132 union { 133 /* user-space address */ 134 void *virtual_address; /* < 4.11 */ 135 unsigned long address; /* >= 4.11 */ 136 }; 137 struct page *page; 138 struct vm_area_struct *vma; 139}; 140 141struct vm_operations_struct { 142 void (*open) (struct vm_area_struct *); 143 void (*close) (struct vm_area_struct *); 144 int (*fault) (struct vm_fault *); 145 int (*access) (struct vm_area_struct *, unsigned long, void *, int, int); 146}; 147 148struct sysinfo { 149 uint64_t totalram; /* Total usable main memory size */ 150 uint64_t freeram; /* Available memory size */ 151 uint64_t totalhigh; /* Total high memory size */ 152 uint64_t freehigh; /* Available high memory size */ 153 uint32_t mem_unit; /* Memory unit size in bytes */ 154}; 155 156static inline struct page * 157virt_to_head_page(const void *p) 158{ 159 160 return (virt_to_page(p)); 161} 162 163/* 164 * Compute log2 of the power of two rounded up count of pages 165 * needed for size bytes. 166 */ 167static inline int 168get_order(unsigned long size) 169{ 170 int order; 171 172 size = (size - 1) >> PAGE_SHIFT; 173 order = 0; 174 while (size) { 175 order++; 176 size >>= 1; 177 } 178 return (order); 179} 180 181static inline void * 182lowmem_page_address(struct page *page) 183{ 184 return (page_address(page)); 185} 186 187/* 188 * This only works via memory map operations. 189 */ 190static inline int 191io_remap_pfn_range(struct vm_area_struct *vma, 192 unsigned long addr, unsigned long pfn, unsigned long size, 193 vm_memattr_t prot) 194{ 195 vma->vm_page_prot = prot; 196 vma->vm_pfn = pfn; 197 vma->vm_len = size; 198 199 return (0); 200} 201 202vm_fault_t 203lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr, 204 unsigned long pfn, pgprot_t prot); 205 206static inline vm_fault_t 207vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 208 unsigned long pfn, pgprot_t prot) 209{ 210 vm_fault_t ret; 211 212 VM_OBJECT_WLOCK(vma->vm_obj); 213 ret = lkpi_vmf_insert_pfn_prot_locked(vma, addr, pfn, prot); 214 VM_OBJECT_WUNLOCK(vma->vm_obj); 215 216 return (ret); 217} 218#define vmf_insert_pfn_prot(...) \ 219 _Static_assert(false, \ 220"This function is always called in a loop. Consider using the locked version") 221 222static inline int 223apply_to_page_range(struct mm_struct *mm, unsigned long address, 224 unsigned long size, pte_fn_t fn, void *data) 225{ 226 return (-ENOTSUP); 227} 228 229int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 230 unsigned long size); 231 232int lkpi_remap_pfn_range(struct vm_area_struct *vma, 233 unsigned long start_addr, unsigned long start_pfn, unsigned long size, 234 pgprot_t prot); 235 236static inline int 237remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 238 unsigned long pfn, unsigned long size, pgprot_t prot) 239{ 240 return (lkpi_remap_pfn_range(vma, addr, pfn, size, prot)); 241} 242 243static inline unsigned long 244vma_pages(struct vm_area_struct *vma) 245{ 246 return ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); 247} 248 249#define offset_in_page(off) ((unsigned long)(off) & (PAGE_SIZE - 1)) 250 251static inline void 252set_page_dirty(struct page *page) 253{ 254 vm_page_dirty(page); 255} 256 257static inline void 258mark_page_accessed(struct page *page) 259{ 260 vm_page_reference(page); 261} 262 263static inline void 264get_page(struct page *page) 265{ 266 vm_page_wire(page); 267} 268 269extern long 270get_user_pages(unsigned long start, unsigned long nr_pages, 271 unsigned int gup_flags, struct page **, 272 struct vm_area_struct **); 273 274static inline long 275pin_user_pages(unsigned long start, unsigned long nr_pages, 276 unsigned int gup_flags, struct page **pages, 277 struct vm_area_struct **vmas) 278{ 279 return get_user_pages(start, nr_pages, gup_flags, pages, vmas); 280} 281 282extern int 283__get_user_pages_fast(unsigned long start, int nr_pages, int write, 284 struct page **); 285 286static inline int 287pin_user_pages_fast(unsigned long start, int nr_pages, 288 unsigned int gup_flags, struct page **pages) 289{ 290 return __get_user_pages_fast( 291 start, nr_pages, !!(gup_flags & FOLL_WRITE), pages); 292} 293 294extern long 295get_user_pages_remote(struct task_struct *, struct mm_struct *, 296 unsigned long start, unsigned long nr_pages, 297 unsigned int gup_flags, struct page **, 298 struct vm_area_struct **); 299 300static inline long 301pin_user_pages_remote(struct task_struct *task, struct mm_struct *mm, 302 unsigned long start, unsigned long nr_pages, 303 unsigned int gup_flags, struct page **pages, 304 struct vm_area_struct **vmas) 305{ 306 return get_user_pages_remote( 307 task, mm, start, nr_pages, gup_flags, pages, vmas); 308} 309 310static inline void 311put_page(struct page *page) 312{ 313 vm_page_unwire(page, PQ_ACTIVE); 314} 315 316#define unpin_user_page(page) put_page(page) 317#define unpin_user_pages(pages, npages) release_pages(pages, npages) 318 319#define copy_highpage(to, from) pmap_copy_page(from, to) 320 321static inline pgprot_t 322vm_get_page_prot(unsigned long vm_flags) 323{ 324 return (vm_flags & VM_PROT_ALL); 325} 326 327static inline void 328vm_flags_set(struct vm_area_struct *vma, unsigned long flags) 329{ 330 vma->vm_flags |= flags; 331} 332 333static inline void 334vm_flags_clear(struct vm_area_struct *vma, unsigned long flags) 335{ 336 vma->vm_flags &= ~flags; 337} 338 339static inline struct page * 340vmalloc_to_page(const void *addr) 341{ 342 vm_paddr_t paddr; 343 344 paddr = pmap_kextract((vm_offset_t)addr); 345 return (PHYS_TO_VM_PAGE(paddr)); 346} 347 348static inline int 349trylock_page(struct page *page) 350{ 351 return (vm_page_trylock(page)); 352} 353 354static inline void 355unlock_page(struct page *page) 356{ 357 358 vm_page_unlock(page); 359} 360 361extern int is_vmalloc_addr(const void *addr); 362void si_meminfo(struct sysinfo *si); 363 364static inline unsigned long 365totalram_pages(void) 366{ 367 return ((unsigned long)physmem); 368} 369 370#define unmap_mapping_range(...) lkpi_unmap_mapping_range(__VA_ARGS__) 371void lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused, 372 loff_t const holelen, int even_cows __unused); 373 374#define PAGE_ALIGNED(p) __is_aligned(p, PAGE_SIZE) 375 376void vma_set_file(struct vm_area_struct *vma, struct linux_file *file); 377 378static inline void 379might_alloc(gfp_t gfp_mask __unused) 380{ 381} 382 383#define is_cow_mapping(flags) (false) 384 385static inline bool 386want_init_on_free(void) 387{ 388 return (false); 389} 390 391#endif /* _LINUXKPI_LINUX_MM_H_ */ 392