1/*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
6 * Copyright (c) 2015 Fran��ois Tigeot
7 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com>
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice unmodified, this list of conditions, and the following
15 *    disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 *    notice, this list of conditions and the following disclaimer in the
18 *    documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31#ifndef	_LINUXKPI_LINUX_MM_H_
32#define	_LINUXKPI_LINUX_MM_H_
33
34#include <linux/spinlock.h>
35#include <linux/gfp.h>
36#include <linux/kernel.h>
37#include <linux/mm_types.h>
38#include <linux/pfn.h>
39#include <linux/list.h>
40#include <linux/mmap_lock.h>
41#include <linux/overflow.h>
42#include <linux/shrinker.h>
43#include <linux/page.h>
44
45#include <asm/pgtable.h>
46
47#define	PAGE_ALIGN(x)	ALIGN(x, PAGE_SIZE)
48
49/*
50 * Make sure our LinuxKPI defined virtual memory flags don't conflict
51 * with the ones defined by FreeBSD:
52 */
53CTASSERT((VM_PROT_ALL & -(1 << 8)) == 0);
54
55#define	VM_READ			VM_PROT_READ
56#define	VM_WRITE		VM_PROT_WRITE
57#define	VM_EXEC			VM_PROT_EXECUTE
58
59#define	VM_ACCESS_FLAGS		(VM_READ | VM_WRITE | VM_EXEC)
60
61#define	VM_PFNINTERNAL		(1 << 8)	/* FreeBSD private flag to vm_insert_pfn() */
62#define	VM_MIXEDMAP		(1 << 9)
63#define	VM_NORESERVE		(1 << 10)
64#define	VM_PFNMAP		(1 << 11)
65#define	VM_IO			(1 << 12)
66#define	VM_MAYWRITE		(1 << 13)
67#define	VM_DONTCOPY		(1 << 14)
68#define	VM_DONTEXPAND		(1 << 15)
69#define	VM_DONTDUMP		(1 << 16)
70#define	VM_SHARED		(1 << 17)
71
72#define	VMA_MAX_PREFAULT_RECORD	1
73
74#define	FOLL_WRITE		(1 << 0)
75#define	FOLL_FORCE		(1 << 1)
76
77#define	VM_FAULT_OOM		(1 << 0)
78#define	VM_FAULT_SIGBUS		(1 << 1)
79#define	VM_FAULT_MAJOR		(1 << 2)
80#define	VM_FAULT_WRITE		(1 << 3)
81#define	VM_FAULT_HWPOISON	(1 << 4)
82#define	VM_FAULT_HWPOISON_LARGE	(1 << 5)
83#define	VM_FAULT_SIGSEGV	(1 << 6)
84#define	VM_FAULT_NOPAGE		(1 << 7)
85#define	VM_FAULT_LOCKED		(1 << 8)
86#define	VM_FAULT_RETRY		(1 << 9)
87#define	VM_FAULT_FALLBACK	(1 << 10)
88
89#define	VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
90	VM_FAULT_HWPOISON |VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK)
91
92#define	FAULT_FLAG_WRITE	(1 << 0)
93#define	FAULT_FLAG_MKWRITE	(1 << 1)
94#define	FAULT_FLAG_ALLOW_RETRY	(1 << 2)
95#define	FAULT_FLAG_RETRY_NOWAIT	(1 << 3)
96#define	FAULT_FLAG_KILLABLE	(1 << 4)
97#define	FAULT_FLAG_TRIED	(1 << 5)
98#define	FAULT_FLAG_USER		(1 << 6)
99#define	FAULT_FLAG_REMOTE	(1 << 7)
100#define	FAULT_FLAG_INSTRUCTION	(1 << 8)
101
102#define fault_flag_allow_retry_first(flags) \
103	(((flags) & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_TRIED)) == FAULT_FLAG_ALLOW_RETRY)
104
105typedef int (*pte_fn_t)(linux_pte_t *, unsigned long addr, void *data);
106
107struct vm_area_struct {
108	vm_offset_t vm_start;
109	vm_offset_t vm_end;
110	vm_offset_t vm_pgoff;
111	pgprot_t vm_page_prot;
112	unsigned long vm_flags;
113	struct mm_struct *vm_mm;
114	void   *vm_private_data;
115	const struct vm_operations_struct *vm_ops;
116	struct linux_file *vm_file;
117
118	/* internal operation */
119	vm_paddr_t vm_pfn;		/* PFN for memory map */
120	vm_size_t vm_len;		/* length for memory map */
121	vm_pindex_t vm_pfn_first;
122	int	vm_pfn_count;
123	int    *vm_pfn_pcount;
124	vm_object_t vm_obj;
125	vm_map_t vm_cached_map;
126	TAILQ_ENTRY(vm_area_struct) vm_entry;
127};
128
129struct vm_fault {
130	unsigned int flags;
131	pgoff_t	pgoff;
132	union {
133		/* user-space address */
134		void *virtual_address;	/* < 4.11 */
135		unsigned long address;	/* >= 4.11 */
136	};
137	struct page *page;
138	struct vm_area_struct *vma;
139};
140
141struct vm_operations_struct {
142	void    (*open) (struct vm_area_struct *);
143	void    (*close) (struct vm_area_struct *);
144	int     (*fault) (struct vm_fault *);
145	int	(*access) (struct vm_area_struct *, unsigned long, void *, int, int);
146};
147
148struct sysinfo {
149	uint64_t totalram;	/* Total usable main memory size */
150	uint64_t freeram;	/* Available memory size */
151	uint64_t totalhigh;	/* Total high memory size */
152	uint64_t freehigh;	/* Available high memory size */
153	uint32_t mem_unit;	/* Memory unit size in bytes */
154};
155
156static inline struct page *
157virt_to_head_page(const void *p)
158{
159
160	return (virt_to_page(p));
161}
162
163/*
164 * Compute log2 of the power of two rounded up count of pages
165 * needed for size bytes.
166 */
167static inline int
168get_order(unsigned long size)
169{
170	int order;
171
172	size = (size - 1) >> PAGE_SHIFT;
173	order = 0;
174	while (size) {
175		order++;
176		size >>= 1;
177	}
178	return (order);
179}
180
181static inline void *
182lowmem_page_address(struct page *page)
183{
184	return (page_address(page));
185}
186
187/*
188 * This only works via memory map operations.
189 */
190static inline int
191io_remap_pfn_range(struct vm_area_struct *vma,
192    unsigned long addr, unsigned long pfn, unsigned long size,
193    vm_memattr_t prot)
194{
195	vma->vm_page_prot = prot;
196	vma->vm_pfn = pfn;
197	vma->vm_len = size;
198
199	return (0);
200}
201
202vm_fault_t
203lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr,
204    unsigned long pfn, pgprot_t prot);
205
206static inline vm_fault_t
207vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
208    unsigned long pfn, pgprot_t prot)
209{
210	vm_fault_t ret;
211
212	VM_OBJECT_WLOCK(vma->vm_obj);
213	ret = lkpi_vmf_insert_pfn_prot_locked(vma, addr, pfn, prot);
214	VM_OBJECT_WUNLOCK(vma->vm_obj);
215
216	return (ret);
217}
218#define	vmf_insert_pfn_prot(...)	\
219	_Static_assert(false,		\
220"This function is always called in a loop. Consider using the locked version")
221
222static inline int
223apply_to_page_range(struct mm_struct *mm, unsigned long address,
224    unsigned long size, pte_fn_t fn, void *data)
225{
226	return (-ENOTSUP);
227}
228
229int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
230    unsigned long size);
231
232int lkpi_remap_pfn_range(struct vm_area_struct *vma,
233    unsigned long start_addr, unsigned long start_pfn, unsigned long size,
234    pgprot_t prot);
235
236static inline int
237remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
238    unsigned long pfn, unsigned long size, pgprot_t prot)
239{
240	return (lkpi_remap_pfn_range(vma, addr, pfn, size, prot));
241}
242
243static inline unsigned long
244vma_pages(struct vm_area_struct *vma)
245{
246	return ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
247}
248
249#define	offset_in_page(off)	((unsigned long)(off) & (PAGE_SIZE - 1))
250
251static inline void
252set_page_dirty(struct page *page)
253{
254	vm_page_dirty(page);
255}
256
257static inline void
258mark_page_accessed(struct page *page)
259{
260	vm_page_reference(page);
261}
262
263static inline void
264get_page(struct page *page)
265{
266	vm_page_wire(page);
267}
268
269extern long
270get_user_pages(unsigned long start, unsigned long nr_pages,
271    unsigned int gup_flags, struct page **,
272    struct vm_area_struct **);
273
274static inline long
275pin_user_pages(unsigned long start, unsigned long nr_pages,
276    unsigned int gup_flags, struct page **pages,
277    struct vm_area_struct **vmas)
278{
279	return get_user_pages(start, nr_pages, gup_flags, pages, vmas);
280}
281
282extern int
283__get_user_pages_fast(unsigned long start, int nr_pages, int write,
284    struct page **);
285
286static inline int
287pin_user_pages_fast(unsigned long start, int nr_pages,
288    unsigned int gup_flags, struct page **pages)
289{
290	return __get_user_pages_fast(
291	    start, nr_pages, !!(gup_flags & FOLL_WRITE), pages);
292}
293
294extern long
295get_user_pages_remote(struct task_struct *, struct mm_struct *,
296    unsigned long start, unsigned long nr_pages,
297    unsigned int gup_flags, struct page **,
298    struct vm_area_struct **);
299
300static inline long
301pin_user_pages_remote(struct task_struct *task, struct mm_struct *mm,
302    unsigned long start, unsigned long nr_pages,
303    unsigned int gup_flags, struct page **pages,
304    struct vm_area_struct **vmas)
305{
306	return get_user_pages_remote(
307	    task, mm, start, nr_pages, gup_flags, pages, vmas);
308}
309
310static inline void
311put_page(struct page *page)
312{
313	vm_page_unwire(page, PQ_ACTIVE);
314}
315
316#define	unpin_user_page(page) put_page(page)
317#define	unpin_user_pages(pages, npages) release_pages(pages, npages)
318
319#define	copy_highpage(to, from) pmap_copy_page(from, to)
320
321static inline pgprot_t
322vm_get_page_prot(unsigned long vm_flags)
323{
324	return (vm_flags & VM_PROT_ALL);
325}
326
327static inline void
328vm_flags_set(struct vm_area_struct *vma, unsigned long flags)
329{
330	vma->vm_flags |= flags;
331}
332
333static inline void
334vm_flags_clear(struct vm_area_struct *vma, unsigned long flags)
335{
336	vma->vm_flags &= ~flags;
337}
338
339static inline struct page *
340vmalloc_to_page(const void *addr)
341{
342	vm_paddr_t paddr;
343
344	paddr = pmap_kextract((vm_offset_t)addr);
345	return (PHYS_TO_VM_PAGE(paddr));
346}
347
348static inline int
349trylock_page(struct page *page)
350{
351	return (vm_page_trylock(page));
352}
353
354static inline void
355unlock_page(struct page *page)
356{
357
358	vm_page_unlock(page);
359}
360
361extern int is_vmalloc_addr(const void *addr);
362void si_meminfo(struct sysinfo *si);
363
364static inline unsigned long
365totalram_pages(void)
366{
367	return ((unsigned long)physmem);
368}
369
370#define	unmap_mapping_range(...)	lkpi_unmap_mapping_range(__VA_ARGS__)
371void lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused,
372    loff_t const holelen, int even_cows __unused);
373
374#define PAGE_ALIGNED(p)	__is_aligned(p, PAGE_SIZE)
375
376void vma_set_file(struct vm_area_struct *vma, struct linux_file *file);
377
378static inline void
379might_alloc(gfp_t gfp_mask __unused)
380{
381}
382
383#define	is_cow_mapping(flags)	(false)
384
385static inline bool
386want_init_on_free(void)
387{
388	return (false);
389}
390
391#endif					/* _LINUXKPI_LINUX_MM_H_ */
392