1/*
2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
49 *  School of Computer Science
50 *  Carnegie Mellon University
51 *  Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 *	File:	vm/vm_pageout.h
60 *	Author:	Avadis Tevanian, Jr.
61 *	Date:	1986
62 *
63 *	Declarations for the pageout daemon interface.
64 */
65
66#ifndef	_VM_VM_PAGEOUT_H_
67#define _VM_VM_PAGEOUT_H_
68
69#ifdef	KERNEL_PRIVATE
70
71#include <mach/mach_types.h>
72#include <mach/boolean.h>
73#include <mach/machine/vm_types.h>
74#include <mach/memory_object_types.h>
75
76#include <kern/kern_types.h>
77#include <kern/locks.h>
78
79#include <libkern/OSAtomic.h>
80
81
82#include <vm/vm_options.h>
83
84#ifdef	MACH_KERNEL_PRIVATE
85#include <vm/vm_page.h>
86#endif
87
88#include <sys/kdebug.h>
89
90#define VM_PAGE_AVAILABLE_COUNT()		((unsigned int)(vm_page_cleaned_count))
91
92/* externally manipulated counters */
93extern unsigned int vm_pageout_cleaned_reactivated, vm_pageout_cleaned_fault_reactivated, vm_pageout_cleaned_commit_reactivated;
94
95#if CONFIG_FREEZE
96extern boolean_t memorystatus_freeze_enabled;
97#define VM_DYNAMIC_PAGING_ENABLED(port) (COMPRESSED_PAGER_IS_ACTIVE || (memorystatus_freeze_enabled == FALSE && IP_VALID(port)))
98#else
99#define VM_DYNAMIC_PAGING_ENABLED(port) (COMPRESSED_PAGER_IS_ACTIVE || IP_VALID(port))
100#endif
101
102#if VM_PRESSURE_EVENTS
103extern boolean_t vm_pressure_events_enabled;
104#endif /* VM_PRESSURE_EVENTS */
105
106extern int	vm_debug_events;
107
108#define VMF_CHECK_ZFDELAY		0x100
109#define VMF_COWDELAY			0x101
110#define VMF_ZFDELAY			0x102
111#define VMF_COMPRESSORDELAY		0x103
112
113#define VM_PAGEOUT_SCAN			0x104
114#define VM_PAGEOUT_BALANCE		0x105
115#define VM_PAGEOUT_FREELIST		0x106
116#define VM_PAGEOUT_PURGEONE		0x107
117#define VM_PAGEOUT_CACHE_EVICT		0x108
118#define VM_PAGEOUT_THREAD_BLOCK		0x109
119#define VM_PAGEOUT_JETSAM		0x10A
120
121#define VM_UPL_PAGE_WAIT		0x120
122#define VM_IOPL_PAGE_WAIT		0x121
123#define VM_PAGE_WAIT_BLOCK		0x122
124
125#if CONFIG_IOSCHED
126#define VM_PAGE_SLEEP			0x123
127#define VM_PAGE_EXPEDITE		0x124
128#endif
129
130#define VM_PRESSURE_EVENT		0x130
131#define VM_EXECVE			0x131
132#define VM_WAKEUP_COMPACTOR_SWAPPER	0x132
133
134#define VM_DEBUG_EVENT(name, event, control, arg1, arg2, arg3, arg4)	\
135	MACRO_BEGIN						\
136	if (vm_debug_events) {					\
137		KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \
138	}							\
139	MACRO_END
140
141extern void memoryshot(unsigned int event, unsigned int control);
142
143extern kern_return_t vm_map_create_upl(
144	vm_map_t		map,
145	vm_map_address_t	offset,
146	upl_size_t		*upl_size,
147	upl_t			*upl,
148	upl_page_info_array_t	page_list,
149	unsigned int		*count,
150	int			*flags);
151
152extern ppnum_t upl_get_highest_page(
153	upl_t			upl);
154
155extern upl_size_t upl_get_size(
156	upl_t			upl);
157
158extern void iopl_valid_data(
159	upl_t			upl_ptr);
160
161#ifndef	MACH_KERNEL_PRIVATE
162typedef struct vm_page	*vm_page_t;
163#endif
164
165extern void                vm_page_free_list(
166                            vm_page_t	mem,
167                            boolean_t	prepare_object);
168
169extern kern_return_t      vm_page_alloc_list(
170                            int         page_count,
171                            int			flags,
172                            vm_page_t * list);
173
174extern void               vm_page_set_offset(vm_page_t page, vm_object_offset_t offset);
175extern vm_object_offset_t vm_page_get_offset(vm_page_t page);
176extern ppnum_t            vm_page_get_phys_page(vm_page_t page);
177extern vm_page_t          vm_page_get_next(vm_page_t page);
178
179extern kern_return_t	mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level);
180
181#ifdef	MACH_KERNEL_PRIVATE
182
183#include <vm/vm_page.h>
184
185extern unsigned int	vm_pageout_scan_event_counter;
186extern unsigned int	vm_page_anonymous_count;
187
188
189/*
190 * must hold the page queues lock to
191 * manipulate this structure
192 */
193struct vm_pageout_queue {
194        queue_head_t	pgo_pending;	/* laundry pages to be processed by pager's iothread */
195        unsigned int	pgo_laundry;	/* current count of laundry pages on queue or in flight */
196        unsigned int	pgo_maxlaundry;
197	uint64_t	pgo_tid;	/* thread ID of I/O thread that services this queue */
198	uint8_t		pgo_lowpriority; /* iothread is set to use low priority I/O */
199
200        unsigned int	pgo_idle:1,	/* iothread is blocked waiting for work to do */
201	                pgo_busy:1,     /* iothread is currently processing request from pgo_pending */
202			pgo_throttled:1,/* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */
203		        pgo_draining:1,
204			pgo_inited:1,
205			:0;
206};
207
208#define VM_PAGE_Q_THROTTLED(q)		\
209        ((q)->pgo_laundry >= (q)->pgo_maxlaundry)
210
211extern struct	vm_pageout_queue	vm_pageout_queue_internal;
212extern struct	vm_pageout_queue	vm_pageout_queue_external;
213
214
215/*
216 *	Routines exported to Mach.
217 */
218extern void		vm_pageout(void);
219
220extern kern_return_t	vm_pageout_internal_start(void);
221
222extern void		vm_pageout_object_terminate(
223					vm_object_t	object);
224
225extern void		vm_pageout_cluster(
226	                                vm_page_t	m,
227					boolean_t	pageout);
228
229extern void		vm_pageout_initialize_page(
230					vm_page_t	m);
231
232extern void		vm_pageclean_setup(
233					vm_page_t		m,
234					vm_page_t		new_m,
235					vm_object_t		new_object,
236					vm_object_offset_t	new_offset);
237
238/* UPL exported routines and structures */
239
240#define upl_lock_init(object)	lck_mtx_init(&(object)->Lock, &vm_object_lck_grp, &vm_object_lck_attr)
241#define upl_lock_destroy(object)	lck_mtx_destroy(&(object)->Lock, &vm_object_lck_grp)
242#define upl_lock(object)	lck_mtx_lock(&(object)->Lock)
243#define upl_unlock(object)	lck_mtx_unlock(&(object)->Lock)
244
245#define MAX_VECTOR_UPL_ELEMENTS	8
246
247struct _vector_upl_iostates{
248	upl_offset_t offset;
249	upl_size_t   size;
250};
251
252typedef struct _vector_upl_iostates vector_upl_iostates_t;
253
254struct _vector_upl {
255	upl_size_t		size;
256	uint32_t		num_upls;
257	uint32_t		invalid_upls;
258	uint32_t		_reserved;
259	vm_map_t		submap;
260	vm_offset_t		submap_dst_addr;
261	vm_object_offset_t	offset;
262	upl_t			upl_elems[MAX_VECTOR_UPL_ELEMENTS];
263	upl_page_info_array_t	pagelist;
264	vector_upl_iostates_t	upl_iostates[MAX_VECTOR_UPL_ELEMENTS];
265};
266
267typedef struct _vector_upl* vector_upl_t;
268
269/* universal page list structure */
270
271#if UPL_DEBUG
272#define	UPL_DEBUG_STACK_FRAMES	16
273#define UPL_DEBUG_COMMIT_RECORDS 4
274
275struct ucd {
276	upl_offset_t	c_beg;
277	upl_offset_t	c_end;
278	int		c_aborted;
279	void *		c_retaddr[UPL_DEBUG_STACK_FRAMES];
280};
281#endif
282
283
284struct upl {
285	decl_lck_mtx_data(,	Lock)	/* Synchronization */
286	int		ref_count;
287	int		ext_ref_count;
288	int		flags;
289	vm_object_t	src_object; /* object derived from */
290	vm_object_offset_t offset;
291	upl_size_t	size;	    /* size in bytes of the address space */
292	vm_offset_t	kaddr;      /* secondary mapping in kernel */
293	vm_object_t	map_object;
294	ppnum_t		highest_page;
295	void*		vector_upl;
296#if CONFIG_IOSCHED
297	int 		upl_priority;
298	uint64_t        *upl_reprio_info;
299	void 		*decmp_io_upl;
300#endif
301#if CONFIG_IOSCHED || UPL_DEBUG
302	thread_t        upl_creator;
303	queue_chain_t	uplq;	    /* List of outstanding upls on an obj */
304#endif
305#if	UPL_DEBUG
306	uintptr_t	ubc_alias1;
307	uintptr_t 	ubc_alias2;
308
309	uint32_t	upl_state;
310	uint32_t	upl_commit_index;
311	void	*upl_create_retaddr[UPL_DEBUG_STACK_FRAMES];
312
313	struct  ucd	upl_commit_records[UPL_DEBUG_COMMIT_RECORDS];
314#endif	/* UPL_DEBUG */
315};
316
317/* upl struct flags */
318#define UPL_PAGE_LIST_MAPPED	0x1
319#define UPL_KERNEL_MAPPED 	0x2
320#define	UPL_CLEAR_DIRTY		0x4
321#define UPL_COMPOSITE_LIST	0x8
322#define UPL_INTERNAL		0x10
323#define UPL_PAGE_SYNC_DONE	0x20
324#define UPL_DEVICE_MEMORY	0x40
325#define UPL_PAGEOUT		0x80
326#define UPL_LITE		0x100
327#define UPL_IO_WIRE		0x200
328#define UPL_ACCESS_BLOCKED	0x400
329#define UPL_ENCRYPTED		0x800
330#define UPL_SHADOWED		0x1000
331#define UPL_KERNEL_OBJECT	0x2000
332#define UPL_VECTOR		0x4000
333#define UPL_SET_DIRTY		0x8000
334#define UPL_HAS_BUSY		0x10000
335#define UPL_TRACKED_BY_OBJECT	0x20000
336#define UPL_EXPEDITE_SUPPORTED 	0x40000
337#define UPL_DECMP_REQ 		0x80000
338#define UPL_DECMP_REAL_IO 	0x100000
339
340/* flags for upl_create flags parameter */
341#define UPL_CREATE_EXTERNAL	0
342#define UPL_CREATE_INTERNAL	0x1
343#define UPL_CREATE_LITE		0x2
344#define UPL_CREATE_IO_TRACKING	0x4
345#define UPL_CREATE_EXPEDITE_SUP 0x8
346
347extern upl_t vector_upl_create(vm_offset_t);
348extern void vector_upl_deallocate(upl_t);
349extern boolean_t vector_upl_is_valid(upl_t);
350extern boolean_t vector_upl_set_subupl(upl_t, upl_t, u_int32_t);
351extern void vector_upl_set_pagelist(upl_t);
352extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t);
353extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*);
354extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t);
355extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*);
356extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*);
357extern upl_t vector_upl_subupl_byindex(upl_t , uint32_t);
358extern upl_t vector_upl_subupl_byoffset(upl_t , upl_offset_t*, upl_size_t*);
359
360extern void vm_object_set_pmap_cache_attr(
361		vm_object_t		object,
362		upl_page_info_array_t	user_page_list,
363		unsigned int		num_pages,
364		boolean_t		batch_pmap_op);
365
366extern kern_return_t vm_object_iopl_request(
367	vm_object_t		object,
368	vm_object_offset_t	offset,
369	upl_size_t		size,
370	upl_t			*upl_ptr,
371	upl_page_info_array_t	user_page_list,
372	unsigned int		*page_list_count,
373	int			cntrl_flags);
374
375extern kern_return_t vm_object_super_upl_request(
376	vm_object_t		object,
377	vm_object_offset_t	offset,
378	upl_size_t		size,
379	upl_size_t		super_cluster,
380	upl_t			*upl,
381	upl_page_info_t		*user_page_list,
382	unsigned int		*page_list_count,
383	int			cntrl_flags);
384
385/* should be just a regular vm_map_enter() */
386extern kern_return_t vm_map_enter_upl(
387	vm_map_t		map,
388	upl_t			upl,
389	vm_map_offset_t		*dst_addr);
390
391/* should be just a regular vm_map_remove() */
392extern kern_return_t vm_map_remove_upl(
393	vm_map_t		map,
394	upl_t			upl);
395
396/* wired  page list structure */
397typedef uint32_t *wpl_array_t;
398
399extern void vm_page_free_reserve(int pages);
400
401extern void vm_pageout_throttle_down(vm_page_t page);
402extern void vm_pageout_throttle_up(vm_page_t page);
403
404/*
405 * ENCRYPTED SWAP:
406 */
407extern void upl_encrypt(
408	upl_t			upl,
409	upl_offset_t		crypt_offset,
410	upl_size_t		crypt_size);
411extern void vm_page_encrypt(
412	vm_page_t		page,
413	vm_map_offset_t		kernel_map_offset);
414extern boolean_t vm_pages_encrypted; /* are there encrypted pages ? */
415extern void vm_page_decrypt(
416	vm_page_t		page,
417	vm_map_offset_t		kernel_map_offset);
418extern kern_return_t vm_paging_map_object(
419	vm_page_t		page,
420	vm_object_t		object,
421	vm_object_offset_t	offset,
422	vm_prot_t		protection,
423	boolean_t		can_unlock_object,
424	vm_map_size_t		*size,		/* IN/OUT */
425	vm_map_offset_t		*address,	/* OUT */
426	boolean_t		*need_unmap);	/* OUT */
427extern void vm_paging_unmap_object(
428	vm_object_t		object,
429	vm_map_offset_t		start,
430	vm_map_offset_t		end);
431decl_simple_lock_data(extern, vm_paging_lock)
432
433/*
434 * Backing store throttle when BS is exhausted
435 */
436extern unsigned int    vm_backing_store_low;
437
438extern void vm_pageout_steal_laundry(
439	vm_page_t page,
440	boolean_t queues_locked);
441
442extern boolean_t vm_page_is_slideable(vm_page_t m);
443
444extern kern_return_t vm_page_slide(vm_page_t page, vm_map_offset_t kernel_mapping_offset);
445#endif  /* MACH_KERNEL_PRIVATE */
446
447#if UPL_DEBUG
448extern kern_return_t  upl_ubc_alias_set(
449	upl_t upl,
450	uintptr_t alias1,
451	uintptr_t alias2);
452extern int  upl_ubc_alias_get(
453	upl_t upl,
454	uintptr_t * al,
455	uintptr_t * al2);
456#endif /* UPL_DEBUG */
457
458extern void vm_countdirtypages(void);
459
460extern void vm_backing_store_disable(
461			boolean_t	suspend);
462
463extern kern_return_t upl_transpose(
464	upl_t	upl1,
465	upl_t	upl2);
466
467extern kern_return_t mach_vm_pressure_monitor(
468	boolean_t	wait_for_pressure,
469	unsigned int	nsecs_monitored,
470	unsigned int	*pages_reclaimed_p,
471	unsigned int	*pages_wanted_p);
472
473extern kern_return_t
474vm_set_buffer_cleanup_callout(
475	boolean_t	(*func)(int));
476
477struct vm_page_stats_reusable {
478	SInt32		reusable_count;
479	uint64_t	reusable;
480	uint64_t	reused;
481	uint64_t	reused_wire;
482	uint64_t	reused_remove;
483	uint64_t	all_reusable_calls;
484	uint64_t	partial_reusable_calls;
485	uint64_t	all_reuse_calls;
486	uint64_t	partial_reuse_calls;
487	uint64_t	reusable_pages_success;
488	uint64_t	reusable_pages_failure;
489	uint64_t	reusable_pages_shared;
490	uint64_t	reuse_pages_success;
491	uint64_t	reuse_pages_failure;
492	uint64_t	can_reuse_success;
493	uint64_t	can_reuse_failure;
494	uint64_t	reusable_reclaimed;
495};
496extern struct vm_page_stats_reusable vm_page_stats_reusable;
497
498extern int hibernate_flush_memory(void);
499extern void hibernate_reset_stats(void);
500extern void hibernate_create_paddr_map(void);
501
502extern int vm_compressor_mode;
503extern int vm_compressor_thread_count;
504
505#define VM_PAGER_DEFAULT				0x1	/* Use default pager. */
506#define VM_PAGER_COMPRESSOR_NO_SWAP			0x2	/* In-core compressor only. */
507#define VM_PAGER_COMPRESSOR_WITH_SWAP			0x4	/* In-core compressor + swap backend. */
508#define VM_PAGER_FREEZER_DEFAULT			0x8	/* Freezer backed by default pager.*/
509#define VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP		0x10	/* Freezer backed by in-core compressor only i.e. frozen data remain in-core compressed.*/
510#define VM_PAGER_FREEZER_COMPRESSOR_WITH_SWAP		0x20	/* Freezer backed by in-core compressor with swap support too.*/
511
512#define VM_PAGER_MAX_MODES				6	/* Total number of vm compressor modes supported */
513
514#define DEFAULT_PAGER_IS_ACTIVE		((vm_compressor_mode & VM_PAGER_DEFAULT) == VM_PAGER_DEFAULT)
515
516#define COMPRESSED_PAGER_IS_ACTIVE	(vm_compressor_mode & (VM_PAGER_COMPRESSOR_NO_SWAP | VM_PAGER_COMPRESSOR_WITH_SWAP))
517
518#define DEFAULT_FREEZER_IS_ACTIVE	((vm_compressor_mode & VM_PAGER_FREEZER_DEFAULT) == VM_PAGER_FREEZER_DEFAULT)
519
520#define DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE		(vm_compressor_mode & (VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP | VM_PAGER_FREEZER_COMPRESSOR_WITH_SWAP))
521#define DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS		((vm_compressor_mode & VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP) == VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP)
522#define DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED		((vm_compressor_mode & VM_PAGER_FREEZER_COMPRESSOR_WITH_SWAP) == VM_PAGER_FREEZER_COMPRESSOR_WITH_SWAP)
523
524
525#endif	/* KERNEL_PRIVATE */
526
527#endif	/* _VM_VM_PAGEOUT_H_ */
528