xref: /macosx-10.9.5/xnu-2422.115.4/bsd/kern/ubc_subr.c

/*
 * Copyright (c) 1999-2008 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 *	File:	ubc_subr.c
 *	Author:	Umesh Vaishampayan [umeshv@apple.com]
 *		05-Aug-1999	umeshv	Created.
 *
 *	Functions related to Unified Buffer cache.
 *
 * Caller of UBC functions MUST have a valid reference on the vnode.
 *
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/lock.h>
#include <sys/mman.h>
#include <sys/mount_internal.h>
#include <sys/vnode_internal.h>
#include <sys/ubc_internal.h>
#include <sys/ucred.h>
#include <sys/proc_internal.h>
#include <sys/kauth.h>
#include <sys/buf.h>
#include <sys/user.h>
#include <sys/codesign.h>

#include <mach/mach_types.h>
#include <mach/memory_object_types.h>
#include <mach/memory_object_control.h>
#include <mach/vm_map.h>
#include <mach/mach_vm.h>
#include <mach/upl.h>

#include <kern/kern_types.h>
#include <kern/kalloc.h>
#include <kern/zalloc.h>
#include <kern/thread.h>
#include <vm/vm_kern.h>
#include <vm/vm_protos.h> /* last */

#include <libkern/crypto/sha1.h>
#include <libkern/libkern.h>

#include <sys/kasl.h>
#include <sys/syslog.h>

#include <security/mac_framework.h>

/* XXX These should be in a BSD accessible Mach header, but aren't. */
extern kern_return_t memory_object_pages_resident(memory_object_control_t,
							boolean_t *);
extern kern_return_t	memory_object_signed(memory_object_control_t control,
					     boolean_t is_signed);
extern boolean_t	memory_object_is_slid(memory_object_control_t	control);
extern boolean_t	memory_object_is_signed(memory_object_control_t);

extern void Debugger(const char *message);


/* XXX no one uses this interface! */
kern_return_t ubc_page_op_with_control(
	memory_object_control_t	 control,
	off_t		         f_offset,
	int		         ops,
	ppnum_t	                 *phys_entryp,
	int		         *flagsp);


#if DIAGNOSTIC
#if defined(assert)
#undef assert
#endif
#define assert(cond)    \
    ((void) ((cond) ? 0 : panic("Assert failed: %s", # cond)))
#else
#include <kern/assert.h>
#endif /* DIAGNOSTIC */

static int ubc_info_init_internal(struct vnode *vp, int withfsize, off_t filesize);
static int ubc_umcallback(vnode_t, void *);
static int ubc_msync_internal(vnode_t, off_t, off_t, off_t *, int, int *);
static void ubc_cs_free(struct ubc_info *uip);

struct zone	*ubc_info_zone;


/*
 * CODESIGNING
 * Routines to navigate code signing data structures in the kernel...
 */

extern int cs_debug;

static boolean_t
cs_valid_range(
	const void *start,
	const void *end,
	const void *lower_bound,
	const void *upper_bound)
{
	if (upper_bound < lower_bound ||
	    end < start) {
		return FALSE;
	}

	if (start < lower_bound ||
	    end > upper_bound) {
		return FALSE;
	}

	return TRUE;
}

static void
hex_str(
	const unsigned char *hash,
	size_t len,
	char *buf)
{
	unsigned int n;
	for (n = 0; n < len; n++)
		snprintf(buf + 2*n, 3, "%02.2x", hash[n]);
}


/*
 * Locate the CodeDirectory from an embedded signature blob
 */
static const
CS_CodeDirectory *findCodeDirectory(
	const CS_SuperBlob *embedded,
	char *lower_bound,
	char *upper_bound)
{
	const CS_CodeDirectory *cd = NULL;

	if (embedded &&
	    cs_valid_range(embedded, embedded + 1, lower_bound, upper_bound) &&
	    ntohl(embedded->magic) == CSMAGIC_EMBEDDED_SIGNATURE) {
		const CS_BlobIndex *limit;
		const CS_BlobIndex *p;

		limit = &embedded->index[ntohl(embedded->count)];
		if (!cs_valid_range(&embedded->index[0], limit,
				    lower_bound, upper_bound)) {
			return NULL;
		}
		for (p = embedded->index; p < limit; ++p) {
			if (ntohl(p->type) == CSSLOT_CODEDIRECTORY) {
				const unsigned char *base;

				base = (const unsigned char *)embedded;
				cd = (const CS_CodeDirectory *)(base + ntohl(p->offset));
				break;
			}
		}
	} else {
		/*
		 * Detached signatures come as a bare CS_CodeDirectory,
		 * without a blob.
		 */
		cd = (const CS_CodeDirectory *) embedded;
	}

	if (cd &&
	    cs_valid_range(cd, cd + 1, lower_bound, upper_bound) &&
	    cs_valid_range(cd, (const char *) cd + ntohl(cd->length),
			   lower_bound, upper_bound) &&
	    cs_valid_range(cd, (const char *) cd + ntohl(cd->hashOffset),
			   lower_bound, upper_bound) &&
	    cs_valid_range(cd, (const char *) cd +
			   ntohl(cd->hashOffset) +
			   (ntohl(cd->nCodeSlots) * SHA1_RESULTLEN),
			   lower_bound, upper_bound) &&

	    ntohl(cd->magic) == CSMAGIC_CODEDIRECTORY) {
		return cd;
	}

	// not found or not a valid code directory
	return NULL;
}


/*
 * Locating a page hash
 */
static const unsigned char *
hashes(
	const CS_CodeDirectory *cd,
	unsigned page,
	char *lower_bound,
	char *upper_bound)
{
	const unsigned char *base, *top, *hash;
	uint32_t nCodeSlots = ntohl(cd->nCodeSlots);

	assert(cs_valid_range(cd, cd + 1, lower_bound, upper_bound));

	if((ntohl(cd->version) >= CS_SUPPORTSSCATTER) && (ntohl(cd->scatterOffset))) {
		/* Get first scatter struct */
		const SC_Scatter *scatter = (const SC_Scatter*)
			((const char*)cd + ntohl(cd->scatterOffset));
		uint32_t hashindex=0, scount, sbase=0;
		/* iterate all scatter structs */
		do {
			if((const char*)scatter > (const char*)cd + ntohl(cd->length)) {
				if(cs_debug) {
					printf("CODE SIGNING: Scatter extends past Code Directory\n");
				}
				return NULL;
			}

			scount = ntohl(scatter->count);
			uint32_t new_base = ntohl(scatter->base);

			/* last scatter? */
			if (scount == 0) {
				return NULL;
			}

			if((hashindex > 0) && (new_base <= sbase)) {
				if(cs_debug) {
					printf("CODE SIGNING: unordered Scatter, prev base %d, cur base %d\n",
					sbase, new_base);
				}
				return NULL;	/* unordered scatter array */
			}
			sbase = new_base;

			/* this scatter beyond page we're looking for? */
			if (sbase > page) {
				return NULL;
			}

			if (sbase+scount >= page) {
				/* Found the scatter struct that is
				 * referencing our page */

				/* base = address of first hash covered by scatter */
				base = (const unsigned char *)cd + ntohl(cd->hashOffset) +
					hashindex * SHA1_RESULTLEN;
				/* top = address of first hash after this scatter */
				top = base + scount * SHA1_RESULTLEN;
				if (!cs_valid_range(base, top, lower_bound,
						    upper_bound) ||
				    hashindex > nCodeSlots) {
					return NULL;
				}

				break;
			}

			/* this scatter struct is before the page we're looking
			 * for. Iterate. */
			hashindex+=scount;
			scatter++;
		} while(1);

		hash = base + (page - sbase) * SHA1_RESULTLEN;
	} else {
		base = (const unsigned char *)cd + ntohl(cd->hashOffset);
		top = base + nCodeSlots * SHA1_RESULTLEN;
		if (!cs_valid_range(base, top, lower_bound, upper_bound) ||
		    page > nCodeSlots) {
			return NULL;
		}
		assert(page < nCodeSlots);

		hash = base + page * SHA1_RESULTLEN;
	}

	if (!cs_valid_range(hash, hash + SHA1_RESULTLEN,
			    lower_bound, upper_bound)) {
		hash = NULL;
	}

	return hash;
}

/*
 * cs_validate_codedirectory
 *
 * Validate that pointers inside the code directory to make sure that
 * all offsets and lengths are constrained within the buffer.
 *
 * Parameters:	cd			Pointer to code directory buffer
 *		length			Length of buffer
 *
 * Returns:	0			Success
 *		EBADEXEC		Invalid code signature
 */

static int
cs_validate_codedirectory(const CS_CodeDirectory *cd, size_t length)
{

	if (length < sizeof(*cd))
		return EBADEXEC;
	if (ntohl(cd->magic) != CSMAGIC_CODEDIRECTORY)
		return EBADEXEC;
	if (cd->hashSize != SHA1_RESULTLEN)
		return EBADEXEC;
	if (cd->pageSize != PAGE_SHIFT)
		return EBADEXEC;
	if (cd->hashType != CS_HASHTYPE_SHA1)
		return EBADEXEC;

	if (length < ntohl(cd->hashOffset))
		return EBADEXEC;

	/* check that nSpecialSlots fits in the buffer in front of hashOffset */
	if (ntohl(cd->hashOffset) / SHA1_RESULTLEN < ntohl(cd->nSpecialSlots))
		return EBADEXEC;

	/* check that codeslots fits in the buffer */
	if ((length - ntohl(cd->hashOffset)) / SHA1_RESULTLEN <  ntohl(cd->nCodeSlots))
		return EBADEXEC;

	if (ntohl(cd->version) >= CS_SUPPORTSSCATTER && cd->scatterOffset) {

		if (length < ntohl(cd->scatterOffset))
			return EBADEXEC;

		SC_Scatter *scatter = (SC_Scatter *)
			(((uint8_t *)cd) + ntohl(cd->scatterOffset));
		uint32_t nPages = 0;

		/*
		 * Check each scatter buffer, since we don't know the
		 * length of the scatter buffer array, we have to
		 * check each entry.
		 */
		while(1) {
			/* check that the end of each scatter buffer in within the length */
			if (((const uint8_t *)scatter) + sizeof(scatter[0]) > (const uint8_t *)cd + length)
				return EBADEXEC;
			uint32_t scount = ntohl(scatter->count);
			if (scount == 0)
				break;
			if (nPages + scount < nPages)
				return EBADEXEC;
			nPages += scount;
			scatter++;

			/* XXX check that basees doesn't overlap */
			/* XXX check that targetOffset doesn't overlap */
		}
#if 0 /* rdar://12579439 */
		if (nPages != ntohl(cd->nCodeSlots))
			return EBADEXEC;
#endif
	}

	if (length < ntohl(cd->identOffset))
		return EBADEXEC;

	/* identifier is NUL terminated string */
	if (cd->identOffset) {
		uint8_t *ptr = (uint8_t *)cd + ntohl(cd->identOffset);
		if (memchr(ptr, 0, length - ntohl(cd->identOffset)) == NULL)
			return EBADEXEC;
	}

	return 0;
}

/*
 *
 */

static int
cs_validate_blob(const CS_GenericBlob *blob, size_t length)
{
	if (length < sizeof(CS_GenericBlob) || length < ntohl(blob->length))
		return EBADEXEC;
	return 0;
}

/*
 * cs_validate_csblob
 *
 * Validate that superblob/embedded code directory to make sure that
 * all internal pointers are valid.
 *
 * Will validate both a superblob csblob and a "raw" code directory.
 *
 *
 * Parameters:	buffer			Pointer to code signature
 *		length			Length of buffer
 *		rcd			returns pointer to code directory
 *
 * Returns:	0			Success
 *		EBADEXEC		Invalid code signature
 */

static int
cs_validate_csblob(const uint8_t *addr, size_t length,
		   const CS_CodeDirectory **rcd)
{
	const CS_GenericBlob *blob = (const CS_GenericBlob *)(void *)addr;
	int error;

	*rcd = NULL;

	error = cs_validate_blob(blob, length);
	if (error)
		return error;

	length = ntohl(blob->length);

	if (ntohl(blob->magic) == CSMAGIC_EMBEDDED_SIGNATURE) {
		const CS_SuperBlob *sb = (const CS_SuperBlob *)blob;
		uint32_t n, count = ntohl(sb->count);

		if (length < sizeof(CS_SuperBlob))
			return EBADEXEC;

		/* check that the array of BlobIndex fits in the rest of the data */
		if ((length - sizeof(CS_SuperBlob)) / sizeof(CS_BlobIndex) < count)
			return EBADEXEC;

		/* now check each BlobIndex */
		for (n = 0; n < count; n++) {
			const CS_BlobIndex *blobIndex = &sb->index[n];
			if (length < ntohl(blobIndex->offset))
				return EBADEXEC;

			const CS_GenericBlob *subBlob =
				(const CS_GenericBlob *)(void *)(addr + ntohl(blobIndex->offset));

			size_t subLength = length - ntohl(blobIndex->offset);

			if ((error = cs_validate_blob(subBlob, subLength)) != 0)
				return error;
			subLength = ntohl(subBlob->length);

			/* extra validation for CDs, that is also returned */
			if (ntohl(blobIndex->type) == CSSLOT_CODEDIRECTORY) {
				const CS_CodeDirectory *cd = (const CS_CodeDirectory *)subBlob;
				if ((error = cs_validate_codedirectory(cd, subLength)) != 0)
					return error;
				*rcd = cd;
			}
		}

	} else if (ntohl(blob->magic) == CSMAGIC_CODEDIRECTORY) {

		if ((error = cs_validate_codedirectory((const CS_CodeDirectory *)(void *)addr, length)) != 0)
			return error;
		*rcd = (const CS_CodeDirectory *)blob;
	} else {
		return EBADEXEC;
	}

	if (*rcd == NULL)
		return EBADEXEC;

	return 0;
}

/*
 * cs_find_blob_bytes
 *
 * Find an blob from the superblob/code directory. The blob must have
 * been been validated by cs_validate_csblob() before calling
 * this. Use cs_find_blob() instead.
 *
 * Will also find a "raw" code directory if its stored as well as
 * searching the superblob.
 *
 * Parameters:	buffer			Pointer to code signature
 *		length			Length of buffer
 *		type			type of blob to find
 *		magic			the magic number for that blob
 *
 * Returns:	pointer			Success
 *		NULL			Buffer not found
 */

static const CS_GenericBlob *
cs_find_blob_bytes(const uint8_t *addr, size_t length, uint32_t type, uint32_t magic)
{
	const CS_GenericBlob *blob = (const CS_GenericBlob *)(void *)addr;

	if (ntohl(blob->magic) == CSMAGIC_EMBEDDED_SIGNATURE) {
		const CS_SuperBlob *sb = (const CS_SuperBlob *)blob;
		size_t n, count = ntohl(sb->count);

		for (n = 0; n < count; n++) {
			if (ntohl(sb->index[n].type) != type)
				continue;
			uint32_t offset = ntohl(sb->index[n].offset);
			if (length - sizeof(const CS_GenericBlob) < offset)
				return NULL;
			blob = (const CS_GenericBlob *)(void *)(addr + offset);
			if (ntohl(blob->magic) != magic)
				continue;
			return blob;
		}
	} else if (type == CSSLOT_CODEDIRECTORY
		   && ntohl(blob->magic) == CSMAGIC_CODEDIRECTORY
		   && magic == CSMAGIC_CODEDIRECTORY)
		return blob;
	return NULL;
}


static const CS_GenericBlob *
cs_find_blob(struct cs_blob *csblob, uint32_t type, uint32_t magic)
{
	if ((csblob->csb_flags & CS_VALID) == 0)
		return NULL;
	return cs_find_blob_bytes((const uint8_t *)csblob->csb_mem_kaddr, csblob->csb_mem_size, type, magic);
}

static const uint8_t *
cs_find_special_slot(const CS_CodeDirectory *cd, uint32_t slot)
{
	/* there is no zero special slot since that is the first code slot */
	if (ntohl(cd->nSpecialSlots) < slot || slot == 0)
		return NULL;

	return ((const uint8_t *)cd + ntohl(cd->hashOffset) - (SHA1_RESULTLEN * slot));
}

/*
 * CODESIGNING
 * End of routines to navigate code signing data structures in the kernel.
 */

/*
 * ENTITLEMENTS
 * Routines to navigate entitlements in the kernel.
 */

/* Retrieve the entitlements blob for a process.
 * Returns:
 *   EINVAL	no text vnode associated with the process
 *   EBADEXEC   invalid code signing data
 *   0		no error occurred
 *
 * On success, out_start and out_length will point to the
 * entitlements blob if found; or will be set to NULL/zero
 * if there were no entitlements.
 */

static uint8_t sha1_zero[SHA1_RESULTLEN] = { 0 };

int
cs_entitlements_blob_get(proc_t p, void **out_start, size_t *out_length)
{
	uint8_t computed_hash[SHA1_RESULTLEN];
	const CS_GenericBlob *entitlements;
	const CS_CodeDirectory *code_dir;
	struct cs_blob *csblob;
	const uint8_t *embedded_hash;
	SHA1_CTX context;

	*out_start = NULL;
	*out_length = 0;

	if (NULL == p->p_textvp)
		return EINVAL;

	if ((csblob = ubc_cs_blob_get(p->p_textvp, -1, p->p_textoff)) == NULL)
		return 0;

	if ((code_dir = (const CS_CodeDirectory *)cs_find_blob(csblob, CSSLOT_CODEDIRECTORY, CSMAGIC_CODEDIRECTORY)) == NULL)
		return 0;

	entitlements = cs_find_blob(csblob, CSSLOT_ENTITLEMENTS, CSMAGIC_EMBEDDED_ENTITLEMENTS);
	embedded_hash = cs_find_special_slot(code_dir, CSSLOT_ENTITLEMENTS);

	if (embedded_hash == NULL) {
		if (entitlements)
			return EBADEXEC;
		return 0;
	} else if (entitlements == NULL && memcmp(embedded_hash, sha1_zero, SHA1_RESULTLEN) != 0) {
		return EBADEXEC;
	}

	SHA1Init(&context);
	SHA1Update(&context, entitlements, ntohl(entitlements->length));
	SHA1Final(computed_hash, &context);
	if (memcmp(computed_hash, embedded_hash, SHA1_RESULTLEN) != 0)
		return EBADEXEC;

	*out_start = (void *)entitlements;
	*out_length = ntohl(entitlements->length);

	return 0;
}

/* Retrieve the codesign identity for a process.
 * Returns:
 *   NULL	an error occured
 *   string	the cs_identity
 */

const char *
cs_identity_get(proc_t p)
{
	const CS_CodeDirectory *code_dir;
	struct cs_blob *csblob;

	if (NULL == p->p_textvp)
		return NULL;

	if ((csblob = ubc_cs_blob_get(p->p_textvp, -1, p->p_textoff)) == NULL)
		return NULL;

	if ((code_dir = (const CS_CodeDirectory *)cs_find_blob(csblob, CSSLOT_CODEDIRECTORY, CSMAGIC_CODEDIRECTORY)) == NULL)
		return NULL;

	if (code_dir->identOffset == 0)
		return NULL;

	return ((const char *)code_dir) + ntohl(code_dir->identOffset);
}



/* Retrieve the codesign blob for a process.
 * Returns:
 *   EINVAL	no text vnode associated with the process
 *   0		no error occurred
 *
 * On success, out_start and out_length will point to the
 * cms blob if found; or will be set to NULL/zero
 * if there were no blob.
 */

int
cs_blob_get(proc_t p, void **out_start, size_t *out_length)
{
	struct cs_blob *csblob;

	*out_start = NULL;
	*out_length = 0;

	if (NULL == p->p_textvp)
		return EINVAL;

	if ((csblob = ubc_cs_blob_get(p->p_textvp, -1, p->p_textoff)) == NULL)
		return 0;

	*out_start = (void *)csblob->csb_mem_kaddr;
	*out_length = csblob->csb_mem_size;

	return 0;
}

uint8_t *
cs_get_cdhash(struct proc *p)
{
	struct cs_blob *csblob;

	if (NULL == p->p_textvp)
		return NULL;

	if ((csblob = ubc_cs_blob_get(p->p_textvp, -1, p->p_textoff)) == NULL)
		return NULL;

	return csblob->csb_sha1;
}

/*
 * ENTITLEMENTS
 * End of routines to navigate entitlements in the kernel.
 */



/*
 * ubc_init
 *
 * Initialization of the zone for Unified Buffer Cache.
 *
 * Parameters:	(void)
 *
 * Returns:	(void)
 *
 * Implicit returns:
 *		ubc_info_zone(global)	initialized for subsequent allocations
 */
__private_extern__ void
ubc_init(void)
{
	int	i;

	i = (vm_size_t) sizeof (struct ubc_info);

	ubc_info_zone = zinit (i, 10000*i, 8192, "ubc_info zone");

	zone_change(ubc_info_zone, Z_NOENCRYPT, TRUE);
}


/*
 * ubc_info_init
 *
 * Allocate and attach an empty ubc_info structure to a vnode
 *
 * Parameters:	vp			Pointer to the vnode
 *
 * Returns:	0			Success
 *	vnode_size:ENOMEM		Not enough space
 *	vnode_size:???			Other error from vnode_getattr
 *
 */
int
ubc_info_init(struct vnode *vp)
{
	return(ubc_info_init_internal(vp, 0, 0));
}


/*
 * ubc_info_init_withsize
 *
 * Allocate and attach a sized ubc_info structure to a vnode
 *
 * Parameters:	vp			Pointer to the vnode
 *		filesize		The size of the file
 *
 * Returns:	0			Success
 *	vnode_size:ENOMEM		Not enough space
 *	vnode_size:???			Other error from vnode_getattr
 */
int
ubc_info_init_withsize(struct vnode *vp, off_t filesize)
{
	return(ubc_info_init_internal(vp, 1, filesize));
}


/*
 * ubc_info_init_internal
 *
 * Allocate and attach a ubc_info structure to a vnode
 *
 * Parameters:	vp			Pointer to the vnode
 *		withfsize{0,1}		Zero if the size should be obtained
 *					from the vnode; otherwise, use filesize
 *		filesize		The size of the file, if withfsize == 1
 *
 * Returns:	0			Success
 *	vnode_size:ENOMEM		Not enough space
 *	vnode_size:???			Other error from vnode_getattr
 *
 * Notes:	We call a blocking zalloc(), and the zone was created as an
 *		expandable and collectable zone, so if no memory is available,
 *		it is possible for zalloc() to block indefinitely.  zalloc()
 *		may also panic if the zone of zones is exhausted, since it's
 *		NOT expandable.
 *
 *		We unconditionally call vnode_pager_setup(), even if this is
 *		a reuse of a ubc_info; in that case, we should probably assert
 *		that it does not already have a pager association, but do not.
 *
 *		Since memory_object_create_named() can only fail from receiving
 *		an invalid pager argument, the explicit check and panic is
 *		merely precautionary.
 */
static int
ubc_info_init_internal(vnode_t vp, int withfsize, off_t filesize)
{
	register struct ubc_info	*uip;
	void *  pager;
	int error = 0;
	kern_return_t kret;
	memory_object_control_t control;

	uip = vp->v_ubcinfo;

	/*
	 * If there is not already a ubc_info attached to the vnode, we
	 * attach one; otherwise, we will reuse the one that's there.
	 */
	if (uip == UBC_INFO_NULL) {

		uip = (struct ubc_info *) zalloc(ubc_info_zone);
		bzero((char *)uip, sizeof(struct ubc_info));

		uip->ui_vnode = vp;
		uip->ui_flags = UI_INITED;
		uip->ui_ucred = NOCRED;
	}
	assert(uip->ui_flags != UI_NONE);
	assert(uip->ui_vnode == vp);

	/* now set this ubc_info in the vnode */
	vp->v_ubcinfo = uip;

	/*
	 * Allocate a pager object for this vnode
	 *
	 * XXX The value of the pager parameter is currently ignored.
	 * XXX Presumably, this API changed to avoid the race between
	 * XXX setting the pager and the UI_HASPAGER flag.
	 */
	pager = (void *)vnode_pager_setup(vp, uip->ui_pager);
	assert(pager);

	/*
	 * Explicitly set the pager into the ubc_info, after setting the
	 * UI_HASPAGER flag.
	 */
	SET(uip->ui_flags, UI_HASPAGER);
	uip->ui_pager = pager;

	/*
	 * Note: We can not use VNOP_GETATTR() to get accurate
	 * value of ui_size because this may be an NFS vnode, and
	 * nfs_getattr() can call vinvalbuf(); if this happens,
	 * ubc_info is not set up to deal with that event.
	 * So use bogus size.
	 */

	/*
	 * create a vnode - vm_object association
	 * memory_object_create_named() creates a "named" reference on the
	 * memory object we hold this reference as long as the vnode is
	 * "alive."  Since memory_object_create_named() took its own reference
	 * on the vnode pager we passed it, we can drop the reference
	 * vnode_pager_setup() returned here.
	 */
	kret = memory_object_create_named(pager,
		(memory_object_size_t)uip->ui_size, &control);
	vnode_pager_deallocate(pager);
	if (kret != KERN_SUCCESS)
		panic("ubc_info_init: memory_object_create_named returned %d", kret);

	assert(control);
	uip->ui_control = control;	/* cache the value of the mo control */
	SET(uip->ui_flags, UI_HASOBJREF);	/* with a named reference */

	if (withfsize == 0) {
		/* initialize the size */
		error = vnode_size(vp, &uip->ui_size, vfs_context_current());
		if (error)
			uip->ui_size = 0;
	} else {
		uip->ui_size = filesize;
	}
	vp->v_lflag |= VNAMED_UBC;	/* vnode has a named ubc reference */

	return (error);
}


/*
 * ubc_info_free
 *
 * Free a ubc_info structure
 *
 * Parameters:	uip			A pointer to the ubc_info to free
 *
 * Returns:	(void)
 *
 * Notes:	If there is a credential that has subsequently been associated
 *		with the ubc_info via a call to ubc_setcred(), the reference
 *		to the credential is dropped.
 *
 *		It's actually impossible for a ubc_info.ui_control to take the
 *		value MEMORY_OBJECT_CONTROL_NULL.
 */
static void
ubc_info_free(struct ubc_info *uip)
{
	if (IS_VALID_CRED(uip->ui_ucred)) {
		kauth_cred_unref(&uip->ui_ucred);
	}

	if (uip->ui_control != MEMORY_OBJECT_CONTROL_NULL)
		memory_object_control_deallocate(uip->ui_control);

	cluster_release(uip);
	ubc_cs_free(uip);

	zfree(ubc_info_zone, uip);
	return;
}


void
ubc_info_deallocate(struct ubc_info *uip)
{
        ubc_info_free(uip);
}


/*
 * ubc_setsize
 *
 * Tell the  VM that the the size of the file represented by the vnode has
 * changed
 *
 * Parameters:	vp			The vp whose backing file size is
 *					being changed
 *		nsize			The new size of the backing file
 *
 * Returns:	1			Success
 *		0			Failure
 *
 * Notes:	This function will indicate failure if the new size that's
 *		being attempted to be set is negative.
 *
 *		This function will fail if there is no ubc_info currently
 *		associated with the vnode.
 *
 *		This function will indicate success it the new size is the
 *		same or larger than the old size (in this case, the remainder
 *		of the file will require modification or use of an existing upl
 *		to access successfully).
 *
 *		This function will fail if the new file size is smaller, and
 *		the memory region being invalidated was unable to actually be
 *		invalidated and/or the last page could not be flushed, if the
 *		new size is not aligned to a page boundary.  This is usually
 *		indicative of an I/O error.
 */
int
ubc_setsize(struct vnode *vp, off_t nsize)
{
	off_t osize;	/* ui_size before change */
	off_t lastpg, olastpgend, lastoff;
	struct ubc_info *uip;
	memory_object_control_t control;
	kern_return_t kret = KERN_SUCCESS;

	if (nsize < (off_t)0)
		return (0);

	if (!UBCINFOEXISTS(vp))
		return (0);

	uip = vp->v_ubcinfo;
	osize = uip->ui_size;
	/*
	 * Update the size before flushing the VM
	 */
	uip->ui_size = nsize;

	if (nsize >= osize) {	/* Nothing more to do */
		if (nsize > osize) {
			lock_vnode_and_post(vp, NOTE_EXTEND);
		}

		return (1);		/* return success */
	}

	/*
	 * When the file shrinks, invalidate the pages beyond the
	 * new size. Also get rid of garbage beyond nsize on the
	 * last page. The ui_size already has the nsize, so any
	 * subsequent page-in will zero-fill the tail properly
	 */
	lastpg = trunc_page_64(nsize);
	olastpgend = round_page_64(osize);
	control = uip->ui_control;
	assert(control);
	lastoff = (nsize & PAGE_MASK_64);

	if (lastoff) {
	        upl_t		upl;
		upl_page_info_t	*pl;


	        /*
		 * new EOF ends up in the middle of a page
		 * zero the tail of this page if its currently
		 * present in the cache
		 */
	        kret = ubc_create_upl(vp, lastpg, PAGE_SIZE, &upl, &pl, UPL_SET_LITE);

		if (kret != KERN_SUCCESS)
		        panic("ubc_setsize: ubc_create_upl (error = %d)\n", kret);

		if (upl_valid_page(pl, 0))
		        cluster_zero(upl, (uint32_t)lastoff, PAGE_SIZE - (uint32_t)lastoff, NULL);

		ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);

		lastpg += PAGE_SIZE_64;
	}
	if (olastpgend > lastpg) {
		int	flags;

		if (lastpg == 0)
			flags = MEMORY_OBJECT_DATA_FLUSH_ALL;
		else
			flags = MEMORY_OBJECT_DATA_FLUSH;
	        /*
		 * invalidate the pages beyond the new EOF page
		 *
		 */
	        kret = memory_object_lock_request(control,
						  (memory_object_offset_t)lastpg,
						  (memory_object_size_t)(olastpgend - lastpg), NULL, NULL,
						  MEMORY_OBJECT_RETURN_NONE, flags, VM_PROT_NO_CHANGE);
		if (kret != KERN_SUCCESS)
		        printf("ubc_setsize: invalidate failed (error = %d)\n", kret);
	}
	return ((kret == KERN_SUCCESS) ? 1 : 0);
}


/*
 * ubc_getsize
 *
 * Get the size of the file assocated with the specified vnode
 *
 * Parameters:	vp			The vnode whose size is of interest
 *
 * Returns:	0			There is no ubc_info associated with
 *					this vnode, or the size is zero
 *		!0			The size of the file
 *
 * Notes:	Using this routine, it is not possible for a caller to
 *		successfully distinguish between a vnode associate with a zero
 *		length file, and a vnode with no associated ubc_info.  The
 *		caller therefore needs to not care, or needs to ensure that
 *		they have previously successfully called ubc_info_init() or
 *		ubc_info_init_withsize().
 */
off_t
ubc_getsize(struct vnode *vp)
{
	/* people depend on the side effect of this working this way
	 * as they call this for directory
	 */
	if (!UBCINFOEXISTS(vp))
		return ((off_t)0);
	return (vp->v_ubcinfo->ui_size);
}


/*
 * ubc_umount
 *
 * Call ubc_sync_range(vp, 0, EOF, UBC_PUSHALL) on all the vnodes for this
 * mount point
 *
 * Parameters:	mp			The mount point
 *
 * Returns:	0			Success
 *
 * Notes:	There is no failure indication for this function.
 *
 *		This function is used in the unmount path; since it may block
 *		I/O indefinitely, it should not be used in the forced unmount
 *		path, since a device unavailability could also block that
 *		indefinitely.
 *
 *		Because there is no device ejection interlock on USB, FireWire,
 *		or similar devices, it's possible that an ejection that begins
 *		subsequent to the vnode_iterate() completing, either on one of
 *		those devices, or a network mount for which the server quits
 *		responding, etc., may cause the caller to block indefinitely.
 */
__private_extern__ int
ubc_umount(struct mount *mp)
{
	vnode_iterate(mp, 0, ubc_umcallback, 0);
	return(0);
}


/*
 * ubc_umcallback
 *
 * Used by ubc_umount() as an internal implementation detail; see ubc_umount()
 * and vnode_iterate() for details of implementation.
 */
static int
ubc_umcallback(vnode_t vp, __unused void * args)
{

	if (UBCINFOEXISTS(vp)) {

		(void) ubc_msync(vp, (off_t)0, ubc_getsize(vp), NULL, UBC_PUSHALL);
	}
	return (VNODE_RETURNED);
}


/*
 * ubc_getcred
 *
 * Get the credentials currently active for the ubc_info associated with the
 * vnode.
 *
 * Parameters:	vp			The vnode whose ubc_info credentials
 *					are to be retrieved
 *
 * Returns:	!NOCRED			The credentials
 *		NOCRED			If there is no ubc_info for the vnode,
 *					or if there is one, but it has not had
 *					any credentials associated with it via
 *					a call to ubc_setcred()
 */
kauth_cred_t
ubc_getcred(struct vnode *vp)
{
        if (UBCINFOEXISTS(vp))
	        return (vp->v_ubcinfo->ui_ucred);

	return (NOCRED);
}


/*
 * ubc_setthreadcred
 *
 * If they are not already set, set the credentials of the ubc_info structure
 * associated with the vnode to those of the supplied thread; otherwise leave
 * them alone.
 *
 * Parameters:	vp			The vnode whose ubc_info creds are to
 *					be set
 *		p			The process whose credentials are to
 *					be used, if not running on an assumed
 *					credential
 *		thread			The thread whose credentials are to
 *					be used
 *
 * Returns:	1			This vnode has no associated ubc_info
 *		0			Success
 *
 * Notes:	This function takes a proc parameter to account for bootstrap
 *		issues where a task or thread may call this routine, either
 *		before credentials have been initialized by bsd_init(), or if
 *		there is no BSD info asscoiate with a mach thread yet.  This
 *		is known to happen in both the initial swap and memory mapping
 *		calls.
 *
 *		This function is generally used only in the following cases:
 *
 *		o	a memory mapped file via the mmap() system call
 *		o	a memory mapped file via the deprecated map_fd() call
 *		o	a swap store backing file
 *		o	subsequent to a successful write via vn_write()
 *
 *		The information is then used by the NFS client in order to
 *		cons up a wire message in either the page-in or page-out path.
 *
 *		There are two potential problems with the use of this API:
 *
 *		o	Because the write path only set it on a successful
 *			write, there is a race window between setting the
 *			credential and its use to evict the pages to the
 *			remote file server
 *
 *		o	Because a page-in may occur prior to a write, the
 *			credential may not be set at this time, if the page-in
 *			is not the result of a mapping established via mmap()
 *			or map_fd().
 *
 *		In both these cases, this will be triggered from the paging
 *		path, which will instead use the credential of the current
 *		process, which in this case is either the dynamic_pager or
 *		the kernel task, both of which utilize "root" credentials.
 *
 *		This may potentially permit operations to occur which should
 *		be denied, or it may cause to be denied operations which
 *		should be permitted, depending on the configuration of the NFS
 *		server.
 */
int
ubc_setthreadcred(struct vnode *vp, proc_t p, thread_t thread)
{
	struct ubc_info *uip;
	kauth_cred_t credp;
	struct uthread  *uthread = get_bsdthread_info(thread);

	if (!UBCINFOEXISTS(vp))
		return (1);

	vnode_lock(vp);

	uip = vp->v_ubcinfo;
	credp = uip->ui_ucred;

	if (!IS_VALID_CRED(credp)) {
		/* use per-thread cred, if assumed identity, else proc cred */
		if (uthread == NULL || (uthread->uu_flag & UT_SETUID) == 0) {
			uip->ui_ucred = kauth_cred_proc_ref(p);
		} else {
			uip->ui_ucred = uthread->uu_ucred;
			kauth_cred_ref(uip->ui_ucred);
		}
	}
	vnode_unlock(vp);

	return (0);
}


/*
 * ubc_setcred
 *
 * If they are not already set, set the credentials of the ubc_info structure
 * associated with the vnode to those of the process; otherwise leave them
 * alone.
 *
 * Parameters:	vp			The vnode whose ubc_info creds are to
 *					be set
 *		p			The process whose credentials are to
 *					be used
 *
 * Returns:	0			This vnode has no associated ubc_info
 *		1			Success
 *
 * Notes:	The return values for this function are inverted from nearly
 *		all other uses in the kernel.
 *
 *		See also ubc_setthreadcred(), above.
 *
 *		This function is considered deprecated, and generally should
 *		not be used, as it is incompatible with per-thread credentials;
 *		it exists for legacy KPI reasons.
 *
 * DEPRECATION:	ubc_setcred() is being deprecated. Please use
 *		ubc_setthreadcred() instead.
 */
int
ubc_setcred(struct vnode *vp, proc_t p)
{
	struct ubc_info *uip;
	kauth_cred_t credp;

	/* If there is no ubc_info, deny the operation */
	if ( !UBCINFOEXISTS(vp))
		return (0);

	/*
	 * Check to see if there is already a credential reference in the
	 * ubc_info; if there is not, take one on the supplied credential.
	 */
	vnode_lock(vp);
	uip = vp->v_ubcinfo;
	credp = uip->ui_ucred;
	if (!IS_VALID_CRED(credp)) {
		uip->ui_ucred = kauth_cred_proc_ref(p);
	}
	vnode_unlock(vp);

	return (1);
}

/*
 * ubc_getpager
 *
 * Get the pager associated with the ubc_info associated with the vnode.
 *
 * Parameters:	vp			The vnode to obtain the pager from
 *
 * Returns:	!VNODE_PAGER_NULL	The memory_object_t for the pager
 *		VNODE_PAGER_NULL	There is no ubc_info for this vnode
 *
 * Notes:	For each vnode that has a ubc_info associated with it, that
 *		ubc_info SHALL have a pager associated with it, so in the
 *		normal case, it's impossible to return VNODE_PAGER_NULL for
 *		a vnode with an associated ubc_info.
 */
__private_extern__ memory_object_t
ubc_getpager(struct vnode *vp)
{
        if (UBCINFOEXISTS(vp))
	        return (vp->v_ubcinfo->ui_pager);

	return (0);
}


/*
 * ubc_getobject
 *
 * Get the memory object control associated with the ubc_info associated with
 * the vnode
 *
 * Parameters:	vp			The vnode to obtain the memory object
 *					from
 *		flags			DEPRECATED
 *
 * Returns:	!MEMORY_OBJECT_CONTROL_NULL
 *		MEMORY_OBJECT_CONTROL_NULL
 *
 * Notes:	Historically, if the flags were not "do not reactivate", this
 *		function would look up the memory object using the pager if
 *		it did not exist (this could be the case if the vnode had
 *		been previously reactivated).  The flags would also permit a
 *		hold to be requested, which would have created an object
 *		reference, if one had not already existed.  This usage is
 *		deprecated, as it would permit a race between finding and
 *		taking the reference vs. a single reference being dropped in
 *		another thread.
 */
memory_object_control_t
ubc_getobject(struct vnode *vp, __unused int flags)
{
        if (UBCINFOEXISTS(vp))
	        return((vp->v_ubcinfo->ui_control));

	return (MEMORY_OBJECT_CONTROL_NULL);
}

boolean_t
ubc_strict_uncached_IO(struct vnode *vp)
{
        boolean_t result = FALSE;

	if (UBCINFOEXISTS(vp)) {
	        result = memory_object_is_slid(vp->v_ubcinfo->ui_control);
	}
	return result;
}

/*
 * ubc_blktooff
 *
 * Convert a given block number to a memory backing object (file) offset for a
 * given vnode
 *
 * Parameters:	vp			The vnode in which the block is located
 *		blkno			The block number to convert
 *
 * Returns:	!-1			The offset into the backing object
 *		-1			There is no ubc_info associated with
 *					the vnode
 *		-1			An error occurred in the underlying VFS
 *					while translating the block to an
 *					offset; the most likely cause is that
 *					the caller specified a block past the
 *					end of the file, but this could also be
 *					any other error from VNOP_BLKTOOFF().
 *
 * Note:	Representing the error in band loses some information, but does
 *		not occlude a valid offset, since an off_t of -1 is normally
 *		used to represent EOF.  If we had a more reliable constant in
 *		our header files for it (i.e. explicitly cast to an off_t), we
 *		would use it here instead.
 */
off_t
ubc_blktooff(vnode_t vp, daddr64_t blkno)
{
	off_t file_offset = -1;
	int error;

	if (UBCINFOEXISTS(vp)) {
		error = VNOP_BLKTOOFF(vp, blkno, &file_offset);
		if (error)
			file_offset = -1;
	}

	return (file_offset);
}


/*
 * ubc_offtoblk
 *
 * Convert a given offset in a memory backing object into a block number for a
 * given vnode
 *
 * Parameters:	vp			The vnode in which the offset is
 *					located
 *		offset			The offset into the backing object
 *
 * Returns:	!-1			The returned block number
 *		-1			There is no ubc_info associated with
 *					the vnode
 *		-1			An error occurred in the underlying VFS
 *					while translating the block to an
 *					offset; the most likely cause is that
 *					the caller specified a block past the
 *					end of the file, but this could also be
 *					any other error from VNOP_OFFTOBLK().
 *
 * Note:	Representing the error in band loses some information, but does
 *		not occlude a valid block number, since block numbers exceed
 *		the valid range for offsets, due to their relative sizes.  If
 *		we had a more reliable constant than -1 in our header files
 *		for it (i.e. explicitly cast to an daddr64_t), we would use it
 *		here instead.
 */
daddr64_t
ubc_offtoblk(vnode_t vp, off_t offset)
{
	daddr64_t blkno = -1;
	int error = 0;

	if (UBCINFOEXISTS(vp)) {
		error = VNOP_OFFTOBLK(vp, offset, &blkno);
		if (error)
			blkno = -1;
	}

	return (blkno);
}


/*
 * ubc_pages_resident
 *
 * Determine whether or not a given vnode has pages resident via the memory
 * object control associated with the ubc_info associated with the vnode
 *
 * Parameters:	vp			The vnode we want to know about
 *
 * Returns:	1			Yes
 *		0			No
 */
int
ubc_pages_resident(vnode_t vp)
{
	kern_return_t		kret;
	boolean_t			has_pages_resident;

	if (!UBCINFOEXISTS(vp))
		return (0);

	/*
	 * The following call may fail if an invalid ui_control is specified,
	 * or if there is no VM object associated with the control object.  In
	 * either case, reacting to it as if there were no pages resident will
	 * result in correct behavior.
	 */
	kret = memory_object_pages_resident(vp->v_ubcinfo->ui_control, &has_pages_resident);

	if (kret != KERN_SUCCESS)
		return (0);

	if (has_pages_resident == TRUE)
		return (1);

	return (0);
}


/*
 * ubc_sync_range
 *
 * Clean and/or invalidate a range in the memory object that backs this vnode
 *
 * Parameters:	vp			The vnode whose associated ubc_info's
 *					associated memory object is to have a
 *					range invalidated within it
 *		beg_off			The start of the range, as an offset
 *		end_off			The end of the range, as an offset
 *		flags			See ubc_msync_internal()
 *
 * Returns:	1			Success
 *		0			Failure
 *
 * Notes:	see ubc_msync_internal() for more detailed information.
 *
 * DEPRECATED:	This interface is obsolete due to a failure to return error
 *		information needed in order to correct failures.  The currently
 *		recommended interface is ubc_msync().
 */
int
ubc_sync_range(vnode_t vp, off_t beg_off, off_t end_off, int flags)
{
        return (ubc_msync_internal(vp, beg_off, end_off, NULL, flags, NULL));
}


/*
 * ubc_msync
 *
 * Clean and/or invalidate a range in the memory object that backs this vnode
 *
 * Parameters:	vp			The vnode whose associated ubc_info's
 *					associated memory object is to have a
 *					range invalidated within it
 *		beg_off			The start of the range, as an offset
 *		end_off			The end of the range, as an offset
 *		resid_off		The address of an off_t supplied by the
 *					caller; may be set to NULL to ignore
 *		flags			See ubc_msync_internal()
 *
 * Returns:	0			Success
 *		!0			Failure; an errno is returned
 *
 * Implicit Returns:
 *		*resid_off, modified	If non-NULL, the  contents are ALWAYS
 *					modified; they are initialized to the
 *					beg_off, and in case of an I/O error,
 *					the difference between beg_off and the
 *					current value will reflect what was
 *					able to be written before the error
 *					occurred.  If no error is returned, the
 *					value of the resid_off is undefined; do
 *					NOT use it in place of end_off if you
 *					intend to increment from the end of the
 *					last call and call iteratively.
 *
 * Notes:	see ubc_msync_internal() for more detailed information.
 *
 */
errno_t
ubc_msync(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, int flags)
{
        int retval;
	int io_errno = 0;

	if (resid_off)
	        *resid_off = beg_off;

        retval = ubc_msync_internal(vp, beg_off, end_off, resid_off, flags, &io_errno);

	if (retval == 0 && io_errno == 0)
	        return (EINVAL);
	return (io_errno);
}


/*
 * Clean and/or invalidate a range in the memory object that backs this vnode
 *
 * Parameters:	vp			The vnode whose associated ubc_info's
 *					associated memory object is to have a
 *					range invalidated within it
 *		beg_off			The start of the range, as an offset
 *		end_off			The end of the range, as an offset
 *		resid_off		The address of an off_t supplied by the
 *					caller; may be set to NULL to ignore
 *		flags			MUST contain at least one of the flags
 *					UBC_INVALIDATE, UBC_PUSHDIRTY, or
 *					UBC_PUSHALL; if UBC_PUSHDIRTY is used,
 *					UBC_SYNC may also be specified to cause
 *					this function to block until the
 *					operation is complete.  The behavior
 *					of UBC_SYNC is otherwise undefined.
 *		io_errno		The address of an int to contain the
 *					errno from a failed I/O operation, if
 *					one occurs; may be set to NULL to
 *					ignore
 *
 * Returns:	1			Success
 *		0			Failure
 *
 * Implicit Returns:
 *		*resid_off, modified	The contents of this offset MAY be
 *					modified; in case of an I/O error, the
 *					difference between beg_off and the
 *					current value will reflect what was
 *					able to be written before the error
 *					occurred.
 *		*io_errno, modified	The contents of this offset are set to
 *					an errno, if an error occurs; if the
 *					caller supplies an io_errno parameter,
 *					they should be careful to initialize it
 *					to 0 before calling this function to
 *					enable them to distinguish an error
 *					with a valid *resid_off from an invalid
 *					one, and to avoid potentially falsely
 *					reporting an error, depending on use.
 *
 * Notes:	If there is no ubc_info associated with the vnode supplied,
 *		this function immediately returns success.
 *
 *		If the value of end_off is less than or equal to beg_off, this
 *		function immediately returns success; that is, end_off is NOT
 *		inclusive.
 *
 *		IMPORTANT: one of the flags UBC_INVALIDATE, UBC_PUSHDIRTY, or
 *		UBC_PUSHALL MUST be specified; that is, it is NOT possible to
 *		attempt to block on in-progress I/O by calling this function
 *		with UBC_PUSHDIRTY, and then later call it with just UBC_SYNC
 *		in order to block pending on the I/O already in progress.
 *
 *		The start offset is truncated to the page boundary and the
 *		size is adjusted to include the last page in the range; that
 *		is, end_off on exactly a page boundary will not change if it
 *		is rounded, and the range of bytes written will be from the
 *		truncate beg_off to the rounded (end_off - 1).
 */
static int
ubc_msync_internal(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, int flags, int *io_errno)
{
	memory_object_size_t	tsize;
	kern_return_t		kret;
	int request_flags = 0;
	int flush_flags   = MEMORY_OBJECT_RETURN_NONE;

	if ( !UBCINFOEXISTS(vp))
	        return (0);
	if ((flags & (UBC_INVALIDATE | UBC_PUSHDIRTY | UBC_PUSHALL)) == 0)
	        return (0);
	if (end_off <= beg_off)
	        return (1);

	if (flags & UBC_INVALIDATE)
	        /*
		 * discard the resident pages
		 */
		request_flags = (MEMORY_OBJECT_DATA_FLUSH | MEMORY_OBJECT_DATA_NO_CHANGE);

	if (flags & UBC_SYNC)
	        /*
		 * wait for all the I/O to complete before returning
		 */
	        request_flags |= MEMORY_OBJECT_IO_SYNC;

	if (flags & UBC_PUSHDIRTY)
	        /*
		 * we only return the dirty pages in the range
		 */
	        flush_flags = MEMORY_OBJECT_RETURN_DIRTY;

	if (flags & UBC_PUSHALL)
	        /*
		 * then return all the interesting pages in the range (both
		 * dirty and precious) to the pager
		 */
	        flush_flags = MEMORY_OBJECT_RETURN_ALL;

	beg_off = trunc_page_64(beg_off);
	end_off = round_page_64(end_off);
	tsize   = (memory_object_size_t)end_off - beg_off;

	/* flush and/or invalidate pages in the range requested */
	kret = memory_object_lock_request(vp->v_ubcinfo->ui_control,
					  beg_off, tsize,
					  (memory_object_offset_t *)resid_off,
					  io_errno, flush_flags, request_flags,
					  VM_PROT_NO_CHANGE);

	return ((kret == KERN_SUCCESS) ? 1 : 0);
}


/*
 * ubc_msync_internal
 *
 * Explicitly map a vnode that has an associate ubc_info, and add a reference
 * to it for the ubc system, if there isn't one already, so it will not be
 * recycled while it's in use, and set flags on the ubc_info to indicate that
 * we have done this
 *
 * Parameters:	vp			The vnode to map
 *		flags			The mapping flags for the vnode; this
 *					will be a combination of one or more of
 *					PROT_READ, PROT_WRITE, and PROT_EXEC
 *
 * Returns:	0			Success
 *		EPERM			Permission was denied
 *
 * Notes:	An I/O reference on the vnode must already be held on entry
 *
 *		If there is no ubc_info associated with the vnode, this function
 *		will return success.
 *
 *		If a permission error occurs, this function will return
 *		failure; all other failures will cause this function to return
 *		success.
 *
 *		IMPORTANT: This is an internal use function, and its symbols
 *		are not exported, hence its error checking is not very robust.
 *		It is primarily used by:
 *
 *		o	mmap(), when mapping a file
 *		o	The deprecated map_fd() interface, when mapping a file
 *		o	When mapping a shared file (a shared library in the
 *			shared segment region)
 *		o	When loading a program image during the exec process
 *
 *		...all of these uses ignore the return code, and any fault that
 *		results later because of a failure is handled in the fix-up path
 *		of the fault handler.  The interface exists primarily as a
 *		performance hint.
 *
 *		Given that third party implementation of the type of interfaces
 *		that would use this function, such as alternative executable
 *		formats, etc., are unsupported, this function is not exported
 *		for general use.
 *
 *		The extra reference is held until the VM system unmaps the
 *		vnode from its own context to maintain a vnode reference in
 *		cases like open()/mmap()/close(), which leave the backing
 *		object referenced by a mapped memory region in a process
 *		address space.
 */
__private_extern__ int
ubc_map(vnode_t vp, int flags)
{
	struct ubc_info *uip;
	int error = 0;
	int need_ref = 0;
	int need_wakeup = 0;

	if (UBCINFOEXISTS(vp)) {

		vnode_lock(vp);
		uip = vp->v_ubcinfo;

		while (ISSET(uip->ui_flags, UI_MAPBUSY)) {
			SET(uip->ui_flags, UI_MAPWAITING);
			(void) msleep(&uip->ui_flags, &vp->v_lock,
				      PRIBIO, "ubc_map", NULL);
		}
		SET(uip->ui_flags, UI_MAPBUSY);
		vnode_unlock(vp);

		error = VNOP_MMAP(vp, flags, vfs_context_current());

		if (error != EPERM)
		        error = 0;

		vnode_lock_spin(vp);

		if (error == 0) {
			if ( !ISSET(uip->ui_flags, UI_ISMAPPED))
			        need_ref = 1;
			SET(uip->ui_flags, (UI_WASMAPPED | UI_ISMAPPED));
			if (flags & PROT_WRITE) {
				SET(uip->ui_flags, UI_MAPPEDWRITE);
			}
		}
		CLR(uip->ui_flags, UI_MAPBUSY);

		if (ISSET(uip->ui_flags, UI_MAPWAITING)) {
			CLR(uip->ui_flags, UI_MAPWAITING);
			need_wakeup = 1;
		}
		vnode_unlock(vp);

		if (need_wakeup)
			wakeup(&uip->ui_flags);

		if (need_ref)
			vnode_ref(vp);
	}
	return (error);
}


/*
 * ubc_destroy_named
 *
 * Destroy the named memory object associated with the ubc_info control object
 * associated with the designated vnode, if there is a ubc_info associated
 * with the vnode, and a control object is associated with it
 *
 * Parameters:	vp			The designated vnode
 *
 * Returns:	(void)
 *
 * Notes:	This function is called on vnode termination for all vnodes,
 *		and must therefore not assume that there is a ubc_info that is
 *		associated with the vnode, nor that there is a control object
 *		associated with the ubc_info.
 *
 *		If all the conditions necessary are present, this function
 *		calls memory_object_destory(), which will in turn end up
 *		calling ubc_unmap() to release any vnode references that were
 *		established via ubc_map().
 *
 *		IMPORTANT: This is an internal use function that is used
 *		exclusively by the internal use function vclean().
 */
__private_extern__ void
ubc_destroy_named(vnode_t vp)
{
	memory_object_control_t control;
	struct ubc_info *uip;
	kern_return_t kret;

	if (UBCINFOEXISTS(vp)) {
	        uip = vp->v_ubcinfo;

		/* Terminate the memory object  */
		control = ubc_getobject(vp, UBC_HOLDOBJECT);
		if (control != MEMORY_OBJECT_CONTROL_NULL) {
		        kret = memory_object_destroy(control, 0);
			if (kret != KERN_SUCCESS)
			        panic("ubc_destroy_named: memory_object_destroy failed");
		}
	}
}


/*
 * ubc_isinuse
 *
 * Determine whether or not a vnode is currently in use by ubc at a level in
 * excess of the requested busycount
 *
 * Parameters:	vp			The vnode to check
 *		busycount		The threshold busy count, used to bias
 *					the count usually already held by the
 *					caller to avoid races
 *
 * Returns:	1			The vnode is in use over the threshold
 *		0			The vnode is not in use over the
 *					threshold
 *
 * Notes:	Because the vnode is only held locked while actually asking
 *		the use count, this function only represents a snapshot of the
 *		current state of the vnode.  If more accurate information is
 *		required, an additional busycount should be held by the caller
 *		and a non-zero busycount used.
 *
 *		If there is no ubc_info associated with the vnode, this
 *		function will report that the vnode is not in use by ubc.
 */
int
ubc_isinuse(struct vnode *vp, int busycount)
{
	if ( !UBCINFOEXISTS(vp))
		return (0);
	return(ubc_isinuse_locked(vp, busycount, 0));
}


/*
 * ubc_isinuse_locked
 *
 * Determine whether or not a vnode is currently in use by ubc at a level in
 * excess of the requested busycount
 *
 * Parameters:	vp			The vnode to check
 *		busycount		The threshold busy count, used to bias
 *					the count usually already held by the
 *					caller to avoid races
 *		locked			True if the vnode is already locked by
 *					the caller
 *
 * Returns:	1			The vnode is in use over the threshold
 *		0			The vnode is not in use over the
 *					threshold
 *
 * Notes:	If the vnode is not locked on entry, it is locked while
 *		actually asking the use count.  If this is the case, this
 *		function only represents a snapshot of the current state of
 *		the vnode.  If more accurate information is required, the
 *		vnode lock should be held by the caller, otherwise an
 *		additional busycount should be held by the caller and a
 *		non-zero busycount used.
 *
 *		If there is no ubc_info associated with the vnode, this
 *		function will report that the vnode is not in use by ubc.
 */
int
ubc_isinuse_locked(struct vnode *vp, int busycount, int locked)
{
	int retval = 0;


	if (!locked)
		vnode_lock_spin(vp);

	if ((vp->v_usecount - vp->v_kusecount) > busycount)
		retval = 1;

	if (!locked)
		vnode_unlock(vp);
	return (retval);
}


/*
 * ubc_unmap
 *
 * Reverse the effects of a ubc_map() call for a given vnode
 *
 * Parameters:	vp			vnode to unmap from ubc
 *
 * Returns:	(void)
 *
 * Notes:	This is an internal use function used by vnode_pager_unmap().
 *		It will attempt to obtain a reference on the supplied vnode,
 *		and if it can do so, and there is an associated ubc_info, and
 *		the flags indicate that it was mapped via ubc_map(), then the
 *		flag is cleared, the mapping removed, and the reference taken
 *		by ubc_map() is released.
 *
 *		IMPORTANT: This MUST only be called by the VM
 *		to prevent race conditions.
 */
__private_extern__ void
ubc_unmap(struct vnode *vp)
{
	struct ubc_info *uip;
	int	need_rele = 0;
	int	need_wakeup = 0;

	if (vnode_getwithref(vp))
	        return;

	if (UBCINFOEXISTS(vp)) {
		vnode_lock(vp);
		uip = vp->v_ubcinfo;

		while (ISSET(uip->ui_flags, UI_MAPBUSY)) {
			SET(uip->ui_flags, UI_MAPWAITING);
			(void) msleep(&uip->ui_flags, &vp->v_lock,
				      PRIBIO, "ubc_unmap", NULL);
		}
		SET(uip->ui_flags, UI_MAPBUSY);

		if (ISSET(uip->ui_flags, UI_ISMAPPED)) {
		        CLR(uip->ui_flags, UI_ISMAPPED);
			need_rele = 1;
		}
		vnode_unlock(vp);

		if (need_rele) {
		        (void)VNOP_MNOMAP(vp, vfs_context_current());
		        vnode_rele(vp);
		}

		vnode_lock_spin(vp);

		CLR(uip->ui_flags, UI_MAPBUSY);
		if (ISSET(uip->ui_flags, UI_MAPWAITING)) {
			CLR(uip->ui_flags, UI_MAPWAITING);
			need_wakeup = 1;
		}
		vnode_unlock(vp);

		if (need_wakeup)
		        wakeup(&uip->ui_flags);

	}
	/*
	 * the drop of the vnode ref will cleanup
	 */
	vnode_put(vp);
}


/*
 * ubc_page_op
 *
 * Manipulate individual page state for a vnode with an associated ubc_info
 * with an associated memory object control.
 *
 * Parameters:	vp			The vnode backing the page
 *		f_offset		A file offset interior to the page
 *		ops			The operations to perform, as a bitmap
 *					(see below for more information)
 *		phys_entryp		The address of a ppnum_t; may be NULL
 *					to ignore
 *		flagsp			A pointer to an int to contain flags;
 *					may be NULL to ignore
 *
 * Returns:	KERN_SUCCESS		Success
 *		KERN_INVALID_ARGUMENT	If the memory object control has no VM
 *					object associated
 *		KERN_INVALID_OBJECT	If UPL_POP_PHYSICAL and the object is
 *					not physically contiguous
 *		KERN_INVALID_OBJECT	If !UPL_POP_PHYSICAL and the object is
 *					physically contiguous
 *		KERN_FAILURE		If the page cannot be looked up
 *
 * Implicit Returns:
 *		*phys_entryp (modified)	If phys_entryp is non-NULL and
 *					UPL_POP_PHYSICAL
 *		*flagsp (modified)	If flagsp is non-NULL and there was
 *					!UPL_POP_PHYSICAL and a KERN_SUCCESS
 *
 * Notes:	For object boundaries, it is considerably more efficient to
 *		ensure that f_offset is in fact on a page boundary, as this
 *		will avoid internal use of the hash table to identify the
 *		page, and would therefore skip a number of early optimizations.
 *		Since this is a page operation anyway, the caller should try
 *		to pass only a page aligned offset because of this.
 *
 *		*flagsp may be modified even if this function fails.  If it is
 *		modified, it will contain the condition of the page before the
 *		requested operation was attempted; these will only include the
 *		bitmap flags, and not the PL_POP_PHYSICAL, UPL_POP_DUMP,
 *		UPL_POP_SET, or UPL_POP_CLR bits.
 *
 *		The flags field may contain a specific operation, such as
 *		UPL_POP_PHYSICAL or UPL_POP_DUMP:
 *
 *		o	UPL_POP_PHYSICAL	Fail if not contiguous; if
 *						*phys_entryp and successful, set
 *						*phys_entryp
 *		o	UPL_POP_DUMP		Dump the specified page
 *
 *		Otherwise, it is treated as a bitmap of one or more page
 *		operations to perform on the final memory object; allowable
 *		bit values are:
 *
 *		o	UPL_POP_DIRTY		The page is dirty
 *		o	UPL_POP_PAGEOUT		The page is paged out
 *		o	UPL_POP_PRECIOUS	The page is precious
 *		o	UPL_POP_ABSENT		The page is absent
 *		o	UPL_POP_BUSY		The page is busy
 *
 *		If the page status is only being queried and not modified, then
 *		not other bits should be specified.  However, if it is being
 *		modified, exactly ONE of the following bits should be set:
 *
 *		o	UPL_POP_SET		Set the current bitmap bits
 *		o	UPL_POP_CLR		Clear the current bitmap bits
 *
 *		Thus to effect a combination of setting an clearing, it may be
 *		necessary to call this function twice.  If this is done, the
 *		set should be used before the clear, since clearing may trigger
 *		a wakeup on the destination page, and if the page is backed by
 *		an encrypted swap file, setting will trigger the decryption
 *		needed before the wakeup occurs.
 */
kern_return_t
ubc_page_op(
	struct vnode 	*vp,
	off_t		f_offset,
	int		ops,
	ppnum_t	*phys_entryp,
	int		*flagsp)
{
	memory_object_control_t		control;

	control = ubc_getobject(vp, UBC_FLAGS_NONE);
	if (control == MEMORY_OBJECT_CONTROL_NULL)
		return KERN_INVALID_ARGUMENT;

	return (memory_object_page_op(control,
				      (memory_object_offset_t)f_offset,
				      ops,
				      phys_entryp,
				      flagsp));
}


/*
 * ubc_range_op
 *
 * Manipulate page state for a range of memory for a vnode with an associated
 * ubc_info with an associated memory object control, when page level state is
 * not required to be returned from the call (i.e. there are no phys_entryp or
 * flagsp parameters to this call, and it takes a range which may contain
 * multiple pages, rather than an offset interior to a single page).
 *
 * Parameters:	vp			The vnode backing the page
 *		f_offset_beg		A file offset interior to the start page
 *		f_offset_end		A file offset interior to the end page
 *		ops			The operations to perform, as a bitmap
 *					(see below for more information)
 *		range			The address of an int; may be NULL to
 *					ignore
 *
 * Returns:	KERN_SUCCESS		Success
 *		KERN_INVALID_ARGUMENT	If the memory object control has no VM
 *					object associated
 *		KERN_INVALID_OBJECT	If the object is physically contiguous
 *
 * Implicit Returns:
 *		*range (modified)	If range is non-NULL, its contents will
 *					be modified to contain the number of
 *					bytes successfully operated upon.
 *
 * Notes:	IMPORTANT: This function cannot be used on a range that
 *		consists of physically contiguous pages.
 *
 *		For object boundaries, it is considerably more efficient to
 *		ensure that f_offset_beg and f_offset_end are in fact on page
 *		boundaries, as this will avoid internal use of the hash table
 *		to identify the page, and would therefore skip a number of
 *		early optimizations.  Since this is an operation on a set of
 *		pages anyway, the caller should try to pass only a page aligned
 *		offsets because of this.
 *
 *		*range will be modified only if this function succeeds.
 *
 *		The flags field MUST contain a specific operation; allowable
 *		values are:
 *
 *		o	UPL_ROP_ABSENT	Returns the extent of the range
 *					presented which is absent, starting
 *					with the start address presented
 *
 *		o	UPL_ROP_PRESENT	Returns the extent of the range
 *					presented which is present (resident),
 *					starting with the start address
 *					presented
 *		o	UPL_ROP_DUMP	Dump the pages which are found in the
 *					target object for the target range.
 *
 *		IMPORTANT: For UPL_ROP_ABSENT and UPL_ROP_PRESENT; if there are
 *		multiple regions in the range, only the first matching region
 *		is returned.
 */
kern_return_t
ubc_range_op(
	struct vnode 	*vp,
	off_t		f_offset_beg,
	off_t		f_offset_end,
	int             ops,
	int             *range)
{
	memory_object_control_t		control;

	control = ubc_getobject(vp, UBC_FLAGS_NONE);
	if (control == MEMORY_OBJECT_CONTROL_NULL)
		return KERN_INVALID_ARGUMENT;

	return (memory_object_range_op(control,
				      (memory_object_offset_t)f_offset_beg,
				      (memory_object_offset_t)f_offset_end,
				      ops,
				      range));
}


/*
 * ubc_create_upl
 *
 * Given a vnode, cause the population of a portion of the vm_object; based on
 * the nature of the request, the pages returned may contain valid data, or
 * they may be uninitialized.
 *
 * Parameters:	vp			The vnode from which to create the upl
 *		f_offset		The start offset into the backing store
 *					represented by the vnode
 *		bufsize			The size of the upl to create
 *		uplp			Pointer to the upl_t to receive the
 *					created upl; MUST NOT be NULL
 *		plp			Pointer to receive the internal page
 *					list for the created upl; MAY be NULL
 *					to ignore
 *
 * Returns:	KERN_SUCCESS		The requested upl has been created
 *		KERN_INVALID_ARGUMENT	The bufsize argument is not an even
 *					multiple of the page size
 *		KERN_INVALID_ARGUMENT	There is no ubc_info associated with
 *					the vnode, or there is no memory object
 *					control associated with the ubc_info
 *	memory_object_upl_request:KERN_INVALID_VALUE
 *					The supplied upl_flags argument is
 *					invalid
 * Implicit Returns:
 *		*uplp (modified)
 *		*plp (modified)		If non-NULL, the value of *plp will be
 *					modified to point to the internal page
 *					list; this modification may occur even
 *					if this function is unsuccessful, in
 *					which case the contents may be invalid
 *
 * Note:	If successful, the returned *uplp MUST subsequently be freed
 *		via a call to ubc_upl_commit(), ubc_upl_commit_range(),
 *		ubc_upl_abort(), or ubc_upl_abort_range().
 */
kern_return_t
ubc_create_upl(
	struct vnode	*vp,
	off_t 		f_offset,
	int		bufsize,
	upl_t		*uplp,
	upl_page_info_t	**plp,
	int		uplflags)
{
	memory_object_control_t		control;
	kern_return_t			kr;

	if (plp != NULL)
		*plp = NULL;
	*uplp = NULL;

	if (bufsize & 0xfff)
		return KERN_INVALID_ARGUMENT;

	if (bufsize > MAX_UPL_SIZE * PAGE_SIZE)
		return KERN_INVALID_ARGUMENT;

	if (uplflags & (UPL_UBC_MSYNC | UPL_UBC_PAGEOUT | UPL_UBC_PAGEIN)) {

		if (uplflags & UPL_UBC_MSYNC) {
			uplflags &= UPL_RET_ONLY_DIRTY;

			uplflags |= UPL_COPYOUT_FROM | UPL_CLEAN_IN_PLACE |
				    UPL_SET_INTERNAL | UPL_SET_LITE;

		} else if (uplflags & UPL_UBC_PAGEOUT) {
			uplflags &= UPL_RET_ONLY_DIRTY;

			if (uplflags & UPL_RET_ONLY_DIRTY)
				uplflags |= UPL_NOBLOCK;

			uplflags |= UPL_FOR_PAGEOUT | UPL_CLEAN_IN_PLACE |
                                    UPL_COPYOUT_FROM | UPL_SET_INTERNAL | UPL_SET_LITE;
		} else {
			uplflags |= UPL_RET_ONLY_ABSENT |
				    UPL_NO_SYNC | UPL_CLEAN_IN_PLACE |
				    UPL_SET_INTERNAL | UPL_SET_LITE;

			/*
			 * if the requested size == PAGE_SIZE, we don't want to set
			 * the UPL_NOBLOCK since we may be trying to recover from a
			 * previous partial pagein I/O that occurred because we were low
			 * on memory and bailed early in order to honor the UPL_NOBLOCK...
			 * since we're only asking for a single page, we can block w/o fear
			 * of tying up pages while waiting for more to become available
			 */
			if (bufsize > PAGE_SIZE)
				uplflags |= UPL_NOBLOCK;
		}
	} else {
		uplflags &= ~UPL_FOR_PAGEOUT;

		if (uplflags & UPL_WILL_BE_DUMPED) {
			uplflags &= ~UPL_WILL_BE_DUMPED;
			uplflags |= (UPL_NO_SYNC|UPL_SET_INTERNAL);
		} else
			uplflags |= (UPL_NO_SYNC|UPL_CLEAN_IN_PLACE|UPL_SET_INTERNAL);
	}
	control = ubc_getobject(vp, UBC_FLAGS_NONE);
	if (control == MEMORY_OBJECT_CONTROL_NULL)
		return KERN_INVALID_ARGUMENT;

	kr = memory_object_upl_request(control, f_offset, bufsize, uplp, NULL, NULL, uplflags);
	if (kr == KERN_SUCCESS && plp != NULL)
		*plp = UPL_GET_INTERNAL_PAGE_LIST(*uplp);
	return kr;
}


/*
 * ubc_upl_maxbufsize
 *
 * Return the maximum bufsize ubc_create_upl( ) will take.
 *
 * Parameters:	none
 *
 * Returns:	maximum size buffer (in bytes) ubc_create_upl( ) will take.
 */
upl_size_t
ubc_upl_maxbufsize(
	void)
{
	return(MAX_UPL_SIZE * PAGE_SIZE);
}

/*
 * ubc_upl_map
 *
 * Map the page list assocated with the supplied upl into the kernel virtual
 * address space at the virtual address indicated by the dst_addr argument;
 * the entire upl is mapped
 *
 * Parameters:	upl			The upl to map
 *		dst_addr		The address at which to map the upl
 *
 * Returns:	KERN_SUCCESS		The upl has been mapped
 *		KERN_INVALID_ARGUMENT	The upl is UPL_NULL
 *		KERN_FAILURE		The upl is already mapped
 *	vm_map_enter:KERN_INVALID_ARGUMENT
 *					A failure code from vm_map_enter() due
 *					to an invalid argument
 */
kern_return_t
ubc_upl_map(
	upl_t		upl,
	vm_offset_t	*dst_addr)
{
	return (vm_upl_map(kernel_map, upl, dst_addr));
}


/*
 * ubc_upl_unmap
 *
 * Unmap the page list assocated with the supplied upl from the kernel virtual
 * address space; the entire upl is unmapped.
 *
 * Parameters:	upl			The upl to unmap
 *
 * Returns:	KERN_SUCCESS		The upl has been unmapped
 *		KERN_FAILURE		The upl is not currently mapped
 *		KERN_INVALID_ARGUMENT	If the upl is UPL_NULL
 */
kern_return_t
ubc_upl_unmap(
	upl_t	upl)
{
	return(vm_upl_unmap(kernel_map, upl));
}


/*
 * ubc_upl_commit
 *
 * Commit the contents of the upl to the backing store
 *
 * Parameters:	upl			The upl to commit
 *
 * Returns:	KERN_SUCCESS		The upl has been committed
 *		KERN_INVALID_ARGUMENT	The supplied upl was UPL_NULL
 *		KERN_FAILURE		The supplied upl does not represent
 *					device memory, and the offset plus the
 *					size would exceed the actual size of
 *					the upl
 *
 * Notes:	In practice, the only return value for this function should be
 *		KERN_SUCCESS, unless there has been data structure corruption;
 *		since the upl is deallocated regardless of success or failure,
 *		there's really nothing to do about this other than panic.
 *
 *		IMPORTANT: Use of this function should not be mixed with use of
 *		ubc_upl_commit_range(), due to the unconditional deallocation
 *		by this function.
 */
kern_return_t
ubc_upl_commit(
	upl_t 			upl)
{
	upl_page_info_t	*pl;
	kern_return_t 	kr;

	pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
	kr = upl_commit(upl, pl, MAX_UPL_SIZE);
	upl_deallocate(upl);
	return kr;
}


/*
 * ubc_upl_commit
 *
 * Commit the contents of the specified range of the upl to the backing store
 *
 * Parameters:	upl			The upl to commit
 *		offset			The offset into the upl
 *		size			The size of the region to be committed,
 *					starting at the specified offset
 *		flags			commit type (see below)
 *
 * Returns:	KERN_SUCCESS		The range has been committed
 *		KERN_INVALID_ARGUMENT	The supplied upl was UPL_NULL
 *		KERN_FAILURE		The supplied upl does not represent
 *					device memory, and the offset plus the
 *					size would exceed the actual size of
 *					the upl
 *
 * Notes:	IMPORTANT: If the commit is successful, and the object is now
 *		empty, the upl will be deallocated.  Since the caller cannot
 *		check that this is the case, the UPL_COMMIT_FREE_ON_EMPTY flag
 *		should generally only be used when the offset is 0 and the size
 *		is equal to the upl size.
 *
 *		The flags argument is a bitmap of flags on the rage of pages in
 *		the upl to be committed; allowable flags are:
 *
 *		o	UPL_COMMIT_FREE_ON_EMPTY	Free the upl when it is
 *							both empty and has been
 *							successfully committed
 *		o	UPL_COMMIT_CLEAR_DIRTY		Clear each pages dirty
 *							bit; will prevent a
 *							later pageout
 *		o	UPL_COMMIT_SET_DIRTY		Set each pages dirty
 *							bit; will cause a later
 *							pageout
 *		o	UPL_COMMIT_INACTIVATE		Clear each pages
 *							reference bit; the page
 *							will not be accessed
 *		o	UPL_COMMIT_ALLOW_ACCESS		Unbusy each page; pages
 *							become busy when an
 *							IOMemoryDescriptor is
 *							mapped or redirected,
 *							and we have to wait for
 *							an IOKit driver
 *
 *		The flag UPL_COMMIT_NOTIFY_EMPTY is used internally, and should
 *		not be specified by the caller.
 *
 *		The UPL_COMMIT_CLEAR_DIRTY and UPL_COMMIT_SET_DIRTY flags are
 *		mutually exclusive, and should not be combined.
 */
kern_return_t
ubc_upl_commit_range(
	upl_t 			upl,
	upl_offset_t		offset,
	upl_size_t		size,
	int				flags)
{
	upl_page_info_t	*pl;
	boolean_t		empty;
	kern_return_t 	kr;

	if (flags & UPL_COMMIT_FREE_ON_EMPTY)
		flags |= UPL_COMMIT_NOTIFY_EMPTY;

	if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
		return KERN_INVALID_ARGUMENT;
	}

	pl = UPL_GET_INTERNAL_PAGE_LIST(upl);

	kr = upl_commit_range(upl, offset, size, flags,
						  pl, MAX_UPL_SIZE, &empty);

	if((flags & UPL_COMMIT_FREE_ON_EMPTY) && empty)
		upl_deallocate(upl);

	return kr;
}


/*
 * ubc_upl_abort_range
 *
 * Abort the contents of the specified range of the specified upl
 *
 * Parameters:	upl			The upl to abort
 *		offset			The offset into the upl
 *		size			The size of the region to be aborted,
 *					starting at the specified offset
 *		abort_flags		abort type (see below)
 *
 * Returns:	KERN_SUCCESS		The range has been aborted
 *		KERN_INVALID_ARGUMENT	The supplied upl was UPL_NULL
 *		KERN_FAILURE		The supplied upl does not represent
 *					device memory, and the offset plus the
 *					size would exceed the actual size of
 *					the upl
 *
 * Notes:	IMPORTANT: If the abort is successful, and the object is now
 *		empty, the upl will be deallocated.  Since the caller cannot
 *		check that this is the case, the UPL_ABORT_FREE_ON_EMPTY flag
 *		should generally only be used when the offset is 0 and the size
 *		is equal to the upl size.
 *
 *		The abort_flags argument is a bitmap of flags on the range of
 *		pages in the upl to be aborted; allowable flags are:
 *
 *		o	UPL_ABORT_FREE_ON_EMPTY	Free the upl when it is both
 *						empty and has been successfully
 *						aborted
 *		o	UPL_ABORT_RESTART	The operation must be restarted
 *		o	UPL_ABORT_UNAVAILABLE	The pages are unavailable
 *		o	UPL_ABORT_ERROR		An I/O error occurred
 *		o	UPL_ABORT_DUMP_PAGES	Just free the pages
 *		o	UPL_ABORT_NOTIFY_EMPTY	RESERVED
 *		o	UPL_ABORT_ALLOW_ACCESS	RESERVED
 *
 *		The UPL_ABORT_NOTIFY_EMPTY is an internal use flag and should
 *		not be specified by the caller.  It is intended to fulfill the
 *		same role as UPL_COMMIT_NOTIFY_EMPTY does in the function
 *		ubc_upl_commit_range(), but is never referenced internally.
 *
 *		The UPL_ABORT_ALLOW_ACCESS is defined, but neither set nor
 *		referenced; do not use it.
 */
kern_return_t
ubc_upl_abort_range(
	upl_t			upl,
	upl_offset_t		offset,
	upl_size_t		size,
	int				abort_flags)
{
	kern_return_t 	kr;
	boolean_t		empty = FALSE;

	if (abort_flags & UPL_ABORT_FREE_ON_EMPTY)
		abort_flags |= UPL_ABORT_NOTIFY_EMPTY;

	kr = upl_abort_range(upl, offset, size, abort_flags, &empty);

	if((abort_flags & UPL_ABORT_FREE_ON_EMPTY) && empty)
		upl_deallocate(upl);

	return kr;
}


/*
 * ubc_upl_abort
 *
 * Abort the contents of the specified upl
 *
 * Parameters:	upl			The upl to abort
 *		abort_type		abort type (see below)
 *
 * Returns:	KERN_SUCCESS		The range has been aborted
 *		KERN_INVALID_ARGUMENT	The supplied upl was UPL_NULL
 *		KERN_FAILURE		The supplied upl does not represent
 *					device memory, and the offset plus the
 *					size would exceed the actual size of
 *					the upl
 *
 * Notes:	IMPORTANT: If the abort is successful, and the object is now
 *		empty, the upl will be deallocated.  Since the caller cannot
 *		check that this is the case, the UPL_ABORT_FREE_ON_EMPTY flag
 *		should generally only be used when the offset is 0 and the size
 *		is equal to the upl size.
 *
 *		The abort_type is a bitmap of flags on the range of
 *		pages in the upl to be aborted; allowable flags are:
 *
 *		o	UPL_ABORT_FREE_ON_EMPTY	Free the upl when it is both
 *						empty and has been successfully
 *						aborted
 *		o	UPL_ABORT_RESTART	The operation must be restarted
 *		o	UPL_ABORT_UNAVAILABLE	The pages are unavailable
 *		o	UPL_ABORT_ERROR		An I/O error occurred
 *		o	UPL_ABORT_DUMP_PAGES	Just free the pages
 *		o	UPL_ABORT_NOTIFY_EMPTY	RESERVED
 *		o	UPL_ABORT_ALLOW_ACCESS	RESERVED
 *
 *		The UPL_ABORT_NOTIFY_EMPTY is an internal use flag and should
 *		not be specified by the caller.  It is intended to fulfill the
 *		same role as UPL_COMMIT_NOTIFY_EMPTY does in the function
 *		ubc_upl_commit_range(), but is never referenced internally.
 *
 *		The UPL_ABORT_ALLOW_ACCESS is defined, but neither set nor
 *		referenced; do not use it.
 */
kern_return_t
ubc_upl_abort(
	upl_t			upl,
	int				abort_type)
{
	kern_return_t	kr;

	kr = upl_abort(upl, abort_type);
	upl_deallocate(upl);
	return kr;
}


/*
 * ubc_upl_pageinfo
 *
 *  Retrieve the internal page list for the specified upl
 *
 * Parameters:	upl			The upl to obtain the page list from
 *
 * Returns:	!NULL			The (upl_page_info_t *) for the page
 *					list internal to the upl
 *		NULL			Error/no page list associated
 *
 * Notes:	IMPORTANT: The function is only valid on internal objects
 *		where the list request was made with the UPL_INTERNAL flag.
 *
 *		This function is a utility helper function, since some callers
 *		may not have direct access to the header defining the macro,
 *		due to abstraction layering constraints.
 */
upl_page_info_t *
ubc_upl_pageinfo(
	upl_t			upl)
{
	return (UPL_GET_INTERNAL_PAGE_LIST(upl));
}


int
UBCINFOEXISTS(struct vnode * vp)
{
        return((vp) && ((vp)->v_type == VREG) && ((vp)->v_ubcinfo != UBC_INFO_NULL));
}


void
ubc_upl_range_needed(
	upl_t		upl,
	int		index,
	int		count)
{
	upl_range_needed(upl, index, count);
}


/*
 * CODE SIGNING
 */
#define CS_BLOB_PAGEABLE 0
static volatile SInt32 cs_blob_size = 0;
static volatile SInt32 cs_blob_count = 0;
static SInt32 cs_blob_size_peak = 0;
static UInt32 cs_blob_size_max = 0;
static SInt32 cs_blob_count_peak = 0;

int cs_validation = 1;

#ifndef SECURE_KERNEL
SYSCTL_INT(_vm, OID_AUTO, cs_validation, CTLFLAG_RW | CTLFLAG_LOCKED, &cs_validation, 0, "Do validate code signatures");
#endif
SYSCTL_INT(_vm, OID_AUTO, cs_blob_count, CTLFLAG_RD | CTLFLAG_LOCKED, (int *)(uintptr_t)&cs_blob_count, 0, "Current number of code signature blobs");
SYSCTL_INT(_vm, OID_AUTO, cs_blob_size, CTLFLAG_RD | CTLFLAG_LOCKED, (int *)(uintptr_t)&cs_blob_size, 0, "Current size of all code signature blobs");
SYSCTL_INT(_vm, OID_AUTO, cs_blob_count_peak, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_blob_count_peak, 0, "Peak number of code signature blobs");
SYSCTL_INT(_vm, OID_AUTO, cs_blob_size_peak, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_blob_size_peak, 0, "Peak size of code signature blobs");
SYSCTL_INT(_vm, OID_AUTO, cs_blob_size_max, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_blob_size_max, 0, "Size of biggest code signature blob");


kern_return_t
ubc_cs_blob_allocate(
	vm_offset_t	*blob_addr_p,
	vm_size_t	*blob_size_p)
{
	kern_return_t	kr;

#if CS_BLOB_PAGEABLE
	*blob_size_p = round_page(*blob_size_p);
	kr = kmem_alloc(kernel_map, blob_addr_p, *blob_size_p);
#else	/* CS_BLOB_PAGEABLE */
	*blob_addr_p = (vm_offset_t) kalloc(*blob_size_p);
	if (*blob_addr_p == 0) {
		kr = KERN_NO_SPACE;
	} else {
		kr = KERN_SUCCESS;
	}
#endif	/* CS_BLOB_PAGEABLE */
	return kr;
}

void
ubc_cs_blob_deallocate(
	vm_offset_t	blob_addr,
	vm_size_t	blob_size)
{
#if CS_BLOB_PAGEABLE
	kmem_free(kernel_map, blob_addr, blob_size);
#else	/* CS_BLOB_PAGEABLE */
	kfree((void *) blob_addr, blob_size);
#endif	/* CS_BLOB_PAGEABLE */
}

int
ubc_cs_sigpup_add(
	struct vnode	*vp,
	vm_address_t	address,
	vm_size_t	size)
{
	kern_return_t		kr;
	struct ubc_info		*uip;
	struct cs_blob		*blob;
	memory_object_control_t control;
	const CS_CodeDirectory *cd;
	int			error;

	control = ubc_getobject(vp, UBC_FLAGS_NONE);
	if (control == MEMORY_OBJECT_CONTROL_NULL)
		return KERN_INVALID_ARGUMENT;

	if (memory_object_is_signed(control))
		return 0;

	blob = (struct cs_blob *) kalloc(sizeof (struct cs_blob));
	if (blob == NULL)
		return ENOMEM;

	/* fill in the new blob */
	blob->csb_cpu_type = CPU_TYPE_ANY;
	blob->csb_base_offset = 0;
	blob->csb_mem_size = size;
	blob->csb_mem_offset = 0;
	blob->csb_mem_handle = IPC_PORT_NULL;
	blob->csb_mem_kaddr = address;
	blob->csb_sigpup = 1;

	/*
	 * Validate the blob's contents
	 */
	cd = findCodeDirectory(
		(const CS_SuperBlob *) address,
		(char *) address,
		(char *) address + blob->csb_mem_size);
	if (cd == NULL) {
		/* no code directory => useless blob ! */
		error = EINVAL;
		goto out;
	}

	blob->csb_flags = ntohl(cd->flags) | CS_VALID;
	blob->csb_end_offset = round_page(ntohl(cd->codeLimit));
	if((ntohl(cd->version) >= CS_SUPPORTSSCATTER) && (ntohl(cd->scatterOffset))) {
		const SC_Scatter *scatter = (const SC_Scatter*)
		    ((const char*)cd + ntohl(cd->scatterOffset));
		blob->csb_start_offset = ntohl(scatter->base) * PAGE_SIZE;
	} else {
		blob->csb_start_offset = (blob->csb_end_offset - (ntohl(cd->nCodeSlots) * PAGE_SIZE));
	}

	/*
	 * We don't need to check with the policy module, since the input data is supposed to be already checked
	 */

	vnode_lock(vp);
	if (! UBCINFOEXISTS(vp)) {
		vnode_unlock(vp);
		if (cs_debug)
			printf("out ubc object\n");
		error = ENOENT;
		goto out;
	}
	uip = vp->v_ubcinfo;

	/* someone raced us to adding the code directory */
	if (uip->cs_blobs != NULL) {
		if (cs_debug)
			printf("sigpup: vnode already have CD ?\n");
		vnode_unlock(vp);
		error = EEXIST;
		goto out;
	}

	blob->csb_next = uip->cs_blobs;
	uip->cs_blobs = blob;

	OSAddAtomic(+1, &cs_blob_count);
	OSAddAtomic((SInt32) +blob->csb_mem_size, &cs_blob_size);

	/* mark this vnode's VM object as having "signed pages" */
	kr = memory_object_signed(uip->ui_control, TRUE);
	if (kr != KERN_SUCCESS) {
		vnode_unlock(vp);
		if (cs_debug)
			printf("sigpup: not signable ?\n");
		error = ENOENT;
		goto out;
	}

	vnode_unlock(vp);

	error = 0;
out:
	if (error) {
		if (cs_debug)
			printf("sigpup: not signable ?\n");
		/* we failed; release what we allocated */
		if (blob) {
			kfree(blob, sizeof (*blob));
			blob = NULL;
		}
	}

	return error;
}

int
ubc_cs_blob_add(
	struct vnode	*vp,
	cpu_type_t	cputype,
	off_t		base_offset,
	vm_address_t	addr,
	off_t		blob_offset,
	vm_size_t	size)
{
	kern_return_t		kr;
	struct ubc_info		*uip;
	struct cs_blob		*blob, *oblob;
	int			error;
	ipc_port_t		blob_handle;
	memory_object_size_t	blob_size;
	const CS_CodeDirectory *cd;
	off_t			blob_start_offset, blob_end_offset;
	SHA1_CTX		sha1ctxt;
	boolean_t		record_mtime;

	record_mtime = FALSE;

	blob_handle = IPC_PORT_NULL;

	blob = (struct cs_blob *) kalloc(sizeof (struct cs_blob));
	if (blob == NULL) {
		return ENOMEM;
	}

#if CS_BLOB_PAGEABLE
	/* get a memory entry on the blob */
	blob_size = (memory_object_size_t) size;
	kr = mach_make_memory_entry_64(kernel_map,
				       &blob_size,
				       addr,
				       VM_PROT_READ,
				       &blob_handle,
				       IPC_PORT_NULL);
	if (kr != KERN_SUCCESS) {
		error = ENOMEM;
		goto out;
	}
	if (memory_object_round_page(blob_size) !=
	    (memory_object_size_t) round_page(size)) {
		printf("ubc_cs_blob_add: size mismatch 0x%llx 0x%lx !?\n",
		       blob_size, (size_t)size);
		panic("XXX FBDP size mismatch 0x%llx 0x%lx\n", blob_size, (size_t)size);
		error = EINVAL;
		goto out;
	}
#else
	blob_size = (memory_object_size_t) size;
	blob_handle = IPC_PORT_NULL;
#endif

	/* fill in the new blob */
	blob->csb_cpu_type = cputype;
	blob->csb_sigpup = 0;
	blob->csb_base_offset = base_offset;
	blob->csb_blob_offset = blob_offset;
	blob->csb_mem_size = size;
	blob->csb_mem_offset = 0;
	blob->csb_mem_handle = blob_handle;
	blob->csb_mem_kaddr = addr;
	blob->csb_flags = 0;

	/*
	 * Validate the blob's contents
	 */

	error = cs_validate_csblob((const uint8_t *)addr, size, &cd);
	if (error) {
		if (cs_debug)
			printf("CODESIGNING: csblob invalid: %d\n", error);
		blob->csb_flags = 0;
		blob->csb_start_offset = 0;
		blob->csb_end_offset = 0;
		memset(blob->csb_sha1, 0, SHA1_RESULTLEN);
		/* let the vnode checker determine if the signature is valid or not */
	} else {
		const unsigned char *sha1_base;
		int sha1_size;

		blob->csb_flags = (ntohl(cd->flags) & CS_ALLOWED_MACHO) | CS_VALID;
		blob->csb_end_offset = round_page(ntohl(cd->codeLimit));
		if((ntohl(cd->version) >= CS_SUPPORTSSCATTER) && (ntohl(cd->scatterOffset))) {
			const SC_Scatter *scatter = (const SC_Scatter*)
				((const char*)cd + ntohl(cd->scatterOffset));
			blob->csb_start_offset = ntohl(scatter->base) * PAGE_SIZE;
		} else {
			blob->csb_start_offset = (blob->csb_end_offset -
						  (ntohl(cd->nCodeSlots) * PAGE_SIZE));
		}
		/* compute the blob's SHA1 hash */
		sha1_base = (const unsigned char *) cd;
		sha1_size = ntohl(cd->length);
		SHA1Init(&sha1ctxt);
		SHA1Update(&sha1ctxt, sha1_base, sha1_size);
		SHA1Final(blob->csb_sha1, &sha1ctxt);
	}

	/*
	 * Let policy module check whether the blob's signature is accepted.
	 */
#if CONFIG_MACF
	error = mac_vnode_check_signature(vp, base_offset, blob->csb_sha1, (void*)addr, size);
	if (error)
		goto out;
#endif

	/*
	 * Validate the blob's coverage
	 */
	blob_start_offset = blob->csb_base_offset + blob->csb_start_offset;
	blob_end_offset = blob->csb_base_offset + blob->csb_end_offset;

	if (blob_start_offset >= blob_end_offset ||
	    blob_start_offset < 0 ||
	    blob_end_offset <= 0) {
		/* reject empty or backwards blob */
		error = EINVAL;
		goto out;
	}

	vnode_lock(vp);
	if (! UBCINFOEXISTS(vp)) {
		vnode_unlock(vp);
		error = ENOENT;
		goto out;
	}
	uip = vp->v_ubcinfo;

	/* check if this new blob overlaps with an existing blob */
	for (oblob = uip->cs_blobs;
	     oblob != NULL;
	     oblob = oblob->csb_next) {
		 off_t oblob_start_offset, oblob_end_offset;

		 oblob_start_offset = (oblob->csb_base_offset +
				       oblob->csb_start_offset);
		 oblob_end_offset = (oblob->csb_base_offset +
				     oblob->csb_end_offset);
		 if (blob_start_offset >= oblob_end_offset ||
		     blob_end_offset <= oblob_start_offset) {
			 /* no conflict with this existing blob */
		 } else {
			 /* conflict ! */
			 if (blob_start_offset == oblob_start_offset &&
			     blob_end_offset == oblob_end_offset &&
			     blob->csb_mem_size == oblob->csb_mem_size &&
			     blob->csb_flags == oblob->csb_flags &&
			     (blob->csb_cpu_type == CPU_TYPE_ANY ||
			      oblob->csb_cpu_type == CPU_TYPE_ANY ||
			      blob->csb_cpu_type == oblob->csb_cpu_type) &&
			     !bcmp(blob->csb_sha1,
				   oblob->csb_sha1,
				   SHA1_RESULTLEN)) {
				 /*
				  * We already have this blob:
				  * we'll return success but
				  * throw away the new blob.
				  */
				 if (oblob->csb_cpu_type == CPU_TYPE_ANY) {
					 /*
					  * The old blob matches this one
					  * but doesn't have any CPU type.
					  * Update it with whatever the caller
					  * provided this time.
					  */
					 oblob->csb_cpu_type = cputype;
				 }
				 /*
				  * If the same blob moved around in the Mach-O, we
				  * want to remember the new blob offset to avoid
				  * coming back here again and again.
				  */
				 oblob->csb_blob_offset = blob_offset;

				 vnode_unlock(vp);
				 error = EAGAIN;
				 goto out;
			 } else {
				 /* different blob: reject the new one */
				 char pathbuf[MAXPATHLEN];
				 char new_sha1_str[2*SHA1_RESULTLEN+1];
				 char old_sha1_str[2*SHA1_RESULTLEN+1];
				 char arch_str[20];
				 const char *pathp = "?unknown";
				 int pblen = sizeof(pathbuf);
				 if (vn_getpath(vp, pathbuf, &pblen) == 0) {
				 	/* pblen == strlen(pathbuf) + 1. Assume strlen(pathbuf) > 0 */
					for (pathp = pathbuf + pblen - 2; pathp > pathbuf && pathp[-1] != '/'; pathp--) ;
				 }
				 snprintf(arch_str, sizeof(arch_str), "%x", cputype);
				 hex_str(oblob->csb_sha1, SHA1_RESULTLEN, old_sha1_str);
				 hex_str(blob->csb_sha1, SHA1_RESULTLEN, new_sha1_str);
				 kern_asl_msg(LOG_NOTICE, "messagetracer",
					6,
					"com.apple.message.domain", "com.apple.kernel.cs.replace",
					"com.apple.message.signature", pathp,
					"com.apple.message.signature2", arch_str,
					"com.apple.message.signature3", old_sha1_str,
					"com.apple.message.result", new_sha1_str,
					"com.apple.message.summarize", "YES",
					NULL
				 );
				 printf("CODESIGNING: rejected new signature for architecture %d of file %s\n",
					cputype, pathbuf);
				 vnode_unlock(vp);
				 error = EALREADY;
				 goto out;
			 }
		 }

	}

	/* mark this vnode's VM object as having "signed pages" */
	kr = memory_object_signed(uip->ui_control, TRUE);
	if (kr != KERN_SUCCESS) {
		vnode_unlock(vp);
		error = ENOENT;
		goto out;
	}

	if (uip->cs_blobs == NULL) {
		/* loading 1st blob: record the file's current "modify time" */
		record_mtime = TRUE;
	}

	/*
	 * Add this blob to the list of blobs for this vnode.
	 * We always add at the front of the list and we never remove a
	 * blob from the list, so ubc_cs_get_blobs() can return whatever
	 * the top of the list was and that list will remain valid
	 * while we validate a page, even after we release the vnode's lock.
	 */
	blob->csb_next = uip->cs_blobs;
	uip->cs_blobs = blob;

	OSAddAtomic(+1, &cs_blob_count);
	if (cs_blob_count > cs_blob_count_peak) {
		cs_blob_count_peak = cs_blob_count; /* XXX atomic ? */
	}
	OSAddAtomic((SInt32) +blob->csb_mem_size, &cs_blob_size);
	if ((SInt32) cs_blob_size > cs_blob_size_peak) {
		cs_blob_size_peak = (SInt32) cs_blob_size; /* XXX atomic ? */
	}
	if ((UInt32) blob->csb_mem_size > cs_blob_size_max) {
		cs_blob_size_max = (UInt32) blob->csb_mem_size;
	}

	if (cs_debug > 1) {
		proc_t p;
		const char *name = vnode_getname_printable(vp);
		p = current_proc();
		printf("CODE SIGNING: proc %d(%s) "
		       "loaded %s signatures for file (%s) "
		       "range 0x%llx:0x%llx flags 0x%x\n",
		       p->p_pid, p->p_comm,
		       blob->csb_cpu_type == -1 ? "detached" : "embedded",
		       name,
		       blob->csb_base_offset + blob->csb_start_offset,
		       blob->csb_base_offset + blob->csb_end_offset,
		       blob->csb_flags);
		vnode_putname_printable(name);
	}

	vnode_unlock(vp);

	if (record_mtime) {
		vnode_mtime(vp, &uip->cs_mtime, vfs_context_current());
	}

	error = 0;	/* success ! */

out:
	if (error) {
		/* we failed; release what we allocated */
		if (blob) {
			kfree(blob, sizeof (*blob));
			blob = NULL;
		}
		if (blob_handle != IPC_PORT_NULL) {
			mach_memory_entry_port_release(blob_handle);
			blob_handle = IPC_PORT_NULL;
		}
	}

	if (error == EAGAIN) {
		/*
		 * See above:  error is EAGAIN if we were asked
		 * to add an existing blob again.  We cleaned the new
		 * blob and we want to return success.
		 */
		error = 0;
		/*
		 * Since we're not failing, consume the data we received.
		 */
		ubc_cs_blob_deallocate(addr, size);
	}

	return error;
}


struct cs_blob *
ubc_cs_blob_get(
	struct vnode	*vp,
	cpu_type_t	cputype,
	off_t		offset)
{
	struct ubc_info	*uip;
	struct cs_blob	*blob;
	off_t offset_in_blob;

	vnode_lock_spin(vp);

	if (! UBCINFOEXISTS(vp)) {
		blob = NULL;
		goto out;
	}

	uip = vp->v_ubcinfo;
	for (blob = uip->cs_blobs;
	     blob != NULL;
	     blob = blob->csb_next) {
		if (cputype != -1 && blob->csb_cpu_type == cputype) {
			break;
		}
		if (offset != -1) {
			offset_in_blob = offset - blob->csb_base_offset;
			if (offset_in_blob >= blob->csb_start_offset &&
			    offset_in_blob < blob->csb_end_offset) {
				/* our offset is covered by this blob */
				break;
			}
		}
	}

	if (cs_debug && blob != NULL && blob->csb_sigpup)
		printf("found sig pup blob\n");
out:
	vnode_unlock(vp);

	return blob;
}

static void
ubc_cs_free(
	struct ubc_info	*uip)
{
	struct cs_blob	*blob, *next_blob;

	for (blob = uip->cs_blobs;
	     blob != NULL;
	     blob = next_blob) {
		next_blob = blob->csb_next;
		if (blob->csb_mem_kaddr != 0 && !blob->csb_sigpup) {
			ubc_cs_blob_deallocate(blob->csb_mem_kaddr,
					       blob->csb_mem_size);
			blob->csb_mem_kaddr = 0;
		}
		if (blob->csb_mem_handle != IPC_PORT_NULL) {
			mach_memory_entry_port_release(blob->csb_mem_handle);
		}
		blob->csb_mem_handle = IPC_PORT_NULL;
		OSAddAtomic(-1, &cs_blob_count);
		OSAddAtomic((SInt32) -blob->csb_mem_size, &cs_blob_size);
		kfree(blob, sizeof (*blob));
	}
#if CHECK_CS_VALIDATION_BITMAP
	ubc_cs_validation_bitmap_deallocate( uip->ui_vnode );
#endif
	uip->cs_blobs = NULL;
}

struct cs_blob *
ubc_get_cs_blobs(
	struct vnode	*vp)
{
	struct ubc_info	*uip;
	struct cs_blob	*blobs;

	/*
	 * No need to take the vnode lock here.  The caller must be holding
	 * a reference on the vnode (via a VM mapping or open file descriptor),
	 * so the vnode will not go away.  The ubc_info stays until the vnode
	 * goes away.  And we only modify "blobs" by adding to the head of the
	 * list.
	 * The ubc_info could go away entirely if the vnode gets reclaimed as
	 * part of a forced unmount.  In the case of a code-signature validation
	 * during a page fault, the "paging_in_progress" reference on the VM
	 * object guarantess that the vnode pager (and the ubc_info) won't go
	 * away during the fault.
	 * Other callers need to protect against vnode reclaim by holding the
	 * vnode lock, for example.
	 */

	if (! UBCINFOEXISTS(vp)) {
		blobs = NULL;
		goto out;
	}

	uip = vp->v_ubcinfo;
	blobs = uip->cs_blobs;

out:
	return blobs;
}

void
ubc_get_cs_mtime(
	struct vnode	*vp,
	struct timespec	*cs_mtime)
{
	struct ubc_info	*uip;

	if (! UBCINFOEXISTS(vp)) {
		cs_mtime->tv_sec = 0;
		cs_mtime->tv_nsec = 0;
		return;
	}

	uip = vp->v_ubcinfo;
	cs_mtime->tv_sec = uip->cs_mtime.tv_sec;
	cs_mtime->tv_nsec = uip->cs_mtime.tv_nsec;
}

unsigned long cs_validate_page_no_hash = 0;
unsigned long cs_validate_page_bad_hash = 0;
boolean_t
cs_validate_page(
	void			*_blobs,
	memory_object_t		pager,
	memory_object_offset_t	page_offset,
	const void		*data,
	boolean_t		*tainted)
{
	SHA1_CTX		sha1ctxt;
	unsigned char		actual_hash[SHA1_RESULTLEN];
	unsigned char		expected_hash[SHA1_RESULTLEN];
	boolean_t		found_hash;
	struct cs_blob		*blobs, *blob;
	const CS_CodeDirectory	*cd;
	const CS_SuperBlob	*embedded;
	const unsigned char	*hash;
	boolean_t		validated;
	off_t			offset;	/* page offset in the file */
	size_t			size;
	off_t			codeLimit = 0;
	char			*lower_bound, *upper_bound;
	vm_offset_t		kaddr, blob_addr;
	vm_size_t		ksize;
	kern_return_t		kr;

	offset = page_offset;

	/* retrieve the expected hash */
	found_hash = FALSE;
	blobs = (struct cs_blob *) _blobs;

	for (blob = blobs;
	     blob != NULL;
	     blob = blob->csb_next) {
		offset = page_offset - blob->csb_base_offset;
		if (offset < blob->csb_start_offset ||
		    offset >= blob->csb_end_offset) {
			/* our page is not covered by this blob */
			continue;
		}

		/* map the blob in the kernel address space */
		kaddr = blob->csb_mem_kaddr;
		if (kaddr == 0) {
			ksize = (vm_size_t) (blob->csb_mem_size +
					     blob->csb_mem_offset);
			kr = vm_map(kernel_map,
				    &kaddr,
				    ksize,
				    0,
				    VM_FLAGS_ANYWHERE,
				    blob->csb_mem_handle,
				    0,
				    TRUE,
				    VM_PROT_READ,
				    VM_PROT_READ,
				    VM_INHERIT_NONE);
			if (kr != KERN_SUCCESS) {
				/* XXX FBDP what to do !? */
				printf("cs_validate_page: failed to map blob, "
				       "size=0x%lx kr=0x%x\n",
				       (size_t)blob->csb_mem_size, kr);
				break;
			}
		}
		if (blob->csb_sigpup && cs_debug)
			printf("checking for a sigpup CD\n");

		blob_addr = kaddr + blob->csb_mem_offset;

		lower_bound = CAST_DOWN(char *, blob_addr);
		upper_bound = lower_bound + blob->csb_mem_size;

		embedded = (const CS_SuperBlob *) blob_addr;
		cd = findCodeDirectory(embedded, lower_bound, upper_bound);
		if (cd != NULL) {
			if (cd->pageSize != PAGE_SHIFT ||
			    cd->hashType != CS_HASHTYPE_SHA1 ||
			    cd->hashSize != SHA1_RESULTLEN) {
				/* bogus blob ? */
				if (blob->csb_sigpup && cs_debug)
					printf("page foo bogus sigpup CD\n");
				continue;
			}

			offset = page_offset - blob->csb_base_offset;
			if (offset < blob->csb_start_offset ||
			    offset >= blob->csb_end_offset) {
				/* our page is not covered by this blob */
				if (blob->csb_sigpup && cs_debug)
					printf("OOB sigpup CD\n");
				continue;
			}

			codeLimit = ntohl(cd->codeLimit);
			if (blob->csb_sigpup && cs_debug)
				printf("sigpup codesize %d\n", (int)codeLimit);

			hash = hashes(cd, (unsigned)atop(offset),
				      lower_bound, upper_bound);
			if (hash != NULL) {
				bcopy(hash, expected_hash,
				      sizeof (expected_hash));
				found_hash = TRUE;
				if (blob->csb_sigpup && cs_debug)
					printf("sigpup hash\n");
			}

			break;
		} else {
			if (blob->csb_sigpup && cs_debug)
				printf("sig pup had no valid CD\n");

		}
	}

	if (found_hash == FALSE) {
		/*
		 * We can't verify this page because there is no signature
		 * for it (yet).  It's possible that this part of the object
		 * is not signed, or that signatures for that part have not
		 * been loaded yet.
		 * Report that the page has not been validated and let the
		 * caller decide if it wants to accept it or not.
		 */
		cs_validate_page_no_hash++;
		if (cs_debug > 1) {
			printf("CODE SIGNING: cs_validate_page: "
			       "mobj %p off 0x%llx: no hash to validate !?\n",
			       pager, page_offset);
		}
		validated = FALSE;
		*tainted = FALSE;
	} else {

		size = PAGE_SIZE;
		if ((off_t)(offset + size) > codeLimit) {
			/* partial page at end of segment */
			assert(offset < codeLimit);
			size = (size_t) (codeLimit & PAGE_MASK);
		}
		/* compute the actual page's SHA1 hash */
		SHA1Init(&sha1ctxt);
		SHA1UpdateUsePhysicalAddress(&sha1ctxt, data, size);
		SHA1Final(actual_hash, &sha1ctxt);

		if (bcmp(expected_hash, actual_hash, SHA1_RESULTLEN) != 0) {
			char asha1_str[2*SHA1_RESULTLEN+1];
			char esha1_str[2*SHA1_RESULTLEN+1];
			hex_str(actual_hash, SHA1_RESULTLEN, asha1_str);
			hex_str(expected_hash, SHA1_RESULTLEN, esha1_str);
			if (cs_debug) {
				printf("CODE SIGNING: cs_validate_page: "
				       "mobj %p off 0x%llx size 0x%lx: actual %s expected %s\n",
				       pager, page_offset, size, asha1_str, esha1_str);
			}
			cs_validate_page_bad_hash++;
			if (!*tainted) {
				char page_offset_str[20];
				snprintf(page_offset_str, sizeof(page_offset_str), "%llx", page_offset);
				 kern_asl_msg(LOG_NOTICE, "messagetracer",
					5,
					"com.apple.message.domain", "com.apple.kernel.cs.mismatch",
					"com.apple.message.signature", page_offset_str,
					"com.apple.message.signature2", asha1_str,
					"com.apple.message.signature3", esha1_str,
					"com.apple.message.summarize", "YES",
					NULL
				 );
			}
			*tainted = TRUE;
		} else {
			if (cs_debug > 10) {
				printf("CODE SIGNING: cs_validate_page: "
				       "mobj %p off 0x%llx size 0x%lx: "
				       "SHA1 OK\n",
				       pager, page_offset, size);
			}
			*tainted = FALSE;
		}
		validated = TRUE;
	}

	return validated;
}

int
ubc_cs_getcdhash(
	vnode_t		vp,
	off_t		offset,
	unsigned char	*cdhash)
{
	struct cs_blob	*blobs, *blob;
	off_t		rel_offset;
	int		ret;

	vnode_lock(vp);

	blobs = ubc_get_cs_blobs(vp);
	for (blob = blobs;
	     blob != NULL;
	     blob = blob->csb_next) {
		/* compute offset relative to this blob */
		rel_offset = offset - blob->csb_base_offset;
		if (rel_offset >= blob->csb_start_offset &&
		    rel_offset < blob->csb_end_offset) {
			/* this blob does cover our "offset" ! */
			break;
		}
	}

	if (blob == NULL) {
		/* we didn't find a blob covering "offset" */
		ret = EBADEXEC; /* XXX any better error ? */
	} else {
		/* get the SHA1 hash of that blob */
		bcopy(blob->csb_sha1, cdhash, sizeof (blob->csb_sha1));
		ret = 0;
	}

	vnode_unlock(vp);

	return ret;
}

#if CHECK_CS_VALIDATION_BITMAP
#define stob(s)	((atop_64((s)) + 07) >> 3)
extern	boolean_t	root_fs_upgrade_try;

/*
 * Should we use the code-sign bitmap to avoid repeated code-sign validation?
 * Depends:
 * a) Is the target vnode on the root filesystem?
 * b) Has someone tried to mount the root filesystem read-write?
 * If answers are (a) yes AND (b) no, then we can use the bitmap.
 */
#define USE_CODE_SIGN_BITMAP(vp)	( (vp != NULL) && (vp->v_mount != NULL) && (vp->v_mount->mnt_flag & MNT_ROOTFS) && !root_fs_upgrade_try)
kern_return_t
ubc_cs_validation_bitmap_allocate(
	vnode_t		vp)
{
	kern_return_t	kr = KERN_SUCCESS;
	struct ubc_info *uip;
	char		*target_bitmap;
	vm_object_size_t	bitmap_size;

	if ( ! USE_CODE_SIGN_BITMAP(vp) || (! UBCINFOEXISTS(vp))) {
		kr = KERN_INVALID_ARGUMENT;
	} else {
		uip = vp->v_ubcinfo;

		if ( uip->cs_valid_bitmap == NULL ) {
			bitmap_size = stob(uip->ui_size);
			target_bitmap = (char*) kalloc( (vm_size_t)bitmap_size );
			if (target_bitmap == 0) {
				kr = KERN_NO_SPACE;
			} else {
				kr = KERN_SUCCESS;
			}
			if( kr == KERN_SUCCESS ) {
				memset( target_bitmap, 0, (size_t)bitmap_size);
				uip->cs_valid_bitmap = (void*)target_bitmap;
				uip->cs_valid_bitmap_size = bitmap_size;
			}
		}
	}
	return kr;
}

kern_return_t
ubc_cs_check_validation_bitmap (
	vnode_t			vp,
	memory_object_offset_t		offset,
	int			optype)
{
	kern_return_t	kr = KERN_SUCCESS;

	if ( ! USE_CODE_SIGN_BITMAP(vp) || ! UBCINFOEXISTS(vp)) {
		kr = KERN_INVALID_ARGUMENT;
	} else {
		struct ubc_info *uip = vp->v_ubcinfo;
		char		*target_bitmap = uip->cs_valid_bitmap;

		if ( target_bitmap == NULL ) {
		       kr = KERN_INVALID_ARGUMENT;
		} else {
			uint64_t	bit, byte;
			bit = atop_64( offset );
			byte = bit >> 3;

			if ( byte > uip->cs_valid_bitmap_size ) {
			       kr = KERN_INVALID_ARGUMENT;
			} else {

				if (optype == CS_BITMAP_SET) {
					target_bitmap[byte] |= (1 << (bit & 07));
					kr = KERN_SUCCESS;
				} else if (optype == CS_BITMAP_CLEAR) {
					target_bitmap[byte] &= ~(1 << (bit & 07));
					kr = KERN_SUCCESS;
				} else if (optype == CS_BITMAP_CHECK) {
					if ( target_bitmap[byte] & (1 << (bit & 07))) {
						kr = KERN_SUCCESS;
					} else {
						kr = KERN_FAILURE;
					}
				}
			}
		}
	}
	return kr;
}

void
ubc_cs_validation_bitmap_deallocate(
	vnode_t		vp)
{
	struct ubc_info *uip;
	void		*target_bitmap;
	vm_object_size_t	bitmap_size;

	if ( UBCINFOEXISTS(vp)) {
		uip = vp->v_ubcinfo;

		if ( (target_bitmap = uip->cs_valid_bitmap) != NULL ) {
			bitmap_size = uip->cs_valid_bitmap_size;
			kfree( target_bitmap, (vm_size_t) bitmap_size );
			uip->cs_valid_bitmap = NULL;
		}
	}
}
#else
kern_return_t	ubc_cs_validation_bitmap_allocate(__unused vnode_t vp){
	return KERN_INVALID_ARGUMENT;
}

kern_return_t ubc_cs_check_validation_bitmap(
	__unused struct vnode *vp,
	__unused memory_object_offset_t offset,
	__unused int optype){

	return KERN_INVALID_ARGUMENT;
}

void	ubc_cs_validation_bitmap_deallocate(__unused vnode_t vp){
	return;
}
#endif /* CHECK_CS_VALIDATION_BITMAP */