bsd/vfs/vfs_cluster.c

85 #include <mach/upl.h>
162 static int cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
165 static int cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags, vnode_t vp);
172 static void cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int take_reference);
511 cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags, vnode_t vp)
521 	        ubc_upl_commit_range(upl, upl_offset, abort_size, UPL_COMMIT_FREE_ON_EMPTY);
544 		ubc_upl_abort_range(upl, upl_offset, abort_size, upl_abort_code);
563 	upl_t	upl;
626 	upl        = cbp->b_upl;
671 	        cluster_zero(upl, zero_offset, PAGE_SIZE - (zero_offset & PAGE_MASK), real_bp);
709 			upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, commit_size, error, b_flags, vp);
719 			ubc_upl_commit_range(upl, upl_offset - pg_offset, commit_size, upl_flags);
732 		     upl, upl_offset - pg_offset, commit_size, (error << 24) | upl_flags, 0);
750 cluster_zero(upl_t upl, upl_offset_t upl_offset, int size, buf_t bp)
760 	        pl = ubc_upl_pageinfo(upl);
883 cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
1095 			if (upl_get_internal_vectorupl(upl))
1200 			if (vnode_pageout(vp, upl, trunc_page(upl_offset), trunc_page_64(f_offset), PAGE_SIZE, pageout_flags, NULL) != PAGER_SUCCESS) {
1261 			 * out the affected area in the upl
1265 				 * if this upl contains the EOF and it is not a multiple of PAGE_SIZE
1270 				 * last page in this upl... we need to zero both the hole and the tail
1283 			cluster_zero(upl, upl_offset, bytes_to_zero, real_bp);
1333 			        ubc_upl_commit_range(upl, commit_offset, pg_count * PAGE_SIZE,
1446 		if (buf_setupl(cbp, upl, upl_offset))
1565 			 * Rewind the upl offset to the beginning of the
1614 			upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, abort_size, error, io_flags, vp);
1617 				     upl, upl_offset - pg_offset, abort_size, (error << 24) | upl_flags, 0);
1768 cluster_pageout(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
1771         return cluster_pageout_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
1777 cluster_pageout_ext(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
1803 	 * big the upl really is
1810 		        ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1822 			ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1836 			ubc_upl_abort_range(upl, upl_offset + rounded_size, size - rounded_size,
1839 	return (cluster_io(vp, upl, upl_offset, f_offset, io_size,
1845 cluster_pagein(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
1848         return cluster_pagein_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
1853 cluster_pagein_ext(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
1862 	if (upl == NULL || size < 0)
1863 	        panic("cluster_pagein: NULL upl passed in");
1887 		        ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1900 		ubc_upl_abort_range(upl, upl_offset + rounded_size,
1903 	retval = cluster_io(vp, upl, upl_offset, f_offset, io_size,
2089 	upl_t            upl;
2258 					      &upl,
2276 			pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
2288 			 * needed... release this upl and try again
2290 			ubc_upl_abort(upl, 0);
2319 		        ubc_upl_abort(upl, 0);
2365 		        ubc_upl_abort(upl, 0);
2374 			retval = cluster_io(vp, upl, upl_offset, uio->uio_offset,
2384 			vector_upl_set_subupl(vector_upl,upl,upl_size);
2385 			vector_upl_set_iostate(vector_upl, upl, vector_upl_size, upl_size);
2484  	upl_t            upl[MAX_VECTS];
2535 			      &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
2556 	pl = ubc_upl_pageinfo(upl[cur_upl]);
2623 	        error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset,
2670 	        ubc_upl_abort(upl[n], 0);
2687 cluster_zero_range(upl_t upl, upl_page_info_t *pl, int flags, int io_offset, off_t zero_off, off_t upl_f_offset, int bytes_to_zero)
2706 		cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2717 	upl_t            upl;
2891 		 * compute the size of the upl needed to encompass
2920 				      &upl,
2927 			upl, (int)upl_f_offset, start_offset, 0, 0);
2933 			 * we're starting in the middle of the first page of the upl
2942 		        retval = cluster_io(vp, upl, 0, upl_f_offset, read_size,
2948 				 * to release the rest of the pages in the upl without modifying
2951 				ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES|UPL_ABORT_FREE_ON_EMPTY);
2954 				        ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2957 					     upl, 0, 0, retval, 0);
2963 			 * the last offset we're writing to in this upl does not end on a page
2978 			        retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, read_size,
2984 					 * need to release the rest of the pages in the upl without
2987 					ubc_upl_abort_range(upl, upl_offset, PAGE_SIZE, UPL_ABORT_DUMP_PAGES|UPL_ABORT_FREE_ON_EMPTY);
2990 					        ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2993 						     upl, 0, 0, retval, 0);
3008 			bytes_to_zero = cluster_zero_range(upl, pl, flags, io_offset, zero_off, upl_f_offset, bytes_to_zero);
3021 			retval = cluster_copy_upl_data(uio, upl, io_offset, (int *)&io_requested);
3024 				ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
3027 					     upl, 0, 0, retval, 0);
3041 			bytes_to_zero = cluster_zero_range(upl, pl, flags, io_offset, zero_off1, upl_f_offset, bytes_to_zero);
3061 			        cluster_zero(upl, io_size, upl_size - io_size, NULL);
3064 			 * release the upl now if we hold one since...
3068 			 *    this upl, a deadlock may result on page BUSY
3081 			ubc_upl_commit_range(upl, 0, upl_size,
3338 			 * we've already dropped the current upl, so pick it back up with COPYOUT_FROM set
3448 cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int take_reference)
3457 		ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, range * PAGE_SIZE, abort_flags);
3466 	upl_t            upl;
3683 		 * compute the size of the upl needed to encompass
3708 			     upl, (int)upl_f_offset, upl_size, start_offset, 0);
3713 				      &upl,
3720 			     upl, (int)upl_f_offset, upl_size, start_offset, 0);
3723 		 * scan from the beginning of the upl looking for the first
3735 		 * page before the end of the upl is reached, if we
3766 			error = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset,
3786 			 * pages that were present in the upl when we acquired it.
3801 			        ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
3868 			        retval = cluster_copy_upl_data(uio, upl, start_offset, (int *)&io_requested);
3885 			KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START, upl, start_pg * PAGE_SIZE, io_size, error, 0);
3888 			        ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, io_size,
3898 			        ubc_upl_commit_range(upl, start_pg * PAGE_SIZE, io_size, commit_flags);
3900 			KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, upl, start_pg * PAGE_SIZE, io_size, error, 0);
3905 			 * the entire upl... so just release these without modifying
3909 				ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
3913 					     upl, -1, pages_in_upl - (last_pg - start_pg), 0, 0);
3917 				 * the upl... release these appropriately
3919 				cluster_read_upl_release(upl, 0, start_pg, take_reference);
3925 				cluster_read_upl_release(upl, last_pg, uio_last, take_reference);
3927 				KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, upl, -1, -1, 0, 0);
3991 	upl_t            upl;
4320 						 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags);
4335 			pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
4344 			ubc_upl_abort(upl, 0);
4362 			ubc_upl_abort(upl, 0);
4397 			ubc_upl_abort(upl, 0);
4402 			     upl, (int)upl_offset, (int)uio->uio_offset, io_size, 0);
4411 			retval = cluster_io(vp, upl, upl_offset, uio->uio_offset, io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
4421 			vector_upl_set_subupl(vector_upl,upl, upl_size);
4422 			vector_upl_set_iostate(vector_upl, upl, vector_upl_size, upl_size);
4455 			     upl, (int)uio->uio_offset, io_req_size, retval, 0);
4514 	upl_t            upl[MAX_VECTS];
4584 			      &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
4605 	pl = ubc_upl_pageinfo(upl[cur_upl]);
4670 	        error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset, xsize,
4717 	        ubc_upl_abort(upl[n], 0);
4728 	upl_t            upl;
4758 				    &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
4802 	upl_t            upl;
4837 		 * compute the size of the upl needed to encompass
4887 			     upl, (int)upl_f_offset, upl_size, start_offset, 0);
4892 				      &upl,
4902 		 * upl
4912 			     upl, (int)upl_f_offset, upl_size, start_offset, 0);
4917 			 * scan from the beginning of the upl looking for the first
4929 			 * page before the end of the upl is reached, if we
4954 				retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
4961 		        ubc_upl_abort(upl, 0);
5310 	upl_t            upl;
5383 			      	&upl,
5389 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END, upl, upl_f_offset, 0, 0, 0);
5396 	 * end up working with a freed upl due to the FREE_ON_EMPTY semantics
5406 	        ubc_upl_abort(upl, 0);
5427 			        ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
5462 		retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
5574         upl_t            upl;
5611                               &upl,
5622                 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
5625                           ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
5650 		error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
5661 	ubc_upl_abort_range(upl, 0, PAGE_SIZE, abort_flags);
5669 cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int *io_resid)
5708 	pl = ubc_upl_pageinfo(upl);