bsd/vfs/vfs_journal.c

65  * Set sysctl vfs.generic.jnl.kdebug.trim=1 to enable KERNEL_DEBUG_CONSTANT
66  * logging of trim-related calls within the journal.  (They're
75 SYSCTL_INT(_vfs_generic_jnl_kdebug, OID_AUTO, trim, CTLFLAG_RW|CTLFLAG_LOCKED, &jnl_kdebug, 0, "Enable kdebug logging for journal TRIM");
127 // By default, we grow the list of extents to trim by 4K at a time.
3029 ;	trim		- The trim list to be resized.
3035 ;	 The allocated_count and extents fields of tr->trim are updated
3040 trim_realloc(journal *jnl, struct jnl_trim_list *trim)
3047 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REALLOC | DBG_FUNC_START, VM_KERNEL_ADDRPERM(trim), 0, trim->allocated_count, trim->extent_count, 0);
3049 	new_allocated_count = trim->allocated_count + JOURNAL_DEFAULT_TRIM_EXTENTS;
3071 		trim->extent_count = 0;
3073 			KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REALLOC | DBG_FUNC_END, ENOMEM, 0, trim->allocated_count, 0, 0);
3078 	if (trim->extents != NULL) {
3080 				trim->extents,
3081 				trim->allocated_count * sizeof(dk_extent_t));
3082 		kfree(trim->extents,
3083 			  trim->allocated_count * sizeof(dk_extent_t));
3086 	trim->allocated_count = new_allocated_count;
3087 	trim->extents = new_extents;
3090 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REALLOC | DBG_FUNC_END, 0, 0, new_allocated_count, trim->extent_count, 0);
3104  ;	trim		- The trim list to be searched.
3115 trim_search_extent(struct jnl_trim_list *trim, uint64_t offset,
3120 	uint32_t upper = trim->extent_count;	/* Highest index to search + 1 */
3127 		if (trim->extents[middle].offset >= end)
3129 		else if (trim->extents[middle].offset + trim->extents[middle].length <= offset)
3133 				*overlap_start = trim->extents[middle].offset;
3136 				*overlap_len = trim->extents[middle].length;
3155 ;				SSDs can support trim/unmap and avoid having to recopy those
3183 	/* TODO: Is it OK to manipulate the trim list even if JOURNAL_INVALID is set?  I think so... */
3192 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_START, VM_KERNEL_ADDRPERM(jnl), offset, length, tr->trim.extent_count, 0);
3209 	extent = tr->trim.extents;
3211 	while (insert_index < tr->trim.extent_count && extent->offset + extent->length < offset) {
3216 	while (insert_index + replace_count < tr->trim.extent_count && extent->offset <= end) {
3227 		if (tr->trim.extent_count == tr->trim.allocated_count) {
3228 			if (trim_realloc(jnl, &tr->trim) != 0) {
3231 					KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, ENOMEM, 0, 0, tr->trim.extent_count, 0);
3237 		if (insert_index < tr->trim.extent_count) {
3238 			memmove(&tr->trim.extents[insert_index+1],
3239 					&tr->trim.extents[insert_index],
3240 					(tr->trim.extent_count - insert_index) * sizeof(dk_extent_t));
3242 		tr->trim.extent_count++;
3245 		tr->trim.extents[insert_index].offset = offset;
3246 		tr->trim.extents[insert_index].length = length;
3250 			KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, 0, 0, 0, tr->trim.extent_count, 0);
3258 	if (tr->trim.extents[insert_index].offset < offset)
3259 		offset = tr->trim.extents[insert_index].offset;
3260 	extent = &tr->trim.extents[insert_index + replace_count - 1];
3263 	tr->trim.extents[insert_index].offset = offset;
3264 	tr->trim.extents[insert_index].length = end - offset;
3274 	if (replace_count > 1 && (insert_index + replace_count) < tr->trim.extent_count) {
3275 		memmove(&tr->trim.extents[insert_index + 1],
3276 				&tr->trim.extents[insert_index + replace_count],
3277 				(tr->trim.extent_count - insert_index - replace_count) * sizeof(dk_extent_t));
3279 	tr->trim.extent_count -= replace_count - 1;
3282 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, 0, 0, 0, tr->trim.extent_count, 0);
3310 	overlap = trim_search_extent (&tr->trim, offset, length, &overlap_start, &overlap_len);
3313 		 * The second is the async trim list, which is only done if the current
3315 		 * extent. This async trim list is the set of all previously
3317 		 * trim lock in order to search this list.  If we grab the list before the
3380 ;				Any overlapping ranges currently in the journal's trim list will
3390 ;	trim		- The trim list to update.
3396 trim_remove_extent(journal *jnl, struct jnl_trim_list *trim, uint64_t offset, uint64_t length)
3410 	extent = trim->extents;
3412 	while (keep_before < trim->extent_count && extent->offset < offset) {
3422 	while (keep_after < trim->extent_count && (extent->offset + extent->length) <= end) {
3442 		if (trim->extent_count == trim->allocated_count) {
3443 			if (trim_realloc(jnl, trim) != 0) {
3455 		memmove(&trim->extents[keep_before],
3456 				&trim->extents[keep_after],
3457 				(trim->extent_count - keep_after) * sizeof(dk_extent_t));
3458 		++trim->extent_count;
3473 		extent = &trim->extents[keep_before - 1];
3483 	if (keep_after < trim->extent_count) {
3484 		extent = &trim->extents[keep_after];
3495 	if (keep_after > keep_before && keep_after < trim->extent_count) {
3496 		memmove(&trim->extents[keep_before],
3497 				&trim->extents[keep_after],
3498 				(trim->extent_count - keep_after) * sizeof(dk_extent_t));
3500 	trim->extent_count -= keep_after - keep_before;
3513  ;				this transaction, or a pending trim of a transaction being
3530 	/* TODO: Is it OK to manipulate the trim list even if JOURNAL_INVALID is set?  I think so... */
3539 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE | DBG_FUNC_START, VM_KERNEL_ADDRPERM(jnl), offset, length, tr->trim.extent_count, 0);
3548 	error = trim_remove_extent(jnl, &tr->trim, offset, length);
3553 		 * See if a pending trim has any extents that overlap with the
3584 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE | DBG_FUNC_END, error, 0, 0, tr->trim.extent_count, 0);
3596 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_FLUSH | DBG_FUNC_START, VM_KERNEL_ADDRPERM(jnl), tr, 0, tr->trim.extent_count, 0);
3610 	if (tr->trim.extent_count > 0) {
3615 			unmap.extents = tr->trim.extents;
3616 			unmap.extentsCount = tr->trim.extent_count;
3618 				KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_UNMAP | DBG_FUNC_START, VM_KERNEL_ADDRPERM(jnl), tr, 0, tr->trim.extent_count, 0);
3634 			jnl->trim_callback(jnl->trim_callback_arg, tr->trim.extent_count, tr->trim.extents);
3642 	 * tell the current transaction that there is no pending trim
3651 	if (jnl->async_trim == &tr->trim)
3657 	 * of "tr", so it is safe for us to manipulate tr->trim without
3660 	if (tr->trim.extents) {
3661 		kfree(tr->trim.extents, tr->trim.allocated_count * sizeof(dk_extent_t));
3662 		tr->trim.allocated_count = 0;
3663 		tr->trim.extent_count = 0;
3664 		tr->trim.extents = NULL;
3759     // trim error, then we stop issuing trims for this
3766 		   && (!(jnl->flags & JOURNAL_USE_UNMAP) || (tr->trim.extent_count < jnl_trim_flush_limit))) {
3793 	 * Store a pointer to this transaction's trim list so that
3796 	 * Note: if there are no extents in the trim list, then don't
3799 	 * there is a trim pending).
3804 	if (tr->trim.extent_count > 0)
3805 		jnl->async_trim = &tr->trim;
4452 	 * tell the current transaction that there is no pending trim
4456 	if (jnl->async_trim == &tr->trim)
4461 	if (tr->trim.extents) {
4462 		kfree(tr->trim.extents, tr->trim.allocated_count * sizeof(dk_extent_t));
4464 	tr->trim.allocated_count = 0;
4465 	tr->trim.extent_count = 0;
4466 	tr->trim.extents = NULL;