bsd/vfs/vfs_journal.c

158 static void finish_end_thread(transaction *tr);
160 static int finish_end_transaction(transaction *tr, errno_t (*callback)(void*), void *callback_arg);
161 static int end_transaction(transaction *tr, int force_it, errno_t (*callback)(void*), void *callback_arg, boolean_t drop_lock, boolean_t must_wait);
162 static void abort_transaction(journal *jnl, transaction *tr);
219 #define CHECK_TRANSACTION(tr) \
221 	if (tr == NULL) {					\
224 	if (tr->jnl == NULL) {						\
225 		panic("%s:%d: null tr->jnl ptr?\n", __FILE__, __LINE__); \
227 	if (tr->blhdr != (block_list_header *)tr->tbuffer) {		\
228 		panic("%s:%d: blhdr (%p) != tbuffer (%p)\n", __FILE__, __LINE__, tr->blhdr, tr->tbuffer); \
230 	if (tr->total_bytes < 0) {					\
231 		panic("%s:%d: tr total_bytes looks bad: %d\n", __FILE__, __LINE__, tr->total_bytes); \
233 	if (tr->journal_start < 0) {					\
234 		panic("%s:%d: tr journal start looks bad: 0x%llx\n", __FILE__, __LINE__, tr->journal_start); \
236 	if (tr->journal_end < 0) {					\
237 		panic("%s:%d: tr journal end looks bad: 0x%llx\n", __FILE__, __LINE__, tr->journal_end); \
239 	if (tr->blhdr && (tr->blhdr->max_blocks <= 0 || tr->blhdr->max_blocks > (tr->jnl->jhdr->size/tr->jnl->jhdr->jhdr_size))) { \
240 		panic("%s:%d: tr blhdr max_blocks looks bad: %d\n", __FILE__, __LINE__, tr->blhdr->max_blocks);	\
517 	transaction *tr, *next;
524 	tr = jnl->tr_freeme;
528 	for(; tr; tr=next) {
529 		for (blhdr = tr->blhdr; blhdr; blhdr = next_blhdr) {
533 			kmem_free(kernel_map, (vm_offset_t)blhdr, tr->tbuffer_size);
535 			KERNEL_DEBUG(0xbbbbc01c, jnl, tr, tr->tbuffer_size, 0, 0);
537 		next = tr->next;
538 		FREE_ZONE(tr, sizeof(transaction), M_JNL_TR);
553 	transaction  *tr;
560 	//printf("jnl: buf flush: bp @ 0x%x l/blkno %qd/%qd vp 0x%x tr @ 0x%x\n",
565 	tr      = (transaction *)arg;
568 	if (tr == NULL) {
572 	CHECK_TRANSACTION(tr);
574 	jnl = tr->jnl;
581 	amt_flushed = tr->num_killed;
582 	total_bytes = tr->total_bytes;
588 	// OSAddAtomic() returns the value of tr->num_flushed before the add
590 	amt_flushed += OSAddAtomic(bufsize, &tr->num_flushed);
597 	//       the tr pointer after doing the OSAddAtomic().  if
599 	//       and then it's ok to dereference "tr".
608 	if (tr->total_bytes == (int)0xfbadc0de) {
616 	tr->total_bytes = 0xfbadc0de;
618 	//printf("jnl: tr 0x%x (0x%llx 0x%llx) in jnl 0x%x completed.\n",
619 	//   tr, tr->journal_start, tr->journal_end, jnl);
624 		if ((off_t)(jnl->old_start[i] & ~(0x8000000000000000ULL)) == tr->journal_start) {
631 		panic("jnl: buffer_flushed: did not find tr w/start @ %lld (tr %p, jnl %p)\n",
632 		      tr->journal_start, tr, jnl);
638 	if (tr->journal_start == jnl->active_start) {
639 		jnl->active_start = tr->journal_end;
640 		tr->journal_start = tr->journal_end = (off_t)0;
659 		} else if (tr->journal_end == ctr->journal_start) {
660 			ctr->journal_start = tr->journal_start;
663 			tr->journal_start  = tr->journal_end = (off_t)0;
664 		} else if (tr->journal_start == ctr->journal_end) {
665 			ctr->journal_end  = tr->journal_end;
667 			tr->journal_start = tr->journal_end = (off_t)0;
687 	if (tr->journal_start != 0) {
693 		for (ctr = jnl->completed_trs; ctr && tr->journal_start > ctr->journal_start; prev=ctr, ctr=ctr->next) {
698 			jnl->completed_trs = tr;
699 			tr->next = NULL;
701 			tr->next = jnl->completed_trs;
702 			jnl->completed_trs = tr;
704 			tr->next = prev->next;
705 			prev->next = tr;
708 		// if we're here this tr got merged with someone else so
710 		tr->next       = jnl->tr_freeme;
711 		jnl->tr_freeme = tr;
2218 			transaction *tr = jnl->cur_tr;
2225 			end_transaction(tr, 1, NULL, NULL, FALSE, TRUE);
2261 			transaction *tr;
2264 				tr = jnl->active_tr;
2267 				tr = jnl->cur_tr;
2270 			abort_transaction(jnl, tr);
2273 				panic("jnl: %s: close: jnl @ %p had both an active and cur tr\n", jnl->jdev_name, jnl);
2384 					panic("jnl: check_free_space: tr starting @ 0x%llx not flushing (jnl %p).\n",
2462 	transaction *tr;
2464 	MALLOC_ZONE(tr, transaction *, sizeof(transaction), M_JNL_TR, M_WAITOK);
2465 	memset(tr, 0, sizeof(transaction));
2467 	tr->tbuffer_size = jnl->tbuffer_size;
2469 	if (kmem_alloc_kobject(kernel_map, (vm_offset_t *)&tr->tbuffer, tr->tbuffer_size)) {
2470 		FREE_ZONE(tr, sizeof(transaction), M_JNL_TR);
2476 	memset(tr->tbuffer, 0, BLHDR_CHECKSUM_SIZE);
2478 	memset(tr->tbuffer + BLHDR_CHECKSUM_SIZE, 0x5a, jnl->jhdr->blhdr_size - BLHDR_CHECKSUM_SIZE);
2480 	tr->blhdr = (block_list_header *)tr->tbuffer;
2481 	tr->blhdr->max_blocks = (jnl->jhdr->blhdr_size / sizeof(block_info)) - 1;
2482 	tr->blhdr->num_blocks = 1;      // accounts for this header block
2483 	tr->blhdr->bytes_used = jnl->jhdr->blhdr_size;
2484 	tr->blhdr->flags = BLHDR_CHECK_CHECKSUMS | BLHDR_FIRST_HEADER;
2486 	tr->sequence_num = ++jnl->sequence_num;
2487 	tr->num_blhdrs  = 1;
2488 	tr->total_bytes = jnl->jhdr->blhdr_size;
2489 	tr->jnl         = jnl;
2491 	jnl->active_tr  = tr;
2556 	// printf("jnl: start_tr: owner 0x%x new tr @ 0x%x\n", jnl->owner, jnl->active_tr);
2572 	transaction *tr;
2589 	tr = jnl->active_tr;
2590 	CHECK_TRANSACTION(tr);
2598 	//   bp, buf_vnode(bp), buf_lblkno(bp), buf_blkno(bp), buf_size(bp), tr->total_bytes);
2650 	if (tr->total_bytes+buf_size(bp) >= (jnl->jhdr->size - jnl->jhdr->jhdr_size)) {
2651 		panic("jnl: transaction too big (%d >= %lld bytes, bufsize %d, tr %p bp %p)\n",
2652 		      tr->total_bytes, (tr->jnl->jhdr->size - jnl->jhdr->jhdr_size), buf_size(bp), tr, bp);
2680 	transaction	*tr;
2688 	tr = jnl->active_tr;
2693 	// that happened to be modified as part of another tr.
2695 	if (tr == NULL) {
2706 	CHECK_TRANSACTION(tr);
2713 	// printf("jnl: modify_block_abort: tr 0x%x bp 0x%x\n", jnl->active_tr, bp);
2716 	for (blhdr = tr->blhdr; blhdr; blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum)) {
2750 	transaction	*tr;
2762 	tr = jnl->active_tr;
2763 	CHECK_TRANSACTION(tr);
2771 	//   bp, buf_vnode(bp), buf_lblkno(bp), buf_blkno(bp), buf_size(bp), tr->total_bytes);
2778 	for (blhdr = tr->blhdr; blhdr; prev = blhdr, blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum)) {
2800 	    && (prev->bytes_used+buf_size(bp)) <= (uint32_t)tr->tbuffer_size) {
2816 		if (kmem_alloc_kobject(kernel_map, (vm_offset_t *)&nblhdr, tr->tbuffer_size)) {
2817 			panic("jnl: end_tr: no space for new block tr @ %p (total bytes: %d)!\n",
2818 			      tr, tr->total_bytes);
2832 		tr->num_blhdrs++;
2833 		tr->total_bytes += jnl->jhdr->blhdr_size;
2873 		tr->total_bytes   += bsize;
2888 	transaction	*tr;
2898 	tr = jnl->active_tr;
2899 	CHECK_TRANSACTION(tr);
2915 	for (blhdr = tr->blhdr; blhdr; blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum)) {
2935 					tr->num_killed += buf_size(bp);
3004 ;	 The allocated_count and extents fields of tr->trim are updated
3130 	transaction *tr;
3142 	tr = jnl->active_tr;
3143 	CHECK_TRANSACTION(tr);
3146 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_START, jnl, offset, length, tr->trim.extent_count, 0);
3163 	extent = tr->trim.extents;
3165 	while (insert_index < tr->trim.extent_count && extent->offset + extent->length < offset) {
3170 	while (insert_index + replace_count < tr->trim.extent_count && extent->offset <= end) {
3181 		if (tr->trim.extent_count == tr->trim.allocated_count) {
3182 			if (trim_realloc(&tr->trim) != 0) {
3185 					KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, ENOMEM, 0, 0, tr->trim.extent_count, 0);
3191 		if (insert_index < tr->trim.extent_count) {
3192 			memmove(&tr->trim.extents[insert_index+1],
3193 					&tr->trim.extents[insert_index],
3194 					(tr->trim.extent_count - insert_index) * sizeof(dk_extent_t));
3196 		tr->trim.extent_count++;
3199 		tr->trim.extents[insert_index].offset = offset;
3200 		tr->trim.extents[insert_index].length = length;
3204 			KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, 0, 0, 0, tr->trim.extent_count, 0);
3212 	if (tr->trim.extents[insert_index].offset < offset)
3213 		offset = tr->trim.extents[insert_index].offset;
3214 	extent = &tr->trim.extents[insert_index + replace_count - 1];
3217 	tr->trim.extents[insert_index].offset = offset;
3218 	tr->trim.extents[insert_index].length = end - offset;
3228 	if (replace_count > 1 && (insert_index + replace_count) < tr->trim.extent_count) {
3229 		memmove(&tr->trim.extents[insert_index + 1],
3230 				&tr->trim.extents[insert_index + replace_count],
3231 				(tr->trim.extent_count - insert_index - replace_count) * sizeof(dk_extent_t));
3233 	tr->trim.extent_count -= replace_count - 1;
3236 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, 0, 0, 0, tr->trim.extent_count, 0);
3248 	transaction *tr = NULL;
3253 	tr = jnl->active_tr;
3254 	CHECK_TRANSACTION(tr);
3264 	overlap = trim_search_extent (&tr->trim, offset, length, &overlap_start, &overlap_len);
3309 	transaction *tr = NULL;
3314 	tr = jnl->active_tr;
3315 	if (tr != NULL) {
3316 		CHECK_TRANSACTION(tr);
3317 		tr->flush_on_completion = TRUE;
3480 	transaction *tr;
3489 	tr = jnl->active_tr;
3490 	CHECK_TRANSACTION(tr);
3493 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE | DBG_FUNC_START, jnl, offset, length, tr->trim.extent_count, 0);
3502 	error = trim_remove_extent(&tr->trim, offset, length);
3538 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE | DBG_FUNC_END, error, 0, 0, tr->trim.extent_count, 0);
3544 journal_trim_flush(journal *jnl, transaction *tr)
3549 		KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_FLUSH | DBG_FUNC_START, jnl, tr, 0, tr->trim.extent_count, 0);
3552 	if (tr->trim.extent_count > 0) {
3557 			unmap.extents = tr->trim.extents;
3558 			unmap.extentsCount = tr->trim.extent_count;
3560 				KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_UNMAP | DBG_FUNC_START, jnl, tr, 0, tr->trim.extent_count, 0);
3576 			jnl->trim_callback(jnl->trim_callback_arg, tr->trim.extent_count, tr->trim.extents);
3591 	if (jnl->async_trim == &tr->trim)
3597 	 * of "tr", so it is safe for us to manipulate tr->trim without
3600 	if (tr->trim.extents) {
3601 		kfree(tr->trim.extents, tr->trim.allocated_count * sizeof(dk_extent_t));
3602 		tr->trim.allocated_count = 0;
3603 		tr->trim.extent_count = 0;
3604 		tr->trim.extents = NULL;
3657  *	tr			 Transaction to add to the journal
3667 end_transaction(transaction *tr, int force_it, errno_t (*callback)(void*), void *callback_arg, boolean_t drop_lock, boolean_t must_wait)
3672 	journal		*jnl = tr->jnl;
3678 		panic("jnl: jnl @ %p already has cur_tr %p, new tr: %p\n",
3679 			  jnl, jnl->cur_tr, tr);
3684 	if (tr->total_bytes == jnl->jhdr->blhdr_size) {
3685 		jnl->cur_tr = tr;
3704 		   && tr->num_blhdrs < 3
3705 		   && (tr->total_bytes <= ((tr->tbuffer_size*tr->num_blhdrs) - tr->tbuffer_size/8))
3706 		   && (!(jnl->flags & JOURNAL_USE_UNMAP) || (tr->trim.extent_count < jnl_trim_flush_limit))) {
3708 		jnl->cur_tr = tr;
3712 	KERNEL_DEBUG(0xbbbbc018|DBG_FUNC_START, jnl, tr, drop_lock, must_wait, 0);
3726 		abort_transaction(jnl, tr);
3728 		KERNEL_DEBUG(0xbbbbc018|DBG_FUNC_END, jnl, tr, ret_val, 0, 0);
3744 	if (tr->trim.extent_count > 0)
3745 		jnl->async_trim = &tr->trim;
3767 	check_free_space(jnl, tr->total_bytes, &tr->delayed_header_write, jnl->saved_sequence_num);
3769 	KERNEL_DEBUG(0xbbbbc030 | DBG_FUNC_END, jnl, tr->delayed_header_write, 0, 0, 0);
3776 	if (tr->delayed_header_write == TRUE) {
3792 	tr->journal_start = jnl->jhdr->end;
3801 	jnl->old_start[sizeof(jnl->old_start)/sizeof(jnl->old_start[0]) - 1] = tr->journal_start | 0x8000000000000000LL;
3806 	for (blhdr = tr->blhdr; blhdr; blhdr = next) {
3822 					panic("jnl: inconsistent binfo (NULL bp w/bnum %lld; jnl @ %p, tr %p)\n",
3823 						blhdr->binfo[i].bnum, jnl, tr);
3922 		ret_val = finish_end_transaction(tr, callback, callback_arg);
3931 		kernel_thread_start((thread_continue_t)finish_end_thread, tr, &thread);
3933 	KERNEL_DEBUG(0xbbbbc018|DBG_FUNC_END, jnl, tr, ret_val, 0, 0);
3944 finish_end_thread(transaction *tr)
3949 	finish_end_transaction(tr, NULL, NULL);
3972 finish_end_transaction(transaction *tr, errno_t (*callback)(void*), void *callback_arg)
3977 	journal		*jnl = tr->jnl;
3985 	KERNEL_DEBUG(0xbbbbc028|DBG_FUNC_START, jnl, tr, 0, 0, 0);
3989 	for (blhdr = tr->blhdr; blhdr; blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum)) {
3993 		blhdr->binfo[0].u.bi.b.sequence_num = tr->sequence_num;
4023 					printf("jnl: %s: end_tr: bad news! bp @ %p w/null vp and l/blkno = %qd/%qd.  aborting the transaction (tr %p jnl %p).\n",
4024 					       jnl->jdev_name, bp, lblkno, blkno, tr, jnl);
4061 				buf_setfilter(bp, buffer_flushed_callback, tr, &func, &arg);
4114 	tr->journal_end = end;    // the transaction ends here too
4116 	if (tr->journal_start == 0 || tr->journal_end == 0) {
4117 		panic("jnl: end_transaction: bad tr journal start/end: 0x%llx 0x%llx\n",
4118 		      tr->journal_start, tr->journal_end);
4139 	journal_trim_flush(jnl, tr);
4145 	tr->num_flushed = tr->num_blhdrs * jnl->jhdr->blhdr_size;
4152 	for (blhdr = tr->blhdr; blhdr; blhdr = next) {
4205 		tr->next       = jnl->tr_freeme;
4206 		jnl->tr_freeme = tr;
4213 	//printf("jnl: end_tr: tr @ 0x%x, jnl-blocks: 0x%llx - 0x%llx. exit!\n",
4214 	//   tr, tr->journal_start, tr->journal_end);
4235 		abort_transaction(jnl, tr);		// cleans up list of extents to be trimmed
4241 	KERNEL_DEBUG(0xbbbbc028|DBG_FUNC_END, jnl, tr, bufs_written, ret_val, 0);
4295 abort_transaction(journal *jnl, transaction *tr)
4309 	KERNEL_DEBUG(0xbbbbc034|DBG_FUNC_START, jnl, tr, 0, 0, 0);
4311 	for (blhdr = tr->blhdr; blhdr; blhdr = next) {
4354 					KERNEL_DEBUG(0xbbbbc034|DBG_FUNC_NONE, jnl, tr, bp, 0, 0);
4375 		kmem_free(kernel_map, (vm_offset_t)blhdr, tr->tbuffer_size);
4384 	if (jnl->async_trim == &tr->trim)
4389 	if (tr->trim.extents) {
4390 		kfree(tr->trim.extents, tr->trim.allocated_count * sizeof(dk_extent_t));
4392 	tr->trim.allocated_count = 0;
4393 	tr->trim.extent_count = 0;
4394 	tr->trim.extents = NULL;
4395 	tr->tbuffer     = NULL;
4396 	tr->blhdr       = NULL;
4397 	tr->total_bytes = 0xdbadc0de;
4398 	FREE_ZONE(tr, sizeof(transaction), M_JNL_TR);
4400 	KERNEL_DEBUG(0xbbbbc034|DBG_FUNC_END, jnl, tr, 0, 0, 0);
4408 	transaction *tr;
4433 				panic("jnl: journal @ %p has active tr (%p) and cur tr (%p)\n",
4436 			tr             = jnl->active_tr;
4439 			abort_transaction(jnl, tr);
4447 	tr = jnl->active_tr;
4448 	CHECK_TRANSACTION(tr);
4458 	if (tr->flush_on_completion == TRUE) {
4463 		ret = end_transaction(tr, 1, NULL, NULL, TRUE, TRUE);
4467 		ret = end_transaction(tr, 0, NULL, NULL, TRUE, FALSE);
4522 		transaction *tr = jnl->cur_tr;
4536 		end_transaction(tr, 1, NULL, NULL, drop_lock, TRUE);   // force it to get flushed
4644 	transaction	*tr;
4681 	tr = jnl->active_tr;
4682 	CHECK_TRANSACTION(tr);
4685 	jnl->active_tr = tr;
4720 	ret = end_transaction(tr, 1, callback, callback_arg, FALSE, TRUE);
4740 	abort_transaction(jnl, tr);