Lines Matching refs:abd
109 abd_verify(abd_t *abd)
111 ASSERT3U(abd->abd_size, >, 0);
112 ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE);
113 ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR |
118 IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER));
120 IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER);
121 if (abd_is_linear(abd)) {
122 ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL);
123 } else if (abd_is_gang(abd)) {
125 for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain);
127 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
132 ASSERT3U(abd->abd_size, ==, child_sizes);
134 abd_verify_scatter(abd);
139 abd_init_struct(abd_t *abd)
141 list_link_init(&abd->abd_gang_link);
142 mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
143 abd->abd_flags = 0;
145 zfs_refcount_create(&abd->abd_children);
146 abd->abd_parent = NULL;
148 abd->abd_size = 0;
152 abd_fini_struct(abd_t *abd)
154 mutex_destroy(&abd->abd_mtx);
155 ASSERT(!list_link_active(&abd->abd_gang_link));
157 zfs_refcount_destroy(&abd->abd_children);
164 abd_t *abd = abd_alloc_struct_impl(size);
165 abd_init_struct(abd);
166 abd->abd_flags |= ABD_FLAG_ALLOCD;
167 return (abd);
171 abd_free_struct(abd_t *abd)
173 abd_fini_struct(abd);
174 abd_free_struct_impl(abd);
189 abd_t *abd = abd_alloc_struct(size);
190 abd->abd_flags |= ABD_FLAG_OWNER;
191 abd->abd_u.abd_scatter.abd_offset = 0;
192 abd_alloc_chunks(abd, size);
195 abd->abd_flags |= ABD_FLAG_META;
197 abd->abd_size = size;
199 abd_update_scatter_stats(abd, ABDSTAT_INCR);
201 return (abd);
212 abd_t *abd = abd_alloc_struct(0);
216 abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_OWNER;
218 abd->abd_flags |= ABD_FLAG_META;
220 abd->abd_size = size;
223 ABD_LINEAR_BUF(abd) = zio_buf_alloc(size);
225 ABD_LINEAR_BUF(abd) = zio_data_buf_alloc(size);
228 abd_update_linear_stats(abd, ABDSTAT_INCR);
230 return (abd);
234 abd_free_linear(abd_t *abd)
236 if (abd_is_linear_page(abd)) {
237 abd_free_linear_page(abd);
240 if (abd->abd_flags & ABD_FLAG_META) {
241 zio_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
243 zio_data_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
246 abd_update_linear_stats(abd, ABDSTAT_DECR);
250 abd_free_gang(abd_t *abd)
252 ASSERT(abd_is_gang(abd));
255 while ((cabd = list_head(&ABD_GANG(abd).abd_gang_chain)) != NULL) {
264 list_remove(&ABD_GANG(abd).abd_gang_chain, cabd);
269 list_destroy(&ABD_GANG(abd).abd_gang_chain);
273 abd_free_scatter(abd_t *abd)
275 abd_free_chunks(abd);
276 abd_update_scatter_stats(abd, ABDSTAT_DECR);
280 * Free an ABD. Use with any kind of abd: those created with abd_alloc_*()
291 abd_free(abd_t *abd)
293 if (abd == NULL)
296 abd_verify(abd);
298 IMPLY(abd->abd_flags & ABD_FLAG_OWNER, abd->abd_parent == NULL);
301 if (abd_is_gang(abd)) {
302 abd_free_gang(abd);
303 } else if (abd_is_linear(abd)) {
304 if (abd->abd_flags & ABD_FLAG_OWNER)
305 abd_free_linear(abd);
307 if (abd->abd_flags & ABD_FLAG_OWNER)
308 abd_free_scatter(abd);
312 if (abd->abd_parent != NULL) {
313 (void) zfs_refcount_remove_many(&abd->abd_parent->abd_children,
314 abd->abd_size, abd);
318 abd_fini_struct(abd);
319 if (abd->abd_flags & ABD_FLAG_ALLOCD)
320 abd_free_struct_impl(abd);
342 * IO's. To free this abd, abd_free() must be called.
347 abd_t *abd = abd_alloc_struct(0);
348 abd->abd_flags |= ABD_FLAG_GANG | ABD_FLAG_OWNER;
349 list_create(&ABD_GANG(abd).abd_gang_chain,
351 return (abd);
471 abd_gang_get_offset(abd_t *abd, size_t *off)
475 ASSERT(abd_is_gang(abd));
476 ASSERT3U(*off, <, abd->abd_size);
477 for (cabd = list_head(&ABD_GANG(abd).abd_gang_chain); cabd != NULL;
478 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
495 abd_get_offset_impl(abd_t *abd, abd_t *sabd, size_t off, size_t size)
501 if (abd == NULL)
502 abd = abd_alloc_struct(0);
508 abd->abd_flags |= ABD_FLAG_LINEAR;
510 ABD_LINEAR_BUF(abd) = (char *)ABD_LINEAR_BUF(sabd) + off;
513 if (abd == NULL) {
514 abd = abd_alloc_gang();
516 abd->abd_flags |= ABD_FLAG_GANG;
517 list_create(&ABD_GANG(abd).abd_gang_chain,
521 abd->abd_flags &= ~ABD_FLAG_OWNER;
528 abd_gang_add(abd, nabd, B_TRUE);
534 abd = abd_get_offset_scatter(abd, sabd, off);
537 ASSERT3P(abd, !=, NULL);
538 abd->abd_size = size;
540 abd->abd_parent = sabd;
541 (void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd);
543 return (abd);
549 * of allocating memory for the abd_t struct, and updating the abd stats.
550 * Usually, the provided abd is returned, but in some circumstances (FreeBSD,
556 abd_get_offset_struct(abd_t *abd, abd_t *sabd, size_t off, size_t size)
559 abd_init_struct(abd);
560 result = abd_get_offset_impl(abd, sabd, off, size);
561 if (result != abd)
562 abd_fini_struct(abd);
598 abd_t *abd = abd_alloc_struct(0);
607 abd->abd_flags |= ABD_FLAG_LINEAR;
608 abd->abd_size = size;
610 ABD_LINEAR_BUF(abd) = buf;
612 return (abd);
619 abd_to_buf(abd_t *abd)
621 ASSERT(abd_is_linear(abd));
622 abd_verify(abd);
623 return (ABD_LINEAR_BUF(abd));
633 abd_borrow_buf(abd_t *abd, size_t n)
636 abd_verify(abd);
637 ASSERT3U(abd->abd_size, >=, n);
638 if (abd_is_linear(abd)) {
639 buf = abd_to_buf(abd);
644 (void) zfs_refcount_add_many(&abd->abd_children, n, buf);
650 abd_borrow_buf_copy(abd_t *abd, size_t n)
652 void *buf = abd_borrow_buf(abd, n);
653 if (!abd_is_linear(abd)) {
654 abd_copy_to_buf(buf, abd, n);
663 * be copied back to abd, use abd_return_buf_copy() instead.
666 abd_return_buf(abd_t *abd, void *buf, size_t n)
668 abd_verify(abd);
669 ASSERT3U(abd->abd_size, >=, n);
670 if (abd_is_linear(abd)) {
671 ASSERT3P(buf, ==, abd_to_buf(abd));
673 ASSERT0(abd_cmp_buf(abd, buf, n));
677 (void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
682 abd_return_buf_copy(abd_t *abd, void *buf, size_t n)
684 if (!abd_is_linear(abd)) {
685 abd_copy_from_buf(abd, buf, n);
687 abd_return_buf(abd, buf, n);
691 abd_release_ownership_of_buf(abd_t *abd)
693 ASSERT(abd_is_linear(abd));
694 ASSERT(abd->abd_flags & ABD_FLAG_OWNER);
703 ASSERT(!abd_is_linear_page(abd));
705 abd_verify(abd);
707 abd->abd_flags &= ~ABD_FLAG_OWNER;
709 abd->abd_flags &= ~ABD_FLAG_META;
711 abd_update_linear_stats(abd, ABDSTAT_DECR);
722 abd_take_ownership_of_buf(abd_t *abd, boolean_t is_metadata)
724 ASSERT(abd_is_linear(abd));
725 ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER));
726 abd_verify(abd);
728 abd->abd_flags |= ABD_FLAG_OWNER;
730 abd->abd_flags |= ABD_FLAG_META;
733 abd_update_linear_stats(abd, ABDSTAT_INCR);
737 * Initializes an abd_iter based on whether the abd is a gang ABD
741 abd_init_abd_iter(abd_t *abd, struct abd_iter *aiter, size_t off)
745 if (abd_is_gang(abd)) {
746 cabd = abd_gang_get_offset(abd, &off);
752 abd_iter_init(aiter, abd);
764 abd_advance_abd_iter(abd_t *abd, abd_t *cabd, struct abd_iter *aiter,
768 if (abd_is_gang(abd) && abd_iter_at_end(aiter)) {
770 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd);
780 abd_iterate_func(abd_t *abd, size_t off, size_t size,
789 abd_verify(abd);
790 ASSERT3U(off + size, <=, abd->abd_size);
792 boolean_t gang = abd_is_gang(abd);
793 abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);
813 c_abd = abd_advance_abd_iter(abd, c_abd, &aiter, len);
835 * Copy abd to buf. (off is the offset in abd.)
838 abd_copy_to_buf_off(void *buf, abd_t *abd, size_t off, size_t size)
842 (void) abd_iterate_func(abd, off, size, abd_copy_to_buf_off_cb,
859 * Compare the contents of abd to buf. (off is the offset in abd.)
862 abd_cmp_buf_off(abd_t *abd, const void *buf, size_t off, size_t size)
866 return (abd_iterate_func(abd, off, size, abd_cmp_buf_off_cb, &ba_ptr));
881 * Copy from buf to abd. (off is the offset in abd.)
884 abd_copy_from_buf_off(abd_t *abd, const void *buf, size_t off, size_t size)
888 (void) abd_iterate_func(abd, off, size, abd_copy_from_buf_off_cb,
901 * Zero out the abd from a particular offset to the end.
904 abd_zero_off(abd_t *abd, size_t off, size_t size)
906 (void) abd_iterate_func(abd, off, size, abd_zero_off_cb, NULL);