1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (c) 2014 by Chunwei Chen. All rights reserved. 23 * Copyright (c) 2016, 2019 by Delphix. All rights reserved. 24 * Copyright (c) 2023, 2024, Klara Inc. 25 */ 26 27#ifndef _ABD_IMPL_H 28#define _ABD_IMPL_H 29 30#include <sys/abd.h> 31#include <sys/wmsum.h> 32 33#ifdef __cplusplus 34extern "C" { 35#endif 36 37typedef enum abd_stats_op { 38 ABDSTAT_INCR, /* Increase abdstat values */ 39 ABDSTAT_DECR /* Decrease abdstat values */ 40} abd_stats_op_t; 41 42/* forward declarations */ 43struct scatterlist; 44struct page; 45 46struct abd_iter { 47 /* public interface */ 48 union { 49 /* for abd_iter_map()/abd_iter_unmap() */ 50 struct { 51 /* addr corresponding to iter_pos */ 52 void *iter_mapaddr; 53 /* length of data valid at mapaddr */ 54 size_t iter_mapsize; 55 }; 56 /* for abd_iter_page() */ 57 struct { 58 /* current page */ 59 struct page *iter_page; 60 /* offset of data in page */ 61 size_t iter_page_doff; 62 /* size of data in page */ 63 size_t iter_page_dsize; 64 }; 65 }; 66 67 /* private */ 68 abd_t *iter_abd; /* ABD being iterated through */ 69 size_t iter_pos; 70 size_t iter_offset; /* offset in current sg/abd_buf, */ 71 /* abd_offset included */ 72 struct scatterlist *iter_sg; /* current sg */ 73}; 74 75extern abd_t *abd_zero_scatter; 76 77abd_t *abd_gang_get_offset(abd_t *, size_t *); 78abd_t *abd_alloc_struct(size_t); 79void abd_free_struct(abd_t *); 80 81/* 82 * OS specific functions 83 */ 84 85abd_t *abd_alloc_struct_impl(size_t); 86abd_t *abd_get_offset_scatter(abd_t *, abd_t *, size_t, size_t); 87void abd_free_struct_impl(abd_t *); 88void abd_alloc_chunks(abd_t *, size_t); 89void abd_free_chunks(abd_t *); 90void abd_update_scatter_stats(abd_t *, abd_stats_op_t); 91void abd_update_linear_stats(abd_t *, abd_stats_op_t); 92void abd_verify_scatter(abd_t *); 93void abd_free_linear_page(abd_t *); 94/* OS specific abd_iter functions */ 95void abd_iter_init(struct abd_iter *, abd_t *); 96boolean_t abd_iter_at_end(struct abd_iter *); 97void abd_iter_advance(struct abd_iter *, size_t); 98void abd_iter_map(struct abd_iter *); 99void abd_iter_unmap(struct abd_iter *); 100void abd_iter_page(struct abd_iter *); 101 102/* 103 * Helper macros 104 */ 105#define ABDSTAT_INCR(stat, val) \ 106 wmsum_add(&abd_sums.stat, (val)) 107#define ABDSTAT_BUMP(stat) ABDSTAT_INCR(stat, 1) 108#define ABDSTAT_BUMPDOWN(stat) ABDSTAT_INCR(stat, -1) 109 110#define ABD_SCATTER(abd) (abd->abd_u.abd_scatter) 111#define ABD_LINEAR_BUF(abd) (abd->abd_u.abd_linear.abd_buf) 112#define ABD_GANG(abd) (abd->abd_u.abd_gang) 113 114#if defined(_KERNEL) 115#if defined(__FreeBSD__) 116#define abd_enter_critical(flags) critical_enter() 117#define abd_exit_critical(flags) critical_exit() 118#else 119#define abd_enter_critical(flags) local_irq_save(flags) 120#define abd_exit_critical(flags) local_irq_restore(flags) 121#endif 122#else /* !_KERNEL */ 123#define abd_enter_critical(flags) ((void)0) 124#define abd_exit_critical(flags) ((void)0) 125#endif 126 127#ifdef __cplusplus 128} 129#endif 130 131#endif /* _ABD_IMPL_H */ 132