1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
23 * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
24 * Copyright (c) 2023, 2024, Klara Inc.
25 */
26
27#ifndef _ABD_IMPL_H
28#define	_ABD_IMPL_H
29
30#include <sys/abd.h>
31#include <sys/wmsum.h>
32
33#ifdef __cplusplus
34extern "C" {
35#endif
36
37typedef enum abd_stats_op {
38	ABDSTAT_INCR, /* Increase abdstat values */
39	ABDSTAT_DECR  /* Decrease abdstat values */
40} abd_stats_op_t;
41
42/* forward declarations */
43struct scatterlist;
44struct page;
45
46struct abd_iter {
47	/* public interface */
48	union {
49		/* for abd_iter_map()/abd_iter_unmap() */
50		struct {
51			/* addr corresponding to iter_pos */
52			void		*iter_mapaddr;
53			/* length of data valid at mapaddr */
54			size_t		iter_mapsize;
55		};
56		/* for abd_iter_page() */
57		struct {
58			/* current page */
59			struct page	*iter_page;
60			/* offset of data in page */
61			size_t		iter_page_doff;
62			/* size of data in page */
63			size_t		iter_page_dsize;
64		};
65	};
66
67	/* private */
68	abd_t		*iter_abd;	/* ABD being iterated through */
69	size_t		iter_pos;
70	size_t		iter_offset;	/* offset in current sg/abd_buf, */
71					/* abd_offset included */
72	struct scatterlist *iter_sg;	/* current sg */
73};
74
75extern abd_t *abd_zero_scatter;
76
77abd_t *abd_gang_get_offset(abd_t *, size_t *);
78abd_t *abd_alloc_struct(size_t);
79void abd_free_struct(abd_t *);
80
81/*
82 * OS specific functions
83 */
84
85abd_t *abd_alloc_struct_impl(size_t);
86abd_t *abd_get_offset_scatter(abd_t *, abd_t *, size_t, size_t);
87void abd_free_struct_impl(abd_t *);
88void abd_alloc_chunks(abd_t *, size_t);
89void abd_free_chunks(abd_t *);
90void abd_update_scatter_stats(abd_t *, abd_stats_op_t);
91void abd_update_linear_stats(abd_t *, abd_stats_op_t);
92void abd_verify_scatter(abd_t *);
93void abd_free_linear_page(abd_t *);
94/* OS specific abd_iter functions */
95void abd_iter_init(struct abd_iter  *, abd_t *);
96boolean_t abd_iter_at_end(struct abd_iter *);
97void abd_iter_advance(struct abd_iter *, size_t);
98void abd_iter_map(struct abd_iter *);
99void abd_iter_unmap(struct abd_iter *);
100void abd_iter_page(struct abd_iter *);
101
102/*
103 * Helper macros
104 */
105#define	ABDSTAT_INCR(stat, val) \
106	wmsum_add(&abd_sums.stat, (val))
107#define	ABDSTAT_BUMP(stat)	ABDSTAT_INCR(stat, 1)
108#define	ABDSTAT_BUMPDOWN(stat)	ABDSTAT_INCR(stat, -1)
109
110#define	ABD_SCATTER(abd)	(abd->abd_u.abd_scatter)
111#define	ABD_LINEAR_BUF(abd)	(abd->abd_u.abd_linear.abd_buf)
112#define	ABD_GANG(abd)		(abd->abd_u.abd_gang)
113
114#if defined(_KERNEL)
115#if defined(__FreeBSD__)
116#define	abd_enter_critical(flags)	critical_enter()
117#define	abd_exit_critical(flags)	critical_exit()
118#else
119#define	abd_enter_critical(flags)	local_irq_save(flags)
120#define	abd_exit_critical(flags)	local_irq_restore(flags)
121#endif
122#else /* !_KERNEL */
123#define	abd_enter_critical(flags)	((void)0)
124#define	abd_exit_critical(flags)	((void)0)
125#endif
126
127#ifdef __cplusplus
128}
129#endif
130
131#endif	/* _ABD_IMPL_H */
132