scatterlist.h revision 289682 
1/*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013-2015 Mellanox Technologies, Ltd.
6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com>
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice unmodified, this list of conditions, and the following
14 *    disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * $FreeBSD: head/sys/ofed/include/linux/scatterlist.h 289682 2015-10-21 08:51:49Z hselasky $
31 */
32#ifndef	_LINUX_SCATTERLIST_H_
33#define	_LINUX_SCATTERLIST_H_
34
35#include <linux/page.h>
36#include <linux/slab.h>
37
38struct scatterlist {
39	union {
40		struct page *page;
41		struct scatterlist *sg;
42	}	sl_un;
43	dma_addr_t address;
44	unsigned long offset;
45	uint32_t length;
46	uint32_t flags;
47};
48
49struct sg_table {
50	struct scatterlist *sgl;
51	unsigned int nents;
52	unsigned int orig_nents;
53};
54
55struct sg_page_iter {
56	struct scatterlist *sg;
57	unsigned int sg_pgoffset;
58	unsigned int maxents;
59};
60
61#define	SG_MAX_SINGLE_ALLOC	(PAGE_SIZE / sizeof(struct scatterlist))
62
63#define	sg_dma_address(sg)	(sg)->address
64#define	sg_dma_len(sg)		(sg)->length
65#define	sg_page(sg)		(sg)->sl_un.page
66#define	sg_scatternext(sg)	(sg)->sl_un.sg
67
68#define	SG_END		0x01
69#define	SG_CHAIN	0x02
70
71static inline void
72sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len,
73    unsigned int offset)
74{
75	sg_page(sg) = page;
76	sg_dma_len(sg) = len;
77	sg->offset = offset;
78	if (offset > PAGE_SIZE)
79		panic("sg_set_page: Invalid offset %d\n", offset);
80}
81
82static inline void
83sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen)
84{
85	sg_set_page(sg, virt_to_page(buf), buflen,
86	    ((uintptr_t)buf) & (PAGE_SIZE - 1));
87}
88
89static inline void
90sg_init_table(struct scatterlist *sg, unsigned int nents)
91{
92	bzero(sg, sizeof(*sg) * nents);
93	sg[nents - 1].flags = SG_END;
94}
95
96static inline struct scatterlist *
97sg_next(struct scatterlist *sg)
98{
99	if (sg->flags & SG_END)
100		return (NULL);
101	sg++;
102	if (sg->flags & SG_CHAIN)
103		sg = sg_scatternext(sg);
104	return (sg);
105}
106
107static inline vm_paddr_t
108sg_phys(struct scatterlist *sg)
109{
110	return sg_page(sg)->phys_addr + sg->offset;
111}
112
113static inline void
114sg_chain(struct scatterlist *prv, unsigned int prv_nents,
115    struct scatterlist *sgl)
116{
117	struct scatterlist *sg = &prv[prv_nents - 1];
118
119	sg->offset = 0;
120	sg->length = 0;
121	sg->flags = SG_CHAIN;
122	sg->sl_un.sg = sgl;
123}
124
125static inline void
126sg_mark_end(struct scatterlist *sg)
127{
128	sg->flags = SG_END;
129}
130
131static inline void
132__sg_free_table(struct sg_table *table, unsigned int max_ents)
133{
134	struct scatterlist *sgl, *next;
135
136	if (unlikely(!table->sgl))
137		return;
138
139	sgl = table->sgl;
140	while (table->orig_nents) {
141		unsigned int alloc_size = table->orig_nents;
142		unsigned int sg_size;
143
144		if (alloc_size > max_ents) {
145			next = sgl[max_ents - 1].sl_un.sg;
146			alloc_size = max_ents;
147			sg_size = alloc_size - 1;
148		} else {
149			sg_size = alloc_size;
150			next = NULL;
151		}
152
153		table->orig_nents -= sg_size;
154		kfree(sgl);
155		sgl = next;
156	}
157
158	table->sgl = NULL;
159}
160
161static inline void
162sg_free_table(struct sg_table *table)
163{
164	__sg_free_table(table, SG_MAX_SINGLE_ALLOC);
165}
166
167static inline int
168__sg_alloc_table(struct sg_table *table, unsigned int nents,
169    unsigned int max_ents, gfp_t gfp_mask)
170{
171	struct scatterlist *sg, *prv;
172	unsigned int left;
173
174	memset(table, 0, sizeof(*table));
175
176	if (nents == 0)
177		return -EINVAL;
178	left = nents;
179	prv = NULL;
180	do {
181		unsigned int sg_size;
182		unsigned int alloc_size = left;
183
184		if (alloc_size > max_ents) {
185			alloc_size = max_ents;
186			sg_size = alloc_size - 1;
187		} else
188			sg_size = alloc_size;
189
190		left -= sg_size;
191
192		sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask);
193		if (unlikely(!sg)) {
194			if (prv)
195				table->nents = ++table->orig_nents;
196
197			return -ENOMEM;
198		}
199		sg_init_table(sg, alloc_size);
200		table->nents = table->orig_nents += sg_size;
201
202		if (prv)
203			sg_chain(prv, max_ents, sg);
204		else
205			table->sgl = sg;
206
207		if (!left)
208			sg_mark_end(&sg[sg_size - 1]);
209
210		prv = sg;
211	} while (left);
212
213	return 0;
214}
215
216static inline int
217sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
218{
219	int ret;
220
221	ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
222	    gfp_mask);
223	if (unlikely(ret))
224		__sg_free_table(table, SG_MAX_SINGLE_ALLOC);
225
226	return ret;
227}
228
229static inline void
230_sg_iter_next(struct sg_page_iter *iter)
231{
232	struct scatterlist *sg;
233	unsigned int pgcount;
234
235	sg = iter->sg;
236	pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
237
238	++iter->sg_pgoffset;
239	while (iter->sg_pgoffset >= pgcount) {
240		iter->sg_pgoffset -= pgcount;
241		sg = sg_next(sg);
242		--iter->maxents;
243		if (sg == NULL || iter->maxents == 0)
244			break;
245		pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
246	}
247	iter->sg = sg;
248}
249
250static inline void
251_sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter,
252    unsigned int nents, unsigned long pgoffset)
253{
254	if (nents) {
255		iter->sg = sgl;
256		iter->sg_pgoffset = pgoffset - 1;
257		iter->maxents = nents;
258		_sg_iter_next(iter);
259	} else {
260		iter->sg = NULL;
261		iter->sg_pgoffset = 0;
262		iter->maxents = 0;
263	}
264}
265
266static inline dma_addr_t
267sg_page_iter_dma_address(struct sg_page_iter *spi)
268{
269	return spi->sg->address + (spi->sg_pgoffset << PAGE_SHIFT);
270}
271
272#define	for_each_sg_page(sgl, iter, nents, pgoffset)			\
273	for (_sg_iter_init(sgl, iter, nents, pgoffset);			\
274	     (iter)->sg; _sg_iter_next(iter))
275
276#define	for_each_sg(sglist, sg, sgmax, _itr)				\
277	for (_itr = 0, sg = (sglist); _itr < (sgmax); _itr++, sg = sg_next(sg))
278
279#endif					/* _LINUX_SCATTERLIST_H_ */
280