1/*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com>
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice unmodified, this list of conditions, and the following
14 *    disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * $FreeBSD$
31 */
32#ifndef	_LINUX_SCATTERLIST_H_
33#define	_LINUX_SCATTERLIST_H_
34
35#include <sys/types.h>
36#include <sys/sf_buf.h>
37
38#include <linux/page.h>
39#include <linux/slab.h>
40#include <linux/mm.h>
41
42struct bus_dmamap;
43struct scatterlist {
44	unsigned long page_link;
45#define	SG_PAGE_LINK_CHAIN	0x1UL
46#define	SG_PAGE_LINK_LAST	0x2UL
47#define	SG_PAGE_LINK_MASK	0x3UL
48	unsigned int offset;
49	unsigned int length;
50	dma_addr_t dma_address;
51	struct bus_dmamap *dma_map;	/* FreeBSD specific */
52};
53
54CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0);
55
56struct sg_table {
57	struct scatterlist *sgl;
58	unsigned int nents;
59	unsigned int orig_nents;
60};
61
62struct sg_page_iter {
63	struct scatterlist *sg;
64	unsigned int sg_pgoffset;
65	unsigned int maxents;
66	struct {
67		unsigned int nents;
68		int	pg_advance;
69	} internal;
70};
71
72struct sg_dma_page_iter {
73	struct sg_page_iter base;
74};
75
76#define	SCATTERLIST_MAX_SEGMENT	(-1U & ~(PAGE_SIZE - 1))
77
78#define	SG_MAX_SINGLE_ALLOC	(PAGE_SIZE / sizeof(struct scatterlist))
79
80#define	SG_MAGIC		0x87654321UL
81#define	SG_CHAIN		SG_PAGE_LINK_CHAIN
82#define	SG_END			SG_PAGE_LINK_LAST
83
84#define	sg_is_chain(sg)		((sg)->page_link & SG_PAGE_LINK_CHAIN)
85#define	sg_is_last(sg)		((sg)->page_link & SG_PAGE_LINK_LAST)
86#define	sg_chain_ptr(sg)	\
87	((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK))
88
89#define	sg_dma_address(sg)	(sg)->dma_address
90#define	sg_dma_len(sg)		(sg)->length
91
92#define	for_each_sg_page(sgl, iter, nents, pgoffset)			\
93	for (_sg_iter_init(sgl, iter, nents, pgoffset);			\
94	     (iter)->sg; _sg_iter_next(iter))
95#define	for_each_sg_dma_page(sgl, iter, nents, pgoffset) 		\
96	for_each_sg_page(sgl, &(iter)->base, nents, pgoffset)
97
98#define	for_each_sg(sglist, sg, sgmax, iter)				\
99	for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg))
100
101typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t);
102typedef void (sg_free_fn) (struct scatterlist *, unsigned int);
103
104static inline void
105sg_assign_page(struct scatterlist *sg, struct page *page)
106{
107	unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK;
108
109	sg->page_link = page_link | (unsigned long)page;
110}
111
112static inline void
113sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len,
114    unsigned int offset)
115{
116	sg_assign_page(sg, page);
117	sg->offset = offset;
118	sg->length = len;
119}
120
121static inline struct page *
122sg_page(struct scatterlist *sg)
123{
124	return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK));
125}
126
127static inline void
128sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen)
129{
130	sg_set_page(sg, virt_to_page(buf), buflen,
131	    ((uintptr_t)buf) & (PAGE_SIZE - 1));
132}
133
134static inline struct scatterlist *
135sg_next(struct scatterlist *sg)
136{
137	if (sg_is_last(sg))
138		return (NULL);
139	sg++;
140	if (sg_is_chain(sg))
141		sg = sg_chain_ptr(sg);
142	return (sg);
143}
144
145static inline vm_paddr_t
146sg_phys(struct scatterlist *sg)
147{
148	return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset);
149}
150
151static inline void *
152sg_virt(struct scatterlist *sg)
153{
154
155	return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset));
156}
157
158static inline void
159sg_chain(struct scatterlist *prv, unsigned int prv_nents,
160    struct scatterlist *sgl)
161{
162	struct scatterlist *sg = &prv[prv_nents - 1];
163
164	sg->offset = 0;
165	sg->length = 0;
166	sg->page_link = ((unsigned long)sgl |
167	    SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST;
168}
169
170static inline void
171sg_mark_end(struct scatterlist *sg)
172{
173	sg->page_link |= SG_PAGE_LINK_LAST;
174	sg->page_link &= ~SG_PAGE_LINK_CHAIN;
175}
176
177static inline void
178sg_init_table(struct scatterlist *sg, unsigned int nents)
179{
180	bzero(sg, sizeof(*sg) * nents);
181	sg_mark_end(&sg[nents - 1]);
182}
183
184static struct scatterlist *
185sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
186{
187	if (nents == SG_MAX_SINGLE_ALLOC) {
188		return ((void *)__get_free_page(gfp_mask));
189	} else
190		return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask));
191}
192
193static inline void
194sg_kfree(struct scatterlist *sg, unsigned int nents)
195{
196	if (nents == SG_MAX_SINGLE_ALLOC) {
197		free_page((unsigned long)sg);
198	} else
199		kfree(sg);
200}
201
202static inline void
203__sg_free_table(struct sg_table *table, unsigned int max_ents,
204    bool skip_first_chunk, sg_free_fn * free_fn)
205{
206	struct scatterlist *sgl, *next;
207
208	if (unlikely(!table->sgl))
209		return;
210
211	sgl = table->sgl;
212	while (table->orig_nents) {
213		unsigned int alloc_size = table->orig_nents;
214		unsigned int sg_size;
215
216		if (alloc_size > max_ents) {
217			next = sg_chain_ptr(&sgl[max_ents - 1]);
218			alloc_size = max_ents;
219			sg_size = alloc_size - 1;
220		} else {
221			sg_size = alloc_size;
222			next = NULL;
223		}
224
225		table->orig_nents -= sg_size;
226		if (skip_first_chunk)
227			skip_first_chunk = 0;
228		else
229			free_fn(sgl, alloc_size);
230		sgl = next;
231	}
232
233	table->sgl = NULL;
234}
235
236static inline void
237sg_free_table(struct sg_table *table)
238{
239	__sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree);
240}
241
242static inline int
243__sg_alloc_table(struct sg_table *table, unsigned int nents,
244    unsigned int max_ents, struct scatterlist *first_chunk,
245    gfp_t gfp_mask, sg_alloc_fn *alloc_fn)
246{
247	struct scatterlist *sg, *prv;
248	unsigned int left;
249
250	memset(table, 0, sizeof(*table));
251
252	if (nents == 0)
253		return (-EINVAL);
254	left = nents;
255	prv = NULL;
256	do {
257		unsigned int sg_size;
258		unsigned int alloc_size = left;
259
260		if (alloc_size > max_ents) {
261			alloc_size = max_ents;
262			sg_size = alloc_size - 1;
263		} else
264			sg_size = alloc_size;
265
266		left -= sg_size;
267
268		if (first_chunk) {
269			sg = first_chunk;
270			first_chunk = NULL;
271		} else {
272			sg = alloc_fn(alloc_size, gfp_mask);
273		}
274		if (unlikely(!sg)) {
275			if (prv)
276				table->nents = ++table->orig_nents;
277
278			return (-ENOMEM);
279		}
280		sg_init_table(sg, alloc_size);
281		table->nents = table->orig_nents += sg_size;
282
283		if (prv)
284			sg_chain(prv, max_ents, sg);
285		else
286			table->sgl = sg;
287
288		if (!left)
289			sg_mark_end(&sg[sg_size - 1]);
290
291		prv = sg;
292	} while (left);
293
294	return (0);
295}
296
297static inline int
298sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
299{
300	int ret;
301
302	ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
303	    NULL, gfp_mask, sg_kmalloc);
304	if (unlikely(ret))
305		__sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree);
306
307	return (ret);
308}
309
310static inline int
311__sg_alloc_table_from_pages(struct sg_table *sgt,
312    struct page **pages, unsigned int count,
313    unsigned long off, unsigned long size,
314    unsigned int max_segment, gfp_t gfp_mask)
315{
316	unsigned int i, segs, cur, len;
317	int rc;
318	struct scatterlist *s;
319
320	if (__predict_false(!max_segment || offset_in_page(max_segment)))
321		return (-EINVAL);
322
323	len = 0;
324	for (segs = i = 1; i < count; ++i) {
325		len += PAGE_SIZE;
326		if (len >= max_segment ||
327		    page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) {
328			++segs;
329			len = 0;
330		}
331	}
332	if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask))))
333		return (rc);
334
335	cur = 0;
336	for_each_sg(sgt->sgl, s, sgt->orig_nents, i) {
337		unsigned long seg_size;
338		unsigned int j;
339
340		len = 0;
341		for (j = cur + 1; j < count; ++j) {
342			len += PAGE_SIZE;
343			if (len >= max_segment || page_to_pfn(pages[j]) !=
344			    page_to_pfn(pages[j - 1]) + 1)
345				break;
346		}
347
348		seg_size = ((j - cur) << PAGE_SHIFT) - off;
349		sg_set_page(s, pages[cur], MIN(size, seg_size), off);
350		size -= seg_size;
351		off = 0;
352		cur = j;
353	}
354	return (0);
355}
356
357static inline int
358sg_alloc_table_from_pages(struct sg_table *sgt,
359    struct page **pages, unsigned int count,
360    unsigned long off, unsigned long size,
361    gfp_t gfp_mask)
362{
363
364	return (__sg_alloc_table_from_pages(sgt, pages, count, off, size,
365	    SCATTERLIST_MAX_SEGMENT, gfp_mask));
366}
367
368static inline int
369sg_nents(struct scatterlist *sg)
370{
371	int nents;
372
373	for (nents = 0; sg; sg = sg_next(sg))
374		nents++;
375	return (nents);
376}
377
378static inline void
379__sg_page_iter_start(struct sg_page_iter *piter,
380    struct scatterlist *sglist, unsigned int nents,
381    unsigned long pgoffset)
382{
383	piter->internal.pg_advance = 0;
384	piter->internal.nents = nents;
385
386	piter->sg = sglist;
387	piter->sg_pgoffset = pgoffset;
388}
389
390static inline void
391_sg_iter_next(struct sg_page_iter *iter)
392{
393	struct scatterlist *sg;
394	unsigned int pgcount;
395
396	sg = iter->sg;
397	pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
398
399	++iter->sg_pgoffset;
400	while (iter->sg_pgoffset >= pgcount) {
401		iter->sg_pgoffset -= pgcount;
402		sg = sg_next(sg);
403		--iter->maxents;
404		if (sg == NULL || iter->maxents == 0)
405			break;
406		pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
407	}
408	iter->sg = sg;
409}
410
411static inline int
412sg_page_count(struct scatterlist *sg)
413{
414	return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT);
415}
416#define	sg_dma_page_count(sg) \
417	sg_page_count(sg)
418
419static inline bool
420__sg_page_iter_next(struct sg_page_iter *piter)
421{
422	unsigned int pgcount;
423
424	if (piter->internal.nents == 0)
425		return (0);
426	if (piter->sg == NULL)
427		return (0);
428
429	piter->sg_pgoffset += piter->internal.pg_advance;
430	piter->internal.pg_advance = 1;
431
432	while (1) {
433		pgcount = sg_page_count(piter->sg);
434		if (likely(piter->sg_pgoffset < pgcount))
435			break;
436		piter->sg_pgoffset -= pgcount;
437		piter->sg = sg_next(piter->sg);
438		if (--piter->internal.nents == 0)
439			return (0);
440		if (piter->sg == NULL)
441			return (0);
442	}
443	return (1);
444}
445#define	__sg_page_iter_dma_next(itr) \
446	__sg_page_iter_next(&(itr)->base)
447
448static inline void
449_sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter,
450    unsigned int nents, unsigned long pgoffset)
451{
452	if (nents) {
453		iter->sg = sgl;
454		iter->sg_pgoffset = pgoffset - 1;
455		iter->maxents = nents;
456		_sg_iter_next(iter);
457	} else {
458		iter->sg = NULL;
459		iter->sg_pgoffset = 0;
460		iter->maxents = 0;
461	}
462}
463
464/*
465 * sg_page_iter_dma_address() is implemented as a macro because it
466 * needs to accept two different and identical structure types. This
467 * allows both old and new code to co-exist. The compile time assert
468 * adds some safety, that the structure sizes match.
469 */
470#define	sg_page_iter_dma_address(spi) ({		\
471	struct sg_page_iter *__spi = (void *)(spi);	\
472	dma_addr_t __dma_address;			\
473	CTASSERT(sizeof(*(spi)) == sizeof(*__spi));	\
474	__dma_address = __spi->sg->dma_address +	\
475	    (__spi->sg_pgoffset << PAGE_SHIFT);		\
476	__dma_address;					\
477})
478
479static inline struct page *
480sg_page_iter_page(struct sg_page_iter *piter)
481{
482	return (nth_page(sg_page(piter->sg), piter->sg_pgoffset));
483}
484
485static __inline size_t
486sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
487    const void *buf, size_t buflen, off_t skip)
488{
489	struct sg_page_iter piter;
490	struct page *page;
491	struct sf_buf *sf;
492	size_t len, copied;
493	char *p, *b;
494
495	if (buflen == 0)
496		return (0);
497
498	b = __DECONST(char *, buf);
499	copied = 0;
500	sched_pin();
501	for_each_sg_page(sgl, &piter, nents, 0) {
502
503		/* Skip to the start. */
504		if (piter.sg->length <= skip) {
505			skip -= piter.sg->length;
506			continue;
507		}
508
509		/* See how much to copy. */
510		KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)),
511		    ("%s: sg len %u - skip %ju || buflen %zu is 0\n",
512		    __func__, piter.sg->length, (uintmax_t)skip, buflen));
513		len = min(piter.sg->length - skip, buflen);
514
515		page = sg_page_iter_page(&piter);
516		sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT);
517		if (sf == NULL)
518			break;
519		p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip;
520		memcpy(p, b, len);
521		sf_buf_free(sf);
522
523		copied += len;
524		/* Either we exactly filled the page, or we are done. */
525		buflen -= len;
526		if (buflen == 0)
527			break;
528		skip -= len;
529		b += len;
530	}
531	sched_unpin();
532
533	return (copied);
534}
535
536#endif					/* _LINUX_SCATTERLIST_H_ */
537