scatterlist.h revision 345938
1/*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD: stable/11/sys/compat/linuxkpi/common/include/linux/scatterlist.h 345938 2019-04-05 11:31:52Z hselasky $ 31 */ 32#ifndef _LINUX_SCATTERLIST_H_ 33#define _LINUX_SCATTERLIST_H_ 34 35#include <linux/page.h> 36#include <linux/slab.h> 37#include <linux/mm.h> 38 39struct scatterlist { 40 unsigned long page_link; 41#define SG_PAGE_LINK_CHAIN 0x1UL 42#define SG_PAGE_LINK_LAST 0x2UL 43#define SG_PAGE_LINK_MASK 0x3UL 44 unsigned int offset; 45 unsigned int length; 46 dma_addr_t address; 47}; 48 49CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 50 51struct sg_table { 52 struct scatterlist *sgl; 53 unsigned int nents; 54 unsigned int orig_nents; 55}; 56 57struct sg_page_iter { 58 struct scatterlist *sg; 59 unsigned int sg_pgoffset; 60 unsigned int maxents; 61 struct { 62 unsigned int nents; 63 int pg_advance; 64 } internal; 65}; 66 67#define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 68 69#define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 70 71#define SG_MAGIC 0x87654321UL 72#define SG_CHAIN SG_PAGE_LINK_CHAIN 73#define SG_END SG_PAGE_LINK_LAST 74 75#define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 76#define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 77#define sg_chain_ptr(sg) \ 78 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 79 80#define sg_dma_address(sg) (sg)->address 81#define sg_dma_len(sg) (sg)->length 82 83#define for_each_sg_page(sgl, iter, nents, pgoffset) \ 84 for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 85 (iter)->sg; _sg_iter_next(iter)) 86 87#define for_each_sg(sglist, sg, sgmax, iter) \ 88 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 89 90typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 91typedef void (sg_free_fn) (struct scatterlist *, unsigned int); 92 93static inline void 94sg_assign_page(struct scatterlist *sg, struct page *page) 95{ 96 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 97 98 sg->page_link = page_link | (unsigned long)page; 99} 100 101static inline void 102sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 103 unsigned int offset) 104{ 105 sg_assign_page(sg, page); 106 sg->offset = offset; 107 sg->length = len; 108} 109 110static inline struct page * 111sg_page(struct scatterlist *sg) 112{ 113 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 114} 115 116static inline void 117sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 118{ 119 sg_set_page(sg, virt_to_page(buf), buflen, 120 ((uintptr_t)buf) & (PAGE_SIZE - 1)); 121} 122 123static inline struct scatterlist * 124sg_next(struct scatterlist *sg) 125{ 126 if (sg_is_last(sg)) 127 return (NULL); 128 sg++; 129 if (sg_is_chain(sg)) 130 sg = sg_chain_ptr(sg); 131 return (sg); 132} 133 134static inline vm_paddr_t 135sg_phys(struct scatterlist *sg) 136{ 137 return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); 138} 139 140static inline void 141sg_chain(struct scatterlist *prv, unsigned int prv_nents, 142 struct scatterlist *sgl) 143{ 144 struct scatterlist *sg = &prv[prv_nents - 1]; 145 146 sg->offset = 0; 147 sg->length = 0; 148 sg->page_link = ((unsigned long)sgl | 149 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 150} 151 152static inline void 153sg_mark_end(struct scatterlist *sg) 154{ 155 sg->page_link |= SG_PAGE_LINK_LAST; 156 sg->page_link &= ~SG_PAGE_LINK_CHAIN; 157} 158 159static inline void 160sg_init_table(struct scatterlist *sg, unsigned int nents) 161{ 162 bzero(sg, sizeof(*sg) * nents); 163 sg_mark_end(&sg[nents - 1]); 164} 165 166static struct scatterlist * 167sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 168{ 169 if (nents == SG_MAX_SINGLE_ALLOC) { 170 return ((void *)__get_free_page(gfp_mask)); 171 } else 172 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 173} 174 175static inline void 176sg_kfree(struct scatterlist *sg, unsigned int nents) 177{ 178 if (nents == SG_MAX_SINGLE_ALLOC) { 179 free_page((unsigned long)sg); 180 } else 181 kfree(sg); 182} 183 184static inline void 185__sg_free_table(struct sg_table *table, unsigned int max_ents, 186 bool skip_first_chunk, sg_free_fn * free_fn) 187{ 188 struct scatterlist *sgl, *next; 189 190 if (unlikely(!table->sgl)) 191 return; 192 193 sgl = table->sgl; 194 while (table->orig_nents) { 195 unsigned int alloc_size = table->orig_nents; 196 unsigned int sg_size; 197 198 if (alloc_size > max_ents) { 199 next = sg_chain_ptr(&sgl[max_ents - 1]); 200 alloc_size = max_ents; 201 sg_size = alloc_size - 1; 202 } else { 203 sg_size = alloc_size; 204 next = NULL; 205 } 206 207 table->orig_nents -= sg_size; 208 if (skip_first_chunk) 209 skip_first_chunk = 0; 210 else 211 free_fn(sgl, alloc_size); 212 sgl = next; 213 } 214 215 table->sgl = NULL; 216} 217 218static inline void 219sg_free_table(struct sg_table *table) 220{ 221 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 222} 223 224static inline int 225__sg_alloc_table(struct sg_table *table, unsigned int nents, 226 unsigned int max_ents, struct scatterlist *first_chunk, 227 gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 228{ 229 struct scatterlist *sg, *prv; 230 unsigned int left; 231 232 memset(table, 0, sizeof(*table)); 233 234 if (nents == 0) 235 return (-EINVAL); 236 left = nents; 237 prv = NULL; 238 do { 239 unsigned int sg_size; 240 unsigned int alloc_size = left; 241 242 if (alloc_size > max_ents) { 243 alloc_size = max_ents; 244 sg_size = alloc_size - 1; 245 } else 246 sg_size = alloc_size; 247 248 left -= sg_size; 249 250 if (first_chunk) { 251 sg = first_chunk; 252 first_chunk = NULL; 253 } else { 254 sg = alloc_fn(alloc_size, gfp_mask); 255 } 256 if (unlikely(!sg)) { 257 if (prv) 258 table->nents = ++table->orig_nents; 259 260 return (-ENOMEM); 261 } 262 sg_init_table(sg, alloc_size); 263 table->nents = table->orig_nents += sg_size; 264 265 if (prv) 266 sg_chain(prv, max_ents, sg); 267 else 268 table->sgl = sg; 269 270 if (!left) 271 sg_mark_end(&sg[sg_size - 1]); 272 273 prv = sg; 274 } while (left); 275 276 return (0); 277} 278 279static inline int 280sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 281{ 282 int ret; 283 284 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 285 NULL, gfp_mask, sg_kmalloc); 286 if (unlikely(ret)) 287 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 288 289 return (ret); 290} 291 292static inline int 293__sg_alloc_table_from_pages(struct sg_table *sgt, 294 struct page **pages, unsigned int count, 295 unsigned long off, unsigned long size, 296 unsigned int max_segment, gfp_t gfp_mask) 297{ 298 unsigned int i, segs, cur, len; 299 int rc; 300 struct scatterlist *s; 301 302 if (__predict_false(!max_segment || offset_in_page(max_segment))) 303 return (-EINVAL); 304 305 len = 0; 306 for (segs = i = 1; i < count; ++i) { 307 len += PAGE_SIZE; 308 if (len >= max_segment || 309 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 310 ++segs; 311 len = 0; 312 } 313 } 314 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 315 return (rc); 316 317 cur = 0; 318 for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { 319 unsigned long seg_size; 320 unsigned int j; 321 322 len = 0; 323 for (j = cur + 1; j < count; ++j) { 324 len += PAGE_SIZE; 325 if (len >= max_segment || page_to_pfn(pages[j]) != 326 page_to_pfn(pages[j - 1]) + 1) 327 break; 328 } 329 330 seg_size = ((j - cur) << PAGE_SHIFT) - off; 331 sg_set_page(s, pages[cur], min(size, seg_size), off); 332 size -= seg_size; 333 off = 0; 334 cur = j; 335 } 336 return (0); 337} 338 339static inline int 340sg_alloc_table_from_pages(struct sg_table *sgt, 341 struct page **pages, unsigned int count, 342 unsigned long off, unsigned long size, 343 gfp_t gfp_mask) 344{ 345 346 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 347 SCATTERLIST_MAX_SEGMENT, gfp_mask)); 348} 349 350static inline int 351sg_nents(struct scatterlist *sg) 352{ 353 int nents; 354 355 for (nents = 0; sg; sg = sg_next(sg)) 356 nents++; 357 return (nents); 358} 359 360static inline void 361__sg_page_iter_start(struct sg_page_iter *piter, 362 struct scatterlist *sglist, unsigned int nents, 363 unsigned long pgoffset) 364{ 365 piter->internal.pg_advance = 0; 366 piter->internal.nents = nents; 367 368 piter->sg = sglist; 369 piter->sg_pgoffset = pgoffset; 370} 371 372static inline void 373_sg_iter_next(struct sg_page_iter *iter) 374{ 375 struct scatterlist *sg; 376 unsigned int pgcount; 377 378 sg = iter->sg; 379 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 380 381 ++iter->sg_pgoffset; 382 while (iter->sg_pgoffset >= pgcount) { 383 iter->sg_pgoffset -= pgcount; 384 sg = sg_next(sg); 385 --iter->maxents; 386 if (sg == NULL || iter->maxents == 0) 387 break; 388 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 389 } 390 iter->sg = sg; 391} 392 393static inline int 394sg_page_count(struct scatterlist *sg) 395{ 396 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 397} 398 399static inline bool 400__sg_page_iter_next(struct sg_page_iter *piter) 401{ 402 if (piter->internal.nents == 0) 403 return (0); 404 if (piter->sg == NULL) 405 return (0); 406 407 piter->sg_pgoffset += piter->internal.pg_advance; 408 piter->internal.pg_advance = 1; 409 410 while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { 411 piter->sg_pgoffset -= sg_page_count(piter->sg); 412 piter->sg = sg_next(piter->sg); 413 if (--piter->internal.nents == 0) 414 return (0); 415 if (piter->sg == NULL) 416 return (0); 417 } 418 return (1); 419} 420 421static inline void 422_sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 423 unsigned int nents, unsigned long pgoffset) 424{ 425 if (nents) { 426 iter->sg = sgl; 427 iter->sg_pgoffset = pgoffset - 1; 428 iter->maxents = nents; 429 _sg_iter_next(iter); 430 } else { 431 iter->sg = NULL; 432 iter->sg_pgoffset = 0; 433 iter->maxents = 0; 434 } 435} 436 437static inline dma_addr_t 438sg_page_iter_dma_address(struct sg_page_iter *spi) 439{ 440 return (spi->sg->address + (spi->sg_pgoffset << PAGE_SHIFT)); 441} 442 443static inline struct page * 444sg_page_iter_page(struct sg_page_iter *piter) 445{ 446 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 447} 448 449 450#endif /* _LINUX_SCATTERLIST_H_ */ 451