scatterlist.h revision 361204
1279377Simp/*- 2279377Simp * Copyright (c) 2010 Isilon Systems, Inc. 3279377Simp * Copyright (c) 2010 iX Systems, Inc. 4279377Simp * Copyright (c) 2010 Panasas, Inc. 5279377Simp * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6279377Simp * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7279377Simp * All rights reserved. 8279377Simp * 9279377Simp * Redistribution and use in source and binary forms, with or without 10279377Simp * modification, are permitted provided that the following conditions 11279377Simp * are met: 12279377Simp * 1. Redistributions of source code must retain the above copyright 13279377Simp * notice unmodified, this list of conditions, and the following 14279377Simp * disclaimer. 15279377Simp * 2. Redistributions in binary form must reproduce the above copyright 16279377Simp * notice, this list of conditions and the following disclaimer in the 17279377Simp * documentation and/or other materials provided with the distribution. 18279377Simp * 19279377Simp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20279377Simp * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21279377Simp * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22279377Simp * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23279377Simp * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24279377Simp * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25279377Simp * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26279377Simp * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27279377Simp * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28279377Simp * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29279377Simp * 30279377Simp * $FreeBSD: stable/11/sys/compat/linuxkpi/common/include/linux/scatterlist.h 361204 2020-05-18 09:43:31Z hselasky $ 31279377Simp */ 32279377Simp#ifndef _LINUX_SCATTERLIST_H_ 33279377Simp#define _LINUX_SCATTERLIST_H_ 34279377Simp 35279377Simp#include <linux/page.h> 36279377Simp#include <linux/slab.h> 37279377Simp#include <linux/mm.h> 38279377Simp 39279377Simpstruct scatterlist { 40279377Simp unsigned long page_link; 41279377Simp#define SG_PAGE_LINK_CHAIN 0x1UL 42279377Simp#define SG_PAGE_LINK_LAST 0x2UL 43279377Simp#define SG_PAGE_LINK_MASK 0x3UL 44279377Simp unsigned int offset; 45279377Simp unsigned int length; 46279377Simp dma_addr_t address; 47279377Simp}; 48279377Simp 49279377SimpCTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); 50279377Simp 51279377Simpstruct sg_table { 52279377Simp struct scatterlist *sgl; 53279377Simp unsigned int nents; 54279377Simp unsigned int orig_nents; 55279377Simp}; 56279377Simp 57279377Simpstruct sg_page_iter { 58279377Simp struct scatterlist *sg; 59279377Simp unsigned int sg_pgoffset; 60279377Simp unsigned int maxents; 61279377Simp struct { 62279377Simp unsigned int nents; 63279377Simp int pg_advance; 64279377Simp } internal; 65279377Simp}; 66279377Simp 67279377Simpstruct sg_dma_page_iter { 68279377Simp struct sg_page_iter base; 69279377Simp}; 70279377Simp 71279377Simp#define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) 72279377Simp 73279377Simp#define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 74279377Simp 75279377Simp#define SG_MAGIC 0x87654321UL 76279377Simp#define SG_CHAIN SG_PAGE_LINK_CHAIN 77279377Simp#define SG_END SG_PAGE_LINK_LAST 78279377Simp 79279377Simp#define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) 80279377Simp#define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) 81279377Simp#define sg_chain_ptr(sg) \ 82279377Simp ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) 83279377Simp 84279377Simp#define sg_dma_address(sg) (sg)->address 85279377Simp#define sg_dma_len(sg) (sg)->length 86279377Simp 87279377Simp#define for_each_sg_page(sgl, iter, nents, pgoffset) \ 88279377Simp for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 89279377Simp (iter)->sg; _sg_iter_next(iter)) 90279377Simp#define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ 91279377Simp for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) 92279377Simp 93279377Simp#define for_each_sg(sglist, sg, sgmax, iter) \ 94279377Simp for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) 95279377Simp 96279377Simptypedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); 97279377Simptypedef void (sg_free_fn) (struct scatterlist *, unsigned int); 98279377Simp 99279377Simpstatic inline void 100279377Simpsg_assign_page(struct scatterlist *sg, struct page *page) 101279377Simp{ 102279377Simp unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; 103279377Simp 104279377Simp sg->page_link = page_link | (unsigned long)page; 105279377Simp} 106279377Simp 107279377Simpstatic inline void 108279377Simpsg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 109279377Simp unsigned int offset) 110279377Simp{ 111279377Simp sg_assign_page(sg, page); 112279377Simp sg->offset = offset; 113279377Simp sg->length = len; 114279377Simp} 115279377Simp 116279377Simpstatic inline struct page * 117279377Simpsg_page(struct scatterlist *sg) 118279377Simp{ 119279377Simp return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); 120279377Simp} 121279377Simp 122279377Simpstatic inline void 123279377Simpsg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 124279377Simp{ 125279377Simp sg_set_page(sg, virt_to_page(buf), buflen, 126279377Simp ((uintptr_t)buf) & (PAGE_SIZE - 1)); 127279377Simp} 128279377Simp 129279377Simpstatic inline struct scatterlist * 130279377Simpsg_next(struct scatterlist *sg) 131279377Simp{ 132279377Simp if (sg_is_last(sg)) 133279377Simp return (NULL); 134279377Simp sg++; 135279377Simp if (sg_is_chain(sg)) 136279377Simp sg = sg_chain_ptr(sg); 137279377Simp return (sg); 138279377Simp} 139279377Simp 140279377Simpstatic inline vm_paddr_t 141279377Simpsg_phys(struct scatterlist *sg) 142279377Simp{ 143279377Simp return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); 144279377Simp} 145279377Simp 146279377Simpstatic inline void * 147279377Simpsg_virt(struct scatterlist *sg) 148279377Simp{ 149279377Simp 150279377Simp return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); 151279377Simp} 152279377Simp 153279377Simpstatic inline void 154279377Simpsg_chain(struct scatterlist *prv, unsigned int prv_nents, 155279377Simp struct scatterlist *sgl) 156279377Simp{ 157279377Simp struct scatterlist *sg = &prv[prv_nents - 1]; 158279377Simp 159279377Simp sg->offset = 0; 160279377Simp sg->length = 0; 161279377Simp sg->page_link = ((unsigned long)sgl | 162279377Simp SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; 163279377Simp} 164279377Simp 165279377Simpstatic inline void 166279377Simpsg_mark_end(struct scatterlist *sg) 167279377Simp{ 168279377Simp sg->page_link |= SG_PAGE_LINK_LAST; 169279377Simp sg->page_link &= ~SG_PAGE_LINK_CHAIN; 170279377Simp} 171279377Simp 172279377Simpstatic inline void 173279377Simpsg_init_table(struct scatterlist *sg, unsigned int nents) 174279377Simp{ 175279377Simp bzero(sg, sizeof(*sg) * nents); 176279377Simp sg_mark_end(&sg[nents - 1]); 177279377Simp} 178279377Simp 179279377Simpstatic struct scatterlist * 180279377Simpsg_kmalloc(unsigned int nents, gfp_t gfp_mask) 181279377Simp{ 182279377Simp if (nents == SG_MAX_SINGLE_ALLOC) { 183279377Simp return ((void *)__get_free_page(gfp_mask)); 184279377Simp } else 185279377Simp return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); 186279377Simp} 187279377Simp 188279377Simpstatic inline void 189279377Simpsg_kfree(struct scatterlist *sg, unsigned int nents) 190279377Simp{ 191279377Simp if (nents == SG_MAX_SINGLE_ALLOC) { 192279377Simp free_page((unsigned long)sg); 193279377Simp } else 194279377Simp kfree(sg); 195279377Simp} 196279377Simp 197279377Simpstatic inline void 198279377Simp__sg_free_table(struct sg_table *table, unsigned int max_ents, 199279377Simp bool skip_first_chunk, sg_free_fn * free_fn) 200279377Simp{ 201279377Simp struct scatterlist *sgl, *next; 202279377Simp 203279377Simp if (unlikely(!table->sgl)) 204279377Simp return; 205279377Simp 206279377Simp sgl = table->sgl; 207279377Simp while (table->orig_nents) { 208279377Simp unsigned int alloc_size = table->orig_nents; 209279377Simp unsigned int sg_size; 210279377Simp 211279377Simp if (alloc_size > max_ents) { 212279377Simp next = sg_chain_ptr(&sgl[max_ents - 1]); 213279377Simp alloc_size = max_ents; 214279377Simp sg_size = alloc_size - 1; 215279377Simp } else { 216279377Simp sg_size = alloc_size; 217279377Simp next = NULL; 218279377Simp } 219279377Simp 220279377Simp table->orig_nents -= sg_size; 221279377Simp if (skip_first_chunk) 222279377Simp skip_first_chunk = 0; 223279377Simp else 224279377Simp free_fn(sgl, alloc_size); 225279377Simp sgl = next; 226279377Simp } 227279377Simp 228279377Simp table->sgl = NULL; 229279377Simp} 230279377Simp 231279377Simpstatic inline void 232279377Simpsg_free_table(struct sg_table *table) 233279377Simp{ 234279377Simp __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 235279377Simp} 236279377Simp 237279377Simpstatic inline int 238279377Simp__sg_alloc_table(struct sg_table *table, unsigned int nents, 239279377Simp unsigned int max_ents, struct scatterlist *first_chunk, 240279377Simp gfp_t gfp_mask, sg_alloc_fn *alloc_fn) 241279377Simp{ 242279377Simp struct scatterlist *sg, *prv; 243279377Simp unsigned int left; 244279377Simp 245279377Simp memset(table, 0, sizeof(*table)); 246279377Simp 247279377Simp if (nents == 0) 248279377Simp return (-EINVAL); 249279377Simp left = nents; 250279377Simp prv = NULL; 251279377Simp do { 252279377Simp unsigned int sg_size; 253279377Simp unsigned int alloc_size = left; 254279377Simp 255279377Simp if (alloc_size > max_ents) { 256279377Simp alloc_size = max_ents; 257279377Simp sg_size = alloc_size - 1; 258279377Simp } else 259279377Simp sg_size = alloc_size; 260279377Simp 261279377Simp left -= sg_size; 262279377Simp 263279377Simp if (first_chunk) { 264279377Simp sg = first_chunk; 265279377Simp first_chunk = NULL; 266279377Simp } else { 267279377Simp sg = alloc_fn(alloc_size, gfp_mask); 268279377Simp } 269279377Simp if (unlikely(!sg)) { 270279377Simp if (prv) 271279377Simp table->nents = ++table->orig_nents; 272279377Simp 273279377Simp return (-ENOMEM); 274279377Simp } 275279377Simp sg_init_table(sg, alloc_size); 276279377Simp table->nents = table->orig_nents += sg_size; 277279377Simp 278279377Simp if (prv) 279279377Simp sg_chain(prv, max_ents, sg); 280279377Simp else 281279377Simp table->sgl = sg; 282279377Simp 283279377Simp if (!left) 284279377Simp sg_mark_end(&sg[sg_size - 1]); 285279377Simp 286279377Simp prv = sg; 287279377Simp } while (left); 288279377Simp 289279377Simp return (0); 290279377Simp} 291279377Simp 292279377Simpstatic inline int 293279377Simpsg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 294279377Simp{ 295279377Simp int ret; 296279377Simp 297279377Simp ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 298279377Simp NULL, gfp_mask, sg_kmalloc); 299279377Simp if (unlikely(ret)) 300279377Simp __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); 301279377Simp 302279377Simp return (ret); 303279377Simp} 304279377Simp 305279377Simpstatic inline int 306279377Simp__sg_alloc_table_from_pages(struct sg_table *sgt, 307279377Simp struct page **pages, unsigned int count, 308279377Simp unsigned long off, unsigned long size, 309279377Simp unsigned int max_segment, gfp_t gfp_mask) 310279377Simp{ 311279377Simp unsigned int i, segs, cur, len; 312279377Simp int rc; 313279377Simp struct scatterlist *s; 314279377Simp 315279377Simp if (__predict_false(!max_segment || offset_in_page(max_segment))) 316279377Simp return (-EINVAL); 317279377Simp 318279377Simp len = 0; 319279377Simp for (segs = i = 1; i < count; ++i) { 320279377Simp len += PAGE_SIZE; 321279377Simp if (len >= max_segment || 322279377Simp page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { 323279377Simp ++segs; 324279377Simp len = 0; 325279377Simp } 326279377Simp } 327279377Simp if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) 328279377Simp return (rc); 329279377Simp 330279377Simp cur = 0; 331279377Simp for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { 332279377Simp unsigned long seg_size; 333279377Simp unsigned int j; 334279377Simp 335279377Simp len = 0; 336279377Simp for (j = cur + 1; j < count; ++j) { 337279377Simp len += PAGE_SIZE; 338279377Simp if (len >= max_segment || page_to_pfn(pages[j]) != 339279377Simp page_to_pfn(pages[j - 1]) + 1) 340279377Simp break; 341279377Simp } 342279377Simp 343279377Simp seg_size = ((j - cur) << PAGE_SHIFT) - off; 344279377Simp sg_set_page(s, pages[cur], min(size, seg_size), off); 345279377Simp size -= seg_size; 346279377Simp off = 0; 347279377Simp cur = j; 348279377Simp } 349279377Simp return (0); 350279377Simp} 351279377Simp 352279377Simpstatic inline int 353279377Simpsg_alloc_table_from_pages(struct sg_table *sgt, 354279377Simp struct page **pages, unsigned int count, 355279377Simp unsigned long off, unsigned long size, 356279377Simp gfp_t gfp_mask) 357279377Simp{ 358279377Simp 359279377Simp return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, 360279377Simp SCATTERLIST_MAX_SEGMENT, gfp_mask)); 361279377Simp} 362279377Simp 363279377Simpstatic inline int 364279377Simpsg_nents(struct scatterlist *sg) 365279377Simp{ 366279377Simp int nents; 367279377Simp 368279377Simp for (nents = 0; sg; sg = sg_next(sg)) 369279377Simp nents++; 370279377Simp return (nents); 371279377Simp} 372279377Simp 373279377Simpstatic inline void 374279377Simp__sg_page_iter_start(struct sg_page_iter *piter, 375279377Simp struct scatterlist *sglist, unsigned int nents, 376279377Simp unsigned long pgoffset) 377279377Simp{ 378279377Simp piter->internal.pg_advance = 0; 379279377Simp piter->internal.nents = nents; 380279377Simp 381279377Simp piter->sg = sglist; 382279377Simp piter->sg_pgoffset = pgoffset; 383279377Simp} 384279377Simp 385279377Simpstatic inline void 386279377Simp_sg_iter_next(struct sg_page_iter *iter) 387279377Simp{ 388279377Simp struct scatterlist *sg; 389279377Simp unsigned int pgcount; 390279377Simp 391279377Simp sg = iter->sg; 392279377Simp pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 393279377Simp 394279377Simp ++iter->sg_pgoffset; 395279377Simp while (iter->sg_pgoffset >= pgcount) { 396279377Simp iter->sg_pgoffset -= pgcount; 397279377Simp sg = sg_next(sg); 398279377Simp --iter->maxents; 399279377Simp if (sg == NULL || iter->maxents == 0) 400279377Simp break; 401279377Simp pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 402279377Simp } 403279377Simp iter->sg = sg; 404279377Simp} 405279377Simp 406279377Simpstatic inline int 407279377Simpsg_page_count(struct scatterlist *sg) 408279377Simp{ 409279377Simp return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); 410279377Simp} 411279377Simp#define sg_dma_page_count(sg) \ 412279377Simp sg_page_count(sg) 413279377Simp 414279377Simpstatic inline bool 415279377Simp__sg_page_iter_next(struct sg_page_iter *piter) 416279377Simp{ 417279377Simp if (piter->internal.nents == 0) 418279377Simp return (0); 419279377Simp if (piter->sg == NULL) 420279377Simp return (0); 421279377Simp 422279377Simp piter->sg_pgoffset += piter->internal.pg_advance; 423279377Simp piter->internal.pg_advance = 1; 424279377Simp 425279377Simp while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { 426279377Simp piter->sg_pgoffset -= sg_page_count(piter->sg); 427279377Simp piter->sg = sg_next(piter->sg); 428279377Simp if (--piter->internal.nents == 0) 429279377Simp return (0); 430279377Simp if (piter->sg == NULL) 431279377Simp return (0); 432279377Simp } 433279377Simp return (1); 434279377Simp} 435279377Simp#define __sg_page_iter_dma_next(itr) \ 436279377Simp __sg_page_iter_next(&(itr)->base) 437279377Simp 438279377Simpstatic inline void 439279377Simp_sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 440279377Simp unsigned int nents, unsigned long pgoffset) 441279377Simp{ 442279377Simp if (nents) { 443279377Simp iter->sg = sgl; 444279377Simp iter->sg_pgoffset = pgoffset - 1; 445279377Simp iter->maxents = nents; 446279377Simp _sg_iter_next(iter); 447279377Simp } else { 448279377Simp iter->sg = NULL; 449279377Simp iter->sg_pgoffset = 0; 450279377Simp iter->maxents = 0; 451279377Simp } 452279377Simp} 453279377Simp 454279377Simp/* 455279377Simp * sg_page_iter_dma_address() is implemented as a macro because it 456279377Simp * needs to accept two different and identical structure types. This 457279377Simp * allows both old and new code to co-exist. The compile time assert 458279377Simp * adds some safety, that the structure sizes match. 459279377Simp */ 460279377Simp#define sg_page_iter_dma_address(spi) ({ \ 461279377Simp struct sg_page_iter *__spi = (void *)(spi); \ 462279377Simp dma_addr_t __dma_address; \ 463279377Simp CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ 464279377Simp __dma_address = __spi->sg->address + \ 465279377Simp (__spi->sg_pgoffset << PAGE_SHIFT); \ 466279377Simp __dma_address; \ 467279377Simp}) 468279377Simp 469279377Simpstatic inline struct page * 470279377Simpsg_page_iter_page(struct sg_page_iter *piter) 471279377Simp{ 472279377Simp return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); 473279377Simp} 474279377Simp 475279377Simp 476279377Simp#endif /* _LINUX_SCATTERLIST_H_ */ 477279377Simp