1219820Sjeff/*- 2219820Sjeff * Copyright (c) 2010 Isilon Systems, Inc. 3219820Sjeff * Copyright (c) 2010 iX Systems, Inc. 4219820Sjeff * Copyright (c) 2010 Panasas, Inc. 5289682Shselasky * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. 6289567Shselasky * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 7219820Sjeff * All rights reserved. 8219820Sjeff * 9219820Sjeff * Redistribution and use in source and binary forms, with or without 10219820Sjeff * modification, are permitted provided that the following conditions 11219820Sjeff * are met: 12219820Sjeff * 1. Redistributions of source code must retain the above copyright 13219820Sjeff * notice unmodified, this list of conditions, and the following 14219820Sjeff * disclaimer. 15219820Sjeff * 2. Redistributions in binary form must reproduce the above copyright 16219820Sjeff * notice, this list of conditions and the following disclaimer in the 17219820Sjeff * documentation and/or other materials provided with the distribution. 18219820Sjeff * 19219820Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20219820Sjeff * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21219820Sjeff * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22219820Sjeff * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23219820Sjeff * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24219820Sjeff * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25219820Sjeff * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26219820Sjeff * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27219820Sjeff * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28219820Sjeff * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29289644Shselasky * 30289644Shselasky * $FreeBSD: releng/11.0/sys/compat/linuxkpi/common/include/linux/scatterlist.h 290135 2015-10-29 08:28:39Z hselasky $ 31219820Sjeff */ 32219820Sjeff#ifndef _LINUX_SCATTERLIST_H_ 33219820Sjeff#define _LINUX_SCATTERLIST_H_ 34219820Sjeff 35219820Sjeff#include <linux/page.h> 36273135Shselasky#include <linux/slab.h> 37219820Sjeff 38219820Sjeffstruct scatterlist { 39219820Sjeff union { 40289682Shselasky struct page *page; 41289682Shselasky struct scatterlist *sg; 42289682Shselasky } sl_un; 43289682Shselasky dma_addr_t address; 44289682Shselasky unsigned long offset; 45289682Shselasky uint32_t length; 46289682Shselasky uint32_t flags; 47219820Sjeff}; 48219820Sjeff 49270710Shselaskystruct sg_table { 50289682Shselasky struct scatterlist *sgl; 51289682Shselasky unsigned int nents; 52289682Shselasky unsigned int orig_nents; 53270710Shselasky}; 54270710Shselasky 55289567Shselaskystruct sg_page_iter { 56289682Shselasky struct scatterlist *sg; 57289682Shselasky unsigned int sg_pgoffset; 58289682Shselasky unsigned int maxents; 59289567Shselasky}; 60289567Shselasky 61289682Shselasky#define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 62273135Shselasky 63219820Sjeff#define sg_dma_address(sg) (sg)->address 64219820Sjeff#define sg_dma_len(sg) (sg)->length 65219820Sjeff#define sg_page(sg) (sg)->sl_un.page 66219820Sjeff#define sg_scatternext(sg) (sg)->sl_un.sg 67219820Sjeff 68219820Sjeff#define SG_END 0x01 69219820Sjeff#define SG_CHAIN 0x02 70219820Sjeff 71219820Sjeffstatic inline void 72219820Sjeffsg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 73219820Sjeff unsigned int offset) 74219820Sjeff{ 75219820Sjeff sg_page(sg) = page; 76219820Sjeff sg_dma_len(sg) = len; 77219820Sjeff sg->offset = offset; 78219820Sjeff if (offset > PAGE_SIZE) 79219820Sjeff panic("sg_set_page: Invalid offset %d\n", offset); 80219820Sjeff} 81219820Sjeff 82219820Sjeffstatic inline void 83219820Sjeffsg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 84219820Sjeff{ 85219820Sjeff sg_set_page(sg, virt_to_page(buf), buflen, 86289567Shselasky ((uintptr_t)buf) & (PAGE_SIZE - 1)); 87219820Sjeff} 88219820Sjeff 89219820Sjeffstatic inline void 90219820Sjeffsg_init_table(struct scatterlist *sg, unsigned int nents) 91219820Sjeff{ 92219820Sjeff bzero(sg, sizeof(*sg) * nents); 93219820Sjeff sg[nents - 1].flags = SG_END; 94219820Sjeff} 95219820Sjeff 96219820Sjeffstatic inline struct scatterlist * 97219820Sjeffsg_next(struct scatterlist *sg) 98219820Sjeff{ 99219820Sjeff if (sg->flags & SG_END) 100219820Sjeff return (NULL); 101219820Sjeff sg++; 102219820Sjeff if (sg->flags & SG_CHAIN) 103219820Sjeff sg = sg_scatternext(sg); 104219820Sjeff return (sg); 105219820Sjeff} 106219820Sjeff 107219820Sjeffstatic inline vm_paddr_t 108219820Sjeffsg_phys(struct scatterlist *sg) 109219820Sjeff{ 110219820Sjeff return sg_page(sg)->phys_addr + sg->offset; 111219820Sjeff} 112219820Sjeff 113273135Shselaskystatic inline void 114273135Shselaskysg_chain(struct scatterlist *prv, unsigned int prv_nents, 115289682Shselasky struct scatterlist *sgl) 116273135Shselasky{ 117273135Shselasky struct scatterlist *sg = &prv[prv_nents - 1]; 118273135Shselasky 119273135Shselasky sg->offset = 0; 120273135Shselasky sg->length = 0; 121273135Shselasky sg->flags = SG_CHAIN; 122273135Shselasky sg->sl_un.sg = sgl; 123273135Shselasky} 124273135Shselasky 125289682Shselaskystatic inline void 126289682Shselaskysg_mark_end(struct scatterlist *sg) 127273135Shselasky{ 128289682Shselasky sg->flags = SG_END; 129273135Shselasky} 130273135Shselasky 131273135Shselaskystatic inline void 132273135Shselasky__sg_free_table(struct sg_table *table, unsigned int max_ents) 133273135Shselasky{ 134273135Shselasky struct scatterlist *sgl, *next; 135273135Shselasky 136273135Shselasky if (unlikely(!table->sgl)) 137273135Shselasky return; 138273135Shselasky 139273135Shselasky sgl = table->sgl; 140273135Shselasky while (table->orig_nents) { 141273135Shselasky unsigned int alloc_size = table->orig_nents; 142273135Shselasky unsigned int sg_size; 143273135Shselasky 144273135Shselasky if (alloc_size > max_ents) { 145273135Shselasky next = sgl[max_ents - 1].sl_un.sg; 146273135Shselasky alloc_size = max_ents; 147273135Shselasky sg_size = alloc_size - 1; 148273135Shselasky } else { 149273135Shselasky sg_size = alloc_size; 150273135Shselasky next = NULL; 151273135Shselasky } 152273135Shselasky 153273135Shselasky table->orig_nents -= sg_size; 154273135Shselasky kfree(sgl); 155273135Shselasky sgl = next; 156273135Shselasky } 157273135Shselasky 158273135Shselasky table->sgl = NULL; 159273135Shselasky} 160273135Shselasky 161273135Shselaskystatic inline void 162273135Shselaskysg_free_table(struct sg_table *table) 163273135Shselasky{ 164273135Shselasky __sg_free_table(table, SG_MAX_SINGLE_ALLOC); 165273135Shselasky} 166273135Shselasky 167273135Shselaskystatic inline int 168273135Shselasky__sg_alloc_table(struct sg_table *table, unsigned int nents, 169289682Shselasky unsigned int max_ents, gfp_t gfp_mask) 170273135Shselasky{ 171273135Shselasky struct scatterlist *sg, *prv; 172273135Shselasky unsigned int left; 173273135Shselasky 174273135Shselasky memset(table, 0, sizeof(*table)); 175273135Shselasky 176273135Shselasky if (nents == 0) 177273135Shselasky return -EINVAL; 178273135Shselasky left = nents; 179273135Shselasky prv = NULL; 180273135Shselasky do { 181289682Shselasky unsigned int sg_size; 182289682Shselasky unsigned int alloc_size = left; 183273135Shselasky 184273135Shselasky if (alloc_size > max_ents) { 185273135Shselasky alloc_size = max_ents; 186273135Shselasky sg_size = alloc_size - 1; 187273135Shselasky } else 188273135Shselasky sg_size = alloc_size; 189273135Shselasky 190273135Shselasky left -= sg_size; 191273135Shselasky 192273135Shselasky sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); 193273135Shselasky if (unlikely(!sg)) { 194273135Shselasky if (prv) 195273135Shselasky table->nents = ++table->orig_nents; 196273135Shselasky 197273135Shselasky return -ENOMEM; 198273135Shselasky } 199273135Shselasky sg_init_table(sg, alloc_size); 200273135Shselasky table->nents = table->orig_nents += sg_size; 201273135Shselasky 202273135Shselasky if (prv) 203273135Shselasky sg_chain(prv, max_ents, sg); 204273135Shselasky else 205273135Shselasky table->sgl = sg; 206273135Shselasky 207273135Shselasky if (!left) 208273135Shselasky sg_mark_end(&sg[sg_size - 1]); 209273135Shselasky 210273135Shselasky prv = sg; 211273135Shselasky } while (left); 212273135Shselasky 213273135Shselasky return 0; 214273135Shselasky} 215273135Shselasky 216273135Shselaskystatic inline int 217273135Shselaskysg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 218273135Shselasky{ 219273135Shselasky int ret; 220273135Shselasky 221273135Shselasky ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 222289682Shselasky gfp_mask); 223273135Shselasky if (unlikely(ret)) 224273135Shselasky __sg_free_table(table, SG_MAX_SINGLE_ALLOC); 225273135Shselasky 226273135Shselasky return ret; 227273135Shselasky} 228273135Shselasky 229289567Shselaskystatic inline void 230289567Shselasky_sg_iter_next(struct sg_page_iter *iter) 231289567Shselasky{ 232289567Shselasky struct scatterlist *sg; 233289567Shselasky unsigned int pgcount; 234289567Shselasky 235289567Shselasky sg = iter->sg; 236289567Shselasky pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 237289567Shselasky 238289567Shselasky ++iter->sg_pgoffset; 239289567Shselasky while (iter->sg_pgoffset >= pgcount) { 240289567Shselasky iter->sg_pgoffset -= pgcount; 241289567Shselasky sg = sg_next(sg); 242289567Shselasky --iter->maxents; 243289567Shselasky if (sg == NULL || iter->maxents == 0) 244289567Shselasky break; 245289567Shselasky pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; 246289567Shselasky } 247289567Shselasky iter->sg = sg; 248289567Shselasky} 249289567Shselasky 250289567Shselaskystatic inline void 251289567Shselasky_sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, 252289682Shselasky unsigned int nents, unsigned long pgoffset) 253289567Shselasky{ 254289567Shselasky if (nents) { 255289567Shselasky iter->sg = sgl; 256289567Shselasky iter->sg_pgoffset = pgoffset - 1; 257289567Shselasky iter->maxents = nents; 258289567Shselasky _sg_iter_next(iter); 259289567Shselasky } else { 260289567Shselasky iter->sg = NULL; 261289567Shselasky iter->sg_pgoffset = 0; 262289567Shselasky iter->maxents = 0; 263289567Shselasky } 264289567Shselasky} 265289567Shselasky 266289567Shselaskystatic inline dma_addr_t 267289567Shselaskysg_page_iter_dma_address(struct sg_page_iter *spi) 268289567Shselasky{ 269289567Shselasky return spi->sg->address + (spi->sg_pgoffset << PAGE_SHIFT); 270289567Shselasky} 271289567Shselasky 272289567Shselasky#define for_each_sg_page(sgl, iter, nents, pgoffset) \ 273289567Shselasky for (_sg_iter_init(sgl, iter, nents, pgoffset); \ 274289567Shselasky (iter)->sg; _sg_iter_next(iter)) 275289567Shselasky 276219820Sjeff#define for_each_sg(sglist, sg, sgmax, _itr) \ 277219820Sjeff for (_itr = 0, sg = (sglist); _itr < (sgmax); _itr++, sg = sg_next(sg)) 278219820Sjeff 279289682Shselasky#endif /* _LINUX_SCATTERLIST_H_ */ 280