scatterlist.h revision 273135
1219820Sjeff/*- 2219820Sjeff * Copyright (c) 2010 Isilon Systems, Inc. 3219820Sjeff * Copyright (c) 2010 iX Systems, Inc. 4219820Sjeff * Copyright (c) 2010 Panasas, Inc. 5270710Shselasky * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. 6219820Sjeff * All rights reserved. 7219820Sjeff * 8219820Sjeff * Redistribution and use in source and binary forms, with or without 9219820Sjeff * modification, are permitted provided that the following conditions 10219820Sjeff * are met: 11219820Sjeff * 1. Redistributions of source code must retain the above copyright 12219820Sjeff * notice unmodified, this list of conditions, and the following 13219820Sjeff * disclaimer. 14219820Sjeff * 2. Redistributions in binary form must reproduce the above copyright 15219820Sjeff * notice, this list of conditions and the following disclaimer in the 16219820Sjeff * documentation and/or other materials provided with the distribution. 17219820Sjeff * 18219820Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19219820Sjeff * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20219820Sjeff * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21219820Sjeff * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22219820Sjeff * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23219820Sjeff * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24219820Sjeff * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25219820Sjeff * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26219820Sjeff * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27219820Sjeff * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28219820Sjeff */ 29270710Shselasky 30219820Sjeff#ifndef _LINUX_SCATTERLIST_H_ 31219820Sjeff#define _LINUX_SCATTERLIST_H_ 32219820Sjeff 33219820Sjeff#include <linux/page.h> 34273135Shselasky#include <linux/slab.h> 35219820Sjeff 36273135Shselasky/* 37273135Shselasky * SG table design. 38273135Shselasky * 39273135Shselasky * If flags bit 0 is set, then the sg field contains a pointer to the next sg 40273135Shselasky * table list. Otherwise the next entry is at sg + 1, can be determined using 41273135Shselasky * the sg_is_chain() function. 42273135Shselasky * 43273135Shselasky * If flags bit 1 is set, then this sg entry is the last element in a list, 44273135Shselasky * can be determined using the sg_is_last() function. 45273135Shselasky * 46273135Shselasky * See sg_next(). 47273135Shselasky * 48273135Shselasky */ 49273135Shselasky 50219820Sjeffstruct scatterlist { 51219820Sjeff union { 52219820Sjeff struct page *page; 53219820Sjeff struct scatterlist *sg; 54219820Sjeff } sl_un; 55219846Skib dma_addr_t address; 56219820Sjeff unsigned long offset; 57219820Sjeff uint32_t length; 58219820Sjeff uint32_t flags; 59219820Sjeff}; 60219820Sjeff 61270710Shselaskystruct sg_table { 62270710Shselasky struct scatterlist *sgl; /* the list */ 63270710Shselasky unsigned int nents; /* number of mapped entries */ 64270710Shselasky unsigned int orig_nents; /* original size of list */ 65270710Shselasky}; 66270710Shselasky 67273135Shselasky/* 68273135Shselasky * Maximum number of entries that will be allocated in one piece, if 69273135Shselasky * a list larger than this is required then chaining will be utilized. 70273135Shselasky */ 71273135Shselasky#define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 72273135Shselasky 73219820Sjeff#define sg_dma_address(sg) (sg)->address 74219820Sjeff#define sg_dma_len(sg) (sg)->length 75219820Sjeff#define sg_page(sg) (sg)->sl_un.page 76219820Sjeff#define sg_scatternext(sg) (sg)->sl_un.sg 77219820Sjeff 78219820Sjeff#define SG_END 0x01 79219820Sjeff#define SG_CHAIN 0x02 80219820Sjeff 81219820Sjeffstatic inline void 82219820Sjeffsg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, 83219820Sjeff unsigned int offset) 84219820Sjeff{ 85219820Sjeff sg_page(sg) = page; 86219820Sjeff sg_dma_len(sg) = len; 87219820Sjeff sg->offset = offset; 88219820Sjeff if (offset > PAGE_SIZE) 89219820Sjeff panic("sg_set_page: Invalid offset %d\n", offset); 90219820Sjeff} 91219820Sjeff 92219820Sjeffstatic inline void 93219820Sjeffsg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) 94219820Sjeff{ 95219820Sjeff sg_set_page(sg, virt_to_page(buf), buflen, 96219820Sjeff ((uintptr_t)buf) & ~PAGE_MASK); 97219820Sjeff} 98219820Sjeff 99219820Sjeffstatic inline void 100219820Sjeffsg_init_table(struct scatterlist *sg, unsigned int nents) 101219820Sjeff{ 102219820Sjeff bzero(sg, sizeof(*sg) * nents); 103219820Sjeff sg[nents - 1].flags = SG_END; 104219820Sjeff} 105219820Sjeff 106219820Sjeffstatic inline struct scatterlist * 107219820Sjeffsg_next(struct scatterlist *sg) 108219820Sjeff{ 109219820Sjeff if (sg->flags & SG_END) 110219820Sjeff return (NULL); 111219820Sjeff sg++; 112219820Sjeff if (sg->flags & SG_CHAIN) 113219820Sjeff sg = sg_scatternext(sg); 114219820Sjeff return (sg); 115219820Sjeff} 116219820Sjeff 117219820Sjeffstatic inline vm_paddr_t 118219820Sjeffsg_phys(struct scatterlist *sg) 119219820Sjeff{ 120219820Sjeff return sg_page(sg)->phys_addr + sg->offset; 121219820Sjeff} 122219820Sjeff 123273135Shselasky/** 124273135Shselasky * sg_chain - Chain two sglists together 125273135Shselasky * @prv: First scatterlist 126273135Shselasky * @prv_nents: Number of entries in prv 127273135Shselasky * @sgl: Second scatterlist 128273135Shselasky * 129273135Shselasky * Description: 130273135Shselasky * Links @prv@ and @sgl@ together, to form a longer scatterlist. 131273135Shselasky * 132273135Shselasky **/ 133273135Shselaskystatic inline void 134273135Shselaskysg_chain(struct scatterlist *prv, unsigned int prv_nents, 135273135Shselasky struct scatterlist *sgl) 136273135Shselasky{ 137273135Shselasky/* 138273135Shselasky * offset and length are unused for chain entry. Clear them. 139273135Shselasky */ 140273135Shselasky struct scatterlist *sg = &prv[prv_nents - 1]; 141273135Shselasky 142273135Shselasky sg->offset = 0; 143273135Shselasky sg->length = 0; 144273135Shselasky 145273135Shselasky /* 146273135Shselasky * Indicate a link pointer, and set the link to the second list. 147273135Shselasky */ 148273135Shselasky sg->flags = SG_CHAIN; 149273135Shselasky sg->sl_un.sg = sgl; 150273135Shselasky} 151273135Shselasky 152273135Shselasky/** 153273135Shselasky * sg_mark_end - Mark the end of the scatterlist 154273135Shselasky * @sg: SG entryScatterlist 155273135Shselasky * 156273135Shselasky * Description: 157273135Shselasky * Marks the passed in sg entry as the termination point for the sg 158273135Shselasky * table. A call to sg_next() on this entry will return NULL. 159273135Shselasky * 160273135Shselasky **/ 161273135Shselaskystatic inline void sg_mark_end(struct scatterlist *sg) 162273135Shselasky{ 163273135Shselasky sg->flags = SG_END; 164273135Shselasky} 165273135Shselasky 166273135Shselasky/** 167273135Shselasky * __sg_free_table - Free a previously mapped sg table 168273135Shselasky * @table: The sg table header to use 169273135Shselasky * @max_ents: The maximum number of entries per single scatterlist 170273135Shselasky * 171273135Shselasky * Description: 172273135Shselasky * Free an sg table previously allocated and setup with 173273135Shselasky * __sg_alloc_table(). The @max_ents value must be identical to 174273135Shselasky * that previously used with __sg_alloc_table(). 175273135Shselasky * 176273135Shselasky **/ 177273135Shselaskystatic inline void 178273135Shselasky__sg_free_table(struct sg_table *table, unsigned int max_ents) 179273135Shselasky{ 180273135Shselasky struct scatterlist *sgl, *next; 181273135Shselasky 182273135Shselasky if (unlikely(!table->sgl)) 183273135Shselasky return; 184273135Shselasky 185273135Shselasky sgl = table->sgl; 186273135Shselasky while (table->orig_nents) { 187273135Shselasky unsigned int alloc_size = table->orig_nents; 188273135Shselasky unsigned int sg_size; 189273135Shselasky 190273135Shselasky /* 191273135Shselasky * If we have more than max_ents segments left, 192273135Shselasky * then assign 'next' to the sg table after the current one. 193273135Shselasky * sg_size is then one less than alloc size, since the last 194273135Shselasky * element is the chain pointer. 195273135Shselasky */ 196273135Shselasky if (alloc_size > max_ents) { 197273135Shselasky next = sgl[max_ents - 1].sl_un.sg; 198273135Shselasky alloc_size = max_ents; 199273135Shselasky sg_size = alloc_size - 1; 200273135Shselasky } else { 201273135Shselasky sg_size = alloc_size; 202273135Shselasky next = NULL; 203273135Shselasky } 204273135Shselasky 205273135Shselasky table->orig_nents -= sg_size; 206273135Shselasky kfree(sgl); 207273135Shselasky sgl = next; 208273135Shselasky } 209273135Shselasky 210273135Shselasky table->sgl = NULL; 211273135Shselasky} 212273135Shselasky 213273135Shselasky/** 214273135Shselasky * sg_free_table - Free a previously allocated sg table 215273135Shselasky * @table: The mapped sg table header 216273135Shselasky * 217273135Shselasky **/ 218273135Shselaskystatic inline void 219273135Shselaskysg_free_table(struct sg_table *table) 220273135Shselasky{ 221273135Shselasky __sg_free_table(table, SG_MAX_SINGLE_ALLOC); 222273135Shselasky} 223273135Shselasky 224273135Shselasky/** 225273135Shselasky * __sg_alloc_table - Allocate and initialize an sg table with given allocator 226273135Shselasky * @table: The sg table header to use 227273135Shselasky * @nents: Number of entries in sg list 228273135Shselasky * @max_ents: The maximum number of entries the allocator returns per call 229273135Shselasky * @gfp_mask: GFP allocation mask 230273135Shselasky * 231273135Shselasky * Description: 232273135Shselasky * This function returns a @table @nents long. The allocator is 233273135Shselasky * defined to return scatterlist chunks of maximum size @max_ents. 234273135Shselasky * Thus if @nents is bigger than @max_ents, the scatterlists will be 235273135Shselasky * chained in units of @max_ents. 236273135Shselasky * 237273135Shselasky * Notes: 238273135Shselasky * If this function returns non-0 (eg failure), the caller must call 239273135Shselasky * __sg_free_table() to cleanup any leftover allocations. 240273135Shselasky * 241273135Shselasky **/ 242273135Shselaskystatic inline int 243273135Shselasky__sg_alloc_table(struct sg_table *table, unsigned int nents, 244273135Shselasky unsigned int max_ents, gfp_t gfp_mask) 245273135Shselasky{ 246273135Shselasky struct scatterlist *sg, *prv; 247273135Shselasky unsigned int left; 248273135Shselasky 249273135Shselasky memset(table, 0, sizeof(*table)); 250273135Shselasky 251273135Shselasky if (nents == 0) 252273135Shselasky return -EINVAL; 253273135Shselasky left = nents; 254273135Shselasky prv = NULL; 255273135Shselasky do { 256273135Shselasky unsigned int sg_size, alloc_size = left; 257273135Shselasky 258273135Shselasky if (alloc_size > max_ents) { 259273135Shselasky alloc_size = max_ents; 260273135Shselasky sg_size = alloc_size - 1; 261273135Shselasky } else 262273135Shselasky sg_size = alloc_size; 263273135Shselasky 264273135Shselasky left -= sg_size; 265273135Shselasky 266273135Shselasky sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); 267273135Shselasky if (unlikely(!sg)) { 268273135Shselasky /* 269273135Shselasky * Adjust entry count to reflect that the last 270273135Shselasky * entry of the previous table won't be used for 271273135Shselasky * linkage. Without this, sg_kfree() may get 272273135Shselasky * confused. 273273135Shselasky */ 274273135Shselasky if (prv) 275273135Shselasky table->nents = ++table->orig_nents; 276273135Shselasky 277273135Shselasky return -ENOMEM; 278273135Shselasky } 279273135Shselasky 280273135Shselasky sg_init_table(sg, alloc_size); 281273135Shselasky table->nents = table->orig_nents += sg_size; 282273135Shselasky 283273135Shselasky /* 284273135Shselasky * If this is the first mapping, assign the sg table header. 285273135Shselasky * If this is not the first mapping, chain previous part. 286273135Shselasky */ 287273135Shselasky if (prv) 288273135Shselasky sg_chain(prv, max_ents, sg); 289273135Shselasky else 290273135Shselasky table->sgl = sg; 291273135Shselasky 292273135Shselasky /* 293273135Shselasky * If no more entries after this one, mark the end 294273135Shselasky */ 295273135Shselasky if (!left) 296273135Shselasky sg_mark_end(&sg[sg_size - 1]); 297273135Shselasky 298273135Shselasky prv = sg; 299273135Shselasky } while (left); 300273135Shselasky 301273135Shselasky return 0; 302273135Shselasky} 303273135Shselasky 304273135Shselasky/** 305273135Shselasky * sg_alloc_table - Allocate and initialize an sg table 306273135Shselasky * @table: The sg table header to use 307273135Shselasky * @nents: Number of entries in sg list 308273135Shselasky * @gfp_mask: GFP allocation mask 309273135Shselasky * 310273135Shselasky * Description: 311273135Shselasky * Allocate and initialize an sg table. If @nents@ is larger than 312273135Shselasky * SG_MAX_SINGLE_ALLOC a chained sg table will be setup. 313273135Shselasky * 314273135Shselasky **/ 315273135Shselasky 316273135Shselaskystatic inline int 317273135Shselaskysg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 318273135Shselasky{ 319273135Shselasky int ret; 320273135Shselasky 321273135Shselasky ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 322273135Shselasky gfp_mask); 323273135Shselasky if (unlikely(ret)) 324273135Shselasky __sg_free_table(table, SG_MAX_SINGLE_ALLOC); 325273135Shselasky 326273135Shselasky return ret; 327273135Shselasky} 328273135Shselasky 329219820Sjeff#define for_each_sg(sglist, sg, sgmax, _itr) \ 330219820Sjeff for (_itr = 0, sg = (sglist); _itr < (sgmax); _itr++, sg = sg_next(sg)) 331219820Sjeff 332219820Sjeff#endif /* _LINUX_SCATTERLIST_H_ */ 333