include/linux/scatterlist.h

27  * You should only work with the number of sg entries dma_map_sg
28  * returns, or alternatively stop on the first sg_dma_len(sg) which
31 #define sg_dma_address(sg)	((sg)->dma_address)
34 #define sg_dma_len(sg)		((sg)->dma_length)
36 #define sg_dma_len(sg)		((sg)->length)
55  * the page pointer AND encode information about the sg table as well. The two
58  * If bit 0 is set, then the page_link contains a pointer to the next sg
59  * table list. Otherwise the next entry is at sg + 1.
61  * If bit 1 is set, then this sg entry is the last element in a list.
72  * a valid sg entry, or whether it points to the start of a new scatterlist.
77 static inline unsigned int __sg_flags(struct scatterlist *sg)
79 	return sg->page_link & SG_PAGE_LINK_MASK;
82 static inline struct scatterlist *sg_chain_ptr(struct scatterlist *sg)
84 	return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK);
87 static inline bool sg_is_chain(struct scatterlist *sg)
89 	return __sg_flags(sg) & SG_CHAIN;
92 static inline bool sg_is_last(struct scatterlist *sg)
94 	return __sg_flags(sg) & SG_END;
99  * @sg:		    SG entry
103  *   Assign page to sg entry. Also see sg_set_page(), the most commonly used
107 static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
109 	unsigned long page_link = sg->page_link & (SG_CHAIN | SG_END);
117 	BUG_ON(sg_is_chain(sg));
119 	sg->page_link = page_link | (unsigned long) page;
123  * sg_set_page - Set sg entry to point at given page
124  * @sg:		 SG entry
130  *   Use this function to set an sg entry pointing at a page, never assign
131  *   the page directly. We encode sg table information in the lower bits
133  *   to an sg entry.
136 static inline void sg_set_page(struct scatterlist *sg, struct page *page,
139 	sg_assign_page(sg, page);
140 	sg->offset = offset;
141 	sg->length = len;
145  * sg_set_folio - Set sg entry to point at given folio
146  * @sg:		 SG entry
152  *   Use this function to set an sg entry pointing at a folio, never assign
153  *   the folio directly. We encode sg table information in the lower bits
155  *   to an sg entry.
158 static inline void sg_set_folio(struct scatterlist *sg, struct folio *folio,
163 	sg_assign_page(sg, &folio->page);
164 	sg->offset = offset;
165 	sg->length = len;
168 static inline struct page *sg_page(struct scatterlist *sg)
171 	BUG_ON(sg_is_chain(sg));
173 	return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK);
177  * sg_set_buf - Set sg entry to point at given data
178  * @sg:		 SG entry
183 static inline void sg_set_buf(struct scatterlist *sg, const void *buf,
189 	sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));
193  * Loop over each sg element, following the pointer to a new list if necessary
195 #define for_each_sg(sglist, sg, nr, __i)	\
196 	for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))
199  * Loop over each sg element in the given sg_table object.
201 #define for_each_sgtable_sg(sgt, sg, i)		\
202 	for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i)
205  * Loop over each sg element in the given *DMA mapped* sg_table object.
206  * Please use sg_dma_address(sg) and sg_dma_len(sg) to extract DMA addresses
209 #define for_each_sgtable_dma_sg(sgt, sg, i)	\
210 	for_each_sg((sgt)->sgl, sg, (sgt)->nents, i)
246  * @sg:		 SG entryScatterlist
249  *   Marks the passed in sg entry as the termination point for the sg
253 static inline void sg_mark_end(struct scatterlist *sg)
258 	sg->page_link |= SG_END;
259 	sg->page_link &= ~SG_CHAIN;
264  * @sg:		 SG entryScatterlist
270 static inline void sg_unmark_end(struct scatterlist *sg)
272 	sg->page_link &= ~SG_END;
289  * @sg:		 SG entry
295 static inline bool sg_dma_is_bus_address(struct scatterlist *sg)
297 	return sg->dma_flags & SG_DMA_BUS_ADDRESS;
302  * @sg:		 SG entry
305  *   Marks the passed in sg entry to indicate that the dma_address is
310 static inline void sg_dma_mark_bus_address(struct scatterlist *sg)
312 	sg->dma_flags |= SG_DMA_BUS_ADDRESS;
317  * @sg:		 SG entry
322 static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)
324 	sg->dma_flags &= ~SG_DMA_BUS_ADDRESS;
330  * @sg:		SG entry
337 static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)
339 	return sg->dma_flags & SG_DMA_SWIOTLB;
344  * @sg:		SG entry
350 static inline void sg_dma_mark_swiotlb(struct scatterlist *sg)
352 	sg->dma_flags |= SG_DMA_SWIOTLB;
357 static inline bool sg_dma_is_bus_address(struct scatterlist *sg)
361 static inline void sg_dma_mark_bus_address(struct scatterlist *sg)
364 static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)
367 static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)
371 static inline void sg_dma_mark_swiotlb(struct scatterlist *sg)
378  * sg_phys - Return physical address of an sg entry
379  * @sg:	     SG entry
382  *   This calls page_to_phys() on the page in this sg entry, and adds the
383  *   sg offset. The caller must know that it is legal to call page_to_phys()
384  *   on the sg page.
387 static inline dma_addr_t sg_phys(struct scatterlist *sg)
389 	return page_to_phys(sg_page(sg)) + sg->offset;
393  * sg_virt - Return virtual address of an sg entry
394  * @sg:      SG entry
397  *   This calls page_address() on the page in this sg entry, and adds the
398  *   sg offset. The caller must know that the sg page has a valid virtual
402 static inline void *sg_virt(struct scatterlist *sg)
404 	return page_address(sg_page(sg)) + sg->offset;
408  * sg_init_marker - Initialize markers in sg table
419 int sg_nents(struct scatterlist *sg);
420 int sg_nents_for_len(struct scatterlist *sg, u64 len);
452  * sg_alloc_table_from_pages - Allocate and initialize an sg table from
454  * @sgt:	 The sg table header to use
462  *    Allocate and initialize an sg table from a list of pages. Contiguous
465  *    specified by the page array. The returned sg table is released by
523  * Like SG_CHUNK_SIZE, but for archs that have sg chaining. This limit
541  * sg page iterator
543  * Iterates over sg entries page-by-page.  On each successful iteration, you
545  * @piter->sg will point to the sg holding this page and @piter->sg_pgoffset to
546  * the page's page offset within the sg. The iteration will stop either when a
547  * maximum number of sg entries was reached or a terminating sg
548  * (sg_last(sg) == true) was reached.
551 	struct scatterlist	*sg;		/* sg holding the page */
552 	unsigned int		sg_pgoffset;	/* page offset within the sg */
555 	unsigned int		__nents;	/* remaining sg entries */
561  * sg page iterator for DMA addresses
582 	return nth_page(sg_page(piter->sg), piter->sg_pgoffset);
593 	return sg_dma_address(dma_iter->base.sg) +
598  * for_each_sg_page - iterate over the pages of the given sg list
600  * @piter:	page iterator to hold current page, sg, sg_pgoffset
601  * @nents:	maximum number of sg entries to iterate over
612  * for_each_sg_dma_page - iterate over the pages of the given sg list
615  * @dma_nents:	maximum number of sg entries to iterate over, this is the value
656  * Mapping sg iterator
658  * Iterates over sg entries mapping page-by-page.  On each successful