1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef __LINUX_SWIOTLB_H 3#define __LINUX_SWIOTLB_H 4 5#include <linux/device.h> 6#include <linux/dma-direction.h> 7#include <linux/init.h> 8#include <linux/types.h> 9#include <linux/limits.h> 10#include <linux/spinlock.h> 11#include <linux/workqueue.h> 12 13struct device; 14struct page; 15struct scatterlist; 16 17#define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */ 18#define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */ 19#define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */ 20 21/* 22 * Maximum allowable number of contiguous slabs to map, 23 * must be a power of 2. What is the appropriate value ? 24 * The complexity of {map,unmap}_single is linearly dependent on this value. 25 */ 26#define IO_TLB_SEGSIZE 128 27 28/* 29 * log of the size of each IO TLB slab. The number of slabs is command line 30 * controllable. 31 */ 32#define IO_TLB_SHIFT 11 33#define IO_TLB_SIZE (1 << IO_TLB_SHIFT) 34 35/* default to 64MB */ 36#define IO_TLB_DEFAULT_SIZE (64UL<<20) 37 38unsigned long swiotlb_size_or_default(void); 39void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, 40 int (*remap)(void *tlb, unsigned long nslabs)); 41int swiotlb_init_late(size_t size, gfp_t gfp_mask, 42 int (*remap)(void *tlb, unsigned long nslabs)); 43extern void __init swiotlb_update_mem_attributes(void); 44 45phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, 46 size_t mapping_size, size_t alloc_size, 47 unsigned int alloc_aligned_mask, enum dma_data_direction dir, 48 unsigned long attrs); 49 50extern void swiotlb_tbl_unmap_single(struct device *hwdev, 51 phys_addr_t tlb_addr, 52 size_t mapping_size, 53 enum dma_data_direction dir, 54 unsigned long attrs); 55 56void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr, 57 size_t size, enum dma_data_direction dir); 58void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr, 59 size_t size, enum dma_data_direction dir); 60dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, 61 size_t size, enum dma_data_direction dir, unsigned long attrs); 62 63#ifdef CONFIG_SWIOTLB 64 65/** 66 * struct io_tlb_pool - IO TLB memory pool descriptor 67 * @start: The start address of the swiotlb memory pool. Used to do a quick 68 * range check to see if the memory was in fact allocated by this 69 * API. 70 * @end: The end address of the swiotlb memory pool. Used to do a quick 71 * range check to see if the memory was in fact allocated by this 72 * API. 73 * @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool 74 * may be remapped in the memory encrypted case and store virtual 75 * address for bounce buffer operation. 76 * @nslabs: The number of IO TLB slots between @start and @end. For the 77 * default swiotlb, this can be adjusted with a boot parameter, 78 * see setup_io_tlb_npages(). 79 * @late_alloc: %true if allocated using the page allocator. 80 * @nareas: Number of areas in the pool. 81 * @area_nslabs: Number of slots in each area. 82 * @areas: Array of memory area descriptors. 83 * @slots: Array of slot descriptors. 84 * @node: Member of the IO TLB memory pool list. 85 * @rcu: RCU head for swiotlb_dyn_free(). 86 * @transient: %true if transient memory pool. 87 */ 88struct io_tlb_pool { 89 phys_addr_t start; 90 phys_addr_t end; 91 void *vaddr; 92 unsigned long nslabs; 93 bool late_alloc; 94 unsigned int nareas; 95 unsigned int area_nslabs; 96 struct io_tlb_area *areas; 97 struct io_tlb_slot *slots; 98#ifdef CONFIG_SWIOTLB_DYNAMIC 99 struct list_head node; 100 struct rcu_head rcu; 101 bool transient; 102#endif 103}; 104 105/** 106 * struct io_tlb_mem - Software IO TLB allocator 107 * @defpool: Default (initial) IO TLB memory pool descriptor. 108 * @pool: IO TLB memory pool descriptor (if not dynamic). 109 * @nslabs: Total number of IO TLB slabs in all pools. 110 * @debugfs: The dentry to debugfs. 111 * @force_bounce: %true if swiotlb bouncing is forced 112 * @for_alloc: %true if the pool is used for memory allocation 113 * @can_grow: %true if more pools can be allocated dynamically. 114 * @phys_limit: Maximum allowed physical address. 115 * @lock: Lock to synchronize changes to the list. 116 * @pools: List of IO TLB memory pool descriptors (if dynamic). 117 * @dyn_alloc: Dynamic IO TLB pool allocation work. 118 * @total_used: The total number of slots in the pool that are currently used 119 * across all areas. Used only for calculating used_hiwater in 120 * debugfs. 121 * @used_hiwater: The high water mark for total_used. Used only for reporting 122 * in debugfs. 123 * @transient_nslabs: The total number of slots in all transient pools that 124 * are currently used across all areas. 125 */ 126struct io_tlb_mem { 127 struct io_tlb_pool defpool; 128 unsigned long nslabs; 129 struct dentry *debugfs; 130 bool force_bounce; 131 bool for_alloc; 132#ifdef CONFIG_SWIOTLB_DYNAMIC 133 bool can_grow; 134 u64 phys_limit; 135 spinlock_t lock; 136 struct list_head pools; 137 struct work_struct dyn_alloc; 138#endif 139#ifdef CONFIG_DEBUG_FS 140 atomic_long_t total_used; 141 atomic_long_t used_hiwater; 142 atomic_long_t transient_nslabs; 143#endif 144}; 145 146#ifdef CONFIG_SWIOTLB_DYNAMIC 147 148struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr); 149 150#else 151 152static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev, 153 phys_addr_t paddr) 154{ 155 return &dev->dma_io_tlb_mem->defpool; 156} 157 158#endif 159 160/** 161 * is_swiotlb_buffer() - check if a physical address belongs to a swiotlb 162 * @dev: Device which has mapped the buffer. 163 * @paddr: Physical address within the DMA buffer. 164 * 165 * Check if @paddr points into a bounce buffer. 166 * 167 * Return: 168 * * %true if @paddr points into a bounce buffer 169 * * %false otherwise 170 */ 171static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) 172{ 173 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; 174 175 if (!mem) 176 return false; 177 178#ifdef CONFIG_SWIOTLB_DYNAMIC 179 /* 180 * All SWIOTLB buffer addresses must have been returned by 181 * swiotlb_tbl_map_single() and passed to a device driver. 182 * If a SWIOTLB address is checked on another CPU, then it was 183 * presumably loaded by the device driver from an unspecified private 184 * data structure. Make sure that this load is ordered before reading 185 * dev->dma_uses_io_tlb here and mem->pools in swiotlb_find_pool(). 186 * 187 * This barrier pairs with smp_mb() in swiotlb_find_slots(). 188 */ 189 smp_rmb(); 190 return READ_ONCE(dev->dma_uses_io_tlb) && 191 swiotlb_find_pool(dev, paddr); 192#else 193 return paddr >= mem->defpool.start && paddr < mem->defpool.end; 194#endif 195} 196 197static inline bool is_swiotlb_force_bounce(struct device *dev) 198{ 199 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; 200 201 return mem && mem->force_bounce; 202} 203 204void swiotlb_init(bool addressing_limited, unsigned int flags); 205void __init swiotlb_exit(void); 206void swiotlb_dev_init(struct device *dev); 207size_t swiotlb_max_mapping_size(struct device *dev); 208bool is_swiotlb_allocated(void); 209bool is_swiotlb_active(struct device *dev); 210void __init swiotlb_adjust_size(unsigned long size); 211phys_addr_t default_swiotlb_base(void); 212phys_addr_t default_swiotlb_limit(void); 213#else 214static inline void swiotlb_init(bool addressing_limited, unsigned int flags) 215{ 216} 217 218static inline void swiotlb_dev_init(struct device *dev) 219{ 220} 221 222static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) 223{ 224 return false; 225} 226static inline bool is_swiotlb_force_bounce(struct device *dev) 227{ 228 return false; 229} 230static inline void swiotlb_exit(void) 231{ 232} 233static inline size_t swiotlb_max_mapping_size(struct device *dev) 234{ 235 return SIZE_MAX; 236} 237 238static inline bool is_swiotlb_allocated(void) 239{ 240 return false; 241} 242 243static inline bool is_swiotlb_active(struct device *dev) 244{ 245 return false; 246} 247 248static inline void swiotlb_adjust_size(unsigned long size) 249{ 250} 251 252static inline phys_addr_t default_swiotlb_base(void) 253{ 254 return 0; 255} 256 257static inline phys_addr_t default_swiotlb_limit(void) 258{ 259 return 0; 260} 261#endif /* CONFIG_SWIOTLB */ 262 263extern void swiotlb_print_info(void); 264 265#ifdef CONFIG_DMA_RESTRICTED_POOL 266struct page *swiotlb_alloc(struct device *dev, size_t size); 267bool swiotlb_free(struct device *dev, struct page *page, size_t size); 268 269static inline bool is_swiotlb_for_alloc(struct device *dev) 270{ 271 return dev->dma_io_tlb_mem->for_alloc; 272} 273#else 274static inline struct page *swiotlb_alloc(struct device *dev, size_t size) 275{ 276 return NULL; 277} 278static inline bool swiotlb_free(struct device *dev, struct page *page, 279 size_t size) 280{ 281 return false; 282} 283static inline bool is_swiotlb_for_alloc(struct device *dev) 284{ 285 return false; 286} 287#endif /* CONFIG_DMA_RESTRICTED_POOL */ 288 289#endif /* __LINUX_SWIOTLB_H */ 290