1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _NET_PAGE_POOL_TYPES_H
4#define _NET_PAGE_POOL_TYPES_H
5
6#include <linux/dma-direction.h>
7#include <linux/ptr_ring.h>
8#include <linux/types.h>
9
10#define PP_FLAG_DMA_MAP		BIT(0) /* Should page_pool do the DMA
11					* map/unmap
12					*/
13#define PP_FLAG_DMA_SYNC_DEV	BIT(1) /* If set all pages that the driver gets
14					* from page_pool will be
15					* DMA-synced-for-device according to
16					* the length provided by the device
17					* driver.
18					* Please note DMA-sync-for-CPU is still
19					* device driver responsibility
20					*/
21#define PP_FLAG_SYSTEM_POOL	BIT(2) /* Global system page_pool */
22#define PP_FLAG_ALL		(PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | \
23				 PP_FLAG_SYSTEM_POOL)
24
25/*
26 * Fast allocation side cache array/stack
27 *
28 * The cache size and refill watermark is related to the network
29 * use-case.  The NAPI budget is 64 packets.  After a NAPI poll the RX
30 * ring is usually refilled and the max consumed elements will be 64,
31 * thus a natural max size of objects needed in the cache.
32 *
33 * Keeping room for more objects, is due to XDP_DROP use-case.  As
34 * XDP_DROP allows the opportunity to recycle objects directly into
35 * this array, as it shares the same softirq/NAPI protection.  If
36 * cache is already full (or partly full) then the XDP_DROP recycles
37 * would have to take a slower code path.
38 */
39#define PP_ALLOC_CACHE_SIZE	128
40#define PP_ALLOC_CACHE_REFILL	64
41struct pp_alloc_cache {
42	u32 count;
43	struct page *cache[PP_ALLOC_CACHE_SIZE];
44};
45
46/**
47 * struct page_pool_params - page pool parameters
48 * @flags:	PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV
49 * @order:	2^order pages on allocation
50 * @pool_size:	size of the ptr_ring
51 * @nid:	NUMA node id to allocate from pages from
52 * @dev:	device, for DMA pre-mapping purposes
53 * @netdev:	netdev this pool will serve (leave as NULL if none or multiple)
54 * @napi:	NAPI which is the sole consumer of pages, otherwise NULL
55 * @dma_dir:	DMA mapping direction
56 * @max_len:	max DMA sync memory size for PP_FLAG_DMA_SYNC_DEV
57 * @offset:	DMA sync address offset for PP_FLAG_DMA_SYNC_DEV
58 */
59struct page_pool_params {
60	struct_group_tagged(page_pool_params_fast, fast,
61		unsigned int	flags;
62		unsigned int	order;
63		unsigned int	pool_size;
64		int		nid;
65		struct device	*dev;
66		struct napi_struct *napi;
67		enum dma_data_direction dma_dir;
68		unsigned int	max_len;
69		unsigned int	offset;
70	);
71	struct_group_tagged(page_pool_params_slow, slow,
72		struct net_device *netdev;
73/* private: used by test code only */
74		void (*init_callback)(struct page *page, void *arg);
75		void *init_arg;
76	);
77};
78
79#ifdef CONFIG_PAGE_POOL_STATS
80/**
81 * struct page_pool_alloc_stats - allocation statistics
82 * @fast:	successful fast path allocations
83 * @slow:	slow path order-0 allocations
84 * @slow_high_order: slow path high order allocations
85 * @empty:	ptr ring is empty, so a slow path allocation was forced
86 * @refill:	an allocation which triggered a refill of the cache
87 * @waive:	pages obtained from the ptr ring that cannot be added to
88 *		the cache due to a NUMA mismatch
89 */
90struct page_pool_alloc_stats {
91	u64 fast;
92	u64 slow;
93	u64 slow_high_order;
94	u64 empty;
95	u64 refill;
96	u64 waive;
97};
98
99/**
100 * struct page_pool_recycle_stats - recycling (freeing) statistics
101 * @cached:	recycling placed page in the page pool cache
102 * @cache_full:	page pool cache was full
103 * @ring:	page placed into the ptr ring
104 * @ring_full:	page released from page pool because the ptr ring was full
105 * @released_refcnt:	page released (and not recycled) because refcnt > 1
106 */
107struct page_pool_recycle_stats {
108	u64 cached;
109	u64 cache_full;
110	u64 ring;
111	u64 ring_full;
112	u64 released_refcnt;
113};
114
115/**
116 * struct page_pool_stats - combined page pool use statistics
117 * @alloc_stats:	see struct page_pool_alloc_stats
118 * @recycle_stats:	see struct page_pool_recycle_stats
119 *
120 * Wrapper struct for combining page pool stats with different storage
121 * requirements.
122 */
123struct page_pool_stats {
124	struct page_pool_alloc_stats alloc_stats;
125	struct page_pool_recycle_stats recycle_stats;
126};
127#endif
128
129struct page_pool {
130	struct page_pool_params_fast p;
131
132	int cpuid;
133	bool has_init_callback;
134
135	long frag_users;
136	struct page *frag_page;
137	unsigned int frag_offset;
138	u32 pages_state_hold_cnt;
139
140	struct delayed_work release_dw;
141	void (*disconnect)(void *pool);
142	unsigned long defer_start;
143	unsigned long defer_warn;
144
145#ifdef CONFIG_PAGE_POOL_STATS
146	/* these stats are incremented while in softirq context */
147	struct page_pool_alloc_stats alloc_stats;
148#endif
149	u32 xdp_mem_id;
150
151	/*
152	 * Data structure for allocation side
153	 *
154	 * Drivers allocation side usually already perform some kind
155	 * of resource protection.  Piggyback on this protection, and
156	 * require driver to protect allocation side.
157	 *
158	 * For NIC drivers this means, allocate a page_pool per
159	 * RX-queue. As the RX-queue is already protected by
160	 * Softirq/BH scheduling and napi_schedule. NAPI schedule
161	 * guarantee that a single napi_struct will only be scheduled
162	 * on a single CPU (see napi_schedule).
163	 */
164	struct pp_alloc_cache alloc ____cacheline_aligned_in_smp;
165
166	/* Data structure for storing recycled pages.
167	 *
168	 * Returning/freeing pages is more complicated synchronization
169	 * wise, because free's can happen on remote CPUs, with no
170	 * association with allocation resource.
171	 *
172	 * Use ptr_ring, as it separates consumer and producer
173	 * efficiently, it a way that doesn't bounce cache-lines.
174	 *
175	 * TODO: Implement bulk return pages into this structure.
176	 */
177	struct ptr_ring ring;
178
179#ifdef CONFIG_PAGE_POOL_STATS
180	/* recycle stats are per-cpu to avoid locking */
181	struct page_pool_recycle_stats __percpu *recycle_stats;
182#endif
183	atomic_t pages_state_release_cnt;
184
185	/* A page_pool is strictly tied to a single RX-queue being
186	 * protected by NAPI, due to above pp_alloc_cache. This
187	 * refcnt serves purpose is to simplify drivers error handling.
188	 */
189	refcount_t user_cnt;
190
191	u64 destroy_cnt;
192
193	/* Slow/Control-path information follows */
194	struct page_pool_params_slow slow;
195	/* User-facing fields, protected by page_pools_lock */
196	struct {
197		struct hlist_node list;
198		u64 detach_time;
199		u32 napi_id;
200		u32 id;
201	} user;
202};
203
204struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
205struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
206				  unsigned int size, gfp_t gfp);
207struct page_pool *page_pool_create(const struct page_pool_params *params);
208struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,
209					  int cpuid);
210
211struct xdp_mem_info;
212
213#ifdef CONFIG_PAGE_POOL
214void page_pool_destroy(struct page_pool *pool);
215void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
216			   struct xdp_mem_info *mem);
217void page_pool_put_page_bulk(struct page_pool *pool, void **data,
218			     int count);
219#else
220static inline void page_pool_destroy(struct page_pool *pool)
221{
222}
223
224static inline void page_pool_use_xdp_mem(struct page_pool *pool,
225					 void (*disconnect)(void *),
226					 struct xdp_mem_info *mem)
227{
228}
229
230static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
231					   int count)
232{
233}
234#endif
235
236void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
237				unsigned int dma_sync_size,
238				bool allow_direct);
239
240static inline bool is_page_pool_compiled_in(void)
241{
242#ifdef CONFIG_PAGE_POOL
243	return true;
244#else
245	return false;
246#endif
247}
248
249/* Caller must provide appropriate safe context, e.g. NAPI. */
250void page_pool_update_nid(struct page_pool *pool, int new_nid);
251
252#endif /* _NET_PAGE_POOL_H */
253