1/*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian K��nig
23 */
24
25#ifndef _TTM_RESOURCE_H_
26#define _TTM_RESOURCE_H_
27
28#include <linux/types.h>
29#include <linux/list.h>
30#include <linux/mutex.h>
31#include <linux/iosys-map.h>
32#include <linux/dma-fence.h>
33
34#include <drm/drm_print.h>
35#include <drm/ttm/ttm_caching.h>
36#include <drm/ttm/ttm_kmap_iter.h>
37
38#define TTM_MAX_BO_PRIORITY	4U
39#define TTM_NUM_MEM_TYPES 8
40
41struct ttm_device;
42struct ttm_resource_manager;
43struct ttm_resource;
44struct ttm_place;
45struct ttm_buffer_object;
46struct ttm_placement;
47struct iosys_map;
48struct io_mapping;
49struct sg_table;
50struct scatterlist;
51
52struct ttm_resource_manager_func {
53	/**
54	 * struct ttm_resource_manager_func member alloc
55	 *
56	 * @man: Pointer to a memory type manager.
57	 * @bo: Pointer to the buffer object we're allocating space for.
58	 * @place: Placement details.
59	 * @res: Resulting pointer to the ttm_resource.
60	 *
61	 * This function should allocate space in the memory type managed
62	 * by @man. Placement details if applicable are given by @place. If
63	 * successful, a filled in ttm_resource object should be returned in
64	 * @res. @res::start should be set to a value identifying the beginning
65	 * of the range allocated, and the function should return zero.
66	 * If the manager can't fulfill the request -ENOSPC should be returned.
67	 * If a system error occurred, preventing the request to be fulfilled,
68	 * the function should return a negative error code.
69	 *
70	 * This function may not be called from within atomic context and needs
71	 * to take care of its own locking to protect any data structures
72	 * managing the space.
73	 */
74	int  (*alloc)(struct ttm_resource_manager *man,
75		      struct ttm_buffer_object *bo,
76		      const struct ttm_place *place,
77		      struct ttm_resource **res);
78
79	/**
80	 * struct ttm_resource_manager_func member free
81	 *
82	 * @man: Pointer to a memory type manager.
83	 * @res: Pointer to a struct ttm_resource to be freed.
84	 *
85	 * This function frees memory type resources previously allocated.
86	 * May not be called from within atomic context.
87	 */
88	void (*free)(struct ttm_resource_manager *man,
89		     struct ttm_resource *res);
90
91	/**
92	 * struct ttm_resource_manager_func member intersects
93	 *
94	 * @man: Pointer to a memory type manager.
95	 * @res: Pointer to a struct ttm_resource to be checked.
96	 * @place: Placement to check against.
97	 * @size: Size of the check.
98	 *
99	 * Test if @res intersects with @place + @size. Used to judge if
100	 * evictions are valueable or not.
101	 */
102	bool (*intersects)(struct ttm_resource_manager *man,
103			   struct ttm_resource *res,
104			   const struct ttm_place *place,
105			   size_t size);
106
107	/**
108	 * struct ttm_resource_manager_func member compatible
109	 *
110	 * @man: Pointer to a memory type manager.
111	 * @res: Pointer to a struct ttm_resource to be checked.
112	 * @place: Placement to check against.
113	 * @size: Size of the check.
114	 *
115	 * Test if @res compatible with @place + @size. Used to check of
116	 * the need to move the backing store or not.
117	 */
118	bool (*compatible)(struct ttm_resource_manager *man,
119			   struct ttm_resource *res,
120			   const struct ttm_place *place,
121			   size_t size);
122
123	/**
124	 * struct ttm_resource_manager_func member debug
125	 *
126	 * @man: Pointer to a memory type manager.
127	 * @printer: Prefix to be used in printout to identify the caller.
128	 *
129	 * This function is called to print out the state of the memory
130	 * type manager to aid debugging of out-of-memory conditions.
131	 * It may not be called from within atomic context.
132	 */
133	void (*debug)(struct ttm_resource_manager *man,
134		      struct drm_printer *printer);
135};
136
137/**
138 * struct ttm_resource_manager
139 *
140 * @use_type: The memory type is enabled.
141 * @use_tt: If a TT object should be used for the backing store.
142 * @size: Size of the managed region.
143 * @bdev: ttm device this manager belongs to
144 * @func: structure pointer implementing the range manager. See above
145 * @move_lock: lock for move fence
146 * @move: The fence of the last pipelined move operation.
147 * @lru: The lru list for this memory type.
148 *
149 * This structure is used to identify and manage memory types for a device.
150 */
151struct ttm_resource_manager {
152	/*
153	 * No protection. Constant from start.
154	 */
155	bool use_type;
156	bool use_tt;
157	struct ttm_device *bdev;
158	uint64_t size;
159	const struct ttm_resource_manager_func *func;
160	spinlock_t move_lock;
161
162	/*
163	 * Protected by @move_lock.
164	 */
165	struct dma_fence *move;
166
167	/*
168	 * Protected by the bdev->lru_lock.
169	 */
170	struct list_head lru[TTM_MAX_BO_PRIORITY];
171
172	/**
173	 * @usage: How much of the resources are used, protected by the
174	 * bdev->lru_lock.
175	 */
176	uint64_t usage;
177};
178
179/**
180 * struct ttm_bus_placement
181 *
182 * @addr:		mapped virtual address
183 * @offset:		physical addr
184 * @is_iomem:		is this io memory ?
185 * @caching:		See enum ttm_caching
186 *
187 * Structure indicating the bus placement of an object.
188 */
189struct ttm_bus_placement {
190	void			*addr;
191	phys_addr_t		offset;
192	bool			is_iomem;
193	enum ttm_caching	caching;
194};
195
196/**
197 * struct ttm_resource
198 *
199 * @start: Start of the allocation.
200 * @size: Actual size of resource in bytes.
201 * @mem_type: Resource type of the allocation.
202 * @placement: Placement flags.
203 * @bus: Placement on io bus accessible to the CPU
204 * @bo: weak reference to the BO, protected by ttm_device::lru_lock
205 *
206 * Structure indicating the placement and space resources used by a
207 * buffer object.
208 */
209struct ttm_resource {
210	unsigned long start;
211	size_t size;
212	uint32_t mem_type;
213	uint32_t placement;
214	struct ttm_bus_placement bus;
215	struct ttm_buffer_object *bo;
216
217	/**
218	 * @lru: Least recently used list, see &ttm_resource_manager.lru
219	 */
220	struct list_head lru;
221};
222
223/**
224 * struct ttm_resource_cursor
225 *
226 * @priority: the current priority
227 *
228 * Cursor to iterate over the resources in a manager.
229 */
230struct ttm_resource_cursor {
231	unsigned int priority;
232};
233
234/**
235 * struct ttm_lru_bulk_move_pos
236 *
237 * @first: first res in the bulk move range
238 * @last: last res in the bulk move range
239 *
240 * Range of resources for a lru bulk move.
241 */
242struct ttm_lru_bulk_move_pos {
243	struct ttm_resource *first;
244	struct ttm_resource *last;
245};
246
247/**
248 * struct ttm_lru_bulk_move
249 *
250 * @pos: first/last lru entry for resources in the each domain/priority
251 *
252 * Container for the current bulk move state. Should be used with
253 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
254 */
255struct ttm_lru_bulk_move {
256	struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
257};
258
259/**
260 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
261 * struct sg_table backed struct ttm_resource.
262 * @base: Embedded struct ttm_kmap_iter providing the usage interface.
263 * @iomap: struct io_mapping representing the underlying linear io_memory.
264 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
265 * @start: Offset that needs to be subtracted from @st to make
266 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
267 * @cache: Scatterlist traversal cache for fast lookups.
268 * @cache.sg: Pointer to the currently cached scatterlist segment.
269 * @cache.i: First index of @sg. PAGE_SIZE granularity.
270 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
271 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
272 */
273struct ttm_kmap_iter_iomap {
274	struct ttm_kmap_iter base;
275	struct io_mapping *iomap;
276	struct sg_table *st;
277	resource_size_t start;
278	struct {
279		struct scatterlist *sg;
280		pgoff_t i;
281		pgoff_t end;
282		pgoff_t offs;
283	} cache;
284};
285
286/**
287 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
288 * @base: The base iterator
289 * @dmap: Points to the starting address of the region
290 * @needs_unmap: Whether we need to unmap on fini
291 */
292struct ttm_kmap_iter_linear_io {
293	struct ttm_kmap_iter base;
294	struct iosys_map dmap;
295	bool needs_unmap;
296};
297
298/**
299 * ttm_resource_manager_set_used
300 *
301 * @man: A memory manager object.
302 * @used: usage state to set.
303 *
304 * Set the manager in use flag. If disabled the manager is no longer
305 * used for object placement.
306 */
307static inline void
308ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)
309{
310	int i;
311
312	for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)
313		WARN_ON(!list_empty(&man->lru[i]));
314	man->use_type = used;
315}
316
317/**
318 * ttm_resource_manager_used
319 *
320 * @man: Manager to get used state for
321 *
322 * Get the in use flag for a manager.
323 * Returns:
324 * true is used, false if not.
325 */
326static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)
327{
328	return man->use_type;
329}
330
331/**
332 * ttm_resource_manager_cleanup
333 *
334 * @man: A memory manager object.
335 *
336 * Cleanup the move fences from the memory manager object.
337 */
338static inline void
339ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
340{
341	dma_fence_put(man->move);
342	man->move = NULL;
343}
344
345void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
346void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
347
348void ttm_resource_add_bulk_move(struct ttm_resource *res,
349				struct ttm_buffer_object *bo);
350void ttm_resource_del_bulk_move(struct ttm_resource *res,
351				struct ttm_buffer_object *bo);
352void ttm_resource_move_to_lru_tail(struct ttm_resource *res);
353
354void ttm_resource_init(struct ttm_buffer_object *bo,
355                       const struct ttm_place *place,
356                       struct ttm_resource *res);
357void ttm_resource_fini(struct ttm_resource_manager *man,
358		       struct ttm_resource *res);
359
360int ttm_resource_alloc(struct ttm_buffer_object *bo,
361		       const struct ttm_place *place,
362		       struct ttm_resource **res);
363void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
364bool ttm_resource_intersects(struct ttm_device *bdev,
365			     struct ttm_resource *res,
366			     const struct ttm_place *place,
367			     size_t size);
368bool ttm_resource_compatible(struct ttm_resource *res,
369			     struct ttm_placement *placement);
370void ttm_resource_set_bo(struct ttm_resource *res,
371			 struct ttm_buffer_object *bo);
372
373void ttm_resource_manager_init(struct ttm_resource_manager *man,
374			       struct ttm_device *bdev,
375			       uint64_t size);
376
377int ttm_resource_manager_evict_all(struct ttm_device *bdev,
378				   struct ttm_resource_manager *man);
379
380uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);
381void ttm_resource_manager_debug(struct ttm_resource_manager *man,
382				struct drm_printer *p);
383
384struct ttm_resource *
385ttm_resource_manager_first(struct ttm_resource_manager *man,
386			   struct ttm_resource_cursor *cursor);
387struct ttm_resource *
388ttm_resource_manager_next(struct ttm_resource_manager *man,
389			  struct ttm_resource_cursor *cursor,
390			  struct ttm_resource *res);
391
392/**
393 * ttm_resource_manager_for_each_res - iterate over all resources
394 * @man: the resource manager
395 * @cursor: struct ttm_resource_cursor for the current position
396 * @res: the current resource
397 *
398 * Iterate over all the evictable resources in a resource manager.
399 */
400#define ttm_resource_manager_for_each_res(man, cursor, res)		\
401	for (res = ttm_resource_manager_first(man, cursor); res;	\
402	     res = ttm_resource_manager_next(man, cursor, res))
403
404struct ttm_kmap_iter *
405ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
406			 struct io_mapping *iomap,
407			 struct sg_table *st,
408			 resource_size_t start);
409
410struct ttm_kmap_iter_linear_io;
411
412struct ttm_kmap_iter *
413ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
414			     struct ttm_device *bdev,
415			     struct ttm_resource *mem);
416
417void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
418				  struct ttm_device *bdev,
419				  struct ttm_resource *mem);
420
421void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
422					 struct dentry * parent,
423					 const char *name);
424#endif
425