1// SPDX-License-Identifier: MIT
2/*
3 * Copyright �� 2023 Intel Corporation
4 */
5
6#include <linux/align.h>
7
8#include <drm/drm_managed.h>
9
10#include "regs/xe_sriov_regs.h"
11
12#include "xe_assert.h"
13#include "xe_bo.h"
14#include "xe_lmtt.h"
15#include "xe_map.h"
16#include "xe_mmio.h"
17#include "xe_res_cursor.h"
18#include "xe_sriov.h"
19#include "xe_sriov_printk.h"
20
21/**
22 * DOC: Local Memory Translation Table
23 *
24 * The Local Memory Translation Table (LMTT) provides additional abstraction
25 * when Virtual Function (VF) is accessing device Local Memory (VRAM).
26 *
27 * The Root LMTT Page Directory contains one entry for each VF. Entries are
28 * indexed by the function number (1-based, index 0 is unused).
29 *
30 * See `Two-Level LMTT Structure`_ and `Multi-Level LMTT Structure`_.
31 */
32
33#define lmtt_assert(lmtt, condition)	xe_tile_assert(lmtt_to_tile(lmtt), condition)
34#define lmtt_debug(lmtt, msg...)	xe_sriov_dbg_verbose(lmtt_to_xe(lmtt), "LMTT: " msg)
35
36static bool xe_has_multi_level_lmtt(struct xe_device *xe)
37{
38	return xe->info.platform == XE_PVC;
39}
40
41static struct xe_tile *lmtt_to_tile(struct xe_lmtt *lmtt)
42{
43	return container_of(lmtt, struct xe_tile, sriov.pf.lmtt);
44}
45
46static struct xe_device *lmtt_to_xe(struct xe_lmtt *lmtt)
47{
48	return tile_to_xe(lmtt_to_tile(lmtt));
49}
50
51static u64 lmtt_page_size(struct xe_lmtt *lmtt)
52{
53	return BIT_ULL(lmtt->ops->lmtt_pte_shift(0));
54}
55
56static struct xe_lmtt_pt *lmtt_pt_alloc(struct xe_lmtt *lmtt, unsigned int level)
57{
58	unsigned int num_entries = level ? lmtt->ops->lmtt_pte_num(level) : 0;
59	struct xe_lmtt_pt *pt;
60	struct xe_bo *bo;
61	int err;
62
63	pt = kzalloc(struct_size(pt, entries, num_entries), GFP_KERNEL);
64	if (!pt) {
65		err = -ENOMEM;
66		goto out;
67	}
68
69	bo = xe_bo_create_pin_map(lmtt_to_xe(lmtt), lmtt_to_tile(lmtt), NULL,
70				  PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *
71					     lmtt->ops->lmtt_pte_num(level)),
72				  ttm_bo_type_kernel,
73				  XE_BO_CREATE_VRAM_IF_DGFX(lmtt_to_tile(lmtt)) |
74				  XE_BO_CREATE_PINNED_BIT);
75	if (IS_ERR(bo)) {
76		err = PTR_ERR(bo);
77		goto out_free_pt;
78	}
79
80	lmtt_assert(lmtt, xe_bo_is_vram(bo));
81
82	pt->level = level;
83	pt->bo = bo;
84	return pt;
85
86out_free_pt:
87	kfree(pt);
88out:
89	return ERR_PTR(err);
90}
91
92static void lmtt_pt_free(struct xe_lmtt_pt *pt)
93{
94	xe_bo_unpin_map_no_vm(pt->bo);
95	kfree(pt);
96}
97
98static int lmtt_init_pd(struct xe_lmtt *lmtt)
99{
100	struct xe_lmtt_pt *pd;
101
102	lmtt_assert(lmtt, !lmtt->pd);
103	lmtt_assert(lmtt, lmtt->ops->lmtt_root_pd_level());
104
105	pd = lmtt_pt_alloc(lmtt, lmtt->ops->lmtt_root_pd_level());
106	if (IS_ERR(pd))
107		return PTR_ERR(pd);
108
109	lmtt->pd = pd;
110	return 0;
111}
112
113static void lmtt_fini_pd(struct xe_lmtt *lmtt)
114{
115	struct xe_lmtt_pt *pd = lmtt->pd;
116	unsigned int num_entries = lmtt->ops->lmtt_pte_num(pd->level);
117	unsigned int n = 0;
118
119	/* make sure we don't leak */
120	for (n = 0; n < num_entries; n++)
121		lmtt_assert(lmtt, !pd->entries[n]);
122
123	lmtt->pd = NULL;
124	lmtt_pt_free(pd);
125}
126
127static void fini_lmtt(struct drm_device *drm, void *arg)
128{
129	struct xe_lmtt *lmtt = arg;
130
131	lmtt_assert(lmtt, !(!!lmtt->ops ^ !!lmtt->pd));
132
133	if (!lmtt->pd)
134		return;
135
136	lmtt_fini_pd(lmtt);
137	lmtt->ops = NULL;
138}
139
140/**
141 * xe_lmtt_init - LMTT software initialization.
142 * @lmtt: the &xe_lmtt to initialize
143 *
144 * The LMTT initialization requires two steps.
145 *
146 * The xe_lmtt_init() checks if LMTT is required on current device and selects
147 * and initialize proper variant of the LMTT Root Directory. Currently supported
148 * variants are `Two-Level LMTT Structure`_ and `Multi-Level LMTT Structure`_.
149 *
150 * In next step xe_lmtt_init_hw() will register this directory on the hardware.
151 *
152 * Notes:
153 * The LMTT allocations are managed and will be implicitly released on driver unload.
154 * This function shall be called only once and only when running as a PF driver.
155 * Any LMTT initialization failure should block VFs enabling.
156 *
157 * Return: 0 on success or a negative error code on failure.
158 */
159int xe_lmtt_init(struct xe_lmtt *lmtt)
160{
161	struct xe_device *xe = lmtt_to_xe(lmtt);
162	int err;
163
164	lmtt_assert(lmtt, IS_SRIOV_PF(xe));
165	lmtt_assert(lmtt, !lmtt->ops);
166
167	if (!IS_DGFX(xe))
168		return 0;
169
170	if (xe_has_multi_level_lmtt(xe))
171		lmtt->ops = &lmtt_ml_ops;
172	else
173		lmtt->ops = &lmtt_2l_ops;
174
175	err = lmtt_init_pd(lmtt);
176	if (unlikely(err))
177		goto fail;
178
179	return drmm_add_action_or_reset(&xe->drm, fini_lmtt, lmtt);
180
181fail:
182	lmtt->ops = NULL;
183	return err;
184}
185
186static void lmtt_setup_dir_ptr(struct xe_lmtt *lmtt)
187{
188	struct xe_tile *tile = lmtt_to_tile(lmtt);
189	struct xe_device *xe = tile_to_xe(tile);
190	dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE);
191
192	lmtt_debug(lmtt, "DIR offset %pad\n", &offset);
193	lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo));
194	lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K));
195
196	xe_mmio_write32(tile->primary_gt,
197			GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG,
198			LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K));
199}
200
201/**
202 * xe_lmtt_init_hw - Perform LMTT hardware initialization.
203 * @lmtt: the &xe_lmtt to initialize
204 *
205 * This function is a second step of the LMTT initialization.
206 * This function registers LMTT Root Directory prepared in xe_lmtt_init().
207 *
208 * This function shall be called after every hardware reset.
209 * This function shall be called only when running as a PF driver.
210 */
211void xe_lmtt_init_hw(struct xe_lmtt *lmtt)
212{
213	if (!lmtt->pd)
214		return;
215
216	lmtt_setup_dir_ptr(lmtt);
217}
218
219static void lmtt_write_pte(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pt,
220			   u64 pte, unsigned int idx)
221{
222	unsigned int level = pt->level;
223
224	lmtt_assert(lmtt, idx <= lmtt->ops->lmtt_pte_num(level));
225	lmtt_debug(lmtt, "WRITE level=%u index=%u pte=%#llx\n", level, idx, pte);
226
227	switch (lmtt->ops->lmtt_pte_size(level)) {
228	case sizeof(u32):
229		xe_map_wr(lmtt_to_xe(lmtt), &pt->bo->vmap, idx * sizeof(u32), u32, pte);
230		break;
231	case sizeof(u64):
232		xe_map_wr(lmtt_to_xe(lmtt), &pt->bo->vmap, idx * sizeof(u64), u64, pte);
233		break;
234	default:
235		lmtt_assert(lmtt, !!!"invalid pte size");
236	}
237}
238
239static void lmtt_destroy_pt(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pd)
240{
241	unsigned int num_entries = pd->level ? lmtt->ops->lmtt_pte_num(pd->level) : 0;
242	struct xe_lmtt_pt *pt;
243	unsigned int i;
244
245	for (i = 0; i < num_entries; i++) {
246		pt = pd->entries[i];
247		pd->entries[i] = NULL;
248		if (!pt)
249			continue;
250
251		lmtt_destroy_pt(lmtt, pt);
252	}
253
254	lmtt_pt_free(pd);
255}
256
257static void lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid)
258{
259	struct xe_lmtt_pt *pd = lmtt->pd;
260	struct xe_lmtt_pt *pt;
261
262	pt = pd->entries[vfid];
263	pd->entries[vfid] = NULL;
264	if (!pt)
265		return;
266
267	lmtt_write_pte(lmtt, pd, LMTT_PTE_INVALID, vfid);
268
269	lmtt_assert(lmtt, pd->level > 0);
270	lmtt_assert(lmtt, pt->level == pd->level - 1);
271	lmtt_destroy_pt(lmtt, pt);
272}
273
274static int __lmtt_alloc_range(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pd,
275			      u64 start, u64 end)
276{
277	u64 pte_addr_shift = BIT_ULL(lmtt->ops->lmtt_pte_shift(pd->level));
278	u64 offset;
279	int err;
280
281	lmtt_assert(lmtt, pd->level > 0);
282
283	offset = start;
284	while (offset < end) {
285		struct xe_lmtt_pt *pt;
286		u64 next, pde, pt_addr;
287		unsigned int idx;
288
289		pt = lmtt_pt_alloc(lmtt, pd->level - 1);
290		if (IS_ERR(pt))
291			return PTR_ERR(pt);
292
293		pt_addr = xe_bo_main_addr(pt->bo, XE_PAGE_SIZE);
294
295		idx = lmtt->ops->lmtt_pte_index(offset, pd->level);
296		pde = lmtt->ops->lmtt_pte_encode(pt_addr, pd->level);
297
298		lmtt_write_pte(lmtt, pd, pde, idx);
299
300		pd->entries[idx] = pt;
301
302		next = min(end, round_up(offset + 1, pte_addr_shift));
303
304		if (pt->level != 0) {
305			err = __lmtt_alloc_range(lmtt, pt, offset, next);
306			if (err)
307				return err;
308		}
309
310		offset = next;
311	}
312
313	return 0;
314}
315
316static int lmtt_alloc_range(struct xe_lmtt *lmtt, unsigned int vfid, u64 start, u64 end)
317{
318	struct xe_lmtt_pt *pd = lmtt->pd;
319	struct xe_lmtt_pt *pt;
320	u64 pt_addr;
321	u64 pde;
322	int err;
323
324	lmtt_assert(lmtt, pd->level > 0);
325	lmtt_assert(lmtt, vfid <= lmtt->ops->lmtt_pte_num(pd->level));
326	lmtt_assert(lmtt, IS_ALIGNED(start, lmtt_page_size(lmtt)));
327	lmtt_assert(lmtt, IS_ALIGNED(end, lmtt_page_size(lmtt)));
328
329	if (pd->entries[vfid])
330		return -ENOTEMPTY;
331
332	pt = lmtt_pt_alloc(lmtt, pd->level - 1);
333	if (IS_ERR(pt))
334		return PTR_ERR(pt);
335
336	pt_addr = xe_bo_main_addr(pt->bo, XE_PAGE_SIZE);
337
338	pde = lmtt->ops->lmtt_pte_encode(pt_addr, pd->level);
339
340	lmtt_write_pte(lmtt, pd, pde, vfid);
341
342	pd->entries[vfid] = pt;
343
344	if (pt->level != 0) {
345		err = __lmtt_alloc_range(lmtt, pt, start, end);
346		if (err)
347			goto out_free_pt;
348	}
349
350	return 0;
351
352out_free_pt:
353	lmtt_pt_free(pt);
354	return err;
355}
356
357static struct xe_lmtt_pt *lmtt_leaf_pt(struct xe_lmtt *lmtt, unsigned int vfid, u64 addr)
358{
359	struct xe_lmtt_pt *pd = lmtt->pd;
360	struct xe_lmtt_pt *pt;
361
362	lmtt_assert(lmtt, vfid <= lmtt->ops->lmtt_pte_num(pd->level));
363	pt = pd->entries[vfid];
364
365	while (pt->level) {
366		lmtt_assert(lmtt, lmtt->ops->lmtt_pte_index(addr, pt->level) <=
367			    lmtt->ops->lmtt_pte_num(pt->level));
368
369		pt = pt->entries[lmtt->ops->lmtt_pte_index(addr, pt->level)];
370
371		addr >>= lmtt->ops->lmtt_pte_shift(pt->level);
372	}
373
374	lmtt_assert(lmtt, lmtt->ops->lmtt_pte_index(addr, pt->level) <=
375		    lmtt->ops->lmtt_pte_num(pt->level));
376	lmtt_assert(lmtt, pt->level != pd->level);
377	lmtt_assert(lmtt, pt->level == 0);
378	return pt;
379}
380
381static void lmtt_insert_bo(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 start)
382{
383	u64 page_size = lmtt_page_size(lmtt);
384	struct xe_res_cursor cur;
385	struct xe_lmtt_pt *pt;
386	u64 addr, vram_offset;
387
388	lmtt_assert(lmtt, IS_ALIGNED(start, page_size));
389	lmtt_assert(lmtt, IS_ALIGNED(bo->size, page_size));
390	lmtt_assert(lmtt, xe_bo_is_vram(bo));
391
392	vram_offset = vram_region_gpu_offset(bo->ttm.resource);
393	xe_res_first(bo->ttm.resource, 0, bo->size, &cur);
394	while (cur.remaining) {
395		addr = xe_res_dma(&cur);
396		addr += vram_offset; /* XXX */
397
398		pt = lmtt_leaf_pt(lmtt, vfid, start);
399
400		lmtt_write_pte(lmtt, pt, lmtt->ops->lmtt_pte_encode(addr, 0),
401					 lmtt->ops->lmtt_pte_index(start, 0));
402
403		xe_res_next(&cur, page_size);
404		start += page_size;
405	}
406}
407
408/**
409 * xe_lmtt_prepare_pages - Create VF's LMTT Page Tables.
410 * @lmtt: the &xe_lmtt to update
411 * @vfid: the VF identifier (1-based)
412 * @range: top range of LMEM offset to be supported
413 *
414 * This function creates empty LMTT page tables for given VF to support
415 * up to maximum #range LMEM offset. The LMTT page tables created by this
416 * function must be released using xe_lmtt_drop_pages() function.
417 *
418 * Notes:
419 * This function shall be called only after successful LMTT initialization.
420 * See xe_lmtt_init().
421 *
422 * Return: 0 on success or a negative error code on failure.
423 */
424int xe_lmtt_prepare_pages(struct xe_lmtt *lmtt, unsigned int vfid, u64 range)
425{
426	lmtt_assert(lmtt, lmtt->pd);
427	lmtt_assert(lmtt, vfid);
428
429	return lmtt_alloc_range(lmtt, vfid, 0, range);
430}
431
432/**
433 * xe_lmtt_populate_pages - Update VF's LMTT Page Table Entries.
434 * @lmtt: the &xe_lmtt to update
435 * @vfid: the VF identifier (1-based)
436 * @bo: the buffer object with LMEM allocation to be mapped
437 * @offset: the offset at which #bo should be mapped
438 *
439 * This function updates VF's LMTT entries to use given buffer object as a backstore.
440 *
441 * Notes:
442 * This function shall be called only after successful preparation of the
443 * VF's LMTT Page Tables. See xe_lmtt_prepare().
444 *
445 * Return: 0 on success or a negative error code on failure.
446 */
447int xe_lmtt_populate_pages(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 offset)
448{
449	lmtt_assert(lmtt, lmtt->pd);
450	lmtt_assert(lmtt, vfid);
451
452	lmtt_insert_bo(lmtt, vfid, bo, offset);
453	return 0;
454}
455
456/**
457 * xe_lmtt_drop_pages - Remove VF's LMTT Pages.
458 * @lmtt: the &xe_lmtt to update
459 * @vfid: the VF identifier (1-based)
460 *
461 * This function removes all LMTT Page Tables prepared by xe_lmtt_prepare_pages().
462 *
463 * This function shall be called only after successful LMTT initialization.
464 * See xe_lmtt_init().
465 */
466void xe_lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid)
467{
468	lmtt_assert(lmtt, lmtt->pd);
469	lmtt_assert(lmtt, vfid);
470
471	lmtt_drop_pages(lmtt, vfid);
472}
473
474/**
475 * xe_lmtt_estimate_pt_size - Estimate size of LMTT PT allocations.
476 * @lmtt: the &xe_lmtt
477 * @size: the size of the LMEM to be mapped over LMTT (including any offset)
478 *
479 * This function shall be called only by PF.
480 *
481 * Return: size of the PT allocation(s) needed to support given LMEM size.
482 */
483u64 xe_lmtt_estimate_pt_size(struct xe_lmtt *lmtt, u64 size)
484{
485	unsigned int level = 0;
486	u64 pt_size;
487
488	lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt)));
489	lmtt_assert(lmtt, IS_DGFX(lmtt_to_xe(lmtt)));
490	lmtt_assert(lmtt, lmtt->ops);
491
492	pt_size = PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *
493			     lmtt->ops->lmtt_pte_num(level));
494
495	while (++level < lmtt->ops->lmtt_root_pd_level()) {
496		pt_size *= lmtt->ops->lmtt_pte_index(size, level) + 1;
497		pt_size += PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *
498				      lmtt->ops->lmtt_pte_num(level));
499	}
500
501	return pt_size;
502}
503
504#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
505#include "tests/xe_lmtt_test.c"
506#endif
507