1// SPDX-License-Identifier: MIT
2/*
3 * Copyright �� 2021 Intel Corporation
4 */
5
6#include "i915_drv.h"
7#include "intel_display_types.h"
8#include "intel_dpt.h"
9#include "intel_fb.h"
10#include "intel_fb_pin.h"
11#include "xe_ggtt.h"
12#include "xe_gt.h"
13
14#include <drm/ttm/ttm_bo.h>
15
16static void
17write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs,
18		  u32 width, u32 height, u32 src_stride, u32 dst_stride)
19{
20	struct xe_device *xe = xe_bo_device(bo);
21	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
22	u32 column, row;
23
24	/* TODO: Maybe rewrite so we can traverse the bo addresses sequentially,
25	 * by writing dpt/ggtt in a different order?
26	 */
27
28	for (column = 0; column < width; column++) {
29		u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
30
31		for (row = 0; row < height; row++) {
32			u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
33							      xe->pat.idx[XE_CACHE_WB]);
34
35			iosys_map_wr(map, *dpt_ofs, u64, pte);
36			*dpt_ofs += 8;
37			src_idx -= src_stride;
38		}
39
40		/* The DE ignores the PTEs for the padding tiles */
41		*dpt_ofs += (dst_stride - height) * 8;
42	}
43
44	/* Align to next page */
45	*dpt_ofs = ALIGN(*dpt_ofs, 4096);
46}
47
48static void
49write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs,
50		   u32 bo_ofs, u32 width, u32 height, u32 src_stride,
51		   u32 dst_stride)
52{
53	struct xe_device *xe = xe_bo_device(bo);
54	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
55	u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index)
56		= ggtt->pt_ops->pte_encode_bo;
57	u32 column, row;
58
59	for (row = 0; row < height; row++) {
60		u32 src_idx = src_stride * row + bo_ofs;
61
62		for (column = 0; column < width; column++) {
63			iosys_map_wr(map, *dpt_ofs, u64,
64				     pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
65				     xe->pat.idx[XE_CACHE_WB]));
66
67			*dpt_ofs += 8;
68			src_idx++;
69		}
70
71		/* The DE ignores the PTEs for the padding tiles */
72		*dpt_ofs += (dst_stride - width) * 8;
73	}
74
75	/* Align to next page */
76	*dpt_ofs = ALIGN(*dpt_ofs, 4096);
77}
78
79static int __xe_pin_fb_vma_dpt(struct intel_framebuffer *fb,
80			       const struct i915_gtt_view *view,
81			       struct i915_vma *vma)
82{
83	struct xe_device *xe = to_xe_device(fb->base.dev);
84	struct xe_tile *tile0 = xe_device_get_root_tile(xe);
85	struct xe_ggtt *ggtt = tile0->mem.ggtt;
86	struct xe_bo *bo = intel_fb_obj(&fb->base), *dpt;
87	u32 dpt_size, size = bo->ttm.base.size;
88
89	if (view->type == I915_GTT_VIEW_NORMAL)
90		dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE);
91	else if (view->type == I915_GTT_VIEW_REMAPPED)
92		dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8,
93				 XE_PAGE_SIZE);
94	else
95		/* display uses 4K tiles instead of bytes here, convert to entries.. */
96		dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8,
97				 XE_PAGE_SIZE);
98
99	if (IS_DGFX(xe))
100		dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,
101					   ttm_bo_type_kernel,
102					   XE_BO_CREATE_VRAM0_BIT |
103					   XE_BO_CREATE_GGTT_BIT);
104	else
105		dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,
106					   ttm_bo_type_kernel,
107					   XE_BO_CREATE_STOLEN_BIT |
108					   XE_BO_CREATE_GGTT_BIT);
109	if (IS_ERR(dpt))
110		dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,
111					   ttm_bo_type_kernel,
112					   XE_BO_CREATE_SYSTEM_BIT |
113					   XE_BO_CREATE_GGTT_BIT);
114	if (IS_ERR(dpt))
115		return PTR_ERR(dpt);
116
117	if (view->type == I915_GTT_VIEW_NORMAL) {
118		u32 x;
119
120		for (x = 0; x < size / XE_PAGE_SIZE; x++) {
121			u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE,
122							      xe->pat.idx[XE_CACHE_WB]);
123
124			iosys_map_wr(&dpt->vmap, x * 8, u64, pte);
125		}
126	} else if (view->type == I915_GTT_VIEW_REMAPPED) {
127		const struct intel_remapped_info *remap_info = &view->remapped;
128		u32 i, dpt_ofs = 0;
129
130		for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++)
131			write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs,
132					   remap_info->plane[i].offset,
133					   remap_info->plane[i].width,
134					   remap_info->plane[i].height,
135					   remap_info->plane[i].src_stride,
136					   remap_info->plane[i].dst_stride);
137
138	} else {
139		const struct intel_rotation_info *rot_info = &view->rotated;
140		u32 i, dpt_ofs = 0;
141
142		for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
143			write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs,
144					  rot_info->plane[i].offset,
145					  rot_info->plane[i].width,
146					  rot_info->plane[i].height,
147					  rot_info->plane[i].src_stride,
148					  rot_info->plane[i].dst_stride);
149	}
150
151	vma->dpt = dpt;
152	vma->node = dpt->ggtt_node;
153	return 0;
154}
155
156static void
157write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs,
158		   u32 width, u32 height, u32 src_stride, u32 dst_stride)
159{
160	struct xe_device *xe = xe_bo_device(bo);
161	u32 column, row;
162
163	for (column = 0; column < width; column++) {
164		u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
165
166		for (row = 0; row < height; row++) {
167			u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
168							      xe->pat.idx[XE_CACHE_WB]);
169
170			xe_ggtt_set_pte(ggtt, *ggtt_ofs, pte);
171			*ggtt_ofs += XE_PAGE_SIZE;
172			src_idx -= src_stride;
173		}
174
175		/* The DE ignores the PTEs for the padding tiles */
176		*ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE;
177	}
178}
179
180static int __xe_pin_fb_vma_ggtt(struct intel_framebuffer *fb,
181				const struct i915_gtt_view *view,
182				struct i915_vma *vma)
183{
184	struct xe_bo *bo = intel_fb_obj(&fb->base);
185	struct xe_device *xe = to_xe_device(fb->base.dev);
186	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
187	u32 align;
188	int ret;
189
190	/* TODO: Consider sharing framebuffer mapping?
191	 * embed i915_vma inside intel_framebuffer
192	 */
193	xe_device_mem_access_get(tile_to_xe(ggtt->tile));
194	ret = mutex_lock_interruptible(&ggtt->lock);
195	if (ret)
196		goto out;
197
198	align = XE_PAGE_SIZE;
199	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
200		align = max_t(u32, align, SZ_64K);
201
202	if (bo->ggtt_node.size && view->type == I915_GTT_VIEW_NORMAL) {
203		vma->node = bo->ggtt_node;
204	} else if (view->type == I915_GTT_VIEW_NORMAL) {
205		u32 x, size = bo->ttm.base.size;
206
207		ret = xe_ggtt_insert_special_node_locked(ggtt, &vma->node, size,
208							 align, 0);
209		if (ret)
210			goto out_unlock;
211
212		for (x = 0; x < size; x += XE_PAGE_SIZE) {
213			u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x,
214							      xe->pat.idx[XE_CACHE_WB]);
215
216			xe_ggtt_set_pte(ggtt, vma->node.start + x, pte);
217		}
218	} else {
219		u32 i, ggtt_ofs;
220		const struct intel_rotation_info *rot_info = &view->rotated;
221
222		/* display seems to use tiles instead of bytes here, so convert it back.. */
223		u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE;
224
225		ret = xe_ggtt_insert_special_node_locked(ggtt, &vma->node, size,
226							 align, 0);
227		if (ret)
228			goto out_unlock;
229
230		ggtt_ofs = vma->node.start;
231
232		for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
233			write_ggtt_rotated(bo, ggtt, &ggtt_ofs,
234					   rot_info->plane[i].offset,
235					   rot_info->plane[i].width,
236					   rot_info->plane[i].height,
237					   rot_info->plane[i].src_stride,
238					   rot_info->plane[i].dst_stride);
239	}
240
241	xe_ggtt_invalidate(ggtt);
242out_unlock:
243	mutex_unlock(&ggtt->lock);
244out:
245	xe_device_mem_access_put(tile_to_xe(ggtt->tile));
246	return ret;
247}
248
249static struct i915_vma *__xe_pin_fb_vma(struct intel_framebuffer *fb,
250					const struct i915_gtt_view *view)
251{
252	struct drm_device *dev = fb->base.dev;
253	struct xe_device *xe = to_xe_device(dev);
254	struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
255	struct xe_bo *bo = intel_fb_obj(&fb->base);
256	int ret;
257
258	if (!vma)
259		return ERR_PTR(-ENODEV);
260
261	if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) &&
262	    intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 &&
263	    !(bo->flags & XE_BO_NEEDS_CPU_ACCESS)) {
264		struct xe_tile *tile = xe_device_get_root_tile(xe);
265
266		/*
267		 * If we need to able to access the clear-color value stored in
268		 * the buffer, then we require that such buffers are also CPU
269		 * accessible.  This is important on small-bar systems where
270		 * only some subset of VRAM is CPU accessible.
271		 */
272		if (tile->mem.vram.io_size < tile->mem.vram.usable_size) {
273			ret = -EINVAL;
274			goto err;
275		}
276	}
277
278	/*
279	 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the
280	 * assumptions are incorrect for framebuffers
281	 */
282	ret = ttm_bo_reserve(&bo->ttm, false, false, NULL);
283	if (ret)
284		goto err;
285
286	if (IS_DGFX(xe))
287		ret = xe_bo_migrate(bo, XE_PL_VRAM0);
288	else
289		ret = xe_bo_validate(bo, NULL, true);
290	if (!ret)
291		ttm_bo_pin(&bo->ttm);
292	ttm_bo_unreserve(&bo->ttm);
293	if (ret)
294		goto err;
295
296	vma->bo = bo;
297	if (intel_fb_uses_dpt(&fb->base))
298		ret = __xe_pin_fb_vma_dpt(fb, view, vma);
299	else
300		ret = __xe_pin_fb_vma_ggtt(fb, view, vma);
301	if (ret)
302		goto err_unpin;
303
304	return vma;
305
306err_unpin:
307	ttm_bo_reserve(&bo->ttm, false, false, NULL);
308	ttm_bo_unpin(&bo->ttm);
309	ttm_bo_unreserve(&bo->ttm);
310err:
311	kfree(vma);
312	return ERR_PTR(ret);
313}
314
315static void __xe_unpin_fb_vma(struct i915_vma *vma)
316{
317	struct xe_device *xe = to_xe_device(vma->bo->ttm.base.dev);
318	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
319
320	if (vma->dpt)
321		xe_bo_unpin_map_no_vm(vma->dpt);
322	else if (!drm_mm_node_allocated(&vma->bo->ggtt_node) ||
323		 vma->bo->ggtt_node.start != vma->node.start)
324		xe_ggtt_remove_node(ggtt, &vma->node);
325
326	ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);
327	ttm_bo_unpin(&vma->bo->ttm);
328	ttm_bo_unreserve(&vma->bo->ttm);
329	kfree(vma);
330}
331
332struct i915_vma *
333intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
334			   bool phys_cursor,
335			   const struct i915_gtt_view *view,
336			   bool uses_fence,
337			   unsigned long *out_flags)
338{
339	*out_flags = 0;
340
341	return __xe_pin_fb_vma(to_intel_framebuffer(fb), view);
342}
343
344void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
345{
346	__xe_unpin_fb_vma(vma);
347}
348
349int intel_plane_pin_fb(struct intel_plane_state *plane_state)
350{
351	struct drm_framebuffer *fb = plane_state->hw.fb;
352	struct xe_bo *bo = intel_fb_obj(fb);
353	struct i915_vma *vma;
354
355	/* We reject creating !SCANOUT fb's, so this is weird.. */
356	drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_SCANOUT_BIT));
357
358	vma = __xe_pin_fb_vma(to_intel_framebuffer(fb), &plane_state->view.gtt);
359	if (IS_ERR(vma))
360		return PTR_ERR(vma);
361
362	plane_state->ggtt_vma = vma;
363	return 0;
364}
365
366void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
367{
368	__xe_unpin_fb_vma(old_plane_state->ggtt_vma);
369	old_plane_state->ggtt_vma = NULL;
370}
371
372/*
373 * For Xe introduce dummy intel_dpt_create which just return NULL and
374 * intel_dpt_destroy which does nothing.
375 */
376struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb)
377{
378	return NULL;
379}
380
381void intel_dpt_destroy(struct i915_address_space *vm)
382{
383	return;
384}