1// SPDX-License-Identifier: MIT
2/*
3 * Copyright �� 2021 Intel Corporation
4 */
5
6#include "gem/i915_gem_domain.h"
7#include "gem/i915_gem_internal.h"
8#include "gem/i915_gem_lmem.h"
9#include "gt/gen8_ppgtt.h"
10
11#include "i915_drv.h"
12#include "i915_reg.h"
13#include "intel_de.h"
14#include "intel_display_types.h"
15#include "intel_dpt.h"
16#include "intel_fb.h"
17
18struct i915_dpt {
19	struct i915_address_space vm;
20
21	struct drm_i915_gem_object *obj;
22	struct i915_vma *vma;
23	void __iomem *iomem;
24};
25
26#define i915_is_dpt(vm) ((vm)->is_dpt)
27
28static inline struct i915_dpt *
29i915_vm_to_dpt(struct i915_address_space *vm)
30{
31	BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
32	GEM_BUG_ON(!i915_is_dpt(vm));
33	return container_of(vm, struct i915_dpt, vm);
34}
35
36#define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT)
37
38static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
39{
40	writeq(pte, addr);
41}
42
43static void dpt_insert_page(struct i915_address_space *vm,
44			    dma_addr_t addr,
45			    u64 offset,
46			    unsigned int pat_index,
47			    u32 flags)
48{
49	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
50	gen8_pte_t __iomem *base = dpt->iomem;
51
52	gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
53		     vm->pte_encode(addr, pat_index, flags));
54}
55
56static void dpt_insert_entries(struct i915_address_space *vm,
57			       struct i915_vma_resource *vma_res,
58			       unsigned int pat_index,
59			       u32 flags)
60{
61	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
62	gen8_pte_t __iomem *base = dpt->iomem;
63	const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);
64	struct sgt_iter sgt_iter;
65	dma_addr_t addr;
66	int i;
67
68	/*
69	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
70	 * not to allow the user to override access to a read only page.
71	 */
72
73	i = vma_res->start / I915_GTT_PAGE_SIZE;
74	for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages)
75		gen8_set_pte(&base[i++], pte_encode | addr);
76}
77
78static void dpt_clear_range(struct i915_address_space *vm,
79			    u64 start, u64 length)
80{
81}
82
83static void dpt_bind_vma(struct i915_address_space *vm,
84			 struct i915_vm_pt_stash *stash,
85			 struct i915_vma_resource *vma_res,
86			 unsigned int pat_index,
87			 u32 flags)
88{
89	u32 pte_flags;
90
91	if (vma_res->bound_flags)
92		return;
93
94	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
95	pte_flags = 0;
96	if (vm->has_read_only && vma_res->bi.readonly)
97		pte_flags |= PTE_READ_ONLY;
98	if (vma_res->bi.lmem)
99		pte_flags |= PTE_LM;
100
101	vm->insert_entries(vm, vma_res, pat_index, pte_flags);
102
103	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
104
105	/*
106	 * Without aliasing PPGTT there's no difference between
107	 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
108	 * upgrade to both bound if we bind either to avoid double-binding.
109	 */
110	vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
111}
112
113static void dpt_unbind_vma(struct i915_address_space *vm,
114			   struct i915_vma_resource *vma_res)
115{
116	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
117}
118
119static void dpt_cleanup(struct i915_address_space *vm)
120{
121	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
122
123	i915_gem_object_put(dpt->obj);
124}
125
126struct i915_vma *intel_dpt_pin(struct i915_address_space *vm)
127{
128	struct drm_i915_private *i915 = vm->i915;
129	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
130	intel_wakeref_t wakeref;
131	struct i915_vma *vma;
132	void __iomem *iomem;
133	struct i915_gem_ww_ctx ww;
134	u64 pin_flags = 0;
135	int err;
136
137	if (i915_gem_object_is_stolen(dpt->obj))
138		pin_flags |= PIN_MAPPABLE;
139
140	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
141	atomic_inc(&i915->gpu_error.pending_fb_pin);
142
143	for_i915_gem_ww(&ww, err, true) {
144		err = i915_gem_object_lock(dpt->obj, &ww);
145		if (err)
146			continue;
147
148		vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 4096,
149						  pin_flags);
150		if (IS_ERR(vma)) {
151			err = PTR_ERR(vma);
152			continue;
153		}
154
155		iomem = i915_vma_pin_iomap(vma);
156		i915_vma_unpin(vma);
157
158		if (IS_ERR(iomem)) {
159			err = PTR_ERR(iomem);
160			continue;
161		}
162
163		dpt->vma = vma;
164		dpt->iomem = iomem;
165
166		i915_vma_get(vma);
167	}
168
169	dpt->obj->mm.dirty = true;
170
171	atomic_dec(&i915->gpu_error.pending_fb_pin);
172	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
173
174	return err ? ERR_PTR(err) : vma;
175}
176
177void intel_dpt_unpin(struct i915_address_space *vm)
178{
179	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
180
181	i915_vma_unpin_iomap(dpt->vma);
182	i915_vma_put(dpt->vma);
183}
184
185/**
186 * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume
187 * @i915: device instance
188 *
189 * Restore the memory mapping during system resume for all framebuffers which
190 * are mapped to HW via a GGTT->DPT page table. The content of these page
191 * tables are not stored in the hibernation image during S4 and S3RST->S4
192 * transitions, so here we reprogram the PTE entries in those tables.
193 *
194 * This function must be called after the mappings in GGTT have been restored calling
195 * i915_ggtt_resume().
196 */
197void intel_dpt_resume(struct drm_i915_private *i915)
198{
199	struct drm_framebuffer *drm_fb;
200
201	if (!HAS_DISPLAY(i915))
202		return;
203
204	mutex_lock(&i915->drm.mode_config.fb_lock);
205	drm_for_each_fb(drm_fb, &i915->drm) {
206		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
207
208		if (fb->dpt_vm)
209			i915_ggtt_resume_vm(fb->dpt_vm);
210	}
211	mutex_unlock(&i915->drm.mode_config.fb_lock);
212}
213
214/**
215 * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend
216 * @i915: device instance
217 *
218 * Suspend the memory mapping during system suspend for all framebuffers which
219 * are mapped to HW via a GGTT->DPT page table.
220 *
221 * This function must be called before the mappings in GGTT are suspended calling
222 * i915_ggtt_suspend().
223 */
224void intel_dpt_suspend(struct drm_i915_private *i915)
225{
226	struct drm_framebuffer *drm_fb;
227
228	if (!HAS_DISPLAY(i915))
229		return;
230
231	mutex_lock(&i915->drm.mode_config.fb_lock);
232
233	drm_for_each_fb(drm_fb, &i915->drm) {
234		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
235
236		if (fb->dpt_vm)
237			i915_ggtt_suspend_vm(fb->dpt_vm);
238	}
239
240	mutex_unlock(&i915->drm.mode_config.fb_lock);
241}
242
243struct i915_address_space *
244intel_dpt_create(struct intel_framebuffer *fb)
245{
246	struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base;
247	struct drm_i915_private *i915 = to_i915(obj->dev);
248	struct drm_i915_gem_object *dpt_obj;
249	struct i915_address_space *vm;
250	struct i915_dpt *dpt;
251	size_t size;
252	int ret;
253
254	if (intel_fb_needs_pot_stride_remap(fb))
255		size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
256	else
257		size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
258
259	size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
260
261	dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS);
262	if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt))
263		dpt_obj = i915_gem_object_create_stolen(i915, size);
264	if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) {
265		drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n");
266		dpt_obj = i915_gem_object_create_shmem(i915, size);
267	}
268	if (IS_ERR(dpt_obj))
269		return ERR_CAST(dpt_obj);
270
271	ret = i915_gem_object_lock_interruptible(dpt_obj, NULL);
272	if (!ret) {
273		ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
274		i915_gem_object_unlock(dpt_obj);
275	}
276	if (ret) {
277		i915_gem_object_put(dpt_obj);
278		return ERR_PTR(ret);
279	}
280
281	dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);
282	if (!dpt) {
283		i915_gem_object_put(dpt_obj);
284		return ERR_PTR(-ENOMEM);
285	}
286
287	vm = &dpt->vm;
288
289	vm->gt = to_gt(i915);
290	vm->i915 = i915;
291	vm->dma = i915->drm.dev;
292	vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
293	vm->is_dpt = true;
294
295	i915_address_space_init(vm, VM_CLASS_DPT);
296
297	vm->insert_page = dpt_insert_page;
298	vm->clear_range = dpt_clear_range;
299	vm->insert_entries = dpt_insert_entries;
300	vm->cleanup = dpt_cleanup;
301
302	vm->vma_ops.bind_vma    = dpt_bind_vma;
303	vm->vma_ops.unbind_vma  = dpt_unbind_vma;
304
305	vm->pte_encode = vm->gt->ggtt->vm.pte_encode;
306
307	dpt->obj = dpt_obj;
308	dpt->obj->is_dpt = true;
309
310	return &dpt->vm;
311}
312
313void intel_dpt_destroy(struct i915_address_space *vm)
314{
315	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
316
317	dpt->obj->is_dpt = false;
318	i915_vm_put(&dpt->vm);
319}
320
321void intel_dpt_configure(struct intel_crtc *crtc)
322{
323	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
324
325	if (DISPLAY_VER(i915) == 14) {
326		enum pipe pipe = crtc->pipe;
327		enum plane_id plane_id;
328
329		for_each_plane_id_on_crtc(crtc, plane_id) {
330			if (plane_id == PLANE_CURSOR)
331				continue;
332
333			intel_de_rmw(i915, PLANE_CHICKEN(pipe, plane_id),
334				     PLANE_CHICKEN_DISABLE_DPT,
335				     i915->params.enable_dpt ? 0 : PLANE_CHICKEN_DISABLE_DPT);
336		}
337	} else if (DISPLAY_VER(i915) == 13) {
338		intel_de_rmw(i915, CHICKEN_MISC_2,
339			     CHICKEN_MISC_DISABLE_DPT,
340			     i915->params.enable_dpt ? 0 : CHICKEN_MISC_DISABLE_DPT);
341	}
342}
343