1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/slab.h>
5#include <linux/dma-mapping.h>
6
7#include "lima_device.h"
8#include "lima_vm.h"
9#include "lima_gem.h"
10#include "lima_regs.h"
11
12struct lima_bo_va {
13	struct list_head list;
14	unsigned int ref_count;
15
16	struct drm_mm_node node;
17
18	struct lima_vm *vm;
19};
20
21#define LIMA_VM_PD_SHIFT 22
22#define LIMA_VM_PT_SHIFT 12
23#define LIMA_VM_PB_SHIFT (LIMA_VM_PD_SHIFT + LIMA_VM_NUM_PT_PER_BT_SHIFT)
24#define LIMA_VM_BT_SHIFT LIMA_VM_PT_SHIFT
25
26#define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1)
27#define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1)
28
29#define LIMA_PDE(va) (va >> LIMA_VM_PD_SHIFT)
30#define LIMA_PTE(va) ((va & LIMA_VM_PT_MASK) >> LIMA_VM_PT_SHIFT)
31#define LIMA_PBE(va) (va >> LIMA_VM_PB_SHIFT)
32#define LIMA_BTE(va) ((va & LIMA_VM_BT_MASK) >> LIMA_VM_BT_SHIFT)
33
34
35static void lima_vm_unmap_range(struct lima_vm *vm, u32 start, u32 end)
36{
37	u32 addr;
38
39	for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) {
40		u32 pbe = LIMA_PBE(addr);
41		u32 bte = LIMA_BTE(addr);
42
43		vm->bts[pbe].cpu[bte] = 0;
44	}
45}
46
47static int lima_vm_map_page(struct lima_vm *vm, dma_addr_t pa, u32 va)
48{
49	u32 pbe = LIMA_PBE(va);
50	u32 bte = LIMA_BTE(va);
51
52	if (!vm->bts[pbe].cpu) {
53		dma_addr_t pts;
54		u32 *pd;
55		int j;
56
57		vm->bts[pbe].cpu = dma_alloc_wc(
58			vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
59			&vm->bts[pbe].dma, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
60		if (!vm->bts[pbe].cpu)
61			return -ENOMEM;
62
63		pts = vm->bts[pbe].dma;
64		pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT);
65		for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
66			pd[j] = pts | LIMA_VM_FLAG_PRESENT;
67			pts += LIMA_PAGE_SIZE;
68		}
69	}
70
71	vm->bts[pbe].cpu[bte] = pa | LIMA_VM_FLAGS_CACHE;
72
73	return 0;
74}
75
76static struct lima_bo_va *
77lima_vm_bo_find(struct lima_vm *vm, struct lima_bo *bo)
78{
79	struct lima_bo_va *bo_va, *ret = NULL;
80
81	list_for_each_entry(bo_va, &bo->va, list) {
82		if (bo_va->vm == vm) {
83			ret = bo_va;
84			break;
85		}
86	}
87
88	return ret;
89}
90
91int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create)
92{
93	struct lima_bo_va *bo_va;
94	struct sg_dma_page_iter sg_iter;
95	int offset = 0, err;
96
97	mutex_lock(&bo->lock);
98
99	bo_va = lima_vm_bo_find(vm, bo);
100	if (bo_va) {
101		bo_va->ref_count++;
102		mutex_unlock(&bo->lock);
103		return 0;
104	}
105
106	/* should not create new bo_va if not asked by caller */
107	if (!create) {
108		mutex_unlock(&bo->lock);
109		return -ENOENT;
110	}
111
112	bo_va = kzalloc(sizeof(*bo_va), GFP_KERNEL);
113	if (!bo_va) {
114		err = -ENOMEM;
115		goto err_out0;
116	}
117
118	bo_va->vm = vm;
119	bo_va->ref_count = 1;
120
121	mutex_lock(&vm->lock);
122
123	err = drm_mm_insert_node(&vm->mm, &bo_va->node, lima_bo_size(bo));
124	if (err)
125		goto err_out1;
126
127	for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, 0) {
128		err = lima_vm_map_page(vm, sg_page_iter_dma_address(&sg_iter),
129				       bo_va->node.start + offset);
130		if (err)
131			goto err_out2;
132
133		offset += PAGE_SIZE;
134	}
135
136	mutex_unlock(&vm->lock);
137
138	list_add_tail(&bo_va->list, &bo->va);
139
140	mutex_unlock(&bo->lock);
141	return 0;
142
143err_out2:
144	if (offset)
145		lima_vm_unmap_range(vm, bo_va->node.start, bo_va->node.start + offset - 1);
146	drm_mm_remove_node(&bo_va->node);
147err_out1:
148	mutex_unlock(&vm->lock);
149	kfree(bo_va);
150err_out0:
151	mutex_unlock(&bo->lock);
152	return err;
153}
154
155void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo)
156{
157	struct lima_bo_va *bo_va;
158	u32 size;
159
160	mutex_lock(&bo->lock);
161
162	bo_va = lima_vm_bo_find(vm, bo);
163	if (--bo_va->ref_count > 0) {
164		mutex_unlock(&bo->lock);
165		return;
166	}
167
168	mutex_lock(&vm->lock);
169
170	size = bo->heap_size ? bo->heap_size : bo_va->node.size;
171	lima_vm_unmap_range(vm, bo_va->node.start,
172			    bo_va->node.start + size - 1);
173
174	drm_mm_remove_node(&bo_va->node);
175
176	mutex_unlock(&vm->lock);
177
178	list_del(&bo_va->list);
179
180	mutex_unlock(&bo->lock);
181
182	kfree(bo_va);
183}
184
185u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo)
186{
187	struct lima_bo_va *bo_va;
188	u32 ret;
189
190	mutex_lock(&bo->lock);
191
192	bo_va = lima_vm_bo_find(vm, bo);
193	ret = bo_va->node.start;
194
195	mutex_unlock(&bo->lock);
196
197	return ret;
198}
199
200struct lima_vm *lima_vm_create(struct lima_device *dev)
201{
202	struct lima_vm *vm;
203
204	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
205	if (!vm)
206		return NULL;
207
208	vm->dev = dev;
209	mutex_init(&vm->lock);
210	kref_init(&vm->refcount);
211
212	vm->pd.cpu = dma_alloc_wc(dev->dev, LIMA_PAGE_SIZE, &vm->pd.dma,
213				  GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
214	if (!vm->pd.cpu)
215		goto err_out0;
216
217	if (dev->dlbu_cpu) {
218		int err = lima_vm_map_page(
219			vm, dev->dlbu_dma, LIMA_VA_RESERVE_DLBU);
220		if (err)
221			goto err_out1;
222	}
223
224	drm_mm_init(&vm->mm, dev->va_start, dev->va_end - dev->va_start);
225
226	return vm;
227
228err_out1:
229	dma_free_wc(dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);
230err_out0:
231	kfree(vm);
232	return NULL;
233}
234
235void lima_vm_release(struct kref *kref)
236{
237	struct lima_vm *vm = container_of(kref, struct lima_vm, refcount);
238	int i;
239
240	drm_mm_takedown(&vm->mm);
241
242	for (i = 0; i < LIMA_VM_NUM_BT; i++) {
243		if (vm->bts[i].cpu)
244			dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
245				    vm->bts[i].cpu, vm->bts[i].dma);
246	}
247
248	if (vm->pd.cpu)
249		dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);
250
251	kfree(vm);
252}
253
254void lima_vm_print(struct lima_vm *vm)
255{
256	int i, j, k;
257	u32 *pd, *pt;
258
259	if (!vm->pd.cpu)
260		return;
261
262	pd = vm->pd.cpu;
263	for (i = 0; i < LIMA_VM_NUM_BT; i++) {
264		if (!vm->bts[i].cpu)
265			continue;
266
267		pt = vm->bts[i].cpu;
268		for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
269			int idx = (i << LIMA_VM_NUM_PT_PER_BT_SHIFT) + j;
270
271			printk(KERN_INFO "lima vm pd %03x:%08x\n", idx, pd[idx]);
272
273			for (k = 0; k < LIMA_PAGE_ENT_NUM; k++) {
274				u32 pte = *pt++;
275
276				if (pte)
277					printk(KERN_INFO "  pt %03x:%08x\n", k, pte);
278			}
279		}
280	}
281}
282
283int lima_vm_map_bo(struct lima_vm *vm, struct lima_bo *bo, int pageoff)
284{
285	struct lima_bo_va *bo_va;
286	struct sg_dma_page_iter sg_iter;
287	int offset = 0, err;
288	u32 base;
289
290	mutex_lock(&bo->lock);
291
292	bo_va = lima_vm_bo_find(vm, bo);
293	if (!bo_va) {
294		err = -ENOENT;
295		goto err_out0;
296	}
297
298	mutex_lock(&vm->lock);
299
300	base = bo_va->node.start + (pageoff << PAGE_SHIFT);
301	for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, pageoff) {
302		err = lima_vm_map_page(vm, sg_page_iter_dma_address(&sg_iter),
303				       base + offset);
304		if (err)
305			goto err_out1;
306
307		offset += PAGE_SIZE;
308	}
309
310	mutex_unlock(&vm->lock);
311
312	mutex_unlock(&bo->lock);
313	return 0;
314
315err_out1:
316	if (offset)
317		lima_vm_unmap_range(vm, base, base + offset - 1);
318	mutex_unlock(&vm->lock);
319err_out0:
320	mutex_unlock(&bo->lock);
321	return err;
322}
323