1/*
2 * Copyright 2017 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22#include "vmm.h"
23
24#include <subdev/fb.h>
25#include <subdev/ltc.h>
26#include <subdev/timer.h>
27
28#include <nvif/if900d.h>
29#include <nvif/unpack.h>
30
31static inline void
32gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
33		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
34{
35	u64 base = (addr >> 8) | map->type;
36	u64 data = base;
37
38	if (map->ctag && !(map->next & (1ULL << 44))) {
39		while (ptes--) {
40			data = base | ((map->ctag >> 1) << 44);
41			if (!(map->ctag++ & 1))
42				data |= BIT_ULL(60);
43
44			VMM_WO064(pt, vmm, ptei++ * 8, data);
45			base += map->next;
46		}
47	} else {
48		map->type += ptes * map->ctag;
49
50		while (ptes--) {
51			VMM_WO064(pt, vmm, ptei++ * 8, data);
52			data += map->next;
53		}
54	}
55}
56
57void
58gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
59		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
60{
61	VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
62}
63
64void
65gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
66		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
67{
68	if (map->page->shift == PAGE_SHIFT) {
69		VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
70		nvkm_kmap(pt->memory);
71		while (ptes--) {
72			const u64 data = (*map->dma++ >> 8) | map->type;
73			VMM_WO064(pt, vmm, ptei++ * 8, data);
74			map->type += map->ctag;
75		}
76		nvkm_done(pt->memory);
77		return;
78	}
79
80	VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
81}
82
83void
84gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
85		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
86{
87	VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
88}
89
90void
91gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm,
92		    struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
93{
94	VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes);
95}
96
97const struct nvkm_vmm_desc_func
98gf100_vmm_pgt = {
99	.unmap = gf100_vmm_pgt_unmap,
100	.mem = gf100_vmm_pgt_mem,
101	.dma = gf100_vmm_pgt_dma,
102	.sgl = gf100_vmm_pgt_sgl,
103};
104
105void
106gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
107{
108	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
109	struct nvkm_mmu_pt *pd = pgd->pt[0];
110	struct nvkm_mmu_pt *pt;
111	u64 data = 0;
112
113	if ((pt = pgt->pt[0])) {
114		switch (nvkm_memory_target(pt->memory)) {
115		case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break;
116		case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0;
117			data |= BIT_ULL(35); /* VOL */
118			break;
119		case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break;
120		default:
121			WARN_ON(1);
122			return;
123		}
124		data |= pt->addr >> 8;
125	}
126
127	if ((pt = pgt->pt[1])) {
128		switch (nvkm_memory_target(pt->memory)) {
129		case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break;
130		case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32;
131			data |= BIT_ULL(34); /* VOL */
132			break;
133		case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break;
134		default:
135			WARN_ON(1);
136			return;
137		}
138		data |= pt->addr << 24;
139	}
140
141	nvkm_kmap(pd->memory);
142	VMM_WO064(pd, vmm, pdei * 8, data);
143	nvkm_done(pd->memory);
144}
145
146const struct nvkm_vmm_desc_func
147gf100_vmm_pgd = {
148	.unmap = gf100_vmm_pgt_unmap,
149	.pde = gf100_vmm_pgd_pde,
150};
151
152static const struct nvkm_vmm_desc
153gf100_vmm_desc_17_12[] = {
154	{ SPT, 15, 8, 0x1000, &gf100_vmm_pgt },
155	{ PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
156	{}
157};
158
159static const struct nvkm_vmm_desc
160gf100_vmm_desc_17_17[] = {
161	{ LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
162	{ PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
163	{}
164};
165
166static const struct nvkm_vmm_desc
167gf100_vmm_desc_16_12[] = {
168	{ SPT, 14, 8, 0x1000, &gf100_vmm_pgt },
169	{ PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
170	{}
171};
172
173static const struct nvkm_vmm_desc
174gf100_vmm_desc_16_16[] = {
175	{ LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
176	{ PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
177	{}
178};
179
180void
181gf100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr)
182{
183	struct nvkm_device *device = vmm->mmu->subdev.device;
184	nvkm_wr32(device, 0x100cb8, addr);
185}
186
187void
188gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type)
189{
190	struct nvkm_device *device = vmm->mmu->subdev.device;
191	struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
192	u64 addr = 0;
193
194	mutex_lock(&vmm->mmu->mutex);
195	/* Looks like maybe a "free flush slots" counter, the
196	 * faster you write to 0x100cbc to more it decreases.
197	 */
198	nvkm_msec(device, 2000,
199		if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
200			break;
201	);
202
203	if (!(type & 0x00000002) /* ALL_PDB. */) {
204		switch (nvkm_memory_target(pd->memory)) {
205		case NVKM_MEM_TARGET_VRAM: addr |= 0x00000000; break;
206		case NVKM_MEM_TARGET_HOST: addr |= 0x00000002; break;
207		case NVKM_MEM_TARGET_NCOH: addr |= 0x00000003; break;
208		default:
209			WARN_ON(1);
210			break;
211		}
212		addr |= (vmm->pd->pt[0]->addr >> 12) << 4;
213
214		vmm->func->invalidate_pdb(vmm, addr);
215	}
216
217	nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
218
219	/* Wait for flush to be queued? */
220	nvkm_msec(device, 2000,
221		if (nvkm_rd32(device, 0x100c80) & 0x00008000)
222			break;
223	);
224	mutex_unlock(&vmm->mmu->mutex);
225}
226
227void
228gf100_vmm_flush(struct nvkm_vmm *vmm, int depth)
229{
230	u32 type = 0x00000001; /* PAGE_ALL */
231	if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))
232		type |= 0x00000004; /* HUB_ONLY */
233	gf100_vmm_invalidate(vmm, type);
234}
235
236int
237gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
238		struct nvkm_vmm_map *map)
239{
240	const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
241	const struct nvkm_vmm_page *page = map->page;
242	const bool gm20x = page->desc->func->sparse != NULL;
243	union {
244		struct gf100_vmm_map_vn vn;
245		struct gf100_vmm_map_v0 v0;
246	} *args = argv;
247	struct nvkm_device *device = vmm->mmu->subdev.device;
248	struct nvkm_memory *memory = map->memory;
249	u8  kind, kind_inv, priv, ro, vol;
250	int kindn, aper, ret = -ENOSYS;
251	const u8 *kindm;
252
253	map->next = (1 << page->shift) >> 8;
254	map->type = map->ctag = 0;
255
256	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
257		vol  = !!args->v0.vol;
258		ro   = !!args->v0.ro;
259		priv = !!args->v0.priv;
260		kind =   args->v0.kind;
261	} else
262	if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
263		vol  = target == NVKM_MEM_TARGET_HOST;
264		ro   = 0;
265		priv = 0;
266		kind = 0x00;
267	} else {
268		VMM_DEBUG(vmm, "args");
269		return ret;
270	}
271
272	aper = vmm->func->aper(target);
273	if (WARN_ON(aper < 0))
274		return aper;
275
276	kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv);
277	if (kind >= kindn || kindm[kind] == kind_inv) {
278		VMM_DEBUG(vmm, "kind %02x", kind);
279		return -EINVAL;
280	}
281
282	if (kindm[kind] != kind) {
283		u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17;
284		u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp;
285		if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
286			VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
287			return -EINVAL;
288		}
289
290		if (!map->no_comp) {
291			ret = nvkm_memory_tags_get(memory, device, tags,
292						   nvkm_ltc_tags_clear,
293						   &map->tags);
294			if (ret) {
295				VMM_DEBUG(vmm, "comp %d", ret);
296				return ret;
297			}
298		}
299
300		if (!map->no_comp && map->tags->mn) {
301			u64 tags = map->tags->mn->offset + (map->offset >> 17);
302			if (page->shift == 17 || !gm20x) {
303				map->type |= tags << 44;
304				map->ctag |= 1ULL << 44;
305				map->next |= 1ULL << 44;
306			} else {
307				map->ctag |= tags << 1 | 1;
308			}
309		} else {
310			kind = kindm[kind];
311		}
312	}
313
314	map->type |= BIT(0);
315	map->type |= (u64)priv << 1;
316	map->type |= (u64)  ro << 2;
317	map->type |= (u64) vol << 32;
318	map->type |= (u64)aper << 33;
319	map->type |= (u64)kind << 36;
320	return 0;
321}
322
323int
324gf100_vmm_aper(enum nvkm_memory_target target)
325{
326	switch (target) {
327	case NVKM_MEM_TARGET_VRAM: return 0;
328	case NVKM_MEM_TARGET_HOST: return 2;
329	case NVKM_MEM_TARGET_NCOH: return 3;
330	default:
331		return -EINVAL;
332	}
333}
334
335void
336gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
337{
338	nvkm_fo64(inst, 0x0200, 0x00000000, 2);
339}
340
341int
342gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base)
343{
344	struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
345
346	switch (nvkm_memory_target(pd->memory)) {
347	case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break;
348	case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0;
349		base |= BIT_ULL(2) /* VOL. */;
350		break;
351	case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break;
352	default:
353		WARN_ON(1);
354		return -EINVAL;
355	}
356	base |= pd->addr;
357
358	nvkm_kmap(inst);
359	nvkm_wo64(inst, 0x0200, base);
360	nvkm_wo64(inst, 0x0208, vmm->limit - 1);
361	nvkm_done(inst);
362	return 0;
363}
364
365int
366gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
367{
368	return gf100_vmm_join_(vmm, inst, 0);
369}
370
371static const struct nvkm_vmm_func
372gf100_vmm_17 = {
373	.join = gf100_vmm_join,
374	.part = gf100_vmm_part,
375	.aper = gf100_vmm_aper,
376	.valid = gf100_vmm_valid,
377	.flush = gf100_vmm_flush,
378	.invalidate_pdb = gf100_vmm_invalidate_pdb,
379	.page = {
380		{ 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC },
381		{ 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx },
382		{}
383	}
384};
385
386static const struct nvkm_vmm_func
387gf100_vmm_16 = {
388	.join = gf100_vmm_join,
389	.part = gf100_vmm_part,
390	.aper = gf100_vmm_aper,
391	.valid = gf100_vmm_valid,
392	.flush = gf100_vmm_flush,
393	.invalidate_pdb = gf100_vmm_invalidate_pdb,
394	.page = {
395		{ 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC },
396		{ 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx },
397		{}
398	}
399};
400
401int
402gf100_vmm_new_(const struct nvkm_vmm_func *func_16,
403	       const struct nvkm_vmm_func *func_17,
404	       struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
405	       void *argv, u32 argc, struct lock_class_key *key,
406	       const char *name, struct nvkm_vmm **pvmm)
407{
408	switch (mmu->subdev.device->fb->page) {
409	case 16: return nv04_vmm_new_(func_16, mmu, 0, managed, addr, size,
410				      argv, argc, key, name, pvmm);
411	case 17: return nv04_vmm_new_(func_17, mmu, 0, managed, addr, size,
412				      argv, argc, key, name, pvmm);
413	default:
414		WARN_ON(1);
415		return -EINVAL;
416	}
417}
418
419int
420gf100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
421	      void *argv, u32 argc, struct lock_class_key *key,
422	      const char *name, struct nvkm_vmm **pvmm)
423{
424	return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, managed, addr,
425			      size, argv, argc, key, name, pvmm);
426}
427