subdev/mmu/vmm.c

30 	struct nvkm_vmm_pt *pgt = *ppgt;
31 	if (pgt) {
32 		kvfree(pgt->pde);
33 		kfree(pgt);
44 	struct nvkm_vmm_pt *pgt;
56 	if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
58 	pgt->page = page ? page->shift : 0;
59 	pgt->sparse = sparse;
62 		pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
63 		if (!pgt->pde) {
64 			kfree(pgt);
69 	return pgt;
143 	struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
144 	struct nvkm_mmu_pt *pt = pgt->pt[type];
154 		pgt->pt[type] = NULL;
155 		if (!pgt->refs[!type]) {
158 				if (pgt->sparse) {
191 	if (!pgt->refs[!type])
192 		nvkm_vmm_pt_del(&pgt);
197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
211 		pgt->pte[lpti] -= pten;
216 	if (!pgt->refs[0])
221 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
223 				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
235 		pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
237 			if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
239 			pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
242 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
244 			pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
252 			pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
262 	struct nvkm_vmm_pt *pgt = it->pt[0];
267 		dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
272 			desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
277 	pgt->refs[type] -= ptes;
280 	if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
281 		nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
284 	if (!pgt->refs[type]) {
296 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
310 		pgt->pte[lpti] += pten;
315 	if (!pgt->refs[0])
320 		if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
322 				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
334 		pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
336 			if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
338 			pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
341 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
348 			desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
351 			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
358 			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
368 	struct nvkm_vmm_pt *pgt = it->pt[0];
371 	pgt->refs[type] += ptes;
375 		nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
382 		     struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
386 			pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
389 		memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
417 	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
418 	const bool zero = !pgt->sparse && !desc->func->invalid;
428 	pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
429 	if (!pgt->pt[type]) {
438 	pt = pgt->pt[type];
440 	if (desc->type == LPT && pgt->refs[1]) {
448 			bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
450 				bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
456 				if (pgt->sparse)
460 				memset(&pgt->pte[pteb], 0x00, ptes);
464 					pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
468 		if (pgt->sparse) {
469 			nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
487 	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
489 	pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
490 	if (!pgt) {
496 	pgd->pde[pdei] = pgt;
532 		struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
541 			struct nvkm_vmm_pt *pgd = pgt;
548 			it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
556 			if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
564 			struct nvkm_mmu_pt *pt = pgt->pt[type];