// SPDX-License-Identifier: MIT /* * Locking: * * The uvmm mutex protects any operations on the GPU VA space provided by the * DRM GPU VA manager. * * The GEMs dma_resv lock protects the GEMs GPUVA list, hence link/unlink of a * mapping to it's backing GEM must be performed under this lock. * * Actual map/unmap operations within the fence signalling critical path are * protected by installing DMA fences to the corresponding GEMs DMA * reservations, such that concurrent BO moves, which itself walk the GEMs GPUVA * list in order to map/unmap it's entries, can't occur concurrently. * * Accessing the DRM_GPUVA_INVALIDATED flag doesn't need any separate * protection, since there are no accesses other than from BO move callbacks * and from the fence signalling critical path, which are already protected by * the corresponding GEMs DMA reservation fence. */ #include "nouveau_drv.h" #include "nouveau_gem.h" #include "nouveau_mem.h" #include "nouveau_uvmm.h" #include #include #include #include #include #define NOUVEAU_VA_SPACE_BITS 47 /* FIXME */ #define NOUVEAU_VA_SPACE_START 0x0 #define NOUVEAU_VA_SPACE_END (1ULL << NOUVEAU_VA_SPACE_BITS) #define list_last_op(_ops) list_last_entry(_ops, struct bind_job_op, entry) #define list_prev_op(_op) list_prev_entry(_op, entry) #define list_for_each_op(_op, _ops) list_for_each_entry(_op, _ops, entry) #define list_for_each_op_from_reverse(_op, _ops) \ list_for_each_entry_from_reverse(_op, _ops, entry) #define list_for_each_op_safe(_op, _n, _ops) list_for_each_entry_safe(_op, _n, _ops, entry) enum vm_bind_op { OP_MAP = DRM_NOUVEAU_VM_BIND_OP_MAP, OP_UNMAP = DRM_NOUVEAU_VM_BIND_OP_UNMAP, OP_MAP_SPARSE, OP_UNMAP_SPARSE, }; struct nouveau_uvma_prealloc { struct nouveau_uvma *map; struct nouveau_uvma *prev; struct nouveau_uvma *next; }; struct bind_job_op { struct list_head entry; enum vm_bind_op op; u32 flags; struct drm_gpuvm_bo *vm_bo; struct { u64 addr; u64 range; } va; struct { u32 handle; u64 offset; struct drm_gem_object *obj; } gem; struct nouveau_uvma_region *reg; struct nouveau_uvma_prealloc new; struct drm_gpuva_ops *ops; }; struct uvmm_map_args { struct nouveau_uvma_region *region; u64 addr; u64 range; u8 kind; }; static int nouveau_uvmm_vmm_sparse_ref(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { struct nvif_vmm *vmm = &uvmm->vmm.vmm; return nvif_vmm_raw_sparse(vmm, addr, range, true); } static int nouveau_uvmm_vmm_sparse_unref(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { struct nvif_vmm *vmm = &uvmm->vmm.vmm; return nvif_vmm_raw_sparse(vmm, addr, range, false); } static int nouveau_uvmm_vmm_get(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { struct nvif_vmm *vmm = &uvmm->vmm.vmm; return nvif_vmm_raw_get(vmm, addr, range, PAGE_SHIFT); } static int nouveau_uvmm_vmm_put(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { struct nvif_vmm *vmm = &uvmm->vmm.vmm; return nvif_vmm_raw_put(vmm, addr, range, PAGE_SHIFT); } static int nouveau_uvmm_vmm_unmap(struct nouveau_uvmm *uvmm, u64 addr, u64 range, bool sparse) { struct nvif_vmm *vmm = &uvmm->vmm.vmm; return nvif_vmm_raw_unmap(vmm, addr, range, PAGE_SHIFT, sparse); } static int nouveau_uvmm_vmm_map(struct nouveau_uvmm *uvmm, u64 addr, u64 range, u64 bo_offset, u8 kind, struct nouveau_mem *mem) { struct nvif_vmm *vmm = &uvmm->vmm.vmm; union { struct gf100_vmm_map_v0 gf100; } args; u32 argc = 0; switch (vmm->object.oclass) { case NVIF_CLASS_VMM_GF100: case NVIF_CLASS_VMM_GM200: case NVIF_CLASS_VMM_GP100: args.gf100.version = 0; if (mem->mem.type & NVIF_MEM_VRAM) args.gf100.vol = 0; else args.gf100.vol = 1; args.gf100.ro = 0; args.gf100.priv = 0; args.gf100.kind = kind; argc = sizeof(args.gf100); break; default: WARN_ON(1); return -ENOSYS; } return nvif_vmm_raw_map(vmm, addr, range, PAGE_SHIFT, &args, argc, &mem->mem, bo_offset); } static int nouveau_uvma_region_sparse_unref(struct nouveau_uvma_region *reg) { u64 addr = reg->va.addr; u64 range = reg->va.range; return nouveau_uvmm_vmm_sparse_unref(reg->uvmm, addr, range); } static int nouveau_uvma_vmm_put(struct nouveau_uvma *uvma) { u64 addr = uvma->va.va.addr; u64 range = uvma->va.va.range; return nouveau_uvmm_vmm_put(to_uvmm(uvma), addr, range); } static int nouveau_uvma_map(struct nouveau_uvma *uvma, struct nouveau_mem *mem) { u64 addr = uvma->va.va.addr; u64 offset = uvma->va.gem.offset; u64 range = uvma->va.va.range; return nouveau_uvmm_vmm_map(to_uvmm(uvma), addr, range, offset, uvma->kind, mem); } static int nouveau_uvma_unmap(struct nouveau_uvma *uvma) { u64 addr = uvma->va.va.addr; u64 range = uvma->va.va.range; bool sparse = !!uvma->region; if (drm_gpuva_invalidated(&uvma->va)) return 0; return nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse); } static int nouveau_uvma_alloc(struct nouveau_uvma **puvma) { *puvma = kzalloc(sizeof(**puvma), GFP_KERNEL); if (!*puvma) return -ENOMEM; return 0; } static void nouveau_uvma_free(struct nouveau_uvma *uvma) { kfree(uvma); } static void nouveau_uvma_gem_get(struct nouveau_uvma *uvma) { drm_gem_object_get(uvma->va.gem.obj); } static void nouveau_uvma_gem_put(struct nouveau_uvma *uvma) { drm_gem_object_put(uvma->va.gem.obj); } static int nouveau_uvma_region_alloc(struct nouveau_uvma_region **preg) { *preg = kzalloc(sizeof(**preg), GFP_KERNEL); if (!*preg) return -ENOMEM; kref_init(&(*preg)->kref); return 0; } static void nouveau_uvma_region_free(struct kref *kref) { struct nouveau_uvma_region *reg = container_of(kref, struct nouveau_uvma_region, kref); kfree(reg); } static void nouveau_uvma_region_get(struct nouveau_uvma_region *reg) { kref_get(®->kref); } static void nouveau_uvma_region_put(struct nouveau_uvma_region *reg) { kref_put(®->kref, nouveau_uvma_region_free); } static int __nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm, struct nouveau_uvma_region *reg) { u64 addr = reg->va.addr; u64 range = reg->va.range; u64 last = addr + range - 1; MA_STATE(mas, &uvmm->region_mt, addr, addr); if (unlikely(mas_walk(&mas))) return -EEXIST; if (unlikely(mas.last < last)) return -EEXIST; mas.index = addr; mas.last = last; mas_store_gfp(&mas, reg, GFP_KERNEL); reg->uvmm = uvmm; return 0; } static int nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm, struct nouveau_uvma_region *reg, u64 addr, u64 range) { int ret; reg->uvmm = uvmm; reg->va.addr = addr; reg->va.range = range; ret = __nouveau_uvma_region_insert(uvmm, reg); if (ret) return ret; return 0; } static void nouveau_uvma_region_remove(struct nouveau_uvma_region *reg) { struct nouveau_uvmm *uvmm = reg->uvmm; MA_STATE(mas, &uvmm->region_mt, reg->va.addr, 0); mas_erase(&mas); } static int nouveau_uvma_region_create(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { struct nouveau_uvma_region *reg; int ret; if (!drm_gpuvm_interval_empty(&uvmm->base, addr, range)) return -ENOSPC; ret = nouveau_uvma_region_alloc(®); if (ret) return ret; ret = nouveau_uvma_region_insert(uvmm, reg, addr, range); if (ret) goto err_free_region; ret = nouveau_uvmm_vmm_sparse_ref(uvmm, addr, range); if (ret) goto err_region_remove; return 0; err_region_remove: nouveau_uvma_region_remove(reg); err_free_region: nouveau_uvma_region_put(reg); return ret; } static struct nouveau_uvma_region * nouveau_uvma_region_find_first(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { MA_STATE(mas, &uvmm->region_mt, addr, 0); return mas_find(&mas, addr + range - 1); } static struct nouveau_uvma_region * nouveau_uvma_region_find(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { struct nouveau_uvma_region *reg; reg = nouveau_uvma_region_find_first(uvmm, addr, range); if (!reg) return NULL; if (reg->va.addr != addr || reg->va.range != range) return NULL; return reg; } static bool nouveau_uvma_region_empty(struct nouveau_uvma_region *reg) { struct nouveau_uvmm *uvmm = reg->uvmm; return drm_gpuvm_interval_empty(&uvmm->base, reg->va.addr, reg->va.range); } static int __nouveau_uvma_region_destroy(struct nouveau_uvma_region *reg) { struct nouveau_uvmm *uvmm = reg->uvmm; u64 addr = reg->va.addr; u64 range = reg->va.range; if (!nouveau_uvma_region_empty(reg)) return -EBUSY; nouveau_uvma_region_remove(reg); nouveau_uvmm_vmm_sparse_unref(uvmm, addr, range); nouveau_uvma_region_put(reg); return 0; } static int nouveau_uvma_region_destroy(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { struct nouveau_uvma_region *reg; reg = nouveau_uvma_region_find(uvmm, addr, range); if (!reg) return -ENOENT; return __nouveau_uvma_region_destroy(reg); } static void nouveau_uvma_region_dirty(struct nouveau_uvma_region *reg) { init_completion(®->complete); reg->dirty = true; } static void nouveau_uvma_region_complete(struct nouveau_uvma_region *reg) { complete_all(®->complete); } static void op_map_prepare_unwind(struct nouveau_uvma *uvma) { struct drm_gpuva *va = &uvma->va; nouveau_uvma_gem_put(uvma); drm_gpuva_remove(va); nouveau_uvma_free(uvma); } static void op_unmap_prepare_unwind(struct drm_gpuva *va) { drm_gpuva_insert(va->vm, va); } static void nouveau_uvmm_sm_prepare_unwind(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops, struct drm_gpuva_op *last, struct uvmm_map_args *args) { struct drm_gpuva_op *op = last; u64 vmm_get_start = args ? args->addr : 0; u64 vmm_get_end = args ? args->addr + args->range : 0; /* Unwind GPUVA space. */ drm_gpuva_for_each_op_from_reverse(op, ops) { switch (op->op) { case DRM_GPUVA_OP_MAP: op_map_prepare_unwind(new->map); break; case DRM_GPUVA_OP_REMAP: { struct drm_gpuva_op_remap *r = &op->remap; struct drm_gpuva *va = r->unmap->va; if (r->next) op_map_prepare_unwind(new->next); if (r->prev) op_map_prepare_unwind(new->prev); op_unmap_prepare_unwind(va); break; } case DRM_GPUVA_OP_UNMAP: op_unmap_prepare_unwind(op->unmap.va); break; default: break; } } /* Unmap operation don't allocate page tables, hence skip the following * page table unwind. */ if (!args) return; drm_gpuva_for_each_op(op, ops) { switch (op->op) { case DRM_GPUVA_OP_MAP: { u64 vmm_get_range = vmm_get_end - vmm_get_start; if (vmm_get_range) nouveau_uvmm_vmm_put(uvmm, vmm_get_start, vmm_get_range); break; } case DRM_GPUVA_OP_REMAP: { struct drm_gpuva_op_remap *r = &op->remap; struct drm_gpuva *va = r->unmap->va; u64 ustart = va->va.addr; u64 urange = va->va.range; u64 uend = ustart + urange; if (r->prev) vmm_get_start = uend; if (r->next) vmm_get_end = ustart; if (r->prev && r->next) vmm_get_start = vmm_get_end = 0; break; } case DRM_GPUVA_OP_UNMAP: { struct drm_gpuva_op_unmap *u = &op->unmap; struct drm_gpuva *va = u->va; u64 ustart = va->va.addr; u64 urange = va->va.range; u64 uend = ustart + urange; /* Nothing to do for mappings we merge with. */ if (uend == vmm_get_start || ustart == vmm_get_end) break; if (ustart > vmm_get_start) { u64 vmm_get_range = ustart - vmm_get_start; nouveau_uvmm_vmm_put(uvmm, vmm_get_start, vmm_get_range); } vmm_get_start = uend; break; } default: break; } if (op == last) break; } } static void nouveau_uvmm_sm_map_prepare_unwind(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops, u64 addr, u64 range) { struct drm_gpuva_op *last = drm_gpuva_last_op(ops); struct uvmm_map_args args = { .addr = addr, .range = range, }; nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, &args); } static void nouveau_uvmm_sm_unmap_prepare_unwind(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops) { struct drm_gpuva_op *last = drm_gpuva_last_op(ops); nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, NULL); } static int op_map_prepare(struct nouveau_uvmm *uvmm, struct nouveau_uvma **puvma, struct drm_gpuva_op_map *op, struct uvmm_map_args *args) { struct nouveau_uvma *uvma; int ret; ret = nouveau_uvma_alloc(&uvma); if (ret) return ret; uvma->region = args->region; uvma->kind = args->kind; drm_gpuva_map(&uvmm->base, &uvma->va, op); /* Keep a reference until this uvma is destroyed. */ nouveau_uvma_gem_get(uvma); *puvma = uvma; return 0; } static void op_unmap_prepare(struct drm_gpuva_op_unmap *u) { drm_gpuva_unmap(u); } /* * Note: @args should not be NULL when calling for a map operation. */ static int nouveau_uvmm_sm_prepare(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops, struct uvmm_map_args *args) { struct drm_gpuva_op *op; u64 vmm_get_start = args ? args->addr : 0; u64 vmm_get_end = args ? args->addr + args->range : 0; int ret; drm_gpuva_for_each_op(op, ops) { switch (op->op) { case DRM_GPUVA_OP_MAP: { u64 vmm_get_range = vmm_get_end - vmm_get_start; ret = op_map_prepare(uvmm, &new->map, &op->map, args); if (ret) goto unwind; if (vmm_get_range) { ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start, vmm_get_range); if (ret) { op_map_prepare_unwind(new->map); goto unwind; } } break; } case DRM_GPUVA_OP_REMAP: { struct drm_gpuva_op_remap *r = &op->remap; struct drm_gpuva *va = r->unmap->va; struct uvmm_map_args remap_args = { .kind = uvma_from_va(va)->kind, .region = uvma_from_va(va)->region, }; u64 ustart = va->va.addr; u64 urange = va->va.range; u64 uend = ustart + urange; op_unmap_prepare(r->unmap); if (r->prev) { ret = op_map_prepare(uvmm, &new->prev, r->prev, &remap_args); if (ret) goto unwind; if (args) vmm_get_start = uend; } if (r->next) { ret = op_map_prepare(uvmm, &new->next, r->next, &remap_args); if (ret) { if (r->prev) op_map_prepare_unwind(new->prev); goto unwind; } if (args) vmm_get_end = ustart; } if (args && (r->prev && r->next)) vmm_get_start = vmm_get_end = 0; break; } case DRM_GPUVA_OP_UNMAP: { struct drm_gpuva_op_unmap *u = &op->unmap; struct drm_gpuva *va = u->va; u64 ustart = va->va.addr; u64 urange = va->va.range; u64 uend = ustart + urange; op_unmap_prepare(u); if (!args) break; /* Nothing to do for mappings we merge with. */ if (uend == vmm_get_start || ustart == vmm_get_end) break; if (ustart > vmm_get_start) { u64 vmm_get_range = ustart - vmm_get_start; ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start, vmm_get_range); if (ret) { op_unmap_prepare_unwind(va); goto unwind; } } vmm_get_start = uend; break; } default: ret = -EINVAL; goto unwind; } } return 0; unwind: if (op != drm_gpuva_first_op(ops)) nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, drm_gpuva_prev_op(op), args); return ret; } static int nouveau_uvmm_sm_map_prepare(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct nouveau_uvma_region *region, struct drm_gpuva_ops *ops, u64 addr, u64 range, u8 kind) { struct uvmm_map_args args = { .region = region, .addr = addr, .range = range, .kind = kind, }; return nouveau_uvmm_sm_prepare(uvmm, new, ops, &args); } static int nouveau_uvmm_sm_unmap_prepare(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops) { return nouveau_uvmm_sm_prepare(uvmm, new, ops, NULL); } static struct drm_gem_object * op_gem_obj(struct drm_gpuva_op *op) { switch (op->op) { case DRM_GPUVA_OP_MAP: return op->map.gem.obj; case DRM_GPUVA_OP_REMAP: /* Actually, we're looking for the GEMs backing remap.prev and * remap.next, but since this is a remap they're identical to * the GEM backing the unmapped GPUVA. */ return op->remap.unmap->va->gem.obj; case DRM_GPUVA_OP_UNMAP: return op->unmap.va->gem.obj; default: WARN(1, "Unknown operation.\n"); return NULL; } } static void op_map(struct nouveau_uvma *uvma) { struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj); nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource)); } static void op_unmap(struct drm_gpuva_op_unmap *u) { struct drm_gpuva *va = u->va; struct nouveau_uvma *uvma = uvma_from_va(va); /* nouveau_uvma_unmap() does not unmap if backing BO is evicted. */ if (!u->keep) nouveau_uvma_unmap(uvma); } static void op_unmap_range(struct drm_gpuva_op_unmap *u, u64 addr, u64 range) { struct nouveau_uvma *uvma = uvma_from_va(u->va); bool sparse = !!uvma->region; if (!drm_gpuva_invalidated(u->va)) nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse); } static void op_remap(struct drm_gpuva_op_remap *r, struct nouveau_uvma_prealloc *new) { struct drm_gpuva_op_unmap *u = r->unmap; struct nouveau_uvma *uvma = uvma_from_va(u->va); u64 addr = uvma->va.va.addr; u64 end = uvma->va.va.addr + uvma->va.va.range; if (r->prev) addr = r->prev->va.addr + r->prev->va.range; if (r->next) end = r->next->va.addr; op_unmap_range(u, addr, end - addr); } static int nouveau_uvmm_sm(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops) { struct drm_gpuva_op *op; drm_gpuva_for_each_op(op, ops) { switch (op->op) { case DRM_GPUVA_OP_MAP: op_map(new->map); break; case DRM_GPUVA_OP_REMAP: op_remap(&op->remap, new); break; case DRM_GPUVA_OP_UNMAP: op_unmap(&op->unmap); break; default: break; } } return 0; } static int nouveau_uvmm_sm_map(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops) { return nouveau_uvmm_sm(uvmm, new, ops); } static int nouveau_uvmm_sm_unmap(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops) { return nouveau_uvmm_sm(uvmm, new, ops); } static void nouveau_uvmm_sm_cleanup(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops, bool unmap) { struct drm_gpuva_op *op; drm_gpuva_for_each_op(op, ops) { switch (op->op) { case DRM_GPUVA_OP_MAP: break; case DRM_GPUVA_OP_REMAP: { struct drm_gpuva_op_remap *r = &op->remap; struct drm_gpuva_op_map *p = r->prev; struct drm_gpuva_op_map *n = r->next; struct drm_gpuva *va = r->unmap->va; struct nouveau_uvma *uvma = uvma_from_va(va); if (unmap) { u64 addr = va->va.addr; u64 end = addr + va->va.range; if (p) addr = p->va.addr + p->va.range; if (n) end = n->va.addr; nouveau_uvmm_vmm_put(uvmm, addr, end - addr); } nouveau_uvma_gem_put(uvma); nouveau_uvma_free(uvma); break; } case DRM_GPUVA_OP_UNMAP: { struct drm_gpuva_op_unmap *u = &op->unmap; struct drm_gpuva *va = u->va; struct nouveau_uvma *uvma = uvma_from_va(va); if (unmap) nouveau_uvma_vmm_put(uvma); nouveau_uvma_gem_put(uvma); nouveau_uvma_free(uvma); break; } default: break; } } } static void nouveau_uvmm_sm_map_cleanup(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops) { nouveau_uvmm_sm_cleanup(uvmm, new, ops, false); } static void nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm, struct nouveau_uvma_prealloc *new, struct drm_gpuva_ops *ops) { nouveau_uvmm_sm_cleanup(uvmm, new, ops, true); } static int nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range) { if (addr & ~PAGE_MASK) return -EINVAL; if (range & ~PAGE_MASK) return -EINVAL; if (!drm_gpuvm_range_valid(&uvmm->base, addr, range)) return -EINVAL; return 0; } static int nouveau_uvmm_bind_job_alloc(struct nouveau_uvmm_bind_job **pjob) { *pjob = kzalloc(sizeof(**pjob), GFP_KERNEL); if (!*pjob) return -ENOMEM; kref_init(&(*pjob)->kref); return 0; } static void nouveau_uvmm_bind_job_free(struct kref *kref) { struct nouveau_uvmm_bind_job *job = container_of(kref, struct nouveau_uvmm_bind_job, kref); struct bind_job_op *op, *next; list_for_each_op_safe(op, next, &job->ops) { list_del(&op->entry); kfree(op); } nouveau_job_free(&job->base); kfree(job); } static void nouveau_uvmm_bind_job_get(struct nouveau_uvmm_bind_job *job) { kref_get(&job->kref); } static void nouveau_uvmm_bind_job_put(struct nouveau_uvmm_bind_job *job) { kref_put(&job->kref, nouveau_uvmm_bind_job_free); } static int bind_validate_op(struct nouveau_job *job, struct bind_job_op *op) { struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); struct drm_gem_object *obj = op->gem.obj; if (op->op == OP_MAP) { if (op->gem.offset & ~PAGE_MASK) return -EINVAL; if (obj->size <= op->gem.offset) return -EINVAL; if (op->va.range > (obj->size - op->gem.offset)) return -EINVAL; } return nouveau_uvmm_validate_range(uvmm, op->va.addr, op->va.range); } static void bind_validate_map_sparse(struct nouveau_job *job, u64 addr, u64 range) { struct nouveau_sched *sched = job->sched; struct nouveau_job *__job; struct bind_job_op *op; u64 end = addr + range; again: spin_lock(&sched->job.list.lock); list_for_each_entry(__job, &sched->job.list.head, entry) { struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(__job); list_for_each_op(op, &bind_job->ops) { if (op->op == OP_UNMAP) { u64 op_addr = op->va.addr; u64 op_end = op_addr + op->va.range; if (!(end <= op_addr || addr >= op_end)) { nouveau_uvmm_bind_job_get(bind_job); spin_unlock(&sched->job.list.lock); wait_for_completion(&bind_job->complete); nouveau_uvmm_bind_job_put(bind_job); goto again; } } } } spin_unlock(&sched->job.list.lock); } static int bind_validate_map_common(struct nouveau_job *job, u64 addr, u64 range, bool sparse) { struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); struct nouveau_uvma_region *reg; u64 reg_addr, reg_end; u64 end = addr + range; again: nouveau_uvmm_lock(uvmm); reg = nouveau_uvma_region_find_first(uvmm, addr, range); if (!reg) { nouveau_uvmm_unlock(uvmm); return 0; } /* Generally, job submits are serialized, hence only * dirty regions can be modified concurrently. */ if (reg->dirty) { nouveau_uvma_region_get(reg); nouveau_uvmm_unlock(uvmm); wait_for_completion(®->complete); nouveau_uvma_region_put(reg); goto again; } nouveau_uvmm_unlock(uvmm); if (sparse) return -ENOSPC; reg_addr = reg->va.addr; reg_end = reg_addr + reg->va.range; /* Make sure the mapping is either outside of a * region or fully enclosed by a region. */ if (reg_addr > addr || reg_end < end) return -ENOSPC; return 0; } static int bind_validate_region(struct nouveau_job *job) { struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); struct bind_job_op *op; int ret; list_for_each_op(op, &bind_job->ops) { u64 op_addr = op->va.addr; u64 op_range = op->va.range; bool sparse = false; switch (op->op) { case OP_MAP_SPARSE: sparse = true; bind_validate_map_sparse(job, op_addr, op_range); fallthrough; case OP_MAP: ret = bind_validate_map_common(job, op_addr, op_range, sparse); if (ret) return ret; break; default: break; } } return 0; } static void bind_link_gpuvas(struct bind_job_op *bop) { struct nouveau_uvma_prealloc *new = &bop->new; struct drm_gpuvm_bo *vm_bo = bop->vm_bo; struct drm_gpuva_ops *ops = bop->ops; struct drm_gpuva_op *op; drm_gpuva_for_each_op(op, ops) { switch (op->op) { case DRM_GPUVA_OP_MAP: drm_gpuva_link(&new->map->va, vm_bo); break; case DRM_GPUVA_OP_REMAP: { struct drm_gpuva *va = op->remap.unmap->va; if (op->remap.prev) drm_gpuva_link(&new->prev->va, va->vm_bo); if (op->remap.next) drm_gpuva_link(&new->next->va, va->vm_bo); drm_gpuva_unlink(va); break; } case DRM_GPUVA_OP_UNMAP: drm_gpuva_unlink(op->unmap.va); break; default: break; } } } static int bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec, unsigned int num_fences) { struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); struct bind_job_op *op; int ret; list_for_each_op(op, &bind_job->ops) { struct drm_gpuva_op *va_op; if (!op->ops) continue; drm_gpuva_for_each_op(va_op, op->ops) { struct drm_gem_object *obj = op_gem_obj(va_op); if (unlikely(!obj)) continue; ret = drm_exec_prepare_obj(exec, obj, num_fences); if (ret) return ret; /* Don't validate GEMs backing mappings we're about to * unmap, it's not worth the effort. */ if (va_op->op == DRM_GPUVA_OP_UNMAP) continue; ret = nouveau_bo_validate(nouveau_gem_object(obj), true, false); if (ret) return ret; } } return 0; } static int nouveau_uvmm_bind_job_submit(struct nouveau_job *job, struct drm_gpuvm_exec *vme) { struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); struct drm_exec *exec = &vme->exec; struct bind_job_op *op; int ret; list_for_each_op(op, &bind_job->ops) { if (op->op == OP_MAP) { struct drm_gem_object *obj = op->gem.obj = drm_gem_object_lookup(job->file_priv, op->gem.handle); if (!obj) return -ENOENT; dma_resv_lock(obj->resv, NULL); op->vm_bo = drm_gpuvm_bo_obtain(&uvmm->base, obj); dma_resv_unlock(obj->resv); if (IS_ERR(op->vm_bo)) return PTR_ERR(op->vm_bo); drm_gpuvm_bo_extobj_add(op->vm_bo); } ret = bind_validate_op(job, op); if (ret) return ret; } /* If a sparse region or mapping overlaps a dirty region, we need to * wait for the region to complete the unbind process. This is due to * how page table management is currently implemented. A future * implementation might change this. */ ret = bind_validate_region(job); if (ret) return ret; /* Once we start modifying the GPU VA space we need to keep holding the * uvmm lock until we can't fail anymore. This is due to the set of GPU * VA space changes must appear atomically and we need to be able to * unwind all GPU VA space changes on failure. */ nouveau_uvmm_lock(uvmm); list_for_each_op(op, &bind_job->ops) { switch (op->op) { case OP_MAP_SPARSE: ret = nouveau_uvma_region_create(uvmm, op->va.addr, op->va.range); if (ret) goto unwind_continue; break; case OP_UNMAP_SPARSE: op->reg = nouveau_uvma_region_find(uvmm, op->va.addr, op->va.range); if (!op->reg || op->reg->dirty) { ret = -ENOENT; goto unwind_continue; } op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base, op->va.addr, op->va.range); if (IS_ERR(op->ops)) { ret = PTR_ERR(op->ops); goto unwind_continue; } ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new, op->ops); if (ret) { drm_gpuva_ops_free(&uvmm->base, op->ops); op->ops = NULL; op->reg = NULL; goto unwind_continue; } nouveau_uvma_region_dirty(op->reg); break; case OP_MAP: { struct nouveau_uvma_region *reg; reg = nouveau_uvma_region_find_first(uvmm, op->va.addr, op->va.range); if (reg) { u64 reg_addr = reg->va.addr; u64 reg_end = reg_addr + reg->va.range; u64 op_addr = op->va.addr; u64 op_end = op_addr + op->va.range; if (unlikely(reg->dirty)) { ret = -EINVAL; goto unwind_continue; } /* Make sure the mapping is either outside of a * region or fully enclosed by a region. */ if (reg_addr > op_addr || reg_end < op_end) { ret = -ENOSPC; goto unwind_continue; } } op->ops = drm_gpuvm_sm_map_ops_create(&uvmm->base, op->va.addr, op->va.range, op->gem.obj, op->gem.offset); if (IS_ERR(op->ops)) { ret = PTR_ERR(op->ops); goto unwind_continue; } ret = nouveau_uvmm_sm_map_prepare(uvmm, &op->new, reg, op->ops, op->va.addr, op->va.range, op->flags & 0xff); if (ret) { drm_gpuva_ops_free(&uvmm->base, op->ops); op->ops = NULL; goto unwind_continue; } break; } case OP_UNMAP: op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base, op->va.addr, op->va.range); if (IS_ERR(op->ops)) { ret = PTR_ERR(op->ops); goto unwind_continue; } ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new, op->ops); if (ret) { drm_gpuva_ops_free(&uvmm->base, op->ops); op->ops = NULL; goto unwind_continue; } break; default: ret = -EINVAL; goto unwind_continue; } } drm_exec_init(exec, vme->flags, 0); drm_exec_until_all_locked(exec) { ret = bind_lock_validate(job, exec, vme->num_fences); drm_exec_retry_on_contention(exec); if (ret) { op = list_last_op(&bind_job->ops); goto unwind; } } /* Link and unlink GPUVAs while holding the dma_resv lock. * * As long as we validate() all GEMs and add fences to all GEMs DMA * reservations backing map and remap operations we can be sure there * won't be any concurrent (in)validations during job execution, hence * we're safe to check drm_gpuva_invalidated() within the fence * signalling critical path without holding a separate lock. * * GPUVAs about to be unmapped are safe as well, since they're unlinked * already. * * GEMs from map and remap operations must be validated before linking * their corresponding mappings to prevent the actual PT update to * happen right away in validate() rather than asynchronously as * intended. * * Note that after linking and unlinking the GPUVAs in this loop this * function cannot fail anymore, hence there is no need for an unwind * path. */ list_for_each_op(op, &bind_job->ops) { switch (op->op) { case OP_UNMAP_SPARSE: case OP_MAP: case OP_UNMAP: bind_link_gpuvas(op); break; default: break; } } nouveau_uvmm_unlock(uvmm); return 0; unwind_continue: op = list_prev_op(op); unwind: list_for_each_op_from_reverse(op, &bind_job->ops) { switch (op->op) { case OP_MAP_SPARSE: nouveau_uvma_region_destroy(uvmm, op->va.addr, op->va.range); break; case OP_UNMAP_SPARSE: __nouveau_uvma_region_insert(uvmm, op->reg); nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new, op->ops); break; case OP_MAP: nouveau_uvmm_sm_map_prepare_unwind(uvmm, &op->new, op->ops, op->va.addr, op->va.range); break; case OP_UNMAP: nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new, op->ops); break; } drm_gpuva_ops_free(&uvmm->base, op->ops); op->ops = NULL; op->reg = NULL; } nouveau_uvmm_unlock(uvmm); drm_gpuvm_exec_unlock(vme); return ret; } static void nouveau_uvmm_bind_job_armed_submit(struct nouveau_job *job, struct drm_gpuvm_exec *vme) { drm_gpuvm_exec_resv_add_fence(vme, job->done_fence, job->resv_usage, job->resv_usage); drm_gpuvm_exec_unlock(vme); } static struct dma_fence * nouveau_uvmm_bind_job_run(struct nouveau_job *job) { struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); struct bind_job_op *op; int ret = 0; list_for_each_op(op, &bind_job->ops) { switch (op->op) { case OP_MAP_SPARSE: /* noop */ break; case OP_MAP: ret = nouveau_uvmm_sm_map(uvmm, &op->new, op->ops); if (ret) goto out; break; case OP_UNMAP_SPARSE: fallthrough; case OP_UNMAP: ret = nouveau_uvmm_sm_unmap(uvmm, &op->new, op->ops); if (ret) goto out; break; } } out: if (ret) NV_PRINTK(err, job->cli, "bind job failed: %d\n", ret); return ERR_PTR(ret); } static void nouveau_uvmm_bind_job_cleanup(struct nouveau_job *job) { struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); struct bind_job_op *op; list_for_each_op(op, &bind_job->ops) { struct drm_gem_object *obj = op->gem.obj; /* When nouveau_uvmm_bind_job_submit() fails op->ops and op->reg * will be NULL, hence skip the cleanup. */ switch (op->op) { case OP_MAP_SPARSE: /* noop */ break; case OP_UNMAP_SPARSE: if (!IS_ERR_OR_NULL(op->ops)) nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new, op->ops); if (op->reg) { nouveau_uvma_region_sparse_unref(op->reg); nouveau_uvmm_lock(uvmm); nouveau_uvma_region_remove(op->reg); nouveau_uvmm_unlock(uvmm); nouveau_uvma_region_complete(op->reg); nouveau_uvma_region_put(op->reg); } break; case OP_MAP: if (!IS_ERR_OR_NULL(op->ops)) nouveau_uvmm_sm_map_cleanup(uvmm, &op->new, op->ops); break; case OP_UNMAP: if (!IS_ERR_OR_NULL(op->ops)) nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new, op->ops); break; } if (!IS_ERR_OR_NULL(op->ops)) drm_gpuva_ops_free(&uvmm->base, op->ops); if (!IS_ERR_OR_NULL(op->vm_bo)) { dma_resv_lock(obj->resv, NULL); drm_gpuvm_bo_put(op->vm_bo); dma_resv_unlock(obj->resv); } if (obj) drm_gem_object_put(obj); } nouveau_job_done(job); complete_all(&bind_job->complete); nouveau_uvmm_bind_job_put(bind_job); } static struct nouveau_job_ops nouveau_bind_job_ops = { .submit = nouveau_uvmm_bind_job_submit, .armed_submit = nouveau_uvmm_bind_job_armed_submit, .run = nouveau_uvmm_bind_job_run, .free = nouveau_uvmm_bind_job_cleanup, }; static int bind_job_op_from_uop(struct bind_job_op **pop, struct drm_nouveau_vm_bind_op *uop) { struct bind_job_op *op; op = *pop = kzalloc(sizeof(*op), GFP_KERNEL); if (!op) return -ENOMEM; switch (uop->op) { case OP_MAP: op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ? OP_MAP_SPARSE : OP_MAP; break; case OP_UNMAP: op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ? OP_UNMAP_SPARSE : OP_UNMAP; break; default: op->op = uop->op; break; } op->flags = uop->flags; op->va.addr = uop->addr; op->va.range = uop->range; op->gem.handle = uop->handle; op->gem.offset = uop->bo_offset; return 0; } static void bind_job_ops_free(struct list_head *ops) { struct bind_job_op *op, *next; list_for_each_op_safe(op, next, ops) { list_del(&op->entry); kfree(op); } } static int nouveau_uvmm_bind_job_init(struct nouveau_uvmm_bind_job **pjob, struct nouveau_uvmm_bind_job_args *__args) { struct nouveau_uvmm_bind_job *job; struct nouveau_job_args args = {}; struct bind_job_op *op; int i, ret; ret = nouveau_uvmm_bind_job_alloc(&job); if (ret) return ret; INIT_LIST_HEAD(&job->ops); for (i = 0; i < __args->op.count; i++) { ret = bind_job_op_from_uop(&op, &__args->op.s[i]); if (ret) goto err_free; list_add_tail(&op->entry, &job->ops); } init_completion(&job->complete); args.file_priv = __args->file_priv; args.sched = __args->sched; args.credits = 1; args.in_sync.count = __args->in_sync.count; args.in_sync.s = __args->in_sync.s; args.out_sync.count = __args->out_sync.count; args.out_sync.s = __args->out_sync.s; args.sync = !(__args->flags & DRM_NOUVEAU_VM_BIND_RUN_ASYNC); args.ops = &nouveau_bind_job_ops; args.resv_usage = DMA_RESV_USAGE_BOOKKEEP; ret = nouveau_job_init(&job->base, &args); if (ret) goto err_free; *pjob = job; return 0; err_free: bind_job_ops_free(&job->ops); kfree(job); *pjob = NULL; return ret; } static int nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args) { struct nouveau_uvmm_bind_job *job; int ret; ret = nouveau_uvmm_bind_job_init(&job, args); if (ret) return ret; ret = nouveau_job_submit(&job->base); if (ret) goto err_job_fini; return 0; err_job_fini: nouveau_job_fini(&job->base); return ret; } static int nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args, struct drm_nouveau_vm_bind *req) { struct drm_nouveau_sync **s; u32 inc = req->wait_count; u64 ins = req->wait_ptr; u32 outc = req->sig_count; u64 outs = req->sig_ptr; u32 opc = req->op_count; u64 ops = req->op_ptr; int ret; args->flags = req->flags; if (opc) { args->op.count = opc; args->op.s = u_memcpya(ops, opc, sizeof(*args->op.s)); if (IS_ERR(args->op.s)) return PTR_ERR(args->op.s); } if (inc) { s = &args->in_sync.s; args->in_sync.count = inc; *s = u_memcpya(ins, inc, sizeof(**s)); if (IS_ERR(*s)) { ret = PTR_ERR(*s); goto err_free_ops; } } if (outc) { s = &args->out_sync.s; args->out_sync.count = outc; *s = u_memcpya(outs, outc, sizeof(**s)); if (IS_ERR(*s)) { ret = PTR_ERR(*s); goto err_free_ins; } } return 0; err_free_ops: u_free(args->op.s); err_free_ins: u_free(args->in_sync.s); return ret; } static void nouveau_uvmm_vm_bind_ufree(struct nouveau_uvmm_bind_job_args *args) { u_free(args->op.s); u_free(args->in_sync.s); u_free(args->out_sync.s); } int nouveau_uvmm_ioctl_vm_bind(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct nouveau_cli *cli = nouveau_cli(file_priv); struct nouveau_uvmm_bind_job_args args = {}; struct drm_nouveau_vm_bind *req = data; int ret = 0; if (unlikely(!nouveau_cli_uvmm_locked(cli))) return -ENOSYS; ret = nouveau_uvmm_vm_bind_ucopy(&args, req); if (ret) return ret; args.sched = cli->sched; args.file_priv = file_priv; ret = nouveau_uvmm_vm_bind(&args); if (ret) goto out_free_args; out_free_args: nouveau_uvmm_vm_bind_ufree(&args); return ret; } void nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem) { struct drm_gem_object *obj = &nvbo->bo.base; struct drm_gpuvm_bo *vm_bo; struct drm_gpuva *va; dma_resv_assert_held(obj->resv); drm_gem_for_each_gpuvm_bo(vm_bo, obj) { drm_gpuvm_bo_for_each_va(va, vm_bo) { struct nouveau_uvma *uvma = uvma_from_va(va); nouveau_uvma_map(uvma, mem); drm_gpuva_invalidate(va, false); } } } void nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo) { struct drm_gem_object *obj = &nvbo->bo.base; struct drm_gpuvm_bo *vm_bo; struct drm_gpuva *va; dma_resv_assert_held(obj->resv); drm_gem_for_each_gpuvm_bo(vm_bo, obj) { drm_gpuvm_bo_for_each_va(va, vm_bo) { struct nouveau_uvma *uvma = uvma_from_va(va); nouveau_uvma_unmap(uvma); drm_gpuva_invalidate(va, true); } } } static void nouveau_uvmm_free(struct drm_gpuvm *gpuvm) { struct nouveau_uvmm *uvmm = uvmm_from_gpuvm(gpuvm); kfree(uvmm); } static int nouveau_uvmm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) { struct nouveau_bo *nvbo = nouveau_gem_object(vm_bo->obj); return nouveau_bo_validate(nvbo, true, false); } static const struct drm_gpuvm_ops gpuvm_ops = { .vm_free = nouveau_uvmm_free, .vm_bo_validate = nouveau_uvmm_bo_validate, }; int nouveau_uvmm_ioctl_vm_init(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct nouveau_uvmm *uvmm; struct nouveau_cli *cli = nouveau_cli(file_priv); struct drm_device *drm = cli->drm->dev; struct drm_gem_object *r_obj; struct drm_nouveau_vm_init *init = data; u64 kernel_managed_end; int ret; if (check_add_overflow(init->kernel_managed_addr, init->kernel_managed_size, &kernel_managed_end)) return -EINVAL; if (kernel_managed_end > NOUVEAU_VA_SPACE_END) return -EINVAL; mutex_lock(&cli->mutex); if (unlikely(cli->uvmm.disabled)) { ret = -ENOSYS; goto out_unlock; } uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL); if (!uvmm) { ret = -ENOMEM; goto out_unlock; } r_obj = drm_gpuvm_resv_object_alloc(drm); if (!r_obj) { kfree(uvmm); ret = -ENOMEM; goto out_unlock; } mutex_init(&uvmm->mutex); mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN); mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex); drm_gpuvm_init(&uvmm->base, cli->name, 0, drm, r_obj, NOUVEAU_VA_SPACE_START, NOUVEAU_VA_SPACE_END, init->kernel_managed_addr, init->kernel_managed_size, &gpuvm_ops); /* GPUVM takes care from here on. */ drm_gem_object_put(r_obj); ret = nvif_vmm_ctor(&cli->mmu, "uvmm", cli->vmm.vmm.object.oclass, RAW, init->kernel_managed_addr, init->kernel_managed_size, NULL, 0, &uvmm->vmm.vmm); if (ret) goto out_gpuvm_fini; uvmm->vmm.cli = cli; cli->uvmm.ptr = uvmm; mutex_unlock(&cli->mutex); return 0; out_gpuvm_fini: drm_gpuvm_put(&uvmm->base); out_unlock: mutex_unlock(&cli->mutex); return ret; } void nouveau_uvmm_fini(struct nouveau_uvmm *uvmm) { MA_STATE(mas, &uvmm->region_mt, 0, 0); struct nouveau_uvma_region *reg; struct nouveau_cli *cli = uvmm->vmm.cli; struct drm_gpuva *va, *next; nouveau_uvmm_lock(uvmm); drm_gpuvm_for_each_va_safe(va, next, &uvmm->base) { struct nouveau_uvma *uvma = uvma_from_va(va); struct drm_gem_object *obj = va->gem.obj; if (unlikely(va == &uvmm->base.kernel_alloc_node)) continue; drm_gpuva_remove(va); dma_resv_lock(obj->resv, NULL); drm_gpuva_unlink(va); dma_resv_unlock(obj->resv); nouveau_uvma_unmap(uvma); nouveau_uvma_vmm_put(uvma); nouveau_uvma_gem_put(uvma); nouveau_uvma_free(uvma); } mas_for_each(&mas, reg, ULONG_MAX) { mas_erase(&mas); nouveau_uvma_region_sparse_unref(reg); nouveau_uvma_region_put(reg); } WARN(!mtree_empty(&uvmm->region_mt), "nouveau_uvma_region tree not empty, potentially leaking memory."); __mt_destroy(&uvmm->region_mt); nouveau_uvmm_unlock(uvmm); mutex_lock(&cli->mutex); nouveau_vmm_fini(&uvmm->vmm); drm_gpuvm_put(&uvmm->base); mutex_unlock(&cli->mutex); }