Lines Matching defs:mem
79 struct kgd_mem *mem)
83 list_for_each_entry(entry, &mem->attachments, list)
116 uint64_t mem;
122 mem = si.totalram - si.totalhigh;
123 mem *= si.mem_unit;
126 kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6);
257 "adev reference can't be null when alloc mem flags vram is set");
308 * @mem: BO of peer device that is being DMA mapped. Provides parameters
314 struct kgd_mem *mem, struct amdgpu_bo **bo_out)
320 ret = amdgpu_bo_reserve(mem->bo, false);
324 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
325 flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
328 ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1,
330 ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0);
332 amdgpu_bo_unreserve(mem->bo);
340 (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
499 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
504 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)
506 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)
545 kfd_mem_dmamap_userptr(struct kgd_mem *mem,
549 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
554 struct ttm_tt *src_ttm = mem->bo->tbo.ttm;
613 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
638 kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
651 mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
657 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
659 dma_addr = mem->bo->tbo.sg->sgl->dma_address;
660 pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
663 mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
669 ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
687 dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
693 kfd_mem_dmamap_attachment(struct kgd_mem *mem,
700 return kfd_mem_dmamap_userptr(mem, attachment);
704 return kfd_mem_dmamap_sg_bo(mem, attachment);
712 kfd_mem_dmaunmap_userptr(struct kgd_mem *mem,
716 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
746 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
761 kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
778 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
789 kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
796 kfd_mem_dmaunmap_userptr(mem, attachment);
802 kfd_mem_dmaunmap_sg_bo(mem, attachment);
809 static int kfd_mem_export_dmabuf(struct kgd_mem *mem)
811 if (!mem->dmabuf) {
816 bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);
818 mem->gem_handle,
819 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
827 mem->dmabuf = dmabuf;
834 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
840 ret = kfd_mem_export_dmabuf(mem);
844 gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf);
867 static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
870 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);
871 unsigned long bo_size = mem->bo->tbo.base.size;
872 uint64_t va = mem->va;
893 ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
894 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
895 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
896 if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
912 if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
913 (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) ||
914 (mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) ||
921 bo[i] = mem->bo;
928 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
931 ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
935 } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
936 WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
937 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
940 ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
944 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
945 mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
947 ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
977 attachment[i]->pte_flags = get_pte_flags(adev, mem);
979 list_add(&attachment[i]->list, &mem->attachments);
1017 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
1023 list_add_tail(&mem->validate_list,
1026 list_add_tail(&mem->validate_list, &process_info->kfd_bo_list);
1030 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
1034 list_del(&mem->validate_list);
1050 static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
1053 struct amdkfd_process_info *process_info = mem->process_info;
1054 struct amdgpu_bo *bo = mem->bo;
1082 mem->invalid++;
1099 amdgpu_bo_placement_from_domain(bo, mem->domain);
1137 * @mem: KFD BO structure.
1141 static int reserve_bo_and_vm(struct kgd_mem *mem,
1145 struct amdgpu_bo *bo = mem->bo;
1151 ctx->sync = &mem->sync;
1174 * @mem: KFD BO structure.
1182 static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
1187 struct amdgpu_bo *bo = mem->bo;
1190 ctx->sync = &mem->sync;
1194 list_for_each_entry(entry, &mem->attachments, list) {
1244 static void unmap_bo_from_gpuvm(struct kgd_mem *mem,
1259 static int update_gpuvm_pte(struct kgd_mem *mem,
1267 ret = kfd_mem_dmamap_attachment(mem, entry);
1281 static int map_bo_to_gpuvm(struct kgd_mem *mem,
1301 ret = update_gpuvm_pte(mem, entry, sync);
1310 unmap_bo_from_gpuvm(mem, entry, sync);
1311 kfd_mem_dmaunmap_attachment(mem, entry);
1685 void *drm_priv, struct kgd_mem **mem,
1750 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1751 if (!*mem) {
1755 INIT_LIST_HEAD(&(*mem)->attachments);
1756 mutex_init(&(*mem)->lock);
1757 (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1763 if ((*mem)->aql_queue)
1767 (*mem)->alloc_flags = flags;
1769 amdgpu_sync_create(&(*mem)->sync);
1779 va, (*mem)->aql_queue ? size << 1 : size,
1794 ret = drm_gem_handle_create(adev->kfd.client.file, gobj, &(*mem)->gem_handle);
1802 bo->kfd_bo = *mem;
1803 (*mem)->bo = bo;
1807 (*mem)->va = va;
1808 (*mem)->domain = domain;
1809 (*mem)->mapped_to_gpu_memory = 0;
1810 (*mem)->process_info = avm->process_info;
1812 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);
1816 ret = init_user_pages(*mem, user_addr, criu_resume);
1847 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
1848 drm_gem_handle_delete(adev->kfd.client.file, (*mem)->gem_handle);
1852 /* Don't unreserve system mem limit twice */
1857 amdgpu_sync_free(&(*mem)->sync);
1858 mutex_destroy(&(*mem)->lock);
1862 kfree(*mem);
1872 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
1875 struct amdkfd_process_info *process_info = mem->process_info;
1876 unsigned long bo_size = mem->bo->tbo.base.size;
1877 bool use_release_notifier = (mem->bo->kfd_bo == mem);
1884 mutex_lock(&mem->lock);
1887 if (mem->alloc_flags &
1890 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
1893 mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
1894 is_imported = mem->is_imported;
1895 mutex_unlock(&mem->lock);
1896 /* lock is not needed after this, since mem is unused and will
1902 mem->va, bo_size);
1908 list_del(&mem->validate_list);
1912 if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
1913 amdgpu_hmm_unregister(mem->bo);
1915 amdgpu_ttm_tt_discard_user_pages(mem->bo->tbo.ttm, mem->range);
1919 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
1923 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1925 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1926 mem->va + bo_size * (1 + mem->aql_queue));
1929 list_for_each_entry_safe(entry, tmp, &mem->attachments, list) {
1930 kfd_mem_dmaunmap_attachment(mem, entry);
1937 amdgpu_sync_free(&mem->sync);
1942 if (mem->bo->tbo.sg) {
1943 sg_free_table(mem->bo->tbo.sg);
1944 kfree(mem->bo->tbo.sg);
1953 (mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM ||
1955 mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT)))
1962 drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);
1963 drm_gem_handle_delete(adev->kfd.client.file, mem->gem_handle);
1964 if (mem->dmabuf) {
1965 dma_buf_put(mem->dmabuf);
1966 mem->dmabuf = NULL;
1968 mutex_destroy(&mem->lock);
1971 * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why
1974 drm_gem_object_put(&mem->bo->tbo.base);
1981 kfree(mem);
1987 struct amdgpu_device *adev, struct kgd_mem *mem,
1999 bo = mem->bo;
2009 mutex_lock(&mem->process_info->lock);
2016 mutex_lock(&mem->process_info->notifier_lock);
2017 is_invalid_userptr = !!mem->invalid;
2018 mutex_unlock(&mem->process_info->notifier_lock);
2021 mutex_lock(&mem->lock);
2023 domain = mem->domain;
2027 mem->va,
2028 mem->va + bo_size * (1 + mem->aql_queue),
2031 if (!kfd_mem_is_attached(avm, mem)) {
2032 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue);
2037 ret = reserve_bo_and_vm(mem, avm, &ctx);
2054 list_for_each_entry(entry, &mem->attachments, list) {
2061 ret = map_bo_to_gpuvm(mem, entry, ctx.sync,
2075 mem->mapped_to_gpu_memory++;
2077 mem->mapped_to_gpu_memory);
2087 mutex_unlock(&mem->process_info->lock);
2088 mutex_unlock(&mem->lock);
2092 int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv)
2100 mutex_lock(&mem->lock);
2102 ret = amdgpu_bo_reserve(mem->bo, true);
2106 list_for_each_entry(entry, &mem->attachments, list) {
2113 kfd_mem_dmaunmap_attachment(mem, entry);
2116 amdgpu_bo_unreserve(mem->bo);
2118 mutex_unlock(&mem->lock);
2124 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
2127 unsigned long bo_size = mem->bo->tbo.base.size;
2132 mutex_lock(&mem->lock);
2134 ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx);
2148 mem->va,
2149 mem->va + bo_size * (1 + mem->aql_queue),
2152 list_for_each_entry(entry, &mem->attachments, list) {
2159 unmap_bo_from_gpuvm(mem, entry, ctx.sync);
2162 mem->mapped_to_gpu_memory--;
2164 mem->mapped_to_gpu_memory);
2170 mutex_unlock(&mem->lock);
2175 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)
2182 mutex_lock(&mem->lock);
2183 amdgpu_sync_clone(&mem->sync, &sync);
2184 mutex_unlock(&mem->lock);
2240 * @mem: Buffer object to be mapped for CPU access
2251 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
2255 struct amdgpu_bo *bo = mem->bo;
2262 mutex_lock(&mem->process_info->lock);
2283 bo, mem->process_info->eviction_fence);
2290 mutex_unlock(&mem->process_info->lock);
2298 mutex_unlock(&mem->process_info->lock);
2305 * @mem: Buffer object to be unmapped for CPU access
2311 void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)
2313 struct amdgpu_bo *bo = mem->bo;
2322 struct kfd_vm_fault_info *mem)
2325 *mem = *adev->gmc.vm_fault_info;
2336 struct kgd_mem **mem, uint64_t *size,
2349 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2350 if (!*mem)
2363 INIT_LIST_HEAD(&(*mem)->attachments);
2364 mutex_init(&(*mem)->lock);
2366 (*mem)->alloc_flags =
2373 (*mem)->dmabuf = dma_buf;
2374 (*mem)->bo = bo;
2375 (*mem)->va = va;
2376 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && !adev->gmc.is_app_apu ?
2379 (*mem)->mapped_to_gpu_memory = 0;
2380 (*mem)->process_info = avm->process_info;
2381 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);
2382 amdgpu_sync_create(&(*mem)->sync);
2383 (*mem)->is_imported = true;
2388 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, (*mem)->domain,
2397 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
2400 kfree(*mem);
2406 struct kgd_mem **mem, uint64_t *size,
2423 ret = import_obj_create(adev, obj->dma_buf, obj, va, drm_priv, mem, size,
2428 (*mem)->gem_handle = handle;
2439 int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,
2444 mutex_lock(&mem->lock);
2445 ret = kfd_mem_export_dmabuf(mem);
2449 get_dma_buf(mem->dmabuf);
2450 *dma_buf = mem->dmabuf;
2452 mutex_unlock(&mem->lock);
2467 unsigned long cur_seq, struct kgd_mem *mem)
2469 struct amdkfd_process_info *process_info = mem->process_info;
2481 mem->invalid++;
2506 struct kgd_mem *mem, *tmp_mem;
2515 list_for_each_entry_safe(mem, tmp_mem,
2518 if (mem->invalid)
2519 list_move_tail(&mem->validate_list,
2523 list_for_each_entry(mem, &process_info->userptr_inval_list,
2525 invalid = mem->invalid;
2532 bo = mem->bo;
2534 amdgpu_ttm_tt_discard_user_pages(bo->tbo.ttm, mem->range);
2535 mem->range = NULL;
2560 &mem->range);
2582 if (mem->invalid != invalid) {
2586 /* set mem valid if mem has hmm range associated */
2587 if (mem->range)
2588 mem->invalid = 0;
2609 struct kgd_mem *mem, *tmp_mem;
2628 list_for_each_entry(mem, &process_info->userptr_inval_list,
2632 gobj = &mem->bo->tbo.base;
2645 list_for_each_entry_safe(mem, tmp_mem,
2650 bo = mem->bo;
2654 amdgpu_bo_placement_from_domain(bo, mem->domain);
2668 list_for_each_entry(attachment, &mem->attachments, list) {
2672 kfd_mem_dmaunmap_attachment(mem, attachment);
2673 ret = update_gpuvm_pte(mem, attachment, &sync);
2678 mem->invalid++;
2702 struct kgd_mem *mem, *tmp_mem;
2705 list_for_each_entry_safe(mem, tmp_mem,
2710 /* keep mem without hmm range at userptr_inval_list */
2711 if (!mem->range)
2714 /* Only check mem with hmm range associated */
2716 mem->bo->tbo.ttm, mem->range);
2718 mem->range = NULL;
2720 WARN(!mem->invalid, "Invalid BO not marked invalid");
2725 if (mem->invalid) {
2731 list_move_tail(&mem->validate_list,
2861 struct kgd_mem *mem;
2888 list_for_each_entry(mem, &process_info->kfd_bo_list,
2892 gobj = &mem->bo->tbo.base;
2905 list_for_each_entry(mem, &process_info->kfd_bo_list,
2908 struct amdgpu_bo *bo = mem->bo;
2909 uint32_t domain = mem->domain;
2949 list_for_each_entry(mem, &process_info->kfd_bo_list,
2953 list_for_each_entry(attachment, &mem->attachments, list) {
2960 kfd_mem_dmaunmap_attachment(mem, attachment);
2961 ret = update_gpuvm_pte(mem, attachment, &sync_obj);
3032 list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {
3033 if (mem->bo->tbo.pin_count)
3036 dma_resv_add_fence(mem->bo->tbo.base.resv,
3058 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
3067 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
3068 if (!*mem)
3071 mutex_init(&(*mem)->lock);
3072 INIT_LIST_HEAD(&(*mem)->attachments);
3073 (*mem)->bo = amdgpu_bo_ref(gws_bo);
3074 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
3075 (*mem)->process_info = process_info;
3076 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);
3077 amdgpu_sync_create(&(*mem)->sync);
3081 mutex_lock(&(*mem)->process_info->lock);
3104 mutex_unlock(&(*mem)->process_info->lock);
3112 mutex_unlock(&(*mem)->process_info->lock);
3113 amdgpu_sync_free(&(*mem)->sync);
3114 remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
3116 mutex_destroy(&(*mem)->lock);
3117 kfree(*mem);
3118 *mem = NULL;
3122 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
3126 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
3146 kfree(mem);
3170 bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem)
3174 list_for_each_entry(entry, &mem->attachments, list) {
3187 seq_printf(m, "System mem used %lldM out of %lluM\n",
3190 seq_printf(m, "TTM mem used %lldM out of %lluM\n",