amd/amdkfd/kfd_migrate.c

213 svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
218 	svm_range_bo_ref(prange->svm_bo);
219 	page->zone_device_data = prange->svm_bo;
277 svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
290 	pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,
291 		 prange->last, ttm_res_offset);
296 	amdgpu_res_first(prange->ttm_res, ttm_res_offset,
303 		svm_migrate_get_vram_page(prange, migrate->dst[i]);
381 svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
385 	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
416 				      0, node->id, prange->prefetch_loc,
417 				      prange->preferred_loc, trigger);
422 			__func__, r, prange->start, prange->last);
429 			 prange->start, prange->last);
438 	r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);
458 		pdd = svm_range_get_pdd_by_node(prange, node);
469  * @prange: range structure
476  * Context: Process context, caller hold mmap read lock, svms lock, prange lock
482 svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
493 	if (start_mgr < prange->start || last_mgr > prange->last) {
494 		pr_debug("range [0x%lx 0x%lx] out prange [0x%lx 0x%lx]\n",
495 			 start_mgr, last_mgr, prange->start, prange->last);
499 	node = svm_range_get_node_by_id(prange, best_loc);
506 		prange->svms, start_mgr, last_mgr, prange->start, prange->last,
513 					prange->npages * PAGE_SIZE,
521 	r = svm_range_vram_node_new(node, prange, true);
526 	ttm_res_offset = (start_mgr - prange->start + prange->offset) << PAGE_SHIFT;
536 		r = svm_migrate_vma_to_vram(node, prange, vma, addr, next, trigger, ttm_res_offset);
548 		prange->actual_loc = best_loc;
549 		prange->vram_pages += mpages;
550 	} else if (!prange->actual_loc) {
551 		/* if no page migrated and all pages from prange are at
554 		svm_range_vram_node_free(prange);
559 					prange->npages * PAGE_SIZE,
576 svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
588 	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
589 		 prange->last);
602 				 prange->svms, prange->start, prange->last);
628 				 prange->svms, prange->start, prange->last);
665  * @prange: svm range structure
673  * Context: Process context, caller hold mmap read lock, prange->migrate_mutex
680 svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
684 	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
720 				      node->id, 0, prange->prefetch_loc,
721 				      prange->preferred_loc, trigger);
726 			__func__, r, prange->start, prange->last);
733 			 prange->start, prange->last);
743 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,
765 		pdd = svm_range_get_pdd_by_node(prange, node);
775  * @prange: range structure
782  * Context: Process context, caller hold mmap read lock, prange->migrate_mutex
787 int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,
800 	if (!prange->actual_loc) {
802 			 prange->start, prange->last);
806 	if (start_mgr < prange->start || last_mgr > prange->last) {
807 		pr_debug("range [0x%lx 0x%lx] out prange [0x%lx 0x%lx]\n",
808 			 start_mgr, last_mgr, prange->start, prange->last);
812 	node = svm_range_get_node_by_id(prange, prange->actual_loc);
814 		pr_debug("failed to get kfd node by id 0x%x\n", prange->actual_loc);
817 	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] from gpu 0x%x to ram\n",
818 		 prange->svms, prange, start_mgr, last_mgr,
819 		 prange->actual_loc);
829 			pr_debug("failed to find vma for prange %p\n", prange);
835 		r = svm_migrate_vma_to_ram(node, prange, vma, addr, next, trigger,
838 			pr_debug("failed %ld to migrate prange %p\n", r, prange);
847 		prange->vram_pages -= mpages;
849 		/* prange does not have vram page set its actual_loc to system
852 		if (prange->vram_pages == 0 && prange->ttm_res) {
853 			prange->actual_loc = 0;
854 			svm_range_vram_node_free(prange);
863  * @prange: range structure
870  * Context: Process context, caller hold mmap read lock, svms lock, prange lock
872  * migrate all vram pages in prange to sys ram, then migrate
879 svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
890 	pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);
893 		r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last,
897 	} while (prange->actual_loc && --retries);
899 	if (prange->actual_loc)
902 	return svm_migrate_ram_to_vram(prange, best_loc, start, last, mm, trigger);
906 svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
910 	if  (!prange->actual_loc || prange->actual_loc == best_loc)
911 		return svm_migrate_ram_to_vram(prange, best_loc, start, last,
915 		return svm_migrate_vram_to_vram(prange, best_loc, start, last,
935 	struct svm_range *prange;
971 	prange = svm_range_from_addr(&p->svms, addr, NULL);
972 	if (!prange) {
978 	mutex_lock(&prange->migrate_mutex);
980 	if (!prange->actual_loc)
984 	size = 1UL << prange->granularity;
985 	start = max(ALIGN_DOWN(addr, size), prange->start);
986 	last = min(ALIGN(addr + 1, size) - 1, prange->last);
988 	r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm, start, last,
992 			r, prange->svms, prange, start, last);
995 	mutex_unlock(&prange->migrate_mutex);