amd/amdkfd/kfd_migrate.c

27 #include <linux/migrate.h>
181  * @mfence: migrate fence
263 static unsigned long svm_migrate_unsuccessful_pages(struct migrate_vma *migrate)
268 	for (i = 0; i < migrate->npages; i++) {
269 		if (migrate->src[i] & MIGRATE_PFN_VALID &&
270 		    !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
278 			 struct migrate_vma *migrate, struct dma_fence **mfence,
281 	uint64_t npages = migrate->cpages;
302 		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
303 		svm_migrate_get_vram_page(prange, migrate->dst[i]);
304 		migrate->dst[i] = migrate_pfn(migrate->dst[i]);
306 		spage = migrate_pfn_to_page(migrate->src[i]);
358 			migrate->dst[i] = 0;
367 		migrate->dst[i] = 0;
369 		migrate->dst[i + 1] = 0;
371 		migrate->dst[i + 2] = 0;
373 		migrate->dst[i + 3] = 0;
390 	struct migrate_vma migrate = { 0 };
397 	memset(&migrate, 0, sizeof(migrate));
398 	migrate.vma = vma;
399 	migrate.start = start;
400 	migrate.end = end;
401 	migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
402 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
405 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),
410 	migrate.src = buf;
411 	migrate.dst = migrate.src + npages;
412 	scratch = (dma_addr_t *)(migrate.dst + npages);
419 	r = migrate_vma_setup(&migrate);
426 	cpages = migrate.cpages;
428 		pr_debug("failed collect migrate sys pages [0x%lx 0x%lx]\n",
438 	r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);
439 	migrate_vma_pages(&migrate);
442 	migrate_vma_finalize(&migrate);
444 	mpages = cpages - svm_migrate_unsuccessful_pages(&migrate);
446 			 mpages, cpages, migrate.npages);
468  * svm_migrate_ram_to_vram - migrate svm range from system to device
470  * @best_loc: the device to migrate to
471  * @start_mgr: start page to migrate
472  * @last_mgr: last page to migrate
538 			pr_debug("failed %ld to migrate\n", r);
577 			struct migrate_vma *migrate, struct dma_fence **mfence,
591 	addr = migrate->start;
599 		spage = migrate_pfn_to_page(migrate->src[i]);
625 		dpage = svm_migrate_get_sys_page(migrate->vma, addr);
643 		migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
655 			migrate->dst[i] = 0;
663  * svm_migrate_vma_to_ram - migrate range inside one vma from device to system
669  * @node: kfd node device to migrate from
692 	struct migrate_vma migrate = { 0 };
697 	memset(&migrate, 0, sizeof(migrate));
698 	migrate.vma = vma;
699 	migrate.start = start;
700 	migrate.end = end;
701 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
703 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_COHERENT;
705 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
708 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),
713 	migrate.src = buf;
714 	migrate.dst = migrate.src + npages;
715 	migrate.fault_page = fault_page;
716 	scratch = (dma_addr_t *)(migrate.dst + npages);
723 	r = migrate_vma_setup(&migrate);
730 	cpages = migrate.cpages;
732 		pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",
734 		upages = svm_migrate_unsuccessful_pages(&migrate);
743 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,
745 	migrate_vma_pages(&migrate);
747 	upages = svm_migrate_unsuccessful_pages(&migrate);
749 		 upages, cpages, migrate.npages);
752 	migrate_vma_finalize(&migrate);
774  * svm_migrate_vram_to_ram - migrate svm range from device to system
799 	/* this pragne has no any vram page to migrate to sys ram */
838 			pr_debug("failed %ld to migrate prange %p\n", r, prange);
862  * svm_migrate_vram_to_vram - migrate svm range from device to device
864  * @best_loc: the device to migrate to
872  * migrate all vram pages in prange to sys ram, then migrate
991 		pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",