linux-master/lib/test_hmm.c

83 	struct dmirror			*dmirror;
90 struct dmirror {
147 dmirror_select_device(struct dmirror *dmirror)
149 	return (dmirror->mdevice->zone_device_type ==
163 	struct dmirror *dmirror;
167 	dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
168 	if (dmirror == NULL)
171 	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
172 	mutex_init(&dmirror->mutex);
173 	xa_init(&dmirror->pt);
175 	ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
178 		kfree(dmirror);
182 	filp->private_data = dmirror;
188 	struct dmirror *dmirror = filp->private_data;
190 	mmu_interval_notifier_remove(&dmirror->notifier);
191 	xa_destroy(&dmirror->pt);
192 	kfree(dmirror);
207 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
233 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
241 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
252 	xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
254 		xa_erase(&dmirror->pt, pfn);
261 	struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
268 	    range->owner == dmirror->mdevice)
272 		mutex_lock(&dmirror->mutex);
273 	else if (!mutex_trylock(&dmirror->mutex))
277 	dmirror_do_update(dmirror, range->start, range->end);
279 	mutex_unlock(&dmirror->mutex);
287 static int dmirror_range_fault(struct dmirror *dmirror,
290 	struct mm_struct *mm = dmirror->notifier.mm;
311 		mutex_lock(&dmirror->mutex);
314 			mutex_unlock(&dmirror->mutex);
320 	ret = dmirror_do_fault(dmirror, range);
322 	mutex_unlock(&dmirror->mutex);
327 static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
330 	struct mm_struct *mm = dmirror->notifier.mm;
334 		.notifier = &dmirror->notifier,
339 		.dev_private_owner = dmirror->mdevice,
351 		ret = dmirror_range_fault(dmirror, &range);
360 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
372 		entry = xa_load(&dmirror->pt, pfn);
386 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
403 		mutex_lock(&dmirror->mutex);
404 		ret = dmirror_do_read(dmirror, start, end, &bounce);
405 		mutex_unlock(&dmirror->mutex);
410 		ret = dmirror_fault(dmirror, start, end, false);
426 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
438 		entry = xa_load(&dmirror->pt, pfn);
452 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
474 		mutex_lock(&dmirror->mutex);
475 		ret = dmirror_do_write(dmirror, start, end, &bounce);
476 		mutex_unlock(&dmirror->mutex);
481 		ret = dmirror_fault(dmirror, start, end, true);
640 					   struct dmirror *dmirror)
642 	struct dmirror_device *mdevice = dmirror->mdevice;
682 		rpage->zone_device_data = dmirror;
693 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,
701 		entry = xa_load(&dmirror->pt, pfn);
710 			      struct page **pages, struct dmirror *dmirror)
716 	mutex_lock(&dmirror->mutex);
726 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
728 			mutex_unlock(&dmirror->mutex);
735 	mutex_unlock(&dmirror->mutex);
740 					    struct dmirror *dmirror)
749 	mutex_lock(&dmirror->mutex);
766 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
768 			mutex_unlock(&dmirror->mutex);
773 	mutex_unlock(&dmirror->mutex);
777 static int dmirror_exclusive(struct dmirror *dmirror,
782 	struct mm_struct *mm = dmirror->notifier.mm;
814 			mapped = dmirror_atomic_map(addr, next, pages, dmirror);
835 	mutex_lock(&dmirror->mutex);
836 	ret = dmirror_do_read(dmirror, start, end, &bounce);
837 	mutex_unlock(&dmirror->mutex);
850 						      struct dmirror *dmirror)
877 		xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
900 static int dmirror_migrate_to_system(struct dmirror *dmirror,
905 	struct mm_struct *mm = dmirror->notifier.mm;
939 		args.pgmap_owner = dmirror->mdevice;
940 		args.flags = dmirror_select_device(dmirror);
947 		dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
960 static int dmirror_migrate_to_device(struct dmirror *dmirror,
965 	struct mm_struct *mm = dmirror->notifier.mm;
999 		args.pgmap_owner = dmirror->mdevice;
1006 		dmirror_migrate_alloc_and_copy(&args, dmirror);
1008 		dmirror_migrate_finalize_and_map(&args, dmirror);
1021 	mutex_lock(&dmirror->mutex);
1022 	ret = dmirror_do_read(dmirror, start, end, &bounce);
1023 	mutex_unlock(&dmirror->mutex);
1039 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
1056 		if (dmirror->mdevice == dmirror_page_to_device(page))
1062 		if (dmirror->mdevice == dmirror_page_to_device(page))
1086 	struct dmirror *dmirror = dmi->dmirror;
1089 		mutex_lock(&dmirror->mutex);
1090 	else if (!mutex_trylock(&dmirror->mutex))
1099 	mutex_unlock(&dmirror->mutex);
1107 static int dmirror_range_snapshot(struct dmirror *dmirror,
1111 	struct mm_struct *mm = dmirror->notifier.mm;
1119 	notifier.dmirror = dmirror;
1145 		mutex_lock(&dmirror->mutex);
1148 			mutex_unlock(&dmirror->mutex);
1156 		dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
1158 	mutex_unlock(&dmirror->mutex);
1164 static int dmirror_snapshot(struct dmirror *dmirror,
1167 	struct mm_struct *mm = dmirror->notifier.mm;
1177 		.dev_private_owner = dmirror->mdevice,
1202 		ret = dmirror_range_snapshot(dmirror, &range, perm);
1305 	struct dmirror *dmirror;
1308 	dmirror = filp->private_data;
1309 	if (!dmirror)
1325 		ret = dmirror_read(dmirror, &cmd);
1329 		ret = dmirror_write(dmirror, &cmd);
1333 		ret = dmirror_migrate_to_device(dmirror, &cmd);
1337 		ret = dmirror_migrate_to_system(dmirror, &cmd);
1341 		ret = dmirror_exclusive(dmirror, &cmd);
1345 		ret = dmirror_check_atomic(dmirror, cmd.addr,
1350 		ret = dmirror_snapshot(dmirror, &cmd);
1354 		dmirror_device_remove_chunks(dmirror->mdevice);
1428 	struct dmirror *dmirror;
1437 	dmirror = rpage->zone_device_data;
1445 	args.pgmap_owner = dmirror->mdevice;
1446 	args.flags = dmirror_select_device(dmirror);
1452 	ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);