Lines Matching defs:prange
85 * @prange: svm range structure to be removed
92 static void svm_range_unlink(struct svm_range *prange)
94 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
95 prange, prange->start, prange->last);
97 if (prange->svm_bo) {
98 spin_lock(&prange->svm_bo->list_lock);
99 list_del(&prange->svm_bo_list);
100 spin_unlock(&prange->svm_bo->list_lock);
103 list_del(&prange->list);
104 if (prange->it_node.start != 0 && prange->it_node.last != 0)
105 interval_tree_remove(&prange->it_node, &prange->svms->objects);
109 svm_range_add_notifier_locked(struct mm_struct *mm, struct svm_range *prange)
111 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
112 prange, prange->start, prange->last);
114 mmu_interval_notifier_insert_locked(&prange->notifier, mm,
115 prange->start << PAGE_SHIFT,
116 prange->npages << PAGE_SHIFT,
122 * @prange: svm range structure to be added
128 static void svm_range_add_to_svms(struct svm_range *prange)
130 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
131 prange, prange->start, prange->last);
133 list_move_tail(&prange->list, &prange->svms->list);
134 prange->it_node.start = prange->start;
135 prange->it_node.last = prange->last;
136 interval_tree_insert(&prange->it_node, &prange->svms->objects);
139 static void svm_range_remove_notifier(struct svm_range *prange)
141 pr_debug("remove notifier svms 0x%p prange 0x%p [0x%lx 0x%lx]\n",
142 prange->svms, prange,
143 prange->notifier.interval_tree.start >> PAGE_SHIFT,
144 prange->notifier.interval_tree.last >> PAGE_SHIFT);
146 if (prange->notifier.interval_tree.start != 0 &&
147 prange->notifier.interval_tree.last != 0)
148 mmu_interval_notifier_remove(&prange->notifier);
159 svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,
164 dma_addr_t *addr = prange->dma_addr[gpuidx];
170 addr = kvcalloc(prange->npages, sizeof(*addr), GFP_KERNEL);
173 prange->dma_addr[gpuidx] = addr;
183 struct amdgpu_device *bo_adev = prange->svm_bo->node->adev;
206 svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap,
214 p = container_of(prange->svms, struct kfd_process, svms);
226 r = svm_range_dma_map_dev(pdd->dev->adev, prange, offset, npages,
253 void svm_range_dma_unmap(struct svm_range *prange)
261 p = container_of(prange->svms, struct kfd_process, svms);
264 dma_addr = prange->dma_addr[gpuidx];
275 svm_range_dma_unmap_dev(dev, dma_addr, 0, prange->npages);
279 static void svm_range_free(struct svm_range *prange, bool do_unmap)
281 uint64_t size = (prange->last - prange->start + 1) << PAGE_SHIFT;
282 struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
285 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange,
286 prange->start, prange->last);
288 svm_range_vram_node_free(prange);
290 svm_range_dma_unmap(prange);
293 pr_debug("unreserve prange 0x%p size: 0x%llx\n", prange, size);
300 if (prange->dma_addr[gpuidx]) {
301 kvfree(prange->dma_addr[gpuidx]);
302 prange->dma_addr[gpuidx] = NULL;
306 mutex_destroy(&prange->lock);
307 mutex_destroy(&prange->migrate_mutex);
308 kfree(prange);
327 struct svm_range *prange;
330 prange = kzalloc(sizeof(*prange), GFP_KERNEL);
331 if (!prange)
339 kfree(prange);
342 prange->npages = size;
343 prange->svms = svms;
344 prange->start = start;
345 prange->last = last;
346 INIT_LIST_HEAD(&prange->list);
347 INIT_LIST_HEAD(&prange->update_list);
348 INIT_LIST_HEAD(&prange->svm_bo_list);
349 INIT_LIST_HEAD(&prange->deferred_list);
350 INIT_LIST_HEAD(&prange->child_list);
351 atomic_set(&prange->invalid, 0);
352 prange->validate_timestamp = 0;
353 prange->vram_pages = 0;
354 mutex_init(&prange->migrate_mutex);
355 mutex_init(&prange->lock);
358 bitmap_copy(prange->bitmap_access, svms->bitmap_supported,
361 svm_range_set_default_attributes(&prange->preferred_loc,
362 &prange->prefetch_loc,
363 &prange->granularity, &prange->flags);
367 return prange;
387 struct svm_range *prange =
393 list_del_init(&prange->svm_bo_list);
396 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
397 prange->start, prange->last);
398 mutex_lock(&prange->lock);
399 prange->svm_bo = NULL;
400 /* prange should not hold vram page now */
401 WARN_ONCE(prange->actual_loc, "prange should not hold vram page");
402 mutex_unlock(&prange->lock);
445 svm_range_validate_svm_bo(struct kfd_node *node, struct svm_range *prange)
447 mutex_lock(&prange->lock);
448 if (!prange->svm_bo) {
449 mutex_unlock(&prange->lock);
452 if (prange->ttm_res) {
454 mutex_unlock(&prange->lock);
457 if (svm_bo_ref_unless_zero(prange->svm_bo)) {
463 if (prange->svm_bo->node != node) {
464 mutex_unlock(&prange->lock);
466 spin_lock(&prange->svm_bo->list_lock);
467 list_del_init(&prange->svm_bo_list);
468 spin_unlock(&prange->svm_bo->list_lock);
470 svm_range_bo_unref(prange->svm_bo);
473 if (READ_ONCE(prange->svm_bo->evicting)) {
479 mutex_unlock(&prange->lock);
480 svm_bo = prange->svm_bo;
482 svm_range_bo_unref(prange->svm_bo);
492 mutex_unlock(&prange->lock);
494 prange->svms, prange->start, prange->last);
496 prange->ttm_res = prange->svm_bo->bo->tbo.resource;
501 mutex_unlock(&prange->lock);
506 * its range list and set prange->svm_bo to null. After this,
509 while (!list_empty_careful(&prange->svm_bo_list) || prange->svm_bo)
531 svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
542 p = container_of(prange->svms, struct kfd_process, svms);
543 pr_debug("pasid: %x svms 0x%p [0x%lx 0x%lx]\n", p->pasid, prange->svms,
544 prange->start, prange->last);
546 if (svm_range_validate_svm_bo(node, prange))
569 bp.size = prange->npages * PAGE_SIZE;
617 prange->svm_bo = svm_bo;
618 prange->ttm_res = bo->tbo.resource;
619 prange->offset = 0;
622 list_add(&prange->svm_bo_list, &svm_bo->range_list);
632 prange->ttm_res = NULL;
637 void svm_range_vram_node_free(struct svm_range *prange)
639 /* serialize prange->svm_bo unref */
640 mutex_lock(&prange->lock);
641 /* prange->svm_bo has not been unref */
642 if (prange->ttm_res) {
643 prange->ttm_res = NULL;
644 mutex_unlock(&prange->lock);
645 svm_range_bo_unref(prange->svm_bo);
647 mutex_unlock(&prange->lock);
651 svm_range_get_node_by_id(struct svm_range *prange, uint32_t gpu_id)
656 p = container_of(prange->svms, struct kfd_process, svms);
667 svm_range_get_pdd_by_node(struct svm_range *prange, struct kfd_node *node)
671 p = container_of(prange->svms, struct kfd_process, svms);
735 svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange,
745 prange->preferred_loc = attrs[i].value;
748 prange->prefetch_loc = attrs[i].value;
759 bitmap_clear(prange->bitmap_access, gpuidx, 1);
760 bitmap_clear(prange->bitmap_aip, gpuidx, 1);
762 bitmap_set(prange->bitmap_access, gpuidx, 1);
763 bitmap_clear(prange->bitmap_aip, gpuidx, 1);
765 bitmap_clear(prange->bitmap_access, gpuidx, 1);
766 bitmap_set(prange->bitmap_aip, gpuidx, 1);
771 prange->flags |= attrs[i].value;
775 prange->flags &= ~attrs[i].value;
778 prange->granularity = min_t(uint32_t, attrs[i].value, 0x3F);
787 svm_range_is_same_attrs(struct kfd_process *p, struct svm_range *prange,
796 if (prange->preferred_loc != attrs[i].value)
810 if (test_bit(gpuidx, prange->bitmap_access) ||
811 test_bit(gpuidx, prange->bitmap_aip))
814 if (!test_bit(gpuidx, prange->bitmap_access))
817 if (!test_bit(gpuidx, prange->bitmap_aip))
822 if ((prange->flags & attrs[i].value) != attrs[i].value)
826 if ((prange->flags & attrs[i].value) != 0)
830 if (prange->granularity != attrs[i].value)
853 struct svm_range *prange;
858 list_for_each_entry(prange, &svms->list, list) {
860 prange, prange->start, prange->npages,
861 prange->start + prange->npages - 1,
862 prange->actual_loc);
869 prange = container_of(node, struct svm_range, it_node);
871 prange, prange->start, prange->npages,
872 prange->start + prange->npages - 1,
873 prange->actual_loc);
983 pr_debug("svms 0x%p new prange 0x%p start 0x%lx [0x%llx 0x%llx]\n",
1061 * @prange: the svm range to split
1067 * case 1: if start == prange->start
1068 * prange ==> prange[start, last]
1069 * new range [last + 1, prange->last]
1071 * case 2: if last == prange->last
1072 * prange ==> prange[start, last]
1073 * new range [prange->start, start - 1]
1079 svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last,
1082 uint64_t old_start = prange->start;
1083 uint64_t old_last = prange->last;
1087 pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms,
1095 svms = prange->svms;
1103 r = svm_range_split_adjust(*new, prange, start, last);
1115 svm_range_split_tail(struct svm_range *prange, uint64_t new_last,
1119 int r = svm_range_split(prange, prange->start, new_last, &tail);
1123 if (!IS_ALIGNED(new_last + 1, 1UL << prange->granularity))
1130 svm_range_split_head(struct svm_range *prange, uint64_t new_start,
1134 int r = svm_range_split(prange, new_start, prange->last, &head);
1138 if (!IS_ALIGNED(new_start, 1UL << prange->granularity))
1145 svm_range_add_child(struct svm_range *prange, struct mm_struct *mm,
1148 pr_debug("add child 0x%p [0x%lx 0x%lx] to prange 0x%p child list %d\n",
1149 pchild, pchild->start, pchild->last, prange, op);
1153 list_add_tail(&pchild->child_list, &prange->child_list);
1165 struct svm_range *prange, int domain)
1168 uint32_t flags = prange->flags;
1178 bo_node = prange->svm_bo->node;
1286 svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
1296 if (!prange->mapped_to_gpu) {
1297 pr_debug("prange 0x%p [0x%lx 0x%lx] not mapped to GPU\n",
1298 prange, prange->start, prange->last);
1302 if (prange->start == start && prange->last == last) {
1303 pr_debug("unmap svms 0x%p prange 0x%p\n", prange->svms, prange);
1304 prange->mapped_to_gpu = false;
1307 bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,
1309 p = container_of(prange->svms, struct kfd_process, svms);
1342 svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
1355 last_start = prange->start + offset;
1357 pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms,
1372 last_start, prange->start + i, last_domain ? "GPU" : "CPU");
1374 pte_flags = svm_range_get_pte_flags(pdd->dev, prange, last_domain);
1379 prange->svms, last_start, prange->start + i,
1388 NULL, last_start, prange->start + i,
1390 (last_start - prange->start) << PAGE_SHIFT,
1394 for (j = last_start - prange->start; j <= i; j++)
1398 pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start);
1401 last_start = prange->start + i + 1;
1407 prange->start);
1419 svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
1430 if (prange->svm_bo && prange->ttm_res)
1431 bo_adev = prange->svm_bo->node->adev;
1433 p = container_of(prange->svms, struct kfd_process, svms);
1452 r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,
1453 prange->dma_addr[gpuidx],
1477 struct svm_range *prange;
1557 * prange->migrate_mutex must be held.
1576 struct svm_range *prange, int32_t gpuidx,
1589 ctx->process = container_of(prange->svms, struct kfd_process, svms);
1590 ctx->prange = prange;
1597 bitmap_copy(ctx->bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);
1603 if (prange->actual_loc) {
1605 prange->actual_loc);
1608 prange->actual_loc);
1612 if (test_bit(gpuidx, prange->bitmap_access))
1617 * If prange is already mapped or with always mapped flag,
1621 if (prange->mapped_to_gpu ||
1622 prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)
1623 bitmap_copy(ctx->bitmap, prange->bitmap_access, MAX_GPU_INSTANCE);
1626 bitmap_or(ctx->bitmap, prange->bitmap_access,
1627 prange->bitmap_aip, MAX_GPU_INSTANCE);
1635 if (prange->actual_loc && !prange->ttm_res) {
1648 p = container_of(prange->svms, struct kfd_process, svms);
1677 r = amdgpu_hmm_range_get_pages(&prange->notifier, addr, npages,
1691 offset = (addr >> PAGE_SHIFT) - prange->start;
1692 r = svm_range_dma_map(prange, ctx->bitmap, offset, npages,
1698 svm_range_lock(prange);
1704 if (!r && !list_empty(&prange->child_list)) {
1710 map_start_vma = max(map_start, prange->start + offset);
1711 map_last_vma = min(map_last, prange->start + offset + npages - 1);
1713 offset = map_start_vma - prange->start;
1715 r = svm_range_map_to_gpus(prange, offset, npages, readonly,
1721 prange->mapped_to_gpu = true;
1723 svm_range_unlock(prange);
1730 prange->validate_timestamp = ktime_get_boottime();
1767 struct svm_range *prange;
1797 list_for_each_entry(prange, &svms->list, list) {
1798 invalid = atomic_read(&prange->invalid);
1802 pr_debug("restoring svms 0x%p prange 0x%p [0x%lx %lx] inv %d\n",
1803 prange->svms, prange, prange->start, prange->last,
1809 mutex_lock(&prange->migrate_mutex);
1811 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,
1815 prange->start);
1817 mutex_unlock(&prange->migrate_mutex);
1821 if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid)
1859 * @prange: svm range structure
1874 svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
1878 struct svm_range_list *svms = prange->svms;
1885 pr_debug("invalidate svms 0x%p prange [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
1886 svms, prange->start, prange->last, start, last);
1889 (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) {
1891 bool mapped = prange->mapped_to_gpu;
1893 list_for_each_entry(pchild, &prange->child_list, child_list) {
1909 if (prange->start <= last && prange->last >= start)
1910 atomic_inc(&prange->invalid);
1917 prange->svms, prange->start, prange->last);
1937 prange->svms, start, last);
1938 list_for_each_entry(pchild, &prange->child_list, child_list) {
1946 s = max(start, prange->start);
1947 l = min(last, prange->last);
1949 svm_range_unmap_from_gpus(prange, s, l, trigger);
2016 struct svm_range *prange;
2025 prange = svm_range_new(svms, start, l, true);
2026 if (!prange)
2028 list_add(&prange->list, insert_list);
2029 list_add(&prange->update_list, update_list);
2075 struct svm_range *prange;
2096 prange = container_of(node, struct svm_range, it_node);
2100 if (svm_range_is_same_attrs(p, prange, nattr, attrs) &&
2101 prange->mapped_to_gpu) {
2108 struct svm_range *old = prange;
2110 prange = svm_range_clone(old);
2111 if (!prange) {
2117 list_add(&prange->list, insert_list);
2118 list_add(&prange->update_list, update_list);
2122 r = svm_range_split_head(prange, start,
2129 r = svm_range_split_tail(prange, last,
2138 list_add(&prange->update_list, update_list);
2162 list_for_each_entry_safe(prange, tmp, insert_list, list)
2163 svm_range_free(prange, false);
2164 list_for_each_entry_safe(prange, tmp, &new_list, list)
2165 svm_range_free(prange, true);
2175 struct svm_range *prange)
2180 start = prange->notifier.interval_tree.start >> PAGE_SHIFT;
2181 last = prange->notifier.interval_tree.last >> PAGE_SHIFT;
2183 if (prange->start == start && prange->last == last)
2186 pr_debug("up notifier 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
2187 prange->svms, prange, start, last, prange->start,
2188 prange->last);
2191 interval_tree_remove(&prange->it_node, &prange->svms->objects);
2192 svm_range_remove_notifier(prange);
2194 prange->it_node.start = prange->start;
2195 prange->it_node.last = prange->last;
2197 interval_tree_insert(&prange->it_node, &prange->svms->objects);
2198 svm_range_add_notifier_locked(mm, prange);
2202 svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange,
2205 switch (prange->work_item.op) {
2207 pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n",
2208 svms, prange, prange->start, prange->last);
2211 pr_debug("remove 0x%p prange 0x%p [0x%lx 0x%lx]\n",
2212 svms, prange, prange->start, prange->last);
2213 svm_range_unlink(prange);
2214 svm_range_remove_notifier(prange);
2215 svm_range_free(prange, true);
2218 pr_debug("update notifier 0x%p prange 0x%p [0x%lx 0x%lx]\n",
2219 svms, prange, prange->start, prange->last);
2220 svm_range_update_notifier_and_interval_tree(mm, prange);
2223 pr_debug("update and map 0x%p prange 0x%p [0x%lx 0x%lx]\n",
2224 svms, prange, prange->start, prange->last);
2225 svm_range_update_notifier_and_interval_tree(mm, prange);
2229 pr_debug("add 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms, prange,
2230 prange->start, prange->last);
2231 svm_range_add_to_svms(prange);
2232 svm_range_add_notifier_locked(mm, prange);
2235 pr_debug("add and map 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms,
2236 prange, prange->start, prange->last);
2237 svm_range_add_to_svms(prange);
2238 svm_range_add_notifier_locked(mm, prange);
2242 WARN_ONCE(1, "Unknown prange 0x%p work op %d\n", prange,
2243 prange->work_item.op);
2287 struct svm_range *prange;
2295 prange = list_first_entry(&svms->deferred_range_list,
2299 pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange,
2300 prange->start, prange->last, prange->work_item.op);
2302 mm = prange->work_item.mm;
2324 list_del_init(&prange->deferred_list);
2328 mutex_lock(&prange->migrate_mutex);
2329 while (!list_empty(&prange->child_list)) {
2332 pchild = list_first_entry(&prange->child_list,
2334 pr_debug("child prange 0x%p op %d\n", pchild,
2339 mutex_unlock(&prange->migrate_mutex);
2341 svm_range_handle_list_op(svms, prange, mm);
2357 svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange,
2361 /* if prange is on the deferred list */
2362 if (!list_empty(&prange->deferred_list)) {
2363 pr_debug("update exist prange 0x%p work op %d\n", prange, op);
2364 WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n");
2366 prange->work_item.op != SVM_OP_UNMAP_RANGE)
2367 prange->work_item.op = op;
2369 prange->work_item.op = op;
2373 prange->work_item.mm = mm;
2374 list_add_tail(&prange->deferred_list,
2375 &prange->svms->deferred_range_list);
2376 pr_debug("add prange 0x%p [0x%lx 0x%lx] to work list op %d\n",
2377 prange, prange->start, prange->last, op);
2392 struct svm_range *prange, unsigned long start,
2398 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
2399 pr_debug("prange 0x%p [0x%lx 0x%lx] is already freed\n", prange,
2400 prange->start, prange->last);
2403 if (start > prange->last || last < prange->start)
2406 head = tail = prange;
2407 if (start > prange->start)
2408 svm_range_split(prange, prange->start, start - 1, &tail);
2412 if (head != prange && tail != prange) {
2415 } else if (tail != prange) {
2417 } else if (head != prange) {
2419 } else if (parent != prange) {
2420 prange->work_item.op = SVM_OP_UNMAP_RANGE;
2425 svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,
2440 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms,
2441 prange, prange->start, prange->last, start, last);
2449 unmap_parent = start <= prange->start && last >= prange->last;
2451 list_for_each_entry(pchild, &prange->child_list, child_list) {
2457 svm_range_unmap_split(mm, prange, pchild, start, last);
2460 s = max(start, prange->start);
2461 l = min(last, prange->last);
2463 svm_range_unmap_from_gpus(prange, s, l, trigger);
2464 svm_range_unmap_split(mm, prange, prange, start, last);
2467 svm_range_add_list_work(svms, prange, mm, SVM_OP_UNMAP_RANGE);
2469 svm_range_add_list_work(svms, prange, mm,
2485 * For unmap event, unmap range from GPUs, remove prange from svms in a delayed
2486 * work thread, and split prange if only part of prange is unmapped.
2494 * for invalidate event, prange lock is held if this is from migration
2501 struct svm_range *prange;
2520 prange = container_of(mni, struct svm_range, notifier);
2522 svm_range_lock(prange);
2527 svm_range_unmap_from_cpu(mni->mm, prange, start, last);
2530 svm_range_evict(prange, mni->mm, start, last, range->event);
2534 svm_range_unlock(prange);
2555 struct svm_range *prange;
2562 prange = container_of(node, struct svm_range, it_node);
2563 pr_debug("address 0x%lx prange [0x%lx 0x%lx] node [0x%lx 0x%lx]\n",
2564 addr, prange->start, prange->last, node->start, node->last);
2566 if (addr >= prange->start && addr <= prange->last) {
2568 *parent = prange;
2569 return prange;
2571 list_for_each_entry(pchild, &prange->child_list, child_list)
2576 *parent = prange;
2584 * @prange: svm range structure
2605 svm_range_best_restore_location(struct svm_range *prange,
2614 p = container_of(prange->svms, struct kfd_process, svms);
2625 if (prange->preferred_loc == gpuid ||
2626 prange->preferred_loc == KFD_IOCTL_SVM_LOCATION_SYSMEM) {
2627 return prange->preferred_loc;
2628 } else if (prange->preferred_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED) {
2629 preferred_node = svm_range_get_node_by_id(prange, prange->preferred_loc);
2631 return prange->preferred_loc;
2635 if (test_bit(*gpuidx, prange->bitmap_access))
2638 if (test_bit(*gpuidx, prange->bitmap_aip)) {
2639 if (!prange->actual_loc)
2642 bo_node = svm_range_get_node_by_id(prange, prange->actual_loc);
2644 return prange->actual_loc;
2762 struct svm_range *prange = NULL;
2787 prange = svm_range_new(&p->svms, start, last, true);
2788 if (!prange) {
2789 pr_debug("Failed to create prange in address [0x%llx]\n", addr);
2794 svm_range_free(prange, true);
2799 prange->preferred_loc = KFD_IOCTL_SVM_LOCATION_SYSMEM;
2801 svm_range_add_to_svms(prange);
2802 svm_range_add_notifier_locked(mm, prange);
2804 return prange;
2807 /* svm_range_skip_recover - decide if prange can be recovered
2808 * @prange: svm range structure
2811 * 1. prange is on deferred list to be removed after unmap, it is stale fault,
2812 * deferred list work will drain the stale fault before free the prange.
2813 * 2. prange is on deferred list to add interval notifier after split, or
2814 * 3. prange is child range, it is split from parent prange, recover later
2819 static bool svm_range_skip_recover(struct svm_range *prange)
2821 struct svm_range_list *svms = prange->svms;
2824 if (list_empty(&prange->deferred_list) &&
2825 list_empty(&prange->child_list)) {
2831 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
2832 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] unmapped\n",
2833 svms, prange, prange->start, prange->last);
2836 if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP ||
2837 prange->work_item.op == SVM_OP_ADD_RANGE) {
2838 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] not added yet\n",
2839 svms, prange, prange->start, prange->last);
2893 struct svm_range *prange;
2950 prange = svm_range_from_addr(svms, addr, NULL);
2951 if (!prange) {
2952 pr_debug("failed to find prange svms 0x%p address [0x%llx]\n",
2965 prange = svm_range_create_unregistered_range(node, p, mm, addr);
2966 if (!prange) {
2977 mutex_lock(&prange->migrate_mutex);
2979 if (svm_range_skip_recover(prange)) {
2986 if (ktime_before(timestamp, ktime_add_ns(prange->validate_timestamp,
2989 svms, prange->start, prange->last);
3011 best_loc = svm_range_best_restore_location(prange, node, &gpuidx);
3014 svms, prange->start, prange->last);
3020 svms, prange->start, prange->last, best_loc,
3021 prange->actual_loc);
3027 size = 1UL << prange->granularity;
3028 start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);
3029 last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);
3030 if (prange->actual_loc != 0 || best_loc != 0) {
3034 r = svm_migrate_to_vram(prange, best_loc, start, last,
3042 if (prange->actual_loc && prange->actual_loc != best_loc)
3043 r = svm_migrate_vram_to_ram(prange, mm, start, last,
3049 r = svm_migrate_vram_to_ram(prange, mm, start, last,
3059 r = svm_range_validate_and_map(mm, start, last, prange, gpuidx, false,
3069 mutex_unlock(&prange->migrate_mutex);
3091 struct svm_range *prange, *pchild;
3100 list_for_each_entry(prange, &p->svms.list, list) {
3101 svm_range_lock(prange);
3102 list_for_each_entry(pchild, &prange->child_list, child_list) {
3116 size = (prange->last - prange->start + 1) << PAGE_SHIFT;
3128 svm_range_unlock(prange);
3148 struct svm_range *prange;
3165 list_for_each_entry_safe(prange, next, &p->svms.list, list) {
3166 svm_range_unlink(prange);
3167 svm_range_remove_notifier(prange);
3168 svm_range_free(prange, true);
3294 * @prange: svm range structure
3319 svm_range_best_prefetch_location(struct svm_range *prange)
3322 uint32_t best_loc = prange->prefetch_loc;
3328 p = container_of(prange->svms, struct kfd_process, svms);
3333 bo_node = svm_range_get_node_by_id(prange, best_loc);
3346 bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);
3348 bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,
3369 p->xnack_enabled, &p->svms, prange->start, prange->last,
3377 * @prange: svm range structure
3390 * a. svm_range_validate_vram takes prange->migrate_mutex
3400 svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange,
3407 best_loc = svm_range_best_prefetch_location(prange);
3409 /* when best_loc is a gpu node and same as prange->actual_loc
3410 * we still need do migration as prange->actual_loc !=0 does
3411 * not mean all pages in prange are vram. hmm migrate will pick
3415 (best_loc == 0 && prange->actual_loc == 0))
3419 r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last,
3425 r = svm_migrate_to_vram(prange, best_loc, prange->start, prange->last,
3467 struct svm_range *prange =
3472 list_del_init(&prange->svm_bo_list);
3475 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
3476 prange->start, prange->last);
3478 mutex_lock(&prange->migrate_mutex);
3480 /* migrate all vram pages in this prange to sys ram
3481 * after that prange->actual_loc should be zero
3483 r = svm_migrate_vram_to_ram(prange, mm,
3484 prange->start, prange->last,
3486 } while (!r && prange->actual_loc && --retries);
3488 if (!r && prange->actual_loc)
3491 if (!prange->actual_loc) {
3492 mutex_lock(&prange->lock);
3493 prange->svm_bo = NULL;
3494 mutex_unlock(&prange->lock);
3496 mutex_unlock(&prange->migrate_mutex);
3524 struct svm_range *prange;
3561 list_for_each_entry_safe(prange, next, &insert_list, list) {
3562 svm_range_add_to_svms(prange);
3563 svm_range_add_notifier_locked(mm, prange);
3565 list_for_each_entry(prange, &update_list, update_list) {
3566 svm_range_apply_attrs(p, prange, nattr, attrs, &update_mapping);
3569 list_for_each_entry_safe(prange, next, &remove_list, update_list) {
3570 pr_debug("unlink old 0x%p prange 0x%p [0x%lx 0x%lx]\n",
3571 prange->svms, prange, prange->start,
3572 prange->last);
3573 svm_range_unlink(prange);
3574 svm_range_remove_notifier(prange);
3575 svm_range_free(prange, false);
3584 list_for_each_entry(prange, &update_list, update_list) {
3587 mutex_lock(&prange->migrate_mutex);
3589 r = svm_range_trigger_migration(mm, prange, &migrated);
3594 (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) &&
3595 prange->mapped_to_gpu) {
3597 mutex_unlock(&prange->migrate_mutex);
3602 mutex_unlock(&prange->migrate_mutex);
3606 flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu;
3608 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,
3614 mutex_unlock(&prange->migrate_mutex);
3619 list_for_each_entry(prange, &remap_list, update_list) {
3620 pr_debug("Remapping prange 0x%p [0x%lx 0x%lx]\n",
3621 prange, prange->start, prange->last);
3622 mutex_lock(&prange->migrate_mutex);
3623 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,
3624 MAX_GPU_INSTANCE, true, true, prange->mapped_to_gpu);
3627 mutex_unlock(&prange->migrate_mutex);
3661 struct svm_range *prange;
3740 prange = container_of(node, struct svm_range, it_node);
3744 if (prange->preferred_loc ==
3747 location != prange->preferred_loc)) {
3751 location = prange->preferred_loc;
3755 if (prange->prefetch_loc ==
3758 prefetch_loc != prange->prefetch_loc)) {
3762 prefetch_loc = prange->prefetch_loc;
3767 prange->bitmap_access, MAX_GPU_INSTANCE);
3769 prange->bitmap_aip, MAX_GPU_INSTANCE);
3772 flags_and &= prange->flags;
3773 flags_or |= prange->flags;
3776 if (get_granularity && prange->granularity < granularity)
3777 granularity = prange->granularity;
3985 struct svm_range *prange;
3995 list_for_each_entry(prange, &svms->list, list) {
3996 pr_debug("prange: 0x%p start: 0x%lx\t npages: 0x%llx\t end: 0x%llx\n",
3997 prange, prange->start, prange->npages,
3998 prange->start + prange->npages - 1);
4047 struct svm_range *prange;
4091 list_for_each_entry(prange, &svms->list, list) {
4094 svm_priv->start_addr = prange->start;
4095 svm_priv->size = prange->npages;
4097 pr_debug("CRIU: prange: 0x%p start: 0x%lx\t npages: 0x%llx end: 0x%llx\t size: 0x%llx\n",
4098 prange, prange->start, prange->npages,
4099 prange->start + prange->npages - 1,
4100 prange->npages * PAGE_SIZE);