1// SPDX-License-Identifier: MIT
2/*
3 * Copyright �� 2022 Intel Corporation
4 */
5
6#include "xe_bo_evict.h"
7
8#include "xe_bo.h"
9#include "xe_device.h"
10#include "xe_ggtt.h"
11#include "xe_tile.h"
12
13/**
14 * xe_bo_evict_all - evict all BOs from VRAM
15 *
16 * @xe: xe device
17 *
18 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next
19 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU.
20 * All eviction magic done via TTM calls.
21 *
22 * Evict == move VRAM BOs to temporary (typically system) memory.
23 *
24 * This function should be called before the device goes into a suspend state
25 * where the VRAM loses power.
26 */
27int xe_bo_evict_all(struct xe_device *xe)
28{
29	struct ttm_device *bdev = &xe->ttm;
30	struct xe_bo *bo;
31	struct xe_tile *tile;
32	struct list_head still_in_list;
33	u32 mem_type;
34	u8 id;
35	int ret;
36
37	if (!IS_DGFX(xe))
38		return 0;
39
40	/* User memory */
41	for (mem_type = XE_PL_VRAM0; mem_type <= XE_PL_VRAM1; ++mem_type) {
42		struct ttm_resource_manager *man =
43			ttm_manager_type(bdev, mem_type);
44
45		if (man) {
46			ret = ttm_resource_manager_evict_all(bdev, man);
47			if (ret)
48				return ret;
49		}
50	}
51
52	/* Pinned user memory in VRAM */
53	INIT_LIST_HEAD(&still_in_list);
54	spin_lock(&xe->pinned.lock);
55	for (;;) {
56		bo = list_first_entry_or_null(&xe->pinned.external_vram,
57					      typeof(*bo), pinned_link);
58		if (!bo)
59			break;
60		xe_bo_get(bo);
61		list_move_tail(&bo->pinned_link, &still_in_list);
62		spin_unlock(&xe->pinned.lock);
63
64		xe_bo_lock(bo, false);
65		ret = xe_bo_evict_pinned(bo);
66		xe_bo_unlock(bo);
67		xe_bo_put(bo);
68		if (ret) {
69			spin_lock(&xe->pinned.lock);
70			list_splice_tail(&still_in_list,
71					 &xe->pinned.external_vram);
72			spin_unlock(&xe->pinned.lock);
73			return ret;
74		}
75
76		spin_lock(&xe->pinned.lock);
77	}
78	list_splice_tail(&still_in_list, &xe->pinned.external_vram);
79	spin_unlock(&xe->pinned.lock);
80
81	/*
82	 * Wait for all user BO to be evicted as those evictions depend on the
83	 * memory moved below.
84	 */
85	for_each_tile(tile, xe, id)
86		xe_tile_migrate_wait(tile);
87
88	spin_lock(&xe->pinned.lock);
89	for (;;) {
90		bo = list_first_entry_or_null(&xe->pinned.kernel_bo_present,
91					      typeof(*bo), pinned_link);
92		if (!bo)
93			break;
94		xe_bo_get(bo);
95		list_move_tail(&bo->pinned_link, &xe->pinned.evicted);
96		spin_unlock(&xe->pinned.lock);
97
98		xe_bo_lock(bo, false);
99		ret = xe_bo_evict_pinned(bo);
100		xe_bo_unlock(bo);
101		xe_bo_put(bo);
102		if (ret)
103			return ret;
104
105		spin_lock(&xe->pinned.lock);
106	}
107	spin_unlock(&xe->pinned.lock);
108
109	return 0;
110}
111
112/**
113 * xe_bo_restore_kernel - restore kernel BOs to VRAM
114 *
115 * @xe: xe device
116 *
117 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All
118 * moves done via TTM calls.
119 *
120 * This function should be called early, before trying to init the GT, on device
121 * resume.
122 */
123int xe_bo_restore_kernel(struct xe_device *xe)
124{
125	struct xe_bo *bo;
126	int ret;
127
128	if (!IS_DGFX(xe))
129		return 0;
130
131	spin_lock(&xe->pinned.lock);
132	for (;;) {
133		bo = list_first_entry_or_null(&xe->pinned.evicted,
134					      typeof(*bo), pinned_link);
135		if (!bo)
136			break;
137		xe_bo_get(bo);
138		list_move_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present);
139		spin_unlock(&xe->pinned.lock);
140
141		xe_bo_lock(bo, false);
142		ret = xe_bo_restore_pinned(bo);
143		xe_bo_unlock(bo);
144		if (ret) {
145			xe_bo_put(bo);
146			return ret;
147		}
148
149		if (bo->flags & XE_BO_CREATE_GGTT_BIT) {
150			struct xe_tile *tile = bo->tile;
151
152			mutex_lock(&tile->mem.ggtt->lock);
153			xe_ggtt_map_bo(tile->mem.ggtt, bo);
154			mutex_unlock(&tile->mem.ggtt->lock);
155		}
156
157		/*
158		 * We expect validate to trigger a move VRAM and our move code
159		 * should setup the iosys map.
160		 */
161		xe_assert(xe, !iosys_map_is_null(&bo->vmap));
162		xe_assert(xe, xe_bo_is_vram(bo));
163
164		xe_bo_put(bo);
165
166		spin_lock(&xe->pinned.lock);
167	}
168	spin_unlock(&xe->pinned.lock);
169
170	return 0;
171}
172
173/**
174 * xe_bo_restore_user - restore pinned user BOs to VRAM
175 *
176 * @xe: xe device
177 *
178 * Move pinned user BOs from temporary (typically system) memory to VRAM via
179 * CPU. All moves done via TTM calls.
180 *
181 * This function should be called late, after GT init, on device resume.
182 */
183int xe_bo_restore_user(struct xe_device *xe)
184{
185	struct xe_bo *bo;
186	struct xe_tile *tile;
187	struct list_head still_in_list;
188	u8 id;
189	int ret;
190
191	if (!IS_DGFX(xe))
192		return 0;
193
194	/* Pinned user memory in VRAM should be validated on resume */
195	INIT_LIST_HEAD(&still_in_list);
196	spin_lock(&xe->pinned.lock);
197	for (;;) {
198		bo = list_first_entry_or_null(&xe->pinned.external_vram,
199					      typeof(*bo), pinned_link);
200		if (!bo)
201			break;
202		list_move_tail(&bo->pinned_link, &still_in_list);
203		xe_bo_get(bo);
204		spin_unlock(&xe->pinned.lock);
205
206		xe_bo_lock(bo, false);
207		ret = xe_bo_restore_pinned(bo);
208		xe_bo_unlock(bo);
209		xe_bo_put(bo);
210		if (ret) {
211			spin_lock(&xe->pinned.lock);
212			list_splice_tail(&still_in_list,
213					 &xe->pinned.external_vram);
214			spin_unlock(&xe->pinned.lock);
215			return ret;
216		}
217
218		spin_lock(&xe->pinned.lock);
219	}
220	list_splice_tail(&still_in_list, &xe->pinned.external_vram);
221	spin_unlock(&xe->pinned.lock);
222
223	/* Wait for validate to complete */
224	for_each_tile(tile, xe, id)
225		xe_tile_migrate_wait(tile);
226
227	return 0;
228}
229