1/*
2 * Copyright 2022 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include "amdgpu.h"
24#include "amdgpu_xcp.h"
25#include "amdgpu_drv.h"
26
27#include <drm/drm_drv.h>
28#include "../amdxcp/amdgpu_xcp_drv.h"
29
30static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr,
31			    struct amdgpu_xcp_ip *xcp_ip, int xcp_state)
32{
33	int (*run_func)(void *handle, uint32_t inst_mask);
34	int ret = 0;
35
36	if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs)
37		return 0;
38
39	run_func = NULL;
40
41	switch (xcp_state) {
42	case AMDGPU_XCP_PREPARE_SUSPEND:
43		run_func = xcp_ip->ip_funcs->prepare_suspend;
44		break;
45	case AMDGPU_XCP_SUSPEND:
46		run_func = xcp_ip->ip_funcs->suspend;
47		break;
48	case AMDGPU_XCP_PREPARE_RESUME:
49		run_func = xcp_ip->ip_funcs->prepare_resume;
50		break;
51	case AMDGPU_XCP_RESUME:
52		run_func = xcp_ip->ip_funcs->resume;
53		break;
54	}
55
56	if (run_func)
57		ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask);
58
59	return ret;
60}
61
62static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
63				     int state)
64{
65	struct amdgpu_xcp_ip *xcp_ip;
66	struct amdgpu_xcp *xcp;
67	int i, ret;
68
69	if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid)
70		return -EINVAL;
71
72	xcp = &xcp_mgr->xcp[xcp_id];
73	for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) {
74		xcp_ip = &xcp->ip[i];
75		ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state);
76		if (ret)
77			break;
78	}
79
80	return ret;
81}
82
83int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
84{
85	return amdgpu_xcp_run_transition(xcp_mgr, xcp_id,
86					 AMDGPU_XCP_PREPARE_SUSPEND);
87}
88
89int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
90{
91	return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND);
92}
93
94int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
95{
96	return amdgpu_xcp_run_transition(xcp_mgr, xcp_id,
97					 AMDGPU_XCP_PREPARE_RESUME);
98}
99
100int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
101{
102	return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME);
103}
104
105static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
106				   struct amdgpu_xcp_ip *ip)
107{
108	struct amdgpu_xcp *xcp;
109
110	if (!ip)
111		return;
112
113	xcp = &xcp_mgr->xcp[xcp_id];
114	xcp->ip[ip->ip_id] = *ip;
115	xcp->ip[ip->ip_id].valid = true;
116
117	xcp->valid = true;
118}
119
120int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode)
121{
122	struct amdgpu_device *adev = xcp_mgr->adev;
123	struct amdgpu_xcp_ip ip;
124	uint8_t mem_id;
125	int i, j, ret;
126
127	if (!num_xcps || num_xcps > MAX_XCP)
128		return -EINVAL;
129
130	xcp_mgr->mode = mode;
131
132	for (i = 0; i < MAX_XCP; ++i)
133		xcp_mgr->xcp[i].valid = false;
134
135	/* This is needed for figuring out memory id of xcp */
136	xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions;
137
138	for (i = 0; i < num_xcps; ++i) {
139		for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) {
140			ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j,
141							     &ip);
142			if (ret)
143				continue;
144
145			__amdgpu_xcp_add_block(xcp_mgr, i, &ip);
146		}
147
148		xcp_mgr->xcp[i].id = i;
149
150		if (xcp_mgr->funcs->get_xcp_mem_id) {
151			ret = xcp_mgr->funcs->get_xcp_mem_id(
152				xcp_mgr, &xcp_mgr->xcp[i], &mem_id);
153			if (ret)
154				continue;
155			else
156				xcp_mgr->xcp[i].mem_id = mem_id;
157		}
158	}
159
160	xcp_mgr->num_xcps = num_xcps;
161	amdgpu_xcp_update_partition_sched_list(adev);
162
163	return 0;
164}
165
166static int __amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
167					      int mode)
168{
169	int ret, curr_mode, num_xcps = 0;
170
171	if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode)
172		return 0;
173
174	mutex_lock(&xcp_mgr->xcp_lock);
175
176	curr_mode = xcp_mgr->mode;
177	/* State set to transient mode */
178	xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS;
179
180	ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps);
181
182	if (ret) {
183		/* Failed, get whatever mode it's at now */
184		if (xcp_mgr->funcs->query_partition_mode)
185			xcp_mgr->mode = amdgpu_xcp_query_partition_mode(
186				xcp_mgr, AMDGPU_XCP_FL_LOCKED);
187		else
188			xcp_mgr->mode = curr_mode;
189
190		goto out;
191	}
192
193out:
194	mutex_unlock(&xcp_mgr->xcp_lock);
195
196	return ret;
197}
198
199int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode)
200{
201	if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE)
202		return -EINVAL;
203
204	if (xcp_mgr->mode == mode)
205		return 0;
206
207	return __amdgpu_xcp_switch_partition_mode(xcp_mgr, mode);
208}
209
210int amdgpu_xcp_restore_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr)
211{
212	if (!xcp_mgr || xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
213		return 0;
214
215	return __amdgpu_xcp_switch_partition_mode(xcp_mgr, xcp_mgr->mode);
216}
217
218int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
219{
220	int mode;
221
222	if (xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
223		return xcp_mgr->mode;
224
225	if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode)
226		return xcp_mgr->mode;
227
228	if (!(flags & AMDGPU_XCP_FL_LOCKED))
229		mutex_lock(&xcp_mgr->xcp_lock);
230	mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
231	if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode)
232		dev_WARN(
233			xcp_mgr->adev->dev,
234			"Cached partition mode %d not matching with device mode %d",
235			xcp_mgr->mode, mode);
236
237	if (!(flags & AMDGPU_XCP_FL_LOCKED))
238		mutex_unlock(&xcp_mgr->xcp_lock);
239
240	return mode;
241}
242
243static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev)
244{
245	struct drm_device *p_ddev;
246	struct drm_device *ddev;
247	int i, ret;
248
249	ddev = adev_to_drm(adev);
250
251	/* xcp #0 shares drm device setting with adev */
252	adev->xcp_mgr->xcp->ddev = ddev;
253
254	for (i = 1; i < MAX_XCP; i++) {
255		ret = amdgpu_xcp_drm_dev_alloc(&p_ddev);
256		if (ret == -ENOSPC) {
257			dev_warn(adev->dev,
258			"Skip xcp node #%d when out of drm node resource.", i);
259			return 0;
260		} else if (ret) {
261			return ret;
262		}
263
264		/* Redirect all IOCTLs to the primary device */
265		adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev;
266		adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev;
267		adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver;
268		adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager;
269		p_ddev->render->dev = ddev;
270		p_ddev->primary->dev = ddev;
271		p_ddev->vma_offset_manager = ddev->vma_offset_manager;
272		p_ddev->driver = &amdgpu_partition_driver;
273		adev->xcp_mgr->xcp[i].ddev = p_ddev;
274	}
275
276	return 0;
277}
278
279int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode,
280			int init_num_xcps,
281			struct amdgpu_xcp_mgr_funcs *xcp_funcs)
282{
283	struct amdgpu_xcp_mgr *xcp_mgr;
284
285	if (!xcp_funcs || !xcp_funcs->switch_partition_mode ||
286	    !xcp_funcs->get_ip_details)
287		return -EINVAL;
288
289	xcp_mgr = kzalloc(sizeof(*xcp_mgr), GFP_KERNEL);
290
291	if (!xcp_mgr)
292		return -ENOMEM;
293
294	xcp_mgr->adev = adev;
295	xcp_mgr->funcs = xcp_funcs;
296	xcp_mgr->mode = init_mode;
297	mutex_init(&xcp_mgr->xcp_lock);
298
299	if (init_mode != AMDGPU_XCP_MODE_NONE)
300		amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode);
301
302	adev->xcp_mgr = xcp_mgr;
303
304	return amdgpu_xcp_dev_alloc(adev);
305}
306
307int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr,
308			     enum AMDGPU_XCP_IP_BLOCK ip, int instance)
309{
310	struct amdgpu_xcp *xcp;
311	int i, id_mask = 0;
312
313	if (ip >= AMDGPU_XCP_MAX_BLOCKS)
314		return -EINVAL;
315
316	for (i = 0; i < xcp_mgr->num_xcps; ++i) {
317		xcp = &xcp_mgr->xcp[i];
318		if ((xcp->valid) && (xcp->ip[ip].valid) &&
319		    (xcp->ip[ip].inst_mask & BIT(instance)))
320			id_mask |= BIT(i);
321	}
322
323	if (!id_mask)
324		id_mask = -ENXIO;
325
326	return id_mask;
327}
328
329int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp,
330				enum AMDGPU_XCP_IP_BLOCK ip,
331				uint32_t *inst_mask)
332{
333	if (!xcp->valid || !inst_mask || !(xcp->ip[ip].valid))
334		return -EINVAL;
335
336	*inst_mask = xcp->ip[ip].inst_mask;
337
338	return 0;
339}
340
341int amdgpu_xcp_dev_register(struct amdgpu_device *adev,
342			const struct pci_device_id *ent)
343{
344	int i, ret;
345
346	if (!adev->xcp_mgr)
347		return 0;
348
349	for (i = 1; i < MAX_XCP; i++) {
350		if (!adev->xcp_mgr->xcp[i].ddev)
351			break;
352
353		ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data);
354		if (ret)
355			return ret;
356	}
357
358	return 0;
359}
360
361void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev)
362{
363	struct drm_device *p_ddev;
364	int i;
365
366	if (!adev->xcp_mgr)
367		return;
368
369	for (i = 1; i < MAX_XCP; i++) {
370		if (!adev->xcp_mgr->xcp[i].ddev)
371			break;
372
373		p_ddev = adev->xcp_mgr->xcp[i].ddev;
374		drm_dev_unplug(p_ddev);
375		p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev;
376		p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev;
377		p_ddev->driver =  adev->xcp_mgr->xcp[i].driver;
378		p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager;
379	}
380}
381
382int amdgpu_xcp_open_device(struct amdgpu_device *adev,
383			   struct amdgpu_fpriv *fpriv,
384			   struct drm_file *file_priv)
385{
386	int i;
387
388	if (!adev->xcp_mgr)
389		return 0;
390
391	fpriv->xcp_id = AMDGPU_XCP_NO_PARTITION;
392	for (i = 0; i < MAX_XCP; ++i) {
393		if (!adev->xcp_mgr->xcp[i].ddev)
394			break;
395
396		if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) {
397			if (adev->xcp_mgr->xcp[i].valid == FALSE) {
398				dev_err(adev->dev, "renderD%d partition %d not valid!",
399						file_priv->minor->index, i);
400				return -ENOENT;
401			}
402			dev_dbg(adev->dev, "renderD%d partition %d opened!",
403					file_priv->minor->index, i);
404			fpriv->xcp_id = i;
405			break;
406		}
407	}
408
409	fpriv->vm.mem_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? -1 :
410				adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id;
411	return 0;
412}
413
414void amdgpu_xcp_release_sched(struct amdgpu_device *adev,
415				  struct amdgpu_ctx_entity *entity)
416{
417	struct drm_gpu_scheduler *sched;
418	struct amdgpu_ring *ring;
419
420	if (!adev->xcp_mgr)
421		return;
422
423	sched = entity->entity.rq->sched;
424	if (sched->ready) {
425		ring = to_amdgpu_ring(entity->entity.rq->sched);
426		atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt);
427	}
428}
429
430