1/*
2 * Copyright 2012 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24#include "priv.h"
25#include "chan.h"
26#include "chid.h"
27#include "runl.h"
28#include "runq.h"
29
30#include <core/gpuobj.h>
31#include <subdev/bar.h>
32#include <subdev/mc.h>
33#include <subdev/mmu.h>
34
35#include <nvif/cl0080.h>
36#include <nvif/unpack.h>
37
38bool
39nvkm_fifo_ctxsw_in_progress(struct nvkm_engine *engine)
40{
41	struct nvkm_runl *runl;
42	struct nvkm_engn *engn;
43
44	nvkm_runl_foreach(runl, engine->subdev.device->fifo) {
45		nvkm_runl_foreach_engn(engn, runl) {
46			if (engn->engine == engine)
47				return engn->func->chsw ? engn->func->chsw(engn) : false;
48		}
49	}
50
51	return false;
52}
53
54void
55nvkm_fifo_pause(struct nvkm_fifo *fifo, unsigned long *flags)
56{
57	return fifo->func->pause(fifo, flags);
58}
59
60void
61nvkm_fifo_start(struct nvkm_fifo *fifo, unsigned long *flags)
62{
63	return fifo->func->start(fifo, flags);
64}
65
66void
67nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info)
68{
69	return fifo->func->mmu_fault->recover(fifo, info);
70}
71
72static int
73nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass,
74		    void *argv, u32 argc, struct nvkm_object **pobject)
75{
76	struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine);
77
78	if (oclass->engn == &fifo->func->cgrp.user)
79		return nvkm_ucgrp_new(fifo, oclass, argv, argc, pobject);
80
81	if (oclass->engn == &fifo->func->chan.user)
82		return nvkm_uchan_new(fifo, NULL, oclass, argv, argc, pobject);
83
84	WARN_ON(1);
85	return -ENOSYS;
86}
87
88static const struct nvkm_device_oclass
89nvkm_fifo_class = {
90	.ctor = nvkm_fifo_class_new,
91};
92
93static int
94nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class)
95{
96	struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine);
97	const struct nvkm_fifo_func_cgrp *cgrp = &fifo->func->cgrp;
98	const struct nvkm_fifo_func_chan *chan = &fifo->func->chan;
99	int c = 0;
100
101	/* *_CHANNEL_GROUP_* */
102	if (cgrp->user.oclass) {
103		if (c++ == index) {
104			oclass->base = cgrp->user;
105			oclass->engn = &fifo->func->cgrp.user;
106			*class = &nvkm_fifo_class;
107			return 0;
108		}
109	}
110
111	/* *_CHANNEL_DMA, *_CHANNEL_GPFIFO_* */
112	if (chan->user.oclass) {
113		if (c++ == index) {
114			oclass->base = chan->user;
115			oclass->engn = &fifo->func->chan.user;
116			*class = &nvkm_fifo_class;
117			return 0;
118		}
119	}
120
121	return c;
122}
123
124static int
125nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend)
126{
127	struct nvkm_fifo *fifo = nvkm_fifo(engine);
128	struct nvkm_runl *runl;
129
130	nvkm_inth_block(&fifo->engine.subdev.inth);
131
132	nvkm_runl_foreach(runl, fifo)
133		nvkm_runl_fini(runl);
134
135	return 0;
136}
137
138static int
139nvkm_fifo_init(struct nvkm_engine *engine)
140{
141	struct nvkm_fifo *fifo = nvkm_fifo(engine);
142	struct nvkm_runq *runq;
143	struct nvkm_runl *runl;
144	u32 mask = 0;
145
146	if (fifo->func->init_pbdmas) {
147		nvkm_runq_foreach(runq, fifo)
148			mask |= BIT(runq->id);
149
150		fifo->func->init_pbdmas(fifo, mask);
151
152		nvkm_runq_foreach(runq, fifo)
153			runq->func->init(runq);
154	}
155
156	nvkm_runl_foreach(runl, fifo) {
157		if (runl->func->init)
158			runl->func->init(runl);
159	}
160
161	if (fifo->func->init)
162		fifo->func->init(fifo);
163
164	nvkm_inth_allow(&fifo->engine.subdev.inth);
165	return 0;
166}
167
168static int
169nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data)
170{
171	struct nvkm_fifo *fifo = nvkm_fifo(engine);
172	struct nvkm_runl *runl;
173	struct nvkm_engn *engn;
174	int ret;
175
176	ret = nvkm_subdev_oneinit(&fifo->engine.subdev);
177	if (ret)
178		return ret;
179
180	switch (mthd) {
181	case NV_DEVICE_HOST_CHANNELS: *data = fifo->chid ? fifo->chid->nr : 0; return 0;
182	case NV_DEVICE_HOST_RUNLISTS:
183		*data = 0;
184		nvkm_runl_foreach(runl, fifo)
185			*data |= BIT(runl->id);
186		return 0;
187	case NV_DEVICE_HOST_RUNLIST_ENGINES:
188		runl = nvkm_runl_get(fifo, *data, 0);
189		if (runl) {
190			*data = 0;
191			nvkm_runl_foreach_engn(engn, runl) {
192#define CASE(n) case NVKM_ENGINE_##n: *data |= NV_DEVICE_HOST_RUNLIST_ENGINES_##n; break
193				switch (engn->engine->subdev.type) {
194				case NVKM_ENGINE_DMAOBJ:
195					break;
196				CASE(SW    );
197				CASE(GR    );
198				CASE(MPEG  );
199				CASE(ME    );
200				CASE(CIPHER);
201				CASE(BSP   );
202				CASE(VP    );
203				CASE(CE    );
204				CASE(SEC   );
205				CASE(MSVLD );
206				CASE(MSPDEC);
207				CASE(MSPPP );
208				CASE(MSENC );
209				CASE(VIC   );
210				CASE(SEC2  );
211				CASE(NVDEC );
212				CASE(NVENC );
213				CASE(NVJPG );
214				CASE(OFA   );
215				default:
216					WARN_ON(1);
217					break;
218				}
219#undef CASE
220			}
221			return 0;
222		}
223		return -EINVAL;
224	case NV_DEVICE_HOST_RUNLIST_CHANNELS:
225		if (!fifo->chid) {
226			runl = nvkm_runl_get(fifo, *data, 0);
227			if (runl) {
228				*data = runl->chid->nr;
229				return 0;
230			}
231		}
232		return -EINVAL;
233	default:
234		break;
235	}
236
237	return -ENOSYS;
238}
239
240static int
241nvkm_fifo_oneinit(struct nvkm_engine *engine)
242{
243	struct nvkm_subdev *subdev = &engine->subdev;
244	struct nvkm_device *device = subdev->device;
245	struct nvkm_fifo *fifo = nvkm_fifo(engine);
246	struct nvkm_runl *runl;
247	struct nvkm_engn *engn;
248	int ret, nr, i;
249
250	/* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */
251	if (fifo->func->chid_nr) {
252		ret = fifo->func->chid_ctor(fifo, fifo->func->chid_nr(fifo));
253		if (ret)
254			return ret;
255	}
256
257	/* Create runqueues for each PBDMA. */
258	if (fifo->func->runq_nr) {
259		for (nr = fifo->func->runq_nr(fifo), i = 0; i < nr; i++) {
260			if (!nvkm_runq_new(fifo, i))
261				return -ENOMEM;
262		}
263	}
264
265	/* Create runlists. */
266	ret = fifo->func->runl_ctor(fifo);
267	if (ret)
268		return ret;
269
270	nvkm_runl_foreach(runl, fifo) {
271		RUNL_DEBUG(runl, "chan:%06x", runl->chan);
272		nvkm_runl_foreach_engn(engn, runl) {
273			ENGN_DEBUG(engn, "");
274		}
275	}
276
277	/* Register interrupt handler. */
278	if (fifo->func->intr) {
279		ret = nvkm_inth_add(&device->mc->intr, NVKM_INTR_SUBDEV, NVKM_INTR_PRIO_NORMAL,
280				    subdev, fifo->func->intr, &subdev->inth);
281		if (ret) {
282			nvkm_error(subdev, "intr %d\n", ret);
283			return ret;
284		}
285	}
286
287	/* Initialise non-stall intr handling. */
288	if (fifo->func->nonstall) {
289		if (fifo->func->nonstall_ctor) {
290			ret = fifo->func->nonstall_ctor(fifo);
291			if (ret < 0) {
292				nvkm_error(subdev, "nonstall %d\n", ret);
293				return ret;
294			}
295		} else {
296			ret = 1;
297		}
298
299		ret = nvkm_event_init(fifo->func->nonstall, &fifo->engine.subdev, 1, ret,
300				      &fifo->nonstall.event);
301		if (ret)
302			return ret;
303	}
304
305	/* Allocate USERD + BAR1 polling area. */
306	if (fifo->func->chan.func->userd->bar == 1) {
307		struct nvkm_vmm *bar1 = nvkm_bar_bar1_vmm(device);
308
309		ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, fifo->chid->nr *
310				      fifo->func->chan.func->userd->size, 0, true,
311				      &fifo->userd.mem);
312		if (ret)
313			return ret;
314
315		ret = nvkm_vmm_get(bar1, 12, nvkm_memory_size(fifo->userd.mem), &fifo->userd.bar1);
316		if (ret)
317			return ret;
318
319		ret = nvkm_memory_map(fifo->userd.mem, 0, bar1, fifo->userd.bar1, NULL, 0);
320		if (ret)
321			return ret;
322	}
323
324	return 0;
325}
326
327static void
328nvkm_fifo_preinit(struct nvkm_engine *engine)
329{
330	nvkm_mc_reset(engine->subdev.device, NVKM_ENGINE_FIFO, 0);
331}
332
333static void *
334nvkm_fifo_dtor(struct nvkm_engine *engine)
335{
336	struct nvkm_fifo *fifo = nvkm_fifo(engine);
337	struct nvkm_runl *runl, *runt;
338	struct nvkm_runq *runq, *rtmp;
339
340	if (fifo->userd.bar1)
341		nvkm_vmm_put(nvkm_bar_bar1_vmm(engine->subdev.device), &fifo->userd.bar1);
342	nvkm_memory_unref(&fifo->userd.mem);
343
344	list_for_each_entry_safe(runl, runt, &fifo->runls, head)
345		nvkm_runl_del(runl);
346	list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head)
347		nvkm_runq_del(runq);
348
349	nvkm_chid_unref(&fifo->cgid);
350	nvkm_chid_unref(&fifo->chid);
351
352	mutex_destroy(&fifo->userd.mutex);
353
354	nvkm_event_fini(&fifo->nonstall.event);
355	mutex_destroy(&fifo->mutex);
356
357	if (fifo->func->dtor)
358		fifo->func->dtor(fifo);
359
360	return fifo;
361}
362
363static const struct nvkm_engine_func
364nvkm_fifo = {
365	.dtor = nvkm_fifo_dtor,
366	.preinit = nvkm_fifo_preinit,
367	.oneinit = nvkm_fifo_oneinit,
368	.info = nvkm_fifo_info,
369	.init = nvkm_fifo_init,
370	.fini = nvkm_fifo_fini,
371	.base.sclass = nvkm_fifo_class_get,
372};
373
374int
375nvkm_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device,
376	       enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo)
377{
378	struct nvkm_fifo *fifo;
379
380	if (!(fifo = *pfifo = kzalloc(sizeof(*fifo), GFP_KERNEL)))
381		return -ENOMEM;
382
383	fifo->func = func;
384	INIT_LIST_HEAD(&fifo->runqs);
385	INIT_LIST_HEAD(&fifo->runls);
386	/*TODO: Needs to be >CTXSW_TIMEOUT, so RC can recover before this is hit.
387	 *      CTXSW_TIMEOUT HW default seems to differ between GPUs, so just a
388	 *      large number for now until we support changing it.
389	 */
390	fifo->timeout.chan_msec = 10000;
391	spin_lock_init(&fifo->lock);
392	mutex_init(&fifo->mutex);
393
394	INIT_LIST_HEAD(&fifo->userd.list);
395	mutex_init(&fifo->userd.mutex);
396
397	return nvkm_engine_ctor(&nvkm_fifo, device, type, inst, true, &fifo->engine);
398}
399