1/*
2 * Copyright 2021 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22#include <core/intr.h>
23#include <core/device.h>
24#include <core/subdev.h>
25#include <subdev/pci.h>
26#include <subdev/top.h>
27
28static int
29nvkm_intr_xlat(struct nvkm_subdev *subdev, struct nvkm_intr *intr,
30	       enum nvkm_intr_type type, int *leaf, u32 *mask)
31{
32	struct nvkm_device *device = subdev->device;
33
34	if (type < NVKM_INTR_VECTOR_0) {
35		if (type == NVKM_INTR_SUBDEV) {
36			const struct nvkm_intr_data *data = intr->data;
37			struct nvkm_top_device *tdev;
38
39			while (data && data->mask) {
40				if (data->type == NVKM_SUBDEV_TOP) {
41					list_for_each_entry(tdev, &device->top->device, head) {
42						if (tdev->intr >= 0 &&
43						    tdev->type == subdev->type &&
44						    tdev->inst == subdev->inst) {
45							if (data->mask & BIT(tdev->intr)) {
46								*leaf = data->leaf;
47								*mask = BIT(tdev->intr);
48								return 0;
49							}
50						}
51					}
52				} else
53				if (data->type == subdev->type && data->inst == subdev->inst) {
54					*leaf = data->leaf;
55					*mask = data->mask;
56					return 0;
57				}
58
59				data++;
60			}
61		} else {
62			return -ENOSYS;
63		}
64	} else {
65		if (type < intr->leaves * sizeof(*intr->stat) * 8) {
66			*leaf = type / 32;
67			*mask = BIT(type % 32);
68			return 0;
69		}
70	}
71
72	return -EINVAL;
73}
74
75static struct nvkm_intr *
76nvkm_intr_find(struct nvkm_subdev *subdev, enum nvkm_intr_type type, int *leaf, u32 *mask)
77{
78	struct nvkm_intr *intr;
79	int ret;
80
81	list_for_each_entry(intr, &subdev->device->intr.intr, head) {
82		ret = nvkm_intr_xlat(subdev, intr, type, leaf, mask);
83		if (ret == 0)
84			return intr;
85	}
86
87	return NULL;
88}
89
90static void
91nvkm_intr_allow_locked(struct nvkm_intr *intr, int leaf, u32 mask)
92{
93	intr->mask[leaf] |= mask;
94	if (intr->func->allow) {
95		if (intr->func->reset)
96			intr->func->reset(intr, leaf, mask);
97		intr->func->allow(intr, leaf, mask);
98	}
99}
100
101void
102nvkm_intr_allow(struct nvkm_subdev *subdev, enum nvkm_intr_type type)
103{
104	struct nvkm_device *device = subdev->device;
105	struct nvkm_intr *intr;
106	unsigned long flags;
107	int leaf;
108	u32 mask;
109
110	intr = nvkm_intr_find(subdev, type, &leaf, &mask);
111	if (intr) {
112		nvkm_debug(intr->subdev, "intr %d/%08x allowed by %s\n", leaf, mask, subdev->name);
113		spin_lock_irqsave(&device->intr.lock, flags);
114		nvkm_intr_allow_locked(intr, leaf, mask);
115		spin_unlock_irqrestore(&device->intr.lock, flags);
116	}
117}
118
119static void
120nvkm_intr_block_locked(struct nvkm_intr *intr, int leaf, u32 mask)
121{
122	intr->mask[leaf] &= ~mask;
123	if (intr->func->block)
124		intr->func->block(intr, leaf, mask);
125}
126
127void
128nvkm_intr_block(struct nvkm_subdev *subdev, enum nvkm_intr_type type)
129{
130	struct nvkm_device *device = subdev->device;
131	struct nvkm_intr *intr;
132	unsigned long flags;
133	int leaf;
134	u32 mask;
135
136	intr = nvkm_intr_find(subdev, type, &leaf, &mask);
137	if (intr) {
138		nvkm_debug(intr->subdev, "intr %d/%08x blocked by %s\n", leaf, mask, subdev->name);
139		spin_lock_irqsave(&device->intr.lock, flags);
140		nvkm_intr_block_locked(intr, leaf, mask);
141		spin_unlock_irqrestore(&device->intr.lock, flags);
142	}
143}
144
145static void
146nvkm_intr_rearm_locked(struct nvkm_device *device)
147{
148	struct nvkm_intr *intr;
149
150	list_for_each_entry(intr, &device->intr.intr, head)
151		intr->func->rearm(intr);
152}
153
154static void
155nvkm_intr_unarm_locked(struct nvkm_device *device)
156{
157	struct nvkm_intr *intr;
158
159	list_for_each_entry(intr, &device->intr.intr, head)
160		intr->func->unarm(intr);
161}
162
163static irqreturn_t
164nvkm_intr(int irq, void *arg)
165{
166	struct nvkm_device *device = arg;
167	struct nvkm_intr *intr;
168	struct nvkm_inth *inth;
169	irqreturn_t ret = IRQ_NONE;
170	bool pending = false;
171	int prio, leaf;
172
173	/* Disable all top-level interrupt sources, and re-arm MSI interrupts. */
174	spin_lock(&device->intr.lock);
175	if (!device->intr.armed)
176		goto done_unlock;
177
178	nvkm_intr_unarm_locked(device);
179	nvkm_pci_msi_rearm(device);
180
181	/* Fetch pending interrupt masks. */
182	list_for_each_entry(intr, &device->intr.intr, head) {
183		if (intr->func->pending(intr))
184			pending = true;
185	}
186
187	if (!pending)
188		goto done;
189
190	/* Check that GPU is still on the bus by reading NV_PMC_BOOT_0. */
191	if (WARN_ON(nvkm_rd32(device, 0x000000) == 0xffffffff))
192		goto done;
193
194	/* Execute handlers. */
195	for (prio = 0; prio < ARRAY_SIZE(device->intr.prio); prio++) {
196		list_for_each_entry(inth, &device->intr.prio[prio], head) {
197			struct nvkm_intr *intr = inth->intr;
198
199			if (intr->stat[inth->leaf] & inth->mask) {
200				if (atomic_read(&inth->allowed)) {
201					if (intr->func->reset)
202						intr->func->reset(intr, inth->leaf, inth->mask);
203					if (inth->func(inth) == IRQ_HANDLED)
204						ret = IRQ_HANDLED;
205				}
206			}
207		}
208	}
209
210	/* Nothing handled?  Some debugging/protection from IRQ storms is in order... */
211	if (ret == IRQ_NONE) {
212		list_for_each_entry(intr, &device->intr.intr, head) {
213			for (leaf = 0; leaf < intr->leaves; leaf++) {
214				if (intr->stat[leaf]) {
215					nvkm_debug(intr->subdev, "intr%d: %08x\n",
216						   leaf, intr->stat[leaf]);
217					nvkm_intr_block_locked(intr, leaf, intr->stat[leaf]);
218				}
219			}
220		}
221	}
222
223done:
224	/* Re-enable all top-level interrupt sources. */
225	nvkm_intr_rearm_locked(device);
226done_unlock:
227	spin_unlock(&device->intr.lock);
228	return ret;
229}
230
231int
232nvkm_intr_add(const struct nvkm_intr_func *func, const struct nvkm_intr_data *data,
233	      struct nvkm_subdev *subdev, int leaves, struct nvkm_intr *intr)
234{
235	struct nvkm_device *device = subdev->device;
236	int i;
237
238	intr->func = func;
239	intr->data = data;
240	intr->subdev = subdev;
241	intr->leaves = leaves;
242	intr->stat = kcalloc(leaves, sizeof(*intr->stat), GFP_KERNEL);
243	intr->mask = kcalloc(leaves, sizeof(*intr->mask), GFP_KERNEL);
244	if (!intr->stat || !intr->mask) {
245		kfree(intr->stat);
246		return -ENOMEM;
247	}
248
249	if (intr->subdev->debug >= NV_DBG_DEBUG) {
250		for (i = 0; i < intr->leaves; i++)
251			intr->mask[i] = ~0;
252	}
253
254	spin_lock_irq(&device->intr.lock);
255	list_add_tail(&intr->head, &device->intr.intr);
256	spin_unlock_irq(&device->intr.lock);
257	return 0;
258}
259
260static irqreturn_t
261nvkm_intr_subdev(struct nvkm_inth *inth)
262{
263	struct nvkm_subdev *subdev = container_of(inth, typeof(*subdev), inth);
264
265	nvkm_subdev_intr(subdev);
266	return IRQ_HANDLED;
267}
268
269static void
270nvkm_intr_subdev_add_dev(struct nvkm_intr *intr, enum nvkm_subdev_type type, int inst)
271{
272	struct nvkm_subdev *subdev;
273	enum nvkm_intr_prio prio;
274	int ret;
275
276	subdev = nvkm_device_subdev(intr->subdev->device, type, inst);
277	if (!subdev || !subdev->func->intr)
278		return;
279
280	if (type == NVKM_ENGINE_DISP)
281		prio = NVKM_INTR_PRIO_VBLANK;
282	else
283		prio = NVKM_INTR_PRIO_NORMAL;
284
285	ret = nvkm_inth_add(intr, NVKM_INTR_SUBDEV, prio, subdev, nvkm_intr_subdev, &subdev->inth);
286	if (WARN_ON(ret))
287		return;
288
289	nvkm_inth_allow(&subdev->inth);
290}
291
292static void
293nvkm_intr_subdev_add(struct nvkm_intr *intr)
294{
295	const struct nvkm_intr_data *data;
296	struct nvkm_device *device = intr->subdev->device;
297	struct nvkm_top_device *tdev;
298
299	for (data = intr->data; data && data->mask; data++) {
300		if (data->legacy) {
301			if (data->type == NVKM_SUBDEV_TOP) {
302				list_for_each_entry(tdev, &device->top->device, head) {
303					if (tdev->intr < 0 || !(data->mask & BIT(tdev->intr)))
304						continue;
305
306					nvkm_intr_subdev_add_dev(intr, tdev->type, tdev->inst);
307				}
308			} else {
309				nvkm_intr_subdev_add_dev(intr, data->type, data->inst);
310			}
311		}
312	}
313}
314
315void
316nvkm_intr_rearm(struct nvkm_device *device)
317{
318	struct nvkm_intr *intr;
319	int i;
320
321	if (unlikely(!device->intr.legacy_done)) {
322		list_for_each_entry(intr, &device->intr.intr, head)
323			nvkm_intr_subdev_add(intr);
324		device->intr.legacy_done = true;
325	}
326
327	spin_lock_irq(&device->intr.lock);
328	list_for_each_entry(intr, &device->intr.intr, head) {
329		for (i = 0; intr->func->block && i < intr->leaves; i++) {
330			intr->func->block(intr, i, ~0);
331			intr->func->allow(intr, i, intr->mask[i]);
332		}
333	}
334
335	nvkm_intr_rearm_locked(device);
336	device->intr.armed = true;
337	spin_unlock_irq(&device->intr.lock);
338}
339
340void
341nvkm_intr_unarm(struct nvkm_device *device)
342{
343	spin_lock_irq(&device->intr.lock);
344	nvkm_intr_unarm_locked(device);
345	device->intr.armed = false;
346	spin_unlock_irq(&device->intr.lock);
347}
348
349int
350nvkm_intr_install(struct nvkm_device *device)
351{
352	int ret;
353
354	device->intr.irq = device->func->irq(device);
355	if (device->intr.irq < 0)
356		return device->intr.irq;
357
358	ret = request_irq(device->intr.irq, nvkm_intr, IRQF_SHARED, "nvkm", device);
359	if (ret)
360		return ret;
361
362	device->intr.alloc = true;
363	return 0;
364}
365
366void
367nvkm_intr_dtor(struct nvkm_device *device)
368{
369	struct nvkm_intr *intr, *intt;
370
371	list_for_each_entry_safe(intr, intt, &device->intr.intr, head) {
372		list_del(&intr->head);
373		kfree(intr->mask);
374		kfree(intr->stat);
375	}
376
377	if (device->intr.alloc)
378		free_irq(device->intr.irq, device);
379}
380
381void
382nvkm_intr_ctor(struct nvkm_device *device)
383{
384	int i;
385
386	INIT_LIST_HEAD(&device->intr.intr);
387	for (i = 0; i < ARRAY_SIZE(device->intr.prio); i++)
388		INIT_LIST_HEAD(&device->intr.prio[i]);
389
390	spin_lock_init(&device->intr.lock);
391	device->intr.armed = false;
392}
393
394void
395nvkm_inth_block(struct nvkm_inth *inth)
396{
397	if (unlikely(!inth->intr))
398		return;
399
400	atomic_set(&inth->allowed, 0);
401}
402
403void
404nvkm_inth_allow(struct nvkm_inth *inth)
405{
406	struct nvkm_intr *intr = inth->intr;
407	unsigned long flags;
408
409	if (unlikely(!inth->intr))
410		return;
411
412	spin_lock_irqsave(&intr->subdev->device->intr.lock, flags);
413	if (!atomic_xchg(&inth->allowed, 1)) {
414		if ((intr->mask[inth->leaf] & inth->mask) != inth->mask)
415			nvkm_intr_allow_locked(intr, inth->leaf, inth->mask);
416	}
417	spin_unlock_irqrestore(&intr->subdev->device->intr.lock, flags);
418}
419
420int
421nvkm_inth_add(struct nvkm_intr *intr, enum nvkm_intr_type type, enum nvkm_intr_prio prio,
422	      struct nvkm_subdev *subdev, nvkm_inth_func func, struct nvkm_inth *inth)
423{
424	struct nvkm_device *device = subdev->device;
425	int ret;
426
427	if (WARN_ON(inth->mask))
428		return -EBUSY;
429
430	ret = nvkm_intr_xlat(subdev, intr, type, &inth->leaf, &inth->mask);
431	if (ret)
432		return ret;
433
434	nvkm_debug(intr->subdev, "intr %d/%08x requested by %s\n",
435		   inth->leaf, inth->mask, subdev->name);
436
437	inth->intr = intr;
438	inth->func = func;
439	atomic_set(&inth->allowed, 0);
440	list_add_tail(&inth->head, &device->intr.prio[prio]);
441	return 0;
442}
443