1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright 2011 Nathan Whitehorn
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD$");
30
31#include "opt_platform.h"
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/module.h>
36#include <sys/bus.h>
37#include <sys/conf.h>
38#include <sys/kernel.h>
39#include <sys/malloc.h>
40#include <sys/smp.h>
41
42#include <vm/vm.h>
43#include <vm/pmap.h>
44
45#include <machine/bus.h>
46#include <machine/intr_machdep.h>
47#include <machine/md_var.h>
48#include <machine/rtas.h>
49
50#include <dev/ofw/ofw_bus.h>
51#include <dev/ofw/ofw_bus_subr.h>
52
53#ifdef POWERNV
54#include <powerpc/powernv/opal.h>
55#endif
56
57#include "phyp-hvcall.h"
58#include "pic_if.h"
59
60#define XICP_PRIORITY	5	/* Random non-zero number */
61#define XICP_IPI	2
62#define MAX_XICP_IRQS	(1<<24)	/* 24-bit XIRR field */
63
64#define	XIVE_XICS_MODE_EMU	0
65#define	XIVE_XICS_MODE_EXP	1
66
67static int	xicp_probe(device_t);
68static int	xicp_attach(device_t);
69static int	xics_probe(device_t);
70static int	xics_attach(device_t);
71
72static void	xicp_bind(device_t dev, u_int irq, cpuset_t cpumask);
73static void	xicp_dispatch(device_t, struct trapframe *);
74static void	xicp_enable(device_t, u_int, u_int);
75static void	xicp_eoi(device_t, u_int);
76static void	xicp_ipi(device_t, u_int);
77static void	xicp_mask(device_t, u_int);
78static void	xicp_unmask(device_t, u_int);
79
80#ifdef POWERNV
81void	xicp_smp_cpu_startup(void);
82#endif
83
84static device_method_t  xicp_methods[] = {
85	/* Device interface */
86	DEVMETHOD(device_probe,		xicp_probe),
87	DEVMETHOD(device_attach,	xicp_attach),
88
89	/* PIC interface */
90	DEVMETHOD(pic_bind,		xicp_bind),
91	DEVMETHOD(pic_dispatch,		xicp_dispatch),
92	DEVMETHOD(pic_enable,		xicp_enable),
93	DEVMETHOD(pic_eoi,		xicp_eoi),
94	DEVMETHOD(pic_ipi,		xicp_ipi),
95	DEVMETHOD(pic_mask,		xicp_mask),
96	DEVMETHOD(pic_unmask,		xicp_unmask),
97
98	DEVMETHOD_END
99};
100
101static device_method_t  xics_methods[] = {
102	/* Device interface */
103	DEVMETHOD(device_probe,		xics_probe),
104	DEVMETHOD(device_attach,	xics_attach),
105
106	DEVMETHOD_END
107};
108
109struct xicp_softc {
110	struct mtx sc_mtx;
111	struct resource *mem[MAXCPU];
112
113	int cpu_range[2];
114
115	int ibm_int_on;
116	int ibm_int_off;
117	int ibm_get_xive;
118	int ibm_set_xive;
119
120	/* XXX: inefficient -- hash table? tree? */
121	struct {
122		int irq;
123		int vector;
124		int cpu;
125	} intvecs[256];
126	int nintvecs;
127	bool xics_emu;
128};
129
130static driver_t xicp_driver = {
131	"xicp",
132	xicp_methods,
133	sizeof(struct xicp_softc)
134};
135
136static driver_t xics_driver = {
137	"xics",
138	xics_methods,
139	0
140};
141
142#ifdef POWERNV
143/* We can only pass physical addresses into OPAL.  Kernel stacks are in the KVA,
144 * not in the direct map, so we need to somehow extract the physical address.
145 * However, pmap_kextract() takes locks, which is forbidden in a critical region
146 * (which PMAP_DISPATCH() operates in).  The kernel is mapped into the Direct
147 * Map (0xc000....), and the CPU implicitly drops the top two bits when doing
148 * real address by nature that the bus width is smaller than 64-bits.  Placing
149 * cpu_xirr into the DMAP lets us take advantage of this and avoids the
150 * pmap_kextract() that would otherwise be needed if using the stack variable.
151 */
152static uint32_t cpu_xirr[MAXCPU];
153#endif
154
155static devclass_t xicp_devclass;
156static devclass_t xics_devclass;
157
158EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, xicp_devclass, 0, 0,
159    BUS_PASS_INTERRUPT-1);
160EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, xics_devclass, 0, 0,
161    BUS_PASS_INTERRUPT);
162
163#ifdef POWERNV
164static struct resource *
165xicp_mem_for_cpu(int cpu)
166{
167	device_t dev;
168	struct xicp_softc *sc;
169	int i;
170
171	for (i = 0; (dev = devclass_get_device(xicp_devclass, i)) != NULL; i++){
172		sc = device_get_softc(dev);
173		if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1])
174			return (sc->mem[cpu - sc->cpu_range[0]]);
175	}
176
177	return (NULL);
178}
179#endif
180
181static int
182xicp_probe(device_t dev)
183{
184
185	if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") &&
186	    !ofw_bus_is_compatible(dev, "ibm,opal-intc"))
187		return (ENXIO);
188
189	device_set_desc(dev, "External Interrupt Presentation Controller");
190	return (BUS_PROBE_GENERIC);
191}
192
193static int
194xics_probe(device_t dev)
195{
196
197	if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") &&
198	    !ofw_bus_is_compatible(dev, "IBM,opal-xics"))
199		return (ENXIO);
200
201	device_set_desc(dev, "External Interrupt Source Controller");
202	return (BUS_PROBE_GENERIC);
203}
204
205static int
206xicp_attach(device_t dev)
207{
208	struct xicp_softc *sc = device_get_softc(dev);
209	phandle_t phandle = ofw_bus_get_node(dev);
210
211	if (rtas_exists()) {
212		sc->ibm_int_on = rtas_token_lookup("ibm,int-on");
213		sc->ibm_int_off = rtas_token_lookup("ibm,int-off");
214		sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive");
215		sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive");
216#ifdef POWERNV
217	} else if (opal_check() == 0) {
218		/* No init needed */
219#endif
220	} else {
221		device_printf(dev, "Cannot attach without RTAS or OPAL\n");
222		return (ENXIO);
223	}
224
225	if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) {
226		OF_getencprop(phandle, "ibm,interrupt-server-ranges",
227		    sc->cpu_range, sizeof(sc->cpu_range));
228		sc->cpu_range[1] += sc->cpu_range[0];
229		device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0],
230		    sc->cpu_range[1]-1);
231#ifdef POWERNV
232	} else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) {
233			/*
234			 * For now run POWER9 XIVE interrupt controller in XICS
235			 * compatibility mode.
236			 */
237			sc->xics_emu = true;
238			opal_call(OPAL_XIVE_RESET, XIVE_XICS_MODE_EMU);
239#endif
240	} else {
241		sc->cpu_range[0] = 0;
242		sc->cpu_range[1] = mp_ncpus;
243	}
244
245#ifdef POWERNV
246	if (mfmsr() & PSL_HV) {
247		int i;
248
249		if (sc->xics_emu) {
250			opal_call(OPAL_INT_SET_CPPR, 0xff);
251			for (i = 0; i < mp_ncpus; i++) {
252				opal_call(OPAL_INT_SET_MFRR,
253				    pcpu_find(i)->pc_hwref, 0xff);
254			}
255		} else {
256			for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) {
257				sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
258				    &i, RF_ACTIVE);
259				if (sc->mem[i] == NULL) {
260					device_printf(dev, "Could not alloc mem "
261					    "resource %d\n", i);
262					return (ENXIO);
263				}
264
265				/* Unmask interrupts on all cores */
266				bus_write_1(sc->mem[i], 4, 0xff);
267				bus_write_1(sc->mem[i], 12, 0xff);
268			}
269		}
270	}
271#endif
272
273	mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF);
274	sc->nintvecs = 0;
275
276	powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS,
277	    1 /* Number of IPIs */, FALSE);
278	root_pic = dev;
279
280	return (0);
281}
282
283static int
284xics_attach(device_t dev)
285{
286	phandle_t phandle = ofw_bus_get_node(dev);
287
288	/* The XICP (root PIC) will handle all our interrupts */
289	powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
290	    MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE);
291
292	return (0);
293}
294
295/*
296 * PIC I/F methods.
297 */
298
299static void
300xicp_bind(device_t dev, u_int irq, cpuset_t cpumask)
301{
302	struct xicp_softc *sc = device_get_softc(dev);
303	cell_t status, cpu;
304	int ncpus, i, error;
305
306	/* Ignore IPIs */
307	if (irq == MAX_XICP_IRQS)
308		return;
309
310	/*
311	 * This doesn't appear to actually support affinity groups, so pick a
312	 * random CPU.
313	 */
314	ncpus = 0;
315	CPU_FOREACH(cpu)
316		if (CPU_ISSET(cpu, &cpumask)) ncpus++;
317
318	i = mftb() % ncpus;
319	ncpus = 0;
320	CPU_FOREACH(cpu) {
321		if (!CPU_ISSET(cpu, &cpumask))
322			continue;
323		if (ncpus == i)
324			break;
325		ncpus++;
326	}
327
328	cpu = pcpu_find(cpu)->pc_hwref;
329
330	/* XXX: super inefficient */
331	for (i = 0; i < sc->nintvecs; i++) {
332		if (sc->intvecs[i].irq == irq) {
333			sc->intvecs[i].cpu = cpu;
334			break;
335		}
336	}
337	KASSERT(i < sc->nintvecs, ("Binding non-configured interrupt"));
338
339	if (rtas_exists())
340		error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
341		    XICP_PRIORITY, &status);
342#ifdef POWERNV
343	else
344		error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
345#endif
346
347	if (error < 0)
348		panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
349}
350
351static void
352xicp_dispatch(device_t dev, struct trapframe *tf)
353{
354	struct xicp_softc *sc;
355	struct resource *regs = NULL;
356	uint64_t xirr, junk;
357	int i;
358
359	sc = device_get_softc(dev);
360#ifdef POWERNV
361	if ((mfmsr() & PSL_HV) && !sc->xics_emu) {
362		regs = xicp_mem_for_cpu(PCPU_GET(hwref));
363		KASSERT(regs != NULL,
364		    ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref)));
365	}
366#endif
367
368	for (;;) {
369		/* Return value in R4, use the PFT call */
370		if (regs) {
371			xirr = bus_read_4(regs, 4);
372#ifdef POWERNV
373		} else if (sc->xics_emu) {
374			opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)],
375			    false);
376			xirr = cpu_xirr[PCPU_GET(cpuid)];
377#endif
378		} else {
379			/* Return value in R4, use the PFT call */
380			phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk);
381		}
382		xirr &= 0x00ffffff;
383
384		if (xirr == 0) /* No more pending interrupts? */
385			break;
386
387		if (xirr == XICP_IPI) {		/* Magic number for IPIs */
388			xirr = MAX_XICP_IRQS;	/* Map to FreeBSD magic */
389
390			/* Clear IPI */
391			if (regs)
392				bus_write_1(regs, 12, 0xff);
393#ifdef POWERNV
394			else if (sc->xics_emu)
395				opal_call(OPAL_INT_SET_MFRR,
396				    PCPU_GET(hwref), 0xff);
397#endif
398			else
399				phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)),
400				    0xff);
401		}
402
403		/* XXX: super inefficient */
404		for (i = 0; i < sc->nintvecs; i++) {
405			if (sc->intvecs[i].irq == xirr)
406				break;
407		}
408
409		KASSERT(i < sc->nintvecs, ("Unmapped XIRR"));
410		powerpc_dispatch_intr(sc->intvecs[i].vector, tf);
411	}
412}
413
414static void
415xicp_enable(device_t dev, u_int irq, u_int vector)
416{
417	struct xicp_softc *sc;
418	cell_t status, cpu;
419
420	sc = device_get_softc(dev);
421
422	KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs),
423		("Too many XICP interrupts"));
424
425	/* Bind to this CPU to start: distrib. ID is last entry in gserver# */
426	cpu = PCPU_GET(hwref);
427
428	mtx_lock(&sc->sc_mtx);
429	sc->intvecs[sc->nintvecs].irq = irq;
430	sc->intvecs[sc->nintvecs].vector = vector;
431	sc->intvecs[sc->nintvecs].cpu = cpu;
432	mb();
433	sc->nintvecs++;
434	mtx_unlock(&sc->sc_mtx);
435
436	/* IPIs are also enabled */
437	if (irq == MAX_XICP_IRQS)
438		return;
439
440	if (rtas_exists()) {
441		rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
442		    XICP_PRIORITY, &status);
443		xicp_unmask(dev, irq);
444#ifdef POWERNV
445	} else {
446		status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
447		/* Unmask implicit for OPAL */
448
449		if (status != 0)
450			panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
451			    cpu, status);
452#endif
453	}
454}
455
456static void
457xicp_eoi(device_t dev, u_int irq)
458{
459#ifdef POWERNV
460	struct xicp_softc *sc;
461#endif
462	uint64_t xirr;
463
464	if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */
465		irq = XICP_IPI;
466	xirr = irq | (0xff << 24);
467
468#ifdef POWERNV
469	if (mfmsr() & PSL_HV) {
470		sc = device_get_softc(dev);
471		if (sc->xics_emu)
472			opal_call(OPAL_INT_EOI, xirr);
473		else
474			bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr);
475	} else
476#endif
477		phyp_hcall(H_EOI, xirr);
478}
479
480static void
481xicp_ipi(device_t dev, u_int cpu)
482{
483
484#ifdef POWERNV
485	struct xicp_softc *sc;
486	cpu = pcpu_find(cpu)->pc_hwref;
487
488	if (mfmsr() & PSL_HV) {
489		sc = device_get_softc(dev);
490		if (sc->xics_emu) {
491			int64_t rv;
492			rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY);
493			if (rv != 0)
494			    device_printf(dev, "IPI SET_MFRR result: %ld\n", rv);
495		} else
496			bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY);
497	} else
498#endif
499		phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY);
500}
501
502static void
503xicp_mask(device_t dev, u_int irq)
504{
505	struct xicp_softc *sc = device_get_softc(dev);
506	cell_t status;
507
508	if (irq == MAX_XICP_IRQS)
509		return;
510
511	if (rtas_exists()) {
512		rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status);
513#ifdef POWERNV
514	} else {
515		int i;
516
517		for (i = 0; i < sc->nintvecs; i++) {
518			if (sc->intvecs[i].irq == irq) {
519				break;
520			}
521		}
522		KASSERT(i < sc->nintvecs, ("Masking unconfigured interrupt"));
523		opal_call(OPAL_SET_XIVE, irq, sc->intvecs[i].cpu << 2, 0xff);
524#endif
525	}
526}
527
528static void
529xicp_unmask(device_t dev, u_int irq)
530{
531	struct xicp_softc *sc = device_get_softc(dev);
532	cell_t status;
533
534	if (irq == MAX_XICP_IRQS)
535		return;
536
537	if (rtas_exists()) {
538		rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status);
539#ifdef POWERNV
540	} else {
541		int i;
542
543		for (i = 0; i < sc->nintvecs; i++) {
544			if (sc->intvecs[i].irq == irq) {
545				break;
546			}
547		}
548		KASSERT(i < sc->nintvecs, ("Unmasking unconfigured interrupt"));
549		opal_call(OPAL_SET_XIVE, irq, sc->intvecs[i].cpu << 2,
550		    XICP_PRIORITY);
551#endif
552	}
553}
554
555#ifdef POWERNV
556/* This is only used on POWER9 systems with the XIVE's XICS emulation. */
557void
558xicp_smp_cpu_startup(void)
559{
560	struct xicp_softc *sc;
561
562	if (mfmsr() & PSL_HV) {
563		sc = device_get_softc(root_pic);
564
565		if (sc->xics_emu)
566			opal_call(OPAL_INT_SET_CPPR, 0xff);
567	}
568}
569#endif
570