1/*-
2 * Copyright (c) 2015-2016 Svatopluk Kraus
3 * Copyright (c) 2015-2016 Michal Meloun
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include "opt_platform.h"
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/syslog.h>
36#include <sys/kernel.h>
37#include <sys/malloc.h>
38#include <sys/proc.h>
39#include <sys/bus.h>
40#include <sys/interrupt.h>
41#include <sys/conf.h>
42#include <sys/pmc.h>
43#include <sys/pmckern.h>
44#include <sys/smp.h>
45
46#include <machine/atomic.h>
47#include <machine/bus.h>
48#include <machine/intr.h>
49#include <machine/cpu.h>
50#include <machine/smp.h>
51
52#ifdef INTRNG
53#include "pic_if.h"
54
55#ifdef SMP
56#define INTR_IPI_NAMELEN	(MAXCOMLEN + 1)
57
58struct intr_ipi {
59	intr_ipi_handler_t *	ii_handler;
60	void *			ii_handler_arg;
61	intr_ipi_send_t *	ii_send;
62	void *			ii_send_arg;
63	char			ii_name[INTR_IPI_NAMELEN];
64	u_long *		ii_count;
65};
66
67static struct intr_ipi ipi_sources[INTR_IPI_COUNT];
68#endif
69#endif
70
71/*
72 * arm_irq_memory_barrier()
73 *
74 * Ensure all writes to device memory have reached devices before proceeding.
75 *
76 * This is intended to be called from the post-filter and post-thread routines
77 * of an interrupt controller implementation.  A peripheral device driver should
78 * use bus_space_barrier() if it needs to ensure a write has reached the
79 * hardware for some reason other than clearing interrupt conditions.
80 *
81 * The need for this function arises from the ARM weak memory ordering model.
82 * Writes to locations mapped with the Device attribute bypass any caches, but
83 * are buffered.  Multiple writes to the same device will be observed by that
84 * device in the order issued by the cpu.  Writes to different devices may
85 * appear at those devices in a different order than issued by the cpu.  That
86 * is, if the cpu writes to device A then device B, the write to device B could
87 * complete before the write to device A.
88 *
89 * Consider a typical device interrupt handler which services the interrupt and
90 * writes to a device status-acknowledge register to clear the interrupt before
91 * returning.  That write is posted to the L2 controller which "immediately"
92 * places it in a store buffer and automatically drains that buffer.  This can
93 * be less immediate than you'd think... There may be no free slots in the store
94 * buffers, so an existing buffer has to be drained first to make room.  The
95 * target bus may be busy with other traffic (such as DMA for various devices),
96 * delaying the drain of the store buffer for some indeterminate time.  While
97 * all this delay is happening, execution proceeds on the CPU, unwinding its way
98 * out of the interrupt call stack to the point where the interrupt driver code
99 * is ready to EOI and unmask the interrupt.  The interrupt controller may be
100 * accessed via a faster bus than the hardware whose handler just ran; the write
101 * to unmask and EOI the interrupt may complete quickly while the device write
102 * to ack and clear the interrupt source is still lingering in a store buffer
103 * waiting for access to a slower bus.  With the interrupt unmasked at the
104 * interrupt controller but still active at the device, as soon as interrupts
105 * are enabled on the core the device re-interrupts immediately: now you've got
106 * a spurious interrupt on your hands.
107 *
108 * The right way to fix this problem is for every device driver to use the
109 * proper bus_space_barrier() calls in its interrupt handler.  For ARM a single
110 * barrier call at the end of the handler would work.  This would have to be
111 * done to every driver in the system, not just arm-specific drivers.
112 *
113 * Another potential fix is to map all device memory as Strongly-Ordered rather
114 * than Device memory, which takes the store buffers out of the picture.  This
115 * has a pretty big impact on overall system performance, because each strongly
116 * ordered memory access causes all L2 store buffers to be drained.
117 *
118 * A compromise solution is to have the interrupt controller implementation call
119 * this function to establish a barrier between writes to the interrupt-source
120 * device and writes to the interrupt controller device.
121 *
122 * This takes the interrupt number as an argument, and currently doesn't use it.
123 * The plan is that maybe some day there is a way to flag certain interrupts as
124 * "memory barrier safe" and we can avoid this overhead with them.
125 */
126void
127arm_irq_memory_barrier(uintptr_t irq)
128{
129
130	dsb();
131	cpu_l2cache_drain_writebuf();
132}
133
134#ifdef INTRNG
135#ifdef SMP
136static inline struct intr_ipi *
137intr_ipi_lookup(u_int ipi)
138{
139
140	if (ipi >= INTR_IPI_COUNT)
141		panic("%s: no such IPI %u", __func__, ipi);
142
143	return (&ipi_sources[ipi]);
144}
145
146void
147intr_ipi_dispatch(u_int ipi, struct trapframe *tf)
148{
149	void *arg;
150	struct intr_ipi *ii;
151
152	ii = intr_ipi_lookup(ipi);
153	if (ii->ii_count == NULL)
154		panic("%s: not setup IPI %u", __func__, ipi);
155
156	intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid));
157
158	/*
159	 * Supply ipi filter with trapframe argument
160	 * if none is registered.
161	 */
162	arg = ii->ii_handler_arg != NULL ? ii->ii_handler_arg : tf;
163	ii->ii_handler(arg);
164}
165
166void
167intr_ipi_send(cpuset_t cpus, u_int ipi)
168{
169	struct intr_ipi *ii;
170
171	ii = intr_ipi_lookup(ipi);
172	if (ii->ii_count == NULL)
173		panic("%s: not setup IPI %u", __func__, ipi);
174
175	ii->ii_send(ii->ii_send_arg, cpus, ipi);
176}
177
178void
179intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
180    void *h_arg, intr_ipi_send_t *send, void *s_arg)
181{
182	struct intr_ipi *ii;
183
184	ii = intr_ipi_lookup(ipi);
185
186	KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi));
187	KASSERT(send != NULL, ("%s: ipi %u no sender", __func__, ipi));
188	KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi));
189
190	ii->ii_handler = hand;
191	ii->ii_handler_arg = h_arg;
192	ii->ii_send = send;
193	ii->ii_send_arg = s_arg;
194	strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN);
195	ii->ii_count = intr_ipi_setup_counters(name);
196}
197
198/*
199 *  Send IPI thru interrupt controller.
200 */
201static void
202pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi)
203{
204
205	KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
206	PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi);
207}
208
209/*
210 *  Setup IPI handler on interrupt controller.
211 *
212 *  Not SMP coherent.
213 */
214int
215intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
216    void *arg)
217{
218	int error;
219	struct intr_irqsrc *isrc;
220
221	KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
222
223	error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc);
224	if (error != 0)
225		return (error);
226
227	isrc->isrc_handlers++;
228	intr_ipi_setup(ipi, name, hand, arg, pic_ipi_send, isrc);
229	return (0);
230}
231#endif
232#endif
233