pic.c revision 1.6 
1/*	$NetBSD: pic.c,v 1.6 2010/08/31 14:33:41 kiyohara Exp $	*/
2/*-
3 * Copyright (c) 2008 The NetBSD Foundation, Inc.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to The NetBSD Foundation
7 * by Matt Thomas.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 */
30#include <sys/cdefs.h>
31__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.6 2010/08/31 14:33:41 kiyohara Exp $");
32
33#define _INTR_PRIVATE
34#include <sys/param.h>
35#include <sys/evcnt.h>
36#include <sys/atomic.h>
37#include <sys/malloc.h>
38#include <sys/mallocvar.h>
39#include <sys/atomic.h>
40
41#include <arm/armreg.h>
42#include <arm/cpu.h>
43#include <arm/cpufunc.h>
44
45#include <arm/pic/picvar.h>
46
47MALLOC_DEFINE(M_INTRSOURCE, "intrsource", "interrupt source");
48
49static uint32_t
50	pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int);
51static struct pic_softc *
52	pic_list_find_pic_by_pending_ipl(uint32_t);
53static void
54	pic_deliver_irqs(struct pic_softc *, int, void *);
55static void
56	pic_list_deliver_irqs(register_t, int, void *);
57
58struct pic_softc *pic_list[PIC_MAXPICS];
59#if PIC_MAXPICS > 32
60#error PIC_MAXPICS > 32 not supported
61#endif
62volatile uint32_t pic_blocked_pics;
63volatile uint32_t pic_pending_pics;
64volatile uint32_t pic_pending_ipls;
65struct intrsource *pic_sources[PIC_MAXMAXSOURCES];
66struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES];
67struct intrsource **pic_iplsource[NIPL] = {
68	[0 ... NIPL-1] = pic__iplsources,
69};
70size_t pic_ipl_offset[NIPL+1];
71size_t pic_sourcebase;
72static struct evcnt pic_deferral_ev =
73    EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "deferred", "intr");
74EVCNT_ATTACH_STATIC(pic_deferral_ev);
75
76
77
78int
79pic_handle_intr(void *arg)
80{
81	struct pic_softc * const pic = arg;
82	int rv;
83
84	rv = (*pic->pic_ops->pic_find_pending_irqs)(pic);
85
86	return rv > 0;
87}
88
89void
90pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is)
91{
92	const uint32_t ipl_mask = __BIT(is->is_ipl);
93
94	atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5],
95	    __BIT(is->is_irq & 0x1f));
96
97	atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
98	atomic_or_32(&pic_pending_ipls, ipl_mask);
99	atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id));
100}
101
102void
103pic_mark_pending(struct pic_softc *pic, int irq)
104{
105	struct intrsource * const is = pic->pic_sources[irq];
106
107	KASSERT(irq < pic->pic_maxsources);
108	KASSERT(is != NULL);
109
110	pic_mark_pending_source(pic, is);
111}
112
113uint32_t
114pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base,
115	uint32_t pending)
116{
117	struct intrsource ** const isbase = &pic->pic_sources[irq_base];
118	struct intrsource *is;
119	volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5];
120	uint32_t ipl_mask = 0;
121
122	if (pending == 0)
123		return ipl_mask;
124
125	KASSERT((irq_base & 31) == 0);
126
127	(*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending);
128
129	atomic_or_32(ipending, pending);
130        while (pending != 0) {
131		int n = ffs(pending);
132		if (n-- == 0)
133			break;
134		is = isbase[n];
135		KASSERT(is != NULL);
136		KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32);
137		pending &= ~__BIT(n);
138		ipl_mask |= __BIT(is->is_ipl);
139	}
140
141	atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
142	atomic_or_32(&pic_pending_ipls, ipl_mask);
143	atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id));
144
145	return ipl_mask;
146}
147
148uint32_t
149pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base,
150	uint32_t pending, int ipl)
151{
152	uint32_t ipl_irq_mask = 0;
153	uint32_t irq_mask;
154
155	for (;;) {
156		int irq = ffs(pending);
157		if (irq-- == 0)
158			return ipl_irq_mask;
159
160		irq_mask = __BIT(irq);
161		KASSERT(pic->pic_sources[irq_base + irq] != NULL);
162		if (pic->pic_sources[irq_base + irq]->is_ipl == ipl)
163			ipl_irq_mask |= irq_mask;
164
165		pending &= ~irq_mask;
166	}
167}
168
169void
170pic_dispatch(struct intrsource *is, void *frame)
171{
172	int rv;
173
174	if (__predict_false(is->is_arg == NULL)
175	    && __predict_true(frame != NULL)) {
176		rv = (*is->is_func)(frame);
177	} else if (__predict_true(is->is_arg != NULL)) {
178		rv = (*is->is_func)(is->is_arg);
179	} else {
180		pic_deferral_ev.ev_count++;
181		return;
182	}
183	is->is_ev.ev_count++;
184}
185
186void
187pic_deliver_irqs(struct pic_softc *pic, int ipl, void *frame)
188{
189	const uint32_t ipl_mask = __BIT(ipl);
190	struct intrsource *is;
191	volatile uint32_t *ipending = pic->pic_pending_irqs;
192	volatile uint32_t *iblocked = pic->pic_blocked_irqs;
193	size_t irq_base;
194#if PIC_MAXSOURCES > 32
195	size_t irq_count;
196	int poi = 0;		/* Possibility of interrupting */
197#endif
198	uint32_t pending_irqs;
199	uint32_t blocked_irqs;
200	int irq;
201	bool progress = false;
202
203	KASSERT(pic->pic_pending_ipls & ipl_mask);
204
205	irq_base = 0;
206#if PIC_MAXSOURCES > 32
207	irq_count = 0;
208#endif
209
210	for (;;) {
211		pending_irqs = pic_find_pending_irqs_by_ipl(pic, irq_base,
212		    *ipending, ipl);
213		KASSERT((pending_irqs & *ipending) == pending_irqs);
214		KASSERT((pending_irqs & ~(*ipending)) == 0);
215		if (pending_irqs == 0) {
216#if PIC_MAXSOURCES > 32
217			irq_count += 32;
218			if (__predict_true(irq_count >= pic->pic_maxsources)) {
219				if (!poi)
220					/*Interrupt at this level was handled.*/
221					break;
222				irq_base = 0;
223				irq_count = 0;
224				poi = 0;
225				ipending = pic->pic_pending_irqs;
226				iblocked = pic->pic_blocked_irqs;
227			} else {
228				irq_base += 32;
229				ipending++;
230				iblocked++;
231				KASSERT(irq_base <= pic->pic_maxsources);
232			}
233			continue;
234#else
235			break;
236#endif
237		}
238		progress = true;
239		blocked_irqs = 0;
240		do {
241			irq = ffs(pending_irqs) - 1;
242			KASSERT(irq >= 0);
243
244			atomic_and_32(ipending, ~__BIT(irq));
245			is = pic->pic_sources[irq_base + irq];
246			if (is != NULL) {
247				cpsie(I32_bit);
248				pic_dispatch(is, frame);
249				cpsid(I32_bit);
250#if PIC_MAXSOURCES > 32
251				/*
252				 * There is a possibility of interrupting
253				 * from cpsie() to cpsid().
254				 */
255				poi = 1;
256#endif
257				blocked_irqs |= __BIT(irq);
258			} else {
259				KASSERT(0);
260			}
261			pending_irqs = pic_find_pending_irqs_by_ipl(pic,
262			    irq_base, *ipending, ipl);
263		} while (pending_irqs);
264		if (blocked_irqs) {
265			atomic_or_32(iblocked, blocked_irqs);
266			atomic_or_32(&pic_blocked_pics, __BIT(pic->pic_id));
267		}
268	}
269
270	KASSERT(progress);
271	/*
272	 * Since interrupts are disabled, we don't have to be too careful
273	 * about these.
274	 */
275	if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0)
276		atomic_and_32(&pic_pending_pics, ~__BIT(pic->pic_id));
277}
278
279static void
280pic_list_unblock_irqs(void)
281{
282	uint32_t blocked_pics = pic_blocked_pics;
283
284	pic_blocked_pics = 0;
285	for (;;) {
286		struct pic_softc *pic;
287#if PIC_MAXSOURCES > 32
288		volatile uint32_t *iblocked;
289		uint32_t blocked;
290		size_t irq_base;
291#endif
292
293		int pic_id = ffs(blocked_pics);
294		if (pic_id-- == 0)
295			return;
296
297		pic = pic_list[pic_id];
298		KASSERT(pic != NULL);
299#if PIC_MAXSOURCES > 32
300		for (irq_base = 0, iblocked = pic->pic_blocked_irqs;
301		     irq_base < pic->pic_maxsources;
302		     irq_base += 32, iblocked++) {
303			if ((blocked = *iblocked) != 0) {
304				(*pic->pic_ops->pic_unblock_irqs)(pic,
305				    irq_base, blocked);
306				atomic_and_32(iblocked, ~blocked);
307			}
308		}
309#else
310		KASSERT(pic->pic_blocked_irqs[0] != 0);
311		(*pic->pic_ops->pic_unblock_irqs)(pic,
312		    0, pic->pic_blocked_irqs[0]);
313		pic->pic_blocked_irqs[0] = 0;
314#endif
315		blocked_pics &= ~__BIT(pic_id);
316	}
317}
318
319
320struct pic_softc *
321pic_list_find_pic_by_pending_ipl(uint32_t ipl_mask)
322{
323	uint32_t pending_pics = pic_pending_pics;
324	struct pic_softc *pic;
325
326	for (;;) {
327		int pic_id = ffs(pending_pics);
328		if (pic_id-- == 0)
329			return NULL;
330
331		pic = pic_list[pic_id];
332		KASSERT(pic != NULL);
333		if (pic->pic_pending_ipls & ipl_mask)
334			return pic;
335		pending_pics &= ~__BIT(pic_id);
336	}
337}
338
339void
340pic_list_deliver_irqs(register_t psw, int ipl, void *frame)
341{
342	const uint32_t ipl_mask = __BIT(ipl);
343	struct pic_softc *pic;
344
345	while ((pic = pic_list_find_pic_by_pending_ipl(ipl_mask)) != NULL) {
346		pic_deliver_irqs(pic, ipl, frame);
347		KASSERT((pic->pic_pending_ipls & ipl_mask) == 0);
348	}
349	atomic_and_32(&pic_pending_ipls, ~ipl_mask);
350}
351
352void
353pic_do_pending_ints(register_t psw, int newipl, void *frame)
354{
355	struct cpu_info * const ci = curcpu();
356	if (__predict_false(newipl == IPL_HIGH))
357		return;
358	while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
359		KASSERT(pic_pending_ipls < __BIT(NIPL));
360		for (;;) {
361			int ipl = 31 - __builtin_clz(pic_pending_ipls);
362			KASSERT(ipl < NIPL);
363			if (ipl <= newipl)
364				break;
365
366			ci->ci_cpl = ipl;
367			pic_list_deliver_irqs(psw, ipl, frame);
368			pic_list_unblock_irqs();
369		}
370	}
371	if (ci->ci_cpl != newipl)
372		ci->ci_cpl = newipl;
373#ifdef __HAVE_FAST_SOFTINTS
374	cpu_dosoftints();
375#endif
376}
377
378void
379pic_add(struct pic_softc *pic, int irqbase)
380{
381	int slot, maybe_slot = -1;
382
383	for (slot = 0; slot < PIC_MAXPICS; slot++) {
384		struct pic_softc * const xpic = pic_list[slot];
385		if (xpic == NULL) {
386			if (maybe_slot < 0)
387				maybe_slot = slot;
388			if (irqbase < 0)
389				break;
390			continue;
391		}
392		if (irqbase < 0 || xpic->pic_irqbase < 0)
393			continue;
394		if (irqbase >= xpic->pic_irqbase + xpic->pic_maxsources)
395			continue;
396		if (irqbase + pic->pic_maxsources <= xpic->pic_irqbase)
397			continue;
398		panic("pic_add: pic %s (%zu sources @ irq %u) conflicts"
399		    " with pic %s (%zu sources @ irq %u)",
400		    pic->pic_name, pic->pic_maxsources, irqbase,
401		    xpic->pic_name, xpic->pic_maxsources, xpic->pic_irqbase);
402	}
403	slot = maybe_slot;
404#if 0
405	printf("%s: pic_sourcebase=%zu pic_maxsources=%zu\n",
406	    pic->pic_name, pic_sourcebase, pic->pic_maxsources);
407#endif
408	KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES);
409
410	pic->pic_sources = &pic_sources[pic_sourcebase];
411	pic->pic_irqbase = irqbase;
412	pic_sourcebase += pic->pic_maxsources;
413	pic->pic_id = slot;
414	pic_list[slot] = pic;
415}
416
417int
418pic_alloc_irq(struct pic_softc *pic)
419{
420	int irq;
421
422	for (irq = 0; irq < pic->pic_maxsources; irq++) {
423		if (pic->pic_sources[irq] == NULL)
424			return irq;
425	}
426
427	return -1;
428}
429
430void *
431pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type,
432	int (*func)(void *), void *arg)
433{
434	struct intrsource *is;
435	int off, nipl;
436
437	if (pic->pic_sources[irq]) {
438		printf("pic_establish_intr: pic %s irq %d already present\n",
439		    pic->pic_name, irq);
440		return NULL;
441	}
442
443	is = malloc(sizeof(*is), M_INTRSOURCE, M_NOWAIT|M_ZERO);
444	if (is == NULL)
445		return NULL;
446
447	is->is_pic = pic;
448	is->is_irq = irq;
449	is->is_ipl = ipl;
450	is->is_type = type;
451	is->is_func = func;
452	is->is_arg = arg;
453
454	if (pic->pic_ops->pic_source_name)
455		(*pic->pic_ops->pic_source_name)(pic, irq, is->is_source,
456		    sizeof(is->is_source));
457	else
458		snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq);
459
460	evcnt_attach_dynamic(&is->is_ev, EVCNT_TYPE_INTR, NULL,
461	    pic->pic_name, is->is_source);
462
463	pic->pic_sources[irq] = is;
464
465	/*
466	 * First try to use an existing slot which is empty.
467	 */
468	for (off = pic_ipl_offset[ipl]; off < pic_ipl_offset[ipl+1]; off++) {
469		if (pic__iplsources[off] == NULL) {
470			is->is_iplidx = off - pic_ipl_offset[ipl];
471			pic__iplsources[off] = is;
472			return is;
473		}
474	}
475
476	/*
477	 * Move up all the sources by one.
478 	 */
479	if (ipl < NIPL) {
480		off = pic_ipl_offset[ipl+1];
481		memmove(&pic__iplsources[off+1], &pic__iplsources[off],
482		    sizeof(pic__iplsources[0]) * (pic_ipl_offset[NIPL] - off));
483	}
484
485	/*
486	 * Advance the offset of all IPLs higher than this.  Include an
487	 * extra one as well.  Thus the number of sources per ipl is
488	 * pic_ipl_offset[ipl+1] - pic_ipl_offset[ipl].
489	 */
490	for (nipl = ipl + 1; nipl <= NIPL; nipl++)
491		pic_ipl_offset[nipl]++;
492
493	/*
494	 * Insert into the previously made position at the end of this IPL's
495	 * sources.
496	 */
497	off = pic_ipl_offset[ipl + 1] - 1;
498	is->is_iplidx = off - pic_ipl_offset[ipl];
499	pic__iplsources[off] = is;
500
501	(*pic->pic_ops->pic_establish_irq)(pic, is);
502
503	(*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f,
504	    __BIT(is->is_irq & 0x1f));
505
506	/* We're done. */
507	return is;
508}
509
510void
511pic_disestablish_source(struct intrsource *is)
512{
513	struct pic_softc * const pic = is->is_pic;
514	const int irq = is->is_irq;
515
516	(*pic->pic_ops->pic_block_irqs)(pic, irq & ~31, __BIT(irq));
517	pic->pic_sources[irq] = NULL;
518	pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL;
519	evcnt_detach(&is->is_ev);
520
521	free(is, M_INTRSOURCE);
522}
523
524int
525_splraise(int newipl)
526{
527	struct cpu_info * const ci = curcpu();
528	const int oldipl = ci->ci_cpl;
529	KASSERT(newipl < NIPL);
530	if (newipl > ci->ci_cpl)
531		ci->ci_cpl = newipl;
532	return oldipl;
533}
534int
535_spllower(int newipl)
536{
537	struct cpu_info * const ci = curcpu();
538	const int oldipl = ci->ci_cpl;
539	KASSERT(panicstr || newipl <= ci->ci_cpl);
540	if (newipl < ci->ci_cpl) {
541		register_t psw = disable_interrupts(I32_bit);
542		pic_do_pending_ints(psw, newipl, NULL);
543		restore_interrupts(psw);
544	}
545	return oldipl;
546}
547
548void
549splx(int savedipl)
550{
551	struct cpu_info * const ci = curcpu();
552	KASSERT(savedipl < NIPL);
553	if (savedipl < ci->ci_cpl) {
554		register_t psw = disable_interrupts(I32_bit);
555		pic_do_pending_ints(psw, savedipl, NULL);
556		restore_interrupts(psw);
557	}
558	ci->ci_cpl = savedipl;
559}
560
561void *
562intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg)
563{
564	int slot;
565
566	for (slot = 0; slot < PIC_MAXPICS; slot++) {
567		struct pic_softc * const pic = pic_list[slot];
568		if (pic == NULL || pic->pic_irqbase < 0)
569			continue;
570		if (pic->pic_irqbase <= irq
571		    && irq < pic->pic_irqbase + pic->pic_maxsources) {
572			return pic_establish_intr(pic, irq - pic->pic_irqbase,
573			    ipl, type, func, arg);
574		}
575	}
576
577	return NULL;
578}
579
580void
581intr_disestablish(void *ih)
582{
583	struct intrsource * const is = ih;
584	pic_disestablish_source(is);
585}
586