1/******************************************************************************
2 * xen_intr.c
3 *
4 * Xen event and interrupt services for x86 PV and HVM guests.
5 *
6 * Copyright (c) 2002-2005, K A Fraser
7 * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com>
8 * Copyright (c) 2012, Spectra Logic Corporation
9 *
10 * This file may be distributed separately from the Linux kernel, or
11 * incorporated into other software packages, subject to the following license:
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
17 * and to permit persons to whom the Software is furnished to do so, subject to
18 * the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 * IN THE SOFTWARE.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/x86/xen/xen_intr.c 255726 2013-09-20 05:06:03Z gibbs $");
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/bus.h>
38#include <sys/malloc.h>
39#include <sys/kernel.h>
40#include <sys/limits.h>
41#include <sys/lock.h>
42#include <sys/mutex.h>
43#include <sys/interrupt.h>
44#include <sys/pcpu.h>
45#include <sys/smp.h>
46
47#include <vm/vm.h>
48#include <vm/pmap.h>
49
50#include <machine/intr_machdep.h>
51#include <machine/apicvar.h>
52#include <machine/smp.h>
53#include <machine/stdarg.h>
54
55#include <machine/xen/synch_bitops.h>
56#include <machine/xen/xen-os.h>
57#include <machine/xen/xenvar.h>
58
59#include <xen/hypervisor.h>
60#include <xen/xen_intr.h>
61#include <xen/evtchn/evtchnvar.h>
62
63#include <dev/xen/xenpci/xenpcivar.h>
64
65static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services");
66
67/**
68 * Per-cpu event channel processing state.
69 */
70struct xen_intr_pcpu_data {
71 /**
72 * The last event channel bitmap section (level one bit) processed.
73 * This is used to ensure we scan all ports before
74 * servicing an already servied port again.
75 */
76 u_int last_processed_l1i;
77
78 /**
79 * The last event channel processed within the event channel
80 * bitmap being scanned.
81 */
82 u_int last_processed_l2i;
83
84 /** Pointer to this CPU's interrupt statistic counter. */
85 u_long *evtchn_intrcnt;
86
87 /**
88 * A bitmap of ports that can be serviced from this CPU.
89 * A set bit means interrupt handling is enabled.
90 */
91 u_long evtchn_enabled[sizeof(u_long) * 8];
92};
93
94/*
95 * Start the scan at port 0 by initializing the last scanned
96 * location as the highest numbered event channel port.
97 */
98DPCPU_DEFINE(struct xen_intr_pcpu_data, xen_intr_pcpu) = {
99 .last_processed_l1i = LONG_BIT - 1,
100 .last_processed_l2i = LONG_BIT - 1
101};
102
103DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
104
105#define is_valid_evtchn(x) ((x) != 0)
106
107struct xenisrc {
108 struct intsrc xi_intsrc;
109 enum evtchn_type xi_type;
110 int xi_cpu; /* VCPU for delivery. */
111 int xi_vector; /* Global isrc vector number. */
112 evtchn_port_t xi_port;
113 int xi_pirq;
114 int xi_virq;
115 u_int xi_close:1; /* close on unbind? */
116 u_int xi_needs_eoi:1;
117 u_int xi_shared:1; /* Shared with other domains. */
118};
119
120#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
121
122static void xen_intr_suspend(struct pic *);
123static void xen_intr_resume(struct pic *, bool suspend_cancelled);
124static void xen_intr_enable_source(struct intsrc *isrc);
125static void xen_intr_disable_source(struct intsrc *isrc, int eoi);
126static void xen_intr_eoi_source(struct intsrc *isrc);
127static void xen_intr_enable_intr(struct intsrc *isrc);
128static void xen_intr_disable_intr(struct intsrc *isrc);
129static int xen_intr_vector(struct intsrc *isrc);
130static int xen_intr_source_pending(struct intsrc *isrc);
131static int xen_intr_config_intr(struct intsrc *isrc,
132 enum intr_trigger trig, enum intr_polarity pol);
133static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id);
134
135static void xen_intr_pirq_enable_source(struct intsrc *isrc);
136static void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi);
137static void xen_intr_pirq_eoi_source(struct intsrc *isrc);
138static void xen_intr_pirq_enable_intr(struct intsrc *isrc);
139
140/**
141 * PIC interface for all event channel port types except physical IRQs.
142 */
143struct pic xen_intr_pic = {
144 .pic_enable_source = xen_intr_enable_source,
145 .pic_disable_source = xen_intr_disable_source,
146 .pic_eoi_source = xen_intr_eoi_source,
147 .pic_enable_intr = xen_intr_enable_intr,
148 .pic_disable_intr = xen_intr_disable_intr,
149 .pic_vector = xen_intr_vector,
150 .pic_source_pending = xen_intr_source_pending,
151 .pic_suspend = xen_intr_suspend,
152 .pic_resume = xen_intr_resume,
153 .pic_config_intr = xen_intr_config_intr,
154 .pic_assign_cpu = xen_intr_assign_cpu
155};
156
157/**
158 * PIC interface for all event channel representing
159 * physical interrupt sources.
160 */
161struct pic xen_intr_pirq_pic = {
162 .pic_enable_source = xen_intr_pirq_enable_source,
163 .pic_disable_source = xen_intr_pirq_disable_source,
164 .pic_eoi_source = xen_intr_pirq_eoi_source,
165 .pic_enable_intr = xen_intr_pirq_enable_intr,
166 .pic_disable_intr = xen_intr_disable_intr,
167 .pic_vector = xen_intr_vector,
168 .pic_source_pending = xen_intr_source_pending,
169 .pic_suspend = xen_intr_suspend,
170 .pic_resume = xen_intr_resume,
171 .pic_config_intr = xen_intr_config_intr,
172 .pic_assign_cpu = xen_intr_assign_cpu
173};
174
175static struct mtx xen_intr_isrc_lock;
176static int xen_intr_isrc_count;
177static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS];
178
179/*------------------------- Private Functions --------------------------------*/
180/**
181 * Disable signal delivery for an event channel port on the
182 * specified CPU.
183 *
184 * \param port The event channel port to mask.
185 *
186 * This API is used to manage the port<=>CPU binding of event
187 * channel handlers.
188 *
189 * \note This operation does not preclude reception of an event
190 * for this event channel on another CPU. To mask the
191 * event channel globally, use evtchn_mask().
192 */
193static inline void
194evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port)
195{
196 struct xen_intr_pcpu_data *pcpu;
197
198 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
199 clear_bit(port, pcpu->evtchn_enabled);
200}
201
202/**
203 * Enable signal delivery for an event channel port on the
204 * specified CPU.
205 *
206 * \param port The event channel port to unmask.
207 *
208 * This API is used to manage the port<=>CPU binding of event
209 * channel handlers.
210 *
211 * \note This operation does not guarantee that event delivery
212 * is enabled for this event channel port. The port must
213 * also be globally enabled. See evtchn_unmask().
214 */
215static inline void
216evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port)
217{
218 struct xen_intr_pcpu_data *pcpu;
219
220 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
221 set_bit(port, pcpu->evtchn_enabled);
222}
223
224/**
225 * Allocate and register a per-cpu Xen upcall interrupt counter.
226 *
227 * \param cpu The cpu for which to register this interrupt count.
228 */
229static void
230xen_intr_intrcnt_add(u_int cpu)
231{
232 char buf[MAXCOMLEN + 1];
233 struct xen_intr_pcpu_data *pcpu;
234
235 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
236 if (pcpu->evtchn_intrcnt != NULL)
237 return;
238
239 snprintf(buf, sizeof(buf), "cpu%d:xen", cpu);
240 intrcnt_add(buf, &pcpu->evtchn_intrcnt);
241}
242
243/**
244 * Search for an already allocated but currently unused Xen interrupt
245 * source object.
246 *
247 * \param type Restrict the search to interrupt sources of the given
248 * type.
249 *
250 * \return A pointer to a free Xen interrupt source object or NULL.
251 */
252static struct xenisrc *
253xen_intr_find_unused_isrc(enum evtchn_type type)
254{
255 int isrc_idx;
256
257 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held"));
258
259 for (isrc_idx = 0; isrc_idx < xen_intr_isrc_count; isrc_idx ++) {
260 struct xenisrc *isrc;
261 u_int vector;
262
263 vector = FIRST_EVTCHN_INT + isrc_idx;
264 isrc = (struct xenisrc *)intr_lookup_source(vector);
265 if (isrc != NULL
266 && isrc->xi_type == EVTCHN_TYPE_UNBOUND) {
267 KASSERT(isrc->xi_intsrc.is_handlers == 0,
268 ("Free evtchn still has handlers"));
269 isrc->xi_type = type;
270 return (isrc);
271 }
272 }
273 return (NULL);
274}
275
276/**
277 * Allocate a Xen interrupt source object.
278 *
279 * \param type The type of interrupt source to create.
280 *
281 * \return A pointer to a newly allocated Xen interrupt source
282 * object or NULL.
283 */
284static struct xenisrc *
285xen_intr_alloc_isrc(enum evtchn_type type)
286{
287 static int warned;
288 struct xenisrc *isrc;
289 int vector;
290
291 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held"));
292
293 if (xen_intr_isrc_count > NR_EVENT_CHANNELS) {
294 if (!warned) {
295 warned = 1;
296 printf("xen_intr_alloc: Event channels exhausted.\n");
297 }
298 return (NULL);
299 }
300 vector = FIRST_EVTCHN_INT + xen_intr_isrc_count;
301 xen_intr_isrc_count++;
302
303 mtx_unlock(&xen_intr_isrc_lock);
304 isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO);
305 isrc->xi_intsrc.is_pic = &xen_intr_pic;
306 isrc->xi_vector = vector;
307 isrc->xi_type = type;
308 intr_register_source(&isrc->xi_intsrc);
309 mtx_lock(&xen_intr_isrc_lock);
310
311 return (isrc);
312}
313
314/**
315 * Attempt to free an active Xen interrupt source object.
316 *
317 * \param isrc The interrupt source object to release.
318 *
319 * \returns EBUSY if the source is still in use, otherwise 0.
320 */
321static int
322xen_intr_release_isrc(struct xenisrc *isrc)
323{
324
325 mtx_lock(&xen_intr_isrc_lock);
326 if (isrc->xi_intsrc.is_handlers != 0) {
327 mtx_unlock(&xen_intr_isrc_lock);
328 return (EBUSY);
329 }
330 evtchn_mask_port(isrc->xi_port);
331 evtchn_clear_port(isrc->xi_port);
332
333 /* Rebind port to CPU 0. */
334 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
335 evtchn_cpu_unmask_port(0, isrc->xi_port);
336
337 if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) {
338 struct evtchn_close close = { .port = isrc->xi_port };
339 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
340 panic("EVTCHNOP_close failed");
341 }
342
343 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
344 isrc->xi_cpu = 0;
345 isrc->xi_type = EVTCHN_TYPE_UNBOUND;
346 isrc->xi_port = 0;
347 mtx_unlock(&xen_intr_isrc_lock);
348 return (0);
349}
350
351/**
352 * Associate an interrupt handler with an already allocated local Xen
353 * event channel port.
354 *
355 * \param isrcp The returned Xen interrupt object associated with
356 * the specified local port.
357 * \param local_port The event channel to bind.
358 * \param type The event channel type of local_port.
359 * \param intr_owner The device making this bind request.
360 * \param filter An interrupt filter handler. Specify NULL
361 * to always dispatch to the ithread handler.
362 * \param handler An interrupt ithread handler. Optional (can
363 * specify NULL) if all necessary event actions
364 * are performed by filter.
365 * \param arg Argument to present to both filter and handler.
366 * \param irqflags Interrupt handler flags. See sys/bus.h.
367 * \param handlep Pointer to an opaque handle used to manage this
368 * registration.
369 *
370 * \returns 0 on success, otherwise an errno.
371 */
372static int
373xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port,
374 enum evtchn_type type, device_t intr_owner, driver_filter_t filter,
375 driver_intr_t handler, void *arg, enum intr_type flags,
376 xen_intr_handle_t *port_handlep)
377{
378 struct xenisrc *isrc;
379 int error;
380
381 *isrcp = NULL;
382 if (port_handlep == NULL) {
383 device_printf(intr_owner,
384 "xen_intr_bind_isrc: Bad event handle\n");
385 return (EINVAL);
386 }
387
388 mtx_lock(&xen_intr_isrc_lock);
389 isrc = xen_intr_find_unused_isrc(type);
390 if (isrc == NULL) {
391 isrc = xen_intr_alloc_isrc(type);
392 if (isrc == NULL) {
393 mtx_unlock(&xen_intr_isrc_lock);
394 return (ENOSPC);
395 }
396 }
397 isrc->xi_port = local_port;
398 xen_intr_port_to_isrc[local_port] = isrc;
399 mtx_unlock(&xen_intr_isrc_lock);
400
401 error = intr_add_handler(device_get_nameunit(intr_owner),
402 isrc->xi_vector, filter, handler, arg,
403 flags|INTR_EXCL, port_handlep);
404 if (error != 0) {
405 device_printf(intr_owner,
406 "xen_intr_bind_irq: intr_add_handler failed\n");
407 xen_intr_release_isrc(isrc);
408 return (error);
409 }
410 *isrcp = isrc;
411 evtchn_unmask_port(local_port);
412 return (0);
413}
414
415/**
416 * Lookup a Xen interrupt source object given an interrupt binding handle.
417 *
418 * \param handle A handle initialized by a previous call to
419 * xen_intr_bind_isrc().
420 *
421 * \returns A pointer to the Xen interrupt source object associated
422 * with the given interrupt handle. NULL if no association
423 * currently exists.
424 */
425static struct xenisrc *
426xen_intr_isrc(xen_intr_handle_t handle)
427{
428 struct intr_handler *ih;
429
430 ih = handle;
431 if (ih == NULL || ih->ih_event == NULL)
432 return (NULL);
433
434 return (ih->ih_event->ie_source);
435}
436
437/**
438 * Determine the event channel ports at the given section of the
439 * event port bitmap which have pending events for the given cpu.
440 *
441 * \param pcpu The Xen interrupt pcpu data for the cpu being querried.
442 * \param sh The Xen shared info area.
443 * \param idx The index of the section of the event channel bitmap to
444 * inspect.
445 *
446 * \returns A u_long with bits set for every event channel with pending
447 * events.
448 */
449static inline u_long
450xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh,
451 u_int idx)
452{
453 return (sh->evtchn_pending[idx]
454 & ~sh->evtchn_mask[idx]
455 & pcpu->evtchn_enabled[idx]);
456}
457
458/**
459 * Interrupt handler for processing all Xen event channel events.
460 *
461 * \param trap_frame The trap frame context for the current interrupt.
462 */
463void
464xen_intr_handle_upcall(struct trapframe *trap_frame)
465{
466 u_int l1i, l2i, port, cpu;
467 u_long masked_l1, masked_l2;
468 struct xenisrc *isrc;
469 shared_info_t *s;
470 vcpu_info_t *v;
471 struct xen_intr_pcpu_data *pc;
472 u_long l1, l2;
473
474 /*
475 * Disable preemption in order to always check and fire events
476 * on the right vCPU
477 */
478 critical_enter();
479
480 cpu = PCPU_GET(cpuid);
481 pc = DPCPU_PTR(xen_intr_pcpu);
482 s = HYPERVISOR_shared_info;
483 v = DPCPU_GET(vcpu_info);
484
485 if (xen_hvm_domain() && !xen_vector_callback_enabled) {
486 KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU"));
487 }
488
489 v->evtchn_upcall_pending = 0;
490
491#if 0
492#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
493 /* Clear master flag /before/ clearing selector flag. */
494 wmb();
495#endif
496#endif
497
498 l1 = atomic_readandclear_long(&v->evtchn_pending_sel);
499
500 l1i = pc->last_processed_l1i;
501 l2i = pc->last_processed_l2i;
502 (*pc->evtchn_intrcnt)++;
503
504 while (l1 != 0) {
505
506 l1i = (l1i + 1) % LONG_BIT;
507 masked_l1 = l1 & ((~0UL) << l1i);
508
509 if (masked_l1 == 0) {
510 /*
511 * if we masked out all events, wrap around
512 * to the beginning.
513 */
514 l1i = LONG_BIT - 1;
515 l2i = LONG_BIT - 1;
516 continue;
517 }
518 l1i = ffsl(masked_l1) - 1;
519
520 do {
521 l2 = xen_intr_active_ports(pc, s, l1i);
522
523 l2i = (l2i + 1) % LONG_BIT;
524 masked_l2 = l2 & ((~0UL) << l2i);
525
526 if (masked_l2 == 0) {
527 /* if we masked out all events, move on */
528 l2i = LONG_BIT - 1;
529 break;
530 }
531 l2i = ffsl(masked_l2) - 1;
532
533 /* process port */
534 port = (l1i * LONG_BIT) + l2i;
535 synch_clear_bit(port, &s->evtchn_pending[0]);
536
537 isrc = xen_intr_port_to_isrc[port];
538 if (__predict_false(isrc == NULL))
539 continue;
540
541 /* Make sure we are firing on the right vCPU */
542 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)),
543 ("Received unexpected event on vCPU#%d, event bound to vCPU#%d",
544 PCPU_GET(cpuid), isrc->xi_cpu));
545
546 intr_execute_handlers(&isrc->xi_intsrc, trap_frame);
547
548 /*
549 * If this is the final port processed,
550 * we'll pick up here+1 next time.
551 */
552 pc->last_processed_l1i = l1i;
553 pc->last_processed_l2i = l2i;
554
555 } while (l2i != LONG_BIT - 1);
556
557 l2 = xen_intr_active_ports(pc, s, l1i);
558 if (l2 == 0) {
559 /*
560 * We handled all ports, so we can clear the
561 * selector bit.
562 */
563 l1 &= ~(1UL << l1i);
564 }
565 }
566 critical_exit();
567}
568
569static int
570xen_intr_init(void *dummy __unused)
571{
572 struct xen_intr_pcpu_data *pcpu;
573 int i;
574
575 if (!xen_domain())
576 return (0);
577
578 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF);
579
580 /*
581 * Register interrupt count manually as we aren't
582 * guaranteed to see a call to xen_intr_assign_cpu()
583 * before our first interrupt. Also set the per-cpu
584 * mask of CPU#0 to enable all, since by default
585 * all event channels are bound to CPU#0.
586 */
587 CPU_FOREACH(i) {
588 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
589 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
590 sizeof(pcpu->evtchn_enabled));
591 xen_intr_intrcnt_add(i);
592 }
593
594 intr_register_pic(&xen_intr_pic);
595
596 return (0);
597}
598SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intr_init, NULL);
599
600/*--------------------------- Common PIC Functions ---------------------------*/
601/**
602 * Prepare this PIC for system suspension.
603 */
604static void
605xen_intr_suspend(struct pic *unused)
606{
607}
608
609static void
610xen_rebind_ipi(struct xenisrc *isrc)
611{
612#ifdef SMP
613 int cpu = isrc->xi_cpu;
614 int acpi_id = pcpu_find(cpu)->pc_acpi_id;
615 int error;
616 struct evtchn_bind_ipi bind_ipi = { .vcpu = acpi_id };
617
618 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
619 &bind_ipi);
620 if (error != 0)
621 panic("unable to rebind xen IPI: %d", error);
622
623 isrc->xi_port = bind_ipi.port;
624 isrc->xi_cpu = 0;
625 xen_intr_port_to_isrc[bind_ipi.port] = isrc;
626
627 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
628 cpu_apic_ids[cpu]);
629 if (error)
630 panic("unable to bind xen IPI to CPU#%d: %d",
631 cpu, error);
632
633 evtchn_unmask_port(bind_ipi.port);
634#else
635 panic("Resume IPI event channel on UP");
636#endif
637}
638
639static void
640xen_rebind_virq(struct xenisrc *isrc)
641{
642 int cpu = isrc->xi_cpu;
643 int acpi_id = pcpu_find(cpu)->pc_acpi_id;
644 int error;
645 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq,
646 .vcpu = acpi_id };
647
648 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
649 &bind_virq);
650 if (error != 0)
651 panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error);
652
653 isrc->xi_port = bind_virq.port;
654 isrc->xi_cpu = 0;
655 xen_intr_port_to_isrc[bind_virq.port] = isrc;
656
657#ifdef SMP
658 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
659 cpu_apic_ids[cpu]);
660 if (error)
661 panic("unable to bind xen VIRQ#%d to CPU#%d: %d",
662 isrc->xi_virq, cpu, error);
663#endif
664
665 evtchn_unmask_port(bind_virq.port);
666}
667
668/**
669 * Return this PIC to service after being suspended.
670 */
671static void
672xen_intr_resume(struct pic *unused, bool suspend_cancelled)
673{
674 shared_info_t *s = HYPERVISOR_shared_info;
675 struct xenisrc *isrc;
676 u_int isrc_idx;
677 int i;
678
679 if (suspend_cancelled)
680 return;
681
682 /* Reset the per-CPU masks */
683 CPU_FOREACH(i) {
684 struct xen_intr_pcpu_data *pcpu;
685
686 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
687 memset(pcpu->evtchn_enabled,
688 i == 0 ? ~0 : 0, sizeof(pcpu->evtchn_enabled));
689 }
690
691 /* Mask all event channels. */
692 for (i = 0; i < nitems(s->evtchn_mask); i++)
693 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
694
695 /* Remove port -> isrc mappings */
696 memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc));
697
698 /* Free unused isrcs and rebind VIRQs and IPIs */
699 for (isrc_idx = 0; isrc_idx < xen_intr_isrc_count; isrc_idx++) {
700 u_int vector;
701
702 vector = FIRST_EVTCHN_INT + isrc_idx;
703 isrc = (struct xenisrc *)intr_lookup_source(vector);
704 if (isrc != NULL) {
705 isrc->xi_port = 0;
706 switch (isrc->xi_type) {
707 case EVTCHN_TYPE_IPI:
708 xen_rebind_ipi(isrc);
709 break;
710 case EVTCHN_TYPE_VIRQ:
711 xen_rebind_virq(isrc);
712 break;
713 default:
714 isrc->xi_cpu = 0;
715 break;
716 }
717 }
718 }
719}
720
721/**
722 * Disable a Xen interrupt source.
723 *
724 * \param isrc The interrupt source to disable.
725 */
726static void
727xen_intr_disable_intr(struct intsrc *base_isrc)
728{
729 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
730
731 evtchn_mask_port(isrc->xi_port);
732}
733
734/**
735 * Determine the global interrupt vector number for
736 * a Xen interrupt source.
737 *
738 * \param isrc The interrupt source to query.
739 *
740 * \return The vector number corresponding to the given interrupt source.
741 */
742static int
743xen_intr_vector(struct intsrc *base_isrc)
744{
745 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
746
747 return (isrc->xi_vector);
748}
749
750/**
751 * Determine whether or not interrupt events are pending on the
752 * the given interrupt source.
753 *
754 * \param isrc The interrupt source to query.
755 *
756 * \returns 0 if no events are pending, otherwise non-zero.
757 */
758static int
759xen_intr_source_pending(struct intsrc *isrc)
760{
761 /*
762 * EventChannels are edge triggered and never masked.
763 * There can be no pending events.
764 */
765 return (0);
766}
767
768/**
769 * Perform configuration of an interrupt source.
770 *
771 * \param isrc The interrupt source to configure.
772 * \param trig Edge or level.
773 * \param pol Active high or low.
774 *
775 * \returns 0 if no events are pending, otherwise non-zero.
776 */
777static int
778xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig,
779 enum intr_polarity pol)
780{
781 /* Configuration is only possible via the evtchn apis. */
782 return (ENODEV);
783}
784
785/**
786 * Configure CPU affinity for interrupt source event delivery.
787 *
788 * \param isrc The interrupt source to configure.
789 * \param apic_id The apic id of the CPU for handling future events.
790 *
791 * \returns 0 if successful, otherwise an errno.
792 */
793static int
794xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id)
795{
796#ifdef SMP
797 struct evtchn_bind_vcpu bind_vcpu;
798 struct xenisrc *isrc;
799 u_int to_cpu, acpi_id;
800 int error;
801
802#ifdef XENHVM
803 if (xen_vector_callback_enabled == 0)
804 return (EOPNOTSUPP);
805#endif
806
807 to_cpu = apic_cpuid(apic_id);
808 acpi_id = pcpu_find(to_cpu)->pc_acpi_id;
809 xen_intr_intrcnt_add(to_cpu);
810
811 mtx_lock(&xen_intr_isrc_lock);
812 isrc = (struct xenisrc *)base_isrc;
813 if (!is_valid_evtchn(isrc->xi_port)) {
814 mtx_unlock(&xen_intr_isrc_lock);
815 return (EINVAL);
816 }
817
818 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) ||
819 (isrc->xi_type == EVTCHN_TYPE_IPI)) {
820 /*
821 * Virtual IRQs are associated with a cpu by
822 * the Hypervisor at evtchn_bind_virq time, so
823 * all we need to do is update the per-CPU masks.
824 */
825 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
826 isrc->xi_cpu = to_cpu;
827 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
828 mtx_unlock(&xen_intr_isrc_lock);
829 return (0);
830 }
831
832 bind_vcpu.port = isrc->xi_port;
833 bind_vcpu.vcpu = acpi_id;
834
835 /*
836 * Allow interrupts to be fielded on the new VCPU before
837 * we ask the hypervisor to deliver them there.
838 */
839 evtchn_cpu_unmask_port(to_cpu, isrc->xi_port);
840 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu);
841 if (isrc->xi_cpu != to_cpu) {
842 if (error == 0) {
843 /* Commit to new binding by removing the old one. */
844 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
845 isrc->xi_cpu = to_cpu;
846 } else {
847 /* Roll-back to previous binding. */
848 evtchn_cpu_mask_port(to_cpu, isrc->xi_port);
849 }
850 }
851 mtx_unlock(&xen_intr_isrc_lock);
852 return (0);
853#else
854 return (EOPNOTSUPP);
855#endif
856}
857
858/*------------------- Virtual Interrupt Source PIC Functions -----------------*/
859/*
860 * Mask a level triggered interrupt source.
861 *
862 * \param isrc The interrupt source to mask (if necessary).
863 * \param eoi If non-zero, perform any necessary end-of-interrupt
864 * acknowledgements.
865 */
866static void
867xen_intr_disable_source(struct intsrc *isrc, int eoi)
868{
869}
870
871/*
872 * Unmask a level triggered interrupt source.
873 *
874 * \param isrc The interrupt source to unmask (if necessary).
875 */
876static void
877xen_intr_enable_source(struct intsrc *isrc)
878{
879}
880
881/*
882 * Perform any necessary end-of-interrupt acknowledgements.
883 *
884 * \param isrc The interrupt source to EOI.
885 */
886static void
887xen_intr_eoi_source(struct intsrc *isrc)
888{
889}
890
891/*
892 * Enable and unmask the interrupt source.
893 *
894 * \param isrc The interrupt source to enable.
895 */
896static void
897xen_intr_enable_intr(struct intsrc *base_isrc)
898{
899 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
900
901 evtchn_unmask_port(isrc->xi_port);
902}
903
904/*------------------ Physical Interrupt Source PIC Functions -----------------*/
905/*
906 * Mask a level triggered interrupt source.
907 *
908 * \param isrc The interrupt source to mask (if necessary).
909 * \param eoi If non-zero, perform any necessary end-of-interrupt
910 * acknowledgements.
911 */
912static void
913xen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi)
914{
915 struct xenisrc *isrc;
916
917 isrc = (struct xenisrc *)base_isrc;
918 evtchn_mask_port(isrc->xi_port);
919}
920
921/*
922 * Unmask a level triggered interrupt source.
923 *
924 * \param isrc The interrupt source to unmask (if necessary).
925 */
926static void
927xen_intr_pirq_enable_source(struct intsrc *base_isrc)
928{
929 struct xenisrc *isrc;
930
931 isrc = (struct xenisrc *)base_isrc;
932 evtchn_unmask_port(isrc->xi_port);
933}
934
935/*
936 * Perform any necessary end-of-interrupt acknowledgements.
937 *
938 * \param isrc The interrupt source to EOI.
939 */
940static void
941xen_intr_pirq_eoi_source(struct intsrc *base_isrc)
942{
943 struct xenisrc *isrc;
944
945 /* XXX Use shared page of flags for this. */
946 isrc = (struct xenisrc *)base_isrc;
947 if (isrc->xi_needs_eoi != 0) {
948 struct physdev_eoi eoi = { .irq = isrc->xi_pirq };
949
950 (void)HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
951 }
952}
953
954/*
955 * Enable and unmask the interrupt source.
956 *
957 * \param isrc The interrupt source to enable.
958 */
959static void
960xen_intr_pirq_enable_intr(struct intsrc *isrc)
961{
962}
963
964/*--------------------------- Public Functions -------------------------------*/
965/*------- API comments for these methods can be found in xen/xenintr.h -------*/
966int
967xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port,
968 driver_filter_t filter, driver_intr_t handler, void *arg,
969 enum intr_type flags, xen_intr_handle_t *port_handlep)
970{
971 struct xenisrc *isrc;
972 int error;
973
974 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, dev,
975 filter, handler, arg, flags, port_handlep);
976 if (error != 0)
977 return (error);
978
979 /*
980 * The Event Channel API didn't open this port, so it is not
981 * responsible for closing it automatically on unbind.
982 */
983 isrc->xi_close = 0;
984 return (0);
985}
986
987int
988xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain,
989 driver_filter_t filter, driver_intr_t handler, void *arg,
990 enum intr_type flags, xen_intr_handle_t *port_handlep)
991{
992 struct xenisrc *isrc;
993 struct evtchn_alloc_unbound alloc_unbound;
994 int error;
995
996 alloc_unbound.dom = DOMID_SELF;
997 alloc_unbound.remote_dom = remote_domain;
998 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
999 &alloc_unbound);
1000 if (error != 0) {
1001 /*
1002 * XXX Trap Hypercall error code Linuxisms in
1003 * the HYPERCALL layer.
1004 */
1005 return (-error);
1006 }
1007
1008 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT,
1009 dev, filter, handler, arg, flags,
1010 port_handlep);
1011 if (error != 0) {
1012 evtchn_close_t close = { .port = alloc_unbound.port };
1013 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1014 panic("EVTCHNOP_close failed");
1015 return (error);
1016 }
1017
1018 isrc->xi_close = 1;
1019 return (0);
1020}
1021
1022int
1023xen_intr_bind_remote_port(device_t dev, u_int remote_domain,
1024 u_int remote_port, driver_filter_t filter, driver_intr_t handler,
1025 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep)
1026{
1027 struct xenisrc *isrc;
1028 struct evtchn_bind_interdomain bind_interdomain;
1029 int error;
1030
1031 bind_interdomain.remote_dom = remote_domain;
1032 bind_interdomain.remote_port = remote_port;
1033 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1034 &bind_interdomain);
1035 if (error != 0) {
1036 /*
1037 * XXX Trap Hypercall error code Linuxisms in
1038 * the HYPERCALL layer.
1039 */
1040 return (-error);
1041 }
1042
1043 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port,
1044 EVTCHN_TYPE_PORT, dev, filter, handler,
1045 arg, flags, port_handlep);
1046 if (error) {
1047 evtchn_close_t close = { .port = bind_interdomain.local_port };
1048 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1049 panic("EVTCHNOP_close failed");
1050 return (error);
1051 }
1052
1053 /*
1054 * The Event Channel API opened this port, so it is
1055 * responsible for closing it automatically on unbind.
1056 */
1057 isrc->xi_close = 1;
1058 return (0);
1059}
1060
1061int
1062xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu,
1063 driver_filter_t filter, driver_intr_t handler, void *arg,
1064 enum intr_type flags, xen_intr_handle_t *port_handlep)
1065{
1066 int acpi_id = pcpu_find(cpu)->pc_acpi_id;
1067 struct xenisrc *isrc;
1068 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = acpi_id };
1069 int error;
1070
1071 /* Ensure the target CPU is ready to handle evtchn interrupts. */
1072 xen_intr_intrcnt_add(cpu);
1073
1074 isrc = NULL;
1075 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
1076 if (error != 0) {
1077 /*
1078 * XXX Trap Hypercall error code Linuxisms in
1079 * the HYPERCALL layer.
1080 */
1081 return (-error);
1082 }
1083
1084 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, dev,
1085 filter, handler, arg, flags, port_handlep);
1086
1087#ifdef SMP
1088 if (error == 0)
1089 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1090#endif
1091
1092 if (error != 0) {
1093 evtchn_close_t close = { .port = bind_virq.port };
1094
1095 xen_intr_unbind(*port_handlep);
1096 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1097 panic("EVTCHNOP_close failed");
1098 return (error);
1099 }
1100
1101#ifdef SMP
1102 if (isrc->xi_cpu != cpu) {
1103 /*
1104 * Too early in the boot process for the generic interrupt
1105 * code to perform the binding. Update our event channel
1106 * masks manually so events can't fire on the wrong cpu
1107 * during AP startup.
1108 */
1109 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1110 }
1111#endif
1112
1113 /*
1114 * The Event Channel API opened this port, so it is
1115 * responsible for closing it automatically on unbind.
1116 */
1117 isrc->xi_close = 1;
1118 isrc->xi_virq = virq;
1119
1120 return (0);
1121}
1122
1123int
1124xen_intr_alloc_and_bind_ipi(device_t dev, u_int cpu,
1125 driver_filter_t filter, enum intr_type flags,
1126 xen_intr_handle_t *port_handlep)
1127{
1128#ifdef SMP
1129 int acpi_id = pcpu_find(cpu)->pc_acpi_id;
1130 struct xenisrc *isrc;
1131 struct evtchn_bind_ipi bind_ipi = { .vcpu = acpi_id };
1132 int error;
1133
1134 /* Ensure the target CPU is ready to handle evtchn interrupts. */
1135 xen_intr_intrcnt_add(cpu);
1136
1137 isrc = NULL;
1138 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
1139 if (error != 0) {
1140 /*
1141 * XXX Trap Hypercall error code Linuxisms in
1142 * the HYPERCALL layer.
1143 */
1144 return (-error);
1145 }
1146
1147 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI,
1148 dev, filter, NULL, NULL, flags,
1149 port_handlep);
1150 if (error == 0)
1151 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1152
1153 if (error != 0) {
1154 evtchn_close_t close = { .port = bind_ipi.port };
1155
1156 xen_intr_unbind(*port_handlep);
1157 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1158 panic("EVTCHNOP_close failed");
1159 return (error);
1160 }
1161
1162 if (isrc->xi_cpu != cpu) {
1163 /*
1164 * Too early in the boot process for the generic interrupt
1165 * code to perform the binding. Update our event channel
1166 * masks manually so events can't fire on the wrong cpu
1167 * during AP startup.
1168 */
1169 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1170 }
1171
1172 /*
1173 * The Event Channel API opened this port, so it is
1174 * responsible for closing it automatically on unbind.
1175 */
1176 isrc->xi_close = 1;
1177 return (0);
1178#else
1179 return (EOPNOTSUPP);
1180#endif
1181}
1182
1183int
1184xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...)
1185{
1186 char descr[MAXCOMLEN + 1];
1187 struct xenisrc *isrc;
1188 va_list ap;
1189
1190 isrc = xen_intr_isrc(port_handle);
1191 if (isrc == NULL)
1192 return (EINVAL);
1193
1194 va_start(ap, fmt);
1195 vsnprintf(descr, sizeof(descr), fmt, ap);
1196 va_end(ap);
1197 return (intr_describe(isrc->xi_vector, port_handle, descr));
1198}
1199
1200void
1201xen_intr_unbind(xen_intr_handle_t *port_handlep)
1202{
1203 struct intr_handler *handler;
1204 struct xenisrc *isrc;
1205
1206 handler = *port_handlep;
1207 *port_handlep = NULL;
1208 isrc = xen_intr_isrc(handler);
1209 if (isrc == NULL)
1210 return;
1211
1212 intr_remove_handler(handler);
1213 xen_intr_release_isrc(isrc);
1214}
1215
1216void
1217xen_intr_signal(xen_intr_handle_t handle)
1218{
1219 struct xenisrc *isrc;
1220
1221 isrc = xen_intr_isrc(handle);
1222 if (isrc != NULL) {
1223 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT ||
1224 isrc->xi_type == EVTCHN_TYPE_IPI,
1225 ("evtchn_signal on something other than a local port"));
1226 struct evtchn_send send = { .port = isrc->xi_port };
1227 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
1228 }
1229}
1230
1231evtchn_port_t
1232xen_intr_port(xen_intr_handle_t handle)
1233{
1234 struct xenisrc *isrc;
1235
1236 isrc = xen_intr_isrc(handle);
1237 if (isrc == NULL)
1238 return (0);
1239
1240 return (isrc->xi_port);
1241}
2 * xen_intr.c
3 *
4 * Xen event and interrupt services for x86 PV and HVM guests.
5 *
6 * Copyright (c) 2002-2005, K A Fraser
7 * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com>
8 * Copyright (c) 2012, Spectra Logic Corporation
9 *
10 * This file may be distributed separately from the Linux kernel, or
11 * incorporated into other software packages, subject to the following license:
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
17 * and to permit persons to whom the Software is furnished to do so, subject to
18 * the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 * IN THE SOFTWARE.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/x86/xen/xen_intr.c 255726 2013-09-20 05:06:03Z gibbs $");
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/bus.h>
38#include <sys/malloc.h>
39#include <sys/kernel.h>
40#include <sys/limits.h>
41#include <sys/lock.h>
42#include <sys/mutex.h>
43#include <sys/interrupt.h>
44#include <sys/pcpu.h>
45#include <sys/smp.h>
46
47#include <vm/vm.h>
48#include <vm/pmap.h>
49
50#include <machine/intr_machdep.h>
51#include <machine/apicvar.h>
52#include <machine/smp.h>
53#include <machine/stdarg.h>
54
55#include <machine/xen/synch_bitops.h>
56#include <machine/xen/xen-os.h>
57#include <machine/xen/xenvar.h>
58
59#include <xen/hypervisor.h>
60#include <xen/xen_intr.h>
61#include <xen/evtchn/evtchnvar.h>
62
63#include <dev/xen/xenpci/xenpcivar.h>
64
65static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services");
66
67/**
68 * Per-cpu event channel processing state.
69 */
70struct xen_intr_pcpu_data {
71 /**
72 * The last event channel bitmap section (level one bit) processed.
73 * This is used to ensure we scan all ports before
74 * servicing an already servied port again.
75 */
76 u_int last_processed_l1i;
77
78 /**
79 * The last event channel processed within the event channel
80 * bitmap being scanned.
81 */
82 u_int last_processed_l2i;
83
84 /** Pointer to this CPU's interrupt statistic counter. */
85 u_long *evtchn_intrcnt;
86
87 /**
88 * A bitmap of ports that can be serviced from this CPU.
89 * A set bit means interrupt handling is enabled.
90 */
91 u_long evtchn_enabled[sizeof(u_long) * 8];
92};
93
94/*
95 * Start the scan at port 0 by initializing the last scanned
96 * location as the highest numbered event channel port.
97 */
98DPCPU_DEFINE(struct xen_intr_pcpu_data, xen_intr_pcpu) = {
99 .last_processed_l1i = LONG_BIT - 1,
100 .last_processed_l2i = LONG_BIT - 1
101};
102
103DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
104
105#define is_valid_evtchn(x) ((x) != 0)
106
107struct xenisrc {
108 struct intsrc xi_intsrc;
109 enum evtchn_type xi_type;
110 int xi_cpu; /* VCPU for delivery. */
111 int xi_vector; /* Global isrc vector number. */
112 evtchn_port_t xi_port;
113 int xi_pirq;
114 int xi_virq;
115 u_int xi_close:1; /* close on unbind? */
116 u_int xi_needs_eoi:1;
117 u_int xi_shared:1; /* Shared with other domains. */
118};
119
120#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
121
122static void xen_intr_suspend(struct pic *);
123static void xen_intr_resume(struct pic *, bool suspend_cancelled);
124static void xen_intr_enable_source(struct intsrc *isrc);
125static void xen_intr_disable_source(struct intsrc *isrc, int eoi);
126static void xen_intr_eoi_source(struct intsrc *isrc);
127static void xen_intr_enable_intr(struct intsrc *isrc);
128static void xen_intr_disable_intr(struct intsrc *isrc);
129static int xen_intr_vector(struct intsrc *isrc);
130static int xen_intr_source_pending(struct intsrc *isrc);
131static int xen_intr_config_intr(struct intsrc *isrc,
132 enum intr_trigger trig, enum intr_polarity pol);
133static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id);
134
135static void xen_intr_pirq_enable_source(struct intsrc *isrc);
136static void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi);
137static void xen_intr_pirq_eoi_source(struct intsrc *isrc);
138static void xen_intr_pirq_enable_intr(struct intsrc *isrc);
139
140/**
141 * PIC interface for all event channel port types except physical IRQs.
142 */
143struct pic xen_intr_pic = {
144 .pic_enable_source = xen_intr_enable_source,
145 .pic_disable_source = xen_intr_disable_source,
146 .pic_eoi_source = xen_intr_eoi_source,
147 .pic_enable_intr = xen_intr_enable_intr,
148 .pic_disable_intr = xen_intr_disable_intr,
149 .pic_vector = xen_intr_vector,
150 .pic_source_pending = xen_intr_source_pending,
151 .pic_suspend = xen_intr_suspend,
152 .pic_resume = xen_intr_resume,
153 .pic_config_intr = xen_intr_config_intr,
154 .pic_assign_cpu = xen_intr_assign_cpu
155};
156
157/**
158 * PIC interface for all event channel representing
159 * physical interrupt sources.
160 */
161struct pic xen_intr_pirq_pic = {
162 .pic_enable_source = xen_intr_pirq_enable_source,
163 .pic_disable_source = xen_intr_pirq_disable_source,
164 .pic_eoi_source = xen_intr_pirq_eoi_source,
165 .pic_enable_intr = xen_intr_pirq_enable_intr,
166 .pic_disable_intr = xen_intr_disable_intr,
167 .pic_vector = xen_intr_vector,
168 .pic_source_pending = xen_intr_source_pending,
169 .pic_suspend = xen_intr_suspend,
170 .pic_resume = xen_intr_resume,
171 .pic_config_intr = xen_intr_config_intr,
172 .pic_assign_cpu = xen_intr_assign_cpu
173};
174
175static struct mtx xen_intr_isrc_lock;
176static int xen_intr_isrc_count;
177static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS];
178
179/*------------------------- Private Functions --------------------------------*/
180/**
181 * Disable signal delivery for an event channel port on the
182 * specified CPU.
183 *
184 * \param port The event channel port to mask.
185 *
186 * This API is used to manage the port<=>CPU binding of event
187 * channel handlers.
188 *
189 * \note This operation does not preclude reception of an event
190 * for this event channel on another CPU. To mask the
191 * event channel globally, use evtchn_mask().
192 */
193static inline void
194evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port)
195{
196 struct xen_intr_pcpu_data *pcpu;
197
198 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
199 clear_bit(port, pcpu->evtchn_enabled);
200}
201
202/**
203 * Enable signal delivery for an event channel port on the
204 * specified CPU.
205 *
206 * \param port The event channel port to unmask.
207 *
208 * This API is used to manage the port<=>CPU binding of event
209 * channel handlers.
210 *
211 * \note This operation does not guarantee that event delivery
212 * is enabled for this event channel port. The port must
213 * also be globally enabled. See evtchn_unmask().
214 */
215static inline void
216evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port)
217{
218 struct xen_intr_pcpu_data *pcpu;
219
220 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
221 set_bit(port, pcpu->evtchn_enabled);
222}
223
224/**
225 * Allocate and register a per-cpu Xen upcall interrupt counter.
226 *
227 * \param cpu The cpu for which to register this interrupt count.
228 */
229static void
230xen_intr_intrcnt_add(u_int cpu)
231{
232 char buf[MAXCOMLEN + 1];
233 struct xen_intr_pcpu_data *pcpu;
234
235 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
236 if (pcpu->evtchn_intrcnt != NULL)
237 return;
238
239 snprintf(buf, sizeof(buf), "cpu%d:xen", cpu);
240 intrcnt_add(buf, &pcpu->evtchn_intrcnt);
241}
242
243/**
244 * Search for an already allocated but currently unused Xen interrupt
245 * source object.
246 *
247 * \param type Restrict the search to interrupt sources of the given
248 * type.
249 *
250 * \return A pointer to a free Xen interrupt source object or NULL.
251 */
252static struct xenisrc *
253xen_intr_find_unused_isrc(enum evtchn_type type)
254{
255 int isrc_idx;
256
257 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held"));
258
259 for (isrc_idx = 0; isrc_idx < xen_intr_isrc_count; isrc_idx ++) {
260 struct xenisrc *isrc;
261 u_int vector;
262
263 vector = FIRST_EVTCHN_INT + isrc_idx;
264 isrc = (struct xenisrc *)intr_lookup_source(vector);
265 if (isrc != NULL
266 && isrc->xi_type == EVTCHN_TYPE_UNBOUND) {
267 KASSERT(isrc->xi_intsrc.is_handlers == 0,
268 ("Free evtchn still has handlers"));
269 isrc->xi_type = type;
270 return (isrc);
271 }
272 }
273 return (NULL);
274}
275
276/**
277 * Allocate a Xen interrupt source object.
278 *
279 * \param type The type of interrupt source to create.
280 *
281 * \return A pointer to a newly allocated Xen interrupt source
282 * object or NULL.
283 */
284static struct xenisrc *
285xen_intr_alloc_isrc(enum evtchn_type type)
286{
287 static int warned;
288 struct xenisrc *isrc;
289 int vector;
290
291 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held"));
292
293 if (xen_intr_isrc_count > NR_EVENT_CHANNELS) {
294 if (!warned) {
295 warned = 1;
296 printf("xen_intr_alloc: Event channels exhausted.\n");
297 }
298 return (NULL);
299 }
300 vector = FIRST_EVTCHN_INT + xen_intr_isrc_count;
301 xen_intr_isrc_count++;
302
303 mtx_unlock(&xen_intr_isrc_lock);
304 isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO);
305 isrc->xi_intsrc.is_pic = &xen_intr_pic;
306 isrc->xi_vector = vector;
307 isrc->xi_type = type;
308 intr_register_source(&isrc->xi_intsrc);
309 mtx_lock(&xen_intr_isrc_lock);
310
311 return (isrc);
312}
313
314/**
315 * Attempt to free an active Xen interrupt source object.
316 *
317 * \param isrc The interrupt source object to release.
318 *
319 * \returns EBUSY if the source is still in use, otherwise 0.
320 */
321static int
322xen_intr_release_isrc(struct xenisrc *isrc)
323{
324
325 mtx_lock(&xen_intr_isrc_lock);
326 if (isrc->xi_intsrc.is_handlers != 0) {
327 mtx_unlock(&xen_intr_isrc_lock);
328 return (EBUSY);
329 }
330 evtchn_mask_port(isrc->xi_port);
331 evtchn_clear_port(isrc->xi_port);
332
333 /* Rebind port to CPU 0. */
334 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
335 evtchn_cpu_unmask_port(0, isrc->xi_port);
336
337 if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) {
338 struct evtchn_close close = { .port = isrc->xi_port };
339 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
340 panic("EVTCHNOP_close failed");
341 }
342
343 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
344 isrc->xi_cpu = 0;
345 isrc->xi_type = EVTCHN_TYPE_UNBOUND;
346 isrc->xi_port = 0;
347 mtx_unlock(&xen_intr_isrc_lock);
348 return (0);
349}
350
351/**
352 * Associate an interrupt handler with an already allocated local Xen
353 * event channel port.
354 *
355 * \param isrcp The returned Xen interrupt object associated with
356 * the specified local port.
357 * \param local_port The event channel to bind.
358 * \param type The event channel type of local_port.
359 * \param intr_owner The device making this bind request.
360 * \param filter An interrupt filter handler. Specify NULL
361 * to always dispatch to the ithread handler.
362 * \param handler An interrupt ithread handler. Optional (can
363 * specify NULL) if all necessary event actions
364 * are performed by filter.
365 * \param arg Argument to present to both filter and handler.
366 * \param irqflags Interrupt handler flags. See sys/bus.h.
367 * \param handlep Pointer to an opaque handle used to manage this
368 * registration.
369 *
370 * \returns 0 on success, otherwise an errno.
371 */
372static int
373xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port,
374 enum evtchn_type type, device_t intr_owner, driver_filter_t filter,
375 driver_intr_t handler, void *arg, enum intr_type flags,
376 xen_intr_handle_t *port_handlep)
377{
378 struct xenisrc *isrc;
379 int error;
380
381 *isrcp = NULL;
382 if (port_handlep == NULL) {
383 device_printf(intr_owner,
384 "xen_intr_bind_isrc: Bad event handle\n");
385 return (EINVAL);
386 }
387
388 mtx_lock(&xen_intr_isrc_lock);
389 isrc = xen_intr_find_unused_isrc(type);
390 if (isrc == NULL) {
391 isrc = xen_intr_alloc_isrc(type);
392 if (isrc == NULL) {
393 mtx_unlock(&xen_intr_isrc_lock);
394 return (ENOSPC);
395 }
396 }
397 isrc->xi_port = local_port;
398 xen_intr_port_to_isrc[local_port] = isrc;
399 mtx_unlock(&xen_intr_isrc_lock);
400
401 error = intr_add_handler(device_get_nameunit(intr_owner),
402 isrc->xi_vector, filter, handler, arg,
403 flags|INTR_EXCL, port_handlep);
404 if (error != 0) {
405 device_printf(intr_owner,
406 "xen_intr_bind_irq: intr_add_handler failed\n");
407 xen_intr_release_isrc(isrc);
408 return (error);
409 }
410 *isrcp = isrc;
411 evtchn_unmask_port(local_port);
412 return (0);
413}
414
415/**
416 * Lookup a Xen interrupt source object given an interrupt binding handle.
417 *
418 * \param handle A handle initialized by a previous call to
419 * xen_intr_bind_isrc().
420 *
421 * \returns A pointer to the Xen interrupt source object associated
422 * with the given interrupt handle. NULL if no association
423 * currently exists.
424 */
425static struct xenisrc *
426xen_intr_isrc(xen_intr_handle_t handle)
427{
428 struct intr_handler *ih;
429
430 ih = handle;
431 if (ih == NULL || ih->ih_event == NULL)
432 return (NULL);
433
434 return (ih->ih_event->ie_source);
435}
436
437/**
438 * Determine the event channel ports at the given section of the
439 * event port bitmap which have pending events for the given cpu.
440 *
441 * \param pcpu The Xen interrupt pcpu data for the cpu being querried.
442 * \param sh The Xen shared info area.
443 * \param idx The index of the section of the event channel bitmap to
444 * inspect.
445 *
446 * \returns A u_long with bits set for every event channel with pending
447 * events.
448 */
449static inline u_long
450xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh,
451 u_int idx)
452{
453 return (sh->evtchn_pending[idx]
454 & ~sh->evtchn_mask[idx]
455 & pcpu->evtchn_enabled[idx]);
456}
457
458/**
459 * Interrupt handler for processing all Xen event channel events.
460 *
461 * \param trap_frame The trap frame context for the current interrupt.
462 */
463void
464xen_intr_handle_upcall(struct trapframe *trap_frame)
465{
466 u_int l1i, l2i, port, cpu;
467 u_long masked_l1, masked_l2;
468 struct xenisrc *isrc;
469 shared_info_t *s;
470 vcpu_info_t *v;
471 struct xen_intr_pcpu_data *pc;
472 u_long l1, l2;
473
474 /*
475 * Disable preemption in order to always check and fire events
476 * on the right vCPU
477 */
478 critical_enter();
479
480 cpu = PCPU_GET(cpuid);
481 pc = DPCPU_PTR(xen_intr_pcpu);
482 s = HYPERVISOR_shared_info;
483 v = DPCPU_GET(vcpu_info);
484
485 if (xen_hvm_domain() && !xen_vector_callback_enabled) {
486 KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU"));
487 }
488
489 v->evtchn_upcall_pending = 0;
490
491#if 0
492#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
493 /* Clear master flag /before/ clearing selector flag. */
494 wmb();
495#endif
496#endif
497
498 l1 = atomic_readandclear_long(&v->evtchn_pending_sel);
499
500 l1i = pc->last_processed_l1i;
501 l2i = pc->last_processed_l2i;
502 (*pc->evtchn_intrcnt)++;
503
504 while (l1 != 0) {
505
506 l1i = (l1i + 1) % LONG_BIT;
507 masked_l1 = l1 & ((~0UL) << l1i);
508
509 if (masked_l1 == 0) {
510 /*
511 * if we masked out all events, wrap around
512 * to the beginning.
513 */
514 l1i = LONG_BIT - 1;
515 l2i = LONG_BIT - 1;
516 continue;
517 }
518 l1i = ffsl(masked_l1) - 1;
519
520 do {
521 l2 = xen_intr_active_ports(pc, s, l1i);
522
523 l2i = (l2i + 1) % LONG_BIT;
524 masked_l2 = l2 & ((~0UL) << l2i);
525
526 if (masked_l2 == 0) {
527 /* if we masked out all events, move on */
528 l2i = LONG_BIT - 1;
529 break;
530 }
531 l2i = ffsl(masked_l2) - 1;
532
533 /* process port */
534 port = (l1i * LONG_BIT) + l2i;
535 synch_clear_bit(port, &s->evtchn_pending[0]);
536
537 isrc = xen_intr_port_to_isrc[port];
538 if (__predict_false(isrc == NULL))
539 continue;
540
541 /* Make sure we are firing on the right vCPU */
542 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)),
543 ("Received unexpected event on vCPU#%d, event bound to vCPU#%d",
544 PCPU_GET(cpuid), isrc->xi_cpu));
545
546 intr_execute_handlers(&isrc->xi_intsrc, trap_frame);
547
548 /*
549 * If this is the final port processed,
550 * we'll pick up here+1 next time.
551 */
552 pc->last_processed_l1i = l1i;
553 pc->last_processed_l2i = l2i;
554
555 } while (l2i != LONG_BIT - 1);
556
557 l2 = xen_intr_active_ports(pc, s, l1i);
558 if (l2 == 0) {
559 /*
560 * We handled all ports, so we can clear the
561 * selector bit.
562 */
563 l1 &= ~(1UL << l1i);
564 }
565 }
566 critical_exit();
567}
568
569static int
570xen_intr_init(void *dummy __unused)
571{
572 struct xen_intr_pcpu_data *pcpu;
573 int i;
574
575 if (!xen_domain())
576 return (0);
577
578 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF);
579
580 /*
581 * Register interrupt count manually as we aren't
582 * guaranteed to see a call to xen_intr_assign_cpu()
583 * before our first interrupt. Also set the per-cpu
584 * mask of CPU#0 to enable all, since by default
585 * all event channels are bound to CPU#0.
586 */
587 CPU_FOREACH(i) {
588 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
589 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
590 sizeof(pcpu->evtchn_enabled));
591 xen_intr_intrcnt_add(i);
592 }
593
594 intr_register_pic(&xen_intr_pic);
595
596 return (0);
597}
598SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intr_init, NULL);
599
600/*--------------------------- Common PIC Functions ---------------------------*/
601/**
602 * Prepare this PIC for system suspension.
603 */
604static void
605xen_intr_suspend(struct pic *unused)
606{
607}
608
609static void
610xen_rebind_ipi(struct xenisrc *isrc)
611{
612#ifdef SMP
613 int cpu = isrc->xi_cpu;
614 int acpi_id = pcpu_find(cpu)->pc_acpi_id;
615 int error;
616 struct evtchn_bind_ipi bind_ipi = { .vcpu = acpi_id };
617
618 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
619 &bind_ipi);
620 if (error != 0)
621 panic("unable to rebind xen IPI: %d", error);
622
623 isrc->xi_port = bind_ipi.port;
624 isrc->xi_cpu = 0;
625 xen_intr_port_to_isrc[bind_ipi.port] = isrc;
626
627 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
628 cpu_apic_ids[cpu]);
629 if (error)
630 panic("unable to bind xen IPI to CPU#%d: %d",
631 cpu, error);
632
633 evtchn_unmask_port(bind_ipi.port);
634#else
635 panic("Resume IPI event channel on UP");
636#endif
637}
638
639static void
640xen_rebind_virq(struct xenisrc *isrc)
641{
642 int cpu = isrc->xi_cpu;
643 int acpi_id = pcpu_find(cpu)->pc_acpi_id;
644 int error;
645 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq,
646 .vcpu = acpi_id };
647
648 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
649 &bind_virq);
650 if (error != 0)
651 panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error);
652
653 isrc->xi_port = bind_virq.port;
654 isrc->xi_cpu = 0;
655 xen_intr_port_to_isrc[bind_virq.port] = isrc;
656
657#ifdef SMP
658 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
659 cpu_apic_ids[cpu]);
660 if (error)
661 panic("unable to bind xen VIRQ#%d to CPU#%d: %d",
662 isrc->xi_virq, cpu, error);
663#endif
664
665 evtchn_unmask_port(bind_virq.port);
666}
667
668/**
669 * Return this PIC to service after being suspended.
670 */
671static void
672xen_intr_resume(struct pic *unused, bool suspend_cancelled)
673{
674 shared_info_t *s = HYPERVISOR_shared_info;
675 struct xenisrc *isrc;
676 u_int isrc_idx;
677 int i;
678
679 if (suspend_cancelled)
680 return;
681
682 /* Reset the per-CPU masks */
683 CPU_FOREACH(i) {
684 struct xen_intr_pcpu_data *pcpu;
685
686 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
687 memset(pcpu->evtchn_enabled,
688 i == 0 ? ~0 : 0, sizeof(pcpu->evtchn_enabled));
689 }
690
691 /* Mask all event channels. */
692 for (i = 0; i < nitems(s->evtchn_mask); i++)
693 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
694
695 /* Remove port -> isrc mappings */
696 memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc));
697
698 /* Free unused isrcs and rebind VIRQs and IPIs */
699 for (isrc_idx = 0; isrc_idx < xen_intr_isrc_count; isrc_idx++) {
700 u_int vector;
701
702 vector = FIRST_EVTCHN_INT + isrc_idx;
703 isrc = (struct xenisrc *)intr_lookup_source(vector);
704 if (isrc != NULL) {
705 isrc->xi_port = 0;
706 switch (isrc->xi_type) {
707 case EVTCHN_TYPE_IPI:
708 xen_rebind_ipi(isrc);
709 break;
710 case EVTCHN_TYPE_VIRQ:
711 xen_rebind_virq(isrc);
712 break;
713 default:
714 isrc->xi_cpu = 0;
715 break;
716 }
717 }
718 }
719}
720
721/**
722 * Disable a Xen interrupt source.
723 *
724 * \param isrc The interrupt source to disable.
725 */
726static void
727xen_intr_disable_intr(struct intsrc *base_isrc)
728{
729 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
730
731 evtchn_mask_port(isrc->xi_port);
732}
733
734/**
735 * Determine the global interrupt vector number for
736 * a Xen interrupt source.
737 *
738 * \param isrc The interrupt source to query.
739 *
740 * \return The vector number corresponding to the given interrupt source.
741 */
742static int
743xen_intr_vector(struct intsrc *base_isrc)
744{
745 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
746
747 return (isrc->xi_vector);
748}
749
750/**
751 * Determine whether or not interrupt events are pending on the
752 * the given interrupt source.
753 *
754 * \param isrc The interrupt source to query.
755 *
756 * \returns 0 if no events are pending, otherwise non-zero.
757 */
758static int
759xen_intr_source_pending(struct intsrc *isrc)
760{
761 /*
762 * EventChannels are edge triggered and never masked.
763 * There can be no pending events.
764 */
765 return (0);
766}
767
768/**
769 * Perform configuration of an interrupt source.
770 *
771 * \param isrc The interrupt source to configure.
772 * \param trig Edge or level.
773 * \param pol Active high or low.
774 *
775 * \returns 0 if no events are pending, otherwise non-zero.
776 */
777static int
778xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig,
779 enum intr_polarity pol)
780{
781 /* Configuration is only possible via the evtchn apis. */
782 return (ENODEV);
783}
784
785/**
786 * Configure CPU affinity for interrupt source event delivery.
787 *
788 * \param isrc The interrupt source to configure.
789 * \param apic_id The apic id of the CPU for handling future events.
790 *
791 * \returns 0 if successful, otherwise an errno.
792 */
793static int
794xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id)
795{
796#ifdef SMP
797 struct evtchn_bind_vcpu bind_vcpu;
798 struct xenisrc *isrc;
799 u_int to_cpu, acpi_id;
800 int error;
801
802#ifdef XENHVM
803 if (xen_vector_callback_enabled == 0)
804 return (EOPNOTSUPP);
805#endif
806
807 to_cpu = apic_cpuid(apic_id);
808 acpi_id = pcpu_find(to_cpu)->pc_acpi_id;
809 xen_intr_intrcnt_add(to_cpu);
810
811 mtx_lock(&xen_intr_isrc_lock);
812 isrc = (struct xenisrc *)base_isrc;
813 if (!is_valid_evtchn(isrc->xi_port)) {
814 mtx_unlock(&xen_intr_isrc_lock);
815 return (EINVAL);
816 }
817
818 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) ||
819 (isrc->xi_type == EVTCHN_TYPE_IPI)) {
820 /*
821 * Virtual IRQs are associated with a cpu by
822 * the Hypervisor at evtchn_bind_virq time, so
823 * all we need to do is update the per-CPU masks.
824 */
825 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
826 isrc->xi_cpu = to_cpu;
827 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
828 mtx_unlock(&xen_intr_isrc_lock);
829 return (0);
830 }
831
832 bind_vcpu.port = isrc->xi_port;
833 bind_vcpu.vcpu = acpi_id;
834
835 /*
836 * Allow interrupts to be fielded on the new VCPU before
837 * we ask the hypervisor to deliver them there.
838 */
839 evtchn_cpu_unmask_port(to_cpu, isrc->xi_port);
840 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu);
841 if (isrc->xi_cpu != to_cpu) {
842 if (error == 0) {
843 /* Commit to new binding by removing the old one. */
844 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
845 isrc->xi_cpu = to_cpu;
846 } else {
847 /* Roll-back to previous binding. */
848 evtchn_cpu_mask_port(to_cpu, isrc->xi_port);
849 }
850 }
851 mtx_unlock(&xen_intr_isrc_lock);
852 return (0);
853#else
854 return (EOPNOTSUPP);
855#endif
856}
857
858/*------------------- Virtual Interrupt Source PIC Functions -----------------*/
859/*
860 * Mask a level triggered interrupt source.
861 *
862 * \param isrc The interrupt source to mask (if necessary).
863 * \param eoi If non-zero, perform any necessary end-of-interrupt
864 * acknowledgements.
865 */
866static void
867xen_intr_disable_source(struct intsrc *isrc, int eoi)
868{
869}
870
871/*
872 * Unmask a level triggered interrupt source.
873 *
874 * \param isrc The interrupt source to unmask (if necessary).
875 */
876static void
877xen_intr_enable_source(struct intsrc *isrc)
878{
879}
880
881/*
882 * Perform any necessary end-of-interrupt acknowledgements.
883 *
884 * \param isrc The interrupt source to EOI.
885 */
886static void
887xen_intr_eoi_source(struct intsrc *isrc)
888{
889}
890
891/*
892 * Enable and unmask the interrupt source.
893 *
894 * \param isrc The interrupt source to enable.
895 */
896static void
897xen_intr_enable_intr(struct intsrc *base_isrc)
898{
899 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
900
901 evtchn_unmask_port(isrc->xi_port);
902}
903
904/*------------------ Physical Interrupt Source PIC Functions -----------------*/
905/*
906 * Mask a level triggered interrupt source.
907 *
908 * \param isrc The interrupt source to mask (if necessary).
909 * \param eoi If non-zero, perform any necessary end-of-interrupt
910 * acknowledgements.
911 */
912static void
913xen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi)
914{
915 struct xenisrc *isrc;
916
917 isrc = (struct xenisrc *)base_isrc;
918 evtchn_mask_port(isrc->xi_port);
919}
920
921/*
922 * Unmask a level triggered interrupt source.
923 *
924 * \param isrc The interrupt source to unmask (if necessary).
925 */
926static void
927xen_intr_pirq_enable_source(struct intsrc *base_isrc)
928{
929 struct xenisrc *isrc;
930
931 isrc = (struct xenisrc *)base_isrc;
932 evtchn_unmask_port(isrc->xi_port);
933}
934
935/*
936 * Perform any necessary end-of-interrupt acknowledgements.
937 *
938 * \param isrc The interrupt source to EOI.
939 */
940static void
941xen_intr_pirq_eoi_source(struct intsrc *base_isrc)
942{
943 struct xenisrc *isrc;
944
945 /* XXX Use shared page of flags for this. */
946 isrc = (struct xenisrc *)base_isrc;
947 if (isrc->xi_needs_eoi != 0) {
948 struct physdev_eoi eoi = { .irq = isrc->xi_pirq };
949
950 (void)HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
951 }
952}
953
954/*
955 * Enable and unmask the interrupt source.
956 *
957 * \param isrc The interrupt source to enable.
958 */
959static void
960xen_intr_pirq_enable_intr(struct intsrc *isrc)
961{
962}
963
964/*--------------------------- Public Functions -------------------------------*/
965/*------- API comments for these methods can be found in xen/xenintr.h -------*/
966int
967xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port,
968 driver_filter_t filter, driver_intr_t handler, void *arg,
969 enum intr_type flags, xen_intr_handle_t *port_handlep)
970{
971 struct xenisrc *isrc;
972 int error;
973
974 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, dev,
975 filter, handler, arg, flags, port_handlep);
976 if (error != 0)
977 return (error);
978
979 /*
980 * The Event Channel API didn't open this port, so it is not
981 * responsible for closing it automatically on unbind.
982 */
983 isrc->xi_close = 0;
984 return (0);
985}
986
987int
988xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain,
989 driver_filter_t filter, driver_intr_t handler, void *arg,
990 enum intr_type flags, xen_intr_handle_t *port_handlep)
991{
992 struct xenisrc *isrc;
993 struct evtchn_alloc_unbound alloc_unbound;
994 int error;
995
996 alloc_unbound.dom = DOMID_SELF;
997 alloc_unbound.remote_dom = remote_domain;
998 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
999 &alloc_unbound);
1000 if (error != 0) {
1001 /*
1002 * XXX Trap Hypercall error code Linuxisms in
1003 * the HYPERCALL layer.
1004 */
1005 return (-error);
1006 }
1007
1008 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT,
1009 dev, filter, handler, arg, flags,
1010 port_handlep);
1011 if (error != 0) {
1012 evtchn_close_t close = { .port = alloc_unbound.port };
1013 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1014 panic("EVTCHNOP_close failed");
1015 return (error);
1016 }
1017
1018 isrc->xi_close = 1;
1019 return (0);
1020}
1021
1022int
1023xen_intr_bind_remote_port(device_t dev, u_int remote_domain,
1024 u_int remote_port, driver_filter_t filter, driver_intr_t handler,
1025 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep)
1026{
1027 struct xenisrc *isrc;
1028 struct evtchn_bind_interdomain bind_interdomain;
1029 int error;
1030
1031 bind_interdomain.remote_dom = remote_domain;
1032 bind_interdomain.remote_port = remote_port;
1033 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1034 &bind_interdomain);
1035 if (error != 0) {
1036 /*
1037 * XXX Trap Hypercall error code Linuxisms in
1038 * the HYPERCALL layer.
1039 */
1040 return (-error);
1041 }
1042
1043 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port,
1044 EVTCHN_TYPE_PORT, dev, filter, handler,
1045 arg, flags, port_handlep);
1046 if (error) {
1047 evtchn_close_t close = { .port = bind_interdomain.local_port };
1048 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1049 panic("EVTCHNOP_close failed");
1050 return (error);
1051 }
1052
1053 /*
1054 * The Event Channel API opened this port, so it is
1055 * responsible for closing it automatically on unbind.
1056 */
1057 isrc->xi_close = 1;
1058 return (0);
1059}
1060
1061int
1062xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu,
1063 driver_filter_t filter, driver_intr_t handler, void *arg,
1064 enum intr_type flags, xen_intr_handle_t *port_handlep)
1065{
1066 int acpi_id = pcpu_find(cpu)->pc_acpi_id;
1067 struct xenisrc *isrc;
1068 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = acpi_id };
1069 int error;
1070
1071 /* Ensure the target CPU is ready to handle evtchn interrupts. */
1072 xen_intr_intrcnt_add(cpu);
1073
1074 isrc = NULL;
1075 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
1076 if (error != 0) {
1077 /*
1078 * XXX Trap Hypercall error code Linuxisms in
1079 * the HYPERCALL layer.
1080 */
1081 return (-error);
1082 }
1083
1084 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, dev,
1085 filter, handler, arg, flags, port_handlep);
1086
1087#ifdef SMP
1088 if (error == 0)
1089 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1090#endif
1091
1092 if (error != 0) {
1093 evtchn_close_t close = { .port = bind_virq.port };
1094
1095 xen_intr_unbind(*port_handlep);
1096 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1097 panic("EVTCHNOP_close failed");
1098 return (error);
1099 }
1100
1101#ifdef SMP
1102 if (isrc->xi_cpu != cpu) {
1103 /*
1104 * Too early in the boot process for the generic interrupt
1105 * code to perform the binding. Update our event channel
1106 * masks manually so events can't fire on the wrong cpu
1107 * during AP startup.
1108 */
1109 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1110 }
1111#endif
1112
1113 /*
1114 * The Event Channel API opened this port, so it is
1115 * responsible for closing it automatically on unbind.
1116 */
1117 isrc->xi_close = 1;
1118 isrc->xi_virq = virq;
1119
1120 return (0);
1121}
1122
1123int
1124xen_intr_alloc_and_bind_ipi(device_t dev, u_int cpu,
1125 driver_filter_t filter, enum intr_type flags,
1126 xen_intr_handle_t *port_handlep)
1127{
1128#ifdef SMP
1129 int acpi_id = pcpu_find(cpu)->pc_acpi_id;
1130 struct xenisrc *isrc;
1131 struct evtchn_bind_ipi bind_ipi = { .vcpu = acpi_id };
1132 int error;
1133
1134 /* Ensure the target CPU is ready to handle evtchn interrupts. */
1135 xen_intr_intrcnt_add(cpu);
1136
1137 isrc = NULL;
1138 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
1139 if (error != 0) {
1140 /*
1141 * XXX Trap Hypercall error code Linuxisms in
1142 * the HYPERCALL layer.
1143 */
1144 return (-error);
1145 }
1146
1147 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI,
1148 dev, filter, NULL, NULL, flags,
1149 port_handlep);
1150 if (error == 0)
1151 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1152
1153 if (error != 0) {
1154 evtchn_close_t close = { .port = bind_ipi.port };
1155
1156 xen_intr_unbind(*port_handlep);
1157 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1158 panic("EVTCHNOP_close failed");
1159 return (error);
1160 }
1161
1162 if (isrc->xi_cpu != cpu) {
1163 /*
1164 * Too early in the boot process for the generic interrupt
1165 * code to perform the binding. Update our event channel
1166 * masks manually so events can't fire on the wrong cpu
1167 * during AP startup.
1168 */
1169 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1170 }
1171
1172 /*
1173 * The Event Channel API opened this port, so it is
1174 * responsible for closing it automatically on unbind.
1175 */
1176 isrc->xi_close = 1;
1177 return (0);
1178#else
1179 return (EOPNOTSUPP);
1180#endif
1181}
1182
1183int
1184xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...)
1185{
1186 char descr[MAXCOMLEN + 1];
1187 struct xenisrc *isrc;
1188 va_list ap;
1189
1190 isrc = xen_intr_isrc(port_handle);
1191 if (isrc == NULL)
1192 return (EINVAL);
1193
1194 va_start(ap, fmt);
1195 vsnprintf(descr, sizeof(descr), fmt, ap);
1196 va_end(ap);
1197 return (intr_describe(isrc->xi_vector, port_handle, descr));
1198}
1199
1200void
1201xen_intr_unbind(xen_intr_handle_t *port_handlep)
1202{
1203 struct intr_handler *handler;
1204 struct xenisrc *isrc;
1205
1206 handler = *port_handlep;
1207 *port_handlep = NULL;
1208 isrc = xen_intr_isrc(handler);
1209 if (isrc == NULL)
1210 return;
1211
1212 intr_remove_handler(handler);
1213 xen_intr_release_isrc(isrc);
1214}
1215
1216void
1217xen_intr_signal(xen_intr_handle_t handle)
1218{
1219 struct xenisrc *isrc;
1220
1221 isrc = xen_intr_isrc(handle);
1222 if (isrc != NULL) {
1223 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT ||
1224 isrc->xi_type == EVTCHN_TYPE_IPI,
1225 ("evtchn_signal on something other than a local port"));
1226 struct evtchn_send send = { .port = isrc->xi_port };
1227 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
1228 }
1229}
1230
1231evtchn_port_t
1232xen_intr_port(xen_intr_handle_t handle)
1233{
1234 struct xenisrc *isrc;
1235
1236 isrc = xen_intr_isrc(handle);
1237 if (isrc == NULL)
1238 return (0);
1239
1240 return (isrc->xi_port);
1241}