1255040Sgibbs/****************************************************************************** 2255040Sgibbs * xen_intr.c 3255040Sgibbs * 4255040Sgibbs * Xen event and interrupt services for x86 PV and HVM guests. 5255040Sgibbs * 6255040Sgibbs * Copyright (c) 2002-2005, K A Fraser 7255040Sgibbs * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com> 8255040Sgibbs * Copyright (c) 2012, Spectra Logic Corporation 9255040Sgibbs * 10255040Sgibbs * This file may be distributed separately from the Linux kernel, or 11255040Sgibbs * incorporated into other software packages, subject to the following license: 12255040Sgibbs * 13255040Sgibbs * Permission is hereby granted, free of charge, to any person obtaining a copy 14255040Sgibbs * of this source file (the "Software"), to deal in the Software without 15255040Sgibbs * restriction, including without limitation the rights to use, copy, modify, 16255040Sgibbs * merge, publish, distribute, sublicense, and/or sell copies of the Software, 17255040Sgibbs * and to permit persons to whom the Software is furnished to do so, subject to 18255040Sgibbs * the following conditions: 19255040Sgibbs * 20255040Sgibbs * The above copyright notice and this permission notice shall be included in 21255040Sgibbs * all copies or substantial portions of the Software. 22255040Sgibbs * 23255040Sgibbs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24255040Sgibbs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25255040Sgibbs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 26255040Sgibbs * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27255040Sgibbs * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28255040Sgibbs * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 29255040Sgibbs * IN THE SOFTWARE. 30255040Sgibbs */ 31255040Sgibbs 32255040Sgibbs#include <sys/cdefs.h> 33255040Sgibbs__FBSDID("$FreeBSD$"); 34255040Sgibbs 35255040Sgibbs#include <sys/param.h> 36255040Sgibbs#include <sys/systm.h> 37255040Sgibbs#include <sys/bus.h> 38255040Sgibbs#include <sys/malloc.h> 39255040Sgibbs#include <sys/kernel.h> 40255040Sgibbs#include <sys/limits.h> 41255040Sgibbs#include <sys/lock.h> 42255040Sgibbs#include <sys/mutex.h> 43255040Sgibbs#include <sys/interrupt.h> 44255040Sgibbs#include <sys/pcpu.h> 45255040Sgibbs#include <sys/smp.h> 46255040Sgibbs 47255040Sgibbs#include <vm/vm.h> 48255040Sgibbs#include <vm/pmap.h> 49255040Sgibbs 50255040Sgibbs#include <machine/intr_machdep.h> 51255040Sgibbs#include <machine/apicvar.h> 52255040Sgibbs#include <machine/smp.h> 53255040Sgibbs#include <machine/stdarg.h> 54255040Sgibbs 55255040Sgibbs#include <machine/xen/synch_bitops.h> 56255040Sgibbs#include <machine/xen/xen-os.h> 57255040Sgibbs#include <machine/xen/xenvar.h> 58255040Sgibbs 59255040Sgibbs#include <xen/hypervisor.h> 60255040Sgibbs#include <xen/xen_intr.h> 61255040Sgibbs#include <xen/evtchn/evtchnvar.h> 62255040Sgibbs 63255040Sgibbs#include <dev/xen/xenpci/xenpcivar.h> 64255040Sgibbs 65255040Sgibbsstatic MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); 66255040Sgibbs 67255040Sgibbs/** 68255040Sgibbs * Per-cpu event channel processing state. 69255040Sgibbs */ 70255040Sgibbsstruct xen_intr_pcpu_data { 71255040Sgibbs /** 72255040Sgibbs * The last event channel bitmap section (level one bit) processed. 73255040Sgibbs * This is used to ensure we scan all ports before 74255040Sgibbs * servicing an already servied port again. 75255040Sgibbs */ 76255040Sgibbs u_int last_processed_l1i; 77255040Sgibbs 78255040Sgibbs /** 79255040Sgibbs * The last event channel processed within the event channel 80255040Sgibbs * bitmap being scanned. 81255040Sgibbs */ 82255040Sgibbs u_int last_processed_l2i; 83255040Sgibbs 84255040Sgibbs /** Pointer to this CPU's interrupt statistic counter. */ 85255040Sgibbs u_long *evtchn_intrcnt; 86255040Sgibbs 87255040Sgibbs /** 88255040Sgibbs * A bitmap of ports that can be serviced from this CPU. 89255040Sgibbs * A set bit means interrupt handling is enabled. 90255040Sgibbs */ 91255040Sgibbs u_long evtchn_enabled[sizeof(u_long) * 8]; 92255040Sgibbs}; 93255040Sgibbs 94255040Sgibbs/* 95255040Sgibbs * Start the scan at port 0 by initializing the last scanned 96255040Sgibbs * location as the highest numbered event channel port. 97255040Sgibbs */ 98255040SgibbsDPCPU_DEFINE(struct xen_intr_pcpu_data, xen_intr_pcpu) = { 99255040Sgibbs .last_processed_l1i = LONG_BIT - 1, 100255040Sgibbs .last_processed_l2i = LONG_BIT - 1 101255040Sgibbs}; 102255040Sgibbs 103255040SgibbsDPCPU_DECLARE(struct vcpu_info *, vcpu_info); 104255040Sgibbs 105255040Sgibbs#define is_valid_evtchn(x) ((x) != 0) 106255040Sgibbs 107255040Sgibbsstruct xenisrc { 108255040Sgibbs struct intsrc xi_intsrc; 109255040Sgibbs enum evtchn_type xi_type; 110255040Sgibbs int xi_cpu; /* VCPU for delivery. */ 111255040Sgibbs int xi_vector; /* Global isrc vector number. */ 112255040Sgibbs evtchn_port_t xi_port; 113255040Sgibbs int xi_pirq; 114255040Sgibbs int xi_virq; 115255040Sgibbs u_int xi_close:1; /* close on unbind? */ 116255040Sgibbs u_int xi_needs_eoi:1; 117255040Sgibbs u_int xi_shared:1; /* Shared with other domains. */ 118255040Sgibbs}; 119255040Sgibbs 120255040Sgibbs#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) 121255040Sgibbs 122255040Sgibbsstatic void xen_intr_suspend(struct pic *); 123255726Sgibbsstatic void xen_intr_resume(struct pic *, bool suspend_cancelled); 124255040Sgibbsstatic void xen_intr_enable_source(struct intsrc *isrc); 125255040Sgibbsstatic void xen_intr_disable_source(struct intsrc *isrc, int eoi); 126255040Sgibbsstatic void xen_intr_eoi_source(struct intsrc *isrc); 127255040Sgibbsstatic void xen_intr_enable_intr(struct intsrc *isrc); 128255040Sgibbsstatic void xen_intr_disable_intr(struct intsrc *isrc); 129255040Sgibbsstatic int xen_intr_vector(struct intsrc *isrc); 130255040Sgibbsstatic int xen_intr_source_pending(struct intsrc *isrc); 131255040Sgibbsstatic int xen_intr_config_intr(struct intsrc *isrc, 132255040Sgibbs enum intr_trigger trig, enum intr_polarity pol); 133255040Sgibbsstatic int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id); 134255040Sgibbs 135255040Sgibbsstatic void xen_intr_pirq_enable_source(struct intsrc *isrc); 136255040Sgibbsstatic void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi); 137255040Sgibbsstatic void xen_intr_pirq_eoi_source(struct intsrc *isrc); 138255040Sgibbsstatic void xen_intr_pirq_enable_intr(struct intsrc *isrc); 139255040Sgibbs 140255040Sgibbs/** 141255040Sgibbs * PIC interface for all event channel port types except physical IRQs. 142255040Sgibbs */ 143255040Sgibbsstruct pic xen_intr_pic = { 144255040Sgibbs .pic_enable_source = xen_intr_enable_source, 145255040Sgibbs .pic_disable_source = xen_intr_disable_source, 146255040Sgibbs .pic_eoi_source = xen_intr_eoi_source, 147255040Sgibbs .pic_enable_intr = xen_intr_enable_intr, 148255040Sgibbs .pic_disable_intr = xen_intr_disable_intr, 149255040Sgibbs .pic_vector = xen_intr_vector, 150255040Sgibbs .pic_source_pending = xen_intr_source_pending, 151255040Sgibbs .pic_suspend = xen_intr_suspend, 152255040Sgibbs .pic_resume = xen_intr_resume, 153255040Sgibbs .pic_config_intr = xen_intr_config_intr, 154255040Sgibbs .pic_assign_cpu = xen_intr_assign_cpu 155255040Sgibbs}; 156255040Sgibbs 157255040Sgibbs/** 158255040Sgibbs * PIC interface for all event channel representing 159255040Sgibbs * physical interrupt sources. 160255040Sgibbs */ 161255040Sgibbsstruct pic xen_intr_pirq_pic = { 162255040Sgibbs .pic_enable_source = xen_intr_pirq_enable_source, 163255040Sgibbs .pic_disable_source = xen_intr_pirq_disable_source, 164255040Sgibbs .pic_eoi_source = xen_intr_pirq_eoi_source, 165255040Sgibbs .pic_enable_intr = xen_intr_pirq_enable_intr, 166255040Sgibbs .pic_disable_intr = xen_intr_disable_intr, 167255040Sgibbs .pic_vector = xen_intr_vector, 168255040Sgibbs .pic_source_pending = xen_intr_source_pending, 169255040Sgibbs .pic_suspend = xen_intr_suspend, 170255040Sgibbs .pic_resume = xen_intr_resume, 171255040Sgibbs .pic_config_intr = xen_intr_config_intr, 172255040Sgibbs .pic_assign_cpu = xen_intr_assign_cpu 173255040Sgibbs}; 174255040Sgibbs 175255040Sgibbsstatic struct mtx xen_intr_isrc_lock; 176255040Sgibbsstatic int xen_intr_isrc_count; 177255040Sgibbsstatic struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS]; 178255040Sgibbs 179255040Sgibbs/*------------------------- Private Functions --------------------------------*/ 180255040Sgibbs/** 181255040Sgibbs * Disable signal delivery for an event channel port on the 182255040Sgibbs * specified CPU. 183255040Sgibbs * 184255040Sgibbs * \param port The event channel port to mask. 185255040Sgibbs * 186255040Sgibbs * This API is used to manage the port<=>CPU binding of event 187255040Sgibbs * channel handlers. 188255040Sgibbs * 189255040Sgibbs * \note This operation does not preclude reception of an event 190255040Sgibbs * for this event channel on another CPU. To mask the 191255040Sgibbs * event channel globally, use evtchn_mask(). 192255040Sgibbs */ 193255040Sgibbsstatic inline void 194255040Sgibbsevtchn_cpu_mask_port(u_int cpu, evtchn_port_t port) 195255040Sgibbs{ 196255040Sgibbs struct xen_intr_pcpu_data *pcpu; 197255040Sgibbs 198255040Sgibbs pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); 199255040Sgibbs clear_bit(port, pcpu->evtchn_enabled); 200255040Sgibbs} 201255040Sgibbs 202255040Sgibbs/** 203255040Sgibbs * Enable signal delivery for an event channel port on the 204255040Sgibbs * specified CPU. 205255040Sgibbs * 206255040Sgibbs * \param port The event channel port to unmask. 207255040Sgibbs * 208255040Sgibbs * This API is used to manage the port<=>CPU binding of event 209255040Sgibbs * channel handlers. 210255040Sgibbs * 211255040Sgibbs * \note This operation does not guarantee that event delivery 212255040Sgibbs * is enabled for this event channel port. The port must 213255040Sgibbs * also be globally enabled. See evtchn_unmask(). 214255040Sgibbs */ 215255040Sgibbsstatic inline void 216255040Sgibbsevtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port) 217255040Sgibbs{ 218255040Sgibbs struct xen_intr_pcpu_data *pcpu; 219255040Sgibbs 220255040Sgibbs pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); 221255040Sgibbs set_bit(port, pcpu->evtchn_enabled); 222255040Sgibbs} 223255040Sgibbs 224255040Sgibbs/** 225255040Sgibbs * Allocate and register a per-cpu Xen upcall interrupt counter. 226255040Sgibbs * 227255040Sgibbs * \param cpu The cpu for which to register this interrupt count. 228255040Sgibbs */ 229255040Sgibbsstatic void 230255040Sgibbsxen_intr_intrcnt_add(u_int cpu) 231255040Sgibbs{ 232255040Sgibbs char buf[MAXCOMLEN + 1]; 233255040Sgibbs struct xen_intr_pcpu_data *pcpu; 234255040Sgibbs 235255040Sgibbs pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); 236255040Sgibbs if (pcpu->evtchn_intrcnt != NULL) 237255040Sgibbs return; 238255040Sgibbs 239255040Sgibbs snprintf(buf, sizeof(buf), "cpu%d:xen", cpu); 240255040Sgibbs intrcnt_add(buf, &pcpu->evtchn_intrcnt); 241255040Sgibbs} 242255040Sgibbs 243255040Sgibbs/** 244255040Sgibbs * Search for an already allocated but currently unused Xen interrupt 245255040Sgibbs * source object. 246255040Sgibbs * 247255040Sgibbs * \param type Restrict the search to interrupt sources of the given 248255040Sgibbs * type. 249255040Sgibbs * 250255040Sgibbs * \return A pointer to a free Xen interrupt source object or NULL. 251255040Sgibbs */ 252255040Sgibbsstatic struct xenisrc * 253255040Sgibbsxen_intr_find_unused_isrc(enum evtchn_type type) 254255040Sgibbs{ 255255040Sgibbs int isrc_idx; 256255040Sgibbs 257255040Sgibbs KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held")); 258255040Sgibbs 259255040Sgibbs for (isrc_idx = 0; isrc_idx < xen_intr_isrc_count; isrc_idx ++) { 260255040Sgibbs struct xenisrc *isrc; 261255040Sgibbs u_int vector; 262255040Sgibbs 263255040Sgibbs vector = FIRST_EVTCHN_INT + isrc_idx; 264255040Sgibbs isrc = (struct xenisrc *)intr_lookup_source(vector); 265255040Sgibbs if (isrc != NULL 266255040Sgibbs && isrc->xi_type == EVTCHN_TYPE_UNBOUND) { 267255040Sgibbs KASSERT(isrc->xi_intsrc.is_handlers == 0, 268255040Sgibbs ("Free evtchn still has handlers")); 269255040Sgibbs isrc->xi_type = type; 270255040Sgibbs return (isrc); 271255040Sgibbs } 272255040Sgibbs } 273255040Sgibbs return (NULL); 274255040Sgibbs} 275255040Sgibbs 276255040Sgibbs/** 277255040Sgibbs * Allocate a Xen interrupt source object. 278255040Sgibbs * 279255040Sgibbs * \param type The type of interrupt source to create. 280255040Sgibbs * 281255040Sgibbs * \return A pointer to a newly allocated Xen interrupt source 282255040Sgibbs * object or NULL. 283255040Sgibbs */ 284255040Sgibbsstatic struct xenisrc * 285255040Sgibbsxen_intr_alloc_isrc(enum evtchn_type type) 286255040Sgibbs{ 287255040Sgibbs static int warned; 288255040Sgibbs struct xenisrc *isrc; 289255040Sgibbs int vector; 290255040Sgibbs 291255040Sgibbs KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held")); 292255040Sgibbs 293255040Sgibbs if (xen_intr_isrc_count > NR_EVENT_CHANNELS) { 294255040Sgibbs if (!warned) { 295255040Sgibbs warned = 1; 296255040Sgibbs printf("xen_intr_alloc: Event channels exhausted.\n"); 297255040Sgibbs } 298255040Sgibbs return (NULL); 299255040Sgibbs } 300255040Sgibbs vector = FIRST_EVTCHN_INT + xen_intr_isrc_count; 301255040Sgibbs xen_intr_isrc_count++; 302255040Sgibbs 303255040Sgibbs mtx_unlock(&xen_intr_isrc_lock); 304255040Sgibbs isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO); 305255040Sgibbs isrc->xi_intsrc.is_pic = &xen_intr_pic; 306255040Sgibbs isrc->xi_vector = vector; 307255040Sgibbs isrc->xi_type = type; 308255040Sgibbs intr_register_source(&isrc->xi_intsrc); 309255040Sgibbs mtx_lock(&xen_intr_isrc_lock); 310255040Sgibbs 311255040Sgibbs return (isrc); 312255040Sgibbs} 313255040Sgibbs 314255040Sgibbs/** 315255040Sgibbs * Attempt to free an active Xen interrupt source object. 316255040Sgibbs * 317255040Sgibbs * \param isrc The interrupt source object to release. 318255040Sgibbs * 319255040Sgibbs * \returns EBUSY if the source is still in use, otherwise 0. 320255040Sgibbs */ 321255040Sgibbsstatic int 322255040Sgibbsxen_intr_release_isrc(struct xenisrc *isrc) 323255040Sgibbs{ 324255040Sgibbs 325255040Sgibbs mtx_lock(&xen_intr_isrc_lock); 326255040Sgibbs if (isrc->xi_intsrc.is_handlers != 0) { 327255040Sgibbs mtx_unlock(&xen_intr_isrc_lock); 328255040Sgibbs return (EBUSY); 329255040Sgibbs } 330255040Sgibbs evtchn_mask_port(isrc->xi_port); 331255040Sgibbs evtchn_clear_port(isrc->xi_port); 332255040Sgibbs 333255040Sgibbs /* Rebind port to CPU 0. */ 334255040Sgibbs evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 335255040Sgibbs evtchn_cpu_unmask_port(0, isrc->xi_port); 336255040Sgibbs 337255726Sgibbs if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) { 338255040Sgibbs struct evtchn_close close = { .port = isrc->xi_port }; 339255040Sgibbs if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 340255040Sgibbs panic("EVTCHNOP_close failed"); 341255040Sgibbs } 342255040Sgibbs 343255040Sgibbs xen_intr_port_to_isrc[isrc->xi_port] = NULL; 344255040Sgibbs isrc->xi_cpu = 0; 345255040Sgibbs isrc->xi_type = EVTCHN_TYPE_UNBOUND; 346255040Sgibbs isrc->xi_port = 0; 347255040Sgibbs mtx_unlock(&xen_intr_isrc_lock); 348255040Sgibbs return (0); 349255040Sgibbs} 350255040Sgibbs 351255040Sgibbs/** 352255040Sgibbs * Associate an interrupt handler with an already allocated local Xen 353255040Sgibbs * event channel port. 354255040Sgibbs * 355255040Sgibbs * \param isrcp The returned Xen interrupt object associated with 356255040Sgibbs * the specified local port. 357255040Sgibbs * \param local_port The event channel to bind. 358255040Sgibbs * \param type The event channel type of local_port. 359255040Sgibbs * \param intr_owner The device making this bind request. 360255040Sgibbs * \param filter An interrupt filter handler. Specify NULL 361255040Sgibbs * to always dispatch to the ithread handler. 362255040Sgibbs * \param handler An interrupt ithread handler. Optional (can 363255040Sgibbs * specify NULL) if all necessary event actions 364255040Sgibbs * are performed by filter. 365255040Sgibbs * \param arg Argument to present to both filter and handler. 366255040Sgibbs * \param irqflags Interrupt handler flags. See sys/bus.h. 367255040Sgibbs * \param handlep Pointer to an opaque handle used to manage this 368255040Sgibbs * registration. 369255040Sgibbs * 370255040Sgibbs * \returns 0 on success, otherwise an errno. 371255040Sgibbs */ 372255040Sgibbsstatic int 373255040Sgibbsxen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port, 374255040Sgibbs enum evtchn_type type, device_t intr_owner, driver_filter_t filter, 375255040Sgibbs driver_intr_t handler, void *arg, enum intr_type flags, 376255040Sgibbs xen_intr_handle_t *port_handlep) 377255040Sgibbs{ 378255040Sgibbs struct xenisrc *isrc; 379255040Sgibbs int error; 380255040Sgibbs 381255040Sgibbs *isrcp = NULL; 382255040Sgibbs if (port_handlep == NULL) { 383255040Sgibbs device_printf(intr_owner, 384255040Sgibbs "xen_intr_bind_isrc: Bad event handle\n"); 385255040Sgibbs return (EINVAL); 386255040Sgibbs } 387255040Sgibbs 388255040Sgibbs mtx_lock(&xen_intr_isrc_lock); 389255040Sgibbs isrc = xen_intr_find_unused_isrc(type); 390255040Sgibbs if (isrc == NULL) { 391255040Sgibbs isrc = xen_intr_alloc_isrc(type); 392255040Sgibbs if (isrc == NULL) { 393255040Sgibbs mtx_unlock(&xen_intr_isrc_lock); 394255040Sgibbs return (ENOSPC); 395255040Sgibbs } 396255040Sgibbs } 397255040Sgibbs isrc->xi_port = local_port; 398255040Sgibbs xen_intr_port_to_isrc[local_port] = isrc; 399255040Sgibbs mtx_unlock(&xen_intr_isrc_lock); 400255040Sgibbs 401255040Sgibbs error = intr_add_handler(device_get_nameunit(intr_owner), 402255040Sgibbs isrc->xi_vector, filter, handler, arg, 403255040Sgibbs flags|INTR_EXCL, port_handlep); 404255040Sgibbs if (error != 0) { 405255040Sgibbs device_printf(intr_owner, 406255040Sgibbs "xen_intr_bind_irq: intr_add_handler failed\n"); 407255040Sgibbs xen_intr_release_isrc(isrc); 408255040Sgibbs return (error); 409255040Sgibbs } 410255040Sgibbs *isrcp = isrc; 411255726Sgibbs evtchn_unmask_port(local_port); 412255040Sgibbs return (0); 413255040Sgibbs} 414255040Sgibbs 415255040Sgibbs/** 416255040Sgibbs * Lookup a Xen interrupt source object given an interrupt binding handle. 417255040Sgibbs * 418255040Sgibbs * \param handle A handle initialized by a previous call to 419255040Sgibbs * xen_intr_bind_isrc(). 420255040Sgibbs * 421255040Sgibbs * \returns A pointer to the Xen interrupt source object associated 422255040Sgibbs * with the given interrupt handle. NULL if no association 423255040Sgibbs * currently exists. 424255040Sgibbs */ 425255040Sgibbsstatic struct xenisrc * 426255040Sgibbsxen_intr_isrc(xen_intr_handle_t handle) 427255040Sgibbs{ 428255040Sgibbs struct intr_handler *ih; 429255040Sgibbs 430255040Sgibbs ih = handle; 431255040Sgibbs if (ih == NULL || ih->ih_event == NULL) 432255040Sgibbs return (NULL); 433255040Sgibbs 434255040Sgibbs return (ih->ih_event->ie_source); 435255040Sgibbs} 436255040Sgibbs 437255040Sgibbs/** 438255040Sgibbs * Determine the event channel ports at the given section of the 439255040Sgibbs * event port bitmap which have pending events for the given cpu. 440255040Sgibbs * 441255040Sgibbs * \param pcpu The Xen interrupt pcpu data for the cpu being querried. 442255040Sgibbs * \param sh The Xen shared info area. 443255040Sgibbs * \param idx The index of the section of the event channel bitmap to 444255040Sgibbs * inspect. 445255040Sgibbs * 446255040Sgibbs * \returns A u_long with bits set for every event channel with pending 447255040Sgibbs * events. 448255040Sgibbs */ 449255040Sgibbsstatic inline u_long 450255040Sgibbsxen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh, 451255040Sgibbs u_int idx) 452255040Sgibbs{ 453255040Sgibbs return (sh->evtchn_pending[idx] 454255040Sgibbs & ~sh->evtchn_mask[idx] 455255040Sgibbs & pcpu->evtchn_enabled[idx]); 456255040Sgibbs} 457255040Sgibbs 458255040Sgibbs/** 459255040Sgibbs * Interrupt handler for processing all Xen event channel events. 460255040Sgibbs * 461255040Sgibbs * \param trap_frame The trap frame context for the current interrupt. 462255040Sgibbs */ 463255040Sgibbsvoid 464255040Sgibbsxen_intr_handle_upcall(struct trapframe *trap_frame) 465255040Sgibbs{ 466255040Sgibbs u_int l1i, l2i, port, cpu; 467255040Sgibbs u_long masked_l1, masked_l2; 468255040Sgibbs struct xenisrc *isrc; 469255040Sgibbs shared_info_t *s; 470255040Sgibbs vcpu_info_t *v; 471255040Sgibbs struct xen_intr_pcpu_data *pc; 472255040Sgibbs u_long l1, l2; 473255040Sgibbs 474255040Sgibbs /* 475255040Sgibbs * Disable preemption in order to always check and fire events 476255040Sgibbs * on the right vCPU 477255040Sgibbs */ 478255040Sgibbs critical_enter(); 479255040Sgibbs 480255040Sgibbs cpu = PCPU_GET(cpuid); 481255040Sgibbs pc = DPCPU_PTR(xen_intr_pcpu); 482255040Sgibbs s = HYPERVISOR_shared_info; 483255040Sgibbs v = DPCPU_GET(vcpu_info); 484255040Sgibbs 485255040Sgibbs if (xen_hvm_domain() && !xen_vector_callback_enabled) { 486255040Sgibbs KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU")); 487255040Sgibbs } 488255040Sgibbs 489255040Sgibbs v->evtchn_upcall_pending = 0; 490255040Sgibbs 491255040Sgibbs#if 0 492255040Sgibbs#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */ 493255040Sgibbs /* Clear master flag /before/ clearing selector flag. */ 494255040Sgibbs wmb(); 495255040Sgibbs#endif 496255040Sgibbs#endif 497255040Sgibbs 498255040Sgibbs l1 = atomic_readandclear_long(&v->evtchn_pending_sel); 499255040Sgibbs 500255040Sgibbs l1i = pc->last_processed_l1i; 501255040Sgibbs l2i = pc->last_processed_l2i; 502255040Sgibbs (*pc->evtchn_intrcnt)++; 503255040Sgibbs 504255040Sgibbs while (l1 != 0) { 505255040Sgibbs 506255040Sgibbs l1i = (l1i + 1) % LONG_BIT; 507255040Sgibbs masked_l1 = l1 & ((~0UL) << l1i); 508255040Sgibbs 509255040Sgibbs if (masked_l1 == 0) { 510255040Sgibbs /* 511255040Sgibbs * if we masked out all events, wrap around 512255040Sgibbs * to the beginning. 513255040Sgibbs */ 514255040Sgibbs l1i = LONG_BIT - 1; 515255040Sgibbs l2i = LONG_BIT - 1; 516255040Sgibbs continue; 517255040Sgibbs } 518255040Sgibbs l1i = ffsl(masked_l1) - 1; 519255040Sgibbs 520255040Sgibbs do { 521255040Sgibbs l2 = xen_intr_active_ports(pc, s, l1i); 522255040Sgibbs 523255040Sgibbs l2i = (l2i + 1) % LONG_BIT; 524255040Sgibbs masked_l2 = l2 & ((~0UL) << l2i); 525255040Sgibbs 526255040Sgibbs if (masked_l2 == 0) { 527255040Sgibbs /* if we masked out all events, move on */ 528255040Sgibbs l2i = LONG_BIT - 1; 529255040Sgibbs break; 530255040Sgibbs } 531255040Sgibbs l2i = ffsl(masked_l2) - 1; 532255040Sgibbs 533255040Sgibbs /* process port */ 534255040Sgibbs port = (l1i * LONG_BIT) + l2i; 535255040Sgibbs synch_clear_bit(port, &s->evtchn_pending[0]); 536255040Sgibbs 537255040Sgibbs isrc = xen_intr_port_to_isrc[port]; 538255040Sgibbs if (__predict_false(isrc == NULL)) 539255040Sgibbs continue; 540255040Sgibbs 541255040Sgibbs /* Make sure we are firing on the right vCPU */ 542255040Sgibbs KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), 543255040Sgibbs ("Received unexpected event on vCPU#%d, event bound to vCPU#%d", 544255040Sgibbs PCPU_GET(cpuid), isrc->xi_cpu)); 545255040Sgibbs 546255040Sgibbs intr_execute_handlers(&isrc->xi_intsrc, trap_frame); 547255040Sgibbs 548255040Sgibbs /* 549255040Sgibbs * If this is the final port processed, 550255040Sgibbs * we'll pick up here+1 next time. 551255040Sgibbs */ 552255040Sgibbs pc->last_processed_l1i = l1i; 553255040Sgibbs pc->last_processed_l2i = l2i; 554255040Sgibbs 555255040Sgibbs } while (l2i != LONG_BIT - 1); 556255040Sgibbs 557255040Sgibbs l2 = xen_intr_active_ports(pc, s, l1i); 558255040Sgibbs if (l2 == 0) { 559255040Sgibbs /* 560255040Sgibbs * We handled all ports, so we can clear the 561255040Sgibbs * selector bit. 562255040Sgibbs */ 563255040Sgibbs l1 &= ~(1UL << l1i); 564255040Sgibbs } 565255040Sgibbs } 566255040Sgibbs critical_exit(); 567255040Sgibbs} 568255040Sgibbs 569255040Sgibbsstatic int 570255040Sgibbsxen_intr_init(void *dummy __unused) 571255040Sgibbs{ 572255040Sgibbs struct xen_intr_pcpu_data *pcpu; 573255040Sgibbs int i; 574255040Sgibbs 575255726Sgibbs if (!xen_domain()) 576255726Sgibbs return (0); 577255726Sgibbs 578255040Sgibbs mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); 579255040Sgibbs 580255040Sgibbs /* 581255040Sgibbs * Register interrupt count manually as we aren't 582255040Sgibbs * guaranteed to see a call to xen_intr_assign_cpu() 583255040Sgibbs * before our first interrupt. Also set the per-cpu 584255040Sgibbs * mask of CPU#0 to enable all, since by default 585255040Sgibbs * all event channels are bound to CPU#0. 586255040Sgibbs */ 587255040Sgibbs CPU_FOREACH(i) { 588255040Sgibbs pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); 589255040Sgibbs memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, 590255040Sgibbs sizeof(pcpu->evtchn_enabled)); 591255040Sgibbs xen_intr_intrcnt_add(i); 592255040Sgibbs } 593255040Sgibbs 594255040Sgibbs intr_register_pic(&xen_intr_pic); 595255040Sgibbs 596255040Sgibbs return (0); 597255040Sgibbs} 598255040SgibbsSYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intr_init, NULL); 599255040Sgibbs 600255040Sgibbs/*--------------------------- Common PIC Functions ---------------------------*/ 601255040Sgibbs/** 602255040Sgibbs * Prepare this PIC for system suspension. 603255040Sgibbs */ 604255040Sgibbsstatic void 605255040Sgibbsxen_intr_suspend(struct pic *unused) 606255040Sgibbs{ 607255040Sgibbs} 608255040Sgibbs 609255726Sgibbsstatic void 610255726Sgibbsxen_rebind_ipi(struct xenisrc *isrc) 611255726Sgibbs{ 612255726Sgibbs#ifdef SMP 613255726Sgibbs int cpu = isrc->xi_cpu; 614256073Sgibbs int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; 615255726Sgibbs int error; 616256073Sgibbs struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; 617255726Sgibbs 618255726Sgibbs error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 619255726Sgibbs &bind_ipi); 620255726Sgibbs if (error != 0) 621255726Sgibbs panic("unable to rebind xen IPI: %d", error); 622255726Sgibbs 623255726Sgibbs isrc->xi_port = bind_ipi.port; 624255726Sgibbs isrc->xi_cpu = 0; 625255726Sgibbs xen_intr_port_to_isrc[bind_ipi.port] = isrc; 626255726Sgibbs 627255726Sgibbs error = xen_intr_assign_cpu(&isrc->xi_intsrc, 628255726Sgibbs cpu_apic_ids[cpu]); 629255726Sgibbs if (error) 630255726Sgibbs panic("unable to bind xen IPI to CPU#%d: %d", 631255726Sgibbs cpu, error); 632255726Sgibbs 633255726Sgibbs evtchn_unmask_port(bind_ipi.port); 634255726Sgibbs#else 635255726Sgibbs panic("Resume IPI event channel on UP"); 636255726Sgibbs#endif 637255726Sgibbs} 638255726Sgibbs 639255726Sgibbsstatic void 640255726Sgibbsxen_rebind_virq(struct xenisrc *isrc) 641255726Sgibbs{ 642255726Sgibbs int cpu = isrc->xi_cpu; 643256073Sgibbs int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; 644255726Sgibbs int error; 645255726Sgibbs struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq, 646256073Sgibbs .vcpu = vcpu_id }; 647255726Sgibbs 648255726Sgibbs error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 649255726Sgibbs &bind_virq); 650255726Sgibbs if (error != 0) 651255726Sgibbs panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error); 652255726Sgibbs 653255726Sgibbs isrc->xi_port = bind_virq.port; 654255726Sgibbs isrc->xi_cpu = 0; 655255726Sgibbs xen_intr_port_to_isrc[bind_virq.port] = isrc; 656255726Sgibbs 657255726Sgibbs#ifdef SMP 658255726Sgibbs error = xen_intr_assign_cpu(&isrc->xi_intsrc, 659255726Sgibbs cpu_apic_ids[cpu]); 660255726Sgibbs if (error) 661255726Sgibbs panic("unable to bind xen VIRQ#%d to CPU#%d: %d", 662255726Sgibbs isrc->xi_virq, cpu, error); 663255726Sgibbs#endif 664255726Sgibbs 665255726Sgibbs evtchn_unmask_port(bind_virq.port); 666255726Sgibbs} 667255726Sgibbs 668255040Sgibbs/** 669255040Sgibbs * Return this PIC to service after being suspended. 670255040Sgibbs */ 671255040Sgibbsstatic void 672255726Sgibbsxen_intr_resume(struct pic *unused, bool suspend_cancelled) 673255040Sgibbs{ 674255726Sgibbs shared_info_t *s = HYPERVISOR_shared_info; 675255726Sgibbs struct xenisrc *isrc; 676255726Sgibbs u_int isrc_idx; 677255726Sgibbs int i; 678255040Sgibbs 679255726Sgibbs if (suspend_cancelled) 680255726Sgibbs return; 681255726Sgibbs 682255726Sgibbs /* Reset the per-CPU masks */ 683255726Sgibbs CPU_FOREACH(i) { 684255726Sgibbs struct xen_intr_pcpu_data *pcpu; 685255726Sgibbs 686255726Sgibbs pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); 687255726Sgibbs memset(pcpu->evtchn_enabled, 688255726Sgibbs i == 0 ? ~0 : 0, sizeof(pcpu->evtchn_enabled)); 689255726Sgibbs } 690255726Sgibbs 691255726Sgibbs /* Mask all event channels. */ 692255726Sgibbs for (i = 0; i < nitems(s->evtchn_mask); i++) 693255726Sgibbs atomic_store_rel_long(&s->evtchn_mask[i], ~0); 694255726Sgibbs 695255726Sgibbs /* Remove port -> isrc mappings */ 696255726Sgibbs memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc)); 697255726Sgibbs 698255726Sgibbs /* Free unused isrcs and rebind VIRQs and IPIs */ 699255726Sgibbs for (isrc_idx = 0; isrc_idx < xen_intr_isrc_count; isrc_idx++) { 700255726Sgibbs u_int vector; 701255726Sgibbs 702255726Sgibbs vector = FIRST_EVTCHN_INT + isrc_idx; 703255726Sgibbs isrc = (struct xenisrc *)intr_lookup_source(vector); 704255726Sgibbs if (isrc != NULL) { 705255726Sgibbs isrc->xi_port = 0; 706255726Sgibbs switch (isrc->xi_type) { 707255726Sgibbs case EVTCHN_TYPE_IPI: 708255726Sgibbs xen_rebind_ipi(isrc); 709255726Sgibbs break; 710255726Sgibbs case EVTCHN_TYPE_VIRQ: 711255726Sgibbs xen_rebind_virq(isrc); 712255726Sgibbs break; 713255726Sgibbs default: 714255726Sgibbs isrc->xi_cpu = 0; 715255726Sgibbs break; 716255726Sgibbs } 717255726Sgibbs } 718255726Sgibbs } 719255040Sgibbs} 720255040Sgibbs 721255040Sgibbs/** 722255040Sgibbs * Disable a Xen interrupt source. 723255040Sgibbs * 724255040Sgibbs * \param isrc The interrupt source to disable. 725255040Sgibbs */ 726255040Sgibbsstatic void 727255040Sgibbsxen_intr_disable_intr(struct intsrc *base_isrc) 728255040Sgibbs{ 729255040Sgibbs struct xenisrc *isrc = (struct xenisrc *)base_isrc; 730255040Sgibbs 731255040Sgibbs evtchn_mask_port(isrc->xi_port); 732255040Sgibbs} 733255040Sgibbs 734255040Sgibbs/** 735255040Sgibbs * Determine the global interrupt vector number for 736255040Sgibbs * a Xen interrupt source. 737255040Sgibbs * 738255040Sgibbs * \param isrc The interrupt source to query. 739255040Sgibbs * 740255040Sgibbs * \return The vector number corresponding to the given interrupt source. 741255040Sgibbs */ 742255040Sgibbsstatic int 743255040Sgibbsxen_intr_vector(struct intsrc *base_isrc) 744255040Sgibbs{ 745255040Sgibbs struct xenisrc *isrc = (struct xenisrc *)base_isrc; 746255040Sgibbs 747255040Sgibbs return (isrc->xi_vector); 748255040Sgibbs} 749255040Sgibbs 750255040Sgibbs/** 751255040Sgibbs * Determine whether or not interrupt events are pending on the 752255040Sgibbs * the given interrupt source. 753255040Sgibbs * 754255040Sgibbs * \param isrc The interrupt source to query. 755255040Sgibbs * 756255040Sgibbs * \returns 0 if no events are pending, otherwise non-zero. 757255040Sgibbs */ 758255040Sgibbsstatic int 759255040Sgibbsxen_intr_source_pending(struct intsrc *isrc) 760255040Sgibbs{ 761255040Sgibbs /* 762255040Sgibbs * EventChannels are edge triggered and never masked. 763255040Sgibbs * There can be no pending events. 764255040Sgibbs */ 765255040Sgibbs return (0); 766255040Sgibbs} 767255040Sgibbs 768255040Sgibbs/** 769255040Sgibbs * Perform configuration of an interrupt source. 770255040Sgibbs * 771255040Sgibbs * \param isrc The interrupt source to configure. 772255040Sgibbs * \param trig Edge or level. 773255040Sgibbs * \param pol Active high or low. 774255040Sgibbs * 775255040Sgibbs * \returns 0 if no events are pending, otherwise non-zero. 776255040Sgibbs */ 777255040Sgibbsstatic int 778255040Sgibbsxen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig, 779255040Sgibbs enum intr_polarity pol) 780255040Sgibbs{ 781255040Sgibbs /* Configuration is only possible via the evtchn apis. */ 782255040Sgibbs return (ENODEV); 783255040Sgibbs} 784255040Sgibbs 785255040Sgibbs/** 786255040Sgibbs * Configure CPU affinity for interrupt source event delivery. 787255040Sgibbs * 788255040Sgibbs * \param isrc The interrupt source to configure. 789255040Sgibbs * \param apic_id The apic id of the CPU for handling future events. 790255040Sgibbs * 791255040Sgibbs * \returns 0 if successful, otherwise an errno. 792255040Sgibbs */ 793255040Sgibbsstatic int 794255040Sgibbsxen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id) 795255040Sgibbs{ 796255726Sgibbs#ifdef SMP 797255040Sgibbs struct evtchn_bind_vcpu bind_vcpu; 798255040Sgibbs struct xenisrc *isrc; 799256073Sgibbs u_int to_cpu, vcpu_id; 800255040Sgibbs int error; 801255040Sgibbs 802255040Sgibbs#ifdef XENHVM 803255040Sgibbs if (xen_vector_callback_enabled == 0) 804255040Sgibbs return (EOPNOTSUPP); 805255040Sgibbs#endif 806255040Sgibbs 807255040Sgibbs to_cpu = apic_cpuid(apic_id); 808256073Sgibbs vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id; 809255040Sgibbs xen_intr_intrcnt_add(to_cpu); 810255040Sgibbs 811255040Sgibbs mtx_lock(&xen_intr_isrc_lock); 812255040Sgibbs isrc = (struct xenisrc *)base_isrc; 813255040Sgibbs if (!is_valid_evtchn(isrc->xi_port)) { 814255040Sgibbs mtx_unlock(&xen_intr_isrc_lock); 815255040Sgibbs return (EINVAL); 816255040Sgibbs } 817255040Sgibbs 818255040Sgibbs if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) || 819255040Sgibbs (isrc->xi_type == EVTCHN_TYPE_IPI)) { 820255040Sgibbs /* 821255040Sgibbs * Virtual IRQs are associated with a cpu by 822255040Sgibbs * the Hypervisor at evtchn_bind_virq time, so 823255040Sgibbs * all we need to do is update the per-CPU masks. 824255040Sgibbs */ 825255040Sgibbs evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 826255040Sgibbs isrc->xi_cpu = to_cpu; 827255040Sgibbs evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); 828255040Sgibbs mtx_unlock(&xen_intr_isrc_lock); 829255040Sgibbs return (0); 830255040Sgibbs } 831255040Sgibbs 832255040Sgibbs bind_vcpu.port = isrc->xi_port; 833256073Sgibbs bind_vcpu.vcpu = vcpu_id; 834255040Sgibbs 835255040Sgibbs /* 836255040Sgibbs * Allow interrupts to be fielded on the new VCPU before 837255040Sgibbs * we ask the hypervisor to deliver them there. 838255040Sgibbs */ 839255040Sgibbs evtchn_cpu_unmask_port(to_cpu, isrc->xi_port); 840255040Sgibbs error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu); 841255040Sgibbs if (isrc->xi_cpu != to_cpu) { 842255040Sgibbs if (error == 0) { 843255040Sgibbs /* Commit to new binding by removing the old one. */ 844255040Sgibbs evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 845255040Sgibbs isrc->xi_cpu = to_cpu; 846255040Sgibbs } else { 847255040Sgibbs /* Roll-back to previous binding. */ 848255040Sgibbs evtchn_cpu_mask_port(to_cpu, isrc->xi_port); 849255040Sgibbs } 850255040Sgibbs } 851255040Sgibbs mtx_unlock(&xen_intr_isrc_lock); 852255040Sgibbs return (0); 853255726Sgibbs#else 854255726Sgibbs return (EOPNOTSUPP); 855255726Sgibbs#endif 856255040Sgibbs} 857255040Sgibbs 858255040Sgibbs/*------------------- Virtual Interrupt Source PIC Functions -----------------*/ 859255040Sgibbs/* 860255040Sgibbs * Mask a level triggered interrupt source. 861255040Sgibbs * 862255040Sgibbs * \param isrc The interrupt source to mask (if necessary). 863255040Sgibbs * \param eoi If non-zero, perform any necessary end-of-interrupt 864255040Sgibbs * acknowledgements. 865255040Sgibbs */ 866255040Sgibbsstatic void 867255040Sgibbsxen_intr_disable_source(struct intsrc *isrc, int eoi) 868255040Sgibbs{ 869255040Sgibbs} 870255040Sgibbs 871255040Sgibbs/* 872255040Sgibbs * Unmask a level triggered interrupt source. 873255040Sgibbs * 874255040Sgibbs * \param isrc The interrupt source to unmask (if necessary). 875255040Sgibbs */ 876255040Sgibbsstatic void 877255040Sgibbsxen_intr_enable_source(struct intsrc *isrc) 878255040Sgibbs{ 879255040Sgibbs} 880255040Sgibbs 881255040Sgibbs/* 882255040Sgibbs * Perform any necessary end-of-interrupt acknowledgements. 883255040Sgibbs * 884255040Sgibbs * \param isrc The interrupt source to EOI. 885255040Sgibbs */ 886255040Sgibbsstatic void 887255040Sgibbsxen_intr_eoi_source(struct intsrc *isrc) 888255040Sgibbs{ 889255040Sgibbs} 890255040Sgibbs 891255040Sgibbs/* 892255040Sgibbs * Enable and unmask the interrupt source. 893255040Sgibbs * 894255040Sgibbs * \param isrc The interrupt source to enable. 895255040Sgibbs */ 896255040Sgibbsstatic void 897255040Sgibbsxen_intr_enable_intr(struct intsrc *base_isrc) 898255040Sgibbs{ 899255040Sgibbs struct xenisrc *isrc = (struct xenisrc *)base_isrc; 900255040Sgibbs 901255040Sgibbs evtchn_unmask_port(isrc->xi_port); 902255040Sgibbs} 903255040Sgibbs 904255040Sgibbs/*------------------ Physical Interrupt Source PIC Functions -----------------*/ 905255040Sgibbs/* 906255040Sgibbs * Mask a level triggered interrupt source. 907255040Sgibbs * 908255040Sgibbs * \param isrc The interrupt source to mask (if necessary). 909255040Sgibbs * \param eoi If non-zero, perform any necessary end-of-interrupt 910255040Sgibbs * acknowledgements. 911255040Sgibbs */ 912255040Sgibbsstatic void 913255040Sgibbsxen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi) 914255040Sgibbs{ 915255040Sgibbs struct xenisrc *isrc; 916255040Sgibbs 917255040Sgibbs isrc = (struct xenisrc *)base_isrc; 918255040Sgibbs evtchn_mask_port(isrc->xi_port); 919255040Sgibbs} 920255040Sgibbs 921255040Sgibbs/* 922255040Sgibbs * Unmask a level triggered interrupt source. 923255040Sgibbs * 924255040Sgibbs * \param isrc The interrupt source to unmask (if necessary). 925255040Sgibbs */ 926255040Sgibbsstatic void 927255040Sgibbsxen_intr_pirq_enable_source(struct intsrc *base_isrc) 928255040Sgibbs{ 929255040Sgibbs struct xenisrc *isrc; 930255040Sgibbs 931255040Sgibbs isrc = (struct xenisrc *)base_isrc; 932255040Sgibbs evtchn_unmask_port(isrc->xi_port); 933255040Sgibbs} 934255040Sgibbs 935255040Sgibbs/* 936255040Sgibbs * Perform any necessary end-of-interrupt acknowledgements. 937255040Sgibbs * 938255040Sgibbs * \param isrc The interrupt source to EOI. 939255040Sgibbs */ 940255040Sgibbsstatic void 941255040Sgibbsxen_intr_pirq_eoi_source(struct intsrc *base_isrc) 942255040Sgibbs{ 943255040Sgibbs struct xenisrc *isrc; 944255040Sgibbs 945255040Sgibbs /* XXX Use shared page of flags for this. */ 946255040Sgibbs isrc = (struct xenisrc *)base_isrc; 947255040Sgibbs if (isrc->xi_needs_eoi != 0) { 948255040Sgibbs struct physdev_eoi eoi = { .irq = isrc->xi_pirq }; 949255040Sgibbs 950255040Sgibbs (void)HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi); 951255040Sgibbs } 952255040Sgibbs} 953255040Sgibbs 954255040Sgibbs/* 955255040Sgibbs * Enable and unmask the interrupt source. 956255040Sgibbs * 957255040Sgibbs * \param isrc The interrupt source to enable. 958255040Sgibbs */ 959255040Sgibbsstatic void 960255040Sgibbsxen_intr_pirq_enable_intr(struct intsrc *isrc) 961255040Sgibbs{ 962255040Sgibbs} 963255040Sgibbs 964255040Sgibbs/*--------------------------- Public Functions -------------------------------*/ 965255040Sgibbs/*------- API comments for these methods can be found in xen/xenintr.h -------*/ 966255040Sgibbsint 967255040Sgibbsxen_intr_bind_local_port(device_t dev, evtchn_port_t local_port, 968255040Sgibbs driver_filter_t filter, driver_intr_t handler, void *arg, 969255040Sgibbs enum intr_type flags, xen_intr_handle_t *port_handlep) 970255040Sgibbs{ 971255040Sgibbs struct xenisrc *isrc; 972255040Sgibbs int error; 973255040Sgibbs 974255040Sgibbs error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, dev, 975255040Sgibbs filter, handler, arg, flags, port_handlep); 976255040Sgibbs if (error != 0) 977255040Sgibbs return (error); 978255040Sgibbs 979255040Sgibbs /* 980255040Sgibbs * The Event Channel API didn't open this port, so it is not 981255040Sgibbs * responsible for closing it automatically on unbind. 982255040Sgibbs */ 983255040Sgibbs isrc->xi_close = 0; 984255040Sgibbs return (0); 985255040Sgibbs} 986255040Sgibbs 987255040Sgibbsint 988255040Sgibbsxen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain, 989255040Sgibbs driver_filter_t filter, driver_intr_t handler, void *arg, 990255040Sgibbs enum intr_type flags, xen_intr_handle_t *port_handlep) 991255040Sgibbs{ 992255040Sgibbs struct xenisrc *isrc; 993255040Sgibbs struct evtchn_alloc_unbound alloc_unbound; 994255040Sgibbs int error; 995255040Sgibbs 996255040Sgibbs alloc_unbound.dom = DOMID_SELF; 997255040Sgibbs alloc_unbound.remote_dom = remote_domain; 998255040Sgibbs error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, 999255040Sgibbs &alloc_unbound); 1000255040Sgibbs if (error != 0) { 1001255040Sgibbs /* 1002255040Sgibbs * XXX Trap Hypercall error code Linuxisms in 1003255040Sgibbs * the HYPERCALL layer. 1004255040Sgibbs */ 1005255040Sgibbs return (-error); 1006255040Sgibbs } 1007255040Sgibbs 1008255040Sgibbs error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT, 1009255040Sgibbs dev, filter, handler, arg, flags, 1010255040Sgibbs port_handlep); 1011255040Sgibbs if (error != 0) { 1012255040Sgibbs evtchn_close_t close = { .port = alloc_unbound.port }; 1013255040Sgibbs if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 1014255040Sgibbs panic("EVTCHNOP_close failed"); 1015255040Sgibbs return (error); 1016255040Sgibbs } 1017255040Sgibbs 1018255040Sgibbs isrc->xi_close = 1; 1019255040Sgibbs return (0); 1020255040Sgibbs} 1021255040Sgibbs 1022255040Sgibbsint 1023255040Sgibbsxen_intr_bind_remote_port(device_t dev, u_int remote_domain, 1024255040Sgibbs u_int remote_port, driver_filter_t filter, driver_intr_t handler, 1025255040Sgibbs void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) 1026255040Sgibbs{ 1027255040Sgibbs struct xenisrc *isrc; 1028255040Sgibbs struct evtchn_bind_interdomain bind_interdomain; 1029255040Sgibbs int error; 1030255040Sgibbs 1031255040Sgibbs bind_interdomain.remote_dom = remote_domain; 1032255040Sgibbs bind_interdomain.remote_port = remote_port; 1033255040Sgibbs error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, 1034255040Sgibbs &bind_interdomain); 1035255040Sgibbs if (error != 0) { 1036255040Sgibbs /* 1037255040Sgibbs * XXX Trap Hypercall error code Linuxisms in 1038255040Sgibbs * the HYPERCALL layer. 1039255040Sgibbs */ 1040255040Sgibbs return (-error); 1041255040Sgibbs } 1042255040Sgibbs 1043255040Sgibbs error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port, 1044255040Sgibbs EVTCHN_TYPE_PORT, dev, filter, handler, 1045255040Sgibbs arg, flags, port_handlep); 1046255040Sgibbs if (error) { 1047255040Sgibbs evtchn_close_t close = { .port = bind_interdomain.local_port }; 1048255040Sgibbs if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 1049255040Sgibbs panic("EVTCHNOP_close failed"); 1050255040Sgibbs return (error); 1051255040Sgibbs } 1052255040Sgibbs 1053255040Sgibbs /* 1054255040Sgibbs * The Event Channel API opened this port, so it is 1055255040Sgibbs * responsible for closing it automatically on unbind. 1056255040Sgibbs */ 1057255040Sgibbs isrc->xi_close = 1; 1058255040Sgibbs return (0); 1059255040Sgibbs} 1060255040Sgibbs 1061255040Sgibbsint 1062255040Sgibbsxen_intr_bind_virq(device_t dev, u_int virq, u_int cpu, 1063255040Sgibbs driver_filter_t filter, driver_intr_t handler, void *arg, 1064255040Sgibbs enum intr_type flags, xen_intr_handle_t *port_handlep) 1065255040Sgibbs{ 1066256073Sgibbs int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; 1067255040Sgibbs struct xenisrc *isrc; 1068256073Sgibbs struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id }; 1069255040Sgibbs int error; 1070255040Sgibbs 1071255040Sgibbs /* Ensure the target CPU is ready to handle evtchn interrupts. */ 1072255040Sgibbs xen_intr_intrcnt_add(cpu); 1073255040Sgibbs 1074255040Sgibbs isrc = NULL; 1075255040Sgibbs error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq); 1076255040Sgibbs if (error != 0) { 1077255040Sgibbs /* 1078255040Sgibbs * XXX Trap Hypercall error code Linuxisms in 1079255040Sgibbs * the HYPERCALL layer. 1080255040Sgibbs */ 1081255040Sgibbs return (-error); 1082255040Sgibbs } 1083255040Sgibbs 1084255040Sgibbs error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, dev, 1085255040Sgibbs filter, handler, arg, flags, port_handlep); 1086255726Sgibbs 1087255726Sgibbs#ifdef SMP 1088255040Sgibbs if (error == 0) 1089255040Sgibbs error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); 1090255726Sgibbs#endif 1091255040Sgibbs 1092255040Sgibbs if (error != 0) { 1093255040Sgibbs evtchn_close_t close = { .port = bind_virq.port }; 1094255040Sgibbs 1095255040Sgibbs xen_intr_unbind(*port_handlep); 1096255040Sgibbs if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 1097255040Sgibbs panic("EVTCHNOP_close failed"); 1098255040Sgibbs return (error); 1099255040Sgibbs } 1100255040Sgibbs 1101255726Sgibbs#ifdef SMP 1102255040Sgibbs if (isrc->xi_cpu != cpu) { 1103255040Sgibbs /* 1104255040Sgibbs * Too early in the boot process for the generic interrupt 1105255040Sgibbs * code to perform the binding. Update our event channel 1106255040Sgibbs * masks manually so events can't fire on the wrong cpu 1107255040Sgibbs * during AP startup. 1108255040Sgibbs */ 1109255040Sgibbs xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]); 1110255040Sgibbs } 1111255726Sgibbs#endif 1112255040Sgibbs 1113255040Sgibbs /* 1114255040Sgibbs * The Event Channel API opened this port, so it is 1115255040Sgibbs * responsible for closing it automatically on unbind. 1116255040Sgibbs */ 1117255040Sgibbs isrc->xi_close = 1; 1118255726Sgibbs isrc->xi_virq = virq; 1119255726Sgibbs 1120255040Sgibbs return (0); 1121255040Sgibbs} 1122255040Sgibbs 1123255040Sgibbsint 1124255331Sgibbsxen_intr_alloc_and_bind_ipi(device_t dev, u_int cpu, 1125255040Sgibbs driver_filter_t filter, enum intr_type flags, 1126255040Sgibbs xen_intr_handle_t *port_handlep) 1127255040Sgibbs{ 1128255726Sgibbs#ifdef SMP 1129256073Sgibbs int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; 1130255040Sgibbs struct xenisrc *isrc; 1131256073Sgibbs struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; 1132255040Sgibbs int error; 1133255040Sgibbs 1134255040Sgibbs /* Ensure the target CPU is ready to handle evtchn interrupts. */ 1135255040Sgibbs xen_intr_intrcnt_add(cpu); 1136255040Sgibbs 1137255040Sgibbs isrc = NULL; 1138255040Sgibbs error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi); 1139255040Sgibbs if (error != 0) { 1140255040Sgibbs /* 1141255040Sgibbs * XXX Trap Hypercall error code Linuxisms in 1142255040Sgibbs * the HYPERCALL layer. 1143255040Sgibbs */ 1144255040Sgibbs return (-error); 1145255040Sgibbs } 1146255040Sgibbs 1147255040Sgibbs error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI, 1148255040Sgibbs dev, filter, NULL, NULL, flags, 1149255040Sgibbs port_handlep); 1150255040Sgibbs if (error == 0) 1151255040Sgibbs error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); 1152255040Sgibbs 1153255040Sgibbs if (error != 0) { 1154255040Sgibbs evtchn_close_t close = { .port = bind_ipi.port }; 1155255040Sgibbs 1156255040Sgibbs xen_intr_unbind(*port_handlep); 1157255040Sgibbs if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 1158255040Sgibbs panic("EVTCHNOP_close failed"); 1159255040Sgibbs return (error); 1160255040Sgibbs } 1161255040Sgibbs 1162255040Sgibbs if (isrc->xi_cpu != cpu) { 1163255040Sgibbs /* 1164255040Sgibbs * Too early in the boot process for the generic interrupt 1165255040Sgibbs * code to perform the binding. Update our event channel 1166255040Sgibbs * masks manually so events can't fire on the wrong cpu 1167255040Sgibbs * during AP startup. 1168255040Sgibbs */ 1169255040Sgibbs xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]); 1170255040Sgibbs } 1171255040Sgibbs 1172255040Sgibbs /* 1173255040Sgibbs * The Event Channel API opened this port, so it is 1174255040Sgibbs * responsible for closing it automatically on unbind. 1175255040Sgibbs */ 1176255040Sgibbs isrc->xi_close = 1; 1177255040Sgibbs return (0); 1178255726Sgibbs#else 1179255726Sgibbs return (EOPNOTSUPP); 1180255726Sgibbs#endif 1181255040Sgibbs} 1182255040Sgibbs 1183255040Sgibbsint 1184255040Sgibbsxen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...) 1185255040Sgibbs{ 1186255040Sgibbs char descr[MAXCOMLEN + 1]; 1187255040Sgibbs struct xenisrc *isrc; 1188255040Sgibbs va_list ap; 1189255040Sgibbs 1190255040Sgibbs isrc = xen_intr_isrc(port_handle); 1191255040Sgibbs if (isrc == NULL) 1192255040Sgibbs return (EINVAL); 1193255040Sgibbs 1194255040Sgibbs va_start(ap, fmt); 1195255040Sgibbs vsnprintf(descr, sizeof(descr), fmt, ap); 1196255040Sgibbs va_end(ap); 1197255040Sgibbs return (intr_describe(isrc->xi_vector, port_handle, descr)); 1198255040Sgibbs} 1199255040Sgibbs 1200255040Sgibbsvoid 1201255040Sgibbsxen_intr_unbind(xen_intr_handle_t *port_handlep) 1202255040Sgibbs{ 1203255040Sgibbs struct intr_handler *handler; 1204255040Sgibbs struct xenisrc *isrc; 1205255040Sgibbs 1206255040Sgibbs handler = *port_handlep; 1207255040Sgibbs *port_handlep = NULL; 1208255040Sgibbs isrc = xen_intr_isrc(handler); 1209255040Sgibbs if (isrc == NULL) 1210255040Sgibbs return; 1211255040Sgibbs 1212255040Sgibbs intr_remove_handler(handler); 1213255040Sgibbs xen_intr_release_isrc(isrc); 1214255040Sgibbs} 1215255040Sgibbs 1216255040Sgibbsvoid 1217255040Sgibbsxen_intr_signal(xen_intr_handle_t handle) 1218255040Sgibbs{ 1219255040Sgibbs struct xenisrc *isrc; 1220255040Sgibbs 1221255040Sgibbs isrc = xen_intr_isrc(handle); 1222255040Sgibbs if (isrc != NULL) { 1223255040Sgibbs KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT || 1224255040Sgibbs isrc->xi_type == EVTCHN_TYPE_IPI, 1225255040Sgibbs ("evtchn_signal on something other than a local port")); 1226255040Sgibbs struct evtchn_send send = { .port = isrc->xi_port }; 1227255040Sgibbs (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send); 1228255040Sgibbs } 1229255040Sgibbs} 1230255040Sgibbs 1231255040Sgibbsevtchn_port_t 1232255040Sgibbsxen_intr_port(xen_intr_handle_t handle) 1233255040Sgibbs{ 1234255040Sgibbs struct xenisrc *isrc; 1235255040Sgibbs 1236255040Sgibbs isrc = xen_intr_isrc(handle); 1237255040Sgibbs if (isrc == NULL) 1238255040Sgibbs return (0); 1239255040Sgibbs 1240255040Sgibbs return (isrc->xi_port); 1241255040Sgibbs} 1242