1164265Sjhb/*- 2172937Sjhb * Copyright (c) 2006 Yahoo!, Inc. 3164265Sjhb * All rights reserved. 4172937Sjhb * Written by: John Baldwin <jhb@FreeBSD.org> 5164265Sjhb * 6164265Sjhb * Redistribution and use in source and binary forms, with or without 7164265Sjhb * modification, are permitted provided that the following conditions 8164265Sjhb * are met: 9164265Sjhb * 1. Redistributions of source code must retain the above copyright 10164265Sjhb * notice, this list of conditions and the following disclaimer. 11164265Sjhb * 2. Redistributions in binary form must reproduce the above copyright 12164265Sjhb * notice, this list of conditions and the following disclaimer in the 13164265Sjhb * documentation and/or other materials provided with the distribution. 14164265Sjhb * 3. Neither the name of the author nor the names of any co-contributors 15164265Sjhb * may be used to endorse or promote products derived from this software 16164265Sjhb * without specific prior written permission. 17164265Sjhb * 18164265Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19164265Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20164265Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21164265Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22164265Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23164265Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24164265Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25164265Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26164265Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27164265Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28164265Sjhb * SUCH DAMAGE. 29164265Sjhb */ 30164265Sjhb 31164265Sjhb/* 32164265Sjhb * Support for PCI Message Signalled Interrupts (MSI). MSI interrupts on 33164265Sjhb * x86 are basically APIC messages that the northbridge delivers directly 34164265Sjhb * to the local APICs as if they had come from an I/O APIC. 35164265Sjhb */ 36164265Sjhb 37164265Sjhb#include <sys/cdefs.h> 38164265Sjhb__FBSDID("$FreeBSD: stable/10/sys/x86/x86/msi.c 342587 2018-12-29 01:19:14Z jhb $"); 39164265Sjhb 40164265Sjhb#include <sys/param.h> 41164265Sjhb#include <sys/bus.h> 42164265Sjhb#include <sys/kernel.h> 43164265Sjhb#include <sys/lock.h> 44164265Sjhb#include <sys/malloc.h> 45164265Sjhb#include <sys/mutex.h> 46164265Sjhb#include <sys/sx.h> 47303776Sjhb#include <sys/sysctl.h> 48164265Sjhb#include <sys/systm.h> 49214631Sjhb#include <x86/apicreg.h> 50185341Sjkim#include <machine/cputypes.h> 51164265Sjhb#include <machine/md_var.h> 52164265Sjhb#include <machine/frame.h> 53164265Sjhb#include <machine/intr_machdep.h> 54164265Sjhb#include <machine/apicvar.h> 55187157Sjkim#include <machine/specialreg.h> 56164265Sjhb#include <dev/pci/pcivar.h> 57164265Sjhb 58164265Sjhb/* Fields in address for Intel MSI messages. */ 59164265Sjhb#define MSI_INTEL_ADDR_DEST 0x000ff000 60164265Sjhb#define MSI_INTEL_ADDR_RH 0x00000008 61164265Sjhb# define MSI_INTEL_ADDR_RH_ON 0x00000008 62164265Sjhb# define MSI_INTEL_ADDR_RH_OFF 0x00000000 63164265Sjhb#define MSI_INTEL_ADDR_DM 0x00000004 64164265Sjhb# define MSI_INTEL_ADDR_DM_PHYSICAL 0x00000000 65164265Sjhb# define MSI_INTEL_ADDR_DM_LOGICAL 0x00000004 66164265Sjhb 67164265Sjhb/* Fields in data for Intel MSI messages. */ 68164265Sjhb#define MSI_INTEL_DATA_TRGRMOD IOART_TRGRMOD /* Trigger mode. */ 69164265Sjhb# define MSI_INTEL_DATA_TRGREDG IOART_TRGREDG 70164265Sjhb# define MSI_INTEL_DATA_TRGRLVL IOART_TRGRLVL 71164265Sjhb#define MSI_INTEL_DATA_LEVEL 0x00004000 /* Polarity. */ 72164265Sjhb# define MSI_INTEL_DATA_DEASSERT 0x00000000 73164265Sjhb# define MSI_INTEL_DATA_ASSERT 0x00004000 74164265Sjhb#define MSI_INTEL_DATA_DELMOD IOART_DELMOD /* Delivery mode. */ 75164265Sjhb# define MSI_INTEL_DATA_DELFIXED IOART_DELFIXED 76164265Sjhb# define MSI_INTEL_DATA_DELLOPRI IOART_DELLOPRI 77164265Sjhb# define MSI_INTEL_DATA_DELSMI IOART_DELSMI 78164265Sjhb# define MSI_INTEL_DATA_DELNMI IOART_DELNMI 79164265Sjhb# define MSI_INTEL_DATA_DELINIT IOART_DELINIT 80164265Sjhb# define MSI_INTEL_DATA_DELEXINT IOART_DELEXINT 81164265Sjhb#define MSI_INTEL_DATA_INTVEC IOART_INTVEC /* Interrupt vector. */ 82164265Sjhb 83164265Sjhb/* 84164265Sjhb * Build Intel MSI message and data values from a source. AMD64 systems 85164265Sjhb * seem to be compatible, so we use the same function for both. 86164265Sjhb */ 87164265Sjhb#define INTEL_ADDR(msi) \ 88164265Sjhb (MSI_INTEL_ADDR_BASE | (msi)->msi_cpu << 12 | \ 89164265Sjhb MSI_INTEL_ADDR_RH_OFF | MSI_INTEL_ADDR_DM_PHYSICAL) 90164265Sjhb#define INTEL_DATA(msi) \ 91164265Sjhb (MSI_INTEL_DATA_TRGREDG | MSI_INTEL_DATA_DELFIXED | (msi)->msi_vector) 92164265Sjhb 93164265Sjhbstatic MALLOC_DEFINE(M_MSI, "msi", "PCI MSI"); 94164265Sjhb 95164265Sjhb/* 96164265Sjhb * MSI sources are bunched into groups. This is because MSI forces 97164265Sjhb * all of the messages to share the address and data registers and 98164265Sjhb * thus certain properties (such as the local APIC ID target on x86). 99164265Sjhb * Each group has a 'first' source that contains information global to 100164265Sjhb * the group. These fields are marked with (g) below. 101164265Sjhb * 102164265Sjhb * Note that local APIC ID is kind of special. Each message will be 103164265Sjhb * assigned an ID by the system; however, a group will use the ID from 104164265Sjhb * the first message. 105164265Sjhb * 106169221Sjhb * For MSI-X, each message is isolated. 107164265Sjhb */ 108164265Sjhbstruct msi_intsrc { 109164265Sjhb struct intsrc msi_intsrc; 110164265Sjhb device_t msi_dev; /* Owning device. (g) */ 111164265Sjhb struct msi_intsrc *msi_first; /* First source in group. */ 112164265Sjhb u_int msi_irq; /* IRQ cookie. */ 113164265Sjhb u_int msi_msix; /* MSI-X message. */ 114164265Sjhb u_int msi_vector:8; /* IDT vector. */ 115164265Sjhb u_int msi_cpu:8; /* Local APIC ID. (g) */ 116164265Sjhb u_int msi_count:8; /* Messages in this group. (g) */ 117195249Sjhb u_int msi_maxcount:8; /* Alignment for this group. (g) */ 118195249Sjhb int *msi_irqs; /* Group's IRQ list. (g) */ 119164265Sjhb}; 120164265Sjhb 121169391Sjhbstatic void msi_create_source(void); 122164265Sjhbstatic void msi_enable_source(struct intsrc *isrc); 123164265Sjhbstatic void msi_disable_source(struct intsrc *isrc, int eoi); 124164265Sjhbstatic void msi_eoi_source(struct intsrc *isrc); 125164265Sjhbstatic void msi_enable_intr(struct intsrc *isrc); 126169391Sjhbstatic void msi_disable_intr(struct intsrc *isrc); 127164265Sjhbstatic int msi_vector(struct intsrc *isrc); 128164265Sjhbstatic int msi_source_pending(struct intsrc *isrc); 129164265Sjhbstatic int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig, 130164265Sjhb enum intr_polarity pol); 131195249Sjhbstatic int msi_assign_cpu(struct intsrc *isrc, u_int apic_id); 132164265Sjhb 133164265Sjhbstruct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source, 134169391Sjhb msi_enable_intr, msi_disable_intr, msi_vector, 135169391Sjhb msi_source_pending, NULL, NULL, msi_config_intr, 136169391Sjhb msi_assign_cpu }; 137164265Sjhb 138305672Sjhb#ifdef SMP 139303776Sjhb/** 140303776Sjhb * Xen hypervisors prior to 4.6.0 do not properly handle updates to 141303776Sjhb * enabled MSI-X table entries. Allow migration of MSI-X interrupts 142303776Sjhb * to be disabled via a tunable. Values have the following meaning: 143303776Sjhb * 144303776Sjhb * -1: automatic detection by FreeBSD 145303776Sjhb * 0: enable migration 146303776Sjhb * 1: disable migration 147303776Sjhb */ 148303776Sjhbint msix_disable_migration = -1; 149303776SjhbSYSCTL_INT(_machdep, OID_AUTO, disable_msix_migration, CTLFLAG_RDTUN, 150303776Sjhb &msix_disable_migration, 0, 151303776Sjhb "Disable migration of MSI-X interrupts between CPUs"); 152305672Sjhb#endif 153303776Sjhb 154164265Sjhbstatic int msi_enabled; 155169391Sjhbstatic int msi_last_irq; 156169391Sjhbstatic struct mtx msi_lock; 157164265Sjhb 158164265Sjhbstatic void 159164265Sjhbmsi_enable_source(struct intsrc *isrc) 160164265Sjhb{ 161164265Sjhb} 162164265Sjhb 163164265Sjhbstatic void 164164265Sjhbmsi_disable_source(struct intsrc *isrc, int eoi) 165164265Sjhb{ 166164265Sjhb 167164265Sjhb if (eoi == PIC_EOI) 168164265Sjhb lapic_eoi(); 169164265Sjhb} 170164265Sjhb 171164265Sjhbstatic void 172164265Sjhbmsi_eoi_source(struct intsrc *isrc) 173164265Sjhb{ 174164265Sjhb 175164265Sjhb lapic_eoi(); 176164265Sjhb} 177164265Sjhb 178164265Sjhbstatic void 179164265Sjhbmsi_enable_intr(struct intsrc *isrc) 180164265Sjhb{ 181164265Sjhb struct msi_intsrc *msi = (struct msi_intsrc *)isrc; 182164265Sjhb 183187880Sjeff apic_enable_vector(msi->msi_cpu, msi->msi_vector); 184164265Sjhb} 185164265Sjhb 186169391Sjhbstatic void 187169391Sjhbmsi_disable_intr(struct intsrc *isrc) 188169391Sjhb{ 189169391Sjhb struct msi_intsrc *msi = (struct msi_intsrc *)isrc; 190169391Sjhb 191187880Sjeff apic_disable_vector(msi->msi_cpu, msi->msi_vector); 192169391Sjhb} 193169391Sjhb 194164265Sjhbstatic int 195164265Sjhbmsi_vector(struct intsrc *isrc) 196164265Sjhb{ 197164265Sjhb struct msi_intsrc *msi = (struct msi_intsrc *)isrc; 198164265Sjhb 199164265Sjhb return (msi->msi_irq); 200164265Sjhb} 201164265Sjhb 202164265Sjhbstatic int 203164265Sjhbmsi_source_pending(struct intsrc *isrc) 204164265Sjhb{ 205164265Sjhb 206164265Sjhb return (0); 207164265Sjhb} 208164265Sjhb 209164265Sjhbstatic int 210164265Sjhbmsi_config_intr(struct intsrc *isrc, enum intr_trigger trig, 211164265Sjhb enum intr_polarity pol) 212164265Sjhb{ 213164265Sjhb 214164265Sjhb return (ENODEV); 215164265Sjhb} 216164265Sjhb 217195249Sjhbstatic int 218164265Sjhbmsi_assign_cpu(struct intsrc *isrc, u_int apic_id) 219164265Sjhb{ 220195249Sjhb struct msi_intsrc *sib, *msi = (struct msi_intsrc *)isrc; 221187880Sjeff int old_vector; 222187880Sjeff u_int old_id; 223195249Sjhb int i, vector; 224164265Sjhb 225195249Sjhb /* 226195249Sjhb * Only allow CPUs to be assigned to the first message for an 227195249Sjhb * MSI group. 228195249Sjhb */ 229195249Sjhb if (msi->msi_first != msi) 230195249Sjhb return (EINVAL); 231195249Sjhb 232305672Sjhb#ifdef SMP 233303776Sjhb if (msix_disable_migration && msi->msi_msix) 234303776Sjhb return (EINVAL); 235305672Sjhb#endif 236303776Sjhb 237187880Sjeff /* Store information to free existing irq. */ 238187880Sjeff old_vector = msi->msi_vector; 239187880Sjeff old_id = msi->msi_cpu; 240194985Sjhb if (old_id == apic_id) 241195249Sjhb return (0); 242194985Sjhb 243195249Sjhb /* Allocate IDT vectors on this cpu. */ 244195249Sjhb if (msi->msi_count > 1) { 245195249Sjhb KASSERT(msi->msi_msix == 0, ("MSI-X message group")); 246195249Sjhb vector = apic_alloc_vectors(apic_id, msi->msi_irqs, 247195249Sjhb msi->msi_count, msi->msi_maxcount); 248195249Sjhb } else 249195249Sjhb vector = apic_alloc_vector(apic_id, msi->msi_irq); 250187880Sjeff if (vector == 0) 251195249Sjhb return (ENOSPC); 252195249Sjhb 253164265Sjhb msi->msi_cpu = apic_id; 254187880Sjeff msi->msi_vector = vector; 255195415Sjhb if (msi->msi_intsrc.is_handlers > 0) 256195415Sjhb apic_enable_vector(msi->msi_cpu, msi->msi_vector); 257164265Sjhb if (bootverbose) 258187880Sjeff printf("msi: Assigning %s IRQ %d to local APIC %u vector %u\n", 259169221Sjhb msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq, 260187880Sjeff msi->msi_cpu, msi->msi_vector); 261195249Sjhb for (i = 1; i < msi->msi_count; i++) { 262195249Sjhb sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]); 263195249Sjhb sib->msi_cpu = apic_id; 264195249Sjhb sib->msi_vector = vector + i; 265195415Sjhb if (sib->msi_intsrc.is_handlers > 0) 266195415Sjhb apic_enable_vector(sib->msi_cpu, sib->msi_vector); 267195249Sjhb if (bootverbose) 268195249Sjhb printf( 269195249Sjhb "msi: Assigning MSI IRQ %d to local APIC %u vector %u\n", 270195249Sjhb sib->msi_irq, sib->msi_cpu, sib->msi_vector); 271195249Sjhb } 272209154Smav BUS_REMAP_INTR(device_get_parent(msi->msi_dev), msi->msi_dev, 273209154Smav msi->msi_irq); 274194985Sjhb 275187880Sjeff /* 276187880Sjeff * Free the old vector after the new one is established. This is done 277187880Sjeff * to prevent races where we could miss an interrupt. 278187880Sjeff */ 279195415Sjhb if (msi->msi_intsrc.is_handlers > 0) 280195415Sjhb apic_disable_vector(old_id, old_vector); 281194985Sjhb apic_free_vector(old_id, old_vector, msi->msi_irq); 282195415Sjhb for (i = 1; i < msi->msi_count; i++) { 283195415Sjhb sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]); 284195415Sjhb if (sib->msi_intsrc.is_handlers > 0) 285195415Sjhb apic_disable_vector(old_id, old_vector + i); 286195249Sjhb apic_free_vector(old_id, old_vector + i, msi->msi_irqs[i]); 287195415Sjhb } 288195249Sjhb return (0); 289164265Sjhb} 290164265Sjhb 291164265Sjhbvoid 292164265Sjhbmsi_init(void) 293164265Sjhb{ 294164265Sjhb 295164265Sjhb /* Check if we have a supported CPU. */ 296187157Sjkim switch (cpu_vendor_id) { 297187157Sjkim case CPU_VENDOR_INTEL: 298187157Sjkim case CPU_VENDOR_AMD: 299187157Sjkim break; 300187157Sjkim case CPU_VENDOR_CENTAUR: 301197070Sjkim if (CPUID_TO_FAMILY(cpu_id) == 0x6 && 302197070Sjkim CPUID_TO_MODEL(cpu_id) >= 0xf) 303187157Sjkim break; 304187157Sjkim /* FALLTHROUGH */ 305187157Sjkim default: 306164265Sjhb return; 307187157Sjkim } 308164265Sjhb 309305672Sjhb#ifdef SMP 310303776Sjhb if (msix_disable_migration == -1) { 311303776Sjhb /* The default is to allow migration of MSI-X interrupts. */ 312303776Sjhb msix_disable_migration = 0; 313303776Sjhb } 314305672Sjhb#endif 315303776Sjhb 316164265Sjhb msi_enabled = 1; 317164265Sjhb intr_register_pic(&msi_pic); 318169391Sjhb mtx_init(&msi_lock, "msi", NULL, MTX_DEF); 319164265Sjhb} 320164265Sjhb 321203160Savgstatic void 322169391Sjhbmsi_create_source(void) 323169221Sjhb{ 324169221Sjhb struct msi_intsrc *msi; 325169391Sjhb u_int irq; 326169221Sjhb 327169391Sjhb mtx_lock(&msi_lock); 328169391Sjhb if (msi_last_irq >= NUM_MSI_INTS) { 329169391Sjhb mtx_unlock(&msi_lock); 330169391Sjhb return; 331169391Sjhb } 332169391Sjhb irq = msi_last_irq + FIRST_MSI_INT; 333169391Sjhb msi_last_irq++; 334169391Sjhb mtx_unlock(&msi_lock); 335169391Sjhb 336195249Sjhb msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO); 337169221Sjhb msi->msi_intsrc.is_pic = &msi_pic; 338169221Sjhb msi->msi_irq = irq; 339169221Sjhb intr_register_source(&msi->msi_intsrc); 340169391Sjhb nexus_add_irq(irq); 341169221Sjhb} 342169221Sjhb 343164265Sjhb/* 344195249Sjhb * Try to allocate 'count' interrupt sources with contiguous IDT values. 345164265Sjhb */ 346164265Sjhbint 347169391Sjhbmsi_alloc(device_t dev, int count, int maxcount, int *irqs) 348164265Sjhb{ 349164265Sjhb struct msi_intsrc *msi, *fsrc; 350194985Sjhb u_int cpu; 351195249Sjhb int cnt, i, *mirqs, vector; 352164265Sjhb 353164265Sjhb if (!msi_enabled) 354164265Sjhb return (ENXIO); 355164265Sjhb 356195249Sjhb if (count > 1) 357195249Sjhb mirqs = malloc(count * sizeof(*mirqs), M_MSI, M_WAITOK); 358195249Sjhb else 359195249Sjhb mirqs = NULL; 360169391Sjhbagain: 361169391Sjhb mtx_lock(&msi_lock); 362164265Sjhb 363164265Sjhb /* Try to find 'count' free IRQs. */ 364164265Sjhb cnt = 0; 365164265Sjhb for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) { 366164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(i); 367164265Sjhb 368164265Sjhb /* End of allocated sources, so break. */ 369164265Sjhb if (msi == NULL) 370164265Sjhb break; 371164265Sjhb 372164265Sjhb /* If this is a free one, save its IRQ in the array. */ 373164265Sjhb if (msi->msi_dev == NULL) { 374164265Sjhb irqs[cnt] = i; 375164265Sjhb cnt++; 376164265Sjhb if (cnt == count) 377164265Sjhb break; 378164265Sjhb } 379164265Sjhb } 380164265Sjhb 381164265Sjhb /* Do we need to create some new sources? */ 382164265Sjhb if (cnt < count) { 383164265Sjhb /* If we would exceed the max, give up. */ 384342587Sjhb if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) { 385169391Sjhb mtx_unlock(&msi_lock); 386195249Sjhb free(mirqs, M_MSI); 387164265Sjhb return (ENXIO); 388164265Sjhb } 389169391Sjhb mtx_unlock(&msi_lock); 390164265Sjhb 391169391Sjhb /* We need count - cnt more sources. */ 392169391Sjhb while (cnt < count) { 393169391Sjhb msi_create_source(); 394164265Sjhb cnt++; 395164265Sjhb } 396169391Sjhb goto again; 397164265Sjhb } 398164265Sjhb 399164265Sjhb /* Ok, we now have the IRQs allocated. */ 400164265Sjhb KASSERT(cnt == count, ("count mismatch")); 401164265Sjhb 402194985Sjhb /* Allocate 'count' IDT vectors. */ 403194985Sjhb cpu = intr_next_cpu(); 404194985Sjhb vector = apic_alloc_vectors(cpu, irqs, count, maxcount); 405194985Sjhb if (vector == 0) { 406194985Sjhb mtx_unlock(&msi_lock); 407195249Sjhb free(mirqs, M_MSI); 408194985Sjhb return (ENOSPC); 409194985Sjhb } 410194985Sjhb 411164265Sjhb /* Assign IDT vectors and make these messages owned by 'dev'. */ 412164265Sjhb fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]); 413164265Sjhb for (i = 0; i < count; i++) { 414164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); 415194985Sjhb msi->msi_cpu = cpu; 416164265Sjhb msi->msi_dev = dev; 417194985Sjhb msi->msi_vector = vector + i; 418194985Sjhb if (bootverbose) 419194985Sjhb printf( 420194985Sjhb "msi: routing MSI IRQ %d to local APIC %u vector %u\n", 421194985Sjhb msi->msi_irq, msi->msi_cpu, msi->msi_vector); 422164265Sjhb msi->msi_first = fsrc; 423169391Sjhb KASSERT(msi->msi_intsrc.is_handlers == 0, 424169391Sjhb ("dead MSI has handlers")); 425164265Sjhb } 426164265Sjhb fsrc->msi_count = count; 427195249Sjhb fsrc->msi_maxcount = maxcount; 428195249Sjhb if (count > 1) 429195249Sjhb bcopy(irqs, mirqs, count * sizeof(*mirqs)); 430195249Sjhb fsrc->msi_irqs = mirqs; 431169391Sjhb mtx_unlock(&msi_lock); 432164265Sjhb 433164265Sjhb return (0); 434164265Sjhb} 435164265Sjhb 436164265Sjhbint 437164265Sjhbmsi_release(int *irqs, int count) 438164265Sjhb{ 439164265Sjhb struct msi_intsrc *msi, *first; 440164265Sjhb int i; 441164265Sjhb 442169391Sjhb mtx_lock(&msi_lock); 443164265Sjhb first = (struct msi_intsrc *)intr_lookup_source(irqs[0]); 444164265Sjhb if (first == NULL) { 445169391Sjhb mtx_unlock(&msi_lock); 446164265Sjhb return (ENOENT); 447164265Sjhb } 448164265Sjhb 449164265Sjhb /* Make sure this isn't an MSI-X message. */ 450164265Sjhb if (first->msi_msix) { 451169391Sjhb mtx_unlock(&msi_lock); 452164265Sjhb return (EINVAL); 453164265Sjhb } 454164265Sjhb 455164265Sjhb /* Make sure this message is allocated to a group. */ 456164265Sjhb if (first->msi_first == NULL) { 457169391Sjhb mtx_unlock(&msi_lock); 458164265Sjhb return (ENXIO); 459164265Sjhb } 460164265Sjhb 461164265Sjhb /* 462164265Sjhb * Make sure this is the start of a group and that we are releasing 463164265Sjhb * the entire group. 464164265Sjhb */ 465164265Sjhb if (first->msi_first != first || first->msi_count != count) { 466169391Sjhb mtx_unlock(&msi_lock); 467164265Sjhb return (EINVAL); 468164265Sjhb } 469164265Sjhb KASSERT(first->msi_dev != NULL, ("unowned group")); 470164265Sjhb 471164265Sjhb /* Clear all the extra messages in the group. */ 472164265Sjhb for (i = 1; i < count; i++) { 473164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); 474164265Sjhb KASSERT(msi->msi_first == first, ("message not in group")); 475164265Sjhb KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch")); 476164265Sjhb msi->msi_first = NULL; 477164265Sjhb msi->msi_dev = NULL; 478194985Sjhb apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); 479164265Sjhb msi->msi_vector = 0; 480164265Sjhb } 481164265Sjhb 482164265Sjhb /* Clear out the first message. */ 483164265Sjhb first->msi_first = NULL; 484164265Sjhb first->msi_dev = NULL; 485194985Sjhb apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq); 486164265Sjhb first->msi_vector = 0; 487164265Sjhb first->msi_count = 0; 488195249Sjhb first->msi_maxcount = 0; 489195249Sjhb free(first->msi_irqs, M_MSI); 490195249Sjhb first->msi_irqs = NULL; 491164265Sjhb 492169391Sjhb mtx_unlock(&msi_lock); 493164265Sjhb return (0); 494164265Sjhb} 495164265Sjhb 496164265Sjhbint 497169221Sjhbmsi_map(int irq, uint64_t *addr, uint32_t *data) 498164265Sjhb{ 499164265Sjhb struct msi_intsrc *msi; 500169221Sjhb 501169391Sjhb mtx_lock(&msi_lock); 502169221Sjhb msi = (struct msi_intsrc *)intr_lookup_source(irq); 503169221Sjhb if (msi == NULL) { 504169391Sjhb mtx_unlock(&msi_lock); 505169221Sjhb return (ENOENT); 506169221Sjhb } 507169221Sjhb 508169221Sjhb /* Make sure this message is allocated to a device. */ 509169221Sjhb if (msi->msi_dev == NULL) { 510169391Sjhb mtx_unlock(&msi_lock); 511169221Sjhb return (ENXIO); 512169221Sjhb } 513169221Sjhb 514169221Sjhb /* 515169221Sjhb * If this message isn't an MSI-X message, make sure it's part 516169391Sjhb * of a group, and switch to the first message in the 517169221Sjhb * group. 518169221Sjhb */ 519169221Sjhb if (!msi->msi_msix) { 520169221Sjhb if (msi->msi_first == NULL) { 521169391Sjhb mtx_unlock(&msi_lock); 522169221Sjhb return (ENXIO); 523169221Sjhb } 524169221Sjhb msi = msi->msi_first; 525169221Sjhb } 526169221Sjhb 527169221Sjhb *addr = INTEL_ADDR(msi); 528169221Sjhb *data = INTEL_DATA(msi); 529169391Sjhb mtx_unlock(&msi_lock); 530169221Sjhb return (0); 531169221Sjhb} 532169221Sjhb 533169221Sjhbint 534169391Sjhbmsix_alloc(device_t dev, int *irq) 535169221Sjhb{ 536169221Sjhb struct msi_intsrc *msi; 537194985Sjhb u_int cpu; 538194985Sjhb int i, vector; 539164265Sjhb 540164265Sjhb if (!msi_enabled) 541164265Sjhb return (ENXIO); 542164265Sjhb 543169391Sjhbagain: 544169391Sjhb mtx_lock(&msi_lock); 545164265Sjhb 546164265Sjhb /* Find a free IRQ. */ 547164265Sjhb for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) { 548164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(i); 549164265Sjhb 550164265Sjhb /* End of allocated sources, so break. */ 551164265Sjhb if (msi == NULL) 552164265Sjhb break; 553164265Sjhb 554169391Sjhb /* Stop at the first free source. */ 555164265Sjhb if (msi->msi_dev == NULL) 556164265Sjhb break; 557164265Sjhb } 558164265Sjhb 559342587Sjhb /* Are all IRQs in use? */ 560342587Sjhb if (i == FIRST_MSI_INT + NUM_MSI_INTS) { 561342587Sjhb mtx_unlock(&msi_lock); 562342587Sjhb return (ENXIO); 563342587Sjhb } 564342587Sjhb 565164265Sjhb /* Do we need to create a new source? */ 566164265Sjhb if (msi == NULL) { 567169391Sjhb mtx_unlock(&msi_lock); 568164265Sjhb 569164265Sjhb /* Create a new source. */ 570169391Sjhb msi_create_source(); 571169391Sjhb goto again; 572164265Sjhb } 573164265Sjhb 574194985Sjhb /* Allocate an IDT vector. */ 575194985Sjhb cpu = intr_next_cpu(); 576194985Sjhb vector = apic_alloc_vector(cpu, i); 577195249Sjhb if (vector == 0) { 578195249Sjhb mtx_unlock(&msi_lock); 579195249Sjhb return (ENOSPC); 580195249Sjhb } 581194985Sjhb if (bootverbose) 582194985Sjhb printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n", 583194985Sjhb msi->msi_irq, cpu, vector); 584195249Sjhb 585164265Sjhb /* Setup source. */ 586194985Sjhb msi->msi_cpu = cpu; 587164265Sjhb msi->msi_dev = dev; 588195249Sjhb msi->msi_first = msi; 589194985Sjhb msi->msi_vector = vector; 590164265Sjhb msi->msi_msix = 1; 591195249Sjhb msi->msi_count = 1; 592195249Sjhb msi->msi_maxcount = 1; 593195249Sjhb msi->msi_irqs = NULL; 594164265Sjhb 595169391Sjhb KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers")); 596169391Sjhb mtx_unlock(&msi_lock); 597164265Sjhb 598164265Sjhb *irq = i; 599164265Sjhb return (0); 600164265Sjhb} 601164265Sjhb 602164265Sjhbint 603164265Sjhbmsix_release(int irq) 604164265Sjhb{ 605164265Sjhb struct msi_intsrc *msi; 606164265Sjhb 607169391Sjhb mtx_lock(&msi_lock); 608164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(irq); 609164265Sjhb if (msi == NULL) { 610169391Sjhb mtx_unlock(&msi_lock); 611164265Sjhb return (ENOENT); 612164265Sjhb } 613164265Sjhb 614164265Sjhb /* Make sure this is an MSI-X message. */ 615164265Sjhb if (!msi->msi_msix) { 616169391Sjhb mtx_unlock(&msi_lock); 617164265Sjhb return (EINVAL); 618164265Sjhb } 619164265Sjhb 620164265Sjhb KASSERT(msi->msi_dev != NULL, ("unowned message")); 621164265Sjhb 622164265Sjhb /* Clear out the message. */ 623195249Sjhb msi->msi_first = NULL; 624164265Sjhb msi->msi_dev = NULL; 625194985Sjhb apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); 626164265Sjhb msi->msi_vector = 0; 627164265Sjhb msi->msi_msix = 0; 628195249Sjhb msi->msi_count = 0; 629195249Sjhb msi->msi_maxcount = 0; 630164265Sjhb 631169391Sjhb mtx_unlock(&msi_lock); 632164265Sjhb return (0); 633164265Sjhb} 634