1164265Sjhb/*- 2172937Sjhb * Copyright (c) 2006 Yahoo!, Inc. 3164265Sjhb * All rights reserved. 4172937Sjhb * Written by: John Baldwin <jhb@FreeBSD.org> 5164265Sjhb * 6164265Sjhb * Redistribution and use in source and binary forms, with or without 7164265Sjhb * modification, are permitted provided that the following conditions 8164265Sjhb * are met: 9164265Sjhb * 1. Redistributions of source code must retain the above copyright 10164265Sjhb * notice, this list of conditions and the following disclaimer. 11164265Sjhb * 2. Redistributions in binary form must reproduce the above copyright 12164265Sjhb * notice, this list of conditions and the following disclaimer in the 13164265Sjhb * documentation and/or other materials provided with the distribution. 14164265Sjhb * 3. Neither the name of the author nor the names of any co-contributors 15164265Sjhb * may be used to endorse or promote products derived from this software 16164265Sjhb * without specific prior written permission. 17164265Sjhb * 18164265Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19164265Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20164265Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21164265Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22164265Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23164265Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24164265Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25164265Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26164265Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27164265Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28164265Sjhb * SUCH DAMAGE. 29164265Sjhb */ 30164265Sjhb 31164265Sjhb/* 32164265Sjhb * Support for PCI Message Signalled Interrupts (MSI). MSI interrupts on 33164265Sjhb * x86 are basically APIC messages that the northbridge delivers directly 34164265Sjhb * to the local APICs as if they had come from an I/O APIC. 35164265Sjhb */ 36164265Sjhb 37164265Sjhb#include <sys/cdefs.h> 38164265Sjhb__FBSDID("$FreeBSD$"); 39164265Sjhb 40164265Sjhb#include <sys/param.h> 41164265Sjhb#include <sys/bus.h> 42164265Sjhb#include <sys/kernel.h> 43164265Sjhb#include <sys/lock.h> 44164265Sjhb#include <sys/malloc.h> 45164265Sjhb#include <sys/mutex.h> 46164265Sjhb#include <sys/sx.h> 47164265Sjhb#include <sys/systm.h> 48214631Sjhb#include <x86/apicreg.h> 49185341Sjkim#include <machine/cputypes.h> 50164265Sjhb#include <machine/md_var.h> 51164265Sjhb#include <machine/frame.h> 52164265Sjhb#include <machine/intr_machdep.h> 53164265Sjhb#include <machine/apicvar.h> 54187157Sjkim#include <machine/specialreg.h> 55164265Sjhb#include <dev/pci/pcivar.h> 56164265Sjhb 57164265Sjhb/* Fields in address for Intel MSI messages. */ 58164265Sjhb#define MSI_INTEL_ADDR_DEST 0x000ff000 59164265Sjhb#define MSI_INTEL_ADDR_RH 0x00000008 60164265Sjhb# define MSI_INTEL_ADDR_RH_ON 0x00000008 61164265Sjhb# define MSI_INTEL_ADDR_RH_OFF 0x00000000 62164265Sjhb#define MSI_INTEL_ADDR_DM 0x00000004 63164265Sjhb# define MSI_INTEL_ADDR_DM_PHYSICAL 0x00000000 64164265Sjhb# define MSI_INTEL_ADDR_DM_LOGICAL 0x00000004 65164265Sjhb 66164265Sjhb/* Fields in data for Intel MSI messages. */ 67164265Sjhb#define MSI_INTEL_DATA_TRGRMOD IOART_TRGRMOD /* Trigger mode. */ 68164265Sjhb# define MSI_INTEL_DATA_TRGREDG IOART_TRGREDG 69164265Sjhb# define MSI_INTEL_DATA_TRGRLVL IOART_TRGRLVL 70164265Sjhb#define MSI_INTEL_DATA_LEVEL 0x00004000 /* Polarity. */ 71164265Sjhb# define MSI_INTEL_DATA_DEASSERT 0x00000000 72164265Sjhb# define MSI_INTEL_DATA_ASSERT 0x00004000 73164265Sjhb#define MSI_INTEL_DATA_DELMOD IOART_DELMOD /* Delivery mode. */ 74164265Sjhb# define MSI_INTEL_DATA_DELFIXED IOART_DELFIXED 75164265Sjhb# define MSI_INTEL_DATA_DELLOPRI IOART_DELLOPRI 76164265Sjhb# define MSI_INTEL_DATA_DELSMI IOART_DELSMI 77164265Sjhb# define MSI_INTEL_DATA_DELNMI IOART_DELNMI 78164265Sjhb# define MSI_INTEL_DATA_DELINIT IOART_DELINIT 79164265Sjhb# define MSI_INTEL_DATA_DELEXINT IOART_DELEXINT 80164265Sjhb#define MSI_INTEL_DATA_INTVEC IOART_INTVEC /* Interrupt vector. */ 81164265Sjhb 82164265Sjhb/* 83164265Sjhb * Build Intel MSI message and data values from a source. AMD64 systems 84164265Sjhb * seem to be compatible, so we use the same function for both. 85164265Sjhb */ 86164265Sjhb#define INTEL_ADDR(msi) \ 87164265Sjhb (MSI_INTEL_ADDR_BASE | (msi)->msi_cpu << 12 | \ 88164265Sjhb MSI_INTEL_ADDR_RH_OFF | MSI_INTEL_ADDR_DM_PHYSICAL) 89164265Sjhb#define INTEL_DATA(msi) \ 90164265Sjhb (MSI_INTEL_DATA_TRGREDG | MSI_INTEL_DATA_DELFIXED | (msi)->msi_vector) 91164265Sjhb 92164265Sjhbstatic MALLOC_DEFINE(M_MSI, "msi", "PCI MSI"); 93164265Sjhb 94164265Sjhb/* 95164265Sjhb * MSI sources are bunched into groups. This is because MSI forces 96164265Sjhb * all of the messages to share the address and data registers and 97164265Sjhb * thus certain properties (such as the local APIC ID target on x86). 98164265Sjhb * Each group has a 'first' source that contains information global to 99164265Sjhb * the group. These fields are marked with (g) below. 100164265Sjhb * 101164265Sjhb * Note that local APIC ID is kind of special. Each message will be 102164265Sjhb * assigned an ID by the system; however, a group will use the ID from 103164265Sjhb * the first message. 104164265Sjhb * 105169221Sjhb * For MSI-X, each message is isolated. 106164265Sjhb */ 107164265Sjhbstruct msi_intsrc { 108164265Sjhb struct intsrc msi_intsrc; 109164265Sjhb device_t msi_dev; /* Owning device. (g) */ 110164265Sjhb struct msi_intsrc *msi_first; /* First source in group. */ 111164265Sjhb u_int msi_irq; /* IRQ cookie. */ 112164265Sjhb u_int msi_msix; /* MSI-X message. */ 113164265Sjhb u_int msi_vector:8; /* IDT vector. */ 114164265Sjhb u_int msi_cpu:8; /* Local APIC ID. (g) */ 115164265Sjhb u_int msi_count:8; /* Messages in this group. (g) */ 116195249Sjhb u_int msi_maxcount:8; /* Alignment for this group. (g) */ 117195249Sjhb int *msi_irqs; /* Group's IRQ list. (g) */ 118164265Sjhb}; 119164265Sjhb 120169391Sjhbstatic void msi_create_source(void); 121164265Sjhbstatic void msi_enable_source(struct intsrc *isrc); 122164265Sjhbstatic void msi_disable_source(struct intsrc *isrc, int eoi); 123164265Sjhbstatic void msi_eoi_source(struct intsrc *isrc); 124164265Sjhbstatic void msi_enable_intr(struct intsrc *isrc); 125169391Sjhbstatic void msi_disable_intr(struct intsrc *isrc); 126164265Sjhbstatic int msi_vector(struct intsrc *isrc); 127164265Sjhbstatic int msi_source_pending(struct intsrc *isrc); 128164265Sjhbstatic int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig, 129164265Sjhb enum intr_polarity pol); 130195249Sjhbstatic int msi_assign_cpu(struct intsrc *isrc, u_int apic_id); 131164265Sjhb 132164265Sjhbstruct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source, 133169391Sjhb msi_enable_intr, msi_disable_intr, msi_vector, 134169391Sjhb msi_source_pending, NULL, NULL, msi_config_intr, 135169391Sjhb msi_assign_cpu }; 136164265Sjhb 137164265Sjhbstatic int msi_enabled; 138169391Sjhbstatic int msi_last_irq; 139169391Sjhbstatic struct mtx msi_lock; 140164265Sjhb 141164265Sjhbstatic void 142164265Sjhbmsi_enable_source(struct intsrc *isrc) 143164265Sjhb{ 144164265Sjhb} 145164265Sjhb 146164265Sjhbstatic void 147164265Sjhbmsi_disable_source(struct intsrc *isrc, int eoi) 148164265Sjhb{ 149164265Sjhb 150164265Sjhb if (eoi == PIC_EOI) 151164265Sjhb lapic_eoi(); 152164265Sjhb} 153164265Sjhb 154164265Sjhbstatic void 155164265Sjhbmsi_eoi_source(struct intsrc *isrc) 156164265Sjhb{ 157164265Sjhb 158164265Sjhb lapic_eoi(); 159164265Sjhb} 160164265Sjhb 161164265Sjhbstatic void 162164265Sjhbmsi_enable_intr(struct intsrc *isrc) 163164265Sjhb{ 164164265Sjhb struct msi_intsrc *msi = (struct msi_intsrc *)isrc; 165164265Sjhb 166187880Sjeff apic_enable_vector(msi->msi_cpu, msi->msi_vector); 167164265Sjhb} 168164265Sjhb 169169391Sjhbstatic void 170169391Sjhbmsi_disable_intr(struct intsrc *isrc) 171169391Sjhb{ 172169391Sjhb struct msi_intsrc *msi = (struct msi_intsrc *)isrc; 173169391Sjhb 174187880Sjeff apic_disable_vector(msi->msi_cpu, msi->msi_vector); 175169391Sjhb} 176169391Sjhb 177164265Sjhbstatic int 178164265Sjhbmsi_vector(struct intsrc *isrc) 179164265Sjhb{ 180164265Sjhb struct msi_intsrc *msi = (struct msi_intsrc *)isrc; 181164265Sjhb 182164265Sjhb return (msi->msi_irq); 183164265Sjhb} 184164265Sjhb 185164265Sjhbstatic int 186164265Sjhbmsi_source_pending(struct intsrc *isrc) 187164265Sjhb{ 188164265Sjhb 189164265Sjhb return (0); 190164265Sjhb} 191164265Sjhb 192164265Sjhbstatic int 193164265Sjhbmsi_config_intr(struct intsrc *isrc, enum intr_trigger trig, 194164265Sjhb enum intr_polarity pol) 195164265Sjhb{ 196164265Sjhb 197164265Sjhb return (ENODEV); 198164265Sjhb} 199164265Sjhb 200195249Sjhbstatic int 201164265Sjhbmsi_assign_cpu(struct intsrc *isrc, u_int apic_id) 202164265Sjhb{ 203195249Sjhb struct msi_intsrc *sib, *msi = (struct msi_intsrc *)isrc; 204187880Sjeff int old_vector; 205187880Sjeff u_int old_id; 206195249Sjhb int i, vector; 207164265Sjhb 208195249Sjhb /* 209195249Sjhb * Only allow CPUs to be assigned to the first message for an 210195249Sjhb * MSI group. 211195249Sjhb */ 212195249Sjhb if (msi->msi_first != msi) 213195249Sjhb return (EINVAL); 214195249Sjhb 215187880Sjeff /* Store information to free existing irq. */ 216187880Sjeff old_vector = msi->msi_vector; 217187880Sjeff old_id = msi->msi_cpu; 218194985Sjhb if (old_id == apic_id) 219195249Sjhb return (0); 220194985Sjhb 221195249Sjhb /* Allocate IDT vectors on this cpu. */ 222195249Sjhb if (msi->msi_count > 1) { 223195249Sjhb KASSERT(msi->msi_msix == 0, ("MSI-X message group")); 224195249Sjhb vector = apic_alloc_vectors(apic_id, msi->msi_irqs, 225195249Sjhb msi->msi_count, msi->msi_maxcount); 226195249Sjhb } else 227195249Sjhb vector = apic_alloc_vector(apic_id, msi->msi_irq); 228187880Sjeff if (vector == 0) 229195249Sjhb return (ENOSPC); 230195249Sjhb 231164265Sjhb msi->msi_cpu = apic_id; 232187880Sjeff msi->msi_vector = vector; 233195415Sjhb if (msi->msi_intsrc.is_handlers > 0) 234195415Sjhb apic_enable_vector(msi->msi_cpu, msi->msi_vector); 235164265Sjhb if (bootverbose) 236187880Sjeff printf("msi: Assigning %s IRQ %d to local APIC %u vector %u\n", 237169221Sjhb msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq, 238187880Sjeff msi->msi_cpu, msi->msi_vector); 239195249Sjhb for (i = 1; i < msi->msi_count; i++) { 240195249Sjhb sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]); 241195249Sjhb sib->msi_cpu = apic_id; 242195249Sjhb sib->msi_vector = vector + i; 243195415Sjhb if (sib->msi_intsrc.is_handlers > 0) 244195415Sjhb apic_enable_vector(sib->msi_cpu, sib->msi_vector); 245195249Sjhb if (bootverbose) 246195249Sjhb printf( 247195249Sjhb "msi: Assigning MSI IRQ %d to local APIC %u vector %u\n", 248195249Sjhb sib->msi_irq, sib->msi_cpu, sib->msi_vector); 249195249Sjhb } 250209154Smav BUS_REMAP_INTR(device_get_parent(msi->msi_dev), msi->msi_dev, 251209154Smav msi->msi_irq); 252194985Sjhb 253187880Sjeff /* 254187880Sjeff * Free the old vector after the new one is established. This is done 255187880Sjeff * to prevent races where we could miss an interrupt. 256187880Sjeff */ 257195415Sjhb if (msi->msi_intsrc.is_handlers > 0) 258195415Sjhb apic_disable_vector(old_id, old_vector); 259194985Sjhb apic_free_vector(old_id, old_vector, msi->msi_irq); 260195415Sjhb for (i = 1; i < msi->msi_count; i++) { 261195415Sjhb sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]); 262195415Sjhb if (sib->msi_intsrc.is_handlers > 0) 263195415Sjhb apic_disable_vector(old_id, old_vector + i); 264195249Sjhb apic_free_vector(old_id, old_vector + i, msi->msi_irqs[i]); 265195415Sjhb } 266195249Sjhb return (0); 267164265Sjhb} 268164265Sjhb 269164265Sjhbvoid 270164265Sjhbmsi_init(void) 271164265Sjhb{ 272164265Sjhb 273164265Sjhb /* Check if we have a supported CPU. */ 274187157Sjkim switch (cpu_vendor_id) { 275187157Sjkim case CPU_VENDOR_INTEL: 276187157Sjkim case CPU_VENDOR_AMD: 277187157Sjkim break; 278187157Sjkim case CPU_VENDOR_CENTAUR: 279197070Sjkim if (CPUID_TO_FAMILY(cpu_id) == 0x6 && 280197070Sjkim CPUID_TO_MODEL(cpu_id) >= 0xf) 281187157Sjkim break; 282187157Sjkim /* FALLTHROUGH */ 283187157Sjkim default: 284164265Sjhb return; 285187157Sjkim } 286164265Sjhb 287164265Sjhb msi_enabled = 1; 288164265Sjhb intr_register_pic(&msi_pic); 289169391Sjhb mtx_init(&msi_lock, "msi", NULL, MTX_DEF); 290164265Sjhb} 291164265Sjhb 292203160Savgstatic void 293169391Sjhbmsi_create_source(void) 294169221Sjhb{ 295169221Sjhb struct msi_intsrc *msi; 296169391Sjhb u_int irq; 297169221Sjhb 298169391Sjhb mtx_lock(&msi_lock); 299169391Sjhb if (msi_last_irq >= NUM_MSI_INTS) { 300169391Sjhb mtx_unlock(&msi_lock); 301169391Sjhb return; 302169391Sjhb } 303169391Sjhb irq = msi_last_irq + FIRST_MSI_INT; 304169391Sjhb msi_last_irq++; 305169391Sjhb mtx_unlock(&msi_lock); 306169391Sjhb 307195249Sjhb msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO); 308169221Sjhb msi->msi_intsrc.is_pic = &msi_pic; 309169221Sjhb msi->msi_irq = irq; 310169221Sjhb intr_register_source(&msi->msi_intsrc); 311169391Sjhb nexus_add_irq(irq); 312169221Sjhb} 313169221Sjhb 314164265Sjhb/* 315195249Sjhb * Try to allocate 'count' interrupt sources with contiguous IDT values. 316164265Sjhb */ 317164265Sjhbint 318169391Sjhbmsi_alloc(device_t dev, int count, int maxcount, int *irqs) 319164265Sjhb{ 320164265Sjhb struct msi_intsrc *msi, *fsrc; 321194985Sjhb u_int cpu; 322195249Sjhb int cnt, i, *mirqs, vector; 323164265Sjhb 324164265Sjhb if (!msi_enabled) 325164265Sjhb return (ENXIO); 326164265Sjhb 327195249Sjhb if (count > 1) 328195249Sjhb mirqs = malloc(count * sizeof(*mirqs), M_MSI, M_WAITOK); 329195249Sjhb else 330195249Sjhb mirqs = NULL; 331169391Sjhbagain: 332169391Sjhb mtx_lock(&msi_lock); 333164265Sjhb 334164265Sjhb /* Try to find 'count' free IRQs. */ 335164265Sjhb cnt = 0; 336164265Sjhb for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) { 337164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(i); 338164265Sjhb 339164265Sjhb /* End of allocated sources, so break. */ 340164265Sjhb if (msi == NULL) 341164265Sjhb break; 342164265Sjhb 343164265Sjhb /* If this is a free one, save its IRQ in the array. */ 344164265Sjhb if (msi->msi_dev == NULL) { 345164265Sjhb irqs[cnt] = i; 346164265Sjhb cnt++; 347164265Sjhb if (cnt == count) 348164265Sjhb break; 349164265Sjhb } 350164265Sjhb } 351164265Sjhb 352164265Sjhb /* Do we need to create some new sources? */ 353164265Sjhb if (cnt < count) { 354164265Sjhb /* If we would exceed the max, give up. */ 355164265Sjhb if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) { 356169391Sjhb mtx_unlock(&msi_lock); 357195249Sjhb free(mirqs, M_MSI); 358164265Sjhb return (ENXIO); 359164265Sjhb } 360169391Sjhb mtx_unlock(&msi_lock); 361164265Sjhb 362169391Sjhb /* We need count - cnt more sources. */ 363169391Sjhb while (cnt < count) { 364169391Sjhb msi_create_source(); 365164265Sjhb cnt++; 366164265Sjhb } 367169391Sjhb goto again; 368164265Sjhb } 369164265Sjhb 370164265Sjhb /* Ok, we now have the IRQs allocated. */ 371164265Sjhb KASSERT(cnt == count, ("count mismatch")); 372164265Sjhb 373194985Sjhb /* Allocate 'count' IDT vectors. */ 374194985Sjhb cpu = intr_next_cpu(); 375194985Sjhb vector = apic_alloc_vectors(cpu, irqs, count, maxcount); 376194985Sjhb if (vector == 0) { 377194985Sjhb mtx_unlock(&msi_lock); 378195249Sjhb free(mirqs, M_MSI); 379194985Sjhb return (ENOSPC); 380194985Sjhb } 381194985Sjhb 382164265Sjhb /* Assign IDT vectors and make these messages owned by 'dev'. */ 383164265Sjhb fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]); 384164265Sjhb for (i = 0; i < count; i++) { 385164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); 386194985Sjhb msi->msi_cpu = cpu; 387164265Sjhb msi->msi_dev = dev; 388194985Sjhb msi->msi_vector = vector + i; 389194985Sjhb if (bootverbose) 390194985Sjhb printf( 391194985Sjhb "msi: routing MSI IRQ %d to local APIC %u vector %u\n", 392194985Sjhb msi->msi_irq, msi->msi_cpu, msi->msi_vector); 393164265Sjhb msi->msi_first = fsrc; 394169391Sjhb KASSERT(msi->msi_intsrc.is_handlers == 0, 395169391Sjhb ("dead MSI has handlers")); 396164265Sjhb } 397164265Sjhb fsrc->msi_count = count; 398195249Sjhb fsrc->msi_maxcount = maxcount; 399195249Sjhb if (count > 1) 400195249Sjhb bcopy(irqs, mirqs, count * sizeof(*mirqs)); 401195249Sjhb fsrc->msi_irqs = mirqs; 402169391Sjhb mtx_unlock(&msi_lock); 403164265Sjhb 404164265Sjhb return (0); 405164265Sjhb} 406164265Sjhb 407164265Sjhbint 408164265Sjhbmsi_release(int *irqs, int count) 409164265Sjhb{ 410164265Sjhb struct msi_intsrc *msi, *first; 411164265Sjhb int i; 412164265Sjhb 413169391Sjhb mtx_lock(&msi_lock); 414164265Sjhb first = (struct msi_intsrc *)intr_lookup_source(irqs[0]); 415164265Sjhb if (first == NULL) { 416169391Sjhb mtx_unlock(&msi_lock); 417164265Sjhb return (ENOENT); 418164265Sjhb } 419164265Sjhb 420164265Sjhb /* Make sure this isn't an MSI-X message. */ 421164265Sjhb if (first->msi_msix) { 422169391Sjhb mtx_unlock(&msi_lock); 423164265Sjhb return (EINVAL); 424164265Sjhb } 425164265Sjhb 426164265Sjhb /* Make sure this message is allocated to a group. */ 427164265Sjhb if (first->msi_first == NULL) { 428169391Sjhb mtx_unlock(&msi_lock); 429164265Sjhb return (ENXIO); 430164265Sjhb } 431164265Sjhb 432164265Sjhb /* 433164265Sjhb * Make sure this is the start of a group and that we are releasing 434164265Sjhb * the entire group. 435164265Sjhb */ 436164265Sjhb if (first->msi_first != first || first->msi_count != count) { 437169391Sjhb mtx_unlock(&msi_lock); 438164265Sjhb return (EINVAL); 439164265Sjhb } 440164265Sjhb KASSERT(first->msi_dev != NULL, ("unowned group")); 441164265Sjhb 442164265Sjhb /* Clear all the extra messages in the group. */ 443164265Sjhb for (i = 1; i < count; i++) { 444164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); 445164265Sjhb KASSERT(msi->msi_first == first, ("message not in group")); 446164265Sjhb KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch")); 447164265Sjhb msi->msi_first = NULL; 448164265Sjhb msi->msi_dev = NULL; 449194985Sjhb apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); 450164265Sjhb msi->msi_vector = 0; 451164265Sjhb } 452164265Sjhb 453164265Sjhb /* Clear out the first message. */ 454164265Sjhb first->msi_first = NULL; 455164265Sjhb first->msi_dev = NULL; 456194985Sjhb apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq); 457164265Sjhb first->msi_vector = 0; 458164265Sjhb first->msi_count = 0; 459195249Sjhb first->msi_maxcount = 0; 460195249Sjhb free(first->msi_irqs, M_MSI); 461195249Sjhb first->msi_irqs = NULL; 462164265Sjhb 463169391Sjhb mtx_unlock(&msi_lock); 464164265Sjhb return (0); 465164265Sjhb} 466164265Sjhb 467164265Sjhbint 468169221Sjhbmsi_map(int irq, uint64_t *addr, uint32_t *data) 469164265Sjhb{ 470164265Sjhb struct msi_intsrc *msi; 471169221Sjhb 472169391Sjhb mtx_lock(&msi_lock); 473169221Sjhb msi = (struct msi_intsrc *)intr_lookup_source(irq); 474169221Sjhb if (msi == NULL) { 475169391Sjhb mtx_unlock(&msi_lock); 476169221Sjhb return (ENOENT); 477169221Sjhb } 478169221Sjhb 479169221Sjhb /* Make sure this message is allocated to a device. */ 480169221Sjhb if (msi->msi_dev == NULL) { 481169391Sjhb mtx_unlock(&msi_lock); 482169221Sjhb return (ENXIO); 483169221Sjhb } 484169221Sjhb 485169221Sjhb /* 486169221Sjhb * If this message isn't an MSI-X message, make sure it's part 487169391Sjhb * of a group, and switch to the first message in the 488169221Sjhb * group. 489169221Sjhb */ 490169221Sjhb if (!msi->msi_msix) { 491169221Sjhb if (msi->msi_first == NULL) { 492169391Sjhb mtx_unlock(&msi_lock); 493169221Sjhb return (ENXIO); 494169221Sjhb } 495169221Sjhb msi = msi->msi_first; 496169221Sjhb } 497169221Sjhb 498169221Sjhb *addr = INTEL_ADDR(msi); 499169221Sjhb *data = INTEL_DATA(msi); 500169391Sjhb mtx_unlock(&msi_lock); 501169221Sjhb return (0); 502169221Sjhb} 503169221Sjhb 504169221Sjhbint 505169391Sjhbmsix_alloc(device_t dev, int *irq) 506169221Sjhb{ 507169221Sjhb struct msi_intsrc *msi; 508194985Sjhb u_int cpu; 509194985Sjhb int i, vector; 510164265Sjhb 511164265Sjhb if (!msi_enabled) 512164265Sjhb return (ENXIO); 513164265Sjhb 514169391Sjhbagain: 515169391Sjhb mtx_lock(&msi_lock); 516164265Sjhb 517164265Sjhb /* Find a free IRQ. */ 518164265Sjhb for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) { 519164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(i); 520164265Sjhb 521164265Sjhb /* End of allocated sources, so break. */ 522164265Sjhb if (msi == NULL) 523164265Sjhb break; 524164265Sjhb 525169391Sjhb /* Stop at the first free source. */ 526164265Sjhb if (msi->msi_dev == NULL) 527164265Sjhb break; 528164265Sjhb } 529164265Sjhb 530164265Sjhb /* Do we need to create a new source? */ 531164265Sjhb if (msi == NULL) { 532164265Sjhb /* If we would exceed the max, give up. */ 533164265Sjhb if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) { 534169391Sjhb mtx_unlock(&msi_lock); 535164265Sjhb return (ENXIO); 536164265Sjhb } 537169391Sjhb mtx_unlock(&msi_lock); 538164265Sjhb 539164265Sjhb /* Create a new source. */ 540169391Sjhb msi_create_source(); 541169391Sjhb goto again; 542164265Sjhb } 543164265Sjhb 544194985Sjhb /* Allocate an IDT vector. */ 545194985Sjhb cpu = intr_next_cpu(); 546194985Sjhb vector = apic_alloc_vector(cpu, i); 547195249Sjhb if (vector == 0) { 548195249Sjhb mtx_unlock(&msi_lock); 549195249Sjhb return (ENOSPC); 550195249Sjhb } 551194985Sjhb if (bootverbose) 552194985Sjhb printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n", 553194985Sjhb msi->msi_irq, cpu, vector); 554195249Sjhb 555164265Sjhb /* Setup source. */ 556194985Sjhb msi->msi_cpu = cpu; 557164265Sjhb msi->msi_dev = dev; 558195249Sjhb msi->msi_first = msi; 559194985Sjhb msi->msi_vector = vector; 560164265Sjhb msi->msi_msix = 1; 561195249Sjhb msi->msi_count = 1; 562195249Sjhb msi->msi_maxcount = 1; 563195249Sjhb msi->msi_irqs = NULL; 564164265Sjhb 565169391Sjhb KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers")); 566169391Sjhb mtx_unlock(&msi_lock); 567164265Sjhb 568164265Sjhb *irq = i; 569164265Sjhb return (0); 570164265Sjhb} 571164265Sjhb 572164265Sjhbint 573164265Sjhbmsix_release(int irq) 574164265Sjhb{ 575164265Sjhb struct msi_intsrc *msi; 576164265Sjhb 577169391Sjhb mtx_lock(&msi_lock); 578164265Sjhb msi = (struct msi_intsrc *)intr_lookup_source(irq); 579164265Sjhb if (msi == NULL) { 580169391Sjhb mtx_unlock(&msi_lock); 581164265Sjhb return (ENOENT); 582164265Sjhb } 583164265Sjhb 584164265Sjhb /* Make sure this is an MSI-X message. */ 585164265Sjhb if (!msi->msi_msix) { 586169391Sjhb mtx_unlock(&msi_lock); 587164265Sjhb return (EINVAL); 588164265Sjhb } 589164265Sjhb 590164265Sjhb KASSERT(msi->msi_dev != NULL, ("unowned message")); 591164265Sjhb 592164265Sjhb /* Clear out the message. */ 593195249Sjhb msi->msi_first = NULL; 594164265Sjhb msi->msi_dev = NULL; 595194985Sjhb apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); 596164265Sjhb msi->msi_vector = 0; 597164265Sjhb msi->msi_msix = 0; 598195249Sjhb msi->msi_count = 0; 599195249Sjhb msi->msi_maxcount = 0; 600164265Sjhb 601169391Sjhb mtx_unlock(&msi_lock); 602164265Sjhb return (0); 603164265Sjhb} 604