1/*-
2 * Copyright (c) 2006 Yahoo!, Inc.
3 * All rights reserved.
4 * Written by: John Baldwin <jhb@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. Neither the name of the author nor the names of any co-contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31/*
32 * Support for PCI Message Signalled Interrupts (MSI).  MSI interrupts on
33 * x86 are basically APIC messages that the northbridge delivers directly
34 * to the local APICs as if they had come from an I/O APIC.
35 */
36
37#include <sys/cdefs.h>
38__FBSDID("$FreeBSD: stable/11/sys/x86/x86/msi.c 344912 2019-03-08 01:04:19Z jhb $");
39
40#include "opt_acpi.h"
41
42#include <sys/param.h>
43#include <sys/bus.h>
44#include <sys/kernel.h>
45#include <sys/limits.h>
46#include <sys/lock.h>
47#include <sys/malloc.h>
48#include <sys/mutex.h>
49#include <sys/sx.h>
50#include <sys/sysctl.h>
51#include <sys/systm.h>
52#include <x86/apicreg.h>
53#include <machine/cputypes.h>
54#include <machine/md_var.h>
55#include <machine/frame.h>
56#include <machine/intr_machdep.h>
57#include <x86/apicvar.h>
58#include <x86/iommu/iommu_intrmap.h>
59#include <machine/specialreg.h>
60#include <dev/pci/pcivar.h>
61
62/* Fields in address for Intel MSI messages. */
63#define	MSI_INTEL_ADDR_DEST		0x000ff000
64#define	MSI_INTEL_ADDR_RH		0x00000008
65# define MSI_INTEL_ADDR_RH_ON		0x00000008
66# define MSI_INTEL_ADDR_RH_OFF		0x00000000
67#define	MSI_INTEL_ADDR_DM		0x00000004
68# define MSI_INTEL_ADDR_DM_PHYSICAL	0x00000000
69# define MSI_INTEL_ADDR_DM_LOGICAL	0x00000004
70
71/* Fields in data for Intel MSI messages. */
72#define	MSI_INTEL_DATA_TRGRMOD		IOART_TRGRMOD	/* Trigger mode. */
73# define MSI_INTEL_DATA_TRGREDG		IOART_TRGREDG
74# define MSI_INTEL_DATA_TRGRLVL		IOART_TRGRLVL
75#define	MSI_INTEL_DATA_LEVEL		0x00004000	/* Polarity. */
76# define MSI_INTEL_DATA_DEASSERT	0x00000000
77# define MSI_INTEL_DATA_ASSERT		0x00004000
78#define	MSI_INTEL_DATA_DELMOD		IOART_DELMOD	/* Delivery mode. */
79# define MSI_INTEL_DATA_DELFIXED	IOART_DELFIXED
80# define MSI_INTEL_DATA_DELLOPRI	IOART_DELLOPRI
81# define MSI_INTEL_DATA_DELSMI		IOART_DELSMI
82# define MSI_INTEL_DATA_DELNMI		IOART_DELNMI
83# define MSI_INTEL_DATA_DELINIT		IOART_DELINIT
84# define MSI_INTEL_DATA_DELEXINT	IOART_DELEXINT
85#define	MSI_INTEL_DATA_INTVEC		IOART_INTVEC	/* Interrupt vector. */
86
87/*
88 * Build Intel MSI message and data values from a source.  AMD64 systems
89 * seem to be compatible, so we use the same function for both.
90 */
91#define	INTEL_ADDR(msi)							\
92	(MSI_INTEL_ADDR_BASE | (msi)->msi_cpu << 12 |			\
93	    MSI_INTEL_ADDR_RH_OFF | MSI_INTEL_ADDR_DM_PHYSICAL)
94#define	INTEL_DATA(msi)							\
95	(MSI_INTEL_DATA_TRGREDG | MSI_INTEL_DATA_DELFIXED | (msi)->msi_vector)
96
97static MALLOC_DEFINE(M_MSI, "msi", "PCI MSI");
98
99/*
100 * MSI sources are bunched into groups.  This is because MSI forces
101 * all of the messages to share the address and data registers and
102 * thus certain properties (such as the local APIC ID target on x86).
103 * Each group has a 'first' source that contains information global to
104 * the group.  These fields are marked with (g) below.
105 *
106 * Note that local APIC ID is kind of special.  Each message will be
107 * assigned an ID by the system; however, a group will use the ID from
108 * the first message.
109 *
110 * For MSI-X, each message is isolated.
111 */
112struct msi_intsrc {
113	struct intsrc msi_intsrc;
114	device_t msi_dev;		/* Owning device. (g) */
115	struct msi_intsrc *msi_first;	/* First source in group. */
116	u_int msi_irq;			/* IRQ cookie. */
117	u_int msi_msix;			/* MSI-X message. */
118	u_int msi_vector:8;		/* IDT vector. */
119	u_int msi_cpu;			/* Local APIC ID. (g) */
120	u_int msi_count:8;		/* Messages in this group. (g) */
121	u_int msi_maxcount:8;		/* Alignment for this group. (g) */
122	u_int *msi_irqs;		/* Group's IRQ list. (g) */
123	u_int msi_remap_cookie;
124};
125
126static void	msi_create_source(void);
127static void	msi_enable_source(struct intsrc *isrc);
128static void	msi_disable_source(struct intsrc *isrc, int eoi);
129static void	msi_eoi_source(struct intsrc *isrc);
130static void	msi_enable_intr(struct intsrc *isrc);
131static void	msi_disable_intr(struct intsrc *isrc);
132static int	msi_vector(struct intsrc *isrc);
133static int	msi_source_pending(struct intsrc *isrc);
134static int	msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
135		    enum intr_polarity pol);
136static int	msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
137
138struct pic msi_pic = {
139	.pic_enable_source = msi_enable_source,
140	.pic_disable_source = msi_disable_source,
141	.pic_eoi_source = msi_eoi_source,
142	.pic_enable_intr = msi_enable_intr,
143	.pic_disable_intr = msi_disable_intr,
144	.pic_vector = msi_vector,
145	.pic_source_pending = msi_source_pending,
146	.pic_suspend = NULL,
147	.pic_resume = NULL,
148	.pic_config_intr = msi_config_intr,
149	.pic_assign_cpu = msi_assign_cpu,
150	.pic_reprogram_pin = NULL,
151};
152
153u_int first_msi_irq;
154
155u_int num_msi_irqs = 512;
156SYSCTL_UINT(_machdep, OID_AUTO, num_msi_irqs, CTLFLAG_RDTUN, &num_msi_irqs, 0,
157    "Number of IRQs reserved for MSI and MSI-X interrupts");
158
159#ifdef SMP
160/**
161 * Xen hypervisors prior to 4.6.0 do not properly handle updates to
162 * enabled MSI-X table entries.  Allow migration of MSI-X interrupts
163 * to be disabled via a tunable. Values have the following meaning:
164 *
165 * -1: automatic detection by FreeBSD
166 *  0: enable migration
167 *  1: disable migration
168 */
169int msix_disable_migration = -1;
170SYSCTL_INT(_machdep, OID_AUTO, disable_msix_migration, CTLFLAG_RDTUN,
171    &msix_disable_migration, 0,
172    "Disable migration of MSI-X interrupts between CPUs");
173#endif
174
175static int msi_enabled;
176static u_int msi_last_irq;
177static struct mtx msi_lock;
178
179static void
180msi_enable_source(struct intsrc *isrc)
181{
182}
183
184static void
185msi_disable_source(struct intsrc *isrc, int eoi)
186{
187
188	if (eoi == PIC_EOI)
189		lapic_eoi();
190}
191
192static void
193msi_eoi_source(struct intsrc *isrc)
194{
195
196	lapic_eoi();
197}
198
199static void
200msi_enable_intr(struct intsrc *isrc)
201{
202	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
203
204	apic_enable_vector(msi->msi_cpu, msi->msi_vector);
205}
206
207static void
208msi_disable_intr(struct intsrc *isrc)
209{
210	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
211
212	apic_disable_vector(msi->msi_cpu, msi->msi_vector);
213}
214
215static int
216msi_vector(struct intsrc *isrc)
217{
218	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
219
220	return (msi->msi_irq);
221}
222
223static int
224msi_source_pending(struct intsrc *isrc)
225{
226
227	return (0);
228}
229
230static int
231msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
232    enum intr_polarity pol)
233{
234
235	return (ENODEV);
236}
237
238static int
239msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
240{
241	struct msi_intsrc *sib, *msi = (struct msi_intsrc *)isrc;
242	int old_vector;
243	u_int old_id;
244	int i, vector;
245
246	/*
247	 * Only allow CPUs to be assigned to the first message for an
248	 * MSI group.
249	 */
250	if (msi->msi_first != msi)
251		return (EINVAL);
252
253#ifdef SMP
254	if (msix_disable_migration && msi->msi_msix)
255		return (EINVAL);
256#endif
257
258	/* Store information to free existing irq. */
259	old_vector = msi->msi_vector;
260	old_id = msi->msi_cpu;
261	if (old_id == apic_id)
262		return (0);
263
264	/* Allocate IDT vectors on this cpu. */
265	if (msi->msi_count > 1) {
266		KASSERT(msi->msi_msix == 0, ("MSI-X message group"));
267		vector = apic_alloc_vectors(apic_id, msi->msi_irqs,
268		    msi->msi_count, msi->msi_maxcount);
269	} else
270		vector = apic_alloc_vector(apic_id, msi->msi_irq);
271	if (vector == 0)
272		return (ENOSPC);
273
274	msi->msi_cpu = apic_id;
275	msi->msi_vector = vector;
276	if (msi->msi_intsrc.is_handlers > 0)
277		apic_enable_vector(msi->msi_cpu, msi->msi_vector);
278	if (bootverbose)
279		printf("msi: Assigning %s IRQ %d to local APIC %u vector %u\n",
280		    msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
281		    msi->msi_cpu, msi->msi_vector);
282	for (i = 1; i < msi->msi_count; i++) {
283		sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]);
284		sib->msi_cpu = apic_id;
285		sib->msi_vector = vector + i;
286		if (sib->msi_intsrc.is_handlers > 0)
287			apic_enable_vector(sib->msi_cpu, sib->msi_vector);
288		if (bootverbose)
289			printf(
290		    "msi: Assigning MSI IRQ %d to local APIC %u vector %u\n",
291			    sib->msi_irq, sib->msi_cpu, sib->msi_vector);
292	}
293	BUS_REMAP_INTR(device_get_parent(msi->msi_dev), msi->msi_dev,
294	    msi->msi_irq);
295
296	/*
297	 * Free the old vector after the new one is established.  This is done
298	 * to prevent races where we could miss an interrupt.
299	 */
300	if (msi->msi_intsrc.is_handlers > 0)
301		apic_disable_vector(old_id, old_vector);
302	apic_free_vector(old_id, old_vector, msi->msi_irq);
303	for (i = 1; i < msi->msi_count; i++) {
304		sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]);
305		if (sib->msi_intsrc.is_handlers > 0)
306			apic_disable_vector(old_id, old_vector + i);
307		apic_free_vector(old_id, old_vector + i, msi->msi_irqs[i]);
308	}
309	return (0);
310}
311
312void
313msi_init(void)
314{
315
316	/* Check if we have a supported CPU. */
317	switch (cpu_vendor_id) {
318	case CPU_VENDOR_INTEL:
319	case CPU_VENDOR_AMD:
320		break;
321	case CPU_VENDOR_CENTAUR:
322		if (CPUID_TO_FAMILY(cpu_id) == 0x6 &&
323		    CPUID_TO_MODEL(cpu_id) >= 0xf)
324			break;
325		/* FALLTHROUGH */
326	default:
327		return;
328	}
329
330#ifdef SMP
331	if (msix_disable_migration == -1) {
332		/* The default is to allow migration of MSI-X interrupts. */
333		msix_disable_migration = 0;
334	}
335#endif
336
337	if (num_msi_irqs == 0)
338		return;
339
340	first_msi_irq = max(MINIMUM_MSI_INT, num_io_irqs);
341	if (num_msi_irqs > UINT_MAX - first_msi_irq)
342		panic("num_msi_irqs too high");
343	num_io_irqs = first_msi_irq + num_msi_irqs;
344
345	msi_enabled = 1;
346	intr_register_pic(&msi_pic);
347	mtx_init(&msi_lock, "msi", NULL, MTX_DEF);
348}
349
350static void
351msi_create_source(void)
352{
353	struct msi_intsrc *msi;
354	u_int irq;
355
356	mtx_lock(&msi_lock);
357	if (msi_last_irq >= num_msi_irqs) {
358		mtx_unlock(&msi_lock);
359		return;
360	}
361	irq = msi_last_irq + first_msi_irq;
362	msi_last_irq++;
363	mtx_unlock(&msi_lock);
364
365	msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);
366	msi->msi_intsrc.is_pic = &msi_pic;
367	msi->msi_irq = irq;
368	intr_register_source(&msi->msi_intsrc);
369	nexus_add_irq(irq);
370}
371
372/*
373 * Try to allocate 'count' interrupt sources with contiguous IDT values.
374 */
375int
376msi_alloc(device_t dev, int count, int maxcount, int *irqs)
377{
378	struct msi_intsrc *msi, *fsrc;
379	u_int cpu, *mirqs;
380	int cnt, i, vector;
381#ifdef ACPI_DMAR
382	u_int cookies[count];
383	int error;
384#endif
385
386	if (!msi_enabled)
387		return (ENXIO);
388
389	if (count > 1)
390		mirqs = malloc(count * sizeof(*mirqs), M_MSI, M_WAITOK);
391	else
392		mirqs = NULL;
393again:
394	mtx_lock(&msi_lock);
395
396	/* Try to find 'count' free IRQs. */
397	cnt = 0;
398	for (i = first_msi_irq; i < first_msi_irq + num_msi_irqs; i++) {
399		msi = (struct msi_intsrc *)intr_lookup_source(i);
400
401		/* End of allocated sources, so break. */
402		if (msi == NULL)
403			break;
404
405		/* If this is a free one, save its IRQ in the array. */
406		if (msi->msi_dev == NULL) {
407			irqs[cnt] = i;
408			cnt++;
409			if (cnt == count)
410				break;
411		}
412	}
413
414	/* Do we need to create some new sources? */
415	if (cnt < count) {
416		/* If we would exceed the max, give up. */
417		if (i + (count - cnt) > first_msi_irq + num_msi_irqs) {
418			mtx_unlock(&msi_lock);
419			free(mirqs, M_MSI);
420			return (ENXIO);
421		}
422		mtx_unlock(&msi_lock);
423
424		/* We need count - cnt more sources. */
425		while (cnt < count) {
426			msi_create_source();
427			cnt++;
428		}
429		goto again;
430	}
431
432	/* Ok, we now have the IRQs allocated. */
433	KASSERT(cnt == count, ("count mismatch"));
434
435	/* Allocate 'count' IDT vectors. */
436	cpu = intr_next_cpu();
437	vector = apic_alloc_vectors(cpu, irqs, count, maxcount);
438	if (vector == 0) {
439		mtx_unlock(&msi_lock);
440		free(mirqs, M_MSI);
441		return (ENOSPC);
442	}
443
444#ifdef ACPI_DMAR
445	mtx_unlock(&msi_lock);
446	error = iommu_alloc_msi_intr(dev, cookies, count);
447	mtx_lock(&msi_lock);
448	if (error == EOPNOTSUPP)
449		error = 0;
450	if (error != 0) {
451		for (i = 0; i < count; i++)
452			apic_free_vector(cpu, vector + i, irqs[i]);
453		free(mirqs, M_MSI);
454		return (error);
455	}
456	for (i = 0; i < count; i++) {
457		msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
458		msi->msi_remap_cookie = cookies[i];
459	}
460#endif
461
462	/* Assign IDT vectors and make these messages owned by 'dev'. */
463	fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
464	for (i = 0; i < count; i++) {
465		msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
466		msi->msi_cpu = cpu;
467		msi->msi_dev = dev;
468		msi->msi_vector = vector + i;
469		if (bootverbose)
470			printf(
471		    "msi: routing MSI IRQ %d to local APIC %u vector %u\n",
472			    msi->msi_irq, msi->msi_cpu, msi->msi_vector);
473		msi->msi_first = fsrc;
474		KASSERT(msi->msi_intsrc.is_handlers == 0,
475		    ("dead MSI has handlers"));
476	}
477	fsrc->msi_count = count;
478	fsrc->msi_maxcount = maxcount;
479	if (count > 1)
480		bcopy(irqs, mirqs, count * sizeof(*mirqs));
481	fsrc->msi_irqs = mirqs;
482	mtx_unlock(&msi_lock);
483	return (0);
484}
485
486int
487msi_release(int *irqs, int count)
488{
489	struct msi_intsrc *msi, *first;
490	int i;
491
492	mtx_lock(&msi_lock);
493	first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
494	if (first == NULL) {
495		mtx_unlock(&msi_lock);
496		return (ENOENT);
497	}
498
499	/* Make sure this isn't an MSI-X message. */
500	if (first->msi_msix) {
501		mtx_unlock(&msi_lock);
502		return (EINVAL);
503	}
504
505	/* Make sure this message is allocated to a group. */
506	if (first->msi_first == NULL) {
507		mtx_unlock(&msi_lock);
508		return (ENXIO);
509	}
510
511	/*
512	 * Make sure this is the start of a group and that we are releasing
513	 * the entire group.
514	 */
515	if (first->msi_first != first || first->msi_count != count) {
516		mtx_unlock(&msi_lock);
517		return (EINVAL);
518	}
519	KASSERT(first->msi_dev != NULL, ("unowned group"));
520
521	/* Clear all the extra messages in the group. */
522	for (i = 1; i < count; i++) {
523		msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
524		KASSERT(msi->msi_first == first, ("message not in group"));
525		KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
526#ifdef ACPI_DMAR
527		iommu_unmap_msi_intr(first->msi_dev, msi->msi_remap_cookie);
528#endif
529		msi->msi_first = NULL;
530		msi->msi_dev = NULL;
531		apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
532		msi->msi_vector = 0;
533	}
534
535	/* Clear out the first message. */
536#ifdef ACPI_DMAR
537	mtx_unlock(&msi_lock);
538	iommu_unmap_msi_intr(first->msi_dev, first->msi_remap_cookie);
539	mtx_lock(&msi_lock);
540#endif
541	first->msi_first = NULL;
542	first->msi_dev = NULL;
543	apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq);
544	first->msi_vector = 0;
545	first->msi_count = 0;
546	first->msi_maxcount = 0;
547	free(first->msi_irqs, M_MSI);
548	first->msi_irqs = NULL;
549
550	mtx_unlock(&msi_lock);
551	return (0);
552}
553
554int
555msi_map(int irq, uint64_t *addr, uint32_t *data)
556{
557	struct msi_intsrc *msi;
558	int error;
559#ifdef ACPI_DMAR
560	struct msi_intsrc *msi1;
561	int i, k;
562#endif
563
564	mtx_lock(&msi_lock);
565	msi = (struct msi_intsrc *)intr_lookup_source(irq);
566	if (msi == NULL) {
567		mtx_unlock(&msi_lock);
568		return (ENOENT);
569	}
570
571	/* Make sure this message is allocated to a device. */
572	if (msi->msi_dev == NULL) {
573		mtx_unlock(&msi_lock);
574		return (ENXIO);
575	}
576
577	/*
578	 * If this message isn't an MSI-X message, make sure it's part
579	 * of a group, and switch to the first message in the
580	 * group.
581	 */
582	if (!msi->msi_msix) {
583		if (msi->msi_first == NULL) {
584			mtx_unlock(&msi_lock);
585			return (ENXIO);
586		}
587		msi = msi->msi_first;
588	}
589
590#ifdef ACPI_DMAR
591	if (!msi->msi_msix) {
592		for (k = msi->msi_count - 1, i = first_msi_irq; k > 0 &&
593		    i < first_msi_irq + num_msi_irqs; i++) {
594			if (i == msi->msi_irq)
595				continue;
596			msi1 = (struct msi_intsrc *)intr_lookup_source(i);
597			if (!msi1->msi_msix && msi1->msi_first == msi) {
598				mtx_unlock(&msi_lock);
599				iommu_map_msi_intr(msi1->msi_dev,
600				    msi1->msi_cpu, msi1->msi_vector,
601				    msi1->msi_remap_cookie, NULL, NULL);
602				k--;
603				mtx_lock(&msi_lock);
604			}
605		}
606	}
607	mtx_unlock(&msi_lock);
608	error = iommu_map_msi_intr(msi->msi_dev, msi->msi_cpu,
609	    msi->msi_vector, msi->msi_remap_cookie, addr, data);
610#else
611	mtx_unlock(&msi_lock);
612	error = EOPNOTSUPP;
613#endif
614	if (error == EOPNOTSUPP) {
615		*addr = INTEL_ADDR(msi);
616		*data = INTEL_DATA(msi);
617		error = 0;
618	}
619	return (error);
620}
621
622int
623msix_alloc(device_t dev, int *irq)
624{
625	struct msi_intsrc *msi;
626	u_int cpu;
627	int i, vector;
628#ifdef ACPI_DMAR
629	u_int cookie;
630	int error;
631#endif
632
633	if (!msi_enabled)
634		return (ENXIO);
635
636again:
637	mtx_lock(&msi_lock);
638
639	/* Find a free IRQ. */
640	for (i = first_msi_irq; i < first_msi_irq + num_msi_irqs; i++) {
641		msi = (struct msi_intsrc *)intr_lookup_source(i);
642
643		/* End of allocated sources, so break. */
644		if (msi == NULL)
645			break;
646
647		/* Stop at the first free source. */
648		if (msi->msi_dev == NULL)
649			break;
650	}
651
652	/* Are all IRQs in use? */
653	if (i == first_msi_irq + num_msi_irqs) {
654		mtx_unlock(&msi_lock);
655		return (ENXIO);
656	}
657
658	/* Do we need to create a new source? */
659	if (msi == NULL) {
660		mtx_unlock(&msi_lock);
661
662		/* Create a new source. */
663		msi_create_source();
664		goto again;
665	}
666
667	/* Allocate an IDT vector. */
668	cpu = intr_next_cpu();
669	vector = apic_alloc_vector(cpu, i);
670	if (vector == 0) {
671		mtx_unlock(&msi_lock);
672		return (ENOSPC);
673	}
674
675	msi->msi_dev = dev;
676#ifdef ACPI_DMAR
677	mtx_unlock(&msi_lock);
678	error = iommu_alloc_msi_intr(dev, &cookie, 1);
679	mtx_lock(&msi_lock);
680	if (error == EOPNOTSUPP)
681		error = 0;
682	if (error != 0) {
683		msi->msi_dev = NULL;
684		apic_free_vector(cpu, vector, i);
685		return (error);
686	}
687	msi->msi_remap_cookie = cookie;
688#endif
689
690	if (bootverbose)
691		printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n",
692		    msi->msi_irq, cpu, vector);
693
694	/* Setup source. */
695	msi->msi_cpu = cpu;
696	msi->msi_first = msi;
697	msi->msi_vector = vector;
698	msi->msi_msix = 1;
699	msi->msi_count = 1;
700	msi->msi_maxcount = 1;
701	msi->msi_irqs = NULL;
702
703	KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
704	mtx_unlock(&msi_lock);
705
706	*irq = i;
707	return (0);
708}
709
710int
711msix_release(int irq)
712{
713	struct msi_intsrc *msi;
714
715	mtx_lock(&msi_lock);
716	msi = (struct msi_intsrc *)intr_lookup_source(irq);
717	if (msi == NULL) {
718		mtx_unlock(&msi_lock);
719		return (ENOENT);
720	}
721
722	/* Make sure this is an MSI-X message. */
723	if (!msi->msi_msix) {
724		mtx_unlock(&msi_lock);
725		return (EINVAL);
726	}
727
728	KASSERT(msi->msi_dev != NULL, ("unowned message"));
729
730	/* Clear out the message. */
731#ifdef ACPI_DMAR
732	mtx_unlock(&msi_lock);
733	iommu_unmap_msi_intr(msi->msi_dev, msi->msi_remap_cookie);
734	mtx_lock(&msi_lock);
735#endif
736	msi->msi_first = NULL;
737	msi->msi_dev = NULL;
738	apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
739	msi->msi_vector = 0;
740	msi->msi_msix = 0;
741	msi->msi_count = 0;
742	msi->msi_maxcount = 0;
743
744	mtx_unlock(&msi_lock);
745	return (0);
746}
747