iommu.c revision 306471 
1/*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/11/sys/amd64/vmm/io/iommu.c 306471 2016-09-30 01:39:18Z jhb $
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/11/sys/amd64/vmm/io/iommu.c 306471 2016-09-30 01:39:18Z jhb $");
31
32#include <sys/param.h>
33#include <sys/types.h>
34#include <sys/systm.h>
35#include <sys/bus.h>
36#include <sys/sysctl.h>
37
38#include <dev/pci/pcivar.h>
39#include <dev/pci/pcireg.h>
40
41#include <machine/cpu.h>
42#include <machine/md_var.h>
43
44#include "vmm_util.h"
45#include "vmm_mem.h"
46#include "iommu.h"
47
48SYSCTL_DECL(_hw_vmm);
49SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters");
50
51static int iommu_avail;
52SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail,
53    0, "bhyve iommu initialized?");
54
55static int iommu_enable = 1;
56SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
57    "Enable use of I/O MMU (required for PCI passthrough).");
58
59static struct iommu_ops *ops;
60static void *host_domain;
61static eventhandler_tag add_tag, delete_tag;
62
63static __inline int
64IOMMU_INIT(void)
65{
66	if (ops != NULL)
67		return ((*ops->init)());
68	else
69		return (ENXIO);
70}
71
72static __inline void
73IOMMU_CLEANUP(void)
74{
75	if (ops != NULL && iommu_avail)
76		(*ops->cleanup)();
77}
78
79static __inline void *
80IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
81{
82
83	if (ops != NULL && iommu_avail)
84		return ((*ops->create_domain)(maxaddr));
85	else
86		return (NULL);
87}
88
89static __inline void
90IOMMU_DESTROY_DOMAIN(void *dom)
91{
92
93	if (ops != NULL && iommu_avail)
94		(*ops->destroy_domain)(dom);
95}
96
97static __inline uint64_t
98IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
99{
100
101	if (ops != NULL && iommu_avail)
102		return ((*ops->create_mapping)(domain, gpa, hpa, len));
103	else
104		return (len);		/* XXX */
105}
106
107static __inline uint64_t
108IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len)
109{
110
111	if (ops != NULL && iommu_avail)
112		return ((*ops->remove_mapping)(domain, gpa, len));
113	else
114		return (len);		/* XXX */
115}
116
117static __inline void
118IOMMU_ADD_DEVICE(void *domain, uint16_t rid)
119{
120
121	if (ops != NULL && iommu_avail)
122		(*ops->add_device)(domain, rid);
123}
124
125static __inline void
126IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid)
127{
128
129	if (ops != NULL && iommu_avail)
130		(*ops->remove_device)(domain, rid);
131}
132
133static __inline void
134IOMMU_INVALIDATE_TLB(void *domain)
135{
136
137	if (ops != NULL && iommu_avail)
138		(*ops->invalidate_tlb)(domain);
139}
140
141static __inline void
142IOMMU_ENABLE(void)
143{
144
145	if (ops != NULL && iommu_avail)
146		(*ops->enable)();
147}
148
149static __inline void
150IOMMU_DISABLE(void)
151{
152
153	if (ops != NULL && iommu_avail)
154		(*ops->disable)();
155}
156
157static void
158iommu_pci_add(void *arg, device_t dev)
159{
160
161	/* Add new devices to the host domain. */
162	iommu_add_device(host_domain, pci_get_rid(dev));
163}
164
165static void
166iommu_pci_delete(void *arg, device_t dev)
167{
168
169	iommu_remove_device(host_domain, pci_get_rid(dev));
170}
171
172static void
173iommu_init(void)
174{
175	int error, bus, slot, func;
176	vm_paddr_t maxaddr;
177	device_t dev;
178
179	if (!iommu_enable)
180		return;
181
182	if (vmm_is_intel())
183		ops = &iommu_ops_intel;
184	else if (vmm_is_amd())
185		ops = &iommu_ops_amd;
186	else
187		ops = NULL;
188
189	error = IOMMU_INIT();
190	if (error)
191		return;
192
193	iommu_avail = 1;
194
195	/*
196	 * Create a domain for the devices owned by the host
197	 */
198	maxaddr = vmm_mem_maxaddr();
199	host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
200	if (host_domain == NULL) {
201		printf("iommu_init: unable to create a host domain");
202		IOMMU_CLEANUP();
203		ops = NULL;
204		iommu_avail = 0;
205		return;
206	}
207
208	/*
209	 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
210	 * the host
211	 */
212	iommu_create_mapping(host_domain, 0, 0, maxaddr);
213
214	add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0);
215	delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete,
216	    NULL, 0);
217	for (bus = 0; bus <= PCI_BUSMAX; bus++) {
218		for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
219			for (func = 0; func <= PCI_FUNCMAX; func++) {
220				dev = pci_find_dbsf(0, bus, slot, func);
221				if (dev == NULL)
222					continue;
223
224				/* Everything belongs to the host domain. */
225				iommu_add_device(host_domain,
226				    pci_get_rid(dev));
227			}
228		}
229	}
230	IOMMU_ENABLE();
231
232}
233
234void
235iommu_cleanup(void)
236{
237
238	if (add_tag != NULL) {
239		EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
240		add_tag = NULL;
241	}
242	if (delete_tag != NULL) {
243		EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
244		delete_tag = NULL;
245	}
246	IOMMU_DISABLE();
247	IOMMU_DESTROY_DOMAIN(host_domain);
248	IOMMU_CLEANUP();
249}
250
251void *
252iommu_create_domain(vm_paddr_t maxaddr)
253{
254	static volatile int iommu_initted;
255
256	if (iommu_initted < 2) {
257		if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
258			iommu_init();
259			atomic_store_rel_int(&iommu_initted, 2);
260		} else
261			while (iommu_initted == 1)
262				cpu_spinwait();
263	}
264	return (IOMMU_CREATE_DOMAIN(maxaddr));
265}
266
267void
268iommu_destroy_domain(void *dom)
269{
270
271	IOMMU_DESTROY_DOMAIN(dom);
272}
273
274void
275iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
276{
277	uint64_t mapped, remaining;
278
279	remaining = len;
280
281	while (remaining > 0) {
282		mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining);
283		gpa += mapped;
284		hpa += mapped;
285		remaining -= mapped;
286	}
287}
288
289void
290iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
291{
292	uint64_t unmapped, remaining;
293
294	remaining = len;
295
296	while (remaining > 0) {
297		unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining);
298		gpa += unmapped;
299		remaining -= unmapped;
300	}
301}
302
303void *
304iommu_host_domain(void)
305{
306
307	return (host_domain);
308}
309
310void
311iommu_add_device(void *dom, uint16_t rid)
312{
313
314	IOMMU_ADD_DEVICE(dom, rid);
315}
316
317void
318iommu_remove_device(void *dom, uint16_t rid)
319{
320
321	IOMMU_REMOVE_DEVICE(dom, rid);
322}
323
324void
325iommu_invalidate_tlb(void *domain)
326{
327
328	IOMMU_INVALIDATE_TLB(domain);
329}
330