iommu.c revision 325900 
1/*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/10/sys/amd64/vmm/io/iommu.c 325900 2017-11-16 18:22:03Z jhb $
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/iommu.c 325900 2017-11-16 18:22:03Z jhb $");
31
32#include <sys/param.h>
33#include <sys/types.h>
34#include <sys/systm.h>
35#include <sys/bus.h>
36#include <sys/sysctl.h>
37
38#include <dev/pci/pcivar.h>
39#include <dev/pci/pcireg.h>
40
41#include <machine/cpu.h>
42#include <machine/md_var.h>
43
44#include "vmm_util.h"
45#include "vmm_mem.h"
46#include "iommu.h"
47
48SYSCTL_DECL(_hw_vmm);
49SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters");
50
51static int iommu_avail;
52SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail,
53    0, "bhyve iommu initialized?");
54
55static int iommu_enable = 1;
56SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
57    "Enable use of I/O MMU (required for PCI passthrough).");
58
59static struct iommu_ops *ops;
60static void *host_domain;
61
62static __inline int
63IOMMU_INIT(void)
64{
65	if (ops != NULL)
66		return ((*ops->init)());
67	else
68		return (ENXIO);
69}
70
71static __inline void
72IOMMU_CLEANUP(void)
73{
74	if (ops != NULL && iommu_avail)
75		(*ops->cleanup)();
76}
77
78static __inline void *
79IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
80{
81
82	if (ops != NULL && iommu_avail)
83		return ((*ops->create_domain)(maxaddr));
84	else
85		return (NULL);
86}
87
88static __inline void
89IOMMU_DESTROY_DOMAIN(void *dom)
90{
91
92	if (ops != NULL && iommu_avail)
93		(*ops->destroy_domain)(dom);
94}
95
96static __inline uint64_t
97IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
98{
99
100	if (ops != NULL && iommu_avail)
101		return ((*ops->create_mapping)(domain, gpa, hpa, len));
102	else
103		return (len);		/* XXX */
104}
105
106static __inline uint64_t
107IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len)
108{
109
110	if (ops != NULL && iommu_avail)
111		return ((*ops->remove_mapping)(domain, gpa, len));
112	else
113		return (len);		/* XXX */
114}
115
116static __inline void
117IOMMU_ADD_DEVICE(void *domain, uint16_t rid)
118{
119
120	if (ops != NULL && iommu_avail)
121		(*ops->add_device)(domain, rid);
122}
123
124static __inline void
125IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid)
126{
127
128	if (ops != NULL && iommu_avail)
129		(*ops->remove_device)(domain, rid);
130}
131
132static __inline void
133IOMMU_INVALIDATE_TLB(void *domain)
134{
135
136	if (ops != NULL && iommu_avail)
137		(*ops->invalidate_tlb)(domain);
138}
139
140static __inline void
141IOMMU_ENABLE(void)
142{
143
144	if (ops != NULL && iommu_avail)
145		(*ops->enable)();
146}
147
148static __inline void
149IOMMU_DISABLE(void)
150{
151
152	if (ops != NULL && iommu_avail)
153		(*ops->disable)();
154}
155
156static void
157iommu_init(void)
158{
159	int error, bus, slot, func;
160	vm_paddr_t maxaddr;
161	devclass_t dc;
162	device_t dev;
163
164	if (!iommu_enable)
165		return;
166
167	if (vmm_is_intel())
168		ops = &iommu_ops_intel;
169	else if (vmm_is_amd())
170		ops = &iommu_ops_amd;
171	else
172		ops = NULL;
173
174	error = IOMMU_INIT();
175	if (error)
176		return;
177
178	iommu_avail = 1;
179
180	/*
181	 * Create a domain for the devices owned by the host
182	 */
183	maxaddr = vmm_mem_maxaddr();
184	host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
185	if (host_domain == NULL) {
186		printf("iommu_init: unable to create a host domain");
187		IOMMU_CLEANUP();
188		ops = NULL;
189		iommu_avail = 0;
190		return;
191	}
192
193	/*
194	 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
195	 * the host
196	 */
197	iommu_create_mapping(host_domain, 0, 0, maxaddr);
198
199	dc = devclass_find("ppt");
200	for (bus = 0; bus <= PCI_BUSMAX; bus++) {
201		for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
202			for (func = 0; func <= PCI_FUNCMAX; func++) {
203				dev = pci_find_dbsf(0, bus, slot, func);
204				if (dev == NULL)
205					continue;
206
207				/* Skip passthrough devices. */
208				if (dc != NULL &&
209				    device_get_devclass(dev) == dc)
210					continue;
211
212				/*
213				 * Everything else belongs to the host
214				 * domain.
215				 */
216				iommu_add_device(host_domain,
217				    pci_get_rid(dev));
218			}
219		}
220	}
221	IOMMU_ENABLE();
222
223}
224
225void
226iommu_cleanup(void)
227{
228	IOMMU_DISABLE();
229	IOMMU_DESTROY_DOMAIN(host_domain);
230	IOMMU_CLEANUP();
231}
232
233void *
234iommu_create_domain(vm_paddr_t maxaddr)
235{
236	static volatile int iommu_initted;
237
238	if (iommu_initted < 2) {
239		if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
240			iommu_init();
241			atomic_store_rel_int(&iommu_initted, 2);
242		} else
243			while (iommu_initted == 1)
244				cpu_spinwait();
245	}
246	return (IOMMU_CREATE_DOMAIN(maxaddr));
247}
248
249void
250iommu_destroy_domain(void *dom)
251{
252
253	IOMMU_DESTROY_DOMAIN(dom);
254}
255
256void
257iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
258{
259	uint64_t mapped, remaining;
260
261	remaining = len;
262
263	while (remaining > 0) {
264		mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining);
265		gpa += mapped;
266		hpa += mapped;
267		remaining -= mapped;
268	}
269}
270
271void
272iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
273{
274	uint64_t unmapped, remaining;
275
276	remaining = len;
277
278	while (remaining > 0) {
279		unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining);
280		gpa += unmapped;
281		remaining -= unmapped;
282	}
283}
284
285void *
286iommu_host_domain(void)
287{
288
289	return (host_domain);
290}
291
292void
293iommu_add_device(void *dom, uint16_t rid)
294{
295
296	IOMMU_ADD_DEVICE(dom, rid);
297}
298
299void
300iommu_remove_device(void *dom, uint16_t rid)
301{
302
303	IOMMU_REMOVE_DEVICE(dom, rid);
304}
305
306void
307iommu_invalidate_tlb(void *domain)
308{
309
310	IOMMU_INVALIDATE_TLB(domain);
311}
312