1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD$");
33
34#include <sys/param.h>
35#include <sys/types.h>
36#include <sys/systm.h>
37#include <sys/bus.h>
38#include <sys/sysctl.h>
39
40#include <dev/pci/pcivar.h>
41#include <dev/pci/pcireg.h>
42
43#include <machine/cpu.h>
44#include <machine/md_var.h>
45
46#include "vmm_util.h"
47#include "vmm_mem.h"
48#include "iommu.h"
49
50SYSCTL_DECL(_hw_vmm);
51SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters");
52
53static int iommu_avail;
54SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail,
55    0, "bhyve iommu initialized?");
56
57static int iommu_enable = 1;
58SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
59    "Enable use of I/O MMU (required for PCI passthrough).");
60
61static struct iommu_ops *ops;
62static void *host_domain;
63static eventhandler_tag add_tag, delete_tag;
64
65static __inline int
66IOMMU_INIT(void)
67{
68	if (ops != NULL)
69		return ((*ops->init)());
70	else
71		return (ENXIO);
72}
73
74static __inline void
75IOMMU_CLEANUP(void)
76{
77	if (ops != NULL && iommu_avail)
78		(*ops->cleanup)();
79}
80
81static __inline void *
82IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
83{
84
85	if (ops != NULL && iommu_avail)
86		return ((*ops->create_domain)(maxaddr));
87	else
88		return (NULL);
89}
90
91static __inline void
92IOMMU_DESTROY_DOMAIN(void *dom)
93{
94
95	if (ops != NULL && iommu_avail)
96		(*ops->destroy_domain)(dom);
97}
98
99static __inline uint64_t
100IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
101{
102
103	if (ops != NULL && iommu_avail)
104		return ((*ops->create_mapping)(domain, gpa, hpa, len));
105	else
106		return (len);		/* XXX */
107}
108
109static __inline uint64_t
110IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len)
111{
112
113	if (ops != NULL && iommu_avail)
114		return ((*ops->remove_mapping)(domain, gpa, len));
115	else
116		return (len);		/* XXX */
117}
118
119static __inline void
120IOMMU_ADD_DEVICE(void *domain, uint16_t rid)
121{
122
123	if (ops != NULL && iommu_avail)
124		(*ops->add_device)(domain, rid);
125}
126
127static __inline void
128IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid)
129{
130
131	if (ops != NULL && iommu_avail)
132		(*ops->remove_device)(domain, rid);
133}
134
135static __inline void
136IOMMU_INVALIDATE_TLB(void *domain)
137{
138
139	if (ops != NULL && iommu_avail)
140		(*ops->invalidate_tlb)(domain);
141}
142
143static __inline void
144IOMMU_ENABLE(void)
145{
146
147	if (ops != NULL && iommu_avail)
148		(*ops->enable)();
149}
150
151static __inline void
152IOMMU_DISABLE(void)
153{
154
155	if (ops != NULL && iommu_avail)
156		(*ops->disable)();
157}
158
159static void
160iommu_pci_add(void *arg, device_t dev)
161{
162
163	/* Add new devices to the host domain. */
164	iommu_add_device(host_domain, pci_get_rid(dev));
165}
166
167static void
168iommu_pci_delete(void *arg, device_t dev)
169{
170
171	iommu_remove_device(host_domain, pci_get_rid(dev));
172}
173
174static void
175iommu_init(void)
176{
177	int error, bus, slot, func;
178	vm_paddr_t maxaddr;
179	devclass_t dc;
180	device_t dev;
181
182	if (!iommu_enable)
183		return;
184
185	if (vmm_is_intel())
186		ops = &iommu_ops_intel;
187	else if (vmm_is_svm())
188		ops = &iommu_ops_amd;
189	else
190		ops = NULL;
191
192	error = IOMMU_INIT();
193	if (error)
194		return;
195
196	iommu_avail = 1;
197
198	/*
199	 * Create a domain for the devices owned by the host
200	 */
201	maxaddr = vmm_mem_maxaddr();
202	host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
203	if (host_domain == NULL) {
204		printf("iommu_init: unable to create a host domain");
205		IOMMU_CLEANUP();
206		ops = NULL;
207		iommu_avail = 0;
208		return;
209	}
210
211	/*
212	 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
213	 * the host
214	 */
215	iommu_create_mapping(host_domain, 0, 0, maxaddr);
216
217	add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0);
218	delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete,
219	    NULL, 0);
220	dc = devclass_find("ppt");
221	for (bus = 0; bus <= PCI_BUSMAX; bus++) {
222		for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
223			for (func = 0; func <= PCI_FUNCMAX; func++) {
224				dev = pci_find_dbsf(0, bus, slot, func);
225				if (dev == NULL)
226					continue;
227
228				/* Skip passthrough devices. */
229				if (dc != NULL &&
230				    device_get_devclass(dev) == dc)
231					continue;
232
233				/*
234				 * Everything else belongs to the host
235				 * domain.
236				 */
237				iommu_add_device(host_domain,
238				    pci_get_rid(dev));
239			}
240		}
241	}
242	IOMMU_ENABLE();
243
244}
245
246void
247iommu_cleanup(void)
248{
249
250	if (add_tag != NULL) {
251		EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
252		add_tag = NULL;
253	}
254	if (delete_tag != NULL) {
255		EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
256		delete_tag = NULL;
257	}
258	IOMMU_DISABLE();
259	IOMMU_DESTROY_DOMAIN(host_domain);
260	IOMMU_CLEANUP();
261}
262
263void *
264iommu_create_domain(vm_paddr_t maxaddr)
265{
266	static volatile int iommu_initted;
267
268	if (iommu_initted < 2) {
269		if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
270			iommu_init();
271			atomic_store_rel_int(&iommu_initted, 2);
272		} else
273			while (iommu_initted == 1)
274				cpu_spinwait();
275	}
276	return (IOMMU_CREATE_DOMAIN(maxaddr));
277}
278
279void
280iommu_destroy_domain(void *dom)
281{
282
283	IOMMU_DESTROY_DOMAIN(dom);
284}
285
286void
287iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
288{
289	uint64_t mapped, remaining;
290
291	remaining = len;
292
293	while (remaining > 0) {
294		mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining);
295		gpa += mapped;
296		hpa += mapped;
297		remaining -= mapped;
298	}
299}
300
301void
302iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
303{
304	uint64_t unmapped, remaining;
305
306	remaining = len;
307
308	while (remaining > 0) {
309		unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining);
310		gpa += unmapped;
311		remaining -= unmapped;
312	}
313}
314
315void *
316iommu_host_domain(void)
317{
318
319	return (host_domain);
320}
321
322void
323iommu_add_device(void *dom, uint16_t rid)
324{
325
326	IOMMU_ADD_DEVICE(dom, rid);
327}
328
329void
330iommu_remove_device(void *dom, uint16_t rid)
331{
332
333	IOMMU_REMOVE_DEVICE(dom, rid);
334}
335
336void
337iommu_invalidate_tlb(void *domain)
338{
339
340	IOMMU_INVALIDATE_TLB(domain);
341}
342