1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD$"); 33 34#include <sys/param.h> 35#include <sys/types.h> 36#include <sys/systm.h> 37#include <sys/bus.h> 38#include <sys/sysctl.h> 39 40#include <dev/pci/pcivar.h> 41#include <dev/pci/pcireg.h> 42 43#include <machine/cpu.h> 44#include <machine/md_var.h> 45 46#include "vmm_util.h" 47#include "vmm_mem.h" 48#include "iommu.h" 49 50SYSCTL_DECL(_hw_vmm); 51SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters"); 52 53static int iommu_avail; 54SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail, 55 0, "bhyve iommu initialized?"); 56 57static int iommu_enable = 1; 58SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0, 59 "Enable use of I/O MMU (required for PCI passthrough)."); 60 61static struct iommu_ops *ops; 62static void *host_domain; 63static eventhandler_tag add_tag, delete_tag; 64 65static __inline int 66IOMMU_INIT(void) 67{ 68 if (ops != NULL) 69 return ((*ops->init)()); 70 else 71 return (ENXIO); 72} 73 74static __inline void 75IOMMU_CLEANUP(void) 76{ 77 if (ops != NULL && iommu_avail) 78 (*ops->cleanup)(); 79} 80 81static __inline void * 82IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr) 83{ 84 85 if (ops != NULL && iommu_avail) 86 return ((*ops->create_domain)(maxaddr)); 87 else 88 return (NULL); 89} 90 91static __inline void 92IOMMU_DESTROY_DOMAIN(void *dom) 93{ 94 95 if (ops != NULL && iommu_avail) 96 (*ops->destroy_domain)(dom); 97} 98 99static __inline uint64_t 100IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 101{ 102 103 if (ops != NULL && iommu_avail) 104 return ((*ops->create_mapping)(domain, gpa, hpa, len)); 105 else 106 return (len); /* XXX */ 107} 108 109static __inline uint64_t 110IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len) 111{ 112 113 if (ops != NULL && iommu_avail) 114 return ((*ops->remove_mapping)(domain, gpa, len)); 115 else 116 return (len); /* XXX */ 117} 118 119static __inline void 120IOMMU_ADD_DEVICE(void *domain, uint16_t rid) 121{ 122 123 if (ops != NULL && iommu_avail) 124 (*ops->add_device)(domain, rid); 125} 126 127static __inline void 128IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid) 129{ 130 131 if (ops != NULL && iommu_avail) 132 (*ops->remove_device)(domain, rid); 133} 134 135static __inline void 136IOMMU_INVALIDATE_TLB(void *domain) 137{ 138 139 if (ops != NULL && iommu_avail) 140 (*ops->invalidate_tlb)(domain); 141} 142 143static __inline void 144IOMMU_ENABLE(void) 145{ 146 147 if (ops != NULL && iommu_avail) 148 (*ops->enable)(); 149} 150 151static __inline void 152IOMMU_DISABLE(void) 153{ 154 155 if (ops != NULL && iommu_avail) 156 (*ops->disable)(); 157} 158 159static void 160iommu_pci_add(void *arg, device_t dev) 161{ 162 163 /* Add new devices to the host domain. */ 164 iommu_add_device(host_domain, pci_get_rid(dev)); 165} 166 167static void 168iommu_pci_delete(void *arg, device_t dev) 169{ 170 171 iommu_remove_device(host_domain, pci_get_rid(dev)); 172} 173 174static void 175iommu_init(void) 176{ 177 int error, bus, slot, func; 178 vm_paddr_t maxaddr; 179 devclass_t dc; 180 device_t dev; 181 182 if (!iommu_enable) 183 return; 184 185 if (vmm_is_intel()) 186 ops = &iommu_ops_intel; 187 else if (vmm_is_svm()) 188 ops = &iommu_ops_amd; 189 else 190 ops = NULL; 191 192 error = IOMMU_INIT(); 193 if (error) 194 return; 195 196 iommu_avail = 1; 197 198 /* 199 * Create a domain for the devices owned by the host 200 */ 201 maxaddr = vmm_mem_maxaddr(); 202 host_domain = IOMMU_CREATE_DOMAIN(maxaddr); 203 if (host_domain == NULL) { 204 printf("iommu_init: unable to create a host domain"); 205 IOMMU_CLEANUP(); 206 ops = NULL; 207 iommu_avail = 0; 208 return; 209 } 210 211 /* 212 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to 213 * the host 214 */ 215 iommu_create_mapping(host_domain, 0, 0, maxaddr); 216 217 add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0); 218 delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete, 219 NULL, 0); 220 dc = devclass_find("ppt"); 221 for (bus = 0; bus <= PCI_BUSMAX; bus++) { 222 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 223 for (func = 0; func <= PCI_FUNCMAX; func++) { 224 dev = pci_find_dbsf(0, bus, slot, func); 225 if (dev == NULL) 226 continue; 227 228 /* Skip passthrough devices. */ 229 if (dc != NULL && 230 device_get_devclass(dev) == dc) 231 continue; 232 233 /* 234 * Everything else belongs to the host 235 * domain. 236 */ 237 iommu_add_device(host_domain, 238 pci_get_rid(dev)); 239 } 240 } 241 } 242 IOMMU_ENABLE(); 243 244} 245 246void 247iommu_cleanup(void) 248{ 249 250 if (add_tag != NULL) { 251 EVENTHANDLER_DEREGISTER(pci_add_device, add_tag); 252 add_tag = NULL; 253 } 254 if (delete_tag != NULL) { 255 EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag); 256 delete_tag = NULL; 257 } 258 IOMMU_DISABLE(); 259 IOMMU_DESTROY_DOMAIN(host_domain); 260 IOMMU_CLEANUP(); 261} 262 263void * 264iommu_create_domain(vm_paddr_t maxaddr) 265{ 266 static volatile int iommu_initted; 267 268 if (iommu_initted < 2) { 269 if (atomic_cmpset_int(&iommu_initted, 0, 1)) { 270 iommu_init(); 271 atomic_store_rel_int(&iommu_initted, 2); 272 } else 273 while (iommu_initted == 1) 274 cpu_spinwait(); 275 } 276 return (IOMMU_CREATE_DOMAIN(maxaddr)); 277} 278 279void 280iommu_destroy_domain(void *dom) 281{ 282 283 IOMMU_DESTROY_DOMAIN(dom); 284} 285 286void 287iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 288{ 289 uint64_t mapped, remaining; 290 291 remaining = len; 292 293 while (remaining > 0) { 294 mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining); 295 gpa += mapped; 296 hpa += mapped; 297 remaining -= mapped; 298 } 299} 300 301void 302iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) 303{ 304 uint64_t unmapped, remaining; 305 306 remaining = len; 307 308 while (remaining > 0) { 309 unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining); 310 gpa += unmapped; 311 remaining -= unmapped; 312 } 313} 314 315void * 316iommu_host_domain(void) 317{ 318 319 return (host_domain); 320} 321 322void 323iommu_add_device(void *dom, uint16_t rid) 324{ 325 326 IOMMU_ADD_DEVICE(dom, rid); 327} 328 329void 330iommu_remove_device(void *dom, uint16_t rid) 331{ 332 333 IOMMU_REMOVE_DEVICE(dom, rid); 334} 335 336void 337iommu_invalidate_tlb(void *domain) 338{ 339 340 IOMMU_INVALIDATE_TLB(domain); 341} 342