1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016, Anish Gupta (anish@freebsd.org) 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/param.h> 30#include <sys/systm.h> 31#include <sys/bus.h> 32#include <sys/kernel.h> 33#include <sys/module.h> 34#include <sys/malloc.h> 35#include <sys/pcpu.h> 36#include <sys/rman.h> 37#include <sys/smp.h> 38#include <sys/sysctl.h> 39 40#include <vm/vm.h> 41#include <vm/pmap.h> 42 43#include <dev/pci/pcivar.h> 44#include <dev/pci/pcireg.h> 45 46#include <machine/resource.h> 47#include <machine/vmm.h> 48#include <machine/pmap.h> 49#include <machine/vmparam.h> 50#include <machine/pci_cfgreg.h> 51 52#include "ivhd_if.h" 53#include "pcib_if.h" 54 55#include "io/iommu.h" 56#include "amdvi_priv.h" 57 58SYSCTL_DECL(_hw_vmm); 59SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 60 NULL); 61 62#define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s))) 63#define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s))) 64 65/* Print RID or device ID in PCI string format. */ 66#define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d) 67 68static void amdvi_dump_cmds(struct amdvi_softc *softc, int count); 69static void amdvi_print_dev_cap(struct amdvi_softc *softc); 70 71MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi"); 72 73extern device_t *ivhd_devs; 74 75extern int ivhd_count; 76SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count, 77 0, NULL); 78 79static int amdvi_enable_user = 0; 80SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN, 81 &amdvi_enable_user, 0, NULL); 82TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user); 83 84#ifdef AMDVI_ATS_ENABLE 85/* XXX: ATS is not tested. */ 86static int amdvi_enable_iotlb = 1; 87SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN, 88 &amdvi_enable_iotlb, 0, NULL); 89TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb); 90#endif 91 92static int amdvi_host_ptp = 1; /* Use page tables for host. */ 93SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN, 94 &amdvi_host_ptp, 0, NULL); 95TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp); 96 97/* Page table level used <= supported by h/w[v1=7]. */ 98int amdvi_ptp_level = 4; 99SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN, 100 &amdvi_ptp_level, 0, NULL); 101TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level); 102 103/* Disable fault event reporting. */ 104static int amdvi_disable_io_fault = 0; 105SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN, 106 &amdvi_disable_io_fault, 0, NULL); 107TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault); 108 109static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */ 110SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD, 111 &amdvi_dom_id, 0, NULL); 112/* 113 * Device table entry. 114 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes). 115 * = 256 * 2 * PAGE_SIZE. 116 */ 117static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE); 118CTASSERT(PCI_NUM_DEV_MAX == 0x10000); 119CTASSERT(sizeof(amdvi_dte) == 0x200000); 120 121static SLIST_HEAD (, amdvi_domain) dom_head; 122 123static inline uint32_t 124amdvi_pci_read(struct amdvi_softc *softc, int off) 125{ 126 127 return (pci_cfgregread(softc->pci_seg, PCI_RID2BUS(softc->pci_rid), 128 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid), 129 off, 4)); 130} 131 132#ifdef AMDVI_ATS_ENABLE 133/* XXX: Should be in pci.c */ 134/* 135 * Check if device has ATS capability and its enabled. 136 * If ATS is absent or disabled, return (-1), otherwise ATS 137 * queue length. 138 */ 139static int 140amdvi_find_ats_qlen(uint16_t devid) 141{ 142 device_t dev; 143 uint32_t off, cap; 144 int qlen = -1; 145 146 dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid), 147 PCI_RID2FUNC(devid)); 148 149 if (!dev) { 150 return (-1); 151 } 152#define PCIM_ATS_EN BIT(31) 153 154 if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) { 155 cap = pci_read_config(dev, off + 4, 4); 156 qlen = (cap & 0x1F); 157 qlen = qlen ? qlen : 32; 158 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n", 159 RID2PCI_STR(devid), 160 (cap & PCIM_ATS_EN) ? "enabled" : "Disabled", 161 qlen); 162 qlen = (cap & PCIM_ATS_EN) ? qlen : -1; 163 } 164 165 return (qlen); 166} 167 168/* 169 * Check if an endpoint device support device IOTLB or ATS. 170 */ 171static inline bool 172amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid) 173{ 174 struct ivhd_dev_cfg *cfg; 175 int qlen, i; 176 bool pci_ats, ivhd_ats; 177 178 qlen = amdvi_find_ats_qlen(devid); 179 if (qlen < 0) 180 return (false); 181 182 KASSERT(softc, ("softc is NULL")); 183 cfg = softc->dev_cfg; 184 185 ivhd_ats = false; 186 for (i = 0; i < softc->dev_cfg_cnt; i++) { 187 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) { 188 ivhd_ats = cfg->enable_ats; 189 break; 190 } 191 cfg++; 192 } 193 194 pci_ats = (qlen < 0) ? false : true; 195 if (pci_ats != ivhd_ats) 196 device_printf(softc->dev, 197 "BIOS bug: mismatch in ATS setting for %d.%d.%d," 198 "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen); 199 200 /* Ignore IVRS setting and respect PCI setting. */ 201 return (pci_ats); 202} 203#endif 204 205/* Enable IOTLB support for IOMMU if its supported. */ 206static inline void 207amdvi_hw_enable_iotlb(struct amdvi_softc *softc) 208{ 209#ifndef AMDVI_ATS_ENABLE 210 softc->iotlb = false; 211#else 212 bool supported; 213 214 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false; 215 216 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) { 217 if (!supported) 218 device_printf(softc->dev, "IOTLB disabled by BIOS.\n"); 219 220 if (supported && !amdvi_enable_iotlb) { 221 device_printf(softc->dev, "IOTLB disabled by user.\n"); 222 supported = false; 223 } 224 } else 225 supported = false; 226 227 softc->iotlb = supported; 228 229#endif 230} 231 232static int 233amdvi_init_cmd(struct amdvi_softc *softc) 234{ 235 struct amdvi_ctrl *ctrl = softc->ctrl; 236 237 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */ 238 softc->cmd_max = 1 << ctrl->cmd.len; 239 240 softc->cmd = malloc(sizeof(struct amdvi_cmd) * 241 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO); 242 243 if ((uintptr_t)softc->cmd & PAGE_MASK) 244 panic("AMDVi: Command buffer not aligned on page boundary."); 245 246 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE; 247 /* 248 * XXX: Reset the h/w pointers in case IOMMU is restarting, 249 * h/w doesn't clear these pointers based on empirical data. 250 */ 251 ctrl->cmd_tail = 0; 252 ctrl->cmd_head = 0; 253 254 return (0); 255} 256 257/* 258 * Note: Update tail pointer after we have written the command since tail 259 * pointer update cause h/w to execute new commands, see section 3.3 260 * of AMD IOMMU spec ver 2.0. 261 */ 262/* Get the command tail pointer w/o updating it. */ 263static struct amdvi_cmd * 264amdvi_get_cmd_tail(struct amdvi_softc *softc) 265{ 266 struct amdvi_ctrl *ctrl; 267 struct amdvi_cmd *tail; 268 269 KASSERT(softc, ("softc is NULL")); 270 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 271 272 ctrl = softc->ctrl; 273 KASSERT(ctrl != NULL, ("ctrl is NULL")); 274 275 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd + 276 ctrl->cmd_tail); 277 278 return (tail); 279} 280 281/* 282 * Update the command tail pointer which will start command execution. 283 */ 284static void 285amdvi_update_cmd_tail(struct amdvi_softc *softc) 286{ 287 struct amdvi_ctrl *ctrl; 288 int size; 289 290 size = sizeof(struct amdvi_cmd); 291 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 292 293 ctrl = softc->ctrl; 294 KASSERT(ctrl != NULL, ("ctrl is NULL")); 295 296 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max); 297 softc->total_cmd++; 298 299#ifdef AMDVI_DEBUG_CMD 300 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n", 301 ctrl->cmd_tail, 302 ctrl->cmd_head); 303#endif 304 305} 306 307/* 308 * Various commands supported by IOMMU. 309 */ 310 311/* Completion wait command. */ 312static void 313amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data) 314{ 315 struct amdvi_cmd *cmd; 316 uint64_t pa; 317 318 cmd = amdvi_get_cmd_tail(softc); 319 KASSERT(cmd != NULL, ("Cmd is NULL")); 320 321 pa = vtophys(&softc->cmp_data); 322 cmd->opcode = AMDVI_CMP_WAIT_OPCODE; 323 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE; 324 cmd->word1 = (pa >> 32) & 0xFFFFF; 325 cmd->addr = data; 326 327 amdvi_update_cmd_tail(softc); 328} 329 330/* Invalidate device table entry. */ 331static void 332amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid) 333{ 334 struct amdvi_cmd *cmd; 335 336 cmd = amdvi_get_cmd_tail(softc); 337 KASSERT(cmd != NULL, ("Cmd is NULL")); 338 cmd->opcode = AMDVI_INVD_DTE_OPCODE; 339 cmd->word0 = devid; 340 amdvi_update_cmd_tail(softc); 341#ifdef AMDVI_DEBUG_CMD 342 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid); 343#endif 344} 345 346/* Invalidate IOMMU page, use for invalidation of domain. */ 347static void 348amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id, 349 uint64_t addr, bool guest_nested, 350 bool pde, bool page) 351{ 352 struct amdvi_cmd *cmd; 353 354 cmd = amdvi_get_cmd_tail(softc); 355 KASSERT(cmd != NULL, ("Cmd is NULL")); 356 357 cmd->opcode = AMDVI_INVD_PAGE_OPCODE; 358 cmd->word1 = domain_id; 359 /* 360 * Invalidate all addresses for this domain. 361 */ 362 cmd->addr = addr; 363 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0; 364 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0; 365 366 amdvi_update_cmd_tail(softc); 367} 368 369#ifdef AMDVI_ATS_ENABLE 370/* Invalidate device IOTLB. */ 371static void 372amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid) 373{ 374 struct amdvi_cmd *cmd; 375 int qlen; 376 377 if (!softc->iotlb) 378 return; 379 380 qlen = amdvi_find_ats_qlen(devid); 381 if (qlen < 0) { 382 panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n", 383 qlen, RID2PCI_STR(devid)); 384 } 385 cmd = amdvi_get_cmd_tail(softc); 386 KASSERT(cmd != NULL, ("Cmd is NULL")); 387 388#ifdef AMDVI_DEBUG_CMD 389 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x" 390 " Qlen:%d\n", devid, qlen); 391#endif 392 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE; 393 cmd->word0 = devid; 394 cmd->word1 = qlen; 395 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR | 396 AMDVI_INVD_IOTLB_S; 397 amdvi_update_cmd_tail(softc); 398} 399#endif 400 401#ifdef notyet /* For Interrupt Remap. */ 402static void 403amdvi_cmd_inv_intr_map(struct amdvi_softc *softc, 404 uint16_t devid) 405{ 406 struct amdvi_cmd *cmd; 407 408 cmd = amdvi_get_cmd_tail(softc); 409 KASSERT(cmd != NULL, ("Cmd is NULL")); 410 cmd->opcode = AMDVI_INVD_INTR_OPCODE; 411 cmd->word0 = devid; 412 amdvi_update_cmd_tail(softc); 413#ifdef AMDVI_DEBUG_CMD 414 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid); 415#endif 416} 417#endif 418 419/* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */ 420static void 421amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id) 422{ 423 struct amdvi_cmd *cmd __diagused; 424 425 cmd = amdvi_get_cmd_tail(softc); 426 KASSERT(cmd != NULL, ("Cmd is NULL")); 427 428 /* 429 * See section 3.3.3 of IOMMU spec rev 2.0, software note 430 * for invalidating domain. 431 */ 432 amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR, 433 false, true, true); 434 435#ifdef AMDVI_DEBUG_CMD 436 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id); 437 438#endif 439} 440 441static bool 442amdvi_cmp_wait(struct amdvi_softc *softc) 443{ 444#ifdef AMDVI_DEBUG_CMD 445 struct amdvi_ctrl *ctrl = softc->ctrl; 446#endif 447 const uint64_t VERIFY = 0xA5A5; 448 volatile uint64_t *read; 449 int i; 450 bool status; 451 452 read = &softc->cmp_data; 453 *read = 0; 454 amdvi_cmd_cmp(softc, VERIFY); 455 /* Wait for h/w to update completion data. */ 456 for (i = 0; i < 100 && (*read != VERIFY); i++) { 457 DELAY(1000); /* 1 ms */ 458 } 459 status = (VERIFY == softc->cmp_data) ? true : false; 460 461#ifdef AMDVI_DEBUG_CMD 462 if (status) 463 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, " 464 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail, 465 ctrl->cmd_head, loop); 466#endif 467 return (status); 468} 469 470static void 471amdvi_wait(struct amdvi_softc *softc) 472{ 473 struct amdvi_ctrl *ctrl; 474 int i; 475 476 KASSERT(softc, ("softc is NULL")); 477 478 ctrl = softc->ctrl; 479 KASSERT(ctrl != NULL, ("ctrl is NULL")); 480 /* Don't wait if h/w is not enabled. */ 481 if ((ctrl->control & AMDVI_CTRL_EN) == 0) 482 return; 483 484 for (i = 0; i < 10; i++) { 485 if (amdvi_cmp_wait(softc)) 486 return; 487 } 488 489 device_printf(softc->dev, "Error: completion failed" 490 " tail:0x%x, head:0x%x.\n", 491 ctrl->cmd_tail, ctrl->cmd_head); 492 /* Dump the last command. */ 493 amdvi_dump_cmds(softc, 1); 494} 495 496static void 497amdvi_dump_cmds(struct amdvi_softc *softc, int count) 498{ 499 struct amdvi_ctrl *ctrl; 500 struct amdvi_cmd *cmd; 501 int off, i; 502 503 ctrl = softc->ctrl; 504 device_printf(softc->dev, "Dump last %d command(s):\n", count); 505 /* 506 * If h/w is stuck in completion, it is the previous command, 507 * start dumping from previous command onward. 508 */ 509 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd), 510 softc->cmd_max); 511 for (i = 0; off != ctrl->cmd_tail && i < count; i++) { 512 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off); 513 printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x" 514 " 0x%x 0x%lx\n", i, off, cmd->opcode, 515 cmd->word0, cmd->word1, cmd->addr); 516 off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max); 517 } 518} 519 520static int 521amdvi_init_event(struct amdvi_softc *softc) 522{ 523 struct amdvi_ctrl *ctrl; 524 525 ctrl = softc->ctrl; 526 ctrl->event.len = 8; 527 softc->event_max = 1 << ctrl->event.len; 528 softc->event = malloc(sizeof(struct amdvi_event) * 529 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO); 530 if ((uintptr_t)softc->event & PAGE_MASK) { 531 device_printf(softc->dev, "Event buffer not aligned on page."); 532 return (false); 533 } 534 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE; 535 536 /* Reset the pointers. */ 537 ctrl->evt_head = 0; 538 ctrl->evt_tail = 0; 539 540 return (0); 541} 542 543static inline void 544amdvi_decode_evt_flag(uint16_t flag) 545{ 546 547 flag &= AMDVI_EVENT_FLAG_MASK; 548 printf(" 0x%b]\n", flag, 549 "\020" 550 "\001GN" 551 "\002NX" 552 "\003US" 553 "\004I" 554 "\005PR" 555 "\006RW" 556 "\007PE" 557 "\010RZ" 558 "\011TR" 559 ); 560} 561 562/* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/ 563static inline void 564amdvi_decode_evt_flag_type(uint8_t type) 565{ 566 567 switch (AMDVI_EVENT_FLAG_TYPE(type)) { 568 case 0: 569 printf("RSVD\n"); 570 break; 571 case 1: 572 printf("Master Abort\n"); 573 break; 574 case 2: 575 printf("Target Abort\n"); 576 break; 577 case 3: 578 printf("Data Err\n"); 579 break; 580 default: 581 break; 582 } 583} 584 585static void 586amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr, 587 uint16_t flag) 588{ 589 590 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 591 " Addr:0x%lx", 592 devid, domid, addr); 593 amdvi_decode_evt_flag(flag); 594} 595 596static void 597amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr, 598 uint16_t flag) 599{ 600 601 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 602 " Addr:0x%lx", 603 devid, domid, addr); 604 amdvi_decode_evt_flag(flag); 605} 606 607static void 608amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid, 609 uint64_t addr, uint16_t flag) 610{ 611 612 printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 613 " Addr:0x%lx", devid, domid, addr); 614 amdvi_decode_evt_flag(flag); 615 amdvi_decode_evt_flag_type(flag); 616} 617 618static void 619amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr, 620 uint16_t flag) 621{ 622 623 printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 624 " Addr:0x%lx", devid, domid, addr); 625 amdvi_decode_evt_flag(flag); 626 amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag)); 627} 628 629static void 630amdvi_decode_evt(struct amdvi_event *evt) 631{ 632 struct amdvi_cmd *cmd; 633 634 switch (evt->opcode) { 635 case AMDVI_EVENT_INVALID_DTE: 636 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid, 637 evt->addr, evt->flag); 638 break; 639 640 case AMDVI_EVENT_PFAULT: 641 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid, 642 evt->addr, evt->flag); 643 break; 644 645 case AMDVI_EVENT_DTE_HW_ERROR: 646 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid, 647 evt->addr, evt->flag); 648 break; 649 650 case AMDVI_EVENT_PAGE_HW_ERROR: 651 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid, 652 evt->addr, evt->flag); 653 break; 654 655 case AMDVI_EVENT_ILLEGAL_CMD: 656 /* FALL THROUGH */ 657 case AMDVI_EVENT_CMD_HW_ERROR: 658 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ? 659 "ILLEGAL CMD" : "CMD HW ERR"); 660 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr); 661 printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n", 662 cmd->opcode, cmd->word0, cmd->word1, cmd->addr); 663 break; 664 665 case AMDVI_EVENT_IOTLB_TIMEOUT: 666 printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n", 667 evt->devid, evt->addr); 668 break; 669 670 case AMDVI_EVENT_INVALID_DTE_REQ: 671 printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n", 672 evt->devid, evt->addr, evt->flag >> 9, 673 (evt->flag >> 8) & 1); 674 break; 675 676 case AMDVI_EVENT_INVALID_PPR_REQ: 677 case AMDVI_EVENT_COUNTER_ZERO: 678 printf("AMD-Vi: v2 events.\n"); 679 break; 680 681 default: 682 printf("Unsupported AMD-Vi event:%d\n", evt->opcode); 683 } 684} 685 686static void 687amdvi_print_events(struct amdvi_softc *softc) 688{ 689 struct amdvi_ctrl *ctrl; 690 struct amdvi_event *event; 691 int i, size; 692 693 ctrl = softc->ctrl; 694 size = sizeof(struct amdvi_event); 695 for (i = 0; i < softc->event_max; i++) { 696 event = &softc->event[ctrl->evt_head / size]; 697 if (!event->opcode) 698 break; 699 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n", 700 i, ctrl->evt_head, ctrl->evt_tail); 701 amdvi_decode_evt(event); 702 ctrl->evt_head = MOD_INC(ctrl->evt_head, size, 703 softc->event_max); 704 } 705} 706 707static int 708amdvi_init_dte(struct amdvi_softc *softc) 709{ 710 struct amdvi_ctrl *ctrl; 711 712 ctrl = softc->ctrl; 713 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE; 714 ctrl->dte.size = 0x1FF; /* 2MB device table. */ 715 716 return (0); 717} 718 719/* 720 * Not all capabilities of IOMMU are available in ACPI IVHD flag 721 * or EFR entry, read directly from device. 722 */ 723static int 724amdvi_print_pci_cap(device_t dev) 725{ 726 struct amdvi_softc *softc; 727 uint32_t off, cap; 728 729 softc = device_get_softc(dev); 730 off = softc->cap_off; 731 732 /* 733 * Section 3.7.1 of IOMMU sepc rev 2.0. 734 * Read capability from device. 735 */ 736 cap = amdvi_pci_read(softc, off); 737 738 /* Make sure capability type[18:16] is 3. */ 739 KASSERT((((cap >> 16) & 0x7) == 0x3), 740 ("Not a IOMMU capability 0x%x@0x%x", cap, off)); 741 742 softc->pci_cap = cap >> 24; 743 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n", 744 cap, off, softc->pci_cap, 745 "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt"); 746 747 return (0); 748} 749 750static void 751amdvi_event_intr(void *arg) 752{ 753 struct amdvi_softc *softc; 754 struct amdvi_ctrl *ctrl; 755 756 softc = (struct amdvi_softc *)arg; 757 ctrl = softc->ctrl; 758 device_printf(softc->dev, "EVT INTR %ld Status:0x%x" 759 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++, 760 ctrl->status, ctrl->evt_head, ctrl->evt_tail); 761 printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n", 762 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head); 763 764 amdvi_print_events(softc); 765 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 766} 767 768static void 769amdvi_free_evt_intr_res(device_t dev) 770{ 771 772 struct amdvi_softc *softc; 773 device_t mmio_dev; 774 775 softc = device_get_softc(dev); 776 mmio_dev = softc->pci_dev; 777 778 IVHD_TEARDOWN_INTR(mmio_dev); 779} 780 781static bool 782amdvi_alloc_intr_resources(struct amdvi_softc *softc) 783{ 784 struct amdvi_ctrl *ctrl; 785 device_t dev, mmio_dev; 786 int err; 787 788 dev = softc->dev; 789 mmio_dev = softc->pci_dev; 790 791 /* Clear interrupt status bits. */ 792 ctrl = softc->ctrl; 793 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 794 795 err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault"); 796 if (err) 797 device_printf(dev, "Interrupt setup failed on %s\n", 798 device_get_nameunit(mmio_dev)); 799 return (err); 800} 801 802static void 803amdvi_print_dev_cap(struct amdvi_softc *softc) 804{ 805 struct ivhd_dev_cfg *cfg; 806 int i; 807 808 cfg = softc->dev_cfg; 809 for (i = 0; i < softc->dev_cfg_cnt; i++) { 810 device_printf(softc->dev, "device [0x%x - 0x%x] " 811 "config:%b%s\n", cfg->start_id, cfg->end_id, 812 cfg->data, 813 "\020\001INIT\002ExtInt\003NMI" 814 "\007LINT0\010LINT1", 815 cfg->enable_ats ? "ATS enabled" : ""); 816 cfg++; 817 } 818} 819 820static int 821amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS) 822{ 823 struct amdvi_softc *softc; 824 int result, type, error = 0; 825 826 softc = (struct amdvi_softc *)arg1; 827 type = arg2; 828 829 switch (type) { 830 case 0: 831 result = softc->ctrl->cmd_head; 832 error = sysctl_handle_int(oidp, &result, 0, 833 req); 834 break; 835 case 1: 836 result = softc->ctrl->cmd_tail; 837 error = sysctl_handle_int(oidp, &result, 0, 838 req); 839 break; 840 case 2: 841 result = softc->ctrl->evt_head; 842 error = sysctl_handle_int(oidp, &result, 0, 843 req); 844 break; 845 case 3: 846 result = softc->ctrl->evt_tail; 847 error = sysctl_handle_int(oidp, &result, 0, 848 req); 849 break; 850 851 default: 852 device_printf(softc->dev, "Unknown sysctl:%d\n", type); 853 } 854 855 return (error); 856} 857 858static void 859amdvi_add_sysctl(struct amdvi_softc *softc) 860{ 861 struct sysctl_oid_list *child; 862 struct sysctl_ctx_list *ctx; 863 device_t dev; 864 865 dev = softc->dev; 866 ctx = device_get_sysctl_ctx(dev); 867 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 868 869 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD, 870 &softc->event_intr_cnt, "Event interrupt count"); 871 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD, 872 &softc->total_cmd, "Command submitted count"); 873 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD, 874 &softc->pci_rid, 0, "IOMMU RID"); 875 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head", 876 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0, 877 amdvi_handle_sysctl, "IU", "Command head"); 878 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail", 879 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1, 880 amdvi_handle_sysctl, "IU", "Command tail"); 881 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head", 882 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2, 883 amdvi_handle_sysctl, "IU", "Command head"); 884 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail", 885 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3, 886 amdvi_handle_sysctl, "IU", "Command tail"); 887} 888 889int 890amdvi_setup_hw(struct amdvi_softc *softc) 891{ 892 device_t dev; 893 int status; 894 895 dev = softc->dev; 896 897 amdvi_hw_enable_iotlb(softc); 898 899 amdvi_print_dev_cap(softc); 900 901 if ((status = amdvi_print_pci_cap(dev)) != 0) { 902 device_printf(dev, "PCI capability.\n"); 903 return (status); 904 } 905 if ((status = amdvi_init_cmd(softc)) != 0) { 906 device_printf(dev, "Couldn't configure command buffer.\n"); 907 return (status); 908 } 909 if ((status = amdvi_init_event(softc)) != 0) { 910 device_printf(dev, "Couldn't configure event buffer.\n"); 911 return (status); 912 } 913 if ((status = amdvi_init_dte(softc)) != 0) { 914 device_printf(dev, "Couldn't configure device table.\n"); 915 return (status); 916 } 917 if ((status = amdvi_alloc_intr_resources(softc)) != 0) { 918 return (status); 919 } 920 amdvi_add_sysctl(softc); 921 return (0); 922} 923 924int 925amdvi_teardown_hw(struct amdvi_softc *softc) 926{ 927 device_t dev; 928 929 dev = softc->dev; 930 931 /* 932 * Called after disable, h/w is stopped by now, free all the resources. 933 */ 934 amdvi_free_evt_intr_res(dev); 935 936 if (softc->cmd) 937 free(softc->cmd, M_AMDVI); 938 939 if (softc->event) 940 free(softc->event, M_AMDVI); 941 942 return (0); 943} 944 945/*********** bhyve interfaces *********************/ 946static int 947amdvi_init(void) 948{ 949 if (!ivhd_count) { 950 return (EIO); 951 } 952 if (!amdvi_enable_user && ivhd_count) { 953 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), " 954 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n", 955 ivhd_count); 956 return (EINVAL); 957 } 958 return (0); 959} 960 961static void 962amdvi_cleanup(void) 963{ 964 /* Nothing. */ 965} 966 967static uint16_t 968amdvi_domainId(void) 969{ 970 971 /* 972 * If we hit maximum domain limit, rollover leaving host 973 * domain(0). 974 * XXX: make sure that this domain is not used. 975 */ 976 if (amdvi_dom_id == AMDVI_MAX_DOMAIN) 977 amdvi_dom_id = 1; 978 979 return ((uint16_t)amdvi_dom_id++); 980} 981 982static void 983amdvi_do_inv_domain(uint16_t domain_id, bool create) 984{ 985 struct amdvi_softc *softc; 986 int i; 987 988 for (i = 0; i < ivhd_count; i++) { 989 softc = device_get_softc(ivhd_devs[i]); 990 KASSERT(softc, ("softc is NULL")); 991 /* 992 * If not present pages are cached, invalidate page after 993 * creating domain. 994 */ 995#if 0 996 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0)) 997 continue; 998#endif 999 amdvi_inv_domain(softc, domain_id); 1000 amdvi_wait(softc); 1001 } 1002} 1003 1004static void * 1005amdvi_create_domain(vm_paddr_t maxaddr) 1006{ 1007 struct amdvi_domain *dom; 1008 1009 dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK); 1010 dom->id = amdvi_domainId(); 1011 //dom->maxaddr = maxaddr; 1012#ifdef AMDVI_DEBUG_CMD 1013 printf("Created domain #%d\n", dom->id); 1014#endif 1015 /* 1016 * Host domain(#0) don't create translation table. 1017 */ 1018 if (dom->id || amdvi_host_ptp) 1019 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1020 1021 dom->ptp_level = amdvi_ptp_level; 1022 1023 amdvi_do_inv_domain(dom->id, true); 1024 SLIST_INSERT_HEAD(&dom_head, dom, next); 1025 1026 return (dom); 1027} 1028 1029static void 1030amdvi_free_ptp(uint64_t *ptp, int level) 1031{ 1032 int i; 1033 1034 if (level < 1) 1035 return; 1036 1037 for (i = 0; i < NPTEPG ; i++) { 1038 if ((ptp[i] & AMDVI_PT_PRESENT) == 0) 1039 continue; 1040 /* XXX: Add super-page or PTE mapping > 4KB. */ 1041#ifdef notyet 1042 /* Super-page mapping. */ 1043 if (AMDVI_PD_SUPER(ptp[i])) 1044 continue; 1045#endif 1046 1047 amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i] 1048 & AMDVI_PT_MASK), level - 1); 1049 } 1050 1051 free(ptp, M_AMDVI); 1052} 1053 1054static void 1055amdvi_destroy_domain(void *arg) 1056{ 1057 struct amdvi_domain *domain; 1058 1059 domain = (struct amdvi_domain *)arg; 1060 KASSERT(domain, ("domain is NULL")); 1061#ifdef AMDVI_DEBUG_CMD 1062 printf("Destroying domain %d\n", domain->id); 1063#endif 1064 if (domain->ptp) 1065 amdvi_free_ptp(domain->ptp, domain->ptp_level); 1066 1067 amdvi_do_inv_domain(domain->id, false); 1068 SLIST_REMOVE(&dom_head, domain, amdvi_domain, next); 1069 free(domain, M_AMDVI); 1070} 1071 1072static uint64_t 1073amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa, 1074 vm_paddr_t hpa, uint64_t pg_size, bool create) 1075{ 1076 uint64_t *page, pa; 1077 int shift, index; 1078 const int PT_SHIFT = 9; 1079 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */ 1080 1081 if (!pg_size) 1082 return (0); 1083 1084 if (hpa & (pg_size - 1)) { 1085 printf("HPA is not size aligned.\n"); 1086 return (0); 1087 } 1088 if (gpa & (pg_size - 1)) { 1089 printf("HPA is not size aligned.\n"); 1090 return (0); 1091 } 1092 shift = PML4SHIFT; 1093 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) { 1094 index = (gpa >> shift) & PT_INDEX_MASK; 1095 1096 if ((pt[index] == 0) && create) { 1097 page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1098 pa = vtophys(page); 1099 pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW | 1100 ((level - 1) << AMDVI_PD_LEVEL_SHIFT); 1101 } 1102#ifdef AMDVI_DEBUG_PTE 1103 if ((gpa % 0x1000000) == 0) 1104 printf("[level%d, shift = %d]PTE:0x%lx\n", 1105 level, shift, pt[index]); 1106#endif 1107#define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK) 1108 pa = PTE2PA(pt[index]); 1109 pt = (uint64_t *)PHYS_TO_DMAP(pa); 1110 shift -= PT_SHIFT; 1111 level--; 1112 } 1113 1114 /* Leaf entry. */ 1115 index = (gpa >> shift) & PT_INDEX_MASK; 1116 1117 if (create) { 1118 pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT; 1119 } else 1120 pt[index] = 0; 1121 1122#ifdef AMDVI_DEBUG_PTE 1123 if ((gpa % 0x1000000) == 0) 1124 printf("[Last level%d, shift = %d]PTE:0x%lx\n", 1125 level, shift, pt[index]); 1126#endif 1127 return (1ULL << shift); 1128} 1129 1130static uint64_t 1131amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa, 1132 vm_paddr_t hpa, uint64_t size, bool create) 1133{ 1134 uint64_t mapped, *ptp, len; 1135 int level; 1136 1137 KASSERT(domain, ("domain is NULL")); 1138 level = domain->ptp_level; 1139 KASSERT(level, ("Page table level is 0")); 1140 1141 ptp = domain->ptp; 1142 KASSERT(ptp, ("PTP is NULL")); 1143 mapped = 0; 1144 while (mapped < size) { 1145 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped, 1146 PAGE_SIZE, create); 1147 if (!len) { 1148 printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n", 1149 hpa, gpa); 1150 return (0); 1151 } 1152 mapped += len; 1153 } 1154 1155 return (mapped); 1156} 1157 1158static uint64_t 1159amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, 1160 uint64_t len) 1161{ 1162 struct amdvi_domain *domain; 1163 1164 domain = (struct amdvi_domain *)arg; 1165 1166 if (domain->id && !domain->ptp) { 1167 printf("ptp is NULL"); 1168 return (-1); 1169 } 1170 1171 /* 1172 * If host domain is created w/o page table, skip IOMMU page 1173 * table set-up. 1174 */ 1175 if (domain->ptp) 1176 return (amdvi_update_mapping(domain, gpa, hpa, len, true)); 1177 else 1178 return (len); 1179} 1180 1181static uint64_t 1182amdvi_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 1183{ 1184 struct amdvi_domain *domain; 1185 1186 domain = (struct amdvi_domain *)arg; 1187 /* 1188 * If host domain is created w/o page table, skip IOMMU page 1189 * table set-up. 1190 */ 1191 if (domain->ptp) 1192 return (amdvi_update_mapping(domain, gpa, 0, len, false)); 1193 return 1194 (len); 1195} 1196 1197static struct amdvi_softc * 1198amdvi_find_iommu(uint16_t devid) 1199{ 1200 struct amdvi_softc *softc; 1201 int i, j; 1202 1203 for (i = 0; i < ivhd_count; i++) { 1204 softc = device_get_softc(ivhd_devs[i]); 1205 for (j = 0; j < softc->dev_cfg_cnt; j++) 1206 if ((devid >= softc->dev_cfg[j].start_id) && 1207 (devid <= softc->dev_cfg[j].end_id)) 1208 return (softc); 1209 } 1210 1211 return (NULL); 1212} 1213 1214/* 1215 * Set-up device table entry. 1216 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must 1217 * be set concurrently, e.g. read and write bits. 1218 */ 1219static void 1220amdvi_set_dte(struct amdvi_domain *domain, struct amdvi_softc *softc, 1221 uint16_t devid, bool enable) 1222{ 1223 struct amdvi_dte* temp; 1224 1225 KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid)); 1226 KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid)); 1227 1228 temp = &amdvi_dte[devid]; 1229 1230#ifdef AMDVI_ATS_ENABLE 1231 /* If IOMMU and device support IOTLB, enable it. */ 1232 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb) 1233 temp->iotlb_enable = 1; 1234#endif 1235 1236 /* Avoid duplicate I/O faults. */ 1237 temp->sup_second_io_fault = 1; 1238 temp->sup_all_io_fault = amdvi_disable_io_fault; 1239 1240 temp->dt_valid = 1; 1241 temp->domain_id = domain->id; 1242 1243 if (enable) { 1244 if (domain->ptp) { 1245 temp->pt_base = vtophys(domain->ptp) >> 12; 1246 temp->pt_level = amdvi_ptp_level; 1247 } 1248 /* 1249 * XXX: Page table valid[TV] bit must be set even if host domain 1250 * page tables are not enabled. 1251 */ 1252 temp->pt_valid = 1; 1253 temp->read_allow = 1; 1254 temp->write_allow = 1; 1255 } 1256} 1257 1258static void 1259amdvi_inv_device(struct amdvi_softc *softc, uint16_t devid) 1260{ 1261 KASSERT(softc, ("softc is NULL")); 1262 1263 amdvi_cmd_inv_dte(softc, devid); 1264#ifdef AMDVI_ATS_ENABLE 1265 if (amdvi_dev_support_iotlb(softc, devid)) 1266 amdvi_cmd_inv_iotlb(softc, devid); 1267#endif 1268 amdvi_wait(softc); 1269} 1270 1271static void 1272amdvi_add_device(void *arg, uint16_t devid) 1273{ 1274 struct amdvi_domain *domain; 1275 struct amdvi_softc *softc; 1276 1277 domain = (struct amdvi_domain *)arg; 1278 KASSERT(domain != NULL, ("domain is NULL")); 1279#ifdef AMDVI_DEBUG_CMD 1280 printf("Assigning device(%d.%d.%d) to domain:%d\n", 1281 RID2PCI_STR(devid), domain->id); 1282#endif 1283 softc = amdvi_find_iommu(devid); 1284 if (softc == NULL) 1285 return; 1286 amdvi_set_dte(domain, softc, devid, true); 1287 amdvi_inv_device(softc, devid); 1288} 1289 1290static void 1291amdvi_remove_device(void *arg, uint16_t devid) 1292{ 1293 struct amdvi_domain *domain; 1294 struct amdvi_softc *softc; 1295 1296 domain = (struct amdvi_domain *)arg; 1297#ifdef AMDVI_DEBUG_CMD 1298 printf("Remove device(0x%x) from domain:%d\n", 1299 devid, domain->id); 1300#endif 1301 softc = amdvi_find_iommu(devid); 1302 if (softc == NULL) 1303 return; 1304 amdvi_set_dte(domain, softc, devid, false); 1305 amdvi_inv_device(softc, devid); 1306} 1307 1308static void 1309amdvi_enable(void) 1310{ 1311 struct amdvi_ctrl *ctrl; 1312 struct amdvi_softc *softc; 1313 uint64_t val; 1314 int i; 1315 1316 for (i = 0; i < ivhd_count; i++) { 1317 softc = device_get_softc(ivhd_devs[i]); 1318 KASSERT(softc, ("softc is NULL\n")); 1319 ctrl = softc->ctrl; 1320 KASSERT(ctrl, ("ctrl is NULL\n")); 1321 1322 val = ( AMDVI_CTRL_EN | 1323 AMDVI_CTRL_CMD | 1324 AMDVI_CTRL_ELOG | 1325 AMDVI_CTRL_ELOGINT | 1326 AMDVI_CTRL_INV_TO_1S); 1327 1328 if (softc->ivhd_flag & IVHD_FLAG_COH) 1329 val |= AMDVI_CTRL_COH; 1330 if (softc->ivhd_flag & IVHD_FLAG_HTT) 1331 val |= AMDVI_CTRL_HTT; 1332 if (softc->ivhd_flag & IVHD_FLAG_RPPW) 1333 val |= AMDVI_CTRL_RPPW; 1334 if (softc->ivhd_flag & IVHD_FLAG_PPW) 1335 val |= AMDVI_CTRL_PPW; 1336 if (softc->ivhd_flag & IVHD_FLAG_ISOC) 1337 val |= AMDVI_CTRL_ISOC; 1338 1339 ctrl->control = val; 1340 } 1341} 1342 1343static void 1344amdvi_disable(void) 1345{ 1346 struct amdvi_ctrl *ctrl; 1347 struct amdvi_softc *softc; 1348 int i; 1349 1350 for (i = 0; i < ivhd_count; i++) { 1351 softc = device_get_softc(ivhd_devs[i]); 1352 KASSERT(softc, ("softc is NULL\n")); 1353 ctrl = softc->ctrl; 1354 KASSERT(ctrl, ("ctrl is NULL\n")); 1355 1356 ctrl->control = 0; 1357 } 1358} 1359 1360static void 1361amdvi_invalidate_tlb(void *arg) 1362{ 1363 struct amdvi_domain *domain; 1364 1365 domain = (struct amdvi_domain *)arg; 1366 KASSERT(domain, ("domain is NULL")); 1367 amdvi_do_inv_domain(domain->id, false); 1368} 1369 1370const struct iommu_ops iommu_ops_amd = { 1371 .init = amdvi_init, 1372 .cleanup = amdvi_cleanup, 1373 .enable = amdvi_enable, 1374 .disable = amdvi_disable, 1375 .create_domain = amdvi_create_domain, 1376 .destroy_domain = amdvi_destroy_domain, 1377 .create_mapping = amdvi_create_mapping, 1378 .remove_mapping = amdvi_remove_mapping, 1379 .add_device = amdvi_add_device, 1380 .remove_device = amdvi_remove_device, 1381 .invalidate_tlb = amdvi_invalidate_tlb 1382}; 1383