Lines Matching refs:iommu
38 static int amd_iommu_fini(amd_iommu_t *iommu, int type);
39 static void amd_iommu_teardown_interrupts(amd_iommu_t *iommu);
40 static void amd_iommu_stop(amd_iommu_t *iommu);
74 static int unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
122 amd_iommu_register(amd_iommu_t *iommu)
124 dev_info_t *dip = iommu->aiomt_dip;
135 iommulib_ops->ilops_data = (void *)iommu;
136 iommu->aiomt_iommulib_ops = iommulib_ops;
141 "failed idx=%d", f, driver, instance, iommu->aiomt_idx);
146 iommu->aiomt_iommulib_handle = handle;
152 amd_iommu_unregister(amd_iommu_t *iommu)
154 if (iommu->aiomt_iommulib_handle == NULL) {
159 if (iommulib_iommu_unregister(iommu->aiomt_iommulib_handle)
164 kmem_free(iommu->aiomt_iommulib_ops, sizeof (iommulib_ops_t));
165 iommu->aiomt_iommulib_ops = NULL;
166 iommu->aiomt_iommulib_handle = NULL;
172 amd_iommu_setup_passthru(amd_iommu_t *iommu)
180 amd_iommu_set_passthru(iommu, NULL);
186 amd_iommu_set_passthru(iommu, dip);
195 amd_iommu_start(amd_iommu_t *iommu)
197 dev_info_t *dip = iommu->aiomt_dip;
209 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
217 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
219 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
221 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
225 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
239 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
245 iommu->aiomt_idx);
248 instance, iommu->aiomt_idx);
254 amd_iommu_stop(amd_iommu_t *iommu)
256 dev_info_t *dip = iommu->aiomt_dip;
261 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
263 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
265 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
267 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
273 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
275 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
280 iommu->aiomt_idx);
284 amd_iommu_setup_tables_and_buffers(amd_iommu_t *iommu)
286 dev_info_t *dip = iommu->aiomt_dip;
306 iommu->aiomt_devtbl_sz = (1<<AMD_IOMMU_DEVTBL_SZ) * AMD_IOMMU_DEVENT_SZ;
307 iommu->aiomt_cmdbuf_sz = (1<<AMD_IOMMU_CMDBUF_SZ) * AMD_IOMMU_CMD_SZ;
308 iommu->aiomt_eventlog_sz =
311 dma_bufsz = iommu->aiomt_devtbl_sz + iommu->aiomt_cmdbuf_sz
312 + iommu->aiomt_eventlog_sz;
318 DDI_DMA_SLEEP, NULL, &iommu->aiomt_dmahdl);
329 err = ddi_dma_mem_alloc(iommu->aiomt_dmahdl, dma_bufsz,
331 DDI_DMA_SLEEP, NULL, (caddr_t *)&iommu->aiomt_dma_bufva,
332 (size_t *)&iommu->aiomt_dma_mem_realsz, &iommu->aiomt_dma_mem_hdl);
336 iommu->aiomt_dma_bufva = NULL;
337 iommu->aiomt_dma_mem_realsz = 0;
338 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
339 iommu->aiomt_dmahdl = NULL;
346 ASSERT(((uintptr_t)iommu->aiomt_dma_bufva &
348 ASSERT(iommu->aiomt_dma_mem_realsz >= dma_bufsz);
353 err = ddi_dma_addr_bind_handle(iommu->aiomt_dmahdl, NULL,
354 iommu->aiomt_dma_bufva, iommu->aiomt_dma_mem_realsz,
356 NULL, &iommu->aiomt_buf_dma_cookie, &iommu->aiomt_buf_dma_ncookie);
361 (void *)(uintptr_t)iommu->aiomt_dma_mem_realsz);
362 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
363 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
364 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
365 iommu->aiomt_buf_dma_ncookie = 0;
366 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
367 iommu->aiomt_dma_mem_hdl = NULL;
368 iommu->aiomt_dma_bufva = NULL;
369 iommu->aiomt_dma_mem_realsz = 0;
370 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
371 iommu->aiomt_dmahdl = NULL;
380 if (iommu->aiomt_buf_dma_ncookie != 1) {
384 iommu->aiomt_buf_dma_ncookie);
385 (void) ddi_dma_unbind_handle(iommu->aiomt_dmahdl);
386 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
387 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
388 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
389 iommu->aiomt_buf_dma_ncookie = 0;
390 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
391 iommu->aiomt_dma_mem_hdl = NULL;
392 iommu->aiomt_dma_bufva = NULL;
393 iommu->aiomt_dma_mem_realsz = 0;
394 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
395 iommu->aiomt_dmahdl = NULL;
402 ASSERT((iommu->aiomt_buf_dma_cookie.dmac_cookie_addr
404 ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size
405 <= iommu->aiomt_dma_mem_realsz);
406 ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size >= dma_bufsz);
409 * Setup the device table pointers in the iommu struct as
412 iommu->aiomt_devtbl = iommu->aiomt_dma_bufva;
413 bzero(iommu->aiomt_devtbl, iommu->aiomt_devtbl_sz);
422 dentry = (uint64_t *)&iommu->aiomt_devtbl
428 addr = (caddr_t)(uintptr_t)iommu->aiomt_buf_dma_cookie.dmac_cookie_addr;
429 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
431 sz = (iommu->aiomt_devtbl_sz >> 12) - 1;
433 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
439 iommu->aiomt_cmdbuf = iommu->aiomt_devtbl +
440 iommu->aiomt_devtbl_sz;
441 bzero(iommu->aiomt_cmdbuf, iommu->aiomt_cmdbuf_sz);
442 addr += iommu->aiomt_devtbl_sz;
443 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
449 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
452 iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf;
453 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
455 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va),
461 iommu->aiomt_eventlog = iommu->aiomt_cmdbuf +
462 iommu->aiomt_eventlog_sz;
463 bzero(iommu->aiomt_eventlog, iommu->aiomt_eventlog_sz);
464 addr += iommu->aiomt_cmdbuf_sz;
465 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
470 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
473 iommu->aiomt_event_head = (uint32_t *)iommu->aiomt_eventlog;
474 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
476 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_tail_va),
480 SYNC_FORDEV(iommu->aiomt_dmahdl);
484 "tables, idx=%d", f, driver, instance, iommu->aiomt_idx);
491 amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu, int type)
493 dev_info_t *dip = iommu->aiomt_dip;
498 iommu->aiomt_eventlog = NULL;
499 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
501 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
503 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
505 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
509 iommu->aiomt_cmdbuf = NULL;
510 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
512 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
514 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
516 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
520 iommu->aiomt_devtbl = NULL;
521 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
523 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
526 if (iommu->aiomt_dmahdl == NULL || type == AMD_IOMMU_QUIESCE)
530 if (ddi_dma_unbind_handle(iommu->aiomt_dmahdl) != DDI_SUCCESS) {
533 (void *)iommu->aiomt_dmahdl, iommu->aiomt_idx);
535 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
536 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
537 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
538 iommu->aiomt_buf_dma_ncookie = 0;
541 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
542 iommu->aiomt_dma_mem_hdl = NULL;
543 iommu->aiomt_dma_bufva = NULL;
544 iommu->aiomt_dma_mem_realsz = 0;
547 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
548 iommu->aiomt_dmahdl = NULL;
552 amd_iommu_enable_interrupts(amd_iommu_t *iommu)
554 ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
556 ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
561 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
564 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
566 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
568 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
573 amd_iommu_setup_exclusion(amd_iommu_t *iommu)
581 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
584 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
586 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
588 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
595 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
597 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
599 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
601 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
609 amd_iommu_teardown_exclusion(amd_iommu_t *iommu)
611 (void) amd_iommu_setup_exclusion(iommu);
618 amd_iommu_t *iommu = (amd_iommu_t *)arg1;
619 dev_info_t *dip = iommu->aiomt_dip;
629 f, driver, instance, iommu->aiomt_idx);
632 if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
637 iommu->aiomt_idx);
639 (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISPLAY);
641 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
646 if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
650 iommu->aiomt_idx);
651 (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISCARD);
652 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
654 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
664 amd_iommu_setup_interrupts(amd_iommu_t *iommu)
666 dev_info_t *dip = iommu->aiomt_dip;
683 "failed: idx=%d", f, driver, instance, iommu->aiomt_idx);
690 iommu->aiomt_idx, type);
699 f, driver, instance, iommu->aiomt_idx);
705 f, driver, instance, iommu->aiomt_idx);
712 f, driver, instance, iommu->aiomt_idx, err);
719 f, driver, instance, iommu->aiomt_idx, req);
725 driver, instance, iommu->aiomt_idx);
733 driver, instance, iommu->aiomt_idx, err);
740 f, driver, instance, iommu->aiomt_idx, avail);
746 driver, instance, iommu->aiomt_idx);
753 "Failing init", f, driver, instance, iommu->aiomt_idx,
759 iommu->aiomt_intr_htable_sz = req * sizeof (ddi_intr_handle_t);
760 iommu->aiomt_intr_htable = kmem_zalloc(iommu->aiomt_intr_htable_sz,
763 iommu->aiomt_intr_state = AMD_IOMMU_INTR_TABLE;
775 f, driver, instance, iommu->aiomt_idx, p2req, req);
778 err = ddi_intr_alloc(iommu->aiomt_dip, iommu->aiomt_intr_htable,
783 f, driver, instance, iommu->aiomt_idx, err);
784 amd_iommu_teardown_interrupts(iommu);
788 iommu->aiomt_actual_intrs = actual;
789 iommu->aiomt_intr_state = AMD_IOMMU_INTR_ALLOCED;
794 f, driver, instance, iommu->aiomt_idx, actual);
797 if (iommu->aiomt_actual_intrs < req) {
800 f, driver, instance, iommu->aiomt_idx,
801 iommu->aiomt_actual_intrs, req);
802 amd_iommu_teardown_interrupts(iommu);
806 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
807 if (ddi_intr_add_handler(iommu->aiomt_intr_htable[i],
808 amd_iommu_intr_handler, (void *)iommu, NULL)
812 f, driver, instance, iommu->aiomt_idx, i, err);
815 iommu->aiomt_intr_htable[j]);
817 amd_iommu_teardown_interrupts(iommu);
821 iommu->aiomt_intr_state = AMD_IOMMU_INTR_HANDLER;
824 if (ddi_intr_get_cap(iommu->aiomt_intr_htable[0], &intrcap0)
827 iommu->aiomt_intr_htable[iommu->aiomt_actual_intrs - 1], &intrcapN)
832 f, driver, instance, iommu->aiomt_idx, intrcap0, intrcapN);
833 amd_iommu_teardown_interrupts(iommu);
836 iommu->aiomt_intr_cap = intrcap0;
843 f, driver, instance, iommu->aiomt_idx);
845 if (ddi_intr_block_enable(iommu->aiomt_intr_htable,
846 iommu->aiomt_actual_intrs) != DDI_SUCCESS) {
849 instance, iommu->aiomt_idx);
850 (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
851 iommu->aiomt_actual_intrs);
852 amd_iommu_teardown_interrupts(iommu);
859 f, driver, instance, iommu->aiomt_idx);
861 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
862 if (ddi_intr_enable(iommu->aiomt_intr_htable[i])
866 driver, instance, iommu->aiomt_idx, i);
869 iommu->aiomt_intr_htable[j]);
871 amd_iommu_teardown_interrupts(iommu);
876 iommu->aiomt_intr_state = AMD_IOMMU_INTR_ENABLED;
881 f, driver, instance, iommu->aiomt_idx,
883 "(individually)", iommu->aiomt_actual_intrs);
890 amd_iommu_teardown_interrupts(amd_iommu_t *iommu)
894 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ENABLED) {
895 if (iommu->aiomt_intr_cap & DDI_INTR_FLAG_BLOCK) {
896 (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
897 iommu->aiomt_actual_intrs);
899 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
901 iommu->aiomt_intr_htable[i]);
906 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_HANDLER) {
907 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
909 iommu->aiomt_intr_htable[i]);
913 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ALLOCED) {
914 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
915 (void) ddi_intr_free(iommu->aiomt_intr_htable[i]);
918 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_TABLE) {
919 kmem_free(iommu->aiomt_intr_htable,
920 iommu->aiomt_intr_htable_sz);
922 iommu->aiomt_intr_htable = NULL;
923 iommu->aiomt_intr_htable_sz = 0;
924 iommu->aiomt_intr_state = AMD_IOMMU_INTR_INVALID;
931 amd_iommu_t *iommu;
954 iommu = kmem_zalloc(sizeof (amd_iommu_t), KM_SLEEP);
955 mutex_init(&iommu->aiomt_mutex, NULL, MUTEX_DRIVER, NULL);
956 mutex_enter(&iommu->aiomt_mutex);
958 mutex_init(&iommu->aiomt_cmdlock, NULL, MUTEX_DRIVER, NULL);
959 mutex_init(&iommu->aiomt_eventlock, NULL, MUTEX_DRIVER, NULL);
961 iommu->aiomt_dip = dip;
962 iommu->aiomt_idx = idx;
964 if (acpica_get_bdf(iommu->aiomt_dip, &bus, &device, &func)
972 iommu->aiomt_bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) |
982 iommu->aiomt_cap_hdr = caphdr;
983 iommu->aiomt_npcache = AMD_IOMMU_REG_GET32(&caphdr,
985 iommu->aiomt_httun = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_HTTUN);
988 hinfop = amd_iommu_lookup_any_ivhd(iommu);
991 iommu->aiomt_iotlb = hinfop->ach_IotlbSup;
993 iommu->aiomt_iotlb =
996 iommu->aiomt_captype = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
997 iommu->aiomt_capid = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
1004 iommu->aiomt_low_addr32 = low_addr32;
1005 iommu->aiomt_hi_addr32 = hi_addr32;
1009 iommu->aiomt_reg_pa = hinfop->ach_IOMMU_reg_base;
1012 iommu->aiomt_reg_pa = ((uint64_t)hi_addr32 << 32 | low_addr32);
1019 iommu->aiomt_range = range;
1020 iommu->aiomt_rng_valid = AMD_IOMMU_REG_GET32(&range,
1022 if (iommu->aiomt_rng_valid) {
1023 iommu->aiomt_rng_bus = AMD_IOMMU_REG_GET32(&range,
1025 iommu->aiomt_first_devfn = AMD_IOMMU_REG_GET32(&range,
1027 iommu->aiomt_last_devfn = AMD_IOMMU_REG_GET32(&range,
1030 iommu->aiomt_rng_bus = 0;
1031 iommu->aiomt_first_devfn = 0;
1032 iommu->aiomt_last_devfn = 0;
1036 iommu->aiomt_ht_unitid = hinfop->ach_IOMMU_UnitID;
1038 iommu->aiomt_ht_unitid = AMD_IOMMU_REG_GET32(&range,
1045 iommu->aiomt_misc = misc;
1048 iommu->aiomt_htatsresv = global->acg_HtAtsResv;
1049 iommu->aiomt_vasize = global->acg_VAsize;
1050 iommu->aiomt_pasize = global->acg_PAsize;
1052 iommu->aiomt_htatsresv = AMD_IOMMU_REG_GET32(&misc,
1054 iommu->aiomt_vasize = AMD_IOMMU_REG_GET32(&misc,
1056 iommu->aiomt_pasize = AMD_IOMMU_REG_GET32(&misc,
1061 iommu->aiomt_msinum = hinfop->ach_IOMMU_MSInum;
1063 iommu->aiomt_msinum =
1070 pgoffset = iommu->aiomt_reg_pa & MMU_PAGEOFFSET;
1072 iommu->aiomt_reg_pages = mmu_btopr(AMD_IOMMU_REG_SIZE + pgoffset);
1073 iommu->aiomt_reg_size = mmu_ptob(iommu->aiomt_reg_pages);
1075 iommu->aiomt_va = (uintptr_t)device_arena_alloc(
1076 ptob(iommu->aiomt_reg_pages), VM_SLEEP);
1077 if (iommu->aiomt_va == 0) {
1081 mutex_exit(&iommu->aiomt_mutex);
1082 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1086 hat_devload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
1087 iommu->aiomt_reg_size,
1088 mmu_btop(iommu->aiomt_reg_pa), PROT_READ | PROT_WRITE
1091 iommu->aiomt_reg_va = iommu->aiomt_va + pgoffset;
1096 iommu->aiomt_reg_devtbl_va = iommu->aiomt_reg_va +
1098 iommu->aiomt_reg_cmdbuf_va = iommu->aiomt_reg_va +
1100 iommu->aiomt_reg_eventlog_va = iommu->aiomt_reg_va +
1102 iommu->aiomt_reg_ctrl_va = iommu->aiomt_reg_va +
1104 iommu->aiomt_reg_excl_base_va = iommu->aiomt_reg_va +
1106 iommu->aiomt_reg_excl_lim_va = iommu->aiomt_reg_va +
1108 iommu->aiomt_reg_cmdbuf_head_va = iommu->aiomt_reg_va +
1110 iommu->aiomt_reg_cmdbuf_tail_va = iommu->aiomt_reg_va +
1112 iommu->aiomt_reg_eventlog_head_va = iommu->aiomt_reg_va +
1114 iommu->aiomt_reg_eventlog_tail_va = iommu->aiomt_reg_va +
1116 iommu->aiomt_reg_status_va = iommu->aiomt_reg_va +
1124 if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) {
1125 mutex_exit(&iommu->aiomt_mutex);
1126 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1130 if (amd_iommu_setup_exclusion(iommu) != DDI_SUCCESS) {
1131 mutex_exit(&iommu->aiomt_mutex);
1132 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1136 amd_iommu_enable_interrupts(iommu);
1138 if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) {
1139 mutex_exit(&iommu->aiomt_mutex);
1140 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1147 amd_iommu_init_page_tables(iommu);
1155 if (amd_iommu_setup_passthru(iommu) != DDI_SUCCESS) {
1156 mutex_exit(&iommu->aiomt_mutex);
1157 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1162 if (amd_iommu_acpi_init_devtbl(iommu) != DDI_SUCCESS) {
1165 mutex_exit(&iommu->aiomt_mutex);
1166 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1170 if (amd_iommu_start(iommu) != DDI_SUCCESS) {
1171 mutex_exit(&iommu->aiomt_mutex);
1172 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1177 if (amd_iommu_register(iommu) != DDI_SUCCESS) {
1178 mutex_exit(&iommu->aiomt_mutex);
1179 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1188 return (iommu);
1192 amd_iommu_fini(amd_iommu_t *iommu, int type)
1194 int idx = iommu->aiomt_idx;
1195 dev_info_t *dip = iommu->aiomt_dip;
1201 mutex_enter(&iommu->aiomt_mutex);
1202 if (amd_iommu_unregister(iommu) != DDI_SUCCESS) {
1209 amd_iommu_stop(iommu);
1212 amd_iommu_fini_page_tables(iommu);
1213 amd_iommu_teardown_interrupts(iommu);
1214 amd_iommu_teardown_exclusion(iommu);
1217 amd_iommu_teardown_tables_and_buffers(iommu, type);
1222 if (iommu->aiomt_va != NULL) {
1223 hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
1224 iommu->aiomt_reg_size, HAT_UNLOAD_UNLOCK);
1225 device_arena_free((void *)(uintptr_t)iommu->aiomt_va,
1226 ptob(iommu->aiomt_reg_pages));
1227 iommu->aiomt_va = NULL;
1228 iommu->aiomt_reg_va = NULL;
1230 mutex_destroy(&iommu->aiomt_eventlock);
1231 mutex_destroy(&iommu->aiomt_cmdlock);
1232 mutex_exit(&iommu->aiomt_mutex);
1233 mutex_destroy(&iommu->aiomt_mutex);
1234 kmem_free(iommu, sizeof (amd_iommu_t));
1257 amd_iommu_t *iommu;
1322 iommu = amd_iommu_init(dip, handle, idx, cap_base);
1323 if (iommu == NULL) {
1331 statep->aioms_iommu_start = iommu;
1333 statep->aioms_iommu_end->aiomt_next = iommu;
1335 statep->aioms_iommu_end = iommu;
1355 amd_iommu_t *iommu, *next_iommu;
1361 for (iommu = statep->aioms_iommu_start; iommu;
1362 iommu = next_iommu) {
1364 next_iommu = iommu->aiomt_next;
1365 if (amd_iommu_fini(iommu, type) != DDI_SUCCESS) {
1427 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1429 int instance = ddi_get_instance(iommu->aiomt_dip);
1430 const char *idriver = ddi_driver_name(iommu->aiomt_dip);
1440 amd_iommu_set_passthru(iommu, rdip);
1454 f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
1463 f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
1476 if (hinfop && hinfop->ach_IOMMU_deviceid == iommu->aiomt_bdf)
1502 map_current_window(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
1506 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1507 int instance = ddi_get_instance(iommu->aiomt_dip);
1508 int idx = iommu->aiomt_idx;
1531 if ((error = amd_iommu_map_pa2va(iommu, rdip, attrp, dmareq,
1544 (void) unmap_current_window(iommu, rdip, cookie_array,
1562 unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
1565 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1566 int instance = ddi_get_instance(iommu->aiomt_dip);
1567 int idx = iommu->aiomt_idx;
1590 if (amd_iommu_unmap_va(iommu, rdip,
1598 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, NULL, 0, 0)
1636 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1677 error = map_current_window(iommu, rdip, attrp, dmareq,
1715 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1748 if (unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0)
1818 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1856 (void) unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0);
1878 error = map_current_window(iommu, rdip, attrp, &sdmareq,
1976 * if "amd-iommu = no/false" boot property is set,
1977 * ignore AMD iommu
1980 DDI_PROP_DONTPASS, "amd-iommu", &propval) == DDI_SUCCESS) {
1992 DDI_PROP_DONTPASS, "amd-iommu-disable-list", &propval)
2008 DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu", &disable)
2017 DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu-disable-list",