io/pci/pci_dma.c

354 pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, iommu_t *iommu_p,
359 	uint64_t syslo		= iommu_p->iommu_dvma_base;
360 	uint64_t syshi		= iommu_p->iommu_dvma_end;
361 	uint64_t fasthi		= iommu_p->iommu_dvma_fast_end;
424 	iommu_t *iommu_p = pci_p->pci_iommu_p;
455 		syslo = iommu_p->iommu_dma_bypass_base;
456 		syshi = iommu_p->iommu_dma_bypass_end;
467 		syslo = iommu_p->iommu_dvma_base;
468 		syshi = iommu_p->iommu_dvma_end;
498 		syshi = iommu_p->iommu_dvma_fast_end;
906 pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp)
912 	int i = iommu_p->iommu_dvma_addr_scan_start;
913 	uint8_t *lock_addr = iommu_p->iommu_dvma_cache_locks + i;
927 		lock_addr = iommu_p->iommu_dvma_cache_locks;
938 	iommu_p->iommu_dvma_addr_scan_start = (i + 1) & (entries - 1);
946 	tte_addr = iommu_p->iommu_tsb_vaddr + i;
947 	dvma_pg = iommu_p->dvma_base_pg + i;
971 	if (DVMA_DBG_ON(iommu_p))
972 		pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping,
982 pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, iommu_t *iommu_p)
999 		dvma_addr = vmem_alloc(iommu_p->iommu_dvma_map,
1006 		dvma_addr = vmem_xalloc(iommu_p->iommu_dvma_map,
1019 	dvma_pg_index = dvma_pg - iommu_p->dvma_base_pg;
1028 		if (ctx = pci_iommu_get_dvma_context(iommu_p, dvma_pg_index)) {
1036 	iommu_map_pages(iommu_p, mp, dvma_pg, npages, 0);
1042 		    &iommu_p->iommu_dvma_clid);
1049 pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp)
1057 		iopfn_t index = dvma_pg - iommu_p->dvma_base_pg;
1061 		iommu_p->iommu_dvma_cache_locks[index] = 0;
1068 	pci_vmem_free(iommu_p, mp, (void *)dvma_addr, npages);
1071 		pci_iommu_free_dvma_context(iommu_p, MP2CTX(mp));
1078 	iommu_t *iommu_p = pci_p->pci_iommu_p;
1085 		SYNC_BUF_PA(mp) = IOMMU_PAGE_TTEPA(iommu_p, dvma_pg);
1092 		iommu_unmap_window(iommu_p, mp);
1094 		iommu_unmap_window(iommu_p, mp);
1687 pci_dvma_debug_init(iommu_t *iommu_p)
1690 	ASSERT(MUTEX_HELD(&iommu_p->dvma_debug_lock));
1691 	cmn_err(CE_NOTE, "PCI DVMA %p stat ON", iommu_p);
1693 	iommu_p->dvma_alloc_rec = kmem_zalloc(sz, KM_SLEEP);
1694 	iommu_p->dvma_free_rec = kmem_zalloc(sz, KM_SLEEP);
1696 	iommu_p->dvma_active_list = NULL;
1697 	iommu_p->dvma_alloc_rec_index = 0;
1698 	iommu_p->dvma_free_rec_index = 0;
1699 	iommu_p->dvma_active_count = 0;
1703 pci_dvma_debug_fini(iommu_t *iommu_p)
1707 	uint64_t mask = ~(1ull << iommu_p->iommu_inst);
1708 	cmn_err(CE_NOTE, "PCI DVMA %p stat OFF", iommu_p);
1710 	kmem_free(iommu_p->dvma_alloc_rec, sz);
1711 	kmem_free(iommu_p->dvma_free_rec, sz);
1712 	iommu_p->dvma_alloc_rec = iommu_p->dvma_free_rec = NULL;
1714 	prev = iommu_p->dvma_active_list;
1721 	iommu_p->dvma_active_list = NULL;
1722 	iommu_p->dvma_alloc_rec_index = 0;
1723 	iommu_p->dvma_free_rec_index = 0;
1724 	iommu_p->dvma_active_count = 0;
1731 pci_dvma_alloc_debug(iommu_t *iommu_p, char *address, uint_t len,
1735 	mutex_enter(&iommu_p->dvma_debug_lock);
1737 	if (!iommu_p->dvma_alloc_rec)
1738 		pci_dvma_debug_init(iommu_p);
1739 	if (DVMA_DBG_OFF(iommu_p)) {
1740 		pci_dvma_debug_fini(iommu_p);
1744 	ptr = &iommu_p->dvma_alloc_rec[iommu_p->dvma_alloc_rec_index];
1748 	if (++iommu_p->dvma_alloc_rec_index == pci_dvma_debug_rec)
1749 		iommu_p->dvma_alloc_rec_index = 0;
1756 	ptr->next = iommu_p->dvma_active_list;
1757 	iommu_p->dvma_active_list = ptr;
1758 	iommu_p->dvma_active_count++;
1760 	mutex_exit(&iommu_p->dvma_debug_lock);
1764 pci_dvma_free_debug(iommu_t *iommu_p, char *address, uint_t len,
1768 	mutex_enter(&iommu_p->dvma_debug_lock);
1770 	if (!iommu_p->dvma_alloc_rec)
1771 		pci_dvma_debug_init(iommu_p);
1772 	if (DVMA_DBG_OFF(iommu_p)) {
1773 		pci_dvma_debug_fini(iommu_p);
1777 	ptr = &iommu_p->dvma_free_rec[iommu_p->dvma_free_rec_index];
1781 	if (++iommu_p->dvma_free_rec_index == pci_dvma_debug_rec)
1782 		iommu_p->dvma_free_rec_index = 0;
1784 	ptr_save = iommu_p->dvma_active_list;
1795 	if (ptr == iommu_p->dvma_active_list)
1796 		iommu_p->dvma_active_list = ptr->next;
1800 	iommu_p->dvma_active_count--;
1802 	mutex_exit(&iommu_p->dvma_debug_lock);
1823 pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages,
1826 	vmem_t *map_p = iommu_p->iommu_dvma_map;