io/pci/pci_iommu.c

45 static void iommu_tlb_flushall(iommu_t *iommu_p);
46 static void iommu_preserve_tsb(iommu_t *iommu_p);
52 	iommu_t *iommu_p;
66 	iommu_p = (iommu_t *)kmem_zalloc(sizeof (iommu_t), KM_SLEEP);
67 	pci_p->pci_iommu_p = iommu_p;
68 	iommu_p->iommu_pci_p = pci_p;
69 	iommu_p->iommu_inst = ddi_get_instance(dip);
74 	iommu_p->iommu_dvma_end = pci_iommu_dvma_end;
75 	a = pci_iommu_setup(iommu_p);
80 	iommu_p->iommu_ctrl_reg =
82 	iommu_p->iommu_tsb_base_addr_reg =
84 	iommu_p->iommu_flush_page_reg =
91 	iommu_p->iommu_tsb_vaddr = /* retrieve TSB VA reserved by system */
93 	iommu_p->iommu_tsb_entries = tsb_entries =
94 		IOMMU_TSBSIZE_TO_TSBENTRIES(iommu_p->iommu_tsb_size);
95 	iommu_p->iommu_tsb_paddr = va_to_pa((caddr_t)iommu_p->iommu_tsb_vaddr);
96 	iommu_p->iommu_dvma_cache_locks =
99 	iommu_p->iommu_dvma_base = iommu_p->iommu_dvma_end + 1
101 	iommu_p->dvma_base_pg = IOMMU_BTOP(iommu_p->iommu_dvma_base);
102 	iommu_p->iommu_dvma_reserve = tsb_entries >> 1;
103 	iommu_p->dvma_end_pg = IOMMU_BTOP(iommu_p->iommu_dvma_end);
104 	iommu_p->iommu_dma_bypass_base = COMMON_IOMMU_BYPASS_BASE;
105 	iommu_p->iommu_dma_bypass_end = pci_iommu_bypass_end_configure();
111 	pci_dvma_range.dvma_base = (uint32_t)iommu_p->iommu_dvma_base;
113 		iommu_p->iommu_dvma_end - iommu_p->iommu_dvma_base + 1;
119 		iommu_p->iommu_ctrl_reg, iommu_p->iommu_tsb_base_addr_reg);
121 		iommu_p->iommu_flush_page_reg, iommu_p->iommu_flush_ctx_reg);
123 		iommu_p->iommu_tsb_vaddr, iommu_p->iommu_tsb_paddr);
127 		iommu_p->iommu_tsb_vaddr,
131 		iommu_p->iommu_tsb_size, iommu_p->iommu_tsb_entries,
132 		iommu_p->iommu_dvma_base);
135 		iommu_p->iommu_dvma_cache_locks, pci_dvma_page_cache_entries);
140 	bzero(iommu_p->iommu_tsb_vaddr, tsb_entries << 3);
151 	iommu_p->iommu_dvma_fast_end = iommu_p->iommu_dvma_base +
153 	iommu_p->iommu_dvma_map = vmem_create(map_name,
154 		(void *)(iommu_p->iommu_dvma_fast_end + 1),
158 	mutex_init(&iommu_p->dvma_debug_lock, NULL, MUTEX_DRIVER, NULL);
164 	if (pci_preserve_iommu_tsb && *iommu_p->iommu_tsb_base_addr_reg)
165 		iommu_preserve_tsb(iommu_p);
167 	iommu_configure(iommu_p);
176 	iommu_t *iommu_p = pci_p->pci_iommu_p;
177 	volatile uint64_t ctl_val = *iommu_p->iommu_ctrl_reg;
187 	*iommu_p->iommu_ctrl_reg = ctl_val;
188 	*iommu_p->iommu_tsb_base_addr_reg = 0;
199 	pci_iommu_teardown(iommu_p);
201 	if (DVMA_DBG_ON(iommu_p))
202 		pci_dvma_debug_fini(iommu_p);
203 	mutex_destroy(&iommu_p->dvma_debug_lock);
208 	vmem_destroy(iommu_p->iommu_dvma_map);
210 	kmem_free(iommu_p->iommu_dvma_cache_locks,
216 	kmem_free(iommu_p, sizeof (iommu_t));
225 iommu_configure(iommu_t *iommu_p)
227 	pci_t *pci_p = iommu_p->iommu_pci_p;
229 	dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
232 		((iommu_p->iommu_tsb_size << COMMON_IOMMU_CTRL_TSB_SZ_SHIFT) |
240 	if (!pci_preserve_iommu_tsb || !(*iommu_p->iommu_tsb_base_addr_reg)) {
241 		*iommu_p->iommu_ctrl_reg = COMMON_IOMMU_CTRL_DIAG_ENABLE;
242 		iommu_tlb_flushall(iommu_p);
258 	pci_iommu_config(iommu_p, ctl_val, cdip ? cfgpa : 0);
262 iommu_map_pages(iommu_t *iommu_p, ddi_dma_impl_t *mp,
266 	dvma_addr_t pg_index = dvma_pg - iommu_p->dvma_base_pg;
267 	uint64_t *tte_addr = iommu_p->iommu_tsb_vaddr + pg_index;
271 	dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
278 		(uint_t)iommu_p->dvma_base_pg, (uint_t)pg_index, dvma_pg,
299 	ASSERT(tte_addr == iommu_p->iommu_tsb_vaddr + pg_index);
304 		ASSERT(TTE_IS_INVALID(iommu_p->iommu_tsb_vaddr[pg_index]));
307 	if (DVMA_DBG_ON(iommu_p))
308 		pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping,
321 iommu_map_window(iommu_t *iommu_p, ddi_dma_impl_t *mp, window_t win_no)
341 	iommu_map_pages(iommu_p, mp, dvma_pg, IOMMU_BTOPR(win_size), pfn_index);
345 iommu_unmap_pages(iommu_t *iommu_p, dvma_addr_t dvma_pg, uint_t npages)
347 	dvma_addr_t pg_index = IOMMU_PAGE_INDEX(iommu_p, dvma_pg);
351 		IOMMU_UNLOAD_TTE(iommu_p, pg_index);
354 			IOMMU_PAGE_FLUSH(iommu_p, dvma_pg);
359 iommu_remap_pages(iommu_t *iommu_p, ddi_dma_impl_t *mp, dvma_addr_t dvma_pg,
362 	iommu_unmap_pages(iommu_p, dvma_pg, npages);
363 	iommu_map_pages(iommu_p, mp, dvma_pg, npages, pfn_index);
379 iommu_unmap_window(iommu_t *iommu_p, ddi_dma_impl_t *mp)
382 	dvma_addr_t pg_index = IOMMU_PAGE_INDEX(iommu_p, dvma_pg);
385 	dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
396 			IOMMU_UNLOAD_TTE(iommu_p, pg_index);
399 		*iommu_p->iommu_flush_ctx_reg = ctx;
401 		iommu_unmap_pages(iommu_p, dvma_pg, npages);
405 	if (DVMA_DBG_ON(iommu_p))
406 		pci_dvma_free_debug(iommu_p, (char *)mp->dmai_mapping,
464 iommu_tlb_flushall(iommu_t *iommu_p)
467 	uint64_t base = (uint64_t)(iommu_p->iommu_ctrl_reg) -
478 iommu_preserve_tsb(iommu_t *iommu_p)
481 	dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
484 	uint64_t ctl = *iommu_p->iommu_ctrl_reg;
485 	uint64_t obp_tsb_pa = *iommu_p->iommu_tsb_base_addr_reg;
490 		iommu_p->iommu_tsb_base_addr_reg,
491 		(uint32_t)(*iommu_p->iommu_tsb_base_addr_reg >> 32),
492 		(uint32_t)(*iommu_p->iommu_tsb_base_addr_reg & 0xffffffff));
496 	base_pg_index = iommu_p->dvma_end_pg - obp_tsb_entries + 1;
497 	base_tte_addr = iommu_p->iommu_tsb_vaddr +
498 		(iommu_p->iommu_tsb_entries - obp_tsb_entries);
504 		(CHIP_TYPE(iommu_p->iommu_pci_p) == PCI_CHIP_SABRE))
509 		"base_pg_index=%x\n", iommu_p->iommu_tsb_vaddr,
536 		(void) vmem_xalloc(iommu_p->iommu_dvma_map, IOMMU_PAGE_SIZE,