i86pc/io/immu_dvma.c

44 #include <sys/immu.h>
71 static domain_t *domain_create(immu_t *immu, dev_info_t *ddip,
254 immu_devi_set_spclist(dev_info_t *dip, immu_t *immu)
261 		spclist = &(immu->immu_dvma_gfx_list);
263 		spclist = &(immu->immu_dvma_lpc_list);
267 		mutex_enter(&(immu->immu_lock));
269 		mutex_exit(&(immu->immu_lock));
327 get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)
330 	dvarg.dva_list = &(immu->immu_dvma_lpc_list);
353 	immu_t *immu;
362 	immu = list_head(&immu_list);
363 	for (; immu; immu = list_next(&immu_list, immu)) {
364 		list_gfx = &(immu->immu_dvma_gfx_list);
432 	immu_t *immu = arg;
450 	if (!immu->immu_dvma_coherent)
511 pgtable_alloc(immu_t *immu, immu_flags_t immu_flags)
518 	pgtable = kmem_cache_alloc(immu->immu_pgtable_cache, kmflags);
533 pgtable_free(immu_t *immu, pgtable_t *pgtable)
535 	kmem_cache_free(immu->immu_pgtable_cache, pgtable);
614  *   get the immu unit structure for a dev_info node
620 	immu_t *immu;
623 	 * check if immu unit was already found earlier.
642 		immu = immu_devi->imd_immu;
644 		return (immu);
648 	immu = immu_dmar_get_immu(dip);
649 	if (immu == NULL) {
656 	 * Check if some other thread found immu
670 		immu_devi->imd_immu = immu;
671 		immu_devi_set_spclist(dip, immu);
674 		 * if some other thread got immu before
677 		if (immu_devi->imd_immu != immu) {
679 			    "immu units found for device. Expected (%p), "
680 			    "actual (%p)", (void *)immu,
688 	return (immu);
811 did_alloc(immu_t *immu, dev_info_t *rdip,
816 	did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1,
821 		    " domain-device: %s%d. immu unit is %s. Using "
824 		    immu->immu_name, immu->immu_unity_domain->dom_did);
825 		did = immu->immu_unity_domain->dom_did;
837 	immu_t *immu;
858 	immu = immu_devi->imd_immu;
859 	if (immu == NULL)
860 		immu = immu_dvma_get_immu(pdip, dvp->dva_flags);
886 		dvp->dva_domain = immu->immu_unity_domain;
1025 create_xlate_arena(immu_t *immu, domain_t *domain,
1042 	    "%s-domain-%d-xlate-DVMA-arena", immu->immu_name,
1048 	mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);
1180 	immu_t *immu;
1198 	immu = immu_dvma_get_immu(rdip, immu_flags);
1199 	if (immu == NULL) {
1252 	domain = domain_create(immu, ddip, rdip, immu_flags);
1272 create_unity_domain(immu_t *immu)
1284 	domain->dom_immu = immu;
1285 	immu->immu_unity_domain = domain;
1291 	domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1299 	if (!IMMU_ECAP_GET_PT(immu->immu_regs_excap))
1318 domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,
1336 	did = did_alloc(immu, rdip, ddip, immu_flags);
1339 		ASSERT(immu->immu_unity_domain);
1340 		return (immu->immu_unity_domain);
1347 		    "structure for device. IOMMU unit: %s", immu->immu_name);
1354 	    "immu%s-domain%d-pava-hash", immu->immu_name, did);
1357 	domain->dom_immu = immu;
1364 	create_xlate_arena(immu, domain, rdip, immu_flags);
1369 	domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);
1373 		    domain->dom_did, immu->immu_name);
1379 	 * Since this is a immu unit-specific domain, put it on
1380 	 * the per-immu domain list.
1382 	mutex_enter(&(immu->immu_lock));
1383 	list_insert_head(&immu->immu_domain_list, domain);
1384 	mutex_exit(&(immu->immu_lock));
1424 did_init(immu_t *immu)
1426 	(void) snprintf(immu->immu_did_arena_name,
1427 	    sizeof (immu->immu_did_arena_name),
1428 	    "%s_domainid_arena", immu->immu_name);
1430 	ddi_err(DER_VERB, immu->immu_dip, "creating domainid arena %s",
1431 	    immu->immu_did_arena_name);
1433 	immu->immu_did_arena = vmem_create(
1434 	    immu->immu_did_arena_name,
1436 	    immu->immu_max_domains - IMMU_UNITY_DID,
1445 	if (immu->immu_did_arena == NULL) {
1447 		    "IOMMU domainid allocator: %s", immu->immu_name,
1448 		    immu->immu_did_arena_name);
1455 context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,
1478 	rw_enter(&(immu->immu_ctx_rwlock), RW_READER);
1484 			rw_exit(&(immu->immu_ctx_rwlock));
1495 	    rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) {
1496 		rw_exit(&(immu->immu_ctx_rwlock));
1497 		rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
1506 		immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t));
1515 		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
1518 		immu_flush_context_fsi(immu, 0, sid, domain->dom_did,
1519 		    &immu->immu_ctx_inv_wait);
1523 		CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
1526 		    IMMU_ECAP_GET_PT(immu->immu_regs_excap))
1532 		if (IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {
1537 		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
1539 	rw_exit(&(immu->immu_ctx_rwlock));
1543 context_create(immu_t *immu)
1555 	root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1565 		context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1573 			    immu->immu_unity_domain->dom_pgtable_root;
1575 			    immu->immu_unity_domain->dom_did);
1576 			CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
1578 			if (IMMU_ECAP_GET_PT(immu->immu_regs_excap))
1586 		immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE);
1597 context_init(immu_t *immu)
1599 	rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL);
1601 	immu_init_inv_wait(&immu->immu_ctx_inv_wait, "ctxglobal", B_TRUE);
1603 	immu_regs_wbf_flush(immu);
1605 	immu->immu_ctx_root = context_create(immu);
1607 	immu_regs_set_root_table(immu);
1609 	rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
1610 	immu_flush_context_gbl(immu, &immu->immu_ctx_inv_wait);
1611 	immu_flush_iotlb_gbl(immu, &immu->immu_ctx_inv_wait);
1612 	rw_exit(&(immu->immu_ctx_rwlock));
1635 immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,
1683 		context_set(immu, domain, immu->immu_ctx_root, r_bus,
1692 		context_set(immu, domain, immu->immu_ctx_root, r_bus,
1702 		context_set(immu, domain, immu->immu_ctx_root,
1705 			context_set(immu, domain, immu->immu_ctx_root,
1715 				context_set(immu, domain, immu->immu_ctx_root,
1724 		context_set(immu, domain, immu->immu_ctx_root, d_bus,
1731 		context_set(immu, domain, immu->immu_ctx_root, d_bus,
1749 PDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr,
1769 	if (next == NULL && immu->immu_TM_reserved == B_FALSE) {
1808 	if (next == NULL && immu->immu_SNP_reserved == B_FALSE) {
1852 PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
1977 	immu_t *immu;
1987 	immu = domain->dom_immu;
1996 	nlevels = immu->immu_dvma_nlevels;
2017 PTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr,
2023 	pte = immu->immu_ptemask;
2041 	if (immu->immu_TM_reserved == B_FALSE) {
2054 	if (immu->immu_SNP_reserved == B_FALSE) {
2081 	pte |= immu->immu_ptemask;
2100 PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
2141 		PTE_set_one(immu, hwp, paddr, rdip, immu_flags);
2144 		ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)
2177 PDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next,
2247 PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,
2285 				new = pgtable_alloc(immu, immu_flags);
2305 			PDE_set_one(immu, hwp, next, rdip, immu_flags);
2329 		ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)
2337 		pgtable_free(immu, new);
2347  *     immu: IOMMU unit for which we are generating DVMA cookies
2362 	immu_t *immu = domain->dom_immu;
2363 	int nlevels = immu->immu_dvma_nlevels;
2374 		if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)
2380 		PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,
2391  * immu: IOMMU unit state
2402 	immu_t *immu = domain->dom_immu;
2403 	int nlevels = immu->immu_dvma_nlevels;
2422 		PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);
2466 	immu_t *immu;
2471 	immu = domain->dom_immu;
2472 	nlevels = immu->immu_dvma_nlevels;
2505 		(void) PDE_set_all(immu, domain, xlate, nlevels, rdip,
2513 		PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie,
2561 	immu_t *immu;
2577 	immu = domain->dom_immu;
2625 		rwmask = PDTE_MASK_R | PDTE_MASK_W | immu->immu_ptemask;
2627 		rwmask = immu->immu_ptemask;
2722 	immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc,
2782 	immu_t *immu;
2804 	immu = list_head(listp);
2805 	for (; immu; immu = list_next(listp, immu)) {
2806 		create_unity_domain(immu);
2807 		did_init(immu);
2808 		context_init(immu);
2809 		immu->immu_dvma_setup = B_TRUE;
2817 immu_dvma_startup(immu_t *immu)
2820 	    immu->immu_dvma_gfx_only == B_TRUE) {
2827 	immu->immu_dvma_running = B_TRUE;
2882 	immu_t *immu;
2887 	immu = immu_dvma_get_immu(rdip, immu_flags);
2888 	if (immu == NULL) {
2901 		rdip = get_lpc_devinfo(immu, rdip, immu_flags);
2908 	/* Reset immu, as redirection can change IMMU */
2909 	immu = NULL;
2934 	immu = domain->dom_immu;
2953 	if (immu_context_update(immu, domain, ddip, rdip, immu_flags)
2967 	immu_t *immu;
2975 	immu = domain->dom_immu;
2983 	immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start,
3066 	immu_t *immu;
3071 		immu = IMMU_DEVI(rdip)->imd_immu;
3073 		ihp = kmem_cache_alloc(immu->immu_hdl_cache,