1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Implementation of the IOMMU SVA API for the ARM SMMUv3 4 */ 5 6#include <linux/mm.h> 7#include <linux/mmu_context.h> 8#include <linux/mmu_notifier.h> 9#include <linux/sched/mm.h> 10#include <linux/slab.h> 11 12#include "arm-smmu-v3.h" 13#include "../../io-pgtable-arm.h" 14 15struct arm_smmu_mmu_notifier { 16 struct mmu_notifier mn; 17 struct arm_smmu_ctx_desc *cd; 18 bool cleared; 19 refcount_t refs; 20 struct list_head list; 21 struct arm_smmu_domain *domain; 22}; 23 24#define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn) 25 26struct arm_smmu_bond { 27 struct mm_struct *mm; 28 struct arm_smmu_mmu_notifier *smmu_mn; 29 struct list_head list; 30}; 31 32#define sva_to_bond(handle) \ 33 container_of(handle, struct arm_smmu_bond, sva) 34 35static DEFINE_MUTEX(sva_lock); 36 37/* 38 * Write the CD to the CD tables for all masters that this domain is attached 39 * to. Note that this is only used to update existing CD entries in the target 40 * CD table, for which it's assumed that arm_smmu_write_ctx_desc can't fail. 41 */ 42static void arm_smmu_update_ctx_desc_devices(struct arm_smmu_domain *smmu_domain, 43 int ssid, 44 struct arm_smmu_ctx_desc *cd) 45{ 46 struct arm_smmu_master *master; 47 unsigned long flags; 48 49 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 50 list_for_each_entry(master, &smmu_domain->devices, domain_head) { 51 arm_smmu_write_ctx_desc(master, ssid, cd); 52 } 53 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 54} 55 56/* 57 * Check if the CPU ASID is available on the SMMU side. If a private context 58 * descriptor is using it, try to replace it. 59 */ 60static struct arm_smmu_ctx_desc * 61arm_smmu_share_asid(struct mm_struct *mm, u16 asid) 62{ 63 int ret; 64 u32 new_asid; 65 struct arm_smmu_ctx_desc *cd; 66 struct arm_smmu_device *smmu; 67 struct arm_smmu_domain *smmu_domain; 68 69 cd = xa_load(&arm_smmu_asid_xa, asid); 70 if (!cd) 71 return NULL; 72 73 if (cd->mm) { 74 if (WARN_ON(cd->mm != mm)) 75 return ERR_PTR(-EINVAL); 76 /* All devices bound to this mm use the same cd struct. */ 77 refcount_inc(&cd->refs); 78 return cd; 79 } 80 81 smmu_domain = container_of(cd, struct arm_smmu_domain, cd); 82 smmu = smmu_domain->smmu; 83 84 ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd, 85 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 86 if (ret) 87 return ERR_PTR(-ENOSPC); 88 /* 89 * Race with unmap: TLB invalidations will start targeting the new ASID, 90 * which isn't assigned yet. We'll do an invalidate-all on the old ASID 91 * later, so it doesn't matter. 92 */ 93 cd->asid = new_asid; 94 /* 95 * Update ASID and invalidate CD in all associated masters. There will 96 * be some overlap between use of both ASIDs, until we invalidate the 97 * TLB. 98 */ 99 arm_smmu_update_ctx_desc_devices(smmu_domain, IOMMU_NO_PASID, cd); 100 101 /* Invalidate TLB entries previously associated with that context */ 102 arm_smmu_tlb_inv_asid(smmu, asid); 103 104 xa_erase(&arm_smmu_asid_xa, asid); 105 return NULL; 106} 107 108static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) 109{ 110 u16 asid; 111 int err = 0; 112 u64 tcr, par, reg; 113 struct arm_smmu_ctx_desc *cd; 114 struct arm_smmu_ctx_desc *ret = NULL; 115 116 /* Don't free the mm until we release the ASID */ 117 mmgrab(mm); 118 119 asid = arm64_mm_context_get(mm); 120 if (!asid) { 121 err = -ESRCH; 122 goto out_drop_mm; 123 } 124 125 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 126 if (!cd) { 127 err = -ENOMEM; 128 goto out_put_context; 129 } 130 131 refcount_set(&cd->refs, 1); 132 133 mutex_lock(&arm_smmu_asid_lock); 134 ret = arm_smmu_share_asid(mm, asid); 135 if (ret) { 136 mutex_unlock(&arm_smmu_asid_lock); 137 goto out_free_cd; 138 } 139 140 err = xa_insert(&arm_smmu_asid_xa, asid, cd, GFP_KERNEL); 141 mutex_unlock(&arm_smmu_asid_lock); 142 143 if (err) 144 goto out_free_asid; 145 146 tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | 147 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | 148 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) | 149 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) | 150 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; 151 152 switch (PAGE_SIZE) { 153 case SZ_4K: 154 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K); 155 break; 156 case SZ_16K: 157 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K); 158 break; 159 case SZ_64K: 160 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K); 161 break; 162 default: 163 WARN_ON(1); 164 err = -EINVAL; 165 goto out_free_asid; 166 } 167 168 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 169 par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); 170 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par); 171 172 cd->ttbr = virt_to_phys(mm->pgd); 173 cd->tcr = tcr; 174 /* 175 * MAIR value is pretty much constant and global, so we can just get it 176 * from the current CPU register 177 */ 178 cd->mair = read_sysreg(mair_el1); 179 cd->asid = asid; 180 cd->mm = mm; 181 182 return cd; 183 184out_free_asid: 185 arm_smmu_free_asid(cd); 186out_free_cd: 187 kfree(cd); 188out_put_context: 189 arm64_mm_context_put(mm); 190out_drop_mm: 191 mmdrop(mm); 192 return err < 0 ? ERR_PTR(err) : ret; 193} 194 195static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) 196{ 197 if (arm_smmu_free_asid(cd)) { 198 /* Unpin ASID */ 199 arm64_mm_context_put(cd->mm); 200 mmdrop(cd->mm); 201 kfree(cd); 202 } 203} 204 205/* 206 * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this 207 * is used as a threshold to replace per-page TLBI commands to issue in the 208 * command queue with an address-space TLBI command, when SMMU w/o a range 209 * invalidation feature handles too many per-page TLBI commands, which will 210 * otherwise result in a soft lockup. 211 */ 212#define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3)) 213 214static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, 215 struct mm_struct *mm, 216 unsigned long start, 217 unsigned long end) 218{ 219 struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); 220 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 221 size_t size; 222 223 /* 224 * The mm_types defines vm_end as the first byte after the end address, 225 * different from IOMMU subsystem using the last address of an address 226 * range. So do a simple translation here by calculating size correctly. 227 */ 228 size = end - start; 229 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) { 230 if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE) 231 size = 0; 232 } else { 233 if (size == ULONG_MAX) 234 size = 0; 235 } 236 237 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) { 238 if (!size) 239 arm_smmu_tlb_inv_asid(smmu_domain->smmu, 240 smmu_mn->cd->asid); 241 else 242 arm_smmu_tlb_inv_range_asid(start, size, 243 smmu_mn->cd->asid, 244 PAGE_SIZE, false, 245 smmu_domain); 246 } 247 248 arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), start, 249 size); 250} 251 252static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 253{ 254 struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); 255 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 256 257 mutex_lock(&sva_lock); 258 if (smmu_mn->cleared) { 259 mutex_unlock(&sva_lock); 260 return; 261 } 262 263 /* 264 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, 265 * but disable translation. 266 */ 267 arm_smmu_update_ctx_desc_devices(smmu_domain, mm_get_enqcmd_pasid(mm), 268 &quiet_cd); 269 270 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid); 271 arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), 0, 0); 272 273 smmu_mn->cleared = true; 274 mutex_unlock(&sva_lock); 275} 276 277static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) 278{ 279 kfree(mn_to_smmu(mn)); 280} 281 282static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { 283 .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs, 284 .release = arm_smmu_mm_release, 285 .free_notifier = arm_smmu_mmu_notifier_free, 286}; 287 288/* Allocate or get existing MMU notifier for this {domain, mm} pair */ 289static struct arm_smmu_mmu_notifier * 290arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain, 291 struct mm_struct *mm) 292{ 293 int ret; 294 struct arm_smmu_ctx_desc *cd; 295 struct arm_smmu_mmu_notifier *smmu_mn; 296 297 list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) { 298 if (smmu_mn->mn.mm == mm) { 299 refcount_inc(&smmu_mn->refs); 300 return smmu_mn; 301 } 302 } 303 304 cd = arm_smmu_alloc_shared_cd(mm); 305 if (IS_ERR(cd)) 306 return ERR_CAST(cd); 307 308 smmu_mn = kzalloc(sizeof(*smmu_mn), GFP_KERNEL); 309 if (!smmu_mn) { 310 ret = -ENOMEM; 311 goto err_free_cd; 312 } 313 314 refcount_set(&smmu_mn->refs, 1); 315 smmu_mn->cd = cd; 316 smmu_mn->domain = smmu_domain; 317 smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops; 318 319 ret = mmu_notifier_register(&smmu_mn->mn, mm); 320 if (ret) { 321 kfree(smmu_mn); 322 goto err_free_cd; 323 } 324 325 list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers); 326 return smmu_mn; 327 328err_free_cd: 329 arm_smmu_free_shared_cd(cd); 330 return ERR_PTR(ret); 331} 332 333static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn) 334{ 335 struct mm_struct *mm = smmu_mn->mn.mm; 336 struct arm_smmu_ctx_desc *cd = smmu_mn->cd; 337 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 338 339 if (!refcount_dec_and_test(&smmu_mn->refs)) 340 return; 341 342 list_del(&smmu_mn->list); 343 344 /* 345 * If we went through clear(), we've already invalidated, and no 346 * new TLB entry can have been formed. 347 */ 348 if (!smmu_mn->cleared) { 349 arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid); 350 arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), 0, 351 0); 352 } 353 354 /* Frees smmu_mn */ 355 mmu_notifier_put(&smmu_mn->mn); 356 arm_smmu_free_shared_cd(cd); 357} 358 359static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, 360 struct mm_struct *mm) 361{ 362 int ret; 363 struct arm_smmu_bond *bond; 364 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 365 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 366 struct arm_smmu_domain *smmu_domain; 367 368 if (!(domain->type & __IOMMU_DOMAIN_PAGING)) 369 return -ENODEV; 370 smmu_domain = to_smmu_domain(domain); 371 if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) 372 return -ENODEV; 373 374 if (!master || !master->sva_enabled) 375 return -ENODEV; 376 377 bond = kzalloc(sizeof(*bond), GFP_KERNEL); 378 if (!bond) 379 return -ENOMEM; 380 381 bond->mm = mm; 382 383 bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm); 384 if (IS_ERR(bond->smmu_mn)) { 385 ret = PTR_ERR(bond->smmu_mn); 386 goto err_free_bond; 387 } 388 389 ret = arm_smmu_write_ctx_desc(master, pasid, bond->smmu_mn->cd); 390 if (ret) 391 goto err_put_notifier; 392 393 list_add(&bond->list, &master->bonds); 394 return 0; 395 396err_put_notifier: 397 arm_smmu_mmu_notifier_put(bond->smmu_mn); 398err_free_bond: 399 kfree(bond); 400 return ret; 401} 402 403bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 404{ 405 unsigned long reg, fld; 406 unsigned long oas; 407 unsigned long asid_bits; 408 u32 feat_mask = ARM_SMMU_FEAT_COHERENCY; 409 410 if (vabits_actual == 52) 411 feat_mask |= ARM_SMMU_FEAT_VAX; 412 413 if ((smmu->features & feat_mask) != feat_mask) 414 return false; 415 416 if (!(smmu->pgsize_bitmap & PAGE_SIZE)) 417 return false; 418 419 /* 420 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're 421 * not even pretending to support AArch32 here. Abort if the MMU outputs 422 * addresses larger than what we support. 423 */ 424 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 425 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); 426 oas = id_aa64mmfr0_parange_to_phys_shift(fld); 427 if (smmu->oas < oas) 428 return false; 429 430 /* We can support bigger ASIDs than the CPU, but not smaller */ 431 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); 432 asid_bits = fld ? 16 : 8; 433 if (smmu->asid_bits < asid_bits) 434 return false; 435 436 /* 437 * See max_pinned_asids in arch/arm64/mm/context.c. The following is 438 * generally the maximum number of bindable processes. 439 */ 440 if (arm64_kernel_unmapped_at_el0()) 441 asid_bits--; 442 dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) - 443 num_possible_cpus() - 2); 444 445 return true; 446} 447 448bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) 449{ 450 /* We're not keeping track of SIDs in fault events */ 451 if (master->num_streams != 1) 452 return false; 453 454 return master->stall_enabled; 455} 456 457bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) 458{ 459 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 460 return false; 461 462 /* SSID support is mandatory for the moment */ 463 return master->ssid_bits; 464} 465 466bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) 467{ 468 bool enabled; 469 470 mutex_lock(&sva_lock); 471 enabled = master->sva_enabled; 472 mutex_unlock(&sva_lock); 473 return enabled; 474} 475 476static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) 477{ 478 struct device *dev = master->dev; 479 480 /* 481 * Drivers for devices supporting PRI or stall should enable IOPF first. 482 * Others have device-specific fault handlers and don't need IOPF. 483 */ 484 if (!arm_smmu_master_iopf_supported(master)) 485 return 0; 486 487 if (!master->iopf_enabled) 488 return -EINVAL; 489 490 return iopf_queue_add_device(master->smmu->evtq.iopf, dev); 491} 492 493static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) 494{ 495 struct device *dev = master->dev; 496 497 if (!master->iopf_enabled) 498 return; 499 500 iopf_queue_remove_device(master->smmu->evtq.iopf, dev); 501} 502 503int arm_smmu_master_enable_sva(struct arm_smmu_master *master) 504{ 505 int ret; 506 507 mutex_lock(&sva_lock); 508 ret = arm_smmu_master_sva_enable_iopf(master); 509 if (!ret) 510 master->sva_enabled = true; 511 mutex_unlock(&sva_lock); 512 513 return ret; 514} 515 516int arm_smmu_master_disable_sva(struct arm_smmu_master *master) 517{ 518 mutex_lock(&sva_lock); 519 if (!list_empty(&master->bonds)) { 520 dev_err(master->dev, "cannot disable SVA, device is bound\n"); 521 mutex_unlock(&sva_lock); 522 return -EBUSY; 523 } 524 arm_smmu_master_sva_disable_iopf(master); 525 master->sva_enabled = false; 526 mutex_unlock(&sva_lock); 527 528 return 0; 529} 530 531void arm_smmu_sva_notifier_synchronize(void) 532{ 533 /* 534 * Some MMU notifiers may still be waiting to be freed, using 535 * arm_smmu_mmu_notifier_free(). Wait for them. 536 */ 537 mmu_notifier_synchronize(); 538} 539 540void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 541 struct device *dev, ioasid_t id) 542{ 543 struct mm_struct *mm = domain->mm; 544 struct arm_smmu_bond *bond = NULL, *t; 545 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 546 547 mutex_lock(&sva_lock); 548 549 arm_smmu_write_ctx_desc(master, id, NULL); 550 551 list_for_each_entry(t, &master->bonds, list) { 552 if (t->mm == mm) { 553 bond = t; 554 break; 555 } 556 } 557 558 if (!WARN_ON(!bond)) { 559 list_del(&bond->list); 560 arm_smmu_mmu_notifier_put(bond->smmu_mn); 561 kfree(bond); 562 } 563 mutex_unlock(&sva_lock); 564} 565 566static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, 567 struct device *dev, ioasid_t id) 568{ 569 int ret = 0; 570 struct mm_struct *mm = domain->mm; 571 572 mutex_lock(&sva_lock); 573 ret = __arm_smmu_sva_bind(dev, id, mm); 574 mutex_unlock(&sva_lock); 575 576 return ret; 577} 578 579static void arm_smmu_sva_domain_free(struct iommu_domain *domain) 580{ 581 kfree(domain); 582} 583 584static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { 585 .set_dev_pasid = arm_smmu_sva_set_dev_pasid, 586 .free = arm_smmu_sva_domain_free 587}; 588 589struct iommu_domain *arm_smmu_sva_domain_alloc(void) 590{ 591 struct iommu_domain *domain; 592 593 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 594 if (!domain) 595 return NULL; 596 domain->ops = &arm_smmu_sva_domain_ops; 597 598 return domain; 599} 600