1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Implementation of the IOMMU SVA API for the ARM SMMUv3 4 */ 5 6#include <linux/mm.h> 7#include <linux/mmu_context.h> 8#include <linux/mmu_notifier.h> 9#include <linux/sched/mm.h> 10#include <linux/slab.h> 11#include <kunit/visibility.h> 12 13#include "arm-smmu-v3.h" 14#include "../../io-pgtable-arm.h" 15 16struct arm_smmu_mmu_notifier { 17 struct mmu_notifier mn; 18 struct arm_smmu_ctx_desc *cd; 19 bool cleared; 20 refcount_t refs; 21 struct list_head list; 22 struct arm_smmu_domain *domain; 23}; 24 25#define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn) 26 27struct arm_smmu_bond { 28 struct mm_struct *mm; 29 struct arm_smmu_mmu_notifier *smmu_mn; 30 struct list_head list; 31}; 32 33#define sva_to_bond(handle) \ 34 container_of(handle, struct arm_smmu_bond, sva) 35 36static DEFINE_MUTEX(sva_lock); 37 38static void 39arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) 40{ 41 struct arm_smmu_master *master; 42 struct arm_smmu_cd target_cd; 43 unsigned long flags; 44 45 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 46 list_for_each_entry(master, &smmu_domain->devices, domain_head) { 47 struct arm_smmu_cd *cdptr; 48 49 /* S1 domains only support RID attachment right now */ 50 cdptr = arm_smmu_get_cd_ptr(master, IOMMU_NO_PASID); 51 if (WARN_ON(!cdptr)) 52 continue; 53 54 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); 55 arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr, 56 &target_cd); 57 } 58 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 59} 60 61/* 62 * Check if the CPU ASID is available on the SMMU side. If a private context 63 * descriptor is using it, try to replace it. 64 */ 65static struct arm_smmu_ctx_desc * 66arm_smmu_share_asid(struct mm_struct *mm, u16 asid) 67{ 68 int ret; 69 u32 new_asid; 70 struct arm_smmu_ctx_desc *cd; 71 struct arm_smmu_device *smmu; 72 struct arm_smmu_domain *smmu_domain; 73 74 cd = xa_load(&arm_smmu_asid_xa, asid); 75 if (!cd) 76 return NULL; 77 78 if (cd->mm) { 79 if (WARN_ON(cd->mm != mm)) 80 return ERR_PTR(-EINVAL); 81 /* All devices bound to this mm use the same cd struct. */ 82 refcount_inc(&cd->refs); 83 return cd; 84 } 85 86 smmu_domain = container_of(cd, struct arm_smmu_domain, cd); 87 smmu = smmu_domain->smmu; 88 89 ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd, 90 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 91 if (ret) 92 return ERR_PTR(-ENOSPC); 93 /* 94 * Race with unmap: TLB invalidations will start targeting the new ASID, 95 * which isn't assigned yet. We'll do an invalidate-all on the old ASID 96 * later, so it doesn't matter. 97 */ 98 cd->asid = new_asid; 99 /* 100 * Update ASID and invalidate CD in all associated masters. There will 101 * be some overlap between use of both ASIDs, until we invalidate the 102 * TLB. 103 */ 104 arm_smmu_update_s1_domain_cd_entry(smmu_domain); 105 106 /* Invalidate TLB entries previously associated with that context */ 107 arm_smmu_tlb_inv_asid(smmu, asid); 108 109 xa_erase(&arm_smmu_asid_xa, asid); 110 return NULL; 111} 112 113static u64 page_size_to_cd(void) 114{ 115 static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || 116 PAGE_SIZE == SZ_64K); 117 if (PAGE_SIZE == SZ_64K) 118 return ARM_LPAE_TCR_TG0_64K; 119 if (PAGE_SIZE == SZ_16K) 120 return ARM_LPAE_TCR_TG0_16K; 121 return ARM_LPAE_TCR_TG0_4K; 122} 123 124VISIBLE_IF_KUNIT 125void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, 126 struct arm_smmu_master *master, struct mm_struct *mm, 127 u16 asid) 128{ 129 u64 par; 130 131 memset(target, 0, sizeof(*target)); 132 133 par = cpuid_feature_extract_unsigned_field( 134 read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1), 135 ID_AA64MMFR0_EL1_PARANGE_SHIFT); 136 137 target->data[0] = cpu_to_le64( 138 CTXDESC_CD_0_TCR_EPD1 | 139#ifdef __BIG_ENDIAN 140 CTXDESC_CD_0_ENDI | 141#endif 142 CTXDESC_CD_0_V | 143 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) | 144 CTXDESC_CD_0_AA64 | 145 (master->stall_enabled ? CTXDESC_CD_0_S : 0) | 146 CTXDESC_CD_0_R | 147 CTXDESC_CD_0_A | 148 CTXDESC_CD_0_ASET | 149 FIELD_PREP(CTXDESC_CD_0_ASID, asid)); 150 151 /* 152 * If no MM is passed then this creates a SVA entry that faults 153 * everything. arm_smmu_write_cd_entry() can hitlessly go between these 154 * two entries types since TTB0 is ignored by HW when EPD0 is set. 155 */ 156 if (mm) { 157 target->data[0] |= cpu_to_le64( 158 FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 159 64ULL - vabits_actual) | 160 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | 161 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, 162 ARM_LPAE_TCR_RGN_WBWA) | 163 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, 164 ARM_LPAE_TCR_RGN_WBWA) | 165 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS)); 166 167 target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) & 168 CTXDESC_CD_1_TTB0_MASK); 169 } else { 170 target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0); 171 172 /* 173 * Disable stall and immediately generate an abort if stall 174 * disable is permitted. This speeds up cleanup for an unclean 175 * exit if the device is still doing a lot of DMA. 176 */ 177 if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) 178 target->data[0] &= 179 cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R)); 180 } 181 182 /* 183 * MAIR value is pretty much constant and global, so we can just get it 184 * from the current CPU register 185 */ 186 target->data[3] = cpu_to_le64(read_sysreg(mair_el1)); 187} 188EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd); 189 190static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) 191{ 192 u16 asid; 193 int err = 0; 194 struct arm_smmu_ctx_desc *cd; 195 struct arm_smmu_ctx_desc *ret = NULL; 196 197 /* Don't free the mm until we release the ASID */ 198 mmgrab(mm); 199 200 asid = arm64_mm_context_get(mm); 201 if (!asid) { 202 err = -ESRCH; 203 goto out_drop_mm; 204 } 205 206 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 207 if (!cd) { 208 err = -ENOMEM; 209 goto out_put_context; 210 } 211 212 refcount_set(&cd->refs, 1); 213 214 mutex_lock(&arm_smmu_asid_lock); 215 ret = arm_smmu_share_asid(mm, asid); 216 if (ret) { 217 mutex_unlock(&arm_smmu_asid_lock); 218 goto out_free_cd; 219 } 220 221 err = xa_insert(&arm_smmu_asid_xa, asid, cd, GFP_KERNEL); 222 mutex_unlock(&arm_smmu_asid_lock); 223 224 if (err) 225 goto out_free_asid; 226 227 cd->asid = asid; 228 cd->mm = mm; 229 230 return cd; 231 232out_free_asid: 233 arm_smmu_free_asid(cd); 234out_free_cd: 235 kfree(cd); 236out_put_context: 237 arm64_mm_context_put(mm); 238out_drop_mm: 239 mmdrop(mm); 240 return err < 0 ? ERR_PTR(err) : ret; 241} 242 243static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) 244{ 245 if (arm_smmu_free_asid(cd)) { 246 /* Unpin ASID */ 247 arm64_mm_context_put(cd->mm); 248 mmdrop(cd->mm); 249 kfree(cd); 250 } 251} 252 253/* 254 * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this 255 * is used as a threshold to replace per-page TLBI commands to issue in the 256 * command queue with an address-space TLBI command, when SMMU w/o a range 257 * invalidation feature handles too many per-page TLBI commands, which will 258 * otherwise result in a soft lockup. 259 */ 260#define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3)) 261 262static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, 263 struct mm_struct *mm, 264 unsigned long start, 265 unsigned long end) 266{ 267 struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); 268 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 269 size_t size; 270 271 /* 272 * The mm_types defines vm_end as the first byte after the end address, 273 * different from IOMMU subsystem using the last address of an address 274 * range. So do a simple translation here by calculating size correctly. 275 */ 276 size = end - start; 277 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) { 278 if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE) 279 size = 0; 280 } else { 281 if (size == ULONG_MAX) 282 size = 0; 283 } 284 285 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) { 286 if (!size) 287 arm_smmu_tlb_inv_asid(smmu_domain->smmu, 288 smmu_mn->cd->asid); 289 else 290 arm_smmu_tlb_inv_range_asid(start, size, 291 smmu_mn->cd->asid, 292 PAGE_SIZE, false, 293 smmu_domain); 294 } 295 296 arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), start, 297 size); 298} 299 300static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 301{ 302 struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); 303 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 304 struct arm_smmu_master *master; 305 unsigned long flags; 306 307 mutex_lock(&sva_lock); 308 if (smmu_mn->cleared) { 309 mutex_unlock(&sva_lock); 310 return; 311 } 312 313 /* 314 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, 315 * but disable translation. 316 */ 317 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 318 list_for_each_entry(master, &smmu_domain->devices, domain_head) { 319 struct arm_smmu_cd target; 320 struct arm_smmu_cd *cdptr; 321 322 cdptr = arm_smmu_get_cd_ptr(master, mm_get_enqcmd_pasid(mm)); 323 if (WARN_ON(!cdptr)) 324 continue; 325 arm_smmu_make_sva_cd(&target, master, NULL, smmu_mn->cd->asid); 326 arm_smmu_write_cd_entry(master, mm_get_enqcmd_pasid(mm), cdptr, 327 &target); 328 } 329 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 330 331 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid); 332 arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), 0, 0); 333 334 smmu_mn->cleared = true; 335 mutex_unlock(&sva_lock); 336} 337 338static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) 339{ 340 kfree(mn_to_smmu(mn)); 341} 342 343static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { 344 .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs, 345 .release = arm_smmu_mm_release, 346 .free_notifier = arm_smmu_mmu_notifier_free, 347}; 348 349/* Allocate or get existing MMU notifier for this {domain, mm} pair */ 350static struct arm_smmu_mmu_notifier * 351arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain, 352 struct mm_struct *mm) 353{ 354 int ret; 355 struct arm_smmu_ctx_desc *cd; 356 struct arm_smmu_mmu_notifier *smmu_mn; 357 358 list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) { 359 if (smmu_mn->mn.mm == mm) { 360 refcount_inc(&smmu_mn->refs); 361 return smmu_mn; 362 } 363 } 364 365 cd = arm_smmu_alloc_shared_cd(mm); 366 if (IS_ERR(cd)) 367 return ERR_CAST(cd); 368 369 smmu_mn = kzalloc(sizeof(*smmu_mn), GFP_KERNEL); 370 if (!smmu_mn) { 371 ret = -ENOMEM; 372 goto err_free_cd; 373 } 374 375 refcount_set(&smmu_mn->refs, 1); 376 smmu_mn->cd = cd; 377 smmu_mn->domain = smmu_domain; 378 smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops; 379 380 ret = mmu_notifier_register(&smmu_mn->mn, mm); 381 if (ret) { 382 kfree(smmu_mn); 383 goto err_free_cd; 384 } 385 386 list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers); 387 return smmu_mn; 388 389err_free_cd: 390 arm_smmu_free_shared_cd(cd); 391 return ERR_PTR(ret); 392} 393 394static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn) 395{ 396 struct mm_struct *mm = smmu_mn->mn.mm; 397 struct arm_smmu_ctx_desc *cd = smmu_mn->cd; 398 struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 399 400 if (!refcount_dec_and_test(&smmu_mn->refs)) 401 return; 402 403 list_del(&smmu_mn->list); 404 405 /* 406 * If we went through clear(), we've already invalidated, and no 407 * new TLB entry can have been formed. 408 */ 409 if (!smmu_mn->cleared) { 410 arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid); 411 arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), 0, 412 0); 413 } 414 415 /* Frees smmu_mn */ 416 mmu_notifier_put(&smmu_mn->mn); 417 arm_smmu_free_shared_cd(cd); 418} 419 420static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, 421 struct mm_struct *mm) 422{ 423 int ret; 424 struct arm_smmu_cd target; 425 struct arm_smmu_cd *cdptr; 426 struct arm_smmu_bond *bond; 427 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 428 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 429 struct arm_smmu_domain *smmu_domain; 430 431 if (!(domain->type & __IOMMU_DOMAIN_PAGING)) 432 return -ENODEV; 433 smmu_domain = to_smmu_domain(domain); 434 if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) 435 return -ENODEV; 436 437 if (!master || !master->sva_enabled) 438 return -ENODEV; 439 440 bond = kzalloc(sizeof(*bond), GFP_KERNEL); 441 if (!bond) 442 return -ENOMEM; 443 444 bond->mm = mm; 445 446 bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm); 447 if (IS_ERR(bond->smmu_mn)) { 448 ret = PTR_ERR(bond->smmu_mn); 449 goto err_free_bond; 450 } 451 452 cdptr = arm_smmu_alloc_cd_ptr(master, mm_get_enqcmd_pasid(mm)); 453 if (!cdptr) { 454 ret = -ENOMEM; 455 goto err_put_notifier; 456 } 457 arm_smmu_make_sva_cd(&target, master, mm, bond->smmu_mn->cd->asid); 458 arm_smmu_write_cd_entry(master, pasid, cdptr, &target); 459 460 list_add(&bond->list, &master->bonds); 461 return 0; 462 463err_put_notifier: 464 arm_smmu_mmu_notifier_put(bond->smmu_mn); 465err_free_bond: 466 kfree(bond); 467 return ret; 468} 469 470bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 471{ 472 unsigned long reg, fld; 473 unsigned long oas; 474 unsigned long asid_bits; 475 u32 feat_mask = ARM_SMMU_FEAT_COHERENCY; 476 477 if (vabits_actual == 52) 478 feat_mask |= ARM_SMMU_FEAT_VAX; 479 480 if ((smmu->features & feat_mask) != feat_mask) 481 return false; 482 483 if (!(smmu->pgsize_bitmap & PAGE_SIZE)) 484 return false; 485 486 /* 487 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're 488 * not even pretending to support AArch32 here. Abort if the MMU outputs 489 * addresses larger than what we support. 490 */ 491 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 492 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); 493 oas = id_aa64mmfr0_parange_to_phys_shift(fld); 494 if (smmu->oas < oas) 495 return false; 496 497 /* We can support bigger ASIDs than the CPU, but not smaller */ 498 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); 499 asid_bits = fld ? 16 : 8; 500 if (smmu->asid_bits < asid_bits) 501 return false; 502 503 /* 504 * See max_pinned_asids in arch/arm64/mm/context.c. The following is 505 * generally the maximum number of bindable processes. 506 */ 507 if (arm64_kernel_unmapped_at_el0()) 508 asid_bits--; 509 dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) - 510 num_possible_cpus() - 2); 511 512 return true; 513} 514 515bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) 516{ 517 /* We're not keeping track of SIDs in fault events */ 518 if (master->num_streams != 1) 519 return false; 520 521 return master->stall_enabled; 522} 523 524bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) 525{ 526 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 527 return false; 528 529 /* SSID support is mandatory for the moment */ 530 return master->ssid_bits; 531} 532 533bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) 534{ 535 bool enabled; 536 537 mutex_lock(&sva_lock); 538 enabled = master->sva_enabled; 539 mutex_unlock(&sva_lock); 540 return enabled; 541} 542 543static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) 544{ 545 struct device *dev = master->dev; 546 547 /* 548 * Drivers for devices supporting PRI or stall should enable IOPF first. 549 * Others have device-specific fault handlers and don't need IOPF. 550 */ 551 if (!arm_smmu_master_iopf_supported(master)) 552 return 0; 553 554 if (!master->iopf_enabled) 555 return -EINVAL; 556 557 return iopf_queue_add_device(master->smmu->evtq.iopf, dev); 558} 559 560static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) 561{ 562 struct device *dev = master->dev; 563 564 if (!master->iopf_enabled) 565 return; 566 567 iopf_queue_remove_device(master->smmu->evtq.iopf, dev); 568} 569 570int arm_smmu_master_enable_sva(struct arm_smmu_master *master) 571{ 572 int ret; 573 574 mutex_lock(&sva_lock); 575 ret = arm_smmu_master_sva_enable_iopf(master); 576 if (!ret) 577 master->sva_enabled = true; 578 mutex_unlock(&sva_lock); 579 580 return ret; 581} 582 583int arm_smmu_master_disable_sva(struct arm_smmu_master *master) 584{ 585 mutex_lock(&sva_lock); 586 if (!list_empty(&master->bonds)) { 587 dev_err(master->dev, "cannot disable SVA, device is bound\n"); 588 mutex_unlock(&sva_lock); 589 return -EBUSY; 590 } 591 arm_smmu_master_sva_disable_iopf(master); 592 master->sva_enabled = false; 593 mutex_unlock(&sva_lock); 594 595 return 0; 596} 597 598void arm_smmu_sva_notifier_synchronize(void) 599{ 600 /* 601 * Some MMU notifiers may still be waiting to be freed, using 602 * arm_smmu_mmu_notifier_free(). Wait for them. 603 */ 604 mmu_notifier_synchronize(); 605} 606 607void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 608 struct device *dev, ioasid_t id) 609{ 610 struct mm_struct *mm = domain->mm; 611 struct arm_smmu_bond *bond = NULL, *t; 612 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 613 614 mutex_lock(&sva_lock); 615 616 arm_smmu_clear_cd(master, id); 617 618 list_for_each_entry(t, &master->bonds, list) { 619 if (t->mm == mm) { 620 bond = t; 621 break; 622 } 623 } 624 625 if (!WARN_ON(!bond)) { 626 list_del(&bond->list); 627 arm_smmu_mmu_notifier_put(bond->smmu_mn); 628 kfree(bond); 629 } 630 mutex_unlock(&sva_lock); 631} 632 633static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, 634 struct device *dev, ioasid_t id) 635{ 636 int ret = 0; 637 struct mm_struct *mm = domain->mm; 638 639 if (mm_get_enqcmd_pasid(mm) != id) 640 return -EINVAL; 641 642 mutex_lock(&sva_lock); 643 ret = __arm_smmu_sva_bind(dev, id, mm); 644 mutex_unlock(&sva_lock); 645 646 return ret; 647} 648 649static void arm_smmu_sva_domain_free(struct iommu_domain *domain) 650{ 651 kfree(domain); 652} 653 654static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { 655 .set_dev_pasid = arm_smmu_sva_set_dev_pasid, 656 .free = arm_smmu_sva_domain_free 657}; 658 659struct iommu_domain *arm_smmu_sva_domain_alloc(void) 660{ 661 struct iommu_domain *domain; 662 663 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 664 if (!domain) 665 return NULL; 666 domain->ops = &arm_smmu_sva_domain_ops; 667 668 return domain; 669} 670