1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2013--2024 Intel Corporation 4 */ 5#include <asm/barrier.h> 6 7#include <linux/align.h> 8#include <linux/atomic.h> 9#include <linux/bitops.h> 10#include <linux/bits.h> 11#include <linux/bug.h> 12#include <linux/cacheflush.h> 13#include <linux/dma-mapping.h> 14#include <linux/err.h> 15#include <linux/gfp.h> 16#include <linux/io.h> 17#include <linux/iova.h> 18#include <linux/math.h> 19#include <linux/minmax.h> 20#include <linux/mm.h> 21#include <linux/pfn.h> 22#include <linux/slab.h> 23#include <linux/spinlock.h> 24#include <linux/types.h> 25#include <linux/vmalloc.h> 26 27#include "ipu6.h" 28#include "ipu6-dma.h" 29#include "ipu6-mmu.h" 30#include "ipu6-platform-regs.h" 31 32#define ISP_PAGE_SHIFT 12 33#define ISP_PAGE_SIZE BIT(ISP_PAGE_SHIFT) 34#define ISP_PAGE_MASK (~(ISP_PAGE_SIZE - 1)) 35 36#define ISP_L1PT_SHIFT 22 37#define ISP_L1PT_MASK (~((1U << ISP_L1PT_SHIFT) - 1)) 38 39#define ISP_L2PT_SHIFT 12 40#define ISP_L2PT_MASK (~(ISP_L1PT_MASK | (~(ISP_PAGE_MASK)))) 41 42#define ISP_L1PT_PTES 1024 43#define ISP_L2PT_PTES 1024 44 45#define ISP_PADDR_SHIFT 12 46 47#define REG_TLB_INVALIDATE 0x0000 48 49#define REG_L1_PHYS 0x0004 /* 27-bit pfn */ 50#define REG_INFO 0x0008 51 52#define TBL_PHYS_ADDR(a) ((phys_addr_t)(a) << ISP_PADDR_SHIFT) 53 54static void tlb_invalidate(struct ipu6_mmu *mmu) 55{ 56 unsigned long flags; 57 unsigned int i; 58 59 spin_lock_irqsave(&mmu->ready_lock, flags); 60 if (!mmu->ready) { 61 spin_unlock_irqrestore(&mmu->ready_lock, flags); 62 return; 63 } 64 65 for (i = 0; i < mmu->nr_mmus; i++) { 66 /* 67 * To avoid the HW bug induced dead lock in some of the IPU6 68 * MMUs on successive invalidate calls, we need to first do a 69 * read to the page table base before writing the invalidate 70 * register. MMUs which need to implement this WA, will have 71 * the insert_read_before_invalidate flags set as true. 72 * Disregard the return value of the read. 73 */ 74 if (mmu->mmu_hw[i].insert_read_before_invalidate) 75 readl(mmu->mmu_hw[i].base + REG_L1_PHYS); 76 77 writel(0xffffffff, mmu->mmu_hw[i].base + 78 REG_TLB_INVALIDATE); 79 /* 80 * The TLB invalidation is a "single cycle" (IOMMU clock cycles) 81 * When the actual MMIO write reaches the IPU6 TLB Invalidate 82 * register, wmb() will force the TLB invalidate out if the CPU 83 * attempts to update the IOMMU page table (or sooner). 84 */ 85 wmb(); 86 } 87 spin_unlock_irqrestore(&mmu->ready_lock, flags); 88} 89 90#ifdef DEBUG 91static void page_table_dump(struct ipu6_mmu_info *mmu_info) 92{ 93 u32 l1_idx; 94 95 dev_dbg(mmu_info->dev, "begin IOMMU page table dump\n"); 96 97 for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) { 98 u32 l2_idx; 99 u32 iova = (phys_addr_t)l1_idx << ISP_L1PT_SHIFT; 100 101 if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) 102 continue; 103 dev_dbg(mmu_info->dev, 104 "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %pa\n", 105 l1_idx, iova, iova + ISP_PAGE_SIZE, 106 TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx])); 107 108 for (l2_idx = 0; l2_idx < ISP_L2PT_PTES; l2_idx++) { 109 u32 *l2_pt = mmu_info->l2_pts[l1_idx]; 110 u32 iova2 = iova + (l2_idx << ISP_L2PT_SHIFT); 111 112 if (l2_pt[l2_idx] == mmu_info->dummy_page_pteval) 113 continue; 114 115 dev_dbg(mmu_info->dev, 116 "\tl2 entry %u; iova 0x%8.8x, phys %pa\n", 117 l2_idx, iova2, 118 TBL_PHYS_ADDR(l2_pt[l2_idx])); 119 } 120 } 121 122 dev_dbg(mmu_info->dev, "end IOMMU page table dump\n"); 123} 124#endif /* DEBUG */ 125 126static dma_addr_t map_single(struct ipu6_mmu_info *mmu_info, void *ptr) 127{ 128 dma_addr_t dma; 129 130 dma = dma_map_single(mmu_info->dev, ptr, PAGE_SIZE, DMA_BIDIRECTIONAL); 131 if (dma_mapping_error(mmu_info->dev, dma)) 132 return 0; 133 134 return dma; 135} 136 137static int get_dummy_page(struct ipu6_mmu_info *mmu_info) 138{ 139 void *pt = (void *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 140 dma_addr_t dma; 141 142 if (!pt) 143 return -ENOMEM; 144 145 dev_dbg(mmu_info->dev, "dummy_page: get_zeroed_page() == %p\n", pt); 146 147 dma = map_single(mmu_info, pt); 148 if (!dma) { 149 dev_err(mmu_info->dev, "Failed to map dummy page\n"); 150 goto err_free_page; 151 } 152 153 mmu_info->dummy_page = pt; 154 mmu_info->dummy_page_pteval = dma >> ISP_PAGE_SHIFT; 155 156 return 0; 157 158err_free_page: 159 free_page((unsigned long)pt); 160 return -ENOMEM; 161} 162 163static void free_dummy_page(struct ipu6_mmu_info *mmu_info) 164{ 165 dma_unmap_single(mmu_info->dev, 166 TBL_PHYS_ADDR(mmu_info->dummy_page_pteval), 167 PAGE_SIZE, DMA_BIDIRECTIONAL); 168 free_page((unsigned long)mmu_info->dummy_page); 169} 170 171static int alloc_dummy_l2_pt(struct ipu6_mmu_info *mmu_info) 172{ 173 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 174 dma_addr_t dma; 175 unsigned int i; 176 177 if (!pt) 178 return -ENOMEM; 179 180 dev_dbg(mmu_info->dev, "dummy_l2: get_zeroed_page() = %p\n", pt); 181 182 dma = map_single(mmu_info, pt); 183 if (!dma) { 184 dev_err(mmu_info->dev, "Failed to map l2pt page\n"); 185 goto err_free_page; 186 } 187 188 for (i = 0; i < ISP_L2PT_PTES; i++) 189 pt[i] = mmu_info->dummy_page_pteval; 190 191 mmu_info->dummy_l2_pt = pt; 192 mmu_info->dummy_l2_pteval = dma >> ISP_PAGE_SHIFT; 193 194 return 0; 195 196err_free_page: 197 free_page((unsigned long)pt); 198 return -ENOMEM; 199} 200 201static void free_dummy_l2_pt(struct ipu6_mmu_info *mmu_info) 202{ 203 dma_unmap_single(mmu_info->dev, 204 TBL_PHYS_ADDR(mmu_info->dummy_l2_pteval), 205 PAGE_SIZE, DMA_BIDIRECTIONAL); 206 free_page((unsigned long)mmu_info->dummy_l2_pt); 207} 208 209static u32 *alloc_l1_pt(struct ipu6_mmu_info *mmu_info) 210{ 211 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 212 dma_addr_t dma; 213 unsigned int i; 214 215 if (!pt) 216 return NULL; 217 218 dev_dbg(mmu_info->dev, "alloc_l1: get_zeroed_page() = %p\n", pt); 219 220 for (i = 0; i < ISP_L1PT_PTES; i++) 221 pt[i] = mmu_info->dummy_l2_pteval; 222 223 dma = map_single(mmu_info, pt); 224 if (!dma) { 225 dev_err(mmu_info->dev, "Failed to map l1pt page\n"); 226 goto err_free_page; 227 } 228 229 mmu_info->l1_pt_dma = dma >> ISP_PADDR_SHIFT; 230 dev_dbg(mmu_info->dev, "l1 pt %p mapped at %llx\n", pt, dma); 231 232 return pt; 233 234err_free_page: 235 free_page((unsigned long)pt); 236 return NULL; 237} 238 239static u32 *alloc_l2_pt(struct ipu6_mmu_info *mmu_info) 240{ 241 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 242 unsigned int i; 243 244 if (!pt) 245 return NULL; 246 247 dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt); 248 249 for (i = 0; i < ISP_L1PT_PTES; i++) 250 pt[i] = mmu_info->dummy_page_pteval; 251 252 return pt; 253} 254 255static int l2_map(struct ipu6_mmu_info *mmu_info, unsigned long iova, 256 phys_addr_t paddr, size_t size) 257{ 258 u32 l1_idx = iova >> ISP_L1PT_SHIFT; 259 u32 iova_start = iova; 260 u32 *l2_pt, *l2_virt; 261 unsigned int l2_idx; 262 unsigned long flags; 263 dma_addr_t dma; 264 u32 l1_entry; 265 266 dev_dbg(mmu_info->dev, 267 "mapping l2 page table for l1 index %u (iova %8.8x)\n", 268 l1_idx, (u32)iova); 269 270 spin_lock_irqsave(&mmu_info->lock, flags); 271 l1_entry = mmu_info->l1_pt[l1_idx]; 272 if (l1_entry == mmu_info->dummy_l2_pteval) { 273 l2_virt = mmu_info->l2_pts[l1_idx]; 274 if (likely(!l2_virt)) { 275 l2_virt = alloc_l2_pt(mmu_info); 276 if (!l2_virt) { 277 spin_unlock_irqrestore(&mmu_info->lock, flags); 278 return -ENOMEM; 279 } 280 } 281 282 dma = map_single(mmu_info, l2_virt); 283 if (!dma) { 284 dev_err(mmu_info->dev, "Failed to map l2pt page\n"); 285 free_page((unsigned long)l2_virt); 286 spin_unlock_irqrestore(&mmu_info->lock, flags); 287 return -EINVAL; 288 } 289 290 l1_entry = dma >> ISP_PADDR_SHIFT; 291 292 dev_dbg(mmu_info->dev, "page for l1_idx %u %p allocated\n", 293 l1_idx, l2_virt); 294 mmu_info->l1_pt[l1_idx] = l1_entry; 295 mmu_info->l2_pts[l1_idx] = l2_virt; 296 clflush_cache_range((void *)&mmu_info->l1_pt[l1_idx], 297 sizeof(mmu_info->l1_pt[l1_idx])); 298 } 299 300 l2_pt = mmu_info->l2_pts[l1_idx]; 301 302 dev_dbg(mmu_info->dev, "l2_pt at %p with dma 0x%x\n", l2_pt, l1_entry); 303 304 paddr = ALIGN(paddr, ISP_PAGE_SIZE); 305 306 l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT; 307 308 dev_dbg(mmu_info->dev, "l2_idx %u, phys 0x%8.8x\n", l2_idx, 309 l2_pt[l2_idx]); 310 if (l2_pt[l2_idx] != mmu_info->dummy_page_pteval) { 311 spin_unlock_irqrestore(&mmu_info->lock, flags); 312 return -EINVAL; 313 } 314 315 l2_pt[l2_idx] = paddr >> ISP_PADDR_SHIFT; 316 317 clflush_cache_range((void *)&l2_pt[l2_idx], sizeof(l2_pt[l2_idx])); 318 spin_unlock_irqrestore(&mmu_info->lock, flags); 319 320 dev_dbg(mmu_info->dev, "l2 index %u mapped as 0x%8.8x\n", l2_idx, 321 l2_pt[l2_idx]); 322 323 return 0; 324} 325 326static int __ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova, 327 phys_addr_t paddr, size_t size) 328{ 329 u32 iova_start = round_down(iova, ISP_PAGE_SIZE); 330 u32 iova_end = ALIGN(iova + size, ISP_PAGE_SIZE); 331 332 dev_dbg(mmu_info->dev, 333 "mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr 0x%10.10llx\n", 334 iova_start, iova_end, size, paddr); 335 336 return l2_map(mmu_info, iova_start, paddr, size); 337} 338 339static size_t l2_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova, 340 phys_addr_t dummy, size_t size) 341{ 342 u32 l1_idx = iova >> ISP_L1PT_SHIFT; 343 u32 iova_start = iova; 344 unsigned int l2_idx; 345 size_t unmapped = 0; 346 unsigned long flags; 347 u32 *l2_pt; 348 349 dev_dbg(mmu_info->dev, "unmapping l2 page table for l1 index %u (iova 0x%8.8lx)\n", 350 l1_idx, iova); 351 352 spin_lock_irqsave(&mmu_info->lock, flags); 353 if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) { 354 spin_unlock_irqrestore(&mmu_info->lock, flags); 355 dev_err(mmu_info->dev, 356 "unmap iova 0x%8.8lx l1 idx %u which was not mapped\n", 357 iova, l1_idx); 358 return 0; 359 } 360 361 for (l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT; 362 (iova_start & ISP_L1PT_MASK) + (l2_idx << ISP_PAGE_SHIFT) 363 < iova_start + size && l2_idx < ISP_L2PT_PTES; l2_idx++) { 364 l2_pt = mmu_info->l2_pts[l1_idx]; 365 dev_dbg(mmu_info->dev, 366 "unmap l2 index %u with pteval 0x%10.10llx\n", 367 l2_idx, TBL_PHYS_ADDR(l2_pt[l2_idx])); 368 l2_pt[l2_idx] = mmu_info->dummy_page_pteval; 369 370 clflush_cache_range((void *)&l2_pt[l2_idx], 371 sizeof(l2_pt[l2_idx])); 372 unmapped++; 373 } 374 spin_unlock_irqrestore(&mmu_info->lock, flags); 375 376 return unmapped << ISP_PAGE_SHIFT; 377} 378 379static size_t __ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info, 380 unsigned long iova, size_t size) 381{ 382 return l2_unmap(mmu_info, iova, 0, size); 383} 384 385static int allocate_trash_buffer(struct ipu6_mmu *mmu) 386{ 387 unsigned int n_pages = PHYS_PFN(PAGE_ALIGN(IPU6_MMUV2_TRASH_RANGE)); 388 struct iova *iova; 389 unsigned int i; 390 dma_addr_t dma; 391 unsigned long iova_addr; 392 int ret; 393 394 /* Allocate 8MB in iova range */ 395 iova = alloc_iova(&mmu->dmap->iovad, n_pages, 396 PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0); 397 if (!iova) { 398 dev_err(mmu->dev, "cannot allocate iova range for trash\n"); 399 return -ENOMEM; 400 } 401 402 dma = dma_map_page(mmu->dmap->mmu_info->dev, mmu->trash_page, 0, 403 PAGE_SIZE, DMA_BIDIRECTIONAL); 404 if (dma_mapping_error(mmu->dmap->mmu_info->dev, dma)) { 405 dev_err(mmu->dmap->mmu_info->dev, "Failed to map trash page\n"); 406 ret = -ENOMEM; 407 goto out_free_iova; 408 } 409 410 mmu->pci_trash_page = dma; 411 412 /* 413 * Map the 8MB iova address range to the same physical trash page 414 * mmu->trash_page which is already reserved at the probe 415 */ 416 iova_addr = iova->pfn_lo; 417 for (i = 0; i < n_pages; i++) { 418 ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr), 419 mmu->pci_trash_page, PAGE_SIZE); 420 if (ret) { 421 dev_err(mmu->dev, 422 "mapping trash buffer range failed\n"); 423 goto out_unmap; 424 } 425 426 iova_addr++; 427 } 428 429 mmu->iova_trash_page = PFN_PHYS(iova->pfn_lo); 430 dev_dbg(mmu->dev, "iova trash buffer for MMUID: %d is %u\n", 431 mmu->mmid, (unsigned int)mmu->iova_trash_page); 432 return 0; 433 434out_unmap: 435 ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo), 436 PFN_PHYS(iova_size(iova))); 437 dma_unmap_page(mmu->dmap->mmu_info->dev, mmu->pci_trash_page, 438 PAGE_SIZE, DMA_BIDIRECTIONAL); 439out_free_iova: 440 __free_iova(&mmu->dmap->iovad, iova); 441 return ret; 442} 443 444int ipu6_mmu_hw_init(struct ipu6_mmu *mmu) 445{ 446 struct ipu6_mmu_info *mmu_info; 447 unsigned long flags; 448 unsigned int i; 449 450 mmu_info = mmu->dmap->mmu_info; 451 452 /* Initialise the each MMU HW block */ 453 for (i = 0; i < mmu->nr_mmus; i++) { 454 struct ipu6_mmu_hw *mmu_hw = &mmu->mmu_hw[i]; 455 unsigned int j; 456 u16 block_addr; 457 458 /* Write page table address per MMU */ 459 writel((phys_addr_t)mmu_info->l1_pt_dma, 460 mmu->mmu_hw[i].base + REG_L1_PHYS); 461 462 /* Set info bits per MMU */ 463 writel(mmu->mmu_hw[i].info_bits, 464 mmu->mmu_hw[i].base + REG_INFO); 465 466 /* Configure MMU TLB stream configuration for L1 */ 467 for (j = 0, block_addr = 0; j < mmu_hw->nr_l1streams; 468 block_addr += mmu->mmu_hw[i].l1_block_sz[j], j++) { 469 if (block_addr > IPU6_MAX_LI_BLOCK_ADDR) { 470 dev_err(mmu->dev, "invalid L1 configuration\n"); 471 return -EINVAL; 472 } 473 474 /* Write block start address for each streams */ 475 writel(block_addr, mmu_hw->base + 476 mmu_hw->l1_stream_id_reg_offset + 4 * j); 477 } 478 479 /* Configure MMU TLB stream configuration for L2 */ 480 for (j = 0, block_addr = 0; j < mmu_hw->nr_l2streams; 481 block_addr += mmu->mmu_hw[i].l2_block_sz[j], j++) { 482 if (block_addr > IPU6_MAX_L2_BLOCK_ADDR) { 483 dev_err(mmu->dev, "invalid L2 configuration\n"); 484 return -EINVAL; 485 } 486 487 writel(block_addr, mmu_hw->base + 488 mmu_hw->l2_stream_id_reg_offset + 4 * j); 489 } 490 } 491 492 if (!mmu->trash_page) { 493 int ret; 494 495 mmu->trash_page = alloc_page(GFP_KERNEL); 496 if (!mmu->trash_page) { 497 dev_err(mmu->dev, "insufficient memory for trash buffer\n"); 498 return -ENOMEM; 499 } 500 501 ret = allocate_trash_buffer(mmu); 502 if (ret) { 503 __free_page(mmu->trash_page); 504 mmu->trash_page = NULL; 505 dev_err(mmu->dev, "trash buffer allocation failed\n"); 506 return ret; 507 } 508 } 509 510 spin_lock_irqsave(&mmu->ready_lock, flags); 511 mmu->ready = true; 512 spin_unlock_irqrestore(&mmu->ready_lock, flags); 513 514 return 0; 515} 516EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_init, INTEL_IPU6); 517 518static struct ipu6_mmu_info *ipu6_mmu_alloc(struct ipu6_device *isp) 519{ 520 struct ipu6_mmu_info *mmu_info; 521 int ret; 522 523 mmu_info = kzalloc(sizeof(*mmu_info), GFP_KERNEL); 524 if (!mmu_info) 525 return NULL; 526 527 mmu_info->aperture_start = 0; 528 mmu_info->aperture_end = DMA_BIT_MASK(isp->secure_mode ? 529 IPU6_MMU_ADDR_BITS : 530 IPU6_MMU_ADDR_BITS_NON_SECURE); 531 mmu_info->pgsize_bitmap = SZ_4K; 532 mmu_info->dev = &isp->pdev->dev; 533 534 ret = get_dummy_page(mmu_info); 535 if (ret) 536 goto err_free_info; 537 538 ret = alloc_dummy_l2_pt(mmu_info); 539 if (ret) 540 goto err_free_dummy_page; 541 542 mmu_info->l2_pts = vzalloc(ISP_L2PT_PTES * sizeof(*mmu_info->l2_pts)); 543 if (!mmu_info->l2_pts) 544 goto err_free_dummy_l2_pt; 545 546 /* 547 * We always map the L1 page table (a single page as well as 548 * the L2 page tables). 549 */ 550 mmu_info->l1_pt = alloc_l1_pt(mmu_info); 551 if (!mmu_info->l1_pt) 552 goto err_free_l2_pts; 553 554 spin_lock_init(&mmu_info->lock); 555 556 dev_dbg(mmu_info->dev, "domain initialised\n"); 557 558 return mmu_info; 559 560err_free_l2_pts: 561 vfree(mmu_info->l2_pts); 562err_free_dummy_l2_pt: 563 free_dummy_l2_pt(mmu_info); 564err_free_dummy_page: 565 free_dummy_page(mmu_info); 566err_free_info: 567 kfree(mmu_info); 568 569 return NULL; 570} 571 572void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu) 573{ 574 unsigned long flags; 575 576 spin_lock_irqsave(&mmu->ready_lock, flags); 577 mmu->ready = false; 578 spin_unlock_irqrestore(&mmu->ready_lock, flags); 579} 580EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_cleanup, INTEL_IPU6); 581 582static struct ipu6_dma_mapping *alloc_dma_mapping(struct ipu6_device *isp) 583{ 584 struct ipu6_dma_mapping *dmap; 585 586 dmap = kzalloc(sizeof(*dmap), GFP_KERNEL); 587 if (!dmap) 588 return NULL; 589 590 dmap->mmu_info = ipu6_mmu_alloc(isp); 591 if (!dmap->mmu_info) { 592 kfree(dmap); 593 return NULL; 594 } 595 596 init_iova_domain(&dmap->iovad, SZ_4K, 1); 597 dmap->mmu_info->dmap = dmap; 598 599 dev_dbg(&isp->pdev->dev, "alloc mapping\n"); 600 601 iova_cache_get(); 602 603 return dmap; 604} 605 606phys_addr_t ipu6_mmu_iova_to_phys(struct ipu6_mmu_info *mmu_info, 607 dma_addr_t iova) 608{ 609 phys_addr_t phy_addr; 610 unsigned long flags; 611 u32 *l2_pt; 612 613 spin_lock_irqsave(&mmu_info->lock, flags); 614 l2_pt = mmu_info->l2_pts[iova >> ISP_L1PT_SHIFT]; 615 phy_addr = (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT]; 616 phy_addr <<= ISP_PAGE_SHIFT; 617 spin_unlock_irqrestore(&mmu_info->lock, flags); 618 619 return phy_addr; 620} 621 622static size_t ipu6_mmu_pgsize(unsigned long pgsize_bitmap, 623 unsigned long addr_merge, size_t size) 624{ 625 unsigned int pgsize_idx; 626 size_t pgsize; 627 628 /* Max page size that still fits into 'size' */ 629 pgsize_idx = __fls(size); 630 631 if (likely(addr_merge)) { 632 /* Max page size allowed by address */ 633 unsigned int align_pgsize_idx = __ffs(addr_merge); 634 635 pgsize_idx = min(pgsize_idx, align_pgsize_idx); 636 } 637 638 pgsize = (1UL << (pgsize_idx + 1)) - 1; 639 pgsize &= pgsize_bitmap; 640 641 WARN_ON(!pgsize); 642 643 /* pick the biggest page */ 644 pgsize_idx = __fls(pgsize); 645 pgsize = 1UL << pgsize_idx; 646 647 return pgsize; 648} 649 650size_t ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova, 651 size_t size) 652{ 653 size_t unmapped_page, unmapped = 0; 654 unsigned int min_pagesz; 655 656 /* find out the minimum page size supported */ 657 min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap); 658 659 /* 660 * The virtual address and the size of the mapping must be 661 * aligned (at least) to the size of the smallest page supported 662 * by the hardware 663 */ 664 if (!IS_ALIGNED(iova | size, min_pagesz)) { 665 dev_err(NULL, "unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", 666 iova, size, min_pagesz); 667 return -EINVAL; 668 } 669 670 /* 671 * Keep iterating until we either unmap 'size' bytes (or more) 672 * or we hit an area that isn't mapped. 673 */ 674 while (unmapped < size) { 675 size_t pgsize = ipu6_mmu_pgsize(mmu_info->pgsize_bitmap, 676 iova, size - unmapped); 677 678 unmapped_page = __ipu6_mmu_unmap(mmu_info, iova, pgsize); 679 if (!unmapped_page) 680 break; 681 682 dev_dbg(mmu_info->dev, "unmapped: iova 0x%lx size 0x%zx\n", 683 iova, unmapped_page); 684 685 iova += unmapped_page; 686 unmapped += unmapped_page; 687 } 688 689 return unmapped; 690} 691 692int ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova, 693 phys_addr_t paddr, size_t size) 694{ 695 unsigned long orig_iova = iova; 696 unsigned int min_pagesz; 697 size_t orig_size = size; 698 int ret = 0; 699 700 if (mmu_info->pgsize_bitmap == 0UL) 701 return -ENODEV; 702 703 /* find out the minimum page size supported */ 704 min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap); 705 706 /* 707 * both the virtual address and the physical one, as well as 708 * the size of the mapping, must be aligned (at least) to the 709 * size of the smallest page supported by the hardware 710 */ 711 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 712 dev_err(mmu_info->dev, 713 "unaligned: iova %lx pa %pa size %zx min_pagesz %x\n", 714 iova, &paddr, size, min_pagesz); 715 return -EINVAL; 716 } 717 718 dev_dbg(mmu_info->dev, "map: iova 0x%lx pa %pa size 0x%zx\n", 719 iova, &paddr, size); 720 721 while (size) { 722 size_t pgsize = ipu6_mmu_pgsize(mmu_info->pgsize_bitmap, 723 iova | paddr, size); 724 725 dev_dbg(mmu_info->dev, 726 "mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", 727 iova, &paddr, pgsize); 728 729 ret = __ipu6_mmu_map(mmu_info, iova, paddr, pgsize); 730 if (ret) 731 break; 732 733 iova += pgsize; 734 paddr += pgsize; 735 size -= pgsize; 736 } 737 738 /* unroll mapping in case something went wrong */ 739 if (ret) 740 ipu6_mmu_unmap(mmu_info, orig_iova, orig_size - size); 741 742 return ret; 743} 744 745static void ipu6_mmu_destroy(struct ipu6_mmu *mmu) 746{ 747 struct ipu6_dma_mapping *dmap = mmu->dmap; 748 struct ipu6_mmu_info *mmu_info = dmap->mmu_info; 749 struct iova *iova; 750 u32 l1_idx; 751 752 if (mmu->iova_trash_page) { 753 iova = find_iova(&dmap->iovad, PHYS_PFN(mmu->iova_trash_page)); 754 if (iova) { 755 /* unmap and free the trash buffer iova */ 756 ipu6_mmu_unmap(mmu_info, PFN_PHYS(iova->pfn_lo), 757 PFN_PHYS(iova_size(iova))); 758 __free_iova(&dmap->iovad, iova); 759 } else { 760 dev_err(mmu->dev, "trash buffer iova not found.\n"); 761 } 762 763 mmu->iova_trash_page = 0; 764 dma_unmap_page(mmu_info->dev, mmu->pci_trash_page, 765 PAGE_SIZE, DMA_BIDIRECTIONAL); 766 mmu->pci_trash_page = 0; 767 __free_page(mmu->trash_page); 768 } 769 770 for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) { 771 if (mmu_info->l1_pt[l1_idx] != mmu_info->dummy_l2_pteval) { 772 dma_unmap_single(mmu_info->dev, 773 TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]), 774 PAGE_SIZE, DMA_BIDIRECTIONAL); 775 free_page((unsigned long)mmu_info->l2_pts[l1_idx]); 776 } 777 } 778 779 vfree(mmu_info->l2_pts); 780 free_dummy_page(mmu_info); 781 dma_unmap_single(mmu_info->dev, TBL_PHYS_ADDR(mmu_info->l1_pt_dma), 782 PAGE_SIZE, DMA_BIDIRECTIONAL); 783 free_page((unsigned long)mmu_info->dummy_l2_pt); 784 free_page((unsigned long)mmu_info->l1_pt); 785 kfree(mmu_info); 786} 787 788struct ipu6_mmu *ipu6_mmu_init(struct device *dev, 789 void __iomem *base, int mmid, 790 const struct ipu6_hw_variants *hw) 791{ 792 struct ipu6_device *isp = pci_get_drvdata(to_pci_dev(dev)); 793 struct ipu6_mmu_pdata *pdata; 794 struct ipu6_mmu *mmu; 795 unsigned int i; 796 797 if (hw->nr_mmus > IPU6_MMU_MAX_DEVICES) 798 return ERR_PTR(-EINVAL); 799 800 pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); 801 if (!pdata) 802 return ERR_PTR(-ENOMEM); 803 804 for (i = 0; i < hw->nr_mmus; i++) { 805 struct ipu6_mmu_hw *pdata_mmu = &pdata->mmu_hw[i]; 806 const struct ipu6_mmu_hw *src_mmu = &hw->mmu_hw[i]; 807 808 if (src_mmu->nr_l1streams > IPU6_MMU_MAX_TLB_L1_STREAMS || 809 src_mmu->nr_l2streams > IPU6_MMU_MAX_TLB_L2_STREAMS) 810 return ERR_PTR(-EINVAL); 811 812 *pdata_mmu = *src_mmu; 813 pdata_mmu->base = base + src_mmu->offset; 814 } 815 816 mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL); 817 if (!mmu) 818 return ERR_PTR(-ENOMEM); 819 820 mmu->mmid = mmid; 821 mmu->mmu_hw = pdata->mmu_hw; 822 mmu->nr_mmus = hw->nr_mmus; 823 mmu->tlb_invalidate = tlb_invalidate; 824 mmu->ready = false; 825 INIT_LIST_HEAD(&mmu->vma_list); 826 spin_lock_init(&mmu->ready_lock); 827 828 mmu->dmap = alloc_dma_mapping(isp); 829 if (!mmu->dmap) { 830 dev_err(dev, "can't alloc dma mapping\n"); 831 return ERR_PTR(-ENOMEM); 832 } 833 834 return mmu; 835} 836 837void ipu6_mmu_cleanup(struct ipu6_mmu *mmu) 838{ 839 struct ipu6_dma_mapping *dmap = mmu->dmap; 840 841 ipu6_mmu_destroy(mmu); 842 mmu->dmap = NULL; 843 iova_cache_put(); 844 put_iova_domain(&dmap->iovad); 845 kfree(dmap); 846} 847