1// SPDX-License-Identifier: GPL-2.0 2/* 3 * linux/arch/alpha/kernel/pci_iommu.c 4 */ 5 6#include <linux/kernel.h> 7#include <linux/mm.h> 8#include <linux/pci.h> 9#include <linux/gfp.h> 10#include <linux/memblock.h> 11#include <linux/export.h> 12#include <linux/scatterlist.h> 13#include <linux/log2.h> 14#include <linux/dma-map-ops.h> 15#include <linux/iommu-helper.h> 16 17#include <asm/io.h> 18#include <asm/hwrpb.h> 19 20#include "proto.h" 21#include "pci_impl.h" 22 23 24#define DEBUG_ALLOC 0 25#if DEBUG_ALLOC > 0 26# define DBGA(args...) printk(KERN_DEBUG args) 27#else 28# define DBGA(args...) 29#endif 30#if DEBUG_ALLOC > 1 31# define DBGA2(args...) printk(KERN_DEBUG args) 32#else 33# define DBGA2(args...) 34#endif 35 36#define DEBUG_NODIRECT 0 37 38#define ISA_DMA_MASK 0x00ffffff 39 40static inline unsigned long 41mk_iommu_pte(unsigned long paddr) 42{ 43 return (paddr >> (PAGE_SHIFT-1)) | 1; 44} 45 46/* Return the minimum of MAX or the first power of two larger 47 than main memory. */ 48 49unsigned long 50size_for_memory(unsigned long max) 51{ 52 unsigned long mem = max_low_pfn << PAGE_SHIFT; 53 if (mem < max) 54 max = roundup_pow_of_two(mem); 55 return max; 56} 57 58struct pci_iommu_arena * __init 59iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base, 60 unsigned long window_size, unsigned long align) 61{ 62 unsigned long mem_size; 63 struct pci_iommu_arena *arena; 64 65 mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long)); 66 67 /* Note that the TLB lookup logic uses bitwise concatenation, 68 not addition, so the required arena alignment is based on 69 the size of the window. Retain the align parameter so that 70 particular systems can over-align the arena. */ 71 if (align < mem_size) 72 align = mem_size; 73 74 arena = memblock_alloc(sizeof(*arena), SMP_CACHE_BYTES); 75 if (!arena) 76 panic("%s: Failed to allocate %zu bytes\n", __func__, 77 sizeof(*arena)); 78 arena->ptes = memblock_alloc(mem_size, align); 79 if (!arena->ptes) 80 panic("%s: Failed to allocate %lu bytes align=0x%lx\n", 81 __func__, mem_size, align); 82 83 spin_lock_init(&arena->lock); 84 arena->hose = hose; 85 arena->dma_base = base; 86 arena->size = window_size; 87 arena->next_entry = 0; 88 89 /* Align allocations to a multiple of a page size. Not needed 90 unless there are chip bugs. */ 91 arena->align_entry = 1; 92 93 return arena; 94} 95 96struct pci_iommu_arena * __init 97iommu_arena_new(struct pci_controller *hose, dma_addr_t base, 98 unsigned long window_size, unsigned long align) 99{ 100 return iommu_arena_new_node(0, hose, base, window_size, align); 101} 102 103/* Must be called with the arena lock held */ 104static long 105iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena, 106 long n, long mask) 107{ 108 unsigned long *ptes; 109 long i, p, nent; 110 int pass = 0; 111 unsigned long base; 112 unsigned long boundary_size; 113 114 base = arena->dma_base >> PAGE_SHIFT; 115 boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT); 116 117 /* Search forward for the first mask-aligned sequence of N free ptes */ 118 ptes = arena->ptes; 119 nent = arena->size >> PAGE_SHIFT; 120 p = ALIGN(arena->next_entry, mask + 1); 121 i = 0; 122 123again: 124 while (i < n && p+i < nent) { 125 if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) { 126 p = ALIGN(p + 1, mask + 1); 127 goto again; 128 } 129 130 if (ptes[p+i]) { 131 p = ALIGN(p + i + 1, mask + 1); 132 i = 0; 133 } else { 134 i = i + 1; 135 } 136 } 137 138 if (i < n) { 139 if (pass < 1) { 140 /* 141 * Reached the end. Flush the TLB and restart 142 * the search from the beginning. 143 */ 144 alpha_mv.mv_pci_tbi(arena->hose, 0, -1); 145 146 pass++; 147 p = 0; 148 i = 0; 149 goto again; 150 } else 151 return -1; 152 } 153 154 /* Success. It's the responsibility of the caller to mark them 155 in use before releasing the lock */ 156 return p; 157} 158 159static long 160iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n, 161 unsigned int align) 162{ 163 unsigned long flags; 164 unsigned long *ptes; 165 long i, p, mask; 166 167 spin_lock_irqsave(&arena->lock, flags); 168 169 /* Search for N empty ptes */ 170 ptes = arena->ptes; 171 mask = max(align, arena->align_entry) - 1; 172 p = iommu_arena_find_pages(dev, arena, n, mask); 173 if (p < 0) { 174 spin_unlock_irqrestore(&arena->lock, flags); 175 return -1; 176 } 177 178 /* Success. Mark them all in use, ie not zero and invalid 179 for the iommu tlb that could load them from under us. 180 The chip specific bits will fill this in with something 181 kosher when we return. */ 182 for (i = 0; i < n; ++i) 183 ptes[p+i] = IOMMU_INVALID_PTE; 184 185 arena->next_entry = p + n; 186 spin_unlock_irqrestore(&arena->lock, flags); 187 188 return p; 189} 190 191static void 192iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n) 193{ 194 unsigned long *p; 195 long i; 196 197 p = arena->ptes + ofs; 198 for (i = 0; i < n; ++i) 199 p[i] = 0; 200} 201 202/* 203 * True if the machine supports DAC addressing, and DEV can 204 * make use of it given MASK. 205 */ 206static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask) 207{ 208 dma_addr_t dac_offset = alpha_mv.pci_dac_offset; 209 int ok = 1; 210 211 /* If this is not set, the machine doesn't support DAC at all. */ 212 if (dac_offset == 0) 213 ok = 0; 214 215 /* The device has to be able to address our DAC bit. */ 216 if ((dac_offset & dev->dma_mask) != dac_offset) 217 ok = 0; 218 219 /* If both conditions above are met, we are fine. */ 220 DBGA("pci_dac_dma_supported %s from %ps\n", 221 ok ? "yes" : "no", __builtin_return_address(0)); 222 223 return ok; 224} 225 226/* Map a single buffer of the indicated size for PCI DMA in streaming 227 mode. The 32-bit PCI bus mastering address to use is returned. 228 Once the device is given the dma address, the device owns this memory 229 until either pci_unmap_single or pci_dma_sync_single is performed. */ 230 231static dma_addr_t 232pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size, 233 int dac_allowed) 234{ 235 struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; 236 dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; 237 struct pci_iommu_arena *arena; 238 long npages, dma_ofs, i; 239 unsigned long paddr; 240 dma_addr_t ret; 241 unsigned int align = 0; 242 struct device *dev = pdev ? &pdev->dev : NULL; 243 244 paddr = __pa(cpu_addr); 245 246#if !DEBUG_NODIRECT 247 /* First check to see if we can use the direct map window. */ 248 if (paddr + size + __direct_map_base - 1 <= max_dma 249 && paddr + size <= __direct_map_size) { 250 ret = paddr + __direct_map_base; 251 252 DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %ps\n", 253 cpu_addr, size, ret, __builtin_return_address(0)); 254 255 return ret; 256 } 257#endif 258 259 /* Next, use DAC if selected earlier. */ 260 if (dac_allowed) { 261 ret = paddr + alpha_mv.pci_dac_offset; 262 263 DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %ps\n", 264 cpu_addr, size, ret, __builtin_return_address(0)); 265 266 return ret; 267 } 268 269 /* If the machine doesn't define a pci_tbi routine, we have to 270 assume it doesn't support sg mapping, and, since we tried to 271 use direct_map above, it now must be considered an error. */ 272 if (! alpha_mv.mv_pci_tbi) { 273 printk_once(KERN_WARNING "pci_map_single: no HW sg\n"); 274 return DMA_MAPPING_ERROR; 275 } 276 277 arena = hose->sg_pci; 278 if (!arena || arena->dma_base + arena->size - 1 > max_dma) 279 arena = hose->sg_isa; 280 281 npages = iommu_num_pages(paddr, size, PAGE_SIZE); 282 283 /* Force allocation to 64KB boundary for ISA bridges. */ 284 if (pdev && pdev == isa_bridge) 285 align = 8; 286 dma_ofs = iommu_arena_alloc(dev, arena, npages, align); 287 if (dma_ofs < 0) { 288 printk(KERN_WARNING "pci_map_single failed: " 289 "could not allocate dma page tables\n"); 290 return DMA_MAPPING_ERROR; 291 } 292 293 paddr &= PAGE_MASK; 294 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) 295 arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr); 296 297 ret = arena->dma_base + dma_ofs * PAGE_SIZE; 298 ret += (unsigned long)cpu_addr & ~PAGE_MASK; 299 300 DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %ps\n", 301 cpu_addr, size, npages, ret, __builtin_return_address(0)); 302 303 return ret; 304} 305 306/* Helper for generic DMA-mapping functions. */ 307static struct pci_dev *alpha_gendev_to_pci(struct device *dev) 308{ 309 if (dev && dev_is_pci(dev)) 310 return to_pci_dev(dev); 311 312 /* Assume that non-PCI devices asking for DMA are either ISA or EISA, 313 BUG() otherwise. */ 314 BUG_ON(!isa_bridge); 315 316 /* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA 317 bridge is bus master then). */ 318 if (!dev || !dev->dma_mask || !*dev->dma_mask) 319 return isa_bridge; 320 321 /* For EISA bus masters, return isa_bridge (it might have smaller 322 dma_mask due to wiring limitations). */ 323 if (*dev->dma_mask >= isa_bridge->dma_mask) 324 return isa_bridge; 325 326 /* This assumes ISA bus master with dma_mask 0xffffff. */ 327 return NULL; 328} 329 330static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page, 331 unsigned long offset, size_t size, 332 enum dma_data_direction dir, 333 unsigned long attrs) 334{ 335 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 336 int dac_allowed; 337 338 BUG_ON(dir == DMA_NONE); 339 340 dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; 341 return pci_map_single_1(pdev, (char *)page_address(page) + offset, 342 size, dac_allowed); 343} 344 345/* Unmap a single streaming mode DMA translation. The DMA_ADDR and 346 SIZE must match what was provided for in a previous pci_map_single 347 call. All other usages are undefined. After this call, reads by 348 the cpu to the buffer are guaranteed to see whatever the device 349 wrote there. */ 350 351static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr, 352 size_t size, enum dma_data_direction dir, 353 unsigned long attrs) 354{ 355 unsigned long flags; 356 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 357 struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; 358 struct pci_iommu_arena *arena; 359 long dma_ofs, npages; 360 361 BUG_ON(dir == DMA_NONE); 362 363 if (dma_addr >= __direct_map_base 364 && dma_addr < __direct_map_base + __direct_map_size) { 365 /* Nothing to do. */ 366 367 DBGA2("pci_unmap_single: direct [%llx,%zx] from %ps\n", 368 dma_addr, size, __builtin_return_address(0)); 369 370 return; 371 } 372 373 if (dma_addr > 0xffffffff) { 374 DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %ps\n", 375 dma_addr, size, __builtin_return_address(0)); 376 return; 377 } 378 379 arena = hose->sg_pci; 380 if (!arena || dma_addr < arena->dma_base) 381 arena = hose->sg_isa; 382 383 dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT; 384 if (dma_ofs * PAGE_SIZE >= arena->size) { 385 printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx " 386 " base %llx size %x\n", 387 dma_addr, arena->dma_base, arena->size); 388 return; 389 BUG(); 390 } 391 392 npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); 393 394 spin_lock_irqsave(&arena->lock, flags); 395 396 iommu_arena_free(arena, dma_ofs, npages); 397 398 /* If we're freeing ptes above the `next_entry' pointer (they 399 may have snuck back into the TLB since the last wrap flush), 400 we need to flush the TLB before reallocating the latter. */ 401 if (dma_ofs >= arena->next_entry) 402 alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1); 403 404 spin_unlock_irqrestore(&arena->lock, flags); 405 406 DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %ps\n", 407 dma_addr, size, npages, __builtin_return_address(0)); 408} 409 410/* Allocate and map kernel buffer using consistent mode DMA for PCI 411 device. Returns non-NULL cpu-view pointer to the buffer if 412 successful and sets *DMA_ADDRP to the pci side dma address as well, 413 else DMA_ADDRP is undefined. */ 414 415static void *alpha_pci_alloc_coherent(struct device *dev, size_t size, 416 dma_addr_t *dma_addrp, gfp_t gfp, 417 unsigned long attrs) 418{ 419 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 420 void *cpu_addr; 421 long order = get_order(size); 422 423 gfp &= ~GFP_DMA; 424 425try_again: 426 cpu_addr = (void *)__get_free_pages(gfp | __GFP_ZERO, order); 427 if (! cpu_addr) { 428 printk(KERN_INFO "pci_alloc_consistent: " 429 "get_free_pages failed from %ps\n", 430 __builtin_return_address(0)); 431 /* ??? Really atomic allocation? Otherwise we could play 432 with vmalloc and sg if we can't find contiguous memory. */ 433 return NULL; 434 } 435 memset(cpu_addr, 0, size); 436 437 *dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0); 438 if (*dma_addrp == DMA_MAPPING_ERROR) { 439 free_pages((unsigned long)cpu_addr, order); 440 if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA)) 441 return NULL; 442 /* The address doesn't fit required mask and we 443 do not have iommu. Try again with GFP_DMA. */ 444 gfp |= GFP_DMA; 445 goto try_again; 446 } 447 448 DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %ps\n", 449 size, cpu_addr, *dma_addrp, __builtin_return_address(0)); 450 451 return cpu_addr; 452} 453 454/* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must 455 be values that were returned from pci_alloc_consistent. SIZE must 456 be the same as what as passed into pci_alloc_consistent. 457 References to the memory and mappings associated with CPU_ADDR or 458 DMA_ADDR past this call are illegal. */ 459 460static void alpha_pci_free_coherent(struct device *dev, size_t size, 461 void *cpu_addr, dma_addr_t dma_addr, 462 unsigned long attrs) 463{ 464 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 465 dma_unmap_single(&pdev->dev, dma_addr, size, DMA_BIDIRECTIONAL); 466 free_pages((unsigned long)cpu_addr, get_order(size)); 467 468 DBGA2("pci_free_consistent: [%llx,%zx] from %ps\n", 469 dma_addr, size, __builtin_return_address(0)); 470} 471 472/* Classify the elements of the scatterlist. Write dma_address 473 of each element with: 474 0 : Followers all physically adjacent. 475 1 : Followers all virtually adjacent. 476 -1 : Not leader, physically adjacent to previous. 477 -2 : Not leader, virtually adjacent to previous. 478 Write dma_length of each leader with the combined lengths of 479 the mergable followers. */ 480 481#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) 482#define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG)) 483 484static void 485sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end, 486 int virt_ok) 487{ 488 unsigned long next_paddr; 489 struct scatterlist *leader; 490 long leader_flag, leader_length; 491 unsigned int max_seg_size; 492 493 leader = sg; 494 leader_flag = 0; 495 leader_length = leader->length; 496 next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length; 497 498 /* we will not marge sg without device. */ 499 max_seg_size = dev ? dma_get_max_seg_size(dev) : 0; 500 for (++sg; sg < end; ++sg) { 501 unsigned long addr, len; 502 addr = SG_ENT_PHYS_ADDRESS(sg); 503 len = sg->length; 504 505 if (leader_length + len > max_seg_size) 506 goto new_segment; 507 508 if (next_paddr == addr) { 509 sg->dma_address = -1; 510 leader_length += len; 511 } else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) { 512 sg->dma_address = -2; 513 leader_flag = 1; 514 leader_length += len; 515 } else { 516new_segment: 517 leader->dma_address = leader_flag; 518 leader->dma_length = leader_length; 519 leader = sg; 520 leader_flag = 0; 521 leader_length = len; 522 } 523 524 next_paddr = addr + len; 525 } 526 527 leader->dma_address = leader_flag; 528 leader->dma_length = leader_length; 529} 530 531/* Given a scatterlist leader, choose an allocation method and fill 532 in the blanks. */ 533 534static int 535sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end, 536 struct scatterlist *out, struct pci_iommu_arena *arena, 537 dma_addr_t max_dma, int dac_allowed) 538{ 539 unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader); 540 long size = leader->dma_length; 541 struct scatterlist *sg; 542 unsigned long *ptes; 543 long npages, dma_ofs, i; 544 545#if !DEBUG_NODIRECT 546 /* If everything is physically contiguous, and the addresses 547 fall into the direct-map window, use it. */ 548 if (leader->dma_address == 0 549 && paddr + size + __direct_map_base - 1 <= max_dma 550 && paddr + size <= __direct_map_size) { 551 out->dma_address = paddr + __direct_map_base; 552 out->dma_length = size; 553 554 DBGA(" sg_fill: [%p,%lx] -> direct %llx\n", 555 __va(paddr), size, out->dma_address); 556 557 return 0; 558 } 559#endif 560 561 /* If physically contiguous and DAC is available, use it. */ 562 if (leader->dma_address == 0 && dac_allowed) { 563 out->dma_address = paddr + alpha_mv.pci_dac_offset; 564 out->dma_length = size; 565 566 DBGA(" sg_fill: [%p,%lx] -> DAC %llx\n", 567 __va(paddr), size, out->dma_address); 568 569 return 0; 570 } 571 572 /* Otherwise, we'll use the iommu to make the pages virtually 573 contiguous. */ 574 575 paddr &= ~PAGE_MASK; 576 npages = iommu_num_pages(paddr, size, PAGE_SIZE); 577 dma_ofs = iommu_arena_alloc(dev, arena, npages, 0); 578 if (dma_ofs < 0) { 579 /* If we attempted a direct map above but failed, die. */ 580 if (leader->dma_address == 0) 581 return -1; 582 583 /* Otherwise, break up the remaining virtually contiguous 584 hunks into individual direct maps and retry. */ 585 sg_classify(dev, leader, end, 0); 586 return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed); 587 } 588 589 out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr; 590 out->dma_length = size; 591 592 DBGA(" sg_fill: [%p,%lx] -> sg %llx np %ld\n", 593 __va(paddr), size, out->dma_address, npages); 594 595 /* All virtually contiguous. We need to find the length of each 596 physically contiguous subsegment to fill in the ptes. */ 597 ptes = &arena->ptes[dma_ofs]; 598 sg = leader; 599 do { 600#if DEBUG_ALLOC > 0 601 struct scatterlist *last_sg = sg; 602#endif 603 604 size = sg->length; 605 paddr = SG_ENT_PHYS_ADDRESS(sg); 606 607 while (sg+1 < end && (int) sg[1].dma_address == -1) { 608 size += sg[1].length; 609 sg = sg_next(sg); 610 } 611 612 npages = iommu_num_pages(paddr, size, PAGE_SIZE); 613 614 paddr &= PAGE_MASK; 615 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) 616 *ptes++ = mk_iommu_pte(paddr); 617 618#if DEBUG_ALLOC > 0 619 DBGA(" (%ld) [%p,%x] np %ld\n", 620 last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg), 621 last_sg->length, npages); 622 while (++last_sg <= sg) { 623 DBGA(" (%ld) [%p,%x] cont\n", 624 last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg), 625 last_sg->length); 626 } 627#endif 628 } while (++sg < end && (int) sg->dma_address < 0); 629 630 return 1; 631} 632 633static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg, 634 int nents, enum dma_data_direction dir, 635 unsigned long attrs) 636{ 637 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 638 struct scatterlist *start, *end, *out; 639 struct pci_controller *hose; 640 struct pci_iommu_arena *arena; 641 dma_addr_t max_dma; 642 int dac_allowed; 643 644 BUG_ON(dir == DMA_NONE); 645 646 dac_allowed = dev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; 647 648 /* Fast path single entry scatterlists. */ 649 if (nents == 1) { 650 sg->dma_length = sg->length; 651 sg->dma_address 652 = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg), 653 sg->length, dac_allowed); 654 if (sg->dma_address == DMA_MAPPING_ERROR) 655 return -EIO; 656 return 1; 657 } 658 659 start = sg; 660 end = sg + nents; 661 662 /* First, prepare information about the entries. */ 663 sg_classify(dev, sg, end, alpha_mv.mv_pci_tbi != 0); 664 665 /* Second, figure out where we're going to map things. */ 666 if (alpha_mv.mv_pci_tbi) { 667 hose = pdev ? pdev->sysdata : pci_isa_hose; 668 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; 669 arena = hose->sg_pci; 670 if (!arena || arena->dma_base + arena->size - 1 > max_dma) 671 arena = hose->sg_isa; 672 } else { 673 max_dma = -1; 674 arena = NULL; 675 hose = NULL; 676 } 677 678 /* Third, iterate over the scatterlist leaders and allocate 679 dma space as needed. */ 680 for (out = sg; sg < end; ++sg) { 681 if ((int) sg->dma_address < 0) 682 continue; 683 if (sg_fill(dev, sg, end, out, arena, max_dma, dac_allowed) < 0) 684 goto error; 685 out++; 686 } 687 688 /* Mark the end of the list for pci_unmap_sg. */ 689 if (out < end) 690 out->dma_length = 0; 691 692 if (out - start == 0) { 693 printk(KERN_WARNING "pci_map_sg failed: no entries?\n"); 694 return -ENOMEM; 695 } 696 DBGA("pci_map_sg: %ld entries\n", out - start); 697 698 return out - start; 699 700 error: 701 printk(KERN_WARNING "pci_map_sg failed: " 702 "could not allocate dma page tables\n"); 703 704 /* Some allocation failed while mapping the scatterlist 705 entries. Unmap them now. */ 706 if (out > start) 707 dma_unmap_sg(&pdev->dev, start, out - start, dir); 708 return -ENOMEM; 709} 710 711/* Unmap a set of streaming mode DMA translations. Again, cpu read 712 rules concerning calls here are the same as for pci_unmap_single() 713 above. */ 714 715static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg, 716 int nents, enum dma_data_direction dir, 717 unsigned long attrs) 718{ 719 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 720 unsigned long flags; 721 struct pci_controller *hose; 722 struct pci_iommu_arena *arena; 723 struct scatterlist *end; 724 dma_addr_t max_dma; 725 dma_addr_t fbeg, fend; 726 727 BUG_ON(dir == DMA_NONE); 728 729 if (! alpha_mv.mv_pci_tbi) 730 return; 731 732 hose = pdev ? pdev->sysdata : pci_isa_hose; 733 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; 734 arena = hose->sg_pci; 735 if (!arena || arena->dma_base + arena->size - 1 > max_dma) 736 arena = hose->sg_isa; 737 738 fbeg = -1, fend = 0; 739 740 spin_lock_irqsave(&arena->lock, flags); 741 742 for (end = sg + nents; sg < end; ++sg) { 743 dma_addr_t addr; 744 size_t size; 745 long npages, ofs; 746 dma_addr_t tend; 747 748 addr = sg->dma_address; 749 size = sg->dma_length; 750 if (!size) 751 break; 752 753 if (addr > 0xffffffff) { 754 /* It's a DAC address -- nothing to do. */ 755 DBGA(" (%ld) DAC [%llx,%zx]\n", 756 sg - end + nents, addr, size); 757 continue; 758 } 759 760 if (addr >= __direct_map_base 761 && addr < __direct_map_base + __direct_map_size) { 762 /* Nothing to do. */ 763 DBGA(" (%ld) direct [%llx,%zx]\n", 764 sg - end + nents, addr, size); 765 continue; 766 } 767 768 DBGA(" (%ld) sg [%llx,%zx]\n", 769 sg - end + nents, addr, size); 770 771 npages = iommu_num_pages(addr, size, PAGE_SIZE); 772 ofs = (addr - arena->dma_base) >> PAGE_SHIFT; 773 iommu_arena_free(arena, ofs, npages); 774 775 tend = addr + size - 1; 776 if (fbeg > addr) fbeg = addr; 777 if (fend < tend) fend = tend; 778 } 779 780 /* If we're freeing ptes above the `next_entry' pointer (they 781 may have snuck back into the TLB since the last wrap flush), 782 we need to flush the TLB before reallocating the latter. */ 783 if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry) 784 alpha_mv.mv_pci_tbi(hose, fbeg, fend); 785 786 spin_unlock_irqrestore(&arena->lock, flags); 787 788 DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg)); 789} 790 791/* Return whether the given PCI device DMA address mask can be 792 supported properly. */ 793 794static int alpha_pci_supported(struct device *dev, u64 mask) 795{ 796 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 797 struct pci_controller *hose; 798 struct pci_iommu_arena *arena; 799 800 /* If there exists a direct map, and the mask fits either 801 the entire direct mapped space or the total system memory as 802 shifted by the map base */ 803 if (__direct_map_size != 0 804 && (__direct_map_base + __direct_map_size - 1 <= mask || 805 __direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask)) 806 return 1; 807 808 /* Check that we have a scatter-gather arena that fits. */ 809 hose = pdev ? pdev->sysdata : pci_isa_hose; 810 arena = hose->sg_isa; 811 if (arena && arena->dma_base + arena->size - 1 <= mask) 812 return 1; 813 arena = hose->sg_pci; 814 if (arena && arena->dma_base + arena->size - 1 <= mask) 815 return 1; 816 817 /* As last resort try ZONE_DMA. */ 818 if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask) 819 return 1; 820 821 return 0; 822} 823 824 825/* 826 * AGP GART extensions to the IOMMU 827 */ 828int 829iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask) 830{ 831 unsigned long flags; 832 unsigned long *ptes; 833 long i, p; 834 835 if (!arena) return -EINVAL; 836 837 spin_lock_irqsave(&arena->lock, flags); 838 839 /* Search for N empty ptes. */ 840 ptes = arena->ptes; 841 p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask); 842 if (p < 0) { 843 spin_unlock_irqrestore(&arena->lock, flags); 844 return -1; 845 } 846 847 /* Success. Mark them all reserved (ie not zero and invalid) 848 for the iommu tlb that could load them from under us. 849 They will be filled in with valid bits by _bind() */ 850 for (i = 0; i < pg_count; ++i) 851 ptes[p+i] = IOMMU_RESERVED_PTE; 852 853 arena->next_entry = p + pg_count; 854 spin_unlock_irqrestore(&arena->lock, flags); 855 856 return p; 857} 858 859int 860iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count) 861{ 862 unsigned long *ptes; 863 long i; 864 865 if (!arena) return -EINVAL; 866 867 ptes = arena->ptes; 868 869 /* Make sure they're all reserved first... */ 870 for(i = pg_start; i < pg_start + pg_count; i++) 871 if (ptes[i] != IOMMU_RESERVED_PTE) 872 return -EBUSY; 873 874 iommu_arena_free(arena, pg_start, pg_count); 875 return 0; 876} 877 878int 879iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count, 880 struct page **pages) 881{ 882 unsigned long flags; 883 unsigned long *ptes; 884 long i, j; 885 886 if (!arena) return -EINVAL; 887 888 spin_lock_irqsave(&arena->lock, flags); 889 890 ptes = arena->ptes; 891 892 for(j = pg_start; j < pg_start + pg_count; j++) { 893 if (ptes[j] != IOMMU_RESERVED_PTE) { 894 spin_unlock_irqrestore(&arena->lock, flags); 895 return -EBUSY; 896 } 897 } 898 899 for(i = 0, j = pg_start; i < pg_count; i++, j++) 900 ptes[j] = mk_iommu_pte(page_to_phys(pages[i])); 901 902 spin_unlock_irqrestore(&arena->lock, flags); 903 904 return 0; 905} 906 907int 908iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count) 909{ 910 unsigned long *p; 911 long i; 912 913 if (!arena) return -EINVAL; 914 915 p = arena->ptes + pg_start; 916 for(i = 0; i < pg_count; i++) 917 p[i] = IOMMU_RESERVED_PTE; 918 919 return 0; 920} 921 922const struct dma_map_ops alpha_pci_ops = { 923 .alloc = alpha_pci_alloc_coherent, 924 .free = alpha_pci_free_coherent, 925 .map_page = alpha_pci_map_page, 926 .unmap_page = alpha_pci_unmap_page, 927 .map_sg = alpha_pci_map_sg, 928 .unmap_sg = alpha_pci_unmap_sg, 929 .dma_supported = alpha_pci_supported, 930 .mmap = dma_common_mmap, 931 .get_sgtable = dma_common_get_sgtable, 932 .alloc_pages = dma_common_alloc_pages, 933 .free_pages = dma_common_free_pages, 934}; 935EXPORT_SYMBOL(alpha_pci_ops); 936