1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Dynamic DMA mapping support for AMD Hammer. 4 * 5 * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI. 6 * This allows to use PCI devices that only support 32bit addresses on systems 7 * with more than 4GB. 8 * 9 * See Documentation/core-api/dma-api-howto.rst for the interface specification. 10 * 11 * Copyright 2002 Andi Kleen, SuSE Labs. 12 */ 13 14#include <linux/types.h> 15#include <linux/ctype.h> 16#include <linux/agp_backend.h> 17#include <linux/init.h> 18#include <linux/mm.h> 19#include <linux/sched.h> 20#include <linux/sched/debug.h> 21#include <linux/string.h> 22#include <linux/spinlock.h> 23#include <linux/pci.h> 24#include <linux/topology.h> 25#include <linux/interrupt.h> 26#include <linux/bitmap.h> 27#include <linux/kdebug.h> 28#include <linux/scatterlist.h> 29#include <linux/iommu-helper.h> 30#include <linux/syscore_ops.h> 31#include <linux/io.h> 32#include <linux/gfp.h> 33#include <linux/atomic.h> 34#include <linux/dma-direct.h> 35#include <linux/dma-map-ops.h> 36#include <asm/mtrr.h> 37#include <asm/proto.h> 38#include <asm/iommu.h> 39#include <asm/gart.h> 40#include <asm/set_memory.h> 41#include <asm/dma.h> 42#include <asm/amd_nb.h> 43#include <asm/x86_init.h> 44 45static unsigned long iommu_bus_base; /* GART remapping area (physical) */ 46static unsigned long iommu_size; /* size of remapping area bytes */ 47static unsigned long iommu_pages; /* .. and in pages */ 48 49static u32 *iommu_gatt_base; /* Remapping table */ 50 51/* 52 * If this is disabled the IOMMU will use an optimized flushing strategy 53 * of only flushing when an mapping is reused. With it true the GART is 54 * flushed for every mapping. Problem is that doing the lazy flush seems 55 * to trigger bugs with some popular PCI cards, in particular 3ware (but 56 * has been also seen with Qlogic at least). 57 */ 58static int iommu_fullflush = 1; 59 60/* Allocation bitmap for the remapping area: */ 61static DEFINE_SPINLOCK(iommu_bitmap_lock); 62/* Guarded by iommu_bitmap_lock: */ 63static unsigned long *iommu_gart_bitmap; 64 65static u32 gart_unmapped_entry; 66 67#define GPTE_VALID 1 68#define GPTE_COHERENT 2 69#define GPTE_ENCODE(x) \ 70 (((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT) 71#define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28)) 72 73#ifdef CONFIG_AGP 74#define AGPEXTERN extern 75#else 76#define AGPEXTERN 77#endif 78 79/* GART can only remap to physical addresses < 1TB */ 80#define GART_MAX_PHYS_ADDR (1ULL << 40) 81 82/* backdoor interface to AGP driver */ 83AGPEXTERN int agp_memory_reserved; 84AGPEXTERN __u32 *agp_gatt_table; 85 86static unsigned long next_bit; /* protected by iommu_bitmap_lock */ 87static bool need_flush; /* global flush state. set for each gart wrap */ 88 89static unsigned long alloc_iommu(struct device *dev, int size, 90 unsigned long align_mask) 91{ 92 unsigned long offset, flags; 93 unsigned long boundary_size; 94 unsigned long base_index; 95 96 base_index = ALIGN(iommu_bus_base & dma_get_seg_boundary(dev), 97 PAGE_SIZE) >> PAGE_SHIFT; 98 boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT); 99 100 spin_lock_irqsave(&iommu_bitmap_lock, flags); 101 offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, next_bit, 102 size, base_index, boundary_size, align_mask); 103 if (offset == -1) { 104 need_flush = true; 105 offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, 0, 106 size, base_index, boundary_size, 107 align_mask); 108 } 109 if (offset != -1) { 110 next_bit = offset+size; 111 if (next_bit >= iommu_pages) { 112 next_bit = 0; 113 need_flush = true; 114 } 115 } 116 if (iommu_fullflush) 117 need_flush = true; 118 spin_unlock_irqrestore(&iommu_bitmap_lock, flags); 119 120 return offset; 121} 122 123static void free_iommu(unsigned long offset, int size) 124{ 125 unsigned long flags; 126 127 spin_lock_irqsave(&iommu_bitmap_lock, flags); 128 bitmap_clear(iommu_gart_bitmap, offset, size); 129 if (offset >= next_bit) 130 next_bit = offset + size; 131 spin_unlock_irqrestore(&iommu_bitmap_lock, flags); 132} 133 134/* 135 * Use global flush state to avoid races with multiple flushers. 136 */ 137static void flush_gart(void) 138{ 139 unsigned long flags; 140 141 spin_lock_irqsave(&iommu_bitmap_lock, flags); 142 if (need_flush) { 143 amd_flush_garts(); 144 need_flush = false; 145 } 146 spin_unlock_irqrestore(&iommu_bitmap_lock, flags); 147} 148 149#ifdef CONFIG_IOMMU_LEAK 150/* Debugging aid for drivers that don't free their IOMMU tables */ 151static void dump_leak(void) 152{ 153 static int dump; 154 155 if (dump) 156 return; 157 dump = 1; 158 159 show_stack(NULL, NULL, KERN_ERR); 160 debug_dma_dump_mappings(NULL); 161} 162#endif 163 164static void iommu_full(struct device *dev, size_t size, int dir) 165{ 166 /* 167 * Ran out of IOMMU space for this operation. This is very bad. 168 * Unfortunately the drivers cannot handle this operation properly. 169 * Return some non mapped prereserved space in the aperture and 170 * let the Northbridge deal with it. This will result in garbage 171 * in the IO operation. When the size exceeds the prereserved space 172 * memory corruption will occur or random memory will be DMAed 173 * out. Hopefully no network devices use single mappings that big. 174 */ 175 176 dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n", size); 177#ifdef CONFIG_IOMMU_LEAK 178 dump_leak(); 179#endif 180} 181 182static inline int 183need_iommu(struct device *dev, unsigned long addr, size_t size) 184{ 185 return force_iommu || !dma_capable(dev, addr, size, true); 186} 187 188static inline int 189nonforced_iommu(struct device *dev, unsigned long addr, size_t size) 190{ 191 return !dma_capable(dev, addr, size, true); 192} 193 194/* Map a single continuous physical area into the IOMMU. 195 * Caller needs to check if the iommu is needed and flush. 196 */ 197static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, 198 size_t size, int dir, unsigned long align_mask) 199{ 200 unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE); 201 unsigned long iommu_page; 202 int i; 203 204 if (unlikely(phys_mem + size > GART_MAX_PHYS_ADDR)) 205 return DMA_MAPPING_ERROR; 206 207 iommu_page = alloc_iommu(dev, npages, align_mask); 208 if (iommu_page == -1) { 209 if (!nonforced_iommu(dev, phys_mem, size)) 210 return phys_mem; 211 if (panic_on_overflow) 212 panic("dma_map_area overflow %lu bytes\n", size); 213 iommu_full(dev, size, dir); 214 return DMA_MAPPING_ERROR; 215 } 216 217 for (i = 0; i < npages; i++) { 218 iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem); 219 phys_mem += PAGE_SIZE; 220 } 221 return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK); 222} 223 224/* Map a single area into the IOMMU */ 225static dma_addr_t gart_map_page(struct device *dev, struct page *page, 226 unsigned long offset, size_t size, 227 enum dma_data_direction dir, 228 unsigned long attrs) 229{ 230 unsigned long bus; 231 phys_addr_t paddr = page_to_phys(page) + offset; 232 233 if (!need_iommu(dev, paddr, size)) 234 return paddr; 235 236 bus = dma_map_area(dev, paddr, size, dir, 0); 237 flush_gart(); 238 239 return bus; 240} 241 242/* 243 * Free a DMA mapping. 244 */ 245static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr, 246 size_t size, enum dma_data_direction dir, 247 unsigned long attrs) 248{ 249 unsigned long iommu_page; 250 int npages; 251 int i; 252 253 if (WARN_ON_ONCE(dma_addr == DMA_MAPPING_ERROR)) 254 return; 255 256 /* 257 * This driver will not always use a GART mapping, but might have 258 * created a direct mapping instead. If that is the case there is 259 * nothing to unmap here. 260 */ 261 if (dma_addr < iommu_bus_base || 262 dma_addr >= iommu_bus_base + iommu_size) 263 return; 264 265 iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT; 266 npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); 267 for (i = 0; i < npages; i++) { 268 iommu_gatt_base[iommu_page + i] = gart_unmapped_entry; 269 } 270 free_iommu(iommu_page, npages); 271} 272 273/* 274 * Wrapper for pci_unmap_single working with scatterlists. 275 */ 276static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 277 enum dma_data_direction dir, unsigned long attrs) 278{ 279 struct scatterlist *s; 280 int i; 281 282 for_each_sg(sg, s, nents, i) { 283 if (!s->dma_length || !s->length) 284 break; 285 gart_unmap_page(dev, s->dma_address, s->dma_length, dir, 0); 286 } 287} 288 289/* Fallback for dma_map_sg in case of overflow */ 290static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg, 291 int nents, int dir) 292{ 293 struct scatterlist *s; 294 int i; 295 296#ifdef CONFIG_IOMMU_DEBUG 297 pr_debug("dma_map_sg overflow\n"); 298#endif 299 300 for_each_sg(sg, s, nents, i) { 301 unsigned long addr = sg_phys(s); 302 303 if (nonforced_iommu(dev, addr, s->length)) { 304 addr = dma_map_area(dev, addr, s->length, dir, 0); 305 if (addr == DMA_MAPPING_ERROR) { 306 if (i > 0) 307 gart_unmap_sg(dev, sg, i, dir, 0); 308 nents = 0; 309 sg[0].dma_length = 0; 310 break; 311 } 312 } 313 s->dma_address = addr; 314 s->dma_length = s->length; 315 } 316 flush_gart(); 317 318 return nents; 319} 320 321/* Map multiple scatterlist entries continuous into the first. */ 322static int __dma_map_cont(struct device *dev, struct scatterlist *start, 323 int nelems, struct scatterlist *sout, 324 unsigned long pages) 325{ 326 unsigned long iommu_start = alloc_iommu(dev, pages, 0); 327 unsigned long iommu_page = iommu_start; 328 struct scatterlist *s; 329 int i; 330 331 if (iommu_start == -1) 332 return -ENOMEM; 333 334 for_each_sg(start, s, nelems, i) { 335 unsigned long pages, addr; 336 unsigned long phys_addr = s->dma_address; 337 338 BUG_ON(s != start && s->offset); 339 if (s == start) { 340 sout->dma_address = iommu_bus_base; 341 sout->dma_address += iommu_page*PAGE_SIZE + s->offset; 342 sout->dma_length = s->length; 343 } else { 344 sout->dma_length += s->length; 345 } 346 347 addr = phys_addr; 348 pages = iommu_num_pages(s->offset, s->length, PAGE_SIZE); 349 while (pages--) { 350 iommu_gatt_base[iommu_page] = GPTE_ENCODE(addr); 351 addr += PAGE_SIZE; 352 iommu_page++; 353 } 354 } 355 BUG_ON(iommu_page - iommu_start != pages); 356 357 return 0; 358} 359 360static inline int 361dma_map_cont(struct device *dev, struct scatterlist *start, int nelems, 362 struct scatterlist *sout, unsigned long pages, int need) 363{ 364 if (!need) { 365 BUG_ON(nelems != 1); 366 sout->dma_address = start->dma_address; 367 sout->dma_length = start->length; 368 return 0; 369 } 370 return __dma_map_cont(dev, start, nelems, sout, pages); 371} 372 373/* 374 * DMA map all entries in a scatterlist. 375 * Merge chunks that have page aligned sizes into a continuous mapping. 376 */ 377static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, 378 enum dma_data_direction dir, unsigned long attrs) 379{ 380 struct scatterlist *s, *ps, *start_sg, *sgmap; 381 int need = 0, nextneed, i, out, start, ret; 382 unsigned long pages = 0; 383 unsigned int seg_size; 384 unsigned int max_seg_size; 385 386 if (nents == 0) 387 return -EINVAL; 388 389 out = 0; 390 start = 0; 391 start_sg = sg; 392 sgmap = sg; 393 seg_size = 0; 394 max_seg_size = dma_get_max_seg_size(dev); 395 ps = NULL; /* shut up gcc */ 396 397 for_each_sg(sg, s, nents, i) { 398 dma_addr_t addr = sg_phys(s); 399 400 s->dma_address = addr; 401 BUG_ON(s->length == 0); 402 403 nextneed = need_iommu(dev, addr, s->length); 404 405 /* Handle the previous not yet processed entries */ 406 if (i > start) { 407 /* 408 * Can only merge when the last chunk ends on a 409 * page boundary and the new one doesn't have an 410 * offset. 411 */ 412 if (!iommu_merge || !nextneed || !need || s->offset || 413 (s->length + seg_size > max_seg_size) || 414 (ps->offset + ps->length) % PAGE_SIZE) { 415 ret = dma_map_cont(dev, start_sg, i - start, 416 sgmap, pages, need); 417 if (ret < 0) 418 goto error; 419 out++; 420 421 seg_size = 0; 422 sgmap = sg_next(sgmap); 423 pages = 0; 424 start = i; 425 start_sg = s; 426 } 427 } 428 429 seg_size += s->length; 430 need = nextneed; 431 pages += iommu_num_pages(s->offset, s->length, PAGE_SIZE); 432 ps = s; 433 } 434 ret = dma_map_cont(dev, start_sg, i - start, sgmap, pages, need); 435 if (ret < 0) 436 goto error; 437 out++; 438 flush_gart(); 439 if (out < nents) { 440 sgmap = sg_next(sgmap); 441 sgmap->dma_length = 0; 442 } 443 return out; 444 445error: 446 flush_gart(); 447 gart_unmap_sg(dev, sg, out, dir, 0); 448 449 /* When it was forced or merged try again in a dumb way */ 450 if (force_iommu || iommu_merge) { 451 out = dma_map_sg_nonforce(dev, sg, nents, dir); 452 if (out > 0) 453 return out; 454 } 455 if (panic_on_overflow) 456 panic("dma_map_sg: overflow on %lu pages\n", pages); 457 458 iommu_full(dev, pages << PAGE_SHIFT, dir); 459 return ret; 460} 461 462/* allocate and map a coherent mapping */ 463static void * 464gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, 465 gfp_t flag, unsigned long attrs) 466{ 467 void *vaddr; 468 469 vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs); 470 if (!vaddr || 471 !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24)) 472 return vaddr; 473 474 *dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size, 475 DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1); 476 flush_gart(); 477 if (unlikely(*dma_addr == DMA_MAPPING_ERROR)) 478 goto out_free; 479 return vaddr; 480out_free: 481 dma_direct_free(dev, size, vaddr, *dma_addr, attrs); 482 return NULL; 483} 484 485/* free a coherent mapping */ 486static void 487gart_free_coherent(struct device *dev, size_t size, void *vaddr, 488 dma_addr_t dma_addr, unsigned long attrs) 489{ 490 gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); 491 dma_direct_free(dev, size, vaddr, dma_addr, attrs); 492} 493 494static int no_agp; 495 496static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size) 497{ 498 unsigned long a; 499 500 if (!iommu_size) { 501 iommu_size = aper_size; 502 if (!no_agp) 503 iommu_size /= 2; 504 } 505 506 a = aper + iommu_size; 507 iommu_size -= round_up(a, PMD_SIZE) - a; 508 509 if (iommu_size < 64*1024*1024) { 510 pr_warn("PCI-DMA: Warning: Small IOMMU %luMB." 511 " Consider increasing the AGP aperture in BIOS\n", 512 iommu_size >> 20); 513 } 514 515 return iommu_size; 516} 517 518static __init unsigned read_aperture(struct pci_dev *dev, u32 *size) 519{ 520 unsigned aper_size = 0, aper_base_32, aper_order; 521 u64 aper_base; 522 523 pci_read_config_dword(dev, AMD64_GARTAPERTUREBASE, &aper_base_32); 524 pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &aper_order); 525 aper_order = (aper_order >> 1) & 7; 526 527 aper_base = aper_base_32 & 0x7fff; 528 aper_base <<= 25; 529 530 aper_size = (32 * 1024 * 1024) << aper_order; 531 if (aper_base + aper_size > 0x100000000UL || !aper_size) 532 aper_base = 0; 533 534 *size = aper_size; 535 return aper_base; 536} 537 538static void enable_gart_translations(void) 539{ 540 int i; 541 542 if (!amd_nb_has_feature(AMD_NB_GART)) 543 return; 544 545 for (i = 0; i < amd_nb_num(); i++) { 546 struct pci_dev *dev = node_to_amd_nb(i)->misc; 547 548 enable_gart_translation(dev, __pa(agp_gatt_table)); 549 } 550 551 /* Flush the GART-TLB to remove stale entries */ 552 amd_flush_garts(); 553} 554 555/* 556 * If fix_up_north_bridges is set, the north bridges have to be fixed up on 557 * resume in the same way as they are handled in gart_iommu_hole_init(). 558 */ 559static bool fix_up_north_bridges; 560static u32 aperture_order; 561static u32 aperture_alloc; 562 563void set_up_gart_resume(u32 aper_order, u32 aper_alloc) 564{ 565 fix_up_north_bridges = true; 566 aperture_order = aper_order; 567 aperture_alloc = aper_alloc; 568} 569 570static void gart_fixup_northbridges(void) 571{ 572 int i; 573 574 if (!fix_up_north_bridges) 575 return; 576 577 if (!amd_nb_has_feature(AMD_NB_GART)) 578 return; 579 580 pr_info("PCI-DMA: Restoring GART aperture settings\n"); 581 582 for (i = 0; i < amd_nb_num(); i++) { 583 struct pci_dev *dev = node_to_amd_nb(i)->misc; 584 585 /* 586 * Don't enable translations just yet. That is the next 587 * step. Restore the pre-suspend aperture settings. 588 */ 589 gart_set_size_and_enable(dev, aperture_order); 590 pci_write_config_dword(dev, AMD64_GARTAPERTUREBASE, aperture_alloc >> 25); 591 } 592} 593 594static void gart_resume(void) 595{ 596 pr_info("PCI-DMA: Resuming GART IOMMU\n"); 597 598 gart_fixup_northbridges(); 599 600 enable_gart_translations(); 601} 602 603static struct syscore_ops gart_syscore_ops = { 604 .resume = gart_resume, 605 606}; 607 608/* 609 * Private Northbridge GATT initialization in case we cannot use the 610 * AGP driver for some reason. 611 */ 612static __init int init_amd_gatt(struct agp_kern_info *info) 613{ 614 unsigned aper_size, gatt_size, new_aper_size; 615 unsigned aper_base, new_aper_base; 616 struct pci_dev *dev; 617 void *gatt; 618 int i; 619 620 pr_info("PCI-DMA: Disabling AGP.\n"); 621 622 aper_size = aper_base = info->aper_size = 0; 623 dev = NULL; 624 for (i = 0; i < amd_nb_num(); i++) { 625 dev = node_to_amd_nb(i)->misc; 626 new_aper_base = read_aperture(dev, &new_aper_size); 627 if (!new_aper_base) 628 goto nommu; 629 630 if (!aper_base) { 631 aper_size = new_aper_size; 632 aper_base = new_aper_base; 633 } 634 if (aper_size != new_aper_size || aper_base != new_aper_base) 635 goto nommu; 636 } 637 if (!aper_base) 638 goto nommu; 639 640 info->aper_base = aper_base; 641 info->aper_size = aper_size >> 20; 642 643 gatt_size = (aper_size >> PAGE_SHIFT) * sizeof(u32); 644 gatt = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 645 get_order(gatt_size)); 646 if (!gatt) 647 panic("Cannot allocate GATT table"); 648 if (set_memory_uc((unsigned long)gatt, gatt_size >> PAGE_SHIFT)) 649 panic("Could not set GART PTEs to uncacheable pages"); 650 651 agp_gatt_table = gatt; 652 653 register_syscore_ops(&gart_syscore_ops); 654 655 flush_gart(); 656 657 pr_info("PCI-DMA: aperture base @ %x size %u KB\n", 658 aper_base, aper_size>>10); 659 660 return 0; 661 662 nommu: 663 /* Should not happen anymore */ 664 pr_warn("PCI-DMA: More than 4GB of RAM and no IOMMU - falling back to iommu=soft.\n"); 665 return -1; 666} 667 668static const struct dma_map_ops gart_dma_ops = { 669 .map_sg = gart_map_sg, 670 .unmap_sg = gart_unmap_sg, 671 .map_page = gart_map_page, 672 .unmap_page = gart_unmap_page, 673 .alloc = gart_alloc_coherent, 674 .free = gart_free_coherent, 675 .mmap = dma_common_mmap, 676 .get_sgtable = dma_common_get_sgtable, 677 .dma_supported = dma_direct_supported, 678 .get_required_mask = dma_direct_get_required_mask, 679 .alloc_pages = dma_direct_alloc_pages, 680 .free_pages = dma_direct_free_pages, 681}; 682 683static void gart_iommu_shutdown(void) 684{ 685 struct pci_dev *dev; 686 int i; 687 688 /* don't shutdown it if there is AGP installed */ 689 if (!no_agp) 690 return; 691 692 if (!amd_nb_has_feature(AMD_NB_GART)) 693 return; 694 695 for (i = 0; i < amd_nb_num(); i++) { 696 u32 ctl; 697 698 dev = node_to_amd_nb(i)->misc; 699 pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &ctl); 700 701 ctl &= ~GARTEN; 702 703 pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, ctl); 704 } 705} 706 707int __init gart_iommu_init(void) 708{ 709 struct agp_kern_info info; 710 unsigned long iommu_start; 711 unsigned long aper_base, aper_size; 712 unsigned long start_pfn, end_pfn; 713 unsigned long scratch; 714 715 if (!amd_nb_has_feature(AMD_NB_GART)) 716 return 0; 717 718#ifndef CONFIG_AGP_AMD64 719 no_agp = 1; 720#else 721 /* Makefile puts PCI initialization via subsys_initcall first. */ 722 /* Add other AMD AGP bridge drivers here */ 723 no_agp = no_agp || 724 (agp_amd64_init() < 0) || 725 (agp_copy_info(agp_bridge, &info) < 0); 726#endif 727 728 if (no_iommu || 729 (!force_iommu && max_pfn <= MAX_DMA32_PFN) || 730 !gart_iommu_aperture || 731 (no_agp && init_amd_gatt(&info) < 0)) { 732 if (max_pfn > MAX_DMA32_PFN) { 733 pr_warn("More than 4GB of memory but GART IOMMU not available.\n"); 734 pr_warn("falling back to iommu=soft.\n"); 735 } 736 return 0; 737 } 738 739 /* need to map that range */ 740 aper_size = info.aper_size << 20; 741 aper_base = info.aper_base; 742 end_pfn = (aper_base>>PAGE_SHIFT) + (aper_size>>PAGE_SHIFT); 743 744 start_pfn = PFN_DOWN(aper_base); 745 if (!pfn_range_is_mapped(start_pfn, end_pfn)) 746 init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT, 747 PAGE_KERNEL); 748 749 pr_info("PCI-DMA: using GART IOMMU.\n"); 750 iommu_size = check_iommu_size(info.aper_base, aper_size); 751 iommu_pages = iommu_size >> PAGE_SHIFT; 752 753 iommu_gart_bitmap = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, 754 get_order(iommu_pages/8)); 755 if (!iommu_gart_bitmap) 756 panic("Cannot allocate iommu bitmap\n"); 757 758 pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n", 759 iommu_size >> 20); 760 761 agp_memory_reserved = iommu_size; 762 iommu_start = aper_size - iommu_size; 763 iommu_bus_base = info.aper_base + iommu_start; 764 iommu_gatt_base = agp_gatt_table + (iommu_start>>PAGE_SHIFT); 765 766 /* 767 * Unmap the IOMMU part of the GART. The alias of the page is 768 * always mapped with cache enabled and there is no full cache 769 * coherency across the GART remapping. The unmapping avoids 770 * automatic prefetches from the CPU allocating cache lines in 771 * there. All CPU accesses are done via the direct mapping to 772 * the backing memory. The GART address is only used by PCI 773 * devices. 774 */ 775 set_memory_np((unsigned long)__va(iommu_bus_base), 776 iommu_size >> PAGE_SHIFT); 777 /* 778 * Tricky. The GART table remaps the physical memory range, 779 * so the CPU won't notice potential aliases and if the memory 780 * is remapped to UC later on, we might surprise the PCI devices 781 * with a stray writeout of a cacheline. So play it sure and 782 * do an explicit, full-scale wbinvd() _after_ having marked all 783 * the pages as Not-Present: 784 */ 785 wbinvd(); 786 787 /* 788 * Now all caches are flushed and we can safely enable 789 * GART hardware. Doing it early leaves the possibility 790 * of stale cache entries that can lead to GART PTE 791 * errors. 792 */ 793 enable_gart_translations(); 794 795 /* 796 * Try to workaround a bug (thanks to BenH): 797 * Set unmapped entries to a scratch page instead of 0. 798 * Any prefetches that hit unmapped entries won't get an bus abort 799 * then. (P2P bridge may be prefetching on DMA reads). 800 */ 801 scratch = get_zeroed_page(GFP_KERNEL); 802 if (!scratch) 803 panic("Cannot allocate iommu scratch page"); 804 gart_unmapped_entry = GPTE_ENCODE(__pa(scratch)); 805 806 flush_gart(); 807 dma_ops = &gart_dma_ops; 808 x86_platform.iommu_shutdown = gart_iommu_shutdown; 809 x86_swiotlb_enable = false; 810 811 return 0; 812} 813 814void __init gart_parse_options(char *p) 815{ 816 int arg; 817 818 if (isdigit(*p) && get_option(&p, &arg)) 819 iommu_size = arg; 820 if (!strncmp(p, "fullflush", 9)) 821 iommu_fullflush = 1; 822 if (!strncmp(p, "nofullflush", 11)) 823 iommu_fullflush = 0; 824 if (!strncmp(p, "noagp", 5)) 825 no_agp = 1; 826 if (!strncmp(p, "noaperture", 10)) 827 fix_aperture = 0; 828 /* duplicated from pci-dma.c */ 829 if (!strncmp(p, "force", 5)) 830 gart_iommu_aperture_allowed = 1; 831 if (!strncmp(p, "allowed", 7)) 832 gart_iommu_aperture_allowed = 1; 833 if (!strncmp(p, "memaper", 7)) { 834 fallback_aper_force = 1; 835 p += 7; 836 if (*p == '=') { 837 ++p; 838 if (get_option(&p, &arg)) 839 fallback_aper_order = arg; 840 } 841 } 842} 843