1 2#include <linux/kernel.h> 3#include <linux/types.h> 4#include <linux/init.h> 5#include <linux/bootmem.h> 6#include <linux/mmzone.h> 7#include <linux/pci_ids.h> 8#include <linux/pci.h> 9#include <linux/bitops.h> 10#include <linux/ioport.h> 11#include <linux/suspend.h> 12#include <linux/kmemleak.h> 13#include <asm/e820.h> 14#include <asm/io.h> 15#include <asm/iommu.h> 16#include <asm/gart.h> 17#include <asm/pci-direct.h> 18#include <asm/dma.h> 19#include <asm/k8.h> 20#include <asm/x86_init.h> 21 22int gart_iommu_aperture; 23int gart_iommu_aperture_disabled __initdata; 24int gart_iommu_aperture_allowed __initdata; 25 26int fallback_aper_order __initdata = 1; /* 64MB */ 27int fallback_aper_force __initdata; 28 29int fix_aperture __initdata = 1; 30 31struct bus_dev_range { 32 int bus; 33 int dev_base; 34 int dev_limit; 35}; 36 37static struct bus_dev_range bus_dev_ranges[] __initdata = { 38 { 0x00, 0x18, 0x20}, 39 { 0xff, 0x00, 0x20}, 40 { 0xfe, 0x00, 0x20} 41}; 42 43static struct resource gart_resource = { 44 .name = "GART", 45 .flags = IORESOURCE_MEM, 46}; 47 48static void __init insert_aperture_resource(u32 aper_base, u32 aper_size) 49{ 50 gart_resource.start = aper_base; 51 gart_resource.end = aper_base + aper_size - 1; 52 insert_resource(&iomem_resource, &gart_resource); 53} 54 55/* This code runs before the PCI subsystem is initialized, so just 56 access the northbridge directly. */ 57 58static u32 __init allocate_aperture(void) 59{ 60 u32 aper_size; 61 void *p; 62 63 /* aper_size should <= 1G */ 64 if (fallback_aper_order > 5) 65 fallback_aper_order = 5; 66 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 67 68 /* 69 * Aperture has to be naturally aligned. This means a 2GB aperture 70 * won't have much chance of finding a place in the lower 4GB of 71 * memory. Unfortunately we cannot move it up because that would 72 * make the IOMMU useless. 73 */ 74 /* 75 * using 512M as goal, in case kexec will load kernel_big 76 * that will do the on position decompress, and could overlap with 77 * that positon with gart that is used. 78 * sequende: 79 * kernel_small 80 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 81 * ==> kernel_small(gart area become e820_reserved) 82 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart) 83 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 84 * so don't use 512M below as gart iommu, leave the space for kernel 85 * code for safe 86 */ 87 p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20); 88 /* 89 * Kmemleak should not scan this block as it may not be mapped via the 90 * kernel direct mapping. 91 */ 92 kmemleak_ignore(p); 93 if (!p || __pa(p)+aper_size > 0xffffffff) { 94 printk(KERN_ERR 95 "Cannot allocate aperture memory hole (%p,%uK)\n", 96 p, aper_size>>10); 97 if (p) 98 free_bootmem(__pa(p), aper_size); 99 return 0; 100 } 101 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", 102 aper_size >> 10, __pa(p)); 103 insert_aperture_resource((u32)__pa(p), aper_size); 104 register_nosave_region((u32)__pa(p) >> PAGE_SHIFT, 105 (u32)__pa(p+aper_size) >> PAGE_SHIFT); 106 107 return (u32)__pa(p); 108} 109 110 111/* Find a PCI capability */ 112static u32 __init find_cap(int bus, int slot, int func, int cap) 113{ 114 int bytes; 115 u8 pos; 116 117 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 118 PCI_STATUS_CAP_LIST)) 119 return 0; 120 121 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 122 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 123 u8 id; 124 125 pos &= ~3; 126 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 127 if (id == 0xff) 128 break; 129 if (id == cap) 130 return pos; 131 pos = read_pci_config_byte(bus, slot, func, 132 pos+PCI_CAP_LIST_NEXT); 133 } 134 return 0; 135} 136 137/* Read a standard AGPv3 bridge header */ 138static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 139{ 140 u32 apsize; 141 u32 apsizereg; 142 int nbits; 143 u32 aper_low, aper_hi; 144 u64 aper; 145 u32 old_order; 146 147 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func); 148 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 149 if (apsizereg == 0xffffffff) { 150 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); 151 return 0; 152 } 153 154 /* old_order could be the value from NB gart setting */ 155 old_order = *order; 156 157 apsize = apsizereg & 0xfff; 158 /* Some BIOS use weird encodings not in the AGPv3 table. */ 159 if (apsize & 0xff) 160 apsize |= 0xf00; 161 nbits = hweight16(apsize); 162 *order = 7 - nbits; 163 if ((int)*order < 0) /* < 32MB */ 164 *order = 0; 165 166 aper_low = read_pci_config(bus, slot, func, 0x10); 167 aper_hi = read_pci_config(bus, slot, func, 0x14); 168 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 169 170 /* 171 * On some sick chips, APSIZE is 0. It means it wants 4G 172 * so let double check that order, and lets trust AMD NB settings: 173 */ 174 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n", 175 aper, 32 << old_order); 176 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 177 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n", 178 32 << *order, apsizereg); 179 *order = old_order; 180 } 181 182 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", 183 aper, 32 << *order, apsizereg); 184 185 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 186 return 0; 187 return (u32)aper; 188} 189 190static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 191{ 192 int bus, slot, func; 193 194 /* Poor man's PCI discovery */ 195 for (bus = 0; bus < 256; bus++) { 196 for (slot = 0; slot < 32; slot++) { 197 for (func = 0; func < 8; func++) { 198 u32 class, cap; 199 u8 type; 200 class = read_pci_config(bus, slot, func, 201 PCI_CLASS_REVISION); 202 if (class == 0xffffffff) 203 break; 204 205 switch (class >> 16) { 206 case PCI_CLASS_BRIDGE_HOST: 207 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 208 /* AGP bridge? */ 209 cap = find_cap(bus, slot, func, 210 PCI_CAP_ID_AGP); 211 if (!cap) 212 break; 213 *valid_agp = 1; 214 return read_agp(bus, slot, func, cap, 215 order); 216 } 217 218 /* No multi-function device? */ 219 type = read_pci_config_byte(bus, slot, func, 220 PCI_HEADER_TYPE); 221 if (!(type & 0x80)) 222 break; 223 } 224 } 225 } 226 printk(KERN_INFO "No AGP bridge found\n"); 227 228 return 0; 229} 230 231static int gart_fix_e820 __initdata = 1; 232 233static int __init parse_gart_mem(char *p) 234{ 235 if (!p) 236 return -EINVAL; 237 238 if (!strncmp(p, "off", 3)) 239 gart_fix_e820 = 0; 240 else if (!strncmp(p, "on", 2)) 241 gart_fix_e820 = 1; 242 243 return 0; 244} 245early_param("gart_fix_e820", parse_gart_mem); 246 247void __init early_gart_iommu_check(void) 248{ 249 /* 250 * in case it is enabled before, esp for kexec/kdump, 251 * previous kernel already enable that. memset called 252 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 253 * or second kernel have different position for GART hole. and new 254 * kernel could use hole as RAM that is still used by GART set by 255 * first kernel 256 * or BIOS forget to put that in reserved. 257 * try to update e820 to make that region as reserved. 258 */ 259 u32 agp_aper_order = 0; 260 int i, fix, slot, valid_agp = 0; 261 u32 ctl; 262 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 263 u64 aper_base = 0, last_aper_base = 0; 264 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 265 266 if (!early_pci_allowed()) 267 return; 268 269 /* This is mostly duplicate of iommu_hole_init */ 270 search_agp_bridge(&agp_aper_order, &valid_agp); 271 272 fix = 0; 273 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 274 int bus; 275 int dev_base, dev_limit; 276 277 bus = bus_dev_ranges[i].bus; 278 dev_base = bus_dev_ranges[i].dev_base; 279 dev_limit = bus_dev_ranges[i].dev_limit; 280 281 for (slot = dev_base; slot < dev_limit; slot++) { 282 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 283 continue; 284 285 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 286 aper_enabled = ctl & AMD64_GARTEN; 287 aper_order = (ctl >> 1) & 7; 288 aper_size = (32 * 1024 * 1024) << aper_order; 289 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 290 aper_base <<= 25; 291 292 if (last_valid) { 293 if ((aper_order != last_aper_order) || 294 (aper_base != last_aper_base) || 295 (aper_enabled != last_aper_enabled)) { 296 fix = 1; 297 break; 298 } 299 } 300 301 last_aper_order = aper_order; 302 last_aper_base = aper_base; 303 last_aper_enabled = aper_enabled; 304 last_valid = 1; 305 } 306 } 307 308 if (!fix && !aper_enabled) 309 return; 310 311 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 312 fix = 1; 313 314 if (gart_fix_e820 && !fix && aper_enabled) { 315 if (e820_any_mapped(aper_base, aper_base + aper_size, 316 E820_RAM)) { 317 /* reserve it, so we can reuse it in second kernel */ 318 printk(KERN_INFO "update e820 for GART\n"); 319 e820_add_region(aper_base, aper_size, E820_RESERVED); 320 update_e820(); 321 } 322 } 323 324 if (valid_agp) 325 return; 326 327 /* disable them all at first */ 328 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 329 int bus; 330 int dev_base, dev_limit; 331 332 bus = bus_dev_ranges[i].bus; 333 dev_base = bus_dev_ranges[i].dev_base; 334 dev_limit = bus_dev_ranges[i].dev_limit; 335 336 for (slot = dev_base; slot < dev_limit; slot++) { 337 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 338 continue; 339 340 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 341 ctl &= ~AMD64_GARTEN; 342 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 343 } 344 } 345 346} 347 348static int __initdata printed_gart_size_msg; 349 350void __init gart_iommu_hole_init(void) 351{ 352 u32 agp_aper_base = 0, agp_aper_order = 0; 353 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 354 u64 aper_base, last_aper_base = 0; 355 int fix, slot, valid_agp = 0; 356 int i, node; 357 358 if (gart_iommu_aperture_disabled || !fix_aperture || 359 !early_pci_allowed()) 360 return; 361 362 printk(KERN_INFO "Checking aperture...\n"); 363 364 if (!fallback_aper_force) 365 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 366 367 fix = 0; 368 node = 0; 369 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 370 int bus; 371 int dev_base, dev_limit; 372 u32 ctl; 373 374 bus = bus_dev_ranges[i].bus; 375 dev_base = bus_dev_ranges[i].dev_base; 376 dev_limit = bus_dev_ranges[i].dev_limit; 377 378 for (slot = dev_base; slot < dev_limit; slot++) { 379 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 380 continue; 381 382 iommu_detected = 1; 383 gart_iommu_aperture = 1; 384 x86_init.iommu.iommu_init = gart_iommu_init; 385 386 ctl = read_pci_config(bus, slot, 3, 387 AMD64_GARTAPERTURECTL); 388 389 /* 390 * Before we do anything else disable the GART. It may 391 * still be enabled if we boot into a crash-kernel here. 392 * Reconfiguring the GART while it is enabled could have 393 * unknown side-effects. 394 */ 395 ctl &= ~GARTEN; 396 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 397 398 aper_order = (ctl >> 1) & 7; 399 aper_size = (32 * 1024 * 1024) << aper_order; 400 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 401 aper_base <<= 25; 402 403 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", 404 node, aper_base, aper_size >> 20); 405 node++; 406 407 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 408 if (valid_agp && agp_aper_base && 409 agp_aper_base == aper_base && 410 agp_aper_order == aper_order) { 411 /* the same between two setting from NB and agp */ 412 if (!no_iommu && 413 max_pfn > MAX_DMA32_PFN && 414 !printed_gart_size_msg) { 415 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n"); 416 printk(KERN_ERR "please increase GART size in your BIOS setup\n"); 417 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n"); 418 printed_gart_size_msg = 1; 419 } 420 } else { 421 fix = 1; 422 goto out; 423 } 424 } 425 426 if ((last_aper_order && aper_order != last_aper_order) || 427 (last_aper_base && aper_base != last_aper_base)) { 428 fix = 1; 429 goto out; 430 } 431 last_aper_order = aper_order; 432 last_aper_base = aper_base; 433 } 434 } 435 436out: 437 if (!fix && !fallback_aper_force) { 438 if (last_aper_base) { 439 unsigned long n = (32 * 1024 * 1024) << last_aper_order; 440 441 insert_aperture_resource((u32)last_aper_base, n); 442 } 443 return; 444 } 445 446 if (!fallback_aper_force) { 447 aper_alloc = agp_aper_base; 448 aper_order = agp_aper_order; 449 } 450 451 if (aper_alloc) { 452 /* Got the aperture from the AGP bridge */ 453 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 454 force_iommu || 455 valid_agp || 456 fallback_aper_force) { 457 printk(KERN_INFO 458 "Your BIOS doesn't leave a aperture memory hole\n"); 459 printk(KERN_INFO 460 "Please enable the IOMMU option in the BIOS setup\n"); 461 printk(KERN_INFO 462 "This costs you %d MB of RAM\n", 463 32 << fallback_aper_order); 464 465 aper_order = fallback_aper_order; 466 aper_alloc = allocate_aperture(); 467 if (!aper_alloc) { 468 /* 469 * Could disable AGP and IOMMU here, but it's 470 * probably not worth it. But the later users 471 * cannot deal with bad apertures and turning 472 * on the aperture over memory causes very 473 * strange problems, so it's better to panic 474 * early. 475 */ 476 panic("Not enough memory for aperture"); 477 } 478 } else { 479 return; 480 } 481 482 /* Fix up the north bridges */ 483 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { 484 int bus; 485 int dev_base, dev_limit; 486 487 bus = bus_dev_ranges[i].bus; 488 dev_base = bus_dev_ranges[i].dev_base; 489 dev_limit = bus_dev_ranges[i].dev_limit; 490 for (slot = dev_base; slot < dev_limit; slot++) { 491 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00))) 492 continue; 493 494 /* Don't enable translation yet. That is done later. 495 Assume this BIOS didn't initialise the GART so 496 just overwrite all previous bits */ 497 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1); 498 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 499 } 500 } 501 502 set_up_gart_resume(aper_order, aper_alloc); 503} 504