1#include "amd64_edac.h" 2#include <asm/k8.h> 3 4static struct edac_pci_ctl_info *amd64_ctl_pci; 5 6static int report_gart_errors; 7module_param(report_gart_errors, int, 0644); 8 9/* 10 * Set by command line parameter. If BIOS has enabled the ECC, this override is 11 * cleared to prevent re-enabling the hardware by this driver. 12 */ 13static int ecc_enable_override; 14module_param(ecc_enable_override, int, 0644); 15 16static struct msr __percpu *msrs; 17 18/* Lookup table for all possible MC control instances */ 19struct amd64_pvt; 20static struct mem_ctl_info *mci_lookup[EDAC_MAX_NUMNODES]; 21static struct amd64_pvt *pvt_lookup[EDAC_MAX_NUMNODES]; 22 23/* 24 * Address to DRAM bank mapping: see F2x80 for K8 and F2x[1,0]80 for Fam10 and 25 * later. 26 */ 27static int ddr2_dbam_revCG[] = { 28 [0] = 32, 29 [1] = 64, 30 [2] = 128, 31 [3] = 256, 32 [4] = 512, 33 [5] = 1024, 34 [6] = 2048, 35}; 36 37static int ddr2_dbam_revD[] = { 38 [0] = 32, 39 [1] = 64, 40 [2 ... 3] = 128, 41 [4] = 256, 42 [5] = 512, 43 [6] = 256, 44 [7] = 512, 45 [8 ... 9] = 1024, 46 [10] = 2048, 47}; 48 49static int ddr2_dbam[] = { [0] = 128, 50 [1] = 256, 51 [2 ... 4] = 512, 52 [5 ... 6] = 1024, 53 [7 ... 8] = 2048, 54 [9 ... 10] = 4096, 55 [11] = 8192, 56}; 57 58static int ddr3_dbam[] = { [0] = -1, 59 [1] = 256, 60 [2] = 512, 61 [3 ... 4] = -1, 62 [5 ... 6] = 1024, 63 [7 ... 8] = 2048, 64 [9 ... 10] = 4096, 65 [11] = 8192, 66}; 67 68 69struct scrubrate scrubrates[] = { 70 { 0x01, 1600000000UL}, 71 { 0x02, 800000000UL}, 72 { 0x03, 400000000UL}, 73 { 0x04, 200000000UL}, 74 { 0x05, 100000000UL}, 75 { 0x06, 50000000UL}, 76 { 0x07, 25000000UL}, 77 { 0x08, 12284069UL}, 78 { 0x09, 6274509UL}, 79 { 0x0A, 3121951UL}, 80 { 0x0B, 1560975UL}, 81 { 0x0C, 781440UL}, 82 { 0x0D, 390720UL}, 83 { 0x0E, 195300UL}, 84 { 0x0F, 97650UL}, 85 { 0x10, 48854UL}, 86 { 0x11, 24427UL}, 87 { 0x12, 12213UL}, 88 { 0x13, 6101UL}, 89 { 0x14, 3051UL}, 90 { 0x15, 1523UL}, 91 { 0x16, 761UL}, 92 { 0x00, 0UL}, /* scrubbing off */ 93}; 94 95/* 96 * Memory scrubber control interface. For K8, memory scrubbing is handled by 97 * hardware and can involve L2 cache, dcache as well as the main memory. With 98 * F10, this is extended to L3 cache scrubbing on CPU models sporting that 99 * functionality. 100 * 101 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 102 * (dram) over to cache lines. This is nasty, so we will use bandwidth in 103 * bytes/sec for the setting. 104 * 105 * Currently, we only do dram scrubbing. If the scrubbing is done in software on 106 * other archs, we might not have access to the caches directly. 107 */ 108 109/* 110 * scan the scrub rate mapping table for a close or matching bandwidth value to 111 * issue. If requested is too big, then use last maximum value found. 112 */ 113static int amd64_search_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, 114 u32 min_scrubrate) 115{ 116 u32 scrubval; 117 int i; 118 119 /* 120 * map the configured rate (new_bw) to a value specific to the AMD64 121 * memory controller and apply to register. Search for the first 122 * bandwidth entry that is greater or equal than the setting requested 123 * and program that. If at last entry, turn off DRAM scrubbing. 124 */ 125 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 126 /* 127 * skip scrub rates which aren't recommended 128 * (see F10 BKDG, F3x58) 129 */ 130 if (scrubrates[i].scrubval < min_scrubrate) 131 continue; 132 133 if (scrubrates[i].bandwidth <= new_bw) 134 break; 135 136 /* 137 * if no suitable bandwidth found, turn off DRAM scrubbing 138 * entirely by falling back to the last element in the 139 * scrubrates array. 140 */ 141 } 142 143 scrubval = scrubrates[i].scrubval; 144 if (scrubval) 145 edac_printk(KERN_DEBUG, EDAC_MC, 146 "Setting scrub rate bandwidth: %u\n", 147 scrubrates[i].bandwidth); 148 else 149 edac_printk(KERN_DEBUG, EDAC_MC, "Turning scrubbing off.\n"); 150 151 pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F); 152 153 return 0; 154} 155 156static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bandwidth) 157{ 158 struct amd64_pvt *pvt = mci->pvt_info; 159 u32 min_scrubrate = 0x0; 160 161 switch (boot_cpu_data.x86) { 162 case 0xf: 163 min_scrubrate = K8_MIN_SCRUB_RATE_BITS; 164 break; 165 case 0x10: 166 min_scrubrate = F10_MIN_SCRUB_RATE_BITS; 167 break; 168 case 0x11: 169 min_scrubrate = F11_MIN_SCRUB_RATE_BITS; 170 break; 171 172 default: 173 amd64_printk(KERN_ERR, "Unsupported family!\n"); 174 return -EINVAL; 175 } 176 return amd64_search_set_scrub_rate(pvt->misc_f3_ctl, bandwidth, 177 min_scrubrate); 178} 179 180static int amd64_get_scrub_rate(struct mem_ctl_info *mci, u32 *bw) 181{ 182 struct amd64_pvt *pvt = mci->pvt_info; 183 u32 scrubval = 0; 184 int status = -1, i; 185 186 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_SCRCTRL, &scrubval); 187 188 scrubval = scrubval & 0x001F; 189 190 edac_printk(KERN_DEBUG, EDAC_MC, 191 "pci-read, sdram scrub control value: %d \n", scrubval); 192 193 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 194 if (scrubrates[i].scrubval == scrubval) { 195 *bw = scrubrates[i].bandwidth; 196 status = 0; 197 break; 198 } 199 } 200 201 return status; 202} 203 204/* Map from a CSROW entry to the mask entry that operates on it */ 205static inline u32 amd64_map_to_dcs_mask(struct amd64_pvt *pvt, int csrow) 206{ 207 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) 208 return csrow; 209 else 210 return csrow >> 1; 211} 212 213/* return the 'base' address the i'th CS entry of the 'dct' DRAM controller */ 214static u32 amd64_get_dct_base(struct amd64_pvt *pvt, int dct, int csrow) 215{ 216 if (dct == 0) 217 return pvt->dcsb0[csrow]; 218 else 219 return pvt->dcsb1[csrow]; 220} 221 222/* 223 * Return the 'mask' address the i'th CS entry. This function is needed because 224 * there number of DCSM registers on Rev E and prior vs Rev F and later is 225 * different. 226 */ 227static u32 amd64_get_dct_mask(struct amd64_pvt *pvt, int dct, int csrow) 228{ 229 if (dct == 0) 230 return pvt->dcsm0[amd64_map_to_dcs_mask(pvt, csrow)]; 231 else 232 return pvt->dcsm1[amd64_map_to_dcs_mask(pvt, csrow)]; 233} 234 235 236/* 237 * In *base and *limit, pass back the full 40-bit base and limit physical 238 * addresses for the node given by node_id. This information is obtained from 239 * DRAM Base (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers. The 240 * base and limit addresses are of type SysAddr, as defined at the start of 241 * section 3.4.4 (p. 70). They are the lowest and highest physical addresses 242 * in the address range they represent. 243 */ 244static void amd64_get_base_and_limit(struct amd64_pvt *pvt, int node_id, 245 u64 *base, u64 *limit) 246{ 247 *base = pvt->dram_base[node_id]; 248 *limit = pvt->dram_limit[node_id]; 249} 250 251/* 252 * Return 1 if the SysAddr given by sys_addr matches the base/limit associated 253 * with node_id 254 */ 255static int amd64_base_limit_match(struct amd64_pvt *pvt, 256 u64 sys_addr, int node_id) 257{ 258 u64 base, limit, addr; 259 260 amd64_get_base_and_limit(pvt, node_id, &base, &limit); 261 262 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 263 * all ones if the most significant implemented address bit is 1. 264 * Here we discard bits 63-40. See section 3.4.2 of AMD publication 265 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 266 * Application Programming. 267 */ 268 addr = sys_addr & 0x000000ffffffffffull; 269 270 return (addr >= base) && (addr <= limit); 271} 272 273/* 274 * Attempt to map a SysAddr to a node. On success, return a pointer to the 275 * mem_ctl_info structure for the node that the SysAddr maps to. 276 * 277 * On failure, return NULL. 278 */ 279static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 280 u64 sys_addr) 281{ 282 struct amd64_pvt *pvt; 283 int node_id; 284 u32 intlv_en, bits; 285 286 /* 287 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 288 * 3.4.4.2) registers to map the SysAddr to a node ID. 289 */ 290 pvt = mci->pvt_info; 291 292 /* 293 * The value of this field should be the same for all DRAM Base 294 * registers. Therefore we arbitrarily choose to read it from the 295 * register for node 0. 296 */ 297 intlv_en = pvt->dram_IntlvEn[0]; 298 299 if (intlv_en == 0) { 300 for (node_id = 0; node_id < DRAM_REG_COUNT; node_id++) { 301 if (amd64_base_limit_match(pvt, sys_addr, node_id)) 302 goto found; 303 } 304 goto err_no_match; 305 } 306 307 if (unlikely((intlv_en != 0x01) && 308 (intlv_en != 0x03) && 309 (intlv_en != 0x07))) { 310 amd64_printk(KERN_WARNING, "junk value of 0x%x extracted from " 311 "IntlvEn field of DRAM Base Register for node 0: " 312 "this probably indicates a BIOS bug.\n", intlv_en); 313 return NULL; 314 } 315 316 bits = (((u32) sys_addr) >> 12) & intlv_en; 317 318 for (node_id = 0; ; ) { 319 if ((pvt->dram_IntlvSel[node_id] & intlv_en) == bits) 320 break; /* intlv_sel field matches */ 321 322 if (++node_id >= DRAM_REG_COUNT) 323 goto err_no_match; 324 } 325 326 /* sanity test for sys_addr */ 327 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) { 328 amd64_printk(KERN_WARNING, 329 "%s(): sys_addr 0x%llx falls outside base/limit " 330 "address range for node %d with node interleaving " 331 "enabled.\n", 332 __func__, sys_addr, node_id); 333 return NULL; 334 } 335 336found: 337 return edac_mc_find(node_id); 338 339err_no_match: 340 debugf2("sys_addr 0x%lx doesn't match any node\n", 341 (unsigned long)sys_addr); 342 343 return NULL; 344} 345 346/* 347 * Extract the DRAM CS base address from selected csrow register. 348 */ 349static u64 base_from_dct_base(struct amd64_pvt *pvt, int csrow) 350{ 351 return ((u64) (amd64_get_dct_base(pvt, 0, csrow) & pvt->dcsb_base)) << 352 pvt->dcs_shift; 353} 354 355/* 356 * Extract the mask from the dcsb0[csrow] entry in a CPU revision-specific way. 357 */ 358static u64 mask_from_dct_mask(struct amd64_pvt *pvt, int csrow) 359{ 360 u64 dcsm_bits, other_bits; 361 u64 mask; 362 363 /* Extract bits from DRAM CS Mask. */ 364 dcsm_bits = amd64_get_dct_mask(pvt, 0, csrow) & pvt->dcsm_mask; 365 366 other_bits = pvt->dcsm_mask; 367 other_bits = ~(other_bits << pvt->dcs_shift); 368 369 /* 370 * The extracted bits from DCSM belong in the spaces represented by 371 * the cleared bits in other_bits. 372 */ 373 mask = (dcsm_bits << pvt->dcs_shift) | other_bits; 374 375 return mask; 376} 377 378/* 379 * @input_addr is an InputAddr associated with the node given by mci. Return the 380 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 381 */ 382static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 383{ 384 struct amd64_pvt *pvt; 385 int csrow; 386 u64 base, mask; 387 388 pvt = mci->pvt_info; 389 390 /* 391 * Here we use the DRAM CS Base and DRAM CS Mask registers. For each CS 392 * base/mask register pair, test the condition shown near the start of 393 * section 3.5.4 (p. 84, BKDG #26094, K8, revA-E). 394 */ 395 for (csrow = 0; csrow < pvt->cs_count; csrow++) { 396 397 /* This DRAM chip select is disabled on this node */ 398 if ((pvt->dcsb0[csrow] & K8_DCSB_CS_ENABLE) == 0) 399 continue; 400 401 base = base_from_dct_base(pvt, csrow); 402 mask = ~mask_from_dct_mask(pvt, csrow); 403 404 if ((input_addr & mask) == (base & mask)) { 405 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n", 406 (unsigned long)input_addr, csrow, 407 pvt->mc_node_id); 408 409 return csrow; 410 } 411 } 412 413 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n", 414 (unsigned long)input_addr, pvt->mc_node_id); 415 416 return -1; 417} 418 419/* 420 * Return the base value defined by the DRAM Base register for the node 421 * represented by mci. This function returns the full 40-bit value despite the 422 * fact that the register only stores bits 39-24 of the value. See section 423 * 3.4.4.1 (BKDG #26094, K8, revA-E) 424 */ 425static inline u64 get_dram_base(struct mem_ctl_info *mci) 426{ 427 struct amd64_pvt *pvt = mci->pvt_info; 428 429 return pvt->dram_base[pvt->mc_node_id]; 430} 431 432/* 433 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 434 * for the node represented by mci. Info is passed back in *hole_base, 435 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 436 * info is invalid. Info may be invalid for either of the following reasons: 437 * 438 * - The revision of the node is not E or greater. In this case, the DRAM Hole 439 * Address Register does not exist. 440 * 441 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 442 * indicating that its contents are not valid. 443 * 444 * The values passed back in *hole_base, *hole_offset, and *hole_size are 445 * complete 32-bit values despite the fact that the bitfields in the DHAR 446 * only represent bits 31-24 of the base and offset values. 447 */ 448int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 449 u64 *hole_offset, u64 *hole_size) 450{ 451 struct amd64_pvt *pvt = mci->pvt_info; 452 u64 base; 453 454 /* only revE and later have the DRAM Hole Address Register */ 455 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) { 456 debugf1(" revision %d for node %d does not support DHAR\n", 457 pvt->ext_model, pvt->mc_node_id); 458 return 1; 459 } 460 461 /* only valid for Fam10h */ 462 if (boot_cpu_data.x86 == 0x10 && 463 (pvt->dhar & F10_DRAM_MEM_HOIST_VALID) == 0) { 464 debugf1(" Dram Memory Hoisting is DISABLED on this system\n"); 465 return 1; 466 } 467 468 if ((pvt->dhar & DHAR_VALID) == 0) { 469 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n", 470 pvt->mc_node_id); 471 return 1; 472 } 473 474 /* This node has Memory Hoisting */ 475 476 /* +------------------+--------------------+--------------------+----- 477 * | memory | DRAM hole | relocated | 478 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 479 * | | | DRAM hole | 480 * | | | [0x100000000, | 481 * | | | (0x100000000+ | 482 * | | | (0xffffffff-x))] | 483 * +------------------+--------------------+--------------------+----- 484 * 485 * Above is a diagram of physical memory showing the DRAM hole and the 486 * relocated addresses from the DRAM hole. As shown, the DRAM hole 487 * starts at address x (the base address) and extends through address 488 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 489 * addresses in the hole so that they start at 0x100000000. 490 */ 491 492 base = dhar_base(pvt->dhar); 493 494 *hole_base = base; 495 *hole_size = (0x1ull << 32) - base; 496 497 if (boot_cpu_data.x86 > 0xf) 498 *hole_offset = f10_dhar_offset(pvt->dhar); 499 else 500 *hole_offset = k8_dhar_offset(pvt->dhar); 501 502 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 503 pvt->mc_node_id, (unsigned long)*hole_base, 504 (unsigned long)*hole_offset, (unsigned long)*hole_size); 505 506 return 0; 507} 508EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); 509 510/* 511 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 512 * assumed that sys_addr maps to the node given by mci. 513 * 514 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 515 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 516 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 517 * then it is also involved in translating a SysAddr to a DramAddr. Sections 518 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 519 * These parts of the documentation are unclear. I interpret them as follows: 520 * 521 * When node n receives a SysAddr, it processes the SysAddr as follows: 522 * 523 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 524 * Limit registers for node n. If the SysAddr is not within the range 525 * specified by the base and limit values, then node n ignores the Sysaddr 526 * (since it does not map to node n). Otherwise continue to step 2 below. 527 * 528 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 529 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 530 * the range of relocated addresses (starting at 0x100000000) from the DRAM 531 * hole. If not, skip to step 3 below. Else get the value of the 532 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 533 * offset defined by this value from the SysAddr. 534 * 535 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 536 * Base register for node n. To obtain the DramAddr, subtract the base 537 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 538 */ 539static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 540{ 541 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 542 int ret = 0; 543 544 dram_base = get_dram_base(mci); 545 546 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 547 &hole_size); 548 if (!ret) { 549 if ((sys_addr >= (1ull << 32)) && 550 (sys_addr < ((1ull << 32) + hole_size))) { 551 /* use DHAR to translate SysAddr to DramAddr */ 552 dram_addr = sys_addr - hole_offset; 553 554 debugf2("using DHAR to translate SysAddr 0x%lx to " 555 "DramAddr 0x%lx\n", 556 (unsigned long)sys_addr, 557 (unsigned long)dram_addr); 558 559 return dram_addr; 560 } 561 } 562 563 /* 564 * Translate the SysAddr to a DramAddr as shown near the start of 565 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 566 * only deals with 40-bit values. Therefore we discard bits 63-40 of 567 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 568 * discard are all 1s. Otherwise the bits we discard are all 0s. See 569 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 570 * Programmer's Manual Volume 1 Application Programming. 571 */ 572 dram_addr = (sys_addr & 0xffffffffffull) - dram_base; 573 574 debugf2("using DRAM Base register to translate SysAddr 0x%lx to " 575 "DramAddr 0x%lx\n", (unsigned long)sys_addr, 576 (unsigned long)dram_addr); 577 return dram_addr; 578} 579 580/* 581 * @intlv_en is the value of the IntlvEn field from a DRAM Base register 582 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 583 * for node interleaving. 584 */ 585static int num_node_interleave_bits(unsigned intlv_en) 586{ 587 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 588 int n; 589 590 BUG_ON(intlv_en > 7); 591 n = intlv_shift_table[intlv_en]; 592 return n; 593} 594 595/* Translate the DramAddr given by @dram_addr to an InputAddr. */ 596static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 597{ 598 struct amd64_pvt *pvt; 599 int intlv_shift; 600 u64 input_addr; 601 602 pvt = mci->pvt_info; 603 604 /* 605 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 606 * concerning translating a DramAddr to an InputAddr. 607 */ 608 intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); 609 input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) + 610 (dram_addr & 0xfff); 611 612 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 613 intlv_shift, (unsigned long)dram_addr, 614 (unsigned long)input_addr); 615 616 return input_addr; 617} 618 619/* 620 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 621 * assumed that @sys_addr maps to the node given by mci. 622 */ 623static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 624{ 625 u64 input_addr; 626 627 input_addr = 628 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 629 630 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n", 631 (unsigned long)sys_addr, (unsigned long)input_addr); 632 633 return input_addr; 634} 635 636 637/* 638 * @input_addr is an InputAddr associated with the node represented by mci. 639 * Translate @input_addr to a DramAddr and return the result. 640 */ 641static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr) 642{ 643 struct amd64_pvt *pvt; 644 int node_id, intlv_shift; 645 u64 bits, dram_addr; 646 u32 intlv_sel; 647 648 /* 649 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 650 * shows how to translate a DramAddr to an InputAddr. Here we reverse 651 * this procedure. When translating from a DramAddr to an InputAddr, the 652 * bits used for node interleaving are discarded. Here we recover these 653 * bits from the IntlvSel field of the DRAM Limit register (section 654 * 3.4.4.2) for the node that input_addr is associated with. 655 */ 656 pvt = mci->pvt_info; 657 node_id = pvt->mc_node_id; 658 BUG_ON((node_id < 0) || (node_id > 7)); 659 660 intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); 661 662 if (intlv_shift == 0) { 663 debugf1(" InputAddr 0x%lx translates to DramAddr of " 664 "same value\n", (unsigned long)input_addr); 665 666 return input_addr; 667 } 668 669 bits = ((input_addr & 0xffffff000ull) << intlv_shift) + 670 (input_addr & 0xfff); 671 672 intlv_sel = pvt->dram_IntlvSel[node_id] & ((1 << intlv_shift) - 1); 673 dram_addr = bits + (intlv_sel << 12); 674 675 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx " 676 "(%d node interleave bits)\n", (unsigned long)input_addr, 677 (unsigned long)dram_addr, intlv_shift); 678 679 return dram_addr; 680} 681 682/* 683 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert 684 * @dram_addr to a SysAddr. 685 */ 686static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr) 687{ 688 struct amd64_pvt *pvt = mci->pvt_info; 689 u64 hole_base, hole_offset, hole_size, base, limit, sys_addr; 690 int ret = 0; 691 692 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 693 &hole_size); 694 if (!ret) { 695 if ((dram_addr >= hole_base) && 696 (dram_addr < (hole_base + hole_size))) { 697 sys_addr = dram_addr + hole_offset; 698 699 debugf1("using DHAR to translate DramAddr 0x%lx to " 700 "SysAddr 0x%lx\n", (unsigned long)dram_addr, 701 (unsigned long)sys_addr); 702 703 return sys_addr; 704 } 705 } 706 707 amd64_get_base_and_limit(pvt, pvt->mc_node_id, &base, &limit); 708 sys_addr = dram_addr + base; 709 710 /* 711 * The sys_addr we have computed up to this point is a 40-bit value 712 * because the k8 deals with 40-bit values. However, the value we are 713 * supposed to return is a full 64-bit physical address. The AMD 714 * x86-64 architecture specifies that the most significant implemented 715 * address bit through bit 63 of a physical address must be either all 716 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a 717 * 64-bit value below. See section 3.4.2 of AMD publication 24592: 718 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application 719 * Programming. 720 */ 721 sys_addr |= ~((sys_addr & (1ull << 39)) - 1); 722 723 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n", 724 pvt->mc_node_id, (unsigned long)dram_addr, 725 (unsigned long)sys_addr); 726 727 return sys_addr; 728} 729 730/* 731 * @input_addr is an InputAddr associated with the node given by mci. Translate 732 * @input_addr to a SysAddr. 733 */ 734static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci, 735 u64 input_addr) 736{ 737 return dram_addr_to_sys_addr(mci, 738 input_addr_to_dram_addr(mci, input_addr)); 739} 740 741/* 742 * Find the minimum and maximum InputAddr values that map to the given @csrow. 743 * Pass back these values in *input_addr_min and *input_addr_max. 744 */ 745static void find_csrow_limits(struct mem_ctl_info *mci, int csrow, 746 u64 *input_addr_min, u64 *input_addr_max) 747{ 748 struct amd64_pvt *pvt; 749 u64 base, mask; 750 751 pvt = mci->pvt_info; 752 BUG_ON((csrow < 0) || (csrow >= pvt->cs_count)); 753 754 base = base_from_dct_base(pvt, csrow); 755 mask = mask_from_dct_mask(pvt, csrow); 756 757 *input_addr_min = base & ~mask; 758 *input_addr_max = base | mask | pvt->dcs_mask_notused; 759} 760 761/* Map the Error address to a PAGE and PAGE OFFSET. */ 762static inline void error_address_to_page_and_offset(u64 error_address, 763 u32 *page, u32 *offset) 764{ 765 *page = (u32) (error_address >> PAGE_SHIFT); 766 *offset = ((u32) error_address) & ~PAGE_MASK; 767} 768 769/* 770 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 771 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 772 * of a node that detected an ECC memory error. mci represents the node that 773 * the error address maps to (possibly different from the node that detected 774 * the error). Return the number of the csrow that sys_addr maps to, or -1 on 775 * error. 776 */ 777static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 778{ 779 int csrow; 780 781 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 782 783 if (csrow == -1) 784 amd64_mc_printk(mci, KERN_ERR, 785 "Failed to translate InputAddr to csrow for " 786 "address 0x%lx\n", (unsigned long)sys_addr); 787 return csrow; 788} 789 790static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 791 792static u16 extract_syndrome(struct err_regs *err) 793{ 794 return ((err->nbsh >> 15) & 0xff) | ((err->nbsl >> 16) & 0xff00); 795} 796 797static void amd64_cpu_display_info(struct amd64_pvt *pvt) 798{ 799 if (boot_cpu_data.x86 == 0x11) 800 edac_printk(KERN_DEBUG, EDAC_MC, "F11h CPU detected\n"); 801 else if (boot_cpu_data.x86 == 0x10) 802 edac_printk(KERN_DEBUG, EDAC_MC, "F10h CPU detected\n"); 803 else if (boot_cpu_data.x86 == 0xf) 804 edac_printk(KERN_DEBUG, EDAC_MC, "%s detected\n", 805 (pvt->ext_model >= K8_REV_F) ? 806 "Rev F or later" : "Rev E or earlier"); 807 else 808 /* we'll hardly ever ever get here */ 809 edac_printk(KERN_ERR, EDAC_MC, "Unknown cpu!\n"); 810} 811 812/* 813 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 814 * are ECC capable. 815 */ 816static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt) 817{ 818 int bit; 819 enum dev_type edac_cap = EDAC_FLAG_NONE; 820 821 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F) 822 ? 19 823 : 17; 824 825 if (pvt->dclr0 & BIT(bit)) 826 edac_cap = EDAC_FLAG_SECDED; 827 828 return edac_cap; 829} 830 831 832static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt); 833 834static void amd64_dump_dramcfg_low(u32 dclr, int chan) 835{ 836 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 837 838 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n", 839 (dclr & BIT(16)) ? "un" : "", 840 (dclr & BIT(19)) ? "yes" : "no"); 841 842 debugf1(" PAR/ERR parity: %s\n", 843 (dclr & BIT(8)) ? "enabled" : "disabled"); 844 845 debugf1(" DCT 128bit mode width: %s\n", 846 (dclr & BIT(11)) ? "128b" : "64b"); 847 848 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 849 (dclr & BIT(12)) ? "yes" : "no", 850 (dclr & BIT(13)) ? "yes" : "no", 851 (dclr & BIT(14)) ? "yes" : "no", 852 (dclr & BIT(15)) ? "yes" : "no"); 853} 854 855/* Display and decode various NB registers for debug purposes. */ 856static void amd64_dump_misc_regs(struct amd64_pvt *pvt) 857{ 858 int ganged; 859 860 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 861 862 debugf1(" NB two channel DRAM capable: %s\n", 863 (pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "yes" : "no"); 864 865 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n", 866 (pvt->nbcap & K8_NBCAP_SECDED) ? "yes" : "no", 867 (pvt->nbcap & K8_NBCAP_CHIPKILL) ? "yes" : "no"); 868 869 amd64_dump_dramcfg_low(pvt->dclr0, 0); 870 871 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 872 873 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, " 874 "offset: 0x%08x\n", 875 pvt->dhar, 876 dhar_base(pvt->dhar), 877 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt->dhar) 878 : f10_dhar_offset(pvt->dhar)); 879 880 debugf1(" DramHoleValid: %s\n", 881 (pvt->dhar & DHAR_VALID) ? "yes" : "no"); 882 883 /* everything below this point is Fam10h and above */ 884 if (boot_cpu_data.x86 == 0xf) { 885 amd64_debug_display_dimm_sizes(0, pvt); 886 return; 887 } 888 889 amd64_printk(KERN_INFO, "using %s syndromes.\n", 890 ((pvt->syn_type == 8) ? "x8" : "x4")); 891 892 /* Only if NOT ganged does dclr1 have valid info */ 893 if (!dct_ganging_enabled(pvt)) 894 amd64_dump_dramcfg_low(pvt->dclr1, 1); 895 896 /* 897 * Determine if ganged and then dump memory sizes for first controller, 898 * and if NOT ganged dump info for 2nd controller. 899 */ 900 ganged = dct_ganging_enabled(pvt); 901 902 amd64_debug_display_dimm_sizes(0, pvt); 903 904 if (!ganged) 905 amd64_debug_display_dimm_sizes(1, pvt); 906} 907 908/* Read in both of DBAM registers */ 909static void amd64_read_dbam_reg(struct amd64_pvt *pvt) 910{ 911 amd64_read_pci_cfg(pvt->dram_f2_ctl, DBAM0, &pvt->dbam0); 912 913 if (boot_cpu_data.x86 >= 0x10) 914 amd64_read_pci_cfg(pvt->dram_f2_ctl, DBAM1, &pvt->dbam1); 915} 916 917/* 918 * NOTE: CPU Revision Dependent code: Rev E and Rev F 919 * 920 * Set the DCSB and DCSM mask values depending on the CPU revision value. Also 921 * set the shift factor for the DCSB and DCSM values. 922 * 923 * ->dcs_mask_notused, RevE: 924 * 925 * To find the max InputAddr for the csrow, start with the base address and set 926 * all bits that are "don't care" bits in the test at the start of section 927 * 3.5.4 (p. 84). 928 * 929 * The "don't care" bits are all set bits in the mask and all bits in the gaps 930 * between bit ranges [35:25] and [19:13]. The value REV_E_DCS_NOTUSED_BITS 931 * represents bits [24:20] and [12:0], which are all bits in the above-mentioned 932 * gaps. 933 * 934 * ->dcs_mask_notused, RevF and later: 935 * 936 * To find the max InputAddr for the csrow, start with the base address and set 937 * all bits that are "don't care" bits in the test at the start of NPT section 938 * 4.5.4 (p. 87). 939 * 940 * The "don't care" bits are all set bits in the mask and all bits in the gaps 941 * between bit ranges [36:27] and [21:13]. 942 * 943 * The value REV_F_F1Xh_DCS_NOTUSED_BITS represents bits [26:22] and [12:0], 944 * which are all bits in the above-mentioned gaps. 945 */ 946static void amd64_set_dct_base_and_mask(struct amd64_pvt *pvt) 947{ 948 949 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 950 pvt->dcsb_base = REV_E_DCSB_BASE_BITS; 951 pvt->dcsm_mask = REV_E_DCSM_MASK_BITS; 952 pvt->dcs_mask_notused = REV_E_DCS_NOTUSED_BITS; 953 pvt->dcs_shift = REV_E_DCS_SHIFT; 954 pvt->cs_count = 8; 955 pvt->num_dcsm = 8; 956 } else { 957 pvt->dcsb_base = REV_F_F1Xh_DCSB_BASE_BITS; 958 pvt->dcsm_mask = REV_F_F1Xh_DCSM_MASK_BITS; 959 pvt->dcs_mask_notused = REV_F_F1Xh_DCS_NOTUSED_BITS; 960 pvt->dcs_shift = REV_F_F1Xh_DCS_SHIFT; 961 962 if (boot_cpu_data.x86 == 0x11) { 963 pvt->cs_count = 4; 964 pvt->num_dcsm = 2; 965 } else { 966 pvt->cs_count = 8; 967 pvt->num_dcsm = 4; 968 } 969 } 970} 971 972/* 973 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask hw registers 974 */ 975static void amd64_read_dct_base_mask(struct amd64_pvt *pvt) 976{ 977 int cs, reg; 978 979 amd64_set_dct_base_and_mask(pvt); 980 981 for (cs = 0; cs < pvt->cs_count; cs++) { 982 reg = K8_DCSB0 + (cs * 4); 983 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, &pvt->dcsb0[cs])) 984 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n", 985 cs, pvt->dcsb0[cs], reg); 986 987 /* If DCT are NOT ganged, then read in DCT1's base */ 988 if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { 989 reg = F10_DCSB1 + (cs * 4); 990 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, 991 &pvt->dcsb1[cs])) 992 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n", 993 cs, pvt->dcsb1[cs], reg); 994 } else { 995 pvt->dcsb1[cs] = 0; 996 } 997 } 998 999 for (cs = 0; cs < pvt->num_dcsm; cs++) { 1000 reg = K8_DCSM0 + (cs * 4); 1001 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, &pvt->dcsm0[cs])) 1002 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n", 1003 cs, pvt->dcsm0[cs], reg); 1004 1005 /* If DCT are NOT ganged, then read in DCT1's mask */ 1006 if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { 1007 reg = F10_DCSM1 + (cs * 4); 1008 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, 1009 &pvt->dcsm1[cs])) 1010 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n", 1011 cs, pvt->dcsm1[cs], reg); 1012 } else { 1013 pvt->dcsm1[cs] = 0; 1014 } 1015 } 1016} 1017 1018static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt) 1019{ 1020 enum mem_type type; 1021 1022 if (boot_cpu_data.x86 >= 0x10 || pvt->ext_model >= K8_REV_F) { 1023 if (pvt->dchr0 & DDR3_MODE) 1024 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 1025 else 1026 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 1027 } else { 1028 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 1029 } 1030 1031 debugf1(" Memory type is: %s\n", edac_mem_types[type]); 1032 1033 return type; 1034} 1035 1036/* 1037 * Read the DRAM Configuration Low register. It differs between CG, D & E revs 1038 * and the later RevF memory controllers (DDR vs DDR2) 1039 * 1040 * Return: 1041 * number of memory channels in operation 1042 * Pass back: 1043 * contents of the DCL0_LOW register 1044 */ 1045static int k8_early_channel_count(struct amd64_pvt *pvt) 1046{ 1047 int flag, err = 0; 1048 1049 err = amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0); 1050 if (err) 1051 return err; 1052 1053 if ((boot_cpu_data.x86_model >> 4) >= K8_REV_F) { 1054 /* RevF (NPT) and later */ 1055 flag = pvt->dclr0 & F10_WIDTH_128; 1056 } else { 1057 /* RevE and earlier */ 1058 flag = pvt->dclr0 & REVE_WIDTH_128; 1059 } 1060 1061 /* not used */ 1062 pvt->dclr1 = 0; 1063 1064 return (flag) ? 2 : 1; 1065} 1066 1067/* extract the ERROR ADDRESS for the K8 CPUs */ 1068static u64 k8_get_error_address(struct mem_ctl_info *mci, 1069 struct err_regs *info) 1070{ 1071 return (((u64) (info->nbeah & 0xff)) << 32) + 1072 (info->nbeal & ~0x03); 1073} 1074 1075/* 1076 * Read the Base and Limit registers for K8 based Memory controllers; extract 1077 * fields from the 'raw' reg into separate data fields 1078 * 1079 * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN 1080 */ 1081static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram) 1082{ 1083 u32 low; 1084 u32 off = dram << 3; /* 8 bytes between DRAM entries */ 1085 1086 amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DRAM_BASE_LOW + off, &low); 1087 1088 /* Extract parts into separate data entries */ 1089 pvt->dram_base[dram] = ((u64) low & 0xFFFF0000) << 8; 1090 pvt->dram_IntlvEn[dram] = (low >> 8) & 0x7; 1091 pvt->dram_rw_en[dram] = (low & 0x3); 1092 1093 amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DRAM_LIMIT_LOW + off, &low); 1094 1095 /* 1096 * Extract parts into separate data entries. Limit is the HIGHEST memory 1097 * location of the region, so lower 24 bits need to be all ones 1098 */ 1099 pvt->dram_limit[dram] = (((u64) low & 0xFFFF0000) << 8) | 0x00FFFFFF; 1100 pvt->dram_IntlvSel[dram] = (low >> 8) & 0x7; 1101 pvt->dram_DstNode[dram] = (low & 0x7); 1102} 1103 1104static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, 1105 struct err_regs *err_info, u64 sys_addr) 1106{ 1107 struct mem_ctl_info *src_mci; 1108 int channel, csrow; 1109 u32 page, offset; 1110 u16 syndrome; 1111 1112 syndrome = extract_syndrome(err_info); 1113 1114 /* CHIPKILL enabled */ 1115 if (err_info->nbcfg & K8_NBCFG_CHIPKILL) { 1116 channel = get_channel_from_ecc_syndrome(mci, syndrome); 1117 if (channel < 0) { 1118 /* 1119 * Syndrome didn't map, so we don't know which of the 1120 * 2 DIMMs is in error. So we need to ID 'both' of them 1121 * as suspect. 1122 */ 1123 amd64_mc_printk(mci, KERN_WARNING, 1124 "unknown syndrome 0x%04x - possible " 1125 "error reporting race\n", syndrome); 1126 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1127 return; 1128 } 1129 } else { 1130 /* 1131 * non-chipkill ecc mode 1132 * 1133 * The k8 documentation is unclear about how to determine the 1134 * channel number when using non-chipkill memory. This method 1135 * was obtained from email communication with someone at AMD. 1136 * (Wish the email was placed in this comment - norsk) 1137 */ 1138 channel = ((sys_addr & BIT(3)) != 0); 1139 } 1140 1141 /* 1142 * Find out which node the error address belongs to. This may be 1143 * different from the node that detected the error. 1144 */ 1145 src_mci = find_mc_by_sys_addr(mci, sys_addr); 1146 if (!src_mci) { 1147 amd64_mc_printk(mci, KERN_ERR, 1148 "failed to map error address 0x%lx to a node\n", 1149 (unsigned long)sys_addr); 1150 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1151 return; 1152 } 1153 1154 /* Now map the sys_addr to a CSROW */ 1155 csrow = sys_addr_to_csrow(src_mci, sys_addr); 1156 if (csrow < 0) { 1157 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR); 1158 } else { 1159 error_address_to_page_and_offset(sys_addr, &page, &offset); 1160 1161 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow, 1162 channel, EDAC_MOD_STR); 1163 } 1164} 1165 1166static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) 1167{ 1168 int *dbam_map; 1169 1170 if (pvt->ext_model >= K8_REV_F) 1171 dbam_map = ddr2_dbam; 1172 else if (pvt->ext_model >= K8_REV_D) 1173 dbam_map = ddr2_dbam_revD; 1174 else 1175 dbam_map = ddr2_dbam_revCG; 1176 1177 return dbam_map[cs_mode]; 1178} 1179 1180/* 1181 * Get the number of DCT channels in use. 1182 * 1183 * Return: 1184 * number of Memory Channels in operation 1185 * Pass back: 1186 * contents of the DCL0_LOW register 1187 */ 1188static int f10_early_channel_count(struct amd64_pvt *pvt) 1189{ 1190 int dbams[] = { DBAM0, DBAM1 }; 1191 int i, j, channels = 0; 1192 u32 dbam; 1193 1194 /* If we are in 128 bit mode, then we are using 2 channels */ 1195 if (pvt->dclr0 & F10_WIDTH_128) { 1196 channels = 2; 1197 return channels; 1198 } 1199 1200 /* 1201 * Need to check if in unganged mode: In such, there are 2 channels, 1202 * but they are not in 128 bit mode and thus the above 'dclr0' status 1203 * bit will be OFF. 1204 * 1205 * Need to check DCT0[0] and DCT1[0] to see if only one of them has 1206 * their CSEnable bit on. If so, then SINGLE DIMM case. 1207 */ 1208 debugf0("Data width is not 128 bits - need more decoding\n"); 1209 1210 /* 1211 * Check DRAM Bank Address Mapping values for each DIMM to see if there 1212 * is more than just one DIMM present in unganged mode. Need to check 1213 * both controllers since DIMMs can be placed in either one. 1214 */ 1215 for (i = 0; i < ARRAY_SIZE(dbams); i++) { 1216 if (amd64_read_pci_cfg(pvt->dram_f2_ctl, dbams[i], &dbam)) 1217 goto err_reg; 1218 1219 for (j = 0; j < 4; j++) { 1220 if (DBAM_DIMM(j, dbam) > 0) { 1221 channels++; 1222 break; 1223 } 1224 } 1225 } 1226 1227 if (channels > 2) 1228 channels = 2; 1229 1230 debugf0("MCT channel count: %d\n", channels); 1231 1232 return channels; 1233 1234err_reg: 1235 return -1; 1236 1237} 1238 1239static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) 1240{ 1241 int *dbam_map; 1242 1243 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 1244 dbam_map = ddr3_dbam; 1245 else 1246 dbam_map = ddr2_dbam; 1247 1248 return dbam_map[cs_mode]; 1249} 1250 1251/* Enable extended configuration access via 0xCF8 feature */ 1252static void amd64_setup(struct amd64_pvt *pvt) 1253{ 1254 u32 reg; 1255 1256 amd64_read_pci_cfg(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, ®); 1257 1258 pvt->flags.cf8_extcfg = !!(reg & F10_NB_CFG_LOW_ENABLE_EXT_CFG); 1259 reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG; 1260 pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg); 1261} 1262 1263/* Restore the extended configuration access via 0xCF8 feature */ 1264static void amd64_teardown(struct amd64_pvt *pvt) 1265{ 1266 u32 reg; 1267 1268 amd64_read_pci_cfg(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, ®); 1269 1270 reg &= ~F10_NB_CFG_LOW_ENABLE_EXT_CFG; 1271 if (pvt->flags.cf8_extcfg) 1272 reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG; 1273 pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg); 1274} 1275 1276static u64 f10_get_error_address(struct mem_ctl_info *mci, 1277 struct err_regs *info) 1278{ 1279 return (((u64) (info->nbeah & 0xffff)) << 32) + 1280 (info->nbeal & ~0x01); 1281} 1282 1283/* 1284 * Read the Base and Limit registers for F10 based Memory controllers. Extract 1285 * fields from the 'raw' reg into separate data fields. 1286 * 1287 * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN. 1288 */ 1289static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram) 1290{ 1291 u32 high_offset, low_offset, high_base, low_base, high_limit, low_limit; 1292 1293 low_offset = K8_DRAM_BASE_LOW + (dram << 3); 1294 high_offset = F10_DRAM_BASE_HIGH + (dram << 3); 1295 1296 /* read the 'raw' DRAM BASE Address register */ 1297 amd64_read_pci_cfg(pvt->addr_f1_ctl, low_offset, &low_base); 1298 1299 /* Read from the ECS data register */ 1300 amd64_read_pci_cfg(pvt->addr_f1_ctl, high_offset, &high_base); 1301 1302 /* Extract parts into separate data entries */ 1303 pvt->dram_rw_en[dram] = (low_base & 0x3); 1304 1305 if (pvt->dram_rw_en[dram] == 0) 1306 return; 1307 1308 pvt->dram_IntlvEn[dram] = (low_base >> 8) & 0x7; 1309 1310 pvt->dram_base[dram] = (((u64)high_base & 0x000000FF) << 40) | 1311 (((u64)low_base & 0xFFFF0000) << 8); 1312 1313 low_offset = K8_DRAM_LIMIT_LOW + (dram << 3); 1314 high_offset = F10_DRAM_LIMIT_HIGH + (dram << 3); 1315 1316 /* read the 'raw' LIMIT registers */ 1317 amd64_read_pci_cfg(pvt->addr_f1_ctl, low_offset, &low_limit); 1318 1319 /* Read from the ECS data register for the HIGH portion */ 1320 amd64_read_pci_cfg(pvt->addr_f1_ctl, high_offset, &high_limit); 1321 1322 pvt->dram_DstNode[dram] = (low_limit & 0x7); 1323 pvt->dram_IntlvSel[dram] = (low_limit >> 8) & 0x7; 1324 1325 /* 1326 * Extract address values and form a LIMIT address. Limit is the HIGHEST 1327 * memory location of the region, so low 24 bits need to be all ones. 1328 */ 1329 pvt->dram_limit[dram] = (((u64)high_limit & 0x000000FF) << 40) | 1330 (((u64) low_limit & 0xFFFF0000) << 8) | 1331 0x00FFFFFF; 1332} 1333 1334static void f10_read_dram_ctl_register(struct amd64_pvt *pvt) 1335{ 1336 1337 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCTL_SEL_LOW, 1338 &pvt->dram_ctl_select_low)) { 1339 debugf0("F2x110 (DCTL Sel. Low): 0x%08x, " 1340 "High range addresses at: 0x%x\n", 1341 pvt->dram_ctl_select_low, 1342 dct_sel_baseaddr(pvt)); 1343 1344 debugf0(" DCT mode: %s, All DCTs on: %s\n", 1345 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"), 1346 (dct_dram_enabled(pvt) ? "yes" : "no")); 1347 1348 if (!dct_ganging_enabled(pvt)) 1349 debugf0(" Address range split per DCT: %s\n", 1350 (dct_high_range_enabled(pvt) ? "yes" : "no")); 1351 1352 debugf0(" DCT data interleave for ECC: %s, " 1353 "DRAM cleared since last warm reset: %s\n", 1354 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 1355 (dct_memory_cleared(pvt) ? "yes" : "no")); 1356 1357 debugf0(" DCT channel interleave: %s, " 1358 "DCT interleave bits selector: 0x%x\n", 1359 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 1360 dct_sel_interleave_addr(pvt)); 1361 } 1362 1363 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCTL_SEL_HIGH, 1364 &pvt->dram_ctl_select_high); 1365} 1366 1367/* 1368 * determine channel based on the interleaving mode: F10h BKDG, 2.8.9 Memory 1369 * Interleaving Modes. 1370 */ 1371static u32 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 1372 int hi_range_sel, u32 intlv_en) 1373{ 1374 u32 cs, temp, dct_sel_high = (pvt->dram_ctl_select_low >> 1) & 1; 1375 1376 if (dct_ganging_enabled(pvt)) 1377 cs = 0; 1378 else if (hi_range_sel) 1379 cs = dct_sel_high; 1380 else if (dct_interleave_enabled(pvt)) { 1381 /* 1382 * see F2x110[DctSelIntLvAddr] - channel interleave mode 1383 */ 1384 if (dct_sel_interleave_addr(pvt) == 0) 1385 cs = sys_addr >> 6 & 1; 1386 else if ((dct_sel_interleave_addr(pvt) >> 1) & 1) { 1387 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; 1388 1389 if (dct_sel_interleave_addr(pvt) & 1) 1390 cs = (sys_addr >> 9 & 1) ^ temp; 1391 else 1392 cs = (sys_addr >> 6 & 1) ^ temp; 1393 } else if (intlv_en & 4) 1394 cs = sys_addr >> 15 & 1; 1395 else if (intlv_en & 2) 1396 cs = sys_addr >> 14 & 1; 1397 else if (intlv_en & 1) 1398 cs = sys_addr >> 13 & 1; 1399 else 1400 cs = sys_addr >> 12 & 1; 1401 } else if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt)) 1402 cs = ~dct_sel_high & 1; 1403 else 1404 cs = 0; 1405 1406 return cs; 1407} 1408 1409static inline u32 f10_map_intlv_en_to_shift(u32 intlv_en) 1410{ 1411 if (intlv_en == 1) 1412 return 1; 1413 else if (intlv_en == 3) 1414 return 2; 1415 else if (intlv_en == 7) 1416 return 3; 1417 1418 return 0; 1419} 1420 1421/* See F10h BKDG, 2.8.10.2 DctSelBaseOffset Programming */ 1422static inline u64 f10_get_base_addr_offset(u64 sys_addr, int hi_range_sel, 1423 u32 dct_sel_base_addr, 1424 u64 dct_sel_base_off, 1425 u32 hole_valid, u32 hole_off, 1426 u64 dram_base) 1427{ 1428 u64 chan_off; 1429 1430 if (hi_range_sel) { 1431 if (!(dct_sel_base_addr & 0xFFFF0000) && 1432 hole_valid && (sys_addr >= 0x100000000ULL)) 1433 chan_off = hole_off << 16; 1434 else 1435 chan_off = dct_sel_base_off; 1436 } else { 1437 if (hole_valid && (sys_addr >= 0x100000000ULL)) 1438 chan_off = hole_off << 16; 1439 else 1440 chan_off = dram_base & 0xFFFFF8000000ULL; 1441 } 1442 1443 return (sys_addr & 0x0000FFFFFFFFFFC0ULL) - 1444 (chan_off & 0x0000FFFFFF800000ULL); 1445} 1446 1447/* Hack for the time being - Can we get this from BIOS?? */ 1448#define CH0SPARE_RANK 0 1449#define CH1SPARE_RANK 1 1450 1451/* 1452 * checks if the csrow passed in is marked as SPARED, if so returns the new 1453 * spare row 1454 */ 1455static inline int f10_process_possible_spare(int csrow, 1456 u32 cs, struct amd64_pvt *pvt) 1457{ 1458 u32 swap_done; 1459 u32 bad_dram_cs; 1460 1461 /* Depending on channel, isolate respective SPARING info */ 1462 if (cs) { 1463 swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare); 1464 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare); 1465 if (swap_done && (csrow == bad_dram_cs)) 1466 csrow = CH1SPARE_RANK; 1467 } else { 1468 swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare); 1469 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare); 1470 if (swap_done && (csrow == bad_dram_cs)) 1471 csrow = CH0SPARE_RANK; 1472 } 1473 return csrow; 1474} 1475 1476/* 1477 * Iterate over the DRAM DCT "base" and "mask" registers looking for a 1478 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 1479 * 1480 * Return: 1481 * -EINVAL: NOT FOUND 1482 * 0..csrow = Chip-Select Row 1483 */ 1484static int f10_lookup_addr_in_dct(u32 in_addr, u32 nid, u32 cs) 1485{ 1486 struct mem_ctl_info *mci; 1487 struct amd64_pvt *pvt; 1488 u32 cs_base, cs_mask; 1489 int cs_found = -EINVAL; 1490 int csrow; 1491 1492 mci = mci_lookup[nid]; 1493 if (!mci) 1494 return cs_found; 1495 1496 pvt = mci->pvt_info; 1497 1498 debugf1("InputAddr=0x%x channelselect=%d\n", in_addr, cs); 1499 1500 for (csrow = 0; csrow < pvt->cs_count; csrow++) { 1501 1502 cs_base = amd64_get_dct_base(pvt, cs, csrow); 1503 if (!(cs_base & K8_DCSB_CS_ENABLE)) 1504 continue; 1505 1506 /* 1507 * We have an ENABLED CSROW, Isolate just the MASK bits of the 1508 * target: [28:19] and [13:5], which map to [36:27] and [21:13] 1509 * of the actual address. 1510 */ 1511 cs_base &= REV_F_F1Xh_DCSB_BASE_BITS; 1512 1513 /* 1514 * Get the DCT Mask, and ENABLE the reserved bits: [18:16] and 1515 * [4:0] to become ON. Then mask off bits [28:0] ([36:8]) 1516 */ 1517 cs_mask = amd64_get_dct_mask(pvt, cs, csrow); 1518 1519 debugf1(" CSROW=%d CSBase=0x%x RAW CSMask=0x%x\n", 1520 csrow, cs_base, cs_mask); 1521 1522 cs_mask = (cs_mask | 0x0007C01F) & 0x1FFFFFFF; 1523 1524 debugf1(" Final CSMask=0x%x\n", cs_mask); 1525 debugf1(" (InputAddr & ~CSMask)=0x%x " 1526 "(CSBase & ~CSMask)=0x%x\n", 1527 (in_addr & ~cs_mask), (cs_base & ~cs_mask)); 1528 1529 if ((in_addr & ~cs_mask) == (cs_base & ~cs_mask)) { 1530 cs_found = f10_process_possible_spare(csrow, cs, pvt); 1531 1532 debugf1(" MATCH csrow=%d\n", cs_found); 1533 break; 1534 } 1535 } 1536 return cs_found; 1537} 1538 1539/* For a given @dram_range, check if @sys_addr falls within it. */ 1540static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range, 1541 u64 sys_addr, int *nid, int *chan_sel) 1542{ 1543 int node_id, cs_found = -EINVAL, high_range = 0; 1544 u32 intlv_en, intlv_sel, intlv_shift, hole_off; 1545 u32 hole_valid, tmp, dct_sel_base, channel; 1546 u64 dram_base, chan_addr, dct_sel_base_off; 1547 1548 dram_base = pvt->dram_base[dram_range]; 1549 intlv_en = pvt->dram_IntlvEn[dram_range]; 1550 1551 node_id = pvt->dram_DstNode[dram_range]; 1552 intlv_sel = pvt->dram_IntlvSel[dram_range]; 1553 1554 debugf1("(dram=%d) Base=0x%llx SystemAddr= 0x%llx Limit=0x%llx\n", 1555 dram_range, dram_base, sys_addr, pvt->dram_limit[dram_range]); 1556 1557 /* 1558 * This assumes that one node's DHAR is the same as all the other 1559 * nodes' DHAR. 1560 */ 1561 hole_off = (pvt->dhar & 0x0000FF80); 1562 hole_valid = (pvt->dhar & 0x1); 1563 dct_sel_base_off = (pvt->dram_ctl_select_high & 0xFFFFFC00) << 16; 1564 1565 debugf1(" HoleOffset=0x%x HoleValid=0x%x IntlvSel=0x%x\n", 1566 hole_off, hole_valid, intlv_sel); 1567 1568 if (intlv_en && 1569 (intlv_sel != ((sys_addr >> 12) & intlv_en))) 1570 return -EINVAL; 1571 1572 dct_sel_base = dct_sel_baseaddr(pvt); 1573 1574 /* 1575 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 1576 * select between DCT0 and DCT1. 1577 */ 1578 if (dct_high_range_enabled(pvt) && 1579 !dct_ganging_enabled(pvt) && 1580 ((sys_addr >> 27) >= (dct_sel_base >> 11))) 1581 high_range = 1; 1582 1583 channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en); 1584 1585 chan_addr = f10_get_base_addr_offset(sys_addr, high_range, dct_sel_base, 1586 dct_sel_base_off, hole_valid, 1587 hole_off, dram_base); 1588 1589 intlv_shift = f10_map_intlv_en_to_shift(intlv_en); 1590 1591 /* remove Node ID (in case of memory interleaving) */ 1592 tmp = chan_addr & 0xFC0; 1593 1594 chan_addr = ((chan_addr >> intlv_shift) & 0xFFFFFFFFF000ULL) | tmp; 1595 1596 /* remove channel interleave and hash */ 1597 if (dct_interleave_enabled(pvt) && 1598 !dct_high_range_enabled(pvt) && 1599 !dct_ganging_enabled(pvt)) { 1600 if (dct_sel_interleave_addr(pvt) != 1) 1601 chan_addr = (chan_addr >> 1) & 0xFFFFFFFFFFFFFFC0ULL; 1602 else { 1603 tmp = chan_addr & 0xFC0; 1604 chan_addr = ((chan_addr & 0xFFFFFFFFFFFFC000ULL) >> 1) 1605 | tmp; 1606 } 1607 } 1608 1609 debugf1(" (ChannelAddrLong=0x%llx) >> 8 becomes InputAddr=0x%x\n", 1610 chan_addr, (u32)(chan_addr >> 8)); 1611 1612 cs_found = f10_lookup_addr_in_dct(chan_addr >> 8, node_id, channel); 1613 1614 if (cs_found >= 0) { 1615 *nid = node_id; 1616 *chan_sel = channel; 1617 } 1618 return cs_found; 1619} 1620 1621static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, 1622 int *node, int *chan_sel) 1623{ 1624 int dram_range, cs_found = -EINVAL; 1625 u64 dram_base, dram_limit; 1626 1627 for (dram_range = 0; dram_range < DRAM_REG_COUNT; dram_range++) { 1628 1629 if (!pvt->dram_rw_en[dram_range]) 1630 continue; 1631 1632 dram_base = pvt->dram_base[dram_range]; 1633 dram_limit = pvt->dram_limit[dram_range]; 1634 1635 if ((dram_base <= sys_addr) && (sys_addr <= dram_limit)) { 1636 1637 cs_found = f10_match_to_this_node(pvt, dram_range, 1638 sys_addr, node, 1639 chan_sel); 1640 if (cs_found >= 0) 1641 break; 1642 } 1643 } 1644 return cs_found; 1645} 1646 1647/* 1648 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 1649 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 1650 * 1651 * The @sys_addr is usually an error address received from the hardware 1652 * (MCX_ADDR). 1653 */ 1654static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci, 1655 struct err_regs *err_info, 1656 u64 sys_addr) 1657{ 1658 struct amd64_pvt *pvt = mci->pvt_info; 1659 u32 page, offset; 1660 int nid, csrow, chan = 0; 1661 u16 syndrome; 1662 1663 csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan); 1664 1665 if (csrow < 0) { 1666 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1667 return; 1668 } 1669 1670 error_address_to_page_and_offset(sys_addr, &page, &offset); 1671 1672 syndrome = extract_syndrome(err_info); 1673 1674 /* 1675 * We need the syndromes for channel detection only when we're 1676 * ganged. Otherwise @chan should already contain the channel at 1677 * this point. 1678 */ 1679 if (dct_ganging_enabled(pvt) && (pvt->nbcfg & K8_NBCFG_CHIPKILL)) 1680 chan = get_channel_from_ecc_syndrome(mci, syndrome); 1681 1682 if (chan >= 0) 1683 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan, 1684 EDAC_MOD_STR); 1685 else 1686 /* 1687 * Channel unknown, report all channels on this CSROW as failed. 1688 */ 1689 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++) 1690 edac_mc_handle_ce(mci, page, offset, syndrome, 1691 csrow, chan, EDAC_MOD_STR); 1692} 1693 1694/* 1695 * debug routine to display the memory sizes of all logical DIMMs and its 1696 * CSROWs as well 1697 */ 1698static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt) 1699{ 1700 int dimm, size0, size1, factor = 0; 1701 u32 dbam; 1702 u32 *dcsb; 1703 1704 if (boot_cpu_data.x86 == 0xf) { 1705 if (pvt->dclr0 & F10_WIDTH_128) 1706 factor = 1; 1707 1708 /* K8 families < revF not supported yet */ 1709 if (pvt->ext_model < K8_REV_F) 1710 return; 1711 else 1712 WARN_ON(ctrl != 0); 1713 } 1714 1715 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", 1716 ctrl, ctrl ? pvt->dbam1 : pvt->dbam0); 1717 1718 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 1719 dcsb = ctrl ? pvt->dcsb1 : pvt->dcsb0; 1720 1721 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 1722 1723 /* Dump memory sizes for DIMM and its CSROWs */ 1724 for (dimm = 0; dimm < 4; dimm++) { 1725 1726 size0 = 0; 1727 if (dcsb[dimm*2] & K8_DCSB_CS_ENABLE) 1728 size0 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam)); 1729 1730 size1 = 0; 1731 if (dcsb[dimm*2 + 1] & K8_DCSB_CS_ENABLE) 1732 size1 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam)); 1733 1734 edac_printk(KERN_DEBUG, EDAC_MC, " %d: %5dMB %d: %5dMB\n", 1735 dimm * 2, size0 << factor, 1736 dimm * 2 + 1, size1 << factor); 1737 } 1738} 1739 1740/* 1741 * There currently are 3 types type of MC devices for AMD Athlon/Opterons 1742 * (as per PCI DEVICE_IDs): 1743 * 1744 * Family K8: That is the Athlon64 and Opteron CPUs. They all have the same PCI 1745 * DEVICE ID, even though there is differences between the different Revisions 1746 * (CG,D,E,F). 1747 * 1748 * Family F10h and F11h. 1749 * 1750 */ 1751static struct amd64_family_type amd64_family_types[] = { 1752 [K8_CPUS] = { 1753 .ctl_name = "RevF", 1754 .addr_f1_ctl = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, 1755 .misc_f3_ctl = PCI_DEVICE_ID_AMD_K8_NB_MISC, 1756 .ops = { 1757 .early_channel_count = k8_early_channel_count, 1758 .get_error_address = k8_get_error_address, 1759 .read_dram_base_limit = k8_read_dram_base_limit, 1760 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, 1761 .dbam_to_cs = k8_dbam_to_chip_select, 1762 } 1763 }, 1764 [F10_CPUS] = { 1765 .ctl_name = "Family 10h", 1766 .addr_f1_ctl = PCI_DEVICE_ID_AMD_10H_NB_MAP, 1767 .misc_f3_ctl = PCI_DEVICE_ID_AMD_10H_NB_MISC, 1768 .ops = { 1769 .early_channel_count = f10_early_channel_count, 1770 .get_error_address = f10_get_error_address, 1771 .read_dram_base_limit = f10_read_dram_base_limit, 1772 .read_dram_ctl_register = f10_read_dram_ctl_register, 1773 .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow, 1774 .dbam_to_cs = f10_dbam_to_chip_select, 1775 } 1776 }, 1777 [F11_CPUS] = { 1778 .ctl_name = "Family 11h", 1779 .addr_f1_ctl = PCI_DEVICE_ID_AMD_11H_NB_MAP, 1780 .misc_f3_ctl = PCI_DEVICE_ID_AMD_11H_NB_MISC, 1781 .ops = { 1782 .early_channel_count = f10_early_channel_count, 1783 .get_error_address = f10_get_error_address, 1784 .read_dram_base_limit = f10_read_dram_base_limit, 1785 .read_dram_ctl_register = f10_read_dram_ctl_register, 1786 .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow, 1787 .dbam_to_cs = f10_dbam_to_chip_select, 1788 } 1789 }, 1790}; 1791 1792static struct pci_dev *pci_get_related_function(unsigned int vendor, 1793 unsigned int device, 1794 struct pci_dev *related) 1795{ 1796 struct pci_dev *dev = NULL; 1797 1798 dev = pci_get_device(vendor, device, dev); 1799 while (dev) { 1800 if ((dev->bus->number == related->bus->number) && 1801 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 1802 break; 1803 dev = pci_get_device(vendor, device, dev); 1804 } 1805 1806 return dev; 1807} 1808 1809/* 1810 * These are tables of eigenvectors (one per line) which can be used for the 1811 * construction of the syndrome tables. The modified syndrome search algorithm 1812 * uses those to find the symbol in error and thus the DIMM. 1813 * 1814 * Algorithm courtesy of Ross LaFetra from AMD. 1815 */ 1816static u16 x4_vectors[] = { 1817 0x2f57, 0x1afe, 0x66cc, 0xdd88, 1818 0x11eb, 0x3396, 0x7f4c, 0xeac8, 1819 0x0001, 0x0002, 0x0004, 0x0008, 1820 0x1013, 0x3032, 0x4044, 0x8088, 1821 0x106b, 0x30d6, 0x70fc, 0xe0a8, 1822 0x4857, 0xc4fe, 0x13cc, 0x3288, 1823 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 1824 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 1825 0x15c1, 0x2a42, 0x89ac, 0x4758, 1826 0x2b03, 0x1602, 0x4f0c, 0xca08, 1827 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 1828 0x8ba7, 0x465e, 0x244c, 0x1cc8, 1829 0x2b87, 0x164e, 0x642c, 0xdc18, 1830 0x40b9, 0x80de, 0x1094, 0x20e8, 1831 0x27db, 0x1eb6, 0x9dac, 0x7b58, 1832 0x11c1, 0x2242, 0x84ac, 0x4c58, 1833 0x1be5, 0x2d7a, 0x5e34, 0xa718, 1834 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 1835 0x4c97, 0xc87e, 0x11fc, 0x33a8, 1836 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 1837 0x16b3, 0x3d62, 0x4f34, 0x8518, 1838 0x1e2f, 0x391a, 0x5cac, 0xf858, 1839 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 1840 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 1841 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 1842 0x4397, 0xc27e, 0x17fc, 0x3ea8, 1843 0x1617, 0x3d3e, 0x6464, 0xb8b8, 1844 0x23ff, 0x12aa, 0xab6c, 0x56d8, 1845 0x2dfb, 0x1ba6, 0x913c, 0x7328, 1846 0x185d, 0x2ca6, 0x7914, 0x9e28, 1847 0x171b, 0x3e36, 0x7d7c, 0xebe8, 1848 0x4199, 0x82ee, 0x19f4, 0x2e58, 1849 0x4807, 0xc40e, 0x130c, 0x3208, 1850 0x1905, 0x2e0a, 0x5804, 0xac08, 1851 0x213f, 0x132a, 0xadfc, 0x5ba8, 1852 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 1853}; 1854 1855static u16 x8_vectors[] = { 1856 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 1857 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 1858 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 1859 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 1860 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 1861 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 1862 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 1863 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 1864 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 1865 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 1866 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 1867 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 1868 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 1869 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 1870 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 1871 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 1872 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 1873 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 1874 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 1875}; 1876 1877static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs, 1878 int v_dim) 1879{ 1880 unsigned int i, err_sym; 1881 1882 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 1883 u16 s = syndrome; 1884 int v_idx = err_sym * v_dim; 1885 int v_end = (err_sym + 1) * v_dim; 1886 1887 /* walk over all 16 bits of the syndrome */ 1888 for (i = 1; i < (1U << 16); i <<= 1) { 1889 1890 /* if bit is set in that eigenvector... */ 1891 if (v_idx < v_end && vectors[v_idx] & i) { 1892 u16 ev_comp = vectors[v_idx++]; 1893 1894 /* ... and bit set in the modified syndrome, */ 1895 if (s & i) { 1896 /* remove it. */ 1897 s ^= ev_comp; 1898 1899 if (!s) 1900 return err_sym; 1901 } 1902 1903 } else if (s & i) 1904 /* can't get to zero, move to next symbol */ 1905 break; 1906 } 1907 } 1908 1909 debugf0("syndrome(%x) not found\n", syndrome); 1910 return -1; 1911} 1912 1913static int map_err_sym_to_channel(int err_sym, int sym_size) 1914{ 1915 if (sym_size == 4) 1916 switch (err_sym) { 1917 case 0x20: 1918 case 0x21: 1919 return 0; 1920 break; 1921 case 0x22: 1922 case 0x23: 1923 return 1; 1924 break; 1925 default: 1926 return err_sym >> 4; 1927 break; 1928 } 1929 /* x8 symbols */ 1930 else 1931 switch (err_sym) { 1932 /* imaginary bits not in a DIMM */ 1933 case 0x10: 1934 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 1935 err_sym); 1936 return -1; 1937 break; 1938 1939 case 0x11: 1940 return 0; 1941 break; 1942 case 0x12: 1943 return 1; 1944 break; 1945 default: 1946 return err_sym >> 3; 1947 break; 1948 } 1949 return -1; 1950} 1951 1952static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 1953{ 1954 struct amd64_pvt *pvt = mci->pvt_info; 1955 int err_sym = -1; 1956 1957 if (pvt->syn_type == 8) 1958 err_sym = decode_syndrome(syndrome, x8_vectors, 1959 ARRAY_SIZE(x8_vectors), 1960 pvt->syn_type); 1961 else if (pvt->syn_type == 4) 1962 err_sym = decode_syndrome(syndrome, x4_vectors, 1963 ARRAY_SIZE(x4_vectors), 1964 pvt->syn_type); 1965 else { 1966 amd64_printk(KERN_WARNING, "%s: Illegal syndrome type: %u\n", 1967 __func__, pvt->syn_type); 1968 return err_sym; 1969 } 1970 1971 return map_err_sym_to_channel(err_sym, pvt->syn_type); 1972} 1973 1974/* 1975 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR 1976 * ADDRESS and process. 1977 */ 1978static void amd64_handle_ce(struct mem_ctl_info *mci, 1979 struct err_regs *info) 1980{ 1981 struct amd64_pvt *pvt = mci->pvt_info; 1982 u64 sys_addr; 1983 1984 /* Ensure that the Error Address is VALID */ 1985 if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) { 1986 amd64_mc_printk(mci, KERN_ERR, 1987 "HW has no ERROR_ADDRESS available\n"); 1988 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1989 return; 1990 } 1991 1992 sys_addr = pvt->ops->get_error_address(mci, info); 1993 1994 amd64_mc_printk(mci, KERN_ERR, 1995 "CE ERROR_ADDRESS= 0x%llx\n", sys_addr); 1996 1997 pvt->ops->map_sysaddr_to_csrow(mci, info, sys_addr); 1998} 1999 2000/* Handle any Un-correctable Errors (UEs) */ 2001static void amd64_handle_ue(struct mem_ctl_info *mci, 2002 struct err_regs *info) 2003{ 2004 struct amd64_pvt *pvt = mci->pvt_info; 2005 struct mem_ctl_info *log_mci, *src_mci = NULL; 2006 int csrow; 2007 u64 sys_addr; 2008 u32 page, offset; 2009 2010 log_mci = mci; 2011 2012 if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) { 2013 amd64_mc_printk(mci, KERN_CRIT, 2014 "HW has no ERROR_ADDRESS available\n"); 2015 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 2016 return; 2017 } 2018 2019 sys_addr = pvt->ops->get_error_address(mci, info); 2020 2021 /* 2022 * Find out which node the error address belongs to. This may be 2023 * different from the node that detected the error. 2024 */ 2025 src_mci = find_mc_by_sys_addr(mci, sys_addr); 2026 if (!src_mci) { 2027 amd64_mc_printk(mci, KERN_CRIT, 2028 "ERROR ADDRESS (0x%lx) value NOT mapped to a MC\n", 2029 (unsigned long)sys_addr); 2030 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 2031 return; 2032 } 2033 2034 log_mci = src_mci; 2035 2036 csrow = sys_addr_to_csrow(log_mci, sys_addr); 2037 if (csrow < 0) { 2038 amd64_mc_printk(mci, KERN_CRIT, 2039 "ERROR_ADDRESS (0x%lx) value NOT mapped to 'csrow'\n", 2040 (unsigned long)sys_addr); 2041 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 2042 } else { 2043 error_address_to_page_and_offset(sys_addr, &page, &offset); 2044 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR); 2045 } 2046} 2047 2048static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci, 2049 struct err_regs *info) 2050{ 2051 u32 ec = ERROR_CODE(info->nbsl); 2052 u32 xec = EXT_ERROR_CODE(info->nbsl); 2053 int ecc_type = (info->nbsh >> 13) & 0x3; 2054 2055 /* Bail early out if this was an 'observed' error */ 2056 if (PP(ec) == K8_NBSL_PP_OBS) 2057 return; 2058 2059 /* Do only ECC errors */ 2060 if (xec && xec != F10_NBSL_EXT_ERR_ECC) 2061 return; 2062 2063 if (ecc_type == 2) 2064 amd64_handle_ce(mci, info); 2065 else if (ecc_type == 1) 2066 amd64_handle_ue(mci, info); 2067} 2068 2069void amd64_decode_bus_error(int node_id, struct err_regs *regs) 2070{ 2071 struct mem_ctl_info *mci = mci_lookup[node_id]; 2072 2073 __amd64_decode_bus_error(mci, regs); 2074 2075 if (regs->nbsh & K8_NBSH_UC_ERR && !report_gart_errors) 2076 edac_mc_handle_ue_no_info(mci, "UE bit is set"); 2077 2078} 2079 2080/* 2081 * Input: 2082 * 1) struct amd64_pvt which contains pvt->dram_f2_ctl pointer 2083 * 2) AMD Family index value 2084 * 2085 * Ouput: 2086 * Upon return of 0, the following filled in: 2087 * 2088 * struct pvt->addr_f1_ctl 2089 * struct pvt->misc_f3_ctl 2090 * 2091 * Filled in with related device funcitions of 'dram_f2_ctl' 2092 * These devices are "reserved" via the pci_get_device() 2093 * 2094 * Upon return of 1 (error status): 2095 * 2096 * Nothing reserved 2097 */ 2098static int amd64_reserve_mc_sibling_devices(struct amd64_pvt *pvt, int mc_idx) 2099{ 2100 const struct amd64_family_type *amd64_dev = &amd64_family_types[mc_idx]; 2101 2102 /* Reserve the ADDRESS MAP Device */ 2103 pvt->addr_f1_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor, 2104 amd64_dev->addr_f1_ctl, 2105 pvt->dram_f2_ctl); 2106 2107 if (!pvt->addr_f1_ctl) { 2108 amd64_printk(KERN_ERR, "error address map device not found: " 2109 "vendor %x device 0x%x (broken BIOS?)\n", 2110 PCI_VENDOR_ID_AMD, amd64_dev->addr_f1_ctl); 2111 return 1; 2112 } 2113 2114 /* Reserve the MISC Device */ 2115 pvt->misc_f3_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor, 2116 amd64_dev->misc_f3_ctl, 2117 pvt->dram_f2_ctl); 2118 2119 if (!pvt->misc_f3_ctl) { 2120 pci_dev_put(pvt->addr_f1_ctl); 2121 pvt->addr_f1_ctl = NULL; 2122 2123 amd64_printk(KERN_ERR, "error miscellaneous device not found: " 2124 "vendor %x device 0x%x (broken BIOS?)\n", 2125 PCI_VENDOR_ID_AMD, amd64_dev->misc_f3_ctl); 2126 return 1; 2127 } 2128 2129 debugf1(" Addr Map device PCI Bus ID:\t%s\n", 2130 pci_name(pvt->addr_f1_ctl)); 2131 debugf1(" DRAM MEM-CTL PCI Bus ID:\t%s\n", 2132 pci_name(pvt->dram_f2_ctl)); 2133 debugf1(" Misc device PCI Bus ID:\t%s\n", 2134 pci_name(pvt->misc_f3_ctl)); 2135 2136 return 0; 2137} 2138 2139static void amd64_free_mc_sibling_devices(struct amd64_pvt *pvt) 2140{ 2141 pci_dev_put(pvt->addr_f1_ctl); 2142 pci_dev_put(pvt->misc_f3_ctl); 2143} 2144 2145/* 2146 * Retrieve the hardware registers of the memory controller (this includes the 2147 * 'Address Map' and 'Misc' device regs) 2148 */ 2149static void amd64_read_mc_registers(struct amd64_pvt *pvt) 2150{ 2151 u64 msr_val; 2152 u32 tmp; 2153 int dram; 2154 2155 /* 2156 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 2157 * those are Read-As-Zero 2158 */ 2159 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 2160 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem); 2161 2162 /* check first whether TOP_MEM2 is enabled */ 2163 rdmsrl(MSR_K8_SYSCFG, msr_val); 2164 if (msr_val & (1U << 21)) { 2165 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 2166 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 2167 } else 2168 debugf0(" TOP_MEM2 disabled.\n"); 2169 2170 amd64_cpu_display_info(pvt); 2171 2172 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCAP, &pvt->nbcap); 2173 2174 if (pvt->ops->read_dram_ctl_register) 2175 pvt->ops->read_dram_ctl_register(pvt); 2176 2177 for (dram = 0; dram < DRAM_REG_COUNT; dram++) { 2178 /* 2179 * Call CPU specific READ function to get the DRAM Base and 2180 * Limit values from the DCT. 2181 */ 2182 pvt->ops->read_dram_base_limit(pvt, dram); 2183 2184 /* 2185 * Only print out debug info on rows with both R and W Enabled. 2186 * Normal processing, compiler should optimize this whole 'if' 2187 * debug output block away. 2188 */ 2189 if (pvt->dram_rw_en[dram] != 0) { 2190 debugf1(" DRAM-BASE[%d]: 0x%016llx " 2191 "DRAM-LIMIT: 0x%016llx\n", 2192 dram, 2193 pvt->dram_base[dram], 2194 pvt->dram_limit[dram]); 2195 2196 debugf1(" IntlvEn=%s %s %s " 2197 "IntlvSel=%d DstNode=%d\n", 2198 pvt->dram_IntlvEn[dram] ? 2199 "Enabled" : "Disabled", 2200 (pvt->dram_rw_en[dram] & 0x2) ? "W" : "!W", 2201 (pvt->dram_rw_en[dram] & 0x1) ? "R" : "!R", 2202 pvt->dram_IntlvSel[dram], 2203 pvt->dram_DstNode[dram]); 2204 } 2205 } 2206 2207 amd64_read_dct_base_mask(pvt); 2208 2209 amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DHAR, &pvt->dhar); 2210 amd64_read_dbam_reg(pvt); 2211 2212 amd64_read_pci_cfg(pvt->misc_f3_ctl, 2213 F10_ONLINE_SPARE, &pvt->online_spare); 2214 2215 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0); 2216 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCHR_0, &pvt->dchr0); 2217 2218 if (boot_cpu_data.x86 >= 0x10) { 2219 if (!dct_ganging_enabled(pvt)) { 2220 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_1, &pvt->dclr1); 2221 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCHR_1, &pvt->dchr1); 2222 } 2223 amd64_read_pci_cfg(pvt->misc_f3_ctl, EXT_NB_MCA_CFG, &tmp); 2224 } 2225 2226 if (boot_cpu_data.x86 == 0x10 && 2227 boot_cpu_data.x86_model > 7 && 2228 /* F3x180[EccSymbolSize]=1 => x8 symbols */ 2229 tmp & BIT(25)) 2230 pvt->syn_type = 8; 2231 else 2232 pvt->syn_type = 4; 2233 2234 amd64_dump_misc_regs(pvt); 2235} 2236 2237/* 2238 * NOTE: CPU Revision Dependent code 2239 * 2240 * Input: 2241 * @csrow_nr ChipSelect Row Number (0..pvt->cs_count-1) 2242 * k8 private pointer to --> 2243 * DRAM Bank Address mapping register 2244 * node_id 2245 * DCL register where dual_channel_active is 2246 * 2247 * The DBAM register consists of 4 sets of 4 bits each definitions: 2248 * 2249 * Bits: CSROWs 2250 * 0-3 CSROWs 0 and 1 2251 * 4-7 CSROWs 2 and 3 2252 * 8-11 CSROWs 4 and 5 2253 * 12-15 CSROWs 6 and 7 2254 * 2255 * Values range from: 0 to 15 2256 * The meaning of the values depends on CPU revision and dual-channel state, 2257 * see relevant BKDG more info. 2258 * 2259 * The memory controller provides for total of only 8 CSROWs in its current 2260 * architecture. Each "pair" of CSROWs normally represents just one DIMM in 2261 * single channel or two (2) DIMMs in dual channel mode. 2262 * 2263 * The following code logic collapses the various tables for CSROW based on CPU 2264 * revision. 2265 * 2266 * Returns: 2267 * The number of PAGE_SIZE pages on the specified CSROW number it 2268 * encompasses 2269 * 2270 */ 2271static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) 2272{ 2273 u32 cs_mode, nr_pages; 2274 2275 /* 2276 * The math on this doesn't look right on the surface because x/2*4 can 2277 * be simplified to x*2 but this expression makes use of the fact that 2278 * it is integral math where 1/2=0. This intermediate value becomes the 2279 * number of bits to shift the DBAM register to extract the proper CSROW 2280 * field. 2281 */ 2282 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF; 2283 2284 nr_pages = pvt->ops->dbam_to_cs(pvt, cs_mode) << (20 - PAGE_SHIFT); 2285 2286 /* 2287 * If dual channel then double the memory size of single channel. 2288 * Channel count is 1 or 2 2289 */ 2290 nr_pages <<= (pvt->channel_count - 1); 2291 2292 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode); 2293 debugf0(" nr_pages= %u channel-count = %d\n", 2294 nr_pages, pvt->channel_count); 2295 2296 return nr_pages; 2297} 2298 2299/* 2300 * Initialize the array of csrow attribute instances, based on the values 2301 * from pci config hardware registers. 2302 */ 2303static int amd64_init_csrows(struct mem_ctl_info *mci) 2304{ 2305 struct csrow_info *csrow; 2306 struct amd64_pvt *pvt; 2307 u64 input_addr_min, input_addr_max, sys_addr; 2308 int i, empty = 1; 2309 2310 pvt = mci->pvt_info; 2311 2312 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &pvt->nbcfg); 2313 2314 debugf0("NBCFG= 0x%x CHIPKILL= %s DRAM ECC= %s\n", pvt->nbcfg, 2315 (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", 2316 (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled" 2317 ); 2318 2319 for (i = 0; i < pvt->cs_count; i++) { 2320 csrow = &mci->csrows[i]; 2321 2322 if ((pvt->dcsb0[i] & K8_DCSB_CS_ENABLE) == 0) { 2323 debugf1("----CSROW %d EMPTY for node %d\n", i, 2324 pvt->mc_node_id); 2325 continue; 2326 } 2327 2328 debugf1("----CSROW %d VALID for MC node %d\n", 2329 i, pvt->mc_node_id); 2330 2331 empty = 0; 2332 csrow->nr_pages = amd64_csrow_nr_pages(i, pvt); 2333 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max); 2334 sys_addr = input_addr_to_sys_addr(mci, input_addr_min); 2335 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT); 2336 sys_addr = input_addr_to_sys_addr(mci, input_addr_max); 2337 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT); 2338 csrow->page_mask = ~mask_from_dct_mask(pvt, i); 2339 /* 8 bytes of resolution */ 2340 2341 csrow->mtype = amd64_determine_memory_type(pvt); 2342 2343 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i); 2344 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n", 2345 (unsigned long)input_addr_min, 2346 (unsigned long)input_addr_max); 2347 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n", 2348 (unsigned long)sys_addr, csrow->page_mask); 2349 debugf1(" nr_pages: %u first_page: 0x%lx " 2350 "last_page: 0x%lx\n", 2351 (unsigned)csrow->nr_pages, 2352 csrow->first_page, csrow->last_page); 2353 2354 /* 2355 * determine whether CHIPKILL or JUST ECC or NO ECC is operating 2356 */ 2357 if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) 2358 csrow->edac_mode = 2359 (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? 2360 EDAC_S4ECD4ED : EDAC_SECDED; 2361 else 2362 csrow->edac_mode = EDAC_NONE; 2363 } 2364 2365 return empty; 2366} 2367 2368/* get all cores on this DCT */ 2369static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid) 2370{ 2371 int cpu; 2372 2373 for_each_online_cpu(cpu) 2374 if (amd_get_nb_id(cpu) == nid) 2375 cpumask_set_cpu(cpu, mask); 2376} 2377 2378/* check MCG_CTL on all the cpus on this node */ 2379static bool amd64_nb_mce_bank_enabled_on_node(int nid) 2380{ 2381 cpumask_var_t mask; 2382 int cpu, nbe; 2383 bool ret = false; 2384 2385 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 2386 amd64_printk(KERN_WARNING, "%s: error allocating mask\n", 2387 __func__); 2388 return false; 2389 } 2390 2391 get_cpus_on_this_dct_cpumask(mask, nid); 2392 2393 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 2394 2395 for_each_cpu(cpu, mask) { 2396 struct msr *reg = per_cpu_ptr(msrs, cpu); 2397 nbe = reg->l & K8_MSR_MCGCTL_NBE; 2398 2399 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 2400 cpu, reg->q, 2401 (nbe ? "enabled" : "disabled")); 2402 2403 if (!nbe) 2404 goto out; 2405 } 2406 ret = true; 2407 2408out: 2409 free_cpumask_var(mask); 2410 return ret; 2411} 2412 2413static int amd64_toggle_ecc_err_reporting(struct amd64_pvt *pvt, bool on) 2414{ 2415 cpumask_var_t cmask; 2416 int cpu; 2417 2418 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 2419 amd64_printk(KERN_WARNING, "%s: error allocating mask\n", 2420 __func__); 2421 return false; 2422 } 2423 2424 get_cpus_on_this_dct_cpumask(cmask, pvt->mc_node_id); 2425 2426 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2427 2428 for_each_cpu(cpu, cmask) { 2429 2430 struct msr *reg = per_cpu_ptr(msrs, cpu); 2431 2432 if (on) { 2433 if (reg->l & K8_MSR_MCGCTL_NBE) 2434 pvt->flags.nb_mce_enable = 1; 2435 2436 reg->l |= K8_MSR_MCGCTL_NBE; 2437 } else { 2438 /* 2439 * Turn off NB MCE reporting only when it was off before 2440 */ 2441 if (!pvt->flags.nb_mce_enable) 2442 reg->l &= ~K8_MSR_MCGCTL_NBE; 2443 } 2444 } 2445 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2446 2447 free_cpumask_var(cmask); 2448 2449 return 0; 2450} 2451 2452static void amd64_enable_ecc_error_reporting(struct mem_ctl_info *mci) 2453{ 2454 struct amd64_pvt *pvt = mci->pvt_info; 2455 u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; 2456 2457 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCTL, &value); 2458 2459 /* turn on UECCn and CECCEn bits */ 2460 pvt->old_nbctl = value & mask; 2461 pvt->nbctl_mcgctl_saved = 1; 2462 2463 value |= mask; 2464 pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value); 2465 2466 if (amd64_toggle_ecc_err_reporting(pvt, ON)) 2467 amd64_printk(KERN_WARNING, "Error enabling ECC reporting over " 2468 "MCGCTL!\n"); 2469 2470 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value); 2471 2472 debugf0("NBCFG(1)= 0x%x CHIPKILL= %s ECC_ENABLE= %s\n", value, 2473 (value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", 2474 (value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"); 2475 2476 if (!(value & K8_NBCFG_ECC_ENABLE)) { 2477 amd64_printk(KERN_WARNING, 2478 "This node reports that DRAM ECC is " 2479 "currently Disabled; ENABLING now\n"); 2480 2481 pvt->flags.nb_ecc_prev = 0; 2482 2483 /* Attempt to turn on DRAM ECC Enable */ 2484 value |= K8_NBCFG_ECC_ENABLE; 2485 pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCFG, value); 2486 2487 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value); 2488 2489 if (!(value & K8_NBCFG_ECC_ENABLE)) { 2490 amd64_printk(KERN_WARNING, 2491 "Hardware rejects Enabling DRAM ECC checking\n" 2492 "Check memory DIMM configuration\n"); 2493 } else { 2494 amd64_printk(KERN_DEBUG, 2495 "Hardware accepted DRAM ECC Enable\n"); 2496 } 2497 } else { 2498 pvt->flags.nb_ecc_prev = 1; 2499 } 2500 2501 debugf0("NBCFG(2)= 0x%x CHIPKILL= %s ECC_ENABLE= %s\n", value, 2502 (value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", 2503 (value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"); 2504 2505 pvt->ctl_error_info.nbcfg = value; 2506} 2507 2508static void amd64_restore_ecc_error_reporting(struct amd64_pvt *pvt) 2509{ 2510 u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; 2511 2512 if (!pvt->nbctl_mcgctl_saved) 2513 return; 2514 2515 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCTL, &value); 2516 value &= ~mask; 2517 value |= pvt->old_nbctl; 2518 2519 pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value); 2520 2521 /* restore previous BIOS DRAM ECC "off" setting which we force-enabled */ 2522 if (!pvt->flags.nb_ecc_prev) { 2523 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value); 2524 value &= ~K8_NBCFG_ECC_ENABLE; 2525 pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCFG, value); 2526 } 2527 2528 /* restore the NB Enable MCGCTL bit */ 2529 if (amd64_toggle_ecc_err_reporting(pvt, OFF)) 2530 amd64_printk(KERN_WARNING, "Error restoring NB MCGCTL settings!\n"); 2531} 2532 2533/* 2534 * EDAC requires that the BIOS have ECC enabled before taking over the 2535 * processing of ECC errors. This is because the BIOS can properly initialize 2536 * the memory system completely. A command line option allows to force-enable 2537 * hardware ECC later in amd64_enable_ecc_error_reporting(). 2538 */ 2539static const char *ecc_msg = 2540 "ECC disabled in the BIOS or no ECC capability, module will not load.\n" 2541 " Either enable ECC checking or force module loading by setting " 2542 "'ecc_enable_override'.\n" 2543 " (Note that use of the override may cause unknown side effects.)\n"; 2544 2545static int amd64_check_ecc_enabled(struct amd64_pvt *pvt) 2546{ 2547 u32 value; 2548 u8 ecc_enabled = 0; 2549 bool nb_mce_en = false; 2550 2551 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value); 2552 2553 ecc_enabled = !!(value & K8_NBCFG_ECC_ENABLE); 2554 if (!ecc_enabled) 2555 amd64_printk(KERN_NOTICE, "This node reports that Memory ECC " 2556 "is currently disabled, set F3x%x[22] (%s).\n", 2557 K8_NBCFG, pci_name(pvt->misc_f3_ctl)); 2558 else 2559 amd64_printk(KERN_INFO, "ECC is enabled by BIOS.\n"); 2560 2561 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(pvt->mc_node_id); 2562 if (!nb_mce_en) 2563 amd64_printk(KERN_NOTICE, "NB MCE bank disabled, set MSR " 2564 "0x%08x[4] on node %d to enable.\n", 2565 MSR_IA32_MCG_CTL, pvt->mc_node_id); 2566 2567 if (!ecc_enabled || !nb_mce_en) { 2568 if (!ecc_enable_override) { 2569 amd64_printk(KERN_NOTICE, "%s", ecc_msg); 2570 return -ENODEV; 2571 } else { 2572 amd64_printk(KERN_WARNING, "Forcing ECC checking on!\n"); 2573 } 2574 } 2575 2576 return 0; 2577} 2578 2579struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) + 2580 ARRAY_SIZE(amd64_inj_attrs) + 2581 1]; 2582 2583struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } }; 2584 2585static void amd64_set_mc_sysfs_attributes(struct mem_ctl_info *mci) 2586{ 2587 unsigned int i = 0, j = 0; 2588 2589 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++) 2590 sysfs_attrs[i] = amd64_dbg_attrs[i]; 2591 2592 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++) 2593 sysfs_attrs[i] = amd64_inj_attrs[j]; 2594 2595 sysfs_attrs[i] = terminator; 2596 2597 mci->mc_driver_sysfs_attributes = sysfs_attrs; 2598} 2599 2600static void amd64_setup_mci_misc_attributes(struct mem_ctl_info *mci) 2601{ 2602 struct amd64_pvt *pvt = mci->pvt_info; 2603 2604 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 2605 mci->edac_ctl_cap = EDAC_FLAG_NONE; 2606 2607 if (pvt->nbcap & K8_NBCAP_SECDED) 2608 mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 2609 2610 if (pvt->nbcap & K8_NBCAP_CHIPKILL) 2611 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 2612 2613 mci->edac_cap = amd64_determine_edac_cap(pvt); 2614 mci->mod_name = EDAC_MOD_STR; 2615 mci->mod_ver = EDAC_AMD64_VERSION; 2616 mci->ctl_name = get_amd_family_name(pvt->mc_type_index); 2617 mci->dev_name = pci_name(pvt->dram_f2_ctl); 2618 mci->ctl_page_to_phys = NULL; 2619 2620 /* memory scrubber interface */ 2621 mci->set_sdram_scrub_rate = amd64_set_scrub_rate; 2622 mci->get_sdram_scrub_rate = amd64_get_scrub_rate; 2623} 2624 2625/* 2626 * Init stuff for this DRAM Controller device. 2627 * 2628 * Due to a hardware feature on Fam10h CPUs, the Enable Extended Configuration 2629 * Space feature MUST be enabled on ALL Processors prior to actually reading 2630 * from the ECS registers. Since the loading of the module can occur on any 2631 * 'core', and cores don't 'see' all the other processors ECS data when the 2632 * others are NOT enabled. Our solution is to first enable ECS access in this 2633 * routine on all processors, gather some data in a amd64_pvt structure and 2634 * later come back in a finish-setup function to perform that final 2635 * initialization. See also amd64_init_2nd_stage() for that. 2636 */ 2637static int amd64_probe_one_instance(struct pci_dev *dram_f2_ctl, 2638 int mc_type_index) 2639{ 2640 struct amd64_pvt *pvt = NULL; 2641 int err = 0, ret; 2642 2643 ret = -ENOMEM; 2644 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 2645 if (!pvt) 2646 goto err_exit; 2647 2648 pvt->mc_node_id = get_node_id(dram_f2_ctl); 2649 2650 pvt->dram_f2_ctl = dram_f2_ctl; 2651 pvt->ext_model = boot_cpu_data.x86_model >> 4; 2652 pvt->mc_type_index = mc_type_index; 2653 pvt->ops = family_ops(mc_type_index); 2654 2655 /* 2656 * We have the dram_f2_ctl device as an argument, now go reserve its 2657 * sibling devices from the PCI system. 2658 */ 2659 ret = -ENODEV; 2660 err = amd64_reserve_mc_sibling_devices(pvt, mc_type_index); 2661 if (err) 2662 goto err_free; 2663 2664 ret = -EINVAL; 2665 err = amd64_check_ecc_enabled(pvt); 2666 if (err) 2667 goto err_put; 2668 2669 /* 2670 * Key operation here: setup of HW prior to performing ops on it. Some 2671 * setup is required to access ECS data. After this is performed, the 2672 * 'teardown' function must be called upon error and normal exit paths. 2673 */ 2674 if (boot_cpu_data.x86 >= 0x10) 2675 amd64_setup(pvt); 2676 2677 /* 2678 * Save the pointer to the private data for use in 2nd initialization 2679 * stage 2680 */ 2681 pvt_lookup[pvt->mc_node_id] = pvt; 2682 2683 return 0; 2684 2685err_put: 2686 amd64_free_mc_sibling_devices(pvt); 2687 2688err_free: 2689 kfree(pvt); 2690 2691err_exit: 2692 return ret; 2693} 2694 2695/* 2696 * This is the finishing stage of the init code. Needs to be performed after all 2697 * MCs' hardware have been prepped for accessing extended config space. 2698 */ 2699static int amd64_init_2nd_stage(struct amd64_pvt *pvt) 2700{ 2701 int node_id = pvt->mc_node_id; 2702 struct mem_ctl_info *mci; 2703 int ret = -ENODEV; 2704 2705 amd64_read_mc_registers(pvt); 2706 2707 /* 2708 * We need to determine how many memory channels there are. Then use 2709 * that information for calculating the size of the dynamic instance 2710 * tables in the 'mci' structure 2711 */ 2712 pvt->channel_count = pvt->ops->early_channel_count(pvt); 2713 if (pvt->channel_count < 0) 2714 goto err_exit; 2715 2716 ret = -ENOMEM; 2717 mci = edac_mc_alloc(0, pvt->cs_count, pvt->channel_count, node_id); 2718 if (!mci) 2719 goto err_exit; 2720 2721 mci->pvt_info = pvt; 2722 2723 mci->dev = &pvt->dram_f2_ctl->dev; 2724 amd64_setup_mci_misc_attributes(mci); 2725 2726 if (amd64_init_csrows(mci)) 2727 mci->edac_cap = EDAC_FLAG_NONE; 2728 2729 amd64_enable_ecc_error_reporting(mci); 2730 amd64_set_mc_sysfs_attributes(mci); 2731 2732 ret = -ENODEV; 2733 if (edac_mc_add_mc(mci)) { 2734 debugf1("failed edac_mc_add_mc()\n"); 2735 goto err_add_mc; 2736 } 2737 2738 mci_lookup[node_id] = mci; 2739 pvt_lookup[node_id] = NULL; 2740 2741 /* register stuff with EDAC MCE */ 2742 if (report_gart_errors) 2743 amd_report_gart_errors(true); 2744 2745 amd_register_ecc_decoder(amd64_decode_bus_error); 2746 2747 return 0; 2748 2749err_add_mc: 2750 edac_mc_free(mci); 2751 2752err_exit: 2753 debugf0("failure to init 2nd stage: ret=%d\n", ret); 2754 2755 amd64_restore_ecc_error_reporting(pvt); 2756 2757 if (boot_cpu_data.x86 > 0xf) 2758 amd64_teardown(pvt); 2759 2760 amd64_free_mc_sibling_devices(pvt); 2761 2762 kfree(pvt_lookup[pvt->mc_node_id]); 2763 pvt_lookup[node_id] = NULL; 2764 2765 return ret; 2766} 2767 2768 2769static int __devinit amd64_init_one_instance(struct pci_dev *pdev, 2770 const struct pci_device_id *mc_type) 2771{ 2772 int ret = 0; 2773 2774 debugf0("(MC node=%d,mc_type='%s')\n", get_node_id(pdev), 2775 get_amd_family_name(mc_type->driver_data)); 2776 2777 ret = pci_enable_device(pdev); 2778 if (ret < 0) 2779 ret = -EIO; 2780 else 2781 ret = amd64_probe_one_instance(pdev, mc_type->driver_data); 2782 2783 if (ret < 0) 2784 debugf0("ret=%d\n", ret); 2785 2786 return ret; 2787} 2788 2789static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) 2790{ 2791 struct mem_ctl_info *mci; 2792 struct amd64_pvt *pvt; 2793 2794 /* Remove from EDAC CORE tracking list */ 2795 mci = edac_mc_del_mc(&pdev->dev); 2796 if (!mci) 2797 return; 2798 2799 pvt = mci->pvt_info; 2800 2801 amd64_restore_ecc_error_reporting(pvt); 2802 2803 if (boot_cpu_data.x86 > 0xf) 2804 amd64_teardown(pvt); 2805 2806 amd64_free_mc_sibling_devices(pvt); 2807 2808 /* unregister from EDAC MCE */ 2809 amd_report_gart_errors(false); 2810 amd_unregister_ecc_decoder(amd64_decode_bus_error); 2811 2812 /* Free the EDAC CORE resources */ 2813 mci->pvt_info = NULL; 2814 mci_lookup[pvt->mc_node_id] = NULL; 2815 2816 kfree(pvt); 2817 edac_mc_free(mci); 2818} 2819 2820/* 2821 * This table is part of the interface for loading drivers for PCI devices. The 2822 * PCI core identifies what devices are on a system during boot, and then 2823 * inquiry this table to see if this driver is for a given device found. 2824 */ 2825static const struct pci_device_id amd64_pci_table[] __devinitdata = { 2826 { 2827 .vendor = PCI_VENDOR_ID_AMD, 2828 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, 2829 .subvendor = PCI_ANY_ID, 2830 .subdevice = PCI_ANY_ID, 2831 .class = 0, 2832 .class_mask = 0, 2833 .driver_data = K8_CPUS 2834 }, 2835 { 2836 .vendor = PCI_VENDOR_ID_AMD, 2837 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM, 2838 .subvendor = PCI_ANY_ID, 2839 .subdevice = PCI_ANY_ID, 2840 .class = 0, 2841 .class_mask = 0, 2842 .driver_data = F10_CPUS 2843 }, 2844 { 2845 .vendor = PCI_VENDOR_ID_AMD, 2846 .device = PCI_DEVICE_ID_AMD_11H_NB_DRAM, 2847 .subvendor = PCI_ANY_ID, 2848 .subdevice = PCI_ANY_ID, 2849 .class = 0, 2850 .class_mask = 0, 2851 .driver_data = F11_CPUS 2852 }, 2853 {0, } 2854}; 2855MODULE_DEVICE_TABLE(pci, amd64_pci_table); 2856 2857static struct pci_driver amd64_pci_driver = { 2858 .name = EDAC_MOD_STR, 2859 .probe = amd64_init_one_instance, 2860 .remove = __devexit_p(amd64_remove_one_instance), 2861 .id_table = amd64_pci_table, 2862}; 2863 2864static void amd64_setup_pci_device(void) 2865{ 2866 struct mem_ctl_info *mci; 2867 struct amd64_pvt *pvt; 2868 2869 if (amd64_ctl_pci) 2870 return; 2871 2872 mci = mci_lookup[0]; 2873 if (mci) { 2874 2875 pvt = mci->pvt_info; 2876 amd64_ctl_pci = 2877 edac_pci_create_generic_ctl(&pvt->dram_f2_ctl->dev, 2878 EDAC_MOD_STR); 2879 2880 if (!amd64_ctl_pci) { 2881 pr_warning("%s(): Unable to create PCI control\n", 2882 __func__); 2883 2884 pr_warning("%s(): PCI error report via EDAC not set\n", 2885 __func__); 2886 } 2887 } 2888} 2889 2890static int __init amd64_edac_init(void) 2891{ 2892 int nb, err = -ENODEV; 2893 bool load_ok = false; 2894 2895 edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n"); 2896 2897 opstate_init(); 2898 2899 if (cache_k8_northbridges() < 0) 2900 goto err_ret; 2901 2902 msrs = msrs_alloc(); 2903 if (!msrs) 2904 goto err_ret; 2905 2906 err = pci_register_driver(&amd64_pci_driver); 2907 if (err) 2908 goto err_pci; 2909 2910 /* 2911 * At this point, the array 'pvt_lookup[]' contains pointers to alloc'd 2912 * amd64_pvt structs. These will be used in the 2nd stage init function 2913 * to finish initialization of the MC instances. 2914 */ 2915 err = -ENODEV; 2916 for (nb = 0; nb < num_k8_northbridges; nb++) { 2917 if (!pvt_lookup[nb]) 2918 continue; 2919 2920 err = amd64_init_2nd_stage(pvt_lookup[nb]); 2921 if (err) 2922 goto err_2nd_stage; 2923 2924 load_ok = true; 2925 } 2926 2927 if (load_ok) { 2928 amd64_setup_pci_device(); 2929 return 0; 2930 } 2931 2932err_2nd_stage: 2933 pci_unregister_driver(&amd64_pci_driver); 2934err_pci: 2935 msrs_free(msrs); 2936 msrs = NULL; 2937err_ret: 2938 return err; 2939} 2940 2941static void __exit amd64_edac_exit(void) 2942{ 2943 if (amd64_ctl_pci) 2944 edac_pci_release_generic_ctl(amd64_ctl_pci); 2945 2946 pci_unregister_driver(&amd64_pci_driver); 2947 2948 msrs_free(msrs); 2949 msrs = NULL; 2950} 2951 2952module_init(amd64_edac_init); 2953module_exit(amd64_edac_exit); 2954 2955MODULE_LICENSE("GPL"); 2956MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 2957 "Dave Peterson, Thayne Harbaugh"); 2958MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 2959 EDAC_AMD64_VERSION); 2960 2961module_param(edac_op_state, int, 0444); 2962MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 2963