1/* 2 * (c) 2005, 2006 Advanced Micro Devices, Inc. 3 * Your use of this code is subject to the terms and conditions of the 4 * GNU general public license version 2. See "COPYING" or 5 * http://www.gnu.org/licenses/gpl.html 6 * 7 * Written by Jacob Shin - AMD, Inc. 8 * 9 * Support : jacob.shin@amd.com 10 * 11 * April 2006 12 * - added support for AMD Family 0x10 processors 13 * 14 * All MC4_MISCi registers are shared between multi-cores 15 */ 16#include <linux/interrupt.h> 17#include <linux/notifier.h> 18#include <linux/kobject.h> 19#include <linux/percpu.h> 20#include <linux/sysdev.h> 21#include <linux/errno.h> 22#include <linux/sched.h> 23#include <linux/sysfs.h> 24#include <linux/slab.h> 25#include <linux/init.h> 26#include <linux/cpu.h> 27#include <linux/smp.h> 28 29#include <asm/apic.h> 30#include <asm/idle.h> 31#include <asm/mce.h> 32#include <asm/msr.h> 33 34#define PFX "mce_threshold: " 35#define VERSION "version 1.1.1" 36#define NR_BANKS 6 37#define NR_BLOCKS 9 38#define THRESHOLD_MAX 0xFFF 39#define INT_TYPE_APIC 0x00020000 40#define MASK_VALID_HI 0x80000000 41#define MASK_CNTP_HI 0x40000000 42#define MASK_LOCKED_HI 0x20000000 43#define MASK_LVTOFF_HI 0x00F00000 44#define MASK_COUNT_EN_HI 0x00080000 45#define MASK_INT_TYPE_HI 0x00060000 46#define MASK_OVERFLOW_HI 0x00010000 47#define MASK_ERR_COUNT_HI 0x00000FFF 48#define MASK_BLKPTR_LO 0xFF000000 49#define MCG_XBLK_ADDR 0xC0000400 50 51struct threshold_block { 52 unsigned int block; 53 unsigned int bank; 54 unsigned int cpu; 55 u32 address; 56 u16 interrupt_enable; 57 u16 threshold_limit; 58 struct kobject kobj; 59 struct list_head miscj; 60}; 61 62/* defaults used early on boot */ 63static struct threshold_block threshold_defaults = { 64 .interrupt_enable = 0, 65 .threshold_limit = THRESHOLD_MAX, 66}; 67 68struct threshold_bank { 69 struct kobject *kobj; 70 struct threshold_block *blocks; 71 cpumask_var_t cpus; 72}; 73static DEFINE_PER_CPU(struct threshold_bank * [NR_BANKS], threshold_banks); 74 75#ifdef CONFIG_SMP 76static unsigned char shared_bank[NR_BANKS] = { 77 0, 0, 0, 0, 1 78}; 79#endif 80 81static DEFINE_PER_CPU(unsigned char, bank_map); /* see which banks are on */ 82 83static void amd_threshold_interrupt(void); 84 85/* 86 * CPU Initialization 87 */ 88 89struct thresh_restart { 90 struct threshold_block *b; 91 int reset; 92 u16 old_limit; 93}; 94 95/* must be called with correct cpu affinity */ 96/* Called via smp_call_function_single() */ 97static void threshold_restart_bank(void *_tr) 98{ 99 struct thresh_restart *tr = _tr; 100 u32 mci_misc_hi, mci_misc_lo; 101 102 rdmsr(tr->b->address, mci_misc_lo, mci_misc_hi); 103 104 if (tr->b->threshold_limit < (mci_misc_hi & THRESHOLD_MAX)) 105 tr->reset = 1; /* limit cannot be lower than err count */ 106 107 if (tr->reset) { /* reset err count and overflow bit */ 108 mci_misc_hi = 109 (mci_misc_hi & ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI)) | 110 (THRESHOLD_MAX - tr->b->threshold_limit); 111 } else if (tr->old_limit) { /* change limit w/o reset */ 112 int new_count = (mci_misc_hi & THRESHOLD_MAX) + 113 (tr->old_limit - tr->b->threshold_limit); 114 115 mci_misc_hi = (mci_misc_hi & ~MASK_ERR_COUNT_HI) | 116 (new_count & THRESHOLD_MAX); 117 } 118 119 tr->b->interrupt_enable ? 120 (mci_misc_hi = (mci_misc_hi & ~MASK_INT_TYPE_HI) | INT_TYPE_APIC) : 121 (mci_misc_hi &= ~MASK_INT_TYPE_HI); 122 123 mci_misc_hi |= MASK_COUNT_EN_HI; 124 wrmsr(tr->b->address, mci_misc_lo, mci_misc_hi); 125} 126 127/* cpu init entry point, called from mce.c with preempt off */ 128void mce_amd_feature_init(struct cpuinfo_x86 *c) 129{ 130 unsigned int cpu = smp_processor_id(); 131 u32 low = 0, high = 0, address = 0; 132 unsigned int bank, block; 133 struct thresh_restart tr; 134 u8 lvt_off; 135 136 for (bank = 0; bank < NR_BANKS; ++bank) { 137 for (block = 0; block < NR_BLOCKS; ++block) { 138 if (block == 0) 139 address = MSR_IA32_MC0_MISC + bank * 4; 140 else if (block == 1) { 141 address = (low & MASK_BLKPTR_LO) >> 21; 142 if (!address) 143 break; 144 145 address += MCG_XBLK_ADDR; 146 } else 147 ++address; 148 149 if (rdmsr_safe(address, &low, &high)) 150 break; 151 152 if (!(high & MASK_VALID_HI)) 153 continue; 154 155 if (!(high & MASK_CNTP_HI) || 156 (high & MASK_LOCKED_HI)) 157 continue; 158 159 if (!block) 160 per_cpu(bank_map, cpu) |= (1 << bank); 161#ifdef CONFIG_SMP 162 if (shared_bank[bank] && c->cpu_core_id) 163 break; 164#endif 165 lvt_off = setup_APIC_eilvt_mce(THRESHOLD_APIC_VECTOR, 166 APIC_EILVT_MSG_FIX, 0); 167 168 high &= ~MASK_LVTOFF_HI; 169 high |= lvt_off << 20; 170 wrmsr(address, low, high); 171 172 threshold_defaults.address = address; 173 tr.b = &threshold_defaults; 174 tr.reset = 0; 175 tr.old_limit = 0; 176 threshold_restart_bank(&tr); 177 178 mce_threshold_vector = amd_threshold_interrupt; 179 } 180 } 181} 182 183/* 184 * APIC Interrupt Handler 185 */ 186 187/* 188 * threshold interrupt handler will service THRESHOLD_APIC_VECTOR. 189 * the interrupt goes off when error_count reaches threshold_limit. 190 * the handler will simply log mcelog w/ software defined bank number. 191 */ 192static void amd_threshold_interrupt(void) 193{ 194 u32 low = 0, high = 0, address = 0; 195 unsigned int bank, block; 196 struct mce m; 197 198 mce_setup(&m); 199 200 /* assume first bank caused it */ 201 for (bank = 0; bank < NR_BANKS; ++bank) { 202 if (!(per_cpu(bank_map, m.cpu) & (1 << bank))) 203 continue; 204 for (block = 0; block < NR_BLOCKS; ++block) { 205 if (block == 0) { 206 address = MSR_IA32_MC0_MISC + bank * 4; 207 } else if (block == 1) { 208 address = (low & MASK_BLKPTR_LO) >> 21; 209 if (!address) 210 break; 211 address += MCG_XBLK_ADDR; 212 } else { 213 ++address; 214 } 215 216 if (rdmsr_safe(address, &low, &high)) 217 break; 218 219 if (!(high & MASK_VALID_HI)) { 220 if (block) 221 continue; 222 else 223 break; 224 } 225 226 if (!(high & MASK_CNTP_HI) || 227 (high & MASK_LOCKED_HI)) 228 continue; 229 230 /* 231 * Log the machine check that caused the threshold 232 * event. 233 */ 234 machine_check_poll(MCP_TIMESTAMP, 235 &__get_cpu_var(mce_poll_banks)); 236 237 if (high & MASK_OVERFLOW_HI) { 238 rdmsrl(address, m.misc); 239 rdmsrl(MSR_IA32_MC0_STATUS + bank * 4, 240 m.status); 241 m.bank = K8_MCE_THRESHOLD_BASE 242 + bank * NR_BLOCKS 243 + block; 244 mce_log(&m); 245 return; 246 } 247 } 248 } 249} 250 251/* 252 * Sysfs Interface 253 */ 254 255struct threshold_attr { 256 struct attribute attr; 257 ssize_t (*show) (struct threshold_block *, char *); 258 ssize_t (*store) (struct threshold_block *, const char *, size_t count); 259}; 260 261#define SHOW_FIELDS(name) \ 262static ssize_t show_ ## name(struct threshold_block *b, char *buf) \ 263{ \ 264 return sprintf(buf, "%lx\n", (unsigned long) b->name); \ 265} 266SHOW_FIELDS(interrupt_enable) 267SHOW_FIELDS(threshold_limit) 268 269static ssize_t 270store_interrupt_enable(struct threshold_block *b, const char *buf, size_t size) 271{ 272 struct thresh_restart tr; 273 unsigned long new; 274 275 if (strict_strtoul(buf, 0, &new) < 0) 276 return -EINVAL; 277 278 b->interrupt_enable = !!new; 279 280 tr.b = b; 281 tr.reset = 0; 282 tr.old_limit = 0; 283 284 smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1); 285 286 return size; 287} 288 289static ssize_t 290store_threshold_limit(struct threshold_block *b, const char *buf, size_t size) 291{ 292 struct thresh_restart tr; 293 unsigned long new; 294 295 if (strict_strtoul(buf, 0, &new) < 0) 296 return -EINVAL; 297 298 if (new > THRESHOLD_MAX) 299 new = THRESHOLD_MAX; 300 if (new < 1) 301 new = 1; 302 303 tr.old_limit = b->threshold_limit; 304 b->threshold_limit = new; 305 tr.b = b; 306 tr.reset = 0; 307 308 smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1); 309 310 return size; 311} 312 313struct threshold_block_cross_cpu { 314 struct threshold_block *tb; 315 long retval; 316}; 317 318static void local_error_count_handler(void *_tbcc) 319{ 320 struct threshold_block_cross_cpu *tbcc = _tbcc; 321 struct threshold_block *b = tbcc->tb; 322 u32 low, high; 323 324 rdmsr(b->address, low, high); 325 tbcc->retval = (high & 0xFFF) - (THRESHOLD_MAX - b->threshold_limit); 326} 327 328static ssize_t show_error_count(struct threshold_block *b, char *buf) 329{ 330 struct threshold_block_cross_cpu tbcc = { .tb = b, }; 331 332 smp_call_function_single(b->cpu, local_error_count_handler, &tbcc, 1); 333 return sprintf(buf, "%lx\n", tbcc.retval); 334} 335 336static ssize_t store_error_count(struct threshold_block *b, 337 const char *buf, size_t count) 338{ 339 struct thresh_restart tr = { .b = b, .reset = 1, .old_limit = 0 }; 340 341 smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1); 342 return 1; 343} 344 345#define RW_ATTR(val) \ 346static struct threshold_attr val = { \ 347 .attr = {.name = __stringify(val), .mode = 0644 }, \ 348 .show = show_## val, \ 349 .store = store_## val, \ 350}; 351 352RW_ATTR(interrupt_enable); 353RW_ATTR(threshold_limit); 354RW_ATTR(error_count); 355 356static struct attribute *default_attrs[] = { 357 &interrupt_enable.attr, 358 &threshold_limit.attr, 359 &error_count.attr, 360 NULL 361}; 362 363#define to_block(k) container_of(k, struct threshold_block, kobj) 364#define to_attr(a) container_of(a, struct threshold_attr, attr) 365 366static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 367{ 368 struct threshold_block *b = to_block(kobj); 369 struct threshold_attr *a = to_attr(attr); 370 ssize_t ret; 371 372 ret = a->show ? a->show(b, buf) : -EIO; 373 374 return ret; 375} 376 377static ssize_t store(struct kobject *kobj, struct attribute *attr, 378 const char *buf, size_t count) 379{ 380 struct threshold_block *b = to_block(kobj); 381 struct threshold_attr *a = to_attr(attr); 382 ssize_t ret; 383 384 ret = a->store ? a->store(b, buf, count) : -EIO; 385 386 return ret; 387} 388 389static const struct sysfs_ops threshold_ops = { 390 .show = show, 391 .store = store, 392}; 393 394static struct kobj_type threshold_ktype = { 395 .sysfs_ops = &threshold_ops, 396 .default_attrs = default_attrs, 397}; 398 399static __cpuinit int allocate_threshold_blocks(unsigned int cpu, 400 unsigned int bank, 401 unsigned int block, 402 u32 address) 403{ 404 struct threshold_block *b = NULL; 405 u32 low, high; 406 int err; 407 408 if ((bank >= NR_BANKS) || (block >= NR_BLOCKS)) 409 return 0; 410 411 if (rdmsr_safe_on_cpu(cpu, address, &low, &high)) 412 return 0; 413 414 if (!(high & MASK_VALID_HI)) { 415 if (block) 416 goto recurse; 417 else 418 return 0; 419 } 420 421 if (!(high & MASK_CNTP_HI) || 422 (high & MASK_LOCKED_HI)) 423 goto recurse; 424 425 b = kzalloc(sizeof(struct threshold_block), GFP_KERNEL); 426 if (!b) 427 return -ENOMEM; 428 429 b->block = block; 430 b->bank = bank; 431 b->cpu = cpu; 432 b->address = address; 433 b->interrupt_enable = 0; 434 b->threshold_limit = THRESHOLD_MAX; 435 436 INIT_LIST_HEAD(&b->miscj); 437 438 if (per_cpu(threshold_banks, cpu)[bank]->blocks) { 439 list_add(&b->miscj, 440 &per_cpu(threshold_banks, cpu)[bank]->blocks->miscj); 441 } else { 442 per_cpu(threshold_banks, cpu)[bank]->blocks = b; 443 } 444 445 err = kobject_init_and_add(&b->kobj, &threshold_ktype, 446 per_cpu(threshold_banks, cpu)[bank]->kobj, 447 "misc%i", block); 448 if (err) 449 goto out_free; 450recurse: 451 if (!block) { 452 address = (low & MASK_BLKPTR_LO) >> 21; 453 if (!address) 454 return 0; 455 address += MCG_XBLK_ADDR; 456 } else { 457 ++address; 458 } 459 460 err = allocate_threshold_blocks(cpu, bank, ++block, address); 461 if (err) 462 goto out_free; 463 464 if (b) 465 kobject_uevent(&b->kobj, KOBJ_ADD); 466 467 return err; 468 469out_free: 470 if (b) { 471 kobject_put(&b->kobj); 472 kfree(b); 473 } 474 return err; 475} 476 477static __cpuinit long 478local_allocate_threshold_blocks(int cpu, unsigned int bank) 479{ 480 return allocate_threshold_blocks(cpu, bank, 0, 481 MSR_IA32_MC0_MISC + bank * 4); 482} 483 484/* symlinks sibling shared banks to first core. first core owns dir/files. */ 485static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) 486{ 487 int i, err = 0; 488 struct threshold_bank *b = NULL; 489 char name[32]; 490#ifdef CONFIG_SMP 491 struct cpuinfo_x86 *c = &cpu_data(cpu); 492#endif 493 494 sprintf(name, "threshold_bank%i", bank); 495 496#ifdef CONFIG_SMP 497 if (cpu_data(cpu).cpu_core_id && shared_bank[bank]) { /* symlink */ 498 i = cpumask_first(c->llc_shared_map); 499 500 /* first core not up yet */ 501 if (cpu_data(i).cpu_core_id) 502 goto out; 503 504 /* already linked */ 505 if (per_cpu(threshold_banks, cpu)[bank]) 506 goto out; 507 508 b = per_cpu(threshold_banks, i)[bank]; 509 510 if (!b) 511 goto out; 512 513 err = sysfs_create_link(&per_cpu(mce_dev, cpu).kobj, 514 b->kobj, name); 515 if (err) 516 goto out; 517 518 cpumask_copy(b->cpus, c->llc_shared_map); 519 per_cpu(threshold_banks, cpu)[bank] = b; 520 521 goto out; 522 } 523#endif 524 525 b = kzalloc(sizeof(struct threshold_bank), GFP_KERNEL); 526 if (!b) { 527 err = -ENOMEM; 528 goto out; 529 } 530 if (!zalloc_cpumask_var(&b->cpus, GFP_KERNEL)) { 531 kfree(b); 532 err = -ENOMEM; 533 goto out; 534 } 535 536 b->kobj = kobject_create_and_add(name, &per_cpu(mce_dev, cpu).kobj); 537 if (!b->kobj) 538 goto out_free; 539 540#ifndef CONFIG_SMP 541 cpumask_setall(b->cpus); 542#else 543 cpumask_set_cpu(cpu, b->cpus); 544#endif 545 546 per_cpu(threshold_banks, cpu)[bank] = b; 547 548 err = local_allocate_threshold_blocks(cpu, bank); 549 if (err) 550 goto out_free; 551 552 for_each_cpu(i, b->cpus) { 553 if (i == cpu) 554 continue; 555 556 err = sysfs_create_link(&per_cpu(mce_dev, i).kobj, 557 b->kobj, name); 558 if (err) 559 goto out; 560 561 per_cpu(threshold_banks, i)[bank] = b; 562 } 563 564 goto out; 565 566out_free: 567 per_cpu(threshold_banks, cpu)[bank] = NULL; 568 free_cpumask_var(b->cpus); 569 kfree(b); 570out: 571 return err; 572} 573 574/* create dir/files for all valid threshold banks */ 575static __cpuinit int threshold_create_device(unsigned int cpu) 576{ 577 unsigned int bank; 578 int err = 0; 579 580 for (bank = 0; bank < NR_BANKS; ++bank) { 581 if (!(per_cpu(bank_map, cpu) & (1 << bank))) 582 continue; 583 err = threshold_create_bank(cpu, bank); 584 if (err) 585 goto out; 586 } 587out: 588 return err; 589} 590 591/* 592 * let's be hotplug friendly. 593 * in case of multiple core processors, the first core always takes ownership 594 * of shared sysfs dir/files, and rest of the cores will be symlinked to it. 595 */ 596 597static void deallocate_threshold_block(unsigned int cpu, 598 unsigned int bank) 599{ 600 struct threshold_block *pos = NULL; 601 struct threshold_block *tmp = NULL; 602 struct threshold_bank *head = per_cpu(threshold_banks, cpu)[bank]; 603 604 if (!head) 605 return; 606 607 list_for_each_entry_safe(pos, tmp, &head->blocks->miscj, miscj) { 608 kobject_put(&pos->kobj); 609 list_del(&pos->miscj); 610 kfree(pos); 611 } 612 613 kfree(per_cpu(threshold_banks, cpu)[bank]->blocks); 614 per_cpu(threshold_banks, cpu)[bank]->blocks = NULL; 615} 616 617static void threshold_remove_bank(unsigned int cpu, int bank) 618{ 619 struct threshold_bank *b; 620 char name[32]; 621 int i = 0; 622 623 b = per_cpu(threshold_banks, cpu)[bank]; 624 if (!b) 625 return; 626 if (!b->blocks) 627 goto free_out; 628 629 sprintf(name, "threshold_bank%i", bank); 630 631#ifdef CONFIG_SMP 632 /* sibling symlink */ 633 if (shared_bank[bank] && b->blocks->cpu != cpu) { 634 sysfs_remove_link(&per_cpu(mce_dev, cpu).kobj, name); 635 per_cpu(threshold_banks, cpu)[bank] = NULL; 636 637 return; 638 } 639#endif 640 641 /* remove all sibling symlinks before unregistering */ 642 for_each_cpu(i, b->cpus) { 643 if (i == cpu) 644 continue; 645 646 sysfs_remove_link(&per_cpu(mce_dev, i).kobj, name); 647 per_cpu(threshold_banks, i)[bank] = NULL; 648 } 649 650 deallocate_threshold_block(cpu, bank); 651 652free_out: 653 kobject_del(b->kobj); 654 kobject_put(b->kobj); 655 free_cpumask_var(b->cpus); 656 kfree(b); 657 per_cpu(threshold_banks, cpu)[bank] = NULL; 658} 659 660static void threshold_remove_device(unsigned int cpu) 661{ 662 unsigned int bank; 663 664 for (bank = 0; bank < NR_BANKS; ++bank) { 665 if (!(per_cpu(bank_map, cpu) & (1 << bank))) 666 continue; 667 threshold_remove_bank(cpu, bank); 668 } 669} 670 671/* get notified when a cpu comes on/off */ 672static void __cpuinit 673amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu) 674{ 675 switch (action) { 676 case CPU_ONLINE: 677 case CPU_ONLINE_FROZEN: 678 threshold_create_device(cpu); 679 break; 680 case CPU_DEAD: 681 case CPU_DEAD_FROZEN: 682 threshold_remove_device(cpu); 683 break; 684 default: 685 break; 686 } 687} 688 689static __init int threshold_init_device(void) 690{ 691 unsigned lcpu = 0; 692 693 /* to hit CPUs online before the notifier is up */ 694 for_each_online_cpu(lcpu) { 695 int err = threshold_create_device(lcpu); 696 697 if (err) 698 return err; 699 } 700 threshold_cpu_callback = amd_64_threshold_cpu_callback; 701 702 return 0; 703} 704device_initcall(threshold_init_device); 705