1/* 2 * (c) 2005-2012 Advanced Micro Devices, Inc. 3 * Your use of this code is subject to the terms and conditions of the 4 * GNU general public license version 2. See "COPYING" or 5 * http://www.gnu.org/licenses/gpl.html 6 * 7 * Written by Jacob Shin - AMD, Inc. 8 * 9 * Maintained by: Borislav Petkov <bp@alien8.de> 10 * 11 * April 2006 12 * - added support for AMD Family 0x10 processors 13 * May 2012 14 * - major scrubbing 15 * 16 * All MC4_MISCi registers are shared between multi-cores 17 */ 18#include <linux/interrupt.h> 19#include <linux/notifier.h> 20#include <linux/kobject.h> 21#include <linux/percpu.h> 22#include <linux/errno.h> 23#include <linux/sched.h> 24#include <linux/sysfs.h> 25#include <linux/slab.h> 26#include <linux/init.h> 27#include <linux/cpu.h> 28#include <linux/smp.h> 29 30#include <asm/amd_nb.h> 31#include <asm/apic.h> 32#include <asm/idle.h> 33#include <asm/mce.h> 34#include <asm/msr.h> 35 36#define NR_BLOCKS 9 37#define THRESHOLD_MAX 0xFFF 38#define INT_TYPE_APIC 0x00020000 39#define MASK_VALID_HI 0x80000000 40#define MASK_CNTP_HI 0x40000000 41#define MASK_LOCKED_HI 0x20000000 42#define MASK_LVTOFF_HI 0x00F00000 43#define MASK_COUNT_EN_HI 0x00080000 44#define MASK_INT_TYPE_HI 0x00060000 45#define MASK_OVERFLOW_HI 0x00010000 46#define MASK_ERR_COUNT_HI 0x00000FFF 47#define MASK_BLKPTR_LO 0xFF000000 48#define MCG_XBLK_ADDR 0xC0000400 49 50static const char * const th_names[] = { 51 "load_store", 52 "insn_fetch", 53 "combined_unit", 54 "", 55 "northbridge", 56 "execution_unit", 57}; 58 59static DEFINE_PER_CPU(struct threshold_bank **, threshold_banks); 60static DEFINE_PER_CPU(unsigned char, bank_map); /* see which banks are on */ 61 62static void amd_threshold_interrupt(void); 63 64/* 65 * CPU Initialization 66 */ 67 68struct thresh_restart { 69 struct threshold_block *b; 70 int reset; 71 int set_lvt_off; 72 int lvt_off; 73 u16 old_limit; 74}; 75 76static inline bool is_shared_bank(int bank) 77{ 78 /* Bank 4 is for northbridge reporting and is thus shared */ 79 return (bank == 4); 80} 81 82static const char *bank4_names(const struct threshold_block *b) 83{ 84 switch (b->address) { 85 /* MSR4_MISC0 */ 86 case 0x00000413: 87 return "dram"; 88 89 case 0xc0000408: 90 return "ht_links"; 91 92 case 0xc0000409: 93 return "l3_cache"; 94 95 default: 96 WARN(1, "Funny MSR: 0x%08x\n", b->address); 97 return ""; 98 } 99}; 100 101 102static bool lvt_interrupt_supported(unsigned int bank, u32 msr_high_bits) 103{ 104 /* 105 * bank 4 supports APIC LVT interrupts implicitly since forever. 106 */ 107 if (bank == 4) 108 return true; 109 110 /* 111 * IntP: interrupt present; if this bit is set, the thresholding 112 * bank can generate APIC LVT interrupts 113 */ 114 return msr_high_bits & BIT(28); 115} 116 117static int lvt_off_valid(struct threshold_block *b, int apic, u32 lo, u32 hi) 118{ 119 int msr = (hi & MASK_LVTOFF_HI) >> 20; 120 121 if (apic < 0) { 122 pr_err(FW_BUG "cpu %d, failed to setup threshold interrupt " 123 "for bank %d, block %d (MSR%08X=0x%x%08x)\n", b->cpu, 124 b->bank, b->block, b->address, hi, lo); 125 return 0; 126 } 127 128 if (apic != msr) { 129 pr_err(FW_BUG "cpu %d, invalid threshold interrupt offset %d " 130 "for bank %d, block %d (MSR%08X=0x%x%08x)\n", 131 b->cpu, apic, b->bank, b->block, b->address, hi, lo); 132 return 0; 133 } 134 135 return 1; 136}; 137 138/* 139 * Called via smp_call_function_single(), must be called with correct 140 * cpu affinity. 141 */ 142static void threshold_restart_bank(void *_tr) 143{ 144 struct thresh_restart *tr = _tr; 145 u32 hi, lo; 146 147 rdmsr(tr->b->address, lo, hi); 148 149 if (tr->b->threshold_limit < (hi & THRESHOLD_MAX)) 150 tr->reset = 1; /* limit cannot be lower than err count */ 151 152 if (tr->reset) { /* reset err count and overflow bit */ 153 hi = 154 (hi & ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI)) | 155 (THRESHOLD_MAX - tr->b->threshold_limit); 156 } else if (tr->old_limit) { /* change limit w/o reset */ 157 int new_count = (hi & THRESHOLD_MAX) + 158 (tr->old_limit - tr->b->threshold_limit); 159 160 hi = (hi & ~MASK_ERR_COUNT_HI) | 161 (new_count & THRESHOLD_MAX); 162 } 163 164 /* clear IntType */ 165 hi &= ~MASK_INT_TYPE_HI; 166 167 if (!tr->b->interrupt_capable) 168 goto done; 169 170 if (tr->set_lvt_off) { 171 if (lvt_off_valid(tr->b, tr->lvt_off, lo, hi)) { 172 /* set new lvt offset */ 173 hi &= ~MASK_LVTOFF_HI; 174 hi |= tr->lvt_off << 20; 175 } 176 } 177 178 if (tr->b->interrupt_enable) 179 hi |= INT_TYPE_APIC; 180 181 done: 182 183 hi |= MASK_COUNT_EN_HI; 184 wrmsr(tr->b->address, lo, hi); 185} 186 187static void mce_threshold_block_init(struct threshold_block *b, int offset) 188{ 189 struct thresh_restart tr = { 190 .b = b, 191 .set_lvt_off = 1, 192 .lvt_off = offset, 193 }; 194 195 b->threshold_limit = THRESHOLD_MAX; 196 threshold_restart_bank(&tr); 197}; 198 199static int setup_APIC_mce(int reserved, int new) 200{ 201 if (reserved < 0 && !setup_APIC_eilvt(new, THRESHOLD_APIC_VECTOR, 202 APIC_EILVT_MSG_FIX, 0)) 203 return new; 204 205 return reserved; 206} 207 208/* cpu init entry point, called from mce.c with preempt off */ 209void mce_amd_feature_init(struct cpuinfo_x86 *c) 210{ 211 struct threshold_block b; 212 unsigned int cpu = smp_processor_id(); 213 u32 low = 0, high = 0, address = 0; 214 unsigned int bank, block; 215 int offset = -1, new; 216 217 for (bank = 0; bank < mca_cfg.banks; ++bank) { 218 for (block = 0; block < NR_BLOCKS; ++block) { 219 if (block == 0) 220 address = MSR_IA32_MCx_MISC(bank); 221 else if (block == 1) { 222 address = (low & MASK_BLKPTR_LO) >> 21; 223 if (!address) 224 break; 225 226 address += MCG_XBLK_ADDR; 227 } else 228 ++address; 229 230 if (rdmsr_safe(address, &low, &high)) 231 break; 232 233 if (!(high & MASK_VALID_HI)) 234 continue; 235 236 if (!(high & MASK_CNTP_HI) || 237 (high & MASK_LOCKED_HI)) 238 continue; 239 240 if (!block) 241 per_cpu(bank_map, cpu) |= (1 << bank); 242 243 memset(&b, 0, sizeof(b)); 244 b.cpu = cpu; 245 b.bank = bank; 246 b.block = block; 247 b.address = address; 248 b.interrupt_capable = lvt_interrupt_supported(bank, high); 249 250 if (!b.interrupt_capable) 251 goto init; 252 253 b.interrupt_enable = 1; 254 new = (high & MASK_LVTOFF_HI) >> 20; 255 offset = setup_APIC_mce(offset, new); 256 257 if ((offset == new) && 258 (mce_threshold_vector != amd_threshold_interrupt)) 259 mce_threshold_vector = amd_threshold_interrupt; 260 261init: 262 mce_threshold_block_init(&b, offset); 263 } 264 } 265} 266 267/* 268 * APIC Interrupt Handler 269 */ 270 271/* 272 * threshold interrupt handler will service THRESHOLD_APIC_VECTOR. 273 * the interrupt goes off when error_count reaches threshold_limit. 274 * the handler will simply log mcelog w/ software defined bank number. 275 */ 276static void amd_threshold_interrupt(void) 277{ 278 u32 low = 0, high = 0, address = 0; 279 int cpu = smp_processor_id(); 280 unsigned int bank, block; 281 struct mce m; 282 283 /* assume first bank caused it */ 284 for (bank = 0; bank < mca_cfg.banks; ++bank) { 285 if (!(per_cpu(bank_map, cpu) & (1 << bank))) 286 continue; 287 for (block = 0; block < NR_BLOCKS; ++block) { 288 if (block == 0) { 289 address = MSR_IA32_MCx_MISC(bank); 290 } else if (block == 1) { 291 address = (low & MASK_BLKPTR_LO) >> 21; 292 if (!address) 293 break; 294 address += MCG_XBLK_ADDR; 295 } else { 296 ++address; 297 } 298 299 if (rdmsr_safe(address, &low, &high)) 300 break; 301 302 if (!(high & MASK_VALID_HI)) { 303 if (block) 304 continue; 305 else 306 break; 307 } 308 309 if (!(high & MASK_CNTP_HI) || 310 (high & MASK_LOCKED_HI)) 311 continue; 312 313 /* 314 * Log the machine check that caused the threshold 315 * event. 316 */ 317 if (high & MASK_OVERFLOW_HI) 318 goto log; 319 } 320 } 321 return; 322 323log: 324 mce_setup(&m); 325 rdmsrl(MSR_IA32_MCx_STATUS(bank), m.status); 326 if (!(m.status & MCI_STATUS_VAL)) 327 return; 328 m.misc = ((u64)high << 32) | low; 329 m.bank = bank; 330 mce_log(&m); 331 332 wrmsrl(MSR_IA32_MCx_STATUS(bank), 0); 333} 334 335/* 336 * Sysfs Interface 337 */ 338 339struct threshold_attr { 340 struct attribute attr; 341 ssize_t (*show) (struct threshold_block *, char *); 342 ssize_t (*store) (struct threshold_block *, const char *, size_t count); 343}; 344 345#define SHOW_FIELDS(name) \ 346static ssize_t show_ ## name(struct threshold_block *b, char *buf) \ 347{ \ 348 return sprintf(buf, "%lu\n", (unsigned long) b->name); \ 349} 350SHOW_FIELDS(interrupt_enable) 351SHOW_FIELDS(threshold_limit) 352 353static ssize_t 354store_interrupt_enable(struct threshold_block *b, const char *buf, size_t size) 355{ 356 struct thresh_restart tr; 357 unsigned long new; 358 359 if (!b->interrupt_capable) 360 return -EINVAL; 361 362 if (kstrtoul(buf, 0, &new) < 0) 363 return -EINVAL; 364 365 b->interrupt_enable = !!new; 366 367 memset(&tr, 0, sizeof(tr)); 368 tr.b = b; 369 370 smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1); 371 372 return size; 373} 374 375static ssize_t 376store_threshold_limit(struct threshold_block *b, const char *buf, size_t size) 377{ 378 struct thresh_restart tr; 379 unsigned long new; 380 381 if (kstrtoul(buf, 0, &new) < 0) 382 return -EINVAL; 383 384 if (new > THRESHOLD_MAX) 385 new = THRESHOLD_MAX; 386 if (new < 1) 387 new = 1; 388 389 memset(&tr, 0, sizeof(tr)); 390 tr.old_limit = b->threshold_limit; 391 b->threshold_limit = new; 392 tr.b = b; 393 394 smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1); 395 396 return size; 397} 398 399static ssize_t show_error_count(struct threshold_block *b, char *buf) 400{ 401 u32 lo, hi; 402 403 rdmsr_on_cpu(b->cpu, b->address, &lo, &hi); 404 405 return sprintf(buf, "%u\n", ((hi & THRESHOLD_MAX) - 406 (THRESHOLD_MAX - b->threshold_limit))); 407} 408 409static struct threshold_attr error_count = { 410 .attr = {.name = __stringify(error_count), .mode = 0444 }, 411 .show = show_error_count, 412}; 413 414#define RW_ATTR(val) \ 415static struct threshold_attr val = { \ 416 .attr = {.name = __stringify(val), .mode = 0644 }, \ 417 .show = show_## val, \ 418 .store = store_## val, \ 419}; 420 421RW_ATTR(interrupt_enable); 422RW_ATTR(threshold_limit); 423 424static struct attribute *default_attrs[] = { 425 &threshold_limit.attr, 426 &error_count.attr, 427 NULL, /* possibly interrupt_enable if supported, see below */ 428 NULL, 429}; 430 431#define to_block(k) container_of(k, struct threshold_block, kobj) 432#define to_attr(a) container_of(a, struct threshold_attr, attr) 433 434static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 435{ 436 struct threshold_block *b = to_block(kobj); 437 struct threshold_attr *a = to_attr(attr); 438 ssize_t ret; 439 440 ret = a->show ? a->show(b, buf) : -EIO; 441 442 return ret; 443} 444 445static ssize_t store(struct kobject *kobj, struct attribute *attr, 446 const char *buf, size_t count) 447{ 448 struct threshold_block *b = to_block(kobj); 449 struct threshold_attr *a = to_attr(attr); 450 ssize_t ret; 451 452 ret = a->store ? a->store(b, buf, count) : -EIO; 453 454 return ret; 455} 456 457static const struct sysfs_ops threshold_ops = { 458 .show = show, 459 .store = store, 460}; 461 462static struct kobj_type threshold_ktype = { 463 .sysfs_ops = &threshold_ops, 464 .default_attrs = default_attrs, 465}; 466 467static int allocate_threshold_blocks(unsigned int cpu, unsigned int bank, 468 unsigned int block, u32 address) 469{ 470 struct threshold_block *b = NULL; 471 u32 low, high; 472 int err; 473 474 if ((bank >= mca_cfg.banks) || (block >= NR_BLOCKS)) 475 return 0; 476 477 if (rdmsr_safe_on_cpu(cpu, address, &low, &high)) 478 return 0; 479 480 if (!(high & MASK_VALID_HI)) { 481 if (block) 482 goto recurse; 483 else 484 return 0; 485 } 486 487 if (!(high & MASK_CNTP_HI) || 488 (high & MASK_LOCKED_HI)) 489 goto recurse; 490 491 b = kzalloc(sizeof(struct threshold_block), GFP_KERNEL); 492 if (!b) 493 return -ENOMEM; 494 495 b->block = block; 496 b->bank = bank; 497 b->cpu = cpu; 498 b->address = address; 499 b->interrupt_enable = 0; 500 b->interrupt_capable = lvt_interrupt_supported(bank, high); 501 b->threshold_limit = THRESHOLD_MAX; 502 503 if (b->interrupt_capable) { 504 threshold_ktype.default_attrs[2] = &interrupt_enable.attr; 505 b->interrupt_enable = 1; 506 } else { 507 threshold_ktype.default_attrs[2] = NULL; 508 } 509 510 INIT_LIST_HEAD(&b->miscj); 511 512 if (per_cpu(threshold_banks, cpu)[bank]->blocks) { 513 list_add(&b->miscj, 514 &per_cpu(threshold_banks, cpu)[bank]->blocks->miscj); 515 } else { 516 per_cpu(threshold_banks, cpu)[bank]->blocks = b; 517 } 518 519 err = kobject_init_and_add(&b->kobj, &threshold_ktype, 520 per_cpu(threshold_banks, cpu)[bank]->kobj, 521 (bank == 4 ? bank4_names(b) : th_names[bank])); 522 if (err) 523 goto out_free; 524recurse: 525 if (!block) { 526 address = (low & MASK_BLKPTR_LO) >> 21; 527 if (!address) 528 return 0; 529 address += MCG_XBLK_ADDR; 530 } else { 531 ++address; 532 } 533 534 err = allocate_threshold_blocks(cpu, bank, ++block, address); 535 if (err) 536 goto out_free; 537 538 if (b) 539 kobject_uevent(&b->kobj, KOBJ_ADD); 540 541 return err; 542 543out_free: 544 if (b) { 545 kobject_put(&b->kobj); 546 list_del(&b->miscj); 547 kfree(b); 548 } 549 return err; 550} 551 552static int __threshold_add_blocks(struct threshold_bank *b) 553{ 554 struct list_head *head = &b->blocks->miscj; 555 struct threshold_block *pos = NULL; 556 struct threshold_block *tmp = NULL; 557 int err = 0; 558 559 err = kobject_add(&b->blocks->kobj, b->kobj, b->blocks->kobj.name); 560 if (err) 561 return err; 562 563 list_for_each_entry_safe(pos, tmp, head, miscj) { 564 565 err = kobject_add(&pos->kobj, b->kobj, pos->kobj.name); 566 if (err) { 567 list_for_each_entry_safe_reverse(pos, tmp, head, miscj) 568 kobject_del(&pos->kobj); 569 570 return err; 571 } 572 } 573 return err; 574} 575 576static int threshold_create_bank(unsigned int cpu, unsigned int bank) 577{ 578 struct device *dev = per_cpu(mce_device, cpu); 579 struct amd_northbridge *nb = NULL; 580 struct threshold_bank *b = NULL; 581 const char *name = th_names[bank]; 582 int err = 0; 583 584 if (is_shared_bank(bank)) { 585 nb = node_to_amd_nb(amd_get_nb_id(cpu)); 586 587 /* threshold descriptor already initialized on this node? */ 588 if (nb && nb->bank4) { 589 /* yes, use it */ 590 b = nb->bank4; 591 err = kobject_add(b->kobj, &dev->kobj, name); 592 if (err) 593 goto out; 594 595 per_cpu(threshold_banks, cpu)[bank] = b; 596 atomic_inc(&b->cpus); 597 598 err = __threshold_add_blocks(b); 599 600 goto out; 601 } 602 } 603 604 b = kzalloc(sizeof(struct threshold_bank), GFP_KERNEL); 605 if (!b) { 606 err = -ENOMEM; 607 goto out; 608 } 609 610 b->kobj = kobject_create_and_add(name, &dev->kobj); 611 if (!b->kobj) { 612 err = -EINVAL; 613 goto out_free; 614 } 615 616 per_cpu(threshold_banks, cpu)[bank] = b; 617 618 if (is_shared_bank(bank)) { 619 atomic_set(&b->cpus, 1); 620 621 /* nb is already initialized, see above */ 622 if (nb) { 623 WARN_ON(nb->bank4); 624 nb->bank4 = b; 625 } 626 } 627 628 err = allocate_threshold_blocks(cpu, bank, 0, MSR_IA32_MCx_MISC(bank)); 629 if (!err) 630 goto out; 631 632 out_free: 633 kfree(b); 634 635 out: 636 return err; 637} 638 639/* create dir/files for all valid threshold banks */ 640static int threshold_create_device(unsigned int cpu) 641{ 642 unsigned int bank; 643 struct threshold_bank **bp; 644 int err = 0; 645 646 bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks, 647 GFP_KERNEL); 648 if (!bp) 649 return -ENOMEM; 650 651 per_cpu(threshold_banks, cpu) = bp; 652 653 for (bank = 0; bank < mca_cfg.banks; ++bank) { 654 if (!(per_cpu(bank_map, cpu) & (1 << bank))) 655 continue; 656 err = threshold_create_bank(cpu, bank); 657 if (err) 658 return err; 659 } 660 661 return err; 662} 663 664static void deallocate_threshold_block(unsigned int cpu, 665 unsigned int bank) 666{ 667 struct threshold_block *pos = NULL; 668 struct threshold_block *tmp = NULL; 669 struct threshold_bank *head = per_cpu(threshold_banks, cpu)[bank]; 670 671 if (!head) 672 return; 673 674 list_for_each_entry_safe(pos, tmp, &head->blocks->miscj, miscj) { 675 kobject_put(&pos->kobj); 676 list_del(&pos->miscj); 677 kfree(pos); 678 } 679 680 kfree(per_cpu(threshold_banks, cpu)[bank]->blocks); 681 per_cpu(threshold_banks, cpu)[bank]->blocks = NULL; 682} 683 684static void __threshold_remove_blocks(struct threshold_bank *b) 685{ 686 struct threshold_block *pos = NULL; 687 struct threshold_block *tmp = NULL; 688 689 kobject_del(b->kobj); 690 691 list_for_each_entry_safe(pos, tmp, &b->blocks->miscj, miscj) 692 kobject_del(&pos->kobj); 693} 694 695static void threshold_remove_bank(unsigned int cpu, int bank) 696{ 697 struct amd_northbridge *nb; 698 struct threshold_bank *b; 699 700 b = per_cpu(threshold_banks, cpu)[bank]; 701 if (!b) 702 return; 703 704 if (!b->blocks) 705 goto free_out; 706 707 if (is_shared_bank(bank)) { 708 if (!atomic_dec_and_test(&b->cpus)) { 709 __threshold_remove_blocks(b); 710 per_cpu(threshold_banks, cpu)[bank] = NULL; 711 return; 712 } else { 713 /* 714 * the last CPU on this node using the shared bank is 715 * going away, remove that bank now. 716 */ 717 nb = node_to_amd_nb(amd_get_nb_id(cpu)); 718 nb->bank4 = NULL; 719 } 720 } 721 722 deallocate_threshold_block(cpu, bank); 723 724free_out: 725 kobject_del(b->kobj); 726 kobject_put(b->kobj); 727 kfree(b); 728 per_cpu(threshold_banks, cpu)[bank] = NULL; 729} 730 731static void threshold_remove_device(unsigned int cpu) 732{ 733 unsigned int bank; 734 735 for (bank = 0; bank < mca_cfg.banks; ++bank) { 736 if (!(per_cpu(bank_map, cpu) & (1 << bank))) 737 continue; 738 threshold_remove_bank(cpu, bank); 739 } 740 kfree(per_cpu(threshold_banks, cpu)); 741} 742 743/* get notified when a cpu comes on/off */ 744static void 745amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu) 746{ 747 switch (action) { 748 case CPU_ONLINE: 749 case CPU_ONLINE_FROZEN: 750 threshold_create_device(cpu); 751 break; 752 case CPU_DEAD: 753 case CPU_DEAD_FROZEN: 754 threshold_remove_device(cpu); 755 break; 756 default: 757 break; 758 } 759} 760 761static __init int threshold_init_device(void) 762{ 763 unsigned lcpu = 0; 764 765 /* to hit CPUs online before the notifier is up */ 766 for_each_online_cpu(lcpu) { 767 int err = threshold_create_device(lcpu); 768 769 if (err) 770 return err; 771 } 772 threshold_cpu_callback = amd_64_threshold_cpu_callback; 773 774 return 0; 775} 776/* 777 * there are 3 funcs which need to be _initcalled in a logic sequence: 778 * 1. xen_late_init_mcelog 779 * 2. mcheck_init_device 780 * 3. threshold_init_device 781 * 782 * xen_late_init_mcelog must register xen_mce_chrdev_device before 783 * native mce_chrdev_device registration if running under xen platform; 784 * 785 * mcheck_init_device should be inited before threshold_init_device to 786 * initialize mce_device, otherwise a NULL ptr dereference will cause panic. 787 * 788 * so we use following _initcalls 789 * 1. device_initcall(xen_late_init_mcelog); 790 * 2. device_initcall_sync(mcheck_init_device); 791 * 3. late_initcall(threshold_init_device); 792 * 793 * when running under xen, the initcall order is 1,2,3; 794 * on baremetal, we skip 1 and we do only 2 and 3. 795 */ 796late_initcall(threshold_init_device); 797