1// SPDX-License-Identifier: GPL-2.0-only 2#include <linux/module.h> 3 4#include <asm/cpu_device_id.h> 5#include <asm/intel-family.h> 6#include "uncore.h" 7#include "uncore_discovery.h" 8 9static bool uncore_no_discover; 10module_param(uncore_no_discover, bool, 0); 11MODULE_PARM_DESC(uncore_no_discover, "Don't enable the Intel uncore PerfMon discovery mechanism " 12 "(default: enable the discovery mechanism)."); 13struct intel_uncore_type *empty_uncore[] = { NULL, }; 14struct intel_uncore_type **uncore_msr_uncores = empty_uncore; 15struct intel_uncore_type **uncore_pci_uncores = empty_uncore; 16struct intel_uncore_type **uncore_mmio_uncores = empty_uncore; 17 18static bool pcidrv_registered; 19struct pci_driver *uncore_pci_driver; 20/* The PCI driver for the device which the uncore doesn't own. */ 21struct pci_driver *uncore_pci_sub_driver; 22/* pci bus to socket mapping */ 23DEFINE_RAW_SPINLOCK(pci2phy_map_lock); 24struct list_head pci2phy_map_head = LIST_HEAD_INIT(pci2phy_map_head); 25struct pci_extra_dev *uncore_extra_pci_dev; 26int __uncore_max_dies; 27 28/* mask of cpus that collect uncore events */ 29static cpumask_t uncore_cpu_mask; 30 31/* constraint for the fixed counter */ 32static struct event_constraint uncore_constraint_fixed = 33 EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL); 34struct event_constraint uncore_constraint_empty = 35 EVENT_CONSTRAINT(0, 0, 0); 36 37MODULE_LICENSE("GPL"); 38 39int uncore_pcibus_to_dieid(struct pci_bus *bus) 40{ 41 struct pci2phy_map *map; 42 int die_id = -1; 43 44 raw_spin_lock(&pci2phy_map_lock); 45 list_for_each_entry(map, &pci2phy_map_head, list) { 46 if (map->segment == pci_domain_nr(bus)) { 47 die_id = map->pbus_to_dieid[bus->number]; 48 break; 49 } 50 } 51 raw_spin_unlock(&pci2phy_map_lock); 52 53 return die_id; 54} 55 56int uncore_die_to_segment(int die) 57{ 58 struct pci_bus *bus = NULL; 59 60 /* Find first pci bus which attributes to specified die. */ 61 while ((bus = pci_find_next_bus(bus)) && 62 (die != uncore_pcibus_to_dieid(bus))) 63 ; 64 65 return bus ? pci_domain_nr(bus) : -EINVAL; 66} 67 68int uncore_device_to_die(struct pci_dev *dev) 69{ 70 int node = pcibus_to_node(dev->bus); 71 int cpu; 72 73 for_each_cpu(cpu, cpumask_of_pcibus(dev->bus)) { 74 struct cpuinfo_x86 *c = &cpu_data(cpu); 75 76 if (c->initialized && cpu_to_node(cpu) == node) 77 return c->topo.logical_die_id; 78 } 79 80 return -1; 81} 82 83static void uncore_free_pcibus_map(void) 84{ 85 struct pci2phy_map *map, *tmp; 86 87 list_for_each_entry_safe(map, tmp, &pci2phy_map_head, list) { 88 list_del(&map->list); 89 kfree(map); 90 } 91} 92 93struct pci2phy_map *__find_pci2phy_map(int segment) 94{ 95 struct pci2phy_map *map, *alloc = NULL; 96 int i; 97 98 lockdep_assert_held(&pci2phy_map_lock); 99 100lookup: 101 list_for_each_entry(map, &pci2phy_map_head, list) { 102 if (map->segment == segment) 103 goto end; 104 } 105 106 if (!alloc) { 107 raw_spin_unlock(&pci2phy_map_lock); 108 alloc = kmalloc(sizeof(struct pci2phy_map), GFP_KERNEL); 109 raw_spin_lock(&pci2phy_map_lock); 110 111 if (!alloc) 112 return NULL; 113 114 goto lookup; 115 } 116 117 map = alloc; 118 alloc = NULL; 119 map->segment = segment; 120 for (i = 0; i < 256; i++) 121 map->pbus_to_dieid[i] = -1; 122 list_add_tail(&map->list, &pci2phy_map_head); 123 124end: 125 kfree(alloc); 126 return map; 127} 128 129ssize_t uncore_event_show(struct device *dev, 130 struct device_attribute *attr, char *buf) 131{ 132 struct uncore_event_desc *event = 133 container_of(attr, struct uncore_event_desc, attr); 134 return sprintf(buf, "%s", event->config); 135} 136 137struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu) 138{ 139 unsigned int dieid = topology_logical_die_id(cpu); 140 141 /* 142 * The unsigned check also catches the '-1' return value for non 143 * existent mappings in the topology map. 144 */ 145 return dieid < uncore_max_dies() ? pmu->boxes[dieid] : NULL; 146} 147 148u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event) 149{ 150 u64 count; 151 152 rdmsrl(event->hw.event_base, count); 153 154 return count; 155} 156 157void uncore_mmio_exit_box(struct intel_uncore_box *box) 158{ 159 if (box->io_addr) 160 iounmap(box->io_addr); 161} 162 163u64 uncore_mmio_read_counter(struct intel_uncore_box *box, 164 struct perf_event *event) 165{ 166 if (!box->io_addr) 167 return 0; 168 169 if (!uncore_mmio_is_valid_offset(box, event->hw.event_base)) 170 return 0; 171 172 return readq(box->io_addr + event->hw.event_base); 173} 174 175/* 176 * generic get constraint function for shared match/mask registers. 177 */ 178struct event_constraint * 179uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event) 180{ 181 struct intel_uncore_extra_reg *er; 182 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 183 struct hw_perf_event_extra *reg2 = &event->hw.branch_reg; 184 unsigned long flags; 185 bool ok = false; 186 187 /* 188 * reg->alloc can be set due to existing state, so for fake box we 189 * need to ignore this, otherwise we might fail to allocate proper 190 * fake state for this extra reg constraint. 191 */ 192 if (reg1->idx == EXTRA_REG_NONE || 193 (!uncore_box_is_fake(box) && reg1->alloc)) 194 return NULL; 195 196 er = &box->shared_regs[reg1->idx]; 197 raw_spin_lock_irqsave(&er->lock, flags); 198 if (!atomic_read(&er->ref) || 199 (er->config1 == reg1->config && er->config2 == reg2->config)) { 200 atomic_inc(&er->ref); 201 er->config1 = reg1->config; 202 er->config2 = reg2->config; 203 ok = true; 204 } 205 raw_spin_unlock_irqrestore(&er->lock, flags); 206 207 if (ok) { 208 if (!uncore_box_is_fake(box)) 209 reg1->alloc = 1; 210 return NULL; 211 } 212 213 return &uncore_constraint_empty; 214} 215 216void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event) 217{ 218 struct intel_uncore_extra_reg *er; 219 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 220 221 /* 222 * Only put constraint if extra reg was actually allocated. Also 223 * takes care of event which do not use an extra shared reg. 224 * 225 * Also, if this is a fake box we shouldn't touch any event state 226 * (reg->alloc) and we don't care about leaving inconsistent box 227 * state either since it will be thrown out. 228 */ 229 if (uncore_box_is_fake(box) || !reg1->alloc) 230 return; 231 232 er = &box->shared_regs[reg1->idx]; 233 atomic_dec(&er->ref); 234 reg1->alloc = 0; 235} 236 237u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx) 238{ 239 struct intel_uncore_extra_reg *er; 240 unsigned long flags; 241 u64 config; 242 243 er = &box->shared_regs[idx]; 244 245 raw_spin_lock_irqsave(&er->lock, flags); 246 config = er->config; 247 raw_spin_unlock_irqrestore(&er->lock, flags); 248 249 return config; 250} 251 252static void uncore_assign_hw_event(struct intel_uncore_box *box, 253 struct perf_event *event, int idx) 254{ 255 struct hw_perf_event *hwc = &event->hw; 256 257 hwc->idx = idx; 258 hwc->last_tag = ++box->tags[idx]; 259 260 if (uncore_pmc_fixed(hwc->idx)) { 261 hwc->event_base = uncore_fixed_ctr(box); 262 hwc->config_base = uncore_fixed_ctl(box); 263 return; 264 } 265 266 hwc->config_base = uncore_event_ctl(box, hwc->idx); 267 hwc->event_base = uncore_perf_ctr(box, hwc->idx); 268} 269 270void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event) 271{ 272 u64 prev_count, new_count, delta; 273 int shift; 274 275 if (uncore_pmc_freerunning(event->hw.idx)) 276 shift = 64 - uncore_freerunning_bits(box, event); 277 else if (uncore_pmc_fixed(event->hw.idx)) 278 shift = 64 - uncore_fixed_ctr_bits(box); 279 else 280 shift = 64 - uncore_perf_ctr_bits(box); 281 282 /* the hrtimer might modify the previous event value */ 283again: 284 prev_count = local64_read(&event->hw.prev_count); 285 new_count = uncore_read_counter(box, event); 286 if (local64_xchg(&event->hw.prev_count, new_count) != prev_count) 287 goto again; 288 289 delta = (new_count << shift) - (prev_count << shift); 290 delta >>= shift; 291 292 local64_add(delta, &event->count); 293} 294 295/* 296 * The overflow interrupt is unavailable for SandyBridge-EP, is broken 297 * for SandyBridge. So we use hrtimer to periodically poll the counter 298 * to avoid overflow. 299 */ 300static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer) 301{ 302 struct intel_uncore_box *box; 303 struct perf_event *event; 304 unsigned long flags; 305 int bit; 306 307 box = container_of(hrtimer, struct intel_uncore_box, hrtimer); 308 if (!box->n_active || box->cpu != smp_processor_id()) 309 return HRTIMER_NORESTART; 310 /* 311 * disable local interrupt to prevent uncore_pmu_event_start/stop 312 * to interrupt the update process 313 */ 314 local_irq_save(flags); 315 316 /* 317 * handle boxes with an active event list as opposed to active 318 * counters 319 */ 320 list_for_each_entry(event, &box->active_list, active_entry) { 321 uncore_perf_event_update(box, event); 322 } 323 324 for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX) 325 uncore_perf_event_update(box, box->events[bit]); 326 327 local_irq_restore(flags); 328 329 hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration)); 330 return HRTIMER_RESTART; 331} 332 333void uncore_pmu_start_hrtimer(struct intel_uncore_box *box) 334{ 335 hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration), 336 HRTIMER_MODE_REL_PINNED); 337} 338 339void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box) 340{ 341 hrtimer_cancel(&box->hrtimer); 342} 343 344static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box) 345{ 346 hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 347 box->hrtimer.function = uncore_pmu_hrtimer; 348} 349 350static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, 351 int node) 352{ 353 int i, size, numshared = type->num_shared_regs ; 354 struct intel_uncore_box *box; 355 356 size = sizeof(*box) + numshared * sizeof(struct intel_uncore_extra_reg); 357 358 box = kzalloc_node(size, GFP_KERNEL, node); 359 if (!box) 360 return NULL; 361 362 for (i = 0; i < numshared; i++) 363 raw_spin_lock_init(&box->shared_regs[i].lock); 364 365 uncore_pmu_init_hrtimer(box); 366 box->cpu = -1; 367 box->dieid = -1; 368 369 /* set default hrtimer timeout */ 370 box->hrtimer_duration = UNCORE_PMU_HRTIMER_INTERVAL; 371 372 INIT_LIST_HEAD(&box->active_list); 373 374 return box; 375} 376 377/* 378 * Using uncore_pmu_event_init pmu event_init callback 379 * as a detection point for uncore events. 380 */ 381static int uncore_pmu_event_init(struct perf_event *event); 382 383static bool is_box_event(struct intel_uncore_box *box, struct perf_event *event) 384{ 385 return &box->pmu->pmu == event->pmu; 386} 387 388static int 389uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader, 390 bool dogrp) 391{ 392 struct perf_event *event; 393 int n, max_count; 394 395 max_count = box->pmu->type->num_counters; 396 if (box->pmu->type->fixed_ctl) 397 max_count++; 398 399 if (box->n_events >= max_count) 400 return -EINVAL; 401 402 n = box->n_events; 403 404 if (is_box_event(box, leader)) { 405 box->event_list[n] = leader; 406 n++; 407 } 408 409 if (!dogrp) 410 return n; 411 412 for_each_sibling_event(event, leader) { 413 if (!is_box_event(box, event) || 414 event->state <= PERF_EVENT_STATE_OFF) 415 continue; 416 417 if (n >= max_count) 418 return -EINVAL; 419 420 box->event_list[n] = event; 421 n++; 422 } 423 return n; 424} 425 426static struct event_constraint * 427uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event) 428{ 429 struct intel_uncore_type *type = box->pmu->type; 430 struct event_constraint *c; 431 432 if (type->ops->get_constraint) { 433 c = type->ops->get_constraint(box, event); 434 if (c) 435 return c; 436 } 437 438 if (event->attr.config == UNCORE_FIXED_EVENT) 439 return &uncore_constraint_fixed; 440 441 if (type->constraints) { 442 for_each_event_constraint(c, type->constraints) { 443 if ((event->hw.config & c->cmask) == c->code) 444 return c; 445 } 446 } 447 448 return &type->unconstrainted; 449} 450 451static void uncore_put_event_constraint(struct intel_uncore_box *box, 452 struct perf_event *event) 453{ 454 if (box->pmu->type->ops->put_constraint) 455 box->pmu->type->ops->put_constraint(box, event); 456} 457 458static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n) 459{ 460 unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)]; 461 struct event_constraint *c; 462 int i, wmin, wmax, ret = 0; 463 struct hw_perf_event *hwc; 464 465 bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX); 466 467 for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) { 468 c = uncore_get_event_constraint(box, box->event_list[i]); 469 box->event_constraint[i] = c; 470 wmin = min(wmin, c->weight); 471 wmax = max(wmax, c->weight); 472 } 473 474 /* fastpath, try to reuse previous register */ 475 for (i = 0; i < n; i++) { 476 hwc = &box->event_list[i]->hw; 477 c = box->event_constraint[i]; 478 479 /* never assigned */ 480 if (hwc->idx == -1) 481 break; 482 483 /* constraint still honored */ 484 if (!test_bit(hwc->idx, c->idxmsk)) 485 break; 486 487 /* not already used */ 488 if (test_bit(hwc->idx, used_mask)) 489 break; 490 491 __set_bit(hwc->idx, used_mask); 492 if (assign) 493 assign[i] = hwc->idx; 494 } 495 /* slow path */ 496 if (i != n) 497 ret = perf_assign_events(box->event_constraint, n, 498 wmin, wmax, n, assign); 499 500 if (!assign || ret) { 501 for (i = 0; i < n; i++) 502 uncore_put_event_constraint(box, box->event_list[i]); 503 } 504 return ret ? -EINVAL : 0; 505} 506 507void uncore_pmu_event_start(struct perf_event *event, int flags) 508{ 509 struct intel_uncore_box *box = uncore_event_to_box(event); 510 int idx = event->hw.idx; 511 512 if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX)) 513 return; 514 515 /* 516 * Free running counter is read-only and always active. 517 * Use the current counter value as start point. 518 * There is no overflow interrupt for free running counter. 519 * Use hrtimer to periodically poll the counter to avoid overflow. 520 */ 521 if (uncore_pmc_freerunning(event->hw.idx)) { 522 list_add_tail(&event->active_entry, &box->active_list); 523 local64_set(&event->hw.prev_count, 524 uncore_read_counter(box, event)); 525 if (box->n_active++ == 0) 526 uncore_pmu_start_hrtimer(box); 527 return; 528 } 529 530 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 531 return; 532 533 event->hw.state = 0; 534 box->events[idx] = event; 535 box->n_active++; 536 __set_bit(idx, box->active_mask); 537 538 local64_set(&event->hw.prev_count, uncore_read_counter(box, event)); 539 uncore_enable_event(box, event); 540 541 if (box->n_active == 1) 542 uncore_pmu_start_hrtimer(box); 543} 544 545void uncore_pmu_event_stop(struct perf_event *event, int flags) 546{ 547 struct intel_uncore_box *box = uncore_event_to_box(event); 548 struct hw_perf_event *hwc = &event->hw; 549 550 /* Cannot disable free running counter which is read-only */ 551 if (uncore_pmc_freerunning(hwc->idx)) { 552 list_del(&event->active_entry); 553 if (--box->n_active == 0) 554 uncore_pmu_cancel_hrtimer(box); 555 uncore_perf_event_update(box, event); 556 return; 557 } 558 559 if (__test_and_clear_bit(hwc->idx, box->active_mask)) { 560 uncore_disable_event(box, event); 561 box->n_active--; 562 box->events[hwc->idx] = NULL; 563 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 564 hwc->state |= PERF_HES_STOPPED; 565 566 if (box->n_active == 0) 567 uncore_pmu_cancel_hrtimer(box); 568 } 569 570 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 571 /* 572 * Drain the remaining delta count out of a event 573 * that we are disabling: 574 */ 575 uncore_perf_event_update(box, event); 576 hwc->state |= PERF_HES_UPTODATE; 577 } 578} 579 580int uncore_pmu_event_add(struct perf_event *event, int flags) 581{ 582 struct intel_uncore_box *box = uncore_event_to_box(event); 583 struct hw_perf_event *hwc = &event->hw; 584 int assign[UNCORE_PMC_IDX_MAX]; 585 int i, n, ret; 586 587 if (!box) 588 return -ENODEV; 589 590 /* 591 * The free funning counter is assigned in event_init(). 592 * The free running counter event and free running counter 593 * are 1:1 mapped. It doesn't need to be tracked in event_list. 594 */ 595 if (uncore_pmc_freerunning(hwc->idx)) { 596 if (flags & PERF_EF_START) 597 uncore_pmu_event_start(event, 0); 598 return 0; 599 } 600 601 ret = n = uncore_collect_events(box, event, false); 602 if (ret < 0) 603 return ret; 604 605 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 606 if (!(flags & PERF_EF_START)) 607 hwc->state |= PERF_HES_ARCH; 608 609 ret = uncore_assign_events(box, assign, n); 610 if (ret) 611 return ret; 612 613 /* save events moving to new counters */ 614 for (i = 0; i < box->n_events; i++) { 615 event = box->event_list[i]; 616 hwc = &event->hw; 617 618 if (hwc->idx == assign[i] && 619 hwc->last_tag == box->tags[assign[i]]) 620 continue; 621 /* 622 * Ensure we don't accidentally enable a stopped 623 * counter simply because we rescheduled. 624 */ 625 if (hwc->state & PERF_HES_STOPPED) 626 hwc->state |= PERF_HES_ARCH; 627 628 uncore_pmu_event_stop(event, PERF_EF_UPDATE); 629 } 630 631 /* reprogram moved events into new counters */ 632 for (i = 0; i < n; i++) { 633 event = box->event_list[i]; 634 hwc = &event->hw; 635 636 if (hwc->idx != assign[i] || 637 hwc->last_tag != box->tags[assign[i]]) 638 uncore_assign_hw_event(box, event, assign[i]); 639 else if (i < box->n_events) 640 continue; 641 642 if (hwc->state & PERF_HES_ARCH) 643 continue; 644 645 uncore_pmu_event_start(event, 0); 646 } 647 box->n_events = n; 648 649 return 0; 650} 651 652void uncore_pmu_event_del(struct perf_event *event, int flags) 653{ 654 struct intel_uncore_box *box = uncore_event_to_box(event); 655 int i; 656 657 uncore_pmu_event_stop(event, PERF_EF_UPDATE); 658 659 /* 660 * The event for free running counter is not tracked by event_list. 661 * It doesn't need to force event->hw.idx = -1 to reassign the counter. 662 * Because the event and the free running counter are 1:1 mapped. 663 */ 664 if (uncore_pmc_freerunning(event->hw.idx)) 665 return; 666 667 for (i = 0; i < box->n_events; i++) { 668 if (event == box->event_list[i]) { 669 uncore_put_event_constraint(box, event); 670 671 for (++i; i < box->n_events; i++) 672 box->event_list[i - 1] = box->event_list[i]; 673 674 --box->n_events; 675 break; 676 } 677 } 678 679 event->hw.idx = -1; 680 event->hw.last_tag = ~0ULL; 681} 682 683void uncore_pmu_event_read(struct perf_event *event) 684{ 685 struct intel_uncore_box *box = uncore_event_to_box(event); 686 uncore_perf_event_update(box, event); 687} 688 689/* 690 * validation ensures the group can be loaded onto the 691 * PMU if it was the only group available. 692 */ 693static int uncore_validate_group(struct intel_uncore_pmu *pmu, 694 struct perf_event *event) 695{ 696 struct perf_event *leader = event->group_leader; 697 struct intel_uncore_box *fake_box; 698 int ret = -EINVAL, n; 699 700 /* The free running counter is always active. */ 701 if (uncore_pmc_freerunning(event->hw.idx)) 702 return 0; 703 704 fake_box = uncore_alloc_box(pmu->type, NUMA_NO_NODE); 705 if (!fake_box) 706 return -ENOMEM; 707 708 fake_box->pmu = pmu; 709 /* 710 * the event is not yet connected with its 711 * siblings therefore we must first collect 712 * existing siblings, then add the new event 713 * before we can simulate the scheduling 714 */ 715 n = uncore_collect_events(fake_box, leader, true); 716 if (n < 0) 717 goto out; 718 719 fake_box->n_events = n; 720 n = uncore_collect_events(fake_box, event, false); 721 if (n < 0) 722 goto out; 723 724 fake_box->n_events = n; 725 726 ret = uncore_assign_events(fake_box, NULL, n); 727out: 728 kfree(fake_box); 729 return ret; 730} 731 732static int uncore_pmu_event_init(struct perf_event *event) 733{ 734 struct intel_uncore_pmu *pmu; 735 struct intel_uncore_box *box; 736 struct hw_perf_event *hwc = &event->hw; 737 int ret; 738 739 if (event->attr.type != event->pmu->type) 740 return -ENOENT; 741 742 pmu = uncore_event_to_pmu(event); 743 /* no device found for this pmu */ 744 if (pmu->func_id < 0) 745 return -ENOENT; 746 747 /* Sampling not supported yet */ 748 if (hwc->sample_period) 749 return -EINVAL; 750 751 /* 752 * Place all uncore events for a particular physical package 753 * onto a single cpu 754 */ 755 if (event->cpu < 0) 756 return -EINVAL; 757 box = uncore_pmu_to_box(pmu, event->cpu); 758 if (!box || box->cpu < 0) 759 return -EINVAL; 760 event->cpu = box->cpu; 761 event->pmu_private = box; 762 763 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; 764 765 event->hw.idx = -1; 766 event->hw.last_tag = ~0ULL; 767 event->hw.extra_reg.idx = EXTRA_REG_NONE; 768 event->hw.branch_reg.idx = EXTRA_REG_NONE; 769 770 if (event->attr.config == UNCORE_FIXED_EVENT) { 771 /* no fixed counter */ 772 if (!pmu->type->fixed_ctl) 773 return -EINVAL; 774 /* 775 * if there is only one fixed counter, only the first pmu 776 * can access the fixed counter 777 */ 778 if (pmu->type->single_fixed && pmu->pmu_idx > 0) 779 return -EINVAL; 780 781 /* fixed counters have event field hardcoded to zero */ 782 hwc->config = 0ULL; 783 } else if (is_freerunning_event(event)) { 784 hwc->config = event->attr.config; 785 if (!check_valid_freerunning_event(box, event)) 786 return -EINVAL; 787 event->hw.idx = UNCORE_PMC_IDX_FREERUNNING; 788 /* 789 * The free running counter event and free running counter 790 * are always 1:1 mapped. 791 * The free running counter is always active. 792 * Assign the free running counter here. 793 */ 794 event->hw.event_base = uncore_freerunning_counter(box, event); 795 } else { 796 hwc->config = event->attr.config & 797 (pmu->type->event_mask | ((u64)pmu->type->event_mask_ext << 32)); 798 if (pmu->type->ops->hw_config) { 799 ret = pmu->type->ops->hw_config(box, event); 800 if (ret) 801 return ret; 802 } 803 } 804 805 if (event->group_leader != event) 806 ret = uncore_validate_group(pmu, event); 807 else 808 ret = 0; 809 810 return ret; 811} 812 813static void uncore_pmu_enable(struct pmu *pmu) 814{ 815 struct intel_uncore_pmu *uncore_pmu; 816 struct intel_uncore_box *box; 817 818 uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu); 819 820 box = uncore_pmu_to_box(uncore_pmu, smp_processor_id()); 821 if (!box) 822 return; 823 824 if (uncore_pmu->type->ops->enable_box) 825 uncore_pmu->type->ops->enable_box(box); 826} 827 828static void uncore_pmu_disable(struct pmu *pmu) 829{ 830 struct intel_uncore_pmu *uncore_pmu; 831 struct intel_uncore_box *box; 832 833 uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu); 834 835 box = uncore_pmu_to_box(uncore_pmu, smp_processor_id()); 836 if (!box) 837 return; 838 839 if (uncore_pmu->type->ops->disable_box) 840 uncore_pmu->type->ops->disable_box(box); 841} 842 843static ssize_t uncore_get_attr_cpumask(struct device *dev, 844 struct device_attribute *attr, char *buf) 845{ 846 return cpumap_print_to_pagebuf(true, buf, &uncore_cpu_mask); 847} 848 849static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL); 850 851static struct attribute *uncore_pmu_attrs[] = { 852 &dev_attr_cpumask.attr, 853 NULL, 854}; 855 856static const struct attribute_group uncore_pmu_attr_group = { 857 .attrs = uncore_pmu_attrs, 858}; 859 860static inline int uncore_get_box_id(struct intel_uncore_type *type, 861 struct intel_uncore_pmu *pmu) 862{ 863 return type->box_ids ? type->box_ids[pmu->pmu_idx] : pmu->pmu_idx; 864} 865 866void uncore_get_alias_name(char *pmu_name, struct intel_uncore_pmu *pmu) 867{ 868 struct intel_uncore_type *type = pmu->type; 869 870 if (type->num_boxes == 1) 871 sprintf(pmu_name, "uncore_type_%u", type->type_id); 872 else { 873 sprintf(pmu_name, "uncore_type_%u_%d", 874 type->type_id, uncore_get_box_id(type, pmu)); 875 } 876} 877 878static void uncore_get_pmu_name(struct intel_uncore_pmu *pmu) 879{ 880 struct intel_uncore_type *type = pmu->type; 881 882 /* 883 * No uncore block name in discovery table. 884 * Use uncore_type_&typeid_&boxid as name. 885 */ 886 if (!type->name) { 887 uncore_get_alias_name(pmu->name, pmu); 888 return; 889 } 890 891 if (type->num_boxes == 1) { 892 if (strlen(type->name) > 0) 893 sprintf(pmu->name, "uncore_%s", type->name); 894 else 895 sprintf(pmu->name, "uncore"); 896 } else { 897 /* 898 * Use the box ID from the discovery table if applicable. 899 */ 900 sprintf(pmu->name, "uncore_%s_%d", type->name, 901 uncore_get_box_id(type, pmu)); 902 } 903} 904 905static int uncore_pmu_register(struct intel_uncore_pmu *pmu) 906{ 907 int ret; 908 909 if (!pmu->type->pmu) { 910 pmu->pmu = (struct pmu) { 911 .attr_groups = pmu->type->attr_groups, 912 .task_ctx_nr = perf_invalid_context, 913 .pmu_enable = uncore_pmu_enable, 914 .pmu_disable = uncore_pmu_disable, 915 .event_init = uncore_pmu_event_init, 916 .add = uncore_pmu_event_add, 917 .del = uncore_pmu_event_del, 918 .start = uncore_pmu_event_start, 919 .stop = uncore_pmu_event_stop, 920 .read = uncore_pmu_event_read, 921 .module = THIS_MODULE, 922 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 923 .attr_update = pmu->type->attr_update, 924 }; 925 } else { 926 pmu->pmu = *pmu->type->pmu; 927 pmu->pmu.attr_groups = pmu->type->attr_groups; 928 pmu->pmu.attr_update = pmu->type->attr_update; 929 } 930 931 uncore_get_pmu_name(pmu); 932 933 ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); 934 if (!ret) 935 pmu->registered = true; 936 return ret; 937} 938 939static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) 940{ 941 if (!pmu->registered) 942 return; 943 perf_pmu_unregister(&pmu->pmu); 944 pmu->registered = false; 945} 946 947static void uncore_free_boxes(struct intel_uncore_pmu *pmu) 948{ 949 int die; 950 951 for (die = 0; die < uncore_max_dies(); die++) 952 kfree(pmu->boxes[die]); 953 kfree(pmu->boxes); 954} 955 956static void uncore_type_exit(struct intel_uncore_type *type) 957{ 958 struct intel_uncore_pmu *pmu = type->pmus; 959 int i; 960 961 if (type->cleanup_mapping) 962 type->cleanup_mapping(type); 963 964 if (pmu) { 965 for (i = 0; i < type->num_boxes; i++, pmu++) { 966 uncore_pmu_unregister(pmu); 967 uncore_free_boxes(pmu); 968 } 969 kfree(type->pmus); 970 type->pmus = NULL; 971 } 972 if (type->box_ids) { 973 kfree(type->box_ids); 974 type->box_ids = NULL; 975 } 976 kfree(type->events_group); 977 type->events_group = NULL; 978} 979 980static void uncore_types_exit(struct intel_uncore_type **types) 981{ 982 for (; *types; types++) 983 uncore_type_exit(*types); 984} 985 986static int __init uncore_type_init(struct intel_uncore_type *type, bool setid) 987{ 988 struct intel_uncore_pmu *pmus; 989 size_t size; 990 int i, j; 991 992 pmus = kcalloc(type->num_boxes, sizeof(*pmus), GFP_KERNEL); 993 if (!pmus) 994 return -ENOMEM; 995 996 size = uncore_max_dies() * sizeof(struct intel_uncore_box *); 997 998 for (i = 0; i < type->num_boxes; i++) { 999 pmus[i].func_id = setid ? i : -1; 1000 pmus[i].pmu_idx = i; 1001 pmus[i].type = type; 1002 pmus[i].boxes = kzalloc(size, GFP_KERNEL); 1003 if (!pmus[i].boxes) 1004 goto err; 1005 } 1006 1007 type->pmus = pmus; 1008 type->unconstrainted = (struct event_constraint) 1009 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1, 1010 0, type->num_counters, 0, 0); 1011 1012 if (type->event_descs) { 1013 struct { 1014 struct attribute_group group; 1015 struct attribute *attrs[]; 1016 } *attr_group; 1017 for (i = 0; type->event_descs[i].attr.attr.name; i++); 1018 1019 attr_group = kzalloc(struct_size(attr_group, attrs, i + 1), 1020 GFP_KERNEL); 1021 if (!attr_group) 1022 goto err; 1023 1024 attr_group->group.name = "events"; 1025 attr_group->group.attrs = attr_group->attrs; 1026 1027 for (j = 0; j < i; j++) 1028 attr_group->attrs[j] = &type->event_descs[j].attr.attr; 1029 1030 type->events_group = &attr_group->group; 1031 } 1032 1033 type->pmu_group = &uncore_pmu_attr_group; 1034 1035 if (type->set_mapping) 1036 type->set_mapping(type); 1037 1038 return 0; 1039 1040err: 1041 for (i = 0; i < type->num_boxes; i++) 1042 kfree(pmus[i].boxes); 1043 kfree(pmus); 1044 1045 return -ENOMEM; 1046} 1047 1048static int __init 1049uncore_types_init(struct intel_uncore_type **types, bool setid) 1050{ 1051 int ret; 1052 1053 for (; *types; types++) { 1054 ret = uncore_type_init(*types, setid); 1055 if (ret) 1056 return ret; 1057 } 1058 return 0; 1059} 1060 1061/* 1062 * Get the die information of a PCI device. 1063 * @pdev: The PCI device. 1064 * @die: The die id which the device maps to. 1065 */ 1066static int uncore_pci_get_dev_die_info(struct pci_dev *pdev, int *die) 1067{ 1068 *die = uncore_pcibus_to_dieid(pdev->bus); 1069 if (*die < 0) 1070 return -EINVAL; 1071 1072 return 0; 1073} 1074 1075static struct intel_uncore_pmu * 1076uncore_pci_find_dev_pmu_from_types(struct pci_dev *pdev) 1077{ 1078 struct intel_uncore_type **types = uncore_pci_uncores; 1079 struct intel_uncore_type *type; 1080 u64 box_ctl; 1081 int i, die; 1082 1083 for (; *types; types++) { 1084 type = *types; 1085 for (die = 0; die < __uncore_max_dies; die++) { 1086 for (i = 0; i < type->num_boxes; i++) { 1087 if (!type->box_ctls[die]) 1088 continue; 1089 box_ctl = type->box_ctls[die] + type->pci_offsets[i]; 1090 if (pdev->devfn == UNCORE_DISCOVERY_PCI_DEVFN(box_ctl) && 1091 pdev->bus->number == UNCORE_DISCOVERY_PCI_BUS(box_ctl) && 1092 pci_domain_nr(pdev->bus) == UNCORE_DISCOVERY_PCI_DOMAIN(box_ctl)) 1093 return &type->pmus[i]; 1094 } 1095 } 1096 } 1097 1098 return NULL; 1099} 1100 1101/* 1102 * Find the PMU of a PCI device. 1103 * @pdev: The PCI device. 1104 * @ids: The ID table of the available PCI devices with a PMU. 1105 * If NULL, search the whole uncore_pci_uncores. 1106 */ 1107static struct intel_uncore_pmu * 1108uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids) 1109{ 1110 struct intel_uncore_pmu *pmu = NULL; 1111 struct intel_uncore_type *type; 1112 kernel_ulong_t data; 1113 unsigned int devfn; 1114 1115 if (!ids) 1116 return uncore_pci_find_dev_pmu_from_types(pdev); 1117 1118 while (ids && ids->vendor) { 1119 if ((ids->vendor == pdev->vendor) && 1120 (ids->device == pdev->device)) { 1121 data = ids->driver_data; 1122 devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(data), 1123 UNCORE_PCI_DEV_FUNC(data)); 1124 if (devfn == pdev->devfn) { 1125 type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(data)]; 1126 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(data)]; 1127 break; 1128 } 1129 } 1130 ids++; 1131 } 1132 return pmu; 1133} 1134 1135/* 1136 * Register the PMU for a PCI device 1137 * @pdev: The PCI device. 1138 * @type: The corresponding PMU type of the device. 1139 * @pmu: The corresponding PMU of the device. 1140 * @die: The die id which the device maps to. 1141 */ 1142static int uncore_pci_pmu_register(struct pci_dev *pdev, 1143 struct intel_uncore_type *type, 1144 struct intel_uncore_pmu *pmu, 1145 int die) 1146{ 1147 struct intel_uncore_box *box; 1148 int ret; 1149 1150 if (WARN_ON_ONCE(pmu->boxes[die] != NULL)) 1151 return -EINVAL; 1152 1153 box = uncore_alloc_box(type, NUMA_NO_NODE); 1154 if (!box) 1155 return -ENOMEM; 1156 1157 if (pmu->func_id < 0) 1158 pmu->func_id = pdev->devfn; 1159 else 1160 WARN_ON_ONCE(pmu->func_id != pdev->devfn); 1161 1162 atomic_inc(&box->refcnt); 1163 box->dieid = die; 1164 box->pci_dev = pdev; 1165 box->pmu = pmu; 1166 uncore_box_init(box); 1167 1168 pmu->boxes[die] = box; 1169 if (atomic_inc_return(&pmu->activeboxes) > 1) 1170 return 0; 1171 1172 /* First active box registers the pmu */ 1173 ret = uncore_pmu_register(pmu); 1174 if (ret) { 1175 pmu->boxes[die] = NULL; 1176 uncore_box_exit(box); 1177 kfree(box); 1178 } 1179 return ret; 1180} 1181 1182/* 1183 * add a pci uncore device 1184 */ 1185static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1186{ 1187 struct intel_uncore_type *type; 1188 struct intel_uncore_pmu *pmu = NULL; 1189 int die, ret; 1190 1191 ret = uncore_pci_get_dev_die_info(pdev, &die); 1192 if (ret) 1193 return ret; 1194 1195 if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) { 1196 int idx = UNCORE_PCI_DEV_IDX(id->driver_data); 1197 1198 uncore_extra_pci_dev[die].dev[idx] = pdev; 1199 pci_set_drvdata(pdev, NULL); 1200 return 0; 1201 } 1202 1203 type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)]; 1204 1205 /* 1206 * Some platforms, e.g. Knights Landing, use a common PCI device ID 1207 * for multiple instances of an uncore PMU device type. We should check 1208 * PCI slot and func to indicate the uncore box. 1209 */ 1210 if (id->driver_data & ~0xffff) { 1211 struct pci_driver *pci_drv = to_pci_driver(pdev->dev.driver); 1212 1213 pmu = uncore_pci_find_dev_pmu(pdev, pci_drv->id_table); 1214 if (pmu == NULL) 1215 return -ENODEV; 1216 } else { 1217 /* 1218 * for performance monitoring unit with multiple boxes, 1219 * each box has a different function id. 1220 */ 1221 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)]; 1222 } 1223 1224 ret = uncore_pci_pmu_register(pdev, type, pmu, die); 1225 1226 pci_set_drvdata(pdev, pmu->boxes[die]); 1227 1228 return ret; 1229} 1230 1231/* 1232 * Unregister the PMU of a PCI device 1233 * @pmu: The corresponding PMU is unregistered. 1234 * @die: The die id which the device maps to. 1235 */ 1236static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) 1237{ 1238 struct intel_uncore_box *box = pmu->boxes[die]; 1239 1240 pmu->boxes[die] = NULL; 1241 if (atomic_dec_return(&pmu->activeboxes) == 0) 1242 uncore_pmu_unregister(pmu); 1243 uncore_box_exit(box); 1244 kfree(box); 1245} 1246 1247static void uncore_pci_remove(struct pci_dev *pdev) 1248{ 1249 struct intel_uncore_box *box; 1250 struct intel_uncore_pmu *pmu; 1251 int i, die; 1252 1253 if (uncore_pci_get_dev_die_info(pdev, &die)) 1254 return; 1255 1256 box = pci_get_drvdata(pdev); 1257 if (!box) { 1258 for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) { 1259 if (uncore_extra_pci_dev[die].dev[i] == pdev) { 1260 uncore_extra_pci_dev[die].dev[i] = NULL; 1261 break; 1262 } 1263 } 1264 WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); 1265 return; 1266 } 1267 1268 pmu = box->pmu; 1269 1270 pci_set_drvdata(pdev, NULL); 1271 1272 uncore_pci_pmu_unregister(pmu, die); 1273} 1274 1275static int uncore_bus_notify(struct notifier_block *nb, 1276 unsigned long action, void *data, 1277 const struct pci_device_id *ids) 1278{ 1279 struct device *dev = data; 1280 struct pci_dev *pdev = to_pci_dev(dev); 1281 struct intel_uncore_pmu *pmu; 1282 int die; 1283 1284 /* Unregister the PMU when the device is going to be deleted. */ 1285 if (action != BUS_NOTIFY_DEL_DEVICE) 1286 return NOTIFY_DONE; 1287 1288 pmu = uncore_pci_find_dev_pmu(pdev, ids); 1289 if (!pmu) 1290 return NOTIFY_DONE; 1291 1292 if (uncore_pci_get_dev_die_info(pdev, &die)) 1293 return NOTIFY_DONE; 1294 1295 uncore_pci_pmu_unregister(pmu, die); 1296 1297 return NOTIFY_OK; 1298} 1299 1300static int uncore_pci_sub_bus_notify(struct notifier_block *nb, 1301 unsigned long action, void *data) 1302{ 1303 return uncore_bus_notify(nb, action, data, 1304 uncore_pci_sub_driver->id_table); 1305} 1306 1307static struct notifier_block uncore_pci_sub_notifier = { 1308 .notifier_call = uncore_pci_sub_bus_notify, 1309}; 1310 1311static void uncore_pci_sub_driver_init(void) 1312{ 1313 const struct pci_device_id *ids = uncore_pci_sub_driver->id_table; 1314 struct intel_uncore_type *type; 1315 struct intel_uncore_pmu *pmu; 1316 struct pci_dev *pci_sub_dev; 1317 bool notify = false; 1318 unsigned int devfn; 1319 int die; 1320 1321 while (ids && ids->vendor) { 1322 pci_sub_dev = NULL; 1323 type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(ids->driver_data)]; 1324 /* 1325 * Search the available device, and register the 1326 * corresponding PMU. 1327 */ 1328 while ((pci_sub_dev = pci_get_device(PCI_VENDOR_ID_INTEL, 1329 ids->device, pci_sub_dev))) { 1330 devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data), 1331 UNCORE_PCI_DEV_FUNC(ids->driver_data)); 1332 if (devfn != pci_sub_dev->devfn) 1333 continue; 1334 1335 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)]; 1336 if (!pmu) 1337 continue; 1338 1339 if (uncore_pci_get_dev_die_info(pci_sub_dev, &die)) 1340 continue; 1341 1342 if (!uncore_pci_pmu_register(pci_sub_dev, type, pmu, 1343 die)) 1344 notify = true; 1345 } 1346 ids++; 1347 } 1348 1349 if (notify && bus_register_notifier(&pci_bus_type, &uncore_pci_sub_notifier)) 1350 notify = false; 1351 1352 if (!notify) 1353 uncore_pci_sub_driver = NULL; 1354} 1355 1356static int uncore_pci_bus_notify(struct notifier_block *nb, 1357 unsigned long action, void *data) 1358{ 1359 return uncore_bus_notify(nb, action, data, NULL); 1360} 1361 1362static struct notifier_block uncore_pci_notifier = { 1363 .notifier_call = uncore_pci_bus_notify, 1364}; 1365 1366 1367static void uncore_pci_pmus_register(void) 1368{ 1369 struct intel_uncore_type **types = uncore_pci_uncores; 1370 struct intel_uncore_type *type; 1371 struct intel_uncore_pmu *pmu; 1372 struct pci_dev *pdev; 1373 u64 box_ctl; 1374 int i, die; 1375 1376 for (; *types; types++) { 1377 type = *types; 1378 for (die = 0; die < __uncore_max_dies; die++) { 1379 for (i = 0; i < type->num_boxes; i++) { 1380 if (!type->box_ctls[die]) 1381 continue; 1382 box_ctl = type->box_ctls[die] + type->pci_offsets[i]; 1383 pdev = pci_get_domain_bus_and_slot(UNCORE_DISCOVERY_PCI_DOMAIN(box_ctl), 1384 UNCORE_DISCOVERY_PCI_BUS(box_ctl), 1385 UNCORE_DISCOVERY_PCI_DEVFN(box_ctl)); 1386 if (!pdev) 1387 continue; 1388 pmu = &type->pmus[i]; 1389 1390 uncore_pci_pmu_register(pdev, type, pmu, die); 1391 } 1392 } 1393 } 1394 1395 bus_register_notifier(&pci_bus_type, &uncore_pci_notifier); 1396} 1397 1398static int __init uncore_pci_init(void) 1399{ 1400 size_t size; 1401 int ret; 1402 1403 size = uncore_max_dies() * sizeof(struct pci_extra_dev); 1404 uncore_extra_pci_dev = kzalloc(size, GFP_KERNEL); 1405 if (!uncore_extra_pci_dev) { 1406 ret = -ENOMEM; 1407 goto err; 1408 } 1409 1410 ret = uncore_types_init(uncore_pci_uncores, false); 1411 if (ret) 1412 goto errtype; 1413 1414 if (uncore_pci_driver) { 1415 uncore_pci_driver->probe = uncore_pci_probe; 1416 uncore_pci_driver->remove = uncore_pci_remove; 1417 1418 ret = pci_register_driver(uncore_pci_driver); 1419 if (ret) 1420 goto errtype; 1421 } else 1422 uncore_pci_pmus_register(); 1423 1424 if (uncore_pci_sub_driver) 1425 uncore_pci_sub_driver_init(); 1426 1427 pcidrv_registered = true; 1428 return 0; 1429 1430errtype: 1431 uncore_types_exit(uncore_pci_uncores); 1432 kfree(uncore_extra_pci_dev); 1433 uncore_extra_pci_dev = NULL; 1434 uncore_free_pcibus_map(); 1435err: 1436 uncore_pci_uncores = empty_uncore; 1437 return ret; 1438} 1439 1440static void uncore_pci_exit(void) 1441{ 1442 if (pcidrv_registered) { 1443 pcidrv_registered = false; 1444 if (uncore_pci_sub_driver) 1445 bus_unregister_notifier(&pci_bus_type, &uncore_pci_sub_notifier); 1446 if (uncore_pci_driver) 1447 pci_unregister_driver(uncore_pci_driver); 1448 else 1449 bus_unregister_notifier(&pci_bus_type, &uncore_pci_notifier); 1450 uncore_types_exit(uncore_pci_uncores); 1451 kfree(uncore_extra_pci_dev); 1452 uncore_free_pcibus_map(); 1453 } 1454} 1455 1456static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, 1457 int new_cpu) 1458{ 1459 struct intel_uncore_pmu *pmu = type->pmus; 1460 struct intel_uncore_box *box; 1461 int i, die; 1462 1463 die = topology_logical_die_id(old_cpu < 0 ? new_cpu : old_cpu); 1464 for (i = 0; i < type->num_boxes; i++, pmu++) { 1465 box = pmu->boxes[die]; 1466 if (!box) 1467 continue; 1468 1469 if (old_cpu < 0) { 1470 WARN_ON_ONCE(box->cpu != -1); 1471 box->cpu = new_cpu; 1472 continue; 1473 } 1474 1475 WARN_ON_ONCE(box->cpu != old_cpu); 1476 box->cpu = -1; 1477 if (new_cpu < 0) 1478 continue; 1479 1480 uncore_pmu_cancel_hrtimer(box); 1481 perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); 1482 box->cpu = new_cpu; 1483 } 1484} 1485 1486static void uncore_change_context(struct intel_uncore_type **uncores, 1487 int old_cpu, int new_cpu) 1488{ 1489 for (; *uncores; uncores++) 1490 uncore_change_type_ctx(*uncores, old_cpu, new_cpu); 1491} 1492 1493static void uncore_box_unref(struct intel_uncore_type **types, int id) 1494{ 1495 struct intel_uncore_type *type; 1496 struct intel_uncore_pmu *pmu; 1497 struct intel_uncore_box *box; 1498 int i; 1499 1500 for (; *types; types++) { 1501 type = *types; 1502 pmu = type->pmus; 1503 for (i = 0; i < type->num_boxes; i++, pmu++) { 1504 box = pmu->boxes[id]; 1505 if (box && atomic_dec_return(&box->refcnt) == 0) 1506 uncore_box_exit(box); 1507 } 1508 } 1509} 1510 1511static int uncore_event_cpu_offline(unsigned int cpu) 1512{ 1513 int die, target; 1514 1515 /* Check if exiting cpu is used for collecting uncore events */ 1516 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) 1517 goto unref; 1518 /* Find a new cpu to collect uncore events */ 1519 target = cpumask_any_but(topology_die_cpumask(cpu), cpu); 1520 1521 /* Migrate uncore events to the new target */ 1522 if (target < nr_cpu_ids) 1523 cpumask_set_cpu(target, &uncore_cpu_mask); 1524 else 1525 target = -1; 1526 1527 uncore_change_context(uncore_msr_uncores, cpu, target); 1528 uncore_change_context(uncore_mmio_uncores, cpu, target); 1529 uncore_change_context(uncore_pci_uncores, cpu, target); 1530 1531unref: 1532 /* Clear the references */ 1533 die = topology_logical_die_id(cpu); 1534 uncore_box_unref(uncore_msr_uncores, die); 1535 uncore_box_unref(uncore_mmio_uncores, die); 1536 return 0; 1537} 1538 1539static int allocate_boxes(struct intel_uncore_type **types, 1540 unsigned int die, unsigned int cpu) 1541{ 1542 struct intel_uncore_box *box, *tmp; 1543 struct intel_uncore_type *type; 1544 struct intel_uncore_pmu *pmu; 1545 LIST_HEAD(allocated); 1546 int i; 1547 1548 /* Try to allocate all required boxes */ 1549 for (; *types; types++) { 1550 type = *types; 1551 pmu = type->pmus; 1552 for (i = 0; i < type->num_boxes; i++, pmu++) { 1553 if (pmu->boxes[die]) 1554 continue; 1555 box = uncore_alloc_box(type, cpu_to_node(cpu)); 1556 if (!box) 1557 goto cleanup; 1558 box->pmu = pmu; 1559 box->dieid = die; 1560 list_add(&box->active_list, &allocated); 1561 } 1562 } 1563 /* Install them in the pmus */ 1564 list_for_each_entry_safe(box, tmp, &allocated, active_list) { 1565 list_del_init(&box->active_list); 1566 box->pmu->boxes[die] = box; 1567 } 1568 return 0; 1569 1570cleanup: 1571 list_for_each_entry_safe(box, tmp, &allocated, active_list) { 1572 list_del_init(&box->active_list); 1573 kfree(box); 1574 } 1575 return -ENOMEM; 1576} 1577 1578static int uncore_box_ref(struct intel_uncore_type **types, 1579 int id, unsigned int cpu) 1580{ 1581 struct intel_uncore_type *type; 1582 struct intel_uncore_pmu *pmu; 1583 struct intel_uncore_box *box; 1584 int i, ret; 1585 1586 ret = allocate_boxes(types, id, cpu); 1587 if (ret) 1588 return ret; 1589 1590 for (; *types; types++) { 1591 type = *types; 1592 pmu = type->pmus; 1593 for (i = 0; i < type->num_boxes; i++, pmu++) { 1594 box = pmu->boxes[id]; 1595 if (box && atomic_inc_return(&box->refcnt) == 1) 1596 uncore_box_init(box); 1597 } 1598 } 1599 return 0; 1600} 1601 1602static int uncore_event_cpu_online(unsigned int cpu) 1603{ 1604 int die, target, msr_ret, mmio_ret; 1605 1606 die = topology_logical_die_id(cpu); 1607 msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); 1608 mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); 1609 if (msr_ret && mmio_ret) 1610 return -ENOMEM; 1611 1612 /* 1613 * Check if there is an online cpu in the package 1614 * which collects uncore events already. 1615 */ 1616 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); 1617 if (target < nr_cpu_ids) 1618 return 0; 1619 1620 cpumask_set_cpu(cpu, &uncore_cpu_mask); 1621 1622 if (!msr_ret) 1623 uncore_change_context(uncore_msr_uncores, -1, cpu); 1624 if (!mmio_ret) 1625 uncore_change_context(uncore_mmio_uncores, -1, cpu); 1626 uncore_change_context(uncore_pci_uncores, -1, cpu); 1627 return 0; 1628} 1629 1630static int __init type_pmu_register(struct intel_uncore_type *type) 1631{ 1632 int i, ret; 1633 1634 for (i = 0; i < type->num_boxes; i++) { 1635 ret = uncore_pmu_register(&type->pmus[i]); 1636 if (ret) 1637 return ret; 1638 } 1639 return 0; 1640} 1641 1642static int __init uncore_msr_pmus_register(void) 1643{ 1644 struct intel_uncore_type **types = uncore_msr_uncores; 1645 int ret; 1646 1647 for (; *types; types++) { 1648 ret = type_pmu_register(*types); 1649 if (ret) 1650 return ret; 1651 } 1652 return 0; 1653} 1654 1655static int __init uncore_cpu_init(void) 1656{ 1657 int ret; 1658 1659 ret = uncore_types_init(uncore_msr_uncores, true); 1660 if (ret) 1661 goto err; 1662 1663 ret = uncore_msr_pmus_register(); 1664 if (ret) 1665 goto err; 1666 return 0; 1667err: 1668 uncore_types_exit(uncore_msr_uncores); 1669 uncore_msr_uncores = empty_uncore; 1670 return ret; 1671} 1672 1673static int __init uncore_mmio_init(void) 1674{ 1675 struct intel_uncore_type **types = uncore_mmio_uncores; 1676 int ret; 1677 1678 ret = uncore_types_init(types, true); 1679 if (ret) 1680 goto err; 1681 1682 for (; *types; types++) { 1683 ret = type_pmu_register(*types); 1684 if (ret) 1685 goto err; 1686 } 1687 return 0; 1688err: 1689 uncore_types_exit(uncore_mmio_uncores); 1690 uncore_mmio_uncores = empty_uncore; 1691 return ret; 1692} 1693 1694struct intel_uncore_init_fun { 1695 void (*cpu_init)(void); 1696 int (*pci_init)(void); 1697 void (*mmio_init)(void); 1698 /* Discovery table is required */ 1699 bool use_discovery; 1700 /* The units in the discovery table should be ignored. */ 1701 int *uncore_units_ignore; 1702}; 1703 1704static const struct intel_uncore_init_fun nhm_uncore_init __initconst = { 1705 .cpu_init = nhm_uncore_cpu_init, 1706}; 1707 1708static const struct intel_uncore_init_fun snb_uncore_init __initconst = { 1709 .cpu_init = snb_uncore_cpu_init, 1710 .pci_init = snb_uncore_pci_init, 1711}; 1712 1713static const struct intel_uncore_init_fun ivb_uncore_init __initconst = { 1714 .cpu_init = snb_uncore_cpu_init, 1715 .pci_init = ivb_uncore_pci_init, 1716}; 1717 1718static const struct intel_uncore_init_fun hsw_uncore_init __initconst = { 1719 .cpu_init = snb_uncore_cpu_init, 1720 .pci_init = hsw_uncore_pci_init, 1721}; 1722 1723static const struct intel_uncore_init_fun bdw_uncore_init __initconst = { 1724 .cpu_init = snb_uncore_cpu_init, 1725 .pci_init = bdw_uncore_pci_init, 1726}; 1727 1728static const struct intel_uncore_init_fun snbep_uncore_init __initconst = { 1729 .cpu_init = snbep_uncore_cpu_init, 1730 .pci_init = snbep_uncore_pci_init, 1731}; 1732 1733static const struct intel_uncore_init_fun nhmex_uncore_init __initconst = { 1734 .cpu_init = nhmex_uncore_cpu_init, 1735}; 1736 1737static const struct intel_uncore_init_fun ivbep_uncore_init __initconst = { 1738 .cpu_init = ivbep_uncore_cpu_init, 1739 .pci_init = ivbep_uncore_pci_init, 1740}; 1741 1742static const struct intel_uncore_init_fun hswep_uncore_init __initconst = { 1743 .cpu_init = hswep_uncore_cpu_init, 1744 .pci_init = hswep_uncore_pci_init, 1745}; 1746 1747static const struct intel_uncore_init_fun bdx_uncore_init __initconst = { 1748 .cpu_init = bdx_uncore_cpu_init, 1749 .pci_init = bdx_uncore_pci_init, 1750}; 1751 1752static const struct intel_uncore_init_fun knl_uncore_init __initconst = { 1753 .cpu_init = knl_uncore_cpu_init, 1754 .pci_init = knl_uncore_pci_init, 1755}; 1756 1757static const struct intel_uncore_init_fun skl_uncore_init __initconst = { 1758 .cpu_init = skl_uncore_cpu_init, 1759 .pci_init = skl_uncore_pci_init, 1760}; 1761 1762static const struct intel_uncore_init_fun skx_uncore_init __initconst = { 1763 .cpu_init = skx_uncore_cpu_init, 1764 .pci_init = skx_uncore_pci_init, 1765}; 1766 1767static const struct intel_uncore_init_fun icl_uncore_init __initconst = { 1768 .cpu_init = icl_uncore_cpu_init, 1769 .pci_init = skl_uncore_pci_init, 1770}; 1771 1772static const struct intel_uncore_init_fun tgl_uncore_init __initconst = { 1773 .cpu_init = tgl_uncore_cpu_init, 1774 .mmio_init = tgl_uncore_mmio_init, 1775}; 1776 1777static const struct intel_uncore_init_fun tgl_l_uncore_init __initconst = { 1778 .cpu_init = tgl_uncore_cpu_init, 1779 .mmio_init = tgl_l_uncore_mmio_init, 1780}; 1781 1782static const struct intel_uncore_init_fun rkl_uncore_init __initconst = { 1783 .cpu_init = tgl_uncore_cpu_init, 1784 .pci_init = skl_uncore_pci_init, 1785}; 1786 1787static const struct intel_uncore_init_fun adl_uncore_init __initconst = { 1788 .cpu_init = adl_uncore_cpu_init, 1789 .mmio_init = adl_uncore_mmio_init, 1790}; 1791 1792static const struct intel_uncore_init_fun mtl_uncore_init __initconst = { 1793 .cpu_init = mtl_uncore_cpu_init, 1794 .mmio_init = adl_uncore_mmio_init, 1795}; 1796 1797static const struct intel_uncore_init_fun icx_uncore_init __initconst = { 1798 .cpu_init = icx_uncore_cpu_init, 1799 .pci_init = icx_uncore_pci_init, 1800 .mmio_init = icx_uncore_mmio_init, 1801}; 1802 1803static const struct intel_uncore_init_fun snr_uncore_init __initconst = { 1804 .cpu_init = snr_uncore_cpu_init, 1805 .pci_init = snr_uncore_pci_init, 1806 .mmio_init = snr_uncore_mmio_init, 1807}; 1808 1809static const struct intel_uncore_init_fun spr_uncore_init __initconst = { 1810 .cpu_init = spr_uncore_cpu_init, 1811 .pci_init = spr_uncore_pci_init, 1812 .mmio_init = spr_uncore_mmio_init, 1813 .use_discovery = true, 1814 .uncore_units_ignore = spr_uncore_units_ignore, 1815}; 1816 1817static const struct intel_uncore_init_fun gnr_uncore_init __initconst = { 1818 .cpu_init = gnr_uncore_cpu_init, 1819 .pci_init = gnr_uncore_pci_init, 1820 .mmio_init = gnr_uncore_mmio_init, 1821 .use_discovery = true, 1822 .uncore_units_ignore = gnr_uncore_units_ignore, 1823}; 1824 1825static const struct intel_uncore_init_fun generic_uncore_init __initconst = { 1826 .cpu_init = intel_uncore_generic_uncore_cpu_init, 1827 .pci_init = intel_uncore_generic_uncore_pci_init, 1828 .mmio_init = intel_uncore_generic_uncore_mmio_init, 1829}; 1830 1831static const struct x86_cpu_id intel_uncore_match[] __initconst = { 1832 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &nhm_uncore_init), 1833 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &nhm_uncore_init), 1834 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &nhm_uncore_init), 1835 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &nhm_uncore_init), 1836 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &snb_uncore_init), 1837 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &ivb_uncore_init), 1838 X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &hsw_uncore_init), 1839 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &hsw_uncore_init), 1840 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &hsw_uncore_init), 1841 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &bdw_uncore_init), 1842 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &bdw_uncore_init), 1843 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &snbep_uncore_init), 1844 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &nhmex_uncore_init), 1845 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &nhmex_uncore_init), 1846 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ivbep_uncore_init), 1847 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &hswep_uncore_init), 1848 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &bdx_uncore_init), 1849 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &bdx_uncore_init), 1850 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &knl_uncore_init), 1851 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &knl_uncore_init), 1852 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &skl_uncore_init), 1853 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &skl_uncore_init), 1854 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &skx_uncore_init), 1855 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &skl_uncore_init), 1856 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &skl_uncore_init), 1857 X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &skl_uncore_init), 1858 X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &skl_uncore_init), 1859 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_uncore_init), 1860 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_NNPI, &icl_uncore_init), 1861 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_uncore_init), 1862 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &icx_uncore_init), 1863 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &icx_uncore_init), 1864 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &tgl_l_uncore_init), 1865 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &tgl_uncore_init), 1866 X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &rkl_uncore_init), 1867 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_uncore_init), 1868 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &adl_uncore_init), 1869 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_uncore_init), 1870 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &adl_uncore_init), 1871 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &adl_uncore_init), 1872 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, &mtl_uncore_init), 1873 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &mtl_uncore_init), 1874 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &spr_uncore_init), 1875 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &spr_uncore_init), 1876 X86_MATCH_INTEL_FAM6_MODEL(GRANITERAPIDS_X, &gnr_uncore_init), 1877 X86_MATCH_INTEL_FAM6_MODEL(GRANITERAPIDS_D, &gnr_uncore_init), 1878 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init), 1879 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, &adl_uncore_init), 1880 X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT_X, &gnr_uncore_init), 1881 X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT, &gnr_uncore_init), 1882 {}, 1883}; 1884MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match); 1885 1886static int __init intel_uncore_init(void) 1887{ 1888 const struct x86_cpu_id *id; 1889 struct intel_uncore_init_fun *uncore_init; 1890 int pret = 0, cret = 0, mret = 0, ret; 1891 1892 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 1893 return -ENODEV; 1894 1895 __uncore_max_dies = 1896 topology_max_packages() * topology_max_dies_per_package(); 1897 1898 id = x86_match_cpu(intel_uncore_match); 1899 if (!id) { 1900 if (!uncore_no_discover && intel_uncore_has_discovery_tables(NULL)) 1901 uncore_init = (struct intel_uncore_init_fun *)&generic_uncore_init; 1902 else 1903 return -ENODEV; 1904 } else { 1905 uncore_init = (struct intel_uncore_init_fun *)id->driver_data; 1906 if (uncore_no_discover && uncore_init->use_discovery) 1907 return -ENODEV; 1908 if (uncore_init->use_discovery && 1909 !intel_uncore_has_discovery_tables(uncore_init->uncore_units_ignore)) 1910 return -ENODEV; 1911 } 1912 1913 if (uncore_init->pci_init) { 1914 pret = uncore_init->pci_init(); 1915 if (!pret) 1916 pret = uncore_pci_init(); 1917 } 1918 1919 if (uncore_init->cpu_init) { 1920 uncore_init->cpu_init(); 1921 cret = uncore_cpu_init(); 1922 } 1923 1924 if (uncore_init->mmio_init) { 1925 uncore_init->mmio_init(); 1926 mret = uncore_mmio_init(); 1927 } 1928 1929 if (cret && pret && mret) { 1930 ret = -ENODEV; 1931 goto free_discovery; 1932 } 1933 1934 /* Install hotplug callbacks to setup the targets for each package */ 1935 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE, 1936 "perf/x86/intel/uncore:online", 1937 uncore_event_cpu_online, 1938 uncore_event_cpu_offline); 1939 if (ret) 1940 goto err; 1941 return 0; 1942 1943err: 1944 uncore_types_exit(uncore_msr_uncores); 1945 uncore_types_exit(uncore_mmio_uncores); 1946 uncore_pci_exit(); 1947free_discovery: 1948 intel_uncore_clear_discovery_tables(); 1949 return ret; 1950} 1951module_init(intel_uncore_init); 1952 1953static void __exit intel_uncore_exit(void) 1954{ 1955 cpuhp_remove_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE); 1956 uncore_types_exit(uncore_msr_uncores); 1957 uncore_types_exit(uncore_mmio_uncores); 1958 uncore_pci_exit(); 1959 intel_uncore_clear_discovery_tables(); 1960} 1961module_exit(intel_uncore_exit); 1962