1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Hypervisor supplied "24x7" performance counter support 4 * 5 * Author: Cody P Schafer <cody@linux.vnet.ibm.com> 6 * Copyright 2014 IBM Corporation. 7 */ 8 9#define pr_fmt(fmt) "hv-24x7: " fmt 10 11#include <linux/perf_event.h> 12#include <linux/rbtree.h> 13#include <linux/module.h> 14#include <linux/slab.h> 15#include <linux/vmalloc.h> 16 17#include <asm/cputhreads.h> 18#include <asm/firmware.h> 19#include <asm/hvcall.h> 20#include <asm/io.h> 21#include <asm/papr-sysparm.h> 22#include <linux/byteorder/generic.h> 23 24#include <asm/rtas.h> 25#include "hv-24x7.h" 26#include "hv-24x7-catalog.h" 27#include "hv-common.h" 28 29/* Version of the 24x7 hypervisor API that we should use in this machine. */ 30static int interface_version; 31 32/* Whether we have to aggregate result data for some domains. */ 33static bool aggregate_result_elements; 34 35static cpumask_t hv_24x7_cpumask; 36 37static bool domain_is_valid(unsigned int domain) 38{ 39 switch (domain) { 40#define DOMAIN(n, v, x, c) \ 41 case HV_PERF_DOMAIN_##n: \ 42 /* fall through */ 43#include "hv-24x7-domains.h" 44#undef DOMAIN 45 return true; 46 default: 47 return false; 48 } 49} 50 51static bool is_physical_domain(unsigned int domain) 52{ 53 switch (domain) { 54#define DOMAIN(n, v, x, c) \ 55 case HV_PERF_DOMAIN_##n: \ 56 return c; 57#include "hv-24x7-domains.h" 58#undef DOMAIN 59 default: 60 return false; 61 } 62} 63 64/* 65 * The Processor Module Information system parameter allows transferring 66 * of certain processor module information from the platform to the OS. 67 * Refer PAPR+ document to get parameter token value as '43'. 68 */ 69 70static u32 phys_sockets; /* Physical sockets */ 71static u32 phys_chipspersocket; /* Physical chips per socket*/ 72static u32 phys_coresperchip; /* Physical cores per chip */ 73 74/* 75 * read_24x7_sys_info() 76 * Retrieve the number of sockets and chips per socket and cores per 77 * chip details through the get-system-parameter rtas call. 78 */ 79void read_24x7_sys_info(void) 80{ 81 struct papr_sysparm_buf *buf; 82 83 /* 84 * Making system parameter: chips and sockets and cores per chip 85 * default to 1. 86 */ 87 phys_sockets = 1; 88 phys_chipspersocket = 1; 89 phys_coresperchip = 1; 90 91 buf = papr_sysparm_buf_alloc(); 92 if (!buf) 93 return; 94 95 if (!papr_sysparm_get(PAPR_SYSPARM_PROC_MODULE_INFO, buf)) { 96 int ntypes = be16_to_cpup((__be16 *)&buf->val[0]); 97 int len = be16_to_cpu(buf->len); 98 99 if (len >= 8 && ntypes != 0) { 100 phys_sockets = be16_to_cpup((__be16 *)&buf->val[2]); 101 phys_chipspersocket = be16_to_cpup((__be16 *)&buf->val[4]); 102 phys_coresperchip = be16_to_cpup((__be16 *)&buf->val[6]); 103 } 104 } 105 106 papr_sysparm_buf_free(buf); 107} 108 109/* Domains for which more than one result element are returned for each event. */ 110static bool domain_needs_aggregation(unsigned int domain) 111{ 112 return aggregate_result_elements && 113 (domain == HV_PERF_DOMAIN_PHYS_CORE || 114 (domain >= HV_PERF_DOMAIN_VCPU_HOME_CORE && 115 domain <= HV_PERF_DOMAIN_VCPU_REMOTE_NODE)); 116} 117 118static const char *domain_name(unsigned int domain) 119{ 120 if (!domain_is_valid(domain)) 121 return NULL; 122 123 switch (domain) { 124 case HV_PERF_DOMAIN_PHYS_CHIP: return "Physical Chip"; 125 case HV_PERF_DOMAIN_PHYS_CORE: return "Physical Core"; 126 case HV_PERF_DOMAIN_VCPU_HOME_CORE: return "VCPU Home Core"; 127 case HV_PERF_DOMAIN_VCPU_HOME_CHIP: return "VCPU Home Chip"; 128 case HV_PERF_DOMAIN_VCPU_HOME_NODE: return "VCPU Home Node"; 129 case HV_PERF_DOMAIN_VCPU_REMOTE_NODE: return "VCPU Remote Node"; 130 } 131 132 WARN_ON_ONCE(domain); 133 return NULL; 134} 135 136static bool catalog_entry_domain_is_valid(unsigned int domain) 137{ 138 /* POWER8 doesn't support virtual domains. */ 139 if (interface_version == 1) 140 return is_physical_domain(domain); 141 else 142 return domain_is_valid(domain); 143} 144 145/* 146 * TODO: Merging events: 147 * - Think of the hcall as an interface to a 4d array of counters: 148 * - x = domains 149 * - y = indexes in the domain (core, chip, vcpu, node, etc) 150 * - z = offset into the counter space 151 * - w = lpars (guest vms, "logical partitions") 152 * - A single request is: x,y,y_last,z,z_last,w,w_last 153 * - this means we can retrieve a rectangle of counters in y,z for a single x. 154 * 155 * - Things to consider (ignoring w): 156 * - input cost_per_request = 16 157 * - output cost_per_result(ys,zs) = 8 + 8 * ys + ys * zs 158 * - limited number of requests per hcall (must fit into 4K bytes) 159 * - 4k = 16 [buffer header] - 16 [request size] * request_count 160 * - 255 requests per hcall 161 * - sometimes it will be more efficient to read extra data and discard 162 */ 163 164/* 165 * Example usage: 166 * perf stat -e 'hv_24x7/domain=2,offset=8,vcpu=0,lpar=0xffffffff/' 167 */ 168 169/* u3 0-6, one of HV_24X7_PERF_DOMAIN */ 170EVENT_DEFINE_RANGE_FORMAT(domain, config, 0, 3); 171/* u16 */ 172EVENT_DEFINE_RANGE_FORMAT(core, config, 16, 31); 173EVENT_DEFINE_RANGE_FORMAT(chip, config, 16, 31); 174EVENT_DEFINE_RANGE_FORMAT(vcpu, config, 16, 31); 175/* u32, see "data_offset" */ 176EVENT_DEFINE_RANGE_FORMAT(offset, config, 32, 63); 177/* u16 */ 178EVENT_DEFINE_RANGE_FORMAT(lpar, config1, 0, 15); 179 180EVENT_DEFINE_RANGE(reserved1, config, 4, 15); 181EVENT_DEFINE_RANGE(reserved2, config1, 16, 63); 182EVENT_DEFINE_RANGE(reserved3, config2, 0, 63); 183 184static struct attribute *format_attrs[] = { 185 &format_attr_domain.attr, 186 &format_attr_offset.attr, 187 &format_attr_core.attr, 188 &format_attr_chip.attr, 189 &format_attr_vcpu.attr, 190 &format_attr_lpar.attr, 191 NULL, 192}; 193 194static const struct attribute_group format_group = { 195 .name = "format", 196 .attrs = format_attrs, 197}; 198 199static struct attribute_group event_group = { 200 .name = "events", 201 /* .attrs is set in init */ 202}; 203 204static struct attribute_group event_desc_group = { 205 .name = "event_descs", 206 /* .attrs is set in init */ 207}; 208 209static struct attribute_group event_long_desc_group = { 210 .name = "event_long_descs", 211 /* .attrs is set in init */ 212}; 213 214static struct kmem_cache *hv_page_cache; 215 216static DEFINE_PER_CPU(int, hv_24x7_txn_flags); 217static DEFINE_PER_CPU(int, hv_24x7_txn_err); 218 219struct hv_24x7_hw { 220 struct perf_event *events[255]; 221}; 222 223static DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw); 224 225/* 226 * request_buffer and result_buffer are not required to be 4k aligned, 227 * but are not allowed to cross any 4k boundary. Aligning them to 4k is 228 * the simplest way to ensure that. 229 */ 230#define H24x7_DATA_BUFFER_SIZE 4096 231static DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); 232static DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); 233 234static unsigned int max_num_requests(int interface_version) 235{ 236 return (H24x7_DATA_BUFFER_SIZE - sizeof(struct hv_24x7_request_buffer)) 237 / H24x7_REQUEST_SIZE(interface_version); 238} 239 240static char *event_name(struct hv_24x7_event_data *ev, int *len) 241{ 242 *len = be16_to_cpu(ev->event_name_len) - 2; 243 return (char *)ev->remainder; 244} 245 246static char *event_desc(struct hv_24x7_event_data *ev, int *len) 247{ 248 unsigned int nl = be16_to_cpu(ev->event_name_len); 249 __be16 *desc_len = (__be16 *)(ev->remainder + nl - 2); 250 251 *len = be16_to_cpu(*desc_len) - 2; 252 return (char *)ev->remainder + nl; 253} 254 255static char *event_long_desc(struct hv_24x7_event_data *ev, int *len) 256{ 257 unsigned int nl = be16_to_cpu(ev->event_name_len); 258 __be16 *desc_len_ = (__be16 *)(ev->remainder + nl - 2); 259 unsigned int desc_len = be16_to_cpu(*desc_len_); 260 __be16 *long_desc_len = (__be16 *)(ev->remainder + nl + desc_len - 2); 261 262 *len = be16_to_cpu(*long_desc_len) - 2; 263 return (char *)ev->remainder + nl + desc_len; 264} 265 266static bool event_fixed_portion_is_within(struct hv_24x7_event_data *ev, 267 void *end) 268{ 269 void *start = ev; 270 271 return (start + offsetof(struct hv_24x7_event_data, remainder)) < end; 272} 273 274/* 275 * Things we don't check: 276 * - padding for desc, name, and long/detailed desc is required to be '\0' 277 * bytes. 278 * 279 * Return NULL if we pass end, 280 * Otherwise return the address of the byte just following the event. 281 */ 282static void *event_end(struct hv_24x7_event_data *ev, void *end) 283{ 284 void *start = ev; 285 __be16 *dl_, *ldl_; 286 unsigned int dl, ldl; 287 unsigned int nl = be16_to_cpu(ev->event_name_len); 288 289 if (nl < 2) { 290 pr_debug("%s: name length too short: %d", __func__, nl); 291 return NULL; 292 } 293 294 if (start + nl > end) { 295 pr_debug("%s: start=%p + nl=%u > end=%p", 296 __func__, start, nl, end); 297 return NULL; 298 } 299 300 dl_ = (__be16 *)(ev->remainder + nl - 2); 301 if (!IS_ALIGNED((uintptr_t)dl_, 2)) 302 pr_warn("desc len not aligned %p", dl_); 303 dl = be16_to_cpu(*dl_); 304 if (dl < 2) { 305 pr_debug("%s: desc len too short: %d", __func__, dl); 306 return NULL; 307 } 308 309 if (start + nl + dl > end) { 310 pr_debug("%s: (start=%p + nl=%u + dl=%u)=%p > end=%p", 311 __func__, start, nl, dl, start + nl + dl, end); 312 return NULL; 313 } 314 315 ldl_ = (__be16 *)(ev->remainder + nl + dl - 2); 316 if (!IS_ALIGNED((uintptr_t)ldl_, 2)) 317 pr_warn("long desc len not aligned %p", ldl_); 318 ldl = be16_to_cpu(*ldl_); 319 if (ldl < 2) { 320 pr_debug("%s: long desc len too short (ldl=%u)", 321 __func__, ldl); 322 return NULL; 323 } 324 325 if (start + nl + dl + ldl > end) { 326 pr_debug("%s: start=%p + nl=%u + dl=%u + ldl=%u > end=%p", 327 __func__, start, nl, dl, ldl, end); 328 return NULL; 329 } 330 331 return start + nl + dl + ldl; 332} 333 334static long h_get_24x7_catalog_page_(unsigned long phys_4096, 335 unsigned long version, unsigned long index) 336{ 337 pr_devel("h_get_24x7_catalog_page(0x%lx, %lu, %lu)", 338 phys_4096, version, index); 339 340 WARN_ON(!IS_ALIGNED(phys_4096, 4096)); 341 342 return plpar_hcall_norets(H_GET_24X7_CATALOG_PAGE, 343 phys_4096, version, index); 344} 345 346static long h_get_24x7_catalog_page(char page[], u64 version, u32 index) 347{ 348 return h_get_24x7_catalog_page_(virt_to_phys(page), 349 version, index); 350} 351 352/* 353 * Each event we find in the catalog, will have a sysfs entry. Format the 354 * data for this sysfs entry based on the event's domain. 355 * 356 * Events belonging to the Chip domain can only be monitored in that domain. 357 * i.e the domain for these events is a fixed/knwon value. 358 * 359 * Events belonging to the Core domain can be monitored either in the physical 360 * core or in one of the virtual CPU domains. So the domain value for these 361 * events must be specified by the user (i.e is a required parameter). Format 362 * the Core events with 'domain=?' so the perf-tool can error check required 363 * parameters. 364 * 365 * NOTE: For the Core domain events, rather than making domain a required 366 * parameter we could default it to PHYS_CORE and allowe users to 367 * override the domain to one of the VCPU domains. 368 * 369 * However, this can make the interface a little inconsistent. 370 * 371 * If we set domain=2 (PHYS_CHIP) and allow user to override this field 372 * the user may be tempted to also modify the "offset=x" field in which 373 * can lead to confusing usage. Consider the HPM_PCYC (offset=0x18) and 374 * HPM_INST (offset=0x20) events. With: 375 * 376 * perf stat -e hv_24x7/HPM_PCYC,offset=0x20/ 377 * 378 * we end up monitoring HPM_INST, while the command line has HPM_PCYC. 379 * 380 * By not assigning a default value to the domain for the Core events, 381 * we can have simple guidelines: 382 * 383 * - Specifying values for parameters with "=?" is required. 384 * 385 * - Specifying (i.e overriding) values for other parameters 386 * is undefined. 387 */ 388static char *event_fmt(struct hv_24x7_event_data *event, unsigned int domain) 389{ 390 const char *sindex; 391 const char *lpar; 392 const char *domain_str; 393 char buf[8]; 394 395 switch (domain) { 396 case HV_PERF_DOMAIN_PHYS_CHIP: 397 snprintf(buf, sizeof(buf), "%d", domain); 398 domain_str = buf; 399 lpar = "0x0"; 400 sindex = "chip"; 401 break; 402 case HV_PERF_DOMAIN_PHYS_CORE: 403 domain_str = "?"; 404 lpar = "0x0"; 405 sindex = "core"; 406 break; 407 default: 408 domain_str = "?"; 409 lpar = "?"; 410 sindex = "vcpu"; 411 } 412 413 return kasprintf(GFP_KERNEL, 414 "domain=%s,offset=0x%x,%s=?,lpar=%s", 415 domain_str, 416 be16_to_cpu(event->event_counter_offs) + 417 be16_to_cpu(event->event_group_record_offs), 418 sindex, 419 lpar); 420} 421 422/* Avoid trusting fw to NUL terminate strings */ 423static char *memdup_to_str(char *maybe_str, int max_len, gfp_t gfp) 424{ 425 return kasprintf(gfp, "%.*s", max_len, maybe_str); 426} 427 428static ssize_t device_show_string(struct device *dev, 429 struct device_attribute *attr, char *buf) 430{ 431 struct dev_ext_attribute *d; 432 433 d = container_of(attr, struct dev_ext_attribute, attr); 434 435 return sprintf(buf, "%s\n", (char *)d->var); 436} 437 438static ssize_t cpumask_show(struct device *dev, 439 struct device_attribute *attr, char *buf) 440{ 441 return cpumap_print_to_pagebuf(true, buf, &hv_24x7_cpumask); 442} 443 444static ssize_t sockets_show(struct device *dev, 445 struct device_attribute *attr, char *buf) 446{ 447 return sprintf(buf, "%d\n", phys_sockets); 448} 449 450static ssize_t chipspersocket_show(struct device *dev, 451 struct device_attribute *attr, char *buf) 452{ 453 return sprintf(buf, "%d\n", phys_chipspersocket); 454} 455 456static ssize_t coresperchip_show(struct device *dev, 457 struct device_attribute *attr, char *buf) 458{ 459 return sprintf(buf, "%d\n", phys_coresperchip); 460} 461 462static struct attribute *device_str_attr_create_(char *name, char *str) 463{ 464 struct dev_ext_attribute *attr = kzalloc(sizeof(*attr), GFP_KERNEL); 465 466 if (!attr) 467 return NULL; 468 469 sysfs_attr_init(&attr->attr.attr); 470 471 attr->var = str; 472 attr->attr.attr.name = name; 473 attr->attr.attr.mode = 0444; 474 attr->attr.show = device_show_string; 475 476 return &attr->attr.attr; 477} 478 479/* 480 * Allocate and initialize strings representing event attributes. 481 * 482 * NOTE: The strings allocated here are never destroyed and continue to 483 * exist till shutdown. This is to allow us to create as many events 484 * from the catalog as possible, even if we encounter errors with some. 485 * In case of changes to error paths in future, these may need to be 486 * freed by the caller. 487 */ 488static struct attribute *device_str_attr_create(char *name, int name_max, 489 int name_nonce, 490 char *str, size_t str_max) 491{ 492 char *n; 493 char *s = memdup_to_str(str, str_max, GFP_KERNEL); 494 struct attribute *a; 495 496 if (!s) 497 return NULL; 498 499 if (!name_nonce) 500 n = kasprintf(GFP_KERNEL, "%.*s", name_max, name); 501 else 502 n = kasprintf(GFP_KERNEL, "%.*s__%d", name_max, name, 503 name_nonce); 504 if (!n) 505 goto out_s; 506 507 a = device_str_attr_create_(n, s); 508 if (!a) 509 goto out_n; 510 511 return a; 512out_n: 513 kfree(n); 514out_s: 515 kfree(s); 516 return NULL; 517} 518 519static struct attribute *event_to_attr(unsigned int ix, 520 struct hv_24x7_event_data *event, 521 unsigned int domain, 522 int nonce) 523{ 524 int event_name_len; 525 char *ev_name, *a_ev_name, *val; 526 struct attribute *attr; 527 528 if (!domain_is_valid(domain)) { 529 pr_warn("catalog event %u has invalid domain %u\n", 530 ix, domain); 531 return NULL; 532 } 533 534 val = event_fmt(event, domain); 535 if (!val) 536 return NULL; 537 538 ev_name = event_name(event, &event_name_len); 539 if (!nonce) 540 a_ev_name = kasprintf(GFP_KERNEL, "%.*s", 541 (int)event_name_len, ev_name); 542 else 543 a_ev_name = kasprintf(GFP_KERNEL, "%.*s__%d", 544 (int)event_name_len, ev_name, nonce); 545 546 if (!a_ev_name) 547 goto out_val; 548 549 attr = device_str_attr_create_(a_ev_name, val); 550 if (!attr) 551 goto out_name; 552 553 return attr; 554out_name: 555 kfree(a_ev_name); 556out_val: 557 kfree(val); 558 return NULL; 559} 560 561static struct attribute *event_to_desc_attr(struct hv_24x7_event_data *event, 562 int nonce) 563{ 564 int nl, dl; 565 char *name = event_name(event, &nl); 566 char *desc = event_desc(event, &dl); 567 568 /* If there isn't a description, don't create the sysfs file */ 569 if (!dl) 570 return NULL; 571 572 return device_str_attr_create(name, nl, nonce, desc, dl); 573} 574 575static struct attribute * 576event_to_long_desc_attr(struct hv_24x7_event_data *event, int nonce) 577{ 578 int nl, dl; 579 char *name = event_name(event, &nl); 580 char *desc = event_long_desc(event, &dl); 581 582 /* If there isn't a description, don't create the sysfs file */ 583 if (!dl) 584 return NULL; 585 586 return device_str_attr_create(name, nl, nonce, desc, dl); 587} 588 589static int event_data_to_attrs(unsigned int ix, struct attribute **attrs, 590 struct hv_24x7_event_data *event, int nonce) 591{ 592 *attrs = event_to_attr(ix, event, event->domain, nonce); 593 if (!*attrs) 594 return -1; 595 596 return 0; 597} 598 599/* */ 600struct event_uniq { 601 struct rb_node node; 602 const char *name; 603 int nl; 604 unsigned int ct; 605 unsigned int domain; 606}; 607 608static int memord(const void *d1, size_t s1, const void *d2, size_t s2) 609{ 610 if (s1 < s2) 611 return 1; 612 if (s1 > s2) 613 return -1; 614 615 return memcmp(d1, d2, s1); 616} 617 618static int ev_uniq_ord(const void *v1, size_t s1, unsigned int d1, 619 const void *v2, size_t s2, unsigned int d2) 620{ 621 int r = memord(v1, s1, v2, s2); 622 623 if (r) 624 return r; 625 if (d1 > d2) 626 return 1; 627 if (d2 > d1) 628 return -1; 629 return 0; 630} 631 632static int event_uniq_add(struct rb_root *root, const char *name, int nl, 633 unsigned int domain) 634{ 635 struct rb_node **new = &(root->rb_node), *parent = NULL; 636 struct event_uniq *data; 637 638 /* Figure out where to put new node */ 639 while (*new) { 640 struct event_uniq *it; 641 int result; 642 643 it = rb_entry(*new, struct event_uniq, node); 644 result = ev_uniq_ord(name, nl, domain, it->name, it->nl, 645 it->domain); 646 647 parent = *new; 648 if (result < 0) 649 new = &((*new)->rb_left); 650 else if (result > 0) 651 new = &((*new)->rb_right); 652 else { 653 it->ct++; 654 pr_info("found a duplicate event %.*s, ct=%u\n", nl, 655 name, it->ct); 656 return it->ct; 657 } 658 } 659 660 data = kmalloc(sizeof(*data), GFP_KERNEL); 661 if (!data) 662 return -ENOMEM; 663 664 *data = (struct event_uniq) { 665 .name = name, 666 .nl = nl, 667 .ct = 0, 668 .domain = domain, 669 }; 670 671 /* Add new node and rebalance tree. */ 672 rb_link_node(&data->node, parent, new); 673 rb_insert_color(&data->node, root); 674 675 /* data->ct */ 676 return 0; 677} 678 679static void event_uniq_destroy(struct rb_root *root) 680{ 681 /* 682 * the strings we point to are in the giant block of memory filled by 683 * the catalog, and are freed separately. 684 */ 685 struct event_uniq *pos, *n; 686 687 rbtree_postorder_for_each_entry_safe(pos, n, root, node) 688 kfree(pos); 689} 690 691 692/* 693 * ensure the event structure's sizes are self consistent and don't cause us to 694 * read outside of the event 695 * 696 * On success, return the event length in bytes. 697 * Otherwise, return -1 (and print as appropriate). 698 */ 699static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, 700 size_t event_idx, 701 size_t event_data_bytes, 702 size_t event_entry_count, 703 size_t offset, void *end) 704{ 705 ssize_t ev_len; 706 void *ev_end, *calc_ev_end; 707 708 if (offset >= event_data_bytes) 709 return -1; 710 711 if (event_idx >= event_entry_count) { 712 pr_devel("catalog event data has %zu bytes of padding after last event\n", 713 event_data_bytes - offset); 714 return -1; 715 } 716 717 if (!event_fixed_portion_is_within(event, end)) { 718 pr_warn("event %zu fixed portion is not within range\n", 719 event_idx); 720 return -1; 721 } 722 723 ev_len = be16_to_cpu(event->length); 724 725 if (ev_len % 16) 726 pr_info("event %zu has length %zu not divisible by 16: event=%pK\n", 727 event_idx, ev_len, event); 728 729 ev_end = (__u8 *)event + ev_len; 730 if (ev_end > end) { 731 pr_warn("event %zu has .length=%zu, ends after buffer end: ev_end=%pK > end=%pK, offset=%zu\n", 732 event_idx, ev_len, ev_end, end, 733 offset); 734 return -1; 735 } 736 737 calc_ev_end = event_end(event, end); 738 if (!calc_ev_end) { 739 pr_warn("event %zu has a calculated length which exceeds buffer length %zu: event=%pK end=%pK, offset=%zu\n", 740 event_idx, event_data_bytes, event, end, 741 offset); 742 return -1; 743 } 744 745 if (calc_ev_end > ev_end) { 746 pr_warn("event %zu exceeds its own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n", 747 event_idx, event, ev_end, offset, calc_ev_end); 748 return -1; 749 } 750 751 return ev_len; 752} 753 754/* 755 * Return true incase of invalid or dummy events with names like RESERVED* 756 */ 757static bool ignore_event(const char *name) 758{ 759 return strncmp(name, "RESERVED", 8) == 0; 760} 761 762#define MAX_4K (SIZE_MAX / 4096) 763 764static int create_events_from_catalog(struct attribute ***events_, 765 struct attribute ***event_descs_, 766 struct attribute ***event_long_descs_) 767{ 768 long hret; 769 size_t catalog_len, catalog_page_len, event_entry_count, 770 event_data_len, event_data_offs, 771 event_data_bytes, junk_events, event_idx, event_attr_ct, i, 772 attr_max, event_idx_last, desc_ct, long_desc_ct; 773 ssize_t ct, ev_len; 774 uint64_t catalog_version_num; 775 struct attribute **events, **event_descs, **event_long_descs; 776 struct hv_24x7_catalog_page_0 *page_0 = 777 kmem_cache_alloc(hv_page_cache, GFP_KERNEL); 778 void *page = page_0; 779 void *event_data, *end; 780 struct hv_24x7_event_data *event; 781 struct rb_root ev_uniq = RB_ROOT; 782 int ret = 0; 783 784 if (!page) { 785 ret = -ENOMEM; 786 goto e_out; 787 } 788 789 hret = h_get_24x7_catalog_page(page, 0, 0); 790 if (hret) { 791 ret = -EIO; 792 goto e_free; 793 } 794 795 catalog_version_num = be64_to_cpu(page_0->version); 796 catalog_page_len = be32_to_cpu(page_0->length); 797 798 if (MAX_4K < catalog_page_len) { 799 pr_err("invalid page count: %zu\n", catalog_page_len); 800 ret = -EIO; 801 goto e_free; 802 } 803 804 catalog_len = catalog_page_len * 4096; 805 806 event_entry_count = be16_to_cpu(page_0->event_entry_count); 807 event_data_offs = be16_to_cpu(page_0->event_data_offs); 808 event_data_len = be16_to_cpu(page_0->event_data_len); 809 810 pr_devel("cv %llu cl %zu eec %zu edo %zu edl %zu\n", 811 catalog_version_num, catalog_len, 812 event_entry_count, event_data_offs, event_data_len); 813 814 if ((MAX_4K < event_data_len) 815 || (MAX_4K < event_data_offs) 816 || (MAX_4K - event_data_offs < event_data_len)) { 817 pr_err("invalid event data offs %zu and/or len %zu\n", 818 event_data_offs, event_data_len); 819 ret = -EIO; 820 goto e_free; 821 } 822 823 if ((event_data_offs + event_data_len) > catalog_page_len) { 824 pr_err("event data %zu-%zu does not fit inside catalog 0-%zu\n", 825 event_data_offs, 826 event_data_offs + event_data_len, 827 catalog_page_len); 828 ret = -EIO; 829 goto e_free; 830 } 831 832 if (SIZE_MAX - 1 < event_entry_count) { 833 pr_err("event_entry_count %zu is invalid\n", event_entry_count); 834 ret = -EIO; 835 goto e_free; 836 } 837 838 event_data_bytes = event_data_len * 4096; 839 840 /* 841 * event data can span several pages, events can cross between these 842 * pages. Use vmalloc to make this easier. 843 */ 844 event_data = vmalloc(event_data_bytes); 845 if (!event_data) { 846 pr_err("could not allocate event data\n"); 847 ret = -ENOMEM; 848 goto e_free; 849 } 850 851 end = event_data + event_data_bytes; 852 853 /* 854 * using vmalloc_to_phys() like this only works if PAGE_SIZE is 855 * divisible by 4096 856 */ 857 BUILD_BUG_ON(PAGE_SIZE % 4096); 858 859 for (i = 0; i < event_data_len; i++) { 860 hret = h_get_24x7_catalog_page_( 861 vmalloc_to_phys(event_data + i * 4096), 862 catalog_version_num, 863 i + event_data_offs); 864 if (hret) { 865 pr_err("Failed to get event data in page %zu: rc=%ld\n", 866 i + event_data_offs, hret); 867 ret = -EIO; 868 goto e_event_data; 869 } 870 } 871 872 /* 873 * scan the catalog to determine the number of attributes we need, and 874 * verify it at the same time. 875 */ 876 for (junk_events = 0, event = event_data, event_idx = 0, attr_max = 0; 877 ; 878 event_idx++, event = (void *)event + ev_len) { 879 size_t offset = (void *)event - (void *)event_data; 880 char *name; 881 int nl; 882 883 ev_len = catalog_event_len_validate(event, event_idx, 884 event_data_bytes, 885 event_entry_count, 886 offset, end); 887 if (ev_len < 0) 888 break; 889 890 name = event_name(event, &nl); 891 892 if (ignore_event(name)) { 893 junk_events++; 894 continue; 895 } 896 if (event->event_group_record_len == 0) { 897 pr_devel("invalid event %zu (%.*s): group_record_len == 0, skipping\n", 898 event_idx, nl, name); 899 junk_events++; 900 continue; 901 } 902 903 if (!catalog_entry_domain_is_valid(event->domain)) { 904 pr_info("event %zu (%.*s) has invalid domain %d\n", 905 event_idx, nl, name, event->domain); 906 junk_events++; 907 continue; 908 } 909 910 attr_max++; 911 } 912 913 event_idx_last = event_idx; 914 if (event_idx_last != event_entry_count) 915 pr_warn("event buffer ended before listed # of events were parsed (got %zu, wanted %zu, junk %zu)\n", 916 event_idx_last, event_entry_count, junk_events); 917 918 events = kmalloc_array(attr_max + 1, sizeof(*events), GFP_KERNEL); 919 if (!events) { 920 ret = -ENOMEM; 921 goto e_event_data; 922 } 923 924 event_descs = kmalloc_array(event_idx + 1, sizeof(*event_descs), 925 GFP_KERNEL); 926 if (!event_descs) { 927 ret = -ENOMEM; 928 goto e_event_attrs; 929 } 930 931 event_long_descs = kmalloc_array(event_idx + 1, 932 sizeof(*event_long_descs), GFP_KERNEL); 933 if (!event_long_descs) { 934 ret = -ENOMEM; 935 goto e_event_descs; 936 } 937 938 /* Iterate over the catalog filling in the attribute vector */ 939 for (junk_events = 0, event_attr_ct = 0, desc_ct = 0, long_desc_ct = 0, 940 event = event_data, event_idx = 0; 941 event_idx < event_idx_last; 942 event_idx++, ev_len = be16_to_cpu(event->length), 943 event = (void *)event + ev_len) { 944 char *name; 945 int nl; 946 int nonce; 947 /* 948 * these are the only "bad" events that are intermixed and that 949 * we can ignore without issue. make sure to skip them here 950 */ 951 if (event->event_group_record_len == 0) 952 continue; 953 if (!catalog_entry_domain_is_valid(event->domain)) 954 continue; 955 956 name = event_name(event, &nl); 957 if (ignore_event(name)) 958 continue; 959 960 nonce = event_uniq_add(&ev_uniq, name, nl, event->domain); 961 ct = event_data_to_attrs(event_idx, events + event_attr_ct, 962 event, nonce); 963 if (ct < 0) { 964 pr_warn("event %zu (%.*s) creation failure, skipping\n", 965 event_idx, nl, name); 966 junk_events++; 967 } else { 968 event_attr_ct++; 969 event_descs[desc_ct] = event_to_desc_attr(event, nonce); 970 if (event_descs[desc_ct]) 971 desc_ct++; 972 event_long_descs[long_desc_ct] = 973 event_to_long_desc_attr(event, nonce); 974 if (event_long_descs[long_desc_ct]) 975 long_desc_ct++; 976 } 977 } 978 979 pr_info("read %zu catalog entries, created %zu event attrs (%zu failures), %zu descs\n", 980 event_idx, event_attr_ct, junk_events, desc_ct); 981 982 events[event_attr_ct] = NULL; 983 event_descs[desc_ct] = NULL; 984 event_long_descs[long_desc_ct] = NULL; 985 986 event_uniq_destroy(&ev_uniq); 987 vfree(event_data); 988 kmem_cache_free(hv_page_cache, page); 989 990 *events_ = events; 991 *event_descs_ = event_descs; 992 *event_long_descs_ = event_long_descs; 993 return 0; 994 995e_event_descs: 996 kfree(event_descs); 997e_event_attrs: 998 kfree(events); 999e_event_data: 1000 vfree(event_data); 1001e_free: 1002 kmem_cache_free(hv_page_cache, page); 1003e_out: 1004 *events_ = NULL; 1005 *event_descs_ = NULL; 1006 *event_long_descs_ = NULL; 1007 return ret; 1008} 1009 1010static ssize_t catalog_read(struct file *filp, struct kobject *kobj, 1011 struct bin_attribute *bin_attr, char *buf, 1012 loff_t offset, size_t count) 1013{ 1014 long hret; 1015 ssize_t ret = 0; 1016 size_t catalog_len = 0, catalog_page_len = 0; 1017 loff_t page_offset = 0; 1018 loff_t offset_in_page; 1019 size_t copy_len; 1020 uint64_t catalog_version_num = 0; 1021 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); 1022 struct hv_24x7_catalog_page_0 *page_0 = page; 1023 1024 if (!page) 1025 return -ENOMEM; 1026 1027 hret = h_get_24x7_catalog_page(page, 0, 0); 1028 if (hret) { 1029 ret = -EIO; 1030 goto e_free; 1031 } 1032 1033 catalog_version_num = be64_to_cpu(page_0->version); 1034 catalog_page_len = be32_to_cpu(page_0->length); 1035 catalog_len = catalog_page_len * 4096; 1036 1037 page_offset = offset / 4096; 1038 offset_in_page = offset % 4096; 1039 1040 if (page_offset >= catalog_page_len) 1041 goto e_free; 1042 1043 if (page_offset != 0) { 1044 hret = h_get_24x7_catalog_page(page, catalog_version_num, 1045 page_offset); 1046 if (hret) { 1047 ret = -EIO; 1048 goto e_free; 1049 } 1050 } 1051 1052 copy_len = 4096 - offset_in_page; 1053 if (copy_len > count) 1054 copy_len = count; 1055 1056 memcpy(buf, page+offset_in_page, copy_len); 1057 ret = copy_len; 1058 1059e_free: 1060 if (hret) 1061 pr_err("h_get_24x7_catalog_page(ver=%lld, page=%lld) failed:" 1062 " rc=%ld\n", 1063 catalog_version_num, page_offset, hret); 1064 kmem_cache_free(hv_page_cache, page); 1065 1066 pr_devel("catalog_read: offset=%lld(%lld) count=%zu " 1067 "catalog_len=%zu(%zu) => %zd\n", offset, page_offset, 1068 count, catalog_len, catalog_page_len, ret); 1069 1070 return ret; 1071} 1072 1073static ssize_t domains_show(struct device *dev, struct device_attribute *attr, 1074 char *page) 1075{ 1076 int d, n, count = 0; 1077 const char *str; 1078 1079 for (d = 0; d < HV_PERF_DOMAIN_MAX; d++) { 1080 str = domain_name(d); 1081 if (!str) 1082 continue; 1083 1084 n = sprintf(page, "%d: %s\n", d, str); 1085 if (n < 0) 1086 break; 1087 1088 count += n; 1089 page += n; 1090 } 1091 return count; 1092} 1093 1094#define PAGE_0_ATTR(_name, _fmt, _expr) \ 1095static ssize_t _name##_show(struct device *dev, \ 1096 struct device_attribute *dev_attr, \ 1097 char *buf) \ 1098{ \ 1099 long hret; \ 1100 ssize_t ret = 0; \ 1101 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); \ 1102 struct hv_24x7_catalog_page_0 *page_0 = page; \ 1103 if (!page) \ 1104 return -ENOMEM; \ 1105 hret = h_get_24x7_catalog_page(page, 0, 0); \ 1106 if (hret) { \ 1107 ret = -EIO; \ 1108 goto e_free; \ 1109 } \ 1110 ret = sprintf(buf, _fmt, _expr); \ 1111e_free: \ 1112 kmem_cache_free(hv_page_cache, page); \ 1113 return ret; \ 1114} \ 1115static DEVICE_ATTR_RO(_name) 1116 1117PAGE_0_ATTR(catalog_version, "%lld\n", 1118 (unsigned long long)be64_to_cpu(page_0->version)); 1119PAGE_0_ATTR(catalog_len, "%lld\n", 1120 (unsigned long long)be32_to_cpu(page_0->length) * 4096); 1121static BIN_ATTR_RO(catalog, 0/* real length varies */); 1122static DEVICE_ATTR_RO(domains); 1123static DEVICE_ATTR_RO(sockets); 1124static DEVICE_ATTR_RO(chipspersocket); 1125static DEVICE_ATTR_RO(coresperchip); 1126static DEVICE_ATTR_RO(cpumask); 1127 1128static struct bin_attribute *if_bin_attrs[] = { 1129 &bin_attr_catalog, 1130 NULL, 1131}; 1132 1133static struct attribute *cpumask_attrs[] = { 1134 &dev_attr_cpumask.attr, 1135 NULL, 1136}; 1137 1138static const struct attribute_group cpumask_attr_group = { 1139 .attrs = cpumask_attrs, 1140}; 1141 1142static struct attribute *if_attrs[] = { 1143 &dev_attr_catalog_len.attr, 1144 &dev_attr_catalog_version.attr, 1145 &dev_attr_domains.attr, 1146 &dev_attr_sockets.attr, 1147 &dev_attr_chipspersocket.attr, 1148 &dev_attr_coresperchip.attr, 1149 NULL, 1150}; 1151 1152static const struct attribute_group if_group = { 1153 .name = "interface", 1154 .bin_attrs = if_bin_attrs, 1155 .attrs = if_attrs, 1156}; 1157 1158static const struct attribute_group *attr_groups[] = { 1159 &format_group, 1160 &event_group, 1161 &event_desc_group, 1162 &event_long_desc_group, 1163 &if_group, 1164 &cpumask_attr_group, 1165 NULL, 1166}; 1167 1168/* 1169 * Start the process for a new H_GET_24x7_DATA hcall. 1170 */ 1171static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer, 1172 struct hv_24x7_data_result_buffer *result_buffer) 1173{ 1174 1175 memset(request_buffer, 0, H24x7_DATA_BUFFER_SIZE); 1176 memset(result_buffer, 0, H24x7_DATA_BUFFER_SIZE); 1177 1178 request_buffer->interface_version = interface_version; 1179 /* memset above set request_buffer->num_requests to 0 */ 1180} 1181 1182/* 1183 * Commit (i.e perform) the H_GET_24x7_DATA hcall using the data collected 1184 * by 'init_24x7_request()' and 'add_event_to_24x7_request()'. 1185 */ 1186static int make_24x7_request(struct hv_24x7_request_buffer *request_buffer, 1187 struct hv_24x7_data_result_buffer *result_buffer) 1188{ 1189 long ret; 1190 1191 /* 1192 * NOTE: Due to variable number of array elements in request and 1193 * result buffer(s), sizeof() is not reliable. Use the actual 1194 * allocated buffer size, H24x7_DATA_BUFFER_SIZE. 1195 */ 1196 ret = plpar_hcall_norets(H_GET_24X7_DATA, 1197 virt_to_phys(request_buffer), H24x7_DATA_BUFFER_SIZE, 1198 virt_to_phys(result_buffer), H24x7_DATA_BUFFER_SIZE); 1199 1200 if (ret) { 1201 struct hv_24x7_request *req; 1202 1203 req = request_buffer->requests; 1204 pr_notice_ratelimited("hcall failed: [%d %#x %#x %d] => ret 0x%lx (%ld) detail=0x%x failing ix=%x\n", 1205 req->performance_domain, req->data_offset, 1206 req->starting_ix, req->starting_lpar_ix, 1207 ret, ret, result_buffer->detailed_rc, 1208 result_buffer->failing_request_ix); 1209 return -EIO; 1210 } 1211 1212 return 0; 1213} 1214 1215/* 1216 * Add the given @event to the next slot in the 24x7 request_buffer. 1217 * 1218 * Note that H_GET_24X7_DATA hcall allows reading several counters' 1219 * values in a single HCALL. We expect the caller to add events to the 1220 * request buffer one by one, make the HCALL and process the results. 1221 */ 1222static int add_event_to_24x7_request(struct perf_event *event, 1223 struct hv_24x7_request_buffer *request_buffer) 1224{ 1225 u16 idx; 1226 int i; 1227 size_t req_size; 1228 struct hv_24x7_request *req; 1229 1230 if (request_buffer->num_requests >= 1231 max_num_requests(request_buffer->interface_version)) { 1232 pr_devel("Too many requests for 24x7 HCALL %d\n", 1233 request_buffer->num_requests); 1234 return -EINVAL; 1235 } 1236 1237 switch (event_get_domain(event)) { 1238 case HV_PERF_DOMAIN_PHYS_CHIP: 1239 idx = event_get_chip(event); 1240 break; 1241 case HV_PERF_DOMAIN_PHYS_CORE: 1242 idx = event_get_core(event); 1243 break; 1244 default: 1245 idx = event_get_vcpu(event); 1246 } 1247 1248 req_size = H24x7_REQUEST_SIZE(request_buffer->interface_version); 1249 1250 i = request_buffer->num_requests++; 1251 req = (void *) request_buffer->requests + i * req_size; 1252 1253 req->performance_domain = event_get_domain(event); 1254 req->data_size = cpu_to_be16(8); 1255 req->data_offset = cpu_to_be32(event_get_offset(event)); 1256 req->starting_lpar_ix = cpu_to_be16(event_get_lpar(event)); 1257 req->max_num_lpars = cpu_to_be16(1); 1258 req->starting_ix = cpu_to_be16(idx); 1259 req->max_ix = cpu_to_be16(1); 1260 1261 if (request_buffer->interface_version > 1) { 1262 if (domain_needs_aggregation(req->performance_domain)) 1263 req->max_num_thread_groups = -1; 1264 else if (req->performance_domain != HV_PERF_DOMAIN_PHYS_CHIP) { 1265 req->starting_thread_group_ix = idx % 2; 1266 req->max_num_thread_groups = 1; 1267 } 1268 } 1269 1270 return 0; 1271} 1272 1273/** 1274 * get_count_from_result - get event count from all result elements in result 1275 * 1276 * If the event corresponding to this result needs aggregation of the result 1277 * element values, then this function does that. 1278 * 1279 * @event: Event associated with @res. 1280 * @resb: Result buffer containing @res. 1281 * @res: Result to work on. 1282 * @countp: Output variable containing the event count. 1283 * @next: Optional output variable pointing to the next result in @resb. 1284 */ 1285static int get_count_from_result(struct perf_event *event, 1286 struct hv_24x7_data_result_buffer *resb, 1287 struct hv_24x7_result *res, u64 *countp, 1288 struct hv_24x7_result **next) 1289{ 1290 u16 num_elements = be16_to_cpu(res->num_elements_returned); 1291 u16 data_size = be16_to_cpu(res->result_element_data_size); 1292 unsigned int data_offset; 1293 void *element_data; 1294 int i; 1295 u64 count; 1296 1297 /* 1298 * We can bail out early if the result is empty. 1299 */ 1300 if (!num_elements) { 1301 pr_debug("Result of request %hhu is empty, nothing to do\n", 1302 res->result_ix); 1303 1304 if (next) 1305 *next = (struct hv_24x7_result *) res->elements; 1306 1307 return -ENODATA; 1308 } 1309 1310 /* 1311 * Since we always specify 1 as the maximum for the smallest resource 1312 * we're requesting, there should to be only one element per result. 1313 * Except when an event needs aggregation, in which case there are more. 1314 */ 1315 if (num_elements != 1 && 1316 !domain_needs_aggregation(event_get_domain(event))) { 1317 pr_err("Error: result of request %hhu has %hu elements\n", 1318 res->result_ix, num_elements); 1319 1320 return -EIO; 1321 } 1322 1323 if (data_size != sizeof(u64)) { 1324 pr_debug("Error: result of request %hhu has data of %hu bytes\n", 1325 res->result_ix, data_size); 1326 1327 return -ENOTSUPP; 1328 } 1329 1330 if (resb->interface_version == 1) 1331 data_offset = offsetof(struct hv_24x7_result_element_v1, 1332 element_data); 1333 else 1334 data_offset = offsetof(struct hv_24x7_result_element_v2, 1335 element_data); 1336 1337 /* Go through the result elements in the result. */ 1338 for (i = count = 0, element_data = res->elements + data_offset; 1339 i < num_elements; 1340 i++, element_data += data_size + data_offset) 1341 count += be64_to_cpu(*((__be64 *)element_data)); 1342 1343 *countp = count; 1344 1345 /* The next result is after the last result element. */ 1346 if (next) 1347 *next = element_data - data_offset; 1348 1349 return 0; 1350} 1351 1352static int single_24x7_request(struct perf_event *event, u64 *count) 1353{ 1354 int ret; 1355 struct hv_24x7_request_buffer *request_buffer; 1356 struct hv_24x7_data_result_buffer *result_buffer; 1357 1358 BUILD_BUG_ON(sizeof(*request_buffer) > 4096); 1359 BUILD_BUG_ON(sizeof(*result_buffer) > 4096); 1360 1361 request_buffer = (void *)get_cpu_var(hv_24x7_reqb); 1362 result_buffer = (void *)get_cpu_var(hv_24x7_resb); 1363 1364 init_24x7_request(request_buffer, result_buffer); 1365 1366 ret = add_event_to_24x7_request(event, request_buffer); 1367 if (ret) 1368 goto out; 1369 1370 ret = make_24x7_request(request_buffer, result_buffer); 1371 if (ret) 1372 goto out; 1373 1374 /* process result from hcall */ 1375 ret = get_count_from_result(event, result_buffer, 1376 result_buffer->results, count, NULL); 1377 1378out: 1379 put_cpu_var(hv_24x7_reqb); 1380 put_cpu_var(hv_24x7_resb); 1381 return ret; 1382} 1383 1384 1385static int h_24x7_event_init(struct perf_event *event) 1386{ 1387 struct hv_perf_caps caps; 1388 unsigned int domain; 1389 unsigned long hret; 1390 u64 ct; 1391 1392 /* Not our event */ 1393 if (event->attr.type != event->pmu->type) 1394 return -ENOENT; 1395 1396 /* Unused areas must be 0 */ 1397 if (event_get_reserved1(event) || 1398 event_get_reserved2(event) || 1399 event_get_reserved3(event)) { 1400 pr_devel("reserved set when forbidden 0x%llx(0x%llx) 0x%llx(0x%llx) 0x%llx(0x%llx)\n", 1401 event->attr.config, 1402 event_get_reserved1(event), 1403 event->attr.config1, 1404 event_get_reserved2(event), 1405 event->attr.config2, 1406 event_get_reserved3(event)); 1407 return -EINVAL; 1408 } 1409 1410 /* no branch sampling */ 1411 if (has_branch_stack(event)) 1412 return -EOPNOTSUPP; 1413 1414 /* offset must be 8 byte aligned */ 1415 if (event_get_offset(event) % 8) { 1416 pr_devel("bad alignment\n"); 1417 return -EINVAL; 1418 } 1419 1420 domain = event_get_domain(event); 1421 if (domain == 0 || domain >= HV_PERF_DOMAIN_MAX) { 1422 pr_devel("invalid domain %d\n", domain); 1423 return -EINVAL; 1424 } 1425 1426 hret = hv_perf_caps_get(&caps); 1427 if (hret) { 1428 pr_devel("could not get capabilities: rc=%ld\n", hret); 1429 return -EIO; 1430 } 1431 1432 /* Physical domains & other lpars require extra capabilities */ 1433 if (!caps.collect_privileged && (is_physical_domain(domain) || 1434 (event_get_lpar(event) != event_get_lpar_max()))) { 1435 pr_devel("hv permissions disallow: is_physical_domain:%d, lpar=0x%llx\n", 1436 is_physical_domain(domain), 1437 event_get_lpar(event)); 1438 return -EACCES; 1439 } 1440 1441 /* Get the initial value of the counter for this event */ 1442 if (single_24x7_request(event, &ct)) { 1443 pr_devel("test hcall failed\n"); 1444 return -EIO; 1445 } 1446 (void)local64_xchg(&event->hw.prev_count, ct); 1447 1448 return 0; 1449} 1450 1451static u64 h_24x7_get_value(struct perf_event *event) 1452{ 1453 u64 ct; 1454 1455 if (single_24x7_request(event, &ct)) 1456 /* We checked this in event init, shouldn't fail here... */ 1457 return 0; 1458 1459 return ct; 1460} 1461 1462static void update_event_count(struct perf_event *event, u64 now) 1463{ 1464 s64 prev; 1465 1466 prev = local64_xchg(&event->hw.prev_count, now); 1467 local64_add(now - prev, &event->count); 1468} 1469 1470static void h_24x7_event_read(struct perf_event *event) 1471{ 1472 u64 now; 1473 struct hv_24x7_request_buffer *request_buffer; 1474 struct hv_24x7_hw *h24x7hw; 1475 int txn_flags; 1476 1477 txn_flags = __this_cpu_read(hv_24x7_txn_flags); 1478 1479 /* 1480 * If in a READ transaction, add this counter to the list of 1481 * counters to read during the next HCALL (i.e commit_txn()). 1482 * If not in a READ transaction, go ahead and make the HCALL 1483 * to read this counter by itself. 1484 */ 1485 1486 if (txn_flags & PERF_PMU_TXN_READ) { 1487 int i; 1488 int ret; 1489 1490 if (__this_cpu_read(hv_24x7_txn_err)) 1491 return; 1492 1493 request_buffer = (void *)get_cpu_var(hv_24x7_reqb); 1494 1495 ret = add_event_to_24x7_request(event, request_buffer); 1496 if (ret) { 1497 __this_cpu_write(hv_24x7_txn_err, ret); 1498 } else { 1499 /* 1500 * Associate the event with the HCALL request index, 1501 * so ->commit_txn() can quickly find/update count. 1502 */ 1503 i = request_buffer->num_requests - 1; 1504 1505 h24x7hw = &get_cpu_var(hv_24x7_hw); 1506 h24x7hw->events[i] = event; 1507 put_cpu_var(h24x7hw); 1508 } 1509 1510 put_cpu_var(hv_24x7_reqb); 1511 } else { 1512 now = h_24x7_get_value(event); 1513 update_event_count(event, now); 1514 } 1515} 1516 1517static void h_24x7_event_start(struct perf_event *event, int flags) 1518{ 1519 if (flags & PERF_EF_RELOAD) 1520 local64_set(&event->hw.prev_count, h_24x7_get_value(event)); 1521} 1522 1523static void h_24x7_event_stop(struct perf_event *event, int flags) 1524{ 1525 h_24x7_event_read(event); 1526} 1527 1528static int h_24x7_event_add(struct perf_event *event, int flags) 1529{ 1530 if (flags & PERF_EF_START) 1531 h_24x7_event_start(event, flags); 1532 1533 return 0; 1534} 1535 1536/* 1537 * 24x7 counters only support READ transactions. They are 1538 * always counting and dont need/support ADD transactions. 1539 * Cache the flags, but otherwise ignore transactions that 1540 * are not PERF_PMU_TXN_READ. 1541 */ 1542static void h_24x7_event_start_txn(struct pmu *pmu, unsigned int flags) 1543{ 1544 struct hv_24x7_request_buffer *request_buffer; 1545 struct hv_24x7_data_result_buffer *result_buffer; 1546 1547 /* We should not be called if we are already in a txn */ 1548 WARN_ON_ONCE(__this_cpu_read(hv_24x7_txn_flags)); 1549 1550 __this_cpu_write(hv_24x7_txn_flags, flags); 1551 if (flags & ~PERF_PMU_TXN_READ) 1552 return; 1553 1554 request_buffer = (void *)get_cpu_var(hv_24x7_reqb); 1555 result_buffer = (void *)get_cpu_var(hv_24x7_resb); 1556 1557 init_24x7_request(request_buffer, result_buffer); 1558 1559 put_cpu_var(hv_24x7_resb); 1560 put_cpu_var(hv_24x7_reqb); 1561} 1562 1563/* 1564 * Clean up transaction state. 1565 * 1566 * NOTE: Ignore state of request and result buffers for now. 1567 * We will initialize them during the next read/txn. 1568 */ 1569static void reset_txn(void) 1570{ 1571 __this_cpu_write(hv_24x7_txn_flags, 0); 1572 __this_cpu_write(hv_24x7_txn_err, 0); 1573} 1574 1575/* 1576 * 24x7 counters only support READ transactions. They are always counting 1577 * and dont need/support ADD transactions. Clear ->txn_flags but otherwise 1578 * ignore transactions that are not of type PERF_PMU_TXN_READ. 1579 * 1580 * For READ transactions, submit all pending 24x7 requests (i.e requests 1581 * that were queued by h_24x7_event_read()), to the hypervisor and update 1582 * the event counts. 1583 */ 1584static int h_24x7_event_commit_txn(struct pmu *pmu) 1585{ 1586 struct hv_24x7_request_buffer *request_buffer; 1587 struct hv_24x7_data_result_buffer *result_buffer; 1588 struct hv_24x7_result *res, *next_res; 1589 u64 count; 1590 int i, ret, txn_flags; 1591 struct hv_24x7_hw *h24x7hw; 1592 1593 txn_flags = __this_cpu_read(hv_24x7_txn_flags); 1594 WARN_ON_ONCE(!txn_flags); 1595 1596 ret = 0; 1597 if (txn_flags & ~PERF_PMU_TXN_READ) 1598 goto out; 1599 1600 ret = __this_cpu_read(hv_24x7_txn_err); 1601 if (ret) 1602 goto out; 1603 1604 request_buffer = (void *)get_cpu_var(hv_24x7_reqb); 1605 result_buffer = (void *)get_cpu_var(hv_24x7_resb); 1606 1607 ret = make_24x7_request(request_buffer, result_buffer); 1608 if (ret) 1609 goto put_reqb; 1610 1611 h24x7hw = &get_cpu_var(hv_24x7_hw); 1612 1613 /* Go through results in the result buffer to update event counts. */ 1614 for (i = 0, res = result_buffer->results; 1615 i < result_buffer->num_results; i++, res = next_res) { 1616 struct perf_event *event = h24x7hw->events[res->result_ix]; 1617 1618 ret = get_count_from_result(event, result_buffer, res, &count, 1619 &next_res); 1620 if (ret) 1621 break; 1622 1623 update_event_count(event, count); 1624 } 1625 1626 put_cpu_var(hv_24x7_hw); 1627 1628put_reqb: 1629 put_cpu_var(hv_24x7_resb); 1630 put_cpu_var(hv_24x7_reqb); 1631out: 1632 reset_txn(); 1633 return ret; 1634} 1635 1636/* 1637 * 24x7 counters only support READ transactions. They are always counting 1638 * and dont need/support ADD transactions. However, regardless of type 1639 * of transaction, all we need to do is cleanup, so we don't have to check 1640 * the type of transaction. 1641 */ 1642static void h_24x7_event_cancel_txn(struct pmu *pmu) 1643{ 1644 WARN_ON_ONCE(!__this_cpu_read(hv_24x7_txn_flags)); 1645 reset_txn(); 1646} 1647 1648static struct pmu h_24x7_pmu = { 1649 .task_ctx_nr = perf_invalid_context, 1650 1651 .name = "hv_24x7", 1652 .attr_groups = attr_groups, 1653 .event_init = h_24x7_event_init, 1654 .add = h_24x7_event_add, 1655 .del = h_24x7_event_stop, 1656 .start = h_24x7_event_start, 1657 .stop = h_24x7_event_stop, 1658 .read = h_24x7_event_read, 1659 .start_txn = h_24x7_event_start_txn, 1660 .commit_txn = h_24x7_event_commit_txn, 1661 .cancel_txn = h_24x7_event_cancel_txn, 1662 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 1663}; 1664 1665static int ppc_hv_24x7_cpu_online(unsigned int cpu) 1666{ 1667 if (cpumask_empty(&hv_24x7_cpumask)) 1668 cpumask_set_cpu(cpu, &hv_24x7_cpumask); 1669 1670 return 0; 1671} 1672 1673static int ppc_hv_24x7_cpu_offline(unsigned int cpu) 1674{ 1675 int target; 1676 1677 /* Check if exiting cpu is used for collecting 24x7 events */ 1678 if (!cpumask_test_and_clear_cpu(cpu, &hv_24x7_cpumask)) 1679 return 0; 1680 1681 /* Find a new cpu to collect 24x7 events */ 1682 target = cpumask_last(cpu_active_mask); 1683 1684 if (target < 0 || target >= nr_cpu_ids) { 1685 pr_err("hv_24x7: CPU hotplug init failed\n"); 1686 return -1; 1687 } 1688 1689 /* Migrate 24x7 events to the new target */ 1690 cpumask_set_cpu(target, &hv_24x7_cpumask); 1691 perf_pmu_migrate_context(&h_24x7_pmu, cpu, target); 1692 1693 return 0; 1694} 1695 1696static int hv_24x7_cpu_hotplug_init(void) 1697{ 1698 return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE, 1699 "perf/powerpc/hv_24x7:online", 1700 ppc_hv_24x7_cpu_online, 1701 ppc_hv_24x7_cpu_offline); 1702} 1703 1704static int hv_24x7_init(void) 1705{ 1706 int r; 1707 unsigned long hret; 1708 unsigned int pvr = mfspr(SPRN_PVR); 1709 struct hv_perf_caps caps; 1710 1711 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 1712 pr_debug("not a virtualized system, not enabling\n"); 1713 return -ENODEV; 1714 } 1715 1716 /* POWER8 only supports v1, while POWER9 only supports v2. */ 1717 if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E || 1718 PVR_VER(pvr) == PVR_POWER8NVL) 1719 interface_version = 1; 1720 else { 1721 interface_version = 2; 1722 1723 /* SMT8 in POWER9 needs to aggregate result elements. */ 1724 if (threads_per_core == 8) 1725 aggregate_result_elements = true; 1726 } 1727 1728 hret = hv_perf_caps_get(&caps); 1729 if (hret) { 1730 pr_debug("could not obtain capabilities, not enabling, rc=%ld\n", 1731 hret); 1732 return -ENODEV; 1733 } 1734 1735 hv_page_cache = kmem_cache_create("hv-page-4096", 4096, 4096, 0, NULL); 1736 if (!hv_page_cache) 1737 return -ENOMEM; 1738 1739 /* sampling not supported */ 1740 h_24x7_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1741 1742 r = create_events_from_catalog(&event_group.attrs, 1743 &event_desc_group.attrs, 1744 &event_long_desc_group.attrs); 1745 1746 if (r) 1747 return r; 1748 1749 /* init cpuhotplug */ 1750 r = hv_24x7_cpu_hotplug_init(); 1751 if (r) 1752 return r; 1753 1754 r = perf_pmu_register(&h_24x7_pmu, h_24x7_pmu.name, -1); 1755 if (r) 1756 return r; 1757 1758 read_24x7_sys_info(); 1759 1760 return 0; 1761} 1762 1763device_initcall(hv_24x7_init); 1764