1// SPDX-License-Identifier: GPL-2.0 2#include <api/fs/fs.h> 3#include "cpumap.h" 4#include "debug.h" 5#include "event.h" 6#include <assert.h> 7#include <dirent.h> 8#include <stdio.h> 9#include <stdlib.h> 10#include <linux/bitmap.h> 11#include "asm/bug.h" 12 13#include <linux/ctype.h> 14#include <linux/zalloc.h> 15#include <internal/cpumap.h> 16 17static struct perf_cpu max_cpu_num; 18static struct perf_cpu max_present_cpu_num; 19static int max_node_num; 20/** 21 * The numa node X as read from /sys/devices/system/node/nodeX indexed by the 22 * CPU number. 23 */ 24static int *cpunode_map; 25 26bool perf_record_cpu_map_data__test_bit(int i, 27 const struct perf_record_cpu_map_data *data) 28{ 29 int bit_word32 = i / 32; 30 __u32 bit_mask32 = 1U << (i & 31); 31 int bit_word64 = i / 64; 32 __u64 bit_mask64 = ((__u64)1) << (i & 63); 33 34 return (data->mask32_data.long_size == 4) 35 ? (bit_word32 < data->mask32_data.nr) && 36 (data->mask32_data.mask[bit_word32] & bit_mask32) != 0 37 : (bit_word64 < data->mask64_data.nr) && 38 (data->mask64_data.mask[bit_word64] & bit_mask64) != 0; 39} 40 41/* Read ith mask value from data into the given 64-bit sized bitmap */ 42static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu_map_data *data, 43 int i, unsigned long *bitmap) 44{ 45#if __SIZEOF_LONG__ == 8 46 if (data->mask32_data.long_size == 4) 47 bitmap[0] = data->mask32_data.mask[i]; 48 else 49 bitmap[0] = data->mask64_data.mask[i]; 50#else 51 if (data->mask32_data.long_size == 4) { 52 bitmap[0] = data->mask32_data.mask[i]; 53 bitmap[1] = 0; 54 } else { 55#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 56 bitmap[0] = (unsigned long)(data->mask64_data.mask[i] >> 32); 57 bitmap[1] = (unsigned long)data->mask64_data.mask[i]; 58#else 59 bitmap[0] = (unsigned long)data->mask64_data.mask[i]; 60 bitmap[1] = (unsigned long)(data->mask64_data.mask[i] >> 32); 61#endif 62 } 63#endif 64} 65static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_map_data *data) 66{ 67 struct perf_cpu_map *map; 68 69 map = perf_cpu_map__empty_new(data->cpus_data.nr); 70 if (map) { 71 unsigned i; 72 73 for (i = 0; i < data->cpus_data.nr; i++) { 74 /* 75 * Special treatment for -1, which is not real cpu number, 76 * and we need to use (int) -1 to initialize map[i], 77 * otherwise it would become 65535. 78 */ 79 if (data->cpus_data.cpu[i] == (u16) -1) 80 RC_CHK_ACCESS(map)->map[i].cpu = -1; 81 else 82 RC_CHK_ACCESS(map)->map[i].cpu = (int) data->cpus_data.cpu[i]; 83 } 84 } 85 86 return map; 87} 88 89static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_data *data) 90{ 91 DECLARE_BITMAP(local_copy, 64); 92 int weight = 0, mask_nr = data->mask32_data.nr; 93 struct perf_cpu_map *map; 94 95 for (int i = 0; i < mask_nr; i++) { 96 perf_record_cpu_map_data__read_one_mask(data, i, local_copy); 97 weight += bitmap_weight(local_copy, 64); 98 } 99 100 map = perf_cpu_map__empty_new(weight); 101 if (!map) 102 return NULL; 103 104 for (int i = 0, j = 0; i < mask_nr; i++) { 105 int cpus_per_i = (i * data->mask32_data.long_size * BITS_PER_BYTE); 106 int cpu; 107 108 perf_record_cpu_map_data__read_one_mask(data, i, local_copy); 109 for_each_set_bit(cpu, local_copy, 64) 110 RC_CHK_ACCESS(map)->map[j++].cpu = cpu + cpus_per_i; 111 } 112 return map; 113 114} 115 116static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) 117{ 118 struct perf_cpu_map *map; 119 unsigned int i = 0; 120 121 map = perf_cpu_map__empty_new(data->range_cpu_data.end_cpu - 122 data->range_cpu_data.start_cpu + 1 + data->range_cpu_data.any_cpu); 123 if (!map) 124 return NULL; 125 126 if (data->range_cpu_data.any_cpu) 127 RC_CHK_ACCESS(map)->map[i++].cpu = -1; 128 129 for (int cpu = data->range_cpu_data.start_cpu; cpu <= data->range_cpu_data.end_cpu; 130 i++, cpu++) 131 RC_CHK_ACCESS(map)->map[i].cpu = cpu; 132 133 return map; 134} 135 136struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data) 137{ 138 switch (data->type) { 139 case PERF_CPU_MAP__CPUS: 140 return cpu_map__from_entries(data); 141 case PERF_CPU_MAP__MASK: 142 return cpu_map__from_mask(data); 143 case PERF_CPU_MAP__RANGE_CPUS: 144 return cpu_map__from_range(data); 145 default: 146 pr_err("cpu_map__new_data unknown type %d\n", data->type); 147 return NULL; 148 } 149} 150 151size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp) 152{ 153#define BUFSIZE 1024 154 char buf[BUFSIZE]; 155 156 cpu_map__snprint(map, buf, sizeof(buf)); 157 return fprintf(fp, "%s\n", buf); 158#undef BUFSIZE 159} 160 161struct perf_cpu_map *perf_cpu_map__empty_new(int nr) 162{ 163 struct perf_cpu_map *cpus = perf_cpu_map__alloc(nr); 164 165 if (cpus != NULL) { 166 for (int i = 0; i < nr; i++) 167 RC_CHK_ACCESS(cpus)->map[i].cpu = -1; 168 } 169 170 return cpus; 171} 172 173struct cpu_aggr_map *cpu_aggr_map__empty_new(int nr) 174{ 175 struct cpu_aggr_map *cpus = malloc(sizeof(*cpus) + sizeof(struct aggr_cpu_id) * nr); 176 177 if (cpus != NULL) { 178 int i; 179 180 cpus->nr = nr; 181 for (i = 0; i < nr; i++) 182 cpus->map[i] = aggr_cpu_id__empty(); 183 184 refcount_set(&cpus->refcnt, 1); 185 } 186 187 return cpus; 188} 189 190static int cpu__get_topology_int(int cpu, const char *name, int *value) 191{ 192 char path[PATH_MAX]; 193 194 snprintf(path, PATH_MAX, 195 "devices/system/cpu/cpu%d/topology/%s", cpu, name); 196 197 return sysfs__read_int(path, value); 198} 199 200int cpu__get_socket_id(struct perf_cpu cpu) 201{ 202 int value, ret = cpu__get_topology_int(cpu.cpu, "physical_package_id", &value); 203 return ret ?: value; 204} 205 206struct aggr_cpu_id aggr_cpu_id__socket(struct perf_cpu cpu, void *data __maybe_unused) 207{ 208 struct aggr_cpu_id id = aggr_cpu_id__empty(); 209 210 id.socket = cpu__get_socket_id(cpu); 211 return id; 212} 213 214static int aggr_cpu_id__cmp(const void *a_pointer, const void *b_pointer) 215{ 216 struct aggr_cpu_id *a = (struct aggr_cpu_id *)a_pointer; 217 struct aggr_cpu_id *b = (struct aggr_cpu_id *)b_pointer; 218 219 if (a->node != b->node) 220 return a->node - b->node; 221 else if (a->socket != b->socket) 222 return a->socket - b->socket; 223 else if (a->die != b->die) 224 return a->die - b->die; 225 else if (a->cluster != b->cluster) 226 return a->cluster - b->cluster; 227 else if (a->cache_lvl != b->cache_lvl) 228 return a->cache_lvl - b->cache_lvl; 229 else if (a->cache != b->cache) 230 return a->cache - b->cache; 231 else if (a->core != b->core) 232 return a->core - b->core; 233 else 234 return a->thread_idx - b->thread_idx; 235} 236 237struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, 238 aggr_cpu_id_get_t get_id, 239 void *data, bool needs_sort) 240{ 241 int idx; 242 struct perf_cpu cpu; 243 struct cpu_aggr_map *c = cpu_aggr_map__empty_new(perf_cpu_map__nr(cpus)); 244 245 if (!c) 246 return NULL; 247 248 /* Reset size as it may only be partially filled */ 249 c->nr = 0; 250 251 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 252 bool duplicate = false; 253 struct aggr_cpu_id cpu_id = get_id(cpu, data); 254 255 for (int j = 0; j < c->nr; j++) { 256 if (aggr_cpu_id__equal(&cpu_id, &c->map[j])) { 257 duplicate = true; 258 break; 259 } 260 } 261 if (!duplicate) { 262 c->map[c->nr] = cpu_id; 263 c->nr++; 264 } 265 } 266 /* Trim. */ 267 if (c->nr != perf_cpu_map__nr(cpus)) { 268 struct cpu_aggr_map *trimmed_c = 269 realloc(c, 270 sizeof(struct cpu_aggr_map) + sizeof(struct aggr_cpu_id) * c->nr); 271 272 if (trimmed_c) 273 c = trimmed_c; 274 } 275 276 /* ensure we process id in increasing order */ 277 if (needs_sort) 278 qsort(c->map, c->nr, sizeof(struct aggr_cpu_id), aggr_cpu_id__cmp); 279 280 return c; 281 282} 283 284int cpu__get_die_id(struct perf_cpu cpu) 285{ 286 int value, ret = cpu__get_topology_int(cpu.cpu, "die_id", &value); 287 288 return ret ?: value; 289} 290 291struct aggr_cpu_id aggr_cpu_id__die(struct perf_cpu cpu, void *data) 292{ 293 struct aggr_cpu_id id; 294 int die; 295 296 die = cpu__get_die_id(cpu); 297 /* There is no die_id on legacy system. */ 298 if (die == -1) 299 die = 0; 300 301 /* 302 * die_id is relative to socket, so start 303 * with the socket ID and then add die to 304 * make a unique ID. 305 */ 306 id = aggr_cpu_id__socket(cpu, data); 307 if (aggr_cpu_id__is_empty(&id)) 308 return id; 309 310 id.die = die; 311 return id; 312} 313 314int cpu__get_cluster_id(struct perf_cpu cpu) 315{ 316 int value, ret = cpu__get_topology_int(cpu.cpu, "cluster_id", &value); 317 318 return ret ?: value; 319} 320 321struct aggr_cpu_id aggr_cpu_id__cluster(struct perf_cpu cpu, void *data) 322{ 323 int cluster = cpu__get_cluster_id(cpu); 324 struct aggr_cpu_id id; 325 326 /* There is no cluster_id on legacy system. */ 327 if (cluster == -1) 328 cluster = 0; 329 330 id = aggr_cpu_id__die(cpu, data); 331 if (aggr_cpu_id__is_empty(&id)) 332 return id; 333 334 id.cluster = cluster; 335 return id; 336} 337 338int cpu__get_core_id(struct perf_cpu cpu) 339{ 340 int value, ret = cpu__get_topology_int(cpu.cpu, "core_id", &value); 341 return ret ?: value; 342} 343 344struct aggr_cpu_id aggr_cpu_id__core(struct perf_cpu cpu, void *data) 345{ 346 struct aggr_cpu_id id; 347 int core = cpu__get_core_id(cpu); 348 349 /* aggr_cpu_id__die returns a struct with socket die, and cluster set. */ 350 id = aggr_cpu_id__cluster(cpu, data); 351 if (aggr_cpu_id__is_empty(&id)) 352 return id; 353 354 /* 355 * core_id is relative to socket and die, we need a global id. 356 * So we combine the result from cpu_map__get_die with the core id 357 */ 358 id.core = core; 359 return id; 360 361} 362 363struct aggr_cpu_id aggr_cpu_id__cpu(struct perf_cpu cpu, void *data) 364{ 365 struct aggr_cpu_id id; 366 367 /* aggr_cpu_id__core returns a struct with socket, die and core set. */ 368 id = aggr_cpu_id__core(cpu, data); 369 if (aggr_cpu_id__is_empty(&id)) 370 return id; 371 372 id.cpu = cpu; 373 return id; 374 375} 376 377struct aggr_cpu_id aggr_cpu_id__node(struct perf_cpu cpu, void *data __maybe_unused) 378{ 379 struct aggr_cpu_id id = aggr_cpu_id__empty(); 380 381 id.node = cpu__get_node(cpu); 382 return id; 383} 384 385struct aggr_cpu_id aggr_cpu_id__global(struct perf_cpu cpu, void *data __maybe_unused) 386{ 387 struct aggr_cpu_id id = aggr_cpu_id__empty(); 388 389 /* it always aggregates to the cpu 0 */ 390 cpu.cpu = 0; 391 id.cpu = cpu; 392 return id; 393} 394 395/* setup simple routines to easily access node numbers given a cpu number */ 396static int get_max_num(char *path, int *max) 397{ 398 size_t num; 399 char *buf; 400 int err = 0; 401 402 if (filename__read_str(path, &buf, &num)) 403 return -1; 404 405 buf[num] = '\0'; 406 407 /* start on the right, to find highest node num */ 408 while (--num) { 409 if ((buf[num] == ',') || (buf[num] == '-')) { 410 num++; 411 break; 412 } 413 } 414 if (sscanf(&buf[num], "%d", max) < 1) { 415 err = -1; 416 goto out; 417 } 418 419 /* convert from 0-based to 1-based */ 420 (*max)++; 421 422out: 423 free(buf); 424 return err; 425} 426 427/* Determine highest possible cpu in the system for sparse allocation */ 428static void set_max_cpu_num(void) 429{ 430 const char *mnt; 431 char path[PATH_MAX]; 432 int ret = -1; 433 434 /* set up default */ 435 max_cpu_num.cpu = 4096; 436 max_present_cpu_num.cpu = 4096; 437 438 mnt = sysfs__mountpoint(); 439 if (!mnt) 440 goto out; 441 442 /* get the highest possible cpu number for a sparse allocation */ 443 ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/possible", mnt); 444 if (ret >= PATH_MAX) { 445 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 446 goto out; 447 } 448 449 ret = get_max_num(path, &max_cpu_num.cpu); 450 if (ret) 451 goto out; 452 453 /* get the highest present cpu number for a sparse allocation */ 454 ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/present", mnt); 455 if (ret >= PATH_MAX) { 456 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 457 goto out; 458 } 459 460 ret = get_max_num(path, &max_present_cpu_num.cpu); 461 462out: 463 if (ret) 464 pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); 465} 466 467/* Determine highest possible node in the system for sparse allocation */ 468static void set_max_node_num(void) 469{ 470 const char *mnt; 471 char path[PATH_MAX]; 472 int ret = -1; 473 474 /* set up default */ 475 max_node_num = 8; 476 477 mnt = sysfs__mountpoint(); 478 if (!mnt) 479 goto out; 480 481 /* get the highest possible cpu number for a sparse allocation */ 482 ret = snprintf(path, PATH_MAX, "%s/devices/system/node/possible", mnt); 483 if (ret >= PATH_MAX) { 484 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 485 goto out; 486 } 487 488 ret = get_max_num(path, &max_node_num); 489 490out: 491 if (ret) 492 pr_err("Failed to read max nodes, using default of %d\n", max_node_num); 493} 494 495int cpu__max_node(void) 496{ 497 if (unlikely(!max_node_num)) 498 set_max_node_num(); 499 500 return max_node_num; 501} 502 503struct perf_cpu cpu__max_cpu(void) 504{ 505 if (unlikely(!max_cpu_num.cpu)) 506 set_max_cpu_num(); 507 508 return max_cpu_num; 509} 510 511struct perf_cpu cpu__max_present_cpu(void) 512{ 513 if (unlikely(!max_present_cpu_num.cpu)) 514 set_max_cpu_num(); 515 516 return max_present_cpu_num; 517} 518 519 520int cpu__get_node(struct perf_cpu cpu) 521{ 522 if (unlikely(cpunode_map == NULL)) { 523 pr_debug("cpu_map not initialized\n"); 524 return -1; 525 } 526 527 return cpunode_map[cpu.cpu]; 528} 529 530static int init_cpunode_map(void) 531{ 532 int i; 533 534 set_max_cpu_num(); 535 set_max_node_num(); 536 537 cpunode_map = calloc(max_cpu_num.cpu, sizeof(int)); 538 if (!cpunode_map) { 539 pr_err("%s: calloc failed\n", __func__); 540 return -1; 541 } 542 543 for (i = 0; i < max_cpu_num.cpu; i++) 544 cpunode_map[i] = -1; 545 546 return 0; 547} 548 549int cpu__setup_cpunode_map(void) 550{ 551 struct dirent *dent1, *dent2; 552 DIR *dir1, *dir2; 553 unsigned int cpu, mem; 554 char buf[PATH_MAX]; 555 char path[PATH_MAX]; 556 const char *mnt; 557 int n; 558 559 /* initialize globals */ 560 if (init_cpunode_map()) 561 return -1; 562 563 mnt = sysfs__mountpoint(); 564 if (!mnt) 565 return 0; 566 567 n = snprintf(path, PATH_MAX, "%s/devices/system/node", mnt); 568 if (n >= PATH_MAX) { 569 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 570 return -1; 571 } 572 573 dir1 = opendir(path); 574 if (!dir1) 575 return 0; 576 577 /* walk tree and setup map */ 578 while ((dent1 = readdir(dir1)) != NULL) { 579 if (dent1->d_type != DT_DIR || sscanf(dent1->d_name, "node%u", &mem) < 1) 580 continue; 581 582 n = snprintf(buf, PATH_MAX, "%s/%s", path, dent1->d_name); 583 if (n >= PATH_MAX) { 584 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 585 continue; 586 } 587 588 dir2 = opendir(buf); 589 if (!dir2) 590 continue; 591 while ((dent2 = readdir(dir2)) != NULL) { 592 if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) 593 continue; 594 cpunode_map[cpu] = mem; 595 } 596 closedir(dir2); 597 } 598 closedir(dir1); 599 return 0; 600} 601 602size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) 603{ 604 int i, start = -1; 605 bool first = true; 606 size_t ret = 0; 607 608#define COMMA first ? "" : "," 609 610 for (i = 0; i < perf_cpu_map__nr(map) + 1; i++) { 611 struct perf_cpu cpu = { .cpu = INT_MAX }; 612 bool last = i == perf_cpu_map__nr(map); 613 614 if (!last) 615 cpu = perf_cpu_map__cpu(map, i); 616 617 if (start == -1) { 618 start = i; 619 if (last) { 620 ret += snprintf(buf + ret, size - ret, 621 "%s%d", COMMA, 622 perf_cpu_map__cpu(map, i).cpu); 623 } 624 } else if (((i - start) != (cpu.cpu - perf_cpu_map__cpu(map, start).cpu)) || last) { 625 int end = i - 1; 626 627 if (start == end) { 628 ret += snprintf(buf + ret, size - ret, 629 "%s%d", COMMA, 630 perf_cpu_map__cpu(map, start).cpu); 631 } else { 632 ret += snprintf(buf + ret, size - ret, 633 "%s%d-%d", COMMA, 634 perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); 635 } 636 first = false; 637 start = i; 638 } 639 } 640 641#undef COMMA 642 643 pr_debug2("cpumask list: %s\n", buf); 644 return ret; 645} 646 647static char hex_char(unsigned char val) 648{ 649 if (val < 10) 650 return val + '0'; 651 if (val < 16) 652 return val - 10 + 'a'; 653 return '?'; 654} 655 656size_t cpu_map__snprint_mask(struct perf_cpu_map *map, char *buf, size_t size) 657{ 658 int i, cpu; 659 char *ptr = buf; 660 unsigned char *bitmap; 661 struct perf_cpu last_cpu = perf_cpu_map__cpu(map, perf_cpu_map__nr(map) - 1); 662 663 if (buf == NULL) 664 return 0; 665 666 bitmap = zalloc(last_cpu.cpu / 8 + 1); 667 if (bitmap == NULL) { 668 buf[0] = '\0'; 669 return 0; 670 } 671 672 for (i = 0; i < perf_cpu_map__nr(map); i++) { 673 cpu = perf_cpu_map__cpu(map, i).cpu; 674 bitmap[cpu / 8] |= 1 << (cpu % 8); 675 } 676 677 for (cpu = last_cpu.cpu / 4 * 4; cpu >= 0; cpu -= 4) { 678 unsigned char bits = bitmap[cpu / 8]; 679 680 if (cpu % 8) 681 bits >>= 4; 682 else 683 bits &= 0xf; 684 685 *ptr++ = hex_char(bits); 686 if ((cpu % 32) == 0 && cpu > 0) 687 *ptr++ = ','; 688 } 689 *ptr = '\0'; 690 free(bitmap); 691 692 buf[size - 1] = '\0'; 693 return ptr - buf; 694} 695 696struct perf_cpu_map *cpu_map__online(void) /* thread unsafe */ 697{ 698 static struct perf_cpu_map *online; 699 700 if (!online) 701 online = perf_cpu_map__new_online_cpus(); /* from /sys/devices/system/cpu/online */ 702 703 return online; 704} 705 706bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b) 707{ 708 return a->thread_idx == b->thread_idx && 709 a->node == b->node && 710 a->socket == b->socket && 711 a->die == b->die && 712 a->cluster == b->cluster && 713 a->cache_lvl == b->cache_lvl && 714 a->cache == b->cache && 715 a->core == b->core && 716 a->cpu.cpu == b->cpu.cpu; 717} 718 719bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a) 720{ 721 return a->thread_idx == -1 && 722 a->node == -1 && 723 a->socket == -1 && 724 a->die == -1 && 725 a->cluster == -1 && 726 a->cache_lvl == -1 && 727 a->cache == -1 && 728 a->core == -1 && 729 a->cpu.cpu == -1; 730} 731 732struct aggr_cpu_id aggr_cpu_id__empty(void) 733{ 734 struct aggr_cpu_id ret = { 735 .thread_idx = -1, 736 .node = -1, 737 .socket = -1, 738 .die = -1, 739 .cluster = -1, 740 .cache_lvl = -1, 741 .cache = -1, 742 .core = -1, 743 .cpu = (struct perf_cpu){ .cpu = -1 }, 744 }; 745 return ret; 746} 747