1// SPDX-License-Identifier: GPL-2.0 2#include <errno.h> 3#include <unistd.h> 4#include <sys/syscall.h> 5#include <perf/evsel.h> 6#include <perf/cpumap.h> 7#include <perf/threadmap.h> 8#include <linux/list.h> 9#include <internal/evsel.h> 10#include <linux/zalloc.h> 11#include <stdlib.h> 12#include <internal/xyarray.h> 13#include <internal/cpumap.h> 14#include <internal/mmap.h> 15#include <internal/threadmap.h> 16#include <internal/lib.h> 17#include <linux/string.h> 18#include <sys/ioctl.h> 19#include <sys/mman.h> 20#include <asm/bug.h> 21 22void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr, 23 int idx) 24{ 25 INIT_LIST_HEAD(&evsel->node); 26 evsel->attr = *attr; 27 evsel->idx = idx; 28 evsel->leader = evsel; 29} 30 31struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr) 32{ 33 struct perf_evsel *evsel = zalloc(sizeof(*evsel)); 34 35 if (evsel != NULL) 36 perf_evsel__init(evsel, attr, 0); 37 38 return evsel; 39} 40 41void perf_evsel__delete(struct perf_evsel *evsel) 42{ 43 free(evsel); 44} 45 46#define FD(_evsel, _cpu_map_idx, _thread) \ 47 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread)) 48#define MMAP(_evsel, _cpu_map_idx, _thread) \ 49 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \ 50 : NULL) 51 52int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads) 53{ 54 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int)); 55 56 if (evsel->fd) { 57 int idx, thread; 58 59 for (idx = 0; idx < ncpus; idx++) { 60 for (thread = 0; thread < nthreads; thread++) { 61 int *fd = FD(evsel, idx, thread); 62 63 if (fd) 64 *fd = -1; 65 } 66 } 67 } 68 69 return evsel->fd != NULL ? 0 : -ENOMEM; 70} 71 72static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads) 73{ 74 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap)); 75 76 return evsel->mmap != NULL ? 0 : -ENOMEM; 77} 78 79static int 80sys_perf_event_open(struct perf_event_attr *attr, 81 pid_t pid, struct perf_cpu cpu, int group_fd, 82 unsigned long flags) 83{ 84 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags); 85} 86 87static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd) 88{ 89 struct perf_evsel *leader = evsel->leader; 90 int *fd; 91 92 if (evsel == leader) { 93 *group_fd = -1; 94 return 0; 95 } 96 97 /* 98 * Leader must be already processed/open, 99 * if not it's a bug. 100 */ 101 if (!leader->fd) 102 return -ENOTCONN; 103 104 fd = FD(leader, cpu_map_idx, thread); 105 if (fd == NULL || *fd == -1) 106 return -EBADF; 107 108 *group_fd = *fd; 109 110 return 0; 111} 112 113int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus, 114 struct perf_thread_map *threads) 115{ 116 struct perf_cpu cpu; 117 int idx, thread, err = 0; 118 119 if (cpus == NULL) { 120 static struct perf_cpu_map *empty_cpu_map; 121 122 if (empty_cpu_map == NULL) { 123 empty_cpu_map = perf_cpu_map__new_any_cpu(); 124 if (empty_cpu_map == NULL) 125 return -ENOMEM; 126 } 127 128 cpus = empty_cpu_map; 129 } 130 131 if (threads == NULL) { 132 static struct perf_thread_map *empty_thread_map; 133 134 if (empty_thread_map == NULL) { 135 empty_thread_map = perf_thread_map__new_dummy(); 136 if (empty_thread_map == NULL) 137 return -ENOMEM; 138 } 139 140 threads = empty_thread_map; 141 } 142 143 if (evsel->fd == NULL && 144 perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0) 145 return -ENOMEM; 146 147 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 148 for (thread = 0; thread < threads->nr; thread++) { 149 int fd, group_fd, *evsel_fd; 150 151 evsel_fd = FD(evsel, idx, thread); 152 if (evsel_fd == NULL) { 153 err = -EINVAL; 154 goto out; 155 } 156 157 err = get_group_fd(evsel, idx, thread, &group_fd); 158 if (err < 0) 159 goto out; 160 161 fd = sys_perf_event_open(&evsel->attr, 162 threads->map[thread].pid, 163 cpu, group_fd, 0); 164 165 if (fd < 0) { 166 err = -errno; 167 goto out; 168 } 169 170 *evsel_fd = fd; 171 } 172 } 173out: 174 if (err) 175 perf_evsel__close(evsel); 176 177 return err; 178} 179 180static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx) 181{ 182 int thread; 183 184 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) { 185 int *fd = FD(evsel, cpu_map_idx, thread); 186 187 if (fd && *fd >= 0) { 188 close(*fd); 189 *fd = -1; 190 } 191 } 192} 193 194void perf_evsel__close_fd(struct perf_evsel *evsel) 195{ 196 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++) 197 perf_evsel__close_fd_cpu(evsel, idx); 198} 199 200void perf_evsel__free_fd(struct perf_evsel *evsel) 201{ 202 xyarray__delete(evsel->fd); 203 evsel->fd = NULL; 204} 205 206void perf_evsel__close(struct perf_evsel *evsel) 207{ 208 if (evsel->fd == NULL) 209 return; 210 211 perf_evsel__close_fd(evsel); 212 perf_evsel__free_fd(evsel); 213} 214 215void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx) 216{ 217 if (evsel->fd == NULL) 218 return; 219 220 perf_evsel__close_fd_cpu(evsel, cpu_map_idx); 221} 222 223void perf_evsel__munmap(struct perf_evsel *evsel) 224{ 225 int idx, thread; 226 227 if (evsel->fd == NULL || evsel->mmap == NULL) 228 return; 229 230 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 231 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 232 int *fd = FD(evsel, idx, thread); 233 234 if (fd == NULL || *fd < 0) 235 continue; 236 237 perf_mmap__munmap(MMAP(evsel, idx, thread)); 238 } 239 } 240 241 xyarray__delete(evsel->mmap); 242 evsel->mmap = NULL; 243} 244 245int perf_evsel__mmap(struct perf_evsel *evsel, int pages) 246{ 247 int ret, idx, thread; 248 struct perf_mmap_param mp = { 249 .prot = PROT_READ | PROT_WRITE, 250 .mask = (pages * page_size) - 1, 251 }; 252 253 if (evsel->fd == NULL || evsel->mmap) 254 return -EINVAL; 255 256 if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0) 257 return -ENOMEM; 258 259 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 260 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 261 int *fd = FD(evsel, idx, thread); 262 struct perf_mmap *map; 263 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx); 264 265 if (fd == NULL || *fd < 0) 266 continue; 267 268 map = MMAP(evsel, idx, thread); 269 perf_mmap__init(map, NULL, false, NULL); 270 271 ret = perf_mmap__mmap(map, &mp, *fd, cpu); 272 if (ret) { 273 perf_evsel__munmap(evsel); 274 return ret; 275 } 276 } 277 } 278 279 return 0; 280} 281 282void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread) 283{ 284 int *fd = FD(evsel, cpu_map_idx, thread); 285 286 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL) 287 return NULL; 288 289 return MMAP(evsel, cpu_map_idx, thread)->base; 290} 291 292int perf_evsel__read_size(struct perf_evsel *evsel) 293{ 294 u64 read_format = evsel->attr.read_format; 295 int entry = sizeof(u64); /* value */ 296 int size = 0; 297 int nr = 1; 298 299 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 300 size += sizeof(u64); 301 302 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 303 size += sizeof(u64); 304 305 if (read_format & PERF_FORMAT_ID) 306 entry += sizeof(u64); 307 308 if (read_format & PERF_FORMAT_LOST) 309 entry += sizeof(u64); 310 311 if (read_format & PERF_FORMAT_GROUP) { 312 nr = evsel->nr_members; 313 size += sizeof(u64); 314 } 315 316 size += entry * nr; 317 return size; 318} 319 320/* This only reads values for the leader */ 321static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx, 322 int thread, struct perf_counts_values *count) 323{ 324 size_t size = perf_evsel__read_size(evsel); 325 int *fd = FD(evsel, cpu_map_idx, thread); 326 u64 read_format = evsel->attr.read_format; 327 u64 *data; 328 int idx = 1; 329 330 if (fd == NULL || *fd < 0) 331 return -EINVAL; 332 333 data = calloc(1, size); 334 if (data == NULL) 335 return -ENOMEM; 336 337 if (readn(*fd, data, size) <= 0) { 338 free(data); 339 return -errno; 340 } 341 342 /* 343 * This reads only the leader event intentionally since we don't have 344 * perf counts values for sibling events. 345 */ 346 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 347 count->ena = data[idx++]; 348 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 349 count->run = data[idx++]; 350 351 /* value is always available */ 352 count->val = data[idx++]; 353 if (read_format & PERF_FORMAT_ID) 354 count->id = data[idx++]; 355 if (read_format & PERF_FORMAT_LOST) 356 count->lost = data[idx++]; 357 358 free(data); 359 return 0; 360} 361 362/* 363 * The perf read format is very flexible. It needs to set the proper 364 * values according to the read format. 365 */ 366static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf, 367 struct perf_counts_values *count) 368{ 369 u64 read_format = evsel->attr.read_format; 370 int n = 0; 371 372 count->val = buf[n++]; 373 374 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 375 count->ena = buf[n++]; 376 377 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 378 count->run = buf[n++]; 379 380 if (read_format & PERF_FORMAT_ID) 381 count->id = buf[n++]; 382 383 if (read_format & PERF_FORMAT_LOST) 384 count->lost = buf[n++]; 385} 386 387int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread, 388 struct perf_counts_values *count) 389{ 390 size_t size = perf_evsel__read_size(evsel); 391 int *fd = FD(evsel, cpu_map_idx, thread); 392 u64 read_format = evsel->attr.read_format; 393 struct perf_counts_values buf; 394 395 memset(count, 0, sizeof(*count)); 396 397 if (fd == NULL || *fd < 0) 398 return -EINVAL; 399 400 if (read_format & PERF_FORMAT_GROUP) 401 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count); 402 403 if (MMAP(evsel, cpu_map_idx, thread) && 404 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) && 405 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count)) 406 return 0; 407 408 if (readn(*fd, buf.values, size) <= 0) 409 return -errno; 410 411 perf_evsel__adjust_values(evsel, buf.values, count); 412 return 0; 413} 414 415static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg, 416 int cpu_map_idx, int thread) 417{ 418 int *fd = FD(evsel, cpu_map_idx, thread); 419 420 if (fd == NULL || *fd < 0) 421 return -1; 422 423 return ioctl(*fd, ioc, arg); 424} 425 426static int perf_evsel__run_ioctl(struct perf_evsel *evsel, 427 int ioc, void *arg, 428 int cpu_map_idx) 429{ 430 int thread; 431 432 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 433 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread); 434 435 if (err) 436 return err; 437 } 438 439 return 0; 440} 441 442int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 443{ 444 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx); 445} 446 447int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread) 448{ 449 struct perf_cpu cpu __maybe_unused; 450 int idx; 451 int err; 452 453 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) { 454 err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread); 455 if (err) 456 return err; 457 } 458 459 return 0; 460} 461 462int perf_evsel__enable(struct perf_evsel *evsel) 463{ 464 int i; 465 int err = 0; 466 467 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 468 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i); 469 return err; 470} 471 472int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 473{ 474 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx); 475} 476 477int perf_evsel__disable(struct perf_evsel *evsel) 478{ 479 int i; 480 int err = 0; 481 482 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 483 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i); 484 return err; 485} 486 487int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter) 488{ 489 int err = 0, i; 490 491 for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) 492 err = perf_evsel__run_ioctl(evsel, 493 PERF_EVENT_IOC_SET_FILTER, 494 (void *)filter, i); 495 return err; 496} 497 498struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel) 499{ 500 return evsel->cpus; 501} 502 503struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel) 504{ 505 return evsel->threads; 506} 507 508struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel) 509{ 510 return &evsel->attr; 511} 512 513int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads) 514{ 515 if (ncpus == 0 || nthreads == 0) 516 return 0; 517 518 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id)); 519 if (evsel->sample_id == NULL) 520 return -ENOMEM; 521 522 evsel->id = zalloc(ncpus * nthreads * sizeof(u64)); 523 if (evsel->id == NULL) { 524 xyarray__delete(evsel->sample_id); 525 evsel->sample_id = NULL; 526 return -ENOMEM; 527 } 528 529 return 0; 530} 531 532void perf_evsel__free_id(struct perf_evsel *evsel) 533{ 534 xyarray__delete(evsel->sample_id); 535 evsel->sample_id = NULL; 536 zfree(&evsel->id); 537 evsel->ids = 0; 538} 539 540void perf_counts_values__scale(struct perf_counts_values *count, 541 bool scale, __s8 *pscaled) 542{ 543 s8 scaled = 0; 544 545 if (scale) { 546 if (count->run == 0) { 547 scaled = -1; 548 count->val = 0; 549 } else if (count->run < count->ena) { 550 scaled = 1; 551 count->val = (u64)((double)count->val * count->ena / count->run); 552 } 553 } 554 555 if (pscaled) 556 *pscaled = scaled; 557} 558