1// SPDX-License-Identifier: GPL-2.0
2#include <errno.h>
3#include <unistd.h>
4#include <sys/syscall.h>
5#include <perf/evsel.h>
6#include <perf/cpumap.h>
7#include <perf/threadmap.h>
8#include <linux/list.h>
9#include <internal/evsel.h>
10#include <linux/zalloc.h>
11#include <stdlib.h>
12#include <internal/xyarray.h>
13#include <internal/cpumap.h>
14#include <internal/mmap.h>
15#include <internal/threadmap.h>
16#include <internal/lib.h>
17#include <linux/string.h>
18#include <sys/ioctl.h>
19#include <sys/mman.h>
20#include <asm/bug.h>
21
22void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr,
23		      int idx)
24{
25	INIT_LIST_HEAD(&evsel->node);
26	evsel->attr = *attr;
27	evsel->idx  = idx;
28	evsel->leader = evsel;
29}
30
31struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr)
32{
33	struct perf_evsel *evsel = zalloc(sizeof(*evsel));
34
35	if (evsel != NULL)
36		perf_evsel__init(evsel, attr, 0);
37
38	return evsel;
39}
40
41void perf_evsel__delete(struct perf_evsel *evsel)
42{
43	free(evsel);
44}
45
46#define FD(_evsel, _cpu_map_idx, _thread)				\
47	((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread))
48#define MMAP(_evsel, _cpu_map_idx, _thread)				\
49	(_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \
50		      : NULL)
51
52int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
53{
54	evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
55
56	if (evsel->fd) {
57		int idx, thread;
58
59		for (idx = 0; idx < ncpus; idx++) {
60			for (thread = 0; thread < nthreads; thread++) {
61				int *fd = FD(evsel, idx, thread);
62
63				if (fd)
64					*fd = -1;
65			}
66		}
67	}
68
69	return evsel->fd != NULL ? 0 : -ENOMEM;
70}
71
72static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads)
73{
74	evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap));
75
76	return evsel->mmap != NULL ? 0 : -ENOMEM;
77}
78
79static int
80sys_perf_event_open(struct perf_event_attr *attr,
81		    pid_t pid, struct perf_cpu cpu, int group_fd,
82		    unsigned long flags)
83{
84	return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags);
85}
86
87static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd)
88{
89	struct perf_evsel *leader = evsel->leader;
90	int *fd;
91
92	if (evsel == leader) {
93		*group_fd = -1;
94		return 0;
95	}
96
97	/*
98	 * Leader must be already processed/open,
99	 * if not it's a bug.
100	 */
101	if (!leader->fd)
102		return -ENOTCONN;
103
104	fd = FD(leader, cpu_map_idx, thread);
105	if (fd == NULL || *fd == -1)
106		return -EBADF;
107
108	*group_fd = *fd;
109
110	return 0;
111}
112
113int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus,
114		     struct perf_thread_map *threads)
115{
116	struct perf_cpu cpu;
117	int idx, thread, err = 0;
118
119	if (cpus == NULL) {
120		static struct perf_cpu_map *empty_cpu_map;
121
122		if (empty_cpu_map == NULL) {
123			empty_cpu_map = perf_cpu_map__new_any_cpu();
124			if (empty_cpu_map == NULL)
125				return -ENOMEM;
126		}
127
128		cpus = empty_cpu_map;
129	}
130
131	if (threads == NULL) {
132		static struct perf_thread_map *empty_thread_map;
133
134		if (empty_thread_map == NULL) {
135			empty_thread_map = perf_thread_map__new_dummy();
136			if (empty_thread_map == NULL)
137				return -ENOMEM;
138		}
139
140		threads = empty_thread_map;
141	}
142
143	if (evsel->fd == NULL &&
144	    perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0)
145		return -ENOMEM;
146
147	perf_cpu_map__for_each_cpu(cpu, idx, cpus) {
148		for (thread = 0; thread < threads->nr; thread++) {
149			int fd, group_fd, *evsel_fd;
150
151			evsel_fd = FD(evsel, idx, thread);
152			if (evsel_fd == NULL) {
153				err = -EINVAL;
154				goto out;
155			}
156
157			err = get_group_fd(evsel, idx, thread, &group_fd);
158			if (err < 0)
159				goto out;
160
161			fd = sys_perf_event_open(&evsel->attr,
162						 threads->map[thread].pid,
163						 cpu, group_fd, 0);
164
165			if (fd < 0) {
166				err = -errno;
167				goto out;
168			}
169
170			*evsel_fd = fd;
171		}
172	}
173out:
174	if (err)
175		perf_evsel__close(evsel);
176
177	return err;
178}
179
180static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx)
181{
182	int thread;
183
184	for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) {
185		int *fd = FD(evsel, cpu_map_idx, thread);
186
187		if (fd && *fd >= 0) {
188			close(*fd);
189			*fd = -1;
190		}
191	}
192}
193
194void perf_evsel__close_fd(struct perf_evsel *evsel)
195{
196	for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++)
197		perf_evsel__close_fd_cpu(evsel, idx);
198}
199
200void perf_evsel__free_fd(struct perf_evsel *evsel)
201{
202	xyarray__delete(evsel->fd);
203	evsel->fd = NULL;
204}
205
206void perf_evsel__close(struct perf_evsel *evsel)
207{
208	if (evsel->fd == NULL)
209		return;
210
211	perf_evsel__close_fd(evsel);
212	perf_evsel__free_fd(evsel);
213}
214
215void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx)
216{
217	if (evsel->fd == NULL)
218		return;
219
220	perf_evsel__close_fd_cpu(evsel, cpu_map_idx);
221}
222
223void perf_evsel__munmap(struct perf_evsel *evsel)
224{
225	int idx, thread;
226
227	if (evsel->fd == NULL || evsel->mmap == NULL)
228		return;
229
230	for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
231		for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
232			int *fd = FD(evsel, idx, thread);
233
234			if (fd == NULL || *fd < 0)
235				continue;
236
237			perf_mmap__munmap(MMAP(evsel, idx, thread));
238		}
239	}
240
241	xyarray__delete(evsel->mmap);
242	evsel->mmap = NULL;
243}
244
245int perf_evsel__mmap(struct perf_evsel *evsel, int pages)
246{
247	int ret, idx, thread;
248	struct perf_mmap_param mp = {
249		.prot = PROT_READ | PROT_WRITE,
250		.mask = (pages * page_size) - 1,
251	};
252
253	if (evsel->fd == NULL || evsel->mmap)
254		return -EINVAL;
255
256	if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0)
257		return -ENOMEM;
258
259	for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
260		for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
261			int *fd = FD(evsel, idx, thread);
262			struct perf_mmap *map;
263			struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx);
264
265			if (fd == NULL || *fd < 0)
266				continue;
267
268			map = MMAP(evsel, idx, thread);
269			perf_mmap__init(map, NULL, false, NULL);
270
271			ret = perf_mmap__mmap(map, &mp, *fd, cpu);
272			if (ret) {
273				perf_evsel__munmap(evsel);
274				return ret;
275			}
276		}
277	}
278
279	return 0;
280}
281
282void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread)
283{
284	int *fd = FD(evsel, cpu_map_idx, thread);
285
286	if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL)
287		return NULL;
288
289	return MMAP(evsel, cpu_map_idx, thread)->base;
290}
291
292int perf_evsel__read_size(struct perf_evsel *evsel)
293{
294	u64 read_format = evsel->attr.read_format;
295	int entry = sizeof(u64); /* value */
296	int size = 0;
297	int nr = 1;
298
299	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
300		size += sizeof(u64);
301
302	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
303		size += sizeof(u64);
304
305	if (read_format & PERF_FORMAT_ID)
306		entry += sizeof(u64);
307
308	if (read_format & PERF_FORMAT_LOST)
309		entry += sizeof(u64);
310
311	if (read_format & PERF_FORMAT_GROUP) {
312		nr = evsel->nr_members;
313		size += sizeof(u64);
314	}
315
316	size += entry * nr;
317	return size;
318}
319
320/* This only reads values for the leader */
321static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
322				  int thread, struct perf_counts_values *count)
323{
324	size_t size = perf_evsel__read_size(evsel);
325	int *fd = FD(evsel, cpu_map_idx, thread);
326	u64 read_format = evsel->attr.read_format;
327	u64 *data;
328	int idx = 1;
329
330	if (fd == NULL || *fd < 0)
331		return -EINVAL;
332
333	data = calloc(1, size);
334	if (data == NULL)
335		return -ENOMEM;
336
337	if (readn(*fd, data, size) <= 0) {
338		free(data);
339		return -errno;
340	}
341
342	/*
343	 * This reads only the leader event intentionally since we don't have
344	 * perf counts values for sibling events.
345	 */
346	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
347		count->ena = data[idx++];
348	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
349		count->run = data[idx++];
350
351	/* value is always available */
352	count->val = data[idx++];
353	if (read_format & PERF_FORMAT_ID)
354		count->id = data[idx++];
355	if (read_format & PERF_FORMAT_LOST)
356		count->lost = data[idx++];
357
358	free(data);
359	return 0;
360}
361
362/*
363 * The perf read format is very flexible.  It needs to set the proper
364 * values according to the read format.
365 */
366static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf,
367				      struct perf_counts_values *count)
368{
369	u64 read_format = evsel->attr.read_format;
370	int n = 0;
371
372	count->val = buf[n++];
373
374	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
375		count->ena = buf[n++];
376
377	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
378		count->run = buf[n++];
379
380	if (read_format & PERF_FORMAT_ID)
381		count->id = buf[n++];
382
383	if (read_format & PERF_FORMAT_LOST)
384		count->lost = buf[n++];
385}
386
387int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread,
388		     struct perf_counts_values *count)
389{
390	size_t size = perf_evsel__read_size(evsel);
391	int *fd = FD(evsel, cpu_map_idx, thread);
392	u64 read_format = evsel->attr.read_format;
393	struct perf_counts_values buf;
394
395	memset(count, 0, sizeof(*count));
396
397	if (fd == NULL || *fd < 0)
398		return -EINVAL;
399
400	if (read_format & PERF_FORMAT_GROUP)
401		return perf_evsel__read_group(evsel, cpu_map_idx, thread, count);
402
403	if (MMAP(evsel, cpu_map_idx, thread) &&
404	    !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) &&
405	    !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count))
406		return 0;
407
408	if (readn(*fd, buf.values, size) <= 0)
409		return -errno;
410
411	perf_evsel__adjust_values(evsel, buf.values, count);
412	return 0;
413}
414
415static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg,
416			     int cpu_map_idx, int thread)
417{
418	int *fd = FD(evsel, cpu_map_idx, thread);
419
420	if (fd == NULL || *fd < 0)
421		return -1;
422
423	return ioctl(*fd, ioc, arg);
424}
425
426static int perf_evsel__run_ioctl(struct perf_evsel *evsel,
427				 int ioc,  void *arg,
428				 int cpu_map_idx)
429{
430	int thread;
431
432	for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
433		int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread);
434
435		if (err)
436			return err;
437	}
438
439	return 0;
440}
441
442int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
443{
444	return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx);
445}
446
447int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread)
448{
449	struct perf_cpu cpu __maybe_unused;
450	int idx;
451	int err;
452
453	perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) {
454		err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread);
455		if (err)
456			return err;
457	}
458
459	return 0;
460}
461
462int perf_evsel__enable(struct perf_evsel *evsel)
463{
464	int i;
465	int err = 0;
466
467	for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
468		err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i);
469	return err;
470}
471
472int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
473{
474	return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx);
475}
476
477int perf_evsel__disable(struct perf_evsel *evsel)
478{
479	int i;
480	int err = 0;
481
482	for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
483		err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i);
484	return err;
485}
486
487int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter)
488{
489	int err = 0, i;
490
491	for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++)
492		err = perf_evsel__run_ioctl(evsel,
493				     PERF_EVENT_IOC_SET_FILTER,
494				     (void *)filter, i);
495	return err;
496}
497
498struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel)
499{
500	return evsel->cpus;
501}
502
503struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel)
504{
505	return evsel->threads;
506}
507
508struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel)
509{
510	return &evsel->attr;
511}
512
513int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
514{
515	if (ncpus == 0 || nthreads == 0)
516		return 0;
517
518	evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
519	if (evsel->sample_id == NULL)
520		return -ENOMEM;
521
522	evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
523	if (evsel->id == NULL) {
524		xyarray__delete(evsel->sample_id);
525		evsel->sample_id = NULL;
526		return -ENOMEM;
527	}
528
529	return 0;
530}
531
532void perf_evsel__free_id(struct perf_evsel *evsel)
533{
534	xyarray__delete(evsel->sample_id);
535	evsel->sample_id = NULL;
536	zfree(&evsel->id);
537	evsel->ids = 0;
538}
539
540void perf_counts_values__scale(struct perf_counts_values *count,
541			       bool scale, __s8 *pscaled)
542{
543	s8 scaled = 0;
544
545	if (scale) {
546		if (count->run == 0) {
547			scaled = -1;
548			count->val = 0;
549		} else if (count->run < count->ena) {
550			scaled = 1;
551			count->val = (u64)((double)count->val * count->ena / count->run);
552		}
553	}
554
555	if (pscaled)
556		*pscaled = scaled;
557}
558