1// SPDX-License-Identifier: GPL-2.0
2/* Manage affinity to optimize IPIs inside the kernel perf API. */
3#define _GNU_SOURCE 1
4#include <sched.h>
5#include <stdlib.h>
6#include <linux/bitmap.h>
7#include <linux/zalloc.h>
8#include "perf.h"
9#include "cpumap.h"
10#include "affinity.h"
11
12static int get_cpu_set_size(void)
13{
14	int sz = cpu__max_cpu().cpu + 8 - 1;
15	/*
16	 * sched_getaffinity doesn't like masks smaller than the kernel.
17	 * Hopefully that's big enough.
18	 */
19	if (sz < 4096)
20		sz = 4096;
21	return sz / 8;
22}
23
24int affinity__setup(struct affinity *a)
25{
26	int cpu_set_size = get_cpu_set_size();
27
28	a->orig_cpus = bitmap_zalloc(cpu_set_size * 8);
29	if (!a->orig_cpus)
30		return -1;
31	sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
32	a->sched_cpus = bitmap_zalloc(cpu_set_size * 8);
33	if (!a->sched_cpus) {
34		zfree(&a->orig_cpus);
35		return -1;
36	}
37	bitmap_zero((unsigned long *)a->sched_cpus, cpu_set_size);
38	a->changed = false;
39	return 0;
40}
41
42/*
43 * perf_event_open does an IPI internally to the target CPU.
44 * It is more efficient to change perf's affinity to the target
45 * CPU and then set up all events on that CPU, so we amortize
46 * CPU communication.
47 */
48void affinity__set(struct affinity *a, int cpu)
49{
50	int cpu_set_size = get_cpu_set_size();
51
52	/*
53	 * Return:
54	 * - if cpu is -1
55	 * - restrict out of bound access to sched_cpus
56	 */
57	if (cpu == -1 || ((cpu >= (cpu_set_size * 8))))
58		return;
59
60	a->changed = true;
61	__set_bit(cpu, a->sched_cpus);
62	/*
63	 * We ignore errors because affinity is just an optimization.
64	 * This could happen for example with isolated CPUs or cpusets.
65	 * In this case the IPIs inside the kernel's perf API still work.
66	 */
67	sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus);
68	__clear_bit(cpu, a->sched_cpus);
69}
70
71static void __affinity__cleanup(struct affinity *a)
72{
73	int cpu_set_size = get_cpu_set_size();
74
75	if (a->changed)
76		sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
77	zfree(&a->sched_cpus);
78	zfree(&a->orig_cpus);
79}
80
81void affinity__cleanup(struct affinity *a)
82{
83	if (a != NULL)
84		__affinity__cleanup(a);
85}
86