1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 *  Housekeeping management. Manage the targets for routine code that can run on
4 *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10
11enum hk_flags {
12	HK_FLAG_TIMER		= BIT(HK_TYPE_TIMER),
13	HK_FLAG_RCU		= BIT(HK_TYPE_RCU),
14	HK_FLAG_MISC		= BIT(HK_TYPE_MISC),
15	HK_FLAG_SCHED		= BIT(HK_TYPE_SCHED),
16	HK_FLAG_TICK		= BIT(HK_TYPE_TICK),
17	HK_FLAG_DOMAIN		= BIT(HK_TYPE_DOMAIN),
18	HK_FLAG_WQ		= BIT(HK_TYPE_WQ),
19	HK_FLAG_MANAGED_IRQ	= BIT(HK_TYPE_MANAGED_IRQ),
20	HK_FLAG_KTHREAD		= BIT(HK_TYPE_KTHREAD),
21};
22
23DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
24EXPORT_SYMBOL_GPL(housekeeping_overridden);
25
26struct housekeeping {
27	cpumask_var_t cpumasks[HK_TYPE_MAX];
28	unsigned long flags;
29};
30
31static struct housekeeping housekeeping;
32
33bool housekeeping_enabled(enum hk_type type)
34{
35	return !!(housekeeping.flags & BIT(type));
36}
37EXPORT_SYMBOL_GPL(housekeeping_enabled);
38
39int housekeeping_any_cpu(enum hk_type type)
40{
41	int cpu;
42
43	if (static_branch_unlikely(&housekeeping_overridden)) {
44		if (housekeeping.flags & BIT(type)) {
45			cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
46			if (cpu < nr_cpu_ids)
47				return cpu;
48
49			cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
50			if (likely(cpu < nr_cpu_ids))
51				return cpu;
52			/*
53			 * Unless we have another problem this can only happen
54			 * at boot time before start_secondary() brings the 1st
55			 * housekeeping CPU up.
56			 */
57			WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
58				     type != HK_TYPE_TIMER);
59		}
60	}
61	return smp_processor_id();
62}
63EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
64
65const struct cpumask *housekeeping_cpumask(enum hk_type type)
66{
67	if (static_branch_unlikely(&housekeeping_overridden))
68		if (housekeeping.flags & BIT(type))
69			return housekeeping.cpumasks[type];
70	return cpu_possible_mask;
71}
72EXPORT_SYMBOL_GPL(housekeeping_cpumask);
73
74void housekeeping_affine(struct task_struct *t, enum hk_type type)
75{
76	if (static_branch_unlikely(&housekeeping_overridden))
77		if (housekeeping.flags & BIT(type))
78			set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
79}
80EXPORT_SYMBOL_GPL(housekeeping_affine);
81
82bool housekeeping_test_cpu(int cpu, enum hk_type type)
83{
84	if (static_branch_unlikely(&housekeeping_overridden))
85		if (housekeeping.flags & BIT(type))
86			return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
87	return true;
88}
89EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
90
91void __init housekeeping_init(void)
92{
93	enum hk_type type;
94
95	if (!housekeeping.flags)
96		return;
97
98	static_branch_enable(&housekeeping_overridden);
99
100	if (housekeeping.flags & HK_FLAG_TICK)
101		sched_tick_offload_init();
102
103	for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
104		/* We need at least one CPU to handle housekeeping work */
105		WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
106	}
107}
108
109static void __init housekeeping_setup_type(enum hk_type type,
110					   cpumask_var_t housekeeping_staging)
111{
112
113	alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
114	cpumask_copy(housekeeping.cpumasks[type],
115		     housekeeping_staging);
116}
117
118static int __init housekeeping_setup(char *str, unsigned long flags)
119{
120	cpumask_var_t non_housekeeping_mask, housekeeping_staging;
121	unsigned int first_cpu;
122	int err = 0;
123
124	if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
125		if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
126			pr_warn("Housekeeping: nohz unsupported."
127				" Build with CONFIG_NO_HZ_FULL\n");
128			return 0;
129		}
130	}
131
132	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
133	if (cpulist_parse(str, non_housekeeping_mask) < 0) {
134		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
135		goto free_non_housekeeping_mask;
136	}
137
138	alloc_bootmem_cpumask_var(&housekeeping_staging);
139	cpumask_andnot(housekeeping_staging,
140		       cpu_possible_mask, non_housekeeping_mask);
141
142	first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging);
143	if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
144		__cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
145		__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
146		if (!housekeeping.flags) {
147			pr_warn("Housekeeping: must include one present CPU, "
148				"using boot CPU:%d\n", smp_processor_id());
149		}
150	}
151
152	if (cpumask_empty(non_housekeeping_mask))
153		goto free_housekeeping_staging;
154
155	if (!housekeeping.flags) {
156		/* First setup call ("nohz_full=" or "isolcpus=") */
157		enum hk_type type;
158
159		for_each_set_bit(type, &flags, HK_TYPE_MAX)
160			housekeeping_setup_type(type, housekeeping_staging);
161	} else {
162		/* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
163		enum hk_type type;
164		unsigned long iter_flags = flags & housekeeping.flags;
165
166		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
167			if (!cpumask_equal(housekeeping_staging,
168					   housekeeping.cpumasks[type])) {
169				pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
170				goto free_housekeeping_staging;
171			}
172		}
173
174		iter_flags = flags & ~housekeeping.flags;
175
176		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
177			housekeeping_setup_type(type, housekeeping_staging);
178	}
179
180	if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
181		tick_nohz_full_setup(non_housekeeping_mask);
182
183	housekeeping.flags |= flags;
184	err = 1;
185
186free_housekeeping_staging:
187	free_bootmem_cpumask_var(housekeeping_staging);
188free_non_housekeeping_mask:
189	free_bootmem_cpumask_var(non_housekeeping_mask);
190
191	return err;
192}
193
194static int __init housekeeping_nohz_full_setup(char *str)
195{
196	unsigned long flags;
197
198	flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
199		HK_FLAG_MISC | HK_FLAG_KTHREAD;
200
201	return housekeeping_setup(str, flags);
202}
203__setup("nohz_full=", housekeeping_nohz_full_setup);
204
205static int __init housekeeping_isolcpus_setup(char *str)
206{
207	unsigned long flags = 0;
208	bool illegal = false;
209	char *par;
210	int len;
211
212	while (isalpha(*str)) {
213		if (!strncmp(str, "nohz,", 5)) {
214			str += 5;
215			flags |= HK_FLAG_TICK;
216			continue;
217		}
218
219		if (!strncmp(str, "domain,", 7)) {
220			str += 7;
221			flags |= HK_FLAG_DOMAIN;
222			continue;
223		}
224
225		if (!strncmp(str, "managed_irq,", 12)) {
226			str += 12;
227			flags |= HK_FLAG_MANAGED_IRQ;
228			continue;
229		}
230
231		/*
232		 * Skip unknown sub-parameter and validate that it is not
233		 * containing an invalid character.
234		 */
235		for (par = str, len = 0; *str && *str != ','; str++, len++) {
236			if (!isalpha(*str) && *str != '_')
237				illegal = true;
238		}
239
240		if (illegal) {
241			pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
242			return 0;
243		}
244
245		pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
246		str++;
247	}
248
249	/* Default behaviour for isolcpus without flags */
250	if (!flags)
251		flags |= HK_FLAG_DOMAIN;
252
253	return housekeeping_setup(str, flags);
254}
255__setup("isolcpus=", housekeeping_isolcpus_setup);
256