1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_SCHED_IDLE_H
3#define _LINUX_SCHED_IDLE_H
4
5#include <linux/sched.h>
6
7enum cpu_idle_type {
8	CPU_IDLE,
9	CPU_NOT_IDLE,
10	CPU_NEWLY_IDLE,
11	CPU_MAX_IDLE_TYPES
12};
13
14#ifdef CONFIG_SMP
15extern void wake_up_if_idle(int cpu);
16#else
17static inline void wake_up_if_idle(int cpu) { }
18#endif
19
20/*
21 * Idle thread specific functions to determine the need_resched
22 * polling state.
23 */
24#ifdef TIF_POLLING_NRFLAG
25
26#ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H
27
28static __always_inline void __current_set_polling(void)
29{
30	arch_set_bit(TIF_POLLING_NRFLAG,
31		     (unsigned long *)(&current_thread_info()->flags));
32}
33
34static __always_inline void __current_clr_polling(void)
35{
36	arch_clear_bit(TIF_POLLING_NRFLAG,
37		       (unsigned long *)(&current_thread_info()->flags));
38}
39
40#else
41
42static __always_inline void __current_set_polling(void)
43{
44	set_bit(TIF_POLLING_NRFLAG,
45		(unsigned long *)(&current_thread_info()->flags));
46}
47
48static __always_inline void __current_clr_polling(void)
49{
50	clear_bit(TIF_POLLING_NRFLAG,
51		  (unsigned long *)(&current_thread_info()->flags));
52}
53
54#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H */
55
56static __always_inline bool __must_check current_set_polling_and_test(void)
57{
58	__current_set_polling();
59
60	/*
61	 * Polling state must be visible before we test NEED_RESCHED,
62	 * paired by resched_curr()
63	 */
64	smp_mb__after_atomic();
65
66	return unlikely(tif_need_resched());
67}
68
69static __always_inline bool __must_check current_clr_polling_and_test(void)
70{
71	__current_clr_polling();
72
73	/*
74	 * Polling state must be visible before we test NEED_RESCHED,
75	 * paired by resched_curr()
76	 */
77	smp_mb__after_atomic();
78
79	return unlikely(tif_need_resched());
80}
81
82#else
83static inline void __current_set_polling(void) { }
84static inline void __current_clr_polling(void) { }
85
86static inline bool __must_check current_set_polling_and_test(void)
87{
88	return unlikely(tif_need_resched());
89}
90static inline bool __must_check current_clr_polling_and_test(void)
91{
92	return unlikely(tif_need_resched());
93}
94#endif
95
96static __always_inline void current_clr_polling(void)
97{
98	__current_clr_polling();
99
100	/*
101	 * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
102	 * Once the bit is cleared, we'll get IPIs with every new
103	 * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
104	 * fold.
105	 */
106	smp_mb(); /* paired with resched_curr() */
107
108	preempt_fold_need_resched();
109}
110
111#endif /* _LINUX_SCHED_IDLE_H */
112