1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/* delayacct.h - per-task delay accounting
3 *
4 * Copyright (C) Shailabh Nagar, IBM Corp. 2006
5 */
6
7#ifndef _LINUX_DELAYACCT_H
8#define _LINUX_DELAYACCT_H
9
10#include <uapi/linux/taskstats.h>
11
12#ifdef CONFIG_TASK_DELAY_ACCT
13struct task_delay_info {
14	raw_spinlock_t	lock;
15
16	/* For each stat XXX, add following, aligned appropriately
17	 *
18	 * struct timespec XXX_start, XXX_end;
19	 * u64 XXX_delay;
20	 * u32 XXX_count;
21	 *
22	 * Atomicity of updates to XXX_delay, XXX_count protected by
23	 * single lock above (split into XXX_lock if contention is an issue).
24	 */
25
26	/*
27	 * XXX_count is incremented on every XXX operation, the delay
28	 * associated with the operation is added to XXX_delay.
29	 * XXX_delay contains the accumulated delay time in nanoseconds.
30	 */
31	u64 blkio_start;
32	u64 blkio_delay;	/* wait for sync block io completion */
33	u64 swapin_start;
34	u64 swapin_delay;	/* wait for swapin */
35	u32 blkio_count;	/* total count of the number of sync block */
36				/* io operations performed */
37	u32 swapin_count;	/* total count of swapin */
38
39	u64 freepages_start;
40	u64 freepages_delay;	/* wait for memory reclaim */
41
42	u64 thrashing_start;
43	u64 thrashing_delay;	/* wait for thrashing page */
44
45	u64 compact_start;
46	u64 compact_delay;	/* wait for memory compact */
47
48	u64 wpcopy_start;
49	u64 wpcopy_delay;	/* wait for write-protect copy */
50
51	u64 irq_delay;	/* wait for IRQ/SOFTIRQ */
52
53	u32 freepages_count;	/* total count of memory reclaim */
54	u32 thrashing_count;	/* total count of thrash waits */
55	u32 compact_count;	/* total count of memory compact */
56	u32 wpcopy_count;	/* total count of write-protect copy */
57	u32 irq_count;	/* total count of IRQ/SOFTIRQ */
58};
59#endif
60
61#include <linux/sched.h>
62#include <linux/slab.h>
63#include <linux/jump_label.h>
64
65#ifdef CONFIG_TASK_DELAY_ACCT
66DECLARE_STATIC_KEY_FALSE(delayacct_key);
67extern int delayacct_on;	/* Delay accounting turned on/off */
68extern struct kmem_cache *delayacct_cache;
69extern void delayacct_init(void);
70
71extern void __delayacct_tsk_init(struct task_struct *);
72extern void __delayacct_tsk_exit(struct task_struct *);
73extern void __delayacct_blkio_start(void);
74extern void __delayacct_blkio_end(struct task_struct *);
75extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
76extern __u64 __delayacct_blkio_ticks(struct task_struct *);
77extern void __delayacct_freepages_start(void);
78extern void __delayacct_freepages_end(void);
79extern void __delayacct_thrashing_start(bool *in_thrashing);
80extern void __delayacct_thrashing_end(bool *in_thrashing);
81extern void __delayacct_swapin_start(void);
82extern void __delayacct_swapin_end(void);
83extern void __delayacct_compact_start(void);
84extern void __delayacct_compact_end(void);
85extern void __delayacct_wpcopy_start(void);
86extern void __delayacct_wpcopy_end(void);
87extern void __delayacct_irq(struct task_struct *task, u32 delta);
88
89static inline void delayacct_tsk_init(struct task_struct *tsk)
90{
91	/* reinitialize in case parent's non-null pointer was dup'ed*/
92	tsk->delays = NULL;
93	if (delayacct_on)
94		__delayacct_tsk_init(tsk);
95}
96
97/* Free tsk->delays. Called from bad fork and __put_task_struct
98 * where there's no risk of tsk->delays being accessed elsewhere
99 */
100static inline void delayacct_tsk_free(struct task_struct *tsk)
101{
102	if (tsk->delays)
103		kmem_cache_free(delayacct_cache, tsk->delays);
104	tsk->delays = NULL;
105}
106
107static inline void delayacct_blkio_start(void)
108{
109	if (!static_branch_unlikely(&delayacct_key))
110		return;
111
112	if (current->delays)
113		__delayacct_blkio_start();
114}
115
116static inline void delayacct_blkio_end(struct task_struct *p)
117{
118	if (!static_branch_unlikely(&delayacct_key))
119		return;
120
121	if (p->delays)
122		__delayacct_blkio_end(p);
123}
124
125static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
126{
127	if (tsk->delays)
128		return __delayacct_blkio_ticks(tsk);
129	return 0;
130}
131
132static inline void delayacct_freepages_start(void)
133{
134	if (!static_branch_unlikely(&delayacct_key))
135		return;
136
137	if (current->delays)
138		__delayacct_freepages_start();
139}
140
141static inline void delayacct_freepages_end(void)
142{
143	if (!static_branch_unlikely(&delayacct_key))
144		return;
145
146	if (current->delays)
147		__delayacct_freepages_end();
148}
149
150static inline void delayacct_thrashing_start(bool *in_thrashing)
151{
152	if (!static_branch_unlikely(&delayacct_key))
153		return;
154
155	if (current->delays)
156		__delayacct_thrashing_start(in_thrashing);
157}
158
159static inline void delayacct_thrashing_end(bool *in_thrashing)
160{
161	if (!static_branch_unlikely(&delayacct_key))
162		return;
163
164	if (current->delays)
165		__delayacct_thrashing_end(in_thrashing);
166}
167
168static inline void delayacct_swapin_start(void)
169{
170	if (!static_branch_unlikely(&delayacct_key))
171		return;
172
173	if (current->delays)
174		__delayacct_swapin_start();
175}
176
177static inline void delayacct_swapin_end(void)
178{
179	if (!static_branch_unlikely(&delayacct_key))
180		return;
181
182	if (current->delays)
183		__delayacct_swapin_end();
184}
185
186static inline void delayacct_compact_start(void)
187{
188	if (!static_branch_unlikely(&delayacct_key))
189		return;
190
191	if (current->delays)
192		__delayacct_compact_start();
193}
194
195static inline void delayacct_compact_end(void)
196{
197	if (!static_branch_unlikely(&delayacct_key))
198		return;
199
200	if (current->delays)
201		__delayacct_compact_end();
202}
203
204static inline void delayacct_wpcopy_start(void)
205{
206	if (!static_branch_unlikely(&delayacct_key))
207		return;
208
209	if (current->delays)
210		__delayacct_wpcopy_start();
211}
212
213static inline void delayacct_wpcopy_end(void)
214{
215	if (!static_branch_unlikely(&delayacct_key))
216		return;
217
218	if (current->delays)
219		__delayacct_wpcopy_end();
220}
221
222static inline void delayacct_irq(struct task_struct *task, u32 delta)
223{
224	if (!static_branch_unlikely(&delayacct_key))
225		return;
226
227	if (task->delays)
228		__delayacct_irq(task, delta);
229}
230
231#else
232static inline void delayacct_init(void)
233{}
234static inline void delayacct_tsk_init(struct task_struct *tsk)
235{}
236static inline void delayacct_tsk_free(struct task_struct *tsk)
237{}
238static inline void delayacct_blkio_start(void)
239{}
240static inline void delayacct_blkio_end(struct task_struct *p)
241{}
242static inline int delayacct_add_tsk(struct taskstats *d,
243					struct task_struct *tsk)
244{ return 0; }
245static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
246{ return 0; }
247static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
248{ return 0; }
249static inline void delayacct_freepages_start(void)
250{}
251static inline void delayacct_freepages_end(void)
252{}
253static inline void delayacct_thrashing_start(bool *in_thrashing)
254{}
255static inline void delayacct_thrashing_end(bool *in_thrashing)
256{}
257static inline void delayacct_swapin_start(void)
258{}
259static inline void delayacct_swapin_end(void)
260{}
261static inline void delayacct_compact_start(void)
262{}
263static inline void delayacct_compact_end(void)
264{}
265static inline void delayacct_wpcopy_start(void)
266{}
267static inline void delayacct_wpcopy_end(void)
268{}
269static inline void delayacct_irq(struct task_struct *task, u32 delta)
270{}
271
272#endif /* CONFIG_TASK_DELAY_ACCT */
273
274#endif
275