1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 *
4 * Authors: Waiman Long <longman@redhat.com>
5 */
6
7#include "lock_events.h"
8
9#ifdef CONFIG_LOCK_EVENT_COUNTS
10#ifdef CONFIG_PARAVIRT_SPINLOCKS
11/*
12 * Collect pvqspinlock locking event counts
13 */
14#include <linux/sched.h>
15#include <linux/sched/clock.h>
16#include <linux/fs.h>
17
18#define EVENT_COUNT(ev)	lockevents[LOCKEVENT_ ## ev]
19
20/*
21 * PV specific per-cpu counter
22 */
23static DEFINE_PER_CPU(u64, pv_kick_time);
24
25/*
26 * Function to read and return the PV qspinlock counts.
27 *
28 * The following counters are handled specially:
29 * 1. pv_latency_kick
30 *    Average kick latency (ns) = pv_latency_kick/pv_kick_unlock
31 * 2. pv_latency_wake
32 *    Average wake latency (ns) = pv_latency_wake/pv_kick_wake
33 * 3. pv_hash_hops
34 *    Average hops/hash = pv_hash_hops/pv_kick_unlock
35 */
36ssize_t lockevent_read(struct file *file, char __user *user_buf,
37		       size_t count, loff_t *ppos)
38{
39	char buf[64];
40	int cpu, id, len;
41	u64 sum = 0, kicks = 0;
42
43	/*
44	 * Get the counter ID stored in file->f_inode->i_private
45	 */
46	id = (long)file_inode(file)->i_private;
47
48	if (id >= lockevent_num)
49		return -EBADF;
50
51	for_each_possible_cpu(cpu) {
52		sum += per_cpu(lockevents[id], cpu);
53		/*
54		 * Need to sum additional counters for some of them
55		 */
56		switch (id) {
57
58		case LOCKEVENT_pv_latency_kick:
59		case LOCKEVENT_pv_hash_hops:
60			kicks += per_cpu(EVENT_COUNT(pv_kick_unlock), cpu);
61			break;
62
63		case LOCKEVENT_pv_latency_wake:
64			kicks += per_cpu(EVENT_COUNT(pv_kick_wake), cpu);
65			break;
66		}
67	}
68
69	if (id == LOCKEVENT_pv_hash_hops) {
70		u64 frac = 0;
71
72		if (kicks) {
73			frac = 100ULL * do_div(sum, kicks);
74			frac = DIV_ROUND_CLOSEST_ULL(frac, kicks);
75		}
76
77		/*
78		 * Return a X.XX decimal number
79		 */
80		len = snprintf(buf, sizeof(buf) - 1, "%llu.%02llu\n",
81			       sum, frac);
82	} else {
83		/*
84		 * Round to the nearest ns
85		 */
86		if ((id == LOCKEVENT_pv_latency_kick) ||
87		    (id == LOCKEVENT_pv_latency_wake)) {
88			if (kicks)
89				sum = DIV_ROUND_CLOSEST_ULL(sum, kicks);
90		}
91		len = snprintf(buf, sizeof(buf) - 1, "%llu\n", sum);
92	}
93
94	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
95}
96
97/*
98 * PV hash hop count
99 */
100static inline void lockevent_pv_hop(int hopcnt)
101{
102	this_cpu_add(EVENT_COUNT(pv_hash_hops), hopcnt);
103}
104
105/*
106 * Replacement function for pv_kick()
107 */
108static inline void __pv_kick(int cpu)
109{
110	u64 start = sched_clock();
111
112	per_cpu(pv_kick_time, cpu) = start;
113	pv_kick(cpu);
114	this_cpu_add(EVENT_COUNT(pv_latency_kick), sched_clock() - start);
115}
116
117/*
118 * Replacement function for pv_wait()
119 */
120static inline void __pv_wait(u8 *ptr, u8 val)
121{
122	u64 *pkick_time = this_cpu_ptr(&pv_kick_time);
123
124	*pkick_time = 0;
125	pv_wait(ptr, val);
126	if (*pkick_time) {
127		this_cpu_add(EVENT_COUNT(pv_latency_wake),
128			     sched_clock() - *pkick_time);
129		lockevent_inc(pv_kick_wake);
130	}
131}
132
133#define pv_kick(c)	__pv_kick(c)
134#define pv_wait(p, v)	__pv_wait(p, v)
135
136#endif /* CONFIG_PARAVIRT_SPINLOCKS */
137
138#else /* CONFIG_LOCK_EVENT_COUNTS */
139
140static inline void lockevent_pv_hop(int hopcnt)	{ }
141
142#endif /* CONFIG_LOCK_EVENT_COUNTS */
143