1/*
2 * Fast batching percpu counters.
3 */
4
5#include <linux/percpu_counter.h>
6#include <linux/notifier.h>
7#include <linux/mutex.h>
8#include <linux/init.h>
9#include <linux/cpu.h>
10#include <linux/module.h>
11
12static LIST_HEAD(percpu_counters);
13static DEFINE_MUTEX(percpu_counters_lock);
14
15void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
16{
17	int cpu;
18
19	spin_lock(&fbc->lock);
20	for_each_possible_cpu(cpu) {
21		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
22		*pcount = 0;
23	}
24	fbc->count = amount;
25	spin_unlock(&fbc->lock);
26}
27EXPORT_SYMBOL(percpu_counter_set);
28
29void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
30{
31	s64 count;
32	s32 *pcount;
33	int cpu = get_cpu();
34
35	pcount = per_cpu_ptr(fbc->counters, cpu);
36	count = *pcount + amount;
37	if (count >= batch || count <= -batch) {
38		spin_lock(&fbc->lock);
39		fbc->count += count;
40		*pcount = 0;
41		spin_unlock(&fbc->lock);
42	} else {
43		*pcount = count;
44	}
45	put_cpu();
46}
47EXPORT_SYMBOL(__percpu_counter_add);
48
49/*
50 * Add up all the per-cpu counts, return the result.  This is a more accurate
51 * but much slower version of percpu_counter_read_positive()
52 */
53s64 __percpu_counter_sum(struct percpu_counter *fbc)
54{
55	s64 ret;
56	int cpu;
57
58	spin_lock(&fbc->lock);
59	ret = fbc->count;
60	for_each_online_cpu(cpu) {
61		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
62		ret += *pcount;
63	}
64	spin_unlock(&fbc->lock);
65	return ret;
66}
67EXPORT_SYMBOL(__percpu_counter_sum);
68
69int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
70			  struct lock_class_key *key)
71{
72	spin_lock_init(&fbc->lock);
73	lockdep_set_class(&fbc->lock, key);
74	fbc->count = amount;
75	fbc->counters = alloc_percpu(s32);
76	if (!fbc->counters)
77		return -ENOMEM;
78#ifdef CONFIG_HOTPLUG_CPU
79	INIT_LIST_HEAD(&fbc->list);
80	mutex_lock(&percpu_counters_lock);
81	list_add(&fbc->list, &percpu_counters);
82	mutex_unlock(&percpu_counters_lock);
83#endif
84	return 0;
85}
86EXPORT_SYMBOL(__percpu_counter_init);
87
88void percpu_counter_destroy(struct percpu_counter *fbc)
89{
90	if (!fbc->counters)
91		return;
92
93#ifdef CONFIG_HOTPLUG_CPU
94	mutex_lock(&percpu_counters_lock);
95	list_del(&fbc->list);
96	mutex_unlock(&percpu_counters_lock);
97#endif
98	free_percpu(fbc->counters);
99	fbc->counters = NULL;
100}
101EXPORT_SYMBOL(percpu_counter_destroy);
102
103int percpu_counter_batch __read_mostly = 32;
104EXPORT_SYMBOL(percpu_counter_batch);
105
106static void compute_batch_value(void)
107{
108	int nr = num_online_cpus();
109
110	percpu_counter_batch = max(32, nr*2);
111}
112
113static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
114					unsigned long action, void *hcpu)
115{
116#ifdef CONFIG_HOTPLUG_CPU
117	unsigned int cpu;
118	struct percpu_counter *fbc;
119
120	compute_batch_value();
121	if (action != CPU_DEAD)
122		return NOTIFY_OK;
123
124	cpu = (unsigned long)hcpu;
125	mutex_lock(&percpu_counters_lock);
126	list_for_each_entry(fbc, &percpu_counters, list) {
127		s32 *pcount;
128		unsigned long flags;
129
130		spin_lock_irqsave(&fbc->lock, flags);
131		pcount = per_cpu_ptr(fbc->counters, cpu);
132		fbc->count += *pcount;
133		*pcount = 0;
134		spin_unlock_irqrestore(&fbc->lock, flags);
135	}
136	mutex_unlock(&percpu_counters_lock);
137#endif
138	return NOTIFY_OK;
139}
140
141/*
142 * Compare counter against given value.
143 * Return 1 if greater, 0 if equal and -1 if less
144 */
145int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
146{
147	s64	count;
148
149	count = percpu_counter_read(fbc);
150	/* Check to see if rough count will be sufficient for comparison */
151	if (abs(count - rhs) > (percpu_counter_batch*num_online_cpus())) {
152		if (count > rhs)
153			return 1;
154		else
155			return -1;
156	}
157	/* Need to use precise count */
158	count = percpu_counter_sum(fbc);
159	if (count > rhs)
160		return 1;
161	else if (count < rhs)
162		return -1;
163	else
164		return 0;
165}
166EXPORT_SYMBOL(percpu_counter_compare);
167
168static int __init percpu_counter_startup(void)
169{
170	compute_batch_value();
171	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
172	return 0;
173}
174module_init(percpu_counter_startup);
175