1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2018 Mellanox Technologies */
3
4#include <linux/mlx5/vport.h>
5#include <linux/list.h>
6#include "lib/devcom.h"
7#include "mlx5_core.h"
8
9static LIST_HEAD(devcom_dev_list);
10static LIST_HEAD(devcom_comp_list);
11/* protect device list */
12static DEFINE_MUTEX(dev_list_lock);
13/* protect component list */
14static DEFINE_MUTEX(comp_list_lock);
15
16#define devcom_for_each_component(iter) \
17	list_for_each_entry(iter, &devcom_comp_list, comp_list)
18
19struct mlx5_devcom_dev {
20	struct list_head list;
21	struct mlx5_core_dev *dev;
22	struct kref ref;
23};
24
25struct mlx5_devcom_comp {
26	struct list_head comp_list;
27	enum mlx5_devcom_component id;
28	u64 key;
29	struct list_head comp_dev_list_head;
30	mlx5_devcom_event_handler_t handler;
31	struct kref ref;
32	bool ready;
33	struct rw_semaphore sem;
34	struct lock_class_key lock_key;
35};
36
37struct mlx5_devcom_comp_dev {
38	struct list_head list;
39	struct mlx5_devcom_comp *comp;
40	struct mlx5_devcom_dev *devc;
41	void __rcu *data;
42};
43
44static bool devcom_dev_exists(struct mlx5_core_dev *dev)
45{
46	struct mlx5_devcom_dev *iter;
47
48	list_for_each_entry(iter, &devcom_dev_list, list)
49		if (iter->dev == dev)
50			return true;
51
52	return false;
53}
54
55static struct mlx5_devcom_dev *
56mlx5_devcom_dev_alloc(struct mlx5_core_dev *dev)
57{
58	struct mlx5_devcom_dev *devc;
59
60	devc = kzalloc(sizeof(*devc), GFP_KERNEL);
61	if (!devc)
62		return NULL;
63
64	devc->dev = dev;
65	kref_init(&devc->ref);
66	return devc;
67}
68
69struct mlx5_devcom_dev *
70mlx5_devcom_register_device(struct mlx5_core_dev *dev)
71{
72	struct mlx5_devcom_dev *devc;
73
74	mutex_lock(&dev_list_lock);
75
76	if (devcom_dev_exists(dev)) {
77		devc = ERR_PTR(-EEXIST);
78		goto out;
79	}
80
81	devc = mlx5_devcom_dev_alloc(dev);
82	if (!devc) {
83		devc = ERR_PTR(-ENOMEM);
84		goto out;
85	}
86
87	list_add_tail(&devc->list, &devcom_dev_list);
88out:
89	mutex_unlock(&dev_list_lock);
90	return devc;
91}
92
93static void
94mlx5_devcom_dev_release(struct kref *ref)
95{
96	struct mlx5_devcom_dev *devc = container_of(ref, struct mlx5_devcom_dev, ref);
97
98	mutex_lock(&dev_list_lock);
99	list_del(&devc->list);
100	mutex_unlock(&dev_list_lock);
101	kfree(devc);
102}
103
104void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc)
105{
106	if (!IS_ERR_OR_NULL(devc))
107		kref_put(&devc->ref, mlx5_devcom_dev_release);
108}
109
110static struct mlx5_devcom_comp *
111mlx5_devcom_comp_alloc(u64 id, u64 key, mlx5_devcom_event_handler_t handler)
112{
113	struct mlx5_devcom_comp *comp;
114
115	comp = kzalloc(sizeof(*comp), GFP_KERNEL);
116	if (!comp)
117		return ERR_PTR(-ENOMEM);
118
119	comp->id = id;
120	comp->key = key;
121	comp->handler = handler;
122	init_rwsem(&comp->sem);
123	lockdep_register_key(&comp->lock_key);
124	lockdep_set_class(&comp->sem, &comp->lock_key);
125	kref_init(&comp->ref);
126	INIT_LIST_HEAD(&comp->comp_dev_list_head);
127
128	return comp;
129}
130
131static void
132mlx5_devcom_comp_release(struct kref *ref)
133{
134	struct mlx5_devcom_comp *comp = container_of(ref, struct mlx5_devcom_comp, ref);
135
136	mutex_lock(&comp_list_lock);
137	list_del(&comp->comp_list);
138	mutex_unlock(&comp_list_lock);
139	lockdep_unregister_key(&comp->lock_key);
140	kfree(comp);
141}
142
143static struct mlx5_devcom_comp_dev *
144devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc,
145		      struct mlx5_devcom_comp *comp,
146		      void *data)
147{
148	struct mlx5_devcom_comp_dev *devcom;
149
150	devcom = kzalloc(sizeof(*devcom), GFP_KERNEL);
151	if (!devcom)
152		return ERR_PTR(-ENOMEM);
153
154	kref_get(&devc->ref);
155	devcom->devc = devc;
156	devcom->comp = comp;
157	rcu_assign_pointer(devcom->data, data);
158
159	down_write(&comp->sem);
160	list_add_tail(&devcom->list, &comp->comp_dev_list_head);
161	up_write(&comp->sem);
162
163	return devcom;
164}
165
166static void
167devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom)
168{
169	struct mlx5_devcom_comp *comp = devcom->comp;
170
171	down_write(&comp->sem);
172	list_del(&devcom->list);
173	up_write(&comp->sem);
174
175	kref_put(&devcom->devc->ref, mlx5_devcom_dev_release);
176	kfree(devcom);
177	kref_put(&comp->ref, mlx5_devcom_comp_release);
178}
179
180static bool
181devcom_component_equal(struct mlx5_devcom_comp *devcom,
182		       enum mlx5_devcom_component id,
183		       u64 key)
184{
185	return devcom->id == id && devcom->key == key;
186}
187
188static struct mlx5_devcom_comp *
189devcom_component_get(struct mlx5_devcom_dev *devc,
190		     enum mlx5_devcom_component id,
191		     u64 key,
192		     mlx5_devcom_event_handler_t handler)
193{
194	struct mlx5_devcom_comp *comp;
195
196	devcom_for_each_component(comp) {
197		if (devcom_component_equal(comp, id, key)) {
198			if (handler == comp->handler) {
199				kref_get(&comp->ref);
200				return comp;
201			}
202
203			mlx5_core_err(devc->dev,
204				      "Cannot register existing devcom component with different handler\n");
205			return ERR_PTR(-EINVAL);
206		}
207	}
208
209	return NULL;
210}
211
212struct mlx5_devcom_comp_dev *
213mlx5_devcom_register_component(struct mlx5_devcom_dev *devc,
214			       enum mlx5_devcom_component id,
215			       u64 key,
216			       mlx5_devcom_event_handler_t handler,
217			       void *data)
218{
219	struct mlx5_devcom_comp_dev *devcom;
220	struct mlx5_devcom_comp *comp;
221
222	if (IS_ERR_OR_NULL(devc))
223		return ERR_PTR(-EINVAL);
224
225	mutex_lock(&comp_list_lock);
226	comp = devcom_component_get(devc, id, key, handler);
227	if (IS_ERR(comp)) {
228		devcom = ERR_PTR(-EINVAL);
229		goto out_unlock;
230	}
231
232	if (!comp) {
233		comp = mlx5_devcom_comp_alloc(id, key, handler);
234		if (IS_ERR(comp)) {
235			devcom = ERR_CAST(comp);
236			goto out_unlock;
237		}
238		list_add_tail(&comp->comp_list, &devcom_comp_list);
239	}
240	mutex_unlock(&comp_list_lock);
241
242	devcom = devcom_alloc_comp_dev(devc, comp, data);
243	if (IS_ERR(devcom))
244		kref_put(&comp->ref, mlx5_devcom_comp_release);
245
246	return devcom;
247
248out_unlock:
249	mutex_unlock(&comp_list_lock);
250	return devcom;
251}
252
253void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom)
254{
255	if (!IS_ERR_OR_NULL(devcom))
256		devcom_free_comp_dev(devcom);
257}
258
259int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom)
260{
261	struct mlx5_devcom_comp *comp = devcom->comp;
262
263	return kref_read(&comp->ref);
264}
265
266int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom,
267			   int event, int rollback_event,
268			   void *event_data)
269{
270	struct mlx5_devcom_comp_dev *pos;
271	struct mlx5_devcom_comp *comp;
272	int err = 0;
273	void *data;
274
275	if (IS_ERR_OR_NULL(devcom))
276		return -ENODEV;
277
278	comp = devcom->comp;
279	down_write(&comp->sem);
280	list_for_each_entry(pos, &comp->comp_dev_list_head, list) {
281		data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem));
282
283		if (pos != devcom && data) {
284			err = comp->handler(event, data, event_data);
285			if (err)
286				goto rollback;
287		}
288	}
289
290	up_write(&comp->sem);
291	return 0;
292
293rollback:
294	if (list_entry_is_head(pos, &comp->comp_dev_list_head, list))
295		goto out;
296	pos = list_prev_entry(pos, list);
297	list_for_each_entry_from_reverse(pos, &comp->comp_dev_list_head, list) {
298		data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem));
299
300		if (pos != devcom && data)
301			comp->handler(rollback_event, data, event_data);
302	}
303out:
304	up_write(&comp->sem);
305	return err;
306}
307
308void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready)
309{
310	WARN_ON(!rwsem_is_locked(&devcom->comp->sem));
311
312	WRITE_ONCE(devcom->comp->ready, ready);
313}
314
315bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom)
316{
317	if (IS_ERR_OR_NULL(devcom))
318		return false;
319
320	return READ_ONCE(devcom->comp->ready);
321}
322
323bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev *devcom)
324{
325	struct mlx5_devcom_comp *comp;
326
327	if (IS_ERR_OR_NULL(devcom))
328		return false;
329
330	comp = devcom->comp;
331	down_read(&comp->sem);
332	if (!READ_ONCE(comp->ready)) {
333		up_read(&comp->sem);
334		return false;
335	}
336
337	return true;
338}
339
340void mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev *devcom)
341{
342	up_read(&devcom->comp->sem);
343}
344
345void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev *devcom,
346				     struct mlx5_devcom_comp_dev **pos)
347{
348	struct mlx5_devcom_comp *comp = devcom->comp;
349	struct mlx5_devcom_comp_dev *tmp;
350	void *data;
351
352	tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list);
353
354	list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) {
355		if (tmp != devcom) {
356			data = rcu_dereference_protected(tmp->data, lockdep_is_held(&comp->sem));
357			if (data)
358				break;
359		}
360	}
361
362	if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list))
363		return NULL;
364
365	*pos = tmp;
366	return data;
367}
368
369void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom,
370					 struct mlx5_devcom_comp_dev **pos)
371{
372	struct mlx5_devcom_comp *comp = devcom->comp;
373	struct mlx5_devcom_comp_dev *tmp;
374	void *data;
375
376	tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list);
377
378	list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) {
379		if (tmp != devcom) {
380			/* This can change concurrently, however 'data' pointer will remain
381			 * valid for the duration of RCU read section.
382			 */
383			if (!READ_ONCE(comp->ready))
384				return NULL;
385			data = rcu_dereference(tmp->data);
386			if (data)
387				break;
388		}
389	}
390
391	if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list))
392		return NULL;
393
394	*pos = tmp;
395	return data;
396}
397
398void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom)
399{
400	if (IS_ERR_OR_NULL(devcom))
401		return;
402	down_write(&devcom->comp->sem);
403}
404
405void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom)
406{
407	if (IS_ERR_OR_NULL(devcom))
408		return;
409	up_write(&devcom->comp->sem);
410}
411
412int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom)
413{
414	if (IS_ERR_OR_NULL(devcom))
415		return 0;
416	return down_write_trylock(&devcom->comp->sem);
417}
418