1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 */
6
7#include <net/genetlink.h>
8#define CREATE_TRACE_POINTS
9#include <trace/events/devlink.h>
10
11#include "devl_internal.h"
12
13EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
14EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
15EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
16
17DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
18
19static struct devlink *devlinks_xa_get(unsigned long index)
20{
21	struct devlink *devlink;
22
23	rcu_read_lock();
24	devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED);
25	if (!devlink || !devlink_try_get(devlink))
26		devlink = NULL;
27	rcu_read_unlock();
28	return devlink;
29}
30
31/* devlink_rels xarray contains 1:1 relationships between
32 * devlink object and related nested devlink instance.
33 * The xarray index is used to get the nested object from
34 * the nested-in object code.
35 */
36static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1);
37
38#define DEVLINK_REL_IN_USE XA_MARK_0
39
40struct devlink_rel {
41	u32 index;
42	refcount_t refcount;
43	u32 devlink_index;
44	struct {
45		u32 devlink_index;
46		u32 obj_index;
47		devlink_rel_notify_cb_t *notify_cb;
48		devlink_rel_cleanup_cb_t *cleanup_cb;
49		struct delayed_work notify_work;
50	} nested_in;
51};
52
53static void devlink_rel_free(struct devlink_rel *rel)
54{
55	xa_erase(&devlink_rels, rel->index);
56	kfree(rel);
57}
58
59static void __devlink_rel_get(struct devlink_rel *rel)
60{
61	refcount_inc(&rel->refcount);
62}
63
64static void __devlink_rel_put(struct devlink_rel *rel)
65{
66	if (refcount_dec_and_test(&rel->refcount))
67		devlink_rel_free(rel);
68}
69
70static void devlink_rel_nested_in_notify_work(struct work_struct *work)
71{
72	struct devlink_rel *rel = container_of(work, struct devlink_rel,
73					       nested_in.notify_work.work);
74	struct devlink *devlink;
75
76	devlink = devlinks_xa_get(rel->nested_in.devlink_index);
77	if (!devlink)
78		goto rel_put;
79	if (!devl_trylock(devlink)) {
80		devlink_put(devlink);
81		goto reschedule_work;
82	}
83	if (!devl_is_registered(devlink)) {
84		devl_unlock(devlink);
85		devlink_put(devlink);
86		goto rel_put;
87	}
88	if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE))
89		rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index);
90	rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index);
91	devl_unlock(devlink);
92	devlink_put(devlink);
93
94rel_put:
95	__devlink_rel_put(rel);
96	return;
97
98reschedule_work:
99	schedule_delayed_work(&rel->nested_in.notify_work, 1);
100}
101
102static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel)
103{
104	__devlink_rel_get(rel);
105	schedule_delayed_work(&rel->nested_in.notify_work, 0);
106}
107
108static struct devlink_rel *devlink_rel_alloc(void)
109{
110	struct devlink_rel *rel;
111	static u32 next;
112	int err;
113
114	rel = kzalloc(sizeof(*rel), GFP_KERNEL);
115	if (!rel)
116		return ERR_PTR(-ENOMEM);
117
118	err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel,
119			      xa_limit_32b, &next, GFP_KERNEL);
120	if (err) {
121		kfree(rel);
122		return ERR_PTR(err);
123	}
124
125	refcount_set(&rel->refcount, 1);
126	INIT_DELAYED_WORK(&rel->nested_in.notify_work,
127			  &devlink_rel_nested_in_notify_work);
128	return rel;
129}
130
131static void devlink_rel_put(struct devlink *devlink)
132{
133	struct devlink_rel *rel = devlink->rel;
134
135	if (!rel)
136		return;
137	xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
138	devlink_rel_nested_in_notify_work_schedule(rel);
139	__devlink_rel_put(rel);
140	devlink->rel = NULL;
141}
142
143void devlink_rel_nested_in_clear(u32 rel_index)
144{
145	xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE);
146}
147
148int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index,
149			      u32 obj_index, devlink_rel_notify_cb_t *notify_cb,
150			      devlink_rel_cleanup_cb_t *cleanup_cb,
151			      struct devlink *devlink)
152{
153	struct devlink_rel *rel = devlink_rel_alloc();
154
155	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
156
157	if (IS_ERR(rel))
158		return PTR_ERR(rel);
159
160	rel->devlink_index = devlink->index;
161	rel->nested_in.devlink_index = devlink_index;
162	rel->nested_in.obj_index = obj_index;
163	rel->nested_in.notify_cb = notify_cb;
164	rel->nested_in.cleanup_cb = cleanup_cb;
165	*rel_index = rel->index;
166	xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
167	devlink->rel = rel;
168	return 0;
169}
170
171/**
172 * devlink_rel_nested_in_notify - Notify the object this devlink
173 *				  instance is nested in.
174 * @devlink: devlink
175 *
176 * This is called upon network namespace change of devlink instance.
177 * In case this devlink instance is nested in another devlink object,
178 * a notification of a change of this object should be sent
179 * over netlink. The parent devlink instance lock needs to be
180 * taken during the notification preparation.
181 * However, since the devlink lock of nested instance is held here,
182 * we would end with wrong devlink instance lock ordering and
183 * deadlock. Therefore the work is utilized to avoid that.
184 */
185void devlink_rel_nested_in_notify(struct devlink *devlink)
186{
187	struct devlink_rel *rel = devlink->rel;
188
189	if (!rel)
190		return;
191	devlink_rel_nested_in_notify_work_schedule(rel);
192}
193
194static struct devlink_rel *devlink_rel_find(unsigned long rel_index)
195{
196	return xa_find(&devlink_rels, &rel_index, rel_index,
197		       DEVLINK_REL_IN_USE);
198}
199
200static struct devlink *devlink_rel_devlink_get(u32 rel_index)
201{
202	struct devlink_rel *rel;
203	u32 devlink_index;
204
205	if (!rel_index)
206		return NULL;
207	xa_lock(&devlink_rels);
208	rel = devlink_rel_find(rel_index);
209	if (rel)
210		devlink_index = rel->devlink_index;
211	xa_unlock(&devlink_rels);
212	if (!rel)
213		return NULL;
214	return devlinks_xa_get(devlink_index);
215}
216
217int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink,
218				   u32 rel_index, int attrtype,
219				   bool *msg_updated)
220{
221	struct net *net = devlink_net(devlink);
222	struct devlink *rel_devlink;
223	int err;
224
225	rel_devlink = devlink_rel_devlink_get(rel_index);
226	if (!rel_devlink)
227		return 0;
228	err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype);
229	devlink_put(rel_devlink);
230	if (!err && msg_updated)
231		*msg_updated = true;
232	return err;
233}
234
235void *devlink_priv(struct devlink *devlink)
236{
237	return &devlink->priv;
238}
239EXPORT_SYMBOL_GPL(devlink_priv);
240
241struct devlink *priv_to_devlink(void *priv)
242{
243	return container_of(priv, struct devlink, priv);
244}
245EXPORT_SYMBOL_GPL(priv_to_devlink);
246
247struct device *devlink_to_dev(const struct devlink *devlink)
248{
249	return devlink->dev;
250}
251EXPORT_SYMBOL_GPL(devlink_to_dev);
252
253struct net *devlink_net(const struct devlink *devlink)
254{
255	return read_pnet(&devlink->_net);
256}
257EXPORT_SYMBOL_GPL(devlink_net);
258
259void devl_assert_locked(struct devlink *devlink)
260{
261	lockdep_assert_held(&devlink->lock);
262}
263EXPORT_SYMBOL_GPL(devl_assert_locked);
264
265#ifdef CONFIG_LOCKDEP
266/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
267bool devl_lock_is_held(struct devlink *devlink)
268{
269	return lockdep_is_held(&devlink->lock);
270}
271EXPORT_SYMBOL_GPL(devl_lock_is_held);
272#endif
273
274void devl_lock(struct devlink *devlink)
275{
276	mutex_lock(&devlink->lock);
277}
278EXPORT_SYMBOL_GPL(devl_lock);
279
280int devl_trylock(struct devlink *devlink)
281{
282	return mutex_trylock(&devlink->lock);
283}
284EXPORT_SYMBOL_GPL(devl_trylock);
285
286void devl_unlock(struct devlink *devlink)
287{
288	mutex_unlock(&devlink->lock);
289}
290EXPORT_SYMBOL_GPL(devl_unlock);
291
292/**
293 * devlink_try_get() - try to obtain a reference on a devlink instance
294 * @devlink: instance to reference
295 *
296 * Obtain a reference on a devlink instance. A reference on a devlink instance
297 * only implies that it's safe to take the instance lock. It does not imply
298 * that the instance is registered, use devl_is_registered() after taking
299 * the instance lock to check registration status.
300 */
301struct devlink *__must_check devlink_try_get(struct devlink *devlink)
302{
303	if (refcount_inc_not_zero(&devlink->refcount))
304		return devlink;
305	return NULL;
306}
307
308static void devlink_release(struct work_struct *work)
309{
310	struct devlink *devlink;
311
312	devlink = container_of(to_rcu_work(work), struct devlink, rwork);
313
314	mutex_destroy(&devlink->lock);
315	lockdep_unregister_key(&devlink->lock_key);
316	put_device(devlink->dev);
317	kfree(devlink);
318}
319
320void devlink_put(struct devlink *devlink)
321{
322	if (refcount_dec_and_test(&devlink->refcount))
323		queue_rcu_work(system_wq, &devlink->rwork);
324}
325
326struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp)
327{
328	struct devlink *devlink = NULL;
329
330	rcu_read_lock();
331retry:
332	devlink = xa_find(&devlinks, indexp, ULONG_MAX, DEVLINK_REGISTERED);
333	if (!devlink)
334		goto unlock;
335
336	if (!devlink_try_get(devlink))
337		goto next;
338	if (!net_eq(devlink_net(devlink), net)) {
339		devlink_put(devlink);
340		goto next;
341	}
342unlock:
343	rcu_read_unlock();
344	return devlink;
345
346next:
347	(*indexp)++;
348	goto retry;
349}
350
351/**
352 * devl_register - Register devlink instance
353 * @devlink: devlink
354 */
355int devl_register(struct devlink *devlink)
356{
357	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
358	devl_assert_locked(devlink);
359
360	xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
361	devlink_notify_register(devlink);
362	devlink_rel_nested_in_notify(devlink);
363
364	return 0;
365}
366EXPORT_SYMBOL_GPL(devl_register);
367
368void devlink_register(struct devlink *devlink)
369{
370	devl_lock(devlink);
371	devl_register(devlink);
372	devl_unlock(devlink);
373}
374EXPORT_SYMBOL_GPL(devlink_register);
375
376/**
377 * devl_unregister - Unregister devlink instance
378 * @devlink: devlink
379 */
380void devl_unregister(struct devlink *devlink)
381{
382	ASSERT_DEVLINK_REGISTERED(devlink);
383	devl_assert_locked(devlink);
384
385	devlink_notify_unregister(devlink);
386	xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
387	devlink_rel_put(devlink);
388}
389EXPORT_SYMBOL_GPL(devl_unregister);
390
391void devlink_unregister(struct devlink *devlink)
392{
393	devl_lock(devlink);
394	devl_unregister(devlink);
395	devl_unlock(devlink);
396}
397EXPORT_SYMBOL_GPL(devlink_unregister);
398
399/**
400 *	devlink_alloc_ns - Allocate new devlink instance resources
401 *	in specific namespace
402 *
403 *	@ops: ops
404 *	@priv_size: size of user private data
405 *	@net: net namespace
406 *	@dev: parent device
407 *
408 *	Allocate new devlink instance resources, including devlink index
409 *	and name.
410 */
411struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
412				 size_t priv_size, struct net *net,
413				 struct device *dev)
414{
415	struct devlink *devlink;
416	static u32 last_id;
417	int ret;
418
419	WARN_ON(!ops || !dev);
420	if (!devlink_reload_actions_valid(ops))
421		return NULL;
422
423	devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
424	if (!devlink)
425		return NULL;
426
427	ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
428			      &last_id, GFP_KERNEL);
429	if (ret < 0)
430		goto err_xa_alloc;
431
432	devlink->dev = get_device(dev);
433	devlink->ops = ops;
434	xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC);
435	xa_init_flags(&devlink->params, XA_FLAGS_ALLOC);
436	xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
437	xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC);
438	write_pnet(&devlink->_net, net);
439	INIT_LIST_HEAD(&devlink->rate_list);
440	INIT_LIST_HEAD(&devlink->linecard_list);
441	INIT_LIST_HEAD(&devlink->sb_list);
442	INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
443	INIT_LIST_HEAD(&devlink->resource_list);
444	INIT_LIST_HEAD(&devlink->region_list);
445	INIT_LIST_HEAD(&devlink->reporter_list);
446	INIT_LIST_HEAD(&devlink->trap_list);
447	INIT_LIST_HEAD(&devlink->trap_group_list);
448	INIT_LIST_HEAD(&devlink->trap_policer_list);
449	INIT_RCU_WORK(&devlink->rwork, devlink_release);
450	lockdep_register_key(&devlink->lock_key);
451	mutex_init(&devlink->lock);
452	lockdep_set_class(&devlink->lock, &devlink->lock_key);
453	refcount_set(&devlink->refcount, 1);
454
455	return devlink;
456
457err_xa_alloc:
458	kfree(devlink);
459	return NULL;
460}
461EXPORT_SYMBOL_GPL(devlink_alloc_ns);
462
463/**
464 *	devlink_free - Free devlink instance resources
465 *
466 *	@devlink: devlink
467 */
468void devlink_free(struct devlink *devlink)
469{
470	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
471
472	WARN_ON(!list_empty(&devlink->trap_policer_list));
473	WARN_ON(!list_empty(&devlink->trap_group_list));
474	WARN_ON(!list_empty(&devlink->trap_list));
475	WARN_ON(!list_empty(&devlink->reporter_list));
476	WARN_ON(!list_empty(&devlink->region_list));
477	WARN_ON(!list_empty(&devlink->resource_list));
478	WARN_ON(!list_empty(&devlink->dpipe_table_list));
479	WARN_ON(!list_empty(&devlink->sb_list));
480	WARN_ON(!list_empty(&devlink->rate_list));
481	WARN_ON(!list_empty(&devlink->linecard_list));
482	WARN_ON(!xa_empty(&devlink->ports));
483
484	xa_destroy(&devlink->nested_rels);
485	xa_destroy(&devlink->snapshot_ids);
486	xa_destroy(&devlink->params);
487	xa_destroy(&devlink->ports);
488
489	xa_erase(&devlinks, devlink->index);
490
491	devlink_put(devlink);
492}
493EXPORT_SYMBOL_GPL(devlink_free);
494
495static void __net_exit devlink_pernet_pre_exit(struct net *net)
496{
497	struct devlink *devlink;
498	u32 actions_performed;
499	unsigned long index;
500	int err;
501
502	/* In case network namespace is getting destroyed, reload
503	 * all devlink instances from this namespace into init_net.
504	 */
505	devlinks_xa_for_each_registered_get(net, index, devlink) {
506		devl_dev_lock(devlink, true);
507		err = 0;
508		if (devl_is_registered(devlink))
509			err = devlink_reload(devlink, &init_net,
510					     DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
511					     DEVLINK_RELOAD_LIMIT_UNSPEC,
512					     &actions_performed, NULL);
513		devl_dev_unlock(devlink, true);
514		devlink_put(devlink);
515		if (err && err != -EOPNOTSUPP)
516			pr_warn("Failed to reload devlink instance into init_net\n");
517	}
518}
519
520static struct pernet_operations devlink_pernet_ops __net_initdata = {
521	.pre_exit = devlink_pernet_pre_exit,
522};
523
524static struct notifier_block devlink_port_netdevice_nb = {
525	.notifier_call = devlink_port_netdevice_event,
526};
527
528static int __init devlink_init(void)
529{
530	int err;
531
532	err = register_pernet_subsys(&devlink_pernet_ops);
533	if (err)
534		goto out;
535	err = genl_register_family(&devlink_nl_family);
536	if (err)
537		goto out_unreg_pernet_subsys;
538	err = register_netdevice_notifier(&devlink_port_netdevice_nb);
539	if (!err)
540		return 0;
541
542	genl_unregister_family(&devlink_nl_family);
543
544out_unreg_pernet_subsys:
545	unregister_pernet_subsys(&devlink_pernet_ops);
546out:
547	WARN_ON(err);
548	return err;
549}
550
551subsys_initcall(devlink_init);
552