1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> 5 */ 6 7#include <net/genetlink.h> 8#define CREATE_TRACE_POINTS 9#include <trace/events/devlink.h> 10 11#include "devl_internal.h" 12 13EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg); 14EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr); 15EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report); 16 17DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC); 18 19static struct devlink *devlinks_xa_get(unsigned long index) 20{ 21 struct devlink *devlink; 22 23 rcu_read_lock(); 24 devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED); 25 if (!devlink || !devlink_try_get(devlink)) 26 devlink = NULL; 27 rcu_read_unlock(); 28 return devlink; 29} 30 31/* devlink_rels xarray contains 1:1 relationships between 32 * devlink object and related nested devlink instance. 33 * The xarray index is used to get the nested object from 34 * the nested-in object code. 35 */ 36static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1); 37 38#define DEVLINK_REL_IN_USE XA_MARK_0 39 40struct devlink_rel { 41 u32 index; 42 refcount_t refcount; 43 u32 devlink_index; 44 struct { 45 u32 devlink_index; 46 u32 obj_index; 47 devlink_rel_notify_cb_t *notify_cb; 48 devlink_rel_cleanup_cb_t *cleanup_cb; 49 struct delayed_work notify_work; 50 } nested_in; 51}; 52 53static void devlink_rel_free(struct devlink_rel *rel) 54{ 55 xa_erase(&devlink_rels, rel->index); 56 kfree(rel); 57} 58 59static void __devlink_rel_get(struct devlink_rel *rel) 60{ 61 refcount_inc(&rel->refcount); 62} 63 64static void __devlink_rel_put(struct devlink_rel *rel) 65{ 66 if (refcount_dec_and_test(&rel->refcount)) 67 devlink_rel_free(rel); 68} 69 70static void devlink_rel_nested_in_notify_work(struct work_struct *work) 71{ 72 struct devlink_rel *rel = container_of(work, struct devlink_rel, 73 nested_in.notify_work.work); 74 struct devlink *devlink; 75 76 devlink = devlinks_xa_get(rel->nested_in.devlink_index); 77 if (!devlink) 78 goto rel_put; 79 if (!devl_trylock(devlink)) { 80 devlink_put(devlink); 81 goto reschedule_work; 82 } 83 if (!devl_is_registered(devlink)) { 84 devl_unlock(devlink); 85 devlink_put(devlink); 86 goto rel_put; 87 } 88 if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE)) 89 rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index); 90 rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index); 91 devl_unlock(devlink); 92 devlink_put(devlink); 93 94rel_put: 95 __devlink_rel_put(rel); 96 return; 97 98reschedule_work: 99 schedule_delayed_work(&rel->nested_in.notify_work, 1); 100} 101 102static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel) 103{ 104 __devlink_rel_get(rel); 105 schedule_delayed_work(&rel->nested_in.notify_work, 0); 106} 107 108static struct devlink_rel *devlink_rel_alloc(void) 109{ 110 struct devlink_rel *rel; 111 static u32 next; 112 int err; 113 114 rel = kzalloc(sizeof(*rel), GFP_KERNEL); 115 if (!rel) 116 return ERR_PTR(-ENOMEM); 117 118 err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel, 119 xa_limit_32b, &next, GFP_KERNEL); 120 if (err) { 121 kfree(rel); 122 return ERR_PTR(err); 123 } 124 125 refcount_set(&rel->refcount, 1); 126 INIT_DELAYED_WORK(&rel->nested_in.notify_work, 127 &devlink_rel_nested_in_notify_work); 128 return rel; 129} 130 131static void devlink_rel_put(struct devlink *devlink) 132{ 133 struct devlink_rel *rel = devlink->rel; 134 135 if (!rel) 136 return; 137 xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); 138 devlink_rel_nested_in_notify_work_schedule(rel); 139 __devlink_rel_put(rel); 140 devlink->rel = NULL; 141} 142 143void devlink_rel_nested_in_clear(u32 rel_index) 144{ 145 xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE); 146} 147 148int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index, 149 u32 obj_index, devlink_rel_notify_cb_t *notify_cb, 150 devlink_rel_cleanup_cb_t *cleanup_cb, 151 struct devlink *devlink) 152{ 153 struct devlink_rel *rel = devlink_rel_alloc(); 154 155 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 156 157 if (IS_ERR(rel)) 158 return PTR_ERR(rel); 159 160 rel->devlink_index = devlink->index; 161 rel->nested_in.devlink_index = devlink_index; 162 rel->nested_in.obj_index = obj_index; 163 rel->nested_in.notify_cb = notify_cb; 164 rel->nested_in.cleanup_cb = cleanup_cb; 165 *rel_index = rel->index; 166 xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); 167 devlink->rel = rel; 168 return 0; 169} 170 171/** 172 * devlink_rel_nested_in_notify - Notify the object this devlink 173 * instance is nested in. 174 * @devlink: devlink 175 * 176 * This is called upon network namespace change of devlink instance. 177 * In case this devlink instance is nested in another devlink object, 178 * a notification of a change of this object should be sent 179 * over netlink. The parent devlink instance lock needs to be 180 * taken during the notification preparation. 181 * However, since the devlink lock of nested instance is held here, 182 * we would end with wrong devlink instance lock ordering and 183 * deadlock. Therefore the work is utilized to avoid that. 184 */ 185void devlink_rel_nested_in_notify(struct devlink *devlink) 186{ 187 struct devlink_rel *rel = devlink->rel; 188 189 if (!rel) 190 return; 191 devlink_rel_nested_in_notify_work_schedule(rel); 192} 193 194static struct devlink_rel *devlink_rel_find(unsigned long rel_index) 195{ 196 return xa_find(&devlink_rels, &rel_index, rel_index, 197 DEVLINK_REL_IN_USE); 198} 199 200static struct devlink *devlink_rel_devlink_get(u32 rel_index) 201{ 202 struct devlink_rel *rel; 203 u32 devlink_index; 204 205 if (!rel_index) 206 return NULL; 207 xa_lock(&devlink_rels); 208 rel = devlink_rel_find(rel_index); 209 if (rel) 210 devlink_index = rel->devlink_index; 211 xa_unlock(&devlink_rels); 212 if (!rel) 213 return NULL; 214 return devlinks_xa_get(devlink_index); 215} 216 217int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink, 218 u32 rel_index, int attrtype, 219 bool *msg_updated) 220{ 221 struct net *net = devlink_net(devlink); 222 struct devlink *rel_devlink; 223 int err; 224 225 rel_devlink = devlink_rel_devlink_get(rel_index); 226 if (!rel_devlink) 227 return 0; 228 err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype); 229 devlink_put(rel_devlink); 230 if (!err && msg_updated) 231 *msg_updated = true; 232 return err; 233} 234 235void *devlink_priv(struct devlink *devlink) 236{ 237 return &devlink->priv; 238} 239EXPORT_SYMBOL_GPL(devlink_priv); 240 241struct devlink *priv_to_devlink(void *priv) 242{ 243 return container_of(priv, struct devlink, priv); 244} 245EXPORT_SYMBOL_GPL(priv_to_devlink); 246 247struct device *devlink_to_dev(const struct devlink *devlink) 248{ 249 return devlink->dev; 250} 251EXPORT_SYMBOL_GPL(devlink_to_dev); 252 253struct net *devlink_net(const struct devlink *devlink) 254{ 255 return read_pnet(&devlink->_net); 256} 257EXPORT_SYMBOL_GPL(devlink_net); 258 259void devl_assert_locked(struct devlink *devlink) 260{ 261 lockdep_assert_held(&devlink->lock); 262} 263EXPORT_SYMBOL_GPL(devl_assert_locked); 264 265#ifdef CONFIG_LOCKDEP 266/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */ 267bool devl_lock_is_held(struct devlink *devlink) 268{ 269 return lockdep_is_held(&devlink->lock); 270} 271EXPORT_SYMBOL_GPL(devl_lock_is_held); 272#endif 273 274void devl_lock(struct devlink *devlink) 275{ 276 mutex_lock(&devlink->lock); 277} 278EXPORT_SYMBOL_GPL(devl_lock); 279 280int devl_trylock(struct devlink *devlink) 281{ 282 return mutex_trylock(&devlink->lock); 283} 284EXPORT_SYMBOL_GPL(devl_trylock); 285 286void devl_unlock(struct devlink *devlink) 287{ 288 mutex_unlock(&devlink->lock); 289} 290EXPORT_SYMBOL_GPL(devl_unlock); 291 292/** 293 * devlink_try_get() - try to obtain a reference on a devlink instance 294 * @devlink: instance to reference 295 * 296 * Obtain a reference on a devlink instance. A reference on a devlink instance 297 * only implies that it's safe to take the instance lock. It does not imply 298 * that the instance is registered, use devl_is_registered() after taking 299 * the instance lock to check registration status. 300 */ 301struct devlink *__must_check devlink_try_get(struct devlink *devlink) 302{ 303 if (refcount_inc_not_zero(&devlink->refcount)) 304 return devlink; 305 return NULL; 306} 307 308static void devlink_release(struct work_struct *work) 309{ 310 struct devlink *devlink; 311 312 devlink = container_of(to_rcu_work(work), struct devlink, rwork); 313 314 mutex_destroy(&devlink->lock); 315 lockdep_unregister_key(&devlink->lock_key); 316 put_device(devlink->dev); 317 kfree(devlink); 318} 319 320void devlink_put(struct devlink *devlink) 321{ 322 if (refcount_dec_and_test(&devlink->refcount)) 323 queue_rcu_work(system_wq, &devlink->rwork); 324} 325 326struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp) 327{ 328 struct devlink *devlink = NULL; 329 330 rcu_read_lock(); 331retry: 332 devlink = xa_find(&devlinks, indexp, ULONG_MAX, DEVLINK_REGISTERED); 333 if (!devlink) 334 goto unlock; 335 336 if (!devlink_try_get(devlink)) 337 goto next; 338 if (!net_eq(devlink_net(devlink), net)) { 339 devlink_put(devlink); 340 goto next; 341 } 342unlock: 343 rcu_read_unlock(); 344 return devlink; 345 346next: 347 (*indexp)++; 348 goto retry; 349} 350 351/** 352 * devl_register - Register devlink instance 353 * @devlink: devlink 354 */ 355int devl_register(struct devlink *devlink) 356{ 357 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 358 devl_assert_locked(devlink); 359 360 xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); 361 devlink_notify_register(devlink); 362 devlink_rel_nested_in_notify(devlink); 363 364 return 0; 365} 366EXPORT_SYMBOL_GPL(devl_register); 367 368void devlink_register(struct devlink *devlink) 369{ 370 devl_lock(devlink); 371 devl_register(devlink); 372 devl_unlock(devlink); 373} 374EXPORT_SYMBOL_GPL(devlink_register); 375 376/** 377 * devl_unregister - Unregister devlink instance 378 * @devlink: devlink 379 */ 380void devl_unregister(struct devlink *devlink) 381{ 382 ASSERT_DEVLINK_REGISTERED(devlink); 383 devl_assert_locked(devlink); 384 385 devlink_notify_unregister(devlink); 386 xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); 387 devlink_rel_put(devlink); 388} 389EXPORT_SYMBOL_GPL(devl_unregister); 390 391void devlink_unregister(struct devlink *devlink) 392{ 393 devl_lock(devlink); 394 devl_unregister(devlink); 395 devl_unlock(devlink); 396} 397EXPORT_SYMBOL_GPL(devlink_unregister); 398 399/** 400 * devlink_alloc_ns - Allocate new devlink instance resources 401 * in specific namespace 402 * 403 * @ops: ops 404 * @priv_size: size of user private data 405 * @net: net namespace 406 * @dev: parent device 407 * 408 * Allocate new devlink instance resources, including devlink index 409 * and name. 410 */ 411struct devlink *devlink_alloc_ns(const struct devlink_ops *ops, 412 size_t priv_size, struct net *net, 413 struct device *dev) 414{ 415 struct devlink *devlink; 416 static u32 last_id; 417 int ret; 418 419 WARN_ON(!ops || !dev); 420 if (!devlink_reload_actions_valid(ops)) 421 return NULL; 422 423 devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL); 424 if (!devlink) 425 return NULL; 426 427 ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b, 428 &last_id, GFP_KERNEL); 429 if (ret < 0) 430 goto err_xa_alloc; 431 432 devlink->dev = get_device(dev); 433 devlink->ops = ops; 434 xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC); 435 xa_init_flags(&devlink->params, XA_FLAGS_ALLOC); 436 xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC); 437 xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC); 438 write_pnet(&devlink->_net, net); 439 INIT_LIST_HEAD(&devlink->rate_list); 440 INIT_LIST_HEAD(&devlink->linecard_list); 441 INIT_LIST_HEAD(&devlink->sb_list); 442 INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list); 443 INIT_LIST_HEAD(&devlink->resource_list); 444 INIT_LIST_HEAD(&devlink->region_list); 445 INIT_LIST_HEAD(&devlink->reporter_list); 446 INIT_LIST_HEAD(&devlink->trap_list); 447 INIT_LIST_HEAD(&devlink->trap_group_list); 448 INIT_LIST_HEAD(&devlink->trap_policer_list); 449 INIT_RCU_WORK(&devlink->rwork, devlink_release); 450 lockdep_register_key(&devlink->lock_key); 451 mutex_init(&devlink->lock); 452 lockdep_set_class(&devlink->lock, &devlink->lock_key); 453 refcount_set(&devlink->refcount, 1); 454 455 return devlink; 456 457err_xa_alloc: 458 kfree(devlink); 459 return NULL; 460} 461EXPORT_SYMBOL_GPL(devlink_alloc_ns); 462 463/** 464 * devlink_free - Free devlink instance resources 465 * 466 * @devlink: devlink 467 */ 468void devlink_free(struct devlink *devlink) 469{ 470 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 471 472 WARN_ON(!list_empty(&devlink->trap_policer_list)); 473 WARN_ON(!list_empty(&devlink->trap_group_list)); 474 WARN_ON(!list_empty(&devlink->trap_list)); 475 WARN_ON(!list_empty(&devlink->reporter_list)); 476 WARN_ON(!list_empty(&devlink->region_list)); 477 WARN_ON(!list_empty(&devlink->resource_list)); 478 WARN_ON(!list_empty(&devlink->dpipe_table_list)); 479 WARN_ON(!list_empty(&devlink->sb_list)); 480 WARN_ON(!list_empty(&devlink->rate_list)); 481 WARN_ON(!list_empty(&devlink->linecard_list)); 482 WARN_ON(!xa_empty(&devlink->ports)); 483 484 xa_destroy(&devlink->nested_rels); 485 xa_destroy(&devlink->snapshot_ids); 486 xa_destroy(&devlink->params); 487 xa_destroy(&devlink->ports); 488 489 xa_erase(&devlinks, devlink->index); 490 491 devlink_put(devlink); 492} 493EXPORT_SYMBOL_GPL(devlink_free); 494 495static void __net_exit devlink_pernet_pre_exit(struct net *net) 496{ 497 struct devlink *devlink; 498 u32 actions_performed; 499 unsigned long index; 500 int err; 501 502 /* In case network namespace is getting destroyed, reload 503 * all devlink instances from this namespace into init_net. 504 */ 505 devlinks_xa_for_each_registered_get(net, index, devlink) { 506 devl_dev_lock(devlink, true); 507 err = 0; 508 if (devl_is_registered(devlink)) 509 err = devlink_reload(devlink, &init_net, 510 DEVLINK_RELOAD_ACTION_DRIVER_REINIT, 511 DEVLINK_RELOAD_LIMIT_UNSPEC, 512 &actions_performed, NULL); 513 devl_dev_unlock(devlink, true); 514 devlink_put(devlink); 515 if (err && err != -EOPNOTSUPP) 516 pr_warn("Failed to reload devlink instance into init_net\n"); 517 } 518} 519 520static struct pernet_operations devlink_pernet_ops __net_initdata = { 521 .pre_exit = devlink_pernet_pre_exit, 522}; 523 524static struct notifier_block devlink_port_netdevice_nb = { 525 .notifier_call = devlink_port_netdevice_event, 526}; 527 528static int __init devlink_init(void) 529{ 530 int err; 531 532 err = register_pernet_subsys(&devlink_pernet_ops); 533 if (err) 534 goto out; 535 err = genl_register_family(&devlink_nl_family); 536 if (err) 537 goto out_unreg_pernet_subsys; 538 err = register_netdevice_notifier(&devlink_port_netdevice_nb); 539 if (!err) 540 return 0; 541 542 genl_unregister_family(&devlink_nl_family); 543 544out_unreg_pernet_subsys: 545 unregister_pernet_subsys(&devlink_pernet_ops); 546out: 547 WARN_ON(err); 548 return err; 549} 550 551subsys_initcall(devlink_init); 552