1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* Copyright (c) 2020 Mellanox Technologies Ltd */ 3 4#include <linux/mlx5/driver.h> 5#include "eswitch.h" 6#include "priv.h" 7#include "sf/dev/dev.h" 8#include "mlx5_ifc_vhca_event.h" 9#include "vhca_event.h" 10#include "ecpf.h" 11#define CREATE_TRACE_POINTS 12#include "diag/sf_tracepoint.h" 13 14struct mlx5_sf { 15 struct mlx5_devlink_port dl_port; 16 unsigned int port_index; 17 u32 controller; 18 u16 id; 19 u16 hw_fn_id; 20 u16 hw_state; 21}; 22 23static void *mlx5_sf_by_dl_port(struct devlink_port *dl_port) 24{ 25 struct mlx5_devlink_port *mlx5_dl_port = mlx5_devlink_port_get(dl_port); 26 27 return container_of(mlx5_dl_port, struct mlx5_sf, dl_port); 28} 29 30struct mlx5_sf_table { 31 struct mlx5_core_dev *dev; /* To refer from notifier context. */ 32 struct xarray function_ids; /* function id based lookup. */ 33 struct mutex sf_state_lock; /* Serializes sf state among user cmds & vhca event handler. */ 34 struct notifier_block esw_nb; 35 struct notifier_block vhca_nb; 36 struct notifier_block mdev_nb; 37}; 38 39static struct mlx5_sf * 40mlx5_sf_lookup_by_function_id(struct mlx5_sf_table *table, unsigned int fn_id) 41{ 42 return xa_load(&table->function_ids, fn_id); 43} 44 45static int mlx5_sf_function_id_insert(struct mlx5_sf_table *table, struct mlx5_sf *sf) 46{ 47 return xa_insert(&table->function_ids, sf->hw_fn_id, sf, GFP_KERNEL); 48} 49 50static void mlx5_sf_function_id_erase(struct mlx5_sf_table *table, struct mlx5_sf *sf) 51{ 52 xa_erase(&table->function_ids, sf->hw_fn_id); 53} 54 55static struct mlx5_sf * 56mlx5_sf_alloc(struct mlx5_sf_table *table, struct mlx5_eswitch *esw, 57 u32 controller, u32 sfnum, struct netlink_ext_ack *extack) 58{ 59 unsigned int dl_port_index; 60 struct mlx5_sf *sf; 61 u16 hw_fn_id; 62 int id_err; 63 int err; 64 65 if (!mlx5_esw_offloads_controller_valid(esw, controller)) { 66 NL_SET_ERR_MSG_MOD(extack, "Invalid controller number"); 67 return ERR_PTR(-EINVAL); 68 } 69 70 id_err = mlx5_sf_hw_table_sf_alloc(table->dev, controller, sfnum); 71 if (id_err < 0) { 72 err = id_err; 73 goto id_err; 74 } 75 76 sf = kzalloc(sizeof(*sf), GFP_KERNEL); 77 if (!sf) { 78 err = -ENOMEM; 79 goto alloc_err; 80 } 81 sf->id = id_err; 82 hw_fn_id = mlx5_sf_sw_to_hw_id(table->dev, controller, sf->id); 83 dl_port_index = mlx5_esw_vport_to_devlink_port_index(table->dev, hw_fn_id); 84 sf->port_index = dl_port_index; 85 sf->hw_fn_id = hw_fn_id; 86 sf->hw_state = MLX5_VHCA_STATE_ALLOCATED; 87 sf->controller = controller; 88 89 err = mlx5_sf_function_id_insert(table, sf); 90 if (err) 91 goto insert_err; 92 93 return sf; 94 95insert_err: 96 kfree(sf); 97alloc_err: 98 mlx5_sf_hw_table_sf_free(table->dev, controller, id_err); 99id_err: 100 if (err == -EEXIST) 101 NL_SET_ERR_MSG_MOD(extack, "SF already exist. Choose different sfnum"); 102 return ERR_PTR(err); 103} 104 105static void mlx5_sf_free(struct mlx5_sf_table *table, struct mlx5_sf *sf) 106{ 107 mlx5_sf_hw_table_sf_free(table->dev, sf->controller, sf->id); 108 trace_mlx5_sf_free(table->dev, sf->port_index, sf->controller, sf->hw_fn_id); 109 kfree(sf); 110} 111 112static enum devlink_port_fn_state mlx5_sf_to_devlink_state(u8 hw_state) 113{ 114 switch (hw_state) { 115 case MLX5_VHCA_STATE_ACTIVE: 116 case MLX5_VHCA_STATE_IN_USE: 117 return DEVLINK_PORT_FN_STATE_ACTIVE; 118 case MLX5_VHCA_STATE_INVALID: 119 case MLX5_VHCA_STATE_ALLOCATED: 120 case MLX5_VHCA_STATE_TEARDOWN_REQUEST: 121 default: 122 return DEVLINK_PORT_FN_STATE_INACTIVE; 123 } 124} 125 126static enum devlink_port_fn_opstate mlx5_sf_to_devlink_opstate(u8 hw_state) 127{ 128 switch (hw_state) { 129 case MLX5_VHCA_STATE_IN_USE: 130 case MLX5_VHCA_STATE_TEARDOWN_REQUEST: 131 return DEVLINK_PORT_FN_OPSTATE_ATTACHED; 132 case MLX5_VHCA_STATE_INVALID: 133 case MLX5_VHCA_STATE_ALLOCATED: 134 case MLX5_VHCA_STATE_ACTIVE: 135 default: 136 return DEVLINK_PORT_FN_OPSTATE_DETACHED; 137 } 138} 139 140static bool mlx5_sf_is_active(const struct mlx5_sf *sf) 141{ 142 return sf->hw_state == MLX5_VHCA_STATE_ACTIVE || sf->hw_state == MLX5_VHCA_STATE_IN_USE; 143} 144 145int mlx5_devlink_sf_port_fn_state_get(struct devlink_port *dl_port, 146 enum devlink_port_fn_state *state, 147 enum devlink_port_fn_opstate *opstate, 148 struct netlink_ext_ack *extack) 149{ 150 struct mlx5_core_dev *dev = devlink_priv(dl_port->devlink); 151 struct mlx5_sf_table *table = dev->priv.sf_table; 152 struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); 153 154 mutex_lock(&table->sf_state_lock); 155 *state = mlx5_sf_to_devlink_state(sf->hw_state); 156 *opstate = mlx5_sf_to_devlink_opstate(sf->hw_state); 157 mutex_unlock(&table->sf_state_lock); 158 return 0; 159} 160 161static int mlx5_sf_activate(struct mlx5_core_dev *dev, struct mlx5_sf *sf, 162 struct netlink_ext_ack *extack) 163{ 164 int err; 165 166 if (mlx5_sf_is_active(sf)) 167 return 0; 168 if (sf->hw_state != MLX5_VHCA_STATE_ALLOCATED) { 169 NL_SET_ERR_MSG_MOD(extack, "SF is inactivated but it is still attached"); 170 return -EBUSY; 171 } 172 173 err = mlx5_cmd_sf_enable_hca(dev, sf->hw_fn_id); 174 if (err) 175 return err; 176 177 sf->hw_state = MLX5_VHCA_STATE_ACTIVE; 178 trace_mlx5_sf_activate(dev, sf->port_index, sf->controller, sf->hw_fn_id); 179 return 0; 180} 181 182static int mlx5_sf_deactivate(struct mlx5_core_dev *dev, struct mlx5_sf *sf) 183{ 184 int err; 185 186 if (!mlx5_sf_is_active(sf)) 187 return 0; 188 189 err = mlx5_cmd_sf_disable_hca(dev, sf->hw_fn_id); 190 if (err) 191 return err; 192 193 sf->hw_state = MLX5_VHCA_STATE_TEARDOWN_REQUEST; 194 trace_mlx5_sf_deactivate(dev, sf->port_index, sf->controller, sf->hw_fn_id); 195 return 0; 196} 197 198static int mlx5_sf_state_set(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, 199 struct mlx5_sf *sf, 200 enum devlink_port_fn_state state, 201 struct netlink_ext_ack *extack) 202{ 203 int err = 0; 204 205 mutex_lock(&table->sf_state_lock); 206 if (state == mlx5_sf_to_devlink_state(sf->hw_state)) 207 goto out; 208 if (state == DEVLINK_PORT_FN_STATE_ACTIVE) 209 err = mlx5_sf_activate(dev, sf, extack); 210 else if (state == DEVLINK_PORT_FN_STATE_INACTIVE) 211 err = mlx5_sf_deactivate(dev, sf); 212 else 213 err = -EINVAL; 214out: 215 mutex_unlock(&table->sf_state_lock); 216 return err; 217} 218 219int mlx5_devlink_sf_port_fn_state_set(struct devlink_port *dl_port, 220 enum devlink_port_fn_state state, 221 struct netlink_ext_ack *extack) 222{ 223 struct mlx5_core_dev *dev = devlink_priv(dl_port->devlink); 224 struct mlx5_sf_table *table = dev->priv.sf_table; 225 struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); 226 227 return mlx5_sf_state_set(dev, table, sf, state, extack); 228} 229 230static int mlx5_sf_add(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, 231 const struct devlink_port_new_attrs *new_attr, 232 struct netlink_ext_ack *extack, 233 struct devlink_port **dl_port) 234{ 235 struct mlx5_eswitch *esw = dev->priv.eswitch; 236 struct mlx5_sf *sf; 237 int err; 238 239 sf = mlx5_sf_alloc(table, esw, new_attr->controller, new_attr->sfnum, extack); 240 if (IS_ERR(sf)) 241 return PTR_ERR(sf); 242 243 err = mlx5_eswitch_load_sf_vport(esw, sf->hw_fn_id, MLX5_VPORT_UC_ADDR_CHANGE, 244 &sf->dl_port, new_attr->controller, new_attr->sfnum); 245 if (err) 246 goto esw_err; 247 *dl_port = &sf->dl_port.dl_port; 248 trace_mlx5_sf_add(dev, sf->port_index, sf->controller, sf->hw_fn_id, new_attr->sfnum); 249 return 0; 250 251esw_err: 252 mlx5_sf_free(table, sf); 253 return err; 254} 255 256static int 257mlx5_sf_new_check_attr(struct mlx5_core_dev *dev, const struct devlink_port_new_attrs *new_attr, 258 struct netlink_ext_ack *extack) 259{ 260 if (new_attr->flavour != DEVLINK_PORT_FLAVOUR_PCI_SF) { 261 NL_SET_ERR_MSG_MOD(extack, "Driver supports only SF port addition"); 262 return -EOPNOTSUPP; 263 } 264 if (new_attr->port_index_valid) { 265 NL_SET_ERR_MSG_MOD(extack, 266 "Driver does not support user defined port index assignment"); 267 return -EOPNOTSUPP; 268 } 269 if (!new_attr->sfnum_valid) { 270 NL_SET_ERR_MSG_MOD(extack, 271 "User must provide unique sfnum. Driver does not support auto assignment"); 272 return -EOPNOTSUPP; 273 } 274 if (new_attr->controller_valid && new_attr->controller && 275 !mlx5_core_is_ecpf_esw_manager(dev)) { 276 NL_SET_ERR_MSG_MOD(extack, "External controller is unsupported"); 277 return -EOPNOTSUPP; 278 } 279 if (new_attr->pfnum != mlx5_get_dev_index(dev)) { 280 NL_SET_ERR_MSG_MOD(extack, "Invalid pfnum supplied"); 281 return -EOPNOTSUPP; 282 } 283 return 0; 284} 285 286static bool mlx5_sf_table_supported(const struct mlx5_core_dev *dev) 287{ 288 return dev->priv.eswitch && MLX5_ESWITCH_MANAGER(dev) && 289 mlx5_sf_hw_table_supported(dev); 290} 291 292int mlx5_devlink_sf_port_new(struct devlink *devlink, 293 const struct devlink_port_new_attrs *new_attr, 294 struct netlink_ext_ack *extack, 295 struct devlink_port **dl_port) 296{ 297 struct mlx5_core_dev *dev = devlink_priv(devlink); 298 struct mlx5_sf_table *table = dev->priv.sf_table; 299 int err; 300 301 err = mlx5_sf_new_check_attr(dev, new_attr, extack); 302 if (err) 303 return err; 304 305 if (!mlx5_sf_table_supported(dev)) { 306 NL_SET_ERR_MSG_MOD(extack, "SF ports are not supported."); 307 return -EOPNOTSUPP; 308 } 309 310 if (!is_mdev_switchdev_mode(dev)) { 311 NL_SET_ERR_MSG_MOD(extack, 312 "SF ports are only supported in eswitch switchdev mode."); 313 return -EOPNOTSUPP; 314 } 315 316 return mlx5_sf_add(dev, table, new_attr, extack, dl_port); 317} 318 319static void mlx5_sf_dealloc(struct mlx5_sf_table *table, struct mlx5_sf *sf) 320{ 321 mutex_lock(&table->sf_state_lock); 322 323 mlx5_sf_function_id_erase(table, sf); 324 325 if (sf->hw_state == MLX5_VHCA_STATE_ALLOCATED) { 326 mlx5_sf_free(table, sf); 327 } else if (mlx5_sf_is_active(sf)) { 328 /* Even if its active, it is treated as in_use because by the time, 329 * it is disabled here, it may getting used. So it is safe to 330 * always look for the event to ensure that it is recycled only after 331 * firmware gives confirmation that it is detached by the driver. 332 */ 333 mlx5_cmd_sf_disable_hca(table->dev, sf->hw_fn_id); 334 mlx5_sf_hw_table_sf_deferred_free(table->dev, sf->controller, sf->id); 335 kfree(sf); 336 } else { 337 mlx5_sf_hw_table_sf_deferred_free(table->dev, sf->controller, sf->id); 338 kfree(sf); 339 } 340 341 mutex_unlock(&table->sf_state_lock); 342} 343 344static void mlx5_sf_del(struct mlx5_sf_table *table, struct mlx5_sf *sf) 345{ 346 struct mlx5_eswitch *esw = table->dev->priv.eswitch; 347 348 mlx5_eswitch_unload_sf_vport(esw, sf->hw_fn_id); 349 mlx5_sf_dealloc(table, sf); 350} 351 352int mlx5_devlink_sf_port_del(struct devlink *devlink, 353 struct devlink_port *dl_port, 354 struct netlink_ext_ack *extack) 355{ 356 struct mlx5_core_dev *dev = devlink_priv(devlink); 357 struct mlx5_sf_table *table = dev->priv.sf_table; 358 struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); 359 360 mlx5_sf_del(table, sf); 361 return 0; 362} 363 364static bool mlx5_sf_state_update_check(const struct mlx5_sf *sf, u8 new_state) 365{ 366 if (sf->hw_state == MLX5_VHCA_STATE_ACTIVE && new_state == MLX5_VHCA_STATE_IN_USE) 367 return true; 368 369 if (sf->hw_state == MLX5_VHCA_STATE_IN_USE && new_state == MLX5_VHCA_STATE_ACTIVE) 370 return true; 371 372 if (sf->hw_state == MLX5_VHCA_STATE_TEARDOWN_REQUEST && 373 new_state == MLX5_VHCA_STATE_ALLOCATED) 374 return true; 375 376 return false; 377} 378 379static int mlx5_sf_vhca_event(struct notifier_block *nb, unsigned long opcode, void *data) 380{ 381 struct mlx5_sf_table *table = container_of(nb, struct mlx5_sf_table, vhca_nb); 382 const struct mlx5_vhca_state_event *event = data; 383 bool update = false; 384 struct mlx5_sf *sf; 385 386 mutex_lock(&table->sf_state_lock); 387 sf = mlx5_sf_lookup_by_function_id(table, event->function_id); 388 if (!sf) 389 goto unlock; 390 391 /* When driver is attached or detached to a function, an event 392 * notifies such state change. 393 */ 394 update = mlx5_sf_state_update_check(sf, event->new_vhca_state); 395 if (update) 396 sf->hw_state = event->new_vhca_state; 397 trace_mlx5_sf_update_state(table->dev, sf->port_index, sf->controller, 398 sf->hw_fn_id, sf->hw_state); 399unlock: 400 mutex_unlock(&table->sf_state_lock); 401 return 0; 402} 403 404static void mlx5_sf_del_all(struct mlx5_sf_table *table) 405{ 406 unsigned long index; 407 struct mlx5_sf *sf; 408 409 xa_for_each(&table->function_ids, index, sf) 410 mlx5_sf_del(table, sf); 411} 412 413static int mlx5_sf_esw_event(struct notifier_block *nb, unsigned long event, void *data) 414{ 415 struct mlx5_sf_table *table = container_of(nb, struct mlx5_sf_table, esw_nb); 416 const struct mlx5_esw_event_info *mode = data; 417 418 switch (mode->new_mode) { 419 case MLX5_ESWITCH_LEGACY: 420 mlx5_sf_del_all(table); 421 break; 422 default: 423 break; 424 } 425 426 return 0; 427} 428 429static int mlx5_sf_mdev_event(struct notifier_block *nb, unsigned long event, void *data) 430{ 431 struct mlx5_sf_table *table = container_of(nb, struct mlx5_sf_table, mdev_nb); 432 struct mlx5_sf_peer_devlink_event_ctx *event_ctx = data; 433 int ret = NOTIFY_DONE; 434 struct mlx5_sf *sf; 435 436 if (event != MLX5_DRIVER_EVENT_SF_PEER_DEVLINK) 437 return NOTIFY_DONE; 438 439 440 mutex_lock(&table->sf_state_lock); 441 sf = mlx5_sf_lookup_by_function_id(table, event_ctx->fn_id); 442 if (!sf) 443 goto out; 444 445 event_ctx->err = devl_port_fn_devlink_set(&sf->dl_port.dl_port, 446 event_ctx->devlink); 447 448 ret = NOTIFY_OK; 449out: 450 mutex_unlock(&table->sf_state_lock); 451 return ret; 452} 453 454int mlx5_sf_table_init(struct mlx5_core_dev *dev) 455{ 456 struct mlx5_sf_table *table; 457 int err; 458 459 if (!mlx5_sf_table_supported(dev) || !mlx5_vhca_event_supported(dev)) 460 return 0; 461 462 table = kzalloc(sizeof(*table), GFP_KERNEL); 463 if (!table) 464 return -ENOMEM; 465 466 mutex_init(&table->sf_state_lock); 467 table->dev = dev; 468 xa_init(&table->function_ids); 469 dev->priv.sf_table = table; 470 table->esw_nb.notifier_call = mlx5_sf_esw_event; 471 err = mlx5_esw_event_notifier_register(dev->priv.eswitch, &table->esw_nb); 472 if (err) 473 goto reg_err; 474 475 table->vhca_nb.notifier_call = mlx5_sf_vhca_event; 476 err = mlx5_vhca_event_notifier_register(table->dev, &table->vhca_nb); 477 if (err) 478 goto vhca_err; 479 480 table->mdev_nb.notifier_call = mlx5_sf_mdev_event; 481 mlx5_blocking_notifier_register(dev, &table->mdev_nb); 482 483 return 0; 484 485vhca_err: 486 mlx5_esw_event_notifier_unregister(dev->priv.eswitch, &table->esw_nb); 487reg_err: 488 mutex_destroy(&table->sf_state_lock); 489 kfree(table); 490 dev->priv.sf_table = NULL; 491 return err; 492} 493 494void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev) 495{ 496 struct mlx5_sf_table *table = dev->priv.sf_table; 497 498 if (!table) 499 return; 500 501 mlx5_blocking_notifier_unregister(dev, &table->mdev_nb); 502 mlx5_vhca_event_notifier_unregister(table->dev, &table->vhca_nb); 503 mlx5_esw_event_notifier_unregister(dev->priv.eswitch, &table->esw_nb); 504 mutex_destroy(&table->sf_state_lock); 505 WARN_ON(!xa_empty(&table->function_ids)); 506 kfree(table); 507} 508