1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ 3 4#include "lib/sd.h" 5#include "mlx5_core.h" 6#include "lib/mlx5.h" 7#include "fs_cmd.h" 8#include <linux/mlx5/vport.h> 9#include <linux/debugfs.h> 10 11#define sd_info(__dev, format, ...) \ 12 dev_info((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__) 13#define sd_warn(__dev, format, ...) \ 14 dev_warn((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__) 15 16struct mlx5_sd { 17 u32 group_id; 18 u8 host_buses; 19 struct mlx5_devcom_comp_dev *devcom; 20 struct dentry *dfs; 21 bool primary; 22 union { 23 struct { /* primary */ 24 struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1]; 25 struct mlx5_flow_table *tx_ft; 26 }; 27 struct { /* secondary */ 28 struct mlx5_core_dev *primary_dev; 29 u32 alias_obj_id; 30 }; 31 }; 32}; 33 34static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev) 35{ 36 struct mlx5_sd *sd = mlx5_get_sd(dev); 37 38 if (!sd) 39 return 1; 40 41 return sd->host_buses; 42} 43 44static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev) 45{ 46 struct mlx5_sd *sd = mlx5_get_sd(dev); 47 48 if (!sd) 49 return dev; 50 51 return sd->primary ? dev : sd->primary_dev; 52} 53 54struct mlx5_core_dev * 55mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx) 56{ 57 struct mlx5_sd *sd; 58 59 if (idx == 0) 60 return primary; 61 62 if (idx >= mlx5_sd_get_host_buses(primary)) 63 return NULL; 64 65 sd = mlx5_get_sd(primary); 66 return sd->secondaries[idx - 1]; 67} 68 69int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix) 70{ 71 return ch_ix % mlx5_sd_get_host_buses(dev); 72} 73 74int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix) 75{ 76 return ch_ix / mlx5_sd_get_host_buses(dev); 77} 78 79struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix) 80{ 81 int mdev_idx = mlx5_sd_ch_ix_get_dev_ix(primary, ch_ix); 82 83 return mlx5_sd_primary_get_peer(primary, mdev_idx); 84} 85 86static bool ft_create_alias_supported(struct mlx5_core_dev *dev) 87{ 88 u64 obj_allowed = MLX5_CAP_GEN_2_64(dev, allowed_object_for_other_vhca_access); 89 u32 obj_supp = MLX5_CAP_GEN_2(dev, cross_vhca_object_to_object_supported); 90 91 if (!(obj_supp & 92 MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE)) 93 return false; 94 95 if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE)) 96 return false; 97 98 return true; 99} 100 101static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses) 102{ 103 /* Feature is currently implemented for PFs only */ 104 if (!mlx5_core_is_pf(dev)) 105 return false; 106 107 /* Honor the SW implementation limit */ 108 if (host_buses > MLX5_SD_MAX_GROUP_SZ) 109 return false; 110 111 /* Disconnect secondaries from the network */ 112 if (!MLX5_CAP_GEN(dev, eswitch_manager)) 113 return false; 114 if (!MLX5_CAP_GEN(dev, silent_mode)) 115 return false; 116 117 /* RX steering from primary to secondaries */ 118 if (!MLX5_CAP_GEN(dev, cross_vhca_rqt)) 119 return false; 120 if (host_buses > MLX5_CAP_GEN_2(dev, max_rqt_vhca_id)) 121 return false; 122 123 /* TX steering from secondaries to primary */ 124 if (!ft_create_alias_supported(dev)) 125 return false; 126 if (!MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default)) 127 return false; 128 129 return true; 130} 131 132static int mlx5_query_sd(struct mlx5_core_dev *dev, bool *sdm, 133 u8 *host_buses, u8 *sd_group) 134{ 135 u32 out[MLX5_ST_SZ_DW(mpir_reg)]; 136 int err; 137 138 err = mlx5_query_mpir_reg(dev, out); 139 if (err) 140 return err; 141 142 err = mlx5_query_nic_vport_sd_group(dev, sd_group); 143 if (err) 144 return err; 145 146 *sdm = MLX5_GET(mpir_reg, out, sdm); 147 *host_buses = MLX5_GET(mpir_reg, out, host_buses); 148 149 return 0; 150} 151 152static u32 mlx5_sd_group_id(struct mlx5_core_dev *dev, u8 sd_group) 153{ 154 return (u32)((MLX5_CAP_GEN(dev, native_port_num) << 8) | sd_group); 155} 156 157static int sd_init(struct mlx5_core_dev *dev) 158{ 159 u8 host_buses, sd_group; 160 struct mlx5_sd *sd; 161 u32 group_id; 162 bool sdm; 163 int err; 164 165 if (!MLX5_CAP_MCAM_REG(dev, mpir)) 166 return 0; 167 168 err = mlx5_query_sd(dev, &sdm, &host_buses, &sd_group); 169 if (err) 170 return err; 171 172 if (!sdm) 173 return 0; 174 175 if (!sd_group) 176 return 0; 177 178 group_id = mlx5_sd_group_id(dev, sd_group); 179 180 if (!mlx5_sd_is_supported(dev, host_buses)) { 181 sd_warn(dev, "can't support requested netdev combining for group id 0x%x), skipping\n", 182 group_id); 183 return 0; 184 } 185 186 sd = kzalloc(sizeof(*sd), GFP_KERNEL); 187 if (!sd) 188 return -ENOMEM; 189 190 sd->host_buses = host_buses; 191 sd->group_id = group_id; 192 193 mlx5_set_sd(dev, sd); 194 195 return 0; 196} 197 198static void sd_cleanup(struct mlx5_core_dev *dev) 199{ 200 struct mlx5_sd *sd = mlx5_get_sd(dev); 201 202 mlx5_set_sd(dev, NULL); 203 kfree(sd); 204} 205 206static int sd_register(struct mlx5_core_dev *dev) 207{ 208 struct mlx5_devcom_comp_dev *devcom, *pos; 209 struct mlx5_core_dev *peer, *primary; 210 struct mlx5_sd *sd, *primary_sd; 211 int err, i; 212 213 sd = mlx5_get_sd(dev); 214 devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP, 215 sd->group_id, NULL, dev); 216 if (IS_ERR(devcom)) 217 return PTR_ERR(devcom); 218 219 sd->devcom = devcom; 220 221 if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses) 222 return 0; 223 224 mlx5_devcom_comp_lock(devcom); 225 mlx5_devcom_comp_set_ready(devcom, true); 226 mlx5_devcom_comp_unlock(devcom); 227 228 if (!mlx5_devcom_for_each_peer_begin(devcom)) { 229 err = -ENODEV; 230 goto err_devcom_unreg; 231 } 232 233 primary = dev; 234 mlx5_devcom_for_each_peer_entry(devcom, peer, pos) 235 if (peer->pdev->bus->number < primary->pdev->bus->number) 236 primary = peer; 237 238 primary_sd = mlx5_get_sd(primary); 239 primary_sd->primary = true; 240 i = 0; 241 /* loop the secondaries */ 242 mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) { 243 struct mlx5_sd *peer_sd = mlx5_get_sd(peer); 244 245 primary_sd->secondaries[i++] = peer; 246 peer_sd->primary = false; 247 peer_sd->primary_dev = primary; 248 } 249 250 mlx5_devcom_for_each_peer_end(devcom); 251 return 0; 252 253err_devcom_unreg: 254 mlx5_devcom_comp_lock(sd->devcom); 255 mlx5_devcom_comp_set_ready(sd->devcom, false); 256 mlx5_devcom_comp_unlock(sd->devcom); 257 mlx5_devcom_unregister_component(sd->devcom); 258 return err; 259} 260 261static void sd_unregister(struct mlx5_core_dev *dev) 262{ 263 struct mlx5_sd *sd = mlx5_get_sd(dev); 264 265 mlx5_devcom_comp_lock(sd->devcom); 266 mlx5_devcom_comp_set_ready(sd->devcom, false); 267 mlx5_devcom_comp_unlock(sd->devcom); 268 mlx5_devcom_unregister_component(sd->devcom); 269} 270 271static int sd_cmd_set_primary(struct mlx5_core_dev *primary, u8 *alias_key) 272{ 273 struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {}; 274 struct mlx5_sd *sd = mlx5_get_sd(primary); 275 struct mlx5_flow_table_attr ft_attr = {}; 276 struct mlx5_flow_namespace *nic_ns; 277 struct mlx5_flow_table *ft; 278 int err; 279 280 nic_ns = mlx5_get_flow_namespace(primary, MLX5_FLOW_NAMESPACE_EGRESS); 281 if (!nic_ns) 282 return -EOPNOTSUPP; 283 284 ft = mlx5_create_flow_table(nic_ns, &ft_attr); 285 if (IS_ERR(ft)) { 286 err = PTR_ERR(ft); 287 return err; 288 } 289 sd->tx_ft = ft; 290 memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN); 291 allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; 292 allow_attr.obj_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; 293 294 err = mlx5_cmd_allow_other_vhca_access(primary, &allow_attr); 295 if (err) { 296 mlx5_core_err(primary, "Failed to allow other vhca access err=%d\n", 297 err); 298 mlx5_destroy_flow_table(ft); 299 return err; 300 } 301 302 return 0; 303} 304 305static void sd_cmd_unset_primary(struct mlx5_core_dev *primary) 306{ 307 struct mlx5_sd *sd = mlx5_get_sd(primary); 308 309 mlx5_destroy_flow_table(sd->tx_ft); 310} 311 312static int sd_secondary_create_alias_ft(struct mlx5_core_dev *secondary, 313 struct mlx5_core_dev *primary, 314 struct mlx5_flow_table *ft, 315 u32 *obj_id, u8 *alias_key) 316{ 317 u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; 318 u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(primary, vhca_id); 319 struct mlx5_cmd_alias_obj_create_attr alias_attr = {}; 320 int ret; 321 322 memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN); 323 alias_attr.obj_id = aliased_object_id; 324 alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; 325 alias_attr.vhca_id = vhca_id_to_be_accessed; 326 ret = mlx5_cmd_alias_obj_create(secondary, &alias_attr, obj_id); 327 if (ret) { 328 mlx5_core_err(secondary, "Failed to create alias object err=%d\n", 329 ret); 330 return ret; 331 } 332 333 return 0; 334} 335 336static void sd_secondary_destroy_alias_ft(struct mlx5_core_dev *secondary) 337{ 338 struct mlx5_sd *sd = mlx5_get_sd(secondary); 339 340 mlx5_cmd_alias_obj_destroy(secondary, sd->alias_obj_id, 341 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); 342} 343 344static int sd_cmd_set_secondary(struct mlx5_core_dev *secondary, 345 struct mlx5_core_dev *primary, 346 u8 *alias_key) 347{ 348 struct mlx5_sd *primary_sd = mlx5_get_sd(primary); 349 struct mlx5_sd *sd = mlx5_get_sd(secondary); 350 int err; 351 352 err = mlx5_fs_cmd_set_l2table_entry_silent(secondary, 1); 353 if (err) 354 return err; 355 356 err = sd_secondary_create_alias_ft(secondary, primary, primary_sd->tx_ft, 357 &sd->alias_obj_id, alias_key); 358 if (err) 359 goto err_unset_silent; 360 361 err = mlx5_fs_cmd_set_tx_flow_table_root(secondary, sd->alias_obj_id, false); 362 if (err) 363 goto err_destroy_alias_ft; 364 365 return 0; 366 367err_destroy_alias_ft: 368 sd_secondary_destroy_alias_ft(secondary); 369err_unset_silent: 370 mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0); 371 return err; 372} 373 374static void sd_cmd_unset_secondary(struct mlx5_core_dev *secondary) 375{ 376 mlx5_fs_cmd_set_tx_flow_table_root(secondary, 0, true); 377 sd_secondary_destroy_alias_ft(secondary); 378 mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0); 379} 380 381static void sd_print_group(struct mlx5_core_dev *primary) 382{ 383 struct mlx5_sd *sd = mlx5_get_sd(primary); 384 struct mlx5_core_dev *pos; 385 int i; 386 387 sd_info(primary, "group id %#x, primary %s, vhca %#x\n", 388 sd->group_id, pci_name(primary->pdev), 389 MLX5_CAP_GEN(primary, vhca_id)); 390 mlx5_sd_for_each_secondary(i, primary, pos) 391 sd_info(primary, "group id %#x, secondary_%d %s, vhca %#x\n", 392 sd->group_id, i - 1, pci_name(pos->pdev), 393 MLX5_CAP_GEN(pos, vhca_id)); 394} 395 396static ssize_t dev_read(struct file *filp, char __user *buf, size_t count, 397 loff_t *pos) 398{ 399 struct mlx5_core_dev *dev; 400 char tbuf[32]; 401 int ret; 402 403 dev = filp->private_data; 404 ret = snprintf(tbuf, sizeof(tbuf), "%s vhca %#x\n", pci_name(dev->pdev), 405 MLX5_CAP_GEN(dev, vhca_id)); 406 407 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 408} 409 410static const struct file_operations dev_fops = { 411 .owner = THIS_MODULE, 412 .open = simple_open, 413 .read = dev_read, 414}; 415 416int mlx5_sd_init(struct mlx5_core_dev *dev) 417{ 418 struct mlx5_core_dev *primary, *pos, *to; 419 struct mlx5_sd *sd = mlx5_get_sd(dev); 420 u8 alias_key[ACCESS_KEY_LEN]; 421 int err, i; 422 423 err = sd_init(dev); 424 if (err) 425 return err; 426 427 sd = mlx5_get_sd(dev); 428 if (!sd) 429 return 0; 430 431 err = sd_register(dev); 432 if (err) 433 goto err_sd_cleanup; 434 435 if (!mlx5_devcom_comp_is_ready(sd->devcom)) 436 return 0; 437 438 primary = mlx5_sd_get_primary(dev); 439 440 for (i = 0; i < ACCESS_KEY_LEN; i++) 441 alias_key[i] = get_random_u8(); 442 443 err = sd_cmd_set_primary(primary, alias_key); 444 if (err) 445 goto err_sd_unregister; 446 447 sd->dfs = debugfs_create_dir("multi-pf", mlx5_debugfs_get_dev_root(primary)); 448 debugfs_create_x32("group_id", 0400, sd->dfs, &sd->group_id); 449 debugfs_create_file("primary", 0400, sd->dfs, primary, &dev_fops); 450 451 mlx5_sd_for_each_secondary(i, primary, pos) { 452 char name[32]; 453 454 err = sd_cmd_set_secondary(pos, primary, alias_key); 455 if (err) 456 goto err_unset_secondaries; 457 458 snprintf(name, sizeof(name), "secondary_%d", i - 1); 459 debugfs_create_file(name, 0400, sd->dfs, pos, &dev_fops); 460 461 } 462 463 sd_info(primary, "group id %#x, size %d, combined\n", 464 sd->group_id, mlx5_devcom_comp_get_size(sd->devcom)); 465 sd_print_group(primary); 466 467 return 0; 468 469err_unset_secondaries: 470 to = pos; 471 mlx5_sd_for_each_secondary_to(i, primary, to, pos) 472 sd_cmd_unset_secondary(pos); 473 sd_cmd_unset_primary(primary); 474 debugfs_remove_recursive(sd->dfs); 475err_sd_unregister: 476 sd_unregister(dev); 477err_sd_cleanup: 478 sd_cleanup(dev); 479 return err; 480} 481 482void mlx5_sd_cleanup(struct mlx5_core_dev *dev) 483{ 484 struct mlx5_sd *sd = mlx5_get_sd(dev); 485 struct mlx5_core_dev *primary, *pos; 486 int i; 487 488 if (!sd) 489 return; 490 491 if (!mlx5_devcom_comp_is_ready(sd->devcom)) 492 goto out; 493 494 primary = mlx5_sd_get_primary(dev); 495 mlx5_sd_for_each_secondary(i, primary, pos) 496 sd_cmd_unset_secondary(pos); 497 sd_cmd_unset_primary(primary); 498 debugfs_remove_recursive(sd->dfs); 499 500 sd_info(primary, "group id %#x, uncombined\n", sd->group_id); 501out: 502 sd_unregister(dev); 503 sd_cleanup(dev); 504} 505 506struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev, 507 struct auxiliary_device *adev, 508 int idx) 509{ 510 struct mlx5_sd *sd = mlx5_get_sd(dev); 511 struct mlx5_core_dev *primary; 512 513 if (!sd) 514 return adev; 515 516 if (!mlx5_devcom_comp_is_ready(sd->devcom)) 517 return NULL; 518 519 primary = mlx5_sd_get_primary(dev); 520 if (dev == primary) 521 return adev; 522 523 return &primary->priv.adev[idx]->adev; 524} 525