mlx5_main.c revision 347802
1/*- 2 * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c 347802 2019-05-16 17:15:00Z hselasky $ 26 */ 27 28#define LINUXKPI_PARAM_PREFIX mlx5_ 29 30#include <linux/kmod.h> 31#include <linux/module.h> 32#include <linux/errno.h> 33#include <linux/pci.h> 34#include <linux/dma-mapping.h> 35#include <linux/slab.h> 36#include <linux/io-mapping.h> 37#include <linux/interrupt.h> 38#include <linux/hardirq.h> 39#include <dev/mlx5/driver.h> 40#include <dev/mlx5/cq.h> 41#include <dev/mlx5/qp.h> 42#include <dev/mlx5/srq.h> 43#include <linux/delay.h> 44#include <dev/mlx5/mlx5_ifc.h> 45#include <dev/mlx5/mlx5_fpga/core.h> 46#include <dev/mlx5/mlx5_lib/mlx5.h> 47#include "mlx5_core.h" 48#include "fs_core.h" 49 50static const char mlx5_version[] = "Mellanox Core driver " 51 DRIVER_VERSION " (" DRIVER_RELDATE ")"; 52MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 53MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 54MODULE_LICENSE("Dual BSD/GPL"); 55#if (__FreeBSD_version >= 1100000) 56MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1); 57#endif 58MODULE_VERSION(mlx5, 1); 59 60int mlx5_core_debug_mask; 61module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644); 62MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 63 64#define MLX5_DEFAULT_PROF 2 65static int prof_sel = MLX5_DEFAULT_PROF; 66module_param_named(prof_sel, prof_sel, int, 0444); 67MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 68 69SYSCTL_NODE(_hw, OID_AUTO, mlx5, CTLFLAG_RW, 0, "mlx5 HW controls"); 70 71#define NUMA_NO_NODE -1 72 73static LIST_HEAD(intf_list); 74static LIST_HEAD(dev_list); 75static DEFINE_MUTEX(intf_mutex); 76 77struct mlx5_device_context { 78 struct list_head list; 79 struct mlx5_interface *intf; 80 void *context; 81}; 82 83enum { 84 MLX5_ATOMIC_REQ_MODE_BE = 0x0, 85 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, 86}; 87 88static struct mlx5_profile profiles[] = { 89 [0] = { 90 .mask = 0, 91 }, 92 [1] = { 93 .mask = MLX5_PROF_MASK_QP_SIZE, 94 .log_max_qp = 12, 95 }, 96 [2] = { 97 .mask = MLX5_PROF_MASK_QP_SIZE | 98 MLX5_PROF_MASK_MR_CACHE, 99 .log_max_qp = 17, 100 .mr_cache[0] = { 101 .size = 500, 102 .limit = 250 103 }, 104 .mr_cache[1] = { 105 .size = 500, 106 .limit = 250 107 }, 108 .mr_cache[2] = { 109 .size = 500, 110 .limit = 250 111 }, 112 .mr_cache[3] = { 113 .size = 500, 114 .limit = 250 115 }, 116 .mr_cache[4] = { 117 .size = 500, 118 .limit = 250 119 }, 120 .mr_cache[5] = { 121 .size = 500, 122 .limit = 250 123 }, 124 .mr_cache[6] = { 125 .size = 500, 126 .limit = 250 127 }, 128 .mr_cache[7] = { 129 .size = 500, 130 .limit = 250 131 }, 132 .mr_cache[8] = { 133 .size = 500, 134 .limit = 250 135 }, 136 .mr_cache[9] = { 137 .size = 500, 138 .limit = 250 139 }, 140 .mr_cache[10] = { 141 .size = 500, 142 .limit = 250 143 }, 144 .mr_cache[11] = { 145 .size = 500, 146 .limit = 250 147 }, 148 .mr_cache[12] = { 149 .size = 64, 150 .limit = 32 151 }, 152 .mr_cache[13] = { 153 .size = 32, 154 .limit = 16 155 }, 156 .mr_cache[14] = { 157 .size = 16, 158 .limit = 8 159 }, 160 }, 161 [3] = { 162 .mask = MLX5_PROF_MASK_QP_SIZE, 163 .log_max_qp = 17, 164 }, 165}; 166 167static int set_dma_caps(struct pci_dev *pdev) 168{ 169 int err; 170 171 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 172 if (err) { 173 device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit PCI DMA mask\n"); 174 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 175 if (err) { 176 device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set PCI DMA mask, aborting\n"); 177 return err; 178 } 179 } 180 181 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 182 if (err) { 183 device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 184 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 185 if (err) { 186 device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set consistent PCI DMA mask, aborting\n"); 187 return err; 188 } 189 } 190 191 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 192 return err; 193} 194 195static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) 196{ 197 struct pci_dev *pdev = dev->pdev; 198 int err = 0; 199 200 mutex_lock(&dev->pci_status_mutex); 201 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { 202 err = pci_enable_device(pdev); 203 if (!err) 204 dev->pci_status = MLX5_PCI_STATUS_ENABLED; 205 } 206 mutex_unlock(&dev->pci_status_mutex); 207 208 return err; 209} 210 211static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) 212{ 213 struct pci_dev *pdev = dev->pdev; 214 215 mutex_lock(&dev->pci_status_mutex); 216 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { 217 pci_disable_device(pdev); 218 dev->pci_status = MLX5_PCI_STATUS_DISABLED; 219 } 220 mutex_unlock(&dev->pci_status_mutex); 221} 222 223static int request_bar(struct pci_dev *pdev) 224{ 225 int err = 0; 226 227 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 228 device_printf((&pdev->dev)->bsddev, "ERR: ""Missing registers BAR, aborting\n"); 229 return -ENODEV; 230 } 231 232 err = pci_request_regions(pdev, DRIVER_NAME); 233 if (err) 234 device_printf((&pdev->dev)->bsddev, "ERR: ""Couldn't get PCI resources, aborting\n"); 235 236 return err; 237} 238 239static void release_bar(struct pci_dev *pdev) 240{ 241 pci_release_regions(pdev); 242} 243 244static int mlx5_enable_msix(struct mlx5_core_dev *dev) 245{ 246 struct mlx5_priv *priv = &dev->priv; 247 struct mlx5_eq_table *table = &priv->eq_table; 248 int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 249 int limit = dev->msix_eqvec; 250 int nvec = MLX5_EQ_VEC_COMP_BASE; 251 int i; 252 253 if (limit > 0) 254 nvec += limit; 255 else 256 nvec += MLX5_CAP_GEN(dev, num_ports) * num_online_cpus(); 257 258 nvec = min_t(int, nvec, num_eqs); 259 if (nvec <= MLX5_EQ_VEC_COMP_BASE) 260 return -ENOMEM; 261 262 priv->msix_arr = kzalloc(nvec * sizeof(*priv->msix_arr), GFP_KERNEL); 263 264 priv->irq_info = kzalloc(nvec * sizeof(*priv->irq_info), GFP_KERNEL); 265 266 for (i = 0; i < nvec; i++) 267 priv->msix_arr[i].entry = i; 268 269 nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 270 MLX5_EQ_VEC_COMP_BASE + 1, nvec); 271 if (nvec < 0) 272 return nvec; 273 274 table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 275 276 return 0; 277 278} 279 280static void mlx5_disable_msix(struct mlx5_core_dev *dev) 281{ 282 struct mlx5_priv *priv = &dev->priv; 283 284 pci_disable_msix(dev->pdev); 285 kfree(priv->irq_info); 286 kfree(priv->msix_arr); 287} 288 289struct mlx5_reg_host_endianess { 290 u8 he; 291 u8 rsvd[15]; 292}; 293 294 295#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 296 297enum { 298 MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 299 MLX5_DEV_CAP_FLAG_DCT | 300 MLX5_DEV_CAP_FLAG_DRAIN_SIGERR, 301}; 302 303static u16 to_fw_pkey_sz(u32 size) 304{ 305 switch (size) { 306 case 128: 307 return 0; 308 case 256: 309 return 1; 310 case 512: 311 return 2; 312 case 1024: 313 return 3; 314 case 2048: 315 return 4; 316 case 4096: 317 return 5; 318 default: 319 printf("mlx5_core: WARN: ""invalid pkey table size %d\n", size); 320 return 0; 321 } 322} 323 324static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, 325 enum mlx5_cap_type cap_type, 326 enum mlx5_cap_mode cap_mode) 327{ 328 u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 329 int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 330 void *out, *hca_caps; 331 u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 332 int err; 333 334 memset(in, 0, sizeof(in)); 335 out = kzalloc(out_sz, GFP_KERNEL); 336 337 MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 338 MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 339 err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 340 if (err) { 341 mlx5_core_warn(dev, 342 "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 343 cap_type, cap_mode, err); 344 goto query_ex; 345 } 346 347 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 348 349 switch (cap_mode) { 350 case HCA_CAP_OPMOD_GET_MAX: 351 memcpy(dev->hca_caps_max[cap_type], hca_caps, 352 MLX5_UN_SZ_BYTES(hca_cap_union)); 353 break; 354 case HCA_CAP_OPMOD_GET_CUR: 355 memcpy(dev->hca_caps_cur[cap_type], hca_caps, 356 MLX5_UN_SZ_BYTES(hca_cap_union)); 357 break; 358 default: 359 mlx5_core_warn(dev, 360 "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 361 cap_type, cap_mode); 362 err = -EINVAL; 363 break; 364 } 365query_ex: 366 kfree(out); 367 return err; 368} 369 370int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type) 371{ 372 int ret; 373 374 ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR); 375 if (ret) 376 return ret; 377 378 return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX); 379} 380 381static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz) 382{ 383 u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)] = {0}; 384 385 MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 386 387 return mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 388} 389 390static int handle_hca_cap(struct mlx5_core_dev *dev) 391{ 392 void *set_ctx = NULL; 393 struct mlx5_profile *prof = dev->profile; 394 int err = -ENOMEM; 395 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 396 void *set_hca_cap; 397 398 set_ctx = kzalloc(set_sz, GFP_KERNEL); 399 400 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); 401 if (err) 402 goto query_ex; 403 404 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 405 capability); 406 memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL], 407 MLX5_ST_SZ_BYTES(cmd_hca_cap)); 408 409 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 410 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 411 128); 412 /* we limit the size of the pkey table to 128 entries for now */ 413 MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 414 to_fw_pkey_sz(128)); 415 416 if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 417 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 418 prof->log_max_qp); 419 420 /* disable cmdif checksum */ 421 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 422 423 /* enable drain sigerr */ 424 MLX5_SET(cmd_hca_cap, set_hca_cap, drain_sigerr, 1); 425 426 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 427 428 err = set_caps(dev, set_ctx, set_sz); 429 430query_ex: 431 kfree(set_ctx); 432 return err; 433} 434 435static int handle_hca_cap_atomic(struct mlx5_core_dev *dev) 436{ 437 void *set_ctx; 438 void *set_hca_cap; 439 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 440 int req_endianness; 441 int err; 442 443 if (MLX5_CAP_GEN(dev, atomic)) { 444 err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC); 445 if (err) 446 return err; 447 } else { 448 return 0; 449 } 450 451 req_endianness = 452 MLX5_CAP_ATOMIC(dev, 453 supported_atomic_req_8B_endianess_mode_1); 454 455 if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) 456 return 0; 457 458 set_ctx = kzalloc(set_sz, GFP_KERNEL); 459 if (!set_ctx) 460 return -ENOMEM; 461 462 MLX5_SET(set_hca_cap_in, set_ctx, op_mod, 463 MLX5_SET_HCA_CAP_OP_MOD_ATOMIC << 1); 464 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 465 466 /* Set requestor to host endianness */ 467 MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode, 468 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); 469 470 err = set_caps(dev, set_ctx, set_sz); 471 472 kfree(set_ctx); 473 return err; 474} 475 476static int set_hca_ctrl(struct mlx5_core_dev *dev) 477{ 478 struct mlx5_reg_host_endianess he_in; 479 struct mlx5_reg_host_endianess he_out; 480 int err; 481 482 if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH && 483 !MLX5_CAP_GEN(dev, roce)) 484 return 0; 485 486 memset(&he_in, 0, sizeof(he_in)); 487 he_in.he = MLX5_SET_HOST_ENDIANNESS; 488 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 489 &he_out, sizeof(he_out), 490 MLX5_REG_HOST_ENDIANNESS, 0, 1); 491 return err; 492} 493 494static int mlx5_core_enable_hca(struct mlx5_core_dev *dev) 495{ 496 u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0}; 497 u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {0}; 498 499 MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); 500 return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 501} 502 503static int mlx5_core_disable_hca(struct mlx5_core_dev *dev) 504{ 505 u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {0}; 506 u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {0}; 507 508 MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); 509 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 510} 511 512static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 513{ 514 u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {0}; 515 u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {0}; 516 u32 sup_issi; 517 int err; 518 519 MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 520 521 err = mlx5_cmd_exec(dev, query_in, sizeof(query_in), query_out, sizeof(query_out)); 522 if (err) { 523 u32 syndrome; 524 u8 status; 525 526 mlx5_cmd_mbox_status(query_out, &status, &syndrome); 527 if (status == MLX5_CMD_STAT_BAD_OP_ERR) { 528 pr_debug("Only ISSI 0 is supported\n"); 529 return 0; 530 } 531 532 printf("mlx5_core: ERR: ""failed to query ISSI\n"); 533 return err; 534 } 535 536 sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 537 538 if (sup_issi & (1 << 1)) { 539 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {0}; 540 u32 set_out[MLX5_ST_SZ_DW(set_issi_out)] = {0}; 541 542 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 543 MLX5_SET(set_issi_in, set_in, current_issi, 1); 544 545 err = mlx5_cmd_exec(dev, set_in, sizeof(set_in), set_out, sizeof(set_out)); 546 if (err) { 547 printf("mlx5_core: ERR: ""failed to set ISSI=1 err(%d)\n", err); 548 return err; 549 } 550 551 dev->issi = 1; 552 553 return 0; 554 } else if (sup_issi & (1 << 0)) { 555 return 0; 556 } 557 558 return -ENOTSUPP; 559} 560 561 562int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn) 563{ 564 struct mlx5_eq_table *table = &dev->priv.eq_table; 565 struct mlx5_eq *eq; 566 int err = -ENOENT; 567 568 spin_lock(&table->lock); 569 list_for_each_entry(eq, &table->comp_eqs_list, list) { 570 if (eq->index == vector) { 571 *eqn = eq->eqn; 572 *irqn = eq->irqn; 573 err = 0; 574 break; 575 } 576 } 577 spin_unlock(&table->lock); 578 579 return err; 580} 581EXPORT_SYMBOL(mlx5_vector2eqn); 582 583int mlx5_rename_eq(struct mlx5_core_dev *dev, int eq_ix, char *name) 584{ 585 struct mlx5_priv *priv = &dev->priv; 586 struct mlx5_eq_table *table = &priv->eq_table; 587 struct mlx5_eq *eq; 588 int err = -ENOENT; 589 590 spin_lock(&table->lock); 591 list_for_each_entry(eq, &table->comp_eqs_list, list) { 592 if (eq->index == eq_ix) { 593 int irq_ix = eq_ix + MLX5_EQ_VEC_COMP_BASE; 594 595 snprintf(priv->irq_info[irq_ix].name, MLX5_MAX_IRQ_NAME, 596 "%s-%d", name, eq_ix); 597 598 err = 0; 599 break; 600 } 601 } 602 spin_unlock(&table->lock); 603 604 return err; 605} 606 607static void free_comp_eqs(struct mlx5_core_dev *dev) 608{ 609 struct mlx5_eq_table *table = &dev->priv.eq_table; 610 struct mlx5_eq *eq, *n; 611 612 spin_lock(&table->lock); 613 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 614 list_del(&eq->list); 615 spin_unlock(&table->lock); 616 if (mlx5_destroy_unmap_eq(dev, eq)) 617 mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 618 eq->eqn); 619 kfree(eq); 620 spin_lock(&table->lock); 621 } 622 spin_unlock(&table->lock); 623} 624 625static int alloc_comp_eqs(struct mlx5_core_dev *dev) 626{ 627 struct mlx5_eq_table *table = &dev->priv.eq_table; 628 char name[MLX5_MAX_IRQ_NAME]; 629 struct mlx5_eq *eq; 630 int ncomp_vec; 631 int nent; 632 int err; 633 int i; 634 635 INIT_LIST_HEAD(&table->comp_eqs_list); 636 ncomp_vec = table->num_comp_vectors; 637 nent = MLX5_COMP_EQ_SIZE; 638 for (i = 0; i < ncomp_vec; i++) { 639 eq = kzalloc(sizeof(*eq), GFP_KERNEL); 640 641 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 642 err = mlx5_create_map_eq(dev, eq, 643 i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 644 name, &dev->priv.uuari.uars[0]); 645 if (err) { 646 kfree(eq); 647 goto clean; 648 } 649 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 650 eq->index = i; 651 spin_lock(&table->lock); 652 list_add_tail(&eq->list, &table->comp_eqs_list); 653 spin_unlock(&table->lock); 654 } 655 656 return 0; 657 658clean: 659 free_comp_eqs(dev); 660 return err; 661} 662 663static int map_bf_area(struct mlx5_core_dev *dev) 664{ 665 resource_size_t bf_start = pci_resource_start(dev->pdev, 0); 666 resource_size_t bf_len = pci_resource_len(dev->pdev, 0); 667 668 dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len); 669 670 return dev->priv.bf_mapping ? 0 : -ENOMEM; 671} 672 673static void unmap_bf_area(struct mlx5_core_dev *dev) 674{ 675 if (dev->priv.bf_mapping) 676 io_mapping_free(dev->priv.bf_mapping); 677} 678 679static inline int fw_initializing(struct mlx5_core_dev *dev) 680{ 681 return ioread32be(&dev->iseg->initializing) >> 31; 682} 683 684static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) 685{ 686 u64 end = jiffies + msecs_to_jiffies(max_wait_mili); 687 int err = 0; 688 689 while (fw_initializing(dev)) { 690 if (time_after(jiffies, end)) { 691 err = -EBUSY; 692 break; 693 } 694 msleep(FW_INIT_WAIT_MS); 695 } 696 697 return err; 698} 699 700static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 701{ 702 struct mlx5_device_context *dev_ctx; 703 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 704 705 dev_ctx = kzalloc(sizeof(*dev_ctx), GFP_KERNEL); 706 if (!dev_ctx) 707 return; 708 709 dev_ctx->intf = intf; 710 CURVNET_SET_QUIET(vnet0); 711 dev_ctx->context = intf->add(dev); 712 CURVNET_RESTORE(); 713 714 if (dev_ctx->context) { 715 spin_lock_irq(&priv->ctx_lock); 716 list_add_tail(&dev_ctx->list, &priv->ctx_list); 717 spin_unlock_irq(&priv->ctx_lock); 718 } else { 719 kfree(dev_ctx); 720 } 721} 722 723static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 724{ 725 struct mlx5_device_context *dev_ctx; 726 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 727 728 list_for_each_entry(dev_ctx, &priv->ctx_list, list) 729 if (dev_ctx->intf == intf) { 730 spin_lock_irq(&priv->ctx_lock); 731 list_del(&dev_ctx->list); 732 spin_unlock_irq(&priv->ctx_lock); 733 734 intf->remove(dev, dev_ctx->context); 735 kfree(dev_ctx); 736 return; 737 } 738} 739 740int 741mlx5_register_device(struct mlx5_core_dev *dev) 742{ 743 struct mlx5_priv *priv = &dev->priv; 744 struct mlx5_interface *intf; 745 746 mutex_lock(&intf_mutex); 747 list_add_tail(&priv->dev_list, &dev_list); 748 list_for_each_entry(intf, &intf_list, list) 749 mlx5_add_device(intf, priv); 750 mutex_unlock(&intf_mutex); 751 752 return 0; 753} 754 755void 756mlx5_unregister_device(struct mlx5_core_dev *dev) 757{ 758 struct mlx5_priv *priv = &dev->priv; 759 struct mlx5_interface *intf; 760 761 mutex_lock(&intf_mutex); 762 list_for_each_entry(intf, &intf_list, list) 763 mlx5_remove_device(intf, priv); 764 list_del(&priv->dev_list); 765 mutex_unlock(&intf_mutex); 766} 767 768int mlx5_register_interface(struct mlx5_interface *intf) 769{ 770 struct mlx5_priv *priv; 771 772 if (!intf->add || !intf->remove) 773 return -EINVAL; 774 775 mutex_lock(&intf_mutex); 776 list_add_tail(&intf->list, &intf_list); 777 list_for_each_entry(priv, &dev_list, dev_list) 778 mlx5_add_device(intf, priv); 779 mutex_unlock(&intf_mutex); 780 781 return 0; 782} 783EXPORT_SYMBOL(mlx5_register_interface); 784 785void mlx5_unregister_interface(struct mlx5_interface *intf) 786{ 787 struct mlx5_priv *priv; 788 789 mutex_lock(&intf_mutex); 790 list_for_each_entry(priv, &dev_list, dev_list) 791 mlx5_remove_device(intf, priv); 792 list_del(&intf->list); 793 mutex_unlock(&intf_mutex); 794} 795EXPORT_SYMBOL(mlx5_unregister_interface); 796 797void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol) 798{ 799 struct mlx5_priv *priv = &mdev->priv; 800 struct mlx5_device_context *dev_ctx; 801 unsigned long flags; 802 void *result = NULL; 803 804 spin_lock_irqsave(&priv->ctx_lock, flags); 805 806 list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list) 807 if ((dev_ctx->intf->protocol == protocol) && 808 dev_ctx->intf->get_dev) { 809 result = dev_ctx->intf->get_dev(dev_ctx->context); 810 break; 811 } 812 813 spin_unlock_irqrestore(&priv->ctx_lock, flags); 814 815 return result; 816} 817EXPORT_SYMBOL(mlx5_get_protocol_dev); 818 819static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 820{ 821 struct pci_dev *pdev = dev->pdev; 822 int err = 0; 823 824 pci_set_drvdata(dev->pdev, dev); 825 strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 826 priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 827 828 mutex_init(&priv->pgdir_mutex); 829 INIT_LIST_HEAD(&priv->pgdir_list); 830 spin_lock_init(&priv->mkey_lock); 831 832 priv->numa_node = NUMA_NO_NODE; 833 834 err = mlx5_pci_enable_device(dev); 835 if (err) { 836 device_printf((&pdev->dev)->bsddev, "ERR: ""Cannot enable PCI device, aborting\n"); 837 goto err_dbg; 838 } 839 840 err = request_bar(pdev); 841 if (err) { 842 device_printf((&pdev->dev)->bsddev, "ERR: ""error requesting BARs, aborting\n"); 843 goto err_disable; 844 } 845 846 pci_set_master(pdev); 847 848 err = set_dma_caps(pdev); 849 if (err) { 850 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed setting DMA capabilities mask, aborting\n"); 851 goto err_clr_master; 852 } 853 854 dev->iseg_base = pci_resource_start(dev->pdev, 0); 855 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 856 if (!dev->iseg) { 857 err = -ENOMEM; 858 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed mapping initialization segment, aborting\n"); 859 goto err_clr_master; 860 } 861 862 return 0; 863 864err_clr_master: 865 pci_clear_master(dev->pdev); 866 release_bar(dev->pdev); 867err_disable: 868 mlx5_pci_disable_device(dev); 869err_dbg: 870 return err; 871} 872 873static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 874{ 875 iounmap(dev->iseg); 876 pci_clear_master(dev->pdev); 877 release_bar(dev->pdev); 878 mlx5_pci_disable_device(dev); 879} 880 881static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 882{ 883 struct pci_dev *pdev = dev->pdev; 884 int err; 885 886 err = mlx5_vsc_find_cap(dev); 887 if (err) 888 dev_err(&pdev->dev, "Unable to find vendor specific capabilities\n"); 889 890 err = mlx5_query_hca_caps(dev); 891 if (err) { 892 dev_err(&pdev->dev, "query hca failed\n"); 893 goto out; 894 } 895 896 err = mlx5_query_board_id(dev); 897 if (err) { 898 dev_err(&pdev->dev, "query board id failed\n"); 899 goto out; 900 } 901 902 err = mlx5_eq_init(dev); 903 if (err) { 904 dev_err(&pdev->dev, "failed to initialize eq\n"); 905 goto out; 906 } 907 908 MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); 909 910 err = mlx5_init_cq_table(dev); 911 if (err) { 912 dev_err(&pdev->dev, "failed to initialize cq table\n"); 913 goto err_eq_cleanup; 914 } 915 916 mlx5_init_qp_table(dev); 917 mlx5_init_srq_table(dev); 918 mlx5_init_mr_table(dev); 919 920 mlx5_init_reserved_gids(dev); 921 mlx5_fpga_init(dev); 922 923 return 0; 924 925err_eq_cleanup: 926 mlx5_eq_cleanup(dev); 927 928out: 929 return err; 930} 931 932static void mlx5_cleanup_once(struct mlx5_core_dev *dev) 933{ 934 mlx5_fpga_cleanup(dev); 935 mlx5_cleanup_reserved_gids(dev); 936 mlx5_cleanup_mr_table(dev); 937 mlx5_cleanup_srq_table(dev); 938 mlx5_cleanup_qp_table(dev); 939 mlx5_cleanup_cq_table(dev); 940 mlx5_eq_cleanup(dev); 941} 942 943static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, 944 bool boot) 945{ 946 struct pci_dev *pdev = dev->pdev; 947 int err; 948 949 mutex_lock(&dev->intf_state_mutex); 950 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { 951 dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n", 952 __func__); 953 goto out; 954 } 955 956 device_printf((&pdev->dev)->bsddev, "INFO: ""firmware version: %d.%d.%d\n", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); 957 958 /* 959 * On load removing any previous indication of internal error, 960 * device is up 961 */ 962 dev->state = MLX5_DEVICE_STATE_UP; 963 964 err = mlx5_cmd_init(dev); 965 if (err) { 966 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed initializing command interface, aborting\n"); 967 goto out_err; 968 } 969 970 err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); 971 if (err) { 972 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI); 973 goto err_cmd_cleanup; 974 } 975 976 err = mlx5_core_enable_hca(dev); 977 if (err) { 978 device_printf((&pdev->dev)->bsddev, "ERR: ""enable hca failed\n"); 979 goto err_cmd_cleanup; 980 } 981 982 err = mlx5_core_set_issi(dev); 983 if (err) { 984 device_printf((&pdev->dev)->bsddev, "ERR: ""failed to set issi\n"); 985 goto err_disable_hca; 986 } 987 988 err = mlx5_pagealloc_start(dev); 989 if (err) { 990 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_pagealloc_start failed\n"); 991 goto err_disable_hca; 992 } 993 994 err = mlx5_satisfy_startup_pages(dev, 1); 995 if (err) { 996 device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate boot pages\n"); 997 goto err_pagealloc_stop; 998 } 999 1000 err = set_hca_ctrl(dev); 1001 if (err) { 1002 device_printf((&pdev->dev)->bsddev, "ERR: ""set_hca_ctrl failed\n"); 1003 goto reclaim_boot_pages; 1004 } 1005 1006 err = handle_hca_cap(dev); 1007 if (err) { 1008 device_printf((&pdev->dev)->bsddev, "ERR: ""handle_hca_cap failed\n"); 1009 goto reclaim_boot_pages; 1010 } 1011 1012 err = handle_hca_cap_atomic(dev); 1013 if (err) { 1014 device_printf((&pdev->dev)->bsddev, "ERR: ""handle_hca_cap_atomic failed\n"); 1015 goto reclaim_boot_pages; 1016 } 1017 1018 err = mlx5_satisfy_startup_pages(dev, 0); 1019 if (err) { 1020 device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate init pages\n"); 1021 goto reclaim_boot_pages; 1022 } 1023 1024 err = mlx5_cmd_init_hca(dev); 1025 if (err) { 1026 device_printf((&pdev->dev)->bsddev, "ERR: ""init hca failed\n"); 1027 goto reclaim_boot_pages; 1028 } 1029 1030 mlx5_start_health_poll(dev); 1031 1032 if (boot && mlx5_init_once(dev, priv)) { 1033 dev_err(&pdev->dev, "sw objs init failed\n"); 1034 goto err_stop_poll; 1035 } 1036 1037 err = mlx5_enable_msix(dev); 1038 if (err) { 1039 device_printf((&pdev->dev)->bsddev, "ERR: ""enable msix failed\n"); 1040 goto err_cleanup_once; 1041 } 1042 1043 err = mlx5_alloc_uuars(dev, &priv->uuari); 1044 if (err) { 1045 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed allocating uar, aborting\n"); 1046 goto err_disable_msix; 1047 } 1048 1049 err = mlx5_start_eqs(dev); 1050 if (err) { 1051 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to start pages and async EQs\n"); 1052 goto err_free_uar; 1053 } 1054 1055 err = alloc_comp_eqs(dev); 1056 if (err) { 1057 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to alloc completion EQs\n"); 1058 goto err_stop_eqs; 1059 } 1060 1061 if (map_bf_area(dev)) 1062 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to map blue flame area\n"); 1063 1064 err = mlx5_init_fs(dev); 1065 if (err) { 1066 mlx5_core_err(dev, "flow steering init %d\n", err); 1067 goto err_free_comp_eqs; 1068 } 1069 1070 err = mlx5_fpga_device_start(dev); 1071 if (err) { 1072 dev_err(&pdev->dev, "fpga device start failed %d\n", err); 1073 goto err_fpga_start; 1074 } 1075 1076 err = mlx5_register_device(dev); 1077 if (err) { 1078 dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); 1079 goto err_fs; 1080 } 1081 1082 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1083 1084out: 1085 mutex_unlock(&dev->intf_state_mutex); 1086 return 0; 1087 1088err_fpga_start: 1089err_fs: 1090 mlx5_cleanup_fs(dev); 1091 1092err_free_comp_eqs: 1093 free_comp_eqs(dev); 1094 unmap_bf_area(dev); 1095 1096err_stop_eqs: 1097 mlx5_stop_eqs(dev); 1098 1099err_free_uar: 1100 mlx5_free_uuars(dev, &priv->uuari); 1101 1102err_disable_msix: 1103 mlx5_disable_msix(dev); 1104 1105err_cleanup_once: 1106 if (boot) 1107 mlx5_cleanup_once(dev); 1108 1109err_stop_poll: 1110 mlx5_stop_health_poll(dev, boot); 1111 if (mlx5_cmd_teardown_hca(dev)) { 1112 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n"); 1113 goto out_err; 1114 } 1115 1116reclaim_boot_pages: 1117 mlx5_reclaim_startup_pages(dev); 1118 1119err_pagealloc_stop: 1120 mlx5_pagealloc_stop(dev); 1121 1122err_disable_hca: 1123 mlx5_core_disable_hca(dev); 1124 1125err_cmd_cleanup: 1126 mlx5_cmd_cleanup(dev); 1127 1128out_err: 1129 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1130 mutex_unlock(&dev->intf_state_mutex); 1131 1132 return err; 1133} 1134 1135static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, 1136 bool cleanup) 1137{ 1138 int err = 0; 1139 1140 if (cleanup) 1141 mlx5_drain_health_recovery(dev); 1142 1143 mutex_lock(&dev->intf_state_mutex); 1144 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { 1145 dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", __func__); 1146 if (cleanup) 1147 mlx5_cleanup_once(dev); 1148 goto out; 1149 } 1150 1151 mlx5_unregister_device(dev); 1152 1153 mlx5_fpga_device_stop(dev); 1154 mlx5_cleanup_fs(dev); 1155 unmap_bf_area(dev); 1156 mlx5_wait_for_reclaim_vfs_pages(dev); 1157 free_comp_eqs(dev); 1158 mlx5_stop_eqs(dev); 1159 mlx5_free_uuars(dev, &priv->uuari); 1160 mlx5_disable_msix(dev); 1161 if (cleanup) 1162 mlx5_cleanup_once(dev); 1163 mlx5_stop_health_poll(dev, cleanup); 1164 err = mlx5_cmd_teardown_hca(dev); 1165 if (err) { 1166 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n"); 1167 goto out; 1168 } 1169 mlx5_pagealloc_stop(dev); 1170 mlx5_reclaim_startup_pages(dev); 1171 mlx5_core_disable_hca(dev); 1172 mlx5_cmd_cleanup(dev); 1173 1174out: 1175 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1176 mutex_unlock(&dev->intf_state_mutex); 1177 return err; 1178} 1179 1180void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 1181 unsigned long param) 1182{ 1183 struct mlx5_priv *priv = &dev->priv; 1184 struct mlx5_device_context *dev_ctx; 1185 unsigned long flags; 1186 1187 spin_lock_irqsave(&priv->ctx_lock, flags); 1188 1189 list_for_each_entry(dev_ctx, &priv->ctx_list, list) 1190 if (dev_ctx->intf->event) 1191 dev_ctx->intf->event(dev, dev_ctx->context, event, param); 1192 1193 spin_unlock_irqrestore(&priv->ctx_lock, flags); 1194} 1195 1196struct mlx5_core_event_handler { 1197 void (*event)(struct mlx5_core_dev *dev, 1198 enum mlx5_dev_event event, 1199 void *data); 1200}; 1201 1202static int init_one(struct pci_dev *pdev, 1203 const struct pci_device_id *id) 1204{ 1205 struct mlx5_core_dev *dev; 1206 struct mlx5_priv *priv; 1207 device_t bsddev = pdev->dev.bsddev; 1208 int err; 1209 1210 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1211 priv = &dev->priv; 1212 if (id) 1213 priv->pci_dev_data = id->driver_data; 1214 1215 if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profiles)) { 1216 device_printf(bsddev, "WARN: selected profile out of range, selecting default (%d)\n", MLX5_DEFAULT_PROF); 1217 prof_sel = MLX5_DEFAULT_PROF; 1218 } 1219 dev->profile = &profiles[prof_sel]; 1220 dev->pdev = pdev; 1221 dev->event = mlx5_core_event; 1222 1223 /* Set desc */ 1224 device_set_desc(bsddev, mlx5_version); 1225 1226 sysctl_ctx_init(&dev->sysctl_ctx); 1227 SYSCTL_ADD_INT(&dev->sysctl_ctx, 1228 SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)), 1229 OID_AUTO, "msix_eqvec", CTLFLAG_RDTUN, &dev->msix_eqvec, 0, 1230 "Maximum number of MSIX event queue vectors, if set"); 1231 1232 INIT_LIST_HEAD(&priv->ctx_list); 1233 spin_lock_init(&priv->ctx_lock); 1234 mutex_init(&dev->pci_status_mutex); 1235 mutex_init(&dev->intf_state_mutex); 1236 err = mlx5_pci_init(dev, priv); 1237 if (err) { 1238 device_printf(bsddev, "ERR: mlx5_pci_init failed %d\n", err); 1239 goto clean_dev; 1240 } 1241 1242 err = mlx5_health_init(dev); 1243 if (err) { 1244 device_printf(bsddev, "ERR: mlx5_health_init failed %d\n", err); 1245 goto close_pci; 1246 } 1247 1248 mlx5_pagealloc_init(dev); 1249 1250 err = mlx5_load_one(dev, priv, true); 1251 if (err) { 1252 device_printf(bsddev, "ERR: mlx5_load_one failed %d\n", err); 1253 goto clean_health; 1254 } 1255 1256 mlx5_fwdump_prep(dev); 1257 1258 pci_save_state(bsddev); 1259 return 0; 1260 1261clean_health: 1262 mlx5_pagealloc_cleanup(dev); 1263 mlx5_health_cleanup(dev); 1264close_pci: 1265 mlx5_pci_close(dev, priv); 1266clean_dev: 1267 sysctl_ctx_free(&dev->sysctl_ctx); 1268 kfree(dev); 1269 return err; 1270} 1271 1272static void remove_one(struct pci_dev *pdev) 1273{ 1274 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1275 struct mlx5_priv *priv = &dev->priv; 1276 1277 if (mlx5_unload_one(dev, priv, true)) { 1278 dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n"); 1279 mlx5_health_cleanup(dev); 1280 return; 1281 } 1282 1283 mlx5_fwdump_clean(dev); 1284 mlx5_pagealloc_cleanup(dev); 1285 mlx5_health_cleanup(dev); 1286 mlx5_pci_close(dev, priv); 1287 pci_set_drvdata(pdev, NULL); 1288 sysctl_ctx_free(&dev->sysctl_ctx); 1289 kfree(dev); 1290} 1291 1292static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, 1293 pci_channel_state_t state) 1294{ 1295 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1296 struct mlx5_priv *priv = &dev->priv; 1297 1298 dev_info(&pdev->dev, "%s was called\n", __func__); 1299 mlx5_enter_error_state(dev, false); 1300 mlx5_unload_one(dev, priv, false); 1301 1302 if (state) { 1303 mlx5_drain_health_wq(dev); 1304 mlx5_pci_disable_device(dev); 1305 } 1306 1307 return state == pci_channel_io_perm_failure ? 1308 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 1309} 1310 1311static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) 1312{ 1313 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1314 int err = 0; 1315 1316 dev_info(&pdev->dev, "%s was called\n", __func__); 1317 1318 err = mlx5_pci_enable_device(dev); 1319 if (err) { 1320 dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" 1321 , __func__, err); 1322 return PCI_ERS_RESULT_DISCONNECT; 1323 } 1324 pci_set_master(pdev); 1325 pci_set_powerstate(pdev->dev.bsddev, PCI_POWERSTATE_D0); 1326 pci_restore_state(pdev->dev.bsddev); 1327 pci_save_state(pdev->dev.bsddev); 1328 1329 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 1330} 1331 1332/* wait for the device to show vital signs. For now we check 1333 * that we can read the device ID and that the health buffer 1334 * shows a non zero value which is different than 0xffffffff 1335 */ 1336static void wait_vital(struct pci_dev *pdev) 1337{ 1338 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1339 struct mlx5_core_health *health = &dev->priv.health; 1340 const int niter = 100; 1341 u32 count; 1342 u16 did; 1343 int i; 1344 1345 /* Wait for firmware to be ready after reset */ 1346 msleep(1000); 1347 for (i = 0; i < niter; i++) { 1348 if (pci_read_config_word(pdev, 2, &did)) { 1349 dev_warn(&pdev->dev, "failed reading config word\n"); 1350 break; 1351 } 1352 if (did == pdev->device) { 1353 dev_info(&pdev->dev, "device ID correctly read after %d iterations\n", i); 1354 break; 1355 } 1356 msleep(50); 1357 } 1358 if (i == niter) 1359 dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__); 1360 1361 for (i = 0; i < niter; i++) { 1362 count = ioread32be(health->health_counter); 1363 if (count && count != 0xffffffff) { 1364 dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i); 1365 break; 1366 } 1367 msleep(50); 1368 } 1369 1370 if (i == niter) 1371 dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__); 1372} 1373 1374static void mlx5_pci_resume(struct pci_dev *pdev) 1375{ 1376 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1377 struct mlx5_priv *priv = &dev->priv; 1378 int err; 1379 1380 dev_info(&pdev->dev, "%s was called\n", __func__); 1381 1382 wait_vital(pdev); 1383 1384 err = mlx5_load_one(dev, priv, false); 1385 if (err) 1386 dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n" 1387 , __func__, err); 1388 else 1389 dev_info(&pdev->dev, "%s: device recovered\n", __func__); 1390} 1391 1392static const struct pci_error_handlers mlx5_err_handler = { 1393 .error_detected = mlx5_pci_err_detected, 1394 .slot_reset = mlx5_pci_slot_reset, 1395 .resume = mlx5_pci_resume 1396}; 1397 1398static int mlx5_try_fast_unload(struct mlx5_core_dev *dev) 1399{ 1400 int err; 1401 1402 if (!MLX5_CAP_GEN(dev, force_teardown)) { 1403 mlx5_core_dbg(dev, "force teardown is not supported in the firmware\n"); 1404 return -EOPNOTSUPP; 1405 } 1406 1407 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 1408 mlx5_core_dbg(dev, "Device in internal error state, giving up\n"); 1409 return -EAGAIN; 1410 } 1411 1412 /* Panic tear down fw command will stop the PCI bus communication 1413 * with the HCA, so the health polll is no longer needed. 1414 */ 1415 mlx5_drain_health_wq(dev); 1416 mlx5_stop_health_poll(dev, false); 1417 1418 err = mlx5_cmd_force_teardown_hca(dev); 1419 if (err) { 1420 mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", err); 1421 return err; 1422 } 1423 1424 mlx5_enter_error_state(dev, true); 1425 1426 return 0; 1427} 1428 1429static void mlx5_disable_interrupts(struct mlx5_core_dev *mdev) 1430{ 1431 int nvec = mdev->priv.eq_table.num_comp_vectors + MLX5_EQ_VEC_COMP_BASE; 1432 int x; 1433 1434 mdev->priv.disable_irqs = 1; 1435 1436 /* wait for all IRQ handlers to finish processing */ 1437 for (x = 0; x != nvec; x++) 1438 synchronize_irq(mdev->priv.msix_arr[x].vector); 1439} 1440 1441static void shutdown_one(struct pci_dev *pdev) 1442{ 1443 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1444 struct mlx5_priv *priv = &dev->priv; 1445 int err; 1446 1447 /* enter polling mode */ 1448 mlx5_cmd_use_polling(dev); 1449 1450 /* disable all interrupts */ 1451 mlx5_disable_interrupts(dev); 1452 1453 err = mlx5_try_fast_unload(dev); 1454 if (err) 1455 mlx5_unload_one(dev, priv, false); 1456 mlx5_pci_disable_device(dev); 1457} 1458 1459static const struct pci_device_id mlx5_core_pci_table[] = { 1460 { PCI_VDEVICE(MELLANOX, 4113) }, /* Connect-IB */ 1461 { PCI_VDEVICE(MELLANOX, 4114) }, /* Connect-IB VF */ 1462 { PCI_VDEVICE(MELLANOX, 4115) }, /* ConnectX-4 */ 1463 { PCI_VDEVICE(MELLANOX, 4116) }, /* ConnectX-4 VF */ 1464 { PCI_VDEVICE(MELLANOX, 4117) }, /* ConnectX-4LX */ 1465 { PCI_VDEVICE(MELLANOX, 4118) }, /* ConnectX-4LX VF */ 1466 { PCI_VDEVICE(MELLANOX, 4119) }, /* ConnectX-5 */ 1467 { PCI_VDEVICE(MELLANOX, 4120) }, /* ConnectX-5 VF */ 1468 { PCI_VDEVICE(MELLANOX, 4121) }, 1469 { PCI_VDEVICE(MELLANOX, 4122) }, 1470 { PCI_VDEVICE(MELLANOX, 4123) }, 1471 { PCI_VDEVICE(MELLANOX, 4124) }, 1472 { PCI_VDEVICE(MELLANOX, 4125) }, 1473 { PCI_VDEVICE(MELLANOX, 4126) }, 1474 { PCI_VDEVICE(MELLANOX, 4127) }, 1475 { PCI_VDEVICE(MELLANOX, 4128) }, 1476 { PCI_VDEVICE(MELLANOX, 4129) }, 1477 { PCI_VDEVICE(MELLANOX, 4130) }, 1478 { PCI_VDEVICE(MELLANOX, 4131) }, 1479 { PCI_VDEVICE(MELLANOX, 4132) }, 1480 { PCI_VDEVICE(MELLANOX, 4133) }, 1481 { PCI_VDEVICE(MELLANOX, 4134) }, 1482 { PCI_VDEVICE(MELLANOX, 4135) }, 1483 { PCI_VDEVICE(MELLANOX, 4136) }, 1484 { PCI_VDEVICE(MELLANOX, 4137) }, 1485 { PCI_VDEVICE(MELLANOX, 4138) }, 1486 { PCI_VDEVICE(MELLANOX, 4139) }, 1487 { PCI_VDEVICE(MELLANOX, 4140) }, 1488 { PCI_VDEVICE(MELLANOX, 4141) }, 1489 { PCI_VDEVICE(MELLANOX, 4142) }, 1490 { PCI_VDEVICE(MELLANOX, 4143) }, 1491 { PCI_VDEVICE(MELLANOX, 4144) }, 1492 { 0, } 1493}; 1494 1495MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 1496 1497void mlx5_disable_device(struct mlx5_core_dev *dev) 1498{ 1499 mlx5_pci_err_detected(dev->pdev, 0); 1500} 1501 1502void mlx5_recover_device(struct mlx5_core_dev *dev) 1503{ 1504 mlx5_pci_disable_device(dev); 1505 if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED) 1506 mlx5_pci_resume(dev->pdev); 1507} 1508 1509struct pci_driver mlx5_core_driver = { 1510 .name = DRIVER_NAME, 1511 .id_table = mlx5_core_pci_table, 1512 .shutdown = shutdown_one, 1513 .probe = init_one, 1514 .remove = remove_one, 1515 .err_handler = &mlx5_err_handler 1516}; 1517 1518static int __init init(void) 1519{ 1520 int err; 1521 1522 err = pci_register_driver(&mlx5_core_driver); 1523 if (err) 1524 goto err_debug; 1525 1526 err = mlx5_fwdump_init(); 1527 if (err) 1528 goto err_fwdump; 1529 1530 return 0; 1531 1532err_fwdump: 1533 pci_unregister_driver(&mlx5_core_driver); 1534 1535err_debug: 1536 return err; 1537} 1538 1539static void __exit cleanup(void) 1540{ 1541 mlx5_fwdump_fini(); 1542 pci_unregister_driver(&mlx5_core_driver); 1543} 1544 1545module_init(init); 1546module_exit(cleanup); 1547