mlx5_main.c revision 329212
1290650Shselasky/*- 2329200Shselasky * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c 329212 2018-02-13 15:16:10Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28300676Shselasky#define LINUXKPI_PARAM_PREFIX mlx5_ 29300676Shselasky 30290650Shselasky#include <linux/kmod.h> 31290650Shselasky#include <linux/module.h> 32290650Shselasky#include <linux/errno.h> 33290650Shselasky#include <linux/pci.h> 34290650Shselasky#include <linux/dma-mapping.h> 35290650Shselasky#include <linux/slab.h> 36290650Shselasky#include <linux/io-mapping.h> 37290650Shselasky#include <linux/interrupt.h> 38290650Shselasky#include <dev/mlx5/driver.h> 39290650Shselasky#include <dev/mlx5/cq.h> 40290650Shselasky#include <dev/mlx5/qp.h> 41290650Shselasky#include <dev/mlx5/srq.h> 42290650Shselasky#include <linux/delay.h> 43290650Shselasky#include <dev/mlx5/mlx5_ifc.h> 44290650Shselasky#include "mlx5_core.h" 45329200Shselasky#include "fs_core.h" 46290650Shselasky 47290650ShselaskyMODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 48290650ShselaskyMODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 49290650ShselaskyMODULE_LICENSE("Dual BSD/GPL"); 50290650Shselasky#if (__FreeBSD_version >= 1100000) 51290650ShselaskyMODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1); 52290650Shselasky#endif 53290650ShselaskyMODULE_VERSION(mlx5, 1); 54290650Shselasky 55290650Shselaskyint mlx5_core_debug_mask; 56290650Shselaskymodule_param_named(debug_mask, mlx5_core_debug_mask, int, 0644); 57290650ShselaskyMODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 58290650Shselasky 59290650Shselasky#define MLX5_DEFAULT_PROF 2 60290650Shselaskystatic int prof_sel = MLX5_DEFAULT_PROF; 61290650Shselaskymodule_param_named(prof_sel, prof_sel, int, 0444); 62290650ShselaskyMODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 63290650Shselasky 64290650Shselasky#define NUMA_NO_NODE -1 65290650Shselasky 66290650Shselaskystruct workqueue_struct *mlx5_core_wq; 67290650Shselaskystatic LIST_HEAD(intf_list); 68290650Shselaskystatic LIST_HEAD(dev_list); 69290650Shselaskystatic DEFINE_MUTEX(intf_mutex); 70290650Shselasky 71290650Shselaskystruct mlx5_device_context { 72290650Shselasky struct list_head list; 73290650Shselasky struct mlx5_interface *intf; 74290650Shselasky void *context; 75290650Shselasky}; 76290650Shselasky 77329209Shselaskyenum { 78329209Shselasky MLX5_ATOMIC_REQ_MODE_BE = 0x0, 79329209Shselasky MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, 80329209Shselasky}; 81329209Shselasky 82290650Shselaskystatic struct mlx5_profile profiles[] = { 83290650Shselasky [0] = { 84290650Shselasky .mask = 0, 85290650Shselasky }, 86290650Shselasky [1] = { 87290650Shselasky .mask = MLX5_PROF_MASK_QP_SIZE, 88290650Shselasky .log_max_qp = 12, 89290650Shselasky }, 90290650Shselasky [2] = { 91290650Shselasky .mask = MLX5_PROF_MASK_QP_SIZE | 92290650Shselasky MLX5_PROF_MASK_MR_CACHE, 93290650Shselasky .log_max_qp = 17, 94290650Shselasky .mr_cache[0] = { 95290650Shselasky .size = 500, 96290650Shselasky .limit = 250 97290650Shselasky }, 98290650Shselasky .mr_cache[1] = { 99290650Shselasky .size = 500, 100290650Shselasky .limit = 250 101290650Shselasky }, 102290650Shselasky .mr_cache[2] = { 103290650Shselasky .size = 500, 104290650Shselasky .limit = 250 105290650Shselasky }, 106290650Shselasky .mr_cache[3] = { 107290650Shselasky .size = 500, 108290650Shselasky .limit = 250 109290650Shselasky }, 110290650Shselasky .mr_cache[4] = { 111290650Shselasky .size = 500, 112290650Shselasky .limit = 250 113290650Shselasky }, 114290650Shselasky .mr_cache[5] = { 115290650Shselasky .size = 500, 116290650Shselasky .limit = 250 117290650Shselasky }, 118290650Shselasky .mr_cache[6] = { 119290650Shselasky .size = 500, 120290650Shselasky .limit = 250 121290650Shselasky }, 122290650Shselasky .mr_cache[7] = { 123290650Shselasky .size = 500, 124290650Shselasky .limit = 250 125290650Shselasky }, 126290650Shselasky .mr_cache[8] = { 127290650Shselasky .size = 500, 128290650Shselasky .limit = 250 129290650Shselasky }, 130290650Shselasky .mr_cache[9] = { 131290650Shselasky .size = 500, 132290650Shselasky .limit = 250 133290650Shselasky }, 134290650Shselasky .mr_cache[10] = { 135290650Shselasky .size = 500, 136290650Shselasky .limit = 250 137290650Shselasky }, 138290650Shselasky .mr_cache[11] = { 139290650Shselasky .size = 500, 140290650Shselasky .limit = 250 141290650Shselasky }, 142290650Shselasky .mr_cache[12] = { 143290650Shselasky .size = 64, 144290650Shselasky .limit = 32 145290650Shselasky }, 146290650Shselasky .mr_cache[13] = { 147290650Shselasky .size = 32, 148290650Shselasky .limit = 16 149290650Shselasky }, 150290650Shselasky .mr_cache[14] = { 151290650Shselasky .size = 16, 152290650Shselasky .limit = 8 153290650Shselasky }, 154290650Shselasky }, 155290650Shselasky [3] = { 156290650Shselasky .mask = MLX5_PROF_MASK_QP_SIZE, 157290650Shselasky .log_max_qp = 17, 158290650Shselasky }, 159290650Shselasky}; 160290650Shselasky 161290650Shselaskystatic int set_dma_caps(struct pci_dev *pdev) 162290650Shselasky{ 163290650Shselasky int err; 164290650Shselasky 165290650Shselasky err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 166290650Shselasky if (err) { 167290650Shselasky device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit PCI DMA mask\n"); 168290650Shselasky err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 169290650Shselasky if (err) { 170290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set PCI DMA mask, aborting\n"); 171290650Shselasky return err; 172290650Shselasky } 173290650Shselasky } 174290650Shselasky 175290650Shselasky err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 176290650Shselasky if (err) { 177290650Shselasky device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 178290650Shselasky err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 179290650Shselasky if (err) { 180290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set consistent PCI DMA mask, aborting\n"); 181290650Shselasky return err; 182290650Shselasky } 183290650Shselasky } 184290650Shselasky 185290650Shselasky dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 186290650Shselasky return err; 187290650Shselasky} 188290650Shselasky 189290650Shselaskystatic int request_bar(struct pci_dev *pdev) 190290650Shselasky{ 191290650Shselasky int err = 0; 192290650Shselasky 193290650Shselasky if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 194290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Missing registers BAR, aborting\n"); 195290650Shselasky return -ENODEV; 196290650Shselasky } 197290650Shselasky 198290650Shselasky err = pci_request_regions(pdev, DRIVER_NAME); 199290650Shselasky if (err) 200290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Couldn't get PCI resources, aborting\n"); 201290650Shselasky 202290650Shselasky return err; 203290650Shselasky} 204290650Shselasky 205290650Shselaskystatic void release_bar(struct pci_dev *pdev) 206290650Shselasky{ 207290650Shselasky pci_release_regions(pdev); 208290650Shselasky} 209290650Shselasky 210290650Shselaskystatic int mlx5_enable_msix(struct mlx5_core_dev *dev) 211290650Shselasky{ 212290650Shselasky struct mlx5_priv *priv = &dev->priv; 213290650Shselasky struct mlx5_eq_table *table = &priv->eq_table; 214290650Shselasky int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 215290650Shselasky int nvec; 216290650Shselasky int i; 217290650Shselasky 218290650Shselasky nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 219290650Shselasky MLX5_EQ_VEC_COMP_BASE; 220290650Shselasky nvec = min_t(int, nvec, num_eqs); 221290650Shselasky if (nvec <= MLX5_EQ_VEC_COMP_BASE) 222290650Shselasky return -ENOMEM; 223290650Shselasky 224290650Shselasky priv->msix_arr = kzalloc(nvec * sizeof(*priv->msix_arr), GFP_KERNEL); 225290650Shselasky 226290650Shselasky priv->irq_info = kzalloc(nvec * sizeof(*priv->irq_info), GFP_KERNEL); 227290650Shselasky 228290650Shselasky for (i = 0; i < nvec; i++) 229290650Shselasky priv->msix_arr[i].entry = i; 230290650Shselasky 231290650Shselasky nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 232290650Shselasky MLX5_EQ_VEC_COMP_BASE + 1, nvec); 233290650Shselasky if (nvec < 0) 234290650Shselasky return nvec; 235290650Shselasky 236290650Shselasky table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 237290650Shselasky 238290650Shselasky return 0; 239290650Shselasky 240290650Shselasky} 241290650Shselasky 242290650Shselaskystatic void mlx5_disable_msix(struct mlx5_core_dev *dev) 243290650Shselasky{ 244290650Shselasky struct mlx5_priv *priv = &dev->priv; 245290650Shselasky 246290650Shselasky pci_disable_msix(dev->pdev); 247290650Shselasky kfree(priv->irq_info); 248290650Shselasky kfree(priv->msix_arr); 249290650Shselasky} 250290650Shselasky 251290650Shselaskystruct mlx5_reg_host_endianess { 252290650Shselasky u8 he; 253290650Shselasky u8 rsvd[15]; 254290650Shselasky}; 255290650Shselasky 256290650Shselasky 257290650Shselasky#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 258290650Shselasky 259290650Shselaskyenum { 260290650Shselasky MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 261306233Shselasky MLX5_DEV_CAP_FLAG_DCT | 262306233Shselasky MLX5_DEV_CAP_FLAG_DRAIN_SIGERR, 263290650Shselasky}; 264290650Shselasky 265290650Shselaskystatic u16 to_fw_pkey_sz(u32 size) 266290650Shselasky{ 267290650Shselasky switch (size) { 268290650Shselasky case 128: 269290650Shselasky return 0; 270290650Shselasky case 256: 271290650Shselasky return 1; 272290650Shselasky case 512: 273290650Shselasky return 2; 274290650Shselasky case 1024: 275290650Shselasky return 3; 276290650Shselasky case 2048: 277290650Shselasky return 4; 278290650Shselasky case 4096: 279290650Shselasky return 5; 280290650Shselasky default: 281290650Shselasky printf("mlx5_core: WARN: ""invalid pkey table size %d\n", size); 282290650Shselasky return 0; 283290650Shselasky } 284290650Shselasky} 285290650Shselasky 286290650Shselaskyint mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, 287290650Shselasky enum mlx5_cap_mode cap_mode) 288290650Shselasky{ 289290650Shselasky u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 290290650Shselasky int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 291290650Shselasky void *out, *hca_caps; 292290650Shselasky u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 293290650Shselasky int err; 294290650Shselasky 295290650Shselasky memset(in, 0, sizeof(in)); 296290650Shselasky out = kzalloc(out_sz, GFP_KERNEL); 297290650Shselasky 298290650Shselasky MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 299290650Shselasky MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 300290650Shselasky err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 301290650Shselasky if (err) 302290650Shselasky goto query_ex; 303290650Shselasky 304290650Shselasky err = mlx5_cmd_status_to_err_v2(out); 305290650Shselasky if (err) { 306290650Shselasky mlx5_core_warn(dev, 307290650Shselasky "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 308290650Shselasky cap_type, cap_mode, err); 309290650Shselasky goto query_ex; 310290650Shselasky } 311290650Shselasky 312290650Shselasky hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 313290650Shselasky 314290650Shselasky switch (cap_mode) { 315290650Shselasky case HCA_CAP_OPMOD_GET_MAX: 316290650Shselasky memcpy(dev->hca_caps_max[cap_type], hca_caps, 317290650Shselasky MLX5_UN_SZ_BYTES(hca_cap_union)); 318290650Shselasky break; 319290650Shselasky case HCA_CAP_OPMOD_GET_CUR: 320290650Shselasky memcpy(dev->hca_caps_cur[cap_type], hca_caps, 321290650Shselasky MLX5_UN_SZ_BYTES(hca_cap_union)); 322290650Shselasky break; 323290650Shselasky default: 324290650Shselasky mlx5_core_warn(dev, 325290650Shselasky "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 326290650Shselasky cap_type, cap_mode); 327290650Shselasky err = -EINVAL; 328290650Shselasky break; 329290650Shselasky } 330290650Shselaskyquery_ex: 331290650Shselasky kfree(out); 332290650Shselasky return err; 333290650Shselasky} 334290650Shselasky 335290650Shselaskystatic int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz) 336290650Shselasky{ 337290650Shselasky u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)]; 338290650Shselasky int err; 339290650Shselasky 340290650Shselasky memset(out, 0, sizeof(out)); 341290650Shselasky 342290650Shselasky MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 343290650Shselasky err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 344290650Shselasky if (err) 345290650Shselasky return err; 346290650Shselasky 347290650Shselasky err = mlx5_cmd_status_to_err_v2(out); 348290650Shselasky 349290650Shselasky return err; 350290650Shselasky} 351290650Shselasky 352290650Shselaskystatic int handle_hca_cap(struct mlx5_core_dev *dev) 353290650Shselasky{ 354290650Shselasky void *set_ctx = NULL; 355290650Shselasky struct mlx5_profile *prof = dev->profile; 356290650Shselasky int err = -ENOMEM; 357290650Shselasky int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 358290650Shselasky void *set_hca_cap; 359290650Shselasky 360290650Shselasky set_ctx = kzalloc(set_sz, GFP_KERNEL); 361290650Shselasky 362290650Shselasky err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_MAX); 363290650Shselasky if (err) 364290650Shselasky goto query_ex; 365290650Shselasky 366290650Shselasky err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_CUR); 367290650Shselasky if (err) 368290650Shselasky goto query_ex; 369290650Shselasky 370290650Shselasky set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 371290650Shselasky capability); 372290650Shselasky memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL], 373290650Shselasky MLX5_ST_SZ_BYTES(cmd_hca_cap)); 374290650Shselasky 375290650Shselasky mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 376290650Shselasky mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 377290650Shselasky 128); 378290650Shselasky /* we limit the size of the pkey table to 128 entries for now */ 379290650Shselasky MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 380290650Shselasky to_fw_pkey_sz(128)); 381290650Shselasky 382290650Shselasky if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 383290650Shselasky MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 384290650Shselasky prof->log_max_qp); 385290650Shselasky 386290650Shselasky /* disable cmdif checksum */ 387290650Shselasky MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 388290650Shselasky 389306233Shselasky /* enable drain sigerr */ 390306233Shselasky MLX5_SET(cmd_hca_cap, set_hca_cap, drain_sigerr, 1); 391306233Shselasky 392290650Shselasky MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 393290650Shselasky 394290650Shselasky err = set_caps(dev, set_ctx, set_sz); 395290650Shselasky 396290650Shselaskyquery_ex: 397290650Shselasky kfree(set_ctx); 398290650Shselasky return err; 399290650Shselasky} 400290650Shselasky 401329209Shselaskystatic int handle_hca_cap_atomic(struct mlx5_core_dev *dev) 402329209Shselasky{ 403329209Shselasky void *set_ctx; 404329209Shselasky void *set_hca_cap; 405329209Shselasky int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 406329209Shselasky int req_endianness; 407329209Shselasky int err; 408329209Shselasky 409329209Shselasky if (MLX5_CAP_GEN(dev, atomic)) { 410329209Shselasky err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC, 411329209Shselasky HCA_CAP_OPMOD_GET_MAX); 412329209Shselasky if (err) 413329209Shselasky return err; 414329209Shselasky 415329209Shselasky err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC, 416329209Shselasky HCA_CAP_OPMOD_GET_CUR); 417329209Shselasky if (err) 418329209Shselasky return err; 419329209Shselasky } else { 420329209Shselasky return 0; 421329209Shselasky } 422329209Shselasky 423329209Shselasky req_endianness = 424329209Shselasky MLX5_CAP_ATOMIC(dev, 425329209Shselasky supported_atomic_req_8B_endianess_mode_1); 426329209Shselasky 427329209Shselasky if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) 428329209Shselasky return 0; 429329209Shselasky 430329209Shselasky set_ctx = kzalloc(set_sz, GFP_KERNEL); 431329209Shselasky if (!set_ctx) 432329209Shselasky return -ENOMEM; 433329209Shselasky 434329209Shselasky MLX5_SET(set_hca_cap_in, set_ctx, op_mod, 435329209Shselasky MLX5_SET_HCA_CAP_OP_MOD_ATOMIC << 1); 436329209Shselasky set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 437329209Shselasky 438329209Shselasky /* Set requestor to host endianness */ 439329209Shselasky MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode, 440329209Shselasky MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); 441329209Shselasky 442329209Shselasky err = set_caps(dev, set_ctx, set_sz); 443329209Shselasky 444329209Shselasky kfree(set_ctx); 445329209Shselasky return err; 446329209Shselasky} 447329209Shselasky 448290650Shselaskystatic int set_hca_ctrl(struct mlx5_core_dev *dev) 449290650Shselasky{ 450290650Shselasky struct mlx5_reg_host_endianess he_in; 451290650Shselasky struct mlx5_reg_host_endianess he_out; 452290650Shselasky int err; 453290650Shselasky 454306233Shselasky if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH && 455306233Shselasky !MLX5_CAP_GEN(dev, roce)) 456306233Shselasky return 0; 457306233Shselasky 458290650Shselasky memset(&he_in, 0, sizeof(he_in)); 459290650Shselasky he_in.he = MLX5_SET_HOST_ENDIANNESS; 460290650Shselasky err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 461290650Shselasky &he_out, sizeof(he_out), 462290650Shselasky MLX5_REG_HOST_ENDIANNESS, 0, 1); 463290650Shselasky return err; 464290650Shselasky} 465290650Shselasky 466290650Shselaskystatic int mlx5_core_enable_hca(struct mlx5_core_dev *dev) 467290650Shselasky{ 468290650Shselasky u32 in[MLX5_ST_SZ_DW(enable_hca_in)]; 469290650Shselasky u32 out[MLX5_ST_SZ_DW(enable_hca_out)]; 470290650Shselasky 471290650Shselasky memset(in, 0, sizeof(in)); 472290650Shselasky MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); 473290650Shselasky memset(out, 0, sizeof(out)); 474290650Shselasky return mlx5_cmd_exec_check_status(dev, in, sizeof(in), 475290650Shselasky out, sizeof(out)); 476290650Shselasky} 477290650Shselasky 478290650Shselaskystatic int mlx5_core_disable_hca(struct mlx5_core_dev *dev) 479290650Shselasky{ 480290650Shselasky u32 in[MLX5_ST_SZ_DW(disable_hca_in)]; 481290650Shselasky u32 out[MLX5_ST_SZ_DW(disable_hca_out)]; 482290650Shselasky 483290650Shselasky memset(in, 0, sizeof(in)); 484290650Shselasky 485290650Shselasky MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); 486290650Shselasky memset(out, 0, sizeof(out)); 487290650Shselasky return mlx5_cmd_exec_check_status(dev, in, sizeof(in), 488290650Shselasky out, sizeof(out)); 489290650Shselasky} 490290650Shselasky 491290650Shselaskystatic int mlx5_core_set_issi(struct mlx5_core_dev *dev) 492290650Shselasky{ 493290650Shselasky u32 query_in[MLX5_ST_SZ_DW(query_issi_in)]; 494290650Shselasky u32 query_out[MLX5_ST_SZ_DW(query_issi_out)]; 495290650Shselasky u32 set_in[MLX5_ST_SZ_DW(set_issi_in)]; 496290650Shselasky u32 set_out[MLX5_ST_SZ_DW(set_issi_out)]; 497290650Shselasky int err; 498290650Shselasky u32 sup_issi; 499290650Shselasky 500290650Shselasky memset(query_in, 0, sizeof(query_in)); 501290650Shselasky memset(query_out, 0, sizeof(query_out)); 502290650Shselasky 503290650Shselasky MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 504290650Shselasky 505290650Shselasky err = mlx5_cmd_exec_check_status(dev, query_in, sizeof(query_in), 506290650Shselasky query_out, sizeof(query_out)); 507290650Shselasky if (err) { 508290650Shselasky if (((struct mlx5_outbox_hdr *)query_out)->status == 509290650Shselasky MLX5_CMD_STAT_BAD_OP_ERR) { 510290650Shselasky pr_debug("Only ISSI 0 is supported\n"); 511290650Shselasky return 0; 512290650Shselasky } 513290650Shselasky 514290650Shselasky printf("mlx5_core: ERR: ""failed to query ISSI\n"); 515290650Shselasky return err; 516290650Shselasky } 517290650Shselasky 518290650Shselasky sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 519290650Shselasky 520290650Shselasky if (sup_issi & (1 << 1)) { 521290650Shselasky memset(set_in, 0, sizeof(set_in)); 522290650Shselasky memset(set_out, 0, sizeof(set_out)); 523290650Shselasky 524290650Shselasky MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 525290650Shselasky MLX5_SET(set_issi_in, set_in, current_issi, 1); 526290650Shselasky 527290650Shselasky err = mlx5_cmd_exec_check_status(dev, set_in, sizeof(set_in), 528290650Shselasky set_out, sizeof(set_out)); 529290650Shselasky if (err) { 530290650Shselasky printf("mlx5_core: ERR: ""failed to set ISSI=1\n"); 531290650Shselasky return err; 532290650Shselasky } 533290650Shselasky 534290650Shselasky dev->issi = 1; 535290650Shselasky 536290650Shselasky return 0; 537290650Shselasky } else if (sup_issi & (1 << 0)) { 538290650Shselasky return 0; 539290650Shselasky } 540290650Shselasky 541290650Shselasky return -ENOTSUPP; 542290650Shselasky} 543290650Shselasky 544290650Shselasky 545290650Shselaskyint mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn) 546290650Shselasky{ 547290650Shselasky struct mlx5_eq_table *table = &dev->priv.eq_table; 548290650Shselasky struct mlx5_eq *eq; 549290650Shselasky int err = -ENOENT; 550290650Shselasky 551290650Shselasky spin_lock(&table->lock); 552290650Shselasky list_for_each_entry(eq, &table->comp_eqs_list, list) { 553290650Shselasky if (eq->index == vector) { 554290650Shselasky *eqn = eq->eqn; 555290650Shselasky *irqn = eq->irqn; 556290650Shselasky err = 0; 557290650Shselasky break; 558290650Shselasky } 559290650Shselasky } 560290650Shselasky spin_unlock(&table->lock); 561290650Shselasky 562290650Shselasky return err; 563290650Shselasky} 564290650ShselaskyEXPORT_SYMBOL(mlx5_vector2eqn); 565290650Shselasky 566290650Shselaskyint mlx5_rename_eq(struct mlx5_core_dev *dev, int eq_ix, char *name) 567290650Shselasky{ 568290650Shselasky struct mlx5_priv *priv = &dev->priv; 569290650Shselasky struct mlx5_eq_table *table = &priv->eq_table; 570290650Shselasky struct mlx5_eq *eq; 571290650Shselasky int err = -ENOENT; 572290650Shselasky 573290650Shselasky spin_lock(&table->lock); 574290650Shselasky list_for_each_entry(eq, &table->comp_eqs_list, list) { 575290650Shselasky if (eq->index == eq_ix) { 576290650Shselasky int irq_ix = eq_ix + MLX5_EQ_VEC_COMP_BASE; 577290650Shselasky 578290650Shselasky snprintf(priv->irq_info[irq_ix].name, MLX5_MAX_IRQ_NAME, 579290650Shselasky "%s-%d", name, eq_ix); 580290650Shselasky 581290650Shselasky err = 0; 582290650Shselasky break; 583290650Shselasky } 584290650Shselasky } 585290650Shselasky spin_unlock(&table->lock); 586290650Shselasky 587290650Shselasky return err; 588290650Shselasky} 589290650Shselasky 590290650Shselaskystatic void free_comp_eqs(struct mlx5_core_dev *dev) 591290650Shselasky{ 592290650Shselasky struct mlx5_eq_table *table = &dev->priv.eq_table; 593290650Shselasky struct mlx5_eq *eq, *n; 594290650Shselasky 595290650Shselasky spin_lock(&table->lock); 596290650Shselasky list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 597290650Shselasky list_del(&eq->list); 598290650Shselasky spin_unlock(&table->lock); 599290650Shselasky if (mlx5_destroy_unmap_eq(dev, eq)) 600290650Shselasky mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 601290650Shselasky eq->eqn); 602290650Shselasky kfree(eq); 603290650Shselasky spin_lock(&table->lock); 604290650Shselasky } 605290650Shselasky spin_unlock(&table->lock); 606290650Shselasky} 607290650Shselasky 608290650Shselaskystatic int alloc_comp_eqs(struct mlx5_core_dev *dev) 609290650Shselasky{ 610290650Shselasky struct mlx5_eq_table *table = &dev->priv.eq_table; 611290650Shselasky char name[MLX5_MAX_IRQ_NAME]; 612290650Shselasky struct mlx5_eq *eq; 613290650Shselasky int ncomp_vec; 614290650Shselasky int nent; 615290650Shselasky int err; 616290650Shselasky int i; 617290650Shselasky 618290650Shselasky INIT_LIST_HEAD(&table->comp_eqs_list); 619290650Shselasky ncomp_vec = table->num_comp_vectors; 620290650Shselasky nent = MLX5_COMP_EQ_SIZE; 621290650Shselasky for (i = 0; i < ncomp_vec; i++) { 622290650Shselasky eq = kzalloc(sizeof(*eq), GFP_KERNEL); 623290650Shselasky 624290650Shselasky snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 625290650Shselasky err = mlx5_create_map_eq(dev, eq, 626290650Shselasky i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 627290650Shselasky name, &dev->priv.uuari.uars[0]); 628290650Shselasky if (err) { 629290650Shselasky kfree(eq); 630290650Shselasky goto clean; 631290650Shselasky } 632290650Shselasky mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 633290650Shselasky eq->index = i; 634290650Shselasky spin_lock(&table->lock); 635290650Shselasky list_add_tail(&eq->list, &table->comp_eqs_list); 636290650Shselasky spin_unlock(&table->lock); 637290650Shselasky } 638290650Shselasky 639290650Shselasky return 0; 640290650Shselasky 641290650Shselaskyclean: 642290650Shselasky free_comp_eqs(dev); 643290650Shselasky return err; 644290650Shselasky} 645290650Shselasky 646290650Shselaskystatic int map_bf_area(struct mlx5_core_dev *dev) 647290650Shselasky{ 648290650Shselasky resource_size_t bf_start = pci_resource_start(dev->pdev, 0); 649290650Shselasky resource_size_t bf_len = pci_resource_len(dev->pdev, 0); 650290650Shselasky 651290650Shselasky dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len); 652290650Shselasky 653290650Shselasky return dev->priv.bf_mapping ? 0 : -ENOMEM; 654290650Shselasky} 655290650Shselasky 656290650Shselaskystatic void unmap_bf_area(struct mlx5_core_dev *dev) 657290650Shselasky{ 658290650Shselasky if (dev->priv.bf_mapping) 659290650Shselasky io_mapping_free(dev->priv.bf_mapping); 660290650Shselasky} 661290650Shselasky 662290650Shselaskystatic inline int fw_initializing(struct mlx5_core_dev *dev) 663290650Shselasky{ 664290650Shselasky return ioread32be(&dev->iseg->initializing) >> 31; 665290650Shselasky} 666290650Shselasky 667290650Shselaskystatic int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) 668290650Shselasky{ 669290650Shselasky u64 end = jiffies + msecs_to_jiffies(max_wait_mili); 670290650Shselasky int err = 0; 671290650Shselasky 672290650Shselasky while (fw_initializing(dev)) { 673290650Shselasky if (time_after(jiffies, end)) { 674290650Shselasky err = -EBUSY; 675290650Shselasky break; 676290650Shselasky } 677290650Shselasky msleep(FW_INIT_WAIT_MS); 678290650Shselasky } 679290650Shselasky 680290650Shselasky return err; 681290650Shselasky} 682290650Shselasky 683290650Shselaskystatic int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev) 684290650Shselasky{ 685290650Shselasky struct mlx5_priv *priv = &dev->priv; 686290650Shselasky int err; 687290650Shselasky 688290650Shselasky dev->pdev = pdev; 689290650Shselasky pci_set_drvdata(dev->pdev, dev); 690290650Shselasky strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 691290650Shselasky priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 692290650Shselasky 693290650Shselasky mutex_init(&priv->pgdir_mutex); 694290650Shselasky INIT_LIST_HEAD(&priv->pgdir_list); 695290650Shselasky spin_lock_init(&priv->mkey_lock); 696290650Shselasky 697290650Shselasky priv->numa_node = NUMA_NO_NODE; 698290650Shselasky 699290650Shselasky err = pci_enable_device(pdev); 700290650Shselasky if (err) { 701290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Cannot enable PCI device, aborting\n"); 702290650Shselasky goto err_dbg; 703290650Shselasky } 704290650Shselasky 705290650Shselasky err = request_bar(pdev); 706290650Shselasky if (err) { 707290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""error requesting BARs, aborting\n"); 708290650Shselasky goto err_disable; 709290650Shselasky } 710290650Shselasky 711290650Shselasky pci_set_master(pdev); 712290650Shselasky 713290650Shselasky err = set_dma_caps(pdev); 714290650Shselasky if (err) { 715290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Failed setting DMA capabilities mask, aborting\n"); 716290650Shselasky goto err_clr_master; 717290650Shselasky } 718290650Shselasky 719329212Shselasky dev->iseg_base = pci_resource_start(dev->pdev, 0); 720329212Shselasky dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 721290650Shselasky if (!dev->iseg) { 722290650Shselasky err = -ENOMEM; 723290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Failed mapping initialization segment, aborting\n"); 724290650Shselasky goto err_clr_master; 725290650Shselasky } 726290650Shselasky device_printf((&pdev->dev)->bsddev, "INFO: ""firmware version: %d.%d.%d\n", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); 727290650Shselasky 728306233Shselasky /* 729306233Shselasky * On load removing any previous indication of internal error, 730306233Shselasky * device is up 731306233Shselasky */ 732306233Shselasky dev->state = MLX5_DEVICE_STATE_UP; 733306233Shselasky 734290650Shselasky err = mlx5_cmd_init(dev); 735290650Shselasky if (err) { 736290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Failed initializing command interface, aborting\n"); 737290650Shselasky goto err_unmap; 738290650Shselasky } 739290650Shselasky 740290650Shselasky err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); 741290650Shselasky if (err) { 742290650Shselasky device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI); 743290650Shselasky goto err_cmd_cleanup; 744290650Shselasky } 745290650Shselasky 746290650Shselasky mlx5_pagealloc_init(dev); 747290650Shselasky 748290650Shselasky err = mlx5_core_enable_hca(dev); 749290650Shselasky if (err) { 750290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""enable hca failed\n"); 751290650Shselasky goto err_pagealloc_cleanup; 752290650Shselasky } 753290650Shselasky 754290650Shselasky err = mlx5_core_set_issi(dev); 755290650Shselasky if (err) { 756290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""failed to set issi\n"); 757290650Shselasky goto err_disable_hca; 758290650Shselasky } 759290650Shselasky 760290650Shselasky err = mlx5_pagealloc_start(dev); 761290650Shselasky if (err) { 762290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_pagealloc_start failed\n"); 763290650Shselasky goto err_disable_hca; 764290650Shselasky } 765290650Shselasky 766290650Shselasky err = mlx5_satisfy_startup_pages(dev, 1); 767290650Shselasky if (err) { 768290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate boot pages\n"); 769290650Shselasky goto err_pagealloc_stop; 770290650Shselasky } 771290650Shselasky 772329209Shselasky err = set_hca_ctrl(dev); 773329209Shselasky if (err) { 774329209Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""set_hca_ctrl failed\n"); 775329209Shselasky goto reclaim_boot_pages; 776329209Shselasky } 777329209Shselasky 778306233Shselasky err = handle_hca_cap(dev); 779290650Shselasky if (err) { 780306233Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""handle_hca_cap failed\n"); 781290650Shselasky goto reclaim_boot_pages; 782290650Shselasky } 783290650Shselasky 784329209Shselasky err = handle_hca_cap_atomic(dev); 785290650Shselasky if (err) { 786329209Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""handle_hca_cap_atomic failed\n"); 787290650Shselasky goto reclaim_boot_pages; 788290650Shselasky } 789290650Shselasky 790290650Shselasky err = mlx5_satisfy_startup_pages(dev, 0); 791290650Shselasky if (err) { 792290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate init pages\n"); 793290650Shselasky goto reclaim_boot_pages; 794290650Shselasky } 795290650Shselasky 796290650Shselasky err = mlx5_cmd_init_hca(dev); 797290650Shselasky if (err) { 798290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""init hca failed\n"); 799290650Shselasky goto reclaim_boot_pages; 800290650Shselasky } 801290650Shselasky 802290650Shselasky mlx5_start_health_poll(dev); 803290650Shselasky 804290650Shselasky err = mlx5_query_hca_caps(dev); 805290650Shselasky if (err) { 806290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""query hca failed\n"); 807290650Shselasky goto err_stop_poll; 808290650Shselasky } 809290650Shselasky 810290650Shselasky err = mlx5_query_board_id(dev); 811290650Shselasky if (err) { 812290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""query board id failed\n"); 813290650Shselasky goto err_stop_poll; 814290650Shselasky } 815290650Shselasky 816290650Shselasky err = mlx5_enable_msix(dev); 817290650Shselasky if (err) { 818290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""enable msix failed\n"); 819290650Shselasky goto err_stop_poll; 820290650Shselasky } 821290650Shselasky 822290650Shselasky err = mlx5_eq_init(dev); 823290650Shselasky if (err) { 824290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""failed to initialize eq\n"); 825290650Shselasky goto disable_msix; 826290650Shselasky } 827290650Shselasky 828290650Shselasky err = mlx5_alloc_uuars(dev, &priv->uuari); 829290650Shselasky if (err) { 830290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Failed allocating uar, aborting\n"); 831290650Shselasky goto err_eq_cleanup; 832290650Shselasky } 833290650Shselasky 834290650Shselasky err = mlx5_start_eqs(dev); 835290650Shselasky if (err) { 836290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to start pages and async EQs\n"); 837290650Shselasky goto err_free_uar; 838290650Shselasky } 839290650Shselasky 840290650Shselasky err = alloc_comp_eqs(dev); 841290650Shselasky if (err) { 842290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to alloc completion EQs\n"); 843290650Shselasky goto err_stop_eqs; 844290650Shselasky } 845290650Shselasky 846290650Shselasky if (map_bf_area(dev)) 847290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to map blue flame area\n"); 848290650Shselasky 849290650Shselasky MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); 850290650Shselasky 851290650Shselasky mlx5_init_cq_table(dev); 852290650Shselasky mlx5_init_qp_table(dev); 853290650Shselasky mlx5_init_srq_table(dev); 854290650Shselasky mlx5_init_mr_table(dev); 855290650Shselasky 856329200Shselasky err = mlx5_init_fs(dev); 857329200Shselasky if (err) { 858329200Shselasky mlx5_core_err(dev, "flow steering init %d\n", err); 859329200Shselasky goto err_init_tables; 860329200Shselasky } 861329200Shselasky 862290650Shselasky return 0; 863290650Shselasky 864329200Shselaskyerr_init_tables: 865329200Shselasky mlx5_cleanup_mr_table(dev); 866329200Shselasky mlx5_cleanup_srq_table(dev); 867329200Shselasky mlx5_cleanup_qp_table(dev); 868329200Shselasky mlx5_cleanup_cq_table(dev); 869329200Shselasky unmap_bf_area(dev); 870329200Shselasky 871290650Shselaskyerr_stop_eqs: 872290650Shselasky mlx5_stop_eqs(dev); 873290650Shselasky 874290650Shselaskyerr_free_uar: 875290650Shselasky mlx5_free_uuars(dev, &priv->uuari); 876290650Shselasky 877290650Shselaskyerr_eq_cleanup: 878290650Shselasky mlx5_eq_cleanup(dev); 879290650Shselasky 880290650Shselaskydisable_msix: 881290650Shselasky mlx5_disable_msix(dev); 882290650Shselasky 883290650Shselaskyerr_stop_poll: 884290650Shselasky mlx5_stop_health_poll(dev); 885290650Shselasky if (mlx5_cmd_teardown_hca(dev)) { 886290650Shselasky device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n"); 887290650Shselasky return err; 888290650Shselasky } 889290650Shselasky 890290650Shselaskyreclaim_boot_pages: 891290650Shselasky mlx5_reclaim_startup_pages(dev); 892290650Shselasky 893290650Shselaskyerr_pagealloc_stop: 894290650Shselasky mlx5_pagealloc_stop(dev); 895290650Shselasky 896290650Shselaskyerr_disable_hca: 897290650Shselasky mlx5_core_disable_hca(dev); 898290650Shselasky 899290650Shselaskyerr_pagealloc_cleanup: 900290650Shselasky mlx5_pagealloc_cleanup(dev); 901290650Shselaskyerr_cmd_cleanup: 902290650Shselasky mlx5_cmd_cleanup(dev); 903290650Shselasky 904290650Shselaskyerr_unmap: 905290650Shselasky iounmap(dev->iseg); 906290650Shselasky 907290650Shselaskyerr_clr_master: 908290650Shselasky pci_clear_master(dev->pdev); 909290650Shselasky release_bar(dev->pdev); 910290650Shselasky 911290650Shselaskyerr_disable: 912290650Shselasky pci_disable_device(dev->pdev); 913290650Shselasky 914290650Shselaskyerr_dbg: 915306233Shselasky dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 916290650Shselasky return err; 917290650Shselasky} 918290650Shselasky 919290650Shselaskystatic void mlx5_dev_cleanup(struct mlx5_core_dev *dev) 920290650Shselasky{ 921290650Shselasky struct mlx5_priv *priv = &dev->priv; 922290650Shselasky 923329200Shselasky mlx5_cleanup_fs(dev); 924290650Shselasky mlx5_cleanup_mr_table(dev); 925290650Shselasky mlx5_cleanup_srq_table(dev); 926290650Shselasky mlx5_cleanup_qp_table(dev); 927290650Shselasky mlx5_cleanup_cq_table(dev); 928290650Shselasky unmap_bf_area(dev); 929322144Shselasky mlx5_wait_for_reclaim_vfs_pages(dev); 930290650Shselasky free_comp_eqs(dev); 931290650Shselasky mlx5_stop_eqs(dev); 932290650Shselasky mlx5_free_uuars(dev, &priv->uuari); 933290650Shselasky mlx5_eq_cleanup(dev); 934290650Shselasky mlx5_disable_msix(dev); 935290650Shselasky mlx5_stop_health_poll(dev); 936290650Shselasky if (mlx5_cmd_teardown_hca(dev)) { 937290650Shselasky device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n"); 938290650Shselasky return; 939290650Shselasky } 940290650Shselasky mlx5_pagealloc_stop(dev); 941290650Shselasky mlx5_reclaim_startup_pages(dev); 942290650Shselasky mlx5_core_disable_hca(dev); 943290650Shselasky mlx5_pagealloc_cleanup(dev); 944290650Shselasky mlx5_cmd_cleanup(dev); 945290650Shselasky iounmap(dev->iseg); 946290650Shselasky pci_clear_master(dev->pdev); 947290650Shselasky release_bar(dev->pdev); 948290650Shselasky pci_disable_device(dev->pdev); 949290650Shselasky} 950290650Shselasky 951290650Shselaskystatic void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 952290650Shselasky{ 953290650Shselasky struct mlx5_device_context *dev_ctx; 954290650Shselasky struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 955290650Shselasky 956290650Shselasky dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL); 957290650Shselasky 958290650Shselasky dev_ctx->intf = intf; 959290650Shselasky dev_ctx->context = intf->add(dev); 960290650Shselasky 961290650Shselasky if (dev_ctx->context) { 962290650Shselasky spin_lock_irq(&priv->ctx_lock); 963290650Shselasky list_add_tail(&dev_ctx->list, &priv->ctx_list); 964290650Shselasky spin_unlock_irq(&priv->ctx_lock); 965290650Shselasky } else { 966290650Shselasky kfree(dev_ctx); 967290650Shselasky } 968290650Shselasky} 969290650Shselasky 970290650Shselaskystatic void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 971290650Shselasky{ 972290650Shselasky struct mlx5_device_context *dev_ctx; 973290650Shselasky struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 974290650Shselasky 975290650Shselasky list_for_each_entry(dev_ctx, &priv->ctx_list, list) 976290650Shselasky if (dev_ctx->intf == intf) { 977290650Shselasky spin_lock_irq(&priv->ctx_lock); 978290650Shselasky list_del(&dev_ctx->list); 979290650Shselasky spin_unlock_irq(&priv->ctx_lock); 980290650Shselasky 981290650Shselasky intf->remove(dev, dev_ctx->context); 982290650Shselasky kfree(dev_ctx); 983290650Shselasky return; 984290650Shselasky } 985290650Shselasky} 986290650Shselaskystatic int mlx5_register_device(struct mlx5_core_dev *dev) 987290650Shselasky{ 988290650Shselasky struct mlx5_priv *priv = &dev->priv; 989290650Shselasky struct mlx5_interface *intf; 990290650Shselasky 991290650Shselasky mutex_lock(&intf_mutex); 992290650Shselasky list_add_tail(&priv->dev_list, &dev_list); 993290650Shselasky list_for_each_entry(intf, &intf_list, list) 994290650Shselasky mlx5_add_device(intf, priv); 995290650Shselasky mutex_unlock(&intf_mutex); 996290650Shselasky 997290650Shselasky return 0; 998290650Shselasky} 999290650Shselaskystatic void mlx5_unregister_device(struct mlx5_core_dev *dev) 1000290650Shselasky{ 1001290650Shselasky struct mlx5_priv *priv = &dev->priv; 1002290650Shselasky struct mlx5_interface *intf; 1003290650Shselasky 1004290650Shselasky mutex_lock(&intf_mutex); 1005290650Shselasky list_for_each_entry(intf, &intf_list, list) 1006290650Shselasky mlx5_remove_device(intf, priv); 1007290650Shselasky list_del(&priv->dev_list); 1008290650Shselasky mutex_unlock(&intf_mutex); 1009290650Shselasky} 1010290650Shselasky 1011290650Shselaskyint mlx5_register_interface(struct mlx5_interface *intf) 1012290650Shselasky{ 1013290650Shselasky struct mlx5_priv *priv; 1014290650Shselasky 1015290650Shselasky if (!intf->add || !intf->remove) 1016290650Shselasky return -EINVAL; 1017290650Shselasky 1018290650Shselasky mutex_lock(&intf_mutex); 1019290650Shselasky list_add_tail(&intf->list, &intf_list); 1020290650Shselasky list_for_each_entry(priv, &dev_list, dev_list) 1021290650Shselasky mlx5_add_device(intf, priv); 1022290650Shselasky mutex_unlock(&intf_mutex); 1023290650Shselasky 1024290650Shselasky return 0; 1025290650Shselasky} 1026290650ShselaskyEXPORT_SYMBOL(mlx5_register_interface); 1027290650Shselasky 1028290650Shselaskyvoid mlx5_unregister_interface(struct mlx5_interface *intf) 1029290650Shselasky{ 1030290650Shselasky struct mlx5_priv *priv; 1031290650Shselasky 1032290650Shselasky mutex_lock(&intf_mutex); 1033290650Shselasky list_for_each_entry(priv, &dev_list, dev_list) 1034290650Shselasky mlx5_remove_device(intf, priv); 1035290650Shselasky list_del(&intf->list); 1036290650Shselasky mutex_unlock(&intf_mutex); 1037290650Shselasky} 1038290650ShselaskyEXPORT_SYMBOL(mlx5_unregister_interface); 1039290650Shselasky 1040290650Shselaskyvoid *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol) 1041290650Shselasky{ 1042290650Shselasky struct mlx5_priv *priv = &mdev->priv; 1043290650Shselasky struct mlx5_device_context *dev_ctx; 1044290650Shselasky unsigned long flags; 1045290650Shselasky void *result = NULL; 1046290650Shselasky 1047290650Shselasky spin_lock_irqsave(&priv->ctx_lock, flags); 1048290650Shselasky 1049290650Shselasky list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list) 1050290650Shselasky if ((dev_ctx->intf->protocol == protocol) && 1051290650Shselasky dev_ctx->intf->get_dev) { 1052290650Shselasky result = dev_ctx->intf->get_dev(dev_ctx->context); 1053290650Shselasky break; 1054290650Shselasky } 1055290650Shselasky 1056290650Shselasky spin_unlock_irqrestore(&priv->ctx_lock, flags); 1057290650Shselasky 1058290650Shselasky return result; 1059290650Shselasky} 1060290650ShselaskyEXPORT_SYMBOL(mlx5_get_protocol_dev); 1061290650Shselasky 1062290650Shselaskystatic void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 1063290650Shselasky unsigned long param) 1064290650Shselasky{ 1065290650Shselasky struct mlx5_priv *priv = &dev->priv; 1066290650Shselasky struct mlx5_device_context *dev_ctx; 1067290650Shselasky unsigned long flags; 1068290650Shselasky 1069290650Shselasky spin_lock_irqsave(&priv->ctx_lock, flags); 1070290650Shselasky 1071290650Shselasky list_for_each_entry(dev_ctx, &priv->ctx_list, list) 1072290650Shselasky if (dev_ctx->intf->event) 1073290650Shselasky dev_ctx->intf->event(dev, dev_ctx->context, event, param); 1074290650Shselasky 1075290650Shselasky spin_unlock_irqrestore(&priv->ctx_lock, flags); 1076290650Shselasky} 1077290650Shselasky 1078290650Shselaskystruct mlx5_core_event_handler { 1079290650Shselasky void (*event)(struct mlx5_core_dev *dev, 1080290650Shselasky enum mlx5_dev_event event, 1081290650Shselasky void *data); 1082290650Shselasky}; 1083290650Shselasky 1084290650Shselasky 1085290650Shselaskystatic int init_one(struct pci_dev *pdev, 1086290650Shselasky const struct pci_device_id *id) 1087290650Shselasky{ 1088290650Shselasky struct mlx5_core_dev *dev; 1089290650Shselasky struct mlx5_priv *priv; 1090290650Shselasky int err; 1091290650Shselasky 1092290650Shselasky dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1093290650Shselasky priv = &dev->priv; 1094306233Shselasky if (id) 1095306233Shselasky priv->pci_dev_data = id->driver_data; 1096290650Shselasky 1097290650Shselasky if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profiles)) { 1098290650Shselasky printf("mlx5_core: WARN: ""selected profile out of range, selecting default (%d)\n", MLX5_DEFAULT_PROF); 1099290650Shselasky prof_sel = MLX5_DEFAULT_PROF; 1100290650Shselasky } 1101290650Shselasky dev->profile = &profiles[prof_sel]; 1102290650Shselasky dev->event = mlx5_core_event; 1103290650Shselasky 1104290650Shselasky INIT_LIST_HEAD(&priv->ctx_list); 1105290650Shselasky spin_lock_init(&priv->ctx_lock); 1106290650Shselasky err = mlx5_dev_init(dev, pdev); 1107290650Shselasky if (err) { 1108290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_dev_init failed %d\n", err); 1109290650Shselasky goto out; 1110290650Shselasky } 1111290650Shselasky 1112290650Shselasky err = mlx5_register_device(dev); 1113290650Shselasky if (err) { 1114290650Shselasky device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_register_device failed %d\n", err); 1115290650Shselasky goto out_init; 1116290650Shselasky } 1117290650Shselasky 1118290650Shselasky 1119290650Shselasky return 0; 1120290650Shselasky 1121290650Shselaskyout_init: 1122290650Shselasky mlx5_dev_cleanup(dev); 1123290650Shselaskyout: 1124290650Shselasky kfree(dev); 1125290650Shselasky return err; 1126290650Shselasky} 1127290650Shselasky 1128290650Shselaskystatic void remove_one(struct pci_dev *pdev) 1129290650Shselasky{ 1130290650Shselasky struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1131290650Shselasky 1132290650Shselasky mlx5_unregister_device(dev); 1133290650Shselasky mlx5_dev_cleanup(dev); 1134290650Shselasky kfree(dev); 1135290650Shselasky} 1136290650Shselasky 1137329211Shselaskystatic void shutdown_one(struct pci_dev *pdev) 1138329211Shselasky{ 1139329211Shselasky /* prevent device from accessing host memory after shutdown */ 1140329211Shselasky pci_clear_master(pdev); 1141329211Shselasky} 1142329211Shselasky 1143290650Shselaskystatic const struct pci_device_id mlx5_core_pci_table[] = { 1144290650Shselasky { PCI_VDEVICE(MELLANOX, 4113) }, /* Connect-IB */ 1145290650Shselasky { PCI_VDEVICE(MELLANOX, 4114) }, /* Connect-IB VF */ 1146290650Shselasky { PCI_VDEVICE(MELLANOX, 4115) }, /* ConnectX-4 */ 1147290650Shselasky { PCI_VDEVICE(MELLANOX, 4116) }, /* ConnectX-4 VF */ 1148290650Shselasky { PCI_VDEVICE(MELLANOX, 4117) }, /* ConnectX-4LX */ 1149290650Shselasky { PCI_VDEVICE(MELLANOX, 4118) }, /* ConnectX-4LX VF */ 1150306233Shselasky { PCI_VDEVICE(MELLANOX, 4119) }, /* ConnectX-5 */ 1151306233Shselasky { PCI_VDEVICE(MELLANOX, 4120) }, /* ConnectX-5 VF */ 1152290650Shselasky { PCI_VDEVICE(MELLANOX, 4121) }, 1153290650Shselasky { PCI_VDEVICE(MELLANOX, 4122) }, 1154290650Shselasky { PCI_VDEVICE(MELLANOX, 4123) }, 1155290650Shselasky { PCI_VDEVICE(MELLANOX, 4124) }, 1156290650Shselasky { PCI_VDEVICE(MELLANOX, 4125) }, 1157290650Shselasky { PCI_VDEVICE(MELLANOX, 4126) }, 1158290650Shselasky { PCI_VDEVICE(MELLANOX, 4127) }, 1159290650Shselasky { PCI_VDEVICE(MELLANOX, 4128) }, 1160290650Shselasky { PCI_VDEVICE(MELLANOX, 4129) }, 1161290650Shselasky { PCI_VDEVICE(MELLANOX, 4130) }, 1162290650Shselasky { PCI_VDEVICE(MELLANOX, 4131) }, 1163290650Shselasky { PCI_VDEVICE(MELLANOX, 4132) }, 1164290650Shselasky { PCI_VDEVICE(MELLANOX, 4133) }, 1165290650Shselasky { PCI_VDEVICE(MELLANOX, 4134) }, 1166290650Shselasky { PCI_VDEVICE(MELLANOX, 4135) }, 1167290650Shselasky { PCI_VDEVICE(MELLANOX, 4136) }, 1168290650Shselasky { PCI_VDEVICE(MELLANOX, 4137) }, 1169290650Shselasky { PCI_VDEVICE(MELLANOX, 4138) }, 1170290650Shselasky { PCI_VDEVICE(MELLANOX, 4139) }, 1171290650Shselasky { PCI_VDEVICE(MELLANOX, 4140) }, 1172290650Shselasky { PCI_VDEVICE(MELLANOX, 4141) }, 1173290650Shselasky { PCI_VDEVICE(MELLANOX, 4142) }, 1174290650Shselasky { PCI_VDEVICE(MELLANOX, 4143) }, 1175290650Shselasky { PCI_VDEVICE(MELLANOX, 4144) }, 1176290650Shselasky { 0, } 1177290650Shselasky}; 1178290650Shselasky 1179290650ShselaskyMODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 1180290650Shselasky 1181290650Shselaskystatic struct pci_driver mlx5_core_driver = { 1182290650Shselasky .name = DRIVER_NAME, 1183290650Shselasky .id_table = mlx5_core_pci_table, 1184329211Shselasky .shutdown = shutdown_one, 1185290650Shselasky .probe = init_one, 1186290650Shselasky .remove = remove_one 1187290650Shselasky}; 1188290650Shselasky 1189290650Shselaskystatic int __init init(void) 1190290650Shselasky{ 1191290650Shselasky int err; 1192290650Shselasky 1193290650Shselasky mlx5_core_wq = create_singlethread_workqueue("mlx5_core_wq"); 1194290650Shselasky if (!mlx5_core_wq) { 1195290650Shselasky err = -ENOMEM; 1196290650Shselasky goto err_debug; 1197290650Shselasky } 1198290650Shselasky mlx5_health_init(); 1199290650Shselasky 1200290650Shselasky err = pci_register_driver(&mlx5_core_driver); 1201290650Shselasky if (err) 1202290650Shselasky goto err_health; 1203290650Shselasky 1204290650Shselasky 1205290650Shselasky return 0; 1206290650Shselasky 1207290650Shselaskyerr_health: 1208290650Shselasky mlx5_health_cleanup(); 1209290650Shselasky destroy_workqueue(mlx5_core_wq); 1210290650Shselaskyerr_debug: 1211290650Shselasky return err; 1212290650Shselasky} 1213290650Shselasky 1214290650Shselaskystatic void __exit cleanup(void) 1215290650Shselasky{ 1216290650Shselasky pci_unregister_driver(&mlx5_core_driver); 1217290650Shselasky mlx5_health_cleanup(); 1218290650Shselasky destroy_workqueue(mlx5_core_wq); 1219290650Shselasky} 1220290650Shselasky 1221290650Shselaskymodule_init(init); 1222290650Shselaskymodule_exit(cleanup); 1223322148Shselasky 1224322148Shselaskyvoid mlx5_enter_error_state(struct mlx5_core_dev *dev) 1225322148Shselasky{ 1226322148Shselasky if (dev->state != MLX5_DEVICE_STATE_UP) 1227322148Shselasky return; 1228322148Shselasky 1229322148Shselasky dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1230322148Shselasky mlx5_trigger_cmd_completions(dev); 1231322148Shselasky} 1232322148ShselaskyEXPORT_SYMBOL(mlx5_enter_error_state); 1233