mlx5_ib_main.c revision 325599
1220176Sobrien/*- 2220176Sobrien * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3220176Sobrien * 4220176Sobrien * Redistribution and use in source and binary forms, with or without 5220176Sobrien * modification, are permitted provided that the following conditions 6220176Sobrien * are met: 7220176Sobrien * 1. Redistributions of source code must retain the above copyright 8220176Sobrien * notice, this list of conditions and the following disclaimer. 9220176Sobrien * 2. Redistributions in binary form must reproduce the above copyright 10220176Sobrien * notice, this list of conditions and the following disclaimer in the 11220176Sobrien * documentation and/or other materials provided with the distribution. 12220176Sobrien * 13220176Sobrien * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14220176Sobrien * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15220176Sobrien * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16220176Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17220176Sobrien * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18220176Sobrien * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19220176Sobrien * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20220176Sobrien * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21220176Sobrien * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22220176Sobrien * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23220176Sobrien * SUCH DAMAGE. 24220176Sobrien * 25220176Sobrien * $FreeBSD: stable/10/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c 325599 2017-11-09 17:02:20Z hselasky $ 26220176Sobrien */ 27220176Sobrien 28220176Sobrien#include <linux/errno.h> 29220176Sobrien#include <linux/pci.h> 30220176Sobrien#include <linux/dma-mapping.h> 31220176Sobrien#include <linux/slab.h> 32220176Sobrien#include <linux/io-mapping.h> 33220176Sobrien#include <linux/sched.h> 34220176Sobrien#include <linux/netdevice.h> 35220176Sobrien#include <linux/etherdevice.h> 36220176Sobrien#include <net/ipv6.h> 37220176Sobrien#include <linux/list.h> 38220176Sobrien#include <dev/mlx5/driver.h> 39220176Sobrien#include <dev/mlx5/vport.h> 40220176Sobrien#include <asm/pgtable.h> 41220176Sobrien#include <linux/fs.h> 42220176Sobrien#undef inode 43220176Sobrien 44220176Sobrien#include <rdma/ib_user_verbs.h> 45220176Sobrien#include <rdma/ib_smi.h> 46220176Sobrien#include <rdma/ib_umem.h> 47220176Sobrien#include "user.h" 48220176Sobrien#include "mlx5_ib.h" 49220176Sobrien 50220176Sobrien#include <sys/unistd.h> 51220176Sobrien 52220176Sobrien#define DRIVER_NAME "mlx5_ib" 53220176Sobrien#define DRIVER_VERSION "3.2-rc1" 54220176Sobrien#define DRIVER_RELDATE "May 2016" 55220176Sobrien 56220176Sobrien#undef MODULE_VERSION 57220176Sobrien#include <sys/module.h> 58220176Sobrien 59220176SobrienMODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 60220176SobrienMODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver"); 61220176SobrienMODULE_LICENSE("Dual BSD/GPL"); 62220176SobrienMODULE_DEPEND(mlx5ib, mlx5, 1, 1, 1); 63220176SobrienMODULE_DEPEND(mlx5ib, ibcore, 1, 1, 1); 64220176SobrienMODULE_VERSION(mlx5ib, 1); 65220176Sobrien 66220176Sobrienstatic int deprecated_prof_sel = 2; 67220176Sobrienmodule_param_named(prof_sel, deprecated_prof_sel, int, 0444); 68220176SobrienMODULE_PARM_DESC(prof_sel, "profile selector. Deprecated here. Moved to module mlx5_core"); 69220176Sobrien 70220176Sobrienenum { 71220176Sobrien MLX5_STANDARD_ATOMIC_SIZE = 0x8, 72220176Sobrien}; 73220176Sobrien 74220176Sobrienstruct workqueue_struct *mlx5_ib_wq; 75220176Sobrien 76220176Sobrienstatic char mlx5_version[] = 77220176Sobrien DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v" 78220176Sobrien DRIVER_VERSION " (" DRIVER_RELDATE ")\n"; 79220176Sobrien 80220176Sobrienstatic void get_atomic_caps(struct mlx5_ib_dev *dev, 81220176Sobrien struct ib_device_attr *props) 82220176Sobrien{ 83220176Sobrien int tmp; 84220176Sobrien u8 atomic_operations; 85220176Sobrien u8 atomic_size_qp; 86220176Sobrien u8 atomic_req_endianess; 87220176Sobrien 88220176Sobrien atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations); 89220176Sobrien atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp); 90220176Sobrien atomic_req_endianess = MLX5_CAP_ATOMIC(dev->mdev, 91220176Sobrien atomic_req_8B_endianess_mode) || 92220176Sobrien !mlx5_host_is_le(); 93220176Sobrien 94220176Sobrien tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD; 95220176Sobrien if (((atomic_operations & tmp) == tmp) 96220176Sobrien && (atomic_size_qp & 8)) { 97220176Sobrien if (atomic_req_endianess) { 98220176Sobrien props->atomic_cap = IB_ATOMIC_HCA; 99220176Sobrien } else { 100220176Sobrien props->atomic_cap = IB_ATOMIC_NONE; 101220176Sobrien } 102220176Sobrien } else { 103220176Sobrien props->atomic_cap = IB_ATOMIC_NONE; 104220176Sobrien } 105220176Sobrien 106220176Sobrien tmp = MLX5_ATOMIC_OPS_MASKED_CMP_SWAP | MLX5_ATOMIC_OPS_MASKED_FETCH_ADD; 107220176Sobrien if (((atomic_operations & tmp) == tmp) 108220176Sobrien &&(atomic_size_qp & 8)) { 109220176Sobrien if (atomic_req_endianess) 110220176Sobrien props->masked_atomic_cap = IB_ATOMIC_HCA; 111220176Sobrien else { 112220176Sobrien props->masked_atomic_cap = IB_ATOMIC_NONE; 113220176Sobrien } 114220176Sobrien } else { 115220176Sobrien props->masked_atomic_cap = IB_ATOMIC_NONE; 116220176Sobrien } 117220176Sobrien} 118220176Sobrien 119220176Sobrienstatic enum rdma_link_layer 120220176Sobrienmlx5_ib_port_link_layer(struct ib_device *device, u8 port_num) 121220176Sobrien{ 122220176Sobrien struct mlx5_ib_dev *dev = to_mdev(device); 123220176Sobrien 124220176Sobrien switch (MLX5_CAP_GEN(dev->mdev, port_type)) { 125220176Sobrien case MLX5_CAP_PORT_TYPE_IB: 126220176Sobrien return IB_LINK_LAYER_INFINIBAND; 127220176Sobrien case MLX5_CAP_PORT_TYPE_ETH: 128220176Sobrien return IB_LINK_LAYER_ETHERNET; 129220176Sobrien default: 130220176Sobrien return IB_LINK_LAYER_UNSPECIFIED; 131220176Sobrien } 132220176Sobrien} 133220176Sobrien 134220176Sobrienstatic int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev) 135220176Sobrien{ 136220176Sobrien return !dev->mdev->issi; 137220176Sobrien} 138220176Sobrien 139220176Sobrienenum { 140220176Sobrien MLX5_VPORT_ACCESS_METHOD_MAD, 141220176Sobrien MLX5_VPORT_ACCESS_METHOD_HCA, 142220176Sobrien MLX5_VPORT_ACCESS_METHOD_NIC, 143220176Sobrien}; 144220176Sobrien 145220176Sobrienstatic int mlx5_get_vport_access_method(struct ib_device *ibdev) 146220176Sobrien{ 147220176Sobrien if (mlx5_use_mad_ifc(to_mdev(ibdev))) 148220176Sobrien return MLX5_VPORT_ACCESS_METHOD_MAD; 149220176Sobrien 150220176Sobrien if (mlx5_ib_port_link_layer(ibdev, 1) == 151220176Sobrien IB_LINK_LAYER_ETHERNET) 152220176Sobrien return MLX5_VPORT_ACCESS_METHOD_NIC; 153220176Sobrien 154220176Sobrien return MLX5_VPORT_ACCESS_METHOD_HCA; 155220176Sobrien} 156220176Sobrien 157220176Sobrienstatic int mlx5_query_system_image_guid(struct ib_device *ibdev, 158220176Sobrien __be64 *sys_image_guid) 159220176Sobrien{ 160220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 161220176Sobrien struct mlx5_core_dev *mdev = dev->mdev; 162220176Sobrien u64 tmp; 163220176Sobrien int err; 164220176Sobrien 165220176Sobrien switch (mlx5_get_vport_access_method(ibdev)) { 166220176Sobrien case MLX5_VPORT_ACCESS_METHOD_MAD: 167220176Sobrien return mlx5_query_system_image_guid_mad_ifc(ibdev, 168220176Sobrien sys_image_guid); 169220176Sobrien 170220176Sobrien case MLX5_VPORT_ACCESS_METHOD_HCA: 171220176Sobrien err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp); 172220176Sobrien if (!err) 173220176Sobrien *sys_image_guid = cpu_to_be64(tmp); 174220176Sobrien return err; 175220176Sobrien 176220176Sobrien case MLX5_VPORT_ACCESS_METHOD_NIC: 177220176Sobrien err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp); 178220176Sobrien if (!err) 179220176Sobrien *sys_image_guid = cpu_to_be64(tmp); 180220176Sobrien return err; 181220176Sobrien 182220176Sobrien default: 183220176Sobrien return -EINVAL; 184220176Sobrien } 185220176Sobrien} 186220176Sobrien 187220176Sobrienstatic int mlx5_query_max_pkeys(struct ib_device *ibdev, 188220176Sobrien u16 *max_pkeys) 189220176Sobrien{ 190220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 191220176Sobrien struct mlx5_core_dev *mdev = dev->mdev; 192220176Sobrien 193220176Sobrien switch (mlx5_get_vport_access_method(ibdev)) { 194220176Sobrien case MLX5_VPORT_ACCESS_METHOD_MAD: 195220176Sobrien return mlx5_query_max_pkeys_mad_ifc(ibdev, max_pkeys); 196220176Sobrien 197220176Sobrien case MLX5_VPORT_ACCESS_METHOD_HCA: 198220176Sobrien case MLX5_VPORT_ACCESS_METHOD_NIC: 199220176Sobrien *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, 200220176Sobrien pkey_table_size)); 201220176Sobrien return 0; 202220176Sobrien 203220176Sobrien default: 204220176Sobrien return -EINVAL; 205220176Sobrien } 206220176Sobrien} 207220176Sobrien 208220176Sobrienstatic int mlx5_query_vendor_id(struct ib_device *ibdev, 209220176Sobrien u32 *vendor_id) 210220176Sobrien{ 211220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 212220176Sobrien 213220176Sobrien switch (mlx5_get_vport_access_method(ibdev)) { 214220176Sobrien case MLX5_VPORT_ACCESS_METHOD_MAD: 215220176Sobrien return mlx5_query_vendor_id_mad_ifc(ibdev, vendor_id); 216220176Sobrien 217220176Sobrien case MLX5_VPORT_ACCESS_METHOD_HCA: 218220176Sobrien case MLX5_VPORT_ACCESS_METHOD_NIC: 219220176Sobrien return mlx5_core_query_vendor_id(dev->mdev, vendor_id); 220220176Sobrien 221220176Sobrien default: 222220176Sobrien return -EINVAL; 223220176Sobrien } 224220176Sobrien} 225220176Sobrien 226220176Sobrienstatic int mlx5_query_node_guid(struct mlx5_ib_dev *dev, 227220176Sobrien __be64 *node_guid) 228220176Sobrien{ 229220176Sobrien u64 tmp; 230220176Sobrien int err; 231220176Sobrien 232220176Sobrien switch (mlx5_get_vport_access_method(&dev->ib_dev)) { 233220176Sobrien case MLX5_VPORT_ACCESS_METHOD_MAD: 234220176Sobrien return mlx5_query_node_guid_mad_ifc(dev, node_guid); 235220176Sobrien 236220176Sobrien case MLX5_VPORT_ACCESS_METHOD_HCA: 237220176Sobrien err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp); 238220176Sobrien if (!err) 239220176Sobrien *node_guid = cpu_to_be64(tmp); 240220176Sobrien return err; 241220176Sobrien 242220176Sobrien case MLX5_VPORT_ACCESS_METHOD_NIC: 243220176Sobrien err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp); 244220176Sobrien if (!err) 245220176Sobrien *node_guid = cpu_to_be64(tmp); 246220176Sobrien return err; 247220176Sobrien 248220176Sobrien default: 249220176Sobrien return -EINVAL; 250220176Sobrien } 251220176Sobrien} 252220176Sobrien 253220176Sobrienstruct mlx5_reg_node_desc { 254220176Sobrien u8 desc[64]; 255220176Sobrien}; 256220176Sobrien 257220176Sobrienstatic int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc) 258220176Sobrien{ 259220176Sobrien struct mlx5_reg_node_desc in; 260220176Sobrien 261220176Sobrien if (mlx5_use_mad_ifc(dev)) 262220176Sobrien return mlx5_query_node_desc_mad_ifc(dev, node_desc); 263220176Sobrien 264220176Sobrien memset(&in, 0, sizeof(in)); 265220176Sobrien 266220176Sobrien return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc, 267220176Sobrien sizeof(struct mlx5_reg_node_desc), 268220176Sobrien MLX5_REG_NODE_DESC, 0, 0); 269220176Sobrien} 270220176Sobrien 271220176Sobrienstatic int mlx5_ib_query_device(struct ib_device *ibdev, 272220176Sobrien struct ib_device_attr *props) 273220176Sobrien{ 274220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 275220176Sobrien struct mlx5_core_dev *mdev = dev->mdev; 276220176Sobrien int max_sq_desc; 277220176Sobrien int max_rq_sg; 278220176Sobrien int max_sq_sg; 279220176Sobrien int err; 280220176Sobrien 281220176Sobrien 282220176Sobrien memset(props, 0, sizeof(*props)); 283220176Sobrien 284220176Sobrien err = mlx5_query_system_image_guid(ibdev, 285220176Sobrien &props->sys_image_guid); 286220176Sobrien if (err) 287220176Sobrien return err; 288220176Sobrien 289220176Sobrien err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys); 290220176Sobrien if (err) 291220176Sobrien return err; 292220176Sobrien 293220176Sobrien err = mlx5_query_vendor_id(ibdev, &props->vendor_id); 294220176Sobrien if (err) 295220176Sobrien return err; 296220176Sobrien 297220176Sobrien props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) | 298220176Sobrien ((u64)fw_rev_min(dev->mdev) << 16) | 299220176Sobrien fw_rev_sub(dev->mdev); 300220176Sobrien props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT | 301220176Sobrien IB_DEVICE_PORT_ACTIVE_EVENT | 302220176Sobrien IB_DEVICE_SYS_IMAGE_GUID | 303220176Sobrien IB_DEVICE_RC_RNR_NAK_GEN; 304220176Sobrien 305220176Sobrien if (MLX5_CAP_GEN(mdev, pkv)) 306220176Sobrien props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR; 307220176Sobrien if (MLX5_CAP_GEN(mdev, qkv)) 308220176Sobrien props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR; 309220176Sobrien if (MLX5_CAP_GEN(mdev, apm)) 310220176Sobrien props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; 311220176Sobrien props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; 312220176Sobrien if (MLX5_CAP_GEN(mdev, xrc)) 313220176Sobrien props->device_cap_flags |= IB_DEVICE_XRC; 314220176Sobrien props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; 315220176Sobrien if (MLX5_CAP_GEN(mdev, block_lb_mc)) 316220176Sobrien props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK; 317220176Sobrien 318220176Sobrien props->vendor_part_id = mdev->pdev->device; 319220176Sobrien props->hw_ver = mdev->pdev->revision; 320220176Sobrien 321220176Sobrien props->max_mr_size = ~0ull; 322220176Sobrien props->page_size_cap = ~(u32)((1ull << MLX5_CAP_GEN(mdev, log_pg_sz)) -1); 323220176Sobrien props->max_qp = 1 << MLX5_CAP_GEN(mdev, log_max_qp); 324220176Sobrien props->max_qp_wr = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz); 325220176Sobrien max_rq_sg = MLX5_CAP_GEN(mdev, max_wqe_sz_rq) / 326220176Sobrien sizeof(struct mlx5_wqe_data_seg); 327220176Sobrien max_sq_desc = min((int)MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512); 328220176Sobrien max_sq_sg = (max_sq_desc - 329220176Sobrien sizeof(struct mlx5_wqe_ctrl_seg) - 330220176Sobrien sizeof(struct mlx5_wqe_raddr_seg)) / sizeof(struct mlx5_wqe_data_seg); 331220176Sobrien props->max_sge = min(max_rq_sg, max_sq_sg); 332220176Sobrien props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq); 333220176Sobrien props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1; 334220176Sobrien props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey); 335220176Sobrien props->max_pd = 1 << MLX5_CAP_GEN(mdev, log_max_pd); 336220176Sobrien props->max_qp_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp); 337220176Sobrien props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp); 338220176Sobrien props->max_srq = 1 << MLX5_CAP_GEN(mdev, log_max_srq); 339220176Sobrien props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1; 340220176Sobrien props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay); 341220176Sobrien props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; 342220176Sobrien props->max_srq_sge = max_rq_sg - 1; 343220176Sobrien props->max_fast_reg_page_list_len = (unsigned int)-1; 344220176Sobrien get_atomic_caps(dev, props); 345220176Sobrien props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg); 346220176Sobrien props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg); 347220176Sobrien props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 348220176Sobrien props->max_mcast_grp; 349220176Sobrien props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */ 350220176Sobrien props->max_ah = INT_MAX; 351220176Sobrien 352220176Sobrien return 0; 353220176Sobrien} 354220176Sobrien 355220176Sobrienenum mlx5_ib_width { 356220176Sobrien MLX5_IB_WIDTH_1X = 1 << 0, 357220176Sobrien MLX5_IB_WIDTH_2X = 1 << 1, 358220176Sobrien MLX5_IB_WIDTH_4X = 1 << 2, 359220176Sobrien MLX5_IB_WIDTH_8X = 1 << 3, 360220176Sobrien MLX5_IB_WIDTH_12X = 1 << 4 361220176Sobrien}; 362220176Sobrien 363220176Sobrienstatic int translate_active_width(struct ib_device *ibdev, u8 active_width, 364220176Sobrien u8 *ib_width) 365220176Sobrien{ 366220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 367220176Sobrien int err = 0; 368220176Sobrien 369220176Sobrien if (active_width & MLX5_IB_WIDTH_1X) { 370220176Sobrien *ib_width = IB_WIDTH_1X; 371220176Sobrien } else if (active_width & MLX5_IB_WIDTH_2X) { 372220176Sobrien mlx5_ib_warn(dev, "active_width %d is not supported by IB spec\n", 373220176Sobrien (int)active_width); 374220176Sobrien err = -EINVAL; 375220176Sobrien } else if (active_width & MLX5_IB_WIDTH_4X) { 376220176Sobrien *ib_width = IB_WIDTH_4X; 377220176Sobrien } else if (active_width & MLX5_IB_WIDTH_8X) { 378220176Sobrien *ib_width = IB_WIDTH_8X; 379220176Sobrien } else if (active_width & MLX5_IB_WIDTH_12X) { 380220176Sobrien *ib_width = IB_WIDTH_12X; 381220176Sobrien } else { 382220176Sobrien mlx5_ib_dbg(dev, "Invalid active_width %d\n", 383220176Sobrien (int)active_width); 384220176Sobrien err = -EINVAL; 385220176Sobrien } 386220176Sobrien 387220176Sobrien return err; 388220176Sobrien} 389220176Sobrien 390220176Sobrien/* 391220176Sobrien * TODO: Move to IB core 392220176Sobrien */ 393220176Sobrienenum ib_max_vl_num { 394220176Sobrien __IB_MAX_VL_0 = 1, 395220176Sobrien __IB_MAX_VL_0_1 = 2, 396220176Sobrien __IB_MAX_VL_0_3 = 3, 397220176Sobrien __IB_MAX_VL_0_7 = 4, 398220176Sobrien __IB_MAX_VL_0_14 = 5, 399220176Sobrien}; 400220176Sobrien 401220176Sobrienenum mlx5_vl_hw_cap { 402220176Sobrien MLX5_VL_HW_0 = 1, 403220176Sobrien MLX5_VL_HW_0_1 = 2, 404220176Sobrien MLX5_VL_HW_0_2 = 3, 405220176Sobrien MLX5_VL_HW_0_3 = 4, 406220176Sobrien MLX5_VL_HW_0_4 = 5, 407220176Sobrien MLX5_VL_HW_0_5 = 6, 408220176Sobrien MLX5_VL_HW_0_6 = 7, 409220176Sobrien MLX5_VL_HW_0_7 = 8, 410220176Sobrien MLX5_VL_HW_0_14 = 15 411220176Sobrien}; 412220176Sobrien 413220176Sobrienstatic int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap, 414220176Sobrien u8 *max_vl_num) 415220176Sobrien{ 416220176Sobrien switch (vl_hw_cap) { 417220176Sobrien case MLX5_VL_HW_0: 418220176Sobrien *max_vl_num = __IB_MAX_VL_0; 419220176Sobrien break; 420220176Sobrien case MLX5_VL_HW_0_1: 421220176Sobrien *max_vl_num = __IB_MAX_VL_0_1; 422220176Sobrien break; 423220176Sobrien case MLX5_VL_HW_0_3: 424220176Sobrien *max_vl_num = __IB_MAX_VL_0_3; 425220176Sobrien break; 426220176Sobrien case MLX5_VL_HW_0_7: 427220176Sobrien *max_vl_num = __IB_MAX_VL_0_7; 428220176Sobrien break; 429220176Sobrien case MLX5_VL_HW_0_14: 430220176Sobrien *max_vl_num = __IB_MAX_VL_0_14; 431220176Sobrien break; 432220176Sobrien 433220176Sobrien default: 434220176Sobrien return -EINVAL; 435220176Sobrien } 436220176Sobrien 437220176Sobrien return 0; 438220176Sobrien} 439220176Sobrien 440220176Sobrienstatic int mlx5_query_port_ib(struct ib_device *ibdev, u8 port, 441220176Sobrien struct ib_port_attr *props) 442220176Sobrien{ 443220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 444220176Sobrien struct mlx5_core_dev *mdev = dev->mdev; 445220176Sobrien u32 *rep; 446220176Sobrien int outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out); 447220176Sobrien struct mlx5_ptys_reg *ptys; 448220176Sobrien struct mlx5_pmtu_reg *pmtu; 449220176Sobrien struct mlx5_pvlc_reg pvlc; 450220176Sobrien void *ctx; 451220176Sobrien int err; 452220176Sobrien 453220176Sobrien rep = mlx5_vzalloc(outlen); 454220176Sobrien ptys = kzalloc(sizeof(*ptys), GFP_KERNEL); 455220176Sobrien pmtu = kzalloc(sizeof(*pmtu), GFP_KERNEL); 456220176Sobrien if (!rep || !ptys || !pmtu) { 457220176Sobrien err = -ENOMEM; 458220176Sobrien goto out; 459220176Sobrien } 460220176Sobrien 461220176Sobrien memset(props, 0, sizeof(*props)); 462220176Sobrien 463220176Sobrien /* what if I am pf with dual port */ 464220176Sobrien err = mlx5_query_hca_vport_context(mdev, port, 0, rep, outlen); 465220176Sobrien if (err) 466220176Sobrien goto out; 467220176Sobrien 468220176Sobrien ctx = MLX5_ADDR_OF(query_hca_vport_context_out, rep, hca_vport_context); 469220176Sobrien 470220176Sobrien props->lid = MLX5_GET(hca_vport_context, ctx, lid); 471220176Sobrien props->lmc = MLX5_GET(hca_vport_context, ctx, lmc); 472220176Sobrien props->sm_lid = MLX5_GET(hca_vport_context, ctx, sm_lid); 473220176Sobrien props->sm_sl = MLX5_GET(hca_vport_context, ctx, sm_sl); 474220176Sobrien props->state = MLX5_GET(hca_vport_context, ctx, vport_state); 475220176Sobrien props->phys_state = MLX5_GET(hca_vport_context, ctx, 476220176Sobrien port_physical_state); 477220176Sobrien props->port_cap_flags = MLX5_GET(hca_vport_context, ctx, cap_mask1); 478220176Sobrien props->gid_tbl_len = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size)); 479220176Sobrien props->max_msg_sz = 1 << MLX5_CAP_GEN(mdev, log_max_msg); 480220176Sobrien props->pkey_tbl_len = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size)); 481220176Sobrien props->bad_pkey_cntr = MLX5_GET(hca_vport_context, ctx, 482220176Sobrien pkey_violation_counter); 483220176Sobrien props->qkey_viol_cntr = MLX5_GET(hca_vport_context, ctx, 484220176Sobrien qkey_violation_counter); 485220176Sobrien props->subnet_timeout = MLX5_GET(hca_vport_context, ctx, 486220176Sobrien subnet_timeout); 487220176Sobrien props->init_type_reply = MLX5_GET(hca_vport_context, ctx, 488220176Sobrien init_type_reply); 489220176Sobrien 490220176Sobrien ptys->proto_mask |= MLX5_PTYS_IB; 491220176Sobrien ptys->local_port = port; 492220176Sobrien err = mlx5_core_access_ptys(mdev, ptys, 0); 493220176Sobrien if (err) 494220176Sobrien goto out; 495220176Sobrien 496220176Sobrien err = translate_active_width(ibdev, ptys->ib_link_width_oper, 497220176Sobrien &props->active_width); 498220176Sobrien if (err) 499220176Sobrien goto out; 500220176Sobrien 501220176Sobrien props->active_speed = (u8)ptys->ib_proto_oper; 502220176Sobrien 503220176Sobrien pmtu->local_port = port; 504220176Sobrien err = mlx5_core_access_pmtu(mdev, pmtu, 0); 505220176Sobrien if (err) 506220176Sobrien goto out; 507220176Sobrien 508220176Sobrien props->max_mtu = pmtu->max_mtu; 509220176Sobrien props->active_mtu = pmtu->oper_mtu; 510220176Sobrien 511220176Sobrien memset(&pvlc, 0, sizeof(pvlc)); 512220176Sobrien pvlc.local_port = port; 513220176Sobrien err = mlx5_core_access_pvlc(mdev, &pvlc, 0); 514220176Sobrien if (err) 515220176Sobrien goto out; 516220176Sobrien 517220176Sobrien err = translate_max_vl_num(ibdev, pvlc.vl_hw_cap, 518220176Sobrien &props->max_vl_num); 519220176Sobrienout: 520220176Sobrien kvfree(rep); 521220176Sobrien kfree(ptys); 522220176Sobrien kfree(pmtu); 523220176Sobrien return err; 524220176Sobrien} 525220176Sobrien 526220176Sobrienint mlx5_ib_query_port(struct ib_device *ibdev, u8 port, 527220176Sobrien struct ib_port_attr *props) 528220176Sobrien{ 529220176Sobrien switch (mlx5_get_vport_access_method(ibdev)) { 530220176Sobrien case MLX5_VPORT_ACCESS_METHOD_MAD: 531220176Sobrien return mlx5_query_port_mad_ifc(ibdev, port, props); 532220176Sobrien 533220176Sobrien case MLX5_VPORT_ACCESS_METHOD_HCA: 534220176Sobrien return mlx5_query_port_ib(ibdev, port, props); 535220176Sobrien 536220176Sobrien case MLX5_VPORT_ACCESS_METHOD_NIC: 537220176Sobrien return mlx5_query_port_roce(ibdev, port, props); 538220176Sobrien 539220176Sobrien default: 540220176Sobrien return -EINVAL; 541220176Sobrien } 542220176Sobrien} 543220176Sobrien 544220176Sobrienstatic void 545220176Sobrienmlx5_addrconf_ifid_eui48(u8 *eui, u16 vlan_id, struct net_device *dev) 546220176Sobrien{ 547220176Sobrien if (dev->if_addrlen != ETH_ALEN) 548220176Sobrien return; 549220176Sobrien 550220176Sobrien memcpy(eui, IF_LLADDR(dev), 3); 551220176Sobrien memcpy(eui + 5, IF_LLADDR(dev) + 3, 3); 552220176Sobrien 553220176Sobrien if (vlan_id < 0x1000) { 554220176Sobrien eui[3] = vlan_id >> 8; 555220176Sobrien eui[4] = vlan_id & 0xff; 556220176Sobrien } else { 557220176Sobrien eui[3] = 0xFF; 558220176Sobrien eui[4] = 0xFE; 559220176Sobrien } 560220176Sobrien eui[0] ^= 2; 561220176Sobrien} 562220176Sobrien 563220176Sobrienstatic void 564220176Sobrienmlx5_make_default_gid(struct net_device *dev, union ib_gid *gid) 565220176Sobrien{ 566220176Sobrien gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); 567220176Sobrien mlx5_addrconf_ifid_eui48(&gid->raw[8], 0xFFFF, dev); 568220176Sobrien} 569220176Sobrien 570220176Sobrienstatic void 571220176Sobrienmlx5_ib_roce_port_update(void *arg) 572220176Sobrien{ 573220176Sobrien struct mlx5_ib_port *port = (struct mlx5_ib_port *)arg; 574220176Sobrien struct mlx5_ib_dev *dev = port->dev; 575220176Sobrien struct mlx5_core_dev *mdev = dev->mdev; 576220176Sobrien struct net_device *xdev[MLX5_IB_GID_MAX]; 577220176Sobrien struct net_device *idev; 578220176Sobrien struct net_device *ndev; 579220176Sobrien union ib_gid gid_temp; 580220176Sobrien 581220176Sobrien while (port->port_gone == 0) { 582220176Sobrien int update = 0; 583220176Sobrien int gid_index = 0; 584220176Sobrien int j; 585220176Sobrien int error; 586220176Sobrien 587220176Sobrien ndev = mlx5_get_protocol_dev(mdev, MLX5_INTERFACE_PROTOCOL_ETH); 588220176Sobrien if (ndev == NULL) { 589220176Sobrien pause("W", hz); 590220176Sobrien continue; 591220176Sobrien } 592220176Sobrien 593220176Sobrien CURVNET_SET_QUIET(ndev->if_vnet); 594220176Sobrien 595220176Sobrien memset(&gid_temp, 0, sizeof(gid_temp)); 596220176Sobrien mlx5_make_default_gid(ndev, &gid_temp); 597220176Sobrien if (bcmp(&gid_temp, &port->gid_table[gid_index], sizeof(gid_temp))) { 598220176Sobrien port->gid_table[gid_index] = gid_temp; 599220176Sobrien update = 1; 600220176Sobrien } 601220176Sobrien xdev[gid_index] = ndev; 602220176Sobrien gid_index++; 603220176Sobrien 604220176Sobrien IFNET_RLOCK(); 605220176Sobrien TAILQ_FOREACH(idev, &V_ifnet, if_link) { 606220176Sobrien if (idev == ndev) 607220176Sobrien break; 608220176Sobrien } 609220176Sobrien if (idev != NULL) { 610220176Sobrien TAILQ_FOREACH(idev, &V_ifnet, if_link) { 611220176Sobrien u16 vid; 612220176Sobrien 613220176Sobrien if (idev != ndev) { 614220176Sobrien if (idev->if_type != IFT_L2VLAN) 615220176Sobrien continue; 616220176Sobrien if (ndev != rdma_vlan_dev_real_dev(idev)) 617220176Sobrien continue; 618220176Sobrien } 619220176Sobrien 620220176Sobrien /* setup valid MAC-based GID */ 621220176Sobrien memset(&gid_temp, 0, sizeof(gid_temp)); 622220176Sobrien gid_temp.global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); 623220176Sobrien vid = rdma_vlan_dev_vlan_id(idev); 624220176Sobrien mlx5_addrconf_ifid_eui48(&gid_temp.raw[8], vid, idev); 625220176Sobrien 626220176Sobrien /* check for existing entry */ 627220176Sobrien for (j = 0; j != gid_index; j++) { 628220176Sobrien if (bcmp(&gid_temp, &port->gid_table[j], sizeof(gid_temp)) == 0) 629220176Sobrien break; 630220176Sobrien } 631220176Sobrien 632220176Sobrien /* check if new entry should be added */ 633220176Sobrien if (j == gid_index && gid_index < MLX5_IB_GID_MAX) { 634220176Sobrien if (bcmp(&gid_temp, &port->gid_table[gid_index], sizeof(gid_temp))) { 635220176Sobrien port->gid_table[gid_index] = gid_temp; 636220176Sobrien update = 1; 637220176Sobrien } 638220176Sobrien xdev[gid_index] = idev; 639220176Sobrien gid_index++; 640220176Sobrien } 641220176Sobrien } 642220176Sobrien } 643220176Sobrien IFNET_RUNLOCK(); 644220176Sobrien CURVNET_RESTORE(); 645220176Sobrien 646220176Sobrien if (update != 0 && 647220176Sobrien mlx5_ib_port_link_layer(&dev->ib_dev, 1) == IB_LINK_LAYER_ETHERNET) { 648220176Sobrien struct ib_event event = { 649220176Sobrien .device = &dev->ib_dev, 650220176Sobrien .element.port_num = port->port_num + 1, 651220176Sobrien .event = IB_EVENT_GID_CHANGE, 652220176Sobrien }; 653220176Sobrien 654220176Sobrien /* add new entries, if any */ 655220176Sobrien for (j = 0; j != gid_index; j++) { 656220176Sobrien error = modify_gid_roce(&dev->ib_dev, port->port_num, j, 657220176Sobrien port->gid_table + j, xdev[j]); 658220176Sobrien if (error != 0) 659220176Sobrien printf("mlx5_ib: Failed to update ROCE GID table: %d\n", error); 660220176Sobrien } 661220176Sobrien memset(&gid_temp, 0, sizeof(gid_temp)); 662220176Sobrien 663220176Sobrien /* clear old entries, if any */ 664220176Sobrien for (; j != MLX5_IB_GID_MAX; j++) { 665220176Sobrien if (bcmp(&gid_temp, port->gid_table + j, sizeof(gid_temp)) == 0) 666220176Sobrien continue; 667220176Sobrien port->gid_table[j] = gid_temp; 668220176Sobrien (void) modify_gid_roce(&dev->ib_dev, port->port_num, j, 669220176Sobrien port->gid_table + j, ndev); 670220176Sobrien } 671220176Sobrien 672220176Sobrien /* make sure ibcore gets updated */ 673220176Sobrien ib_dispatch_event(&event); 674220176Sobrien } 675220176Sobrien pause("W", hz); 676220176Sobrien } 677220176Sobrien do { 678220176Sobrien struct ib_event event = { 679220176Sobrien .device = &dev->ib_dev, 680220176Sobrien .element.port_num = port->port_num + 1, 681220176Sobrien .event = IB_EVENT_GID_CHANGE, 682220176Sobrien }; 683220176Sobrien /* make sure ibcore gets updated */ 684220176Sobrien ib_dispatch_event(&event); 685220176Sobrien 686220176Sobrien /* wait a bit */ 687220176Sobrien pause("W", hz); 688220176Sobrien } while (0); 689220176Sobrien port->port_gone = 2; 690220176Sobrien kthread_exit(); 691220176Sobrien} 692220176Sobrien 693220176Sobrienstatic int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index, 694220176Sobrien union ib_gid *gid) 695220176Sobrien{ 696220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 697220176Sobrien struct mlx5_core_dev *mdev = dev->mdev; 698220176Sobrien 699220176Sobrien switch (mlx5_get_vport_access_method(ibdev)) { 700220176Sobrien case MLX5_VPORT_ACCESS_METHOD_MAD: 701220176Sobrien return mlx5_query_gids_mad_ifc(ibdev, port, index, gid); 702220176Sobrien 703220176Sobrien case MLX5_VPORT_ACCESS_METHOD_HCA: 704220176Sobrien return mlx5_query_hca_vport_gid(mdev, port, 0, index, gid); 705220176Sobrien 706220176Sobrien case MLX5_VPORT_ACCESS_METHOD_NIC: 707220176Sobrien if (port == 0 || port > MLX5_CAP_GEN(mdev, num_ports) || 708220176Sobrien index < 0 || index >= MLX5_IB_GID_MAX || 709220176Sobrien dev->port[port - 1].port_gone != 0) 710220176Sobrien memset(gid, 0, sizeof(*gid)); 711220176Sobrien else 712220176Sobrien *gid = dev->port[port - 1].gid_table[index]; 713220176Sobrien return 0; 714220176Sobrien 715220176Sobrien default: 716220176Sobrien return -EINVAL; 717220176Sobrien } 718220176Sobrien} 719220176Sobrien 720220176Sobrienstatic int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, 721220176Sobrien u16 *pkey) 722220176Sobrien{ 723220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 724220176Sobrien struct mlx5_core_dev *mdev = dev->mdev; 725220176Sobrien 726220176Sobrien switch (mlx5_get_vport_access_method(ibdev)) { 727220176Sobrien case MLX5_VPORT_ACCESS_METHOD_MAD: 728220176Sobrien return mlx5_query_pkey_mad_ifc(ibdev, port, index, pkey); 729220176Sobrien 730220176Sobrien case MLX5_VPORT_ACCESS_METHOD_HCA: 731220176Sobrien case MLX5_VPORT_ACCESS_METHOD_NIC: 732220176Sobrien return mlx5_query_hca_vport_pkey(mdev, 0, port, 0, index, 733220176Sobrien pkey); 734220176Sobrien 735220176Sobrien default: 736220176Sobrien return -EINVAL; 737220176Sobrien } 738220176Sobrien} 739220176Sobrien 740220176Sobrienstatic int mlx5_ib_modify_device(struct ib_device *ibdev, int mask, 741220176Sobrien struct ib_device_modify *props) 742220176Sobrien{ 743220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 744220176Sobrien struct mlx5_reg_node_desc in; 745220176Sobrien struct mlx5_reg_node_desc out; 746220176Sobrien int err; 747220176Sobrien 748220176Sobrien if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) 749220176Sobrien return -EOPNOTSUPP; 750220176Sobrien 751220176Sobrien if (!(mask & IB_DEVICE_MODIFY_NODE_DESC)) 752220176Sobrien return 0; 753220176Sobrien 754220176Sobrien /* 755220176Sobrien * If possible, pass node desc to FW, so it can generate 756220176Sobrien * a 144 trap. If cmd fails, just ignore. 757220176Sobrien */ 758220176Sobrien memcpy(&in, props->node_desc, 64); 759220176Sobrien err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out, 760220176Sobrien sizeof(out), MLX5_REG_NODE_DESC, 0, 1); 761220176Sobrien if (err) 762220176Sobrien return err; 763220176Sobrien 764220176Sobrien memcpy(ibdev->node_desc, props->node_desc, 64); 765220176Sobrien 766220176Sobrien return err; 767220176Sobrien} 768220176Sobrien 769220176Sobrienstatic int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, 770220176Sobrien struct ib_port_modify *props) 771220176Sobrien{ 772220176Sobrien u8 is_eth = (mlx5_ib_port_link_layer(ibdev, port) == 773220176Sobrien IB_LINK_LAYER_ETHERNET); 774220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 775220176Sobrien struct ib_port_attr attr; 776220176Sobrien u32 tmp; 777220176Sobrien int err; 778220176Sobrien 779220176Sobrien /* return OK if this is RoCE. CM calls ib_modify_port() regardless 780220176Sobrien * of whether port link layer is ETH or IB. For ETH ports, qkey 781220176Sobrien * violations and port capabilities are not valid. 782220176Sobrien */ 783220176Sobrien if (is_eth) 784220176Sobrien return 0; 785220176Sobrien 786220176Sobrien mutex_lock(&dev->cap_mask_mutex); 787220176Sobrien 788220176Sobrien err = mlx5_ib_query_port(ibdev, port, &attr); 789220176Sobrien if (err) 790220176Sobrien goto out; 791220176Sobrien 792220176Sobrien tmp = (attr.port_cap_flags | props->set_port_cap_mask) & 793220176Sobrien ~props->clr_port_cap_mask; 794220176Sobrien 795220176Sobrien err = mlx5_set_port_caps(dev->mdev, port, tmp); 796220176Sobrien 797220176Sobrienout: 798220176Sobrien mutex_unlock(&dev->cap_mask_mutex); 799220176Sobrien return err; 800220176Sobrien} 801220176Sobrien 802220176Sobrienenum mlx5_cap_flags { 803220176Sobrien MLX5_CAP_COMPACT_AV = 1 << 0, 804220176Sobrien}; 805220176Sobrien 806220176Sobrienstatic void set_mlx5_flags(u32 *flags, struct mlx5_core_dev *dev) 807220176Sobrien{ 808220176Sobrien *flags |= MLX5_CAP_GEN(dev, compact_address_vector) ? 809220176Sobrien MLX5_CAP_COMPACT_AV : 0; 810220176Sobrien} 811220176Sobrien 812220176Sobrienstatic struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev, 813220176Sobrien struct ib_udata *udata) 814220176Sobrien{ 815220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 816220176Sobrien struct mlx5_ib_alloc_ucontext_req_v2 req; 817220176Sobrien struct mlx5_ib_alloc_ucontext_resp resp; 818220176Sobrien struct mlx5_ib_ucontext *context; 819220176Sobrien struct mlx5_uuar_info *uuari; 820220176Sobrien struct mlx5_uar *uars; 821220176Sobrien int gross_uuars; 822220176Sobrien int num_uars; 823220176Sobrien int ver; 824220176Sobrien int uuarn; 825220176Sobrien int err; 826220176Sobrien int i; 827220176Sobrien size_t reqlen; 828220176Sobrien 829220176Sobrien if (!dev->ib_active) 830220176Sobrien return ERR_PTR(-EAGAIN); 831220176Sobrien 832220176Sobrien memset(&req, 0, sizeof(req)); 833220176Sobrien memset(&resp, 0, sizeof(resp)); 834220176Sobrien 835220176Sobrien reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr); 836220176Sobrien if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req)) 837220176Sobrien ver = 0; 838220176Sobrien else if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req_v2)) 839220176Sobrien ver = 2; 840220176Sobrien else { 841220176Sobrien mlx5_ib_err(dev, "request malformed, reqlen: %ld\n", (long)reqlen); 842220176Sobrien return ERR_PTR(-EINVAL); 843220176Sobrien } 844220176Sobrien 845220176Sobrien err = ib_copy_from_udata(&req, udata, reqlen); 846220176Sobrien if (err) { 847220176Sobrien mlx5_ib_err(dev, "copy failed\n"); 848220176Sobrien return ERR_PTR(err); 849220176Sobrien } 850220176Sobrien 851220176Sobrien if (req.reserved) { 852220176Sobrien mlx5_ib_err(dev, "request corrupted\n"); 853220176Sobrien return ERR_PTR(-EINVAL); 854220176Sobrien } 855220176Sobrien 856220176Sobrien if (req.total_num_uuars == 0 || req.total_num_uuars > MLX5_MAX_UUARS) { 857220176Sobrien mlx5_ib_warn(dev, "wrong num_uuars: %d\n", req.total_num_uuars); 858220176Sobrien return ERR_PTR(-ENOMEM); 859220176Sobrien } 860220176Sobrien 861220176Sobrien req.total_num_uuars = ALIGN(req.total_num_uuars, 862220176Sobrien MLX5_NON_FP_BF_REGS_PER_PAGE); 863220176Sobrien if (req.num_low_latency_uuars > req.total_num_uuars - 1) { 864220176Sobrien mlx5_ib_warn(dev, "wrong num_low_latency_uuars: %d ( > %d)\n", 865220176Sobrien req.total_num_uuars, req.total_num_uuars); 866220176Sobrien return ERR_PTR(-EINVAL); 867220176Sobrien } 868220176Sobrien 869220176Sobrien num_uars = req.total_num_uuars / MLX5_NON_FP_BF_REGS_PER_PAGE; 870220176Sobrien gross_uuars = num_uars * MLX5_BF_REGS_PER_PAGE; 871220176Sobrien resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp); 872220176Sobrien if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf)) 873220176Sobrien resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size); 874220176Sobrien resp.cache_line_size = L1_CACHE_BYTES; 875220176Sobrien resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq); 876220176Sobrien resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq); 877220176Sobrien resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz); 878220176Sobrien resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz); 879220176Sobrien resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); 880220176Sobrien set_mlx5_flags(&resp.flags, dev->mdev); 881220176Sobrien 882220176Sobrien if (offsetof(struct mlx5_ib_alloc_ucontext_resp, max_desc_sz_sq_dc) < udata->outlen) 883220176Sobrien resp.max_desc_sz_sq_dc = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq_dc); 884220176Sobrien 885220176Sobrien if (offsetof(struct mlx5_ib_alloc_ucontext_resp, atomic_arg_sizes_dc) < udata->outlen) 886220176Sobrien resp.atomic_arg_sizes_dc = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc); 887220176Sobrien 888220176Sobrien context = kzalloc(sizeof(*context), GFP_KERNEL); 889220176Sobrien if (!context) 890220176Sobrien return ERR_PTR(-ENOMEM); 891220176Sobrien 892220176Sobrien uuari = &context->uuari; 893220176Sobrien mutex_init(&uuari->lock); 894220176Sobrien uars = kcalloc(num_uars, sizeof(*uars), GFP_KERNEL); 895220176Sobrien if (!uars) { 896220176Sobrien err = -ENOMEM; 897220176Sobrien goto out_ctx; 898220176Sobrien } 899220176Sobrien 900220176Sobrien uuari->bitmap = kcalloc(BITS_TO_LONGS(gross_uuars), 901220176Sobrien sizeof(*uuari->bitmap), 902220176Sobrien GFP_KERNEL); 903220176Sobrien if (!uuari->bitmap) { 904220176Sobrien err = -ENOMEM; 905220176Sobrien goto out_uar_ctx; 906220176Sobrien } 907220176Sobrien /* 908220176Sobrien * clear all fast path uuars 909220176Sobrien */ 910220176Sobrien for (i = 0; i < gross_uuars; i++) { 911220176Sobrien uuarn = i & 3; 912220176Sobrien if (uuarn == 2 || uuarn == 3) 913220176Sobrien set_bit(i, uuari->bitmap); 914220176Sobrien } 915220176Sobrien 916220176Sobrien uuari->count = kcalloc(gross_uuars, sizeof(*uuari->count), GFP_KERNEL); 917220176Sobrien if (!uuari->count) { 918220176Sobrien err = -ENOMEM; 919220176Sobrien goto out_bitmap; 920220176Sobrien } 921220176Sobrien 922220176Sobrien for (i = 0; i < num_uars; i++) { 923220176Sobrien err = mlx5_cmd_alloc_uar(dev->mdev, &uars[i].index); 924220176Sobrien if (err) { 925220176Sobrien mlx5_ib_err(dev, "uar alloc failed at %d\n", i); 926220176Sobrien goto out_uars; 927220176Sobrien } 928220176Sobrien } 929220176Sobrien for (i = 0; i < MLX5_IB_MAX_CTX_DYNAMIC_UARS; i++) 930220176Sobrien context->dynamic_wc_uar_index[i] = MLX5_IB_INVALID_UAR_INDEX; 931220176Sobrien 932220176Sobrien INIT_LIST_HEAD(&context->db_page_list); 933220176Sobrien mutex_init(&context->db_page_mutex); 934220176Sobrien 935220176Sobrien resp.tot_uuars = req.total_num_uuars; 936220176Sobrien resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports); 937220176Sobrien err = ib_copy_to_udata(udata, &resp, 938220176Sobrien min_t(size_t, udata->outlen, sizeof(resp))); 939220176Sobrien if (err) 940220176Sobrien goto out_uars; 941220176Sobrien 942220176Sobrien uuari->ver = ver; 943220176Sobrien uuari->num_low_latency_uuars = req.num_low_latency_uuars; 944220176Sobrien uuari->uars = uars; 945220176Sobrien uuari->num_uars = num_uars; 946220176Sobrien 947220176Sobrien if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) == 948220176Sobrien IB_LINK_LAYER_ETHERNET) { 949220176Sobrien err = mlx5_alloc_transport_domain(dev->mdev, &context->tdn); 950220176Sobrien if (err) 951220176Sobrien goto out_uars; 952220176Sobrien } 953220176Sobrien 954220176Sobrien return &context->ibucontext; 955220176Sobrien 956220176Sobrienout_uars: 957220176Sobrien for (i--; i >= 0; i--) 958220176Sobrien mlx5_cmd_free_uar(dev->mdev, uars[i].index); 959220176Sobrien kfree(uuari->count); 960220176Sobrien 961220176Sobrienout_bitmap: 962220176Sobrien kfree(uuari->bitmap); 963220176Sobrien 964220176Sobrienout_uar_ctx: 965220176Sobrien kfree(uars); 966220176Sobrien 967220176Sobrienout_ctx: 968220176Sobrien kfree(context); 969220176Sobrien return ERR_PTR(err); 970220176Sobrien} 971220176Sobrien 972220176Sobrienstatic int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) 973220176Sobrien{ 974220176Sobrien struct mlx5_ib_ucontext *context = to_mucontext(ibcontext); 975220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibcontext->device); 976220176Sobrien struct mlx5_uuar_info *uuari = &context->uuari; 977220176Sobrien int i; 978220176Sobrien 979220176Sobrien if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) == 980220176Sobrien IB_LINK_LAYER_ETHERNET) 981220176Sobrien mlx5_dealloc_transport_domain(dev->mdev, context->tdn); 982220176Sobrien 983220176Sobrien for (i = 0; i < uuari->num_uars; i++) { 984220176Sobrien if (mlx5_cmd_free_uar(dev->mdev, uuari->uars[i].index)) 985220176Sobrien mlx5_ib_warn(dev, "failed to free UAR 0x%x\n", uuari->uars[i].index); 986220176Sobrien } 987220176Sobrien for (i = 0; i < MLX5_IB_MAX_CTX_DYNAMIC_UARS; i++) { 988220176Sobrien if (context->dynamic_wc_uar_index[i] != MLX5_IB_INVALID_UAR_INDEX) 989220176Sobrien mlx5_cmd_free_uar(dev->mdev, context->dynamic_wc_uar_index[i]); 990220176Sobrien } 991220176Sobrien 992220176Sobrien kfree(uuari->count); 993220176Sobrien kfree(uuari->bitmap); 994220176Sobrien kfree(uuari->uars); 995220176Sobrien kfree(context); 996220176Sobrien 997220176Sobrien return 0; 998220176Sobrien} 999220176Sobrien 1000220176Sobrienstatic phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int index) 1001220176Sobrien{ 1002220176Sobrien return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + index; 1003220176Sobrien} 1004220176Sobrien 1005220176Sobrienstatic int get_command(unsigned long offset) 1006220176Sobrien{ 1007220176Sobrien return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK; 1008220176Sobrien} 1009220176Sobrien 1010220176Sobrienstatic int get_arg(unsigned long offset) 1011220176Sobrien{ 1012220176Sobrien return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1); 1013220176Sobrien} 1014220176Sobrien 1015220176Sobrienstatic int get_index(unsigned long offset) 1016220176Sobrien{ 1017220176Sobrien return get_arg(offset); 1018220176Sobrien} 1019220176Sobrien 1020220176Sobrienstatic int uar_mmap(struct vm_area_struct *vma, pgprot_t prot, bool is_wc, 1021220176Sobrien struct mlx5_uuar_info *uuari, struct mlx5_ib_dev *dev, 1022220176Sobrien struct mlx5_ib_ucontext *context) 1023220176Sobrien{ 1024220176Sobrien unsigned long idx; 1025220176Sobrien phys_addr_t pfn; 1026220176Sobrien 1027220176Sobrien if (vma->vm_end - vma->vm_start != PAGE_SIZE) { 1028220176Sobrien mlx5_ib_warn(dev, "wrong size, expected PAGE_SIZE(%ld) got %ld\n", 1029220176Sobrien (long)PAGE_SIZE, (long)(vma->vm_end - vma->vm_start)); 1030220176Sobrien return -EINVAL; 1031220176Sobrien } 1032220176Sobrien 1033220176Sobrien idx = get_index(vma->vm_pgoff); 1034220176Sobrien if (idx >= uuari->num_uars) { 1035220176Sobrien mlx5_ib_warn(dev, "wrong offset, idx:%ld num_uars:%d\n", 1036220176Sobrien idx, uuari->num_uars); 1037220176Sobrien return -EINVAL; 1038220176Sobrien } 1039220176Sobrien 1040220176Sobrien pfn = uar_index2pfn(dev, uuari->uars[idx].index); 1041220176Sobrien mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx, 1042220176Sobrien (unsigned long long)pfn); 1043220176Sobrien 1044220176Sobrien vma->vm_page_prot = prot; 1045220176Sobrien if (io_remap_pfn_range(vma, vma->vm_start, pfn, 1046220176Sobrien PAGE_SIZE, vma->vm_page_prot)) { 1047220176Sobrien mlx5_ib_err(dev, "io remap failed\n"); 1048220176Sobrien return -EAGAIN; 1049220176Sobrien } 1050220176Sobrien 1051220176Sobrien mlx5_ib_dbg(dev, "mapped %s at 0x%lx, PA 0x%llx\n", is_wc ? "WC" : "NC", 1052220176Sobrien (long)vma->vm_start, (unsigned long long)pfn << PAGE_SHIFT); 1053220176Sobrien 1054220176Sobrien return 0; 1055220176Sobrien} 1056220176Sobrien 1057220176Sobrienstatic int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) 1058220176Sobrien{ 1059220176Sobrien struct mlx5_ib_ucontext *context = to_mucontext(ibcontext); 1060220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibcontext->device); 1061220176Sobrien struct mlx5_uuar_info *uuari = &context->uuari; 1062220176Sobrien unsigned long command; 1063220176Sobrien 1064220176Sobrien command = get_command(vma->vm_pgoff); 1065220176Sobrien switch (command) { 1066220176Sobrien case MLX5_IB_MMAP_REGULAR_PAGE: 1067220176Sobrien return uar_mmap(vma, pgprot_writecombine(vma->vm_page_prot), 1068220176Sobrien true, 1069220176Sobrien uuari, dev, context); 1070220176Sobrien 1071220176Sobrien break; 1072220176Sobrien 1073220176Sobrien case MLX5_IB_MMAP_WC_PAGE: 1074220176Sobrien return uar_mmap(vma, pgprot_writecombine(vma->vm_page_prot), 1075220176Sobrien true, uuari, dev, context); 1076220176Sobrien break; 1077220176Sobrien 1078220176Sobrien case MLX5_IB_MMAP_NC_PAGE: 1079220176Sobrien return uar_mmap(vma, pgprot_noncached(vma->vm_page_prot), 1080220176Sobrien false, uuari, dev, context); 1081220176Sobrien break; 1082220176Sobrien 1083220176Sobrien default: 1084220176Sobrien return -EINVAL; 1085220176Sobrien } 1086220176Sobrien 1087220176Sobrien return 0; 1088220176Sobrien} 1089220176Sobrien 1090220176Sobrienstatic int alloc_pa_mkey(struct mlx5_ib_dev *dev, u32 *key, u32 pdn) 1091220176Sobrien{ 1092220176Sobrien struct mlx5_create_mkey_mbox_in *in; 1093220176Sobrien struct mlx5_mkey_seg *seg; 1094220176Sobrien struct mlx5_core_mr mr; 1095220176Sobrien int err; 1096220176Sobrien 1097220176Sobrien in = kzalloc(sizeof(*in), GFP_KERNEL); 1098220176Sobrien if (!in) 1099220176Sobrien return -ENOMEM; 1100220176Sobrien 1101220176Sobrien seg = &in->seg; 1102220176Sobrien seg->flags = MLX5_PERM_LOCAL_READ | MLX5_ACCESS_MODE_PA; 1103220176Sobrien seg->flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64); 1104220176Sobrien seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 1105220176Sobrien seg->start_addr = 0; 1106220176Sobrien 1107220176Sobrien err = mlx5_core_create_mkey(dev->mdev, &mr, in, sizeof(*in), 1108220176Sobrien NULL, NULL, NULL); 1109220176Sobrien if (err) { 1110220176Sobrien mlx5_ib_warn(dev, "failed to create mkey, %d\n", err); 1111220176Sobrien goto err_in; 1112220176Sobrien } 1113220176Sobrien 1114220176Sobrien kfree(in); 1115220176Sobrien *key = mr.key; 1116220176Sobrien 1117220176Sobrien return 0; 1118220176Sobrien 1119220176Sobrienerr_in: 1120220176Sobrien kfree(in); 1121220176Sobrien 1122220176Sobrien return err; 1123220176Sobrien} 1124220176Sobrien 1125220176Sobrienstatic void free_pa_mkey(struct mlx5_ib_dev *dev, u32 key) 1126220176Sobrien{ 1127220176Sobrien struct mlx5_core_mr mr; 1128220176Sobrien int err; 1129220176Sobrien 1130220176Sobrien memset(&mr, 0, sizeof(mr)); 1131220176Sobrien mr.key = key; 1132220176Sobrien err = mlx5_core_destroy_mkey(dev->mdev, &mr); 1133220176Sobrien if (err) 1134220176Sobrien mlx5_ib_warn(dev, "failed to destroy mkey 0x%x\n", key); 1135220176Sobrien} 1136220176Sobrien 1137220176Sobrienstatic struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev, 1138220176Sobrien struct ib_ucontext *context, 1139220176Sobrien struct ib_udata *udata) 1140220176Sobrien{ 1141220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibdev); 1142220176Sobrien struct mlx5_ib_alloc_pd_resp resp; 1143220176Sobrien struct mlx5_ib_pd *pd; 1144220176Sobrien int err; 1145220176Sobrien 1146220176Sobrien pd = kmalloc(sizeof(*pd), GFP_KERNEL); 1147220176Sobrien if (!pd) 1148220176Sobrien return ERR_PTR(-ENOMEM); 1149220176Sobrien 1150220176Sobrien err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn); 1151220176Sobrien if (err) { 1152220176Sobrien mlx5_ib_warn(dev, "pd alloc failed\n"); 1153220176Sobrien kfree(pd); 1154220176Sobrien return ERR_PTR(err); 1155220176Sobrien } 1156220176Sobrien 1157220176Sobrien if (context) { 1158220176Sobrien resp.pdn = pd->pdn; 1159220176Sobrien if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { 1160220176Sobrien mlx5_ib_err(dev, "copy failed\n"); 1161220176Sobrien mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn); 1162220176Sobrien kfree(pd); 1163220176Sobrien return ERR_PTR(-EFAULT); 1164220176Sobrien } 1165220176Sobrien } else { 1166220176Sobrien err = alloc_pa_mkey(to_mdev(ibdev), &pd->pa_lkey, pd->pdn); 1167220176Sobrien if (err) { 1168220176Sobrien mlx5_ib_err(dev, "alloc mkey failed\n"); 1169220176Sobrien mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn); 1170220176Sobrien kfree(pd); 1171220176Sobrien return ERR_PTR(err); 1172220176Sobrien } 1173220176Sobrien } 1174220176Sobrien 1175220176Sobrien return &pd->ibpd; 1176220176Sobrien} 1177220176Sobrien 1178220176Sobrienstatic int mlx5_ib_dealloc_pd(struct ib_pd *pd) 1179220176Sobrien{ 1180220176Sobrien struct mlx5_ib_dev *mdev = to_mdev(pd->device); 1181220176Sobrien struct mlx5_ib_pd *mpd = to_mpd(pd); 1182220176Sobrien 1183220176Sobrien if (!pd->uobject) 1184220176Sobrien free_pa_mkey(mdev, mpd->pa_lkey); 1185220176Sobrien 1186220176Sobrien mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn); 1187220176Sobrien kfree(mpd); 1188220176Sobrien 1189220176Sobrien return 0; 1190220176Sobrien} 1191220176Sobrien 1192220176Sobrienstatic int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 1193220176Sobrien{ 1194220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 1195220176Sobrien int err; 1196220176Sobrien 1197220176Sobrien if (ibqp->qp_type == IB_QPT_RAW_PACKET) 1198220176Sobrien err = -EOPNOTSUPP; 1199220176Sobrien else 1200220176Sobrien err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num); 1201220176Sobrien if (err) 1202220176Sobrien mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n", 1203220176Sobrien ibqp->qp_num, gid->raw); 1204220176Sobrien 1205220176Sobrien return err; 1206220176Sobrien} 1207220176Sobrien 1208220176Sobrienstatic int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 1209220176Sobrien{ 1210220176Sobrien struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 1211220176Sobrien int err; 1212220176Sobrien 1213220176Sobrien if (ibqp->qp_type == IB_QPT_RAW_PACKET) 1214220176Sobrien err = -EOPNOTSUPP; 1215220176Sobrien else 1216220176Sobrien err = mlx5_core_detach_mcg(dev->mdev, gid, ibqp->qp_num); 1217220176Sobrien if (err) 1218220176Sobrien mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n", 1219220176Sobrien ibqp->qp_num, gid->raw); 1220220176Sobrien 1221220176Sobrien return err; 1222220176Sobrien} 1223220176Sobrien 1224220176Sobrienstatic int init_node_data(struct mlx5_ib_dev *dev) 1225220176Sobrien{ 1226220176Sobrien int err; 1227220176Sobrien 1228220176Sobrien err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc); 1229220176Sobrien if (err) 1230220176Sobrien return err; 1231220176Sobrien 1232220176Sobrien return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid); 1233220176Sobrien} 1234220176Sobrien 1235220176Sobrienstatic ssize_t show_fw_pages(struct device *device, struct device_attribute *attr, 1236220176Sobrien char *buf) 1237220176Sobrien{ 1238220176Sobrien struct mlx5_ib_dev *dev = 1239220176Sobrien container_of(device, struct mlx5_ib_dev, ib_dev.dev); 1240220176Sobrien 1241220176Sobrien return sprintf(buf, "%lld\n", (long long)dev->mdev->priv.fw_pages); 1242220176Sobrien} 1243220176Sobrien 1244220176Sobrienstatic ssize_t show_reg_pages(struct device *device, 1245220176Sobrien struct device_attribute *attr, char *buf) 1246220176Sobrien{ 1247220176Sobrien struct mlx5_ib_dev *dev = 1248220176Sobrien container_of(device, struct mlx5_ib_dev, ib_dev.dev); 1249220176Sobrien 1250220176Sobrien return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages)); 1251220176Sobrien} 1252220176Sobrien 1253220176Sobrienstatic ssize_t show_hca(struct device *device, struct device_attribute *attr, 1254220176Sobrien char *buf) 1255220176Sobrien{ 1256220176Sobrien struct mlx5_ib_dev *dev = 1257220176Sobrien container_of(device, struct mlx5_ib_dev, ib_dev.dev); 1258220176Sobrien return sprintf(buf, "MT%d\n", dev->mdev->pdev->device); 1259220176Sobrien} 1260220176Sobrien 1261220176Sobrienstatic ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, 1262220176Sobrien char *buf) 1263220176Sobrien{ 1264220176Sobrien struct mlx5_ib_dev *dev = 1265220176Sobrien container_of(device, struct mlx5_ib_dev, ib_dev.dev); 1266220176Sobrien return sprintf(buf, "%d.%d.%04d\n", fw_rev_maj(dev->mdev), 1267220176Sobrien fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev)); 1268220176Sobrien} 1269220176Sobrien 1270220176Sobrienstatic ssize_t show_rev(struct device *device, struct device_attribute *attr, 1271220176Sobrien char *buf) 1272220176Sobrien{ 1273220176Sobrien struct mlx5_ib_dev *dev = 1274220176Sobrien container_of(device, struct mlx5_ib_dev, ib_dev.dev); 1275220176Sobrien return sprintf(buf, "%x\n", (unsigned)dev->mdev->pdev->revision); 1276220176Sobrien} 1277220176Sobrien 1278220176Sobrienstatic ssize_t show_board(struct device *device, struct device_attribute *attr, 1279220176Sobrien char *buf) 1280220176Sobrien{ 1281220176Sobrien struct mlx5_ib_dev *dev = 1282220176Sobrien container_of(device, struct mlx5_ib_dev, ib_dev.dev); 1283220176Sobrien return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN, 1284220176Sobrien dev->mdev->board_id); 1285220176Sobrien} 1286220176Sobrien 1287220176Sobrienstatic DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); 1288220176Sobrienstatic DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); 1289220176Sobrienstatic DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); 1290220176Sobrienstatic DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); 1291220176Sobrienstatic DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL); 1292220176Sobrienstatic DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL); 1293220176Sobrien 1294220176Sobrienstatic struct device_attribute *mlx5_class_attributes[] = { 1295220176Sobrien &dev_attr_hw_rev, 1296220176Sobrien &dev_attr_fw_ver, 1297220176Sobrien &dev_attr_hca_type, 1298220176Sobrien &dev_attr_board_id, 1299220176Sobrien &dev_attr_fw_pages, 1300220176Sobrien &dev_attr_reg_pages, 1301220176Sobrien}; 1302220176Sobrien 1303220176Sobrienstatic void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev) 1304220176Sobrien{ 1305220176Sobrien struct mlx5_ib_qp *mqp; 1306220176Sobrien struct mlx5_ib_cq *send_mcq, *recv_mcq; 1307220176Sobrien struct mlx5_core_cq *mcq; 1308220176Sobrien struct list_head cq_armed_list; 1309220176Sobrien unsigned long flags_qp; 1310220176Sobrien unsigned long flags_cq; 1311220176Sobrien unsigned long flags; 1312220176Sobrien 1313220176Sobrien mlx5_ib_warn(ibdev, " started\n"); 1314220176Sobrien INIT_LIST_HEAD(&cq_armed_list); 1315220176Sobrien 1316220176Sobrien /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/ 1317220176Sobrien spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags); 1318220176Sobrien list_for_each_entry(mqp, &ibdev->qp_list, qps_list) { 1319220176Sobrien spin_lock_irqsave(&mqp->sq.lock, flags_qp); 1320220176Sobrien if (mqp->sq.tail != mqp->sq.head) { 1321220176Sobrien send_mcq = to_mcq(mqp->ibqp.send_cq); 1322220176Sobrien spin_lock_irqsave(&send_mcq->lock, flags_cq); 1323220176Sobrien if (send_mcq->mcq.comp && 1324220176Sobrien mqp->ibqp.send_cq->comp_handler) { 1325220176Sobrien if (!send_mcq->mcq.reset_notify_added) { 1326220176Sobrien send_mcq->mcq.reset_notify_added = 1; 1327220176Sobrien list_add_tail(&send_mcq->mcq.reset_notify, 1328220176Sobrien &cq_armed_list); 1329220176Sobrien } 1330220176Sobrien } 1331220176Sobrien spin_unlock_irqrestore(&send_mcq->lock, flags_cq); 1332220176Sobrien } 1333220176Sobrien spin_unlock_irqrestore(&mqp->sq.lock, flags_qp); 1334220176Sobrien spin_lock_irqsave(&mqp->rq.lock, flags_qp); 1335220176Sobrien /* no handling is needed for SRQ */ 1336220176Sobrien if (!mqp->ibqp.srq) { 1337220176Sobrien if (mqp->rq.tail != mqp->rq.head) { 1338220176Sobrien recv_mcq = to_mcq(mqp->ibqp.recv_cq); 1339220176Sobrien spin_lock_irqsave(&recv_mcq->lock, flags_cq); 1340220176Sobrien if (recv_mcq->mcq.comp && 1341220176Sobrien mqp->ibqp.recv_cq->comp_handler) { 1342220176Sobrien if (!recv_mcq->mcq.reset_notify_added) { 1343220176Sobrien recv_mcq->mcq.reset_notify_added = 1; 1344220176Sobrien list_add_tail(&recv_mcq->mcq.reset_notify, 1345220176Sobrien &cq_armed_list); 1346220176Sobrien } 1347220176Sobrien } 1348220176Sobrien spin_unlock_irqrestore(&recv_mcq->lock, 1349220176Sobrien flags_cq); 1350220176Sobrien } 1351220176Sobrien } 1352220176Sobrien spin_unlock_irqrestore(&mqp->rq.lock, flags_qp); 1353220176Sobrien } 1354220176Sobrien /*At that point all inflight post send were put to be executed as of we 1355220176Sobrien * lock/unlock above locks Now need to arm all involved CQs. 1356220176Sobrien */ 1357220176Sobrien list_for_each_entry(mcq, &cq_armed_list, reset_notify) { 1358220176Sobrien mcq->comp(mcq); 1359220176Sobrien } 1360220176Sobrien spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags); 1361220176Sobrien mlx5_ib_warn(ibdev, " ended\n"); 1362220176Sobrien return; 1363220176Sobrien} 1364220176Sobrien 1365220176Sobrienstatic void mlx5_ib_event(struct mlx5_core_dev *dev, void *context, 1366220176Sobrien enum mlx5_dev_event event, unsigned long param) 1367220176Sobrien{ 1368220176Sobrien struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context; 1369220176Sobrien struct ib_event ibev; 1370220176Sobrien 1371220176Sobrien u8 port = 0; 1372220176Sobrien 1373220176Sobrien switch (event) { 1374220176Sobrien case MLX5_DEV_EVENT_SYS_ERROR: 1375220176Sobrien ibdev->ib_active = false; 1376220176Sobrien ibev.event = IB_EVENT_DEVICE_FATAL; 1377220176Sobrien mlx5_ib_handle_internal_error(ibdev); 1378220176Sobrien break; 1379220176Sobrien 1380220176Sobrien case MLX5_DEV_EVENT_PORT_UP: 1381220176Sobrien ibev.event = IB_EVENT_PORT_ACTIVE; 1382220176Sobrien port = (u8)param; 1383220176Sobrien break; 1384220176Sobrien 1385220176Sobrien case MLX5_DEV_EVENT_PORT_DOWN: 1386220176Sobrien case MLX5_DEV_EVENT_PORT_INITIALIZED: 1387220176Sobrien ibev.event = IB_EVENT_PORT_ERR; 1388220176Sobrien port = (u8)param; 1389220176Sobrien break; 1390220176Sobrien 1391220176Sobrien case MLX5_DEV_EVENT_LID_CHANGE: 1392220176Sobrien ibev.event = IB_EVENT_LID_CHANGE; 1393220176Sobrien port = (u8)param; 1394220176Sobrien break; 1395220176Sobrien 1396220176Sobrien case MLX5_DEV_EVENT_PKEY_CHANGE: 1397220176Sobrien ibev.event = IB_EVENT_PKEY_CHANGE; 1398220176Sobrien port = (u8)param; 1399220176Sobrien break; 1400220176Sobrien 1401220176Sobrien case MLX5_DEV_EVENT_GUID_CHANGE: 1402220176Sobrien ibev.event = IB_EVENT_GID_CHANGE; 1403220176Sobrien port = (u8)param; 1404220176Sobrien break; 1405220176Sobrien 1406220176Sobrien case MLX5_DEV_EVENT_CLIENT_REREG: 1407220176Sobrien ibev.event = IB_EVENT_CLIENT_REREGISTER; 1408220176Sobrien port = (u8)param; 1409220176Sobrien break; 1410220176Sobrien 1411220176Sobrien default: 1412220176Sobrien break; 1413220176Sobrien } 1414220176Sobrien 1415220176Sobrien ibev.device = &ibdev->ib_dev; 1416220176Sobrien ibev.element.port_num = port; 1417220176Sobrien 1418220176Sobrien if ((event != MLX5_DEV_EVENT_SYS_ERROR) && 1419220176Sobrien (port < 1 || port > ibdev->num_ports)) { 1420220176Sobrien mlx5_ib_warn(ibdev, "warning: event on port %d\n", port); 1421220176Sobrien return; 1422220176Sobrien } 1423220176Sobrien 1424220176Sobrien if (ibdev->ib_active) 1425220176Sobrien ib_dispatch_event(&ibev); 1426220176Sobrien} 1427220176Sobrien 1428220176Sobrienstatic void get_ext_port_caps(struct mlx5_ib_dev *dev) 1429220176Sobrien{ 1430220176Sobrien int port; 1431220176Sobrien 1432220176Sobrien for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) 1433220176Sobrien mlx5_query_ext_port_caps(dev, port); 1434220176Sobrien} 1435220176Sobrien 1436220176Sobrienstatic void config_atomic_responder(struct mlx5_ib_dev *dev, 1437220176Sobrien struct ib_device_attr *props) 1438220176Sobrien{ 1439220176Sobrien enum ib_atomic_cap cap = props->atomic_cap; 1440220176Sobrien 1441220176Sobrien#if 0 1442220176Sobrien if (cap == IB_ATOMIC_HCA || 1443220176Sobrien cap == IB_ATOMIC_GLOB) 1444220176Sobrien#endif 1445220176Sobrien dev->enable_atomic_resp = 1; 1446220176Sobrien 1447220176Sobrien dev->atomic_cap = cap; 1448220176Sobrien} 1449220176Sobrien 1450220176Sobrienenum mlx5_addr_align { 1451220176Sobrien MLX5_ADDR_ALIGN_0 = 0, 1452220176Sobrien MLX5_ADDR_ALIGN_64 = 64, 1453220176Sobrien MLX5_ADDR_ALIGN_128 = 128, 1454220176Sobrien}; 1455220176Sobrien 1456220176Sobrienstatic int get_port_caps(struct mlx5_ib_dev *dev) 1457220176Sobrien{ 1458220176Sobrien struct ib_device_attr *dprops = NULL; 1459220176Sobrien struct ib_port_attr *pprops = NULL; 1460220176Sobrien int err = -ENOMEM; 1461220176Sobrien int port; 1462220176Sobrien 1463220176Sobrien pprops = kmalloc(sizeof(*pprops), GFP_KERNEL); 1464220176Sobrien if (!pprops) 1465220176Sobrien goto out; 1466220176Sobrien 1467220176Sobrien dprops = kmalloc(sizeof(*dprops), GFP_KERNEL); 1468220176Sobrien if (!dprops) 1469220176Sobrien goto out; 1470220176Sobrien 1471220176Sobrien err = mlx5_ib_query_device(&dev->ib_dev, dprops); 1472220176Sobrien if (err) { 1473220176Sobrien mlx5_ib_warn(dev, "query_device failed %d\n", err); 1474220176Sobrien goto out; 1475220176Sobrien } 1476220176Sobrien config_atomic_responder(dev, dprops); 1477220176Sobrien 1478220176Sobrien for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) { 1479220176Sobrien err = mlx5_ib_query_port(&dev->ib_dev, port, pprops); 1480220176Sobrien if (err) { 1481220176Sobrien mlx5_ib_warn(dev, "query_port %d failed %d\n", 1482220176Sobrien port, err); 1483220176Sobrien break; 1484220176Sobrien } 1485220176Sobrien dev->mdev->port_caps[port - 1].pkey_table_len = dprops->max_pkeys; 1486220176Sobrien dev->mdev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len; 1487220176Sobrien mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n", 1488220176Sobrien dprops->max_pkeys, pprops->gid_tbl_len); 1489220176Sobrien } 1490220176Sobrien 1491220176Sobrienout: 1492220176Sobrien kfree(pprops); 1493220176Sobrien kfree(dprops); 1494220176Sobrien 1495220176Sobrien return err; 1496220176Sobrien} 1497220176Sobrien 1498220176Sobrienstatic void destroy_umrc_res(struct mlx5_ib_dev *dev) 1499220176Sobrien{ 1500220176Sobrien int err; 1501220176Sobrien 1502220176Sobrien err = mlx5_mr_cache_cleanup(dev); 1503220176Sobrien if (err) 1504220176Sobrien mlx5_ib_warn(dev, "mr cache cleanup failed\n"); 1505220176Sobrien 1506220176Sobrien ib_dereg_mr(dev->umrc.mr); 1507220176Sobrien ib_dealloc_pd(dev->umrc.pd); 1508220176Sobrien} 1509220176Sobrien 1510220176Sobrienenum { 1511220176Sobrien MAX_UMR_WR = 128, 1512220176Sobrien}; 1513220176Sobrien 1514220176Sobrienstatic int create_umr_res(struct mlx5_ib_dev *dev) 1515220176Sobrien{ 1516220176Sobrien struct ib_pd *pd; 1517220176Sobrien struct ib_mr *mr; 1518220176Sobrien int ret; 1519220176Sobrien 1520220176Sobrien pd = ib_alloc_pd(&dev->ib_dev); 1521220176Sobrien if (IS_ERR(pd)) { 1522220176Sobrien mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n"); 1523220176Sobrien ret = PTR_ERR(pd); 1524220176Sobrien goto error_0; 1525220176Sobrien } 1526220176Sobrien 1527220176Sobrien mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE); 1528220176Sobrien if (IS_ERR(mr)) { 1529220176Sobrien mlx5_ib_dbg(dev, "Couldn't create DMA MR for sync UMR QP\n"); 1530220176Sobrien ret = PTR_ERR(mr); 1531220176Sobrien goto error_1; 1532220176Sobrien } 1533220176Sobrien 1534220176Sobrien dev->umrc.mr = mr; 1535220176Sobrien dev->umrc.pd = pd; 1536220176Sobrien 1537220176Sobrien ret = mlx5_mr_cache_init(dev); 1538220176Sobrien if (ret) { 1539220176Sobrien mlx5_ib_warn(dev, "mr cache init failed %d\n", ret); 1540220176Sobrien goto error_4; 1541220176Sobrien } 1542220176Sobrien 1543220176Sobrien return 0; 1544220176Sobrien 1545220176Sobrienerror_4: 1546220176Sobrien ib_dereg_mr(mr); 1547220176Sobrienerror_1: 1548220176Sobrien ib_dealloc_pd(pd); 1549220176Sobrienerror_0: 1550220176Sobrien return ret; 1551220176Sobrien} 1552220176Sobrien 1553220176Sobrienstatic int create_dev_resources(struct mlx5_ib_resources *devr) 1554220176Sobrien{ 1555220176Sobrien struct ib_srq_init_attr attr; 1556220176Sobrien struct mlx5_ib_dev *dev; 1557220176Sobrien int ret = 0; 1558220176Sobrien 1559220176Sobrien dev = container_of(devr, struct mlx5_ib_dev, devr); 1560220176Sobrien 1561220176Sobrien devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL); 1562220176Sobrien if (IS_ERR(devr->p0)) { 1563220176Sobrien ret = PTR_ERR(devr->p0); 1564220176Sobrien goto error0; 1565220176Sobrien } 1566220176Sobrien devr->p0->device = &dev->ib_dev; 1567220176Sobrien devr->p0->uobject = NULL; 1568220176Sobrien atomic_set(&devr->p0->usecnt, 0); 1569220176Sobrien 1570220176Sobrien devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, 1, 0, NULL, NULL); 1571220176Sobrien if (IS_ERR(devr->c0)) { 1572220176Sobrien ret = PTR_ERR(devr->c0); 1573220176Sobrien goto error1; 1574220176Sobrien } 1575220176Sobrien devr->c0->device = &dev->ib_dev; 1576220176Sobrien devr->c0->uobject = NULL; 1577220176Sobrien devr->c0->comp_handler = NULL; 1578220176Sobrien devr->c0->event_handler = NULL; 1579220176Sobrien devr->c0->cq_context = NULL; 1580220176Sobrien atomic_set(&devr->c0->usecnt, 0); 1581220176Sobrien 1582220176Sobrien devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL); 1583220176Sobrien if (IS_ERR(devr->x0)) { 1584220176Sobrien ret = PTR_ERR(devr->x0); 1585220176Sobrien goto error2; 1586220176Sobrien } 1587220176Sobrien devr->x0->device = &dev->ib_dev; 1588220176Sobrien devr->x0->inode = NULL; 1589220176Sobrien atomic_set(&devr->x0->usecnt, 0); 1590220176Sobrien mutex_init(&devr->x0->tgt_qp_mutex); 1591220176Sobrien INIT_LIST_HEAD(&devr->x0->tgt_qp_list); 1592220176Sobrien 1593220176Sobrien devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL); 1594220176Sobrien if (IS_ERR(devr->x1)) { 1595220176Sobrien ret = PTR_ERR(devr->x1); 1596220176Sobrien goto error3; 1597220176Sobrien } 1598220176Sobrien devr->x1->device = &dev->ib_dev; 1599220176Sobrien devr->x1->inode = NULL; 1600220176Sobrien atomic_set(&devr->x1->usecnt, 0); 1601220176Sobrien mutex_init(&devr->x1->tgt_qp_mutex); 1602220176Sobrien INIT_LIST_HEAD(&devr->x1->tgt_qp_list); 1603220176Sobrien 1604220176Sobrien memset(&attr, 0, sizeof(attr)); 1605220176Sobrien attr.attr.max_sge = 1; 1606220176Sobrien attr.attr.max_wr = 1; 1607220176Sobrien attr.srq_type = IB_SRQT_XRC; 1608220176Sobrien attr.ext.xrc.cq = devr->c0; 1609220176Sobrien attr.ext.xrc.xrcd = devr->x0; 1610220176Sobrien 1611220176Sobrien devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL); 1612220176Sobrien if (IS_ERR(devr->s0)) { 1613220176Sobrien ret = PTR_ERR(devr->s0); 1614220176Sobrien goto error4; 1615220176Sobrien } 1616220176Sobrien devr->s0->device = &dev->ib_dev; 1617220176Sobrien devr->s0->pd = devr->p0; 1618220176Sobrien devr->s0->uobject = NULL; 1619220176Sobrien devr->s0->event_handler = NULL; 1620220176Sobrien devr->s0->srq_context = NULL; 1621220176Sobrien devr->s0->srq_type = IB_SRQT_XRC; 1622220176Sobrien devr->s0->ext.xrc.xrcd = devr->x0; 1623220176Sobrien devr->s0->ext.xrc.cq = devr->c0; 1624220176Sobrien atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt); 1625220176Sobrien atomic_inc(&devr->s0->ext.xrc.cq->usecnt); 1626220176Sobrien atomic_inc(&devr->p0->usecnt); 1627220176Sobrien atomic_set(&devr->s0->usecnt, 0); 1628220176Sobrien 1629220176Sobrien memset(&attr, 0, sizeof(attr)); 1630220176Sobrien attr.attr.max_sge = 1; 1631220176Sobrien attr.attr.max_wr = 1; 1632220176Sobrien attr.srq_type = IB_SRQT_BASIC; 1633220176Sobrien devr->s1 = mlx5_ib_create_srq(devr->p0, &attr, NULL); 1634220176Sobrien if (IS_ERR(devr->s1)) { 1635220176Sobrien ret = PTR_ERR(devr->s1); 1636220176Sobrien goto error5; 1637220176Sobrien } 1638220176Sobrien devr->s1->device = &dev->ib_dev; 1639220176Sobrien devr->s1->pd = devr->p0; 1640220176Sobrien devr->s1->uobject = NULL; 1641220176Sobrien devr->s1->event_handler = NULL; 1642220176Sobrien devr->s1->srq_context = NULL; 1643220176Sobrien devr->s1->srq_type = IB_SRQT_BASIC; 1644220176Sobrien devr->s1->ext.xrc.cq = devr->c0; 1645220176Sobrien atomic_inc(&devr->p0->usecnt); 1646220176Sobrien atomic_set(&devr->s1->usecnt, 0); 1647220176Sobrien 1648220176Sobrien return 0; 1649220176Sobrien 1650220176Sobrienerror5: 1651220176Sobrien mlx5_ib_destroy_srq(devr->s0); 1652220176Sobrienerror4: 1653220176Sobrien mlx5_ib_dealloc_xrcd(devr->x1); 1654220176Sobrienerror3: 1655220176Sobrien mlx5_ib_dealloc_xrcd(devr->x0); 1656220176Sobrienerror2: 1657220176Sobrien mlx5_ib_destroy_cq(devr->c0); 1658220176Sobrienerror1: 1659220176Sobrien mlx5_ib_dealloc_pd(devr->p0); 1660220176Sobrienerror0: 1661220176Sobrien return ret; 1662220176Sobrien} 1663220176Sobrien 1664220176Sobrienstatic void destroy_dev_resources(struct mlx5_ib_resources *devr) 1665{ 1666 mlx5_ib_destroy_srq(devr->s1); 1667 mlx5_ib_destroy_srq(devr->s0); 1668 mlx5_ib_dealloc_xrcd(devr->x0); 1669 mlx5_ib_dealloc_xrcd(devr->x1); 1670 mlx5_ib_destroy_cq(devr->c0); 1671 mlx5_ib_dealloc_pd(devr->p0); 1672} 1673 1674static void enable_dc_tracer(struct mlx5_ib_dev *dev) 1675{ 1676 struct device *device = dev->ib_dev.dma_device; 1677 struct mlx5_dc_tracer *dct = &dev->dctr; 1678 int order; 1679 void *tmp; 1680 int size; 1681 int err; 1682 1683 size = MLX5_CAP_GEN(dev->mdev, num_ports) * 4096; 1684 if (size <= PAGE_SIZE) 1685 order = 0; 1686 else 1687 order = 1; 1688 1689 dct->pg = alloc_pages(GFP_KERNEL, order); 1690 if (!dct->pg) { 1691 mlx5_ib_err(dev, "failed to allocate %d pages\n", order); 1692 return; 1693 } 1694 1695 tmp = page_address(dct->pg); 1696 memset(tmp, 0xff, size); 1697 1698 dct->size = size; 1699 dct->order = order; 1700 dct->dma = dma_map_page(device, dct->pg, 0, size, DMA_FROM_DEVICE); 1701 if (dma_mapping_error(device, dct->dma)) { 1702 mlx5_ib_err(dev, "dma mapping error\n"); 1703 goto map_err; 1704 } 1705 1706 err = mlx5_core_set_dc_cnak_trace(dev->mdev, 1, dct->dma); 1707 if (err) { 1708 mlx5_ib_warn(dev, "failed to enable DC tracer\n"); 1709 goto cmd_err; 1710 } 1711 1712 return; 1713 1714cmd_err: 1715 dma_unmap_page(device, dct->dma, size, DMA_FROM_DEVICE); 1716map_err: 1717 __free_pages(dct->pg, dct->order); 1718 dct->pg = NULL; 1719} 1720 1721static void disable_dc_tracer(struct mlx5_ib_dev *dev) 1722{ 1723 struct device *device = dev->ib_dev.dma_device; 1724 struct mlx5_dc_tracer *dct = &dev->dctr; 1725 int err; 1726 1727 if (!dct->pg) 1728 return; 1729 1730 err = mlx5_core_set_dc_cnak_trace(dev->mdev, 0, dct->dma); 1731 if (err) { 1732 mlx5_ib_warn(dev, "failed to disable DC tracer\n"); 1733 return; 1734 } 1735 1736 dma_unmap_page(device, dct->dma, dct->size, DMA_FROM_DEVICE); 1737 __free_pages(dct->pg, dct->order); 1738 dct->pg = NULL; 1739} 1740 1741enum { 1742 MLX5_DC_CNAK_SIZE = 128, 1743 MLX5_NUM_BUF_IN_PAGE = PAGE_SIZE / MLX5_DC_CNAK_SIZE, 1744 MLX5_CNAK_TX_CQ_SIGNAL_FACTOR = 128, 1745 MLX5_DC_CNAK_SL = 0, 1746 MLX5_DC_CNAK_VL = 0, 1747}; 1748 1749static int init_dc_improvements(struct mlx5_ib_dev *dev) 1750{ 1751 if (!mlx5_core_is_pf(dev->mdev)) 1752 return 0; 1753 1754 if (!(MLX5_CAP_GEN(dev->mdev, dc_cnak_trace))) 1755 return 0; 1756 1757 enable_dc_tracer(dev); 1758 1759 return 0; 1760} 1761 1762static void cleanup_dc_improvements(struct mlx5_ib_dev *dev) 1763{ 1764 1765 disable_dc_tracer(dev); 1766} 1767 1768static void mlx5_ib_dealloc_q_port_counter(struct mlx5_ib_dev *dev, u8 port_num) 1769{ 1770 mlx5_vport_dealloc_q_counter(dev->mdev, 1771 MLX5_INTERFACE_PROTOCOL_IB, 1772 dev->port[port_num].q_cnt_id); 1773 dev->port[port_num].q_cnt_id = 0; 1774} 1775 1776static void mlx5_ib_dealloc_q_counters(struct mlx5_ib_dev *dev) 1777{ 1778 unsigned int i; 1779 1780 for (i = 0; i < dev->num_ports; i++) 1781 mlx5_ib_dealloc_q_port_counter(dev, i); 1782} 1783 1784static int mlx5_ib_alloc_q_counters(struct mlx5_ib_dev *dev) 1785{ 1786 int i; 1787 int ret; 1788 1789 for (i = 0; i < dev->num_ports; i++) { 1790 ret = mlx5_vport_alloc_q_counter(dev->mdev, 1791 MLX5_INTERFACE_PROTOCOL_IB, 1792 &dev->port[i].q_cnt_id); 1793 if (ret) { 1794 mlx5_ib_warn(dev, 1795 "couldn't allocate queue counter for port %d\n", 1796 i + 1); 1797 goto dealloc_counters; 1798 } 1799 } 1800 1801 return 0; 1802 1803dealloc_counters: 1804 while (--i >= 0) 1805 mlx5_ib_dealloc_q_port_counter(dev, i); 1806 1807 return ret; 1808} 1809 1810struct port_attribute { 1811 struct attribute attr; 1812 ssize_t (*show)(struct mlx5_ib_port *, 1813 struct port_attribute *, char *buf); 1814 ssize_t (*store)(struct mlx5_ib_port *, 1815 struct port_attribute *, 1816 const char *buf, size_t count); 1817}; 1818 1819struct port_counter_attribute { 1820 struct port_attribute attr; 1821 size_t offset; 1822}; 1823 1824static ssize_t port_attr_show(struct kobject *kobj, 1825 struct attribute *attr, char *buf) 1826{ 1827 struct port_attribute *port_attr = 1828 container_of(attr, struct port_attribute, attr); 1829 struct mlx5_ib_port_sysfs_group *p = 1830 container_of(kobj, struct mlx5_ib_port_sysfs_group, 1831 kobj); 1832 struct mlx5_ib_port *mibport = container_of(p, struct mlx5_ib_port, 1833 group); 1834 1835 if (!port_attr->show) 1836 return -EIO; 1837 1838 return port_attr->show(mibport, port_attr, buf); 1839} 1840 1841static ssize_t show_port_counter(struct mlx5_ib_port *p, 1842 struct port_attribute *port_attr, 1843 char *buf) 1844{ 1845 int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out); 1846 struct port_counter_attribute *counter_attr = 1847 container_of(port_attr, struct port_counter_attribute, attr); 1848 void *out; 1849 int ret; 1850 1851 out = mlx5_vzalloc(outlen); 1852 if (!out) 1853 return -ENOMEM; 1854 1855 ret = mlx5_vport_query_q_counter(p->dev->mdev, 1856 p->q_cnt_id, 0, 1857 out, outlen); 1858 if (ret) 1859 goto free; 1860 1861 ret = sprintf(buf, "%d\n", 1862 be32_to_cpu(*(__be32 *)(out + counter_attr->offset))); 1863 1864free: 1865 kfree(out); 1866 return ret; 1867} 1868 1869#define PORT_COUNTER_ATTR(_name) \ 1870struct port_counter_attribute port_counter_attr_##_name = { \ 1871 .attr = __ATTR(_name, S_IRUGO, show_port_counter, NULL), \ 1872 .offset = MLX5_BYTE_OFF(query_q_counter_out, _name) \ 1873} 1874 1875static PORT_COUNTER_ATTR(rx_write_requests); 1876static PORT_COUNTER_ATTR(rx_read_requests); 1877static PORT_COUNTER_ATTR(rx_atomic_requests); 1878static PORT_COUNTER_ATTR(rx_dct_connect); 1879static PORT_COUNTER_ATTR(out_of_buffer); 1880static PORT_COUNTER_ATTR(out_of_sequence); 1881static PORT_COUNTER_ATTR(duplicate_request); 1882static PORT_COUNTER_ATTR(rnr_nak_retry_err); 1883static PORT_COUNTER_ATTR(packet_seq_err); 1884static PORT_COUNTER_ATTR(implied_nak_seq_err); 1885static PORT_COUNTER_ATTR(local_ack_timeout_err); 1886 1887static struct attribute *counter_attrs[] = { 1888 &port_counter_attr_rx_write_requests.attr.attr, 1889 &port_counter_attr_rx_read_requests.attr.attr, 1890 &port_counter_attr_rx_atomic_requests.attr.attr, 1891 &port_counter_attr_rx_dct_connect.attr.attr, 1892 &port_counter_attr_out_of_buffer.attr.attr, 1893 &port_counter_attr_out_of_sequence.attr.attr, 1894 &port_counter_attr_duplicate_request.attr.attr, 1895 &port_counter_attr_rnr_nak_retry_err.attr.attr, 1896 &port_counter_attr_packet_seq_err.attr.attr, 1897 &port_counter_attr_implied_nak_seq_err.attr.attr, 1898 &port_counter_attr_local_ack_timeout_err.attr.attr, 1899 NULL 1900}; 1901 1902static struct attribute_group port_counters_group = { 1903 .name = "counters", 1904 .attrs = counter_attrs 1905}; 1906 1907static const struct sysfs_ops port_sysfs_ops = { 1908 .show = port_attr_show 1909}; 1910 1911static struct kobj_type port_type = { 1912 .sysfs_ops = &port_sysfs_ops, 1913}; 1914 1915static int add_port_attrs(struct mlx5_ib_dev *dev, 1916 struct kobject *parent, 1917 struct mlx5_ib_port_sysfs_group *port, 1918 u8 port_num) 1919{ 1920 int ret; 1921 1922 ret = kobject_init_and_add(&port->kobj, &port_type, 1923 parent, 1924 "%d", port_num); 1925 if (ret) 1926 return ret; 1927 1928 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt) && 1929 MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) { 1930 ret = sysfs_create_group(&port->kobj, &port_counters_group); 1931 if (ret) 1932 goto put_kobj; 1933 } 1934 1935 port->enabled = true; 1936 return ret; 1937 1938put_kobj: 1939 kobject_put(&port->kobj); 1940 return ret; 1941} 1942 1943static void destroy_ports_attrs(struct mlx5_ib_dev *dev, 1944 unsigned int num_ports) 1945{ 1946 unsigned int i; 1947 1948 for (i = 0; i < num_ports; i++) { 1949 struct mlx5_ib_port_sysfs_group *port = 1950 &dev->port[i].group; 1951 1952 if (!port->enabled) 1953 continue; 1954 1955 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt) && 1956 MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) 1957 sysfs_remove_group(&port->kobj, 1958 &port_counters_group); 1959 kobject_put(&port->kobj); 1960 port->enabled = false; 1961 } 1962 1963 if (dev->ports_parent) { 1964 kobject_put(dev->ports_parent); 1965 dev->ports_parent = NULL; 1966 } 1967} 1968 1969static int create_port_attrs(struct mlx5_ib_dev *dev) 1970{ 1971 int ret = 0; 1972 unsigned int i = 0; 1973 struct device *device = &dev->ib_dev.dev; 1974 1975 dev->ports_parent = kobject_create_and_add("mlx5_ports", 1976 &device->kobj); 1977 if (!dev->ports_parent) 1978 return -ENOMEM; 1979 1980 for (i = 0; i < dev->num_ports; i++) { 1981 ret = add_port_attrs(dev, 1982 dev->ports_parent, 1983 &dev->port[i].group, 1984 i + 1); 1985 1986 if (ret) 1987 goto _destroy_ports_attrs; 1988 } 1989 1990 return 0; 1991 1992_destroy_ports_attrs: 1993 destroy_ports_attrs(dev, i); 1994 return ret; 1995} 1996 1997static void *mlx5_ib_add(struct mlx5_core_dev *mdev) 1998{ 1999 struct mlx5_ib_dev *dev; 2000 int err; 2001 int i; 2002 2003 printk_once(KERN_INFO "%s", mlx5_version); 2004 2005 dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev)); 2006 if (!dev) 2007 return NULL; 2008 2009 dev->mdev = mdev; 2010 2011 dev->port = kcalloc(MLX5_CAP_GEN(mdev, num_ports), sizeof(*dev->port), 2012 GFP_KERNEL); 2013 if (!dev->port) 2014 goto err_dealloc; 2015 2016 for (i = 0; i < MLX5_CAP_GEN(mdev, num_ports); i++) { 2017 dev->port[i].dev = dev; 2018 dev->port[i].port_num = i; 2019 dev->port[i].port_gone = 0; 2020 memset(dev->port[i].gid_table, 0, sizeof(dev->port[i].gid_table)); 2021 } 2022 2023 err = get_port_caps(dev); 2024 if (err) 2025 goto err_free_port; 2026 2027 if (mlx5_use_mad_ifc(dev)) 2028 get_ext_port_caps(dev); 2029 2030 if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) == 2031 IB_LINK_LAYER_ETHERNET) { 2032 if (MLX5_CAP_GEN(mdev, roce)) { 2033 err = mlx5_nic_vport_enable_roce(mdev); 2034 if (err) 2035 goto err_free_port; 2036 } else { 2037 goto err_free_port; 2038 } 2039 } 2040 2041 MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock); 2042 2043 strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX); 2044 dev->ib_dev.owner = THIS_MODULE; 2045 dev->ib_dev.node_type = RDMA_NODE_IB_CA; 2046 dev->ib_dev.local_dma_lkey = mdev->special_contexts.resd_lkey; 2047 dev->num_ports = MLX5_CAP_GEN(mdev, num_ports); 2048 dev->ib_dev.phys_port_cnt = dev->num_ports; 2049 dev->ib_dev.num_comp_vectors = 2050 dev->mdev->priv.eq_table.num_comp_vectors; 2051 dev->ib_dev.dma_device = &mdev->pdev->dev; 2052 2053 dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION; 2054 dev->ib_dev.uverbs_cmd_mask = 2055 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 2056 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 2057 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 2058 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 2059 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 2060 (1ull << IB_USER_VERBS_CMD_REG_MR) | 2061 (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 2062 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 2063 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 2064 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | 2065 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 2066 (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 2067 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 2068 (1ull << IB_USER_VERBS_CMD_QUERY_QP) | 2069 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 2070 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | 2071 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) | 2072 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | 2073 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | 2074 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | 2075 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | 2076 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) | 2077 (1ull << IB_USER_VERBS_CMD_OPEN_QP); 2078 2079 dev->ib_dev.query_device = mlx5_ib_query_device; 2080 dev->ib_dev.query_port = mlx5_ib_query_port; 2081 dev->ib_dev.get_link_layer = mlx5_ib_port_link_layer; 2082 dev->ib_dev.query_gid = mlx5_ib_query_gid; 2083 dev->ib_dev.query_pkey = mlx5_ib_query_pkey; 2084 dev->ib_dev.modify_device = mlx5_ib_modify_device; 2085 dev->ib_dev.modify_port = mlx5_ib_modify_port; 2086 dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext; 2087 dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext; 2088 dev->ib_dev.mmap = mlx5_ib_mmap; 2089 dev->ib_dev.alloc_pd = mlx5_ib_alloc_pd; 2090 dev->ib_dev.dealloc_pd = mlx5_ib_dealloc_pd; 2091 dev->ib_dev.create_ah = mlx5_ib_create_ah; 2092 dev->ib_dev.query_ah = mlx5_ib_query_ah; 2093 dev->ib_dev.destroy_ah = mlx5_ib_destroy_ah; 2094 dev->ib_dev.create_srq = mlx5_ib_create_srq; 2095 dev->ib_dev.modify_srq = mlx5_ib_modify_srq; 2096 dev->ib_dev.query_srq = mlx5_ib_query_srq; 2097 dev->ib_dev.destroy_srq = mlx5_ib_destroy_srq; 2098 dev->ib_dev.post_srq_recv = mlx5_ib_post_srq_recv; 2099 dev->ib_dev.create_qp = mlx5_ib_create_qp; 2100 dev->ib_dev.modify_qp = mlx5_ib_modify_qp; 2101 dev->ib_dev.query_qp = mlx5_ib_query_qp; 2102 dev->ib_dev.destroy_qp = mlx5_ib_destroy_qp; 2103 dev->ib_dev.post_send = mlx5_ib_post_send; 2104 dev->ib_dev.post_recv = mlx5_ib_post_recv; 2105 dev->ib_dev.create_cq = mlx5_ib_create_cq; 2106 dev->ib_dev.modify_cq = mlx5_ib_modify_cq; 2107 dev->ib_dev.resize_cq = mlx5_ib_resize_cq; 2108 dev->ib_dev.destroy_cq = mlx5_ib_destroy_cq; 2109 dev->ib_dev.poll_cq = mlx5_ib_poll_cq; 2110 dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq; 2111 dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr; 2112 dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr; 2113 dev->ib_dev.reg_phys_mr = mlx5_ib_reg_phys_mr; 2114 dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr; 2115 dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach; 2116 dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach; 2117 dev->ib_dev.process_mad = mlx5_ib_process_mad; 2118 dev->ib_dev.alloc_fast_reg_mr = mlx5_ib_alloc_fast_reg_mr; 2119 dev->ib_dev.alloc_fast_reg_page_list = mlx5_ib_alloc_fast_reg_page_list; 2120 dev->ib_dev.free_fast_reg_page_list = mlx5_ib_free_fast_reg_page_list; 2121 2122 if (MLX5_CAP_GEN(mdev, xrc)) { 2123 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd; 2124 dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd; 2125 dev->ib_dev.uverbs_cmd_mask |= 2126 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) | 2127 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD); 2128 } 2129 2130 err = init_node_data(dev); 2131 if (err) 2132 goto err_disable_roce; 2133 2134 mutex_init(&dev->cap_mask_mutex); 2135 INIT_LIST_HEAD(&dev->qp_list); 2136 spin_lock_init(&dev->reset_flow_resource_lock); 2137 2138 err = create_dev_resources(&dev->devr); 2139 if (err) 2140 goto err_disable_roce; 2141 2142 2143 err = mlx5_ib_alloc_q_counters(dev); 2144 if (err) 2145 goto err_odp; 2146 2147 err = ib_register_device(&dev->ib_dev, NULL); 2148 if (err) 2149 goto err_q_cnt; 2150 2151 err = create_umr_res(dev); 2152 if (err) 2153 goto err_dev; 2154 2155 if (MLX5_CAP_GEN(dev->mdev, port_type) == 2156 MLX5_CAP_PORT_TYPE_IB) { 2157 if (init_dc_improvements(dev)) 2158 mlx5_ib_dbg(dev, "init_dc_improvements - continuing\n"); 2159 } 2160 2161 err = create_port_attrs(dev); 2162 if (err) 2163 goto err_dc; 2164 2165 for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) { 2166 err = device_create_file(&dev->ib_dev.dev, 2167 mlx5_class_attributes[i]); 2168 if (err) 2169 goto err_port_attrs; 2170 } 2171 2172 if (1) { 2173 struct thread *rl_thread = NULL; 2174 struct proc *rl_proc = NULL; 2175 2176 for (i = 0; i < MLX5_CAP_GEN(mdev, num_ports); i++) { 2177 (void) kproc_kthread_add(mlx5_ib_roce_port_update, dev->port + i, &rl_proc, &rl_thread, 2178 RFHIGHPID, 0, "mlx5-ib-roce-port", "mlx5-ib-roce_port-%d", i); 2179 } 2180 } 2181 2182 dev->ib_active = true; 2183 2184 return dev; 2185 2186err_port_attrs: 2187 destroy_ports_attrs(dev, dev->num_ports); 2188 2189err_dc: 2190 if (MLX5_CAP_GEN(dev->mdev, port_type) == 2191 MLX5_CAP_PORT_TYPE_IB) 2192 cleanup_dc_improvements(dev); 2193 destroy_umrc_res(dev); 2194 2195err_dev: 2196 ib_unregister_device(&dev->ib_dev); 2197 2198err_q_cnt: 2199 mlx5_ib_dealloc_q_counters(dev); 2200 2201err_odp: 2202 destroy_dev_resources(&dev->devr); 2203 2204err_disable_roce: 2205 if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) == 2206 IB_LINK_LAYER_ETHERNET && MLX5_CAP_GEN(mdev, roce)) 2207 mlx5_nic_vport_disable_roce(mdev); 2208err_free_port: 2209 kfree(dev->port); 2210 2211err_dealloc: 2212 ib_dealloc_device((struct ib_device *)dev); 2213 2214 return NULL; 2215} 2216 2217static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context) 2218{ 2219 struct mlx5_ib_dev *dev = context; 2220 int i; 2221 2222 for (i = 0; i < MLX5_CAP_GEN(mdev, num_ports); i++) { 2223 dev->port[i].port_gone = 1; 2224 while (dev->port[i].port_gone != 2) 2225 pause("W", hz); 2226 } 2227 2228 for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) { 2229 device_remove_file(&dev->ib_dev.dev, 2230 mlx5_class_attributes[i]); 2231 } 2232 2233 destroy_ports_attrs(dev, dev->num_ports); 2234 if (MLX5_CAP_GEN(dev->mdev, port_type) == 2235 MLX5_CAP_PORT_TYPE_IB) 2236 cleanup_dc_improvements(dev); 2237 mlx5_ib_dealloc_q_counters(dev); 2238 ib_unregister_device(&dev->ib_dev); 2239 destroy_umrc_res(dev); 2240 destroy_dev_resources(&dev->devr); 2241 2242 if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) == 2243 IB_LINK_LAYER_ETHERNET && MLX5_CAP_GEN(mdev, roce)) 2244 mlx5_nic_vport_disable_roce(mdev); 2245 2246 kfree(dev->port); 2247 ib_dealloc_device(&dev->ib_dev); 2248} 2249 2250static struct mlx5_interface mlx5_ib_interface = { 2251 .add = mlx5_ib_add, 2252 .remove = mlx5_ib_remove, 2253 .event = mlx5_ib_event, 2254 .protocol = MLX5_INTERFACE_PROTOCOL_IB, 2255}; 2256 2257static int __init mlx5_ib_init(void) 2258{ 2259 int err; 2260 2261 if (deprecated_prof_sel != 2) 2262 printf("mlx5_ib: WARN: ""prof_sel is deprecated for mlx5_ib, set it for mlx5_core\n"); 2263 2264 err = mlx5_register_interface(&mlx5_ib_interface); 2265 if (err) 2266 goto clean_odp; 2267 2268 mlx5_ib_wq = create_singlethread_workqueue("mlx5_ib_wq"); 2269 if (!mlx5_ib_wq) { 2270 printf("mlx5_ib: ERR: ""%s: failed to create mlx5_ib_wq\n", __func__); 2271 goto err_unreg; 2272 } 2273 2274 return err; 2275 2276err_unreg: 2277 mlx5_unregister_interface(&mlx5_ib_interface); 2278 2279clean_odp: 2280 return err; 2281} 2282 2283static void __exit mlx5_ib_cleanup(void) 2284{ 2285 destroy_workqueue(mlx5_ib_wq); 2286 mlx5_unregister_interface(&mlx5_ib_interface); 2287} 2288 2289module_init_order(mlx5_ib_init, SI_ORDER_THIRD); 2290module_exit_order(mlx5_ib_cleanup, SI_ORDER_THIRD); 2291