1219820Sjeff/* 2219820Sjeff * Copyright (c) 2007 Cisco, Inc. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff */ 32219820Sjeff 33219820Sjeff#if HAVE_CONFIG_H 34219820Sjeff# include <config.h> 35219820Sjeff#endif /* HAVE_CONFIG_H */ 36219820Sjeff 37219820Sjeff#include <stdio.h> 38219820Sjeff#include <stdlib.h> 39219820Sjeff#include <unistd.h> 40219820Sjeff#include <errno.h> 41219820Sjeff#include <sys/mman.h> 42219820Sjeff#include <pthread.h> 43219820Sjeff#include <string.h> 44219820Sjeff 45219820Sjeff#ifndef HAVE_IBV_REGISTER_DRIVER 46219820Sjeff#include <sysfs/libsysfs.h> 47219820Sjeff#endif 48219820Sjeff 49219820Sjeff#include "mlx4.h" 50219820Sjeff#include "mlx4-abi.h" 51219820Sjeff 52219820Sjeff#ifndef PCI_VENDOR_ID_MELLANOX 53219820Sjeff#define PCI_VENDOR_ID_MELLANOX 0x15b3 54219820Sjeff#endif 55219820Sjeff 56219820Sjeff#define HCA(v, d) \ 57219820Sjeff { .vendor = PCI_VENDOR_ID_##v, \ 58219820Sjeff .device = d } 59219820Sjeff 60219820Sjeffstruct { 61219820Sjeff unsigned vendor; 62219820Sjeff unsigned device; 63219820Sjeff} hca_table[] = { 64219820Sjeff HCA(MELLANOX, 0x6340), /* MT25408 "Hermon" SDR */ 65219820Sjeff HCA(MELLANOX, 0x634a), /* MT25408 "Hermon" DDR */ 66219820Sjeff HCA(MELLANOX, 0x6354), /* MT25408 "Hermon" QDR */ 67219820Sjeff HCA(MELLANOX, 0x6732), /* MT25408 "Hermon" DDR PCIe gen2 */ 68219820Sjeff HCA(MELLANOX, 0x673c), /* MT25408 "Hermon" QDR PCIe gen2 */ 69219820Sjeff HCA(MELLANOX, 0x6368), /* MT25448 [ConnectX EN 10GigE, PCIe 2.0 2.5GT/s] */ 70219820Sjeff HCA(MELLANOX, 0x6750), /* MT26448 [ConnectX EN 10GigE, PCIe 2.0 5GT/s] */ 71219820Sjeff HCA(MELLANOX, 0x6372), /* MT25408 [ConnectX EN 10GigE 10GBaseT, PCIe 2.0 2.5GT/s] */ 72219820Sjeff HCA(MELLANOX, 0x675a), /* MT25408 [ConnectX EN 10GigE 10GBaseT, PCIe Gen2 5GT/s] */ 73219820Sjeff HCA(MELLANOX, 0x6764), /* MT26468 [ConnectX EN 10GigE, PCIe 2.0 5GT/s] */ 74219820Sjeff HCA(MELLANOX, 0x6746), /* MT26438 ConnectX VPI PCIe 2.0 5GT/s - IB QDR / 10GigE Virt+ */ 75219820Sjeff HCA(MELLANOX, 0x676e), /* MT26478 ConnectX EN 40GigE PCIe 2.0 5GT/s */ 76219820Sjeff HCA(MELLANOX, 0x6778), /* MT26488 ConnectX VPI PCIe 2.0 5GT/s - IB DDR / 10GigE Virt+ */ 77219820Sjeff HCA(MELLANOX, 0x1000), 78219820Sjeff HCA(MELLANOX, 0x1001), 79219820Sjeff HCA(MELLANOX, 0x1002), 80219820Sjeff HCA(MELLANOX, 0x1003), 81219820Sjeff HCA(MELLANOX, 0x1004), 82219820Sjeff HCA(MELLANOX, 0x1005), 83219820Sjeff HCA(MELLANOX, 0x1006), 84219820Sjeff HCA(MELLANOX, 0x1007), 85219820Sjeff HCA(MELLANOX, 0x1008), 86219820Sjeff HCA(MELLANOX, 0x1009), 87219820Sjeff HCA(MELLANOX, 0x100a), 88219820Sjeff HCA(MELLANOX, 0x100b), 89219820Sjeff HCA(MELLANOX, 0x100c), 90219820Sjeff HCA(MELLANOX, 0x100d), 91219820Sjeff HCA(MELLANOX, 0x100e), 92219820Sjeff HCA(MELLANOX, 0x100f), 93219820Sjeff}; 94219820Sjeff 95219820Sjeff#ifdef HAVE_IBV_MORE_OPS 96219820Sjeffstatic struct ibv_more_ops mlx4_more_ops = { 97219820Sjeff#ifdef HAVE_IBV_XRC_OPS 98219820Sjeff .create_xrc_srq = mlx4_create_xrc_srq, 99219820Sjeff .open_xrc_domain = mlx4_open_xrc_domain, 100219820Sjeff .close_xrc_domain = mlx4_close_xrc_domain, 101219820Sjeff .create_xrc_rcv_qp = mlx4_create_xrc_rcv_qp, 102219820Sjeff .modify_xrc_rcv_qp = mlx4_modify_xrc_rcv_qp, 103219820Sjeff .query_xrc_rcv_qp = mlx4_query_xrc_rcv_qp, 104219820Sjeff .reg_xrc_rcv_qp = mlx4_reg_xrc_rcv_qp, 105219820Sjeff .unreg_xrc_rcv_qp = mlx4_unreg_xrc_rcv_qp, 106219820Sjeff#endif 107219820Sjeff}; 108219820Sjeff#endif 109219820Sjeff 110219820Sjeffstatic struct ibv_context_ops mlx4_ctx_ops = { 111219820Sjeff .query_device = mlx4_query_device, 112219820Sjeff .query_port = mlx4_query_port, 113219820Sjeff .alloc_pd = mlx4_alloc_pd, 114219820Sjeff .dealloc_pd = mlx4_free_pd, 115219820Sjeff .reg_mr = mlx4_reg_mr, 116219820Sjeff .dereg_mr = mlx4_dereg_mr, 117219820Sjeff .create_cq = mlx4_create_cq, 118219820Sjeff .poll_cq = mlx4_poll_cq, 119219820Sjeff .req_notify_cq = mlx4_arm_cq, 120219820Sjeff .cq_event = mlx4_cq_event, 121219820Sjeff .resize_cq = mlx4_resize_cq, 122219820Sjeff .destroy_cq = mlx4_destroy_cq, 123219820Sjeff .create_srq = mlx4_create_srq, 124219820Sjeff .modify_srq = mlx4_modify_srq, 125219820Sjeff .query_srq = mlx4_query_srq, 126219820Sjeff .destroy_srq = mlx4_destroy_srq, 127219820Sjeff .post_srq_recv = mlx4_post_srq_recv, 128219820Sjeff .create_qp = mlx4_create_qp, 129219820Sjeff .query_qp = mlx4_query_qp, 130219820Sjeff .modify_qp = mlx4_modify_qp, 131219820Sjeff .destroy_qp = mlx4_destroy_qp, 132219820Sjeff .post_send = mlx4_post_send, 133219820Sjeff .post_recv = mlx4_post_recv, 134219820Sjeff .create_ah = mlx4_create_ah, 135219820Sjeff .destroy_ah = mlx4_destroy_ah, 136219820Sjeff .attach_mcast = ibv_cmd_attach_mcast, 137219820Sjeff .detach_mcast = ibv_cmd_detach_mcast 138219820Sjeff}; 139219820Sjeff 140219820Sjeffstatic struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_fd) 141219820Sjeff{ 142219820Sjeff struct mlx4_context *context; 143219820Sjeff struct ibv_get_context cmd; 144219820Sjeff struct mlx4_alloc_ucontext_resp resp; 145277160Shselasky struct mlx4_alloc_ucontext_resp_v3 resp_v3; 146219820Sjeff int i; 147219820Sjeff struct ibv_device_attr dev_attrs; 148277160Shselasky unsigned int bf_reg_size; 149219820Sjeff 150219820Sjeff context = calloc(1, sizeof *context); 151219820Sjeff if (!context) 152219820Sjeff return NULL; 153219820Sjeff 154219820Sjeff context->ibv_ctx.cmd_fd = cmd_fd; 155219820Sjeff 156277160Shselasky if (to_mdev(ibdev)->driver_abi_ver > 3) { 157277160Shselasky if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd, 158277160Shselasky &resp.ibv_resp, sizeof resp)) 159277160Shselasky goto err_free; 160219820Sjeff 161277160Shselasky context->num_qps = resp.qp_tab_size; 162277160Shselasky context->num_xrc_srqs = resp.qp_tab_size; 163277160Shselasky bf_reg_size = resp.bf_reg_size; 164277160Shselasky context->cqe_size = resp.cqe_size; 165277160Shselasky } else { 166277160Shselasky if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd, 167277160Shselasky &resp_v3.ibv_resp, sizeof resp_v3)) 168277160Shselasky goto err_free; 169277160Shselasky 170277160Shselasky context->num_qps = resp_v3.qp_tab_size; 171277160Shselasky context->num_xrc_srqs = resp_v3.qp_tab_size; 172277160Shselasky bf_reg_size = resp_v3.bf_reg_size; 173277160Shselasky context->cqe_size = 32; 174277160Shselasky } 175277160Shselasky 176219820Sjeff context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS; 177219820Sjeff context->qp_table_mask = (1 << context->qp_table_shift) - 1; 178219820Sjeff 179219820Sjeff pthread_mutex_init(&context->qp_table_mutex, NULL); 180219820Sjeff for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i) 181219820Sjeff context->qp_table[i].refcnt = 0; 182219820Sjeff 183219820Sjeff context->xrc_srq_table_shift = ffs(context->num_xrc_srqs) - 1 184219820Sjeff - MLX4_XRC_SRQ_TABLE_BITS; 185219820Sjeff context->xrc_srq_table_mask = (1 << context->xrc_srq_table_shift) - 1; 186219820Sjeff 187219820Sjeff pthread_mutex_init(&context->xrc_srq_table_mutex, NULL); 188219820Sjeff for (i = 0; i < MLX4_XRC_SRQ_TABLE_SIZE; ++i) 189219820Sjeff context->xrc_srq_table[i].refcnt = 0; 190219820Sjeff 191219820Sjeff for (i = 0; i < MLX4_NUM_DB_TYPE; ++i) 192219820Sjeff context->db_list[i] = NULL; 193219820Sjeff 194219820Sjeff pthread_mutex_init(&context->db_list_mutex, NULL); 195219820Sjeff 196219820Sjeff context->uar = mmap(NULL, to_mdev(ibdev)->page_size, PROT_WRITE, 197219820Sjeff MAP_SHARED, cmd_fd, 0); 198219820Sjeff if (context->uar == MAP_FAILED) 199219820Sjeff goto err_free; 200219820Sjeff 201277160Shselasky if (bf_reg_size) { 202219820Sjeff context->bf_page = mmap(NULL, to_mdev(ibdev)->page_size, 203219820Sjeff PROT_WRITE, MAP_SHARED, cmd_fd, 204219820Sjeff to_mdev(ibdev)->page_size); 205219820Sjeff if (context->bf_page == MAP_FAILED) { 206219820Sjeff fprintf(stderr, PFX "Warning: BlueFlame available, " 207219820Sjeff "but failed to mmap() BlueFlame page.\n"); 208219820Sjeff context->bf_page = NULL; 209219820Sjeff context->bf_buf_size = 0; 210219820Sjeff } else { 211277160Shselasky context->bf_buf_size = bf_reg_size / 2; 212219820Sjeff context->bf_offset = 0; 213219820Sjeff pthread_spin_init(&context->bf_lock, PTHREAD_PROCESS_PRIVATE); 214219820Sjeff } 215219820Sjeff } else { 216219820Sjeff context->bf_page = NULL; 217219820Sjeff context->bf_buf_size = 0; 218219820Sjeff } 219219820Sjeff 220219820Sjeff pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE); 221219820Sjeff 222219820Sjeff context->ibv_ctx.ops = mlx4_ctx_ops; 223219820Sjeff#ifdef HAVE_IBV_XRC_OPS 224219820Sjeff context->ibv_ctx.more_ops = &mlx4_more_ops; 225219820Sjeff#endif 226219820Sjeff 227219820Sjeff if (mlx4_query_device(&context->ibv_ctx, &dev_attrs)) 228219820Sjeff goto query_free; 229219820Sjeff 230219820Sjeff context->max_qp_wr = dev_attrs.max_qp_wr; 231219820Sjeff context->max_sge = dev_attrs.max_sge; 232219820Sjeff context->max_cqe = dev_attrs.max_cqe; 233219820Sjeff if (!(dev_attrs.device_cap_flags & IBV_DEVICE_XRC)) { 234219820Sjeff fprintf(stderr, PFX "There is a mismatch between " 235219820Sjeff "the kernel and the userspace libraries: " 236219820Sjeff "Kernel does not support XRC. Exiting.\n"); 237219820Sjeff goto query_free; 238219820Sjeff } 239219820Sjeff 240219820Sjeff return &context->ibv_ctx; 241219820Sjeff 242219820Sjeffquery_free: 243219820Sjeff munmap(context->uar, to_mdev(ibdev)->page_size); 244219820Sjeff if (context->bf_page) 245219820Sjeff munmap(context->bf_page, to_mdev(ibdev)->page_size); 246219820Sjeff 247219820Sjefferr_free: 248219820Sjeff free(context); 249219820Sjeff return NULL; 250219820Sjeff} 251219820Sjeff 252219820Sjeffstatic void mlx4_free_context(struct ibv_context *ibctx) 253219820Sjeff{ 254219820Sjeff struct mlx4_context *context = to_mctx(ibctx); 255219820Sjeff 256219820Sjeff munmap(context->uar, to_mdev(ibctx->device)->page_size); 257219820Sjeff if (context->bf_page) 258219820Sjeff munmap(context->bf_page, to_mdev(ibctx->device)->page_size); 259219820Sjeff free(context); 260219820Sjeff} 261219820Sjeff 262219820Sjeffstatic struct ibv_device_ops mlx4_dev_ops = { 263219820Sjeff .alloc_context = mlx4_alloc_context, 264219820Sjeff .free_context = mlx4_free_context 265219820Sjeff}; 266219820Sjeff 267219820Sjeffstatic struct ibv_device *mlx4_driver_init(const char *uverbs_sys_path, 268219820Sjeff int abi_version) 269219820Sjeff{ 270219820Sjeff char value[8]; 271219820Sjeff struct mlx4_device *dev; 272219820Sjeff unsigned vendor, device; 273219820Sjeff int i; 274219820Sjeff 275219820Sjeff if (ibv_read_sysfs_file(uverbs_sys_path, "device/vendor", 276219820Sjeff value, sizeof value) < 0) 277219820Sjeff return NULL; 278219820Sjeff sscanf(value, "%i", &vendor); 279219820Sjeff 280219820Sjeff if (ibv_read_sysfs_file(uverbs_sys_path, "device/device", 281219820Sjeff value, sizeof value) < 0) 282219820Sjeff return NULL; 283219820Sjeff sscanf(value, "%i", &device); 284219820Sjeff 285219820Sjeff for (i = 0; i < sizeof hca_table / sizeof hca_table[0]; ++i) 286219820Sjeff if (vendor == hca_table[i].vendor && 287219820Sjeff device == hca_table[i].device) 288219820Sjeff goto found; 289219820Sjeff 290219820Sjeff return NULL; 291219820Sjeff 292219820Sjefffound: 293219820Sjeff if (abi_version < MLX4_UVERBS_MIN_ABI_VERSION || 294219820Sjeff abi_version > MLX4_UVERBS_MAX_ABI_VERSION) { 295219820Sjeff fprintf(stderr, PFX "Fatal: ABI version %d of %s is not supported " 296219820Sjeff "(min supported %d, max supported %d)\n", 297219820Sjeff abi_version, uverbs_sys_path, 298219820Sjeff MLX4_UVERBS_MIN_ABI_VERSION, 299219820Sjeff MLX4_UVERBS_MAX_ABI_VERSION); 300219820Sjeff return NULL; 301219820Sjeff } 302219820Sjeff 303219820Sjeff dev = malloc(sizeof *dev); 304219820Sjeff if (!dev) { 305219820Sjeff fprintf(stderr, PFX "Fatal: couldn't allocate device for %s\n", 306219820Sjeff uverbs_sys_path); 307219820Sjeff return NULL; 308219820Sjeff } 309219820Sjeff 310219820Sjeff dev->ibv_dev.ops = mlx4_dev_ops; 311219820Sjeff dev->page_size = sysconf(_SC_PAGESIZE); 312277160Shselasky dev->driver_abi_ver = abi_version; 313219820Sjeff 314219820Sjeff return &dev->ibv_dev; 315219820Sjeff} 316219820Sjeff 317219820Sjeff#ifdef HAVE_IBV_REGISTER_DRIVER 318219820Sjeffstatic __attribute__((constructor)) void mlx4_register_driver(void) 319219820Sjeff{ 320219820Sjeff ibv_register_driver("mlx4", mlx4_driver_init); 321219820Sjeff} 322219820Sjeff#else 323219820Sjeff/* 324219820Sjeff * Export the old libsysfs sysfs_class_device-based driver entry point 325219820Sjeff * if libibverbs does not export an ibv_register_driver() function. 326219820Sjeff */ 327219820Sjeffstruct ibv_device *openib_driver_init(struct sysfs_class_device *sysdev) 328219820Sjeff{ 329219820Sjeff int abi_ver = 0; 330219820Sjeff char value[8]; 331219820Sjeff 332219820Sjeff if (ibv_read_sysfs_file(sysdev->path, "abi_version", 333219820Sjeff value, sizeof value) > 0) 334219820Sjeff abi_ver = strtol(value, NULL, 10); 335219820Sjeff 336219820Sjeff return mlx4_driver_init(sysdev->path, abi_ver); 337219820Sjeff} 338219820Sjeff#endif /* HAVE_IBV_REGISTER_DRIVER */ 339