1/* 2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 5 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36#include <linux/module.h> 37#include <linux/init.h> 38#include <linux/errno.h> 39#include <linux/pci.h> 40#include <linux/dma-mapping.h> 41 42#include <linux/mlx4/device.h> 43#include <linux/mlx4/doorbell.h> 44 45#include "mlx4.h" 46#include "fw.h" 47#include "icm.h" 48 49MODULE_AUTHOR("Roland Dreier"); 50MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver"); 51MODULE_LICENSE("Dual BSD/GPL"); 52MODULE_VERSION(DRV_VERSION); 53 54#ifdef CONFIG_MLX4_DEBUG 55 56int mlx4_debug_level = 0; 57module_param_named(debug_level, mlx4_debug_level, int, 0644); 58MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); 59 60#endif /* CONFIG_MLX4_DEBUG */ 61 62#ifdef CONFIG_PCI_MSI 63 64static int msi_x; 65module_param(msi_x, int, 0444); 66MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero"); 67 68#else /* CONFIG_PCI_MSI */ 69 70#define msi_x (0) 71 72#endif /* CONFIG_PCI_MSI */ 73 74static const char mlx4_version[] __devinitdata = 75 DRV_NAME ": Mellanox ConnectX core driver v" 76 DRV_VERSION " (" DRV_RELDATE ")\n"; 77 78static struct mlx4_profile default_profile = { 79 .num_qp = 1 << 16, 80 .num_srq = 1 << 16, 81 .rdmarc_per_qp = 4, 82 .num_cq = 1 << 16, 83 .num_mcg = 1 << 13, 84 .num_mpt = 1 << 17, 85 .num_mtt = 1 << 20, 86}; 87 88static int __devinit mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) 89{ 90 int err; 91 int i; 92 93 err = mlx4_QUERY_DEV_CAP(dev, dev_cap); 94 if (err) { 95 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); 96 return err; 97 } 98 99 if (dev_cap->min_page_sz > PAGE_SIZE) { 100 mlx4_err(dev, "HCA minimum page size of %d bigger than " 101 "kernel PAGE_SIZE of %ld, aborting.\n", 102 dev_cap->min_page_sz, PAGE_SIZE); 103 return -ENODEV; 104 } 105 if (dev_cap->num_ports > MLX4_MAX_PORTS) { 106 mlx4_err(dev, "HCA has %d ports, but we only support %d, " 107 "aborting.\n", 108 dev_cap->num_ports, MLX4_MAX_PORTS); 109 return -ENODEV; 110 } 111 112 if (dev_cap->uar_size > pci_resource_len(dev->pdev, 2)) { 113 mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than " 114 "PCI resource 2 size of 0x%llx, aborting.\n", 115 dev_cap->uar_size, 116 (unsigned long long) pci_resource_len(dev->pdev, 2)); 117 return -ENODEV; 118 } 119 120 dev->caps.num_ports = dev_cap->num_ports; 121 for (i = 1; i <= dev->caps.num_ports; ++i) { 122 dev->caps.vl_cap[i] = dev_cap->max_vl[i]; 123 dev->caps.mtu_cap[i] = dev_cap->max_mtu[i]; 124 dev->caps.gid_table_len[i] = dev_cap->max_gids[i]; 125 dev->caps.pkey_table_len[i] = dev_cap->max_pkeys[i]; 126 dev->caps.port_width_cap[i] = dev_cap->max_port_width[i]; 127 } 128 129 dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE; 130 dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay; 131 dev->caps.bf_reg_size = dev_cap->bf_reg_size; 132 dev->caps.bf_regs_per_page = dev_cap->bf_regs_per_page; 133 dev->caps.max_sq_sg = dev_cap->max_sq_sg; 134 dev->caps.max_rq_sg = dev_cap->max_rq_sg; 135 dev->caps.max_wqes = dev_cap->max_qp_sz; 136 dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp; 137 dev->caps.reserved_qps = dev_cap->reserved_qps; 138 dev->caps.max_srq_wqes = dev_cap->max_srq_sz; 139 dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1; 140 dev->caps.reserved_srqs = dev_cap->reserved_srqs; 141 dev->caps.max_sq_desc_sz = dev_cap->max_sq_desc_sz; 142 dev->caps.max_rq_desc_sz = dev_cap->max_rq_desc_sz; 143 dev->caps.num_qp_per_mgm = MLX4_QP_PER_MGM; 144 /* 145 * Subtract 1 from the limit because we need to allocate a 146 * spare CQE so the HCA HW can tell the difference between an 147 * empty CQ and a full CQ. 148 */ 149 dev->caps.max_cqes = dev_cap->max_cq_sz - 1; 150 dev->caps.reserved_cqs = dev_cap->reserved_cqs; 151 dev->caps.reserved_eqs = dev_cap->reserved_eqs; 152 dev->caps.reserved_mtts = dev_cap->reserved_mtts; 153 dev->caps.reserved_mrws = dev_cap->reserved_mrws; 154 dev->caps.reserved_uars = dev_cap->reserved_uars; 155 dev->caps.reserved_pds = dev_cap->reserved_pds; 156 dev->caps.mtt_entry_sz = MLX4_MTT_ENTRY_PER_SEG * dev_cap->mtt_entry_sz; 157 dev->caps.page_size_cap = ~(u32) (dev_cap->min_page_sz - 1); 158 dev->caps.flags = dev_cap->flags; 159 dev->caps.stat_rate_support = dev_cap->stat_rate_support; 160 161 return 0; 162} 163 164static int __devinit mlx4_load_fw(struct mlx4_dev *dev) 165{ 166 struct mlx4_priv *priv = mlx4_priv(dev); 167 int err; 168 169 priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages, 170 GFP_HIGHUSER | __GFP_NOWARN); 171 if (!priv->fw.fw_icm) { 172 mlx4_err(dev, "Couldn't allocate FW area, aborting.\n"); 173 return -ENOMEM; 174 } 175 176 err = mlx4_MAP_FA(dev, priv->fw.fw_icm); 177 if (err) { 178 mlx4_err(dev, "MAP_FA command failed, aborting.\n"); 179 goto err_free; 180 } 181 182 err = mlx4_RUN_FW(dev); 183 if (err) { 184 mlx4_err(dev, "RUN_FW command failed, aborting.\n"); 185 goto err_unmap_fa; 186 } 187 188 return 0; 189 190err_unmap_fa: 191 mlx4_UNMAP_FA(dev); 192 193err_free: 194 mlx4_free_icm(dev, priv->fw.fw_icm); 195 return err; 196} 197 198static int __devinit mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base, 199 int cmpt_entry_sz) 200{ 201 struct mlx4_priv *priv = mlx4_priv(dev); 202 int err; 203 204 err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table, 205 cmpt_base + 206 ((u64) (MLX4_CMPT_TYPE_QP * 207 cmpt_entry_sz) << MLX4_CMPT_SHIFT), 208 cmpt_entry_sz, dev->caps.num_qps, 209 dev->caps.reserved_qps, 0); 210 if (err) 211 goto err; 212 213 err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table, 214 cmpt_base + 215 ((u64) (MLX4_CMPT_TYPE_SRQ * 216 cmpt_entry_sz) << MLX4_CMPT_SHIFT), 217 cmpt_entry_sz, dev->caps.num_srqs, 218 dev->caps.reserved_srqs, 0); 219 if (err) 220 goto err_qp; 221 222 err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table, 223 cmpt_base + 224 ((u64) (MLX4_CMPT_TYPE_CQ * 225 cmpt_entry_sz) << MLX4_CMPT_SHIFT), 226 cmpt_entry_sz, dev->caps.num_cqs, 227 dev->caps.reserved_cqs, 0); 228 if (err) 229 goto err_srq; 230 231 err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table, 232 cmpt_base + 233 ((u64) (MLX4_CMPT_TYPE_EQ * 234 cmpt_entry_sz) << MLX4_CMPT_SHIFT), 235 cmpt_entry_sz, 236 roundup_pow_of_two(MLX4_NUM_EQ + 237 dev->caps.reserved_eqs), 238 MLX4_NUM_EQ + dev->caps.reserved_eqs, 0); 239 if (err) 240 goto err_cq; 241 242 return 0; 243 244err_cq: 245 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); 246 247err_srq: 248 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); 249 250err_qp: 251 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); 252 253err: 254 return err; 255} 256 257static int __devinit mlx4_init_icm(struct mlx4_dev *dev, 258 struct mlx4_dev_cap *dev_cap, 259 struct mlx4_init_hca_param *init_hca, 260 u64 icm_size) 261{ 262 struct mlx4_priv *priv = mlx4_priv(dev); 263 u64 aux_pages; 264 int err; 265 266 err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages); 267 if (err) { 268 mlx4_err(dev, "SET_ICM_SIZE command failed, aborting.\n"); 269 return err; 270 } 271 272 mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory.\n", 273 (unsigned long long) icm_size >> 10, 274 (unsigned long long) aux_pages << 2); 275 276 priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages, 277 GFP_HIGHUSER | __GFP_NOWARN); 278 if (!priv->fw.aux_icm) { 279 mlx4_err(dev, "Couldn't allocate aux memory, aborting.\n"); 280 return -ENOMEM; 281 } 282 283 err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm); 284 if (err) { 285 mlx4_err(dev, "MAP_ICM_AUX command failed, aborting.\n"); 286 goto err_free_aux; 287 } 288 289 err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz); 290 if (err) { 291 mlx4_err(dev, "Failed to map cMPT context memory, aborting.\n"); 292 goto err_unmap_aux; 293 } 294 295 err = mlx4_map_eq_icm(dev, init_hca->eqc_base); 296 if (err) { 297 mlx4_err(dev, "Failed to map EQ context memory, aborting.\n"); 298 goto err_unmap_cmpt; 299 } 300 301 err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table, 302 init_hca->mtt_base, 303 dev->caps.mtt_entry_sz, 304 dev->caps.num_mtt_segs, 305 dev->caps.reserved_mtts, 1); 306 if (err) { 307 mlx4_err(dev, "Failed to map MTT context memory, aborting.\n"); 308 goto err_unmap_eq; 309 } 310 311 err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table, 312 init_hca->dmpt_base, 313 dev_cap->dmpt_entry_sz, 314 dev->caps.num_mpts, 315 dev->caps.reserved_mrws, 1); 316 if (err) { 317 mlx4_err(dev, "Failed to map dMPT context memory, aborting.\n"); 318 goto err_unmap_mtt; 319 } 320 321 err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table, 322 init_hca->qpc_base, 323 dev_cap->qpc_entry_sz, 324 dev->caps.num_qps, 325 dev->caps.reserved_qps, 0); 326 if (err) { 327 mlx4_err(dev, "Failed to map QP context memory, aborting.\n"); 328 goto err_unmap_dmpt; 329 } 330 331 err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table, 332 init_hca->auxc_base, 333 dev_cap->aux_entry_sz, 334 dev->caps.num_qps, 335 dev->caps.reserved_qps, 0); 336 if (err) { 337 mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n"); 338 goto err_unmap_qp; 339 } 340 341 err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table, 342 init_hca->altc_base, 343 dev_cap->altc_entry_sz, 344 dev->caps.num_qps, 345 dev->caps.reserved_qps, 0); 346 if (err) { 347 mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n"); 348 goto err_unmap_auxc; 349 } 350 351 err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table, 352 init_hca->rdmarc_base, 353 dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift, 354 dev->caps.num_qps, 355 dev->caps.reserved_qps, 0); 356 if (err) { 357 mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n"); 358 goto err_unmap_altc; 359 } 360 361 err = mlx4_init_icm_table(dev, &priv->cq_table.table, 362 init_hca->cqc_base, 363 dev_cap->cqc_entry_sz, 364 dev->caps.num_cqs, 365 dev->caps.reserved_cqs, 0); 366 if (err) { 367 mlx4_err(dev, "Failed to map CQ context memory, aborting.\n"); 368 goto err_unmap_rdmarc; 369 } 370 371 err = mlx4_init_icm_table(dev, &priv->srq_table.table, 372 init_hca->srqc_base, 373 dev_cap->srq_entry_sz, 374 dev->caps.num_srqs, 375 dev->caps.reserved_srqs, 0); 376 if (err) { 377 mlx4_err(dev, "Failed to map SRQ context memory, aborting.\n"); 378 goto err_unmap_cq; 379 } 380 381 /* 382 * It's not strictly required, but for simplicity just map the 383 * whole multicast group table now. The table isn't very big 384 * and it's a lot easier than trying to track ref counts. 385 */ 386 err = mlx4_init_icm_table(dev, &priv->mcg_table.table, 387 init_hca->mc_base, MLX4_MGM_ENTRY_SIZE, 388 dev->caps.num_mgms + dev->caps.num_amgms, 389 dev->caps.num_mgms + dev->caps.num_amgms, 390 0); 391 if (err) { 392 mlx4_err(dev, "Failed to map MCG context memory, aborting.\n"); 393 goto err_unmap_srq; 394 } 395 396 return 0; 397 398err_unmap_srq: 399 mlx4_cleanup_icm_table(dev, &priv->srq_table.table); 400 401err_unmap_cq: 402 mlx4_cleanup_icm_table(dev, &priv->cq_table.table); 403 404err_unmap_rdmarc: 405 mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); 406 407err_unmap_altc: 408 mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); 409 410err_unmap_auxc: 411 mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); 412 413err_unmap_qp: 414 mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); 415 416err_unmap_dmpt: 417 mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); 418 419err_unmap_mtt: 420 mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); 421 422err_unmap_eq: 423 mlx4_unmap_eq_icm(dev); 424 425err_unmap_cmpt: 426 mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); 427 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); 428 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); 429 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); 430 431err_unmap_aux: 432 mlx4_UNMAP_ICM_AUX(dev); 433 434err_free_aux: 435 mlx4_free_icm(dev, priv->fw.aux_icm); 436 437 return err; 438} 439 440static void mlx4_free_icms(struct mlx4_dev *dev) 441{ 442 struct mlx4_priv *priv = mlx4_priv(dev); 443 444 mlx4_cleanup_icm_table(dev, &priv->mcg_table.table); 445 mlx4_cleanup_icm_table(dev, &priv->srq_table.table); 446 mlx4_cleanup_icm_table(dev, &priv->cq_table.table); 447 mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); 448 mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); 449 mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); 450 mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); 451 mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); 452 mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); 453 mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); 454 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); 455 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); 456 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); 457 mlx4_unmap_eq_icm(dev); 458 459 mlx4_UNMAP_ICM_AUX(dev); 460 mlx4_free_icm(dev, priv->fw.aux_icm); 461} 462 463static void mlx4_close_hca(struct mlx4_dev *dev) 464{ 465 mlx4_CLOSE_HCA(dev, 0); 466 mlx4_free_icms(dev); 467 mlx4_UNMAP_FA(dev); 468 mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm); 469} 470 471static int __devinit mlx4_init_hca(struct mlx4_dev *dev) 472{ 473 struct mlx4_priv *priv = mlx4_priv(dev); 474 struct mlx4_adapter adapter; 475 struct mlx4_dev_cap dev_cap; 476 struct mlx4_profile profile; 477 struct mlx4_init_hca_param init_hca; 478 u64 icm_size; 479 int err; 480 481 err = mlx4_QUERY_FW(dev); 482 if (err) { 483 mlx4_err(dev, "QUERY_FW command failed, aborting.\n"); 484 return err; 485 } 486 487 err = mlx4_load_fw(dev); 488 if (err) { 489 mlx4_err(dev, "Failed to start FW, aborting.\n"); 490 return err; 491 } 492 493 err = mlx4_dev_cap(dev, &dev_cap); 494 if (err) { 495 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); 496 goto err_stop_fw; 497 } 498 499 profile = default_profile; 500 501 icm_size = mlx4_make_profile(dev, &profile, &dev_cap, &init_hca); 502 if ((long long) icm_size < 0) { 503 err = icm_size; 504 goto err_stop_fw; 505 } 506 507 init_hca.log_uar_sz = ilog2(dev->caps.num_uars); 508 509 err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size); 510 if (err) 511 goto err_stop_fw; 512 513 err = mlx4_INIT_HCA(dev, &init_hca); 514 if (err) { 515 mlx4_err(dev, "INIT_HCA command failed, aborting.\n"); 516 goto err_free_icm; 517 } 518 519 err = mlx4_QUERY_ADAPTER(dev, &adapter); 520 if (err) { 521 mlx4_err(dev, "QUERY_ADAPTER command failed, aborting.\n"); 522 goto err_close; 523 } 524 525 priv->eq_table.inta_pin = adapter.inta_pin; 526 priv->rev_id = adapter.revision_id; 527 memcpy(priv->board_id, adapter.board_id, sizeof priv->board_id); 528 529 return 0; 530 531err_close: 532 mlx4_close_hca(dev); 533 534err_free_icm: 535 mlx4_free_icms(dev); 536 537err_stop_fw: 538 mlx4_UNMAP_FA(dev); 539 mlx4_free_icm(dev, priv->fw.fw_icm); 540 541 return err; 542} 543 544static int __devinit mlx4_setup_hca(struct mlx4_dev *dev) 545{ 546 struct mlx4_priv *priv = mlx4_priv(dev); 547 int err; 548 549 err = mlx4_init_uar_table(dev); 550 if (err) { 551 mlx4_err(dev, "Failed to initialize " 552 "user access region table, aborting.\n"); 553 return err; 554 } 555 556 err = mlx4_uar_alloc(dev, &priv->driver_uar); 557 if (err) { 558 mlx4_err(dev, "Failed to allocate driver access region, " 559 "aborting.\n"); 560 goto err_uar_table_free; 561 } 562 563 priv->kar = ioremap(priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); 564 if (!priv->kar) { 565 mlx4_err(dev, "Couldn't map kernel access region, " 566 "aborting.\n"); 567 err = -ENOMEM; 568 goto err_uar_free; 569 } 570 571 err = mlx4_init_pd_table(dev); 572 if (err) { 573 mlx4_err(dev, "Failed to initialize " 574 "protection domain table, aborting.\n"); 575 goto err_kar_unmap; 576 } 577 578 err = mlx4_init_mr_table(dev); 579 if (err) { 580 mlx4_err(dev, "Failed to initialize " 581 "memory region table, aborting.\n"); 582 goto err_pd_table_free; 583 } 584 585 mlx4_map_catas_buf(dev); 586 587 err = mlx4_init_eq_table(dev); 588 if (err) { 589 mlx4_err(dev, "Failed to initialize " 590 "event queue table, aborting.\n"); 591 goto err_catas_buf; 592 } 593 594 err = mlx4_cmd_use_events(dev); 595 if (err) { 596 mlx4_err(dev, "Failed to switch to event-driven " 597 "firmware commands, aborting.\n"); 598 goto err_eq_table_free; 599 } 600 601 err = mlx4_NOP(dev); 602 if (err) { 603 mlx4_err(dev, "NOP command failed to generate interrupt " 604 "(IRQ %d), aborting.\n", 605 priv->eq_table.eq[MLX4_EQ_ASYNC].irq); 606 if (dev->flags & MLX4_FLAG_MSI_X) 607 mlx4_err(dev, "Try again with MSI-X disabled.\n"); 608 else 609 mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n"); 610 611 goto err_cmd_poll; 612 } 613 614 mlx4_dbg(dev, "NOP command IRQ test passed\n"); 615 616 err = mlx4_init_cq_table(dev); 617 if (err) { 618 mlx4_err(dev, "Failed to initialize " 619 "completion queue table, aborting.\n"); 620 goto err_cmd_poll; 621 } 622 623 err = mlx4_init_srq_table(dev); 624 if (err) { 625 mlx4_err(dev, "Failed to initialize " 626 "shared receive queue table, aborting.\n"); 627 goto err_cq_table_free; 628 } 629 630 err = mlx4_init_qp_table(dev); 631 if (err) { 632 mlx4_err(dev, "Failed to initialize " 633 "queue pair table, aborting.\n"); 634 goto err_srq_table_free; 635 } 636 637 err = mlx4_init_mcg_table(dev); 638 if (err) { 639 mlx4_err(dev, "Failed to initialize " 640 "multicast group table, aborting.\n"); 641 goto err_qp_table_free; 642 } 643 644 return 0; 645 646err_qp_table_free: 647 mlx4_cleanup_qp_table(dev); 648 649err_srq_table_free: 650 mlx4_cleanup_srq_table(dev); 651 652err_cq_table_free: 653 mlx4_cleanup_cq_table(dev); 654 655err_cmd_poll: 656 mlx4_cmd_use_polling(dev); 657 658err_eq_table_free: 659 mlx4_cleanup_eq_table(dev); 660 661err_catas_buf: 662 mlx4_unmap_catas_buf(dev); 663 mlx4_cleanup_mr_table(dev); 664 665err_pd_table_free: 666 mlx4_cleanup_pd_table(dev); 667 668err_kar_unmap: 669 iounmap(priv->kar); 670 671err_uar_free: 672 mlx4_uar_free(dev, &priv->driver_uar); 673 674err_uar_table_free: 675 mlx4_cleanup_uar_table(dev); 676 return err; 677} 678 679static void __devinit mlx4_enable_msi_x(struct mlx4_dev *dev) 680{ 681 struct mlx4_priv *priv = mlx4_priv(dev); 682 struct msix_entry entries[MLX4_NUM_EQ]; 683 int err; 684 int i; 685 686 if (msi_x) { 687 for (i = 0; i < MLX4_NUM_EQ; ++i) 688 entries[i].entry = i; 689 690 err = pci_enable_msix(dev->pdev, entries, ARRAY_SIZE(entries)); 691 if (err) { 692 if (err > 0) 693 mlx4_info(dev, "Only %d MSI-X vectors available, " 694 "not using MSI-X\n", err); 695 goto no_msi; 696 } 697 698 for (i = 0; i < MLX4_NUM_EQ; ++i) 699 priv->eq_table.eq[i].irq = entries[i].vector; 700 701 dev->flags |= MLX4_FLAG_MSI_X; 702 return; 703 } 704 705no_msi: 706 for (i = 0; i < MLX4_NUM_EQ; ++i) 707 priv->eq_table.eq[i].irq = dev->pdev->irq; 708} 709 710static int __devinit mlx4_init_one(struct pci_dev *pdev, 711 const struct pci_device_id *id) 712{ 713 static int mlx4_version_printed; 714 struct mlx4_priv *priv; 715 struct mlx4_dev *dev; 716 int err; 717 718 if (!mlx4_version_printed) { 719 printk(KERN_INFO "%s", mlx4_version); 720 ++mlx4_version_printed; 721 } 722 723 printk(KERN_INFO PFX "Initializing %s\n", 724 pci_name(pdev)); 725 726 err = pci_enable_device(pdev); 727 if (err) { 728 dev_err(&pdev->dev, "Cannot enable PCI device, " 729 "aborting.\n"); 730 return err; 731 } 732 733 /* 734 * Check for BARs. We expect 0: 1MB, 2: 8MB, 4: DDR (may not 735 * be present) 736 */ 737 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || 738 pci_resource_len(pdev, 0) != 1 << 20) { 739 dev_err(&pdev->dev, "Missing DCS, aborting.\n"); 740 err = -ENODEV; 741 goto err_disable_pdev; 742 } 743 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { 744 dev_err(&pdev->dev, "Missing UAR, aborting.\n"); 745 err = -ENODEV; 746 goto err_disable_pdev; 747 } 748 749 err = pci_request_region(pdev, 0, DRV_NAME); 750 if (err) { 751 dev_err(&pdev->dev, "Cannot request control region, aborting.\n"); 752 goto err_disable_pdev; 753 } 754 755 err = pci_request_region(pdev, 2, DRV_NAME); 756 if (err) { 757 dev_err(&pdev->dev, "Cannot request UAR region, aborting.\n"); 758 goto err_release_bar0; 759 } 760 761 pci_set_master(pdev); 762 763 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); 764 if (err) { 765 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n"); 766 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); 767 if (err) { 768 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n"); 769 goto err_release_bar2; 770 } 771 } 772 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK); 773 if (err) { 774 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit " 775 "consistent PCI DMA mask.\n"); 776 err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK); 777 if (err) { 778 dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, " 779 "aborting.\n"); 780 goto err_release_bar2; 781 } 782 } 783 784 priv = kzalloc(sizeof *priv, GFP_KERNEL); 785 if (!priv) { 786 dev_err(&pdev->dev, "Device struct alloc failed, " 787 "aborting.\n"); 788 err = -ENOMEM; 789 goto err_release_bar2; 790 } 791 792 dev = &priv->dev; 793 dev->pdev = pdev; 794 INIT_LIST_HEAD(&priv->ctx_list); 795 spin_lock_init(&priv->ctx_lock); 796 797 /* 798 * Now reset the HCA before we touch the PCI capabilities or 799 * attempt a firmware command, since a boot ROM may have left 800 * the HCA in an undefined state. 801 */ 802 err = mlx4_reset(dev); 803 if (err) { 804 mlx4_err(dev, "Failed to reset HCA, aborting.\n"); 805 goto err_free_dev; 806 } 807 808 mlx4_enable_msi_x(dev); 809 810 if (mlx4_cmd_init(dev)) { 811 mlx4_err(dev, "Failed to init command interface, aborting.\n"); 812 goto err_free_dev; 813 } 814 815 err = mlx4_init_hca(dev); 816 if (err) 817 goto err_cmd; 818 819 err = mlx4_setup_hca(dev); 820 if (err) 821 goto err_close; 822 823 err = mlx4_register_device(dev); 824 if (err) 825 goto err_cleanup; 826 827 pci_set_drvdata(pdev, dev); 828 829 return 0; 830 831err_cleanup: 832 mlx4_cleanup_mcg_table(dev); 833 mlx4_cleanup_qp_table(dev); 834 mlx4_cleanup_srq_table(dev); 835 mlx4_cleanup_cq_table(dev); 836 mlx4_cmd_use_polling(dev); 837 mlx4_cleanup_eq_table(dev); 838 839 mlx4_unmap_catas_buf(dev); 840 841 mlx4_cleanup_mr_table(dev); 842 mlx4_cleanup_pd_table(dev); 843 mlx4_cleanup_uar_table(dev); 844 845err_close: 846 mlx4_close_hca(dev); 847 848err_cmd: 849 mlx4_cmd_cleanup(dev); 850 851err_free_dev: 852 if (dev->flags & MLX4_FLAG_MSI_X) 853 pci_disable_msix(pdev); 854 855 kfree(priv); 856 857err_release_bar2: 858 pci_release_region(pdev, 2); 859 860err_release_bar0: 861 pci_release_region(pdev, 0); 862 863err_disable_pdev: 864 pci_disable_device(pdev); 865 pci_set_drvdata(pdev, NULL); 866 return err; 867} 868 869static void __devexit mlx4_remove_one(struct pci_dev *pdev) 870{ 871 struct mlx4_dev *dev = pci_get_drvdata(pdev); 872 struct mlx4_priv *priv = mlx4_priv(dev); 873 int p; 874 875 if (dev) { 876 mlx4_unregister_device(dev); 877 878 for (p = 1; p <= dev->caps.num_ports; ++p) 879 mlx4_CLOSE_PORT(dev, p); 880 881 mlx4_cleanup_mcg_table(dev); 882 mlx4_cleanup_qp_table(dev); 883 mlx4_cleanup_srq_table(dev); 884 mlx4_cleanup_cq_table(dev); 885 mlx4_cmd_use_polling(dev); 886 mlx4_cleanup_eq_table(dev); 887 888 mlx4_unmap_catas_buf(dev); 889 890 mlx4_cleanup_mr_table(dev); 891 mlx4_cleanup_pd_table(dev); 892 893 iounmap(priv->kar); 894 mlx4_uar_free(dev, &priv->driver_uar); 895 mlx4_cleanup_uar_table(dev); 896 mlx4_close_hca(dev); 897 mlx4_cmd_cleanup(dev); 898 899 if (dev->flags & MLX4_FLAG_MSI_X) 900 pci_disable_msix(pdev); 901 902 kfree(priv); 903 pci_release_region(pdev, 2); 904 pci_release_region(pdev, 0); 905 pci_disable_device(pdev); 906 pci_set_drvdata(pdev, NULL); 907 } 908} 909 910static struct pci_device_id mlx4_pci_table[] = { 911 { PCI_VDEVICE(MELLANOX, 0x6340) }, /* MT25408 "Hermon" SDR */ 912 { PCI_VDEVICE(MELLANOX, 0x634a) }, /* MT25408 "Hermon" DDR */ 913 { PCI_VDEVICE(MELLANOX, 0x6354) }, /* MT25408 "Hermon" QDR */ 914 { PCI_VDEVICE(MELLANOX, 0x6732) }, /* MT25408 "Hermon" DDR PCIe gen2 */ 915 { PCI_VDEVICE(MELLANOX, 0x673c) }, /* MT25408 "Hermon" QDR PCIe gen2 */ 916 { 0, } 917}; 918 919MODULE_DEVICE_TABLE(pci, mlx4_pci_table); 920 921static struct pci_driver mlx4_driver = { 922 .name = DRV_NAME, 923 .id_table = mlx4_pci_table, 924 .probe = mlx4_init_one, 925 .remove = __devexit_p(mlx4_remove_one) 926}; 927 928static int __init mlx4_init(void) 929{ 930 int ret; 931 932 ret = pci_register_driver(&mlx4_driver); 933 return ret < 0 ? ret : 0; 934} 935 936static void __exit mlx4_cleanup(void) 937{ 938 pci_unregister_driver(&mlx4_driver); 939} 940 941module_init(mlx4_init); 942module_exit(mlx4_cleanup); 943