1/* This file is part of the Emulex RoCE Device Driver for 2 * RoCE (RDMA over Converged Ethernet) adapters. 3 * Copyright (C) 2012-2015 Emulex. All rights reserved. 4 * EMULEX and SLI are trademarks of Emulex. 5 * www.emulex.com 6 * 7 * This software is available to you under a choice of one of two licenses. 8 * You may choose to be licensed under the terms of the GNU General Public 9 * License (GPL) Version 2, available from the file COPYING in the main 10 * directory of this source tree, or the BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * - Redistributions of source code must retain the above copyright notice, 17 * this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * Contact Information: 36 * linux-drivers@emulex.com 37 * 38 * Emulex 39 * 3333 Susan Street 40 * Costa Mesa, CA 92626 41 */ 42 43#ifndef __OCRDMA_H__ 44#define __OCRDMA_H__ 45 46#include <linux/mutex.h> 47#include <linux/list.h> 48#include <linux/spinlock.h> 49#include <linux/pci.h> 50 51#include <rdma/ib_verbs.h> 52#include <rdma/ib_user_verbs.h> 53#include <rdma/ib_addr.h> 54 55#include <be_roce.h> 56#include "ocrdma_sli.h" 57 58#define OCRDMA_ROCE_DRV_VERSION "11.0.0.0" 59 60#define OCRDMA_ROCE_DRV_DESC "Emulex OneConnect RoCE Driver" 61#define OCRDMA_NODE_DESC "Emulex OneConnect RoCE HCA" 62 63#define OC_NAME_SH OCRDMA_NODE_DESC "(Skyhawk)" 64#define OC_NAME_UNKNOWN OCRDMA_NODE_DESC "(Unknown)" 65 66#define OC_SKH_DEVICE_PF 0x720 67#define OC_SKH_DEVICE_VF 0x728 68#define OCRDMA_MAX_AH 512 69 70#define OCRDMA_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME) 71 72#define convert_to_64bit(lo, hi) ((u64)hi << 32 | (u64)lo) 73#define EQ_INTR_PER_SEC_THRSH_HI 150000 74#define EQ_INTR_PER_SEC_THRSH_LOW 100000 75#define EQ_AIC_MAX_EQD 20 76#define EQ_AIC_MIN_EQD 0 77 78void ocrdma_eqd_set_task(struct work_struct *work); 79 80struct ocrdma_dev_attr { 81 u8 fw_ver[32]; 82 u32 vendor_id; 83 u32 device_id; 84 u16 max_pd; 85 u16 max_dpp_pds; 86 u16 max_cq; 87 u16 max_cqe; 88 u16 max_qp; 89 u16 max_wqe; 90 u16 max_rqe; 91 u16 max_srq; 92 u32 max_inline_data; 93 int max_send_sge; 94 int max_recv_sge; 95 int max_srq_sge; 96 int max_rdma_sge; 97 int max_mr; 98 u64 max_mr_size; 99 u32 max_num_mr_pbl; 100 int max_mw; 101 int max_map_per_fmr; 102 int max_pages_per_frmr; 103 u16 max_ord_per_qp; 104 u16 max_ird_per_qp; 105 106 int device_cap_flags; 107 u8 cq_overflow_detect; 108 u8 srq_supported; 109 110 u32 wqe_size; 111 u32 rqe_size; 112 u32 ird_page_size; 113 u8 local_ca_ack_delay; 114 u8 ird; 115 u8 num_ird_pages; 116 u8 udp_encap; 117}; 118 119struct ocrdma_dma_mem { 120 void *va; 121 dma_addr_t pa; 122 u32 size; 123}; 124 125struct ocrdma_pbl { 126 void *va; 127 dma_addr_t pa; 128}; 129 130struct ocrdma_queue_info { 131 void *va; 132 dma_addr_t dma; 133 u32 size; 134 u16 len; 135 u16 entry_size; /* Size of an element in the queue */ 136 u16 id; /* qid, where to ring the doorbell. */ 137 u16 head, tail; 138 bool created; 139}; 140 141struct ocrdma_aic_obj { /* Adaptive interrupt coalescing (AIC) info */ 142 u32 prev_eqd; 143 u64 eq_intr_cnt; 144 u64 prev_eq_intr_cnt; 145}; 146 147struct ocrdma_eq { 148 struct ocrdma_queue_info q; 149 u32 vector; 150 int cq_cnt; 151 struct ocrdma_dev *dev; 152 char irq_name[32]; 153 struct ocrdma_aic_obj aic_obj; 154}; 155 156struct ocrdma_mq { 157 struct ocrdma_queue_info sq; 158 struct ocrdma_queue_info cq; 159 bool rearm_cq; 160}; 161 162struct mqe_ctx { 163 struct mutex lock; /* for serializing mailbox commands on MQ */ 164 wait_queue_head_t cmd_wait; 165 u32 tag; 166 u16 cqe_status; 167 u16 ext_status; 168 bool cmd_done; 169 bool fw_error_state; 170}; 171 172struct ocrdma_hw_mr { 173 u32 lkey; 174 u8 fr_mr; 175 u8 remote_atomic; 176 u8 remote_rd; 177 u8 remote_wr; 178 u8 local_rd; 179 u8 local_wr; 180 u8 mw_bind; 181 u8 rsvd; 182 u64 len; 183 struct ocrdma_pbl *pbl_table; 184 u32 num_pbls; 185 u32 num_pbes; 186 u32 pbl_size; 187 u32 pbe_size; 188 u64 va; 189}; 190 191struct ocrdma_mr { 192 struct ib_mr ibmr; 193 struct ib_umem *umem; 194 struct ocrdma_hw_mr hwmr; 195 u64 *pages; 196 u32 npages; 197}; 198 199struct ocrdma_stats { 200 u8 type; 201 struct ocrdma_dev *dev; 202}; 203 204struct ocrdma_pd_resource_mgr { 205 u32 pd_norm_start; 206 u16 pd_norm_count; 207 u16 pd_norm_thrsh; 208 u16 max_normal_pd; 209 u32 pd_dpp_start; 210 u16 pd_dpp_count; 211 u16 pd_dpp_thrsh; 212 u16 max_dpp_pd; 213 u16 dpp_page_index; 214 unsigned long *pd_norm_bitmap; 215 unsigned long *pd_dpp_bitmap; 216 bool pd_prealloc_valid; 217}; 218 219struct stats_mem { 220 struct ocrdma_mqe mqe; 221 void *va; 222 dma_addr_t pa; 223 u32 size; 224 char *debugfs_mem; 225}; 226 227struct phy_info { 228 u16 auto_speeds_supported; 229 u16 fixed_speeds_supported; 230 u16 phy_type; 231 u16 interface_type; 232}; 233 234enum ocrdma_flags { 235 OCRDMA_FLAGS_LINK_STATUS_INIT = 0x01 236}; 237 238struct ocrdma_dev { 239 struct ib_device ibdev; 240 struct ocrdma_dev_attr attr; 241 242 struct mutex dev_lock; /* provides syncronise access to device data */ 243 spinlock_t flush_q_lock ____cacheline_aligned; 244 245 struct ocrdma_cq **cq_tbl; 246 struct ocrdma_qp **qp_tbl; 247 248 struct ocrdma_eq *eq_tbl; 249 int eq_cnt; 250 struct delayed_work eqd_work; 251 u16 base_eqid; 252 u16 max_eq; 253 254 /* provided synchronization to sgid table for 255 * updating gid entries triggered by notifier. 256 */ 257 spinlock_t sgid_lock; 258 259 int gsi_qp_created; 260 struct ocrdma_cq *gsi_sqcq; 261 struct ocrdma_cq *gsi_rqcq; 262 263 struct { 264 struct ocrdma_av *va; 265 dma_addr_t pa; 266 u32 size; 267 u32 num_ah; 268 /* provide synchronization for av 269 * entry allocations. 270 */ 271 spinlock_t lock; 272 u32 ahid; 273 struct ocrdma_pbl pbl; 274 } av_tbl; 275 276 void *mbx_cmd; 277 struct ocrdma_mq mq; 278 struct mqe_ctx mqe_ctx; 279 280 struct be_dev_info nic_info; 281 struct phy_info phy; 282 char model_number[32]; 283 u32 hba_port_num; 284 285 struct list_head entry; 286 int id; 287 u64 *stag_arr; 288 u8 sl; /* service level */ 289 bool pfc_state; 290 atomic_t update_sl; 291 u16 pvid; 292 u32 asic_id; 293 u32 flags; 294 295 ulong last_stats_time; 296 struct mutex stats_lock; /* provide synch for debugfs operations */ 297 struct stats_mem stats_mem; 298 struct ocrdma_stats rsrc_stats; 299 struct ocrdma_stats rx_stats; 300 struct ocrdma_stats wqe_stats; 301 struct ocrdma_stats tx_stats; 302 struct ocrdma_stats db_err_stats; 303 struct ocrdma_stats tx_qp_err_stats; 304 struct ocrdma_stats rx_qp_err_stats; 305 struct ocrdma_stats tx_dbg_stats; 306 struct ocrdma_stats rx_dbg_stats; 307 struct ocrdma_stats driver_stats; 308 struct ocrdma_stats reset_stats; 309 struct dentry *dir; 310 atomic_t async_err_stats[OCRDMA_MAX_ASYNC_ERRORS]; 311 atomic_t cqe_err_stats[OCRDMA_MAX_CQE_ERR]; 312 struct ocrdma_pd_resource_mgr *pd_mgr; 313}; 314 315struct ocrdma_cq { 316 struct ib_cq ibcq; 317 struct ocrdma_cqe *va; 318 u32 phase; 319 u32 getp; /* pointer to pending wrs to 320 * return to stack, wrap arounds 321 * at max_hw_cqe 322 */ 323 u32 max_hw_cqe; 324 bool phase_change; 325 spinlock_t cq_lock ____cacheline_aligned; /* provide synchronization 326 * to cq polling 327 */ 328 /* syncronizes cq completion handler invoked from multiple context */ 329 spinlock_t comp_handler_lock ____cacheline_aligned; 330 u16 id; 331 u16 eqn; 332 333 struct ocrdma_ucontext *ucontext; 334 dma_addr_t pa; 335 u32 len; 336 u32 cqe_cnt; 337 338 /* head of all qp's sq and rq for which cqes need to be flushed 339 * by the software. 340 */ 341 struct list_head sq_head, rq_head; 342}; 343 344struct ocrdma_pd { 345 struct ib_pd ibpd; 346 struct ocrdma_ucontext *uctx; 347 u32 id; 348 int num_dpp_qp; 349 u32 dpp_page; 350 bool dpp_enabled; 351}; 352 353struct ocrdma_ah { 354 struct ib_ah ibah; 355 struct ocrdma_av *av; 356 u16 sgid_index; 357 u32 id; 358 u8 hdr_type; 359}; 360 361struct ocrdma_qp_hwq_info { 362 u8 *va; /* virtual address */ 363 u32 max_sges; 364 u32 head, tail; 365 u32 entry_size; 366 u32 max_cnt; 367 u32 max_wqe_idx; 368 u16 dbid; /* qid, where to ring the doorbell. */ 369 u32 len; 370 dma_addr_t pa; 371}; 372 373struct ocrdma_srq { 374 struct ib_srq ibsrq; 375 u8 __iomem *db; 376 struct ocrdma_qp_hwq_info rq; 377 u64 *rqe_wr_id_tbl; 378 u32 *idx_bit_fields; 379 u32 bit_fields_len; 380 381 /* provide synchronization to multiple context(s) posting rqe */ 382 spinlock_t q_lock ____cacheline_aligned; 383 384 struct ocrdma_pd *pd; 385 u32 id; 386}; 387 388struct ocrdma_qp { 389 struct ib_qp ibqp; 390 391 u8 __iomem *sq_db; 392 struct ocrdma_qp_hwq_info sq; 393 struct { 394 uint64_t wrid; 395 uint16_t dpp_wqe_idx; 396 uint16_t dpp_wqe; 397 uint8_t signaled; 398 uint8_t rsvd[3]; 399 } *wqe_wr_id_tbl; 400 u32 max_inline_data; 401 402 /* provide synchronization to multiple context(s) posting wqe, rqe */ 403 spinlock_t q_lock ____cacheline_aligned; 404 struct ocrdma_cq *sq_cq; 405 /* list maintained per CQ to flush SQ errors */ 406 struct list_head sq_entry; 407 408 u8 __iomem *rq_db; 409 struct ocrdma_qp_hwq_info rq; 410 u64 *rqe_wr_id_tbl; 411 struct ocrdma_cq *rq_cq; 412 struct ocrdma_srq *srq; 413 /* list maintained per CQ to flush RQ errors */ 414 struct list_head rq_entry; 415 416 enum ocrdma_qp_state state; /* QP state */ 417 int cap_flags; 418 u32 max_ord, max_ird; 419 420 u32 id; 421 struct ocrdma_pd *pd; 422 423 enum ib_qp_type qp_type; 424 425 int sgid_idx; 426 u32 qkey; 427 bool dpp_enabled; 428 u8 *ird_q_va; 429 bool signaled; 430}; 431 432struct ocrdma_ucontext { 433 struct ib_ucontext ibucontext; 434 435 struct list_head mm_head; 436 struct mutex mm_list_lock; /* protects list entries of mm type */ 437 struct ocrdma_pd *cntxt_pd; 438 int pd_in_use; 439 440 struct { 441 u32 *va; 442 dma_addr_t pa; 443 u32 len; 444 } ah_tbl; 445}; 446 447struct ocrdma_mm { 448 struct { 449 u64 phy_addr; 450 unsigned long len; 451 } key; 452 struct list_head entry; 453}; 454 455static inline struct ocrdma_dev *get_ocrdma_dev(struct ib_device *ibdev) 456{ 457 return container_of(ibdev, struct ocrdma_dev, ibdev); 458} 459 460static inline struct ocrdma_ucontext *get_ocrdma_ucontext(struct ib_ucontext 461 *ibucontext) 462{ 463 return container_of(ibucontext, struct ocrdma_ucontext, ibucontext); 464} 465 466static inline struct ocrdma_pd *get_ocrdma_pd(struct ib_pd *ibpd) 467{ 468 return container_of(ibpd, struct ocrdma_pd, ibpd); 469} 470 471static inline struct ocrdma_cq *get_ocrdma_cq(struct ib_cq *ibcq) 472{ 473 return container_of(ibcq, struct ocrdma_cq, ibcq); 474} 475 476static inline struct ocrdma_qp *get_ocrdma_qp(struct ib_qp *ibqp) 477{ 478 return container_of(ibqp, struct ocrdma_qp, ibqp); 479} 480 481static inline struct ocrdma_mr *get_ocrdma_mr(struct ib_mr *ibmr) 482{ 483 return container_of(ibmr, struct ocrdma_mr, ibmr); 484} 485 486static inline struct ocrdma_ah *get_ocrdma_ah(struct ib_ah *ibah) 487{ 488 return container_of(ibah, struct ocrdma_ah, ibah); 489} 490 491static inline struct ocrdma_srq *get_ocrdma_srq(struct ib_srq *ibsrq) 492{ 493 return container_of(ibsrq, struct ocrdma_srq, ibsrq); 494} 495 496static inline int is_cqe_valid(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe) 497{ 498 int cqe_valid; 499 cqe_valid = le32_to_cpu(cqe->flags_status_srcqpn) & OCRDMA_CQE_VALID; 500 return (cqe_valid == cq->phase); 501} 502 503static inline int is_cqe_for_sq(struct ocrdma_cqe *cqe) 504{ 505 return (le32_to_cpu(cqe->flags_status_srcqpn) & 506 OCRDMA_CQE_QTYPE) ? 0 : 1; 507} 508 509static inline int is_cqe_invalidated(struct ocrdma_cqe *cqe) 510{ 511 return (le32_to_cpu(cqe->flags_status_srcqpn) & 512 OCRDMA_CQE_INVALIDATE) ? 1 : 0; 513} 514 515static inline int is_cqe_imm(struct ocrdma_cqe *cqe) 516{ 517 return (le32_to_cpu(cqe->flags_status_srcqpn) & 518 OCRDMA_CQE_IMM) ? 1 : 0; 519} 520 521static inline int is_cqe_wr_imm(struct ocrdma_cqe *cqe) 522{ 523 return (le32_to_cpu(cqe->flags_status_srcqpn) & 524 OCRDMA_CQE_WRITE_IMM) ? 1 : 0; 525} 526 527static inline int ocrdma_resolve_dmac(struct ocrdma_dev *dev, 528 struct rdma_ah_attr *ah_attr, u8 *mac_addr) 529{ 530 struct in6_addr in6; 531 532 memcpy(&in6, rdma_ah_read_grh(ah_attr)->dgid.raw, sizeof(in6)); 533 if (rdma_is_multicast_addr(&in6)) 534 rdma_get_mcast_mac(&in6, mac_addr); 535 else if (rdma_link_local_addr(&in6)) 536 rdma_get_ll_mac(&in6, mac_addr); 537 else 538 memcpy(mac_addr, ah_attr->roce.dmac, ETH_ALEN); 539 return 0; 540} 541 542static inline char *hca_name(struct ocrdma_dev *dev) 543{ 544 switch (dev->nic_info.pdev->device) { 545 case OC_SKH_DEVICE_PF: 546 case OC_SKH_DEVICE_VF: 547 return OC_NAME_SH; 548 default: 549 return OC_NAME_UNKNOWN; 550 } 551} 552 553static inline int ocrdma_get_eq_table_index(struct ocrdma_dev *dev, 554 int eqid) 555{ 556 int indx; 557 558 for (indx = 0; indx < dev->eq_cnt; indx++) { 559 if (dev->eq_tbl[indx].q.id == eqid) 560 return indx; 561 } 562 563 return -EINVAL; 564} 565 566static inline u8 ocrdma_get_asic_type(struct ocrdma_dev *dev) 567{ 568 if (dev->nic_info.dev_family == 0xF && !dev->asic_id) { 569 pci_read_config_dword( 570 dev->nic_info.pdev, 571 OCRDMA_SLI_ASIC_ID_OFFSET, &dev->asic_id); 572 } 573 574 return (dev->asic_id & OCRDMA_SLI_ASIC_GEN_NUM_MASK) >> 575 OCRDMA_SLI_ASIC_GEN_NUM_SHIFT; 576} 577 578static inline u8 ocrdma_get_pfc_prio(u8 *pfc, u8 prio) 579{ 580 return *(pfc + prio); 581} 582 583static inline u8 ocrdma_get_app_prio(u8 *app_prio, u8 prio) 584{ 585 return *(app_prio + prio); 586} 587 588static inline u8 ocrdma_is_enabled_and_synced(u32 state) 589{ /* May also be used to interpret TC-state, QCN-state 590 * Appl-state and Logical-link-state in future. 591 */ 592 return (state & OCRDMA_STATE_FLAG_ENABLED) && 593 (state & OCRDMA_STATE_FLAG_SYNC); 594} 595 596static inline u8 ocrdma_get_ae_link_state(u32 ae_state) 597{ 598 return ((ae_state & OCRDMA_AE_LSC_LS_MASK) >> OCRDMA_AE_LSC_LS_SHIFT); 599} 600 601static inline bool ocrdma_is_udp_encap_supported(struct ocrdma_dev *dev) 602{ 603 return (dev->attr.udp_encap & OCRDMA_L3_TYPE_IPV4) || 604 (dev->attr.udp_encap & OCRDMA_L3_TYPE_IPV6); 605} 606 607#endif 608