1273806Snp/* 2273806Snp * Copyright (c) 2006-2014 Chelsio, Inc. All rights reserved. 3273806Snp * 4273806Snp * This software is available to you under a choice of one of two 5273806Snp * licenses. You may choose to be licensed under the terms of the GNU 6273806Snp * General Public License (GPL) Version 2, available from the file 7273806Snp * COPYING in the main directory of this source tree, or the 8273806Snp * OpenIB.org BSD license below: 9273806Snp * 10273806Snp * Redistribution and use in source and binary forms, with or 11273806Snp * without modification, are permitted provided that the following 12273806Snp * conditions are met: 13273806Snp * 14273806Snp * - Redistributions of source code must retain the above 15273806Snp * copyright notice, this list of conditions and the following 16273806Snp * disclaimer. 17273806Snp * 18273806Snp * - Redistributions in binary form must reproduce the above 19273806Snp * copyright notice, this list of conditions and the following 20273806Snp * disclaimer in the documentation and/or other materials 21273806Snp * provided with the distribution. 22273806Snp * 23273806Snp * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24273806Snp * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25273806Snp * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26273806Snp * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27273806Snp * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28273806Snp * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29273806Snp * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30273806Snp * SOFTWARE. 31273806Snp */ 32273806Snp#if HAVE_CONFIG_H 33273806Snp# include <config.h> 34273806Snp#endif /* HAVE_CONFIG_H */ 35273806Snp 36273806Snp#include <stdlib.h> 37273806Snp#include <stdio.h> 38273806Snp#include <string.h> 39273806Snp#include <errno.h> 40273806Snp#include <pthread.h> 41273806Snp#include <sys/mman.h> 42273806Snp#include <netinet/in.h> 43273806Snp#include <inttypes.h> 44273806Snp#include <assert.h> 45273806Snp 46273806Snp#include "libcxgb4.h" 47273806Snp#include "cxgb4-abi.h" 48273806Snp 49273806Snp#define MASKED(x) (void *)((unsigned long)(x) & c4iw_page_mask) 50273806Snp 51273806Snpint c4iw_query_device(struct ibv_context *context, struct ibv_device_attr *attr) 52273806Snp{ 53273806Snp struct ibv_query_device cmd; 54273806Snp uint64_t raw_fw_ver; 55273806Snp unsigned major, minor, sub_minor; 56273806Snp int ret; 57273806Snp 58273806Snp ret = ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, 59273806Snp sizeof cmd); 60273806Snp if (ret) 61273806Snp return ret; 62273806Snp 63273806Snp major = (raw_fw_ver >> 32) & 0xffff; 64273806Snp minor = (raw_fw_ver >> 16) & 0xffff; 65273806Snp sub_minor = raw_fw_ver & 0xffff; 66273806Snp 67273806Snp snprintf(attr->fw_ver, sizeof attr->fw_ver, 68273806Snp "%d.%d.%d", major, minor, sub_minor); 69273806Snp 70273806Snp return 0; 71273806Snp} 72273806Snp 73273806Snpint c4iw_query_port(struct ibv_context *context, uint8_t port, 74273806Snp struct ibv_port_attr *attr) 75273806Snp{ 76273806Snp struct ibv_query_port cmd; 77273806Snp 78273806Snp return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd); 79273806Snp} 80273806Snp 81273806Snpstruct ibv_pd *c4iw_alloc_pd(struct ibv_context *context) 82273806Snp{ 83273806Snp struct ibv_alloc_pd cmd; 84273806Snp struct c4iw_alloc_pd_resp resp; 85273806Snp struct c4iw_pd *pd; 86273806Snp 87273806Snp pd = malloc(sizeof *pd); 88273806Snp if (!pd) 89273806Snp return NULL; 90273806Snp 91273806Snp if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd, 92273806Snp &resp.ibv_resp, sizeof resp)) { 93273806Snp free(pd); 94273806Snp return NULL; 95273806Snp } 96273806Snp 97273806Snp return &pd->ibv_pd; 98273806Snp} 99273806Snp 100273806Snpint c4iw_free_pd(struct ibv_pd *pd) 101273806Snp{ 102273806Snp int ret; 103273806Snp 104273806Snp ret = ibv_cmd_dealloc_pd(pd); 105273806Snp if (ret) 106273806Snp return ret; 107273806Snp 108273806Snp free(pd); 109273806Snp return 0; 110273806Snp} 111273806Snp 112273806Snpstatic struct ibv_mr *__c4iw_reg_mr(struct ibv_pd *pd, void *addr, 113273806Snp size_t length, uint64_t hca_va, 114273806Snp int access) 115273806Snp{ 116273806Snp struct c4iw_mr *mhp; 117273806Snp struct ibv_reg_mr cmd; 118273806Snp struct ibv_reg_mr_resp resp; 119273806Snp struct c4iw_dev *dev = to_c4iw_dev(pd->context->device); 120273806Snp 121273806Snp mhp = malloc(sizeof *mhp); 122273806Snp if (!mhp) 123273806Snp return NULL; 124273806Snp 125273806Snp if (ibv_cmd_reg_mr(pd, addr, length, hca_va, 126273806Snp access, &mhp->ibv_mr, &cmd, sizeof cmd, 127273806Snp &resp, sizeof resp)) { 128273806Snp free(mhp); 129273806Snp return NULL; 130273806Snp } 131273806Snp 132273806Snp mhp->va_fbo = hca_va; 133273806Snp mhp->len = length; 134273806Snp 135273806Snp PDBG("%s stag 0x%x va_fbo 0x%" PRIx64 " len %d\n", 136273806Snp __func__, mhp->ibv_mr.rkey, mhp->va_fbo, mhp->len); 137273806Snp 138273806Snp pthread_spin_lock(&dev->lock); 139273806Snp dev->mmid2ptr[c4iw_mmid(mhp->ibv_mr.lkey)] = mhp; 140273806Snp pthread_spin_unlock(&dev->lock); 141273806Snp INC_STAT(mr); 142273806Snp return &mhp->ibv_mr; 143273806Snp} 144273806Snp 145273806Snpstruct ibv_mr *c4iw_reg_mr(struct ibv_pd *pd, void *addr, 146273806Snp size_t length, int access) 147273806Snp{ 148273806Snp PDBG("%s addr %p length %ld\n", __func__, addr, length); 149273806Snp return __c4iw_reg_mr(pd, addr, length, (uintptr_t) addr, access); 150273806Snp} 151273806Snp 152273806Snpint c4iw_dereg_mr(struct ibv_mr *mr) 153273806Snp{ 154273806Snp int ret; 155273806Snp struct c4iw_dev *dev = to_c4iw_dev(mr->pd->context->device); 156273806Snp 157273806Snp ret = ibv_cmd_dereg_mr(mr); 158273806Snp if (ret) 159273806Snp return ret; 160273806Snp 161273806Snp pthread_spin_lock(&dev->lock); 162273806Snp dev->mmid2ptr[c4iw_mmid(mr->lkey)] = NULL; 163273806Snp pthread_spin_unlock(&dev->lock); 164273806Snp 165273806Snp free(to_c4iw_mr(mr)); 166273806Snp 167273806Snp return 0; 168273806Snp} 169273806Snp 170273806Snpstruct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe, 171273806Snp struct ibv_comp_channel *channel, int comp_vector) 172273806Snp{ 173273806Snp struct ibv_create_cq cmd; 174273806Snp struct c4iw_create_cq_resp resp; 175273806Snp struct c4iw_cq *chp; 176273806Snp struct c4iw_dev *dev = to_c4iw_dev(context->device); 177273806Snp int ret; 178273806Snp 179273806Snp chp = calloc(1, sizeof *chp); 180273806Snp if (!chp) { 181273806Snp return NULL; 182273806Snp } 183273806Snp 184273806Snp resp.reserved = 0; 185273806Snp ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector, 186273806Snp &chp->ibv_cq, &cmd, sizeof cmd, 187273806Snp &resp.ibv_resp, sizeof resp); 188273806Snp if (ret) 189273806Snp goto err1; 190273806Snp 191273806Snp if (resp.reserved) 192273806Snp PDBG("%s c4iw_create_cq_resp reserved field modified by kernel\n", 193273806Snp __FUNCTION__); 194273806Snp 195273806Snp pthread_spin_init(&chp->lock, PTHREAD_PROCESS_PRIVATE); 196273806Snp#ifdef STALL_DETECTION 197273806Snp gettimeofday(&chp->time, NULL); 198273806Snp#endif 199273806Snp chp->rhp = dev; 200273806Snp chp->cq.qid_mask = resp.qid_mask; 201273806Snp chp->cq.cqid = resp.cqid; 202273806Snp chp->cq.size = resp.size; 203273806Snp chp->cq.memsize = resp.memsize; 204273806Snp chp->cq.gen = 1; 205273806Snp chp->cq.queue = mmap(NULL, chp->cq.memsize, PROT_READ|PROT_WRITE, 206273806Snp MAP_SHARED, context->cmd_fd, resp.key); 207273806Snp if (chp->cq.queue == MAP_FAILED) 208273806Snp goto err2; 209273806Snp 210273806Snp chp->cq.ugts = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, 211273806Snp context->cmd_fd, resp.gts_key); 212273806Snp if (chp->cq.ugts == MAP_FAILED) 213273806Snp goto err3; 214273806Snp 215319255Snp if (dev_is_t4(chp->rhp)) 216319255Snp chp->cq.ugts += 1; 217319255Snp else 218309378Sjhb chp->cq.ugts += 5; 219273806Snp chp->cq.sw_queue = calloc(chp->cq.size, sizeof *chp->cq.queue); 220273806Snp if (!chp->cq.sw_queue) 221273806Snp goto err4; 222273806Snp 223273806Snp PDBG("%s cqid 0x%x key %" PRIx64 " va %p memsize %lu gts_key %" 224273806Snp PRIx64 " va %p qid_mask 0x%x\n", 225273806Snp __func__, chp->cq.cqid, resp.key, chp->cq.queue, 226273806Snp chp->cq.memsize, resp.gts_key, chp->cq.ugts, chp->cq.qid_mask); 227273806Snp 228273806Snp pthread_spin_lock(&dev->lock); 229273806Snp dev->cqid2ptr[chp->cq.cqid] = chp; 230273806Snp pthread_spin_unlock(&dev->lock); 231273806Snp INC_STAT(cq); 232273806Snp return &chp->ibv_cq; 233273806Snperr4: 234273806Snp munmap(MASKED(chp->cq.ugts), c4iw_page_size); 235273806Snperr3: 236273806Snp munmap(chp->cq.queue, chp->cq.memsize); 237273806Snperr2: 238273806Snp (void)ibv_cmd_destroy_cq(&chp->ibv_cq); 239273806Snperr1: 240273806Snp free(chp); 241273806Snp return NULL; 242273806Snp} 243273806Snp 244273806Snpint c4iw_resize_cq(struct ibv_cq *ibcq, int cqe) 245273806Snp{ 246273806Snp#if 0 247273806Snp int ret; 248273806Snp 249273806Snp struct ibv_resize_cq cmd; 250273806Snp struct ibv_resize_cq_resp resp; 251273806Snp ret = ibv_cmd_resize_cq(ibcq, cqe, &cmd, sizeof cmd, &resp, sizeof resp); 252273806Snp PDBG("%s ret %d\n", __func__, ret); 253273806Snp return ret; 254273806Snp#else 255273806Snp return -ENOSYS; 256273806Snp#endif 257273806Snp} 258273806Snp 259273806Snpint c4iw_destroy_cq(struct ibv_cq *ibcq) 260273806Snp{ 261273806Snp int ret; 262273806Snp struct c4iw_cq *chp = to_c4iw_cq(ibcq); 263273806Snp struct c4iw_dev *dev = to_c4iw_dev(ibcq->context->device); 264273806Snp 265273806Snp chp->cq.error = 1; 266273806Snp ret = ibv_cmd_destroy_cq(ibcq); 267273806Snp if (ret) { 268273806Snp return ret; 269273806Snp } 270273806Snp munmap(MASKED(chp->cq.ugts), c4iw_page_size); 271273806Snp munmap(chp->cq.queue, chp->cq.memsize); 272273806Snp 273273806Snp pthread_spin_lock(&dev->lock); 274273806Snp dev->cqid2ptr[chp->cq.cqid] = NULL; 275273806Snp pthread_spin_unlock(&dev->lock); 276273806Snp 277273806Snp free(chp->cq.sw_queue); 278273806Snp free(chp); 279273806Snp return 0; 280273806Snp} 281273806Snp 282273806Snpstruct ibv_srq *c4iw_create_srq(struct ibv_pd *pd, 283273806Snp struct ibv_srq_init_attr *attr) 284273806Snp{ 285273806Snp return NULL; 286273806Snp} 287273806Snp 288273806Snpint c4iw_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, 289273806Snp int attr_mask) 290273806Snp{ 291273806Snp return ENOSYS; 292273806Snp} 293273806Snp 294273806Snpint c4iw_destroy_srq(struct ibv_srq *srq) 295273806Snp{ 296273806Snp return ENOSYS; 297273806Snp} 298273806Snp 299273806Snpint c4iw_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr, 300273806Snp struct ibv_recv_wr **bad_wr) 301273806Snp{ 302273806Snp return ENOSYS; 303273806Snp} 304273806Snp 305273806Snpstatic struct ibv_qp *create_qp_v0(struct ibv_pd *pd, 306273806Snp struct ibv_qp_init_attr *attr) 307273806Snp{ 308273806Snp struct ibv_create_qp cmd; 309273806Snp struct c4iw_create_qp_resp_v0 resp; 310273806Snp struct c4iw_qp *qhp; 311273806Snp struct c4iw_dev *dev = to_c4iw_dev(pd->context->device); 312273806Snp int ret; 313273806Snp void *dbva; 314273806Snp 315273806Snp PDBG("%s enter qp\n", __func__); 316273806Snp qhp = calloc(1, sizeof *qhp); 317273806Snp if (!qhp) 318273806Snp goto err1; 319273806Snp 320273806Snp ret = ibv_cmd_create_qp(pd, &qhp->ibv_qp, attr, &cmd, 321273806Snp sizeof cmd, &resp.ibv_resp, sizeof resp); 322273806Snp if (ret) 323273806Snp goto err2; 324273806Snp 325273806Snp PDBG("%s sqid 0x%x sq key %" PRIx64 " sq db/gts key %" PRIx64 326273806Snp " rqid 0x%x rq key %" PRIx64 " rq db/gts key %" PRIx64 327273806Snp " qid_mask 0x%x\n", 328273806Snp __func__, 329273806Snp resp.sqid, resp.sq_key, resp.sq_db_gts_key, 330273806Snp resp.rqid, resp.rq_key, resp.rq_db_gts_key, resp.qid_mask); 331273806Snp 332273806Snp qhp->wq.qid_mask = resp.qid_mask; 333273806Snp qhp->rhp = dev; 334273806Snp qhp->wq.sq.qid = resp.sqid; 335273806Snp qhp->wq.sq.size = resp.sq_size; 336273806Snp qhp->wq.sq.memsize = resp.sq_memsize; 337273806Snp qhp->wq.sq.flags = 0; 338273806Snp qhp->wq.rq.msn = 1; 339273806Snp qhp->wq.rq.qid = resp.rqid; 340273806Snp qhp->wq.rq.size = resp.rq_size; 341273806Snp qhp->wq.rq.memsize = resp.rq_memsize; 342273806Snp pthread_spin_init(&qhp->lock, PTHREAD_PROCESS_PRIVATE); 343273806Snp 344273806Snp dbva = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, 345273806Snp pd->context->cmd_fd, resp.sq_db_gts_key); 346273806Snp if (dbva == MAP_FAILED) 347273806Snp goto err3; 348273806Snp 349273806Snp qhp->wq.sq.udb = dbva; 350273806Snp qhp->wq.sq.queue = mmap(NULL, qhp->wq.sq.memsize, 351273806Snp PROT_WRITE, MAP_SHARED, 352273806Snp pd->context->cmd_fd, resp.sq_key); 353273806Snp if (qhp->wq.sq.queue == MAP_FAILED) 354273806Snp goto err4; 355273806Snp 356273806Snp dbva = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, 357273806Snp pd->context->cmd_fd, resp.rq_db_gts_key); 358273806Snp if (dbva == MAP_FAILED) 359273806Snp goto err5; 360273806Snp qhp->wq.rq.udb = dbva; 361273806Snp qhp->wq.rq.queue = mmap(NULL, qhp->wq.rq.memsize, 362273806Snp PROT_WRITE, MAP_SHARED, 363273806Snp pd->context->cmd_fd, resp.rq_key); 364273806Snp if (qhp->wq.rq.queue == MAP_FAILED) 365273806Snp goto err6; 366273806Snp 367273806Snp qhp->wq.sq.sw_sq = calloc(qhp->wq.sq.size, sizeof (struct t4_swsqe)); 368273806Snp if (!qhp->wq.sq.sw_sq) 369273806Snp goto err7; 370273806Snp 371273806Snp qhp->wq.rq.sw_rq = calloc(qhp->wq.rq.size, sizeof (uint64_t)); 372273806Snp if (!qhp->wq.rq.sw_rq) 373273806Snp goto err8; 374273806Snp 375273806Snp PDBG("%s sq dbva %p sq qva %p sq depth %u sq memsize %lu " 376273806Snp " rq dbva %p rq qva %p rq depth %u rq memsize %lu\n", 377273806Snp __func__, 378273806Snp qhp->wq.sq.udb, qhp->wq.sq.queue, 379273806Snp qhp->wq.sq.size, qhp->wq.sq.memsize, 380273806Snp qhp->wq.rq.udb, qhp->wq.rq.queue, 381273806Snp qhp->wq.rq.size, qhp->wq.rq.memsize); 382273806Snp 383273806Snp qhp->sq_sig_all = attr->sq_sig_all; 384273806Snp 385273806Snp pthread_spin_lock(&dev->lock); 386273806Snp dev->qpid2ptr[qhp->wq.sq.qid] = qhp; 387273806Snp pthread_spin_unlock(&dev->lock); 388273806Snp INC_STAT(qp); 389273806Snp return &qhp->ibv_qp; 390273806Snperr8: 391273806Snp free(qhp->wq.sq.sw_sq); 392273806Snperr7: 393273806Snp munmap((void *)qhp->wq.rq.queue, qhp->wq.rq.memsize); 394273806Snperr6: 395273806Snp munmap(MASKED(qhp->wq.rq.udb), c4iw_page_size); 396273806Snperr5: 397273806Snp munmap((void *)qhp->wq.sq.queue, qhp->wq.sq.memsize); 398273806Snperr4: 399273806Snp munmap(MASKED(qhp->wq.sq.udb), c4iw_page_size); 400273806Snperr3: 401273806Snp (void)ibv_cmd_destroy_qp(&qhp->ibv_qp); 402273806Snperr2: 403273806Snp free(qhp); 404273806Snperr1: 405273806Snp return NULL; 406273806Snp} 407273806Snp 408273806Snpstatic struct ibv_qp *create_qp(struct ibv_pd *pd, 409273806Snp struct ibv_qp_init_attr *attr) 410273806Snp{ 411273806Snp struct ibv_create_qp cmd; 412273806Snp struct c4iw_create_qp_resp resp; 413273806Snp struct c4iw_qp *qhp; 414273806Snp struct c4iw_dev *dev = to_c4iw_dev(pd->context->device); 415273806Snp struct c4iw_context *ctx = to_c4iw_context(pd->context); 416273806Snp int ret; 417273806Snp void *dbva; 418273806Snp 419273806Snp PDBG("%s enter qp\n", __func__); 420273806Snp qhp = calloc(1, sizeof *qhp); 421273806Snp if (!qhp) 422273806Snp goto err1; 423273806Snp 424273806Snp ret = ibv_cmd_create_qp(pd, &qhp->ibv_qp, attr, &cmd, 425273806Snp sizeof cmd, &resp.ibv_resp, sizeof resp); 426273806Snp if (ret) 427273806Snp goto err2; 428273806Snp 429273806Snp PDBG("%s sqid 0x%x sq key %" PRIx64 " sq db/gts key %" PRIx64 430273806Snp " rqid 0x%x rq key %" PRIx64 " rq db/gts key %" PRIx64 431273806Snp " qid_mask 0x%x\n", 432273806Snp __func__, 433273806Snp resp.sqid, resp.sq_key, resp.sq_db_gts_key, 434273806Snp resp.rqid, resp.rq_key, resp.rq_db_gts_key, resp.qid_mask); 435273806Snp 436273806Snp qhp->wq.qid_mask = resp.qid_mask; 437273806Snp qhp->rhp = dev; 438273806Snp qhp->wq.sq.qid = resp.sqid; 439273806Snp qhp->wq.sq.size = resp.sq_size; 440273806Snp qhp->wq.sq.memsize = resp.sq_memsize; 441273806Snp qhp->wq.sq.flags = resp.flags & C4IW_QPF_ONCHIP ? T4_SQ_ONCHIP : 0; 442273806Snp qhp->wq.sq.flush_cidx = -1; 443273806Snp qhp->wq.rq.msn = 1; 444273806Snp qhp->wq.rq.qid = resp.rqid; 445273806Snp qhp->wq.rq.size = resp.rq_size; 446273806Snp qhp->wq.rq.memsize = resp.rq_memsize; 447273806Snp if (ma_wr && resp.sq_memsize < (resp.sq_size + 1) * 448273806Snp sizeof *qhp->wq.sq.queue + 16*sizeof(__be64) ) { 449273806Snp ma_wr = 0; 450273806Snp fprintf(stderr, "libcxgb4 warning - downlevel iw_cxgb4 driver. " 451273806Snp "MA workaround disabled.\n"); 452273806Snp } 453273806Snp pthread_spin_init(&qhp->lock, PTHREAD_PROCESS_PRIVATE); 454273806Snp 455273806Snp dbva = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, 456273806Snp pd->context->cmd_fd, resp.sq_db_gts_key); 457273806Snp if (dbva == MAP_FAILED) { 458273806Snp PDBG(" %s mmap for sq db failed\n", __func__); 459273806Snp abort(); 460273806Snp goto err3; 461273806Snp } 462273806Snp qhp->wq.sq.udb = dbva; 463309378Sjhb if (!dev_is_t4(qhp->rhp)) { 464309378Sjhb unsigned long segment_offset = 128 * (qhp->wq.sq.qid & qhp->wq.qid_mask); 465309378Sjhb 466309378Sjhb if (segment_offset < c4iw_page_size) { 467309378Sjhb qhp->wq.sq.udb += segment_offset / 4; 468309378Sjhb qhp->wq.sq.wc_reg_available = 1; 469309378Sjhb } else 470309378Sjhb qhp->wq.sq.bar2_qid = qhp->wq.sq.qid & qhp->wq.qid_mask; 471273806Snp qhp->wq.sq.udb += 2; 472273806Snp } 473273806Snp 474273806Snp qhp->wq.sq.queue = mmap(NULL, qhp->wq.sq.memsize, 475273806Snp PROT_READ | PROT_WRITE, MAP_SHARED, 476273806Snp pd->context->cmd_fd, resp.sq_key); 477273806Snp if (qhp->wq.sq.queue == MAP_FAILED) { 478273806Snp PDBG(" %s mmap for sq q failed size is qhp->wq.sq.memsize %zu \n", __func__, qhp->wq.sq.memsize); 479273806Snp abort(); 480273806Snp goto err4; 481273806Snp } 482273806Snp 483273806Snp dbva = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, 484273806Snp pd->context->cmd_fd, resp.rq_db_gts_key); 485273806Snp if (dbva == MAP_FAILED) 486273806Snp goto err5; 487273806Snp qhp->wq.rq.udb = dbva; 488309378Sjhb if (!dev_is_t4(qhp->rhp)) { 489309378Sjhb unsigned long segment_offset = 128 * (qhp->wq.rq.qid & qhp->wq.qid_mask); 490309378Sjhb 491309378Sjhb if (segment_offset < c4iw_page_size) { 492309378Sjhb qhp->wq.rq.udb += segment_offset / 4; 493309378Sjhb qhp->wq.rq.wc_reg_available = 1; 494309378Sjhb } else 495309378Sjhb qhp->wq.rq.bar2_qid = qhp->wq.rq.qid & qhp->wq.qid_mask; 496273806Snp qhp->wq.rq.udb += 2; 497273806Snp } 498273806Snp qhp->wq.rq.queue = mmap(NULL, qhp->wq.rq.memsize, 499273806Snp PROT_READ | PROT_WRITE, MAP_SHARED, 500273806Snp pd->context->cmd_fd, resp.rq_key); 501273806Snp if (qhp->wq.rq.queue == MAP_FAILED) 502273806Snp goto err6; 503273806Snp 504273806Snp qhp->wq.sq.sw_sq = calloc(qhp->wq.sq.size, sizeof (struct t4_swsqe)); 505273806Snp if (!qhp->wq.sq.sw_sq) 506273806Snp goto err7; 507273806Snp 508273806Snp qhp->wq.rq.sw_rq = calloc(qhp->wq.rq.size, sizeof (uint64_t)); 509273806Snp if (!qhp->wq.rq.sw_rq) 510273806Snp goto err8; 511273806Snp 512273806Snp if (t4_sq_onchip(&qhp->wq)) { 513273806Snp qhp->wq.sq.ma_sync = mmap(NULL, c4iw_page_size, PROT_WRITE, 514273806Snp MAP_SHARED, pd->context->cmd_fd, 515273806Snp resp.ma_sync_key); 516273806Snp if (qhp->wq.sq.ma_sync == MAP_FAILED) 517273806Snp goto err9; 518273806Snp qhp->wq.sq.ma_sync += (A_PCIE_MA_SYNC & (c4iw_page_size - 1)); 519273806Snp } 520273806Snp 521273806Snp if (ctx->status_page_size) { 522273806Snp qhp->wq.db_offp = &ctx->status_page->db_off; 523273806Snp } else { 524273806Snp qhp->wq.db_offp = 525273806Snp &qhp->wq.rq.queue[qhp->wq.rq.size].status.db_off; 526273806Snp } 527273806Snp 528273806Snp PDBG("%s sq dbva %p sq qva %p sq depth %u sq memsize %lu " 529273806Snp " rq dbva %p rq qva %p rq depth %u rq memsize %lu\n", 530273806Snp __func__, 531273806Snp qhp->wq.sq.udb, qhp->wq.sq.queue, 532273806Snp qhp->wq.sq.size, qhp->wq.sq.memsize, 533273806Snp qhp->wq.rq.udb, qhp->wq.rq.queue, 534273806Snp qhp->wq.rq.size, qhp->wq.rq.memsize); 535273806Snp 536273806Snp qhp->sq_sig_all = attr->sq_sig_all; 537273806Snp 538273806Snp pthread_spin_lock(&dev->lock); 539273806Snp dev->qpid2ptr[qhp->wq.sq.qid] = qhp; 540273806Snp pthread_spin_unlock(&dev->lock); 541273806Snp INC_STAT(qp); 542273806Snp return &qhp->ibv_qp; 543273806Snperr9: 544273806Snp free(qhp->wq.rq.sw_rq); 545273806Snperr8: 546273806Snp free(qhp->wq.sq.sw_sq); 547273806Snperr7: 548273806Snp munmap((void *)qhp->wq.rq.queue, qhp->wq.rq.memsize); 549273806Snperr6: 550273806Snp munmap(MASKED(qhp->wq.rq.udb), c4iw_page_size); 551273806Snperr5: 552273806Snp munmap((void *)qhp->wq.sq.queue, qhp->wq.sq.memsize); 553273806Snperr4: 554273806Snp munmap(MASKED(qhp->wq.sq.udb), c4iw_page_size); 555273806Snperr3: 556273806Snp (void)ibv_cmd_destroy_qp(&qhp->ibv_qp); 557273806Snperr2: 558273806Snp free(qhp); 559273806Snperr1: 560273806Snp return NULL; 561273806Snp} 562273806Snp 563273806Snpstruct ibv_qp *c4iw_create_qp(struct ibv_pd *pd, 564273806Snp struct ibv_qp_init_attr *attr) 565273806Snp{ 566273806Snp struct c4iw_dev *dev = to_c4iw_dev(pd->context->device); 567273806Snp 568273806Snp if (dev->abi_version == 0) 569273806Snp return create_qp_v0(pd, attr); 570273806Snp return create_qp(pd, attr); 571273806Snp} 572273806Snp 573273806Snpstatic void reset_qp(struct c4iw_qp *qhp) 574273806Snp{ 575273806Snp PDBG("%s enter qp %p\n", __func__, qhp); 576273806Snp qhp->wq.sq.cidx = 0; 577273806Snp qhp->wq.sq.wq_pidx = qhp->wq.sq.pidx = qhp->wq.sq.in_use = 0; 578273806Snp qhp->wq.rq.cidx = qhp->wq.rq.pidx = qhp->wq.rq.in_use = 0; 579273806Snp qhp->wq.sq.oldest_read = NULL; 580273806Snp memset(qhp->wq.sq.queue, 0, qhp->wq.sq.memsize); 581273806Snp memset(qhp->wq.rq.queue, 0, qhp->wq.rq.memsize); 582273806Snp} 583273806Snp 584273806Snpint c4iw_modify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, 585273806Snp int attr_mask) 586273806Snp{ 587273806Snp struct ibv_modify_qp cmd; 588273806Snp struct c4iw_qp *qhp = to_c4iw_qp(ibqp); 589273806Snp int ret; 590273806Snp 591273806Snp PDBG("%s enter qp %p new state %d\n", __func__, ibqp, attr_mask & IBV_QP_STATE ? attr->qp_state : -1); 592273806Snp pthread_spin_lock(&qhp->lock); 593273806Snp if (t4_wq_in_error(&qhp->wq)) 594273806Snp c4iw_flush_qp(qhp); 595273806Snp ret = ibv_cmd_modify_qp(ibqp, attr, attr_mask, &cmd, sizeof cmd); 596273806Snp if (!ret && (attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RESET) 597273806Snp reset_qp(qhp); 598273806Snp pthread_spin_unlock(&qhp->lock); 599273806Snp return ret; 600273806Snp} 601273806Snp 602273806Snpint c4iw_destroy_qp(struct ibv_qp *ibqp) 603273806Snp{ 604273806Snp int ret; 605273806Snp struct c4iw_qp *qhp = to_c4iw_qp(ibqp); 606273806Snp struct c4iw_dev *dev = to_c4iw_dev(ibqp->context->device); 607273806Snp 608273806Snp PDBG("%s enter qp %p\n", __func__, ibqp); 609273806Snp pthread_spin_lock(&qhp->lock); 610273806Snp c4iw_flush_qp(qhp); 611273806Snp pthread_spin_unlock(&qhp->lock); 612273806Snp 613273806Snp ret = ibv_cmd_destroy_qp(ibqp); 614273806Snp if (ret) { 615273806Snp return ret; 616273806Snp } 617273806Snp if (t4_sq_onchip(&qhp->wq)) { 618273806Snp qhp->wq.sq.ma_sync -= (A_PCIE_MA_SYNC & (c4iw_page_size - 1)); 619273806Snp munmap((void *)qhp->wq.sq.ma_sync, c4iw_page_size); 620273806Snp } 621273806Snp munmap(MASKED(qhp->wq.sq.udb), c4iw_page_size); 622273806Snp munmap(MASKED(qhp->wq.rq.udb), c4iw_page_size); 623273806Snp munmap(qhp->wq.sq.queue, qhp->wq.sq.memsize); 624273806Snp munmap(qhp->wq.rq.queue, qhp->wq.rq.memsize); 625273806Snp 626273806Snp pthread_spin_lock(&dev->lock); 627273806Snp dev->qpid2ptr[qhp->wq.sq.qid] = NULL; 628273806Snp pthread_spin_unlock(&dev->lock); 629273806Snp 630273806Snp free(qhp->wq.rq.sw_rq); 631273806Snp free(qhp->wq.sq.sw_sq); 632273806Snp free(qhp); 633273806Snp return 0; 634273806Snp} 635273806Snp 636273806Snpint c4iw_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, 637273806Snp int attr_mask, struct ibv_qp_init_attr *init_attr) 638273806Snp{ 639273806Snp struct ibv_query_qp cmd; 640273806Snp struct c4iw_qp *qhp = to_c4iw_qp(ibqp); 641273806Snp int ret; 642273806Snp 643273806Snp pthread_spin_lock(&qhp->lock); 644273806Snp if (t4_wq_in_error(&qhp->wq)) 645273806Snp c4iw_flush_qp(qhp); 646273806Snp ret = ibv_cmd_query_qp(ibqp, attr, attr_mask, init_attr, &cmd, sizeof cmd); 647273806Snp pthread_spin_unlock(&qhp->lock); 648273806Snp return ret; 649273806Snp} 650273806Snp 651273806Snpstruct ibv_ah *c4iw_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) 652273806Snp{ 653273806Snp return NULL; 654273806Snp} 655273806Snp 656273806Snpint c4iw_destroy_ah(struct ibv_ah *ah) 657273806Snp{ 658273806Snp return ENOSYS; 659273806Snp} 660273806Snp 661273806Snpint c4iw_attach_mcast(struct ibv_qp *ibqp, const union ibv_gid *gid, 662273806Snp uint16_t lid) 663273806Snp{ 664273806Snp struct c4iw_qp *qhp = to_c4iw_qp(ibqp); 665273806Snp int ret; 666273806Snp 667273806Snp pthread_spin_lock(&qhp->lock); 668273806Snp if (t4_wq_in_error(&qhp->wq)) 669273806Snp c4iw_flush_qp(qhp); 670273806Snp ret = ibv_cmd_attach_mcast(ibqp, gid, lid); 671273806Snp pthread_spin_unlock(&qhp->lock); 672273806Snp return ret; 673273806Snp} 674273806Snp 675273806Snpint c4iw_detach_mcast(struct ibv_qp *ibqp, const union ibv_gid *gid, 676273806Snp uint16_t lid) 677273806Snp{ 678273806Snp struct c4iw_qp *qhp = to_c4iw_qp(ibqp); 679273806Snp int ret; 680273806Snp 681273806Snp pthread_spin_lock(&qhp->lock); 682273806Snp if (t4_wq_in_error(&qhp->wq)) 683273806Snp c4iw_flush_qp(qhp); 684273806Snp ret = ibv_cmd_detach_mcast(ibqp, gid, lid); 685273806Snp pthread_spin_unlock(&qhp->lock); 686273806Snp return ret; 687273806Snp} 688273806Snp 689273806Snpvoid c4iw_async_event(struct ibv_async_event *event) 690273806Snp{ 691273806Snp PDBG("%s type %d obj %p\n", __func__, event->event_type, 692273806Snp event->element.cq); 693273806Snp 694273806Snp switch (event->event_type) { 695273806Snp case IBV_EVENT_CQ_ERR: 696273806Snp break; 697273806Snp case IBV_EVENT_QP_FATAL: 698273806Snp case IBV_EVENT_QP_REQ_ERR: 699273806Snp case IBV_EVENT_QP_ACCESS_ERR: 700273806Snp case IBV_EVENT_PATH_MIG_ERR: { 701273806Snp struct c4iw_qp *qhp = to_c4iw_qp(event->element.qp); 702273806Snp pthread_spin_lock(&qhp->lock); 703273806Snp c4iw_flush_qp(qhp); 704273806Snp pthread_spin_unlock(&qhp->lock); 705273806Snp break; 706273806Snp } 707273806Snp case IBV_EVENT_SQ_DRAINED: 708273806Snp case IBV_EVENT_PATH_MIG: 709273806Snp case IBV_EVENT_COMM_EST: 710273806Snp case IBV_EVENT_QP_LAST_WQE_REACHED: 711273806Snp default: 712273806Snp break; 713273806Snp } 714273806Snp} 715