1/* 2 * Copyright (c) 2006 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32#include "iwch_provider.h" 33#include "iwch.h" 34#include "iwch_cm.h" 35#include "cxio_hal.h" 36 37#define NO_SUPPORT -1 38 39static int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, 40 u8 * flit_cnt) 41{ 42 int i; 43 u32 plen; 44 45 switch (wr->opcode) { 46 case IB_WR_SEND: 47 case IB_WR_SEND_WITH_IMM: 48 if (wr->send_flags & IB_SEND_SOLICITED) 49 wqe->send.rdmaop = T3_SEND_WITH_SE; 50 else 51 wqe->send.rdmaop = T3_SEND; 52 wqe->send.rem_stag = 0; 53 break; 54 default: 55 break; 56 } 57 if (wr->num_sge > T3_MAX_SGE) 58 return -EINVAL; 59 wqe->send.reserved[0] = 0; 60 wqe->send.reserved[1] = 0; 61 wqe->send.reserved[2] = 0; 62 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 63 plen = 4; 64 wqe->send.sgl[0].stag = wr->imm_data; 65 wqe->send.sgl[0].len = __constant_cpu_to_be32(0); 66 wqe->send.num_sgle = __constant_cpu_to_be32(0); 67 *flit_cnt = 5; 68 } else { 69 plen = 0; 70 for (i = 0; i < wr->num_sge; i++) { 71 if ((plen + wr->sg_list[i].length) < plen) { 72 return -EMSGSIZE; 73 } 74 plen += wr->sg_list[i].length; 75 wqe->send.sgl[i].stag = 76 cpu_to_be32(wr->sg_list[i].lkey); 77 wqe->send.sgl[i].len = 78 cpu_to_be32(wr->sg_list[i].length); 79 wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); 80 } 81 wqe->send.num_sgle = cpu_to_be32(wr->num_sge); 82 *flit_cnt = 4 + ((wr->num_sge) << 1); 83 } 84 wqe->send.plen = cpu_to_be32(plen); 85 return 0; 86} 87 88static int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, 89 u8 *flit_cnt) 90{ 91 int i; 92 u32 plen; 93 if (wr->num_sge > T3_MAX_SGE) 94 return -EINVAL; 95 wqe->write.rdmaop = T3_RDMA_WRITE; 96 wqe->write.reserved[0] = 0; 97 wqe->write.reserved[1] = 0; 98 wqe->write.reserved[2] = 0; 99 wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); 100 wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); 101 102 if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 103 plen = 4; 104 wqe->write.sgl[0].stag = wr->imm_data; 105 wqe->write.sgl[0].len = __constant_cpu_to_be32(0); 106 wqe->write.num_sgle = __constant_cpu_to_be32(0); 107 *flit_cnt = 6; 108 } else { 109 plen = 0; 110 for (i = 0; i < wr->num_sge; i++) { 111 if ((plen + wr->sg_list[i].length) < plen) { 112 return -EMSGSIZE; 113 } 114 plen += wr->sg_list[i].length; 115 wqe->write.sgl[i].stag = 116 cpu_to_be32(wr->sg_list[i].lkey); 117 wqe->write.sgl[i].len = 118 cpu_to_be32(wr->sg_list[i].length); 119 wqe->write.sgl[i].to = 120 cpu_to_be64(wr->sg_list[i].addr); 121 } 122 wqe->write.num_sgle = cpu_to_be32(wr->num_sge); 123 *flit_cnt = 5 + ((wr->num_sge) << 1); 124 } 125 wqe->write.plen = cpu_to_be32(plen); 126 return 0; 127} 128 129static int iwch_build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, 130 u8 *flit_cnt) 131{ 132 if (wr->num_sge > 1) 133 return -EINVAL; 134 wqe->read.rdmaop = T3_READ_REQ; 135 wqe->read.reserved[0] = 0; 136 wqe->read.reserved[1] = 0; 137 wqe->read.reserved[2] = 0; 138 wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); 139 wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr); 140 wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); 141 wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); 142 wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); 143 *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; 144 return 0; 145} 146 147/* 148 * TBD: this is going to be moved to firmware. Missing pdid/qpid check for now. 149 */ 150static int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, 151 u32 num_sgle, u32 * pbl_addr, u8 * page_size) 152{ 153 int i; 154 struct iwch_mr *mhp; 155 u32 offset; 156 for (i = 0; i < num_sgle; i++) { 157 158 mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); 159 if (!mhp) { 160 PDBG("%s %d\n", __FUNCTION__, __LINE__); 161 return -EIO; 162 } 163 if (!mhp->attr.state) { 164 PDBG("%s %d\n", __FUNCTION__, __LINE__); 165 return -EIO; 166 } 167 if (mhp->attr.zbva) { 168 PDBG("%s %d\n", __FUNCTION__, __LINE__); 169 return -EIO; 170 } 171 172 if (sg_list[i].addr < mhp->attr.va_fbo) { 173 PDBG("%s %d\n", __FUNCTION__, __LINE__); 174 return -EINVAL; 175 } 176 if (sg_list[i].addr + ((u64) sg_list[i].length) < 177 sg_list[i].addr) { 178 PDBG("%s %d\n", __FUNCTION__, __LINE__); 179 return -EINVAL; 180 } 181 if (sg_list[i].addr + ((u64) sg_list[i].length) > 182 mhp->attr.va_fbo + ((u64) mhp->attr.len)) { 183 PDBG("%s %d\n", __FUNCTION__, __LINE__); 184 return -EINVAL; 185 } 186 offset = sg_list[i].addr - mhp->attr.va_fbo; 187 offset += ((u32) mhp->attr.va_fbo) % 188 (1UL << (12 + mhp->attr.page_size)); 189 pbl_addr[i] = ((mhp->attr.pbl_addr - 190 rhp->rdev.rnic_info.pbl_base) >> 3) + 191 (offset >> (12 + mhp->attr.page_size)); 192 page_size[i] = mhp->attr.page_size; 193 } 194 return 0; 195} 196 197static int iwch_build_rdma_recv(struct iwch_dev *rhp, union t3_wr *wqe, 198 struct ib_recv_wr *wr) 199{ 200 int i, err = 0; 201 u32 pbl_addr[4]; 202 u8 page_size[4]; 203 if (wr->num_sge > T3_MAX_SGE) 204 return -EINVAL; 205 err = iwch_sgl2pbl_map(rhp, wr->sg_list, wr->num_sge, pbl_addr, 206 page_size); 207 if (err) 208 return err; 209 wqe->recv.pagesz[0] = page_size[0]; 210 wqe->recv.pagesz[1] = page_size[1]; 211 wqe->recv.pagesz[2] = page_size[2]; 212 wqe->recv.pagesz[3] = page_size[3]; 213 wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); 214 for (i = 0; i < wr->num_sge; i++) { 215 wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); 216 wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); 217 218 /* to in the WQE == the offset into the page */ 219 wqe->recv.sgl[i].to = cpu_to_be64(((u32) wr->sg_list[i].addr) % 220 (1UL << (12 + page_size[i]))); 221 222 /* pbl_addr is the adapters address in the PBL */ 223 wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]); 224 } 225 for (; i < T3_MAX_SGE; i++) { 226 wqe->recv.sgl[i].stag = 0; 227 wqe->recv.sgl[i].len = 0; 228 wqe->recv.sgl[i].to = 0; 229 wqe->recv.pbl_addr[i] = 0; 230 } 231 return 0; 232} 233 234int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 235 struct ib_send_wr **bad_wr) 236{ 237 int err = 0; 238 u8 t3_wr_flit_cnt; 239 enum t3_wr_opcode t3_wr_opcode = 0; 240 enum t3_wr_flags t3_wr_flags; 241 struct iwch_qp *qhp; 242 u32 idx; 243 union t3_wr *wqe; 244 u32 num_wrs; 245 unsigned long flag; 246 struct t3_swsq *sqp; 247 248 qhp = to_iwch_qp(ibqp); 249 spin_lock_irqsave(&qhp->lock, flag); 250 if (qhp->attr.state > IWCH_QP_STATE_RTS) { 251 spin_unlock_irqrestore(&qhp->lock, flag); 252 return -EINVAL; 253 } 254 num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 255 qhp->wq.sq_size_log2); 256 if (num_wrs <= 0) { 257 spin_unlock_irqrestore(&qhp->lock, flag); 258 return -ENOMEM; 259 } 260 while (wr) { 261 if (num_wrs == 0) { 262 err = -ENOMEM; 263 *bad_wr = wr; 264 break; 265 } 266 idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 267 wqe = (union t3_wr *) (qhp->wq.queue + idx); 268 t3_wr_flags = 0; 269 if (wr->send_flags & IB_SEND_SOLICITED) 270 t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; 271 if (wr->send_flags & IB_SEND_FENCE) 272 t3_wr_flags |= T3_READ_FENCE_FLAG; 273 if (wr->send_flags & IB_SEND_SIGNALED) 274 t3_wr_flags |= T3_COMPLETION_FLAG; 275 sqp = qhp->wq.sq + 276 Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 277 switch (wr->opcode) { 278 case IB_WR_SEND: 279 case IB_WR_SEND_WITH_IMM: 280 t3_wr_opcode = T3_WR_SEND; 281 err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt); 282 break; 283 case IB_WR_RDMA_WRITE: 284 case IB_WR_RDMA_WRITE_WITH_IMM: 285 t3_wr_opcode = T3_WR_WRITE; 286 err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt); 287 break; 288 case IB_WR_RDMA_READ: 289 t3_wr_opcode = T3_WR_READ; 290 t3_wr_flags = 0; /* T3 reads are always signaled */ 291 err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt); 292 if (err) 293 break; 294 sqp->read_len = wqe->read.local_len; 295 if (!qhp->wq.oldest_read) 296 qhp->wq.oldest_read = sqp; 297 break; 298 default: 299 PDBG("%s post of type=%d TBD!\n", __FUNCTION__, 300 wr->opcode); 301 err = -EINVAL; 302 } 303 if (err) { 304 *bad_wr = wr; 305 break; 306 } 307 wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 308 sqp->wr_id = wr->wr_id; 309 sqp->opcode = wr2opcode(t3_wr_opcode); 310 sqp->sq_wptr = qhp->wq.sq_wptr; 311 sqp->complete = 0; 312 sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); 313 314 build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, 315 Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 316 0, t3_wr_flit_cnt); 317 PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n", 318 __FUNCTION__, (unsigned long long) wr->wr_id, idx, 319 Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), 320 sqp->opcode); 321 wr = wr->next; 322 num_wrs--; 323 ++(qhp->wq.wptr); 324 ++(qhp->wq.sq_wptr); 325 } 326 spin_unlock_irqrestore(&qhp->lock, flag); 327 ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 328 return err; 329} 330 331int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 332 struct ib_recv_wr **bad_wr) 333{ 334 int err = 0; 335 struct iwch_qp *qhp; 336 u32 idx; 337 union t3_wr *wqe; 338 u32 num_wrs; 339 unsigned long flag; 340 341 qhp = to_iwch_qp(ibqp); 342 spin_lock_irqsave(&qhp->lock, flag); 343 if (qhp->attr.state > IWCH_QP_STATE_RTS) { 344 spin_unlock_irqrestore(&qhp->lock, flag); 345 return -EINVAL; 346 } 347 num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, 348 qhp->wq.rq_size_log2) - 1; 349 if (!wr) { 350 spin_unlock_irqrestore(&qhp->lock, flag); 351 return -EINVAL; 352 } 353 while (wr) { 354 idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 355 wqe = (union t3_wr *) (qhp->wq.queue + idx); 356 if (num_wrs) 357 err = iwch_build_rdma_recv(qhp->rhp, wqe, wr); 358 else 359 err = -ENOMEM; 360 if (err) { 361 *bad_wr = wr; 362 break; 363 } 364 qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] = 365 wr->wr_id; 366 build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, 367 Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 368 0, sizeof(struct t3_receive_wr) >> 3); 369 PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " 370 "wqe %p \n", __FUNCTION__, (unsigned long long) wr->wr_id, 371 idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); 372 ++(qhp->wq.rq_wptr); 373 ++(qhp->wq.wptr); 374 wr = wr->next; 375 num_wrs--; 376 } 377 spin_unlock_irqrestore(&qhp->lock, flag); 378 ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 379 return err; 380} 381 382int iwch_bind_mw(struct ib_qp *qp, 383 struct ib_mw *mw, 384 struct ib_mw_bind *mw_bind) 385{ 386 struct iwch_dev *rhp; 387 struct iwch_mw *mhp; 388 struct iwch_qp *qhp; 389 union t3_wr *wqe; 390 u32 pbl_addr; 391 u8 page_size; 392 u32 num_wrs; 393 unsigned long flag; 394 struct ib_sge sgl; 395 int err=0; 396 enum t3_wr_flags t3_wr_flags; 397 u32 idx; 398 struct t3_swsq *sqp; 399 400 qhp = to_iwch_qp(qp); 401 mhp = to_iwch_mw(mw); 402 rhp = qhp->rhp; 403 404 spin_lock_irqsave(&qhp->lock, flag); 405 if (qhp->attr.state > IWCH_QP_STATE_RTS) { 406 spin_unlock_irqrestore(&qhp->lock, flag); 407 return -EINVAL; 408 } 409 num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 410 qhp->wq.sq_size_log2); 411 if ((num_wrs) <= 0) { 412 spin_unlock_irqrestore(&qhp->lock, flag); 413 return -ENOMEM; 414 } 415 idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 416 PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __FUNCTION__, idx, 417 mw, mw_bind); 418 wqe = (union t3_wr *) (qhp->wq.queue + idx); 419 420 t3_wr_flags = 0; 421 if (mw_bind->send_flags & IB_SEND_SIGNALED) 422 t3_wr_flags = T3_COMPLETION_FLAG; 423 424 sgl.addr = mw_bind->addr; 425 sgl.lkey = mw_bind->mr->lkey; 426 sgl.length = mw_bind->length; 427 wqe->bind.reserved = 0; 428 wqe->bind.type = T3_VA_BASED_TO; 429 430 /* TBD: check perms */ 431 wqe->bind.perms = iwch_ib_to_mwbind_access(mw_bind->mw_access_flags); 432 wqe->bind.mr_stag = cpu_to_be32(mw_bind->mr->lkey); 433 wqe->bind.mw_stag = cpu_to_be32(mw->rkey); 434 wqe->bind.mw_len = cpu_to_be32(mw_bind->length); 435 wqe->bind.mw_va = cpu_to_be64(mw_bind->addr); 436 err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); 437 if (err) { 438 spin_unlock_irqrestore(&qhp->lock, flag); 439 return err; 440 } 441 wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 442 sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 443 sqp->wr_id = mw_bind->wr_id; 444 sqp->opcode = T3_BIND_MW; 445 sqp->sq_wptr = qhp->wq.sq_wptr; 446 sqp->complete = 0; 447 sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); 448 wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr); 449 wqe->bind.mr_pagesz = page_size; 450 wqe->flit[T3_SQ_COOKIE_FLIT] = mw_bind->wr_id; 451 build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, 452 Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, 453 sizeof(struct t3_bind_mw_wr) >> 3); 454 ++(qhp->wq.wptr); 455 ++(qhp->wq.sq_wptr); 456 spin_unlock_irqrestore(&qhp->lock, flag); 457 458 ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 459 460 return err; 461} 462 463static inline void build_term_codes(struct respQ_msg_t *rsp_msg, 464 u8 *layer_type, u8 *ecode) 465{ 466 int status = TPT_ERR_INTERNAL_ERR; 467 int tagged = 0; 468 int opcode = -1; 469 int rqtype = 0; 470 int send_inv = 0; 471 472 if (rsp_msg) { 473 status = CQE_STATUS(rsp_msg->cqe); 474 opcode = CQE_OPCODE(rsp_msg->cqe); 475 rqtype = RQ_TYPE(rsp_msg->cqe); 476 send_inv = (opcode == T3_SEND_WITH_INV) || 477 (opcode == T3_SEND_WITH_SE_INV); 478 tagged = (opcode == T3_RDMA_WRITE) || 479 (rqtype && (opcode == T3_READ_RESP)); 480 } 481 482 switch (status) { 483 case TPT_ERR_STAG: 484 if (send_inv) { 485 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 486 *ecode = RDMAP_CANT_INV_STAG; 487 } else { 488 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 489 *ecode = RDMAP_INV_STAG; 490 } 491 break; 492 case TPT_ERR_PDID: 493 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 494 if ((opcode == T3_SEND_WITH_INV) || 495 (opcode == T3_SEND_WITH_SE_INV)) 496 *ecode = RDMAP_CANT_INV_STAG; 497 else 498 *ecode = RDMAP_STAG_NOT_ASSOC; 499 break; 500 case TPT_ERR_QPID: 501 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 502 *ecode = RDMAP_STAG_NOT_ASSOC; 503 break; 504 case TPT_ERR_ACCESS: 505 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 506 *ecode = RDMAP_ACC_VIOL; 507 break; 508 case TPT_ERR_WRAP: 509 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 510 *ecode = RDMAP_TO_WRAP; 511 break; 512 case TPT_ERR_BOUND: 513 if (tagged) { 514 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 515 *ecode = DDPT_BASE_BOUNDS; 516 } else { 517 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 518 *ecode = RDMAP_BASE_BOUNDS; 519 } 520 break; 521 case TPT_ERR_INVALIDATE_SHARED_MR: 522 case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 523 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 524 *ecode = RDMAP_CANT_INV_STAG; 525 break; 526 case TPT_ERR_ECC: 527 case TPT_ERR_ECC_PSTAG: 528 case TPT_ERR_INTERNAL_ERR: 529 *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; 530 *ecode = 0; 531 break; 532 case TPT_ERR_OUT_OF_RQE: 533 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 534 *ecode = DDPU_INV_MSN_NOBUF; 535 break; 536 case TPT_ERR_PBL_ADDR_BOUND: 537 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 538 *ecode = DDPT_BASE_BOUNDS; 539 break; 540 case TPT_ERR_CRC: 541 *layer_type = LAYER_MPA|DDP_LLP; 542 *ecode = MPA_CRC_ERR; 543 break; 544 case TPT_ERR_MARKER: 545 *layer_type = LAYER_MPA|DDP_LLP; 546 *ecode = MPA_MARKER_ERR; 547 break; 548 case TPT_ERR_PDU_LEN_ERR: 549 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 550 *ecode = DDPU_MSG_TOOBIG; 551 break; 552 case TPT_ERR_DDP_VERSION: 553 if (tagged) { 554 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 555 *ecode = DDPT_INV_VERS; 556 } else { 557 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 558 *ecode = DDPU_INV_VERS; 559 } 560 break; 561 case TPT_ERR_RDMA_VERSION: 562 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 563 *ecode = RDMAP_INV_VERS; 564 break; 565 case TPT_ERR_OPCODE: 566 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 567 *ecode = RDMAP_INV_OPCODE; 568 break; 569 case TPT_ERR_DDP_QUEUE_NUM: 570 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 571 *ecode = DDPU_INV_QN; 572 break; 573 case TPT_ERR_MSN: 574 case TPT_ERR_MSN_GAP: 575 case TPT_ERR_MSN_RANGE: 576 case TPT_ERR_IRD_OVERFLOW: 577 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 578 *ecode = DDPU_INV_MSN_RANGE; 579 break; 580 case TPT_ERR_TBIT: 581 *layer_type = LAYER_DDP|DDP_LOCAL_CATA; 582 *ecode = 0; 583 break; 584 case TPT_ERR_MO: 585 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 586 *ecode = DDPU_INV_MO; 587 break; 588 default: 589 *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 590 *ecode = 0; 591 break; 592 } 593} 594 595/* 596 * This posts a TERMINATE with layer=RDMA, type=catastrophic. 597 */ 598int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) 599{ 600 union t3_wr *wqe; 601 struct terminate_message *term; 602 struct sk_buff *skb; 603 604 PDBG("%s %d\n", __FUNCTION__, __LINE__); 605 skb = alloc_skb(40, GFP_ATOMIC); 606 if (!skb) { 607 printk(KERN_ERR "%s cannot send TERMINATE!\n", __FUNCTION__); 608 return -ENOMEM; 609 } 610 wqe = (union t3_wr *)skb_put(skb, 40); 611 memset(wqe, 0, 40); 612 wqe->send.rdmaop = T3_TERMINATE; 613 614 /* immediate data length */ 615 wqe->send.plen = htonl(4); 616 617 /* immediate data starts here. */ 618 term = (struct terminate_message *)wqe->send.sgl; 619 build_term_codes(rsp_msg, &term->layer_etype, &term->ecode); 620 build_fw_riwrh((void *)wqe, T3_WR_SEND, 621 T3_COMPLETION_FLAG | T3_NOTIFY_FLAG, 1, 622 qhp->ep->hwtid, 5); 623 skb->priority = CPL_PRIORITY_DATA; 624 return cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); 625} 626 627/* 628 * Assumes qhp lock is held. 629 */ 630static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) 631{ 632 struct iwch_cq *rchp, *schp; 633 int count; 634 635 rchp = get_chp(qhp->rhp, qhp->attr.rcq); 636 schp = get_chp(qhp->rhp, qhp->attr.scq); 637 638 PDBG("%s qhp %p rchp %p schp %p\n", __FUNCTION__, qhp, rchp, schp); 639 /* take a ref on the qhp since we must release the lock */ 640 atomic_inc(&qhp->refcnt); 641 spin_unlock_irqrestore(&qhp->lock, *flag); 642 643 /* locking heirarchy: cq lock first, then qp lock. */ 644 spin_lock_irqsave(&rchp->lock, *flag); 645 spin_lock(&qhp->lock); 646 cxio_flush_hw_cq(&rchp->cq); 647 cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); 648 cxio_flush_rq(&qhp->wq, &rchp->cq, count); 649 spin_unlock(&qhp->lock); 650 spin_unlock_irqrestore(&rchp->lock, *flag); 651 652 /* locking heirarchy: cq lock first, then qp lock. */ 653 spin_lock_irqsave(&schp->lock, *flag); 654 spin_lock(&qhp->lock); 655 cxio_flush_hw_cq(&schp->cq); 656 cxio_count_scqes(&schp->cq, &qhp->wq, &count); 657 cxio_flush_sq(&qhp->wq, &schp->cq, count); 658 spin_unlock(&qhp->lock); 659 spin_unlock_irqrestore(&schp->lock, *flag); 660 661 /* deref */ 662 if (atomic_dec_and_test(&qhp->refcnt)) 663 wake_up(&qhp->wait); 664 665 spin_lock_irqsave(&qhp->lock, *flag); 666} 667 668static void flush_qp(struct iwch_qp *qhp, unsigned long *flag) 669{ 670 if (t3b_device(qhp->rhp)) 671 cxio_set_wq_in_error(&qhp->wq); 672 else 673 __flush_qp(qhp, flag); 674} 675 676 677/* 678 * Return non zero if at least one RECV was pre-posted. 679 */ 680static int rqes_posted(struct iwch_qp *qhp) 681{ 682 return fw_riwrh_opcode((struct fw_riwrh *)qhp->wq.queue) == T3_WR_RCV; 683} 684 685static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, 686 enum iwch_qp_attr_mask mask, 687 struct iwch_qp_attributes *attrs) 688{ 689 struct t3_rdma_init_attr init_attr; 690 int ret; 691 692 init_attr.tid = qhp->ep->hwtid; 693 init_attr.qpid = qhp->wq.qpid; 694 init_attr.pdid = qhp->attr.pd; 695 init_attr.scqid = qhp->attr.scq; 696 init_attr.rcqid = qhp->attr.rcq; 697 init_attr.rq_addr = qhp->wq.rq_addr; 698 init_attr.rq_size = 1 << qhp->wq.rq_size_log2; 699 init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | 700 qhp->attr.mpa_attr.recv_marker_enabled | 701 (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | 702 (qhp->attr.mpa_attr.crc_enabled << 2); 703 704 init_attr.qpcaps = 0x1f; 705 init_attr.tcp_emss = qhp->ep->emss; 706 init_attr.ord = qhp->attr.max_ord; 707 init_attr.ird = qhp->attr.max_ird; 708 init_attr.qp_dma_addr = qhp->wq.dma_addr; 709 init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); 710 init_attr.flags = rqes_posted(qhp) ? RECVS_POSTED : 0; 711 PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " 712 "flags 0x%x qpcaps 0x%x\n", __FUNCTION__, 713 init_attr.rq_addr, init_attr.rq_size, 714 init_attr.flags, init_attr.qpcaps); 715 ret = cxio_rdma_init(&rhp->rdev, &init_attr); 716 PDBG("%s ret %d\n", __FUNCTION__, ret); 717 return ret; 718} 719 720int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, 721 enum iwch_qp_attr_mask mask, 722 struct iwch_qp_attributes *attrs, 723 int internal) 724{ 725 int ret = 0; 726 struct iwch_qp_attributes newattr = qhp->attr; 727 unsigned long flag; 728 int disconnect = 0; 729 int terminate = 0; 730 int abort = 0; 731 int free = 0; 732 struct iwch_ep *ep = NULL; 733 734 PDBG("%s qhp %p qpid 0x%x ep %p state %d -> %d\n", __FUNCTION__, 735 qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, 736 (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); 737 738 spin_lock_irqsave(&qhp->lock, flag); 739 740 /* Process attr changes if in IDLE */ 741 if (mask & IWCH_QP_ATTR_VALID_MODIFY) { 742 if (qhp->attr.state != IWCH_QP_STATE_IDLE) { 743 ret = -EIO; 744 goto out; 745 } 746 if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) 747 newattr.enable_rdma_read = attrs->enable_rdma_read; 748 if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) 749 newattr.enable_rdma_write = attrs->enable_rdma_write; 750 if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) 751 newattr.enable_bind = attrs->enable_bind; 752 if (mask & IWCH_QP_ATTR_MAX_ORD) { 753 if (attrs->max_ord > 754 rhp->attr.max_rdma_read_qp_depth) { 755 ret = -EINVAL; 756 goto out; 757 } 758 newattr.max_ord = attrs->max_ord; 759 } 760 if (mask & IWCH_QP_ATTR_MAX_IRD) { 761 if (attrs->max_ird > 762 rhp->attr.max_rdma_reads_per_qp) { 763 ret = -EINVAL; 764 goto out; 765 } 766 newattr.max_ird = attrs->max_ird; 767 } 768 qhp->attr = newattr; 769 } 770 771 if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) 772 goto out; 773 if (qhp->attr.state == attrs->next_state) 774 goto out; 775 776 switch (qhp->attr.state) { 777 case IWCH_QP_STATE_IDLE: 778 switch (attrs->next_state) { 779 case IWCH_QP_STATE_RTS: 780 if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { 781 ret = -EINVAL; 782 goto out; 783 } 784 if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { 785 ret = -EINVAL; 786 goto out; 787 } 788 qhp->attr.mpa_attr = attrs->mpa_attr; 789 qhp->attr.llp_stream_handle = attrs->llp_stream_handle; 790 qhp->ep = qhp->attr.llp_stream_handle; 791 qhp->attr.state = IWCH_QP_STATE_RTS; 792 793 /* 794 * Ref the endpoint here and deref when we 795 * disassociate the endpoint from the QP. This 796 * happens in CLOSING->IDLE transition or *->ERROR 797 * transition. 798 */ 799 get_ep(&qhp->ep->com); 800 spin_unlock_irqrestore(&qhp->lock, flag); 801 ret = rdma_init(rhp, qhp, mask, attrs); 802 spin_lock_irqsave(&qhp->lock, flag); 803 if (ret) 804 goto err; 805 break; 806 case IWCH_QP_STATE_ERROR: 807 qhp->attr.state = IWCH_QP_STATE_ERROR; 808 flush_qp(qhp, &flag); 809 break; 810 default: 811 ret = -EINVAL; 812 goto out; 813 } 814 break; 815 case IWCH_QP_STATE_RTS: 816 switch (attrs->next_state) { 817 case IWCH_QP_STATE_CLOSING: 818 BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); 819 qhp->attr.state = IWCH_QP_STATE_CLOSING; 820 if (!internal) { 821 abort=0; 822 disconnect = 1; 823 ep = qhp->ep; 824 } 825 break; 826 case IWCH_QP_STATE_TERMINATE: 827 qhp->attr.state = IWCH_QP_STATE_TERMINATE; 828 if (t3b_device(qhp->rhp)) 829 cxio_set_wq_in_error(&qhp->wq); 830 if (!internal) 831 terminate = 1; 832 break; 833 case IWCH_QP_STATE_ERROR: 834 qhp->attr.state = IWCH_QP_STATE_ERROR; 835 if (!internal) { 836 abort=1; 837 disconnect = 1; 838 ep = qhp->ep; 839 } 840 goto err; 841 break; 842 default: 843 ret = -EINVAL; 844 goto out; 845 } 846 break; 847 case IWCH_QP_STATE_CLOSING: 848 if (!internal) { 849 ret = -EINVAL; 850 goto out; 851 } 852 switch (attrs->next_state) { 853 case IWCH_QP_STATE_IDLE: 854 qhp->attr.state = IWCH_QP_STATE_IDLE; 855 qhp->attr.llp_stream_handle = NULL; 856 put_ep(&qhp->ep->com); 857 qhp->ep = NULL; 858 wake_up(&qhp->wait); 859 break; 860 case IWCH_QP_STATE_ERROR: 861 goto err; 862 default: 863 ret = -EINVAL; 864 goto err; 865 } 866 break; 867 case IWCH_QP_STATE_ERROR: 868 if (attrs->next_state != IWCH_QP_STATE_IDLE) { 869 ret = -EINVAL; 870 goto out; 871 } 872 873 if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || 874 !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { 875 ret = -EINVAL; 876 goto out; 877 } 878 qhp->attr.state = IWCH_QP_STATE_IDLE; 879 memset(&qhp->attr, 0, sizeof(qhp->attr)); 880 break; 881 case IWCH_QP_STATE_TERMINATE: 882 if (!internal) { 883 ret = -EINVAL; 884 goto out; 885 } 886 goto err; 887 break; 888 default: 889 printk(KERN_ERR "%s in a bad state %d\n", 890 __FUNCTION__, qhp->attr.state); 891 ret = -EINVAL; 892 goto err; 893 break; 894 } 895 goto out; 896err: 897 PDBG("%s disassociating ep %p qpid 0x%x\n", __FUNCTION__, qhp->ep, 898 qhp->wq.qpid); 899 900 /* disassociate the LLP connection */ 901 qhp->attr.llp_stream_handle = NULL; 902 ep = qhp->ep; 903 qhp->ep = NULL; 904 qhp->attr.state = IWCH_QP_STATE_ERROR; 905 free=1; 906 wake_up(&qhp->wait); 907 BUG_ON(!ep); 908 flush_qp(qhp, &flag); 909out: 910 spin_unlock_irqrestore(&qhp->lock, flag); 911 912 if (terminate) 913 iwch_post_terminate(qhp, NULL); 914 915 /* 916 * If disconnect is 1, then we need to initiate a disconnect 917 * on the EP. This can be a normal close (RTS->CLOSING) or 918 * an abnormal close (RTS/CLOSING->ERROR). 919 */ 920 if (disconnect) 921 iwch_ep_disconnect(ep, abort, GFP_KERNEL); 922 923 /* 924 * If free is 1, then we've disassociated the EP from the QP 925 * and we need to dereference the EP. 926 */ 927 if (free) 928 put_ep(&ep->com); 929 930 PDBG("%s exit state %d\n", __FUNCTION__, qhp->attr.state); 931 return ret; 932} 933 934static int quiesce_qp(struct iwch_qp *qhp) 935{ 936 spin_lock_irq(&qhp->lock); 937 iwch_quiesce_tid(qhp->ep); 938 qhp->flags |= QP_QUIESCED; 939 spin_unlock_irq(&qhp->lock); 940 return 0; 941} 942 943static int resume_qp(struct iwch_qp *qhp) 944{ 945 spin_lock_irq(&qhp->lock); 946 iwch_resume_tid(qhp->ep); 947 qhp->flags &= ~QP_QUIESCED; 948 spin_unlock_irq(&qhp->lock); 949 return 0; 950} 951 952int iwch_quiesce_qps(struct iwch_cq *chp) 953{ 954 int i; 955 struct iwch_qp *qhp; 956 957 for (i=0; i < T3_MAX_NUM_QP; i++) { 958 qhp = get_qhp(chp->rhp, i); 959 if (!qhp) 960 continue; 961 if ((qhp->attr.rcq == chp->cq.cqid) && !qp_quiesced(qhp)) { 962 quiesce_qp(qhp); 963 continue; 964 } 965 if ((qhp->attr.scq == chp->cq.cqid) && !qp_quiesced(qhp)) 966 quiesce_qp(qhp); 967 } 968 return 0; 969} 970 971int iwch_resume_qps(struct iwch_cq *chp) 972{ 973 int i; 974 struct iwch_qp *qhp; 975 976 for (i=0; i < T3_MAX_NUM_QP; i++) { 977 qhp = get_qhp(chp->rhp, i); 978 if (!qhp) 979 continue; 980 if ((qhp->attr.rcq == chp->cq.cqid) && qp_quiesced(qhp)) { 981 resume_qp(qhp); 982 continue; 983 } 984 if ((qhp->attr.scq == chp->cq.cqid) && qp_quiesced(qhp)) 985 resume_qp(qhp); 986 } 987 return 0; 988} 989