1/* 2 * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved. 3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#include <linux/sched.h> 35#include <linux/spinlock.h> 36 37#include "ipath_verbs.h" 38#include "ipath_kernel.h" 39 40/* 41 * Convert the AETH RNR timeout code into the number of milliseconds. 42 */ 43const u32 ib_ipath_rnr_table[32] = { 44 656, /* 0 */ 45 1, /* 1 */ 46 1, /* 2 */ 47 1, /* 3 */ 48 1, /* 4 */ 49 1, /* 5 */ 50 1, /* 6 */ 51 1, /* 7 */ 52 1, /* 8 */ 53 1, /* 9 */ 54 1, /* A */ 55 1, /* B */ 56 1, /* C */ 57 1, /* D */ 58 2, /* E */ 59 2, /* F */ 60 3, /* 10 */ 61 4, /* 11 */ 62 6, /* 12 */ 63 8, /* 13 */ 64 11, /* 14 */ 65 16, /* 15 */ 66 21, /* 16 */ 67 31, /* 17 */ 68 41, /* 18 */ 69 62, /* 19 */ 70 82, /* 1A */ 71 123, /* 1B */ 72 164, /* 1C */ 73 246, /* 1D */ 74 328, /* 1E */ 75 492 /* 1F */ 76}; 77 78void ipath_insert_rnr_queue(struct ipath_qp *qp) 79{ 80 struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 81 82 /* We already did a spin_lock_irqsave(), so just use spin_lock */ 83 spin_lock(&dev->pending_lock); 84 if (list_empty(&dev->rnrwait)) 85 list_add(&qp->timerwait, &dev->rnrwait); 86 else { 87 struct list_head *l = &dev->rnrwait; 88 struct ipath_qp *nqp = list_entry(l->next, struct ipath_qp, 89 timerwait); 90 91 while (qp->s_rnr_timeout >= nqp->s_rnr_timeout) { 92 qp->s_rnr_timeout -= nqp->s_rnr_timeout; 93 l = l->next; 94 if (l->next == &dev->rnrwait) { 95 nqp = NULL; 96 break; 97 } 98 nqp = list_entry(l->next, struct ipath_qp, 99 timerwait); 100 } 101 if (nqp) 102 nqp->s_rnr_timeout -= qp->s_rnr_timeout; 103 list_add(&qp->timerwait, l); 104 } 105 spin_unlock(&dev->pending_lock); 106} 107 108/** 109 * ipath_init_sge - Validate a RWQE and fill in the SGE state 110 * @qp: the QP 111 * 112 * Return 1 if OK. 113 */ 114int ipath_init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe, 115 u32 *lengthp, struct ipath_sge_state *ss) 116{ 117 int i, j, ret; 118 struct ib_wc wc; 119 120 *lengthp = 0; 121 for (i = j = 0; i < wqe->num_sge; i++) { 122 if (wqe->sg_list[i].length == 0) 123 continue; 124 /* Check LKEY */ 125 if (!ipath_lkey_ok(qp, j ? &ss->sg_list[j - 1] : &ss->sge, 126 &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE)) 127 goto bad_lkey; 128 *lengthp += wqe->sg_list[i].length; 129 j++; 130 } 131 ss->num_sge = j; 132 ret = 1; 133 goto bail; 134 135bad_lkey: 136 memset(&wc, 0, sizeof(wc)); 137 wc.wr_id = wqe->wr_id; 138 wc.status = IB_WC_LOC_PROT_ERR; 139 wc.opcode = IB_WC_RECV; 140 wc.qp = &qp->ibqp; 141 /* Signal solicited completion event. */ 142 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1); 143 ret = 0; 144bail: 145 return ret; 146} 147 148/** 149 * ipath_get_rwqe - copy the next RWQE into the QP's RWQE 150 * @qp: the QP 151 * @wr_id_only: update qp->r_wr_id only, not qp->r_sge 152 * 153 * Return 0 if no RWQE is available, otherwise return 1. 154 * 155 * Can be called from interrupt level. 156 */ 157int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only) 158{ 159 unsigned long flags; 160 struct ipath_rq *rq; 161 struct ipath_rwq *wq; 162 struct ipath_srq *srq; 163 struct ipath_rwqe *wqe; 164 void (*handler)(struct ib_event *, void *); 165 u32 tail; 166 int ret; 167 168 if (qp->ibqp.srq) { 169 srq = to_isrq(qp->ibqp.srq); 170 handler = srq->ibsrq.event_handler; 171 rq = &srq->rq; 172 } else { 173 srq = NULL; 174 handler = NULL; 175 rq = &qp->r_rq; 176 } 177 178 spin_lock_irqsave(&rq->lock, flags); 179 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) { 180 ret = 0; 181 goto unlock; 182 } 183 184 wq = rq->wq; 185 tail = wq->tail; 186 /* Validate tail before using it since it is user writable. */ 187 if (tail >= rq->size) 188 tail = 0; 189 do { 190 if (unlikely(tail == wq->head)) { 191 ret = 0; 192 goto unlock; 193 } 194 /* Make sure entry is read after head index is read. */ 195 smp_rmb(); 196 wqe = get_rwqe_ptr(rq, tail); 197 if (++tail >= rq->size) 198 tail = 0; 199 if (wr_id_only) 200 break; 201 qp->r_sge.sg_list = qp->r_sg_list; 202 } while (!ipath_init_sge(qp, wqe, &qp->r_len, &qp->r_sge)); 203 qp->r_wr_id = wqe->wr_id; 204 wq->tail = tail; 205 206 ret = 1; 207 set_bit(IPATH_R_WRID_VALID, &qp->r_aflags); 208 if (handler) { 209 u32 n; 210 211 /* 212 * validate head pointer value and compute 213 * the number of remaining WQEs. 214 */ 215 n = wq->head; 216 if (n >= rq->size) 217 n = 0; 218 if (n < tail) 219 n += rq->size - tail; 220 else 221 n -= tail; 222 if (n < srq->limit) { 223 struct ib_event ev; 224 225 srq->limit = 0; 226 spin_unlock_irqrestore(&rq->lock, flags); 227 ev.device = qp->ibqp.device; 228 ev.element.srq = qp->ibqp.srq; 229 ev.event = IB_EVENT_SRQ_LIMIT_REACHED; 230 handler(&ev, srq->ibsrq.srq_context); 231 goto bail; 232 } 233 } 234unlock: 235 spin_unlock_irqrestore(&rq->lock, flags); 236bail: 237 return ret; 238} 239 240/** 241 * ipath_ruc_loopback - handle UC and RC lookback requests 242 * @sqp: the sending QP 243 * 244 * This is called from ipath_do_send() to 245 * forward a WQE addressed to the same HCA. 246 * Note that although we are single threaded due to the tasklet, we still 247 * have to protect against post_send(). We don't have to worry about 248 * receive interrupts since this is a connected protocol and all packets 249 * will pass through here. 250 */ 251static void ipath_ruc_loopback(struct ipath_qp *sqp) 252{ 253 struct ipath_ibdev *dev = to_idev(sqp->ibqp.device); 254 struct ipath_qp *qp; 255 struct ipath_swqe *wqe; 256 struct ipath_sge *sge; 257 unsigned long flags; 258 struct ib_wc wc; 259 u64 sdata; 260 atomic64_t *maddr; 261 enum ib_wc_status send_status; 262 263 /* 264 * Note that we check the responder QP state after 265 * checking the requester's state. 266 */ 267 qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn); 268 269 spin_lock_irqsave(&sqp->s_lock, flags); 270 271 /* Return if we are already busy processing a work request. */ 272 if ((sqp->s_flags & (IPATH_S_BUSY | IPATH_S_ANY_WAIT)) || 273 !(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_OR_FLUSH_SEND)) 274 goto unlock; 275 276 sqp->s_flags |= IPATH_S_BUSY; 277 278again: 279 if (sqp->s_last == sqp->s_head) 280 goto clr_busy; 281 wqe = get_swqe_ptr(sqp, sqp->s_last); 282 283 /* Return if it is not OK to start a new work reqeust. */ 284 if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_NEXT_SEND_OK)) { 285 if (!(ib_ipath_state_ops[sqp->state] & IPATH_FLUSH_SEND)) 286 goto clr_busy; 287 /* We are in the error state, flush the work request. */ 288 send_status = IB_WC_WR_FLUSH_ERR; 289 goto flush_send; 290 } 291 292 /* 293 * We can rely on the entry not changing without the s_lock 294 * being held until we update s_last. 295 * We increment s_cur to indicate s_last is in progress. 296 */ 297 if (sqp->s_last == sqp->s_cur) { 298 if (++sqp->s_cur >= sqp->s_size) 299 sqp->s_cur = 0; 300 } 301 spin_unlock_irqrestore(&sqp->s_lock, flags); 302 303 if (!qp || !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) { 304 dev->n_pkt_drops++; 305 /* 306 * For RC, the requester would timeout and retry so 307 * shortcut the timeouts and just signal too many retries. 308 */ 309 if (sqp->ibqp.qp_type == IB_QPT_RC) 310 send_status = IB_WC_RETRY_EXC_ERR; 311 else 312 send_status = IB_WC_SUCCESS; 313 goto serr; 314 } 315 316 memset(&wc, 0, sizeof wc); 317 send_status = IB_WC_SUCCESS; 318 319 sqp->s_sge.sge = wqe->sg_list[0]; 320 sqp->s_sge.sg_list = wqe->sg_list + 1; 321 sqp->s_sge.num_sge = wqe->wr.num_sge; 322 sqp->s_len = wqe->length; 323 switch (wqe->wr.opcode) { 324 case IB_WR_SEND_WITH_IMM: 325 wc.wc_flags = IB_WC_WITH_IMM; 326 wc.ex.imm_data = wqe->wr.ex.imm_data; 327 /* FALLTHROUGH */ 328 case IB_WR_SEND: 329 if (!ipath_get_rwqe(qp, 0)) 330 goto rnr_nak; 331 break; 332 333 case IB_WR_RDMA_WRITE_WITH_IMM: 334 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 335 goto inv_err; 336 wc.wc_flags = IB_WC_WITH_IMM; 337 wc.ex.imm_data = wqe->wr.ex.imm_data; 338 if (!ipath_get_rwqe(qp, 1)) 339 goto rnr_nak; 340 /* FALLTHROUGH */ 341 case IB_WR_RDMA_WRITE: 342 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 343 goto inv_err; 344 if (wqe->length == 0) 345 break; 346 if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, wqe->length, 347 wqe->wr.wr.rdma.remote_addr, 348 wqe->wr.wr.rdma.rkey, 349 IB_ACCESS_REMOTE_WRITE))) 350 goto acc_err; 351 break; 352 353 case IB_WR_RDMA_READ: 354 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 355 goto inv_err; 356 if (unlikely(!ipath_rkey_ok(qp, &sqp->s_sge, wqe->length, 357 wqe->wr.wr.rdma.remote_addr, 358 wqe->wr.wr.rdma.rkey, 359 IB_ACCESS_REMOTE_READ))) 360 goto acc_err; 361 qp->r_sge.sge = wqe->sg_list[0]; 362 qp->r_sge.sg_list = wqe->sg_list + 1; 363 qp->r_sge.num_sge = wqe->wr.num_sge; 364 break; 365 366 case IB_WR_ATOMIC_CMP_AND_SWP: 367 case IB_WR_ATOMIC_FETCH_AND_ADD: 368 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) 369 goto inv_err; 370 if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, sizeof(u64), 371 wqe->wr.wr.atomic.remote_addr, 372 wqe->wr.wr.atomic.rkey, 373 IB_ACCESS_REMOTE_ATOMIC))) 374 goto acc_err; 375 /* Perform atomic OP and save result. */ 376 maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 377 sdata = wqe->wr.wr.atomic.compare_add; 378 *(u64 *) sqp->s_sge.sge.vaddr = 379 (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ? 380 (u64) atomic64_add_return(sdata, maddr) - sdata : 381 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 382 sdata, wqe->wr.wr.atomic.swap); 383 goto send_comp; 384 385 default: 386 send_status = IB_WC_LOC_QP_OP_ERR; 387 goto serr; 388 } 389 390 sge = &sqp->s_sge.sge; 391 while (sqp->s_len) { 392 u32 len = sqp->s_len; 393 394 if (len > sge->length) 395 len = sge->length; 396 if (len > sge->sge_length) 397 len = sge->sge_length; 398 BUG_ON(len == 0); 399 ipath_copy_sge(&qp->r_sge, sge->vaddr, len); 400 sge->vaddr += len; 401 sge->length -= len; 402 sge->sge_length -= len; 403 if (sge->sge_length == 0) { 404 if (--sqp->s_sge.num_sge) 405 *sge = *sqp->s_sge.sg_list++; 406 } else if (sge->length == 0 && sge->mr != NULL) { 407 if (++sge->n >= IPATH_SEGSZ) { 408 if (++sge->m >= sge->mr->mapsz) 409 break; 410 sge->n = 0; 411 } 412 sge->vaddr = 413 sge->mr->map[sge->m]->segs[sge->n].vaddr; 414 sge->length = 415 sge->mr->map[sge->m]->segs[sge->n].length; 416 } 417 sqp->s_len -= len; 418 } 419 420 if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) 421 goto send_comp; 422 423 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM) 424 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 425 else 426 wc.opcode = IB_WC_RECV; 427 wc.wr_id = qp->r_wr_id; 428 wc.status = IB_WC_SUCCESS; 429 wc.byte_len = wqe->length; 430 wc.qp = &qp->ibqp; 431 wc.src_qp = qp->remote_qpn; 432 wc.slid = qp->remote_ah_attr.dlid; 433 wc.sl = qp->remote_ah_attr.sl; 434 wc.port_num = 1; 435 /* Signal completion event if the solicited bit is set. */ 436 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 437 wqe->wr.send_flags & IB_SEND_SOLICITED); 438 439send_comp: 440 spin_lock_irqsave(&sqp->s_lock, flags); 441flush_send: 442 sqp->s_rnr_retry = sqp->s_rnr_retry_cnt; 443 ipath_send_complete(sqp, wqe, send_status); 444 goto again; 445 446rnr_nak: 447 /* Handle RNR NAK */ 448 if (qp->ibqp.qp_type == IB_QPT_UC) 449 goto send_comp; 450 /* 451 * Note: we don't need the s_lock held since the BUSY flag 452 * makes this single threaded. 453 */ 454 if (sqp->s_rnr_retry == 0) { 455 send_status = IB_WC_RNR_RETRY_EXC_ERR; 456 goto serr; 457 } 458 if (sqp->s_rnr_retry_cnt < 7) 459 sqp->s_rnr_retry--; 460 spin_lock_irqsave(&sqp->s_lock, flags); 461 if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_RECV_OK)) 462 goto clr_busy; 463 sqp->s_flags |= IPATH_S_WAITING; 464 dev->n_rnr_naks++; 465 sqp->s_rnr_timeout = ib_ipath_rnr_table[qp->r_min_rnr_timer]; 466 ipath_insert_rnr_queue(sqp); 467 goto clr_busy; 468 469inv_err: 470 send_status = IB_WC_REM_INV_REQ_ERR; 471 wc.status = IB_WC_LOC_QP_OP_ERR; 472 goto err; 473 474acc_err: 475 send_status = IB_WC_REM_ACCESS_ERR; 476 wc.status = IB_WC_LOC_PROT_ERR; 477err: 478 /* responder goes to error state */ 479 ipath_rc_error(qp, wc.status); 480 481serr: 482 spin_lock_irqsave(&sqp->s_lock, flags); 483 ipath_send_complete(sqp, wqe, send_status); 484 if (sqp->ibqp.qp_type == IB_QPT_RC) { 485 int lastwqe = ipath_error_qp(sqp, IB_WC_WR_FLUSH_ERR); 486 487 sqp->s_flags &= ~IPATH_S_BUSY; 488 spin_unlock_irqrestore(&sqp->s_lock, flags); 489 if (lastwqe) { 490 struct ib_event ev; 491 492 ev.device = sqp->ibqp.device; 493 ev.element.qp = &sqp->ibqp; 494 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 495 sqp->ibqp.event_handler(&ev, sqp->ibqp.qp_context); 496 } 497 goto done; 498 } 499clr_busy: 500 sqp->s_flags &= ~IPATH_S_BUSY; 501unlock: 502 spin_unlock_irqrestore(&sqp->s_lock, flags); 503done: 504 if (qp && atomic_dec_and_test(&qp->refcount)) 505 wake_up(&qp->wait); 506} 507 508static void want_buffer(struct ipath_devdata *dd, struct ipath_qp *qp) 509{ 510 if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA) || 511 qp->ibqp.qp_type == IB_QPT_SMI) { 512 unsigned long flags; 513 514 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); 515 dd->ipath_sendctrl |= INFINIPATH_S_PIOINTBUFAVAIL; 516 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, 517 dd->ipath_sendctrl); 518 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 519 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); 520 } 521} 522 523/** 524 * ipath_no_bufs_available - tell the layer driver we need buffers 525 * @qp: the QP that caused the problem 526 * @dev: the device we ran out of buffers on 527 * 528 * Called when we run out of PIO buffers. 529 * If we are now in the error state, return zero to flush the 530 * send work request. 531 */ 532static int ipath_no_bufs_available(struct ipath_qp *qp, 533 struct ipath_ibdev *dev) 534{ 535 unsigned long flags; 536 int ret = 1; 537 538 /* 539 * Note that as soon as want_buffer() is called and 540 * possibly before it returns, ipath_ib_piobufavail() 541 * could be called. Therefore, put QP on the piowait list before 542 * enabling the PIO avail interrupt. 543 */ 544 spin_lock_irqsave(&qp->s_lock, flags); 545 if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) { 546 dev->n_piowait++; 547 qp->s_flags |= IPATH_S_WAITING; 548 qp->s_flags &= ~IPATH_S_BUSY; 549 spin_lock(&dev->pending_lock); 550 if (list_empty(&qp->piowait)) 551 list_add_tail(&qp->piowait, &dev->piowait); 552 spin_unlock(&dev->pending_lock); 553 } else 554 ret = 0; 555 spin_unlock_irqrestore(&qp->s_lock, flags); 556 if (ret) 557 want_buffer(dev->dd, qp); 558 return ret; 559} 560 561/** 562 * ipath_make_grh - construct a GRH header 563 * @dev: a pointer to the ipath device 564 * @hdr: a pointer to the GRH header being constructed 565 * @grh: the global route address to send to 566 * @hwords: the number of 32 bit words of header being sent 567 * @nwords: the number of 32 bit words of data being sent 568 * 569 * Return the size of the header in 32 bit words. 570 */ 571u32 ipath_make_grh(struct ipath_ibdev *dev, struct ib_grh *hdr, 572 struct ib_global_route *grh, u32 hwords, u32 nwords) 573{ 574 hdr->version_tclass_flow = 575 cpu_to_be32((6 << 28) | 576 (grh->traffic_class << 20) | 577 grh->flow_label); 578 hdr->paylen = cpu_to_be16((hwords - 2 + nwords + SIZE_OF_CRC) << 2); 579 /* next_hdr is defined by C8-7 in ch. 8.4.1 */ 580 hdr->next_hdr = 0x1B; 581 hdr->hop_limit = grh->hop_limit; 582 /* The SGID is 32-bit aligned. */ 583 hdr->sgid.global.subnet_prefix = dev->gid_prefix; 584 hdr->sgid.global.interface_id = dev->dd->ipath_guid; 585 hdr->dgid = grh->dgid; 586 587 /* GRH header size in 32-bit words. */ 588 return sizeof(struct ib_grh) / sizeof(u32); 589} 590 591void ipath_make_ruc_header(struct ipath_ibdev *dev, struct ipath_qp *qp, 592 struct ipath_other_headers *ohdr, 593 u32 bth0, u32 bth2) 594{ 595 u16 lrh0; 596 u32 nwords; 597 u32 extra_bytes; 598 599 /* Construct the header. */ 600 extra_bytes = -qp->s_cur_size & 3; 601 nwords = (qp->s_cur_size + extra_bytes) >> 2; 602 lrh0 = IPATH_LRH_BTH; 603 if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { 604 qp->s_hdrwords += ipath_make_grh(dev, &qp->s_hdr.u.l.grh, 605 &qp->remote_ah_attr.grh, 606 qp->s_hdrwords, nwords); 607 lrh0 = IPATH_LRH_GRH; 608 } 609 lrh0 |= qp->remote_ah_attr.sl << 4; 610 qp->s_hdr.lrh[0] = cpu_to_be16(lrh0); 611 qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); 612 qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC); 613 qp->s_hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid | 614 qp->remote_ah_attr.src_path_bits); 615 bth0 |= ipath_get_pkey(dev->dd, qp->s_pkey_index); 616 bth0 |= extra_bytes << 20; 617 ohdr->bth[0] = cpu_to_be32(bth0 | (1 << 22)); 618 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 619 ohdr->bth[2] = cpu_to_be32(bth2); 620} 621 622/** 623 * ipath_do_send - perform a send on a QP 624 * @data: contains a pointer to the QP 625 * 626 * Process entries in the send work queue until credit or queue is 627 * exhausted. Only allow one CPU to send a packet per QP (tasklet). 628 * Otherwise, two threads could send packets out of order. 629 */ 630void ipath_do_send(unsigned long data) 631{ 632 struct ipath_qp *qp = (struct ipath_qp *)data; 633 struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 634 int (*make_req)(struct ipath_qp *qp); 635 unsigned long flags; 636 637 if ((qp->ibqp.qp_type == IB_QPT_RC || 638 qp->ibqp.qp_type == IB_QPT_UC) && 639 qp->remote_ah_attr.dlid == dev->dd->ipath_lid) { 640 ipath_ruc_loopback(qp); 641 goto bail; 642 } 643 644 if (qp->ibqp.qp_type == IB_QPT_RC) 645 make_req = ipath_make_rc_req; 646 else if (qp->ibqp.qp_type == IB_QPT_UC) 647 make_req = ipath_make_uc_req; 648 else 649 make_req = ipath_make_ud_req; 650 651 spin_lock_irqsave(&qp->s_lock, flags); 652 653 /* Return if we are already busy processing a work request. */ 654 if ((qp->s_flags & (IPATH_S_BUSY | IPATH_S_ANY_WAIT)) || 655 !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_OR_FLUSH_SEND)) { 656 spin_unlock_irqrestore(&qp->s_lock, flags); 657 goto bail; 658 } 659 660 qp->s_flags |= IPATH_S_BUSY; 661 662 spin_unlock_irqrestore(&qp->s_lock, flags); 663 664again: 665 /* Check for a constructed packet to be sent. */ 666 if (qp->s_hdrwords != 0) { 667 /* 668 * If no PIO bufs are available, return. An interrupt will 669 * call ipath_ib_piobufavail() when one is available. 670 */ 671 if (ipath_verbs_send(qp, &qp->s_hdr, qp->s_hdrwords, 672 qp->s_cur_sge, qp->s_cur_size)) { 673 if (ipath_no_bufs_available(qp, dev)) 674 goto bail; 675 } 676 dev->n_unicast_xmit++; 677 /* Record that we sent the packet and s_hdr is empty. */ 678 qp->s_hdrwords = 0; 679 } 680 681 if (make_req(qp)) 682 goto again; 683 684bail:; 685} 686 687/* 688 * This should be called with s_lock held. 689 */ 690void ipath_send_complete(struct ipath_qp *qp, struct ipath_swqe *wqe, 691 enum ib_wc_status status) 692{ 693 u32 old_last, last; 694 695 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_OR_FLUSH_SEND)) 696 return; 697 698 /* See ch. 11.2.4.1 and 10.7.3.1 */ 699 if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) || 700 (wqe->wr.send_flags & IB_SEND_SIGNALED) || 701 status != IB_WC_SUCCESS) { 702 struct ib_wc wc; 703 704 memset(&wc, 0, sizeof wc); 705 wc.wr_id = wqe->wr.wr_id; 706 wc.status = status; 707 wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 708 wc.qp = &qp->ibqp; 709 if (status == IB_WC_SUCCESS) 710 wc.byte_len = wqe->length; 711 ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 712 status != IB_WC_SUCCESS); 713 } 714 715 old_last = last = qp->s_last; 716 if (++last >= qp->s_size) 717 last = 0; 718 qp->s_last = last; 719 if (qp->s_cur == old_last) 720 qp->s_cur = last; 721 if (qp->s_tail == old_last) 722 qp->s_tail = last; 723 if (qp->state == IB_QPS_SQD && last == qp->s_cur) 724 qp->s_draining = 0; 725} 726