1/* 2 * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved. 3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#include <linux/err.h> 35#include <linux/sched.h> 36#include <linux/slab.h> 37#include <linux/vmalloc.h> 38 39#include "ipath_verbs.h" 40#include "ipath_kernel.h" 41 42#define BITS_PER_PAGE (PAGE_SIZE*BITS_PER_BYTE) 43#define BITS_PER_PAGE_MASK (BITS_PER_PAGE-1) 44#define mk_qpn(qpt, map, off) (((map) - (qpt)->map) * BITS_PER_PAGE + \ 45 (off)) 46#define find_next_offset(map, off) find_next_zero_bit((map)->page, \ 47 BITS_PER_PAGE, off) 48 49/* 50 * Convert the AETH credit code into the number of credits. 51 */ 52static u32 credit_table[31] = { 53 0, /* 0 */ 54 1, /* 1 */ 55 2, /* 2 */ 56 3, /* 3 */ 57 4, /* 4 */ 58 6, /* 5 */ 59 8, /* 6 */ 60 12, /* 7 */ 61 16, /* 8 */ 62 24, /* 9 */ 63 32, /* A */ 64 48, /* B */ 65 64, /* C */ 66 96, /* D */ 67 128, /* E */ 68 192, /* F */ 69 256, /* 10 */ 70 384, /* 11 */ 71 512, /* 12 */ 72 768, /* 13 */ 73 1024, /* 14 */ 74 1536, /* 15 */ 75 2048, /* 16 */ 76 3072, /* 17 */ 77 4096, /* 18 */ 78 6144, /* 19 */ 79 8192, /* 1A */ 80 12288, /* 1B */ 81 16384, /* 1C */ 82 24576, /* 1D */ 83 32768 /* 1E */ 84}; 85 86 87static void get_map_page(struct ipath_qp_table *qpt, struct qpn_map *map) 88{ 89 unsigned long page = get_zeroed_page(GFP_KERNEL); 90 unsigned long flags; 91 92 /* 93 * Free the page if someone raced with us installing it. 94 */ 95 96 spin_lock_irqsave(&qpt->lock, flags); 97 if (map->page) 98 free_page(page); 99 else 100 map->page = (void *)page; 101 spin_unlock_irqrestore(&qpt->lock, flags); 102} 103 104 105static int alloc_qpn(struct ipath_qp_table *qpt, enum ib_qp_type type) 106{ 107 u32 i, offset, max_scan, qpn; 108 struct qpn_map *map; 109 u32 ret = -1; 110 111 if (type == IB_QPT_SMI) 112 ret = 0; 113 else if (type == IB_QPT_GSI) 114 ret = 1; 115 116 if (ret != -1) { 117 map = &qpt->map[0]; 118 if (unlikely(!map->page)) { 119 get_map_page(qpt, map); 120 if (unlikely(!map->page)) { 121 ret = -ENOMEM; 122 goto bail; 123 } 124 } 125 if (!test_and_set_bit(ret, map->page)) 126 atomic_dec(&map->n_free); 127 else 128 ret = -EBUSY; 129 goto bail; 130 } 131 132 qpn = qpt->last + 1; 133 if (qpn >= QPN_MAX) 134 qpn = 2; 135 offset = qpn & BITS_PER_PAGE_MASK; 136 map = &qpt->map[qpn / BITS_PER_PAGE]; 137 max_scan = qpt->nmaps - !offset; 138 for (i = 0;;) { 139 if (unlikely(!map->page)) { 140 get_map_page(qpt, map); 141 if (unlikely(!map->page)) 142 break; 143 } 144 if (likely(atomic_read(&map->n_free))) { 145 do { 146 if (!test_and_set_bit(offset, map->page)) { 147 atomic_dec(&map->n_free); 148 qpt->last = qpn; 149 ret = qpn; 150 goto bail; 151 } 152 offset = find_next_offset(map, offset); 153 qpn = mk_qpn(qpt, map, offset); 154 /* 155 * This test differs from alloc_pidmap(). 156 * If find_next_offset() does find a zero 157 * bit, we don't need to check for QPN 158 * wrapping around past our starting QPN. 159 * We just need to be sure we don't loop 160 * forever. 161 */ 162 } while (offset < BITS_PER_PAGE && qpn < QPN_MAX); 163 } 164 /* 165 * In order to keep the number of pages allocated to a 166 * minimum, we scan the all existing pages before increasing 167 * the size of the bitmap table. 168 */ 169 if (++i > max_scan) { 170 if (qpt->nmaps == QPNMAP_ENTRIES) 171 break; 172 map = &qpt->map[qpt->nmaps++]; 173 offset = 0; 174 } else if (map < &qpt->map[qpt->nmaps]) { 175 ++map; 176 offset = 0; 177 } else { 178 map = &qpt->map[0]; 179 offset = 2; 180 } 181 qpn = mk_qpn(qpt, map, offset); 182 } 183 184 ret = -ENOMEM; 185 186bail: 187 return ret; 188} 189 190static void free_qpn(struct ipath_qp_table *qpt, u32 qpn) 191{ 192 struct qpn_map *map; 193 194 map = qpt->map + qpn / BITS_PER_PAGE; 195 if (map->page) 196 clear_bit(qpn & BITS_PER_PAGE_MASK, map->page); 197 atomic_inc(&map->n_free); 198} 199 200/** 201 * ipath_alloc_qpn - allocate a QP number 202 * @qpt: the QP table 203 * @qp: the QP 204 * @type: the QP type (IB_QPT_SMI and IB_QPT_GSI are special) 205 * 206 * Allocate the next available QPN and put the QP into the hash table. 207 * The hash table holds a reference to the QP. 208 */ 209static int ipath_alloc_qpn(struct ipath_qp_table *qpt, struct ipath_qp *qp, 210 enum ib_qp_type type) 211{ 212 unsigned long flags; 213 int ret; 214 215 ret = alloc_qpn(qpt, type); 216 if (ret < 0) 217 goto bail; 218 qp->ibqp.qp_num = ret; 219 220 /* Add the QP to the hash table. */ 221 spin_lock_irqsave(&qpt->lock, flags); 222 223 ret %= qpt->max; 224 qp->next = qpt->table[ret]; 225 qpt->table[ret] = qp; 226 atomic_inc(&qp->refcount); 227 228 spin_unlock_irqrestore(&qpt->lock, flags); 229 ret = 0; 230 231bail: 232 return ret; 233} 234 235/** 236 * ipath_free_qp - remove a QP from the QP table 237 * @qpt: the QP table 238 * @qp: the QP to remove 239 * 240 * Remove the QP from the table so it can't be found asynchronously by 241 * the receive interrupt routine. 242 */ 243static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp) 244{ 245 struct ipath_qp *q, **qpp; 246 unsigned long flags; 247 248 spin_lock_irqsave(&qpt->lock, flags); 249 250 /* Remove QP from the hash table. */ 251 qpp = &qpt->table[qp->ibqp.qp_num % qpt->max]; 252 for (; (q = *qpp) != NULL; qpp = &q->next) { 253 if (q == qp) { 254 *qpp = qp->next; 255 qp->next = NULL; 256 atomic_dec(&qp->refcount); 257 break; 258 } 259 } 260 261 spin_unlock_irqrestore(&qpt->lock, flags); 262} 263 264/** 265 * ipath_free_all_qps - check for QPs still in use 266 * @qpt: the QP table to empty 267 * 268 * There should not be any QPs still in use. 269 * Free memory for table. 270 */ 271unsigned ipath_free_all_qps(struct ipath_qp_table *qpt) 272{ 273 unsigned long flags; 274 struct ipath_qp *qp; 275 u32 n, qp_inuse = 0; 276 277 spin_lock_irqsave(&qpt->lock, flags); 278 for (n = 0; n < qpt->max; n++) { 279 qp = qpt->table[n]; 280 qpt->table[n] = NULL; 281 282 for (; qp; qp = qp->next) 283 qp_inuse++; 284 } 285 spin_unlock_irqrestore(&qpt->lock, flags); 286 287 for (n = 0; n < ARRAY_SIZE(qpt->map); n++) 288 if (qpt->map[n].page) 289 free_page((unsigned long) qpt->map[n].page); 290 return qp_inuse; 291} 292 293/** 294 * ipath_lookup_qpn - return the QP with the given QPN 295 * @qpt: the QP table 296 * @qpn: the QP number to look up 297 * 298 * The caller is responsible for decrementing the QP reference count 299 * when done. 300 */ 301struct ipath_qp *ipath_lookup_qpn(struct ipath_qp_table *qpt, u32 qpn) 302{ 303 unsigned long flags; 304 struct ipath_qp *qp; 305 306 spin_lock_irqsave(&qpt->lock, flags); 307 308 for (qp = qpt->table[qpn % qpt->max]; qp; qp = qp->next) { 309 if (qp->ibqp.qp_num == qpn) { 310 atomic_inc(&qp->refcount); 311 break; 312 } 313 } 314 315 spin_unlock_irqrestore(&qpt->lock, flags); 316 return qp; 317} 318 319/** 320 * ipath_reset_qp - initialize the QP state to the reset state 321 * @qp: the QP to reset 322 * @type: the QP type 323 */ 324static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type) 325{ 326 qp->remote_qpn = 0; 327 qp->qkey = 0; 328 qp->qp_access_flags = 0; 329 atomic_set(&qp->s_dma_busy, 0); 330 qp->s_flags &= IPATH_S_SIGNAL_REQ_WR; 331 qp->s_hdrwords = 0; 332 qp->s_wqe = NULL; 333 qp->s_pkt_delay = 0; 334 qp->s_draining = 0; 335 qp->s_psn = 0; 336 qp->r_psn = 0; 337 qp->r_msn = 0; 338 if (type == IB_QPT_RC) { 339 qp->s_state = IB_OPCODE_RC_SEND_LAST; 340 qp->r_state = IB_OPCODE_RC_SEND_LAST; 341 } else { 342 qp->s_state = IB_OPCODE_UC_SEND_LAST; 343 qp->r_state = IB_OPCODE_UC_SEND_LAST; 344 } 345 qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE; 346 qp->r_nak_state = 0; 347 qp->r_aflags = 0; 348 qp->r_flags = 0; 349 qp->s_rnr_timeout = 0; 350 qp->s_head = 0; 351 qp->s_tail = 0; 352 qp->s_cur = 0; 353 qp->s_last = 0; 354 qp->s_ssn = 1; 355 qp->s_lsn = 0; 356 memset(qp->s_ack_queue, 0, sizeof(qp->s_ack_queue)); 357 qp->r_head_ack_queue = 0; 358 qp->s_tail_ack_queue = 0; 359 qp->s_num_rd_atomic = 0; 360 if (qp->r_rq.wq) { 361 qp->r_rq.wq->head = 0; 362 qp->r_rq.wq->tail = 0; 363 } 364} 365 366/** 367 * ipath_error_qp - put a QP into the error state 368 * @qp: the QP to put into the error state 369 * @err: the receive completion error to signal if a RWQE is active 370 * 371 * Flushes both send and receive work queues. 372 * Returns true if last WQE event should be generated. 373 * The QP s_lock should be held and interrupts disabled. 374 * If we are already in error state, just return. 375 */ 376 377int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err) 378{ 379 struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 380 struct ib_wc wc; 381 int ret = 0; 382 383 if (qp->state == IB_QPS_ERR) 384 goto bail; 385 386 qp->state = IB_QPS_ERR; 387 388 spin_lock(&dev->pending_lock); 389 if (!list_empty(&qp->timerwait)) 390 list_del_init(&qp->timerwait); 391 if (!list_empty(&qp->piowait)) 392 list_del_init(&qp->piowait); 393 spin_unlock(&dev->pending_lock); 394 395 /* Schedule the sending tasklet to drain the send work queue. */ 396 if (qp->s_last != qp->s_head) 397 ipath_schedule_send(qp); 398 399 memset(&wc, 0, sizeof(wc)); 400 wc.qp = &qp->ibqp; 401 wc.opcode = IB_WC_RECV; 402 403 if (test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) { 404 wc.wr_id = qp->r_wr_id; 405 wc.status = err; 406 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1); 407 } 408 wc.status = IB_WC_WR_FLUSH_ERR; 409 410 if (qp->r_rq.wq) { 411 struct ipath_rwq *wq; 412 u32 head; 413 u32 tail; 414 415 spin_lock(&qp->r_rq.lock); 416 417 /* sanity check pointers before trusting them */ 418 wq = qp->r_rq.wq; 419 head = wq->head; 420 if (head >= qp->r_rq.size) 421 head = 0; 422 tail = wq->tail; 423 if (tail >= qp->r_rq.size) 424 tail = 0; 425 while (tail != head) { 426 wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id; 427 if (++tail >= qp->r_rq.size) 428 tail = 0; 429 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1); 430 } 431 wq->tail = tail; 432 433 spin_unlock(&qp->r_rq.lock); 434 } else if (qp->ibqp.event_handler) 435 ret = 1; 436 437bail: 438 return ret; 439} 440 441/** 442 * ipath_modify_qp - modify the attributes of a queue pair 443 * @ibqp: the queue pair who's attributes we're modifying 444 * @attr: the new attributes 445 * @attr_mask: the mask of attributes to modify 446 * @udata: user data for ipathverbs.so 447 * 448 * Returns 0 on success, otherwise returns an errno. 449 */ 450int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 451 int attr_mask, struct ib_udata *udata) 452{ 453 struct ipath_ibdev *dev = to_idev(ibqp->device); 454 struct ipath_qp *qp = to_iqp(ibqp); 455 enum ib_qp_state cur_state, new_state; 456 int lastwqe = 0; 457 int ret; 458 459 spin_lock_irq(&qp->s_lock); 460 461 cur_state = attr_mask & IB_QP_CUR_STATE ? 462 attr->cur_qp_state : qp->state; 463 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 464 465 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 466 attr_mask)) 467 goto inval; 468 469 if (attr_mask & IB_QP_AV) { 470 if (attr->ah_attr.dlid == 0 || 471 attr->ah_attr.dlid >= IPATH_MULTICAST_LID_BASE) 472 goto inval; 473 474 if ((attr->ah_attr.ah_flags & IB_AH_GRH) && 475 (attr->ah_attr.grh.sgid_index > 1)) 476 goto inval; 477 } 478 479 if (attr_mask & IB_QP_PKEY_INDEX) 480 if (attr->pkey_index >= ipath_get_npkeys(dev->dd)) 481 goto inval; 482 483 if (attr_mask & IB_QP_MIN_RNR_TIMER) 484 if (attr->min_rnr_timer > 31) 485 goto inval; 486 487 if (attr_mask & IB_QP_PORT) 488 if (attr->port_num == 0 || 489 attr->port_num > ibqp->device->phys_port_cnt) 490 goto inval; 491 492 /* 493 * don't allow invalid Path MTU values or greater than 2048 494 * unless we are configured for a 4KB MTU 495 */ 496 if ((attr_mask & IB_QP_PATH_MTU) && 497 (ib_mtu_enum_to_int(attr->path_mtu) == -1 || 498 (attr->path_mtu > IB_MTU_2048 && !ipath_mtu4096))) 499 goto inval; 500 501 if (attr_mask & IB_QP_PATH_MIG_STATE) 502 if (attr->path_mig_state != IB_MIG_MIGRATED && 503 attr->path_mig_state != IB_MIG_REARM) 504 goto inval; 505 506 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 507 if (attr->max_dest_rd_atomic > IPATH_MAX_RDMA_ATOMIC) 508 goto inval; 509 510 switch (new_state) { 511 case IB_QPS_RESET: 512 if (qp->state != IB_QPS_RESET) { 513 qp->state = IB_QPS_RESET; 514 spin_lock(&dev->pending_lock); 515 if (!list_empty(&qp->timerwait)) 516 list_del_init(&qp->timerwait); 517 if (!list_empty(&qp->piowait)) 518 list_del_init(&qp->piowait); 519 spin_unlock(&dev->pending_lock); 520 qp->s_flags &= ~IPATH_S_ANY_WAIT; 521 spin_unlock_irq(&qp->s_lock); 522 /* Stop the sending tasklet */ 523 tasklet_kill(&qp->s_task); 524 wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy)); 525 spin_lock_irq(&qp->s_lock); 526 } 527 ipath_reset_qp(qp, ibqp->qp_type); 528 break; 529 530 case IB_QPS_SQD: 531 qp->s_draining = qp->s_last != qp->s_cur; 532 qp->state = new_state; 533 break; 534 535 case IB_QPS_SQE: 536 if (qp->ibqp.qp_type == IB_QPT_RC) 537 goto inval; 538 qp->state = new_state; 539 break; 540 541 case IB_QPS_ERR: 542 lastwqe = ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR); 543 break; 544 545 default: 546 qp->state = new_state; 547 break; 548 } 549 550 if (attr_mask & IB_QP_PKEY_INDEX) 551 qp->s_pkey_index = attr->pkey_index; 552 553 if (attr_mask & IB_QP_DEST_QPN) 554 qp->remote_qpn = attr->dest_qp_num; 555 556 if (attr_mask & IB_QP_SQ_PSN) { 557 qp->s_psn = qp->s_next_psn = attr->sq_psn; 558 qp->s_last_psn = qp->s_next_psn - 1; 559 } 560 561 if (attr_mask & IB_QP_RQ_PSN) 562 qp->r_psn = attr->rq_psn; 563 564 if (attr_mask & IB_QP_ACCESS_FLAGS) 565 qp->qp_access_flags = attr->qp_access_flags; 566 567 if (attr_mask & IB_QP_AV) { 568 qp->remote_ah_attr = attr->ah_attr; 569 qp->s_dmult = ipath_ib_rate_to_mult(attr->ah_attr.static_rate); 570 } 571 572 if (attr_mask & IB_QP_PATH_MTU) 573 qp->path_mtu = attr->path_mtu; 574 575 if (attr_mask & IB_QP_RETRY_CNT) 576 qp->s_retry = qp->s_retry_cnt = attr->retry_cnt; 577 578 if (attr_mask & IB_QP_RNR_RETRY) { 579 qp->s_rnr_retry = attr->rnr_retry; 580 if (qp->s_rnr_retry > 7) 581 qp->s_rnr_retry = 7; 582 qp->s_rnr_retry_cnt = qp->s_rnr_retry; 583 } 584 585 if (attr_mask & IB_QP_MIN_RNR_TIMER) 586 qp->r_min_rnr_timer = attr->min_rnr_timer; 587 588 if (attr_mask & IB_QP_TIMEOUT) 589 qp->timeout = attr->timeout; 590 591 if (attr_mask & IB_QP_QKEY) 592 qp->qkey = attr->qkey; 593 594 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 595 qp->r_max_rd_atomic = attr->max_dest_rd_atomic; 596 597 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) 598 qp->s_max_rd_atomic = attr->max_rd_atomic; 599 600 spin_unlock_irq(&qp->s_lock); 601 602 if (lastwqe) { 603 struct ib_event ev; 604 605 ev.device = qp->ibqp.device; 606 ev.element.qp = &qp->ibqp; 607 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 608 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 609 } 610 ret = 0; 611 goto bail; 612 613inval: 614 spin_unlock_irq(&qp->s_lock); 615 ret = -EINVAL; 616 617bail: 618 return ret; 619} 620 621int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 622 int attr_mask, struct ib_qp_init_attr *init_attr) 623{ 624 struct ipath_qp *qp = to_iqp(ibqp); 625 626 attr->qp_state = qp->state; 627 attr->cur_qp_state = attr->qp_state; 628 attr->path_mtu = qp->path_mtu; 629 attr->path_mig_state = 0; 630 attr->qkey = qp->qkey; 631 attr->rq_psn = qp->r_psn; 632 attr->sq_psn = qp->s_next_psn; 633 attr->dest_qp_num = qp->remote_qpn; 634 attr->qp_access_flags = qp->qp_access_flags; 635 attr->cap.max_send_wr = qp->s_size - 1; 636 attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1; 637 attr->cap.max_send_sge = qp->s_max_sge; 638 attr->cap.max_recv_sge = qp->r_rq.max_sge; 639 attr->cap.max_inline_data = 0; 640 attr->ah_attr = qp->remote_ah_attr; 641 memset(&attr->alt_ah_attr, 0, sizeof(attr->alt_ah_attr)); 642 attr->pkey_index = qp->s_pkey_index; 643 attr->alt_pkey_index = 0; 644 attr->en_sqd_async_notify = 0; 645 attr->sq_draining = qp->s_draining; 646 attr->max_rd_atomic = qp->s_max_rd_atomic; 647 attr->max_dest_rd_atomic = qp->r_max_rd_atomic; 648 attr->min_rnr_timer = qp->r_min_rnr_timer; 649 attr->port_num = 1; 650 attr->timeout = qp->timeout; 651 attr->retry_cnt = qp->s_retry_cnt; 652 attr->rnr_retry = qp->s_rnr_retry_cnt; 653 attr->alt_port_num = 0; 654 attr->alt_timeout = 0; 655 656 init_attr->event_handler = qp->ibqp.event_handler; 657 init_attr->qp_context = qp->ibqp.qp_context; 658 init_attr->send_cq = qp->ibqp.send_cq; 659 init_attr->recv_cq = qp->ibqp.recv_cq; 660 init_attr->srq = qp->ibqp.srq; 661 init_attr->cap = attr->cap; 662 if (qp->s_flags & IPATH_S_SIGNAL_REQ_WR) 663 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 664 else 665 init_attr->sq_sig_type = IB_SIGNAL_ALL_WR; 666 init_attr->qp_type = qp->ibqp.qp_type; 667 init_attr->port_num = 1; 668 return 0; 669} 670 671/** 672 * ipath_compute_aeth - compute the AETH (syndrome + MSN) 673 * @qp: the queue pair to compute the AETH for 674 * 675 * Returns the AETH. 676 */ 677__be32 ipath_compute_aeth(struct ipath_qp *qp) 678{ 679 u32 aeth = qp->r_msn & IPATH_MSN_MASK; 680 681 if (qp->ibqp.srq) { 682 /* 683 * Shared receive queues don't generate credits. 684 * Set the credit field to the invalid value. 685 */ 686 aeth |= IPATH_AETH_CREDIT_INVAL << IPATH_AETH_CREDIT_SHIFT; 687 } else { 688 u32 min, max, x; 689 u32 credits; 690 struct ipath_rwq *wq = qp->r_rq.wq; 691 u32 head; 692 u32 tail; 693 694 /* sanity check pointers before trusting them */ 695 head = wq->head; 696 if (head >= qp->r_rq.size) 697 head = 0; 698 tail = wq->tail; 699 if (tail >= qp->r_rq.size) 700 tail = 0; 701 credits = head - tail; 702 if ((int)credits < 0) 703 credits += qp->r_rq.size; 704 /* 705 * Binary search the credit table to find the code to 706 * use. 707 */ 708 min = 0; 709 max = 31; 710 for (;;) { 711 x = (min + max) / 2; 712 if (credit_table[x] == credits) 713 break; 714 if (credit_table[x] > credits) 715 max = x; 716 else if (min == x) 717 break; 718 else 719 min = x; 720 } 721 aeth |= x << IPATH_AETH_CREDIT_SHIFT; 722 } 723 return cpu_to_be32(aeth); 724} 725 726/** 727 * ipath_create_qp - create a queue pair for a device 728 * @ibpd: the protection domain who's device we create the queue pair for 729 * @init_attr: the attributes of the queue pair 730 * @udata: unused by InfiniPath 731 * 732 * Returns the queue pair on success, otherwise returns an errno. 733 * 734 * Called by the ib_create_qp() core verbs function. 735 */ 736struct ib_qp *ipath_create_qp(struct ib_pd *ibpd, 737 struct ib_qp_init_attr *init_attr, 738 struct ib_udata *udata) 739{ 740 struct ipath_qp *qp; 741 int err; 742 struct ipath_swqe *swq = NULL; 743 struct ipath_ibdev *dev; 744 size_t sz; 745 size_t sg_list_sz; 746 struct ib_qp *ret; 747 748 if (init_attr->create_flags) { 749 ret = ERR_PTR(-EINVAL); 750 goto bail; 751 } 752 753 if (init_attr->cap.max_send_sge > ib_ipath_max_sges || 754 init_attr->cap.max_send_wr > ib_ipath_max_qp_wrs) { 755 ret = ERR_PTR(-EINVAL); 756 goto bail; 757 } 758 759 /* Check receive queue parameters if no SRQ is specified. */ 760 if (!init_attr->srq) { 761 if (init_attr->cap.max_recv_sge > ib_ipath_max_sges || 762 init_attr->cap.max_recv_wr > ib_ipath_max_qp_wrs) { 763 ret = ERR_PTR(-EINVAL); 764 goto bail; 765 } 766 if (init_attr->cap.max_send_sge + 767 init_attr->cap.max_send_wr + 768 init_attr->cap.max_recv_sge + 769 init_attr->cap.max_recv_wr == 0) { 770 ret = ERR_PTR(-EINVAL); 771 goto bail; 772 } 773 } 774 775 switch (init_attr->qp_type) { 776 case IB_QPT_UC: 777 case IB_QPT_RC: 778 case IB_QPT_UD: 779 case IB_QPT_SMI: 780 case IB_QPT_GSI: 781 sz = sizeof(struct ipath_sge) * 782 init_attr->cap.max_send_sge + 783 sizeof(struct ipath_swqe); 784 swq = vmalloc((init_attr->cap.max_send_wr + 1) * sz); 785 if (swq == NULL) { 786 ret = ERR_PTR(-ENOMEM); 787 goto bail; 788 } 789 sz = sizeof(*qp); 790 sg_list_sz = 0; 791 if (init_attr->srq) { 792 struct ipath_srq *srq = to_isrq(init_attr->srq); 793 794 if (srq->rq.max_sge > 1) 795 sg_list_sz = sizeof(*qp->r_sg_list) * 796 (srq->rq.max_sge - 1); 797 } else if (init_attr->cap.max_recv_sge > 1) 798 sg_list_sz = sizeof(*qp->r_sg_list) * 799 (init_attr->cap.max_recv_sge - 1); 800 qp = kmalloc(sz + sg_list_sz, GFP_KERNEL); 801 if (!qp) { 802 ret = ERR_PTR(-ENOMEM); 803 goto bail_swq; 804 } 805 if (sg_list_sz && (init_attr->qp_type == IB_QPT_UD || 806 init_attr->qp_type == IB_QPT_SMI || 807 init_attr->qp_type == IB_QPT_GSI)) { 808 qp->r_ud_sg_list = kmalloc(sg_list_sz, GFP_KERNEL); 809 if (!qp->r_ud_sg_list) { 810 ret = ERR_PTR(-ENOMEM); 811 goto bail_qp; 812 } 813 } else 814 qp->r_ud_sg_list = NULL; 815 if (init_attr->srq) { 816 sz = 0; 817 qp->r_rq.size = 0; 818 qp->r_rq.max_sge = 0; 819 qp->r_rq.wq = NULL; 820 init_attr->cap.max_recv_wr = 0; 821 init_attr->cap.max_recv_sge = 0; 822 } else { 823 qp->r_rq.size = init_attr->cap.max_recv_wr + 1; 824 qp->r_rq.max_sge = init_attr->cap.max_recv_sge; 825 sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) + 826 sizeof(struct ipath_rwqe); 827 qp->r_rq.wq = vmalloc_user(sizeof(struct ipath_rwq) + 828 qp->r_rq.size * sz); 829 if (!qp->r_rq.wq) { 830 ret = ERR_PTR(-ENOMEM); 831 goto bail_sg_list; 832 } 833 } 834 835 /* 836 * ib_create_qp() will initialize qp->ibqp 837 * except for qp->ibqp.qp_num. 838 */ 839 spin_lock_init(&qp->s_lock); 840 spin_lock_init(&qp->r_rq.lock); 841 atomic_set(&qp->refcount, 0); 842 init_waitqueue_head(&qp->wait); 843 init_waitqueue_head(&qp->wait_dma); 844 tasklet_init(&qp->s_task, ipath_do_send, (unsigned long)qp); 845 INIT_LIST_HEAD(&qp->piowait); 846 INIT_LIST_HEAD(&qp->timerwait); 847 qp->state = IB_QPS_RESET; 848 qp->s_wq = swq; 849 qp->s_size = init_attr->cap.max_send_wr + 1; 850 qp->s_max_sge = init_attr->cap.max_send_sge; 851 if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR) 852 qp->s_flags = IPATH_S_SIGNAL_REQ_WR; 853 else 854 qp->s_flags = 0; 855 dev = to_idev(ibpd->device); 856 err = ipath_alloc_qpn(&dev->qp_table, qp, 857 init_attr->qp_type); 858 if (err) { 859 ret = ERR_PTR(err); 860 vfree(qp->r_rq.wq); 861 goto bail_sg_list; 862 } 863 qp->ip = NULL; 864 qp->s_tx = NULL; 865 ipath_reset_qp(qp, init_attr->qp_type); 866 break; 867 868 default: 869 /* Don't support raw QPs */ 870 ret = ERR_PTR(-ENOSYS); 871 goto bail; 872 } 873 874 init_attr->cap.max_inline_data = 0; 875 876 /* 877 * Return the address of the RWQ as the offset to mmap. 878 * See ipath_mmap() for details. 879 */ 880 if (udata && udata->outlen >= sizeof(__u64)) { 881 if (!qp->r_rq.wq) { 882 __u64 offset = 0; 883 884 err = ib_copy_to_udata(udata, &offset, 885 sizeof(offset)); 886 if (err) { 887 ret = ERR_PTR(err); 888 goto bail_ip; 889 } 890 } else { 891 u32 s = sizeof(struct ipath_rwq) + 892 qp->r_rq.size * sz; 893 894 qp->ip = 895 ipath_create_mmap_info(dev, s, 896 ibpd->uobject->context, 897 qp->r_rq.wq); 898 if (!qp->ip) { 899 ret = ERR_PTR(-ENOMEM); 900 goto bail_ip; 901 } 902 903 err = ib_copy_to_udata(udata, &(qp->ip->offset), 904 sizeof(qp->ip->offset)); 905 if (err) { 906 ret = ERR_PTR(err); 907 goto bail_ip; 908 } 909 } 910 } 911 912 spin_lock(&dev->n_qps_lock); 913 if (dev->n_qps_allocated == ib_ipath_max_qps) { 914 spin_unlock(&dev->n_qps_lock); 915 ret = ERR_PTR(-ENOMEM); 916 goto bail_ip; 917 } 918 919 dev->n_qps_allocated++; 920 spin_unlock(&dev->n_qps_lock); 921 922 if (qp->ip) { 923 spin_lock_irq(&dev->pending_lock); 924 list_add(&qp->ip->pending_mmaps, &dev->pending_mmaps); 925 spin_unlock_irq(&dev->pending_lock); 926 } 927 928 ret = &qp->ibqp; 929 goto bail; 930 931bail_ip: 932 if (qp->ip) 933 kref_put(&qp->ip->ref, ipath_release_mmap_info); 934 else 935 vfree(qp->r_rq.wq); 936 ipath_free_qp(&dev->qp_table, qp); 937 free_qpn(&dev->qp_table, qp->ibqp.qp_num); 938bail_sg_list: 939 kfree(qp->r_ud_sg_list); 940bail_qp: 941 kfree(qp); 942bail_swq: 943 vfree(swq); 944bail: 945 return ret; 946} 947 948/** 949 * ipath_destroy_qp - destroy a queue pair 950 * @ibqp: the queue pair to destroy 951 * 952 * Returns 0 on success. 953 * 954 * Note that this can be called while the QP is actively sending or 955 * receiving! 956 */ 957int ipath_destroy_qp(struct ib_qp *ibqp) 958{ 959 struct ipath_qp *qp = to_iqp(ibqp); 960 struct ipath_ibdev *dev = to_idev(ibqp->device); 961 962 /* Make sure HW and driver activity is stopped. */ 963 spin_lock_irq(&qp->s_lock); 964 if (qp->state != IB_QPS_RESET) { 965 qp->state = IB_QPS_RESET; 966 spin_lock(&dev->pending_lock); 967 if (!list_empty(&qp->timerwait)) 968 list_del_init(&qp->timerwait); 969 if (!list_empty(&qp->piowait)) 970 list_del_init(&qp->piowait); 971 spin_unlock(&dev->pending_lock); 972 qp->s_flags &= ~IPATH_S_ANY_WAIT; 973 spin_unlock_irq(&qp->s_lock); 974 /* Stop the sending tasklet */ 975 tasklet_kill(&qp->s_task); 976 wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy)); 977 } else 978 spin_unlock_irq(&qp->s_lock); 979 980 ipath_free_qp(&dev->qp_table, qp); 981 982 if (qp->s_tx) { 983 atomic_dec(&qp->refcount); 984 if (qp->s_tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEBUF) 985 kfree(qp->s_tx->txreq.map_addr); 986 spin_lock_irq(&dev->pending_lock); 987 list_add(&qp->s_tx->txreq.list, &dev->txreq_free); 988 spin_unlock_irq(&dev->pending_lock); 989 qp->s_tx = NULL; 990 } 991 992 wait_event(qp->wait, !atomic_read(&qp->refcount)); 993 994 /* all user's cleaned up, mark it available */ 995 free_qpn(&dev->qp_table, qp->ibqp.qp_num); 996 spin_lock(&dev->n_qps_lock); 997 dev->n_qps_allocated--; 998 spin_unlock(&dev->n_qps_lock); 999 1000 if (qp->ip) 1001 kref_put(&qp->ip->ref, ipath_release_mmap_info); 1002 else 1003 vfree(qp->r_rq.wq); 1004 kfree(qp->r_ud_sg_list); 1005 vfree(qp->s_wq); 1006 kfree(qp); 1007 return 0; 1008} 1009 1010/** 1011 * ipath_init_qp_table - initialize the QP table for a device 1012 * @idev: the device who's QP table we're initializing 1013 * @size: the size of the QP table 1014 * 1015 * Returns 0 on success, otherwise returns an errno. 1016 */ 1017int ipath_init_qp_table(struct ipath_ibdev *idev, int size) 1018{ 1019 int i; 1020 int ret; 1021 1022 idev->qp_table.last = 1; /* QPN 0 and 1 are special. */ 1023 idev->qp_table.max = size; 1024 idev->qp_table.nmaps = 1; 1025 idev->qp_table.table = kzalloc(size * sizeof(*idev->qp_table.table), 1026 GFP_KERNEL); 1027 if (idev->qp_table.table == NULL) { 1028 ret = -ENOMEM; 1029 goto bail; 1030 } 1031 1032 for (i = 0; i < ARRAY_SIZE(idev->qp_table.map); i++) { 1033 atomic_set(&idev->qp_table.map[i].n_free, BITS_PER_PAGE); 1034 idev->qp_table.map[i].page = NULL; 1035 } 1036 1037 ret = 0; 1038 1039bail: 1040 return ret; 1041} 1042 1043/** 1044 * ipath_get_credit - flush the send work queue of a QP 1045 * @qp: the qp who's send work queue to flush 1046 * @aeth: the Acknowledge Extended Transport Header 1047 * 1048 * The QP s_lock should be held. 1049 */ 1050void ipath_get_credit(struct ipath_qp *qp, u32 aeth) 1051{ 1052 u32 credit = (aeth >> IPATH_AETH_CREDIT_SHIFT) & IPATH_AETH_CREDIT_MASK; 1053 1054 /* 1055 * If the credit is invalid, we can send 1056 * as many packets as we like. Otherwise, we have to 1057 * honor the credit field. 1058 */ 1059 if (credit == IPATH_AETH_CREDIT_INVAL) 1060 qp->s_lsn = (u32) -1; 1061 else if (qp->s_lsn != (u32) -1) { 1062 /* Compute new LSN (i.e., MSN + credit) */ 1063 credit = (aeth + credit_table[credit]) & IPATH_MSN_MASK; 1064 if (ipath_cmp24(credit, qp->s_lsn) > 0) 1065 qp->s_lsn = credit; 1066 } 1067 1068 /* Restart sending if it was blocked due to lack of credits. */ 1069 if ((qp->s_flags & IPATH_S_WAIT_SSN_CREDIT) && 1070 qp->s_cur != qp->s_head && 1071 (qp->s_lsn == (u32) -1 || 1072 ipath_cmp24(get_swqe_ptr(qp, qp->s_cur)->ssn, 1073 qp->s_lsn + 1) <= 0)) 1074 ipath_schedule_send(qp); 1075} 1076