1/* 2 * Copyright (c) 2006 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32#include <linux/module.h> 33#include <linux/list.h> 34#include <linux/slab.h> 35#include <linux/workqueue.h> 36#include <linux/skbuff.h> 37#include <linux/timer.h> 38#include <linux/notifier.h> 39#include <linux/inetdevice.h> 40 41#include <net/neighbour.h> 42#include <net/netevent.h> 43#include <net/route.h> 44 45#include "tcb.h" 46#include "cxgb3_offload.h" 47#include "iwch.h" 48#include "iwch_provider.h" 49#include "iwch_cm.h" 50 51static char *states[] = { 52 "idle", 53 "listen", 54 "connecting", 55 "mpa_wait_req", 56 "mpa_req_sent", 57 "mpa_req_rcvd", 58 "mpa_rep_sent", 59 "fpdu_mode", 60 "aborting", 61 "closing", 62 "moribund", 63 "dead", 64 NULL, 65}; 66 67int peer2peer = 0; 68module_param(peer2peer, int, 0644); 69MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)"); 70 71static int ep_timeout_secs = 60; 72module_param(ep_timeout_secs, int, 0644); 73MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout " 74 "in seconds (default=60)"); 75 76static int mpa_rev = 1; 77module_param(mpa_rev, int, 0644); 78MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, " 79 "1 is spec compliant. (default=1)"); 80 81static int markers_enabled = 0; 82module_param(markers_enabled, int, 0644); 83MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)"); 84 85static int crc_enabled = 1; 86module_param(crc_enabled, int, 0644); 87MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)"); 88 89static int rcv_win = 256 * 1024; 90module_param(rcv_win, int, 0644); 91MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256)"); 92 93static int snd_win = 32 * 1024; 94module_param(snd_win, int, 0644); 95MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=32KB)"); 96 97static unsigned int nocong = 0; 98module_param(nocong, uint, 0644); 99MODULE_PARM_DESC(nocong, "Turn off congestion control (default=0)"); 100 101static unsigned int cong_flavor = 1; 102module_param(cong_flavor, uint, 0644); 103MODULE_PARM_DESC(cong_flavor, "TCP Congestion control flavor (default=1)"); 104 105static struct workqueue_struct *workq; 106 107static struct sk_buff_head rxq; 108 109static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp); 110static void ep_timeout(unsigned long arg); 111static void connect_reply_upcall(struct iwch_ep *ep, int status); 112 113static void start_ep_timer(struct iwch_ep *ep) 114{ 115 PDBG("%s ep %p\n", __func__, ep); 116 if (timer_pending(&ep->timer)) { 117 PDBG("%s stopped / restarted timer ep %p\n", __func__, ep); 118 del_timer_sync(&ep->timer); 119 } else 120 get_ep(&ep->com); 121 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 122 ep->timer.data = (unsigned long)ep; 123 ep->timer.function = ep_timeout; 124 add_timer(&ep->timer); 125} 126 127static void stop_ep_timer(struct iwch_ep *ep) 128{ 129 PDBG("%s ep %p\n", __func__, ep); 130 if (!timer_pending(&ep->timer)) { 131 printk(KERN_ERR "%s timer stopped when its not running! ep %p state %u\n", 132 __func__, ep, ep->com.state); 133 WARN_ON(1); 134 return; 135 } 136 del_timer_sync(&ep->timer); 137 put_ep(&ep->com); 138} 139 140static int iwch_l2t_send(struct t3cdev *tdev, struct sk_buff *skb, struct l2t_entry *l2e) 141{ 142 int error = 0; 143 struct cxio_rdev *rdev; 144 145 rdev = (struct cxio_rdev *)tdev->ulp; 146 if (cxio_fatal_error(rdev)) { 147 kfree_skb(skb); 148 return -EIO; 149 } 150 error = l2t_send(tdev, skb, l2e); 151 if (error < 0) 152 kfree_skb(skb); 153 return error; 154} 155 156int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb) 157{ 158 int error = 0; 159 struct cxio_rdev *rdev; 160 161 rdev = (struct cxio_rdev *)tdev->ulp; 162 if (cxio_fatal_error(rdev)) { 163 kfree_skb(skb); 164 return -EIO; 165 } 166 error = cxgb3_ofld_send(tdev, skb); 167 if (error < 0) 168 kfree_skb(skb); 169 return error; 170} 171 172static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb) 173{ 174 struct cpl_tid_release *req; 175 176 skb = get_skb(skb, sizeof *req, GFP_KERNEL); 177 if (!skb) 178 return; 179 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req)); 180 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 181 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid)); 182 skb->priority = CPL_PRIORITY_SETUP; 183 iwch_cxgb3_ofld_send(tdev, skb); 184 return; 185} 186 187int iwch_quiesce_tid(struct iwch_ep *ep) 188{ 189 struct cpl_set_tcb_field *req; 190 struct sk_buff *skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 191 192 if (!skb) 193 return -ENOMEM; 194 req = (struct cpl_set_tcb_field *) skb_put(skb, sizeof(*req)); 195 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 196 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid)); 197 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep->hwtid)); 198 req->reply = 0; 199 req->cpu_idx = 0; 200 req->word = htons(W_TCB_RX_QUIESCE); 201 req->mask = cpu_to_be64(1ULL << S_TCB_RX_QUIESCE); 202 req->val = cpu_to_be64(1 << S_TCB_RX_QUIESCE); 203 204 skb->priority = CPL_PRIORITY_DATA; 205 return iwch_cxgb3_ofld_send(ep->com.tdev, skb); 206} 207 208int iwch_resume_tid(struct iwch_ep *ep) 209{ 210 struct cpl_set_tcb_field *req; 211 struct sk_buff *skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 212 213 if (!skb) 214 return -ENOMEM; 215 req = (struct cpl_set_tcb_field *) skb_put(skb, sizeof(*req)); 216 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 217 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid)); 218 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep->hwtid)); 219 req->reply = 0; 220 req->cpu_idx = 0; 221 req->word = htons(W_TCB_RX_QUIESCE); 222 req->mask = cpu_to_be64(1ULL << S_TCB_RX_QUIESCE); 223 req->val = 0; 224 225 skb->priority = CPL_PRIORITY_DATA; 226 return iwch_cxgb3_ofld_send(ep->com.tdev, skb); 227} 228 229static void set_emss(struct iwch_ep *ep, u16 opt) 230{ 231 PDBG("%s ep %p opt %u\n", __func__, ep, opt); 232 ep->emss = T3C_DATA(ep->com.tdev)->mtus[G_TCPOPT_MSS(opt)] - 40; 233 if (G_TCPOPT_TSTAMP(opt)) 234 ep->emss -= 12; 235 if (ep->emss < 128) 236 ep->emss = 128; 237 PDBG("emss=%d\n", ep->emss); 238} 239 240static enum iwch_ep_state state_read(struct iwch_ep_common *epc) 241{ 242 unsigned long flags; 243 enum iwch_ep_state state; 244 245 spin_lock_irqsave(&epc->lock, flags); 246 state = epc->state; 247 spin_unlock_irqrestore(&epc->lock, flags); 248 return state; 249} 250 251static void __state_set(struct iwch_ep_common *epc, enum iwch_ep_state new) 252{ 253 epc->state = new; 254} 255 256static void state_set(struct iwch_ep_common *epc, enum iwch_ep_state new) 257{ 258 unsigned long flags; 259 260 spin_lock_irqsave(&epc->lock, flags); 261 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]); 262 __state_set(epc, new); 263 spin_unlock_irqrestore(&epc->lock, flags); 264 return; 265} 266 267static void *alloc_ep(int size, gfp_t gfp) 268{ 269 struct iwch_ep_common *epc; 270 271 epc = kzalloc(size, gfp); 272 if (epc) { 273 kref_init(&epc->kref); 274 spin_lock_init(&epc->lock); 275 init_waitqueue_head(&epc->waitq); 276 } 277 PDBG("%s alloc ep %p\n", __func__, epc); 278 return epc; 279} 280 281void __free_ep(struct kref *kref) 282{ 283 struct iwch_ep *ep; 284 ep = container_of(container_of(kref, struct iwch_ep_common, kref), 285 struct iwch_ep, com); 286 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); 287 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { 288 cxgb3_remove_tid(ep->com.tdev, (void *)ep, ep->hwtid); 289 dst_release(ep->dst); 290 l2t_release(L2DATA(ep->com.tdev), ep->l2t); 291 } 292 kfree(ep); 293} 294 295static void release_ep_resources(struct iwch_ep *ep) 296{ 297 PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid); 298 set_bit(RELEASE_RESOURCES, &ep->com.flags); 299 put_ep(&ep->com); 300} 301 302static int status2errno(int status) 303{ 304 switch (status) { 305 case CPL_ERR_NONE: 306 return 0; 307 case CPL_ERR_CONN_RESET: 308 return -ECONNRESET; 309 case CPL_ERR_ARP_MISS: 310 return -EHOSTUNREACH; 311 case CPL_ERR_CONN_TIMEDOUT: 312 return -ETIMEDOUT; 313 case CPL_ERR_TCAM_FULL: 314 return -ENOMEM; 315 case CPL_ERR_CONN_EXIST: 316 return -EADDRINUSE; 317 default: 318 return -EIO; 319 } 320} 321 322/* 323 * Try and reuse skbs already allocated... 324 */ 325static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp) 326{ 327 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) { 328 skb_trim(skb, 0); 329 skb_get(skb); 330 } else { 331 skb = alloc_skb(len, gfp); 332 } 333 return skb; 334} 335 336static struct rtable *find_route(struct t3cdev *dev, __be32 local_ip, 337 __be32 peer_ip, __be16 local_port, 338 __be16 peer_port, u8 tos) 339{ 340 struct rtable *rt; 341 struct flowi fl = { 342 .oif = 0, 343 .nl_u = { 344 .ip4_u = { 345 .daddr = peer_ip, 346 .saddr = local_ip, 347 .tos = tos} 348 }, 349 .proto = IPPROTO_TCP, 350 .uli_u = { 351 .ports = { 352 .sport = local_port, 353 .dport = peer_port} 354 } 355 }; 356 357 if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0)) 358 return NULL; 359 return rt; 360} 361 362static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu) 363{ 364 int i = 0; 365 366 while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu) 367 ++i; 368 return i; 369} 370 371static void arp_failure_discard(struct t3cdev *dev, struct sk_buff *skb) 372{ 373 PDBG("%s t3cdev %p\n", __func__, dev); 374 kfree_skb(skb); 375} 376 377/* 378 * Handle an ARP failure for an active open. 379 */ 380static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb) 381{ 382 printk(KERN_ERR MOD "ARP failure duing connect\n"); 383 kfree_skb(skb); 384} 385 386/* 387 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 388 * and send it along. 389 */ 390static void abort_arp_failure(struct t3cdev *dev, struct sk_buff *skb) 391{ 392 struct cpl_abort_req *req = cplhdr(skb); 393 394 PDBG("%s t3cdev %p\n", __func__, dev); 395 req->cmd = CPL_ABORT_NO_RST; 396 iwch_cxgb3_ofld_send(dev, skb); 397} 398 399static int send_halfclose(struct iwch_ep *ep, gfp_t gfp) 400{ 401 struct cpl_close_con_req *req; 402 struct sk_buff *skb; 403 404 PDBG("%s ep %p\n", __func__, ep); 405 skb = get_skb(NULL, sizeof(*req), gfp); 406 if (!skb) { 407 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 408 return -ENOMEM; 409 } 410 skb->priority = CPL_PRIORITY_DATA; 411 set_arp_failure_handler(skb, arp_failure_discard); 412 req = (struct cpl_close_con_req *) skb_put(skb, sizeof(*req)); 413 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); 414 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid)); 415 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, ep->hwtid)); 416 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t); 417} 418 419static int send_abort(struct iwch_ep *ep, struct sk_buff *skb, gfp_t gfp) 420{ 421 struct cpl_abort_req *req; 422 423 PDBG("%s ep %p\n", __func__, ep); 424 skb = get_skb(skb, sizeof(*req), gfp); 425 if (!skb) { 426 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 427 __func__); 428 return -ENOMEM; 429 } 430 skb->priority = CPL_PRIORITY_DATA; 431 set_arp_failure_handler(skb, abort_arp_failure); 432 req = (struct cpl_abort_req *) skb_put(skb, sizeof(*req)); 433 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); 434 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid)); 435 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid)); 436 req->cmd = CPL_ABORT_SEND_RST; 437 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t); 438} 439 440static int send_connect(struct iwch_ep *ep) 441{ 442 struct cpl_act_open_req *req; 443 struct sk_buff *skb; 444 u32 opt0h, opt0l, opt2; 445 unsigned int mtu_idx; 446 int wscale; 447 448 PDBG("%s ep %p\n", __func__, ep); 449 450 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 451 if (!skb) { 452 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 453 __func__); 454 return -ENOMEM; 455 } 456 mtu_idx = find_best_mtu(T3C_DATA(ep->com.tdev), dst_mtu(ep->dst)); 457 wscale = compute_wscale(rcv_win); 458 opt0h = V_NAGLE(0) | 459 V_NO_CONG(nocong) | 460 V_KEEP_ALIVE(1) | 461 F_TCAM_BYPASS | 462 V_WND_SCALE(wscale) | 463 V_MSS_IDX(mtu_idx) | 464 V_L2T_IDX(ep->l2t->idx) | V_TX_CHANNEL(ep->l2t->smt_idx); 465 opt0l = V_TOS((ep->tos >> 2) & M_TOS) | V_RCV_BUFSIZ(rcv_win>>10); 466 opt2 = F_RX_COALESCE_VALID | V_RX_COALESCE(0) | V_FLAVORS_VALID(1) | 467 V_CONG_CONTROL_FLAVOR(cong_flavor); 468 skb->priority = CPL_PRIORITY_SETUP; 469 set_arp_failure_handler(skb, act_open_req_arp_failure); 470 471 req = (struct cpl_act_open_req *) skb_put(skb, sizeof(*req)); 472 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 473 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ep->atid)); 474 req->local_port = ep->com.local_addr.sin_port; 475 req->peer_port = ep->com.remote_addr.sin_port; 476 req->local_ip = ep->com.local_addr.sin_addr.s_addr; 477 req->peer_ip = ep->com.remote_addr.sin_addr.s_addr; 478 req->opt0h = htonl(opt0h); 479 req->opt0l = htonl(opt0l); 480 req->params = 0; 481 req->opt2 = htonl(opt2); 482 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t); 483} 484 485static void send_mpa_req(struct iwch_ep *ep, struct sk_buff *skb) 486{ 487 int mpalen; 488 struct tx_data_wr *req; 489 struct mpa_message *mpa; 490 int len; 491 492 PDBG("%s ep %p pd_len %d\n", __func__, ep, ep->plen); 493 494 BUG_ON(skb_cloned(skb)); 495 496 mpalen = sizeof(*mpa) + ep->plen; 497 if (skb->data + mpalen + sizeof(*req) > skb_end_pointer(skb)) { 498 kfree_skb(skb); 499 skb=alloc_skb(mpalen + sizeof(*req), GFP_KERNEL); 500 if (!skb) { 501 connect_reply_upcall(ep, -ENOMEM); 502 return; 503 } 504 } 505 skb_trim(skb, 0); 506 skb_reserve(skb, sizeof(*req)); 507 skb_put(skb, mpalen); 508 skb->priority = CPL_PRIORITY_DATA; 509 mpa = (struct mpa_message *) skb->data; 510 memset(mpa, 0, sizeof(*mpa)); 511 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 512 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 513 (markers_enabled ? MPA_MARKERS : 0); 514 mpa->private_data_size = htons(ep->plen); 515 mpa->revision = mpa_rev; 516 517 if (ep->plen) 518 memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen); 519 520 /* 521 * Reference the mpa skb. This ensures the data area 522 * will remain in memory until the hw acks the tx. 523 * Function tx_ack() will deref it. 524 */ 525 skb_get(skb); 526 set_arp_failure_handler(skb, arp_failure_discard); 527 skb_reset_transport_header(skb); 528 len = skb->len; 529 req = (struct tx_data_wr *) skb_push(skb, sizeof(*req)); 530 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL); 531 req->wr_lo = htonl(V_WR_TID(ep->hwtid)); 532 req->len = htonl(len); 533 req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) | 534 V_TX_SNDBUF(snd_win>>15)); 535 req->flags = htonl(F_TX_INIT); 536 req->sndseq = htonl(ep->snd_seq); 537 BUG_ON(ep->mpa_skb); 538 ep->mpa_skb = skb; 539 iwch_l2t_send(ep->com.tdev, skb, ep->l2t); 540 start_ep_timer(ep); 541 state_set(&ep->com, MPA_REQ_SENT); 542 return; 543} 544 545static int send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen) 546{ 547 int mpalen; 548 struct tx_data_wr *req; 549 struct mpa_message *mpa; 550 struct sk_buff *skb; 551 552 PDBG("%s ep %p plen %d\n", __func__, ep, plen); 553 554 mpalen = sizeof(*mpa) + plen; 555 556 skb = get_skb(NULL, mpalen + sizeof(*req), GFP_KERNEL); 557 if (!skb) { 558 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 559 return -ENOMEM; 560 } 561 skb_reserve(skb, sizeof(*req)); 562 mpa = (struct mpa_message *) skb_put(skb, mpalen); 563 memset(mpa, 0, sizeof(*mpa)); 564 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 565 mpa->flags = MPA_REJECT; 566 mpa->revision = mpa_rev; 567 mpa->private_data_size = htons(plen); 568 if (plen) 569 memcpy(mpa->private_data, pdata, plen); 570 571 /* 572 * Reference the mpa skb again. This ensures the data area 573 * will remain in memory until the hw acks the tx. 574 * Function tx_ack() will deref it. 575 */ 576 skb_get(skb); 577 skb->priority = CPL_PRIORITY_DATA; 578 set_arp_failure_handler(skb, arp_failure_discard); 579 skb_reset_transport_header(skb); 580 req = (struct tx_data_wr *) skb_push(skb, sizeof(*req)); 581 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL); 582 req->wr_lo = htonl(V_WR_TID(ep->hwtid)); 583 req->len = htonl(mpalen); 584 req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) | 585 V_TX_SNDBUF(snd_win>>15)); 586 req->flags = htonl(F_TX_INIT); 587 req->sndseq = htonl(ep->snd_seq); 588 BUG_ON(ep->mpa_skb); 589 ep->mpa_skb = skb; 590 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t); 591} 592 593static int send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen) 594{ 595 int mpalen; 596 struct tx_data_wr *req; 597 struct mpa_message *mpa; 598 int len; 599 struct sk_buff *skb; 600 601 PDBG("%s ep %p plen %d\n", __func__, ep, plen); 602 603 mpalen = sizeof(*mpa) + plen; 604 605 skb = get_skb(NULL, mpalen + sizeof(*req), GFP_KERNEL); 606 if (!skb) { 607 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 608 return -ENOMEM; 609 } 610 skb->priority = CPL_PRIORITY_DATA; 611 skb_reserve(skb, sizeof(*req)); 612 mpa = (struct mpa_message *) skb_put(skb, mpalen); 613 memset(mpa, 0, sizeof(*mpa)); 614 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 615 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 616 (markers_enabled ? MPA_MARKERS : 0); 617 mpa->revision = mpa_rev; 618 mpa->private_data_size = htons(plen); 619 if (plen) 620 memcpy(mpa->private_data, pdata, plen); 621 622 /* 623 * Reference the mpa skb. This ensures the data area 624 * will remain in memory until the hw acks the tx. 625 * Function tx_ack() will deref it. 626 */ 627 skb_get(skb); 628 set_arp_failure_handler(skb, arp_failure_discard); 629 skb_reset_transport_header(skb); 630 len = skb->len; 631 req = (struct tx_data_wr *) skb_push(skb, sizeof(*req)); 632 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL); 633 req->wr_lo = htonl(V_WR_TID(ep->hwtid)); 634 req->len = htonl(len); 635 req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) | 636 V_TX_SNDBUF(snd_win>>15)); 637 req->flags = htonl(F_TX_INIT); 638 req->sndseq = htonl(ep->snd_seq); 639 ep->mpa_skb = skb; 640 state_set(&ep->com, MPA_REP_SENT); 641 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t); 642} 643 644static int act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 645{ 646 struct iwch_ep *ep = ctx; 647 struct cpl_act_establish *req = cplhdr(skb); 648 unsigned int tid = GET_TID(req); 649 650 PDBG("%s ep %p tid %d\n", __func__, ep, tid); 651 652 dst_confirm(ep->dst); 653 654 /* setup the hwtid for this connection */ 655 ep->hwtid = tid; 656 cxgb3_insert_tid(ep->com.tdev, &t3c_client, ep, tid); 657 658 ep->snd_seq = ntohl(req->snd_isn); 659 ep->rcv_seq = ntohl(req->rcv_isn); 660 661 set_emss(ep, ntohs(req->tcp_opt)); 662 663 /* dealloc the atid */ 664 cxgb3_free_atid(ep->com.tdev, ep->atid); 665 666 /* start MPA negotiation */ 667 send_mpa_req(ep, skb); 668 669 return 0; 670} 671 672static void abort_connection(struct iwch_ep *ep, struct sk_buff *skb, gfp_t gfp) 673{ 674 PDBG("%s ep %p\n", __FILE__, ep); 675 state_set(&ep->com, ABORTING); 676 send_abort(ep, skb, gfp); 677} 678 679static void close_complete_upcall(struct iwch_ep *ep) 680{ 681 struct iw_cm_event event; 682 683 PDBG("%s ep %p\n", __func__, ep); 684 memset(&event, 0, sizeof(event)); 685 event.event = IW_CM_EVENT_CLOSE; 686 if (ep->com.cm_id) { 687 PDBG("close complete delivered ep %p cm_id %p tid %d\n", 688 ep, ep->com.cm_id, ep->hwtid); 689 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 690 ep->com.cm_id->rem_ref(ep->com.cm_id); 691 ep->com.cm_id = NULL; 692 ep->com.qp = NULL; 693 } 694} 695 696static void peer_close_upcall(struct iwch_ep *ep) 697{ 698 struct iw_cm_event event; 699 700 PDBG("%s ep %p\n", __func__, ep); 701 memset(&event, 0, sizeof(event)); 702 event.event = IW_CM_EVENT_DISCONNECT; 703 if (ep->com.cm_id) { 704 PDBG("peer close delivered ep %p cm_id %p tid %d\n", 705 ep, ep->com.cm_id, ep->hwtid); 706 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 707 } 708} 709 710static void peer_abort_upcall(struct iwch_ep *ep) 711{ 712 struct iw_cm_event event; 713 714 PDBG("%s ep %p\n", __func__, ep); 715 memset(&event, 0, sizeof(event)); 716 event.event = IW_CM_EVENT_CLOSE; 717 event.status = -ECONNRESET; 718 if (ep->com.cm_id) { 719 PDBG("abort delivered ep %p cm_id %p tid %d\n", ep, 720 ep->com.cm_id, ep->hwtid); 721 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 722 ep->com.cm_id->rem_ref(ep->com.cm_id); 723 ep->com.cm_id = NULL; 724 ep->com.qp = NULL; 725 } 726} 727 728static void connect_reply_upcall(struct iwch_ep *ep, int status) 729{ 730 struct iw_cm_event event; 731 732 PDBG("%s ep %p status %d\n", __func__, ep, status); 733 memset(&event, 0, sizeof(event)); 734 event.event = IW_CM_EVENT_CONNECT_REPLY; 735 event.status = status; 736 event.local_addr = ep->com.local_addr; 737 event.remote_addr = ep->com.remote_addr; 738 739 if ((status == 0) || (status == -ECONNREFUSED)) { 740 event.private_data_len = ep->plen; 741 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 742 } 743 if (ep->com.cm_id) { 744 PDBG("%s ep %p tid %d status %d\n", __func__, ep, 745 ep->hwtid, status); 746 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 747 } 748 if (status < 0) { 749 ep->com.cm_id->rem_ref(ep->com.cm_id); 750 ep->com.cm_id = NULL; 751 ep->com.qp = NULL; 752 } 753} 754 755static void connect_request_upcall(struct iwch_ep *ep) 756{ 757 struct iw_cm_event event; 758 759 PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid); 760 memset(&event, 0, sizeof(event)); 761 event.event = IW_CM_EVENT_CONNECT_REQUEST; 762 event.local_addr = ep->com.local_addr; 763 event.remote_addr = ep->com.remote_addr; 764 event.private_data_len = ep->plen; 765 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 766 event.provider_data = ep; 767 if (state_read(&ep->parent_ep->com) != DEAD) { 768 get_ep(&ep->com); 769 ep->parent_ep->com.cm_id->event_handler( 770 ep->parent_ep->com.cm_id, 771 &event); 772 } 773 put_ep(&ep->parent_ep->com); 774 ep->parent_ep = NULL; 775} 776 777static void established_upcall(struct iwch_ep *ep) 778{ 779 struct iw_cm_event event; 780 781 PDBG("%s ep %p\n", __func__, ep); 782 memset(&event, 0, sizeof(event)); 783 event.event = IW_CM_EVENT_ESTABLISHED; 784 if (ep->com.cm_id) { 785 PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid); 786 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 787 } 788} 789 790static int update_rx_credits(struct iwch_ep *ep, u32 credits) 791{ 792 struct cpl_rx_data_ack *req; 793 struct sk_buff *skb; 794 795 PDBG("%s ep %p credits %u\n", __func__, ep, credits); 796 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 797 if (!skb) { 798 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n"); 799 return 0; 800 } 801 802 req = (struct cpl_rx_data_ack *) skb_put(skb, sizeof(*req)); 803 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 804 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, ep->hwtid)); 805 req->credit_dack = htonl(V_RX_CREDITS(credits) | V_RX_FORCE_ACK(1)); 806 skb->priority = CPL_PRIORITY_ACK; 807 iwch_cxgb3_ofld_send(ep->com.tdev, skb); 808 return credits; 809} 810 811static void process_mpa_reply(struct iwch_ep *ep, struct sk_buff *skb) 812{ 813 struct mpa_message *mpa; 814 u16 plen; 815 struct iwch_qp_attributes attrs; 816 enum iwch_qp_attr_mask mask; 817 int err; 818 819 PDBG("%s ep %p\n", __func__, ep); 820 821 /* 822 * Stop mpa timer. If it expired, then the state has 823 * changed and we bail since ep_timeout already aborted 824 * the connection. 825 */ 826 stop_ep_timer(ep); 827 if (state_read(&ep->com) != MPA_REQ_SENT) 828 return; 829 830 /* 831 * If we get more than the supported amount of private data 832 * then we must fail this connection. 833 */ 834 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 835 err = -EINVAL; 836 goto err; 837 } 838 839 /* 840 * copy the new data into our accumulation buffer. 841 */ 842 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 843 skb->len); 844 ep->mpa_pkt_len += skb->len; 845 846 /* 847 * if we don't even have the mpa message, then bail. 848 */ 849 if (ep->mpa_pkt_len < sizeof(*mpa)) 850 return; 851 mpa = (struct mpa_message *) ep->mpa_pkt; 852 853 /* Validate MPA header. */ 854 if (mpa->revision != mpa_rev) { 855 err = -EPROTO; 856 goto err; 857 } 858 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 859 err = -EPROTO; 860 goto err; 861 } 862 863 plen = ntohs(mpa->private_data_size); 864 865 /* 866 * Fail if there's too much private data. 867 */ 868 if (plen > MPA_MAX_PRIVATE_DATA) { 869 err = -EPROTO; 870 goto err; 871 } 872 873 /* 874 * If plen does not account for pkt size 875 */ 876 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 877 err = -EPROTO; 878 goto err; 879 } 880 881 ep->plen = (u8) plen; 882 883 /* 884 * If we don't have all the pdata yet, then bail. 885 * We'll continue process when more data arrives. 886 */ 887 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 888 return; 889 890 if (mpa->flags & MPA_REJECT) { 891 err = -ECONNREFUSED; 892 goto err; 893 } 894 895 /* 896 * If we get here we have accumulated the entire mpa 897 * start reply message including private data. And 898 * the MPA header is valid. 899 */ 900 state_set(&ep->com, FPDU_MODE); 901 ep->mpa_attr.initiator = 1; 902 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 903 ep->mpa_attr.recv_marker_enabled = markers_enabled; 904 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 905 ep->mpa_attr.version = mpa_rev; 906 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 907 "xmit_marker_enabled=%d, version=%d\n", __func__, 908 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 909 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 910 911 attrs.mpa_attr = ep->mpa_attr; 912 attrs.max_ird = ep->ird; 913 attrs.max_ord = ep->ord; 914 attrs.llp_stream_handle = ep; 915 attrs.next_state = IWCH_QP_STATE_RTS; 916 917 mask = IWCH_QP_ATTR_NEXT_STATE | 918 IWCH_QP_ATTR_LLP_STREAM_HANDLE | IWCH_QP_ATTR_MPA_ATTR | 919 IWCH_QP_ATTR_MAX_IRD | IWCH_QP_ATTR_MAX_ORD; 920 921 /* bind QP and TID with INIT_WR */ 922 err = iwch_modify_qp(ep->com.qp->rhp, 923 ep->com.qp, mask, &attrs, 1); 924 if (err) 925 goto err; 926 927 if (peer2peer && iwch_rqes_posted(ep->com.qp) == 0) { 928 iwch_post_zb_read(ep->com.qp); 929 } 930 931 goto out; 932err: 933 abort_connection(ep, skb, GFP_KERNEL); 934out: 935 connect_reply_upcall(ep, err); 936 return; 937} 938 939static void process_mpa_request(struct iwch_ep *ep, struct sk_buff *skb) 940{ 941 struct mpa_message *mpa; 942 u16 plen; 943 944 PDBG("%s ep %p\n", __func__, ep); 945 946 /* 947 * Stop mpa timer. If it expired, then the state has 948 * changed and we bail since ep_timeout already aborted 949 * the connection. 950 */ 951 stop_ep_timer(ep); 952 if (state_read(&ep->com) != MPA_REQ_WAIT) 953 return; 954 955 /* 956 * If we get more than the supported amount of private data 957 * then we must fail this connection. 958 */ 959 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 960 abort_connection(ep, skb, GFP_KERNEL); 961 return; 962 } 963 964 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 965 966 /* 967 * Copy the new data into our accumulation buffer. 968 */ 969 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 970 skb->len); 971 ep->mpa_pkt_len += skb->len; 972 973 /* 974 * If we don't even have the mpa message, then bail. 975 * We'll continue process when more data arrives. 976 */ 977 if (ep->mpa_pkt_len < sizeof(*mpa)) 978 return; 979 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 980 mpa = (struct mpa_message *) ep->mpa_pkt; 981 982 /* 983 * Validate MPA Header. 984 */ 985 if (mpa->revision != mpa_rev) { 986 abort_connection(ep, skb, GFP_KERNEL); 987 return; 988 } 989 990 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 991 abort_connection(ep, skb, GFP_KERNEL); 992 return; 993 } 994 995 plen = ntohs(mpa->private_data_size); 996 997 /* 998 * Fail if there's too much private data. 999 */ 1000 if (plen > MPA_MAX_PRIVATE_DATA) { 1001 abort_connection(ep, skb, GFP_KERNEL); 1002 return; 1003 } 1004 1005 /* 1006 * If plen does not account for pkt size 1007 */ 1008 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1009 abort_connection(ep, skb, GFP_KERNEL); 1010 return; 1011 } 1012 ep->plen = (u8) plen; 1013 1014 /* 1015 * If we don't have all the pdata yet, then bail. 1016 */ 1017 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1018 return; 1019 1020 /* 1021 * If we get here we have accumulated the entire mpa 1022 * start reply message including private data. 1023 */ 1024 ep->mpa_attr.initiator = 0; 1025 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1026 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1027 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1028 ep->mpa_attr.version = mpa_rev; 1029 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1030 "xmit_marker_enabled=%d, version=%d\n", __func__, 1031 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1032 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 1033 1034 state_set(&ep->com, MPA_REQ_RCVD); 1035 1036 /* drive upcall */ 1037 connect_request_upcall(ep); 1038 return; 1039} 1040 1041static int rx_data(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1042{ 1043 struct iwch_ep *ep = ctx; 1044 struct cpl_rx_data *hdr = cplhdr(skb); 1045 unsigned int dlen = ntohs(hdr->len); 1046 1047 PDBG("%s ep %p dlen %u\n", __func__, ep, dlen); 1048 1049 skb_pull(skb, sizeof(*hdr)); 1050 skb_trim(skb, dlen); 1051 1052 ep->rcv_seq += dlen; 1053 BUG_ON(ep->rcv_seq != (ntohl(hdr->seq) + dlen)); 1054 1055 switch (state_read(&ep->com)) { 1056 case MPA_REQ_SENT: 1057 process_mpa_reply(ep, skb); 1058 break; 1059 case MPA_REQ_WAIT: 1060 process_mpa_request(ep, skb); 1061 break; 1062 case MPA_REP_SENT: 1063 break; 1064 default: 1065 printk(KERN_ERR MOD "%s Unexpected streaming data." 1066 " ep %p state %d tid %d\n", 1067 __func__, ep, state_read(&ep->com), ep->hwtid); 1068 1069 /* 1070 * The ep will timeout and inform the ULP of the failure. 1071 * See ep_timeout(). 1072 */ 1073 break; 1074 } 1075 1076 /* update RX credits */ 1077 update_rx_credits(ep, dlen); 1078 1079 return CPL_RET_BUF_DONE; 1080} 1081 1082/* 1083 * Upcall from the adapter indicating data has been transmitted. 1084 * For us its just the single MPA request or reply. We can now free 1085 * the skb holding the mpa message. 1086 */ 1087static int tx_ack(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1088{ 1089 struct iwch_ep *ep = ctx; 1090 struct cpl_wr_ack *hdr = cplhdr(skb); 1091 unsigned int credits = ntohs(hdr->credits); 1092 1093 PDBG("%s ep %p credits %u\n", __func__, ep, credits); 1094 1095 if (credits == 0) { 1096 PDBG(KERN_ERR "%s 0 credit ack ep %p state %u\n", 1097 __func__, ep, state_read(&ep->com)); 1098 return CPL_RET_BUF_DONE; 1099 } 1100 1101 BUG_ON(credits != 1); 1102 dst_confirm(ep->dst); 1103 if (!ep->mpa_skb) { 1104 PDBG("%s rdma_init wr_ack ep %p state %u\n", 1105 __func__, ep, state_read(&ep->com)); 1106 if (ep->mpa_attr.initiator) { 1107 PDBG("%s initiator ep %p state %u\n", 1108 __func__, ep, state_read(&ep->com)); 1109 if (peer2peer) 1110 iwch_post_zb_read(ep->com.qp); 1111 } else { 1112 PDBG("%s responder ep %p state %u\n", 1113 __func__, ep, state_read(&ep->com)); 1114 ep->com.rpl_done = 1; 1115 wake_up(&ep->com.waitq); 1116 } 1117 } else { 1118 PDBG("%s lsm ack ep %p state %u freeing skb\n", 1119 __func__, ep, state_read(&ep->com)); 1120 kfree_skb(ep->mpa_skb); 1121 ep->mpa_skb = NULL; 1122 } 1123 return CPL_RET_BUF_DONE; 1124} 1125 1126static int abort_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1127{ 1128 struct iwch_ep *ep = ctx; 1129 unsigned long flags; 1130 int release = 0; 1131 1132 PDBG("%s ep %p\n", __func__, ep); 1133 BUG_ON(!ep); 1134 1135 /* 1136 * We get 2 abort replies from the HW. The first one must 1137 * be ignored except for scribbling that we need one more. 1138 */ 1139 if (!test_and_set_bit(ABORT_REQ_IN_PROGRESS, &ep->com.flags)) { 1140 return CPL_RET_BUF_DONE; 1141 } 1142 1143 spin_lock_irqsave(&ep->com.lock, flags); 1144 switch (ep->com.state) { 1145 case ABORTING: 1146 close_complete_upcall(ep); 1147 __state_set(&ep->com, DEAD); 1148 release = 1; 1149 break; 1150 default: 1151 printk(KERN_ERR "%s ep %p state %d\n", 1152 __func__, ep, ep->com.state); 1153 break; 1154 } 1155 spin_unlock_irqrestore(&ep->com.lock, flags); 1156 1157 if (release) 1158 release_ep_resources(ep); 1159 return CPL_RET_BUF_DONE; 1160} 1161 1162/* 1163 * Return whether a failed active open has allocated a TID 1164 */ 1165static inline int act_open_has_tid(int status) 1166{ 1167 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1168 status != CPL_ERR_ARP_MISS; 1169} 1170 1171static int act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1172{ 1173 struct iwch_ep *ep = ctx; 1174 struct cpl_act_open_rpl *rpl = cplhdr(skb); 1175 1176 PDBG("%s ep %p status %u errno %d\n", __func__, ep, rpl->status, 1177 status2errno(rpl->status)); 1178 connect_reply_upcall(ep, status2errno(rpl->status)); 1179 state_set(&ep->com, DEAD); 1180 if (ep->com.tdev->type != T3A && act_open_has_tid(rpl->status)) 1181 release_tid(ep->com.tdev, GET_TID(rpl), NULL); 1182 cxgb3_free_atid(ep->com.tdev, ep->atid); 1183 dst_release(ep->dst); 1184 l2t_release(L2DATA(ep->com.tdev), ep->l2t); 1185 put_ep(&ep->com); 1186 return CPL_RET_BUF_DONE; 1187} 1188 1189static int listen_start(struct iwch_listen_ep *ep) 1190{ 1191 struct sk_buff *skb; 1192 struct cpl_pass_open_req *req; 1193 1194 PDBG("%s ep %p\n", __func__, ep); 1195 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1196 if (!skb) { 1197 printk(KERN_ERR MOD "t3c_listen_start failed to alloc skb!\n"); 1198 return -ENOMEM; 1199 } 1200 1201 req = (struct cpl_pass_open_req *) skb_put(skb, sizeof(*req)); 1202 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1203 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, ep->stid)); 1204 req->local_port = ep->com.local_addr.sin_port; 1205 req->local_ip = ep->com.local_addr.sin_addr.s_addr; 1206 req->peer_port = 0; 1207 req->peer_ip = 0; 1208 req->peer_netmask = 0; 1209 req->opt0h = htonl(F_DELACK | F_TCAM_BYPASS); 1210 req->opt0l = htonl(V_RCV_BUFSIZ(rcv_win>>10)); 1211 req->opt1 = htonl(V_CONN_POLICY(CPL_CONN_POLICY_ASK)); 1212 1213 skb->priority = 1; 1214 return iwch_cxgb3_ofld_send(ep->com.tdev, skb); 1215} 1216 1217static int pass_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1218{ 1219 struct iwch_listen_ep *ep = ctx; 1220 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1221 1222 PDBG("%s ep %p status %d error %d\n", __func__, ep, 1223 rpl->status, status2errno(rpl->status)); 1224 ep->com.rpl_err = status2errno(rpl->status); 1225 ep->com.rpl_done = 1; 1226 wake_up(&ep->com.waitq); 1227 1228 return CPL_RET_BUF_DONE; 1229} 1230 1231static int listen_stop(struct iwch_listen_ep *ep) 1232{ 1233 struct sk_buff *skb; 1234 struct cpl_close_listserv_req *req; 1235 1236 PDBG("%s ep %p\n", __func__, ep); 1237 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1238 if (!skb) { 1239 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 1240 return -ENOMEM; 1241 } 1242 req = (struct cpl_close_listserv_req *) skb_put(skb, sizeof(*req)); 1243 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1244 req->cpu_idx = 0; 1245 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, ep->stid)); 1246 skb->priority = 1; 1247 return iwch_cxgb3_ofld_send(ep->com.tdev, skb); 1248} 1249 1250static int close_listsrv_rpl(struct t3cdev *tdev, struct sk_buff *skb, 1251 void *ctx) 1252{ 1253 struct iwch_listen_ep *ep = ctx; 1254 struct cpl_close_listserv_rpl *rpl = cplhdr(skb); 1255 1256 PDBG("%s ep %p\n", __func__, ep); 1257 ep->com.rpl_err = status2errno(rpl->status); 1258 ep->com.rpl_done = 1; 1259 wake_up(&ep->com.waitq); 1260 return CPL_RET_BUF_DONE; 1261} 1262 1263static void accept_cr(struct iwch_ep *ep, __be32 peer_ip, struct sk_buff *skb) 1264{ 1265 struct cpl_pass_accept_rpl *rpl; 1266 unsigned int mtu_idx; 1267 u32 opt0h, opt0l, opt2; 1268 int wscale; 1269 1270 PDBG("%s ep %p\n", __func__, ep); 1271 BUG_ON(skb_cloned(skb)); 1272 skb_trim(skb, sizeof(*rpl)); 1273 skb_get(skb); 1274 mtu_idx = find_best_mtu(T3C_DATA(ep->com.tdev), dst_mtu(ep->dst)); 1275 wscale = compute_wscale(rcv_win); 1276 opt0h = V_NAGLE(0) | 1277 V_NO_CONG(nocong) | 1278 V_KEEP_ALIVE(1) | 1279 F_TCAM_BYPASS | 1280 V_WND_SCALE(wscale) | 1281 V_MSS_IDX(mtu_idx) | 1282 V_L2T_IDX(ep->l2t->idx) | V_TX_CHANNEL(ep->l2t->smt_idx); 1283 opt0l = V_TOS((ep->tos >> 2) & M_TOS) | V_RCV_BUFSIZ(rcv_win>>10); 1284 opt2 = F_RX_COALESCE_VALID | V_RX_COALESCE(0) | V_FLAVORS_VALID(1) | 1285 V_CONG_CONTROL_FLAVOR(cong_flavor); 1286 1287 rpl = cplhdr(skb); 1288 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1289 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, ep->hwtid)); 1290 rpl->peer_ip = peer_ip; 1291 rpl->opt0h = htonl(opt0h); 1292 rpl->opt0l_status = htonl(opt0l | CPL_PASS_OPEN_ACCEPT); 1293 rpl->opt2 = htonl(opt2); 1294 rpl->rsvd = rpl->opt2; 1295 skb->priority = CPL_PRIORITY_SETUP; 1296 iwch_l2t_send(ep->com.tdev, skb, ep->l2t); 1297 1298 return; 1299} 1300 1301static void reject_cr(struct t3cdev *tdev, u32 hwtid, __be32 peer_ip, 1302 struct sk_buff *skb) 1303{ 1304 PDBG("%s t3cdev %p tid %u peer_ip %x\n", __func__, tdev, hwtid, 1305 peer_ip); 1306 BUG_ON(skb_cloned(skb)); 1307 skb_trim(skb, sizeof(struct cpl_tid_release)); 1308 skb_get(skb); 1309 1310 if (tdev->type != T3A) 1311 release_tid(tdev, hwtid, skb); 1312 else { 1313 struct cpl_pass_accept_rpl *rpl; 1314 1315 rpl = cplhdr(skb); 1316 skb->priority = CPL_PRIORITY_SETUP; 1317 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1318 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 1319 hwtid)); 1320 rpl->peer_ip = peer_ip; 1321 rpl->opt0h = htonl(F_TCAM_BYPASS); 1322 rpl->opt0l_status = htonl(CPL_PASS_OPEN_REJECT); 1323 rpl->opt2 = 0; 1324 rpl->rsvd = rpl->opt2; 1325 iwch_cxgb3_ofld_send(tdev, skb); 1326 } 1327} 1328 1329static int pass_accept_req(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1330{ 1331 struct iwch_ep *child_ep, *parent_ep = ctx; 1332 struct cpl_pass_accept_req *req = cplhdr(skb); 1333 unsigned int hwtid = GET_TID(req); 1334 struct dst_entry *dst; 1335 struct l2t_entry *l2t; 1336 struct rtable *rt; 1337 struct iff_mac tim; 1338 1339 PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid); 1340 1341 if (state_read(&parent_ep->com) != LISTEN) { 1342 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 1343 __func__); 1344 goto reject; 1345 } 1346 1347 /* 1348 * Find the netdev for this connection request. 1349 */ 1350 tim.mac_addr = req->dst_mac; 1351 tim.vlan_tag = ntohs(req->vlan_tag); 1352 if (tdev->ctl(tdev, GET_IFF_FROM_MAC, &tim) < 0 || !tim.dev) { 1353 printk(KERN_ERR "%s bad dst mac %pM\n", 1354 __func__, req->dst_mac); 1355 goto reject; 1356 } 1357 1358 /* Find output route */ 1359 rt = find_route(tdev, 1360 req->local_ip, 1361 req->peer_ip, 1362 req->local_port, 1363 req->peer_port, G_PASS_OPEN_TOS(ntohl(req->tos_tid))); 1364 if (!rt) { 1365 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 1366 __func__); 1367 goto reject; 1368 } 1369 dst = &rt->dst; 1370 l2t = t3_l2t_get(tdev, dst->neighbour, dst->neighbour->dev); 1371 if (!l2t) { 1372 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 1373 __func__); 1374 dst_release(dst); 1375 goto reject; 1376 } 1377 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 1378 if (!child_ep) { 1379 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n", 1380 __func__); 1381 l2t_release(L2DATA(tdev), l2t); 1382 dst_release(dst); 1383 goto reject; 1384 } 1385 state_set(&child_ep->com, CONNECTING); 1386 child_ep->com.tdev = tdev; 1387 child_ep->com.cm_id = NULL; 1388 child_ep->com.local_addr.sin_family = PF_INET; 1389 child_ep->com.local_addr.sin_port = req->local_port; 1390 child_ep->com.local_addr.sin_addr.s_addr = req->local_ip; 1391 child_ep->com.remote_addr.sin_family = PF_INET; 1392 child_ep->com.remote_addr.sin_port = req->peer_port; 1393 child_ep->com.remote_addr.sin_addr.s_addr = req->peer_ip; 1394 get_ep(&parent_ep->com); 1395 child_ep->parent_ep = parent_ep; 1396 child_ep->tos = G_PASS_OPEN_TOS(ntohl(req->tos_tid)); 1397 child_ep->l2t = l2t; 1398 child_ep->dst = dst; 1399 child_ep->hwtid = hwtid; 1400 init_timer(&child_ep->timer); 1401 cxgb3_insert_tid(tdev, &t3c_client, child_ep, hwtid); 1402 accept_cr(child_ep, req->peer_ip, skb); 1403 goto out; 1404reject: 1405 reject_cr(tdev, hwtid, req->peer_ip, skb); 1406out: 1407 return CPL_RET_BUF_DONE; 1408} 1409 1410static int pass_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1411{ 1412 struct iwch_ep *ep = ctx; 1413 struct cpl_pass_establish *req = cplhdr(skb); 1414 1415 PDBG("%s ep %p\n", __func__, ep); 1416 ep->snd_seq = ntohl(req->snd_isn); 1417 ep->rcv_seq = ntohl(req->rcv_isn); 1418 1419 set_emss(ep, ntohs(req->tcp_opt)); 1420 1421 dst_confirm(ep->dst); 1422 state_set(&ep->com, MPA_REQ_WAIT); 1423 start_ep_timer(ep); 1424 1425 return CPL_RET_BUF_DONE; 1426} 1427 1428static int peer_close(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1429{ 1430 struct iwch_ep *ep = ctx; 1431 struct iwch_qp_attributes attrs; 1432 unsigned long flags; 1433 int disconnect = 1; 1434 int release = 0; 1435 1436 PDBG("%s ep %p\n", __func__, ep); 1437 dst_confirm(ep->dst); 1438 1439 spin_lock_irqsave(&ep->com.lock, flags); 1440 switch (ep->com.state) { 1441 case MPA_REQ_WAIT: 1442 __state_set(&ep->com, CLOSING); 1443 break; 1444 case MPA_REQ_SENT: 1445 __state_set(&ep->com, CLOSING); 1446 connect_reply_upcall(ep, -ECONNRESET); 1447 break; 1448 case MPA_REQ_RCVD: 1449 1450 /* 1451 * We're gonna mark this puppy DEAD, but keep 1452 * the reference on it until the ULP accepts or 1453 * rejects the CR. Also wake up anyone waiting 1454 * in rdma connection migration (see iwch_accept_cr()). 1455 */ 1456 __state_set(&ep->com, CLOSING); 1457 ep->com.rpl_done = 1; 1458 ep->com.rpl_err = -ECONNRESET; 1459 PDBG("waking up ep %p\n", ep); 1460 wake_up(&ep->com.waitq); 1461 break; 1462 case MPA_REP_SENT: 1463 __state_set(&ep->com, CLOSING); 1464 ep->com.rpl_done = 1; 1465 ep->com.rpl_err = -ECONNRESET; 1466 PDBG("waking up ep %p\n", ep); 1467 wake_up(&ep->com.waitq); 1468 break; 1469 case FPDU_MODE: 1470 start_ep_timer(ep); 1471 __state_set(&ep->com, CLOSING); 1472 attrs.next_state = IWCH_QP_STATE_CLOSING; 1473 iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, 1474 IWCH_QP_ATTR_NEXT_STATE, &attrs, 1); 1475 peer_close_upcall(ep); 1476 break; 1477 case ABORTING: 1478 disconnect = 0; 1479 break; 1480 case CLOSING: 1481 __state_set(&ep->com, MORIBUND); 1482 disconnect = 0; 1483 break; 1484 case MORIBUND: 1485 stop_ep_timer(ep); 1486 if (ep->com.cm_id && ep->com.qp) { 1487 attrs.next_state = IWCH_QP_STATE_IDLE; 1488 iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, 1489 IWCH_QP_ATTR_NEXT_STATE, &attrs, 1); 1490 } 1491 close_complete_upcall(ep); 1492 __state_set(&ep->com, DEAD); 1493 release = 1; 1494 disconnect = 0; 1495 break; 1496 case DEAD: 1497 disconnect = 0; 1498 break; 1499 default: 1500 BUG_ON(1); 1501 } 1502 spin_unlock_irqrestore(&ep->com.lock, flags); 1503 if (disconnect) 1504 iwch_ep_disconnect(ep, 0, GFP_KERNEL); 1505 if (release) 1506 release_ep_resources(ep); 1507 return CPL_RET_BUF_DONE; 1508} 1509 1510/* 1511 * Returns whether an ABORT_REQ_RSS message is a negative advice. 1512 */ 1513static int is_neg_adv_abort(unsigned int status) 1514{ 1515 return status == CPL_ERR_RTX_NEG_ADVICE || 1516 status == CPL_ERR_PERSIST_NEG_ADVICE; 1517} 1518 1519static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1520{ 1521 struct cpl_abort_req_rss *req = cplhdr(skb); 1522 struct iwch_ep *ep = ctx; 1523 struct cpl_abort_rpl *rpl; 1524 struct sk_buff *rpl_skb; 1525 struct iwch_qp_attributes attrs; 1526 int ret; 1527 int release = 0; 1528 unsigned long flags; 1529 1530 if (is_neg_adv_abort(req->status)) { 1531 PDBG("%s neg_adv_abort ep %p tid %d\n", __func__, ep, 1532 ep->hwtid); 1533 t3_l2t_send_event(ep->com.tdev, ep->l2t); 1534 return CPL_RET_BUF_DONE; 1535 } 1536 1537 /* 1538 * We get 2 peer aborts from the HW. The first one must 1539 * be ignored except for scribbling that we need one more. 1540 */ 1541 if (!test_and_set_bit(PEER_ABORT_IN_PROGRESS, &ep->com.flags)) { 1542 return CPL_RET_BUF_DONE; 1543 } 1544 1545 spin_lock_irqsave(&ep->com.lock, flags); 1546 PDBG("%s ep %p state %u\n", __func__, ep, ep->com.state); 1547 switch (ep->com.state) { 1548 case CONNECTING: 1549 break; 1550 case MPA_REQ_WAIT: 1551 stop_ep_timer(ep); 1552 break; 1553 case MPA_REQ_SENT: 1554 stop_ep_timer(ep); 1555 connect_reply_upcall(ep, -ECONNRESET); 1556 break; 1557 case MPA_REP_SENT: 1558 ep->com.rpl_done = 1; 1559 ep->com.rpl_err = -ECONNRESET; 1560 PDBG("waking up ep %p\n", ep); 1561 wake_up(&ep->com.waitq); 1562 break; 1563 case MPA_REQ_RCVD: 1564 1565 /* 1566 * We're gonna mark this puppy DEAD, but keep 1567 * the reference on it until the ULP accepts or 1568 * rejects the CR. Also wake up anyone waiting 1569 * in rdma connection migration (see iwch_accept_cr()). 1570 */ 1571 ep->com.rpl_done = 1; 1572 ep->com.rpl_err = -ECONNRESET; 1573 PDBG("waking up ep %p\n", ep); 1574 wake_up(&ep->com.waitq); 1575 break; 1576 case MORIBUND: 1577 case CLOSING: 1578 stop_ep_timer(ep); 1579 /*FALLTHROUGH*/ 1580 case FPDU_MODE: 1581 if (ep->com.cm_id && ep->com.qp) { 1582 attrs.next_state = IWCH_QP_STATE_ERROR; 1583 ret = iwch_modify_qp(ep->com.qp->rhp, 1584 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, 1585 &attrs, 1); 1586 if (ret) 1587 printk(KERN_ERR MOD 1588 "%s - qp <- error failed!\n", 1589 __func__); 1590 } 1591 peer_abort_upcall(ep); 1592 break; 1593 case ABORTING: 1594 break; 1595 case DEAD: 1596 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__); 1597 spin_unlock_irqrestore(&ep->com.lock, flags); 1598 return CPL_RET_BUF_DONE; 1599 default: 1600 BUG_ON(1); 1601 break; 1602 } 1603 dst_confirm(ep->dst); 1604 if (ep->com.state != ABORTING) { 1605 __state_set(&ep->com, DEAD); 1606 release = 1; 1607 } 1608 spin_unlock_irqrestore(&ep->com.lock, flags); 1609 1610 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL); 1611 if (!rpl_skb) { 1612 printk(KERN_ERR MOD "%s - cannot allocate skb!\n", 1613 __func__); 1614 release = 1; 1615 goto out; 1616 } 1617 rpl_skb->priority = CPL_PRIORITY_DATA; 1618 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl)); 1619 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 1620 rpl->wr.wr_lo = htonl(V_WR_TID(ep->hwtid)); 1621 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid)); 1622 rpl->cmd = CPL_ABORT_NO_RST; 1623 iwch_cxgb3_ofld_send(ep->com.tdev, rpl_skb); 1624out: 1625 if (release) 1626 release_ep_resources(ep); 1627 return CPL_RET_BUF_DONE; 1628} 1629 1630static int close_con_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1631{ 1632 struct iwch_ep *ep = ctx; 1633 struct iwch_qp_attributes attrs; 1634 unsigned long flags; 1635 int release = 0; 1636 1637 PDBG("%s ep %p\n", __func__, ep); 1638 BUG_ON(!ep); 1639 1640 /* The cm_id may be null if we failed to connect */ 1641 spin_lock_irqsave(&ep->com.lock, flags); 1642 switch (ep->com.state) { 1643 case CLOSING: 1644 __state_set(&ep->com, MORIBUND); 1645 break; 1646 case MORIBUND: 1647 stop_ep_timer(ep); 1648 if ((ep->com.cm_id) && (ep->com.qp)) { 1649 attrs.next_state = IWCH_QP_STATE_IDLE; 1650 iwch_modify_qp(ep->com.qp->rhp, 1651 ep->com.qp, 1652 IWCH_QP_ATTR_NEXT_STATE, 1653 &attrs, 1); 1654 } 1655 close_complete_upcall(ep); 1656 __state_set(&ep->com, DEAD); 1657 release = 1; 1658 break; 1659 case ABORTING: 1660 case DEAD: 1661 break; 1662 default: 1663 BUG_ON(1); 1664 break; 1665 } 1666 spin_unlock_irqrestore(&ep->com.lock, flags); 1667 if (release) 1668 release_ep_resources(ep); 1669 return CPL_RET_BUF_DONE; 1670} 1671 1672/* 1673 * T3A does 3 things when a TERM is received: 1674 * 1) send up a CPL_RDMA_TERMINATE message with the TERM packet 1675 * 2) generate an async event on the QP with the TERMINATE opcode 1676 * 3) post a TERMINATE opcde cqe into the associated CQ. 1677 * 1678 * For (1), we save the message in the qp for later consumer consumption. 1679 * For (2), we move the QP into TERMINATE, post a QP event and disconnect. 1680 * For (3), we toss the CQE in cxio_poll_cq(). 1681 * 1682 * terminate() handles case (1)... 1683 */ 1684static int terminate(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1685{ 1686 struct iwch_ep *ep = ctx; 1687 1688 if (state_read(&ep->com) != FPDU_MODE) 1689 return CPL_RET_BUF_DONE; 1690 1691 PDBG("%s ep %p\n", __func__, ep); 1692 skb_pull(skb, sizeof(struct cpl_rdma_terminate)); 1693 PDBG("%s saving %d bytes of term msg\n", __func__, skb->len); 1694 skb_copy_from_linear_data(skb, ep->com.qp->attr.terminate_buffer, 1695 skb->len); 1696 ep->com.qp->attr.terminate_msg_len = skb->len; 1697 ep->com.qp->attr.is_terminate_local = 0; 1698 return CPL_RET_BUF_DONE; 1699} 1700 1701static int ec_status(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 1702{ 1703 struct cpl_rdma_ec_status *rep = cplhdr(skb); 1704 struct iwch_ep *ep = ctx; 1705 1706 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, 1707 rep->status); 1708 if (rep->status) { 1709 struct iwch_qp_attributes attrs; 1710 1711 printk(KERN_ERR MOD "%s BAD CLOSE - Aborting tid %u\n", 1712 __func__, ep->hwtid); 1713 stop_ep_timer(ep); 1714 attrs.next_state = IWCH_QP_STATE_ERROR; 1715 iwch_modify_qp(ep->com.qp->rhp, 1716 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, 1717 &attrs, 1); 1718 abort_connection(ep, NULL, GFP_KERNEL); 1719 } 1720 return CPL_RET_BUF_DONE; 1721} 1722 1723static void ep_timeout(unsigned long arg) 1724{ 1725 struct iwch_ep *ep = (struct iwch_ep *)arg; 1726 struct iwch_qp_attributes attrs; 1727 unsigned long flags; 1728 int abort = 1; 1729 1730 spin_lock_irqsave(&ep->com.lock, flags); 1731 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid, 1732 ep->com.state); 1733 switch (ep->com.state) { 1734 case MPA_REQ_SENT: 1735 __state_set(&ep->com, ABORTING); 1736 connect_reply_upcall(ep, -ETIMEDOUT); 1737 break; 1738 case MPA_REQ_WAIT: 1739 __state_set(&ep->com, ABORTING); 1740 break; 1741 case CLOSING: 1742 case MORIBUND: 1743 if (ep->com.cm_id && ep->com.qp) { 1744 attrs.next_state = IWCH_QP_STATE_ERROR; 1745 iwch_modify_qp(ep->com.qp->rhp, 1746 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, 1747 &attrs, 1); 1748 } 1749 __state_set(&ep->com, ABORTING); 1750 break; 1751 default: 1752 printk(KERN_ERR "%s unexpected state ep %p state %u\n", 1753 __func__, ep, ep->com.state); 1754 WARN_ON(1); 1755 abort = 0; 1756 } 1757 spin_unlock_irqrestore(&ep->com.lock, flags); 1758 if (abort) 1759 abort_connection(ep, NULL, GFP_ATOMIC); 1760 put_ep(&ep->com); 1761} 1762 1763int iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 1764{ 1765 int err; 1766 struct iwch_ep *ep = to_ep(cm_id); 1767 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1768 1769 if (state_read(&ep->com) == DEAD) { 1770 put_ep(&ep->com); 1771 return -ECONNRESET; 1772 } 1773 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 1774 if (mpa_rev == 0) 1775 abort_connection(ep, NULL, GFP_KERNEL); 1776 else { 1777 err = send_mpa_reject(ep, pdata, pdata_len); 1778 err = iwch_ep_disconnect(ep, 0, GFP_KERNEL); 1779 } 1780 put_ep(&ep->com); 1781 return 0; 1782} 1783 1784int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 1785{ 1786 int err; 1787 struct iwch_qp_attributes attrs; 1788 enum iwch_qp_attr_mask mask; 1789 struct iwch_ep *ep = to_ep(cm_id); 1790 struct iwch_dev *h = to_iwch_dev(cm_id->device); 1791 struct iwch_qp *qp = get_qhp(h, conn_param->qpn); 1792 1793 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1794 if (state_read(&ep->com) == DEAD) { 1795 err = -ECONNRESET; 1796 goto err; 1797 } 1798 1799 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 1800 BUG_ON(!qp); 1801 1802 if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) || 1803 (conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) { 1804 abort_connection(ep, NULL, GFP_KERNEL); 1805 err = -EINVAL; 1806 goto err; 1807 } 1808 1809 cm_id->add_ref(cm_id); 1810 ep->com.cm_id = cm_id; 1811 ep->com.qp = qp; 1812 1813 ep->ird = conn_param->ird; 1814 ep->ord = conn_param->ord; 1815 1816 if (peer2peer && ep->ird == 0) 1817 ep->ird = 1; 1818 1819 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); 1820 1821 /* bind QP to EP and move to RTS */ 1822 attrs.mpa_attr = ep->mpa_attr; 1823 attrs.max_ird = ep->ird; 1824 attrs.max_ord = ep->ord; 1825 attrs.llp_stream_handle = ep; 1826 attrs.next_state = IWCH_QP_STATE_RTS; 1827 1828 /* bind QP and TID with INIT_WR */ 1829 mask = IWCH_QP_ATTR_NEXT_STATE | 1830 IWCH_QP_ATTR_LLP_STREAM_HANDLE | 1831 IWCH_QP_ATTR_MPA_ATTR | 1832 IWCH_QP_ATTR_MAX_IRD | 1833 IWCH_QP_ATTR_MAX_ORD; 1834 1835 err = iwch_modify_qp(ep->com.qp->rhp, 1836 ep->com.qp, mask, &attrs, 1); 1837 if (err) 1838 goto err1; 1839 1840 /* if needed, wait for wr_ack */ 1841 if (iwch_rqes_posted(qp)) { 1842 wait_event(ep->com.waitq, ep->com.rpl_done); 1843 err = ep->com.rpl_err; 1844 if (err) 1845 goto err1; 1846 } 1847 1848 err = send_mpa_reply(ep, conn_param->private_data, 1849 conn_param->private_data_len); 1850 if (err) 1851 goto err1; 1852 1853 1854 state_set(&ep->com, FPDU_MODE); 1855 established_upcall(ep); 1856 put_ep(&ep->com); 1857 return 0; 1858err1: 1859 ep->com.cm_id = NULL; 1860 ep->com.qp = NULL; 1861 cm_id->rem_ref(cm_id); 1862err: 1863 put_ep(&ep->com); 1864 return err; 1865} 1866 1867static int is_loopback_dst(struct iw_cm_id *cm_id) 1868{ 1869 struct net_device *dev; 1870 1871 dev = ip_dev_find(&init_net, cm_id->remote_addr.sin_addr.s_addr); 1872 if (!dev) 1873 return 0; 1874 dev_put(dev); 1875 return 1; 1876} 1877 1878int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 1879{ 1880 int err = 0; 1881 struct iwch_dev *h = to_iwch_dev(cm_id->device); 1882 struct iwch_ep *ep; 1883 struct rtable *rt; 1884 1885 if (is_loopback_dst(cm_id)) { 1886 err = -ENOSYS; 1887 goto out; 1888 } 1889 1890 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 1891 if (!ep) { 1892 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 1893 err = -ENOMEM; 1894 goto out; 1895 } 1896 init_timer(&ep->timer); 1897 ep->plen = conn_param->private_data_len; 1898 if (ep->plen) 1899 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 1900 conn_param->private_data, ep->plen); 1901 ep->ird = conn_param->ird; 1902 ep->ord = conn_param->ord; 1903 1904 if (peer2peer && ep->ord == 0) 1905 ep->ord = 1; 1906 1907 ep->com.tdev = h->rdev.t3cdev_p; 1908 1909 cm_id->add_ref(cm_id); 1910 ep->com.cm_id = cm_id; 1911 ep->com.qp = get_qhp(h, conn_param->qpn); 1912 BUG_ON(!ep->com.qp); 1913 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 1914 ep->com.qp, cm_id); 1915 1916 /* 1917 * Allocate an active TID to initiate a TCP connection. 1918 */ 1919 ep->atid = cxgb3_alloc_atid(h->rdev.t3cdev_p, &t3c_client, ep); 1920 if (ep->atid == -1) { 1921 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 1922 err = -ENOMEM; 1923 goto fail2; 1924 } 1925 1926 /* find a route */ 1927 rt = find_route(h->rdev.t3cdev_p, 1928 cm_id->local_addr.sin_addr.s_addr, 1929 cm_id->remote_addr.sin_addr.s_addr, 1930 cm_id->local_addr.sin_port, 1931 cm_id->remote_addr.sin_port, IPTOS_LOWDELAY); 1932 if (!rt) { 1933 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 1934 err = -EHOSTUNREACH; 1935 goto fail3; 1936 } 1937 ep->dst = &rt->dst; 1938 1939 /* get a l2t entry */ 1940 ep->l2t = t3_l2t_get(ep->com.tdev, ep->dst->neighbour, 1941 ep->dst->neighbour->dev); 1942 if (!ep->l2t) { 1943 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 1944 err = -ENOMEM; 1945 goto fail4; 1946 } 1947 1948 state_set(&ep->com, CONNECTING); 1949 ep->tos = IPTOS_LOWDELAY; 1950 ep->com.local_addr = cm_id->local_addr; 1951 ep->com.remote_addr = cm_id->remote_addr; 1952 1953 /* send connect request to rnic */ 1954 err = send_connect(ep); 1955 if (!err) 1956 goto out; 1957 1958 l2t_release(L2DATA(h->rdev.t3cdev_p), ep->l2t); 1959fail4: 1960 dst_release(ep->dst); 1961fail3: 1962 cxgb3_free_atid(ep->com.tdev, ep->atid); 1963fail2: 1964 cm_id->rem_ref(cm_id); 1965 put_ep(&ep->com); 1966out: 1967 return err; 1968} 1969 1970int iwch_create_listen(struct iw_cm_id *cm_id, int backlog) 1971{ 1972 int err = 0; 1973 struct iwch_dev *h = to_iwch_dev(cm_id->device); 1974 struct iwch_listen_ep *ep; 1975 1976 1977 might_sleep(); 1978 1979 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 1980 if (!ep) { 1981 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 1982 err = -ENOMEM; 1983 goto fail1; 1984 } 1985 PDBG("%s ep %p\n", __func__, ep); 1986 ep->com.tdev = h->rdev.t3cdev_p; 1987 cm_id->add_ref(cm_id); 1988 ep->com.cm_id = cm_id; 1989 ep->backlog = backlog; 1990 ep->com.local_addr = cm_id->local_addr; 1991 1992 /* 1993 * Allocate a server TID. 1994 */ 1995 ep->stid = cxgb3_alloc_stid(h->rdev.t3cdev_p, &t3c_client, ep); 1996 if (ep->stid == -1) { 1997 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 1998 err = -ENOMEM; 1999 goto fail2; 2000 } 2001 2002 state_set(&ep->com, LISTEN); 2003 err = listen_start(ep); 2004 if (err) 2005 goto fail3; 2006 2007 /* wait for pass_open_rpl */ 2008 wait_event(ep->com.waitq, ep->com.rpl_done); 2009 err = ep->com.rpl_err; 2010 if (!err) { 2011 cm_id->provider_data = ep; 2012 goto out; 2013 } 2014fail3: 2015 cxgb3_free_stid(ep->com.tdev, ep->stid); 2016fail2: 2017 cm_id->rem_ref(cm_id); 2018 put_ep(&ep->com); 2019fail1: 2020out: 2021 return err; 2022} 2023 2024int iwch_destroy_listen(struct iw_cm_id *cm_id) 2025{ 2026 int err; 2027 struct iwch_listen_ep *ep = to_listen_ep(cm_id); 2028 2029 PDBG("%s ep %p\n", __func__, ep); 2030 2031 might_sleep(); 2032 state_set(&ep->com, DEAD); 2033 ep->com.rpl_done = 0; 2034 ep->com.rpl_err = 0; 2035 err = listen_stop(ep); 2036 if (err) 2037 goto done; 2038 wait_event(ep->com.waitq, ep->com.rpl_done); 2039 cxgb3_free_stid(ep->com.tdev, ep->stid); 2040done: 2041 err = ep->com.rpl_err; 2042 cm_id->rem_ref(cm_id); 2043 put_ep(&ep->com); 2044 return err; 2045} 2046 2047int iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, gfp_t gfp) 2048{ 2049 int ret=0; 2050 unsigned long flags; 2051 int close = 0; 2052 int fatal = 0; 2053 struct t3cdev *tdev; 2054 struct cxio_rdev *rdev; 2055 2056 spin_lock_irqsave(&ep->com.lock, flags); 2057 2058 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep, 2059 states[ep->com.state], abrupt); 2060 2061 tdev = (struct t3cdev *)ep->com.tdev; 2062 rdev = (struct cxio_rdev *)tdev->ulp; 2063 if (cxio_fatal_error(rdev)) { 2064 fatal = 1; 2065 close_complete_upcall(ep); 2066 ep->com.state = DEAD; 2067 } 2068 switch (ep->com.state) { 2069 case MPA_REQ_WAIT: 2070 case MPA_REQ_SENT: 2071 case MPA_REQ_RCVD: 2072 case MPA_REP_SENT: 2073 case FPDU_MODE: 2074 close = 1; 2075 if (abrupt) 2076 ep->com.state = ABORTING; 2077 else { 2078 ep->com.state = CLOSING; 2079 start_ep_timer(ep); 2080 } 2081 set_bit(CLOSE_SENT, &ep->com.flags); 2082 break; 2083 case CLOSING: 2084 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2085 close = 1; 2086 if (abrupt) { 2087 stop_ep_timer(ep); 2088 ep->com.state = ABORTING; 2089 } else 2090 ep->com.state = MORIBUND; 2091 } 2092 break; 2093 case MORIBUND: 2094 case ABORTING: 2095 case DEAD: 2096 PDBG("%s ignoring disconnect ep %p state %u\n", 2097 __func__, ep, ep->com.state); 2098 break; 2099 default: 2100 BUG(); 2101 break; 2102 } 2103 2104 spin_unlock_irqrestore(&ep->com.lock, flags); 2105 if (close) { 2106 if (abrupt) 2107 ret = send_abort(ep, NULL, gfp); 2108 else 2109 ret = send_halfclose(ep, gfp); 2110 if (ret) 2111 fatal = 1; 2112 } 2113 if (fatal) 2114 release_ep_resources(ep); 2115 return ret; 2116} 2117 2118int iwch_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, 2119 struct l2t_entry *l2t) 2120{ 2121 struct iwch_ep *ep = ctx; 2122 2123 if (ep->dst != old) 2124 return 0; 2125 2126 PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new, 2127 l2t); 2128 dst_hold(new); 2129 l2t_release(L2DATA(ep->com.tdev), ep->l2t); 2130 ep->l2t = l2t; 2131 dst_release(old); 2132 ep->dst = new; 2133 return 1; 2134} 2135 2136/* 2137 * All the CM events are handled on a work queue to have a safe context. 2138 * These are the real handlers that are called from the work queue. 2139 */ 2140static const cxgb3_cpl_handler_func work_handlers[NUM_CPL_CMDS] = { 2141 [CPL_ACT_ESTABLISH] = act_establish, 2142 [CPL_ACT_OPEN_RPL] = act_open_rpl, 2143 [CPL_RX_DATA] = rx_data, 2144 [CPL_TX_DMA_ACK] = tx_ack, 2145 [CPL_ABORT_RPL_RSS] = abort_rpl, 2146 [CPL_ABORT_RPL] = abort_rpl, 2147 [CPL_PASS_OPEN_RPL] = pass_open_rpl, 2148 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl, 2149 [CPL_PASS_ACCEPT_REQ] = pass_accept_req, 2150 [CPL_PASS_ESTABLISH] = pass_establish, 2151 [CPL_PEER_CLOSE] = peer_close, 2152 [CPL_ABORT_REQ_RSS] = peer_abort, 2153 [CPL_CLOSE_CON_RPL] = close_con_rpl, 2154 [CPL_RDMA_TERMINATE] = terminate, 2155 [CPL_RDMA_EC_STATUS] = ec_status, 2156}; 2157 2158static void process_work(struct work_struct *work) 2159{ 2160 struct sk_buff *skb = NULL; 2161 void *ep; 2162 struct t3cdev *tdev; 2163 int ret; 2164 2165 while ((skb = skb_dequeue(&rxq))) { 2166 ep = *((void **) (skb->cb)); 2167 tdev = *((struct t3cdev **) (skb->cb + sizeof(void *))); 2168 ret = work_handlers[G_OPCODE(ntohl((__force __be32)skb->csum))](tdev, skb, ep); 2169 if (ret & CPL_RET_BUF_DONE) 2170 kfree_skb(skb); 2171 2172 /* 2173 * ep was referenced in sched(), and is freed here. 2174 */ 2175 put_ep((struct iwch_ep_common *)ep); 2176 } 2177} 2178 2179static DECLARE_WORK(skb_work, process_work); 2180 2181static int sched(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 2182{ 2183 struct iwch_ep_common *epc = ctx; 2184 2185 get_ep(epc); 2186 2187 /* 2188 * Save ctx and tdev in the skb->cb area. 2189 */ 2190 *((void **) skb->cb) = ctx; 2191 *((struct t3cdev **) (skb->cb + sizeof(void *))) = tdev; 2192 2193 /* 2194 * Queue the skb and schedule the worker thread. 2195 */ 2196 skb_queue_tail(&rxq, skb); 2197 queue_work(workq, &skb_work); 2198 return 0; 2199} 2200 2201static int set_tcb_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 2202{ 2203 struct cpl_set_tcb_rpl *rpl = cplhdr(skb); 2204 2205 if (rpl->status != CPL_ERR_NONE) { 2206 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u " 2207 "for tid %u\n", rpl->status, GET_TID(rpl)); 2208 } 2209 return CPL_RET_BUF_DONE; 2210} 2211 2212/* 2213 * All upcalls from the T3 Core go to sched() to schedule the 2214 * processing on a work queue. 2215 */ 2216cxgb3_cpl_handler_func t3c_handlers[NUM_CPL_CMDS] = { 2217 [CPL_ACT_ESTABLISH] = sched, 2218 [CPL_ACT_OPEN_RPL] = sched, 2219 [CPL_RX_DATA] = sched, 2220 [CPL_TX_DMA_ACK] = sched, 2221 [CPL_ABORT_RPL_RSS] = sched, 2222 [CPL_ABORT_RPL] = sched, 2223 [CPL_PASS_OPEN_RPL] = sched, 2224 [CPL_CLOSE_LISTSRV_RPL] = sched, 2225 [CPL_PASS_ACCEPT_REQ] = sched, 2226 [CPL_PASS_ESTABLISH] = sched, 2227 [CPL_PEER_CLOSE] = sched, 2228 [CPL_CLOSE_CON_RPL] = sched, 2229 [CPL_ABORT_REQ_RSS] = sched, 2230 [CPL_RDMA_TERMINATE] = sched, 2231 [CPL_RDMA_EC_STATUS] = sched, 2232 [CPL_SET_TCB_RPL] = set_tcb_rpl, 2233}; 2234 2235int __init iwch_cm_init(void) 2236{ 2237 skb_queue_head_init(&rxq); 2238 2239 workq = create_singlethread_workqueue("iw_cxgb3"); 2240 if (!workq) 2241 return -ENOMEM; 2242 2243 return 0; 2244} 2245 2246void __exit iwch_cm_term(void) 2247{ 2248 flush_workqueue(workq); 2249 destroy_workqueue(workq); 2250} 2251