1// SPDX-License-Identifier: GPL-2.0-or-later 2/******************************************************************************* 3 * This file contains iSCSI extentions for RDMA (iSER) Verbs 4 * 5 * (c) Copyright 2013 Datera, Inc. 6 * 7 * Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ****************************************************************************/ 10 11#include <linux/string.h> 12#include <linux/module.h> 13#include <linux/scatterlist.h> 14#include <linux/socket.h> 15#include <linux/in.h> 16#include <linux/in6.h> 17#include <rdma/ib_verbs.h> 18#include <rdma/ib_cm.h> 19#include <rdma/rdma_cm.h> 20#include <target/target_core_base.h> 21#include <target/target_core_fabric.h> 22#include <target/iscsi/iscsi_transport.h> 23#include <linux/semaphore.h> 24 25#include "ib_isert.h" 26 27static int isert_debug_level; 28module_param_named(debug_level, isert_debug_level, int, 0644); 29MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 30 31static int isert_sg_tablesize_set(const char *val, 32 const struct kernel_param *kp); 33static const struct kernel_param_ops sg_tablesize_ops = { 34 .set = isert_sg_tablesize_set, 35 .get = param_get_int, 36}; 37 38static int isert_sg_tablesize = ISCSI_ISER_MIN_SG_TABLESIZE; 39module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644); 40MODULE_PARM_DESC(sg_tablesize, 41 "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 128, max: 4096)"); 42 43static DEFINE_MUTEX(device_list_mutex); 44static LIST_HEAD(device_list); 45static struct workqueue_struct *isert_login_wq; 46static struct workqueue_struct *isert_comp_wq; 47static struct workqueue_struct *isert_release_wq; 48 49static int 50isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd); 51static int 52isert_login_post_recv(struct isert_conn *isert_conn); 53static int 54isert_rdma_accept(struct isert_conn *isert_conn); 55struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 56 57static void isert_release_work(struct work_struct *work); 58static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 59static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 60static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 61static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 62 63static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) 64{ 65 int n = 0, ret; 66 67 ret = kstrtoint(val, 10, &n); 68 if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE || 69 n > ISCSI_ISER_MAX_SG_TABLESIZE) 70 return -EINVAL; 71 72 return param_set_int(val, kp); 73} 74 75static inline bool 76isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 77{ 78 return (conn->pi_support && 79 cmd->prot_op != TARGET_PROT_NORMAL); 80} 81 82static void 83isert_qp_event_callback(struct ib_event *e, void *context) 84{ 85 struct isert_conn *isert_conn = context; 86 87 isert_err("%s (%d): conn %p\n", 88 ib_event_msg(e->event), e->event, isert_conn); 89 90 switch (e->event) { 91 case IB_EVENT_COMM_EST: 92 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 93 break; 94 case IB_EVENT_QP_LAST_WQE_REACHED: 95 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 96 break; 97 default: 98 break; 99 } 100} 101 102static struct ib_qp * 103isert_create_qp(struct isert_conn *isert_conn, 104 struct rdma_cm_id *cma_id) 105{ 106 u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2; 107 struct isert_device *device = isert_conn->device; 108 struct ib_device *ib_dev = device->ib_device; 109 struct ib_qp_init_attr attr; 110 int ret, factor; 111 112 isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE); 113 if (IS_ERR(isert_conn->cq)) { 114 isert_err("Unable to allocate cq\n"); 115 ret = PTR_ERR(isert_conn->cq); 116 return ERR_PTR(ret); 117 } 118 isert_conn->cq_size = cq_size; 119 120 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 121 attr.event_handler = isert_qp_event_callback; 122 attr.qp_context = isert_conn; 123 attr.send_cq = isert_conn->cq; 124 attr.recv_cq = isert_conn->cq; 125 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 126 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 127 factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num, 128 isert_sg_tablesize); 129 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor; 130 attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 131 attr.cap.max_recv_sge = 1; 132 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 133 attr.qp_type = IB_QPT_RC; 134 if (device->pi_capable) 135 attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 136 137 ret = rdma_create_qp(cma_id, device->pd, &attr); 138 if (ret) { 139 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 140 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 141 142 return ERR_PTR(ret); 143 } 144 145 return cma_id->qp; 146} 147 148static int 149isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 150{ 151 struct isert_device *device = isert_conn->device; 152 struct ib_device *ib_dev = device->ib_device; 153 struct iser_rx_desc *rx_desc; 154 struct ib_sge *rx_sg; 155 u64 dma_addr; 156 int i, j; 157 158 isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS, 159 sizeof(struct iser_rx_desc), 160 GFP_KERNEL); 161 if (!isert_conn->rx_descs) 162 return -ENOMEM; 163 164 rx_desc = isert_conn->rx_descs; 165 166 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 167 dma_addr = ib_dma_map_single(ib_dev, rx_desc->buf, 168 ISER_RX_SIZE, DMA_FROM_DEVICE); 169 if (ib_dma_mapping_error(ib_dev, dma_addr)) 170 goto dma_map_fail; 171 172 rx_desc->dma_addr = dma_addr; 173 174 rx_sg = &rx_desc->rx_sg; 175 rx_sg->addr = rx_desc->dma_addr + isert_get_hdr_offset(rx_desc); 176 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 177 rx_sg->lkey = device->pd->local_dma_lkey; 178 rx_desc->rx_cqe.done = isert_recv_done; 179 } 180 181 return 0; 182 183dma_map_fail: 184 rx_desc = isert_conn->rx_descs; 185 for (j = 0; j < i; j++, rx_desc++) { 186 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 187 ISER_RX_SIZE, DMA_FROM_DEVICE); 188 } 189 kfree(isert_conn->rx_descs); 190 isert_conn->rx_descs = NULL; 191 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 192 return -ENOMEM; 193} 194 195static void 196isert_free_rx_descriptors(struct isert_conn *isert_conn) 197{ 198 struct ib_device *ib_dev = isert_conn->device->ib_device; 199 struct iser_rx_desc *rx_desc; 200 int i; 201 202 if (!isert_conn->rx_descs) 203 return; 204 205 rx_desc = isert_conn->rx_descs; 206 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 207 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 208 ISER_RX_SIZE, DMA_FROM_DEVICE); 209 } 210 211 kfree(isert_conn->rx_descs); 212 isert_conn->rx_descs = NULL; 213} 214 215static int 216isert_create_device_ib_res(struct isert_device *device) 217{ 218 struct ib_device *ib_dev = device->ib_device; 219 int ret; 220 221 isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n", 222 ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 223 isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 224 225 device->pd = ib_alloc_pd(ib_dev, 0); 226 if (IS_ERR(device->pd)) { 227 ret = PTR_ERR(device->pd); 228 isert_err("failed to allocate pd, device %p, ret=%d\n", 229 device, ret); 230 return ret; 231 } 232 233 /* Check signature cap */ 234 if (ib_dev->attrs.kernel_cap_flags & IBK_INTEGRITY_HANDOVER) 235 device->pi_capable = true; 236 else 237 device->pi_capable = false; 238 239 return 0; 240} 241 242static void 243isert_free_device_ib_res(struct isert_device *device) 244{ 245 isert_info("device %p\n", device); 246 247 ib_dealloc_pd(device->pd); 248} 249 250static void 251isert_device_put(struct isert_device *device) 252{ 253 mutex_lock(&device_list_mutex); 254 device->refcount--; 255 isert_info("device %p refcount %d\n", device, device->refcount); 256 if (!device->refcount) { 257 isert_free_device_ib_res(device); 258 list_del(&device->dev_node); 259 kfree(device); 260 } 261 mutex_unlock(&device_list_mutex); 262} 263 264static struct isert_device * 265isert_device_get(struct rdma_cm_id *cma_id) 266{ 267 struct isert_device *device; 268 int ret; 269 270 mutex_lock(&device_list_mutex); 271 list_for_each_entry(device, &device_list, dev_node) { 272 if (device->ib_device->node_guid == cma_id->device->node_guid) { 273 device->refcount++; 274 isert_info("Found iser device %p refcount %d\n", 275 device, device->refcount); 276 mutex_unlock(&device_list_mutex); 277 return device; 278 } 279 } 280 281 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 282 if (!device) { 283 mutex_unlock(&device_list_mutex); 284 return ERR_PTR(-ENOMEM); 285 } 286 287 INIT_LIST_HEAD(&device->dev_node); 288 289 device->ib_device = cma_id->device; 290 ret = isert_create_device_ib_res(device); 291 if (ret) { 292 kfree(device); 293 mutex_unlock(&device_list_mutex); 294 return ERR_PTR(ret); 295 } 296 297 device->refcount++; 298 list_add_tail(&device->dev_node, &device_list); 299 isert_info("Created a new iser device %p refcount %d\n", 300 device, device->refcount); 301 mutex_unlock(&device_list_mutex); 302 303 return device; 304} 305 306static void 307isert_init_conn(struct isert_conn *isert_conn) 308{ 309 isert_conn->state = ISER_CONN_INIT; 310 INIT_LIST_HEAD(&isert_conn->node); 311 init_completion(&isert_conn->login_comp); 312 init_completion(&isert_conn->login_req_comp); 313 init_waitqueue_head(&isert_conn->rem_wait); 314 kref_init(&isert_conn->kref); 315 mutex_init(&isert_conn->mutex); 316 INIT_WORK(&isert_conn->release_work, isert_release_work); 317} 318 319static void 320isert_free_login_buf(struct isert_conn *isert_conn) 321{ 322 struct ib_device *ib_dev = isert_conn->device->ib_device; 323 324 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 325 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 326 kfree(isert_conn->login_rsp_buf); 327 328 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 329 ISER_RX_SIZE, DMA_FROM_DEVICE); 330 kfree(isert_conn->login_desc); 331} 332 333static int 334isert_alloc_login_buf(struct isert_conn *isert_conn, 335 struct ib_device *ib_dev) 336{ 337 int ret; 338 339 isert_conn->login_desc = kzalloc(sizeof(*isert_conn->login_desc), 340 GFP_KERNEL); 341 if (!isert_conn->login_desc) 342 return -ENOMEM; 343 344 isert_conn->login_desc->dma_addr = ib_dma_map_single(ib_dev, 345 isert_conn->login_desc->buf, 346 ISER_RX_SIZE, DMA_FROM_DEVICE); 347 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_desc->dma_addr); 348 if (ret) { 349 isert_err("login_desc dma mapping error: %d\n", ret); 350 isert_conn->login_desc->dma_addr = 0; 351 goto out_free_login_desc; 352 } 353 354 isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 355 if (!isert_conn->login_rsp_buf) { 356 ret = -ENOMEM; 357 goto out_unmap_login_desc; 358 } 359 360 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 361 isert_conn->login_rsp_buf, 362 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 363 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 364 if (ret) { 365 isert_err("login_rsp_dma mapping error: %d\n", ret); 366 isert_conn->login_rsp_dma = 0; 367 goto out_free_login_rsp_buf; 368 } 369 370 return 0; 371 372out_free_login_rsp_buf: 373 kfree(isert_conn->login_rsp_buf); 374out_unmap_login_desc: 375 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 376 ISER_RX_SIZE, DMA_FROM_DEVICE); 377out_free_login_desc: 378 kfree(isert_conn->login_desc); 379 return ret; 380} 381 382static void 383isert_set_nego_params(struct isert_conn *isert_conn, 384 struct rdma_conn_param *param) 385{ 386 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 387 388 /* Set max inflight RDMA READ requests */ 389 isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 390 attr->max_qp_init_rd_atom); 391 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 392 393 if (param->private_data) { 394 u8 flags = *(u8 *)param->private_data; 395 396 /* 397 * use remote invalidation if the both initiator 398 * and the HCA support it 399 */ 400 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 401 (attr->device_cap_flags & 402 IB_DEVICE_MEM_MGT_EXTENSIONS); 403 if (isert_conn->snd_w_inv) 404 isert_info("Using remote invalidation\n"); 405 } 406} 407 408static void 409isert_destroy_qp(struct isert_conn *isert_conn) 410{ 411 ib_destroy_qp(isert_conn->qp); 412 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 413} 414 415static int 416isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 417{ 418 struct isert_np *isert_np = cma_id->context; 419 struct iscsi_np *np = isert_np->np; 420 struct isert_conn *isert_conn; 421 struct isert_device *device; 422 int ret = 0; 423 424 spin_lock_bh(&np->np_thread_lock); 425 if (!np->enabled) { 426 spin_unlock_bh(&np->np_thread_lock); 427 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 428 return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 429 } 430 spin_unlock_bh(&np->np_thread_lock); 431 432 isert_dbg("cma_id: %p, portal: %p\n", 433 cma_id, cma_id->context); 434 435 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 436 if (!isert_conn) 437 return -ENOMEM; 438 439 isert_init_conn(isert_conn); 440 isert_conn->cm_id = cma_id; 441 442 device = isert_device_get(cma_id); 443 if (IS_ERR(device)) { 444 ret = PTR_ERR(device); 445 goto out; 446 } 447 isert_conn->device = device; 448 449 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 450 if (ret) 451 goto out_conn_dev; 452 453 isert_set_nego_params(isert_conn, &event->param.conn); 454 455 isert_conn->qp = isert_create_qp(isert_conn, cma_id); 456 if (IS_ERR(isert_conn->qp)) { 457 ret = PTR_ERR(isert_conn->qp); 458 goto out_rsp_dma_map; 459 } 460 461 ret = isert_login_post_recv(isert_conn); 462 if (ret) 463 goto out_destroy_qp; 464 465 ret = isert_rdma_accept(isert_conn); 466 if (ret) 467 goto out_destroy_qp; 468 469 mutex_lock(&isert_np->mutex); 470 list_add_tail(&isert_conn->node, &isert_np->accepted); 471 mutex_unlock(&isert_np->mutex); 472 473 return 0; 474 475out_destroy_qp: 476 isert_destroy_qp(isert_conn); 477out_rsp_dma_map: 478 isert_free_login_buf(isert_conn); 479out_conn_dev: 480 isert_device_put(device); 481out: 482 kfree(isert_conn); 483 rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 484 return ret; 485} 486 487static void 488isert_connect_release(struct isert_conn *isert_conn) 489{ 490 struct isert_device *device = isert_conn->device; 491 492 isert_dbg("conn %p\n", isert_conn); 493 494 BUG_ON(!device); 495 496 isert_free_rx_descriptors(isert_conn); 497 if (isert_conn->cm_id && 498 !isert_conn->dev_removed) 499 rdma_destroy_id(isert_conn->cm_id); 500 501 if (isert_conn->qp) 502 isert_destroy_qp(isert_conn); 503 504 if (isert_conn->login_desc) 505 isert_free_login_buf(isert_conn); 506 507 isert_device_put(device); 508 509 if (isert_conn->dev_removed) 510 wake_up_interruptible(&isert_conn->rem_wait); 511 else 512 kfree(isert_conn); 513} 514 515static void 516isert_connected_handler(struct rdma_cm_id *cma_id) 517{ 518 struct isert_conn *isert_conn = cma_id->qp->qp_context; 519 struct isert_np *isert_np = cma_id->context; 520 521 isert_info("conn %p\n", isert_conn); 522 523 mutex_lock(&isert_conn->mutex); 524 isert_conn->state = ISER_CONN_UP; 525 kref_get(&isert_conn->kref); 526 mutex_unlock(&isert_conn->mutex); 527 528 mutex_lock(&isert_np->mutex); 529 list_move_tail(&isert_conn->node, &isert_np->pending); 530 mutex_unlock(&isert_np->mutex); 531 532 isert_info("np %p: Allow accept_np to continue\n", isert_np); 533 up(&isert_np->sem); 534} 535 536static void 537isert_release_kref(struct kref *kref) 538{ 539 struct isert_conn *isert_conn = container_of(kref, 540 struct isert_conn, kref); 541 542 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 543 current->pid); 544 545 isert_connect_release(isert_conn); 546} 547 548static void 549isert_put_conn(struct isert_conn *isert_conn) 550{ 551 kref_put(&isert_conn->kref, isert_release_kref); 552} 553 554static void 555isert_handle_unbound_conn(struct isert_conn *isert_conn) 556{ 557 struct isert_np *isert_np = isert_conn->cm_id->context; 558 559 mutex_lock(&isert_np->mutex); 560 if (!list_empty(&isert_conn->node)) { 561 /* 562 * This means iscsi doesn't know this connection 563 * so schedule a cleanup ourselves 564 */ 565 list_del_init(&isert_conn->node); 566 isert_put_conn(isert_conn); 567 queue_work(isert_release_wq, &isert_conn->release_work); 568 } 569 mutex_unlock(&isert_np->mutex); 570} 571 572/** 573 * isert_conn_terminate() - Initiate connection termination 574 * @isert_conn: isert connection struct 575 * 576 * Notes: 577 * In case the connection state is BOUND, move state 578 * to TEMINATING and start teardown sequence (rdma_disconnect). 579 * In case the connection state is UP, complete flush as well. 580 * 581 * This routine must be called with mutex held. Thus it is 582 * safe to call multiple times. 583 */ 584static void 585isert_conn_terminate(struct isert_conn *isert_conn) 586{ 587 int err; 588 589 if (isert_conn->state >= ISER_CONN_TERMINATING) 590 return; 591 592 isert_info("Terminating conn %p state %d\n", 593 isert_conn, isert_conn->state); 594 isert_conn->state = ISER_CONN_TERMINATING; 595 err = rdma_disconnect(isert_conn->cm_id); 596 if (err) 597 isert_warn("Failed rdma_disconnect isert_conn %p\n", 598 isert_conn); 599} 600 601static int 602isert_np_cma_handler(struct isert_np *isert_np, 603 enum rdma_cm_event_type event) 604{ 605 isert_dbg("%s (%d): isert np %p\n", 606 rdma_event_msg(event), event, isert_np); 607 608 switch (event) { 609 case RDMA_CM_EVENT_DEVICE_REMOVAL: 610 isert_np->cm_id = NULL; 611 break; 612 case RDMA_CM_EVENT_ADDR_CHANGE: 613 isert_np->cm_id = isert_setup_id(isert_np); 614 if (IS_ERR(isert_np->cm_id)) { 615 isert_err("isert np %p setup id failed: %ld\n", 616 isert_np, PTR_ERR(isert_np->cm_id)); 617 isert_np->cm_id = NULL; 618 } 619 break; 620 default: 621 isert_err("isert np %p Unexpected event %d\n", 622 isert_np, event); 623 } 624 625 return -1; 626} 627 628static int 629isert_disconnected_handler(struct rdma_cm_id *cma_id, 630 enum rdma_cm_event_type event) 631{ 632 struct isert_conn *isert_conn = cma_id->qp->qp_context; 633 634 mutex_lock(&isert_conn->mutex); 635 switch (isert_conn->state) { 636 case ISER_CONN_TERMINATING: 637 break; 638 case ISER_CONN_UP: 639 isert_conn_terminate(isert_conn); 640 ib_drain_qp(isert_conn->qp); 641 isert_handle_unbound_conn(isert_conn); 642 break; 643 case ISER_CONN_BOUND: 644 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 645 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 646 break; 647 default: 648 isert_warn("conn %p terminating in state %d\n", 649 isert_conn, isert_conn->state); 650 } 651 mutex_unlock(&isert_conn->mutex); 652 653 return 0; 654} 655 656static int 657isert_connect_error(struct rdma_cm_id *cma_id) 658{ 659 struct isert_conn *isert_conn = cma_id->qp->qp_context; 660 struct isert_np *isert_np = cma_id->context; 661 662 ib_drain_qp(isert_conn->qp); 663 664 mutex_lock(&isert_np->mutex); 665 list_del_init(&isert_conn->node); 666 mutex_unlock(&isert_np->mutex); 667 isert_conn->cm_id = NULL; 668 isert_put_conn(isert_conn); 669 670 return -1; 671} 672 673static int 674isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 675{ 676 struct isert_np *isert_np = cma_id->context; 677 struct isert_conn *isert_conn; 678 int ret = 0; 679 680 isert_info("%s (%d): status %d id %p np %p\n", 681 rdma_event_msg(event->event), event->event, 682 event->status, cma_id, cma_id->context); 683 684 if (isert_np->cm_id == cma_id) 685 return isert_np_cma_handler(cma_id->context, event->event); 686 687 switch (event->event) { 688 case RDMA_CM_EVENT_CONNECT_REQUEST: 689 ret = isert_connect_request(cma_id, event); 690 if (ret) 691 isert_err("failed handle connect request %d\n", ret); 692 break; 693 case RDMA_CM_EVENT_ESTABLISHED: 694 isert_connected_handler(cma_id); 695 break; 696 case RDMA_CM_EVENT_ADDR_CHANGE: 697 case RDMA_CM_EVENT_DISCONNECTED: 698 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 699 ret = isert_disconnected_handler(cma_id, event->event); 700 break; 701 case RDMA_CM_EVENT_DEVICE_REMOVAL: 702 isert_conn = cma_id->qp->qp_context; 703 isert_conn->dev_removed = true; 704 isert_disconnected_handler(cma_id, event->event); 705 wait_event_interruptible(isert_conn->rem_wait, 706 isert_conn->state == ISER_CONN_DOWN); 707 kfree(isert_conn); 708 /* 709 * return non-zero from the callback to destroy 710 * the rdma cm id 711 */ 712 return 1; 713 case RDMA_CM_EVENT_REJECTED: 714 isert_info("Connection rejected: %s\n", 715 rdma_reject_msg(cma_id, event->status)); 716 fallthrough; 717 case RDMA_CM_EVENT_UNREACHABLE: 718 case RDMA_CM_EVENT_CONNECT_ERROR: 719 ret = isert_connect_error(cma_id); 720 break; 721 default: 722 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 723 break; 724 } 725 726 return ret; 727} 728 729static int 730isert_post_recvm(struct isert_conn *isert_conn, u32 count) 731{ 732 struct ib_recv_wr *rx_wr; 733 int i, ret; 734 struct iser_rx_desc *rx_desc; 735 736 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 737 rx_desc = &isert_conn->rx_descs[i]; 738 739 rx_wr->wr_cqe = &rx_desc->rx_cqe; 740 rx_wr->sg_list = &rx_desc->rx_sg; 741 rx_wr->num_sge = 1; 742 rx_wr->next = rx_wr + 1; 743 rx_desc->in_use = false; 744 } 745 rx_wr--; 746 rx_wr->next = NULL; /* mark end of work requests list */ 747 748 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 749 if (ret) 750 isert_err("ib_post_recv() failed with ret: %d\n", ret); 751 752 return ret; 753} 754 755static int 756isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 757{ 758 struct ib_recv_wr rx_wr; 759 int ret; 760 761 if (!rx_desc->in_use) { 762 /* 763 * if the descriptor is not in-use we already reposted it 764 * for recv, so just silently return 765 */ 766 return 0; 767 } 768 769 rx_desc->in_use = false; 770 rx_wr.wr_cqe = &rx_desc->rx_cqe; 771 rx_wr.sg_list = &rx_desc->rx_sg; 772 rx_wr.num_sge = 1; 773 rx_wr.next = NULL; 774 775 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 776 if (ret) 777 isert_err("ib_post_recv() failed with ret: %d\n", ret); 778 779 return ret; 780} 781 782static int 783isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 784{ 785 struct ib_device *ib_dev = isert_conn->cm_id->device; 786 struct ib_send_wr send_wr; 787 int ret; 788 789 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 790 ISER_HEADERS_LEN, DMA_TO_DEVICE); 791 792 tx_desc->tx_cqe.done = isert_login_send_done; 793 794 send_wr.next = NULL; 795 send_wr.wr_cqe = &tx_desc->tx_cqe; 796 send_wr.sg_list = tx_desc->tx_sg; 797 send_wr.num_sge = tx_desc->num_sge; 798 send_wr.opcode = IB_WR_SEND; 799 send_wr.send_flags = IB_SEND_SIGNALED; 800 801 ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 802 if (ret) 803 isert_err("ib_post_send() failed, ret: %d\n", ret); 804 805 return ret; 806} 807 808static void 809__isert_create_send_desc(struct isert_device *device, 810 struct iser_tx_desc *tx_desc) 811{ 812 813 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 814 tx_desc->iser_header.flags = ISCSI_CTRL; 815 816 tx_desc->num_sge = 1; 817 818 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 819 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 820 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 821 } 822} 823 824static void 825isert_create_send_desc(struct isert_conn *isert_conn, 826 struct isert_cmd *isert_cmd, 827 struct iser_tx_desc *tx_desc) 828{ 829 struct isert_device *device = isert_conn->device; 830 struct ib_device *ib_dev = device->ib_device; 831 832 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 833 ISER_HEADERS_LEN, DMA_TO_DEVICE); 834 835 __isert_create_send_desc(device, tx_desc); 836} 837 838static int 839isert_init_tx_hdrs(struct isert_conn *isert_conn, 840 struct iser_tx_desc *tx_desc) 841{ 842 struct isert_device *device = isert_conn->device; 843 struct ib_device *ib_dev = device->ib_device; 844 u64 dma_addr; 845 846 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 847 ISER_HEADERS_LEN, DMA_TO_DEVICE); 848 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 849 isert_err("ib_dma_mapping_error() failed\n"); 850 return -ENOMEM; 851 } 852 853 tx_desc->dma_addr = dma_addr; 854 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 855 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 856 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 857 858 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 859 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 860 tx_desc->tx_sg[0].lkey); 861 862 return 0; 863} 864 865static void 866isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 867 struct ib_send_wr *send_wr) 868{ 869 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 870 871 tx_desc->tx_cqe.done = isert_send_done; 872 send_wr->wr_cqe = &tx_desc->tx_cqe; 873 874 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 875 send_wr->opcode = IB_WR_SEND_WITH_INV; 876 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 877 } else { 878 send_wr->opcode = IB_WR_SEND; 879 } 880 881 send_wr->sg_list = &tx_desc->tx_sg[0]; 882 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 883 send_wr->send_flags = IB_SEND_SIGNALED; 884} 885 886static int 887isert_login_post_recv(struct isert_conn *isert_conn) 888{ 889 struct ib_recv_wr rx_wr; 890 struct ib_sge sge; 891 int ret; 892 893 memset(&sge, 0, sizeof(struct ib_sge)); 894 sge.addr = isert_conn->login_desc->dma_addr + 895 isert_get_hdr_offset(isert_conn->login_desc); 896 sge.length = ISER_RX_PAYLOAD_SIZE; 897 sge.lkey = isert_conn->device->pd->local_dma_lkey; 898 899 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 900 sge.addr, sge.length, sge.lkey); 901 902 isert_conn->login_desc->rx_cqe.done = isert_login_recv_done; 903 904 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 905 rx_wr.wr_cqe = &isert_conn->login_desc->rx_cqe; 906 rx_wr.sg_list = &sge; 907 rx_wr.num_sge = 1; 908 909 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 910 if (ret) 911 isert_err("ib_post_recv() failed: %d\n", ret); 912 913 return ret; 914} 915 916static int 917isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, 918 u32 length) 919{ 920 struct isert_conn *isert_conn = conn->context; 921 struct isert_device *device = isert_conn->device; 922 struct ib_device *ib_dev = device->ib_device; 923 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 924 int ret; 925 926 __isert_create_send_desc(device, tx_desc); 927 928 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 929 sizeof(struct iscsi_hdr)); 930 931 isert_init_tx_hdrs(isert_conn, tx_desc); 932 933 if (length > 0) { 934 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 935 936 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 937 length, DMA_TO_DEVICE); 938 939 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 940 941 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 942 length, DMA_TO_DEVICE); 943 944 tx_dsg->addr = isert_conn->login_rsp_dma; 945 tx_dsg->length = length; 946 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 947 tx_desc->num_sge = 2; 948 } 949 if (!login->login_failed) { 950 if (login->login_complete) { 951 ret = isert_alloc_rx_descriptors(isert_conn); 952 if (ret) 953 return ret; 954 955 ret = isert_post_recvm(isert_conn, 956 ISERT_QP_MAX_RECV_DTOS); 957 if (ret) 958 return ret; 959 960 /* Now we are in FULL_FEATURE phase */ 961 mutex_lock(&isert_conn->mutex); 962 isert_conn->state = ISER_CONN_FULL_FEATURE; 963 mutex_unlock(&isert_conn->mutex); 964 goto post_send; 965 } 966 967 ret = isert_login_post_recv(isert_conn); 968 if (ret) 969 return ret; 970 } 971post_send: 972 ret = isert_login_post_send(isert_conn, tx_desc); 973 if (ret) 974 return ret; 975 976 return 0; 977} 978 979static void 980isert_rx_login_req(struct isert_conn *isert_conn) 981{ 982 struct iser_rx_desc *rx_desc = isert_conn->login_desc; 983 int rx_buflen = isert_conn->login_req_len; 984 struct iscsit_conn *conn = isert_conn->conn; 985 struct iscsi_login *login = conn->conn_login; 986 int size; 987 988 isert_info("conn %p\n", isert_conn); 989 990 WARN_ON_ONCE(!login); 991 992 if (login->first_request) { 993 struct iscsi_login_req *login_req = 994 (struct iscsi_login_req *)isert_get_iscsi_hdr(rx_desc); 995 /* 996 * Setup the initial iscsi_login values from the leading 997 * login request PDU. 998 */ 999 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1000 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1001 login_req->flags); 1002 login->version_min = login_req->min_version; 1003 login->version_max = login_req->max_version; 1004 memcpy(login->isid, login_req->isid, 6); 1005 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1006 login->init_task_tag = login_req->itt; 1007 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1008 login->cid = be16_to_cpu(login_req->cid); 1009 login->tsih = be16_to_cpu(login_req->tsih); 1010 } 1011 1012 memcpy(&login->req[0], isert_get_iscsi_hdr(rx_desc), ISCSI_HDR_LEN); 1013 1014 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1015 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1016 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1017 MAX_KEY_VALUE_PAIRS); 1018 memcpy(login->req_buf, isert_get_data(rx_desc), size); 1019 1020 if (login->first_request) { 1021 complete(&isert_conn->login_comp); 1022 return; 1023 } 1024 queue_delayed_work(isert_login_wq, &conn->login_work, 0); 1025} 1026 1027static struct iscsit_cmd 1028*isert_allocate_cmd(struct iscsit_conn *conn, struct iser_rx_desc *rx_desc) 1029{ 1030 struct isert_conn *isert_conn = conn->context; 1031 struct isert_cmd *isert_cmd; 1032 struct iscsit_cmd *cmd; 1033 1034 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1035 if (!cmd) { 1036 isert_err("Unable to allocate iscsit_cmd + isert_cmd\n"); 1037 return NULL; 1038 } 1039 isert_cmd = iscsit_priv_cmd(cmd); 1040 isert_cmd->conn = isert_conn; 1041 isert_cmd->iscsit_cmd = cmd; 1042 isert_cmd->rx_desc = rx_desc; 1043 1044 return cmd; 1045} 1046 1047static int 1048isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1049 struct isert_cmd *isert_cmd, struct iscsit_cmd *cmd, 1050 struct iser_rx_desc *rx_desc, unsigned char *buf) 1051{ 1052 struct iscsit_conn *conn = isert_conn->conn; 1053 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1054 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1055 bool dump_payload = false; 1056 unsigned int data_len; 1057 1058 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1059 if (rc < 0) 1060 return rc; 1061 1062 imm_data = cmd->immediate_data; 1063 imm_data_len = cmd->first_burst_len; 1064 unsol_data = cmd->unsolicited_data; 1065 data_len = cmd->se_cmd.data_length; 1066 1067 if (imm_data && imm_data_len == data_len) 1068 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1069 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1070 if (rc < 0) { 1071 return 0; 1072 } else if (rc > 0) { 1073 dump_payload = true; 1074 goto sequence_cmd; 1075 } 1076 1077 if (!imm_data) 1078 return 0; 1079 1080 if (imm_data_len != data_len) { 1081 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1082 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1083 isert_get_data(rx_desc), imm_data_len); 1084 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1085 sg_nents, imm_data_len); 1086 } else { 1087 sg_init_table(&isert_cmd->sg, 1); 1088 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1089 cmd->se_cmd.t_data_nents = 1; 1090 sg_set_buf(&isert_cmd->sg, isert_get_data(rx_desc), 1091 imm_data_len); 1092 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1093 imm_data_len); 1094 } 1095 1096 cmd->write_data_done += imm_data_len; 1097 1098 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1099 spin_lock_bh(&cmd->istate_lock); 1100 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1101 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1102 spin_unlock_bh(&cmd->istate_lock); 1103 } 1104 1105sequence_cmd: 1106 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1107 1108 if (!rc && !dump_payload && unsol_data) 1109 iscsit_set_unsolicited_dataout(cmd); 1110 else if (dump_payload && imm_data) 1111 target_put_sess_cmd(&cmd->se_cmd); 1112 1113 return 0; 1114} 1115 1116static int 1117isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1118 struct iser_rx_desc *rx_desc, unsigned char *buf) 1119{ 1120 struct scatterlist *sg_start; 1121 struct iscsit_conn *conn = isert_conn->conn; 1122 struct iscsit_cmd *cmd = NULL; 1123 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1124 u32 unsol_data_len = ntoh24(hdr->dlength); 1125 int rc, sg_nents, sg_off, page_off; 1126 1127 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1128 if (rc < 0) 1129 return rc; 1130 else if (!cmd) 1131 return 0; 1132 /* 1133 * FIXME: Unexpected unsolicited_data out 1134 */ 1135 if (!cmd->unsolicited_data) { 1136 isert_err("Received unexpected solicited data payload\n"); 1137 dump_stack(); 1138 return -1; 1139 } 1140 1141 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1142 "write_data_done: %u, data_length: %u\n", 1143 unsol_data_len, cmd->write_data_done, 1144 cmd->se_cmd.data_length); 1145 1146 sg_off = cmd->write_data_done / PAGE_SIZE; 1147 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1148 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1149 page_off = cmd->write_data_done % PAGE_SIZE; 1150 /* 1151 * FIXME: Non page-aligned unsolicited_data out 1152 */ 1153 if (page_off) { 1154 isert_err("unexpected non-page aligned data payload\n"); 1155 dump_stack(); 1156 return -1; 1157 } 1158 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1159 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1160 sg_nents, isert_get_data(rx_desc), unsol_data_len); 1161 1162 sg_copy_from_buffer(sg_start, sg_nents, isert_get_data(rx_desc), 1163 unsol_data_len); 1164 1165 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1166 if (rc < 0) 1167 return rc; 1168 1169 /* 1170 * multiple data-outs on the same command can arrive - 1171 * so post the buffer before hand 1172 */ 1173 return isert_post_recv(isert_conn, rx_desc); 1174} 1175 1176static int 1177isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1178 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1179 unsigned char *buf) 1180{ 1181 struct iscsit_conn *conn = isert_conn->conn; 1182 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1183 int rc; 1184 1185 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1186 if (rc < 0) 1187 return rc; 1188 /* 1189 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1190 */ 1191 1192 return iscsit_process_nop_out(conn, cmd, hdr); 1193} 1194 1195static int 1196isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1197 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1198 struct iscsi_text *hdr) 1199{ 1200 struct iscsit_conn *conn = isert_conn->conn; 1201 u32 payload_length = ntoh24(hdr->dlength); 1202 int rc; 1203 unsigned char *text_in = NULL; 1204 1205 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1206 if (rc < 0) 1207 return rc; 1208 1209 if (payload_length) { 1210 text_in = kzalloc(payload_length, GFP_KERNEL); 1211 if (!text_in) 1212 return -ENOMEM; 1213 } 1214 cmd->text_in_ptr = text_in; 1215 1216 memcpy(cmd->text_in_ptr, isert_get_data(rx_desc), payload_length); 1217 1218 return iscsit_process_text_cmd(conn, cmd, hdr); 1219} 1220 1221static int 1222isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1223 uint32_t read_stag, uint64_t read_va, 1224 uint32_t write_stag, uint64_t write_va) 1225{ 1226 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1227 struct iscsit_conn *conn = isert_conn->conn; 1228 struct iscsit_cmd *cmd; 1229 struct isert_cmd *isert_cmd; 1230 int ret = -EINVAL; 1231 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1232 1233 if (conn->sess->sess_ops->SessionType && 1234 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1235 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1236 " ignoring\n", opcode); 1237 return 0; 1238 } 1239 1240 switch (opcode) { 1241 case ISCSI_OP_SCSI_CMD: 1242 cmd = isert_allocate_cmd(conn, rx_desc); 1243 if (!cmd) 1244 break; 1245 1246 isert_cmd = iscsit_priv_cmd(cmd); 1247 isert_cmd->read_stag = read_stag; 1248 isert_cmd->read_va = read_va; 1249 isert_cmd->write_stag = write_stag; 1250 isert_cmd->write_va = write_va; 1251 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1252 1253 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1254 rx_desc, (unsigned char *)hdr); 1255 break; 1256 case ISCSI_OP_NOOP_OUT: 1257 cmd = isert_allocate_cmd(conn, rx_desc); 1258 if (!cmd) 1259 break; 1260 1261 isert_cmd = iscsit_priv_cmd(cmd); 1262 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1263 rx_desc, (unsigned char *)hdr); 1264 break; 1265 case ISCSI_OP_SCSI_DATA_OUT: 1266 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1267 (unsigned char *)hdr); 1268 break; 1269 case ISCSI_OP_SCSI_TMFUNC: 1270 cmd = isert_allocate_cmd(conn, rx_desc); 1271 if (!cmd) 1272 break; 1273 1274 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1275 (unsigned char *)hdr); 1276 break; 1277 case ISCSI_OP_LOGOUT: 1278 cmd = isert_allocate_cmd(conn, rx_desc); 1279 if (!cmd) 1280 break; 1281 1282 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1283 break; 1284 case ISCSI_OP_TEXT: 1285 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1286 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1287 else 1288 cmd = isert_allocate_cmd(conn, rx_desc); 1289 1290 if (!cmd) 1291 break; 1292 1293 isert_cmd = iscsit_priv_cmd(cmd); 1294 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1295 rx_desc, (struct iscsi_text *)hdr); 1296 break; 1297 default: 1298 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1299 dump_stack(); 1300 break; 1301 } 1302 1303 return ret; 1304} 1305 1306static void 1307isert_print_wc(struct ib_wc *wc, const char *type) 1308{ 1309 if (wc->status != IB_WC_WR_FLUSH_ERR) 1310 isert_err("%s failure: %s (%d) vend_err %x\n", type, 1311 ib_wc_status_msg(wc->status), wc->status, 1312 wc->vendor_err); 1313 else 1314 isert_dbg("%s failure: %s (%d)\n", type, 1315 ib_wc_status_msg(wc->status), wc->status); 1316} 1317 1318static void 1319isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1320{ 1321 struct isert_conn *isert_conn = wc->qp->qp_context; 1322 struct ib_device *ib_dev = isert_conn->cm_id->device; 1323 struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 1324 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1325 struct iser_ctrl *iser_ctrl = isert_get_iser_hdr(rx_desc); 1326 uint64_t read_va = 0, write_va = 0; 1327 uint32_t read_stag = 0, write_stag = 0; 1328 1329 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1330 isert_print_wc(wc, "recv"); 1331 if (wc->status != IB_WC_WR_FLUSH_ERR) 1332 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1333 return; 1334 } 1335 1336 rx_desc->in_use = true; 1337 1338 ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 1339 ISER_RX_SIZE, DMA_FROM_DEVICE); 1340 1341 isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1342 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 1343 (int)(wc->byte_len - ISER_HEADERS_LEN)); 1344 1345 switch (iser_ctrl->flags & 0xF0) { 1346 case ISCSI_CTRL: 1347 if (iser_ctrl->flags & ISER_RSV) { 1348 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1349 read_va = be64_to_cpu(iser_ctrl->read_va); 1350 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1351 read_stag, (unsigned long long)read_va); 1352 } 1353 if (iser_ctrl->flags & ISER_WSV) { 1354 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1355 write_va = be64_to_cpu(iser_ctrl->write_va); 1356 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1357 write_stag, (unsigned long long)write_va); 1358 } 1359 1360 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1361 break; 1362 case ISER_HELLO: 1363 isert_err("iSER Hello message\n"); 1364 break; 1365 default: 1366 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1367 break; 1368 } 1369 1370 isert_rx_opcode(isert_conn, rx_desc, 1371 read_stag, read_va, write_stag, write_va); 1372 1373 ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 1374 ISER_RX_SIZE, DMA_FROM_DEVICE); 1375} 1376 1377static void 1378isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1379{ 1380 struct isert_conn *isert_conn = wc->qp->qp_context; 1381 struct ib_device *ib_dev = isert_conn->device->ib_device; 1382 1383 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1384 isert_print_wc(wc, "login recv"); 1385 return; 1386 } 1387 1388 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_desc->dma_addr, 1389 ISER_RX_SIZE, DMA_FROM_DEVICE); 1390 1391 isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1392 1393 if (isert_conn->conn) { 1394 struct iscsi_login *login = isert_conn->conn->conn_login; 1395 1396 if (login && !login->first_request) 1397 isert_rx_login_req(isert_conn); 1398 } 1399 1400 mutex_lock(&isert_conn->mutex); 1401 complete(&isert_conn->login_req_comp); 1402 mutex_unlock(&isert_conn->mutex); 1403 1404 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_desc->dma_addr, 1405 ISER_RX_SIZE, DMA_FROM_DEVICE); 1406} 1407 1408static void 1409isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1410{ 1411 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 1412 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1413 1414 if (!cmd->rw.nr_ops) 1415 return; 1416 1417 if (isert_prot_cmd(conn, se_cmd)) { 1418 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 1419 conn->cm_id->port_num, se_cmd->t_data_sg, 1420 se_cmd->t_data_nents, se_cmd->t_prot_sg, 1421 se_cmd->t_prot_nents, dir); 1422 } else { 1423 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 1424 se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1425 } 1426 1427 cmd->rw.nr_ops = 0; 1428} 1429 1430static void 1431isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1432{ 1433 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1434 struct isert_conn *isert_conn = isert_cmd->conn; 1435 struct iscsit_conn *conn = isert_conn->conn; 1436 struct iscsi_text_rsp *hdr; 1437 1438 isert_dbg("Cmd %p\n", isert_cmd); 1439 1440 switch (cmd->iscsi_opcode) { 1441 case ISCSI_OP_SCSI_CMD: 1442 spin_lock_bh(&conn->cmd_lock); 1443 if (!list_empty(&cmd->i_conn_node)) 1444 list_del_init(&cmd->i_conn_node); 1445 spin_unlock_bh(&conn->cmd_lock); 1446 1447 if (cmd->data_direction == DMA_TO_DEVICE) { 1448 iscsit_stop_dataout_timer(cmd); 1449 /* 1450 * Check for special case during comp_err where 1451 * WRITE_PENDING has been handed off from core, 1452 * but requires an extra target_put_sess_cmd() 1453 * before transport_generic_free_cmd() below. 1454 */ 1455 if (comp_err && 1456 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1457 struct se_cmd *se_cmd = &cmd->se_cmd; 1458 1459 target_put_sess_cmd(se_cmd); 1460 } 1461 } 1462 1463 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1464 transport_generic_free_cmd(&cmd->se_cmd, 0); 1465 break; 1466 case ISCSI_OP_SCSI_TMFUNC: 1467 spin_lock_bh(&conn->cmd_lock); 1468 if (!list_empty(&cmd->i_conn_node)) 1469 list_del_init(&cmd->i_conn_node); 1470 spin_unlock_bh(&conn->cmd_lock); 1471 1472 transport_generic_free_cmd(&cmd->se_cmd, 0); 1473 break; 1474 case ISCSI_OP_REJECT: 1475 case ISCSI_OP_NOOP_OUT: 1476 case ISCSI_OP_TEXT: 1477 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1478 /* If the continue bit is on, keep the command alive */ 1479 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1480 break; 1481 1482 spin_lock_bh(&conn->cmd_lock); 1483 if (!list_empty(&cmd->i_conn_node)) 1484 list_del_init(&cmd->i_conn_node); 1485 spin_unlock_bh(&conn->cmd_lock); 1486 1487 /* 1488 * Handle special case for REJECT when iscsi_add_reject*() has 1489 * overwritten the original iscsi_opcode assignment, and the 1490 * associated cmd->se_cmd needs to be released. 1491 */ 1492 if (cmd->se_cmd.se_tfo != NULL) { 1493 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1494 cmd->iscsi_opcode); 1495 transport_generic_free_cmd(&cmd->se_cmd, 0); 1496 break; 1497 } 1498 fallthrough; 1499 default: 1500 iscsit_release_cmd(cmd); 1501 break; 1502 } 1503} 1504 1505static void 1506isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1507{ 1508 if (tx_desc->dma_addr != 0) { 1509 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1510 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1511 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1512 tx_desc->dma_addr = 0; 1513 } 1514} 1515 1516static void 1517isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1518 struct ib_device *ib_dev, bool comp_err) 1519{ 1520 if (isert_cmd->pdu_buf_dma != 0) { 1521 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1522 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1523 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1524 isert_cmd->pdu_buf_dma = 0; 1525 } 1526 1527 isert_unmap_tx_desc(tx_desc, ib_dev); 1528 isert_put_cmd(isert_cmd, comp_err); 1529} 1530 1531static int 1532isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1533{ 1534 struct ib_mr_status mr_status; 1535 int ret; 1536 1537 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1538 if (ret) { 1539 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1540 goto fail_mr_status; 1541 } 1542 1543 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1544 u64 sec_offset_err; 1545 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1546 1547 switch (mr_status.sig_err.err_type) { 1548 case IB_SIG_BAD_GUARD: 1549 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1550 break; 1551 case IB_SIG_BAD_REFTAG: 1552 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1553 break; 1554 case IB_SIG_BAD_APPTAG: 1555 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1556 break; 1557 } 1558 sec_offset_err = mr_status.sig_err.sig_err_offset; 1559 do_div(sec_offset_err, block_size); 1560 se_cmd->sense_info = sec_offset_err + se_cmd->t_task_lba; 1561 1562 isert_err("PI error found type %d at sector 0x%llx " 1563 "expected 0x%x vs actual 0x%x\n", 1564 mr_status.sig_err.err_type, 1565 (unsigned long long)se_cmd->sense_info, 1566 mr_status.sig_err.expected, 1567 mr_status.sig_err.actual); 1568 ret = 1; 1569 } 1570 1571fail_mr_status: 1572 return ret; 1573} 1574 1575static void 1576isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 1577{ 1578 struct isert_conn *isert_conn = wc->qp->qp_context; 1579 struct isert_device *device = isert_conn->device; 1580 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1581 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1582 struct se_cmd *cmd = &isert_cmd->iscsit_cmd->se_cmd; 1583 int ret = 0; 1584 1585 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1586 isert_print_wc(wc, "rdma write"); 1587 if (wc->status != IB_WC_WR_FLUSH_ERR) 1588 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1589 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1590 return; 1591 } 1592 1593 isert_dbg("Cmd %p\n", isert_cmd); 1594 1595 ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 1596 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1597 1598 if (ret) { 1599 /* 1600 * transport_generic_request_failure() expects to have 1601 * plus two references to handle queue-full, so re-add 1602 * one here as target-core will have already dropped 1603 * it after the first isert_put_datain() callback. 1604 */ 1605 kref_get(&cmd->cmd_kref); 1606 transport_generic_request_failure(cmd, cmd->pi_err); 1607 } else { 1608 /* 1609 * XXX: isert_put_response() failure is not retried. 1610 */ 1611 ret = isert_put_response(isert_conn->conn, isert_cmd->iscsit_cmd); 1612 if (ret) 1613 pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1614 } 1615} 1616 1617static void 1618isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1619{ 1620 struct isert_conn *isert_conn = wc->qp->qp_context; 1621 struct isert_device *device = isert_conn->device; 1622 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1623 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1624 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1625 struct se_cmd *se_cmd = &cmd->se_cmd; 1626 int ret = 0; 1627 1628 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1629 isert_print_wc(wc, "rdma read"); 1630 if (wc->status != IB_WC_WR_FLUSH_ERR) 1631 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1632 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1633 return; 1634 } 1635 1636 isert_dbg("Cmd %p\n", isert_cmd); 1637 1638 iscsit_stop_dataout_timer(cmd); 1639 1640 if (isert_prot_cmd(isert_conn, se_cmd)) 1641 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 1642 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1643 cmd->write_data_done = 0; 1644 1645 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1646 spin_lock_bh(&cmd->istate_lock); 1647 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1648 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1649 spin_unlock_bh(&cmd->istate_lock); 1650 1651 /* 1652 * transport_generic_request_failure() will drop the extra 1653 * se_cmd->cmd_kref reference after T10-PI error, and handle 1654 * any non-zero ->queue_status() callback error retries. 1655 */ 1656 if (ret) 1657 transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1658 else 1659 target_execute_cmd(se_cmd); 1660} 1661 1662static void 1663isert_do_control_comp(struct work_struct *work) 1664{ 1665 struct isert_cmd *isert_cmd = container_of(work, 1666 struct isert_cmd, comp_work); 1667 struct isert_conn *isert_conn = isert_cmd->conn; 1668 struct ib_device *ib_dev = isert_conn->cm_id->device; 1669 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1670 1671 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1672 1673 switch (cmd->i_state) { 1674 case ISTATE_SEND_TASKMGTRSP: 1675 iscsit_tmr_post_handler(cmd, cmd->conn); 1676 fallthrough; 1677 case ISTATE_SEND_REJECT: 1678 case ISTATE_SEND_TEXTRSP: 1679 cmd->i_state = ISTATE_SENT_STATUS; 1680 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1681 ib_dev, false); 1682 break; 1683 case ISTATE_SEND_LOGOUTRSP: 1684 iscsit_logout_post_handler(cmd, cmd->conn); 1685 break; 1686 default: 1687 isert_err("Unknown i_state %d\n", cmd->i_state); 1688 dump_stack(); 1689 break; 1690 } 1691} 1692 1693static void 1694isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1695{ 1696 struct isert_conn *isert_conn = wc->qp->qp_context; 1697 struct ib_device *ib_dev = isert_conn->cm_id->device; 1698 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1699 1700 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1701 isert_print_wc(wc, "login send"); 1702 if (wc->status != IB_WC_WR_FLUSH_ERR) 1703 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1704 } 1705 1706 isert_unmap_tx_desc(tx_desc, ib_dev); 1707} 1708 1709static void 1710isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 1711{ 1712 struct isert_conn *isert_conn = wc->qp->qp_context; 1713 struct ib_device *ib_dev = isert_conn->cm_id->device; 1714 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1715 struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 1716 1717 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1718 isert_print_wc(wc, "send"); 1719 if (wc->status != IB_WC_WR_FLUSH_ERR) 1720 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1721 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 1722 return; 1723 } 1724 1725 isert_dbg("Cmd %p\n", isert_cmd); 1726 1727 switch (isert_cmd->iscsit_cmd->i_state) { 1728 case ISTATE_SEND_TASKMGTRSP: 1729 case ISTATE_SEND_LOGOUTRSP: 1730 case ISTATE_SEND_REJECT: 1731 case ISTATE_SEND_TEXTRSP: 1732 isert_unmap_tx_desc(tx_desc, ib_dev); 1733 1734 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1735 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1736 return; 1737 default: 1738 isert_cmd->iscsit_cmd->i_state = ISTATE_SENT_STATUS; 1739 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1740 break; 1741 } 1742} 1743 1744static int 1745isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1746{ 1747 int ret; 1748 1749 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 1750 if (ret) 1751 return ret; 1752 1753 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1754 if (ret) { 1755 isert_err("ib_post_send failed with %d\n", ret); 1756 return ret; 1757 } 1758 return ret; 1759} 1760 1761static int 1762isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1763{ 1764 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1765 struct isert_conn *isert_conn = conn->context; 1766 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1767 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1768 &isert_cmd->tx_desc.iscsi_header; 1769 1770 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1771 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1772 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1773 /* 1774 * Attach SENSE DATA payload to iSCSI Response PDU 1775 */ 1776 if (cmd->se_cmd.sense_buffer && 1777 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1778 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1779 struct isert_device *device = isert_conn->device; 1780 struct ib_device *ib_dev = device->ib_device; 1781 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1782 u32 padding, pdu_len; 1783 1784 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1785 cmd->sense_buffer); 1786 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1787 1788 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1789 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1790 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1791 1792 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1793 (void *)cmd->sense_buffer, pdu_len, 1794 DMA_TO_DEVICE); 1795 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1796 return -ENOMEM; 1797 1798 isert_cmd->pdu_buf_len = pdu_len; 1799 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1800 tx_dsg->length = pdu_len; 1801 tx_dsg->lkey = device->pd->local_dma_lkey; 1802 isert_cmd->tx_desc.num_sge = 2; 1803 } 1804 1805 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1806 1807 isert_dbg("Posting SCSI Response\n"); 1808 1809 return isert_post_response(isert_conn, isert_cmd); 1810} 1811 1812static void 1813isert_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1814{ 1815 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1816 struct isert_conn *isert_conn = conn->context; 1817 1818 spin_lock_bh(&conn->cmd_lock); 1819 if (!list_empty(&cmd->i_conn_node)) 1820 list_del_init(&cmd->i_conn_node); 1821 spin_unlock_bh(&conn->cmd_lock); 1822 1823 if (cmd->data_direction == DMA_TO_DEVICE) 1824 iscsit_stop_dataout_timer(cmd); 1825 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1826} 1827 1828static enum target_prot_op 1829isert_get_sup_prot_ops(struct iscsit_conn *conn) 1830{ 1831 struct isert_conn *isert_conn = conn->context; 1832 struct isert_device *device = isert_conn->device; 1833 1834 if (conn->tpg->tpg_attrib.t10_pi) { 1835 if (device->pi_capable) { 1836 isert_info("conn %p PI offload enabled\n", isert_conn); 1837 isert_conn->pi_support = true; 1838 return TARGET_PROT_ALL; 1839 } 1840 } 1841 1842 isert_info("conn %p PI offload disabled\n", isert_conn); 1843 isert_conn->pi_support = false; 1844 1845 return TARGET_PROT_NORMAL; 1846} 1847 1848static int 1849isert_put_nopin(struct iscsit_cmd *cmd, struct iscsit_conn *conn, 1850 bool nopout_response) 1851{ 1852 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1853 struct isert_conn *isert_conn = conn->context; 1854 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1855 1856 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1857 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1858 &isert_cmd->tx_desc.iscsi_header, 1859 nopout_response); 1860 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1861 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1862 1863 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1864 1865 return isert_post_response(isert_conn, isert_cmd); 1866} 1867 1868static int 1869isert_put_logout_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1870{ 1871 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1872 struct isert_conn *isert_conn = conn->context; 1873 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1874 1875 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1876 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1877 &isert_cmd->tx_desc.iscsi_header); 1878 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1879 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1880 1881 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1882 1883 return isert_post_response(isert_conn, isert_cmd); 1884} 1885 1886static int 1887isert_put_tm_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1888{ 1889 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1890 struct isert_conn *isert_conn = conn->context; 1891 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1892 1893 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1894 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1895 &isert_cmd->tx_desc.iscsi_header); 1896 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1897 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1898 1899 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1900 1901 return isert_post_response(isert_conn, isert_cmd); 1902} 1903 1904static int 1905isert_put_reject(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1906{ 1907 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1908 struct isert_conn *isert_conn = conn->context; 1909 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1910 struct isert_device *device = isert_conn->device; 1911 struct ib_device *ib_dev = device->ib_device; 1912 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1913 struct iscsi_reject *hdr = 1914 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1915 1916 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1917 iscsit_build_reject(cmd, conn, hdr); 1918 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1919 1920 hton24(hdr->dlength, ISCSI_HDR_LEN); 1921 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1922 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1923 DMA_TO_DEVICE); 1924 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1925 return -ENOMEM; 1926 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1927 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1928 tx_dsg->length = ISCSI_HDR_LEN; 1929 tx_dsg->lkey = device->pd->local_dma_lkey; 1930 isert_cmd->tx_desc.num_sge = 2; 1931 1932 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1933 1934 isert_dbg("conn %p Posting Reject\n", isert_conn); 1935 1936 return isert_post_response(isert_conn, isert_cmd); 1937} 1938 1939static int 1940isert_put_text_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1941{ 1942 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1943 struct isert_conn *isert_conn = conn->context; 1944 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1945 struct iscsi_text_rsp *hdr = 1946 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1947 u32 txt_rsp_len; 1948 int rc; 1949 1950 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1951 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 1952 if (rc < 0) 1953 return rc; 1954 1955 txt_rsp_len = rc; 1956 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1957 1958 if (txt_rsp_len) { 1959 struct isert_device *device = isert_conn->device; 1960 struct ib_device *ib_dev = device->ib_device; 1961 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1962 void *txt_rsp_buf = cmd->buf_ptr; 1963 1964 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1965 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1966 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1967 return -ENOMEM; 1968 1969 isert_cmd->pdu_buf_len = txt_rsp_len; 1970 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1971 tx_dsg->length = txt_rsp_len; 1972 tx_dsg->lkey = device->pd->local_dma_lkey; 1973 isert_cmd->tx_desc.num_sge = 2; 1974 } 1975 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1976 1977 isert_dbg("conn %p Text Response\n", isert_conn); 1978 1979 return isert_post_response(isert_conn, isert_cmd); 1980} 1981 1982static inline void 1983isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 1984{ 1985 domain->sig_type = IB_SIG_TYPE_T10_DIF; 1986 domain->sig.dif.bg_type = IB_T10DIF_CRC; 1987 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 1988 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 1989 /* 1990 * At the moment we hard code those, but if in the future 1991 * the target core would like to use it, we will take it 1992 * from se_cmd. 1993 */ 1994 domain->sig.dif.apptag_check_mask = 0xffff; 1995 domain->sig.dif.app_escape = true; 1996 domain->sig.dif.ref_escape = true; 1997 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 1998 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 1999 domain->sig.dif.ref_remap = true; 2000} 2001 2002static int 2003isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2004{ 2005 memset(sig_attrs, 0, sizeof(*sig_attrs)); 2006 2007 switch (se_cmd->prot_op) { 2008 case TARGET_PROT_DIN_INSERT: 2009 case TARGET_PROT_DOUT_STRIP: 2010 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2011 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2012 break; 2013 case TARGET_PROT_DOUT_INSERT: 2014 case TARGET_PROT_DIN_STRIP: 2015 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2016 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2017 break; 2018 case TARGET_PROT_DIN_PASS: 2019 case TARGET_PROT_DOUT_PASS: 2020 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2021 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2022 break; 2023 default: 2024 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2025 return -EINVAL; 2026 } 2027 2028 if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 2029 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 2030 if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 2031 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 2032 if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 2033 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 2034 2035 return 0; 2036} 2037 2038static int 2039isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 2040 struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 2041{ 2042 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 2043 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 2044 u8 port_num = conn->cm_id->port_num; 2045 u64 addr; 2046 u32 rkey, offset; 2047 int ret; 2048 2049 if (cmd->ctx_init_done) 2050 goto rdma_ctx_post; 2051 2052 if (dir == DMA_FROM_DEVICE) { 2053 addr = cmd->write_va; 2054 rkey = cmd->write_stag; 2055 offset = cmd->iscsit_cmd->write_data_done; 2056 } else { 2057 addr = cmd->read_va; 2058 rkey = cmd->read_stag; 2059 offset = 0; 2060 } 2061 2062 if (isert_prot_cmd(conn, se_cmd)) { 2063 struct ib_sig_attrs sig_attrs; 2064 2065 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2066 if (ret) 2067 return ret; 2068 2069 WARN_ON_ONCE(offset); 2070 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 2071 se_cmd->t_data_sg, se_cmd->t_data_nents, 2072 se_cmd->t_prot_sg, se_cmd->t_prot_nents, 2073 &sig_attrs, addr, rkey, dir); 2074 } else { 2075 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 2076 se_cmd->t_data_sg, se_cmd->t_data_nents, 2077 offset, addr, rkey, dir); 2078 } 2079 2080 if (ret < 0) { 2081 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 2082 return ret; 2083 } 2084 2085 cmd->ctx_init_done = true; 2086 2087rdma_ctx_post: 2088 ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 2089 if (ret < 0) 2090 isert_err("Cmd: %p failed to post RDMA res\n", cmd); 2091 return ret; 2092} 2093 2094static int 2095isert_put_datain(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 2096{ 2097 struct se_cmd *se_cmd = &cmd->se_cmd; 2098 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2099 struct isert_conn *isert_conn = conn->context; 2100 struct ib_cqe *cqe = NULL; 2101 struct ib_send_wr *chain_wr = NULL; 2102 int rc; 2103 2104 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2105 isert_cmd, se_cmd->data_length); 2106 2107 if (isert_prot_cmd(isert_conn, se_cmd)) { 2108 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 2109 cqe = &isert_cmd->tx_desc.tx_cqe; 2110 } else { 2111 /* 2112 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2113 */ 2114 isert_create_send_desc(isert_conn, isert_cmd, 2115 &isert_cmd->tx_desc); 2116 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2117 &isert_cmd->tx_desc.iscsi_header); 2118 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2119 isert_init_send_wr(isert_conn, isert_cmd, 2120 &isert_cmd->tx_desc.send_wr); 2121 2122 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2123 if (rc) 2124 return rc; 2125 2126 chain_wr = &isert_cmd->tx_desc.send_wr; 2127 } 2128 2129 rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2130 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2131 isert_cmd, rc); 2132 return rc; 2133} 2134 2135static int 2136isert_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, bool recovery) 2137{ 2138 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2139 int ret; 2140 2141 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2142 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2143 2144 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2145 ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 2146 &isert_cmd->tx_desc.tx_cqe, NULL); 2147 2148 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2149 isert_cmd, ret); 2150 return ret; 2151} 2152 2153static int 2154isert_immediate_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2155{ 2156 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2157 int ret = 0; 2158 2159 switch (state) { 2160 case ISTATE_REMOVE: 2161 spin_lock_bh(&conn->cmd_lock); 2162 list_del_init(&cmd->i_conn_node); 2163 spin_unlock_bh(&conn->cmd_lock); 2164 isert_put_cmd(isert_cmd, true); 2165 break; 2166 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2167 ret = isert_put_nopin(cmd, conn, false); 2168 break; 2169 default: 2170 isert_err("Unknown immediate state: 0x%02x\n", state); 2171 ret = -EINVAL; 2172 break; 2173 } 2174 2175 return ret; 2176} 2177 2178static int 2179isert_response_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2180{ 2181 struct isert_conn *isert_conn = conn->context; 2182 int ret; 2183 2184 switch (state) { 2185 case ISTATE_SEND_LOGOUTRSP: 2186 ret = isert_put_logout_rsp(cmd, conn); 2187 if (!ret) 2188 isert_conn->logout_posted = true; 2189 break; 2190 case ISTATE_SEND_NOPIN: 2191 ret = isert_put_nopin(cmd, conn, true); 2192 break; 2193 case ISTATE_SEND_TASKMGTRSP: 2194 ret = isert_put_tm_rsp(cmd, conn); 2195 break; 2196 case ISTATE_SEND_REJECT: 2197 ret = isert_put_reject(cmd, conn); 2198 break; 2199 case ISTATE_SEND_TEXTRSP: 2200 ret = isert_put_text_rsp(cmd, conn); 2201 break; 2202 case ISTATE_SEND_STATUS: 2203 /* 2204 * Special case for sending non GOOD SCSI status from TX thread 2205 * context during pre se_cmd excecution failure. 2206 */ 2207 ret = isert_put_response(conn, cmd); 2208 break; 2209 default: 2210 isert_err("Unknown response state: 0x%02x\n", state); 2211 ret = -EINVAL; 2212 break; 2213 } 2214 2215 return ret; 2216} 2217 2218struct rdma_cm_id * 2219isert_setup_id(struct isert_np *isert_np) 2220{ 2221 struct iscsi_np *np = isert_np->np; 2222 struct rdma_cm_id *id; 2223 struct sockaddr *sa; 2224 int ret; 2225 2226 sa = (struct sockaddr *)&np->np_sockaddr; 2227 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2228 2229 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2230 RDMA_PS_TCP, IB_QPT_RC); 2231 if (IS_ERR(id)) { 2232 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2233 ret = PTR_ERR(id); 2234 goto out; 2235 } 2236 isert_dbg("id %p context %p\n", id, id->context); 2237 2238 /* 2239 * Allow both IPv4 and IPv6 sockets to bind a single port 2240 * at the same time. 2241 */ 2242 ret = rdma_set_afonly(id, 1); 2243 if (ret) { 2244 isert_err("rdma_set_afonly() failed: %d\n", ret); 2245 goto out_id; 2246 } 2247 2248 ret = rdma_bind_addr(id, sa); 2249 if (ret) { 2250 isert_err("rdma_bind_addr() failed: %d\n", ret); 2251 goto out_id; 2252 } 2253 2254 ret = rdma_listen(id, 0); 2255 if (ret) { 2256 isert_err("rdma_listen() failed: %d\n", ret); 2257 goto out_id; 2258 } 2259 2260 return id; 2261out_id: 2262 rdma_destroy_id(id); 2263out: 2264 return ERR_PTR(ret); 2265} 2266 2267static int 2268isert_setup_np(struct iscsi_np *np, 2269 struct sockaddr_storage *ksockaddr) 2270{ 2271 struct isert_np *isert_np; 2272 struct rdma_cm_id *isert_lid; 2273 int ret; 2274 2275 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2276 if (!isert_np) 2277 return -ENOMEM; 2278 2279 sema_init(&isert_np->sem, 0); 2280 mutex_init(&isert_np->mutex); 2281 INIT_LIST_HEAD(&isert_np->accepted); 2282 INIT_LIST_HEAD(&isert_np->pending); 2283 isert_np->np = np; 2284 2285 /* 2286 * Setup the np->np_sockaddr from the passed sockaddr setup 2287 * in iscsi_target_configfs.c code.. 2288 */ 2289 memcpy(&np->np_sockaddr, ksockaddr, 2290 sizeof(struct sockaddr_storage)); 2291 2292 isert_lid = isert_setup_id(isert_np); 2293 if (IS_ERR(isert_lid)) { 2294 ret = PTR_ERR(isert_lid); 2295 goto out; 2296 } 2297 2298 isert_np->cm_id = isert_lid; 2299 np->np_context = isert_np; 2300 2301 return 0; 2302 2303out: 2304 kfree(isert_np); 2305 2306 return ret; 2307} 2308 2309static int 2310isert_rdma_accept(struct isert_conn *isert_conn) 2311{ 2312 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2313 struct rdma_conn_param cp; 2314 int ret; 2315 struct iser_cm_hdr rsp_hdr; 2316 2317 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2318 cp.initiator_depth = isert_conn->initiator_depth; 2319 cp.retry_count = 7; 2320 cp.rnr_retry_count = 7; 2321 2322 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2323 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2324 if (!isert_conn->snd_w_inv) 2325 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 2326 cp.private_data = (void *)&rsp_hdr; 2327 cp.private_data_len = sizeof(rsp_hdr); 2328 2329 ret = rdma_accept(cm_id, &cp); 2330 if (ret) { 2331 isert_err("rdma_accept() failed with: %d\n", ret); 2332 return ret; 2333 } 2334 2335 return 0; 2336} 2337 2338static int 2339isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login) 2340{ 2341 struct isert_conn *isert_conn = conn->context; 2342 int ret; 2343 2344 isert_info("before login_req comp conn: %p\n", isert_conn); 2345 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 2346 if (ret) { 2347 isert_err("isert_conn %p interrupted before got login req\n", 2348 isert_conn); 2349 return ret; 2350 } 2351 reinit_completion(&isert_conn->login_req_comp); 2352 2353 /* 2354 * For login requests after the first PDU, isert_rx_login_req() will 2355 * kick queue_delayed_work(isert_login_wq, &conn->login_work) as 2356 * the packet is received, which turns this callback from 2357 * iscsi_target_do_login_rx() into a NOP. 2358 */ 2359 if (!login->first_request) 2360 return 0; 2361 2362 isert_rx_login_req(isert_conn); 2363 2364 isert_info("before login_comp conn: %p\n", conn); 2365 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2366 if (ret) 2367 return ret; 2368 2369 isert_info("processing login->req: %p\n", login->req); 2370 2371 return 0; 2372} 2373 2374static void 2375isert_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn, 2376 struct isert_conn *isert_conn) 2377{ 2378 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2379 struct rdma_route *cm_route = &cm_id->route; 2380 2381 conn->login_family = np->np_sockaddr.ss_family; 2382 2383 conn->login_sockaddr = cm_route->addr.dst_addr; 2384 conn->local_sockaddr = cm_route->addr.src_addr; 2385} 2386 2387static int 2388isert_accept_np(struct iscsi_np *np, struct iscsit_conn *conn) 2389{ 2390 struct isert_np *isert_np = np->np_context; 2391 struct isert_conn *isert_conn; 2392 int ret; 2393 2394accept_wait: 2395 ret = down_interruptible(&isert_np->sem); 2396 if (ret) 2397 return -ENODEV; 2398 2399 spin_lock_bh(&np->np_thread_lock); 2400 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2401 spin_unlock_bh(&np->np_thread_lock); 2402 isert_dbg("np_thread_state %d\n", 2403 np->np_thread_state); 2404 /* 2405 * No point in stalling here when np_thread 2406 * is in state RESET/SHUTDOWN/EXIT - bail 2407 */ 2408 return -ENODEV; 2409 } 2410 spin_unlock_bh(&np->np_thread_lock); 2411 2412 mutex_lock(&isert_np->mutex); 2413 if (list_empty(&isert_np->pending)) { 2414 mutex_unlock(&isert_np->mutex); 2415 goto accept_wait; 2416 } 2417 isert_conn = list_first_entry(&isert_np->pending, 2418 struct isert_conn, node); 2419 list_del_init(&isert_conn->node); 2420 mutex_unlock(&isert_np->mutex); 2421 2422 conn->context = isert_conn; 2423 isert_conn->conn = conn; 2424 isert_conn->state = ISER_CONN_BOUND; 2425 2426 isert_set_conn_info(np, conn, isert_conn); 2427 2428 isert_dbg("Processing isert_conn: %p\n", isert_conn); 2429 2430 return 0; 2431} 2432 2433static void 2434isert_free_np(struct iscsi_np *np) 2435{ 2436 struct isert_np *isert_np = np->np_context; 2437 struct isert_conn *isert_conn, *n; 2438 LIST_HEAD(drop_conn_list); 2439 2440 if (isert_np->cm_id) 2441 rdma_destroy_id(isert_np->cm_id); 2442 2443 /* 2444 * FIXME: At this point we don't have a good way to insure 2445 * that at this point we don't have hanging connections that 2446 * completed RDMA establishment but didn't start iscsi login 2447 * process. So work-around this by cleaning up what ever piled 2448 * up in accepted and pending lists. 2449 */ 2450 mutex_lock(&isert_np->mutex); 2451 if (!list_empty(&isert_np->pending)) { 2452 isert_info("Still have isert pending connections\n"); 2453 list_for_each_entry_safe(isert_conn, n, 2454 &isert_np->pending, 2455 node) { 2456 isert_info("cleaning isert_conn %p state (%d)\n", 2457 isert_conn, isert_conn->state); 2458 list_move_tail(&isert_conn->node, &drop_conn_list); 2459 } 2460 } 2461 2462 if (!list_empty(&isert_np->accepted)) { 2463 isert_info("Still have isert accepted connections\n"); 2464 list_for_each_entry_safe(isert_conn, n, 2465 &isert_np->accepted, 2466 node) { 2467 isert_info("cleaning isert_conn %p state (%d)\n", 2468 isert_conn, isert_conn->state); 2469 list_move_tail(&isert_conn->node, &drop_conn_list); 2470 } 2471 } 2472 mutex_unlock(&isert_np->mutex); 2473 2474 list_for_each_entry_safe(isert_conn, n, &drop_conn_list, node) { 2475 list_del_init(&isert_conn->node); 2476 isert_connect_release(isert_conn); 2477 } 2478 2479 np->np_context = NULL; 2480 kfree(isert_np); 2481} 2482 2483static void isert_release_work(struct work_struct *work) 2484{ 2485 struct isert_conn *isert_conn = container_of(work, 2486 struct isert_conn, 2487 release_work); 2488 2489 isert_info("Starting release conn %p\n", isert_conn); 2490 2491 mutex_lock(&isert_conn->mutex); 2492 isert_conn->state = ISER_CONN_DOWN; 2493 mutex_unlock(&isert_conn->mutex); 2494 2495 isert_info("Destroying conn %p\n", isert_conn); 2496 isert_put_conn(isert_conn); 2497} 2498 2499static void 2500isert_wait4logout(struct isert_conn *isert_conn) 2501{ 2502 struct iscsit_conn *conn = isert_conn->conn; 2503 2504 isert_info("conn %p\n", isert_conn); 2505 2506 if (isert_conn->logout_posted) { 2507 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2508 wait_for_completion_timeout(&conn->conn_logout_comp, 2509 SECONDS_FOR_LOGOUT_COMP * HZ); 2510 } 2511} 2512 2513static void 2514isert_wait4cmds(struct iscsit_conn *conn) 2515{ 2516 isert_info("iscsit_conn %p\n", conn); 2517 2518 if (conn->sess) { 2519 target_stop_cmd_counter(conn->cmd_cnt); 2520 target_wait_for_cmds(conn->cmd_cnt); 2521 } 2522} 2523 2524/** 2525 * isert_put_unsol_pending_cmds() - Drop commands waiting for 2526 * unsolicitate dataout 2527 * @conn: iscsi connection 2528 * 2529 * We might still have commands that are waiting for unsolicited 2530 * dataouts messages. We must put the extra reference on those 2531 * before blocking on the target_wait_for_session_cmds 2532 */ 2533static void 2534isert_put_unsol_pending_cmds(struct iscsit_conn *conn) 2535{ 2536 struct iscsit_cmd *cmd, *tmp; 2537 static LIST_HEAD(drop_cmd_list); 2538 2539 spin_lock_bh(&conn->cmd_lock); 2540 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 2541 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 2542 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 2543 (cmd->write_data_done < cmd->se_cmd.data_length)) 2544 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 2545 } 2546 spin_unlock_bh(&conn->cmd_lock); 2547 2548 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 2549 list_del_init(&cmd->i_conn_node); 2550 if (cmd->i_state != ISTATE_REMOVE) { 2551 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2552 2553 isert_info("conn %p dropping cmd %p\n", conn, cmd); 2554 isert_put_cmd(isert_cmd, true); 2555 } 2556 } 2557} 2558 2559static void isert_wait_conn(struct iscsit_conn *conn) 2560{ 2561 struct isert_conn *isert_conn = conn->context; 2562 2563 isert_info("Starting conn %p\n", isert_conn); 2564 2565 mutex_lock(&isert_conn->mutex); 2566 isert_conn_terminate(isert_conn); 2567 mutex_unlock(&isert_conn->mutex); 2568 2569 ib_drain_qp(isert_conn->qp); 2570 isert_put_unsol_pending_cmds(conn); 2571 isert_wait4cmds(conn); 2572 isert_wait4logout(isert_conn); 2573 2574 queue_work(isert_release_wq, &isert_conn->release_work); 2575} 2576 2577static void isert_free_conn(struct iscsit_conn *conn) 2578{ 2579 struct isert_conn *isert_conn = conn->context; 2580 2581 ib_drain_qp(isert_conn->qp); 2582 isert_put_conn(isert_conn); 2583} 2584 2585static void isert_get_rx_pdu(struct iscsit_conn *conn) 2586{ 2587 struct completion comp; 2588 2589 init_completion(&comp); 2590 2591 wait_for_completion_interruptible(&comp); 2592} 2593 2594static struct iscsit_transport iser_target_transport = { 2595 .name = "IB/iSER", 2596 .transport_type = ISCSI_INFINIBAND, 2597 .rdma_shutdown = true, 2598 .priv_size = sizeof(struct isert_cmd), 2599 .owner = THIS_MODULE, 2600 .iscsit_setup_np = isert_setup_np, 2601 .iscsit_accept_np = isert_accept_np, 2602 .iscsit_free_np = isert_free_np, 2603 .iscsit_wait_conn = isert_wait_conn, 2604 .iscsit_free_conn = isert_free_conn, 2605 .iscsit_get_login_rx = isert_get_login_rx, 2606 .iscsit_put_login_tx = isert_put_login_tx, 2607 .iscsit_immediate_queue = isert_immediate_queue, 2608 .iscsit_response_queue = isert_response_queue, 2609 .iscsit_get_dataout = isert_get_dataout, 2610 .iscsit_queue_data_in = isert_put_datain, 2611 .iscsit_queue_status = isert_put_response, 2612 .iscsit_aborted_task = isert_aborted_task, 2613 .iscsit_get_rx_pdu = isert_get_rx_pdu, 2614 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2615}; 2616 2617static int __init isert_init(void) 2618{ 2619 isert_login_wq = alloc_workqueue("isert_login_wq", 0, 0); 2620 if (!isert_login_wq) { 2621 isert_err("Unable to allocate isert_login_wq\n"); 2622 return -ENOMEM; 2623 } 2624 2625 isert_comp_wq = alloc_workqueue("isert_comp_wq", 2626 WQ_UNBOUND | WQ_HIGHPRI, 0); 2627 if (!isert_comp_wq) { 2628 isert_err("Unable to allocate isert_comp_wq\n"); 2629 goto destroy_login_wq; 2630 } 2631 2632 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2633 WQ_UNBOUND_MAX_ACTIVE); 2634 if (!isert_release_wq) { 2635 isert_err("Unable to allocate isert_release_wq\n"); 2636 goto destroy_comp_wq; 2637 } 2638 2639 iscsit_register_transport(&iser_target_transport); 2640 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2641 2642 return 0; 2643 2644destroy_comp_wq: 2645 destroy_workqueue(isert_comp_wq); 2646destroy_login_wq: 2647 destroy_workqueue(isert_login_wq); 2648 2649 return -ENOMEM; 2650} 2651 2652static void __exit isert_exit(void) 2653{ 2654 flush_workqueue(isert_login_wq); 2655 destroy_workqueue(isert_release_wq); 2656 destroy_workqueue(isert_comp_wq); 2657 iscsit_unregister_transport(&iser_target_transport); 2658 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2659 destroy_workqueue(isert_login_wq); 2660} 2661 2662MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2663MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2664MODULE_LICENSE("GPL"); 2665 2666module_init(isert_init); 2667module_exit(isert_exit); 2668