1/* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 * $Id: iser_initiator.c,v 1.1.1.1 2007/08/03 18:52:32 Exp $ 33 */ 34#include <linux/kernel.h> 35#include <linux/slab.h> 36#include <linux/mm.h> 37#include <asm/io.h> 38#include <asm/scatterlist.h> 39#include <linux/scatterlist.h> 40#include <linux/kfifo.h> 41#include <scsi/scsi_cmnd.h> 42#include <scsi/scsi_host.h> 43 44#include "iscsi_iser.h" 45 46/* Constant PDU lengths calculations */ 47#define ISER_TOTAL_HEADERS_LEN (sizeof (struct iser_hdr) + \ 48 sizeof (struct iscsi_hdr)) 49 50/* iser_dto_add_regd_buff - increments the reference count for * 51 * the registered buffer & adds it to the DTO object */ 52static void iser_dto_add_regd_buff(struct iser_dto *dto, 53 struct iser_regd_buf *regd_buf, 54 unsigned long use_offset, 55 unsigned long use_size) 56{ 57 int add_idx; 58 59 atomic_inc(®d_buf->ref_count); 60 61 add_idx = dto->regd_vector_len; 62 dto->regd[add_idx] = regd_buf; 63 dto->used_sz[add_idx] = use_size; 64 dto->offset[add_idx] = use_offset; 65 66 dto->regd_vector_len++; 67} 68 69/* Register user buffer memory and initialize passive rdma 70 * dto descriptor. Total data size is stored in 71 * iser_ctask->data[ISER_DIR_IN].data_len 72 */ 73static int iser_prepare_read_cmd(struct iscsi_cmd_task *ctask, 74 unsigned int edtl) 75 76{ 77 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 78 struct iser_regd_buf *regd_buf; 79 int err; 80 struct iser_hdr *hdr = &iser_ctask->desc.iser_header; 81 struct iser_data_buf *buf_in = &iser_ctask->data[ISER_DIR_IN]; 82 83 err = iser_dma_map_task_data(iser_ctask, 84 buf_in, 85 ISER_DIR_IN, 86 DMA_FROM_DEVICE); 87 if (err) 88 return err; 89 90 if (edtl > iser_ctask->data[ISER_DIR_IN].data_len) { 91 iser_err("Total data length: %ld, less than EDTL: " 92 "%d, in READ cmd BHS itt: %d, conn: 0x%p\n", 93 iser_ctask->data[ISER_DIR_IN].data_len, edtl, 94 ctask->itt, iser_ctask->iser_conn); 95 return -EINVAL; 96 } 97 98 err = iser_reg_rdma_mem(iser_ctask,ISER_DIR_IN); 99 if (err) { 100 iser_err("Failed to set up Data-IN RDMA\n"); 101 return err; 102 } 103 regd_buf = &iser_ctask->rdma_regd[ISER_DIR_IN]; 104 105 hdr->flags |= ISER_RSV; 106 hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey); 107 hdr->read_va = cpu_to_be64(regd_buf->reg.va); 108 109 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n", 110 ctask->itt, regd_buf->reg.rkey, 111 (unsigned long long)regd_buf->reg.va); 112 113 return 0; 114} 115 116/* Register user buffer memory and initialize passive rdma 117 * dto descriptor. Total data size is stored in 118 * ctask->data[ISER_DIR_OUT].data_len 119 */ 120static int 121iser_prepare_write_cmd(struct iscsi_cmd_task *ctask, 122 unsigned int imm_sz, 123 unsigned int unsol_sz, 124 unsigned int edtl) 125{ 126 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 127 struct iser_regd_buf *regd_buf; 128 int err; 129 struct iser_dto *send_dto = &iser_ctask->desc.dto; 130 struct iser_hdr *hdr = &iser_ctask->desc.iser_header; 131 struct iser_data_buf *buf_out = &iser_ctask->data[ISER_DIR_OUT]; 132 133 err = iser_dma_map_task_data(iser_ctask, 134 buf_out, 135 ISER_DIR_OUT, 136 DMA_TO_DEVICE); 137 if (err) 138 return err; 139 140 if (edtl > iser_ctask->data[ISER_DIR_OUT].data_len) { 141 iser_err("Total data length: %ld, less than EDTL: %d, " 142 "in WRITE cmd BHS itt: %d, conn: 0x%p\n", 143 iser_ctask->data[ISER_DIR_OUT].data_len, 144 edtl, ctask->itt, ctask->conn); 145 return -EINVAL; 146 } 147 148 err = iser_reg_rdma_mem(iser_ctask,ISER_DIR_OUT); 149 if (err != 0) { 150 iser_err("Failed to register write cmd RDMA mem\n"); 151 return err; 152 } 153 154 regd_buf = &iser_ctask->rdma_regd[ISER_DIR_OUT]; 155 156 if (unsol_sz < edtl) { 157 hdr->flags |= ISER_WSV; 158 hdr->write_stag = cpu_to_be32(regd_buf->reg.rkey); 159 hdr->write_va = cpu_to_be64(regd_buf->reg.va + unsol_sz); 160 161 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X " 162 "VA:%#llX + unsol:%d\n", 163 ctask->itt, regd_buf->reg.rkey, 164 (unsigned long long)regd_buf->reg.va, unsol_sz); 165 } 166 167 if (imm_sz > 0) { 168 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", 169 ctask->itt, imm_sz); 170 iser_dto_add_regd_buff(send_dto, 171 regd_buf, 172 0, 173 imm_sz); 174 } 175 176 return 0; 177} 178 179/** 180 * iser_post_receive_control - allocates, initializes and posts receive DTO. 181 */ 182static int iser_post_receive_control(struct iscsi_conn *conn) 183{ 184 struct iscsi_iser_conn *iser_conn = conn->dd_data; 185 struct iser_desc *rx_desc; 186 struct iser_regd_buf *regd_hdr; 187 struct iser_regd_buf *regd_data; 188 struct iser_dto *recv_dto = NULL; 189 struct iser_device *device = iser_conn->ib_conn->device; 190 int rx_data_size, err = 0; 191 192 rx_desc = kmem_cache_alloc(ig.desc_cache, GFP_NOIO); 193 if (rx_desc == NULL) { 194 iser_err("Failed to alloc desc for post recv\n"); 195 return -ENOMEM; 196 } 197 rx_desc->type = ISCSI_RX; 198 199 /* for the login sequence we must support rx of upto 8K; login is done 200 * after conn create/bind (connect) and conn stop/bind (reconnect), 201 * what's common for both schemes is that the connection is not started 202 */ 203 if (conn->c_stage != ISCSI_CONN_STARTED) 204 rx_data_size = ISCSI_DEF_MAX_RECV_SEG_LEN; 205 else 206 rx_data_size = 128; 207 208 rx_desc->data = kmalloc(rx_data_size, GFP_NOIO); 209 if (rx_desc->data == NULL) { 210 iser_err("Failed to alloc data buf for post recv\n"); 211 err = -ENOMEM; 212 goto post_rx_kmalloc_failure; 213 } 214 215 recv_dto = &rx_desc->dto; 216 recv_dto->ib_conn = iser_conn->ib_conn; 217 recv_dto->regd_vector_len = 0; 218 219 regd_hdr = &rx_desc->hdr_regd_buf; 220 memset(regd_hdr, 0, sizeof(struct iser_regd_buf)); 221 regd_hdr->device = device; 222 regd_hdr->virt_addr = rx_desc; /* == &rx_desc->iser_header */ 223 regd_hdr->data_size = ISER_TOTAL_HEADERS_LEN; 224 225 iser_reg_single(device, regd_hdr, DMA_FROM_DEVICE); 226 227 iser_dto_add_regd_buff(recv_dto, regd_hdr, 0, 0); 228 229 regd_data = &rx_desc->data_regd_buf; 230 memset(regd_data, 0, sizeof(struct iser_regd_buf)); 231 regd_data->device = device; 232 regd_data->virt_addr = rx_desc->data; 233 regd_data->data_size = rx_data_size; 234 235 iser_reg_single(device, regd_data, DMA_FROM_DEVICE); 236 237 iser_dto_add_regd_buff(recv_dto, regd_data, 0, 0); 238 239 err = iser_post_recv(rx_desc); 240 if (!err) 241 return 0; 242 243 /* iser_post_recv failed */ 244 iser_dto_buffs_release(recv_dto); 245 kfree(rx_desc->data); 246post_rx_kmalloc_failure: 247 kmem_cache_free(ig.desc_cache, rx_desc); 248 return err; 249} 250 251/* creates a new tx descriptor and adds header regd buffer */ 252static void iser_create_send_desc(struct iscsi_iser_conn *iser_conn, 253 struct iser_desc *tx_desc) 254{ 255 struct iser_regd_buf *regd_hdr = &tx_desc->hdr_regd_buf; 256 struct iser_dto *send_dto = &tx_desc->dto; 257 258 memset(regd_hdr, 0, sizeof(struct iser_regd_buf)); 259 regd_hdr->device = iser_conn->ib_conn->device; 260 regd_hdr->virt_addr = tx_desc; /* == &tx_desc->iser_header */ 261 regd_hdr->data_size = ISER_TOTAL_HEADERS_LEN; 262 263 send_dto->ib_conn = iser_conn->ib_conn; 264 send_dto->notify_enable = 1; 265 send_dto->regd_vector_len = 0; 266 267 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 268 tx_desc->iser_header.flags = ISER_VER; 269 270 iser_dto_add_regd_buff(send_dto, regd_hdr, 0, 0); 271} 272 273/** 274 * iser_conn_set_full_featured_mode - (iSER API) 275 */ 276int iser_conn_set_full_featured_mode(struct iscsi_conn *conn) 277{ 278 struct iscsi_iser_conn *iser_conn = conn->dd_data; 279 280 int i; 281 /* no need to keep it in a var, we are after login so if this should 282 * be negotiated, by now the result should be available here */ 283 int initial_post_recv_bufs_num = ISER_MAX_RX_MISC_PDUS; 284 285 iser_dbg("Initially post: %d\n", initial_post_recv_bufs_num); 286 287 /* Check that there is no posted recv or send buffers left - */ 288 /* they must be consumed during the login phase */ 289 BUG_ON(atomic_read(&iser_conn->ib_conn->post_recv_buf_count) != 0); 290 BUG_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0); 291 292 /* Initial post receive buffers */ 293 for (i = 0; i < initial_post_recv_bufs_num; i++) { 294 if (iser_post_receive_control(conn) != 0) { 295 iser_err("Failed to post recv bufs at:%d conn:0x%p\n", 296 i, conn); 297 return -ENOMEM; 298 } 299 } 300 iser_dbg("Posted %d post recv bufs, conn:0x%p\n", i, conn); 301 return 0; 302} 303 304static int 305iser_check_xmit(struct iscsi_conn *conn, void *task) 306{ 307 struct iscsi_iser_conn *iser_conn = conn->dd_data; 308 309 if (atomic_read(&iser_conn->ib_conn->post_send_buf_count) == 310 ISER_QP_MAX_REQ_DTOS) { 311 iser_dbg("%ld can't xmit task %p\n",jiffies,task); 312 return -ENOBUFS; 313 } 314 return 0; 315} 316 317 318/** 319 * iser_send_command - send command PDU 320 */ 321int iser_send_command(struct iscsi_conn *conn, 322 struct iscsi_cmd_task *ctask) 323{ 324 struct iscsi_iser_conn *iser_conn = conn->dd_data; 325 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 326 struct iser_dto *send_dto = NULL; 327 unsigned long edtl; 328 int err = 0; 329 struct iser_data_buf *data_buf; 330 331 struct iscsi_cmd *hdr = ctask->hdr; 332 struct scsi_cmnd *sc = ctask->sc; 333 334 if (!iser_conn_state_comp(iser_conn->ib_conn, ISER_CONN_UP)) { 335 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn->ib_conn); 336 return -EPERM; 337 } 338 if (iser_check_xmit(conn, ctask)) 339 return -ENOBUFS; 340 341 edtl = ntohl(hdr->data_length); 342 343 /* build the tx desc regd header and add it to the tx desc dto */ 344 iser_ctask->desc.type = ISCSI_TX_SCSI_COMMAND; 345 send_dto = &iser_ctask->desc.dto; 346 send_dto->ctask = iser_ctask; 347 iser_create_send_desc(iser_conn, &iser_ctask->desc); 348 349 if (hdr->flags & ISCSI_FLAG_CMD_READ) 350 data_buf = &iser_ctask->data[ISER_DIR_IN]; 351 else 352 data_buf = &iser_ctask->data[ISER_DIR_OUT]; 353 354 if (sc->use_sg) { /* using a scatter list */ 355 data_buf->buf = sc->request_buffer; 356 data_buf->size = sc->use_sg; 357 } else if (sc->request_bufflen) { 358 /* using a single buffer - convert it into one entry SG */ 359 sg_init_one(&data_buf->sg_single, 360 sc->request_buffer, sc->request_bufflen); 361 data_buf->buf = &data_buf->sg_single; 362 data_buf->size = 1; 363 } 364 365 data_buf->data_len = sc->request_bufflen; 366 367 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 368 err = iser_prepare_read_cmd(ctask, edtl); 369 if (err) 370 goto send_command_error; 371 } 372 if (hdr->flags & ISCSI_FLAG_CMD_WRITE) { 373 err = iser_prepare_write_cmd(ctask, 374 ctask->imm_count, 375 ctask->imm_count + 376 ctask->unsol_count, 377 edtl); 378 if (err) 379 goto send_command_error; 380 } 381 382 iser_reg_single(iser_conn->ib_conn->device, 383 send_dto->regd[0], DMA_TO_DEVICE); 384 385 if (iser_post_receive_control(conn) != 0) { 386 iser_err("post_recv failed!\n"); 387 err = -ENOMEM; 388 goto send_command_error; 389 } 390 391 iser_ctask->status = ISER_TASK_STATUS_STARTED; 392 393 err = iser_post_send(&iser_ctask->desc); 394 if (!err) 395 return 0; 396 397send_command_error: 398 iser_dto_buffs_release(send_dto); 399 iser_err("conn %p failed ctask->itt %d err %d\n",conn, ctask->itt, err); 400 return err; 401} 402 403/** 404 * iser_send_data_out - send data out PDU 405 */ 406int iser_send_data_out(struct iscsi_conn *conn, 407 struct iscsi_cmd_task *ctask, 408 struct iscsi_data *hdr) 409{ 410 struct iscsi_iser_conn *iser_conn = conn->dd_data; 411 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 412 struct iser_desc *tx_desc = NULL; 413 struct iser_dto *send_dto = NULL; 414 unsigned long buf_offset; 415 unsigned long data_seg_len; 416 unsigned int itt; 417 int err = 0; 418 419 if (!iser_conn_state_comp(iser_conn->ib_conn, ISER_CONN_UP)) { 420 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn->ib_conn); 421 return -EPERM; 422 } 423 424 if (iser_check_xmit(conn, ctask)) 425 return -ENOBUFS; 426 427 itt = ntohl(hdr->itt); 428 data_seg_len = ntoh24(hdr->dlength); 429 buf_offset = ntohl(hdr->offset); 430 431 iser_dbg("%s itt %d dseg_len %d offset %d\n", 432 __func__,(int)itt,(int)data_seg_len,(int)buf_offset); 433 434 tx_desc = kmem_cache_alloc(ig.desc_cache, GFP_NOIO); 435 if (tx_desc == NULL) { 436 iser_err("Failed to alloc desc for post dataout\n"); 437 return -ENOMEM; 438 } 439 440 tx_desc->type = ISCSI_TX_DATAOUT; 441 memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); 442 443 /* build the tx desc regd header and add it to the tx desc dto */ 444 send_dto = &tx_desc->dto; 445 send_dto->ctask = iser_ctask; 446 iser_create_send_desc(iser_conn, tx_desc); 447 448 iser_reg_single(iser_conn->ib_conn->device, 449 send_dto->regd[0], DMA_TO_DEVICE); 450 451 /* all data was registered for RDMA, we can use the lkey */ 452 iser_dto_add_regd_buff(send_dto, 453 &iser_ctask->rdma_regd[ISER_DIR_OUT], 454 buf_offset, 455 data_seg_len); 456 457 if (buf_offset + data_seg_len > iser_ctask->data[ISER_DIR_OUT].data_len) { 458 iser_err("Offset:%ld & DSL:%ld in Data-Out " 459 "inconsistent with total len:%ld, itt:%d\n", 460 buf_offset, data_seg_len, 461 iser_ctask->data[ISER_DIR_OUT].data_len, itt); 462 err = -EINVAL; 463 goto send_data_out_error; 464 } 465 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n", 466 itt, buf_offset, data_seg_len); 467 468 469 err = iser_post_send(tx_desc); 470 if (!err) 471 return 0; 472 473send_data_out_error: 474 iser_dto_buffs_release(send_dto); 475 kmem_cache_free(ig.desc_cache, tx_desc); 476 iser_err("conn %p failed err %d\n",conn, err); 477 return err; 478} 479 480int iser_send_control(struct iscsi_conn *conn, 481 struct iscsi_mgmt_task *mtask) 482{ 483 struct iscsi_iser_conn *iser_conn = conn->dd_data; 484 struct iser_desc *mdesc = mtask->dd_data; 485 struct iser_dto *send_dto = NULL; 486 unsigned long data_seg_len; 487 int err = 0; 488 struct iser_regd_buf *regd_buf; 489 struct iser_device *device; 490 491 if (!iser_conn_state_comp(iser_conn->ib_conn, ISER_CONN_UP)) { 492 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn->ib_conn); 493 return -EPERM; 494 } 495 496 if (iser_check_xmit(conn,mtask)) 497 return -ENOBUFS; 498 499 /* build the tx desc regd header and add it to the tx desc dto */ 500 mdesc->type = ISCSI_TX_CONTROL; 501 send_dto = &mdesc->dto; 502 send_dto->ctask = NULL; 503 iser_create_send_desc(iser_conn, mdesc); 504 505 device = iser_conn->ib_conn->device; 506 507 iser_reg_single(device, send_dto->regd[0], DMA_TO_DEVICE); 508 509 data_seg_len = ntoh24(mtask->hdr->dlength); 510 511 if (data_seg_len > 0) { 512 regd_buf = &mdesc->data_regd_buf; 513 memset(regd_buf, 0, sizeof(struct iser_regd_buf)); 514 regd_buf->device = device; 515 regd_buf->virt_addr = mtask->data; 516 regd_buf->data_size = mtask->data_count; 517 iser_reg_single(device, regd_buf, 518 DMA_TO_DEVICE); 519 iser_dto_add_regd_buff(send_dto, regd_buf, 520 0, 521 data_seg_len); 522 } 523 524 if (iser_post_receive_control(conn) != 0) { 525 iser_err("post_rcv_buff failed!\n"); 526 err = -ENOMEM; 527 goto send_control_error; 528 } 529 530 err = iser_post_send(mdesc); 531 if (!err) 532 return 0; 533 534send_control_error: 535 iser_dto_buffs_release(send_dto); 536 iser_err("conn %p failed err %d\n",conn, err); 537 return err; 538} 539 540/** 541 * iser_rcv_dto_completion - recv DTO completion 542 */ 543void iser_rcv_completion(struct iser_desc *rx_desc, 544 unsigned long dto_xfer_len) 545{ 546 struct iser_dto *dto = &rx_desc->dto; 547 struct iscsi_iser_conn *conn = dto->ib_conn->iser_conn; 548 struct iscsi_session *session = conn->iscsi_conn->session; 549 struct iscsi_cmd_task *ctask; 550 struct iscsi_iser_cmd_task *iser_ctask; 551 struct iscsi_hdr *hdr; 552 char *rx_data = NULL; 553 int rx_data_len = 0; 554 unsigned int itt; 555 unsigned char opcode; 556 557 hdr = &rx_desc->iscsi_header; 558 559 iser_dbg("op 0x%x itt 0x%x\n", hdr->opcode,hdr->itt); 560 561 if (dto_xfer_len > ISER_TOTAL_HEADERS_LEN) { /* we have data */ 562 rx_data_len = dto_xfer_len - ISER_TOTAL_HEADERS_LEN; 563 rx_data = dto->regd[1]->virt_addr; 564 rx_data += dto->offset[1]; 565 } 566 567 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 568 569 if (opcode == ISCSI_OP_SCSI_CMD_RSP) { 570 itt = get_itt(hdr->itt); /* mask out cid and age bits */ 571 if (!(itt < session->cmds_max)) 572 iser_err("itt can't be matched to task!!!" 573 "conn %p opcode %d cmds_max %d itt %d\n", 574 conn->iscsi_conn,opcode,session->cmds_max,itt); 575 /* use the mapping given with the cmds array indexed by itt */ 576 ctask = (struct iscsi_cmd_task *)session->cmds[itt]; 577 iser_ctask = ctask->dd_data; 578 iser_dbg("itt %d ctask %p\n",itt,ctask); 579 iser_ctask->status = ISER_TASK_STATUS_COMPLETED; 580 iser_ctask_rdma_finalize(iser_ctask); 581 } 582 583 iser_dto_buffs_release(dto); 584 585 iscsi_iser_recv(conn->iscsi_conn, hdr, rx_data, rx_data_len); 586 587 kfree(rx_desc->data); 588 kmem_cache_free(ig.desc_cache, rx_desc); 589 590 /* decrementing conn->post_recv_buf_count only --after-- freeing the * 591 * task eliminates the need to worry on tasks which are completed in * 592 * parallel to the execution of iser_conn_term. So the code that waits * 593 * for the posted rx bufs refcount to become zero handles everything */ 594 atomic_dec(&conn->ib_conn->post_recv_buf_count); 595} 596 597void iser_snd_completion(struct iser_desc *tx_desc) 598{ 599 struct iser_dto *dto = &tx_desc->dto; 600 struct iser_conn *ib_conn = dto->ib_conn; 601 struct iscsi_iser_conn *iser_conn = ib_conn->iser_conn; 602 struct iscsi_conn *conn = iser_conn->iscsi_conn; 603 struct iscsi_mgmt_task *mtask; 604 int resume_tx = 0; 605 606 iser_dbg("Initiator, Data sent dto=0x%p\n", dto); 607 608 iser_dto_buffs_release(dto); 609 610 if (tx_desc->type == ISCSI_TX_DATAOUT) 611 kmem_cache_free(ig.desc_cache, tx_desc); 612 613 if (atomic_read(&iser_conn->ib_conn->post_send_buf_count) == 614 ISER_QP_MAX_REQ_DTOS) 615 resume_tx = 1; 616 617 atomic_dec(&ib_conn->post_send_buf_count); 618 619 if (resume_tx) { 620 iser_dbg("%ld resuming tx\n",jiffies); 621 scsi_queue_work(conn->session->host, &conn->xmitwork); 622 } 623 624 if (tx_desc->type == ISCSI_TX_CONTROL) { 625 /* this arithmetic is legal by libiscsi dd_data allocation */ 626 mtask = (void *) ((long)(void *)tx_desc - 627 sizeof(struct iscsi_mgmt_task)); 628 if (mtask->hdr->itt == RESERVED_ITT) { 629 struct iscsi_session *session = conn->session; 630 631 spin_lock(&conn->session->lock); 632 list_del(&mtask->running); 633 __kfifo_put(session->mgmtpool.queue, (void*)&mtask, 634 sizeof(void*)); 635 spin_unlock(&session->lock); 636 } 637 } 638} 639 640void iser_ctask_rdma_init(struct iscsi_iser_cmd_task *iser_ctask) 641 642{ 643 iser_ctask->status = ISER_TASK_STATUS_INIT; 644 645 iser_ctask->dir[ISER_DIR_IN] = 0; 646 iser_ctask->dir[ISER_DIR_OUT] = 0; 647 648 iser_ctask->data[ISER_DIR_IN].data_len = 0; 649 iser_ctask->data[ISER_DIR_OUT].data_len = 0; 650 651 memset(&iser_ctask->rdma_regd[ISER_DIR_IN], 0, 652 sizeof(struct iser_regd_buf)); 653 memset(&iser_ctask->rdma_regd[ISER_DIR_OUT], 0, 654 sizeof(struct iser_regd_buf)); 655} 656 657void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task *iser_ctask) 658{ 659 int deferred; 660 int is_rdma_aligned = 1; 661 struct iser_regd_buf *regd; 662 663 /* if we were reading, copy back to unaligned sglist, 664 * anyway dma_unmap and free the copy 665 */ 666 if (iser_ctask->data_copy[ISER_DIR_IN].copy_buf != NULL) { 667 is_rdma_aligned = 0; 668 iser_finalize_rdma_unaligned_sg(iser_ctask, ISER_DIR_IN); 669 } 670 if (iser_ctask->data_copy[ISER_DIR_OUT].copy_buf != NULL) { 671 is_rdma_aligned = 0; 672 iser_finalize_rdma_unaligned_sg(iser_ctask, ISER_DIR_OUT); 673 } 674 675 if (iser_ctask->dir[ISER_DIR_IN]) { 676 regd = &iser_ctask->rdma_regd[ISER_DIR_IN]; 677 deferred = iser_regd_buff_release(regd); 678 if (deferred) { 679 iser_err("%d references remain for BUF-IN rdma reg\n", 680 atomic_read(®d->ref_count)); 681 } 682 } 683 684 if (iser_ctask->dir[ISER_DIR_OUT]) { 685 regd = &iser_ctask->rdma_regd[ISER_DIR_OUT]; 686 deferred = iser_regd_buff_release(regd); 687 if (deferred) { 688 iser_err("%d references remain for BUF-OUT rdma reg\n", 689 atomic_read(®d->ref_count)); 690 } 691 } 692 693 /* if the data was unaligned, it was already unmapped and then copied */ 694 if (is_rdma_aligned) 695 iser_dma_unmap_task_data(iser_ctask); 696} 697 698void iser_dto_buffs_release(struct iser_dto *dto) 699{ 700 int i; 701 702 for (i = 0; i < dto->regd_vector_len; i++) 703 iser_regd_buff_release(dto->regd[i]); 704} 705