1300723Strasz/* $FreeBSD$ */ 2300723Strasz/*- 3300723Strasz * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved. 4300723Strasz * 5300723Strasz * Redistribution and use in source and binary forms, with or without 6300723Strasz * modification, are permitted provided that the following conditions 7300723Strasz * are met: 8300723Strasz * 1. Redistributions of source code must retain the above copyright 9300723Strasz * notice, this list of conditions and the following disclaimer. 10300723Strasz * 2. Redistributions in binary form must reproduce the above copyright 11300723Strasz * notice, this list of conditions and the following disclaimer in the 12300723Strasz * documentation and/or other materials provided with the distribution. 13300723Strasz * 14300723Strasz * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15300723Strasz * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16300723Strasz * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17300723Strasz * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18300723Strasz * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19300723Strasz * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20300723Strasz * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21300723Strasz * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22300723Strasz * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23300723Strasz * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24300723Strasz * SUCH DAMAGE. 25300723Strasz */ 26300723Strasz 27300723Strasz#include "icl_iser.h" 28300723Strasz 29300723Straszstatic MALLOC_DEFINE(M_ISER_INITIATOR, "iser_initiator", "iser initiator backend"); 30300723Strasz 31300723Strasz/* Register user buffer memory and initialize passive rdma 32300723Strasz * dto descriptor. Data size is stored in 33300723Strasz * task->data[ISER_DIR_IN].data_len, Protection size 34300723Strasz * os stored in task->prot[ISER_DIR_IN].data_len 35300723Strasz */ 36300723Straszstatic int 37300723Strasziser_prepare_read_cmd(struct icl_iser_pdu *iser_pdu) 38300723Strasz{ 39300723Strasz struct iser_hdr *hdr = &iser_pdu->desc.iser_header; 40300723Strasz struct iser_data_buf *buf_in = &iser_pdu->data[ISER_DIR_IN]; 41300723Strasz struct iser_mem_reg *mem_reg; 42300723Strasz int err; 43300723Strasz 44300723Strasz err = iser_dma_map_task_data(iser_pdu, 45300723Strasz buf_in, 46300723Strasz ISER_DIR_IN, 47300723Strasz DMA_FROM_DEVICE); 48300723Strasz if (err) 49300723Strasz return (err); 50300723Strasz 51300723Strasz err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_IN); 52300723Strasz if (err) { 53300723Strasz ISER_ERR("Failed to set up Data-IN RDMA"); 54300723Strasz return (err); 55300723Strasz } 56300723Strasz 57300723Strasz mem_reg = &iser_pdu->rdma_reg[ISER_DIR_IN]; 58300723Strasz 59300723Strasz hdr->flags |= ISER_RSV; 60300723Strasz hdr->read_stag = cpu_to_be32(mem_reg->rkey); 61300723Strasz hdr->read_va = cpu_to_be64(mem_reg->sge.addr); 62300723Strasz 63300723Strasz return (0); 64300723Strasz} 65300723Strasz 66300723Strasz/* Register user buffer memory and initialize passive rdma 67300723Strasz * dto descriptor. Data size is stored in 68300723Strasz * task->data[ISER_DIR_OUT].data_len, Protection size 69300723Strasz * is stored at task->prot[ISER_DIR_OUT].data_len 70300723Strasz */ 71300723Straszstatic int 72300723Strasziser_prepare_write_cmd(struct icl_iser_pdu *iser_pdu) 73300723Strasz{ 74300723Strasz struct iser_hdr *hdr = &iser_pdu->desc.iser_header; 75300723Strasz struct iser_data_buf *buf_out = &iser_pdu->data[ISER_DIR_OUT]; 76300723Strasz struct iser_mem_reg *mem_reg; 77300723Strasz int err; 78300723Strasz 79300723Strasz err = iser_dma_map_task_data(iser_pdu, 80300723Strasz buf_out, 81300723Strasz ISER_DIR_OUT, 82300723Strasz DMA_TO_DEVICE); 83300723Strasz if (err) 84300723Strasz return (err); 85300723Strasz 86300723Strasz err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_OUT); 87300723Strasz if (err) { 88300723Strasz ISER_ERR("Failed to set up Data-out RDMA"); 89300723Strasz return (err); 90300723Strasz } 91300723Strasz 92300723Strasz mem_reg = &iser_pdu->rdma_reg[ISER_DIR_OUT]; 93300723Strasz 94300723Strasz hdr->flags |= ISER_WSV; 95300723Strasz hdr->write_stag = cpu_to_be32(mem_reg->rkey); 96300723Strasz hdr->write_va = cpu_to_be64(mem_reg->sge.addr); 97300723Strasz 98300723Strasz return (0); 99300723Strasz} 100300723Strasz 101300723Strasz/* creates a new tx descriptor and adds header regd buffer */ 102300723Straszvoid 103300723Strasziser_create_send_desc(struct iser_conn *iser_conn, 104300723Strasz struct iser_tx_desc *tx_desc) 105300723Strasz{ 106300723Strasz struct iser_device *device = iser_conn->ib_conn.device; 107300723Strasz 108300723Strasz ib_dma_sync_single_for_cpu(device->ib_device, 109300723Strasz tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); 110300723Strasz 111300723Strasz memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 112300723Strasz tx_desc->iser_header.flags = ISER_VER; 113300723Strasz 114300723Strasz tx_desc->num_sge = 1; 115300723Strasz 116300723Strasz if (tx_desc->tx_sg[0].lkey != device->mr->lkey) { 117300723Strasz tx_desc->tx_sg[0].lkey = device->mr->lkey; 118300723Strasz ISER_DBG("sdesc %p lkey mismatch, fixing", tx_desc); 119300723Strasz } 120300723Strasz} 121300723Strasz 122300723Straszvoid 123300723Strasziser_free_login_buf(struct iser_conn *iser_conn) 124300723Strasz{ 125300723Strasz struct iser_device *device = iser_conn->ib_conn.device; 126300723Strasz 127300723Strasz if (!iser_conn->login_buf) 128300723Strasz return; 129300723Strasz 130300723Strasz if (iser_conn->login_req_dma) 131300723Strasz ib_dma_unmap_single(device->ib_device, 132300723Strasz iser_conn->login_req_dma, 133300723Strasz ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); 134300723Strasz 135300723Strasz if (iser_conn->login_resp_dma) 136300723Strasz ib_dma_unmap_single(device->ib_device, 137300723Strasz iser_conn->login_resp_dma, 138300723Strasz ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); 139300723Strasz 140300723Strasz free(iser_conn->login_buf, M_ISER_INITIATOR); 141300723Strasz 142300723Strasz /* make sure we never redo any unmapping */ 143300723Strasz iser_conn->login_req_dma = 0; 144300723Strasz iser_conn->login_resp_dma = 0; 145300723Strasz iser_conn->login_buf = NULL; 146300723Strasz} 147300723Strasz 148300723Straszint 149300723Strasziser_alloc_login_buf(struct iser_conn *iser_conn) 150300723Strasz{ 151300723Strasz struct iser_device *device = iser_conn->ib_conn.device; 152300723Strasz int req_err, resp_err; 153300723Strasz 154300723Strasz BUG_ON(device == NULL); 155300723Strasz 156300723Strasz iser_conn->login_buf = malloc(ISCSI_DEF_MAX_RECV_SEG_LEN + ISER_RX_LOGIN_SIZE, 157300723Strasz M_ISER_INITIATOR, M_WAITOK | M_ZERO); 158300723Strasz 159300723Strasz if (!iser_conn->login_buf) 160300723Strasz goto out_err; 161300723Strasz 162300723Strasz iser_conn->login_req_buf = iser_conn->login_buf; 163300723Strasz iser_conn->login_resp_buf = iser_conn->login_buf + 164300723Strasz ISCSI_DEF_MAX_RECV_SEG_LEN; 165300723Strasz 166300723Strasz iser_conn->login_req_dma = ib_dma_map_single(device->ib_device, 167300723Strasz iser_conn->login_req_buf, 168300723Strasz ISCSI_DEF_MAX_RECV_SEG_LEN, 169300723Strasz DMA_TO_DEVICE); 170300723Strasz 171300723Strasz iser_conn->login_resp_dma = ib_dma_map_single(device->ib_device, 172300723Strasz iser_conn->login_resp_buf, 173300723Strasz ISER_RX_LOGIN_SIZE, 174300723Strasz DMA_FROM_DEVICE); 175300723Strasz 176300723Strasz req_err = ib_dma_mapping_error(device->ib_device, 177300723Strasz iser_conn->login_req_dma); 178300723Strasz resp_err = ib_dma_mapping_error(device->ib_device, 179300723Strasz iser_conn->login_resp_dma); 180300723Strasz 181300723Strasz if (req_err || resp_err) { 182300723Strasz if (req_err) 183300723Strasz iser_conn->login_req_dma = 0; 184300723Strasz if (resp_err) 185300723Strasz iser_conn->login_resp_dma = 0; 186300723Strasz goto free_login_buf; 187300723Strasz } 188300723Strasz 189300723Strasz return (0); 190300723Strasz 191300723Straszfree_login_buf: 192300723Strasz iser_free_login_buf(iser_conn); 193300723Strasz 194300723Straszout_err: 195300723Strasz ISER_DBG("unable to alloc or map login buf"); 196300723Strasz return (ENOMEM); 197300723Strasz} 198300723Strasz 199300723Straszint iser_alloc_rx_descriptors(struct iser_conn *iser_conn, int cmds_max) 200300723Strasz{ 201300723Strasz int i, j; 202300723Strasz u64 dma_addr; 203300723Strasz struct iser_rx_desc *rx_desc; 204300723Strasz struct ib_sge *rx_sg; 205300723Strasz struct ib_conn *ib_conn = &iser_conn->ib_conn; 206300723Strasz struct iser_device *device = ib_conn->device; 207300723Strasz 208300723Strasz iser_conn->qp_max_recv_dtos = cmds_max; 209300723Strasz iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2; 210300723Strasz 211300723Strasz if (iser_create_fastreg_pool(ib_conn, cmds_max)) 212300723Strasz goto create_rdma_reg_res_failed; 213300723Strasz 214300723Strasz 215300723Strasz iser_conn->num_rx_descs = cmds_max; 216300723Strasz iser_conn->rx_descs = malloc(iser_conn->num_rx_descs * 217300723Strasz sizeof(struct iser_rx_desc), M_ISER_INITIATOR, 218300723Strasz M_WAITOK | M_ZERO); 219300723Strasz if (!iser_conn->rx_descs) 220300723Strasz goto rx_desc_alloc_fail; 221300723Strasz 222300723Strasz rx_desc = iser_conn->rx_descs; 223300723Strasz 224300723Strasz for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) { 225300723Strasz dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, 226300723Strasz ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 227300723Strasz if (ib_dma_mapping_error(device->ib_device, dma_addr)) 228300723Strasz goto rx_desc_dma_map_failed; 229300723Strasz 230300723Strasz rx_desc->dma_addr = dma_addr; 231300723Strasz 232300723Strasz rx_sg = &rx_desc->rx_sg; 233300723Strasz rx_sg->addr = rx_desc->dma_addr; 234300723Strasz rx_sg->length = ISER_RX_PAYLOAD_SIZE; 235300723Strasz rx_sg->lkey = device->mr->lkey; 236300723Strasz } 237300723Strasz 238300723Strasz iser_conn->rx_desc_head = 0; 239300723Strasz 240300723Strasz return (0); 241300723Strasz 242300723Straszrx_desc_dma_map_failed: 243300723Strasz rx_desc = iser_conn->rx_descs; 244300723Strasz for (j = 0; j < i; j++, rx_desc++) 245300723Strasz ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 246300723Strasz ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 247300723Strasz free(iser_conn->rx_descs, M_ISER_INITIATOR); 248300723Strasz iser_conn->rx_descs = NULL; 249300723Straszrx_desc_alloc_fail: 250300723Strasz iser_free_fastreg_pool(ib_conn); 251300723Straszcreate_rdma_reg_res_failed: 252300723Strasz ISER_ERR("failed allocating rx descriptors / data buffers"); 253300723Strasz 254300723Strasz return (ENOMEM); 255300723Strasz} 256300723Strasz 257300723Straszvoid 258300723Strasziser_free_rx_descriptors(struct iser_conn *iser_conn) 259300723Strasz{ 260300723Strasz int i; 261300723Strasz struct iser_rx_desc *rx_desc; 262300723Strasz struct ib_conn *ib_conn = &iser_conn->ib_conn; 263300723Strasz struct iser_device *device = ib_conn->device; 264300723Strasz 265300723Strasz iser_free_fastreg_pool(ib_conn); 266300723Strasz 267300723Strasz rx_desc = iser_conn->rx_descs; 268300723Strasz for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) 269300723Strasz ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 270300723Strasz ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 271300723Strasz 272300723Strasz free(iser_conn->rx_descs, M_ISER_INITIATOR); 273300723Strasz 274300723Strasz /* make sure we never redo any unmapping */ 275300723Strasz iser_conn->rx_descs = NULL; 276300723Strasz} 277300723Strasz 278300723Straszstatic void 279300723Strasziser_buf_to_sg(void *buf, struct iser_data_buf *data_buf) 280300723Strasz{ 281300723Strasz struct scatterlist *sg; 282300723Strasz int i; 283300723Strasz size_t len, tlen; 284300723Strasz int offset; 285300723Strasz 286300723Strasz tlen = data_buf->data_len; 287300723Strasz 288300723Strasz for (i = 0; 0 < tlen; i++, tlen -= len) { 289300723Strasz sg = &data_buf->sgl[i]; 290300723Strasz offset = ((uintptr_t)buf) & ~PAGE_MASK; 291300723Strasz len = min(PAGE_SIZE - offset, tlen); 292300723Strasz sg_set_buf(sg, buf, len); 293300723Strasz buf = (void *)(((u64)buf) + (u64)len); 294300723Strasz } 295300723Strasz 296300723Strasz data_buf->size = i; 297300723Strasz sg_mark_end(sg); 298300723Strasz} 299300723Strasz 300300723Strasz 301300723Straszstatic void 302300723Strasziser_bio_to_sg(struct bio *bp, struct iser_data_buf *data_buf) 303300723Strasz{ 304300723Strasz struct scatterlist *sg; 305300723Strasz int i; 306300723Strasz size_t len, tlen; 307300723Strasz int offset; 308300723Strasz 309300723Strasz tlen = bp->bio_bcount; 310300723Strasz offset = bp->bio_ma_offset; 311300723Strasz 312300723Strasz for (i = 0; 0 < tlen; i++, tlen -= len) { 313300723Strasz sg = &data_buf->sgl[i]; 314300723Strasz len = min(PAGE_SIZE - offset, tlen); 315300723Strasz sg_set_page(sg, bp->bio_ma[i], len, offset); 316300723Strasz offset = 0; 317300723Strasz } 318300723Strasz 319300723Strasz data_buf->size = i; 320300723Strasz sg_mark_end(sg); 321300723Strasz} 322300723Strasz 323300723Straszstatic int 324300723Strasziser_csio_to_sg(struct ccb_scsiio *csio, struct iser_data_buf *data_buf) 325300723Strasz{ 326300723Strasz struct ccb_hdr *ccbh; 327300723Strasz int err = 0; 328300723Strasz 329300723Strasz ccbh = &csio->ccb_h; 330300723Strasz switch ((ccbh->flags & CAM_DATA_MASK)) { 331300723Strasz case CAM_DATA_BIO: 332300723Strasz iser_bio_to_sg((struct bio *) csio->data_ptr, data_buf); 333300723Strasz break; 334300723Strasz case CAM_DATA_VADDR: 335300723Strasz /* 336300723Strasz * Support KVA buffers for various scsi commands such as: 337300723Strasz * - REPORT_LUNS 338300723Strasz * - MODE_SENSE_6 339300723Strasz * - INQUIRY 340300723Strasz * - SERVICE_ACTION_IN. 341300723Strasz * The data of these commands always mapped into KVA. 342300723Strasz */ 343300723Strasz iser_buf_to_sg(csio->data_ptr, data_buf); 344300723Strasz break; 345300723Strasz default: 346300723Strasz ISER_ERR("flags 0x%X unimplemented", ccbh->flags); 347300723Strasz err = EINVAL; 348300723Strasz } 349300723Strasz return (err); 350300723Strasz} 351300723Strasz 352300723Straszstatic inline bool 353300723Strasziser_signal_comp(u8 sig_count) 354300723Strasz{ 355300723Strasz return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0); 356300723Strasz} 357300723Strasz 358300723Straszint 359300723Strasziser_send_command(struct iser_conn *iser_conn, 360300723Strasz struct icl_iser_pdu *iser_pdu) 361300723Strasz{ 362300723Strasz struct iser_data_buf *data_buf; 363300723Strasz struct iser_tx_desc *tx_desc = &iser_pdu->desc; 364300723Strasz struct iscsi_bhs_scsi_command *hdr = (struct iscsi_bhs_scsi_command *) &(iser_pdu->desc.iscsi_header); 365300723Strasz struct ccb_scsiio *csio = iser_pdu->csio; 366300723Strasz int err = 0; 367300723Strasz u8 sig_count = ++iser_conn->ib_conn.sig_count; 368300723Strasz 369300723Strasz /* build the tx desc regd header and add it to the tx desc dto */ 370300723Strasz tx_desc->type = ISCSI_TX_SCSI_COMMAND; 371300723Strasz iser_create_send_desc(iser_conn, tx_desc); 372300723Strasz 373300723Strasz if (hdr->bhssc_flags & BHSSC_FLAGS_R) { 374300723Strasz data_buf = &iser_pdu->data[ISER_DIR_IN]; 375300723Strasz } else { 376300723Strasz data_buf = &iser_pdu->data[ISER_DIR_OUT]; 377300723Strasz } 378300723Strasz 379300723Strasz data_buf->sg = csio->data_ptr; 380300723Strasz data_buf->data_len = csio->dxfer_len; 381300723Strasz 382300723Strasz if (likely(csio->dxfer_len)) { 383300723Strasz err = iser_csio_to_sg(csio, data_buf); 384300723Strasz if (unlikely(err)) 385300723Strasz goto send_command_error; 386300723Strasz } 387300723Strasz 388300723Strasz if (hdr->bhssc_flags & BHSSC_FLAGS_R) { 389300723Strasz err = iser_prepare_read_cmd(iser_pdu); 390300723Strasz if (err) 391300723Strasz goto send_command_error; 392300723Strasz } else if (hdr->bhssc_flags & BHSSC_FLAGS_W) { 393300723Strasz err = iser_prepare_write_cmd(iser_pdu); 394300723Strasz if (err) 395300723Strasz goto send_command_error; 396300723Strasz } 397300723Strasz 398300723Strasz err = iser_post_send(&iser_conn->ib_conn, tx_desc, 399300723Strasz iser_signal_comp(sig_count)); 400300723Strasz if (!err) 401300723Strasz return (0); 402300723Strasz 403300723Straszsend_command_error: 404300723Strasz ISER_ERR("iser_conn %p itt %u len %u err %d", iser_conn, 405300723Strasz hdr->bhssc_initiator_task_tag, 406300723Strasz hdr->bhssc_expected_data_transfer_length, 407300723Strasz err); 408300723Strasz return (err); 409300723Strasz} 410300723Strasz 411300723Straszint 412300723Strasziser_send_control(struct iser_conn *iser_conn, 413300723Strasz struct icl_iser_pdu *iser_pdu) 414300723Strasz{ 415300723Strasz struct iser_tx_desc *mdesc; 416300723Strasz struct iser_device *device; 417300723Strasz size_t datalen = iser_pdu->icl_pdu.ip_data_len; 418300723Strasz int err; 419300723Strasz 420300723Strasz mdesc = &iser_pdu->desc; 421300723Strasz 422300723Strasz /* build the tx desc regd header and add it to the tx desc dto */ 423300723Strasz mdesc->type = ISCSI_TX_CONTROL; 424300723Strasz iser_create_send_desc(iser_conn, mdesc); 425300723Strasz 426300723Strasz device = iser_conn->ib_conn.device; 427300723Strasz 428300723Strasz if (datalen > 0) { 429300723Strasz struct ib_sge *tx_dsg = &mdesc->tx_sg[1]; 430300723Strasz ib_dma_sync_single_for_cpu(device->ib_device, 431300723Strasz iser_conn->login_req_dma, datalen, 432300723Strasz DMA_TO_DEVICE); 433300723Strasz 434300723Strasz ib_dma_sync_single_for_device(device->ib_device, 435300723Strasz iser_conn->login_req_dma, datalen, 436300723Strasz DMA_TO_DEVICE); 437300723Strasz 438300723Strasz tx_dsg->addr = iser_conn->login_req_dma; 439300723Strasz tx_dsg->length = datalen; 440300723Strasz tx_dsg->lkey = device->mr->lkey; 441300723Strasz mdesc->num_sge = 2; 442300723Strasz } 443300723Strasz 444300727Strasz /* For login phase and discovery session we re-use the login buffer */ 445300727Strasz if (!iser_conn->handoff_done) { 446300723Strasz err = iser_post_recvl(iser_conn); 447300723Strasz if (err) 448300723Strasz goto send_control_error; 449300723Strasz } 450300723Strasz 451300723Strasz err = iser_post_send(&iser_conn->ib_conn, mdesc, true); 452300723Strasz if (!err) 453300723Strasz return (0); 454300723Strasz 455300723Straszsend_control_error: 456300723Strasz ISER_ERR("conn %p failed err %d", iser_conn, err); 457300723Strasz 458300723Strasz return (err); 459300723Strasz 460300723Strasz} 461300723Strasz 462300723Strasz/** 463300723Strasz * iser_rcv_dto_completion - recv DTO completion 464300723Strasz */ 465300723Straszvoid 466300723Strasziser_rcv_completion(struct iser_rx_desc *rx_desc, 467300723Strasz unsigned long rx_xfer_len, 468300723Strasz struct ib_conn *ib_conn) 469300723Strasz{ 470300723Strasz struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 471300723Strasz ib_conn); 472300723Strasz struct icl_conn *ic = &iser_conn->icl_conn; 473300723Strasz struct icl_pdu *response; 474300723Strasz struct iscsi_bhs *hdr; 475300723Strasz u64 rx_dma; 476300723Strasz int rx_buflen; 477300723Strasz int outstanding, count, err; 478300723Strasz 479300723Strasz /* differentiate between login to all other PDUs */ 480300723Strasz if ((char *)rx_desc == iser_conn->login_resp_buf) { 481300723Strasz rx_dma = iser_conn->login_resp_dma; 482300723Strasz rx_buflen = ISER_RX_LOGIN_SIZE; 483300723Strasz } else { 484300723Strasz rx_dma = rx_desc->dma_addr; 485300723Strasz rx_buflen = ISER_RX_PAYLOAD_SIZE; 486300723Strasz } 487300723Strasz 488300723Strasz ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma, 489300723Strasz rx_buflen, DMA_FROM_DEVICE); 490300723Strasz 491300723Strasz hdr = &rx_desc->iscsi_header; 492300723Strasz 493300723Strasz response = iser_new_pdu(ic, M_NOWAIT); 494300723Strasz response->ip_bhs = hdr; 495300723Strasz response->ip_data_len = rx_xfer_len - ISER_HEADERS_LEN; 496300723Strasz 497300723Strasz /* 498300723Strasz * In case we got data in the receive buffer, assign the ip_data_mbuf 499300723Strasz * to the rx_buffer - later we'll copy it to upper layer buffers 500300723Strasz */ 501300723Strasz if (response->ip_data_len) 502300723Strasz response->ip_data_mbuf = (struct mbuf *)(rx_desc->data); 503300723Strasz 504300723Strasz ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma, 505300723Strasz rx_buflen, DMA_FROM_DEVICE); 506300723Strasz 507300723Strasz /* decrementing conn->post_recv_buf_count only --after-- freeing the * 508300723Strasz * task eliminates the need to worry on tasks which are completed in * 509300723Strasz * parallel to the execution of iser_conn_term. So the code that waits * 510300723Strasz * for the posted rx bufs refcount to become zero handles everything */ 511300723Strasz ib_conn->post_recv_buf_count--; 512300723Strasz 513300723Strasz if (rx_dma == iser_conn->login_resp_dma) 514300723Strasz goto receive; 515300723Strasz 516300723Strasz outstanding = ib_conn->post_recv_buf_count; 517300723Strasz if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) { 518300723Strasz count = min(iser_conn->qp_max_recv_dtos - outstanding, 519300723Strasz iser_conn->min_posted_rx); 520300723Strasz err = iser_post_recvm(iser_conn, count); 521300723Strasz if (err) 522300723Strasz ISER_ERR("posting %d rx bufs err %d", count, err); 523300723Strasz } 524300723Strasz 525300723Straszreceive: 526300723Strasz (ic->ic_receive)(response); 527300723Strasz} 528300723Strasz 529300723Straszvoid 530300723Strasziser_snd_completion(struct iser_tx_desc *tx_desc, 531300723Strasz struct ib_conn *ib_conn) 532300723Strasz{ 533300723Strasz struct icl_iser_pdu *iser_pdu = container_of(tx_desc, struct icl_iser_pdu, desc); 534300723Strasz struct iser_conn *iser_conn = iser_pdu->iser_conn; 535300723Strasz 536300723Strasz if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL) 537300723Strasz iser_pdu_free(&iser_conn->icl_conn, &iser_pdu->icl_pdu); 538300723Strasz} 539