1300723Strasz/* $FreeBSD: stable/11/sys/dev/iser/icl_iser.c 331769 2018-03-30 18:06:29Z hselasky $ */ 2300723Strasz/*- 3300723Strasz * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved. 4300723Strasz * 5300723Strasz * Redistribution and use in source and binary forms, with or without 6300723Strasz * modification, are permitted provided that the following conditions 7300723Strasz * are met: 8300723Strasz * 1. Redistributions of source code must retain the above copyright 9300723Strasz * notice, this list of conditions and the following disclaimer. 10300723Strasz * 2. Redistributions in binary form must reproduce the above copyright 11300723Strasz * notice, this list of conditions and the following disclaimer in the 12300723Strasz * documentation and/or other materials provided with the distribution. 13300723Strasz * 14300723Strasz * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15300723Strasz * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16300723Strasz * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17300723Strasz * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18300723Strasz * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19300723Strasz * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20300723Strasz * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21300723Strasz * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22300723Strasz * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23300723Strasz * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24300723Strasz * SUCH DAMAGE. 25300723Strasz */ 26300723Strasz 27300723Strasz#include "icl_iser.h" 28300723Strasz 29300723StraszSYSCTL_NODE(_kern, OID_AUTO, iser, CTLFLAG_RW, 0, "iSER module"); 30300723Straszint iser_debug = 0; 31300723StraszSYSCTL_INT(_kern_iser, OID_AUTO, debug, CTLFLAG_RWTUN, 32300723Strasz &iser_debug, 0, "Enable iser debug messages"); 33300723Strasz 34300723Straszstatic MALLOC_DEFINE(M_ICL_ISER, "icl_iser", "iSCSI iser backend"); 35300723Straszstatic uma_zone_t icl_pdu_zone; 36300723Strasz 37300723Straszstatic volatile u_int icl_iser_ncons; 38300723Straszstruct iser_global ig; 39300723Strasz 40300727Straszstatic void iser_conn_release(struct icl_conn *ic); 41300727Strasz 42300723Straszstatic icl_conn_new_pdu_t iser_conn_new_pdu; 43300723Straszstatic icl_conn_pdu_free_t iser_conn_pdu_free; 44300723Straszstatic icl_conn_pdu_data_segment_length_t iser_conn_pdu_data_segment_length; 45300723Straszstatic icl_conn_pdu_append_data_t iser_conn_pdu_append_data; 46300723Straszstatic icl_conn_pdu_queue_t iser_conn_pdu_queue; 47300723Straszstatic icl_conn_handoff_t iser_conn_handoff; 48300723Straszstatic icl_conn_free_t iser_conn_free; 49300723Straszstatic icl_conn_close_t iser_conn_close; 50300723Straszstatic icl_conn_connect_t iser_conn_connect; 51300723Straszstatic icl_conn_task_setup_t iser_conn_task_setup; 52300723Straszstatic icl_conn_task_done_t iser_conn_task_done; 53300723Straszstatic icl_conn_pdu_get_data_t iser_conn_pdu_get_data; 54300723Strasz 55300723Straszstatic kobj_method_t icl_iser_methods[] = { 56300723Strasz KOBJMETHOD(icl_conn_new_pdu, iser_conn_new_pdu), 57300723Strasz KOBJMETHOD(icl_conn_pdu_free, iser_conn_pdu_free), 58300723Strasz KOBJMETHOD(icl_conn_pdu_data_segment_length, iser_conn_pdu_data_segment_length), 59300723Strasz KOBJMETHOD(icl_conn_pdu_append_data, iser_conn_pdu_append_data), 60300723Strasz KOBJMETHOD(icl_conn_pdu_queue, iser_conn_pdu_queue), 61300723Strasz KOBJMETHOD(icl_conn_handoff, iser_conn_handoff), 62300723Strasz KOBJMETHOD(icl_conn_free, iser_conn_free), 63300723Strasz KOBJMETHOD(icl_conn_close, iser_conn_close), 64300723Strasz KOBJMETHOD(icl_conn_connect, iser_conn_connect), 65300723Strasz KOBJMETHOD(icl_conn_task_setup, iser_conn_task_setup), 66300723Strasz KOBJMETHOD(icl_conn_task_done, iser_conn_task_done), 67300723Strasz KOBJMETHOD(icl_conn_pdu_get_data, iser_conn_pdu_get_data), 68300723Strasz { 0, 0 } 69300723Strasz}; 70300723Strasz 71300723StraszDEFINE_CLASS(icl_iser, icl_iser_methods, sizeof(struct iser_conn)); 72300723Strasz 73300723Strasz/** 74300723Strasz * iser_initialize_headers() - Initialize task headers 75300723Strasz * @pdu: iser pdu 76300723Strasz * @iser_conn: iser connection 77300723Strasz * 78300723Strasz * Notes: 79300723Strasz * This routine may race with iser teardown flow for scsi 80300723Strasz * error handling TMFs. So for TMF we should acquire the 81300723Strasz * state mutex to avoid dereferencing the IB device which 82300723Strasz * may have already been terminated (racing teardown sequence). 83300723Strasz */ 84300723Straszint 85300723Strasziser_initialize_headers(struct icl_iser_pdu *pdu, struct iser_conn *iser_conn) 86300723Strasz{ 87300723Strasz struct iser_tx_desc *tx_desc = &pdu->desc; 88300723Strasz struct iser_device *device = iser_conn->ib_conn.device; 89300723Strasz u64 dma_addr; 90300723Strasz int ret = 0; 91300723Strasz 92300723Strasz dma_addr = ib_dma_map_single(device->ib_device, (void *)tx_desc, 93300723Strasz ISER_HEADERS_LEN, DMA_TO_DEVICE); 94300723Strasz if (ib_dma_mapping_error(device->ib_device, dma_addr)) { 95300723Strasz ret = -ENOMEM; 96300723Strasz goto out; 97300723Strasz } 98300723Strasz 99300723Strasz tx_desc->mapped = true; 100300723Strasz tx_desc->dma_addr = dma_addr; 101300723Strasz tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 102300723Strasz tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 103300723Strasz tx_desc->tx_sg[0].lkey = device->mr->lkey; 104300723Strasz 105300723Straszout: 106300723Strasz 107300723Strasz return (ret); 108300723Strasz} 109300723Strasz 110300723Straszint 111300723Strasziser_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, 112300723Strasz const void *addr, size_t len, int flags) 113300723Strasz{ 114300723Strasz struct iser_conn *iser_conn = icl_to_iser_conn(ic); 115300723Strasz 116300723Strasz if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_LOGIN_REQUEST || 117300723Strasz request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_TEXT_REQUEST) { 118300723Strasz ISER_DBG("copy to login buff"); 119300723Strasz memcpy(iser_conn->login_req_buf, addr, len); 120300723Strasz request->ip_data_len = len; 121300723Strasz } 122300723Strasz 123300723Strasz return (0); 124300723Strasz} 125300723Strasz 126300723Straszvoid 127300723Strasziser_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 128300723Strasz size_t off, void *addr, size_t len) 129300723Strasz{ 130300723Strasz /* If we have a receive data, copy it to upper layer buffer */ 131300723Strasz if (ip->ip_data_mbuf) 132300723Strasz memcpy(addr, ip->ip_data_mbuf + off, len); 133300723Strasz} 134300723Strasz 135300723Strasz/* 136300723Strasz * Allocate icl_pdu with empty BHS to fill up by the caller. 137300723Strasz */ 138300723Straszstruct icl_pdu * 139300723Strasziser_new_pdu(struct icl_conn *ic, int flags) 140300723Strasz{ 141300723Strasz struct icl_iser_pdu *iser_pdu; 142300723Strasz struct icl_pdu *ip; 143300723Strasz struct iser_conn *iser_conn = icl_to_iser_conn(ic); 144300723Strasz 145300723Strasz iser_pdu = uma_zalloc(icl_pdu_zone, flags | M_ZERO); 146300723Strasz if (iser_pdu == NULL) { 147300723Strasz ISER_WARN("failed to allocate %zd bytes", sizeof(*iser_pdu)); 148300723Strasz return (NULL); 149300723Strasz } 150300723Strasz 151300723Strasz iser_pdu->iser_conn = iser_conn; 152300723Strasz ip = &iser_pdu->icl_pdu; 153300723Strasz ip->ip_conn = ic; 154300723Strasz ip->ip_bhs = &iser_pdu->desc.iscsi_header; 155300723Strasz 156300723Strasz return (ip); 157300723Strasz} 158300723Strasz 159300723Straszstruct icl_pdu * 160300723Strasziser_conn_new_pdu(struct icl_conn *ic, int flags) 161300723Strasz{ 162300723Strasz return (iser_new_pdu(ic, flags)); 163300723Strasz} 164300723Strasz 165300723Straszvoid 166300723Strasziser_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 167300723Strasz{ 168300723Strasz struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip); 169300723Strasz 170300723Strasz uma_zfree(icl_pdu_zone, iser_pdu); 171300723Strasz} 172300723Strasz 173300723Straszsize_t 174300723Strasziser_conn_pdu_data_segment_length(struct icl_conn *ic, 175300723Strasz const struct icl_pdu *request) 176300723Strasz{ 177300723Strasz uint32_t len = 0; 178300723Strasz 179300723Strasz len += request->ip_bhs->bhs_data_segment_len[0]; 180300723Strasz len <<= 8; 181300723Strasz len += request->ip_bhs->bhs_data_segment_len[1]; 182300723Strasz len <<= 8; 183300723Strasz len += request->ip_bhs->bhs_data_segment_len[2]; 184300723Strasz 185300723Strasz return (len); 186300723Strasz} 187300723Strasz 188300723Straszvoid 189300723Strasziser_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 190300723Strasz{ 191300723Strasz iser_pdu_free(ic, ip); 192300723Strasz} 193300723Strasz 194300723Straszstatic bool 195300723Straszis_control_opcode(uint8_t opcode) 196300723Strasz{ 197300723Strasz bool is_control = false; 198300723Strasz 199300723Strasz switch (opcode & ISCSI_OPCODE_MASK) { 200300723Strasz case ISCSI_BHS_OPCODE_NOP_OUT: 201300723Strasz case ISCSI_BHS_OPCODE_LOGIN_REQUEST: 202300723Strasz case ISCSI_BHS_OPCODE_LOGOUT_REQUEST: 203300723Strasz case ISCSI_BHS_OPCODE_TEXT_REQUEST: 204300723Strasz is_control = true; 205300723Strasz break; 206300723Strasz case ISCSI_BHS_OPCODE_SCSI_COMMAND: 207300723Strasz is_control = false; 208300723Strasz break; 209300723Strasz default: 210300723Strasz ISER_ERR("unknown opcode %d", opcode); 211300723Strasz } 212300723Strasz 213300723Strasz return (is_control); 214300723Strasz} 215300723Strasz 216300723Straszvoid 217300723Strasziser_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 218300723Strasz{ 219300723Strasz struct iser_conn *iser_conn = icl_to_iser_conn(ic); 220300723Strasz struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip); 221300723Strasz int ret; 222300723Strasz 223300734Strasz if (iser_conn->state != ISER_CONN_UP) 224300734Strasz return; 225300734Strasz 226300723Strasz ret = iser_initialize_headers(iser_pdu, iser_conn); 227300723Strasz if (ret) { 228300723Strasz ISER_ERR("Failed to map TX descriptor pdu %p", iser_pdu); 229300723Strasz return; 230300723Strasz } 231300723Strasz 232300723Strasz if (is_control_opcode(ip->ip_bhs->bhs_opcode)) { 233300723Strasz ret = iser_send_control(iser_conn, iser_pdu); 234300723Strasz if (unlikely(ret)) 235300723Strasz ISER_ERR("Failed to send control pdu %p", iser_pdu); 236300723Strasz } else { 237300723Strasz ret = iser_send_command(iser_conn, iser_pdu); 238300723Strasz if (unlikely(ret)) 239300723Strasz ISER_ERR("Failed to send command pdu %p", iser_pdu); 240300723Strasz } 241300723Strasz} 242300723Strasz 243300723Straszstatic struct icl_conn * 244300723Strasziser_new_conn(const char *name, struct mtx *lock) 245300723Strasz{ 246300723Strasz struct iser_conn *iser_conn; 247300723Strasz struct icl_conn *ic; 248300723Strasz 249300723Strasz refcount_acquire(&icl_iser_ncons); 250300723Strasz 251300723Strasz iser_conn = (struct iser_conn *)kobj_create(&icl_iser_class, M_ICL_ISER, M_WAITOK | M_ZERO); 252300723Strasz if (!iser_conn) { 253300723Strasz ISER_ERR("failed to allocate iser conn"); 254300723Strasz refcount_release(&icl_iser_ncons); 255300723Strasz return (NULL); 256300723Strasz } 257300723Strasz 258300723Strasz cv_init(&iser_conn->up_cv, "iser_cv"); 259300723Strasz sx_init(&iser_conn->state_mutex, "iser_conn_state_mutex"); 260326519Shselasky mtx_init(&iser_conn->ib_conn.beacon.flush_lock, "iser_flush_lock", NULL, MTX_DEF); 261300723Strasz cv_init(&iser_conn->ib_conn.beacon.flush_cv, "flush_cv"); 262326519Shselasky mtx_init(&iser_conn->ib_conn.lock, "iser_lock", NULL, MTX_DEF); 263300723Strasz 264300723Strasz ic = &iser_conn->icl_conn; 265300723Strasz ic->ic_lock = lock; 266300723Strasz ic->ic_name = name; 267300727Strasz ic->ic_offload = strdup("iser", M_TEMP); 268300723Strasz ic->ic_iser = true; 269300727Strasz ic->ic_unmapped = true; 270300723Strasz 271300723Strasz return (ic); 272300723Strasz} 273300723Strasz 274300723Straszvoid 275300723Strasziser_conn_free(struct icl_conn *ic) 276300723Strasz{ 277300723Strasz struct iser_conn *iser_conn = icl_to_iser_conn(ic); 278300723Strasz 279300727Strasz iser_conn_release(ic); 280326519Shselasky mtx_destroy(&iser_conn->ib_conn.lock); 281300723Strasz cv_destroy(&iser_conn->ib_conn.beacon.flush_cv); 282300723Strasz mtx_destroy(&iser_conn->ib_conn.beacon.flush_lock); 283300723Strasz sx_destroy(&iser_conn->state_mutex); 284300723Strasz cv_destroy(&iser_conn->up_cv); 285300723Strasz kobj_delete((struct kobj *)iser_conn, M_ICL_ISER); 286300723Strasz refcount_release(&icl_iser_ncons); 287300723Strasz} 288300723Strasz 289300723Straszint 290300727Strasziser_conn_handoff(struct icl_conn *ic, int fd) 291300723Strasz{ 292300723Strasz struct iser_conn *iser_conn = icl_to_iser_conn(ic); 293300723Strasz int error = 0; 294300723Strasz 295300723Strasz sx_xlock(&iser_conn->state_mutex); 296300723Strasz if (iser_conn->state != ISER_CONN_UP) { 297300723Strasz error = EINVAL; 298300723Strasz ISER_ERR("iser_conn %p state is %d, teardown started\n", 299300723Strasz iser_conn, iser_conn->state); 300300723Strasz goto out; 301300723Strasz } 302300723Strasz 303300727Strasz error = iser_alloc_rx_descriptors(iser_conn, ic->ic_maxtags); 304300723Strasz if (error) 305300723Strasz goto out; 306300723Strasz 307300723Strasz error = iser_post_recvm(iser_conn, iser_conn->min_posted_rx); 308300723Strasz if (error) 309300723Strasz goto post_error; 310300723Strasz 311300727Strasz iser_conn->handoff_done = true; 312300727Strasz 313300723Strasz sx_xunlock(&iser_conn->state_mutex); 314300723Strasz return (error); 315300723Strasz 316300723Straszpost_error: 317300723Strasz iser_free_rx_descriptors(iser_conn); 318300723Straszout: 319300723Strasz sx_xunlock(&iser_conn->state_mutex); 320300723Strasz return (error); 321300723Strasz 322300723Strasz} 323300723Strasz 324300723Strasz/** 325300723Strasz * Frees all conn objects 326300723Strasz */ 327300727Straszstatic void 328300723Strasziser_conn_release(struct icl_conn *ic) 329300723Strasz{ 330300723Strasz struct iser_conn *iser_conn = icl_to_iser_conn(ic); 331300723Strasz struct ib_conn *ib_conn = &iser_conn->ib_conn; 332300723Strasz struct iser_conn *curr, *tmp; 333300723Strasz 334300723Strasz mtx_lock(&ig.connlist_mutex); 335300723Strasz /* 336300723Strasz * Search for iser connection in global list. 337300723Strasz * It may not be there in case of failure in connection establishment 338300723Strasz * stage. 339300723Strasz */ 340300723Strasz list_for_each_entry_safe(curr, tmp, &ig.connlist, conn_list) { 341300723Strasz if (iser_conn == curr) { 342300723Strasz ISER_WARN("found iser_conn %p", iser_conn); 343300723Strasz list_del(&iser_conn->conn_list); 344300723Strasz } 345300723Strasz } 346300723Strasz mtx_unlock(&ig.connlist_mutex); 347300723Strasz 348300723Strasz /* 349300723Strasz * In case we reconnecting or removing session, we need to 350300723Strasz * release IB resources (which is safe to call more than once). 351300723Strasz */ 352300723Strasz sx_xlock(&iser_conn->state_mutex); 353300723Strasz iser_free_ib_conn_res(iser_conn, true); 354300723Strasz sx_xunlock(&iser_conn->state_mutex); 355300723Strasz 356300723Strasz if (ib_conn->cma_id != NULL) { 357300723Strasz rdma_destroy_id(ib_conn->cma_id); 358300723Strasz ib_conn->cma_id = NULL; 359300723Strasz } 360300723Strasz 361300723Strasz} 362300723Strasz 363300723Straszvoid 364300723Strasziser_conn_close(struct icl_conn *ic) 365300723Strasz{ 366300723Strasz struct iser_conn *iser_conn = icl_to_iser_conn(ic); 367300723Strasz 368300723Strasz ISER_INFO("closing conn %p", iser_conn); 369300723Strasz 370300723Strasz sx_xlock(&iser_conn->state_mutex); 371300723Strasz /* 372300723Strasz * In case iser connection is waiting on conditional variable 373300723Strasz * (state PENDING) and we try to close it before connection establishment, 374300723Strasz * we need to signal it to continue releasing connection properly. 375300723Strasz */ 376300723Strasz if (!iser_conn_terminate(iser_conn) && iser_conn->state == ISER_CONN_PENDING) 377300723Strasz cv_signal(&iser_conn->up_cv); 378300723Strasz sx_xunlock(&iser_conn->state_mutex); 379300723Strasz 380300723Strasz} 381300723Strasz 382300723Straszint 383300723Strasziser_conn_connect(struct icl_conn *ic, int domain, int socktype, 384300723Strasz int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa) 385300723Strasz{ 386300723Strasz struct iser_conn *iser_conn = icl_to_iser_conn(ic); 387300723Strasz struct ib_conn *ib_conn = &iser_conn->ib_conn; 388300723Strasz int err = 0; 389300723Strasz 390300727Strasz iser_conn_release(ic); 391300727Strasz 392300723Strasz sx_xlock(&iser_conn->state_mutex); 393300723Strasz /* the device is known only --after-- address resolution */ 394300723Strasz ib_conn->device = NULL; 395300727Strasz iser_conn->handoff_done = false; 396300723Strasz 397300723Strasz iser_conn->state = ISER_CONN_PENDING; 398300723Strasz 399331769Shselasky ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler, (void *)iser_conn, 400300723Strasz RDMA_PS_TCP, IB_QPT_RC); 401300723Strasz if (IS_ERR(ib_conn->cma_id)) { 402300723Strasz err = -PTR_ERR(ib_conn->cma_id); 403300723Strasz ISER_ERR("rdma_create_id failed: %d", err); 404300723Strasz goto id_failure; 405300723Strasz } 406300723Strasz 407300723Strasz err = rdma_resolve_addr(ib_conn->cma_id, from_sa, to_sa, 1000); 408300723Strasz if (err) { 409300723Strasz ISER_ERR("rdma_resolve_addr failed: %d", err); 410300723Strasz if (err < 0) 411300723Strasz err = -err; 412300723Strasz goto addr_failure; 413300723Strasz } 414300723Strasz 415300723Strasz ISER_DBG("before cv_wait: %p", iser_conn); 416300723Strasz cv_wait(&iser_conn->up_cv, &iser_conn->state_mutex); 417300723Strasz ISER_DBG("after cv_wait: %p", iser_conn); 418300723Strasz 419300723Strasz if (iser_conn->state != ISER_CONN_UP) { 420300723Strasz err = EIO; 421300723Strasz goto addr_failure; 422300723Strasz } 423300723Strasz 424300723Strasz err = iser_alloc_login_buf(iser_conn); 425300723Strasz if (err) 426300723Strasz goto addr_failure; 427300723Strasz sx_xunlock(&iser_conn->state_mutex); 428300723Strasz 429300723Strasz mtx_lock(&ig.connlist_mutex); 430300723Strasz list_add(&iser_conn->conn_list, &ig.connlist); 431300723Strasz mtx_unlock(&ig.connlist_mutex); 432300723Strasz 433300723Strasz return (0); 434300723Strasz 435300723Straszid_failure: 436300723Strasz ib_conn->cma_id = NULL; 437300723Straszaddr_failure: 438300723Strasz sx_xunlock(&iser_conn->state_mutex); 439300723Strasz return (err); 440300723Strasz} 441300723Strasz 442300723Straszint 443300727Strasziser_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 444300727Strasz struct ccb_scsiio *csio, 445300727Strasz uint32_t *task_tagp, void **prvp) 446300723Strasz{ 447300723Strasz struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip); 448300723Strasz 449300723Strasz *prvp = ip; 450300723Strasz iser_pdu->csio = csio; 451300723Strasz 452300723Strasz return (0); 453300723Strasz} 454300723Strasz 455300723Straszvoid 456300723Strasziser_conn_task_done(struct icl_conn *ic, void *prv) 457300723Strasz{ 458300723Strasz struct icl_pdu *ip = prv; 459300723Strasz struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip); 460300723Strasz struct iser_device *device = iser_pdu->iser_conn->ib_conn.device; 461300723Strasz struct iser_tx_desc *tx_desc = &iser_pdu->desc; 462300723Strasz 463300723Strasz if (iser_pdu->dir[ISER_DIR_IN]) { 464300723Strasz iser_unreg_rdma_mem(iser_pdu, ISER_DIR_IN); 465300723Strasz iser_dma_unmap_task_data(iser_pdu, 466300723Strasz &iser_pdu->data[ISER_DIR_IN], 467300723Strasz DMA_FROM_DEVICE); 468300723Strasz } 469300723Strasz 470300723Strasz if (iser_pdu->dir[ISER_DIR_OUT]) { 471300723Strasz iser_unreg_rdma_mem(iser_pdu, ISER_DIR_OUT); 472300723Strasz iser_dma_unmap_task_data(iser_pdu, 473300723Strasz &iser_pdu->data[ISER_DIR_OUT], 474300723Strasz DMA_TO_DEVICE); 475300723Strasz } 476300723Strasz 477300723Strasz if (likely(tx_desc->mapped)) { 478300723Strasz ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr, 479300723Strasz ISER_HEADERS_LEN, DMA_TO_DEVICE); 480300723Strasz tx_desc->mapped = false; 481300723Strasz } 482300723Strasz 483300723Strasz iser_pdu_free(ic, ip); 484300723Strasz} 485300723Strasz 486300723Straszstatic int 487300723Strasziser_limits(size_t *limitp) 488300723Strasz{ 489300723Strasz *limitp = 128 * 1024; 490300723Strasz 491300723Strasz return (0); 492300723Strasz} 493300723Strasz 494300723Straszstatic int 495300723Straszicl_iser_load(void) 496300723Strasz{ 497300723Strasz int error; 498300723Strasz 499300723Strasz ISER_DBG("Starting iSER datamover..."); 500300723Strasz 501300723Strasz icl_pdu_zone = uma_zcreate("icl_iser_pdu", sizeof(struct icl_iser_pdu), 502300723Strasz NULL, NULL, NULL, NULL, 503300723Strasz UMA_ALIGN_PTR, 0); 504300723Strasz /* FIXME: Check rc */ 505300723Strasz 506300723Strasz refcount_init(&icl_iser_ncons, 0); 507300723Strasz 508300727Strasz error = icl_register("iser", true, 0, iser_limits, iser_new_conn); 509300723Strasz KASSERT(error == 0, ("failed to register iser")); 510300723Strasz 511300723Strasz memset(&ig, 0, sizeof(struct iser_global)); 512300723Strasz 513300723Strasz /* device init is called only after the first addr resolution */ 514300723Strasz sx_init(&ig.device_list_mutex, "global_device_lock"); 515300723Strasz INIT_LIST_HEAD(&ig.device_list); 516326519Shselasky mtx_init(&ig.connlist_mutex, "iser_global_conn_lock", NULL, MTX_DEF); 517300723Strasz INIT_LIST_HEAD(&ig.connlist); 518300723Strasz sx_init(&ig.close_conns_mutex, "global_close_conns_lock"); 519300723Strasz 520300723Strasz return (error); 521300723Strasz} 522300723Strasz 523300723Straszstatic int 524300723Straszicl_iser_unload(void) 525300723Strasz{ 526300723Strasz ISER_DBG("Removing iSER datamover..."); 527300723Strasz 528300723Strasz if (icl_iser_ncons != 0) 529300723Strasz return (EBUSY); 530300723Strasz 531300723Strasz sx_destroy(&ig.close_conns_mutex); 532300723Strasz mtx_destroy(&ig.connlist_mutex); 533300723Strasz sx_destroy(&ig.device_list_mutex); 534300723Strasz 535300727Strasz icl_unregister("iser", true); 536300723Strasz 537300723Strasz uma_zdestroy(icl_pdu_zone); 538300723Strasz 539300723Strasz return (0); 540300723Strasz} 541300723Strasz 542300723Straszstatic int 543300723Straszicl_iser_modevent(module_t mod, int what, void *arg) 544300723Strasz{ 545300723Strasz switch (what) { 546300723Strasz case MOD_LOAD: 547300723Strasz return (icl_iser_load()); 548300723Strasz case MOD_UNLOAD: 549300723Strasz return (icl_iser_unload()); 550300723Strasz default: 551300723Strasz return (EINVAL); 552300723Strasz } 553300723Strasz} 554300723Strasz 555300723Straszmoduledata_t icl_iser_data = { 556300723Strasz .name = "icl_iser", 557300723Strasz .evhand = icl_iser_modevent, 558300723Strasz .priv = 0 559300723Strasz}; 560300723Strasz 561300723StraszDECLARE_MODULE(icl_iser, icl_iser_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 562300723StraszMODULE_DEPEND(icl_iser, icl, 1, 1, 1); 563300723StraszMODULE_DEPEND(icl_iser, ibcore, 1, 1, 1); 564300723StraszMODULE_DEPEND(icl_iser, linuxkpi, 1, 1, 1); 565300723StraszMODULE_VERSION(icl_iser, 1); 566