srpt_impl.h revision 9684:2e0b0fc2d0d3
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27#ifndef _SRPT_IMPL_H_ 28#define _SRPT_IMPL_H_ 29 30/* 31 * Prototypes and data structures for the SRP Target Port Provider. 32 */ 33 34#include <sys/types.h> 35#include <sys/ddi.h> 36#include <sys/ib/ibtl/ibti.h> 37#include <sys/modctl.h> 38 39#include <stmf.h> 40#include <stmf_ioctl.h> 41#include <portif.h> 42 43#include <sys/ib/mgt/ibdma/ibdma.h> 44 45#ifdef __cplusplus 46extern "C" { 47#endif 48 49/* 50 * We should/could consider making some of these values tunables. 51 * Specifically, SEND_MSG_SIZE and SEND_MSG_DEPTH. 52 */ 53enum { 54 SRPT_DEFAULT_IOC_SRQ_SIZE = 4096, 55 SRPT_DEFAULT_SEND_MSG_DEPTH = 128, 56 SRPT_DEFAULT_SEND_MSG_SIZE = 960, /* must be multiple of 64 */ 57 SRPT_DEFAULT_MAX_RDMA_SIZE = 65536, 58 SRPT_MIN_T_I_IU_LEN = 52, 59 SRPT_EUI_ID_LEN = 20, 60 SRPT_RECV_WC_POLL_SIZE = 16, 61 SRPT_SEND_WC_POLL_SIZE = 16, 62 SRPT_MAX_OUT_IO_PER_CMD = 16, 63 SRPT_FENCE_SEND = 1, 64 SRPT_NO_FENCE_SEND = 0 65}; 66 67struct srpt_target_port_s; 68 69/* 70 * SRP Session - represents a SCSI I_T_Nexus. 71 * 72 * Sessions map 1 or more initiator logins to a specific I/O 73 * Controller SCSI Target Port. Targets create sessions 74 * at initiator login and release when no longer referenced 75 * by a login. 76 */ 77typedef struct srpt_session_s { 78 krwlock_t ss_rwlock; 79 list_node_t ss_node; 80 81 /* 82 * ADVANCED FEATURE, NOT YET SUPPORTED. 83 * In multi-channel mode, multiple RDMA communication 84 * channels may reference the same SCSI session. When 85 * a channel releases its reference to the SCSI session, 86 * it should have no tasks associated with the session. 87 * 88 * If multi-channel is implemented, add a channel list 89 * to this object instead of tracking it on the target. 90 * 91 * Will also need a session state & mode. Mode is to 92 * track if the session is MULTI or SINGLE channel. 93 */ 94 95 stmf_scsi_session_t *ss_ss; 96 struct srpt_target_port_s *ss_tgt; 97 list_t ss_task_list; 98 99 /* 100 * SRP Initiator and target identifiers are 128-bit. 101 * 102 * The specification defines the initiator to be 64-bits of 103 * ID extension and 64 bits of GUID, but these are really 104 * just a recommendation. Generally the extension is used 105 * to create unique I_T_Nexus from the same initiator and 106 * target. Initiators are inconsistent on the GUID they 107 * use, some use the HCA Node, some the HCA port. 108 * 109 * The specification defines the target to be 64-bits of 110 * service ID followed by 64-bits of I/O Controller GUID. 111 * In the case where there is a single default target 112 * service, they will be the same (our default). 113 */ 114 uint8_t ss_i_id[SRP_PORT_ID_LEN]; 115 uint8_t ss_t_id[SRP_PORT_ID_LEN]; 116 117 /* So we can see the full 128-bit initiator login from stmfadm */ 118 char ss_alias[SRP_PORT_ID_LEN * 2 + 2]; 119 uint8_t ss_hw_port; 120} srpt_session_t; 121 122/* 123 * Send work request types. 124 */ 125typedef enum srpt_swqe_type_e { 126 SRPT_SWQE_TYPE_DATA = 1, 127 SRPT_SWQE_TYPE_RESP 128} srpt_swqe_type_t; 129 130typedef struct srpt_swqe_s { 131 srpt_swqe_type_t sw_type; 132 void *sw_addr; 133 ibt_wrid_t sw_next; 134} srpt_swqe_t; 135 136/* 137 * SRP Channel - the RDMA communications channel associated with 138 * a specific SRP login. 139 */ 140typedef enum srpt_channel_state_e { 141 SRPT_CHANNEL_CONNECTING = 0, 142 SRPT_CHANNEL_CONNECTED, 143 SRPT_CHANNEL_DISCONNECTING 144} srpt_channel_state_t; 145 146typedef struct srpt_channel_s { 147 krwlock_t ch_rwlock; 148 149 kmutex_t ch_reflock; 150 uint_t ch_refcnt; 151 kcondvar_t ch_cv_complete; 152 uint_t ch_cv_waiters; 153 154 list_node_t ch_stp_node; 155 srpt_channel_state_t ch_state; 156 ibt_cq_hdl_t ch_scq_hdl; 157 ibt_cq_hdl_t ch_rcq_hdl; 158 ibt_channel_hdl_t ch_chan_hdl; 159 ibt_chan_sizes_t ch_sizes; 160 161 uint32_t ch_req_lim_delta; 162 uint32_t ch_ti_iu_len; 163 struct srpt_target_port_s *ch_tgt; 164 srpt_session_t *ch_session; 165 166 /* 167 * Map IB send WQE request IDs to the 168 * apporpriate operation type (for errors). 169 */ 170 kmutex_t ch_swqe_lock; 171 srpt_swqe_t *ch_swqe; 172 uint32_t ch_num_swqe; 173 uint32_t ch_head; 174 uint32_t ch_tail; 175 uint32_t ch_swqe_posted; 176} srpt_channel_t; 177 178/* 179 * SRP Information Unit (IU). Each IU structure contains 180 * the buffer for the IU itself (received over the RC 181 * channel), and all of the context required by the target 182 * to process this request represented by the IU. 183 * Available IU structures are managed on the I/O Controller 184 * shared receive queue. 185 */ 186enum { 187 SRPT_IU_STMF_ABORTING = 1 << 0, /* STMF called abort */ 188 SRPT_IU_SRP_ABORTING = 1 << 1, /* SRP initiator aborting */ 189 SRPT_IU_ABORTED = 1 << 2, /* Task has been aborted */ 190 SRPT_IU_RESP_SENT = 1 << 3 /* Response queued */ 191}; 192 193typedef struct srpt_iu_s { 194 /* 195 * The buffer for the IU itself. When unused (a 196 * reference count of zero), this buffer is posted 197 * on the I/O Controllers SRPT SRQ. 198 */ 199 void *iu_buf; 200 ibt_wr_ds_t iu_sge; 201 struct srpt_ioc_s *iu_ioc; 202 uint_t iu_pool_ndx; 203 kmutex_t iu_lock; 204 205 /* 206 * The following are reset for each IU request 207 * processed by this buffer. 208 */ 209 list_node_t iu_ss_task_node; 210 srpt_channel_t *iu_ch; 211 212 uint_t iu_num_rdescs; 213 srp_direct_desc_t *iu_rdescs; 214 uint_t iu_tot_xfer_len; 215 216 uint64_t iu_tag; 217 uint_t iu_flags; 218 uint32_t iu_sq_posted_cnt; 219 scsi_task_t *iu_stmf_task; 220} srpt_iu_t; 221 222/* 223 * SRP SCSI Target Port. By default each HCA creates a single 224 * SCSI Target Port based on the associated I/O Controller 225 * (HCA) node GUID and made available through each physical 226 * hardware port of the I/O Controller. 227 */ 228typedef enum srpt_target_state_e { 229 SRPT_TGT_STATE_OFFLINE = 0, 230 SRPT_TGT_STATE_ONLINING, 231 SRPT_TGT_STATE_ONLINE, 232 SRPT_TGT_STATE_OFFLINING 233} srpt_target_state_t; 234 235typedef struct srpt_hw_port_s { 236 ibt_sbind_hdl_t hwp_bind_hdl; 237 ib_gid_t hwp_gid; 238} srpt_hw_port_t; 239 240typedef struct srpt_target_port_s { 241 stmf_local_port_t *tp_lport; 242 struct srpt_ioc_s *tp_ioc; 243 244 kmutex_t tp_lock; 245 srpt_target_state_t tp_state; 246 kcondvar_t tp_offline_complete; 247 uint_t tp_drv_disabled; 248 249 /* 250 * We are using a simple list for channels right now, we 251 * probably should switch this over to the AVL 252 * implementation eventually (but lookups are not done 253 * in the data path so this is not urgent). 254 */ 255 kmutex_t tp_ch_list_lock; 256 list_t tp_ch_list; 257 258 /* 259 * A list of active sessions. Session lifetime is 260 * determined by having active channels, but track 261 * them here for easier determination to when a 262 * target can truly be offlined, and as a step toward 263 * being session-focused rather than channel-focused. 264 * If we ever truly support multi-channel, move the 265 * channels to be part of the session object. 266 * 267 * Sessions should remain on this list until they 268 * are deregistered from STMF. This allows the target 269 * to properly track when it can consider itself 'offline'. 270 */ 271 kmutex_t tp_sess_list_lock; 272 kcondvar_t tp_sess_complete; 273 list_t tp_sess_list; 274 275 uint_t tp_srp_enabled; 276 ibt_srv_hdl_t tp_ibt_svc_hdl; 277 ibt_srv_desc_t tp_ibt_svc_desc; 278 ib_svc_id_t tp_ibt_svc_id; 279 scsi_devid_desc_t *tp_scsi_devid; 280 uint8_t tp_srp_port_id[SRP_PORT_ID_LEN]; 281 282 uint_t tp_nports; 283 srpt_hw_port_t *tp_hw_port; 284} srpt_target_port_t; 285 286/* 287 * SRP Target hardware device. A SRP Target hardware device 288 * is an IB HCA. All ports of the HCA comprise a single 289 * I/O Controller that is registered with the IB Device 290 * Managment Agent. 291 */ 292typedef struct srpt_ioc_s { 293 list_node_t ioc_node; 294 295 krwlock_t ioc_rwlock; 296 ibt_hca_hdl_t ioc_ibt_hdl; 297 ibt_hca_attr_t ioc_attr; 298 ib_guid_t ioc_guid; 299 300 /* 301 * By default each HCA is a single SRP.T10 service based on 302 * the HCA GUID. We have implemented the target here as a 303 * pointer to facilitate moving to a list of targets if 304 * appropriate down the road. 305 */ 306 srpt_target_port_t *ioc_tgt_port; 307 308 309 /* 310 * Each HCA registers a single I/O Controller with the 311 * IB Device Management Agent. 312 */ 313 ibdma_hdl_t ioc_ibdma_hdl; 314 ib_dm_ioc_ctrl_profile_t ioc_profile; 315 ib_dm_srv_t ioc_svc; 316 317 ibt_pd_hdl_t ioc_pd_hdl; 318 ibt_srq_sizes_t ioc_srq_attr; 319 ibt_srq_hdl_t ioc_srq_hdl; 320 321 /* 322 * The I/O Controller pool of IU resources allocated 323 * at controller creation. 324 */ 325 uint32_t ioc_num_iu_entries; 326 srpt_iu_t *ioc_iu_pool; 327 ibt_mr_hdl_t ioc_iu_mr_hdl; 328 void *ioc_iu_bufs; /* iu buffer space */ 329 330 /* 331 * Each I/O Controller has it's own data buffer 332 * vmem arena. Pool is created at controller creation, 333 * and expanded as required. This keeps IB memory 334 * registrations to a minimum in the data path. 335 */ 336 struct srpt_vmem_pool_s *ioc_dbuf_pool; 337 stmf_dbuf_store_t *ioc_stmf_ds; 338} srpt_ioc_t; 339 340/* 341 * Memory regions 342 */ 343typedef struct srpt_mr_s { 344 ibt_mr_hdl_t mr_hdl; 345 ib_vaddr_t mr_va; 346 ib_memlen_t mr_len; 347 ibt_lkey_t mr_lkey; 348 ibt_rkey_t mr_rkey; 349 avl_node_t mr_avl; 350} srpt_mr_t; 351 352/* 353 * SRP Target vmem arena definition 354 */ 355typedef struct srpt_vmem_pool_s { 356 srpt_ioc_t *svp_ioc; 357 ib_memlen_t svp_chunksize; 358 vmem_t *svp_vmem; 359 uint64_t svp_total_size; 360 uint64_t svp_max_size; 361 avl_tree_t svp_mr_list; 362 krwlock_t svp_lock; 363 ibt_mr_flags_t svp_flags; 364} srpt_vmem_pool_t; 365 366/* 367 * SRP port provider data buffer, allocated and freed 368 * via calls to the IOC datastore. 369 */ 370typedef struct srpt_ds_dbuf_s { 371 stmf_data_buf_t *db_stmf_buf; 372 srpt_ioc_t *db_ioc; 373 ibt_mr_hdl_t db_mr_hdl; 374 ibt_wr_ds_t db_sge; 375 srpt_iu_t *db_iu; 376} srpt_ds_dbuf_t; 377 378/* 379 * SRP Target service state 380 */ 381typedef enum { 382 SRPT_SVC_DISABLED, 383 SRPT_SVC_ENABLED 384} srpt_svc_state_t; 385 386typedef struct { 387 ddi_modhandle_t ibdmah; 388 ibdma_hdl_t (*ibdma_register)(ib_guid_t, 389 ib_dm_ioc_ctrl_profile_t *, ib_dm_srv_t *); 390 ibdma_status_t (*ibdma_unregister)(ibdma_hdl_t); 391 ibdma_status_t (*ibdma_update)(ibdma_hdl_t, 392 ib_dm_ioc_ctrl_profile_t *, ib_dm_srv_t *); 393} srpt_ibdma_ops_t; 394 395/* 396 * SRP Target protocol driver context data structure, maintaining 397 * the global state of the protocol. 398 */ 399typedef struct srpt_ctxt_s { 400 dev_info_t *sc_dip; 401 krwlock_t sc_rwlock; 402 srpt_svc_state_t sc_svc_state; 403 404 ibt_clnt_hdl_t sc_ibt_hdl; 405 406 /* 407 * SRP Target I/O Controllers. Each IBT HCA represents an 408 * I/O Controller. Must hold rwlock as a writer to update. 409 */ 410 list_t sc_ioc_list; 411 uint_t sc_num_iocs; 412 413 /* Back-end COMSTAR port provider interface. */ 414 stmf_port_provider_t *sc_pp; 415 416 /* IBDMA entry points */ 417 srpt_ibdma_ops_t sc_ibdma_ops; 418} srpt_ctxt_t; 419 420typedef struct srpt_iu_data_s { 421 union { 422 uint8_t srp_op; 423 srp_cmd_req_t srp_cmd; 424 srp_tsk_mgmt_t srp_tsk_mgmt; 425 srp_i_logout_t srp_i_logout; 426 srp_rsp_t srp_rsp; 427 } rx_iu; 428} srpt_iu_data_t; 429 430extern srpt_ctxt_t *srpt_ctxt; 431 432/* 433 * For Non recoverable or Major Errors 434 */ 435#define SRPT_LOG_L0 0 436 437/* 438 * For additional information on Non recoverable errors and 439 * warnings/informational message for sys-admin types. 440 */ 441#define SRPT_LOG_L1 1 442 443/* 444 * debug only 445 * for more verbose trace than L1, for e.g. recoverable errors, 446 * or intersting trace 447 */ 448#define SRPT_LOG_L2 2 449 450/* 451 * debug only 452 * for more verbose trace than L2, for e.g. printing function entries.... 453 */ 454#define SRPT_LOG_L3 3 455 456/* 457 * debug only 458 * for more verbose trace than L3, for e.g. printing minor function entries... 459 */ 460#define SRPT_LOG_L4 4 461 462/* 463 * srpt_errlevel can be set in the debugger to enable additional logging. 464 * You can also add set srpt:srpt_errlevel={0,1,2,3,4} in /etc/system. 465 * The default log level is L2 for debug builds, otherwise L1. 466 */ 467#ifdef DEBUG 468#define SRPT_LOG_DEFAULT_LEVEL SRPT_LOG_L2 469#else 470#define SRPT_LOG_DEFAULT_LEVEL SRPT_LOG_L1 471#endif 472 473extern uint_t srpt_errlevel; 474 475 476#define SRPT_DPRINTF_L0(...) cmn_err(CE_WARN, __VA_ARGS__) 477#define SRPT_DPRINTF_L1(...) cmn_err(CE_NOTE, __VA_ARGS__) 478#define SRPT_DPRINTF_L2(...) if (srpt_errlevel >= SRPT_LOG_L2) { \ 479 cmn_err(CE_NOTE, __VA_ARGS__);\ 480 } 481#ifdef DEBUG 482#define SRPT_DPRINTF_L3(...) if (srpt_errlevel >= SRPT_LOG_L3) { \ 483 cmn_err(CE_NOTE, __VA_ARGS__);\ 484 } 485#define SRPT_DPRINTF_L4(...) if (srpt_errlevel >= SRPT_LOG_L4) { \ 486 cmn_err(CE_NOTE, __VA_ARGS__);\ 487 } 488#else 489#define SRPT_DPRINTF_L3 0 && 490#define SRPT_DPRINTF_L4 0 && 491#endif 492 493#ifdef __cplusplus 494} 495#endif 496 497#endif /* _SRPT_IMPL_H_ */ 498