1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, 2008 Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy$FreeBSD$ 29178786Skmacy 30178786Skmacy***************************************************************************/ 31178786Skmacy#ifndef __IWCH_PROVIDER_H__ 32178786Skmacy#define __IWCH_PROVIDER_H__ 33178786Skmacy 34237263Snp#include <rdma/ib_verbs.h> 35178786Skmacy 36178786Skmacystruct iwch_pd { 37178786Skmacy struct ib_pd ibpd; 38178786Skmacy u32 pdid; 39178786Skmacy struct iwch_dev *rhp; 40178786Skmacy}; 41178786Skmacy 42178786Skmacy#ifndef container_of 43178786Skmacy#define container_of(p, stype, field) ((stype *)(((uint8_t *)(p)) - offsetof(stype, field))) 44178786Skmacy#endif 45178786Skmacystatic __inline struct iwch_pd * 46178786Skmacyto_iwch_pd(struct ib_pd *ibpd) 47178786Skmacy{ 48178786Skmacy return container_of(ibpd, struct iwch_pd, ibpd); 49178786Skmacy} 50178786Skmacy 51178786Skmacystruct tpt_attributes { 52178786Skmacy u32 stag; 53178786Skmacy u32 state:1; 54178786Skmacy u32 type:2; 55178786Skmacy u32 rsvd:1; 56178786Skmacy enum tpt_mem_perm perms; 57178786Skmacy u32 remote_invaliate_disable:1; 58178786Skmacy u32 zbva:1; 59178786Skmacy u32 mw_bind_enable:1; 60178786Skmacy u32 page_size:5; 61178786Skmacy 62178786Skmacy u32 pdid; 63178786Skmacy u32 qpid; 64178786Skmacy u32 pbl_addr; 65178786Skmacy u32 len; 66178786Skmacy u64 va_fbo; 67178786Skmacy u32 pbl_size; 68178786Skmacy}; 69178786Skmacy 70178786Skmacystruct iwch_mr { 71178786Skmacy struct ib_mr ibmr; 72178786Skmacy struct ib_umem *umem; 73178786Skmacy struct iwch_dev *rhp; 74178786Skmacy u64 kva; 75178786Skmacy struct tpt_attributes attr; 76178786Skmacy}; 77178786Skmacy 78178786Skmacytypedef struct iwch_mw iwch_mw_handle; 79178786Skmacy 80178786Skmacystatic __inline struct iwch_mr * 81178786Skmacyto_iwch_mr(struct ib_mr *ibmr) 82178786Skmacy{ 83178786Skmacy return container_of(ibmr, struct iwch_mr, ibmr); 84178786Skmacy} 85178786Skmacy 86178786Skmacystruct iwch_mw { 87178786Skmacy struct ib_mw ibmw; 88178786Skmacy struct iwch_dev *rhp; 89178786Skmacy u64 kva; 90178786Skmacy struct tpt_attributes attr; 91178786Skmacy}; 92178786Skmacy 93178786Skmacystatic __inline struct iwch_mw * 94178786Skmacyto_iwch_mw(struct ib_mw *ibmw) 95178786Skmacy{ 96178786Skmacy return container_of(ibmw, struct iwch_mw, ibmw); 97178786Skmacy} 98178786Skmacy 99178786Skmacystruct iwch_cq { 100178786Skmacy struct ib_cq ibcq; 101178786Skmacy struct iwch_dev *rhp; 102178786Skmacy struct t3_cq cq; 103178786Skmacy struct mtx lock; 104178786Skmacy int refcnt; 105178786Skmacy u32 /* __user */ *user_rptr_addr; 106178786Skmacy}; 107178786Skmacy 108178786Skmacystatic __inline struct iwch_cq * 109178786Skmacyto_iwch_cq(struct ib_cq *ibcq) 110178786Skmacy{ 111178786Skmacy return container_of(ibcq, struct iwch_cq, ibcq); 112178786Skmacy} 113178786Skmacy 114178786Skmacyenum IWCH_QP_FLAGS { 115178786Skmacy QP_QUIESCED = 0x01 116178786Skmacy}; 117178786Skmacy 118178786Skmacystruct iwch_mpa_attributes { 119237263Snp u8 initiator; 120178786Skmacy u8 recv_marker_enabled; 121178786Skmacy u8 xmit_marker_enabled; /* iWARP: enable inbound Read Resp. */ 122178786Skmacy u8 crc_enabled; 123178786Skmacy u8 version; /* 0 or 1 */ 124178786Skmacy}; 125178786Skmacy 126178786Skmacystruct iwch_qp_attributes { 127178786Skmacy u32 scq; 128178786Skmacy u32 rcq; 129178786Skmacy u32 sq_num_entries; 130178786Skmacy u32 rq_num_entries; 131178786Skmacy u32 sq_max_sges; 132178786Skmacy u32 sq_max_sges_rdma_write; 133178786Skmacy u32 rq_max_sges; 134178786Skmacy u32 state; 135178786Skmacy u8 enable_rdma_read; 136178786Skmacy u8 enable_rdma_write; /* enable inbound Read Resp. */ 137178786Skmacy u8 enable_bind; 138178786Skmacy u8 enable_mmid0_fastreg; /* Enable STAG0 + Fast-register */ 139178786Skmacy /* 140178786Skmacy * Next QP state. If specify the current state, only the 141178786Skmacy * QP attributes will be modified. 142178786Skmacy */ 143178786Skmacy u32 max_ord; 144178786Skmacy u32 max_ird; 145178786Skmacy u32 pd; /* IN */ 146178786Skmacy u32 next_state; 147178786Skmacy char terminate_buffer[52]; 148178786Skmacy u32 terminate_msg_len; 149178786Skmacy u8 is_terminate_local; 150178786Skmacy struct iwch_mpa_attributes mpa_attr; /* IN-OUT */ 151178786Skmacy struct iwch_ep *llp_stream_handle; 152178786Skmacy char *stream_msg_buf; /* Last stream msg. before Idle -> RTS */ 153178786Skmacy u32 stream_msg_buf_len; /* Only on Idle -> RTS */ 154178786Skmacy}; 155178786Skmacy 156178786Skmacystruct iwch_qp { 157178786Skmacy struct ib_qp ibqp; 158178786Skmacy struct iwch_dev *rhp; 159178786Skmacy struct iwch_ep *ep; 160178786Skmacy struct iwch_qp_attributes attr; 161178786Skmacy struct t3_wq wq; 162178786Skmacy struct mtx lock; 163178786Skmacy int refcnt; 164178786Skmacy enum IWCH_QP_FLAGS flags; 165178786Skmacy struct callout timer; 166178786Skmacy}; 167178786Skmacy 168178786Skmacystatic __inline int 169178786Skmacyqp_quiesced(struct iwch_qp *qhp) 170178786Skmacy{ 171178786Skmacy return qhp->flags & QP_QUIESCED; 172178786Skmacy} 173178786Skmacy 174178786Skmacystatic __inline struct iwch_qp * 175178786Skmacyto_iwch_qp(struct ib_qp *ibqp) 176178786Skmacy{ 177178786Skmacy return container_of(ibqp, struct iwch_qp, ibqp); 178178786Skmacy} 179178786Skmacy 180178786Skmacyvoid iwch_qp_add_ref(struct ib_qp *qp); 181178786Skmacyvoid iwch_qp_rem_ref(struct ib_qp *qp); 182178786Skmacy 183178786Skmacystruct iwch_ucontext { 184178786Skmacy struct ib_ucontext ibucontext; 185178786Skmacy struct cxio_ucontext uctx; 186178786Skmacy u32 key; 187178786Skmacy struct mtx mmap_lock; 188178786Skmacy TAILQ_HEAD( ,iwch_mm_entry) mmaps; 189178786Skmacy}; 190178786Skmacy 191178786Skmacystatic __inline struct iwch_ucontext * 192178786Skmacyto_iwch_ucontext(struct ib_ucontext *c) 193178786Skmacy{ 194178786Skmacy return container_of(c, struct iwch_ucontext, ibucontext); 195178786Skmacy} 196178786Skmacy 197178786Skmacystruct iwch_mm_entry { 198178786Skmacy TAILQ_ENTRY(iwch_mm_entry) entry; 199178786Skmacy u64 addr; 200178786Skmacy u32 key; 201178786Skmacy unsigned len; 202178786Skmacy}; 203178786Skmacy 204178786Skmacystatic __inline struct iwch_mm_entry * 205178786Skmacyremove_mmap(struct iwch_ucontext *ucontext, 206178786Skmacy u32 key, unsigned len) 207178786Skmacy{ 208178786Skmacy struct iwch_mm_entry *tmp, *mm; 209178786Skmacy 210178786Skmacy mtx_lock(&ucontext->mmap_lock); 211178786Skmacy TAILQ_FOREACH_SAFE(mm, &ucontext->mmaps, entry, tmp) { 212178786Skmacy if (mm->key == key && mm->len == len) { 213178786Skmacy TAILQ_REMOVE(&ucontext->mmaps, mm, entry); 214178786Skmacy mtx_unlock(&ucontext->mmap_lock); 215178786Skmacy CTR4(KTR_IW_CXGB, "%s key 0x%x addr 0x%llx len %d\n", __FUNCTION__, 216178786Skmacy key, (unsigned long long) mm->addr, mm->len); 217178786Skmacy return mm; 218178786Skmacy } 219178786Skmacy } 220178786Skmacy mtx_unlock(&ucontext->mmap_lock); 221178786Skmacy 222178786Skmacy return NULL; 223178786Skmacy} 224178786Skmacy 225178786Skmacystatic __inline void 226178786Skmacyinsert_mmap(struct iwch_ucontext *ucontext, 227178786Skmacy struct iwch_mm_entry *mm) 228178786Skmacy{ 229178786Skmacy mtx_lock(&ucontext->mmap_lock); 230178786Skmacy CTR4(KTR_IW_CXGB, "%s key 0x%x addr 0x%llx len %d\n", __FUNCTION__, 231178786Skmacy mm->key, (unsigned long long) mm->addr, mm->len); 232178786Skmacy TAILQ_INSERT_TAIL(&ucontext->mmaps, mm, entry); 233178786Skmacy mtx_unlock(&ucontext->mmap_lock); 234178786Skmacy} 235178786Skmacy 236178786Skmacyenum iwch_qp_attr_mask { 237178786Skmacy IWCH_QP_ATTR_NEXT_STATE = 1 << 0, 238178786Skmacy IWCH_QP_ATTR_ENABLE_RDMA_READ = 1 << 7, 239178786Skmacy IWCH_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8, 240178786Skmacy IWCH_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9, 241178786Skmacy IWCH_QP_ATTR_MAX_ORD = 1 << 11, 242178786Skmacy IWCH_QP_ATTR_MAX_IRD = 1 << 12, 243178786Skmacy IWCH_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22, 244178786Skmacy IWCH_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23, 245178786Skmacy IWCH_QP_ATTR_MPA_ATTR = 1 << 24, 246178786Skmacy IWCH_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25, 247178786Skmacy IWCH_QP_ATTR_VALID_MODIFY = (IWCH_QP_ATTR_ENABLE_RDMA_READ | 248178786Skmacy IWCH_QP_ATTR_ENABLE_RDMA_WRITE | 249178786Skmacy IWCH_QP_ATTR_MAX_ORD | 250178786Skmacy IWCH_QP_ATTR_MAX_IRD | 251178786Skmacy IWCH_QP_ATTR_LLP_STREAM_HANDLE | 252178786Skmacy IWCH_QP_ATTR_STREAM_MSG_BUFFER | 253178786Skmacy IWCH_QP_ATTR_MPA_ATTR | 254178786Skmacy IWCH_QP_ATTR_QP_CONTEXT_ACTIVATE) 255178786Skmacy}; 256178786Skmacy 257178786Skmacyint iwch_modify_qp(struct iwch_dev *rhp, 258178786Skmacy struct iwch_qp *qhp, 259178786Skmacy enum iwch_qp_attr_mask mask, 260178786Skmacy struct iwch_qp_attributes *attrs, 261178786Skmacy int internal); 262178786Skmacy 263178786Skmacyenum iwch_qp_state { 264178786Skmacy IWCH_QP_STATE_IDLE, 265178786Skmacy IWCH_QP_STATE_RTS, 266178786Skmacy IWCH_QP_STATE_ERROR, 267178786Skmacy IWCH_QP_STATE_TERMINATE, 268178786Skmacy IWCH_QP_STATE_CLOSING, 269178786Skmacy IWCH_QP_STATE_TOT 270178786Skmacy}; 271178786Skmacy 272178786Skmacystatic __inline int 273178786Skmacyiwch_convert_state(enum ib_qp_state ib_state) 274178786Skmacy{ 275178786Skmacy switch (ib_state) { 276178786Skmacy case IB_QPS_RESET: 277178786Skmacy case IB_QPS_INIT: 278178786Skmacy return IWCH_QP_STATE_IDLE; 279178786Skmacy case IB_QPS_RTS: 280178786Skmacy return IWCH_QP_STATE_RTS; 281178786Skmacy case IB_QPS_SQD: 282178786Skmacy return IWCH_QP_STATE_CLOSING; 283178786Skmacy case IB_QPS_SQE: 284178786Skmacy return IWCH_QP_STATE_TERMINATE; 285178786Skmacy case IB_QPS_ERR: 286178786Skmacy return IWCH_QP_STATE_ERROR; 287178786Skmacy default: 288178786Skmacy return -1; 289178786Skmacy } 290178786Skmacy} 291178786Skmacy 292178786Skmacystatic __inline u32 293178786Skmacyiwch_ib_to_tpt_access(int acc) 294178786Skmacy{ 295178786Skmacy return (acc & IB_ACCESS_REMOTE_WRITE ? TPT_REMOTE_WRITE : 0) | 296178786Skmacy (acc & IB_ACCESS_REMOTE_READ ? TPT_REMOTE_READ : 0) | 297178786Skmacy (acc & IB_ACCESS_LOCAL_WRITE ? TPT_LOCAL_WRITE : 0) | 298178786Skmacy TPT_LOCAL_READ; 299178786Skmacy} 300178786Skmacy 301178786Skmacystatic __inline u32 302178786Skmacyiwch_ib_to_mwbind_access(int acc) 303178786Skmacy{ 304178786Skmacy return (acc & IB_ACCESS_REMOTE_WRITE ? T3_MEM_ACCESS_REM_WRITE : 0) | 305178786Skmacy (acc & IB_ACCESS_REMOTE_READ ? T3_MEM_ACCESS_REM_READ : 0) | 306178786Skmacy (acc & IB_ACCESS_LOCAL_WRITE ? T3_MEM_ACCESS_LOCAL_WRITE : 0) | 307178786Skmacy T3_MEM_ACCESS_LOCAL_READ; 308178786Skmacy} 309178786Skmacy 310178786Skmacyenum iwch_mmid_state { 311178786Skmacy IWCH_STAG_STATE_VALID, 312178786Skmacy IWCH_STAG_STATE_INVALID 313178786Skmacy}; 314178786Skmacy 315178786Skmacyenum iwch_qp_query_flags { 316178786Skmacy IWCH_QP_QUERY_CONTEXT_NONE = 0x0, /* No ctx; Only attrs */ 317178786Skmacy IWCH_QP_QUERY_CONTEXT_GET = 0x1, /* Get ctx + attrs */ 318178786Skmacy IWCH_QP_QUERY_CONTEXT_SUSPEND = 0x2, /* Not Supported */ 319178786Skmacy 320178786Skmacy /* 321178786Skmacy * Quiesce QP context; Consumer 322178786Skmacy * will NOT replay outstanding WR 323178786Skmacy */ 324178786Skmacy IWCH_QP_QUERY_CONTEXT_QUIESCE = 0x4, 325178786Skmacy IWCH_QP_QUERY_CONTEXT_REMOVE = 0x8, 326178786Skmacy IWCH_QP_QUERY_TEST_USERWRITE = 0x32 /* Test special */ 327178786Skmacy}; 328178786Skmacy 329178786Skmacyint iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 330178786Skmacy struct ib_send_wr **bad_wr); 331178786Skmacyint iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 332178786Skmacy struct ib_recv_wr **bad_wr); 333178786Skmacyint iwch_bind_mw(struct ib_qp *qp, 334178786Skmacy struct ib_mw *mw, 335178786Skmacy struct ib_mw_bind *mw_bind); 336178786Skmacyint iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); 337178786Skmacyint iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg); 338178786Skmacyint iwch_register_device(struct iwch_dev *dev); 339178786Skmacyvoid iwch_unregister_device(struct iwch_dev *dev); 340178786Skmacyvoid stop_read_rep_timer(struct iwch_qp *qhp); 341178786Skmacyint iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php, 342178786Skmacy struct iwch_mr *mhp, 343237263Snp int shift); 344178786Skmacyint iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php, 345178786Skmacy struct iwch_mr *mhp, 346178786Skmacy int shift, 347178786Skmacy int npages); 348237263Snpint iwch_alloc_pbl(struct iwch_mr *mhp, int npages); 349237263Snpvoid iwch_free_pbl(struct iwch_mr *mhp); 350237263Snpint iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset); 351178786Skmacyint build_phys_page_list(struct ib_phys_buf *buffer_list, 352178786Skmacy int num_phys_buf, 353178786Skmacy u64 *iova_start, 354178786Skmacy u64 *total_size, 355178786Skmacy int *npages, 356178786Skmacy int *shift, 357178786Skmacy __be64 **page_list); 358178786Skmacy 359178786Skmacy 360178786Skmacy#define IWCH_NODE_DESC "cxgb3 Chelsio Communications" 361178786Skmacy 362178786Skmacy#endif 363