1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, 2008 Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy$FreeBSD: releng/10.3/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_wr.h 237263 2012-06-19 07:34:13Z np $ 29178786Skmacy 30178786Skmacy***************************************************************************/ 31178786Skmacy#ifndef __CXIO_WR_H__ 32178786Skmacy#define __CXIO_WR_H__ 33178786Skmacy#define T3_MAX_SGE 4 34178786Skmacy#define T3_MAX_INLINE 64 35237263Snp#define T3_STAG0_PBL_SIZE (2 * T3_MAX_SGE << 3) 36237263Snp#define T3_STAG0_MAX_PBE_LEN (128 * 1024 * 1024) 37237263Snp#define T3_STAG0_PAGE_SHIFT 15 38178786Skmacy 39178786Skmacy#define Q_EMPTY(rptr,wptr) ((rptr)==(wptr)) 40178786Skmacy#define Q_FULL(rptr,wptr,size_log2) ( (((wptr)-(rptr))>>(size_log2)) && \ 41178786Skmacy ((rptr)!=(wptr)) ) 42178786Skmacy#define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1)) 43178786Skmacy#define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr))) 44178786Skmacy#define Q_COUNT(rptr,wptr) ((wptr)-(rptr)) 45178786Skmacy#define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1)) 46178786Skmacy 47178786Skmacystatic __inline void 48178786Skmacyring_doorbell(void /* __iomem */ *doorbell, u32 qpid) 49178786Skmacy{ 50178786Skmacy writel(doorbell, ((1<<31) | qpid)); 51178786Skmacy} 52178786Skmacy 53178786Skmacy#define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) & 0x80000000 )) 54178786Skmacy 55178786Skmacyenum t3_wr_flags { 56178786Skmacy T3_COMPLETION_FLAG = 0x01, 57178786Skmacy T3_NOTIFY_FLAG = 0x02, 58178786Skmacy T3_SOLICITED_EVENT_FLAG = 0x04, 59178786Skmacy T3_READ_FENCE_FLAG = 0x08, 60178786Skmacy T3_LOCAL_FENCE_FLAG = 0x10 61178786Skmacy} __attribute__ ((packed)); 62178786Skmacy 63178786Skmacyenum t3_wr_opcode { 64178786Skmacy T3_WR_BP = FW_WROPCODE_RI_BYPASS, 65178786Skmacy T3_WR_SEND = FW_WROPCODE_RI_SEND, 66178786Skmacy T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE, 67178786Skmacy T3_WR_READ = FW_WROPCODE_RI_RDMA_READ, 68178786Skmacy T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV, 69178786Skmacy T3_WR_BIND = FW_WROPCODE_RI_BIND_MW, 70178786Skmacy T3_WR_RCV = FW_WROPCODE_RI_RECEIVE, 71178786Skmacy T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT, 72178786Skmacy T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP 73178786Skmacy} __attribute__ ((packed)); 74178786Skmacy 75178786Skmacyenum t3_rdma_opcode { 76178786Skmacy T3_RDMA_WRITE, /* IETF RDMAP v1.0 ... */ 77178786Skmacy T3_READ_REQ, 78178786Skmacy T3_READ_RESP, 79178786Skmacy T3_SEND, 80178786Skmacy T3_SEND_WITH_INV, 81178786Skmacy T3_SEND_WITH_SE, 82178786Skmacy T3_SEND_WITH_SE_INV, 83178786Skmacy T3_TERMINATE, 84178786Skmacy T3_RDMA_INIT, /* CHELSIO RI specific ... */ 85178786Skmacy T3_BIND_MW, 86178786Skmacy T3_FAST_REGISTER, 87178786Skmacy T3_LOCAL_INV, 88178786Skmacy T3_QP_MOD, 89178786Skmacy T3_BYPASS 90178786Skmacy} __attribute__ ((packed)); 91178786Skmacy 92178786Skmacystatic inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop) 93178786Skmacy{ 94178786Skmacy switch (wrop) { 95178786Skmacy case T3_WR_BP: return T3_BYPASS; 96178786Skmacy case T3_WR_SEND: return T3_SEND; 97178786Skmacy case T3_WR_WRITE: return T3_RDMA_WRITE; 98178786Skmacy case T3_WR_READ: return T3_READ_REQ; 99178786Skmacy case T3_WR_INV_STAG: return T3_LOCAL_INV; 100178786Skmacy case T3_WR_BIND: return T3_BIND_MW; 101178786Skmacy case T3_WR_INIT: return T3_RDMA_INIT; 102178786Skmacy case T3_WR_QP_MOD: return T3_QP_MOD; 103178786Skmacy default: break; 104178786Skmacy } 105178786Skmacy return -1; 106178786Skmacy} 107178786Skmacy 108178786Skmacy 109178786Skmacy/* Work request id */ 110178786Skmacyunion t3_wrid { 111178786Skmacy struct { 112178786Skmacy u32 hi; 113178786Skmacy u32 low; 114178786Skmacy } id0; 115178786Skmacy u64 id1; 116178786Skmacy}; 117178786Skmacy 118178786Skmacy#define WRID(wrid) (wrid.id1) 119178786Skmacy#define WRID_GEN(wrid) (wrid.id0.wr_gen) 120178786Skmacy#define WRID_IDX(wrid) (wrid.id0.wr_idx) 121178786Skmacy#define WRID_LO(wrid) (wrid.id0.wr_lo) 122178786Skmacy 123178786Skmacystruct fw_riwrh { 124178786Skmacy __be32 op_seop_flags; 125178786Skmacy __be32 gen_tid_len; 126178786Skmacy}; 127178786Skmacy 128178786Skmacy#define S_FW_RIWR_OP 24 129178786Skmacy#define M_FW_RIWR_OP 0xff 130178786Skmacy#define V_FW_RIWR_OP(x) ((x) << S_FW_RIWR_OP) 131178786Skmacy#define G_FW_RIWR_OP(x) ((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP) 132178786Skmacy 133178786Skmacy#define S_FW_RIWR_SOPEOP 22 134178786Skmacy#define M_FW_RIWR_SOPEOP 0x3 135178786Skmacy#define V_FW_RIWR_SOPEOP(x) ((x) << S_FW_RIWR_SOPEOP) 136178786Skmacy 137178786Skmacy#define S_FW_RIWR_FLAGS 8 138178786Skmacy#define M_FW_RIWR_FLAGS 0x3fffff 139178786Skmacy#define V_FW_RIWR_FLAGS(x) ((x) << S_FW_RIWR_FLAGS) 140178786Skmacy#define G_FW_RIWR_FLAGS(x) ((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS) 141178786Skmacy 142178786Skmacy#define S_FW_RIWR_TID 8 143178786Skmacy#define V_FW_RIWR_TID(x) ((x) << S_FW_RIWR_TID) 144178786Skmacy 145178786Skmacy#define S_FW_RIWR_LEN 0 146178786Skmacy#define V_FW_RIWR_LEN(x) ((x) << S_FW_RIWR_LEN) 147178786Skmacy 148178786Skmacy#define S_FW_RIWR_GEN 31 149178786Skmacy#define V_FW_RIWR_GEN(x) ((x) << S_FW_RIWR_GEN) 150178786Skmacy 151178786Skmacystruct t3_sge { 152178786Skmacy __be32 stag; 153178786Skmacy __be32 len; 154178786Skmacy __be64 to; 155178786Skmacy}; 156178786Skmacy 157178786Skmacy/* If num_sgle is zero, flit 5+ contains immediate data.*/ 158178786Skmacystruct t3_send_wr { 159178786Skmacy struct fw_riwrh wrh; /* 0 */ 160178786Skmacy union t3_wrid wrid; /* 1 */ 161178786Skmacy 162178786Skmacy u8 rdmaop; /* 2 */ 163178786Skmacy u8 reserved[3]; 164178786Skmacy __be32 rem_stag; 165178786Skmacy __be32 plen; /* 3 */ 166178786Skmacy __be32 num_sgle; 167178786Skmacy struct t3_sge sgl[T3_MAX_SGE]; /* 4+ */ 168178786Skmacy}; 169178786Skmacy 170178786Skmacystruct t3_local_inv_wr { 171178786Skmacy struct fw_riwrh wrh; /* 0 */ 172178786Skmacy union t3_wrid wrid; /* 1 */ 173178786Skmacy __be32 stag; /* 2 */ 174178786Skmacy __be32 reserved3; 175178786Skmacy}; 176178786Skmacy 177178786Skmacystruct t3_rdma_write_wr { 178178786Skmacy struct fw_riwrh wrh; /* 0 */ 179178786Skmacy union t3_wrid wrid; /* 1 */ 180178786Skmacy u8 rdmaop; /* 2 */ 181178786Skmacy u8 reserved[3]; 182178786Skmacy __be32 stag_sink; 183178786Skmacy __be64 to_sink; /* 3 */ 184178786Skmacy __be32 plen; /* 4 */ 185178786Skmacy __be32 num_sgle; 186178786Skmacy struct t3_sge sgl[T3_MAX_SGE]; /* 5+ */ 187178786Skmacy}; 188178786Skmacy 189178786Skmacystruct t3_rdma_read_wr { 190178786Skmacy struct fw_riwrh wrh; /* 0 */ 191178786Skmacy union t3_wrid wrid; /* 1 */ 192178786Skmacy u8 rdmaop; /* 2 */ 193178786Skmacy u8 reserved[3]; 194178786Skmacy __be32 rem_stag; 195178786Skmacy __be64 rem_to; /* 3 */ 196178786Skmacy __be32 local_stag; /* 4 */ 197178786Skmacy __be32 local_len; 198178786Skmacy __be64 local_to; /* 5 */ 199178786Skmacy}; 200178786Skmacy 201178786Skmacyenum t3_addr_type { 202178786Skmacy T3_VA_BASED_TO = 0x0, 203178786Skmacy T3_ZERO_BASED_TO = 0x1 204178786Skmacy} __attribute__ ((packed)); 205178786Skmacy 206178786Skmacyenum t3_mem_perms { 207178786Skmacy T3_MEM_ACCESS_LOCAL_READ = 0x1, 208178786Skmacy T3_MEM_ACCESS_LOCAL_WRITE = 0x2, 209178786Skmacy T3_MEM_ACCESS_REM_READ = 0x4, 210178786Skmacy T3_MEM_ACCESS_REM_WRITE = 0x8 211178786Skmacy} __attribute__ ((packed)); 212178786Skmacy 213178786Skmacystruct t3_bind_mw_wr { 214178786Skmacy struct fw_riwrh wrh; /* 0 */ 215178786Skmacy union t3_wrid wrid; /* 1 */ 216178786Skmacy u16 reserved; /* 2 */ 217178786Skmacy u8 type; 218178786Skmacy u8 perms; 219178786Skmacy __be32 mr_stag; 220178786Skmacy __be32 mw_stag; /* 3 */ 221178786Skmacy __be32 mw_len; 222178786Skmacy __be64 mw_va; /* 4 */ 223178786Skmacy __be32 mr_pbl_addr; /* 5 */ 224178786Skmacy u8 reserved2[3]; 225178786Skmacy u8 mr_pagesz; 226178786Skmacy}; 227178786Skmacy 228178786Skmacystruct t3_receive_wr { 229178786Skmacy struct fw_riwrh wrh; /* 0 */ 230178786Skmacy union t3_wrid wrid; /* 1 */ 231178786Skmacy u8 pagesz[T3_MAX_SGE]; 232178786Skmacy __be32 num_sgle; /* 2 */ 233178786Skmacy struct t3_sge sgl[T3_MAX_SGE]; /* 3+ */ 234178786Skmacy __be32 pbl_addr[T3_MAX_SGE]; 235178786Skmacy}; 236178786Skmacy 237178786Skmacystruct t3_bypass_wr { 238178786Skmacy struct fw_riwrh wrh; 239178786Skmacy union t3_wrid wrid; /* 1 */ 240178786Skmacy}; 241178786Skmacy 242178786Skmacystruct t3_modify_qp_wr { 243178786Skmacy struct fw_riwrh wrh; /* 0 */ 244178786Skmacy union t3_wrid wrid; /* 1 */ 245178786Skmacy __be32 flags; /* 2 */ 246178786Skmacy __be32 quiesce; /* 2 */ 247178786Skmacy __be32 max_ird; /* 3 */ 248178786Skmacy __be32 max_ord; /* 3 */ 249178786Skmacy __be64 sge_cmd; /* 4 */ 250178786Skmacy __be64 ctx1; /* 5 */ 251178786Skmacy __be64 ctx0; /* 6 */ 252178786Skmacy}; 253178786Skmacy 254178786Skmacyenum t3_modify_qp_flags { 255178786Skmacy MODQP_QUIESCE = 0x01, 256178786Skmacy MODQP_MAX_IRD = 0x02, 257178786Skmacy MODQP_MAX_ORD = 0x04, 258178786Skmacy MODQP_WRITE_EC = 0x08, 259178786Skmacy MODQP_READ_EC = 0x10, 260178786Skmacy}; 261178786Skmacy 262178786Skmacy 263178786Skmacyenum t3_mpa_attrs { 264178786Skmacy uP_RI_MPA_RX_MARKER_ENABLE = 0x1, 265178786Skmacy uP_RI_MPA_TX_MARKER_ENABLE = 0x2, 266178786Skmacy uP_RI_MPA_CRC_ENABLE = 0x4, 267178786Skmacy uP_RI_MPA_IETF_ENABLE = 0x8 268178786Skmacy} __attribute__ ((packed)); 269178786Skmacy 270178786Skmacyenum t3_qp_caps { 271178786Skmacy uP_RI_QP_RDMA_READ_ENABLE = 0x01, 272178786Skmacy uP_RI_QP_RDMA_WRITE_ENABLE = 0x02, 273178786Skmacy uP_RI_QP_BIND_ENABLE = 0x04, 274178786Skmacy uP_RI_QP_FAST_REGISTER_ENABLE = 0x08, 275178786Skmacy uP_RI_QP_STAG0_ENABLE = 0x10 276178786Skmacy} __attribute__ ((packed)); 277178786Skmacy 278237263Snpenum rdma_init_rtr_types { 279237263Snp RTR_READ = 1, 280237263Snp RTR_WRITE = 2, 281237263Snp RTR_SEND = 3, 282237263Snp}; 283237263Snp 284237263Snp#define S_RTR_TYPE 2 285237263Snp#define M_RTR_TYPE 0x3 286237263Snp#define V_RTR_TYPE(x) ((x) << S_RTR_TYPE) 287237263Snp#define G_RTR_TYPE(x) ((((x) >> S_RTR_TYPE)) & M_RTR_TYPE) 288237263Snp 289237263Snp#define S_CHAN 4 290237263Snp#define M_CHAN 0x3 291237263Snp#define V_CHAN(x) ((x) << S_CHAN) 292237263Snp#define G_CHAN(x) ((((x) >> S_CHAN)) & M_CHAN) 293237263Snp 294178786Skmacystruct t3_rdma_init_attr { 295178786Skmacy u32 tid; 296178786Skmacy u32 qpid; 297178786Skmacy u32 pdid; 298178786Skmacy u32 scqid; 299178786Skmacy u32 rcqid; 300178786Skmacy u32 rq_addr; 301178786Skmacy u32 rq_size; 302178786Skmacy enum t3_mpa_attrs mpaattrs; 303178786Skmacy enum t3_qp_caps qpcaps; 304178786Skmacy u16 tcp_emss; 305178786Skmacy u32 ord; 306178786Skmacy u32 ird; 307178786Skmacy u64 qp_dma_addr; 308178786Skmacy u32 qp_dma_size; 309237263Snp enum rdma_init_rtr_types rtr_type; 310237263Snp u16 flags; 311237263Snp u16 rqe_count; 312178786Skmacy u32 irs; 313237263Snp u32 chan; 314178786Skmacy}; 315178786Skmacy 316178786Skmacystruct t3_rdma_init_wr { 317178786Skmacy struct fw_riwrh wrh; /* 0 */ 318178786Skmacy union t3_wrid wrid; /* 1 */ 319178786Skmacy __be32 qpid; /* 2 */ 320178786Skmacy __be32 pdid; 321178786Skmacy __be32 scqid; /* 3 */ 322178786Skmacy __be32 rcqid; 323178786Skmacy __be32 rq_addr; /* 4 */ 324178786Skmacy __be32 rq_size; 325178786Skmacy u8 mpaattrs; /* 5 */ 326178786Skmacy u8 qpcaps; 327178786Skmacy __be16 ulpdu_size; 328237263Snp __be16 flags_rtr_type; 329237263Snp __be16 rqe_count; 330178786Skmacy __be32 ord; /* 6 */ 331178786Skmacy __be32 ird; 332178786Skmacy __be64 qp_dma_addr; /* 7 */ 333178786Skmacy __be32 qp_dma_size; /* 8 */ 334237263Snp __be32 irs; 335178786Skmacy}; 336178786Skmacy 337178786Skmacystruct t3_genbit { 338178786Skmacy u64 flit[15]; 339178786Skmacy __be64 genbit; 340178786Skmacy}; 341178786Skmacy 342178786Skmacyenum rdma_init_wr_flags { 343237263Snp MPA_INITIATOR = (1<<0), 344237263Snp PRIV_QP = (1<<1), 345178786Skmacy}; 346178786Skmacy 347178786Skmacyunion t3_wr { 348178786Skmacy struct t3_send_wr send; 349178786Skmacy struct t3_rdma_write_wr write; 350178786Skmacy struct t3_rdma_read_wr read; 351178786Skmacy struct t3_receive_wr recv; 352178786Skmacy struct t3_local_inv_wr local_inv; 353178786Skmacy struct t3_bind_mw_wr bind; 354178786Skmacy struct t3_bypass_wr bypass; 355178786Skmacy struct t3_rdma_init_wr init; 356178786Skmacy struct t3_modify_qp_wr qp_mod; 357178786Skmacy struct t3_genbit genbit; 358178786Skmacy u64 flit[16]; 359178786Skmacy}; 360178786Skmacy 361178786Skmacy#define T3_SQ_CQE_FLIT 13 362178786Skmacy#define T3_SQ_COOKIE_FLIT 14 363178786Skmacy 364178786Skmacy#define T3_RQ_COOKIE_FLIT 13 365178786Skmacy#define T3_RQ_CQE_FLIT 14 366178786Skmacy 367178786Skmacystatic inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe) 368178786Skmacy{ 369178786Skmacy return G_FW_RIWR_OP(be32toh(wqe->op_seop_flags)); 370178786Skmacy} 371178786Skmacy 372178786Skmacystatic inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op, 373178786Skmacy enum t3_wr_flags flags, u8 genbit, u32 tid, 374178786Skmacy u8 len) 375178786Skmacy{ 376178786Skmacy wqe->op_seop_flags = htobe32(V_FW_RIWR_OP(op) | 377178786Skmacy V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) | 378178786Skmacy V_FW_RIWR_FLAGS(flags)); 379178786Skmacy wmb(); 380178786Skmacy wqe->gen_tid_len = htobe32(V_FW_RIWR_GEN(genbit) | 381178786Skmacy V_FW_RIWR_TID(tid) | 382178786Skmacy V_FW_RIWR_LEN(len)); 383178786Skmacy /* 2nd gen bit... */ 384178786Skmacy ((union t3_wr *)wqe)->genbit.genbit = htobe64(genbit); 385178786Skmacy} 386178786Skmacy 387178786Skmacy/* 388178786Skmacy * T3 ULP2_TX commands 389178786Skmacy */ 390178786Skmacyenum t3_utx_mem_op { 391178786Skmacy T3_UTX_MEM_READ = 2, 392178786Skmacy T3_UTX_MEM_WRITE = 3 393178786Skmacy}; 394178786Skmacy 395178786Skmacy/* T3 MC7 RDMA TPT entry format */ 396178786Skmacy 397178786Skmacyenum tpt_mem_type { 398178786Skmacy TPT_NON_SHARED_MR = 0x0, 399178786Skmacy TPT_SHARED_MR = 0x1, 400178786Skmacy TPT_MW = 0x2, 401178786Skmacy TPT_MW_RELAXED_PROTECTION = 0x3 402178786Skmacy}; 403178786Skmacy 404178786Skmacyenum tpt_addr_type { 405178786Skmacy TPT_ZBTO = 0, 406178786Skmacy TPT_VATO = 1 407178786Skmacy}; 408178786Skmacy 409178786Skmacyenum tpt_mem_perm { 410178786Skmacy TPT_LOCAL_READ = 0x8, 411178786Skmacy TPT_LOCAL_WRITE = 0x4, 412178786Skmacy TPT_REMOTE_READ = 0x2, 413178786Skmacy TPT_REMOTE_WRITE = 0x1 414178786Skmacy}; 415178786Skmacy 416178786Skmacystruct tpt_entry { 417178786Skmacy __be32 valid_stag_pdid; 418178786Skmacy __be32 flags_pagesize_qpid; 419178786Skmacy 420178786Skmacy __be32 rsvd_pbl_addr; 421178786Skmacy __be32 len; 422178786Skmacy __be32 va_hi; 423178786Skmacy __be32 va_low_or_fbo; 424178786Skmacy 425178786Skmacy __be32 rsvd_bind_cnt_or_pstag; 426178786Skmacy __be32 rsvd_pbl_size; 427178786Skmacy}; 428178786Skmacy 429178786Skmacy#define S_TPT_VALID 31 430178786Skmacy#define V_TPT_VALID(x) ((x) << S_TPT_VALID) 431178786Skmacy#define F_TPT_VALID V_TPT_VALID(1U) 432178786Skmacy 433178786Skmacy#define S_TPT_STAG_KEY 23 434178786Skmacy#define M_TPT_STAG_KEY 0xFF 435178786Skmacy#define V_TPT_STAG_KEY(x) ((x) << S_TPT_STAG_KEY) 436178786Skmacy#define G_TPT_STAG_KEY(x) (((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY) 437178786Skmacy 438178786Skmacy#define S_TPT_STAG_STATE 22 439178786Skmacy#define V_TPT_STAG_STATE(x) ((x) << S_TPT_STAG_STATE) 440178786Skmacy#define F_TPT_STAG_STATE V_TPT_STAG_STATE(1U) 441178786Skmacy 442178786Skmacy#define S_TPT_STAG_TYPE 20 443178786Skmacy#define M_TPT_STAG_TYPE 0x3 444178786Skmacy#define V_TPT_STAG_TYPE(x) ((x) << S_TPT_STAG_TYPE) 445178786Skmacy#define G_TPT_STAG_TYPE(x) (((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE) 446178786Skmacy 447178786Skmacy#define S_TPT_PDID 0 448178786Skmacy#define M_TPT_PDID 0xFFFFF 449178786Skmacy#define V_TPT_PDID(x) ((x) << S_TPT_PDID) 450178786Skmacy#define G_TPT_PDID(x) (((x) >> S_TPT_PDID) & M_TPT_PDID) 451178786Skmacy 452178786Skmacy#define S_TPT_PERM 28 453178786Skmacy#define M_TPT_PERM 0xF 454178786Skmacy#define V_TPT_PERM(x) ((x) << S_TPT_PERM) 455178786Skmacy#define G_TPT_PERM(x) (((x) >> S_TPT_PERM) & M_TPT_PERM) 456178786Skmacy 457178786Skmacy#define S_TPT_REM_INV_DIS 27 458178786Skmacy#define V_TPT_REM_INV_DIS(x) ((x) << S_TPT_REM_INV_DIS) 459178786Skmacy#define F_TPT_REM_INV_DIS V_TPT_REM_INV_DIS(1U) 460178786Skmacy 461178786Skmacy#define S_TPT_ADDR_TYPE 26 462178786Skmacy#define V_TPT_ADDR_TYPE(x) ((x) << S_TPT_ADDR_TYPE) 463178786Skmacy#define F_TPT_ADDR_TYPE V_TPT_ADDR_TYPE(1U) 464178786Skmacy 465178786Skmacy#define S_TPT_MW_BIND_ENABLE 25 466178786Skmacy#define V_TPT_MW_BIND_ENABLE(x) ((x) << S_TPT_MW_BIND_ENABLE) 467178786Skmacy#define F_TPT_MW_BIND_ENABLE V_TPT_MW_BIND_ENABLE(1U) 468178786Skmacy 469178786Skmacy#define S_TPT_PAGE_SIZE 20 470178786Skmacy#define M_TPT_PAGE_SIZE 0x1F 471178786Skmacy#define V_TPT_PAGE_SIZE(x) ((x) << S_TPT_PAGE_SIZE) 472178786Skmacy#define G_TPT_PAGE_SIZE(x) (((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE) 473178786Skmacy 474178786Skmacy#define S_TPT_PBL_ADDR 0 475178786Skmacy#define M_TPT_PBL_ADDR 0x1FFFFFFF 476178786Skmacy#define V_TPT_PBL_ADDR(x) ((x) << S_TPT_PBL_ADDR) 477178786Skmacy#define G_TPT_PBL_ADDR(x) (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR) 478178786Skmacy 479178786Skmacy#define S_TPT_QPID 0 480178786Skmacy#define M_TPT_QPID 0xFFFFF 481178786Skmacy#define V_TPT_QPID(x) ((x) << S_TPT_QPID) 482178786Skmacy#define G_TPT_QPID(x) (((x) >> S_TPT_QPID) & M_TPT_QPID) 483178786Skmacy 484178786Skmacy#define S_TPT_PSTAG 0 485178786Skmacy#define M_TPT_PSTAG 0xFFFFFF 486178786Skmacy#define V_TPT_PSTAG(x) ((x) << S_TPT_PSTAG) 487178786Skmacy#define G_TPT_PSTAG(x) (((x) >> S_TPT_PSTAG) & M_TPT_PSTAG) 488178786Skmacy 489178786Skmacy#define S_TPT_PBL_SIZE 0 490178786Skmacy#define M_TPT_PBL_SIZE 0xFFFFF 491178786Skmacy#define V_TPT_PBL_SIZE(x) ((x) << S_TPT_PBL_SIZE) 492178786Skmacy#define G_TPT_PBL_SIZE(x) (((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE) 493178786Skmacy 494178786Skmacy/* 495178786Skmacy * CQE defs 496178786Skmacy */ 497178786Skmacystruct t3_cqe { 498178786Skmacy __be32 header; 499178786Skmacy __be32 len; 500178786Skmacy union { 501178786Skmacy struct { 502178786Skmacy __be32 stag; 503178786Skmacy __be32 msn; 504178786Skmacy } rcqe; 505178786Skmacy struct { 506178786Skmacy u32 wrid_hi; 507178786Skmacy u32 wrid_low; 508178786Skmacy } scqe; 509178786Skmacy } u; 510178786Skmacy}; 511178786Skmacy 512178786Skmacy#define S_CQE_OOO 31 513178786Skmacy#define M_CQE_OOO 0x1 514178786Skmacy#define G_CQE_OOO(x) ((((x) >> S_CQE_OOO)) & M_CQE_OOO) 515178786Skmacy#define V_CEQ_OOO(x) ((x)<<S_CQE_OOO) 516178786Skmacy 517178786Skmacy#define S_CQE_QPID 12 518178786Skmacy#define M_CQE_QPID 0x7FFFF 519178786Skmacy#define G_CQE_QPID(x) ((((x) >> S_CQE_QPID)) & M_CQE_QPID) 520178786Skmacy#define V_CQE_QPID(x) ((x)<<S_CQE_QPID) 521178786Skmacy 522178786Skmacy#define S_CQE_SWCQE 11 523178786Skmacy#define M_CQE_SWCQE 0x1 524178786Skmacy#define G_CQE_SWCQE(x) ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE) 525178786Skmacy#define V_CQE_SWCQE(x) ((x)<<S_CQE_SWCQE) 526178786Skmacy 527178786Skmacy#define S_CQE_GENBIT 10 528178786Skmacy#define M_CQE_GENBIT 0x1 529178786Skmacy#define G_CQE_GENBIT(x) (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT) 530178786Skmacy#define V_CQE_GENBIT(x) ((x)<<S_CQE_GENBIT) 531178786Skmacy 532178786Skmacy#define S_CQE_STATUS 5 533178786Skmacy#define M_CQE_STATUS 0x1F 534178786Skmacy#define G_CQE_STATUS(x) ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS) 535178786Skmacy#define V_CQE_STATUS(x) ((x)<<S_CQE_STATUS) 536178786Skmacy 537178786Skmacy#define S_CQE_TYPE 4 538178786Skmacy#define M_CQE_TYPE 0x1 539178786Skmacy#define G_CQE_TYPE(x) ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE) 540178786Skmacy#define V_CQE_TYPE(x) ((x)<<S_CQE_TYPE) 541178786Skmacy 542178786Skmacy#define S_CQE_OPCODE 0 543178786Skmacy#define M_CQE_OPCODE 0xF 544178786Skmacy#define G_CQE_OPCODE(x) ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE) 545178786Skmacy#define V_CQE_OPCODE(x) ((x)<<S_CQE_OPCODE) 546178786Skmacy 547178786Skmacy#define SW_CQE(x) (G_CQE_SWCQE(be32toh((x).header))) 548178786Skmacy#define CQE_OOO(x) (G_CQE_OOO(be32toh((x).header))) 549178786Skmacy#define CQE_QPID(x) (G_CQE_QPID(be32toh((x).header))) 550178786Skmacy#define CQE_GENBIT(x) (G_CQE_GENBIT(be32toh((x).header))) 551178786Skmacy#define CQE_TYPE(x) (G_CQE_TYPE(be32toh((x).header))) 552178786Skmacy#define SQ_TYPE(x) (CQE_TYPE((x))) 553178786Skmacy#define RQ_TYPE(x) (!CQE_TYPE((x))) 554178786Skmacy#define CQE_STATUS(x) (G_CQE_STATUS(be32toh((x).header))) 555178786Skmacy#define CQE_OPCODE(x) (G_CQE_OPCODE(be32toh((x).header))) 556178786Skmacy 557237263Snp#define CQE_SEND_OPCODE(x)( \ 558237263Snp (G_CQE_OPCODE(be32_to_cpu((x).header)) == T3_SEND) || \ 559237263Snp (G_CQE_OPCODE(be32_to_cpu((x).header)) == T3_SEND_WITH_SE) || \ 560237263Snp (G_CQE_OPCODE(be32_to_cpu((x).header)) == T3_SEND_WITH_INV) || \ 561237263Snp (G_CQE_OPCODE(be32_to_cpu((x).header)) == T3_SEND_WITH_SE_INV)) 562237263Snp 563178786Skmacy#define CQE_LEN(x) (be32toh((x).len)) 564178786Skmacy 565178786Skmacy/* used for RQ completion processing */ 566178786Skmacy#define CQE_WRID_STAG(x) (be32toh((x).u.rcqe.stag)) 567178786Skmacy#define CQE_WRID_MSN(x) (be32toh((x).u.rcqe.msn)) 568178786Skmacy 569178786Skmacy/* used for SQ completion processing */ 570178786Skmacy#define CQE_WRID_SQ_WPTR(x) ((x).u.scqe.wrid_hi) 571178786Skmacy#define CQE_WRID_WPTR(x) ((x).u.scqe.wrid_low) 572178786Skmacy 573178786Skmacy/* generic accessor macros */ 574178786Skmacy#define CQE_WRID_HI(x) ((x).u.scqe.wrid_hi) 575178786Skmacy#define CQE_WRID_LOW(x) ((x).u.scqe.wrid_low) 576178786Skmacy 577178786Skmacy#define TPT_ERR_SUCCESS 0x0 578178786Skmacy#define TPT_ERR_STAG 0x1 /* STAG invalid: either the */ 579178786Skmacy /* STAG is offlimt, being 0, */ 580178786Skmacy /* or STAG_key mismatch */ 581178786Skmacy#define TPT_ERR_PDID 0x2 /* PDID mismatch */ 582178786Skmacy#define TPT_ERR_QPID 0x3 /* QPID mismatch */ 583178786Skmacy#define TPT_ERR_ACCESS 0x4 /* Invalid access right */ 584178786Skmacy#define TPT_ERR_WRAP 0x5 /* Wrap error */ 585178786Skmacy#define TPT_ERR_BOUND 0x6 /* base and bounds voilation */ 586178786Skmacy#define TPT_ERR_INVALIDATE_SHARED_MR 0x7 /* attempt to invalidate a */ 587178786Skmacy /* shared memory region */ 588178786Skmacy#define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8 /* attempt to invalidate a */ 589178786Skmacy /* shared memory region */ 590178786Skmacy#define TPT_ERR_ECC 0x9 /* ECC error detected */ 591178786Skmacy#define TPT_ERR_ECC_PSTAG 0xA /* ECC error detected when */ 592178786Skmacy /* reading PSTAG for a MW */ 593178786Skmacy /* Invalidate */ 594178786Skmacy#define TPT_ERR_PBL_ADDR_BOUND 0xB /* pbl addr out of bounds: */ 595178786Skmacy /* software error */ 596178786Skmacy#define TPT_ERR_SWFLUSH 0xC /* SW FLUSHED */ 597178786Skmacy#define TPT_ERR_CRC 0x10 /* CRC error */ 598178786Skmacy#define TPT_ERR_MARKER 0x11 /* Marker error */ 599178786Skmacy#define TPT_ERR_PDU_LEN_ERR 0x12 /* invalid PDU length */ 600178786Skmacy#define TPT_ERR_OUT_OF_RQE 0x13 /* out of RQE */ 601178786Skmacy#define TPT_ERR_DDP_VERSION 0x14 /* wrong DDP version */ 602178786Skmacy#define TPT_ERR_RDMA_VERSION 0x15 /* wrong RDMA version */ 603178786Skmacy#define TPT_ERR_OPCODE 0x16 /* invalid rdma opcode */ 604178786Skmacy#define TPT_ERR_DDP_QUEUE_NUM 0x17 /* invalid ddp queue number */ 605178786Skmacy#define TPT_ERR_MSN 0x18 /* MSN error */ 606178786Skmacy#define TPT_ERR_TBIT 0x19 /* tag bit not set correctly */ 607178786Skmacy#define TPT_ERR_MO 0x1A /* MO not 0 for TERMINATE */ 608178786Skmacy /* or READ_REQ */ 609178786Skmacy#define TPT_ERR_MSN_GAP 0x1B 610178786Skmacy#define TPT_ERR_MSN_RANGE 0x1C 611178786Skmacy#define TPT_ERR_IRD_OVERFLOW 0x1D 612178786Skmacy#define TPT_ERR_RQE_ADDR_BOUND 0x1E /* RQE addr out of bounds: */ 613178786Skmacy /* software error */ 614178786Skmacy#define TPT_ERR_INTERNAL_ERR 0x1F /* internal error (opcode */ 615178786Skmacy /* mismatch) */ 616178786Skmacy 617178786Skmacystruct t3_swsq { 618178786Skmacy uint64_t wr_id; 619178786Skmacy struct t3_cqe cqe; 620178786Skmacy uint32_t sq_wptr; 621237263Snp __be32 read_len; 622178786Skmacy int opcode; 623178786Skmacy int complete; 624178786Skmacy int signaled; 625178786Skmacy}; 626178786Skmacy 627237263Snpstruct t3_swrq { 628237263Snp __u64 wr_id; 629237263Snp __u32 pbl_addr; 630237263Snp}; 631237263Snp 632178786Skmacy/* 633178786Skmacy * A T3 WQ implements both the SQ and RQ. 634178786Skmacy */ 635178786Skmacystruct t3_wq { 636178786Skmacy union t3_wr *queue; /* DMA accessable memory */ 637178786Skmacy bus_addr_t dma_addr; /* DMA address for HW */ 638178786Skmacy u32 error; /* 1 once we go to ERROR */ 639178786Skmacy u32 qpid; 640178786Skmacy u32 wptr; /* idx to next available WR slot */ 641178786Skmacy u32 size_log2; /* total wq size */ 642178786Skmacy struct t3_swsq *sq; /* SW SQ */ 643178786Skmacy struct t3_swsq *oldest_read; /* tracks oldest pending read */ 644178786Skmacy u32 sq_wptr; /* sq_wptr - sq_rptr == count of */ 645178786Skmacy u32 sq_rptr; /* pending wrs */ 646178786Skmacy u32 sq_size_log2; /* sq size */ 647237263Snp struct t3_swrq *rq; /* SW RQ (holds consumer wr_ids */ 648178786Skmacy u32 rq_wptr; /* rq_wptr - rq_rptr == count of */ 649178786Skmacy u32 rq_rptr; /* pending wrs */ 650237263Snp struct t3_swrq *rq_oldest_wr; /* oldest wr on the SW RQ */ 651178786Skmacy u32 rq_size_log2; /* rq size */ 652178786Skmacy u32 rq_addr; /* rq adapter address */ 653237263Snp void *doorbell; /* kernel db */ 654178786Skmacy u64 udb; /* user db if any */ 655237263Snp struct cxio_rdev *rdev; 656178786Skmacy}; 657178786Skmacy 658178786Skmacystruct t3_cq { 659178786Skmacy u32 cqid; 660178786Skmacy u32 rptr; 661178786Skmacy u32 wptr; 662178786Skmacy u32 size_log2; 663178786Skmacy bus_addr_t dma_addr; 664178786Skmacy struct t3_cqe *queue; 665178786Skmacy struct t3_cqe *sw_queue; 666178786Skmacy u32 sw_rptr; 667178786Skmacy u32 sw_wptr; 668178786Skmacy}; 669178786Skmacy 670178786Skmacy#define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \ 671178786Skmacy CQE_GENBIT(*cqe)) 672178786Skmacy 673237263Snpstruct t3_cq_status_page { 674237263Snp u32 cq_err; 675237263Snp}; 676237263Snp 677237263Snpstatic inline int cxio_cq_in_error(struct t3_cq *cq) 678237263Snp{ 679237263Snp return ((struct t3_cq_status_page *) 680237263Snp &cq->queue[1 << cq->size_log2])->cq_err; 681237263Snp} 682237263Snp 683237263Snpstatic inline void cxio_set_cq_in_error(struct t3_cq *cq) 684237263Snp{ 685237263Snp ((struct t3_cq_status_page *) 686237263Snp &cq->queue[1 << cq->size_log2])->cq_err = 1; 687237263Snp} 688237263Snp 689178786Skmacystatic inline void cxio_set_wq_in_error(struct t3_wq *wq) 690178786Skmacy{ 691178786Skmacy wq->queue->flit[13] = 1; 692178786Skmacy} 693178786Skmacy 694178786Skmacystatic inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq) 695178786Skmacy{ 696178786Skmacy struct t3_cqe *cqe; 697178786Skmacy 698178786Skmacy cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2)); 699178786Skmacy if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe)) 700178786Skmacy return cqe; 701178786Skmacy return NULL; 702178786Skmacy} 703178786Skmacy 704178786Skmacystatic inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq) 705178786Skmacy{ 706178786Skmacy struct t3_cqe *cqe; 707178786Skmacy 708178786Skmacy if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) { 709178786Skmacy cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2)); 710178786Skmacy return cqe; 711178786Skmacy } 712178786Skmacy return NULL; 713178786Skmacy} 714178786Skmacy 715178786Skmacystatic inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq) 716178786Skmacy{ 717178786Skmacy struct t3_cqe *cqe; 718178786Skmacy 719178786Skmacy if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) { 720178786Skmacy cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2)); 721178786Skmacy return cqe; 722178786Skmacy } 723178786Skmacy cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2)); 724178786Skmacy if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe)) 725178786Skmacy return cqe; 726178786Skmacy return NULL; 727178786Skmacy} 728178786Skmacy 729178786Skmacy#endif 730