28 29#include "efsys.h" 30#include "efx.h" 31#include "efx_types.h" 32#include "efx_regs.h" 33#include "efx_impl.h" 34 35#if EFSYS_OPT_QSTATS 36#define EFX_TX_QSTAT_INCR(_etp, _stat) \ 37 do { \ 38 (_etp)->et_stat[_stat]++; \ 39 _NOTE(CONSTANTCONDITION) \ 40 } while (B_FALSE) 41#else 42#define EFX_TX_QSTAT_INCR(_etp, _stat) 43#endif 44 45 __checkReturn int 46efx_tx_init( 47 __in efx_nic_t *enp) 48{ 49 efx_oword_t oword; 50 int rc; 51 52 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 53 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 54 55 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 56 rc = EINVAL; 57 goto fail1; 58 } 59 60 if (enp->en_mod_flags & EFX_MOD_TX) { 61 rc = EINVAL; 62 goto fail2; 63 } 64 65 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 66 67 /* 68 * Disable the timer-based TX DMA backoff and allow TX DMA to be 69 * controlled by the RX FIFO fill level (although always allow a 70 * minimal trickle). 71 */ 72 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 73 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 74 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 75 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 76 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 77 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 78 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 79 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 80 81 /* 82 * Filter all packets less than 14 bytes to avoid parsing 83 * errors. 84 */ 85 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 86 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 87 88 /* 89 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 90 * descriptors (which is bad). 91 */ 92 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 93 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 94 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 95 96 enp->en_mod_flags |= EFX_MOD_TX; 97 return (0); 98 99fail2: 100 EFSYS_PROBE(fail2); 101fail1: 102 EFSYS_PROBE1(fail1, int, rc); 103 104 return (rc); 105} 106 107#if EFSYS_OPT_FILTER 108extern __checkReturn int 109efx_tx_filter_insert( 110 __in efx_txq_t *etp, 111 __inout efx_filter_spec_t *spec) 112{ 113 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 114 EFSYS_ASSERT3P(spec, !=, NULL); 115 116 spec->efs_dmaq_id = (uint16_t)etp->et_index; 117 return (efx_filter_insert_filter(etp->et_enp, spec, B_FALSE)); 118} 119#endif 120 121#if EFSYS_OPT_FILTER 122extern __checkReturn int 123efx_tx_filter_remove( 124 __in efx_txq_t *etp, 125 __inout efx_filter_spec_t *spec) 126{ 127 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 128 EFSYS_ASSERT3P(spec, !=, NULL); 129 130 spec->efs_dmaq_id = (uint16_t)etp->et_index; 131 return (efx_filter_remove_filter(etp->et_enp, spec)); 132} 133#endif 134 135#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 136 do { \ 137 unsigned int id; \ 138 size_t offset; \ 139 efx_qword_t qword; \ 140 \ 141 id = (_added)++ & (_etp)->et_mask; \ 142 offset = id * sizeof (efx_qword_t); \ 143 \ 144 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 145 unsigned int, id, efsys_dma_addr_t, (_addr), \ 146 size_t, (_size), boolean_t, (_eop)); \ 147 \ 148 EFX_POPULATE_QWORD_4(qword, \ 149 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 150 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 151 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 152 (uint32_t)((_addr) & 0xffffffff), \ 153 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 154 (uint32_t)((_addr) >> 32)); \ 155 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 156 \ 157 _NOTE(CONSTANTCONDITION) \ 158 } while (B_FALSE) 159 160 __checkReturn int 161efx_tx_qpost( 162 __in efx_txq_t *etp, 163 __in_ecount(n) efx_buffer_t *eb, 164 __in unsigned int n, 165 __in unsigned int completed, 166 __inout unsigned int *addedp) 167{ 168 unsigned int added = *addedp; 169 unsigned int i; 170 int rc = ENOSPC; 171 172 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 173 174 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 175 goto fail1; 176 177 for (i = 0; i < n; i++) { 178 efx_buffer_t *ebp = &eb[i]; 179 efsys_dma_addr_t start = ebp->eb_addr; 180 size_t size = ebp->eb_size; 181 efsys_dma_addr_t end = start + size; 182 183 /* Fragments must not span 4k boundaries. */ 184 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 185 186 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 187 } 188 189 EFX_TX_QSTAT_INCR(etp, TX_POST); 190 191 *addedp = added; 192 return (0); 193 194fail1: 195 EFSYS_PROBE1(fail1, int, rc); 196 197 return (rc); 198} 199 200 void 201efx_tx_qpush( 202 __in efx_txq_t *etp, 203 __in unsigned int added) 204{ 205 efx_nic_t *enp = etp->et_enp; 206 uint32_t wptr; 207 efx_dword_t dword; 208 efx_oword_t oword; 209 210 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 211 212 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 213 EFSYS_PIO_WRITE_BARRIER(); 214 215 /* Push the populated descriptors out */ 216 wptr = added & etp->et_mask; 217 218 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 219 220 /* Only write the third DWORD */ 221 EFX_POPULATE_DWORD_1(dword, 222 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 223 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 224 etp->et_index, &dword, B_FALSE); 225} 226 227 void 228efx_tx_qflush( 229 __in efx_txq_t *etp) 230{ 231 efx_nic_t *enp = etp->et_enp; 232 efx_oword_t oword; 233 uint32_t label; 234 235 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 236 237 label = etp->et_index; 238 239 /* Flush the queue */ 240 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 241 FRF_AZ_TX_FLUSH_DESCQ, label); 242 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 243} 244 245 void 246efx_tx_qenable( 247 __in efx_txq_t *etp) 248{ 249 efx_nic_t *enp = etp->et_enp; 250 efx_oword_t oword; 251 252 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 253 254 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 255 etp->et_index, &oword); 256 257 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 258 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 259 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 260 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 261 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 262 263 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 264 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 265 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 266 267 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 268 etp->et_index, &oword); 269} 270 271 __checkReturn int 272efx_tx_qcreate( 273 __in efx_nic_t *enp, 274 __in unsigned int index, 275 __in unsigned int label, 276 __in efsys_mem_t *esmp, 277 __in size_t n, 278 __in uint32_t id, 279 __in uint16_t flags, 280 __in efx_evq_t *eep, 281 __deref_out efx_txq_t **etpp) 282{ 283 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 284 efx_txq_t *etp; 285 efx_oword_t oword; 286 uint32_t size; 287 int rc; 288 289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 291
|
295 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 296 297 if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) { 298 rc = EINVAL; 299 goto fail1; 300 } 301 if (index >= encp->enc_txq_limit) { 302 rc = EINVAL; 303 goto fail2; 304 } 305 for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS); 306 size++) 307 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 308 break; 309 if (id + (1 << size) >= encp->enc_buftbl_limit) { 310 rc = EINVAL; 311 goto fail3; 312 } 313 314 /* Allocate an TXQ object */ 315 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 316 317 if (etp == NULL) { 318 rc = ENOMEM; 319 goto fail4; 320 } 321 322 etp->et_magic = EFX_TXQ_MAGIC; 323 etp->et_enp = enp; 324 etp->et_index = index; 325 etp->et_mask = n - 1; 326 etp->et_esmp = esmp; 327 328 /* Set up the new descriptor queue */ 329 EFX_POPULATE_OWORD_6(oword, 330 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 331 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 332 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 333 FRF_AZ_TX_DESCQ_LABEL, label, 334 FRF_AZ_TX_DESCQ_SIZE, size, 335 FRF_AZ_TX_DESCQ_TYPE, 0); 336 337 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 338 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 339 (flags & EFX_CKSUM_IPV4) ? 0 : 1); 340 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 341 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1); 342 343 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 344 etp->et_index, &oword); 345 346 enp->en_tx_qcount++; 347 *etpp = etp; 348 return (0); 349 350fail4: 351 EFSYS_PROBE(fail4); 352fail3: 353 EFSYS_PROBE(fail3); 354fail2: 355 EFSYS_PROBE(fail2); 356fail1: 357 EFSYS_PROBE1(fail1, int, rc); 358 359 return (rc); 360} 361 362#if EFSYS_OPT_QSTATS 363#if EFSYS_OPT_NAMES 364/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */ 365static const char __cs * __cs __efx_tx_qstat_name[] = { 366 "post", 367 "unaligned_split", 368}; 369/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 370 371 const char __cs * 372efx_tx_qstat_name( 373 __in efx_nic_t *enp, 374 __in unsigned int id) 375{ 376 _NOTE(ARGUNUSED(enp)) 377 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 378 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 379 380 return (__efx_tx_qstat_name[id]); 381} 382#endif /* EFSYS_OPT_NAMES */ 383#endif /* EFSYS_OPT_QSTATS */ 384 385#if EFSYS_OPT_QSTATS 386 void 387efx_tx_qstats_update( 388 __in efx_txq_t *etp, 389 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 390{ 391 unsigned int id; 392 393 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 394 395 for (id = 0; id < TX_NQSTATS; id++) { 396 efsys_stat_t *essp = &stat[id]; 397 398 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 399 etp->et_stat[id] = 0; 400 } 401} 402#endif /* EFSYS_OPT_QSTATS */ 403 404 void 405efx_tx_qdestroy( 406 __in efx_txq_t *etp) 407{ 408 efx_nic_t *enp = etp->et_enp; 409 efx_oword_t oword; 410 411 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 412 413 EFSYS_ASSERT(enp->en_tx_qcount != 0); 414 --enp->en_tx_qcount; 415 416 /* Purge descriptor queue */ 417 EFX_ZERO_OWORD(oword); 418 419 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 420 etp->et_index, &oword); 421 422 /* Free the TXQ object */ 423 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 424} 425 426 void 427efx_tx_fini( 428 __in efx_nic_t *enp) 429{ 430 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 431 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 432 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 433 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 434 435 enp->en_mod_flags &= ~EFX_MOD_TX; 436}
|