28 29#include "efsys.h" 30#include "efx.h" 31#include "efx_types.h" 32#include "efx_regs.h" 33#include "efx_impl.h" 34 35#if EFSYS_OPT_QSTATS 36#define EFX_TX_QSTAT_INCR(_etp, _stat) \ 37 do { \ 38 (_etp)->et_stat[_stat]++; \ 39 _NOTE(CONSTANTCONDITION) \ 40 } while (B_FALSE) 41#else 42#define EFX_TX_QSTAT_INCR(_etp, _stat) 43#endif 44 45 __checkReturn int 46efx_tx_init( 47 __in efx_nic_t *enp) 48{ 49 efx_oword_t oword; 50 int rc; 51 52 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 53 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 54 55 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 56 rc = EINVAL; 57 goto fail1; 58 } 59 60 if (enp->en_mod_flags & EFX_MOD_TX) { 61 rc = EINVAL; 62 goto fail2; 63 } 64 65 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 66 67 /* 68 * Disable the timer-based TX DMA backoff and allow TX DMA to be 69 * controlled by the RX FIFO fill level (although always allow a 70 * minimal trickle). 71 */ 72 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 73 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 74 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 75 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 76 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 77 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 78 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 79 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 80 81 /* 82 * Filter all packets less than 14 bytes to avoid parsing 83 * errors. 84 */ 85 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 86 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 87 88 /* 89 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 90 * descriptors (which is bad). 91 */ 92 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 93 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 94 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 95 96 enp->en_mod_flags |= EFX_MOD_TX; 97 return (0); 98 99fail2: 100 EFSYS_PROBE(fail2); 101fail1: 102 EFSYS_PROBE1(fail1, int, rc); 103 104 return (rc); 105} 106 107#if EFSYS_OPT_FILTER 108extern __checkReturn int 109efx_tx_filter_insert( 110 __in efx_txq_t *etp, 111 __inout efx_filter_spec_t *spec) 112{ 113 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 114 EFSYS_ASSERT3P(spec, !=, NULL); 115 116 spec->efs_dmaq_id = (uint16_t)etp->et_index;
|
132} 133#endif 134 135#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 136 do { \ 137 unsigned int id; \ 138 size_t offset; \ 139 efx_qword_t qword; \ 140 \ 141 id = (_added)++ & (_etp)->et_mask; \ 142 offset = id * sizeof (efx_qword_t); \ 143 \ 144 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 145 unsigned int, id, efsys_dma_addr_t, (_addr), \ 146 size_t, (_size), boolean_t, (_eop)); \ 147 \ 148 EFX_POPULATE_QWORD_4(qword, \ 149 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 150 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 151 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 152 (uint32_t)((_addr) & 0xffffffff), \ 153 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 154 (uint32_t)((_addr) >> 32)); \ 155 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 156 \ 157 _NOTE(CONSTANTCONDITION) \ 158 } while (B_FALSE) 159 160 __checkReturn int 161efx_tx_qpost( 162 __in efx_txq_t *etp, 163 __in_ecount(n) efx_buffer_t *eb, 164 __in unsigned int n, 165 __in unsigned int completed, 166 __inout unsigned int *addedp) 167{ 168 unsigned int added = *addedp; 169 unsigned int i; 170 int rc = ENOSPC; 171 172 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 173 174 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 175 goto fail1; 176 177 for (i = 0; i < n; i++) { 178 efx_buffer_t *ebp = &eb[i]; 179 efsys_dma_addr_t start = ebp->eb_addr; 180 size_t size = ebp->eb_size; 181 efsys_dma_addr_t end = start + size; 182 183 /* Fragments must not span 4k boundaries. */ 184 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 185 186 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 187 } 188 189 EFX_TX_QSTAT_INCR(etp, TX_POST); 190 191 *addedp = added; 192 return (0); 193 194fail1: 195 EFSYS_PROBE1(fail1, int, rc); 196 197 return (rc); 198} 199 200 void 201efx_tx_qpush( 202 __in efx_txq_t *etp, 203 __in unsigned int added) 204{ 205 efx_nic_t *enp = etp->et_enp; 206 uint32_t wptr; 207 efx_dword_t dword; 208 efx_oword_t oword; 209 210 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 211 212 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 213 EFSYS_PIO_WRITE_BARRIER(); 214 215 /* Push the populated descriptors out */ 216 wptr = added & etp->et_mask; 217 218 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 219 220 /* Only write the third DWORD */ 221 EFX_POPULATE_DWORD_1(dword, 222 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 223 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 224 etp->et_index, &dword, B_FALSE); 225} 226 227 void 228efx_tx_qflush( 229 __in efx_txq_t *etp) 230{ 231 efx_nic_t *enp = etp->et_enp; 232 efx_oword_t oword; 233 uint32_t label; 234 235 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 236 237 label = etp->et_index; 238 239 /* Flush the queue */ 240 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 241 FRF_AZ_TX_FLUSH_DESCQ, label); 242 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 243} 244 245 void 246efx_tx_qenable( 247 __in efx_txq_t *etp) 248{ 249 efx_nic_t *enp = etp->et_enp; 250 efx_oword_t oword; 251 252 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 253 254 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 255 etp->et_index, &oword); 256 257 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 258 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 259 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 260 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 261 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 262 263 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 264 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 265 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 266 267 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 268 etp->et_index, &oword); 269} 270 271 __checkReturn int 272efx_tx_qcreate( 273 __in efx_nic_t *enp, 274 __in unsigned int index, 275 __in unsigned int label, 276 __in efsys_mem_t *esmp, 277 __in size_t n, 278 __in uint32_t id, 279 __in uint16_t flags, 280 __in efx_evq_t *eep, 281 __deref_out efx_txq_t **etpp) 282{ 283 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 284 efx_txq_t *etp; 285 efx_oword_t oword; 286 uint32_t size; 287 int rc; 288 289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 291 292 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 293 /* EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);*/ 294 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 295 296 if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) { 297 rc = EINVAL; 298 goto fail1; 299 } 300 if (index >= encp->enc_txq_limit) { 301 rc = EINVAL; 302 goto fail2; 303 } 304 for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS); 305 size++) 306 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 307 break; 308 if (id + (1 << size) >= encp->enc_buftbl_limit) { 309 rc = EINVAL; 310 goto fail3; 311 } 312 313 /* Allocate an TXQ object */ 314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 315 316 if (etp == NULL) { 317 rc = ENOMEM; 318 goto fail4; 319 } 320 321 etp->et_magic = EFX_TXQ_MAGIC; 322 etp->et_enp = enp; 323 etp->et_index = index; 324 etp->et_mask = n - 1; 325 etp->et_esmp = esmp; 326 327 /* Set up the new descriptor queue */ 328 EFX_POPULATE_OWORD_6(oword, 329 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 330 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 331 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 332 FRF_AZ_TX_DESCQ_LABEL, label, 333 FRF_AZ_TX_DESCQ_SIZE, size, 334 FRF_AZ_TX_DESCQ_TYPE, 0); 335 336 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 337 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 338 (flags & EFX_CKSUM_IPV4) ? 0 : 1); 339 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 340 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1); 341 342 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 343 etp->et_index, &oword); 344 345 enp->en_tx_qcount++; 346 *etpp = etp; 347 return (0); 348 349fail4: 350 EFSYS_PROBE(fail4); 351fail3: 352 EFSYS_PROBE(fail3); 353fail2: 354 EFSYS_PROBE(fail2); 355fail1: 356 EFSYS_PROBE1(fail1, int, rc); 357 358 return (rc); 359} 360 361#if EFSYS_OPT_QSTATS 362#if EFSYS_OPT_NAMES 363/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */ 364static const char __cs * __cs __efx_tx_qstat_name[] = { 365 "post", 366 "unaligned_split", 367}; 368/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 369 370 const char __cs * 371efx_tx_qstat_name( 372 __in efx_nic_t *enp, 373 __in unsigned int id) 374{ 375 _NOTE(ARGUNUSED(enp)) 376 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 377 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 378 379 return (__efx_tx_qstat_name[id]); 380} 381#endif /* EFSYS_OPT_NAMES */ 382#endif /* EFSYS_OPT_QSTATS */ 383 384#if EFSYS_OPT_QSTATS 385 void 386efx_tx_qstats_update( 387 __in efx_txq_t *etp, 388 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 389{ 390 unsigned int id; 391 392 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 393 394 for (id = 0; id < TX_NQSTATS; id++) { 395 efsys_stat_t *essp = &stat[id]; 396 397 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 398 etp->et_stat[id] = 0; 399 } 400} 401#endif /* EFSYS_OPT_QSTATS */ 402 403 void 404efx_tx_qdestroy( 405 __in efx_txq_t *etp) 406{ 407 efx_nic_t *enp = etp->et_enp; 408 efx_oword_t oword; 409 410 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 411 412 EFSYS_ASSERT(enp->en_tx_qcount != 0); 413 --enp->en_tx_qcount; 414 415 /* Purge descriptor queue */ 416 EFX_ZERO_OWORD(oword); 417 418 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 419 etp->et_index, &oword); 420 421 /* Free the TXQ object */ 422 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 423} 424 425 void 426efx_tx_fini( 427 __in efx_nic_t *enp) 428{ 429 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 430 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 431 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 432 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 433 434 enp->en_mod_flags &= ~EFX_MOD_TX; 435}
|