efx_tx.c revision 280589
1/*- 2 * Copyright 2007-2009 Solarflare Communications Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26#include <sys/cdefs.h> 27__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/common/efx_tx.c 280589 2015-03-25 13:12:15Z arybchik $"); 28 29#include "efsys.h" 30#include "efx.h" 31#include "efx_types.h" 32#include "efx_regs.h" 33#include "efx_impl.h" 34 35#if EFSYS_OPT_QSTATS 36#define EFX_TX_QSTAT_INCR(_etp, _stat) \ 37 do { \ 38 (_etp)->et_stat[_stat]++; \ 39 _NOTE(CONSTANTCONDITION) \ 40 } while (B_FALSE) 41#else 42#define EFX_TX_QSTAT_INCR(_etp, _stat) 43#endif 44 45 __checkReturn int 46efx_tx_init( 47 __in efx_nic_t *enp) 48{ 49 efx_oword_t oword; 50 int rc; 51 52 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 53 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 54 55 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 56 rc = EINVAL; 57 goto fail1; 58 } 59 60 if (enp->en_mod_flags & EFX_MOD_TX) { 61 rc = EINVAL; 62 goto fail2; 63 } 64 65 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 66 67 /* 68 * Disable the timer-based TX DMA backoff and allow TX DMA to be 69 * controlled by the RX FIFO fill level (although always allow a 70 * minimal trickle). 71 */ 72 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 73 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 74 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 75 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 76 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 77 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 78 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 79 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 80 81 /* 82 * Filter all packets less than 14 bytes to avoid parsing 83 * errors. 84 */ 85 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 86 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 87 88 /* 89 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 90 * descriptors (which is bad). 91 */ 92 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 93 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 94 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 95 96 enp->en_mod_flags |= EFX_MOD_TX; 97 return (0); 98 99fail2: 100 EFSYS_PROBE(fail2); 101fail1: 102 EFSYS_PROBE1(fail1, int, rc); 103 104 return (rc); 105} 106 107#if EFSYS_OPT_FILTER 108extern __checkReturn int 109efx_tx_filter_insert( 110 __in efx_txq_t *etp, 111 __inout efx_filter_spec_t *spec) 112{ 113 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 114 EFSYS_ASSERT3P(spec, !=, NULL); 115 116 spec->efs_dmaq_id = (uint16_t)etp->et_index; 117 return (efx_filter_insert_filter(etp->et_enp, spec, B_FALSE)); 118} 119#endif 120 121#if EFSYS_OPT_FILTER 122extern __checkReturn int 123efx_tx_filter_remove( 124 __in efx_txq_t *etp, 125 __inout efx_filter_spec_t *spec) 126{ 127 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 128 EFSYS_ASSERT3P(spec, !=, NULL); 129 130 spec->efs_dmaq_id = (uint16_t)etp->et_index; 131 return (efx_filter_remove_filter(etp->et_enp, spec)); 132} 133#endif 134 135#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 136 do { \ 137 unsigned int id; \ 138 size_t offset; \ 139 efx_qword_t qword; \ 140 \ 141 id = (_added)++ & (_etp)->et_mask; \ 142 offset = id * sizeof (efx_qword_t); \ 143 \ 144 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 145 unsigned int, id, efsys_dma_addr_t, (_addr), \ 146 size_t, (_size), boolean_t, (_eop)); \ 147 \ 148 EFX_POPULATE_QWORD_4(qword, \ 149 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 150 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 151 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 152 (uint32_t)((_addr) & 0xffffffff), \ 153 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 154 (uint32_t)((_addr) >> 32)); \ 155 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 156 \ 157 _NOTE(CONSTANTCONDITION) \ 158 } while (B_FALSE) 159 160 __checkReturn int 161efx_tx_qpost( 162 __in efx_txq_t *etp, 163 __in_ecount(n) efx_buffer_t *eb, 164 __in unsigned int n, 165 __in unsigned int completed, 166 __inout unsigned int *addedp) 167{ 168 unsigned int added = *addedp; 169 unsigned int i; 170 int rc = ENOSPC; 171 172 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 173 174 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 175 goto fail1; 176 177 for (i = 0; i < n; i++) { 178 efx_buffer_t *ebp = &eb[i]; 179 efsys_dma_addr_t start = ebp->eb_addr; 180 size_t size = ebp->eb_size; 181 efsys_dma_addr_t end = start + size; 182 183 /* Fragments must not span 4k boundaries. */ 184 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 185 186 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 187 } 188 189 EFX_TX_QSTAT_INCR(etp, TX_POST); 190 191 *addedp = added; 192 return (0); 193 194fail1: 195 EFSYS_PROBE1(fail1, int, rc); 196 197 return (rc); 198} 199 200 void 201efx_tx_qpush( 202 __in efx_txq_t *etp, 203 __in unsigned int added) 204{ 205 efx_nic_t *enp = etp->et_enp; 206 uint32_t wptr; 207 efx_dword_t dword; 208 efx_oword_t oword; 209 210 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 211 212 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 213 EFSYS_PIO_WRITE_BARRIER(); 214 215 /* Push the populated descriptors out */ 216 wptr = added & etp->et_mask; 217 218 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 219 220 /* Only write the third DWORD */ 221 EFX_POPULATE_DWORD_1(dword, 222 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 223 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 224 etp->et_index, &dword, B_FALSE); 225} 226 227#define EFX_MAX_PACE_VALUE 20 228#define EFX_TX_PACE_CLOCK_BASE 104 229 230 __checkReturn int 231efx_tx_qpace( 232 __in efx_txq_t *etp, 233 __in unsigned int ns) 234{ 235 efx_nic_t *enp = etp->et_enp; 236 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 237 efx_oword_t oword; 238 unsigned int pace_val; 239 unsigned int timer_period; 240 int rc; 241 242 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 243 244 if (ns == 0) { 245 pace_val = 0; 246 } else { 247 /* 248 * The pace_val to write into the table is s.t 249 * ns <= timer_period * (2 ^ pace_val) 250 */ 251 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 252 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 253 if ((timer_period << pace_val) >= ns) 254 break; 255 } 256 } 257 if (pace_val > EFX_MAX_PACE_VALUE) { 258 rc = EINVAL; 259 goto fail1; 260 } 261 262 /* Update the pacing table */ 263 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 264 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, &oword); 265 266 return (0); 267 268fail1: 269 EFSYS_PROBE1(fail1, int, rc); 270 271 return (rc); 272} 273 274 void 275efx_tx_qflush( 276 __in efx_txq_t *etp) 277{ 278 efx_nic_t *enp = etp->et_enp; 279 efx_oword_t oword; 280 uint32_t label; 281 282 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 283 284 efx_tx_qpace(etp, 0); 285 286 label = etp->et_index; 287 288 /* Flush the queue */ 289 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 290 FRF_AZ_TX_FLUSH_DESCQ, label); 291 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 292} 293 294 void 295efx_tx_qenable( 296 __in efx_txq_t *etp) 297{ 298 efx_nic_t *enp = etp->et_enp; 299 efx_oword_t oword; 300 301 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 302 303 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 304 etp->et_index, &oword); 305 306 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 307 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 308 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 309 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 310 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 311 312 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 313 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 314 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 315 316 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 317 etp->et_index, &oword); 318} 319 320 __checkReturn int 321efx_tx_qcreate( 322 __in efx_nic_t *enp, 323 __in unsigned int index, 324 __in unsigned int label, 325 __in efsys_mem_t *esmp, 326 __in size_t n, 327 __in uint32_t id, 328 __in uint16_t flags, 329 __in efx_evq_t *eep, 330 __deref_out efx_txq_t **etpp) 331{ 332 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 333 efx_txq_t *etp; 334 efx_oword_t oword; 335 uint32_t size; 336 int rc; 337 338 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 339 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 340 341 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 342 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 343 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 344 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 345 346 if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) { 347 rc = EINVAL; 348 goto fail1; 349 } 350 if (index >= encp->enc_txq_limit) { 351 rc = EINVAL; 352 goto fail2; 353 } 354 for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS); 355 size++) 356 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 357 break; 358 if (id + (1 << size) >= encp->enc_buftbl_limit) { 359 rc = EINVAL; 360 goto fail3; 361 } 362 363 /* Allocate an TXQ object */ 364 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 365 366 if (etp == NULL) { 367 rc = ENOMEM; 368 goto fail4; 369 } 370 371 etp->et_magic = EFX_TXQ_MAGIC; 372 etp->et_enp = enp; 373 etp->et_index = index; 374 etp->et_mask = n - 1; 375 etp->et_esmp = esmp; 376 377 /* Set up the new descriptor queue */ 378 EFX_POPULATE_OWORD_6(oword, 379 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 380 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 381 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 382 FRF_AZ_TX_DESCQ_LABEL, label, 383 FRF_AZ_TX_DESCQ_SIZE, size, 384 FRF_AZ_TX_DESCQ_TYPE, 0); 385 386 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 387 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 388 (flags & EFX_CKSUM_IPV4) ? 0 : 1); 389 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 390 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1); 391 392 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 393 etp->et_index, &oword); 394 395 enp->en_tx_qcount++; 396 *etpp = etp; 397 return (0); 398 399fail4: 400 EFSYS_PROBE(fail4); 401fail3: 402 EFSYS_PROBE(fail3); 403fail2: 404 EFSYS_PROBE(fail2); 405fail1: 406 EFSYS_PROBE1(fail1, int, rc); 407 408 return (rc); 409} 410 411#if EFSYS_OPT_QSTATS 412#if EFSYS_OPT_NAMES 413/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */ 414static const char __cs * __cs __efx_tx_qstat_name[] = { 415 "post", 416 "unaligned_split", 417}; 418/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 419 420 const char __cs * 421efx_tx_qstat_name( 422 __in efx_nic_t *enp, 423 __in unsigned int id) 424{ 425 _NOTE(ARGUNUSED(enp)) 426 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 427 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 428 429 return (__efx_tx_qstat_name[id]); 430} 431#endif /* EFSYS_OPT_NAMES */ 432#endif /* EFSYS_OPT_QSTATS */ 433 434#if EFSYS_OPT_QSTATS 435 void 436efx_tx_qstats_update( 437 __in efx_txq_t *etp, 438 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 439{ 440 unsigned int id; 441 442 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 443 444 for (id = 0; id < TX_NQSTATS; id++) { 445 efsys_stat_t *essp = &stat[id]; 446 447 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 448 etp->et_stat[id] = 0; 449 } 450} 451#endif /* EFSYS_OPT_QSTATS */ 452 453 void 454efx_tx_qdestroy( 455 __in efx_txq_t *etp) 456{ 457 efx_nic_t *enp = etp->et_enp; 458 efx_oword_t oword; 459 460 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 461 462 EFSYS_ASSERT(enp->en_tx_qcount != 0); 463 --enp->en_tx_qcount; 464 465 /* Purge descriptor queue */ 466 EFX_ZERO_OWORD(oword); 467 468 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 469 etp->et_index, &oword); 470 471 /* Free the TXQ object */ 472 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 473} 474 475 void 476efx_tx_fini( 477 __in efx_nic_t *enp) 478{ 479 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 480 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 481 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 482 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 483 484 enp->en_mod_flags &= ~EFX_MOD_TX; 485} 486