Deleted Added
sdiff udiff text old ( 277886 ) new ( 278839 )
full compact
efx_tx.c (277886) efx_tx.c (278839)
1/*-
2 * Copyright 2007-2009 Solarflare Communications Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
1/*-
2 * Copyright 2007-2009 Solarflare Communications Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_tx.c 277886 2015-01-29 18:54:43Z arybchik $");
27__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_tx.c 278839 2015-02-16 06:12:04Z arybchik $");
28
29#include "efsys.h"
30#include "efx.h"
31#include "efx_types.h"
32#include "efx_regs.h"
33#include "efx_impl.h"
34
35#if EFSYS_OPT_QSTATS
36#define EFX_TX_QSTAT_INCR(_etp, _stat) \
37 do { \
38 (_etp)->et_stat[_stat]++; \
39 _NOTE(CONSTANTCONDITION) \
40 } while (B_FALSE)
41#else
42#define EFX_TX_QSTAT_INCR(_etp, _stat)
43#endif
44
45 __checkReturn int
46efx_tx_init(
47 __in efx_nic_t *enp)
48{
49 efx_oword_t oword;
50 int rc;
51
52 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
53 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
54
55 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
56 rc = EINVAL;
57 goto fail1;
58 }
59
60 if (enp->en_mod_flags & EFX_MOD_TX) {
61 rc = EINVAL;
62 goto fail2;
63 }
64
65 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
66
67 /*
68 * Disable the timer-based TX DMA backoff and allow TX DMA to be
69 * controlled by the RX FIFO fill level (although always allow a
70 * minimal trickle).
71 */
72 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
73 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
74 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
75 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
76 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
77 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
78 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
79 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
80
81 /*
82 * Filter all packets less than 14 bytes to avoid parsing
83 * errors.
84 */
85 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
86 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
87
88 /*
89 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
90 * descriptors (which is bad).
91 */
92 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
93 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
94 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
95
96 enp->en_mod_flags |= EFX_MOD_TX;
97 return (0);
98
99fail2:
100 EFSYS_PROBE(fail2);
101fail1:
102 EFSYS_PROBE1(fail1, int, rc);
103
104 return (rc);
105}
106
107#if EFSYS_OPT_FILTER
108extern __checkReturn int
109efx_tx_filter_insert(
110 __in efx_txq_t *etp,
111 __inout efx_filter_spec_t *spec)
112{
113 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
114 EFSYS_ASSERT3P(spec, !=, NULL);
115
116 spec->efs_dmaq_id = (uint16_t)etp->et_index;
28
29#include "efsys.h"
30#include "efx.h"
31#include "efx_types.h"
32#include "efx_regs.h"
33#include "efx_impl.h"
34
35#if EFSYS_OPT_QSTATS
36#define EFX_TX_QSTAT_INCR(_etp, _stat) \
37 do { \
38 (_etp)->et_stat[_stat]++; \
39 _NOTE(CONSTANTCONDITION) \
40 } while (B_FALSE)
41#else
42#define EFX_TX_QSTAT_INCR(_etp, _stat)
43#endif
44
45 __checkReturn int
46efx_tx_init(
47 __in efx_nic_t *enp)
48{
49 efx_oword_t oword;
50 int rc;
51
52 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
53 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
54
55 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
56 rc = EINVAL;
57 goto fail1;
58 }
59
60 if (enp->en_mod_flags & EFX_MOD_TX) {
61 rc = EINVAL;
62 goto fail2;
63 }
64
65 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
66
67 /*
68 * Disable the timer-based TX DMA backoff and allow TX DMA to be
69 * controlled by the RX FIFO fill level (although always allow a
70 * minimal trickle).
71 */
72 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
73 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
74 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
75 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
76 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
77 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
78 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
79 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
80
81 /*
82 * Filter all packets less than 14 bytes to avoid parsing
83 * errors.
84 */
85 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
86 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
87
88 /*
89 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
90 * descriptors (which is bad).
91 */
92 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
93 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
94 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
95
96 enp->en_mod_flags |= EFX_MOD_TX;
97 return (0);
98
99fail2:
100 EFSYS_PROBE(fail2);
101fail1:
102 EFSYS_PROBE1(fail1, int, rc);
103
104 return (rc);
105}
106
107#if EFSYS_OPT_FILTER
108extern __checkReturn int
109efx_tx_filter_insert(
110 __in efx_txq_t *etp,
111 __inout efx_filter_spec_t *spec)
112{
113 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
114 EFSYS_ASSERT3P(spec, !=, NULL);
115
116 spec->efs_dmaq_id = (uint16_t)etp->et_index;
117 return efx_filter_insert_filter(etp->et_enp, spec, B_FALSE);
117 return (efx_filter_insert_filter(etp->et_enp, spec, B_FALSE));
118}
119#endif
120
121#if EFSYS_OPT_FILTER
122extern __checkReturn int
123efx_tx_filter_remove(
124 __in efx_txq_t *etp,
125 __inout efx_filter_spec_t *spec)
126{
127 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
128 EFSYS_ASSERT3P(spec, !=, NULL);
129
130 spec->efs_dmaq_id = (uint16_t)etp->et_index;
118}
119#endif
120
121#if EFSYS_OPT_FILTER
122extern __checkReturn int
123efx_tx_filter_remove(
124 __in efx_txq_t *etp,
125 __inout efx_filter_spec_t *spec)
126{
127 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
128 EFSYS_ASSERT3P(spec, !=, NULL);
129
130 spec->efs_dmaq_id = (uint16_t)etp->et_index;
131 return efx_filter_remove_filter(etp->et_enp, spec);
131 return (efx_filter_remove_filter(etp->et_enp, spec));
132}
133#endif
134
135#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
136 do { \
137 unsigned int id; \
138 size_t offset; \
139 efx_qword_t qword; \
140 \
141 id = (_added)++ & (_etp)->et_mask; \
142 offset = id * sizeof (efx_qword_t); \
143 \
144 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
145 unsigned int, id, efsys_dma_addr_t, (_addr), \
146 size_t, (_size), boolean_t, (_eop)); \
147 \
148 EFX_POPULATE_QWORD_4(qword, \
149 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
150 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
151 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
152 (uint32_t)((_addr) & 0xffffffff), \
153 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
154 (uint32_t)((_addr) >> 32)); \
155 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
156 \
157 _NOTE(CONSTANTCONDITION) \
158 } while (B_FALSE)
159
160 __checkReturn int
161efx_tx_qpost(
162 __in efx_txq_t *etp,
163 __in_ecount(n) efx_buffer_t *eb,
164 __in unsigned int n,
165 __in unsigned int completed,
166 __inout unsigned int *addedp)
167{
168 unsigned int added = *addedp;
169 unsigned int i;
170 int rc = ENOSPC;
171
172 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
173
174 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
175 goto fail1;
176
177 for (i = 0; i < n; i++) {
178 efx_buffer_t *ebp = &eb[i];
179 efsys_dma_addr_t start = ebp->eb_addr;
180 size_t size = ebp->eb_size;
181 efsys_dma_addr_t end = start + size;
182
183 /* Fragments must not span 4k boundaries. */
184 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
185
186 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
187 }
188
189 EFX_TX_QSTAT_INCR(etp, TX_POST);
190
191 *addedp = added;
192 return (0);
193
194fail1:
195 EFSYS_PROBE1(fail1, int, rc);
196
197 return (rc);
198}
199
200 void
201efx_tx_qpush(
202 __in efx_txq_t *etp,
203 __in unsigned int added)
204{
205 efx_nic_t *enp = etp->et_enp;
206 uint32_t wptr;
207 efx_dword_t dword;
208 efx_oword_t oword;
209
210 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
211
212 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
213 EFSYS_PIO_WRITE_BARRIER();
214
215 /* Push the populated descriptors out */
216 wptr = added & etp->et_mask;
217
218 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
219
220 /* Only write the third DWORD */
221 EFX_POPULATE_DWORD_1(dword,
222 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
223 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
224 etp->et_index, &dword, B_FALSE);
225}
226
227 void
228efx_tx_qflush(
229 __in efx_txq_t *etp)
230{
231 efx_nic_t *enp = etp->et_enp;
232 efx_oword_t oword;
233 uint32_t label;
234
235 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
236
237 label = etp->et_index;
238
239 /* Flush the queue */
240 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
241 FRF_AZ_TX_FLUSH_DESCQ, label);
242 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
243}
244
245 void
246efx_tx_qenable(
247 __in efx_txq_t *etp)
248{
249 efx_nic_t *enp = etp->et_enp;
250 efx_oword_t oword;
251
252 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
253
254 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
255 etp->et_index, &oword);
256
257 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
258 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
259 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
260 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
261 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
262
263 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
264 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
265 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
266
267 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
268 etp->et_index, &oword);
269}
270
271 __checkReturn int
272efx_tx_qcreate(
273 __in efx_nic_t *enp,
274 __in unsigned int index,
275 __in unsigned int label,
276 __in efsys_mem_t *esmp,
277 __in size_t n,
278 __in uint32_t id,
279 __in uint16_t flags,
280 __in efx_evq_t *eep,
281 __deref_out efx_txq_t **etpp)
282{
283 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
284 efx_txq_t *etp;
285 efx_oword_t oword;
286 uint32_t size;
287 int rc;
288
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
291
292 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
293 /* EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);*/
294 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
295
296 if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) {
297 rc = EINVAL;
298 goto fail1;
299 }
300 if (index >= encp->enc_txq_limit) {
301 rc = EINVAL;
302 goto fail2;
303 }
304 for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS);
305 size++)
306 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
307 break;
308 if (id + (1 << size) >= encp->enc_buftbl_limit) {
309 rc = EINVAL;
310 goto fail3;
311 }
312
313 /* Allocate an TXQ object */
314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
315
316 if (etp == NULL) {
317 rc = ENOMEM;
318 goto fail4;
319 }
320
321 etp->et_magic = EFX_TXQ_MAGIC;
322 etp->et_enp = enp;
323 etp->et_index = index;
324 etp->et_mask = n - 1;
325 etp->et_esmp = esmp;
326
327 /* Set up the new descriptor queue */
328 EFX_POPULATE_OWORD_6(oword,
329 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
330 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
331 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
332 FRF_AZ_TX_DESCQ_LABEL, label,
333 FRF_AZ_TX_DESCQ_SIZE, size,
334 FRF_AZ_TX_DESCQ_TYPE, 0);
335
336 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
337 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
338 (flags & EFX_CKSUM_IPV4) ? 0 : 1);
339 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
340 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1);
341
342 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
343 etp->et_index, &oword);
344
345 enp->en_tx_qcount++;
346 *etpp = etp;
347 return (0);
348
349fail4:
350 EFSYS_PROBE(fail4);
351fail3:
352 EFSYS_PROBE(fail3);
353fail2:
354 EFSYS_PROBE(fail2);
355fail1:
356 EFSYS_PROBE1(fail1, int, rc);
357
358 return (rc);
359}
360
361#if EFSYS_OPT_QSTATS
362#if EFSYS_OPT_NAMES
363/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */
364static const char __cs * __cs __efx_tx_qstat_name[] = {
365 "post",
366 "unaligned_split",
367};
368/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
369
370 const char __cs *
371efx_tx_qstat_name(
372 __in efx_nic_t *enp,
373 __in unsigned int id)
374{
375 _NOTE(ARGUNUSED(enp))
376 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
377 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
378
379 return (__efx_tx_qstat_name[id]);
380}
381#endif /* EFSYS_OPT_NAMES */
382#endif /* EFSYS_OPT_QSTATS */
383
384#if EFSYS_OPT_QSTATS
385 void
386efx_tx_qstats_update(
387 __in efx_txq_t *etp,
388 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
389{
390 unsigned int id;
391
392 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
393
394 for (id = 0; id < TX_NQSTATS; id++) {
395 efsys_stat_t *essp = &stat[id];
396
397 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
398 etp->et_stat[id] = 0;
399 }
400}
401#endif /* EFSYS_OPT_QSTATS */
402
403 void
404efx_tx_qdestroy(
405 __in efx_txq_t *etp)
406{
407 efx_nic_t *enp = etp->et_enp;
408 efx_oword_t oword;
409
410 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
411
412 EFSYS_ASSERT(enp->en_tx_qcount != 0);
413 --enp->en_tx_qcount;
414
415 /* Purge descriptor queue */
416 EFX_ZERO_OWORD(oword);
417
418 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
419 etp->et_index, &oword);
420
421 /* Free the TXQ object */
422 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
423}
424
425 void
426efx_tx_fini(
427 __in efx_nic_t *enp)
428{
429 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
430 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
431 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
432 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
433
434 enp->en_mod_flags &= ~EFX_MOD_TX;
435}
132}
133#endif
134
135#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
136 do { \
137 unsigned int id; \
138 size_t offset; \
139 efx_qword_t qword; \
140 \
141 id = (_added)++ & (_etp)->et_mask; \
142 offset = id * sizeof (efx_qword_t); \
143 \
144 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
145 unsigned int, id, efsys_dma_addr_t, (_addr), \
146 size_t, (_size), boolean_t, (_eop)); \
147 \
148 EFX_POPULATE_QWORD_4(qword, \
149 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
150 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
151 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
152 (uint32_t)((_addr) & 0xffffffff), \
153 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
154 (uint32_t)((_addr) >> 32)); \
155 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
156 \
157 _NOTE(CONSTANTCONDITION) \
158 } while (B_FALSE)
159
160 __checkReturn int
161efx_tx_qpost(
162 __in efx_txq_t *etp,
163 __in_ecount(n) efx_buffer_t *eb,
164 __in unsigned int n,
165 __in unsigned int completed,
166 __inout unsigned int *addedp)
167{
168 unsigned int added = *addedp;
169 unsigned int i;
170 int rc = ENOSPC;
171
172 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
173
174 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
175 goto fail1;
176
177 for (i = 0; i < n; i++) {
178 efx_buffer_t *ebp = &eb[i];
179 efsys_dma_addr_t start = ebp->eb_addr;
180 size_t size = ebp->eb_size;
181 efsys_dma_addr_t end = start + size;
182
183 /* Fragments must not span 4k boundaries. */
184 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
185
186 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
187 }
188
189 EFX_TX_QSTAT_INCR(etp, TX_POST);
190
191 *addedp = added;
192 return (0);
193
194fail1:
195 EFSYS_PROBE1(fail1, int, rc);
196
197 return (rc);
198}
199
200 void
201efx_tx_qpush(
202 __in efx_txq_t *etp,
203 __in unsigned int added)
204{
205 efx_nic_t *enp = etp->et_enp;
206 uint32_t wptr;
207 efx_dword_t dword;
208 efx_oword_t oword;
209
210 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
211
212 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
213 EFSYS_PIO_WRITE_BARRIER();
214
215 /* Push the populated descriptors out */
216 wptr = added & etp->et_mask;
217
218 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
219
220 /* Only write the third DWORD */
221 EFX_POPULATE_DWORD_1(dword,
222 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
223 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
224 etp->et_index, &dword, B_FALSE);
225}
226
227 void
228efx_tx_qflush(
229 __in efx_txq_t *etp)
230{
231 efx_nic_t *enp = etp->et_enp;
232 efx_oword_t oword;
233 uint32_t label;
234
235 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
236
237 label = etp->et_index;
238
239 /* Flush the queue */
240 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
241 FRF_AZ_TX_FLUSH_DESCQ, label);
242 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
243}
244
245 void
246efx_tx_qenable(
247 __in efx_txq_t *etp)
248{
249 efx_nic_t *enp = etp->et_enp;
250 efx_oword_t oword;
251
252 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
253
254 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
255 etp->et_index, &oword);
256
257 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
258 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
259 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
260 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
261 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
262
263 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
264 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
265 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
266
267 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
268 etp->et_index, &oword);
269}
270
271 __checkReturn int
272efx_tx_qcreate(
273 __in efx_nic_t *enp,
274 __in unsigned int index,
275 __in unsigned int label,
276 __in efsys_mem_t *esmp,
277 __in size_t n,
278 __in uint32_t id,
279 __in uint16_t flags,
280 __in efx_evq_t *eep,
281 __deref_out efx_txq_t **etpp)
282{
283 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
284 efx_txq_t *etp;
285 efx_oword_t oword;
286 uint32_t size;
287 int rc;
288
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
291
292 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
293 /* EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);*/
294 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
295
296 if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) {
297 rc = EINVAL;
298 goto fail1;
299 }
300 if (index >= encp->enc_txq_limit) {
301 rc = EINVAL;
302 goto fail2;
303 }
304 for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS);
305 size++)
306 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
307 break;
308 if (id + (1 << size) >= encp->enc_buftbl_limit) {
309 rc = EINVAL;
310 goto fail3;
311 }
312
313 /* Allocate an TXQ object */
314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
315
316 if (etp == NULL) {
317 rc = ENOMEM;
318 goto fail4;
319 }
320
321 etp->et_magic = EFX_TXQ_MAGIC;
322 etp->et_enp = enp;
323 etp->et_index = index;
324 etp->et_mask = n - 1;
325 etp->et_esmp = esmp;
326
327 /* Set up the new descriptor queue */
328 EFX_POPULATE_OWORD_6(oword,
329 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
330 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
331 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
332 FRF_AZ_TX_DESCQ_LABEL, label,
333 FRF_AZ_TX_DESCQ_SIZE, size,
334 FRF_AZ_TX_DESCQ_TYPE, 0);
335
336 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
337 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
338 (flags & EFX_CKSUM_IPV4) ? 0 : 1);
339 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
340 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1);
341
342 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
343 etp->et_index, &oword);
344
345 enp->en_tx_qcount++;
346 *etpp = etp;
347 return (0);
348
349fail4:
350 EFSYS_PROBE(fail4);
351fail3:
352 EFSYS_PROBE(fail3);
353fail2:
354 EFSYS_PROBE(fail2);
355fail1:
356 EFSYS_PROBE1(fail1, int, rc);
357
358 return (rc);
359}
360
361#if EFSYS_OPT_QSTATS
362#if EFSYS_OPT_NAMES
363/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */
364static const char __cs * __cs __efx_tx_qstat_name[] = {
365 "post",
366 "unaligned_split",
367};
368/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
369
370 const char __cs *
371efx_tx_qstat_name(
372 __in efx_nic_t *enp,
373 __in unsigned int id)
374{
375 _NOTE(ARGUNUSED(enp))
376 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
377 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
378
379 return (__efx_tx_qstat_name[id]);
380}
381#endif /* EFSYS_OPT_NAMES */
382#endif /* EFSYS_OPT_QSTATS */
383
384#if EFSYS_OPT_QSTATS
385 void
386efx_tx_qstats_update(
387 __in efx_txq_t *etp,
388 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
389{
390 unsigned int id;
391
392 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
393
394 for (id = 0; id < TX_NQSTATS; id++) {
395 efsys_stat_t *essp = &stat[id];
396
397 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
398 etp->et_stat[id] = 0;
399 }
400}
401#endif /* EFSYS_OPT_QSTATS */
402
403 void
404efx_tx_qdestroy(
405 __in efx_txq_t *etp)
406{
407 efx_nic_t *enp = etp->et_enp;
408 efx_oword_t oword;
409
410 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
411
412 EFSYS_ASSERT(enp->en_tx_qcount != 0);
413 --enp->en_tx_qcount;
414
415 /* Purge descriptor queue */
416 EFX_ZERO_OWORD(oword);
417
418 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
419 etp->et_index, &oword);
420
421 /* Free the TXQ object */
422 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
423}
424
425 void
426efx_tx_fini(
427 __in efx_nic_t *enp)
428{
429 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
430 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
431 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
432 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
433
434 enp->en_mod_flags &= ~EFX_MOD_TX;
435}