1/*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
| 1/*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
|
67
68 txq = next;
69 } while (txq != NULL);
70 }
71
72 if (rxq->pending != rxq->completed)
73 sfxge_rx_qcomplete(rxq, eop);
74}
75
76static boolean_t
77sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
78 uint16_t flags)
79{
80 struct sfxge_evq *evq;
81 struct sfxge_softc *sc;
82 struct sfxge_rxq *rxq;
83 unsigned int expected;
84 struct sfxge_rx_sw_desc *rx_desc;
85
86 evq = arg;
87 sc = evq->sc;
88
89 if (evq->exception)
90 goto done;
91
92 rxq = sc->rxq[label];
93 KASSERT(rxq != NULL, ("rxq == NULL"));
94 KASSERT(evq->index == rxq->index,
95 ("evq->index != rxq->index"));
96
97 if (rxq->init_state != SFXGE_RXQ_STARTED)
98 goto done;
99
100 expected = rxq->pending++ & rxq->ptr_mask;
101 if (id != expected) {
102 evq->exception = B_TRUE;
103
104 device_printf(sc->dev, "RX completion out of order"
105 " (id=%#x expected=%#x flags=%#x); resetting\n",
106 id, expected, flags);
107 sfxge_schedule_reset(sc);
108
109 goto done;
110 }
111
112 rx_desc = &rxq->queue[id];
113
114 KASSERT(rx_desc->flags == EFX_DISCARD,
115 ("rx_desc->flags != EFX_DISCARD"));
116 rx_desc->flags = flags;
117
118 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
119 rx_desc->size = (uint16_t)size;
120 prefetch_read_many(rx_desc->mbuf);
121
122 evq->rx_done++;
123
124 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
125 sfxge_ev_qcomplete(evq, B_FALSE);
126
127done:
128 return (evq->rx_done >= SFXGE_EV_BATCH);
129}
130
131static boolean_t
132sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
133{
134 struct sfxge_evq *evq;
135 struct sfxge_softc *sc;
136
137 evq = (struct sfxge_evq *)arg;
138 sc = evq->sc;
139
140 evq->exception = B_TRUE;
141
142 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
143 device_printf(sc->dev,
144 "hardware exception (code=%u); resetting\n",
145 code);
146 sfxge_schedule_reset(sc);
147 }
148
149 return (B_FALSE);
150}
151
152static boolean_t
153sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
154{
155 struct sfxge_evq *evq;
156 struct sfxge_softc *sc;
157 struct sfxge_rxq *rxq;
158 unsigned int index;
159 unsigned int label;
160 uint16_t magic;
161
162 evq = (struct sfxge_evq *)arg;
163 sc = evq->sc;
164 rxq = sc->rxq[rxq_index];
165
166 KASSERT(rxq != NULL, ("rxq == NULL"));
167
168 /* Resend a software event on the correct queue */
169 index = rxq->index;
170 evq = sc->evq[index];
171
172 label = rxq_index;
173 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
174 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
175 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
176
177 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
178 ("evq not started"));
179 efx_ev_qpost(evq->common, magic);
180
181 return (B_FALSE);
182}
183
184static boolean_t
185sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
186{
187 struct sfxge_evq *evq;
188 struct sfxge_softc *sc;
189 struct sfxge_rxq *rxq;
190 unsigned int index;
191 unsigned int label;
192 uint16_t magic;
193
194 evq = (struct sfxge_evq *)arg;
195 sc = evq->sc;
196 rxq = sc->rxq[rxq_index];
197
198 KASSERT(rxq != NULL, ("rxq == NULL"));
199
200 /* Resend a software event on the correct queue */
201 index = rxq->index;
202 evq = sc->evq[index];
203
204 label = rxq_index;
205 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
206 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
207 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
208
209 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
210 ("evq not started"));
211 efx_ev_qpost(evq->common, magic);
212
213 return (B_FALSE);
214}
215
216static struct sfxge_txq *
217sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
218{
219 unsigned int index;
220
221 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
222 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
223 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
224 return (evq->sc->txq[index]);
225}
226
227static boolean_t
228sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
229{
230 struct sfxge_evq *evq;
231 struct sfxge_txq *txq;
232 unsigned int stop;
233 unsigned int delta;
234
235 evq = (struct sfxge_evq *)arg;
236 txq = sfxge_get_txq_by_label(evq, label);
237
238 KASSERT(txq != NULL, ("txq == NULL"));
239 KASSERT(evq->index == txq->evq_index,
240 ("evq->index != txq->evq_index"));
241
242 if (txq->init_state != SFXGE_TXQ_STARTED)
243 goto done;
244
245 stop = (id + 1) & txq->ptr_mask;
246 id = txq->pending & txq->ptr_mask;
247
248 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
249 txq->pending += delta;
250
251 evq->tx_done++;
252
253 if (txq->next == NULL &&
254 evq->txqs != &(txq->next)) {
255 *(evq->txqs) = txq;
256 evq->txqs = &(txq->next);
257 }
258
259 if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
| 67
68 txq = next;
69 } while (txq != NULL);
70 }
71
72 if (rxq->pending != rxq->completed)
73 sfxge_rx_qcomplete(rxq, eop);
74}
75
76static boolean_t
77sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
78 uint16_t flags)
79{
80 struct sfxge_evq *evq;
81 struct sfxge_softc *sc;
82 struct sfxge_rxq *rxq;
83 unsigned int expected;
84 struct sfxge_rx_sw_desc *rx_desc;
85
86 evq = arg;
87 sc = evq->sc;
88
89 if (evq->exception)
90 goto done;
91
92 rxq = sc->rxq[label];
93 KASSERT(rxq != NULL, ("rxq == NULL"));
94 KASSERT(evq->index == rxq->index,
95 ("evq->index != rxq->index"));
96
97 if (rxq->init_state != SFXGE_RXQ_STARTED)
98 goto done;
99
100 expected = rxq->pending++ & rxq->ptr_mask;
101 if (id != expected) {
102 evq->exception = B_TRUE;
103
104 device_printf(sc->dev, "RX completion out of order"
105 " (id=%#x expected=%#x flags=%#x); resetting\n",
106 id, expected, flags);
107 sfxge_schedule_reset(sc);
108
109 goto done;
110 }
111
112 rx_desc = &rxq->queue[id];
113
114 KASSERT(rx_desc->flags == EFX_DISCARD,
115 ("rx_desc->flags != EFX_DISCARD"));
116 rx_desc->flags = flags;
117
118 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
119 rx_desc->size = (uint16_t)size;
120 prefetch_read_many(rx_desc->mbuf);
121
122 evq->rx_done++;
123
124 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
125 sfxge_ev_qcomplete(evq, B_FALSE);
126
127done:
128 return (evq->rx_done >= SFXGE_EV_BATCH);
129}
130
131static boolean_t
132sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
133{
134 struct sfxge_evq *evq;
135 struct sfxge_softc *sc;
136
137 evq = (struct sfxge_evq *)arg;
138 sc = evq->sc;
139
140 evq->exception = B_TRUE;
141
142 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
143 device_printf(sc->dev,
144 "hardware exception (code=%u); resetting\n",
145 code);
146 sfxge_schedule_reset(sc);
147 }
148
149 return (B_FALSE);
150}
151
152static boolean_t
153sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
154{
155 struct sfxge_evq *evq;
156 struct sfxge_softc *sc;
157 struct sfxge_rxq *rxq;
158 unsigned int index;
159 unsigned int label;
160 uint16_t magic;
161
162 evq = (struct sfxge_evq *)arg;
163 sc = evq->sc;
164 rxq = sc->rxq[rxq_index];
165
166 KASSERT(rxq != NULL, ("rxq == NULL"));
167
168 /* Resend a software event on the correct queue */
169 index = rxq->index;
170 evq = sc->evq[index];
171
172 label = rxq_index;
173 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
174 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
175 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
176
177 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
178 ("evq not started"));
179 efx_ev_qpost(evq->common, magic);
180
181 return (B_FALSE);
182}
183
184static boolean_t
185sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
186{
187 struct sfxge_evq *evq;
188 struct sfxge_softc *sc;
189 struct sfxge_rxq *rxq;
190 unsigned int index;
191 unsigned int label;
192 uint16_t magic;
193
194 evq = (struct sfxge_evq *)arg;
195 sc = evq->sc;
196 rxq = sc->rxq[rxq_index];
197
198 KASSERT(rxq != NULL, ("rxq == NULL"));
199
200 /* Resend a software event on the correct queue */
201 index = rxq->index;
202 evq = sc->evq[index];
203
204 label = rxq_index;
205 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
206 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
207 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
208
209 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
210 ("evq not started"));
211 efx_ev_qpost(evq->common, magic);
212
213 return (B_FALSE);
214}
215
216static struct sfxge_txq *
217sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
218{
219 unsigned int index;
220
221 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
222 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
223 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
224 return (evq->sc->txq[index]);
225}
226
227static boolean_t
228sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
229{
230 struct sfxge_evq *evq;
231 struct sfxge_txq *txq;
232 unsigned int stop;
233 unsigned int delta;
234
235 evq = (struct sfxge_evq *)arg;
236 txq = sfxge_get_txq_by_label(evq, label);
237
238 KASSERT(txq != NULL, ("txq == NULL"));
239 KASSERT(evq->index == txq->evq_index,
240 ("evq->index != txq->evq_index"));
241
242 if (txq->init_state != SFXGE_TXQ_STARTED)
243 goto done;
244
245 stop = (id + 1) & txq->ptr_mask;
246 id = txq->pending & txq->ptr_mask;
247
248 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
249 txq->pending += delta;
250
251 evq->tx_done++;
252
253 if (txq->next == NULL &&
254 evq->txqs != &(txq->next)) {
255 *(evq->txqs) = txq;
256 evq->txqs = &(txq->next);
257 }
258
259 if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
|
261
262done:
263 return (evq->tx_done >= SFXGE_EV_BATCH);
264}
265
266static boolean_t
267sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
268{
269 struct sfxge_evq *evq;
270 struct sfxge_softc *sc;
271 struct sfxge_txq *txq;
272 unsigned int label;
273 uint16_t magic;
274
275 evq = (struct sfxge_evq *)arg;
276 sc = evq->sc;
277 txq = sc->txq[txq_index];
278
279 KASSERT(txq != NULL, ("txq == NULL"));
280 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
281 ("txq not initialized"));
282
283 /* Resend a software event on the correct queue */
284 evq = sc->evq[txq->evq_index];
285
286 label = txq->type;
287 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
288 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
289 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
290
291 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
292 ("evq not started"));
293 efx_ev_qpost(evq->common, magic);
294
295 return (B_FALSE);
296}
297
298static boolean_t
299sfxge_ev_software(void *arg, uint16_t magic)
300{
301 struct sfxge_evq *evq;
302 struct sfxge_softc *sc;
303 unsigned int label;
304
305 evq = (struct sfxge_evq *)arg;
306 sc = evq->sc;
307
308 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
309 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
310
311 switch (magic) {
312 case SFXGE_MAGIC_RX_QFLUSH_DONE: {
313 struct sfxge_rxq *rxq = sc->rxq[label];
314
315 KASSERT(rxq != NULL, ("rxq == NULL"));
316 KASSERT(evq->index == rxq->index,
317 ("evq->index != rxq->index"));
318
319 sfxge_rx_qflush_done(rxq);
320 break;
321 }
322 case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
323 struct sfxge_rxq *rxq = sc->rxq[label];
324
325 KASSERT(rxq != NULL, ("rxq == NULL"));
326 KASSERT(evq->index == rxq->index,
327 ("evq->index != rxq->index"));
328
329 sfxge_rx_qflush_failed(rxq);
330 break;
331 }
332 case SFXGE_MAGIC_RX_QREFILL: {
333 struct sfxge_rxq *rxq = sc->rxq[label];
334
335 KASSERT(rxq != NULL, ("rxq == NULL"));
336 KASSERT(evq->index == rxq->index,
337 ("evq->index != rxq->index"));
338
339 sfxge_rx_qrefill(rxq);
340 break;
341 }
342 case SFXGE_MAGIC_TX_QFLUSH_DONE: {
343 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
344
345 KASSERT(txq != NULL, ("txq == NULL"));
346 KASSERT(evq->index == txq->evq_index,
347 ("evq->index != txq->evq_index"));
348
349 sfxge_tx_qflush_done(txq);
350 break;
351 }
352 default:
353 break;
354 }
355
356 return (B_FALSE);
357}
358
359static boolean_t
360sfxge_ev_sram(void *arg, uint32_t code)
361{
362 (void)arg;
363 (void)code;
364
365 switch (code) {
366 case EFX_SRAM_UPDATE:
367 EFSYS_PROBE(sram_update);
368 break;
369
370 case EFX_SRAM_CLEAR:
371 EFSYS_PROBE(sram_clear);
372 break;
373
374 case EFX_SRAM_ILLEGAL_CLEAR:
375 EFSYS_PROBE(sram_illegal_clear);
376 break;
377
378 default:
379 KASSERT(B_FALSE, ("Impossible SRAM event"));
380 break;
381 }
382
383 return (B_FALSE);
384}
385
386static boolean_t
387sfxge_ev_timer(void *arg, uint32_t index)
388{
389 (void)arg;
390 (void)index;
391
392 return (B_FALSE);
393}
394
395static boolean_t
396sfxge_ev_wake_up(void *arg, uint32_t index)
397{
398 (void)arg;
399 (void)index;
400
401 return (B_FALSE);
402}
403
404#if EFSYS_OPT_QSTATS
405
406static void
407sfxge_ev_stat_update(struct sfxge_softc *sc)
408{
409 struct sfxge_evq *evq;
410 unsigned int index;
411 clock_t now;
412
413 sx_xlock(&sc->softc_lock);
414
415 if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
416 goto out;
417
418 now = ticks;
419 if (now - sc->ev_stats_update_time < hz)
420 goto out;
421
422 sc->ev_stats_update_time = now;
423
424 /* Add event counts from each event queue in turn */
425 for (index = 0; index < sc->intr.n_alloc; index++) {
426 evq = sc->evq[index];
427 mtx_lock(&evq->lock);
428 efx_ev_qstats_update(evq->common, sc->ev_stats);
429 mtx_unlock(&evq->lock);
430 }
431out:
432 sx_xunlock(&sc->softc_lock);
433}
434
435static int
436sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
437{
438 struct sfxge_softc *sc = arg1;
439 unsigned int id = arg2;
440
441 sfxge_ev_stat_update(sc);
442
443 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
444}
445
446static void
447sfxge_ev_stat_init(struct sfxge_softc *sc)
448{
449 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
450 struct sysctl_oid_list *stat_list;
451 unsigned int id;
452 char name[40];
453
454 stat_list = SYSCTL_CHILDREN(sc->stats_node);
455
456 for (id = 0; id < EV_NQSTATS; id++) {
457 snprintf(name, sizeof(name), "ev_%s",
458 efx_ev_qstat_name(sc->enp, id));
459 SYSCTL_ADD_PROC(
460 ctx, stat_list,
461 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
462 sc, id, sfxge_ev_stat_handler, "Q",
463 "");
464 }
465}
466
467#endif /* EFSYS_OPT_QSTATS */
468
469static void
470sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
471{
472 struct sfxge_evq *evq;
473 efx_evq_t *eep;
474
475 evq = sc->evq[idx];
476 eep = evq->common;
477
478 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
479 ("evq->init_state != SFXGE_EVQ_STARTED"));
480
481 (void)efx_ev_qmoderate(eep, us);
482}
483
484static int
485sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
486{
487 struct sfxge_softc *sc = arg1;
488 struct sfxge_intr *intr = &sc->intr;
489 unsigned int moderation;
490 int error;
491 int index;
492
493 sx_xlock(&sc->softc_lock);
494
495 if (req->newptr != NULL) {
496 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
497 != 0)
498 goto out;
499
500 /* We may not be calling efx_ev_qmoderate() now,
501 * so we have to range-check the value ourselves.
502 */
503 if (moderation >
504 efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
505 error = EINVAL;
506 goto out;
507 }
508
509 sc->ev_moderation = moderation;
510 if (intr->state == SFXGE_INTR_STARTED) {
511 for (index = 0; index < intr->n_alloc; index++)
512 sfxge_ev_qmoderate(sc, index, moderation);
513 }
514 } else {
515 error = SYSCTL_OUT(req, &sc->ev_moderation,
516 sizeof(sc->ev_moderation));
517 }
518
519out:
520 sx_xunlock(&sc->softc_lock);
521
522 return (error);
523}
524
525static boolean_t
526sfxge_ev_initialized(void *arg)
527{
528 struct sfxge_evq *evq;
529
530 evq = (struct sfxge_evq *)arg;
531
532 KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
533 ("evq not starting"));
534
535 evq->init_state = SFXGE_EVQ_STARTED;
536
537 return (0);
538}
539
540static boolean_t
541sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
542{
543 struct sfxge_evq *evq;
544 struct sfxge_softc *sc;
545
546 evq = (struct sfxge_evq *)arg;
547 sc = evq->sc;
548
549 sfxge_mac_link_update(sc, link_mode);
550
551 return (0);
552}
553
554static const efx_ev_callbacks_t sfxge_ev_callbacks = {
555 .eec_initialized = sfxge_ev_initialized,
556 .eec_rx = sfxge_ev_rx,
557 .eec_tx = sfxge_ev_tx,
558 .eec_exception = sfxge_ev_exception,
559 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done,
560 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed,
561 .eec_txq_flush_done = sfxge_ev_txq_flush_done,
562 .eec_software = sfxge_ev_software,
563 .eec_sram = sfxge_ev_sram,
564 .eec_wake_up = sfxge_ev_wake_up,
565 .eec_timer = sfxge_ev_timer,
566 .eec_link_change = sfxge_ev_link_change,
567};
568
569
570int
571sfxge_ev_qpoll(struct sfxge_evq *evq)
572{
573 int rc;
574
575 mtx_lock(&evq->lock);
576
577 if (evq->init_state != SFXGE_EVQ_STARTING &&
578 evq->init_state != SFXGE_EVQ_STARTED) {
579 rc = EINVAL;
580 goto fail;
581 }
582
583 /* Synchronize the DMA memory for reading */
584 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
585 BUS_DMASYNC_POSTREAD);
586
587 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
588 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
589 KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
590 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
591
592 /* Poll the queue */
593 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
594
595 evq->rx_done = 0;
596 evq->tx_done = 0;
597
598 /* Perform any pending completion processing */
599 sfxge_ev_qcomplete(evq, B_TRUE);
600
601 /* Re-prime the event queue for interrupts */
602 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
603 goto fail;
604
605 mtx_unlock(&evq->lock);
606
607 return (0);
608
609fail:
610 mtx_unlock(&(evq->lock));
611 return (rc);
612}
613
614static void
615sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
616{
617 struct sfxge_evq *evq;
618
619 evq = sc->evq[index];
620
621 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
622 ("evq->init_state != SFXGE_EVQ_STARTED"));
623
624 mtx_lock(&evq->lock);
625 evq->init_state = SFXGE_EVQ_INITIALIZED;
626 evq->read_ptr = 0;
627 evq->exception = B_FALSE;
628
629#if EFSYS_OPT_QSTATS
630 /* Add event counts before discarding the common evq state */
631 efx_ev_qstats_update(evq->common, sc->ev_stats);
632#endif
633
634 efx_ev_qdestroy(evq->common);
635 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
636 EFX_EVQ_NBUFS(evq->entries));
637 mtx_unlock(&evq->lock);
638}
639
640static int
641sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
642{
643 struct sfxge_evq *evq;
644 efsys_mem_t *esmp;
645 int count;
646 int rc;
647
648 evq = sc->evq[index];
649 esmp = &evq->mem;
650
651 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
652 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
653
654 /* Clear all events. */
655 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
656
657 /* Program the buffer table. */
658 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
659 EFX_EVQ_NBUFS(evq->entries))) != 0)
660 return (rc);
661
662 /* Create the common code event queue. */
663 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
664 evq->buf_base_id, &evq->common)) != 0)
665 goto fail;
666
667 mtx_lock(&evq->lock);
668
669 /* Set the default moderation */
670 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
671
672 /* Prime the event queue for interrupts */
673 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
674 goto fail2;
675
676 evq->init_state = SFXGE_EVQ_STARTING;
677
678 mtx_unlock(&evq->lock);
679
680 /* Wait for the initialization event */
681 count = 0;
682 do {
683 /* Pause for 100 ms */
684 pause("sfxge evq init", hz / 10);
685
686 /* Check to see if the test event has been processed */
687 if (evq->init_state == SFXGE_EVQ_STARTED)
688 goto done;
689
690 } while (++count < 20);
691
692 rc = ETIMEDOUT;
693 goto fail3;
694
695done:
696 return (0);
697
698fail3:
699 mtx_lock(&evq->lock);
700 evq->init_state = SFXGE_EVQ_INITIALIZED;
701fail2:
702 mtx_unlock(&evq->lock);
703 efx_ev_qdestroy(evq->common);
704fail:
705 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
706 EFX_EVQ_NBUFS(evq->entries));
707
708 return (rc);
709}
710
711void
712sfxge_ev_stop(struct sfxge_softc *sc)
713{
714 struct sfxge_intr *intr;
715 efx_nic_t *enp;
716 int index;
717
718 intr = &sc->intr;
719 enp = sc->enp;
720
721 KASSERT(intr->state == SFXGE_INTR_STARTED,
722 ("Interrupts not started"));
723
724 /* Stop the event queue(s) */
725 index = intr->n_alloc;
726 while (--index >= 0)
727 sfxge_ev_qstop(sc, index);
728
729 /* Tear down the event module */
730 efx_ev_fini(enp);
731}
732
733int
734sfxge_ev_start(struct sfxge_softc *sc)
735{
736 struct sfxge_intr *intr;
737 int index;
738 int rc;
739
740 intr = &sc->intr;
741
742 KASSERT(intr->state == SFXGE_INTR_STARTED,
743 ("intr->state != SFXGE_INTR_STARTED"));
744
745 /* Initialize the event module */
746 if ((rc = efx_ev_init(sc->enp)) != 0)
747 return (rc);
748
749 /* Start the event queues */
750 for (index = 0; index < intr->n_alloc; index++) {
751 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
752 goto fail;
753 }
754
755 return (0);
756
757fail:
758 /* Stop the event queue(s) */
759 while (--index >= 0)
760 sfxge_ev_qstop(sc, index);
761
762 /* Tear down the event module */
763 efx_ev_fini(sc->enp);
764
765 return (rc);
766}
767
768static void
769sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
770{
771 struct sfxge_evq *evq;
772
773 evq = sc->evq[index];
774
775 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
776 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
777 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
778
779 sfxge_dma_free(&evq->mem);
780
781 sc->evq[index] = NULL;
782
783 mtx_destroy(&evq->lock);
784
785 free(evq, M_SFXGE);
786}
787
788static int
789sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
790{
791 struct sfxge_evq *evq;
792 efsys_mem_t *esmp;
793 int rc;
794
795 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
796
797 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
798 evq->sc = sc;
799 evq->index = index;
800 sc->evq[index] = evq;
801 esmp = &evq->mem;
802
803 /* Build an event queue with room for one event per tx and rx buffer,
804 * plus some extra for link state events and MCDI completions.
805 * There are three tx queues in the first event queue and one in
806 * other.
807 */
808 if (index == 0)
809 evq->entries =
810 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
811 3 * sc->txq_entries +
812 128);
813 else
814 evq->entries =
815 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
816 sc->txq_entries +
817 128);
818
819 /* Initialise TX completion list */
820 evq->txqs = &evq->txq;
821
822 /* Allocate DMA space. */
823 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
824 return (rc);
825
826 /* Allocate buffer table entries. */
827 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
828 &evq->buf_base_id);
829
830 mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
831
832 evq->init_state = SFXGE_EVQ_INITIALIZED;
833
834 return (0);
835}
836
837void
838sfxge_ev_fini(struct sfxge_softc *sc)
839{
840 struct sfxge_intr *intr;
841 int index;
842
843 intr = &sc->intr;
844
845 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
846 ("intr->state != SFXGE_INTR_INITIALIZED"));
847
848 sc->ev_moderation = 0;
849
850 /* Tear down the event queue(s). */
851 index = intr->n_alloc;
852 while (--index >= 0)
853 sfxge_ev_qfini(sc, index);
854}
855
856int
857sfxge_ev_init(struct sfxge_softc *sc)
858{
859 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
860 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
861 struct sfxge_intr *intr;
862 int index;
863 int rc;
864
865 intr = &sc->intr;
866
867 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
868 ("intr->state != SFXGE_INTR_INITIALIZED"));
869
870 /* Set default interrupt moderation; add a sysctl to
871 * read and change it.
872 */
873 sc->ev_moderation = 30;
874 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
875 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
876 sc, 0, sfxge_int_mod_handler, "IU",
877 "sfxge interrupt moderation (us)");
878
879 /*
880 * Initialize the event queue(s) - one per interrupt.
881 */
882 for (index = 0; index < intr->n_alloc; index++) {
883 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
884 goto fail;
885 }
886
887#if EFSYS_OPT_QSTATS
888 sfxge_ev_stat_init(sc);
889#endif
890
891 return (0);
892
893fail:
894 while (--index >= 0)
895 sfxge_ev_qfini(sc, index);
896
897 return (rc);
898}
| 261
262done:
263 return (evq->tx_done >= SFXGE_EV_BATCH);
264}
265
266static boolean_t
267sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
268{
269 struct sfxge_evq *evq;
270 struct sfxge_softc *sc;
271 struct sfxge_txq *txq;
272 unsigned int label;
273 uint16_t magic;
274
275 evq = (struct sfxge_evq *)arg;
276 sc = evq->sc;
277 txq = sc->txq[txq_index];
278
279 KASSERT(txq != NULL, ("txq == NULL"));
280 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
281 ("txq not initialized"));
282
283 /* Resend a software event on the correct queue */
284 evq = sc->evq[txq->evq_index];
285
286 label = txq->type;
287 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
288 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
289 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
290
291 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
292 ("evq not started"));
293 efx_ev_qpost(evq->common, magic);
294
295 return (B_FALSE);
296}
297
298static boolean_t
299sfxge_ev_software(void *arg, uint16_t magic)
300{
301 struct sfxge_evq *evq;
302 struct sfxge_softc *sc;
303 unsigned int label;
304
305 evq = (struct sfxge_evq *)arg;
306 sc = evq->sc;
307
308 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
309 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
310
311 switch (magic) {
312 case SFXGE_MAGIC_RX_QFLUSH_DONE: {
313 struct sfxge_rxq *rxq = sc->rxq[label];
314
315 KASSERT(rxq != NULL, ("rxq == NULL"));
316 KASSERT(evq->index == rxq->index,
317 ("evq->index != rxq->index"));
318
319 sfxge_rx_qflush_done(rxq);
320 break;
321 }
322 case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
323 struct sfxge_rxq *rxq = sc->rxq[label];
324
325 KASSERT(rxq != NULL, ("rxq == NULL"));
326 KASSERT(evq->index == rxq->index,
327 ("evq->index != rxq->index"));
328
329 sfxge_rx_qflush_failed(rxq);
330 break;
331 }
332 case SFXGE_MAGIC_RX_QREFILL: {
333 struct sfxge_rxq *rxq = sc->rxq[label];
334
335 KASSERT(rxq != NULL, ("rxq == NULL"));
336 KASSERT(evq->index == rxq->index,
337 ("evq->index != rxq->index"));
338
339 sfxge_rx_qrefill(rxq);
340 break;
341 }
342 case SFXGE_MAGIC_TX_QFLUSH_DONE: {
343 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
344
345 KASSERT(txq != NULL, ("txq == NULL"));
346 KASSERT(evq->index == txq->evq_index,
347 ("evq->index != txq->evq_index"));
348
349 sfxge_tx_qflush_done(txq);
350 break;
351 }
352 default:
353 break;
354 }
355
356 return (B_FALSE);
357}
358
359static boolean_t
360sfxge_ev_sram(void *arg, uint32_t code)
361{
362 (void)arg;
363 (void)code;
364
365 switch (code) {
366 case EFX_SRAM_UPDATE:
367 EFSYS_PROBE(sram_update);
368 break;
369
370 case EFX_SRAM_CLEAR:
371 EFSYS_PROBE(sram_clear);
372 break;
373
374 case EFX_SRAM_ILLEGAL_CLEAR:
375 EFSYS_PROBE(sram_illegal_clear);
376 break;
377
378 default:
379 KASSERT(B_FALSE, ("Impossible SRAM event"));
380 break;
381 }
382
383 return (B_FALSE);
384}
385
386static boolean_t
387sfxge_ev_timer(void *arg, uint32_t index)
388{
389 (void)arg;
390 (void)index;
391
392 return (B_FALSE);
393}
394
395static boolean_t
396sfxge_ev_wake_up(void *arg, uint32_t index)
397{
398 (void)arg;
399 (void)index;
400
401 return (B_FALSE);
402}
403
404#if EFSYS_OPT_QSTATS
405
406static void
407sfxge_ev_stat_update(struct sfxge_softc *sc)
408{
409 struct sfxge_evq *evq;
410 unsigned int index;
411 clock_t now;
412
413 sx_xlock(&sc->softc_lock);
414
415 if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
416 goto out;
417
418 now = ticks;
419 if (now - sc->ev_stats_update_time < hz)
420 goto out;
421
422 sc->ev_stats_update_time = now;
423
424 /* Add event counts from each event queue in turn */
425 for (index = 0; index < sc->intr.n_alloc; index++) {
426 evq = sc->evq[index];
427 mtx_lock(&evq->lock);
428 efx_ev_qstats_update(evq->common, sc->ev_stats);
429 mtx_unlock(&evq->lock);
430 }
431out:
432 sx_xunlock(&sc->softc_lock);
433}
434
435static int
436sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
437{
438 struct sfxge_softc *sc = arg1;
439 unsigned int id = arg2;
440
441 sfxge_ev_stat_update(sc);
442
443 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
444}
445
446static void
447sfxge_ev_stat_init(struct sfxge_softc *sc)
448{
449 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
450 struct sysctl_oid_list *stat_list;
451 unsigned int id;
452 char name[40];
453
454 stat_list = SYSCTL_CHILDREN(sc->stats_node);
455
456 for (id = 0; id < EV_NQSTATS; id++) {
457 snprintf(name, sizeof(name), "ev_%s",
458 efx_ev_qstat_name(sc->enp, id));
459 SYSCTL_ADD_PROC(
460 ctx, stat_list,
461 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
462 sc, id, sfxge_ev_stat_handler, "Q",
463 "");
464 }
465}
466
467#endif /* EFSYS_OPT_QSTATS */
468
469static void
470sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
471{
472 struct sfxge_evq *evq;
473 efx_evq_t *eep;
474
475 evq = sc->evq[idx];
476 eep = evq->common;
477
478 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
479 ("evq->init_state != SFXGE_EVQ_STARTED"));
480
481 (void)efx_ev_qmoderate(eep, us);
482}
483
484static int
485sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
486{
487 struct sfxge_softc *sc = arg1;
488 struct sfxge_intr *intr = &sc->intr;
489 unsigned int moderation;
490 int error;
491 int index;
492
493 sx_xlock(&sc->softc_lock);
494
495 if (req->newptr != NULL) {
496 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
497 != 0)
498 goto out;
499
500 /* We may not be calling efx_ev_qmoderate() now,
501 * so we have to range-check the value ourselves.
502 */
503 if (moderation >
504 efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
505 error = EINVAL;
506 goto out;
507 }
508
509 sc->ev_moderation = moderation;
510 if (intr->state == SFXGE_INTR_STARTED) {
511 for (index = 0; index < intr->n_alloc; index++)
512 sfxge_ev_qmoderate(sc, index, moderation);
513 }
514 } else {
515 error = SYSCTL_OUT(req, &sc->ev_moderation,
516 sizeof(sc->ev_moderation));
517 }
518
519out:
520 sx_xunlock(&sc->softc_lock);
521
522 return (error);
523}
524
525static boolean_t
526sfxge_ev_initialized(void *arg)
527{
528 struct sfxge_evq *evq;
529
530 evq = (struct sfxge_evq *)arg;
531
532 KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
533 ("evq not starting"));
534
535 evq->init_state = SFXGE_EVQ_STARTED;
536
537 return (0);
538}
539
540static boolean_t
541sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
542{
543 struct sfxge_evq *evq;
544 struct sfxge_softc *sc;
545
546 evq = (struct sfxge_evq *)arg;
547 sc = evq->sc;
548
549 sfxge_mac_link_update(sc, link_mode);
550
551 return (0);
552}
553
554static const efx_ev_callbacks_t sfxge_ev_callbacks = {
555 .eec_initialized = sfxge_ev_initialized,
556 .eec_rx = sfxge_ev_rx,
557 .eec_tx = sfxge_ev_tx,
558 .eec_exception = sfxge_ev_exception,
559 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done,
560 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed,
561 .eec_txq_flush_done = sfxge_ev_txq_flush_done,
562 .eec_software = sfxge_ev_software,
563 .eec_sram = sfxge_ev_sram,
564 .eec_wake_up = sfxge_ev_wake_up,
565 .eec_timer = sfxge_ev_timer,
566 .eec_link_change = sfxge_ev_link_change,
567};
568
569
570int
571sfxge_ev_qpoll(struct sfxge_evq *evq)
572{
573 int rc;
574
575 mtx_lock(&evq->lock);
576
577 if (evq->init_state != SFXGE_EVQ_STARTING &&
578 evq->init_state != SFXGE_EVQ_STARTED) {
579 rc = EINVAL;
580 goto fail;
581 }
582
583 /* Synchronize the DMA memory for reading */
584 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
585 BUS_DMASYNC_POSTREAD);
586
587 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
588 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
589 KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
590 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
591
592 /* Poll the queue */
593 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
594
595 evq->rx_done = 0;
596 evq->tx_done = 0;
597
598 /* Perform any pending completion processing */
599 sfxge_ev_qcomplete(evq, B_TRUE);
600
601 /* Re-prime the event queue for interrupts */
602 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
603 goto fail;
604
605 mtx_unlock(&evq->lock);
606
607 return (0);
608
609fail:
610 mtx_unlock(&(evq->lock));
611 return (rc);
612}
613
614static void
615sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
616{
617 struct sfxge_evq *evq;
618
619 evq = sc->evq[index];
620
621 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
622 ("evq->init_state != SFXGE_EVQ_STARTED"));
623
624 mtx_lock(&evq->lock);
625 evq->init_state = SFXGE_EVQ_INITIALIZED;
626 evq->read_ptr = 0;
627 evq->exception = B_FALSE;
628
629#if EFSYS_OPT_QSTATS
630 /* Add event counts before discarding the common evq state */
631 efx_ev_qstats_update(evq->common, sc->ev_stats);
632#endif
633
634 efx_ev_qdestroy(evq->common);
635 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
636 EFX_EVQ_NBUFS(evq->entries));
637 mtx_unlock(&evq->lock);
638}
639
640static int
641sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
642{
643 struct sfxge_evq *evq;
644 efsys_mem_t *esmp;
645 int count;
646 int rc;
647
648 evq = sc->evq[index];
649 esmp = &evq->mem;
650
651 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
652 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
653
654 /* Clear all events. */
655 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
656
657 /* Program the buffer table. */
658 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
659 EFX_EVQ_NBUFS(evq->entries))) != 0)
660 return (rc);
661
662 /* Create the common code event queue. */
663 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
664 evq->buf_base_id, &evq->common)) != 0)
665 goto fail;
666
667 mtx_lock(&evq->lock);
668
669 /* Set the default moderation */
670 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
671
672 /* Prime the event queue for interrupts */
673 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
674 goto fail2;
675
676 evq->init_state = SFXGE_EVQ_STARTING;
677
678 mtx_unlock(&evq->lock);
679
680 /* Wait for the initialization event */
681 count = 0;
682 do {
683 /* Pause for 100 ms */
684 pause("sfxge evq init", hz / 10);
685
686 /* Check to see if the test event has been processed */
687 if (evq->init_state == SFXGE_EVQ_STARTED)
688 goto done;
689
690 } while (++count < 20);
691
692 rc = ETIMEDOUT;
693 goto fail3;
694
695done:
696 return (0);
697
698fail3:
699 mtx_lock(&evq->lock);
700 evq->init_state = SFXGE_EVQ_INITIALIZED;
701fail2:
702 mtx_unlock(&evq->lock);
703 efx_ev_qdestroy(evq->common);
704fail:
705 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
706 EFX_EVQ_NBUFS(evq->entries));
707
708 return (rc);
709}
710
711void
712sfxge_ev_stop(struct sfxge_softc *sc)
713{
714 struct sfxge_intr *intr;
715 efx_nic_t *enp;
716 int index;
717
718 intr = &sc->intr;
719 enp = sc->enp;
720
721 KASSERT(intr->state == SFXGE_INTR_STARTED,
722 ("Interrupts not started"));
723
724 /* Stop the event queue(s) */
725 index = intr->n_alloc;
726 while (--index >= 0)
727 sfxge_ev_qstop(sc, index);
728
729 /* Tear down the event module */
730 efx_ev_fini(enp);
731}
732
733int
734sfxge_ev_start(struct sfxge_softc *sc)
735{
736 struct sfxge_intr *intr;
737 int index;
738 int rc;
739
740 intr = &sc->intr;
741
742 KASSERT(intr->state == SFXGE_INTR_STARTED,
743 ("intr->state != SFXGE_INTR_STARTED"));
744
745 /* Initialize the event module */
746 if ((rc = efx_ev_init(sc->enp)) != 0)
747 return (rc);
748
749 /* Start the event queues */
750 for (index = 0; index < intr->n_alloc; index++) {
751 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
752 goto fail;
753 }
754
755 return (0);
756
757fail:
758 /* Stop the event queue(s) */
759 while (--index >= 0)
760 sfxge_ev_qstop(sc, index);
761
762 /* Tear down the event module */
763 efx_ev_fini(sc->enp);
764
765 return (rc);
766}
767
768static void
769sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
770{
771 struct sfxge_evq *evq;
772
773 evq = sc->evq[index];
774
775 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
776 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
777 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
778
779 sfxge_dma_free(&evq->mem);
780
781 sc->evq[index] = NULL;
782
783 mtx_destroy(&evq->lock);
784
785 free(evq, M_SFXGE);
786}
787
788static int
789sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
790{
791 struct sfxge_evq *evq;
792 efsys_mem_t *esmp;
793 int rc;
794
795 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
796
797 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
798 evq->sc = sc;
799 evq->index = index;
800 sc->evq[index] = evq;
801 esmp = &evq->mem;
802
803 /* Build an event queue with room for one event per tx and rx buffer,
804 * plus some extra for link state events and MCDI completions.
805 * There are three tx queues in the first event queue and one in
806 * other.
807 */
808 if (index == 0)
809 evq->entries =
810 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
811 3 * sc->txq_entries +
812 128);
813 else
814 evq->entries =
815 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
816 sc->txq_entries +
817 128);
818
819 /* Initialise TX completion list */
820 evq->txqs = &evq->txq;
821
822 /* Allocate DMA space. */
823 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
824 return (rc);
825
826 /* Allocate buffer table entries. */
827 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
828 &evq->buf_base_id);
829
830 mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
831
832 evq->init_state = SFXGE_EVQ_INITIALIZED;
833
834 return (0);
835}
836
837void
838sfxge_ev_fini(struct sfxge_softc *sc)
839{
840 struct sfxge_intr *intr;
841 int index;
842
843 intr = &sc->intr;
844
845 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
846 ("intr->state != SFXGE_INTR_INITIALIZED"));
847
848 sc->ev_moderation = 0;
849
850 /* Tear down the event queue(s). */
851 index = intr->n_alloc;
852 while (--index >= 0)
853 sfxge_ev_qfini(sc, index);
854}
855
856int
857sfxge_ev_init(struct sfxge_softc *sc)
858{
859 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
860 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
861 struct sfxge_intr *intr;
862 int index;
863 int rc;
864
865 intr = &sc->intr;
866
867 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
868 ("intr->state != SFXGE_INTR_INITIALIZED"));
869
870 /* Set default interrupt moderation; add a sysctl to
871 * read and change it.
872 */
873 sc->ev_moderation = 30;
874 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
875 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
876 sc, 0, sfxge_int_mod_handler, "IU",
877 "sfxge interrupt moderation (us)");
878
879 /*
880 * Initialize the event queue(s) - one per interrupt.
881 */
882 for (index = 0; index < intr->n_alloc; index++) {
883 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
884 goto fail;
885 }
886
887#if EFSYS_OPT_QSTATS
888 sfxge_ev_stat_init(sc);
889#endif
890
891 return (0);
892
893fail:
894 while (--index >= 0)
895 sfxge_ev_qfini(sc, index);
896
897 return (rc);
898}
|