sfxge_ev.c revision 277893 
1/*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_ev.c 277893 2015-01-29 19:06:14Z arybchik $");
32
33#include <sys/param.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/param.h>
37#include <sys/queue.h>
38#include <sys/systm.h>
39#include <sys/taskqueue.h>
40
41#include "common/efx.h"
42
43#include "sfxge.h"
44
45static void
46sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
47{
48	struct sfxge_softc *sc;
49	unsigned int index;
50	struct sfxge_rxq *rxq;
51	struct sfxge_txq *txq;
52
53	sc = evq->sc;
54	index = evq->index;
55	rxq = sc->rxq[index];
56
57	if ((txq = evq->txq) != NULL) {
58		evq->txq = NULL;
59		evq->txqs = &(evq->txq);
60
61		do {
62			struct sfxge_txq *next;
63
64			next = txq->next;
65			txq->next = NULL;
66
67			KASSERT(txq->evq_index == index,
68			    ("txq->evq_index != index"));
69
70			if (txq->pending != txq->completed)
71				sfxge_tx_qcomplete(txq, evq);
72
73			txq = next;
74		} while (txq != NULL);
75	}
76
77	if (rxq->pending != rxq->completed)
78		sfxge_rx_qcomplete(rxq, eop);
79}
80
81static boolean_t
82sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
83    uint16_t flags)
84{
85	struct sfxge_evq *evq;
86	struct sfxge_softc *sc;
87	struct sfxge_rxq *rxq;
88	unsigned int expected;
89	struct sfxge_rx_sw_desc *rx_desc;
90
91	evq = arg;
92	sc = evq->sc;
93
94	if (evq->exception)
95		goto done;
96
97	rxq = sc->rxq[label];
98	KASSERT(rxq != NULL, ("rxq == NULL"));
99	KASSERT(evq->index == rxq->index,
100	    ("evq->index != rxq->index"));
101
102	if (rxq->init_state != SFXGE_RXQ_STARTED)
103		goto done;
104
105	expected = rxq->pending++ & rxq->ptr_mask;
106	if (id != expected) {
107		evq->exception = B_TRUE;
108
109		device_printf(sc->dev, "RX completion out of order"
110			      " (id=%#x expected=%#x flags=%#x); resetting\n",
111			      id, expected, flags);
112		sfxge_schedule_reset(sc);
113
114		goto done;
115	}
116
117	rx_desc = &rxq->queue[id];
118
119	KASSERT(rx_desc->flags == EFX_DISCARD,
120	    ("rx_desc->flags != EFX_DISCARD"));
121	rx_desc->flags = flags;
122
123	KASSERT(size < (1 << 16), ("size > (1 << 16)"));
124	rx_desc->size = (uint16_t)size;
125	prefetch_read_many(rx_desc->mbuf);
126
127	evq->rx_done++;
128
129	if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
130		sfxge_ev_qcomplete(evq, B_FALSE);
131
132done:
133	return (evq->rx_done >= SFXGE_EV_BATCH);
134}
135
136static boolean_t
137sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
138{
139	struct sfxge_evq *evq;
140	struct sfxge_softc *sc;
141
142	evq = (struct sfxge_evq *)arg;
143	sc = evq->sc;
144
145	evq->exception = B_TRUE;
146
147	if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
148		device_printf(sc->dev,
149			      "hardware exception (code=%u); resetting\n",
150			      code);
151		sfxge_schedule_reset(sc);
152	}
153
154	return (B_FALSE);
155}
156
157static boolean_t
158sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
159{
160	struct sfxge_evq *evq;
161	struct sfxge_softc *sc;
162	struct sfxge_rxq *rxq;
163	unsigned int index;
164	unsigned int label;
165	uint16_t magic;
166
167	evq = (struct sfxge_evq *)arg;
168	sc = evq->sc;
169	rxq = sc->rxq[rxq_index];
170
171	KASSERT(rxq != NULL, ("rxq == NULL"));
172
173	/* Resend a software event on the correct queue */
174	index = rxq->index;
175	evq = sc->evq[index];
176
177	label = rxq_index;
178	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
179	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
180	magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
181
182	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
183	    ("evq not started"));
184	efx_ev_qpost(evq->common, magic);
185
186	return (B_FALSE);
187}
188
189static boolean_t
190sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
191{
192	struct sfxge_evq *evq;
193	struct sfxge_softc *sc;
194	struct sfxge_rxq *rxq;
195	unsigned int index;
196	unsigned int label;
197	uint16_t magic;
198
199	evq = (struct sfxge_evq *)arg;
200	sc = evq->sc;
201	rxq = sc->rxq[rxq_index];
202
203	KASSERT(rxq != NULL, ("rxq == NULL"));
204
205	/* Resend a software event on the correct queue */
206	index = rxq->index;
207	evq = sc->evq[index];
208
209	label = rxq_index;
210	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
211	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
212	magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
213
214	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
215	    ("evq not started"));
216	efx_ev_qpost(evq->common, magic);
217
218	return (B_FALSE);
219}
220
221static struct sfxge_txq *
222sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
223{
224	unsigned int index;
225
226	KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
227	    (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
228	index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
229	return (evq->sc->txq[index]);
230}
231
232static boolean_t
233sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
234{
235	struct sfxge_evq *evq;
236	struct sfxge_txq *txq;
237	unsigned int stop;
238	unsigned int delta;
239
240	evq = (struct sfxge_evq *)arg;
241	txq = sfxge_get_txq_by_label(evq, label);
242
243	KASSERT(txq != NULL, ("txq == NULL"));
244	KASSERT(evq->index == txq->evq_index,
245	    ("evq->index != txq->evq_index"));
246
247	if (txq->init_state != SFXGE_TXQ_STARTED)
248		goto done;
249
250	stop = (id + 1) & txq->ptr_mask;
251	id = txq->pending & txq->ptr_mask;
252
253	delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
254	txq->pending += delta;
255
256	evq->tx_done++;
257
258	if (txq->next == NULL &&
259	    evq->txqs != &(txq->next)) {
260		*(evq->txqs) = txq;
261		evq->txqs = &(txq->next);
262	}
263
264	if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
265		sfxge_tx_qcomplete(txq, evq);
266
267done:
268	return (evq->tx_done >= SFXGE_EV_BATCH);
269}
270
271static boolean_t
272sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
273{
274	struct sfxge_evq *evq;
275	struct sfxge_softc *sc;
276	struct sfxge_txq *txq;
277	unsigned int label;
278	uint16_t magic;
279
280	evq = (struct sfxge_evq *)arg;
281	sc = evq->sc;
282	txq = sc->txq[txq_index];
283
284	KASSERT(txq != NULL, ("txq == NULL"));
285	KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
286	    ("txq not initialized"));
287
288	/* Resend a software event on the correct queue */
289	evq = sc->evq[txq->evq_index];
290
291	label = txq->type;
292	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
293	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
294	magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
295
296	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
297	    ("evq not started"));
298	efx_ev_qpost(evq->common, magic);
299
300	return (B_FALSE);
301}
302
303static boolean_t
304sfxge_ev_software(void *arg, uint16_t magic)
305{
306	struct sfxge_evq *evq;
307	struct sfxge_softc *sc;
308	unsigned int label;
309
310	evq = (struct sfxge_evq *)arg;
311	sc = evq->sc;
312
313	label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
314	magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
315
316	switch (magic) {
317	case SFXGE_MAGIC_RX_QFLUSH_DONE: {
318		struct sfxge_rxq *rxq = sc->rxq[label];
319
320		KASSERT(rxq != NULL, ("rxq == NULL"));
321		KASSERT(evq->index == rxq->index,
322		    ("evq->index != rxq->index"));
323
324		sfxge_rx_qflush_done(rxq);
325		break;
326	}
327	case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
328		struct sfxge_rxq *rxq = sc->rxq[label];
329
330		KASSERT(rxq != NULL, ("rxq == NULL"));
331		KASSERT(evq->index == rxq->index,
332		    ("evq->index != rxq->index"));
333
334		sfxge_rx_qflush_failed(rxq);
335		break;
336	}
337	case SFXGE_MAGIC_RX_QREFILL: {
338		struct sfxge_rxq *rxq = sc->rxq[label];
339
340		KASSERT(rxq != NULL, ("rxq == NULL"));
341		KASSERT(evq->index == rxq->index,
342		    ("evq->index != rxq->index"));
343
344		sfxge_rx_qrefill(rxq);
345		break;
346	}
347	case SFXGE_MAGIC_TX_QFLUSH_DONE: {
348		struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
349
350		KASSERT(txq != NULL, ("txq == NULL"));
351		KASSERT(evq->index == txq->evq_index,
352		    ("evq->index != txq->evq_index"));
353
354		sfxge_tx_qflush_done(txq);
355		break;
356	}
357	default:
358		break;
359	}
360
361	return (B_FALSE);
362}
363
364static boolean_t
365sfxge_ev_sram(void *arg, uint32_t code)
366{
367	(void)arg;
368	(void)code;
369
370	switch (code) {
371	case EFX_SRAM_UPDATE:
372		EFSYS_PROBE(sram_update);
373		break;
374
375	case EFX_SRAM_CLEAR:
376		EFSYS_PROBE(sram_clear);
377		break;
378
379	case EFX_SRAM_ILLEGAL_CLEAR:
380		EFSYS_PROBE(sram_illegal_clear);
381		break;
382
383	default:
384		KASSERT(B_FALSE, ("Impossible SRAM event"));
385		break;
386	}
387
388	return (B_FALSE);
389}
390
391static boolean_t
392sfxge_ev_timer(void *arg, uint32_t index)
393{
394	(void)arg;
395	(void)index;
396
397	return (B_FALSE);
398}
399
400static boolean_t
401sfxge_ev_wake_up(void *arg, uint32_t index)
402{
403	(void)arg;
404	(void)index;
405
406	return (B_FALSE);
407}
408
409#if EFSYS_OPT_QSTATS
410
411static void
412sfxge_ev_stat_update(struct sfxge_softc *sc)
413{
414	struct sfxge_evq *evq;
415	unsigned int index;
416	clock_t now;
417
418	sx_xlock(&sc->softc_lock);
419
420	if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
421		goto out;
422
423	now = ticks;
424	if (now - sc->ev_stats_update_time < hz)
425		goto out;
426
427	sc->ev_stats_update_time = now;
428
429	/* Add event counts from each event queue in turn */
430	for (index = 0; index < sc->intr.n_alloc; index++) {
431		evq = sc->evq[index];
432		mtx_lock(&evq->lock);
433		efx_ev_qstats_update(evq->common, sc->ev_stats);
434		mtx_unlock(&evq->lock);
435	}
436out:
437	sx_xunlock(&sc->softc_lock);
438}
439
440static int
441sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
442{
443	struct sfxge_softc *sc = arg1;
444	unsigned int id = arg2;
445
446	sfxge_ev_stat_update(sc);
447
448	return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
449}
450
451static void
452sfxge_ev_stat_init(struct sfxge_softc *sc)
453{
454	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
455	struct sysctl_oid_list *stat_list;
456	unsigned int id;
457	char name[40];
458
459	stat_list = SYSCTL_CHILDREN(sc->stats_node);
460
461	for (id = 0; id < EV_NQSTATS; id++) {
462		snprintf(name, sizeof(name), "ev_%s",
463			 efx_ev_qstat_name(sc->enp, id));
464		SYSCTL_ADD_PROC(
465			ctx, stat_list,
466			OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
467			sc, id, sfxge_ev_stat_handler, "Q",
468			"");
469	}
470}
471
472#endif /* EFSYS_OPT_QSTATS */
473
474static void
475sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
476{
477	struct sfxge_evq *evq;
478	efx_evq_t *eep;
479
480	evq = sc->evq[idx];
481	eep = evq->common;
482
483	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
484	    ("evq->init_state != SFXGE_EVQ_STARTED"));
485
486	(void)efx_ev_qmoderate(eep, us);
487}
488
489static int
490sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
491{
492	struct sfxge_softc *sc = arg1;
493	struct sfxge_intr *intr = &sc->intr;
494	unsigned int moderation;
495	int error;
496	int index;
497
498	sx_xlock(&sc->softc_lock);
499
500	if (req->newptr != NULL) {
501		if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
502		    != 0)
503			goto out;
504
505		/* We may not be calling efx_ev_qmoderate() now,
506		 * so we have to range-check the value ourselves.
507		 */
508		if (moderation >
509		    efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
510			error = EINVAL;
511			goto out;
512		}
513
514		sc->ev_moderation = moderation;
515		if (intr->state == SFXGE_INTR_STARTED) {
516			for (index = 0; index < intr->n_alloc; index++)
517				sfxge_ev_qmoderate(sc, index, moderation);
518		}
519	} else {
520		error = SYSCTL_OUT(req, &sc->ev_moderation,
521				   sizeof(sc->ev_moderation));
522	}
523
524out:
525	sx_xunlock(&sc->softc_lock);
526
527	return (error);
528}
529
530static boolean_t
531sfxge_ev_initialized(void *arg)
532{
533	struct sfxge_evq *evq;
534
535	evq = (struct sfxge_evq *)arg;
536
537	KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
538	    ("evq not starting"));
539
540	evq->init_state = SFXGE_EVQ_STARTED;
541
542	return (0);
543}
544
545static boolean_t
546sfxge_ev_link_change(void *arg, efx_link_mode_t	link_mode)
547{
548	struct sfxge_evq *evq;
549	struct sfxge_softc *sc;
550
551	evq = (struct sfxge_evq *)arg;
552	sc = evq->sc;
553
554	sfxge_mac_link_update(sc, link_mode);
555
556	return (0);
557}
558
559static const efx_ev_callbacks_t sfxge_ev_callbacks = {
560	.eec_initialized	= sfxge_ev_initialized,
561	.eec_rx			= sfxge_ev_rx,
562	.eec_tx			= sfxge_ev_tx,
563	.eec_exception		= sfxge_ev_exception,
564	.eec_rxq_flush_done	= sfxge_ev_rxq_flush_done,
565	.eec_rxq_flush_failed	= sfxge_ev_rxq_flush_failed,
566	.eec_txq_flush_done	= sfxge_ev_txq_flush_done,
567	.eec_software		= sfxge_ev_software,
568	.eec_sram		= sfxge_ev_sram,
569	.eec_wake_up		= sfxge_ev_wake_up,
570	.eec_timer		= sfxge_ev_timer,
571	.eec_link_change	= sfxge_ev_link_change,
572};
573
574
575int
576sfxge_ev_qpoll(struct sfxge_evq *evq)
577{
578	int rc;
579
580	mtx_lock(&evq->lock);
581
582	if (evq->init_state != SFXGE_EVQ_STARTING &&
583	    evq->init_state != SFXGE_EVQ_STARTED) {
584		rc = EINVAL;
585		goto fail;
586	}
587
588	/* Synchronize the DMA memory for reading */
589	bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
590	    BUS_DMASYNC_POSTREAD);
591
592	KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
593	KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
594	KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
595	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
596
597	/* Poll the queue */
598	efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
599
600	evq->rx_done = 0;
601	evq->tx_done = 0;
602
603	/* Perform any pending completion processing */
604	sfxge_ev_qcomplete(evq, B_TRUE);
605
606	/* Re-prime the event queue for interrupts */
607	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
608		goto fail;
609
610	mtx_unlock(&evq->lock);
611
612	return (0);
613
614fail:
615	mtx_unlock(&(evq->lock));
616	return (rc);
617}
618
619static void
620sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
621{
622	struct sfxge_evq *evq;
623
624	evq = sc->evq[index];
625
626	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
627	    ("evq->init_state != SFXGE_EVQ_STARTED"));
628
629	mtx_lock(&evq->lock);
630	evq->init_state = SFXGE_EVQ_INITIALIZED;
631	evq->read_ptr = 0;
632	evq->exception = B_FALSE;
633
634#if EFSYS_OPT_QSTATS
635	/* Add event counts before discarding the common evq state */
636	efx_ev_qstats_update(evq->common, sc->ev_stats);
637#endif
638
639	efx_ev_qdestroy(evq->common);
640	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
641	    EFX_EVQ_NBUFS(evq->entries));
642	mtx_unlock(&evq->lock);
643}
644
645static int
646sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
647{
648	struct sfxge_evq *evq;
649	efsys_mem_t *esmp;
650	int count;
651	int rc;
652
653	evq = sc->evq[index];
654	esmp = &evq->mem;
655
656	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
657	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
658
659	/* Clear all events. */
660	(void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
661
662	/* Program the buffer table. */
663	if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
664	    EFX_EVQ_NBUFS(evq->entries))) != 0)
665		return (rc);
666
667	/* Create the common code event queue. */
668	if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
669	    evq->buf_base_id, &evq->common)) != 0)
670		goto fail;
671
672	mtx_lock(&evq->lock);
673
674	/* Set the default moderation */
675	(void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
676
677	/* Prime the event queue for interrupts */
678	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
679		goto fail2;
680
681	evq->init_state = SFXGE_EVQ_STARTING;
682
683	mtx_unlock(&evq->lock);
684
685	/* Wait for the initialization event */
686	count = 0;
687	do {
688		/* Pause for 100 ms */
689		pause("sfxge evq init", hz / 10);
690
691		/* Check to see if the test event has been processed */
692		if (evq->init_state == SFXGE_EVQ_STARTED)
693			goto done;
694
695	} while (++count < 20);
696
697	rc = ETIMEDOUT;
698	goto fail3;
699
700done:
701	return (0);
702
703fail3:
704	mtx_lock(&evq->lock);
705	evq->init_state = SFXGE_EVQ_INITIALIZED;
706fail2:
707	mtx_unlock(&evq->lock);
708	efx_ev_qdestroy(evq->common);
709fail:
710	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
711	    EFX_EVQ_NBUFS(evq->entries));
712
713	return (rc);
714}
715
716void
717sfxge_ev_stop(struct sfxge_softc *sc)
718{
719	struct sfxge_intr *intr;
720	efx_nic_t *enp;
721	int index;
722
723	intr = &sc->intr;
724	enp = sc->enp;
725
726	KASSERT(intr->state == SFXGE_INTR_STARTED,
727	    ("Interrupts not started"));
728
729	/* Stop the event queue(s) */
730	index = intr->n_alloc;
731	while (--index >= 0)
732		sfxge_ev_qstop(sc, index);
733
734	/* Tear down the event module */
735	efx_ev_fini(enp);
736}
737
738int
739sfxge_ev_start(struct sfxge_softc *sc)
740{
741	struct sfxge_intr *intr;
742	int index;
743	int rc;
744
745	intr = &sc->intr;
746
747	KASSERT(intr->state == SFXGE_INTR_STARTED,
748	    ("intr->state != SFXGE_INTR_STARTED"));
749
750	/* Initialize the event module */
751	if ((rc = efx_ev_init(sc->enp)) != 0)
752		return (rc);
753
754	/* Start the event queues */
755	for (index = 0; index < intr->n_alloc; index++) {
756		if ((rc = sfxge_ev_qstart(sc, index)) != 0)
757			goto fail;
758	}
759
760	return (0);
761
762fail:
763	/* Stop the event queue(s) */
764	while (--index >= 0)
765		sfxge_ev_qstop(sc, index);
766
767	/* Tear down the event module */
768	efx_ev_fini(sc->enp);
769
770	return (rc);
771}
772
773static void
774sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
775{
776	struct sfxge_evq *evq;
777
778	evq = sc->evq[index];
779
780	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
781	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
782	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
783
784	sfxge_dma_free(&evq->mem);
785
786	sc->evq[index] = NULL;
787
788	mtx_destroy(&evq->lock);
789
790	free(evq, M_SFXGE);
791}
792
793static int
794sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
795{
796	struct sfxge_evq *evq;
797	efsys_mem_t *esmp;
798	int rc;
799
800	KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
801
802	evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
803	evq->sc = sc;
804	evq->index = index;
805	sc->evq[index] = evq;
806	esmp = &evq->mem;
807
808	/* Build an event queue with room for one event per tx and rx buffer,
809	 * plus some extra for link state events and MCDI completions.
810	 * There are three tx queues in the first event queue and one in
811	 * other.
812	 */
813	if (index == 0)
814		evq->entries =
815			ROUNDUP_POW_OF_TWO(sc->rxq_entries +
816					   3 * sc->txq_entries +
817					   128);
818	else
819		evq->entries =
820			ROUNDUP_POW_OF_TWO(sc->rxq_entries +
821					   sc->txq_entries +
822					   128);
823
824	/* Initialise TX completion list */
825	evq->txqs = &evq->txq;
826
827	/* Allocate DMA space. */
828	if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
829		return (rc);
830
831	/* Allocate buffer table entries. */
832	sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
833				 &evq->buf_base_id);
834
835	mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
836
837	evq->init_state = SFXGE_EVQ_INITIALIZED;
838
839	return (0);
840}
841
842void
843sfxge_ev_fini(struct sfxge_softc *sc)
844{
845	struct sfxge_intr *intr;
846	int index;
847
848	intr = &sc->intr;
849
850	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
851	    ("intr->state != SFXGE_INTR_INITIALIZED"));
852
853	sc->ev_moderation = 0;
854
855	/* Tear down the event queue(s). */
856	index = intr->n_alloc;
857	while (--index >= 0)
858		sfxge_ev_qfini(sc, index);
859}
860
861int
862sfxge_ev_init(struct sfxge_softc *sc)
863{
864	struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
865	struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
866	struct sfxge_intr *intr;
867	int index;
868	int rc;
869
870	intr = &sc->intr;
871
872	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
873	    ("intr->state != SFXGE_INTR_INITIALIZED"));
874
875	/* Set default interrupt moderation; add a sysctl to
876	 * read and change it.
877	 */
878	sc->ev_moderation = SFXGE_MODERATION;
879	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
880			OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
881			sc, 0, sfxge_int_mod_handler, "IU",
882			"sfxge interrupt moderation (us)");
883
884	/*
885	 * Initialize the event queue(s) - one per interrupt.
886	 */
887	for (index = 0; index < intr->n_alloc; index++) {
888		if ((rc = sfxge_ev_qinit(sc, index)) != 0)
889			goto fail;
890	}
891
892#if EFSYS_OPT_QSTATS
893	sfxge_ev_stat_init(sc);
894#endif
895
896	return (0);
897
898fail:
899	while (--index >= 0)
900		sfxge_ev_qfini(sc, index);
901
902	return (rc);
903}
904