sfxge_ev.c revision 257179 
1/*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_ev.c 257179 2013-10-26 18:18:50Z glebius $");
32
33#include <sys/param.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/param.h>
37#include <sys/queue.h>
38#include <sys/systm.h>
39#include <sys/taskqueue.h>
40
41#include "common/efx.h"
42
43#include "sfxge.h"
44
45static void
46sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
47{
48	struct sfxge_softc *sc;
49	unsigned int index;
50	struct sfxge_rxq *rxq;
51	struct sfxge_txq *txq;
52
53	sc = evq->sc;
54	index = evq->index;
55	rxq = sc->rxq[index];
56
57	if ((txq = evq->txq) != NULL) {
58		evq->txq = NULL;
59		evq->txqs = &(evq->txq);
60
61		do {
62			struct sfxge_txq *next;
63
64			next = txq->next;
65			txq->next = NULL;
66
67			KASSERT(txq->evq_index == index,
68			    ("txq->evq_index != index"));
69
70			if (txq->pending != txq->completed)
71				sfxge_tx_qcomplete(txq);
72
73			txq = next;
74		} while (txq != NULL);
75	}
76
77	if (rxq->pending != rxq->completed)
78		sfxge_rx_qcomplete(rxq, eop);
79}
80
81static boolean_t
82sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
83    uint16_t flags)
84{
85	struct sfxge_evq *evq;
86	struct sfxge_softc *sc;
87	struct sfxge_rxq *rxq;
88	unsigned int expected;
89	struct sfxge_rx_sw_desc *rx_desc;
90
91	evq = arg;
92	sc = evq->sc;
93
94	if (evq->exception)
95		goto done;
96
97	rxq = sc->rxq[label];
98	KASSERT(rxq != NULL, ("rxq == NULL"));
99	KASSERT(evq->index == rxq->index,
100	    ("evq->index != rxq->index"));
101
102	if (rxq->init_state != SFXGE_RXQ_STARTED)
103		goto done;
104
105	expected = rxq->pending++ & (SFXGE_NDESCS - 1);
106	if (id != expected) {
107		evq->exception = B_TRUE;
108
109		device_printf(sc->dev, "RX completion out of order"
110			      " (id=%#x expected=%#x flags=%#x); resetting\n",
111			      id, expected, flags);
112		sfxge_schedule_reset(sc);
113
114		goto done;
115	}
116
117	rx_desc = &rxq->queue[id];
118
119	KASSERT(rx_desc->flags == EFX_DISCARD,
120	    ("rx_desc->flags != EFX_DISCARD"));
121	rx_desc->flags = flags;
122
123	KASSERT(size < (1 << 16), ("size > (1 << 16)"));
124	rx_desc->size = (uint16_t)size;
125	prefetch_read_many(rx_desc->mbuf);
126
127	evq->rx_done++;
128
129	if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
130		sfxge_ev_qcomplete(evq, B_FALSE);
131
132done:
133	return (evq->rx_done >= SFXGE_EV_BATCH);
134}
135
136static boolean_t
137sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
138{
139	struct sfxge_evq *evq;
140	struct sfxge_softc *sc;
141
142	evq = (struct sfxge_evq *)arg;
143	sc = evq->sc;
144
145	evq->exception = B_TRUE;
146
147	if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
148		device_printf(sc->dev,
149			      "hardware exception (code=%u); resetting\n",
150			      code);
151		sfxge_schedule_reset(sc);
152	}
153
154	return (B_FALSE);
155}
156
157static boolean_t
158sfxge_ev_rxq_flush_done(void *arg, uint32_t label)
159{
160	struct sfxge_evq *evq;
161	struct sfxge_softc *sc;
162	struct sfxge_rxq *rxq;
163	unsigned int index;
164	uint16_t magic;
165
166	evq = (struct sfxge_evq *)arg;
167	sc = evq->sc;
168	rxq = sc->rxq[label];
169
170	KASSERT(rxq != NULL, ("rxq == NULL"));
171
172	/* Resend a software event on the correct queue */
173	index = rxq->index;
174	evq = sc->evq[index];
175
176	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
177	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
178	magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
179
180	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
181	    ("evq not started"));
182	efx_ev_qpost(evq->common, magic);
183
184	return (B_FALSE);
185}
186
187static boolean_t
188sfxge_ev_rxq_flush_failed(void *arg, uint32_t label)
189{
190	struct sfxge_evq *evq;
191	struct sfxge_softc *sc;
192	struct sfxge_rxq *rxq;
193	unsigned int index;
194	uint16_t magic;
195
196	evq = (struct sfxge_evq *)arg;
197	sc = evq->sc;
198	rxq = sc->rxq[label];
199
200	KASSERT(rxq != NULL, ("rxq == NULL"));
201
202	/* Resend a software event on the correct queue */
203	index = rxq->index;
204	evq = sc->evq[index];
205
206	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
207	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
208	magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
209
210	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
211	    ("evq not started"));
212	efx_ev_qpost(evq->common, magic);
213
214	return (B_FALSE);
215}
216
217static boolean_t
218sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
219{
220	struct sfxge_evq *evq;
221	struct sfxge_softc *sc;
222	struct sfxge_txq *txq;
223	unsigned int stop;
224	unsigned int delta;
225
226	evq = (struct sfxge_evq *)arg;
227	sc = evq->sc;
228	txq = sc->txq[label];
229
230	KASSERT(txq != NULL, ("txq == NULL"));
231	KASSERT(evq->index == txq->evq_index,
232	    ("evq->index != txq->evq_index"));
233
234	if (txq->init_state != SFXGE_TXQ_STARTED)
235		goto done;
236
237	stop = (id + 1) & (SFXGE_NDESCS - 1);
238	id = txq->pending & (SFXGE_NDESCS - 1);
239
240	delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop);
241	txq->pending += delta;
242
243	evq->tx_done++;
244
245	if (txq->next == NULL &&
246	    evq->txqs != &(txq->next)) {
247		*(evq->txqs) = txq;
248		evq->txqs = &(txq->next);
249	}
250
251	if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
252		sfxge_tx_qcomplete(txq);
253
254done:
255	return (evq->tx_done >= SFXGE_EV_BATCH);
256}
257
258static boolean_t
259sfxge_ev_txq_flush_done(void *arg, uint32_t label)
260{
261	struct sfxge_evq *evq;
262	struct sfxge_softc *sc;
263	struct sfxge_txq *txq;
264	uint16_t magic;
265
266	evq = (struct sfxge_evq *)arg;
267	sc = evq->sc;
268	txq = sc->txq[label];
269
270	KASSERT(txq != NULL, ("txq == NULL"));
271	KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
272	    ("txq not initialized"));
273
274	/* Resend a software event on the correct queue */
275	evq = sc->evq[txq->evq_index];
276
277	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
278	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
279	magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
280
281	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
282	    ("evq not started"));
283	efx_ev_qpost(evq->common, magic);
284
285	return (B_FALSE);
286}
287
288static boolean_t
289sfxge_ev_software(void *arg, uint16_t magic)
290{
291	struct sfxge_evq *evq;
292	struct sfxge_softc *sc;
293	unsigned int label;
294
295	evq = (struct sfxge_evq *)arg;
296	sc = evq->sc;
297
298	label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
299	magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
300
301	switch (magic) {
302	case SFXGE_MAGIC_RX_QFLUSH_DONE: {
303		struct sfxge_rxq *rxq = sc->rxq[label];
304
305		KASSERT(rxq != NULL, ("rxq == NULL"));
306		KASSERT(evq->index == rxq->index,
307		    ("evq->index != rxq->index"));
308
309		sfxge_rx_qflush_done(rxq);
310		break;
311	}
312	case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
313		struct sfxge_rxq *rxq = sc->rxq[label];
314
315		KASSERT(rxq != NULL, ("rxq == NULL"));
316		KASSERT(evq->index == rxq->index,
317		    ("evq->index != rxq->index"));
318
319		sfxge_rx_qflush_failed(rxq);
320		break;
321	}
322	case SFXGE_MAGIC_RX_QREFILL: {
323		struct sfxge_rxq *rxq = sc->rxq[label];
324
325		KASSERT(rxq != NULL, ("rxq == NULL"));
326		KASSERT(evq->index == rxq->index,
327		    ("evq->index != rxq->index"));
328
329		sfxge_rx_qrefill(rxq);
330		break;
331	}
332	case SFXGE_MAGIC_TX_QFLUSH_DONE: {
333		struct sfxge_txq *txq = sc->txq[label];
334
335		KASSERT(txq != NULL, ("txq == NULL"));
336		KASSERT(evq->index == txq->evq_index,
337		    ("evq->index != txq->evq_index"));
338
339		sfxge_tx_qflush_done(txq);
340		break;
341	}
342	default:
343		break;
344	}
345
346	return (B_FALSE);
347}
348
349static boolean_t
350sfxge_ev_sram(void *arg, uint32_t code)
351{
352	(void)arg;
353	(void)code;
354
355	switch (code) {
356	case EFX_SRAM_UPDATE:
357		EFSYS_PROBE(sram_update);
358		break;
359
360	case EFX_SRAM_CLEAR:
361		EFSYS_PROBE(sram_clear);
362		break;
363
364	case EFX_SRAM_ILLEGAL_CLEAR:
365		EFSYS_PROBE(sram_illegal_clear);
366		break;
367
368	default:
369		KASSERT(B_FALSE, ("Impossible SRAM event"));
370		break;
371	}
372
373	return (B_FALSE);
374}
375
376static boolean_t
377sfxge_ev_timer(void *arg, uint32_t index)
378{
379	(void)arg;
380	(void)index;
381
382	return (B_FALSE);
383}
384
385static boolean_t
386sfxge_ev_wake_up(void *arg, uint32_t index)
387{
388	(void)arg;
389	(void)index;
390
391	return (B_FALSE);
392}
393
394static void
395sfxge_ev_stat_update(struct sfxge_softc *sc)
396{
397	struct sfxge_evq *evq;
398	unsigned int index;
399	clock_t now;
400
401	sx_xlock(&sc->softc_lock);
402
403	if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
404		goto out;
405
406	now = ticks;
407	if (now - sc->ev_stats_update_time < hz)
408		goto out;
409
410	sc->ev_stats_update_time = now;
411
412	/* Add event counts from each event queue in turn */
413	for (index = 0; index < sc->intr.n_alloc; index++) {
414		evq = sc->evq[index];
415		mtx_lock(&evq->lock);
416		efx_ev_qstats_update(evq->common, sc->ev_stats);
417		mtx_unlock(&evq->lock);
418	}
419out:
420	sx_xunlock(&sc->softc_lock);
421}
422
423static int
424sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
425{
426	struct sfxge_softc *sc = arg1;
427	unsigned int id = arg2;
428
429	sfxge_ev_stat_update(sc);
430
431	return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]));
432}
433
434static void
435sfxge_ev_stat_init(struct sfxge_softc *sc)
436{
437	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
438	struct sysctl_oid_list *stat_list;
439	unsigned int id;
440	char name[40];
441
442	stat_list = SYSCTL_CHILDREN(sc->stats_node);
443
444	for (id = 0; id < EV_NQSTATS; id++) {
445		snprintf(name, sizeof(name), "ev_%s",
446			 efx_ev_qstat_name(sc->enp, id));
447		SYSCTL_ADD_PROC(
448			ctx, stat_list,
449			OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
450			sc, id, sfxge_ev_stat_handler, "Q",
451			"");
452	}
453}
454
455static void
456sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
457{
458	struct sfxge_evq *evq;
459	efx_evq_t *eep;
460
461	evq = sc->evq[idx];
462	eep = evq->common;
463
464	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
465	    ("evq->init_state != SFXGE_EVQ_STARTED"));
466
467	(void)efx_ev_qmoderate(eep, us);
468}
469
470static int
471sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
472{
473	struct sfxge_softc *sc = arg1;
474	struct sfxge_intr *intr = &sc->intr;
475	unsigned int moderation;
476	int error;
477	int index;
478
479	sx_xlock(&sc->softc_lock);
480
481	if (req->newptr) {
482		if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
483		    != 0)
484			goto out;
485
486		/* We may not be calling efx_ev_qmoderate() now,
487		 * so we have to range-check the value ourselves.
488		 */
489		if (moderation >
490		    efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
491			error = EINVAL;
492			goto out;
493		}
494
495		sc->ev_moderation = moderation;
496		if (intr->state == SFXGE_INTR_STARTED) {
497			for (index = 0; index < intr->n_alloc; index++)
498				sfxge_ev_qmoderate(sc, index, moderation);
499		}
500	} else {
501		error = SYSCTL_OUT(req, &sc->ev_moderation,
502				   sizeof(sc->ev_moderation));
503	}
504
505out:
506	sx_xunlock(&sc->softc_lock);
507
508	return error;
509}
510
511static boolean_t
512sfxge_ev_initialized(void *arg)
513{
514	struct sfxge_evq *evq;
515
516	evq = (struct sfxge_evq *)arg;
517
518	KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
519	    ("evq not starting"));
520
521	evq->init_state = SFXGE_EVQ_STARTED;
522
523	return (0);
524}
525
526static boolean_t
527sfxge_ev_link_change(void *arg, efx_link_mode_t	link_mode)
528{
529	struct sfxge_evq *evq;
530	struct sfxge_softc *sc;
531
532	evq = (struct sfxge_evq *)arg;
533	sc = evq->sc;
534
535	sfxge_mac_link_update(sc, link_mode);
536
537	return (0);
538}
539
540static const efx_ev_callbacks_t sfxge_ev_callbacks = {
541	.eec_initialized	= sfxge_ev_initialized,
542	.eec_rx			= sfxge_ev_rx,
543	.eec_tx			= sfxge_ev_tx,
544	.eec_exception		= sfxge_ev_exception,
545	.eec_rxq_flush_done	= sfxge_ev_rxq_flush_done,
546	.eec_rxq_flush_failed	= sfxge_ev_rxq_flush_failed,
547	.eec_txq_flush_done	= sfxge_ev_txq_flush_done,
548	.eec_software		= sfxge_ev_software,
549	.eec_sram		= sfxge_ev_sram,
550	.eec_wake_up		= sfxge_ev_wake_up,
551	.eec_timer		= sfxge_ev_timer,
552	.eec_link_change	= sfxge_ev_link_change,
553};
554
555
556int
557sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index)
558{
559	struct sfxge_evq *evq;
560	int rc;
561
562	evq = sc->evq[index];
563
564	mtx_lock(&evq->lock);
565
566	if (evq->init_state != SFXGE_EVQ_STARTING &&
567	    evq->init_state != SFXGE_EVQ_STARTED) {
568		rc = EINVAL;
569		goto fail;
570	}
571
572	/* Synchronize the DMA memory for reading */
573	bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
574	    BUS_DMASYNC_POSTREAD);
575
576	KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
577	KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
578	KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
579	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
580
581	/* Poll the queue */
582	efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
583
584	evq->rx_done = 0;
585	evq->tx_done = 0;
586
587	/* Perform any pending completion processing */
588	sfxge_ev_qcomplete(evq, B_TRUE);
589
590	/* Re-prime the event queue for interrupts */
591	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
592		goto fail;
593
594	mtx_unlock(&evq->lock);
595
596	return (0);
597
598fail:
599	mtx_unlock(&(evq->lock));
600	return (rc);
601}
602
603static void
604sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
605{
606	struct sfxge_evq *evq;
607
608	evq = sc->evq[index];
609
610	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
611	    ("evq->init_state != SFXGE_EVQ_STARTED"));
612
613	mtx_lock(&evq->lock);
614	evq->init_state = SFXGE_EVQ_INITIALIZED;
615	evq->read_ptr = 0;
616	evq->exception = B_FALSE;
617
618	/* Add event counts before discarding the common evq state */
619	efx_ev_qstats_update(evq->common, sc->ev_stats);
620
621	efx_ev_qdestroy(evq->common);
622	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
623	    EFX_EVQ_NBUFS(SFXGE_NEVS));
624	mtx_unlock(&evq->lock);
625}
626
627static int
628sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
629{
630	struct sfxge_evq *evq;
631	efsys_mem_t *esmp;
632	int count;
633	int rc;
634
635	evq = sc->evq[index];
636	esmp = &evq->mem;
637
638	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
639	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
640
641	/* Clear all events. */
642	(void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS));
643
644	/* Program the buffer table. */
645	if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
646	    EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0)
647		return rc;
648
649	/* Create the common code event queue. */
650	if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS,
651	    evq->buf_base_id, &evq->common)) != 0)
652		goto fail;
653
654	mtx_lock(&evq->lock);
655
656	/* Set the default moderation */
657	(void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
658
659	/* Prime the event queue for interrupts */
660	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
661		goto fail2;
662
663	evq->init_state = SFXGE_EVQ_STARTING;
664
665	mtx_unlock(&evq->lock);
666
667	/* Wait for the initialization event */
668	count = 0;
669	do {
670		/* Pause for 100 ms */
671		pause("sfxge evq init", hz / 10);
672
673		/* Check to see if the test event has been processed */
674		if (evq->init_state == SFXGE_EVQ_STARTED)
675			goto done;
676
677	} while (++count < 20);
678
679	rc = ETIMEDOUT;
680	goto fail3;
681
682done:
683	return (0);
684
685fail3:
686	mtx_lock(&evq->lock);
687	evq->init_state = SFXGE_EVQ_INITIALIZED;
688fail2:
689	mtx_unlock(&evq->lock);
690	efx_ev_qdestroy(evq->common);
691fail:
692	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
693	    EFX_EVQ_NBUFS(SFXGE_NEVS));
694
695	return (rc);
696}
697
698void
699sfxge_ev_stop(struct sfxge_softc *sc)
700{
701	struct sfxge_intr *intr;
702	efx_nic_t *enp;
703	int index;
704
705	intr = &sc->intr;
706	enp = sc->enp;
707
708	KASSERT(intr->state == SFXGE_INTR_STARTED,
709	    ("Interrupts not started"));
710
711	/* Stop the event queue(s) */
712	index = intr->n_alloc;
713	while (--index >= 0)
714		sfxge_ev_qstop(sc, index);
715
716	/* Tear down the event module */
717	efx_ev_fini(enp);
718}
719
720int
721sfxge_ev_start(struct sfxge_softc *sc)
722{
723	struct sfxge_intr *intr;
724	int index;
725	int rc;
726
727	intr = &sc->intr;
728
729	KASSERT(intr->state == SFXGE_INTR_STARTED,
730	    ("intr->state != SFXGE_INTR_STARTED"));
731
732	/* Initialize the event module */
733	if ((rc = efx_ev_init(sc->enp)) != 0)
734		return rc;
735
736	/* Start the event queues */
737	for (index = 0; index < intr->n_alloc; index++) {
738		if ((rc = sfxge_ev_qstart(sc, index)) != 0)
739			goto fail;
740	}
741
742	return (0);
743
744fail:
745	/* Stop the event queue(s) */
746	while (--index >= 0)
747		sfxge_ev_qstop(sc, index);
748
749	/* Tear down the event module */
750	efx_ev_fini(sc->enp);
751
752	return (rc);
753}
754
755static void
756sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
757{
758	struct sfxge_evq *evq;
759
760	evq = sc->evq[index];
761
762	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
763	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
764	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
765
766	sfxge_dma_free(&evq->mem);
767
768	sc->evq[index] = NULL;
769
770	mtx_destroy(&evq->lock);
771
772	free(evq, M_SFXGE);
773}
774
775static int
776sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
777{
778	struct sfxge_evq *evq;
779	efsys_mem_t *esmp;
780	int rc;
781
782	KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
783
784	evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
785	evq->sc = sc;
786	evq->index = index;
787	sc->evq[index] = evq;
788	esmp = &evq->mem;
789
790	/* Initialise TX completion list */
791	evq->txqs = &evq->txq;
792
793	/* Allocate DMA space. */
794	if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0)
795		return (rc);
796
797	/* Allocate buffer table entries. */
798	sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS),
799				 &evq->buf_base_id);
800
801	mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
802
803	evq->init_state = SFXGE_EVQ_INITIALIZED;
804
805	return (0);
806}
807
808void
809sfxge_ev_fini(struct sfxge_softc *sc)
810{
811	struct sfxge_intr *intr;
812	int index;
813
814	intr = &sc->intr;
815
816	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
817	    ("intr->state != SFXGE_INTR_INITIALIZED"));
818
819	sc->ev_moderation = 0;
820
821	/* Tear down the event queue(s). */
822	index = intr->n_alloc;
823	while (--index >= 0)
824		sfxge_ev_qfini(sc, index);
825}
826
827int
828sfxge_ev_init(struct sfxge_softc *sc)
829{
830	struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
831	struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
832	struct sfxge_intr *intr;
833	int index;
834	int rc;
835
836	intr = &sc->intr;
837
838	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
839	    ("intr->state != SFXGE_INTR_INITIALIZED"));
840
841	/* Set default interrupt moderation; add a sysctl to
842	 * read and change it.
843	 */
844	sc->ev_moderation = 30;
845	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
846			OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
847			sc, 0, sfxge_int_mod_handler, "IU",
848			"sfxge interrupt moderation (us)");
849
850	/*
851	 * Initialize the event queue(s) - one per interrupt.
852	 */
853	for (index = 0; index < intr->n_alloc; index++) {
854		if ((rc = sfxge_ev_qinit(sc, index)) != 0)
855			goto fail;
856	}
857
858	sfxge_ev_stat_init(sc);
859
860	return (0);
861
862fail:
863	while (--index >= 0)
864		sfxge_ev_qfini(sc, index);
865
866	return (rc);
867}
868