1/*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35
36#include "common/efx.h"
37
38#include "sfxge.h"
39
40static void
41sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
42{
43	struct sfxge_softc *sc;
44	unsigned int index;
45	struct sfxge_rxq *rxq;
46	struct sfxge_txq *txq;
47
48	sc = evq->sc;
49	index = evq->index;
50	rxq = sc->rxq[index];
51
52	if ((txq = evq->txq) != NULL) {
53		evq->txq = NULL;
54		evq->txqs = &(evq->txq);
55
56		do {
57			struct sfxge_txq *next;
58
59			next = txq->next;
60			txq->next = NULL;
61
62			KASSERT(txq->evq_index == index,
63			    ("txq->evq_index != index"));
64
65			if (txq->pending != txq->completed)
66				sfxge_tx_qcomplete(txq);
67
68			txq = next;
69		} while (txq != NULL);
70	}
71
72	if (rxq->pending != rxq->completed)
73		sfxge_rx_qcomplete(rxq, eop);
74}
75
76static boolean_t
77sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
78    uint16_t flags)
79{
80	struct sfxge_evq *evq;
81	struct sfxge_softc *sc;
82	struct sfxge_rxq *rxq;
83	unsigned int expected;
84	struct sfxge_rx_sw_desc *rx_desc;
85
86	evq = arg;
87	sc = evq->sc;
88
89	if (evq->exception)
90		goto done;
91
92	rxq = sc->rxq[label];
93	KASSERT(rxq != NULL, ("rxq == NULL"));
94	KASSERT(evq->index == rxq->index,
95	    ("evq->index != rxq->index"));
96
97	if (rxq->init_state != SFXGE_RXQ_STARTED)
98		goto done;
99
100	expected = rxq->pending++ & (SFXGE_NDESCS - 1);
101	if (id != expected) {
102		evq->exception = B_TRUE;
103
104		device_printf(sc->dev, "RX completion out of order"
105			      " (id=%#x expected=%#x flags=%#x); resetting\n",
106			      id, expected, flags);
107		sfxge_schedule_reset(sc);
108
109		goto done;
110	}
111
112	rx_desc = &rxq->queue[id];
113
114	KASSERT(rx_desc->flags == EFX_DISCARD,
115	    ("rx_desc->flags != EFX_DISCARD"));
116	rx_desc->flags = flags;
117
118	KASSERT(size < (1 << 16), ("size > (1 << 16)"));
119	rx_desc->size = (uint16_t)size;
120	prefetch_read_many(rx_desc->mbuf);
121
122	evq->rx_done++;
123
124	if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
125		sfxge_ev_qcomplete(evq, B_FALSE);
126
127done:
128	return (evq->rx_done >= SFXGE_EV_BATCH);
129}
130
131static boolean_t
132sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
133{
134	struct sfxge_evq *evq;
135	struct sfxge_softc *sc;
136
137	evq = (struct sfxge_evq *)arg;
138	sc = evq->sc;
139
140	evq->exception = B_TRUE;
141
142	if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
143		device_printf(sc->dev,
144			      "hardware exception (code=%u); resetting\n",
145			      code);
146		sfxge_schedule_reset(sc);
147	}
148
149	return (B_FALSE);
150}
151
152static boolean_t
153sfxge_ev_rxq_flush_done(void *arg, uint32_t label)
154{
155	struct sfxge_evq *evq;
156	struct sfxge_softc *sc;
157	struct sfxge_rxq *rxq;
158	unsigned int index;
159	uint16_t magic;
160
161	evq = (struct sfxge_evq *)arg;
162	sc = evq->sc;
163	rxq = sc->rxq[label];
164
165	KASSERT(rxq != NULL, ("rxq == NULL"));
166
167	/* Resend a software event on the correct queue */
168	index = rxq->index;
169	evq = sc->evq[index];
170
171	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
172	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
173	magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
174
175	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
176	    ("evq not started"));
177	efx_ev_qpost(evq->common, magic);
178
179	return (B_FALSE);
180}
181
182static boolean_t
183sfxge_ev_rxq_flush_failed(void *arg, uint32_t label)
184{
185	struct sfxge_evq *evq;
186	struct sfxge_softc *sc;
187	struct sfxge_rxq *rxq;
188	unsigned int index;
189	uint16_t magic;
190
191	evq = (struct sfxge_evq *)arg;
192	sc = evq->sc;
193	rxq = sc->rxq[label];
194
195	KASSERT(rxq != NULL, ("rxq == NULL"));
196
197	/* Resend a software event on the correct queue */
198	index = rxq->index;
199	evq = sc->evq[index];
200
201	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
202	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
203	magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
204
205	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
206	    ("evq not started"));
207	efx_ev_qpost(evq->common, magic);
208
209	return (B_FALSE);
210}
211
212static boolean_t
213sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
214{
215	struct sfxge_evq *evq;
216	struct sfxge_softc *sc;
217	struct sfxge_txq *txq;
218	unsigned int stop;
219	unsigned int delta;
220
221	evq = (struct sfxge_evq *)arg;
222	sc = evq->sc;
223	txq = sc->txq[label];
224
225	KASSERT(txq != NULL, ("txq == NULL"));
226	KASSERT(evq->index == txq->evq_index,
227	    ("evq->index != txq->evq_index"));
228
229	if (txq->init_state != SFXGE_TXQ_STARTED)
230		goto done;
231
232	stop = (id + 1) & (SFXGE_NDESCS - 1);
233	id = txq->pending & (SFXGE_NDESCS - 1);
234
235	delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop);
236	txq->pending += delta;
237
238	evq->tx_done++;
239
240	if (txq->next == NULL &&
241	    evq->txqs != &(txq->next)) {
242		*(evq->txqs) = txq;
243		evq->txqs = &(txq->next);
244	}
245
246	if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
247		sfxge_tx_qcomplete(txq);
248
249done:
250	return (evq->tx_done >= SFXGE_EV_BATCH);
251}
252
253static boolean_t
254sfxge_ev_txq_flush_done(void *arg, uint32_t label)
255{
256	struct sfxge_evq *evq;
257	struct sfxge_softc *sc;
258	struct sfxge_txq *txq;
259	uint16_t magic;
260
261	evq = (struct sfxge_evq *)arg;
262	sc = evq->sc;
263	txq = sc->txq[label];
264
265	KASSERT(txq != NULL, ("txq == NULL"));
266	KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
267	    ("txq not initialized"));
268
269	/* Resend a software event on the correct queue */
270	evq = sc->evq[txq->evq_index];
271
272	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
273	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
274	magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
275
276	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
277	    ("evq not started"));
278	efx_ev_qpost(evq->common, magic);
279
280	return (B_FALSE);
281}
282
283static boolean_t
284sfxge_ev_software(void *arg, uint16_t magic)
285{
286	struct sfxge_evq *evq;
287	struct sfxge_softc *sc;
288	unsigned int label;
289
290	evq = (struct sfxge_evq *)arg;
291	sc = evq->sc;
292
293	label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
294	magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
295
296	switch (magic) {
297	case SFXGE_MAGIC_RX_QFLUSH_DONE: {
298		struct sfxge_rxq *rxq = sc->rxq[label];
299
300		KASSERT(rxq != NULL, ("rxq == NULL"));
301		KASSERT(evq->index == rxq->index,
302		    ("evq->index != rxq->index"));
303
304		sfxge_rx_qflush_done(rxq);
305		break;
306	}
307	case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
308		struct sfxge_rxq *rxq = sc->rxq[label];
309
310		KASSERT(rxq != NULL, ("rxq == NULL"));
311		KASSERT(evq->index == rxq->index,
312		    ("evq->index != rxq->index"));
313
314		sfxge_rx_qflush_failed(rxq);
315		break;
316	}
317	case SFXGE_MAGIC_RX_QREFILL: {
318		struct sfxge_rxq *rxq = sc->rxq[label];
319
320		KASSERT(rxq != NULL, ("rxq == NULL"));
321		KASSERT(evq->index == rxq->index,
322		    ("evq->index != rxq->index"));
323
324		sfxge_rx_qrefill(rxq);
325		break;
326	}
327	case SFXGE_MAGIC_TX_QFLUSH_DONE: {
328		struct sfxge_txq *txq = sc->txq[label];
329
330		KASSERT(txq != NULL, ("txq == NULL"));
331		KASSERT(evq->index == txq->evq_index,
332		    ("evq->index != txq->evq_index"));
333
334		sfxge_tx_qflush_done(txq);
335		break;
336	}
337	default:
338		break;
339	}
340
341	return (B_FALSE);
342}
343
344static boolean_t
345sfxge_ev_sram(void *arg, uint32_t code)
346{
347	(void)arg;
348	(void)code;
349
350	switch (code) {
351	case EFX_SRAM_UPDATE:
352		EFSYS_PROBE(sram_update);
353		break;
354
355	case EFX_SRAM_CLEAR:
356		EFSYS_PROBE(sram_clear);
357		break;
358
359	case EFX_SRAM_ILLEGAL_CLEAR:
360		EFSYS_PROBE(sram_illegal_clear);
361		break;
362
363	default:
364		KASSERT(B_FALSE, ("Impossible SRAM event"));
365		break;
366	}
367
368	return (B_FALSE);
369}
370
371static boolean_t
372sfxge_ev_timer(void *arg, uint32_t index)
373{
374	(void)arg;
375	(void)index;
376
377	return (B_FALSE);
378}
379
380static boolean_t
381sfxge_ev_wake_up(void *arg, uint32_t index)
382{
383	(void)arg;
384	(void)index;
385
386	return (B_FALSE);
387}
388
389static void
390sfxge_ev_stat_update(struct sfxge_softc *sc)
391{
392	struct sfxge_evq *evq;
393	unsigned int index;
394	clock_t now;
395
396	sx_xlock(&sc->softc_lock);
397
398	if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
399		goto out;
400
401	now = ticks;
402	if (now - sc->ev_stats_update_time < hz)
403		goto out;
404
405	sc->ev_stats_update_time = now;
406
407	/* Add event counts from each event queue in turn */
408	for (index = 0; index < sc->intr.n_alloc; index++) {
409		evq = sc->evq[index];
410		mtx_lock(&evq->lock);
411		efx_ev_qstats_update(evq->common, sc->ev_stats);
412		mtx_unlock(&evq->lock);
413	}
414out:
415	sx_xunlock(&sc->softc_lock);
416}
417
418static int
419sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
420{
421	struct sfxge_softc *sc = arg1;
422	unsigned int id = arg2;
423
424	sfxge_ev_stat_update(sc);
425
426	return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]));
427}
428
429static void
430sfxge_ev_stat_init(struct sfxge_softc *sc)
431{
432	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
433	struct sysctl_oid_list *stat_list;
434	unsigned int id;
435	char name[40];
436
437	stat_list = SYSCTL_CHILDREN(sc->stats_node);
438
439	for (id = 0; id < EV_NQSTATS; id++) {
440		snprintf(name, sizeof(name), "ev_%s",
441			 efx_ev_qstat_name(sc->enp, id));
442		SYSCTL_ADD_PROC(
443			ctx, stat_list,
444			OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
445			sc, id, sfxge_ev_stat_handler, "Q",
446			"");
447	}
448}
449
450static void
451sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
452{
453	struct sfxge_evq *evq;
454	efx_evq_t *eep;
455
456	evq = sc->evq[idx];
457	eep = evq->common;
458
459	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
460	    ("evq->init_state != SFXGE_EVQ_STARTED"));
461
462	(void)efx_ev_qmoderate(eep, us);
463}
464
465static int
466sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
467{
468	struct sfxge_softc *sc = arg1;
469	struct sfxge_intr *intr = &sc->intr;
470	unsigned int moderation;
471	int error;
472	int index;
473
474	sx_xlock(&sc->softc_lock);
475
476	if (req->newptr) {
477		if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
478		    != 0)
479			goto out;
480
481		/* We may not be calling efx_ev_qmoderate() now,
482		 * so we have to range-check the value ourselves.
483		 */
484		if (moderation >
485		    efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
486			error = EINVAL;
487			goto out;
488		}
489
490		sc->ev_moderation = moderation;
491		if (intr->state == SFXGE_INTR_STARTED) {
492			for (index = 0; index < intr->n_alloc; index++)
493				sfxge_ev_qmoderate(sc, index, moderation);
494		}
495	} else {
496		error = SYSCTL_OUT(req, &sc->ev_moderation,
497				   sizeof(sc->ev_moderation));
498	}
499
500out:
501	sx_xunlock(&sc->softc_lock);
502
503	return error;
504}
505
506static boolean_t
507sfxge_ev_initialized(void *arg)
508{
509	struct sfxge_evq *evq;
510
511	evq = (struct sfxge_evq *)arg;
512
513	KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
514	    ("evq not starting"));
515
516	evq->init_state = SFXGE_EVQ_STARTED;
517
518	return (0);
519}
520
521static boolean_t
522sfxge_ev_link_change(void *arg, efx_link_mode_t	link_mode)
523{
524	struct sfxge_evq *evq;
525	struct sfxge_softc *sc;
526
527	evq = (struct sfxge_evq *)arg;
528	sc = evq->sc;
529
530	sfxge_mac_link_update(sc, link_mode);
531
532	return (0);
533}
534
535static const efx_ev_callbacks_t sfxge_ev_callbacks = {
536	.eec_initialized	= sfxge_ev_initialized,
537	.eec_rx			= sfxge_ev_rx,
538	.eec_tx			= sfxge_ev_tx,
539	.eec_exception		= sfxge_ev_exception,
540	.eec_rxq_flush_done	= sfxge_ev_rxq_flush_done,
541	.eec_rxq_flush_failed	= sfxge_ev_rxq_flush_failed,
542	.eec_txq_flush_done	= sfxge_ev_txq_flush_done,
543	.eec_software		= sfxge_ev_software,
544	.eec_sram		= sfxge_ev_sram,
545	.eec_wake_up		= sfxge_ev_wake_up,
546	.eec_timer		= sfxge_ev_timer,
547	.eec_link_change	= sfxge_ev_link_change,
548};
549
550
551int
552sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index)
553{
554	struct sfxge_evq *evq;
555	int rc;
556
557	evq = sc->evq[index];
558
559	mtx_lock(&evq->lock);
560
561	if (evq->init_state != SFXGE_EVQ_STARTING &&
562	    evq->init_state != SFXGE_EVQ_STARTED) {
563		rc = EINVAL;
564		goto fail;
565	}
566
567	/* Synchronize the DMA memory for reading */
568	bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
569	    BUS_DMASYNC_POSTREAD);
570
571	KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
572	KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
573	KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
574	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
575
576	/* Poll the queue */
577	efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
578
579	evq->rx_done = 0;
580	evq->tx_done = 0;
581
582	/* Perform any pending completion processing */
583	sfxge_ev_qcomplete(evq, B_TRUE);
584
585	/* Re-prime the event queue for interrupts */
586	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
587		goto fail;
588
589	mtx_unlock(&evq->lock);
590
591	return (0);
592
593fail:
594	mtx_unlock(&(evq->lock));
595	return (rc);
596}
597
598static void
599sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
600{
601	struct sfxge_evq *evq;
602
603	evq = sc->evq[index];
604
605	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
606	    ("evq->init_state != SFXGE_EVQ_STARTED"));
607
608	mtx_lock(&evq->lock);
609	evq->init_state = SFXGE_EVQ_INITIALIZED;
610	evq->read_ptr = 0;
611	evq->exception = B_FALSE;
612
613	/* Add event counts before discarding the common evq state */
614	efx_ev_qstats_update(evq->common, sc->ev_stats);
615
616	efx_ev_qdestroy(evq->common);
617	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
618	    EFX_EVQ_NBUFS(SFXGE_NEVS));
619	mtx_unlock(&evq->lock);
620}
621
622static int
623sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
624{
625	struct sfxge_evq *evq;
626	efsys_mem_t *esmp;
627	int count;
628	int rc;
629
630	evq = sc->evq[index];
631	esmp = &evq->mem;
632
633	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
634	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
635
636	/* Clear all events. */
637	(void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS));
638
639	/* Program the buffer table. */
640	if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
641	    EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0)
642		return rc;
643
644	/* Create the common code event queue. */
645	if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS,
646	    evq->buf_base_id, &evq->common)) != 0)
647		goto fail;
648
649	mtx_lock(&evq->lock);
650
651	/* Set the default moderation */
652	(void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
653
654	/* Prime the event queue for interrupts */
655	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
656		goto fail2;
657
658	evq->init_state = SFXGE_EVQ_STARTING;
659
660	mtx_unlock(&evq->lock);
661
662	/* Wait for the initialization event */
663	count = 0;
664	do {
665		/* Pause for 100 ms */
666		pause("sfxge evq init", hz / 10);
667
668		/* Check to see if the test event has been processed */
669		if (evq->init_state == SFXGE_EVQ_STARTED)
670			goto done;
671
672	} while (++count < 20);
673
674	rc = ETIMEDOUT;
675	goto fail3;
676
677done:
678	return (0);
679
680fail3:
681	mtx_lock(&evq->lock);
682	evq->init_state = SFXGE_EVQ_INITIALIZED;
683fail2:
684	mtx_unlock(&evq->lock);
685	efx_ev_qdestroy(evq->common);
686fail:
687	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
688	    EFX_EVQ_NBUFS(SFXGE_NEVS));
689
690	return (rc);
691}
692
693void
694sfxge_ev_stop(struct sfxge_softc *sc)
695{
696	struct sfxge_intr *intr;
697	efx_nic_t *enp;
698	int index;
699
700	intr = &sc->intr;
701	enp = sc->enp;
702
703	KASSERT(intr->state == SFXGE_INTR_STARTED,
704	    ("Interrupts not started"));
705
706	/* Stop the event queue(s) */
707	index = intr->n_alloc;
708	while (--index >= 0)
709		sfxge_ev_qstop(sc, index);
710
711	/* Tear down the event module */
712	efx_ev_fini(enp);
713}
714
715int
716sfxge_ev_start(struct sfxge_softc *sc)
717{
718	struct sfxge_intr *intr;
719	int index;
720	int rc;
721
722	intr = &sc->intr;
723
724	KASSERT(intr->state == SFXGE_INTR_STARTED,
725	    ("intr->state != SFXGE_INTR_STARTED"));
726
727	/* Initialize the event module */
728	if ((rc = efx_ev_init(sc->enp)) != 0)
729		return rc;
730
731	/* Start the event queues */
732	for (index = 0; index < intr->n_alloc; index++) {
733		if ((rc = sfxge_ev_qstart(sc, index)) != 0)
734			goto fail;
735	}
736
737	return (0);
738
739fail:
740	/* Stop the event queue(s) */
741	while (--index >= 0)
742		sfxge_ev_qstop(sc, index);
743
744	/* Tear down the event module */
745	efx_ev_fini(sc->enp);
746
747	return (rc);
748}
749
750static void
751sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
752{
753	struct sfxge_evq *evq;
754
755	evq = sc->evq[index];
756
757	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
758	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
759	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
760
761	sfxge_dma_free(&evq->mem);
762
763	sc->evq[index] = NULL;
764
765	mtx_destroy(&evq->lock);
766
767	free(evq, M_SFXGE);
768}
769
770static int
771sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
772{
773	struct sfxge_evq *evq;
774	efsys_mem_t *esmp;
775	int rc;
776
777	KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
778
779	evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
780	evq->sc = sc;
781	evq->index = index;
782	sc->evq[index] = evq;
783	esmp = &evq->mem;
784
785	/* Initialise TX completion list */
786	evq->txqs = &evq->txq;
787
788	/* Allocate DMA space. */
789	if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0)
790		return (rc);
791
792	/* Allocate buffer table entries. */
793	sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS),
794				 &evq->buf_base_id);
795
796	mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
797
798	evq->init_state = SFXGE_EVQ_INITIALIZED;
799
800	return (0);
801}
802
803void
804sfxge_ev_fini(struct sfxge_softc *sc)
805{
806	struct sfxge_intr *intr;
807	int index;
808
809	intr = &sc->intr;
810
811	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
812	    ("intr->state != SFXGE_INTR_INITIALIZED"));
813
814	sc->ev_moderation = 0;
815
816	/* Tear down the event queue(s). */
817	index = intr->n_alloc;
818	while (--index >= 0)
819		sfxge_ev_qfini(sc, index);
820}
821
822int
823sfxge_ev_init(struct sfxge_softc *sc)
824{
825	struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
826	struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
827	struct sfxge_intr *intr;
828	int index;
829	int rc;
830
831	intr = &sc->intr;
832
833	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
834	    ("intr->state != SFXGE_INTR_INITIALIZED"));
835
836	/* Set default interrupt moderation; add a sysctl to
837	 * read and change it.
838	 */
839	sc->ev_moderation = 30;
840	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
841			OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
842			sc, 0, sfxge_int_mod_handler, "IU",
843			"sfxge interrupt moderation (us)");
844
845	/*
846	 * Initialize the event queue(s) - one per interrupt.
847	 */
848	for (index = 0; index < intr->n_alloc; index++) {
849		if ((rc = sfxge_ev_qinit(sc, index)) != 0)
850			goto fail;
851	}
852
853	sfxge_ev_stat_init(sc);
854
855	return (0);
856
857fail:
858	while (--index >= 0)
859		sfxge_ev_qfini(sc, index);
860
861	return (rc);
862}
863