dev/sfxge/sfxge_ev.c

50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
57 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
59 	sc = evq->sc;
60 	index = evq->index;
63 	if ((txq = evq->txq) != NULL) {
64 		evq->txq = NULL;
65 		evq->txqs = &(evq->txq);
77 				sfxge_tx_qcomplete(txq, evq);
88 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
94 	rxq = evq->sc->rxq[evq->index];
97 	KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
106 	struct sfxge_evq *evq;
113 	evq = arg;
114 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
116 	sc = evq->sc;
118 	if (evq->exception)
121 	rxq = sfxge_get_rxq_by_label(evq, label);
133 			evq->exception = B_TRUE;
158 	evq->rx_done++;
161 		sfxge_ev_qcomplete(evq, B_FALSE);
164 	return (evq->rx_done >= SFXGE_EV_BATCH);
170 	struct sfxge_evq *evq;
173 	evq = (struct sfxge_evq *)arg;
174 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
176 	sc = evq->sc;
178 	DBGPRINT(sc->dev, "[%d] %s", evq->index,
190 	evq->exception = B_TRUE;
205 	struct sfxge_evq *evq;
211 	evq = (struct sfxge_evq *)arg;
212 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
214 	sc = evq->sc;
221 	if (index == evq->index) {
226 	evq = sc->evq[index];
229 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
230 	    ("evq not started"));
231 	efx_ev_qpost(evq->common, magic);
239 	struct sfxge_evq *evq;
245 	evq = (struct sfxge_evq *)arg;
246 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
248 	sc = evq->sc;
255 	evq = sc->evq[index];
258 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
259 	    ("evq not started"));
260 	efx_ev_qpost(evq->common, magic);
266 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
270 	KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) :
271 		((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
275 	index = (evq->index == 0) ?
276 		label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc));
277 	return (evq->sc->txq[index]);
283 	struct sfxge_evq *evq;
288 	evq = (struct sfxge_evq *)arg;
289 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
291 	txq = sfxge_get_txq_by_label(evq, label);
294 	KASSERT(evq->index == txq->evq_index,
295 	    ("evq->index != txq->evq_index"));
306 	evq->tx_done++;
309 	    evq->txqs != &(txq->next)) {
310 		*(evq->txqs) = txq;
311 		evq->txqs = &(txq->next);
315 		sfxge_tx_qcomplete(txq, evq);
318 	return (evq->tx_done >= SFXGE_EV_BATCH);
324 	struct sfxge_evq *evq;
329 	evq = (struct sfxge_evq *)arg;
330 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
332 	sc = evq->sc;
339 	if (txq->evq_index == evq->index) {
345 	evq = sc->evq[txq->evq_index];
348 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
349 	    ("evq not started"));
350 	efx_ev_qpost(evq->common, magic);
358 	struct sfxge_evq *evq;
362 	evq = (struct sfxge_evq *)arg;
363 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
365 	sc = evq->sc;
372 		sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
376 		sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
380 		sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
384 		struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
387 		KASSERT(evq->index == txq->evq_index,
388 		    ("evq->index != txq->evq_index"));
448 sfxge_evq_stat_update(struct sfxge_evq *evq)
452 	SFXGE_EVQ_LOCK(evq);
454 	if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED))
458 	if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz)
461 	evq->stats_update_time = now;
462 	efx_ev_qstats_update(evq->common, evq->stats);
465 	SFXGE_EVQ_UNLOCK(evq);
471 	struct sfxge_evq *evq = arg1;
472 	struct sfxge_softc *sc = evq->sc;
477 	sfxge_evq_stat_update(evq);
481 	return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id])));
485 sfxge_evq_stat_init(struct sfxge_evq *evq)
487 	struct sfxge_softc *sc = evq->sc;
493 	snprintf(name, sizeof(name), "%u", evq->index);
505 			evq, id, sfxge_evq_stat_handler, "Q",
515 	struct sfxge_evq *evq;
532 		evq = sc->evq[index];
533 		sfxge_evq_stat_update(evq);
535 			sc->ev_stats[id] += evq->stats[id];
578 	struct sfxge_evq *evq;
581 	evq = sc->evq[idx];
582 	eep = evq->common;
584 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
585 	    ("evq->init_state != SFXGE_EVQ_STARTED"));
634 	struct sfxge_evq *evq;
636 	evq = (struct sfxge_evq *)arg;
637 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
640 	KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
641 		evq->init_state == SFXGE_EVQ_STARTED,
642 	    ("evq not starting"));
644 	evq->init_state = SFXGE_EVQ_STARTED;
652 	struct sfxge_evq *evq;
655 	evq = (struct sfxge_evq *)arg;
656 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
658 	sc = evq->sc;
682 sfxge_ev_qpoll(struct sfxge_evq *evq)
686 	SFXGE_EVQ_LOCK(evq);
688 	if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
689 			    evq->init_state != SFXGE_EVQ_STARTED)) {
695 	bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
698 	KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
699 	KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
700 	KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
701 	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
704 	efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
706 	evq->rx_done = 0;
707 	evq->tx_done = 0;
710 	sfxge_ev_qcomplete(evq, B_TRUE);
713 	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
716 	SFXGE_EVQ_UNLOCK(evq);
721 	SFXGE_EVQ_UNLOCK(evq);
728 	struct sfxge_evq *evq;
730 	evq = sc->evq[index];
732 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
733 	    ("evq->init_state != SFXGE_EVQ_STARTED"));
735 	SFXGE_EVQ_LOCK(evq);
736 	evq->init_state = SFXGE_EVQ_INITIALIZED;
737 	evq->read_ptr = 0;
738 	evq->exception = B_FALSE;
741 	/* Add event counts before discarding the common evq state */
742 	efx_ev_qstats_update(evq->common, evq->stats);
745 	efx_ev_qdestroy(evq->common);
746 	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
747 	    EFX_EVQ_NBUFS(evq->entries));
748 	SFXGE_EVQ_UNLOCK(evq);
754 	struct sfxge_evq *evq;
759 	evq = sc->evq[index];
760 	esmp = &evq->mem;
762 	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
763 	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
766 	(void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
769 	if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
770 	    EFX_EVQ_NBUFS(evq->entries))) != 0)
774 	if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
775 	    evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,
776 	    &evq->common)) != 0)
779 	SFXGE_EVQ_LOCK(evq);
782 	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
785 	evq->init_state = SFXGE_EVQ_STARTING;
787 	SFXGE_EVQ_UNLOCK(evq);
793 		pause("sfxge evq init", hz / 10);
796 		if (evq->init_state == SFXGE_EVQ_STARTED)
808 	SFXGE_EVQ_LOCK(evq);
809 	evq->init_state = SFXGE_EVQ_INITIALIZED;
811 	SFXGE_EVQ_UNLOCK(evq);
812 	efx_ev_qdestroy(evq->common);
814 	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
815 	    EFX_EVQ_NBUFS(evq->entries));
880 	struct sfxge_evq *evq;
882 	evq = sc->evq[index];
884 	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
885 	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
886 	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
888 	sfxge_dma_free(&evq->mem);
890 	sc->evq[index] = NULL;
892 	SFXGE_EVQ_LOCK_DESTROY(evq);
894 	free(evq, M_SFXGE);
900 	struct sfxge_evq *evq;
906 	evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
907 	evq->sc = sc;
908 	evq->index = index;
909 	sc->evq[index] = evq;
910 	esmp = &evq->mem;
918 		evq->entries =
923 		evq->entries =
929 	evq->txqs = &evq->txq;
932 	if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
936 	sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
937 				 &evq->buf_base_id);
939 	SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
941 	evq->init_state = SFXGE_EVQ_INITIALIZED;
944 	rc = sfxge_evq_stat_init(evq);
953 	evq->init_state = SFXGE_EVQ_UNINITIALIZED;
954 	SFXGE_EVQ_LOCK_DESTROY(evq);
956 	sc->evq[index] = NULL;
957 	free(evq, M_SFXGE);
1012 		OID_AUTO, "evq", CTLFLAG_RD, NULL, "Event queues stats");