Deleted Added
sdiff udiff text old ( 272325 ) new ( 272328 )
full compact
sfxge_rx.c (272325) sfxge_rx.c (272328)
1/*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 14 unchanged lines hidden (view full) ---

23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 14 unchanged lines hidden (view full) ---

23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_rx.c 272325 2014-09-30 20:18:10Z gnn $");
31__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_rx.c 272328 2014-09-30 20:36:07Z gnn $");
32
33#include <sys/types.h>
34#include <sys/mbuf.h>
35#include <sys/smp.h>
36#include <sys/socket.h>
37#include <sys/sysctl.h>
38#include <sys/limits.h>
39

--- 9 unchanged lines hidden (view full) ---

49#include <machine/in_cksum.h>
50
51#include "common/efx.h"
52
53
54#include "sfxge.h"
55#include "sfxge_rx.h"
56
32
33#include <sys/types.h>
34#include <sys/mbuf.h>
35#include <sys/smp.h>
36#include <sys/socket.h>
37#include <sys/sysctl.h>
38#include <sys/limits.h>
39

--- 9 unchanged lines hidden (view full) ---

49#include <machine/in_cksum.h>
50
51#include "common/efx.h"
52
53
54#include "sfxge.h"
55#include "sfxge_rx.h"
56
57#define RX_REFILL_THRESHOLD (EFX_RXQ_LIMIT(SFXGE_NDESCS) * 9 / 10)
58#define RX_REFILL_THRESHOLD_2 (RX_REFILL_THRESHOLD / 2)
57#define RX_REFILL_THRESHOLD(_entries) (EFX_RXQ_LIMIT(_entries) * 9 / 10)
59
60/* Size of the LRO hash table. Must be a power of 2. A larger table
61 * means we can accelerate a larger number of streams.
62 */
63static unsigned lro_table_size = 128;
64
65/* Maximum length of a hash chain. If chains get too long then the lookup
66 * time increases and may exceed the benefit of LRO.

--- 142 unchanged lines hidden (view full) ---

209 prefetch_read_many(rxq->common);
210
211 mtx_assert(&evq->lock, MA_OWNED);
212
213 if (rxq->init_state != SFXGE_RXQ_STARTED)
214 return;
215
216 rxfill = rxq->added - rxq->completed;
58
59/* Size of the LRO hash table. Must be a power of 2. A larger table
60 * means we can accelerate a larger number of streams.
61 */
62static unsigned lro_table_size = 128;
63
64/* Maximum length of a hash chain. If chains get too long then the lookup
65 * time increases and may exceed the benefit of LRO.

--- 142 unchanged lines hidden (view full) ---

208 prefetch_read_many(rxq->common);
209
210 mtx_assert(&evq->lock, MA_OWNED);
211
212 if (rxq->init_state != SFXGE_RXQ_STARTED)
213 return;
214
215 rxfill = rxq->added - rxq->completed;
217 KASSERT(rxfill <= EFX_RXQ_LIMIT(SFXGE_NDESCS),
218 ("rxfill > EFX_RXQ_LIMIT(SFXGE_NDESCS)"));
219 ntodo = min(EFX_RXQ_LIMIT(SFXGE_NDESCS) - rxfill, target);
220 KASSERT(ntodo <= EFX_RXQ_LIMIT(SFXGE_NDESCS),
221 ("ntodo > EFX_RQX_LIMIT(SFXGE_NDESCS)"));
216 KASSERT(rxfill <= EFX_RXQ_LIMIT(rxq->entries),
217 ("rxfill > EFX_RXQ_LIMIT(rxq->entries)"));
218 ntodo = min(EFX_RXQ_LIMIT(rxq->entries) - rxfill, target);
219 KASSERT(ntodo <= EFX_RXQ_LIMIT(rxq->entries),
220 ("ntodo > EFX_RQX_LIMIT(rxq->entries)"));
222
223 if (ntodo == 0)
224 return;
225
226 batch = 0;
227 mblksize = sc->rx_buffer_size;
228 while (ntodo-- > 0) {
229 unsigned int id;
230 struct sfxge_rx_sw_desc *rx_desc;
231 bus_dma_segment_t seg;
232 struct mbuf *m;
233
221
222 if (ntodo == 0)
223 return;
224
225 batch = 0;
226 mblksize = sc->rx_buffer_size;
227 while (ntodo-- > 0) {
228 unsigned int id;
229 struct sfxge_rx_sw_desc *rx_desc;
230 bus_dma_segment_t seg;
231 struct mbuf *m;
232
234 id = (rxq->added + batch) & (SFXGE_NDESCS - 1);
233 id = (rxq->added + batch) & rxq->ptr_mask;
235 rx_desc = &rxq->queue[id];
236 KASSERT(rx_desc->mbuf == NULL, ("rx_desc->mbuf != NULL"));
237
238 rx_desc->flags = EFX_DISCARD;
239 m = rx_desc->mbuf = sfxge_rx_alloc_mbuf(sc);
240 if (m == NULL)
241 break;
242 sfxge_map_mbuf_fast(rxq->mem.esm_tag, rxq->mem.esm_map, m, &seg);

--- 26 unchanged lines hidden (view full) ---

269void
270sfxge_rx_qrefill(struct sfxge_rxq *rxq)
271{
272
273 if (rxq->init_state != SFXGE_RXQ_STARTED)
274 return;
275
276 /* Make sure the queue is full */
234 rx_desc = &rxq->queue[id];
235 KASSERT(rx_desc->mbuf == NULL, ("rx_desc->mbuf != NULL"));
236
237 rx_desc->flags = EFX_DISCARD;
238 m = rx_desc->mbuf = sfxge_rx_alloc_mbuf(sc);
239 if (m == NULL)
240 break;
241 sfxge_map_mbuf_fast(rxq->mem.esm_tag, rxq->mem.esm_map, m, &seg);

--- 26 unchanged lines hidden (view full) ---

268void
269sfxge_rx_qrefill(struct sfxge_rxq *rxq)
270{
271
272 if (rxq->init_state != SFXGE_RXQ_STARTED)
273 return;
274
275 /* Make sure the queue is full */
277 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(SFXGE_NDESCS), B_TRUE);
276 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(rxq->entries), B_TRUE);
278}
279
280static void __sfxge_rx_deliver(struct sfxge_softc *sc, struct mbuf *m)
281{
282 struct ifnet *ifp = sc->ifnet;
283
284 m->m_pkthdr.rcvif = ifp;
285 m->m_pkthdr.csum_data = 0xffff;

--- 466 unchanged lines hidden (view full) ---

752
753 mtx_assert(&evq->lock, MA_OWNED);
754
755 completed = rxq->completed;
756 while (completed != rxq->pending) {
757 unsigned int id;
758 struct sfxge_rx_sw_desc *rx_desc;
759
277}
278
279static void __sfxge_rx_deliver(struct sfxge_softc *sc, struct mbuf *m)
280{
281 struct ifnet *ifp = sc->ifnet;
282
283 m->m_pkthdr.rcvif = ifp;
284 m->m_pkthdr.csum_data = 0xffff;

--- 466 unchanged lines hidden (view full) ---

751
752 mtx_assert(&evq->lock, MA_OWNED);
753
754 completed = rxq->completed;
755 while (completed != rxq->pending) {
756 unsigned int id;
757 struct sfxge_rx_sw_desc *rx_desc;
758
760 id = completed++ & (SFXGE_NDESCS - 1);
759 id = completed++ & rxq->ptr_mask;
761 rx_desc = &rxq->queue[id];
762 m = rx_desc->mbuf;
763
764 if (rxq->init_state != SFXGE_RXQ_STARTED)
765 goto discard;
766
767 if (rx_desc->flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
768 goto discard;

--- 47 unchanged lines hidden (view full) ---

816 /*
817 * If there are any pending flows and this is the end of the
818 * poll then they must be completed.
819 */
820 if (eop)
821 sfxge_lro_end_of_burst(rxq);
822
823 /* Top up the queue if necessary */
760 rx_desc = &rxq->queue[id];
761 m = rx_desc->mbuf;
762
763 if (rxq->init_state != SFXGE_RXQ_STARTED)
764 goto discard;
765
766 if (rx_desc->flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
767 goto discard;

--- 47 unchanged lines hidden (view full) ---

815 /*
816 * If there are any pending flows and this is the end of the
817 * poll then they must be completed.
818 */
819 if (eop)
820 sfxge_lro_end_of_burst(rxq);
821
822 /* Top up the queue if necessary */
824 if (level < RX_REFILL_THRESHOLD)
825 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(SFXGE_NDESCS), B_FALSE);
823 if (level < rxq->refill_threshold)
824 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(rxq->entries), B_FALSE);
826}
827
828static void
829sfxge_rx_qstop(struct sfxge_softc *sc, unsigned int index)
830{
831 struct sfxge_rxq *rxq;
832 struct sfxge_evq *evq;
833 unsigned int count;

--- 45 unchanged lines hidden (view full) ---

879 rxq->pending = 0;
880 rxq->completed = 0;
881 rxq->loopback = 0;
882
883 /* Destroy the common code receive queue. */
884 efx_rx_qdestroy(rxq->common);
885
886 efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
825}
826
827static void
828sfxge_rx_qstop(struct sfxge_softc *sc, unsigned int index)
829{
830 struct sfxge_rxq *rxq;
831 struct sfxge_evq *evq;
832 unsigned int count;

--- 45 unchanged lines hidden (view full) ---

878 rxq->pending = 0;
879 rxq->completed = 0;
880 rxq->loopback = 0;
881
882 /* Destroy the common code receive queue. */
883 efx_rx_qdestroy(rxq->common);
884
885 efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
887 EFX_RXQ_NBUFS(SFXGE_NDESCS));
886 EFX_RXQ_NBUFS(sc->rxq_entries));
888
889 mtx_unlock(&evq->lock);
890}
891
892static int
893sfxge_rx_qstart(struct sfxge_softc *sc, unsigned int index)
894{
895 struct sfxge_rxq *rxq;

--- 7 unchanged lines hidden (view full) ---

903
904 KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED,
905 ("rxq->init_state != SFXGE_RXQ_INITIALIZED"));
906 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
907 ("evq->init_state != SFXGE_EVQ_STARTED"));
908
909 /* Program the buffer table. */
910 if ((rc = efx_sram_buf_tbl_set(sc->enp, rxq->buf_base_id, esmp,
887
888 mtx_unlock(&evq->lock);
889}
890
891static int
892sfxge_rx_qstart(struct sfxge_softc *sc, unsigned int index)
893{
894 struct sfxge_rxq *rxq;

--- 7 unchanged lines hidden (view full) ---

902
903 KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED,
904 ("rxq->init_state != SFXGE_RXQ_INITIALIZED"));
905 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
906 ("evq->init_state != SFXGE_EVQ_STARTED"));
907
908 /* Program the buffer table. */
909 if ((rc = efx_sram_buf_tbl_set(sc->enp, rxq->buf_base_id, esmp,
911 EFX_RXQ_NBUFS(SFXGE_NDESCS))) != 0)
912 return rc;
910 EFX_RXQ_NBUFS(sc->rxq_entries))) != 0)
911 return (rc);
913
914 /* Create the common code receive queue. */
915 if ((rc = efx_rx_qcreate(sc->enp, index, index, EFX_RXQ_TYPE_DEFAULT,
912
913 /* Create the common code receive queue. */
914 if ((rc = efx_rx_qcreate(sc->enp, index, index, EFX_RXQ_TYPE_DEFAULT,
916 esmp, SFXGE_NDESCS, rxq->buf_base_id, evq->common,
915 esmp, sc->rxq_entries, rxq->buf_base_id, evq->common,
917 &rxq->common)) != 0)
918 goto fail;
919
920 mtx_lock(&evq->lock);
921
922 /* Enable the receive queue. */
923 efx_rx_qenable(rxq->common);
924
925 rxq->init_state = SFXGE_RXQ_STARTED;
926
927 /* Try to fill the queue from the pool. */
916 &rxq->common)) != 0)
917 goto fail;
918
919 mtx_lock(&evq->lock);
920
921 /* Enable the receive queue. */
922 efx_rx_qenable(rxq->common);
923
924 rxq->init_state = SFXGE_RXQ_STARTED;
925
926 /* Try to fill the queue from the pool. */
928 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(SFXGE_NDESCS), B_FALSE);
927 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(sc->rxq_entries), B_FALSE);
929
930 mtx_unlock(&evq->lock);
931
932 return (0);
933
934fail:
935 efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
928
929 mtx_unlock(&evq->lock);
930
931 return (0);
932
933fail:
934 efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
936 EFX_RXQ_NBUFS(SFXGE_NDESCS));
937 return rc;
935 EFX_RXQ_NBUFS(sc->rxq_entries));
936 return (rc);
938}
939
940void
941sfxge_rx_stop(struct sfxge_softc *sc)
942{
943 struct sfxge_intr *intr;
944 int index;
945

--- 154 unchanged lines hidden (view full) ---

1100 efsys_mem_t *esmp;
1101 int rc;
1102
1103 KASSERT(index < sc->intr.n_alloc, ("index >= %d", sc->intr.n_alloc));
1104
1105 rxq = malloc(sizeof(struct sfxge_rxq), M_SFXGE, M_ZERO | M_WAITOK);
1106 rxq->sc = sc;
1107 rxq->index = index;
937}
938
939void
940sfxge_rx_stop(struct sfxge_softc *sc)
941{
942 struct sfxge_intr *intr;
943 int index;
944

--- 154 unchanged lines hidden (view full) ---

1099 efsys_mem_t *esmp;
1100 int rc;
1101
1102 KASSERT(index < sc->intr.n_alloc, ("index >= %d", sc->intr.n_alloc));
1103
1104 rxq = malloc(sizeof(struct sfxge_rxq), M_SFXGE, M_ZERO | M_WAITOK);
1105 rxq->sc = sc;
1106 rxq->index = index;
1107 rxq->entries = sc->rxq_entries;
1108 rxq->ptr_mask = rxq->entries - 1;
1109 rxq->refill_threshold = RX_REFILL_THRESHOLD(rxq->entries);
1108
1109 sc->rxq[index] = rxq;
1110 esmp = &rxq->mem;
1111
1112 evq = sc->evq[index];
1113
1114 /* Allocate and zero DMA space. */
1110
1111 sc->rxq[index] = rxq;
1112 esmp = &rxq->mem;
1113
1114 evq = sc->evq[index];
1115
1116 /* Allocate and zero DMA space. */
1115 if ((rc = sfxge_dma_alloc(sc, EFX_RXQ_SIZE(SFXGE_NDESCS), esmp)) != 0)
1117 if ((rc = sfxge_dma_alloc(sc, EFX_RXQ_SIZE(sc->rxq_entries), esmp)) != 0)
1116 return (rc);
1118 return (rc);
1117 (void)memset(esmp->esm_base, 0, EFX_RXQ_SIZE(SFXGE_NDESCS));
1119 (void)memset(esmp->esm_base, 0, EFX_RXQ_SIZE(sc->rxq_entries));
1118
1119 /* Allocate buffer table entries. */
1120
1121 /* Allocate buffer table entries. */
1120 sfxge_sram_buf_tbl_alloc(sc, EFX_RXQ_NBUFS(SFXGE_NDESCS),
1122 sfxge_sram_buf_tbl_alloc(sc, EFX_RXQ_NBUFS(sc->rxq_entries),
1121 &rxq->buf_base_id);
1122
1123 /* Allocate the context array and the flow table. */
1123 &rxq->buf_base_id);
1124
1125 /* Allocate the context array and the flow table. */
1124 rxq->queue = malloc(sizeof(struct sfxge_rx_sw_desc) * SFXGE_NDESCS,
1126 rxq->queue = malloc(sizeof(struct sfxge_rx_sw_desc) * sc->rxq_entries,
1125 M_SFXGE, M_WAITOK | M_ZERO);
1126 sfxge_lro_init(rxq);
1127
1128 callout_init(&rxq->refill_callout, B_TRUE);
1129
1130 rxq->init_state = SFXGE_RXQ_INITIALIZED;
1131
1132 return (0);

--- 100 unchanged lines hidden --- 		1127 M_SFXGE, M_WAITOK | M_ZERO);
1128 sfxge_lro_init(rxq);
1129
1130 callout_init(&rxq->refill_callout, B_TRUE);
1131
1132 rxq->init_state = SFXGE_RXQ_INITIALIZED;
1133
1134 return (0);

--- 100 unchanged lines hidden ---