Deleted Added
sdiff udiff text old ( 269356 ) new ( 270297 )
full compact
t4_sge.c (269356) t4_sge.c (270297)
1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_sge.c 269356 2014-07-31 23:04:41Z np $");
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_sge.c 270297 2014-08-21 19:54:02Z np $");
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#include <sys/types.h>
35#include <sys/mbuf.h>
36#include <sys/socket.h>
37#include <sys/kernel.h>

--- 12 unchanged lines hidden (view full) ---

50#include <net/if_vlan_var.h>
51#include <netinet/in.h>
52#include <netinet/ip.h>
53#include <netinet/ip6.h>
54#include <netinet/tcp.h>
55#include <machine/md_var.h>
56#include <vm/vm.h>
57#include <vm/pmap.h>
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#include <sys/types.h>
35#include <sys/mbuf.h>
36#include <sys/socket.h>
37#include <sys/kernel.h>

--- 12 unchanged lines hidden (view full) ---

50#include <net/if_vlan_var.h>
51#include <netinet/in.h>
52#include <netinet/ip.h>
53#include <netinet/ip6.h>
54#include <netinet/tcp.h>
55#include <machine/md_var.h>
56#include <vm/vm.h>
57#include <vm/pmap.h>
58#ifdef DEV_NETMAP
59#include <machine/bus.h>
60#include <sys/selinfo.h>
61#include <net/if_var.h>
62#include <net/netmap.h>
63#include <dev/netmap/netmap_kern.h>
64#endif
58
59#include "common/common.h"
60#include "common/t4_regs.h"
61#include "common/t4_regs_values.h"
62#include "common/t4_msg.h"
63
64#ifdef T4_PKT_TIMESTAMP
65#define RX_COPY_THRESHOLD (MINCLSIZE - 8)
66#else
67#define RX_COPY_THRESHOLD MINCLSIZE
68#endif
69
70/*
71 * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
72 * 0-7 are valid values.
73 */
65
66#include "common/common.h"
67#include "common/t4_regs.h"
68#include "common/t4_regs_values.h"
69#include "common/t4_msg.h"
70
71#ifdef T4_PKT_TIMESTAMP
72#define RX_COPY_THRESHOLD (MINCLSIZE - 8)
73#else
74#define RX_COPY_THRESHOLD MINCLSIZE
75#endif
76
77/*
78 * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
79 * 0-7 are valid values.
80 */
74static int fl_pktshift = 2;
81int fl_pktshift = 2;
75TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift);
76
77/*
78 * Pad ethernet payload up to this boundary.
79 * -1: driver should figure out a good value.
80 * 0: disable padding.
81 * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
82 */
82TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift);
83
84/*
85 * Pad ethernet payload up to this boundary.
86 * -1: driver should figure out a good value.
87 * 0: disable padding.
88 * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
89 */
83static int fl_pad = -1;
90int fl_pad = -1;
84TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad);
85
86/*
87 * Status page length.
88 * -1: driver should figure out a good value.
89 * 64 or 128 are the only other valid values.
90 */
91TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad);
92
93/*
94 * Status page length.
95 * -1: driver should figure out a good value.
96 * 64 or 128 are the only other valid values.
97 */
91static int spg_len = -1;
98int spg_len = -1;
92TUNABLE_INT("hw.cxgbe.spg_len", &spg_len);
93
94/*
95 * Congestion drops.
96 * -1: no congestion feedback (not recommended).
97 * 0: backpressure the channel instead of dropping packets right away.
98 * 1: no backpressure, drop packets for the congested queue immediately.
99 */

--- 59 unchanged lines hidden (view full) ---

159/* A packet's SGL. This + m_pkthdr has all info needed for tx */
160struct sgl {
161 int nsegs; /* # of segments in the SGL, 0 means imm. tx */
162 int nflits; /* # of flits needed for the SGL */
163 bus_dma_segment_t seg[TX_SGL_SEGS];
164};
165
166static int service_iq(struct sge_iq *, int);
99TUNABLE_INT("hw.cxgbe.spg_len", &spg_len);
100
101/*
102 * Congestion drops.
103 * -1: no congestion feedback (not recommended).
104 * 0: backpressure the channel instead of dropping packets right away.
105 * 1: no backpressure, drop packets for the congested queue immediately.
106 */

--- 59 unchanged lines hidden (view full) ---

166/* A packet's SGL. This + m_pkthdr has all info needed for tx */
167struct sgl {
168 int nsegs; /* # of segments in the SGL, 0 means imm. tx */
169 int nflits; /* # of flits needed for the SGL */
170 bus_dma_segment_t seg[TX_SGL_SEGS];
171};
172
173static int service_iq(struct sge_iq *, int);
167static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t,
168 int *);
174static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
169static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
175static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
170static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
171 int);
176static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int);
172static inline void init_fl(struct adapter *, struct sge_fl *, int, int, int,
173 char *);
174static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t,
175 char *);
176static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *,
177 bus_addr_t *, void **);
178static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
179 void *);

--- 9 unchanged lines hidden (view full) ---

189static int alloc_rxq(struct port_info *, struct sge_rxq *, int, int,
190 struct sysctl_oid *);
191static int free_rxq(struct port_info *, struct sge_rxq *);
192#ifdef TCP_OFFLOAD
193static int alloc_ofld_rxq(struct port_info *, struct sge_ofld_rxq *, int, int,
194 struct sysctl_oid *);
195static int free_ofld_rxq(struct port_info *, struct sge_ofld_rxq *);
196#endif
177static inline void init_fl(struct adapter *, struct sge_fl *, int, int, int,
178 char *);
179static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t,
180 char *);
181static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *,
182 bus_addr_t *, void **);
183static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
184 void *);

--- 9 unchanged lines hidden (view full) ---

194static int alloc_rxq(struct port_info *, struct sge_rxq *, int, int,
195 struct sysctl_oid *);
196static int free_rxq(struct port_info *, struct sge_rxq *);
197#ifdef TCP_OFFLOAD
198static int alloc_ofld_rxq(struct port_info *, struct sge_ofld_rxq *, int, int,
199 struct sysctl_oid *);
200static int free_ofld_rxq(struct port_info *, struct sge_ofld_rxq *);
201#endif
202#ifdef DEV_NETMAP
203static int alloc_nm_rxq(struct port_info *, struct sge_nm_rxq *, int, int,
204 struct sysctl_oid *);
205static int free_nm_rxq(struct port_info *, struct sge_nm_rxq *);
206static int alloc_nm_txq(struct port_info *, struct sge_nm_txq *, int, int,
207 struct sysctl_oid *);
208static int free_nm_txq(struct port_info *, struct sge_nm_txq *);
209#endif
197static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
198static int eth_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
199#ifdef TCP_OFFLOAD
200static int ofld_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
201#endif
202static int alloc_eq(struct adapter *, struct port_info *, struct sge_eq *);
203static int free_eq(struct adapter *, struct sge_eq *);
204static int alloc_wrq(struct adapter *, struct port_info *, struct sge_wrq *,
205 struct sysctl_oid *);
206static int free_wrq(struct adapter *, struct sge_wrq *);
207static int alloc_txq(struct port_info *, struct sge_txq *, int,
208 struct sysctl_oid *);
209static int free_txq(struct port_info *, struct sge_txq *);
210static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
210static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
211static int eth_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
212#ifdef TCP_OFFLOAD
213static int ofld_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
214#endif
215static int alloc_eq(struct adapter *, struct port_info *, struct sge_eq *);
216static int free_eq(struct adapter *, struct sge_eq *);
217static int alloc_wrq(struct adapter *, struct port_info *, struct sge_wrq *,
218 struct sysctl_oid *);
219static int free_wrq(struct adapter *, struct sge_wrq *);
220static int alloc_txq(struct port_info *, struct sge_txq *, int,
221 struct sysctl_oid *);
222static int free_txq(struct port_info *, struct sge_txq *);
223static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
211static inline bool is_new_response(const struct sge_iq *, struct rsp_ctrl **);
212static inline void iq_next(struct sge_iq *);
213static inline void ring_fl_db(struct adapter *, struct sge_fl *);
214static int refill_fl(struct adapter *, struct sge_fl *, int);
215static void refill_sfl(void *);
216static int alloc_fl_sdesc(struct sge_fl *);
217static void free_fl_sdesc(struct adapter *, struct sge_fl *);
218static void find_best_refill_source(struct adapter *, struct sge_fl *, int);
219static void find_safe_refill_source(struct adapter *, struct sge_fl *);
220static void add_fl_to_sfl(struct adapter *, struct sge_fl *);

--- 610 unchanged lines hidden (view full) ---

831
832 free_mgmtq(sc);
833 free_fwq(sc);
834
835 return (0);
836}
837
838static inline int
224static inline void ring_fl_db(struct adapter *, struct sge_fl *);
225static int refill_fl(struct adapter *, struct sge_fl *, int);
226static void refill_sfl(void *);
227static int alloc_fl_sdesc(struct sge_fl *);
228static void free_fl_sdesc(struct adapter *, struct sge_fl *);
229static void find_best_refill_source(struct adapter *, struct sge_fl *, int);
230static void find_safe_refill_source(struct adapter *, struct sge_fl *);
231static void add_fl_to_sfl(struct adapter *, struct sge_fl *);

--- 610 unchanged lines hidden (view full) ---

842
843 free_mgmtq(sc);
844 free_fwq(sc);
845
846 return (0);
847}
848
849static inline int
850port_intr_count(struct port_info *pi)
851{
852 int rc = 0;
853
854 if (pi->flags & INTR_RXQ)
855 rc += pi->nrxq;
856#ifdef TCP_OFFLOAD
857 if (pi->flags & INTR_OFLD_RXQ)
858 rc += pi->nofldrxq;
859#endif
860#ifdef DEV_NETMAP
861 if (pi->flags & INTR_NM_RXQ)
862 rc += pi->nnmrxq;
863#endif
864 return (rc);
865}
866
867static inline int
839first_vector(struct port_info *pi)
840{
841 struct adapter *sc = pi->adapter;
842 int rc = T4_EXTRA_INTR, i;
843
844 if (sc->intr_count == 1)
845 return (0);
846
847 for_each_port(sc, i) {
868first_vector(struct port_info *pi)
869{
870 struct adapter *sc = pi->adapter;
871 int rc = T4_EXTRA_INTR, i;
872
873 if (sc->intr_count == 1)
874 return (0);
875
876 for_each_port(sc, i) {
848 struct port_info *p = sc->port[i];
849
850 if (i == pi->port_id)
851 break;
852
877 if (i == pi->port_id)
878 break;
879
853#ifdef TCP_OFFLOAD
854 if (sc->flags & INTR_DIRECT)
855 rc += p->nrxq + p->nofldrxq;
856 else
857 rc += max(p->nrxq, p->nofldrxq);
858#else
859 /*
860 * Not compiled with offload support and intr_count > 1. Only
861 * NIC queues exist and they'd better be taking direct
862 * interrupts.
863 */
864 KASSERT(sc->flags & INTR_DIRECT,
865 ("%s: intr_count %d, !INTR_DIRECT", __func__,
866 sc->intr_count));
867
868 rc += p->nrxq;
869#endif
880 rc += port_intr_count(sc->port[i]);
870 }
871
872 return (rc);
873}
874
875/*
876 * Given an arbitrary "index," come up with an iq that can be used by other
877 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
878 * The iq returned is guaranteed to be something that takes direct interrupts.
879 */
880static struct sge_iq *
881port_intr_iq(struct port_info *pi, int idx)
882{
883 struct adapter *sc = pi->adapter;
884 struct sge *s = &sc->sge;
885 struct sge_iq *iq = NULL;
881 }
882
883 return (rc);
884}
885
886/*
887 * Given an arbitrary "index," come up with an iq that can be used by other
888 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
889 * The iq returned is guaranteed to be something that takes direct interrupts.
890 */
891static struct sge_iq *
892port_intr_iq(struct port_info *pi, int idx)
893{
894 struct adapter *sc = pi->adapter;
895 struct sge *s = &sc->sge;
896 struct sge_iq *iq = NULL;
897 int nintr, i;
886
887 if (sc->intr_count == 1)
888 return (&sc->sge.fwq);
889
898
899 if (sc->intr_count == 1)
900 return (&sc->sge.fwq);
901
890#ifdef TCP_OFFLOAD
891 if (sc->flags & INTR_DIRECT) {
892 idx %= pi->nrxq + pi->nofldrxq;
902 nintr = port_intr_count(pi);
903 KASSERT(nintr != 0,
904 ("%s: pi %p has no exclusive interrupts, total interrupts = %d",
905 __func__, pi, sc->intr_count));
906#ifdef DEV_NETMAP
907 /* Exclude netmap queues as they can't take anyone else's interrupts */
908 if (pi->flags & INTR_NM_RXQ)
909 nintr -= pi->nnmrxq;
910 KASSERT(nintr > 0,
911 ("%s: pi %p has nintr %d after netmap adjustment of %d", __func__,
912 pi, nintr, pi->nnmrxq));
913#endif
914 i = idx % nintr;
893
915
894 if (idx >= pi->nrxq) {
895 idx -= pi->nrxq;
896 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
897 } else
898 iq = &s->rxq[pi->first_rxq + idx].iq;
899
900 } else {
901 idx %= max(pi->nrxq, pi->nofldrxq);
902
903 if (pi->nrxq >= pi->nofldrxq)
904 iq = &s->rxq[pi->first_rxq + idx].iq;
905 else
906 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
916 if (pi->flags & INTR_RXQ) {
917 if (i < pi->nrxq) {
918 iq = &s->rxq[pi->first_rxq + i].iq;
919 goto done;
920 }
921 i -= pi->nrxq;
907 }
922 }
908#else
909 /*
910 * Not compiled with offload support and intr_count > 1. Only NIC
911 * queues exist and they'd better be taking direct interrupts.
912 */
913 KASSERT(sc->flags & INTR_DIRECT,
914 ("%s: intr_count %d, !INTR_DIRECT", __func__, sc->intr_count));
915
916 idx %= pi->nrxq;
917 iq = &s->rxq[pi->first_rxq + idx].iq;
923#ifdef TCP_OFFLOAD
924 if (pi->flags & INTR_OFLD_RXQ) {
925 if (i < pi->nofldrxq) {
926 iq = &s->ofld_rxq[pi->first_ofld_rxq + i].iq;
927 goto done;
928 }
929 i -= pi->nofldrxq;
930 }
918#endif
931#endif
919
920 KASSERT(iq->flags & IQ_INTR, ("%s: EDOOFUS", __func__));
932 panic("%s: pi %p, intr_flags 0x%lx, idx %d, total intr %d\n", __func__,
933 pi, pi->flags & INTR_ALL, idx, nintr);
934done:
935 MPASS(iq != NULL);
936 KASSERT(iq->flags & IQ_INTR,
937 ("%s: iq %p (port %p, intr_flags 0x%lx, idx %d)", __func__, iq, pi,
938 pi->flags & INTR_ALL, idx));
921 return (iq);
922}
923
924/* Maximum payload that can be delivered with a single iq descriptor */
925static inline int
926mtu_to_max_payload(struct adapter *sc, int mtu, const int toe)
927{
928 int payload;

--- 20 unchanged lines hidden (view full) ---

949{
950 int rc = 0, i, j, intr_idx, iqid;
951 struct sge_rxq *rxq;
952 struct sge_txq *txq;
953 struct sge_wrq *ctrlq;
954#ifdef TCP_OFFLOAD
955 struct sge_ofld_rxq *ofld_rxq;
956 struct sge_wrq *ofld_txq;
939 return (iq);
940}
941
942/* Maximum payload that can be delivered with a single iq descriptor */
943static inline int
944mtu_to_max_payload(struct adapter *sc, int mtu, const int toe)
945{
946 int payload;

--- 20 unchanged lines hidden (view full) ---

967{
968 int rc = 0, i, j, intr_idx, iqid;
969 struct sge_rxq *rxq;
970 struct sge_txq *txq;
971 struct sge_wrq *ctrlq;
972#ifdef TCP_OFFLOAD
973 struct sge_ofld_rxq *ofld_rxq;
974 struct sge_wrq *ofld_txq;
957 struct sysctl_oid *oid2 = NULL;
958#endif
975#endif
976#ifdef DEV_NETMAP
977 struct sge_nm_rxq *nm_rxq;
978 struct sge_nm_txq *nm_txq;
979#endif
959 char name[16];
960 struct adapter *sc = pi->adapter;
961 struct ifnet *ifp = pi->ifp;
962 struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev);
963 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
964 int maxp, pack, mtu = ifp->if_mtu;
965
980 char name[16];
981 struct adapter *sc = pi->adapter;
982 struct ifnet *ifp = pi->ifp;
983 struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev);
984 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
985 int maxp, pack, mtu = ifp->if_mtu;
986
966 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD,
967 NULL, "rx queues");
968
969#ifdef TCP_OFFLOAD
970 if (is_offload(sc)) {
971 oid2 = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
972 CTLFLAG_RD, NULL,
973 "rx queues for offloaded TCP connections");
974 }
975#endif
976
977 /* Interrupt vector to start from (when using multiple vectors) */
978 intr_idx = first_vector(pi);
979
980 /*
987 /* Interrupt vector to start from (when using multiple vectors) */
988 intr_idx = first_vector(pi);
989
990 /*
981 * First pass over all rx queues (NIC and TOE):
991 * First pass over all NIC and TOE rx queues:
982 * a) initialize iq and fl
983 * b) allocate queue iff it will take direct interrupts.
984 */
985 maxp = mtu_to_max_payload(sc, mtu, 0);
986 pack = enable_buffer_packing(sc);
992 * a) initialize iq and fl
993 * b) allocate queue iff it will take direct interrupts.
994 */
995 maxp = mtu_to_max_payload(sc, mtu, 0);
996 pack = enable_buffer_packing(sc);
997 if (pi->flags & INTR_RXQ) {
998 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq",
999 CTLFLAG_RD, NULL, "rx queues");
1000 }
987 for_each_rxq(pi, i, rxq) {
988
1001 for_each_rxq(pi, i, rxq) {
1002
989 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq,
990 RX_IQ_ESIZE);
1003 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq);
991
992 snprintf(name, sizeof(name), "%s rxq%d-fl",
993 device_get_nameunit(pi->dev), i);
994 init_fl(sc, &rxq->fl, pi->qsize_rxq / 8, maxp, pack, name);
995
1004
1005 snprintf(name, sizeof(name), "%s rxq%d-fl",
1006 device_get_nameunit(pi->dev), i);
1007 init_fl(sc, &rxq->fl, pi->qsize_rxq / 8, maxp, pack, name);
1008
996 if (sc->flags & INTR_DIRECT
997#ifdef TCP_OFFLOAD
998 || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
999#endif
1000 ) {
1009 if (pi->flags & INTR_RXQ) {
1001 rxq->iq.flags |= IQ_INTR;
1002 rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
1003 if (rc != 0)
1004 goto done;
1005 intr_idx++;
1006 }
1007 }
1010 rxq->iq.flags |= IQ_INTR;
1011 rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
1012 if (rc != 0)
1013 goto done;
1014 intr_idx++;
1015 }
1016 }
1008
1009#ifdef TCP_OFFLOAD
1010 maxp = mtu_to_max_payload(sc, mtu, 1);
1017#ifdef TCP_OFFLOAD
1018 maxp = mtu_to_max_payload(sc, mtu, 1);
1019 if (is_offload(sc) && pi->flags & INTR_OFLD_RXQ) {
1020 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
1021 CTLFLAG_RD, NULL,
1022 "rx queues for offloaded TCP connections");
1023 }
1011 for_each_ofld_rxq(pi, i, ofld_rxq) {
1012
1013 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
1024 for_each_ofld_rxq(pi, i, ofld_rxq) {
1025
1026 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
1014 pi->qsize_rxq, RX_IQ_ESIZE);
1027 pi->qsize_rxq);
1015
1016 snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
1017 device_get_nameunit(pi->dev), i);
1018 init_fl(sc, &ofld_rxq->fl, pi->qsize_rxq / 8, maxp, pack, name);
1019
1028
1029 snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
1030 device_get_nameunit(pi->dev), i);
1031 init_fl(sc, &ofld_rxq->fl, pi->qsize_rxq / 8, maxp, pack, name);
1032
1020 if (sc->flags & INTR_DIRECT ||
1021 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
1033 if (pi->flags & INTR_OFLD_RXQ) {
1022 ofld_rxq->iq.flags |= IQ_INTR;
1034 ofld_rxq->iq.flags |= IQ_INTR;
1023 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
1035 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid);
1024 if (rc != 0)
1025 goto done;
1026 intr_idx++;
1027 }
1028 }
1029#endif
1036 if (rc != 0)
1037 goto done;
1038 intr_idx++;
1039 }
1040 }
1041#endif
1042#ifdef DEV_NETMAP
1043 /*
1044 * We don't have buffers to back the netmap rx queues right now so we
1045 * create the queues in a way that doesn't set off any congestion signal
1046 * in the chip.
1047 */
1048 if (pi->flags & INTR_NM_RXQ) {
1049 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "nm_rxq",
1050 CTLFLAG_RD, NULL, "rx queues for netmap");
1051 for_each_nm_rxq(pi, i, nm_rxq) {
1052 rc = alloc_nm_rxq(pi, nm_rxq, intr_idx, i, oid);
1053 if (rc != 0)
1054 goto done;
1055 intr_idx++;
1056 }
1057 }
1058#endif
1030
1031 /*
1059
1060 /*
1032 * Second pass over all rx queues (NIC and TOE). The queues forwarding
1061 * Second pass over all NIC and TOE rx queues. The queues forwarding
1033 * their interrupts are allocated now.
1034 */
1035 j = 0;
1062 * their interrupts are allocated now.
1063 */
1064 j = 0;
1036 for_each_rxq(pi, i, rxq) {
1037 if (rxq->iq.flags & IQ_INTR)
1038 continue;
1065 if (!(pi->flags & INTR_RXQ)) {
1066 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq",
1067 CTLFLAG_RD, NULL, "rx queues");
1068 for_each_rxq(pi, i, rxq) {
1069 MPASS(!(rxq->iq.flags & IQ_INTR));
1039
1070
1040 intr_idx = port_intr_iq(pi, j)->abs_id;
1071 intr_idx = port_intr_iq(pi, j)->abs_id;
1041
1072
1042 rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
1043 if (rc != 0)
1044 goto done;
1045 j++;
1073 rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
1074 if (rc != 0)
1075 goto done;
1076 j++;
1077 }
1046 }
1078 }
1047
1048#ifdef TCP_OFFLOAD
1079#ifdef TCP_OFFLOAD
1049 for_each_ofld_rxq(pi, i, ofld_rxq) {
1050 if (ofld_rxq->iq.flags & IQ_INTR)
1051 continue;
1080 if (is_offload(sc) && !(pi->flags & INTR_OFLD_RXQ)) {
1081 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
1082 CTLFLAG_RD, NULL,
1083 "rx queues for offloaded TCP connections");
1084 for_each_ofld_rxq(pi, i, ofld_rxq) {
1085 MPASS(!(ofld_rxq->iq.flags & IQ_INTR));
1052
1086
1053 intr_idx = port_intr_iq(pi, j)->abs_id;
1087 intr_idx = port_intr_iq(pi, j)->abs_id;
1054
1088
1055 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
1056 if (rc != 0)
1057 goto done;
1058 j++;
1089 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid);
1090 if (rc != 0)
1091 goto done;
1092 j++;
1093 }
1059 }
1060#endif
1094 }
1095#endif
1096#ifdef DEV_NETMAP
1097 if (!(pi->flags & INTR_NM_RXQ))
1098 CXGBE_UNIMPLEMENTED(__func__);
1099#endif
1061
1062 /*
1063 * Now the tx queues. Only one pass needed.
1064 */
1065 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD,
1066 NULL, "tx queues");
1067 j = 0;
1068 for_each_txq(pi, i, txq) {
1100
1101 /*
1102 * Now the tx queues. Only one pass needed.
1103 */
1104 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD,
1105 NULL, "tx queues");
1106 j = 0;
1107 for_each_txq(pi, i, txq) {
1069 uint16_t iqid;
1070
1071 iqid = port_intr_iq(pi, j)->cntxt_id;
1108 iqid = port_intr_iq(pi, j)->cntxt_id;
1072
1073 snprintf(name, sizeof(name), "%s txq%d",
1074 device_get_nameunit(pi->dev), i);
1075 init_eq(&txq->eq, EQ_ETH, pi->qsize_txq, pi->tx_chan, iqid,
1076 name);
1077
1078 rc = alloc_txq(pi, txq, i, oid);
1079 if (rc != 0)
1080 goto done;
1081 j++;
1082 }
1109 snprintf(name, sizeof(name), "%s txq%d",
1110 device_get_nameunit(pi->dev), i);
1111 init_eq(&txq->eq, EQ_ETH, pi->qsize_txq, pi->tx_chan, iqid,
1112 name);
1113
1114 rc = alloc_txq(pi, txq, i, oid);
1115 if (rc != 0)
1116 goto done;
1117 j++;
1118 }
1083
1084#ifdef TCP_OFFLOAD
1085 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_txq",
1086 CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections");
1087 for_each_ofld_txq(pi, i, ofld_txq) {
1119#ifdef TCP_OFFLOAD
1120 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_txq",
1121 CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections");
1122 for_each_ofld_txq(pi, i, ofld_txq) {
1088 uint16_t iqid;
1123 struct sysctl_oid *oid2;
1089
1090 iqid = port_intr_iq(pi, j)->cntxt_id;
1124
1125 iqid = port_intr_iq(pi, j)->cntxt_id;
1091
1092 snprintf(name, sizeof(name), "%s ofld_txq%d",
1093 device_get_nameunit(pi->dev), i);
1094 init_eq(&ofld_txq->eq, EQ_OFLD, pi->qsize_txq, pi->tx_chan,
1095 iqid, name);
1096
1097 snprintf(name, sizeof(name), "%d", i);
1098 oid2 = SYSCTL_ADD_NODE(&pi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
1099 name, CTLFLAG_RD, NULL, "offload tx queue");
1100
1101 rc = alloc_wrq(sc, pi, ofld_txq, oid2);
1102 if (rc != 0)
1103 goto done;
1104 j++;
1105 }
1106#endif
1126 snprintf(name, sizeof(name), "%s ofld_txq%d",
1127 device_get_nameunit(pi->dev), i);
1128 init_eq(&ofld_txq->eq, EQ_OFLD, pi->qsize_txq, pi->tx_chan,
1129 iqid, name);
1130
1131 snprintf(name, sizeof(name), "%d", i);
1132 oid2 = SYSCTL_ADD_NODE(&pi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
1133 name, CTLFLAG_RD, NULL, "offload tx queue");
1134
1135 rc = alloc_wrq(sc, pi, ofld_txq, oid2);
1136 if (rc != 0)
1137 goto done;
1138 j++;
1139 }
1140#endif
1141#ifdef DEV_NETMAP
1142 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "nm_txq",
1143 CTLFLAG_RD, NULL, "tx queues for netmap use");
1144 for_each_nm_txq(pi, i, nm_txq) {
1145 iqid = pi->first_nm_rxq + (j % pi->nnmrxq);
1146 rc = alloc_nm_txq(pi, nm_txq, iqid, i, oid);
1147 if (rc != 0)
1148 goto done;
1149 j++;
1150 }
1151#endif
1107
1108 /*
1109 * Finally, the control queue.
1110 */
1111 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD,
1112 NULL, "ctrl queue");
1113 ctrlq = &sc->sge.ctrlq[pi->port_id];
1114 iqid = port_intr_iq(pi, 0)->cntxt_id;

--- 17 unchanged lines hidden (view full) ---

1132 int i;
1133 struct adapter *sc = pi->adapter;
1134 struct sge_rxq *rxq;
1135 struct sge_txq *txq;
1136#ifdef TCP_OFFLOAD
1137 struct sge_ofld_rxq *ofld_rxq;
1138 struct sge_wrq *ofld_txq;
1139#endif
1152
1153 /*
1154 * Finally, the control queue.
1155 */
1156 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD,
1157 NULL, "ctrl queue");
1158 ctrlq = &sc->sge.ctrlq[pi->port_id];
1159 iqid = port_intr_iq(pi, 0)->cntxt_id;

--- 17 unchanged lines hidden (view full) ---

1177 int i;
1178 struct adapter *sc = pi->adapter;
1179 struct sge_rxq *rxq;
1180 struct sge_txq *txq;
1181#ifdef TCP_OFFLOAD
1182 struct sge_ofld_rxq *ofld_rxq;
1183 struct sge_wrq *ofld_txq;
1184#endif
1185#ifdef DEV_NETMAP
1186 struct sge_nm_rxq *nm_rxq;
1187 struct sge_nm_txq *nm_txq;
1188#endif
1140
1141 /* Do this before freeing the queues */
1142 if (pi->flags & PORT_SYSCTL_CTX) {
1143 sysctl_ctx_free(&pi->ctx);
1144 pi->flags &= ~PORT_SYSCTL_CTX;
1145 }
1146
1147 /*
1148 * Take down all the tx queues first, as they reference the rx queues
1149 * (for egress updates, etc.).
1150 */
1151
1152 free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
1153
1154 for_each_txq(pi, i, txq) {
1155 free_txq(pi, txq);
1156 }
1189
1190 /* Do this before freeing the queues */
1191 if (pi->flags & PORT_SYSCTL_CTX) {
1192 sysctl_ctx_free(&pi->ctx);
1193 pi->flags &= ~PORT_SYSCTL_CTX;
1194 }
1195
1196 /*
1197 * Take down all the tx queues first, as they reference the rx queues
1198 * (for egress updates, etc.).
1199 */
1200
1201 free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
1202
1203 for_each_txq(pi, i, txq) {
1204 free_txq(pi, txq);
1205 }
1157
1158#ifdef TCP_OFFLOAD
1159 for_each_ofld_txq(pi, i, ofld_txq) {
1160 free_wrq(sc, ofld_txq);
1161 }
1162#endif
1206#ifdef TCP_OFFLOAD
1207 for_each_ofld_txq(pi, i, ofld_txq) {
1208 free_wrq(sc, ofld_txq);
1209 }
1210#endif
1211#ifdef DEV_NETMAP
1212 for_each_nm_txq(pi, i, nm_txq)
1213 free_nm_txq(pi, nm_txq);
1214#endif
1163
1164 /*
1165 * Then take down the rx queues that forward their interrupts, as they
1166 * reference other rx queues.
1167 */
1168
1169 for_each_rxq(pi, i, rxq) {
1170 if ((rxq->iq.flags & IQ_INTR) == 0)
1171 free_rxq(pi, rxq);
1172 }
1215
1216 /*
1217 * Then take down the rx queues that forward their interrupts, as they
1218 * reference other rx queues.
1219 */
1220
1221 for_each_rxq(pi, i, rxq) {
1222 if ((rxq->iq.flags & IQ_INTR) == 0)
1223 free_rxq(pi, rxq);
1224 }
1173
1174#ifdef TCP_OFFLOAD
1175 for_each_ofld_rxq(pi, i, ofld_rxq) {
1176 if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
1177 free_ofld_rxq(pi, ofld_rxq);
1178 }
1179#endif
1225#ifdef TCP_OFFLOAD
1226 for_each_ofld_rxq(pi, i, ofld_rxq) {
1227 if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
1228 free_ofld_rxq(pi, ofld_rxq);
1229 }
1230#endif
1231#ifdef DEV_NETMAP
1232 for_each_nm_rxq(pi, i, nm_rxq)
1233 free_nm_rxq(pi, nm_rxq);
1234#endif
1180
1181 /*
1182 * Then take down the rx queues that take direct interrupts.
1183 */
1184
1185 for_each_rxq(pi, i, rxq) {
1186 if (rxq->iq.flags & IQ_INTR)
1187 free_rxq(pi, rxq);
1188 }
1235
1236 /*
1237 * Then take down the rx queues that take direct interrupts.
1238 */
1239
1240 for_each_rxq(pi, i, rxq) {
1241 if (rxq->iq.flags & IQ_INTR)
1242 free_rxq(pi, rxq);
1243 }
1189
1190#ifdef TCP_OFFLOAD
1191 for_each_ofld_rxq(pi, i, ofld_rxq) {
1192 if (ofld_rxq->iq.flags & IQ_INTR)
1193 free_ofld_rxq(pi, ofld_rxq);
1194 }
1195#endif
1244#ifdef TCP_OFFLOAD
1245 for_each_ofld_rxq(pi, i, ofld_rxq) {
1246 if (ofld_rxq->iq.flags & IQ_INTR)
1247 free_ofld_rxq(pi, ofld_rxq);
1248 }
1249#endif
1250#ifdef DEV_NETMAP
1251 CXGBE_UNIMPLEMENTED(__func__);
1252#endif
1196
1197 return (0);
1198}
1199
1200/*
1201 * Deals with errors and the firmware event queue. All data rx queues forward
1202 * their interrupt to the firmware event queue.
1203 */

--- 45 unchanged lines hidden (view full) ---

1249/*
1250 * Deals with anything and everything on the given ingress queue.
1251 */
1252static int
1253service_iq(struct sge_iq *iq, int budget)
1254{
1255 struct sge_iq *q;
1256 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
1253
1254 return (0);
1255}
1256
1257/*
1258 * Deals with errors and the firmware event queue. All data rx queues forward
1259 * their interrupt to the firmware event queue.
1260 */

--- 45 unchanged lines hidden (view full) ---

1306/*
1307 * Deals with anything and everything on the given ingress queue.
1308 */
1309static int
1310service_iq(struct sge_iq *iq, int budget)
1311{
1312 struct sge_iq *q;
1313 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
1257 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
1314 struct sge_fl *fl; /* Use iff IQ_HAS_FL */
1258 struct adapter *sc = iq->adapter;
1315 struct adapter *sc = iq->adapter;
1259 struct rsp_ctrl *ctrl;
1260 const struct rss_header *rss;
1261 int ndescs = 0, limit, fl_bufs_used = 0;
1262 int rsp_type;
1316 struct iq_desc *d = &iq->desc[iq->cidx];
1317 int ndescs = 0, limit;
1318 int rsp_type, refill;
1263 uint32_t lq;
1319 uint32_t lq;
1320 uint16_t fl_hw_cidx;
1264 struct mbuf *m0;
1265 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
1266#if defined(INET) || defined(INET6)
1267 const struct timeval lro_timeout = {0, sc->lro_timeout};
1268#endif
1269
1321 struct mbuf *m0;
1322 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
1323#if defined(INET) || defined(INET6)
1324 const struct timeval lro_timeout = {0, sc->lro_timeout};
1325#endif
1326
1270 limit = budget ? budget : iq->qsize / 8;
1271
1272 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
1273
1327 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
1328
1329 limit = budget ? budget : iq->qsize / 16;
1330
1331 if (iq->flags & IQ_HAS_FL) {
1332 fl = &rxq->fl;
1333 fl_hw_cidx = fl->hw_cidx; /* stable snapshot */
1334 } else {
1335 fl = NULL;
1336 fl_hw_cidx = 0; /* to silence gcc warning */
1337 }
1338
1274 /*
1275 * We always come back and check the descriptor ring for new indirect
1276 * interrupts and other responses after running a single handler.
1277 */
1278 for (;;) {
1339 /*
1340 * We always come back and check the descriptor ring for new indirect
1341 * interrupts and other responses after running a single handler.
1342 */
1343 for (;;) {
1279 while (is_new_response(iq, &ctrl)) {
1344 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
1280
1281 rmb();
1282
1345
1346 rmb();
1347
1348 refill = 0;
1283 m0 = NULL;
1349 m0 = NULL;
1284 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
1285 lq = be32toh(ctrl->pldbuflen_qid);
1286 rss = (const void *)iq->cdesc;
1350 rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
1351 lq = be32toh(d->rsp.pldbuflen_qid);
1287
1288 switch (rsp_type) {
1289 case X_RSPD_TYPE_FLBUF:
1290
1291 KASSERT(iq->flags & IQ_HAS_FL,
1292 ("%s: data for an iq (%p) with no freelist",
1293 __func__, iq));
1294
1352
1353 switch (rsp_type) {
1354 case X_RSPD_TYPE_FLBUF:
1355
1356 KASSERT(iq->flags & IQ_HAS_FL,
1357 ("%s: data for an iq (%p) with no freelist",
1358 __func__, iq));
1359
1295 m0 = get_fl_payload(sc, fl, lq, &fl_bufs_used);
1360 m0 = get_fl_payload(sc, fl, lq);
1296 if (__predict_false(m0 == NULL))
1297 goto process_iql;
1361 if (__predict_false(m0 == NULL))
1362 goto process_iql;
1363 refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
1298#ifdef T4_PKT_TIMESTAMP
1299 /*
1300 * 60 bit timestamp for the payload is
1301 * *(uint64_t *)m0->m_pktdat. Note that it is
1302 * in the leading free-space in the mbuf. The
1303 * kernel can clobber it during a pullup,
1304 * m_copymdata, etc. You need to make sure that
1305 * the mbuf reaches you unmolested if you care
1306 * about the timestamp.
1307 */
1308 *(uint64_t *)m0->m_pktdat =
1309 be64toh(ctrl->u.last_flit) &
1310 0xfffffffffffffff;
1311#endif
1312
1313 /* fall through */
1314
1315 case X_RSPD_TYPE_CPL:
1364#ifdef T4_PKT_TIMESTAMP
1365 /*
1366 * 60 bit timestamp for the payload is
1367 * *(uint64_t *)m0->m_pktdat. Note that it is
1368 * in the leading free-space in the mbuf. The
1369 * kernel can clobber it during a pullup,
1370 * m_copymdata, etc. You need to make sure that
1371 * the mbuf reaches you unmolested if you care
1372 * about the timestamp.
1373 */
1374 *(uint64_t *)m0->m_pktdat =
1375 be64toh(ctrl->u.last_flit) &
1376 0xfffffffffffffff;
1377#endif
1378
1379 /* fall through */
1380
1381 case X_RSPD_TYPE_CPL:
1316 KASSERT(rss->opcode < NUM_CPL_CMDS,
1382 KASSERT(d->rss.opcode < NUM_CPL_CMDS,
1317 ("%s: bad opcode %02x.", __func__,
1383 ("%s: bad opcode %02x.", __func__,
1318 rss->opcode));
1319 sc->cpl_handler[rss->opcode](iq, rss, m0);
1384 d->rss.opcode));
1385 sc->cpl_handler[d->rss.opcode](iq, &d->rss, m0);
1320 break;
1321
1322 case X_RSPD_TYPE_INTR:
1323
1324 /*
1325 * Interrupts should be forwarded only to queues
1326 * that are not forwarding their interrupts.
1327 * This means service_iq can recurse but only 1

--- 5 unchanged lines hidden (view full) ---

1333
1334 /*
1335 * There are 1K interrupt-capable queues (qids 0
1336 * through 1023). A response type indicating a
1337 * forwarded interrupt with a qid >= 1K is an
1338 * iWARP async notification.
1339 */
1340 if (lq >= 1024) {
1386 break;
1387
1388 case X_RSPD_TYPE_INTR:
1389
1390 /*
1391 * Interrupts should be forwarded only to queues
1392 * that are not forwarding their interrupts.
1393 * This means service_iq can recurse but only 1

--- 5 unchanged lines hidden (view full) ---

1399
1400 /*
1401 * There are 1K interrupt-capable queues (qids 0
1402 * through 1023). A response type indicating a
1403 * forwarded interrupt with a qid >= 1K is an
1404 * iWARP async notification.
1405 */
1406 if (lq >= 1024) {
1341 sc->an_handler(iq, ctrl);
1407 sc->an_handler(iq, &d->rsp);
1342 break;
1343 }
1344
1345 q = sc->sge.iqmap[lq - sc->sge.iq_start];
1346 if (atomic_cmpset_int(&q->state, IQS_IDLE,
1347 IQS_BUSY)) {
1408 break;
1409 }
1410
1411 q = sc->sge.iqmap[lq - sc->sge.iq_start];
1412 if (atomic_cmpset_int(&q->state, IQS_IDLE,
1413 IQS_BUSY)) {
1348 if (service_iq(q, q->qsize / 8) == 0) {
1414 if (service_iq(q, q->qsize / 16) == 0) {
1349 atomic_cmpset_int(&q->state,
1350 IQS_BUSY, IQS_IDLE);
1351 } else {
1352 STAILQ_INSERT_TAIL(&iql, q,
1353 link);
1354 }
1355 }
1356 break;
1357
1358 default:
1359 KASSERT(0,
1360 ("%s: illegal response type %d on iq %p",
1361 __func__, rsp_type, iq));
1362 log(LOG_ERR,
1363 "%s: illegal response type %d on iq %p",
1364 device_get_nameunit(sc->dev), rsp_type, iq);
1365 break;
1366 }
1367
1415 atomic_cmpset_int(&q->state,
1416 IQS_BUSY, IQS_IDLE);
1417 } else {
1418 STAILQ_INSERT_TAIL(&iql, q,
1419 link);
1420 }
1421 }
1422 break;
1423
1424 default:
1425 KASSERT(0,
1426 ("%s: illegal response type %d on iq %p",
1427 __func__, rsp_type, iq));
1428 log(LOG_ERR,
1429 "%s: illegal response type %d on iq %p",
1430 device_get_nameunit(sc->dev), rsp_type, iq);
1431 break;
1432 }
1433
1368 if (fl_bufs_used >= 16) {
1369 FL_LOCK(fl);
1370 fl->needed += fl_bufs_used;
1371 refill_fl(sc, fl, 32);
1372 FL_UNLOCK(fl);
1373 fl_bufs_used = 0;
1434 d++;
1435 if (__predict_false(++iq->cidx == iq->sidx)) {
1436 iq->cidx = 0;
1437 iq->gen ^= F_RSPD_GEN;
1438 d = &iq->desc[0];
1374 }
1439 }
1375
1376 iq_next(iq);
1377 if (++ndescs == limit) {
1440 if (__predict_false(++ndescs == limit)) {
1378 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
1379 V_CIDXINC(ndescs) |
1380 V_INGRESSQID(iq->cntxt_id) |
1381 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1382 ndescs = 0;
1383
1384#if defined(INET) || defined(INET6)
1385 if (iq->flags & IQ_LRO_ENABLED &&
1386 sc->lro_timeout != 0) {
1387 tcp_lro_flush_inactive(&rxq->lro,
1388 &lro_timeout);
1389 }
1390#endif
1391
1392 if (budget) {
1441 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
1442 V_CIDXINC(ndescs) |
1443 V_INGRESSQID(iq->cntxt_id) |
1444 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1445 ndescs = 0;
1446
1447#if defined(INET) || defined(INET6)
1448 if (iq->flags & IQ_LRO_ENABLED &&
1449 sc->lro_timeout != 0) {
1450 tcp_lro_flush_inactive(&rxq->lro,
1451 &lro_timeout);
1452 }
1453#endif
1454
1455 if (budget) {
1393 if (fl_bufs_used) {
1456 if (iq->flags & IQ_HAS_FL) {
1394 FL_LOCK(fl);
1457 FL_LOCK(fl);
1395 fl->needed += fl_bufs_used;
1396 refill_fl(sc, fl, 32);
1397 FL_UNLOCK(fl);
1398 }
1399 return (EINPROGRESS);
1400 }
1401 }
1458 refill_fl(sc, fl, 32);
1459 FL_UNLOCK(fl);
1460 }
1461 return (EINPROGRESS);
1462 }
1463 }
1464 if (refill) {
1465 FL_LOCK(fl);
1466 refill_fl(sc, fl, 32);
1467 FL_UNLOCK(fl);
1468 fl_hw_cidx = fl->hw_cidx;
1469 }
1402 }
1403
1404process_iql:
1405 if (STAILQ_EMPTY(&iql))
1406 break;
1407
1408 /*
1409 * Process the head only, and send it to the back of the list if

--- 22 unchanged lines hidden (view full) ---

1432
1433 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
1434 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1435
1436 if (iq->flags & IQ_HAS_FL) {
1437 int starved;
1438
1439 FL_LOCK(fl);
1470 }
1471
1472process_iql:
1473 if (STAILQ_EMPTY(&iql))
1474 break;
1475
1476 /*
1477 * Process the head only, and send it to the back of the list if

--- 22 unchanged lines hidden (view full) ---

1500
1501 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
1502 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1503
1504 if (iq->flags & IQ_HAS_FL) {
1505 int starved;
1506
1507 FL_LOCK(fl);
1440 fl->needed += fl_bufs_used;
1441 starved = refill_fl(sc, fl, 64);
1442 FL_UNLOCK(fl);
1443 if (__predict_false(starved != 0))
1444 add_fl_to_sfl(sc, fl);
1445 }
1446
1447 return (0);
1448}

--- 52 unchanged lines hidden (view full) ---

1501 struct cluster_layout *cll = &sd->cll;
1502 struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx];
1503 struct hw_buf_info *hwb = &sc->sge.hw_buf_info[cll->hwidx];
1504 struct cluster_metadata *clm = cl_metadata(sc, fl, cll, sd->cl);
1505 int len, padded_len;
1506 caddr_t payload;
1507
1508 len = min(total, hwb->size - fl->rx_offset);
1508 starved = refill_fl(sc, fl, 64);
1509 FL_UNLOCK(fl);
1510 if (__predict_false(starved != 0))
1511 add_fl_to_sfl(sc, fl);
1512 }
1513
1514 return (0);
1515}

--- 52 unchanged lines hidden (view full) ---

1568 struct cluster_layout *cll = &sd->cll;
1569 struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx];
1570 struct hw_buf_info *hwb = &sc->sge.hw_buf_info[cll->hwidx];
1571 struct cluster_metadata *clm = cl_metadata(sc, fl, cll, sd->cl);
1572 int len, padded_len;
1573 caddr_t payload;
1574
1575 len = min(total, hwb->size - fl->rx_offset);
1509 padded_len = roundup2(len, fl_pad);
1576 padded_len = roundup2(len, fl->buf_boundary);
1510 payload = sd->cl + cll->region1 + fl->rx_offset;
1511
1512 if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) {
1513
1514 /*
1515 * Copy payload into a freshly allocated mbuf.
1516 */
1517

--- 49 unchanged lines hidden (view full) ---

1567 sd->cl = NULL; /* consumed, not a recycle candidate */
1568 }
1569 }
1570 if (flags & M_PKTHDR)
1571 m->m_pkthdr.len = total;
1572 m->m_len = len;
1573
1574 if (fl->flags & FL_BUF_PACKING) {
1577 payload = sd->cl + cll->region1 + fl->rx_offset;
1578
1579 if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) {
1580
1581 /*
1582 * Copy payload into a freshly allocated mbuf.
1583 */
1584

--- 49 unchanged lines hidden (view full) ---

1634 sd->cl = NULL; /* consumed, not a recycle candidate */
1635 }
1636 }
1637 if (flags & M_PKTHDR)
1638 m->m_pkthdr.len = total;
1639 m->m_len = len;
1640
1641 if (fl->flags & FL_BUF_PACKING) {
1575 fl->rx_offset += roundup2(padded_len, sc->sge.pack_boundary);
1642 fl->rx_offset += padded_len;
1576 MPASS(fl->rx_offset <= hwb->size);
1577 if (fl->rx_offset < hwb->size)
1578 return (m); /* without advancing the cidx */
1579 }
1580
1643 MPASS(fl->rx_offset <= hwb->size);
1644 if (fl->rx_offset < hwb->size)
1645 return (m); /* without advancing the cidx */
1646 }
1647
1581 if (__predict_false(++fl->cidx == fl->cap))
1582 fl->cidx = 0;
1648 if (__predict_false(++fl->cidx % 8 == 0)) {
1649 uint16_t cidx = fl->cidx / 8;
1650
1651 if (__predict_false(cidx == fl->sidx))
1652 fl->cidx = cidx = 0;
1653 fl->hw_cidx = cidx;
1654 }
1583 fl->rx_offset = 0;
1584
1585 return (m);
1586}
1587
1588static struct mbuf *
1655 fl->rx_offset = 0;
1656
1657 return (m);
1658}
1659
1660static struct mbuf *
1589get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
1590 int *fl_bufs_used)
1661get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf)
1591{
1592 struct mbuf *m0, *m, **pnext;
1662{
1663 struct mbuf *m0, *m, **pnext;
1593 u_int nbuf, len;
1664 u_int len;
1594
1665
1595 /*
1596 * No assertion for the fl lock because we don't need it. This routine
1597 * is called only from the rx interrupt handler and it only updates
1598 * fl->cidx. (Contrast that with fl->pidx/fl->needed which could be
1599 * updated in the rx interrupt handler or the starvation helper routine.
1600 * That's why code that manipulates fl->pidx/fl->needed needs the fl
1601 * lock but this routine does not).
1602 */
1603
1604 nbuf = 0;
1605 len = G_RSPD_LEN(len_newbuf);
1666 len = G_RSPD_LEN(len_newbuf);
1606 if (__predict_false(fl->m0 != NULL)) {
1667 if (__predict_false(fl->flags & FL_BUF_RESUME)) {
1607 M_ASSERTPKTHDR(fl->m0);
1608 MPASS(len == fl->m0->m_pkthdr.len);
1609 MPASS(fl->remaining < len);
1610
1611 m0 = fl->m0;
1612 pnext = fl->pnext;
1613 len = fl->remaining;
1668 M_ASSERTPKTHDR(fl->m0);
1669 MPASS(len == fl->m0->m_pkthdr.len);
1670 MPASS(fl->remaining < len);
1671
1672 m0 = fl->m0;
1673 pnext = fl->pnext;
1674 len = fl->remaining;
1614 fl->m0 = NULL;
1675 fl->flags &= ~FL_BUF_RESUME;
1615 goto get_segment;
1616 }
1617
1618 if (fl->rx_offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
1676 goto get_segment;
1677 }
1678
1679 if (fl->rx_offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
1619 nbuf++;
1620 fl->rx_offset = 0;
1680 fl->rx_offset = 0;
1621 if (__predict_false(++fl->cidx == fl->cap))
1622 fl->cidx = 0;
1681 if (__predict_false(++fl->cidx % 8 == 0)) {
1682 uint16_t cidx = fl->cidx / 8;
1683
1684 if (__predict_false(cidx == fl->sidx))
1685 fl->cidx = cidx = 0;
1686 fl->hw_cidx = cidx;
1687 }
1623 }
1624
1625 /*
1626 * Payload starts at rx_offset in the current hw buffer. Its length is
1627 * 'len' and it may span multiple hw buffers.
1628 */
1629
1630 m0 = get_scatter_segment(sc, fl, len, M_PKTHDR);
1631 if (m0 == NULL)
1688 }
1689
1690 /*
1691 * Payload starts at rx_offset in the current hw buffer. Its length is
1692 * 'len' and it may span multiple hw buffers.
1693 */
1694
1695 m0 = get_scatter_segment(sc, fl, len, M_PKTHDR);
1696 if (m0 == NULL)
1632 goto done;
1697 return (NULL);
1633 len -= m0->m_len;
1634 pnext = &m0->m_next;
1635 while (len > 0) {
1698 len -= m0->m_len;
1699 pnext = &m0->m_next;
1700 while (len > 0) {
1636 nbuf++;
1637get_segment:
1638 MPASS(fl->rx_offset == 0);
1639 m = get_scatter_segment(sc, fl, len, 0);
1701get_segment:
1702 MPASS(fl->rx_offset == 0);
1703 m = get_scatter_segment(sc, fl, len, 0);
1640 if (m == NULL) {
1704 if (__predict_false(m == NULL)) {
1641 fl->m0 = m0;
1642 fl->pnext = pnext;
1643 fl->remaining = len;
1705 fl->m0 = m0;
1706 fl->pnext = pnext;
1707 fl->remaining = len;
1644 m0 = NULL;
1645 goto done;
1708 fl->flags |= FL_BUF_RESUME;
1709 return (NULL);
1646 }
1647 *pnext = m;
1648 pnext = &m->m_next;
1649 len -= m->m_len;
1650 }
1651 *pnext = NULL;
1710 }
1711 *pnext = m;
1712 pnext = &m->m_next;
1713 len -= m->m_len;
1714 }
1715 *pnext = NULL;
1652 if (fl->rx_offset == 0)
1653 nbuf++;
1654done:
1655 (*fl_bufs_used) += nbuf;
1716
1656 return (m0);
1657}
1658
1659static int
1660t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
1661{
1662 struct sge_rxq *rxq = iq_to_rxq(iq);
1663 struct ifnet *ifp = rxq->ifp;

--- 371 unchanged lines hidden (view full) ---

2035can_resume_tx(struct sge_eq *eq)
2036{
2037
2038 return (eq->avail + reclaimable(eq) >= tx_resume_threshold(eq));
2039}
2040
2041static inline void
2042init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
1717 return (m0);
1718}
1719
1720static int
1721t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
1722{
1723 struct sge_rxq *rxq = iq_to_rxq(iq);
1724 struct ifnet *ifp = rxq->ifp;

--- 371 unchanged lines hidden (view full) ---

2096can_resume_tx(struct sge_eq *eq)
2097{
2098
2099 return (eq->avail + reclaimable(eq) >= tx_resume_threshold(eq));
2100}
2101
2102static inline void
2103init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
2043 int qsize, int esize)
2104 int qsize)
2044{
2105{
2106
2045 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
2046 ("%s: bad tmr_idx %d", __func__, tmr_idx));
2047 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */
2048 ("%s: bad pktc_idx %d", __func__, pktc_idx));
2049
2050 iq->flags = 0;
2051 iq->adapter = sc;
2052 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
2053 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
2054 if (pktc_idx >= 0) {
2055 iq->intr_params |= F_QINTR_CNT_EN;
2056 iq->intr_pktc_idx = pktc_idx;
2057 }
2058 iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */
2107 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
2108 ("%s: bad tmr_idx %d", __func__, tmr_idx));
2109 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */
2110 ("%s: bad pktc_idx %d", __func__, pktc_idx));
2111
2112 iq->flags = 0;
2113 iq->adapter = sc;
2114 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
2115 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
2116 if (pktc_idx >= 0) {
2117 iq->intr_params |= F_QINTR_CNT_EN;
2118 iq->intr_pktc_idx = pktc_idx;
2119 }
2120 iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */
2059 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
2121 iq->sidx = iq->qsize - spg_len / IQ_ESIZE;
2060}
2061
2062static inline void
2063init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, int pack,
2064 char *name)
2065{
2066
2067 fl->qsize = qsize;
2122}
2123
2124static inline void
2125init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, int pack,
2126 char *name)
2127{
2128
2129 fl->qsize = qsize;
2130 fl->sidx = qsize - spg_len / EQ_ESIZE;
2068 strlcpy(fl->lockname, name, sizeof(fl->lockname));
2069 if (pack)
2070 fl->flags |= FL_BUF_PACKING;
2071 find_best_refill_source(sc, fl, maxp);
2072 find_safe_refill_source(sc, fl);
2073}
2074
2075static inline void

--- 76 unchanged lines hidden (view full) ---

2152 int intr_idx, int cong)
2153{
2154 int rc, i, cntxt_id;
2155 size_t len;
2156 struct fw_iq_cmd c;
2157 struct adapter *sc = iq->adapter;
2158 __be32 v = 0;
2159
2131 strlcpy(fl->lockname, name, sizeof(fl->lockname));
2132 if (pack)
2133 fl->flags |= FL_BUF_PACKING;
2134 find_best_refill_source(sc, fl, maxp);
2135 find_safe_refill_source(sc, fl);
2136}
2137
2138static inline void

--- 76 unchanged lines hidden (view full) ---

2215 int intr_idx, int cong)
2216{
2217 int rc, i, cntxt_id;
2218 size_t len;
2219 struct fw_iq_cmd c;
2220 struct adapter *sc = iq->adapter;
2221 __be32 v = 0;
2222
2160 len = iq->qsize * iq->esize;
2223 len = iq->qsize * IQ_ESIZE;
2161 rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
2162 (void **)&iq->desc);
2163 if (rc != 0)
2164 return (rc);
2165
2166 bzero(&c, sizeof(c));
2167 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
2168 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |

--- 15 unchanged lines hidden (view full) ---

2184
2185 c.type_to_iqandstindex = htobe32(v |
2186 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
2187 V_FW_IQ_CMD_VIID(pi->viid) |
2188 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
2189 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
2190 F_FW_IQ_CMD_IQGTSMODE |
2191 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
2224 rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
2225 (void **)&iq->desc);
2226 if (rc != 0)
2227 return (rc);
2228
2229 bzero(&c, sizeof(c));
2230 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
2231 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |

--- 15 unchanged lines hidden (view full) ---

2247
2248 c.type_to_iqandstindex = htobe32(v |
2249 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
2250 V_FW_IQ_CMD_VIID(pi->viid) |
2251 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
2252 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
2253 F_FW_IQ_CMD_IQGTSMODE |
2254 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
2192 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
2255 V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
2193 c.iqsize = htobe16(iq->qsize);
2194 c.iqaddr = htobe64(iq->ba);
2195 if (cong >= 0)
2196 c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN);
2197
2198 if (fl) {
2199 mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
2200
2256 c.iqsize = htobe16(iq->qsize);
2257 c.iqaddr = htobe64(iq->ba);
2258 if (cong >= 0)
2259 c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN);
2260
2261 if (fl) {
2262 mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
2263
2201 len = fl->qsize * RX_FL_ESIZE;
2264 len = fl->qsize * EQ_ESIZE;
2202 rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
2203 &fl->ba, (void **)&fl->desc);
2204 if (rc)
2205 return (rc);
2206
2207 /* Allocate space for one software descriptor per buffer. */
2265 rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
2266 &fl->ba, (void **)&fl->desc);
2267 if (rc)
2268 return (rc);
2269
2270 /* Allocate space for one software descriptor per buffer. */
2208 fl->cap = (fl->qsize - spg_len / RX_FL_ESIZE) * 8;
2209 rc = alloc_fl_sdesc(fl);
2210 if (rc != 0) {
2211 device_printf(sc->dev,
2212 "failed to setup fl software descriptors: %d\n",
2213 rc);
2214 return (rc);
2215 }
2271 rc = alloc_fl_sdesc(fl);
2272 if (rc != 0) {
2273 device_printf(sc->dev,
2274 "failed to setup fl software descriptors: %d\n",
2275 rc);
2276 return (rc);
2277 }
2216 fl->needed = fl->cap;
2217 fl->lowat = fl->flags & FL_BUF_PACKING ?
2218 roundup2(sc->sge.fl_starve_threshold2, 8) :
2219 roundup2(sc->sge.fl_starve_threshold, 8);
2220
2278
2279 if (fl->flags & FL_BUF_PACKING) {
2280 fl->lowat = roundup2(sc->sge.fl_starve_threshold2, 8);
2281 fl->buf_boundary = max(fl_pad, sc->sge.pack_boundary);
2282 } else {
2283 fl->lowat = roundup2(sc->sge.fl_starve_threshold, 8);
2284 fl->buf_boundary = fl_pad;
2285 }
2286
2221 c.iqns_to_fl0congen |=
2222 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
2223 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
2224 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
2225 (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
2226 0));
2227 if (cong >= 0) {
2228 c.iqns_to_fl0congen |=

--- 10 unchanged lines hidden (view full) ---

2239
2240 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2241 if (rc != 0) {
2242 device_printf(sc->dev,
2243 "failed to create ingress queue: %d\n", rc);
2244 return (rc);
2245 }
2246
2287 c.iqns_to_fl0congen |=
2288 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
2289 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
2290 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
2291 (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
2292 0));
2293 if (cong >= 0) {
2294 c.iqns_to_fl0congen |=

--- 10 unchanged lines hidden (view full) ---

2305
2306 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2307 if (rc != 0) {
2308 device_printf(sc->dev,
2309 "failed to create ingress queue: %d\n", rc);
2310 return (rc);
2311 }
2312
2247 iq->cdesc = iq->desc;
2248 iq->cidx = 0;
2313 iq->cidx = 0;
2249 iq->gen = 1;
2314 iq->gen = F_RSPD_GEN;
2250 iq->intr_next = iq->intr_params;
2251 iq->cntxt_id = be16toh(c.iqid);
2252 iq->abs_id = be16toh(c.physiqid);
2253 iq->flags |= IQ_ALLOCATED;
2254
2255 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
2256 if (cntxt_id >= sc->sge.niq) {
2257 panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
2258 cntxt_id, sc->sge.niq - 1);
2259 }
2260 sc->sge.iqmap[cntxt_id] = iq;
2261
2262 if (fl) {
2315 iq->intr_next = iq->intr_params;
2316 iq->cntxt_id = be16toh(c.iqid);
2317 iq->abs_id = be16toh(c.physiqid);
2318 iq->flags |= IQ_ALLOCATED;
2319
2320 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
2321 if (cntxt_id >= sc->sge.niq) {
2322 panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
2323 cntxt_id, sc->sge.niq - 1);
2324 }
2325 sc->sge.iqmap[cntxt_id] = iq;
2326
2327 if (fl) {
2328 u_int qid;
2329
2330 iq->flags |= IQ_HAS_FL;
2263 fl->cntxt_id = be16toh(c.fl0id);
2264 fl->pidx = fl->cidx = 0;
2265
2266 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
2267 if (cntxt_id >= sc->sge.neq) {
2268 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
2269 __func__, cntxt_id, sc->sge.neq - 1);
2270 }
2271 sc->sge.eqmap[cntxt_id] = (void *)fl;
2272
2331 fl->cntxt_id = be16toh(c.fl0id);
2332 fl->pidx = fl->cidx = 0;
2333
2334 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
2335 if (cntxt_id >= sc->sge.neq) {
2336 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
2337 __func__, cntxt_id, sc->sge.neq - 1);
2338 }
2339 sc->sge.eqmap[cntxt_id] = (void *)fl;
2340
2341 qid = fl->cntxt_id;
2342 if (isset(&sc->doorbells, DOORBELL_UDB)) {
2343 uint32_t s_qpp = sc->sge.eq_s_qpp;
2344 uint32_t mask = (1 << s_qpp) - 1;
2345 volatile uint8_t *udb;
2346
2347 udb = sc->udbs_base + UDBS_DB_OFFSET;
2348 udb += (qid >> s_qpp) << PAGE_SHIFT;
2349 qid &= mask;
2350 if (qid < PAGE_SIZE / UDBS_SEG_SIZE) {
2351 udb += qid << UDBS_SEG_SHIFT;
2352 qid = 0;
2353 }
2354 fl->udb = (volatile void *)udb;
2355 }
2356 fl->dbval = F_DBPRIO | V_QID(qid);
2357 if (is_t5(sc))
2358 fl->dbval |= F_DBTYPE;
2359
2273 FL_LOCK(fl);
2274 /* Enough to make sure the SGE doesn't think it's starved */
2275 refill_fl(sc, fl, fl->lowat);
2276 FL_UNLOCK(fl);
2360 FL_LOCK(fl);
2361 /* Enough to make sure the SGE doesn't think it's starved */
2362 refill_fl(sc, fl, fl->lowat);
2363 FL_UNLOCK(fl);
2277
2278 iq->flags |= IQ_HAS_FL;
2279 }
2280
2281 if (is_t5(sc) && cong >= 0) {
2282 uint32_t param, val;
2283
2284 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2285 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
2286 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);

--- 104 unchanged lines hidden (view full) ---

2391static int
2392alloc_fwq(struct adapter *sc)
2393{
2394 int rc, intr_idx;
2395 struct sge_iq *fwq = &sc->sge.fwq;
2396 struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
2397 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
2398
2364 }
2365
2366 if (is_t5(sc) && cong >= 0) {
2367 uint32_t param, val;
2368
2369 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2370 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
2371 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);

--- 104 unchanged lines hidden (view full) ---

2476static int
2477alloc_fwq(struct adapter *sc)
2478{
2479 int rc, intr_idx;
2480 struct sge_iq *fwq = &sc->sge.fwq;
2481 struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
2482 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
2483
2399 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
2484 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE);
2400 fwq->flags |= IQ_INTR; /* always */
2401 intr_idx = sc->intr_count > 1 ? 1 : 0;
2402 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
2403 if (rc != 0) {
2404 device_printf(sc->dev,
2405 "failed to create firmware event queue: %d\n", rc);
2406 return (rc);
2407 }

--- 72 unchanged lines hidden (view full) ---

2480 int rc;
2481 struct sysctl_oid_list *children;
2482 char name[16];
2483
2484 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, tnl_cong(pi));
2485 if (rc != 0)
2486 return (rc);
2487
2485 fwq->flags |= IQ_INTR; /* always */
2486 intr_idx = sc->intr_count > 1 ? 1 : 0;
2487 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
2488 if (rc != 0) {
2489 device_printf(sc->dev,
2490 "failed to create firmware event queue: %d\n", rc);
2491 return (rc);
2492 }

--- 72 unchanged lines hidden (view full) ---

2565 int rc;
2566 struct sysctl_oid_list *children;
2567 char name[16];
2568
2569 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, tnl_cong(pi));
2570 if (rc != 0)
2571 return (rc);
2572
2573 /*
2574 * The freelist is just barely above the starvation threshold right now,
2575 * fill it up a bit more.
2576 */
2488 FL_LOCK(&rxq->fl);
2577 FL_LOCK(&rxq->fl);
2489 refill_fl(pi->adapter, &rxq->fl, rxq->fl.needed / 8);
2578 refill_fl(pi->adapter, &rxq->fl, 128);
2490 FL_UNLOCK(&rxq->fl);
2491
2492#if defined(INET) || defined(INET6)
2493 rc = tcp_lro_init(&rxq->lro);
2494 if (rc != 0)
2495 return (rc);
2496 rxq->lro.ifp = pi->ifp; /* also indicates LRO init'ed */
2497

--- 98 unchanged lines hidden (view full) ---

2596 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
2597 if (rc == 0)
2598 bzero(ofld_rxq, sizeof(*ofld_rxq));
2599
2600 return (rc);
2601}
2602#endif
2603
2579 FL_UNLOCK(&rxq->fl);
2580
2581#if defined(INET) || defined(INET6)
2582 rc = tcp_lro_init(&rxq->lro);
2583 if (rc != 0)
2584 return (rc);
2585 rxq->lro.ifp = pi->ifp; /* also indicates LRO init'ed */
2586

--- 98 unchanged lines hidden (view full) ---

2685 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
2686 if (rc == 0)
2687 bzero(ofld_rxq, sizeof(*ofld_rxq));
2688
2689 return (rc);
2690}
2691#endif
2692
2693#ifdef DEV_NETMAP
2604static int
2694static int
2695alloc_nm_rxq(struct port_info *pi, struct sge_nm_rxq *nm_rxq, int intr_idx,
2696 int idx, struct sysctl_oid *oid)
2697{
2698 int rc;
2699 struct sysctl_oid_list *children;
2700 struct sysctl_ctx_list *ctx;
2701 char name[16];
2702 size_t len;
2703 struct adapter *sc = pi->adapter;
2704 struct netmap_adapter *na = NA(pi->nm_ifp);
2705
2706 MPASS(na != NULL);
2707
2708 len = pi->qsize_rxq * IQ_ESIZE;
2709 rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map,
2710 &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc);
2711 if (rc != 0)
2712 return (rc);
2713
2714 len = na->num_rx_desc * EQ_ESIZE + spg_len;
2715 rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map,
2716 &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc);
2717 if (rc != 0)
2718 return (rc);
2719
2720 nm_rxq->pi = pi;
2721 nm_rxq->nid = idx;
2722 nm_rxq->iq_cidx = 0;
2723 nm_rxq->iq_sidx = pi->qsize_rxq - spg_len / IQ_ESIZE;
2724 nm_rxq->iq_gen = F_RSPD_GEN;
2725 nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;
2726 nm_rxq->fl_sidx = na->num_rx_desc;
2727 nm_rxq->intr_idx = intr_idx;
2728
2729 ctx = &pi->ctx;
2730 children = SYSCTL_CHILDREN(oid);
2731
2732 snprintf(name, sizeof(name), "%d", idx);
2733 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL,
2734 "rx queue");
2735 children = SYSCTL_CHILDREN(oid);
2736
2737 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id",
2738 CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_abs_id, 0, sysctl_uint16,
2739 "I", "absolute id of the queue");
2740 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
2741 CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cntxt_id, 0, sysctl_uint16,
2742 "I", "SGE context id of the queue");
2743 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
2744 CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cidx, 0, sysctl_uint16, "I",
2745 "consumer index");
2746
2747 children = SYSCTL_CHILDREN(oid);
2748 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL,
2749 "freelist");
2750 children = SYSCTL_CHILDREN(oid);
2751
2752 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
2753 CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->fl_cntxt_id, 0, sysctl_uint16,
2754 "I", "SGE context id of the freelist");
2755 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD,
2756 &nm_rxq->fl_cidx, 0, "consumer index");
2757 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD,
2758 &nm_rxq->fl_pidx, 0, "producer index");
2759
2760 return (rc);
2761}
2762
2763
2764static int
2765free_nm_rxq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
2766{
2767 struct adapter *sc = pi->adapter;
2768
2769 free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba,
2770 nm_rxq->iq_desc);
2771 free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba,
2772 nm_rxq->fl_desc);
2773
2774 return (0);
2775}
2776
2777static int
2778alloc_nm_txq(struct port_info *pi, struct sge_nm_txq *nm_txq, int iqidx, int idx,
2779 struct sysctl_oid *oid)
2780{
2781 int rc;
2782 size_t len;
2783 struct adapter *sc = pi->adapter;
2784 struct netmap_adapter *na = NA(pi->nm_ifp);
2785 char name[16];
2786 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
2787
2788 len = na->num_tx_desc * EQ_ESIZE + spg_len;
2789 rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map,
2790 &nm_txq->ba, (void **)&nm_txq->desc);
2791 if (rc)
2792 return (rc);
2793
2794 nm_txq->pidx = nm_txq->cidx = 0;
2795 nm_txq->sidx = na->num_tx_desc;
2796 nm_txq->nid = idx;
2797 nm_txq->iqidx = iqidx;
2798 nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2799 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf));
2800
2801 snprintf(name, sizeof(name), "%d", idx);
2802 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
2803 NULL, "netmap tx queue");
2804 children = SYSCTL_CHILDREN(oid);
2805
2806 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
2807 &nm_txq->cntxt_id, 0, "SGE context id of the queue");
2808 SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
2809 CTLTYPE_INT | CTLFLAG_RD, &nm_txq->cidx, 0, sysctl_uint16, "I",
2810 "consumer index");
2811 SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "pidx",
2812 CTLTYPE_INT | CTLFLAG_RD, &nm_txq->pidx, 0, sysctl_uint16, "I",
2813 "producer index");
2814
2815 return (rc);
2816}
2817
2818static int
2819free_nm_txq(struct port_info *pi, struct sge_nm_txq *nm_txq)
2820{
2821 struct adapter *sc = pi->adapter;
2822
2823 free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba,
2824 nm_txq->desc);
2825
2826 return (0);
2827}
2828#endif
2829
2830static int
2605ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
2606{
2607 int rc, cntxt_id;
2608 struct fw_eq_ctrl_cmd c;
2609
2610 bzero(&c, sizeof(c));
2611
2612 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |

--- 170 unchanged lines hidden (view full) ---

2783 isset(&eq->doorbells, DOORBELL_WCWR)) {
2784 uint32_t s_qpp = sc->sge.eq_s_qpp;
2785 uint32_t mask = (1 << s_qpp) - 1;
2786 volatile uint8_t *udb;
2787
2788 udb = sc->udbs_base + UDBS_DB_OFFSET;
2789 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
2790 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
2831ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
2832{
2833 int rc, cntxt_id;
2834 struct fw_eq_ctrl_cmd c;
2835
2836 bzero(&c, sizeof(c));
2837
2838 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |

--- 170 unchanged lines hidden (view full) ---

3009 isset(&eq->doorbells, DOORBELL_WCWR)) {
3010 uint32_t s_qpp = sc->sge.eq_s_qpp;
3011 uint32_t mask = (1 << s_qpp) - 1;
3012 volatile uint8_t *udb;
3013
3014 udb = sc->udbs_base + UDBS_DB_OFFSET;
3015 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
3016 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
2791 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
3017 if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
2792 clrbit(&eq->doorbells, DOORBELL_WCWR);
2793 else {
2794 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
2795 eq->udb_qid = 0;
2796 }
2797 eq->udb = (volatile void *)udb;
2798 }
2799

--- 211 unchanged lines hidden (view full) ---

3011 bus_addr_t *ba = arg;
3012
3013 KASSERT(nseg == 1,
3014 ("%s meant for single segment mappings only.", __func__));
3015
3016 *ba = error ? 0 : segs->ds_addr;
3017}
3018
3018 clrbit(&eq->doorbells, DOORBELL_WCWR);
3019 else {
3020 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
3021 eq->udb_qid = 0;
3022 }
3023 eq->udb = (volatile void *)udb;
3024 }
3025

--- 211 unchanged lines hidden (view full) ---

3237 bus_addr_t *ba = arg;
3238
3239 KASSERT(nseg == 1,
3240 ("%s meant for single segment mappings only.", __func__));
3241
3242 *ba = error ? 0 : segs->ds_addr;
3243}
3244
3019static inline bool
3020is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
3021{
3022 *ctrl = (void *)((uintptr_t)iq->cdesc +
3023 (iq->esize - sizeof(struct rsp_ctrl)));
3024
3025 return (((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen);
3026}
3027
3028static inline void
3245static inline void
3029iq_next(struct sge_iq *iq)
3030{
3031 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
3032 if (__predict_false(++iq->cidx == iq->qsize - 1)) {
3033 iq->cidx = 0;
3034 iq->gen ^= 1;
3035 iq->cdesc = iq->desc;
3036 }
3037}
3038
3039#define FL_HW_IDX(x) ((x) >> 3)
3040static inline void
3041ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3042{
3246ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3247{
3043 int ndesc = fl->pending / 8;
3044 uint32_t v;
3248 uint32_t n, v;
3045
3249
3046 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
3047 ndesc--; /* hold back one credit */
3250 n = IDXDIFF(fl->pidx / 8, fl->dbidx, fl->sidx);
3251 MPASS(n > 0);
3048
3252
3049 if (ndesc <= 0)
3050 return; /* nothing to do */
3051
3052 v = F_DBPRIO | V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3053 if (is_t5(sc))
3054 v |= F_DBTYPE;
3055
3056 wmb();
3253 wmb();
3057
3058 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3059 fl->pending -= ndesc * 8;
3254 v = fl->dbval | V_PIDX(n);
3255 if (fl->udb)
3256 *fl->udb = htole32(v);
3257 else
3258 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3259 IDXINCR(fl->dbidx, n, fl->sidx);
3060}
3061
3062/*
3260}
3261
3262/*
3063 * Fill up the freelist by upto nbufs and maybe ring its doorbell.
3263 * Fills up the freelist by allocating upto 'n' buffers. Buffers that are
3264 * recycled do not count towards this allocation budget.
3064 *
3265 *
3065 * Returns non-zero to indicate that it should be added to the list of starving
3066 * freelists.
3266 * Returns non-zero to indicate that this freelist should be added to the list
3267 * of starving freelists.
3067 */
3068static int
3268 */
3269static int
3069refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
3270refill_fl(struct adapter *sc, struct sge_fl *fl, int n)
3070{
3271{
3071 __be64 *d = &fl->desc[fl->pidx];
3072 struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
3272 __be64 *d;
3273 struct fl_sdesc *sd;
3073 uintptr_t pa;
3074 caddr_t cl;
3274 uintptr_t pa;
3275 caddr_t cl;
3075 struct cluster_layout *cll = &fl->cll_def; /* default layout */
3076 struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx];
3276 struct cluster_layout *cll;
3277 struct sw_zone_info *swz;
3077 struct cluster_metadata *clm;
3278 struct cluster_metadata *clm;
3279 uint16_t max_pidx;
3280 uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */
3078
3079 FL_LOCK_ASSERT_OWNED(fl);
3080
3281
3282 FL_LOCK_ASSERT_OWNED(fl);
3283
3081 if (nbufs > fl->needed)
3082 nbufs = fl->needed;
3083 nbufs -= (fl->pidx + nbufs) % 8;
3284 /*
3285 * We always stop at the begining of the hardware descriptor that's just
3286 * before the one with the hw cidx. This is to avoid hw pidx = hw cidx,
3287 * which would mean an empty freelist to the chip.
3288 */
3289 max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1;
3290 if (fl->pidx == max_pidx * 8)
3291 return (0);
3084
3292
3085 while (nbufs--) {
3293 d = &fl->desc[fl->pidx];
3294 sd = &fl->sdesc[fl->pidx];
3295 cll = &fl->cll_def; /* default layout */
3296 swz = &sc->sge.sw_zone_info[cll->zidx];
3086
3297
3298 while (n > 0) {
3299
3087 if (sd->cl != NULL) {
3088
3089 if (sd->nmbuf == 0) {
3090 /*
3091 * Fast recycle without involving any atomics on
3092 * the cluster's metadata (if the cluster has
3093 * metadata). This happens when all frames
3094 * received in the cluster were small enough to

--- 32 unchanged lines hidden (view full) ---

3127 break;
3128
3129 /* fall back to the safe zone */
3130 cll = &fl->cll_alt;
3131 swz = &sc->sge.sw_zone_info[cll->zidx];
3132 goto alloc;
3133 }
3134 fl->cl_allocated++;
3300 if (sd->cl != NULL) {
3301
3302 if (sd->nmbuf == 0) {
3303 /*
3304 * Fast recycle without involving any atomics on
3305 * the cluster's metadata (if the cluster has
3306 * metadata). This happens when all frames
3307 * received in the cluster were small enough to

--- 32 unchanged lines hidden (view full) ---

3340 break;
3341
3342 /* fall back to the safe zone */
3343 cll = &fl->cll_alt;
3344 swz = &sc->sge.sw_zone_info[cll->zidx];
3345 goto alloc;
3346 }
3347 fl->cl_allocated++;
3348 n--;
3135
3136 pa = pmap_kextract((vm_offset_t)cl);
3137 pa += cll->region1;
3138 sd->cl = cl;
3139 sd->cll = *cll;
3140 *d = htobe64(pa | cll->hwidx);
3141 clm = cl_metadata(sc, fl, cll, cl);
3142 if (clm != NULL) {
3143recycled:
3144#ifdef INVARIANTS
3145 clm->sd = sd;
3146#endif
3147 clm->refcount = 1;
3148 }
3149 sd->nmbuf = 0;
3150recycled_fast:
3349
3350 pa = pmap_kextract((vm_offset_t)cl);
3351 pa += cll->region1;
3352 sd->cl = cl;
3353 sd->cll = *cll;
3354 *d = htobe64(pa | cll->hwidx);
3355 clm = cl_metadata(sc, fl, cll, cl);
3356 if (clm != NULL) {
3357recycled:
3358#ifdef INVARIANTS
3359 clm->sd = sd;
3360#endif
3361 clm->refcount = 1;
3362 }
3363 sd->nmbuf = 0;
3364recycled_fast:
3151 fl->pending++;
3152 fl->needed--;
3153 d++;
3154 sd++;
3365 d++;
3366 sd++;
3155 if (__predict_false(++fl->pidx == fl->cap)) {
3156 fl->pidx = 0;
3157 sd = fl->sdesc;
3158 d = fl->desc;
3367 if (__predict_false(++fl->pidx % 8 == 0)) {
3368 uint16_t pidx = fl->pidx / 8;
3369
3370 if (__predict_false(pidx == fl->sidx)) {
3371 fl->pidx = 0;
3372 pidx = 0;
3373 sd = fl->sdesc;
3374 d = fl->desc;
3375 }
3376 if (pidx == max_pidx)
3377 break;
3378
3379 if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4)
3380 ring_fl_db(sc, fl);
3159 }
3160 }
3161
3381 }
3382 }
3383
3162 if (fl->pending >= 8)
3384 if (fl->pidx / 8 != fl->dbidx)
3163 ring_fl_db(sc, fl);
3164
3165 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
3166}
3167
3168/*
3169 * Attempt to refill all starving freelists.
3170 */

--- 18 unchanged lines hidden (view full) ---

3189 callout_schedule(&sc->sfl_callout, hz / 5);
3190 mtx_unlock(&sc->sfl_lock);
3191}
3192
3193static int
3194alloc_fl_sdesc(struct sge_fl *fl)
3195{
3196
3385 ring_fl_db(sc, fl);
3386
3387 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
3388}
3389
3390/*
3391 * Attempt to refill all starving freelists.
3392 */

--- 18 unchanged lines hidden (view full) ---

3411 callout_schedule(&sc->sfl_callout, hz / 5);
3412 mtx_unlock(&sc->sfl_lock);
3413}
3414
3415static int
3416alloc_fl_sdesc(struct sge_fl *fl)
3417{
3418
3197 fl->sdesc = malloc(fl->cap * sizeof(struct fl_sdesc), M_CXGBE,
3419 fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE,
3198 M_ZERO | M_WAITOK);
3199
3200 return (0);
3201}
3202
3203static void
3204free_fl_sdesc(struct adapter *sc, struct sge_fl *fl)
3205{
3206 struct fl_sdesc *sd;
3207 struct cluster_metadata *clm;
3208 struct cluster_layout *cll;
3209 int i;
3210
3211 sd = fl->sdesc;
3420 M_ZERO | M_WAITOK);
3421
3422 return (0);
3423}
3424
3425static void
3426free_fl_sdesc(struct adapter *sc, struct sge_fl *fl)
3427{
3428 struct fl_sdesc *sd;
3429 struct cluster_metadata *clm;
3430 struct cluster_layout *cll;
3431 int i;
3432
3433 sd = fl->sdesc;
3212 for (i = 0; i < fl->cap; i++, sd++) {
3434 for (i = 0; i < fl->sidx * 8; i++, sd++) {
3213 if (sd->cl == NULL)
3214 continue;
3215
3216 cll = &sd->cll;
3217 clm = cl_metadata(sc, fl, cll, sd->cl);
3218 if (sd->nmbuf == 0)
3219 uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl);
3220 else if (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1) {

--- 1117 unchanged lines hidden --- 		3435 if (sd->cl == NULL)
3436 continue;
3437
3438 cll = &sd->cll;
3439 clm = cl_metadata(sc, fl, cll, sd->cl);
3440 if (sd->nmbuf == 0)
3441 uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl);
3442 else if (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1) {

--- 1117 unchanged lines hidden ---