Deleted Added
sdiff udiff text old ( 171469 ) new ( 171471 )
full compact
cxgb_sge.c (171469) cxgb_sge.c (171471)
1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8

--- 14 unchanged lines hidden (view full) ---

23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8

--- 14 unchanged lines hidden (view full) ---

23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_sge.c 171469 2007-07-17 06:12:22Z kmacy $");
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_sge.c 171471 2007-07-17 06:50:35Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/bus.h>
38#include <sys/conf.h>
39#include <machine/bus.h>

--- 24 unchanged lines hidden (view full) ---

64#include <dev/cxgb/cxgb_include.h>
65#endif
66
67uint32_t collapse_free = 0;
68uint32_t mb_free_vec_free = 0;
69int txq_fills = 0;
70int collapse_mbufs = 0;
71static int recycle_enable = 1;
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/bus.h>
38#include <sys/conf.h>
39#include <machine/bus.h>

--- 24 unchanged lines hidden (view full) ---

64#include <dev/cxgb/cxgb_include.h>
65#endif
66
67uint32_t collapse_free = 0;
68uint32_t mb_free_vec_free = 0;
69int txq_fills = 0;
70int collapse_mbufs = 0;
71static int recycle_enable = 1;
72static int bogus_imm = 0;
72
73
73
74/*
75 * XXX GC
76 */
77#define NET_XMIT_CN 2
78#define NET_XMIT_SUCCESS 0
79
80#define USE_GTS 0
81

--- 205 unchanged lines hidden (view full) ---

287}
288
289/**
290 * get_imm_packet - return the next ingress packet buffer from a response
291 * @resp: the response descriptor containing the packet data
292 *
293 * Return a packet containing the immediate data of the given response.
294 */
74/*
75 * XXX GC
76 */
77#define NET_XMIT_CN 2
78#define NET_XMIT_SUCCESS 0
79
80#define USE_GTS 0
81

--- 205 unchanged lines hidden (view full) ---

287}
288
289/**
290 * get_imm_packet - return the next ingress packet buffer from a response
291 * @resp: the response descriptor containing the packet data
292 *
293 * Return a packet containing the immediate data of the given response.
294 */
295static __inline void
296get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl)
295static int
296get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl, uint32_t flags)
297{
297{
298 int len;
299 uint32_t flags = ntohl(resp->flags);
298 int len, error;
300 uint8_t sopeop = G_RSPD_SOP_EOP(flags);
299 uint8_t sopeop = G_RSPD_SOP_EOP(flags);
301
300
302 /*
303 * would be a firmware bug
304 */
301 /*
302 * would be a firmware bug
303 */
305 if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP)
306 return;
307
308 len = G_RSPD_LEN(ntohl(resp->len_cq));
304 len = G_RSPD_LEN(ntohl(resp->len_cq));
305 if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP) {
306 if (cxgb_debug)
307 device_printf(sc->dev, "unexpected value sopeop=%d flags=0x%x len=%din get_imm_packet\n", sopeop, flags, len);
308 bogus_imm++;
309 return (EINVAL);
310 }
311 error = 0;
309 switch (sopeop) {
310 case RSPQ_SOP_EOP:
311 m->m_len = m->m_pkthdr.len = len;
312 memcpy(mtod(m, uint8_t *), resp->imm_data, len);
313 break;
314 case RSPQ_EOP:
315 memcpy(cl, resp->imm_data, len);
316 m_iovappend(m, cl, MSIZE, len, 0);
317 break;
312 switch (sopeop) {
313 case RSPQ_SOP_EOP:
314 m->m_len = m->m_pkthdr.len = len;
315 memcpy(mtod(m, uint8_t *), resp->imm_data, len);
316 break;
317 case RSPQ_EOP:
318 memcpy(cl, resp->imm_data, len);
319 m_iovappend(m, cl, MSIZE, len, 0);
320 break;
321 default:
322 bogus_imm++;
323 error = EINVAL;
318 }
324 }
325
326 return (error);
319}
320
321
322static __inline u_int
323flits_to_desc(u_int n)
324{
325 return (flit_desc_map[n]);
326}

--- 30 unchanged lines hidden (view full) ---

357 /* XXX Does ETHER_ALIGN need to be accounted for here? */
358 p->max_pkt_size = MJUM16BYTES - sizeof(struct cpl_rx_data);
359
360 for (i = 0; i < SGE_QSETS; ++i) {
361 struct qset_params *q = p->qset + i;
362
363 q->polling = adap->params.rev > 0;
364
327}
328
329
330static __inline u_int
331flits_to_desc(u_int n)
332{
333 return (flit_desc_map[n]);
334}

--- 30 unchanged lines hidden (view full) ---

365 /* XXX Does ETHER_ALIGN need to be accounted for here? */
366 p->max_pkt_size = MJUM16BYTES - sizeof(struct cpl_rx_data);
367
368 for (i = 0; i < SGE_QSETS; ++i) {
369 struct qset_params *q = p->qset + i;
370
371 q->polling = adap->params.rev > 0;
372
365 q->coalesce_nsecs = 5000;
366
373 if (adap->params.nports > 2)
374 q->coalesce_nsecs = 50000;
375 else
376 q->coalesce_nsecs = 5000;
377
367 q->rspq_size = RSPQ_Q_SIZE;
368 q->fl_size = FL_Q_SIZE;
369 q->jumbo_size = JUMBO_Q_SIZE;
370 q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE;
371 q->txq_size[TXQ_OFLD] = 1024;
372 q->txq_size[TXQ_CTRL] = 256;
373 q->cong_thres = 0;
374 }

--- 284 unchanged lines hidden (view full) ---

659static void
660sge_slow_intr_handler(void *arg, int ncount)
661{
662 adapter_t *sc = arg;
663
664 t3_slow_intr_handler(sc);
665}
666
378 q->rspq_size = RSPQ_Q_SIZE;
379 q->fl_size = FL_Q_SIZE;
380 q->jumbo_size = JUMBO_Q_SIZE;
381 q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE;
382 q->txq_size[TXQ_OFLD] = 1024;
383 q->txq_size[TXQ_CTRL] = 256;
384 q->cong_thres = 0;
385 }

--- 284 unchanged lines hidden (view full) ---

670static void
671sge_slow_intr_handler(void *arg, int ncount)
672{
673 adapter_t *sc = arg;
674
675 t3_slow_intr_handler(sc);
676}
677
678/**
679 * sge_timer_cb - perform periodic maintenance of an SGE qset
680 * @data: the SGE queue set to maintain
681 *
682 * Runs periodically from a timer to perform maintenance of an SGE queue
683 * set. It performs two tasks:
684 *
685 * a) Cleans up any completed Tx descriptors that may still be pending.
686 * Normal descriptor cleanup happens when new packets are added to a Tx
687 * queue so this timer is relatively infrequent and does any cleanup only
688 * if the Tx queue has not seen any new packets in a while. We make a
689 * best effort attempt to reclaim descriptors, in that we don't wait
690 * around if we cannot get a queue's lock (which most likely is because
691 * someone else is queueing new packets and so will also handle the clean
692 * up). Since control queues use immediate data exclusively we don't
693 * bother cleaning them up here.
694 *
695 * b) Replenishes Rx queues that have run out due to memory shortage.
696 * Normally new Rx buffers are added when existing ones are consumed but
697 * when out of memory a queue can become empty. We try to add only a few
698 * buffers here, the queue will be replenished fully as these new buffers
699 * are used up if memory shortage has subsided.
700 *
701 * c) Return coalesced response queue credits in case a response queue is
702 * starved.
703 *
704 * d) Ring doorbells for T304 tunnel queues since we have seen doorbell
705 * fifo overflows and the FW doesn't implement any recovery scheme yet.
706 */
667static void
668sge_timer_cb(void *arg)
669{
670 adapter_t *sc = arg;
671 struct port_info *p;
672 struct sge_qset *qs;
673 struct sge_txq *txq;
674 int i, j;

--- 8 unchanged lines hidden (view full) ---

683 refill_rx = ((qs->fl[0].credits < qs->fl[0].size) ||
684 (qs->fl[1].credits < qs->fl[1].size));
685 if (reclaim_eth || reclaim_ofl || refill_rx) {
686 p = &sc->port[i];
687 taskqueue_enqueue(p->tq, &p->timer_reclaim_task);
688 break;
689 }
690 }
707static void
708sge_timer_cb(void *arg)
709{
710 adapter_t *sc = arg;
711 struct port_info *p;
712 struct sge_qset *qs;
713 struct sge_txq *txq;
714 int i, j;

--- 8 unchanged lines hidden (view full) ---

723 refill_rx = ((qs->fl[0].credits < qs->fl[0].size) ||
724 (qs->fl[1].credits < qs->fl[1].size));
725 if (reclaim_eth || reclaim_ofl || refill_rx) {
726 p = &sc->port[i];
727 taskqueue_enqueue(p->tq, &p->timer_reclaim_task);
728 break;
729 }
730 }
731 if (sc->params.nports > 2) {
732 int i;
733
734 for_each_port(sc, i) {
735 struct port_info *pi = &sc->port[i];
736
737 t3_write_reg(sc, A_SG_KDOORBELL,
738 F_SELEGRCNTX |
739 (FW_TUNNEL_SGEEC_START + pi->first_qset));
740 }
741 }
691 if (sc->open_device_map != 0)
692 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
693}
694
695/*
696 * This is meant to be a catch-all function to keep sge state private
697 * to sge.c
698 *

--- 216 unchanged lines hidden (view full) ---

915 mtmp = mtmp->m_next;
916 }
917 printf("map_mbufs: bus_dmamap_load_mbuf_sg failed with %d - pkthdr.len==%d nmbufs=%d\n",
918 err, m0->m_pkthdr.len, n);
919 }
920#endif
921 if (err == EFBIG) {
922 /* Too many segments, try to defrag */
742 if (sc->open_device_map != 0)
743 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
744}
745
746/*
747 * This is meant to be a catch-all function to keep sge state private
748 * to sge.c
749 *

--- 216 unchanged lines hidden (view full) ---

966 mtmp = mtmp->m_next;
967 }
968 printf("map_mbufs: bus_dmamap_load_mbuf_sg failed with %d - pkthdr.len==%d nmbufs=%d\n",
969 err, m0->m_pkthdr.len, n);
970 }
971#endif
972 if (err == EFBIG) {
973 /* Too many segments, try to defrag */
923 m0 = m_defrag(m0, M_NOWAIT);
974 m0 = m_defrag(m0, M_DONTWAIT);
924 if (m0 == NULL) {
925 m_freem(*m);
926 *m = NULL;
927 return (ENOBUFS);
928 }
929 *m = m0;
930 err = bus_dmamap_load_mbuf_sg(txq->entry_tag, stx->map, m0, segs, nsegs, 0);
931 }

--- 1114 unchanged lines hidden (view full) ---

2046
2047
2048
2049 q->fl[0].gen = q->fl[1].gen = 1;
2050 q->fl[0].size = p->fl_size;
2051 q->fl[1].size = p->jumbo_size;
2052
2053 q->rspq.gen = 1;
975 if (m0 == NULL) {
976 m_freem(*m);
977 *m = NULL;
978 return (ENOBUFS);
979 }
980 *m = m0;
981 err = bus_dmamap_load_mbuf_sg(txq->entry_tag, stx->map, m0, segs, nsegs, 0);
982 }

--- 1114 unchanged lines hidden (view full) ---

2097
2098
2099
2100 q->fl[0].gen = q->fl[1].gen = 1;
2101 q->fl[0].size = p->fl_size;
2102 q->fl[1].size = p->jumbo_size;
2103
2104 q->rspq.gen = 1;
2105 q->rspq.cidx = 0;
2054 q->rspq.size = p->rspq_size;
2055
2056 q->txq[TXQ_ETH].stop_thres = nports *
2057 flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3);
2058
2059 q->fl[0].buf_size = MCLBYTES;
2060 q->fl[0].zone = zone_clust;
2061 q->fl[0].type = EXT_CLUSTER;

--- 144 unchanged lines hidden (view full) ---

2206 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
2207 uint32_t len = G_RSPD_LEN(len_cq);
2208 uint32_t flags = ntohl(r->flags);
2209 uint8_t sopeop = G_RSPD_SOP_EOP(flags);
2210 void *cl;
2211 int ret = 0;
2212
2213 prefetch(sd->cl);
2106 q->rspq.size = p->rspq_size;
2107
2108 q->txq[TXQ_ETH].stop_thres = nports *
2109 flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3);
2110
2111 q->fl[0].buf_size = MCLBYTES;
2112 q->fl[0].zone = zone_clust;
2113 q->fl[0].type = EXT_CLUSTER;

--- 144 unchanged lines hidden (view full) ---

2258 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
2259 uint32_t len = G_RSPD_LEN(len_cq);
2260 uint32_t flags = ntohl(r->flags);
2261 uint8_t sopeop = G_RSPD_SOP_EOP(flags);
2262 void *cl;
2263 int ret = 0;
2264
2265 prefetch(sd->cl);
2214
2266
2215 fl->credits--;
2216 bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
2217
2218 if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) {
2219 cl = mtod(m, void *);
2220 memcpy(cl, sd->cl, len);
2221 recycle_rx_buf(adap, fl, fl->cidx);
2222 } else {

--- 100 unchanged lines hidden (view full) ---

2323 struct rsp_desc *r = &rspq->desc[rspq->cidx];
2324 int budget_left = budget;
2325 unsigned int sleeping = 0;
2326 int lro = qs->lro.enabled;
2327 struct mbuf *offload_mbufs[RX_BUNDLE_SIZE];
2328 int ngathered = 0;
2329#ifdef DEBUG
2330 static int last_holdoff = 0;
2267 fl->credits--;
2268 bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
2269
2270 if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) {
2271 cl = mtod(m, void *);
2272 memcpy(cl, sd->cl, len);
2273 recycle_rx_buf(adap, fl, fl->cidx);
2274 } else {

--- 100 unchanged lines hidden (view full) ---

2375 struct rsp_desc *r = &rspq->desc[rspq->cidx];
2376 int budget_left = budget;
2377 unsigned int sleeping = 0;
2378 int lro = qs->lro.enabled;
2379 struct mbuf *offload_mbufs[RX_BUNDLE_SIZE];
2380 int ngathered = 0;
2381#ifdef DEBUG
2382 static int last_holdoff = 0;
2331 if (rspq->holdoff_tmr != last_holdoff) {
2383 if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) {
2332 printf("next_holdoff=%d\n", rspq->holdoff_tmr);
2333 last_holdoff = rspq->holdoff_tmr;
2334 }
2335#endif
2336 rspq->next_holdoff = rspq->holdoff_tmr;
2337
2338 while (__predict_true(budget_left && is_new_response(r, rspq))) {
2339 int eth, eop = 0, ethpad = 0;

--- 4 unchanged lines hidden (view full) ---

2344 eth = (r->rss_hdr.opcode == CPL_RX_PKT);
2345
2346 if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) {
2347 /* XXX */
2348 printf("async notification\n");
2349
2350 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2351 struct mbuf *m = NULL;
2384 printf("next_holdoff=%d\n", rspq->holdoff_tmr);
2385 last_holdoff = rspq->holdoff_tmr;
2386 }
2387#endif
2388 rspq->next_holdoff = rspq->holdoff_tmr;
2389
2390 while (__predict_true(budget_left && is_new_response(r, rspq))) {
2391 int eth, eop = 0, ethpad = 0;

--- 4 unchanged lines hidden (view full) ---

2396 eth = (r->rss_hdr.opcode == CPL_RX_PKT);
2397
2398 if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) {
2399 /* XXX */
2400 printf("async notification\n");
2401
2402 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2403 struct mbuf *m = NULL;
2404
2352 if (cxgb_debug)
2405 if (cxgb_debug)
2353 printf("IMM DATA VALID\n");
2406 printf("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx);
2354 if (rspq->m == NULL)
2407 if (rspq->m == NULL)
2355 rspq->m = m_gethdr(M_NOWAIT, MT_DATA);
2408 rspq->m = m_gethdr(M_DONTWAIT, MT_DATA);
2356 else
2409 else
2357 m = m_gethdr(M_NOWAIT, MT_DATA);
2410 m = m_gethdr(M_DONTWAIT, MT_DATA);
2358
2411
2359 if (rspq->m == NULL || m == NULL) {
2412 /*
2413 * XXX revisit me
2414 */
2415 if (rspq->m == NULL && m == NULL) {
2360 rspq->next_holdoff = NOMEM_INTR_DELAY;
2361 budget_left--;
2362 break;
2363 }
2416 rspq->next_holdoff = NOMEM_INTR_DELAY;
2417 budget_left--;
2418 break;
2419 }
2364 get_imm_packet(adap, r, rspq->m, m);
2420 if (get_imm_packet(adap, r, rspq->m, m, flags))
2421 goto skip;
2365 eop = 1;
2366 rspq->imm_data++;
2367 } else if (r->len_cq) {
2368 int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
2369
2370 if (rspq->m == NULL)
2422 eop = 1;
2423 rspq->imm_data++;
2424 } else if (r->len_cq) {
2425 int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
2426
2427 if (rspq->m == NULL)
2371 rspq->m = m_gethdr(M_NOWAIT, MT_DATA);
2428 rspq->m = m_gethdr(M_DONTWAIT, MT_DATA);
2372 if (rspq->m == NULL) {
2373 log(LOG_WARNING, "failed to get mbuf for packet\n");
2374 break;
2375 }
2376
2377 ethpad = 2;
2378 eop = get_packet(adap, drop_thresh, qs, rspq->m, r);
2379 } else {
2380 DPRINTF("pure response\n");
2381 rspq->pure_rsps++;
2382 }
2383
2384 if (flags & RSPD_CTRL_MASK) {
2385 sleeping |= flags & RSPD_GTS_MASK;
2386 handle_rsp_cntrl_info(qs, flags);
2387 }
2429 if (rspq->m == NULL) {
2430 log(LOG_WARNING, "failed to get mbuf for packet\n");
2431 break;
2432 }
2433
2434 ethpad = 2;
2435 eop = get_packet(adap, drop_thresh, qs, rspq->m, r);
2436 } else {
2437 DPRINTF("pure response\n");
2438 rspq->pure_rsps++;
2439 }
2440
2441 if (flags & RSPD_CTRL_MASK) {
2442 sleeping |= flags & RSPD_GTS_MASK;
2443 handle_rsp_cntrl_info(qs, flags);
2444 }
2388
2445 skip:
2389 r++;
2390 if (__predict_false(++rspq->cidx == rspq->size)) {
2391 rspq->cidx = 0;
2392 rspq->gen ^= 1;
2393 r = rspq->desc;
2394 }
2395
2396 prefetch(r);

--- 259 unchanged lines hidden (view full) ---

2656 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2657 "collapse_mbufs",
2658 CTLFLAG_RW, &collapse_mbufs,
2659 0, "collapse mbuf chains into iovecs");
2660 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2661 "txq_overrun",
2662 CTLFLAG_RD, &txq_fills,
2663 0, "#times txq overrun");
2446 r++;
2447 if (__predict_false(++rspq->cidx == rspq->size)) {
2448 rspq->cidx = 0;
2449 rspq->gen ^= 1;
2450 r = rspq->desc;
2451 }
2452
2453 prefetch(r);

--- 259 unchanged lines hidden (view full) ---

2713 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2714 "collapse_mbufs",
2715 CTLFLAG_RW, &collapse_mbufs,
2716 0, "collapse mbuf chains into iovecs");
2717 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2718 "txq_overrun",
2719 CTLFLAG_RD, &txq_fills,
2720 0, "#times txq overrun");
2721 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2722 "bogus_imm",
2723 CTLFLAG_RD, &bogus_imm,
2724 0, "#times a bogus immediate response was seen");
2664}
2665
2666/**
2667 * t3_get_desc - dump an SGE descriptor for debugging purposes
2668 * @qs: the queue set
2669 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
2670 * @idx: the descriptor index in the queue
2671 * @data: where to dump the descriptor contents

--- 31 unchanged lines hidden --- 		2725}
2726
2727/**
2728 * t3_get_desc - dump an SGE descriptor for debugging purposes
2729 * @qs: the queue set
2730 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
2731 * @idx: the descriptor index in the queue
2732 * @data: where to dump the descriptor contents

--- 31 unchanged lines hidden ---