1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
--- 14 unchanged lines hidden (view full) ---
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c 176507 2008-02-24 07:19:31Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/fcntl.h>
36#include <sys/kernel.h>
37#include <sys/limits.h>
38#include <sys/ktr.h>
39#include <sys/lock.h>
--- 593 unchanged lines hidden (view full) ---
633do_rx_urg_notify(struct t3cdev *cdev, struct mbuf *m, void *ctx)
634{
635 struct toepcb *toep = (struct toepcb *)ctx;
636
637 rx_urg_notify(toep, m);
638 return (0);
639}
640
641/*
642 * Set of states for which we should return RX credits.
643 */
644#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2)
645
646/*
647 * Called after some received data has been read. It returns RX credits
648 * to the HW for the amount of data processed.
--- 48 unchanged lines hidden (view full) ---
697 return;
698
699 dev = toep->tp_toedev;
700 thres = TOM_TUNABLE(dev, rx_credit_thres);
701
702 if (__predict_false(thres == 0))
703 return;
704
705 if (toep->tp_ulp_mode)
706 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
707 else {
708 dack_mode = TOM_TUNABLE(dev, delack);
709 if (__predict_false(dack_mode != toep->tp_delack_mode)) {
710 u32 r = tp->rcv_nxt - toep->tp_delack_seq;
711
712 if (r >= tp->rcv_wnd || r >= 16 * toep->tp_mss_clamp)
713 dack = F_RX_DACK_CHANGE |
714 V_RX_DACK_MODE(dack_mode);
715 }
716 }
717
718 /*
719 * For coalescing to work effectively ensure the receive window has
720 * at least 16KB left.
721 */
722 must_send = credits + 16384 >= tp->rcv_wnd;
723
724 if (must_send || credits >= thres)
725 toep->tp_rcv_wup += t3_send_rx_credits(tp, credits, dack, must_send);
--- 156 unchanged lines hidden (view full) ---
882}
883
884void
885t3_set_rcv_coalesce_enable(struct socket *so, int on_off)
886{
887 set_tcb_tflag(so, S_TF_RCV_COALESCE_ENABLE, on_off);
888}
889
890/*
891 * Send a SET_TCB_FIELD CPL message to change a connection's TOS setting.
892 */
893static void
894t3_set_tos(struct socket *so)
895{
896 t3_set_tcb_field(so, W_TCB_TOS, V_TCB_TOS(M_TCB_TOS),
897 V_TCB_TOS(SO_TOS(so)));
--- 1149 unchanged lines hidden (view full) ---
2047{
2048 struct tcpcb *tp;
2049 struct ddp_state *q;
2050 struct ddp_buf_state *bsp;
2051 struct cpl_rx_data_ddp *hdr;
2052 unsigned int ddp_len, rcv_nxt, ddp_report, end_offset, buf_idx;
2053 struct socket *so = toeptoso(toep);
2054 int nomoredata = 0;
2055
2056 tp = sototcpcb(so);
2057
2058 INP_LOCK(tp->t_inpcb);
2059 if (__predict_false(so_no_receive(so))) {
2060
2061 handle_excess_rx(toep, m);
2062 INP_UNLOCK(tp->t_inpcb);
--- 23 unchanged lines hidden (view full) ---
2086 ntohs(hdr->len));
2087 CTR3(KTR_TOM,
2088 "new_rx_data_ddp: offset %u ddp_report 0x%x buf_idx=%d",
2089 G_DDP_OFFSET(ddp_report), ddp_report, buf_idx);
2090
2091 ddp_len = ntohs(hdr->len);
2092 rcv_nxt = ntohl(hdr->seq) + ddp_len;
2093
2094 m->m_seq = tp->rcv_nxt;
2095 tp->rcv_nxt = rcv_nxt;
2096
2097 tp->t_rcvtime = ticks;
2098 /*
2099 * Store the length in m->m_len. We are changing the meaning of
2100 * m->m_len here, we need to be very careful that nothing from now on
2101 * interprets ->len of this packet the usual way.
--- 46 unchanged lines hidden (view full) ---
2148 bsp->flags &= ~DDP_BF_NOCOPY;
2149 }
2150
2151 if (ddp_report & F_DDP_PSH)
2152 m->m_ddp_flags |= DDP_BF_PSH;
2153 if (nomoredata)
2154 m->m_ddp_flags |= DDP_BF_NODATA;
2155
2156 if (__predict_false(G_DDP_DACK_MODE(ddp_report) != toep->tp_delack_mode)) {
2157 toep->tp_delack_mode = G_DDP_DACK_MODE(ddp_report);
2158 toep->tp_delack_seq = tp->rcv_nxt;
2159 }
2160
2161 SBAPPEND(&so->so_rcv, m);
2162
2163 if ((so->so_state & SS_NOFDREF) == 0)
2164 sorwakeup_locked(so);
2165 else
2166 SOCKBUF_UNLOCK(&so->so_rcv);
2167}
2168
--- 28 unchanged lines hidden (view full) ---
2197static void
2198process_ddp_complete(struct toepcb *toep, struct mbuf *m)
2199{
2200 struct tcpcb *tp = toep->tp_tp;
2201 struct socket *so = toeptoso(toep);
2202 struct ddp_state *q;
2203 struct ddp_buf_state *bsp;
2204 struct cpl_rx_ddp_complete *hdr;
2205 unsigned int ddp_report, buf_idx, when;
2206 int nomoredata = 0;
2207
2208 INP_LOCK(tp->t_inpcb);
2209 if (__predict_false(so_no_receive(so))) {
2210 struct inpcb *inp = sotoinpcb(so);
2211
2212 handle_excess_rx(toep, m);
2213 INP_UNLOCK(inp);
--- 7 unchanged lines hidden (view full) ---
2221
2222
2223 SOCKBUF_LOCK(&so->so_rcv);
2224 bsp = &q->buf_state[buf_idx];
2225 when = bsp->cur_offset;
2226 m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when;
2227 tp->rcv_nxt += m->m_len;
2228 tp->t_rcvtime = ticks;
2229 INP_UNLOCK(tp->t_inpcb);
2230
2231 KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
2232#ifdef T3_TRACE
2233 T3_TRACE5(TIDTB(sk),
2234 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2235 "ddp_report 0x%x offset %u, len %u",
2236 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2237 G_DDP_OFFSET(ddp_report), skb->len);
2238#endif
2239 CTR5(KTR_TOM,
2240 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2241 "ddp_report 0x%x offset %u, len %u",
2242 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2243 G_DDP_OFFSET(ddp_report), m->m_len);
2244
2245 bsp->cur_offset += m->m_len;
2246
2247 if (!(bsp->flags & DDP_BF_NOFLIP)) {
2248 q->cur_buf ^= 1; /* flip buffers */
2249 if (G_DDP_OFFSET(ddp_report) < q->kbuf[0]->dgl_length)
2250 nomoredata=1;
2251 }
2252
2253#ifdef T3_TRACE
2254 T3_TRACE4(TIDTB(sk),
2255 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2256 "ddp_report %u offset %u",
2257 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2258 G_DDP_OFFSET(ddp_report));
2259#endif
2260 CTR4(KTR_TOM,
2261 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2262 "ddp_report %u offset %u",
2263 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2264 G_DDP_OFFSET(ddp_report));
2265
2266 m->m_ddp_gl = (unsigned char *)bsp->gl;
2267 m->m_flags |= M_DDP;
2268 m->m_ddp_flags = (bsp->flags & DDP_BF_NOCOPY) | 1;
2269 if (bsp->flags & DDP_BF_NOCOPY)
2270 bsp->flags &= ~DDP_BF_NOCOPY;
2271 if (nomoredata)
2272 m->m_ddp_flags |= DDP_BF_NODATA;
2273
2274 SBAPPEND(&so->so_rcv, m);
2275
2276 if ((so->so_state & SS_NOFDREF) == 0)
2277 sorwakeup_locked(so);
2278 else
2279 SOCKBUF_UNLOCK(&so->so_rcv);
2280}
2281
--- 82 unchanged lines hidden (view full) ---
2364 m->m_cur_offset = bsp->cur_offset;
2365 m->m_ddp_flags =
2366 DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1;
2367 m->m_seq = tp->rcv_nxt;
2368 tp->rcv_nxt = rcv_nxt;
2369 bsp->cur_offset += m->m_pkthdr.len;
2370 if (!(bsp->flags & DDP_BF_NOFLIP))
2371 q->cur_buf ^= 1;
2372 tp->t_rcvtime = ticks;
2373 SBAPPEND(&so->so_rcv, m);
2374 if (__predict_true((so->so_state & SS_NOFDREF) == 0))
2375 sorwakeup_locked(so);
2376 else
2377 SOCKBUF_UNLOCK(&so->so_rcv);
2378 return (1);
2379}
--- 1621 unchanged lines hidden (view full) ---
4001 req->mask = htobe64(mask);
4002 req->val = htobe64(val);
4003}
4004
4005/*
4006 * Build a CPL_RX_DATA_ACK message as payload of a ULP_TX_PKT command.
4007 */
4008static void
4009mk_rx_data_ack_ulp(struct cpl_rx_data_ack *ack, unsigned int tid, unsigned int credits)
4010{
4011 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)ack;
4012
4013 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
4014 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*ack) / 8));
4015 OPCODE_TID(ack) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid));
4016 ack->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE |
4017 V_RX_DACK_MODE(1) | V_RX_CREDITS(credits));
4018}
4019
4020void
4021t3_cancel_ddpbuf(struct toepcb *toep, unsigned int bufidx)
4022{
4023 unsigned int wrlen;
4024 struct mbuf *m;
4025 struct work_request_hdr *wr;
--- 185 unchanged lines hidden (view full) ---
4211 V_TCB_RX_DDP_BUF1_LEN((uint64_t)len1) << 32);
4212 req++;
4213 }
4214
4215 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, flag_mask,
4216 ddp_flags);
4217
4218 if (modulate) {
4219 mk_rx_data_ack_ulp((struct cpl_rx_data_ack *)(req + 1), toep->tp_tid,
4220 toep->tp_copied_seq - toep->tp_rcv_wup);
4221 toep->tp_rcv_wup = toep->tp_copied_seq;
4222 }
4223
4224#ifdef T3_TRACE
4225 T3_TRACE5(TIDTB(sk),
4226 "t3_setup_ddpbufs: len0 %u len1 %u ddp_flags 0x%08x%08x "
4227 "modulate %d",
4228 len0, len1, ddp_flags >> 32, ddp_flags & 0xffffffff,
--- 58 unchanged lines hidden ---
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
--- 14 unchanged lines hidden (view full) ---
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c 176507 2008-02-24 07:19:31Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/fcntl.h>
36#include <sys/kernel.h>
37#include <sys/limits.h>
38#include <sys/ktr.h>
39#include <sys/lock.h>
--- 593 unchanged lines hidden (view full) ---
633do_rx_urg_notify(struct t3cdev *cdev, struct mbuf *m, void *ctx)
634{
635 struct toepcb *toep = (struct toepcb *)ctx;
636
637 rx_urg_notify(toep, m);
638 return (0);
639}
640
641/*
642 * Set of states for which we should return RX credits.
643 */
644#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2)
645
646/*
647 * Called after some received data has been read. It returns RX credits
648 * to the HW for the amount of data processed.
--- 48 unchanged lines hidden (view full) ---
697 return;
698
699 dev = toep->tp_toedev;
700 thres = TOM_TUNABLE(dev, rx_credit_thres);
701
702 if (__predict_false(thres == 0))
703 return;
704
705 if (toep->tp_ulp_mode)
706 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
707 else {
708 dack_mode = TOM_TUNABLE(dev, delack);
709 if (__predict_false(dack_mode != toep->tp_delack_mode)) {
710 u32 r = tp->rcv_nxt - toep->tp_delack_seq;
711
712 if (r >= tp->rcv_wnd || r >= 16 * toep->tp_mss_clamp)
713 dack = F_RX_DACK_CHANGE |
714 V_RX_DACK_MODE(dack_mode);
715 }
716 }
717
718 /*
719 * For coalescing to work effectively ensure the receive window has
720 * at least 16KB left.
721 */
722 must_send = credits + 16384 >= tp->rcv_wnd;
723
724 if (must_send || credits >= thres)
725 toep->tp_rcv_wup += t3_send_rx_credits(tp, credits, dack, must_send);
--- 156 unchanged lines hidden (view full) ---
882}
883
884void
885t3_set_rcv_coalesce_enable(struct socket *so, int on_off)
886{
887 set_tcb_tflag(so, S_TF_RCV_COALESCE_ENABLE, on_off);
888}
889
890/*
891 * Send a SET_TCB_FIELD CPL message to change a connection's TOS setting.
892 */
893static void
894t3_set_tos(struct socket *so)
895{
896 t3_set_tcb_field(so, W_TCB_TOS, V_TCB_TOS(M_TCB_TOS),
897 V_TCB_TOS(SO_TOS(so)));
--- 1149 unchanged lines hidden (view full) ---
2047{
2048 struct tcpcb *tp;
2049 struct ddp_state *q;
2050 struct ddp_buf_state *bsp;
2051 struct cpl_rx_data_ddp *hdr;
2052 unsigned int ddp_len, rcv_nxt, ddp_report, end_offset, buf_idx;
2053 struct socket *so = toeptoso(toep);
2054 int nomoredata = 0;
2055
2056 tp = sototcpcb(so);
2057
2058 INP_LOCK(tp->t_inpcb);
2059 if (__predict_false(so_no_receive(so))) {
2060
2061 handle_excess_rx(toep, m);
2062 INP_UNLOCK(tp->t_inpcb);
--- 23 unchanged lines hidden (view full) ---
2086 ntohs(hdr->len));
2087 CTR3(KTR_TOM,
2088 "new_rx_data_ddp: offset %u ddp_report 0x%x buf_idx=%d",
2089 G_DDP_OFFSET(ddp_report), ddp_report, buf_idx);
2090
2091 ddp_len = ntohs(hdr->len);
2092 rcv_nxt = ntohl(hdr->seq) + ddp_len;
2093
2094 m->m_seq = tp->rcv_nxt;
2095 tp->rcv_nxt = rcv_nxt;
2096
2097 tp->t_rcvtime = ticks;
2098 /*
2099 * Store the length in m->m_len. We are changing the meaning of
2100 * m->m_len here, we need to be very careful that nothing from now on
2101 * interprets ->len of this packet the usual way.
--- 46 unchanged lines hidden (view full) ---
2148 bsp->flags &= ~DDP_BF_NOCOPY;
2149 }
2150
2151 if (ddp_report & F_DDP_PSH)
2152 m->m_ddp_flags |= DDP_BF_PSH;
2153 if (nomoredata)
2154 m->m_ddp_flags |= DDP_BF_NODATA;
2155
2156 if (__predict_false(G_DDP_DACK_MODE(ddp_report) != toep->tp_delack_mode)) {
2157 toep->tp_delack_mode = G_DDP_DACK_MODE(ddp_report);
2158 toep->tp_delack_seq = tp->rcv_nxt;
2159 }
2160
2161 SBAPPEND(&so->so_rcv, m);
2162
2163 if ((so->so_state & SS_NOFDREF) == 0)
2164 sorwakeup_locked(so);
2165 else
2166 SOCKBUF_UNLOCK(&so->so_rcv);
2167}
2168
--- 28 unchanged lines hidden (view full) ---
2197static void
2198process_ddp_complete(struct toepcb *toep, struct mbuf *m)
2199{
2200 struct tcpcb *tp = toep->tp_tp;
2201 struct socket *so = toeptoso(toep);
2202 struct ddp_state *q;
2203 struct ddp_buf_state *bsp;
2204 struct cpl_rx_ddp_complete *hdr;
2205 unsigned int ddp_report, buf_idx, when;
2206 int nomoredata = 0;
2207
2208 INP_LOCK(tp->t_inpcb);
2209 if (__predict_false(so_no_receive(so))) {
2210 struct inpcb *inp = sotoinpcb(so);
2211
2212 handle_excess_rx(toep, m);
2213 INP_UNLOCK(inp);
--- 7 unchanged lines hidden (view full) ---
2221
2222
2223 SOCKBUF_LOCK(&so->so_rcv);
2224 bsp = &q->buf_state[buf_idx];
2225 when = bsp->cur_offset;
2226 m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when;
2227 tp->rcv_nxt += m->m_len;
2228 tp->t_rcvtime = ticks;
2229 INP_UNLOCK(tp->t_inpcb);
2230
2231 KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
2232#ifdef T3_TRACE
2233 T3_TRACE5(TIDTB(sk),
2234 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2235 "ddp_report 0x%x offset %u, len %u",
2236 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2237 G_DDP_OFFSET(ddp_report), skb->len);
2238#endif
2239 CTR5(KTR_TOM,
2240 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2241 "ddp_report 0x%x offset %u, len %u",
2242 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2243 G_DDP_OFFSET(ddp_report), m->m_len);
2244
2245 bsp->cur_offset += m->m_len;
2246
2247 if (!(bsp->flags & DDP_BF_NOFLIP)) {
2248 q->cur_buf ^= 1; /* flip buffers */
2249 if (G_DDP_OFFSET(ddp_report) < q->kbuf[0]->dgl_length)
2250 nomoredata=1;
2251 }
2252
2253#ifdef T3_TRACE
2254 T3_TRACE4(TIDTB(sk),
2255 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2256 "ddp_report %u offset %u",
2257 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2258 G_DDP_OFFSET(ddp_report));
2259#endif
2260 CTR4(KTR_TOM,
2261 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2262 "ddp_report %u offset %u",
2263 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2264 G_DDP_OFFSET(ddp_report));
2265
2266 m->m_ddp_gl = (unsigned char *)bsp->gl;
2267 m->m_flags |= M_DDP;
2268 m->m_ddp_flags = (bsp->flags & DDP_BF_NOCOPY) | 1;
2269 if (bsp->flags & DDP_BF_NOCOPY)
2270 bsp->flags &= ~DDP_BF_NOCOPY;
2271 if (nomoredata)
2272 m->m_ddp_flags |= DDP_BF_NODATA;
2273
2274 SBAPPEND(&so->so_rcv, m);
2275
2276 if ((so->so_state & SS_NOFDREF) == 0)
2277 sorwakeup_locked(so);
2278 else
2279 SOCKBUF_UNLOCK(&so->so_rcv);
2280}
2281
--- 82 unchanged lines hidden (view full) ---
2364 m->m_cur_offset = bsp->cur_offset;
2365 m->m_ddp_flags =
2366 DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1;
2367 m->m_seq = tp->rcv_nxt;
2368 tp->rcv_nxt = rcv_nxt;
2369 bsp->cur_offset += m->m_pkthdr.len;
2370 if (!(bsp->flags & DDP_BF_NOFLIP))
2371 q->cur_buf ^= 1;
2372 tp->t_rcvtime = ticks;
2373 SBAPPEND(&so->so_rcv, m);
2374 if (__predict_true((so->so_state & SS_NOFDREF) == 0))
2375 sorwakeup_locked(so);
2376 else
2377 SOCKBUF_UNLOCK(&so->so_rcv);
2378 return (1);
2379}
--- 1621 unchanged lines hidden (view full) ---
4001 req->mask = htobe64(mask);
4002 req->val = htobe64(val);
4003}
4004
4005/*
4006 * Build a CPL_RX_DATA_ACK message as payload of a ULP_TX_PKT command.
4007 */
4008static void
4009mk_rx_data_ack_ulp(struct cpl_rx_data_ack *ack, unsigned int tid, unsigned int credits)
4010{
4011 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)ack;
4012
4013 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
4014 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*ack) / 8));
4015 OPCODE_TID(ack) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid));
4016 ack->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE |
4017 V_RX_DACK_MODE(1) | V_RX_CREDITS(credits));
4018}
4019
4020void
4021t3_cancel_ddpbuf(struct toepcb *toep, unsigned int bufidx)
4022{
4023 unsigned int wrlen;
4024 struct mbuf *m;
4025 struct work_request_hdr *wr;
--- 185 unchanged lines hidden (view full) ---
4211 V_TCB_RX_DDP_BUF1_LEN((uint64_t)len1) << 32);
4212 req++;
4213 }
4214
4215 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, flag_mask,
4216 ddp_flags);
4217
4218 if (modulate) {
4219 mk_rx_data_ack_ulp((struct cpl_rx_data_ack *)(req + 1), toep->tp_tid,
4220 toep->tp_copied_seq - toep->tp_rcv_wup);
4221 toep->tp_rcv_wup = toep->tp_copied_seq;
4222 }
4223
4224#ifdef T3_TRACE
4225 T3_TRACE5(TIDTB(sk),
4226 "t3_setup_ddpbufs: len0 %u len1 %u ddp_flags 0x%08x%08x "
4227 "modulate %d",
4228 len0, len1, ddp_flags >> 32, ddp_flags & 0xffffffff,
--- 58 unchanged lines hidden ---