Deleted Added
sdiff udiff text old ( 174712 ) new ( 176472 )
full compact
cxgb_cpl_io.c (174712) cxgb_cpl_io.c (176472)
1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8

--- 14 unchanged lines hidden (view full) ---

23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8

--- 14 unchanged lines hidden (view full) ---

23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c 174712 2007-12-17 10:02:29Z kmacy $");
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c 176472 2008-02-23 01:06:17Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/fcntl.h>
36#include <sys/kernel.h>
37#include <sys/limits.h>
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/fcntl.h>
36#include <sys/kernel.h>
37#include <sys/limits.h>
38#include <sys/ktr.h>
38#include <sys/lock.h>
39#include <sys/mbuf.h>
40#include <sys/mutex.h>
41#include <sys/socket.h>
42#include <sys/sysctl.h>
43#include <sys/syslog.h>
44#include <sys/socketvar.h>
45#include <sys/protosw.h>

--- 12 unchanged lines hidden (view full) ---

58#include <dev/cxgb/sys/mbufq.h>
59
60#include <netinet/ip.h>
61#include <netinet/tcp_var.h>
62#include <netinet/tcp_fsm.h>
63#include <netinet/tcp_offload.h>
64#include <netinet/tcp_seq.h>
65#include <netinet/tcp_syncache.h>
39#include <sys/lock.h>
40#include <sys/mbuf.h>
41#include <sys/mutex.h>
42#include <sys/socket.h>
43#include <sys/sysctl.h>
44#include <sys/syslog.h>
45#include <sys/socketvar.h>
46#include <sys/protosw.h>

--- 12 unchanged lines hidden (view full) ---

59#include <dev/cxgb/sys/mbufq.h>
60
61#include <netinet/ip.h>
62#include <netinet/tcp_var.h>
63#include <netinet/tcp_fsm.h>
64#include <netinet/tcp_offload.h>
65#include <netinet/tcp_seq.h>
66#include <netinet/tcp_syncache.h>
67#include <netinet/tcp_timer.h>
66#include <net/route.h>
67
68#include <net/route.h>
69
68
69#include <dev/cxgb/t3cdev.h>
70#include <dev/cxgb/common/cxgb_firmware_exports.h>
71#include <dev/cxgb/common/cxgb_t3_cpl.h>
72#include <dev/cxgb/common/cxgb_tcb.h>
73#include <dev/cxgb/common/cxgb_ctl_defs.h>
74#include <dev/cxgb/cxgb_l2t.h>
75#include <dev/cxgb/cxgb_offload.h>
76#include <vm/vm.h>
77#include <vm/pmap.h>
78#include <machine/bus.h>
79#include <dev/cxgb/sys/mvec.h>
80#include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
81#include <dev/cxgb/ulp/tom/cxgb_defs.h>
82#include <dev/cxgb/ulp/tom/cxgb_tom.h>
83#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h>
84#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
85#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
86
70#include <dev/cxgb/t3cdev.h>
71#include <dev/cxgb/common/cxgb_firmware_exports.h>
72#include <dev/cxgb/common/cxgb_t3_cpl.h>
73#include <dev/cxgb/common/cxgb_tcb.h>
74#include <dev/cxgb/common/cxgb_ctl_defs.h>
75#include <dev/cxgb/cxgb_l2t.h>
76#include <dev/cxgb/cxgb_offload.h>
77#include <vm/vm.h>
78#include <vm/pmap.h>
79#include <machine/bus.h>
80#include <dev/cxgb/sys/mvec.h>
81#include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
82#include <dev/cxgb/ulp/tom/cxgb_defs.h>
83#include <dev/cxgb/ulp/tom/cxgb_tom.h>
84#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h>
85#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
86#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
87
87
88
89/*
90 * For ULP connections HW may add headers, e.g., for digests, that aren't part
91 * of the messages sent by the host but that are part of the TCP payload and
92 * therefore consume TCP sequence space. Tx connection parameters that
93 * operate in TCP sequence space are affected by the HW additions and need to
94 * compensate for them to accurately track TCP sequence numbers. This array
95 * contains the compensating extra lengths for ULP packets. It is indexed by
96 * a packet's ULP submode.

--- 16 unchanged lines hidden (view full) ---

113 */
114static unsigned int wrlen __read_mostly;
115
116/*
117 * The number of WRs needed for an skb depends on the number of page fragments
118 * in the skb and whether it has any payload in its main body. This maps the
119 * length of the gather list represented by an skb into the # of necessary WRs.
120 */
88/*
89 * For ULP connections HW may add headers, e.g., for digests, that aren't part
90 * of the messages sent by the host but that are part of the TCP payload and
91 * therefore consume TCP sequence space. Tx connection parameters that
92 * operate in TCP sequence space are affected by the HW additions and need to
93 * compensate for them to accurately track TCP sequence numbers. This array
94 * contains the compensating extra lengths for ULP packets. It is indexed by
95 * a packet's ULP submode.

--- 16 unchanged lines hidden (view full) ---

112 */
113static unsigned int wrlen __read_mostly;
114
115/*
116 * The number of WRs needed for an skb depends on the number of page fragments
117 * in the skb and whether it has any payload in its main body. This maps the
118 * length of the gather list represented by an skb into the # of necessary WRs.
119 */
121static unsigned int mbuf_wrs[TX_MAX_SEGS] __read_mostly;
120static unsigned int mbuf_wrs[TX_MAX_SEGS + 1] __read_mostly;
122
123/*
124 * Max receive window supported by HW in bytes. Only a small part of it can
125 * be set through option0, the rest needs to be set through RX_DATA_ACK.
126 */
127#define MAX_RCV_WND ((1U << 27) - 1)
128
129/*

--- 12 unchanged lines hidden (view full) ---

142extern int tcp_autorcvbuf_max;
143extern int tcp_autosndbuf_max;
144
145static void t3_send_reset(struct toepcb *toep);
146static void send_abort_rpl(struct mbuf *m, struct toedev *tdev, int rst_status);
147static inline void free_atid(struct t3cdev *cdev, unsigned int tid);
148static void handle_syncache_event(int event, void *arg);
149
121
122/*
123 * Max receive window supported by HW in bytes. Only a small part of it can
124 * be set through option0, the rest needs to be set through RX_DATA_ACK.
125 */
126#define MAX_RCV_WND ((1U << 27) - 1)
127
128/*

--- 12 unchanged lines hidden (view full) ---

141extern int tcp_autorcvbuf_max;
142extern int tcp_autosndbuf_max;
143
144static void t3_send_reset(struct toepcb *toep);
145static void send_abort_rpl(struct mbuf *m, struct toedev *tdev, int rst_status);
146static inline void free_atid(struct t3cdev *cdev, unsigned int tid);
147static void handle_syncache_event(int event, void *arg);
148
149static inline void
150SBAPPEND(struct sockbuf *sb, struct mbuf *n)
151{
152 struct mbuf * m;
150
153
154 m = sb->sb_mb;
155 while (m) {
156 KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) ||
157 !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n",
158 !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len));
159 KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x",
160 m->m_next, m->m_nextpkt, m->m_flags));
161 m = m->m_next;
162 }
163 m = n;
164 while (m) {
165 KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) ||
166 !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n",
167 !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len));
168 KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x",
169 m->m_next, m->m_nextpkt, m->m_flags));
170 m = m->m_next;
171 }
172 sbappend_locked(sb, n);
173 m = sb->sb_mb;
174 while (m) {
175 KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x",
176 m->m_next, m->m_nextpkt, m->m_flags));
177 m = m->m_next;
178 }
179}
180
151static inline int
152is_t3a(const struct toedev *dev)
153{
154 return (dev->tod_ttid == TOE_ID_CHELSIO_T3);
155}
156
157static void
158dump_toepcb(struct toepcb *toep)
159{
160 DPRINTF("qset_idx=%d qset=%d ulp_mode=%d mtu_idx=%d tid=%d\n",
161 toep->tp_qset_idx, toep->tp_qset, toep->tp_ulp_mode,
162 toep->tp_mtu_idx, toep->tp_tid);
163
164 DPRINTF("wr_max=%d wr_avail=%d wr_unacked=%d mss_clamp=%d flags=0x%x\n",
165 toep->tp_wr_max, toep->tp_wr_avail, toep->tp_wr_unacked,
166 toep->tp_mss_clamp, toep->tp_flags);
167}
168
181static inline int
182is_t3a(const struct toedev *dev)
183{
184 return (dev->tod_ttid == TOE_ID_CHELSIO_T3);
185}
186
187static void
188dump_toepcb(struct toepcb *toep)
189{
190 DPRINTF("qset_idx=%d qset=%d ulp_mode=%d mtu_idx=%d tid=%d\n",
191 toep->tp_qset_idx, toep->tp_qset, toep->tp_ulp_mode,
192 toep->tp_mtu_idx, toep->tp_tid);
193
194 DPRINTF("wr_max=%d wr_avail=%d wr_unacked=%d mss_clamp=%d flags=0x%x\n",
195 toep->tp_wr_max, toep->tp_wr_avail, toep->tp_wr_unacked,
196 toep->tp_mss_clamp, toep->tp_flags);
197}
198
199#ifndef RTALLOC2_DEFINED
169static struct rtentry *
170rtalloc2(struct sockaddr *dst, int report, u_long ignflags)
171{
172 struct rtentry *rt = NULL;
173
174 if ((rt = rtalloc1(dst, report, ignflags)) != NULL)
175 RT_UNLOCK(rt);
176
177 return (rt);
178}
200static struct rtentry *
201rtalloc2(struct sockaddr *dst, int report, u_long ignflags)
202{
203 struct rtentry *rt = NULL;
204
205 if ((rt = rtalloc1(dst, report, ignflags)) != NULL)
206 RT_UNLOCK(rt);
207
208 return (rt);
209}
179
210#endif
180/*
181 * Determine whether to send a CPL message now or defer it. A message is
182 * deferred if the connection is in SYN_SENT since we don't know the TID yet.
183 * For connections in other states the message is sent immediately.
184 * If through_l2t is set the message is subject to ARP processing, otherwise
185 * it is sent directly.
186 */
187static inline void
211/*
212 * Determine whether to send a CPL message now or defer it. A message is
213 * deferred if the connection is in SYN_SENT since we don't know the TID yet.
214 * For connections in other states the message is sent immediately.
215 * If through_l2t is set the message is subject to ARP processing, otherwise
216 * it is sent directly.
217 */
218static inline void
188send_or_defer(struct socket *so, struct tcpcb *tp, struct mbuf *m, int through_l2t)
219send_or_defer(struct toepcb *toep, struct mbuf *m, int through_l2t)
189{
220{
190 struct toepcb *toep = tp->t_toe;
221 struct tcpcb *tp = toep->tp_tp;
191
222
192
193 if (__predict_false(tp->t_state == TCPS_SYN_SENT)) {
194 INP_LOCK(tp->t_inpcb);
195 mbufq_tail(&toep->out_of_order_queue, m); // defer
196 INP_UNLOCK(tp->t_inpcb);
197 } else if (through_l2t)
223 if (__predict_false(tp->t_state == TCPS_SYN_SENT)) {
224 INP_LOCK(tp->t_inpcb);
225 mbufq_tail(&toep->out_of_order_queue, m); // defer
226 INP_UNLOCK(tp->t_inpcb);
227 } else if (through_l2t)
198 l2t_send(T3C_DEV(so), m, toep->tp_l2t); // send through L2T
228 l2t_send(TOEP_T3C_DEV(toep), m, toep->tp_l2t); // send through L2T
199 else
229 else
200 cxgb_ofld_send(T3C_DEV(so), m); // send directly
230 cxgb_ofld_send(TOEP_T3C_DEV(toep), m); // send directly
201}
202
203static inline unsigned int
231}
232
233static inline unsigned int
204mkprio(unsigned int cntrl, const struct socket *so)
234mkprio(unsigned int cntrl, const struct toepcb *toep)
205{
235{
206 return cntrl;
236 return (cntrl);
207}
208
209/*
210 * Populate a TID_RELEASE WR. The skb must be already propely sized.
211 */
212static inline void
237}
238
239/*
240 * Populate a TID_RELEASE WR. The skb must be already propely sized.
241 */
242static inline void
213mk_tid_release(struct mbuf *m, const struct socket *so, unsigned int tid)
243mk_tid_release(struct mbuf *m, const struct toepcb *toep, unsigned int tid)
214{
215 struct cpl_tid_release *req;
216
244{
245 struct cpl_tid_release *req;
246
217 m_set_priority(m, mkprio(CPL_PRIORITY_SETUP, so));
247 m_set_priority(m, mkprio(CPL_PRIORITY_SETUP, toep));
218 m->m_pkthdr.len = m->m_len = sizeof(*req);
219 req = mtod(m, struct cpl_tid_release *);
220 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
248 m->m_pkthdr.len = m->m_len = sizeof(*req);
249 req = mtod(m, struct cpl_tid_release *);
250 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
251 req->wr.wr_lo = 0;
221 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
222}
223
224static inline void
225make_tx_data_wr(struct socket *so, struct mbuf *m, int len, struct mbuf *tail)
226{
227 struct tcpcb *tp = sototcpcb(so);
228 struct toepcb *toep = tp->t_toe;

--- 23 unchanged lines hidden (view full) ---

252 if (tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE)
253 req->param |= htonl(V_TX_SNDBUF(tcp_autosndbuf_max >> 15));
254 else
255 req->param |= htonl(V_TX_SNDBUF(so->so_snd.sb_hiwat >> 15));
256 toep->tp_flags |= TP_DATASENT;
257 }
258}
259
252 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
253}
254
255static inline void
256make_tx_data_wr(struct socket *so, struct mbuf *m, int len, struct mbuf *tail)
257{
258 struct tcpcb *tp = sototcpcb(so);
259 struct toepcb *toep = tp->t_toe;

--- 23 unchanged lines hidden (view full) ---

283 if (tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE)
284 req->param |= htonl(V_TX_SNDBUF(tcp_autosndbuf_max >> 15));
285 else
286 req->param |= htonl(V_TX_SNDBUF(so->so_snd.sb_hiwat >> 15));
287 toep->tp_flags |= TP_DATASENT;
288 }
289}
290
291#define IMM_LEN 64 /* XXX - see WR_LEN in the cxgb driver */
292
260int
261t3_push_frames(struct socket *so, int req_completion)
262{
263 struct tcpcb *tp = sototcpcb(so);
264 struct toepcb *toep = tp->t_toe;
265
266 struct mbuf *tail, *m0, *last;
267 struct t3cdev *cdev;
268 struct tom_data *d;
293int
294t3_push_frames(struct socket *so, int req_completion)
295{
296 struct tcpcb *tp = sototcpcb(so);
297 struct toepcb *toep = tp->t_toe;
298
299 struct mbuf *tail, *m0, *last;
300 struct t3cdev *cdev;
301 struct tom_data *d;
269 int bytes, count, total_bytes;
302 int i, bytes, count, total_bytes;
270 bus_dma_segment_t segs[TX_MAX_SEGS], *segp;
303 bus_dma_segment_t segs[TX_MAX_SEGS], *segp;
271 segp = segs;
272
273 if (tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_CLOSED) {
274 DPRINTF("tcp state=%d\n", tp->t_state);
275 return (0);
276 }
277
278 if (so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)) {
279 DPRINTF("disconnecting\n");
280
281 return (0);
282 }
283
304
305 if (tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_CLOSED) {
306 DPRINTF("tcp state=%d\n", tp->t_state);
307 return (0);
308 }
309
310 if (so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)) {
311 DPRINTF("disconnecting\n");
312
313 return (0);
314 }
315
284 INP_LOCK_ASSERT(tp->t_inpcb);
285
316
317 INP_LOCK_ASSERT(tp->t_inpcb);
286 SOCKBUF_LOCK(&so->so_snd);
318 SOCKBUF_LOCK(&so->so_snd);
287
288 d = TOM_DATA(TOE_DEV(so));
289 cdev = d->cdev;
290 last = tail = so->so_snd.sb_sndptr ? so->so_snd.sb_sndptr : so->so_snd.sb_mb;
291 total_bytes = 0;
292 DPRINTF("wr_avail=%d tail=%p snd.cc=%d tp_last=%p\n",
293 toep->tp_wr_avail, tail, so->so_snd.sb_cc, toep->tp_m_last);
294
295 if (last && toep->tp_m_last == last && so->so_snd.sb_sndptroff != 0) {

--- 5 unchanged lines hidden (view full) ---

301 DPRINTF("wr_avail=%d tail=%p\n", toep->tp_wr_avail, tail);
302 SOCKBUF_UNLOCK(&so->so_snd);
303 return (0);
304 }
305
306 toep->tp_m_last = NULL;
307 while (toep->tp_wr_avail && (tail != NULL)) {
308 count = bytes = 0;
319 d = TOM_DATA(TOE_DEV(so));
320 cdev = d->cdev;
321 last = tail = so->so_snd.sb_sndptr ? so->so_snd.sb_sndptr : so->so_snd.sb_mb;
322 total_bytes = 0;
323 DPRINTF("wr_avail=%d tail=%p snd.cc=%d tp_last=%p\n",
324 toep->tp_wr_avail, tail, so->so_snd.sb_cc, toep->tp_m_last);
325
326 if (last && toep->tp_m_last == last && so->so_snd.sb_sndptroff != 0) {

--- 5 unchanged lines hidden (view full) ---

332 DPRINTF("wr_avail=%d tail=%p\n", toep->tp_wr_avail, tail);
333 SOCKBUF_UNLOCK(&so->so_snd);
334 return (0);
335 }
336
337 toep->tp_m_last = NULL;
338 while (toep->tp_wr_avail && (tail != NULL)) {
339 count = bytes = 0;
340 segp = segs;
309 if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) {
310 SOCKBUF_UNLOCK(&so->so_snd);
311 return (0);
312 }
341 if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) {
342 SOCKBUF_UNLOCK(&so->so_snd);
343 return (0);
344 }
313 while ((mbuf_wrs[count + 1] <= toep->tp_wr_avail)
314 && (tail != NULL) && (count < TX_MAX_SEGS)) {
315 bytes += tail->m_len;
316 count++;
345 /*
346 * If the data in tail fits as in-line, then
347 * make an immediate data wr.
348 */
349 if (tail->m_len <= IMM_LEN) {
350 count = 1;
351 bytes = tail->m_len;
317 last = tail;
352 last = tail;
318 /*
319 * technically an abuse to be using this for a VA
320 * but less gross than defining my own structure
321 * or calling pmap_kextract from here :-|
322 */
323 segp->ds_addr = (bus_addr_t)tail->m_data;
324 segp->ds_len = tail->m_len;
325 DPRINTF("count=%d wr_needed=%d ds_addr=%p ds_len=%d\n",
326 count, mbuf_wrs[count], tail->m_data, tail->m_len);
327
328 segp++;
329 tail = tail->m_next;
353 tail = tail->m_next;
354 m_set_sgl(m0, NULL);
355 m_set_sgllen(m0, 0);
356 make_tx_data_wr(so, m0, bytes, tail);
357 m_append(m0, bytes, mtod(last, caddr_t));
358 KASSERT(!m0->m_next, ("bad append"));
359 } else {
360 while ((mbuf_wrs[count + 1] <= toep->tp_wr_avail)
361 && (tail != NULL) && (count < TX_MAX_SEGS-1)) {
362 bytes += tail->m_len;
363 last = tail;
364 count++;
365 /*
366 * technically an abuse to be using this for a VA
367 * but less gross than defining my own structure
368 * or calling pmap_kextract from here :-|
369 */
370 segp->ds_addr = (bus_addr_t)tail->m_data;
371 segp->ds_len = tail->m_len;
372 DPRINTF("count=%d wr_needed=%d ds_addr=%p ds_len=%d\n",
373 count, mbuf_wrs[count], tail->m_data, tail->m_len);
374 segp++;
375 tail = tail->m_next;
376 }
377 DPRINTF("wr_avail=%d mbuf_wrs[%d]=%d tail=%p\n",
378 toep->tp_wr_avail, count, mbuf_wrs[count], tail);
379
380 m_set_sgl(m0, segs);
381 m_set_sgllen(m0, count);
382 make_tx_data_wr(so, m0, bytes, tail);
330 }
383 }
331 DPRINTF("wr_avail=%d mbuf_wrs[%d]=%d tail=%p\n",
332 toep->tp_wr_avail, count, mbuf_wrs[count], tail);
384 m_set_priority(m0, mkprio(CPL_PRIORITY_DATA, toep));
385
333 if (tail) {
334 so->so_snd.sb_sndptr = tail;
335 toep->tp_m_last = NULL;
336 } else
337 toep->tp_m_last = so->so_snd.sb_sndptr = last;
338
386 if (tail) {
387 so->so_snd.sb_sndptr = tail;
388 toep->tp_m_last = NULL;
389 } else
390 toep->tp_m_last = so->so_snd.sb_sndptr = last;
391
392
339 DPRINTF("toep->tp_m_last=%p\n", toep->tp_m_last);
340
341 so->so_snd.sb_sndptroff += bytes;
342 total_bytes += bytes;
343 toep->tp_write_seq += bytes;
393 DPRINTF("toep->tp_m_last=%p\n", toep->tp_m_last);
394
395 so->so_snd.sb_sndptroff += bytes;
396 total_bytes += bytes;
397 toep->tp_write_seq += bytes;
398 CTR6(KTR_TOM, "t3_push_frames: wr_avail=%d mbuf_wrs[%d]=%d tail=%p sndptr=%p sndptroff=%d",
399 toep->tp_wr_avail, count, mbuf_wrs[count], tail, so->so_snd.sb_sndptr, so->so_snd.sb_sndptroff);
400 if (tail)
401 CTR4(KTR_TOM, "t3_push_frames: total_bytes=%d tp_m_last=%p tailbuf=%p snd_una=0x%08x",
402 total_bytes, toep->tp_m_last, tail->m_data, tp->snd_una);
403 else
404 CTR3(KTR_TOM, "t3_push_frames: total_bytes=%d tp_m_last=%p snd_una=0x%08x",
405 total_bytes, toep->tp_m_last, tp->snd_una);
344
345
406
407
346 SOCKBUF_UNLOCK(&so->so_snd);
347
348 /*
349 * XXX can drop socket buffer lock here
350 */
408 i = 0;
409 while (i < count && m_get_sgllen(m0)) {
410 if ((count - i) >= 3) {
411 CTR6(KTR_TOM,
412 "t3_push_frames: pa=0x%zx len=%d pa=0x%zx len=%d pa=0x%zx len=%d",
413 segs[i].ds_addr, segs[i].ds_len, segs[i + 1].ds_addr, segs[i + 1].ds_len,
414 segs[i + 2].ds_addr, segs[i + 2].ds_len);
415 i += 3;
416 } else if ((count - i) == 2) {
417 CTR4(KTR_TOM,
418 "t3_push_frames: pa=0x%zx len=%d pa=0x%zx len=%d",
419 segs[i].ds_addr, segs[i].ds_len, segs[i + 1].ds_addr, segs[i + 1].ds_len);
420 i += 2;
421 } else {
422 CTR2(KTR_TOM, "t3_push_frames: pa=0x%zx len=%d",
423 segs[i].ds_addr, segs[i].ds_len);
424 i++;
425 }
351
426
352 toep->tp_wr_avail -= mbuf_wrs[count];
353 toep->tp_wr_unacked += mbuf_wrs[count];
427 }
354
428
355 make_tx_data_wr(so, m0, bytes, tail);
356 m_set_priority(m0, mkprio(CPL_PRIORITY_DATA, so));
357 m_set_sgl(m0, segs);
358 m_set_sgllen(m0, count);
359 /*
429 /*
360 * remember credits used
361 */
362 m0->m_pkthdr.csum_data = mbuf_wrs[count];
363 m0->m_pkthdr.len = bytes;
430 * remember credits used
431 */
432 m0->m_pkthdr.csum_data = mbuf_wrs[count];
433 m0->m_pkthdr.len = bytes;
434 toep->tp_wr_avail -= mbuf_wrs[count];
435 toep->tp_wr_unacked += mbuf_wrs[count];
436
364 if ((req_completion && toep->tp_wr_unacked == mbuf_wrs[count]) ||
365 toep->tp_wr_unacked >= toep->tp_wr_max / 2) {
366 struct work_request_hdr *wr = cplhdr(m0);
367
368 wr->wr_hi |= htonl(F_WR_COMPL);
369 toep->tp_wr_unacked = 0;
370 }
437 if ((req_completion && toep->tp_wr_unacked == mbuf_wrs[count]) ||
438 toep->tp_wr_unacked >= toep->tp_wr_max / 2) {
439 struct work_request_hdr *wr = cplhdr(m0);
440
441 wr->wr_hi |= htonl(F_WR_COMPL);
442 toep->tp_wr_unacked = 0;
443 }
371
444 KASSERT((m0->m_pkthdr.csum_data > 0) &&
445 (m0->m_pkthdr.csum_data <= 4), ("bad credit count %d",
446 m0->m_pkthdr.csum_data));
372 m0->m_type = MT_DONTFREE;
373 enqueue_wr(toep, m0);
374 DPRINTF("sending offload tx with %d bytes in %d segments\n",
375 bytes, count);
447 m0->m_type = MT_DONTFREE;
448 enqueue_wr(toep, m0);
449 DPRINTF("sending offload tx with %d bytes in %d segments\n",
450 bytes, count);
376
377 l2t_send(cdev, m0, toep->tp_l2t);
451 l2t_send(cdev, m0, toep->tp_l2t);
378 if (toep->tp_wr_avail && (tail != NULL))
379 SOCKBUF_LOCK(&so->so_snd);
380 }
452 }
381
382 SOCKBUF_UNLOCK_ASSERT(&so->so_snd);
453 SOCKBUF_UNLOCK(&so->so_snd);
383 return (total_bytes);
384}
385
386/*
387 * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail
388 * under any circumstances. We take the easy way out and always queue the
389 * message to the write_queue. We can optimize the case where the queue is
390 * already empty though the optimization is probably not worth it.

--- 71 unchanged lines hidden (view full) ---

462 struct toedev *tdev = toep->tp_toedev;
463
464 m = m_gethdr_nofail(sizeof(*req));
465
466 DPRINTF("returning %u credits to HW\n", credits);
467
468 req = mtod(m, struct cpl_rx_data_ack *);
469 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
454 return (total_bytes);
455}
456
457/*
458 * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail
459 * under any circumstances. We take the easy way out and always queue the
460 * message to the write_queue. We can optimize the case where the queue is
461 * already empty though the optimization is probably not worth it.

--- 71 unchanged lines hidden (view full) ---

533 struct toedev *tdev = toep->tp_toedev;
534
535 m = m_gethdr_nofail(sizeof(*req));
536
537 DPRINTF("returning %u credits to HW\n", credits);
538
539 req = mtod(m, struct cpl_rx_data_ack *);
540 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
541 req->wr.wr_lo = 0;
470 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid));
471 req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
542 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid));
543 req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
472 m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toeptoso(toep)));
544 m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toep));
473 cxgb_ofld_send(TOM_DATA(tdev)->cdev, m);
474 return (credits);
475}
476
545 cxgb_ofld_send(TOM_DATA(tdev)->cdev, m);
546 return (credits);
547}
548
549/*
550 * Send RX_DATA_ACK CPL message to request a modulation timer to be scheduled.
551 * This is only used in DDP mode, so we take the opportunity to also set the
552 * DACK mode and flush any Rx credits.
553 */
554void
555t3_send_rx_modulate(struct toepcb *toep)
556{
557 struct mbuf *m;
558 struct cpl_rx_data_ack *req;
477
559
560 m = m_gethdr_nofail(sizeof(*req));
561
562 req = mtod(m, struct cpl_rx_data_ack *);
563 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
564 req->wr.wr_lo = 0;
565 m->m_pkthdr.len = m->m_len = sizeof(*req);
566
567 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid));
568 req->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE |
569 V_RX_DACK_MODE(1) |
570 V_RX_CREDITS(toep->tp_copied_seq - toep->tp_rcv_wup));
571 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
572 cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
573 toep->tp_rcv_wup = toep->tp_copied_seq;
574}
575
478/*
576/*
577 * Handle receipt of an urgent pointer.
578 */
579static void
580handle_urg_ptr(struct socket *so, uint32_t urg_seq)
581{
582#ifdef URGENT_DATA_SUPPORTED
583 struct tcpcb *tp = sototcpcb(so);
584
585 urg_seq--; /* initially points past the urgent data, per BSD */
586
587 if (tp->urg_data && !after(urg_seq, tp->urg_seq))
588 return; /* duplicate pointer */
589 sk_send_sigurg(sk);
590 if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
591 !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) {
592 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
593
594 tp->copied_seq++;
595 if (skb && tp->copied_seq - TCP_SKB_CB(skb)->seq >= skb->len)
596 tom_eat_skb(sk, skb, 0);
597 }
598 tp->urg_data = TCP_URG_NOTYET;
599 tp->urg_seq = urg_seq;
600#endif
601}
602
603/*
604 * Returns true if a socket cannot accept new Rx data.
605 */
606static inline int
607so_no_receive(const struct socket *so)
608{
609 return (so->so_state & (SS_ISDISCONNECTED|SS_ISDISCONNECTING));
610}
611
612/*
613 * Process an urgent data notification.
614 */
615static void
616rx_urg_notify(struct toepcb *toep, struct mbuf *m)
617{
618 struct cpl_rx_urg_notify *hdr = cplhdr(m);
619 struct socket *so = toeptoso(toep);
620
621 VALIDATE_SOCK(so);
622
623 if (!so_no_receive(so))
624 handle_urg_ptr(so, ntohl(hdr->seq));
625
626 m_freem(m);
627}
628
629/*
630 * Handler for RX_URG_NOTIFY CPL messages.
631 */
632static int
633do_rx_urg_notify(struct t3cdev *cdev, struct mbuf *m, void *ctx)
634{
635 struct toepcb *toep = (struct toepcb *)ctx;
636
637 rx_urg_notify(toep, m);
638 return (0);
639}
640
641/*
479 * Set of states for which we should return RX credits.
480 */
481#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2)
482
483/*
484 * Called after some received data has been read. It returns RX credits
485 * to the HW for the amount of data processed.
486 */
487void
642 * Set of states for which we should return RX credits.
643 */
644#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2)
645
646/*
647 * Called after some received data has been read. It returns RX credits
648 * to the HW for the amount of data processed.
649 */
650void
488t3_cleanup_rbuf(struct tcpcb *tp)
651t3_cleanup_rbuf(struct tcpcb *tp, int copied)
489{
490 struct toepcb *toep = tp->t_toe;
491 struct socket *so;
492 struct toedev *dev;
493 int dack_mode, must_send, read;
494 u32 thres, credits, dack = 0;
495
652{
653 struct toepcb *toep = tp->t_toe;
654 struct socket *so;
655 struct toedev *dev;
656 int dack_mode, must_send, read;
657 u32 thres, credits, dack = 0;
658
659 so = tp->t_inpcb->inp_socket;
496 if (!((tp->t_state == TCPS_ESTABLISHED) || (tp->t_state == TCPS_FIN_WAIT_1) ||
660 if (!((tp->t_state == TCPS_ESTABLISHED) || (tp->t_state == TCPS_FIN_WAIT_1) ||
497 (tp->t_state == TCPS_FIN_WAIT_2)))
661 (tp->t_state == TCPS_FIN_WAIT_2))) {
662 if (copied) {
663 SOCKBUF_LOCK(&so->so_rcv);
664 toep->tp_copied_seq += copied;
665 SOCKBUF_UNLOCK(&so->so_rcv);
666 }
667
498 return;
668 return;
499 INP_LOCK_ASSERT(tp->t_inpcb);
669 }
500
670
501 so = tp->t_inpcb->inp_socket;
671 INP_LOCK_ASSERT(tp->t_inpcb);
502 SOCKBUF_LOCK(&so->so_rcv);
672 SOCKBUF_LOCK(&so->so_rcv);
503 read = toep->tp_enqueued_bytes - so->so_rcv.sb_cc;
504 toep->tp_copied_seq += read;
505 toep->tp_enqueued_bytes -= read;
673 if (copied)
674 toep->tp_copied_seq += copied;
675 else {
676 read = toep->tp_enqueued_bytes - so->so_rcv.sb_cc;
677 toep->tp_copied_seq += read;
678 }
506 credits = toep->tp_copied_seq - toep->tp_rcv_wup;
679 credits = toep->tp_copied_seq - toep->tp_rcv_wup;
680 toep->tp_enqueued_bytes = so->so_rcv.sb_cc;
507 SOCKBUF_UNLOCK(&so->so_rcv);
508
681 SOCKBUF_UNLOCK(&so->so_rcv);
682
509 if (credits > so->so_rcv.sb_mbmax)
683 if (credits > so->so_rcv.sb_mbmax) {
510 printf("copied_seq=%u rcv_wup=%u credits=%u\n",
511 toep->tp_copied_seq, toep->tp_rcv_wup, credits);
684 printf("copied_seq=%u rcv_wup=%u credits=%u\n",
685 toep->tp_copied_seq, toep->tp_rcv_wup, credits);
512 /*
686 credits = so->so_rcv.sb_mbmax;
687 }
688
689
690 /*
513 * XXX this won't accurately reflect credit return - we need
514 * to look at the difference between the amount that has been
515 * put in the recv sockbuf and what is there now
516 */
517
518 if (__predict_false(!credits))
519 return;
520

--- 67 unchanged lines hidden (view full) ---

588 t3_push_frames(so, 1);
589 return (0);
590}
591
592static int
593cxgb_toe_rcvd(struct tcpcb *tp)
594{
595 INP_LOCK_ASSERT(tp->t_inpcb);
691 * XXX this won't accurately reflect credit return - we need
692 * to look at the difference between the amount that has been
693 * put in the recv sockbuf and what is there now
694 */
695
696 if (__predict_false(!credits))
697 return;
698

--- 67 unchanged lines hidden (view full) ---

766 t3_push_frames(so, 1);
767 return (0);
768}
769
770static int
771cxgb_toe_rcvd(struct tcpcb *tp)
772{
773 INP_LOCK_ASSERT(tp->t_inpcb);
596 t3_cleanup_rbuf(tp);
774 t3_cleanup_rbuf(tp, 0);
597
598 return (0);
599}
600
601static void
602cxgb_toe_detach(struct tcpcb *tp)
603{
604 struct toepcb *toep;

--- 21 unchanged lines hidden (view full) ---

626 .tu_rcvd = cxgb_toe_rcvd,
627 .tu_detach = cxgb_toe_detach,
628 .tu_detach = cxgb_toe_detach,
629 .tu_syncache_event = handle_syncache_event,
630};
631
632
633static void
775
776 return (0);
777}
778
779static void
780cxgb_toe_detach(struct tcpcb *tp)
781{
782 struct toepcb *toep;

--- 21 unchanged lines hidden (view full) ---

804 .tu_rcvd = cxgb_toe_rcvd,
805 .tu_detach = cxgb_toe_detach,
806 .tu_detach = cxgb_toe_detach,
807 .tu_syncache_event = handle_syncache_event,
808};
809
810
811static void
634__set_tcb_field(struct socket *so, struct mbuf *m, uint16_t word,
812__set_tcb_field(struct toepcb *toep, struct mbuf *m, uint16_t word,
635 uint64_t mask, uint64_t val, int no_reply)
636{
637 struct cpl_set_tcb_field *req;
813 uint64_t mask, uint64_t val, int no_reply)
814{
815 struct cpl_set_tcb_field *req;
638 struct tcpcb *tp = sototcpcb(so);
639 struct toepcb *toep = tp->t_toe;
640
816
817 CTR4(KTR_TCB, "__set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx",
818 toep->tp_tid, word, mask, val);
819
641 req = mtod(m, struct cpl_set_tcb_field *);
642 m->m_pkthdr.len = m->m_len = sizeof(*req);
643 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
820 req = mtod(m, struct cpl_set_tcb_field *);
821 m->m_pkthdr.len = m->m_len = sizeof(*req);
822 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
823 req->wr.wr_lo = 0;
644 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tp_tid));
645 req->reply = V_NO_REPLY(no_reply);
646 req->cpu_idx = 0;
647 req->word = htons(word);
648 req->mask = htobe64(mask);
649 req->val = htobe64(val);
650
824 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tp_tid));
825 req->reply = V_NO_REPLY(no_reply);
826 req->cpu_idx = 0;
827 req->word = htons(word);
828 req->mask = htobe64(mask);
829 req->val = htobe64(val);
830
651 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, so));
652 send_or_defer(so, tp, m, 0);
831 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
832 send_or_defer(toep, m, 0);
653}
654
655static void
656t3_set_tcb_field(struct socket *so, uint16_t word, uint64_t mask, uint64_t val)
657{
658 struct mbuf *m;
659 struct tcpcb *tp = sototcpcb(so);
660 struct toepcb *toep = tp->t_toe;
661
662 if (toep == NULL)
663 return;
833}
834
835static void
836t3_set_tcb_field(struct socket *so, uint16_t word, uint64_t mask, uint64_t val)
837{
838 struct mbuf *m;
839 struct tcpcb *tp = sototcpcb(so);
840 struct toepcb *toep = tp->t_toe;
841
842 if (toep == NULL)
843 return;
664
665 if (tp->t_state == TCPS_CLOSED || (toep->tp_flags & TP_ABORT_SHUTDOWN))
844
845 if (tp->t_state == TCPS_CLOSED || (toep->tp_flags & TP_ABORT_SHUTDOWN)) {
846 printf("not seting field\n");
666 return;
847 return;
667
848 }
849
668 m = m_gethdr_nofail(sizeof(struct cpl_set_tcb_field));
669
850 m = m_gethdr_nofail(sizeof(struct cpl_set_tcb_field));
851
670 __set_tcb_field(so, m, word, mask, val, 1);
852 __set_tcb_field(toep, m, word, mask, val, 1);
671}
672
673/*
674 * Set one of the t_flags bits in the TCB.
675 */
676static void
677set_tcb_tflag(struct socket *so, unsigned int bit_pos, int val)
678{

--- 51 unchanged lines hidden (view full) ---

730#define TP_DDP_TIMER_WORKAROUND_VAL\
731 (V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(0) |\
732 ((V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)1) | V_TCB_RX_DDP_BUF0_LEN((uint64_t)2)) <<\
733 32))
734
735static void
736t3_enable_ddp(struct socket *so, int on)
737{
853}
854
855/*
856 * Set one of the t_flags bits in the TCB.
857 */
858static void
859set_tcb_tflag(struct socket *so, unsigned int bit_pos, int val)
860{

--- 51 unchanged lines hidden (view full) ---

912#define TP_DDP_TIMER_WORKAROUND_VAL\
913 (V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(0) |\
914 ((V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)1) | V_TCB_RX_DDP_BUF0_LEN((uint64_t)2)) <<\
915 32))
916
917static void
918t3_enable_ddp(struct socket *so, int on)
919{
738 if (on)
920 if (on) {
921
739 t3_set_tcb_field(so, W_TCB_RX_DDP_FLAGS, V_TF_DDP_OFF(1),
740 V_TF_DDP_OFF(0));
922 t3_set_tcb_field(so, W_TCB_RX_DDP_FLAGS, V_TF_DDP_OFF(1),
923 V_TF_DDP_OFF(0));
741 else
924 } else
742 t3_set_tcb_field(so, W_TCB_RX_DDP_FLAGS,
743 V_TF_DDP_OFF(1) |
744 TP_DDP_TIMER_WORKAROUND_MASK,
745 V_TF_DDP_OFF(1) |
746 TP_DDP_TIMER_WORKAROUND_VAL);
747
748}
749
925 t3_set_tcb_field(so, W_TCB_RX_DDP_FLAGS,
926 V_TF_DDP_OFF(1) |
927 TP_DDP_TIMER_WORKAROUND_MASK,
928 V_TF_DDP_OFF(1) |
929 TP_DDP_TIMER_WORKAROUND_VAL);
930
931}
932
750
751void
752t3_set_ddp_tag(struct socket *so, int buf_idx, unsigned int tag_color)
753{
754 t3_set_tcb_field(so, W_TCB_RX_DDP_BUF0_TAG + buf_idx,
755 V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG),
756 tag_color);
757}
758

--- 13 unchanged lines hidden (view full) ---

772 V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN << 32),
773 V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset) |
774 V_TCB_RX_DDP_BUF1_LEN(((uint64_t)len) << 32));
775}
776
777static int
778t3_set_cong_control(struct socket *so, const char *name)
779{
933void
934t3_set_ddp_tag(struct socket *so, int buf_idx, unsigned int tag_color)
935{
936 t3_set_tcb_field(so, W_TCB_RX_DDP_BUF0_TAG + buf_idx,
937 V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG),
938 tag_color);
939}
940

--- 13 unchanged lines hidden (view full) ---

954 V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN << 32),
955 V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset) |
956 V_TCB_RX_DDP_BUF1_LEN(((uint64_t)len) << 32));
957}
958
959static int
960t3_set_cong_control(struct socket *so, const char *name)
961{
780#ifdef notyet
962#ifdef CONGESTION_CONTROL_SUPPORTED
781 int cong_algo;
782
783 for (cong_algo = 0; cong_algo < ARRAY_SIZE(t3_cong_ops); cong_algo++)
784 if (!strcmp(name, t3_cong_ops[cong_algo].name))
785 break;
786
787 if (cong_algo >= ARRAY_SIZE(t3_cong_ops))
788 return -EINVAL;

--- 8 unchanged lines hidden (view full) ---

797 struct tcpcb *tp = sototcpcb(so);
798 struct toepcb *toep = tp->t_toe;
799 struct mbuf *m = m_gethdr(M_NOWAIT, MT_DATA);
800
801 if (!m)
802 return (ENOMEM);
803
804 INP_LOCK_ASSERT(tp->t_inpcb);
963 int cong_algo;
964
965 for (cong_algo = 0; cong_algo < ARRAY_SIZE(t3_cong_ops); cong_algo++)
966 if (!strcmp(name, t3_cong_ops[cong_algo].name))
967 break;
968
969 if (cong_algo >= ARRAY_SIZE(t3_cong_ops))
970 return -EINVAL;

--- 8 unchanged lines hidden (view full) ---

979 struct tcpcb *tp = sototcpcb(so);
980 struct toepcb *toep = tp->t_toe;
981 struct mbuf *m = m_gethdr(M_NOWAIT, MT_DATA);
982
983 if (!m)
984 return (ENOMEM);
985
986 INP_LOCK_ASSERT(tp->t_inpcb);
805 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, so));
987 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
806 req = mtod(m, struct cpl_get_tcb *);
807 m->m_pkthdr.len = m->m_len = sizeof(*req);
808 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
988 req = mtod(m, struct cpl_get_tcb *);
989 m->m_pkthdr.len = m->m_len = sizeof(*req);
990 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
991 req->wr.wr_lo = 0;
809 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, toep->tp_tid));
810 req->cpuno = htons(toep->tp_qset);
992 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, toep->tp_tid));
993 req->cpuno = htons(toep->tp_qset);
994 req->rsvd = 0;
811 if (sototcpcb(so)->t_state == TCPS_SYN_SENT)
812 mbufq_tail(&toep->out_of_order_queue, m); // defer
813 else
814 cxgb_ofld_send(T3C_DEV(so), m);
815 return 0;
816}
817
818static inline void

--- 39 unchanged lines hidden (view full) ---

858
859 tp->t_maxseg = td->mtus[idx] - 40;
860 } else
861 idx = find_best_mtu(td, pmtu);
862
863 return (idx);
864}
865
995 if (sototcpcb(so)->t_state == TCPS_SYN_SENT)
996 mbufq_tail(&toep->out_of_order_queue, m); // defer
997 else
998 cxgb_ofld_send(T3C_DEV(so), m);
999 return 0;
1000}
1001
1002static inline void

--- 39 unchanged lines hidden (view full) ---

1042
1043 tp->t_maxseg = td->mtus[idx] - 40;
1044 } else
1045 idx = find_best_mtu(td, pmtu);
1046
1047 return (idx);
1048}
1049
866void
867t3_release_ddp_resources(struct toepcb *toep)
868{
869 /*
870 * This is a no-op until we have DDP support
871 */
872}
873
874static inline void
875free_atid(struct t3cdev *cdev, unsigned int tid)
876{
877 struct toepcb *toep = cxgb_free_atid(cdev, tid);
878
879 if (toep)
880 toepcb_release(toep);
881}

--- 28 unchanged lines hidden (view full) ---

910 purge_wr_queue(toep);
911 reset_wr_list(toep);
912 }
913
914 if (toep->tp_l2t) {
915 l2t_release(L2DATA(cdev), toep->tp_l2t);
916 toep->tp_l2t = NULL;
917 }
1050static inline void
1051free_atid(struct t3cdev *cdev, unsigned int tid)
1052{
1053 struct toepcb *toep = cxgb_free_atid(cdev, tid);
1054
1055 if (toep)
1056 toepcb_release(toep);
1057}

--- 28 unchanged lines hidden (view full) ---

1086 purge_wr_queue(toep);
1087 reset_wr_list(toep);
1088 }
1089
1090 if (toep->tp_l2t) {
1091 l2t_release(L2DATA(cdev), toep->tp_l2t);
1092 toep->tp_l2t = NULL;
1093 }
918 printf("setting toep->tp_tp to NULL\n");
919
920 toep->tp_tp = NULL;
921 if (tp) {
922 INP_LOCK_ASSERT(tp->t_inpcb);
923 tp->t_toe = NULL;
924 tp->t_flags &= ~TF_TOE;
925 }
926
927 if (toep->tp_state == TCPS_SYN_SENT) {

--- 31 unchanged lines hidden (view full) ---

959{
960 int wscale = 0;
961
962 if (space > MAX_RCV_WND)
963 space = MAX_RCV_WND;
964
965 if (tcp_do_rfc1323)
966 for (; space > 65535 && wscale < 14; space >>= 1, ++wscale) ;
1094 toep->tp_tp = NULL;
1095 if (tp) {
1096 INP_LOCK_ASSERT(tp->t_inpcb);
1097 tp->t_toe = NULL;
1098 tp->t_flags &= ~TF_TOE;
1099 }
1100
1101 if (toep->tp_state == TCPS_SYN_SENT) {

--- 31 unchanged lines hidden (view full) ---

1133{
1134 int wscale = 0;
1135
1136 if (space > MAX_RCV_WND)
1137 space = MAX_RCV_WND;
1138
1139 if (tcp_do_rfc1323)
1140 for (; space > 65535 && wscale < 14; space >>= 1, ++wscale) ;
967 return wscale;
1141
1142 return (wscale);
968}
969
970/*
971 * Determine the receive window size for a socket.
972 */
1143}
1144
1145/*
1146 * Determine the receive window size for a socket.
1147 */
973static unsigned int
974select_rcv_wnd(struct socket *so)
1148static unsigned long
1149select_rcv_wnd(struct toedev *dev, struct socket *so)
975{
1150{
976 struct toedev *dev = TOE_DEV(so);
977 struct tom_data *d = TOM_DATA(dev);
978 unsigned int wnd;
979 unsigned int max_rcv_wnd;
980
981 if (tcp_do_autorcvbuf)
982 wnd = tcp_autorcvbuf_max;
983 else
1151 struct tom_data *d = TOM_DATA(dev);
1152 unsigned int wnd;
1153 unsigned int max_rcv_wnd;
1154
1155 if (tcp_do_autorcvbuf)
1156 wnd = tcp_autorcvbuf_max;
1157 else
984 wnd = sbspace(&so->so_rcv);
1158 wnd = so->so_rcv.sb_hiwat;
1159
985
1160
1161
986 /* XXX
987 * For receive coalescing to work effectively we need a receive window
988 * that can accomodate a coalesced segment.
989 */
990 if (wnd < MIN_RCV_WND)
991 wnd = MIN_RCV_WND;
992
993 /* PR 5138 */
1162 /* XXX
1163 * For receive coalescing to work effectively we need a receive window
1164 * that can accomodate a coalesced segment.
1165 */
1166 if (wnd < MIN_RCV_WND)
1167 wnd = MIN_RCV_WND;
1168
1169 /* PR 5138 */
994 max_rcv_wnd = (dev->tod_ttid == TOE_ID_CHELSIO_T3B ?
1170 max_rcv_wnd = (dev->tod_ttid < TOE_ID_CHELSIO_T3C ?
995 (uint32_t)d->rx_page_size * 23 :
996 MAX_RCV_WND);
997
998 return min(wnd, max_rcv_wnd);
999}
1000
1001/*
1002 * Assign offload parameters to some socket fields. This code is used by

--- 9 unchanged lines hidden (view full) ---

1012 SOCK_LOCK_ASSERT(so);
1013
1014 printf("initializing offload socket\n");
1015 /*
1016 * We either need to fix push frames to work with sbcompress
1017 * or we need to add this
1018 */
1019 so->so_snd.sb_flags |= SB_NOCOALESCE;
1171 (uint32_t)d->rx_page_size * 23 :
1172 MAX_RCV_WND);
1173
1174 return min(wnd, max_rcv_wnd);
1175}
1176
1177/*
1178 * Assign offload parameters to some socket fields. This code is used by

--- 9 unchanged lines hidden (view full) ---

1188 SOCK_LOCK_ASSERT(so);
1189
1190 printf("initializing offload socket\n");
1191 /*
1192 * We either need to fix push frames to work with sbcompress
1193 * or we need to add this
1194 */
1195 so->so_snd.sb_flags |= SB_NOCOALESCE;
1020
1196 so->so_rcv.sb_flags |= SB_NOCOALESCE;
1197
1021 tp->t_toe = toep;
1022 toep->tp_tp = tp;
1023 toep->tp_toedev = dev;
1024
1025 toep->tp_tid = tid;
1026 toep->tp_l2t = e;
1027 toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(dev, max_wrs);
1028 toep->tp_wr_unacked = 0;
1029 toep->tp_delack_mode = 0;
1030
1031 toep->tp_mtu_idx = select_mss(td, tp, dst->rt_ifp->if_mtu);
1032 /*
1033 * XXX broken
1034 *
1035 */
1198 tp->t_toe = toep;
1199 toep->tp_tp = tp;
1200 toep->tp_toedev = dev;
1201
1202 toep->tp_tid = tid;
1203 toep->tp_l2t = e;
1204 toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(dev, max_wrs);
1205 toep->tp_wr_unacked = 0;
1206 toep->tp_delack_mode = 0;
1207
1208 toep->tp_mtu_idx = select_mss(td, tp, dst->rt_ifp->if_mtu);
1209 /*
1210 * XXX broken
1211 *
1212 */
1036 tp->rcv_wnd = select_rcv_wnd(so);
1213 tp->rcv_wnd = select_rcv_wnd(dev, so);
1214
1037 toep->tp_ulp_mode = TOM_TUNABLE(dev, ddp) && !(so->so_options & SO_NO_DDP) &&
1038 tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0;
1039 toep->tp_qset_idx = 0;
1040
1041 reset_wr_list(toep);
1042 DPRINTF("initialization done\n");
1043}
1044

--- 26 unchanged lines hidden (view full) ---

1071
1072static inline unsigned int
1073calc_opt2(const struct socket *so, struct toedev *dev)
1074{
1075 int flv_valid;
1076
1077 flv_valid = (TOM_TUNABLE(dev, cong_alg) != -1);
1078
1215 toep->tp_ulp_mode = TOM_TUNABLE(dev, ddp) && !(so->so_options & SO_NO_DDP) &&
1216 tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0;
1217 toep->tp_qset_idx = 0;
1218
1219 reset_wr_list(toep);
1220 DPRINTF("initialization done\n");
1221}
1222

--- 26 unchanged lines hidden (view full) ---

1249
1250static inline unsigned int
1251calc_opt2(const struct socket *so, struct toedev *dev)
1252{
1253 int flv_valid;
1254
1255 flv_valid = (TOM_TUNABLE(dev, cong_alg) != -1);
1256
1079 return V_FLAVORS_VALID(flv_valid) |
1080 V_CONG_CONTROL_FLAVOR(flv_valid ? TOM_TUNABLE(dev, cong_alg) : 0);
1257 return (V_FLAVORS_VALID(flv_valid) |
1258 V_CONG_CONTROL_FLAVOR(flv_valid ? TOM_TUNABLE(dev, cong_alg) : 0));
1081}
1259}
1260
1261#if DEBUG_WR > 1
1262static int
1263count_pending_wrs(const struct toepcb *toep)
1264{
1265 const struct mbuf *m;
1266 int n = 0;
1267
1268 wr_queue_walk(toep, m)
1269 n += m->m_pkthdr.csum_data;
1270 return (n);
1271}
1272#endif
1273
1082#if 0
1083(((*(struct tom_data **)&(dev)->l4opt)->conf.cong_alg) != -1)
1084#endif
1085
1086static void
1087mk_act_open_req(struct socket *so, struct mbuf *m,
1088 unsigned int atid, const struct l2t_entry *e)
1089{
1090 struct cpl_act_open_req *req;
1091 struct inpcb *inp = sotoinpcb(so);
1092 struct tcpcb *tp = intotcpcb(inp);
1093 struct toepcb *toep = tp->t_toe;
1094 struct toedev *tdev = TOE_DEV(so);
1095
1274#if 0
1275(((*(struct tom_data **)&(dev)->l4opt)->conf.cong_alg) != -1)
1276#endif
1277
1278static void
1279mk_act_open_req(struct socket *so, struct mbuf *m,
1280 unsigned int atid, const struct l2t_entry *e)
1281{
1282 struct cpl_act_open_req *req;
1283 struct inpcb *inp = sotoinpcb(so);
1284 struct tcpcb *tp = intotcpcb(inp);
1285 struct toepcb *toep = tp->t_toe;
1286 struct toedev *tdev = TOE_DEV(so);
1287
1096 m_set_priority((struct mbuf *)m, mkprio(CPL_PRIORITY_SETUP, so));
1288 m_set_priority((struct mbuf *)m, mkprio(CPL_PRIORITY_SETUP, toep));
1097
1098 req = mtod(m, struct cpl_act_open_req *);
1099 m->m_pkthdr.len = m->m_len = sizeof(*req);
1289
1290 req = mtod(m, struct cpl_act_open_req *);
1291 m->m_pkthdr.len = m->m_len = sizeof(*req);
1100
1292
1101 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1293 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1294 req->wr.wr_lo = 0;
1102 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
1103 req->local_port = inp->inp_lport;
1104 req->peer_port = inp->inp_fport;
1105 memcpy(&req->local_ip, &inp->inp_laddr, 4);
1106 memcpy(&req->peer_ip, &inp->inp_faddr, 4);
1295 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
1296 req->local_port = inp->inp_lport;
1297 req->peer_port = inp->inp_fport;
1298 memcpy(&req->local_ip, &inp->inp_laddr, 4);
1299 memcpy(&req->peer_ip, &inp->inp_faddr, 4);
1107 DPRINTF("connect smt_idx=%d\n", e->smt_idx);
1108 req->opt0h = htonl(calc_opt0h(so, toep->tp_mtu_idx) | V_L2T_IDX(e->idx) |
1109 V_TX_CHANNEL(e->smt_idx));
1110 req->opt0l = htonl(calc_opt0l(so, toep->tp_ulp_mode));
1111 req->params = 0;
1112 req->opt2 = htonl(calc_opt2(so, tdev));
1113}
1114
1115

--- 23 unchanged lines hidden (view full) ---

1139static void
1140fail_act_open(struct toepcb *toep, int errno)
1141{
1142 struct tcpcb *tp = toep->tp_tp;
1143
1144 t3_release_offload_resources(toep);
1145 if (tp) {
1146 INP_LOCK_ASSERT(tp->t_inpcb);
1300 req->opt0h = htonl(calc_opt0h(so, toep->tp_mtu_idx) | V_L2T_IDX(e->idx) |
1301 V_TX_CHANNEL(e->smt_idx));
1302 req->opt0l = htonl(calc_opt0l(so, toep->tp_ulp_mode));
1303 req->params = 0;
1304 req->opt2 = htonl(calc_opt2(so, tdev));
1305}
1306
1307

--- 23 unchanged lines hidden (view full) ---

1331static void
1332fail_act_open(struct toepcb *toep, int errno)
1333{
1334 struct tcpcb *tp = toep->tp_tp;
1335
1336 t3_release_offload_resources(toep);
1337 if (tp) {
1338 INP_LOCK_ASSERT(tp->t_inpcb);
1147 cxgb_tcp_drop(tp, errno);
1339 tcp_drop(tp, errno);
1148 }
1149
1150#ifdef notyet
1151 TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS);
1152#endif
1153}
1154
1155/*

--- 128 unchanged lines hidden (view full) ---

1284
1285 mk_act_open_req(so, m, atid, e);
1286 SOCK_UNLOCK(so);
1287
1288 soisconnecting(so);
1289 toep = tp->t_toe;
1290 m_set_toep(m, tp->t_toe);
1291
1340 }
1341
1342#ifdef notyet
1343 TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS);
1344#endif
1345}
1346
1347/*

--- 128 unchanged lines hidden (view full) ---

1476
1477 mk_act_open_req(so, m, atid, e);
1478 SOCK_UNLOCK(so);
1479
1480 soisconnecting(so);
1481 toep = tp->t_toe;
1482 m_set_toep(m, tp->t_toe);
1483
1292 printf("sending off request\n");
1293
1294 toep->tp_state = TCPS_SYN_SENT;
1295 l2t_send(d->cdev, (struct mbuf *)m, e);
1296
1297 if (toep->tp_ulp_mode)
1298 t3_enable_ddp(so, 0);
1299 return (0);
1300
1301free_tid:

--- 35 unchanged lines hidden (view full) ---

1337
1338 /* Purge the send queue so we don't send anything after an abort. */
1339 if (so)
1340 sbflush(&so->so_snd);
1341 if ((toep->tp_flags & TP_CLOSE_CON_REQUESTED) && is_t3a(tdev))
1342 mode |= CPL_ABORT_POST_CLOSE_REQ;
1343
1344 m = m_gethdr_nofail(sizeof(*req));
1484 toep->tp_state = TCPS_SYN_SENT;
1485 l2t_send(d->cdev, (struct mbuf *)m, e);
1486
1487 if (toep->tp_ulp_mode)
1488 t3_enable_ddp(so, 0);
1489 return (0);
1490
1491free_tid:

--- 35 unchanged lines hidden (view full) ---

1527
1528 /* Purge the send queue so we don't send anything after an abort. */
1529 if (so)
1530 sbflush(&so->so_snd);
1531 if ((toep->tp_flags & TP_CLOSE_CON_REQUESTED) && is_t3a(tdev))
1532 mode |= CPL_ABORT_POST_CLOSE_REQ;
1533
1534 m = m_gethdr_nofail(sizeof(*req));
1345 m_set_priority(m, mkprio(CPL_PRIORITY_DATA, so));
1535 m_set_priority(m, mkprio(CPL_PRIORITY_DATA, toep));
1346 set_arp_failure_handler(m, abort_arp_failure);
1347
1348 req = mtod(m, struct cpl_abort_req *);
1349 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
1350 req->wr.wr_lo = htonl(V_WR_TID(tid));
1351 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
1352 req->rsvd0 = tp ? htonl(tp->snd_nxt) : 0;
1353 req->rsvd1 = !(toep->tp_flags & TP_DATASENT);

--- 57 unchanged lines hidden (view full) ---

1411 if (copied < 1)
1412 return (EINVAL);
1413
1414 tp = sototcpcb(so);
1415 /*
1416 * XXX I need to revisit this
1417 */
1418 if ((err = t3_set_cong_control(so, name)) == 0) {
1536 set_arp_failure_handler(m, abort_arp_failure);
1537
1538 req = mtod(m, struct cpl_abort_req *);
1539 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
1540 req->wr.wr_lo = htonl(V_WR_TID(tid));
1541 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
1542 req->rsvd0 = tp ? htonl(tp->snd_nxt) : 0;
1543 req->rsvd1 = !(toep->tp_flags & TP_DATASENT);

--- 57 unchanged lines hidden (view full) ---

1601 if (copied < 1)
1602 return (EINVAL);
1603
1604 tp = sototcpcb(so);
1605 /*
1606 * XXX I need to revisit this
1607 */
1608 if ((err = t3_set_cong_control(so, name)) == 0) {
1419#ifdef notyet
1609#ifdef CONGESTION_CONTROL_SUPPORTED
1420 tp->t_cong_control = strdup(name, M_CXGB);
1421#endif
1422 } else
1423 return (err);
1424 } else {
1425 int optval, oldval;
1426 struct inpcb *inp;
1427 struct tcpcb *tp;

--- 32 unchanged lines hidden (view full) ---

1460 if (sopt->sopt_level != IPPROTO_TCP)
1461 err = t3_ip_ctloutput(so, sopt);
1462 else
1463 err = t3_tcp_ctloutput(so, sopt);
1464
1465 if (err != EOPNOTSUPP)
1466 return (err);
1467
1610 tp->t_cong_control = strdup(name, M_CXGB);
1611#endif
1612 } else
1613 return (err);
1614 } else {
1615 int optval, oldval;
1616 struct inpcb *inp;
1617 struct tcpcb *tp;

--- 32 unchanged lines hidden (view full) ---

1650 if (sopt->sopt_level != IPPROTO_TCP)
1651 err = t3_ip_ctloutput(so, sopt);
1652 else
1653 err = t3_tcp_ctloutput(so, sopt);
1654
1655 if (err != EOPNOTSUPP)
1656 return (err);
1657
1468 return tcp_ctloutput(so, sopt);
1658 return (tcp_ctloutput(so, sopt));
1469}
1470
1471/*
1659}
1660
1661/*
1662 * Returns true if we need to explicitly request RST when we receive new data
1663 * on an RX-closed connection.
1664 */
1665static inline int
1666need_rst_on_excess_rx(const struct toepcb *toep)
1667{
1668 return (1);
1669}
1670
1671/*
1672 * Handles Rx data that arrives in a state where the socket isn't accepting
1673 * new data.
1674 */
1675static void
1676handle_excess_rx(struct toepcb *toep, struct mbuf *m)
1677{
1678
1679 if (need_rst_on_excess_rx(toep) && !(toep->tp_flags & TP_ABORT_SHUTDOWN))
1680 t3_send_reset(toep);
1681 m_freem(m);
1682}
1683
1684/*
1685 * Process a get_tcb_rpl as a DDP completion (similar to RX_DDP_COMPLETE)
1686 * by getting the DDP offset from the TCB.
1687 */
1688static void
1689tcb_rpl_as_ddp_complete(struct toepcb *toep, struct mbuf *m)
1690{
1691 struct ddp_state *q = &toep->tp_ddp_state;
1692 struct ddp_buf_state *bsp;
1693 struct cpl_get_tcb_rpl *hdr;
1694 unsigned int ddp_offset;
1695 struct socket *so;
1696 struct tcpcb *tp;
1697
1698 uint64_t t;
1699 __be64 *tcb;
1700
1701 so = toeptoso(toep);
1702 tp = toep->tp_tp;
1703
1704 INP_LOCK_ASSERT(tp->t_inpcb);
1705 SOCKBUF_LOCK(&so->so_rcv);
1706
1707 /* Note that we only accout for CPL_GET_TCB issued by the DDP code. We
1708 * really need a cookie in order to dispatch the RPLs.
1709 */
1710 q->get_tcb_count--;
1711
1712 /* It is a possible that a previous CPL already invalidated UBUF DDP
1713 * and moved the cur_buf idx and hence no further processing of this
1714 * skb is required. However, the app might be sleeping on
1715 * !q->get_tcb_count and we need to wake it up.
1716 */
1717 if (q->cancel_ubuf && !t3_ddp_ubuf_pending(toep)) {
1718 struct socket *so = toeptoso(toep);
1719
1720 m_freem(m);
1721 if (__predict_true((so->so_state & SS_NOFDREF) == 0))
1722 sorwakeup_locked(so);
1723 else
1724 SOCKBUF_UNLOCK(&so->so_rcv);
1725 return;
1726 }
1727
1728 bsp = &q->buf_state[q->cur_buf];
1729 hdr = cplhdr(m);
1730 tcb = (__be64 *)(hdr + 1);
1731 if (q->cur_buf == 0) {
1732 t = be64toh(tcb[(31 - W_TCB_RX_DDP_BUF0_OFFSET) / 2]);
1733 ddp_offset = t >> (32 + S_TCB_RX_DDP_BUF0_OFFSET);
1734 } else {
1735 t = be64toh(tcb[(31 - W_TCB_RX_DDP_BUF1_OFFSET) / 2]);
1736 ddp_offset = t >> S_TCB_RX_DDP_BUF1_OFFSET;
1737 }
1738 ddp_offset &= M_TCB_RX_DDP_BUF0_OFFSET;
1739 m->m_cur_offset = bsp->cur_offset;
1740 bsp->cur_offset = ddp_offset;
1741 m->m_len = m->m_pkthdr.len = ddp_offset - m->m_cur_offset;
1742
1743 CTR5(KTR_TOM,
1744 "tcb_rpl_as_ddp_complete: idx=%d seq=0x%x hwbuf=%u ddp_offset=%u cur_offset=%u",
1745 q->cur_buf, tp->rcv_nxt, q->cur_buf, ddp_offset, m->m_cur_offset);
1746 KASSERT(ddp_offset >= m->m_cur_offset, ("ddp_offset=%u less than cur_offset=%u",
1747 ddp_offset, m->m_cur_offset));
1748
1749#ifdef T3_TRACE
1750 T3_TRACE3(TIDTB(so),
1751 "tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u ddp_offset %u",
1752 tp->rcv_nxt, q->cur_buf, ddp_offset);
1753#endif
1754
1755#if 0
1756{
1757 unsigned int ddp_flags, rcv_nxt, rx_hdr_offset, buf_idx;
1758
1759 t = be64toh(tcb[(31 - W_TCB_RX_DDP_FLAGS) / 2]);
1760 ddp_flags = (t >> S_TCB_RX_DDP_FLAGS) & M_TCB_RX_DDP_FLAGS;
1761
1762 t = be64toh(tcb[(31 - W_TCB_RCV_NXT) / 2]);
1763 rcv_nxt = t >> S_TCB_RCV_NXT;
1764 rcv_nxt &= M_TCB_RCV_NXT;
1765
1766 t = be64toh(tcb[(31 - W_TCB_RX_HDR_OFFSET) / 2]);
1767 rx_hdr_offset = t >> (32 + S_TCB_RX_HDR_OFFSET);
1768 rx_hdr_offset &= M_TCB_RX_HDR_OFFSET;
1769
1770 T3_TRACE2(TIDTB(sk),
1771 "tcb_rpl_as_ddp_complete: DDP FLAGS 0x%x dma up to 0x%x",
1772 ddp_flags, rcv_nxt - rx_hdr_offset);
1773 T3_TRACE4(TB(q),
1774 "tcb_rpl_as_ddp_complete: rcvnxt 0x%x hwbuf %u cur_offset %u cancel %u",
1775 tp->rcv_nxt, q->cur_buf, bsp->cur_offset, q->cancel_ubuf);
1776 T3_TRACE3(TB(q),
1777 "tcb_rpl_as_ddp_complete: TCB rcvnxt 0x%x hwbuf 0x%x ddp_offset %u",
1778 rcv_nxt - rx_hdr_offset, ddp_flags, ddp_offset);
1779 T3_TRACE2(TB(q),
1780 "tcb_rpl_as_ddp_complete: flags0 0x%x flags1 0x%x",
1781 q->buf_state[0].flags, q->buf_state[1].flags);
1782
1783}
1784#endif
1785 if (__predict_false(so_no_receive(so) && m->m_pkthdr.len)) {
1786 handle_excess_rx(toep, m);
1787 return;
1788 }
1789
1790#ifdef T3_TRACE
1791 if ((int)m->m_pkthdr.len < 0) {
1792 t3_ddp_error(so, "tcb_rpl_as_ddp_complete: neg len");
1793 }
1794#endif
1795 if (bsp->flags & DDP_BF_NOCOPY) {
1796#ifdef T3_TRACE
1797 T3_TRACE0(TB(q),
1798 "tcb_rpl_as_ddp_complete: CANCEL UBUF");
1799
1800 if (!q->cancel_ubuf && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1801 printk("!cancel_ubuf");
1802 t3_ddp_error(sk, "tcb_rpl_as_ddp_complete: !cancel_ubuf");
1803 }
1804#endif
1805 m->m_ddp_flags = DDP_BF_PSH | DDP_BF_NOCOPY | 1;
1806 bsp->flags &= ~(DDP_BF_NOCOPY|DDP_BF_NODATA);
1807 q->cur_buf ^= 1;
1808 } else if (bsp->flags & DDP_BF_NOFLIP) {
1809
1810 m->m_ddp_flags = 1; /* always a kernel buffer */
1811
1812 /* now HW buffer carries a user buffer */
1813 bsp->flags &= ~DDP_BF_NOFLIP;
1814 bsp->flags |= DDP_BF_NOCOPY;
1815
1816 /* It is possible that the CPL_GET_TCB_RPL doesn't indicate
1817 * any new data in which case we're done. If in addition the
1818 * offset is 0, then there wasn't a completion for the kbuf
1819 * and we need to decrement the posted count.
1820 */
1821 if (m->m_pkthdr.len == 0) {
1822 if (ddp_offset == 0) {
1823 q->kbuf_posted--;
1824 bsp->flags |= DDP_BF_NODATA;
1825 }
1826 SOCKBUF_UNLOCK(&so->so_rcv);
1827
1828 m_free(m);
1829 return;
1830 }
1831 } else {
1832 SOCKBUF_UNLOCK(&so->so_rcv);
1833 /* This reply is for a CPL_GET_TCB_RPL to cancel the UBUF DDP,
1834 * but it got here way late and nobody cares anymore.
1835 */
1836 m_free(m);
1837 return;
1838 }
1839
1840 m->m_ddp_gl = (unsigned char *)bsp->gl;
1841 m->m_flags |= M_DDP;
1842 m->m_seq = tp->rcv_nxt;
1843 tp->rcv_nxt += m->m_pkthdr.len;
1844 tp->t_rcvtime = ticks;
1845#ifdef T3_TRACE
1846 T3_TRACE3(TB(q),
1847 "tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u lskb->len %u",
1848 m->m_seq, q->cur_buf, m->m_pkthdr.len);
1849#endif
1850 CTR3(KTR_TOM, "tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u m->m_pktlen %u",
1851 m->m_seq, q->cur_buf, m->m_pkthdr.len);
1852 if (m->m_pkthdr.len == 0)
1853 q->user_ddp_pending = 0;
1854 else
1855 SBAPPEND(&so->so_rcv, m);
1856 if (__predict_true((so->so_state & SS_NOFDREF) == 0))
1857 sorwakeup_locked(so);
1858 else
1859 SOCKBUF_UNLOCK(&so->so_rcv);
1860}
1861
1862/*
1863 * Process a CPL_GET_TCB_RPL. These can also be generated by the DDP code,
1864 * in that case they are similar to DDP completions.
1865 */
1866static int
1867do_get_tcb_rpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
1868{
1869 struct toepcb *toep = (struct toepcb *)ctx;
1870
1871 /* OK if socket doesn't exist */
1872 if (toep == NULL) {
1873 printf("null toep in do_get_tcb_rpl\n");
1874 return (CPL_RET_BUF_DONE);
1875 }
1876
1877 INP_LOCK(toep->tp_tp->t_inpcb);
1878 tcb_rpl_as_ddp_complete(toep, m);
1879 INP_UNLOCK(toep->tp_tp->t_inpcb);
1880
1881 return (0);
1882}
1883
1884static void
1885handle_ddp_data(struct toepcb *toep, struct mbuf *m)
1886{
1887 struct tcpcb *tp = toep->tp_tp;
1888 struct socket *so = toeptoso(toep);
1889 struct ddp_state *q;
1890 struct ddp_buf_state *bsp;
1891 struct cpl_rx_data *hdr = cplhdr(m);
1892 unsigned int rcv_nxt = ntohl(hdr->seq);
1893
1894 if (tp->rcv_nxt == rcv_nxt)
1895 return;
1896
1897 INP_LOCK_ASSERT(tp->t_inpcb);
1898 SOCKBUF_LOCK(&so->so_rcv);
1899 q = &toep->tp_ddp_state;
1900 bsp = &q->buf_state[q->cur_buf];
1901 KASSERT(SEQ_GT(rcv_nxt, tp->rcv_nxt), ("tp->rcv_nxt=0x%08x decreased rcv_nxt=0x08%x",
1902 rcv_nxt, tp->rcv_nxt));
1903 m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt;
1904 KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
1905 CTR3(KTR_TOM, "rcv_nxt=0x%x tp->rcv_nxt=0x%x len=%d",
1906 rcv_nxt, tp->rcv_nxt, m->m_pkthdr.len);
1907
1908#ifdef T3_TRACE
1909 if ((int)m->m_pkthdr.len < 0) {
1910 t3_ddp_error(so, "handle_ddp_data: neg len");
1911 }
1912#endif
1913
1914 m->m_ddp_gl = (unsigned char *)bsp->gl;
1915 m->m_flags |= M_DDP;
1916 m->m_cur_offset = bsp->cur_offset;
1917 m->m_ddp_flags = DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1;
1918 if (bsp->flags & DDP_BF_NOCOPY)
1919 bsp->flags &= ~DDP_BF_NOCOPY;
1920
1921 m->m_seq = tp->rcv_nxt;
1922 tp->rcv_nxt = rcv_nxt;
1923 bsp->cur_offset += m->m_pkthdr.len;
1924 if (!(bsp->flags & DDP_BF_NOFLIP))
1925 q->cur_buf ^= 1;
1926 /*
1927 * For now, don't re-enable DDP after a connection fell out of DDP
1928 * mode.
1929 */
1930 q->ubuf_ddp_ready = 0;
1931 SOCKBUF_UNLOCK(&so->so_rcv);
1932}
1933
1934/*
1472 * Process new data received for a connection.
1473 */
1474static void
1475new_rx_data(struct toepcb *toep, struct mbuf *m)
1476{
1477 struct cpl_rx_data *hdr = cplhdr(m);
1478 struct tcpcb *tp = toep->tp_tp;
1479 struct socket *so = toeptoso(toep);
1480 int len = be16toh(hdr->len);
1481
1482 INP_LOCK(tp->t_inpcb);
1483
1935 * Process new data received for a connection.
1936 */
1937static void
1938new_rx_data(struct toepcb *toep, struct mbuf *m)
1939{
1940 struct cpl_rx_data *hdr = cplhdr(m);
1941 struct tcpcb *tp = toep->tp_tp;
1942 struct socket *so = toeptoso(toep);
1943 int len = be16toh(hdr->len);
1944
1945 INP_LOCK(tp->t_inpcb);
1946
1484#ifdef notyet
1485 if (__predict_false(sk_no_receive(sk))) {
1486 handle_excess_rx(so, skb);
1947 if (__predict_false(so_no_receive(so))) {
1948 handle_excess_rx(toep, m);
1949 INP_UNLOCK(tp->t_inpcb);
1950 TRACE_EXIT;
1487 return;
1488 }
1489
1951 return;
1952 }
1953
1490 if (ULP_MODE(tp) == ULP_MODE_TCPDDP)
1491 handle_ddp_data(so, skb);
1954 if (toep->tp_ulp_mode == ULP_MODE_TCPDDP)
1955 handle_ddp_data(toep, m);
1956
1957 m->m_seq = ntohl(hdr->seq);
1958 m->m_ulp_mode = 0; /* for iSCSI */
1492
1959
1493 TCP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1494 TCP_SKB_CB(skb)->flags = 0;
1495 skb_ulp_mode(skb) = 0; /* for iSCSI */
1496#endif
1497#if VALIDATE_SEQ
1960#if VALIDATE_SEQ
1498 if (__predict_false(TCP_SKB_CB(skb)->seq != tp->rcv_nxt)) {
1499 printk(KERN_ERR
1961 if (__predict_false(m->m_seq != tp->rcv_nxt)) {
1962 log(LOG_ERR,
1500 "%s: TID %u: Bad sequence number %u, expected %u\n",
1963 "%s: TID %u: Bad sequence number %u, expected %u\n",
1501 TOE_DEV(sk)->name, TID(tp), TCP_SKB_CB(skb)->seq,
1964 TOE_DEV(toeptoso(toep))->name, toep->tp_tid, m->m_seq,
1502 tp->rcv_nxt);
1965 tp->rcv_nxt);
1503 __kfree_skb(skb);
1966 m_freem(m);
1967 INP_UNLOCK(tp->t_inpcb);
1504 return;
1505 }
1506#endif
1507 m_adj(m, sizeof(*hdr));
1508
1968 return;
1969 }
1970#endif
1971 m_adj(m, sizeof(*hdr));
1972
1509#ifdef notyet
1973#ifdef URGENT_DATA_SUPPORTED
1510 /*
1511 * We don't handle urgent data yet
1512 */
1513 if (__predict_false(hdr->urg))
1514 handle_urg_ptr(so, tp->rcv_nxt + ntohs(hdr->urg));
1515 if (__predict_false(tp->urg_data == TCP_URG_NOTYET &&
1516 tp->urg_seq - tp->rcv_nxt < skb->len))
1517 tp->urg_data = TCP_URG_VALID | skb->data[tp->urg_seq -
1518 tp->rcv_nxt];
1519#endif
1520 if (__predict_false(hdr->dack_mode != toep->tp_delack_mode)) {
1521 toep->tp_delack_mode = hdr->dack_mode;
1522 toep->tp_delack_seq = tp->rcv_nxt;
1523 }
1974 /*
1975 * We don't handle urgent data yet
1976 */
1977 if (__predict_false(hdr->urg))
1978 handle_urg_ptr(so, tp->rcv_nxt + ntohs(hdr->urg));
1979 if (__predict_false(tp->urg_data == TCP_URG_NOTYET &&
1980 tp->urg_seq - tp->rcv_nxt < skb->len))
1981 tp->urg_data = TCP_URG_VALID | skb->data[tp->urg_seq -
1982 tp->rcv_nxt];
1983#endif
1984 if (__predict_false(hdr->dack_mode != toep->tp_delack_mode)) {
1985 toep->tp_delack_mode = hdr->dack_mode;
1986 toep->tp_delack_seq = tp->rcv_nxt;
1987 }
1524
1525 DPRINTF("appending mbuf=%p pktlen=%d m_len=%d len=%d\n", m, m->m_pkthdr.len, m->m_len, len);
1988 CTR6(KTR_TOM, "appending mbuf=%p pktlen=%d m_len=%d len=%d rcv_nxt=0x%x enqueued_bytes=%d",
1989 m, m->m_pkthdr.len, m->m_len, len, tp->rcv_nxt, toep->tp_enqueued_bytes);
1526
1527 if (len < m->m_pkthdr.len)
1528 m->m_pkthdr.len = m->m_len = len;
1529
1530 tp->rcv_nxt += m->m_pkthdr.len;
1531 tp->t_rcvtime = ticks;
1532 toep->tp_enqueued_bytes += m->m_pkthdr.len;
1533#ifdef T3_TRACE
1534 T3_TRACE2(TIDTB(sk),
1990
1991 if (len < m->m_pkthdr.len)
1992 m->m_pkthdr.len = m->m_len = len;
1993
1994 tp->rcv_nxt += m->m_pkthdr.len;
1995 tp->t_rcvtime = ticks;
1996 toep->tp_enqueued_bytes += m->m_pkthdr.len;
1997#ifdef T3_TRACE
1998 T3_TRACE2(TIDTB(sk),
1535 "new_rx_data: seq 0x%x len %u",
1536 TCP_SKB_CB(skb)->seq, skb->len);
1999 "new_rx_data: seq 0x%x len %u",
2000 m->m_seq, m->m_pkthdr.len);
1537#endif
2001#endif
2002 INP_UNLOCK(tp->t_inpcb);
1538 SOCKBUF_LOCK(&so->so_rcv);
1539 if (sb_notify(&so->so_rcv))
1540 DPRINTF("rx_data so=%p flags=0x%x len=%d\n", so, so->so_rcv.sb_flags, m->m_pkthdr.len);
1541
2003 SOCKBUF_LOCK(&so->so_rcv);
2004 if (sb_notify(&so->so_rcv))
2005 DPRINTF("rx_data so=%p flags=0x%x len=%d\n", so, so->so_rcv.sb_flags, m->m_pkthdr.len);
2006
1542 sbappend_locked(&so->so_rcv, m);
1543 KASSERT(so->so_rcv.sb_cc < so->so_rcv.sb_mbmax,
2007 SBAPPEND(&so->so_rcv, m);
1544
2008
2009#ifdef notyet
2010 /*
2011 * We're giving too many credits to the card - but disable this check so we can keep on moving :-|
2012 *
2013 */
2014 KASSERT(so->so_rcv.sb_cc < (so->so_rcv.sb_mbmax << 1),
2015
1545 ("so=%p, data contents exceed mbmax, sb_cc=%d sb_mbmax=%d",
1546 so, so->so_rcv.sb_cc, so->so_rcv.sb_mbmax));
2016 ("so=%p, data contents exceed mbmax, sb_cc=%d sb_mbmax=%d",
2017 so, so->so_rcv.sb_cc, so->so_rcv.sb_mbmax));
2018#endif
1547
2019
1548 INP_UNLOCK(tp->t_inpcb);
1549 DPRINTF("sb_cc=%d sb_mbcnt=%d\n",
2020
2021 CTR2(KTR_TOM, "sb_cc=%d sb_mbcnt=%d",
1550 so->so_rcv.sb_cc, so->so_rcv.sb_mbcnt);
1551
1552 if (__predict_true((so->so_state & SS_NOFDREF) == 0))
1553 sorwakeup_locked(so);
1554 else
1555 SOCKBUF_UNLOCK(&so->so_rcv);
1556}
1557

--- 8 unchanged lines hidden (view full) ---

1566 DPRINTF("rx_data len=%d\n", m->m_pkthdr.len);
1567
1568 new_rx_data(toep, m);
1569
1570 return (0);
1571}
1572
1573static void
2022 so->so_rcv.sb_cc, so->so_rcv.sb_mbcnt);
2023
2024 if (__predict_true((so->so_state & SS_NOFDREF) == 0))
2025 sorwakeup_locked(so);
2026 else
2027 SOCKBUF_UNLOCK(&so->so_rcv);
2028}
2029

--- 8 unchanged lines hidden (view full) ---

2038 DPRINTF("rx_data len=%d\n", m->m_pkthdr.len);
2039
2040 new_rx_data(toep, m);
2041
2042 return (0);
2043}
2044
2045static void
1574new_rx_data_ddp(struct socket *so, struct mbuf *m)
2046new_rx_data_ddp(struct toepcb *toep, struct mbuf *m)
1575{
2047{
1576 struct tcpcb *tp = sototcpcb(so);
1577 struct toepcb *toep = tp->t_toe;
2048 struct tcpcb *tp;
1578 struct ddp_state *q;
1579 struct ddp_buf_state *bsp;
1580 struct cpl_rx_data_ddp *hdr;
1581 unsigned int ddp_len, rcv_nxt, ddp_report, end_offset, buf_idx;
2049 struct ddp_state *q;
2050 struct ddp_buf_state *bsp;
2051 struct cpl_rx_data_ddp *hdr;
2052 unsigned int ddp_len, rcv_nxt, ddp_report, end_offset, buf_idx;
2053 struct socket *so = toeptoso(toep);
2054 int nomoredata = 0;
1582
2055
1583#ifdef notyet
1584 if (unlikely(sk_no_receive(sk))) {
1585 handle_excess_rx(so, m);
2056 tp = sototcpcb(so);
2057
2058 INP_LOCK(tp->t_inpcb);
2059 if (__predict_false(so_no_receive(so))) {
2060
2061 handle_excess_rx(toep, m);
2062 INP_UNLOCK(tp->t_inpcb);
1586 return;
1587 }
2063 return;
2064 }
1588#endif
1589 tp = sototcpcb(so);
2065
1590 q = &toep->tp_ddp_state;
1591 hdr = cplhdr(m);
1592 ddp_report = ntohl(hdr->u.ddp_report);
1593 buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1;
1594 bsp = &q->buf_state[buf_idx];
1595
1596#ifdef T3_TRACE
1597 T3_TRACE5(TIDTB(sk),
1598 "new_rx_data_ddp: tp->rcv_nxt 0x%x cur_offset %u "
1599 "hdr seq 0x%x len %u offset %u",
1600 tp->rcv_nxt, bsp->cur_offset, ntohl(hdr->seq),
1601 ntohs(hdr->len), G_DDP_OFFSET(ddp_report));
1602 T3_TRACE1(TIDTB(sk),
1603 "new_rx_data_ddp: ddp_report 0x%x",
1604 ddp_report);
1605#endif
2066 q = &toep->tp_ddp_state;
2067 hdr = cplhdr(m);
2068 ddp_report = ntohl(hdr->u.ddp_report);
2069 buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1;
2070 bsp = &q->buf_state[buf_idx];
2071
2072#ifdef T3_TRACE
2073 T3_TRACE5(TIDTB(sk),
2074 "new_rx_data_ddp: tp->rcv_nxt 0x%x cur_offset %u "
2075 "hdr seq 0x%x len %u offset %u",
2076 tp->rcv_nxt, bsp->cur_offset, ntohl(hdr->seq),
2077 ntohs(hdr->len), G_DDP_OFFSET(ddp_report));
2078 T3_TRACE1(TIDTB(sk),
2079 "new_rx_data_ddp: ddp_report 0x%x",
2080 ddp_report);
2081#endif
1606
2082 CTR4(KTR_TOM,
2083 "new_rx_data_ddp: tp->rcv_nxt 0x%x cur_offset %u "
2084 "hdr seq 0x%x len %u",
2085 tp->rcv_nxt, bsp->cur_offset, ntohl(hdr->seq),
2086 ntohs(hdr->len));
2087 CTR3(KTR_TOM,
2088 "new_rx_data_ddp: offset %u ddp_report 0x%x buf_idx=%d",
2089 G_DDP_OFFSET(ddp_report), ddp_report, buf_idx);
2090
1607 ddp_len = ntohs(hdr->len);
1608 rcv_nxt = ntohl(hdr->seq) + ddp_len;
1609
2091 ddp_len = ntohs(hdr->len);
2092 rcv_nxt = ntohl(hdr->seq) + ddp_len;
2093
1610 /*
1611 * Overload to store old rcv_next
1612 */
1613 m->m_pkthdr.csum_data = tp->rcv_nxt;
2094 m->m_seq = tp->rcv_nxt;
1614 tp->rcv_nxt = rcv_nxt;
1615
2095 tp->rcv_nxt = rcv_nxt;
2096
2097 tp->t_rcvtime = ticks;
1616 /*
1617 * Store the length in m->m_len. We are changing the meaning of
1618 * m->m_len here, we need to be very careful that nothing from now on
1619 * interprets ->len of this packet the usual way.
1620 */
2098 /*
2099 * Store the length in m->m_len. We are changing the meaning of
2100 * m->m_len here, we need to be very careful that nothing from now on
2101 * interprets ->len of this packet the usual way.
2102 */
1621 m->m_len = tp->rcv_nxt - m->m_pkthdr.csum_data;
1622
2103 m->m_len = m->m_pkthdr.len = rcv_nxt - m->m_seq;
2104 INP_UNLOCK(tp->t_inpcb);
2105 CTR3(KTR_TOM,
2106 "new_rx_data_ddp: m_len=%u rcv_next 0x%08x rcv_nxt_prev=0x%08x ",
2107 m->m_len, rcv_nxt, m->m_seq);
1623 /*
1624 * Figure out where the new data was placed in the buffer and store it
1625 * in when. Assumes the buffer offset starts at 0, consumer needs to
1626 * account for page pod's pg_offset.
1627 */
1628 end_offset = G_DDP_OFFSET(ddp_report) + ddp_len;
2108 /*
2109 * Figure out where the new data was placed in the buffer and store it
2110 * in when. Assumes the buffer offset starts at 0, consumer needs to
2111 * account for page pod's pg_offset.
2112 */
2113 end_offset = G_DDP_OFFSET(ddp_report) + ddp_len;
1629#ifdef notyet
1630 TCP_SKB_CB(skb)->when = end_offset - skb->len;
2114 m->m_cur_offset = end_offset - m->m_pkthdr.len;
1631
2115
1632 /*
1633 * We store in mac.raw the address of the gather list where the
1634 * placement happened.
1635 */
1636 skb->mac.raw = (unsigned char *)bsp->gl;
1637#endif
2116 SOCKBUF_LOCK(&so->so_rcv);
2117 m->m_ddp_gl = (unsigned char *)bsp->gl;
2118 m->m_flags |= M_DDP;
1638 bsp->cur_offset = end_offset;
2119 bsp->cur_offset = end_offset;
2120 toep->tp_enqueued_bytes += m->m_pkthdr.len;
1639
1640 /*
2121
2122 /*
2123 * Length is only meaningful for kbuf
2124 */
2125 if (!(bsp->flags & DDP_BF_NOCOPY))
2126 KASSERT(m->m_len <= bsp->gl->dgl_length,
2127 ("length received exceeds ddp pages: len=%d dgl_length=%d",
2128 m->m_len, bsp->gl->dgl_length));
2129
2130 KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
2131 KASSERT(m->m_next == NULL, ("m_len=%p", m->m_next));
2132
2133
2134 /*
1641 * Bit 0 of flags stores whether the DDP buffer is completed.
1642 * Note that other parts of the code depend on this being in bit 0.
1643 */
1644 if ((bsp->flags & DDP_BF_NOINVAL) && end_offset != bsp->gl->dgl_length) {
2135 * Bit 0 of flags stores whether the DDP buffer is completed.
2136 * Note that other parts of the code depend on this being in bit 0.
2137 */
2138 if ((bsp->flags & DDP_BF_NOINVAL) && end_offset != bsp->gl->dgl_length) {
1645#if 0
1646 TCP_SKB_CB(skb)->flags = 0; /* potential spurious completion */
1647#endif
1648 panic("spurious ddp completion");
1649 } else {
2139 panic("spurious ddp completion");
2140 } else {
1650 m->m_pkthdr.csum_flags = !!(ddp_report & F_DDP_BUF_COMPLETE);
1651 if (m->m_pkthdr.csum_flags && !(bsp->flags & DDP_BF_NOFLIP))
2141 m->m_ddp_flags = !!(ddp_report & F_DDP_BUF_COMPLETE);
2142 if (m->m_ddp_flags && !(bsp->flags & DDP_BF_NOFLIP))
1652 q->cur_buf ^= 1; /* flip buffers */
1653 }
1654
1655 if (bsp->flags & DDP_BF_NOCOPY) {
2143 q->cur_buf ^= 1; /* flip buffers */
2144 }
2145
2146 if (bsp->flags & DDP_BF_NOCOPY) {
1656 m->m_pkthdr.csum_flags |= (bsp->flags & DDP_BF_NOCOPY);
2147 m->m_ddp_flags |= (bsp->flags & DDP_BF_NOCOPY);
1657 bsp->flags &= ~DDP_BF_NOCOPY;
1658 }
1659
1660 if (ddp_report & F_DDP_PSH)
2148 bsp->flags &= ~DDP_BF_NOCOPY;
2149 }
2150
2151 if (ddp_report & F_DDP_PSH)
1661 m->m_pkthdr.csum_flags |= DDP_BF_PSH;
2152 m->m_ddp_flags |= DDP_BF_PSH;
2153 if (nomoredata)
2154 m->m_ddp_flags |= DDP_BF_NODATA;
2155
2156 if (__predict_false(G_DDP_DACK_MODE(ddp_report) != toep->tp_delack_mode)) {
2157 toep->tp_delack_mode = G_DDP_DACK_MODE(ddp_report);
2158 toep->tp_delack_seq = tp->rcv_nxt;
2159 }
2160
2161 SBAPPEND(&so->so_rcv, m);
1662
2162
1663 tp->t_rcvtime = ticks;
1664 sbappendstream_locked(&so->so_rcv, m);
1665#ifdef notyet
1666 if (!sock_flag(sk, SOCK_DEAD))
1667 sk->sk_data_ready(sk, 0);
1668#endif
2163 if ((so->so_state & SS_NOFDREF) == 0)
2164 sorwakeup_locked(so);
2165 else
2166 SOCKBUF_UNLOCK(&so->so_rcv);
1669}
1670
1671#define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\
1672 F_DDP_PPOD_PARITY_ERR | F_DDP_PADDING_ERR | F_DDP_OFFSET_ERR |\
1673 F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\
1674 F_DDP_INVALID_PPOD)
1675
1676/*
1677 * Handler for RX_DATA_DDP CPL messages.
1678 */
1679static int
1680do_rx_data_ddp(struct t3cdev *cdev, struct mbuf *m, void *ctx)
1681{
1682 struct toepcb *toep = ctx;
2167}
2168
2169#define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\
2170 F_DDP_PPOD_PARITY_ERR | F_DDP_PADDING_ERR | F_DDP_OFFSET_ERR |\
2171 F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\
2172 F_DDP_INVALID_PPOD)
2173
2174/*
2175 * Handler for RX_DATA_DDP CPL messages.
2176 */
2177static int
2178do_rx_data_ddp(struct t3cdev *cdev, struct mbuf *m, void *ctx)
2179{
2180 struct toepcb *toep = ctx;
1683 struct socket *so = toeptoso(toep);
1684 const struct cpl_rx_data_ddp *hdr = cplhdr(m);
1685
1686 VALIDATE_SOCK(so);
1687
1688 if (__predict_false(ntohl(hdr->ddpvld_status) & DDP_ERR)) {
1689 log(LOG_ERR, "RX_DATA_DDP for TID %u reported error 0x%x\n",
1690 GET_TID(hdr), G_DDP_VALID(ntohl(hdr->ddpvld_status)));
2181 const struct cpl_rx_data_ddp *hdr = cplhdr(m);
2182
2183 VALIDATE_SOCK(so);
2184
2185 if (__predict_false(ntohl(hdr->ddpvld_status) & DDP_ERR)) {
2186 log(LOG_ERR, "RX_DATA_DDP for TID %u reported error 0x%x\n",
2187 GET_TID(hdr), G_DDP_VALID(ntohl(hdr->ddpvld_status)));
1691 return CPL_RET_BUF_DONE;
2188 return (CPL_RET_BUF_DONE);
1692 }
1693#if 0
1694 skb->h.th = tcphdr_skb->h.th;
1695#endif
2189 }
2190#if 0
2191 skb->h.th = tcphdr_skb->h.th;
2192#endif
1696 new_rx_data_ddp(so, m);
2193 new_rx_data_ddp(toep, m);
1697 return (0);
1698}
1699
1700static void
2194 return (0);
2195}
2196
2197static void
1701process_ddp_complete(struct socket *so, struct mbuf *m)
2198process_ddp_complete(struct toepcb *toep, struct mbuf *m)
1702{
2199{
1703 struct tcpcb *tp = sototcpcb(so);
1704 struct toepcb *toep = tp->t_toe;
2200 struct tcpcb *tp = toep->tp_tp;
2201 struct socket *so = toeptoso(toep);
1705 struct ddp_state *q;
1706 struct ddp_buf_state *bsp;
1707 struct cpl_rx_ddp_complete *hdr;
1708 unsigned int ddp_report, buf_idx, when;
2202 struct ddp_state *q;
2203 struct ddp_buf_state *bsp;
2204 struct cpl_rx_ddp_complete *hdr;
2205 unsigned int ddp_report, buf_idx, when;
2206 int nomoredata = 0;
1709
2207
1710#ifdef notyet
1711 if (unlikely(sk_no_receive(sk))) {
1712 handle_excess_rx(sk, skb);
2208 INP_LOCK(tp->t_inpcb);
2209 if (__predict_false(so_no_receive(so))) {
2210 struct inpcb *inp = sotoinpcb(so);
2211
2212 handle_excess_rx(toep, m);
2213 INP_UNLOCK(inp);
1713 return;
1714 }
2214 return;
2215 }
1715#endif
1716 q = &toep->tp_ddp_state;
1717 hdr = cplhdr(m);
1718 ddp_report = ntohl(hdr->ddp_report);
1719 buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1;
2216 q = &toep->tp_ddp_state;
2217 hdr = cplhdr(m);
2218 ddp_report = ntohl(hdr->ddp_report);
2219 buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1;
1720 bsp = &q->buf_state[buf_idx];
2220 m->m_pkthdr.csum_data = tp->rcv_nxt;
1721
2221
2222
2223 SOCKBUF_LOCK(&so->so_rcv);
2224 bsp = &q->buf_state[buf_idx];
1722 when = bsp->cur_offset;
2225 when = bsp->cur_offset;
1723 m->m_len = G_DDP_OFFSET(ddp_report) - when;
2226 m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when;
2227 tp->rcv_nxt += m->m_len;
2228 tp->t_rcvtime = ticks;
2229 INP_UNLOCK(tp->t_inpcb);
1724
2230
2231 KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
1725#ifdef T3_TRACE
1726 T3_TRACE5(TIDTB(sk),
1727 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
1728 "ddp_report 0x%x offset %u, len %u",
1729 tp->rcv_nxt, bsp->cur_offset, ddp_report,
1730 G_DDP_OFFSET(ddp_report), skb->len);
1731#endif
2232#ifdef T3_TRACE
2233 T3_TRACE5(TIDTB(sk),
2234 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2235 "ddp_report 0x%x offset %u, len %u",
2236 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2237 G_DDP_OFFSET(ddp_report), skb->len);
2238#endif
1732
2239 CTR5(KTR_TOM,
2240 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2241 "ddp_report 0x%x offset %u, len %u",
2242 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2243 G_DDP_OFFSET(ddp_report), m->m_len);
2244
1733 bsp->cur_offset += m->m_len;
1734
2245 bsp->cur_offset += m->m_len;
2246
1735 if (!(bsp->flags & DDP_BF_NOFLIP))
2247 if (!(bsp->flags & DDP_BF_NOFLIP)) {
1736 q->cur_buf ^= 1; /* flip buffers */
2248 q->cur_buf ^= 1; /* flip buffers */
1737
2249 if (G_DDP_OFFSET(ddp_report) < q->kbuf[0]->dgl_length)
2250 nomoredata=1;
2251 }
2252
1738#ifdef T3_TRACE
1739 T3_TRACE4(TIDTB(sk),
1740 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
1741 "ddp_report %u offset %u",
1742 tp->rcv_nxt, bsp->cur_offset, ddp_report,
1743 G_DDP_OFFSET(ddp_report));
1744#endif
2253#ifdef T3_TRACE
2254 T3_TRACE4(TIDTB(sk),
2255 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2256 "ddp_report %u offset %u",
2257 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2258 G_DDP_OFFSET(ddp_report));
2259#endif
1745#if 0
1746 skb->mac.raw = (unsigned char *)bsp->gl;
1747#endif
1748 m->m_pkthdr.csum_flags = (bsp->flags & DDP_BF_NOCOPY) | 1;
2260 CTR4(KTR_TOM,
2261 "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
2262 "ddp_report %u offset %u",
2263 tp->rcv_nxt, bsp->cur_offset, ddp_report,
2264 G_DDP_OFFSET(ddp_report));
2265
2266 m->m_ddp_gl = (unsigned char *)bsp->gl;
2267 m->m_flags |= M_DDP;
2268 m->m_ddp_flags = (bsp->flags & DDP_BF_NOCOPY) | 1;
1749 if (bsp->flags & DDP_BF_NOCOPY)
1750 bsp->flags &= ~DDP_BF_NOCOPY;
2269 if (bsp->flags & DDP_BF_NOCOPY)
2270 bsp->flags &= ~DDP_BF_NOCOPY;
1751 m->m_pkthdr.csum_data = tp->rcv_nxt;
1752 tp->rcv_nxt += m->m_len;
2271 if (nomoredata)
2272 m->m_ddp_flags |= DDP_BF_NODATA;
1753
2273
1754 tp->t_rcvtime = ticks;
1755 sbappendstream_locked(&so->so_rcv, m);
1756#ifdef notyet
1757 if (!sock_flag(sk, SOCK_DEAD))
1758 sk->sk_data_ready(sk, 0);
1759#endif
2274 SBAPPEND(&so->so_rcv, m);
2275
2276 if ((so->so_state & SS_NOFDREF) == 0)
2277 sorwakeup_locked(so);
2278 else
2279 SOCKBUF_UNLOCK(&so->so_rcv);
1760}
1761
1762/*
1763 * Handler for RX_DDP_COMPLETE CPL messages.
1764 */
1765static int
1766do_rx_ddp_complete(struct t3cdev *cdev, struct mbuf *m, void *ctx)
1767{
1768 struct toepcb *toep = ctx;
2280}
2281
2282/*
2283 * Handler for RX_DDP_COMPLETE CPL messages.
2284 */
2285static int
2286do_rx_ddp_complete(struct t3cdev *cdev, struct mbuf *m, void *ctx)
2287{
2288 struct toepcb *toep = ctx;
1769 struct socket *so = toeptoso(toep);
1770
1771 VALIDATE_SOCK(so);
1772#if 0
1773 skb->h.th = tcphdr_skb->h.th;
1774#endif
2289
2290 VALIDATE_SOCK(so);
2291#if 0
2292 skb->h.th = tcphdr_skb->h.th;
2293#endif
1775 process_ddp_complete(so, m);
2294 process_ddp_complete(toep, m);
1776 return (0);
1777}
1778
1779/*
1780 * Move a socket to TIME_WAIT state. We need to make some adjustments to the
1781 * socket state before calling tcp_time_wait to comply with its expectations.
1782 */
1783static void

--- 12 unchanged lines hidden (view full) ---

1796 tp->rcv_nxt++;
1797
1798 tp->ts_recent_age = 0; /* defeat recycling */
1799 tp->t_srtt = 0; /* defeat tcp_update_metrics */
1800 tcp_twstart(tp);
1801}
1802
1803/*
2295 return (0);
2296}
2297
2298/*
2299 * Move a socket to TIME_WAIT state. We need to make some adjustments to the
2300 * socket state before calling tcp_time_wait to comply with its expectations.
2301 */
2302static void

--- 12 unchanged lines hidden (view full) ---

2315 tp->rcv_nxt++;
2316
2317 tp->ts_recent_age = 0; /* defeat recycling */
2318 tp->t_srtt = 0; /* defeat tcp_update_metrics */
2319 tcp_twstart(tp);
2320}
2321
2322/*
2323 * For TCP DDP a PEER_CLOSE may also be an implicit RX_DDP_COMPLETE. This
2324 * function deals with the data that may be reported along with the FIN.
2325 * Returns -1 if no further processing of the PEER_CLOSE is needed, >= 0 to
2326 * perform normal FIN-related processing. In the latter case 1 indicates that
2327 * there was an implicit RX_DDP_COMPLETE and the skb should not be freed, 0 the
2328 * skb can be freed.
2329 */
2330static int
2331handle_peer_close_data(struct socket *so, struct mbuf *m)
2332{
2333 struct tcpcb *tp = sototcpcb(so);
2334 struct toepcb *toep = tp->t_toe;
2335 struct ddp_state *q;
2336 struct ddp_buf_state *bsp;
2337 struct cpl_peer_close *req = cplhdr(m);
2338 unsigned int rcv_nxt = ntohl(req->rcv_nxt) - 1; /* exclude FIN */
2339
2340 if (tp->rcv_nxt == rcv_nxt) /* no data */
2341 return (0);
2342
2343 if (__predict_false(so_no_receive(so))) {
2344 handle_excess_rx(toep, m);
2345
2346 /*
2347 * Although we discard the data we want to process the FIN so
2348 * that PEER_CLOSE + data behaves the same as RX_DATA_DDP +
2349 * PEER_CLOSE without data. In particular this PEER_CLOSE
2350 * may be what will close the connection. We return 1 because
2351 * handle_excess_rx() already freed the packet.
2352 */
2353 return (1);
2354 }
2355
2356 INP_LOCK_ASSERT(tp->t_inpcb);
2357 q = &toep->tp_ddp_state;
2358 SOCKBUF_LOCK(&so->so_rcv);
2359 bsp = &q->buf_state[q->cur_buf];
2360 m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt;
2361 KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
2362 m->m_ddp_gl = (unsigned char *)bsp->gl;
2363 m->m_flags |= M_DDP;
2364 m->m_cur_offset = bsp->cur_offset;
2365 m->m_ddp_flags =
2366 DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1;
2367 m->m_seq = tp->rcv_nxt;
2368 tp->rcv_nxt = rcv_nxt;
2369 bsp->cur_offset += m->m_pkthdr.len;
2370 if (!(bsp->flags & DDP_BF_NOFLIP))
2371 q->cur_buf ^= 1;
2372 tp->t_rcvtime = ticks;
2373 SBAPPEND(&so->so_rcv, m);
2374 if (__predict_true((so->so_state & SS_NOFDREF) == 0))
2375 sorwakeup_locked(so);
2376 else
2377 SOCKBUF_UNLOCK(&so->so_rcv);
2378 return (1);
2379}
2380
2381/*
1804 * Handle a peer FIN.
1805 */
1806static void
1807do_peer_fin(struct socket *so, struct mbuf *m)
1808{
1809 struct tcpcb *tp = sototcpcb(so);
1810 struct toepcb *toep = tp->t_toe;
2382 * Handle a peer FIN.
2383 */
2384static void
2385do_peer_fin(struct socket *so, struct mbuf *m)
2386{
2387 struct tcpcb *tp = sototcpcb(so);
2388 struct toepcb *toep = tp->t_toe;
1811 int keep = 0, dead = (so->so_state & SS_NOFDREF);
1812
1813 DPRINTF("do_peer_fin state=%d dead=%d\n", tp->t_state, !!dead);
2389 int keep = 0;
2390 DPRINTF("do_peer_fin state=%d\n", tp->t_state);
1814
1815#ifdef T3_TRACE
1816 T3_TRACE0(TIDTB(sk),"do_peer_fin:");
1817#endif
1818
1819 if (!is_t3a(TOE_DEV(so)) && (toep->tp_flags & TP_ABORT_RPL_PENDING)) {
1820 printf("abort_pending set\n");
1821
1822 goto out;
1823 }
2391
2392#ifdef T3_TRACE
2393 T3_TRACE0(TIDTB(sk),"do_peer_fin:");
2394#endif
2395
2396 if (!is_t3a(TOE_DEV(so)) && (toep->tp_flags & TP_ABORT_RPL_PENDING)) {
2397 printf("abort_pending set\n");
2398
2399 goto out;
2400 }
1824
1825#ifdef notyet
1826 if (ULP_MODE(tp) == ULP_MODE_TCPDDP) {
1827 keep = handle_peer_close_data(so, skb);
1828 if (keep < 0)
1829 return;
1830 }
1831 sk->sk_shutdown |= RCV_SHUTDOWN;
1832 sock_set_flag(so, SOCK_DONE);
1833#endif
1834 INP_INFO_WLOCK(&tcbinfo);
1835 INP_LOCK(tp->t_inpcb);
2401 INP_INFO_WLOCK(&tcbinfo);
2402 INP_LOCK(tp->t_inpcb);
1836 if (TCPS_HAVERCVDFIN(tp->t_state) == 0)
2403 if (toep->tp_ulp_mode == ULP_MODE_TCPDDP) {
2404 keep = handle_peer_close_data(so, m);
2405 if (keep < 0) {
2406 INP_INFO_WUNLOCK(&tcbinfo);
2407 INP_UNLOCK(tp->t_inpcb);
2408 return;
2409 }
2410 }
2411 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {
1837 socantrcvmore(so);
2412 socantrcvmore(so);
2413 /*
2414 * If connection is half-synchronized
2415 * (ie NEEDSYN flag on) then delay ACK,
2416 * so it may be piggybacked when SYN is sent.
2417 * Otherwise, since we received a FIN then no
2418 * more input can be expected, send ACK now.
2419 */
2420 if (tp->t_flags & TF_NEEDSYN)
2421 tp->t_flags |= TF_DELACK;
2422 else
2423 tp->t_flags |= TF_ACKNOW;
2424 tp->rcv_nxt++;
2425 }
2426
1838 switch (tp->t_state) {
1839 case TCPS_SYN_RECEIVED:
1840 tp->t_starttime = ticks;
1841 /* FALLTHROUGH */
1842 case TCPS_ESTABLISHED:
1843 tp->t_state = TCPS_CLOSE_WAIT;
1844 break;
1845 case TCPS_FIN_WAIT_1:

--- 7 unchanged lines hidden (view full) ---

1853 * be treated as an abort_rpl, i.e., transition the connection
1854 * to TCP_CLOSE (note that the host stack does this at the
1855 * time of generating the RST but we must wait for HW).
1856 * Otherwise we enter TIME_WAIT.
1857 */
1858 t3_release_offload_resources(toep);
1859 if (toep->tp_flags & TP_ABORT_RPL_PENDING) {
1860 tp = tcp_close(tp);
2427 switch (tp->t_state) {
2428 case TCPS_SYN_RECEIVED:
2429 tp->t_starttime = ticks;
2430 /* FALLTHROUGH */
2431 case TCPS_ESTABLISHED:
2432 tp->t_state = TCPS_CLOSE_WAIT;
2433 break;
2434 case TCPS_FIN_WAIT_1:

--- 7 unchanged lines hidden (view full) ---

2442 * be treated as an abort_rpl, i.e., transition the connection
2443 * to TCP_CLOSE (note that the host stack does this at the
2444 * time of generating the RST but we must wait for HW).
2445 * Otherwise we enter TIME_WAIT.
2446 */
2447 t3_release_offload_resources(toep);
2448 if (toep->tp_flags & TP_ABORT_RPL_PENDING) {
2449 tp = tcp_close(tp);
1861 } else
2450 } else {
1862 enter_timewait(so);
2451 enter_timewait(so);
2452 }
1863 break;
1864 default:
1865 log(LOG_ERR,
1866 "%s: TID %u received PEER_CLOSE in bad state %d\n",
1867 TOE_DEV(so)->tod_name, toep->tp_tid, tp->t_state);
1868 }
1869 INP_INFO_WUNLOCK(&tcbinfo);
1870 if (tp)
1871 INP_UNLOCK(tp->t_inpcb);
1872
2453 break;
2454 default:
2455 log(LOG_ERR,
2456 "%s: TID %u received PEER_CLOSE in bad state %d\n",
2457 TOE_DEV(so)->tod_name, toep->tp_tid, tp->t_state);
2458 }
2459 INP_INFO_WUNLOCK(&tcbinfo);
2460 if (tp)
2461 INP_UNLOCK(tp->t_inpcb);
2462
1873 if (!dead) {
1874 DPRINTF("waking up waiters on %p rcv_notify=%d flags=0x%x\n", so, sb_notify(&so->so_rcv), so->so_rcv.sb_flags);
1875
1876 sorwakeup(so);
1877 sowwakeup(so);
1878 wakeup(&so->so_timeo);
1879#ifdef notyet
1880 sk->sk_state_change(sk);
2463 DPRINTF("waking up waiters on %p rcv_notify=%d flags=0x%x\n", so, sb_notify(&so->so_rcv), so->so_rcv.sb_flags);
1881
2464
1882 /* Do not send POLL_HUP for half duplex close. */
1883 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1884 sk->sk_state == TCP_CLOSE)
1885 sk_wake_async(so, 1, POLL_HUP);
1886 else
1887 sk_wake_async(so, 1, POLL_IN);
2465#ifdef notyet
2466 /* Do not send POLL_HUP for half duplex close. */
2467 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
2468 sk->sk_state == TCP_CLOSE)
2469 sk_wake_async(so, 1, POLL_HUP);
2470 else
2471 sk_wake_async(so, 1, POLL_IN);
1888#endif
2472#endif
1889 }
2473
1890out:
1891 if (!keep)
1892 m_free(m);
1893}
1894
1895/*
1896 * Handler for PEER_CLOSE CPL messages.
1897 */

--- 26 unchanged lines hidden (view full) ---

1924 INP_INFO_WLOCK(&tcbinfo);
1925 INP_LOCK(tp->t_inpcb);
1926 switch (tp->t_state) {
1927 case TCPS_CLOSING: /* see FIN_WAIT2 case in do_peer_fin */
1928 t3_release_offload_resources(toep);
1929 if (toep->tp_flags & TP_ABORT_RPL_PENDING) {
1930 tp = tcp_close(tp);
1931
2474out:
2475 if (!keep)
2476 m_free(m);
2477}
2478
2479/*
2480 * Handler for PEER_CLOSE CPL messages.
2481 */

--- 26 unchanged lines hidden (view full) ---

2508 INP_INFO_WLOCK(&tcbinfo);
2509 INP_LOCK(tp->t_inpcb);
2510 switch (tp->t_state) {
2511 case TCPS_CLOSING: /* see FIN_WAIT2 case in do_peer_fin */
2512 t3_release_offload_resources(toep);
2513 if (toep->tp_flags & TP_ABORT_RPL_PENDING) {
2514 tp = tcp_close(tp);
2515
1932 } else
2516 } else {
1933 enter_timewait(so);
2517 enter_timewait(so);
2518 soisdisconnected(so);
2519 }
1934 break;
1935 case TCPS_LAST_ACK:
1936 /*
1937 * In this state we don't care about pending abort_rpl.
1938 * If we've sent abort_req it was post-close and was sent too
1939 * late, this close_con_rpl is the actual last message.
1940 */
1941 t3_release_offload_resources(toep);
1942 tp = tcp_close(tp);
1943 break;
1944 case TCPS_FIN_WAIT_1:
2520 break;
2521 case TCPS_LAST_ACK:
2522 /*
2523 * In this state we don't care about pending abort_rpl.
2524 * If we've sent abort_req it was post-close and was sent too
2525 * late, this close_con_rpl is the actual last message.
2526 */
2527 t3_release_offload_resources(toep);
2528 tp = tcp_close(tp);
2529 break;
2530 case TCPS_FIN_WAIT_1:
1945#ifdef notyet
1946 dst_confirm(sk->sk_dst_cache);
1947#endif
1948 soisdisconnecting(so);
1949
1950 if ((so->so_state & SS_NOFDREF) == 0) {
1951 /*
1952 * Wake up lingering close
1953 */
1954 sowwakeup(so);
1955 sorwakeup(so);
1956 wakeup(&so->so_timeo);
1957 } else if ((so->so_options & SO_LINGER) && so->so_linger == 0 &&
2531 /*
2532 * If we can't receive any more
2533 * data, then closing user can proceed.
2534 * Starting the timer is contrary to the
2535 * specification, but if we don't get a FIN
2536 * we'll hang forever.
2537 *
2538 * XXXjl:
2539 * we should release the tp also, and use a
2540 * compressed state.
2541 */
2542 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
2543 int timeout;
2544
2545 soisdisconnected(so);
2546 timeout = (tcp_fast_finwait2_recycle) ?
2547 tcp_finwait2_timeout : tcp_maxidle;
2548 tcp_timer_activate(tp, TT_2MSL, timeout);
2549 }
2550 tp->t_state = TCPS_FIN_WAIT_2;
2551 if ((so->so_options & SO_LINGER) && so->so_linger == 0 &&
1958 (toep->tp_flags & TP_ABORT_SHUTDOWN) == 0) {
2552 (toep->tp_flags & TP_ABORT_SHUTDOWN) == 0) {
1959 tp = cxgb_tcp_drop(tp, 0);
2553 tp = tcp_drop(tp, 0);
1960 }
1961
1962 break;
1963 default:
1964 log(LOG_ERR,
1965 "%s: TID %u received CLOSE_CON_RPL in bad state %d\n",
1966 TOE_DEV(so)->tod_name, toep->tp_tid,
1967 tp->t_state);
1968 }
1969 INP_INFO_WUNLOCK(&tcbinfo);
1970 if (tp)
1971 INP_UNLOCK(tp->t_inpcb);
1972out:
2554 }
2555
2556 break;
2557 default:
2558 log(LOG_ERR,
2559 "%s: TID %u received CLOSE_CON_RPL in bad state %d\n",
2560 TOE_DEV(so)->tod_name, toep->tp_tid,
2561 tp->t_state);
2562 }
2563 INP_INFO_WUNLOCK(&tcbinfo);
2564 if (tp)
2565 INP_UNLOCK(tp->t_inpcb);
2566out:
1973 m_free(m);
2567 m_freem(m);
1974}
1975
1976/*
1977 * Handler for CLOSE_CON_RPL CPL messages.
1978 */
1979static int
1980do_close_con_rpl(struct t3cdev *cdev, struct mbuf *m,
1981 void *ctx)

--- 19 unchanged lines hidden (view full) ---

2001 struct tcpcb *tp = sototcpcb(so);
2002 struct toepcb *toep = tp->t_toe;
2003
2004#ifdef T3_TRACE
2005 T3_TRACE1(TIDTB(sk),
2006 "process_abort_rpl: GTS rpl pending %d",
2007 sock_flag(sk, ABORT_RPL_PENDING));
2008#endif
2568}
2569
2570/*
2571 * Handler for CLOSE_CON_RPL CPL messages.
2572 */
2573static int
2574do_close_con_rpl(struct t3cdev *cdev, struct mbuf *m,
2575 void *ctx)

--- 19 unchanged lines hidden (view full) ---

2595 struct tcpcb *tp = sototcpcb(so);
2596 struct toepcb *toep = tp->t_toe;
2597
2598#ifdef T3_TRACE
2599 T3_TRACE1(TIDTB(sk),
2600 "process_abort_rpl: GTS rpl pending %d",
2601 sock_flag(sk, ABORT_RPL_PENDING));
2602#endif
2603
2604 INP_INFO_WLOCK(&tcbinfo);
2009 INP_LOCK(tp->t_inpcb);
2010
2011 if (toep->tp_flags & TP_ABORT_RPL_PENDING) {
2012 /*
2013 * XXX panic on tcpdrop
2014 */
2015 if (!(toep->tp_flags & TP_ABORT_RPL_RCVD) && !is_t3a(TOE_DEV(so)))
2016 toep->tp_flags |= TP_ABORT_RPL_RCVD;
2017 else {
2018 toep->tp_flags &= ~(TP_ABORT_RPL_RCVD|TP_ABORT_RPL_PENDING);
2019 if (!(toep->tp_flags & TP_ABORT_REQ_RCVD) ||
2020 !is_t3a(TOE_DEV(so))) {
2021 if (toep->tp_flags & TP_ABORT_REQ_RCVD)
2022 panic("TP_ABORT_REQ_RCVD set");
2605 INP_LOCK(tp->t_inpcb);
2606
2607 if (toep->tp_flags & TP_ABORT_RPL_PENDING) {
2608 /*
2609 * XXX panic on tcpdrop
2610 */
2611 if (!(toep->tp_flags & TP_ABORT_RPL_RCVD) && !is_t3a(TOE_DEV(so)))
2612 toep->tp_flags |= TP_ABORT_RPL_RCVD;
2613 else {
2614 toep->tp_flags &= ~(TP_ABORT_RPL_RCVD|TP_ABORT_RPL_PENDING);
2615 if (!(toep->tp_flags & TP_ABORT_REQ_RCVD) ||
2616 !is_t3a(TOE_DEV(so))) {
2617 if (toep->tp_flags & TP_ABORT_REQ_RCVD)
2618 panic("TP_ABORT_REQ_RCVD set");
2023 INP_INFO_WLOCK(&tcbinfo);
2024 INP_LOCK(tp->t_inpcb);
2025 t3_release_offload_resources(toep);
2026 tp = tcp_close(tp);
2619 t3_release_offload_resources(toep);
2620 tp = tcp_close(tp);
2027 INP_INFO_WUNLOCK(&tcbinfo);
2028 }
2029 }
2030 }
2031 if (tp)
2032 INP_UNLOCK(tp->t_inpcb);
2621 }
2622 }
2623 }
2624 if (tp)
2625 INP_UNLOCK(tp->t_inpcb);
2626 INP_INFO_WUNLOCK(&tcbinfo);
2033
2034 m_free(m);
2035}
2036
2037/*
2038 * Handle an ABORT_RPL_RSS CPL message.
2039 */
2040static int

--- 43 unchanged lines hidden (view full) ---

2084 so = toeptoso(toep); /* <- XXX panic */
2085 toepcb_hold(toep);
2086 process_abort_rpl(so, m);
2087 toepcb_release(toep);
2088 return (0);
2089}
2090
2091/*
2627
2628 m_free(m);
2629}
2630
2631/*
2632 * Handle an ABORT_RPL_RSS CPL message.
2633 */
2634static int

--- 43 unchanged lines hidden (view full) ---

2678 so = toeptoso(toep); /* <- XXX panic */
2679 toepcb_hold(toep);
2680 process_abort_rpl(so, m);
2681 toepcb_release(toep);
2682 return (0);
2683}
2684
2685/*
2092 * Convert the status code of an ABORT_REQ into a Linux error code. Also
2686 * Convert the status code of an ABORT_REQ into a FreeBSD error code. Also
2093 * indicate whether RST should be sent in response.
2094 */
2095static int
2096abort_status_to_errno(struct socket *so, int abort_reason, int *need_rst)
2097{
2098 struct tcpcb *tp = sototcpcb(so);
2099
2100 switch (abort_reason) {

--- 183 unchanged lines hidden (view full) ---

2284 * be ignored and the connection should be closed now.
2285 * c) We have sent a regular abort_req that will get to TP too late.
2286 * That will generate an abort_rpl with status 0, wait for it.
2287 */
2288 if (((toep->tp_flags & TP_ABORT_RPL_PENDING) == 0) ||
2289 (is_t3a(TOE_DEV(so)) && (toep->tp_flags & TP_CLOSE_CON_REQUESTED))) {
2290 so->so_error = abort_status_to_errno(so, req->status,
2291 &rst_status);
2687 * indicate whether RST should be sent in response.
2688 */
2689static int
2690abort_status_to_errno(struct socket *so, int abort_reason, int *need_rst)
2691{
2692 struct tcpcb *tp = sototcpcb(so);
2693
2694 switch (abort_reason) {

--- 183 unchanged lines hidden (view full) ---

2878 * be ignored and the connection should be closed now.
2879 * c) We have sent a regular abort_req that will get to TP too late.
2880 * That will generate an abort_rpl with status 0, wait for it.
2881 */
2882 if (((toep->tp_flags & TP_ABORT_RPL_PENDING) == 0) ||
2883 (is_t3a(TOE_DEV(so)) && (toep->tp_flags & TP_CLOSE_CON_REQUESTED))) {
2884 so->so_error = abort_status_to_errno(so, req->status,
2885 &rst_status);
2292#if 0
2293 if (!sock_flag(sk, SOCK_DEAD))
2294 sk->sk_error_report(sk);
2295#endif
2886 if (__predict_true((so->so_state & SS_NOFDREF) == 0))
2887 sorwakeup(so);
2296 /*
2297 * SYN_RECV needs special processing. If abort_syn_rcv()
2298 * returns 0 is has taken care of the abort.
2299 */
2300 if ((tp->t_state == TCPS_SYN_RECEIVED) && !abort_syn_rcv(so, m))
2301 goto skip;
2302
2303 t3_release_offload_resources(toep);

--- 204 unchanged lines hidden (view full) ---

2508static void
2509syncache_add_accept_req(struct cpl_pass_accept_req *req, struct socket *lso, struct toepcb *toep)
2510{
2511 struct in_conninfo inc;
2512 struct tcpopt to;
2513 struct tcphdr th;
2514 struct inpcb *inp;
2515 int mss, wsf, sack, ts;
2888 /*
2889 * SYN_RECV needs special processing. If abort_syn_rcv()
2890 * returns 0 is has taken care of the abort.
2891 */
2892 if ((tp->t_state == TCPS_SYN_RECEIVED) && !abort_syn_rcv(so, m))
2893 goto skip;
2894
2895 t3_release_offload_resources(toep);

--- 204 unchanged lines hidden (view full) ---

3100static void
3101syncache_add_accept_req(struct cpl_pass_accept_req *req, struct socket *lso, struct toepcb *toep)
3102{
3103 struct in_conninfo inc;
3104 struct tcpopt to;
3105 struct tcphdr th;
3106 struct inpcb *inp;
3107 int mss, wsf, sack, ts;
2516
3108 uint32_t rcv_isn = ntohl(req->rcv_isn);
3109
2517 bzero(&to, sizeof(struct tcpopt));
2518 inp = sotoinpcb(lso);
2519
2520 /*
2521 * Fill out information for entering us into the syncache
2522 */
2523 inc.inc_fport = th.th_sport = req->peer_port;
2524 inc.inc_lport = th.th_dport = req->local_port;
3110 bzero(&to, sizeof(struct tcpopt));
3111 inp = sotoinpcb(lso);
3112
3113 /*
3114 * Fill out information for entering us into the syncache
3115 */
3116 inc.inc_fport = th.th_sport = req->peer_port;
3117 inc.inc_lport = th.th_dport = req->local_port;
2525 toep->tp_iss = th.th_seq = req->rcv_isn;
3118 th.th_seq = req->rcv_isn;
2526 th.th_flags = TH_SYN;
2527
3119 th.th_flags = TH_SYN;
3120
2528 toep->tp_delack_seq = toep->tp_rcv_wup = toep->tp_copied_seq = ntohl(req->rcv_isn);
3121 toep->tp_iss = toep->tp_delack_seq = toep->tp_rcv_wup = toep->tp_copied_seq = rcv_isn + 1;
3122
2529
2530 inc.inc_isipv6 = 0;
2531 inc.inc_len = 0;
2532 inc.inc_faddr.s_addr = req->peer_ip;
2533 inc.inc_laddr.s_addr = req->local_ip;
2534
2535 DPRINTF("syncache add of %d:%d %d:%d\n",
2536 ntohl(req->local_ip), ntohs(req->local_port),
2537 ntohl(req->peer_ip), ntohs(req->peer_port));
2538
2539 mss = req->tcp_options.mss;
2540 wsf = req->tcp_options.wsf;
2541 ts = req->tcp_options.tstamp;
2542 sack = req->tcp_options.sack;
2543 to.to_mss = mss;
2544 to.to_wscale = wsf;
2545 to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0);
3123
3124 inc.inc_isipv6 = 0;
3125 inc.inc_len = 0;
3126 inc.inc_faddr.s_addr = req->peer_ip;
3127 inc.inc_laddr.s_addr = req->local_ip;
3128
3129 DPRINTF("syncache add of %d:%d %d:%d\n",
3130 ntohl(req->local_ip), ntohs(req->local_port),
3131 ntohl(req->peer_ip), ntohs(req->peer_port));
3132
3133 mss = req->tcp_options.mss;
3134 wsf = req->tcp_options.wsf;
3135 ts = req->tcp_options.tstamp;
3136 sack = req->tcp_options.sack;
3137 to.to_mss = mss;
3138 to.to_wscale = wsf;
3139 to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0);
2546
2547 INP_INFO_WLOCK(&tcbinfo);
2548 INP_LOCK(inp);
2549 syncache_offload_add(&inc, &to, &th, inp, &lso, &cxgb_toe_usrreqs, toep);
2550}
2551
2552
2553/*
2554 * Process a CPL_PASS_ACCEPT_REQ message. Does the part that needs the socket

--- 94 unchanged lines hidden (view full) ---

2649 }
2650 /*
2651 * Point to our listen socket until accept
2652 */
2653 newtoep->tp_tp = tp;
2654 newtoep->tp_flags = TP_SYN_RCVD;
2655 newtoep->tp_tid = tid;
2656 newtoep->tp_toedev = tdev;
3140 INP_INFO_WLOCK(&tcbinfo);
3141 INP_LOCK(inp);
3142 syncache_offload_add(&inc, &to, &th, inp, &lso, &cxgb_toe_usrreqs, toep);
3143}
3144
3145
3146/*
3147 * Process a CPL_PASS_ACCEPT_REQ message. Does the part that needs the socket

--- 94 unchanged lines hidden (view full) ---

3242 }
3243 /*
3244 * Point to our listen socket until accept
3245 */
3246 newtoep->tp_tp = tp;
3247 newtoep->tp_flags = TP_SYN_RCVD;
3248 newtoep->tp_tid = tid;
3249 newtoep->tp_toedev = tdev;
3250 tp->rcv_wnd = select_rcv_wnd(tdev, so);
2657
3251
2658 printf("inserting tid=%d\n", tid);
2659 cxgb_insert_tid(cdev, d->client, newtoep, tid);
2660 SOCK_LOCK(so);
2661 LIST_INSERT_HEAD(&lctx->synq_head, newtoep, synq_entry);
2662 SOCK_UNLOCK(so);
2663
3252 cxgb_insert_tid(cdev, d->client, newtoep, tid);
3253 SOCK_LOCK(so);
3254 LIST_INSERT_HEAD(&lctx->synq_head, newtoep, synq_entry);
3255 SOCK_UNLOCK(so);
3256
2664
2665 if (lctx->ulp_mode) {
3257 newtoep->tp_ulp_mode = TOM_TUNABLE(tdev, ddp) && !(so->so_options & SO_NO_DDP) &&
3258 tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0;
3259
3260 if (newtoep->tp_ulp_mode) {
2666 ddp_mbuf = m_gethdr(M_NOWAIT, MT_DATA);
2667
3261 ddp_mbuf = m_gethdr(M_NOWAIT, MT_DATA);
3262
2668 if (!ddp_mbuf)
3263 if (ddp_mbuf == NULL)
2669 newtoep->tp_ulp_mode = 0;
3264 newtoep->tp_ulp_mode = 0;
2670 else
2671 newtoep->tp_ulp_mode = lctx->ulp_mode;
2672 }
3265 }
2673
3266
3267 CTR4(KTR_TOM, "ddp=%d rcv_wnd=%ld min_win=%d ulp_mode=%d",
3268 TOM_TUNABLE(tdev, ddp), tp->rcv_wnd, MIN_DDP_RCV_WIN, newtoep->tp_ulp_mode);
2674 set_arp_failure_handler(reply_mbuf, pass_accept_rpl_arp_failure);
3269 set_arp_failure_handler(reply_mbuf, pass_accept_rpl_arp_failure);
2675
2676 DPRINTF("adding request to syn cache\n");
2677
2678 /*
2679 * XXX workaround for lack of syncache drop
2680 */
2681 toepcb_hold(newtoep);
2682 syncache_add_accept_req(req, so, newtoep);
3270 /*
3271 * XXX workaround for lack of syncache drop
3272 */
3273 toepcb_hold(newtoep);
3274 syncache_add_accept_req(req, so, newtoep);
2683
2684
3275
2685
2686 rpl = cplhdr(reply_mbuf);
2687 reply_mbuf->m_pkthdr.len = reply_mbuf->m_len = sizeof(*rpl);
2688 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
2689 rpl->wr.wr_lo = 0;
2690 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, tid));
2691 rpl->opt2 = htonl(calc_opt2(so, tdev));
2692 rpl->rsvd = rpl->opt2; /* workaround for HW bug */
2693 rpl->peer_ip = req->peer_ip; // req->peer_ip is not overwritten
2694
3276 rpl = cplhdr(reply_mbuf);
3277 reply_mbuf->m_pkthdr.len = reply_mbuf->m_len = sizeof(*rpl);
3278 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3279 rpl->wr.wr_lo = 0;
3280 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, tid));
3281 rpl->opt2 = htonl(calc_opt2(so, tdev));
3282 rpl->rsvd = rpl->opt2; /* workaround for HW bug */
3283 rpl->peer_ip = req->peer_ip; // req->peer_ip is not overwritten
3284
2695 DPRINTF("accept smt_idx=%d\n", e->smt_idx);
2696
2697 rpl->opt0h = htonl(calc_opt0h(so, select_mss(td, NULL, dst->rt_ifp->if_mtu)) |
2698 V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
3285 rpl->opt0h = htonl(calc_opt0h(so, select_mss(td, NULL, dst->rt_ifp->if_mtu)) |
3286 V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
2699 rpl->opt0l_status = htonl(calc_opt0l(so, lctx->ulp_mode) |
3287 rpl->opt0l_status = htonl(calc_opt0l(so, newtoep->tp_ulp_mode) |
2700 CPL_PASS_OPEN_ACCEPT);
2701
2702 DPRINTF("opt0l_status=%08x\n", rpl->opt0l_status);
2703
3288 CPL_PASS_OPEN_ACCEPT);
3289
3290 DPRINTF("opt0l_status=%08x\n", rpl->opt0l_status);
3291
2704 m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, so));
2705
2706#ifdef DEBUG_PRINT
2707 {
2708 int i;
2709
2710 DPRINTF("rpl:\n");
2711 uint32_t *rplbuf = mtod(reply_mbuf, uint32_t *);
3292 m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, newtoep));
2712
3293
2713 for (i = 0; i < sizeof(*rpl)/sizeof(uint32_t); i++)
2714 DPRINTF("[%d] %08x\n", i, rplbuf[i]);
2715 }
2716#endif
2717
2718
2719 l2t_send(cdev, reply_mbuf, e);
2720 m_free(m);
3294 l2t_send(cdev, reply_mbuf, e);
3295 m_free(m);
2721#ifdef notyet
2722 /*
2723 * XXX this call path has to be converted to not depend on sockets
2724 */
2725 if (newtoep->tp_ulp_mode)
2726 __set_tcb_field(newso, ddp_mbuf, W_TCB_RX_DDP_FLAGS,
3296 if (newtoep->tp_ulp_mode) {
3297 __set_tcb_field(newtoep, ddp_mbuf, W_TCB_RX_DDP_FLAGS,
2727 V_TF_DDP_OFF(1) |
2728 TP_DDP_TIMER_WORKAROUND_MASK,
2729 V_TF_DDP_OFF(1) |
3298 V_TF_DDP_OFF(1) |
3299 TP_DDP_TIMER_WORKAROUND_MASK,
3300 V_TF_DDP_OFF(1) |
2730 TP_DDP_TIMER_WORKAROUND_VAL, 1);
3301 TP_DDP_TIMER_WORKAROUND_VAL, 1);
3302 } else
3303 printf("not offloading\n");
3304
3305
2731
3306
2732#endif
2733 return;
2734reject:
2735 if (tdev->tod_ttid == TOE_ID_CHELSIO_T3)
2736 mk_pass_accept_rpl(reply_mbuf, m);
2737 else
3307 return;
3308reject:
3309 if (tdev->tod_ttid == TOE_ID_CHELSIO_T3)
3310 mk_pass_accept_rpl(reply_mbuf, m);
3311 else
2738 mk_tid_release(reply_mbuf, NULL, tid);
3312 mk_tid_release(reply_mbuf, newtoep, tid);
2739 cxgb_ofld_send(cdev, reply_mbuf);
2740 m_free(m);
2741out:
2742#if 0
2743 TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS);
2744#else
2745 return;
2746#endif

--- 41 unchanged lines hidden (view full) ---

2788#endif
2789
2790 process_pass_accept_req(lso, m, &d->tdev, listen_ctx);
2791 return (0);
2792}
2793
2794/*
2795 * Called when a connection is established to translate the TCP options
3313 cxgb_ofld_send(cdev, reply_mbuf);
3314 m_free(m);
3315out:
3316#if 0
3317 TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS);
3318#else
3319 return;
3320#endif

--- 41 unchanged lines hidden (view full) ---

3362#endif
3363
3364 process_pass_accept_req(lso, m, &d->tdev, listen_ctx);
3365 return (0);
3366}
3367
3368/*
3369 * Called when a connection is established to translate the TCP options
2796 * reported by HW to Linux's native format.
3370 * reported by HW to FreeBSD's native format.
2797 */
2798static void
2799assign_rxopt(struct socket *so, unsigned int opt)
2800{
2801 const struct t3c_data *td = T3C_DATA(T3C_DEV(so));
2802 struct tcpcb *tp = sototcpcb(so);
2803 struct toepcb *toep = tp->t_toe;
2804
2805 INP_LOCK_ASSERT(tp->t_inpcb);
2806
2807 toep->tp_mss_clamp = td->mtus[G_TCPOPT_MSS(opt)] - 40;
2808 tp->t_flags |= G_TCPOPT_TSTAMP(opt) ? TF_RCVD_TSTMP : 0;
2809 tp->t_flags |= G_TCPOPT_SACK(opt) ? TF_SACK_PERMIT : 0;
2810 tp->t_flags |= G_TCPOPT_WSCALE_OK(opt) ? TF_RCVD_SCALE : 0;
3371 */
3372static void
3373assign_rxopt(struct socket *so, unsigned int opt)
3374{
3375 const struct t3c_data *td = T3C_DATA(T3C_DEV(so));
3376 struct tcpcb *tp = sototcpcb(so);
3377 struct toepcb *toep = tp->t_toe;
3378
3379 INP_LOCK_ASSERT(tp->t_inpcb);
3380
3381 toep->tp_mss_clamp = td->mtus[G_TCPOPT_MSS(opt)] - 40;
3382 tp->t_flags |= G_TCPOPT_TSTAMP(opt) ? TF_RCVD_TSTMP : 0;
3383 tp->t_flags |= G_TCPOPT_SACK(opt) ? TF_SACK_PERMIT : 0;
3384 tp->t_flags |= G_TCPOPT_WSCALE_OK(opt) ? TF_RCVD_SCALE : 0;
2811 if (tp->t_flags & TF_RCVD_SCALE)
2812 tp->rcv_scale = 0;
3385 if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) ==
3386 (TF_RCVD_SCALE|TF_REQ_SCALE))
3387 tp->rcv_scale = tp->request_r_scale;
2813}
2814
2815/*
2816 * Completes some final bits of initialization for just established connections
2817 * and changes their state to TCP_ESTABLISHED.
2818 *
2819 * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
2820 */

--- 5 unchanged lines hidden (view full) ---

2826
2827 toep->tp_write_seq = tp->iss = tp->snd_max = tp->snd_nxt = tp->snd_una = snd_isn;
2828 assign_rxopt(so, opt);
2829 so->so_proto->pr_ctloutput = t3_ctloutput;
2830
2831#if 0
2832 inet_sk(sk)->id = tp->write_seq ^ jiffies;
2833#endif
3388}
3389
3390/*
3391 * Completes some final bits of initialization for just established connections
3392 * and changes their state to TCP_ESTABLISHED.
3393 *
3394 * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
3395 */

--- 5 unchanged lines hidden (view full) ---

3401
3402 toep->tp_write_seq = tp->iss = tp->snd_max = tp->snd_nxt = tp->snd_una = snd_isn;
3403 assign_rxopt(so, opt);
3404 so->so_proto->pr_ctloutput = t3_ctloutput;
3405
3406#if 0
3407 inet_sk(sk)->id = tp->write_seq ^ jiffies;
3408#endif
2834
2835
2836 /*
2837 * XXX not clear what rcv_wup maps to
2838 */
2839 /*
2840 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
2841 * pass through opt0.
2842 */
2843 if (tp->rcv_wnd > (M_RCV_BUFSIZ << 10))
2844 toep->tp_rcv_wup -= tp->rcv_wnd - (M_RCV_BUFSIZ << 10);
2845
2846 dump_toepcb(toep);
2847
2848#ifdef notyet
2849/*
2850 * no clean interface for marking ARP up to date
2851 */
2852 dst_confirm(sk->sk_dst_cache);
2853#endif
3409 /*
3410 * XXX not clear what rcv_wup maps to
3411 */
3412 /*
3413 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
3414 * pass through opt0.
3415 */
3416 if (tp->rcv_wnd > (M_RCV_BUFSIZ << 10))
3417 toep->tp_rcv_wup -= tp->rcv_wnd - (M_RCV_BUFSIZ << 10);
3418
3419 dump_toepcb(toep);
3420
3421#ifdef notyet
3422/*
3423 * no clean interface for marking ARP up to date
3424 */
3425 dst_confirm(sk->sk_dst_cache);
3426#endif
3427 tp->t_starttime = ticks;
2854 tp->t_state = TCPS_ESTABLISHED;
3428 tp->t_state = TCPS_ESTABLISHED;
3429 soisconnected(so);
2855}
2856
2857static int
2858syncache_expand_establish_req(struct cpl_pass_establish *req, struct socket **so, struct toepcb *toep)
2859{
2860
2861 struct in_conninfo inc;
2862 struct tcpopt to;

--- 80 unchanged lines hidden (view full) ---

2943
2944 /*
2945 * XXX workaround for lack of syncache drop
2946 */
2947 toepcb_release(toep);
2948
2949 tp = sototcpcb(so);
2950 INP_LOCK(tp->t_inpcb);
3430}
3431
3432static int
3433syncache_expand_establish_req(struct cpl_pass_establish *req, struct socket **so, struct toepcb *toep)
3434{
3435
3436 struct in_conninfo inc;
3437 struct tcpopt to;

--- 80 unchanged lines hidden (view full) ---

3518
3519 /*
3520 * XXX workaround for lack of syncache drop
3521 */
3522 toepcb_release(toep);
3523
3524 tp = sototcpcb(so);
3525 INP_LOCK(tp->t_inpcb);
2951#ifdef notyet
2952 so->so_snd.sb_flags |= SB_TOE;
2953 so->so_rcv.sb_flags |= SB_TOE;
2954#endif
3526
3527 so->so_snd.sb_flags |= SB_NOCOALESCE;
3528 so->so_rcv.sb_flags |= SB_NOCOALESCE;
3529
2955 toep->tp_tp = tp;
2956 toep->tp_flags = 0;
2957 tp->t_toe = toep;
2958 reset_wr_list(toep);
3530 toep->tp_tp = tp;
3531 toep->tp_flags = 0;
3532 tp->t_toe = toep;
3533 reset_wr_list(toep);
2959 tp->rcv_wnd = select_rcv_wnd(so);
2960 DPRINTF("rcv_wnd=%ld\n", tp->rcv_wnd);
3534 tp->rcv_wnd = select_rcv_wnd(tdev, so);
3535 tp->rcv_nxt = toep->tp_copied_seq;
2961 install_offload_ops(so);
2962
2963 toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(tdev, max_wrs);
2964 toep->tp_wr_unacked = 0;
2965 toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data));
3536 install_offload_ops(so);
3537
3538 toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(tdev, max_wrs);
3539 toep->tp_wr_unacked = 0;
3540 toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data));
2966 toep->tp_ulp_mode = TOM_TUNABLE(tdev, ddp) && !(so->so_options & SO_NO_DDP) &&
2967 tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0;
2968 toep->tp_qset_idx = 0;
2969 toep->tp_mtu_idx = select_mss(td, tp, toep->tp_l2t->neigh->rt_ifp->if_mtu);
2970
2971 /*
2972 * XXX Cancel any keep alive timer
2973 */
2974
2975 make_established(so, ntohl(req->snd_isn), ntohs(req->tcp_opt));
2976 INP_INFO_WUNLOCK(&tcbinfo);
2977 INP_UNLOCK(tp->t_inpcb);
3541 toep->tp_qset_idx = 0;
3542 toep->tp_mtu_idx = select_mss(td, tp, toep->tp_l2t->neigh->rt_ifp->if_mtu);
3543
3544 /*
3545 * XXX Cancel any keep alive timer
3546 */
3547
3548 make_established(so, ntohl(req->snd_isn), ntohs(req->tcp_opt));
3549 INP_INFO_WUNLOCK(&tcbinfo);
3550 INP_UNLOCK(tp->t_inpcb);
2978 soisconnected(so);
2979
3551
3552 CTR1(KTR_TOM, "do_pass_establish tid=%u", toep->tp_tid);
3553 cxgb_log_tcb(cdev->adapter, toep->tp_tid);
2980#ifdef notyet
2981 /*
2982 * XXX not sure how these checks map to us
2983 */
2984 if (unlikely(sk->sk_socket)) { // simultaneous opens only
2985 sk->sk_state_change(sk);
2986 sk_wake_async(so, 0, POLL_OUT);
2987 }

--- 73 unchanged lines hidden (view full) ---

3061 /*
3062 * Now that we finally have a TID send any CPL messages that we had to
3063 * defer for lack of a TID.
3064 */
3065 if (mbufq_len(&toep->out_of_order_queue))
3066 fixup_and_send_ofo(so);
3067
3068 if (__predict_false(so->so_state & SS_NOFDREF)) {
3554#ifdef notyet
3555 /*
3556 * XXX not sure how these checks map to us
3557 */
3558 if (unlikely(sk->sk_socket)) { // simultaneous opens only
3559 sk->sk_state_change(sk);
3560 sk_wake_async(so, 0, POLL_OUT);
3561 }

--- 73 unchanged lines hidden (view full) ---

3635 /*
3636 * Now that we finally have a TID send any CPL messages that we had to
3637 * defer for lack of a TID.
3638 */
3639 if (mbufq_len(&toep->out_of_order_queue))
3640 fixup_and_send_ofo(so);
3641
3642 if (__predict_false(so->so_state & SS_NOFDREF)) {
3069#ifdef notyet
3070 /*
3071 * XXX not clear what should be done here
3072 * appears to correspond to sorwakeup_locked
3643 /*
3644 * XXX does this even make sense?
3073 */
3645 */
3074 sk->sk_state_change(sk);
3075 sk_wake_async(so, 0, POLL_OUT);
3076#endif
3646 sorwakeup(so);
3077 }
3078 m_free(m);
3079#ifdef notyet
3080/*
3081 * XXX assume no write requests permitted while socket connection is
3082 * incomplete
3083 */
3084 /*

--- 5 unchanged lines hidden (view full) ---

3090 * buffers according to the just learned write_seq, and then we send
3091 * them on their way.
3092 */
3093 fixup_pending_writeq_buffers(sk);
3094 if (t3_push_frames(so, 1))
3095 sk->sk_write_space(sk);
3096#endif
3097
3647 }
3648 m_free(m);
3649#ifdef notyet
3650/*
3651 * XXX assume no write requests permitted while socket connection is
3652 * incomplete
3653 */
3654 /*

--- 5 unchanged lines hidden (view full) ---

3660 * buffers according to the just learned write_seq, and then we send
3661 * them on their way.
3662 */
3663 fixup_pending_writeq_buffers(sk);
3664 if (t3_push_frames(so, 1))
3665 sk->sk_write_space(sk);
3666#endif
3667
3098 soisconnected(so);
3099 toep->tp_state = tp->t_state = TCPS_ESTABLISHED;
3668 toep->tp_state = tp->t_state;
3100 tcpstat.tcps_connects++;
3101
3102}
3103
3104/*
3105 * Process a CPL_ACT_ESTABLISH message.
3106 */
3107static int

--- 26 unchanged lines hidden (view full) ---

3134 toep->tp_tid = tid;
3135 toep->tp_tp = tp;
3136 so_insert_tid(d, so, tid);
3137 free_atid(cdev, atid);
3138 toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data));
3139
3140 socket_act_establish(so, m);
3141 INP_UNLOCK(tp->t_inpcb);
3669 tcpstat.tcps_connects++;
3670
3671}
3672
3673/*
3674 * Process a CPL_ACT_ESTABLISH message.
3675 */
3676static int

--- 26 unchanged lines hidden (view full) ---

3703 toep->tp_tid = tid;
3704 toep->tp_tp = tp;
3705 so_insert_tid(d, so, tid);
3706 free_atid(cdev, atid);
3707 toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data));
3708
3709 socket_act_establish(so, m);
3710 INP_UNLOCK(tp->t_inpcb);
3711 CTR1(KTR_TOM, "do_act_establish tid=%u", toep->tp_tid);
3712 cxgb_log_tcb(cdev->adapter, toep->tp_tid);
3713
3142 return (0);
3143}
3144
3145/*
3146 * Process an acknowledgment of WR completion. Advance snd_una and send the
3147 * next batch of work requests from the write queue.
3148 */
3149static void
3150wr_ack(struct toepcb *toep, struct mbuf *m)
3151{
3152 struct tcpcb *tp = toep->tp_tp;
3153 struct cpl_wr_ack *hdr = cplhdr(m);
3154 struct socket *so = toeptoso(toep);
3155 unsigned int credits = ntohs(hdr->credits);
3156 u32 snd_una = ntohl(hdr->snd_una);
3157 int bytes = 0;
3158
3714 return (0);
3715}
3716
3717/*
3718 * Process an acknowledgment of WR completion. Advance snd_una and send the
3719 * next batch of work requests from the write queue.
3720 */
3721static void
3722wr_ack(struct toepcb *toep, struct mbuf *m)
3723{
3724 struct tcpcb *tp = toep->tp_tp;
3725 struct cpl_wr_ack *hdr = cplhdr(m);
3726 struct socket *so = toeptoso(toep);
3727 unsigned int credits = ntohs(hdr->credits);
3728 u32 snd_una = ntohl(hdr->snd_una);
3729 int bytes = 0;
3730
3159 DPRINTF("wr_ack: snd_una=%u credits=%d\n", snd_una, credits);
3731 CTR2(KTR_SPARE2, "wr_ack: snd_una=%u credits=%d", snd_una, credits);
3160
3161 INP_LOCK(tp->t_inpcb);
3162
3163 toep->tp_wr_avail += credits;
3164 if (toep->tp_wr_unacked > toep->tp_wr_max - toep->tp_wr_avail)
3165 toep->tp_wr_unacked = toep->tp_wr_max - toep->tp_wr_avail;
3166
3167 while (credits) {
3168 struct mbuf *p = peek_wr(toep);
3732
3733 INP_LOCK(tp->t_inpcb);
3734
3735 toep->tp_wr_avail += credits;
3736 if (toep->tp_wr_unacked > toep->tp_wr_max - toep->tp_wr_avail)
3737 toep->tp_wr_unacked = toep->tp_wr_max - toep->tp_wr_avail;
3738
3739 while (credits) {
3740 struct mbuf *p = peek_wr(toep);
3169 DPRINTF("p->credits=%d p->bytes=%d\n", p->m_pkthdr.csum_data, p->m_pkthdr.len) ;
3170
3171 if (__predict_false(!p)) {
3172 log(LOG_ERR, "%u WR_ACK credits for TID %u with "
3741
3742 if (__predict_false(!p)) {
3743 log(LOG_ERR, "%u WR_ACK credits for TID %u with "
3173 "nothing pending, state %u\n",
3174 credits, toep->tp_tid, tp->t_state);
3744 "nothing pending, state %u wr_avail=%u\n",
3745 credits, toep->tp_tid, tp->t_state, toep->tp_wr_avail);
3175 break;
3176 }
3746 break;
3747 }
3748 CTR2(KTR_TOM,
3749 "wr_ack: p->credits=%d p->bytes=%d", p->m_pkthdr.csum_data, p->m_pkthdr.len);
3750
3751 KASSERT(p->m_pkthdr.csum_data != 0, ("empty request still on list"));
3177 if (__predict_false(credits < p->m_pkthdr.csum_data)) {
3752 if (__predict_false(credits < p->m_pkthdr.csum_data)) {
3753
3178#if DEBUG_WR > 1
3179 struct tx_data_wr *w = cplhdr(p);
3754#if DEBUG_WR > 1
3755 struct tx_data_wr *w = cplhdr(p);
3180#ifdef notyet
3181 log(LOG_ERR,
3182 "TID %u got %u WR credits, need %u, len %u, "
3183 "main body %u, frags %u, seq # %u, ACK una %u,"
3184 " ACK nxt %u, WR_AVAIL %u, WRs pending %u\n",
3185 toep->tp_tid, credits, p->csum, p->len,
3186 p->len - p->data_len, skb_shinfo(p)->nr_frags,
3187 ntohl(w->sndseq), snd_una, ntohl(hdr->snd_nxt),
3756 log(LOG_ERR,
3757 "TID %u got %u WR credits, need %u, len %u, "
3758 "main body %u, frags %u, seq # %u, ACK una %u,"
3759 " ACK nxt %u, WR_AVAIL %u, WRs pending %u\n",
3760 toep->tp_tid, credits, p->csum, p->len,
3761 p->len - p->data_len, skb_shinfo(p)->nr_frags,
3762 ntohl(w->sndseq), snd_una, ntohl(hdr->snd_nxt),
3188 WR_AVAIL(tp), count_pending_wrs(tp) - credits);
3189#endif
3763 toep->tp_wr_avail, count_pending_wrs(tp) - credits);
3190#endif
3191 p->m_pkthdr.csum_data -= credits;
3192 break;
3193 } else {
3194 dequeue_wr(toep);
3195 credits -= p->m_pkthdr.csum_data;
3196 bytes += p->m_pkthdr.len;
3764#endif
3765 p->m_pkthdr.csum_data -= credits;
3766 break;
3767 } else {
3768 dequeue_wr(toep);
3769 credits -= p->m_pkthdr.csum_data;
3770 bytes += p->m_pkthdr.len;
3197 DPRINTF("done with wr of %d bytes\n", p->m_pkthdr.len);
3771 CTR3(KTR_TOM,
3772 "wr_ack: done with wr of %d bytes remain credits=%d wr credits=%d",
3773 p->m_pkthdr.len, credits, p->m_pkthdr.csum_data);
3198
3199 m_free(p);
3200 }
3201 }
3202
3203#if DEBUG_WR
3204 check_wr_invariants(tp);
3205#endif

--- 17 unchanged lines hidden (view full) ---

3223 * Keep ARP entry "minty fresh"
3224 */
3225 dst_confirm(sk->sk_dst_cache);
3226#endif
3227 if (tp->snd_una == tp->snd_nxt)
3228 toep->tp_flags &= ~TP_TX_WAIT_IDLE;
3229 }
3230 if (bytes) {
3774
3775 m_free(p);
3776 }
3777 }
3778
3779#if DEBUG_WR
3780 check_wr_invariants(tp);
3781#endif

--- 17 unchanged lines hidden (view full) ---

3799 * Keep ARP entry "minty fresh"
3800 */
3801 dst_confirm(sk->sk_dst_cache);
3802#endif
3803 if (tp->snd_una == tp->snd_nxt)
3804 toep->tp_flags &= ~TP_TX_WAIT_IDLE;
3805 }
3806 if (bytes) {
3231 DPRINTF("sbdrop(%d)\n", bytes);
3807 CTR1(KTR_SPARE2, "wr_ack: sbdrop(%d)", bytes);
3232 SOCKBUF_LOCK(&so->so_snd);
3233 sbdrop_locked(&so->so_snd, bytes);
3234 sowwakeup_locked(so);
3235 }
3236
3237 if (so->so_snd.sb_sndptroff < so->so_snd.sb_cc)
3238 t3_push_frames(so, 0);
3239

--- 5 unchanged lines hidden (view full) ---

3245/*
3246 * Handler for TX_DATA_ACK CPL messages.
3247 */
3248static int
3249do_wr_ack(struct t3cdev *dev, struct mbuf *m, void *ctx)
3250{
3251 struct toepcb *toep = (struct toepcb *)ctx;
3252
3808 SOCKBUF_LOCK(&so->so_snd);
3809 sbdrop_locked(&so->so_snd, bytes);
3810 sowwakeup_locked(so);
3811 }
3812
3813 if (so->so_snd.sb_sndptroff < so->so_snd.sb_cc)
3814 t3_push_frames(so, 0);
3815

--- 5 unchanged lines hidden (view full) ---

3821/*
3822 * Handler for TX_DATA_ACK CPL messages.
3823 */
3824static int
3825do_wr_ack(struct t3cdev *dev, struct mbuf *m, void *ctx)
3826{
3827 struct toepcb *toep = (struct toepcb *)ctx;
3828
3253 DPRINTF("do_wr_ack\n");
3254 dump_toepcb(toep);
3255
3256 VALIDATE_SOCK(so);
3257
3258 wr_ack(toep, m);
3259 return 0;
3260}
3261
3829 VALIDATE_SOCK(so);
3830
3831 wr_ack(toep, m);
3832 return 0;
3833}
3834
3835/*
3836 * Handler for TRACE_PKT CPL messages. Just sink these packets.
3837 */
3838static int
3839do_trace_pkt(struct t3cdev *dev, struct mbuf *m, void *ctx)
3840{
3841 m_freem(m);
3842 return 0;
3843}
3262
3263/*
3264 * Reset a connection that is on a listener's SYN queue or accept queue,
3265 * i.e., one that has not had a struct socket associated with it.
3266 * Must be called from process context.
3267 *
3268 * Modeled after code in inet_csk_listen_stop().
3269 */

--- 45 unchanged lines hidden (view full) ---

3315 toep->tp_tp = NULL;
3316 t3_send_reset(toep);
3317 cxgb_remove_tid(TOEP_T3C_DEV(toep), toep, toep->tp_tid);
3318 toepcb_release(toep);
3319 }
3320 SOCK_UNLOCK(lctx->lso);
3321}
3322
3844
3845/*
3846 * Reset a connection that is on a listener's SYN queue or accept queue,
3847 * i.e., one that has not had a struct socket associated with it.
3848 * Must be called from process context.
3849 *
3850 * Modeled after code in inet_csk_listen_stop().
3851 */

--- 45 unchanged lines hidden (view full) ---

3897 toep->tp_tp = NULL;
3898 t3_send_reset(toep);
3899 cxgb_remove_tid(TOEP_T3C_DEV(toep), toep, toep->tp_tid);
3900 toepcb_release(toep);
3901 }
3902 SOCK_UNLOCK(lctx->lso);
3903}
3904
3905
3906int
3907t3_setup_ppods(struct socket *so, const struct ddp_gather_list *gl,
3908 unsigned int nppods, unsigned int tag, unsigned int maxoff,
3909 unsigned int pg_off, unsigned int color)
3910{
3911 unsigned int i, j, pidx;
3912 struct pagepod *p;
3913 struct mbuf *m;
3914 struct ulp_mem_io *req;
3915 struct tcpcb *tp = sototcpcb(so);
3916 struct toepcb *toep = tp->t_toe;
3917 unsigned int tid = toep->tp_tid;
3918 const struct tom_data *td = TOM_DATA(TOE_DEV(so));
3919 unsigned int ppod_addr = tag * PPOD_SIZE + td->ddp_llimit;
3920
3921 CTR6(KTR_TOM, "t3_setup_ppods(gl=%p nppods=%u tag=%u maxoff=%u pg_off=%u color=%u)",
3922 gl, nppods, tag, maxoff, pg_off, color);
3923
3924 for (i = 0; i < nppods; ++i) {
3925 m = m_gethdr_nofail(sizeof(*req) + PPOD_SIZE);
3926 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
3927 req = mtod(m, struct ulp_mem_io *);
3928 m->m_pkthdr.len = m->m_len = sizeof(*req) + PPOD_SIZE;
3929 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
3930 req->wr.wr_lo = 0;
3931 req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(ppod_addr >> 5) |
3932 V_ULPTX_CMD(ULP_MEM_WRITE));
3933 req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE / 32) |
3934 V_ULPTX_NFLITS(PPOD_SIZE / 8 + 1));
3935
3936 p = (struct pagepod *)(req + 1);
3937 if (__predict_false(i < nppods - NUM_SENTINEL_PPODS)) {
3938 p->pp_vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid));
3939 p->pp_pgsz_tag_color = htonl(V_PPOD_TAG(tag) |
3940 V_PPOD_COLOR(color));
3941 p->pp_max_offset = htonl(maxoff);
3942 p->pp_page_offset = htonl(pg_off);
3943 p->pp_rsvd = 0;
3944 for (pidx = 4 * i, j = 0; j < 5; ++j, ++pidx)
3945 p->pp_addr[j] = pidx < gl->dgl_nelem ?
3946 htobe64(VM_PAGE_TO_PHYS(gl->dgl_pages[pidx])) : 0;
3947 } else
3948 p->pp_vld_tid = 0; /* mark sentinel page pods invalid */
3949 send_or_defer(toep, m, 0);
3950 ppod_addr += PPOD_SIZE;
3951 }
3952 return (0);
3953}
3954
3955/*
3956 * Build a CPL_BARRIER message as payload of a ULP_TX_PKT command.
3957 */
3958static inline void
3959mk_cpl_barrier_ulp(struct cpl_barrier *b)
3960{
3961 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)b;
3962
3963 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3964 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*b) / 8));
3965 b->opcode = CPL_BARRIER;
3966}
3967
3968/*
3969 * Build a CPL_GET_TCB message as payload of a ULP_TX_PKT command.
3970 */
3971static inline void
3972mk_get_tcb_ulp(struct cpl_get_tcb *req, unsigned int tid, unsigned int cpuno)
3973{
3974 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3975
3976 txpkt = (struct ulp_txpkt *)req;
3977 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3978 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3979 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, tid));
3980 req->cpuno = htons(cpuno);
3981}
3982
3983/*
3984 * Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command.
3985 */
3986static inline void
3987mk_set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3988 unsigned int word, uint64_t mask, uint64_t val)
3989{
3990 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3991
3992 CTR4(KTR_TCB, "mk_set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx",
3993 tid, word, mask, val);
3994
3995 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3996 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3997 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3998 req->reply = V_NO_REPLY(1);
3999 req->cpu_idx = 0;
4000 req->word = htons(word);
4001 req->mask = htobe64(mask);
4002 req->val = htobe64(val);
4003}
4004
4005/*
4006 * Build a CPL_RX_DATA_ACK message as payload of a ULP_TX_PKT command.
4007 */
4008static void
4009mk_rx_data_ack_ulp(struct cpl_rx_data_ack *ack, unsigned int tid, unsigned int credits)
4010{
4011 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)ack;
4012
4013 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
4014 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*ack) / 8));
4015 OPCODE_TID(ack) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid));
4016 ack->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE |
4017 V_RX_DACK_MODE(1) | V_RX_CREDITS(credits));
4018}
4019
3323void
4020void
4021t3_cancel_ddpbuf(struct toepcb *toep, unsigned int bufidx)
4022{
4023 unsigned int wrlen;
4024 struct mbuf *m;
4025 struct work_request_hdr *wr;
4026 struct cpl_barrier *lock;
4027 struct cpl_set_tcb_field *req;
4028 struct cpl_get_tcb *getreq;
4029 struct ddp_state *p = &toep->tp_ddp_state;
4030
4031 SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv);
4032 wrlen = sizeof(*wr) + sizeof(*req) + 2 * sizeof(*lock) +
4033 sizeof(*getreq);
4034 m = m_gethdr_nofail(wrlen);
4035 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
4036 wr = mtod(m, struct work_request_hdr *);
4037 bzero(wr, wrlen);
4038
4039 wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
4040 m->m_pkthdr.len = m->m_len = wrlen;
4041
4042 lock = (struct cpl_barrier *)(wr + 1);
4043 mk_cpl_barrier_ulp(lock);
4044
4045 req = (struct cpl_set_tcb_field *)(lock + 1);
4046
4047 CTR1(KTR_TCB, "t3_cancel_ddpbuf(bufidx=%u)", bufidx);
4048
4049 /* Hmmm, not sure if this actually a good thing: reactivating
4050 * the other buffer might be an issue if it has been completed
4051 * already. However, that is unlikely, since the fact that the UBUF
4052 * is not completed indicates that there is no oustanding data.
4053 */
4054 if (bufidx == 0)
4055 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS,
4056 V_TF_DDP_ACTIVE_BUF(1) |
4057 V_TF_DDP_BUF0_VALID(1),
4058 V_TF_DDP_ACTIVE_BUF(1));
4059 else
4060 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS,
4061 V_TF_DDP_ACTIVE_BUF(1) |
4062 V_TF_DDP_BUF1_VALID(1), 0);
4063
4064 getreq = (struct cpl_get_tcb *)(req + 1);
4065 mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset);
4066
4067 mk_cpl_barrier_ulp((struct cpl_barrier *)(getreq + 1));
4068
4069 /* Keep track of the number of oustanding CPL_GET_TCB requests
4070 */
4071 p->get_tcb_count++;
4072
4073#ifdef T3_TRACE
4074 T3_TRACE1(TIDTB(so),
4075 "t3_cancel_ddpbuf: bufidx %u", bufidx);
4076#endif
4077 cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
4078}
4079
4080/**
4081 * t3_overlay_ddpbuf - overlay an existing DDP buffer with a new one
4082 * @sk: the socket associated with the buffers
4083 * @bufidx: index of HW DDP buffer (0 or 1)
4084 * @tag0: new tag for HW buffer 0
4085 * @tag1: new tag for HW buffer 1
4086 * @len: new length for HW buf @bufidx
4087 *
4088 * Sends a compound WR to overlay a new DDP buffer on top of an existing
4089 * buffer by changing the buffer tag and length and setting the valid and
4090 * active flag accordingly. The caller must ensure the new buffer is at
4091 * least as big as the existing one. Since we typically reprogram both HW
4092 * buffers this function sets both tags for convenience. Read the TCB to
4093 * determine how made data was written into the buffer before the overlay
4094 * took place.
4095 */
4096void
4097t3_overlay_ddpbuf(struct toepcb *toep, unsigned int bufidx, unsigned int tag0,
4098 unsigned int tag1, unsigned int len)
4099{
4100 unsigned int wrlen;
4101 struct mbuf *m;
4102 struct work_request_hdr *wr;
4103 struct cpl_get_tcb *getreq;
4104 struct cpl_set_tcb_field *req;
4105 struct ddp_state *p = &toep->tp_ddp_state;
4106
4107 CTR4(KTR_TCB, "t3_setup_ppods(bufidx=%u tag0=%u tag1=%u len=%u)",
4108 bufidx, tag0, tag1, len);
4109 SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv);
4110 wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq);
4111 m = m_gethdr_nofail(wrlen);
4112 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
4113 wr = mtod(m, struct work_request_hdr *);
4114 m->m_pkthdr.len = m->m_len = wrlen;
4115 bzero(wr, wrlen);
4116
4117
4118 /* Set the ATOMIC flag to make sure that TP processes the following
4119 * CPLs in an atomic manner and no wire segments can be interleaved.
4120 */
4121 wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
4122 req = (struct cpl_set_tcb_field *)(wr + 1);
4123 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_TAG,
4124 V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG) |
4125 V_TCB_RX_DDP_BUF1_TAG(M_TCB_RX_DDP_BUF1_TAG) << 32,
4126 V_TCB_RX_DDP_BUF0_TAG(tag0) |
4127 V_TCB_RX_DDP_BUF1_TAG((uint64_t)tag1) << 32);
4128 req++;
4129 if (bufidx == 0) {
4130 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_LEN,
4131 V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN),
4132 V_TCB_RX_DDP_BUF0_LEN((uint64_t)len));
4133 req++;
4134 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS,
4135 V_TF_DDP_PUSH_DISABLE_0(1) |
4136 V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(1),
4137 V_TF_DDP_PUSH_DISABLE_0(0) |
4138 V_TF_DDP_BUF0_VALID(1));
4139 } else {
4140 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_LEN,
4141 V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN),
4142 V_TCB_RX_DDP_BUF1_LEN((uint64_t)len));
4143 req++;
4144 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS,
4145 V_TF_DDP_PUSH_DISABLE_1(1) |
4146 V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1),
4147 V_TF_DDP_PUSH_DISABLE_1(0) |
4148 V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1));
4149 }
4150
4151 getreq = (struct cpl_get_tcb *)(req + 1);
4152 mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset);
4153
4154 /* Keep track of the number of oustanding CPL_GET_TCB requests
4155 */
4156 p->get_tcb_count++;
4157
4158#ifdef T3_TRACE
4159 T3_TRACE4(TIDTB(sk),
4160 "t3_overlay_ddpbuf: bufidx %u tag0 %u tag1 %u "
4161 "len %d",
4162 bufidx, tag0, tag1, len);
4163#endif
4164 cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
4165}
4166
4167/*
4168 * Sends a compound WR containing all the CPL messages needed to program the
4169 * two HW DDP buffers, namely optionally setting up the length and offset of
4170 * each buffer, programming the DDP flags, and optionally sending RX_DATA_ACK.
4171 */
4172void
4173t3_setup_ddpbufs(struct toepcb *toep, unsigned int len0, unsigned int offset0,
4174 unsigned int len1, unsigned int offset1,
4175 uint64_t ddp_flags, uint64_t flag_mask, int modulate)
4176{
4177 unsigned int wrlen;
4178 struct mbuf *m;
4179 struct work_request_hdr *wr;
4180 struct cpl_set_tcb_field *req;
4181
4182 CTR6(KTR_TCB, "t3_setup_ddpbufs(len0=%u offset0=%u len1=%u offset1=%u ddp_flags=0x%08x%08x ",
4183 len0, offset0, len1, offset1, ddp_flags >> 32, ddp_flags & 0xffffffff);
4184
4185 SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv);
4186 wrlen = sizeof(*wr) + sizeof(*req) + (len0 ? sizeof(*req) : 0) +
4187 (len1 ? sizeof(*req) : 0) +
4188 (modulate ? sizeof(struct cpl_rx_data_ack) : 0);
4189 m = m_gethdr_nofail(wrlen);
4190 m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
4191 wr = mtod(m, struct work_request_hdr *);
4192 bzero(wr, wrlen);
4193
4194 wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
4195 m->m_pkthdr.len = m->m_len = wrlen;
4196
4197 req = (struct cpl_set_tcb_field *)(wr + 1);
4198 if (len0) { /* program buffer 0 offset and length */
4199 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_OFFSET,
4200 V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) |
4201 V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN),
4202 V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)offset0) |
4203 V_TCB_RX_DDP_BUF0_LEN((uint64_t)len0));
4204 req++;
4205 }
4206 if (len1) { /* program buffer 1 offset and length */
4207 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_OFFSET,
4208 V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) |
4209 V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN) << 32,
4210 V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset1) |
4211 V_TCB_RX_DDP_BUF1_LEN((uint64_t)len1) << 32);
4212 req++;
4213 }
4214
4215 mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, flag_mask,
4216 ddp_flags);
4217
4218 if (modulate) {
4219 mk_rx_data_ack_ulp((struct cpl_rx_data_ack *)(req + 1), toep->tp_tid,
4220 toep->tp_copied_seq - toep->tp_rcv_wup);
4221 toep->tp_rcv_wup = toep->tp_copied_seq;
4222 }
4223
4224#ifdef T3_TRACE
4225 T3_TRACE5(TIDTB(sk),
4226 "t3_setup_ddpbufs: len0 %u len1 %u ddp_flags 0x%08x%08x "
4227 "modulate %d",
4228 len0, len1, ddp_flags >> 32, ddp_flags & 0xffffffff,
4229 modulate);
4230#endif
4231
4232 cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
4233}
4234
4235void
3324t3_init_wr_tab(unsigned int wr_len)
3325{
3326 int i;
3327
3328 if (mbuf_wrs[1]) /* already initialized */
3329 return;
3330
3331 for (i = 1; i < ARRAY_SIZE(mbuf_wrs); i++) {

--- 16 unchanged lines hidden (view full) ---

3348 log(LOG_ERR,
3349 "Chelsio TCP offload: can't allocate sk_buff\n");
3350 return -1;
3351 }
3352 skb_put(tcphdr_skb, sizeof(struct tcphdr));
3353 tcphdr_skb->h.raw = tcphdr_skb->data;
3354 memset(tcphdr_skb->data, 0, tcphdr_skb->len);
3355#endif
4236t3_init_wr_tab(unsigned int wr_len)
4237{
4238 int i;
4239
4240 if (mbuf_wrs[1]) /* already initialized */
4241 return;
4242
4243 for (i = 1; i < ARRAY_SIZE(mbuf_wrs); i++) {

--- 16 unchanged lines hidden (view full) ---

4260 log(LOG_ERR,
4261 "Chelsio TCP offload: can't allocate sk_buff\n");
4262 return -1;
4263 }
4264 skb_put(tcphdr_skb, sizeof(struct tcphdr));
4265 tcphdr_skb->h.raw = tcphdr_skb->data;
4266 memset(tcphdr_skb->data, 0, tcphdr_skb->len);
4267#endif
3356
3357
3358 t3tom_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
3359 t3tom_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl);
3360 t3tom_register_cpl_handler(CPL_TX_DMA_ACK, do_wr_ack);
3361 t3tom_register_cpl_handler(CPL_RX_DATA, do_rx_data);
3362 t3tom_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl);
3363 t3tom_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close);
3364 t3tom_register_cpl_handler(CPL_PASS_ESTABLISH, do_pass_establish);
3365 t3tom_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_pass_accept_req);
3366 t3tom_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req);
3367 t3tom_register_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl);
3368 t3tom_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp);
3369 t3tom_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete);
4268
4269 t3tom_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
4270 t3tom_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl);
4271 t3tom_register_cpl_handler(CPL_TX_DMA_ACK, do_wr_ack);
4272 t3tom_register_cpl_handler(CPL_RX_DATA, do_rx_data);
4273 t3tom_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl);
4274 t3tom_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close);
4275 t3tom_register_cpl_handler(CPL_PASS_ESTABLISH, do_pass_establish);
4276 t3tom_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_pass_accept_req);
4277 t3tom_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req);
4278 t3tom_register_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl);
4279 t3tom_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp);
4280 t3tom_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete);
3370#ifdef notyet
3371 t3tom_register_cpl_handler(CPL_RX_URG_NOTIFY, do_rx_urg_notify);
3372 t3tom_register_cpl_handler(CPL_TRACE_PKT, do_trace_pkt);
3373 t3tom_register_cpl_handler(CPL_GET_TCB_RPL, do_get_tcb_rpl);
4281 t3tom_register_cpl_handler(CPL_RX_URG_NOTIFY, do_rx_urg_notify);
4282 t3tom_register_cpl_handler(CPL_TRACE_PKT, do_trace_pkt);
4283 t3tom_register_cpl_handler(CPL_GET_TCB_RPL, do_get_tcb_rpl);
3374#endif
3375 return (0);
3376}
3377
4284 return (0);
4285}
4286