1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29#include <sys/cdefs.h>
| 1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29#include <sys/cdefs.h>
|
30__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c 183292 2008-09-23 03:16:54Z kmacy $");
| 30__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c 193272 2009-06-01 21:17:03Z jhb $");
|
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/bus.h>
36#include <sys/module.h>
37#include <sys/pciio.h>
38#include <sys/conf.h>
39#include <machine/bus.h>
40#include <machine/resource.h>
41#include <sys/bus_dma.h>
42#include <sys/rman.h>
43#include <sys/ioccom.h>
44#include <sys/mbuf.h>
45#include <sys/rwlock.h>
46#include <sys/linker.h>
47#include <sys/firmware.h>
48#include <sys/socket.h>
49#include <sys/socketvar.h>
50#include <sys/sockio.h>
51#include <sys/smp.h>
52#include <sys/sysctl.h>
53#include <sys/syslog.h>
54#include <sys/queue.h>
55#include <sys/taskqueue.h>
56#include <sys/proc.h>
57#include <sys/uio.h>
58
59#include <net/route.h>
60#include <netinet/in_systm.h>
61#include <netinet/in.h>
62#include <netinet/in_pcb.h>
63#include <netinet/ip.h>
64#include <netinet/ip_var.h>
65#include <netinet/tcp_var.h>
66#include <netinet/tcp.h>
67#include <netinet/tcpip.h>
68
69#include <contrib/rdma/ib_verbs.h>
70
71#include <cxgb_include.h>
72#include <ulp/tom/cxgb_tom.h>
73#include <ulp/tom/cxgb_t3_ddp.h>
74#include <ulp/tom/cxgb_defs.h>
75#include <ulp/tom/cxgb_toepcb.h>
76#include <ulp/iw_cxgb/iw_cxgb_wr.h>
77#include <ulp/iw_cxgb/iw_cxgb_hal.h>
78#include <ulp/iw_cxgb/iw_cxgb_provider.h>
79#include <ulp/iw_cxgb/iw_cxgb_cm.h>
80#include <ulp/iw_cxgb/iw_cxgb.h>
81
82#ifdef KTR
83static char *states[] = {
84 "idle",
85 "listen",
86 "connecting",
87 "mpa_wait_req",
88 "mpa_req_sent",
89 "mpa_req_rcvd",
90 "mpa_rep_sent",
91 "fpdu_mode",
92 "aborting",
93 "closing",
94 "moribund",
95 "dead",
96 NULL,
97};
98#endif
99
100SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "iw_cxgb driver parameters");
101
102static int ep_timeout_secs = 10;
103TUNABLE_INT("hw.iw_cxgb.ep_timeout_secs", &ep_timeout_secs);
104SYSCTL_UINT(_hw_cxgb, OID_AUTO, ep_timeout_secs, CTLFLAG_RDTUN, &ep_timeout_secs, 0,
105 "CM Endpoint operation timeout in seconds (default=10)");
106
107static int mpa_rev = 1;
108TUNABLE_INT("hw.iw_cxgb.mpa_rev", &mpa_rev);
109SYSCTL_UINT(_hw_cxgb, OID_AUTO, mpa_rev, CTLFLAG_RDTUN, &mpa_rev, 0,
110 "MPA Revision, 0 supports amso1100, 1 is spec compliant. (default=1)");
111
112static int markers_enabled = 0;
113TUNABLE_INT("hw.iw_cxgb.markers_enabled", &markers_enabled);
114SYSCTL_UINT(_hw_cxgb, OID_AUTO, markers_enabled, CTLFLAG_RDTUN, &markers_enabled, 0,
115 "Enable MPA MARKERS (default(0)=disabled)");
116
117static int crc_enabled = 1;
118TUNABLE_INT("hw.iw_cxgb.crc_enabled", &crc_enabled);
119SYSCTL_UINT(_hw_cxgb, OID_AUTO, crc_enabled, CTLFLAG_RDTUN, &crc_enabled, 0,
120 "Enable MPA CRC (default(1)=enabled)");
121
122static int rcv_win = 256 * 1024;
123TUNABLE_INT("hw.iw_cxgb.rcv_win", &rcv_win);
124SYSCTL_UINT(_hw_cxgb, OID_AUTO, rcv_win, CTLFLAG_RDTUN, &rcv_win, 0,
125 "TCP receive window in bytes (default=256KB)");
126
127static int snd_win = 32 * 1024;
128TUNABLE_INT("hw.iw_cxgb.snd_win", &snd_win);
129SYSCTL_UINT(_hw_cxgb, OID_AUTO, snd_win, CTLFLAG_RDTUN, &snd_win, 0,
130 "TCP send window in bytes (default=32KB)");
131
132static unsigned int nocong = 0;
133TUNABLE_INT("hw.iw_cxgb.nocong", &nocong);
134SYSCTL_UINT(_hw_cxgb, OID_AUTO, nocong, CTLFLAG_RDTUN, &nocong, 0,
135 "Turn off congestion control (default=0)");
136
137static unsigned int cong_flavor = 1;
138TUNABLE_INT("hw.iw_cxgb.cong_flavor", &cong_flavor);
139SYSCTL_UINT(_hw_cxgb, OID_AUTO, cong_flavor, CTLFLAG_RDTUN, &cong_flavor, 0,
140 "TCP Congestion control flavor (default=1)");
141
142static void ep_timeout(void *arg);
143static void connect_reply_upcall(struct iwch_ep *ep, int status);
| 31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/bus.h>
36#include <sys/module.h>
37#include <sys/pciio.h>
38#include <sys/conf.h>
39#include <machine/bus.h>
40#include <machine/resource.h>
41#include <sys/bus_dma.h>
42#include <sys/rman.h>
43#include <sys/ioccom.h>
44#include <sys/mbuf.h>
45#include <sys/rwlock.h>
46#include <sys/linker.h>
47#include <sys/firmware.h>
48#include <sys/socket.h>
49#include <sys/socketvar.h>
50#include <sys/sockio.h>
51#include <sys/smp.h>
52#include <sys/sysctl.h>
53#include <sys/syslog.h>
54#include <sys/queue.h>
55#include <sys/taskqueue.h>
56#include <sys/proc.h>
57#include <sys/uio.h>
58
59#include <net/route.h>
60#include <netinet/in_systm.h>
61#include <netinet/in.h>
62#include <netinet/in_pcb.h>
63#include <netinet/ip.h>
64#include <netinet/ip_var.h>
65#include <netinet/tcp_var.h>
66#include <netinet/tcp.h>
67#include <netinet/tcpip.h>
68
69#include <contrib/rdma/ib_verbs.h>
70
71#include <cxgb_include.h>
72#include <ulp/tom/cxgb_tom.h>
73#include <ulp/tom/cxgb_t3_ddp.h>
74#include <ulp/tom/cxgb_defs.h>
75#include <ulp/tom/cxgb_toepcb.h>
76#include <ulp/iw_cxgb/iw_cxgb_wr.h>
77#include <ulp/iw_cxgb/iw_cxgb_hal.h>
78#include <ulp/iw_cxgb/iw_cxgb_provider.h>
79#include <ulp/iw_cxgb/iw_cxgb_cm.h>
80#include <ulp/iw_cxgb/iw_cxgb.h>
81
82#ifdef KTR
83static char *states[] = {
84 "idle",
85 "listen",
86 "connecting",
87 "mpa_wait_req",
88 "mpa_req_sent",
89 "mpa_req_rcvd",
90 "mpa_rep_sent",
91 "fpdu_mode",
92 "aborting",
93 "closing",
94 "moribund",
95 "dead",
96 NULL,
97};
98#endif
99
100SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "iw_cxgb driver parameters");
101
102static int ep_timeout_secs = 10;
103TUNABLE_INT("hw.iw_cxgb.ep_timeout_secs", &ep_timeout_secs);
104SYSCTL_UINT(_hw_cxgb, OID_AUTO, ep_timeout_secs, CTLFLAG_RDTUN, &ep_timeout_secs, 0,
105 "CM Endpoint operation timeout in seconds (default=10)");
106
107static int mpa_rev = 1;
108TUNABLE_INT("hw.iw_cxgb.mpa_rev", &mpa_rev);
109SYSCTL_UINT(_hw_cxgb, OID_AUTO, mpa_rev, CTLFLAG_RDTUN, &mpa_rev, 0,
110 "MPA Revision, 0 supports amso1100, 1 is spec compliant. (default=1)");
111
112static int markers_enabled = 0;
113TUNABLE_INT("hw.iw_cxgb.markers_enabled", &markers_enabled);
114SYSCTL_UINT(_hw_cxgb, OID_AUTO, markers_enabled, CTLFLAG_RDTUN, &markers_enabled, 0,
115 "Enable MPA MARKERS (default(0)=disabled)");
116
117static int crc_enabled = 1;
118TUNABLE_INT("hw.iw_cxgb.crc_enabled", &crc_enabled);
119SYSCTL_UINT(_hw_cxgb, OID_AUTO, crc_enabled, CTLFLAG_RDTUN, &crc_enabled, 0,
120 "Enable MPA CRC (default(1)=enabled)");
121
122static int rcv_win = 256 * 1024;
123TUNABLE_INT("hw.iw_cxgb.rcv_win", &rcv_win);
124SYSCTL_UINT(_hw_cxgb, OID_AUTO, rcv_win, CTLFLAG_RDTUN, &rcv_win, 0,
125 "TCP receive window in bytes (default=256KB)");
126
127static int snd_win = 32 * 1024;
128TUNABLE_INT("hw.iw_cxgb.snd_win", &snd_win);
129SYSCTL_UINT(_hw_cxgb, OID_AUTO, snd_win, CTLFLAG_RDTUN, &snd_win, 0,
130 "TCP send window in bytes (default=32KB)");
131
132static unsigned int nocong = 0;
133TUNABLE_INT("hw.iw_cxgb.nocong", &nocong);
134SYSCTL_UINT(_hw_cxgb, OID_AUTO, nocong, CTLFLAG_RDTUN, &nocong, 0,
135 "Turn off congestion control (default=0)");
136
137static unsigned int cong_flavor = 1;
138TUNABLE_INT("hw.iw_cxgb.cong_flavor", &cong_flavor);
139SYSCTL_UINT(_hw_cxgb, OID_AUTO, cong_flavor, CTLFLAG_RDTUN, &cong_flavor, 0,
140 "TCP Congestion control flavor (default=1)");
141
142static void ep_timeout(void *arg);
143static void connect_reply_upcall(struct iwch_ep *ep, int status);
|
144static void iwch_so_upcall(struct socket *so, void *arg, int waitflag);
| 144static int iwch_so_upcall(struct socket *so, void *arg, int waitflag);
|
145
146/*
147 * Cruft to offload socket upcalls onto thread.
148 */
149static struct mtx req_lock;
150static TAILQ_HEAD(iwch_ep_list, iwch_ep_common) req_list;
151static struct task iw_cxgb_task;
152static struct taskqueue *iw_cxgb_taskq;
153static void process_req(void *ctx, int pending);
154
155static void
156start_ep_timer(struct iwch_ep *ep)
157{
158 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
159 if (callout_pending(&ep->timer)) {
160 CTR2(KTR_IW_CXGB, "%s stopped / restarted timer ep %p", __FUNCTION__, ep);
161 callout_deactivate(&ep->timer);
162 callout_drain(&ep->timer);
163 } else {
164 /*
165 * XXX this looks racy
166 */
167 get_ep(&ep->com);
168 callout_init(&ep->timer, TRUE);
169 }
170 callout_reset(&ep->timer, ep_timeout_secs * hz, ep_timeout, ep);
171}
172
173static void
174stop_ep_timer(struct iwch_ep *ep)
175{
176 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
177 callout_drain(&ep->timer);
178 put_ep(&ep->com);
179}
180
181static int set_tcpinfo(struct iwch_ep *ep)
182{
183 struct tcp_info ti;
184 struct sockopt sopt;
185 int err;
186
187 sopt.sopt_dir = SOPT_GET;
188 sopt.sopt_level = IPPROTO_TCP;
189 sopt.sopt_name = TCP_INFO;
190 sopt.sopt_val = (caddr_t)&ti;
191 sopt.sopt_valsize = sizeof ti;
192 sopt.sopt_td = NULL;
193
194 err = sogetopt(ep->com.so, &sopt);
195 if (err) {
196 printf("%s can't get tcpinfo\n", __FUNCTION__);
197 return -err;
198 }
199 if (!(ti.tcpi_options & TCPI_OPT_TOE)) {
200 printf("%s connection NOT OFFLOADED!\n", __FUNCTION__);
201 return -EINVAL;
202 }
203
204 ep->snd_seq = ti.tcpi_snd_nxt;
205 ep->rcv_seq = ti.tcpi_rcv_nxt;
206 ep->emss = ti.__tcpi_snd_mss - sizeof(struct tcpiphdr);
207 ep->hwtid = TOEPCB(ep->com.so)->tp_tid; /* XXX */
208 if (ti.tcpi_options & TCPI_OPT_TIMESTAMPS)
209 ep->emss -= 12;
210 if (ep->emss < 128)
211 ep->emss = 128;
212 return 0;
213}
214
215static enum iwch_ep_state
216state_read(struct iwch_ep_common *epc)
217{
218 enum iwch_ep_state state;
219
220 mtx_lock(&epc->lock);
221 state = epc->state;
222 mtx_unlock(&epc->lock);
223 return state;
224}
225
226static void
227__state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
228{
229 epc->state = new;
230}
231
232static void
233state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
234{
235
236 mtx_lock(&epc->lock);
237 CTR3(KTR_IW_CXGB, "%s - %s -> %s", __FUNCTION__, states[epc->state], states[new]);
238 __state_set(epc, new);
239 mtx_unlock(&epc->lock);
240 return;
241}
242
243static void *
244alloc_ep(int size, int flags)
245{
246 struct iwch_ep_common *epc;
247
248 epc = malloc(size, M_DEVBUF, flags);
249 if (epc) {
250 memset(epc, 0, size);
251 refcount_init(&epc->refcount, 1);
252 mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_DEF|MTX_DUPOK);
253 cv_init(&epc->waitq, "iwch_epc cv");
254 }
255 CTR2(KTR_IW_CXGB, "%s alloc ep %p", __FUNCTION__, epc);
256 return epc;
257}
258
259void __free_ep(struct iwch_ep_common *epc)
260{
261 CTR3(KTR_IW_CXGB, "%s ep %p state %s", __FUNCTION__, epc, states[state_read(epc)]);
262 KASSERT(!epc->so, ("%s warning ep->so %p \n", __FUNCTION__, epc->so));
263 KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list!\n", __FUNCTION__, epc));
264 free(epc, M_DEVBUF);
265}
266
267int
268iwch_quiesce_tid(struct iwch_ep *ep)
269{
270#ifdef notyet
271 struct cpl_set_tcb_field *req;
272 struct mbuf *m = get_mbuf(NULL, sizeof(*req), M_NOWAIT);
273
274 if (m == NULL)
275 return (-ENOMEM);
276 req = (struct cpl_set_tcb_field *) mbuf_put(m, sizeof(*req));
277 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
278 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
279 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep->hwtid));
280 req->reply = 0;
281 req->cpu_idx = 0;
282 req->word = htons(W_TCB_RX_QUIESCE);
283 req->mask = cpu_to_be64(1ULL << S_TCB_RX_QUIESCE);
284 req->val = cpu_to_be64(1 << S_TCB_RX_QUIESCE);
285
286 m_set_priority(m, CPL_PRIORITY_DATA);
287 cxgb_ofld_send(ep->com.tdev, m);
288#endif
289 return 0;
290}
291
292int
293iwch_resume_tid(struct iwch_ep *ep)
294{
295#ifdef notyet
296 struct cpl_set_tcb_field *req;
297 struct mbuf *m = get_mbuf(NULL, sizeof(*req), M_NOWAIT);
298
299 if (m == NULL)
300 return (-ENOMEM);
301 req = (struct cpl_set_tcb_field *) mbuf_put(m, sizeof(*req));
302 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
303 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
304 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep->hwtid));
305 req->reply = 0;
306 req->cpu_idx = 0;
307 req->word = htons(W_TCB_RX_QUIESCE);
308 req->mask = cpu_to_be64(1ULL << S_TCB_RX_QUIESCE);
309 req->val = 0;
310
311 m_set_priority(m, CPL_PRIORITY_DATA);
312 cxgb_ofld_send(ep->com.tdev, m);
313#endif
314 return 0;
315}
316
317static struct rtentry *
318find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port,
319 __be16 peer_port, u8 tos)
320{
321 struct route iproute;
322 struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
323
324 bzero(&iproute, sizeof iproute);
325 dst->sin_family = AF_INET;
326 dst->sin_len = sizeof *dst;
327 dst->sin_addr.s_addr = peer_ip;
328
329 rtalloc(&iproute);
330 return iproute.ro_rt;
331}
332
333static void
334close_socket(struct iwch_ep_common *epc)
335{
336 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]);
337 SOCK_LOCK(epc->so);
| 145
146/*
147 * Cruft to offload socket upcalls onto thread.
148 */
149static struct mtx req_lock;
150static TAILQ_HEAD(iwch_ep_list, iwch_ep_common) req_list;
151static struct task iw_cxgb_task;
152static struct taskqueue *iw_cxgb_taskq;
153static void process_req(void *ctx, int pending);
154
155static void
156start_ep_timer(struct iwch_ep *ep)
157{
158 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
159 if (callout_pending(&ep->timer)) {
160 CTR2(KTR_IW_CXGB, "%s stopped / restarted timer ep %p", __FUNCTION__, ep);
161 callout_deactivate(&ep->timer);
162 callout_drain(&ep->timer);
163 } else {
164 /*
165 * XXX this looks racy
166 */
167 get_ep(&ep->com);
168 callout_init(&ep->timer, TRUE);
169 }
170 callout_reset(&ep->timer, ep_timeout_secs * hz, ep_timeout, ep);
171}
172
173static void
174stop_ep_timer(struct iwch_ep *ep)
175{
176 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
177 callout_drain(&ep->timer);
178 put_ep(&ep->com);
179}
180
181static int set_tcpinfo(struct iwch_ep *ep)
182{
183 struct tcp_info ti;
184 struct sockopt sopt;
185 int err;
186
187 sopt.sopt_dir = SOPT_GET;
188 sopt.sopt_level = IPPROTO_TCP;
189 sopt.sopt_name = TCP_INFO;
190 sopt.sopt_val = (caddr_t)&ti;
191 sopt.sopt_valsize = sizeof ti;
192 sopt.sopt_td = NULL;
193
194 err = sogetopt(ep->com.so, &sopt);
195 if (err) {
196 printf("%s can't get tcpinfo\n", __FUNCTION__);
197 return -err;
198 }
199 if (!(ti.tcpi_options & TCPI_OPT_TOE)) {
200 printf("%s connection NOT OFFLOADED!\n", __FUNCTION__);
201 return -EINVAL;
202 }
203
204 ep->snd_seq = ti.tcpi_snd_nxt;
205 ep->rcv_seq = ti.tcpi_rcv_nxt;
206 ep->emss = ti.__tcpi_snd_mss - sizeof(struct tcpiphdr);
207 ep->hwtid = TOEPCB(ep->com.so)->tp_tid; /* XXX */
208 if (ti.tcpi_options & TCPI_OPT_TIMESTAMPS)
209 ep->emss -= 12;
210 if (ep->emss < 128)
211 ep->emss = 128;
212 return 0;
213}
214
215static enum iwch_ep_state
216state_read(struct iwch_ep_common *epc)
217{
218 enum iwch_ep_state state;
219
220 mtx_lock(&epc->lock);
221 state = epc->state;
222 mtx_unlock(&epc->lock);
223 return state;
224}
225
226static void
227__state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
228{
229 epc->state = new;
230}
231
232static void
233state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
234{
235
236 mtx_lock(&epc->lock);
237 CTR3(KTR_IW_CXGB, "%s - %s -> %s", __FUNCTION__, states[epc->state], states[new]);
238 __state_set(epc, new);
239 mtx_unlock(&epc->lock);
240 return;
241}
242
243static void *
244alloc_ep(int size, int flags)
245{
246 struct iwch_ep_common *epc;
247
248 epc = malloc(size, M_DEVBUF, flags);
249 if (epc) {
250 memset(epc, 0, size);
251 refcount_init(&epc->refcount, 1);
252 mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_DEF|MTX_DUPOK);
253 cv_init(&epc->waitq, "iwch_epc cv");
254 }
255 CTR2(KTR_IW_CXGB, "%s alloc ep %p", __FUNCTION__, epc);
256 return epc;
257}
258
259void __free_ep(struct iwch_ep_common *epc)
260{
261 CTR3(KTR_IW_CXGB, "%s ep %p state %s", __FUNCTION__, epc, states[state_read(epc)]);
262 KASSERT(!epc->so, ("%s warning ep->so %p \n", __FUNCTION__, epc->so));
263 KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list!\n", __FUNCTION__, epc));
264 free(epc, M_DEVBUF);
265}
266
267int
268iwch_quiesce_tid(struct iwch_ep *ep)
269{
270#ifdef notyet
271 struct cpl_set_tcb_field *req;
272 struct mbuf *m = get_mbuf(NULL, sizeof(*req), M_NOWAIT);
273
274 if (m == NULL)
275 return (-ENOMEM);
276 req = (struct cpl_set_tcb_field *) mbuf_put(m, sizeof(*req));
277 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
278 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
279 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep->hwtid));
280 req->reply = 0;
281 req->cpu_idx = 0;
282 req->word = htons(W_TCB_RX_QUIESCE);
283 req->mask = cpu_to_be64(1ULL << S_TCB_RX_QUIESCE);
284 req->val = cpu_to_be64(1 << S_TCB_RX_QUIESCE);
285
286 m_set_priority(m, CPL_PRIORITY_DATA);
287 cxgb_ofld_send(ep->com.tdev, m);
288#endif
289 return 0;
290}
291
292int
293iwch_resume_tid(struct iwch_ep *ep)
294{
295#ifdef notyet
296 struct cpl_set_tcb_field *req;
297 struct mbuf *m = get_mbuf(NULL, sizeof(*req), M_NOWAIT);
298
299 if (m == NULL)
300 return (-ENOMEM);
301 req = (struct cpl_set_tcb_field *) mbuf_put(m, sizeof(*req));
302 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
303 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
304 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep->hwtid));
305 req->reply = 0;
306 req->cpu_idx = 0;
307 req->word = htons(W_TCB_RX_QUIESCE);
308 req->mask = cpu_to_be64(1ULL << S_TCB_RX_QUIESCE);
309 req->val = 0;
310
311 m_set_priority(m, CPL_PRIORITY_DATA);
312 cxgb_ofld_send(ep->com.tdev, m);
313#endif
314 return 0;
315}
316
317static struct rtentry *
318find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port,
319 __be16 peer_port, u8 tos)
320{
321 struct route iproute;
322 struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
323
324 bzero(&iproute, sizeof iproute);
325 dst->sin_family = AF_INET;
326 dst->sin_len = sizeof *dst;
327 dst->sin_addr.s_addr = peer_ip;
328
329 rtalloc(&iproute);
330 return iproute.ro_rt;
331}
332
333static void
334close_socket(struct iwch_ep_common *epc)
335{
336 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]);
337 SOCK_LOCK(epc->so);
|
338 epc->so->so_upcall = NULL;
339 epc->so->so_upcallarg = NULL;
340 epc->so->so_rcv.sb_flags &= ~SB_UPCALL;
| 338 soupcall_clear(epc->so, SO_RCV);
|
341 SOCK_UNLOCK(epc->so);
342 soshutdown(epc->so, SHUT_WR|SHUT_RD);
343 epc->so = NULL;
344}
345
346static void
347shutdown_socket(struct iwch_ep_common *epc)
348{
349 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]);
350 soshutdown(epc->so, SHUT_WR);
351}
352
353static void
354abort_socket(struct iwch_ep *ep)
355{
356 struct sockopt sopt;
357 int err;
358 struct linger l;
359
360 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
361 l.l_onoff = 1;
362 l.l_linger = 0;
363
364 /* linger_time of 0 forces RST to be sent */
365 sopt.sopt_dir = SOPT_SET;
366 sopt.sopt_level = SOL_SOCKET;
367 sopt.sopt_name = SO_LINGER;
368 sopt.sopt_val = (caddr_t)&l;
369 sopt.sopt_valsize = sizeof l;
370 sopt.sopt_td = NULL;
371 err = sosetopt(ep->com.so, &sopt);
372 if (err)
373 printf("%s can't set linger to 0, no RST! err %d\n", __FUNCTION__, err);
374}
375
376static void
377send_mpa_req(struct iwch_ep *ep)
378{
379 int mpalen;
380 struct mpa_message *mpa;
381 struct mbuf *m;
382 int err;
383
384 CTR3(KTR_IW_CXGB, "%s ep %p pd_len %d", __FUNCTION__, ep, ep->plen);
385
386 mpalen = sizeof(*mpa) + ep->plen;
387 m = m_gethdr(mpalen, M_NOWAIT);
388 if (m == NULL) {
389 connect_reply_upcall(ep, -ENOMEM);
390 return;
391 }
392 mpa = mtod(m, struct mpa_message *);
393 m->m_len = mpalen;
394 m->m_pkthdr.len = mpalen;
395 memset(mpa, 0, sizeof(*mpa));
396 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key));
397 mpa->flags = (crc_enabled ? MPA_CRC : 0) |
398 (markers_enabled ? MPA_MARKERS : 0);
399 mpa->private_data_size = htons(ep->plen);
400 mpa->revision = mpa_rev;
401 if (ep->plen)
402 memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen);
403
404 err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread);
405 if (err) {
406 m_freem(m);
407 connect_reply_upcall(ep, -ENOMEM);
408 return;
409 }
410
411 start_ep_timer(ep);
412 state_set(&ep->com, MPA_REQ_SENT);
413 return;
414}
415
416static int
417send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen)
418{
419 int mpalen;
420 struct mpa_message *mpa;
421 struct mbuf *m;
422 int err;
423
424 CTR3(KTR_IW_CXGB, "%s ep %p plen %d", __FUNCTION__, ep, plen);
425
426 mpalen = sizeof(*mpa) + plen;
427
428 m = m_gethdr(mpalen, M_NOWAIT);
429 if (m == NULL) {
430 printf("%s - cannot alloc mbuf!\n", __FUNCTION__);
431 return (-ENOMEM);
432 }
433 mpa = mtod(m, struct mpa_message *);
434 m->m_len = mpalen;
435 m->m_pkthdr.len = mpalen;
436 memset(mpa, 0, sizeof(*mpa));
437 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
438 mpa->flags = MPA_REJECT;
439 mpa->revision = mpa_rev;
440 mpa->private_data_size = htons(plen);
441 if (plen)
442 memcpy(mpa->private_data, pdata, plen);
443 err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread);
444 PANIC_IF(err);
445 return 0;
446}
447
448static int
449send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)
450{
451 int mpalen;
452 struct mpa_message *mpa;
453 struct mbuf *m;
454
455 CTR4(KTR_IW_CXGB, "%s ep %p so %p plen %d", __FUNCTION__, ep, ep->com.so, plen);
456
457 mpalen = sizeof(*mpa) + plen;
458
459 m = m_gethdr(mpalen, M_NOWAIT);
460 if (m == NULL) {
461 printf("%s - cannot alloc mbuf!\n", __FUNCTION__);
462 return (-ENOMEM);
463 }
464 mpa = mtod(m, struct mpa_message *);
465 m->m_len = mpalen;
466 m->m_pkthdr.len = mpalen;
467 memset(mpa, 0, sizeof(*mpa));
468 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
469 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) |
470 (markers_enabled ? MPA_MARKERS : 0);
471 mpa->revision = mpa_rev;
472 mpa->private_data_size = htons(plen);
473 if (plen)
474 memcpy(mpa->private_data, pdata, plen);
475
476 state_set(&ep->com, MPA_REP_SENT);
477 return sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT,
478 ep->com.thread);
479}
480
481static void
482close_complete_upcall(struct iwch_ep *ep)
483{
484 struct iw_cm_event event;
485
486 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
487 memset(&event, 0, sizeof(event));
488 event.event = IW_CM_EVENT_CLOSE;
489 if (ep->com.cm_id) {
490 CTR3(KTR_IW_CXGB, "close complete delivered ep %p cm_id %p tid %d",
491 ep, ep->com.cm_id, ep->hwtid);
492 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
493 ep->com.cm_id->rem_ref(ep->com.cm_id);
494 ep->com.cm_id = NULL;
495 ep->com.qp = NULL;
496 }
497}
498
499static void
500abort_connection(struct iwch_ep *ep)
501{
502 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
503 state_set(&ep->com, ABORTING);
504 abort_socket(ep);
505 close_socket(&ep->com);
506 close_complete_upcall(ep);
507 state_set(&ep->com, DEAD);
508 put_ep(&ep->com);
509}
510
511static void
512peer_close_upcall(struct iwch_ep *ep)
513{
514 struct iw_cm_event event;
515
516 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
517 memset(&event, 0, sizeof(event));
518 event.event = IW_CM_EVENT_DISCONNECT;
519 if (ep->com.cm_id) {
520 CTR3(KTR_IW_CXGB, "peer close delivered ep %p cm_id %p tid %d",
521 ep, ep->com.cm_id, ep->hwtid);
522 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
523 }
524}
525
526static void
527peer_abort_upcall(struct iwch_ep *ep)
528{
529 struct iw_cm_event event;
530
531 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
532 memset(&event, 0, sizeof(event));
533 event.event = IW_CM_EVENT_CLOSE;
534 event.status = ECONNRESET;
535 if (ep->com.cm_id) {
536 CTR3(KTR_IW_CXGB, "abort delivered ep %p cm_id %p tid %d", ep,
537 ep->com.cm_id, ep->hwtid);
538 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
539 ep->com.cm_id->rem_ref(ep->com.cm_id);
540 ep->com.cm_id = NULL;
541 ep->com.qp = NULL;
542 }
543}
544
545static void
546connect_reply_upcall(struct iwch_ep *ep, int status)
547{
548 struct iw_cm_event event;
549
550 CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s status %d", __FUNCTION__, ep, ep->com.so, states[ep->com.state], status);
551 memset(&event, 0, sizeof(event));
552 event.event = IW_CM_EVENT_CONNECT_REPLY;
553 event.status = status;
554 event.local_addr = ep->com.local_addr;
555 event.remote_addr = ep->com.remote_addr;
556
557 if ((status == 0) || (status == ECONNREFUSED)) {
558 event.private_data_len = ep->plen;
559 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
560 }
561 if (ep->com.cm_id) {
562 CTR4(KTR_IW_CXGB, "%s ep %p tid %d status %d", __FUNCTION__, ep,
563 ep->hwtid, status);
564 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
565 }
566 if (status < 0) {
567 ep->com.cm_id->rem_ref(ep->com.cm_id);
568 ep->com.cm_id = NULL;
569 ep->com.qp = NULL;
570 }
571}
572
573static void
574connect_request_upcall(struct iwch_ep *ep)
575{
576 struct iw_cm_event event;
577
578 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
579 memset(&event, 0, sizeof(event));
580 event.event = IW_CM_EVENT_CONNECT_REQUEST;
581 event.local_addr = ep->com.local_addr;
582 event.remote_addr = ep->com.remote_addr;
583 event.private_data_len = ep->plen;
584 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
585 event.provider_data = ep;
586 event.so = ep->com.so;
587 if (state_read(&ep->parent_ep->com) != DEAD)
588 ep->parent_ep->com.cm_id->event_handler(
589 ep->parent_ep->com.cm_id,
590 &event);
591 put_ep(&ep->parent_ep->com);
592 ep->parent_ep = NULL;
593}
594
595static void
596established_upcall(struct iwch_ep *ep)
597{
598 struct iw_cm_event event;
599
600 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
601 memset(&event, 0, sizeof(event));
602 event.event = IW_CM_EVENT_ESTABLISHED;
603 if (ep->com.cm_id) {
604 CTR3(KTR_IW_CXGB, "%s ep %p tid %d", __FUNCTION__, ep, ep->hwtid);
605 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
606 }
607}
608
609static void
610process_mpa_reply(struct iwch_ep *ep)
611{
612 struct mpa_message *mpa;
613 u16 plen;
614 struct iwch_qp_attributes attrs;
615 enum iwch_qp_attr_mask mask;
616 int err;
617 struct mbuf *top, *m;
618 int flags = MSG_DONTWAIT;
619 struct uio uio;
620 int len;
621
622 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
623
624 /*
625 * Stop mpa timer. If it expired, then the state has
626 * changed and we bail since ep_timeout already aborted
627 * the connection.
628 */
629 stop_ep_timer(ep);
630 if (state_read(&ep->com) != MPA_REQ_SENT)
631 return;
632
633 uio.uio_resid = len = 1000000;
634 uio.uio_td = ep->com.thread;
635 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags);
636 if (err) {
637 if (err == EWOULDBLOCK) {
638 start_ep_timer(ep);
639 return;
640 }
641 err = -err;
642 goto err;
643 }
644
645 if (ep->com.so->so_rcv.sb_mb) {
646 printf("%s data after soreceive called! so %p sb_mb %p top %p\n",
647 __FUNCTION__, ep->com.so, ep->com.so->so_rcv.sb_mb, top);
648 }
649
650 m = top;
651 do {
652 /*
653 * If we get more than the supported amount of private data
654 * then we must fail this connection.
655 */
656 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) {
657 err = (-EINVAL);
658 goto err;
659 }
660
661 /*
662 * copy the new data into our accumulation buffer.
663 */
664 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len]));
665 ep->mpa_pkt_len += m->m_len;
666 if (!m->m_next)
667 m = m->m_nextpkt;
668 else
669 m = m->m_next;
670 } while (m);
671
672 m_freem(top);
673
674 /*
675 * if we don't even have the mpa message, then bail.
676 */
677 if (ep->mpa_pkt_len < sizeof(*mpa))
678 return;
679 mpa = (struct mpa_message *)ep->mpa_pkt;
680
681 /* Validate MPA header. */
682 if (mpa->revision != mpa_rev) {
683 CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision);
684 err = EPROTO;
685 goto err;
686 }
687 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) {
688 CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key);
689 err = EPROTO;
690 goto err;
691 }
692
693 plen = ntohs(mpa->private_data_size);
694
695 /*
696 * Fail if there's too much private data.
697 */
698 if (plen > MPA_MAX_PRIVATE_DATA) {
699 CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen);
700 err = EPROTO;
701 goto err;
702 }
703
704 /*
705 * If plen does not account for pkt size
706 */
707 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
708 CTR2(KTR_IW_CXGB, "%s pkt too big %d", __FUNCTION__, ep->mpa_pkt_len);
709 err = EPROTO;
710 goto err;
711 }
712
713 ep->plen = (u8) plen;
714
715 /*
716 * If we don't have all the pdata yet, then bail.
717 * We'll continue process when more data arrives.
718 */
719 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
720 return;
721
722 if (mpa->flags & MPA_REJECT) {
723 err = ECONNREFUSED;
724 goto err;
725 }
726
727 /*
728 * If we get here we have accumulated the entire mpa
729 * start reply message including private data. And
730 * the MPA header is valid.
731 */
732 CTR1(KTR_IW_CXGB, "%s mpa rpl looks good!", __FUNCTION__);
733 state_set(&ep->com, FPDU_MODE);
734 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
735 ep->mpa_attr.recv_marker_enabled = markers_enabled;
736 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
737 ep->mpa_attr.version = mpa_rev;
738 if (set_tcpinfo(ep)) {
739 printf("%s set_tcpinfo error\n", __FUNCTION__);
740 goto err;
741 }
742 CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, "
743 "xmit_marker_enabled=%d, version=%d", __FUNCTION__,
744 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
745 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
746
747 attrs.mpa_attr = ep->mpa_attr;
748 attrs.max_ird = ep->ird;
749 attrs.max_ord = ep->ord;
750 attrs.llp_stream_handle = ep;
751 attrs.next_state = IWCH_QP_STATE_RTS;
752
753 mask = IWCH_QP_ATTR_NEXT_STATE |
754 IWCH_QP_ATTR_LLP_STREAM_HANDLE | IWCH_QP_ATTR_MPA_ATTR |
755 IWCH_QP_ATTR_MAX_IRD | IWCH_QP_ATTR_MAX_ORD;
756
757 /* bind QP and TID with INIT_WR */
758 err = iwch_modify_qp(ep->com.qp->rhp,
759 ep->com.qp, mask, &attrs, 1);
760 if (!err)
761 goto out;
762err:
763 abort_connection(ep);
764out:
765 connect_reply_upcall(ep, err);
766 return;
767}
768
769static void
770process_mpa_request(struct iwch_ep *ep)
771{
772 struct mpa_message *mpa;
773 u16 plen;
774 int flags = MSG_DONTWAIT;
775 struct mbuf *top, *m;
776 int err;
777 struct uio uio;
778 int len;
779
780 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
781
782 /*
783 * Stop mpa timer. If it expired, then the state has
784 * changed and we bail since ep_timeout already aborted
785 * the connection.
786 */
787 stop_ep_timer(ep);
788 if (state_read(&ep->com) != MPA_REQ_WAIT)
789 return;
790
791 uio.uio_resid = len = 1000000;
792 uio.uio_td = ep->com.thread;
793 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags);
794 if (err) {
795 if (err == EWOULDBLOCK) {
796 start_ep_timer(ep);
797 return;
798 }
799 err = -err;
800 goto err;
801 }
802
803 m = top;
804 do {
805
806 /*
807 * If we get more than the supported amount of private data
808 * then we must fail this connection.
809 */
810 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) {
811 CTR2(KTR_IW_CXGB, "%s mpa message too big %d", __FUNCTION__,
812 ep->mpa_pkt_len + m->m_len);
813 goto err;
814 }
815
816
817 /*
818 * Copy the new data into our accumulation buffer.
819 */
820 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len]));
821 ep->mpa_pkt_len += m->m_len;
822
823 if (!m->m_next)
824 m = m->m_nextpkt;
825 else
826 m = m->m_next;
827 } while (m);
828
829 m_freem(top);
830
831 /*
832 * If we don't even have the mpa message, then bail.
833 * We'll continue process when more data arrives.
834 */
835 if (ep->mpa_pkt_len < sizeof(*mpa)) {
836 start_ep_timer(ep);
837 CTR2(KTR_IW_CXGB, "%s not enough header %d...waiting...", __FUNCTION__,
838 ep->mpa_pkt_len);
839 return;
840 }
841 mpa = (struct mpa_message *) ep->mpa_pkt;
842
843 /*
844 * Validate MPA Header.
845 */
846 if (mpa->revision != mpa_rev) {
847 CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision);
848 goto err;
849 }
850
851 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) {
852 CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key);
853 goto err;
854 }
855
856 plen = ntohs(mpa->private_data_size);
857
858 /*
859 * Fail if there's too much private data.
860 */
861 if (plen > MPA_MAX_PRIVATE_DATA) {
862 CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen);
863 goto err;
864 }
865
866 /*
867 * If plen does not account for pkt size
868 */
869 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
870 CTR2(KTR_IW_CXGB, "%s more data after private data %d", __FUNCTION__,
871 ep->mpa_pkt_len);
872 goto err;
873 }
874 ep->plen = (u8) plen;
875
876 /*
877 * If we don't have all the pdata yet, then bail.
878 */
879 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) {
880 start_ep_timer(ep);
881 CTR2(KTR_IW_CXGB, "%s more mpa msg to come %d", __FUNCTION__,
882 ep->mpa_pkt_len);
883 return;
884 }
885
886 /*
887 * If we get here we have accumulated the entire mpa
888 * start reply message including private data.
889 */
890 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
891 ep->mpa_attr.recv_marker_enabled = markers_enabled;
892 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
893 ep->mpa_attr.version = mpa_rev;
894 if (set_tcpinfo(ep)) {
895 printf("%s set_tcpinfo error\n", __FUNCTION__);
896 goto err;
897 }
898 CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, "
899 "xmit_marker_enabled=%d, version=%d", __FUNCTION__,
900 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
901 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
902
903 state_set(&ep->com, MPA_REQ_RCVD);
904
905 /* drive upcall */
906 connect_request_upcall(ep);
907 return;
908err:
909 abort_connection(ep);
910 return;
911}
912
913static void
914process_peer_close(struct iwch_ep *ep)
915{
916 struct iwch_qp_attributes attrs;
917 int disconnect = 1;
918 int release = 0;
919
920 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
921
922 mtx_lock(&ep->com.lock);
923 switch (ep->com.state) {
924 case MPA_REQ_WAIT:
925 __state_set(&ep->com, CLOSING);
926 break;
927 case MPA_REQ_SENT:
928 __state_set(&ep->com, CLOSING);
929 connect_reply_upcall(ep, -ECONNRESET);
930 break;
931 case MPA_REQ_RCVD:
932
933 /*
934 * We're gonna mark this puppy DEAD, but keep
935 * the reference on it until the ULP accepts or
936 * rejects the CR.
937 */
938 __state_set(&ep->com, CLOSING);
939 get_ep(&ep->com);
940 break;
941 case MPA_REP_SENT:
942 __state_set(&ep->com, CLOSING);
943 break;
944 case FPDU_MODE:
945 start_ep_timer(ep);
946 __state_set(&ep->com, CLOSING);
947 attrs.next_state = IWCH_QP_STATE_CLOSING;
948 iwch_modify_qp(ep->com.qp->rhp, ep->com.qp,
949 IWCH_QP_ATTR_NEXT_STATE, &attrs, 1);
950 peer_close_upcall(ep);
951 break;
952 case ABORTING:
953 disconnect = 0;
954 break;
955 case CLOSING:
956 __state_set(&ep->com, MORIBUND);
957 disconnect = 0;
958 break;
959 case MORIBUND:
960 stop_ep_timer(ep);
961 if (ep->com.cm_id && ep->com.qp) {
962 attrs.next_state = IWCH_QP_STATE_IDLE;
963 iwch_modify_qp(ep->com.qp->rhp, ep->com.qp,
964 IWCH_QP_ATTR_NEXT_STATE, &attrs, 1);
965 }
966 close_socket(&ep->com);
967 close_complete_upcall(ep);
968 __state_set(&ep->com, DEAD);
969 release = 1;
970 disconnect = 0;
971 break;
972 case DEAD:
973 disconnect = 0;
974 break;
975 default:
976 PANIC_IF(1);
977 }
978 mtx_unlock(&ep->com.lock);
979 if (disconnect)
980 iwch_ep_disconnect(ep, 0, M_NOWAIT);
981 if (release)
982 put_ep(&ep->com);
983 return;
984}
985
986static void
987process_conn_error(struct iwch_ep *ep)
988{
989 struct iwch_qp_attributes attrs;
990 int ret;
991 int state;
992
993 state = state_read(&ep->com);
994 CTR5(KTR_IW_CXGB, "%s ep %p so %p so->so_error %u state %s", __FUNCTION__, ep, ep->com.so, ep->com.so->so_error, states[ep->com.state]);
995 switch (state) {
996 case MPA_REQ_WAIT:
997 stop_ep_timer(ep);
998 break;
999 case MPA_REQ_SENT:
1000 stop_ep_timer(ep);
1001 connect_reply_upcall(ep, -ECONNRESET);
1002 break;
1003 case MPA_REP_SENT:
1004 ep->com.rpl_err = ECONNRESET;
1005 CTR1(KTR_IW_CXGB, "waking up ep %p", ep);
1006 break;
1007 case MPA_REQ_RCVD:
1008
1009 /*
1010 * We're gonna mark this puppy DEAD, but keep
1011 * the reference on it until the ULP accepts or
1012 * rejects the CR.
1013 */
1014 get_ep(&ep->com);
1015 break;
1016 case MORIBUND:
1017 case CLOSING:
1018 stop_ep_timer(ep);
1019 /*FALLTHROUGH*/
1020 case FPDU_MODE:
1021 if (ep->com.cm_id && ep->com.qp) {
1022 attrs.next_state = IWCH_QP_STATE_ERROR;
1023 ret = iwch_modify_qp(ep->com.qp->rhp,
1024 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1025 &attrs, 1);
1026 if (ret)
1027 log(LOG_ERR,
1028 "%s - qp <- error failed!\n",
1029 __FUNCTION__);
1030 }
1031 peer_abort_upcall(ep);
1032 break;
1033 case ABORTING:
1034 break;
1035 case DEAD:
1036 CTR2(KTR_IW_CXGB, "%s so_error %d IN DEAD STATE!!!!", __FUNCTION__,
1037 ep->com.so->so_error);
1038 return;
1039 default:
1040 PANIC_IF(1);
1041 break;
1042 }
1043
1044 if (state != ABORTING) {
1045 close_socket(&ep->com);
1046 state_set(&ep->com, DEAD);
1047 put_ep(&ep->com);
1048 }
1049 return;
1050}
1051
1052static void
1053process_close_complete(struct iwch_ep *ep)
1054{
1055 struct iwch_qp_attributes attrs;
1056 int release = 0;
1057
1058 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1059 PANIC_IF(!ep);
1060
1061 /* The cm_id may be null if we failed to connect */
1062 mtx_lock(&ep->com.lock);
1063 switch (ep->com.state) {
1064 case CLOSING:
1065 __state_set(&ep->com, MORIBUND);
1066 break;
1067 case MORIBUND:
1068 stop_ep_timer(ep);
1069 if ((ep->com.cm_id) && (ep->com.qp)) {
1070 attrs.next_state = IWCH_QP_STATE_IDLE;
1071 iwch_modify_qp(ep->com.qp->rhp,
1072 ep->com.qp,
1073 IWCH_QP_ATTR_NEXT_STATE,
1074 &attrs, 1);
1075 }
1076 close_socket(&ep->com);
1077 close_complete_upcall(ep);
1078 __state_set(&ep->com, DEAD);
1079 release = 1;
1080 break;
1081 case ABORTING:
1082 break;
1083 case DEAD:
1084 default:
1085 PANIC_IF(1);
1086 break;
1087 }
1088 mtx_unlock(&ep->com.lock);
1089 if (release)
1090 put_ep(&ep->com);
1091 return;
1092}
1093
1094/*
1095 * T3A does 3 things when a TERM is received:
1096 * 1) send up a CPL_RDMA_TERMINATE message with the TERM packet
1097 * 2) generate an async event on the QP with the TERMINATE opcode
1098 * 3) post a TERMINATE opcde cqe into the associated CQ.
1099 *
1100 * For (1), we save the message in the qp for later consumer consumption.
1101 * For (2), we move the QP into TERMINATE, post a QP event and disconnect.
1102 * For (3), we toss the CQE in cxio_poll_cq().
1103 *
1104 * terminate() handles case (1)...
1105 */
1106static int
1107terminate(struct t3cdev *tdev, struct mbuf *m, void *ctx)
1108{
1109 struct toepcb *toep = (struct toepcb *)ctx;
1110 struct socket *so = toeptoso(toep);
| 339 SOCK_UNLOCK(epc->so);
340 soshutdown(epc->so, SHUT_WR|SHUT_RD);
341 epc->so = NULL;
342}
343
344static void
345shutdown_socket(struct iwch_ep_common *epc)
346{
347 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]);
348 soshutdown(epc->so, SHUT_WR);
349}
350
351static void
352abort_socket(struct iwch_ep *ep)
353{
354 struct sockopt sopt;
355 int err;
356 struct linger l;
357
358 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
359 l.l_onoff = 1;
360 l.l_linger = 0;
361
362 /* linger_time of 0 forces RST to be sent */
363 sopt.sopt_dir = SOPT_SET;
364 sopt.sopt_level = SOL_SOCKET;
365 sopt.sopt_name = SO_LINGER;
366 sopt.sopt_val = (caddr_t)&l;
367 sopt.sopt_valsize = sizeof l;
368 sopt.sopt_td = NULL;
369 err = sosetopt(ep->com.so, &sopt);
370 if (err)
371 printf("%s can't set linger to 0, no RST! err %d\n", __FUNCTION__, err);
372}
373
374static void
375send_mpa_req(struct iwch_ep *ep)
376{
377 int mpalen;
378 struct mpa_message *mpa;
379 struct mbuf *m;
380 int err;
381
382 CTR3(KTR_IW_CXGB, "%s ep %p pd_len %d", __FUNCTION__, ep, ep->plen);
383
384 mpalen = sizeof(*mpa) + ep->plen;
385 m = m_gethdr(mpalen, M_NOWAIT);
386 if (m == NULL) {
387 connect_reply_upcall(ep, -ENOMEM);
388 return;
389 }
390 mpa = mtod(m, struct mpa_message *);
391 m->m_len = mpalen;
392 m->m_pkthdr.len = mpalen;
393 memset(mpa, 0, sizeof(*mpa));
394 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key));
395 mpa->flags = (crc_enabled ? MPA_CRC : 0) |
396 (markers_enabled ? MPA_MARKERS : 0);
397 mpa->private_data_size = htons(ep->plen);
398 mpa->revision = mpa_rev;
399 if (ep->plen)
400 memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen);
401
402 err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread);
403 if (err) {
404 m_freem(m);
405 connect_reply_upcall(ep, -ENOMEM);
406 return;
407 }
408
409 start_ep_timer(ep);
410 state_set(&ep->com, MPA_REQ_SENT);
411 return;
412}
413
414static int
415send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen)
416{
417 int mpalen;
418 struct mpa_message *mpa;
419 struct mbuf *m;
420 int err;
421
422 CTR3(KTR_IW_CXGB, "%s ep %p plen %d", __FUNCTION__, ep, plen);
423
424 mpalen = sizeof(*mpa) + plen;
425
426 m = m_gethdr(mpalen, M_NOWAIT);
427 if (m == NULL) {
428 printf("%s - cannot alloc mbuf!\n", __FUNCTION__);
429 return (-ENOMEM);
430 }
431 mpa = mtod(m, struct mpa_message *);
432 m->m_len = mpalen;
433 m->m_pkthdr.len = mpalen;
434 memset(mpa, 0, sizeof(*mpa));
435 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
436 mpa->flags = MPA_REJECT;
437 mpa->revision = mpa_rev;
438 mpa->private_data_size = htons(plen);
439 if (plen)
440 memcpy(mpa->private_data, pdata, plen);
441 err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread);
442 PANIC_IF(err);
443 return 0;
444}
445
446static int
447send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)
448{
449 int mpalen;
450 struct mpa_message *mpa;
451 struct mbuf *m;
452
453 CTR4(KTR_IW_CXGB, "%s ep %p so %p plen %d", __FUNCTION__, ep, ep->com.so, plen);
454
455 mpalen = sizeof(*mpa) + plen;
456
457 m = m_gethdr(mpalen, M_NOWAIT);
458 if (m == NULL) {
459 printf("%s - cannot alloc mbuf!\n", __FUNCTION__);
460 return (-ENOMEM);
461 }
462 mpa = mtod(m, struct mpa_message *);
463 m->m_len = mpalen;
464 m->m_pkthdr.len = mpalen;
465 memset(mpa, 0, sizeof(*mpa));
466 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
467 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) |
468 (markers_enabled ? MPA_MARKERS : 0);
469 mpa->revision = mpa_rev;
470 mpa->private_data_size = htons(plen);
471 if (plen)
472 memcpy(mpa->private_data, pdata, plen);
473
474 state_set(&ep->com, MPA_REP_SENT);
475 return sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT,
476 ep->com.thread);
477}
478
479static void
480close_complete_upcall(struct iwch_ep *ep)
481{
482 struct iw_cm_event event;
483
484 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
485 memset(&event, 0, sizeof(event));
486 event.event = IW_CM_EVENT_CLOSE;
487 if (ep->com.cm_id) {
488 CTR3(KTR_IW_CXGB, "close complete delivered ep %p cm_id %p tid %d",
489 ep, ep->com.cm_id, ep->hwtid);
490 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
491 ep->com.cm_id->rem_ref(ep->com.cm_id);
492 ep->com.cm_id = NULL;
493 ep->com.qp = NULL;
494 }
495}
496
497static void
498abort_connection(struct iwch_ep *ep)
499{
500 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
501 state_set(&ep->com, ABORTING);
502 abort_socket(ep);
503 close_socket(&ep->com);
504 close_complete_upcall(ep);
505 state_set(&ep->com, DEAD);
506 put_ep(&ep->com);
507}
508
509static void
510peer_close_upcall(struct iwch_ep *ep)
511{
512 struct iw_cm_event event;
513
514 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
515 memset(&event, 0, sizeof(event));
516 event.event = IW_CM_EVENT_DISCONNECT;
517 if (ep->com.cm_id) {
518 CTR3(KTR_IW_CXGB, "peer close delivered ep %p cm_id %p tid %d",
519 ep, ep->com.cm_id, ep->hwtid);
520 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
521 }
522}
523
524static void
525peer_abort_upcall(struct iwch_ep *ep)
526{
527 struct iw_cm_event event;
528
529 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
530 memset(&event, 0, sizeof(event));
531 event.event = IW_CM_EVENT_CLOSE;
532 event.status = ECONNRESET;
533 if (ep->com.cm_id) {
534 CTR3(KTR_IW_CXGB, "abort delivered ep %p cm_id %p tid %d", ep,
535 ep->com.cm_id, ep->hwtid);
536 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
537 ep->com.cm_id->rem_ref(ep->com.cm_id);
538 ep->com.cm_id = NULL;
539 ep->com.qp = NULL;
540 }
541}
542
543static void
544connect_reply_upcall(struct iwch_ep *ep, int status)
545{
546 struct iw_cm_event event;
547
548 CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s status %d", __FUNCTION__, ep, ep->com.so, states[ep->com.state], status);
549 memset(&event, 0, sizeof(event));
550 event.event = IW_CM_EVENT_CONNECT_REPLY;
551 event.status = status;
552 event.local_addr = ep->com.local_addr;
553 event.remote_addr = ep->com.remote_addr;
554
555 if ((status == 0) || (status == ECONNREFUSED)) {
556 event.private_data_len = ep->plen;
557 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
558 }
559 if (ep->com.cm_id) {
560 CTR4(KTR_IW_CXGB, "%s ep %p tid %d status %d", __FUNCTION__, ep,
561 ep->hwtid, status);
562 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
563 }
564 if (status < 0) {
565 ep->com.cm_id->rem_ref(ep->com.cm_id);
566 ep->com.cm_id = NULL;
567 ep->com.qp = NULL;
568 }
569}
570
571static void
572connect_request_upcall(struct iwch_ep *ep)
573{
574 struct iw_cm_event event;
575
576 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
577 memset(&event, 0, sizeof(event));
578 event.event = IW_CM_EVENT_CONNECT_REQUEST;
579 event.local_addr = ep->com.local_addr;
580 event.remote_addr = ep->com.remote_addr;
581 event.private_data_len = ep->plen;
582 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
583 event.provider_data = ep;
584 event.so = ep->com.so;
585 if (state_read(&ep->parent_ep->com) != DEAD)
586 ep->parent_ep->com.cm_id->event_handler(
587 ep->parent_ep->com.cm_id,
588 &event);
589 put_ep(&ep->parent_ep->com);
590 ep->parent_ep = NULL;
591}
592
593static void
594established_upcall(struct iwch_ep *ep)
595{
596 struct iw_cm_event event;
597
598 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
599 memset(&event, 0, sizeof(event));
600 event.event = IW_CM_EVENT_ESTABLISHED;
601 if (ep->com.cm_id) {
602 CTR3(KTR_IW_CXGB, "%s ep %p tid %d", __FUNCTION__, ep, ep->hwtid);
603 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
604 }
605}
606
607static void
608process_mpa_reply(struct iwch_ep *ep)
609{
610 struct mpa_message *mpa;
611 u16 plen;
612 struct iwch_qp_attributes attrs;
613 enum iwch_qp_attr_mask mask;
614 int err;
615 struct mbuf *top, *m;
616 int flags = MSG_DONTWAIT;
617 struct uio uio;
618 int len;
619
620 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
621
622 /*
623 * Stop mpa timer. If it expired, then the state has
624 * changed and we bail since ep_timeout already aborted
625 * the connection.
626 */
627 stop_ep_timer(ep);
628 if (state_read(&ep->com) != MPA_REQ_SENT)
629 return;
630
631 uio.uio_resid = len = 1000000;
632 uio.uio_td = ep->com.thread;
633 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags);
634 if (err) {
635 if (err == EWOULDBLOCK) {
636 start_ep_timer(ep);
637 return;
638 }
639 err = -err;
640 goto err;
641 }
642
643 if (ep->com.so->so_rcv.sb_mb) {
644 printf("%s data after soreceive called! so %p sb_mb %p top %p\n",
645 __FUNCTION__, ep->com.so, ep->com.so->so_rcv.sb_mb, top);
646 }
647
648 m = top;
649 do {
650 /*
651 * If we get more than the supported amount of private data
652 * then we must fail this connection.
653 */
654 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) {
655 err = (-EINVAL);
656 goto err;
657 }
658
659 /*
660 * copy the new data into our accumulation buffer.
661 */
662 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len]));
663 ep->mpa_pkt_len += m->m_len;
664 if (!m->m_next)
665 m = m->m_nextpkt;
666 else
667 m = m->m_next;
668 } while (m);
669
670 m_freem(top);
671
672 /*
673 * if we don't even have the mpa message, then bail.
674 */
675 if (ep->mpa_pkt_len < sizeof(*mpa))
676 return;
677 mpa = (struct mpa_message *)ep->mpa_pkt;
678
679 /* Validate MPA header. */
680 if (mpa->revision != mpa_rev) {
681 CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision);
682 err = EPROTO;
683 goto err;
684 }
685 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) {
686 CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key);
687 err = EPROTO;
688 goto err;
689 }
690
691 plen = ntohs(mpa->private_data_size);
692
693 /*
694 * Fail if there's too much private data.
695 */
696 if (plen > MPA_MAX_PRIVATE_DATA) {
697 CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen);
698 err = EPROTO;
699 goto err;
700 }
701
702 /*
703 * If plen does not account for pkt size
704 */
705 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
706 CTR2(KTR_IW_CXGB, "%s pkt too big %d", __FUNCTION__, ep->mpa_pkt_len);
707 err = EPROTO;
708 goto err;
709 }
710
711 ep->plen = (u8) plen;
712
713 /*
714 * If we don't have all the pdata yet, then bail.
715 * We'll continue process when more data arrives.
716 */
717 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
718 return;
719
720 if (mpa->flags & MPA_REJECT) {
721 err = ECONNREFUSED;
722 goto err;
723 }
724
725 /*
726 * If we get here we have accumulated the entire mpa
727 * start reply message including private data. And
728 * the MPA header is valid.
729 */
730 CTR1(KTR_IW_CXGB, "%s mpa rpl looks good!", __FUNCTION__);
731 state_set(&ep->com, FPDU_MODE);
732 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
733 ep->mpa_attr.recv_marker_enabled = markers_enabled;
734 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
735 ep->mpa_attr.version = mpa_rev;
736 if (set_tcpinfo(ep)) {
737 printf("%s set_tcpinfo error\n", __FUNCTION__);
738 goto err;
739 }
740 CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, "
741 "xmit_marker_enabled=%d, version=%d", __FUNCTION__,
742 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
743 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
744
745 attrs.mpa_attr = ep->mpa_attr;
746 attrs.max_ird = ep->ird;
747 attrs.max_ord = ep->ord;
748 attrs.llp_stream_handle = ep;
749 attrs.next_state = IWCH_QP_STATE_RTS;
750
751 mask = IWCH_QP_ATTR_NEXT_STATE |
752 IWCH_QP_ATTR_LLP_STREAM_HANDLE | IWCH_QP_ATTR_MPA_ATTR |
753 IWCH_QP_ATTR_MAX_IRD | IWCH_QP_ATTR_MAX_ORD;
754
755 /* bind QP and TID with INIT_WR */
756 err = iwch_modify_qp(ep->com.qp->rhp,
757 ep->com.qp, mask, &attrs, 1);
758 if (!err)
759 goto out;
760err:
761 abort_connection(ep);
762out:
763 connect_reply_upcall(ep, err);
764 return;
765}
766
767static void
768process_mpa_request(struct iwch_ep *ep)
769{
770 struct mpa_message *mpa;
771 u16 plen;
772 int flags = MSG_DONTWAIT;
773 struct mbuf *top, *m;
774 int err;
775 struct uio uio;
776 int len;
777
778 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
779
780 /*
781 * Stop mpa timer. If it expired, then the state has
782 * changed and we bail since ep_timeout already aborted
783 * the connection.
784 */
785 stop_ep_timer(ep);
786 if (state_read(&ep->com) != MPA_REQ_WAIT)
787 return;
788
789 uio.uio_resid = len = 1000000;
790 uio.uio_td = ep->com.thread;
791 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags);
792 if (err) {
793 if (err == EWOULDBLOCK) {
794 start_ep_timer(ep);
795 return;
796 }
797 err = -err;
798 goto err;
799 }
800
801 m = top;
802 do {
803
804 /*
805 * If we get more than the supported amount of private data
806 * then we must fail this connection.
807 */
808 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) {
809 CTR2(KTR_IW_CXGB, "%s mpa message too big %d", __FUNCTION__,
810 ep->mpa_pkt_len + m->m_len);
811 goto err;
812 }
813
814
815 /*
816 * Copy the new data into our accumulation buffer.
817 */
818 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len]));
819 ep->mpa_pkt_len += m->m_len;
820
821 if (!m->m_next)
822 m = m->m_nextpkt;
823 else
824 m = m->m_next;
825 } while (m);
826
827 m_freem(top);
828
829 /*
830 * If we don't even have the mpa message, then bail.
831 * We'll continue process when more data arrives.
832 */
833 if (ep->mpa_pkt_len < sizeof(*mpa)) {
834 start_ep_timer(ep);
835 CTR2(KTR_IW_CXGB, "%s not enough header %d...waiting...", __FUNCTION__,
836 ep->mpa_pkt_len);
837 return;
838 }
839 mpa = (struct mpa_message *) ep->mpa_pkt;
840
841 /*
842 * Validate MPA Header.
843 */
844 if (mpa->revision != mpa_rev) {
845 CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision);
846 goto err;
847 }
848
849 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) {
850 CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key);
851 goto err;
852 }
853
854 plen = ntohs(mpa->private_data_size);
855
856 /*
857 * Fail if there's too much private data.
858 */
859 if (plen > MPA_MAX_PRIVATE_DATA) {
860 CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen);
861 goto err;
862 }
863
864 /*
865 * If plen does not account for pkt size
866 */
867 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
868 CTR2(KTR_IW_CXGB, "%s more data after private data %d", __FUNCTION__,
869 ep->mpa_pkt_len);
870 goto err;
871 }
872 ep->plen = (u8) plen;
873
874 /*
875 * If we don't have all the pdata yet, then bail.
876 */
877 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) {
878 start_ep_timer(ep);
879 CTR2(KTR_IW_CXGB, "%s more mpa msg to come %d", __FUNCTION__,
880 ep->mpa_pkt_len);
881 return;
882 }
883
884 /*
885 * If we get here we have accumulated the entire mpa
886 * start reply message including private data.
887 */
888 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
889 ep->mpa_attr.recv_marker_enabled = markers_enabled;
890 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
891 ep->mpa_attr.version = mpa_rev;
892 if (set_tcpinfo(ep)) {
893 printf("%s set_tcpinfo error\n", __FUNCTION__);
894 goto err;
895 }
896 CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, "
897 "xmit_marker_enabled=%d, version=%d", __FUNCTION__,
898 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
899 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
900
901 state_set(&ep->com, MPA_REQ_RCVD);
902
903 /* drive upcall */
904 connect_request_upcall(ep);
905 return;
906err:
907 abort_connection(ep);
908 return;
909}
910
911static void
912process_peer_close(struct iwch_ep *ep)
913{
914 struct iwch_qp_attributes attrs;
915 int disconnect = 1;
916 int release = 0;
917
918 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
919
920 mtx_lock(&ep->com.lock);
921 switch (ep->com.state) {
922 case MPA_REQ_WAIT:
923 __state_set(&ep->com, CLOSING);
924 break;
925 case MPA_REQ_SENT:
926 __state_set(&ep->com, CLOSING);
927 connect_reply_upcall(ep, -ECONNRESET);
928 break;
929 case MPA_REQ_RCVD:
930
931 /*
932 * We're gonna mark this puppy DEAD, but keep
933 * the reference on it until the ULP accepts or
934 * rejects the CR.
935 */
936 __state_set(&ep->com, CLOSING);
937 get_ep(&ep->com);
938 break;
939 case MPA_REP_SENT:
940 __state_set(&ep->com, CLOSING);
941 break;
942 case FPDU_MODE:
943 start_ep_timer(ep);
944 __state_set(&ep->com, CLOSING);
945 attrs.next_state = IWCH_QP_STATE_CLOSING;
946 iwch_modify_qp(ep->com.qp->rhp, ep->com.qp,
947 IWCH_QP_ATTR_NEXT_STATE, &attrs, 1);
948 peer_close_upcall(ep);
949 break;
950 case ABORTING:
951 disconnect = 0;
952 break;
953 case CLOSING:
954 __state_set(&ep->com, MORIBUND);
955 disconnect = 0;
956 break;
957 case MORIBUND:
958 stop_ep_timer(ep);
959 if (ep->com.cm_id && ep->com.qp) {
960 attrs.next_state = IWCH_QP_STATE_IDLE;
961 iwch_modify_qp(ep->com.qp->rhp, ep->com.qp,
962 IWCH_QP_ATTR_NEXT_STATE, &attrs, 1);
963 }
964 close_socket(&ep->com);
965 close_complete_upcall(ep);
966 __state_set(&ep->com, DEAD);
967 release = 1;
968 disconnect = 0;
969 break;
970 case DEAD:
971 disconnect = 0;
972 break;
973 default:
974 PANIC_IF(1);
975 }
976 mtx_unlock(&ep->com.lock);
977 if (disconnect)
978 iwch_ep_disconnect(ep, 0, M_NOWAIT);
979 if (release)
980 put_ep(&ep->com);
981 return;
982}
983
984static void
985process_conn_error(struct iwch_ep *ep)
986{
987 struct iwch_qp_attributes attrs;
988 int ret;
989 int state;
990
991 state = state_read(&ep->com);
992 CTR5(KTR_IW_CXGB, "%s ep %p so %p so->so_error %u state %s", __FUNCTION__, ep, ep->com.so, ep->com.so->so_error, states[ep->com.state]);
993 switch (state) {
994 case MPA_REQ_WAIT:
995 stop_ep_timer(ep);
996 break;
997 case MPA_REQ_SENT:
998 stop_ep_timer(ep);
999 connect_reply_upcall(ep, -ECONNRESET);
1000 break;
1001 case MPA_REP_SENT:
1002 ep->com.rpl_err = ECONNRESET;
1003 CTR1(KTR_IW_CXGB, "waking up ep %p", ep);
1004 break;
1005 case MPA_REQ_RCVD:
1006
1007 /*
1008 * We're gonna mark this puppy DEAD, but keep
1009 * the reference on it until the ULP accepts or
1010 * rejects the CR.
1011 */
1012 get_ep(&ep->com);
1013 break;
1014 case MORIBUND:
1015 case CLOSING:
1016 stop_ep_timer(ep);
1017 /*FALLTHROUGH*/
1018 case FPDU_MODE:
1019 if (ep->com.cm_id && ep->com.qp) {
1020 attrs.next_state = IWCH_QP_STATE_ERROR;
1021 ret = iwch_modify_qp(ep->com.qp->rhp,
1022 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1023 &attrs, 1);
1024 if (ret)
1025 log(LOG_ERR,
1026 "%s - qp <- error failed!\n",
1027 __FUNCTION__);
1028 }
1029 peer_abort_upcall(ep);
1030 break;
1031 case ABORTING:
1032 break;
1033 case DEAD:
1034 CTR2(KTR_IW_CXGB, "%s so_error %d IN DEAD STATE!!!!", __FUNCTION__,
1035 ep->com.so->so_error);
1036 return;
1037 default:
1038 PANIC_IF(1);
1039 break;
1040 }
1041
1042 if (state != ABORTING) {
1043 close_socket(&ep->com);
1044 state_set(&ep->com, DEAD);
1045 put_ep(&ep->com);
1046 }
1047 return;
1048}
1049
1050static void
1051process_close_complete(struct iwch_ep *ep)
1052{
1053 struct iwch_qp_attributes attrs;
1054 int release = 0;
1055
1056 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1057 PANIC_IF(!ep);
1058
1059 /* The cm_id may be null if we failed to connect */
1060 mtx_lock(&ep->com.lock);
1061 switch (ep->com.state) {
1062 case CLOSING:
1063 __state_set(&ep->com, MORIBUND);
1064 break;
1065 case MORIBUND:
1066 stop_ep_timer(ep);
1067 if ((ep->com.cm_id) && (ep->com.qp)) {
1068 attrs.next_state = IWCH_QP_STATE_IDLE;
1069 iwch_modify_qp(ep->com.qp->rhp,
1070 ep->com.qp,
1071 IWCH_QP_ATTR_NEXT_STATE,
1072 &attrs, 1);
1073 }
1074 close_socket(&ep->com);
1075 close_complete_upcall(ep);
1076 __state_set(&ep->com, DEAD);
1077 release = 1;
1078 break;
1079 case ABORTING:
1080 break;
1081 case DEAD:
1082 default:
1083 PANIC_IF(1);
1084 break;
1085 }
1086 mtx_unlock(&ep->com.lock);
1087 if (release)
1088 put_ep(&ep->com);
1089 return;
1090}
1091
1092/*
1093 * T3A does 3 things when a TERM is received:
1094 * 1) send up a CPL_RDMA_TERMINATE message with the TERM packet
1095 * 2) generate an async event on the QP with the TERMINATE opcode
1096 * 3) post a TERMINATE opcde cqe into the associated CQ.
1097 *
1098 * For (1), we save the message in the qp for later consumer consumption.
1099 * For (2), we move the QP into TERMINATE, post a QP event and disconnect.
1100 * For (3), we toss the CQE in cxio_poll_cq().
1101 *
1102 * terminate() handles case (1)...
1103 */
1104static int
1105terminate(struct t3cdev *tdev, struct mbuf *m, void *ctx)
1106{
1107 struct toepcb *toep = (struct toepcb *)ctx;
1108 struct socket *so = toeptoso(toep);
|
1111 struct iwch_ep *ep = so->so_upcallarg;
| 1109 struct iwch_ep *ep = so->so_rcv.sb_upcallarg;
|
1112
1113 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
1114 m_adj(m, sizeof(struct cpl_rdma_terminate));
1115 CTR2(KTR_IW_CXGB, "%s saving %d bytes of term msg", __FUNCTION__, m->m_len);
1116 m_copydata(m, 0, m->m_len, ep->com.qp->attr.terminate_buffer);
1117 ep->com.qp->attr.terminate_msg_len = m->m_len;
1118 ep->com.qp->attr.is_terminate_local = 0;
1119 return CPL_RET_BUF_DONE;
1120}
1121
1122static int
1123ec_status(struct t3cdev *tdev, struct mbuf *m, void *ctx)
1124{
1125 struct toepcb *toep = (struct toepcb *)ctx;
1126 struct socket *so = toeptoso(toep);
1127 struct cpl_rdma_ec_status *rep = cplhdr(m);
1128 struct iwch_ep *ep;
1129 struct iwch_qp_attributes attrs;
1130 int release = 0;
1131
| 1110
1111 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
1112 m_adj(m, sizeof(struct cpl_rdma_terminate));
1113 CTR2(KTR_IW_CXGB, "%s saving %d bytes of term msg", __FUNCTION__, m->m_len);
1114 m_copydata(m, 0, m->m_len, ep->com.qp->attr.terminate_buffer);
1115 ep->com.qp->attr.terminate_msg_len = m->m_len;
1116 ep->com.qp->attr.is_terminate_local = 0;
1117 return CPL_RET_BUF_DONE;
1118}
1119
1120static int
1121ec_status(struct t3cdev *tdev, struct mbuf *m, void *ctx)
1122{
1123 struct toepcb *toep = (struct toepcb *)ctx;
1124 struct socket *so = toeptoso(toep);
1125 struct cpl_rdma_ec_status *rep = cplhdr(m);
1126 struct iwch_ep *ep;
1127 struct iwch_qp_attributes attrs;
1128 int release = 0;
1129
|
1132 ep = so->so_upcallarg;
| 1130 ep = so->so_rcv.sb_upcallarg;
|
1133 CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s ec_status %d", __FUNCTION__, ep, ep->com.so, states[ep->com.state], rep->status);
1134 if (!so || !ep) {
1135 panic("bogosity ep %p state %d, so %p state %x\n", ep, ep ? ep->com.state : -1, so, so ? so->so_state : -1);
1136 }
1137 mtx_lock(&ep->com.lock);
1138 switch (ep->com.state) {
1139 case CLOSING:
1140 if (!rep->status)
1141 __state_set(&ep->com, MORIBUND);
1142 else
1143 __state_set(&ep->com, ABORTING);
1144 break;
1145 case MORIBUND:
1146 stop_ep_timer(ep);
1147 if (!rep->status) {
1148 if ((ep->com.cm_id) && (ep->com.qp)) {
1149 attrs.next_state = IWCH_QP_STATE_IDLE;
1150 iwch_modify_qp(ep->com.qp->rhp,
1151 ep->com.qp,
1152 IWCH_QP_ATTR_NEXT_STATE,
1153 &attrs, 1);
1154 }
1155 close_socket(&ep->com);
1156 close_complete_upcall(ep);
1157 __state_set(&ep->com, DEAD);
1158 release = 1;
1159 }
1160 break;
1161 case DEAD:
1162 break;
1163 default:
1164 panic("unknown state: %d\n", ep->com.state);
1165 }
1166 mtx_unlock(&ep->com.lock);
1167 if (rep->status) {
1168 log(LOG_ERR, "%s BAD CLOSE - Aborting tid %u\n",
1169 __FUNCTION__, ep->hwtid);
1170 attrs.next_state = IWCH_QP_STATE_ERROR;
1171 iwch_modify_qp(ep->com.qp->rhp,
1172 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1173 &attrs, 1);
1174 }
1175 if (release)
1176 put_ep(&ep->com);
1177 return CPL_RET_BUF_DONE;
1178}
1179
1180static void
1181ep_timeout(void *arg)
1182{
1183 struct iwch_ep *ep = (struct iwch_ep *)arg;
1184 struct iwch_qp_attributes attrs;
1185 int err = 0;
1186
1187 mtx_lock(&ep->com.lock);
1188 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1189 switch (ep->com.state) {
1190 case MPA_REQ_SENT:
1191 connect_reply_upcall(ep, -ETIMEDOUT);
1192 break;
1193 case MPA_REQ_WAIT:
1194 break;
1195 case CLOSING:
1196 case MORIBUND:
1197 if (ep->com.cm_id && ep->com.qp)
1198 err = 1;
1199 break;
1200 default:
1201 panic("unknown state: %d\n", ep->com.state);
1202 }
1203 __state_set(&ep->com, ABORTING);
1204 mtx_unlock(&ep->com.lock);
1205 if (err){
1206 attrs.next_state = IWCH_QP_STATE_ERROR;
1207 iwch_modify_qp(ep->com.qp->rhp,
1208 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1209 &attrs, 1);
1210 }
1211 abort_connection(ep);
1212 put_ep(&ep->com);
1213}
1214
1215int
1216iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
1217{
1218 int err;
1219 struct iwch_ep *ep = to_ep(cm_id);
1220 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1221
1222 if (state_read(&ep->com) == DEAD) {
1223 put_ep(&ep->com);
1224 return (-ECONNRESET);
1225 }
1226 PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD);
1227 if (mpa_rev == 0) {
1228 abort_connection(ep);
1229 } else {
1230 err = send_mpa_reject(ep, pdata, pdata_len);
1231 err = soshutdown(ep->com.so, 3);
1232 }
1233 return 0;
1234}
1235
1236int
1237iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1238{
1239 int err;
1240 struct iwch_qp_attributes attrs;
1241 enum iwch_qp_attr_mask mask;
1242 struct iwch_ep *ep = to_ep(cm_id);
1243 struct iwch_dev *h = to_iwch_dev(cm_id->device);
1244 struct iwch_qp *qp = get_qhp(h, conn_param->qpn);
1245
1246 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1247 if (state_read(&ep->com) == DEAD)
1248 return (-ECONNRESET);
1249
1250 PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD);
1251 PANIC_IF(!qp);
1252
1253 if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) ||
1254 (conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) {
1255 abort_connection(ep);
1256 return (-EINVAL);
1257 }
1258
1259 cm_id->add_ref(cm_id);
1260 ep->com.cm_id = cm_id;
1261 ep->com.qp = qp;
1262
1263 ep->com.rpl_err = 0;
1264 ep->com.rpl_done = 0;
1265 ep->ird = conn_param->ird;
1266 ep->ord = conn_param->ord;
1267 CTR3(KTR_IW_CXGB, "%s ird %d ord %d", __FUNCTION__, ep->ird, ep->ord);
1268 get_ep(&ep->com);
1269
1270 /* bind QP to EP and move to RTS */
1271 attrs.mpa_attr = ep->mpa_attr;
1272 attrs.max_ird = ep->ord;
1273 attrs.max_ord = ep->ord;
1274 attrs.llp_stream_handle = ep;
1275 attrs.next_state = IWCH_QP_STATE_RTS;
1276
1277 /* bind QP and TID with INIT_WR */
1278 mask = IWCH_QP_ATTR_NEXT_STATE |
1279 IWCH_QP_ATTR_LLP_STREAM_HANDLE |
1280 IWCH_QP_ATTR_MPA_ATTR |
1281 IWCH_QP_ATTR_MAX_IRD |
1282 IWCH_QP_ATTR_MAX_ORD;
1283
1284 err = iwch_modify_qp(ep->com.qp->rhp,
1285 ep->com.qp, mask, &attrs, 1);
1286
1287 if (err)
1288 goto err;
1289
1290 err = send_mpa_reply(ep, conn_param->private_data,
1291 conn_param->private_data_len);
1292 if (err)
1293 goto err;
1294 state_set(&ep->com, FPDU_MODE);
1295 established_upcall(ep);
1296 put_ep(&ep->com);
1297 return 0;
1298err:
1299 ep->com.cm_id = NULL;
1300 ep->com.qp = NULL;
1301 cm_id->rem_ref(cm_id);
1302 put_ep(&ep->com);
1303 return err;
1304}
1305
1306static int init_sock(struct iwch_ep_common *epc)
1307{
1308 int err;
1309 struct sockopt sopt;
1310 int on=1;
1311
| 1131 CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s ec_status %d", __FUNCTION__, ep, ep->com.so, states[ep->com.state], rep->status);
1132 if (!so || !ep) {
1133 panic("bogosity ep %p state %d, so %p state %x\n", ep, ep ? ep->com.state : -1, so, so ? so->so_state : -1);
1134 }
1135 mtx_lock(&ep->com.lock);
1136 switch (ep->com.state) {
1137 case CLOSING:
1138 if (!rep->status)
1139 __state_set(&ep->com, MORIBUND);
1140 else
1141 __state_set(&ep->com, ABORTING);
1142 break;
1143 case MORIBUND:
1144 stop_ep_timer(ep);
1145 if (!rep->status) {
1146 if ((ep->com.cm_id) && (ep->com.qp)) {
1147 attrs.next_state = IWCH_QP_STATE_IDLE;
1148 iwch_modify_qp(ep->com.qp->rhp,
1149 ep->com.qp,
1150 IWCH_QP_ATTR_NEXT_STATE,
1151 &attrs, 1);
1152 }
1153 close_socket(&ep->com);
1154 close_complete_upcall(ep);
1155 __state_set(&ep->com, DEAD);
1156 release = 1;
1157 }
1158 break;
1159 case DEAD:
1160 break;
1161 default:
1162 panic("unknown state: %d\n", ep->com.state);
1163 }
1164 mtx_unlock(&ep->com.lock);
1165 if (rep->status) {
1166 log(LOG_ERR, "%s BAD CLOSE - Aborting tid %u\n",
1167 __FUNCTION__, ep->hwtid);
1168 attrs.next_state = IWCH_QP_STATE_ERROR;
1169 iwch_modify_qp(ep->com.qp->rhp,
1170 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1171 &attrs, 1);
1172 }
1173 if (release)
1174 put_ep(&ep->com);
1175 return CPL_RET_BUF_DONE;
1176}
1177
1178static void
1179ep_timeout(void *arg)
1180{
1181 struct iwch_ep *ep = (struct iwch_ep *)arg;
1182 struct iwch_qp_attributes attrs;
1183 int err = 0;
1184
1185 mtx_lock(&ep->com.lock);
1186 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1187 switch (ep->com.state) {
1188 case MPA_REQ_SENT:
1189 connect_reply_upcall(ep, -ETIMEDOUT);
1190 break;
1191 case MPA_REQ_WAIT:
1192 break;
1193 case CLOSING:
1194 case MORIBUND:
1195 if (ep->com.cm_id && ep->com.qp)
1196 err = 1;
1197 break;
1198 default:
1199 panic("unknown state: %d\n", ep->com.state);
1200 }
1201 __state_set(&ep->com, ABORTING);
1202 mtx_unlock(&ep->com.lock);
1203 if (err){
1204 attrs.next_state = IWCH_QP_STATE_ERROR;
1205 iwch_modify_qp(ep->com.qp->rhp,
1206 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1207 &attrs, 1);
1208 }
1209 abort_connection(ep);
1210 put_ep(&ep->com);
1211}
1212
1213int
1214iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
1215{
1216 int err;
1217 struct iwch_ep *ep = to_ep(cm_id);
1218 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1219
1220 if (state_read(&ep->com) == DEAD) {
1221 put_ep(&ep->com);
1222 return (-ECONNRESET);
1223 }
1224 PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD);
1225 if (mpa_rev == 0) {
1226 abort_connection(ep);
1227 } else {
1228 err = send_mpa_reject(ep, pdata, pdata_len);
1229 err = soshutdown(ep->com.so, 3);
1230 }
1231 return 0;
1232}
1233
1234int
1235iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1236{
1237 int err;
1238 struct iwch_qp_attributes attrs;
1239 enum iwch_qp_attr_mask mask;
1240 struct iwch_ep *ep = to_ep(cm_id);
1241 struct iwch_dev *h = to_iwch_dev(cm_id->device);
1242 struct iwch_qp *qp = get_qhp(h, conn_param->qpn);
1243
1244 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1245 if (state_read(&ep->com) == DEAD)
1246 return (-ECONNRESET);
1247
1248 PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD);
1249 PANIC_IF(!qp);
1250
1251 if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) ||
1252 (conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) {
1253 abort_connection(ep);
1254 return (-EINVAL);
1255 }
1256
1257 cm_id->add_ref(cm_id);
1258 ep->com.cm_id = cm_id;
1259 ep->com.qp = qp;
1260
1261 ep->com.rpl_err = 0;
1262 ep->com.rpl_done = 0;
1263 ep->ird = conn_param->ird;
1264 ep->ord = conn_param->ord;
1265 CTR3(KTR_IW_CXGB, "%s ird %d ord %d", __FUNCTION__, ep->ird, ep->ord);
1266 get_ep(&ep->com);
1267
1268 /* bind QP to EP and move to RTS */
1269 attrs.mpa_attr = ep->mpa_attr;
1270 attrs.max_ird = ep->ord;
1271 attrs.max_ord = ep->ord;
1272 attrs.llp_stream_handle = ep;
1273 attrs.next_state = IWCH_QP_STATE_RTS;
1274
1275 /* bind QP and TID with INIT_WR */
1276 mask = IWCH_QP_ATTR_NEXT_STATE |
1277 IWCH_QP_ATTR_LLP_STREAM_HANDLE |
1278 IWCH_QP_ATTR_MPA_ATTR |
1279 IWCH_QP_ATTR_MAX_IRD |
1280 IWCH_QP_ATTR_MAX_ORD;
1281
1282 err = iwch_modify_qp(ep->com.qp->rhp,
1283 ep->com.qp, mask, &attrs, 1);
1284
1285 if (err)
1286 goto err;
1287
1288 err = send_mpa_reply(ep, conn_param->private_data,
1289 conn_param->private_data_len);
1290 if (err)
1291 goto err;
1292 state_set(&ep->com, FPDU_MODE);
1293 established_upcall(ep);
1294 put_ep(&ep->com);
1295 return 0;
1296err:
1297 ep->com.cm_id = NULL;
1298 ep->com.qp = NULL;
1299 cm_id->rem_ref(cm_id);
1300 put_ep(&ep->com);
1301 return err;
1302}
1303
1304static int init_sock(struct iwch_ep_common *epc)
1305{
1306 int err;
1307 struct sockopt sopt;
1308 int on=1;
1309
|
1312 epc->so->so_upcall = iwch_so_upcall;
1313 epc->so->so_upcallarg = epc;
1314 epc->so->so_rcv.sb_flags |= SB_UPCALL;
| 1310 SOCK_LOCK(epc->so);
1311 soupcall_set(epc->so, SO_RCV, iwch_so_upcall, epc);
|
1315 epc->so->so_state |= SS_NBIO;
| 1312 epc->so->so_state |= SS_NBIO;
|
| 1313 SOCK_UNLOCK(epc->so);
|
1316 sopt.sopt_dir = SOPT_SET;
1317 sopt.sopt_level = SOL_SOCKET;
1318 sopt.sopt_name = SO_NO_DDP;
1319 sopt.sopt_val = (caddr_t)&on;
1320 sopt.sopt_valsize = sizeof on;
1321 sopt.sopt_td = NULL;
1322 err = sosetopt(epc->so, &sopt);
1323 if (err)
1324 printf("%s can't set SO_NO_DDP err %d\n", __FUNCTION__, err);
1325 sopt.sopt_dir = SOPT_SET;
1326 sopt.sopt_level = IPPROTO_TCP;
1327 sopt.sopt_name = TCP_NODELAY;
1328 sopt.sopt_val = (caddr_t)&on;
1329 sopt.sopt_valsize = sizeof on;
1330 sopt.sopt_td = NULL;
1331 err = sosetopt(epc->so, &sopt);
1332 if (err)
1333 printf("%s can't set TCP_NODELAY err %d\n", __FUNCTION__, err);
1334
1335 return 0;
1336}
1337
1338static int
1339is_loopback_dst(struct iw_cm_id *cm_id)
1340{
1341 uint16_t port = cm_id->remote_addr.sin_port;
1342 struct ifaddr *ifa;
1343
1344 cm_id->remote_addr.sin_port = 0;
1345 ifa = ifa_ifwithaddr((struct sockaddr *)&cm_id->remote_addr);
1346 cm_id->remote_addr.sin_port = port;
1347 return (ifa != NULL);
1348}
1349
1350int
1351iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1352{
1353 int err = 0;
1354 struct iwch_dev *h = to_iwch_dev(cm_id->device);
1355 struct iwch_ep *ep;
1356 struct rtentry *rt;
1357 struct toedev *tdev;
1358
1359 if (is_loopback_dst(cm_id)) {
1360 err = -ENOSYS;
1361 goto out;
1362 }
1363
1364 ep = alloc_ep(sizeof(*ep), M_NOWAIT);
1365 if (!ep) {
1366 printf("%s - cannot alloc ep.\n", __FUNCTION__);
1367 err = (-ENOMEM);
1368 goto out;
1369 }
1370 callout_init(&ep->timer, TRUE);
1371 ep->plen = conn_param->private_data_len;
1372 if (ep->plen)
1373 memcpy(ep->mpa_pkt + sizeof(struct mpa_message),
1374 conn_param->private_data, ep->plen);
1375 ep->ird = conn_param->ird;
1376 ep->ord = conn_param->ord;
1377
1378 cm_id->add_ref(cm_id);
1379 ep->com.cm_id = cm_id;
1380 ep->com.qp = get_qhp(h, conn_param->qpn);
1381 ep->com.thread = curthread;
1382 PANIC_IF(!ep->com.qp);
1383 CTR4(KTR_IW_CXGB, "%s qpn 0x%x qp %p cm_id %p", __FUNCTION__, conn_param->qpn,
1384 ep->com.qp, cm_id);
1385
1386 ep->com.so = cm_id->so;
1387 err = init_sock(&ep->com);
1388 if (err)
1389 goto fail2;
1390
1391 /* find a route */
1392 rt = find_route(cm_id->local_addr.sin_addr.s_addr,
1393 cm_id->remote_addr.sin_addr.s_addr,
1394 cm_id->local_addr.sin_port,
1395 cm_id->remote_addr.sin_port, IPTOS_LOWDELAY);
1396 if (!rt) {
1397 printf("%s - cannot find route.\n", __FUNCTION__);
1398 err = EHOSTUNREACH;
1399 goto fail2;
1400 }
1401
1402 if (!(rt->rt_ifp->if_flags & IFCAP_TOE)) {
1403 printf("%s - interface not TOE capable.\n", __FUNCTION__);
1404 goto fail3;
1405 }
1406 tdev = TOEDEV(rt->rt_ifp);
1407 if (tdev == NULL) {
1408 printf("%s - No toedev for interface.\n", __FUNCTION__);
1409 goto fail3;
1410 }
1411 if (!tdev->tod_can_offload(tdev, ep->com.so)) {
1412 printf("%s - interface cannot offload!.\n", __FUNCTION__);
1413 goto fail3;
1414 }
1415 RTFREE(rt);
1416
1417 state_set(&ep->com, CONNECTING);
1418 ep->com.local_addr = cm_id->local_addr;
1419 ep->com.remote_addr = cm_id->remote_addr;
1420 err = soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr,
1421 ep->com.thread);
1422 if (!err)
1423 goto out;
1424fail3:
1425 RTFREE(ep->dst);
1426fail2:
1427 put_ep(&ep->com);
1428out:
1429 return err;
1430}
1431
1432int
1433iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
1434{
1435 int err = 0;
1436 struct iwch_listen_ep *ep;
1437
1438 ep = alloc_ep(sizeof(*ep), M_NOWAIT);
1439 if (!ep) {
1440 printf("%s - cannot alloc ep.\n", __FUNCTION__);
1441 err = ENOMEM;
1442 goto out;
1443 }
1444 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
1445 cm_id->add_ref(cm_id);
1446 ep->com.cm_id = cm_id;
1447 ep->backlog = backlog;
1448 ep->com.local_addr = cm_id->local_addr;
1449 ep->com.thread = curthread;
1450 state_set(&ep->com, LISTEN);
1451
1452 ep->com.so = cm_id->so;
1453 err = init_sock(&ep->com);
1454 if (err)
1455 goto fail;
1456
1457 err = solisten(ep->com.so, ep->backlog, ep->com.thread);
1458 if (!err) {
1459 cm_id->provider_data = ep;
1460 goto out;
1461 }
1462 close_socket(&ep->com);
1463fail:
1464 cm_id->rem_ref(cm_id);
1465 put_ep(&ep->com);
1466out:
1467 return err;
1468}
1469
1470int
1471iwch_destroy_listen(struct iw_cm_id *cm_id)
1472{
1473 struct iwch_listen_ep *ep = to_listen_ep(cm_id);
1474
1475 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
1476
1477 state_set(&ep->com, DEAD);
1478 close_socket(&ep->com);
1479 cm_id->rem_ref(cm_id);
1480 put_ep(&ep->com);
1481 return 0;
1482}
1483
1484int
1485iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, int flags)
1486{
1487 int close = 0;
1488
1489 mtx_lock(&ep->com.lock);
1490
1491 PANIC_IF(!ep);
1492 PANIC_IF(!ep->com.so);
1493
1494 CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s, abrupt %d", __FUNCTION__, ep,
1495 ep->com.so, states[ep->com.state], abrupt);
1496
1497 if (ep->com.state == DEAD) {
1498 CTR2(KTR_IW_CXGB, "%s already dead ep %p", __FUNCTION__, ep);
1499 goto out;
1500 }
1501
1502 if (abrupt) {
1503 if (ep->com.state != ABORTING) {
1504 ep->com.state = ABORTING;
1505 close = 1;
1506 }
1507 goto out;
1508 }
1509
1510 switch (ep->com.state) {
1511 case MPA_REQ_WAIT:
1512 case MPA_REQ_SENT:
1513 case MPA_REQ_RCVD:
1514 case MPA_REP_SENT:
1515 case FPDU_MODE:
1516 start_ep_timer(ep);
1517 ep->com.state = CLOSING;
1518 close = 1;
1519 break;
1520 case CLOSING:
1521 ep->com.state = MORIBUND;
1522 close = 1;
1523 break;
1524 case MORIBUND:
1525 case ABORTING:
1526 break;
1527 default:
1528 panic("unknown state: %d\n", ep->com.state);
1529 break;
1530 }
1531out:
1532 mtx_unlock(&ep->com.lock);
1533 if (close) {
1534 if (abrupt)
1535 abort_connection(ep);
1536 else
1537 shutdown_socket(&ep->com);
1538 }
1539 return 0;
1540}
1541
1542static void
1543process_data(struct iwch_ep *ep)
1544{
1545 struct sockaddr_in *local, *remote;
1546
1547 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1548
1549 switch (state_read(&ep->com)) {
1550 case MPA_REQ_SENT:
1551 process_mpa_reply(ep);
1552 break;
1553 case MPA_REQ_WAIT:
1554
1555 /*
1556 * XXX
1557 * Set local and remote addrs here because when we
1558 * dequeue the newly accepted socket, they aren't set
1559 * yet in the pcb!
1560 */
1561 in_getsockaddr(ep->com.so, (struct sockaddr **)&local);
1562 in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote);
1563 CTR3(KTR_IW_CXGB, "%s local %s remote %s", __FUNCTION__,
1564 inet_ntoa(local->sin_addr),
1565 inet_ntoa(remote->sin_addr));
1566 ep->com.local_addr = *local;
1567 ep->com.remote_addr = *remote;
1568 free(local, M_SONAME);
1569 free(remote, M_SONAME);
1570 process_mpa_request(ep);
1571 break;
1572 default:
1573 if (ep->com.so->so_rcv.sb_cc)
1574 printf("%s Unexpected streaming data."
1575 " ep %p state %d so %p so_state %x so_rcv.sb_cc %u so_rcv.sb_mb %p\n",
1576 __FUNCTION__, ep, state_read(&ep->com), ep->com.so, ep->com.so->so_state,
1577 ep->com.so->so_rcv.sb_cc, ep->com.so->so_rcv.sb_mb);
1578 break;
1579 }
1580 return;
1581}
1582
1583static void
1584process_connected(struct iwch_ep *ep)
1585{
1586 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1587 if ((ep->com.so->so_state & SS_ISCONNECTED) && !ep->com.so->so_error) {
1588 send_mpa_req(ep);
1589 } else {
1590 connect_reply_upcall(ep, -ep->com.so->so_error);
1591 close_socket(&ep->com);
1592 state_set(&ep->com, DEAD);
1593 put_ep(&ep->com);
1594 }
1595}
1596
1597static struct socket *
1598dequeue_socket(struct socket *head, struct sockaddr_in **remote, struct iwch_ep *child_ep)
1599{
1600 struct socket *so;
1601
1602 ACCEPT_LOCK();
1603 so = TAILQ_FIRST(&head->so_comp);
1604 if (!so) {
1605 ACCEPT_UNLOCK();
1606 return NULL;
1607 }
1608 TAILQ_REMOVE(&head->so_comp, so, so_list);
1609 head->so_qlen--;
1610 SOCK_LOCK(so);
1611 so->so_qstate &= ~SQ_COMP;
1612 so->so_head = NULL;
1613 soref(so);
| 1314 sopt.sopt_dir = SOPT_SET;
1315 sopt.sopt_level = SOL_SOCKET;
1316 sopt.sopt_name = SO_NO_DDP;
1317 sopt.sopt_val = (caddr_t)&on;
1318 sopt.sopt_valsize = sizeof on;
1319 sopt.sopt_td = NULL;
1320 err = sosetopt(epc->so, &sopt);
1321 if (err)
1322 printf("%s can't set SO_NO_DDP err %d\n", __FUNCTION__, err);
1323 sopt.sopt_dir = SOPT_SET;
1324 sopt.sopt_level = IPPROTO_TCP;
1325 sopt.sopt_name = TCP_NODELAY;
1326 sopt.sopt_val = (caddr_t)&on;
1327 sopt.sopt_valsize = sizeof on;
1328 sopt.sopt_td = NULL;
1329 err = sosetopt(epc->so, &sopt);
1330 if (err)
1331 printf("%s can't set TCP_NODELAY err %d\n", __FUNCTION__, err);
1332
1333 return 0;
1334}
1335
1336static int
1337is_loopback_dst(struct iw_cm_id *cm_id)
1338{
1339 uint16_t port = cm_id->remote_addr.sin_port;
1340 struct ifaddr *ifa;
1341
1342 cm_id->remote_addr.sin_port = 0;
1343 ifa = ifa_ifwithaddr((struct sockaddr *)&cm_id->remote_addr);
1344 cm_id->remote_addr.sin_port = port;
1345 return (ifa != NULL);
1346}
1347
1348int
1349iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1350{
1351 int err = 0;
1352 struct iwch_dev *h = to_iwch_dev(cm_id->device);
1353 struct iwch_ep *ep;
1354 struct rtentry *rt;
1355 struct toedev *tdev;
1356
1357 if (is_loopback_dst(cm_id)) {
1358 err = -ENOSYS;
1359 goto out;
1360 }
1361
1362 ep = alloc_ep(sizeof(*ep), M_NOWAIT);
1363 if (!ep) {
1364 printf("%s - cannot alloc ep.\n", __FUNCTION__);
1365 err = (-ENOMEM);
1366 goto out;
1367 }
1368 callout_init(&ep->timer, TRUE);
1369 ep->plen = conn_param->private_data_len;
1370 if (ep->plen)
1371 memcpy(ep->mpa_pkt + sizeof(struct mpa_message),
1372 conn_param->private_data, ep->plen);
1373 ep->ird = conn_param->ird;
1374 ep->ord = conn_param->ord;
1375
1376 cm_id->add_ref(cm_id);
1377 ep->com.cm_id = cm_id;
1378 ep->com.qp = get_qhp(h, conn_param->qpn);
1379 ep->com.thread = curthread;
1380 PANIC_IF(!ep->com.qp);
1381 CTR4(KTR_IW_CXGB, "%s qpn 0x%x qp %p cm_id %p", __FUNCTION__, conn_param->qpn,
1382 ep->com.qp, cm_id);
1383
1384 ep->com.so = cm_id->so;
1385 err = init_sock(&ep->com);
1386 if (err)
1387 goto fail2;
1388
1389 /* find a route */
1390 rt = find_route(cm_id->local_addr.sin_addr.s_addr,
1391 cm_id->remote_addr.sin_addr.s_addr,
1392 cm_id->local_addr.sin_port,
1393 cm_id->remote_addr.sin_port, IPTOS_LOWDELAY);
1394 if (!rt) {
1395 printf("%s - cannot find route.\n", __FUNCTION__);
1396 err = EHOSTUNREACH;
1397 goto fail2;
1398 }
1399
1400 if (!(rt->rt_ifp->if_flags & IFCAP_TOE)) {
1401 printf("%s - interface not TOE capable.\n", __FUNCTION__);
1402 goto fail3;
1403 }
1404 tdev = TOEDEV(rt->rt_ifp);
1405 if (tdev == NULL) {
1406 printf("%s - No toedev for interface.\n", __FUNCTION__);
1407 goto fail3;
1408 }
1409 if (!tdev->tod_can_offload(tdev, ep->com.so)) {
1410 printf("%s - interface cannot offload!.\n", __FUNCTION__);
1411 goto fail3;
1412 }
1413 RTFREE(rt);
1414
1415 state_set(&ep->com, CONNECTING);
1416 ep->com.local_addr = cm_id->local_addr;
1417 ep->com.remote_addr = cm_id->remote_addr;
1418 err = soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr,
1419 ep->com.thread);
1420 if (!err)
1421 goto out;
1422fail3:
1423 RTFREE(ep->dst);
1424fail2:
1425 put_ep(&ep->com);
1426out:
1427 return err;
1428}
1429
1430int
1431iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
1432{
1433 int err = 0;
1434 struct iwch_listen_ep *ep;
1435
1436 ep = alloc_ep(sizeof(*ep), M_NOWAIT);
1437 if (!ep) {
1438 printf("%s - cannot alloc ep.\n", __FUNCTION__);
1439 err = ENOMEM;
1440 goto out;
1441 }
1442 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
1443 cm_id->add_ref(cm_id);
1444 ep->com.cm_id = cm_id;
1445 ep->backlog = backlog;
1446 ep->com.local_addr = cm_id->local_addr;
1447 ep->com.thread = curthread;
1448 state_set(&ep->com, LISTEN);
1449
1450 ep->com.so = cm_id->so;
1451 err = init_sock(&ep->com);
1452 if (err)
1453 goto fail;
1454
1455 err = solisten(ep->com.so, ep->backlog, ep->com.thread);
1456 if (!err) {
1457 cm_id->provider_data = ep;
1458 goto out;
1459 }
1460 close_socket(&ep->com);
1461fail:
1462 cm_id->rem_ref(cm_id);
1463 put_ep(&ep->com);
1464out:
1465 return err;
1466}
1467
1468int
1469iwch_destroy_listen(struct iw_cm_id *cm_id)
1470{
1471 struct iwch_listen_ep *ep = to_listen_ep(cm_id);
1472
1473 CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
1474
1475 state_set(&ep->com, DEAD);
1476 close_socket(&ep->com);
1477 cm_id->rem_ref(cm_id);
1478 put_ep(&ep->com);
1479 return 0;
1480}
1481
1482int
1483iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, int flags)
1484{
1485 int close = 0;
1486
1487 mtx_lock(&ep->com.lock);
1488
1489 PANIC_IF(!ep);
1490 PANIC_IF(!ep->com.so);
1491
1492 CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s, abrupt %d", __FUNCTION__, ep,
1493 ep->com.so, states[ep->com.state], abrupt);
1494
1495 if (ep->com.state == DEAD) {
1496 CTR2(KTR_IW_CXGB, "%s already dead ep %p", __FUNCTION__, ep);
1497 goto out;
1498 }
1499
1500 if (abrupt) {
1501 if (ep->com.state != ABORTING) {
1502 ep->com.state = ABORTING;
1503 close = 1;
1504 }
1505 goto out;
1506 }
1507
1508 switch (ep->com.state) {
1509 case MPA_REQ_WAIT:
1510 case MPA_REQ_SENT:
1511 case MPA_REQ_RCVD:
1512 case MPA_REP_SENT:
1513 case FPDU_MODE:
1514 start_ep_timer(ep);
1515 ep->com.state = CLOSING;
1516 close = 1;
1517 break;
1518 case CLOSING:
1519 ep->com.state = MORIBUND;
1520 close = 1;
1521 break;
1522 case MORIBUND:
1523 case ABORTING:
1524 break;
1525 default:
1526 panic("unknown state: %d\n", ep->com.state);
1527 break;
1528 }
1529out:
1530 mtx_unlock(&ep->com.lock);
1531 if (close) {
1532 if (abrupt)
1533 abort_connection(ep);
1534 else
1535 shutdown_socket(&ep->com);
1536 }
1537 return 0;
1538}
1539
1540static void
1541process_data(struct iwch_ep *ep)
1542{
1543 struct sockaddr_in *local, *remote;
1544
1545 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1546
1547 switch (state_read(&ep->com)) {
1548 case MPA_REQ_SENT:
1549 process_mpa_reply(ep);
1550 break;
1551 case MPA_REQ_WAIT:
1552
1553 /*
1554 * XXX
1555 * Set local and remote addrs here because when we
1556 * dequeue the newly accepted socket, they aren't set
1557 * yet in the pcb!
1558 */
1559 in_getsockaddr(ep->com.so, (struct sockaddr **)&local);
1560 in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote);
1561 CTR3(KTR_IW_CXGB, "%s local %s remote %s", __FUNCTION__,
1562 inet_ntoa(local->sin_addr),
1563 inet_ntoa(remote->sin_addr));
1564 ep->com.local_addr = *local;
1565 ep->com.remote_addr = *remote;
1566 free(local, M_SONAME);
1567 free(remote, M_SONAME);
1568 process_mpa_request(ep);
1569 break;
1570 default:
1571 if (ep->com.so->so_rcv.sb_cc)
1572 printf("%s Unexpected streaming data."
1573 " ep %p state %d so %p so_state %x so_rcv.sb_cc %u so_rcv.sb_mb %p\n",
1574 __FUNCTION__, ep, state_read(&ep->com), ep->com.so, ep->com.so->so_state,
1575 ep->com.so->so_rcv.sb_cc, ep->com.so->so_rcv.sb_mb);
1576 break;
1577 }
1578 return;
1579}
1580
1581static void
1582process_connected(struct iwch_ep *ep)
1583{
1584 CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
1585 if ((ep->com.so->so_state & SS_ISCONNECTED) && !ep->com.so->so_error) {
1586 send_mpa_req(ep);
1587 } else {
1588 connect_reply_upcall(ep, -ep->com.so->so_error);
1589 close_socket(&ep->com);
1590 state_set(&ep->com, DEAD);
1591 put_ep(&ep->com);
1592 }
1593}
1594
1595static struct socket *
1596dequeue_socket(struct socket *head, struct sockaddr_in **remote, struct iwch_ep *child_ep)
1597{
1598 struct socket *so;
1599
1600 ACCEPT_LOCK();
1601 so = TAILQ_FIRST(&head->so_comp);
1602 if (!so) {
1603 ACCEPT_UNLOCK();
1604 return NULL;
1605 }
1606 TAILQ_REMOVE(&head->so_comp, so, so_list);
1607 head->so_qlen--;
1608 SOCK_LOCK(so);
1609 so->so_qstate &= ~SQ_COMP;
1610 so->so_head = NULL;
1611 soref(so);
|
1614 so->so_rcv.sb_flags |= SB_UPCALL;
| 1612 soupcall_set(so, SO_RCV, iwch_so_upcall, child_ep);
|
1615 so->so_state |= SS_NBIO;
| 1613 so->so_state |= SS_NBIO;
|
1616 so->so_upcall = iwch_so_upcall;
1617 so->so_upcallarg = child_ep;
| |
1618 PANIC_IF(!(so->so_state & SS_ISCONNECTED));
1619 PANIC_IF(so->so_error);
1620 SOCK_UNLOCK(so);
1621 ACCEPT_UNLOCK();
1622 soaccept(so, (struct sockaddr **)remote);
1623 return so;
1624}
1625
1626static void
1627process_newconn(struct iwch_ep *parent_ep)
1628{
1629 struct socket *child_so;
1630 struct iwch_ep *child_ep;
1631 struct sockaddr_in *remote;
1632
1633 CTR3(KTR_IW_CXGB, "%s parent ep %p so %p", __FUNCTION__, parent_ep, parent_ep->com.so);
1634 child_ep = alloc_ep(sizeof(*child_ep), M_NOWAIT);
1635 if (!child_ep) {
1636 log(LOG_ERR, "%s - failed to allocate ep entry!\n",
1637 __FUNCTION__);
1638 return;
1639 }
1640 child_so = dequeue_socket(parent_ep->com.so, &remote, child_ep);
1641 if (!child_so) {
1642 log(LOG_ERR, "%s - failed to dequeue child socket!\n",
1643 __FUNCTION__);
1644 __free_ep(&child_ep->com);
1645 return;
1646 }
1647 CTR3(KTR_IW_CXGB, "%s remote addr %s port %d", __FUNCTION__,
1648 inet_ntoa(remote->sin_addr), ntohs(remote->sin_port));
1649 child_ep->com.so = child_so;
1650 child_ep->com.cm_id = NULL;
1651 child_ep->com.thread = parent_ep->com.thread;
1652 child_ep->parent_ep = parent_ep;
1653 free(remote, M_SONAME);
1654 get_ep(&parent_ep->com);
1655 child_ep->parent_ep = parent_ep;
1656 callout_init(&child_ep->timer, TRUE);
1657 state_set(&child_ep->com, MPA_REQ_WAIT);
1658 start_ep_timer(child_ep);
1659
1660 /* maybe the request has already been queued up on the socket... */
1661 process_mpa_request(child_ep);
1662}
1663
| 1614 PANIC_IF(!(so->so_state & SS_ISCONNECTED));
1615 PANIC_IF(so->so_error);
1616 SOCK_UNLOCK(so);
1617 ACCEPT_UNLOCK();
1618 soaccept(so, (struct sockaddr **)remote);
1619 return so;
1620}
1621
1622static void
1623process_newconn(struct iwch_ep *parent_ep)
1624{
1625 struct socket *child_so;
1626 struct iwch_ep *child_ep;
1627 struct sockaddr_in *remote;
1628
1629 CTR3(KTR_IW_CXGB, "%s parent ep %p so %p", __FUNCTION__, parent_ep, parent_ep->com.so);
1630 child_ep = alloc_ep(sizeof(*child_ep), M_NOWAIT);
1631 if (!child_ep) {
1632 log(LOG_ERR, "%s - failed to allocate ep entry!\n",
1633 __FUNCTION__);
1634 return;
1635 }
1636 child_so = dequeue_socket(parent_ep->com.so, &remote, child_ep);
1637 if (!child_so) {
1638 log(LOG_ERR, "%s - failed to dequeue child socket!\n",
1639 __FUNCTION__);
1640 __free_ep(&child_ep->com);
1641 return;
1642 }
1643 CTR3(KTR_IW_CXGB, "%s remote addr %s port %d", __FUNCTION__,
1644 inet_ntoa(remote->sin_addr), ntohs(remote->sin_port));
1645 child_ep->com.so = child_so;
1646 child_ep->com.cm_id = NULL;
1647 child_ep->com.thread = parent_ep->com.thread;
1648 child_ep->parent_ep = parent_ep;
1649 free(remote, M_SONAME);
1650 get_ep(&parent_ep->com);
1651 child_ep->parent_ep = parent_ep;
1652 callout_init(&child_ep->timer, TRUE);
1653 state_set(&child_ep->com, MPA_REQ_WAIT);
1654 start_ep_timer(child_ep);
1655
1656 /* maybe the request has already been queued up on the socket... */
1657 process_mpa_request(child_ep);
1658}
1659
|
1664static void
| 1660static int
|
1665iwch_so_upcall(struct socket *so, void *arg, int waitflag)
1666{
1667 struct iwch_ep *ep = arg;
1668
1669 CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]);
1670 mtx_lock(&req_lock);
1671 if (ep && ep->com.so && !ep->com.entry.tqe_prev) {
1672 get_ep(&ep->com);
1673 TAILQ_INSERT_TAIL(&req_list, &ep->com, entry);
1674 taskqueue_enqueue(iw_cxgb_taskq, &iw_cxgb_task);
1675 }
1676 mtx_unlock(&req_lock);
| 1661iwch_so_upcall(struct socket *so, void *arg, int waitflag)
1662{
1663 struct iwch_ep *ep = arg;
1664
1665 CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]);
1666 mtx_lock(&req_lock);
1667 if (ep && ep->com.so && !ep->com.entry.tqe_prev) {
1668 get_ep(&ep->com);
1669 TAILQ_INSERT_TAIL(&req_list, &ep->com, entry);
1670 taskqueue_enqueue(iw_cxgb_taskq, &iw_cxgb_task);
1671 }
1672 mtx_unlock(&req_lock);
|
| 1673 return (SU_OK);
|
1677}
1678
1679static void
1680process_socket_event(struct iwch_ep *ep)
1681{
1682 int state = state_read(&ep->com);
1683 struct socket *so = ep->com.so;
1684
1685 CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]);
1686 if (state == CONNECTING) {
1687 process_connected(ep);
1688 return;
1689 }
1690
1691 if (state == LISTEN) {
1692 process_newconn(ep);
1693 return;
1694 }
1695
1696 /* connection error */
1697 if (so->so_error) {
1698 process_conn_error(ep);
1699 return;
1700 }
1701
1702 /* peer close */
1703 if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state < CLOSING) {
1704 process_peer_close(ep);
1705 return;
1706 }
1707
1708 /* close complete */
1709 if (so->so_state & (SS_ISDISCONNECTED)) {
1710 process_close_complete(ep);
1711 return;
1712 }
1713
1714 /* rx data */
1715 process_data(ep);
1716 return;
1717}
1718
1719static void
1720process_req(void *ctx, int pending)
1721{
1722 struct iwch_ep_common *epc;
1723
1724 CTR1(KTR_IW_CXGB, "%s enter", __FUNCTION__);
1725 mtx_lock(&req_lock);
1726 while (!TAILQ_EMPTY(&req_list)) {
1727 epc = TAILQ_FIRST(&req_list);
1728 TAILQ_REMOVE(&req_list, epc, entry);
1729 epc->entry.tqe_prev = NULL;
1730 mtx_unlock(&req_lock);
1731 if (epc->so)
1732 process_socket_event((struct iwch_ep *)epc);
1733 put_ep(epc);
1734 mtx_lock(&req_lock);
1735 }
1736 mtx_unlock(&req_lock);
1737}
1738
1739int
1740iwch_cm_init(void)
1741{
1742 TAILQ_INIT(&req_list);
1743 mtx_init(&req_lock, "iw_cxgb req_list lock", NULL, MTX_DEF);
1744 iw_cxgb_taskq = taskqueue_create("iw_cxgb_taskq", M_NOWAIT,
1745 taskqueue_thread_enqueue, &iw_cxgb_taskq);
1746 if (iw_cxgb_taskq == NULL) {
1747 printf("failed to allocate iw_cxgb taskqueue\n");
1748 return (ENOMEM);
1749 }
1750 taskqueue_start_threads(&iw_cxgb_taskq, 1, PI_NET, "iw_cxgb taskq");
1751 TASK_INIT(&iw_cxgb_task, 0, process_req, NULL);
1752 t3tom_register_cpl_handler(CPL_RDMA_TERMINATE, terminate);
1753 t3tom_register_cpl_handler(CPL_RDMA_EC_STATUS, ec_status);
1754 return 0;
1755}
1756
1757void
1758iwch_cm_term(void)
1759{
1760 t3tom_register_cpl_handler(CPL_RDMA_TERMINATE, NULL);
1761 t3tom_register_cpl_handler(CPL_RDMA_EC_STATUS, NULL);
1762 taskqueue_drain(iw_cxgb_taskq, &iw_cxgb_task);
1763 taskqueue_free(iw_cxgb_taskq);
1764}
1765
| 1674}
1675
1676static void
1677process_socket_event(struct iwch_ep *ep)
1678{
1679 int state = state_read(&ep->com);
1680 struct socket *so = ep->com.so;
1681
1682 CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]);
1683 if (state == CONNECTING) {
1684 process_connected(ep);
1685 return;
1686 }
1687
1688 if (state == LISTEN) {
1689 process_newconn(ep);
1690 return;
1691 }
1692
1693 /* connection error */
1694 if (so->so_error) {
1695 process_conn_error(ep);
1696 return;
1697 }
1698
1699 /* peer close */
1700 if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state < CLOSING) {
1701 process_peer_close(ep);
1702 return;
1703 }
1704
1705 /* close complete */
1706 if (so->so_state & (SS_ISDISCONNECTED)) {
1707 process_close_complete(ep);
1708 return;
1709 }
1710
1711 /* rx data */
1712 process_data(ep);
1713 return;
1714}
1715
1716static void
1717process_req(void *ctx, int pending)
1718{
1719 struct iwch_ep_common *epc;
1720
1721 CTR1(KTR_IW_CXGB, "%s enter", __FUNCTION__);
1722 mtx_lock(&req_lock);
1723 while (!TAILQ_EMPTY(&req_list)) {
1724 epc = TAILQ_FIRST(&req_list);
1725 TAILQ_REMOVE(&req_list, epc, entry);
1726 epc->entry.tqe_prev = NULL;
1727 mtx_unlock(&req_lock);
1728 if (epc->so)
1729 process_socket_event((struct iwch_ep *)epc);
1730 put_ep(epc);
1731 mtx_lock(&req_lock);
1732 }
1733 mtx_unlock(&req_lock);
1734}
1735
1736int
1737iwch_cm_init(void)
1738{
1739 TAILQ_INIT(&req_list);
1740 mtx_init(&req_lock, "iw_cxgb req_list lock", NULL, MTX_DEF);
1741 iw_cxgb_taskq = taskqueue_create("iw_cxgb_taskq", M_NOWAIT,
1742 taskqueue_thread_enqueue, &iw_cxgb_taskq);
1743 if (iw_cxgb_taskq == NULL) {
1744 printf("failed to allocate iw_cxgb taskqueue\n");
1745 return (ENOMEM);
1746 }
1747 taskqueue_start_threads(&iw_cxgb_taskq, 1, PI_NET, "iw_cxgb taskq");
1748 TASK_INIT(&iw_cxgb_task, 0, process_req, NULL);
1749 t3tom_register_cpl_handler(CPL_RDMA_TERMINATE, terminate);
1750 t3tom_register_cpl_handler(CPL_RDMA_EC_STATUS, ec_status);
1751 return 0;
1752}
1753
1754void
1755iwch_cm_term(void)
1756{
1757 t3tom_register_cpl_handler(CPL_RDMA_TERMINATE, NULL);
1758 t3tom_register_cpl_handler(CPL_RDMA_EC_STATUS, NULL);
1759 taskqueue_drain(iw_cxgb_taskq, &iw_cxgb_task);
1760 taskqueue_free(iw_cxgb_taskq);
1761}
1762
|