1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_tom.c 176472 2008-02-23 01:06:17Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/fcntl.h>
37#include <sys/ktr.h>
38#include <sys/limits.h>
39#include <sys/lock.h>
40#include <sys/eventhandler.h>
41#include <sys/mbuf.h>
42#include <sys/module.h>
43#include <sys/condvar.h>
44#include <sys/mutex.h>
45#include <sys/socket.h>
46#include <sys/sysctl.h>
47#include <sys/syslog.h>
48#include <sys/socketvar.h>
49#include <sys/taskqueue.h>
50
51#include <net/if.h>
52#include <net/route.h>
53
54#include <netinet/in.h>
55#include <netinet/in_pcb.h>
56#include <netinet/in_systm.h>
57#include <netinet/in_var.h>
58
59#include <dev/cxgb/cxgb_osdep.h>
60#include <dev/cxgb/sys/mbufq.h>
61
62#include <netinet/in_pcb.h>
63#include <netinet/tcp.h>
64#include <netinet/tcp_var.h>
65#include <netinet/tcp_offload.h>
66#include <netinet/tcp_fsm.h>
67#include <net/route.h>
68
69#include <dev/cxgb/t3cdev.h>
70#include <dev/cxgb/common/cxgb_firmware_exports.h>
71#include <dev/cxgb/common/cxgb_tcb.h>
72#include <dev/cxgb/cxgb_include.h>
73#include <dev/cxgb/common/cxgb_ctl_defs.h>
74#include <dev/cxgb/common/cxgb_t3_cpl.h>
75#include <dev/cxgb/cxgb_offload.h>
76#include <dev/cxgb/cxgb_l2t.h>
77#include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
78#include <dev/cxgb/ulp/tom/cxgb_tom.h>
79#include <dev/cxgb/ulp/tom/cxgb_defs.h>
80#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h>
81#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
82#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
83
84
85static int activated = 1;
86TUNABLE_INT("hw.t3toe.activated", &activated);
87SYSCTL_NODE(_hw, OID_AUTO, t3toe, CTLFLAG_RD, 0, "T3 toe driver parameters");
88SYSCTL_UINT(_hw_t3toe, OID_AUTO, activated, CTLFLAG_RDTUN, &activated, 0,
89 "enable TOE at init time");
90
91static TAILQ_HEAD(, tom_data) cxgb_list;
92static struct mtx cxgb_list_lock;
93
94static int t3_toe_attach(struct toedev *dev, const struct offload_id *entry);
95static void cxgb_register_listeners(void);
96
97/*
98 * Handlers for each CPL opcode
99 */
100static cxgb_cpl_handler_func tom_cpl_handlers[256];
101
102
103static eventhandler_tag listen_tag;
104
105static struct offload_id t3_toe_id_tab[] = {
106 { TOE_ID_CHELSIO_T3, 0 },
107 { TOE_ID_CHELSIO_T3B, 0 },
108 { TOE_ID_CHELSIO_T3C, 0 },
109 { 0 }
110};
111
112static struct tom_info t3_tom_info = {
113 .ti_attach = t3_toe_attach,
114 .ti_id_table = t3_toe_id_tab,
115 .ti_name = "Chelsio-T3"
116};
117
118struct cxgb_client t3c_tom_client = {
119 .name = "tom_cxgb3",
120 .remove = NULL,
121 .handlers = tom_cpl_handlers,
122 .redirect = NULL
123};
124
125/*
126 * Add an skb to the deferred skb queue for processing from process context.
127 */
128void
129t3_defer_reply(struct mbuf *m, struct toedev *dev, defer_handler_t handler)
130{
131 struct tom_data *td = TOM_DATA(dev);
132
133 m_set_handler(m, handler);
134 mtx_lock(&td->deferq.lock);
135
136 mbufq_tail(&td->deferq, m);
137 if (mbufq_len(&td->deferq) == 1)
138 taskqueue_enqueue(td->tq, &td->deferq_task);
139 mtx_lock(&td->deferq.lock);
140}
141
142struct toepcb *
143toepcb_alloc(void)
144{
145 struct toepcb *toep;
146
147 toep = malloc(sizeof(struct toepcb), M_DEVBUF, M_NOWAIT|M_ZERO);
148
149 if (toep == NULL)
150 return (NULL);
151
152 toepcb_init(toep);
153 return (toep);
154}
155
156void
157toepcb_init(struct toepcb *toep)
158{
159 toep->tp_refcount = 1;
160 cv_init(&toep->tp_cv, "toep cv");
161}
162
163void
164toepcb_hold(struct toepcb *toep)
165{
166 atomic_add_acq_int(&toep->tp_refcount, 1);
167}
168
169void
170toepcb_release(struct toepcb *toep)
171{
172 if (toep->tp_refcount == 1) {
173 free(toep, M_DEVBUF);
174 return;
175 }
176 atomic_add_acq_int(&toep->tp_refcount, -1);
177}
178
179/*
180 * Add a T3 offload device to the list of devices we are managing.
181 */
182static void
183t3cdev_add(struct tom_data *t)
184{
185 mtx_lock(&cxgb_list_lock);
186 TAILQ_INSERT_TAIL(&cxgb_list, t, entry);
187 mtx_unlock(&cxgb_list_lock);
188}
189
190static inline int
191cdev2type(struct t3cdev *cdev)
192{
193 int type = 0;
194
195 switch (cdev->type) {
196 case T3A:
197 type = TOE_ID_CHELSIO_T3;
198 break;
199 case T3B:
200 type = TOE_ID_CHELSIO_T3B;
201 break;
202 case T3C:
203 type = TOE_ID_CHELSIO_T3C;
204 break;
205 }
206 return (type);
207}
208
209/*
210 * Allocate a TOM data structure,
211 * initialize its cpl_handlers
212 * and register it as a T3C client
213 */
214static void
215t3c_tom_add(struct t3cdev *cdev)
216{
217 int i;
218 unsigned int wr_len;
219 struct tom_data *t;
220 struct toedev *tdev;
221 struct adap_ports *port_info;
222
223 t = malloc(sizeof(*t), M_CXGB, M_NOWAIT|M_ZERO);
224 if (t == NULL)
225 return;
226
227 if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0)
228 goto out_free_tom;
229
230 port_info = malloc(sizeof(*port_info), M_CXGB, M_NOWAIT|M_ZERO);
231 if (!port_info)
232 goto out_free_tom;
233
234 if (cdev->ctl(cdev, GET_PORTS, port_info) < 0)
235 goto out_free_all;
236
237 t3_init_wr_tab(wr_len);
238 t->cdev = cdev;
239 t->client = &t3c_tom_client;
240
241 /* Register TCP offload device */
242 tdev = &t->tdev;
243 tdev->tod_ttid = cdev2type(cdev);
244 tdev->tod_lldev = cdev->lldev;
245
246 if (register_toedev(tdev, "toe%d")) {
247 printf("unable to register offload device");
248 goto out_free_all;
249 }
250 TOM_DATA(tdev) = t;
251
252 for (i = 0; i < port_info->nports; i++) {
253 struct ifnet *ifp = port_info->lldevs[i];
254 TOEDEV(ifp) = tdev;
255
256 CTR1(KTR_TOM, "enabling toe on %p", ifp);
257 ifp->if_capabilities |= IFCAP_TOE4;
258 ifp->if_capenable |= IFCAP_TOE4;
259 }
260 t->ports = port_info;
261
262 /* Add device to the list of offload devices */
263 t3cdev_add(t);
264
265 /* Activate TCP offload device */
266 activate_offload(tdev);
267 cxgb_register_listeners();
268 return;
269
270out_free_all:
271 printf("out_free_all fail\n");
272 free(port_info, M_CXGB);
273out_free_tom:
274 printf("out_free_tom fail\n");
275 free(t, M_CXGB);
276 return;
277}
278
279/*
280 * Process a received packet with an unknown/unexpected CPL opcode.
281 */
282static int
283do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
284{
285 log(LOG_ERR, "%s: received bad CPL command %u\n", cdev->name,
286 0xFF & *mtod(m, unsigned int *));
287 kdb_backtrace();
288 return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
289}
290
291
292/*
293 * Add a new handler to the CPL dispatch table. A NULL handler may be supplied
294 * to unregister an existing handler.
295 */
296void
297t3tom_register_cpl_handler(unsigned int opcode, cxgb_cpl_handler_func h)
298{
299 if (opcode < 256)
300 tom_cpl_handlers[opcode] = h ? h : do_bad_cpl;
301 else
302 log(LOG_ERR, "Chelsio T3 TOM: handler registration for "
303 "opcode %u failed\n", opcode);
304}
305
306/*
307 * Make a preliminary determination if a connection can be offloaded. It's OK
308 * to fail the offload later if we say we can offload here. For now this
309 * always accepts the offload request unless there are IP options.
310 */
311static int
312can_offload(struct toedev *dev, struct socket *so)
313{
314 struct tom_data *tomd = TOM_DATA(dev);
315 struct t3cdev *cdev = T3CDEV(dev->tod_lldev);
316 struct tid_info *t = &(T3C_DATA(cdev))->tid_maps;
317
318 return sotoinpcb(so)->inp_depend4.inp4_options == NULL &&
319 tomd->conf.activated &&
320 (tomd->conf.max_conn < 0 ||
321 atomic_load_acq_int(&t->tids_in_use) + t->atids_in_use < tomd->conf.max_conn);
322}
323
324static int
325tom_ctl(struct toedev *dev, unsigned int req, void *data)
326{
327 struct tom_data *t = TOM_DATA(dev);
328 struct t3cdev *cdev = t->cdev;
329
330 if (cdev->ctl)
331 return cdev->ctl(cdev, req, data);
332
333 return (EOPNOTSUPP);
334}
335
336/*
337 * Initialize the CPL dispatch table.
338 */
339static void
340init_cpl_handlers(void)
341{
342 int i;
343
344 for (i = 0; i < 256; ++i)
345 tom_cpl_handlers[i] = do_bad_cpl;
346
347 t3_init_listen_cpl_handlers();
348}
349
350static int
351t3_toe_attach(struct toedev *dev, const struct offload_id *entry)
352{
353 struct tom_data *t = TOM_DATA(dev);
354 struct t3cdev *cdev = t->cdev;
355 struct ddp_params ddp;
356 struct ofld_page_info rx_page_info;
357 int err;
358
359#if 0
360 skb_queue_head_init(&t->deferq);
361 T3_INIT_WORK(&t->deferq_task, process_deferq, t);
362 spin_lock_init(&t->listen_lock);
363#endif
364 t3_init_tunables(t);
365 mtx_init(&t->listen_lock, "tom data listeners", NULL, MTX_DEF);
366 CTR2(KTR_TOM, "t3_toe_attach dev=%p entry=%p", dev, entry);
367 /* Adjust TOE activation for this module */
368 t->conf.activated = activated;
369
370 dev->tod_can_offload = can_offload;
371 dev->tod_connect = t3_connect;
372 dev->tod_ctl = tom_ctl;
373#if 0
374#ifndef NETEVENT
375 dev->tod_neigh_update = tom_neigh_update;
376#endif
377 dev->tod_failover = t3_failover;
378#endif
379 err = cdev->ctl(cdev, GET_DDP_PARAMS, &ddp);
380 if (err)
381 return err;
382
383 err = cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info);
384 if (err)
385 return err;
386
387 t->ddp_llimit = ddp.llimit;
388 t->ddp_ulimit = ddp.ulimit;
389 t->pdev = ddp.pdev;
390 t->rx_page_size = rx_page_info.page_size;
391 /* OK if this fails, we just can't do DDP */
392 t->nppods = (ddp.ulimit + 1 - ddp.llimit) / PPOD_SIZE;
393 t->ppod_map = malloc(t->nppods, M_DEVBUF, M_WAITOK|M_ZERO);
394
395 mtx_init(&t->ppod_map_lock, "ppod map", NULL, MTX_DEF);
396
397
398 t3_sysctl_register(cdev->adapter, &t->conf);
399 return (0);
400}
401
402static void
403cxgb_toe_listen_start(void *unused, struct tcpcb *tp)
404{
405 struct socket *so = tp->t_inpcb->inp_socket;
406 struct tom_data *p;
407
408 mtx_lock(&cxgb_list_lock);
409 TAILQ_FOREACH(p, &cxgb_list, entry) {
410 t3_listen_start(&p->tdev, so, p->cdev);
411 }
412 mtx_unlock(&cxgb_list_lock);
413}
414
415static void
416cxgb_toe_listen_stop(void *unused, struct tcpcb *tp)
417{
418 struct socket *so = tp->t_inpcb->inp_socket;
419 struct tom_data *p;
420
421 mtx_lock(&cxgb_list_lock);
422 TAILQ_FOREACH(p, &cxgb_list, entry) {
423 if (tp->t_state == TCPS_LISTEN)
424 t3_listen_stop(&p->tdev, so, p->cdev);
425 }
426 mtx_unlock(&cxgb_list_lock);
427}
428
429static void
430cxgb_register_listeners(void)
431{
432 struct inpcb *inp;
433 struct tcpcb *tp;
434
435 INP_INFO_RLOCK(&tcbinfo);
436 LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) {
437 tp = intotcpcb(inp);
438
439 if (tp->t_state == TCPS_LISTEN)
440 cxgb_toe_listen_start(NULL, tp);
441 }
442 INP_INFO_RUNLOCK(&tcbinfo);
443}
444
445static int
446t3_tom_init(void)
447{
448 init_cpl_handlers();
449 if (t3_init_cpl_io() < 0) {
450 log(LOG_ERR,
451 "Unable to initialize cpl io ops\n");
452 return -1;
453 }
454 t3_init_socket_ops();
455
456 /* Register with the TOE device layer. */
457
458 if (register_tom(&t3_tom_info) != 0) {
459 log(LOG_ERR,
460 "Unable to register Chelsio T3 TCP offload module.\n");
461 return -1;
462 }
463 INP_INFO_WLOCK(&tcbinfo);
464 INP_INFO_WUNLOCK(&tcbinfo);
465
466 mtx_init(&cxgb_list_lock, "cxgb tom list", NULL, MTX_DEF);
467 listen_tag = EVENTHANDLER_REGISTER(tcp_offload_listen_start,
468 cxgb_toe_listen_start, NULL, EVENTHANDLER_PRI_ANY);
469 listen_tag = EVENTHANDLER_REGISTER(tcp_offload_listen_stop,
470 cxgb_toe_listen_stop, NULL, EVENTHANDLER_PRI_ANY);
471 TAILQ_INIT(&cxgb_list);
472
473 /* Register to offloading devices */
474 t3c_tom_client.add = t3c_tom_add;
475 cxgb_register_client(&t3c_tom_client);
476 return (0);
477}
478
479static int
480t3_tom_load(module_t mod, int cmd, void *arg)
481{
482 int err = 0;
483
484 switch (cmd) {
485 case MOD_LOAD:
486 t3_tom_init();
487 break;
488 case MOD_QUIESCE:
489 break;
490 case MOD_UNLOAD:
491 printf("uhm, ... unloading isn't really supported for toe\n");
492 break;
493 case MOD_SHUTDOWN:
494 break;
495 default:
496 err = EOPNOTSUPP;
497 break;
498 }
499
500 return (err);
501}
502
503static moduledata_t mod_data= {
504 "t3_tom",
505 t3_tom_load,
506 0
507};
508MODULE_VERSION(t3_tom, 1);
509MODULE_DEPEND(t3_tom, toecore, 1, 1, 1);
510MODULE_DEPEND(t3_tom, if_cxgb, 1, 1, 1);
511DECLARE_MODULE(t3_tom, mod_data, SI_SUB_EXEC, SI_ORDER_ANY);
512
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_tom.c 176472 2008-02-23 01:06:17Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/fcntl.h>
37#include <sys/ktr.h>
38#include <sys/limits.h>
39#include <sys/lock.h>
40#include <sys/eventhandler.h>
41#include <sys/mbuf.h>
42#include <sys/module.h>
43#include <sys/condvar.h>
44#include <sys/mutex.h>
45#include <sys/socket.h>
46#include <sys/sysctl.h>
47#include <sys/syslog.h>
48#include <sys/socketvar.h>
49#include <sys/taskqueue.h>
50
51#include <net/if.h>
52#include <net/route.h>
53
54#include <netinet/in.h>
55#include <netinet/in_pcb.h>
56#include <netinet/in_systm.h>
57#include <netinet/in_var.h>
58
59#include <dev/cxgb/cxgb_osdep.h>
60#include <dev/cxgb/sys/mbufq.h>
61
62#include <netinet/in_pcb.h>
63#include <netinet/tcp.h>
64#include <netinet/tcp_var.h>
65#include <netinet/tcp_offload.h>
66#include <netinet/tcp_fsm.h>
67#include <net/route.h>
68
69#include <dev/cxgb/t3cdev.h>
70#include <dev/cxgb/common/cxgb_firmware_exports.h>
71#include <dev/cxgb/common/cxgb_tcb.h>
72#include <dev/cxgb/cxgb_include.h>
73#include <dev/cxgb/common/cxgb_ctl_defs.h>
74#include <dev/cxgb/common/cxgb_t3_cpl.h>
75#include <dev/cxgb/cxgb_offload.h>
76#include <dev/cxgb/cxgb_l2t.h>
77#include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
78#include <dev/cxgb/ulp/tom/cxgb_tom.h>
79#include <dev/cxgb/ulp/tom/cxgb_defs.h>
80#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h>
81#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
82#include <dev/cxgb/ulp/tom/cxgb_tcp.h>
83
84
85static int activated = 1;
86TUNABLE_INT("hw.t3toe.activated", &activated);
87SYSCTL_NODE(_hw, OID_AUTO, t3toe, CTLFLAG_RD, 0, "T3 toe driver parameters");
88SYSCTL_UINT(_hw_t3toe, OID_AUTO, activated, CTLFLAG_RDTUN, &activated, 0,
89 "enable TOE at init time");
90
91static TAILQ_HEAD(, tom_data) cxgb_list;
92static struct mtx cxgb_list_lock;
93
94static int t3_toe_attach(struct toedev *dev, const struct offload_id *entry);
95static void cxgb_register_listeners(void);
96
97/*
98 * Handlers for each CPL opcode
99 */
100static cxgb_cpl_handler_func tom_cpl_handlers[256];
101
102
103static eventhandler_tag listen_tag;
104
105static struct offload_id t3_toe_id_tab[] = {
106 { TOE_ID_CHELSIO_T3, 0 },
107 { TOE_ID_CHELSIO_T3B, 0 },
108 { TOE_ID_CHELSIO_T3C, 0 },
109 { 0 }
110};
111
112static struct tom_info t3_tom_info = {
113 .ti_attach = t3_toe_attach,
114 .ti_id_table = t3_toe_id_tab,
115 .ti_name = "Chelsio-T3"
116};
117
118struct cxgb_client t3c_tom_client = {
119 .name = "tom_cxgb3",
120 .remove = NULL,
121 .handlers = tom_cpl_handlers,
122 .redirect = NULL
123};
124
125/*
126 * Add an skb to the deferred skb queue for processing from process context.
127 */
128void
129t3_defer_reply(struct mbuf *m, struct toedev *dev, defer_handler_t handler)
130{
131 struct tom_data *td = TOM_DATA(dev);
132
133 m_set_handler(m, handler);
134 mtx_lock(&td->deferq.lock);
135
136 mbufq_tail(&td->deferq, m);
137 if (mbufq_len(&td->deferq) == 1)
138 taskqueue_enqueue(td->tq, &td->deferq_task);
139 mtx_lock(&td->deferq.lock);
140}
141
142struct toepcb *
143toepcb_alloc(void)
144{
145 struct toepcb *toep;
146
147 toep = malloc(sizeof(struct toepcb), M_DEVBUF, M_NOWAIT|M_ZERO);
148
149 if (toep == NULL)
150 return (NULL);
151
152 toepcb_init(toep);
153 return (toep);
154}
155
156void
157toepcb_init(struct toepcb *toep)
158{
159 toep->tp_refcount = 1;
160 cv_init(&toep->tp_cv, "toep cv");
161}
162
163void
164toepcb_hold(struct toepcb *toep)
165{
166 atomic_add_acq_int(&toep->tp_refcount, 1);
167}
168
169void
170toepcb_release(struct toepcb *toep)
171{
172 if (toep->tp_refcount == 1) {
173 free(toep, M_DEVBUF);
174 return;
175 }
176 atomic_add_acq_int(&toep->tp_refcount, -1);
177}
178
179/*
180 * Add a T3 offload device to the list of devices we are managing.
181 */
182static void
183t3cdev_add(struct tom_data *t)
184{
185 mtx_lock(&cxgb_list_lock);
186 TAILQ_INSERT_TAIL(&cxgb_list, t, entry);
187 mtx_unlock(&cxgb_list_lock);
188}
189
190static inline int
191cdev2type(struct t3cdev *cdev)
192{
193 int type = 0;
194
195 switch (cdev->type) {
196 case T3A:
197 type = TOE_ID_CHELSIO_T3;
198 break;
199 case T3B:
200 type = TOE_ID_CHELSIO_T3B;
201 break;
202 case T3C:
203 type = TOE_ID_CHELSIO_T3C;
204 break;
205 }
206 return (type);
207}
208
209/*
210 * Allocate a TOM data structure,
211 * initialize its cpl_handlers
212 * and register it as a T3C client
213 */
214static void
215t3c_tom_add(struct t3cdev *cdev)
216{
217 int i;
218 unsigned int wr_len;
219 struct tom_data *t;
220 struct toedev *tdev;
221 struct adap_ports *port_info;
222
223 t = malloc(sizeof(*t), M_CXGB, M_NOWAIT|M_ZERO);
224 if (t == NULL)
225 return;
226
227 if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0)
228 goto out_free_tom;
229
230 port_info = malloc(sizeof(*port_info), M_CXGB, M_NOWAIT|M_ZERO);
231 if (!port_info)
232 goto out_free_tom;
233
234 if (cdev->ctl(cdev, GET_PORTS, port_info) < 0)
235 goto out_free_all;
236
237 t3_init_wr_tab(wr_len);
238 t->cdev = cdev;
239 t->client = &t3c_tom_client;
240
241 /* Register TCP offload device */
242 tdev = &t->tdev;
243 tdev->tod_ttid = cdev2type(cdev);
244 tdev->tod_lldev = cdev->lldev;
245
246 if (register_toedev(tdev, "toe%d")) {
247 printf("unable to register offload device");
248 goto out_free_all;
249 }
250 TOM_DATA(tdev) = t;
251
252 for (i = 0; i < port_info->nports; i++) {
253 struct ifnet *ifp = port_info->lldevs[i];
254 TOEDEV(ifp) = tdev;
255
256 CTR1(KTR_TOM, "enabling toe on %p", ifp);
257 ifp->if_capabilities |= IFCAP_TOE4;
258 ifp->if_capenable |= IFCAP_TOE4;
259 }
260 t->ports = port_info;
261
262 /* Add device to the list of offload devices */
263 t3cdev_add(t);
264
265 /* Activate TCP offload device */
266 activate_offload(tdev);
267 cxgb_register_listeners();
268 return;
269
270out_free_all:
271 printf("out_free_all fail\n");
272 free(port_info, M_CXGB);
273out_free_tom:
274 printf("out_free_tom fail\n");
275 free(t, M_CXGB);
276 return;
277}
278
279/*
280 * Process a received packet with an unknown/unexpected CPL opcode.
281 */
282static int
283do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
284{
285 log(LOG_ERR, "%s: received bad CPL command %u\n", cdev->name,
286 0xFF & *mtod(m, unsigned int *));
287 kdb_backtrace();
288 return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
289}
290
291
292/*
293 * Add a new handler to the CPL dispatch table. A NULL handler may be supplied
294 * to unregister an existing handler.
295 */
296void
297t3tom_register_cpl_handler(unsigned int opcode, cxgb_cpl_handler_func h)
298{
299 if (opcode < 256)
300 tom_cpl_handlers[opcode] = h ? h : do_bad_cpl;
301 else
302 log(LOG_ERR, "Chelsio T3 TOM: handler registration for "
303 "opcode %u failed\n", opcode);
304}
305
306/*
307 * Make a preliminary determination if a connection can be offloaded. It's OK
308 * to fail the offload later if we say we can offload here. For now this
309 * always accepts the offload request unless there are IP options.
310 */
311static int
312can_offload(struct toedev *dev, struct socket *so)
313{
314 struct tom_data *tomd = TOM_DATA(dev);
315 struct t3cdev *cdev = T3CDEV(dev->tod_lldev);
316 struct tid_info *t = &(T3C_DATA(cdev))->tid_maps;
317
318 return sotoinpcb(so)->inp_depend4.inp4_options == NULL &&
319 tomd->conf.activated &&
320 (tomd->conf.max_conn < 0 ||
321 atomic_load_acq_int(&t->tids_in_use) + t->atids_in_use < tomd->conf.max_conn);
322}
323
324static int
325tom_ctl(struct toedev *dev, unsigned int req, void *data)
326{
327 struct tom_data *t = TOM_DATA(dev);
328 struct t3cdev *cdev = t->cdev;
329
330 if (cdev->ctl)
331 return cdev->ctl(cdev, req, data);
332
333 return (EOPNOTSUPP);
334}
335
336/*
337 * Initialize the CPL dispatch table.
338 */
339static void
340init_cpl_handlers(void)
341{
342 int i;
343
344 for (i = 0; i < 256; ++i)
345 tom_cpl_handlers[i] = do_bad_cpl;
346
347 t3_init_listen_cpl_handlers();
348}
349
350static int
351t3_toe_attach(struct toedev *dev, const struct offload_id *entry)
352{
353 struct tom_data *t = TOM_DATA(dev);
354 struct t3cdev *cdev = t->cdev;
355 struct ddp_params ddp;
356 struct ofld_page_info rx_page_info;
357 int err;
358
359#if 0
360 skb_queue_head_init(&t->deferq);
361 T3_INIT_WORK(&t->deferq_task, process_deferq, t);
362 spin_lock_init(&t->listen_lock);
363#endif
364 t3_init_tunables(t);
365 mtx_init(&t->listen_lock, "tom data listeners", NULL, MTX_DEF);
366 CTR2(KTR_TOM, "t3_toe_attach dev=%p entry=%p", dev, entry);
367 /* Adjust TOE activation for this module */
368 t->conf.activated = activated;
369
370 dev->tod_can_offload = can_offload;
371 dev->tod_connect = t3_connect;
372 dev->tod_ctl = tom_ctl;
373#if 0
374#ifndef NETEVENT
375 dev->tod_neigh_update = tom_neigh_update;
376#endif
377 dev->tod_failover = t3_failover;
378#endif
379 err = cdev->ctl(cdev, GET_DDP_PARAMS, &ddp);
380 if (err)
381 return err;
382
383 err = cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info);
384 if (err)
385 return err;
386
387 t->ddp_llimit = ddp.llimit;
388 t->ddp_ulimit = ddp.ulimit;
389 t->pdev = ddp.pdev;
390 t->rx_page_size = rx_page_info.page_size;
391 /* OK if this fails, we just can't do DDP */
392 t->nppods = (ddp.ulimit + 1 - ddp.llimit) / PPOD_SIZE;
393 t->ppod_map = malloc(t->nppods, M_DEVBUF, M_WAITOK|M_ZERO);
394
395 mtx_init(&t->ppod_map_lock, "ppod map", NULL, MTX_DEF);
396
397
398 t3_sysctl_register(cdev->adapter, &t->conf);
399 return (0);
400}
401
402static void
403cxgb_toe_listen_start(void *unused, struct tcpcb *tp)
404{
405 struct socket *so = tp->t_inpcb->inp_socket;
406 struct tom_data *p;
407
408 mtx_lock(&cxgb_list_lock);
409 TAILQ_FOREACH(p, &cxgb_list, entry) {
410 t3_listen_start(&p->tdev, so, p->cdev);
411 }
412 mtx_unlock(&cxgb_list_lock);
413}
414
415static void
416cxgb_toe_listen_stop(void *unused, struct tcpcb *tp)
417{
418 struct socket *so = tp->t_inpcb->inp_socket;
419 struct tom_data *p;
420
421 mtx_lock(&cxgb_list_lock);
422 TAILQ_FOREACH(p, &cxgb_list, entry) {
423 if (tp->t_state == TCPS_LISTEN)
424 t3_listen_stop(&p->tdev, so, p->cdev);
425 }
426 mtx_unlock(&cxgb_list_lock);
427}
428
429static void
430cxgb_register_listeners(void)
431{
432 struct inpcb *inp;
433 struct tcpcb *tp;
434
435 INP_INFO_RLOCK(&tcbinfo);
436 LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) {
437 tp = intotcpcb(inp);
438
439 if (tp->t_state == TCPS_LISTEN)
440 cxgb_toe_listen_start(NULL, tp);
441 }
442 INP_INFO_RUNLOCK(&tcbinfo);
443}
444
445static int
446t3_tom_init(void)
447{
448 init_cpl_handlers();
449 if (t3_init_cpl_io() < 0) {
450 log(LOG_ERR,
451 "Unable to initialize cpl io ops\n");
452 return -1;
453 }
454 t3_init_socket_ops();
455
456 /* Register with the TOE device layer. */
457
458 if (register_tom(&t3_tom_info) != 0) {
459 log(LOG_ERR,
460 "Unable to register Chelsio T3 TCP offload module.\n");
461 return -1;
462 }
463 INP_INFO_WLOCK(&tcbinfo);
464 INP_INFO_WUNLOCK(&tcbinfo);
465
466 mtx_init(&cxgb_list_lock, "cxgb tom list", NULL, MTX_DEF);
467 listen_tag = EVENTHANDLER_REGISTER(tcp_offload_listen_start,
468 cxgb_toe_listen_start, NULL, EVENTHANDLER_PRI_ANY);
469 listen_tag = EVENTHANDLER_REGISTER(tcp_offload_listen_stop,
470 cxgb_toe_listen_stop, NULL, EVENTHANDLER_PRI_ANY);
471 TAILQ_INIT(&cxgb_list);
472
473 /* Register to offloading devices */
474 t3c_tom_client.add = t3c_tom_add;
475 cxgb_register_client(&t3c_tom_client);
476 return (0);
477}
478
479static int
480t3_tom_load(module_t mod, int cmd, void *arg)
481{
482 int err = 0;
483
484 switch (cmd) {
485 case MOD_LOAD:
486 t3_tom_init();
487 break;
488 case MOD_QUIESCE:
489 break;
490 case MOD_UNLOAD:
491 printf("uhm, ... unloading isn't really supported for toe\n");
492 break;
493 case MOD_SHUTDOWN:
494 break;
495 default:
496 err = EOPNOTSUPP;
497 break;
498 }
499
500 return (err);
501}
502
503static moduledata_t mod_data= {
504 "t3_tom",
505 t3_tom_load,
506 0
507};
508MODULE_VERSION(t3_tom, 1);
509MODULE_DEPEND(t3_tom, toecore, 1, 1, 1);
510MODULE_DEPEND(t3_tom, if_cxgb, 1, 1, 1);
511DECLARE_MODULE(t3_tom, mod_data, SI_SUB_EXEC, SI_ORDER_ANY);
512