1/*-
2 * Copyright (c) 2012 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: releng/11.0/sys/dev/cxgb/ulp/tom/cxgb_tom.c 241394 2012-10-10 08:36:38Z kevlo $");
30
31#include "opt_inet.h"
32
33#include <sys/param.h>
34#include <sys/types.h>
35#include <sys/kernel.h>
36#include <sys/queue.h>
37#include <sys/malloc.h>
38#include <sys/module.h>
39#include <sys/socket.h>
40#include <sys/taskqueue.h>
41#include <netinet/in.h>
42#include <netinet/tcp.h>
43#include <netinet/toecore.h>
44
45#ifdef TCP_OFFLOAD
46#include "cxgb_include.h"
47#include "ulp/tom/cxgb_tom.h"
48#include "ulp/tom/cxgb_l2t.h"
49#include "ulp/tom/cxgb_toepcb.h"
50
51MALLOC_DEFINE(M_CXGB, "cxgb", "Chelsio T3 Offload services");
52
53/* Module ops */
54static int t3_tom_mod_load(void);
55static int t3_tom_mod_unload(void);
56static int t3_tom_modevent(module_t, int, void *);
57
58/* ULD ops and helpers */
59static int t3_tom_activate(struct adapter *);
60static int t3_tom_deactivate(struct adapter *);
61
62static int alloc_tid_tabs(struct tid_info *, u_int, u_int, u_int, u_int, u_int);
63static void free_tid_tabs(struct tid_info *);
64static int write_smt_entry(struct adapter *, int);
65static void free_tom_data(struct tom_data *);
66
67static struct uld_info tom_uld_info = {
68	.uld_id = ULD_TOM,
69	.activate = t3_tom_activate,
70	.deactivate = t3_tom_deactivate,
71};
72
73struct toepcb *
74toepcb_alloc(struct toedev *tod)
75{
76	struct toepcb *toep;
77
78	toep = malloc(sizeof(struct toepcb), M_CXGB, M_NOWAIT | M_ZERO);
79	if (toep == NULL)
80		return (NULL);
81
82	toep->tp_tod = tod;
83	toep->tp_wr_max = toep->tp_wr_avail = 15;
84	toep->tp_wr_unacked = 0;
85	toep->tp_delack_mode = 0;
86
87	return (toep);
88}
89
90void
91toepcb_free(struct toepcb *toep)
92{
93	free(toep, M_CXGB);
94}
95
96static int
97alloc_tid_tabs(struct tid_info *t, u_int ntids, u_int natids, u_int nstids,
98    u_int atid_base, u_int stid_base)
99{
100	unsigned long size = ntids * sizeof(*t->tid_tab) +
101	    natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab);
102
103	t->tid_tab = malloc(size, M_CXGB, M_NOWAIT | M_ZERO);
104	if (!t->tid_tab)
105		return (ENOMEM);
106
107	t->stid_tab = (union listen_entry *)&t->tid_tab[ntids];
108	t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids];
109	t->ntids = ntids;
110	t->nstids = nstids;
111	t->stid_base = stid_base;
112	t->sfree = NULL;
113	t->natids = natids;
114	t->atid_base = atid_base;
115	t->afree = NULL;
116	t->stids_in_use = t->atids_in_use = 0;
117	t->tids_in_use = 0;
118	mtx_init(&t->stid_lock, "stid", NULL, MTX_DEF);
119	mtx_init(&t->atid_lock, "atid", NULL, MTX_DEF);
120
121	/*
122	 * Setup the free lists for stid_tab and atid_tab.
123	 */
124	if (nstids) {
125		while (--nstids)
126			t->stid_tab[nstids - 1].next = &t->stid_tab[nstids];
127		t->sfree = t->stid_tab;
128	}
129	if (natids) {
130		while (--natids)
131			t->atid_tab[natids - 1].next = &t->atid_tab[natids];
132		t->afree = t->atid_tab;
133	}
134	return (0);
135}
136
137static void
138free_tid_tabs(struct tid_info *t)
139{
140	if (mtx_initialized(&t->stid_lock))
141		mtx_destroy(&t->stid_lock);
142	if (mtx_initialized(&t->atid_lock))
143		mtx_destroy(&t->atid_lock);
144	free(t->tid_tab, M_CXGB);
145}
146
147static int
148write_smt_entry(struct adapter *sc, int idx)
149{
150	struct port_info *pi = &sc->port[idx];
151	struct cpl_smt_write_req *req;
152	struct mbuf *m;
153
154	m = M_GETHDR_OFLD(0, CPL_PRIORITY_CONTROL, req);
155	if (m == NULL) {
156		log(LOG_ERR, "%s: no mbuf, can't write SMT entry for %d\n",
157		    __func__, idx);
158		return (ENOMEM);
159	}
160
161	req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
162	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
163	req->mtu_idx = NMTUS - 1;  /* should be 0 but there's a T3 bug */
164	req->iff = idx;
165	memset(req->src_mac1, 0, sizeof(req->src_mac1));
166	memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
167
168	t3_offload_tx(sc, m);
169
170	return (0);
171}
172
173static void
174free_tom_data(struct tom_data *td)
175{
176	KASSERT(TAILQ_EMPTY(&td->toep_list),
177	    ("%s: toep_list not empty", __func__));
178
179	if (td->listen_mask != 0)
180		hashdestroy(td->listen_hash, M_CXGB, td->listen_mask);
181
182	if (mtx_initialized(&td->toep_list_lock))
183		mtx_destroy(&td->toep_list_lock);
184	if (mtx_initialized(&td->lctx_hash_lock))
185		mtx_destroy(&td->lctx_hash_lock);
186	if (mtx_initialized(&td->tid_release_lock))
187		mtx_destroy(&td->tid_release_lock);
188	if (td->l2t)
189		t3_free_l2t(td->l2t);
190	free_tid_tabs(&td->tid_maps);
191	free(td, M_CXGB);
192}
193
194/*
195 * Ground control to Major TOM
196 * Commencing countdown, engines on
197 */
198static int
199t3_tom_activate(struct adapter *sc)
200{
201	struct tom_data *td;
202	struct toedev *tod;
203	int i, rc = 0;
204	struct mc5_params *mc5 = &sc->params.mc5;
205	u_int ntids, natids, mtus;
206
207	ADAPTER_LOCK_ASSERT_OWNED(sc);	/* for sc->flags */
208
209	/* per-adapter softc for TOM */
210	td = malloc(sizeof(*td), M_CXGB, M_ZERO | M_NOWAIT);
211	if (td == NULL)
212		return (ENOMEM);
213
214	/* List of TOE PCBs and associated lock */
215	mtx_init(&td->toep_list_lock, "PCB list lock", NULL, MTX_DEF);
216	TAILQ_INIT(&td->toep_list);
217
218	/* Listen context */
219	mtx_init(&td->lctx_hash_lock, "lctx hash lock", NULL, MTX_DEF);
220	td->listen_hash = hashinit_flags(LISTEN_HASH_SIZE, M_CXGB,
221	    &td->listen_mask, HASH_NOWAIT);
222
223	/* TID release task */
224	TASK_INIT(&td->tid_release_task, 0 , t3_process_tid_release_list, td);
225	mtx_init(&td->tid_release_lock, "tid release", NULL, MTX_DEF);
226
227	/* L2 table */
228	td->l2t = t3_init_l2t(L2T_SIZE);
229	if (td->l2t == NULL) {
230		rc = ENOMEM;
231		goto done;
232	}
233
234	/* TID tables */
235	ntids = t3_mc5_size(&sc->mc5) - mc5->nroutes - mc5->nfilters -
236	    mc5->nservers;
237	natids = min(ntids / 2, 64 * 1024);
238	rc = alloc_tid_tabs(&td->tid_maps, ntids, natids, mc5->nservers,
239	    0x100000 /* ATID_BASE */, ntids);
240	if (rc != 0)
241		goto done;
242
243	/* CPL handlers */
244	t3_init_listen_cpl_handlers(sc);
245	t3_init_l2t_cpl_handlers(sc);
246	t3_init_cpl_io(sc);
247
248	/* toedev ops */
249	tod = &td->tod;
250	init_toedev(tod);
251	tod->tod_softc = sc;
252	tod->tod_connect = t3_connect;
253	tod->tod_listen_start = t3_listen_start;
254	tod->tod_listen_stop = t3_listen_stop;
255	tod->tod_rcvd = t3_rcvd;
256	tod->tod_output = t3_tod_output;
257	tod->tod_send_rst = t3_send_rst;
258	tod->tod_send_fin = t3_send_fin;
259	tod->tod_pcb_detach = t3_pcb_detach;
260	tod->tod_l2_update = t3_l2_update;
261	tod->tod_syncache_added = t3_syncache_added;
262	tod->tod_syncache_removed = t3_syncache_removed;
263	tod->tod_syncache_respond = t3_syncache_respond;
264	tod->tod_offload_socket = t3_offload_socket;
265
266	/* port MTUs */
267	mtus = sc->port[0].ifp->if_mtu;
268	if (sc->params.nports > 1)
269		mtus |= sc->port[1].ifp->if_mtu << 16;
270	t3_write_reg(sc, A_TP_MTU_PORT_TABLE, mtus);
271	t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd,
272	    sc->params.rev == 0 ? sc->port[0].ifp->if_mtu : 0xffff);
273
274	/* SMT entry for each port */
275	for_each_port(sc, i) {
276		write_smt_entry(sc, i);
277		TOEDEV(sc->port[i].ifp) = &td->tod;
278	}
279
280	/* Switch TP to offload mode */
281	t3_tp_set_offload_mode(sc, 1);
282
283	sc->tom_softc = td;
284	sc->flags |= TOM_INIT_DONE;
285	register_toedev(tod);
286
287done:
288	if (rc != 0)
289		free_tom_data(td);
290
291	return (rc);
292}
293
294static int
295t3_tom_deactivate(struct adapter *sc)
296{
297	int rc = 0;
298	struct tom_data *td = sc->tom_softc;
299
300	ADAPTER_LOCK_ASSERT_OWNED(sc);	/* for sc->flags */
301
302	if (td == NULL)
303		return (0);	/* XXX. KASSERT? */
304
305	if (sc->offload_map != 0)
306		return (EBUSY);	/* at least one port has IFCAP_TOE enabled */
307
308	mtx_lock(&td->toep_list_lock);
309	if (!TAILQ_EMPTY(&td->toep_list))
310		rc = EBUSY;
311	mtx_unlock(&td->toep_list_lock);
312
313	mtx_lock(&td->lctx_hash_lock);
314	if (td->lctx_count > 0)
315		rc = EBUSY;
316	mtx_unlock(&td->lctx_hash_lock);
317
318	if (rc == 0) {
319		unregister_toedev(&td->tod);
320		t3_tp_set_offload_mode(sc, 0);
321		free_tom_data(td);
322		sc->tom_softc = NULL;
323		sc->flags &= ~TOM_INIT_DONE;
324	}
325
326	return (rc);
327}
328
329static int
330t3_tom_mod_load(void)
331{
332	int rc;
333
334	rc = t3_register_uld(&tom_uld_info);
335	if (rc != 0)
336		t3_tom_mod_unload();
337
338	return (rc);
339}
340
341static void
342tom_uninit(struct adapter *sc, void *arg __unused)
343{
344	/* Try to free resources (works only if no port has IFCAP_TOE) */
345	ADAPTER_LOCK(sc);
346	if (sc->flags & TOM_INIT_DONE)
347		t3_deactivate_uld(sc, ULD_TOM);
348	ADAPTER_UNLOCK(sc);
349}
350
351static int
352t3_tom_mod_unload(void)
353{
354	t3_iterate(tom_uninit, NULL);
355
356	if (t3_unregister_uld(&tom_uld_info) == EBUSY)
357		return (EBUSY);
358
359	return (0);
360}
361#endif	/* ifdef TCP_OFFLOAD */
362
363static int
364t3_tom_modevent(module_t mod, int cmd, void *arg)
365{
366	int rc = 0;
367
368#ifdef TCP_OFFLOAD
369	switch (cmd) {
370	case MOD_LOAD:
371		rc = t3_tom_mod_load();
372		break;
373
374	case MOD_UNLOAD:
375		rc = t3_tom_mod_unload();
376		break;
377
378	default:
379		rc = EINVAL;
380	}
381#else
382	rc = EOPNOTSUPP;
383#endif
384	return (rc);
385}
386
387static moduledata_t t3_tom_moddata= {
388	"t3_tom",
389	t3_tom_modevent,
390	0
391};
392
393MODULE_VERSION(t3_tom, 1);
394MODULE_DEPEND(t3_tom, toecore, 1, 1, 1);
395MODULE_DEPEND(t3_tom, cxgbc, 1, 1, 1);
396DECLARE_MODULE(t3_tom, t3_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY);
397