1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2012 Chelsio Communications, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28#include <sys/cdefs.h>
29#include "opt_inet.h"
30#include "opt_inet6.h"
31
32#ifdef TCP_OFFLOAD
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/bus.h>
38#include <sys/fnv_hash.h>
39#include <sys/lock.h>
40#include <sys/mutex.h>
41#include <sys/rwlock.h>
42#include <sys/socket.h>
43#include <sys/socketvar.h>
44#include <sys/sbuf.h>
45#include <sys/taskqueue.h>
46#include <net/if.h>
47#include <net/if_types.h>
48#include <net/ethernet.h>
49#include <net/if_vlan_var.h>
50#include <net/route.h>
51#include <netinet/in.h>
52#include <netinet/in_pcb.h>
53#include <netinet/tcp_var.h>
54#include <netinet/toecore.h>
55
56#include "common/common.h"
57#include "common/t4_msg.h"
58#include "tom/t4_tom_l2t.h"
59#include "tom/t4_tom.h"
60
61#define VLAN_NONE	0xfff
62
63static inline void
64l2t_hold(struct l2t_data *d, struct l2t_entry *e)
65{
66
67	if (atomic_fetchadd_int(&e->refcnt, 1) == 0)  /* 0 -> 1 transition */
68		atomic_subtract_int(&d->nfree, 1);
69}
70
71static inline u_int
72l2_hash(struct l2t_data *d, const struct sockaddr *sa, int ifindex)
73{
74	u_int hash, half = d->l2t_size / 2, start = 0;
75	const void *key;
76	size_t len;
77
78	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
79	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
80	    sa->sa_family));
81
82	if (sa->sa_family == AF_INET) {
83		const struct sockaddr_in *sin = (const void *)sa;
84
85		key = &sin->sin_addr;
86		len = sizeof(sin->sin_addr);
87	} else {
88		const struct sockaddr_in6 *sin6 = (const void *)sa;
89
90		key = &sin6->sin6_addr;
91		len = sizeof(sin6->sin6_addr);
92		start = half;
93	}
94
95	hash = fnv_32_buf(key, len, FNV1_32_INIT);
96	hash = fnv_32_buf(&ifindex, sizeof(ifindex), hash);
97	hash %= half;
98
99	return (hash + start);
100}
101
102static inline int
103l2_cmp(const struct sockaddr *sa, struct l2t_entry *e)
104{
105
106	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
107	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
108	    sa->sa_family));
109
110	if (sa->sa_family == AF_INET) {
111		const struct sockaddr_in *sin = (const void *)sa;
112
113		return (e->addr[0] != sin->sin_addr.s_addr);
114	} else {
115		const struct sockaddr_in6 *sin6 = (const void *)sa;
116
117		return (memcmp(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr)));
118	}
119}
120
121static inline void
122l2_store(const struct sockaddr *sa, struct l2t_entry *e)
123{
124
125	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
126	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
127	    sa->sa_family));
128
129	if (sa->sa_family == AF_INET) {
130		const struct sockaddr_in *sin = (const void *)sa;
131
132		e->addr[0] = sin->sin_addr.s_addr;
133		e->ipv6 = 0;
134	} else {
135		const struct sockaddr_in6 *sin6 = (const void *)sa;
136
137		memcpy(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr));
138		e->ipv6 = 1;
139	}
140}
141
142/*
143 * Add a WR to an L2T entry's queue of work requests awaiting resolution.
144 * Must be called with the entry's lock held.
145 */
146static inline void
147arpq_enqueue(struct l2t_entry *e, struct wrqe *wr)
148{
149	mtx_assert(&e->lock, MA_OWNED);
150
151	STAILQ_INSERT_TAIL(&e->wr_list, wr, link);
152}
153
154static inline void
155send_pending(struct adapter *sc, struct l2t_entry *e)
156{
157	struct wrqe *wr;
158
159	mtx_assert(&e->lock, MA_OWNED);
160
161	while ((wr = STAILQ_FIRST(&e->wr_list)) != NULL) {
162		STAILQ_REMOVE_HEAD(&e->wr_list, link);
163		t4_wrq_tx(sc, wr);
164	}
165}
166
167static void
168resolution_failed(struct adapter *sc, struct l2t_entry *e)
169{
170	struct tom_data *td = sc->tom_softc;
171
172	mtx_assert(&e->lock, MA_OWNED);
173
174	mtx_lock(&td->unsent_wr_lock);
175	STAILQ_CONCAT(&td->unsent_wr_list, &e->wr_list);
176	mtx_unlock(&td->unsent_wr_lock);
177
178	taskqueue_enqueue(taskqueue_thread, &td->reclaim_wr_resources);
179}
180
181static void
182update_entry(struct adapter *sc, struct l2t_entry *e, uint8_t *lladdr,
183    uint16_t vtag)
184{
185
186	mtx_assert(&e->lock, MA_OWNED);
187
188	/*
189	 * The entry may be in active use (e->refcount > 0) or not.  We update
190	 * it even when it's not as this simplifies the case where we decide to
191	 * reuse the entry later.
192	 */
193
194	if (lladdr == NULL &&
195	    (e->state == L2T_STATE_RESOLVING || e->state == L2T_STATE_FAILED)) {
196		/*
197		 * Never got a valid L2 address for this one.  Just mark it as
198		 * failed instead of removing it from the hash (for which we'd
199		 * need to wlock the table).
200		 */
201		e->state = L2T_STATE_FAILED;
202		resolution_failed(sc, e);
203		return;
204
205	} else if (lladdr == NULL) {
206
207		/* Valid or already-stale entry was deleted (or expired) */
208
209		KASSERT(e->state == L2T_STATE_VALID ||
210		    e->state == L2T_STATE_STALE,
211		    ("%s: lladdr NULL, state %d", __func__, e->state));
212
213		e->state = L2T_STATE_STALE;
214
215	} else {
216
217		if (e->state == L2T_STATE_RESOLVING ||
218		    e->state == L2T_STATE_FAILED ||
219		    memcmp(e->dmac, lladdr, ETHER_ADDR_LEN)) {
220
221			/* unresolved -> resolved; or dmac changed */
222
223			memcpy(e->dmac, lladdr, ETHER_ADDR_LEN);
224			e->vlan = vtag;
225			t4_write_l2e(e, 1);
226		}
227		e->state = L2T_STATE_VALID;
228	}
229}
230
231static int
232resolve_entry(struct adapter *sc, struct l2t_entry *e)
233{
234	struct tom_data *td = sc->tom_softc;
235	struct toedev *tod = &td->tod;
236	struct sockaddr_in sin = {0};
237	struct sockaddr_in6 sin6 = {0};
238	struct sockaddr *sa;
239	uint8_t dmac[ETHER_HDR_LEN];
240	uint16_t vtag;
241	int rc;
242
243	if (e->ipv6 == 0) {
244		sin.sin_family = AF_INET;
245		sin.sin_len = sizeof(struct sockaddr_in);
246		sin.sin_addr.s_addr = e->addr[0];
247		sa = (void *)&sin;
248	} else {
249		sin6.sin6_family = AF_INET6;
250		sin6.sin6_len = sizeof(struct sockaddr_in6);
251		memcpy(&sin6.sin6_addr, &e->addr[0], sizeof(e->addr));
252		sa = (void *)&sin6;
253	}
254
255	vtag = EVL_MAKETAG(VLAN_NONE, 0, 0);
256	rc = toe_l2_resolve(tod, e->ifp, sa, dmac, &vtag);
257	if (rc == EWOULDBLOCK)
258		return (rc);
259
260	mtx_lock(&e->lock);
261	update_entry(sc, e, rc == 0 ? dmac : NULL, vtag);
262	mtx_unlock(&e->lock);
263
264	return (rc);
265}
266
267int
268t4_l2t_send_slow(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e)
269{
270
271again:
272	switch (e->state) {
273	case L2T_STATE_STALE:     /* entry is stale, kick off revalidation */
274
275		resolve_entry(sc, e);
276
277		/* Fall through */
278
279	case L2T_STATE_VALID:     /* fast-path, send the packet on */
280
281		t4_wrq_tx(sc, wr);
282		return (0);
283
284	case L2T_STATE_RESOLVING:
285	case L2T_STATE_SYNC_WRITE:
286
287		mtx_lock(&e->lock);
288		if (e->state != L2T_STATE_SYNC_WRITE &&
289		    e->state != L2T_STATE_RESOLVING) {
290			/* state changed by the time we got here */
291			mtx_unlock(&e->lock);
292			goto again;
293		}
294		arpq_enqueue(e, wr);
295		mtx_unlock(&e->lock);
296
297		if (resolve_entry(sc, e) == EWOULDBLOCK)
298			break;
299
300		mtx_lock(&e->lock);
301		if (e->state == L2T_STATE_VALID && !STAILQ_EMPTY(&e->wr_list))
302			send_pending(sc, e);
303		if (e->state == L2T_STATE_FAILED)
304			resolution_failed(sc, e);
305		mtx_unlock(&e->lock);
306		break;
307
308	case L2T_STATE_FAILED:
309		return (EHOSTUNREACH);
310	}
311
312	return (0);
313}
314
315int
316do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss,
317    struct mbuf *m)
318{
319	struct adapter *sc = iq->adapter;
320	const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
321	unsigned int tid = GET_TID(rpl);
322	unsigned int idx = tid % L2T_SIZE;
323
324	if (__predict_false(rpl->status != CPL_ERR_NONE)) {
325		log(LOG_ERR,
326		    "Unexpected L2T_WRITE_RPL (%u) for entry at hw_idx %u\n",
327		    rpl->status, idx);
328		return (EINVAL);
329	}
330
331	if (tid & F_SYNC_WR) {
332		struct l2t_entry *e = &sc->l2t->l2tab[idx - sc->vres.l2t.start];
333
334		mtx_lock(&e->lock);
335		if (e->state != L2T_STATE_SWITCHING) {
336			send_pending(sc, e);
337			e->state = L2T_STATE_VALID;
338		}
339		mtx_unlock(&e->lock);
340	}
341
342	return (0);
343}
344
345/*
346 * The TOE wants an L2 table entry that it can use to reach the next hop over
347 * the specified port.  Produce such an entry - create one if needed.
348 *
349 * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on
350 * top of the real cxgbe interface.
351 */
352struct l2t_entry *
353t4_l2t_get(struct port_info *pi, if_t ifp, struct sockaddr *sa)
354{
355	struct l2t_entry *e;
356	struct adapter *sc = pi->adapter;
357	struct l2t_data *d = sc->l2t;
358	u_int hash, smt_idx = pi->port_id;
359	uint16_t vid, pcp, vtag;
360
361	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
362	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
363	    sa->sa_family));
364
365	vid = VLAN_NONE;
366	pcp = 0;
367	if (if_gettype(ifp) == IFT_L2VLAN) {
368		VLAN_TAG(ifp, &vid);
369		VLAN_PCP(ifp, &pcp);
370	} else if ((pcp = if_getpcp(ifp)) != IFNET_PCP_NONE)
371		vid = 0;
372	else
373		pcp = 0;
374	vtag = EVL_MAKETAG(vid, pcp, 0);
375
376	hash = l2_hash(d, sa, if_getindex(ifp));
377	rw_wlock(&d->lock);
378	for (e = d->l2tab[hash].first; e; e = e->next) {
379		if (l2_cmp(sa, e) == 0 && e->ifp == ifp && e->vlan == vtag &&
380		    e->smt_idx == smt_idx) {
381			l2t_hold(d, e);
382			goto done;
383		}
384	}
385
386	/* Need to allocate a new entry */
387	e = t4_alloc_l2e(d);
388	if (e) {
389		mtx_lock(&e->lock);          /* avoid race with t4_l2t_free */
390		e->next = d->l2tab[hash].first;
391		d->l2tab[hash].first = e;
392
393		e->state = L2T_STATE_RESOLVING;
394		l2_store(sa, e);
395		e->ifp = ifp;
396		e->smt_idx = smt_idx;
397		e->hash = hash;
398		e->lport = pi->lport;
399		e->wrq = &sc->sge.ctrlq[pi->port_id];
400		e->iqid = sc->sge.ofld_rxq[pi->vi[0].first_ofld_rxq].iq.abs_id;
401		atomic_store_rel_int(&e->refcnt, 1);
402		e->vlan = vtag;
403		mtx_unlock(&e->lock);
404	}
405done:
406	rw_wunlock(&d->lock);
407	return e;
408}
409
410/*
411 * Called when the host's ARP layer makes a change to some entry that is loaded
412 * into the HW L2 table.
413 */
414void
415t4_l2_update(struct toedev *tod, if_t ifp, struct sockaddr *sa,
416    uint8_t *lladdr, uint16_t vtag)
417{
418	struct adapter *sc = tod->tod_softc;
419	struct l2t_entry *e;
420	struct l2t_data *d = sc->l2t;
421	u_int hash;
422
423	KASSERT(d != NULL, ("%s: no L2 table", __func__));
424
425	hash = l2_hash(d, sa, if_getindex(ifp));
426	rw_rlock(&d->lock);
427	for (e = d->l2tab[hash].first; e; e = e->next) {
428		if (l2_cmp(sa, e) == 0 && e->ifp == ifp) {
429			mtx_lock(&e->lock);
430			if (atomic_load_acq_int(&e->refcnt))
431				goto found;
432			e->state = L2T_STATE_STALE;
433			mtx_unlock(&e->lock);
434			break;
435		}
436	}
437	rw_runlock(&d->lock);
438
439	/*
440	 * This is of no interest to us.  We've never had an offloaded
441	 * connection to this destination, and we aren't attempting one right
442	 * now.
443	 */
444	return;
445
446found:
447	rw_runlock(&d->lock);
448
449	KASSERT(e->state != L2T_STATE_UNUSED,
450	    ("%s: unused entry in the hash.", __func__));
451
452	update_entry(sc, e, lladdr, vtag);
453	mtx_unlock(&e->lock);
454}
455#endif
456