1/*-
2 * Copyright (c) 2012 Chelsio Communications, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD$");
28
29#include "opt_inet.h"
30#include "opt_inet6.h"
31
32#ifdef TCP_OFFLOAD
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/bus.h>
38#include <sys/fnv_hash.h>
39#include <sys/lock.h>
40#include <sys/mutex.h>
41#include <sys/rwlock.h>
42#include <sys/socket.h>
43#include <sys/sbuf.h>
44#include <sys/taskqueue.h>
45#include <net/if.h>
46#include <net/if_types.h>
47#include <net/ethernet.h>
48#include <net/if_vlan_var.h>
49#include <net/route.h>
50#include <netinet/in.h>
51#include <netinet/toecore.h>
52
53#include "common/common.h"
54#include "common/t4_msg.h"
55#include "tom/t4_tom_l2t.h"
56#include "tom/t4_tom.h"
57
58#define VLAN_NONE	0xfff
59
60static inline void
61l2t_hold(struct l2t_data *d, struct l2t_entry *e)
62{
63
64	if (atomic_fetchadd_int(&e->refcnt, 1) == 0)  /* 0 -> 1 transition */
65		atomic_subtract_int(&d->nfree, 1);
66}
67
68static inline u_int
69l2_hash(struct l2t_data *d, const struct sockaddr *sa, int ifindex)
70{
71	u_int hash, half = d->l2t_size / 2, start = 0;
72	const void *key;
73	size_t len;
74
75	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
76	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
77	    sa->sa_family));
78
79	if (sa->sa_family == AF_INET) {
80		const struct sockaddr_in *sin = (const void *)sa;
81
82		key = &sin->sin_addr;
83		len = sizeof(sin->sin_addr);
84	} else {
85		const struct sockaddr_in6 *sin6 = (const void *)sa;
86
87		key = &sin6->sin6_addr;
88		len = sizeof(sin6->sin6_addr);
89		start = half;
90	}
91
92	hash = fnv_32_buf(key, len, FNV1_32_INIT);
93	hash = fnv_32_buf(&ifindex, sizeof(ifindex), hash);
94	hash %= half;
95
96	return (hash + start);
97}
98
99static inline int
100l2_cmp(const struct sockaddr *sa, struct l2t_entry *e)
101{
102
103	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
104	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
105	    sa->sa_family));
106
107	if (sa->sa_family == AF_INET) {
108		const struct sockaddr_in *sin = (const void *)sa;
109
110		return (e->addr[0] != sin->sin_addr.s_addr);
111	} else {
112		const struct sockaddr_in6 *sin6 = (const void *)sa;
113
114		return (memcmp(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr)));
115	}
116}
117
118static inline void
119l2_store(const struct sockaddr *sa, struct l2t_entry *e)
120{
121
122	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
123	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
124	    sa->sa_family));
125
126	if (sa->sa_family == AF_INET) {
127		const struct sockaddr_in *sin = (const void *)sa;
128
129		e->addr[0] = sin->sin_addr.s_addr;
130		e->ipv6 = 0;
131	} else {
132		const struct sockaddr_in6 *sin6 = (const void *)sa;
133
134		memcpy(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr));
135		e->ipv6 = 1;
136	}
137}
138
139/*
140 * Add a WR to an L2T entry's queue of work requests awaiting resolution.
141 * Must be called with the entry's lock held.
142 */
143static inline void
144arpq_enqueue(struct l2t_entry *e, struct wrqe *wr)
145{
146	mtx_assert(&e->lock, MA_OWNED);
147
148	STAILQ_INSERT_TAIL(&e->wr_list, wr, link);
149}
150
151static inline void
152send_pending(struct adapter *sc, struct l2t_entry *e)
153{
154	struct wrqe *wr;
155
156	mtx_assert(&e->lock, MA_OWNED);
157
158	while ((wr = STAILQ_FIRST(&e->wr_list)) != NULL) {
159		STAILQ_REMOVE_HEAD(&e->wr_list, link);
160		t4_wrq_tx(sc, wr);
161	}
162}
163
164static void
165resolution_failed(struct adapter *sc, struct l2t_entry *e)
166{
167	struct tom_data *td = sc->tom_softc;
168
169	mtx_assert(&e->lock, MA_OWNED);
170
171	mtx_lock(&td->unsent_wr_lock);
172	STAILQ_CONCAT(&td->unsent_wr_list, &e->wr_list);
173	mtx_unlock(&td->unsent_wr_lock);
174
175	taskqueue_enqueue(taskqueue_thread, &td->reclaim_wr_resources);
176}
177
178static void
179update_entry(struct adapter *sc, struct l2t_entry *e, uint8_t *lladdr,
180    uint16_t vtag)
181{
182
183	mtx_assert(&e->lock, MA_OWNED);
184
185	/*
186	 * The entry may be in active use (e->refcount > 0) or not.  We update
187	 * it even when it's not as this simplifies the case where we decide to
188	 * reuse the entry later.
189	 */
190
191	if (lladdr == NULL &&
192	    (e->state == L2T_STATE_RESOLVING || e->state == L2T_STATE_FAILED)) {
193		/*
194		 * Never got a valid L2 address for this one.  Just mark it as
195		 * failed instead of removing it from the hash (for which we'd
196		 * need to wlock the table).
197		 */
198		e->state = L2T_STATE_FAILED;
199		resolution_failed(sc, e);
200		return;
201
202	} else if (lladdr == NULL) {
203
204		/* Valid or already-stale entry was deleted (or expired) */
205
206		KASSERT(e->state == L2T_STATE_VALID ||
207		    e->state == L2T_STATE_STALE,
208		    ("%s: lladdr NULL, state %d", __func__, e->state));
209
210		e->state = L2T_STATE_STALE;
211
212	} else {
213
214		if (e->state == L2T_STATE_RESOLVING ||
215		    e->state == L2T_STATE_FAILED ||
216		    memcmp(e->dmac, lladdr, ETHER_ADDR_LEN)) {
217
218			/* unresolved -> resolved; or dmac changed */
219
220			memcpy(e->dmac, lladdr, ETHER_ADDR_LEN);
221			e->vlan = vtag;
222			t4_write_l2e(e, 1);
223		}
224		e->state = L2T_STATE_VALID;
225	}
226}
227
228static int
229resolve_entry(struct adapter *sc, struct l2t_entry *e)
230{
231	struct tom_data *td = sc->tom_softc;
232	struct toedev *tod = &td->tod;
233	struct sockaddr_in sin = {0};
234	struct sockaddr_in6 sin6 = {0};
235	struct sockaddr *sa;
236	uint8_t dmac[ETHER_HDR_LEN];
237	uint16_t vtag = VLAN_NONE;
238	int rc;
239
240	if (e->ipv6 == 0) {
241		sin.sin_family = AF_INET;
242		sin.sin_len = sizeof(struct sockaddr_in);
243		sin.sin_addr.s_addr = e->addr[0];
244		sa = (void *)&sin;
245	} else {
246		sin6.sin6_family = AF_INET6;
247		sin6.sin6_len = sizeof(struct sockaddr_in6);
248		memcpy(&sin6.sin6_addr, &e->addr[0], sizeof(e->addr));
249		sa = (void *)&sin6;
250	}
251
252	rc = toe_l2_resolve(tod, e->ifp, sa, dmac, &vtag);
253	if (rc == EWOULDBLOCK)
254		return (rc);
255
256	mtx_lock(&e->lock);
257	update_entry(sc, e, rc == 0 ? dmac : NULL, vtag);
258	mtx_unlock(&e->lock);
259
260	return (rc);
261}
262
263int
264t4_l2t_send_slow(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e)
265{
266
267again:
268	switch (e->state) {
269	case L2T_STATE_STALE:     /* entry is stale, kick off revalidation */
270
271		if (resolve_entry(sc, e) != EWOULDBLOCK)
272			goto again;	/* entry updated, re-examine state */
273
274		/* Fall through */
275
276	case L2T_STATE_VALID:     /* fast-path, send the packet on */
277
278		t4_wrq_tx(sc, wr);
279		return (0);
280
281	case L2T_STATE_RESOLVING:
282	case L2T_STATE_SYNC_WRITE:
283
284		mtx_lock(&e->lock);
285		if (e->state != L2T_STATE_SYNC_WRITE &&
286		    e->state != L2T_STATE_RESOLVING) {
287			/* state changed by the time we got here */
288			mtx_unlock(&e->lock);
289			goto again;
290		}
291		arpq_enqueue(e, wr);
292		mtx_unlock(&e->lock);
293
294		if (resolve_entry(sc, e) == EWOULDBLOCK)
295			break;
296
297		mtx_lock(&e->lock);
298		if (e->state == L2T_STATE_VALID && !STAILQ_EMPTY(&e->wr_list))
299			send_pending(sc, e);
300		if (e->state == L2T_STATE_FAILED)
301			resolution_failed(sc, e);
302		mtx_unlock(&e->lock);
303		break;
304
305	case L2T_STATE_FAILED:
306		return (EHOSTUNREACH);
307	}
308
309	return (0);
310}
311
312int
313do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss,
314    struct mbuf *m)
315{
316	struct adapter *sc = iq->adapter;
317	const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
318	unsigned int tid = GET_TID(rpl);
319	unsigned int idx = tid % L2T_SIZE;
320
321	if (__predict_false(rpl->status != CPL_ERR_NONE)) {
322		log(LOG_ERR,
323		    "Unexpected L2T_WRITE_RPL (%u) for entry at hw_idx %u\n",
324		    rpl->status, idx);
325		return (EINVAL);
326	}
327
328	if (tid & F_SYNC_WR) {
329		struct l2t_entry *e = &sc->l2t->l2tab[idx - sc->vres.l2t.start];
330
331		mtx_lock(&e->lock);
332		if (e->state != L2T_STATE_SWITCHING) {
333			send_pending(sc, e);
334			e->state = L2T_STATE_VALID;
335		}
336		mtx_unlock(&e->lock);
337	}
338
339	return (0);
340}
341
342/*
343 * The TOE wants an L2 table entry that it can use to reach the next hop over
344 * the specified port.  Produce such an entry - create one if needed.
345 *
346 * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on
347 * top of the real cxgbe interface.
348 */
349struct l2t_entry *
350t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa)
351{
352	struct l2t_entry *e;
353	struct adapter *sc = pi->adapter;
354	struct l2t_data *d = sc->l2t;
355	u_int hash, smt_idx = pi->port_id;
356
357	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
358	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
359	    sa->sa_family));
360
361#ifndef VLAN_TAG
362	if (ifp->if_type == IFT_L2VLAN)
363		return (NULL);
364#endif
365
366	hash = l2_hash(d, sa, ifp->if_index);
367	rw_wlock(&d->lock);
368	for (e = d->l2tab[hash].first; e; e = e->next) {
369		if (l2_cmp(sa, e) == 0 && e->ifp == ifp &&
370		    e->smt_idx == smt_idx) {
371			l2t_hold(d, e);
372			goto done;
373		}
374	}
375
376	/* Need to allocate a new entry */
377	e = t4_alloc_l2e(d);
378	if (e) {
379		mtx_lock(&e->lock);          /* avoid race with t4_l2t_free */
380		e->next = d->l2tab[hash].first;
381		d->l2tab[hash].first = e;
382
383		e->state = L2T_STATE_RESOLVING;
384		l2_store(sa, e);
385		e->ifp = ifp;
386		e->smt_idx = smt_idx;
387		e->hash = hash;
388		e->lport = pi->lport;
389		e->wrq = &sc->sge.ctrlq[pi->port_id];
390		e->iqid = sc->sge.ofld_rxq[pi->vi[0].first_ofld_rxq].iq.abs_id;
391		atomic_store_rel_int(&e->refcnt, 1);
392#ifdef VLAN_TAG
393		if (ifp->if_type == IFT_L2VLAN)
394			VLAN_TAG(ifp, &e->vlan);
395		else
396			e->vlan = VLAN_NONE;
397#endif
398		mtx_unlock(&e->lock);
399	}
400done:
401	rw_wunlock(&d->lock);
402	return e;
403}
404
405/*
406 * Called when the host's ARP layer makes a change to some entry that is loaded
407 * into the HW L2 table.
408 */
409void
410t4_l2_update(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa,
411    uint8_t *lladdr, uint16_t vtag)
412{
413	struct adapter *sc = tod->tod_softc;
414	struct l2t_entry *e;
415	struct l2t_data *d = sc->l2t;
416	u_int hash;
417
418	KASSERT(d != NULL, ("%s: no L2 table", __func__));
419
420	hash = l2_hash(d, sa, ifp->if_index);
421	rw_rlock(&d->lock);
422	for (e = d->l2tab[hash].first; e; e = e->next) {
423		if (l2_cmp(sa, e) == 0 && e->ifp == ifp) {
424			mtx_lock(&e->lock);
425			if (atomic_load_acq_int(&e->refcnt))
426				goto found;
427			e->state = L2T_STATE_STALE;
428			mtx_unlock(&e->lock);
429			break;
430		}
431	}
432	rw_runlock(&d->lock);
433
434	/*
435	 * This is of no interest to us.  We've never had an offloaded
436	 * connection to this destination, and we aren't attempting one right
437	 * now.
438	 */
439	return;
440
441found:
442	rw_runlock(&d->lock);
443
444	KASSERT(e->state != L2T_STATE_UNUSED,
445	    ("%s: unused entry in the hash.", __func__));
446
447	update_entry(sc, e, lladdr, vtag);
448	mtx_unlock(&e->lock);
449}
450#endif
451