ntb_transport.c revision 289152
1250079Scarl/*-
2250079Scarl * Copyright (C) 2013 Intel Corporation
3250079Scarl * All rights reserved.
4250079Scarl *
5250079Scarl * Redistribution and use in source and binary forms, with or without
6250079Scarl * modification, are permitted provided that the following conditions
7250079Scarl * are met:
8250079Scarl * 1. Redistributions of source code must retain the above copyright
9250079Scarl *    notice, this list of conditions and the following disclaimer.
10250079Scarl * 2. Redistributions in binary form must reproduce the above copyright
11250079Scarl *    notice, this list of conditions and the following disclaimer in the
12250079Scarl *    documentation and/or other materials provided with the distribution.
13250079Scarl *
14250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17250079Scarl * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24250079Scarl * SUCH DAMAGE.
25250079Scarl */
26250079Scarl
27250079Scarl#include <sys/cdefs.h>
28250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/if_ntb/if_ntb.c 289152 2015-10-11 20:57:09Z cem $");
29250079Scarl
30250079Scarl#include <sys/param.h>
31250079Scarl#include <sys/kernel.h>
32250079Scarl#include <sys/systm.h>
33250079Scarl#include <sys/bus.h>
34250079Scarl#include <sys/ktr.h>
35250079Scarl#include <sys/lock.h>
36250079Scarl#include <sys/malloc.h>
37250079Scarl#include <sys/module.h>
38250079Scarl#include <sys/mutex.h>
39250079Scarl#include <sys/queue.h>
40250079Scarl#include <sys/socket.h>
41250079Scarl#include <sys/sockio.h>
42250079Scarl#include <sys/taskqueue.h>
43250079Scarl#include <net/if.h>
44250079Scarl#include <net/if_media.h>
45250079Scarl#include <net/if_types.h>
46250079Scarl#include <net/if_var.h>
47250079Scarl#include <net/bpf.h>
48250079Scarl#include <net/ethernet.h>
49250079Scarl#include <vm/vm.h>
50250079Scarl#include <vm/pmap.h>
51250079Scarl#include <machine/bus.h>
52250079Scarl#include <machine/cpufunc.h>
53250079Scarl#include <machine/pmap.h>
54250079Scarl
55250079Scarl#include "../ntb_hw/ntb_hw.h"
56250079Scarl
57250079Scarl/*
58250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that
59250079Scarl * allows you to connect two systems using a PCI-e link.
60250079Scarl *
61250079Scarl * This module contains a protocol for sending and receiving messages, and
62250079Scarl * exposes that protocol through a simulated ethernet device called ntb.
63250079Scarl *
64250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may
65250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license.
66250079Scarl */
67250079Scarl
68250079Scarl/* TODO: These functions should really be part of the kernel */
69250079Scarl#define test_bit(pos, bitmap_addr)  (*(bitmap_addr) & 1UL << (pos))
70250079Scarl#define set_bit(pos, bitmap_addr)   *(bitmap_addr) |= 1UL << (pos)
71250079Scarl#define clear_bit(pos, bitmap_addr) *(bitmap_addr) &= ~(1UL << (pos))
72250079Scarl
73250079Scarl#define KTR_NTB KTR_SPARE3
74250079Scarl
75250079Scarl#define NTB_TRANSPORT_VERSION	3
76250079Scarl#define NTB_RX_MAX_PKTS		64
77250079Scarl#define	NTB_RXQ_SIZE		300
78250079Scarl
79250079Scarlstatic unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN;
80250079Scarlstatic unsigned int max_num_clients = 1;
81250079Scarl
82250079ScarlSTAILQ_HEAD(ntb_queue_list, ntb_queue_entry);
83250079Scarl
84250079Scarlstruct ntb_queue_entry {
85250079Scarl	/* ntb_queue list reference */
86250079Scarl	STAILQ_ENTRY(ntb_queue_entry) entry;
87250079Scarl
88250079Scarl	/* info on data to be transfered */
89250079Scarl	void		*cb_data;
90250079Scarl	void		*buf;
91250079Scarl	uint64_t	len;
92250079Scarl	uint64_t	flags;
93250079Scarl};
94250079Scarl
95250079Scarlstruct ntb_rx_info {
96250079Scarl	unsigned int entry;
97250079Scarl};
98250079Scarl
99250079Scarlstruct ntb_transport_qp {
100250079Scarl	struct ntb_netdev	*transport;
101250079Scarl	struct ntb_softc	*ntb;
102250079Scarl
103250079Scarl	void			*cb_data;
104250079Scarl
105250079Scarl	bool			client_ready;
106250079Scarl	bool			qp_link;
107255281Scarl	uint8_t			qp_num;	/* Only 64 QPs are allowed.  0-63 */
108250079Scarl
109250079Scarl	struct ntb_rx_info	*rx_info;
110250079Scarl	struct ntb_rx_info	*remote_rx_info;
111250079Scarl
112250079Scarl	void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
113250079Scarl	    void *data, int len);
114250079Scarl	struct ntb_queue_list	tx_free_q;
115250079Scarl	struct mtx		ntb_tx_free_q_lock;
116250079Scarl	void			*tx_mw;
117250079Scarl	uint64_t		tx_index;
118250079Scarl	uint64_t		tx_max_entry;
119250079Scarl	uint64_t		tx_max_frame;
120250079Scarl
121250079Scarl	void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
122250079Scarl	    void *data, int len);
123250079Scarl	struct ntb_queue_list	rx_pend_q;
124250079Scarl	struct ntb_queue_list	rx_free_q;
125250079Scarl	struct mtx		ntb_rx_pend_q_lock;
126250079Scarl	struct mtx		ntb_rx_free_q_lock;
127250079Scarl	struct task		rx_completion_task;
128250079Scarl	void			*rx_buff;
129250079Scarl	uint64_t		rx_index;
130250079Scarl	uint64_t		rx_max_entry;
131250079Scarl	uint64_t		rx_max_frame;
132250079Scarl
133250079Scarl	void (*event_handler) (void *data, int status);
134250079Scarl	struct callout		link_work;
135250079Scarl	struct callout		queue_full;
136250079Scarl	struct callout		rx_full;
137250079Scarl
138250079Scarl	uint64_t		last_rx_no_buf;
139250079Scarl
140250079Scarl	/* Stats */
141250079Scarl	uint64_t		rx_bytes;
142250079Scarl	uint64_t		rx_pkts;
143250079Scarl	uint64_t		rx_ring_empty;
144250079Scarl	uint64_t		rx_err_no_buf;
145250079Scarl	uint64_t		rx_err_oflow;
146250079Scarl	uint64_t		rx_err_ver;
147250079Scarl	uint64_t		tx_bytes;
148250079Scarl	uint64_t		tx_pkts;
149250079Scarl	uint64_t		tx_ring_full;
150250079Scarl};
151250079Scarl
152250079Scarlstruct ntb_queue_handlers {
153250079Scarl	void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
154250079Scarl	    void *data, int len);
155250079Scarl	void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
156250079Scarl	    void *data, int len);
157250079Scarl	void (*event_handler) (void *data, int status);
158250079Scarl};
159250079Scarl
160250079Scarl
161250079Scarlstruct ntb_transport_mw {
162250079Scarl	size_t		size;
163250079Scarl	void		*virt_addr;
164250079Scarl	vm_paddr_t	dma_addr;
165250079Scarl};
166250079Scarl
167250079Scarlstruct ntb_netdev {
168250079Scarl	struct ntb_softc	*ntb;
169250079Scarl	struct ifnet		*ifp;
170250079Scarl	struct ntb_transport_mw	mw[NTB_NUM_MW];
171250079Scarl	struct ntb_transport_qp	*qps;
172250079Scarl	uint64_t		max_qps;
173250079Scarl	uint64_t		qp_bitmap;
174250079Scarl	bool			transport_link;
175250079Scarl	struct callout		link_work;
176250079Scarl	struct ntb_transport_qp *qp;
177250079Scarl	uint64_t		bufsize;
178250079Scarl	u_char			eaddr[ETHER_ADDR_LEN];
179250079Scarl	struct mtx		tx_lock;
180250079Scarl	struct mtx		rx_lock;
181250079Scarl};
182250079Scarl
183250079Scarlstatic struct ntb_netdev net_softc;
184250079Scarl
185250079Scarlenum {
186250079Scarl	IF_NTB_DESC_DONE_FLAG = 1 << 0,
187250079Scarl	IF_NTB_LINK_DOWN_FLAG = 1 << 1,
188250079Scarl};
189250079Scarl
190250079Scarlstruct ntb_payload_header {
191250079Scarl	uint64_t ver;
192250079Scarl	uint64_t len;
193250079Scarl	uint64_t flags;
194250079Scarl};
195250079Scarl
196250079Scarlenum {
197250079Scarl	IF_NTB_VERSION = 0,
198250079Scarl	IF_NTB_MW0_SZ,
199250079Scarl	IF_NTB_MW1_SZ,
200250079Scarl	IF_NTB_NUM_QPS,
201250079Scarl	IF_NTB_QP_LINKS,
202250079Scarl	IF_NTB_MAX_SPAD,
203250079Scarl};
204250079Scarl
205250079Scarl#define QP_TO_MW(qp)		((qp) % NTB_NUM_MW)
206250079Scarl#define NTB_QP_DEF_NUM_ENTRIES	100
207250079Scarl#define NTB_LINK_DOWN_TIMEOUT	10
208250079Scarl
209250079Scarlstatic int ntb_handle_module_events(struct module *m, int what, void *arg);
210250079Scarlstatic int ntb_setup_interface(void);
211250079Scarlstatic int ntb_teardown_interface(void);
212250079Scarlstatic void ntb_net_init(void *arg);
213250079Scarlstatic int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
214250079Scarlstatic void ntb_start(struct ifnet *ifp);
215250079Scarlstatic void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data,
216250079Scarl    void *data, int len);
217250079Scarlstatic void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
218250079Scarl    void *data, int len);
219250079Scarlstatic void ntb_net_event_handler(void *data, int status);
220250079Scarlstatic int ntb_transport_init(struct ntb_softc *ntb);
221250079Scarlstatic void ntb_transport_free(void *transport);
222250079Scarlstatic void ntb_transport_init_queue(struct ntb_netdev *nt,
223250079Scarl    unsigned int qp_num);
224250079Scarlstatic void ntb_transport_free_queue(struct ntb_transport_qp *qp);
225250079Scarlstatic struct ntb_transport_qp * ntb_transport_create_queue(void *data,
226250079Scarl    struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers);
227250079Scarlstatic void ntb_transport_link_up(struct ntb_transport_qp *qp);
228250079Scarlstatic int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb,
229250079Scarl    void *data, unsigned int len);
230250079Scarlstatic int ntb_process_tx(struct ntb_transport_qp *qp,
231250079Scarl    struct ntb_queue_entry *entry);
232250079Scarlstatic void ntb_tx_copy_task(struct ntb_transport_qp *qp,
233250079Scarl    struct ntb_queue_entry *entry, void *offset);
234250079Scarlstatic void ntb_qp_full(void *arg);
235250079Scarlstatic void ntb_transport_rxc_db(void *data, int db_num);
236250079Scarlstatic void ntb_rx_pendq_full(void *arg);
237250079Scarlstatic void ntb_transport_rx(struct ntb_transport_qp *qp);
238250079Scarlstatic int ntb_process_rxc(struct ntb_transport_qp *qp);
239250079Scarlstatic void ntb_rx_copy_task(struct ntb_transport_qp *qp,
240250079Scarl    struct ntb_queue_entry *entry, void *offset);
241250079Scarlstatic void ntb_rx_completion_task(void *arg, int pending);
242250079Scarlstatic void ntb_transport_event_callback(void *data, enum ntb_hw_event event);
243250079Scarlstatic void ntb_transport_link_work(void *arg);
244250079Scarlstatic int ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size);
245250079Scarlstatic void ntb_transport_setup_qp_mw(struct ntb_netdev *nt,
246250079Scarl    unsigned int qp_num);
247250079Scarlstatic void ntb_qp_link_work(void *arg);
248250079Scarlstatic void ntb_transport_link_cleanup(struct ntb_netdev *nt);
249250079Scarlstatic void ntb_qp_link_down(struct ntb_transport_qp *qp);
250250079Scarlstatic void ntb_qp_link_cleanup(struct ntb_transport_qp *qp);
251250079Scarlstatic void ntb_transport_link_down(struct ntb_transport_qp *qp);
252250079Scarlstatic void ntb_send_link_down(struct ntb_transport_qp *qp);
253250079Scarlstatic void ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry,
254250079Scarl    struct ntb_queue_list *list);
255250079Scarlstatic struct ntb_queue_entry *ntb_list_rm(struct mtx *lock,
256250079Scarl    struct ntb_queue_list *list);
257250079Scarlstatic void create_random_local_eui48(u_char *eaddr);
258250079Scarlstatic unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp);
259250079Scarl
260250079ScarlMALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver");
261250079Scarl
262250079Scarl/* Module setup and teardown */
263250079Scarlstatic int
264250079Scarlntb_handle_module_events(struct module *m, int what, void *arg)
265250079Scarl{
266250079Scarl	int err = 0;
267250079Scarl
268250079Scarl	switch (what) {
269250079Scarl	case MOD_LOAD:
270250079Scarl		err = ntb_setup_interface();
271250079Scarl		break;
272250079Scarl	case MOD_UNLOAD:
273250079Scarl		err = ntb_teardown_interface();
274250079Scarl		break;
275250079Scarl	default:
276250079Scarl		err = EOPNOTSUPP;
277250079Scarl		break;
278250079Scarl	}
279250079Scarl	return (err);
280250079Scarl}
281250079Scarl
282255271Scarlstatic moduledata_t if_ntb_mod = {
283255271Scarl	"if_ntb",
284250079Scarl	ntb_handle_module_events,
285250079Scarl	NULL
286250079Scarl};
287250079Scarl
288255271ScarlDECLARE_MODULE(if_ntb, if_ntb_mod, SI_SUB_KLD, SI_ORDER_ANY);
289255271ScarlMODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1);
290250079Scarl
291250079Scarlstatic int
292250079Scarlntb_setup_interface()
293250079Scarl{
294250079Scarl	struct ifnet *ifp;
295250079Scarl	struct ntb_queue_handlers handlers = { ntb_net_rx_handler,
296250079Scarl	    ntb_net_tx_handler, ntb_net_event_handler };
297250079Scarl
298250079Scarl	net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0);
299250079Scarl	if (net_softc.ntb == NULL) {
300255281Scarl		printf("ntb: Cannot find devclass\n");
301250079Scarl		return (ENXIO);
302250079Scarl	}
303250079Scarl
304250079Scarl	ntb_transport_init(net_softc.ntb);
305250079Scarl
306250079Scarl	ifp = net_softc.ifp = if_alloc(IFT_ETHER);
307250079Scarl	if (ifp == NULL) {
308250079Scarl		printf("ntb: cannot allocate ifnet structure\n");
309250079Scarl		return (ENOMEM);
310250079Scarl	}
311250079Scarl
312250079Scarl	net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb,
313250079Scarl	    &handlers);
314250079Scarl	if_initname(ifp, "ntb", 0);
315250079Scarl	ifp->if_init = ntb_net_init;
316250079Scarl	ifp->if_softc = &net_softc;
317250079Scarl	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX;
318250079Scarl	ifp->if_ioctl = ntb_ioctl;
319250079Scarl	ifp->if_start = ntb_start;
320250079Scarl	IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN);
321250079Scarl	ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN;
322250079Scarl	IFQ_SET_READY(&ifp->if_snd);
323250079Scarl	create_random_local_eui48(net_softc.eaddr);
324250079Scarl	ether_ifattach(ifp, net_softc.eaddr);
325250079Scarl	ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU;
326250079Scarl	ifp->if_capenable = ifp->if_capabilities;
327250079Scarl
328250079Scarl	ntb_transport_link_up(net_softc.qp);
329250079Scarl	net_softc.bufsize = ntb_transport_max_size(net_softc.qp) +
330250079Scarl	    sizeof(struct ether_header);
331250079Scarl	return (0);
332250079Scarl}
333250079Scarl
334250079Scarlstatic int
335250079Scarlntb_teardown_interface()
336250079Scarl{
337250079Scarl
338255280Scarl	if (net_softc.qp != NULL)
339255280Scarl		ntb_transport_link_down(net_softc.qp);
340250079Scarl
341255280Scarl	if (net_softc.ifp != NULL) {
342255280Scarl		ether_ifdetach(net_softc.ifp);
343255280Scarl		if_free(net_softc.ifp);
344255280Scarl	}
345250079Scarl
346255280Scarl	if (net_softc.qp != NULL) {
347255280Scarl		ntb_transport_free_queue(net_softc.qp);
348255280Scarl		ntb_transport_free(&net_softc);
349255280Scarl	}
350255280Scarl
351250079Scarl	return (0);
352250079Scarl}
353250079Scarl
354250079Scarl/* Network device interface */
355250079Scarl
356250079Scarlstatic void
357250079Scarlntb_net_init(void *arg)
358250079Scarl{
359250079Scarl	struct ntb_netdev *ntb_softc = arg;
360250079Scarl	struct ifnet *ifp = ntb_softc->ifp;
361250079Scarl
362250079Scarl	ifp->if_drv_flags |= IFF_DRV_RUNNING;
363250079Scarl	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
364250079Scarl	ifp->if_flags |= IFF_UP;
365250079Scarl	if_link_state_change(ifp, LINK_STATE_UP);
366250079Scarl}
367250079Scarl
368250079Scarlstatic int
369250079Scarlntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
370250079Scarl{
371250079Scarl	struct ntb_netdev *nt = ifp->if_softc;
372250079Scarl	struct ifreq *ifr = (struct ifreq *)data;
373250079Scarl	int error = 0;
374250079Scarl
375250079Scarl	switch (command) {
376250079Scarl	case SIOCSIFMTU:
377250079Scarl	    {
378250079Scarl		if (ifr->ifr_mtu > ntb_transport_max_size(nt->qp) -
379250079Scarl		    ETHER_HDR_LEN - ETHER_CRC_LEN) {
380250079Scarl			error = EINVAL;
381250079Scarl			break;
382250079Scarl		}
383250079Scarl
384250079Scarl		ifp->if_mtu = ifr->ifr_mtu;
385250079Scarl		break;
386250079Scarl	    }
387250079Scarl	default:
388250079Scarl		error = ether_ioctl(ifp, command, data);
389250079Scarl		break;
390250079Scarl	}
391250079Scarl
392250079Scarl	return (error);
393250079Scarl}
394250079Scarl
395250079Scarl
396250079Scarlstatic void
397250079Scarlntb_start(struct ifnet *ifp)
398250079Scarl{
399250079Scarl	struct mbuf *m_head;
400250079Scarl	struct ntb_netdev *nt = ifp->if_softc;
401250079Scarl	int rc;
402250079Scarl
403250079Scarl	mtx_lock(&nt->tx_lock);
404250079Scarl	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
405250079Scarl	CTR0(KTR_NTB, "TX: ntb_start");
406250079Scarl	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
407250079Scarl		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
408250079Scarl		CTR1(KTR_NTB, "TX: start mbuf %p", m_head);
409250079Scarl		rc = ntb_transport_tx_enqueue(nt->qp, m_head, m_head,
410250079Scarl			     m_length(m_head, NULL));
411250079Scarl		if (rc != 0) {
412250079Scarl			CTR1(KTR_NTB,
413255281Scarl			    "TX: could not tx mbuf %p. Returning to snd q",
414250079Scarl			    m_head);
415250079Scarl			if (rc == EAGAIN) {
416250079Scarl				ifp->if_drv_flags |= IFF_DRV_OACTIVE;
417250079Scarl				IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
418250079Scarl				callout_reset(&nt->qp->queue_full, hz / 1000,
419250079Scarl				    ntb_qp_full, ifp);
420250079Scarl			}
421250079Scarl			break;
422250079Scarl		}
423250079Scarl
424250079Scarl	}
425250079Scarl	mtx_unlock(&nt->tx_lock);
426250079Scarl}
427250079Scarl
428250079Scarl/* Network Device Callbacks */
429250079Scarlstatic void
430250079Scarlntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data,
431250079Scarl    int len)
432250079Scarl{
433250079Scarl
434250079Scarl	m_freem(data);
435250079Scarl	CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data);
436250079Scarl}
437250079Scarl
438250079Scarlstatic void
439250079Scarlntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data,
440250079Scarl    int len)
441250079Scarl{
442250079Scarl	struct mbuf *m = data;
443250079Scarl	struct ifnet *ifp = qp_data;
444250079Scarl
445250079Scarl	CTR0(KTR_NTB, "RX: rx handler");
446250079Scarl	(*ifp->if_input)(ifp, m);
447250079Scarl}
448250079Scarl
449250079Scarlstatic void
450250079Scarlntb_net_event_handler(void *data, int status)
451250079Scarl{
452250079Scarl
453250079Scarl}
454250079Scarl
455250079Scarl/* Transport Init and teardown */
456250079Scarl
457250079Scarlstatic int
458250079Scarlntb_transport_init(struct ntb_softc *ntb)
459250079Scarl{
460250079Scarl	struct ntb_netdev *nt = &net_softc;
461250079Scarl	int rc, i;
462250079Scarl
463250079Scarl	nt->max_qps = max_num_clients;
464250079Scarl	ntb_register_transport(ntb, nt);
465250079Scarl	mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF);
466250079Scarl	mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF);
467250079Scarl
468250079Scarl	nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp),
469250079Scarl			  M_NTB_IF, M_WAITOK|M_ZERO);
470250079Scarl
471250079Scarl	nt->qp_bitmap = ((uint64_t) 1 << nt->max_qps) - 1;
472250079Scarl
473250079Scarl	for (i = 0; i < nt->max_qps; i++)
474250079Scarl		ntb_transport_init_queue(nt, i);
475250079Scarl
476250079Scarl	callout_init(&nt->link_work, 0);
477250079Scarl
478250079Scarl	rc = ntb_register_event_callback(ntb,
479250079Scarl					 ntb_transport_event_callback);
480250079Scarl	if (rc != 0)
481250079Scarl		goto err;
482250079Scarl
483255268Scarl	if (ntb_query_link_status(ntb)) {
484255268Scarl		if (bootverbose)
485255268Scarl			device_printf(ntb_get_device(ntb), "link up\n");
486250079Scarl		callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
487255268Scarl	}
488250079Scarl
489250079Scarl	return (0);
490250079Scarl
491250079Scarlerr:
492250079Scarl	free(nt->qps, M_NTB_IF);
493250079Scarl	ntb_unregister_transport(ntb);
494250079Scarl	return (rc);
495250079Scarl}
496250079Scarl
497250079Scarlstatic void
498250079Scarlntb_transport_free(void *transport)
499250079Scarl{
500250079Scarl	struct ntb_netdev *nt = transport;
501250079Scarl	struct ntb_softc *ntb = nt->ntb;
502250079Scarl	int i;
503250079Scarl
504250079Scarl	nt->transport_link = NTB_LINK_DOWN;
505250079Scarl
506250079Scarl	callout_drain(&nt->link_work);
507250079Scarl
508255281Scarl	/* verify that all the qps are freed */
509250079Scarl	for (i = 0; i < nt->max_qps; i++)
510250079Scarl		if (!test_bit(i, &nt->qp_bitmap))
511250079Scarl			ntb_transport_free_queue(&nt->qps[i]);
512250079Scarl
513250079Scarl
514250079Scarl	ntb_unregister_event_callback(ntb);
515250079Scarl
516250079Scarl	for (i = 0; i < NTB_NUM_MW; i++)
517250079Scarl		if (nt->mw[i].virt_addr != NULL)
518250079Scarl			contigfree(nt->mw[i].virt_addr, nt->mw[i].size,
519250079Scarl					  M_NTB_IF);
520250079Scarl
521250079Scarl	free(nt->qps, M_NTB_IF);
522250079Scarl	ntb_unregister_transport(ntb);
523250079Scarl}
524250079Scarl
525250079Scarlstatic void
526250079Scarlntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num)
527250079Scarl{
528250079Scarl	struct ntb_transport_qp *qp;
529250079Scarl	unsigned int num_qps_mw, tx_size;
530250079Scarl	uint8_t mw_num = QP_TO_MW(qp_num);
531250079Scarl
532250079Scarl	qp = &nt->qps[qp_num];
533250079Scarl	qp->qp_num = qp_num;
534250079Scarl	qp->transport = nt;
535250079Scarl	qp->ntb = nt->ntb;
536250079Scarl	qp->qp_link = NTB_LINK_DOWN;
537250079Scarl	qp->client_ready = NTB_LINK_DOWN;
538250079Scarl	qp->event_handler = NULL;
539250079Scarl
540250079Scarl	if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
541250079Scarl		num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
542250079Scarl	else
543250079Scarl		num_qps_mw = nt->max_qps / NTB_NUM_MW;
544250079Scarl
545250079Scarl	tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw;
546250079Scarl	qp->rx_info = (struct ntb_rx_info *)
547250079Scarl	    ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) +
548250079Scarl	    (qp_num / NTB_NUM_MW * tx_size));
549250079Scarl	tx_size -= sizeof(struct ntb_rx_info);
550250079Scarl
551289152Scem	qp->tx_mw = qp->rx_info + 1;
552250079Scarl	qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
553250079Scarl	    tx_size);
554250079Scarl	qp->tx_max_entry = tx_size / qp->tx_max_frame;
555250079Scarl	qp->tx_index = 0;
556250079Scarl
557250079Scarl	callout_init(&qp->link_work, 0);
558283291Sjkim	callout_init(&qp->queue_full, 1);
559283291Sjkim	callout_init(&qp->rx_full, 1);
560250079Scarl
561250079Scarl	mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN);
562250079Scarl	mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN);
563250079Scarl	mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN);
564250079Scarl	TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp);
565250079Scarl
566250079Scarl	STAILQ_INIT(&qp->rx_pend_q);
567250079Scarl	STAILQ_INIT(&qp->rx_free_q);
568250079Scarl	STAILQ_INIT(&qp->tx_free_q);
569250079Scarl}
570250079Scarl
571250079Scarlstatic void
572250079Scarlntb_transport_free_queue(struct ntb_transport_qp *qp)
573250079Scarl{
574250079Scarl	struct ntb_queue_entry *entry;
575250079Scarl
576250079Scarl	if (qp == NULL)
577250079Scarl		return;
578250079Scarl
579250079Scarl	callout_drain(&qp->link_work);
580250079Scarl
581250079Scarl	ntb_unregister_db_callback(qp->ntb, qp->qp_num);
582250079Scarl
583250079Scarl	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
584250079Scarl		free(entry, M_NTB_IF);
585250079Scarl
586250079Scarl	while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q)))
587250079Scarl		free(entry, M_NTB_IF);
588250079Scarl
589250079Scarl	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
590250079Scarl		free(entry, M_NTB_IF);
591250079Scarl
592250079Scarl	set_bit(qp->qp_num, &qp->transport->qp_bitmap);
593250079Scarl}
594250079Scarl
595250079Scarl/**
596250079Scarl * ntb_transport_create_queue - Create a new NTB transport layer queue
597250079Scarl * @rx_handler: receive callback function
598250079Scarl * @tx_handler: transmit callback function
599250079Scarl * @event_handler: event callback function
600250079Scarl *
601250079Scarl * Create a new NTB transport layer queue and provide the queue with a callback
602250079Scarl * routine for both transmit and receive.  The receive callback routine will be
603250079Scarl * used to pass up data when the transport has received it on the queue.   The
604250079Scarl * transmit callback routine will be called when the transport has completed the
605250079Scarl * transmission of the data on the queue and the data is ready to be freed.
606250079Scarl *
607250079Scarl * RETURNS: pointer to newly created ntb_queue, NULL on error.
608250079Scarl */
609250079Scarlstatic struct ntb_transport_qp *
610250079Scarlntb_transport_create_queue(void *data, struct ntb_softc *pdev,
611250079Scarl    const struct ntb_queue_handlers *handlers)
612250079Scarl{
613250079Scarl	struct ntb_queue_entry *entry;
614250079Scarl	struct ntb_transport_qp *qp;
615250079Scarl	struct ntb_netdev *nt;
616250079Scarl	unsigned int free_queue;
617250079Scarl	int rc, i;
618250079Scarl
619250079Scarl	nt = ntb_find_transport(pdev);
620250079Scarl	if (nt == NULL)
621250079Scarl		goto err;
622250079Scarl
623250079Scarl	free_queue = ffs(nt->qp_bitmap);
624250079Scarl	if (free_queue == 0)
625250079Scarl		goto err;
626250079Scarl
627250079Scarl	/* decrement free_queue to make it zero based */
628250079Scarl	free_queue--;
629250079Scarl
630250079Scarl	clear_bit(free_queue, &nt->qp_bitmap);
631250079Scarl
632250079Scarl	qp = &nt->qps[free_queue];
633250079Scarl	qp->cb_data = data;
634250079Scarl	qp->rx_handler = handlers->rx_handler;
635250079Scarl	qp->tx_handler = handlers->tx_handler;
636250079Scarl	qp->event_handler = handlers->event_handler;
637250079Scarl
638250079Scarl	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
639250079Scarl		entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
640250079Scarl		    M_WAITOK|M_ZERO);
641250079Scarl		entry->cb_data = nt->ifp;
642250079Scarl		entry->buf = NULL;
643250079Scarl		entry->len = transport_mtu;
644250079Scarl		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
645250079Scarl	}
646250079Scarl
647250079Scarl	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
648250079Scarl		entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
649250079Scarl		    M_WAITOK|M_ZERO);
650250079Scarl		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
651250079Scarl	}
652250079Scarl
653250079Scarl	rc = ntb_register_db_callback(qp->ntb, free_queue, qp,
654250079Scarl				      ntb_transport_rxc_db);
655250079Scarl	if (rc != 0)
656250079Scarl		goto err1;
657250079Scarl
658250079Scarl	return (qp);
659250079Scarl
660250079Scarlerr1:
661250079Scarl	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
662250079Scarl		free(entry, M_NTB_IF);
663250079Scarl	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
664250079Scarl		free(entry, M_NTB_IF);
665250079Scarl	set_bit(free_queue, &nt->qp_bitmap);
666250079Scarlerr:
667250079Scarl	return (NULL);
668250079Scarl}
669250079Scarl
670250079Scarl/**
671250079Scarl * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
672250079Scarl * @qp: NTB transport layer queue to be enabled
673250079Scarl *
674250079Scarl * Notify NTB transport layer of client readiness to use queue
675250079Scarl */
676250079Scarlstatic void
677250079Scarlntb_transport_link_up(struct ntb_transport_qp *qp)
678250079Scarl{
679250079Scarl
680250079Scarl	if (qp == NULL)
681250079Scarl		return;
682250079Scarl
683250079Scarl	qp->client_ready = NTB_LINK_UP;
684255268Scarl	if (bootverbose)
685255268Scarl		device_printf(ntb_get_device(qp->ntb), "qp client ready\n");
686250079Scarl
687250079Scarl	if (qp->transport->transport_link == NTB_LINK_UP)
688250079Scarl		callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);
689250079Scarl}
690250079Scarl
691250079Scarl
692250079Scarl
693250079Scarl/* Transport Tx */
694250079Scarl
695250079Scarl/**
696250079Scarl * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
697250079Scarl * @qp: NTB transport layer queue the entry is to be enqueued on
698250079Scarl * @cb: per buffer pointer for callback function to use
699250079Scarl * @data: pointer to data buffer that will be sent
700250079Scarl * @len: length of the data buffer
701250079Scarl *
702250079Scarl * Enqueue a new transmit buffer onto the transport queue from which a NTB
703250079Scarl * payload will be transmitted.  This assumes that a lock is behing held to
704250079Scarl * serialize access to the qp.
705250079Scarl *
706250079Scarl * RETURNS: An appropriate ERRNO error value on error, or zero for success.
707250079Scarl */
708250079Scarlstatic int
709250079Scarlntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
710250079Scarl    unsigned int len)
711250079Scarl{
712250079Scarl	struct ntb_queue_entry *entry;
713250079Scarl	int rc;
714250079Scarl
715250079Scarl	if (qp == NULL || qp->qp_link != NTB_LINK_UP || len == 0) {
716250079Scarl		CTR0(KTR_NTB, "TX: link not up");
717250079Scarl		return (EINVAL);
718250079Scarl	}
719250079Scarl
720250079Scarl	entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
721250079Scarl	if (entry == NULL) {
722255281Scarl		CTR0(KTR_NTB, "TX: could not get entry from tx_free_q");
723250079Scarl		return (ENOMEM);
724250079Scarl	}
725250079Scarl	CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry);
726250079Scarl
727250079Scarl	entry->cb_data = cb;
728250079Scarl	entry->buf = data;
729250079Scarl	entry->len = len;
730250079Scarl	entry->flags = 0;
731250079Scarl
732250079Scarl	rc = ntb_process_tx(qp, entry);
733250079Scarl	if (rc != 0) {
734250079Scarl		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
735250079Scarl		CTR1(KTR_NTB,
736250079Scarl		    "TX: process_tx failed. Returning entry %p to tx_free_q",
737250079Scarl		    entry);
738250079Scarl	}
739250079Scarl	return (rc);
740250079Scarl}
741250079Scarl
742250079Scarlstatic int
743250079Scarlntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry)
744250079Scarl{
745250079Scarl	void *offset;
746250079Scarl
747250079Scarl	offset = (char *)qp->tx_mw + qp->tx_max_frame * qp->tx_index;
748250079Scarl	CTR3(KTR_NTB,
749250079Scarl	    "TX: process_tx: tx_pkts=%u, tx_index=%u, remote entry=%u",
750250079Scarl	    qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry);
751250079Scarl	if (qp->tx_index == qp->remote_rx_info->entry) {
752250079Scarl		CTR0(KTR_NTB, "TX: ring full");
753250079Scarl		qp->tx_ring_full++;
754250079Scarl		return (EAGAIN);
755250079Scarl	}
756250079Scarl
757250079Scarl	if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
758250079Scarl		if (qp->tx_handler != NULL)
759250079Scarl			qp->tx_handler(qp, qp->cb_data, entry->buf,
760250079Scarl				       EIO);
761250079Scarl
762250079Scarl		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
763250079Scarl		CTR1(KTR_NTB,
764250079Scarl		    "TX: frame too big. returning entry %p to tx_free_q",
765250079Scarl		    entry);
766250079Scarl		return (0);
767250079Scarl	}
768250079Scarl	CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset);
769250079Scarl	ntb_tx_copy_task(qp, entry, offset);
770250079Scarl
771250079Scarl	qp->tx_index++;
772250079Scarl	qp->tx_index %= qp->tx_max_entry;
773250079Scarl
774250079Scarl	qp->tx_pkts++;
775250079Scarl
776250079Scarl	return (0);
777250079Scarl}
778250079Scarl
779250079Scarlstatic void
780250079Scarlntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
781250079Scarl    void *offset)
782250079Scarl{
783250079Scarl	struct ntb_payload_header *hdr;
784250079Scarl
785250079Scarl	CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);
786250079Scarl	if (entry->buf != NULL)
787250079Scarl		m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);
788250079Scarl
789250079Scarl	hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame -
790250079Scarl	    sizeof(struct ntb_payload_header));
791250079Scarl	hdr->len = entry->len; /* TODO: replace with bus_space_write */
792250079Scarl	hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */
793250079Scarl	wmb();
794250079Scarl	/* TODO: replace with bus_space_write */
795250079Scarl	hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG;
796250079Scarl
797250079Scarl	ntb_ring_sdb(qp->ntb, qp->qp_num);
798250079Scarl
799250079Scarl	/*
800250079Scarl	 * The entry length can only be zero if the packet is intended to be a
801250079Scarl	 * "link down" or similar.  Since no payload is being sent in these
802250079Scarl	 * cases, there is nothing to add to the completion queue.
803250079Scarl	 */
804250079Scarl	if (entry->len > 0) {
805250079Scarl		qp->tx_bytes += entry->len;
806250079Scarl
807250079Scarl		if (qp->tx_handler)
808250079Scarl			qp->tx_handler(qp, qp->cb_data, entry->cb_data,
809250079Scarl				       entry->len);
810250079Scarl	}
811250079Scarl
812250079Scarl	CTR2(KTR_NTB,
813250079Scarl	    "TX: entry %p sent. hdr->ver = %d, Returning to tx_free_q", entry,
814250079Scarl	    hdr->ver);
815250079Scarl	ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
816250079Scarl}
817250079Scarl
818250079Scarlstatic void
819250079Scarlntb_qp_full(void *arg)
820250079Scarl{
821250079Scarl
822250079Scarl	CTR0(KTR_NTB, "TX: qp_full callout");
823250079Scarl	ntb_start(arg);
824250079Scarl}
825250079Scarl
826250079Scarl/* Transport Rx */
827250079Scarlstatic void
828250079Scarlntb_transport_rxc_db(void *data, int db_num)
829250079Scarl{
830250079Scarl	struct ntb_transport_qp *qp = data;
831250079Scarl
832250079Scarl	ntb_transport_rx(qp);
833250079Scarl}
834250079Scarl
835250079Scarlstatic void
836250079Scarlntb_rx_pendq_full(void *arg)
837250079Scarl{
838250079Scarl
839250079Scarl	CTR0(KTR_NTB, "RX: ntb_rx_pendq_full callout");
840250079Scarl	ntb_transport_rx(arg);
841250079Scarl}
842250079Scarl
843250079Scarlstatic void
844250079Scarlntb_transport_rx(struct ntb_transport_qp *qp)
845250079Scarl{
846250079Scarl	int rc, i;
847250079Scarl
848250079Scarl	/*
849250079Scarl	 * Limit the number of packets processed in a single interrupt to
850250079Scarl	 * provide fairness to others
851250079Scarl	 */
852250079Scarl	mtx_lock(&qp->transport->rx_lock);
853250079Scarl	CTR0(KTR_NTB, "RX: transport_rx");
854250079Scarl	for (i = 0; i < NTB_RX_MAX_PKTS; i++) {
855250079Scarl		rc = ntb_process_rxc(qp);
856250079Scarl		if (rc != 0) {
857250079Scarl			CTR0(KTR_NTB, "RX: process_rxc failed");
858250079Scarl			break;
859250079Scarl		}
860250079Scarl	}
861250079Scarl	mtx_unlock(&qp->transport->rx_lock);
862250079Scarl}
863250079Scarl
864250079Scarlstatic int
865250079Scarlntb_process_rxc(struct ntb_transport_qp *qp)
866250079Scarl{
867250079Scarl	struct ntb_payload_header *hdr;
868250079Scarl	struct ntb_queue_entry *entry;
869250079Scarl	void *offset;
870250079Scarl
871250079Scarl	offset = (void *)
872250079Scarl	    ((char *)qp->rx_buff + qp->rx_max_frame * qp->rx_index);
873250079Scarl	hdr = (void *)
874250079Scarl	    ((char *)offset + qp->rx_max_frame -
875250079Scarl		sizeof(struct ntb_payload_header));
876250079Scarl
877250079Scarl	CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index);
878250079Scarl	entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
879250079Scarl	if (entry == NULL) {
880250079Scarl		qp->rx_err_no_buf++;
881250079Scarl		CTR0(KTR_NTB, "RX: No entries in rx_pend_q");
882250079Scarl		return (ENOMEM);
883250079Scarl	}
884250079Scarl	callout_stop(&qp->rx_full);
885250079Scarl	CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry);
886250079Scarl
887250079Scarl	if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) {
888250079Scarl		CTR1(KTR_NTB,
889250079Scarl		    "RX: hdr not done. Returning entry %p to rx_pend_q", entry);
890250079Scarl		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
891250079Scarl		qp->rx_ring_empty++;
892250079Scarl		return (EAGAIN);
893250079Scarl	}
894250079Scarl
895250079Scarl	if (hdr->ver != (uint32_t) qp->rx_pkts) {
896250079Scarl		CTR3(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). "
897250079Scarl		    "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts,
898250079Scarl		    entry);
899250079Scarl		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
900250079Scarl		qp->rx_err_ver++;
901250079Scarl		return (EIO);
902250079Scarl	}
903250079Scarl
904250079Scarl	if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) {
905250079Scarl		ntb_qp_link_down(qp);
906250079Scarl		CTR1(KTR_NTB,
907250079Scarl		    "RX: link down. adding entry %p back to rx_pend_q", entry);
908250079Scarl		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
909250079Scarl		goto out;
910250079Scarl	}
911250079Scarl
912250079Scarl	if (hdr->len <= entry->len) {
913250079Scarl		entry->len = hdr->len;
914250079Scarl		ntb_rx_copy_task(qp, entry, offset);
915250079Scarl	} else {
916250079Scarl		CTR1(KTR_NTB,
917250079Scarl		    "RX: len too long. Returning entry %p to rx_pend_q", entry);
918250079Scarl		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
919250079Scarl
920250079Scarl		qp->rx_err_oflow++;
921250079Scarl	}
922250079Scarl
923250079Scarl	qp->rx_bytes += hdr->len;
924250079Scarl	qp->rx_pkts++;
925250079Scarl	CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);
926250079Scarl
927250079Scarl
928250079Scarlout:
929250079Scarl	/* Ensure that the data is globally visible before clearing the flag */
930250079Scarl	wmb();
931250079Scarl	hdr->flags = 0;
932250079Scarl	/* TODO: replace with bus_space_write */
933250079Scarl	qp->rx_info->entry = qp->rx_index;
934250079Scarl
935250079Scarl	qp->rx_index++;
936250079Scarl	qp->rx_index %= qp->rx_max_entry;
937250079Scarl
938250079Scarl	return (0);
939250079Scarl}
940250079Scarl
941250079Scarlstatic void
942250079Scarlntb_rx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
943250079Scarl    void *offset)
944250079Scarl{
945250079Scarl	struct ifnet *ifp = entry->cb_data;
946250079Scarl	unsigned int len = entry->len;
947250079Scarl	struct mbuf *m;
948250079Scarl
949250079Scarl	CTR2(KTR_NTB, "RX: copying %d bytes from offset %p", len, offset);
950250079Scarl	m = m_devget(offset, len, 0, ifp, NULL);
951250079Scarl	m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID;
952250079Scarl
953250079Scarl	entry->buf = (void *)m;
954250079Scarl
955250079Scarl	CTR2(KTR_NTB,
956250079Scarl	    "RX: copied entry %p to mbuf %p. Adding entry to rx_free_q", entry,
957250079Scarl	    m);
958250079Scarl	ntb_list_add(&qp->ntb_rx_free_q_lock, entry, &qp->rx_free_q);
959250079Scarl
960250079Scarl	taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task);
961250079Scarl}
962250079Scarl
963250079Scarlstatic void
964250079Scarlntb_rx_completion_task(void *arg, int pending)
965250079Scarl{
966250079Scarl	struct ntb_transport_qp *qp = arg;
967250079Scarl	struct mbuf *m;
968250079Scarl	struct ntb_queue_entry *entry;
969250079Scarl
970250079Scarl	CTR0(KTR_NTB, "RX: rx_completion_task");
971250079Scarl
972250079Scarl	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) {
973250079Scarl		m = entry->buf;
974250079Scarl		CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m);
975250079Scarl		if (qp->rx_handler && qp->client_ready == NTB_LINK_UP)
976250079Scarl			qp->rx_handler(qp, qp->cb_data, m, entry->len);
977250079Scarl
978250079Scarl		entry->buf = NULL;
979250079Scarl		entry->len = qp->transport->bufsize;
980250079Scarl
981250079Scarl		CTR1(KTR_NTB,"RX: entry %p removed from rx_free_q "
982250079Scarl		    "and added to rx_pend_q", entry);
983250079Scarl		ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
984250079Scarl		if (qp->rx_err_no_buf > qp->last_rx_no_buf) {
985250079Scarl			qp->last_rx_no_buf = qp->rx_err_no_buf;
986250079Scarl			CTR0(KTR_NTB, "RX: could spawn rx task");
987250079Scarl			callout_reset(&qp->rx_full, hz / 1000, ntb_rx_pendq_full,
988250079Scarl			    qp);
989250079Scarl		}
990250079Scarl	}
991250079Scarl}
992250079Scarl
993250079Scarl/* Link Event handler */
994250079Scarlstatic void
995250079Scarlntb_transport_event_callback(void *data, enum ntb_hw_event event)
996250079Scarl{
997250079Scarl	struct ntb_netdev *nt = data;
998250079Scarl
999250079Scarl	switch (event) {
1000250079Scarl	case NTB_EVENT_HW_LINK_UP:
1001255268Scarl		if (bootverbose)
1002255268Scarl			device_printf(ntb_get_device(nt->ntb), "HW link up\n");
1003250079Scarl		callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
1004250079Scarl		break;
1005250079Scarl	case NTB_EVENT_HW_LINK_DOWN:
1006255268Scarl		if (bootverbose)
1007255268Scarl			device_printf(ntb_get_device(nt->ntb), "HW link down\n");
1008250079Scarl		ntb_transport_link_cleanup(nt);
1009250079Scarl		break;
1010250079Scarl	default:
1011250079Scarl		panic("ntb: Unknown NTB event");
1012250079Scarl	}
1013250079Scarl}
1014250079Scarl
1015250079Scarl/* Link bring up */
1016250079Scarlstatic void
1017250079Scarlntb_transport_link_work(void *arg)
1018250079Scarl{
1019250079Scarl	struct ntb_netdev *nt = arg;
1020250079Scarl	struct ntb_softc *ntb = nt->ntb;
1021250079Scarl	struct ntb_transport_qp *qp;
1022250079Scarl	uint32_t val;
1023250079Scarl	int rc, i;
1024250079Scarl
1025250079Scarl	/* send the local info */
1026250079Scarl	rc = ntb_write_remote_spad(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
1027250079Scarl	if (rc != 0)
1028250079Scarl		goto out;
1029250079Scarl
1030250079Scarl	rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ, ntb_get_mw_size(ntb, 0));
1031250079Scarl	if (rc != 0)
1032250079Scarl		goto out;
1033250079Scarl
1034250079Scarl	rc = ntb_write_remote_spad(ntb, IF_NTB_MW1_SZ, ntb_get_mw_size(ntb, 1));
1035250079Scarl	if (rc != 0)
1036250079Scarl		goto out;
1037250079Scarl
1038250079Scarl	rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_QPS, nt->max_qps);
1039250079Scarl	if (rc != 0)
1040250079Scarl		goto out;
1041250079Scarl
1042250079Scarl	rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val);
1043250079Scarl	if (rc != 0)
1044250079Scarl		goto out;
1045250079Scarl
1046250079Scarl	rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val);
1047250079Scarl	if (rc != 0)
1048250079Scarl		goto out;
1049250079Scarl
1050250079Scarl	/* Query the remote side for its info */
1051250079Scarl	rc = ntb_read_local_spad(ntb, IF_NTB_VERSION, &val);
1052250079Scarl	if (rc != 0)
1053250079Scarl		goto out;
1054250079Scarl
1055250079Scarl	if (val != NTB_TRANSPORT_VERSION)
1056250079Scarl		goto out;
1057250079Scarl
1058250079Scarl	rc = ntb_read_local_spad(ntb, IF_NTB_NUM_QPS, &val);
1059250079Scarl	if (rc != 0)
1060250079Scarl		goto out;
1061250079Scarl
1062250079Scarl	if (val != nt->max_qps)
1063250079Scarl		goto out;
1064250079Scarl
1065250079Scarl	rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ, &val);
1066250079Scarl	if (rc != 0)
1067250079Scarl		goto out;
1068250079Scarl
1069250079Scarl	if (val == 0)
1070250079Scarl		goto out;
1071250079Scarl
1072250079Scarl	rc = ntb_set_mw(nt, 0, val);
1073250079Scarl	if (rc != 0)
1074250079Scarl		return;
1075250079Scarl
1076250079Scarl	rc = ntb_read_local_spad(ntb, IF_NTB_MW1_SZ, &val);
1077250079Scarl	if (rc != 0)
1078250079Scarl		goto out;
1079250079Scarl
1080250079Scarl	if (val == 0)
1081250079Scarl		goto out;
1082250079Scarl
1083250079Scarl	rc = ntb_set_mw(nt, 1, val);
1084250079Scarl	if (rc != 0)
1085250079Scarl		return;
1086250079Scarl
1087250079Scarl	nt->transport_link = NTB_LINK_UP;
1088255268Scarl	if (bootverbose)
1089255268Scarl		device_printf(ntb_get_device(ntb), "transport link up\n");
1090250079Scarl
1091250079Scarl	for (i = 0; i < nt->max_qps; i++) {
1092250079Scarl		qp = &nt->qps[i];
1093250079Scarl
1094250079Scarl		ntb_transport_setup_qp_mw(nt, i);
1095250079Scarl
1096250079Scarl		if (qp->client_ready == NTB_LINK_UP)
1097250079Scarl			callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);
1098250079Scarl	}
1099250079Scarl
1100250079Scarl	return;
1101250079Scarl
1102250079Scarlout:
1103250079Scarl	if (ntb_query_link_status(ntb))
1104250079Scarl		callout_reset(&nt->link_work,
1105250079Scarl				      NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt);
1106250079Scarl}
1107250079Scarl
1108250079Scarlstatic int
1109250079Scarlntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size)
1110250079Scarl{
1111250079Scarl	struct ntb_transport_mw *mw = &nt->mw[num_mw];
1112250079Scarl
1113250079Scarl	/* Alloc memory for receiving data.  Must be 4k aligned */
1114250079Scarl	mw->size = size;
1115250079Scarl
1116250079Scarl	mw->virt_addr = contigmalloc(mw->size, M_NTB_IF, M_ZERO, 0,
1117250079Scarl	    BUS_SPACE_MAXADDR, mw->size, 0);
1118250079Scarl	if (mw->virt_addr == NULL) {
1119250079Scarl		printf("ntb: Unable to allocate MW buffer of size %d\n",
1120250079Scarl		    (int)mw->size);
1121250079Scarl		return (ENOMEM);
1122250079Scarl	}
1123250079Scarl	/* TODO: replace with bus_space_* functions */
1124250079Scarl	mw->dma_addr = vtophys(mw->virt_addr);
1125250079Scarl
1126250079Scarl	/* Notify HW the memory location of the receive buffer */
1127250079Scarl	ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr);
1128250079Scarl
1129250079Scarl	return (0);
1130250079Scarl}
1131250079Scarl
1132250079Scarlstatic void
1133250079Scarlntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num)
1134250079Scarl{
1135250079Scarl	struct ntb_transport_qp *qp = &nt->qps[qp_num];
1136250079Scarl	void *offset;
1137250079Scarl	unsigned int rx_size, num_qps_mw;
1138250079Scarl	uint8_t mw_num = QP_TO_MW(qp_num);
1139250079Scarl	unsigned int i;
1140250079Scarl
1141250079Scarl	if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
1142250079Scarl		num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
1143250079Scarl	else
1144250079Scarl		num_qps_mw = nt->max_qps / NTB_NUM_MW;
1145250079Scarl
1146250079Scarl	rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw;
1147250079Scarl	qp->remote_rx_info = (void *)((uint8_t *)nt->mw[mw_num].virt_addr +
1148250079Scarl			     (qp_num / NTB_NUM_MW * rx_size));
1149250079Scarl	rx_size -= sizeof(struct ntb_rx_info);
1150250079Scarl
1151289152Scem	qp->rx_buff = qp->remote_rx_info + 1;
1152250079Scarl	qp->rx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
1153250079Scarl	    rx_size);
1154250079Scarl	qp->rx_max_entry = rx_size / qp->rx_max_frame;
1155250079Scarl	qp->rx_index = 0;
1156250079Scarl	qp->tx_index = 0;
1157250079Scarl
1158250079Scarl	qp->remote_rx_info->entry = qp->rx_max_entry;
1159250079Scarl
1160250079Scarl	/* setup the hdr offsets with 0's */
1161250079Scarl	for (i = 0; i < qp->rx_max_entry; i++) {
1162250079Scarl		offset = (void *)((uint8_t *)qp->rx_buff +
1163250079Scarl		    qp->rx_max_frame * (i + 1) -
1164250079Scarl		    sizeof(struct ntb_payload_header));
1165250079Scarl		memset(offset, 0, sizeof(struct ntb_payload_header));
1166250079Scarl	}
1167250079Scarl
1168250079Scarl	qp->rx_pkts = 0;
1169250079Scarl	qp->tx_pkts = 0;
1170250079Scarl}
1171250079Scarl
1172250079Scarlstatic void
1173250079Scarlntb_qp_link_work(void *arg)
1174250079Scarl{
1175250079Scarl	struct ntb_transport_qp *qp = arg;
1176250079Scarl	struct ntb_softc *ntb = qp->ntb;
1177250079Scarl	struct ntb_netdev *nt = qp->transport;
1178250079Scarl	int rc, val;
1179250079Scarl
1180250079Scarl
1181250079Scarl	rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val);
1182250079Scarl	if (rc != 0)
1183250079Scarl		return;
1184250079Scarl
1185250079Scarl	rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num);
1186250079Scarl
1187250079Scarl	/* query remote spad for qp ready bits */
1188250079Scarl	rc = ntb_read_local_spad(ntb, IF_NTB_QP_LINKS, &val);
1189250079Scarl
1190250079Scarl	/* See if the remote side is up */
1191250079Scarl	if ((1 << qp->qp_num & val) != 0) {
1192250079Scarl		qp->qp_link = NTB_LINK_UP;
1193250079Scarl		if (qp->event_handler != NULL)
1194250079Scarl			qp->event_handler(qp->cb_data, NTB_LINK_UP);
1195255268Scarl		if (bootverbose)
1196255268Scarl			device_printf(ntb_get_device(ntb), "qp link up\n");
1197250079Scarl	} else if (nt->transport_link == NTB_LINK_UP) {
1198250079Scarl		callout_reset(&qp->link_work,
1199250079Scarl		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
1200250079Scarl	}
1201250079Scarl}
1202250079Scarl
1203250079Scarl/* Link down event*/
1204250079Scarlstatic void
1205250079Scarlntb_transport_link_cleanup(struct ntb_netdev *nt)
1206250079Scarl{
1207250079Scarl	int i;
1208250079Scarl
1209250079Scarl	if (nt->transport_link == NTB_LINK_DOWN)
1210250079Scarl		callout_drain(&nt->link_work);
1211250079Scarl	else
1212250079Scarl		nt->transport_link = NTB_LINK_DOWN;
1213250079Scarl
1214250079Scarl	/* Pass along the info to any clients */
1215250079Scarl	for (i = 0; i < nt->max_qps; i++)
1216250079Scarl		if (!test_bit(i, &nt->qp_bitmap))
1217250079Scarl			ntb_qp_link_down(&nt->qps[i]);
1218250079Scarl
1219250079Scarl	/*
1220250079Scarl	 * The scratchpad registers keep the values if the remote side
1221250079Scarl	 * goes down, blast them now to give them a sane value the next
1222250079Scarl	 * time they are accessed
1223250079Scarl	 */
1224250079Scarl	for (i = 0; i < IF_NTB_MAX_SPAD; i++)
1225250079Scarl		ntb_write_local_spad(nt->ntb, i, 0);
1226250079Scarl}
1227250079Scarl
1228250079Scarl
1229250079Scarlstatic void
1230250079Scarlntb_qp_link_down(struct ntb_transport_qp *qp)
1231250079Scarl{
1232250079Scarl
1233250079Scarl	ntb_qp_link_cleanup(qp);
1234250079Scarl}
1235250079Scarl
1236250079Scarlstatic void
1237250079Scarlntb_qp_link_cleanup(struct ntb_transport_qp *qp)
1238250079Scarl{
1239250079Scarl	struct ntb_netdev *nt = qp->transport;
1240250079Scarl
1241250079Scarl	if (qp->qp_link == NTB_LINK_DOWN) {
1242250079Scarl		callout_drain(&qp->link_work);
1243250079Scarl		return;
1244250079Scarl	}
1245250079Scarl
1246250079Scarl	if (qp->event_handler != NULL)
1247250079Scarl		qp->event_handler(qp->cb_data, NTB_LINK_DOWN);
1248250079Scarl
1249250079Scarl	qp->qp_link = NTB_LINK_DOWN;
1250250079Scarl
1251250079Scarl	if (nt->transport_link == NTB_LINK_UP)
1252250079Scarl		callout_reset(&qp->link_work,
1253250079Scarl		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
1254250079Scarl}
1255250079Scarl
1256250079Scarl/* Link commanded down */
1257250079Scarl/**
1258250079Scarl * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
1259250079Scarl * @qp: NTB transport layer queue to be disabled
1260250079Scarl *
1261250079Scarl * Notify NTB transport layer of client's desire to no longer receive data on
1262250079Scarl * transport queue specified.  It is the client's responsibility to ensure all
1263250079Scarl * entries on queue are purged or otherwise handled appropraitely.
1264250079Scarl */
1265250079Scarlstatic void
1266250079Scarlntb_transport_link_down(struct ntb_transport_qp *qp)
1267250079Scarl{
1268250079Scarl	int rc, val;
1269250079Scarl
1270250079Scarl	if (qp == NULL)
1271250079Scarl		return;
1272250079Scarl
1273250079Scarl	qp->client_ready = NTB_LINK_DOWN;
1274250079Scarl
1275250079Scarl	rc = ntb_read_remote_spad(qp->ntb, IF_NTB_QP_LINKS, &val);
1276250079Scarl	if (rc != 0)
1277250079Scarl		return;
1278250079Scarl
1279250079Scarl	rc = ntb_write_remote_spad(qp->ntb, IF_NTB_QP_LINKS,
1280250079Scarl	   val & ~(1 << qp->qp_num));
1281250079Scarl
1282250079Scarl	if (qp->qp_link == NTB_LINK_UP)
1283250079Scarl		ntb_send_link_down(qp);
1284250079Scarl	else
1285250079Scarl		callout_drain(&qp->link_work);
1286250079Scarl
1287250079Scarl}
1288250079Scarl
1289250079Scarlstatic void
1290250079Scarlntb_send_link_down(struct ntb_transport_qp *qp)
1291250079Scarl{
1292250079Scarl	struct ntb_queue_entry *entry;
1293250079Scarl	int i, rc;
1294250079Scarl
1295250079Scarl	if (qp->qp_link == NTB_LINK_DOWN)
1296250079Scarl		return;
1297250079Scarl
1298250079Scarl	qp->qp_link = NTB_LINK_DOWN;
1299250079Scarl
1300250079Scarl	for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) {
1301250079Scarl		entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
1302250079Scarl		if (entry != NULL)
1303250079Scarl			break;
1304250079Scarl		pause("NTB Wait for link down", hz / 10);
1305250079Scarl	}
1306250079Scarl
1307250079Scarl	if (entry == NULL)
1308250079Scarl		return;
1309250079Scarl
1310250079Scarl	entry->cb_data = NULL;
1311250079Scarl	entry->buf = NULL;
1312250079Scarl	entry->len = 0;
1313250079Scarl	entry->flags = IF_NTB_LINK_DOWN_FLAG;
1314250079Scarl
1315250079Scarl	mtx_lock(&qp->transport->tx_lock);
1316250079Scarl	rc = ntb_process_tx(qp, entry);
1317250079Scarl	if (rc != 0)
1318250079Scarl		printf("ntb: Failed to send link down\n");
1319250079Scarl	mtx_unlock(&qp->transport->tx_lock);
1320250079Scarl}
1321250079Scarl
1322250079Scarl
1323250079Scarl/* List Management */
1324250079Scarl
1325250079Scarlstatic void
1326250079Scarlntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry,
1327250079Scarl    struct ntb_queue_list *list)
1328250079Scarl{
1329250079Scarl
1330250079Scarl	mtx_lock_spin(lock);
1331250079Scarl	STAILQ_INSERT_TAIL(list, entry, entry);
1332250079Scarl	mtx_unlock_spin(lock);
1333250079Scarl}
1334250079Scarl
1335250079Scarlstatic struct ntb_queue_entry *
1336250079Scarlntb_list_rm(struct mtx *lock, struct ntb_queue_list *list)
1337250079Scarl{
1338250079Scarl	struct ntb_queue_entry *entry;
1339250079Scarl
1340250079Scarl	mtx_lock_spin(lock);
1341250079Scarl	if (STAILQ_EMPTY(list)) {
1342250079Scarl		entry = NULL;
1343250079Scarl		goto out;
1344250079Scarl	}
1345250079Scarl	entry = STAILQ_FIRST(list);
1346250079Scarl	STAILQ_REMOVE_HEAD(list, entry);
1347250079Scarlout:
1348250079Scarl	mtx_unlock_spin(lock);
1349250079Scarl
1350250079Scarl	return (entry);
1351250079Scarl}
1352250079Scarl
1353250079Scarl/* Helper functions */
1354250079Scarl/* TODO: This too should really be part of the kernel */
1355250079Scarl#define EUI48_MULTICAST			1 << 0
1356250079Scarl#define EUI48_LOCALLY_ADMINISTERED	1 << 1
1357250079Scarlstatic void
1358250079Scarlcreate_random_local_eui48(u_char *eaddr)
1359250079Scarl{
1360250079Scarl	static uint8_t counter = 0;
1361250079Scarl	uint32_t seed = ticks;
1362250079Scarl
1363250079Scarl	eaddr[0] = EUI48_LOCALLY_ADMINISTERED;
1364250079Scarl	memcpy(&eaddr[1], &seed, sizeof(uint32_t));
1365250079Scarl	eaddr[5] = counter++;
1366250079Scarl}
1367250079Scarl
1368250079Scarl/**
1369250079Scarl * ntb_transport_max_size - Query the max payload size of a qp
1370250079Scarl * @qp: NTB transport layer queue to be queried
1371250079Scarl *
1372250079Scarl * Query the maximum payload size permissible on the given qp
1373250079Scarl *
1374250079Scarl * RETURNS: the max payload size of a qp
1375250079Scarl */
1376250079Scarlstatic unsigned int
1377250079Scarlntb_transport_max_size(struct ntb_transport_qp *qp)
1378250079Scarl{
1379250079Scarl
1380250079Scarl	if (qp == NULL)
1381250079Scarl		return (0);
1382250079Scarl
1383250079Scarl	return (qp->tx_max_frame - sizeof(struct ntb_payload_header));
1384250079Scarl}
1385