ntb_transport.c revision 289152
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3250079Scarl * All rights reserved. 4250079Scarl * 5250079Scarl * Redistribution and use in source and binary forms, with or without 6250079Scarl * modification, are permitted provided that the following conditions 7250079Scarl * are met: 8250079Scarl * 1. Redistributions of source code must retain the above copyright 9250079Scarl * notice, this list of conditions and the following disclaimer. 10250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer in the 12250079Scarl * documentation and/or other materials provided with the distribution. 13250079Scarl * 14250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24250079Scarl * SUCH DAMAGE. 25250079Scarl */ 26250079Scarl 27250079Scarl#include <sys/cdefs.h> 28250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/if_ntb/if_ntb.c 289152 2015-10-11 20:57:09Z cem $"); 29250079Scarl 30250079Scarl#include <sys/param.h> 31250079Scarl#include <sys/kernel.h> 32250079Scarl#include <sys/systm.h> 33250079Scarl#include <sys/bus.h> 34250079Scarl#include <sys/ktr.h> 35250079Scarl#include <sys/lock.h> 36250079Scarl#include <sys/malloc.h> 37250079Scarl#include <sys/module.h> 38250079Scarl#include <sys/mutex.h> 39250079Scarl#include <sys/queue.h> 40250079Scarl#include <sys/socket.h> 41250079Scarl#include <sys/sockio.h> 42250079Scarl#include <sys/taskqueue.h> 43250079Scarl#include <net/if.h> 44250079Scarl#include <net/if_media.h> 45250079Scarl#include <net/if_types.h> 46250079Scarl#include <net/if_var.h> 47250079Scarl#include <net/bpf.h> 48250079Scarl#include <net/ethernet.h> 49250079Scarl#include <vm/vm.h> 50250079Scarl#include <vm/pmap.h> 51250079Scarl#include <machine/bus.h> 52250079Scarl#include <machine/cpufunc.h> 53250079Scarl#include <machine/pmap.h> 54250079Scarl 55250079Scarl#include "../ntb_hw/ntb_hw.h" 56250079Scarl 57250079Scarl/* 58250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 59250079Scarl * allows you to connect two systems using a PCI-e link. 60250079Scarl * 61250079Scarl * This module contains a protocol for sending and receiving messages, and 62250079Scarl * exposes that protocol through a simulated ethernet device called ntb. 63250079Scarl * 64250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 65250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 66250079Scarl */ 67250079Scarl 68250079Scarl/* TODO: These functions should really be part of the kernel */ 69250079Scarl#define test_bit(pos, bitmap_addr) (*(bitmap_addr) & 1UL << (pos)) 70250079Scarl#define set_bit(pos, bitmap_addr) *(bitmap_addr) |= 1UL << (pos) 71250079Scarl#define clear_bit(pos, bitmap_addr) *(bitmap_addr) &= ~(1UL << (pos)) 72250079Scarl 73250079Scarl#define KTR_NTB KTR_SPARE3 74250079Scarl 75250079Scarl#define NTB_TRANSPORT_VERSION 3 76250079Scarl#define NTB_RX_MAX_PKTS 64 77250079Scarl#define NTB_RXQ_SIZE 300 78250079Scarl 79250079Scarlstatic unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN; 80250079Scarlstatic unsigned int max_num_clients = 1; 81250079Scarl 82250079ScarlSTAILQ_HEAD(ntb_queue_list, ntb_queue_entry); 83250079Scarl 84250079Scarlstruct ntb_queue_entry { 85250079Scarl /* ntb_queue list reference */ 86250079Scarl STAILQ_ENTRY(ntb_queue_entry) entry; 87250079Scarl 88250079Scarl /* info on data to be transfered */ 89250079Scarl void *cb_data; 90250079Scarl void *buf; 91250079Scarl uint64_t len; 92250079Scarl uint64_t flags; 93250079Scarl}; 94250079Scarl 95250079Scarlstruct ntb_rx_info { 96250079Scarl unsigned int entry; 97250079Scarl}; 98250079Scarl 99250079Scarlstruct ntb_transport_qp { 100250079Scarl struct ntb_netdev *transport; 101250079Scarl struct ntb_softc *ntb; 102250079Scarl 103250079Scarl void *cb_data; 104250079Scarl 105250079Scarl bool client_ready; 106250079Scarl bool qp_link; 107255281Scarl uint8_t qp_num; /* Only 64 QPs are allowed. 0-63 */ 108250079Scarl 109250079Scarl struct ntb_rx_info *rx_info; 110250079Scarl struct ntb_rx_info *remote_rx_info; 111250079Scarl 112250079Scarl void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, 113250079Scarl void *data, int len); 114250079Scarl struct ntb_queue_list tx_free_q; 115250079Scarl struct mtx ntb_tx_free_q_lock; 116250079Scarl void *tx_mw; 117250079Scarl uint64_t tx_index; 118250079Scarl uint64_t tx_max_entry; 119250079Scarl uint64_t tx_max_frame; 120250079Scarl 121250079Scarl void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, 122250079Scarl void *data, int len); 123250079Scarl struct ntb_queue_list rx_pend_q; 124250079Scarl struct ntb_queue_list rx_free_q; 125250079Scarl struct mtx ntb_rx_pend_q_lock; 126250079Scarl struct mtx ntb_rx_free_q_lock; 127250079Scarl struct task rx_completion_task; 128250079Scarl void *rx_buff; 129250079Scarl uint64_t rx_index; 130250079Scarl uint64_t rx_max_entry; 131250079Scarl uint64_t rx_max_frame; 132250079Scarl 133250079Scarl void (*event_handler) (void *data, int status); 134250079Scarl struct callout link_work; 135250079Scarl struct callout queue_full; 136250079Scarl struct callout rx_full; 137250079Scarl 138250079Scarl uint64_t last_rx_no_buf; 139250079Scarl 140250079Scarl /* Stats */ 141250079Scarl uint64_t rx_bytes; 142250079Scarl uint64_t rx_pkts; 143250079Scarl uint64_t rx_ring_empty; 144250079Scarl uint64_t rx_err_no_buf; 145250079Scarl uint64_t rx_err_oflow; 146250079Scarl uint64_t rx_err_ver; 147250079Scarl uint64_t tx_bytes; 148250079Scarl uint64_t tx_pkts; 149250079Scarl uint64_t tx_ring_full; 150250079Scarl}; 151250079Scarl 152250079Scarlstruct ntb_queue_handlers { 153250079Scarl void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, 154250079Scarl void *data, int len); 155250079Scarl void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, 156250079Scarl void *data, int len); 157250079Scarl void (*event_handler) (void *data, int status); 158250079Scarl}; 159250079Scarl 160250079Scarl 161250079Scarlstruct ntb_transport_mw { 162250079Scarl size_t size; 163250079Scarl void *virt_addr; 164250079Scarl vm_paddr_t dma_addr; 165250079Scarl}; 166250079Scarl 167250079Scarlstruct ntb_netdev { 168250079Scarl struct ntb_softc *ntb; 169250079Scarl struct ifnet *ifp; 170250079Scarl struct ntb_transport_mw mw[NTB_NUM_MW]; 171250079Scarl struct ntb_transport_qp *qps; 172250079Scarl uint64_t max_qps; 173250079Scarl uint64_t qp_bitmap; 174250079Scarl bool transport_link; 175250079Scarl struct callout link_work; 176250079Scarl struct ntb_transport_qp *qp; 177250079Scarl uint64_t bufsize; 178250079Scarl u_char eaddr[ETHER_ADDR_LEN]; 179250079Scarl struct mtx tx_lock; 180250079Scarl struct mtx rx_lock; 181250079Scarl}; 182250079Scarl 183250079Scarlstatic struct ntb_netdev net_softc; 184250079Scarl 185250079Scarlenum { 186250079Scarl IF_NTB_DESC_DONE_FLAG = 1 << 0, 187250079Scarl IF_NTB_LINK_DOWN_FLAG = 1 << 1, 188250079Scarl}; 189250079Scarl 190250079Scarlstruct ntb_payload_header { 191250079Scarl uint64_t ver; 192250079Scarl uint64_t len; 193250079Scarl uint64_t flags; 194250079Scarl}; 195250079Scarl 196250079Scarlenum { 197250079Scarl IF_NTB_VERSION = 0, 198250079Scarl IF_NTB_MW0_SZ, 199250079Scarl IF_NTB_MW1_SZ, 200250079Scarl IF_NTB_NUM_QPS, 201250079Scarl IF_NTB_QP_LINKS, 202250079Scarl IF_NTB_MAX_SPAD, 203250079Scarl}; 204250079Scarl 205250079Scarl#define QP_TO_MW(qp) ((qp) % NTB_NUM_MW) 206250079Scarl#define NTB_QP_DEF_NUM_ENTRIES 100 207250079Scarl#define NTB_LINK_DOWN_TIMEOUT 10 208250079Scarl 209250079Scarlstatic int ntb_handle_module_events(struct module *m, int what, void *arg); 210250079Scarlstatic int ntb_setup_interface(void); 211250079Scarlstatic int ntb_teardown_interface(void); 212250079Scarlstatic void ntb_net_init(void *arg); 213250079Scarlstatic int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 214250079Scarlstatic void ntb_start(struct ifnet *ifp); 215250079Scarlstatic void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, 216250079Scarl void *data, int len); 217250079Scarlstatic void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, 218250079Scarl void *data, int len); 219250079Scarlstatic void ntb_net_event_handler(void *data, int status); 220250079Scarlstatic int ntb_transport_init(struct ntb_softc *ntb); 221250079Scarlstatic void ntb_transport_free(void *transport); 222250079Scarlstatic void ntb_transport_init_queue(struct ntb_netdev *nt, 223250079Scarl unsigned int qp_num); 224250079Scarlstatic void ntb_transport_free_queue(struct ntb_transport_qp *qp); 225250079Scarlstatic struct ntb_transport_qp * ntb_transport_create_queue(void *data, 226250079Scarl struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers); 227250079Scarlstatic void ntb_transport_link_up(struct ntb_transport_qp *qp); 228250079Scarlstatic int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, 229250079Scarl void *data, unsigned int len); 230250079Scarlstatic int ntb_process_tx(struct ntb_transport_qp *qp, 231250079Scarl struct ntb_queue_entry *entry); 232250079Scarlstatic void ntb_tx_copy_task(struct ntb_transport_qp *qp, 233250079Scarl struct ntb_queue_entry *entry, void *offset); 234250079Scarlstatic void ntb_qp_full(void *arg); 235250079Scarlstatic void ntb_transport_rxc_db(void *data, int db_num); 236250079Scarlstatic void ntb_rx_pendq_full(void *arg); 237250079Scarlstatic void ntb_transport_rx(struct ntb_transport_qp *qp); 238250079Scarlstatic int ntb_process_rxc(struct ntb_transport_qp *qp); 239250079Scarlstatic void ntb_rx_copy_task(struct ntb_transport_qp *qp, 240250079Scarl struct ntb_queue_entry *entry, void *offset); 241250079Scarlstatic void ntb_rx_completion_task(void *arg, int pending); 242250079Scarlstatic void ntb_transport_event_callback(void *data, enum ntb_hw_event event); 243250079Scarlstatic void ntb_transport_link_work(void *arg); 244250079Scarlstatic int ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size); 245250079Scarlstatic void ntb_transport_setup_qp_mw(struct ntb_netdev *nt, 246250079Scarl unsigned int qp_num); 247250079Scarlstatic void ntb_qp_link_work(void *arg); 248250079Scarlstatic void ntb_transport_link_cleanup(struct ntb_netdev *nt); 249250079Scarlstatic void ntb_qp_link_down(struct ntb_transport_qp *qp); 250250079Scarlstatic void ntb_qp_link_cleanup(struct ntb_transport_qp *qp); 251250079Scarlstatic void ntb_transport_link_down(struct ntb_transport_qp *qp); 252250079Scarlstatic void ntb_send_link_down(struct ntb_transport_qp *qp); 253250079Scarlstatic void ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 254250079Scarl struct ntb_queue_list *list); 255250079Scarlstatic struct ntb_queue_entry *ntb_list_rm(struct mtx *lock, 256250079Scarl struct ntb_queue_list *list); 257250079Scarlstatic void create_random_local_eui48(u_char *eaddr); 258250079Scarlstatic unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); 259250079Scarl 260250079ScarlMALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver"); 261250079Scarl 262250079Scarl/* Module setup and teardown */ 263250079Scarlstatic int 264250079Scarlntb_handle_module_events(struct module *m, int what, void *arg) 265250079Scarl{ 266250079Scarl int err = 0; 267250079Scarl 268250079Scarl switch (what) { 269250079Scarl case MOD_LOAD: 270250079Scarl err = ntb_setup_interface(); 271250079Scarl break; 272250079Scarl case MOD_UNLOAD: 273250079Scarl err = ntb_teardown_interface(); 274250079Scarl break; 275250079Scarl default: 276250079Scarl err = EOPNOTSUPP; 277250079Scarl break; 278250079Scarl } 279250079Scarl return (err); 280250079Scarl} 281250079Scarl 282255271Scarlstatic moduledata_t if_ntb_mod = { 283255271Scarl "if_ntb", 284250079Scarl ntb_handle_module_events, 285250079Scarl NULL 286250079Scarl}; 287250079Scarl 288255271ScarlDECLARE_MODULE(if_ntb, if_ntb_mod, SI_SUB_KLD, SI_ORDER_ANY); 289255271ScarlMODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1); 290250079Scarl 291250079Scarlstatic int 292250079Scarlntb_setup_interface() 293250079Scarl{ 294250079Scarl struct ifnet *ifp; 295250079Scarl struct ntb_queue_handlers handlers = { ntb_net_rx_handler, 296250079Scarl ntb_net_tx_handler, ntb_net_event_handler }; 297250079Scarl 298250079Scarl net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0); 299250079Scarl if (net_softc.ntb == NULL) { 300255281Scarl printf("ntb: Cannot find devclass\n"); 301250079Scarl return (ENXIO); 302250079Scarl } 303250079Scarl 304250079Scarl ntb_transport_init(net_softc.ntb); 305250079Scarl 306250079Scarl ifp = net_softc.ifp = if_alloc(IFT_ETHER); 307250079Scarl if (ifp == NULL) { 308250079Scarl printf("ntb: cannot allocate ifnet structure\n"); 309250079Scarl return (ENOMEM); 310250079Scarl } 311250079Scarl 312250079Scarl net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb, 313250079Scarl &handlers); 314250079Scarl if_initname(ifp, "ntb", 0); 315250079Scarl ifp->if_init = ntb_net_init; 316250079Scarl ifp->if_softc = &net_softc; 317250079Scarl ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX; 318250079Scarl ifp->if_ioctl = ntb_ioctl; 319250079Scarl ifp->if_start = ntb_start; 320250079Scarl IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN); 321250079Scarl ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 322250079Scarl IFQ_SET_READY(&ifp->if_snd); 323250079Scarl create_random_local_eui48(net_softc.eaddr); 324250079Scarl ether_ifattach(ifp, net_softc.eaddr); 325250079Scarl ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU; 326250079Scarl ifp->if_capenable = ifp->if_capabilities; 327250079Scarl 328250079Scarl ntb_transport_link_up(net_softc.qp); 329250079Scarl net_softc.bufsize = ntb_transport_max_size(net_softc.qp) + 330250079Scarl sizeof(struct ether_header); 331250079Scarl return (0); 332250079Scarl} 333250079Scarl 334250079Scarlstatic int 335250079Scarlntb_teardown_interface() 336250079Scarl{ 337250079Scarl 338255280Scarl if (net_softc.qp != NULL) 339255280Scarl ntb_transport_link_down(net_softc.qp); 340250079Scarl 341255280Scarl if (net_softc.ifp != NULL) { 342255280Scarl ether_ifdetach(net_softc.ifp); 343255280Scarl if_free(net_softc.ifp); 344255280Scarl } 345250079Scarl 346255280Scarl if (net_softc.qp != NULL) { 347255280Scarl ntb_transport_free_queue(net_softc.qp); 348255280Scarl ntb_transport_free(&net_softc); 349255280Scarl } 350255280Scarl 351250079Scarl return (0); 352250079Scarl} 353250079Scarl 354250079Scarl/* Network device interface */ 355250079Scarl 356250079Scarlstatic void 357250079Scarlntb_net_init(void *arg) 358250079Scarl{ 359250079Scarl struct ntb_netdev *ntb_softc = arg; 360250079Scarl struct ifnet *ifp = ntb_softc->ifp; 361250079Scarl 362250079Scarl ifp->if_drv_flags |= IFF_DRV_RUNNING; 363250079Scarl ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 364250079Scarl ifp->if_flags |= IFF_UP; 365250079Scarl if_link_state_change(ifp, LINK_STATE_UP); 366250079Scarl} 367250079Scarl 368250079Scarlstatic int 369250079Scarlntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 370250079Scarl{ 371250079Scarl struct ntb_netdev *nt = ifp->if_softc; 372250079Scarl struct ifreq *ifr = (struct ifreq *)data; 373250079Scarl int error = 0; 374250079Scarl 375250079Scarl switch (command) { 376250079Scarl case SIOCSIFMTU: 377250079Scarl { 378250079Scarl if (ifr->ifr_mtu > ntb_transport_max_size(nt->qp) - 379250079Scarl ETHER_HDR_LEN - ETHER_CRC_LEN) { 380250079Scarl error = EINVAL; 381250079Scarl break; 382250079Scarl } 383250079Scarl 384250079Scarl ifp->if_mtu = ifr->ifr_mtu; 385250079Scarl break; 386250079Scarl } 387250079Scarl default: 388250079Scarl error = ether_ioctl(ifp, command, data); 389250079Scarl break; 390250079Scarl } 391250079Scarl 392250079Scarl return (error); 393250079Scarl} 394250079Scarl 395250079Scarl 396250079Scarlstatic void 397250079Scarlntb_start(struct ifnet *ifp) 398250079Scarl{ 399250079Scarl struct mbuf *m_head; 400250079Scarl struct ntb_netdev *nt = ifp->if_softc; 401250079Scarl int rc; 402250079Scarl 403250079Scarl mtx_lock(&nt->tx_lock); 404250079Scarl ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 405250079Scarl CTR0(KTR_NTB, "TX: ntb_start"); 406250079Scarl while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 407250079Scarl IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 408250079Scarl CTR1(KTR_NTB, "TX: start mbuf %p", m_head); 409250079Scarl rc = ntb_transport_tx_enqueue(nt->qp, m_head, m_head, 410250079Scarl m_length(m_head, NULL)); 411250079Scarl if (rc != 0) { 412250079Scarl CTR1(KTR_NTB, 413255281Scarl "TX: could not tx mbuf %p. Returning to snd q", 414250079Scarl m_head); 415250079Scarl if (rc == EAGAIN) { 416250079Scarl ifp->if_drv_flags |= IFF_DRV_OACTIVE; 417250079Scarl IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 418250079Scarl callout_reset(&nt->qp->queue_full, hz / 1000, 419250079Scarl ntb_qp_full, ifp); 420250079Scarl } 421250079Scarl break; 422250079Scarl } 423250079Scarl 424250079Scarl } 425250079Scarl mtx_unlock(&nt->tx_lock); 426250079Scarl} 427250079Scarl 428250079Scarl/* Network Device Callbacks */ 429250079Scarlstatic void 430250079Scarlntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 431250079Scarl int len) 432250079Scarl{ 433250079Scarl 434250079Scarl m_freem(data); 435250079Scarl CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data); 436250079Scarl} 437250079Scarl 438250079Scarlstatic void 439250079Scarlntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 440250079Scarl int len) 441250079Scarl{ 442250079Scarl struct mbuf *m = data; 443250079Scarl struct ifnet *ifp = qp_data; 444250079Scarl 445250079Scarl CTR0(KTR_NTB, "RX: rx handler"); 446250079Scarl (*ifp->if_input)(ifp, m); 447250079Scarl} 448250079Scarl 449250079Scarlstatic void 450250079Scarlntb_net_event_handler(void *data, int status) 451250079Scarl{ 452250079Scarl 453250079Scarl} 454250079Scarl 455250079Scarl/* Transport Init and teardown */ 456250079Scarl 457250079Scarlstatic int 458250079Scarlntb_transport_init(struct ntb_softc *ntb) 459250079Scarl{ 460250079Scarl struct ntb_netdev *nt = &net_softc; 461250079Scarl int rc, i; 462250079Scarl 463250079Scarl nt->max_qps = max_num_clients; 464250079Scarl ntb_register_transport(ntb, nt); 465250079Scarl mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF); 466250079Scarl mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF); 467250079Scarl 468250079Scarl nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp), 469250079Scarl M_NTB_IF, M_WAITOK|M_ZERO); 470250079Scarl 471250079Scarl nt->qp_bitmap = ((uint64_t) 1 << nt->max_qps) - 1; 472250079Scarl 473250079Scarl for (i = 0; i < nt->max_qps; i++) 474250079Scarl ntb_transport_init_queue(nt, i); 475250079Scarl 476250079Scarl callout_init(&nt->link_work, 0); 477250079Scarl 478250079Scarl rc = ntb_register_event_callback(ntb, 479250079Scarl ntb_transport_event_callback); 480250079Scarl if (rc != 0) 481250079Scarl goto err; 482250079Scarl 483255268Scarl if (ntb_query_link_status(ntb)) { 484255268Scarl if (bootverbose) 485255268Scarl device_printf(ntb_get_device(ntb), "link up\n"); 486250079Scarl callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 487255268Scarl } 488250079Scarl 489250079Scarl return (0); 490250079Scarl 491250079Scarlerr: 492250079Scarl free(nt->qps, M_NTB_IF); 493250079Scarl ntb_unregister_transport(ntb); 494250079Scarl return (rc); 495250079Scarl} 496250079Scarl 497250079Scarlstatic void 498250079Scarlntb_transport_free(void *transport) 499250079Scarl{ 500250079Scarl struct ntb_netdev *nt = transport; 501250079Scarl struct ntb_softc *ntb = nt->ntb; 502250079Scarl int i; 503250079Scarl 504250079Scarl nt->transport_link = NTB_LINK_DOWN; 505250079Scarl 506250079Scarl callout_drain(&nt->link_work); 507250079Scarl 508255281Scarl /* verify that all the qps are freed */ 509250079Scarl for (i = 0; i < nt->max_qps; i++) 510250079Scarl if (!test_bit(i, &nt->qp_bitmap)) 511250079Scarl ntb_transport_free_queue(&nt->qps[i]); 512250079Scarl 513250079Scarl 514250079Scarl ntb_unregister_event_callback(ntb); 515250079Scarl 516250079Scarl for (i = 0; i < NTB_NUM_MW; i++) 517250079Scarl if (nt->mw[i].virt_addr != NULL) 518250079Scarl contigfree(nt->mw[i].virt_addr, nt->mw[i].size, 519250079Scarl M_NTB_IF); 520250079Scarl 521250079Scarl free(nt->qps, M_NTB_IF); 522250079Scarl ntb_unregister_transport(ntb); 523250079Scarl} 524250079Scarl 525250079Scarlstatic void 526250079Scarlntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num) 527250079Scarl{ 528250079Scarl struct ntb_transport_qp *qp; 529250079Scarl unsigned int num_qps_mw, tx_size; 530250079Scarl uint8_t mw_num = QP_TO_MW(qp_num); 531250079Scarl 532250079Scarl qp = &nt->qps[qp_num]; 533250079Scarl qp->qp_num = qp_num; 534250079Scarl qp->transport = nt; 535250079Scarl qp->ntb = nt->ntb; 536250079Scarl qp->qp_link = NTB_LINK_DOWN; 537250079Scarl qp->client_ready = NTB_LINK_DOWN; 538250079Scarl qp->event_handler = NULL; 539250079Scarl 540250079Scarl if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW) 541250079Scarl num_qps_mw = nt->max_qps / NTB_NUM_MW + 1; 542250079Scarl else 543250079Scarl num_qps_mw = nt->max_qps / NTB_NUM_MW; 544250079Scarl 545250079Scarl tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw; 546250079Scarl qp->rx_info = (struct ntb_rx_info *) 547250079Scarl ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) + 548250079Scarl (qp_num / NTB_NUM_MW * tx_size)); 549250079Scarl tx_size -= sizeof(struct ntb_rx_info); 550250079Scarl 551289152Scem qp->tx_mw = qp->rx_info + 1; 552250079Scarl qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header), 553250079Scarl tx_size); 554250079Scarl qp->tx_max_entry = tx_size / qp->tx_max_frame; 555250079Scarl qp->tx_index = 0; 556250079Scarl 557250079Scarl callout_init(&qp->link_work, 0); 558283291Sjkim callout_init(&qp->queue_full, 1); 559283291Sjkim callout_init(&qp->rx_full, 1); 560250079Scarl 561250079Scarl mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN); 562250079Scarl mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN); 563250079Scarl mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN); 564250079Scarl TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp); 565250079Scarl 566250079Scarl STAILQ_INIT(&qp->rx_pend_q); 567250079Scarl STAILQ_INIT(&qp->rx_free_q); 568250079Scarl STAILQ_INIT(&qp->tx_free_q); 569250079Scarl} 570250079Scarl 571250079Scarlstatic void 572250079Scarlntb_transport_free_queue(struct ntb_transport_qp *qp) 573250079Scarl{ 574250079Scarl struct ntb_queue_entry *entry; 575250079Scarl 576250079Scarl if (qp == NULL) 577250079Scarl return; 578250079Scarl 579250079Scarl callout_drain(&qp->link_work); 580250079Scarl 581250079Scarl ntb_unregister_db_callback(qp->ntb, qp->qp_num); 582250079Scarl 583250079Scarl while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 584250079Scarl free(entry, M_NTB_IF); 585250079Scarl 586250079Scarl while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) 587250079Scarl free(entry, M_NTB_IF); 588250079Scarl 589250079Scarl while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 590250079Scarl free(entry, M_NTB_IF); 591250079Scarl 592250079Scarl set_bit(qp->qp_num, &qp->transport->qp_bitmap); 593250079Scarl} 594250079Scarl 595250079Scarl/** 596250079Scarl * ntb_transport_create_queue - Create a new NTB transport layer queue 597250079Scarl * @rx_handler: receive callback function 598250079Scarl * @tx_handler: transmit callback function 599250079Scarl * @event_handler: event callback function 600250079Scarl * 601250079Scarl * Create a new NTB transport layer queue and provide the queue with a callback 602250079Scarl * routine for both transmit and receive. The receive callback routine will be 603250079Scarl * used to pass up data when the transport has received it on the queue. The 604250079Scarl * transmit callback routine will be called when the transport has completed the 605250079Scarl * transmission of the data on the queue and the data is ready to be freed. 606250079Scarl * 607250079Scarl * RETURNS: pointer to newly created ntb_queue, NULL on error. 608250079Scarl */ 609250079Scarlstatic struct ntb_transport_qp * 610250079Scarlntb_transport_create_queue(void *data, struct ntb_softc *pdev, 611250079Scarl const struct ntb_queue_handlers *handlers) 612250079Scarl{ 613250079Scarl struct ntb_queue_entry *entry; 614250079Scarl struct ntb_transport_qp *qp; 615250079Scarl struct ntb_netdev *nt; 616250079Scarl unsigned int free_queue; 617250079Scarl int rc, i; 618250079Scarl 619250079Scarl nt = ntb_find_transport(pdev); 620250079Scarl if (nt == NULL) 621250079Scarl goto err; 622250079Scarl 623250079Scarl free_queue = ffs(nt->qp_bitmap); 624250079Scarl if (free_queue == 0) 625250079Scarl goto err; 626250079Scarl 627250079Scarl /* decrement free_queue to make it zero based */ 628250079Scarl free_queue--; 629250079Scarl 630250079Scarl clear_bit(free_queue, &nt->qp_bitmap); 631250079Scarl 632250079Scarl qp = &nt->qps[free_queue]; 633250079Scarl qp->cb_data = data; 634250079Scarl qp->rx_handler = handlers->rx_handler; 635250079Scarl qp->tx_handler = handlers->tx_handler; 636250079Scarl qp->event_handler = handlers->event_handler; 637250079Scarl 638250079Scarl for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 639250079Scarl entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF, 640250079Scarl M_WAITOK|M_ZERO); 641250079Scarl entry->cb_data = nt->ifp; 642250079Scarl entry->buf = NULL; 643250079Scarl entry->len = transport_mtu; 644250079Scarl ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 645250079Scarl } 646250079Scarl 647250079Scarl for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 648250079Scarl entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF, 649250079Scarl M_WAITOK|M_ZERO); 650250079Scarl ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 651250079Scarl } 652250079Scarl 653250079Scarl rc = ntb_register_db_callback(qp->ntb, free_queue, qp, 654250079Scarl ntb_transport_rxc_db); 655250079Scarl if (rc != 0) 656250079Scarl goto err1; 657250079Scarl 658250079Scarl return (qp); 659250079Scarl 660250079Scarlerr1: 661250079Scarl while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 662250079Scarl free(entry, M_NTB_IF); 663250079Scarl while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 664250079Scarl free(entry, M_NTB_IF); 665250079Scarl set_bit(free_queue, &nt->qp_bitmap); 666250079Scarlerr: 667250079Scarl return (NULL); 668250079Scarl} 669250079Scarl 670250079Scarl/** 671250079Scarl * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 672250079Scarl * @qp: NTB transport layer queue to be enabled 673250079Scarl * 674250079Scarl * Notify NTB transport layer of client readiness to use queue 675250079Scarl */ 676250079Scarlstatic void 677250079Scarlntb_transport_link_up(struct ntb_transport_qp *qp) 678250079Scarl{ 679250079Scarl 680250079Scarl if (qp == NULL) 681250079Scarl return; 682250079Scarl 683250079Scarl qp->client_ready = NTB_LINK_UP; 684255268Scarl if (bootverbose) 685255268Scarl device_printf(ntb_get_device(qp->ntb), "qp client ready\n"); 686250079Scarl 687250079Scarl if (qp->transport->transport_link == NTB_LINK_UP) 688250079Scarl callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 689250079Scarl} 690250079Scarl 691250079Scarl 692250079Scarl 693250079Scarl/* Transport Tx */ 694250079Scarl 695250079Scarl/** 696250079Scarl * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 697250079Scarl * @qp: NTB transport layer queue the entry is to be enqueued on 698250079Scarl * @cb: per buffer pointer for callback function to use 699250079Scarl * @data: pointer to data buffer that will be sent 700250079Scarl * @len: length of the data buffer 701250079Scarl * 702250079Scarl * Enqueue a new transmit buffer onto the transport queue from which a NTB 703250079Scarl * payload will be transmitted. This assumes that a lock is behing held to 704250079Scarl * serialize access to the qp. 705250079Scarl * 706250079Scarl * RETURNS: An appropriate ERRNO error value on error, or zero for success. 707250079Scarl */ 708250079Scarlstatic int 709250079Scarlntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 710250079Scarl unsigned int len) 711250079Scarl{ 712250079Scarl struct ntb_queue_entry *entry; 713250079Scarl int rc; 714250079Scarl 715250079Scarl if (qp == NULL || qp->qp_link != NTB_LINK_UP || len == 0) { 716250079Scarl CTR0(KTR_NTB, "TX: link not up"); 717250079Scarl return (EINVAL); 718250079Scarl } 719250079Scarl 720250079Scarl entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 721250079Scarl if (entry == NULL) { 722255281Scarl CTR0(KTR_NTB, "TX: could not get entry from tx_free_q"); 723250079Scarl return (ENOMEM); 724250079Scarl } 725250079Scarl CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry); 726250079Scarl 727250079Scarl entry->cb_data = cb; 728250079Scarl entry->buf = data; 729250079Scarl entry->len = len; 730250079Scarl entry->flags = 0; 731250079Scarl 732250079Scarl rc = ntb_process_tx(qp, entry); 733250079Scarl if (rc != 0) { 734250079Scarl ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 735250079Scarl CTR1(KTR_NTB, 736250079Scarl "TX: process_tx failed. Returning entry %p to tx_free_q", 737250079Scarl entry); 738250079Scarl } 739250079Scarl return (rc); 740250079Scarl} 741250079Scarl 742250079Scarlstatic int 743250079Scarlntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry) 744250079Scarl{ 745250079Scarl void *offset; 746250079Scarl 747250079Scarl offset = (char *)qp->tx_mw + qp->tx_max_frame * qp->tx_index; 748250079Scarl CTR3(KTR_NTB, 749250079Scarl "TX: process_tx: tx_pkts=%u, tx_index=%u, remote entry=%u", 750250079Scarl qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry); 751250079Scarl if (qp->tx_index == qp->remote_rx_info->entry) { 752250079Scarl CTR0(KTR_NTB, "TX: ring full"); 753250079Scarl qp->tx_ring_full++; 754250079Scarl return (EAGAIN); 755250079Scarl } 756250079Scarl 757250079Scarl if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 758250079Scarl if (qp->tx_handler != NULL) 759250079Scarl qp->tx_handler(qp, qp->cb_data, entry->buf, 760250079Scarl EIO); 761250079Scarl 762250079Scarl ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 763250079Scarl CTR1(KTR_NTB, 764250079Scarl "TX: frame too big. returning entry %p to tx_free_q", 765250079Scarl entry); 766250079Scarl return (0); 767250079Scarl } 768250079Scarl CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset); 769250079Scarl ntb_tx_copy_task(qp, entry, offset); 770250079Scarl 771250079Scarl qp->tx_index++; 772250079Scarl qp->tx_index %= qp->tx_max_entry; 773250079Scarl 774250079Scarl qp->tx_pkts++; 775250079Scarl 776250079Scarl return (0); 777250079Scarl} 778250079Scarl 779250079Scarlstatic void 780250079Scarlntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 781250079Scarl void *offset) 782250079Scarl{ 783250079Scarl struct ntb_payload_header *hdr; 784250079Scarl 785250079Scarl CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset); 786250079Scarl if (entry->buf != NULL) 787250079Scarl m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset); 788250079Scarl 789250079Scarl hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame - 790250079Scarl sizeof(struct ntb_payload_header)); 791250079Scarl hdr->len = entry->len; /* TODO: replace with bus_space_write */ 792250079Scarl hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */ 793250079Scarl wmb(); 794250079Scarl /* TODO: replace with bus_space_write */ 795250079Scarl hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG; 796250079Scarl 797250079Scarl ntb_ring_sdb(qp->ntb, qp->qp_num); 798250079Scarl 799250079Scarl /* 800250079Scarl * The entry length can only be zero if the packet is intended to be a 801250079Scarl * "link down" or similar. Since no payload is being sent in these 802250079Scarl * cases, there is nothing to add to the completion queue. 803250079Scarl */ 804250079Scarl if (entry->len > 0) { 805250079Scarl qp->tx_bytes += entry->len; 806250079Scarl 807250079Scarl if (qp->tx_handler) 808250079Scarl qp->tx_handler(qp, qp->cb_data, entry->cb_data, 809250079Scarl entry->len); 810250079Scarl } 811250079Scarl 812250079Scarl CTR2(KTR_NTB, 813250079Scarl "TX: entry %p sent. hdr->ver = %d, Returning to tx_free_q", entry, 814250079Scarl hdr->ver); 815250079Scarl ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 816250079Scarl} 817250079Scarl 818250079Scarlstatic void 819250079Scarlntb_qp_full(void *arg) 820250079Scarl{ 821250079Scarl 822250079Scarl CTR0(KTR_NTB, "TX: qp_full callout"); 823250079Scarl ntb_start(arg); 824250079Scarl} 825250079Scarl 826250079Scarl/* Transport Rx */ 827250079Scarlstatic void 828250079Scarlntb_transport_rxc_db(void *data, int db_num) 829250079Scarl{ 830250079Scarl struct ntb_transport_qp *qp = data; 831250079Scarl 832250079Scarl ntb_transport_rx(qp); 833250079Scarl} 834250079Scarl 835250079Scarlstatic void 836250079Scarlntb_rx_pendq_full(void *arg) 837250079Scarl{ 838250079Scarl 839250079Scarl CTR0(KTR_NTB, "RX: ntb_rx_pendq_full callout"); 840250079Scarl ntb_transport_rx(arg); 841250079Scarl} 842250079Scarl 843250079Scarlstatic void 844250079Scarlntb_transport_rx(struct ntb_transport_qp *qp) 845250079Scarl{ 846250079Scarl int rc, i; 847250079Scarl 848250079Scarl /* 849250079Scarl * Limit the number of packets processed in a single interrupt to 850250079Scarl * provide fairness to others 851250079Scarl */ 852250079Scarl mtx_lock(&qp->transport->rx_lock); 853250079Scarl CTR0(KTR_NTB, "RX: transport_rx"); 854250079Scarl for (i = 0; i < NTB_RX_MAX_PKTS; i++) { 855250079Scarl rc = ntb_process_rxc(qp); 856250079Scarl if (rc != 0) { 857250079Scarl CTR0(KTR_NTB, "RX: process_rxc failed"); 858250079Scarl break; 859250079Scarl } 860250079Scarl } 861250079Scarl mtx_unlock(&qp->transport->rx_lock); 862250079Scarl} 863250079Scarl 864250079Scarlstatic int 865250079Scarlntb_process_rxc(struct ntb_transport_qp *qp) 866250079Scarl{ 867250079Scarl struct ntb_payload_header *hdr; 868250079Scarl struct ntb_queue_entry *entry; 869250079Scarl void *offset; 870250079Scarl 871250079Scarl offset = (void *) 872250079Scarl ((char *)qp->rx_buff + qp->rx_max_frame * qp->rx_index); 873250079Scarl hdr = (void *) 874250079Scarl ((char *)offset + qp->rx_max_frame - 875250079Scarl sizeof(struct ntb_payload_header)); 876250079Scarl 877250079Scarl CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index); 878250079Scarl entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 879250079Scarl if (entry == NULL) { 880250079Scarl qp->rx_err_no_buf++; 881250079Scarl CTR0(KTR_NTB, "RX: No entries in rx_pend_q"); 882250079Scarl return (ENOMEM); 883250079Scarl } 884250079Scarl callout_stop(&qp->rx_full); 885250079Scarl CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry); 886250079Scarl 887250079Scarl if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) { 888250079Scarl CTR1(KTR_NTB, 889250079Scarl "RX: hdr not done. Returning entry %p to rx_pend_q", entry); 890250079Scarl ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 891250079Scarl qp->rx_ring_empty++; 892250079Scarl return (EAGAIN); 893250079Scarl } 894250079Scarl 895250079Scarl if (hdr->ver != (uint32_t) qp->rx_pkts) { 896250079Scarl CTR3(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). " 897250079Scarl "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts, 898250079Scarl entry); 899250079Scarl ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 900250079Scarl qp->rx_err_ver++; 901250079Scarl return (EIO); 902250079Scarl } 903250079Scarl 904250079Scarl if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) { 905250079Scarl ntb_qp_link_down(qp); 906250079Scarl CTR1(KTR_NTB, 907250079Scarl "RX: link down. adding entry %p back to rx_pend_q", entry); 908250079Scarl ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 909250079Scarl goto out; 910250079Scarl } 911250079Scarl 912250079Scarl if (hdr->len <= entry->len) { 913250079Scarl entry->len = hdr->len; 914250079Scarl ntb_rx_copy_task(qp, entry, offset); 915250079Scarl } else { 916250079Scarl CTR1(KTR_NTB, 917250079Scarl "RX: len too long. Returning entry %p to rx_pend_q", entry); 918250079Scarl ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 919250079Scarl 920250079Scarl qp->rx_err_oflow++; 921250079Scarl } 922250079Scarl 923250079Scarl qp->rx_bytes += hdr->len; 924250079Scarl qp->rx_pkts++; 925250079Scarl CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts); 926250079Scarl 927250079Scarl 928250079Scarlout: 929250079Scarl /* Ensure that the data is globally visible before clearing the flag */ 930250079Scarl wmb(); 931250079Scarl hdr->flags = 0; 932250079Scarl /* TODO: replace with bus_space_write */ 933250079Scarl qp->rx_info->entry = qp->rx_index; 934250079Scarl 935250079Scarl qp->rx_index++; 936250079Scarl qp->rx_index %= qp->rx_max_entry; 937250079Scarl 938250079Scarl return (0); 939250079Scarl} 940250079Scarl 941250079Scarlstatic void 942250079Scarlntb_rx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 943250079Scarl void *offset) 944250079Scarl{ 945250079Scarl struct ifnet *ifp = entry->cb_data; 946250079Scarl unsigned int len = entry->len; 947250079Scarl struct mbuf *m; 948250079Scarl 949250079Scarl CTR2(KTR_NTB, "RX: copying %d bytes from offset %p", len, offset); 950250079Scarl m = m_devget(offset, len, 0, ifp, NULL); 951250079Scarl m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID; 952250079Scarl 953250079Scarl entry->buf = (void *)m; 954250079Scarl 955250079Scarl CTR2(KTR_NTB, 956250079Scarl "RX: copied entry %p to mbuf %p. Adding entry to rx_free_q", entry, 957250079Scarl m); 958250079Scarl ntb_list_add(&qp->ntb_rx_free_q_lock, entry, &qp->rx_free_q); 959250079Scarl 960250079Scarl taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task); 961250079Scarl} 962250079Scarl 963250079Scarlstatic void 964250079Scarlntb_rx_completion_task(void *arg, int pending) 965250079Scarl{ 966250079Scarl struct ntb_transport_qp *qp = arg; 967250079Scarl struct mbuf *m; 968250079Scarl struct ntb_queue_entry *entry; 969250079Scarl 970250079Scarl CTR0(KTR_NTB, "RX: rx_completion_task"); 971250079Scarl 972250079Scarl while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) { 973250079Scarl m = entry->buf; 974250079Scarl CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m); 975250079Scarl if (qp->rx_handler && qp->client_ready == NTB_LINK_UP) 976250079Scarl qp->rx_handler(qp, qp->cb_data, m, entry->len); 977250079Scarl 978250079Scarl entry->buf = NULL; 979250079Scarl entry->len = qp->transport->bufsize; 980250079Scarl 981250079Scarl CTR1(KTR_NTB,"RX: entry %p removed from rx_free_q " 982250079Scarl "and added to rx_pend_q", entry); 983250079Scarl ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 984250079Scarl if (qp->rx_err_no_buf > qp->last_rx_no_buf) { 985250079Scarl qp->last_rx_no_buf = qp->rx_err_no_buf; 986250079Scarl CTR0(KTR_NTB, "RX: could spawn rx task"); 987250079Scarl callout_reset(&qp->rx_full, hz / 1000, ntb_rx_pendq_full, 988250079Scarl qp); 989250079Scarl } 990250079Scarl } 991250079Scarl} 992250079Scarl 993250079Scarl/* Link Event handler */ 994250079Scarlstatic void 995250079Scarlntb_transport_event_callback(void *data, enum ntb_hw_event event) 996250079Scarl{ 997250079Scarl struct ntb_netdev *nt = data; 998250079Scarl 999250079Scarl switch (event) { 1000250079Scarl case NTB_EVENT_HW_LINK_UP: 1001255268Scarl if (bootverbose) 1002255268Scarl device_printf(ntb_get_device(nt->ntb), "HW link up\n"); 1003250079Scarl callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 1004250079Scarl break; 1005250079Scarl case NTB_EVENT_HW_LINK_DOWN: 1006255268Scarl if (bootverbose) 1007255268Scarl device_printf(ntb_get_device(nt->ntb), "HW link down\n"); 1008250079Scarl ntb_transport_link_cleanup(nt); 1009250079Scarl break; 1010250079Scarl default: 1011250079Scarl panic("ntb: Unknown NTB event"); 1012250079Scarl } 1013250079Scarl} 1014250079Scarl 1015250079Scarl/* Link bring up */ 1016250079Scarlstatic void 1017250079Scarlntb_transport_link_work(void *arg) 1018250079Scarl{ 1019250079Scarl struct ntb_netdev *nt = arg; 1020250079Scarl struct ntb_softc *ntb = nt->ntb; 1021250079Scarl struct ntb_transport_qp *qp; 1022250079Scarl uint32_t val; 1023250079Scarl int rc, i; 1024250079Scarl 1025250079Scarl /* send the local info */ 1026250079Scarl rc = ntb_write_remote_spad(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION); 1027250079Scarl if (rc != 0) 1028250079Scarl goto out; 1029250079Scarl 1030250079Scarl rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ, ntb_get_mw_size(ntb, 0)); 1031250079Scarl if (rc != 0) 1032250079Scarl goto out; 1033250079Scarl 1034250079Scarl rc = ntb_write_remote_spad(ntb, IF_NTB_MW1_SZ, ntb_get_mw_size(ntb, 1)); 1035250079Scarl if (rc != 0) 1036250079Scarl goto out; 1037250079Scarl 1038250079Scarl rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_QPS, nt->max_qps); 1039250079Scarl if (rc != 0) 1040250079Scarl goto out; 1041250079Scarl 1042250079Scarl rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val); 1043250079Scarl if (rc != 0) 1044250079Scarl goto out; 1045250079Scarl 1046250079Scarl rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val); 1047250079Scarl if (rc != 0) 1048250079Scarl goto out; 1049250079Scarl 1050250079Scarl /* Query the remote side for its info */ 1051250079Scarl rc = ntb_read_local_spad(ntb, IF_NTB_VERSION, &val); 1052250079Scarl if (rc != 0) 1053250079Scarl goto out; 1054250079Scarl 1055250079Scarl if (val != NTB_TRANSPORT_VERSION) 1056250079Scarl goto out; 1057250079Scarl 1058250079Scarl rc = ntb_read_local_spad(ntb, IF_NTB_NUM_QPS, &val); 1059250079Scarl if (rc != 0) 1060250079Scarl goto out; 1061250079Scarl 1062250079Scarl if (val != nt->max_qps) 1063250079Scarl goto out; 1064250079Scarl 1065250079Scarl rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ, &val); 1066250079Scarl if (rc != 0) 1067250079Scarl goto out; 1068250079Scarl 1069250079Scarl if (val == 0) 1070250079Scarl goto out; 1071250079Scarl 1072250079Scarl rc = ntb_set_mw(nt, 0, val); 1073250079Scarl if (rc != 0) 1074250079Scarl return; 1075250079Scarl 1076250079Scarl rc = ntb_read_local_spad(ntb, IF_NTB_MW1_SZ, &val); 1077250079Scarl if (rc != 0) 1078250079Scarl goto out; 1079250079Scarl 1080250079Scarl if (val == 0) 1081250079Scarl goto out; 1082250079Scarl 1083250079Scarl rc = ntb_set_mw(nt, 1, val); 1084250079Scarl if (rc != 0) 1085250079Scarl return; 1086250079Scarl 1087250079Scarl nt->transport_link = NTB_LINK_UP; 1088255268Scarl if (bootverbose) 1089255268Scarl device_printf(ntb_get_device(ntb), "transport link up\n"); 1090250079Scarl 1091250079Scarl for (i = 0; i < nt->max_qps; i++) { 1092250079Scarl qp = &nt->qps[i]; 1093250079Scarl 1094250079Scarl ntb_transport_setup_qp_mw(nt, i); 1095250079Scarl 1096250079Scarl if (qp->client_ready == NTB_LINK_UP) 1097250079Scarl callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 1098250079Scarl } 1099250079Scarl 1100250079Scarl return; 1101250079Scarl 1102250079Scarlout: 1103250079Scarl if (ntb_query_link_status(ntb)) 1104250079Scarl callout_reset(&nt->link_work, 1105250079Scarl NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt); 1106250079Scarl} 1107250079Scarl 1108250079Scarlstatic int 1109250079Scarlntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size) 1110250079Scarl{ 1111250079Scarl struct ntb_transport_mw *mw = &nt->mw[num_mw]; 1112250079Scarl 1113250079Scarl /* Alloc memory for receiving data. Must be 4k aligned */ 1114250079Scarl mw->size = size; 1115250079Scarl 1116250079Scarl mw->virt_addr = contigmalloc(mw->size, M_NTB_IF, M_ZERO, 0, 1117250079Scarl BUS_SPACE_MAXADDR, mw->size, 0); 1118250079Scarl if (mw->virt_addr == NULL) { 1119250079Scarl printf("ntb: Unable to allocate MW buffer of size %d\n", 1120250079Scarl (int)mw->size); 1121250079Scarl return (ENOMEM); 1122250079Scarl } 1123250079Scarl /* TODO: replace with bus_space_* functions */ 1124250079Scarl mw->dma_addr = vtophys(mw->virt_addr); 1125250079Scarl 1126250079Scarl /* Notify HW the memory location of the receive buffer */ 1127250079Scarl ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr); 1128250079Scarl 1129250079Scarl return (0); 1130250079Scarl} 1131250079Scarl 1132250079Scarlstatic void 1133250079Scarlntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num) 1134250079Scarl{ 1135250079Scarl struct ntb_transport_qp *qp = &nt->qps[qp_num]; 1136250079Scarl void *offset; 1137250079Scarl unsigned int rx_size, num_qps_mw; 1138250079Scarl uint8_t mw_num = QP_TO_MW(qp_num); 1139250079Scarl unsigned int i; 1140250079Scarl 1141250079Scarl if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW) 1142250079Scarl num_qps_mw = nt->max_qps / NTB_NUM_MW + 1; 1143250079Scarl else 1144250079Scarl num_qps_mw = nt->max_qps / NTB_NUM_MW; 1145250079Scarl 1146250079Scarl rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw; 1147250079Scarl qp->remote_rx_info = (void *)((uint8_t *)nt->mw[mw_num].virt_addr + 1148250079Scarl (qp_num / NTB_NUM_MW * rx_size)); 1149250079Scarl rx_size -= sizeof(struct ntb_rx_info); 1150250079Scarl 1151289152Scem qp->rx_buff = qp->remote_rx_info + 1; 1152250079Scarl qp->rx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header), 1153250079Scarl rx_size); 1154250079Scarl qp->rx_max_entry = rx_size / qp->rx_max_frame; 1155250079Scarl qp->rx_index = 0; 1156250079Scarl qp->tx_index = 0; 1157250079Scarl 1158250079Scarl qp->remote_rx_info->entry = qp->rx_max_entry; 1159250079Scarl 1160250079Scarl /* setup the hdr offsets with 0's */ 1161250079Scarl for (i = 0; i < qp->rx_max_entry; i++) { 1162250079Scarl offset = (void *)((uint8_t *)qp->rx_buff + 1163250079Scarl qp->rx_max_frame * (i + 1) - 1164250079Scarl sizeof(struct ntb_payload_header)); 1165250079Scarl memset(offset, 0, sizeof(struct ntb_payload_header)); 1166250079Scarl } 1167250079Scarl 1168250079Scarl qp->rx_pkts = 0; 1169250079Scarl qp->tx_pkts = 0; 1170250079Scarl} 1171250079Scarl 1172250079Scarlstatic void 1173250079Scarlntb_qp_link_work(void *arg) 1174250079Scarl{ 1175250079Scarl struct ntb_transport_qp *qp = arg; 1176250079Scarl struct ntb_softc *ntb = qp->ntb; 1177250079Scarl struct ntb_netdev *nt = qp->transport; 1178250079Scarl int rc, val; 1179250079Scarl 1180250079Scarl 1181250079Scarl rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val); 1182250079Scarl if (rc != 0) 1183250079Scarl return; 1184250079Scarl 1185250079Scarl rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num); 1186250079Scarl 1187250079Scarl /* query remote spad for qp ready bits */ 1188250079Scarl rc = ntb_read_local_spad(ntb, IF_NTB_QP_LINKS, &val); 1189250079Scarl 1190250079Scarl /* See if the remote side is up */ 1191250079Scarl if ((1 << qp->qp_num & val) != 0) { 1192250079Scarl qp->qp_link = NTB_LINK_UP; 1193250079Scarl if (qp->event_handler != NULL) 1194250079Scarl qp->event_handler(qp->cb_data, NTB_LINK_UP); 1195255268Scarl if (bootverbose) 1196255268Scarl device_printf(ntb_get_device(ntb), "qp link up\n"); 1197250079Scarl } else if (nt->transport_link == NTB_LINK_UP) { 1198250079Scarl callout_reset(&qp->link_work, 1199250079Scarl NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1200250079Scarl } 1201250079Scarl} 1202250079Scarl 1203250079Scarl/* Link down event*/ 1204250079Scarlstatic void 1205250079Scarlntb_transport_link_cleanup(struct ntb_netdev *nt) 1206250079Scarl{ 1207250079Scarl int i; 1208250079Scarl 1209250079Scarl if (nt->transport_link == NTB_LINK_DOWN) 1210250079Scarl callout_drain(&nt->link_work); 1211250079Scarl else 1212250079Scarl nt->transport_link = NTB_LINK_DOWN; 1213250079Scarl 1214250079Scarl /* Pass along the info to any clients */ 1215250079Scarl for (i = 0; i < nt->max_qps; i++) 1216250079Scarl if (!test_bit(i, &nt->qp_bitmap)) 1217250079Scarl ntb_qp_link_down(&nt->qps[i]); 1218250079Scarl 1219250079Scarl /* 1220250079Scarl * The scratchpad registers keep the values if the remote side 1221250079Scarl * goes down, blast them now to give them a sane value the next 1222250079Scarl * time they are accessed 1223250079Scarl */ 1224250079Scarl for (i = 0; i < IF_NTB_MAX_SPAD; i++) 1225250079Scarl ntb_write_local_spad(nt->ntb, i, 0); 1226250079Scarl} 1227250079Scarl 1228250079Scarl 1229250079Scarlstatic void 1230250079Scarlntb_qp_link_down(struct ntb_transport_qp *qp) 1231250079Scarl{ 1232250079Scarl 1233250079Scarl ntb_qp_link_cleanup(qp); 1234250079Scarl} 1235250079Scarl 1236250079Scarlstatic void 1237250079Scarlntb_qp_link_cleanup(struct ntb_transport_qp *qp) 1238250079Scarl{ 1239250079Scarl struct ntb_netdev *nt = qp->transport; 1240250079Scarl 1241250079Scarl if (qp->qp_link == NTB_LINK_DOWN) { 1242250079Scarl callout_drain(&qp->link_work); 1243250079Scarl return; 1244250079Scarl } 1245250079Scarl 1246250079Scarl if (qp->event_handler != NULL) 1247250079Scarl qp->event_handler(qp->cb_data, NTB_LINK_DOWN); 1248250079Scarl 1249250079Scarl qp->qp_link = NTB_LINK_DOWN; 1250250079Scarl 1251250079Scarl if (nt->transport_link == NTB_LINK_UP) 1252250079Scarl callout_reset(&qp->link_work, 1253250079Scarl NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1254250079Scarl} 1255250079Scarl 1256250079Scarl/* Link commanded down */ 1257250079Scarl/** 1258250079Scarl * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 1259250079Scarl * @qp: NTB transport layer queue to be disabled 1260250079Scarl * 1261250079Scarl * Notify NTB transport layer of client's desire to no longer receive data on 1262250079Scarl * transport queue specified. It is the client's responsibility to ensure all 1263250079Scarl * entries on queue are purged or otherwise handled appropraitely. 1264250079Scarl */ 1265250079Scarlstatic void 1266250079Scarlntb_transport_link_down(struct ntb_transport_qp *qp) 1267250079Scarl{ 1268250079Scarl int rc, val; 1269250079Scarl 1270250079Scarl if (qp == NULL) 1271250079Scarl return; 1272250079Scarl 1273250079Scarl qp->client_ready = NTB_LINK_DOWN; 1274250079Scarl 1275250079Scarl rc = ntb_read_remote_spad(qp->ntb, IF_NTB_QP_LINKS, &val); 1276250079Scarl if (rc != 0) 1277250079Scarl return; 1278250079Scarl 1279250079Scarl rc = ntb_write_remote_spad(qp->ntb, IF_NTB_QP_LINKS, 1280250079Scarl val & ~(1 << qp->qp_num)); 1281250079Scarl 1282250079Scarl if (qp->qp_link == NTB_LINK_UP) 1283250079Scarl ntb_send_link_down(qp); 1284250079Scarl else 1285250079Scarl callout_drain(&qp->link_work); 1286250079Scarl 1287250079Scarl} 1288250079Scarl 1289250079Scarlstatic void 1290250079Scarlntb_send_link_down(struct ntb_transport_qp *qp) 1291250079Scarl{ 1292250079Scarl struct ntb_queue_entry *entry; 1293250079Scarl int i, rc; 1294250079Scarl 1295250079Scarl if (qp->qp_link == NTB_LINK_DOWN) 1296250079Scarl return; 1297250079Scarl 1298250079Scarl qp->qp_link = NTB_LINK_DOWN; 1299250079Scarl 1300250079Scarl for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1301250079Scarl entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1302250079Scarl if (entry != NULL) 1303250079Scarl break; 1304250079Scarl pause("NTB Wait for link down", hz / 10); 1305250079Scarl } 1306250079Scarl 1307250079Scarl if (entry == NULL) 1308250079Scarl return; 1309250079Scarl 1310250079Scarl entry->cb_data = NULL; 1311250079Scarl entry->buf = NULL; 1312250079Scarl entry->len = 0; 1313250079Scarl entry->flags = IF_NTB_LINK_DOWN_FLAG; 1314250079Scarl 1315250079Scarl mtx_lock(&qp->transport->tx_lock); 1316250079Scarl rc = ntb_process_tx(qp, entry); 1317250079Scarl if (rc != 0) 1318250079Scarl printf("ntb: Failed to send link down\n"); 1319250079Scarl mtx_unlock(&qp->transport->tx_lock); 1320250079Scarl} 1321250079Scarl 1322250079Scarl 1323250079Scarl/* List Management */ 1324250079Scarl 1325250079Scarlstatic void 1326250079Scarlntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 1327250079Scarl struct ntb_queue_list *list) 1328250079Scarl{ 1329250079Scarl 1330250079Scarl mtx_lock_spin(lock); 1331250079Scarl STAILQ_INSERT_TAIL(list, entry, entry); 1332250079Scarl mtx_unlock_spin(lock); 1333250079Scarl} 1334250079Scarl 1335250079Scarlstatic struct ntb_queue_entry * 1336250079Scarlntb_list_rm(struct mtx *lock, struct ntb_queue_list *list) 1337250079Scarl{ 1338250079Scarl struct ntb_queue_entry *entry; 1339250079Scarl 1340250079Scarl mtx_lock_spin(lock); 1341250079Scarl if (STAILQ_EMPTY(list)) { 1342250079Scarl entry = NULL; 1343250079Scarl goto out; 1344250079Scarl } 1345250079Scarl entry = STAILQ_FIRST(list); 1346250079Scarl STAILQ_REMOVE_HEAD(list, entry); 1347250079Scarlout: 1348250079Scarl mtx_unlock_spin(lock); 1349250079Scarl 1350250079Scarl return (entry); 1351250079Scarl} 1352250079Scarl 1353250079Scarl/* Helper functions */ 1354250079Scarl/* TODO: This too should really be part of the kernel */ 1355250079Scarl#define EUI48_MULTICAST 1 << 0 1356250079Scarl#define EUI48_LOCALLY_ADMINISTERED 1 << 1 1357250079Scarlstatic void 1358250079Scarlcreate_random_local_eui48(u_char *eaddr) 1359250079Scarl{ 1360250079Scarl static uint8_t counter = 0; 1361250079Scarl uint32_t seed = ticks; 1362250079Scarl 1363250079Scarl eaddr[0] = EUI48_LOCALLY_ADMINISTERED; 1364250079Scarl memcpy(&eaddr[1], &seed, sizeof(uint32_t)); 1365250079Scarl eaddr[5] = counter++; 1366250079Scarl} 1367250079Scarl 1368250079Scarl/** 1369250079Scarl * ntb_transport_max_size - Query the max payload size of a qp 1370250079Scarl * @qp: NTB transport layer queue to be queried 1371250079Scarl * 1372250079Scarl * Query the maximum payload size permissible on the given qp 1373250079Scarl * 1374250079Scarl * RETURNS: the max payload size of a qp 1375250079Scarl */ 1376250079Scarlstatic unsigned int 1377250079Scarlntb_transport_max_size(struct ntb_transport_qp *qp) 1378250079Scarl{ 1379250079Scarl 1380250079Scarl if (qp == NULL) 1381250079Scarl return (0); 1382250079Scarl 1383250079Scarl return (qp->tx_max_frame - sizeof(struct ntb_payload_header)); 1384250079Scarl} 1385