ntb_transport.c revision 291705
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3289545Scem * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/if_ntb/if_ntb.c 291705 2015-12-03 17:22:45Z cem $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34289544Scem#include <sys/bitset.h> 35250079Scarl#include <sys/bus.h> 36250079Scarl#include <sys/ktr.h> 37289281Scem#include <sys/limits.h> 38250079Scarl#include <sys/lock.h> 39250079Scarl#include <sys/malloc.h> 40250079Scarl#include <sys/module.h> 41250079Scarl#include <sys/mutex.h> 42250079Scarl#include <sys/queue.h> 43250079Scarl#include <sys/socket.h> 44250079Scarl#include <sys/sockio.h> 45289207Scem#include <sys/sysctl.h> 46250079Scarl#include <sys/taskqueue.h> 47289544Scem 48250079Scarl#include <net/if.h> 49250079Scarl#include <net/if_media.h> 50250079Scarl#include <net/if_types.h> 51250079Scarl#include <net/if_var.h> 52250079Scarl#include <net/bpf.h> 53250079Scarl#include <net/ethernet.h> 54289544Scem 55250079Scarl#include <vm/vm.h> 56250079Scarl#include <vm/pmap.h> 57289544Scem 58250079Scarl#include <machine/bus.h> 59250079Scarl#include <machine/cpufunc.h> 60250079Scarl#include <machine/pmap.h> 61250079Scarl 62291085Scem#include <netinet/in.h> 63291085Scem#include <netinet/ip.h> 64291085Scem 65250079Scarl#include "../ntb_hw/ntb_hw.h" 66250079Scarl 67250079Scarl/* 68250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 69250079Scarl * allows you to connect two systems using a PCI-e link. 70250079Scarl * 71250079Scarl * This module contains a protocol for sending and receiving messages, and 72250079Scarl * exposes that protocol through a simulated ethernet device called ntb. 73250079Scarl * 74250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 75250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 76250079Scarl */ 77250079Scarl 78289544Scem#define QP_SETSIZE 64 79289544ScemBITSET_DEFINE(_qpset, QP_SETSIZE); 80289544Scem#define test_bit(pos, addr) BIT_ISSET(QP_SETSIZE, (pos), (addr)) 81289544Scem#define set_bit(pos, addr) BIT_SET(QP_SETSIZE, (pos), (addr)) 82289544Scem#define clear_bit(pos, addr) BIT_CLR(QP_SETSIZE, (pos), (addr)) 83289544Scem#define ffs_bit(addr) BIT_FFS(QP_SETSIZE, (addr)) 84250079Scarl 85250079Scarl#define KTR_NTB KTR_SPARE3 86250079Scarl 87289546Scem#define NTB_TRANSPORT_VERSION 4 88250079Scarl#define NTB_RX_MAX_PKTS 64 89250079Scarl#define NTB_RXQ_SIZE 300 90250079Scarl 91289546Scemenum ntb_link_event { 92289546Scem NTB_LINK_DOWN = 0, 93289546Scem NTB_LINK_UP, 94289546Scem}; 95289546Scem 96290684Scemstatic SYSCTL_NODE(_hw, OID_AUTO, if_ntb, CTLFLAG_RW, 0, "if_ntb"); 97289208Scem 98290684Scemstatic unsigned g_if_ntb_debug_level; 99290684ScemSYSCTL_UINT(_hw_if_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 100290684Scem &g_if_ntb_debug_level, 0, "if_ntb log level -- higher is more verbose"); 101290684Scem#define ntb_printf(lvl, ...) do { \ 102290684Scem if ((lvl) <= g_if_ntb_debug_level) { \ 103290684Scem if_printf(nt->ifp, __VA_ARGS__); \ 104290684Scem } \ 105290684Scem} while (0) 106290684Scem 107291085Scemstatic unsigned transport_mtu = IP_MAXPACKET + ETHER_HDR_LEN + ETHER_CRC_LEN; 108290684Scem 109289546Scemstatic uint64_t max_mw_size; 110290684ScemSYSCTL_UQUAD(_hw_if_ntb, OID_AUTO, max_mw_size, CTLFLAG_RDTUN, &max_mw_size, 0, 111289546Scem "If enabled (non-zero), limit the size of large memory windows. " 112289546Scem "Both sides of the NTB MUST set the same value here."); 113289546Scem 114290684Scemstatic unsigned max_num_clients; 115290684ScemSYSCTL_UINT(_hw_if_ntb, OID_AUTO, max_num_clients, CTLFLAG_RDTUN, 116289396Scem &max_num_clients, 0, "Maximum number of NTB transport clients. " 117289396Scem "0 (default) - use all available NTB memory windows; " 118289396Scem "positive integer N - Limit to N memory windows."); 119250079Scarl 120291084Scemstatic unsigned enable_xeon_watchdog; 121291084ScemSYSCTL_UINT(_hw_if_ntb, OID_AUTO, enable_xeon_watchdog, CTLFLAG_RDTUN, 122291084Scem &enable_xeon_watchdog, 0, "If non-zero, write a register every second to " 123291084Scem "keep a watchdog from tearing down the NTB link"); 124291084Scem 125250079ScarlSTAILQ_HEAD(ntb_queue_list, ntb_queue_entry); 126250079Scarl 127291028Scemtypedef uint32_t ntb_q_idx_t; 128289653Scem 129250079Scarlstruct ntb_queue_entry { 130250079Scarl /* ntb_queue list reference */ 131250079Scarl STAILQ_ENTRY(ntb_queue_entry) entry; 132250079Scarl 133289546Scem /* info on data to be transferred */ 134250079Scarl void *cb_data; 135250079Scarl void *buf; 136291028Scem uint32_t len; 137291028Scem uint32_t flags; 138289546Scem 139289546Scem struct ntb_transport_qp *qp; 140289546Scem struct ntb_payload_header *x_hdr; 141289653Scem ntb_q_idx_t index; 142250079Scarl}; 143250079Scarl 144250079Scarlstruct ntb_rx_info { 145289653Scem ntb_q_idx_t entry; 146250079Scarl}; 147250079Scarl 148250079Scarlstruct ntb_transport_qp { 149289545Scem struct ntb_transport_ctx *transport; 150250079Scarl struct ntb_softc *ntb; 151250079Scarl 152250079Scarl void *cb_data; 153250079Scarl 154250079Scarl bool client_ready; 155290686Scem volatile bool link_is_up; 156255281Scarl uint8_t qp_num; /* Only 64 QPs are allowed. 0-63 */ 157250079Scarl 158250079Scarl struct ntb_rx_info *rx_info; 159250079Scarl struct ntb_rx_info *remote_rx_info; 160250079Scarl 161289341Scem void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 162250079Scarl void *data, int len); 163250079Scarl struct ntb_queue_list tx_free_q; 164250079Scarl struct mtx ntb_tx_free_q_lock; 165290679Scem caddr_t tx_mw; 166289546Scem bus_addr_t tx_mw_phys; 167289653Scem ntb_q_idx_t tx_index; 168289653Scem ntb_q_idx_t tx_max_entry; 169250079Scarl uint64_t tx_max_frame; 170250079Scarl 171289341Scem void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 172250079Scarl void *data, int len); 173289651Scem struct ntb_queue_list rx_post_q; 174250079Scarl struct ntb_queue_list rx_pend_q; 175289651Scem /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ 176289651Scem struct mtx ntb_rx_q_lock; 177250079Scarl struct task rx_completion_task; 178289546Scem struct task rxc_db_work; 179290679Scem caddr_t rx_buff; 180289653Scem ntb_q_idx_t rx_index; 181289653Scem ntb_q_idx_t rx_max_entry; 182250079Scarl uint64_t rx_max_frame; 183250079Scarl 184289341Scem void (*event_handler)(void *data, enum ntb_link_event status); 185250079Scarl struct callout link_work; 186250079Scarl struct callout queue_full; 187250079Scarl struct callout rx_full; 188250079Scarl 189250079Scarl uint64_t last_rx_no_buf; 190250079Scarl 191250079Scarl /* Stats */ 192250079Scarl uint64_t rx_bytes; 193250079Scarl uint64_t rx_pkts; 194250079Scarl uint64_t rx_ring_empty; 195250079Scarl uint64_t rx_err_no_buf; 196250079Scarl uint64_t rx_err_oflow; 197250079Scarl uint64_t rx_err_ver; 198250079Scarl uint64_t tx_bytes; 199250079Scarl uint64_t tx_pkts; 200250079Scarl uint64_t tx_ring_full; 201289653Scem uint64_t tx_err_no_buf; 202250079Scarl}; 203250079Scarl 204250079Scarlstruct ntb_queue_handlers { 205289341Scem void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 206250079Scarl void *data, int len); 207289341Scem void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 208250079Scarl void *data, int len); 209289341Scem void (*event_handler)(void *data, enum ntb_link_event status); 210250079Scarl}; 211250079Scarl 212250079Scarlstruct ntb_transport_mw { 213289546Scem vm_paddr_t phys_addr; 214289546Scem size_t phys_size; 215289546Scem size_t xlat_align; 216289546Scem size_t xlat_align_size; 217291033Scem bus_addr_t addr_limit; 218289546Scem /* Tx buff is off vbase / phys_addr */ 219290679Scem caddr_t vbase; 220289545Scem size_t xlat_size; 221289546Scem size_t buff_size; 222289546Scem /* Rx buff is off virt_addr / dma_addr */ 223290679Scem caddr_t virt_addr; 224289546Scem bus_addr_t dma_addr; 225250079Scarl}; 226250079Scarl 227289545Scemstruct ntb_transport_ctx { 228250079Scarl struct ntb_softc *ntb; 229250079Scarl struct ifnet *ifp; 230289545Scem struct ntb_transport_mw mw_vec[NTB_MAX_NUM_MW]; 231289545Scem struct ntb_transport_qp *qp_vec; 232289544Scem struct _qpset qp_bitmap; 233289546Scem struct _qpset qp_bitmap_free; 234289545Scem unsigned mw_count; 235289545Scem unsigned qp_count; 236290686Scem volatile bool link_is_up; 237250079Scarl struct callout link_work; 238291084Scem struct callout link_watchdog; 239290683Scem struct task link_cleanup; 240250079Scarl uint64_t bufsize; 241250079Scarl u_char eaddr[ETHER_ADDR_LEN]; 242250079Scarl struct mtx tx_lock; 243250079Scarl struct mtx rx_lock; 244289546Scem 245289546Scem /* The hardcoded single queuepair in ntb_setup_interface() */ 246289546Scem struct ntb_transport_qp *qp; 247250079Scarl}; 248250079Scarl 249289545Scemstatic struct ntb_transport_ctx net_softc; 250250079Scarl 251250079Scarlenum { 252250079Scarl IF_NTB_DESC_DONE_FLAG = 1 << 0, 253250079Scarl IF_NTB_LINK_DOWN_FLAG = 1 << 1, 254250079Scarl}; 255250079Scarl 256250079Scarlstruct ntb_payload_header { 257291028Scem ntb_q_idx_t ver; 258291028Scem uint32_t len; 259291028Scem uint32_t flags; 260250079Scarl}; 261250079Scarl 262250079Scarlenum { 263289153Scem /* 264289153Scem * The order of this enum is part of the if_ntb remote protocol. Do 265289153Scem * not reorder without bumping protocol version (and it's probably best 266289153Scem * to keep the protocol in lock-step with the Linux NTB driver. 267289153Scem */ 268250079Scarl IF_NTB_VERSION = 0, 269289153Scem IF_NTB_QP_LINKS, 270250079Scarl IF_NTB_NUM_QPS, 271289153Scem IF_NTB_NUM_MWS, 272289153Scem /* 273289153Scem * N.B.: transport_link_work assumes MW1 enums = MW0 + 2. 274289153Scem */ 275289153Scem IF_NTB_MW0_SZ_HIGH, 276289153Scem IF_NTB_MW0_SZ_LOW, 277289153Scem IF_NTB_MW1_SZ_HIGH, 278289153Scem IF_NTB_MW1_SZ_LOW, 279250079Scarl IF_NTB_MAX_SPAD, 280291084Scem 281291084Scem /* 282291084Scem * Some NTB-using hardware have a watchdog to work around NTB hangs; if 283291084Scem * a register or doorbell isn't written every few seconds, the link is 284291084Scem * torn down. Write an otherwise unused register every few seconds to 285291084Scem * work around this watchdog. 286291084Scem */ 287291084Scem IF_NTB_WATCHDOG_SPAD = 15 288250079Scarl}; 289291084ScemCTASSERT(IF_NTB_WATCHDOG_SPAD < XEON_SPAD_COUNT && 290291084Scem IF_NTB_WATCHDOG_SPAD < ATOM_SPAD_COUNT); 291250079Scarl 292289545Scem#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 293250079Scarl#define NTB_QP_DEF_NUM_ENTRIES 100 294250079Scarl#define NTB_LINK_DOWN_TIMEOUT 10 295250079Scarl 296250079Scarlstatic int ntb_handle_module_events(struct module *m, int what, void *arg); 297250079Scarlstatic int ntb_setup_interface(void); 298250079Scarlstatic int ntb_teardown_interface(void); 299250079Scarlstatic void ntb_net_init(void *arg); 300250079Scarlstatic int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 301250079Scarlstatic void ntb_start(struct ifnet *ifp); 302250079Scarlstatic void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, 303250079Scarl void *data, int len); 304250079Scarlstatic void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, 305250079Scarl void *data, int len); 306289340Scemstatic void ntb_net_event_handler(void *data, enum ntb_link_event status); 307290682Scemstatic int ntb_transport_probe(struct ntb_softc *ntb); 308289546Scemstatic void ntb_transport_free(struct ntb_transport_ctx *); 309289545Scemstatic void ntb_transport_init_queue(struct ntb_transport_ctx *nt, 310250079Scarl unsigned int qp_num); 311250079Scarlstatic void ntb_transport_free_queue(struct ntb_transport_qp *qp); 312289546Scemstatic struct ntb_transport_qp *ntb_transport_create_queue(void *data, 313250079Scarl struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers); 314250079Scarlstatic void ntb_transport_link_up(struct ntb_transport_qp *qp); 315250079Scarlstatic int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, 316250079Scarl void *data, unsigned int len); 317250079Scarlstatic int ntb_process_tx(struct ntb_transport_qp *qp, 318250079Scarl struct ntb_queue_entry *entry); 319289546Scemstatic void ntb_memcpy_tx(struct ntb_transport_qp *qp, 320250079Scarl struct ntb_queue_entry *entry, void *offset); 321250079Scarlstatic void ntb_qp_full(void *arg); 322289546Scemstatic void ntb_transport_rxc_db(void *arg, int pending); 323250079Scarlstatic int ntb_process_rxc(struct ntb_transport_qp *qp); 324289651Scemstatic void ntb_memcpy_rx(struct ntb_transport_qp *qp, 325250079Scarl struct ntb_queue_entry *entry, void *offset); 326289651Scemstatic inline void ntb_rx_copy_callback(struct ntb_transport_qp *qp, 327289651Scem void *data); 328289546Scemstatic void ntb_complete_rxc(void *arg, int pending); 329289598Scemstatic void ntb_transport_doorbell_callback(void *data, uint32_t vector); 330289546Scemstatic void ntb_transport_event_callback(void *data); 331250079Scarlstatic void ntb_transport_link_work(void *arg); 332289652Scemstatic int ntb_set_mw(struct ntb_transport_ctx *, int num_mw, size_t size); 333289545Scemstatic void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw); 334289546Scemstatic int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, 335250079Scarl unsigned int qp_num); 336250079Scarlstatic void ntb_qp_link_work(void *arg); 337289545Scemstatic void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt); 338290683Scemstatic void ntb_transport_link_cleanup_work(void *, int); 339250079Scarlstatic void ntb_qp_link_down(struct ntb_transport_qp *qp); 340289613Scemstatic void ntb_qp_link_down_reset(struct ntb_transport_qp *qp); 341250079Scarlstatic void ntb_qp_link_cleanup(struct ntb_transport_qp *qp); 342250079Scarlstatic void ntb_transport_link_down(struct ntb_transport_qp *qp); 343250079Scarlstatic void ntb_send_link_down(struct ntb_transport_qp *qp); 344250079Scarlstatic void ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 345250079Scarl struct ntb_queue_list *list); 346250079Scarlstatic struct ntb_queue_entry *ntb_list_rm(struct mtx *lock, 347250079Scarl struct ntb_queue_list *list); 348289651Scemstatic struct ntb_queue_entry *ntb_list_mv(struct mtx *lock, 349289651Scem struct ntb_queue_list *from, struct ntb_queue_list *to); 350250079Scarlstatic void create_random_local_eui48(u_char *eaddr); 351250079Scarlstatic unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); 352291084Scemstatic void xeon_link_watchdog_hb(void *); 353250079Scarl 354289546Scemstatic const struct ntb_ctx_ops ntb_transport_ops = { 355289546Scem .link_event = ntb_transport_event_callback, 356289546Scem .db_event = ntb_transport_doorbell_callback, 357289546Scem}; 358289546Scem 359250079ScarlMALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver"); 360250079Scarl 361291028Scemstatic inline void 362291028Scemiowrite32(uint32_t val, void *addr) 363291028Scem{ 364291028Scem 365291028Scem bus_space_write_4(X86_BUS_SPACE_MEM, 0/* HACK */, (uintptr_t)addr, 366291028Scem val); 367291028Scem} 368291028Scem 369250079Scarl/* Module setup and teardown */ 370250079Scarlstatic int 371250079Scarlntb_handle_module_events(struct module *m, int what, void *arg) 372250079Scarl{ 373250079Scarl int err = 0; 374250079Scarl 375250079Scarl switch (what) { 376250079Scarl case MOD_LOAD: 377250079Scarl err = ntb_setup_interface(); 378250079Scarl break; 379250079Scarl case MOD_UNLOAD: 380250079Scarl err = ntb_teardown_interface(); 381250079Scarl break; 382250079Scarl default: 383250079Scarl err = EOPNOTSUPP; 384250079Scarl break; 385250079Scarl } 386250079Scarl return (err); 387250079Scarl} 388250079Scarl 389255271Scarlstatic moduledata_t if_ntb_mod = { 390255271Scarl "if_ntb", 391250079Scarl ntb_handle_module_events, 392250079Scarl NULL 393250079Scarl}; 394250079Scarl 395255271ScarlDECLARE_MODULE(if_ntb, if_ntb_mod, SI_SUB_KLD, SI_ORDER_ANY); 396255271ScarlMODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1); 397250079Scarl 398250079Scarlstatic int 399289209Scemntb_setup_interface(void) 400250079Scarl{ 401250079Scarl struct ifnet *ifp; 402250079Scarl struct ntb_queue_handlers handlers = { ntb_net_rx_handler, 403250079Scarl ntb_net_tx_handler, ntb_net_event_handler }; 404289546Scem int rc; 405250079Scarl 406250079Scarl net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0); 407250079Scarl if (net_softc.ntb == NULL) { 408255281Scarl printf("ntb: Cannot find devclass\n"); 409250079Scarl return (ENXIO); 410250079Scarl } 411250079Scarl 412250079Scarl ifp = net_softc.ifp = if_alloc(IFT_ETHER); 413250079Scarl if (ifp == NULL) { 414289546Scem ntb_transport_free(&net_softc); 415289546Scem printf("ntb: Cannot allocate ifnet structure\n"); 416250079Scarl return (ENOMEM); 417250079Scarl } 418290684Scem if_initname(ifp, "ntb", 0); 419250079Scarl 420290684Scem rc = ntb_transport_probe(net_softc.ntb); 421290684Scem if (rc != 0) { 422290684Scem printf("ntb: Cannot init transport: %d\n", rc); 423290684Scem if_free(net_softc.ifp); 424290684Scem return (rc); 425290684Scem } 426290684Scem 427250079Scarl net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb, 428250079Scarl &handlers); 429250079Scarl ifp->if_init = ntb_net_init; 430250079Scarl ifp->if_softc = &net_softc; 431250079Scarl ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX; 432250079Scarl ifp->if_ioctl = ntb_ioctl; 433250079Scarl ifp->if_start = ntb_start; 434250079Scarl IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN); 435250079Scarl ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 436250079Scarl IFQ_SET_READY(&ifp->if_snd); 437250079Scarl create_random_local_eui48(net_softc.eaddr); 438250079Scarl ether_ifattach(ifp, net_softc.eaddr); 439250079Scarl ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU; 440250079Scarl ifp->if_capenable = ifp->if_capabilities; 441291085Scem ifp->if_mtu = ntb_transport_max_size(net_softc.qp) - ETHER_HDR_LEN - 442291085Scem ETHER_CRC_LEN; 443250079Scarl 444250079Scarl ntb_transport_link_up(net_softc.qp); 445250079Scarl net_softc.bufsize = ntb_transport_max_size(net_softc.qp) + 446250079Scarl sizeof(struct ether_header); 447250079Scarl return (0); 448250079Scarl} 449250079Scarl 450250079Scarlstatic int 451289209Scemntb_teardown_interface(void) 452250079Scarl{ 453250079Scarl 454290684Scem if (net_softc.qp != NULL) { 455255280Scarl ntb_transport_link_down(net_softc.qp); 456250079Scarl 457290684Scem ntb_transport_free_queue(net_softc.qp); 458290684Scem ntb_transport_free(&net_softc); 459290684Scem } 460290684Scem 461255280Scarl if (net_softc.ifp != NULL) { 462255280Scarl ether_ifdetach(net_softc.ifp); 463255280Scarl if_free(net_softc.ifp); 464290684Scem net_softc.ifp = NULL; 465255280Scarl } 466250079Scarl 467250079Scarl return (0); 468250079Scarl} 469250079Scarl 470250079Scarl/* Network device interface */ 471250079Scarl 472250079Scarlstatic void 473250079Scarlntb_net_init(void *arg) 474250079Scarl{ 475289545Scem struct ntb_transport_ctx *ntb_softc = arg; 476250079Scarl struct ifnet *ifp = ntb_softc->ifp; 477250079Scarl 478250079Scarl ifp->if_drv_flags |= IFF_DRV_RUNNING; 479250079Scarl ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 480250079Scarl ifp->if_flags |= IFF_UP; 481250079Scarl if_link_state_change(ifp, LINK_STATE_UP); 482250079Scarl} 483250079Scarl 484250079Scarlstatic int 485250079Scarlntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 486250079Scarl{ 487289545Scem struct ntb_transport_ctx *nt = ifp->if_softc; 488250079Scarl struct ifreq *ifr = (struct ifreq *)data; 489250079Scarl int error = 0; 490250079Scarl 491250079Scarl switch (command) { 492250079Scarl case SIOCSIFMTU: 493250079Scarl { 494250079Scarl if (ifr->ifr_mtu > ntb_transport_max_size(nt->qp) - 495250079Scarl ETHER_HDR_LEN - ETHER_CRC_LEN) { 496250079Scarl error = EINVAL; 497250079Scarl break; 498250079Scarl } 499250079Scarl 500250079Scarl ifp->if_mtu = ifr->ifr_mtu; 501250079Scarl break; 502250079Scarl } 503250079Scarl default: 504250079Scarl error = ether_ioctl(ifp, command, data); 505250079Scarl break; 506250079Scarl } 507250079Scarl 508250079Scarl return (error); 509250079Scarl} 510250079Scarl 511250079Scarl 512250079Scarlstatic void 513250079Scarlntb_start(struct ifnet *ifp) 514250079Scarl{ 515250079Scarl struct mbuf *m_head; 516289545Scem struct ntb_transport_ctx *nt = ifp->if_softc; 517250079Scarl int rc; 518250079Scarl 519250079Scarl mtx_lock(&nt->tx_lock); 520250079Scarl ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 521250079Scarl CTR0(KTR_NTB, "TX: ntb_start"); 522250079Scarl while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 523250079Scarl IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 524250079Scarl CTR1(KTR_NTB, "TX: start mbuf %p", m_head); 525250079Scarl rc = ntb_transport_tx_enqueue(nt->qp, m_head, m_head, 526250079Scarl m_length(m_head, NULL)); 527250079Scarl if (rc != 0) { 528250079Scarl CTR1(KTR_NTB, 529255281Scarl "TX: could not tx mbuf %p. Returning to snd q", 530250079Scarl m_head); 531250079Scarl if (rc == EAGAIN) { 532250079Scarl ifp->if_drv_flags |= IFF_DRV_OACTIVE; 533250079Scarl IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 534250079Scarl callout_reset(&nt->qp->queue_full, hz / 1000, 535250079Scarl ntb_qp_full, ifp); 536250079Scarl } 537250079Scarl break; 538250079Scarl } 539250079Scarl 540250079Scarl } 541250079Scarl mtx_unlock(&nt->tx_lock); 542250079Scarl} 543250079Scarl 544250079Scarl/* Network Device Callbacks */ 545250079Scarlstatic void 546250079Scarlntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 547250079Scarl int len) 548250079Scarl{ 549250079Scarl 550250079Scarl m_freem(data); 551250079Scarl CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data); 552250079Scarl} 553250079Scarl 554250079Scarlstatic void 555250079Scarlntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 556250079Scarl int len) 557250079Scarl{ 558250079Scarl struct mbuf *m = data; 559250079Scarl struct ifnet *ifp = qp_data; 560250079Scarl 561250079Scarl CTR0(KTR_NTB, "RX: rx handler"); 562250079Scarl (*ifp->if_input)(ifp, m); 563250079Scarl} 564250079Scarl 565250079Scarlstatic void 566289340Scemntb_net_event_handler(void *data, enum ntb_link_event status) 567250079Scarl{ 568289340Scem struct ifnet *ifp; 569250079Scarl 570289340Scem ifp = data; 571289340Scem (void)ifp; 572289340Scem 573289340Scem /* XXX The Linux driver munges with the carrier status here. */ 574289340Scem 575289340Scem switch (status) { 576289340Scem case NTB_LINK_DOWN: 577289340Scem break; 578289340Scem case NTB_LINK_UP: 579289340Scem break; 580289340Scem default: 581289340Scem panic("Bogus ntb_link_event %u\n", status); 582289340Scem } 583250079Scarl} 584250079Scarl 585250079Scarl/* Transport Init and teardown */ 586250079Scarl 587291084Scemstatic void 588291084Scemxeon_link_watchdog_hb(void *arg) 589291084Scem{ 590291084Scem struct ntb_transport_ctx *nt; 591291084Scem 592291084Scem nt = arg; 593291084Scem ntb_spad_write(nt->ntb, IF_NTB_WATCHDOG_SPAD, 0); 594291084Scem callout_reset(&nt->link_watchdog, 1 * hz, xeon_link_watchdog_hb, nt); 595291084Scem} 596291084Scem 597250079Scarlstatic int 598290682Scemntb_transport_probe(struct ntb_softc *ntb) 599250079Scarl{ 600289545Scem struct ntb_transport_ctx *nt = &net_softc; 601289546Scem struct ntb_transport_mw *mw; 602289546Scem uint64_t qp_bitmap; 603289544Scem int rc; 604289546Scem unsigned i; 605250079Scarl 606289545Scem nt->mw_count = ntb_mw_count(ntb); 607289546Scem for (i = 0; i < nt->mw_count; i++) { 608289546Scem mw = &nt->mw_vec[i]; 609289396Scem 610289546Scem rc = ntb_mw_get_range(ntb, i, &mw->phys_addr, &mw->vbase, 611291033Scem &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size, 612291033Scem &mw->addr_limit); 613289546Scem if (rc != 0) 614289546Scem goto err; 615289546Scem 616289546Scem mw->buff_size = 0; 617289546Scem mw->xlat_size = 0; 618290679Scem mw->virt_addr = NULL; 619289546Scem mw->dma_addr = 0; 620289546Scem } 621289546Scem 622289546Scem qp_bitmap = ntb_db_valid_mask(ntb); 623289546Scem nt->qp_count = flsll(qp_bitmap); 624289546Scem KASSERT(nt->qp_count != 0, ("bogus db bitmap")); 625289546Scem nt->qp_count -= 1; 626289546Scem 627289546Scem if (max_num_clients != 0 && max_num_clients < nt->qp_count) 628289546Scem nt->qp_count = max_num_clients; 629289546Scem else if (nt->mw_count < nt->qp_count) 630289546Scem nt->qp_count = nt->mw_count; 631289546Scem KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count")); 632289546Scem 633250079Scarl mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF); 634250079Scarl mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF); 635250079Scarl 636289545Scem nt->qp_vec = malloc(nt->qp_count * sizeof(*nt->qp_vec), M_NTB_IF, 637289545Scem M_WAITOK | M_ZERO); 638250079Scarl 639289545Scem for (i = 0; i < nt->qp_count; i++) { 640289544Scem set_bit(i, &nt->qp_bitmap); 641289546Scem set_bit(i, &nt->qp_bitmap_free); 642250079Scarl ntb_transport_init_queue(nt, i); 643289544Scem } 644250079Scarl 645250079Scarl callout_init(&nt->link_work, 0); 646291084Scem callout_init(&nt->link_watchdog, 0); 647290683Scem TASK_INIT(&nt->link_cleanup, 0, ntb_transport_link_cleanup_work, nt); 648250079Scarl 649289546Scem rc = ntb_set_ctx(ntb, nt, &ntb_transport_ops); 650250079Scarl if (rc != 0) 651250079Scarl goto err; 652250079Scarl 653289546Scem nt->link_is_up = false; 654289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 655289546Scem ntb_link_event(ntb); 656290682Scem 657290682Scem callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 658291084Scem if (enable_xeon_watchdog != 0) 659291084Scem callout_reset(&nt->link_watchdog, 0, xeon_link_watchdog_hb, nt); 660250079Scarl return (0); 661250079Scarl 662250079Scarlerr: 663289545Scem free(nt->qp_vec, M_NTB_IF); 664289546Scem nt->qp_vec = NULL; 665250079Scarl return (rc); 666250079Scarl} 667250079Scarl 668250079Scarlstatic void 669289546Scemntb_transport_free(struct ntb_transport_ctx *nt) 670250079Scarl{ 671250079Scarl struct ntb_softc *ntb = nt->ntb; 672289546Scem struct _qpset qp_bitmap_alloc; 673289544Scem uint8_t i; 674250079Scarl 675289273Scem ntb_transport_link_cleanup(nt); 676290683Scem taskqueue_drain(taskqueue_swi, &nt->link_cleanup); 677250079Scarl callout_drain(&nt->link_work); 678291084Scem callout_drain(&nt->link_watchdog); 679250079Scarl 680289546Scem BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc); 681289546Scem BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free); 682289546Scem 683289546Scem /* Verify that all the QPs are freed */ 684289545Scem for (i = 0; i < nt->qp_count; i++) 685289546Scem if (test_bit(i, &qp_bitmap_alloc)) 686289545Scem ntb_transport_free_queue(&nt->qp_vec[i]); 687250079Scarl 688289546Scem ntb_link_disable(ntb); 689289546Scem ntb_clear_ctx(ntb); 690250079Scarl 691289546Scem for (i = 0; i < nt->mw_count; i++) 692289153Scem ntb_free_mw(nt, i); 693250079Scarl 694289545Scem free(nt->qp_vec, M_NTB_IF); 695250079Scarl} 696250079Scarl 697250079Scarlstatic void 698289545Scemntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num) 699250079Scarl{ 700289546Scem struct ntb_transport_mw *mw; 701250079Scarl struct ntb_transport_qp *qp; 702289546Scem vm_paddr_t mw_base; 703289546Scem uint64_t mw_size, qp_offset; 704289546Scem size_t tx_size; 705289546Scem unsigned num_qps_mw, mw_num, mw_count; 706250079Scarl 707289546Scem mw_count = nt->mw_count; 708289545Scem mw_num = QP_TO_MW(nt, qp_num); 709289546Scem mw = &nt->mw_vec[mw_num]; 710289396Scem 711289545Scem qp = &nt->qp_vec[qp_num]; 712250079Scarl qp->qp_num = qp_num; 713250079Scarl qp->transport = nt; 714250079Scarl qp->ntb = nt->ntb; 715289545Scem qp->client_ready = false; 716250079Scarl qp->event_handler = NULL; 717289613Scem ntb_qp_link_down_reset(qp); 718250079Scarl 719289545Scem if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count) 720289545Scem num_qps_mw = nt->qp_count / mw_count + 1; 721250079Scarl else 722289545Scem num_qps_mw = nt->qp_count / mw_count; 723250079Scarl 724289546Scem mw_base = mw->phys_addr; 725289546Scem mw_size = mw->phys_size; 726289546Scem 727289546Scem tx_size = mw_size / num_qps_mw; 728290688Scem qp_offset = tx_size * (qp_num / mw_count); 729289546Scem 730290679Scem qp->tx_mw = mw->vbase + qp_offset; 731289546Scem KASSERT(qp->tx_mw != NULL, ("uh oh?")); 732289546Scem 733289546Scem /* XXX Assumes that a vm_paddr_t is equivalent to bus_addr_t */ 734289546Scem qp->tx_mw_phys = mw_base + qp_offset; 735289546Scem KASSERT(qp->tx_mw_phys != 0, ("uh oh?")); 736289546Scem 737250079Scarl tx_size -= sizeof(struct ntb_rx_info); 738290679Scem qp->rx_info = (void *)(qp->tx_mw + tx_size); 739250079Scarl 740289156Scem /* Due to house-keeping, there must be at least 2 buffs */ 741289650Scem qp->tx_max_frame = qmin(tx_size / 2, 742289650Scem transport_mtu + sizeof(struct ntb_payload_header)); 743250079Scarl qp->tx_max_entry = tx_size / qp->tx_max_frame; 744250079Scarl 745250079Scarl callout_init(&qp->link_work, 0); 746283291Sjkim callout_init(&qp->queue_full, 1); 747283291Sjkim callout_init(&qp->rx_full, 1); 748250079Scarl 749289651Scem mtx_init(&qp->ntb_rx_q_lock, "ntb rx q", NULL, MTX_SPIN); 750250079Scarl mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN); 751289546Scem TASK_INIT(&qp->rx_completion_task, 0, ntb_complete_rxc, qp); 752289546Scem TASK_INIT(&qp->rxc_db_work, 0, ntb_transport_rxc_db, qp); 753250079Scarl 754289651Scem STAILQ_INIT(&qp->rx_post_q); 755250079Scarl STAILQ_INIT(&qp->rx_pend_q); 756250079Scarl STAILQ_INIT(&qp->tx_free_q); 757290682Scem 758290682Scem callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 759250079Scarl} 760250079Scarl 761250079Scarlstatic void 762250079Scarlntb_transport_free_queue(struct ntb_transport_qp *qp) 763250079Scarl{ 764250079Scarl struct ntb_queue_entry *entry; 765250079Scarl 766250079Scarl if (qp == NULL) 767250079Scarl return; 768250079Scarl 769250079Scarl callout_drain(&qp->link_work); 770250079Scarl 771289546Scem ntb_db_set_mask(qp->ntb, 1ull << qp->qp_num); 772289546Scem taskqueue_drain(taskqueue_swi, &qp->rxc_db_work); 773289546Scem taskqueue_drain(taskqueue_swi, &qp->rx_completion_task); 774250079Scarl 775289546Scem qp->cb_data = NULL; 776289546Scem qp->rx_handler = NULL; 777289546Scem qp->tx_handler = NULL; 778289546Scem qp->event_handler = NULL; 779289546Scem 780289651Scem while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) 781250079Scarl free(entry, M_NTB_IF); 782250079Scarl 783289651Scem while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) 784289651Scem free(entry, M_NTB_IF); 785289651Scem 786250079Scarl while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 787250079Scarl free(entry, M_NTB_IF); 788250079Scarl 789289546Scem set_bit(qp->qp_num, &qp->transport->qp_bitmap_free); 790250079Scarl} 791250079Scarl 792250079Scarl/** 793250079Scarl * ntb_transport_create_queue - Create a new NTB transport layer queue 794250079Scarl * @rx_handler: receive callback function 795250079Scarl * @tx_handler: transmit callback function 796250079Scarl * @event_handler: event callback function 797250079Scarl * 798250079Scarl * Create a new NTB transport layer queue and provide the queue with a callback 799250079Scarl * routine for both transmit and receive. The receive callback routine will be 800250079Scarl * used to pass up data when the transport has received it on the queue. The 801250079Scarl * transmit callback routine will be called when the transport has completed the 802250079Scarl * transmission of the data on the queue and the data is ready to be freed. 803250079Scarl * 804250079Scarl * RETURNS: pointer to newly created ntb_queue, NULL on error. 805250079Scarl */ 806250079Scarlstatic struct ntb_transport_qp * 807289546Scemntb_transport_create_queue(void *data, struct ntb_softc *ntb, 808250079Scarl const struct ntb_queue_handlers *handlers) 809250079Scarl{ 810250079Scarl struct ntb_queue_entry *entry; 811250079Scarl struct ntb_transport_qp *qp; 812289545Scem struct ntb_transport_ctx *nt; 813250079Scarl unsigned int free_queue; 814289546Scem int i; 815250079Scarl 816289546Scem nt = ntb_get_ctx(ntb, NULL); 817289546Scem KASSERT(nt != NULL, ("bogus")); 818250079Scarl 819289544Scem free_queue = ffs_bit(&nt->qp_bitmap); 820250079Scarl if (free_queue == 0) 821289546Scem return (NULL); 822250079Scarl 823250079Scarl /* decrement free_queue to make it zero based */ 824250079Scarl free_queue--; 825250079Scarl 826289545Scem qp = &nt->qp_vec[free_queue]; 827290810Scem clear_bit(qp->qp_num, &nt->qp_bitmap_free); 828250079Scarl qp->cb_data = data; 829250079Scarl qp->rx_handler = handlers->rx_handler; 830250079Scarl qp->tx_handler = handlers->tx_handler; 831250079Scarl qp->event_handler = handlers->event_handler; 832250079Scarl 833250079Scarl for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 834289546Scem entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO); 835250079Scarl entry->cb_data = nt->ifp; 836250079Scarl entry->buf = NULL; 837250079Scarl entry->len = transport_mtu; 838289651Scem ntb_list_add(&qp->ntb_rx_q_lock, entry, &qp->rx_pend_q); 839250079Scarl } 840250079Scarl 841250079Scarl for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 842289546Scem entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO); 843250079Scarl ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 844250079Scarl } 845250079Scarl 846289546Scem ntb_db_clear(ntb, 1ull << qp->qp_num); 847289546Scem ntb_db_clear_mask(ntb, 1ull << qp->qp_num); 848250079Scarl return (qp); 849250079Scarl} 850250079Scarl 851250079Scarl/** 852250079Scarl * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 853250079Scarl * @qp: NTB transport layer queue to be enabled 854250079Scarl * 855250079Scarl * Notify NTB transport layer of client readiness to use queue 856250079Scarl */ 857250079Scarlstatic void 858250079Scarlntb_transport_link_up(struct ntb_transport_qp *qp) 859250079Scarl{ 860290684Scem struct ntb_transport_ctx *nt; 861250079Scarl 862250079Scarl if (qp == NULL) 863250079Scarl return; 864250079Scarl 865289545Scem qp->client_ready = true; 866250079Scarl 867290684Scem nt = qp->transport; 868290684Scem ntb_printf(2, "qp client ready\n"); 869290684Scem 870289545Scem if (qp->transport->link_is_up) 871250079Scarl callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 872250079Scarl} 873250079Scarl 874250079Scarl 875250079Scarl 876250079Scarl/* Transport Tx */ 877250079Scarl 878250079Scarl/** 879250079Scarl * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 880250079Scarl * @qp: NTB transport layer queue the entry is to be enqueued on 881250079Scarl * @cb: per buffer pointer for callback function to use 882250079Scarl * @data: pointer to data buffer that will be sent 883250079Scarl * @len: length of the data buffer 884250079Scarl * 885250079Scarl * Enqueue a new transmit buffer onto the transport queue from which a NTB 886289266Scem * payload will be transmitted. This assumes that a lock is being held to 887250079Scarl * serialize access to the qp. 888250079Scarl * 889250079Scarl * RETURNS: An appropriate ERRNO error value on error, or zero for success. 890250079Scarl */ 891250079Scarlstatic int 892250079Scarlntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 893250079Scarl unsigned int len) 894250079Scarl{ 895250079Scarl struct ntb_queue_entry *entry; 896250079Scarl int rc; 897250079Scarl 898289545Scem if (qp == NULL || !qp->link_is_up || len == 0) { 899250079Scarl CTR0(KTR_NTB, "TX: link not up"); 900250079Scarl return (EINVAL); 901250079Scarl } 902250079Scarl 903250079Scarl entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 904250079Scarl if (entry == NULL) { 905255281Scarl CTR0(KTR_NTB, "TX: could not get entry from tx_free_q"); 906289653Scem qp->tx_err_no_buf++; 907289653Scem return (EBUSY); 908250079Scarl } 909250079Scarl CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry); 910250079Scarl 911250079Scarl entry->cb_data = cb; 912250079Scarl entry->buf = data; 913250079Scarl entry->len = len; 914250079Scarl entry->flags = 0; 915250079Scarl 916250079Scarl rc = ntb_process_tx(qp, entry); 917250079Scarl if (rc != 0) { 918250079Scarl ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 919250079Scarl CTR1(KTR_NTB, 920250079Scarl "TX: process_tx failed. Returning entry %p to tx_free_q", 921250079Scarl entry); 922250079Scarl } 923250079Scarl return (rc); 924250079Scarl} 925250079Scarl 926250079Scarlstatic int 927250079Scarlntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry) 928250079Scarl{ 929250079Scarl void *offset; 930250079Scarl 931291028Scem offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; 932250079Scarl CTR3(KTR_NTB, 933291028Scem "TX: process_tx: tx_pkts=%lu, tx_index=%u, remote entry=%u", 934250079Scarl qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry); 935250079Scarl if (qp->tx_index == qp->remote_rx_info->entry) { 936250079Scarl CTR0(KTR_NTB, "TX: ring full"); 937250079Scarl qp->tx_ring_full++; 938250079Scarl return (EAGAIN); 939250079Scarl } 940250079Scarl 941250079Scarl if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 942250079Scarl if (qp->tx_handler != NULL) 943250079Scarl qp->tx_handler(qp, qp->cb_data, entry->buf, 944291034Scem EIO); 945291034Scem else 946291034Scem m_freem(entry->buf); 947250079Scarl 948291034Scem entry->buf = NULL; 949250079Scarl ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 950250079Scarl CTR1(KTR_NTB, 951250079Scarl "TX: frame too big. returning entry %p to tx_free_q", 952250079Scarl entry); 953250079Scarl return (0); 954250079Scarl } 955250079Scarl CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset); 956289546Scem ntb_memcpy_tx(qp, entry, offset); 957250079Scarl 958250079Scarl qp->tx_index++; 959250079Scarl qp->tx_index %= qp->tx_max_entry; 960250079Scarl 961250079Scarl qp->tx_pkts++; 962250079Scarl 963250079Scarl return (0); 964250079Scarl} 965250079Scarl 966250079Scarlstatic void 967289546Scemntb_memcpy_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 968250079Scarl void *offset) 969250079Scarl{ 970250079Scarl struct ntb_payload_header *hdr; 971250079Scarl 972289546Scem /* This piece is from Linux' ntb_async_tx() */ 973250079Scarl hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame - 974250079Scarl sizeof(struct ntb_payload_header)); 975289546Scem entry->x_hdr = hdr; 976291028Scem iowrite32(entry->len, &hdr->len); 977291028Scem iowrite32(qp->tx_pkts, &hdr->ver); 978289546Scem 979289546Scem /* This piece is ntb_memcpy_tx() */ 980289546Scem CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset); 981289546Scem if (entry->buf != NULL) { 982289546Scem m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset); 983289546Scem 984289546Scem /* 985289546Scem * Ensure that the data is fully copied before setting the 986289546Scem * flags 987289546Scem */ 988289546Scem wmb(); 989289546Scem } 990289546Scem 991289546Scem /* The rest is ntb_tx_copy_callback() */ 992291028Scem iowrite32(entry->flags | IF_NTB_DESC_DONE_FLAG, &hdr->flags); 993291028Scem CTR1(KTR_NTB, "TX: hdr %p set DESC_DONE", hdr); 994250079Scarl 995289545Scem ntb_peer_db_set(qp->ntb, 1ull << qp->qp_num); 996250079Scarl 997289341Scem /* 998250079Scarl * The entry length can only be zero if the packet is intended to be a 999250079Scarl * "link down" or similar. Since no payload is being sent in these 1000250079Scarl * cases, there is nothing to add to the completion queue. 1001250079Scarl */ 1002250079Scarl if (entry->len > 0) { 1003250079Scarl qp->tx_bytes += entry->len; 1004250079Scarl 1005250079Scarl if (qp->tx_handler) 1006291034Scem qp->tx_handler(qp, qp->cb_data, entry->buf, 1007291034Scem entry->len); 1008291034Scem else 1009291034Scem m_freem(entry->buf); 1010291034Scem entry->buf = NULL; 1011250079Scarl } 1012250079Scarl 1013291028Scem CTR3(KTR_NTB, 1014291028Scem "TX: entry %p sent. hdr->ver = %u, hdr->flags = 0x%x, Returning " 1015291028Scem "to tx_free_q", entry, hdr->ver, hdr->flags); 1016250079Scarl ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 1017250079Scarl} 1018250079Scarl 1019250079Scarlstatic void 1020250079Scarlntb_qp_full(void *arg) 1021250079Scarl{ 1022250079Scarl 1023250079Scarl CTR0(KTR_NTB, "TX: qp_full callout"); 1024250079Scarl ntb_start(arg); 1025250079Scarl} 1026250079Scarl 1027250079Scarl/* Transport Rx */ 1028250079Scarlstatic void 1029289546Scemntb_transport_rxc_db(void *arg, int pending __unused) 1030250079Scarl{ 1031289281Scem struct ntb_transport_qp *qp = arg; 1032289653Scem ntb_q_idx_t i; 1033289157Scem int rc; 1034250079Scarl 1035289341Scem /* 1036250079Scarl * Limit the number of packets processed in a single interrupt to 1037250079Scarl * provide fairness to others 1038250079Scarl */ 1039289546Scem CTR0(KTR_NTB, "RX: transport_rx"); 1040250079Scarl mtx_lock(&qp->transport->rx_lock); 1041289546Scem for (i = 0; i < qp->rx_max_entry; i++) { 1042250079Scarl rc = ntb_process_rxc(qp); 1043250079Scarl if (rc != 0) { 1044250079Scarl CTR0(KTR_NTB, "RX: process_rxc failed"); 1045250079Scarl break; 1046250079Scarl } 1047250079Scarl } 1048250079Scarl mtx_unlock(&qp->transport->rx_lock); 1049289281Scem 1050289546Scem if (i == qp->rx_max_entry) 1051289546Scem taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work); 1052289546Scem else if ((ntb_db_read(qp->ntb) & (1ull << qp->qp_num)) != 0) { 1053289546Scem /* If db is set, clear it and read it back to commit clear. */ 1054289546Scem ntb_db_clear(qp->ntb, 1ull << qp->qp_num); 1055289546Scem (void)ntb_db_read(qp->ntb); 1056289546Scem 1057289546Scem /* 1058289546Scem * An interrupt may have arrived between finishing 1059289546Scem * ntb_process_rxc and clearing the doorbell bit: there might 1060289546Scem * be some more work to do. 1061289546Scem */ 1062289546Scem taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work); 1063289546Scem } 1064250079Scarl} 1065250079Scarl 1066250079Scarlstatic int 1067250079Scarlntb_process_rxc(struct ntb_transport_qp *qp) 1068250079Scarl{ 1069250079Scarl struct ntb_payload_header *hdr; 1070250079Scarl struct ntb_queue_entry *entry; 1071291028Scem caddr_t offset; 1072250079Scarl 1073291028Scem offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1074291028Scem hdr = (void *)(offset + qp->rx_max_frame - 1075291028Scem sizeof(struct ntb_payload_header)); 1076250079Scarl 1077250079Scarl CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index); 1078250079Scarl if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) { 1079289546Scem CTR0(KTR_NTB, "RX: hdr not done"); 1080250079Scarl qp->rx_ring_empty++; 1081250079Scarl return (EAGAIN); 1082250079Scarl } 1083250079Scarl 1084289546Scem if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) { 1085289546Scem CTR0(KTR_NTB, "RX: link down"); 1086289546Scem ntb_qp_link_down(qp); 1087289546Scem hdr->flags = 0; 1088289546Scem return (EAGAIN); 1089289546Scem } 1090289546Scem 1091289546Scem if (hdr->ver != (uint32_t)qp->rx_pkts) { 1092289546Scem CTR2(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). " 1093291028Scem "Returning entry to rx_pend_q", hdr->ver, qp->rx_pkts); 1094250079Scarl qp->rx_err_ver++; 1095250079Scarl return (EIO); 1096250079Scarl } 1097250079Scarl 1098289651Scem entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); 1099289546Scem if (entry == NULL) { 1100289546Scem qp->rx_err_no_buf++; 1101289546Scem CTR0(KTR_NTB, "RX: No entries in rx_pend_q"); 1102289546Scem return (EAGAIN); 1103250079Scarl } 1104289546Scem callout_stop(&qp->rx_full); 1105289546Scem CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry); 1106250079Scarl 1107289546Scem entry->x_hdr = hdr; 1108289546Scem entry->index = qp->rx_index; 1109250079Scarl 1110289546Scem if (hdr->len > entry->len) { 1111289546Scem CTR2(KTR_NTB, "RX: len too long. Wanted %ju got %ju", 1112289546Scem (uintmax_t)hdr->len, (uintmax_t)entry->len); 1113250079Scarl qp->rx_err_oflow++; 1114250079Scarl 1115289546Scem entry->len = -EIO; 1116289546Scem entry->flags |= IF_NTB_DESC_DONE_FLAG; 1117250079Scarl 1118289546Scem taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task); 1119289546Scem } else { 1120289546Scem qp->rx_bytes += hdr->len; 1121289546Scem qp->rx_pkts++; 1122250079Scarl 1123289546Scem CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts); 1124250079Scarl 1125289546Scem entry->len = hdr->len; 1126289546Scem 1127289651Scem ntb_memcpy_rx(qp, entry, offset); 1128289546Scem } 1129289546Scem 1130250079Scarl qp->rx_index++; 1131250079Scarl qp->rx_index %= qp->rx_max_entry; 1132250079Scarl return (0); 1133250079Scarl} 1134250079Scarl 1135250079Scarlstatic void 1136289651Scemntb_memcpy_rx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 1137250079Scarl void *offset) 1138250079Scarl{ 1139250079Scarl struct ifnet *ifp = entry->cb_data; 1140250079Scarl unsigned int len = entry->len; 1141250079Scarl struct mbuf *m; 1142250079Scarl 1143250079Scarl CTR2(KTR_NTB, "RX: copying %d bytes from offset %p", len, offset); 1144250079Scarl m = m_devget(offset, len, 0, ifp, NULL); 1145250079Scarl m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID; 1146250079Scarl 1147250079Scarl entry->buf = (void *)m; 1148250079Scarl 1149289546Scem /* Ensure that the data is globally visible before clearing the flag */ 1150289546Scem wmb(); 1151289546Scem 1152289651Scem CTR2(KTR_NTB, "RX: copied entry %p to mbuf %p.", entry, m); 1153289651Scem ntb_rx_copy_callback(qp, entry); 1154289651Scem} 1155250079Scarl 1156289651Scemstatic inline void 1157289651Scemntb_rx_copy_callback(struct ntb_transport_qp *qp, void *data) 1158289651Scem{ 1159289651Scem struct ntb_queue_entry *entry; 1160289651Scem 1161289651Scem entry = data; 1162289651Scem entry->flags |= IF_NTB_DESC_DONE_FLAG; 1163250079Scarl taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task); 1164250079Scarl} 1165250079Scarl 1166250079Scarlstatic void 1167289546Scemntb_complete_rxc(void *arg, int pending) 1168250079Scarl{ 1169250079Scarl struct ntb_transport_qp *qp = arg; 1170289651Scem struct ntb_queue_entry *entry; 1171250079Scarl struct mbuf *m; 1172289651Scem unsigned len; 1173250079Scarl 1174250079Scarl CTR0(KTR_NTB, "RX: rx_completion_task"); 1175250079Scarl 1176289651Scem mtx_lock_spin(&qp->ntb_rx_q_lock); 1177289651Scem 1178289651Scem while (!STAILQ_EMPTY(&qp->rx_post_q)) { 1179289651Scem entry = STAILQ_FIRST(&qp->rx_post_q); 1180289651Scem if ((entry->flags & IF_NTB_DESC_DONE_FLAG) == 0) 1181289651Scem break; 1182289651Scem 1183289651Scem entry->x_hdr->flags = 0; 1184291028Scem iowrite32(entry->index, &qp->rx_info->entry); 1185289651Scem 1186291034Scem STAILQ_REMOVE_HEAD(&qp->rx_post_q, entry); 1187291034Scem 1188289651Scem len = entry->len; 1189250079Scarl m = entry->buf; 1190289651Scem 1191291034Scem /* 1192291034Scem * Re-initialize queue_entry for reuse; rx_handler takes 1193291034Scem * ownership of the mbuf. 1194291034Scem */ 1195291034Scem entry->buf = NULL; 1196291034Scem entry->len = transport_mtu; 1197291034Scem entry->cb_data = qp->transport->ifp; 1198289651Scem 1199291034Scem STAILQ_INSERT_TAIL(&qp->rx_pend_q, entry, entry); 1200291034Scem 1201289651Scem mtx_unlock_spin(&qp->ntb_rx_q_lock); 1202289651Scem 1203250079Scarl CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m); 1204289651Scem if (qp->rx_handler != NULL && qp->client_ready) 1205289651Scem qp->rx_handler(qp, qp->cb_data, m, len); 1206291034Scem else 1207291034Scem m_freem(m); 1208250079Scarl 1209289651Scem mtx_lock_spin(&qp->ntb_rx_q_lock); 1210289651Scem } 1211250079Scarl 1212289651Scem mtx_unlock_spin(&qp->ntb_rx_q_lock); 1213250079Scarl} 1214250079Scarl 1215289546Scemstatic void 1216289598Scemntb_transport_doorbell_callback(void *data, uint32_t vector) 1217289546Scem{ 1218289546Scem struct ntb_transport_ctx *nt = data; 1219289546Scem struct ntb_transport_qp *qp; 1220289546Scem struct _qpset db_bits; 1221289546Scem uint64_t vec_mask; 1222289546Scem unsigned qp_num; 1223289546Scem 1224289546Scem BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &db_bits); 1225289546Scem BIT_NAND(QP_SETSIZE, &db_bits, &nt->qp_bitmap_free); 1226289546Scem 1227289546Scem vec_mask = ntb_db_vector_mask(nt->ntb, vector); 1228289546Scem while (vec_mask != 0) { 1229289775Scem qp_num = ffsll(vec_mask) - 1; 1230289546Scem 1231289546Scem if (test_bit(qp_num, &db_bits)) { 1232289546Scem qp = &nt->qp_vec[qp_num]; 1233289546Scem taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work); 1234289546Scem } 1235289546Scem 1236289546Scem vec_mask &= ~(1ull << qp_num); 1237289546Scem } 1238289546Scem} 1239289546Scem 1240250079Scarl/* Link Event handler */ 1241250079Scarlstatic void 1242289546Scemntb_transport_event_callback(void *data) 1243250079Scarl{ 1244289545Scem struct ntb_transport_ctx *nt = data; 1245250079Scarl 1246289546Scem if (ntb_link_is_up(nt->ntb, NULL, NULL)) { 1247290684Scem ntb_printf(1, "HW link up\n"); 1248250079Scarl callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 1249289546Scem } else { 1250290684Scem ntb_printf(1, "HW link down\n"); 1251290683Scem taskqueue_enqueue(taskqueue_swi, &nt->link_cleanup); 1252250079Scarl } 1253250079Scarl} 1254250079Scarl 1255250079Scarl/* Link bring up */ 1256250079Scarlstatic void 1257250079Scarlntb_transport_link_work(void *arg) 1258250079Scarl{ 1259289545Scem struct ntb_transport_ctx *nt = arg; 1260250079Scarl struct ntb_softc *ntb = nt->ntb; 1261250079Scarl struct ntb_transport_qp *qp; 1262289546Scem uint64_t val64, size; 1263289546Scem uint32_t val; 1264289546Scem unsigned i; 1265289208Scem int rc; 1266250079Scarl 1267289153Scem /* send the local info, in the opposite order of the way we read it */ 1268289546Scem for (i = 0; i < nt->mw_count; i++) { 1269289546Scem size = nt->mw_vec[i].phys_size; 1270250079Scarl 1271289546Scem if (max_mw_size != 0 && size > max_mw_size) 1272289546Scem size = max_mw_size; 1273289546Scem 1274289546Scem ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), 1275289546Scem size >> 32); 1276289546Scem ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), size); 1277289153Scem } 1278250079Scarl 1279289546Scem ntb_peer_spad_write(ntb, IF_NTB_NUM_MWS, nt->mw_count); 1280250079Scarl 1281289546Scem ntb_peer_spad_write(ntb, IF_NTB_NUM_QPS, nt->qp_count); 1282250079Scarl 1283289546Scem ntb_peer_spad_write(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION); 1284250079Scarl 1285250079Scarl /* Query the remote side for its info */ 1286289546Scem val = 0; 1287289546Scem ntb_spad_read(ntb, IF_NTB_VERSION, &val); 1288250079Scarl if (val != NTB_TRANSPORT_VERSION) 1289250079Scarl goto out; 1290250079Scarl 1291289546Scem ntb_spad_read(ntb, IF_NTB_NUM_QPS, &val); 1292289545Scem if (val != nt->qp_count) 1293250079Scarl goto out; 1294250079Scarl 1295289546Scem ntb_spad_read(ntb, IF_NTB_NUM_MWS, &val); 1296289546Scem if (val != nt->mw_count) 1297250079Scarl goto out; 1298250079Scarl 1299289546Scem for (i = 0; i < nt->mw_count; i++) { 1300289546Scem ntb_spad_read(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), &val); 1301289153Scem val64 = (uint64_t)val << 32; 1302250079Scarl 1303289546Scem ntb_spad_read(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), &val); 1304289153Scem val64 |= val; 1305250079Scarl 1306289153Scem rc = ntb_set_mw(nt, i, val64); 1307289153Scem if (rc != 0) 1308289153Scem goto free_mws; 1309289153Scem } 1310289153Scem 1311289545Scem nt->link_is_up = true; 1312290684Scem ntb_printf(1, "transport link up\n"); 1313250079Scarl 1314289545Scem for (i = 0; i < nt->qp_count; i++) { 1315289545Scem qp = &nt->qp_vec[i]; 1316250079Scarl 1317250079Scarl ntb_transport_setup_qp_mw(nt, i); 1318250079Scarl 1319289545Scem if (qp->client_ready) 1320250079Scarl callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 1321250079Scarl } 1322250079Scarl 1323250079Scarl return; 1324250079Scarl 1325289153Scemfree_mws: 1326289546Scem for (i = 0; i < nt->mw_count; i++) 1327289153Scem ntb_free_mw(nt, i); 1328250079Scarlout: 1329289546Scem if (ntb_link_is_up(ntb, NULL, NULL)) 1330250079Scarl callout_reset(&nt->link_work, 1331289153Scem NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt); 1332250079Scarl} 1333250079Scarl 1334250079Scarlstatic int 1335289652Scemntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, size_t size) 1336250079Scarl{ 1337289545Scem struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 1338289652Scem size_t xlat_size, buff_size; 1339289546Scem int rc; 1340250079Scarl 1341289652Scem if (size == 0) 1342289652Scem return (EINVAL); 1343289652Scem 1344289546Scem xlat_size = roundup(size, mw->xlat_align_size); 1345289546Scem buff_size = roundup(size, mw->xlat_align); 1346289546Scem 1347289154Scem /* No need to re-setup */ 1348289546Scem if (mw->xlat_size == xlat_size) 1349289154Scem return (0); 1350289154Scem 1351289546Scem if (mw->buff_size != 0) 1352289154Scem ntb_free_mw(nt, num_mw); 1353289154Scem 1354289546Scem /* Alloc memory for receiving data. Must be aligned */ 1355289546Scem mw->xlat_size = xlat_size; 1356289546Scem mw->buff_size = buff_size; 1357250079Scarl 1358289546Scem mw->virt_addr = contigmalloc(mw->buff_size, M_NTB_IF, M_ZERO, 0, 1359291033Scem mw->addr_limit, mw->xlat_align, 0); 1360250079Scarl if (mw->virt_addr == NULL) { 1361291705Scem ntb_printf(0, "Unable to allocate MW buffer of size %zu/%zu\n", 1362291705Scem mw->buff_size, mw->xlat_size); 1363289545Scem mw->xlat_size = 0; 1364289546Scem mw->buff_size = 0; 1365250079Scarl return (ENOMEM); 1366250079Scarl } 1367250079Scarl /* TODO: replace with bus_space_* functions */ 1368250079Scarl mw->dma_addr = vtophys(mw->virt_addr); 1369250079Scarl 1370289346Scem /* 1371289346Scem * Ensure that the allocation from contigmalloc is aligned as 1372289346Scem * requested. XXX: This may not be needed -- brought in for parity 1373289346Scem * with the Linux driver. 1374289346Scem */ 1375289546Scem if (mw->dma_addr % mw->xlat_align != 0) { 1376290684Scem ntb_printf(0, 1377289652Scem "DMA memory 0x%jx not aligned to BAR size 0x%zx\n", 1378289346Scem (uintmax_t)mw->dma_addr, size); 1379289346Scem ntb_free_mw(nt, num_mw); 1380289346Scem return (ENOMEM); 1381289346Scem } 1382289346Scem 1383250079Scarl /* Notify HW the memory location of the receive buffer */ 1384289546Scem rc = ntb_mw_set_trans(nt->ntb, num_mw, mw->dma_addr, mw->xlat_size); 1385289546Scem if (rc) { 1386290684Scem ntb_printf(0, "Unable to set mw%d translation\n", num_mw); 1387289546Scem ntb_free_mw(nt, num_mw); 1388289546Scem return (rc); 1389289546Scem } 1390250079Scarl 1391250079Scarl return (0); 1392250079Scarl} 1393250079Scarl 1394250079Scarlstatic void 1395289545Scemntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) 1396289153Scem{ 1397289545Scem struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 1398289153Scem 1399289153Scem if (mw->virt_addr == NULL) 1400289153Scem return; 1401289153Scem 1402289546Scem ntb_mw_clear_trans(nt->ntb, num_mw); 1403289545Scem contigfree(mw->virt_addr, mw->xlat_size, M_NTB_IF); 1404289546Scem mw->xlat_size = 0; 1405289546Scem mw->buff_size = 0; 1406289153Scem mw->virt_addr = NULL; 1407289153Scem} 1408289153Scem 1409289546Scemstatic int 1410289545Scemntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, unsigned int qp_num) 1411250079Scarl{ 1412289545Scem struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 1413289546Scem struct ntb_transport_mw *mw; 1414250079Scarl void *offset; 1415289653Scem ntb_q_idx_t i; 1416289546Scem size_t rx_size; 1417289546Scem unsigned num_qps_mw, mw_num, mw_count; 1418250079Scarl 1419289546Scem mw_count = nt->mw_count; 1420289545Scem mw_num = QP_TO_MW(nt, qp_num); 1421289546Scem mw = &nt->mw_vec[mw_num]; 1422289396Scem 1423289546Scem if (mw->virt_addr == NULL) 1424289546Scem return (ENOMEM); 1425289546Scem 1426289545Scem if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count) 1427289545Scem num_qps_mw = nt->qp_count / mw_count + 1; 1428250079Scarl else 1429289545Scem num_qps_mw = nt->qp_count / mw_count; 1430250079Scarl 1431289546Scem rx_size = mw->xlat_size / num_qps_mw; 1432290688Scem qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); 1433250079Scarl rx_size -= sizeof(struct ntb_rx_info); 1434250079Scarl 1435290679Scem qp->remote_rx_info = (void*)(qp->rx_buff + rx_size); 1436289546Scem 1437289156Scem /* Due to house-keeping, there must be at least 2 buffs */ 1438289650Scem qp->rx_max_frame = qmin(rx_size / 2, 1439289650Scem transport_mtu + sizeof(struct ntb_payload_header)); 1440250079Scarl qp->rx_max_entry = rx_size / qp->rx_max_frame; 1441250079Scarl qp->rx_index = 0; 1442250079Scarl 1443289156Scem qp->remote_rx_info->entry = qp->rx_max_entry - 1; 1444250079Scarl 1445289546Scem /* Set up the hdr offsets with 0s */ 1446250079Scarl for (i = 0; i < qp->rx_max_entry; i++) { 1447290679Scem offset = (void *)(qp->rx_buff + qp->rx_max_frame * (i + 1) - 1448250079Scarl sizeof(struct ntb_payload_header)); 1449250079Scarl memset(offset, 0, sizeof(struct ntb_payload_header)); 1450250079Scarl } 1451250079Scarl 1452250079Scarl qp->rx_pkts = 0; 1453250079Scarl qp->tx_pkts = 0; 1454289155Scem qp->tx_index = 0; 1455289546Scem 1456289546Scem return (0); 1457250079Scarl} 1458250079Scarl 1459250079Scarlstatic void 1460250079Scarlntb_qp_link_work(void *arg) 1461250079Scarl{ 1462250079Scarl struct ntb_transport_qp *qp = arg; 1463250079Scarl struct ntb_softc *ntb = qp->ntb; 1464289545Scem struct ntb_transport_ctx *nt = qp->transport; 1465289615Scem uint32_t val, dummy; 1466250079Scarl 1467289546Scem ntb_spad_read(ntb, IF_NTB_QP_LINKS, &val); 1468250079Scarl 1469289546Scem ntb_peer_spad_write(ntb, IF_NTB_QP_LINKS, val | (1ull << qp->qp_num)); 1470250079Scarl 1471250079Scarl /* query remote spad for qp ready bits */ 1472289615Scem ntb_peer_spad_read(ntb, IF_NTB_QP_LINKS, &dummy); 1473250079Scarl 1474250079Scarl /* See if the remote side is up */ 1475289546Scem if ((val & (1ull << qp->qp_num)) != 0) { 1476290684Scem ntb_printf(2, "qp link up\n"); 1477289545Scem qp->link_is_up = true; 1478289546Scem 1479250079Scarl if (qp->event_handler != NULL) 1480250079Scarl qp->event_handler(qp->cb_data, NTB_LINK_UP); 1481289546Scem 1482289546Scem taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work); 1483289546Scem } else if (nt->link_is_up) 1484250079Scarl callout_reset(&qp->link_work, 1485250079Scarl NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1486250079Scarl} 1487250079Scarl 1488250079Scarl/* Link down event*/ 1489250079Scarlstatic void 1490289545Scemntb_transport_link_cleanup(struct ntb_transport_ctx *nt) 1491250079Scarl{ 1492289546Scem struct ntb_transport_qp *qp; 1493289546Scem struct _qpset qp_bitmap_alloc; 1494289546Scem unsigned i; 1495250079Scarl 1496289546Scem BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc); 1497289546Scem BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free); 1498289546Scem 1499289273Scem /* Pass along the info to any clients */ 1500289545Scem for (i = 0; i < nt->qp_count; i++) 1501289546Scem if (test_bit(i, &qp_bitmap_alloc)) { 1502289546Scem qp = &nt->qp_vec[i]; 1503289546Scem ntb_qp_link_cleanup(qp); 1504289546Scem callout_drain(&qp->link_work); 1505289546Scem } 1506289273Scem 1507289545Scem if (!nt->link_is_up) 1508250079Scarl callout_drain(&nt->link_work); 1509250079Scarl 1510289341Scem /* 1511250079Scarl * The scratchpad registers keep the values if the remote side 1512250079Scarl * goes down, blast them now to give them a sane value the next 1513250079Scarl * time they are accessed 1514250079Scarl */ 1515250079Scarl for (i = 0; i < IF_NTB_MAX_SPAD; i++) 1516289545Scem ntb_spad_write(nt->ntb, i, 0); 1517250079Scarl} 1518250079Scarl 1519290683Scemstatic void 1520290683Scemntb_transport_link_cleanup_work(void *arg, int pending __unused) 1521290683Scem{ 1522250079Scarl 1523290683Scem ntb_transport_link_cleanup(arg); 1524290683Scem} 1525290683Scem 1526250079Scarlstatic void 1527250079Scarlntb_qp_link_down(struct ntb_transport_qp *qp) 1528250079Scarl{ 1529250079Scarl 1530250079Scarl ntb_qp_link_cleanup(qp); 1531250079Scarl} 1532250079Scarl 1533250079Scarlstatic void 1534289613Scemntb_qp_link_down_reset(struct ntb_transport_qp *qp) 1535289613Scem{ 1536289613Scem 1537289613Scem qp->link_is_up = false; 1538289613Scem 1539289613Scem qp->tx_index = qp->rx_index = 0; 1540289613Scem qp->tx_bytes = qp->rx_bytes = 0; 1541289613Scem qp->tx_pkts = qp->rx_pkts = 0; 1542289613Scem 1543289613Scem qp->rx_ring_empty = 0; 1544289613Scem qp->tx_ring_full = 0; 1545289613Scem 1546289653Scem qp->rx_err_no_buf = qp->tx_err_no_buf = 0; 1547289653Scem qp->rx_err_oflow = qp->rx_err_ver = 0; 1548289613Scem} 1549289613Scem 1550289613Scemstatic void 1551250079Scarlntb_qp_link_cleanup(struct ntb_transport_qp *qp) 1552250079Scarl{ 1553289545Scem struct ntb_transport_ctx *nt = qp->transport; 1554250079Scarl 1555289613Scem callout_drain(&qp->link_work); 1556289613Scem ntb_qp_link_down_reset(qp); 1557250079Scarl 1558250079Scarl if (qp->event_handler != NULL) 1559250079Scarl qp->event_handler(qp->cb_data, NTB_LINK_DOWN); 1560250079Scarl 1561289545Scem if (nt->link_is_up) 1562250079Scarl callout_reset(&qp->link_work, 1563250079Scarl NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1564250079Scarl} 1565250079Scarl 1566250079Scarl/* Link commanded down */ 1567250079Scarl/** 1568250079Scarl * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 1569250079Scarl * @qp: NTB transport layer queue to be disabled 1570250079Scarl * 1571250079Scarl * Notify NTB transport layer of client's desire to no longer receive data on 1572250079Scarl * transport queue specified. It is the client's responsibility to ensure all 1573289266Scem * entries on queue are purged or otherwise handled appropriately. 1574250079Scarl */ 1575250079Scarlstatic void 1576250079Scarlntb_transport_link_down(struct ntb_transport_qp *qp) 1577250079Scarl{ 1578289546Scem uint32_t val; 1579250079Scarl 1580250079Scarl if (qp == NULL) 1581250079Scarl return; 1582250079Scarl 1583289545Scem qp->client_ready = false; 1584250079Scarl 1585289546Scem ntb_spad_read(qp->ntb, IF_NTB_QP_LINKS, &val); 1586250079Scarl 1587289546Scem ntb_peer_spad_write(qp->ntb, IF_NTB_QP_LINKS, 1588250079Scarl val & ~(1 << qp->qp_num)); 1589250079Scarl 1590289545Scem if (qp->link_is_up) 1591250079Scarl ntb_send_link_down(qp); 1592250079Scarl else 1593250079Scarl callout_drain(&qp->link_work); 1594250079Scarl} 1595250079Scarl 1596250079Scarlstatic void 1597250079Scarlntb_send_link_down(struct ntb_transport_qp *qp) 1598250079Scarl{ 1599250079Scarl struct ntb_queue_entry *entry; 1600250079Scarl int i, rc; 1601250079Scarl 1602289545Scem if (!qp->link_is_up) 1603250079Scarl return; 1604250079Scarl 1605250079Scarl for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1606250079Scarl entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1607250079Scarl if (entry != NULL) 1608250079Scarl break; 1609250079Scarl pause("NTB Wait for link down", hz / 10); 1610250079Scarl } 1611250079Scarl 1612250079Scarl if (entry == NULL) 1613250079Scarl return; 1614250079Scarl 1615250079Scarl entry->cb_data = NULL; 1616250079Scarl entry->buf = NULL; 1617250079Scarl entry->len = 0; 1618250079Scarl entry->flags = IF_NTB_LINK_DOWN_FLAG; 1619250079Scarl 1620250079Scarl mtx_lock(&qp->transport->tx_lock); 1621250079Scarl rc = ntb_process_tx(qp, entry); 1622250079Scarl if (rc != 0) 1623250079Scarl printf("ntb: Failed to send link down\n"); 1624250079Scarl mtx_unlock(&qp->transport->tx_lock); 1625289613Scem 1626289613Scem ntb_qp_link_down_reset(qp); 1627250079Scarl} 1628250079Scarl 1629250079Scarl 1630250079Scarl/* List Management */ 1631250079Scarl 1632250079Scarlstatic void 1633250079Scarlntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 1634250079Scarl struct ntb_queue_list *list) 1635250079Scarl{ 1636250079Scarl 1637250079Scarl mtx_lock_spin(lock); 1638250079Scarl STAILQ_INSERT_TAIL(list, entry, entry); 1639250079Scarl mtx_unlock_spin(lock); 1640250079Scarl} 1641250079Scarl 1642250079Scarlstatic struct ntb_queue_entry * 1643250079Scarlntb_list_rm(struct mtx *lock, struct ntb_queue_list *list) 1644250079Scarl{ 1645250079Scarl struct ntb_queue_entry *entry; 1646250079Scarl 1647250079Scarl mtx_lock_spin(lock); 1648250079Scarl if (STAILQ_EMPTY(list)) { 1649250079Scarl entry = NULL; 1650250079Scarl goto out; 1651250079Scarl } 1652250079Scarl entry = STAILQ_FIRST(list); 1653250079Scarl STAILQ_REMOVE_HEAD(list, entry); 1654250079Scarlout: 1655250079Scarl mtx_unlock_spin(lock); 1656250079Scarl 1657250079Scarl return (entry); 1658250079Scarl} 1659250079Scarl 1660289651Scemstatic struct ntb_queue_entry * 1661289651Scemntb_list_mv(struct mtx *lock, struct ntb_queue_list *from, 1662289651Scem struct ntb_queue_list *to) 1663289651Scem{ 1664289651Scem struct ntb_queue_entry *entry; 1665289651Scem 1666289651Scem mtx_lock_spin(lock); 1667289651Scem if (STAILQ_EMPTY(from)) { 1668289651Scem entry = NULL; 1669289651Scem goto out; 1670289651Scem } 1671289651Scem entry = STAILQ_FIRST(from); 1672289651Scem STAILQ_REMOVE_HEAD(from, entry); 1673289651Scem STAILQ_INSERT_TAIL(to, entry, entry); 1674289651Scem 1675289651Scemout: 1676289651Scem mtx_unlock_spin(lock); 1677289651Scem return (entry); 1678289651Scem} 1679289651Scem 1680250079Scarl/* Helper functions */ 1681250079Scarl/* TODO: This too should really be part of the kernel */ 1682250079Scarl#define EUI48_MULTICAST 1 << 0 1683250079Scarl#define EUI48_LOCALLY_ADMINISTERED 1 << 1 1684250079Scarlstatic void 1685250079Scarlcreate_random_local_eui48(u_char *eaddr) 1686250079Scarl{ 1687250079Scarl static uint8_t counter = 0; 1688250079Scarl uint32_t seed = ticks; 1689250079Scarl 1690250079Scarl eaddr[0] = EUI48_LOCALLY_ADMINISTERED; 1691250079Scarl memcpy(&eaddr[1], &seed, sizeof(uint32_t)); 1692250079Scarl eaddr[5] = counter++; 1693250079Scarl} 1694250079Scarl 1695250079Scarl/** 1696250079Scarl * ntb_transport_max_size - Query the max payload size of a qp 1697250079Scarl * @qp: NTB transport layer queue to be queried 1698250079Scarl * 1699250079Scarl * Query the maximum payload size permissible on the given qp 1700250079Scarl * 1701250079Scarl * RETURNS: the max payload size of a qp 1702250079Scarl */ 1703250079Scarlstatic unsigned int 1704250079Scarlntb_transport_max_size(struct ntb_transport_qp *qp) 1705250079Scarl{ 1706250079Scarl 1707250079Scarl if (qp == NULL) 1708250079Scarl return (0); 1709250079Scarl 1710250079Scarl return (qp->tx_max_frame - sizeof(struct ntb_payload_header)); 1711250079Scarl} 1712