ntb_transport.c revision 283291
1333537Sdes/*- 2333537Sdes * Copyright (C) 2013 Intel Corporation 3333537Sdes * All rights reserved. 4333537Sdes * 5333537Sdes * Redistribution and use in source and binary forms, with or without 6333537Sdes * modification, are permitted provided that the following conditions 7333537Sdes * are met: 8333537Sdes * 1. Redistributions of source code must retain the above copyright 9333537Sdes * notice, this list of conditions and the following disclaimer. 10333537Sdes * 2. Redistributions in binary form must reproduce the above copyright 11333537Sdes * notice, this list of conditions and the following disclaimer in the 12333537Sdes * documentation and/or other materials provided with the distribution. 13333537Sdes * 14333537Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15333537Sdes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16333537Sdes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17333537Sdes * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18333537Sdes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19333537Sdes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20333537Sdes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21333537Sdes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22333537Sdes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23333537Sdes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24333537Sdes * SUCH DAMAGE. 25333537Sdes */ 26333537Sdes 27333537Sdes#include <sys/cdefs.h> 28333537Sdes__FBSDID("$FreeBSD: head/sys/dev/ntb/if_ntb/if_ntb.c 283291 2015-05-22 17:05:21Z jkim $"); 29333537Sdes 30333537Sdes#include <sys/param.h> 31333537Sdes#include <sys/kernel.h> 32333537Sdes#include <sys/systm.h> 33333537Sdes#include <sys/bus.h> 34333537Sdes#include <sys/ktr.h> 35333537Sdes#include <sys/lock.h> 36333537Sdes#include <sys/malloc.h> 37333537Sdes#include <sys/module.h> 38333537Sdes#include <sys/mutex.h> 39333537Sdes#include <sys/queue.h> 40333537Sdes#include <sys/socket.h> 41333537Sdes#include <sys/sockio.h> 42333537Sdes#include <sys/taskqueue.h> 43333537Sdes#include <net/if.h> 44333537Sdes#include <net/if_media.h> 45333537Sdes#include <net/if_types.h> 46333537Sdes#include <net/if_var.h> 47333537Sdes#include <net/bpf.h> 48333537Sdes#include <net/ethernet.h> 49333537Sdes#include <vm/vm.h> 50333537Sdes#include <vm/pmap.h> 51333537Sdes#include <machine/bus.h> 52333537Sdes#include <machine/cpufunc.h> 53333537Sdes#include <machine/pmap.h> 54333537Sdes 55333537Sdes#include "../ntb_hw/ntb_hw.h" 56333537Sdes 57333537Sdes/* 58333537Sdes * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 59333537Sdes * allows you to connect two systems using a PCI-e link. 60333537Sdes * 61333537Sdes * This module contains a protocol for sending and receiving messages, and 62333537Sdes * exposes that protocol through a simulated ethernet device called ntb. 63333537Sdes * 64333537Sdes * NOTE: Much of the code in this module is shared with Linux. Any patches may 65333537Sdes * be picked up and redistributed in Linux with a dual GPL/BSD license. 66333537Sdes */ 67333537Sdes 68333537Sdes/* TODO: These functions should really be part of the kernel */ 69333537Sdes#define test_bit(pos, bitmap_addr) (*(bitmap_addr) & 1UL << (pos)) 70333537Sdes#define set_bit(pos, bitmap_addr) *(bitmap_addr) |= 1UL << (pos) 71333537Sdes#define clear_bit(pos, bitmap_addr) *(bitmap_addr) &= ~(1UL << (pos)) 72333537Sdes 73333537Sdes#define KTR_NTB KTR_SPARE3 74333537Sdes 75333537Sdes#define NTB_TRANSPORT_VERSION 3 76333537Sdes#define NTB_RX_MAX_PKTS 64 77333537Sdes#define NTB_RXQ_SIZE 300 78333537Sdes 79333537Sdesstatic unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN; 80333537Sdesstatic unsigned int max_num_clients = 1; 81333537Sdes 82333537SdesSTAILQ_HEAD(ntb_queue_list, ntb_queue_entry); 83333537Sdes 84333537Sdesstruct ntb_queue_entry { 85333537Sdes /* ntb_queue list reference */ 86333537Sdes STAILQ_ENTRY(ntb_queue_entry) entry; 87333537Sdes 88333537Sdes /* info on data to be transfered */ 89333537Sdes void *cb_data; 90333537Sdes void *buf; 91333537Sdes uint64_t len; 92333537Sdes uint64_t flags; 93333537Sdes}; 94333537Sdes 95333537Sdesstruct ntb_rx_info { 96333537Sdes unsigned int entry; 97333537Sdes}; 98333537Sdes 99333537Sdesstruct ntb_transport_qp { 100333537Sdes struct ntb_netdev *transport; 101333537Sdes struct ntb_softc *ntb; 102333537Sdes 103333537Sdes void *cb_data; 104333537Sdes 105333537Sdes bool client_ready; 106333537Sdes bool qp_link; 107333537Sdes uint8_t qp_num; /* Only 64 QPs are allowed. 0-63 */ 108333537Sdes 109333537Sdes struct ntb_rx_info *rx_info; 110333537Sdes struct ntb_rx_info *remote_rx_info; 111333537Sdes 112333537Sdes void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, 113333537Sdes void *data, int len); 114333537Sdes struct ntb_queue_list tx_free_q; 115333537Sdes struct mtx ntb_tx_free_q_lock; 116333537Sdes void *tx_mw; 117333537Sdes uint64_t tx_index; 118333537Sdes uint64_t tx_max_entry; 119333537Sdes uint64_t tx_max_frame; 120333537Sdes 121333537Sdes void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, 122333537Sdes void *data, int len); 123333537Sdes struct ntb_queue_list rx_pend_q; 124333537Sdes struct ntb_queue_list rx_free_q; 125333537Sdes struct mtx ntb_rx_pend_q_lock; 126333537Sdes struct mtx ntb_rx_free_q_lock; 127333537Sdes struct task rx_completion_task; 128333537Sdes void *rx_buff; 129333537Sdes uint64_t rx_index; 130333537Sdes uint64_t rx_max_entry; 131333537Sdes uint64_t rx_max_frame; 132333537Sdes 133333537Sdes void (*event_handler) (void *data, int status); 134333537Sdes struct callout link_work; 135333537Sdes struct callout queue_full; 136333537Sdes struct callout rx_full; 137333537Sdes 138333537Sdes uint64_t last_rx_no_buf; 139333537Sdes 140333537Sdes /* Stats */ 141333537Sdes uint64_t rx_bytes; 142333537Sdes uint64_t rx_pkts; 143333537Sdes uint64_t rx_ring_empty; 144333537Sdes uint64_t rx_err_no_buf; 145333537Sdes uint64_t rx_err_oflow; 146333537Sdes uint64_t rx_err_ver; 147333537Sdes uint64_t tx_bytes; 148333537Sdes uint64_t tx_pkts; 149333537Sdes uint64_t tx_ring_full; 150333537Sdes}; 151333537Sdes 152333537Sdesstruct ntb_queue_handlers { 153333537Sdes void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, 154333537Sdes void *data, int len); 155333537Sdes void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, 156333537Sdes void *data, int len); 157333537Sdes void (*event_handler) (void *data, int status); 158333537Sdes}; 159 160 161struct ntb_transport_mw { 162 size_t size; 163 void *virt_addr; 164 vm_paddr_t dma_addr; 165}; 166 167struct ntb_netdev { 168 struct ntb_softc *ntb; 169 struct ifnet *ifp; 170 struct ntb_transport_mw mw[NTB_NUM_MW]; 171 struct ntb_transport_qp *qps; 172 uint64_t max_qps; 173 uint64_t qp_bitmap; 174 bool transport_link; 175 struct callout link_work; 176 struct ntb_transport_qp *qp; 177 uint64_t bufsize; 178 u_char eaddr[ETHER_ADDR_LEN]; 179 struct mtx tx_lock; 180 struct mtx rx_lock; 181}; 182 183static struct ntb_netdev net_softc; 184 185enum { 186 IF_NTB_DESC_DONE_FLAG = 1 << 0, 187 IF_NTB_LINK_DOWN_FLAG = 1 << 1, 188}; 189 190struct ntb_payload_header { 191 uint64_t ver; 192 uint64_t len; 193 uint64_t flags; 194}; 195 196enum { 197 IF_NTB_VERSION = 0, 198 IF_NTB_MW0_SZ, 199 IF_NTB_MW1_SZ, 200 IF_NTB_NUM_QPS, 201 IF_NTB_QP_LINKS, 202 IF_NTB_MAX_SPAD, 203}; 204 205#define QP_TO_MW(qp) ((qp) % NTB_NUM_MW) 206#define NTB_QP_DEF_NUM_ENTRIES 100 207#define NTB_LINK_DOWN_TIMEOUT 10 208 209static int ntb_handle_module_events(struct module *m, int what, void *arg); 210static int ntb_setup_interface(void); 211static int ntb_teardown_interface(void); 212static void ntb_net_init(void *arg); 213static int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 214static void ntb_start(struct ifnet *ifp); 215static void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, 216 void *data, int len); 217static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, 218 void *data, int len); 219static void ntb_net_event_handler(void *data, int status); 220static int ntb_transport_init(struct ntb_softc *ntb); 221static void ntb_transport_free(void *transport); 222static void ntb_transport_init_queue(struct ntb_netdev *nt, 223 unsigned int qp_num); 224static void ntb_transport_free_queue(struct ntb_transport_qp *qp); 225static struct ntb_transport_qp * ntb_transport_create_queue(void *data, 226 struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers); 227static void ntb_transport_link_up(struct ntb_transport_qp *qp); 228static int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, 229 void *data, unsigned int len); 230static int ntb_process_tx(struct ntb_transport_qp *qp, 231 struct ntb_queue_entry *entry); 232static void ntb_tx_copy_task(struct ntb_transport_qp *qp, 233 struct ntb_queue_entry *entry, void *offset); 234static void ntb_qp_full(void *arg); 235static void ntb_transport_rxc_db(void *data, int db_num); 236static void ntb_rx_pendq_full(void *arg); 237static void ntb_transport_rx(struct ntb_transport_qp *qp); 238static int ntb_process_rxc(struct ntb_transport_qp *qp); 239static void ntb_rx_copy_task(struct ntb_transport_qp *qp, 240 struct ntb_queue_entry *entry, void *offset); 241static void ntb_rx_completion_task(void *arg, int pending); 242static void ntb_transport_event_callback(void *data, enum ntb_hw_event event); 243static void ntb_transport_link_work(void *arg); 244static int ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size); 245static void ntb_transport_setup_qp_mw(struct ntb_netdev *nt, 246 unsigned int qp_num); 247static void ntb_qp_link_work(void *arg); 248static void ntb_transport_link_cleanup(struct ntb_netdev *nt); 249static void ntb_qp_link_down(struct ntb_transport_qp *qp); 250static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp); 251static void ntb_transport_link_down(struct ntb_transport_qp *qp); 252static void ntb_send_link_down(struct ntb_transport_qp *qp); 253static void ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 254 struct ntb_queue_list *list); 255static struct ntb_queue_entry *ntb_list_rm(struct mtx *lock, 256 struct ntb_queue_list *list); 257static void create_random_local_eui48(u_char *eaddr); 258static unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); 259 260MALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver"); 261 262/* Module setup and teardown */ 263static int 264ntb_handle_module_events(struct module *m, int what, void *arg) 265{ 266 int err = 0; 267 268 switch (what) { 269 case MOD_LOAD: 270 err = ntb_setup_interface(); 271 break; 272 case MOD_UNLOAD: 273 err = ntb_teardown_interface(); 274 break; 275 default: 276 err = EOPNOTSUPP; 277 break; 278 } 279 return (err); 280} 281 282static moduledata_t if_ntb_mod = { 283 "if_ntb", 284 ntb_handle_module_events, 285 NULL 286}; 287 288DECLARE_MODULE(if_ntb, if_ntb_mod, SI_SUB_KLD, SI_ORDER_ANY); 289MODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1); 290 291static int 292ntb_setup_interface() 293{ 294 struct ifnet *ifp; 295 struct ntb_queue_handlers handlers = { ntb_net_rx_handler, 296 ntb_net_tx_handler, ntb_net_event_handler }; 297 298 net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0); 299 if (net_softc.ntb == NULL) { 300 printf("ntb: Cannot find devclass\n"); 301 return (ENXIO); 302 } 303 304 ntb_transport_init(net_softc.ntb); 305 306 ifp = net_softc.ifp = if_alloc(IFT_ETHER); 307 if (ifp == NULL) { 308 printf("ntb: cannot allocate ifnet structure\n"); 309 return (ENOMEM); 310 } 311 312 net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb, 313 &handlers); 314 if_initname(ifp, "ntb", 0); 315 ifp->if_init = ntb_net_init; 316 ifp->if_softc = &net_softc; 317 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX; 318 ifp->if_ioctl = ntb_ioctl; 319 ifp->if_start = ntb_start; 320 IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN); 321 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 322 IFQ_SET_READY(&ifp->if_snd); 323 create_random_local_eui48(net_softc.eaddr); 324 ether_ifattach(ifp, net_softc.eaddr); 325 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU; 326 ifp->if_capenable = ifp->if_capabilities; 327 328 ntb_transport_link_up(net_softc.qp); 329 net_softc.bufsize = ntb_transport_max_size(net_softc.qp) + 330 sizeof(struct ether_header); 331 return (0); 332} 333 334static int 335ntb_teardown_interface() 336{ 337 338 if (net_softc.qp != NULL) 339 ntb_transport_link_down(net_softc.qp); 340 341 if (net_softc.ifp != NULL) { 342 ether_ifdetach(net_softc.ifp); 343 if_free(net_softc.ifp); 344 } 345 346 if (net_softc.qp != NULL) { 347 ntb_transport_free_queue(net_softc.qp); 348 ntb_transport_free(&net_softc); 349 } 350 351 return (0); 352} 353 354/* Network device interface */ 355 356static void 357ntb_net_init(void *arg) 358{ 359 struct ntb_netdev *ntb_softc = arg; 360 struct ifnet *ifp = ntb_softc->ifp; 361 362 ifp->if_drv_flags |= IFF_DRV_RUNNING; 363 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 364 ifp->if_flags |= IFF_UP; 365 if_link_state_change(ifp, LINK_STATE_UP); 366} 367 368static int 369ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 370{ 371 struct ntb_netdev *nt = ifp->if_softc; 372 struct ifreq *ifr = (struct ifreq *)data; 373 int error = 0; 374 375 switch (command) { 376 case SIOCSIFMTU: 377 { 378 if (ifr->ifr_mtu > ntb_transport_max_size(nt->qp) - 379 ETHER_HDR_LEN - ETHER_CRC_LEN) { 380 error = EINVAL; 381 break; 382 } 383 384 ifp->if_mtu = ifr->ifr_mtu; 385 break; 386 } 387 default: 388 error = ether_ioctl(ifp, command, data); 389 break; 390 } 391 392 return (error); 393} 394 395 396static void 397ntb_start(struct ifnet *ifp) 398{ 399 struct mbuf *m_head; 400 struct ntb_netdev *nt = ifp->if_softc; 401 int rc; 402 403 mtx_lock(&nt->tx_lock); 404 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 405 CTR0(KTR_NTB, "TX: ntb_start"); 406 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 407 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 408 CTR1(KTR_NTB, "TX: start mbuf %p", m_head); 409 rc = ntb_transport_tx_enqueue(nt->qp, m_head, m_head, 410 m_length(m_head, NULL)); 411 if (rc != 0) { 412 CTR1(KTR_NTB, 413 "TX: could not tx mbuf %p. Returning to snd q", 414 m_head); 415 if (rc == EAGAIN) { 416 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 417 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 418 callout_reset(&nt->qp->queue_full, hz / 1000, 419 ntb_qp_full, ifp); 420 } 421 break; 422 } 423 424 } 425 mtx_unlock(&nt->tx_lock); 426} 427 428/* Network Device Callbacks */ 429static void 430ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 431 int len) 432{ 433 434 m_freem(data); 435 CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data); 436} 437 438static void 439ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 440 int len) 441{ 442 struct mbuf *m = data; 443 struct ifnet *ifp = qp_data; 444 445 CTR0(KTR_NTB, "RX: rx handler"); 446 (*ifp->if_input)(ifp, m); 447} 448 449static void 450ntb_net_event_handler(void *data, int status) 451{ 452 453} 454 455/* Transport Init and teardown */ 456 457static int 458ntb_transport_init(struct ntb_softc *ntb) 459{ 460 struct ntb_netdev *nt = &net_softc; 461 int rc, i; 462 463 nt->max_qps = max_num_clients; 464 ntb_register_transport(ntb, nt); 465 mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF); 466 mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF); 467 468 nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp), 469 M_NTB_IF, M_WAITOK|M_ZERO); 470 471 nt->qp_bitmap = ((uint64_t) 1 << nt->max_qps) - 1; 472 473 for (i = 0; i < nt->max_qps; i++) 474 ntb_transport_init_queue(nt, i); 475 476 callout_init(&nt->link_work, 0); 477 478 rc = ntb_register_event_callback(ntb, 479 ntb_transport_event_callback); 480 if (rc != 0) 481 goto err; 482 483 if (ntb_query_link_status(ntb)) { 484 if (bootverbose) 485 device_printf(ntb_get_device(ntb), "link up\n"); 486 callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 487 } 488 489 return (0); 490 491err: 492 free(nt->qps, M_NTB_IF); 493 ntb_unregister_transport(ntb); 494 return (rc); 495} 496 497static void 498ntb_transport_free(void *transport) 499{ 500 struct ntb_netdev *nt = transport; 501 struct ntb_softc *ntb = nt->ntb; 502 int i; 503 504 nt->transport_link = NTB_LINK_DOWN; 505 506 callout_drain(&nt->link_work); 507 508 /* verify that all the qps are freed */ 509 for (i = 0; i < nt->max_qps; i++) 510 if (!test_bit(i, &nt->qp_bitmap)) 511 ntb_transport_free_queue(&nt->qps[i]); 512 513 514 ntb_unregister_event_callback(ntb); 515 516 for (i = 0; i < NTB_NUM_MW; i++) 517 if (nt->mw[i].virt_addr != NULL) 518 contigfree(nt->mw[i].virt_addr, nt->mw[i].size, 519 M_NTB_IF); 520 521 free(nt->qps, M_NTB_IF); 522 ntb_unregister_transport(ntb); 523} 524 525static void 526ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num) 527{ 528 struct ntb_transport_qp *qp; 529 unsigned int num_qps_mw, tx_size; 530 uint8_t mw_num = QP_TO_MW(qp_num); 531 532 qp = &nt->qps[qp_num]; 533 qp->qp_num = qp_num; 534 qp->transport = nt; 535 qp->ntb = nt->ntb; 536 qp->qp_link = NTB_LINK_DOWN; 537 qp->client_ready = NTB_LINK_DOWN; 538 qp->event_handler = NULL; 539 540 if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW) 541 num_qps_mw = nt->max_qps / NTB_NUM_MW + 1; 542 else 543 num_qps_mw = nt->max_qps / NTB_NUM_MW; 544 545 tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw; 546 qp->rx_info = (struct ntb_rx_info *) 547 ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) + 548 (qp_num / NTB_NUM_MW * tx_size)); 549 tx_size -= sizeof(struct ntb_rx_info); 550 551 qp->tx_mw = qp->rx_info + sizeof(struct ntb_rx_info); 552 qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header), 553 tx_size); 554 qp->tx_max_entry = tx_size / qp->tx_max_frame; 555 qp->tx_index = 0; 556 557 callout_init(&qp->link_work, 0); 558 callout_init(&qp->queue_full, 1); 559 callout_init(&qp->rx_full, 1); 560 561 mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN); 562 mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN); 563 mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN); 564 TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp); 565 566 STAILQ_INIT(&qp->rx_pend_q); 567 STAILQ_INIT(&qp->rx_free_q); 568 STAILQ_INIT(&qp->tx_free_q); 569} 570 571static void 572ntb_transport_free_queue(struct ntb_transport_qp *qp) 573{ 574 struct ntb_queue_entry *entry; 575 576 if (qp == NULL) 577 return; 578 579 callout_drain(&qp->link_work); 580 581 ntb_unregister_db_callback(qp->ntb, qp->qp_num); 582 583 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 584 free(entry, M_NTB_IF); 585 586 while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) 587 free(entry, M_NTB_IF); 588 589 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 590 free(entry, M_NTB_IF); 591 592 set_bit(qp->qp_num, &qp->transport->qp_bitmap); 593} 594 595/** 596 * ntb_transport_create_queue - Create a new NTB transport layer queue 597 * @rx_handler: receive callback function 598 * @tx_handler: transmit callback function 599 * @event_handler: event callback function 600 * 601 * Create a new NTB transport layer queue and provide the queue with a callback 602 * routine for both transmit and receive. The receive callback routine will be 603 * used to pass up data when the transport has received it on the queue. The 604 * transmit callback routine will be called when the transport has completed the 605 * transmission of the data on the queue and the data is ready to be freed. 606 * 607 * RETURNS: pointer to newly created ntb_queue, NULL on error. 608 */ 609static struct ntb_transport_qp * 610ntb_transport_create_queue(void *data, struct ntb_softc *pdev, 611 const struct ntb_queue_handlers *handlers) 612{ 613 struct ntb_queue_entry *entry; 614 struct ntb_transport_qp *qp; 615 struct ntb_netdev *nt; 616 unsigned int free_queue; 617 int rc, i; 618 619 nt = ntb_find_transport(pdev); 620 if (nt == NULL) 621 goto err; 622 623 free_queue = ffs(nt->qp_bitmap); 624 if (free_queue == 0) 625 goto err; 626 627 /* decrement free_queue to make it zero based */ 628 free_queue--; 629 630 clear_bit(free_queue, &nt->qp_bitmap); 631 632 qp = &nt->qps[free_queue]; 633 qp->cb_data = data; 634 qp->rx_handler = handlers->rx_handler; 635 qp->tx_handler = handlers->tx_handler; 636 qp->event_handler = handlers->event_handler; 637 638 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 639 entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF, 640 M_WAITOK|M_ZERO); 641 entry->cb_data = nt->ifp; 642 entry->buf = NULL; 643 entry->len = transport_mtu; 644 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 645 } 646 647 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 648 entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF, 649 M_WAITOK|M_ZERO); 650 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 651 } 652 653 rc = ntb_register_db_callback(qp->ntb, free_queue, qp, 654 ntb_transport_rxc_db); 655 if (rc != 0) 656 goto err1; 657 658 return (qp); 659 660err1: 661 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 662 free(entry, M_NTB_IF); 663 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 664 free(entry, M_NTB_IF); 665 set_bit(free_queue, &nt->qp_bitmap); 666err: 667 return (NULL); 668} 669 670/** 671 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 672 * @qp: NTB transport layer queue to be enabled 673 * 674 * Notify NTB transport layer of client readiness to use queue 675 */ 676static void 677ntb_transport_link_up(struct ntb_transport_qp *qp) 678{ 679 680 if (qp == NULL) 681 return; 682 683 qp->client_ready = NTB_LINK_UP; 684 if (bootverbose) 685 device_printf(ntb_get_device(qp->ntb), "qp client ready\n"); 686 687 if (qp->transport->transport_link == NTB_LINK_UP) 688 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 689} 690 691 692 693/* Transport Tx */ 694 695/** 696 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 697 * @qp: NTB transport layer queue the entry is to be enqueued on 698 * @cb: per buffer pointer for callback function to use 699 * @data: pointer to data buffer that will be sent 700 * @len: length of the data buffer 701 * 702 * Enqueue a new transmit buffer onto the transport queue from which a NTB 703 * payload will be transmitted. This assumes that a lock is behing held to 704 * serialize access to the qp. 705 * 706 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 707 */ 708static int 709ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 710 unsigned int len) 711{ 712 struct ntb_queue_entry *entry; 713 int rc; 714 715 if (qp == NULL || qp->qp_link != NTB_LINK_UP || len == 0) { 716 CTR0(KTR_NTB, "TX: link not up"); 717 return (EINVAL); 718 } 719 720 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 721 if (entry == NULL) { 722 CTR0(KTR_NTB, "TX: could not get entry from tx_free_q"); 723 return (ENOMEM); 724 } 725 CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry); 726 727 entry->cb_data = cb; 728 entry->buf = data; 729 entry->len = len; 730 entry->flags = 0; 731 732 rc = ntb_process_tx(qp, entry); 733 if (rc != 0) { 734 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 735 CTR1(KTR_NTB, 736 "TX: process_tx failed. Returning entry %p to tx_free_q", 737 entry); 738 } 739 return (rc); 740} 741 742static int 743ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry) 744{ 745 void *offset; 746 747 offset = (char *)qp->tx_mw + qp->tx_max_frame * qp->tx_index; 748 CTR3(KTR_NTB, 749 "TX: process_tx: tx_pkts=%u, tx_index=%u, remote entry=%u", 750 qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry); 751 if (qp->tx_index == qp->remote_rx_info->entry) { 752 CTR0(KTR_NTB, "TX: ring full"); 753 qp->tx_ring_full++; 754 return (EAGAIN); 755 } 756 757 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 758 if (qp->tx_handler != NULL) 759 qp->tx_handler(qp, qp->cb_data, entry->buf, 760 EIO); 761 762 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 763 CTR1(KTR_NTB, 764 "TX: frame too big. returning entry %p to tx_free_q", 765 entry); 766 return (0); 767 } 768 CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset); 769 ntb_tx_copy_task(qp, entry, offset); 770 771 qp->tx_index++; 772 qp->tx_index %= qp->tx_max_entry; 773 774 qp->tx_pkts++; 775 776 return (0); 777} 778 779static void 780ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 781 void *offset) 782{ 783 struct ntb_payload_header *hdr; 784 785 CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset); 786 if (entry->buf != NULL) 787 m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset); 788 789 hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame - 790 sizeof(struct ntb_payload_header)); 791 hdr->len = entry->len; /* TODO: replace with bus_space_write */ 792 hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */ 793 wmb(); 794 /* TODO: replace with bus_space_write */ 795 hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG; 796 797 ntb_ring_sdb(qp->ntb, qp->qp_num); 798 799 /* 800 * The entry length can only be zero if the packet is intended to be a 801 * "link down" or similar. Since no payload is being sent in these 802 * cases, there is nothing to add to the completion queue. 803 */ 804 if (entry->len > 0) { 805 qp->tx_bytes += entry->len; 806 807 if (qp->tx_handler) 808 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 809 entry->len); 810 } 811 812 CTR2(KTR_NTB, 813 "TX: entry %p sent. hdr->ver = %d, Returning to tx_free_q", entry, 814 hdr->ver); 815 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 816} 817 818static void 819ntb_qp_full(void *arg) 820{ 821 822 CTR0(KTR_NTB, "TX: qp_full callout"); 823 ntb_start(arg); 824} 825 826/* Transport Rx */ 827static void 828ntb_transport_rxc_db(void *data, int db_num) 829{ 830 struct ntb_transport_qp *qp = data; 831 832 ntb_transport_rx(qp); 833} 834 835static void 836ntb_rx_pendq_full(void *arg) 837{ 838 839 CTR0(KTR_NTB, "RX: ntb_rx_pendq_full callout"); 840 ntb_transport_rx(arg); 841} 842 843static void 844ntb_transport_rx(struct ntb_transport_qp *qp) 845{ 846 int rc, i; 847 848 /* 849 * Limit the number of packets processed in a single interrupt to 850 * provide fairness to others 851 */ 852 mtx_lock(&qp->transport->rx_lock); 853 CTR0(KTR_NTB, "RX: transport_rx"); 854 for (i = 0; i < NTB_RX_MAX_PKTS; i++) { 855 rc = ntb_process_rxc(qp); 856 if (rc != 0) { 857 CTR0(KTR_NTB, "RX: process_rxc failed"); 858 break; 859 } 860 } 861 mtx_unlock(&qp->transport->rx_lock); 862} 863 864static int 865ntb_process_rxc(struct ntb_transport_qp *qp) 866{ 867 struct ntb_payload_header *hdr; 868 struct ntb_queue_entry *entry; 869 void *offset; 870 871 offset = (void *) 872 ((char *)qp->rx_buff + qp->rx_max_frame * qp->rx_index); 873 hdr = (void *) 874 ((char *)offset + qp->rx_max_frame - 875 sizeof(struct ntb_payload_header)); 876 877 CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index); 878 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 879 if (entry == NULL) { 880 qp->rx_err_no_buf++; 881 CTR0(KTR_NTB, "RX: No entries in rx_pend_q"); 882 return (ENOMEM); 883 } 884 callout_stop(&qp->rx_full); 885 CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry); 886 887 if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) { 888 CTR1(KTR_NTB, 889 "RX: hdr not done. Returning entry %p to rx_pend_q", entry); 890 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 891 qp->rx_ring_empty++; 892 return (EAGAIN); 893 } 894 895 if (hdr->ver != (uint32_t) qp->rx_pkts) { 896 CTR3(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). " 897 "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts, 898 entry); 899 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 900 qp->rx_err_ver++; 901 return (EIO); 902 } 903 904 if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) { 905 ntb_qp_link_down(qp); 906 CTR1(KTR_NTB, 907 "RX: link down. adding entry %p back to rx_pend_q", entry); 908 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 909 goto out; 910 } 911 912 if (hdr->len <= entry->len) { 913 entry->len = hdr->len; 914 ntb_rx_copy_task(qp, entry, offset); 915 } else { 916 CTR1(KTR_NTB, 917 "RX: len too long. Returning entry %p to rx_pend_q", entry); 918 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 919 920 qp->rx_err_oflow++; 921 } 922 923 qp->rx_bytes += hdr->len; 924 qp->rx_pkts++; 925 CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts); 926 927 928out: 929 /* Ensure that the data is globally visible before clearing the flag */ 930 wmb(); 931 hdr->flags = 0; 932 /* TODO: replace with bus_space_write */ 933 qp->rx_info->entry = qp->rx_index; 934 935 qp->rx_index++; 936 qp->rx_index %= qp->rx_max_entry; 937 938 return (0); 939} 940 941static void 942ntb_rx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 943 void *offset) 944{ 945 struct ifnet *ifp = entry->cb_data; 946 unsigned int len = entry->len; 947 struct mbuf *m; 948 949 CTR2(KTR_NTB, "RX: copying %d bytes from offset %p", len, offset); 950 m = m_devget(offset, len, 0, ifp, NULL); 951 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID; 952 953 entry->buf = (void *)m; 954 955 CTR2(KTR_NTB, 956 "RX: copied entry %p to mbuf %p. Adding entry to rx_free_q", entry, 957 m); 958 ntb_list_add(&qp->ntb_rx_free_q_lock, entry, &qp->rx_free_q); 959 960 taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task); 961} 962 963static void 964ntb_rx_completion_task(void *arg, int pending) 965{ 966 struct ntb_transport_qp *qp = arg; 967 struct mbuf *m; 968 struct ntb_queue_entry *entry; 969 970 CTR0(KTR_NTB, "RX: rx_completion_task"); 971 972 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) { 973 m = entry->buf; 974 CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m); 975 if (qp->rx_handler && qp->client_ready == NTB_LINK_UP) 976 qp->rx_handler(qp, qp->cb_data, m, entry->len); 977 978 entry->buf = NULL; 979 entry->len = qp->transport->bufsize; 980 981 CTR1(KTR_NTB,"RX: entry %p removed from rx_free_q " 982 "and added to rx_pend_q", entry); 983 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 984 if (qp->rx_err_no_buf > qp->last_rx_no_buf) { 985 qp->last_rx_no_buf = qp->rx_err_no_buf; 986 CTR0(KTR_NTB, "RX: could spawn rx task"); 987 callout_reset(&qp->rx_full, hz / 1000, ntb_rx_pendq_full, 988 qp); 989 } 990 } 991} 992 993/* Link Event handler */ 994static void 995ntb_transport_event_callback(void *data, enum ntb_hw_event event) 996{ 997 struct ntb_netdev *nt = data; 998 999 switch (event) { 1000 case NTB_EVENT_HW_LINK_UP: 1001 if (bootverbose) 1002 device_printf(ntb_get_device(nt->ntb), "HW link up\n"); 1003 callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 1004 break; 1005 case NTB_EVENT_HW_LINK_DOWN: 1006 if (bootverbose) 1007 device_printf(ntb_get_device(nt->ntb), "HW link down\n"); 1008 ntb_transport_link_cleanup(nt); 1009 break; 1010 default: 1011 panic("ntb: Unknown NTB event"); 1012 } 1013} 1014 1015/* Link bring up */ 1016static void 1017ntb_transport_link_work(void *arg) 1018{ 1019 struct ntb_netdev *nt = arg; 1020 struct ntb_softc *ntb = nt->ntb; 1021 struct ntb_transport_qp *qp; 1022 uint32_t val; 1023 int rc, i; 1024 1025 /* send the local info */ 1026 rc = ntb_write_remote_spad(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION); 1027 if (rc != 0) 1028 goto out; 1029 1030 rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ, ntb_get_mw_size(ntb, 0)); 1031 if (rc != 0) 1032 goto out; 1033 1034 rc = ntb_write_remote_spad(ntb, IF_NTB_MW1_SZ, ntb_get_mw_size(ntb, 1)); 1035 if (rc != 0) 1036 goto out; 1037 1038 rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_QPS, nt->max_qps); 1039 if (rc != 0) 1040 goto out; 1041 1042 rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val); 1043 if (rc != 0) 1044 goto out; 1045 1046 rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val); 1047 if (rc != 0) 1048 goto out; 1049 1050 /* Query the remote side for its info */ 1051 rc = ntb_read_local_spad(ntb, IF_NTB_VERSION, &val); 1052 if (rc != 0) 1053 goto out; 1054 1055 if (val != NTB_TRANSPORT_VERSION) 1056 goto out; 1057 1058 rc = ntb_read_local_spad(ntb, IF_NTB_NUM_QPS, &val); 1059 if (rc != 0) 1060 goto out; 1061 1062 if (val != nt->max_qps) 1063 goto out; 1064 1065 rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ, &val); 1066 if (rc != 0) 1067 goto out; 1068 1069 if (val == 0) 1070 goto out; 1071 1072 rc = ntb_set_mw(nt, 0, val); 1073 if (rc != 0) 1074 return; 1075 1076 rc = ntb_read_local_spad(ntb, IF_NTB_MW1_SZ, &val); 1077 if (rc != 0) 1078 goto out; 1079 1080 if (val == 0) 1081 goto out; 1082 1083 rc = ntb_set_mw(nt, 1, val); 1084 if (rc != 0) 1085 return; 1086 1087 nt->transport_link = NTB_LINK_UP; 1088 if (bootverbose) 1089 device_printf(ntb_get_device(ntb), "transport link up\n"); 1090 1091 for (i = 0; i < nt->max_qps; i++) { 1092 qp = &nt->qps[i]; 1093 1094 ntb_transport_setup_qp_mw(nt, i); 1095 1096 if (qp->client_ready == NTB_LINK_UP) 1097 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 1098 } 1099 1100 return; 1101 1102out: 1103 if (ntb_query_link_status(ntb)) 1104 callout_reset(&nt->link_work, 1105 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt); 1106} 1107 1108static int 1109ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size) 1110{ 1111 struct ntb_transport_mw *mw = &nt->mw[num_mw]; 1112 1113 /* Alloc memory for receiving data. Must be 4k aligned */ 1114 mw->size = size; 1115 1116 mw->virt_addr = contigmalloc(mw->size, M_NTB_IF, M_ZERO, 0, 1117 BUS_SPACE_MAXADDR, mw->size, 0); 1118 if (mw->virt_addr == NULL) { 1119 printf("ntb: Unable to allocate MW buffer of size %d\n", 1120 (int)mw->size); 1121 return (ENOMEM); 1122 } 1123 /* TODO: replace with bus_space_* functions */ 1124 mw->dma_addr = vtophys(mw->virt_addr); 1125 1126 /* Notify HW the memory location of the receive buffer */ 1127 ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr); 1128 1129 return (0); 1130} 1131 1132static void 1133ntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num) 1134{ 1135 struct ntb_transport_qp *qp = &nt->qps[qp_num]; 1136 void *offset; 1137 unsigned int rx_size, num_qps_mw; 1138 uint8_t mw_num = QP_TO_MW(qp_num); 1139 unsigned int i; 1140 1141 if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW) 1142 num_qps_mw = nt->max_qps / NTB_NUM_MW + 1; 1143 else 1144 num_qps_mw = nt->max_qps / NTB_NUM_MW; 1145 1146 rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw; 1147 qp->remote_rx_info = (void *)((uint8_t *)nt->mw[mw_num].virt_addr + 1148 (qp_num / NTB_NUM_MW * rx_size)); 1149 rx_size -= sizeof(struct ntb_rx_info); 1150 1151 qp->rx_buff = qp->remote_rx_info + sizeof(struct ntb_rx_info); 1152 qp->rx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header), 1153 rx_size); 1154 qp->rx_max_entry = rx_size / qp->rx_max_frame; 1155 qp->rx_index = 0; 1156 qp->tx_index = 0; 1157 1158 qp->remote_rx_info->entry = qp->rx_max_entry; 1159 1160 /* setup the hdr offsets with 0's */ 1161 for (i = 0; i < qp->rx_max_entry; i++) { 1162 offset = (void *)((uint8_t *)qp->rx_buff + 1163 qp->rx_max_frame * (i + 1) - 1164 sizeof(struct ntb_payload_header)); 1165 memset(offset, 0, sizeof(struct ntb_payload_header)); 1166 } 1167 1168 qp->rx_pkts = 0; 1169 qp->tx_pkts = 0; 1170} 1171 1172static void 1173ntb_qp_link_work(void *arg) 1174{ 1175 struct ntb_transport_qp *qp = arg; 1176 struct ntb_softc *ntb = qp->ntb; 1177 struct ntb_netdev *nt = qp->transport; 1178 int rc, val; 1179 1180 1181 rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val); 1182 if (rc != 0) 1183 return; 1184 1185 rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num); 1186 1187 /* query remote spad for qp ready bits */ 1188 rc = ntb_read_local_spad(ntb, IF_NTB_QP_LINKS, &val); 1189 1190 /* See if the remote side is up */ 1191 if ((1 << qp->qp_num & val) != 0) { 1192 qp->qp_link = NTB_LINK_UP; 1193 if (qp->event_handler != NULL) 1194 qp->event_handler(qp->cb_data, NTB_LINK_UP); 1195 if (bootverbose) 1196 device_printf(ntb_get_device(ntb), "qp link up\n"); 1197 } else if (nt->transport_link == NTB_LINK_UP) { 1198 callout_reset(&qp->link_work, 1199 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1200 } 1201} 1202 1203/* Link down event*/ 1204static void 1205ntb_transport_link_cleanup(struct ntb_netdev *nt) 1206{ 1207 int i; 1208 1209 if (nt->transport_link == NTB_LINK_DOWN) 1210 callout_drain(&nt->link_work); 1211 else 1212 nt->transport_link = NTB_LINK_DOWN; 1213 1214 /* Pass along the info to any clients */ 1215 for (i = 0; i < nt->max_qps; i++) 1216 if (!test_bit(i, &nt->qp_bitmap)) 1217 ntb_qp_link_down(&nt->qps[i]); 1218 1219 /* 1220 * The scratchpad registers keep the values if the remote side 1221 * goes down, blast them now to give them a sane value the next 1222 * time they are accessed 1223 */ 1224 for (i = 0; i < IF_NTB_MAX_SPAD; i++) 1225 ntb_write_local_spad(nt->ntb, i, 0); 1226} 1227 1228 1229static void 1230ntb_qp_link_down(struct ntb_transport_qp *qp) 1231{ 1232 1233 ntb_qp_link_cleanup(qp); 1234} 1235 1236static void 1237ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 1238{ 1239 struct ntb_netdev *nt = qp->transport; 1240 1241 if (qp->qp_link == NTB_LINK_DOWN) { 1242 callout_drain(&qp->link_work); 1243 return; 1244 } 1245 1246 if (qp->event_handler != NULL) 1247 qp->event_handler(qp->cb_data, NTB_LINK_DOWN); 1248 1249 qp->qp_link = NTB_LINK_DOWN; 1250 1251 if (nt->transport_link == NTB_LINK_UP) 1252 callout_reset(&qp->link_work, 1253 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1254} 1255 1256/* Link commanded down */ 1257/** 1258 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 1259 * @qp: NTB transport layer queue to be disabled 1260 * 1261 * Notify NTB transport layer of client's desire to no longer receive data on 1262 * transport queue specified. It is the client's responsibility to ensure all 1263 * entries on queue are purged or otherwise handled appropraitely. 1264 */ 1265static void 1266ntb_transport_link_down(struct ntb_transport_qp *qp) 1267{ 1268 int rc, val; 1269 1270 if (qp == NULL) 1271 return; 1272 1273 qp->client_ready = NTB_LINK_DOWN; 1274 1275 rc = ntb_read_remote_spad(qp->ntb, IF_NTB_QP_LINKS, &val); 1276 if (rc != 0) 1277 return; 1278 1279 rc = ntb_write_remote_spad(qp->ntb, IF_NTB_QP_LINKS, 1280 val & ~(1 << qp->qp_num)); 1281 1282 if (qp->qp_link == NTB_LINK_UP) 1283 ntb_send_link_down(qp); 1284 else 1285 callout_drain(&qp->link_work); 1286 1287} 1288 1289static void 1290ntb_send_link_down(struct ntb_transport_qp *qp) 1291{ 1292 struct ntb_queue_entry *entry; 1293 int i, rc; 1294 1295 if (qp->qp_link == NTB_LINK_DOWN) 1296 return; 1297 1298 qp->qp_link = NTB_LINK_DOWN; 1299 1300 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1301 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1302 if (entry != NULL) 1303 break; 1304 pause("NTB Wait for link down", hz / 10); 1305 } 1306 1307 if (entry == NULL) 1308 return; 1309 1310 entry->cb_data = NULL; 1311 entry->buf = NULL; 1312 entry->len = 0; 1313 entry->flags = IF_NTB_LINK_DOWN_FLAG; 1314 1315 mtx_lock(&qp->transport->tx_lock); 1316 rc = ntb_process_tx(qp, entry); 1317 if (rc != 0) 1318 printf("ntb: Failed to send link down\n"); 1319 mtx_unlock(&qp->transport->tx_lock); 1320} 1321 1322 1323/* List Management */ 1324 1325static void 1326ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 1327 struct ntb_queue_list *list) 1328{ 1329 1330 mtx_lock_spin(lock); 1331 STAILQ_INSERT_TAIL(list, entry, entry); 1332 mtx_unlock_spin(lock); 1333} 1334 1335static struct ntb_queue_entry * 1336ntb_list_rm(struct mtx *lock, struct ntb_queue_list *list) 1337{ 1338 struct ntb_queue_entry *entry; 1339 1340 mtx_lock_spin(lock); 1341 if (STAILQ_EMPTY(list)) { 1342 entry = NULL; 1343 goto out; 1344 } 1345 entry = STAILQ_FIRST(list); 1346 STAILQ_REMOVE_HEAD(list, entry); 1347out: 1348 mtx_unlock_spin(lock); 1349 1350 return (entry); 1351} 1352 1353/* Helper functions */ 1354/* TODO: This too should really be part of the kernel */ 1355#define EUI48_MULTICAST 1 << 0 1356#define EUI48_LOCALLY_ADMINISTERED 1 << 1 1357static void 1358create_random_local_eui48(u_char *eaddr) 1359{ 1360 static uint8_t counter = 0; 1361 uint32_t seed = ticks; 1362 1363 eaddr[0] = EUI48_LOCALLY_ADMINISTERED; 1364 memcpy(&eaddr[1], &seed, sizeof(uint32_t)); 1365 eaddr[5] = counter++; 1366} 1367 1368/** 1369 * ntb_transport_max_size - Query the max payload size of a qp 1370 * @qp: NTB transport layer queue to be queried 1371 * 1372 * Query the maximum payload size permissible on the given qp 1373 * 1374 * RETURNS: the max payload size of a qp 1375 */ 1376static unsigned int 1377ntb_transport_max_size(struct ntb_transport_qp *qp) 1378{ 1379 1380 if (qp == NULL) 1381 return (0); 1382 1383 return (qp->tx_max_frame - sizeof(struct ntb_payload_header)); 1384} 1385