ntb_hw_intel.c revision 255268
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 255268 2013-09-05 22:46:48Z carl $"); 29 30#include <sys/param.h> 31#include <sys/kernel.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/malloc.h> 35#include <sys/module.h> 36#include <sys/queue.h> 37#include <sys/rman.h> 38#include <vm/vm.h> 39#include <vm/pmap.h> 40#include <machine/bus.h> 41#include <machine/pmap.h> 42#include <machine/resource.h> 43#include <dev/pci/pcireg.h> 44#include <dev/pci/pcivar.h> 45 46#include "ntb_regs.h" 47#include "ntb_hw.h" 48 49/* 50 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 51 * allows you to connect two systems using a PCI-e link. 52 * 53 * This module contains the hardware abstraction layer for the NTB. It allows 54 * you to send and recieve interrupts, map the memory windows and send and 55 * receive messages in the scratch-pad registers. 56 * 57 * NOTE: Much of the code in this module is shared with Linux. Any patches may 58 * be picked up and redistributed in Linux with a dual GPL/BSD license. 59 */ 60 61#define NTB_CONFIG_BAR 0 62#define NTB_B2B_BAR_1 1 63#define NTB_B2B_BAR_2 2 64#define NTB_MAX_BARS 3 65#define NTB_MW_TO_BAR(mw) ((mw) + 1) 66 67#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 68 69#define NTB_HB_TIMEOUT 1 /* second */ 70#define SOC_LINK_RECOVERY_TIME 500 71 72#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 73 74enum ntb_device_type { 75 NTB_XEON, 76 NTB_SOC 77}; 78 79struct ntb_hw_info { 80 uint32_t device_id; 81 enum ntb_device_type type; 82 const char *desc; 83}; 84 85struct ntb_pci_bar_info { 86 bus_space_tag_t pci_bus_tag; 87 bus_space_handle_t pci_bus_handle; 88 int pci_resource_id; 89 struct resource *pci_resource; 90 vm_paddr_t pbase; 91 void *vbase; 92 u_long size; 93}; 94 95struct ntb_int_info { 96 struct resource *res; 97 int rid; 98 void *tag; 99}; 100 101struct ntb_db_cb { 102 ntb_db_callback callback; 103 unsigned int db_num; 104 void *data; 105 struct ntb_softc *ntb; 106}; 107 108struct ntb_softc { 109 device_t device; 110 enum ntb_device_type type; 111 112 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 113 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 114 uint32_t allocated_interrupts; 115 116 struct callout heartbeat_timer; 117 struct callout lr_timer; 118 119 void *ntb_transport; 120 ntb_event_callback event_cb; 121 struct ntb_db_cb *db_cb; 122 123 struct { 124 uint32_t max_spads; 125 uint32_t max_db_bits; 126 uint32_t msix_cnt; 127 } limits; 128 struct { 129 uint32_t pdb; 130 uint32_t pdb_mask; 131 uint32_t sdb; 132 uint32_t sbar2_xlat; 133 uint32_t sbar4_xlat; 134 uint32_t spad_remote; 135 uint32_t spad_local; 136 uint32_t lnk_cntl; 137 uint32_t lnk_stat; 138 uint32_t spci_cmd; 139 } reg_ofs; 140 uint8_t conn_type; 141 uint8_t dev_type; 142 uint8_t bits_per_vector; 143 uint8_t link_status; 144 uint8_t link_width; 145 uint8_t link_speed; 146}; 147 148#define ntb_reg_read(SIZE, offset) \ 149 bus_space_read_ ## SIZE (ntb->bar_info[NTB_CONFIG_BAR].pci_bus_tag, \ 150 ntb->bar_info[NTB_CONFIG_BAR].pci_bus_handle, (offset)) 151#define ntb_reg_write(SIZE, offset, val) \ 152 bus_space_write_ ## SIZE (ntb->bar_info[NTB_CONFIG_BAR].pci_bus_tag, \ 153 ntb->bar_info[NTB_CONFIG_BAR].pci_bus_handle, (offset), (val)) 154 155#define ntb_read_1(offset) ntb_reg_read(1, (offset)) 156#define ntb_read_2(offset) ntb_reg_read(2, (offset)) 157#define ntb_read_4(offset) ntb_reg_read(4, (offset)) 158#define ntb_read_8(offset) ntb_reg_read(8, (offset)) 159#define ntb_write_1(offset, val) ntb_reg_write(1, (offset), (val)) 160#define ntb_write_2(offset, val) ntb_reg_write(2, (offset), (val)) 161#define ntb_write_4(offset, val) ntb_reg_write(4, (offset), (val)) 162#define ntb_write_8(offset, val) ntb_reg_write(8, (offset), (val)) 163 164static int ntb_probe(device_t device); 165static int ntb_attach(device_t device); 166static int ntb_detach(device_t device); 167static int ntb_map_pci_bar(struct ntb_softc *ntb); 168static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 169static int ntb_setup_interrupts(struct ntb_softc *ntb); 170static void ntb_teardown_interrupts(struct ntb_softc *ntb); 171static void handle_soc_irq(void *arg); 172static void handle_xeon_irq(void *arg); 173static void handle_xeon_event_irq(void *arg); 174static void ntb_handle_legacy_interrupt(void *arg); 175static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 176static void ntb_free_callbacks(struct ntb_softc *ntb); 177static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 178static int ntb_initialize_hw(struct ntb_softc *ntb); 179static int ntb_setup_xeon(struct ntb_softc *ntb); 180static int ntb_setup_soc(struct ntb_softc *ntb); 181static void ntb_handle_heartbeat(void *arg); 182static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 183static void recover_soc_link(void *arg); 184static int ntb_check_link_status(struct ntb_softc *ntb); 185static bool is_bar_for_data_transfer(int bar_num); 186 187static struct ntb_hw_info pci_ids[] = { 188 { 0x3C0D8086, NTB_XEON, "Xeon E5/Core i7 Non-Transparent Bridge B2B" }, 189 { 0x0C4E8086, NTB_SOC, "Atom Processor S1200 NTB Primary B2B" }, 190 { 0x0E0D8086, NTB_XEON, "Xeon E5 V2 Non-Transparent Bridge B2B" }, 191 { 0x00000000, NTB_SOC, NULL } 192}; 193 194/* 195 * OS <-> Driver interface structures 196 */ 197MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 198 199static device_method_t ntb_pci_methods[] = { 200 /* Device interface */ 201 DEVMETHOD(device_probe, ntb_probe), 202 DEVMETHOD(device_attach, ntb_attach), 203 DEVMETHOD(device_detach, ntb_detach), 204 DEVMETHOD_END 205}; 206 207static driver_t ntb_pci_driver = { 208 "ntb_hw", 209 ntb_pci_methods, 210 sizeof(struct ntb_softc), 211}; 212 213static devclass_t ntb_devclass; 214DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 215MODULE_VERSION(ntb_hw, 1); 216 217/* 218 * OS <-> Driver linkage functions 219 */ 220static int 221ntb_probe(device_t device) 222{ 223 struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device)); 224 225 if (p != NULL) { 226 device_set_desc(device, p->desc); 227 return (0); 228 } else 229 return (ENXIO); 230} 231 232#define DETACH_ON_ERROR(func) \ 233 error = func; \ 234 if (error < 0) { \ 235 ntb_detach(device); \ 236 return (error); \ 237 } 238 239static int 240ntb_attach(device_t device) 241{ 242 struct ntb_softc *ntb = DEVICE2SOFTC(device); 243 struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device)); 244 int error; 245 246 ntb->device = device; 247 ntb->type = p->type; 248 249 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 250 callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE); 251 callout_init(&ntb->lr_timer, CALLOUT_MPSAFE); 252 253 DETACH_ON_ERROR(ntb_map_pci_bar(ntb)); 254 DETACH_ON_ERROR(ntb_initialize_hw(ntb)); 255 DETACH_ON_ERROR(ntb_setup_interrupts(ntb)); 256 257 pci_enable_busmaster(ntb->device); 258 259 return (error); 260} 261 262static int 263ntb_detach(device_t device) 264{ 265 struct ntb_softc *ntb = DEVICE2SOFTC(device); 266 267 callout_drain(&ntb->heartbeat_timer); 268 callout_drain(&ntb->lr_timer); 269 ntb_teardown_interrupts(ntb); 270 ntb_unmap_pci_bar(ntb); 271 272 return (0); 273} 274 275static int 276ntb_map_pci_bar(struct ntb_softc *ntb) 277{ 278 struct ntb_pci_bar_info *current_bar; 279 int rc, i; 280 281 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 282 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 283 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 284 285 for (i = 0; i< NTB_MAX_BARS; i++) { 286 current_bar = &ntb->bar_info[i]; 287 current_bar->pci_resource = 288 bus_alloc_resource(ntb->device, 289 SYS_RES_MEMORY, 290 ¤t_bar->pci_resource_id, 0, ~0, 1, 291 RF_ACTIVE); 292 293 if (current_bar->pci_resource == NULL) { 294 device_printf(ntb->device, 295 "unable to allocate pci resource\n"); 296 return (ENXIO); 297 } 298 else { 299 current_bar->pci_bus_tag = 300 rman_get_bustag(current_bar->pci_resource); 301 current_bar->pci_bus_handle = 302 rman_get_bushandle(current_bar->pci_resource); 303 current_bar->pbase = 304 rman_get_start(current_bar->pci_resource); 305 current_bar->size = 306 rman_get_size(current_bar->pci_resource); 307 current_bar->vbase = 308 rman_get_virtual(current_bar->pci_resource); 309 if (is_bar_for_data_transfer(i)) { 310 /* 311 * Mark bar region as write combining to improve 312 * performance. 313 */ 314 rc = pmap_change_attr( 315 (vm_offset_t)current_bar->vbase, 316 current_bar->size, 317 VM_MEMATTR_WRITE_COMBINING); 318 if (rc != 0) { 319 device_printf(ntb->device, 320 "Couldn't mark bar as" 321 " WRITE_COMBINING\n"); 322 return (rc); 323 } 324 } 325 device_printf(ntb->device, 326 "Bar size = %lx, v %p, p %p\n", 327 current_bar->size, current_bar->vbase, 328 (void *)(current_bar->pbase)); 329 } 330 } 331 return (0); 332} 333 334static void 335ntb_unmap_pci_bar(struct ntb_softc *ntb) 336{ 337 struct ntb_pci_bar_info *current_bar; 338 int i; 339 340 for (i = 0; i< NTB_MAX_BARS; i++) { 341 current_bar = &ntb->bar_info[i]; 342 if (current_bar->pci_resource != NULL) 343 bus_release_resource(ntb->device, SYS_RES_MEMORY, 344 current_bar->pci_resource_id, 345 current_bar->pci_resource); 346 } 347} 348 349static int 350ntb_setup_interrupts(struct ntb_softc *ntb) 351{ 352 void (*interrupt_handler)(void *); 353 void *int_arg; 354 bool use_msix = 0; 355 uint32_t num_vectors; 356 int i; 357 358 ntb->allocated_interrupts = 0; 359 /* 360 * On SOC, disable all interrupts. On XEON, disable all but Link 361 * Interrupt. The rest will be unmasked as callbacks are registered. 362 */ 363 if (ntb->type == NTB_SOC) 364 ntb_write_8(ntb->reg_ofs.pdb_mask, ~0); 365 else 366 ntb_write_2(ntb->reg_ofs.pdb_mask, 367 ~(1 << ntb->limits.max_db_bits)); 368 369 num_vectors = MIN(pci_msix_count(ntb->device), 370 ntb->limits.max_db_bits); 371 if (num_vectors >= 1) { 372 pci_alloc_msix(ntb->device, &num_vectors); 373 if (num_vectors >= 4) 374 use_msix = TRUE; 375 } 376 377 ntb_create_callbacks(ntb, num_vectors); 378 if (use_msix == TRUE) { 379 for (i = 0; i < num_vectors; i++) { 380 ntb->int_info[i].rid = i + 1; 381 ntb->int_info[i].res = bus_alloc_resource_any( 382 ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 383 RF_ACTIVE); 384 if (ntb->int_info[i].res == NULL) { 385 device_printf(ntb->device, 386 "bus_alloc_resource failed\n"); 387 return (-1); 388 } 389 ntb->int_info[i].tag = NULL; 390 ntb->allocated_interrupts++; 391 if (ntb->type == NTB_SOC) { 392 interrupt_handler = handle_soc_irq; 393 int_arg = &ntb->db_cb[i]; 394 } else { 395 if (i == num_vectors - 1) { 396 interrupt_handler = handle_xeon_event_irq; 397 int_arg = ntb; 398 } else { 399 interrupt_handler = 400 handle_xeon_irq; 401 int_arg = &ntb->db_cb[i]; 402 } 403 } 404 if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 405 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 406 interrupt_handler, int_arg, 407 &ntb->int_info[i].tag) != 0) { 408 device_printf(ntb->device, 409 "bus_setup_intr failed\n"); 410 return (ENXIO); 411 } 412 } 413 } 414 else { 415 ntb->int_info[0].rid = 0; 416 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 417 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 418 interrupt_handler = ntb_handle_legacy_interrupt; 419 if (ntb->int_info[0].res == NULL) { 420 device_printf(ntb->device, 421 "bus_alloc_resource failed\n"); 422 return (-1); 423 } 424 ntb->int_info[0].tag = NULL; 425 ntb->allocated_interrupts = 1; 426 427 if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 428 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 429 interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 430 431 device_printf(ntb->device, "bus_setup_intr failed\n"); 432 return (ENXIO); 433 } 434 } 435 436 return (0); 437} 438 439static void 440ntb_teardown_interrupts(struct ntb_softc *ntb) 441{ 442 struct ntb_int_info *current_int; 443 int i; 444 445 for (i=0; i<ntb->allocated_interrupts; i++) { 446 current_int = &ntb->int_info[i]; 447 if (current_int->tag != NULL) 448 bus_teardown_intr(ntb->device, current_int->res, 449 current_int->tag); 450 451 if (current_int->res != NULL) 452 bus_release_resource(ntb->device, SYS_RES_IRQ, 453 rman_get_rid(current_int->res), current_int->res); 454 } 455 456 ntb_free_callbacks(ntb); 457 pci_release_msi(ntb->device); 458} 459 460static void 461handle_soc_irq(void *arg) 462{ 463 struct ntb_db_cb *db_cb = arg; 464 struct ntb_softc *ntb = db_cb->ntb; 465 466 ntb_write_8(ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num); 467 468 if (db_cb->callback != NULL) 469 db_cb->callback(db_cb->data, db_cb->db_num); 470} 471 472static void 473handle_xeon_irq(void *arg) 474{ 475 struct ntb_db_cb *db_cb = arg; 476 struct ntb_softc *ntb = db_cb->ntb; 477 478 /* 479 * On Xeon, there are 16 bits in the interrupt register 480 * but only 4 vectors. So, 5 bits are assigned to the first 3 481 * vectors, with the 4th having a single bit for link 482 * interrupts. 483 */ 484 ntb_write_2(ntb->reg_ofs.pdb, 485 ((1 << ntb->bits_per_vector) - 1) << 486 (db_cb->db_num * ntb->bits_per_vector)); 487 488 if (db_cb->callback != NULL) 489 db_cb->callback(db_cb->data, db_cb->db_num); 490} 491 492/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 493static void 494handle_xeon_event_irq(void *arg) 495{ 496 struct ntb_softc *ntb = arg; 497 int rc; 498 499 rc = ntb_check_link_status(ntb); 500 if (rc != 0) 501 device_printf(ntb->device, "Error determining link status\n"); 502 503 /* bit 15 is always the link bit */ 504 ntb_write_2(ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits); 505} 506 507static void 508ntb_handle_legacy_interrupt(void *arg) 509{ 510 struct ntb_softc *ntb = arg; 511 unsigned int i = 0; 512 uint64_t pdb64; 513 uint16_t pdb16; 514 515 if (ntb->type == NTB_SOC) { 516 pdb64 = ntb_read_8(ntb->reg_ofs.pdb); 517 518 while (pdb64) { 519 i = ffs(pdb64); 520 pdb64 &= pdb64 - 1; 521 handle_soc_irq(&ntb->db_cb[i]); 522 } 523 } else { 524 pdb16 = ntb_read_2(ntb->reg_ofs.pdb); 525 526 if ((pdb16 & XEON_DB_HW_LINK) != 0) { 527 handle_xeon_event_irq(ntb); 528 pdb16 &= ~XEON_DB_HW_LINK; 529 } 530 531 while (pdb16 != 0) { 532 i = ffs(pdb16); 533 pdb16 &= pdb16 - 1; 534 handle_xeon_irq(&ntb->db_cb[i]); 535 } 536 } 537 538} 539 540static int 541ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 542{ 543 int i; 544 545 ntb->db_cb = malloc(num_vectors * sizeof(struct ntb_db_cb), M_NTB, 546 M_ZERO | M_WAITOK); 547 for (i = 0; i < num_vectors; i++) { 548 ntb->db_cb[i].db_num = i; 549 ntb->db_cb[i].ntb = ntb; 550 } 551 552 return (0); 553} 554 555static void 556ntb_free_callbacks(struct ntb_softc *ntb) 557{ 558 int i; 559 560 for (i = 0; i < ntb->limits.max_db_bits; i++) 561 ntb_unregister_db_callback(ntb, i); 562 563 free(ntb->db_cb, M_NTB); 564} 565 566static struct ntb_hw_info * 567ntb_get_device_info(uint32_t device_id) 568{ 569 struct ntb_hw_info *ep = pci_ids; 570 571 while (ep->device_id) { 572 if (ep->device_id == device_id) 573 return (ep); 574 ++ep; 575 } 576 return (NULL); 577} 578 579static int 580ntb_initialize_hw(struct ntb_softc *ntb) 581{ 582 583 if (ntb->type == NTB_SOC) 584 return (ntb_setup_soc(ntb)); 585 else 586 return (ntb_setup_xeon(ntb)); 587} 588 589static int 590ntb_setup_xeon(struct ntb_softc *ntb) 591{ 592 uint8_t val, connection_type; 593 594 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 595 596 connection_type = val & XEON_PPD_CONN_TYPE; 597 switch (connection_type) { 598 case NTB_CONN_B2B: 599 ntb->conn_type = NTB_CONN_B2B; 600 break; 601 case NTB_CONN_CLASSIC: 602 case NTB_CONN_RP: 603 default: 604 device_printf(ntb->device, "Connection type %d not supported\n", 605 connection_type); 606 return (ENXIO); 607 } 608 609 if ((val & XEON_PPD_DEV_TYPE) != 0) 610 ntb->dev_type = NTB_DEV_DSD; 611 else 612 ntb->dev_type = NTB_DEV_USD; 613 614 ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET; 615 ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET; 616 ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET; 617 ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET; 618 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 619 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 620 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 621 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 622 623 if (ntb->conn_type == NTB_CONN_B2B) { 624 ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET; 625 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 626 ntb->limits.max_spads = XEON_MAX_SPADS; 627 } else { 628 ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET; 629 ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET; 630 ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS; 631 } 632 633 ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 634 ntb->limits.msix_cnt = XEON_MSIX_CNT; 635 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 636 637 /* Enable Bus Master and Memory Space on the secondary side */ 638 ntb_write_2(ntb->reg_ofs.spci_cmd, 639 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 640 641 return (0); 642} 643 644static int 645ntb_setup_soc(struct ntb_softc *ntb) 646{ 647 uint32_t val, connection_type; 648 649 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 650 651 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 652 switch (connection_type) { 653 case NTB_CONN_B2B: 654 ntb->conn_type = NTB_CONN_B2B; 655 break; 656 case NTB_CONN_RP: 657 default: 658 device_printf(ntb->device, "Connection type %d not supported\n", 659 connection_type); 660 return (ENXIO); 661 } 662 663 if ((val & SOC_PPD_DEV_TYPE) != 0) 664 ntb->dev_type = NTB_DEV_DSD; 665 else 666 ntb->dev_type = NTB_DEV_USD; 667 668 /* Initiate PCI-E link training */ 669 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 670 4); 671 672 ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET; 673 ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET; 674 ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET; 675 ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET; 676 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 677 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 678 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 679 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 680 681 if (ntb->conn_type == NTB_CONN_B2B) { 682 ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET; 683 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 684 ntb->limits.max_spads = SOC_MAX_SPADS; 685 } else { 686 ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET; 687 ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET; 688 ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS; 689 } 690 691 ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 692 ntb->limits.msix_cnt = SOC_MSIX_CNT; 693 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 694 695 /* 696 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 697 * resolved. Mask transaction layer internal parity errors. 698 */ 699 pci_write_config(ntb->device, 0xFC, 0x4, 4); 700 701 /* 702 * Some BIOSes aren't filling out the XLAT offsets. 703 * Check and correct the issue. 704 */ 705 if (ntb->dev_type == NTB_DEV_USD) { 706 if (ntb_read_8(SOC_PBAR2XLAT_OFFSET) == 0) 707 ntb_write_8(SOC_PBAR2XLAT_OFFSET, 708 SOC_PBAR2XLAT_USD_ADDR); 709 710 if (ntb_read_8(SOC_PBAR4XLAT_OFFSET) == 0) 711 ntb_write_8(SOC_PBAR4XLAT_OFFSET, 712 SOC_PBAR4XLAT_USD_ADDR); 713 714 if (ntb_read_8(SOC_MBAR23_OFFSET) == 0xC) 715 ntb_write_8(SOC_MBAR23_OFFSET, SOC_MBAR23_USD_ADDR); 716 717 if (ntb_read_8(SOC_MBAR45_OFFSET) == 0xC) 718 ntb_write_8(SOC_MBAR45_OFFSET, SOC_MBAR45_USD_ADDR); 719 } else { 720 if (ntb_read_8(SOC_PBAR2XLAT_OFFSET) == 0) 721 ntb_write_8(SOC_PBAR2XLAT_OFFSET, 722 SOC_PBAR2XLAT_DSD_ADDR); 723 724 if (ntb_read_8(SOC_PBAR4XLAT_OFFSET) == 0) 725 ntb_write_8(SOC_PBAR4XLAT_OFFSET, 726 SOC_PBAR4XLAT_DSD_ADDR); 727 728 if (ntb_read_8(SOC_MBAR23_OFFSET) == 0xC) 729 ntb_write_8(SOC_MBAR23_OFFSET, SOC_MBAR23_DSD_ADDR); 730 731 if (ntb_read_8(SOC_MBAR45_OFFSET) == 0xC) 732 ntb_write_8(SOC_MBAR45_OFFSET, SOC_MBAR45_DSD_ADDR); 733 } 734 735 /* Enable Bus Master and Memory Space on the secondary side */ 736 ntb_write_2(ntb->reg_ofs.spci_cmd, 737 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 738 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 739 740 return (0); 741} 742 743/* SOC doesn't have link status interrupt, poll on that platform */ 744static void 745ntb_handle_heartbeat(void *arg) 746{ 747 struct ntb_softc *ntb = arg; 748 uint32_t status32; 749 int rc = ntb_check_link_status(ntb); 750 751 if (rc != 0) 752 device_printf(ntb->device, 753 "Error determining link status\n"); 754 /* Check to see if a link error is the cause of the link down */ 755 if (ntb->link_status == NTB_LINK_DOWN) { 756 status32 = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET); 757 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 758 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 759 ntb); 760 return; 761 } 762 } 763 764 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 765 ntb_handle_heartbeat, ntb); 766} 767 768static void 769soc_perform_link_restart(struct ntb_softc *ntb) 770{ 771 uint32_t status; 772 773 /* Driver resets the NTB ModPhy lanes - magic! */ 774 ntb_write_1(SOC_MODPHY_PCSREG6, 0xe0); 775 ntb_write_1(SOC_MODPHY_PCSREG4, 0x40); 776 ntb_write_1(SOC_MODPHY_PCSREG4, 0x60); 777 ntb_write_1(SOC_MODPHY_PCSREG6, 0x60); 778 779 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 780 pause("ModPhy", hz / 10); 781 782 /* Clear AER Errors, write to clear */ 783 status = ntb_read_4(SOC_ERRCORSTS_OFFSET); 784 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 785 ntb_write_4(SOC_ERRCORSTS_OFFSET, status); 786 787 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 788 status = ntb_read_4(SOC_LTSSMERRSTS0_OFFSET); 789 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 790 ntb_write_4(SOC_LTSSMERRSTS0_OFFSET, status); 791 792 /* Clear DeSkew Buffer error, write to clear */ 793 status = ntb_read_4(SOC_DESKEWSTS_OFFSET); 794 status |= SOC_DESKEWSTS_DBERR; 795 ntb_write_4(SOC_DESKEWSTS_OFFSET, status); 796 797 status = ntb_read_4(SOC_IBSTERRRCRVSTS0_OFFSET); 798 status &= SOC_IBIST_ERR_OFLOW; 799 ntb_write_4(SOC_IBSTERRRCRVSTS0_OFFSET, status); 800 801 /* Releases the NTB state machine to allow the link to retrain */ 802 status = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET); 803 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 804 ntb_write_4(SOC_LTSSMSTATEJMP_OFFSET, status); 805} 806 807static void 808ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 809{ 810 enum ntb_hw_event event; 811 uint16_t status; 812 813 if (ntb->link_status == link_state) 814 return; 815 816 if (link_state == NTB_LINK_UP) { 817 device_printf(ntb->device, "Link Up\n"); 818 ntb->link_status = NTB_LINK_UP; 819 event = NTB_EVENT_HW_LINK_UP; 820 821 if (ntb->type == NTB_SOC) 822 status = ntb_read_2(ntb->reg_ofs.lnk_stat); 823 else 824 status = pci_read_config(ntb->device, 825 XEON_LINK_STATUS_OFFSET, 2); 826 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 827 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 828 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 829 ntb->link_width, ntb->link_speed); 830 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 831 ntb_handle_heartbeat, ntb); 832 } else { 833 device_printf(ntb->device, "Link Down\n"); 834 ntb->link_status = NTB_LINK_DOWN; 835 event = NTB_EVENT_HW_LINK_DOWN; 836 /* Don't modify link width/speed, we need it in link recovery */ 837 } 838 839 /* notify the upper layer if we have an event change */ 840 if (ntb->event_cb != NULL) 841 ntb->event_cb(ntb->ntb_transport, event); 842} 843 844static void 845recover_soc_link(void *arg) 846{ 847 struct ntb_softc *ntb = arg; 848 uint8_t speed, width; 849 uint32_t status32; 850 uint16_t status16; 851 852 soc_perform_link_restart(ntb); 853 pause("Link", SOC_LINK_RECOVERY_TIME * hz / 1000); 854 855 status32 = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET); 856 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 857 goto retry; 858 859 status32 = ntb_read_4(SOC_IBSTERRRCRVSTS0_OFFSET); 860 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 861 goto retry; 862 863 status16 = ntb_read_2(ntb->reg_ofs.lnk_stat); 864 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 865 speed = (status16 & NTB_LINK_SPEED_MASK); 866 if (ntb->link_width != width || ntb->link_speed != speed) 867 goto retry; 868 869 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 870 ntb_handle_heartbeat, ntb); 871 return; 872 873retry: 874 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 875 ntb); 876} 877 878static int 879ntb_check_link_status(struct ntb_softc *ntb) 880{ 881 int link_state; 882 uint32_t ntb_cntl; 883 uint16_t status; 884 885 if (ntb->type == NTB_SOC) { 886 ntb_cntl = ntb_read_4(ntb->reg_ofs.lnk_cntl); 887 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 888 link_state = NTB_LINK_DOWN; 889 else 890 link_state = NTB_LINK_UP; 891 } else { 892 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 893 2); 894 895 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 896 link_state = NTB_LINK_UP; 897 else 898 link_state = NTB_LINK_DOWN; 899 } 900 901 ntb_handle_link_event(ntb, link_state); 902 903 return (0); 904} 905 906/** 907 * ntb_register_event_callback() - register event callback 908 * @ntb: pointer to ntb_softc instance 909 * @func: callback function to register 910 * 911 * This function registers a callback for any HW driver events such as link 912 * up/down, power management notices and etc. 913 * 914 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 915 */ 916int 917ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 918{ 919 920 if (ntb->event_cb != NULL) 921 return (EINVAL); 922 923 ntb->event_cb = func; 924 925 return (0); 926} 927 928/** 929 * ntb_unregister_event_callback() - unregisters the event callback 930 * @ntb: pointer to ntb_softc instance 931 * 932 * This function unregisters the existing callback from transport 933 */ 934void 935ntb_unregister_event_callback(struct ntb_softc *ntb) 936{ 937 938 ntb->event_cb = NULL; 939} 940 941/** 942 * ntb_register_db_callback() - register a callback for doorbell interrupt 943 * @ntb: pointer to ntb_softc instance 944 * @idx: doorbell index to register callback, zero based 945 * @func: callback function to register 946 * 947 * This function registers a callback function for the doorbell interrupt 948 * on the primary side. The function will unmask the doorbell as well to 949 * allow interrupt. 950 * 951 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 952 */ 953int 954ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 955 ntb_db_callback func) 956{ 957 uint16_t mask; 958 959 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 960 device_printf(ntb->device, "Invalid Index.\n"); 961 return (EINVAL); 962 } 963 964 ntb->db_cb[idx].callback = func; 965 ntb->db_cb[idx].data = data; 966 967 /* unmask interrupt */ 968 mask = ntb_read_2(ntb->reg_ofs.pdb_mask); 969 mask &= ~(1 << (idx * ntb->bits_per_vector)); 970 ntb_write_2(ntb->reg_ofs.pdb_mask, mask); 971 972 return (0); 973} 974 975/** 976 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 977 * @ntb: pointer to ntb_softc instance 978 * @idx: doorbell index to register callback, zero based 979 * 980 * This function unregisters a callback function for the doorbell interrupt 981 * on the primary side. The function will also mask the said doorbell. 982 */ 983void 984ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 985{ 986 unsigned long mask; 987 988 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 989 return; 990 991 mask = ntb_read_2(ntb->reg_ofs.pdb_mask); 992 mask |= 1 << (idx * ntb->bits_per_vector); 993 ntb_write_2(ntb->reg_ofs.pdb_mask, mask); 994 995 ntb->db_cb[idx].callback = NULL; 996} 997 998/** 999 * ntb_find_transport() - find the transport pointer 1000 * @transport: pointer to pci device 1001 * 1002 * Given the pci device pointer, return the transport pointer passed in when 1003 * the transport attached when it was inited. 1004 * 1005 * RETURNS: pointer to transport. 1006 */ 1007void * 1008ntb_find_transport(struct ntb_softc *ntb) 1009{ 1010 1011 return (ntb->ntb_transport); 1012} 1013 1014/** 1015 * ntb_register_transport() - Register NTB transport with NTB HW driver 1016 * @transport: transport identifier 1017 * 1018 * This function allows a transport to reserve the hardware driver for 1019 * NTB usage. 1020 * 1021 * RETURNS: pointer to ntb_softc, NULL on error. 1022 */ 1023struct ntb_softc * 1024ntb_register_transport(struct ntb_softc *ntb, void *transport) 1025{ 1026 1027 /* 1028 * TODO: when we have more than one transport, we will need to rewrite 1029 * this to prevent race conditions 1030 */ 1031 if (ntb->ntb_transport != NULL) 1032 return (NULL); 1033 1034 ntb->ntb_transport = transport; 1035 return (ntb); 1036} 1037 1038/** 1039 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1040 * @ntb - ntb_softc of the transport to be freed 1041 * 1042 * This function unregisters the transport from the HW driver and performs any 1043 * necessary cleanups. 1044 */ 1045void 1046ntb_unregister_transport(struct ntb_softc *ntb) 1047{ 1048 int i; 1049 1050 if (ntb->ntb_transport == NULL) 1051 return; 1052 1053 for (i = 0; i < ntb->allocated_interrupts; i++) 1054 ntb_unregister_db_callback(ntb, i); 1055 1056 ntb_unregister_event_callback(ntb); 1057 ntb->ntb_transport = NULL; 1058} 1059 1060/** 1061 * ntb_get_max_spads() - get the total scratch regs usable 1062 * @ntb: pointer to ntb_softc instance 1063 * 1064 * This function returns the max 32bit scratchpad registers usable by the 1065 * upper layer. 1066 * 1067 * RETURNS: total number of scratch pad registers available 1068 */ 1069int 1070ntb_get_max_spads(struct ntb_softc *ntb) 1071{ 1072 1073 return (ntb->limits.max_spads); 1074} 1075 1076/** 1077 * ntb_write_local_spad() - write to the secondary scratchpad register 1078 * @ntb: pointer to ntb_softc instance 1079 * @idx: index to the scratchpad register, 0 based 1080 * @val: the data value to put into the register 1081 * 1082 * This function allows writing of a 32bit value to the indexed scratchpad 1083 * register. The register resides on the secondary (external) side. 1084 * 1085 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1086 */ 1087int 1088ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1089{ 1090 1091 if (idx >= ntb->limits.max_spads) 1092 return (EINVAL); 1093 1094 ntb_write_4(ntb->reg_ofs.spad_local + idx * 4, val); 1095 1096 return (0); 1097} 1098 1099/** 1100 * ntb_read_local_spad() - read from the primary scratchpad register 1101 * @ntb: pointer to ntb_softc instance 1102 * @idx: index to scratchpad register, 0 based 1103 * @val: pointer to 32bit integer for storing the register value 1104 * 1105 * This function allows reading of the 32bit scratchpad register on 1106 * the primary (internal) side. 1107 * 1108 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1109 */ 1110int 1111ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1112{ 1113 1114 if (idx >= ntb->limits.max_spads) 1115 return (EINVAL); 1116 1117 *val = ntb_read_4(ntb->reg_ofs.spad_local + idx * 4); 1118 1119 return (0); 1120} 1121 1122/** 1123 * ntb_write_remote_spad() - write to the secondary scratchpad register 1124 * @ntb: pointer to ntb_softc instance 1125 * @idx: index to the scratchpad register, 0 based 1126 * @val: the data value to put into the register 1127 * 1128 * This function allows writing of a 32bit value to the indexed scratchpad 1129 * register. The register resides on the secondary (external) side. 1130 * 1131 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1132 */ 1133int 1134ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1135{ 1136 1137 if (idx >= ntb->limits.max_spads) 1138 return (EINVAL); 1139 1140 ntb_write_4(ntb->reg_ofs.spad_remote + idx * 4, val); 1141 1142 return (0); 1143} 1144 1145/** 1146 * ntb_read_remote_spad() - read from the primary scratchpad register 1147 * @ntb: pointer to ntb_softc instance 1148 * @idx: index to scratchpad register, 0 based 1149 * @val: pointer to 32bit integer for storing the register value 1150 * 1151 * This function allows reading of the 32bit scratchpad register on 1152 * the primary (internal) side. 1153 * 1154 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1155 */ 1156int 1157ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1158{ 1159 1160 if (idx >= ntb->limits.max_spads) 1161 return (EINVAL); 1162 1163 *val = ntb_read_4(ntb->reg_ofs.spad_remote + idx * 4); 1164 1165 return (0); 1166} 1167 1168/** 1169 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1170 * @ntb: pointer to ntb_softc instance 1171 * @mw: memory window number 1172 * 1173 * This function provides the base virtual address of the memory window 1174 * specified. 1175 * 1176 * RETURNS: pointer to virtual address, or NULL on error. 1177 */ 1178void * 1179ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1180{ 1181 1182 if (mw >= NTB_NUM_MW) 1183 return (NULL); 1184 1185 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1186} 1187 1188vm_paddr_t 1189ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1190{ 1191 1192 if (mw >= NTB_NUM_MW) 1193 return (0); 1194 1195 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1196} 1197 1198/** 1199 * ntb_get_mw_size() - return size of NTB memory window 1200 * @ntb: pointer to ntb_softc instance 1201 * @mw: memory window number 1202 * 1203 * This function provides the physical size of the memory window specified 1204 * 1205 * RETURNS: the size of the memory window or zero on error 1206 */ 1207u_long 1208ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1209{ 1210 1211 if (mw >= NTB_NUM_MW) 1212 return (0); 1213 1214 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1215} 1216 1217/** 1218 * ntb_set_mw_addr - set the memory window address 1219 * @ntb: pointer to ntb_softc instance 1220 * @mw: memory window number 1221 * @addr: base address for data 1222 * 1223 * This function sets the base physical address of the memory window. This 1224 * memory address is where data from the remote system will be transfered into 1225 * or out of depending on how the transport is configured. 1226 */ 1227void 1228ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1229{ 1230 1231 if (mw >= NTB_NUM_MW) 1232 return; 1233 1234 switch (NTB_MW_TO_BAR(mw)) { 1235 case NTB_B2B_BAR_1: 1236 ntb_write_8(ntb->reg_ofs.sbar2_xlat, addr); 1237 break; 1238 case NTB_B2B_BAR_2: 1239 ntb_write_8(ntb->reg_ofs.sbar4_xlat, addr); 1240 break; 1241 } 1242} 1243 1244/** 1245 * ntb_ring_sdb() - Set the doorbell on the secondary/external side 1246 * @ntb: pointer to ntb_softc instance 1247 * @db: doorbell to ring 1248 * 1249 * This function allows triggering of a doorbell on the secondary/external 1250 * side that will initiate an interrupt on the remote host 1251 * 1252 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1253 */ 1254void 1255ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db) 1256{ 1257 1258 if (ntb->type == NTB_SOC) 1259 ntb_write_8(ntb->reg_ofs.sdb, (uint64_t) 1 << db); 1260 else 1261 ntb_write_2(ntb->reg_ofs.sdb, 1262 ((1 << ntb->bits_per_vector) - 1) << 1263 (db * ntb->bits_per_vector)); 1264} 1265 1266/** 1267 * ntb_query_link_status() - return the hardware link status 1268 * @ndev: pointer to ntb_device instance 1269 * 1270 * Returns true if the hardware is connected to the remote system 1271 * 1272 * RETURNS: true or false based on the hardware link state 1273 */ 1274bool 1275ntb_query_link_status(struct ntb_softc *ntb) 1276{ 1277 1278 return (ntb->link_status == NTB_LINK_UP); 1279} 1280 1281static bool 1282is_bar_for_data_transfer(int bar_num) 1283{ 1284 if ((bar_num > NTB_CONFIG_BAR) && (bar_num < NTB_MAX_BARS)) 1285 return true; 1286 else 1287 return false; 1288} 1289 1290device_t ntb_get_device(struct ntb_softc *ntb) 1291{ 1292 1293 return (ntb->device); 1294} 1295