ntb_hw_intel.c revision 283291
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3250079Scarl * All rights reserved. 4250079Scarl * 5250079Scarl * Redistribution and use in source and binary forms, with or without 6250079Scarl * modification, are permitted provided that the following conditions 7250079Scarl * are met: 8250079Scarl * 1. Redistributions of source code must retain the above copyright 9250079Scarl * notice, this list of conditions and the following disclaimer. 10250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer in the 12250079Scarl * documentation and/or other materials provided with the distribution. 13250079Scarl * 14250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24250079Scarl * SUCH DAMAGE. 25250079Scarl */ 26250079Scarl 27250079Scarl#include <sys/cdefs.h> 28250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 283291 2015-05-22 17:05:21Z jkim $"); 29250079Scarl 30250079Scarl#include <sys/param.h> 31250079Scarl#include <sys/kernel.h> 32250079Scarl#include <sys/systm.h> 33250079Scarl#include <sys/bus.h> 34250079Scarl#include <sys/malloc.h> 35250079Scarl#include <sys/module.h> 36250079Scarl#include <sys/queue.h> 37250079Scarl#include <sys/rman.h> 38250079Scarl#include <vm/vm.h> 39250079Scarl#include <vm/pmap.h> 40250079Scarl#include <machine/bus.h> 41250079Scarl#include <machine/pmap.h> 42250079Scarl#include <machine/resource.h> 43250079Scarl#include <dev/pci/pcireg.h> 44250079Scarl#include <dev/pci/pcivar.h> 45250079Scarl 46250079Scarl#include "ntb_regs.h" 47250079Scarl#include "ntb_hw.h" 48250079Scarl 49250079Scarl/* 50250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 51250079Scarl * allows you to connect two systems using a PCI-e link. 52250079Scarl * 53250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 54250079Scarl * you to send and recieve interrupts, map the memory windows and send and 55250079Scarl * receive messages in the scratch-pad registers. 56250079Scarl * 57250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 58250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 59250079Scarl */ 60250079Scarl 61250079Scarl#define NTB_CONFIG_BAR 0 62250079Scarl#define NTB_B2B_BAR_1 1 63250079Scarl#define NTB_B2B_BAR_2 2 64250079Scarl#define NTB_MAX_BARS 3 65250079Scarl#define NTB_MW_TO_BAR(mw) ((mw) + 1) 66250079Scarl 67250079Scarl#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 68250079Scarl 69250079Scarl#define NTB_HB_TIMEOUT 1 /* second */ 70250079Scarl#define SOC_LINK_RECOVERY_TIME 500 71250079Scarl 72250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 73250079Scarl 74250079Scarlenum ntb_device_type { 75250079Scarl NTB_XEON, 76250079Scarl NTB_SOC 77250079Scarl}; 78250079Scarl 79255274Scarl/* Device features and workarounds */ 80255274Scarl#define HAS_FEATURE(feature) \ 81255274Scarl ((ntb->features & (feature)) != 0) 82255274Scarl 83255274Scarl#define NTB_BAR_SIZE_4K (1 << 0) 84255274Scarl#define NTB_REGS_THRU_MW (1 << 1) 85255274Scarl 86250079Scarlstruct ntb_hw_info { 87250079Scarl uint32_t device_id; 88255274Scarl const char *desc; 89250079Scarl enum ntb_device_type type; 90255274Scarl uint64_t features; 91250079Scarl}; 92250079Scarl 93250079Scarlstruct ntb_pci_bar_info { 94250079Scarl bus_space_tag_t pci_bus_tag; 95250079Scarl bus_space_handle_t pci_bus_handle; 96250079Scarl int pci_resource_id; 97250079Scarl struct resource *pci_resource; 98250079Scarl vm_paddr_t pbase; 99250079Scarl void *vbase; 100250079Scarl u_long size; 101250079Scarl}; 102250079Scarl 103250079Scarlstruct ntb_int_info { 104250079Scarl struct resource *res; 105250079Scarl int rid; 106250079Scarl void *tag; 107250079Scarl}; 108250079Scarl 109250079Scarlstruct ntb_db_cb { 110250079Scarl ntb_db_callback callback; 111250079Scarl unsigned int db_num; 112250079Scarl void *data; 113250079Scarl struct ntb_softc *ntb; 114250079Scarl}; 115250079Scarl 116250079Scarlstruct ntb_softc { 117250079Scarl device_t device; 118250079Scarl enum ntb_device_type type; 119255274Scarl uint64_t features; 120250079Scarl 121250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 122250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 123250079Scarl uint32_t allocated_interrupts; 124250079Scarl 125250079Scarl struct callout heartbeat_timer; 126250079Scarl struct callout lr_timer; 127250079Scarl 128250079Scarl void *ntb_transport; 129250079Scarl ntb_event_callback event_cb; 130250079Scarl struct ntb_db_cb *db_cb; 131250079Scarl 132250079Scarl struct { 133250079Scarl uint32_t max_spads; 134250079Scarl uint32_t max_db_bits; 135250079Scarl uint32_t msix_cnt; 136250079Scarl } limits; 137250079Scarl struct { 138250079Scarl uint32_t pdb; 139250079Scarl uint32_t pdb_mask; 140250079Scarl uint32_t sdb; 141250079Scarl uint32_t sbar2_xlat; 142250079Scarl uint32_t sbar4_xlat; 143250079Scarl uint32_t spad_remote; 144250079Scarl uint32_t spad_local; 145250079Scarl uint32_t lnk_cntl; 146250079Scarl uint32_t lnk_stat; 147250079Scarl uint32_t spci_cmd; 148250079Scarl } reg_ofs; 149250079Scarl uint8_t conn_type; 150250079Scarl uint8_t dev_type; 151250079Scarl uint8_t bits_per_vector; 152250079Scarl uint8_t link_status; 153250079Scarl uint8_t link_width; 154250079Scarl uint8_t link_speed; 155250079Scarl}; 156250079Scarl 157255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 158255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 159255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 160255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 161255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 162255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 163255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 164250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 165255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 166255279Scarl#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 167255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 168255279Scarl ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 169250079Scarl 170255272Scarltypedef int (*bar_map_strategy)(struct ntb_softc *ntb, 171255272Scarl struct ntb_pci_bar_info *bar); 172255272Scarl 173250079Scarlstatic int ntb_probe(device_t device); 174250079Scarlstatic int ntb_attach(device_t device); 175250079Scarlstatic int ntb_detach(device_t device); 176255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 177255272Scarlstatic int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 178255272Scarl struct ntb_pci_bar_info *bar); 179255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 180255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 181255272Scarl struct ntb_pci_bar_info *bar); 182250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 183250079Scarlstatic int ntb_setup_interrupts(struct ntb_softc *ntb); 184250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 185250079Scarlstatic void handle_soc_irq(void *arg); 186250079Scarlstatic void handle_xeon_irq(void *arg); 187250079Scarlstatic void handle_xeon_event_irq(void *arg); 188250079Scarlstatic void ntb_handle_legacy_interrupt(void *arg); 189250079Scarlstatic int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 190250079Scarlstatic void ntb_free_callbacks(struct ntb_softc *ntb); 191250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 192250079Scarlstatic int ntb_initialize_hw(struct ntb_softc *ntb); 193250079Scarlstatic int ntb_setup_xeon(struct ntb_softc *ntb); 194250079Scarlstatic int ntb_setup_soc(struct ntb_softc *ntb); 195255279Scarlstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 196255279Scarlstatic void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 197250079Scarlstatic void ntb_handle_heartbeat(void *arg); 198250079Scarlstatic void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 199250079Scarlstatic void recover_soc_link(void *arg); 200250079Scarlstatic int ntb_check_link_status(struct ntb_softc *ntb); 201255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 202250079Scarl 203250079Scarlstatic struct ntb_hw_info pci_ids[] = { 204255274Scarl { 0x3C0D8086, "Xeon E5/Core i7 Non-Transparent Bridge B2B", NTB_XEON, 205255274Scarl NTB_REGS_THRU_MW }, 206255274Scarl { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 207255274Scarl { 0x0E0D8086, "Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 208255274Scarl NTB_REGS_THRU_MW | NTB_BAR_SIZE_4K }, 209255274Scarl { 0x00000000, NULL, NTB_SOC, 0 } 210250079Scarl}; 211250079Scarl 212250079Scarl/* 213250079Scarl * OS <-> Driver interface structures 214250079Scarl */ 215250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 216250079Scarl 217250079Scarlstatic device_method_t ntb_pci_methods[] = { 218250079Scarl /* Device interface */ 219250079Scarl DEVMETHOD(device_probe, ntb_probe), 220250079Scarl DEVMETHOD(device_attach, ntb_attach), 221250079Scarl DEVMETHOD(device_detach, ntb_detach), 222250079Scarl DEVMETHOD_END 223250079Scarl}; 224250079Scarl 225250079Scarlstatic driver_t ntb_pci_driver = { 226250079Scarl "ntb_hw", 227250079Scarl ntb_pci_methods, 228250079Scarl sizeof(struct ntb_softc), 229250079Scarl}; 230250079Scarl 231250079Scarlstatic devclass_t ntb_devclass; 232250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 233250079ScarlMODULE_VERSION(ntb_hw, 1); 234250079Scarl 235250079Scarl/* 236250079Scarl * OS <-> Driver linkage functions 237250079Scarl */ 238250079Scarlstatic int 239250079Scarlntb_probe(device_t device) 240250079Scarl{ 241250079Scarl struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device)); 242250079Scarl 243250079Scarl if (p != NULL) { 244250079Scarl device_set_desc(device, p->desc); 245250079Scarl return (0); 246250079Scarl } else 247250079Scarl return (ENXIO); 248250079Scarl} 249250079Scarl 250250079Scarl#define DETACH_ON_ERROR(func) \ 251250079Scarl error = func; \ 252250079Scarl if (error < 0) { \ 253250079Scarl ntb_detach(device); \ 254250079Scarl return (error); \ 255250079Scarl } 256250079Scarl 257250079Scarlstatic int 258250079Scarlntb_attach(device_t device) 259250079Scarl{ 260250079Scarl struct ntb_softc *ntb = DEVICE2SOFTC(device); 261250079Scarl struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device)); 262250079Scarl int error; 263250079Scarl 264250079Scarl ntb->device = device; 265250079Scarl ntb->type = p->type; 266255274Scarl ntb->features = p->features; 267250079Scarl 268250079Scarl /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 269283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 270283291Sjkim callout_init(&ntb->lr_timer, 1); 271250079Scarl 272255272Scarl DETACH_ON_ERROR(ntb_map_pci_bars(ntb)); 273250079Scarl DETACH_ON_ERROR(ntb_initialize_hw(ntb)); 274250079Scarl DETACH_ON_ERROR(ntb_setup_interrupts(ntb)); 275250079Scarl 276250079Scarl pci_enable_busmaster(ntb->device); 277250079Scarl 278250079Scarl return (error); 279250079Scarl} 280250079Scarl 281250079Scarlstatic int 282250079Scarlntb_detach(device_t device) 283250079Scarl{ 284250079Scarl struct ntb_softc *ntb = DEVICE2SOFTC(device); 285250079Scarl 286250079Scarl callout_drain(&ntb->heartbeat_timer); 287250079Scarl callout_drain(&ntb->lr_timer); 288250079Scarl ntb_teardown_interrupts(ntb); 289250079Scarl ntb_unmap_pci_bar(ntb); 290250079Scarl 291250079Scarl return (0); 292250079Scarl} 293250079Scarl 294250079Scarlstatic int 295255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 296250079Scarl{ 297255272Scarl int rc; 298250079Scarl 299250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 300255272Scarl rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 301255272Scarl if (rc != 0) 302255272Scarl return rc; 303255272Scarl 304250079Scarl ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 305255272Scarl rc = map_pci_bar(ntb, map_memory_window_bar, 306255272Scarl &ntb->bar_info[NTB_B2B_BAR_1]); 307255272Scarl if (rc != 0) 308255272Scarl return rc; 309255272Scarl 310250079Scarl ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 311255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 312255279Scarl rc = map_pci_bar(ntb, map_mmr_bar, 313255279Scarl &ntb->bar_info[NTB_B2B_BAR_2]); 314255279Scarl else 315255279Scarl rc = map_pci_bar(ntb, map_memory_window_bar, 316255279Scarl &ntb->bar_info[NTB_B2B_BAR_2]); 317255272Scarl if (rc != 0) 318255272Scarl return rc; 319255274Scarl 320255272Scarl return (0); 321255272Scarl} 322250079Scarl 323255272Scarlstatic int 324255272Scarlmap_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 325255272Scarl struct ntb_pci_bar_info *bar) 326255272Scarl{ 327255272Scarl int rc; 328255272Scarl 329255272Scarl rc = strategy(ntb, bar); 330255272Scarl if (rc != 0) { 331255272Scarl device_printf(ntb->device, 332255272Scarl "unable to allocate pci resource\n"); 333255272Scarl } else { 334255279Scarl device_printf(ntb->device, 335255272Scarl "Bar size = %lx, v %p, p %p\n", 336255272Scarl bar->size, bar->vbase, 337255272Scarl (void *)(bar->pbase)); 338255272Scarl } 339255272Scarl return (rc); 340255272Scarl} 341255272Scarl 342255272Scarlstatic int 343255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 344255272Scarl{ 345255272Scarl 346255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 347255275Scarl &bar->pci_resource_id, RF_ACTIVE); 348255272Scarl 349255272Scarl if (bar->pci_resource == NULL) 350255272Scarl return (ENXIO); 351255272Scarl else { 352255272Scarl save_bar_parameters(bar); 353255272Scarl return (0); 354255272Scarl } 355255272Scarl} 356255272Scarl 357255272Scarlstatic int 358255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 359255272Scarl{ 360255272Scarl int rc; 361255276Scarl uint8_t bar_size_bits = 0; 362255272Scarl 363255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, 364255275Scarl SYS_RES_MEMORY, &bar->pci_resource_id, RF_ACTIVE); 365250079Scarl 366255272Scarl if (bar->pci_resource == NULL) 367255272Scarl return (ENXIO); 368255272Scarl else { 369255272Scarl save_bar_parameters(bar); 370255276Scarl /* 371255276Scarl * Ivytown NTB BAR sizes are misreported by the hardware due to 372255276Scarl * a hardware issue. To work around this, query the size it 373255276Scarl * should be configured to by the device and modify the resource 374255276Scarl * to correspond to this new size. The BIOS on systems with this 375255276Scarl * problem is required to provide enough address space to allow 376255276Scarl * the driver to make this change safely. 377255276Scarl * 378255276Scarl * Ideally I could have just specified the size when I allocated 379255276Scarl * the resource like: 380255276Scarl * bus_alloc_resource(ntb->device, 381255276Scarl * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 382255276Scarl * 1ul << bar_size_bits, RF_ACTIVE); 383255276Scarl * but the PCI driver does not honor the size in this call, so 384255276Scarl * we have to modify it after the fact. 385255276Scarl */ 386255279Scarl if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 387255276Scarl if (bar->pci_resource_id == PCIR_BAR(2)) 388255276Scarl bar_size_bits = pci_read_config(ntb->device, 389255276Scarl XEON_PBAR23SZ_OFFSET, 1); 390255276Scarl else 391255276Scarl bar_size_bits = pci_read_config(ntb->device, 392255276Scarl XEON_PBAR45SZ_OFFSET, 1); 393255276Scarl rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 394255276Scarl bar->pci_resource, bar->pbase, 395255276Scarl bar->pbase + (1ul << bar_size_bits) - 1); 396255276Scarl if (rc != 0 ) { 397255276Scarl device_printf(ntb->device, 398255276Scarl "unable to resize bar\n"); 399255276Scarl return (rc); 400255276Scarl } else 401255276Scarl save_bar_parameters(bar); 402255276Scarl } 403255276Scarl 404255272Scarl /* Mark bar region as write combining to improve performance. */ 405255272Scarl rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 406255272Scarl VM_MEMATTR_WRITE_COMBINING); 407255272Scarl if (rc != 0) { 408255272Scarl device_printf(ntb->device, "unable to mark bar as" 409255272Scarl " WRITE_COMBINING\n"); 410255272Scarl return (rc); 411250079Scarl } 412250079Scarl } 413250079Scarl return (0); 414250079Scarl} 415250079Scarl 416250079Scarlstatic void 417250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 418250079Scarl{ 419250079Scarl struct ntb_pci_bar_info *current_bar; 420250079Scarl int i; 421250079Scarl 422250079Scarl for (i = 0; i< NTB_MAX_BARS; i++) { 423250079Scarl current_bar = &ntb->bar_info[i]; 424250079Scarl if (current_bar->pci_resource != NULL) 425250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 426250079Scarl current_bar->pci_resource_id, 427250079Scarl current_bar->pci_resource); 428250079Scarl } 429250079Scarl} 430250079Scarl 431250079Scarlstatic int 432250079Scarlntb_setup_interrupts(struct ntb_softc *ntb) 433250079Scarl{ 434250079Scarl void (*interrupt_handler)(void *); 435250079Scarl void *int_arg; 436250079Scarl bool use_msix = 0; 437250079Scarl uint32_t num_vectors; 438250079Scarl int i; 439250079Scarl 440250079Scarl ntb->allocated_interrupts = 0; 441250079Scarl /* 442250079Scarl * On SOC, disable all interrupts. On XEON, disable all but Link 443250079Scarl * Interrupt. The rest will be unmasked as callbacks are registered. 444250079Scarl */ 445250079Scarl if (ntb->type == NTB_SOC) 446255278Scarl ntb_reg_write(8, ntb->reg_ofs.pdb_mask, ~0); 447250079Scarl else 448255278Scarl ntb_reg_write(2, ntb->reg_ofs.pdb_mask, 449250079Scarl ~(1 << ntb->limits.max_db_bits)); 450250079Scarl 451250079Scarl num_vectors = MIN(pci_msix_count(ntb->device), 452250079Scarl ntb->limits.max_db_bits); 453250079Scarl if (num_vectors >= 1) { 454250079Scarl pci_alloc_msix(ntb->device, &num_vectors); 455250079Scarl if (num_vectors >= 4) 456250079Scarl use_msix = TRUE; 457250079Scarl } 458250079Scarl 459250079Scarl ntb_create_callbacks(ntb, num_vectors); 460250079Scarl if (use_msix == TRUE) { 461250079Scarl for (i = 0; i < num_vectors; i++) { 462250079Scarl ntb->int_info[i].rid = i + 1; 463250079Scarl ntb->int_info[i].res = bus_alloc_resource_any( 464250079Scarl ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 465250079Scarl RF_ACTIVE); 466250079Scarl if (ntb->int_info[i].res == NULL) { 467250079Scarl device_printf(ntb->device, 468250079Scarl "bus_alloc_resource failed\n"); 469250079Scarl return (-1); 470250079Scarl } 471250079Scarl ntb->int_info[i].tag = NULL; 472250079Scarl ntb->allocated_interrupts++; 473250079Scarl if (ntb->type == NTB_SOC) { 474250079Scarl interrupt_handler = handle_soc_irq; 475250079Scarl int_arg = &ntb->db_cb[i]; 476250079Scarl } else { 477250079Scarl if (i == num_vectors - 1) { 478255279Scarl interrupt_handler = 479255279Scarl handle_xeon_event_irq; 480250079Scarl int_arg = ntb; 481250079Scarl } else { 482250079Scarl interrupt_handler = 483250079Scarl handle_xeon_irq; 484250079Scarl int_arg = &ntb->db_cb[i]; 485250079Scarl } 486250079Scarl } 487250079Scarl if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 488250079Scarl INTR_MPSAFE | INTR_TYPE_MISC, NULL, 489250079Scarl interrupt_handler, int_arg, 490250079Scarl &ntb->int_info[i].tag) != 0) { 491250079Scarl device_printf(ntb->device, 492250079Scarl "bus_setup_intr failed\n"); 493250079Scarl return (ENXIO); 494250079Scarl } 495250079Scarl } 496250079Scarl } 497250079Scarl else { 498250079Scarl ntb->int_info[0].rid = 0; 499255279Scarl ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, 500255279Scarl SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 501250079Scarl interrupt_handler = ntb_handle_legacy_interrupt; 502250079Scarl if (ntb->int_info[0].res == NULL) { 503250079Scarl device_printf(ntb->device, 504250079Scarl "bus_alloc_resource failed\n"); 505250079Scarl return (-1); 506250079Scarl } 507250079Scarl ntb->int_info[0].tag = NULL; 508250079Scarl ntb->allocated_interrupts = 1; 509250079Scarl 510250079Scarl if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 511250079Scarl INTR_MPSAFE | INTR_TYPE_MISC, NULL, 512250079Scarl interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 513250079Scarl 514250079Scarl device_printf(ntb->device, "bus_setup_intr failed\n"); 515250079Scarl return (ENXIO); 516250079Scarl } 517250079Scarl } 518250079Scarl 519250079Scarl return (0); 520250079Scarl} 521250079Scarl 522250079Scarlstatic void 523250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 524250079Scarl{ 525250079Scarl struct ntb_int_info *current_int; 526250079Scarl int i; 527250079Scarl 528250079Scarl for (i=0; i<ntb->allocated_interrupts; i++) { 529250079Scarl current_int = &ntb->int_info[i]; 530250079Scarl if (current_int->tag != NULL) 531250079Scarl bus_teardown_intr(ntb->device, current_int->res, 532250079Scarl current_int->tag); 533250079Scarl 534250079Scarl if (current_int->res != NULL) 535250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 536250079Scarl rman_get_rid(current_int->res), current_int->res); 537250079Scarl } 538250079Scarl 539250079Scarl ntb_free_callbacks(ntb); 540250079Scarl pci_release_msi(ntb->device); 541250079Scarl} 542250079Scarl 543250079Scarlstatic void 544250079Scarlhandle_soc_irq(void *arg) 545250079Scarl{ 546250079Scarl struct ntb_db_cb *db_cb = arg; 547250079Scarl struct ntb_softc *ntb = db_cb->ntb; 548250079Scarl 549255278Scarl ntb_reg_write(8, ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num); 550250079Scarl 551250079Scarl if (db_cb->callback != NULL) 552250079Scarl db_cb->callback(db_cb->data, db_cb->db_num); 553250079Scarl} 554250079Scarl 555250079Scarlstatic void 556250079Scarlhandle_xeon_irq(void *arg) 557250079Scarl{ 558250079Scarl struct ntb_db_cb *db_cb = arg; 559250079Scarl struct ntb_softc *ntb = db_cb->ntb; 560250079Scarl 561250079Scarl /* 562250079Scarl * On Xeon, there are 16 bits in the interrupt register 563250079Scarl * but only 4 vectors. So, 5 bits are assigned to the first 3 564250079Scarl * vectors, with the 4th having a single bit for link 565250079Scarl * interrupts. 566250079Scarl */ 567255278Scarl ntb_reg_write(2, ntb->reg_ofs.pdb, 568250079Scarl ((1 << ntb->bits_per_vector) - 1) << 569250079Scarl (db_cb->db_num * ntb->bits_per_vector)); 570250079Scarl 571250079Scarl if (db_cb->callback != NULL) 572250079Scarl db_cb->callback(db_cb->data, db_cb->db_num); 573250079Scarl} 574250079Scarl 575250079Scarl/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 576250079Scarlstatic void 577250079Scarlhandle_xeon_event_irq(void *arg) 578250079Scarl{ 579250079Scarl struct ntb_softc *ntb = arg; 580250079Scarl int rc; 581250079Scarl 582250079Scarl rc = ntb_check_link_status(ntb); 583250079Scarl if (rc != 0) 584250079Scarl device_printf(ntb->device, "Error determining link status\n"); 585250079Scarl 586250079Scarl /* bit 15 is always the link bit */ 587255278Scarl ntb_reg_write(2, ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits); 588250079Scarl} 589250079Scarl 590250079Scarlstatic void 591250079Scarlntb_handle_legacy_interrupt(void *arg) 592250079Scarl{ 593250079Scarl struct ntb_softc *ntb = arg; 594250079Scarl unsigned int i = 0; 595250079Scarl uint64_t pdb64; 596250079Scarl uint16_t pdb16; 597250079Scarl 598250079Scarl if (ntb->type == NTB_SOC) { 599255278Scarl pdb64 = ntb_reg_read(8, ntb->reg_ofs.pdb); 600250079Scarl 601250079Scarl while (pdb64) { 602250079Scarl i = ffs(pdb64); 603250079Scarl pdb64 &= pdb64 - 1; 604250079Scarl handle_soc_irq(&ntb->db_cb[i]); 605250079Scarl } 606250079Scarl } else { 607255278Scarl pdb16 = ntb_reg_read(2, ntb->reg_ofs.pdb); 608250079Scarl 609250079Scarl if ((pdb16 & XEON_DB_HW_LINK) != 0) { 610250079Scarl handle_xeon_event_irq(ntb); 611250079Scarl pdb16 &= ~XEON_DB_HW_LINK; 612250079Scarl } 613250079Scarl 614250079Scarl while (pdb16 != 0) { 615250079Scarl i = ffs(pdb16); 616250079Scarl pdb16 &= pdb16 - 1; 617250079Scarl handle_xeon_irq(&ntb->db_cb[i]); 618250079Scarl } 619250079Scarl } 620250079Scarl 621250079Scarl} 622250079Scarl 623250079Scarlstatic int 624250079Scarlntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 625250079Scarl{ 626250079Scarl int i; 627250079Scarl 628250079Scarl ntb->db_cb = malloc(num_vectors * sizeof(struct ntb_db_cb), M_NTB, 629250079Scarl M_ZERO | M_WAITOK); 630250079Scarl for (i = 0; i < num_vectors; i++) { 631250079Scarl ntb->db_cb[i].db_num = i; 632250079Scarl ntb->db_cb[i].ntb = ntb; 633250079Scarl } 634250079Scarl 635250079Scarl return (0); 636250079Scarl} 637250079Scarl 638250079Scarlstatic void 639250079Scarlntb_free_callbacks(struct ntb_softc *ntb) 640250079Scarl{ 641250079Scarl int i; 642250079Scarl 643250079Scarl for (i = 0; i < ntb->limits.max_db_bits; i++) 644250079Scarl ntb_unregister_db_callback(ntb, i); 645250079Scarl 646250079Scarl free(ntb->db_cb, M_NTB); 647250079Scarl} 648250079Scarl 649250079Scarlstatic struct ntb_hw_info * 650250079Scarlntb_get_device_info(uint32_t device_id) 651250079Scarl{ 652250079Scarl struct ntb_hw_info *ep = pci_ids; 653250079Scarl 654250079Scarl while (ep->device_id) { 655250079Scarl if (ep->device_id == device_id) 656250079Scarl return (ep); 657250079Scarl ++ep; 658250079Scarl } 659250079Scarl return (NULL); 660250079Scarl} 661250079Scarl 662250079Scarlstatic int 663250079Scarlntb_initialize_hw(struct ntb_softc *ntb) 664250079Scarl{ 665250079Scarl 666250079Scarl if (ntb->type == NTB_SOC) 667250079Scarl return (ntb_setup_soc(ntb)); 668250079Scarl else 669250079Scarl return (ntb_setup_xeon(ntb)); 670250079Scarl} 671250079Scarl 672250079Scarlstatic int 673250079Scarlntb_setup_xeon(struct ntb_softc *ntb) 674250079Scarl{ 675250079Scarl uint8_t val, connection_type; 676250079Scarl 677250079Scarl val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 678250079Scarl 679250079Scarl connection_type = val & XEON_PPD_CONN_TYPE; 680250079Scarl switch (connection_type) { 681250079Scarl case NTB_CONN_B2B: 682250079Scarl ntb->conn_type = NTB_CONN_B2B; 683250079Scarl break; 684250079Scarl case NTB_CONN_CLASSIC: 685250079Scarl case NTB_CONN_RP: 686250079Scarl default: 687250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 688250079Scarl connection_type); 689250079Scarl return (ENXIO); 690250079Scarl } 691250079Scarl 692250079Scarl if ((val & XEON_PPD_DEV_TYPE) != 0) 693250079Scarl ntb->dev_type = NTB_DEV_DSD; 694250079Scarl else 695250079Scarl ntb->dev_type = NTB_DEV_USD; 696250079Scarl 697250079Scarl ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET; 698250079Scarl ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET; 699250079Scarl ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET; 700250079Scarl ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET; 701250079Scarl ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 702250079Scarl ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 703250079Scarl ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 704250079Scarl ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 705250079Scarl 706250079Scarl if (ntb->conn_type == NTB_CONN_B2B) { 707250079Scarl ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET; 708250079Scarl ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 709250079Scarl ntb->limits.max_spads = XEON_MAX_SPADS; 710250079Scarl } else { 711250079Scarl ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET; 712250079Scarl ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET; 713250079Scarl ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS; 714250079Scarl } 715250079Scarl 716250079Scarl ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 717250079Scarl ntb->limits.msix_cnt = XEON_MSIX_CNT; 718250079Scarl ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 719250079Scarl 720255279Scarl configure_xeon_secondary_side_bars(ntb); 721250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 722255278Scarl ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 723250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 724255279Scarl 725255269Scarl /* Enable link training */ 726255278Scarl ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, 727255269Scarl NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP); 728250079Scarl 729250079Scarl return (0); 730250079Scarl} 731250079Scarl 732250079Scarlstatic int 733250079Scarlntb_setup_soc(struct ntb_softc *ntb) 734250079Scarl{ 735250079Scarl uint32_t val, connection_type; 736250079Scarl 737250079Scarl val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 738250079Scarl 739250079Scarl connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 740250079Scarl switch (connection_type) { 741250079Scarl case NTB_CONN_B2B: 742250079Scarl ntb->conn_type = NTB_CONN_B2B; 743250079Scarl break; 744250079Scarl case NTB_CONN_RP: 745250079Scarl default: 746250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 747250079Scarl connection_type); 748250079Scarl return (ENXIO); 749250079Scarl } 750250079Scarl 751250079Scarl if ((val & SOC_PPD_DEV_TYPE) != 0) 752250079Scarl ntb->dev_type = NTB_DEV_DSD; 753250079Scarl else 754250079Scarl ntb->dev_type = NTB_DEV_USD; 755250079Scarl 756250079Scarl /* Initiate PCI-E link training */ 757250079Scarl pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 758250079Scarl 4); 759250079Scarl 760250079Scarl ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET; 761250079Scarl ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET; 762250079Scarl ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET; 763250079Scarl ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET; 764250079Scarl ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 765250079Scarl ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 766250079Scarl ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 767250079Scarl ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 768250079Scarl 769250079Scarl if (ntb->conn_type == NTB_CONN_B2B) { 770250079Scarl ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET; 771250079Scarl ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 772250079Scarl ntb->limits.max_spads = SOC_MAX_SPADS; 773250079Scarl } else { 774250079Scarl ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET; 775250079Scarl ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET; 776250079Scarl ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS; 777250079Scarl } 778250079Scarl 779250079Scarl ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 780250079Scarl ntb->limits.msix_cnt = SOC_MSIX_CNT; 781250079Scarl ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 782250079Scarl 783250079Scarl /* 784250079Scarl * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 785250079Scarl * resolved. Mask transaction layer internal parity errors. 786250079Scarl */ 787250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 788250079Scarl 789255279Scarl configure_soc_secondary_side_bars(ntb); 790250079Scarl 791250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 792255278Scarl ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 793250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 794250079Scarl callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 795250079Scarl 796250079Scarl return (0); 797250079Scarl} 798250079Scarl 799255279Scarlstatic void 800255279Scarlconfigure_soc_secondary_side_bars(struct ntb_softc *ntb) 801255279Scarl{ 802255279Scarl 803255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 804255279Scarl ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 805255279Scarl ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 806255279Scarl ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 807255279Scarl ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 808255279Scarl } else { 809255279Scarl ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 810255279Scarl ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 811255279Scarl ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 812255279Scarl ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 813255279Scarl } 814255279Scarl} 815255279Scarl 816255279Scarlstatic void 817255279Scarlconfigure_xeon_secondary_side_bars(struct ntb_softc *ntb) 818255279Scarl{ 819255279Scarl 820255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 821255279Scarl ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 822255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 823255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 824255279Scarl MBAR01_DSD_ADDR); 825255279Scarl else 826255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 827255279Scarl PBAR4XLAT_USD_ADDR); 828255279Scarl ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 829255279Scarl ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 830255279Scarl ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 831255279Scarl } else { 832255279Scarl ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 833255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 834255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 835255279Scarl MBAR01_USD_ADDR); 836255279Scarl else 837255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 838255279Scarl PBAR4XLAT_DSD_ADDR); 839255279Scarl ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 840255279Scarl ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 841255279Scarl ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 842255279Scarl } 843255279Scarl} 844255279Scarl 845255281Scarl/* SOC does not have link status interrupt, poll on that platform */ 846250079Scarlstatic void 847250079Scarlntb_handle_heartbeat(void *arg) 848250079Scarl{ 849250079Scarl struct ntb_softc *ntb = arg; 850250079Scarl uint32_t status32; 851250079Scarl int rc = ntb_check_link_status(ntb); 852250079Scarl 853250079Scarl if (rc != 0) 854250079Scarl device_printf(ntb->device, 855250079Scarl "Error determining link status\n"); 856250079Scarl /* Check to see if a link error is the cause of the link down */ 857250079Scarl if (ntb->link_status == NTB_LINK_DOWN) { 858255278Scarl status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 859250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 860250079Scarl callout_reset(&ntb->lr_timer, 0, recover_soc_link, 861250079Scarl ntb); 862250079Scarl return; 863250079Scarl } 864250079Scarl } 865250079Scarl 866250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 867250079Scarl ntb_handle_heartbeat, ntb); 868250079Scarl} 869250079Scarl 870250079Scarlstatic void 871250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb) 872250079Scarl{ 873250079Scarl uint32_t status; 874250079Scarl 875250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 876255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 877255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 878255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 879255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 880250079Scarl 881250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 882250079Scarl pause("ModPhy", hz / 10); 883250079Scarl 884250079Scarl /* Clear AER Errors, write to clear */ 885255278Scarl status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 886250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 887255278Scarl ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 888250079Scarl 889250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 890255278Scarl status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 891250079Scarl status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 892255278Scarl ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 893250079Scarl 894250079Scarl /* Clear DeSkew Buffer error, write to clear */ 895255278Scarl status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 896250079Scarl status |= SOC_DESKEWSTS_DBERR; 897255278Scarl ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 898250079Scarl 899255278Scarl status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 900250079Scarl status &= SOC_IBIST_ERR_OFLOW; 901255278Scarl ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 902250079Scarl 903250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 904255278Scarl status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 905250079Scarl status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 906255278Scarl ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 907250079Scarl} 908250079Scarl 909250079Scarlstatic void 910250079Scarlntb_handle_link_event(struct ntb_softc *ntb, int link_state) 911250079Scarl{ 912250079Scarl enum ntb_hw_event event; 913250079Scarl uint16_t status; 914250079Scarl 915250079Scarl if (ntb->link_status == link_state) 916250079Scarl return; 917250079Scarl 918250079Scarl if (link_state == NTB_LINK_UP) { 919250079Scarl device_printf(ntb->device, "Link Up\n"); 920250079Scarl ntb->link_status = NTB_LINK_UP; 921250079Scarl event = NTB_EVENT_HW_LINK_UP; 922250079Scarl 923250079Scarl if (ntb->type == NTB_SOC) 924255278Scarl status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 925250079Scarl else 926250079Scarl status = pci_read_config(ntb->device, 927250079Scarl XEON_LINK_STATUS_OFFSET, 2); 928250079Scarl ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 929250079Scarl ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 930250079Scarl device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 931250079Scarl ntb->link_width, ntb->link_speed); 932250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 933250079Scarl ntb_handle_heartbeat, ntb); 934250079Scarl } else { 935250079Scarl device_printf(ntb->device, "Link Down\n"); 936250079Scarl ntb->link_status = NTB_LINK_DOWN; 937250079Scarl event = NTB_EVENT_HW_LINK_DOWN; 938255281Scarl /* Do not modify link width/speed, we need it in link recovery */ 939250079Scarl } 940250079Scarl 941250079Scarl /* notify the upper layer if we have an event change */ 942250079Scarl if (ntb->event_cb != NULL) 943250079Scarl ntb->event_cb(ntb->ntb_transport, event); 944250079Scarl} 945250079Scarl 946250079Scarlstatic void 947250079Scarlrecover_soc_link(void *arg) 948250079Scarl{ 949250079Scarl struct ntb_softc *ntb = arg; 950250079Scarl uint8_t speed, width; 951250079Scarl uint32_t status32; 952250079Scarl uint16_t status16; 953250079Scarl 954250079Scarl soc_perform_link_restart(ntb); 955250079Scarl pause("Link", SOC_LINK_RECOVERY_TIME * hz / 1000); 956250079Scarl 957255278Scarl status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 958250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 959250079Scarl goto retry; 960250079Scarl 961255278Scarl status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 962250079Scarl if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 963250079Scarl goto retry; 964250079Scarl 965255278Scarl status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 966250079Scarl width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 967250079Scarl speed = (status16 & NTB_LINK_SPEED_MASK); 968250079Scarl if (ntb->link_width != width || ntb->link_speed != speed) 969250079Scarl goto retry; 970250079Scarl 971250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 972250079Scarl ntb_handle_heartbeat, ntb); 973250079Scarl return; 974250079Scarl 975250079Scarlretry: 976250079Scarl callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 977250079Scarl ntb); 978250079Scarl} 979250079Scarl 980250079Scarlstatic int 981250079Scarlntb_check_link_status(struct ntb_softc *ntb) 982250079Scarl{ 983250079Scarl int link_state; 984250079Scarl uint32_t ntb_cntl; 985250079Scarl uint16_t status; 986250079Scarl 987250079Scarl if (ntb->type == NTB_SOC) { 988255278Scarl ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 989250079Scarl if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 990250079Scarl link_state = NTB_LINK_DOWN; 991250079Scarl else 992250079Scarl link_state = NTB_LINK_UP; 993250079Scarl } else { 994250079Scarl status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 995250079Scarl 2); 996250079Scarl 997250079Scarl if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 998250079Scarl link_state = NTB_LINK_UP; 999250079Scarl else 1000250079Scarl link_state = NTB_LINK_DOWN; 1001250079Scarl } 1002250079Scarl 1003250079Scarl ntb_handle_link_event(ntb, link_state); 1004250079Scarl 1005250079Scarl return (0); 1006250079Scarl} 1007250079Scarl 1008250079Scarl/** 1009250079Scarl * ntb_register_event_callback() - register event callback 1010250079Scarl * @ntb: pointer to ntb_softc instance 1011250079Scarl * @func: callback function to register 1012250079Scarl * 1013250079Scarl * This function registers a callback for any HW driver events such as link 1014250079Scarl * up/down, power management notices and etc. 1015250079Scarl * 1016250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1017250079Scarl */ 1018250079Scarlint 1019250079Scarlntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1020250079Scarl{ 1021250079Scarl 1022250079Scarl if (ntb->event_cb != NULL) 1023250079Scarl return (EINVAL); 1024250079Scarl 1025250079Scarl ntb->event_cb = func; 1026250079Scarl 1027250079Scarl return (0); 1028250079Scarl} 1029250079Scarl 1030250079Scarl/** 1031250079Scarl * ntb_unregister_event_callback() - unregisters the event callback 1032250079Scarl * @ntb: pointer to ntb_softc instance 1033250079Scarl * 1034250079Scarl * This function unregisters the existing callback from transport 1035250079Scarl */ 1036250079Scarlvoid 1037250079Scarlntb_unregister_event_callback(struct ntb_softc *ntb) 1038250079Scarl{ 1039250079Scarl 1040250079Scarl ntb->event_cb = NULL; 1041250079Scarl} 1042250079Scarl 1043250079Scarl/** 1044250079Scarl * ntb_register_db_callback() - register a callback for doorbell interrupt 1045250079Scarl * @ntb: pointer to ntb_softc instance 1046250079Scarl * @idx: doorbell index to register callback, zero based 1047250079Scarl * @func: callback function to register 1048250079Scarl * 1049250079Scarl * This function registers a callback function for the doorbell interrupt 1050250079Scarl * on the primary side. The function will unmask the doorbell as well to 1051250079Scarl * allow interrupt. 1052250079Scarl * 1053250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1054250079Scarl */ 1055250079Scarlint 1056250079Scarlntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1057250079Scarl ntb_db_callback func) 1058250079Scarl{ 1059250079Scarl uint16_t mask; 1060250079Scarl 1061250079Scarl if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1062250079Scarl device_printf(ntb->device, "Invalid Index.\n"); 1063250079Scarl return (EINVAL); 1064250079Scarl } 1065250079Scarl 1066250079Scarl ntb->db_cb[idx].callback = func; 1067250079Scarl ntb->db_cb[idx].data = data; 1068250079Scarl 1069250079Scarl /* unmask interrupt */ 1070255278Scarl mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask); 1071250079Scarl mask &= ~(1 << (idx * ntb->bits_per_vector)); 1072255278Scarl ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask); 1073250079Scarl 1074250079Scarl return (0); 1075250079Scarl} 1076250079Scarl 1077250079Scarl/** 1078250079Scarl * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1079250079Scarl * @ntb: pointer to ntb_softc instance 1080250079Scarl * @idx: doorbell index to register callback, zero based 1081250079Scarl * 1082250079Scarl * This function unregisters a callback function for the doorbell interrupt 1083250079Scarl * on the primary side. The function will also mask the said doorbell. 1084250079Scarl */ 1085250079Scarlvoid 1086250079Scarlntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1087250079Scarl{ 1088250079Scarl unsigned long mask; 1089250079Scarl 1090250079Scarl if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1091250079Scarl return; 1092250079Scarl 1093255278Scarl mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask); 1094250079Scarl mask |= 1 << (idx * ntb->bits_per_vector); 1095255278Scarl ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask); 1096250079Scarl 1097250079Scarl ntb->db_cb[idx].callback = NULL; 1098250079Scarl} 1099250079Scarl 1100250079Scarl/** 1101250079Scarl * ntb_find_transport() - find the transport pointer 1102250079Scarl * @transport: pointer to pci device 1103250079Scarl * 1104250079Scarl * Given the pci device pointer, return the transport pointer passed in when 1105250079Scarl * the transport attached when it was inited. 1106250079Scarl * 1107250079Scarl * RETURNS: pointer to transport. 1108250079Scarl */ 1109250079Scarlvoid * 1110250079Scarlntb_find_transport(struct ntb_softc *ntb) 1111250079Scarl{ 1112250079Scarl 1113250079Scarl return (ntb->ntb_transport); 1114250079Scarl} 1115250079Scarl 1116250079Scarl/** 1117250079Scarl * ntb_register_transport() - Register NTB transport with NTB HW driver 1118250079Scarl * @transport: transport identifier 1119250079Scarl * 1120250079Scarl * This function allows a transport to reserve the hardware driver for 1121250079Scarl * NTB usage. 1122250079Scarl * 1123250079Scarl * RETURNS: pointer to ntb_softc, NULL on error. 1124250079Scarl */ 1125250079Scarlstruct ntb_softc * 1126250079Scarlntb_register_transport(struct ntb_softc *ntb, void *transport) 1127250079Scarl{ 1128250079Scarl 1129250079Scarl /* 1130250079Scarl * TODO: when we have more than one transport, we will need to rewrite 1131250079Scarl * this to prevent race conditions 1132250079Scarl */ 1133250079Scarl if (ntb->ntb_transport != NULL) 1134250079Scarl return (NULL); 1135250079Scarl 1136250079Scarl ntb->ntb_transport = transport; 1137250079Scarl return (ntb); 1138250079Scarl} 1139250079Scarl 1140250079Scarl/** 1141250079Scarl * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1142250079Scarl * @ntb - ntb_softc of the transport to be freed 1143250079Scarl * 1144250079Scarl * This function unregisters the transport from the HW driver and performs any 1145250079Scarl * necessary cleanups. 1146250079Scarl */ 1147250079Scarlvoid 1148250079Scarlntb_unregister_transport(struct ntb_softc *ntb) 1149250079Scarl{ 1150250079Scarl int i; 1151250079Scarl 1152250079Scarl if (ntb->ntb_transport == NULL) 1153250079Scarl return; 1154250079Scarl 1155250079Scarl for (i = 0; i < ntb->allocated_interrupts; i++) 1156250079Scarl ntb_unregister_db_callback(ntb, i); 1157250079Scarl 1158250079Scarl ntb_unregister_event_callback(ntb); 1159250079Scarl ntb->ntb_transport = NULL; 1160250079Scarl} 1161250079Scarl 1162250079Scarl/** 1163250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 1164250079Scarl * @ntb: pointer to ntb_softc instance 1165250079Scarl * 1166250079Scarl * This function returns the max 32bit scratchpad registers usable by the 1167250079Scarl * upper layer. 1168250079Scarl * 1169250079Scarl * RETURNS: total number of scratch pad registers available 1170250079Scarl */ 1171250079Scarlint 1172250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 1173250079Scarl{ 1174250079Scarl 1175250079Scarl return (ntb->limits.max_spads); 1176250079Scarl} 1177250079Scarl 1178250079Scarl/** 1179250079Scarl * ntb_write_local_spad() - write to the secondary scratchpad register 1180250079Scarl * @ntb: pointer to ntb_softc instance 1181250079Scarl * @idx: index to the scratchpad register, 0 based 1182250079Scarl * @val: the data value to put into the register 1183250079Scarl * 1184250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1185250079Scarl * register. The register resides on the secondary (external) side. 1186250079Scarl * 1187250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1188250079Scarl */ 1189250079Scarlint 1190250079Scarlntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1191250079Scarl{ 1192250079Scarl 1193250079Scarl if (idx >= ntb->limits.max_spads) 1194250079Scarl return (EINVAL); 1195250079Scarl 1196255278Scarl ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1197250079Scarl 1198250079Scarl return (0); 1199250079Scarl} 1200250079Scarl 1201250079Scarl/** 1202250079Scarl * ntb_read_local_spad() - read from the primary scratchpad register 1203250079Scarl * @ntb: pointer to ntb_softc instance 1204250079Scarl * @idx: index to scratchpad register, 0 based 1205250079Scarl * @val: pointer to 32bit integer for storing the register value 1206250079Scarl * 1207250079Scarl * This function allows reading of the 32bit scratchpad register on 1208250079Scarl * the primary (internal) side. 1209250079Scarl * 1210250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1211250079Scarl */ 1212250079Scarlint 1213250079Scarlntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1214250079Scarl{ 1215250079Scarl 1216250079Scarl if (idx >= ntb->limits.max_spads) 1217250079Scarl return (EINVAL); 1218250079Scarl 1219255278Scarl *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1220250079Scarl 1221250079Scarl return (0); 1222250079Scarl} 1223250079Scarl 1224250079Scarl/** 1225250079Scarl * ntb_write_remote_spad() - write to the secondary scratchpad register 1226250079Scarl * @ntb: pointer to ntb_softc instance 1227250079Scarl * @idx: index to the scratchpad register, 0 based 1228250079Scarl * @val: the data value to put into the register 1229250079Scarl * 1230250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1231250079Scarl * register. The register resides on the secondary (external) side. 1232250079Scarl * 1233250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1234250079Scarl */ 1235250079Scarlint 1236250079Scarlntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1237250079Scarl{ 1238250079Scarl 1239250079Scarl if (idx >= ntb->limits.max_spads) 1240250079Scarl return (EINVAL); 1241250079Scarl 1242255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1243255279Scarl ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1244255279Scarl else 1245255279Scarl ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1246250079Scarl 1247250079Scarl return (0); 1248250079Scarl} 1249250079Scarl 1250250079Scarl/** 1251250079Scarl * ntb_read_remote_spad() - read from the primary scratchpad register 1252250079Scarl * @ntb: pointer to ntb_softc instance 1253250079Scarl * @idx: index to scratchpad register, 0 based 1254250079Scarl * @val: pointer to 32bit integer for storing the register value 1255250079Scarl * 1256250079Scarl * This function allows reading of the 32bit scratchpad register on 1257250079Scarl * the primary (internal) side. 1258250079Scarl * 1259250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1260250079Scarl */ 1261250079Scarlint 1262250079Scarlntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1263250079Scarl{ 1264250079Scarl 1265250079Scarl if (idx >= ntb->limits.max_spads) 1266250079Scarl return (EINVAL); 1267250079Scarl 1268255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1269255279Scarl *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1270255279Scarl else 1271255279Scarl *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1272250079Scarl 1273250079Scarl return (0); 1274250079Scarl} 1275250079Scarl 1276250079Scarl/** 1277250079Scarl * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1278250079Scarl * @ntb: pointer to ntb_softc instance 1279250079Scarl * @mw: memory window number 1280250079Scarl * 1281250079Scarl * This function provides the base virtual address of the memory window 1282250079Scarl * specified. 1283250079Scarl * 1284250079Scarl * RETURNS: pointer to virtual address, or NULL on error. 1285250079Scarl */ 1286250079Scarlvoid * 1287250079Scarlntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1288250079Scarl{ 1289250079Scarl 1290250079Scarl if (mw >= NTB_NUM_MW) 1291250079Scarl return (NULL); 1292250079Scarl 1293250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1294250079Scarl} 1295250079Scarl 1296250079Scarlvm_paddr_t 1297250079Scarlntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1298250079Scarl{ 1299250079Scarl 1300250079Scarl if (mw >= NTB_NUM_MW) 1301250079Scarl return (0); 1302250079Scarl 1303250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1304250079Scarl} 1305250079Scarl 1306250079Scarl/** 1307250079Scarl * ntb_get_mw_size() - return size of NTB memory window 1308250079Scarl * @ntb: pointer to ntb_softc instance 1309250079Scarl * @mw: memory window number 1310250079Scarl * 1311250079Scarl * This function provides the physical size of the memory window specified 1312250079Scarl * 1313250079Scarl * RETURNS: the size of the memory window or zero on error 1314250079Scarl */ 1315250079Scarlu_long 1316250079Scarlntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1317250079Scarl{ 1318250079Scarl 1319250079Scarl if (mw >= NTB_NUM_MW) 1320250079Scarl return (0); 1321250079Scarl 1322250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1323250079Scarl} 1324250079Scarl 1325250079Scarl/** 1326250079Scarl * ntb_set_mw_addr - set the memory window address 1327250079Scarl * @ntb: pointer to ntb_softc instance 1328250079Scarl * @mw: memory window number 1329250079Scarl * @addr: base address for data 1330250079Scarl * 1331250079Scarl * This function sets the base physical address of the memory window. This 1332250079Scarl * memory address is where data from the remote system will be transfered into 1333250079Scarl * or out of depending on how the transport is configured. 1334250079Scarl */ 1335250079Scarlvoid 1336250079Scarlntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1337250079Scarl{ 1338250079Scarl 1339250079Scarl if (mw >= NTB_NUM_MW) 1340250079Scarl return; 1341250079Scarl 1342250079Scarl switch (NTB_MW_TO_BAR(mw)) { 1343250079Scarl case NTB_B2B_BAR_1: 1344255278Scarl ntb_reg_write(8, ntb->reg_ofs.sbar2_xlat, addr); 1345250079Scarl break; 1346250079Scarl case NTB_B2B_BAR_2: 1347255278Scarl ntb_reg_write(8, ntb->reg_ofs.sbar4_xlat, addr); 1348250079Scarl break; 1349250079Scarl } 1350250079Scarl} 1351250079Scarl 1352250079Scarl/** 1353250079Scarl * ntb_ring_sdb() - Set the doorbell on the secondary/external side 1354250079Scarl * @ntb: pointer to ntb_softc instance 1355250079Scarl * @db: doorbell to ring 1356250079Scarl * 1357250079Scarl * This function allows triggering of a doorbell on the secondary/external 1358250079Scarl * side that will initiate an interrupt on the remote host 1359250079Scarl * 1360250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1361250079Scarl */ 1362250079Scarlvoid 1363250079Scarlntb_ring_sdb(struct ntb_softc *ntb, unsigned int db) 1364250079Scarl{ 1365250079Scarl 1366250079Scarl if (ntb->type == NTB_SOC) 1367255278Scarl ntb_reg_write(8, ntb->reg_ofs.sdb, (uint64_t) 1 << db); 1368250079Scarl else 1369255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1370255279Scarl ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1371255279Scarl ((1 << ntb->bits_per_vector) - 1) << 1372255279Scarl (db * ntb->bits_per_vector)); 1373255279Scarl else 1374255279Scarl ntb_reg_write(2, ntb->reg_ofs.sdb, 1375255279Scarl ((1 << ntb->bits_per_vector) - 1) << 1376255279Scarl (db * ntb->bits_per_vector)); 1377250079Scarl} 1378250079Scarl 1379250079Scarl/** 1380250079Scarl * ntb_query_link_status() - return the hardware link status 1381250079Scarl * @ndev: pointer to ntb_device instance 1382250079Scarl * 1383250079Scarl * Returns true if the hardware is connected to the remote system 1384250079Scarl * 1385250079Scarl * RETURNS: true or false based on the hardware link state 1386250079Scarl */ 1387250079Scarlbool 1388250079Scarlntb_query_link_status(struct ntb_softc *ntb) 1389250079Scarl{ 1390250079Scarl 1391250079Scarl return (ntb->link_status == NTB_LINK_UP); 1392250079Scarl} 1393250079Scarl 1394255272Scarlstatic void 1395255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 1396250079Scarl{ 1397255272Scarl bar->pci_bus_tag = 1398255272Scarl rman_get_bustag(bar->pci_resource); 1399255272Scarl bar->pci_bus_handle = 1400255272Scarl rman_get_bushandle(bar->pci_resource); 1401255272Scarl bar->pbase = 1402255272Scarl rman_get_start(bar->pci_resource); 1403255272Scarl bar->size = 1404255272Scarl rman_get_size(bar->pci_resource); 1405255272Scarl bar->vbase = 1406255272Scarl rman_get_virtual(bar->pci_resource); 1407255272Scarl 1408250079Scarl} 1409255268Scarl 1410255268Scarldevice_t ntb_get_device(struct ntb_softc *ntb) 1411255268Scarl{ 1412255268Scarl 1413255268Scarl return (ntb->device); 1414255268Scarl} 1415