ntb_hw_intel.c revision 289617
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3289542Scem * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289617 2015-10-20 01:54:52Z cem $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35250079Scarl#include <sys/malloc.h> 36250079Scarl#include <sys/module.h> 37250079Scarl#include <sys/queue.h> 38250079Scarl#include <sys/rman.h> 39289207Scem#include <sys/sysctl.h> 40250079Scarl#include <vm/vm.h> 41250079Scarl#include <vm/pmap.h> 42250079Scarl#include <machine/bus.h> 43250079Scarl#include <machine/pmap.h> 44250079Scarl#include <machine/resource.h> 45250079Scarl#include <dev/pci/pcireg.h> 46250079Scarl#include <dev/pci/pcivar.h> 47250079Scarl 48250079Scarl#include "ntb_regs.h" 49250079Scarl#include "ntb_hw.h" 50250079Scarl 51250079Scarl/* 52250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 53250079Scarl * allows you to connect two systems using a PCI-e link. 54250079Scarl * 55250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 56250079Scarl * you to send and recieve interrupts, map the memory windows and send and 57250079Scarl * receive messages in the scratch-pad registers. 58250079Scarl * 59250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 60250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 61250079Scarl */ 62250079Scarl 63289538Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT) 64250079Scarl 65289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 66289539Scem#define SOC_LINK_RECOVERY_TIME 500 /* ms */ 67250079Scarl 68250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 69250079Scarl 70250079Scarlenum ntb_device_type { 71250079Scarl NTB_XEON, 72250079Scarl NTB_SOC 73250079Scarl}; 74250079Scarl 75289610Scem/* ntb_conn_type are hardware numbers, cannot change. */ 76289610Scemenum ntb_conn_type { 77289610Scem NTB_CONN_TRANSPARENT = 0, 78289610Scem NTB_CONN_B2B = 1, 79289610Scem NTB_CONN_RP = 2, 80289610Scem}; 81289610Scem 82289610Scemenum ntb_b2b_direction { 83289610Scem NTB_DEV_USD = 0, 84289610Scem NTB_DEV_DSD = 1, 85289610Scem}; 86289610Scem 87289539Scemenum ntb_bar { 88289539Scem NTB_CONFIG_BAR = 0, 89289539Scem NTB_B2B_BAR_1, 90289539Scem NTB_B2B_BAR_2, 91289539Scem NTB_B2B_BAR_3, 92289539Scem NTB_MAX_BARS 93289539Scem}; 94289539Scem 95255274Scarl/* Device features and workarounds */ 96255274Scarl#define HAS_FEATURE(feature) \ 97255274Scarl ((ntb->features & (feature)) != 0) 98255274Scarl 99250079Scarlstruct ntb_hw_info { 100250079Scarl uint32_t device_id; 101255274Scarl const char *desc; 102250079Scarl enum ntb_device_type type; 103289397Scem uint32_t features; 104250079Scarl}; 105250079Scarl 106250079Scarlstruct ntb_pci_bar_info { 107250079Scarl bus_space_tag_t pci_bus_tag; 108250079Scarl bus_space_handle_t pci_bus_handle; 109250079Scarl int pci_resource_id; 110250079Scarl struct resource *pci_resource; 111250079Scarl vm_paddr_t pbase; 112250079Scarl void *vbase; 113250079Scarl u_long size; 114289543Scem 115289543Scem /* Configuration register offsets */ 116289543Scem uint32_t psz_off; 117289543Scem uint32_t ssz_off; 118289543Scem uint32_t pbarxlat_off; 119250079Scarl}; 120250079Scarl 121250079Scarlstruct ntb_int_info { 122250079Scarl struct resource *res; 123250079Scarl int rid; 124250079Scarl void *tag; 125250079Scarl}; 126250079Scarl 127289546Scemstruct ntb_vec { 128250079Scarl struct ntb_softc *ntb; 129289546Scem uint32_t num; 130250079Scarl}; 131250079Scarl 132289542Scemstruct ntb_reg { 133289542Scem uint32_t ntb_ctl; 134289542Scem uint32_t lnk_sta; 135289542Scem uint8_t db_size; 136289542Scem unsigned mw_bar[NTB_MAX_BARS]; 137289542Scem}; 138289542Scem 139289542Scemstruct ntb_alt_reg { 140289542Scem uint32_t db_bell; 141289542Scem uint32_t db_mask; 142289542Scem uint32_t spad; 143289542Scem}; 144289542Scem 145289542Scemstruct ntb_xlat_reg { 146289546Scem uint32_t bar0_base; 147289546Scem uint32_t bar2_base; 148289546Scem uint32_t bar4_base; 149289546Scem uint32_t bar5_base; 150289546Scem 151289546Scem uint32_t bar2_xlat; 152289546Scem uint32_t bar4_xlat; 153289546Scem uint32_t bar5_xlat; 154289546Scem 155289546Scem uint32_t bar2_limit; 156289546Scem uint32_t bar4_limit; 157289546Scem uint32_t bar5_limit; 158289542Scem}; 159289542Scem 160289542Scemstruct ntb_b2b_addr { 161289542Scem uint64_t bar0_addr; 162289542Scem uint64_t bar2_addr64; 163289542Scem uint64_t bar4_addr64; 164289542Scem uint64_t bar4_addr32; 165289542Scem uint64_t bar5_addr32; 166289542Scem}; 167289542Scem 168250079Scarlstruct ntb_softc { 169250079Scarl device_t device; 170250079Scarl enum ntb_device_type type; 171255274Scarl uint64_t features; 172250079Scarl 173250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 174250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 175250079Scarl uint32_t allocated_interrupts; 176250079Scarl 177250079Scarl struct callout heartbeat_timer; 178250079Scarl struct callout lr_timer; 179250079Scarl 180289546Scem void *ntb_ctx; 181289546Scem const struct ntb_ctx_ops *ctx_ops; 182289546Scem struct ntb_vec *msix_vec; 183289546Scem#define CTX_LOCK(sc) mtx_lock_spin(&(sc)->ctx_lock) 184289546Scem#define CTX_UNLOCK(sc) mtx_unlock_spin(&(sc)->ctx_lock) 185289546Scem#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 186289546Scem struct mtx ctx_lock; 187250079Scarl 188289610Scem uint32_t ppd; 189289610Scem enum ntb_conn_type conn_type; 190289610Scem enum ntb_b2b_direction dev_type; 191289539Scem 192289542Scem /* Offset of peer bar0 in B2B BAR */ 193289542Scem uint64_t b2b_off; 194289542Scem /* Memory window used to access peer bar0 */ 195289543Scem#define B2B_MW_DISABLED UINT8_MAX 196289542Scem uint8_t b2b_mw_idx; 197289542Scem 198289539Scem uint8_t mw_count; 199289539Scem uint8_t spad_count; 200289539Scem uint8_t db_count; 201289539Scem uint8_t db_vec_count; 202289539Scem uint8_t db_vec_shift; 203289542Scem 204289546Scem /* Protects local db_mask. */ 205289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 206289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 207289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 208289542Scem struct mtx db_mask_lock; 209289542Scem 210289546Scem uint32_t ntb_ctl; 211289546Scem uint32_t lnk_sta; 212289542Scem 213289542Scem uint64_t db_valid_mask; 214289542Scem uint64_t db_link_mask; 215289546Scem uint64_t db_mask; 216289542Scem 217289542Scem int last_ts; /* ticks @ last irq */ 218289542Scem 219289542Scem const struct ntb_reg *reg; 220289542Scem const struct ntb_alt_reg *self_reg; 221289542Scem const struct ntb_alt_reg *peer_reg; 222289542Scem const struct ntb_xlat_reg *xlat_reg; 223250079Scarl}; 224250079Scarl 225289234Scem#ifdef __i386__ 226289234Scemstatic __inline uint64_t 227289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 228289234Scem bus_size_t offset) 229289234Scem{ 230289234Scem 231289234Scem return (bus_space_read_4(tag, handle, offset) | 232289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 233289234Scem} 234289234Scem 235289234Scemstatic __inline void 236289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 237289234Scem bus_size_t offset, uint64_t val) 238289234Scem{ 239289234Scem 240289234Scem bus_space_write_4(tag, handle, offset, val); 241289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 242289234Scem} 243289234Scem#endif 244289234Scem 245255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 246255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 247255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 248255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 249255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 250255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 251255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 252250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 253255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 254289397Scem#define ntb_mw_read(SIZE, offset) \ 255289542Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 256255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 257289542Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 258289397Scem offset, val) 259250079Scarl 260250079Scarlstatic int ntb_probe(device_t device); 261250079Scarlstatic int ntb_attach(device_t device); 262250079Scarlstatic int ntb_detach(device_t device); 263289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 264289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 265289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 266289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 267255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 268289541Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *); 269255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 270255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 271255272Scarl struct ntb_pci_bar_info *bar); 272250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 273289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 274289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 275289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 276289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 277250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 278289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 279289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 280289546Scemstatic void ndev_vec_isr(void *arg); 281289546Scemstatic void ndev_irq_isr(void *arg); 282289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 283289546Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t val); 284289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 285289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 286250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 287289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 288289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 289289348Scemstatic int ntb_detect_soc(struct ntb_softc *ntb); 290289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 291289542Scemstatic int ntb_soc_init_dev(struct ntb_softc *ntb); 292289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 293255279Scarlstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 294289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 295289543Scem enum ntb_bar regbar); 296289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 297289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 298289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 299289543Scem enum ntb_bar idx); 300289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 301289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 302289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 303289546Scemstatic inline bool soc_link_is_err(struct ntb_softc *ntb); 304289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 305289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 306289542Scemstatic void soc_link_hb(void *arg); 307289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 308250079Scarlstatic void recover_soc_link(void *arg); 309289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb); 310255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 311250079Scarl 312250079Scarlstatic struct ntb_hw_info pci_ids[] = { 313289612Scem /* XXX: PS/SS IDs left out until they are supported. */ 314289612Scem { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 315289612Scem NTB_SOC, 0 }, 316289233Scem 317289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 318289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 319289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 320289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 321289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 322289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 323289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 324289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 325289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 326289538Scem NTB_SB01BASE_LOCKUP }, 327289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 328289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 329289538Scem NTB_SB01BASE_LOCKUP }, 330289233Scem 331255274Scarl { 0x00000000, NULL, NTB_SOC, 0 } 332250079Scarl}; 333250079Scarl 334289542Scemstatic const struct ntb_reg soc_reg = { 335289542Scem .ntb_ctl = SOC_NTBCNTL_OFFSET, 336289542Scem .lnk_sta = SOC_LINK_STATUS_OFFSET, 337289542Scem .db_size = sizeof(uint64_t), 338289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 339289542Scem}; 340289542Scem 341289607Scemstatic const struct ntb_alt_reg soc_pri_reg = { 342289607Scem .db_bell = SOC_PDOORBELL_OFFSET, 343289607Scem .db_mask = SOC_PDBMSK_OFFSET, 344289607Scem .spad = SOC_SPAD_OFFSET, 345289607Scem}; 346289607Scem 347289542Scemstatic const struct ntb_alt_reg soc_b2b_reg = { 348289542Scem .db_bell = SOC_B2B_DOORBELL_OFFSET, 349289542Scem .spad = SOC_B2B_SPAD_OFFSET, 350289542Scem}; 351289542Scem 352289542Scemstatic const struct ntb_xlat_reg soc_sec_xlat = { 353289542Scem#if 0 354289542Scem /* "FIXME" says the Linux driver. */ 355289542Scem .bar0_base = SOC_SBAR0BASE_OFFSET, 356289546Scem .bar2_base = SOC_SBAR2BASE_OFFSET, 357289546Scem .bar4_base = SOC_SBAR4BASE_OFFSET, 358289546Scem 359289542Scem .bar2_limit = SOC_SBAR2LMT_OFFSET, 360289546Scem .bar4_limit = SOC_SBAR4LMT_OFFSET, 361289542Scem#endif 362289546Scem 363289542Scem .bar2_xlat = SOC_SBAR2XLAT_OFFSET, 364289546Scem .bar4_xlat = SOC_SBAR4XLAT_OFFSET, 365289542Scem}; 366289542Scem 367289542Scemstatic const struct ntb_reg xeon_reg = { 368289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 369289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 370289542Scem .db_size = sizeof(uint16_t), 371289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 372289542Scem}; 373289542Scem 374289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 375289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 376289607Scem .db_mask = XEON_PDBMSK_OFFSET, 377289607Scem .spad = XEON_SPAD_OFFSET, 378289607Scem}; 379289607Scem 380289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 381289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 382289542Scem .spad = XEON_B2B_SPAD_OFFSET, 383289542Scem}; 384289542Scem 385289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 386289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 387289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 388289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 389289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 390289546Scem 391289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 392289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 393289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 394289546Scem 395289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 396289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 397289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 398289542Scem}; 399289542Scem 400289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 401289542Scem .bar0_addr = XEON_B2B_BAR0_USD_ADDR, 402289542Scem .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, 403289542Scem .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, 404289542Scem .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, 405289542Scem .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, 406289542Scem}; 407289542Scem 408289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 409289542Scem .bar0_addr = XEON_B2B_BAR0_DSD_ADDR, 410289542Scem .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, 411289542Scem .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, 412289542Scem .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, 413289542Scem .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, 414289542Scem}; 415289542Scem 416289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 417289614Scem "B2B MW segment overrides -- MUST be the same on both sides"); 418289614Scem 419289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 420289614Scem &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 421289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 422289614Scem "the window at BAR2, on the upstream side of the link. MUST be the same " 423289614Scem "address on both sides."); 424289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 425289614Scem &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 426289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 427289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 428289614Scem "(split-BAR mode)."); 429289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 430289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR5 " 431289614Scem "(split-BAR mode)."); 432289614Scem 433289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 434289614Scem &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 435289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 436289614Scem "the window at BAR2, on the downstream side of the link. MUST be the same" 437289614Scem " address on both sides."); 438289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 439289614Scem &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 440289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 441289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 442289614Scem "(split-BAR mode)."); 443289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 444289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 445289614Scem "(split-BAR mode)."); 446289614Scem 447250079Scarl/* 448250079Scarl * OS <-> Driver interface structures 449250079Scarl */ 450250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 451250079Scarl 452250079Scarlstatic device_method_t ntb_pci_methods[] = { 453250079Scarl /* Device interface */ 454250079Scarl DEVMETHOD(device_probe, ntb_probe), 455250079Scarl DEVMETHOD(device_attach, ntb_attach), 456250079Scarl DEVMETHOD(device_detach, ntb_detach), 457250079Scarl DEVMETHOD_END 458250079Scarl}; 459250079Scarl 460250079Scarlstatic driver_t ntb_pci_driver = { 461250079Scarl "ntb_hw", 462250079Scarl ntb_pci_methods, 463250079Scarl sizeof(struct ntb_softc), 464250079Scarl}; 465250079Scarl 466250079Scarlstatic devclass_t ntb_devclass; 467250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 468250079ScarlMODULE_VERSION(ntb_hw, 1); 469250079Scarl 470289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 471289207Scem 472250079Scarl/* 473250079Scarl * OS <-> Driver linkage functions 474250079Scarl */ 475250079Scarlstatic int 476250079Scarlntb_probe(device_t device) 477250079Scarl{ 478289209Scem struct ntb_hw_info *p; 479250079Scarl 480289209Scem p = ntb_get_device_info(pci_get_devid(device)); 481289209Scem if (p == NULL) 482250079Scarl return (ENXIO); 483289209Scem 484289209Scem device_set_desc(device, p->desc); 485289209Scem return (0); 486250079Scarl} 487250079Scarl 488250079Scarlstatic int 489250079Scarlntb_attach(device_t device) 490250079Scarl{ 491289209Scem struct ntb_softc *ntb; 492289209Scem struct ntb_hw_info *p; 493250079Scarl int error; 494250079Scarl 495289209Scem ntb = DEVICE2SOFTC(device); 496289209Scem p = ntb_get_device_info(pci_get_devid(device)); 497289209Scem 498250079Scarl ntb->device = device; 499250079Scarl ntb->type = p->type; 500255274Scarl ntb->features = p->features; 501289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 502250079Scarl 503250079Scarl /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 504283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 505283291Sjkim callout_init(&ntb->lr_timer, 1); 506289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 507289546Scem mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN); 508250079Scarl 509289348Scem if (ntb->type == NTB_SOC) 510289348Scem error = ntb_detect_soc(ntb); 511289348Scem else 512289348Scem error = ntb_detect_xeon(ntb); 513289348Scem if (error) 514289348Scem goto out; 515289348Scem 516289397Scem ntb_detect_max_mw(ntb); 517289396Scem 518289209Scem error = ntb_map_pci_bars(ntb); 519289209Scem if (error) 520289209Scem goto out; 521289272Scem if (ntb->type == NTB_SOC) 522289542Scem error = ntb_soc_init_dev(ntb); 523289272Scem else 524289542Scem error = ntb_xeon_init_dev(ntb); 525289209Scem if (error) 526289209Scem goto out; 527289540Scem error = ntb_init_isr(ntb); 528289209Scem if (error) 529289209Scem goto out; 530250079Scarl 531250079Scarl pci_enable_busmaster(ntb->device); 532250079Scarl 533289209Scemout: 534289209Scem if (error != 0) 535289209Scem ntb_detach(device); 536250079Scarl return (error); 537250079Scarl} 538250079Scarl 539250079Scarlstatic int 540250079Scarlntb_detach(device_t device) 541250079Scarl{ 542289209Scem struct ntb_softc *ntb; 543250079Scarl 544289209Scem ntb = DEVICE2SOFTC(device); 545289542Scem 546289617Scem if (ntb->self_reg != NULL) 547289617Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 548250079Scarl callout_drain(&ntb->heartbeat_timer); 549250079Scarl callout_drain(&ntb->lr_timer); 550289272Scem if (ntb->type == NTB_XEON) 551289272Scem ntb_teardown_xeon(ntb); 552250079Scarl ntb_teardown_interrupts(ntb); 553289397Scem 554289542Scem mtx_destroy(&ntb->db_mask_lock); 555289546Scem mtx_destroy(&ntb->ctx_lock); 556289542Scem 557289397Scem /* 558289397Scem * Redetect total MWs so we unmap properly -- in case we lowered the 559289397Scem * maximum to work around Xeon errata. 560289397Scem */ 561289397Scem ntb_detect_max_mw(ntb); 562250079Scarl ntb_unmap_pci_bar(ntb); 563250079Scarl 564250079Scarl return (0); 565250079Scarl} 566250079Scarl 567289542Scem/* 568289542Scem * Driver internal routines 569289542Scem */ 570289539Scemstatic inline enum ntb_bar 571289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 572289539Scem{ 573289539Scem 574289543Scem KASSERT(mw < ntb->mw_count || 575289543Scem (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx), 576289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 577289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 578289539Scem 579289542Scem return (ntb->reg->mw_bar[mw]); 580289539Scem} 581289539Scem 582289546Scemstatic inline bool 583289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 584289546Scem{ 585289546Scem /* XXX This assertion could be stronger. */ 586289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 587289546Scem return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 588289546Scem} 589289546Scem 590289546Scemstatic inline void 591289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 592289546Scem uint32_t *xlat, uint32_t *lmt) 593289546Scem{ 594289546Scem uint32_t basev, lmtv, xlatv; 595289546Scem 596289546Scem switch (bar) { 597289546Scem case NTB_B2B_BAR_1: 598289546Scem basev = ntb->xlat_reg->bar2_base; 599289546Scem lmtv = ntb->xlat_reg->bar2_limit; 600289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 601289546Scem break; 602289546Scem case NTB_B2B_BAR_2: 603289546Scem basev = ntb->xlat_reg->bar4_base; 604289546Scem lmtv = ntb->xlat_reg->bar4_limit; 605289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 606289546Scem break; 607289546Scem case NTB_B2B_BAR_3: 608289546Scem basev = ntb->xlat_reg->bar5_base; 609289546Scem lmtv = ntb->xlat_reg->bar5_limit; 610289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 611289546Scem break; 612289546Scem default: 613289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 614289546Scem ("bad bar")); 615289546Scem basev = lmtv = xlatv = 0; 616289546Scem break; 617289546Scem } 618289546Scem 619289546Scem if (base != NULL) 620289546Scem *base = basev; 621289546Scem if (xlat != NULL) 622289546Scem *xlat = xlatv; 623289546Scem if (lmt != NULL) 624289546Scem *lmt = lmtv; 625289546Scem} 626289546Scem 627250079Scarlstatic int 628255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 629250079Scarl{ 630255272Scarl int rc; 631250079Scarl 632250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 633289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 634255272Scarl if (rc != 0) 635289541Scem goto out; 636255272Scarl 637289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 638289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 639255272Scarl if (rc != 0) 640289541Scem goto out; 641289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 642289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 643289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 644255272Scarl 645289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 646289543Scem /* XXX Are shared MW B2Bs write-combining? */ 647289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 648289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 649255279Scarl else 650289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 651289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 652289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 653289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 654289543Scem 655289397Scem if (!HAS_FEATURE(NTB_SPLIT_BAR)) 656289541Scem goto out; 657289397Scem 658289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 659289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 660289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 661289397Scem else 662289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 663289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 664289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 665289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 666250079Scarl 667289541Scemout: 668289209Scem if (rc != 0) 669255272Scarl device_printf(ntb->device, 670255272Scarl "unable to allocate pci resource\n"); 671255272Scarl return (rc); 672255272Scarl} 673255272Scarl 674289541Scemstatic void 675289541Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 676289541Scem{ 677289541Scem 678289541Scem device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n", 679289541Scem bar->size, bar->vbase, (void *)(bar->pbase)); 680289541Scem} 681289541Scem 682255272Scarlstatic int 683255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 684255272Scarl{ 685255272Scarl 686255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 687289209Scem &bar->pci_resource_id, RF_ACTIVE); 688255272Scarl if (bar->pci_resource == NULL) 689255272Scarl return (ENXIO); 690289209Scem 691289209Scem save_bar_parameters(bar); 692289541Scem print_map_success(ntb, bar); 693289209Scem return (0); 694255272Scarl} 695255272Scarl 696255272Scarlstatic int 697255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 698255272Scarl{ 699255272Scarl int rc; 700255276Scarl uint8_t bar_size_bits = 0; 701255272Scarl 702289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 703289209Scem &bar->pci_resource_id, RF_ACTIVE); 704250079Scarl 705255272Scarl if (bar->pci_resource == NULL) 706255272Scarl return (ENXIO); 707255276Scarl 708289209Scem save_bar_parameters(bar); 709289209Scem /* 710289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 711289209Scem * hardware issue. To work around this, query the size it should be 712289209Scem * configured to by the device and modify the resource to correspond to 713289209Scem * this new size. The BIOS on systems with this problem is required to 714289209Scem * provide enough address space to allow the driver to make this change 715289209Scem * safely. 716289209Scem * 717289209Scem * Ideally I could have just specified the size when I allocated the 718289209Scem * resource like: 719289209Scem * bus_alloc_resource(ntb->device, 720289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 721289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 722289209Scem * but the PCI driver does not honor the size in this call, so we have 723289209Scem * to modify it after the fact. 724289209Scem */ 725289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 726289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 727289209Scem bar_size_bits = pci_read_config(ntb->device, 728289209Scem XEON_PBAR23SZ_OFFSET, 1); 729289209Scem else 730289209Scem bar_size_bits = pci_read_config(ntb->device, 731289209Scem XEON_PBAR45SZ_OFFSET, 1); 732289209Scem 733289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 734289209Scem bar->pci_resource, bar->pbase, 735289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 736255272Scarl if (rc != 0) { 737289209Scem device_printf(ntb->device, 738289209Scem "unable to resize bar\n"); 739255272Scarl return (rc); 740250079Scarl } 741289209Scem 742289209Scem save_bar_parameters(bar); 743250079Scarl } 744289209Scem 745289209Scem /* Mark bar region as write combining to improve performance. */ 746289209Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 747289209Scem VM_MEMATTR_WRITE_COMBINING); 748289209Scem if (rc != 0) { 749289209Scem device_printf(ntb->device, 750289209Scem "unable to mark bar as WRITE_COMBINING\n"); 751289209Scem return (rc); 752289209Scem } 753289541Scem print_map_success(ntb, bar); 754250079Scarl return (0); 755250079Scarl} 756250079Scarl 757250079Scarlstatic void 758250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 759250079Scarl{ 760250079Scarl struct ntb_pci_bar_info *current_bar; 761250079Scarl int i; 762250079Scarl 763289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 764250079Scarl current_bar = &ntb->bar_info[i]; 765250079Scarl if (current_bar->pci_resource != NULL) 766250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 767250079Scarl current_bar->pci_resource_id, 768250079Scarl current_bar->pci_resource); 769250079Scarl } 770250079Scarl} 771250079Scarl 772250079Scarlstatic int 773289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 774250079Scarl{ 775289342Scem uint32_t i; 776289342Scem int rc; 777289342Scem 778289342Scem for (i = 0; i < num_vectors; i++) { 779289342Scem ntb->int_info[i].rid = i + 1; 780289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 781289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 782289342Scem if (ntb->int_info[i].res == NULL) { 783289342Scem device_printf(ntb->device, 784289342Scem "bus_alloc_resource failed\n"); 785289342Scem return (ENOMEM); 786289342Scem } 787289342Scem ntb->int_info[i].tag = NULL; 788289342Scem ntb->allocated_interrupts++; 789289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 790289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 791289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 792289342Scem if (rc != 0) { 793289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 794289342Scem return (ENXIO); 795289342Scem } 796289342Scem } 797289342Scem return (0); 798289342Scem} 799289342Scem 800289344Scem/* 801289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 802289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 803289344Scem * should be okay. This tunable should enable us to test that hypothesis 804289344Scem * when someone gets their hands on some Xeon hardware. 805289344Scem */ 806289344Scemstatic int ntb_force_remap_mode; 807289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 808289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 809289344Scem " to a smaller number of ithreads, even if the desired number are " 810289344Scem "available"); 811289344Scem 812289344Scem/* 813289344Scem * In case it is NOT ok, give consumers an abort button. 814289344Scem */ 815289344Scemstatic int ntb_prefer_intx; 816289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 817289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 818289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 819289344Scem "behavior)"); 820289344Scem 821289344Scem/* 822289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 823289344Scem * round-robin fashion. 824289344Scem */ 825289342Scemstatic int 826289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 827289344Scem{ 828289344Scem u_int *vectors; 829289344Scem uint32_t i; 830289344Scem int rc; 831289344Scem 832289344Scem if (ntb_prefer_intx != 0) 833289344Scem return (ENXIO); 834289344Scem 835289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 836289344Scem 837289344Scem for (i = 0; i < desired; i++) 838289344Scem vectors[i] = (i % avail) + 1; 839289344Scem 840289344Scem rc = pci_remap_msix(dev, desired, vectors); 841289344Scem free(vectors, M_NTB); 842289344Scem return (rc); 843289344Scem} 844289344Scem 845289344Scemstatic int 846289540Scemntb_init_isr(struct ntb_softc *ntb) 847289342Scem{ 848289344Scem uint32_t desired_vectors, num_vectors; 849289342Scem int rc; 850250079Scarl 851250079Scarl ntb->allocated_interrupts = 0; 852289542Scem ntb->last_ts = ticks; 853289347Scem 854250079Scarl /* 855289546Scem * Mask all doorbell interrupts. 856250079Scarl */ 857289546Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 858250079Scarl 859289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 860289539Scem ntb->db_count); 861289344Scem if (desired_vectors >= 1) { 862289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 863250079Scarl 864289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 865289344Scem num_vectors == desired_vectors) 866289344Scem num_vectors--; 867289344Scem 868289344Scem if (rc == 0 && num_vectors < desired_vectors) { 869289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 870289344Scem num_vectors); 871289344Scem if (rc == 0) 872289344Scem num_vectors = desired_vectors; 873289344Scem else 874289344Scem pci_release_msi(ntb->device); 875289344Scem } 876289344Scem if (rc != 0) 877289344Scem num_vectors = 1; 878289344Scem } else 879289344Scem num_vectors = 1; 880289344Scem 881289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 882289539Scem ntb->db_vec_count = 1; 883289539Scem ntb->db_vec_shift = ntb->db_count; 884289539Scem rc = ntb_setup_legacy_interrupt(ntb); 885289539Scem } else { 886289546Scem ntb_create_msix_vec(ntb, num_vectors); 887289540Scem rc = ntb_setup_msix(ntb, num_vectors); 888289539Scem } 889289539Scem if (rc != 0) { 890289539Scem device_printf(ntb->device, 891289539Scem "Error allocating interrupts: %d\n", rc); 892289546Scem ntb_free_msix_vec(ntb); 893289396Scem } 894289396Scem 895289342Scem return (rc); 896289342Scem} 897289342Scem 898289342Scemstatic int 899289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 900289342Scem{ 901289342Scem int rc; 902289342Scem 903289342Scem ntb->int_info[0].rid = 0; 904289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 905289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 906289342Scem if (ntb->int_info[0].res == NULL) { 907289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 908289342Scem return (ENOMEM); 909250079Scarl } 910250079Scarl 911289342Scem ntb->int_info[0].tag = NULL; 912289342Scem ntb->allocated_interrupts = 1; 913289342Scem 914289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 915289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 916289342Scem ntb, &ntb->int_info[0].tag); 917289342Scem if (rc != 0) { 918289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 919289342Scem return (ENXIO); 920289342Scem } 921289342Scem 922250079Scarl return (0); 923250079Scarl} 924250079Scarl 925250079Scarlstatic void 926250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 927250079Scarl{ 928250079Scarl struct ntb_int_info *current_int; 929250079Scarl int i; 930250079Scarl 931289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 932250079Scarl current_int = &ntb->int_info[i]; 933250079Scarl if (current_int->tag != NULL) 934250079Scarl bus_teardown_intr(ntb->device, current_int->res, 935250079Scarl current_int->tag); 936250079Scarl 937250079Scarl if (current_int->res != NULL) 938250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 939250079Scarl rman_get_rid(current_int->res), current_int->res); 940250079Scarl } 941250079Scarl 942289546Scem ntb_free_msix_vec(ntb); 943250079Scarl pci_release_msi(ntb->device); 944250079Scarl} 945250079Scarl 946289347Scem/* 947289347Scem * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon. Abstract it 948289347Scem * out to make code clearer. 949289347Scem */ 950289539Scemstatic inline uint64_t 951289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 952289347Scem{ 953289347Scem 954289347Scem if (ntb->type == NTB_SOC) 955289347Scem return (ntb_reg_read(8, regoff)); 956289347Scem 957289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 958289347Scem 959289347Scem return (ntb_reg_read(2, regoff)); 960289347Scem} 961289347Scem 962289539Scemstatic inline void 963289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 964289347Scem{ 965289347Scem 966289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 967289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 968289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 969289542Scem (uintmax_t)ntb->db_valid_mask)); 970289542Scem 971289607Scem if (regoff == ntb->self_reg->db_mask) 972289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 973289542Scem 974289347Scem if (ntb->type == NTB_SOC) { 975289347Scem ntb_reg_write(8, regoff, val); 976289347Scem return; 977289347Scem } 978289347Scem 979289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 980289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 981289347Scem} 982289347Scem 983289546Scemvoid 984289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 985289542Scem{ 986289542Scem 987289546Scem DB_MASK_LOCK(ntb); 988289542Scem ntb->db_mask |= bits; 989289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 990289546Scem DB_MASK_UNLOCK(ntb); 991289542Scem} 992289542Scem 993289546Scemvoid 994289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 995289542Scem{ 996289542Scem 997289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 998289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 999289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1000289542Scem (uintmax_t)ntb->db_valid_mask)); 1001289542Scem 1002289546Scem DB_MASK_LOCK(ntb); 1003289542Scem ntb->db_mask &= ~bits; 1004289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1005289546Scem DB_MASK_UNLOCK(ntb); 1006289542Scem} 1007289542Scem 1008289546Scemuint64_t 1009289546Scemntb_db_read(struct ntb_softc *ntb) 1010289281Scem{ 1011289281Scem 1012289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 1013289281Scem} 1014289281Scem 1015289546Scemvoid 1016289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1017289281Scem{ 1018289281Scem 1019289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1020289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1021289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1022289546Scem (uintmax_t)ntb->db_valid_mask)); 1023289546Scem 1024289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1025289281Scem} 1026289281Scem 1027289540Scemstatic inline uint64_t 1028289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1029250079Scarl{ 1030289540Scem uint64_t shift, mask; 1031250079Scarl 1032289540Scem shift = ntb->db_vec_shift; 1033289540Scem mask = (1ull << shift) - 1; 1034289540Scem return (mask << (shift * db_vector)); 1035250079Scarl} 1036250079Scarl 1037250079Scarlstatic void 1038289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1039250079Scarl{ 1040289540Scem uint64_t vec_mask; 1041250079Scarl 1042289542Scem ntb->last_ts = ticks; 1043289546Scem vec_mask = ntb_vec_mask(ntb, vec); 1044250079Scarl 1045289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1046289546Scem if (ntb_poll_link(ntb)) 1047289546Scem ntb_link_event(ntb); 1048289540Scem } 1049289540Scem 1050289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1051289546Scem ntb_db_event(ntb, vec); 1052289546Scem} 1053250079Scarl 1054289546Scemstatic void 1055289546Scemndev_vec_isr(void *arg) 1056289546Scem{ 1057289546Scem struct ntb_vec *nvec = arg; 1058250079Scarl 1059289546Scem ntb_interrupt(nvec->ntb, nvec->num); 1060250079Scarl} 1061250079Scarl 1062250079Scarlstatic void 1063289546Scemndev_irq_isr(void *arg) 1064250079Scarl{ 1065289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1066289546Scem ntb_interrupt(arg, 0); 1067250079Scarl} 1068250079Scarl 1069250079Scarlstatic int 1070289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1071250079Scarl{ 1072289342Scem uint32_t i; 1073250079Scarl 1074289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1075250079Scarl M_ZERO | M_WAITOK); 1076250079Scarl for (i = 0; i < num_vectors; i++) { 1077289546Scem ntb->msix_vec[i].num = i; 1078289546Scem ntb->msix_vec[i].ntb = ntb; 1079250079Scarl } 1080250079Scarl 1081250079Scarl return (0); 1082250079Scarl} 1083250079Scarl 1084250079Scarlstatic void 1085289546Scemntb_free_msix_vec(struct ntb_softc *ntb) 1086250079Scarl{ 1087250079Scarl 1088289546Scem if (ntb->msix_vec == NULL) 1089289539Scem return; 1090289539Scem 1091289546Scem free(ntb->msix_vec, M_NTB); 1092289546Scem ntb->msix_vec = NULL; 1093250079Scarl} 1094250079Scarl 1095250079Scarlstatic struct ntb_hw_info * 1096250079Scarlntb_get_device_info(uint32_t device_id) 1097250079Scarl{ 1098250079Scarl struct ntb_hw_info *ep = pci_ids; 1099250079Scarl 1100250079Scarl while (ep->device_id) { 1101250079Scarl if (ep->device_id == device_id) 1102250079Scarl return (ep); 1103250079Scarl ++ep; 1104250079Scarl } 1105250079Scarl return (NULL); 1106250079Scarl} 1107250079Scarl 1108289272Scemstatic void 1109289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 1110250079Scarl{ 1111250079Scarl 1112289617Scem if (ntb->reg != NULL) 1113289617Scem ntb_link_disable(ntb); 1114250079Scarl} 1115250079Scarl 1116289397Scemstatic void 1117289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 1118289397Scem{ 1119289397Scem 1120289397Scem if (ntb->type == NTB_SOC) { 1121289539Scem ntb->mw_count = SOC_MW_COUNT; 1122289397Scem return; 1123289397Scem } 1124289397Scem 1125289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1126289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1127289397Scem else 1128289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1129289397Scem} 1130289397Scem 1131250079Scarlstatic int 1132289348Scemntb_detect_xeon(struct ntb_softc *ntb) 1133250079Scarl{ 1134289348Scem uint8_t ppd, conn_type; 1135250079Scarl 1136289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1137289348Scem ntb->ppd = ppd; 1138250079Scarl 1139289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1140289257Scem ntb->dev_type = NTB_DEV_USD; 1141289257Scem else 1142289257Scem ntb->dev_type = NTB_DEV_DSD; 1143289257Scem 1144289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1145289397Scem ntb->features |= NTB_SPLIT_BAR; 1146289397Scem 1147289542Scem /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1148289542Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1149289542Scem ntb->features |= NTB_SDOORBELL_LOCKUP; 1150289542Scem 1151289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1152289348Scem switch (conn_type) { 1153289348Scem case NTB_CONN_B2B: 1154289348Scem ntb->conn_type = conn_type; 1155289348Scem break; 1156289348Scem case NTB_CONN_RP: 1157289348Scem case NTB_CONN_TRANSPARENT: 1158289348Scem default: 1159289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1160289348Scem (unsigned)conn_type); 1161289348Scem return (ENXIO); 1162289348Scem } 1163289348Scem return (0); 1164289348Scem} 1165289348Scem 1166289348Scemstatic int 1167289348Scemntb_detect_soc(struct ntb_softc *ntb) 1168289348Scem{ 1169289348Scem uint32_t ppd, conn_type; 1170289348Scem 1171289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1172289348Scem ntb->ppd = ppd; 1173289348Scem 1174289348Scem if ((ppd & SOC_PPD_DEV_TYPE) != 0) 1175289348Scem ntb->dev_type = NTB_DEV_DSD; 1176289348Scem else 1177289348Scem ntb->dev_type = NTB_DEV_USD; 1178289348Scem 1179289348Scem conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8; 1180289348Scem switch (conn_type) { 1181289348Scem case NTB_CONN_B2B: 1182289348Scem ntb->conn_type = conn_type; 1183289348Scem break; 1184289348Scem default: 1185289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1186289348Scem return (ENXIO); 1187289348Scem } 1188289348Scem return (0); 1189289348Scem} 1190289348Scem 1191289348Scemstatic int 1192289542Scemntb_xeon_init_dev(struct ntb_softc *ntb) 1193289348Scem{ 1194289542Scem int rc; 1195289348Scem 1196289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1197289542Scem ntb->db_count = XEON_DB_COUNT; 1198289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1199289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1200289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1201289257Scem 1202289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1203250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1204289348Scem ntb->conn_type); 1205250079Scarl return (ENXIO); 1206250079Scarl } 1207250079Scarl 1208289542Scem ntb->reg = &xeon_reg; 1209289607Scem ntb->self_reg = &xeon_pri_reg; 1210289542Scem ntb->peer_reg = &xeon_b2b_reg; 1211289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1212289542Scem 1213289208Scem /* 1214289208Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1215289208Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1216289208Scem * which may hang the system. To workaround this use the second memory 1217289208Scem * window to access the interrupt and scratch pad registers on the 1218289208Scem * remote system. 1219289208Scem */ 1220289543Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1221289543Scem /* Use the last MW for mapping remote spad */ 1222289542Scem ntb->b2b_mw_idx = ntb->mw_count - 1; 1223289543Scem else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1224289208Scem /* 1225289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1226289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1227289542Scem * since bit 14 is the last bit. 1228289542Scem * 1229289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1230289542Scem * anyway. Nor for non-B2B connection types. 1231289542Scem */ 1232289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1233250079Scarl 1234289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1235250079Scarl 1236289542Scem if (ntb->dev_type == NTB_DEV_USD) 1237289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1238289542Scem &xeon_b2b_usd_addr); 1239289542Scem else 1240289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1241289542Scem &xeon_b2b_dsd_addr); 1242289542Scem if (rc != 0) 1243289542Scem return (rc); 1244289271Scem 1245250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1246289607Scem ntb_reg_write(2, XEON_PCICMD_OFFSET, 1247289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1248255279Scarl 1249255269Scarl /* Enable link training */ 1250289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1251250079Scarl 1252250079Scarl return (0); 1253250079Scarl} 1254250079Scarl 1255250079Scarlstatic int 1256289542Scemntb_soc_init_dev(struct ntb_softc *ntb) 1257250079Scarl{ 1258250079Scarl 1259289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1260289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1261250079Scarl 1262289542Scem ntb->spad_count = SOC_SPAD_COUNT; 1263289539Scem ntb->db_count = SOC_DB_COUNT; 1264289539Scem ntb->db_vec_count = SOC_DB_MSIX_VECTOR_COUNT; 1265289539Scem ntb->db_vec_shift = SOC_DB_MSIX_VECTOR_SHIFT; 1266289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1267250079Scarl 1268289542Scem ntb->reg = &soc_reg; 1269289607Scem ntb->self_reg = &soc_pri_reg; 1270289542Scem ntb->peer_reg = &soc_b2b_reg; 1271289542Scem ntb->xlat_reg = &soc_sec_xlat; 1272289542Scem 1273250079Scarl /* 1274250079Scarl * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 1275250079Scarl * resolved. Mask transaction layer internal parity errors. 1276250079Scarl */ 1277250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1278250079Scarl 1279255279Scarl configure_soc_secondary_side_bars(ntb); 1280250079Scarl 1281250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1282289607Scem ntb_reg_write(2, SOC_PCICMD_OFFSET, 1283250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1284289209Scem 1285289542Scem /* Initiate PCI-E link training */ 1286289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1287250079Scarl 1288289542Scem callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb); 1289289542Scem 1290250079Scarl return (0); 1291250079Scarl} 1292250079Scarl 1293289542Scem/* XXX: Linux driver doesn't seem to do any of this for SoC. */ 1294255279Scarlstatic void 1295255279Scarlconfigure_soc_secondary_side_bars(struct ntb_softc *ntb) 1296255279Scarl{ 1297255279Scarl 1298255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1299289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1300289542Scem XEON_B2B_BAR2_DSD_ADDR64); 1301289542Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1302289542Scem XEON_B2B_BAR4_DSD_ADDR64); 1303289542Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64); 1304289542Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64); 1305255279Scarl } else { 1306289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1307289542Scem XEON_B2B_BAR2_USD_ADDR64); 1308289542Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1309289542Scem XEON_B2B_BAR4_USD_ADDR64); 1310289542Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64); 1311289542Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64); 1312255279Scarl } 1313255279Scarl} 1314255279Scarl 1315289543Scem 1316289543Scem/* 1317289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1318289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1319289543Scem * remains for use by a higher layer. 1320289543Scem * 1321289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1322289543Scem * MW size is sufficiently large. 1323289543Scem */ 1324289543Scemstatic unsigned int ntb_b2b_mw_share; 1325289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1326289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1327289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1328289543Scem "value here."); 1329289543Scem 1330289543Scemstatic void 1331289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1332289543Scem enum ntb_bar regbar) 1333289543Scem{ 1334289543Scem struct ntb_pci_bar_info *bar; 1335289543Scem uint8_t bar_sz; 1336289543Scem 1337289543Scem if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1338289543Scem return; 1339289543Scem 1340289543Scem bar = &ntb->bar_info[idx]; 1341289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1342289543Scem if (idx == regbar) { 1343289543Scem if (ntb->b2b_off != 0) 1344289543Scem bar_sz--; 1345289543Scem else 1346289543Scem bar_sz = 0; 1347289543Scem } 1348289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1349289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1350289543Scem (void)bar_sz; 1351289543Scem} 1352289543Scem 1353289543Scemstatic void 1354289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1355289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1356289543Scem{ 1357289546Scem uint64_t reg_val; 1358289546Scem uint32_t base_reg, lmt_reg; 1359289543Scem 1360289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1361289546Scem if (idx == regbar) 1362289546Scem bar_addr += ntb->b2b_off; 1363289543Scem 1364289546Scem if (!bar_is_64bit(ntb, idx)) { 1365289546Scem ntb_reg_write(4, base_reg, bar_addr); 1366289546Scem reg_val = ntb_reg_read(4, base_reg); 1367289546Scem (void)reg_val; 1368289546Scem 1369289546Scem ntb_reg_write(4, lmt_reg, bar_addr); 1370289546Scem reg_val = ntb_reg_read(4, lmt_reg); 1371289546Scem (void)reg_val; 1372289543Scem } else { 1373289546Scem ntb_reg_write(8, base_reg, bar_addr); 1374289546Scem reg_val = ntb_reg_read(8, base_reg); 1375289546Scem (void)reg_val; 1376289546Scem 1377289546Scem ntb_reg_write(8, lmt_reg, bar_addr); 1378289546Scem reg_val = ntb_reg_read(8, lmt_reg); 1379289546Scem (void)reg_val; 1380289543Scem } 1381289543Scem} 1382289543Scem 1383289543Scemstatic void 1384289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1385289543Scem{ 1386289543Scem struct ntb_pci_bar_info *bar; 1387289543Scem 1388289543Scem bar = &ntb->bar_info[idx]; 1389289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1390289543Scem ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1391289543Scem base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1392289543Scem } else { 1393289543Scem ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1394289543Scem base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1395289543Scem } 1396289543Scem (void)base_addr; 1397289543Scem} 1398289543Scem 1399289542Scemstatic int 1400289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1401289542Scem const struct ntb_b2b_addr *peer_addr) 1402255279Scarl{ 1403289543Scem struct ntb_pci_bar_info *b2b_bar; 1404289543Scem vm_size_t bar_size; 1405289543Scem uint64_t bar_addr; 1406289543Scem enum ntb_bar b2b_bar_num, i; 1407255279Scarl 1408289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1409289543Scem b2b_bar = NULL; 1410289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1411289543Scem ntb->b2b_off = 0; 1412289543Scem } else { 1413289543Scem b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1414289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1415289543Scem ("invalid b2b mw bar")); 1416289543Scem 1417289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1418289543Scem bar_size = b2b_bar->size; 1419289543Scem 1420289543Scem if (ntb_b2b_mw_share != 0 && 1421289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1422289543Scem ntb->b2b_off = bar_size >> 1; 1423289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1424289543Scem ntb->b2b_off = 0; 1425289543Scem ntb->mw_count--; 1426289543Scem } else { 1427289543Scem device_printf(ntb->device, 1428289543Scem "B2B bar size is too small!\n"); 1429289543Scem return (EIO); 1430289543Scem } 1431255279Scarl } 1432289542Scem 1433289543Scem /* 1434289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1435289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1436289543Scem */ 1437289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1438289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1439289543Scem 1440289543Scem bar_addr = 0; 1441289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1442289543Scem bar_addr = addr->bar0_addr; 1443289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1444289543Scem bar_addr = addr->bar2_addr64; 1445289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1446289543Scem bar_addr = addr->bar4_addr64; 1447289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1448289543Scem bar_addr = addr->bar4_addr32; 1449289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1450289543Scem bar_addr = addr->bar5_addr32; 1451289543Scem else 1452289543Scem KASSERT(false, ("invalid bar")); 1453289543Scem 1454289543Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1455289543Scem 1456289543Scem /* 1457289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1458289543Scem * register BAR. The B2B BAR is either disabled above or configured 1459289543Scem * half-size. It starts at PBAR xlat + offset. 1460289543Scem * 1461289543Scem * Also set up incoming BAR limits == base (zero length window). 1462289543Scem */ 1463289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1464289543Scem b2b_bar_num); 1465289542Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1466289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1467289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1468289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1469289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1470289542Scem } else 1471289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1472289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1473289543Scem 1474289543Scem /* Zero incoming translation addrs */ 1475289543Scem ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1476289543Scem ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1477289543Scem 1478289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1479289543Scem ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1480289543Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1481289543Scem 1482289543Scem /* Set outgoing translation offsets */ 1483289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1484289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1485289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1486289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1487289543Scem } else 1488289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1489289543Scem 1490289543Scem /* Set the translation offset for B2B registers */ 1491289543Scem bar_addr = 0; 1492289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1493289543Scem bar_addr = peer_addr->bar0_addr; 1494289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1495289543Scem bar_addr = peer_addr->bar2_addr64; 1496289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1497289543Scem bar_addr = peer_addr->bar4_addr64; 1498289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1499289543Scem bar_addr = peer_addr->bar4_addr32; 1500289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1501289543Scem bar_addr = peer_addr->bar5_addr32; 1502289543Scem else 1503289543Scem KASSERT(false, ("invalid bar")); 1504289543Scem 1505289543Scem /* 1506289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1507289543Scem * at a time. 1508289543Scem */ 1509289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1510289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1511289542Scem return (0); 1512255279Scarl} 1513255279Scarl 1514289546Scemstatic inline bool 1515289546Scemlink_is_up(struct ntb_softc *ntb) 1516289546Scem{ 1517289546Scem 1518289611Scem if (ntb->type == NTB_XEON) { 1519289611Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1520289611Scem return (true); 1521289546Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1522289611Scem } 1523289546Scem 1524289546Scem KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1525289546Scem return ((ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0); 1526289546Scem} 1527289546Scem 1528289546Scemstatic inline bool 1529289546Scemsoc_link_is_err(struct ntb_softc *ntb) 1530289546Scem{ 1531289546Scem uint32_t status; 1532289546Scem 1533289546Scem KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1534289546Scem 1535289546Scem status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1536289546Scem if ((status & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1537289546Scem return (true); 1538289546Scem 1539289546Scem status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1540289546Scem return ((status & SOC_IBIST_ERR_OFLOW) != 0); 1541289546Scem} 1542289546Scem 1543255281Scarl/* SOC does not have link status interrupt, poll on that platform */ 1544250079Scarlstatic void 1545289542Scemsoc_link_hb(void *arg) 1546250079Scarl{ 1547250079Scarl struct ntb_softc *ntb = arg; 1548289546Scem sbintime_t timo, poll_ts; 1549250079Scarl 1550289546Scem timo = NTB_HB_TIMEOUT * hz; 1551289546Scem poll_ts = ntb->last_ts + timo; 1552289546Scem 1553289542Scem /* 1554289542Scem * Delay polling the link status if an interrupt was received, unless 1555289542Scem * the cached link status says the link is down. 1556289542Scem */ 1557289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1558289546Scem timo = poll_ts - ticks; 1559289542Scem goto out; 1560289546Scem } 1561289542Scem 1562289546Scem if (ntb_poll_link(ntb)) 1563289546Scem ntb_link_event(ntb); 1564289542Scem 1565289546Scem if (!link_is_up(ntb) && soc_link_is_err(ntb)) { 1566289546Scem /* Link is down with error, proceed with recovery */ 1567289546Scem callout_reset(&ntb->lr_timer, 0, recover_soc_link, ntb); 1568289546Scem return; 1569250079Scarl } 1570250079Scarl 1571289542Scemout: 1572289546Scem callout_reset(&ntb->heartbeat_timer, timo, soc_link_hb, ntb); 1573250079Scarl} 1574250079Scarl 1575250079Scarlstatic void 1576250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb) 1577250079Scarl{ 1578250079Scarl uint32_t status; 1579250079Scarl 1580250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1581255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1582255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1583255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1584255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1585250079Scarl 1586250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1587250079Scarl pause("ModPhy", hz / 10); 1588250079Scarl 1589250079Scarl /* Clear AER Errors, write to clear */ 1590255278Scarl status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1591250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1592255278Scarl ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1593250079Scarl 1594250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1595255278Scarl status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1596250079Scarl status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1597255278Scarl ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1598250079Scarl 1599250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1600255278Scarl status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1601250079Scarl status |= SOC_DESKEWSTS_DBERR; 1602255278Scarl ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1603250079Scarl 1604255278Scarl status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1605250079Scarl status &= SOC_IBIST_ERR_OFLOW; 1606255278Scarl ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1607250079Scarl 1608250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1609255278Scarl status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1610250079Scarl status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1611255278Scarl ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1612250079Scarl} 1613250079Scarl 1614289546Scem/* 1615289546Scem * ntb_set_ctx() - associate a driver context with an ntb device 1616289546Scem * @ntb: NTB device context 1617289546Scem * @ctx: Driver context 1618289546Scem * @ctx_ops: Driver context operations 1619289546Scem * 1620289546Scem * Associate a driver context and operations with a ntb device. The context is 1621289546Scem * provided by the client driver, and the driver may associate a different 1622289546Scem * context with each ntb device. 1623289546Scem * 1624289546Scem * Return: Zero if the context is associated, otherwise an error number. 1625289546Scem */ 1626289546Scemint 1627289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 1628250079Scarl{ 1629250079Scarl 1630289546Scem if (ctx == NULL || ops == NULL) 1631289546Scem return (EINVAL); 1632289546Scem if (ntb->ctx_ops != NULL) 1633289546Scem return (EINVAL); 1634250079Scarl 1635289546Scem CTX_LOCK(ntb); 1636289546Scem if (ntb->ctx_ops != NULL) { 1637289546Scem CTX_UNLOCK(ntb); 1638289546Scem return (EINVAL); 1639250079Scarl } 1640289546Scem ntb->ntb_ctx = ctx; 1641289546Scem ntb->ctx_ops = ops; 1642289546Scem CTX_UNLOCK(ntb); 1643250079Scarl 1644289546Scem return (0); 1645250079Scarl} 1646250079Scarl 1647289546Scem/* 1648289546Scem * It is expected that this will only be used from contexts where the ctx_lock 1649289546Scem * is not needed to protect ntb_ctx lifetime. 1650289546Scem */ 1651289546Scemvoid * 1652289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 1653289546Scem{ 1654289546Scem 1655289546Scem KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 1656289546Scem if (ops != NULL) 1657289546Scem *ops = ntb->ctx_ops; 1658289546Scem return (ntb->ntb_ctx); 1659289546Scem} 1660289546Scem 1661289546Scem/* 1662289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device 1663289546Scem * @ntb: NTB device context 1664289546Scem * 1665289546Scem * Clear any association that may exist between a driver context and the ntb 1666289546Scem * device. 1667289546Scem */ 1668289546Scemvoid 1669289546Scemntb_clear_ctx(struct ntb_softc *ntb) 1670289546Scem{ 1671289546Scem 1672289546Scem CTX_LOCK(ntb); 1673289546Scem ntb->ntb_ctx = NULL; 1674289546Scem ntb->ctx_ops = NULL; 1675289546Scem CTX_UNLOCK(ntb); 1676289546Scem} 1677289546Scem 1678289546Scem/* 1679289546Scem * ntb_link_event() - notify driver context of a change in link status 1680289546Scem * @ntb: NTB device context 1681289546Scem * 1682289546Scem * Notify the driver context that the link status may have changed. The driver 1683289546Scem * should call ntb_link_is_up() to get the current status. 1684289546Scem */ 1685289546Scemvoid 1686289546Scemntb_link_event(struct ntb_softc *ntb) 1687289546Scem{ 1688289546Scem 1689289546Scem CTX_LOCK(ntb); 1690289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 1691289546Scem ntb->ctx_ops->link_event(ntb->ntb_ctx); 1692289546Scem CTX_UNLOCK(ntb); 1693289546Scem} 1694289546Scem 1695289546Scem/* 1696289546Scem * ntb_db_event() - notify driver context of a doorbell event 1697289546Scem * @ntb: NTB device context 1698289546Scem * @vector: Interrupt vector number 1699289546Scem * 1700289546Scem * Notify the driver context of a doorbell event. If hardware supports 1701289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which 1702289546Scem * vector received the interrupt. The vector number is relative to the first 1703289546Scem * vector used for doorbells, starting at zero, and must be less than 1704289546Scem * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 1705289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of 1706289546Scem * those bits are associated with the vector number. 1707289546Scem */ 1708250079Scarlstatic void 1709289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec) 1710289272Scem{ 1711289546Scem 1712289546Scem CTX_LOCK(ntb); 1713289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 1714289546Scem ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 1715289546Scem CTX_UNLOCK(ntb); 1716289546Scem} 1717289546Scem 1718289546Scem/* 1719289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb 1720289546Scem * @ntb: NTB device context 1721289546Scem * @max_speed: The maximum link speed expressed as PCIe generation number[0] 1722289546Scem * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 1723289546Scem * 1724289546Scem * Enable the link on the secondary side of the ntb. This can only be done 1725289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1726289546Scem * should train the link to its maximum speed and width, or the requested speed 1727289546Scem * and width, whichever is smaller, if supported. 1728289546Scem * 1729289546Scem * Return: Zero on success, otherwise an error number. 1730289546Scem * 1731289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 1732289546Scem * and width input will be ignored. 1733289546Scem */ 1734289546Scemint 1735289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 1736289546Scem enum ntb_width w __unused) 1737289546Scem{ 1738289280Scem uint32_t cntl; 1739289272Scem 1740289542Scem if (ntb->type == NTB_SOC) { 1741289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1742289542Scem ntb->ppd | SOC_PPD_INIT_LINK, 4); 1743289546Scem return (0); 1744289542Scem } 1745289542Scem 1746289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1747289546Scem ntb_link_event(ntb); 1748289546Scem return (0); 1749289280Scem } 1750289280Scem 1751289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1752289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1753289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1754289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1755289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1756289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1757289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1758289546Scem return (0); 1759289272Scem} 1760289272Scem 1761289546Scem/* 1762289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb 1763289546Scem * @ntb: NTB device context 1764289546Scem * 1765289546Scem * Disable the link on the secondary side of the ntb. This can only be done 1766289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1767289546Scem * should disable the link. Returning from this call must indicate that a 1768289546Scem * barrier has passed, though with no more writes may pass in either direction 1769289546Scem * across the link, except if this call returns an error number. 1770289546Scem * 1771289546Scem * Return: Zero on success, otherwise an error number. 1772289546Scem */ 1773289546Scemint 1774289542Scemntb_link_disable(struct ntb_softc *ntb) 1775289272Scem{ 1776289272Scem uint32_t cntl; 1777289272Scem 1778289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1779289546Scem ntb_link_event(ntb); 1780289546Scem return (0); 1781289272Scem } 1782289272Scem 1783289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1784289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1785289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1786289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1787289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1788289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1789289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1790289546Scem return (0); 1791289272Scem} 1792289272Scem 1793289272Scemstatic void 1794250079Scarlrecover_soc_link(void *arg) 1795250079Scarl{ 1796250079Scarl struct ntb_softc *ntb = arg; 1797289608Scem unsigned speed, width, oldspeed, oldwidth; 1798250079Scarl uint32_t status32; 1799250079Scarl 1800250079Scarl soc_perform_link_restart(ntb); 1801250079Scarl 1802289232Scem /* 1803289232Scem * There is a potential race between the 2 NTB devices recovering at 1804289232Scem * the same time. If the times are the same, the link will not recover 1805289232Scem * and the driver will be stuck in this loop forever. Add a random 1806289232Scem * interval to the recovery time to prevent this race. 1807289232Scem */ 1808289232Scem status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1809289232Scem pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1810289232Scem 1811289609Scem if (soc_link_is_err(ntb)) 1812250079Scarl goto retry; 1813250079Scarl 1814289542Scem status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1815289232Scem if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1816289232Scem goto out; 1817289232Scem 1818289542Scem status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1819289608Scem width = NTB_LNK_STA_WIDTH(status32); 1820289608Scem speed = status32 & NTB_LINK_SPEED_MASK; 1821289608Scem 1822289608Scem oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 1823289608Scem oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 1824289608Scem if (oldwidth != width || oldspeed != speed) 1825250079Scarl goto retry; 1826250079Scarl 1827289232Scemout: 1828289542Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, 1829289542Scem ntb); 1830250079Scarl return; 1831250079Scarl 1832250079Scarlretry: 1833250079Scarl callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1834250079Scarl ntb); 1835250079Scarl} 1836250079Scarl 1837289546Scem/* 1838289546Scem * Polls the HW link status register(s); returns true if something has changed. 1839289546Scem */ 1840289546Scemstatic bool 1841289542Scemntb_poll_link(struct ntb_softc *ntb) 1842250079Scarl{ 1843250079Scarl uint32_t ntb_cntl; 1844289546Scem uint16_t reg_val; 1845250079Scarl 1846250079Scarl if (ntb->type == NTB_SOC) { 1847289542Scem ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1848289546Scem if (ntb_cntl == ntb->ntb_ctl) 1849289546Scem return (false); 1850289546Scem 1851289542Scem ntb->ntb_ctl = ntb_cntl; 1852289542Scem ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1853250079Scarl } else { 1854289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 1855250079Scarl 1856289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 1857289546Scem if (reg_val == ntb->lnk_sta) 1858289546Scem return (false); 1859250079Scarl 1860289546Scem ntb->lnk_sta = reg_val; 1861289542Scem } 1862289546Scem return (true); 1863289542Scem} 1864289542Scem 1865289546Scemstatic inline enum ntb_speed 1866289546Scemntb_link_sta_speed(struct ntb_softc *ntb) 1867250079Scarl{ 1868250079Scarl 1869289546Scem if (!link_is_up(ntb)) 1870289546Scem return (NTB_SPEED_NONE); 1871289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 1872250079Scarl} 1873250079Scarl 1874289546Scemstatic inline enum ntb_width 1875289546Scemntb_link_sta_width(struct ntb_softc *ntb) 1876250079Scarl{ 1877250079Scarl 1878289546Scem if (!link_is_up(ntb)) 1879289546Scem return (NTB_WIDTH_NONE); 1880289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 1881250079Scarl} 1882250079Scarl 1883289546Scem/* 1884289546Scem * Public API to the rest of the OS 1885250079Scarl */ 1886250079Scarl 1887250079Scarl/** 1888250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 1889250079Scarl * @ntb: pointer to ntb_softc instance 1890250079Scarl * 1891250079Scarl * This function returns the max 32bit scratchpad registers usable by the 1892250079Scarl * upper layer. 1893250079Scarl * 1894250079Scarl * RETURNS: total number of scratch pad registers available 1895250079Scarl */ 1896289208Scemuint8_t 1897250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 1898250079Scarl{ 1899250079Scarl 1900289539Scem return (ntb->spad_count); 1901250079Scarl} 1902250079Scarl 1903289396Scemuint8_t 1904289539Scemntb_mw_count(struct ntb_softc *ntb) 1905289396Scem{ 1906289396Scem 1907289539Scem return (ntb->mw_count); 1908289396Scem} 1909289396Scem 1910250079Scarl/** 1911289545Scem * ntb_spad_write() - write to the secondary scratchpad register 1912250079Scarl * @ntb: pointer to ntb_softc instance 1913250079Scarl * @idx: index to the scratchpad register, 0 based 1914250079Scarl * @val: the data value to put into the register 1915250079Scarl * 1916250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1917250079Scarl * register. The register resides on the secondary (external) side. 1918250079Scarl * 1919289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1920250079Scarl */ 1921250079Scarlint 1922289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1923250079Scarl{ 1924250079Scarl 1925289539Scem if (idx >= ntb->spad_count) 1926250079Scarl return (EINVAL); 1927250079Scarl 1928289607Scem ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 1929250079Scarl 1930250079Scarl return (0); 1931250079Scarl} 1932250079Scarl 1933250079Scarl/** 1934289545Scem * ntb_spad_read() - read from the primary scratchpad register 1935250079Scarl * @ntb: pointer to ntb_softc instance 1936250079Scarl * @idx: index to scratchpad register, 0 based 1937250079Scarl * @val: pointer to 32bit integer for storing the register value 1938250079Scarl * 1939250079Scarl * This function allows reading of the 32bit scratchpad register on 1940250079Scarl * the primary (internal) side. 1941250079Scarl * 1942289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1943250079Scarl */ 1944250079Scarlint 1945289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1946250079Scarl{ 1947250079Scarl 1948289539Scem if (idx >= ntb->spad_count) 1949250079Scarl return (EINVAL); 1950250079Scarl 1951289607Scem *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 1952250079Scarl 1953250079Scarl return (0); 1954250079Scarl} 1955250079Scarl 1956250079Scarl/** 1957289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register 1958250079Scarl * @ntb: pointer to ntb_softc instance 1959250079Scarl * @idx: index to the scratchpad register, 0 based 1960250079Scarl * @val: the data value to put into the register 1961250079Scarl * 1962250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1963250079Scarl * register. The register resides on the secondary (external) side. 1964250079Scarl * 1965289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1966250079Scarl */ 1967250079Scarlint 1968289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1969250079Scarl{ 1970250079Scarl 1971289539Scem if (idx >= ntb->spad_count) 1972250079Scarl return (EINVAL); 1973250079Scarl 1974289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1975255279Scarl ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1976255279Scarl else 1977289542Scem ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 1978250079Scarl 1979250079Scarl return (0); 1980250079Scarl} 1981250079Scarl 1982250079Scarl/** 1983289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register 1984250079Scarl * @ntb: pointer to ntb_softc instance 1985250079Scarl * @idx: index to scratchpad register, 0 based 1986250079Scarl * @val: pointer to 32bit integer for storing the register value 1987250079Scarl * 1988250079Scarl * This function allows reading of the 32bit scratchpad register on 1989250079Scarl * the primary (internal) side. 1990250079Scarl * 1991289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1992250079Scarl */ 1993250079Scarlint 1994289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1995250079Scarl{ 1996250079Scarl 1997289539Scem if (idx >= ntb->spad_count) 1998250079Scarl return (EINVAL); 1999250079Scarl 2000289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2001255279Scarl *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 2002255279Scarl else 2003289542Scem *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2004250079Scarl 2005250079Scarl return (0); 2006250079Scarl} 2007250079Scarl 2008289546Scem/* 2009289546Scem * ntb_mw_get_range() - get the range of a memory window 2010289546Scem * @ntb: NTB device context 2011289546Scem * @idx: Memory window number 2012289546Scem * @base: OUT - the base address for mapping the memory window 2013289546Scem * @size: OUT - the size for mapping the memory window 2014289546Scem * @align: OUT - the base alignment for translating the memory window 2015289546Scem * @align_size: OUT - the size alignment for translating the memory window 2016250079Scarl * 2017289546Scem * Get the range of a memory window. NULL may be given for any output 2018289546Scem * parameter if the value is not needed. The base and size may be used for 2019289546Scem * mapping the memory window, to access the peer memory. The alignment and 2020289546Scem * size may be used for translating the memory window, for the peer to access 2021289546Scem * memory on the local system. 2022250079Scarl * 2023289546Scem * Return: Zero on success, otherwise an error number. 2024250079Scarl */ 2025289546Scemint 2026289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 2027289546Scem void **vbase, size_t *size, size_t *align, size_t *align_size) 2028250079Scarl{ 2029289546Scem struct ntb_pci_bar_info *bar; 2030289546Scem size_t bar_b2b_off; 2031250079Scarl 2032289546Scem if (mw_idx >= ntb_mw_count(ntb)) 2033289546Scem return (EINVAL); 2034250079Scarl 2035289546Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)]; 2036289546Scem bar_b2b_off = 0; 2037289546Scem if (mw_idx == ntb->b2b_mw_idx) { 2038289546Scem KASSERT(ntb->b2b_off != 0, 2039289546Scem ("user shouldn't get non-shared b2b mw")); 2040289546Scem bar_b2b_off = ntb->b2b_off; 2041289546Scem } 2042250079Scarl 2043289546Scem if (base != NULL) 2044289546Scem *base = bar->pbase + bar_b2b_off; 2045289546Scem if (vbase != NULL) 2046289546Scem *vbase = (char *)bar->vbase + bar_b2b_off; 2047289546Scem if (size != NULL) 2048289546Scem *size = bar->size - bar_b2b_off; 2049289546Scem if (align != NULL) 2050289546Scem *align = bar->size; 2051289546Scem if (align_size != NULL) 2052289546Scem *align_size = 1; 2053289546Scem return (0); 2054250079Scarl} 2055250079Scarl 2056289546Scem/* 2057289546Scem * ntb_mw_set_trans() - set the translation of a memory window 2058289546Scem * @ntb: NTB device context 2059289546Scem * @idx: Memory window number 2060289546Scem * @addr: The dma address local memory to expose to the peer 2061289546Scem * @size: The size of the local memory to expose to the peer 2062250079Scarl * 2063289546Scem * Set the translation of a memory window. The peer may access local memory 2064289546Scem * through the window starting at the address, up to the size. The address 2065289546Scem * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2066289546Scem * must be aligned to the size alignment specified by ntb_mw_get_range(). 2067250079Scarl * 2068289546Scem * Return: Zero on success, otherwise an error number. 2069250079Scarl */ 2070289546Scemint 2071289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2072289546Scem size_t size) 2073250079Scarl{ 2074289546Scem struct ntb_pci_bar_info *bar; 2075289546Scem uint64_t base, limit, reg_val; 2076289546Scem size_t bar_size, mw_size; 2077289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2078289546Scem enum ntb_bar bar_num; 2079250079Scarl 2080289546Scem if (idx >= ntb_mw_count(ntb)) 2081289546Scem return (EINVAL); 2082250079Scarl 2083289546Scem bar_num = ntb_mw_to_bar(ntb, idx); 2084289546Scem bar = &ntb->bar_info[bar_num]; 2085250079Scarl 2086289546Scem bar_size = bar->size; 2087289546Scem if (idx == ntb->b2b_mw_idx) 2088289546Scem mw_size = bar_size - ntb->b2b_off; 2089289546Scem else 2090289546Scem mw_size = bar_size; 2091250079Scarl 2092289546Scem /* Hardware requires that addr is aligned to bar size */ 2093289546Scem if ((addr & (bar_size - 1)) != 0) 2094289546Scem return (EINVAL); 2095250079Scarl 2096289546Scem if (size > mw_size) 2097289546Scem return (EINVAL); 2098289546Scem 2099289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2100289546Scem 2101289546Scem limit = 0; 2102289546Scem if (bar_is_64bit(ntb, bar_num)) { 2103289546Scem base = ntb_reg_read(8, base_reg); 2104289546Scem 2105289546Scem if (limit_reg != 0 && size != mw_size) 2106289546Scem limit = base + size; 2107289546Scem 2108289546Scem /* Set and verify translation address */ 2109289546Scem ntb_reg_write(8, xlat_reg, addr); 2110289546Scem reg_val = ntb_reg_read(8, xlat_reg); 2111289546Scem if (reg_val != addr) { 2112289546Scem ntb_reg_write(8, xlat_reg, 0); 2113289546Scem return (EIO); 2114289546Scem } 2115289546Scem 2116289546Scem /* Set and verify the limit */ 2117289546Scem ntb_reg_write(8, limit_reg, limit); 2118289546Scem reg_val = ntb_reg_read(8, limit_reg); 2119289546Scem if (reg_val != limit) { 2120289546Scem ntb_reg_write(8, limit_reg, base); 2121289546Scem ntb_reg_write(8, xlat_reg, 0); 2122289546Scem return (EIO); 2123289546Scem } 2124289546Scem } else { 2125289546Scem /* Configure 32-bit (split) BAR MW */ 2126289546Scem 2127289546Scem if ((addr & ~UINT32_MAX) != 0) 2128289546Scem return (EINVAL); 2129289546Scem if (((addr + size) & ~UINT32_MAX) != 0) 2130289546Scem return (EINVAL); 2131289546Scem 2132289546Scem base = ntb_reg_read(4, base_reg); 2133289546Scem 2134289546Scem if (limit_reg != 0 && size != mw_size) 2135289546Scem limit = base + size; 2136289546Scem 2137289546Scem /* Set and verify translation address */ 2138289546Scem ntb_reg_write(4, xlat_reg, addr); 2139289546Scem reg_val = ntb_reg_read(4, xlat_reg); 2140289546Scem if (reg_val != addr) { 2141289546Scem ntb_reg_write(4, xlat_reg, 0); 2142289546Scem return (EIO); 2143289546Scem } 2144289546Scem 2145289546Scem /* Set and verify the limit */ 2146289546Scem ntb_reg_write(4, limit_reg, limit); 2147289546Scem reg_val = ntb_reg_read(4, limit_reg); 2148289546Scem if (reg_val != limit) { 2149289546Scem ntb_reg_write(4, limit_reg, base); 2150289546Scem ntb_reg_write(4, xlat_reg, 0); 2151289546Scem return (EIO); 2152289546Scem } 2153250079Scarl } 2154289546Scem return (0); 2155250079Scarl} 2156250079Scarl 2157289596Scem/* 2158289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window 2159289596Scem * @ntb: NTB device context 2160289596Scem * @idx: Memory window number 2161289596Scem * 2162289596Scem * Clear the translation of a memory window. The peer may no longer access 2163289596Scem * local memory through the window. 2164289596Scem * 2165289596Scem * Return: Zero on success, otherwise an error number. 2166289596Scem */ 2167289596Scemint 2168289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 2169289596Scem{ 2170289596Scem 2171289596Scem return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 2172289596Scem} 2173289596Scem 2174250079Scarl/** 2175289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side 2176250079Scarl * @ntb: pointer to ntb_softc instance 2177289545Scem * @bit: doorbell bits to ring 2178250079Scarl * 2179250079Scarl * This function allows triggering of a doorbell on the secondary/external 2180250079Scarl * side that will initiate an interrupt on the remote host 2181250079Scarl */ 2182250079Scarlvoid 2183289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 2184250079Scarl{ 2185250079Scarl 2186289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2187289347Scem ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit); 2188289347Scem return; 2189289209Scem } 2190289347Scem 2191289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2192250079Scarl} 2193250079Scarl 2194289542Scem/* 2195289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 2196289542Scem * as well as the size of the register (via *sz_out). 2197289542Scem * 2198289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell 2199289542Scem * ring to its memory window write. 2200289542Scem * 2201289542Scem * Note that writing the peer doorbell via a memory window will *not* generate 2202289542Scem * an interrupt on the remote host; that must be done seperately. 2203289542Scem */ 2204289542Scembus_addr_t 2205289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 2206289542Scem{ 2207289542Scem struct ntb_pci_bar_info *bar; 2208289542Scem uint64_t regoff; 2209289542Scem 2210289542Scem KASSERT(sz_out != NULL, ("must be non-NULL")); 2211289542Scem 2212289542Scem if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2213289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 2214289542Scem regoff = ntb->peer_reg->db_bell; 2215289542Scem } else { 2216289542Scem KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) || 2217289542Scem (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1), 2218289542Scem ("mw_count invalid after setup")); 2219289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 2220289543Scem ("invalid b2b idx")); 2221289542Scem 2222289542Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 2223289542Scem regoff = XEON_SHADOW_PDOORBELL_OFFSET; 2224289542Scem } 2225289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 2226289542Scem 2227289542Scem *sz_out = ntb->reg->db_size; 2228289542Scem /* HACK: Specific to current x86 bus implementation. */ 2229289542Scem return ((uint64_t)bar->pci_bus_handle + regoff); 2230289542Scem} 2231289542Scem 2232289597Scem/* 2233289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 2234289597Scem * @ntb: NTB device context 2235289597Scem * 2236289597Scem * Hardware may support different number or arrangement of doorbell bits. 2237289597Scem * 2238289597Scem * Return: A mask of doorbell bits supported by the ntb. 2239289597Scem */ 2240289597Scemuint64_t 2241289597Scemntb_db_valid_mask(struct ntb_softc *ntb) 2242289597Scem{ 2243289597Scem 2244289597Scem return (ntb->db_valid_mask); 2245289597Scem} 2246289597Scem 2247289598Scem/* 2248289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 2249289598Scem * @ntb: NTB device context 2250289598Scem * @vector: Doorbell vector number 2251289598Scem * 2252289598Scem * Each interrupt vector may have a different number or arrangement of bits. 2253289598Scem * 2254289598Scem * Return: A mask of doorbell bits serviced by a vector. 2255289598Scem */ 2256289598Scemuint64_t 2257289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 2258289598Scem{ 2259289598Scem 2260289598Scem if (vector > ntb->db_vec_count) 2261289598Scem return (0); 2262289598Scem return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 2263289598Scem} 2264289598Scem 2265250079Scarl/** 2266289546Scem * ntb_link_is_up() - get the current ntb link state 2267289546Scem * @ntb: NTB device context 2268289546Scem * @speed: OUT - The link speed expressed as PCIe generation number 2269289546Scem * @width: OUT - The link width expressed as the number of PCIe lanes 2270250079Scarl * 2271250079Scarl * RETURNS: true or false based on the hardware link state 2272250079Scarl */ 2273250079Scarlbool 2274289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 2275289546Scem enum ntb_width *width) 2276250079Scarl{ 2277250079Scarl 2278289546Scem if (speed != NULL) 2279289546Scem *speed = ntb_link_sta_speed(ntb); 2280289546Scem if (width != NULL) 2281289546Scem *width = ntb_link_sta_width(ntb); 2282289546Scem return (link_is_up(ntb)); 2283250079Scarl} 2284250079Scarl 2285255272Scarlstatic void 2286255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 2287250079Scarl{ 2288255272Scarl 2289289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2290289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2291289209Scem bar->pbase = rman_get_start(bar->pci_resource); 2292289209Scem bar->size = rman_get_size(bar->pci_resource); 2293289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 2294250079Scarl} 2295255268Scarl 2296289209Scemdevice_t 2297289209Scemntb_get_device(struct ntb_softc *ntb) 2298255268Scarl{ 2299255268Scarl 2300255268Scarl return (ntb->device); 2301255268Scarl} 2302289208Scem 2303289208Scem/* Export HW-specific errata information. */ 2304289208Scembool 2305289208Scemntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 2306289208Scem{ 2307289208Scem 2308289208Scem return (HAS_FEATURE(feature)); 2309289208Scem} 2310