ntb_hw_intel.c revision 289614
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3289542Scem * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289614 2015-10-20 01:54:25Z cem $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35250079Scarl#include <sys/malloc.h> 36250079Scarl#include <sys/module.h> 37250079Scarl#include <sys/queue.h> 38250079Scarl#include <sys/rman.h> 39289207Scem#include <sys/sysctl.h> 40250079Scarl#include <vm/vm.h> 41250079Scarl#include <vm/pmap.h> 42250079Scarl#include <machine/bus.h> 43250079Scarl#include <machine/pmap.h> 44250079Scarl#include <machine/resource.h> 45250079Scarl#include <dev/pci/pcireg.h> 46250079Scarl#include <dev/pci/pcivar.h> 47250079Scarl 48250079Scarl#include "ntb_regs.h" 49250079Scarl#include "ntb_hw.h" 50250079Scarl 51250079Scarl/* 52250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 53250079Scarl * allows you to connect two systems using a PCI-e link. 54250079Scarl * 55250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 56250079Scarl * you to send and recieve interrupts, map the memory windows and send and 57250079Scarl * receive messages in the scratch-pad registers. 58250079Scarl * 59250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 60250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 61250079Scarl */ 62250079Scarl 63289538Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT) 64250079Scarl 65289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 66289539Scem#define SOC_LINK_RECOVERY_TIME 500 /* ms */ 67250079Scarl 68250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 69250079Scarl 70250079Scarlenum ntb_device_type { 71250079Scarl NTB_XEON, 72250079Scarl NTB_SOC 73250079Scarl}; 74250079Scarl 75289610Scem/* ntb_conn_type are hardware numbers, cannot change. */ 76289610Scemenum ntb_conn_type { 77289610Scem NTB_CONN_TRANSPARENT = 0, 78289610Scem NTB_CONN_B2B = 1, 79289610Scem NTB_CONN_RP = 2, 80289610Scem}; 81289610Scem 82289610Scemenum ntb_b2b_direction { 83289610Scem NTB_DEV_USD = 0, 84289610Scem NTB_DEV_DSD = 1, 85289610Scem}; 86289610Scem 87289539Scemenum ntb_bar { 88289539Scem NTB_CONFIG_BAR = 0, 89289539Scem NTB_B2B_BAR_1, 90289539Scem NTB_B2B_BAR_2, 91289539Scem NTB_B2B_BAR_3, 92289539Scem NTB_MAX_BARS 93289539Scem}; 94289539Scem 95255274Scarl/* Device features and workarounds */ 96255274Scarl#define HAS_FEATURE(feature) \ 97255274Scarl ((ntb->features & (feature)) != 0) 98255274Scarl 99250079Scarlstruct ntb_hw_info { 100250079Scarl uint32_t device_id; 101255274Scarl const char *desc; 102250079Scarl enum ntb_device_type type; 103289397Scem uint32_t features; 104250079Scarl}; 105250079Scarl 106250079Scarlstruct ntb_pci_bar_info { 107250079Scarl bus_space_tag_t pci_bus_tag; 108250079Scarl bus_space_handle_t pci_bus_handle; 109250079Scarl int pci_resource_id; 110250079Scarl struct resource *pci_resource; 111250079Scarl vm_paddr_t pbase; 112250079Scarl void *vbase; 113250079Scarl u_long size; 114289543Scem 115289543Scem /* Configuration register offsets */ 116289543Scem uint32_t psz_off; 117289543Scem uint32_t ssz_off; 118289543Scem uint32_t pbarxlat_off; 119250079Scarl}; 120250079Scarl 121250079Scarlstruct ntb_int_info { 122250079Scarl struct resource *res; 123250079Scarl int rid; 124250079Scarl void *tag; 125250079Scarl}; 126250079Scarl 127289546Scemstruct ntb_vec { 128250079Scarl struct ntb_softc *ntb; 129289546Scem uint32_t num; 130250079Scarl}; 131250079Scarl 132289542Scemstruct ntb_reg { 133289542Scem uint32_t ntb_ctl; 134289542Scem uint32_t lnk_sta; 135289542Scem uint8_t db_size; 136289542Scem unsigned mw_bar[NTB_MAX_BARS]; 137289542Scem}; 138289542Scem 139289542Scemstruct ntb_alt_reg { 140289542Scem uint32_t db_bell; 141289542Scem uint32_t db_mask; 142289542Scem uint32_t spad; 143289542Scem}; 144289542Scem 145289542Scemstruct ntb_xlat_reg { 146289546Scem uint32_t bar0_base; 147289546Scem uint32_t bar2_base; 148289546Scem uint32_t bar4_base; 149289546Scem uint32_t bar5_base; 150289546Scem 151289546Scem uint32_t bar2_xlat; 152289546Scem uint32_t bar4_xlat; 153289546Scem uint32_t bar5_xlat; 154289546Scem 155289546Scem uint32_t bar2_limit; 156289546Scem uint32_t bar4_limit; 157289546Scem uint32_t bar5_limit; 158289542Scem}; 159289542Scem 160289542Scemstruct ntb_b2b_addr { 161289542Scem uint64_t bar0_addr; 162289542Scem uint64_t bar2_addr64; 163289542Scem uint64_t bar4_addr64; 164289542Scem uint64_t bar4_addr32; 165289542Scem uint64_t bar5_addr32; 166289542Scem}; 167289542Scem 168250079Scarlstruct ntb_softc { 169250079Scarl device_t device; 170250079Scarl enum ntb_device_type type; 171255274Scarl uint64_t features; 172250079Scarl 173250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 174250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 175250079Scarl uint32_t allocated_interrupts; 176250079Scarl 177250079Scarl struct callout heartbeat_timer; 178250079Scarl struct callout lr_timer; 179250079Scarl 180289546Scem void *ntb_ctx; 181289546Scem const struct ntb_ctx_ops *ctx_ops; 182289546Scem struct ntb_vec *msix_vec; 183289546Scem#define CTX_LOCK(sc) mtx_lock_spin(&(sc)->ctx_lock) 184289546Scem#define CTX_UNLOCK(sc) mtx_unlock_spin(&(sc)->ctx_lock) 185289546Scem#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 186289546Scem struct mtx ctx_lock; 187250079Scarl 188289610Scem uint32_t ppd; 189289610Scem enum ntb_conn_type conn_type; 190289610Scem enum ntb_b2b_direction dev_type; 191289539Scem 192289542Scem /* Offset of peer bar0 in B2B BAR */ 193289542Scem uint64_t b2b_off; 194289542Scem /* Memory window used to access peer bar0 */ 195289543Scem#define B2B_MW_DISABLED UINT8_MAX 196289542Scem uint8_t b2b_mw_idx; 197289542Scem 198289539Scem uint8_t mw_count; 199289539Scem uint8_t spad_count; 200289539Scem uint8_t db_count; 201289539Scem uint8_t db_vec_count; 202289539Scem uint8_t db_vec_shift; 203289542Scem 204289546Scem /* Protects local db_mask. */ 205289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 206289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 207289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 208289542Scem struct mtx db_mask_lock; 209289542Scem 210289546Scem uint32_t ntb_ctl; 211289546Scem uint32_t lnk_sta; 212289542Scem 213289542Scem uint64_t db_valid_mask; 214289542Scem uint64_t db_link_mask; 215289546Scem uint64_t db_mask; 216289542Scem 217289542Scem int last_ts; /* ticks @ last irq */ 218289542Scem 219289542Scem const struct ntb_reg *reg; 220289542Scem const struct ntb_alt_reg *self_reg; 221289542Scem const struct ntb_alt_reg *peer_reg; 222289542Scem const struct ntb_xlat_reg *xlat_reg; 223250079Scarl}; 224250079Scarl 225289234Scem#ifdef __i386__ 226289234Scemstatic __inline uint64_t 227289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 228289234Scem bus_size_t offset) 229289234Scem{ 230289234Scem 231289234Scem return (bus_space_read_4(tag, handle, offset) | 232289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 233289234Scem} 234289234Scem 235289234Scemstatic __inline void 236289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 237289234Scem bus_size_t offset, uint64_t val) 238289234Scem{ 239289234Scem 240289234Scem bus_space_write_4(tag, handle, offset, val); 241289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 242289234Scem} 243289234Scem#endif 244289234Scem 245255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 246255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 247255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 248255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 249255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 250255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 251255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 252250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 253255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 254289397Scem#define ntb_mw_read(SIZE, offset) \ 255289542Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 256255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 257289542Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 258289397Scem offset, val) 259250079Scarl 260250079Scarlstatic int ntb_probe(device_t device); 261250079Scarlstatic int ntb_attach(device_t device); 262250079Scarlstatic int ntb_detach(device_t device); 263289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 264289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 265289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 266289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 267255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 268289541Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *); 269255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 270255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 271255272Scarl struct ntb_pci_bar_info *bar); 272250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 273289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 274289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 275289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 276289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 277250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 278289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 279289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 280289546Scemstatic void ndev_vec_isr(void *arg); 281289546Scemstatic void ndev_irq_isr(void *arg); 282289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 283289546Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t val); 284289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 285289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 286250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 287289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 288289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 289289348Scemstatic int ntb_detect_soc(struct ntb_softc *ntb); 290289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 291289542Scemstatic int ntb_soc_init_dev(struct ntb_softc *ntb); 292289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 293255279Scarlstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 294289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 295289543Scem enum ntb_bar regbar); 296289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 297289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 298289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 299289543Scem enum ntb_bar idx); 300289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 301289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 302289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 303289546Scemstatic inline bool soc_link_is_err(struct ntb_softc *ntb); 304289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 305289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 306289542Scemstatic void soc_link_hb(void *arg); 307289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 308250079Scarlstatic void recover_soc_link(void *arg); 309289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb); 310255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 311250079Scarl 312250079Scarlstatic struct ntb_hw_info pci_ids[] = { 313289612Scem /* XXX: PS/SS IDs left out until they are supported. */ 314289612Scem { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 315289612Scem NTB_SOC, 0 }, 316289233Scem 317289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 318289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 319289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 320289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 321289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 322289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 323289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 324289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 325289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 326289538Scem NTB_SB01BASE_LOCKUP }, 327289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 328289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 329289538Scem NTB_SB01BASE_LOCKUP }, 330289233Scem 331255274Scarl { 0x00000000, NULL, NTB_SOC, 0 } 332250079Scarl}; 333250079Scarl 334289542Scemstatic const struct ntb_reg soc_reg = { 335289542Scem .ntb_ctl = SOC_NTBCNTL_OFFSET, 336289542Scem .lnk_sta = SOC_LINK_STATUS_OFFSET, 337289542Scem .db_size = sizeof(uint64_t), 338289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 339289542Scem}; 340289542Scem 341289607Scemstatic const struct ntb_alt_reg soc_pri_reg = { 342289607Scem .db_bell = SOC_PDOORBELL_OFFSET, 343289607Scem .db_mask = SOC_PDBMSK_OFFSET, 344289607Scem .spad = SOC_SPAD_OFFSET, 345289607Scem}; 346289607Scem 347289542Scemstatic const struct ntb_alt_reg soc_b2b_reg = { 348289542Scem .db_bell = SOC_B2B_DOORBELL_OFFSET, 349289542Scem .spad = SOC_B2B_SPAD_OFFSET, 350289542Scem}; 351289542Scem 352289542Scemstatic const struct ntb_xlat_reg soc_sec_xlat = { 353289542Scem#if 0 354289542Scem /* "FIXME" says the Linux driver. */ 355289542Scem .bar0_base = SOC_SBAR0BASE_OFFSET, 356289546Scem .bar2_base = SOC_SBAR2BASE_OFFSET, 357289546Scem .bar4_base = SOC_SBAR4BASE_OFFSET, 358289546Scem 359289542Scem .bar2_limit = SOC_SBAR2LMT_OFFSET, 360289546Scem .bar4_limit = SOC_SBAR4LMT_OFFSET, 361289542Scem#endif 362289546Scem 363289542Scem .bar2_xlat = SOC_SBAR2XLAT_OFFSET, 364289546Scem .bar4_xlat = SOC_SBAR4XLAT_OFFSET, 365289542Scem}; 366289542Scem 367289542Scemstatic const struct ntb_reg xeon_reg = { 368289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 369289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 370289542Scem .db_size = sizeof(uint16_t), 371289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 372289542Scem}; 373289542Scem 374289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 375289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 376289607Scem .db_mask = XEON_PDBMSK_OFFSET, 377289607Scem .spad = XEON_SPAD_OFFSET, 378289607Scem}; 379289607Scem 380289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 381289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 382289542Scem .spad = XEON_B2B_SPAD_OFFSET, 383289542Scem}; 384289542Scem 385289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 386289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 387289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 388289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 389289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 390289546Scem 391289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 392289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 393289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 394289546Scem 395289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 396289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 397289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 398289542Scem}; 399289542Scem 400289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 401289542Scem .bar0_addr = XEON_B2B_BAR0_USD_ADDR, 402289542Scem .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, 403289542Scem .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, 404289542Scem .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, 405289542Scem .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, 406289542Scem}; 407289542Scem 408289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 409289542Scem .bar0_addr = XEON_B2B_BAR0_DSD_ADDR, 410289542Scem .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, 411289542Scem .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, 412289542Scem .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, 413289542Scem .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, 414289542Scem}; 415289542Scem 416289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 417289614Scem "B2B MW segment overrides -- MUST be the same on both sides"); 418289614Scem 419289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 420289614Scem &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 421289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 422289614Scem "the window at BAR2, on the upstream side of the link. MUST be the same " 423289614Scem "address on both sides."); 424289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 425289614Scem &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 426289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 427289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 428289614Scem "(split-BAR mode)."); 429289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 430289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR5 " 431289614Scem "(split-BAR mode)."); 432289614Scem 433289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 434289614Scem &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 435289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 436289614Scem "the window at BAR2, on the downstream side of the link. MUST be the same" 437289614Scem " address on both sides."); 438289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 439289614Scem &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 440289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 441289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 442289614Scem "(split-BAR mode)."); 443289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 444289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 445289614Scem "(split-BAR mode)."); 446289614Scem 447250079Scarl/* 448250079Scarl * OS <-> Driver interface structures 449250079Scarl */ 450250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 451250079Scarl 452250079Scarlstatic device_method_t ntb_pci_methods[] = { 453250079Scarl /* Device interface */ 454250079Scarl DEVMETHOD(device_probe, ntb_probe), 455250079Scarl DEVMETHOD(device_attach, ntb_attach), 456250079Scarl DEVMETHOD(device_detach, ntb_detach), 457250079Scarl DEVMETHOD_END 458250079Scarl}; 459250079Scarl 460250079Scarlstatic driver_t ntb_pci_driver = { 461250079Scarl "ntb_hw", 462250079Scarl ntb_pci_methods, 463250079Scarl sizeof(struct ntb_softc), 464250079Scarl}; 465250079Scarl 466250079Scarlstatic devclass_t ntb_devclass; 467250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 468250079ScarlMODULE_VERSION(ntb_hw, 1); 469250079Scarl 470289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 471289207Scem 472250079Scarl/* 473250079Scarl * OS <-> Driver linkage functions 474250079Scarl */ 475250079Scarlstatic int 476250079Scarlntb_probe(device_t device) 477250079Scarl{ 478289209Scem struct ntb_hw_info *p; 479250079Scarl 480289209Scem p = ntb_get_device_info(pci_get_devid(device)); 481289209Scem if (p == NULL) 482250079Scarl return (ENXIO); 483289209Scem 484289209Scem device_set_desc(device, p->desc); 485289209Scem return (0); 486250079Scarl} 487250079Scarl 488250079Scarlstatic int 489250079Scarlntb_attach(device_t device) 490250079Scarl{ 491289209Scem struct ntb_softc *ntb; 492289209Scem struct ntb_hw_info *p; 493250079Scarl int error; 494250079Scarl 495289209Scem ntb = DEVICE2SOFTC(device); 496289209Scem p = ntb_get_device_info(pci_get_devid(device)); 497289209Scem 498250079Scarl ntb->device = device; 499250079Scarl ntb->type = p->type; 500255274Scarl ntb->features = p->features; 501289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 502250079Scarl 503250079Scarl /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 504283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 505283291Sjkim callout_init(&ntb->lr_timer, 1); 506289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 507289546Scem mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN); 508250079Scarl 509289348Scem if (ntb->type == NTB_SOC) 510289348Scem error = ntb_detect_soc(ntb); 511289348Scem else 512289348Scem error = ntb_detect_xeon(ntb); 513289348Scem if (error) 514289348Scem goto out; 515289348Scem 516289397Scem ntb_detect_max_mw(ntb); 517289396Scem 518289209Scem error = ntb_map_pci_bars(ntb); 519289209Scem if (error) 520289209Scem goto out; 521289272Scem if (ntb->type == NTB_SOC) 522289542Scem error = ntb_soc_init_dev(ntb); 523289272Scem else 524289542Scem error = ntb_xeon_init_dev(ntb); 525289209Scem if (error) 526289209Scem goto out; 527289540Scem error = ntb_init_isr(ntb); 528289209Scem if (error) 529289209Scem goto out; 530250079Scarl 531250079Scarl pci_enable_busmaster(ntb->device); 532250079Scarl 533289209Scemout: 534289209Scem if (error != 0) 535289209Scem ntb_detach(device); 536250079Scarl return (error); 537250079Scarl} 538250079Scarl 539250079Scarlstatic int 540250079Scarlntb_detach(device_t device) 541250079Scarl{ 542289209Scem struct ntb_softc *ntb; 543250079Scarl 544289209Scem ntb = DEVICE2SOFTC(device); 545289542Scem 546289542Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 547250079Scarl callout_drain(&ntb->heartbeat_timer); 548250079Scarl callout_drain(&ntb->lr_timer); 549289272Scem if (ntb->type == NTB_XEON) 550289272Scem ntb_teardown_xeon(ntb); 551250079Scarl ntb_teardown_interrupts(ntb); 552289397Scem 553289542Scem mtx_destroy(&ntb->db_mask_lock); 554289546Scem mtx_destroy(&ntb->ctx_lock); 555289542Scem 556289397Scem /* 557289397Scem * Redetect total MWs so we unmap properly -- in case we lowered the 558289397Scem * maximum to work around Xeon errata. 559289397Scem */ 560289397Scem ntb_detect_max_mw(ntb); 561250079Scarl ntb_unmap_pci_bar(ntb); 562250079Scarl 563250079Scarl return (0); 564250079Scarl} 565250079Scarl 566289542Scem/* 567289542Scem * Driver internal routines 568289542Scem */ 569289539Scemstatic inline enum ntb_bar 570289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 571289539Scem{ 572289539Scem 573289543Scem KASSERT(mw < ntb->mw_count || 574289543Scem (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx), 575289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 576289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 577289539Scem 578289542Scem return (ntb->reg->mw_bar[mw]); 579289539Scem} 580289539Scem 581289546Scemstatic inline bool 582289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 583289546Scem{ 584289546Scem /* XXX This assertion could be stronger. */ 585289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 586289546Scem return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 587289546Scem} 588289546Scem 589289546Scemstatic inline void 590289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 591289546Scem uint32_t *xlat, uint32_t *lmt) 592289546Scem{ 593289546Scem uint32_t basev, lmtv, xlatv; 594289546Scem 595289546Scem switch (bar) { 596289546Scem case NTB_B2B_BAR_1: 597289546Scem basev = ntb->xlat_reg->bar2_base; 598289546Scem lmtv = ntb->xlat_reg->bar2_limit; 599289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 600289546Scem break; 601289546Scem case NTB_B2B_BAR_2: 602289546Scem basev = ntb->xlat_reg->bar4_base; 603289546Scem lmtv = ntb->xlat_reg->bar4_limit; 604289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 605289546Scem break; 606289546Scem case NTB_B2B_BAR_3: 607289546Scem basev = ntb->xlat_reg->bar5_base; 608289546Scem lmtv = ntb->xlat_reg->bar5_limit; 609289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 610289546Scem break; 611289546Scem default: 612289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 613289546Scem ("bad bar")); 614289546Scem basev = lmtv = xlatv = 0; 615289546Scem break; 616289546Scem } 617289546Scem 618289546Scem if (base != NULL) 619289546Scem *base = basev; 620289546Scem if (xlat != NULL) 621289546Scem *xlat = xlatv; 622289546Scem if (lmt != NULL) 623289546Scem *lmt = lmtv; 624289546Scem} 625289546Scem 626250079Scarlstatic int 627255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 628250079Scarl{ 629255272Scarl int rc; 630250079Scarl 631250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 632289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 633255272Scarl if (rc != 0) 634289541Scem goto out; 635255272Scarl 636289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 637289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 638255272Scarl if (rc != 0) 639289541Scem goto out; 640289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 641289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 642289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 643255272Scarl 644289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 645289543Scem /* XXX Are shared MW B2Bs write-combining? */ 646289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 647289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 648255279Scarl else 649289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 650289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 651289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 652289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 653289543Scem 654289397Scem if (!HAS_FEATURE(NTB_SPLIT_BAR)) 655289541Scem goto out; 656289397Scem 657289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 658289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 659289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 660289397Scem else 661289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 662289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 663289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 664289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 665250079Scarl 666289541Scemout: 667289209Scem if (rc != 0) 668255272Scarl device_printf(ntb->device, 669255272Scarl "unable to allocate pci resource\n"); 670255272Scarl return (rc); 671255272Scarl} 672255272Scarl 673289541Scemstatic void 674289541Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 675289541Scem{ 676289541Scem 677289541Scem device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n", 678289541Scem bar->size, bar->vbase, (void *)(bar->pbase)); 679289541Scem} 680289541Scem 681255272Scarlstatic int 682255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 683255272Scarl{ 684255272Scarl 685255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 686289209Scem &bar->pci_resource_id, RF_ACTIVE); 687255272Scarl if (bar->pci_resource == NULL) 688255272Scarl return (ENXIO); 689289209Scem 690289209Scem save_bar_parameters(bar); 691289541Scem print_map_success(ntb, bar); 692289209Scem return (0); 693255272Scarl} 694255272Scarl 695255272Scarlstatic int 696255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 697255272Scarl{ 698255272Scarl int rc; 699255276Scarl uint8_t bar_size_bits = 0; 700255272Scarl 701289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 702289209Scem &bar->pci_resource_id, RF_ACTIVE); 703250079Scarl 704255272Scarl if (bar->pci_resource == NULL) 705255272Scarl return (ENXIO); 706255276Scarl 707289209Scem save_bar_parameters(bar); 708289209Scem /* 709289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 710289209Scem * hardware issue. To work around this, query the size it should be 711289209Scem * configured to by the device and modify the resource to correspond to 712289209Scem * this new size. The BIOS on systems with this problem is required to 713289209Scem * provide enough address space to allow the driver to make this change 714289209Scem * safely. 715289209Scem * 716289209Scem * Ideally I could have just specified the size when I allocated the 717289209Scem * resource like: 718289209Scem * bus_alloc_resource(ntb->device, 719289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 720289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 721289209Scem * but the PCI driver does not honor the size in this call, so we have 722289209Scem * to modify it after the fact. 723289209Scem */ 724289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 725289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 726289209Scem bar_size_bits = pci_read_config(ntb->device, 727289209Scem XEON_PBAR23SZ_OFFSET, 1); 728289209Scem else 729289209Scem bar_size_bits = pci_read_config(ntb->device, 730289209Scem XEON_PBAR45SZ_OFFSET, 1); 731289209Scem 732289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 733289209Scem bar->pci_resource, bar->pbase, 734289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 735255272Scarl if (rc != 0) { 736289209Scem device_printf(ntb->device, 737289209Scem "unable to resize bar\n"); 738255272Scarl return (rc); 739250079Scarl } 740289209Scem 741289209Scem save_bar_parameters(bar); 742250079Scarl } 743289209Scem 744289209Scem /* Mark bar region as write combining to improve performance. */ 745289209Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 746289209Scem VM_MEMATTR_WRITE_COMBINING); 747289209Scem if (rc != 0) { 748289209Scem device_printf(ntb->device, 749289209Scem "unable to mark bar as WRITE_COMBINING\n"); 750289209Scem return (rc); 751289209Scem } 752289541Scem print_map_success(ntb, bar); 753250079Scarl return (0); 754250079Scarl} 755250079Scarl 756250079Scarlstatic void 757250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 758250079Scarl{ 759250079Scarl struct ntb_pci_bar_info *current_bar; 760250079Scarl int i; 761250079Scarl 762289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 763250079Scarl current_bar = &ntb->bar_info[i]; 764250079Scarl if (current_bar->pci_resource != NULL) 765250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 766250079Scarl current_bar->pci_resource_id, 767250079Scarl current_bar->pci_resource); 768250079Scarl } 769250079Scarl} 770250079Scarl 771250079Scarlstatic int 772289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 773250079Scarl{ 774289342Scem uint32_t i; 775289342Scem int rc; 776289342Scem 777289342Scem for (i = 0; i < num_vectors; i++) { 778289342Scem ntb->int_info[i].rid = i + 1; 779289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 780289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 781289342Scem if (ntb->int_info[i].res == NULL) { 782289342Scem device_printf(ntb->device, 783289342Scem "bus_alloc_resource failed\n"); 784289342Scem return (ENOMEM); 785289342Scem } 786289342Scem ntb->int_info[i].tag = NULL; 787289342Scem ntb->allocated_interrupts++; 788289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 789289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 790289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 791289342Scem if (rc != 0) { 792289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 793289342Scem return (ENXIO); 794289342Scem } 795289342Scem } 796289342Scem return (0); 797289342Scem} 798289342Scem 799289344Scem/* 800289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 801289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 802289344Scem * should be okay. This tunable should enable us to test that hypothesis 803289344Scem * when someone gets their hands on some Xeon hardware. 804289344Scem */ 805289344Scemstatic int ntb_force_remap_mode; 806289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 807289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 808289344Scem " to a smaller number of ithreads, even if the desired number are " 809289344Scem "available"); 810289344Scem 811289344Scem/* 812289344Scem * In case it is NOT ok, give consumers an abort button. 813289344Scem */ 814289344Scemstatic int ntb_prefer_intx; 815289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 816289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 817289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 818289344Scem "behavior)"); 819289344Scem 820289344Scem/* 821289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 822289344Scem * round-robin fashion. 823289344Scem */ 824289342Scemstatic int 825289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 826289344Scem{ 827289344Scem u_int *vectors; 828289344Scem uint32_t i; 829289344Scem int rc; 830289344Scem 831289344Scem if (ntb_prefer_intx != 0) 832289344Scem return (ENXIO); 833289344Scem 834289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 835289344Scem 836289344Scem for (i = 0; i < desired; i++) 837289344Scem vectors[i] = (i % avail) + 1; 838289344Scem 839289344Scem rc = pci_remap_msix(dev, desired, vectors); 840289344Scem free(vectors, M_NTB); 841289344Scem return (rc); 842289344Scem} 843289344Scem 844289344Scemstatic int 845289540Scemntb_init_isr(struct ntb_softc *ntb) 846289342Scem{ 847289344Scem uint32_t desired_vectors, num_vectors; 848289342Scem int rc; 849250079Scarl 850250079Scarl ntb->allocated_interrupts = 0; 851289542Scem ntb->last_ts = ticks; 852289347Scem 853250079Scarl /* 854289546Scem * Mask all doorbell interrupts. 855250079Scarl */ 856289546Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 857250079Scarl 858289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 859289539Scem ntb->db_count); 860289344Scem if (desired_vectors >= 1) { 861289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 862250079Scarl 863289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 864289344Scem num_vectors == desired_vectors) 865289344Scem num_vectors--; 866289344Scem 867289344Scem if (rc == 0 && num_vectors < desired_vectors) { 868289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 869289344Scem num_vectors); 870289344Scem if (rc == 0) 871289344Scem num_vectors = desired_vectors; 872289344Scem else 873289344Scem pci_release_msi(ntb->device); 874289344Scem } 875289344Scem if (rc != 0) 876289344Scem num_vectors = 1; 877289344Scem } else 878289344Scem num_vectors = 1; 879289344Scem 880289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 881289539Scem ntb->db_vec_count = 1; 882289539Scem ntb->db_vec_shift = ntb->db_count; 883289539Scem rc = ntb_setup_legacy_interrupt(ntb); 884289539Scem } else { 885289546Scem ntb_create_msix_vec(ntb, num_vectors); 886289540Scem rc = ntb_setup_msix(ntb, num_vectors); 887289539Scem } 888289539Scem if (rc != 0) { 889289539Scem device_printf(ntb->device, 890289539Scem "Error allocating interrupts: %d\n", rc); 891289546Scem ntb_free_msix_vec(ntb); 892289396Scem } 893289396Scem 894289342Scem return (rc); 895289342Scem} 896289342Scem 897289342Scemstatic int 898289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 899289342Scem{ 900289342Scem int rc; 901289342Scem 902289342Scem ntb->int_info[0].rid = 0; 903289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 904289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 905289342Scem if (ntb->int_info[0].res == NULL) { 906289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 907289342Scem return (ENOMEM); 908250079Scarl } 909250079Scarl 910289342Scem ntb->int_info[0].tag = NULL; 911289342Scem ntb->allocated_interrupts = 1; 912289342Scem 913289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 914289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 915289342Scem ntb, &ntb->int_info[0].tag); 916289342Scem if (rc != 0) { 917289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 918289342Scem return (ENXIO); 919289342Scem } 920289342Scem 921250079Scarl return (0); 922250079Scarl} 923250079Scarl 924250079Scarlstatic void 925250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 926250079Scarl{ 927250079Scarl struct ntb_int_info *current_int; 928250079Scarl int i; 929250079Scarl 930289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 931250079Scarl current_int = &ntb->int_info[i]; 932250079Scarl if (current_int->tag != NULL) 933250079Scarl bus_teardown_intr(ntb->device, current_int->res, 934250079Scarl current_int->tag); 935250079Scarl 936250079Scarl if (current_int->res != NULL) 937250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 938250079Scarl rman_get_rid(current_int->res), current_int->res); 939250079Scarl } 940250079Scarl 941289546Scem ntb_free_msix_vec(ntb); 942250079Scarl pci_release_msi(ntb->device); 943250079Scarl} 944250079Scarl 945289347Scem/* 946289347Scem * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon. Abstract it 947289347Scem * out to make code clearer. 948289347Scem */ 949289539Scemstatic inline uint64_t 950289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 951289347Scem{ 952289347Scem 953289347Scem if (ntb->type == NTB_SOC) 954289347Scem return (ntb_reg_read(8, regoff)); 955289347Scem 956289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 957289347Scem 958289347Scem return (ntb_reg_read(2, regoff)); 959289347Scem} 960289347Scem 961289539Scemstatic inline void 962289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 963289347Scem{ 964289347Scem 965289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 966289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 967289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 968289542Scem (uintmax_t)ntb->db_valid_mask)); 969289542Scem 970289607Scem if (regoff == ntb->self_reg->db_mask) 971289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 972289542Scem 973289347Scem if (ntb->type == NTB_SOC) { 974289347Scem ntb_reg_write(8, regoff, val); 975289347Scem return; 976289347Scem } 977289347Scem 978289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 979289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 980289347Scem} 981289347Scem 982289546Scemvoid 983289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 984289542Scem{ 985289542Scem 986289546Scem DB_MASK_LOCK(ntb); 987289542Scem ntb->db_mask |= bits; 988289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 989289546Scem DB_MASK_UNLOCK(ntb); 990289542Scem} 991289542Scem 992289546Scemvoid 993289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 994289542Scem{ 995289542Scem 996289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 997289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 998289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 999289542Scem (uintmax_t)ntb->db_valid_mask)); 1000289542Scem 1001289546Scem DB_MASK_LOCK(ntb); 1002289542Scem ntb->db_mask &= ~bits; 1003289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1004289546Scem DB_MASK_UNLOCK(ntb); 1005289542Scem} 1006289542Scem 1007289546Scemuint64_t 1008289546Scemntb_db_read(struct ntb_softc *ntb) 1009289281Scem{ 1010289281Scem 1011289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 1012289281Scem} 1013289281Scem 1014289546Scemvoid 1015289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1016289281Scem{ 1017289281Scem 1018289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1019289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1020289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1021289546Scem (uintmax_t)ntb->db_valid_mask)); 1022289546Scem 1023289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1024289281Scem} 1025289281Scem 1026289540Scemstatic inline uint64_t 1027289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1028250079Scarl{ 1029289540Scem uint64_t shift, mask; 1030250079Scarl 1031289540Scem shift = ntb->db_vec_shift; 1032289540Scem mask = (1ull << shift) - 1; 1033289540Scem return (mask << (shift * db_vector)); 1034250079Scarl} 1035250079Scarl 1036250079Scarlstatic void 1037289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1038250079Scarl{ 1039289540Scem uint64_t vec_mask; 1040250079Scarl 1041289542Scem ntb->last_ts = ticks; 1042289546Scem vec_mask = ntb_vec_mask(ntb, vec); 1043250079Scarl 1044289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1045289546Scem if (ntb_poll_link(ntb)) 1046289546Scem ntb_link_event(ntb); 1047289540Scem } 1048289540Scem 1049289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1050289546Scem ntb_db_event(ntb, vec); 1051289546Scem} 1052250079Scarl 1053289546Scemstatic void 1054289546Scemndev_vec_isr(void *arg) 1055289546Scem{ 1056289546Scem struct ntb_vec *nvec = arg; 1057250079Scarl 1058289546Scem ntb_interrupt(nvec->ntb, nvec->num); 1059250079Scarl} 1060250079Scarl 1061250079Scarlstatic void 1062289546Scemndev_irq_isr(void *arg) 1063250079Scarl{ 1064289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1065289546Scem ntb_interrupt(arg, 0); 1066250079Scarl} 1067250079Scarl 1068250079Scarlstatic int 1069289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1070250079Scarl{ 1071289342Scem uint32_t i; 1072250079Scarl 1073289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1074250079Scarl M_ZERO | M_WAITOK); 1075250079Scarl for (i = 0; i < num_vectors; i++) { 1076289546Scem ntb->msix_vec[i].num = i; 1077289546Scem ntb->msix_vec[i].ntb = ntb; 1078250079Scarl } 1079250079Scarl 1080250079Scarl return (0); 1081250079Scarl} 1082250079Scarl 1083250079Scarlstatic void 1084289546Scemntb_free_msix_vec(struct ntb_softc *ntb) 1085250079Scarl{ 1086250079Scarl 1087289546Scem if (ntb->msix_vec == NULL) 1088289539Scem return; 1089289539Scem 1090289546Scem free(ntb->msix_vec, M_NTB); 1091289546Scem ntb->msix_vec = NULL; 1092250079Scarl} 1093250079Scarl 1094250079Scarlstatic struct ntb_hw_info * 1095250079Scarlntb_get_device_info(uint32_t device_id) 1096250079Scarl{ 1097250079Scarl struct ntb_hw_info *ep = pci_ids; 1098250079Scarl 1099250079Scarl while (ep->device_id) { 1100250079Scarl if (ep->device_id == device_id) 1101250079Scarl return (ep); 1102250079Scarl ++ep; 1103250079Scarl } 1104250079Scarl return (NULL); 1105250079Scarl} 1106250079Scarl 1107289272Scemstatic void 1108289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 1109250079Scarl{ 1110250079Scarl 1111289542Scem ntb_link_disable(ntb); 1112250079Scarl} 1113250079Scarl 1114289397Scemstatic void 1115289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 1116289397Scem{ 1117289397Scem 1118289397Scem if (ntb->type == NTB_SOC) { 1119289539Scem ntb->mw_count = SOC_MW_COUNT; 1120289397Scem return; 1121289397Scem } 1122289397Scem 1123289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1124289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1125289397Scem else 1126289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1127289397Scem} 1128289397Scem 1129250079Scarlstatic int 1130289348Scemntb_detect_xeon(struct ntb_softc *ntb) 1131250079Scarl{ 1132289348Scem uint8_t ppd, conn_type; 1133250079Scarl 1134289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1135289348Scem ntb->ppd = ppd; 1136250079Scarl 1137289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1138289257Scem ntb->dev_type = NTB_DEV_USD; 1139289257Scem else 1140289257Scem ntb->dev_type = NTB_DEV_DSD; 1141289257Scem 1142289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1143289397Scem ntb->features |= NTB_SPLIT_BAR; 1144289397Scem 1145289542Scem /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1146289542Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1147289542Scem ntb->features |= NTB_SDOORBELL_LOCKUP; 1148289542Scem 1149289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1150289348Scem switch (conn_type) { 1151289348Scem case NTB_CONN_B2B: 1152289348Scem ntb->conn_type = conn_type; 1153289348Scem break; 1154289348Scem case NTB_CONN_RP: 1155289348Scem case NTB_CONN_TRANSPARENT: 1156289348Scem default: 1157289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1158289348Scem (unsigned)conn_type); 1159289348Scem return (ENXIO); 1160289348Scem } 1161289348Scem return (0); 1162289348Scem} 1163289348Scem 1164289348Scemstatic int 1165289348Scemntb_detect_soc(struct ntb_softc *ntb) 1166289348Scem{ 1167289348Scem uint32_t ppd, conn_type; 1168289348Scem 1169289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1170289348Scem ntb->ppd = ppd; 1171289348Scem 1172289348Scem if ((ppd & SOC_PPD_DEV_TYPE) != 0) 1173289348Scem ntb->dev_type = NTB_DEV_DSD; 1174289348Scem else 1175289348Scem ntb->dev_type = NTB_DEV_USD; 1176289348Scem 1177289348Scem conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8; 1178289348Scem switch (conn_type) { 1179289348Scem case NTB_CONN_B2B: 1180289348Scem ntb->conn_type = conn_type; 1181289348Scem break; 1182289348Scem default: 1183289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1184289348Scem return (ENXIO); 1185289348Scem } 1186289348Scem return (0); 1187289348Scem} 1188289348Scem 1189289348Scemstatic int 1190289542Scemntb_xeon_init_dev(struct ntb_softc *ntb) 1191289348Scem{ 1192289542Scem int rc; 1193289348Scem 1194289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1195289542Scem ntb->db_count = XEON_DB_COUNT; 1196289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1197289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1198289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1199289257Scem 1200289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1201250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1202289348Scem ntb->conn_type); 1203250079Scarl return (ENXIO); 1204250079Scarl } 1205250079Scarl 1206289542Scem ntb->reg = &xeon_reg; 1207289607Scem ntb->self_reg = &xeon_pri_reg; 1208289542Scem ntb->peer_reg = &xeon_b2b_reg; 1209289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1210289542Scem 1211289208Scem /* 1212289208Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1213289208Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1214289208Scem * which may hang the system. To workaround this use the second memory 1215289208Scem * window to access the interrupt and scratch pad registers on the 1216289208Scem * remote system. 1217289208Scem */ 1218289543Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1219289543Scem /* Use the last MW for mapping remote spad */ 1220289542Scem ntb->b2b_mw_idx = ntb->mw_count - 1; 1221289543Scem else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1222289208Scem /* 1223289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1224289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1225289542Scem * since bit 14 is the last bit. 1226289542Scem * 1227289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1228289542Scem * anyway. Nor for non-B2B connection types. 1229289542Scem */ 1230289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1231250079Scarl 1232289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1233250079Scarl 1234289542Scem if (ntb->dev_type == NTB_DEV_USD) 1235289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1236289542Scem &xeon_b2b_usd_addr); 1237289542Scem else 1238289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1239289542Scem &xeon_b2b_dsd_addr); 1240289542Scem if (rc != 0) 1241289542Scem return (rc); 1242289271Scem 1243250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1244289607Scem ntb_reg_write(2, XEON_PCICMD_OFFSET, 1245289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1246255279Scarl 1247255269Scarl /* Enable link training */ 1248289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1249250079Scarl 1250250079Scarl return (0); 1251250079Scarl} 1252250079Scarl 1253250079Scarlstatic int 1254289542Scemntb_soc_init_dev(struct ntb_softc *ntb) 1255250079Scarl{ 1256250079Scarl 1257289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1258289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1259250079Scarl 1260289542Scem ntb->spad_count = SOC_SPAD_COUNT; 1261289539Scem ntb->db_count = SOC_DB_COUNT; 1262289539Scem ntb->db_vec_count = SOC_DB_MSIX_VECTOR_COUNT; 1263289539Scem ntb->db_vec_shift = SOC_DB_MSIX_VECTOR_SHIFT; 1264289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1265250079Scarl 1266289542Scem ntb->reg = &soc_reg; 1267289607Scem ntb->self_reg = &soc_pri_reg; 1268289542Scem ntb->peer_reg = &soc_b2b_reg; 1269289542Scem ntb->xlat_reg = &soc_sec_xlat; 1270289542Scem 1271250079Scarl /* 1272250079Scarl * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 1273250079Scarl * resolved. Mask transaction layer internal parity errors. 1274250079Scarl */ 1275250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1276250079Scarl 1277255279Scarl configure_soc_secondary_side_bars(ntb); 1278250079Scarl 1279250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1280289607Scem ntb_reg_write(2, SOC_PCICMD_OFFSET, 1281250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1282289209Scem 1283289542Scem /* Initiate PCI-E link training */ 1284289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1285250079Scarl 1286289542Scem callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb); 1287289542Scem 1288250079Scarl return (0); 1289250079Scarl} 1290250079Scarl 1291289542Scem/* XXX: Linux driver doesn't seem to do any of this for SoC. */ 1292255279Scarlstatic void 1293255279Scarlconfigure_soc_secondary_side_bars(struct ntb_softc *ntb) 1294255279Scarl{ 1295255279Scarl 1296255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1297289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1298289542Scem XEON_B2B_BAR2_DSD_ADDR64); 1299289542Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1300289542Scem XEON_B2B_BAR4_DSD_ADDR64); 1301289542Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64); 1302289542Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64); 1303255279Scarl } else { 1304289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1305289542Scem XEON_B2B_BAR2_USD_ADDR64); 1306289542Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1307289542Scem XEON_B2B_BAR4_USD_ADDR64); 1308289542Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64); 1309289542Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64); 1310255279Scarl } 1311255279Scarl} 1312255279Scarl 1313289543Scem 1314289543Scem/* 1315289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1316289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1317289543Scem * remains for use by a higher layer. 1318289543Scem * 1319289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1320289543Scem * MW size is sufficiently large. 1321289543Scem */ 1322289543Scemstatic unsigned int ntb_b2b_mw_share; 1323289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1324289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1325289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1326289543Scem "value here."); 1327289543Scem 1328289543Scemstatic void 1329289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1330289543Scem enum ntb_bar regbar) 1331289543Scem{ 1332289543Scem struct ntb_pci_bar_info *bar; 1333289543Scem uint8_t bar_sz; 1334289543Scem 1335289543Scem if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1336289543Scem return; 1337289543Scem 1338289543Scem bar = &ntb->bar_info[idx]; 1339289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1340289543Scem if (idx == regbar) { 1341289543Scem if (ntb->b2b_off != 0) 1342289543Scem bar_sz--; 1343289543Scem else 1344289543Scem bar_sz = 0; 1345289543Scem } 1346289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1347289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1348289543Scem (void)bar_sz; 1349289543Scem} 1350289543Scem 1351289543Scemstatic void 1352289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1353289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1354289543Scem{ 1355289546Scem uint64_t reg_val; 1356289546Scem uint32_t base_reg, lmt_reg; 1357289543Scem 1358289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1359289546Scem if (idx == regbar) 1360289546Scem bar_addr += ntb->b2b_off; 1361289543Scem 1362289546Scem if (!bar_is_64bit(ntb, idx)) { 1363289546Scem ntb_reg_write(4, base_reg, bar_addr); 1364289546Scem reg_val = ntb_reg_read(4, base_reg); 1365289546Scem (void)reg_val; 1366289546Scem 1367289546Scem ntb_reg_write(4, lmt_reg, bar_addr); 1368289546Scem reg_val = ntb_reg_read(4, lmt_reg); 1369289546Scem (void)reg_val; 1370289543Scem } else { 1371289546Scem ntb_reg_write(8, base_reg, bar_addr); 1372289546Scem reg_val = ntb_reg_read(8, base_reg); 1373289546Scem (void)reg_val; 1374289546Scem 1375289546Scem ntb_reg_write(8, lmt_reg, bar_addr); 1376289546Scem reg_val = ntb_reg_read(8, lmt_reg); 1377289546Scem (void)reg_val; 1378289543Scem } 1379289543Scem} 1380289543Scem 1381289543Scemstatic void 1382289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1383289543Scem{ 1384289543Scem struct ntb_pci_bar_info *bar; 1385289543Scem 1386289543Scem bar = &ntb->bar_info[idx]; 1387289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1388289543Scem ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1389289543Scem base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1390289543Scem } else { 1391289543Scem ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1392289543Scem base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1393289543Scem } 1394289543Scem (void)base_addr; 1395289543Scem} 1396289543Scem 1397289542Scemstatic int 1398289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1399289542Scem const struct ntb_b2b_addr *peer_addr) 1400255279Scarl{ 1401289543Scem struct ntb_pci_bar_info *b2b_bar; 1402289543Scem vm_size_t bar_size; 1403289543Scem uint64_t bar_addr; 1404289543Scem enum ntb_bar b2b_bar_num, i; 1405255279Scarl 1406289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1407289543Scem b2b_bar = NULL; 1408289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1409289543Scem ntb->b2b_off = 0; 1410289543Scem } else { 1411289543Scem b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1412289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1413289543Scem ("invalid b2b mw bar")); 1414289543Scem 1415289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1416289543Scem bar_size = b2b_bar->size; 1417289543Scem 1418289543Scem if (ntb_b2b_mw_share != 0 && 1419289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1420289543Scem ntb->b2b_off = bar_size >> 1; 1421289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1422289543Scem ntb->b2b_off = 0; 1423289543Scem ntb->mw_count--; 1424289543Scem } else { 1425289543Scem device_printf(ntb->device, 1426289543Scem "B2B bar size is too small!\n"); 1427289543Scem return (EIO); 1428289543Scem } 1429255279Scarl } 1430289542Scem 1431289543Scem /* 1432289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1433289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1434289543Scem */ 1435289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1436289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1437289543Scem 1438289543Scem bar_addr = 0; 1439289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1440289543Scem bar_addr = addr->bar0_addr; 1441289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1442289543Scem bar_addr = addr->bar2_addr64; 1443289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1444289543Scem bar_addr = addr->bar4_addr64; 1445289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1446289543Scem bar_addr = addr->bar4_addr32; 1447289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1448289543Scem bar_addr = addr->bar5_addr32; 1449289543Scem else 1450289543Scem KASSERT(false, ("invalid bar")); 1451289543Scem 1452289543Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1453289543Scem 1454289543Scem /* 1455289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1456289543Scem * register BAR. The B2B BAR is either disabled above or configured 1457289543Scem * half-size. It starts at PBAR xlat + offset. 1458289543Scem * 1459289543Scem * Also set up incoming BAR limits == base (zero length window). 1460289543Scem */ 1461289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1462289543Scem b2b_bar_num); 1463289542Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1464289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1465289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1466289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1467289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1468289542Scem } else 1469289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1470289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1471289543Scem 1472289543Scem /* Zero incoming translation addrs */ 1473289543Scem ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1474289543Scem ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1475289543Scem 1476289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1477289543Scem ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1478289543Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1479289543Scem 1480289543Scem /* Set outgoing translation offsets */ 1481289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1482289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1483289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1484289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1485289543Scem } else 1486289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1487289543Scem 1488289543Scem /* Set the translation offset for B2B registers */ 1489289543Scem bar_addr = 0; 1490289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1491289543Scem bar_addr = peer_addr->bar0_addr; 1492289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1493289543Scem bar_addr = peer_addr->bar2_addr64; 1494289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1495289543Scem bar_addr = peer_addr->bar4_addr64; 1496289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1497289543Scem bar_addr = peer_addr->bar4_addr32; 1498289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1499289543Scem bar_addr = peer_addr->bar5_addr32; 1500289543Scem else 1501289543Scem KASSERT(false, ("invalid bar")); 1502289543Scem 1503289543Scem /* 1504289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1505289543Scem * at a time. 1506289543Scem */ 1507289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1508289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1509289542Scem return (0); 1510255279Scarl} 1511255279Scarl 1512289546Scemstatic inline bool 1513289546Scemlink_is_up(struct ntb_softc *ntb) 1514289546Scem{ 1515289546Scem 1516289611Scem if (ntb->type == NTB_XEON) { 1517289611Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1518289611Scem return (true); 1519289546Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1520289611Scem } 1521289546Scem 1522289546Scem KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1523289546Scem return ((ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0); 1524289546Scem} 1525289546Scem 1526289546Scemstatic inline bool 1527289546Scemsoc_link_is_err(struct ntb_softc *ntb) 1528289546Scem{ 1529289546Scem uint32_t status; 1530289546Scem 1531289546Scem KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1532289546Scem 1533289546Scem status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1534289546Scem if ((status & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1535289546Scem return (true); 1536289546Scem 1537289546Scem status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1538289546Scem return ((status & SOC_IBIST_ERR_OFLOW) != 0); 1539289546Scem} 1540289546Scem 1541255281Scarl/* SOC does not have link status interrupt, poll on that platform */ 1542250079Scarlstatic void 1543289542Scemsoc_link_hb(void *arg) 1544250079Scarl{ 1545250079Scarl struct ntb_softc *ntb = arg; 1546289546Scem sbintime_t timo, poll_ts; 1547250079Scarl 1548289546Scem timo = NTB_HB_TIMEOUT * hz; 1549289546Scem poll_ts = ntb->last_ts + timo; 1550289546Scem 1551289542Scem /* 1552289542Scem * Delay polling the link status if an interrupt was received, unless 1553289542Scem * the cached link status says the link is down. 1554289542Scem */ 1555289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1556289546Scem timo = poll_ts - ticks; 1557289542Scem goto out; 1558289546Scem } 1559289542Scem 1560289546Scem if (ntb_poll_link(ntb)) 1561289546Scem ntb_link_event(ntb); 1562289542Scem 1563289546Scem if (!link_is_up(ntb) && soc_link_is_err(ntb)) { 1564289546Scem /* Link is down with error, proceed with recovery */ 1565289546Scem callout_reset(&ntb->lr_timer, 0, recover_soc_link, ntb); 1566289546Scem return; 1567250079Scarl } 1568250079Scarl 1569289542Scemout: 1570289546Scem callout_reset(&ntb->heartbeat_timer, timo, soc_link_hb, ntb); 1571250079Scarl} 1572250079Scarl 1573250079Scarlstatic void 1574250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb) 1575250079Scarl{ 1576250079Scarl uint32_t status; 1577250079Scarl 1578250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1579255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1580255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1581255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1582255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1583250079Scarl 1584250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1585250079Scarl pause("ModPhy", hz / 10); 1586250079Scarl 1587250079Scarl /* Clear AER Errors, write to clear */ 1588255278Scarl status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1589250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1590255278Scarl ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1591250079Scarl 1592250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1593255278Scarl status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1594250079Scarl status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1595255278Scarl ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1596250079Scarl 1597250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1598255278Scarl status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1599250079Scarl status |= SOC_DESKEWSTS_DBERR; 1600255278Scarl ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1601250079Scarl 1602255278Scarl status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1603250079Scarl status &= SOC_IBIST_ERR_OFLOW; 1604255278Scarl ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1605250079Scarl 1606250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1607255278Scarl status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1608250079Scarl status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1609255278Scarl ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1610250079Scarl} 1611250079Scarl 1612289546Scem/* 1613289546Scem * ntb_set_ctx() - associate a driver context with an ntb device 1614289546Scem * @ntb: NTB device context 1615289546Scem * @ctx: Driver context 1616289546Scem * @ctx_ops: Driver context operations 1617289546Scem * 1618289546Scem * Associate a driver context and operations with a ntb device. The context is 1619289546Scem * provided by the client driver, and the driver may associate a different 1620289546Scem * context with each ntb device. 1621289546Scem * 1622289546Scem * Return: Zero if the context is associated, otherwise an error number. 1623289546Scem */ 1624289546Scemint 1625289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 1626250079Scarl{ 1627250079Scarl 1628289546Scem if (ctx == NULL || ops == NULL) 1629289546Scem return (EINVAL); 1630289546Scem if (ntb->ctx_ops != NULL) 1631289546Scem return (EINVAL); 1632250079Scarl 1633289546Scem CTX_LOCK(ntb); 1634289546Scem if (ntb->ctx_ops != NULL) { 1635289546Scem CTX_UNLOCK(ntb); 1636289546Scem return (EINVAL); 1637250079Scarl } 1638289546Scem ntb->ntb_ctx = ctx; 1639289546Scem ntb->ctx_ops = ops; 1640289546Scem CTX_UNLOCK(ntb); 1641250079Scarl 1642289546Scem return (0); 1643250079Scarl} 1644250079Scarl 1645289546Scem/* 1646289546Scem * It is expected that this will only be used from contexts where the ctx_lock 1647289546Scem * is not needed to protect ntb_ctx lifetime. 1648289546Scem */ 1649289546Scemvoid * 1650289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 1651289546Scem{ 1652289546Scem 1653289546Scem KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 1654289546Scem if (ops != NULL) 1655289546Scem *ops = ntb->ctx_ops; 1656289546Scem return (ntb->ntb_ctx); 1657289546Scem} 1658289546Scem 1659289546Scem/* 1660289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device 1661289546Scem * @ntb: NTB device context 1662289546Scem * 1663289546Scem * Clear any association that may exist between a driver context and the ntb 1664289546Scem * device. 1665289546Scem */ 1666289546Scemvoid 1667289546Scemntb_clear_ctx(struct ntb_softc *ntb) 1668289546Scem{ 1669289546Scem 1670289546Scem CTX_LOCK(ntb); 1671289546Scem ntb->ntb_ctx = NULL; 1672289546Scem ntb->ctx_ops = NULL; 1673289546Scem CTX_UNLOCK(ntb); 1674289546Scem} 1675289546Scem 1676289546Scem/* 1677289546Scem * ntb_link_event() - notify driver context of a change in link status 1678289546Scem * @ntb: NTB device context 1679289546Scem * 1680289546Scem * Notify the driver context that the link status may have changed. The driver 1681289546Scem * should call ntb_link_is_up() to get the current status. 1682289546Scem */ 1683289546Scemvoid 1684289546Scemntb_link_event(struct ntb_softc *ntb) 1685289546Scem{ 1686289546Scem 1687289546Scem CTX_LOCK(ntb); 1688289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 1689289546Scem ntb->ctx_ops->link_event(ntb->ntb_ctx); 1690289546Scem CTX_UNLOCK(ntb); 1691289546Scem} 1692289546Scem 1693289546Scem/* 1694289546Scem * ntb_db_event() - notify driver context of a doorbell event 1695289546Scem * @ntb: NTB device context 1696289546Scem * @vector: Interrupt vector number 1697289546Scem * 1698289546Scem * Notify the driver context of a doorbell event. If hardware supports 1699289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which 1700289546Scem * vector received the interrupt. The vector number is relative to the first 1701289546Scem * vector used for doorbells, starting at zero, and must be less than 1702289546Scem * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 1703289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of 1704289546Scem * those bits are associated with the vector number. 1705289546Scem */ 1706250079Scarlstatic void 1707289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec) 1708289272Scem{ 1709289546Scem 1710289546Scem CTX_LOCK(ntb); 1711289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 1712289546Scem ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 1713289546Scem CTX_UNLOCK(ntb); 1714289546Scem} 1715289546Scem 1716289546Scem/* 1717289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb 1718289546Scem * @ntb: NTB device context 1719289546Scem * @max_speed: The maximum link speed expressed as PCIe generation number[0] 1720289546Scem * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 1721289546Scem * 1722289546Scem * Enable the link on the secondary side of the ntb. This can only be done 1723289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1724289546Scem * should train the link to its maximum speed and width, or the requested speed 1725289546Scem * and width, whichever is smaller, if supported. 1726289546Scem * 1727289546Scem * Return: Zero on success, otherwise an error number. 1728289546Scem * 1729289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 1730289546Scem * and width input will be ignored. 1731289546Scem */ 1732289546Scemint 1733289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 1734289546Scem enum ntb_width w __unused) 1735289546Scem{ 1736289280Scem uint32_t cntl; 1737289272Scem 1738289542Scem if (ntb->type == NTB_SOC) { 1739289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1740289542Scem ntb->ppd | SOC_PPD_INIT_LINK, 4); 1741289546Scem return (0); 1742289542Scem } 1743289542Scem 1744289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1745289546Scem ntb_link_event(ntb); 1746289546Scem return (0); 1747289280Scem } 1748289280Scem 1749289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1750289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1751289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1752289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1753289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1754289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1755289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1756289546Scem return (0); 1757289272Scem} 1758289272Scem 1759289546Scem/* 1760289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb 1761289546Scem * @ntb: NTB device context 1762289546Scem * 1763289546Scem * Disable the link on the secondary side of the ntb. This can only be done 1764289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1765289546Scem * should disable the link. Returning from this call must indicate that a 1766289546Scem * barrier has passed, though with no more writes may pass in either direction 1767289546Scem * across the link, except if this call returns an error number. 1768289546Scem * 1769289546Scem * Return: Zero on success, otherwise an error number. 1770289546Scem */ 1771289546Scemint 1772289542Scemntb_link_disable(struct ntb_softc *ntb) 1773289272Scem{ 1774289272Scem uint32_t cntl; 1775289272Scem 1776289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1777289546Scem ntb_link_event(ntb); 1778289546Scem return (0); 1779289272Scem } 1780289272Scem 1781289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1782289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1783289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1784289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1785289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1786289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1787289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1788289546Scem return (0); 1789289272Scem} 1790289272Scem 1791289272Scemstatic void 1792250079Scarlrecover_soc_link(void *arg) 1793250079Scarl{ 1794250079Scarl struct ntb_softc *ntb = arg; 1795289608Scem unsigned speed, width, oldspeed, oldwidth; 1796250079Scarl uint32_t status32; 1797250079Scarl 1798250079Scarl soc_perform_link_restart(ntb); 1799250079Scarl 1800289232Scem /* 1801289232Scem * There is a potential race between the 2 NTB devices recovering at 1802289232Scem * the same time. If the times are the same, the link will not recover 1803289232Scem * and the driver will be stuck in this loop forever. Add a random 1804289232Scem * interval to the recovery time to prevent this race. 1805289232Scem */ 1806289232Scem status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1807289232Scem pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1808289232Scem 1809289609Scem if (soc_link_is_err(ntb)) 1810250079Scarl goto retry; 1811250079Scarl 1812289542Scem status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1813289232Scem if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1814289232Scem goto out; 1815289232Scem 1816289542Scem status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1817289608Scem width = NTB_LNK_STA_WIDTH(status32); 1818289608Scem speed = status32 & NTB_LINK_SPEED_MASK; 1819289608Scem 1820289608Scem oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 1821289608Scem oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 1822289608Scem if (oldwidth != width || oldspeed != speed) 1823250079Scarl goto retry; 1824250079Scarl 1825289232Scemout: 1826289542Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, 1827289542Scem ntb); 1828250079Scarl return; 1829250079Scarl 1830250079Scarlretry: 1831250079Scarl callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1832250079Scarl ntb); 1833250079Scarl} 1834250079Scarl 1835289546Scem/* 1836289546Scem * Polls the HW link status register(s); returns true if something has changed. 1837289546Scem */ 1838289546Scemstatic bool 1839289542Scemntb_poll_link(struct ntb_softc *ntb) 1840250079Scarl{ 1841250079Scarl uint32_t ntb_cntl; 1842289546Scem uint16_t reg_val; 1843250079Scarl 1844250079Scarl if (ntb->type == NTB_SOC) { 1845289542Scem ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1846289546Scem if (ntb_cntl == ntb->ntb_ctl) 1847289546Scem return (false); 1848289546Scem 1849289542Scem ntb->ntb_ctl = ntb_cntl; 1850289542Scem ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1851250079Scarl } else { 1852289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 1853250079Scarl 1854289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 1855289546Scem if (reg_val == ntb->lnk_sta) 1856289546Scem return (false); 1857250079Scarl 1858289546Scem ntb->lnk_sta = reg_val; 1859289542Scem } 1860289546Scem return (true); 1861289542Scem} 1862289542Scem 1863289546Scemstatic inline enum ntb_speed 1864289546Scemntb_link_sta_speed(struct ntb_softc *ntb) 1865250079Scarl{ 1866250079Scarl 1867289546Scem if (!link_is_up(ntb)) 1868289546Scem return (NTB_SPEED_NONE); 1869289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 1870250079Scarl} 1871250079Scarl 1872289546Scemstatic inline enum ntb_width 1873289546Scemntb_link_sta_width(struct ntb_softc *ntb) 1874250079Scarl{ 1875250079Scarl 1876289546Scem if (!link_is_up(ntb)) 1877289546Scem return (NTB_WIDTH_NONE); 1878289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 1879250079Scarl} 1880250079Scarl 1881289546Scem/* 1882289546Scem * Public API to the rest of the OS 1883250079Scarl */ 1884250079Scarl 1885250079Scarl/** 1886250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 1887250079Scarl * @ntb: pointer to ntb_softc instance 1888250079Scarl * 1889250079Scarl * This function returns the max 32bit scratchpad registers usable by the 1890250079Scarl * upper layer. 1891250079Scarl * 1892250079Scarl * RETURNS: total number of scratch pad registers available 1893250079Scarl */ 1894289208Scemuint8_t 1895250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 1896250079Scarl{ 1897250079Scarl 1898289539Scem return (ntb->spad_count); 1899250079Scarl} 1900250079Scarl 1901289396Scemuint8_t 1902289539Scemntb_mw_count(struct ntb_softc *ntb) 1903289396Scem{ 1904289396Scem 1905289539Scem return (ntb->mw_count); 1906289396Scem} 1907289396Scem 1908250079Scarl/** 1909289545Scem * ntb_spad_write() - write to the secondary scratchpad register 1910250079Scarl * @ntb: pointer to ntb_softc instance 1911250079Scarl * @idx: index to the scratchpad register, 0 based 1912250079Scarl * @val: the data value to put into the register 1913250079Scarl * 1914250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1915250079Scarl * register. The register resides on the secondary (external) side. 1916250079Scarl * 1917289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1918250079Scarl */ 1919250079Scarlint 1920289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1921250079Scarl{ 1922250079Scarl 1923289539Scem if (idx >= ntb->spad_count) 1924250079Scarl return (EINVAL); 1925250079Scarl 1926289607Scem ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 1927250079Scarl 1928250079Scarl return (0); 1929250079Scarl} 1930250079Scarl 1931250079Scarl/** 1932289545Scem * ntb_spad_read() - read from the primary scratchpad register 1933250079Scarl * @ntb: pointer to ntb_softc instance 1934250079Scarl * @idx: index to scratchpad register, 0 based 1935250079Scarl * @val: pointer to 32bit integer for storing the register value 1936250079Scarl * 1937250079Scarl * This function allows reading of the 32bit scratchpad register on 1938250079Scarl * the primary (internal) side. 1939250079Scarl * 1940289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1941250079Scarl */ 1942250079Scarlint 1943289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1944250079Scarl{ 1945250079Scarl 1946289539Scem if (idx >= ntb->spad_count) 1947250079Scarl return (EINVAL); 1948250079Scarl 1949289607Scem *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 1950250079Scarl 1951250079Scarl return (0); 1952250079Scarl} 1953250079Scarl 1954250079Scarl/** 1955289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register 1956250079Scarl * @ntb: pointer to ntb_softc instance 1957250079Scarl * @idx: index to the scratchpad register, 0 based 1958250079Scarl * @val: the data value to put into the register 1959250079Scarl * 1960250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1961250079Scarl * register. The register resides on the secondary (external) side. 1962250079Scarl * 1963289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1964250079Scarl */ 1965250079Scarlint 1966289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1967250079Scarl{ 1968250079Scarl 1969289539Scem if (idx >= ntb->spad_count) 1970250079Scarl return (EINVAL); 1971250079Scarl 1972289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1973255279Scarl ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1974255279Scarl else 1975289542Scem ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 1976250079Scarl 1977250079Scarl return (0); 1978250079Scarl} 1979250079Scarl 1980250079Scarl/** 1981289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register 1982250079Scarl * @ntb: pointer to ntb_softc instance 1983250079Scarl * @idx: index to scratchpad register, 0 based 1984250079Scarl * @val: pointer to 32bit integer for storing the register value 1985250079Scarl * 1986250079Scarl * This function allows reading of the 32bit scratchpad register on 1987250079Scarl * the primary (internal) side. 1988250079Scarl * 1989289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1990250079Scarl */ 1991250079Scarlint 1992289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1993250079Scarl{ 1994250079Scarl 1995289539Scem if (idx >= ntb->spad_count) 1996250079Scarl return (EINVAL); 1997250079Scarl 1998289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1999255279Scarl *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 2000255279Scarl else 2001289542Scem *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2002250079Scarl 2003250079Scarl return (0); 2004250079Scarl} 2005250079Scarl 2006289546Scem/* 2007289546Scem * ntb_mw_get_range() - get the range of a memory window 2008289546Scem * @ntb: NTB device context 2009289546Scem * @idx: Memory window number 2010289546Scem * @base: OUT - the base address for mapping the memory window 2011289546Scem * @size: OUT - the size for mapping the memory window 2012289546Scem * @align: OUT - the base alignment for translating the memory window 2013289546Scem * @align_size: OUT - the size alignment for translating the memory window 2014250079Scarl * 2015289546Scem * Get the range of a memory window. NULL may be given for any output 2016289546Scem * parameter if the value is not needed. The base and size may be used for 2017289546Scem * mapping the memory window, to access the peer memory. The alignment and 2018289546Scem * size may be used for translating the memory window, for the peer to access 2019289546Scem * memory on the local system. 2020250079Scarl * 2021289546Scem * Return: Zero on success, otherwise an error number. 2022250079Scarl */ 2023289546Scemint 2024289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 2025289546Scem void **vbase, size_t *size, size_t *align, size_t *align_size) 2026250079Scarl{ 2027289546Scem struct ntb_pci_bar_info *bar; 2028289546Scem size_t bar_b2b_off; 2029250079Scarl 2030289546Scem if (mw_idx >= ntb_mw_count(ntb)) 2031289546Scem return (EINVAL); 2032250079Scarl 2033289546Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)]; 2034289546Scem bar_b2b_off = 0; 2035289546Scem if (mw_idx == ntb->b2b_mw_idx) { 2036289546Scem KASSERT(ntb->b2b_off != 0, 2037289546Scem ("user shouldn't get non-shared b2b mw")); 2038289546Scem bar_b2b_off = ntb->b2b_off; 2039289546Scem } 2040250079Scarl 2041289546Scem if (base != NULL) 2042289546Scem *base = bar->pbase + bar_b2b_off; 2043289546Scem if (vbase != NULL) 2044289546Scem *vbase = (char *)bar->vbase + bar_b2b_off; 2045289546Scem if (size != NULL) 2046289546Scem *size = bar->size - bar_b2b_off; 2047289546Scem if (align != NULL) 2048289546Scem *align = bar->size; 2049289546Scem if (align_size != NULL) 2050289546Scem *align_size = 1; 2051289546Scem return (0); 2052250079Scarl} 2053250079Scarl 2054289546Scem/* 2055289546Scem * ntb_mw_set_trans() - set the translation of a memory window 2056289546Scem * @ntb: NTB device context 2057289546Scem * @idx: Memory window number 2058289546Scem * @addr: The dma address local memory to expose to the peer 2059289546Scem * @size: The size of the local memory to expose to the peer 2060250079Scarl * 2061289546Scem * Set the translation of a memory window. The peer may access local memory 2062289546Scem * through the window starting at the address, up to the size. The address 2063289546Scem * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2064289546Scem * must be aligned to the size alignment specified by ntb_mw_get_range(). 2065250079Scarl * 2066289546Scem * Return: Zero on success, otherwise an error number. 2067250079Scarl */ 2068289546Scemint 2069289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2070289546Scem size_t size) 2071250079Scarl{ 2072289546Scem struct ntb_pci_bar_info *bar; 2073289546Scem uint64_t base, limit, reg_val; 2074289546Scem size_t bar_size, mw_size; 2075289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2076289546Scem enum ntb_bar bar_num; 2077250079Scarl 2078289546Scem if (idx >= ntb_mw_count(ntb)) 2079289546Scem return (EINVAL); 2080250079Scarl 2081289546Scem bar_num = ntb_mw_to_bar(ntb, idx); 2082289546Scem bar = &ntb->bar_info[bar_num]; 2083250079Scarl 2084289546Scem bar_size = bar->size; 2085289546Scem if (idx == ntb->b2b_mw_idx) 2086289546Scem mw_size = bar_size - ntb->b2b_off; 2087289546Scem else 2088289546Scem mw_size = bar_size; 2089250079Scarl 2090289546Scem /* Hardware requires that addr is aligned to bar size */ 2091289546Scem if ((addr & (bar_size - 1)) != 0) 2092289546Scem return (EINVAL); 2093250079Scarl 2094289546Scem if (size > mw_size) 2095289546Scem return (EINVAL); 2096289546Scem 2097289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2098289546Scem 2099289546Scem limit = 0; 2100289546Scem if (bar_is_64bit(ntb, bar_num)) { 2101289546Scem base = ntb_reg_read(8, base_reg); 2102289546Scem 2103289546Scem if (limit_reg != 0 && size != mw_size) 2104289546Scem limit = base + size; 2105289546Scem 2106289546Scem /* Set and verify translation address */ 2107289546Scem ntb_reg_write(8, xlat_reg, addr); 2108289546Scem reg_val = ntb_reg_read(8, xlat_reg); 2109289546Scem if (reg_val != addr) { 2110289546Scem ntb_reg_write(8, xlat_reg, 0); 2111289546Scem return (EIO); 2112289546Scem } 2113289546Scem 2114289546Scem /* Set and verify the limit */ 2115289546Scem ntb_reg_write(8, limit_reg, limit); 2116289546Scem reg_val = ntb_reg_read(8, limit_reg); 2117289546Scem if (reg_val != limit) { 2118289546Scem ntb_reg_write(8, limit_reg, base); 2119289546Scem ntb_reg_write(8, xlat_reg, 0); 2120289546Scem return (EIO); 2121289546Scem } 2122289546Scem } else { 2123289546Scem /* Configure 32-bit (split) BAR MW */ 2124289546Scem 2125289546Scem if ((addr & ~UINT32_MAX) != 0) 2126289546Scem return (EINVAL); 2127289546Scem if (((addr + size) & ~UINT32_MAX) != 0) 2128289546Scem return (EINVAL); 2129289546Scem 2130289546Scem base = ntb_reg_read(4, base_reg); 2131289546Scem 2132289546Scem if (limit_reg != 0 && size != mw_size) 2133289546Scem limit = base + size; 2134289546Scem 2135289546Scem /* Set and verify translation address */ 2136289546Scem ntb_reg_write(4, xlat_reg, addr); 2137289546Scem reg_val = ntb_reg_read(4, xlat_reg); 2138289546Scem if (reg_val != addr) { 2139289546Scem ntb_reg_write(4, xlat_reg, 0); 2140289546Scem return (EIO); 2141289546Scem } 2142289546Scem 2143289546Scem /* Set and verify the limit */ 2144289546Scem ntb_reg_write(4, limit_reg, limit); 2145289546Scem reg_val = ntb_reg_read(4, limit_reg); 2146289546Scem if (reg_val != limit) { 2147289546Scem ntb_reg_write(4, limit_reg, base); 2148289546Scem ntb_reg_write(4, xlat_reg, 0); 2149289546Scem return (EIO); 2150289546Scem } 2151250079Scarl } 2152289546Scem return (0); 2153250079Scarl} 2154250079Scarl 2155289596Scem/* 2156289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window 2157289596Scem * @ntb: NTB device context 2158289596Scem * @idx: Memory window number 2159289596Scem * 2160289596Scem * Clear the translation of a memory window. The peer may no longer access 2161289596Scem * local memory through the window. 2162289596Scem * 2163289596Scem * Return: Zero on success, otherwise an error number. 2164289596Scem */ 2165289596Scemint 2166289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 2167289596Scem{ 2168289596Scem 2169289596Scem return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 2170289596Scem} 2171289596Scem 2172250079Scarl/** 2173289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side 2174250079Scarl * @ntb: pointer to ntb_softc instance 2175289545Scem * @bit: doorbell bits to ring 2176250079Scarl * 2177250079Scarl * This function allows triggering of a doorbell on the secondary/external 2178250079Scarl * side that will initiate an interrupt on the remote host 2179250079Scarl */ 2180250079Scarlvoid 2181289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 2182250079Scarl{ 2183250079Scarl 2184289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2185289347Scem ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit); 2186289347Scem return; 2187289209Scem } 2188289347Scem 2189289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2190250079Scarl} 2191250079Scarl 2192289542Scem/* 2193289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 2194289542Scem * as well as the size of the register (via *sz_out). 2195289542Scem * 2196289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell 2197289542Scem * ring to its memory window write. 2198289542Scem * 2199289542Scem * Note that writing the peer doorbell via a memory window will *not* generate 2200289542Scem * an interrupt on the remote host; that must be done seperately. 2201289542Scem */ 2202289542Scembus_addr_t 2203289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 2204289542Scem{ 2205289542Scem struct ntb_pci_bar_info *bar; 2206289542Scem uint64_t regoff; 2207289542Scem 2208289542Scem KASSERT(sz_out != NULL, ("must be non-NULL")); 2209289542Scem 2210289542Scem if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2211289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 2212289542Scem regoff = ntb->peer_reg->db_bell; 2213289542Scem } else { 2214289542Scem KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) || 2215289542Scem (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1), 2216289542Scem ("mw_count invalid after setup")); 2217289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 2218289543Scem ("invalid b2b idx")); 2219289542Scem 2220289542Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 2221289542Scem regoff = XEON_SHADOW_PDOORBELL_OFFSET; 2222289542Scem } 2223289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 2224289542Scem 2225289542Scem *sz_out = ntb->reg->db_size; 2226289542Scem /* HACK: Specific to current x86 bus implementation. */ 2227289542Scem return ((uint64_t)bar->pci_bus_handle + regoff); 2228289542Scem} 2229289542Scem 2230289597Scem/* 2231289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 2232289597Scem * @ntb: NTB device context 2233289597Scem * 2234289597Scem * Hardware may support different number or arrangement of doorbell bits. 2235289597Scem * 2236289597Scem * Return: A mask of doorbell bits supported by the ntb. 2237289597Scem */ 2238289597Scemuint64_t 2239289597Scemntb_db_valid_mask(struct ntb_softc *ntb) 2240289597Scem{ 2241289597Scem 2242289597Scem return (ntb->db_valid_mask); 2243289597Scem} 2244289597Scem 2245289598Scem/* 2246289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 2247289598Scem * @ntb: NTB device context 2248289598Scem * @vector: Doorbell vector number 2249289598Scem * 2250289598Scem * Each interrupt vector may have a different number or arrangement of bits. 2251289598Scem * 2252289598Scem * Return: A mask of doorbell bits serviced by a vector. 2253289598Scem */ 2254289598Scemuint64_t 2255289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 2256289598Scem{ 2257289598Scem 2258289598Scem if (vector > ntb->db_vec_count) 2259289598Scem return (0); 2260289598Scem return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 2261289598Scem} 2262289598Scem 2263250079Scarl/** 2264289546Scem * ntb_link_is_up() - get the current ntb link state 2265289546Scem * @ntb: NTB device context 2266289546Scem * @speed: OUT - The link speed expressed as PCIe generation number 2267289546Scem * @width: OUT - The link width expressed as the number of PCIe lanes 2268250079Scarl * 2269250079Scarl * RETURNS: true or false based on the hardware link state 2270250079Scarl */ 2271250079Scarlbool 2272289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 2273289546Scem enum ntb_width *width) 2274250079Scarl{ 2275250079Scarl 2276289546Scem if (speed != NULL) 2277289546Scem *speed = ntb_link_sta_speed(ntb); 2278289546Scem if (width != NULL) 2279289546Scem *width = ntb_link_sta_width(ntb); 2280289546Scem return (link_is_up(ntb)); 2281250079Scarl} 2282250079Scarl 2283255272Scarlstatic void 2284255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 2285250079Scarl{ 2286255272Scarl 2287289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2288289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2289289209Scem bar->pbase = rman_get_start(bar->pci_resource); 2290289209Scem bar->size = rman_get_size(bar->pci_resource); 2291289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 2292250079Scarl} 2293255268Scarl 2294289209Scemdevice_t 2295289209Scemntb_get_device(struct ntb_softc *ntb) 2296255268Scarl{ 2297255268Scarl 2298255268Scarl return (ntb->device); 2299255268Scarl} 2300289208Scem 2301289208Scem/* Export HW-specific errata information. */ 2302289208Scembool 2303289208Scemntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 2304289208Scem{ 2305289208Scem 2306289208Scem return (HAS_FEATURE(feature)); 2307289208Scem} 2308