ntb_hw_intel.c revision 290679
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3289542Scem * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 290679 2015-11-11 18:54:58Z cem $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35289774Scem#include <sys/endian.h> 36250079Scarl#include <sys/malloc.h> 37250079Scarl#include <sys/module.h> 38250079Scarl#include <sys/queue.h> 39250079Scarl#include <sys/rman.h> 40289774Scem#include <sys/sbuf.h> 41289207Scem#include <sys/sysctl.h> 42250079Scarl#include <vm/vm.h> 43250079Scarl#include <vm/pmap.h> 44250079Scarl#include <machine/bus.h> 45250079Scarl#include <machine/pmap.h> 46250079Scarl#include <machine/resource.h> 47250079Scarl#include <dev/pci/pcireg.h> 48250079Scarl#include <dev/pci/pcivar.h> 49250079Scarl 50250079Scarl#include "ntb_regs.h" 51250079Scarl#include "ntb_hw.h" 52250079Scarl 53250079Scarl/* 54250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 55250079Scarl * allows you to connect two systems using a PCI-e link. 56250079Scarl * 57250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 58250079Scarl * you to send and recieve interrupts, map the memory windows and send and 59250079Scarl * receive messages in the scratch-pad registers. 60250079Scarl * 61250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 62250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 63250079Scarl */ 64250079Scarl 65289648Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 66250079Scarl 67289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 68289648Scem#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 69250079Scarl 70250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 71250079Scarl 72250079Scarlenum ntb_device_type { 73250079Scarl NTB_XEON, 74289648Scem NTB_ATOM 75250079Scarl}; 76250079Scarl 77289610Scem/* ntb_conn_type are hardware numbers, cannot change. */ 78289610Scemenum ntb_conn_type { 79289610Scem NTB_CONN_TRANSPARENT = 0, 80289610Scem NTB_CONN_B2B = 1, 81289610Scem NTB_CONN_RP = 2, 82289610Scem}; 83289610Scem 84289610Scemenum ntb_b2b_direction { 85289610Scem NTB_DEV_USD = 0, 86289610Scem NTB_DEV_DSD = 1, 87289610Scem}; 88289610Scem 89289539Scemenum ntb_bar { 90289539Scem NTB_CONFIG_BAR = 0, 91289539Scem NTB_B2B_BAR_1, 92289539Scem NTB_B2B_BAR_2, 93289539Scem NTB_B2B_BAR_3, 94289539Scem NTB_MAX_BARS 95289539Scem}; 96289539Scem 97255274Scarl/* Device features and workarounds */ 98255274Scarl#define HAS_FEATURE(feature) \ 99255274Scarl ((ntb->features & (feature)) != 0) 100255274Scarl 101250079Scarlstruct ntb_hw_info { 102250079Scarl uint32_t device_id; 103255274Scarl const char *desc; 104250079Scarl enum ntb_device_type type; 105289397Scem uint32_t features; 106250079Scarl}; 107250079Scarl 108250079Scarlstruct ntb_pci_bar_info { 109250079Scarl bus_space_tag_t pci_bus_tag; 110250079Scarl bus_space_handle_t pci_bus_handle; 111250079Scarl int pci_resource_id; 112250079Scarl struct resource *pci_resource; 113250079Scarl vm_paddr_t pbase; 114290679Scem caddr_t vbase; 115290679Scem vm_size_t size; 116289543Scem 117289543Scem /* Configuration register offsets */ 118289543Scem uint32_t psz_off; 119289543Scem uint32_t ssz_off; 120289543Scem uint32_t pbarxlat_off; 121250079Scarl}; 122250079Scarl 123250079Scarlstruct ntb_int_info { 124250079Scarl struct resource *res; 125250079Scarl int rid; 126250079Scarl void *tag; 127250079Scarl}; 128250079Scarl 129289546Scemstruct ntb_vec { 130250079Scarl struct ntb_softc *ntb; 131289546Scem uint32_t num; 132250079Scarl}; 133250079Scarl 134289542Scemstruct ntb_reg { 135289542Scem uint32_t ntb_ctl; 136289542Scem uint32_t lnk_sta; 137289542Scem uint8_t db_size; 138289542Scem unsigned mw_bar[NTB_MAX_BARS]; 139289542Scem}; 140289542Scem 141289542Scemstruct ntb_alt_reg { 142289542Scem uint32_t db_bell; 143289542Scem uint32_t db_mask; 144289542Scem uint32_t spad; 145289542Scem}; 146289542Scem 147289542Scemstruct ntb_xlat_reg { 148289546Scem uint32_t bar0_base; 149289546Scem uint32_t bar2_base; 150289546Scem uint32_t bar4_base; 151289546Scem uint32_t bar5_base; 152289546Scem 153289546Scem uint32_t bar2_xlat; 154289546Scem uint32_t bar4_xlat; 155289546Scem uint32_t bar5_xlat; 156289546Scem 157289546Scem uint32_t bar2_limit; 158289546Scem uint32_t bar4_limit; 159289546Scem uint32_t bar5_limit; 160289542Scem}; 161289542Scem 162289542Scemstruct ntb_b2b_addr { 163289542Scem uint64_t bar0_addr; 164289542Scem uint64_t bar2_addr64; 165289542Scem uint64_t bar4_addr64; 166289542Scem uint64_t bar4_addr32; 167289542Scem uint64_t bar5_addr32; 168289542Scem}; 169289542Scem 170250079Scarlstruct ntb_softc { 171250079Scarl device_t device; 172250079Scarl enum ntb_device_type type; 173289774Scem uint32_t features; 174250079Scarl 175250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 176250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 177250079Scarl uint32_t allocated_interrupts; 178250079Scarl 179250079Scarl struct callout heartbeat_timer; 180250079Scarl struct callout lr_timer; 181250079Scarl 182289546Scem void *ntb_ctx; 183289546Scem const struct ntb_ctx_ops *ctx_ops; 184289546Scem struct ntb_vec *msix_vec; 185289546Scem#define CTX_LOCK(sc) mtx_lock_spin(&(sc)->ctx_lock) 186289546Scem#define CTX_UNLOCK(sc) mtx_unlock_spin(&(sc)->ctx_lock) 187289546Scem#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 188289546Scem struct mtx ctx_lock; 189250079Scarl 190289610Scem uint32_t ppd; 191289610Scem enum ntb_conn_type conn_type; 192289610Scem enum ntb_b2b_direction dev_type; 193289539Scem 194289542Scem /* Offset of peer bar0 in B2B BAR */ 195289542Scem uint64_t b2b_off; 196289542Scem /* Memory window used to access peer bar0 */ 197289543Scem#define B2B_MW_DISABLED UINT8_MAX 198289542Scem uint8_t b2b_mw_idx; 199289542Scem 200289539Scem uint8_t mw_count; 201289539Scem uint8_t spad_count; 202289539Scem uint8_t db_count; 203289539Scem uint8_t db_vec_count; 204289539Scem uint8_t db_vec_shift; 205289542Scem 206289546Scem /* Protects local db_mask. */ 207289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 208289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 209289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 210289542Scem struct mtx db_mask_lock; 211289542Scem 212289546Scem uint32_t ntb_ctl; 213289546Scem uint32_t lnk_sta; 214289542Scem 215289542Scem uint64_t db_valid_mask; 216289542Scem uint64_t db_link_mask; 217289546Scem uint64_t db_mask; 218289542Scem 219289542Scem int last_ts; /* ticks @ last irq */ 220289542Scem 221289542Scem const struct ntb_reg *reg; 222289542Scem const struct ntb_alt_reg *self_reg; 223289542Scem const struct ntb_alt_reg *peer_reg; 224289542Scem const struct ntb_xlat_reg *xlat_reg; 225250079Scarl}; 226250079Scarl 227289234Scem#ifdef __i386__ 228289234Scemstatic __inline uint64_t 229289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 230289234Scem bus_size_t offset) 231289234Scem{ 232289234Scem 233289234Scem return (bus_space_read_4(tag, handle, offset) | 234289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 235289234Scem} 236289234Scem 237289234Scemstatic __inline void 238289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 239289234Scem bus_size_t offset, uint64_t val) 240289234Scem{ 241289234Scem 242289234Scem bus_space_write_4(tag, handle, offset, val); 243289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 244289234Scem} 245289234Scem#endif 246289234Scem 247255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 248255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 249255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 250255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 251255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 252255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 253255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 254250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 255255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 256289397Scem#define ntb_mw_read(SIZE, offset) \ 257289542Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 258255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 259289542Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 260289397Scem offset, val) 261250079Scarl 262250079Scarlstatic int ntb_probe(device_t device); 263250079Scarlstatic int ntb_attach(device_t device); 264250079Scarlstatic int ntb_detach(device_t device); 265289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 266289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 267289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 268289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 269255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 270289647Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 271289647Scem const char *); 272255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 273255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 274255272Scarl struct ntb_pci_bar_info *bar); 275250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 276289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 277289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 278289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 279289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 280250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 281289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 282289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 283289546Scemstatic void ndev_vec_isr(void *arg); 284289546Scemstatic void ndev_irq_isr(void *arg); 285289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 286290678Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 287290678Scemstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 288289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 289289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 290250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 291289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 292289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 293289648Scemstatic int ntb_detect_atom(struct ntb_softc *ntb); 294289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 295289648Scemstatic int ntb_atom_init_dev(struct ntb_softc *ntb); 296289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 297289648Scemstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 298289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 299289543Scem enum ntb_bar regbar); 300289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 301289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 302289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 303289543Scem enum ntb_bar idx); 304289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 305289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 306289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 307289648Scemstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 308289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 309289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 310289648Scemstatic void atom_link_hb(void *arg); 311289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 312289648Scemstatic void recover_atom_link(void *arg); 313289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb); 314255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 315289774Scemstatic void ntb_sysctl_init(struct ntb_softc *); 316289774Scemstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 317289774Scemstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 318289774Scemstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 319250079Scarl 320250079Scarlstatic struct ntb_hw_info pci_ids[] = { 321289612Scem /* XXX: PS/SS IDs left out until they are supported. */ 322289612Scem { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 323289648Scem NTB_ATOM, 0 }, 324289233Scem 325289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 326289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 327289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 328289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 329289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 330289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 331289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 332289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 333289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 334289538Scem NTB_SB01BASE_LOCKUP }, 335289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 336289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 337289538Scem NTB_SB01BASE_LOCKUP }, 338289233Scem 339289648Scem { 0x00000000, NULL, NTB_ATOM, 0 } 340250079Scarl}; 341250079Scarl 342289648Scemstatic const struct ntb_reg atom_reg = { 343289648Scem .ntb_ctl = ATOM_NTBCNTL_OFFSET, 344289648Scem .lnk_sta = ATOM_LINK_STATUS_OFFSET, 345289542Scem .db_size = sizeof(uint64_t), 346289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 347289542Scem}; 348289542Scem 349289648Scemstatic const struct ntb_alt_reg atom_pri_reg = { 350289648Scem .db_bell = ATOM_PDOORBELL_OFFSET, 351289648Scem .db_mask = ATOM_PDBMSK_OFFSET, 352289648Scem .spad = ATOM_SPAD_OFFSET, 353289607Scem}; 354289607Scem 355289648Scemstatic const struct ntb_alt_reg atom_b2b_reg = { 356289648Scem .db_bell = ATOM_B2B_DOORBELL_OFFSET, 357289648Scem .spad = ATOM_B2B_SPAD_OFFSET, 358289542Scem}; 359289542Scem 360289648Scemstatic const struct ntb_xlat_reg atom_sec_xlat = { 361289542Scem#if 0 362289542Scem /* "FIXME" says the Linux driver. */ 363289648Scem .bar0_base = ATOM_SBAR0BASE_OFFSET, 364289648Scem .bar2_base = ATOM_SBAR2BASE_OFFSET, 365289648Scem .bar4_base = ATOM_SBAR4BASE_OFFSET, 366289546Scem 367289648Scem .bar2_limit = ATOM_SBAR2LMT_OFFSET, 368289648Scem .bar4_limit = ATOM_SBAR4LMT_OFFSET, 369289542Scem#endif 370289546Scem 371289648Scem .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 372289648Scem .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 373289542Scem}; 374289542Scem 375289542Scemstatic const struct ntb_reg xeon_reg = { 376289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 377289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 378289542Scem .db_size = sizeof(uint16_t), 379289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 380289542Scem}; 381289542Scem 382289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 383289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 384289607Scem .db_mask = XEON_PDBMSK_OFFSET, 385289607Scem .spad = XEON_SPAD_OFFSET, 386289607Scem}; 387289607Scem 388289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 389289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 390289542Scem .spad = XEON_B2B_SPAD_OFFSET, 391289542Scem}; 392289542Scem 393289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 394289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 395289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 396289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 397289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 398289546Scem 399289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 400289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 401289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 402289546Scem 403289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 404289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 405289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 406289542Scem}; 407289542Scem 408289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 409289542Scem .bar0_addr = XEON_B2B_BAR0_USD_ADDR, 410289542Scem .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, 411289542Scem .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, 412289542Scem .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, 413289542Scem .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, 414289542Scem}; 415289542Scem 416289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 417289542Scem .bar0_addr = XEON_B2B_BAR0_DSD_ADDR, 418289542Scem .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, 419289542Scem .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, 420289542Scem .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, 421289542Scem .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, 422289542Scem}; 423289542Scem 424289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 425289614Scem "B2B MW segment overrides -- MUST be the same on both sides"); 426289614Scem 427289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 428289614Scem &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 429289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 430289614Scem "the window at BAR2, on the upstream side of the link. MUST be the same " 431289614Scem "address on both sides."); 432289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 433289614Scem &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 434289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 435289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 436289614Scem "(split-BAR mode)."); 437289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 438289646Scem &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 439289614Scem "(split-BAR mode)."); 440289614Scem 441289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 442289614Scem &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 443289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 444289614Scem "the window at BAR2, on the downstream side of the link. MUST be the same" 445289614Scem " address on both sides."); 446289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 447289614Scem &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 448289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 449289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 450289614Scem "(split-BAR mode)."); 451289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 452289646Scem &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 453289614Scem "(split-BAR mode)."); 454289614Scem 455250079Scarl/* 456250079Scarl * OS <-> Driver interface structures 457250079Scarl */ 458250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 459250079Scarl 460250079Scarlstatic device_method_t ntb_pci_methods[] = { 461250079Scarl /* Device interface */ 462250079Scarl DEVMETHOD(device_probe, ntb_probe), 463250079Scarl DEVMETHOD(device_attach, ntb_attach), 464250079Scarl DEVMETHOD(device_detach, ntb_detach), 465250079Scarl DEVMETHOD_END 466250079Scarl}; 467250079Scarl 468250079Scarlstatic driver_t ntb_pci_driver = { 469250079Scarl "ntb_hw", 470250079Scarl ntb_pci_methods, 471250079Scarl sizeof(struct ntb_softc), 472250079Scarl}; 473250079Scarl 474250079Scarlstatic devclass_t ntb_devclass; 475250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 476250079ScarlMODULE_VERSION(ntb_hw, 1); 477250079Scarl 478289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 479289207Scem 480250079Scarl/* 481250079Scarl * OS <-> Driver linkage functions 482250079Scarl */ 483250079Scarlstatic int 484250079Scarlntb_probe(device_t device) 485250079Scarl{ 486289209Scem struct ntb_hw_info *p; 487250079Scarl 488289209Scem p = ntb_get_device_info(pci_get_devid(device)); 489289209Scem if (p == NULL) 490250079Scarl return (ENXIO); 491289209Scem 492289209Scem device_set_desc(device, p->desc); 493289209Scem return (0); 494250079Scarl} 495250079Scarl 496250079Scarlstatic int 497250079Scarlntb_attach(device_t device) 498250079Scarl{ 499289209Scem struct ntb_softc *ntb; 500289209Scem struct ntb_hw_info *p; 501250079Scarl int error; 502250079Scarl 503289209Scem ntb = DEVICE2SOFTC(device); 504289209Scem p = ntb_get_device_info(pci_get_devid(device)); 505289209Scem 506250079Scarl ntb->device = device; 507250079Scarl ntb->type = p->type; 508255274Scarl ntb->features = p->features; 509289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 510250079Scarl 511289648Scem /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 512283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 513283291Sjkim callout_init(&ntb->lr_timer, 1); 514289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 515289546Scem mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN); 516250079Scarl 517289648Scem if (ntb->type == NTB_ATOM) 518289648Scem error = ntb_detect_atom(ntb); 519289348Scem else 520289348Scem error = ntb_detect_xeon(ntb); 521289348Scem if (error) 522289348Scem goto out; 523289348Scem 524289397Scem ntb_detect_max_mw(ntb); 525289396Scem 526289209Scem error = ntb_map_pci_bars(ntb); 527289209Scem if (error) 528289209Scem goto out; 529289648Scem if (ntb->type == NTB_ATOM) 530289648Scem error = ntb_atom_init_dev(ntb); 531289272Scem else 532289542Scem error = ntb_xeon_init_dev(ntb); 533289209Scem if (error) 534289209Scem goto out; 535289540Scem error = ntb_init_isr(ntb); 536289209Scem if (error) 537289209Scem goto out; 538289774Scem ntb_sysctl_init(ntb); 539250079Scarl 540250079Scarl pci_enable_busmaster(ntb->device); 541250079Scarl 542289209Scemout: 543289209Scem if (error != 0) 544289209Scem ntb_detach(device); 545250079Scarl return (error); 546250079Scarl} 547250079Scarl 548250079Scarlstatic int 549250079Scarlntb_detach(device_t device) 550250079Scarl{ 551289209Scem struct ntb_softc *ntb; 552250079Scarl 553289209Scem ntb = DEVICE2SOFTC(device); 554289542Scem 555289617Scem if (ntb->self_reg != NULL) 556289617Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 557250079Scarl callout_drain(&ntb->heartbeat_timer); 558250079Scarl callout_drain(&ntb->lr_timer); 559289272Scem if (ntb->type == NTB_XEON) 560289272Scem ntb_teardown_xeon(ntb); 561250079Scarl ntb_teardown_interrupts(ntb); 562289397Scem 563289542Scem mtx_destroy(&ntb->db_mask_lock); 564289546Scem mtx_destroy(&ntb->ctx_lock); 565289542Scem 566289397Scem /* 567289397Scem * Redetect total MWs so we unmap properly -- in case we lowered the 568289397Scem * maximum to work around Xeon errata. 569289397Scem */ 570289397Scem ntb_detect_max_mw(ntb); 571250079Scarl ntb_unmap_pci_bar(ntb); 572250079Scarl 573250079Scarl return (0); 574250079Scarl} 575250079Scarl 576289542Scem/* 577289542Scem * Driver internal routines 578289542Scem */ 579289539Scemstatic inline enum ntb_bar 580289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 581289539Scem{ 582289539Scem 583289543Scem KASSERT(mw < ntb->mw_count || 584289543Scem (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx), 585289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 586289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 587289539Scem 588289542Scem return (ntb->reg->mw_bar[mw]); 589289539Scem} 590289539Scem 591289546Scemstatic inline bool 592289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 593289546Scem{ 594289546Scem /* XXX This assertion could be stronger. */ 595289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 596289546Scem return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 597289546Scem} 598289546Scem 599289546Scemstatic inline void 600289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 601289546Scem uint32_t *xlat, uint32_t *lmt) 602289546Scem{ 603289546Scem uint32_t basev, lmtv, xlatv; 604289546Scem 605289546Scem switch (bar) { 606289546Scem case NTB_B2B_BAR_1: 607289546Scem basev = ntb->xlat_reg->bar2_base; 608289546Scem lmtv = ntb->xlat_reg->bar2_limit; 609289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 610289546Scem break; 611289546Scem case NTB_B2B_BAR_2: 612289546Scem basev = ntb->xlat_reg->bar4_base; 613289546Scem lmtv = ntb->xlat_reg->bar4_limit; 614289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 615289546Scem break; 616289546Scem case NTB_B2B_BAR_3: 617289546Scem basev = ntb->xlat_reg->bar5_base; 618289546Scem lmtv = ntb->xlat_reg->bar5_limit; 619289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 620289546Scem break; 621289546Scem default: 622289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 623289546Scem ("bad bar")); 624289546Scem basev = lmtv = xlatv = 0; 625289546Scem break; 626289546Scem } 627289546Scem 628289546Scem if (base != NULL) 629289546Scem *base = basev; 630289546Scem if (xlat != NULL) 631289546Scem *xlat = xlatv; 632289546Scem if (lmt != NULL) 633289546Scem *lmt = lmtv; 634289546Scem} 635289546Scem 636250079Scarlstatic int 637255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 638250079Scarl{ 639255272Scarl int rc; 640250079Scarl 641250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 642289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 643255272Scarl if (rc != 0) 644289541Scem goto out; 645255272Scarl 646289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 647289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 648255272Scarl if (rc != 0) 649289541Scem goto out; 650289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 651289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 652289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 653255272Scarl 654289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 655289543Scem /* XXX Are shared MW B2Bs write-combining? */ 656289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 657289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 658255279Scarl else 659289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 660289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 661289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 662289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 663289543Scem 664289397Scem if (!HAS_FEATURE(NTB_SPLIT_BAR)) 665289541Scem goto out; 666289397Scem 667289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 668289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 669289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 670289397Scem else 671289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 672289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 673289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 674289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 675250079Scarl 676289541Scemout: 677289209Scem if (rc != 0) 678255272Scarl device_printf(ntb->device, 679255272Scarl "unable to allocate pci resource\n"); 680255272Scarl return (rc); 681255272Scarl} 682255272Scarl 683289541Scemstatic void 684289647Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 685289647Scem const char *kind) 686289541Scem{ 687289541Scem 688289647Scem device_printf(ntb->device, 689289647Scem "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 690289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 691289647Scem (char *)bar->vbase + bar->size - 1, 692289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 693289647Scem (uintmax_t)bar->size, kind); 694289541Scem} 695289541Scem 696255272Scarlstatic int 697255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 698255272Scarl{ 699255272Scarl 700255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 701289209Scem &bar->pci_resource_id, RF_ACTIVE); 702255272Scarl if (bar->pci_resource == NULL) 703255272Scarl return (ENXIO); 704289209Scem 705289209Scem save_bar_parameters(bar); 706289647Scem print_map_success(ntb, bar, "mmr"); 707289209Scem return (0); 708255272Scarl} 709255272Scarl 710255272Scarlstatic int 711255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 712255272Scarl{ 713255272Scarl int rc; 714255276Scarl uint8_t bar_size_bits = 0; 715255272Scarl 716289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 717289209Scem &bar->pci_resource_id, RF_ACTIVE); 718250079Scarl 719255272Scarl if (bar->pci_resource == NULL) 720255272Scarl return (ENXIO); 721255276Scarl 722289209Scem save_bar_parameters(bar); 723289209Scem /* 724289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 725289209Scem * hardware issue. To work around this, query the size it should be 726289209Scem * configured to by the device and modify the resource to correspond to 727289209Scem * this new size. The BIOS on systems with this problem is required to 728289209Scem * provide enough address space to allow the driver to make this change 729289209Scem * safely. 730289209Scem * 731289209Scem * Ideally I could have just specified the size when I allocated the 732289209Scem * resource like: 733289209Scem * bus_alloc_resource(ntb->device, 734289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 735289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 736289209Scem * but the PCI driver does not honor the size in this call, so we have 737289209Scem * to modify it after the fact. 738289209Scem */ 739289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 740289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 741289209Scem bar_size_bits = pci_read_config(ntb->device, 742289209Scem XEON_PBAR23SZ_OFFSET, 1); 743289209Scem else 744289209Scem bar_size_bits = pci_read_config(ntb->device, 745289209Scem XEON_PBAR45SZ_OFFSET, 1); 746289209Scem 747289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 748289209Scem bar->pci_resource, bar->pbase, 749289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 750255272Scarl if (rc != 0) { 751289209Scem device_printf(ntb->device, 752289209Scem "unable to resize bar\n"); 753255272Scarl return (rc); 754250079Scarl } 755289209Scem 756289209Scem save_bar_parameters(bar); 757250079Scarl } 758289209Scem 759289209Scem /* Mark bar region as write combining to improve performance. */ 760289209Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 761289209Scem VM_MEMATTR_WRITE_COMBINING); 762289647Scem print_map_success(ntb, bar, "mw"); 763289647Scem if (rc == 0) 764289209Scem device_printf(ntb->device, 765289647Scem "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 766289647Scem "WRITE_COMBINING.\n", 767289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 768289647Scem (char *)bar->vbase + bar->size - 1, 769289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1)); 770289647Scem else 771289647Scem device_printf(ntb->device, 772289647Scem "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 773289647Scem "WRITE_COMBINING: %d\n", 774289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 775289647Scem (char *)bar->vbase + bar->size - 1, 776289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 777289647Scem rc); 778289647Scem /* Proceed anyway */ 779250079Scarl return (0); 780250079Scarl} 781250079Scarl 782250079Scarlstatic void 783250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 784250079Scarl{ 785250079Scarl struct ntb_pci_bar_info *current_bar; 786250079Scarl int i; 787250079Scarl 788289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 789250079Scarl current_bar = &ntb->bar_info[i]; 790250079Scarl if (current_bar->pci_resource != NULL) 791250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 792250079Scarl current_bar->pci_resource_id, 793250079Scarl current_bar->pci_resource); 794250079Scarl } 795250079Scarl} 796250079Scarl 797250079Scarlstatic int 798289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 799250079Scarl{ 800289342Scem uint32_t i; 801289342Scem int rc; 802289342Scem 803289342Scem for (i = 0; i < num_vectors; i++) { 804289342Scem ntb->int_info[i].rid = i + 1; 805289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 806289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 807289342Scem if (ntb->int_info[i].res == NULL) { 808289342Scem device_printf(ntb->device, 809289342Scem "bus_alloc_resource failed\n"); 810289342Scem return (ENOMEM); 811289342Scem } 812289342Scem ntb->int_info[i].tag = NULL; 813289342Scem ntb->allocated_interrupts++; 814289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 815289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 816289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 817289342Scem if (rc != 0) { 818289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 819289342Scem return (ENXIO); 820289342Scem } 821289342Scem } 822289342Scem return (0); 823289342Scem} 824289342Scem 825289344Scem/* 826289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 827289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 828289344Scem * should be okay. This tunable should enable us to test that hypothesis 829289344Scem * when someone gets their hands on some Xeon hardware. 830289344Scem */ 831289344Scemstatic int ntb_force_remap_mode; 832289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 833289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 834289344Scem " to a smaller number of ithreads, even if the desired number are " 835289344Scem "available"); 836289344Scem 837289344Scem/* 838289344Scem * In case it is NOT ok, give consumers an abort button. 839289344Scem */ 840289344Scemstatic int ntb_prefer_intx; 841289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 842289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 843289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 844289344Scem "behavior)"); 845289344Scem 846289344Scem/* 847289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 848289344Scem * round-robin fashion. 849289344Scem */ 850289342Scemstatic int 851289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 852289344Scem{ 853289344Scem u_int *vectors; 854289344Scem uint32_t i; 855289344Scem int rc; 856289344Scem 857289344Scem if (ntb_prefer_intx != 0) 858289344Scem return (ENXIO); 859289344Scem 860289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 861289344Scem 862289344Scem for (i = 0; i < desired; i++) 863289344Scem vectors[i] = (i % avail) + 1; 864289344Scem 865289344Scem rc = pci_remap_msix(dev, desired, vectors); 866289344Scem free(vectors, M_NTB); 867289344Scem return (rc); 868289344Scem} 869289344Scem 870289344Scemstatic int 871289540Scemntb_init_isr(struct ntb_softc *ntb) 872289342Scem{ 873289344Scem uint32_t desired_vectors, num_vectors; 874289342Scem int rc; 875250079Scarl 876250079Scarl ntb->allocated_interrupts = 0; 877289542Scem ntb->last_ts = ticks; 878289347Scem 879250079Scarl /* 880289546Scem * Mask all doorbell interrupts. 881250079Scarl */ 882289546Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 883250079Scarl 884289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 885289539Scem ntb->db_count); 886289344Scem if (desired_vectors >= 1) { 887289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 888250079Scarl 889289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 890289344Scem num_vectors == desired_vectors) 891289344Scem num_vectors--; 892289344Scem 893289344Scem if (rc == 0 && num_vectors < desired_vectors) { 894289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 895289344Scem num_vectors); 896289344Scem if (rc == 0) 897289344Scem num_vectors = desired_vectors; 898289344Scem else 899289344Scem pci_release_msi(ntb->device); 900289344Scem } 901289344Scem if (rc != 0) 902289344Scem num_vectors = 1; 903289344Scem } else 904289344Scem num_vectors = 1; 905289344Scem 906289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 907289539Scem ntb->db_vec_count = 1; 908289539Scem ntb->db_vec_shift = ntb->db_count; 909289539Scem rc = ntb_setup_legacy_interrupt(ntb); 910289539Scem } else { 911289546Scem ntb_create_msix_vec(ntb, num_vectors); 912289540Scem rc = ntb_setup_msix(ntb, num_vectors); 913289539Scem } 914289539Scem if (rc != 0) { 915289539Scem device_printf(ntb->device, 916289539Scem "Error allocating interrupts: %d\n", rc); 917289546Scem ntb_free_msix_vec(ntb); 918289396Scem } 919289396Scem 920289342Scem return (rc); 921289342Scem} 922289342Scem 923289342Scemstatic int 924289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 925289342Scem{ 926289342Scem int rc; 927289342Scem 928289342Scem ntb->int_info[0].rid = 0; 929289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 930289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 931289342Scem if (ntb->int_info[0].res == NULL) { 932289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 933289342Scem return (ENOMEM); 934250079Scarl } 935250079Scarl 936289342Scem ntb->int_info[0].tag = NULL; 937289342Scem ntb->allocated_interrupts = 1; 938289342Scem 939289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 940289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 941289342Scem ntb, &ntb->int_info[0].tag); 942289342Scem if (rc != 0) { 943289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 944289342Scem return (ENXIO); 945289342Scem } 946289342Scem 947250079Scarl return (0); 948250079Scarl} 949250079Scarl 950250079Scarlstatic void 951250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 952250079Scarl{ 953250079Scarl struct ntb_int_info *current_int; 954250079Scarl int i; 955250079Scarl 956289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 957250079Scarl current_int = &ntb->int_info[i]; 958250079Scarl if (current_int->tag != NULL) 959250079Scarl bus_teardown_intr(ntb->device, current_int->res, 960250079Scarl current_int->tag); 961250079Scarl 962250079Scarl if (current_int->res != NULL) 963250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 964250079Scarl rman_get_rid(current_int->res), current_int->res); 965250079Scarl } 966250079Scarl 967289546Scem ntb_free_msix_vec(ntb); 968250079Scarl pci_release_msi(ntb->device); 969250079Scarl} 970250079Scarl 971289347Scem/* 972289648Scem * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 973289347Scem * out to make code clearer. 974289347Scem */ 975289539Scemstatic inline uint64_t 976289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 977289347Scem{ 978289347Scem 979289648Scem if (ntb->type == NTB_ATOM) 980289347Scem return (ntb_reg_read(8, regoff)); 981289347Scem 982289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 983289347Scem 984289347Scem return (ntb_reg_read(2, regoff)); 985289347Scem} 986289347Scem 987289539Scemstatic inline void 988289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 989289347Scem{ 990289347Scem 991289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 992289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 993289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 994289542Scem (uintmax_t)ntb->db_valid_mask)); 995289542Scem 996289607Scem if (regoff == ntb->self_reg->db_mask) 997289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 998290678Scem db_iowrite_raw(ntb, regoff, val); 999290678Scem} 1000289542Scem 1001290678Scemstatic inline void 1002290678Scemdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1003290678Scem{ 1004290678Scem 1005289648Scem if (ntb->type == NTB_ATOM) { 1006289347Scem ntb_reg_write(8, regoff, val); 1007289347Scem return; 1008289347Scem } 1009289347Scem 1010289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1011289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 1012289347Scem} 1013289347Scem 1014289546Scemvoid 1015289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1016289542Scem{ 1017289542Scem 1018289546Scem DB_MASK_LOCK(ntb); 1019289542Scem ntb->db_mask |= bits; 1020289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1021289546Scem DB_MASK_UNLOCK(ntb); 1022289542Scem} 1023289542Scem 1024289546Scemvoid 1025289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1026289542Scem{ 1027289542Scem 1028289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1029289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1030289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1031289542Scem (uintmax_t)ntb->db_valid_mask)); 1032289542Scem 1033289546Scem DB_MASK_LOCK(ntb); 1034289542Scem ntb->db_mask &= ~bits; 1035289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1036289546Scem DB_MASK_UNLOCK(ntb); 1037289542Scem} 1038289542Scem 1039289546Scemuint64_t 1040289546Scemntb_db_read(struct ntb_softc *ntb) 1041289281Scem{ 1042289281Scem 1043289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 1044289281Scem} 1045289281Scem 1046289546Scemvoid 1047289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1048289281Scem{ 1049289281Scem 1050289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1051289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1052289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1053289546Scem (uintmax_t)ntb->db_valid_mask)); 1054289546Scem 1055289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1056289281Scem} 1057289281Scem 1058289540Scemstatic inline uint64_t 1059289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1060250079Scarl{ 1061289540Scem uint64_t shift, mask; 1062250079Scarl 1063289540Scem shift = ntb->db_vec_shift; 1064289540Scem mask = (1ull << shift) - 1; 1065289540Scem return (mask << (shift * db_vector)); 1066250079Scarl} 1067250079Scarl 1068250079Scarlstatic void 1069289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1070250079Scarl{ 1071289540Scem uint64_t vec_mask; 1072250079Scarl 1073289542Scem ntb->last_ts = ticks; 1074289546Scem vec_mask = ntb_vec_mask(ntb, vec); 1075250079Scarl 1076289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1077289546Scem if (ntb_poll_link(ntb)) 1078289546Scem ntb_link_event(ntb); 1079289540Scem } 1080289540Scem 1081289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1082289546Scem ntb_db_event(ntb, vec); 1083289546Scem} 1084250079Scarl 1085289546Scemstatic void 1086289546Scemndev_vec_isr(void *arg) 1087289546Scem{ 1088289546Scem struct ntb_vec *nvec = arg; 1089250079Scarl 1090289546Scem ntb_interrupt(nvec->ntb, nvec->num); 1091250079Scarl} 1092250079Scarl 1093250079Scarlstatic void 1094289546Scemndev_irq_isr(void *arg) 1095250079Scarl{ 1096289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1097289546Scem ntb_interrupt(arg, 0); 1098250079Scarl} 1099250079Scarl 1100250079Scarlstatic int 1101289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1102250079Scarl{ 1103289342Scem uint32_t i; 1104250079Scarl 1105289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1106250079Scarl M_ZERO | M_WAITOK); 1107250079Scarl for (i = 0; i < num_vectors; i++) { 1108289546Scem ntb->msix_vec[i].num = i; 1109289546Scem ntb->msix_vec[i].ntb = ntb; 1110250079Scarl } 1111250079Scarl 1112250079Scarl return (0); 1113250079Scarl} 1114250079Scarl 1115250079Scarlstatic void 1116289546Scemntb_free_msix_vec(struct ntb_softc *ntb) 1117250079Scarl{ 1118250079Scarl 1119289546Scem if (ntb->msix_vec == NULL) 1120289539Scem return; 1121289539Scem 1122289546Scem free(ntb->msix_vec, M_NTB); 1123289546Scem ntb->msix_vec = NULL; 1124250079Scarl} 1125250079Scarl 1126250079Scarlstatic struct ntb_hw_info * 1127250079Scarlntb_get_device_info(uint32_t device_id) 1128250079Scarl{ 1129250079Scarl struct ntb_hw_info *ep = pci_ids; 1130250079Scarl 1131250079Scarl while (ep->device_id) { 1132250079Scarl if (ep->device_id == device_id) 1133250079Scarl return (ep); 1134250079Scarl ++ep; 1135250079Scarl } 1136250079Scarl return (NULL); 1137250079Scarl} 1138250079Scarl 1139289272Scemstatic void 1140289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 1141250079Scarl{ 1142250079Scarl 1143289617Scem if (ntb->reg != NULL) 1144289617Scem ntb_link_disable(ntb); 1145250079Scarl} 1146250079Scarl 1147289397Scemstatic void 1148289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 1149289397Scem{ 1150289397Scem 1151289648Scem if (ntb->type == NTB_ATOM) { 1152289648Scem ntb->mw_count = ATOM_MW_COUNT; 1153289397Scem return; 1154289397Scem } 1155289397Scem 1156289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1157289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1158289397Scem else 1159289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1160289397Scem} 1161289397Scem 1162250079Scarlstatic int 1163289348Scemntb_detect_xeon(struct ntb_softc *ntb) 1164250079Scarl{ 1165289348Scem uint8_t ppd, conn_type; 1166250079Scarl 1167289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1168289348Scem ntb->ppd = ppd; 1169250079Scarl 1170289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1171289257Scem ntb->dev_type = NTB_DEV_USD; 1172289257Scem else 1173289257Scem ntb->dev_type = NTB_DEV_DSD; 1174289257Scem 1175289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1176289397Scem ntb->features |= NTB_SPLIT_BAR; 1177289397Scem 1178289542Scem /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1179289542Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1180289542Scem ntb->features |= NTB_SDOORBELL_LOCKUP; 1181289542Scem 1182289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1183289348Scem switch (conn_type) { 1184289348Scem case NTB_CONN_B2B: 1185289348Scem ntb->conn_type = conn_type; 1186289348Scem break; 1187289348Scem case NTB_CONN_RP: 1188289348Scem case NTB_CONN_TRANSPARENT: 1189289348Scem default: 1190289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1191289348Scem (unsigned)conn_type); 1192289348Scem return (ENXIO); 1193289348Scem } 1194289348Scem return (0); 1195289348Scem} 1196289348Scem 1197289348Scemstatic int 1198289648Scemntb_detect_atom(struct ntb_softc *ntb) 1199289348Scem{ 1200289348Scem uint32_t ppd, conn_type; 1201289348Scem 1202289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1203289348Scem ntb->ppd = ppd; 1204289348Scem 1205289648Scem if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1206289348Scem ntb->dev_type = NTB_DEV_DSD; 1207289348Scem else 1208289348Scem ntb->dev_type = NTB_DEV_USD; 1209289348Scem 1210289648Scem conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1211289348Scem switch (conn_type) { 1212289348Scem case NTB_CONN_B2B: 1213289348Scem ntb->conn_type = conn_type; 1214289348Scem break; 1215289348Scem default: 1216289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1217289348Scem return (ENXIO); 1218289348Scem } 1219289348Scem return (0); 1220289348Scem} 1221289348Scem 1222289348Scemstatic int 1223289542Scemntb_xeon_init_dev(struct ntb_softc *ntb) 1224289348Scem{ 1225289542Scem int rc; 1226289348Scem 1227289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1228289542Scem ntb->db_count = XEON_DB_COUNT; 1229289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1230289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1231289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1232289257Scem 1233289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1234250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1235289348Scem ntb->conn_type); 1236250079Scarl return (ENXIO); 1237250079Scarl } 1238250079Scarl 1239289542Scem ntb->reg = &xeon_reg; 1240289607Scem ntb->self_reg = &xeon_pri_reg; 1241289542Scem ntb->peer_reg = &xeon_b2b_reg; 1242289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1243289542Scem 1244289208Scem /* 1245289208Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1246289208Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1247289208Scem * which may hang the system. To workaround this use the second memory 1248289208Scem * window to access the interrupt and scratch pad registers on the 1249289208Scem * remote system. 1250289208Scem */ 1251289543Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1252289543Scem /* Use the last MW for mapping remote spad */ 1253289542Scem ntb->b2b_mw_idx = ntb->mw_count - 1; 1254289543Scem else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1255289208Scem /* 1256289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1257289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1258289542Scem * since bit 14 is the last bit. 1259289542Scem * 1260289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1261289542Scem * anyway. Nor for non-B2B connection types. 1262289542Scem */ 1263289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1264250079Scarl 1265289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1266250079Scarl 1267289542Scem if (ntb->dev_type == NTB_DEV_USD) 1268289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1269289542Scem &xeon_b2b_usd_addr); 1270289542Scem else 1271289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1272289542Scem &xeon_b2b_dsd_addr); 1273289542Scem if (rc != 0) 1274289542Scem return (rc); 1275289271Scem 1276250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1277289607Scem ntb_reg_write(2, XEON_PCICMD_OFFSET, 1278289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1279255279Scarl 1280255269Scarl /* Enable link training */ 1281289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1282250079Scarl 1283250079Scarl return (0); 1284250079Scarl} 1285250079Scarl 1286250079Scarlstatic int 1287289648Scemntb_atom_init_dev(struct ntb_softc *ntb) 1288250079Scarl{ 1289250079Scarl 1290289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1291289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1292250079Scarl 1293289648Scem ntb->spad_count = ATOM_SPAD_COUNT; 1294289648Scem ntb->db_count = ATOM_DB_COUNT; 1295289648Scem ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1296289648Scem ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1297289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1298250079Scarl 1299289648Scem ntb->reg = &atom_reg; 1300289648Scem ntb->self_reg = &atom_pri_reg; 1301289648Scem ntb->peer_reg = &atom_b2b_reg; 1302289648Scem ntb->xlat_reg = &atom_sec_xlat; 1303289542Scem 1304250079Scarl /* 1305289648Scem * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1306250079Scarl * resolved. Mask transaction layer internal parity errors. 1307250079Scarl */ 1308250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1309250079Scarl 1310289648Scem configure_atom_secondary_side_bars(ntb); 1311250079Scarl 1312250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1313289648Scem ntb_reg_write(2, ATOM_PCICMD_OFFSET, 1314250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1315289209Scem 1316289542Scem /* Initiate PCI-E link training */ 1317289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1318250079Scarl 1319289648Scem callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1320289542Scem 1321250079Scarl return (0); 1322250079Scarl} 1323250079Scarl 1324289648Scem/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1325255279Scarlstatic void 1326289648Scemconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1327255279Scarl{ 1328255279Scarl 1329255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1330289648Scem ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1331289542Scem XEON_B2B_BAR2_DSD_ADDR64); 1332289648Scem ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1333289542Scem XEON_B2B_BAR4_DSD_ADDR64); 1334289648Scem ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64); 1335289648Scem ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64); 1336255279Scarl } else { 1337289648Scem ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1338289542Scem XEON_B2B_BAR2_USD_ADDR64); 1339289648Scem ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1340289542Scem XEON_B2B_BAR4_USD_ADDR64); 1341289648Scem ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64); 1342289648Scem ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64); 1343255279Scarl } 1344255279Scarl} 1345255279Scarl 1346289543Scem 1347289543Scem/* 1348289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1349289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1350289543Scem * remains for use by a higher layer. 1351289543Scem * 1352289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1353289543Scem * MW size is sufficiently large. 1354289543Scem */ 1355289543Scemstatic unsigned int ntb_b2b_mw_share; 1356289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1357289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1358289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1359289543Scem "value here."); 1360289543Scem 1361289543Scemstatic void 1362289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1363289543Scem enum ntb_bar regbar) 1364289543Scem{ 1365289543Scem struct ntb_pci_bar_info *bar; 1366289543Scem uint8_t bar_sz; 1367289543Scem 1368289543Scem if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1369289543Scem return; 1370289543Scem 1371289543Scem bar = &ntb->bar_info[idx]; 1372289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1373289543Scem if (idx == regbar) { 1374289543Scem if (ntb->b2b_off != 0) 1375289543Scem bar_sz--; 1376289543Scem else 1377289543Scem bar_sz = 0; 1378289543Scem } 1379289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1380289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1381289543Scem (void)bar_sz; 1382289543Scem} 1383289543Scem 1384289543Scemstatic void 1385289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1386289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1387289543Scem{ 1388289546Scem uint64_t reg_val; 1389289546Scem uint32_t base_reg, lmt_reg; 1390289543Scem 1391289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1392289546Scem if (idx == regbar) 1393289546Scem bar_addr += ntb->b2b_off; 1394289543Scem 1395289546Scem if (!bar_is_64bit(ntb, idx)) { 1396289546Scem ntb_reg_write(4, base_reg, bar_addr); 1397289546Scem reg_val = ntb_reg_read(4, base_reg); 1398289546Scem (void)reg_val; 1399289546Scem 1400289546Scem ntb_reg_write(4, lmt_reg, bar_addr); 1401289546Scem reg_val = ntb_reg_read(4, lmt_reg); 1402289546Scem (void)reg_val; 1403289543Scem } else { 1404289546Scem ntb_reg_write(8, base_reg, bar_addr); 1405289546Scem reg_val = ntb_reg_read(8, base_reg); 1406289546Scem (void)reg_val; 1407289546Scem 1408289546Scem ntb_reg_write(8, lmt_reg, bar_addr); 1409289546Scem reg_val = ntb_reg_read(8, lmt_reg); 1410289546Scem (void)reg_val; 1411289543Scem } 1412289543Scem} 1413289543Scem 1414289543Scemstatic void 1415289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1416289543Scem{ 1417289543Scem struct ntb_pci_bar_info *bar; 1418289543Scem 1419289543Scem bar = &ntb->bar_info[idx]; 1420289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1421289543Scem ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1422289543Scem base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1423289543Scem } else { 1424289543Scem ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1425289543Scem base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1426289543Scem } 1427289543Scem (void)base_addr; 1428289543Scem} 1429289543Scem 1430289542Scemstatic int 1431289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1432289542Scem const struct ntb_b2b_addr *peer_addr) 1433255279Scarl{ 1434289543Scem struct ntb_pci_bar_info *b2b_bar; 1435289543Scem vm_size_t bar_size; 1436289543Scem uint64_t bar_addr; 1437289543Scem enum ntb_bar b2b_bar_num, i; 1438255279Scarl 1439289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1440289543Scem b2b_bar = NULL; 1441289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1442289543Scem ntb->b2b_off = 0; 1443289543Scem } else { 1444289543Scem b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1445289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1446289543Scem ("invalid b2b mw bar")); 1447289543Scem 1448289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1449289543Scem bar_size = b2b_bar->size; 1450289543Scem 1451289543Scem if (ntb_b2b_mw_share != 0 && 1452289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1453289543Scem ntb->b2b_off = bar_size >> 1; 1454289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1455289543Scem ntb->b2b_off = 0; 1456289543Scem ntb->mw_count--; 1457289543Scem } else { 1458289543Scem device_printf(ntb->device, 1459289543Scem "B2B bar size is too small!\n"); 1460289543Scem return (EIO); 1461289543Scem } 1462255279Scarl } 1463289542Scem 1464289543Scem /* 1465289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1466289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1467289543Scem */ 1468289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1469289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1470289543Scem 1471289543Scem bar_addr = 0; 1472289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1473289543Scem bar_addr = addr->bar0_addr; 1474289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1475289543Scem bar_addr = addr->bar2_addr64; 1476289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1477289543Scem bar_addr = addr->bar4_addr64; 1478289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1479289543Scem bar_addr = addr->bar4_addr32; 1480289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1481289543Scem bar_addr = addr->bar5_addr32; 1482289543Scem else 1483289543Scem KASSERT(false, ("invalid bar")); 1484289543Scem 1485289543Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1486289543Scem 1487289543Scem /* 1488289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1489289543Scem * register BAR. The B2B BAR is either disabled above or configured 1490289543Scem * half-size. It starts at PBAR xlat + offset. 1491289543Scem * 1492289543Scem * Also set up incoming BAR limits == base (zero length window). 1493289543Scem */ 1494289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1495289543Scem b2b_bar_num); 1496289542Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1497289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1498289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1499289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1500289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1501289542Scem } else 1502289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1503289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1504289543Scem 1505289543Scem /* Zero incoming translation addrs */ 1506289543Scem ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1507289543Scem ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1508289543Scem 1509289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1510289543Scem ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1511289543Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1512289543Scem 1513289543Scem /* Set outgoing translation offsets */ 1514289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1515289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1516289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1517289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1518289543Scem } else 1519289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1520289543Scem 1521289543Scem /* Set the translation offset for B2B registers */ 1522289543Scem bar_addr = 0; 1523289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1524289543Scem bar_addr = peer_addr->bar0_addr; 1525289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1526289543Scem bar_addr = peer_addr->bar2_addr64; 1527289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1528289543Scem bar_addr = peer_addr->bar4_addr64; 1529289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1530289543Scem bar_addr = peer_addr->bar4_addr32; 1531289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1532289543Scem bar_addr = peer_addr->bar5_addr32; 1533289543Scem else 1534289543Scem KASSERT(false, ("invalid bar")); 1535289543Scem 1536289543Scem /* 1537289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1538289543Scem * at a time. 1539289543Scem */ 1540289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1541289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1542289542Scem return (0); 1543255279Scarl} 1544255279Scarl 1545289546Scemstatic inline bool 1546289546Scemlink_is_up(struct ntb_softc *ntb) 1547289546Scem{ 1548289546Scem 1549289611Scem if (ntb->type == NTB_XEON) { 1550289611Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1551289611Scem return (true); 1552289546Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1553289611Scem } 1554289546Scem 1555289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1556289648Scem return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1557289546Scem} 1558289546Scem 1559289546Scemstatic inline bool 1560289648Scematom_link_is_err(struct ntb_softc *ntb) 1561289546Scem{ 1562289546Scem uint32_t status; 1563289546Scem 1564289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1565289546Scem 1566289648Scem status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1567289648Scem if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1568289546Scem return (true); 1569289546Scem 1570289648Scem status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1571289648Scem return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1572289546Scem} 1573289546Scem 1574289648Scem/* Atom does not have link status interrupt, poll on that platform */ 1575250079Scarlstatic void 1576289648Scematom_link_hb(void *arg) 1577250079Scarl{ 1578250079Scarl struct ntb_softc *ntb = arg; 1579289546Scem sbintime_t timo, poll_ts; 1580250079Scarl 1581289546Scem timo = NTB_HB_TIMEOUT * hz; 1582289546Scem poll_ts = ntb->last_ts + timo; 1583289546Scem 1584289542Scem /* 1585289542Scem * Delay polling the link status if an interrupt was received, unless 1586289542Scem * the cached link status says the link is down. 1587289542Scem */ 1588289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1589289546Scem timo = poll_ts - ticks; 1590289542Scem goto out; 1591289546Scem } 1592289542Scem 1593289546Scem if (ntb_poll_link(ntb)) 1594289546Scem ntb_link_event(ntb); 1595289542Scem 1596289648Scem if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1597289546Scem /* Link is down with error, proceed with recovery */ 1598289648Scem callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1599289546Scem return; 1600250079Scarl } 1601250079Scarl 1602289542Scemout: 1603289648Scem callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1604250079Scarl} 1605250079Scarl 1606250079Scarlstatic void 1607289648Scematom_perform_link_restart(struct ntb_softc *ntb) 1608250079Scarl{ 1609250079Scarl uint32_t status; 1610250079Scarl 1611250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1612289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1613289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1614289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1615289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1616250079Scarl 1617250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1618250079Scarl pause("ModPhy", hz / 10); 1619250079Scarl 1620250079Scarl /* Clear AER Errors, write to clear */ 1621289648Scem status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1622250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1623289648Scem ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1624250079Scarl 1625250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1626289648Scem status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1627289648Scem status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1628289648Scem ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1629250079Scarl 1630250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1631289648Scem status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1632289648Scem status |= ATOM_DESKEWSTS_DBERR; 1633289648Scem ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1634250079Scarl 1635289648Scem status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1636289648Scem status &= ATOM_IBIST_ERR_OFLOW; 1637289648Scem ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1638250079Scarl 1639250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1640289648Scem status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1641289648Scem status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1642289648Scem ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1643250079Scarl} 1644250079Scarl 1645289546Scem/* 1646289546Scem * ntb_set_ctx() - associate a driver context with an ntb device 1647289546Scem * @ntb: NTB device context 1648289546Scem * @ctx: Driver context 1649289546Scem * @ctx_ops: Driver context operations 1650289546Scem * 1651289546Scem * Associate a driver context and operations with a ntb device. The context is 1652289546Scem * provided by the client driver, and the driver may associate a different 1653289546Scem * context with each ntb device. 1654289546Scem * 1655289546Scem * Return: Zero if the context is associated, otherwise an error number. 1656289546Scem */ 1657289546Scemint 1658289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 1659250079Scarl{ 1660250079Scarl 1661289546Scem if (ctx == NULL || ops == NULL) 1662289546Scem return (EINVAL); 1663289546Scem if (ntb->ctx_ops != NULL) 1664289546Scem return (EINVAL); 1665250079Scarl 1666289546Scem CTX_LOCK(ntb); 1667289546Scem if (ntb->ctx_ops != NULL) { 1668289546Scem CTX_UNLOCK(ntb); 1669289546Scem return (EINVAL); 1670250079Scarl } 1671289546Scem ntb->ntb_ctx = ctx; 1672289546Scem ntb->ctx_ops = ops; 1673289546Scem CTX_UNLOCK(ntb); 1674250079Scarl 1675289546Scem return (0); 1676250079Scarl} 1677250079Scarl 1678289546Scem/* 1679289546Scem * It is expected that this will only be used from contexts where the ctx_lock 1680289546Scem * is not needed to protect ntb_ctx lifetime. 1681289546Scem */ 1682289546Scemvoid * 1683289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 1684289546Scem{ 1685289546Scem 1686289546Scem KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 1687289546Scem if (ops != NULL) 1688289546Scem *ops = ntb->ctx_ops; 1689289546Scem return (ntb->ntb_ctx); 1690289546Scem} 1691289546Scem 1692289546Scem/* 1693289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device 1694289546Scem * @ntb: NTB device context 1695289546Scem * 1696289546Scem * Clear any association that may exist between a driver context and the ntb 1697289546Scem * device. 1698289546Scem */ 1699289546Scemvoid 1700289546Scemntb_clear_ctx(struct ntb_softc *ntb) 1701289546Scem{ 1702289546Scem 1703289546Scem CTX_LOCK(ntb); 1704289546Scem ntb->ntb_ctx = NULL; 1705289546Scem ntb->ctx_ops = NULL; 1706289546Scem CTX_UNLOCK(ntb); 1707289546Scem} 1708289546Scem 1709289546Scem/* 1710289546Scem * ntb_link_event() - notify driver context of a change in link status 1711289546Scem * @ntb: NTB device context 1712289546Scem * 1713289546Scem * Notify the driver context that the link status may have changed. The driver 1714289546Scem * should call ntb_link_is_up() to get the current status. 1715289546Scem */ 1716289546Scemvoid 1717289546Scemntb_link_event(struct ntb_softc *ntb) 1718289546Scem{ 1719289546Scem 1720289546Scem CTX_LOCK(ntb); 1721289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 1722289546Scem ntb->ctx_ops->link_event(ntb->ntb_ctx); 1723289546Scem CTX_UNLOCK(ntb); 1724289546Scem} 1725289546Scem 1726289546Scem/* 1727289546Scem * ntb_db_event() - notify driver context of a doorbell event 1728289546Scem * @ntb: NTB device context 1729289546Scem * @vector: Interrupt vector number 1730289546Scem * 1731289546Scem * Notify the driver context of a doorbell event. If hardware supports 1732289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which 1733289546Scem * vector received the interrupt. The vector number is relative to the first 1734289546Scem * vector used for doorbells, starting at zero, and must be less than 1735289546Scem * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 1736289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of 1737289546Scem * those bits are associated with the vector number. 1738289546Scem */ 1739250079Scarlstatic void 1740289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec) 1741289272Scem{ 1742289546Scem 1743289546Scem CTX_LOCK(ntb); 1744289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 1745289546Scem ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 1746289546Scem CTX_UNLOCK(ntb); 1747289546Scem} 1748289546Scem 1749289546Scem/* 1750289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb 1751289546Scem * @ntb: NTB device context 1752289546Scem * @max_speed: The maximum link speed expressed as PCIe generation number[0] 1753289546Scem * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 1754289546Scem * 1755289546Scem * Enable the link on the secondary side of the ntb. This can only be done 1756289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1757289546Scem * should train the link to its maximum speed and width, or the requested speed 1758289546Scem * and width, whichever is smaller, if supported. 1759289546Scem * 1760289546Scem * Return: Zero on success, otherwise an error number. 1761289546Scem * 1762289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 1763289546Scem * and width input will be ignored. 1764289546Scem */ 1765289546Scemint 1766289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 1767289546Scem enum ntb_width w __unused) 1768289546Scem{ 1769289280Scem uint32_t cntl; 1770289272Scem 1771289648Scem if (ntb->type == NTB_ATOM) { 1772289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1773289648Scem ntb->ppd | ATOM_PPD_INIT_LINK, 4); 1774289546Scem return (0); 1775289542Scem } 1776289542Scem 1777289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1778289546Scem ntb_link_event(ntb); 1779289546Scem return (0); 1780289280Scem } 1781289280Scem 1782289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1783289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1784289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1785289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1786289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1787289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1788289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1789289546Scem return (0); 1790289272Scem} 1791289272Scem 1792289546Scem/* 1793289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb 1794289546Scem * @ntb: NTB device context 1795289546Scem * 1796289546Scem * Disable the link on the secondary side of the ntb. This can only be done 1797289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1798289546Scem * should disable the link. Returning from this call must indicate that a 1799289546Scem * barrier has passed, though with no more writes may pass in either direction 1800289546Scem * across the link, except if this call returns an error number. 1801289546Scem * 1802289546Scem * Return: Zero on success, otherwise an error number. 1803289546Scem */ 1804289546Scemint 1805289542Scemntb_link_disable(struct ntb_softc *ntb) 1806289272Scem{ 1807289272Scem uint32_t cntl; 1808289272Scem 1809289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1810289546Scem ntb_link_event(ntb); 1811289546Scem return (0); 1812289272Scem } 1813289272Scem 1814289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1815289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1816289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1817289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1818289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1819289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1820289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1821289546Scem return (0); 1822289272Scem} 1823289272Scem 1824289272Scemstatic void 1825289648Scemrecover_atom_link(void *arg) 1826250079Scarl{ 1827250079Scarl struct ntb_softc *ntb = arg; 1828289608Scem unsigned speed, width, oldspeed, oldwidth; 1829250079Scarl uint32_t status32; 1830250079Scarl 1831289648Scem atom_perform_link_restart(ntb); 1832250079Scarl 1833289232Scem /* 1834289232Scem * There is a potential race between the 2 NTB devices recovering at 1835289232Scem * the same time. If the times are the same, the link will not recover 1836289232Scem * and the driver will be stuck in this loop forever. Add a random 1837289232Scem * interval to the recovery time to prevent this race. 1838289232Scem */ 1839289648Scem status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 1840289648Scem pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 1841289232Scem 1842289648Scem if (atom_link_is_err(ntb)) 1843250079Scarl goto retry; 1844250079Scarl 1845289542Scem status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1846289648Scem if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 1847289232Scem goto out; 1848289232Scem 1849289542Scem status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1850289608Scem width = NTB_LNK_STA_WIDTH(status32); 1851289608Scem speed = status32 & NTB_LINK_SPEED_MASK; 1852289608Scem 1853289608Scem oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 1854289608Scem oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 1855289608Scem if (oldwidth != width || oldspeed != speed) 1856250079Scarl goto retry; 1857250079Scarl 1858289232Scemout: 1859289648Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 1860289542Scem ntb); 1861250079Scarl return; 1862250079Scarl 1863250079Scarlretry: 1864289648Scem callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 1865250079Scarl ntb); 1866250079Scarl} 1867250079Scarl 1868289546Scem/* 1869289546Scem * Polls the HW link status register(s); returns true if something has changed. 1870289546Scem */ 1871289546Scemstatic bool 1872289542Scemntb_poll_link(struct ntb_softc *ntb) 1873250079Scarl{ 1874250079Scarl uint32_t ntb_cntl; 1875289546Scem uint16_t reg_val; 1876250079Scarl 1877289648Scem if (ntb->type == NTB_ATOM) { 1878289542Scem ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1879289546Scem if (ntb_cntl == ntb->ntb_ctl) 1880289546Scem return (false); 1881289546Scem 1882289542Scem ntb->ntb_ctl = ntb_cntl; 1883289542Scem ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1884250079Scarl } else { 1885290678Scem db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 1886250079Scarl 1887289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 1888289546Scem if (reg_val == ntb->lnk_sta) 1889289546Scem return (false); 1890250079Scarl 1891289546Scem ntb->lnk_sta = reg_val; 1892289542Scem } 1893289546Scem return (true); 1894289542Scem} 1895289542Scem 1896289546Scemstatic inline enum ntb_speed 1897289546Scemntb_link_sta_speed(struct ntb_softc *ntb) 1898250079Scarl{ 1899250079Scarl 1900289546Scem if (!link_is_up(ntb)) 1901289546Scem return (NTB_SPEED_NONE); 1902289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 1903250079Scarl} 1904250079Scarl 1905289546Scemstatic inline enum ntb_width 1906289546Scemntb_link_sta_width(struct ntb_softc *ntb) 1907250079Scarl{ 1908250079Scarl 1909289546Scem if (!link_is_up(ntb)) 1910289546Scem return (NTB_WIDTH_NONE); 1911289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 1912250079Scarl} 1913250079Scarl 1914289774ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 1915289774Scem "Driver state, statistics, and HW registers"); 1916289774Scem 1917289774Scem#define NTB_REGSZ_MASK (3ul << 30) 1918289774Scem#define NTB_REG_64 (1ul << 30) 1919289774Scem#define NTB_REG_32 (2ul << 30) 1920289774Scem#define NTB_REG_16 (3ul << 30) 1921289774Scem#define NTB_REG_8 (0ul << 30) 1922289774Scem 1923289774Scem#define NTB_DB_READ (1ul << 29) 1924289774Scem#define NTB_PCI_REG (1ul << 28) 1925289774Scem#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 1926289774Scem 1927289774Scemstatic void 1928289774Scemntb_sysctl_init(struct ntb_softc *ntb) 1929289774Scem{ 1930289774Scem struct sysctl_oid_list *tree_par, *regpar, *statpar, *errpar; 1931289774Scem struct sysctl_ctx_list *ctx; 1932289774Scem struct sysctl_oid *tree, *tmptree; 1933289774Scem 1934289774Scem ctx = device_get_sysctl_ctx(ntb->device); 1935289774Scem 1936289774Scem tree = SYSCTL_ADD_NODE(ctx, 1937289774Scem SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)), OID_AUTO, 1938289774Scem "debug_info", CTLFLAG_RD, NULL, 1939289774Scem "Driver state, statistics, and HW registers"); 1940289774Scem tree_par = SYSCTL_CHILDREN(tree); 1941289774Scem 1942289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 1943289774Scem &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 1944289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 1945289774Scem &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 1946289774Scem 1947289774Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 1948289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 1949289774Scem &ntb->b2b_mw_idx, 0, 1950289774Scem "Index of the MW used for B2B remote register access"); 1951289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 1952289774Scem CTLFLAG_RD, &ntb->b2b_off, 1953289774Scem "If non-zero, offset of B2B register region in shared MW"); 1954289774Scem } 1955289774Scem 1956289774Scem SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 1957289774Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 1958289774Scem "Features/errata of this NTB device"); 1959289774Scem 1960289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 1961289774Scem &ntb->ntb_ctl, 0, "NTB CTL register (cached)"); 1962289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 1963289774Scem &ntb->lnk_sta, 0, "LNK STA register (cached)"); 1964289774Scem 1965289774Scem SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "link_status", 1966289774Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_link_status, 1967289774Scem "A", "Link status"); 1968289774Scem 1969289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 1970289774Scem &ntb->mw_count, 0, "MW count (excl. non-shared B2B register BAR)"); 1971289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 1972289774Scem &ntb->spad_count, 0, "Scratchpad count"); 1973289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 1974289774Scem &ntb->db_count, 0, "Doorbell count"); 1975289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 1976289774Scem &ntb->db_vec_count, 0, "Doorbell vector count"); 1977289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 1978289774Scem &ntb->db_vec_shift, 0, "Doorbell vector shift"); 1979289774Scem 1980289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 1981289774Scem &ntb->db_valid_mask, "Doorbell valid mask"); 1982289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 1983289774Scem &ntb->db_link_mask, "Doorbell link mask"); 1984289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 1985289774Scem &ntb->db_mask, "Doorbell mask (cached)"); 1986289774Scem 1987289774Scem tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 1988289774Scem CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 1989289774Scem regpar = SYSCTL_CHILDREN(tmptree); 1990289774Scem 1991289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 1992289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 1993289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 1994289774Scem sysctl_handle_register, "QU", "Doorbell mask register"); 1995289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 1996289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 1997289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 1998289774Scem sysctl_handle_register, "QU", "Doorbell register"); 1999289774Scem 2000289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2001289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2002289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2003289774Scem sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2004289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2005289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2006289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2007289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2008289774Scem sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2009289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2010289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2011289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2012289774Scem sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2013289774Scem } else { 2014289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2015289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2016289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2017289774Scem sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2018289774Scem } 2019289774Scem 2020289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2021289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2022289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2023289774Scem sysctl_handle_register, "QU", "Incoming LMT23 register"); 2024289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2025289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2026289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2027289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2028289774Scem sysctl_handle_register, "IU", "Incoming LMT4 register"); 2029289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2030289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2031289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2032289774Scem sysctl_handle_register, "IU", "Incoming LMT5 register"); 2033289774Scem } else { 2034289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2035289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2036289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2037289774Scem sysctl_handle_register, "QU", "Incoming LMT45 register"); 2038289774Scem } 2039289774Scem 2040289774Scem if (ntb->type == NTB_ATOM) 2041289774Scem return; 2042289774Scem 2043289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2044289774Scem CTLFLAG_RD, NULL, "Xeon HW statistics"); 2045289774Scem statpar = SYSCTL_CHILDREN(tmptree); 2046289774Scem SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2047289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2048289774Scem NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2049289774Scem sysctl_handle_register, "SU", "Upstream Memory Miss"); 2050289774Scem 2051289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2052289774Scem CTLFLAG_RD, NULL, "Xeon HW errors"); 2053289774Scem errpar = SYSCTL_CHILDREN(tmptree); 2054289774Scem 2055289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "devsts", 2056289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2057289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2058289774Scem sysctl_handle_register, "SU", "DEVSTS"); 2059289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "lnksts", 2060289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2061289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2062289774Scem sysctl_handle_register, "SU", "LNKSTS"); 2063289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2064289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2065289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2066289774Scem sysctl_handle_register, "IU", "UNCERRSTS"); 2067289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2068289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2069289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2070289774Scem sysctl_handle_register, "IU", "CORERRSTS"); 2071289774Scem 2072289774Scem if (ntb->conn_type != NTB_CONN_B2B) 2073289774Scem return; 2074289774Scem 2075289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2076289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2077289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2078289774Scem sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2079289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2080289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2081289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2082289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2083289774Scem sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2084289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2085289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2086289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2087289774Scem sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2088289774Scem } else { 2089289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2090289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2091289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2092289774Scem sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2093289774Scem } 2094289774Scem 2095289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2096289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2097289774Scem NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2098289774Scem sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2099289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2100289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2101289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2102289774Scem NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2103289774Scem sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2104289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2105289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2106289774Scem NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2107289774Scem sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2108289774Scem } else { 2109289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2110289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2111289774Scem NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2112289774Scem sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2113289774Scem } 2114289774Scem 2115289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2116289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2117289774Scem NTB_REG_64 | ntb->xlat_reg->bar0_base, 2118289774Scem sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2119289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2120289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2121289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_base, 2122289774Scem sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2123289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2124289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2125289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2126289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_base, 2127289774Scem sysctl_handle_register, "IU", 2128289774Scem "Secondary BAR4 base register"); 2129289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2130289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2131289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_base, 2132289774Scem sysctl_handle_register, "IU", 2133289774Scem "Secondary BAR5 base register"); 2134289774Scem } else { 2135289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2136289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2137289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_base, 2138289774Scem sysctl_handle_register, "QU", 2139289774Scem "Secondary BAR45 base register"); 2140289774Scem } 2141289774Scem} 2142289774Scem 2143289774Scemstatic int 2144289774Scemsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2145289774Scem{ 2146289774Scem struct ntb_softc *ntb; 2147289774Scem struct sbuf sb; 2148289774Scem int error; 2149289774Scem 2150289774Scem error = 0; 2151289774Scem ntb = arg1; 2152289774Scem 2153289774Scem sbuf_new_for_sysctl(&sb, NULL, 256, req); 2154289774Scem 2155289774Scem sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2156289774Scem error = sbuf_finish(&sb); 2157289774Scem sbuf_delete(&sb); 2158289774Scem 2159289774Scem if (error || !req->newptr) 2160289774Scem return (error); 2161289774Scem return (EINVAL); 2162289774Scem} 2163289774Scem 2164289774Scemstatic int 2165289774Scemsysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2166289774Scem{ 2167289774Scem struct ntb_softc *ntb; 2168289774Scem struct sbuf sb; 2169289774Scem enum ntb_speed speed; 2170289774Scem enum ntb_width width; 2171289774Scem int error; 2172289774Scem 2173289774Scem error = 0; 2174289774Scem ntb = arg1; 2175289774Scem 2176289774Scem sbuf_new_for_sysctl(&sb, NULL, 32, req); 2177289774Scem 2178289774Scem if (ntb_link_is_up(ntb, &speed, &width)) 2179289774Scem sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2180289774Scem (unsigned)speed, (unsigned)width); 2181289774Scem else 2182289774Scem sbuf_printf(&sb, "down"); 2183289774Scem 2184289774Scem error = sbuf_finish(&sb); 2185289774Scem sbuf_delete(&sb); 2186289774Scem 2187289774Scem if (error || !req->newptr) 2188289774Scem return (error); 2189289774Scem return (EINVAL); 2190289774Scem} 2191289774Scem 2192289774Scemstatic int 2193289774Scemsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2194289774Scem{ 2195289774Scem struct ntb_softc *ntb; 2196289774Scem const void *outp; 2197289774Scem uintptr_t sz; 2198289774Scem uint64_t umv; 2199289774Scem char be[sizeof(umv)]; 2200289774Scem size_t outsz; 2201289774Scem uint32_t reg; 2202289774Scem bool db, pci; 2203289774Scem int error; 2204289774Scem 2205289774Scem ntb = arg1; 2206289774Scem reg = arg2 & ~NTB_REGFLAGS_MASK; 2207289774Scem sz = arg2 & NTB_REGSZ_MASK; 2208289774Scem db = (arg2 & NTB_DB_READ) != 0; 2209289774Scem pci = (arg2 & NTB_PCI_REG) != 0; 2210289774Scem 2211289774Scem KASSERT(!(db && pci), ("bogus")); 2212289774Scem 2213289774Scem if (db) { 2214289774Scem KASSERT(sz == NTB_REG_64, ("bogus")); 2215289774Scem umv = db_ioread(ntb, reg); 2216289774Scem outsz = sizeof(uint64_t); 2217289774Scem } else { 2218289774Scem switch (sz) { 2219289774Scem case NTB_REG_64: 2220289774Scem if (pci) 2221289774Scem umv = pci_read_config(ntb->device, reg, 8); 2222289774Scem else 2223289774Scem umv = ntb_reg_read(8, reg); 2224289774Scem outsz = sizeof(uint64_t); 2225289774Scem break; 2226289774Scem case NTB_REG_32: 2227289774Scem if (pci) 2228289774Scem umv = pci_read_config(ntb->device, reg, 4); 2229289774Scem else 2230289774Scem umv = ntb_reg_read(4, reg); 2231289774Scem outsz = sizeof(uint32_t); 2232289774Scem break; 2233289774Scem case NTB_REG_16: 2234289774Scem if (pci) 2235289774Scem umv = pci_read_config(ntb->device, reg, 2); 2236289774Scem else 2237289774Scem umv = ntb_reg_read(2, reg); 2238289774Scem outsz = sizeof(uint16_t); 2239289774Scem break; 2240289774Scem case NTB_REG_8: 2241289774Scem if (pci) 2242289774Scem umv = pci_read_config(ntb->device, reg, 1); 2243289774Scem else 2244289774Scem umv = ntb_reg_read(1, reg); 2245289774Scem outsz = sizeof(uint8_t); 2246289774Scem break; 2247289774Scem default: 2248289774Scem panic("bogus"); 2249289774Scem break; 2250289774Scem } 2251289774Scem } 2252289774Scem 2253289774Scem /* Encode bigendian so that sysctl -x is legible. */ 2254289774Scem be64enc(be, umv); 2255289774Scem outp = ((char *)be) + sizeof(umv) - outsz; 2256289774Scem 2257289774Scem error = SYSCTL_OUT(req, outp, outsz); 2258289774Scem if (error || !req->newptr) 2259289774Scem return (error); 2260289774Scem return (EINVAL); 2261289774Scem} 2262289774Scem 2263289546Scem/* 2264289546Scem * Public API to the rest of the OS 2265250079Scarl */ 2266250079Scarl 2267250079Scarl/** 2268250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 2269250079Scarl * @ntb: pointer to ntb_softc instance 2270250079Scarl * 2271250079Scarl * This function returns the max 32bit scratchpad registers usable by the 2272250079Scarl * upper layer. 2273250079Scarl * 2274250079Scarl * RETURNS: total number of scratch pad registers available 2275250079Scarl */ 2276289208Scemuint8_t 2277250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 2278250079Scarl{ 2279250079Scarl 2280289539Scem return (ntb->spad_count); 2281250079Scarl} 2282250079Scarl 2283289396Scemuint8_t 2284289539Scemntb_mw_count(struct ntb_softc *ntb) 2285289396Scem{ 2286289396Scem 2287289539Scem return (ntb->mw_count); 2288289396Scem} 2289289396Scem 2290250079Scarl/** 2291289545Scem * ntb_spad_write() - write to the secondary scratchpad register 2292250079Scarl * @ntb: pointer to ntb_softc instance 2293250079Scarl * @idx: index to the scratchpad register, 0 based 2294250079Scarl * @val: the data value to put into the register 2295250079Scarl * 2296250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2297250079Scarl * register. The register resides on the secondary (external) side. 2298250079Scarl * 2299289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2300250079Scarl */ 2301250079Scarlint 2302289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2303250079Scarl{ 2304250079Scarl 2305289539Scem if (idx >= ntb->spad_count) 2306250079Scarl return (EINVAL); 2307250079Scarl 2308289607Scem ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2309250079Scarl 2310250079Scarl return (0); 2311250079Scarl} 2312250079Scarl 2313250079Scarl/** 2314289545Scem * ntb_spad_read() - read from the primary scratchpad register 2315250079Scarl * @ntb: pointer to ntb_softc instance 2316250079Scarl * @idx: index to scratchpad register, 0 based 2317250079Scarl * @val: pointer to 32bit integer for storing the register value 2318250079Scarl * 2319250079Scarl * This function allows reading of the 32bit scratchpad register on 2320250079Scarl * the primary (internal) side. 2321250079Scarl * 2322289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2323250079Scarl */ 2324250079Scarlint 2325289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2326250079Scarl{ 2327250079Scarl 2328289539Scem if (idx >= ntb->spad_count) 2329250079Scarl return (EINVAL); 2330250079Scarl 2331289607Scem *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2332250079Scarl 2333250079Scarl return (0); 2334250079Scarl} 2335250079Scarl 2336250079Scarl/** 2337289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register 2338250079Scarl * @ntb: pointer to ntb_softc instance 2339250079Scarl * @idx: index to the scratchpad register, 0 based 2340250079Scarl * @val: the data value to put into the register 2341250079Scarl * 2342250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2343250079Scarl * register. The register resides on the secondary (external) side. 2344250079Scarl * 2345289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2346250079Scarl */ 2347250079Scarlint 2348289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2349250079Scarl{ 2350250079Scarl 2351289539Scem if (idx >= ntb->spad_count) 2352250079Scarl return (EINVAL); 2353250079Scarl 2354289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2355255279Scarl ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 2356255279Scarl else 2357289542Scem ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2358250079Scarl 2359250079Scarl return (0); 2360250079Scarl} 2361250079Scarl 2362250079Scarl/** 2363289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register 2364250079Scarl * @ntb: pointer to ntb_softc instance 2365250079Scarl * @idx: index to scratchpad register, 0 based 2366250079Scarl * @val: pointer to 32bit integer for storing the register value 2367250079Scarl * 2368250079Scarl * This function allows reading of the 32bit scratchpad register on 2369250079Scarl * the primary (internal) side. 2370250079Scarl * 2371289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2372250079Scarl */ 2373250079Scarlint 2374289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2375250079Scarl{ 2376250079Scarl 2377289539Scem if (idx >= ntb->spad_count) 2378250079Scarl return (EINVAL); 2379250079Scarl 2380289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2381255279Scarl *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 2382255279Scarl else 2383289542Scem *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2384250079Scarl 2385250079Scarl return (0); 2386250079Scarl} 2387250079Scarl 2388289546Scem/* 2389289546Scem * ntb_mw_get_range() - get the range of a memory window 2390289546Scem * @ntb: NTB device context 2391289546Scem * @idx: Memory window number 2392289546Scem * @base: OUT - the base address for mapping the memory window 2393289546Scem * @size: OUT - the size for mapping the memory window 2394289546Scem * @align: OUT - the base alignment for translating the memory window 2395289546Scem * @align_size: OUT - the size alignment for translating the memory window 2396250079Scarl * 2397289546Scem * Get the range of a memory window. NULL may be given for any output 2398289546Scem * parameter if the value is not needed. The base and size may be used for 2399289546Scem * mapping the memory window, to access the peer memory. The alignment and 2400289546Scem * size may be used for translating the memory window, for the peer to access 2401289546Scem * memory on the local system. 2402250079Scarl * 2403289546Scem * Return: Zero on success, otherwise an error number. 2404250079Scarl */ 2405289546Scemint 2406289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 2407290679Scem caddr_t *vbase, size_t *size, size_t *align, size_t *align_size) 2408250079Scarl{ 2409289546Scem struct ntb_pci_bar_info *bar; 2410289546Scem size_t bar_b2b_off; 2411250079Scarl 2412289546Scem if (mw_idx >= ntb_mw_count(ntb)) 2413289546Scem return (EINVAL); 2414250079Scarl 2415289546Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)]; 2416289546Scem bar_b2b_off = 0; 2417289546Scem if (mw_idx == ntb->b2b_mw_idx) { 2418289546Scem KASSERT(ntb->b2b_off != 0, 2419289546Scem ("user shouldn't get non-shared b2b mw")); 2420289546Scem bar_b2b_off = ntb->b2b_off; 2421289546Scem } 2422250079Scarl 2423289546Scem if (base != NULL) 2424289546Scem *base = bar->pbase + bar_b2b_off; 2425289546Scem if (vbase != NULL) 2426290679Scem *vbase = bar->vbase + bar_b2b_off; 2427289546Scem if (size != NULL) 2428289546Scem *size = bar->size - bar_b2b_off; 2429289546Scem if (align != NULL) 2430289546Scem *align = bar->size; 2431289546Scem if (align_size != NULL) 2432289546Scem *align_size = 1; 2433289546Scem return (0); 2434250079Scarl} 2435250079Scarl 2436289546Scem/* 2437289546Scem * ntb_mw_set_trans() - set the translation of a memory window 2438289546Scem * @ntb: NTB device context 2439289546Scem * @idx: Memory window number 2440289546Scem * @addr: The dma address local memory to expose to the peer 2441289546Scem * @size: The size of the local memory to expose to the peer 2442250079Scarl * 2443289546Scem * Set the translation of a memory window. The peer may access local memory 2444289546Scem * through the window starting at the address, up to the size. The address 2445289546Scem * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2446289546Scem * must be aligned to the size alignment specified by ntb_mw_get_range(). 2447250079Scarl * 2448289546Scem * Return: Zero on success, otherwise an error number. 2449250079Scarl */ 2450289546Scemint 2451289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2452289546Scem size_t size) 2453250079Scarl{ 2454289546Scem struct ntb_pci_bar_info *bar; 2455289546Scem uint64_t base, limit, reg_val; 2456289546Scem size_t bar_size, mw_size; 2457289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2458289546Scem enum ntb_bar bar_num; 2459250079Scarl 2460289546Scem if (idx >= ntb_mw_count(ntb)) 2461289546Scem return (EINVAL); 2462250079Scarl 2463289546Scem bar_num = ntb_mw_to_bar(ntb, idx); 2464289546Scem bar = &ntb->bar_info[bar_num]; 2465250079Scarl 2466289546Scem bar_size = bar->size; 2467289546Scem if (idx == ntb->b2b_mw_idx) 2468289546Scem mw_size = bar_size - ntb->b2b_off; 2469289546Scem else 2470289546Scem mw_size = bar_size; 2471250079Scarl 2472289546Scem /* Hardware requires that addr is aligned to bar size */ 2473289546Scem if ((addr & (bar_size - 1)) != 0) 2474289546Scem return (EINVAL); 2475250079Scarl 2476289546Scem if (size > mw_size) 2477289546Scem return (EINVAL); 2478289546Scem 2479289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2480289546Scem 2481289546Scem limit = 0; 2482289546Scem if (bar_is_64bit(ntb, bar_num)) { 2483289546Scem base = ntb_reg_read(8, base_reg); 2484289546Scem 2485289546Scem if (limit_reg != 0 && size != mw_size) 2486289546Scem limit = base + size; 2487289546Scem 2488289546Scem /* Set and verify translation address */ 2489289546Scem ntb_reg_write(8, xlat_reg, addr); 2490289546Scem reg_val = ntb_reg_read(8, xlat_reg); 2491289546Scem if (reg_val != addr) { 2492289546Scem ntb_reg_write(8, xlat_reg, 0); 2493289546Scem return (EIO); 2494289546Scem } 2495289546Scem 2496289546Scem /* Set and verify the limit */ 2497289546Scem ntb_reg_write(8, limit_reg, limit); 2498289546Scem reg_val = ntb_reg_read(8, limit_reg); 2499289546Scem if (reg_val != limit) { 2500289546Scem ntb_reg_write(8, limit_reg, base); 2501289546Scem ntb_reg_write(8, xlat_reg, 0); 2502289546Scem return (EIO); 2503289546Scem } 2504289546Scem } else { 2505289546Scem /* Configure 32-bit (split) BAR MW */ 2506289546Scem 2507289546Scem if ((addr & ~UINT32_MAX) != 0) 2508289546Scem return (EINVAL); 2509289546Scem if (((addr + size) & ~UINT32_MAX) != 0) 2510289546Scem return (EINVAL); 2511289546Scem 2512289546Scem base = ntb_reg_read(4, base_reg); 2513289546Scem 2514289546Scem if (limit_reg != 0 && size != mw_size) 2515289546Scem limit = base + size; 2516289546Scem 2517289546Scem /* Set and verify translation address */ 2518289546Scem ntb_reg_write(4, xlat_reg, addr); 2519289546Scem reg_val = ntb_reg_read(4, xlat_reg); 2520289546Scem if (reg_val != addr) { 2521289546Scem ntb_reg_write(4, xlat_reg, 0); 2522289546Scem return (EIO); 2523289546Scem } 2524289546Scem 2525289546Scem /* Set and verify the limit */ 2526289546Scem ntb_reg_write(4, limit_reg, limit); 2527289546Scem reg_val = ntb_reg_read(4, limit_reg); 2528289546Scem if (reg_val != limit) { 2529289546Scem ntb_reg_write(4, limit_reg, base); 2530289546Scem ntb_reg_write(4, xlat_reg, 0); 2531289546Scem return (EIO); 2532289546Scem } 2533250079Scarl } 2534289546Scem return (0); 2535250079Scarl} 2536250079Scarl 2537289596Scem/* 2538289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window 2539289596Scem * @ntb: NTB device context 2540289596Scem * @idx: Memory window number 2541289596Scem * 2542289596Scem * Clear the translation of a memory window. The peer may no longer access 2543289596Scem * local memory through the window. 2544289596Scem * 2545289596Scem * Return: Zero on success, otherwise an error number. 2546289596Scem */ 2547289596Scemint 2548289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 2549289596Scem{ 2550289596Scem 2551289596Scem return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 2552289596Scem} 2553289596Scem 2554250079Scarl/** 2555289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side 2556250079Scarl * @ntb: pointer to ntb_softc instance 2557289545Scem * @bit: doorbell bits to ring 2558250079Scarl * 2559250079Scarl * This function allows triggering of a doorbell on the secondary/external 2560250079Scarl * side that will initiate an interrupt on the remote host 2561250079Scarl */ 2562250079Scarlvoid 2563289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 2564250079Scarl{ 2565250079Scarl 2566289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2567289347Scem ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit); 2568289347Scem return; 2569289209Scem } 2570289347Scem 2571289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2572250079Scarl} 2573250079Scarl 2574289542Scem/* 2575289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 2576289542Scem * as well as the size of the register (via *sz_out). 2577289542Scem * 2578289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell 2579289542Scem * ring to its memory window write. 2580289542Scem * 2581289542Scem * Note that writing the peer doorbell via a memory window will *not* generate 2582289542Scem * an interrupt on the remote host; that must be done seperately. 2583289542Scem */ 2584289542Scembus_addr_t 2585289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 2586289542Scem{ 2587289542Scem struct ntb_pci_bar_info *bar; 2588289542Scem uint64_t regoff; 2589289542Scem 2590289542Scem KASSERT(sz_out != NULL, ("must be non-NULL")); 2591289542Scem 2592289542Scem if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2593289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 2594289542Scem regoff = ntb->peer_reg->db_bell; 2595289542Scem } else { 2596289542Scem KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) || 2597289542Scem (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1), 2598289542Scem ("mw_count invalid after setup")); 2599289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 2600289543Scem ("invalid b2b idx")); 2601289542Scem 2602289542Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 2603289542Scem regoff = XEON_SHADOW_PDOORBELL_OFFSET; 2604289542Scem } 2605289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 2606289542Scem 2607289542Scem *sz_out = ntb->reg->db_size; 2608289542Scem /* HACK: Specific to current x86 bus implementation. */ 2609289542Scem return ((uint64_t)bar->pci_bus_handle + regoff); 2610289542Scem} 2611289542Scem 2612289597Scem/* 2613289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 2614289597Scem * @ntb: NTB device context 2615289597Scem * 2616289597Scem * Hardware may support different number or arrangement of doorbell bits. 2617289597Scem * 2618289597Scem * Return: A mask of doorbell bits supported by the ntb. 2619289597Scem */ 2620289597Scemuint64_t 2621289597Scemntb_db_valid_mask(struct ntb_softc *ntb) 2622289597Scem{ 2623289597Scem 2624289597Scem return (ntb->db_valid_mask); 2625289597Scem} 2626289597Scem 2627289598Scem/* 2628289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 2629289598Scem * @ntb: NTB device context 2630289598Scem * @vector: Doorbell vector number 2631289598Scem * 2632289598Scem * Each interrupt vector may have a different number or arrangement of bits. 2633289598Scem * 2634289598Scem * Return: A mask of doorbell bits serviced by a vector. 2635289598Scem */ 2636289598Scemuint64_t 2637289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 2638289598Scem{ 2639289598Scem 2640289598Scem if (vector > ntb->db_vec_count) 2641289598Scem return (0); 2642289598Scem return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 2643289598Scem} 2644289598Scem 2645250079Scarl/** 2646289546Scem * ntb_link_is_up() - get the current ntb link state 2647289546Scem * @ntb: NTB device context 2648289546Scem * @speed: OUT - The link speed expressed as PCIe generation number 2649289546Scem * @width: OUT - The link width expressed as the number of PCIe lanes 2650250079Scarl * 2651250079Scarl * RETURNS: true or false based on the hardware link state 2652250079Scarl */ 2653250079Scarlbool 2654289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 2655289546Scem enum ntb_width *width) 2656250079Scarl{ 2657250079Scarl 2658289546Scem if (speed != NULL) 2659289546Scem *speed = ntb_link_sta_speed(ntb); 2660289546Scem if (width != NULL) 2661289546Scem *width = ntb_link_sta_width(ntb); 2662289546Scem return (link_is_up(ntb)); 2663250079Scarl} 2664250079Scarl 2665255272Scarlstatic void 2666255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 2667250079Scarl{ 2668255272Scarl 2669289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2670289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2671289209Scem bar->pbase = rman_get_start(bar->pci_resource); 2672289209Scem bar->size = rman_get_size(bar->pci_resource); 2673289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 2674250079Scarl} 2675255268Scarl 2676289209Scemdevice_t 2677289209Scemntb_get_device(struct ntb_softc *ntb) 2678255268Scarl{ 2679255268Scarl 2680255268Scarl return (ntb->device); 2681255268Scarl} 2682289208Scem 2683289208Scem/* Export HW-specific errata information. */ 2684289208Scembool 2685289774Scemntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 2686289208Scem{ 2687289208Scem 2688289208Scem return (HAS_FEATURE(feature)); 2689289208Scem} 2690