ntb_hw_intel.c revision 289610
1185029Spjd/*- 2185029Spjd * Copyright (C) 2013 Intel Corporation 3185029Spjd * Copyright (C) 2015 EMC Corporation 4185029Spjd * All rights reserved. 5185029Spjd * 6185029Spjd * Redistribution and use in source and binary forms, with or without 7185029Spjd * modification, are permitted provided that the following conditions 8185029Spjd * are met: 9185029Spjd * 1. Redistributions of source code must retain the above copyright 10185029Spjd * notice, this list of conditions and the following disclaimer. 11185029Spjd * 2. Redistributions in binary form must reproduce the above copyright 12185029Spjd * notice, this list of conditions and the following disclaimer in the 13185029Spjd * documentation and/or other materials provided with the distribution. 14185029Spjd * 15185029Spjd * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16185029Spjd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17185029Spjd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18185029Spjd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19329175Skevans * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20329175Skevans * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21185029Spjd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22185029Spjd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23185029Spjd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24185096Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25185096Sdfr * SUCH DAMAGE. 26185096Sdfr */ 27185029Spjd 28185029Spjd#include <sys/cdefs.h> 29185029Spjd__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289610 2015-10-20 01:46:05Z cem $"); 30185029Spjd 31185029Spjd#include <sys/param.h> 32200309Sjhb#include <sys/kernel.h> 33185029Spjd#include <sys/systm.h> 34185029Spjd#include <sys/bus.h> 35185029Spjd#include <sys/malloc.h> 36185029Spjd#include <sys/module.h> 37185029Spjd#include <sys/queue.h> 38185029Spjd#include <sys/rman.h> 39185029Spjd#include <sys/sysctl.h> 40185029Spjd#include <vm/vm.h> 41185029Spjd#include <vm/pmap.h> 42213136Spjd#include <machine/bus.h> 43213136Spjd#include <machine/pmap.h> 44329100Skevans#include <machine/resource.h> 45213136Spjd#include <dev/pci/pcireg.h> 46235154Savg#include <dev/pci/pcivar.h> 47294765Simp 48185029Spjd#include "ntb_regs.h" 49235329Savg#include "ntb_hw.h" 50235329Savg 51297629Sallanjude/* 52297629Sallanjude * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 53297629Sallanjude * allows you to connect two systems using a PCI-e link. 54185029Spjd * 55297629Sallanjude * This module contains the hardware abstraction layer for the NTB. It allows 56297629Sallanjude * you to send and recieve interrupts, map the memory windows and send and 57297629Sallanjude * receive messages in the scratch-pad registers. 58185029Spjd * 59297629Sallanjude * NOTE: Much of the code in this module is shared with Linux. Any patches may 60297629Sallanjude * be picked up and redistributed in Linux with a dual GPL/BSD license. 61297629Sallanjude */ 62297629Sallanjude 63185029Spjd#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT) 64297629Sallanjude 65297629Sallanjude#define NTB_HB_TIMEOUT 1 /* second */ 66185029Spjd#define SOC_LINK_RECOVERY_TIME 500 /* ms */ 67185029Spjd 68185096Sdfr#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 69185096Sdfr 70185096Sdfrenum ntb_device_type { 71185029Spjd NTB_XEON, 72185029Spjd NTB_SOC 73185029Spjd}; 74185029Spjd 75185029Spjd/* ntb_conn_type are hardware numbers, cannot change. */ 76185029Spjdenum ntb_conn_type { 77185029Spjd NTB_CONN_TRANSPARENT = 0, 78185029Spjd NTB_CONN_B2B = 1, 79185029Spjd NTB_CONN_RP = 2, 80185029Spjd}; 81185029Spjd 82185029Spjdenum ntb_b2b_direction { 83185029Spjd NTB_DEV_USD = 0, 84185029Spjd NTB_DEV_DSD = 1, 85185029Spjd}; 86185029Spjd 87185029Spjdenum ntb_bar { 88213136Spjd NTB_CONFIG_BAR = 0, 89185029Spjd NTB_B2B_BAR_1, 90348866Skevans NTB_B2B_BAR_2, 91348866Skevans NTB_B2B_BAR_3, 92348866Skevans NTB_MAX_BARS 93348866Skevans}; 94348866Skevans 95348866Skevans/* Device features and workarounds */ 96348866Skevans#define HAS_FEATURE(feature) \ 97348866Skevans ((ntb->features & (feature)) != 0) 98348866Skevans 99348866Skevansstruct ntb_hw_info { 100348866Skevans uint32_t device_id; 101348866Skevans const char *desc; 102348866Skevans enum ntb_device_type type; 103348866Skevans uint32_t features; 104348866Skevans}; 105348866Skevans 106348866Skevansstruct ntb_pci_bar_info { 107348866Skevans bus_space_tag_t pci_bus_tag; 108185029Spjd bus_space_handle_t pci_bus_handle; 109185029Spjd int pci_resource_id; 110185029Spjd struct resource *pci_resource; 111234339Savg vm_paddr_t pbase; 112185029Spjd void *vbase; 113235329Savg u_long size; 114185029Spjd 115185029Spjd /* Configuration register offsets */ 116185029Spjd uint32_t psz_off; 117235329Savg uint32_t ssz_off; 118185029Spjd uint32_t pbarxlat_off; 119200309Sjhb}; 120200309Sjhb 121200309Sjhbstruct ntb_int_info { 122200309Sjhb struct resource *res; 123200309Sjhb int rid; 124200309Sjhb void *tag; 125200309Sjhb}; 126200309Sjhb 127328866Skevansstruct ntb_vec { 128200309Sjhb struct ntb_softc *ntb; 129200309Sjhb uint32_t num; 130200309Sjhb}; 131200309Sjhb 132185029Spjdstruct ntb_reg { 133297629Sallanjude uint32_t ntb_ctl; 134185029Spjd uint32_t lnk_sta; 135185029Spjd uint8_t db_size; 136185029Spjd unsigned mw_bar[NTB_MAX_BARS]; 137185029Spjd}; 138185029Spjd 139185029Spjdstruct ntb_alt_reg { 140185029Spjd uint32_t db_bell; 141308914Savg uint32_t db_mask; 142185029Spjd uint32_t spad; 143329099Skevans}; 144200309Sjhb 145329100Skevansstruct ntb_xlat_reg { 146185029Spjd uint32_t bar0_base; 147296963Sallanjude uint32_t bar2_base; 148344399Skevans uint32_t bar4_base; 149296963Sallanjude uint32_t bar5_base; 150296963Sallanjude 151296963Sallanjude uint32_t bar2_xlat; 152344399Skevans uint32_t bar4_xlat; 153344399Skevans uint32_t bar5_xlat; 154344399Skevans 155344399Skevans uint32_t bar2_limit; 156344399Skevans uint32_t bar4_limit; 157344399Skevans uint32_t bar5_limit; 158344399Skevans}; 159185029Spjd 160185029Spjdstruct ntb_b2b_addr { 161185029Spjd uint64_t bar0_addr; 162185029Spjd uint64_t bar2_addr64; 163185029Spjd uint64_t bar4_addr64; 164185029Spjd uint64_t bar4_addr32; 165185029Spjd uint64_t bar5_addr32; 166185029Spjd}; 167185029Spjd 168185029Spjdstruct ntb_softc { 169185029Spjd device_t device; 170185029Spjd enum ntb_device_type type; 171185029Spjd uint64_t features; 172185029Spjd 173185029Spjd struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 174308914Savg struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 175185029Spjd uint32_t allocated_interrupts; 176185029Spjd 177185029Spjd struct callout heartbeat_timer; 178185029Spjd struct callout lr_timer; 179185029Spjd 180185029Spjd void *ntb_ctx; 181185029Spjd const struct ntb_ctx_ops *ctx_ops; 182185029Spjd struct ntb_vec *msix_vec; 183185029Spjd#define CTX_LOCK(sc) mtx_lock_spin(&(sc)->ctx_lock) 184185029Spjd#define CTX_UNLOCK(sc) mtx_unlock_spin(&(sc)->ctx_lock) 185185029Spjd#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 186235329Savg struct mtx ctx_lock; 187241293Savg 188241293Savg uint32_t ppd; 189185029Spjd enum ntb_conn_type conn_type; 190185029Spjd enum ntb_b2b_direction dev_type; 191185029Spjd 192185029Spjd /* Offset of peer bar0 in B2B BAR */ 193185029Spjd uint64_t b2b_off; 194185029Spjd /* Memory window used to access peer bar0 */ 195329175Skevans#define B2B_MW_DISABLED UINT8_MAX 196185029Spjd uint8_t b2b_mw_idx; 197185029Spjd 198297629Sallanjude uint8_t mw_count; 199300257Sallanjude uint8_t spad_count; 200297629Sallanjude uint8_t db_count; 201344399Skevans uint8_t db_vec_count; 202185029Spjd uint8_t db_vec_shift; 203185029Spjd 204185029Spjd /* Protects local db_mask. */ 205185029Spjd#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 206185029Spjd#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 207185029Spjd#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 208344399Skevans struct mtx db_mask_lock; 209297629Sallanjude 210300257Sallanjude uint32_t ntb_ctl; 211300257Sallanjude uint32_t lnk_sta; 212300257Sallanjude 213300257Sallanjude uint64_t db_valid_mask; 214300257Sallanjude uint64_t db_link_mask; 215297629Sallanjude uint64_t db_mask; 216297629Sallanjude 217297629Sallanjude int last_ts; /* ticks @ last irq */ 218344399Skevans 219297629Sallanjude const struct ntb_reg *reg; 220297629Sallanjude const struct ntb_alt_reg *self_reg; 221185029Spjd const struct ntb_alt_reg *peer_reg; 222185029Spjd const struct ntb_xlat_reg *xlat_reg; 223297629Sallanjude}; 224297629Sallanjude 225297629Sallanjude#ifdef __i386__ 226297629Sallanjudestatic __inline uint64_t 227298949Speterbus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 228298949Speter bus_size_t offset) 229297629Sallanjude{ 230297629Sallanjude 231297629Sallanjude return (bus_space_read_4(tag, handle, offset) | 232297629Sallanjude ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 233298949Speter} 234298949Speter 235297629Sallanjudestatic __inline void 236344399Skevansbus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 237344399Skevans bus_size_t offset, uint64_t val) 238344399Skevans{ 239344399Skevans 240344399Skevans bus_space_write_4(tag, handle, offset, val); 241344399Skevans bus_space_write_4(tag, handle, offset + 4, val >> 32); 242344295Skevans} 243344295Skevans#endif 244344399Skevans 245185029Spjd#define ntb_bar_read(SIZE, bar, offset) \ 246296963Sallanjude bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 247296963Sallanjude ntb->bar_info[(bar)].pci_bus_handle, (offset)) 248344399Skevans#define ntb_bar_write(SIZE, bar, offset, val) \ 249344399Skevans bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 250297629Sallanjude ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 251297629Sallanjude#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 252296963Sallanjude#define ntb_reg_write(SIZE, offset, val) \ 253296963Sallanjude ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 254297629Sallanjude#define ntb_mw_read(SIZE, offset) \ 255185029Spjd ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 256185029Spjd#define ntb_mw_write(SIZE, offset, val) \ 257297629Sallanjude ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 258185029Spjd offset, val) 259297629Sallanjude 260297629Sallanjudestatic int ntb_probe(device_t device); 261185029Spjdstatic int ntb_attach(device_t device); 262185029Spjdstatic int ntb_detach(device_t device); 263185029Spjdstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 264185029Spjdstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 265329175Skevansstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 266329175Skevans uint32_t *base, uint32_t *xlat, uint32_t *lmt); 267329175Skevansstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 268329175Skevansstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *); 269329175Skevansstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 270329175Skevansstatic int map_memory_window_bar(struct ntb_softc *ntb, 271185029Spjd struct ntb_pci_bar_info *bar); 272329175Skevansstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 273185029Spjdstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 274308914Savgstatic int ntb_init_isr(struct ntb_softc *ntb); 275308914Savgstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 276308914Savgstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 277308914Savgstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 278308914Savgstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 279344399Skevansstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 280308914Savgstatic void ndev_vec_isr(void *arg); 281308914Savgstatic void ndev_irq_isr(void *arg); 282308914Savgstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 283308914Savgstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t val); 284308914Savgstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 285308914Savgstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 286344399Skevansstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 287308914Savgstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 288308914Savgstatic int ntb_detect_xeon(struct ntb_softc *ntb); 289308914Savgstatic int ntb_detect_soc(struct ntb_softc *ntb); 290308914Savgstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 291308914Savgstatic int ntb_soc_init_dev(struct ntb_softc *ntb); 292344399Skevansstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 293308914Savgstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 294308914Savgstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 295308914Savg enum ntb_bar regbar); 296308914Savgstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 297308914Savg uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 298308914Savgstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 299308914Savg enum ntb_bar idx); 300308914Savgstatic int xeon_setup_b2b_mw(struct ntb_softc *, 301308914Savg const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 302308914Savgstatic inline bool link_is_up(struct ntb_softc *ntb); 303185029Spjdstatic inline bool soc_link_is_err(struct ntb_softc *ntb); 304185029Spjdstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 305185029Spjdstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 306235329Savgstatic void soc_link_hb(void *arg); 307185029Spjdstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 308185029Spjdstatic void recover_soc_link(void *arg); 309185029Spjdstatic bool ntb_poll_link(struct ntb_softc *ntb); 310185029Spjdstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 311185029Spjd 312308914Savgstatic struct ntb_hw_info pci_ids[] = { 313308914Savg { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 314308914Savg 315308914Savg /* XXX: PS/SS IDs left out until they are supported. */ 316308914Savg { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 317308914Savg NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 318308914Savg { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 319308914Savg NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 320308914Savg { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 321308914Savg NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 322308914Savg NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 323308914Savg { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 324308914Savg NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 325308914Savg NTB_SB01BASE_LOCKUP }, 326308914Savg { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 327308914Savg NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 328308914Savg NTB_SB01BASE_LOCKUP }, 329308914Savg 330308914Savg { 0x00000000, NULL, NTB_SOC, 0 } 331308914Savg}; 332308914Savg 333308914Savgstatic const struct ntb_reg soc_reg = { 334308914Savg .ntb_ctl = SOC_NTBCNTL_OFFSET, 335308914Savg .lnk_sta = SOC_LINK_STATUS_OFFSET, 336308914Savg .db_size = sizeof(uint64_t), 337308914Savg .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 338308914Savg}; 339308914Savg 340308914Savgstatic const struct ntb_alt_reg soc_pri_reg = { 341308914Savg .db_bell = SOC_PDOORBELL_OFFSET, 342308914Savg .db_mask = SOC_PDBMSK_OFFSET, 343308914Savg .spad = SOC_SPAD_OFFSET, 344308914Savg}; 345308914Savg 346308914Savgstatic const struct ntb_alt_reg soc_b2b_reg = { 347308914Savg .db_bell = SOC_B2B_DOORBELL_OFFSET, 348308914Savg .spad = SOC_B2B_SPAD_OFFSET, 349308914Savg}; 350308914Savg 351308914Savgstatic const struct ntb_xlat_reg soc_sec_xlat = { 352308914Savg#if 0 353308914Savg /* "FIXME" says the Linux driver. */ 354308914Savg .bar0_base = SOC_SBAR0BASE_OFFSET, 355308914Savg .bar2_base = SOC_SBAR2BASE_OFFSET, 356308914Savg .bar4_base = SOC_SBAR4BASE_OFFSET, 357308914Savg 358200309Sjhb .bar2_limit = SOC_SBAR2LMT_OFFSET, 359200309Sjhb .bar4_limit = SOC_SBAR4LMT_OFFSET, 360185029Spjd#endif 361200309Sjhb 362185029Spjd .bar2_xlat = SOC_SBAR2XLAT_OFFSET, 363200309Sjhb .bar4_xlat = SOC_SBAR4XLAT_OFFSET, 364200309Sjhb}; 365200309Sjhb 366200309Sjhbstatic const struct ntb_reg xeon_reg = { 367200309Sjhb .ntb_ctl = XEON_NTBCNTL_OFFSET, 368200309Sjhb .lnk_sta = XEON_LINK_STATUS_OFFSET, 369200309Sjhb .db_size = sizeof(uint16_t), 370200309Sjhb .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 371200309Sjhb}; 372200309Sjhb 373200309Sjhbstatic const struct ntb_alt_reg xeon_pri_reg = { 374292682Sjhb .db_bell = XEON_PDOORBELL_OFFSET, 375200309Sjhb .db_mask = XEON_PDBMSK_OFFSET, 376200309Sjhb .spad = XEON_SPAD_OFFSET, 377200309Sjhb}; 378200309Sjhb 379200309Sjhbstatic const struct ntb_alt_reg xeon_b2b_reg = { 380200309Sjhb .db_bell = XEON_B2B_DOORBELL_OFFSET, 381200309Sjhb .spad = XEON_B2B_SPAD_OFFSET, 382200309Sjhb}; 383200309Sjhb 384200309Sjhbstatic const struct ntb_xlat_reg xeon_sec_xlat = { 385200309Sjhb .bar0_base = XEON_SBAR0BASE_OFFSET, 386200309Sjhb .bar2_base = XEON_SBAR2BASE_OFFSET, 387200309Sjhb .bar4_base = XEON_SBAR4BASE_OFFSET, 388200309Sjhb .bar5_base = XEON_SBAR5BASE_OFFSET, 389200309Sjhb 390200309Sjhb .bar2_limit = XEON_SBAR2LMT_OFFSET, 391200309Sjhb .bar4_limit = XEON_SBAR4LMT_OFFSET, 392200309Sjhb .bar5_limit = XEON_SBAR5LMT_OFFSET, 393200309Sjhb 394200309Sjhb .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 395200309Sjhb .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 396200309Sjhb .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 397200309Sjhb}; 398200309Sjhb 399200309Sjhbstatic const struct ntb_b2b_addr xeon_b2b_usd_addr = { 400200309Sjhb .bar0_addr = XEON_B2B_BAR0_USD_ADDR, 401200309Sjhb .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, 402200309Sjhb .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, 403200309Sjhb .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, 404200309Sjhb .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, 405200309Sjhb}; 406200309Sjhb 407200309Sjhbstatic const struct ntb_b2b_addr xeon_b2b_dsd_addr = { 408200309Sjhb .bar0_addr = XEON_B2B_BAR0_DSD_ADDR, 409200309Sjhb .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, 410200309Sjhb .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, 411200309Sjhb .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, 412200309Sjhb .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, 413200309Sjhb}; 414200309Sjhb 415200309Sjhb/* 416200309Sjhb * OS <-> Driver interface structures 417200309Sjhb */ 418200309SjhbMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 419200309Sjhb 420200309Sjhbstatic device_method_t ntb_pci_methods[] = { 421292682Sjhb /* Device interface */ 422200309Sjhb DEVMETHOD(device_probe, ntb_probe), 423200309Sjhb DEVMETHOD(device_attach, ntb_attach), 424200309Sjhb DEVMETHOD(device_detach, ntb_detach), 425200309Sjhb DEVMETHOD_END 426200309Sjhb}; 427200309Sjhb 428200309Sjhbstatic driver_t ntb_pci_driver = { 429200309Sjhb "ntb_hw", 430200309Sjhb ntb_pci_methods, 431200309Sjhb sizeof(struct ntb_softc), 432200309Sjhb}; 433200309Sjhb 434200309Sjhbstatic devclass_t ntb_devclass; 435200309SjhbDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 436200309SjhbMODULE_VERSION(ntb_hw, 1); 437200309Sjhb 438200309SjhbSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 439200309Sjhb 440200309Sjhb/* 441200309Sjhb * OS <-> Driver linkage functions 442296963Sallanjude */ 443200309Sjhbstatic int 444185029Spjdntb_probe(device_t device) 445185029Spjd{ 446185029Spjd struct ntb_hw_info *p; 447185029Spjd 448185029Spjd p = ntb_get_device_info(pci_get_devid(device)); 449185029Spjd if (p == NULL) 450185029Spjd return (ENXIO); 451185029Spjd 452185029Spjd device_set_desc(device, p->desc); 453185029Spjd return (0); 454185029Spjd} 455185029Spjd 456292682Sjhbstatic int 457185029Spjdntb_attach(device_t device) 458185029Spjd{ 459185029Spjd struct ntb_softc *ntb; 460185029Spjd struct ntb_hw_info *p; 461185029Spjd int error; 462185029Spjd 463185029Spjd ntb = DEVICE2SOFTC(device); 464185029Spjd p = ntb_get_device_info(pci_get_devid(device)); 465185029Spjd 466192194Sdfr ntb->device = device; 467192194Sdfr ntb->type = p->type; 468192194Sdfr ntb->features = p->features; 469192194Sdfr ntb->b2b_mw_idx = B2B_MW_DISABLED; 470344399Skevans 471344399Skevans /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 472192194Sdfr callout_init(&ntb->heartbeat_timer, 1); 473344399Skevans callout_init(&ntb->lr_timer, 1); 474192194Sdfr mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 475344399Skevans mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN); 476344399Skevans 477192194Sdfr if (ntb->type == NTB_SOC) 478192194Sdfr error = ntb_detect_soc(ntb); 479192194Sdfr else 480329100Skevans error = ntb_detect_xeon(ntb); 481329100Skevans if (error) 482329100Skevans goto out; 483329100Skevans 484329100Skevans ntb_detect_max_mw(ntb); 485329100Skevans 486329100Skevans error = ntb_map_pci_bars(ntb); 487329100Skevans if (error) 488344399Skevans goto out; 489329100Skevans if (ntb->type == NTB_SOC) 490344399Skevans error = ntb_soc_init_dev(ntb); 491329100Skevans else 492329100Skevans error = ntb_xeon_init_dev(ntb); 493329100Skevans if (error) 494344399Skevans goto out; 495344399Skevans error = ntb_init_isr(ntb); 496329100Skevans if (error) 497329100Skevans goto out; 498329100Skevans 499329100Skevans pci_enable_busmaster(ntb->device); 500329100Skevans 501329100Skevansout: 502329100Skevans if (error != 0) 503329100Skevans ntb_detach(device); 504329100Skevans return (error); 505329100Skevans} 506329100Skevans 507329100Skevansstatic int 508329100Skevansntb_detach(device_t device) 509329100Skevans{ 510329100Skevans struct ntb_softc *ntb; 511329100Skevans 512329100Skevans ntb = DEVICE2SOFTC(device); 513329100Skevans 514329100Skevans ntb_db_set_mask(ntb, ntb->db_valid_mask); 515329100Skevans callout_drain(&ntb->heartbeat_timer); 516329100Skevans callout_drain(&ntb->lr_timer); 517329100Skevans if (ntb->type == NTB_XEON) 518329100Skevans ntb_teardown_xeon(ntb); 519329100Skevans ntb_teardown_interrupts(ntb); 520329100Skevans 521329100Skevans mtx_destroy(&ntb->db_mask_lock); 522329100Skevans mtx_destroy(&ntb->ctx_lock); 523329100Skevans 524329100Skevans /* 525329100Skevans * Redetect total MWs so we unmap properly -- in case we lowered the 526329100Skevans * maximum to work around Xeon errata. 527329100Skevans */ 528329100Skevans ntb_detect_max_mw(ntb); 529329100Skevans ntb_unmap_pci_bar(ntb); 530329100Skevans 531329100Skevans return (0); 532329100Skevans} 533329100Skevans 534329100Skevans/* 535329100Skevans * Driver internal routines 536329100Skevans */ 537329100Skevansstatic inline enum ntb_bar 538329100Skevansntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 539329100Skevans{ 540344399Skevans 541344399Skevans KASSERT(mw < ntb->mw_count || 542329100Skevans (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx), 543344399Skevans ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 544344399Skevans KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 545329100Skevans 546329100Skevans return (ntb->reg->mw_bar[mw]); 547329100Skevans} 548329100Skevans 549185029Spjdstatic inline bool 550344399Skevansbar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 551185029Spjd{ 552185096Sdfr /* XXX This assertion could be stronger. */ 553185096Sdfr KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 554185096Sdfr return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 555185096Sdfr} 556299997Speter 557185096Sdfrstatic inline void 558299997Speterbar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 559299997Speter uint32_t *xlat, uint32_t *lmt) 560299660Sngie{ 561299997Speter uint32_t basev, lmtv, xlatv; 562185029Spjd 563185029Spjd switch (bar) { 564185029Spjd case NTB_B2B_BAR_1: 565185029Spjd basev = ntb->xlat_reg->bar2_base; 566185029Spjd lmtv = ntb->xlat_reg->bar2_limit; 567296963Sallanjude xlatv = ntb->xlat_reg->bar2_xlat; 568185029Spjd break; 569344399Skevans case NTB_B2B_BAR_2: 570185029Spjd basev = ntb->xlat_reg->bar4_base; 571185029Spjd lmtv = ntb->xlat_reg->bar4_limit; 572296963Sallanjude xlatv = ntb->xlat_reg->bar4_xlat; 573296963Sallanjude break; 574296963Sallanjude case NTB_B2B_BAR_3: 575296963Sallanjude basev = ntb->xlat_reg->bar5_base; 576296963Sallanjude lmtv = ntb->xlat_reg->bar5_limit; 577296963Sallanjude xlatv = ntb->xlat_reg->bar5_xlat; 578296963Sallanjude break; 579344399Skevans default: 580296963Sallanjude KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 581296963Sallanjude ("bad bar")); 582296963Sallanjude basev = lmtv = xlatv = 0; 583344399Skevans break; 584344399Skevans } 585344399Skevans 586344399Skevans if (base != NULL) 587344399Skevans *base = basev; 588296963Sallanjude if (xlat != NULL) 589296963Sallanjude *xlat = xlatv; 590296963Sallanjude if (lmt != NULL) 591296963Sallanjude *lmt = lmtv; 592296963Sallanjude} 593296963Sallanjude 594185029Spjdstatic int 595344399Skevansntb_map_pci_bars(struct ntb_softc *ntb) 596185096Sdfr{ 597185096Sdfr int rc; 598185096Sdfr 599185096Sdfr ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 600185096Sdfr rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 601344399Skevans if (rc != 0) 602185096Sdfr goto out; 603185096Sdfr 604185096Sdfr ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 605185096Sdfr rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 606185096Sdfr if (rc != 0) 607185096Sdfr goto out; 608185096Sdfr ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 609185096Sdfr ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 610185096Sdfr ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 611185096Sdfr 612298826Spfg ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 613185096Sdfr /* XXX Are shared MW B2Bs write-combining? */ 614185096Sdfr if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 615185096Sdfr rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 616296963Sallanjude else 617296963Sallanjude rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 618185096Sdfr ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 619185096Sdfr ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 620185096Sdfr ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 621185096Sdfr 622185096Sdfr if (!HAS_FEATURE(NTB_SPLIT_BAR)) 623344399Skevans goto out; 624344399Skevans 625185096Sdfr ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 626185096Sdfr if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 627185096Sdfr rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 628185096Sdfr else 629185096Sdfr rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 630344399Skevans ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 631344399Skevans ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 632344399Skevans ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 633344399Skevans 634344399Skevansout: 635185096Sdfr if (rc != 0) 636185096Sdfr device_printf(ntb->device, 637185096Sdfr "unable to allocate pci resource\n"); 638185096Sdfr return (rc); 639344399Skevans} 640185096Sdfr 641296963Sallanjudestatic void 642344399Skevansprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 643344399Skevans{ 644344399Skevans 645344399Skevans device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n", 646344399Skevans bar->size, bar->vbase, (void *)(bar->pbase)); 647296963Sallanjude} 648296963Sallanjude 649296963Sallanjudestatic int 650344399Skevansmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 651296963Sallanjude{ 652296963Sallanjude 653296963Sallanjude bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 654296963Sallanjude &bar->pci_resource_id, RF_ACTIVE); 655344399Skevans if (bar->pci_resource == NULL) 656296963Sallanjude return (ENXIO); 657296963Sallanjude 658296963Sallanjude save_bar_parameters(bar); 659296963Sallanjude print_map_success(ntb, bar); 660296963Sallanjude return (0); 661185096Sdfr} 662185096Sdfr 663185096Sdfrstatic int 664185096Sdfrmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 665185096Sdfr{ 666185096Sdfr int rc; 667296963Sallanjude uint8_t bar_size_bits = 0; 668185096Sdfr 669344399Skevans bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 670185029Spjd &bar->pci_resource_id, RF_ACTIVE); 671185029Spjd 672185029Spjd if (bar->pci_resource == NULL) 673185029Spjd return (ENXIO); 674185029Spjd 675185029Spjd save_bar_parameters(bar); 676344399Skevans /* 677344399Skevans * Ivytown NTB BAR sizes are misreported by the hardware due to a 678344399Skevans * hardware issue. To work around this, query the size it should be 679344399Skevans * configured to by the device and modify the resource to correspond to 680344399Skevans * this new size. The BIOS on systems with this problem is required to 681185029Spjd * provide enough address space to allow the driver to make this change 682296963Sallanjude * safely. 683344399Skevans * 684344399Skevans * Ideally I could have just specified the size when I allocated the 685344399Skevans * resource like: 686344399Skevans * bus_alloc_resource(ntb->device, 687296963Sallanjude * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 688296963Sallanjude * 1ul << bar_size_bits, RF_ACTIVE); 689296963Sallanjude * but the PCI driver does not honor the size in this call, so we have 690344399Skevans * to modify it after the fact. 691296963Sallanjude */ 692296963Sallanjude if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 693296963Sallanjude if (bar->pci_resource_id == PCIR_BAR(2)) 694296963Sallanjude bar_size_bits = pci_read_config(ntb->device, 695344399Skevans XEON_PBAR23SZ_OFFSET, 1); 696296963Sallanjude else 697296963Sallanjude bar_size_bits = pci_read_config(ntb->device, 698296963Sallanjude XEON_PBAR45SZ_OFFSET, 1); 699296963Sallanjude 700296963Sallanjude rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 701185029Spjd bar->pci_resource, bar->pbase, 702185029Spjd bar->pbase + (1ul << bar_size_bits) - 1); 703185029Spjd if (rc != 0) { 704185029Spjd device_printf(ntb->device, 705185029Spjd "unable to resize bar\n"); 706185029Spjd return (rc); 707185029Spjd } 708185029Spjd 709344399Skevans save_bar_parameters(bar); 710308914Savg } 711308914Savg 712308914Savg /* Mark bar region as write combining to improve performance. */ 713185029Spjd rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 714208388Sjhb VM_MEMATTR_WRITE_COMBINING); 715208388Sjhb if (rc != 0) { 716200309Sjhb device_printf(ntb->device, 717200309Sjhb "unable to mark bar as WRITE_COMBINING\n"); 718200309Sjhb return (rc); 719200309Sjhb } 720200309Sjhb print_map_success(ntb, bar); 721200309Sjhb return (0); 722296963Sallanjude} 723296963Sallanjude 724200309Sjhbstatic void 725329175Skevansntb_unmap_pci_bar(struct ntb_softc *ntb) 726200309Sjhb{ 727344399Skevans struct ntb_pci_bar_info *current_bar; 728344399Skevans int i; 729344399Skevans 730344399Skevans for (i = 0; i < NTB_MAX_BARS; i++) { 731344399Skevans current_bar = &ntb->bar_info[i]; 732344399Skevans if (current_bar->pci_resource != NULL) 733344399Skevans bus_release_resource(ntb->device, SYS_RES_MEMORY, 734344399Skevans current_bar->pci_resource_id, 735185029Spjd current_bar->pci_resource); 736185029Spjd } 737185029Spjd} 738200309Sjhb 739200309Sjhbstatic int 740185029Spjdntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 741344399Skevans{ 742185029Spjd uint32_t i; 743344399Skevans int rc; 744344399Skevans 745185029Spjd for (i = 0; i < num_vectors; i++) { 746185029Spjd ntb->int_info[i].rid = i + 1; 747185029Spjd ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 748185029Spjd SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 749185029Spjd if (ntb->int_info[i].res == NULL) { 750185029Spjd device_printf(ntb->device, 751185029Spjd "bus_alloc_resource failed\n"); 752185029Spjd return (ENOMEM); 753185029Spjd } 754185029Spjd ntb->int_info[i].tag = NULL; 755185029Spjd ntb->allocated_interrupts++; 756344399Skevans rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 757185029Spjd INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 758185029Spjd &ntb->msix_vec[i], &ntb->int_info[i].tag); 759185029Spjd if (rc != 0) { 760185029Spjd device_printf(ntb->device, "bus_setup_intr failed\n"); 761185029Spjd return (ENXIO); 762185029Spjd } 763212805Spjd } 764212805Spjd return (0); 765212805Spjd} 766212805Spjd 767212805Spjd/* 768212805Spjd * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 769185029Spjd * cannot be allocated for each MSI-X message. JHB seems to think remapping 770185029Spjd * should be okay. This tunable should enable us to test that hypothesis 771185029Spjd * when someone gets their hands on some Xeon hardware. 772192194Sdfr */ 773192194Sdfrstatic int ntb_force_remap_mode; 774192194SdfrSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 775344399Skevans &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 776344399Skevans " to a smaller number of ithreads, even if the desired number are " 777344399Skevans "available"); 778344399Skevans 779344399Skevans/* 780344399Skevans * In case it is NOT ok, give consumers an abort button. 781344399Skevans */ 782344399Skevansstatic int ntb_prefer_intx; 783344399SkevansSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 784185029Spjd &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 785185029Spjd "than remapping MSI-X messages over available slots (match Linux driver " 786185029Spjd "behavior)"); 787241294Savg 788185029Spjd/* 789241294Savg * Remap the desired number of MSI-X messages to available ithreads in a simple 790185029Spjd * round-robin fashion. 791241294Savg */ 792241294Savgstatic int 793241294Savgntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 794185029Spjd{ 795185029Spjd u_int *vectors; 796241293Savg uint32_t i; 797241293Savg int rc; 798241293Savg 799308914Savg if (ntb_prefer_intx != 0) 800308914Savg return (ENXIO); 801308914Savg 802308914Savg vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 803308914Savg 804308914Savg for (i = 0; i < desired; i++) 805308914Savg vectors[i] = (i % avail) + 1; 806308914Savg 807308914Savg rc = pci_remap_msix(dev, desired, vectors); 808308914Savg free(vectors, M_NTB); 809308914Savg return (rc); 810308914Savg} 811308914Savg 812308914Savgstatic int 813308914Savgntb_init_isr(struct ntb_softc *ntb) 814308914Savg{ 815308914Savg uint32_t desired_vectors, num_vectors; 816329099Skevans int rc; 817308914Savg 818308914Savg ntb->allocated_interrupts = 0; 819308914Savg ntb->last_ts = ticks; 820308914Savg 821308914Savg /* 822308914Savg * Mask all doorbell interrupts. 823308914Savg */ 824308914Savg ntb_db_set_mask(ntb, ntb->db_valid_mask); 825308914Savg 826308914Savg num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 827308914Savg ntb->db_count); 828308914Savg if (desired_vectors >= 1) { 829308914Savg rc = pci_alloc_msix(ntb->device, &num_vectors); 830308914Savg 831308914Savg if (ntb_force_remap_mode != 0 && rc == 0 && 832235329Savg num_vectors == desired_vectors) 833235329Savg num_vectors--; 834235329Savg 835235329Savg if (rc == 0 && num_vectors < desired_vectors) { 836235329Savg rc = ntb_remap_msix(ntb->device, desired_vectors, 837185029Spjd num_vectors); 838198079Sjhb if (rc == 0) 839185029Spjd num_vectors = desired_vectors; 840185029Spjd else 841185029Spjd pci_release_msi(ntb->device); 842234339Savg } 843329099Skevans if (rc != 0) 844234339Savg num_vectors = 1; 845234339Savg } else 846234339Savg num_vectors = 1; 847329099Skevans 848185029Spjd if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 849234339Savg ntb->db_vec_count = 1; 850241288Savg ntb->db_vec_shift = ntb->db_count; 851185029Spjd rc = ntb_setup_legacy_interrupt(ntb); 852185029Spjd } else { 853185029Spjd ntb_create_msix_vec(ntb, num_vectors); 854185029Spjd rc = ntb_setup_msix(ntb, num_vectors); 855308914Savg } 856308914Savg if (rc != 0) { 857308914Savg device_printf(ntb->device, 858308914Savg "Error allocating interrupts: %d\n", rc); 859185029Spjd ntb_free_msix_vec(ntb); 860348866Skevans } 861348866Skevans 862348866Skevans return (rc); 863348866Skevans} 864348866Skevans 865348866Skevansstatic int 866348866Skevansntb_setup_legacy_interrupt(struct ntb_softc *ntb) 867348866Skevans{ 868348866Skevans int rc; 869185029Spjd 870185029Spjd ntb->int_info[0].rid = 0; 871185029Spjd ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 872185029Spjd &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 873185029Spjd if (ntb->int_info[0].res == NULL) { 874185029Spjd device_printf(ntb->device, "bus_alloc_resource failed\n"); 875185029Spjd return (ENOMEM); 876235329Savg } 877235329Savg 878235329Savg ntb->int_info[0].tag = NULL; 879241288Savg ntb->allocated_interrupts = 1; 880235329Savg 881235329Savg rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 882241288Savg INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 883241288Savg ntb, &ntb->int_info[0].tag); 884241288Savg if (rc != 0) { 885241288Savg device_printf(ntb->device, "bus_setup_intr failed\n"); 886235329Savg return (ENXIO); 887241288Savg } 888235329Savg 889241288Savg return (0); 890235329Savg} 891185029Spjd 892185029Spjdstatic void 893213136Spjdntb_teardown_interrupts(struct ntb_softc *ntb) 894213136Spjd{ 895185029Spjd struct ntb_int_info *current_int; 896185029Spjd int i; 897185029Spjd 898329099Skevans for (i = 0; i < ntb->allocated_interrupts; i++) { 899185029Spjd current_int = &ntb->int_info[i]; 900185029Spjd if (current_int->tag != NULL) 901185029Spjd bus_teardown_intr(ntb->device, current_int->res, 902185029Spjd current_int->tag); 903185029Spjd 904185029Spjd if (current_int->res != NULL) 905185029Spjd bus_release_resource(ntb->device, SYS_RES_IRQ, 906185029Spjd rman_get_rid(current_int->res), current_int->res); 907185029Spjd } 908185029Spjd 909308914Savg ntb_free_msix_vec(ntb); 910185029Spjd pci_release_msi(ntb->device); 911185029Spjd} 912308914Savg 913308914Savg/* 914308914Savg * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon. Abstract it 915308914Savg * out to make code clearer. 916308914Savg */ 917308914Savgstatic inline uint64_t 918185029Spjddb_ioread(struct ntb_softc *ntb, uint64_t regoff) 919185029Spjd{ 920185029Spjd 921185029Spjd if (ntb->type == NTB_SOC) 922185029Spjd return (ntb_reg_read(8, regoff)); 923185029Spjd 924185029Spjd KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 925185029Spjd 926185029Spjd return (ntb_reg_read(2, regoff)); 927185029Spjd} 928185029Spjd 929185029Spjdstatic inline void 930185029Spjddb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 931185029Spjd{ 932185029Spjd 933235329Savg KASSERT((val & ~ntb->db_valid_mask) == 0, 934235329Savg ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 935185029Spjd (uintmax_t)(val & ~ntb->db_valid_mask), 936185029Spjd (uintmax_t)ntb->db_valid_mask)); 937185029Spjd 938185029Spjd if (regoff == ntb->self_reg->db_mask) 939185029Spjd DB_MASK_ASSERT(ntb, MA_OWNED); 940185029Spjd 941185029Spjd if (ntb->type == NTB_SOC) { 942185029Spjd ntb_reg_write(8, regoff, val); 943185029Spjd return; 944185029Spjd } 945185029Spjd 946185029Spjd KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 947185029Spjd ntb_reg_write(2, regoff, (uint16_t)val); 948185029Spjd} 949185029Spjd 950185029Spjdvoid 951185029Spjdntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 952185029Spjd{ 953185029Spjd 954185029Spjd DB_MASK_LOCK(ntb); 955185029Spjd ntb->db_mask |= bits; 956185029Spjd db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 957185029Spjd DB_MASK_UNLOCK(ntb); 958185029Spjd} 959185029Spjd 960185029Spjdvoid 961185029Spjdntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 962185029Spjd{ 963185029Spjd 964185029Spjd KASSERT((bits & ~ntb->db_valid_mask) == 0, 965185029Spjd ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 966185029Spjd (uintmax_t)(bits & ~ntb->db_valid_mask), 967185029Spjd (uintmax_t)ntb->db_valid_mask)); 968185029Spjd 969185029Spjd DB_MASK_LOCK(ntb); 970185029Spjd ntb->db_mask &= ~bits; 971185029Spjd db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 972185029Spjd DB_MASK_UNLOCK(ntb); 973185029Spjd} 974185029Spjd 975185029Spjduint64_t 976185029Spjdntb_db_read(struct ntb_softc *ntb) 977185029Spjd{ 978185029Spjd 979185029Spjd return (db_ioread(ntb, ntb->self_reg->db_bell)); 980185029Spjd} 981185029Spjd 982185029Spjdvoid 983185029Spjdntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 984185029Spjd{ 985185029Spjd 986185029Spjd KASSERT((bits & ~ntb->db_valid_mask) == 0, 987185029Spjd ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 988185029Spjd (uintmax_t)(bits & ~ntb->db_valid_mask), 989185029Spjd (uintmax_t)ntb->db_valid_mask)); 990185029Spjd 991185029Spjd db_iowrite(ntb, ntb->self_reg->db_bell, bits); 992185029Spjd} 993185029Spjd 994185029Spjdstatic inline uint64_t 995185029Spjdntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 996185029Spjd{ 997185029Spjd uint64_t shift, mask; 998185029Spjd 999185029Spjd shift = ntb->db_vec_shift; 1000185029Spjd mask = (1ull << shift) - 1; 1001185029Spjd return (mask << (shift * db_vector)); 1002185029Spjd} 1003185029Spjd 1004185029Spjdstatic void 1005185029Spjdntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1006185029Spjd{ 1007185029Spjd uint64_t vec_mask; 1008235329Savg 1009235329Savg ntb->last_ts = ticks; 1010235329Savg vec_mask = ntb_vec_mask(ntb, vec); 1011241293Savg 1012296963Sallanjude if ((vec_mask & ntb->db_link_mask) != 0) { 1013329099Skevans if (ntb_poll_link(ntb)) 1014344399Skevans ntb_link_event(ntb); 1015296963Sallanjude } 1016241293Savg 1017241293Savg if ((vec_mask & ntb->db_valid_mask) != 0) 1018241293Savg ntb_db_event(ntb, vec); 1019241293Savg} 1020344399Skevans 1021344399Skevansstatic void 1022344399Skevansndev_vec_isr(void *arg) 1023344399Skevans{ 1024344399Skevans struct ntb_vec *nvec = arg; 1025185029Spjd 1026185029Spjd ntb_interrupt(nvec->ntb, nvec->num); 1027235329Savg} 1028185029Spjd 1029185029Spjdstatic void 1030235329Savgndev_irq_isr(void *arg) 1031235329Savg{ 1032185029Spjd /* If we couldn't set up MSI-X, we only have the one vector. */ 1033185029Spjd ntb_interrupt(arg, 0); 1034185029Spjd} 1035241288Savg 1036241288Savgstatic int 1037241288Savgntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1038241288Savg{ 1039241288Savg uint32_t i; 1040241288Savg 1041241288Savg ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1042241288Savg M_ZERO | M_WAITOK); 1043241288Savg for (i = 0; i < num_vectors; i++) { 1044241288Savg ntb->msix_vec[i].num = i; 1045241288Savg ntb->msix_vec[i].ntb = ntb; 1046241288Savg } 1047241288Savg 1048241288Savg return (0); 1049241288Savg} 1050241288Savg 1051241288Savgstatic void 1052241288Savgntb_free_msix_vec(struct ntb_softc *ntb) 1053241288Savg{ 1054241288Savg 1055241288Savg if (ntb->msix_vec == NULL) 1056241288Savg return; 1057241288Savg 1058241288Savg free(ntb->msix_vec, M_NTB); 1059241288Savg ntb->msix_vec = NULL; 1060241288Savg} 1061241288Savg 1062241288Savgstatic struct ntb_hw_info * 1063241288Savgntb_get_device_info(uint32_t device_id) 1064241288Savg{ 1065241288Savg struct ntb_hw_info *ep = pci_ids; 1066241288Savg 1067241288Savg while (ep->device_id) { 1068241288Savg if (ep->device_id == device_id) 1069241288Savg return (ep); 1070329099Skevans ++ep; 1071185029Spjd } 1072185029Spjd return (NULL); 1073185029Spjd} 1074185029Spjd 1075185029Spjdstatic void 1076185029Spjdntb_teardown_xeon(struct ntb_softc *ntb) 1077185029Spjd{ 1078185029Spjd 1079185029Spjd ntb_link_disable(ntb); 1080185029Spjd} 1081185029Spjd 1082185029Spjdstatic void 1083185029Spjdntb_detect_max_mw(struct ntb_softc *ntb) 1084185029Spjd{ 1085185029Spjd 1086185029Spjd if (ntb->type == NTB_SOC) { 1087185029Spjd ntb->mw_count = SOC_MW_COUNT; 1088185029Spjd return; 1089185029Spjd } 1090185029Spjd 1091185029Spjd if (HAS_FEATURE(NTB_SPLIT_BAR)) 1092185029Spjd ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1093185029Spjd else 1094185029Spjd ntb->mw_count = XEON_SNB_MW_COUNT; 1095185029Spjd} 1096185029Spjd 1097185029Spjdstatic int 1098185029Spjdntb_detect_xeon(struct ntb_softc *ntb) 1099185029Spjd{ 1100185029Spjd uint8_t ppd, conn_type; 1101185029Spjd 1102185029Spjd ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1103185029Spjd ntb->ppd = ppd; 1104185029Spjd 1105185029Spjd if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1106185029Spjd ntb->dev_type = NTB_DEV_USD; 1107185029Spjd else 1108185029Spjd ntb->dev_type = NTB_DEV_DSD; 1109185029Spjd 1110185029Spjd if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1111185029Spjd ntb->features |= NTB_SPLIT_BAR; 1112241301Savg 1113241301Savg /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1114241301Savg if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1115241301Savg ntb->features |= NTB_SDOORBELL_LOCKUP; 1116185029Spjd 1117185029Spjd conn_type = ppd & XEON_PPD_CONN_TYPE; 1118185029Spjd switch (conn_type) { 1119235329Savg case NTB_CONN_B2B: 1120185029Spjd ntb->conn_type = conn_type; 1121185029Spjd break; 1122185029Spjd case NTB_CONN_RP: 1123185029Spjd case NTB_CONN_TRANSPARENT: 1124185029Spjd default: 1125185029Spjd device_printf(ntb->device, "Unsupported connection type: %u\n", 1126185029Spjd (unsigned)conn_type); 1127185029Spjd return (ENXIO); 1128185029Spjd } 1129185029Spjd return (0); 1130185029Spjd} 1131185029Spjd 1132185029Spjdstatic int 1133185029Spjdntb_detect_soc(struct ntb_softc *ntb) 1134185029Spjd{ 1135185029Spjd uint32_t ppd, conn_type; 1136241288Savg 1137241288Savg ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1138241288Savg ntb->ppd = ppd; 1139241288Savg 1140241288Savg if ((ppd & SOC_PPD_DEV_TYPE) != 0) 1141241288Savg ntb->dev_type = NTB_DEV_DSD; 1142185029Spjd else 1143185029Spjd ntb->dev_type = NTB_DEV_USD; 1144241288Savg 1145185029Spjd conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8; 1146241288Savg switch (conn_type) { 1147241288Savg case NTB_CONN_B2B: 1148185029Spjd ntb->conn_type = conn_type; 1149241288Savg break; 1150185029Spjd default: 1151185029Spjd device_printf(ntb->device, "Unsupported NTB configuration\n"); 1152185029Spjd return (ENXIO); 1153185029Spjd } 1154185029Spjd return (0); 1155185029Spjd} 1156185029Spjd 1157185029Spjdstatic int 1158185029Spjdntb_xeon_init_dev(struct ntb_softc *ntb) 1159185029Spjd{ 1160185029Spjd int rc; 1161 1162 ntb->spad_count = XEON_SPAD_COUNT; 1163 ntb->db_count = XEON_DB_COUNT; 1164 ntb->db_link_mask = XEON_DB_LINK_BIT; 1165 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1166 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1167 1168 if (ntb->conn_type != NTB_CONN_B2B) { 1169 device_printf(ntb->device, "Connection type %d not supported\n", 1170 ntb->conn_type); 1171 return (ENXIO); 1172 } 1173 1174 ntb->reg = &xeon_reg; 1175 ntb->self_reg = &xeon_pri_reg; 1176 ntb->peer_reg = &xeon_b2b_reg; 1177 ntb->xlat_reg = &xeon_sec_xlat; 1178 1179 /* 1180 * There is a Xeon hardware errata related to writes to SDOORBELL or 1181 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1182 * which may hang the system. To workaround this use the second memory 1183 * window to access the interrupt and scratch pad registers on the 1184 * remote system. 1185 */ 1186 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1187 /* Use the last MW for mapping remote spad */ 1188 ntb->b2b_mw_idx = ntb->mw_count - 1; 1189 else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1190 /* 1191 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1192 * mirrored to the remote system. Shrink the number of bits by one, 1193 * since bit 14 is the last bit. 1194 * 1195 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1196 * anyway. Nor for non-B2B connection types. 1197 */ 1198 ntb->db_count = XEON_DB_COUNT - 1; 1199 1200 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1201 1202 if (ntb->dev_type == NTB_DEV_USD) 1203 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1204 &xeon_b2b_usd_addr); 1205 else 1206 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1207 &xeon_b2b_dsd_addr); 1208 if (rc != 0) 1209 return (rc); 1210 1211 /* Enable Bus Master and Memory Space on the secondary side */ 1212 ntb_reg_write(2, XEON_PCICMD_OFFSET, 1213 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1214 1215 /* Enable link training */ 1216 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1217 1218 return (0); 1219} 1220 1221static int 1222ntb_soc_init_dev(struct ntb_softc *ntb) 1223{ 1224 1225 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1226 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1227 1228 ntb->spad_count = SOC_SPAD_COUNT; 1229 ntb->db_count = SOC_DB_COUNT; 1230 ntb->db_vec_count = SOC_DB_MSIX_VECTOR_COUNT; 1231 ntb->db_vec_shift = SOC_DB_MSIX_VECTOR_SHIFT; 1232 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1233 1234 ntb->reg = &soc_reg; 1235 ntb->self_reg = &soc_pri_reg; 1236 ntb->peer_reg = &soc_b2b_reg; 1237 ntb->xlat_reg = &soc_sec_xlat; 1238 1239 /* 1240 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 1241 * resolved. Mask transaction layer internal parity errors. 1242 */ 1243 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1244 1245 configure_soc_secondary_side_bars(ntb); 1246 1247 /* Enable Bus Master and Memory Space on the secondary side */ 1248 ntb_reg_write(2, SOC_PCICMD_OFFSET, 1249 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1250 1251 /* Initiate PCI-E link training */ 1252 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1253 1254 callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb); 1255 1256 return (0); 1257} 1258 1259/* XXX: Linux driver doesn't seem to do any of this for SoC. */ 1260static void 1261configure_soc_secondary_side_bars(struct ntb_softc *ntb) 1262{ 1263 1264 if (ntb->dev_type == NTB_DEV_USD) { 1265 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1266 XEON_B2B_BAR2_DSD_ADDR64); 1267 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1268 XEON_B2B_BAR4_DSD_ADDR64); 1269 ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64); 1270 ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64); 1271 } else { 1272 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1273 XEON_B2B_BAR2_USD_ADDR64); 1274 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1275 XEON_B2B_BAR4_USD_ADDR64); 1276 ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64); 1277 ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64); 1278 } 1279} 1280 1281 1282/* 1283 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1284 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1285 * remains for use by a higher layer. 1286 * 1287 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1288 * MW size is sufficiently large. 1289 */ 1290static unsigned int ntb_b2b_mw_share; 1291SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1292 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1293 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1294 "value here."); 1295 1296static void 1297xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1298 enum ntb_bar regbar) 1299{ 1300 struct ntb_pci_bar_info *bar; 1301 uint8_t bar_sz; 1302 1303 if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1304 return; 1305 1306 bar = &ntb->bar_info[idx]; 1307 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1308 if (idx == regbar) { 1309 if (ntb->b2b_off != 0) 1310 bar_sz--; 1311 else 1312 bar_sz = 0; 1313 } 1314 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1315 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1316 (void)bar_sz; 1317} 1318 1319static void 1320xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1321 enum ntb_bar idx, enum ntb_bar regbar) 1322{ 1323 uint64_t reg_val; 1324 uint32_t base_reg, lmt_reg; 1325 1326 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1327 if (idx == regbar) 1328 bar_addr += ntb->b2b_off; 1329 1330 if (!bar_is_64bit(ntb, idx)) { 1331 ntb_reg_write(4, base_reg, bar_addr); 1332 reg_val = ntb_reg_read(4, base_reg); 1333 (void)reg_val; 1334 1335 ntb_reg_write(4, lmt_reg, bar_addr); 1336 reg_val = ntb_reg_read(4, lmt_reg); 1337 (void)reg_val; 1338 } else { 1339 ntb_reg_write(8, base_reg, bar_addr); 1340 reg_val = ntb_reg_read(8, base_reg); 1341 (void)reg_val; 1342 1343 ntb_reg_write(8, lmt_reg, bar_addr); 1344 reg_val = ntb_reg_read(8, lmt_reg); 1345 (void)reg_val; 1346 } 1347} 1348 1349static void 1350xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1351{ 1352 struct ntb_pci_bar_info *bar; 1353 1354 bar = &ntb->bar_info[idx]; 1355 if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1356 ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1357 base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1358 } else { 1359 ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1360 base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1361 } 1362 (void)base_addr; 1363} 1364 1365static int 1366xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1367 const struct ntb_b2b_addr *peer_addr) 1368{ 1369 struct ntb_pci_bar_info *b2b_bar; 1370 vm_size_t bar_size; 1371 uint64_t bar_addr; 1372 enum ntb_bar b2b_bar_num, i; 1373 1374 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1375 b2b_bar = NULL; 1376 b2b_bar_num = NTB_CONFIG_BAR; 1377 ntb->b2b_off = 0; 1378 } else { 1379 b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1380 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1381 ("invalid b2b mw bar")); 1382 1383 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1384 bar_size = b2b_bar->size; 1385 1386 if (ntb_b2b_mw_share != 0 && 1387 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1388 ntb->b2b_off = bar_size >> 1; 1389 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1390 ntb->b2b_off = 0; 1391 ntb->mw_count--; 1392 } else { 1393 device_printf(ntb->device, 1394 "B2B bar size is too small!\n"); 1395 return (EIO); 1396 } 1397 } 1398 1399 /* 1400 * Reset the secondary bar sizes to match the primary bar sizes. 1401 * (Except, disable or halve the size of the B2B secondary bar.) 1402 */ 1403 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1404 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1405 1406 bar_addr = 0; 1407 if (b2b_bar_num == NTB_CONFIG_BAR) 1408 bar_addr = addr->bar0_addr; 1409 else if (b2b_bar_num == NTB_B2B_BAR_1) 1410 bar_addr = addr->bar2_addr64; 1411 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1412 bar_addr = addr->bar4_addr64; 1413 else if (b2b_bar_num == NTB_B2B_BAR_2) 1414 bar_addr = addr->bar4_addr32; 1415 else if (b2b_bar_num == NTB_B2B_BAR_3) 1416 bar_addr = addr->bar5_addr32; 1417 else 1418 KASSERT(false, ("invalid bar")); 1419 1420 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1421 1422 /* 1423 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1424 * register BAR. The B2B BAR is either disabled above or configured 1425 * half-size. It starts at PBAR xlat + offset. 1426 * 1427 * Also set up incoming BAR limits == base (zero length window). 1428 */ 1429 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1430 b2b_bar_num); 1431 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1432 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1433 NTB_B2B_BAR_2, b2b_bar_num); 1434 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1435 NTB_B2B_BAR_3, b2b_bar_num); 1436 } else 1437 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1438 NTB_B2B_BAR_2, b2b_bar_num); 1439 1440 /* Zero incoming translation addrs */ 1441 ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1442 ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1443 1444 /* Zero outgoing translation limits (whole bar size windows) */ 1445 ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1446 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1447 1448 /* Set outgoing translation offsets */ 1449 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1450 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1451 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1452 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1453 } else 1454 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1455 1456 /* Set the translation offset for B2B registers */ 1457 bar_addr = 0; 1458 if (b2b_bar_num == NTB_CONFIG_BAR) 1459 bar_addr = peer_addr->bar0_addr; 1460 else if (b2b_bar_num == NTB_B2B_BAR_1) 1461 bar_addr = peer_addr->bar2_addr64; 1462 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1463 bar_addr = peer_addr->bar4_addr64; 1464 else if (b2b_bar_num == NTB_B2B_BAR_2) 1465 bar_addr = peer_addr->bar4_addr32; 1466 else if (b2b_bar_num == NTB_B2B_BAR_3) 1467 bar_addr = peer_addr->bar5_addr32; 1468 else 1469 KASSERT(false, ("invalid bar")); 1470 1471 /* 1472 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1473 * at a time. 1474 */ 1475 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1476 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1477 return (0); 1478} 1479 1480static inline bool 1481link_is_up(struct ntb_softc *ntb) 1482{ 1483 1484 if (ntb->type == NTB_XEON) 1485 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1486 1487 KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1488 return ((ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0); 1489} 1490 1491static inline bool 1492soc_link_is_err(struct ntb_softc *ntb) 1493{ 1494 uint32_t status; 1495 1496 KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1497 1498 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1499 if ((status & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1500 return (true); 1501 1502 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1503 return ((status & SOC_IBIST_ERR_OFLOW) != 0); 1504} 1505 1506/* SOC does not have link status interrupt, poll on that platform */ 1507static void 1508soc_link_hb(void *arg) 1509{ 1510 struct ntb_softc *ntb = arg; 1511 sbintime_t timo, poll_ts; 1512 1513 timo = NTB_HB_TIMEOUT * hz; 1514 poll_ts = ntb->last_ts + timo; 1515 1516 /* 1517 * Delay polling the link status if an interrupt was received, unless 1518 * the cached link status says the link is down. 1519 */ 1520 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1521 timo = poll_ts - ticks; 1522 goto out; 1523 } 1524 1525 if (ntb_poll_link(ntb)) 1526 ntb_link_event(ntb); 1527 1528 if (!link_is_up(ntb) && soc_link_is_err(ntb)) { 1529 /* Link is down with error, proceed with recovery */ 1530 callout_reset(&ntb->lr_timer, 0, recover_soc_link, ntb); 1531 return; 1532 } 1533 1534out: 1535 callout_reset(&ntb->heartbeat_timer, timo, soc_link_hb, ntb); 1536} 1537 1538static void 1539soc_perform_link_restart(struct ntb_softc *ntb) 1540{ 1541 uint32_t status; 1542 1543 /* Driver resets the NTB ModPhy lanes - magic! */ 1544 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1545 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1546 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1547 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1548 1549 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1550 pause("ModPhy", hz / 10); 1551 1552 /* Clear AER Errors, write to clear */ 1553 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1554 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1555 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1556 1557 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1558 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1559 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1560 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1561 1562 /* Clear DeSkew Buffer error, write to clear */ 1563 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1564 status |= SOC_DESKEWSTS_DBERR; 1565 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1566 1567 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1568 status &= SOC_IBIST_ERR_OFLOW; 1569 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1570 1571 /* Releases the NTB state machine to allow the link to retrain */ 1572 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1573 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1574 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1575} 1576 1577/* 1578 * ntb_set_ctx() - associate a driver context with an ntb device 1579 * @ntb: NTB device context 1580 * @ctx: Driver context 1581 * @ctx_ops: Driver context operations 1582 * 1583 * Associate a driver context and operations with a ntb device. The context is 1584 * provided by the client driver, and the driver may associate a different 1585 * context with each ntb device. 1586 * 1587 * Return: Zero if the context is associated, otherwise an error number. 1588 */ 1589int 1590ntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 1591{ 1592 1593 if (ctx == NULL || ops == NULL) 1594 return (EINVAL); 1595 if (ntb->ctx_ops != NULL) 1596 return (EINVAL); 1597 1598 CTX_LOCK(ntb); 1599 if (ntb->ctx_ops != NULL) { 1600 CTX_UNLOCK(ntb); 1601 return (EINVAL); 1602 } 1603 ntb->ntb_ctx = ctx; 1604 ntb->ctx_ops = ops; 1605 CTX_UNLOCK(ntb); 1606 1607 return (0); 1608} 1609 1610/* 1611 * It is expected that this will only be used from contexts where the ctx_lock 1612 * is not needed to protect ntb_ctx lifetime. 1613 */ 1614void * 1615ntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 1616{ 1617 1618 KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 1619 if (ops != NULL) 1620 *ops = ntb->ctx_ops; 1621 return (ntb->ntb_ctx); 1622} 1623 1624/* 1625 * ntb_clear_ctx() - disassociate any driver context from an ntb device 1626 * @ntb: NTB device context 1627 * 1628 * Clear any association that may exist between a driver context and the ntb 1629 * device. 1630 */ 1631void 1632ntb_clear_ctx(struct ntb_softc *ntb) 1633{ 1634 1635 CTX_LOCK(ntb); 1636 ntb->ntb_ctx = NULL; 1637 ntb->ctx_ops = NULL; 1638 CTX_UNLOCK(ntb); 1639} 1640 1641/* 1642 * ntb_link_event() - notify driver context of a change in link status 1643 * @ntb: NTB device context 1644 * 1645 * Notify the driver context that the link status may have changed. The driver 1646 * should call ntb_link_is_up() to get the current status. 1647 */ 1648void 1649ntb_link_event(struct ntb_softc *ntb) 1650{ 1651 1652 CTX_LOCK(ntb); 1653 if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 1654 ntb->ctx_ops->link_event(ntb->ntb_ctx); 1655 CTX_UNLOCK(ntb); 1656} 1657 1658/* 1659 * ntb_db_event() - notify driver context of a doorbell event 1660 * @ntb: NTB device context 1661 * @vector: Interrupt vector number 1662 * 1663 * Notify the driver context of a doorbell event. If hardware supports 1664 * multiple interrupt vectors for doorbells, the vector number indicates which 1665 * vector received the interrupt. The vector number is relative to the first 1666 * vector used for doorbells, starting at zero, and must be less than 1667 * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 1668 * doorbell bits need service, and ntb_db_vector_mask() to determine which of 1669 * those bits are associated with the vector number. 1670 */ 1671static void 1672ntb_db_event(struct ntb_softc *ntb, uint32_t vec) 1673{ 1674 1675 CTX_LOCK(ntb); 1676 if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 1677 ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 1678 CTX_UNLOCK(ntb); 1679} 1680 1681/* 1682 * ntb_link_enable() - enable the link on the secondary side of the ntb 1683 * @ntb: NTB device context 1684 * @max_speed: The maximum link speed expressed as PCIe generation number[0] 1685 * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 1686 * 1687 * Enable the link on the secondary side of the ntb. This can only be done 1688 * from the primary side of the ntb in primary or b2b topology. The ntb device 1689 * should train the link to its maximum speed and width, or the requested speed 1690 * and width, whichever is smaller, if supported. 1691 * 1692 * Return: Zero on success, otherwise an error number. 1693 * 1694 * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 1695 * and width input will be ignored. 1696 */ 1697int 1698ntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 1699 enum ntb_width w __unused) 1700{ 1701 uint32_t cntl; 1702 1703 if (ntb->type == NTB_SOC) { 1704 pci_write_config(ntb->device, NTB_PPD_OFFSET, 1705 ntb->ppd | SOC_PPD_INIT_LINK, 4); 1706 return (0); 1707 } 1708 1709 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1710 ntb_link_event(ntb); 1711 return (0); 1712 } 1713 1714 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1715 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1716 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1717 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1718 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1719 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1720 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1721 return (0); 1722} 1723 1724/* 1725 * ntb_link_disable() - disable the link on the secondary side of the ntb 1726 * @ntb: NTB device context 1727 * 1728 * Disable the link on the secondary side of the ntb. This can only be done 1729 * from the primary side of the ntb in primary or b2b topology. The ntb device 1730 * should disable the link. Returning from this call must indicate that a 1731 * barrier has passed, though with no more writes may pass in either direction 1732 * across the link, except if this call returns an error number. 1733 * 1734 * Return: Zero on success, otherwise an error number. 1735 */ 1736int 1737ntb_link_disable(struct ntb_softc *ntb) 1738{ 1739 uint32_t cntl; 1740 1741 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1742 ntb_link_event(ntb); 1743 return (0); 1744 } 1745 1746 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1747 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1748 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1749 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1750 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1751 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1752 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1753 return (0); 1754} 1755 1756static void 1757recover_soc_link(void *arg) 1758{ 1759 struct ntb_softc *ntb = arg; 1760 unsigned speed, width, oldspeed, oldwidth; 1761 uint32_t status32; 1762 1763 soc_perform_link_restart(ntb); 1764 1765 /* 1766 * There is a potential race between the 2 NTB devices recovering at 1767 * the same time. If the times are the same, the link will not recover 1768 * and the driver will be stuck in this loop forever. Add a random 1769 * interval to the recovery time to prevent this race. 1770 */ 1771 status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1772 pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1773 1774 if (soc_link_is_err(ntb)) 1775 goto retry; 1776 1777 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1778 if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1779 goto out; 1780 1781 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1782 width = NTB_LNK_STA_WIDTH(status32); 1783 speed = status32 & NTB_LINK_SPEED_MASK; 1784 1785 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 1786 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 1787 if (oldwidth != width || oldspeed != speed) 1788 goto retry; 1789 1790out: 1791 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, 1792 ntb); 1793 return; 1794 1795retry: 1796 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1797 ntb); 1798} 1799 1800/* 1801 * Polls the HW link status register(s); returns true if something has changed. 1802 */ 1803static bool 1804ntb_poll_link(struct ntb_softc *ntb) 1805{ 1806 uint32_t ntb_cntl; 1807 uint16_t reg_val; 1808 1809 if (ntb->type == NTB_SOC) { 1810 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1811 if (ntb_cntl == ntb->ntb_ctl) 1812 return (false); 1813 1814 ntb->ntb_ctl = ntb_cntl; 1815 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1816 } else { 1817 db_iowrite(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 1818 1819 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 1820 if (reg_val == ntb->lnk_sta) 1821 return (false); 1822 1823 ntb->lnk_sta = reg_val; 1824 } 1825 return (true); 1826} 1827 1828static inline enum ntb_speed 1829ntb_link_sta_speed(struct ntb_softc *ntb) 1830{ 1831 1832 if (!link_is_up(ntb)) 1833 return (NTB_SPEED_NONE); 1834 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 1835} 1836 1837static inline enum ntb_width 1838ntb_link_sta_width(struct ntb_softc *ntb) 1839{ 1840 1841 if (!link_is_up(ntb)) 1842 return (NTB_WIDTH_NONE); 1843 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 1844} 1845 1846/* 1847 * Public API to the rest of the OS 1848 */ 1849 1850/** 1851 * ntb_get_max_spads() - get the total scratch regs usable 1852 * @ntb: pointer to ntb_softc instance 1853 * 1854 * This function returns the max 32bit scratchpad registers usable by the 1855 * upper layer. 1856 * 1857 * RETURNS: total number of scratch pad registers available 1858 */ 1859uint8_t 1860ntb_get_max_spads(struct ntb_softc *ntb) 1861{ 1862 1863 return (ntb->spad_count); 1864} 1865 1866uint8_t 1867ntb_mw_count(struct ntb_softc *ntb) 1868{ 1869 1870 return (ntb->mw_count); 1871} 1872 1873/** 1874 * ntb_spad_write() - write to the secondary scratchpad register 1875 * @ntb: pointer to ntb_softc instance 1876 * @idx: index to the scratchpad register, 0 based 1877 * @val: the data value to put into the register 1878 * 1879 * This function allows writing of a 32bit value to the indexed scratchpad 1880 * register. The register resides on the secondary (external) side. 1881 * 1882 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1883 */ 1884int 1885ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1886{ 1887 1888 if (idx >= ntb->spad_count) 1889 return (EINVAL); 1890 1891 ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 1892 1893 return (0); 1894} 1895 1896/** 1897 * ntb_spad_read() - read from the primary scratchpad register 1898 * @ntb: pointer to ntb_softc instance 1899 * @idx: index to scratchpad register, 0 based 1900 * @val: pointer to 32bit integer for storing the register value 1901 * 1902 * This function allows reading of the 32bit scratchpad register on 1903 * the primary (internal) side. 1904 * 1905 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1906 */ 1907int 1908ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1909{ 1910 1911 if (idx >= ntb->spad_count) 1912 return (EINVAL); 1913 1914 *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 1915 1916 return (0); 1917} 1918 1919/** 1920 * ntb_peer_spad_write() - write to the secondary scratchpad register 1921 * @ntb: pointer to ntb_softc instance 1922 * @idx: index to the scratchpad register, 0 based 1923 * @val: the data value to put into the register 1924 * 1925 * This function allows writing of a 32bit value to the indexed scratchpad 1926 * register. The register resides on the secondary (external) side. 1927 * 1928 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1929 */ 1930int 1931ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1932{ 1933 1934 if (idx >= ntb->spad_count) 1935 return (EINVAL); 1936 1937 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1938 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1939 else 1940 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 1941 1942 return (0); 1943} 1944 1945/** 1946 * ntb_peer_spad_read() - read from the primary scratchpad register 1947 * @ntb: pointer to ntb_softc instance 1948 * @idx: index to scratchpad register, 0 based 1949 * @val: pointer to 32bit integer for storing the register value 1950 * 1951 * This function allows reading of the 32bit scratchpad register on 1952 * the primary (internal) side. 1953 * 1954 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1955 */ 1956int 1957ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1958{ 1959 1960 if (idx >= ntb->spad_count) 1961 return (EINVAL); 1962 1963 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1964 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1965 else 1966 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 1967 1968 return (0); 1969} 1970 1971/* 1972 * ntb_mw_get_range() - get the range of a memory window 1973 * @ntb: NTB device context 1974 * @idx: Memory window number 1975 * @base: OUT - the base address for mapping the memory window 1976 * @size: OUT - the size for mapping the memory window 1977 * @align: OUT - the base alignment for translating the memory window 1978 * @align_size: OUT - the size alignment for translating the memory window 1979 * 1980 * Get the range of a memory window. NULL may be given for any output 1981 * parameter if the value is not needed. The base and size may be used for 1982 * mapping the memory window, to access the peer memory. The alignment and 1983 * size may be used for translating the memory window, for the peer to access 1984 * memory on the local system. 1985 * 1986 * Return: Zero on success, otherwise an error number. 1987 */ 1988int 1989ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 1990 void **vbase, size_t *size, size_t *align, size_t *align_size) 1991{ 1992 struct ntb_pci_bar_info *bar; 1993 size_t bar_b2b_off; 1994 1995 if (mw_idx >= ntb_mw_count(ntb)) 1996 return (EINVAL); 1997 1998 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)]; 1999 bar_b2b_off = 0; 2000 if (mw_idx == ntb->b2b_mw_idx) { 2001 KASSERT(ntb->b2b_off != 0, 2002 ("user shouldn't get non-shared b2b mw")); 2003 bar_b2b_off = ntb->b2b_off; 2004 } 2005 2006 if (base != NULL) 2007 *base = bar->pbase + bar_b2b_off; 2008 if (vbase != NULL) 2009 *vbase = (char *)bar->vbase + bar_b2b_off; 2010 if (size != NULL) 2011 *size = bar->size - bar_b2b_off; 2012 if (align != NULL) 2013 *align = bar->size; 2014 if (align_size != NULL) 2015 *align_size = 1; 2016 return (0); 2017} 2018 2019/* 2020 * ntb_mw_set_trans() - set the translation of a memory window 2021 * @ntb: NTB device context 2022 * @idx: Memory window number 2023 * @addr: The dma address local memory to expose to the peer 2024 * @size: The size of the local memory to expose to the peer 2025 * 2026 * Set the translation of a memory window. The peer may access local memory 2027 * through the window starting at the address, up to the size. The address 2028 * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2029 * must be aligned to the size alignment specified by ntb_mw_get_range(). 2030 * 2031 * Return: Zero on success, otherwise an error number. 2032 */ 2033int 2034ntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2035 size_t size) 2036{ 2037 struct ntb_pci_bar_info *bar; 2038 uint64_t base, limit, reg_val; 2039 size_t bar_size, mw_size; 2040 uint32_t base_reg, xlat_reg, limit_reg; 2041 enum ntb_bar bar_num; 2042 2043 if (idx >= ntb_mw_count(ntb)) 2044 return (EINVAL); 2045 2046 bar_num = ntb_mw_to_bar(ntb, idx); 2047 bar = &ntb->bar_info[bar_num]; 2048 2049 bar_size = bar->size; 2050 if (idx == ntb->b2b_mw_idx) 2051 mw_size = bar_size - ntb->b2b_off; 2052 else 2053 mw_size = bar_size; 2054 2055 /* Hardware requires that addr is aligned to bar size */ 2056 if ((addr & (bar_size - 1)) != 0) 2057 return (EINVAL); 2058 2059 if (size > mw_size) 2060 return (EINVAL); 2061 2062 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2063 2064 limit = 0; 2065 if (bar_is_64bit(ntb, bar_num)) { 2066 base = ntb_reg_read(8, base_reg); 2067 2068 if (limit_reg != 0 && size != mw_size) 2069 limit = base + size; 2070 2071 /* Set and verify translation address */ 2072 ntb_reg_write(8, xlat_reg, addr); 2073 reg_val = ntb_reg_read(8, xlat_reg); 2074 if (reg_val != addr) { 2075 ntb_reg_write(8, xlat_reg, 0); 2076 return (EIO); 2077 } 2078 2079 /* Set and verify the limit */ 2080 ntb_reg_write(8, limit_reg, limit); 2081 reg_val = ntb_reg_read(8, limit_reg); 2082 if (reg_val != limit) { 2083 ntb_reg_write(8, limit_reg, base); 2084 ntb_reg_write(8, xlat_reg, 0); 2085 return (EIO); 2086 } 2087 } else { 2088 /* Configure 32-bit (split) BAR MW */ 2089 2090 if ((addr & ~UINT32_MAX) != 0) 2091 return (EINVAL); 2092 if (((addr + size) & ~UINT32_MAX) != 0) 2093 return (EINVAL); 2094 2095 base = ntb_reg_read(4, base_reg); 2096 2097 if (limit_reg != 0 && size != mw_size) 2098 limit = base + size; 2099 2100 /* Set and verify translation address */ 2101 ntb_reg_write(4, xlat_reg, addr); 2102 reg_val = ntb_reg_read(4, xlat_reg); 2103 if (reg_val != addr) { 2104 ntb_reg_write(4, xlat_reg, 0); 2105 return (EIO); 2106 } 2107 2108 /* Set and verify the limit */ 2109 ntb_reg_write(4, limit_reg, limit); 2110 reg_val = ntb_reg_read(4, limit_reg); 2111 if (reg_val != limit) { 2112 ntb_reg_write(4, limit_reg, base); 2113 ntb_reg_write(4, xlat_reg, 0); 2114 return (EIO); 2115 } 2116 } 2117 return (0); 2118} 2119 2120/* 2121 * ntb_mw_clear_trans() - clear the translation of a memory window 2122 * @ntb: NTB device context 2123 * @idx: Memory window number 2124 * 2125 * Clear the translation of a memory window. The peer may no longer access 2126 * local memory through the window. 2127 * 2128 * Return: Zero on success, otherwise an error number. 2129 */ 2130int 2131ntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 2132{ 2133 2134 return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 2135} 2136 2137/** 2138 * ntb_peer_db_set() - Set the doorbell on the secondary/external side 2139 * @ntb: pointer to ntb_softc instance 2140 * @bit: doorbell bits to ring 2141 * 2142 * This function allows triggering of a doorbell on the secondary/external 2143 * side that will initiate an interrupt on the remote host 2144 */ 2145void 2146ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 2147{ 2148 2149 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2150 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit); 2151 return; 2152 } 2153 2154 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2155} 2156 2157/* 2158 * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 2159 * as well as the size of the register (via *sz_out). 2160 * 2161 * This function allows a caller using I/OAT DMA to chain the remote doorbell 2162 * ring to its memory window write. 2163 * 2164 * Note that writing the peer doorbell via a memory window will *not* generate 2165 * an interrupt on the remote host; that must be done seperately. 2166 */ 2167bus_addr_t 2168ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 2169{ 2170 struct ntb_pci_bar_info *bar; 2171 uint64_t regoff; 2172 2173 KASSERT(sz_out != NULL, ("must be non-NULL")); 2174 2175 if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2176 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 2177 regoff = ntb->peer_reg->db_bell; 2178 } else { 2179 KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) || 2180 (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1), 2181 ("mw_count invalid after setup")); 2182 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 2183 ("invalid b2b idx")); 2184 2185 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 2186 regoff = XEON_SHADOW_PDOORBELL_OFFSET; 2187 } 2188 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 2189 2190 *sz_out = ntb->reg->db_size; 2191 /* HACK: Specific to current x86 bus implementation. */ 2192 return ((uint64_t)bar->pci_bus_handle + regoff); 2193} 2194 2195/* 2196 * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 2197 * @ntb: NTB device context 2198 * 2199 * Hardware may support different number or arrangement of doorbell bits. 2200 * 2201 * Return: A mask of doorbell bits supported by the ntb. 2202 */ 2203uint64_t 2204ntb_db_valid_mask(struct ntb_softc *ntb) 2205{ 2206 2207 return (ntb->db_valid_mask); 2208} 2209 2210/* 2211 * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 2212 * @ntb: NTB device context 2213 * @vector: Doorbell vector number 2214 * 2215 * Each interrupt vector may have a different number or arrangement of bits. 2216 * 2217 * Return: A mask of doorbell bits serviced by a vector. 2218 */ 2219uint64_t 2220ntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 2221{ 2222 2223 if (vector > ntb->db_vec_count) 2224 return (0); 2225 return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 2226} 2227 2228/** 2229 * ntb_link_is_up() - get the current ntb link state 2230 * @ntb: NTB device context 2231 * @speed: OUT - The link speed expressed as PCIe generation number 2232 * @width: OUT - The link width expressed as the number of PCIe lanes 2233 * 2234 * RETURNS: true or false based on the hardware link state 2235 */ 2236bool 2237ntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 2238 enum ntb_width *width) 2239{ 2240 2241 if (speed != NULL) 2242 *speed = ntb_link_sta_speed(ntb); 2243 if (width != NULL) 2244 *width = ntb_link_sta_width(ntb); 2245 return (link_is_up(ntb)); 2246} 2247 2248static void 2249save_bar_parameters(struct ntb_pci_bar_info *bar) 2250{ 2251 2252 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2253 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2254 bar->pbase = rman_get_start(bar->pci_resource); 2255 bar->size = rman_get_size(bar->pci_resource); 2256 bar->vbase = rman_get_virtual(bar->pci_resource); 2257} 2258 2259device_t 2260ntb_get_device(struct ntb_softc *ntb) 2261{ 2262 2263 return (ntb->device); 2264} 2265 2266/* Export HW-specific errata information. */ 2267bool 2268ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 2269{ 2270 2271 return (HAS_FEATURE(feature)); 2272} 2273