ntb_hw_intel.c revision 289607
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3289542Scem * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289607 2015-10-20 01:45:38Z cem $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35250079Scarl#include <sys/malloc.h> 36250079Scarl#include <sys/module.h> 37250079Scarl#include <sys/queue.h> 38250079Scarl#include <sys/rman.h> 39289207Scem#include <sys/sysctl.h> 40250079Scarl#include <vm/vm.h> 41250079Scarl#include <vm/pmap.h> 42250079Scarl#include <machine/bus.h> 43250079Scarl#include <machine/pmap.h> 44250079Scarl#include <machine/resource.h> 45250079Scarl#include <dev/pci/pcireg.h> 46250079Scarl#include <dev/pci/pcivar.h> 47250079Scarl 48250079Scarl#include "ntb_regs.h" 49250079Scarl#include "ntb_hw.h" 50250079Scarl 51250079Scarl/* 52250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 53250079Scarl * allows you to connect two systems using a PCI-e link. 54250079Scarl * 55250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 56250079Scarl * you to send and recieve interrupts, map the memory windows and send and 57250079Scarl * receive messages in the scratch-pad registers. 58250079Scarl * 59250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 60250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 61250079Scarl */ 62250079Scarl 63289538Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT) 64250079Scarl 65289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 66289539Scem#define SOC_LINK_RECOVERY_TIME 500 /* ms */ 67250079Scarl 68250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 69250079Scarl 70250079Scarlenum ntb_device_type { 71250079Scarl NTB_XEON, 72250079Scarl NTB_SOC 73250079Scarl}; 74250079Scarl 75289539Scemenum ntb_bar { 76289539Scem NTB_CONFIG_BAR = 0, 77289539Scem NTB_B2B_BAR_1, 78289539Scem NTB_B2B_BAR_2, 79289539Scem NTB_B2B_BAR_3, 80289539Scem NTB_MAX_BARS 81289539Scem}; 82289539Scem 83255274Scarl/* Device features and workarounds */ 84255274Scarl#define HAS_FEATURE(feature) \ 85255274Scarl ((ntb->features & (feature)) != 0) 86255274Scarl 87250079Scarlstruct ntb_hw_info { 88250079Scarl uint32_t device_id; 89255274Scarl const char *desc; 90250079Scarl enum ntb_device_type type; 91289397Scem uint32_t features; 92250079Scarl}; 93250079Scarl 94250079Scarlstruct ntb_pci_bar_info { 95250079Scarl bus_space_tag_t pci_bus_tag; 96250079Scarl bus_space_handle_t pci_bus_handle; 97250079Scarl int pci_resource_id; 98250079Scarl struct resource *pci_resource; 99250079Scarl vm_paddr_t pbase; 100250079Scarl void *vbase; 101250079Scarl u_long size; 102289543Scem 103289543Scem /* Configuration register offsets */ 104289543Scem uint32_t psz_off; 105289543Scem uint32_t ssz_off; 106289543Scem uint32_t pbarxlat_off; 107250079Scarl}; 108250079Scarl 109250079Scarlstruct ntb_int_info { 110250079Scarl struct resource *res; 111250079Scarl int rid; 112250079Scarl void *tag; 113250079Scarl}; 114250079Scarl 115289546Scemstruct ntb_vec { 116250079Scarl struct ntb_softc *ntb; 117289546Scem uint32_t num; 118250079Scarl}; 119250079Scarl 120289542Scemstruct ntb_reg { 121289542Scem uint32_t ntb_ctl; 122289542Scem uint32_t lnk_sta; 123289542Scem uint8_t db_size; 124289542Scem unsigned mw_bar[NTB_MAX_BARS]; 125289542Scem}; 126289542Scem 127289542Scemstruct ntb_alt_reg { 128289542Scem uint32_t db_bell; 129289542Scem uint32_t db_mask; 130289542Scem uint32_t spad; 131289542Scem}; 132289542Scem 133289542Scemstruct ntb_xlat_reg { 134289546Scem uint32_t bar0_base; 135289546Scem uint32_t bar2_base; 136289546Scem uint32_t bar4_base; 137289546Scem uint32_t bar5_base; 138289546Scem 139289546Scem uint32_t bar2_xlat; 140289546Scem uint32_t bar4_xlat; 141289546Scem uint32_t bar5_xlat; 142289546Scem 143289546Scem uint32_t bar2_limit; 144289546Scem uint32_t bar4_limit; 145289546Scem uint32_t bar5_limit; 146289542Scem}; 147289542Scem 148289542Scemstruct ntb_b2b_addr { 149289542Scem uint64_t bar0_addr; 150289542Scem uint64_t bar2_addr64; 151289542Scem uint64_t bar4_addr64; 152289542Scem uint64_t bar4_addr32; 153289542Scem uint64_t bar5_addr32; 154289542Scem}; 155289542Scem 156250079Scarlstruct ntb_softc { 157250079Scarl device_t device; 158250079Scarl enum ntb_device_type type; 159255274Scarl uint64_t features; 160250079Scarl 161250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 162250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 163250079Scarl uint32_t allocated_interrupts; 164250079Scarl 165250079Scarl struct callout heartbeat_timer; 166250079Scarl struct callout lr_timer; 167250079Scarl 168289546Scem void *ntb_ctx; 169289546Scem const struct ntb_ctx_ops *ctx_ops; 170289546Scem struct ntb_vec *msix_vec; 171289546Scem#define CTX_LOCK(sc) mtx_lock_spin(&(sc)->ctx_lock) 172289546Scem#define CTX_UNLOCK(sc) mtx_unlock_spin(&(sc)->ctx_lock) 173289546Scem#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 174289546Scem struct mtx ctx_lock; 175250079Scarl 176289348Scem uint32_t ppd; 177250079Scarl uint8_t conn_type; 178250079Scarl uint8_t dev_type; 179250079Scarl uint8_t link_width; 180250079Scarl uint8_t link_speed; 181289539Scem 182289542Scem /* Offset of peer bar0 in B2B BAR */ 183289542Scem uint64_t b2b_off; 184289542Scem /* Memory window used to access peer bar0 */ 185289543Scem#define B2B_MW_DISABLED UINT8_MAX 186289542Scem uint8_t b2b_mw_idx; 187289542Scem 188289539Scem uint8_t mw_count; 189289539Scem uint8_t spad_count; 190289539Scem uint8_t db_count; 191289539Scem uint8_t db_vec_count; 192289539Scem uint8_t db_vec_shift; 193289542Scem 194289546Scem /* Protects local db_mask. */ 195289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 196289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 197289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 198289542Scem struct mtx db_mask_lock; 199289542Scem 200289546Scem uint32_t ntb_ctl; 201289546Scem uint32_t lnk_sta; 202289542Scem 203289542Scem uint64_t db_valid_mask; 204289542Scem uint64_t db_link_mask; 205289546Scem uint64_t db_mask; 206289542Scem 207289542Scem int last_ts; /* ticks @ last irq */ 208289542Scem 209289542Scem const struct ntb_reg *reg; 210289542Scem const struct ntb_alt_reg *self_reg; 211289542Scem const struct ntb_alt_reg *peer_reg; 212289542Scem const struct ntb_xlat_reg *xlat_reg; 213250079Scarl}; 214250079Scarl 215289234Scem#ifdef __i386__ 216289234Scemstatic __inline uint64_t 217289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 218289234Scem bus_size_t offset) 219289234Scem{ 220289234Scem 221289234Scem return (bus_space_read_4(tag, handle, offset) | 222289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 223289234Scem} 224289234Scem 225289234Scemstatic __inline void 226289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 227289234Scem bus_size_t offset, uint64_t val) 228289234Scem{ 229289234Scem 230289234Scem bus_space_write_4(tag, handle, offset, val); 231289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 232289234Scem} 233289234Scem#endif 234289234Scem 235255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 236255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 237255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 238255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 239255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 240255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 241255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 242250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 243255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 244289397Scem#define ntb_mw_read(SIZE, offset) \ 245289542Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 246255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 247289542Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 248289397Scem offset, val) 249250079Scarl 250250079Scarlstatic int ntb_probe(device_t device); 251250079Scarlstatic int ntb_attach(device_t device); 252250079Scarlstatic int ntb_detach(device_t device); 253289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 254289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 255289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 256289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 257255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 258289541Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *); 259255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 260255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 261255272Scarl struct ntb_pci_bar_info *bar); 262250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 263289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 264289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 265289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 266289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 267250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 268289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 269289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 270289546Scemstatic void ndev_vec_isr(void *arg); 271289546Scemstatic void ndev_irq_isr(void *arg); 272289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 273289546Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t val); 274289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 275289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 276250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 277289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 278289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 279289348Scemstatic int ntb_detect_soc(struct ntb_softc *ntb); 280289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 281289542Scemstatic int ntb_soc_init_dev(struct ntb_softc *ntb); 282289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 283255279Scarlstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 284289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 285289543Scem enum ntb_bar regbar); 286289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 287289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 288289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 289289543Scem enum ntb_bar idx); 290289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 291289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 292289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 293289546Scemstatic inline bool soc_link_is_err(struct ntb_softc *ntb); 294289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 295289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 296289542Scemstatic void soc_link_hb(void *arg); 297289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 298250079Scarlstatic void recover_soc_link(void *arg); 299289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb); 300255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 301250079Scarl 302250079Scarlstatic struct ntb_hw_info pci_ids[] = { 303255274Scarl { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 304289233Scem 305289233Scem /* XXX: PS/SS IDs left out until they are supported. */ 306289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 307289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 308289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 309289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 310289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 311289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 312289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 313289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 314289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 315289538Scem NTB_SB01BASE_LOCKUP }, 316289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 317289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 318289538Scem NTB_SB01BASE_LOCKUP }, 319289233Scem 320255274Scarl { 0x00000000, NULL, NTB_SOC, 0 } 321250079Scarl}; 322250079Scarl 323289542Scemstatic const struct ntb_reg soc_reg = { 324289542Scem .ntb_ctl = SOC_NTBCNTL_OFFSET, 325289542Scem .lnk_sta = SOC_LINK_STATUS_OFFSET, 326289542Scem .db_size = sizeof(uint64_t), 327289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 328289542Scem}; 329289542Scem 330289607Scemstatic const struct ntb_alt_reg soc_pri_reg = { 331289607Scem .db_bell = SOC_PDOORBELL_OFFSET, 332289607Scem .db_mask = SOC_PDBMSK_OFFSET, 333289607Scem .spad = SOC_SPAD_OFFSET, 334289607Scem}; 335289607Scem 336289542Scemstatic const struct ntb_alt_reg soc_b2b_reg = { 337289542Scem .db_bell = SOC_B2B_DOORBELL_OFFSET, 338289542Scem .spad = SOC_B2B_SPAD_OFFSET, 339289542Scem}; 340289542Scem 341289542Scemstatic const struct ntb_xlat_reg soc_sec_xlat = { 342289542Scem#if 0 343289542Scem /* "FIXME" says the Linux driver. */ 344289542Scem .bar0_base = SOC_SBAR0BASE_OFFSET, 345289546Scem .bar2_base = SOC_SBAR2BASE_OFFSET, 346289546Scem .bar4_base = SOC_SBAR4BASE_OFFSET, 347289546Scem 348289542Scem .bar2_limit = SOC_SBAR2LMT_OFFSET, 349289546Scem .bar4_limit = SOC_SBAR4LMT_OFFSET, 350289542Scem#endif 351289546Scem 352289542Scem .bar2_xlat = SOC_SBAR2XLAT_OFFSET, 353289546Scem .bar4_xlat = SOC_SBAR4XLAT_OFFSET, 354289542Scem}; 355289542Scem 356289542Scemstatic const struct ntb_reg xeon_reg = { 357289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 358289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 359289542Scem .db_size = sizeof(uint16_t), 360289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 361289542Scem}; 362289542Scem 363289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 364289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 365289607Scem .db_mask = XEON_PDBMSK_OFFSET, 366289607Scem .spad = XEON_SPAD_OFFSET, 367289607Scem}; 368289607Scem 369289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 370289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 371289542Scem .spad = XEON_B2B_SPAD_OFFSET, 372289542Scem}; 373289542Scem 374289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 375289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 376289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 377289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 378289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 379289546Scem 380289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 381289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 382289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 383289546Scem 384289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 385289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 386289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 387289542Scem}; 388289542Scem 389289542Scemstatic const struct ntb_b2b_addr xeon_b2b_usd_addr = { 390289542Scem .bar0_addr = XEON_B2B_BAR0_USD_ADDR, 391289542Scem .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, 392289542Scem .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, 393289542Scem .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, 394289542Scem .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, 395289542Scem}; 396289542Scem 397289542Scemstatic const struct ntb_b2b_addr xeon_b2b_dsd_addr = { 398289542Scem .bar0_addr = XEON_B2B_BAR0_DSD_ADDR, 399289542Scem .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, 400289542Scem .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, 401289542Scem .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, 402289542Scem .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, 403289542Scem}; 404289542Scem 405250079Scarl/* 406250079Scarl * OS <-> Driver interface structures 407250079Scarl */ 408250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 409250079Scarl 410250079Scarlstatic device_method_t ntb_pci_methods[] = { 411250079Scarl /* Device interface */ 412250079Scarl DEVMETHOD(device_probe, ntb_probe), 413250079Scarl DEVMETHOD(device_attach, ntb_attach), 414250079Scarl DEVMETHOD(device_detach, ntb_detach), 415250079Scarl DEVMETHOD_END 416250079Scarl}; 417250079Scarl 418250079Scarlstatic driver_t ntb_pci_driver = { 419250079Scarl "ntb_hw", 420250079Scarl ntb_pci_methods, 421250079Scarl sizeof(struct ntb_softc), 422250079Scarl}; 423250079Scarl 424250079Scarlstatic devclass_t ntb_devclass; 425250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 426250079ScarlMODULE_VERSION(ntb_hw, 1); 427250079Scarl 428289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 429289207Scem 430250079Scarl/* 431250079Scarl * OS <-> Driver linkage functions 432250079Scarl */ 433250079Scarlstatic int 434250079Scarlntb_probe(device_t device) 435250079Scarl{ 436289209Scem struct ntb_hw_info *p; 437250079Scarl 438289209Scem p = ntb_get_device_info(pci_get_devid(device)); 439289209Scem if (p == NULL) 440250079Scarl return (ENXIO); 441289209Scem 442289209Scem device_set_desc(device, p->desc); 443289209Scem return (0); 444250079Scarl} 445250079Scarl 446250079Scarlstatic int 447250079Scarlntb_attach(device_t device) 448250079Scarl{ 449289209Scem struct ntb_softc *ntb; 450289209Scem struct ntb_hw_info *p; 451250079Scarl int error; 452250079Scarl 453289209Scem ntb = DEVICE2SOFTC(device); 454289209Scem p = ntb_get_device_info(pci_get_devid(device)); 455289209Scem 456250079Scarl ntb->device = device; 457250079Scarl ntb->type = p->type; 458255274Scarl ntb->features = p->features; 459289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 460250079Scarl 461250079Scarl /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 462283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 463283291Sjkim callout_init(&ntb->lr_timer, 1); 464289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 465289546Scem mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN); 466250079Scarl 467289348Scem if (ntb->type == NTB_SOC) 468289348Scem error = ntb_detect_soc(ntb); 469289348Scem else 470289348Scem error = ntb_detect_xeon(ntb); 471289348Scem if (error) 472289348Scem goto out; 473289348Scem 474289397Scem ntb_detect_max_mw(ntb); 475289396Scem 476289209Scem error = ntb_map_pci_bars(ntb); 477289209Scem if (error) 478289209Scem goto out; 479289272Scem if (ntb->type == NTB_SOC) 480289542Scem error = ntb_soc_init_dev(ntb); 481289272Scem else 482289542Scem error = ntb_xeon_init_dev(ntb); 483289209Scem if (error) 484289209Scem goto out; 485289540Scem error = ntb_init_isr(ntb); 486289209Scem if (error) 487289209Scem goto out; 488250079Scarl 489250079Scarl pci_enable_busmaster(ntb->device); 490250079Scarl 491289209Scemout: 492289209Scem if (error != 0) 493289209Scem ntb_detach(device); 494250079Scarl return (error); 495250079Scarl} 496250079Scarl 497250079Scarlstatic int 498250079Scarlntb_detach(device_t device) 499250079Scarl{ 500289209Scem struct ntb_softc *ntb; 501250079Scarl 502289209Scem ntb = DEVICE2SOFTC(device); 503289542Scem 504289542Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 505250079Scarl callout_drain(&ntb->heartbeat_timer); 506250079Scarl callout_drain(&ntb->lr_timer); 507289272Scem if (ntb->type == NTB_XEON) 508289272Scem ntb_teardown_xeon(ntb); 509250079Scarl ntb_teardown_interrupts(ntb); 510289397Scem 511289542Scem mtx_destroy(&ntb->db_mask_lock); 512289546Scem mtx_destroy(&ntb->ctx_lock); 513289542Scem 514289397Scem /* 515289397Scem * Redetect total MWs so we unmap properly -- in case we lowered the 516289397Scem * maximum to work around Xeon errata. 517289397Scem */ 518289397Scem ntb_detect_max_mw(ntb); 519250079Scarl ntb_unmap_pci_bar(ntb); 520250079Scarl 521250079Scarl return (0); 522250079Scarl} 523250079Scarl 524289542Scem/* 525289542Scem * Driver internal routines 526289542Scem */ 527289539Scemstatic inline enum ntb_bar 528289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 529289539Scem{ 530289539Scem 531289543Scem KASSERT(mw < ntb->mw_count || 532289543Scem (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx), 533289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 534289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 535289539Scem 536289542Scem return (ntb->reg->mw_bar[mw]); 537289539Scem} 538289539Scem 539289546Scemstatic inline bool 540289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 541289546Scem{ 542289546Scem /* XXX This assertion could be stronger. */ 543289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 544289546Scem return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 545289546Scem} 546289546Scem 547289546Scemstatic inline void 548289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 549289546Scem uint32_t *xlat, uint32_t *lmt) 550289546Scem{ 551289546Scem uint32_t basev, lmtv, xlatv; 552289546Scem 553289546Scem switch (bar) { 554289546Scem case NTB_B2B_BAR_1: 555289546Scem basev = ntb->xlat_reg->bar2_base; 556289546Scem lmtv = ntb->xlat_reg->bar2_limit; 557289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 558289546Scem break; 559289546Scem case NTB_B2B_BAR_2: 560289546Scem basev = ntb->xlat_reg->bar4_base; 561289546Scem lmtv = ntb->xlat_reg->bar4_limit; 562289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 563289546Scem break; 564289546Scem case NTB_B2B_BAR_3: 565289546Scem basev = ntb->xlat_reg->bar5_base; 566289546Scem lmtv = ntb->xlat_reg->bar5_limit; 567289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 568289546Scem break; 569289546Scem default: 570289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 571289546Scem ("bad bar")); 572289546Scem basev = lmtv = xlatv = 0; 573289546Scem break; 574289546Scem } 575289546Scem 576289546Scem if (base != NULL) 577289546Scem *base = basev; 578289546Scem if (xlat != NULL) 579289546Scem *xlat = xlatv; 580289546Scem if (lmt != NULL) 581289546Scem *lmt = lmtv; 582289546Scem} 583289546Scem 584250079Scarlstatic int 585255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 586250079Scarl{ 587255272Scarl int rc; 588250079Scarl 589250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 590289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 591255272Scarl if (rc != 0) 592289541Scem goto out; 593255272Scarl 594289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 595289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 596255272Scarl if (rc != 0) 597289541Scem goto out; 598289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 599289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 600289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 601255272Scarl 602289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 603289543Scem /* XXX Are shared MW B2Bs write-combining? */ 604289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 605289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 606255279Scarl else 607289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 608289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 609289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 610289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 611289543Scem 612289397Scem if (!HAS_FEATURE(NTB_SPLIT_BAR)) 613289541Scem goto out; 614289397Scem 615289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 616289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 617289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 618289397Scem else 619289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 620289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 621289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 622289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 623250079Scarl 624289541Scemout: 625289209Scem if (rc != 0) 626255272Scarl device_printf(ntb->device, 627255272Scarl "unable to allocate pci resource\n"); 628255272Scarl return (rc); 629255272Scarl} 630255272Scarl 631289541Scemstatic void 632289541Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 633289541Scem{ 634289541Scem 635289541Scem device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n", 636289541Scem bar->size, bar->vbase, (void *)(bar->pbase)); 637289541Scem} 638289541Scem 639255272Scarlstatic int 640255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 641255272Scarl{ 642255272Scarl 643255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 644289209Scem &bar->pci_resource_id, RF_ACTIVE); 645255272Scarl if (bar->pci_resource == NULL) 646255272Scarl return (ENXIO); 647289209Scem 648289209Scem save_bar_parameters(bar); 649289541Scem print_map_success(ntb, bar); 650289209Scem return (0); 651255272Scarl} 652255272Scarl 653255272Scarlstatic int 654255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 655255272Scarl{ 656255272Scarl int rc; 657255276Scarl uint8_t bar_size_bits = 0; 658255272Scarl 659289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 660289209Scem &bar->pci_resource_id, RF_ACTIVE); 661250079Scarl 662255272Scarl if (bar->pci_resource == NULL) 663255272Scarl return (ENXIO); 664255276Scarl 665289209Scem save_bar_parameters(bar); 666289209Scem /* 667289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 668289209Scem * hardware issue. To work around this, query the size it should be 669289209Scem * configured to by the device and modify the resource to correspond to 670289209Scem * this new size. The BIOS on systems with this problem is required to 671289209Scem * provide enough address space to allow the driver to make this change 672289209Scem * safely. 673289209Scem * 674289209Scem * Ideally I could have just specified the size when I allocated the 675289209Scem * resource like: 676289209Scem * bus_alloc_resource(ntb->device, 677289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 678289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 679289209Scem * but the PCI driver does not honor the size in this call, so we have 680289209Scem * to modify it after the fact. 681289209Scem */ 682289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 683289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 684289209Scem bar_size_bits = pci_read_config(ntb->device, 685289209Scem XEON_PBAR23SZ_OFFSET, 1); 686289209Scem else 687289209Scem bar_size_bits = pci_read_config(ntb->device, 688289209Scem XEON_PBAR45SZ_OFFSET, 1); 689289209Scem 690289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 691289209Scem bar->pci_resource, bar->pbase, 692289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 693255272Scarl if (rc != 0) { 694289209Scem device_printf(ntb->device, 695289209Scem "unable to resize bar\n"); 696255272Scarl return (rc); 697250079Scarl } 698289209Scem 699289209Scem save_bar_parameters(bar); 700250079Scarl } 701289209Scem 702289209Scem /* Mark bar region as write combining to improve performance. */ 703289209Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 704289209Scem VM_MEMATTR_WRITE_COMBINING); 705289209Scem if (rc != 0) { 706289209Scem device_printf(ntb->device, 707289209Scem "unable to mark bar as WRITE_COMBINING\n"); 708289209Scem return (rc); 709289209Scem } 710289541Scem print_map_success(ntb, bar); 711250079Scarl return (0); 712250079Scarl} 713250079Scarl 714250079Scarlstatic void 715250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 716250079Scarl{ 717250079Scarl struct ntb_pci_bar_info *current_bar; 718250079Scarl int i; 719250079Scarl 720289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 721250079Scarl current_bar = &ntb->bar_info[i]; 722250079Scarl if (current_bar->pci_resource != NULL) 723250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 724250079Scarl current_bar->pci_resource_id, 725250079Scarl current_bar->pci_resource); 726250079Scarl } 727250079Scarl} 728250079Scarl 729250079Scarlstatic int 730289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 731250079Scarl{ 732289342Scem uint32_t i; 733289342Scem int rc; 734289342Scem 735289342Scem for (i = 0; i < num_vectors; i++) { 736289342Scem ntb->int_info[i].rid = i + 1; 737289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 738289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 739289342Scem if (ntb->int_info[i].res == NULL) { 740289342Scem device_printf(ntb->device, 741289342Scem "bus_alloc_resource failed\n"); 742289342Scem return (ENOMEM); 743289342Scem } 744289342Scem ntb->int_info[i].tag = NULL; 745289342Scem ntb->allocated_interrupts++; 746289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 747289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 748289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 749289342Scem if (rc != 0) { 750289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 751289342Scem return (ENXIO); 752289342Scem } 753289342Scem } 754289342Scem return (0); 755289342Scem} 756289342Scem 757289344Scem/* 758289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 759289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 760289344Scem * should be okay. This tunable should enable us to test that hypothesis 761289344Scem * when someone gets their hands on some Xeon hardware. 762289344Scem */ 763289344Scemstatic int ntb_force_remap_mode; 764289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 765289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 766289344Scem " to a smaller number of ithreads, even if the desired number are " 767289344Scem "available"); 768289344Scem 769289344Scem/* 770289344Scem * In case it is NOT ok, give consumers an abort button. 771289344Scem */ 772289344Scemstatic int ntb_prefer_intx; 773289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 774289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 775289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 776289344Scem "behavior)"); 777289344Scem 778289344Scem/* 779289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 780289344Scem * round-robin fashion. 781289344Scem */ 782289342Scemstatic int 783289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 784289344Scem{ 785289344Scem u_int *vectors; 786289344Scem uint32_t i; 787289344Scem int rc; 788289344Scem 789289344Scem if (ntb_prefer_intx != 0) 790289344Scem return (ENXIO); 791289344Scem 792289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 793289344Scem 794289344Scem for (i = 0; i < desired; i++) 795289344Scem vectors[i] = (i % avail) + 1; 796289344Scem 797289344Scem rc = pci_remap_msix(dev, desired, vectors); 798289344Scem free(vectors, M_NTB); 799289344Scem return (rc); 800289344Scem} 801289344Scem 802289344Scemstatic int 803289540Scemntb_init_isr(struct ntb_softc *ntb) 804289342Scem{ 805289344Scem uint32_t desired_vectors, num_vectors; 806289342Scem int rc; 807250079Scarl 808250079Scarl ntb->allocated_interrupts = 0; 809289542Scem ntb->last_ts = ticks; 810289347Scem 811250079Scarl /* 812289546Scem * Mask all doorbell interrupts. 813250079Scarl */ 814289546Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 815250079Scarl 816289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 817289539Scem ntb->db_count); 818289344Scem if (desired_vectors >= 1) { 819289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 820250079Scarl 821289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 822289344Scem num_vectors == desired_vectors) 823289344Scem num_vectors--; 824289344Scem 825289344Scem if (rc == 0 && num_vectors < desired_vectors) { 826289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 827289344Scem num_vectors); 828289344Scem if (rc == 0) 829289344Scem num_vectors = desired_vectors; 830289344Scem else 831289344Scem pci_release_msi(ntb->device); 832289344Scem } 833289344Scem if (rc != 0) 834289344Scem num_vectors = 1; 835289344Scem } else 836289344Scem num_vectors = 1; 837289344Scem 838289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 839289539Scem ntb->db_vec_count = 1; 840289539Scem ntb->db_vec_shift = ntb->db_count; 841289539Scem rc = ntb_setup_legacy_interrupt(ntb); 842289539Scem } else { 843289546Scem ntb_create_msix_vec(ntb, num_vectors); 844289540Scem rc = ntb_setup_msix(ntb, num_vectors); 845289539Scem } 846289539Scem if (rc != 0) { 847289539Scem device_printf(ntb->device, 848289539Scem "Error allocating interrupts: %d\n", rc); 849289546Scem ntb_free_msix_vec(ntb); 850289396Scem } 851289396Scem 852289342Scem return (rc); 853289342Scem} 854289342Scem 855289342Scemstatic int 856289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 857289342Scem{ 858289342Scem int rc; 859289342Scem 860289342Scem ntb->int_info[0].rid = 0; 861289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 862289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 863289342Scem if (ntb->int_info[0].res == NULL) { 864289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 865289342Scem return (ENOMEM); 866250079Scarl } 867250079Scarl 868289342Scem ntb->int_info[0].tag = NULL; 869289342Scem ntb->allocated_interrupts = 1; 870289342Scem 871289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 872289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 873289342Scem ntb, &ntb->int_info[0].tag); 874289342Scem if (rc != 0) { 875289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 876289342Scem return (ENXIO); 877289342Scem } 878289342Scem 879250079Scarl return (0); 880250079Scarl} 881250079Scarl 882250079Scarlstatic void 883250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 884250079Scarl{ 885250079Scarl struct ntb_int_info *current_int; 886250079Scarl int i; 887250079Scarl 888289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 889250079Scarl current_int = &ntb->int_info[i]; 890250079Scarl if (current_int->tag != NULL) 891250079Scarl bus_teardown_intr(ntb->device, current_int->res, 892250079Scarl current_int->tag); 893250079Scarl 894250079Scarl if (current_int->res != NULL) 895250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 896250079Scarl rman_get_rid(current_int->res), current_int->res); 897250079Scarl } 898250079Scarl 899289546Scem ntb_free_msix_vec(ntb); 900250079Scarl pci_release_msi(ntb->device); 901250079Scarl} 902250079Scarl 903289347Scem/* 904289347Scem * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon. Abstract it 905289347Scem * out to make code clearer. 906289347Scem */ 907289539Scemstatic inline uint64_t 908289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 909289347Scem{ 910289347Scem 911289347Scem if (ntb->type == NTB_SOC) 912289347Scem return (ntb_reg_read(8, regoff)); 913289347Scem 914289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 915289347Scem 916289347Scem return (ntb_reg_read(2, regoff)); 917289347Scem} 918289347Scem 919289539Scemstatic inline void 920289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 921289347Scem{ 922289347Scem 923289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 924289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 925289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 926289542Scem (uintmax_t)ntb->db_valid_mask)); 927289542Scem 928289607Scem if (regoff == ntb->self_reg->db_mask) 929289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 930289542Scem 931289347Scem if (ntb->type == NTB_SOC) { 932289347Scem ntb_reg_write(8, regoff, val); 933289347Scem return; 934289347Scem } 935289347Scem 936289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 937289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 938289347Scem} 939289347Scem 940289546Scemvoid 941289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 942289542Scem{ 943289542Scem 944289546Scem DB_MASK_LOCK(ntb); 945289542Scem ntb->db_mask |= bits; 946289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 947289546Scem DB_MASK_UNLOCK(ntb); 948289542Scem} 949289542Scem 950289546Scemvoid 951289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 952289542Scem{ 953289542Scem 954289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 955289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 956289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 957289542Scem (uintmax_t)ntb->db_valid_mask)); 958289542Scem 959289546Scem DB_MASK_LOCK(ntb); 960289542Scem ntb->db_mask &= ~bits; 961289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 962289546Scem DB_MASK_UNLOCK(ntb); 963289542Scem} 964289542Scem 965289546Scemuint64_t 966289546Scemntb_db_read(struct ntb_softc *ntb) 967289281Scem{ 968289281Scem 969289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 970289281Scem} 971289281Scem 972289546Scemvoid 973289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 974289281Scem{ 975289281Scem 976289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 977289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 978289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 979289546Scem (uintmax_t)ntb->db_valid_mask)); 980289546Scem 981289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 982289281Scem} 983289281Scem 984289540Scemstatic inline uint64_t 985289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 986250079Scarl{ 987289540Scem uint64_t shift, mask; 988250079Scarl 989289540Scem shift = ntb->db_vec_shift; 990289540Scem mask = (1ull << shift) - 1; 991289540Scem return (mask << (shift * db_vector)); 992250079Scarl} 993250079Scarl 994250079Scarlstatic void 995289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 996250079Scarl{ 997289540Scem uint64_t vec_mask; 998250079Scarl 999289542Scem ntb->last_ts = ticks; 1000289546Scem vec_mask = ntb_vec_mask(ntb, vec); 1001250079Scarl 1002289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1003289546Scem if (ntb_poll_link(ntb)) 1004289546Scem ntb_link_event(ntb); 1005289540Scem } 1006289540Scem 1007289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1008289546Scem ntb_db_event(ntb, vec); 1009289546Scem} 1010250079Scarl 1011289546Scemstatic void 1012289546Scemndev_vec_isr(void *arg) 1013289546Scem{ 1014289546Scem struct ntb_vec *nvec = arg; 1015250079Scarl 1016289546Scem ntb_interrupt(nvec->ntb, nvec->num); 1017250079Scarl} 1018250079Scarl 1019250079Scarlstatic void 1020289546Scemndev_irq_isr(void *arg) 1021250079Scarl{ 1022289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1023289546Scem ntb_interrupt(arg, 0); 1024250079Scarl} 1025250079Scarl 1026250079Scarlstatic int 1027289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1028250079Scarl{ 1029289342Scem uint32_t i; 1030250079Scarl 1031289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1032250079Scarl M_ZERO | M_WAITOK); 1033250079Scarl for (i = 0; i < num_vectors; i++) { 1034289546Scem ntb->msix_vec[i].num = i; 1035289546Scem ntb->msix_vec[i].ntb = ntb; 1036250079Scarl } 1037250079Scarl 1038250079Scarl return (0); 1039250079Scarl} 1040250079Scarl 1041250079Scarlstatic void 1042289546Scemntb_free_msix_vec(struct ntb_softc *ntb) 1043250079Scarl{ 1044250079Scarl 1045289546Scem if (ntb->msix_vec == NULL) 1046289539Scem return; 1047289539Scem 1048289546Scem free(ntb->msix_vec, M_NTB); 1049289546Scem ntb->msix_vec = NULL; 1050250079Scarl} 1051250079Scarl 1052250079Scarlstatic struct ntb_hw_info * 1053250079Scarlntb_get_device_info(uint32_t device_id) 1054250079Scarl{ 1055250079Scarl struct ntb_hw_info *ep = pci_ids; 1056250079Scarl 1057250079Scarl while (ep->device_id) { 1058250079Scarl if (ep->device_id == device_id) 1059250079Scarl return (ep); 1060250079Scarl ++ep; 1061250079Scarl } 1062250079Scarl return (NULL); 1063250079Scarl} 1064250079Scarl 1065289272Scemstatic void 1066289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 1067250079Scarl{ 1068250079Scarl 1069289542Scem ntb_link_disable(ntb); 1070250079Scarl} 1071250079Scarl 1072289397Scemstatic void 1073289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 1074289397Scem{ 1075289397Scem 1076289397Scem if (ntb->type == NTB_SOC) { 1077289539Scem ntb->mw_count = SOC_MW_COUNT; 1078289397Scem return; 1079289397Scem } 1080289397Scem 1081289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1082289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1083289397Scem else 1084289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1085289397Scem} 1086289397Scem 1087250079Scarlstatic int 1088289348Scemntb_detect_xeon(struct ntb_softc *ntb) 1089250079Scarl{ 1090289348Scem uint8_t ppd, conn_type; 1091250079Scarl 1092289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1093289348Scem ntb->ppd = ppd; 1094250079Scarl 1095289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1096289257Scem ntb->dev_type = NTB_DEV_USD; 1097289257Scem else 1098289257Scem ntb->dev_type = NTB_DEV_DSD; 1099289257Scem 1100289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1101289397Scem ntb->features |= NTB_SPLIT_BAR; 1102289397Scem 1103289542Scem /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1104289542Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1105289542Scem ntb->features |= NTB_SDOORBELL_LOCKUP; 1106289542Scem 1107289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1108289348Scem switch (conn_type) { 1109289348Scem case NTB_CONN_B2B: 1110289348Scem ntb->conn_type = conn_type; 1111289348Scem break; 1112289348Scem case NTB_CONN_RP: 1113289348Scem case NTB_CONN_TRANSPARENT: 1114289348Scem default: 1115289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1116289348Scem (unsigned)conn_type); 1117289348Scem return (ENXIO); 1118289348Scem } 1119289348Scem return (0); 1120289348Scem} 1121289348Scem 1122289348Scemstatic int 1123289348Scemntb_detect_soc(struct ntb_softc *ntb) 1124289348Scem{ 1125289348Scem uint32_t ppd, conn_type; 1126289348Scem 1127289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1128289348Scem ntb->ppd = ppd; 1129289348Scem 1130289348Scem if ((ppd & SOC_PPD_DEV_TYPE) != 0) 1131289348Scem ntb->dev_type = NTB_DEV_DSD; 1132289348Scem else 1133289348Scem ntb->dev_type = NTB_DEV_USD; 1134289348Scem 1135289348Scem conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8; 1136289348Scem switch (conn_type) { 1137289348Scem case NTB_CONN_B2B: 1138289348Scem ntb->conn_type = conn_type; 1139289348Scem break; 1140289348Scem default: 1141289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1142289348Scem return (ENXIO); 1143289348Scem } 1144289348Scem return (0); 1145289348Scem} 1146289348Scem 1147289348Scemstatic int 1148289542Scemntb_xeon_init_dev(struct ntb_softc *ntb) 1149289348Scem{ 1150289542Scem int rc; 1151289348Scem 1152289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1153289542Scem ntb->db_count = XEON_DB_COUNT; 1154289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1155289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1156289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1157289257Scem 1158289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1159250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1160289348Scem ntb->conn_type); 1161250079Scarl return (ENXIO); 1162250079Scarl } 1163250079Scarl 1164289542Scem ntb->reg = &xeon_reg; 1165289607Scem ntb->self_reg = &xeon_pri_reg; 1166289542Scem ntb->peer_reg = &xeon_b2b_reg; 1167289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1168289542Scem 1169289208Scem /* 1170289208Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1171289208Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1172289208Scem * which may hang the system. To workaround this use the second memory 1173289208Scem * window to access the interrupt and scratch pad registers on the 1174289208Scem * remote system. 1175289208Scem */ 1176289543Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1177289543Scem /* Use the last MW for mapping remote spad */ 1178289542Scem ntb->b2b_mw_idx = ntb->mw_count - 1; 1179289543Scem else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1180289208Scem /* 1181289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1182289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1183289542Scem * since bit 14 is the last bit. 1184289542Scem * 1185289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1186289542Scem * anyway. Nor for non-B2B connection types. 1187289542Scem */ 1188289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1189250079Scarl 1190289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1191250079Scarl 1192289542Scem if (ntb->dev_type == NTB_DEV_USD) 1193289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1194289542Scem &xeon_b2b_usd_addr); 1195289542Scem else 1196289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1197289542Scem &xeon_b2b_dsd_addr); 1198289542Scem if (rc != 0) 1199289542Scem return (rc); 1200289271Scem 1201250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1202289607Scem ntb_reg_write(2, XEON_PCICMD_OFFSET, 1203289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1204255279Scarl 1205255269Scarl /* Enable link training */ 1206289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1207250079Scarl 1208250079Scarl return (0); 1209250079Scarl} 1210250079Scarl 1211250079Scarlstatic int 1212289542Scemntb_soc_init_dev(struct ntb_softc *ntb) 1213250079Scarl{ 1214250079Scarl 1215289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1216289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1217250079Scarl 1218289542Scem ntb->spad_count = SOC_SPAD_COUNT; 1219289539Scem ntb->db_count = SOC_DB_COUNT; 1220289539Scem ntb->db_vec_count = SOC_DB_MSIX_VECTOR_COUNT; 1221289539Scem ntb->db_vec_shift = SOC_DB_MSIX_VECTOR_SHIFT; 1222289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1223250079Scarl 1224289542Scem ntb->reg = &soc_reg; 1225289607Scem ntb->self_reg = &soc_pri_reg; 1226289542Scem ntb->peer_reg = &soc_b2b_reg; 1227289542Scem ntb->xlat_reg = &soc_sec_xlat; 1228289542Scem 1229250079Scarl /* 1230250079Scarl * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 1231250079Scarl * resolved. Mask transaction layer internal parity errors. 1232250079Scarl */ 1233250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1234250079Scarl 1235255279Scarl configure_soc_secondary_side_bars(ntb); 1236250079Scarl 1237250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1238289607Scem ntb_reg_write(2, SOC_PCICMD_OFFSET, 1239250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1240289209Scem 1241289542Scem /* Initiate PCI-E link training */ 1242289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1243250079Scarl 1244289542Scem callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb); 1245289542Scem 1246250079Scarl return (0); 1247250079Scarl} 1248250079Scarl 1249289542Scem/* XXX: Linux driver doesn't seem to do any of this for SoC. */ 1250255279Scarlstatic void 1251255279Scarlconfigure_soc_secondary_side_bars(struct ntb_softc *ntb) 1252255279Scarl{ 1253255279Scarl 1254255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1255289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1256289542Scem XEON_B2B_BAR2_DSD_ADDR64); 1257289542Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1258289542Scem XEON_B2B_BAR4_DSD_ADDR64); 1259289542Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64); 1260289542Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64); 1261255279Scarl } else { 1262289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1263289542Scem XEON_B2B_BAR2_USD_ADDR64); 1264289542Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1265289542Scem XEON_B2B_BAR4_USD_ADDR64); 1266289542Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64); 1267289542Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64); 1268255279Scarl } 1269255279Scarl} 1270255279Scarl 1271289543Scem 1272289543Scem/* 1273289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1274289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1275289543Scem * remains for use by a higher layer. 1276289543Scem * 1277289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1278289543Scem * MW size is sufficiently large. 1279289543Scem */ 1280289543Scemstatic unsigned int ntb_b2b_mw_share; 1281289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1282289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1283289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1284289543Scem "value here."); 1285289543Scem 1286289543Scemstatic void 1287289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1288289543Scem enum ntb_bar regbar) 1289289543Scem{ 1290289543Scem struct ntb_pci_bar_info *bar; 1291289543Scem uint8_t bar_sz; 1292289543Scem 1293289543Scem if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1294289543Scem return; 1295289543Scem 1296289543Scem bar = &ntb->bar_info[idx]; 1297289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1298289543Scem if (idx == regbar) { 1299289543Scem if (ntb->b2b_off != 0) 1300289543Scem bar_sz--; 1301289543Scem else 1302289543Scem bar_sz = 0; 1303289543Scem } 1304289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1305289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1306289543Scem (void)bar_sz; 1307289543Scem} 1308289543Scem 1309289543Scemstatic void 1310289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1311289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1312289543Scem{ 1313289546Scem uint64_t reg_val; 1314289546Scem uint32_t base_reg, lmt_reg; 1315289543Scem 1316289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1317289546Scem if (idx == regbar) 1318289546Scem bar_addr += ntb->b2b_off; 1319289543Scem 1320289546Scem if (!bar_is_64bit(ntb, idx)) { 1321289546Scem ntb_reg_write(4, base_reg, bar_addr); 1322289546Scem reg_val = ntb_reg_read(4, base_reg); 1323289546Scem (void)reg_val; 1324289546Scem 1325289546Scem ntb_reg_write(4, lmt_reg, bar_addr); 1326289546Scem reg_val = ntb_reg_read(4, lmt_reg); 1327289546Scem (void)reg_val; 1328289543Scem } else { 1329289546Scem ntb_reg_write(8, base_reg, bar_addr); 1330289546Scem reg_val = ntb_reg_read(8, base_reg); 1331289546Scem (void)reg_val; 1332289546Scem 1333289546Scem ntb_reg_write(8, lmt_reg, bar_addr); 1334289546Scem reg_val = ntb_reg_read(8, lmt_reg); 1335289546Scem (void)reg_val; 1336289543Scem } 1337289543Scem} 1338289543Scem 1339289543Scemstatic void 1340289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1341289543Scem{ 1342289543Scem struct ntb_pci_bar_info *bar; 1343289543Scem 1344289543Scem bar = &ntb->bar_info[idx]; 1345289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1346289543Scem ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1347289543Scem base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1348289543Scem } else { 1349289543Scem ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1350289543Scem base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1351289543Scem } 1352289543Scem (void)base_addr; 1353289543Scem} 1354289543Scem 1355289542Scemstatic int 1356289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1357289542Scem const struct ntb_b2b_addr *peer_addr) 1358255279Scarl{ 1359289543Scem struct ntb_pci_bar_info *b2b_bar; 1360289543Scem vm_size_t bar_size; 1361289543Scem uint64_t bar_addr; 1362289543Scem enum ntb_bar b2b_bar_num, i; 1363255279Scarl 1364289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1365289543Scem b2b_bar = NULL; 1366289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1367289543Scem ntb->b2b_off = 0; 1368289543Scem } else { 1369289543Scem b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1370289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1371289543Scem ("invalid b2b mw bar")); 1372289543Scem 1373289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1374289543Scem bar_size = b2b_bar->size; 1375289543Scem 1376289543Scem if (ntb_b2b_mw_share != 0 && 1377289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1378289543Scem ntb->b2b_off = bar_size >> 1; 1379289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1380289543Scem ntb->b2b_off = 0; 1381289543Scem ntb->mw_count--; 1382289543Scem } else { 1383289543Scem device_printf(ntb->device, 1384289543Scem "B2B bar size is too small!\n"); 1385289543Scem return (EIO); 1386289543Scem } 1387255279Scarl } 1388289542Scem 1389289543Scem /* 1390289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1391289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1392289543Scem */ 1393289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1394289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1395289543Scem 1396289543Scem bar_addr = 0; 1397289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1398289543Scem bar_addr = addr->bar0_addr; 1399289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1400289543Scem bar_addr = addr->bar2_addr64; 1401289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1402289543Scem bar_addr = addr->bar4_addr64; 1403289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1404289543Scem bar_addr = addr->bar4_addr32; 1405289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1406289543Scem bar_addr = addr->bar5_addr32; 1407289543Scem else 1408289543Scem KASSERT(false, ("invalid bar")); 1409289543Scem 1410289543Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1411289543Scem 1412289543Scem /* 1413289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1414289543Scem * register BAR. The B2B BAR is either disabled above or configured 1415289543Scem * half-size. It starts at PBAR xlat + offset. 1416289543Scem * 1417289543Scem * Also set up incoming BAR limits == base (zero length window). 1418289543Scem */ 1419289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1420289543Scem b2b_bar_num); 1421289542Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1422289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1423289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1424289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1425289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1426289542Scem } else 1427289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1428289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1429289543Scem 1430289543Scem /* Zero incoming translation addrs */ 1431289543Scem ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1432289543Scem ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1433289543Scem 1434289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1435289543Scem ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1436289543Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1437289543Scem 1438289543Scem /* Set outgoing translation offsets */ 1439289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1440289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1441289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1442289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1443289543Scem } else 1444289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1445289543Scem 1446289543Scem /* Set the translation offset for B2B registers */ 1447289543Scem bar_addr = 0; 1448289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1449289543Scem bar_addr = peer_addr->bar0_addr; 1450289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1451289543Scem bar_addr = peer_addr->bar2_addr64; 1452289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1453289543Scem bar_addr = peer_addr->bar4_addr64; 1454289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1455289543Scem bar_addr = peer_addr->bar4_addr32; 1456289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1457289543Scem bar_addr = peer_addr->bar5_addr32; 1458289543Scem else 1459289543Scem KASSERT(false, ("invalid bar")); 1460289543Scem 1461289543Scem /* 1462289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1463289543Scem * at a time. 1464289543Scem */ 1465289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1466289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1467289542Scem return (0); 1468255279Scarl} 1469255279Scarl 1470289546Scemstatic inline bool 1471289546Scemlink_is_up(struct ntb_softc *ntb) 1472289546Scem{ 1473289546Scem 1474289546Scem if (ntb->type == NTB_XEON) 1475289546Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1476289546Scem 1477289546Scem KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1478289546Scem return ((ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0); 1479289546Scem} 1480289546Scem 1481289546Scemstatic inline bool 1482289546Scemsoc_link_is_err(struct ntb_softc *ntb) 1483289546Scem{ 1484289546Scem uint32_t status; 1485289546Scem 1486289546Scem KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1487289546Scem 1488289546Scem status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1489289546Scem if ((status & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1490289546Scem return (true); 1491289546Scem 1492289546Scem status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1493289546Scem return ((status & SOC_IBIST_ERR_OFLOW) != 0); 1494289546Scem} 1495289546Scem 1496255281Scarl/* SOC does not have link status interrupt, poll on that platform */ 1497250079Scarlstatic void 1498289542Scemsoc_link_hb(void *arg) 1499250079Scarl{ 1500250079Scarl struct ntb_softc *ntb = arg; 1501289546Scem sbintime_t timo, poll_ts; 1502250079Scarl 1503289546Scem timo = NTB_HB_TIMEOUT * hz; 1504289546Scem poll_ts = ntb->last_ts + timo; 1505289546Scem 1506289542Scem /* 1507289542Scem * Delay polling the link status if an interrupt was received, unless 1508289542Scem * the cached link status says the link is down. 1509289542Scem */ 1510289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1511289546Scem timo = poll_ts - ticks; 1512289542Scem goto out; 1513289546Scem } 1514289542Scem 1515289546Scem if (ntb_poll_link(ntb)) 1516289546Scem ntb_link_event(ntb); 1517289542Scem 1518289546Scem if (!link_is_up(ntb) && soc_link_is_err(ntb)) { 1519289546Scem /* Link is down with error, proceed with recovery */ 1520289546Scem callout_reset(&ntb->lr_timer, 0, recover_soc_link, ntb); 1521289546Scem return; 1522250079Scarl } 1523250079Scarl 1524289542Scemout: 1525289546Scem callout_reset(&ntb->heartbeat_timer, timo, soc_link_hb, ntb); 1526250079Scarl} 1527250079Scarl 1528250079Scarlstatic void 1529250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb) 1530250079Scarl{ 1531250079Scarl uint32_t status; 1532250079Scarl 1533250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1534255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1535255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1536255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1537255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1538250079Scarl 1539250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1540250079Scarl pause("ModPhy", hz / 10); 1541250079Scarl 1542250079Scarl /* Clear AER Errors, write to clear */ 1543255278Scarl status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1544250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1545255278Scarl ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1546250079Scarl 1547250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1548255278Scarl status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1549250079Scarl status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1550255278Scarl ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1551250079Scarl 1552250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1553255278Scarl status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1554250079Scarl status |= SOC_DESKEWSTS_DBERR; 1555255278Scarl ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1556250079Scarl 1557255278Scarl status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1558250079Scarl status &= SOC_IBIST_ERR_OFLOW; 1559255278Scarl ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1560250079Scarl 1561250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1562255278Scarl status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1563250079Scarl status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1564255278Scarl ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1565250079Scarl} 1566250079Scarl 1567289546Scem/* 1568289546Scem * ntb_set_ctx() - associate a driver context with an ntb device 1569289546Scem * @ntb: NTB device context 1570289546Scem * @ctx: Driver context 1571289546Scem * @ctx_ops: Driver context operations 1572289546Scem * 1573289546Scem * Associate a driver context and operations with a ntb device. The context is 1574289546Scem * provided by the client driver, and the driver may associate a different 1575289546Scem * context with each ntb device. 1576289546Scem * 1577289546Scem * Return: Zero if the context is associated, otherwise an error number. 1578289546Scem */ 1579289546Scemint 1580289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 1581250079Scarl{ 1582250079Scarl 1583289546Scem if (ctx == NULL || ops == NULL) 1584289546Scem return (EINVAL); 1585289546Scem if (ntb->ctx_ops != NULL) 1586289546Scem return (EINVAL); 1587250079Scarl 1588289546Scem CTX_LOCK(ntb); 1589289546Scem if (ntb->ctx_ops != NULL) { 1590289546Scem CTX_UNLOCK(ntb); 1591289546Scem return (EINVAL); 1592250079Scarl } 1593289546Scem ntb->ntb_ctx = ctx; 1594289546Scem ntb->ctx_ops = ops; 1595289546Scem CTX_UNLOCK(ntb); 1596250079Scarl 1597289546Scem return (0); 1598250079Scarl} 1599250079Scarl 1600289546Scem/* 1601289546Scem * It is expected that this will only be used from contexts where the ctx_lock 1602289546Scem * is not needed to protect ntb_ctx lifetime. 1603289546Scem */ 1604289546Scemvoid * 1605289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 1606289546Scem{ 1607289546Scem 1608289546Scem KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 1609289546Scem if (ops != NULL) 1610289546Scem *ops = ntb->ctx_ops; 1611289546Scem return (ntb->ntb_ctx); 1612289546Scem} 1613289546Scem 1614289546Scem/* 1615289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device 1616289546Scem * @ntb: NTB device context 1617289546Scem * 1618289546Scem * Clear any association that may exist between a driver context and the ntb 1619289546Scem * device. 1620289546Scem */ 1621289546Scemvoid 1622289546Scemntb_clear_ctx(struct ntb_softc *ntb) 1623289546Scem{ 1624289546Scem 1625289546Scem CTX_LOCK(ntb); 1626289546Scem ntb->ntb_ctx = NULL; 1627289546Scem ntb->ctx_ops = NULL; 1628289546Scem CTX_UNLOCK(ntb); 1629289546Scem} 1630289546Scem 1631289546Scem/* 1632289546Scem * ntb_link_event() - notify driver context of a change in link status 1633289546Scem * @ntb: NTB device context 1634289546Scem * 1635289546Scem * Notify the driver context that the link status may have changed. The driver 1636289546Scem * should call ntb_link_is_up() to get the current status. 1637289546Scem */ 1638289546Scemvoid 1639289546Scemntb_link_event(struct ntb_softc *ntb) 1640289546Scem{ 1641289546Scem 1642289546Scem CTX_LOCK(ntb); 1643289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 1644289546Scem ntb->ctx_ops->link_event(ntb->ntb_ctx); 1645289546Scem CTX_UNLOCK(ntb); 1646289546Scem} 1647289546Scem 1648289546Scem/* 1649289546Scem * ntb_db_event() - notify driver context of a doorbell event 1650289546Scem * @ntb: NTB device context 1651289546Scem * @vector: Interrupt vector number 1652289546Scem * 1653289546Scem * Notify the driver context of a doorbell event. If hardware supports 1654289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which 1655289546Scem * vector received the interrupt. The vector number is relative to the first 1656289546Scem * vector used for doorbells, starting at zero, and must be less than 1657289546Scem * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 1658289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of 1659289546Scem * those bits are associated with the vector number. 1660289546Scem */ 1661250079Scarlstatic void 1662289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec) 1663289272Scem{ 1664289546Scem 1665289546Scem CTX_LOCK(ntb); 1666289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 1667289546Scem ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 1668289546Scem CTX_UNLOCK(ntb); 1669289546Scem} 1670289546Scem 1671289546Scem/* 1672289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb 1673289546Scem * @ntb: NTB device context 1674289546Scem * @max_speed: The maximum link speed expressed as PCIe generation number[0] 1675289546Scem * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 1676289546Scem * 1677289546Scem * Enable the link on the secondary side of the ntb. This can only be done 1678289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1679289546Scem * should train the link to its maximum speed and width, or the requested speed 1680289546Scem * and width, whichever is smaller, if supported. 1681289546Scem * 1682289546Scem * Return: Zero on success, otherwise an error number. 1683289546Scem * 1684289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 1685289546Scem * and width input will be ignored. 1686289546Scem */ 1687289546Scemint 1688289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 1689289546Scem enum ntb_width w __unused) 1690289546Scem{ 1691289280Scem uint32_t cntl; 1692289272Scem 1693289542Scem if (ntb->type == NTB_SOC) { 1694289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1695289542Scem ntb->ppd | SOC_PPD_INIT_LINK, 4); 1696289546Scem return (0); 1697289542Scem } 1698289542Scem 1699289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1700289546Scem ntb_link_event(ntb); 1701289546Scem return (0); 1702289280Scem } 1703289280Scem 1704289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1705289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1706289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1707289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1708289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1709289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1710289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1711289546Scem return (0); 1712289272Scem} 1713289272Scem 1714289546Scem/* 1715289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb 1716289546Scem * @ntb: NTB device context 1717289546Scem * 1718289546Scem * Disable the link on the secondary side of the ntb. This can only be done 1719289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1720289546Scem * should disable the link. Returning from this call must indicate that a 1721289546Scem * barrier has passed, though with no more writes may pass in either direction 1722289546Scem * across the link, except if this call returns an error number. 1723289546Scem * 1724289546Scem * Return: Zero on success, otherwise an error number. 1725289546Scem */ 1726289546Scemint 1727289542Scemntb_link_disable(struct ntb_softc *ntb) 1728289272Scem{ 1729289272Scem uint32_t cntl; 1730289272Scem 1731289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1732289546Scem ntb_link_event(ntb); 1733289546Scem return (0); 1734289272Scem } 1735289272Scem 1736289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1737289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1738289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1739289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1740289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1741289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1742289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1743289546Scem return (0); 1744289272Scem} 1745289272Scem 1746289272Scemstatic void 1747250079Scarlrecover_soc_link(void *arg) 1748250079Scarl{ 1749250079Scarl struct ntb_softc *ntb = arg; 1750250079Scarl uint8_t speed, width; 1751250079Scarl uint32_t status32; 1752250079Scarl 1753250079Scarl soc_perform_link_restart(ntb); 1754250079Scarl 1755289232Scem /* 1756289232Scem * There is a potential race between the 2 NTB devices recovering at 1757289232Scem * the same time. If the times are the same, the link will not recover 1758289232Scem * and the driver will be stuck in this loop forever. Add a random 1759289232Scem * interval to the recovery time to prevent this race. 1760289232Scem */ 1761289232Scem status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1762289232Scem pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1763289232Scem 1764255278Scarl status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1765250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1766250079Scarl goto retry; 1767250079Scarl 1768255278Scarl status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1769250079Scarl if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1770250079Scarl goto retry; 1771250079Scarl 1772289542Scem status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1773289232Scem if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1774289232Scem goto out; 1775289232Scem 1776289542Scem status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1777289542Scem width = (status32 & NTB_LINK_WIDTH_MASK) >> 4; 1778289542Scem speed = (status32 & NTB_LINK_SPEED_MASK); 1779250079Scarl if (ntb->link_width != width || ntb->link_speed != speed) 1780250079Scarl goto retry; 1781250079Scarl 1782289232Scemout: 1783289542Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, 1784289542Scem ntb); 1785250079Scarl return; 1786250079Scarl 1787250079Scarlretry: 1788250079Scarl callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1789250079Scarl ntb); 1790250079Scarl} 1791250079Scarl 1792289546Scem/* 1793289546Scem * Polls the HW link status register(s); returns true if something has changed. 1794289546Scem */ 1795289546Scemstatic bool 1796289542Scemntb_poll_link(struct ntb_softc *ntb) 1797250079Scarl{ 1798250079Scarl uint32_t ntb_cntl; 1799289546Scem uint16_t reg_val; 1800250079Scarl 1801250079Scarl if (ntb->type == NTB_SOC) { 1802289542Scem ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1803289546Scem if (ntb_cntl == ntb->ntb_ctl) 1804289546Scem return (false); 1805289546Scem 1806289542Scem ntb->ntb_ctl = ntb_cntl; 1807289542Scem ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1808250079Scarl } else { 1809289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 1810250079Scarl 1811289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 1812289546Scem if (reg_val == ntb->lnk_sta) 1813289546Scem return (false); 1814250079Scarl 1815289546Scem ntb->lnk_sta = reg_val; 1816289542Scem } 1817289546Scem return (true); 1818289542Scem} 1819289542Scem 1820289546Scemstatic inline enum ntb_speed 1821289546Scemntb_link_sta_speed(struct ntb_softc *ntb) 1822250079Scarl{ 1823250079Scarl 1824289546Scem if (!link_is_up(ntb)) 1825289546Scem return (NTB_SPEED_NONE); 1826289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 1827250079Scarl} 1828250079Scarl 1829289546Scemstatic inline enum ntb_width 1830289546Scemntb_link_sta_width(struct ntb_softc *ntb) 1831250079Scarl{ 1832250079Scarl 1833289546Scem if (!link_is_up(ntb)) 1834289546Scem return (NTB_WIDTH_NONE); 1835289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 1836250079Scarl} 1837250079Scarl 1838289546Scem/* 1839289546Scem * Public API to the rest of the OS 1840250079Scarl */ 1841250079Scarl 1842250079Scarl/** 1843250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 1844250079Scarl * @ntb: pointer to ntb_softc instance 1845250079Scarl * 1846250079Scarl * This function returns the max 32bit scratchpad registers usable by the 1847250079Scarl * upper layer. 1848250079Scarl * 1849250079Scarl * RETURNS: total number of scratch pad registers available 1850250079Scarl */ 1851289208Scemuint8_t 1852250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 1853250079Scarl{ 1854250079Scarl 1855289539Scem return (ntb->spad_count); 1856250079Scarl} 1857250079Scarl 1858289396Scemuint8_t 1859289539Scemntb_mw_count(struct ntb_softc *ntb) 1860289396Scem{ 1861289396Scem 1862289539Scem return (ntb->mw_count); 1863289396Scem} 1864289396Scem 1865250079Scarl/** 1866289545Scem * ntb_spad_write() - write to the secondary scratchpad register 1867250079Scarl * @ntb: pointer to ntb_softc instance 1868250079Scarl * @idx: index to the scratchpad register, 0 based 1869250079Scarl * @val: the data value to put into the register 1870250079Scarl * 1871250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1872250079Scarl * register. The register resides on the secondary (external) side. 1873250079Scarl * 1874289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1875250079Scarl */ 1876250079Scarlint 1877289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1878250079Scarl{ 1879250079Scarl 1880289539Scem if (idx >= ntb->spad_count) 1881250079Scarl return (EINVAL); 1882250079Scarl 1883289607Scem ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 1884250079Scarl 1885250079Scarl return (0); 1886250079Scarl} 1887250079Scarl 1888250079Scarl/** 1889289545Scem * ntb_spad_read() - read from the primary scratchpad register 1890250079Scarl * @ntb: pointer to ntb_softc instance 1891250079Scarl * @idx: index to scratchpad register, 0 based 1892250079Scarl * @val: pointer to 32bit integer for storing the register value 1893250079Scarl * 1894250079Scarl * This function allows reading of the 32bit scratchpad register on 1895250079Scarl * the primary (internal) side. 1896250079Scarl * 1897289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1898250079Scarl */ 1899250079Scarlint 1900289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1901250079Scarl{ 1902250079Scarl 1903289539Scem if (idx >= ntb->spad_count) 1904250079Scarl return (EINVAL); 1905250079Scarl 1906289607Scem *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 1907250079Scarl 1908250079Scarl return (0); 1909250079Scarl} 1910250079Scarl 1911250079Scarl/** 1912289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register 1913250079Scarl * @ntb: pointer to ntb_softc instance 1914250079Scarl * @idx: index to the scratchpad register, 0 based 1915250079Scarl * @val: the data value to put into the register 1916250079Scarl * 1917250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1918250079Scarl * register. The register resides on the secondary (external) side. 1919250079Scarl * 1920289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1921250079Scarl */ 1922250079Scarlint 1923289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1924250079Scarl{ 1925250079Scarl 1926289539Scem if (idx >= ntb->spad_count) 1927250079Scarl return (EINVAL); 1928250079Scarl 1929289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1930255279Scarl ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1931255279Scarl else 1932289542Scem ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 1933250079Scarl 1934250079Scarl return (0); 1935250079Scarl} 1936250079Scarl 1937250079Scarl/** 1938289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register 1939250079Scarl * @ntb: pointer to ntb_softc instance 1940250079Scarl * @idx: index to scratchpad register, 0 based 1941250079Scarl * @val: pointer to 32bit integer for storing the register value 1942250079Scarl * 1943250079Scarl * This function allows reading of the 32bit scratchpad register on 1944250079Scarl * the primary (internal) side. 1945250079Scarl * 1946289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1947250079Scarl */ 1948250079Scarlint 1949289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1950250079Scarl{ 1951250079Scarl 1952289539Scem if (idx >= ntb->spad_count) 1953250079Scarl return (EINVAL); 1954250079Scarl 1955289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1956255279Scarl *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1957255279Scarl else 1958289542Scem *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 1959250079Scarl 1960250079Scarl return (0); 1961250079Scarl} 1962250079Scarl 1963289546Scem/* 1964289546Scem * ntb_mw_get_range() - get the range of a memory window 1965289546Scem * @ntb: NTB device context 1966289546Scem * @idx: Memory window number 1967289546Scem * @base: OUT - the base address for mapping the memory window 1968289546Scem * @size: OUT - the size for mapping the memory window 1969289546Scem * @align: OUT - the base alignment for translating the memory window 1970289546Scem * @align_size: OUT - the size alignment for translating the memory window 1971250079Scarl * 1972289546Scem * Get the range of a memory window. NULL may be given for any output 1973289546Scem * parameter if the value is not needed. The base and size may be used for 1974289546Scem * mapping the memory window, to access the peer memory. The alignment and 1975289546Scem * size may be used for translating the memory window, for the peer to access 1976289546Scem * memory on the local system. 1977250079Scarl * 1978289546Scem * Return: Zero on success, otherwise an error number. 1979250079Scarl */ 1980289546Scemint 1981289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 1982289546Scem void **vbase, size_t *size, size_t *align, size_t *align_size) 1983250079Scarl{ 1984289546Scem struct ntb_pci_bar_info *bar; 1985289546Scem size_t bar_b2b_off; 1986250079Scarl 1987289546Scem if (mw_idx >= ntb_mw_count(ntb)) 1988289546Scem return (EINVAL); 1989250079Scarl 1990289546Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)]; 1991289546Scem bar_b2b_off = 0; 1992289546Scem if (mw_idx == ntb->b2b_mw_idx) { 1993289546Scem KASSERT(ntb->b2b_off != 0, 1994289546Scem ("user shouldn't get non-shared b2b mw")); 1995289546Scem bar_b2b_off = ntb->b2b_off; 1996289546Scem } 1997250079Scarl 1998289546Scem if (base != NULL) 1999289546Scem *base = bar->pbase + bar_b2b_off; 2000289546Scem if (vbase != NULL) 2001289546Scem *vbase = (char *)bar->vbase + bar_b2b_off; 2002289546Scem if (size != NULL) 2003289546Scem *size = bar->size - bar_b2b_off; 2004289546Scem if (align != NULL) 2005289546Scem *align = bar->size; 2006289546Scem if (align_size != NULL) 2007289546Scem *align_size = 1; 2008289546Scem return (0); 2009250079Scarl} 2010250079Scarl 2011289546Scem/* 2012289546Scem * ntb_mw_set_trans() - set the translation of a memory window 2013289546Scem * @ntb: NTB device context 2014289546Scem * @idx: Memory window number 2015289546Scem * @addr: The dma address local memory to expose to the peer 2016289546Scem * @size: The size of the local memory to expose to the peer 2017250079Scarl * 2018289546Scem * Set the translation of a memory window. The peer may access local memory 2019289546Scem * through the window starting at the address, up to the size. The address 2020289546Scem * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2021289546Scem * must be aligned to the size alignment specified by ntb_mw_get_range(). 2022250079Scarl * 2023289546Scem * Return: Zero on success, otherwise an error number. 2024250079Scarl */ 2025289546Scemint 2026289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2027289546Scem size_t size) 2028250079Scarl{ 2029289546Scem struct ntb_pci_bar_info *bar; 2030289546Scem uint64_t base, limit, reg_val; 2031289546Scem size_t bar_size, mw_size; 2032289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2033289546Scem enum ntb_bar bar_num; 2034250079Scarl 2035289546Scem if (idx >= ntb_mw_count(ntb)) 2036289546Scem return (EINVAL); 2037250079Scarl 2038289546Scem bar_num = ntb_mw_to_bar(ntb, idx); 2039289546Scem bar = &ntb->bar_info[bar_num]; 2040250079Scarl 2041289546Scem bar_size = bar->size; 2042289546Scem if (idx == ntb->b2b_mw_idx) 2043289546Scem mw_size = bar_size - ntb->b2b_off; 2044289546Scem else 2045289546Scem mw_size = bar_size; 2046250079Scarl 2047289546Scem /* Hardware requires that addr is aligned to bar size */ 2048289546Scem if ((addr & (bar_size - 1)) != 0) 2049289546Scem return (EINVAL); 2050250079Scarl 2051289546Scem if (size > mw_size) 2052289546Scem return (EINVAL); 2053289546Scem 2054289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2055289546Scem 2056289546Scem limit = 0; 2057289546Scem if (bar_is_64bit(ntb, bar_num)) { 2058289546Scem base = ntb_reg_read(8, base_reg); 2059289546Scem 2060289546Scem if (limit_reg != 0 && size != mw_size) 2061289546Scem limit = base + size; 2062289546Scem 2063289546Scem /* Set and verify translation address */ 2064289546Scem ntb_reg_write(8, xlat_reg, addr); 2065289546Scem reg_val = ntb_reg_read(8, xlat_reg); 2066289546Scem if (reg_val != addr) { 2067289546Scem ntb_reg_write(8, xlat_reg, 0); 2068289546Scem return (EIO); 2069289546Scem } 2070289546Scem 2071289546Scem /* Set and verify the limit */ 2072289546Scem ntb_reg_write(8, limit_reg, limit); 2073289546Scem reg_val = ntb_reg_read(8, limit_reg); 2074289546Scem if (reg_val != limit) { 2075289546Scem ntb_reg_write(8, limit_reg, base); 2076289546Scem ntb_reg_write(8, xlat_reg, 0); 2077289546Scem return (EIO); 2078289546Scem } 2079289546Scem } else { 2080289546Scem /* Configure 32-bit (split) BAR MW */ 2081289546Scem 2082289546Scem if ((addr & ~UINT32_MAX) != 0) 2083289546Scem return (EINVAL); 2084289546Scem if (((addr + size) & ~UINT32_MAX) != 0) 2085289546Scem return (EINVAL); 2086289546Scem 2087289546Scem base = ntb_reg_read(4, base_reg); 2088289546Scem 2089289546Scem if (limit_reg != 0 && size != mw_size) 2090289546Scem limit = base + size; 2091289546Scem 2092289546Scem /* Set and verify translation address */ 2093289546Scem ntb_reg_write(4, xlat_reg, addr); 2094289546Scem reg_val = ntb_reg_read(4, xlat_reg); 2095289546Scem if (reg_val != addr) { 2096289546Scem ntb_reg_write(4, xlat_reg, 0); 2097289546Scem return (EIO); 2098289546Scem } 2099289546Scem 2100289546Scem /* Set and verify the limit */ 2101289546Scem ntb_reg_write(4, limit_reg, limit); 2102289546Scem reg_val = ntb_reg_read(4, limit_reg); 2103289546Scem if (reg_val != limit) { 2104289546Scem ntb_reg_write(4, limit_reg, base); 2105289546Scem ntb_reg_write(4, xlat_reg, 0); 2106289546Scem return (EIO); 2107289546Scem } 2108250079Scarl } 2109289546Scem return (0); 2110250079Scarl} 2111250079Scarl 2112289596Scem/* 2113289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window 2114289596Scem * @ntb: NTB device context 2115289596Scem * @idx: Memory window number 2116289596Scem * 2117289596Scem * Clear the translation of a memory window. The peer may no longer access 2118289596Scem * local memory through the window. 2119289596Scem * 2120289596Scem * Return: Zero on success, otherwise an error number. 2121289596Scem */ 2122289596Scemint 2123289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 2124289596Scem{ 2125289596Scem 2126289596Scem return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 2127289596Scem} 2128289596Scem 2129250079Scarl/** 2130289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side 2131250079Scarl * @ntb: pointer to ntb_softc instance 2132289545Scem * @bit: doorbell bits to ring 2133250079Scarl * 2134250079Scarl * This function allows triggering of a doorbell on the secondary/external 2135250079Scarl * side that will initiate an interrupt on the remote host 2136250079Scarl */ 2137250079Scarlvoid 2138289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 2139250079Scarl{ 2140250079Scarl 2141289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2142289347Scem ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit); 2143289347Scem return; 2144289209Scem } 2145289347Scem 2146289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2147250079Scarl} 2148250079Scarl 2149289542Scem/* 2150289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 2151289542Scem * as well as the size of the register (via *sz_out). 2152289542Scem * 2153289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell 2154289542Scem * ring to its memory window write. 2155289542Scem * 2156289542Scem * Note that writing the peer doorbell via a memory window will *not* generate 2157289542Scem * an interrupt on the remote host; that must be done seperately. 2158289542Scem */ 2159289542Scembus_addr_t 2160289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 2161289542Scem{ 2162289542Scem struct ntb_pci_bar_info *bar; 2163289542Scem uint64_t regoff; 2164289542Scem 2165289542Scem KASSERT(sz_out != NULL, ("must be non-NULL")); 2166289542Scem 2167289542Scem if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2168289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 2169289542Scem regoff = ntb->peer_reg->db_bell; 2170289542Scem } else { 2171289542Scem KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) || 2172289542Scem (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1), 2173289542Scem ("mw_count invalid after setup")); 2174289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 2175289543Scem ("invalid b2b idx")); 2176289542Scem 2177289542Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 2178289542Scem regoff = XEON_SHADOW_PDOORBELL_OFFSET; 2179289542Scem } 2180289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 2181289542Scem 2182289542Scem *sz_out = ntb->reg->db_size; 2183289542Scem /* HACK: Specific to current x86 bus implementation. */ 2184289542Scem return ((uint64_t)bar->pci_bus_handle + regoff); 2185289542Scem} 2186289542Scem 2187289597Scem/* 2188289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 2189289597Scem * @ntb: NTB device context 2190289597Scem * 2191289597Scem * Hardware may support different number or arrangement of doorbell bits. 2192289597Scem * 2193289597Scem * Return: A mask of doorbell bits supported by the ntb. 2194289597Scem */ 2195289597Scemuint64_t 2196289597Scemntb_db_valid_mask(struct ntb_softc *ntb) 2197289597Scem{ 2198289597Scem 2199289597Scem return (ntb->db_valid_mask); 2200289597Scem} 2201289597Scem 2202289598Scem/* 2203289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 2204289598Scem * @ntb: NTB device context 2205289598Scem * @vector: Doorbell vector number 2206289598Scem * 2207289598Scem * Each interrupt vector may have a different number or arrangement of bits. 2208289598Scem * 2209289598Scem * Return: A mask of doorbell bits serviced by a vector. 2210289598Scem */ 2211289598Scemuint64_t 2212289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 2213289598Scem{ 2214289598Scem 2215289598Scem if (vector > ntb->db_vec_count) 2216289598Scem return (0); 2217289598Scem return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 2218289598Scem} 2219289598Scem 2220250079Scarl/** 2221289546Scem * ntb_link_is_up() - get the current ntb link state 2222289546Scem * @ntb: NTB device context 2223289546Scem * @speed: OUT - The link speed expressed as PCIe generation number 2224289546Scem * @width: OUT - The link width expressed as the number of PCIe lanes 2225250079Scarl * 2226250079Scarl * RETURNS: true or false based on the hardware link state 2227250079Scarl */ 2228250079Scarlbool 2229289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 2230289546Scem enum ntb_width *width) 2231250079Scarl{ 2232250079Scarl 2233289546Scem if (speed != NULL) 2234289546Scem *speed = ntb_link_sta_speed(ntb); 2235289546Scem if (width != NULL) 2236289546Scem *width = ntb_link_sta_width(ntb); 2237289546Scem return (link_is_up(ntb)); 2238250079Scarl} 2239250079Scarl 2240255272Scarlstatic void 2241255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 2242250079Scarl{ 2243255272Scarl 2244289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2245289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2246289209Scem bar->pbase = rman_get_start(bar->pci_resource); 2247289209Scem bar->size = rman_get_size(bar->pci_resource); 2248289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 2249250079Scarl} 2250255268Scarl 2251289209Scemdevice_t 2252289209Scemntb_get_device(struct ntb_softc *ntb) 2253255268Scarl{ 2254255268Scarl 2255255268Scarl return (ntb->device); 2256255268Scarl} 2257289208Scem 2258289208Scem/* Export HW-specific errata information. */ 2259289208Scembool 2260289208Scemntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 2261289208Scem{ 2262289208Scem 2263289208Scem return (HAS_FEATURE(feature)); 2264289208Scem} 2265