ntb_hw_intel.c revision 289598
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3289542Scem * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289598 2015-10-19 18:06:35Z cem $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35250079Scarl#include <sys/malloc.h> 36250079Scarl#include <sys/module.h> 37250079Scarl#include <sys/queue.h> 38250079Scarl#include <sys/rman.h> 39289207Scem#include <sys/sysctl.h> 40250079Scarl#include <vm/vm.h> 41250079Scarl#include <vm/pmap.h> 42250079Scarl#include <machine/bus.h> 43250079Scarl#include <machine/pmap.h> 44250079Scarl#include <machine/resource.h> 45250079Scarl#include <dev/pci/pcireg.h> 46250079Scarl#include <dev/pci/pcivar.h> 47250079Scarl 48250079Scarl#include "ntb_regs.h" 49250079Scarl#include "ntb_hw.h" 50250079Scarl 51250079Scarl/* 52250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 53250079Scarl * allows you to connect two systems using a PCI-e link. 54250079Scarl * 55250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 56250079Scarl * you to send and recieve interrupts, map the memory windows and send and 57250079Scarl * receive messages in the scratch-pad registers. 58250079Scarl * 59250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 60250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 61250079Scarl */ 62250079Scarl 63289538Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT) 64250079Scarl 65289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 66289539Scem#define SOC_LINK_RECOVERY_TIME 500 /* ms */ 67250079Scarl 68250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 69250079Scarl 70250079Scarlenum ntb_device_type { 71250079Scarl NTB_XEON, 72250079Scarl NTB_SOC 73250079Scarl}; 74250079Scarl 75289539Scemenum ntb_bar { 76289539Scem NTB_CONFIG_BAR = 0, 77289539Scem NTB_B2B_BAR_1, 78289539Scem NTB_B2B_BAR_2, 79289539Scem NTB_B2B_BAR_3, 80289539Scem NTB_MAX_BARS 81289539Scem}; 82289539Scem 83255274Scarl/* Device features and workarounds */ 84255274Scarl#define HAS_FEATURE(feature) \ 85255274Scarl ((ntb->features & (feature)) != 0) 86255274Scarl 87250079Scarlstruct ntb_hw_info { 88250079Scarl uint32_t device_id; 89255274Scarl const char *desc; 90250079Scarl enum ntb_device_type type; 91289397Scem uint32_t features; 92250079Scarl}; 93250079Scarl 94250079Scarlstruct ntb_pci_bar_info { 95250079Scarl bus_space_tag_t pci_bus_tag; 96250079Scarl bus_space_handle_t pci_bus_handle; 97250079Scarl int pci_resource_id; 98250079Scarl struct resource *pci_resource; 99250079Scarl vm_paddr_t pbase; 100250079Scarl void *vbase; 101250079Scarl u_long size; 102289543Scem 103289543Scem /* Configuration register offsets */ 104289543Scem uint32_t psz_off; 105289543Scem uint32_t ssz_off; 106289543Scem uint32_t pbarxlat_off; 107250079Scarl}; 108250079Scarl 109250079Scarlstruct ntb_int_info { 110250079Scarl struct resource *res; 111250079Scarl int rid; 112250079Scarl void *tag; 113250079Scarl}; 114250079Scarl 115289546Scemstruct ntb_vec { 116250079Scarl struct ntb_softc *ntb; 117289546Scem uint32_t num; 118250079Scarl}; 119250079Scarl 120289542Scemstruct ntb_reg { 121289542Scem uint32_t ntb_ctl; 122289542Scem uint32_t lnk_sta; 123289542Scem uint8_t db_size; 124289542Scem unsigned mw_bar[NTB_MAX_BARS]; 125289542Scem}; 126289542Scem 127289542Scemstruct ntb_alt_reg { 128289542Scem uint32_t db_bell; 129289542Scem uint32_t db_mask; 130289542Scem uint32_t spad; 131289542Scem}; 132289542Scem 133289542Scemstruct ntb_xlat_reg { 134289546Scem uint32_t bar0_base; 135289546Scem uint32_t bar2_base; 136289546Scem uint32_t bar4_base; 137289546Scem uint32_t bar5_base; 138289546Scem 139289546Scem uint32_t bar2_xlat; 140289546Scem uint32_t bar4_xlat; 141289546Scem uint32_t bar5_xlat; 142289546Scem 143289546Scem uint32_t bar2_limit; 144289546Scem uint32_t bar4_limit; 145289546Scem uint32_t bar5_limit; 146289542Scem}; 147289542Scem 148289542Scemstruct ntb_b2b_addr { 149289542Scem uint64_t bar0_addr; 150289542Scem uint64_t bar2_addr64; 151289542Scem uint64_t bar4_addr64; 152289542Scem uint64_t bar4_addr32; 153289542Scem uint64_t bar5_addr32; 154289542Scem}; 155289542Scem 156250079Scarlstruct ntb_softc { 157250079Scarl device_t device; 158250079Scarl enum ntb_device_type type; 159255274Scarl uint64_t features; 160250079Scarl 161250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 162250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 163250079Scarl uint32_t allocated_interrupts; 164250079Scarl 165250079Scarl struct callout heartbeat_timer; 166250079Scarl struct callout lr_timer; 167250079Scarl 168289546Scem void *ntb_ctx; 169289546Scem const struct ntb_ctx_ops *ctx_ops; 170289546Scem struct ntb_vec *msix_vec; 171289546Scem#define CTX_LOCK(sc) mtx_lock_spin(&(sc)->ctx_lock) 172289546Scem#define CTX_UNLOCK(sc) mtx_unlock_spin(&(sc)->ctx_lock) 173289546Scem#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 174289546Scem struct mtx ctx_lock; 175250079Scarl 176250079Scarl struct { 177289255Scem uint32_t ldb; 178289255Scem uint32_t ldb_mask; 179289255Scem uint32_t bar4_xlat; 180289397Scem uint32_t bar5_xlat; 181250079Scarl uint32_t spad_local; 182250079Scarl uint32_t spci_cmd; 183250079Scarl } reg_ofs; 184289348Scem uint32_t ppd; 185250079Scarl uint8_t conn_type; 186250079Scarl uint8_t dev_type; 187250079Scarl uint8_t link_width; 188250079Scarl uint8_t link_speed; 189289539Scem 190289542Scem /* Offset of peer bar0 in B2B BAR */ 191289542Scem uint64_t b2b_off; 192289542Scem /* Memory window used to access peer bar0 */ 193289543Scem#define B2B_MW_DISABLED UINT8_MAX 194289542Scem uint8_t b2b_mw_idx; 195289542Scem 196289539Scem uint8_t mw_count; 197289539Scem uint8_t spad_count; 198289539Scem uint8_t db_count; 199289539Scem uint8_t db_vec_count; 200289539Scem uint8_t db_vec_shift; 201289542Scem 202289546Scem /* Protects local db_mask. */ 203289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 204289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 205289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 206289542Scem struct mtx db_mask_lock; 207289542Scem 208289546Scem uint32_t ntb_ctl; 209289546Scem uint32_t lnk_sta; 210289542Scem 211289542Scem uint64_t db_valid_mask; 212289542Scem uint64_t db_link_mask; 213289546Scem uint64_t db_mask; 214289542Scem 215289542Scem int last_ts; /* ticks @ last irq */ 216289542Scem 217289542Scem const struct ntb_reg *reg; 218289542Scem const struct ntb_alt_reg *self_reg; 219289542Scem const struct ntb_alt_reg *peer_reg; 220289542Scem const struct ntb_xlat_reg *xlat_reg; 221250079Scarl}; 222250079Scarl 223289234Scem#ifdef __i386__ 224289234Scemstatic __inline uint64_t 225289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 226289234Scem bus_size_t offset) 227289234Scem{ 228289234Scem 229289234Scem return (bus_space_read_4(tag, handle, offset) | 230289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 231289234Scem} 232289234Scem 233289234Scemstatic __inline void 234289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 235289234Scem bus_size_t offset, uint64_t val) 236289234Scem{ 237289234Scem 238289234Scem bus_space_write_4(tag, handle, offset, val); 239289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 240289234Scem} 241289234Scem#endif 242289234Scem 243255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 244255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 245255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 246255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 247255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 248255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 249255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 250250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 251255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 252289397Scem#define ntb_mw_read(SIZE, offset) \ 253289542Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 254255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 255289542Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 256289397Scem offset, val) 257250079Scarl 258250079Scarlstatic int ntb_probe(device_t device); 259250079Scarlstatic int ntb_attach(device_t device); 260250079Scarlstatic int ntb_detach(device_t device); 261289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 262289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 263289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 264289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 265255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 266289541Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *); 267255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 268255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 269255272Scarl struct ntb_pci_bar_info *bar); 270250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 271289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 272289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 273289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 274289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 275250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 276289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 277289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 278289546Scemstatic void ndev_vec_isr(void *arg); 279289546Scemstatic void ndev_irq_isr(void *arg); 280289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 281289546Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t val); 282289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 283289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 284250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 285289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 286289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 287289348Scemstatic int ntb_detect_soc(struct ntb_softc *ntb); 288289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 289289542Scemstatic int ntb_soc_init_dev(struct ntb_softc *ntb); 290289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 291255279Scarlstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 292289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 293289543Scem enum ntb_bar regbar); 294289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 295289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 296289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 297289543Scem enum ntb_bar idx); 298289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 299289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 300289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 301289546Scemstatic inline bool soc_link_is_err(struct ntb_softc *ntb); 302289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 303289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 304289542Scemstatic void soc_link_hb(void *arg); 305289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 306250079Scarlstatic void recover_soc_link(void *arg); 307289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb); 308255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 309250079Scarl 310250079Scarlstatic struct ntb_hw_info pci_ids[] = { 311255274Scarl { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 312289233Scem 313289233Scem /* XXX: PS/SS IDs left out until they are supported. */ 314289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 315289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 316289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 317289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 318289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 319289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 320289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 321289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 322289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 323289538Scem NTB_SB01BASE_LOCKUP }, 324289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 325289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 326289538Scem NTB_SB01BASE_LOCKUP }, 327289233Scem 328255274Scarl { 0x00000000, NULL, NTB_SOC, 0 } 329250079Scarl}; 330250079Scarl 331289542Scemstatic const struct ntb_reg soc_reg = { 332289542Scem .ntb_ctl = SOC_NTBCNTL_OFFSET, 333289542Scem .lnk_sta = SOC_LINK_STATUS_OFFSET, 334289542Scem .db_size = sizeof(uint64_t), 335289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 336289542Scem}; 337289542Scem 338289542Scemstatic const struct ntb_alt_reg soc_b2b_reg = { 339289542Scem .db_bell = SOC_B2B_DOORBELL_OFFSET, 340289542Scem .spad = SOC_B2B_SPAD_OFFSET, 341289542Scem}; 342289542Scem 343289542Scemstatic const struct ntb_xlat_reg soc_sec_xlat = { 344289542Scem#if 0 345289542Scem /* "FIXME" says the Linux driver. */ 346289542Scem .bar0_base = SOC_SBAR0BASE_OFFSET, 347289546Scem .bar2_base = SOC_SBAR2BASE_OFFSET, 348289546Scem .bar4_base = SOC_SBAR4BASE_OFFSET, 349289546Scem 350289542Scem .bar2_limit = SOC_SBAR2LMT_OFFSET, 351289546Scem .bar4_limit = SOC_SBAR4LMT_OFFSET, 352289542Scem#endif 353289546Scem 354289542Scem .bar2_xlat = SOC_SBAR2XLAT_OFFSET, 355289546Scem .bar4_xlat = SOC_SBAR4XLAT_OFFSET, 356289542Scem}; 357289542Scem 358289542Scemstatic const struct ntb_reg xeon_reg = { 359289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 360289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 361289542Scem .db_size = sizeof(uint16_t), 362289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 363289542Scem}; 364289542Scem 365289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 366289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 367289542Scem .spad = XEON_B2B_SPAD_OFFSET, 368289542Scem}; 369289542Scem 370289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 371289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 372289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 373289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 374289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 375289546Scem 376289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 377289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 378289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 379289546Scem 380289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 381289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 382289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 383289542Scem}; 384289542Scem 385289542Scemstatic const struct ntb_b2b_addr xeon_b2b_usd_addr = { 386289542Scem .bar0_addr = XEON_B2B_BAR0_USD_ADDR, 387289542Scem .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, 388289542Scem .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, 389289542Scem .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, 390289542Scem .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, 391289542Scem}; 392289542Scem 393289542Scemstatic const struct ntb_b2b_addr xeon_b2b_dsd_addr = { 394289542Scem .bar0_addr = XEON_B2B_BAR0_DSD_ADDR, 395289542Scem .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, 396289542Scem .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, 397289542Scem .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, 398289542Scem .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, 399289542Scem}; 400289542Scem 401250079Scarl/* 402250079Scarl * OS <-> Driver interface structures 403250079Scarl */ 404250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 405250079Scarl 406250079Scarlstatic device_method_t ntb_pci_methods[] = { 407250079Scarl /* Device interface */ 408250079Scarl DEVMETHOD(device_probe, ntb_probe), 409250079Scarl DEVMETHOD(device_attach, ntb_attach), 410250079Scarl DEVMETHOD(device_detach, ntb_detach), 411250079Scarl DEVMETHOD_END 412250079Scarl}; 413250079Scarl 414250079Scarlstatic driver_t ntb_pci_driver = { 415250079Scarl "ntb_hw", 416250079Scarl ntb_pci_methods, 417250079Scarl sizeof(struct ntb_softc), 418250079Scarl}; 419250079Scarl 420250079Scarlstatic devclass_t ntb_devclass; 421250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 422250079ScarlMODULE_VERSION(ntb_hw, 1); 423250079Scarl 424289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 425289207Scem 426250079Scarl/* 427250079Scarl * OS <-> Driver linkage functions 428250079Scarl */ 429250079Scarlstatic int 430250079Scarlntb_probe(device_t device) 431250079Scarl{ 432289209Scem struct ntb_hw_info *p; 433250079Scarl 434289209Scem p = ntb_get_device_info(pci_get_devid(device)); 435289209Scem if (p == NULL) 436250079Scarl return (ENXIO); 437289209Scem 438289209Scem device_set_desc(device, p->desc); 439289209Scem return (0); 440250079Scarl} 441250079Scarl 442250079Scarlstatic int 443250079Scarlntb_attach(device_t device) 444250079Scarl{ 445289209Scem struct ntb_softc *ntb; 446289209Scem struct ntb_hw_info *p; 447250079Scarl int error; 448250079Scarl 449289209Scem ntb = DEVICE2SOFTC(device); 450289209Scem p = ntb_get_device_info(pci_get_devid(device)); 451289209Scem 452250079Scarl ntb->device = device; 453250079Scarl ntb->type = p->type; 454255274Scarl ntb->features = p->features; 455289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 456250079Scarl 457250079Scarl /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 458283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 459283291Sjkim callout_init(&ntb->lr_timer, 1); 460289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 461289546Scem mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN); 462250079Scarl 463289348Scem if (ntb->type == NTB_SOC) 464289348Scem error = ntb_detect_soc(ntb); 465289348Scem else 466289348Scem error = ntb_detect_xeon(ntb); 467289348Scem if (error) 468289348Scem goto out; 469289348Scem 470289397Scem ntb_detect_max_mw(ntb); 471289396Scem 472289209Scem error = ntb_map_pci_bars(ntb); 473289209Scem if (error) 474289209Scem goto out; 475289272Scem if (ntb->type == NTB_SOC) 476289542Scem error = ntb_soc_init_dev(ntb); 477289272Scem else 478289542Scem error = ntb_xeon_init_dev(ntb); 479289209Scem if (error) 480289209Scem goto out; 481289540Scem error = ntb_init_isr(ntb); 482289209Scem if (error) 483289209Scem goto out; 484250079Scarl 485250079Scarl pci_enable_busmaster(ntb->device); 486250079Scarl 487289209Scemout: 488289209Scem if (error != 0) 489289209Scem ntb_detach(device); 490250079Scarl return (error); 491250079Scarl} 492250079Scarl 493250079Scarlstatic int 494250079Scarlntb_detach(device_t device) 495250079Scarl{ 496289209Scem struct ntb_softc *ntb; 497250079Scarl 498289209Scem ntb = DEVICE2SOFTC(device); 499289542Scem 500289542Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 501250079Scarl callout_drain(&ntb->heartbeat_timer); 502250079Scarl callout_drain(&ntb->lr_timer); 503289272Scem if (ntb->type == NTB_XEON) 504289272Scem ntb_teardown_xeon(ntb); 505250079Scarl ntb_teardown_interrupts(ntb); 506289397Scem 507289542Scem mtx_destroy(&ntb->db_mask_lock); 508289546Scem mtx_destroy(&ntb->ctx_lock); 509289542Scem 510289397Scem /* 511289397Scem * Redetect total MWs so we unmap properly -- in case we lowered the 512289397Scem * maximum to work around Xeon errata. 513289397Scem */ 514289397Scem ntb_detect_max_mw(ntb); 515250079Scarl ntb_unmap_pci_bar(ntb); 516250079Scarl 517250079Scarl return (0); 518250079Scarl} 519250079Scarl 520289542Scem/* 521289542Scem * Driver internal routines 522289542Scem */ 523289539Scemstatic inline enum ntb_bar 524289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 525289539Scem{ 526289539Scem 527289543Scem KASSERT(mw < ntb->mw_count || 528289543Scem (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx), 529289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 530289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 531289539Scem 532289542Scem return (ntb->reg->mw_bar[mw]); 533289539Scem} 534289539Scem 535289546Scemstatic inline bool 536289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 537289546Scem{ 538289546Scem /* XXX This assertion could be stronger. */ 539289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 540289546Scem return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 541289546Scem} 542289546Scem 543289546Scemstatic inline void 544289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 545289546Scem uint32_t *xlat, uint32_t *lmt) 546289546Scem{ 547289546Scem uint32_t basev, lmtv, xlatv; 548289546Scem 549289546Scem switch (bar) { 550289546Scem case NTB_B2B_BAR_1: 551289546Scem basev = ntb->xlat_reg->bar2_base; 552289546Scem lmtv = ntb->xlat_reg->bar2_limit; 553289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 554289546Scem break; 555289546Scem case NTB_B2B_BAR_2: 556289546Scem basev = ntb->xlat_reg->bar4_base; 557289546Scem lmtv = ntb->xlat_reg->bar4_limit; 558289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 559289546Scem break; 560289546Scem case NTB_B2B_BAR_3: 561289546Scem basev = ntb->xlat_reg->bar5_base; 562289546Scem lmtv = ntb->xlat_reg->bar5_limit; 563289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 564289546Scem break; 565289546Scem default: 566289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 567289546Scem ("bad bar")); 568289546Scem basev = lmtv = xlatv = 0; 569289546Scem break; 570289546Scem } 571289546Scem 572289546Scem if (base != NULL) 573289546Scem *base = basev; 574289546Scem if (xlat != NULL) 575289546Scem *xlat = xlatv; 576289546Scem if (lmt != NULL) 577289546Scem *lmt = lmtv; 578289546Scem} 579289546Scem 580250079Scarlstatic int 581255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 582250079Scarl{ 583255272Scarl int rc; 584250079Scarl 585250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 586289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 587255272Scarl if (rc != 0) 588289541Scem goto out; 589255272Scarl 590289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 591289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 592255272Scarl if (rc != 0) 593289541Scem goto out; 594289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 595289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 596289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 597255272Scarl 598289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 599289543Scem /* XXX Are shared MW B2Bs write-combining? */ 600289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 601289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 602255279Scarl else 603289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 604289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 605289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 606289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 607289543Scem 608289397Scem if (!HAS_FEATURE(NTB_SPLIT_BAR)) 609289541Scem goto out; 610289397Scem 611289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 612289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 613289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 614289397Scem else 615289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 616289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 617289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 618289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 619250079Scarl 620289541Scemout: 621289209Scem if (rc != 0) 622255272Scarl device_printf(ntb->device, 623255272Scarl "unable to allocate pci resource\n"); 624255272Scarl return (rc); 625255272Scarl} 626255272Scarl 627289541Scemstatic void 628289541Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 629289541Scem{ 630289541Scem 631289541Scem device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n", 632289541Scem bar->size, bar->vbase, (void *)(bar->pbase)); 633289541Scem} 634289541Scem 635255272Scarlstatic int 636255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 637255272Scarl{ 638255272Scarl 639255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 640289209Scem &bar->pci_resource_id, RF_ACTIVE); 641255272Scarl if (bar->pci_resource == NULL) 642255272Scarl return (ENXIO); 643289209Scem 644289209Scem save_bar_parameters(bar); 645289541Scem print_map_success(ntb, bar); 646289209Scem return (0); 647255272Scarl} 648255272Scarl 649255272Scarlstatic int 650255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 651255272Scarl{ 652255272Scarl int rc; 653255276Scarl uint8_t bar_size_bits = 0; 654255272Scarl 655289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 656289209Scem &bar->pci_resource_id, RF_ACTIVE); 657250079Scarl 658255272Scarl if (bar->pci_resource == NULL) 659255272Scarl return (ENXIO); 660255276Scarl 661289209Scem save_bar_parameters(bar); 662289209Scem /* 663289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 664289209Scem * hardware issue. To work around this, query the size it should be 665289209Scem * configured to by the device and modify the resource to correspond to 666289209Scem * this new size. The BIOS on systems with this problem is required to 667289209Scem * provide enough address space to allow the driver to make this change 668289209Scem * safely. 669289209Scem * 670289209Scem * Ideally I could have just specified the size when I allocated the 671289209Scem * resource like: 672289209Scem * bus_alloc_resource(ntb->device, 673289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 674289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 675289209Scem * but the PCI driver does not honor the size in this call, so we have 676289209Scem * to modify it after the fact. 677289209Scem */ 678289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 679289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 680289209Scem bar_size_bits = pci_read_config(ntb->device, 681289209Scem XEON_PBAR23SZ_OFFSET, 1); 682289209Scem else 683289209Scem bar_size_bits = pci_read_config(ntb->device, 684289209Scem XEON_PBAR45SZ_OFFSET, 1); 685289209Scem 686289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 687289209Scem bar->pci_resource, bar->pbase, 688289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 689255272Scarl if (rc != 0) { 690289209Scem device_printf(ntb->device, 691289209Scem "unable to resize bar\n"); 692255272Scarl return (rc); 693250079Scarl } 694289209Scem 695289209Scem save_bar_parameters(bar); 696250079Scarl } 697289209Scem 698289209Scem /* Mark bar region as write combining to improve performance. */ 699289209Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 700289209Scem VM_MEMATTR_WRITE_COMBINING); 701289209Scem if (rc != 0) { 702289209Scem device_printf(ntb->device, 703289209Scem "unable to mark bar as WRITE_COMBINING\n"); 704289209Scem return (rc); 705289209Scem } 706289541Scem print_map_success(ntb, bar); 707250079Scarl return (0); 708250079Scarl} 709250079Scarl 710250079Scarlstatic void 711250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 712250079Scarl{ 713250079Scarl struct ntb_pci_bar_info *current_bar; 714250079Scarl int i; 715250079Scarl 716289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 717250079Scarl current_bar = &ntb->bar_info[i]; 718250079Scarl if (current_bar->pci_resource != NULL) 719250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 720250079Scarl current_bar->pci_resource_id, 721250079Scarl current_bar->pci_resource); 722250079Scarl } 723250079Scarl} 724250079Scarl 725250079Scarlstatic int 726289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 727250079Scarl{ 728289342Scem uint32_t i; 729289342Scem int rc; 730289342Scem 731289342Scem for (i = 0; i < num_vectors; i++) { 732289342Scem ntb->int_info[i].rid = i + 1; 733289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 734289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 735289342Scem if (ntb->int_info[i].res == NULL) { 736289342Scem device_printf(ntb->device, 737289342Scem "bus_alloc_resource failed\n"); 738289342Scem return (ENOMEM); 739289342Scem } 740289342Scem ntb->int_info[i].tag = NULL; 741289342Scem ntb->allocated_interrupts++; 742289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 743289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 744289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 745289342Scem if (rc != 0) { 746289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 747289342Scem return (ENXIO); 748289342Scem } 749289342Scem } 750289342Scem return (0); 751289342Scem} 752289342Scem 753289344Scem/* 754289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 755289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 756289344Scem * should be okay. This tunable should enable us to test that hypothesis 757289344Scem * when someone gets their hands on some Xeon hardware. 758289344Scem */ 759289344Scemstatic int ntb_force_remap_mode; 760289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 761289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 762289344Scem " to a smaller number of ithreads, even if the desired number are " 763289344Scem "available"); 764289344Scem 765289344Scem/* 766289344Scem * In case it is NOT ok, give consumers an abort button. 767289344Scem */ 768289344Scemstatic int ntb_prefer_intx; 769289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 770289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 771289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 772289344Scem "behavior)"); 773289344Scem 774289344Scem/* 775289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 776289344Scem * round-robin fashion. 777289344Scem */ 778289342Scemstatic int 779289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 780289344Scem{ 781289344Scem u_int *vectors; 782289344Scem uint32_t i; 783289344Scem int rc; 784289344Scem 785289344Scem if (ntb_prefer_intx != 0) 786289344Scem return (ENXIO); 787289344Scem 788289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 789289344Scem 790289344Scem for (i = 0; i < desired; i++) 791289344Scem vectors[i] = (i % avail) + 1; 792289344Scem 793289344Scem rc = pci_remap_msix(dev, desired, vectors); 794289344Scem free(vectors, M_NTB); 795289344Scem return (rc); 796289344Scem} 797289344Scem 798289344Scemstatic int 799289540Scemntb_init_isr(struct ntb_softc *ntb) 800289342Scem{ 801289344Scem uint32_t desired_vectors, num_vectors; 802289342Scem int rc; 803250079Scarl 804250079Scarl ntb->allocated_interrupts = 0; 805289542Scem ntb->last_ts = ticks; 806289347Scem 807250079Scarl /* 808289546Scem * Mask all doorbell interrupts. 809250079Scarl */ 810289546Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 811250079Scarl 812289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 813289539Scem ntb->db_count); 814289344Scem if (desired_vectors >= 1) { 815289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 816250079Scarl 817289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 818289344Scem num_vectors == desired_vectors) 819289344Scem num_vectors--; 820289344Scem 821289344Scem if (rc == 0 && num_vectors < desired_vectors) { 822289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 823289344Scem num_vectors); 824289344Scem if (rc == 0) 825289344Scem num_vectors = desired_vectors; 826289344Scem else 827289344Scem pci_release_msi(ntb->device); 828289344Scem } 829289344Scem if (rc != 0) 830289344Scem num_vectors = 1; 831289344Scem } else 832289344Scem num_vectors = 1; 833289344Scem 834289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 835289539Scem ntb->db_vec_count = 1; 836289539Scem ntb->db_vec_shift = ntb->db_count; 837289539Scem rc = ntb_setup_legacy_interrupt(ntb); 838289539Scem } else { 839289546Scem ntb_create_msix_vec(ntb, num_vectors); 840289540Scem rc = ntb_setup_msix(ntb, num_vectors); 841289539Scem } 842289539Scem if (rc != 0) { 843289539Scem device_printf(ntb->device, 844289539Scem "Error allocating interrupts: %d\n", rc); 845289546Scem ntb_free_msix_vec(ntb); 846289396Scem } 847289396Scem 848289342Scem return (rc); 849289342Scem} 850289342Scem 851289342Scemstatic int 852289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 853289342Scem{ 854289342Scem int rc; 855289342Scem 856289342Scem ntb->int_info[0].rid = 0; 857289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 858289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 859289342Scem if (ntb->int_info[0].res == NULL) { 860289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 861289342Scem return (ENOMEM); 862250079Scarl } 863250079Scarl 864289342Scem ntb->int_info[0].tag = NULL; 865289342Scem ntb->allocated_interrupts = 1; 866289342Scem 867289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 868289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 869289342Scem ntb, &ntb->int_info[0].tag); 870289342Scem if (rc != 0) { 871289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 872289342Scem return (ENXIO); 873289342Scem } 874289342Scem 875250079Scarl return (0); 876250079Scarl} 877250079Scarl 878250079Scarlstatic void 879250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 880250079Scarl{ 881250079Scarl struct ntb_int_info *current_int; 882250079Scarl int i; 883250079Scarl 884289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 885250079Scarl current_int = &ntb->int_info[i]; 886250079Scarl if (current_int->tag != NULL) 887250079Scarl bus_teardown_intr(ntb->device, current_int->res, 888250079Scarl current_int->tag); 889250079Scarl 890250079Scarl if (current_int->res != NULL) 891250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 892250079Scarl rman_get_rid(current_int->res), current_int->res); 893250079Scarl } 894250079Scarl 895289546Scem ntb_free_msix_vec(ntb); 896250079Scarl pci_release_msi(ntb->device); 897250079Scarl} 898250079Scarl 899289347Scem/* 900289347Scem * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon. Abstract it 901289347Scem * out to make code clearer. 902289347Scem */ 903289539Scemstatic inline uint64_t 904289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 905289347Scem{ 906289347Scem 907289347Scem if (ntb->type == NTB_SOC) 908289347Scem return (ntb_reg_read(8, regoff)); 909289347Scem 910289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 911289347Scem 912289347Scem return (ntb_reg_read(2, regoff)); 913289347Scem} 914289347Scem 915289539Scemstatic inline void 916289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 917289347Scem{ 918289347Scem 919289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 920289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 921289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 922289542Scem (uintmax_t)ntb->db_valid_mask)); 923289542Scem 924289542Scem if (regoff == ntb->reg_ofs.ldb_mask) 925289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 926289542Scem 927289347Scem if (ntb->type == NTB_SOC) { 928289347Scem ntb_reg_write(8, regoff, val); 929289347Scem return; 930289347Scem } 931289347Scem 932289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 933289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 934289347Scem} 935289347Scem 936289546Scemvoid 937289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 938289542Scem{ 939289542Scem 940289546Scem DB_MASK_LOCK(ntb); 941289542Scem ntb->db_mask |= bits; 942289546Scem db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask); 943289546Scem DB_MASK_UNLOCK(ntb); 944289542Scem} 945289542Scem 946289546Scemvoid 947289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 948289542Scem{ 949289542Scem 950289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 951289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 952289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 953289542Scem (uintmax_t)ntb->db_valid_mask)); 954289542Scem 955289546Scem DB_MASK_LOCK(ntb); 956289542Scem ntb->db_mask &= ~bits; 957289546Scem db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask); 958289546Scem DB_MASK_UNLOCK(ntb); 959289542Scem} 960289542Scem 961289546Scemuint64_t 962289546Scemntb_db_read(struct ntb_softc *ntb) 963289281Scem{ 964289281Scem 965289546Scem return (db_ioread(ntb, ntb->reg_ofs.ldb)); 966289281Scem} 967289281Scem 968289546Scemvoid 969289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 970289281Scem{ 971289281Scem 972289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 973289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 974289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 975289546Scem (uintmax_t)ntb->db_valid_mask)); 976289546Scem 977289546Scem db_iowrite(ntb, ntb->reg_ofs.ldb, bits); 978289281Scem} 979289281Scem 980289540Scemstatic inline uint64_t 981289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 982250079Scarl{ 983289540Scem uint64_t shift, mask; 984250079Scarl 985289540Scem shift = ntb->db_vec_shift; 986289540Scem mask = (1ull << shift) - 1; 987289540Scem return (mask << (shift * db_vector)); 988250079Scarl} 989250079Scarl 990250079Scarlstatic void 991289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 992250079Scarl{ 993289540Scem uint64_t vec_mask; 994250079Scarl 995289542Scem ntb->last_ts = ticks; 996289546Scem vec_mask = ntb_vec_mask(ntb, vec); 997250079Scarl 998289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 999289546Scem if (ntb_poll_link(ntb)) 1000289546Scem ntb_link_event(ntb); 1001289540Scem } 1002289540Scem 1003289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1004289546Scem ntb_db_event(ntb, vec); 1005289546Scem} 1006250079Scarl 1007289546Scemstatic void 1008289546Scemndev_vec_isr(void *arg) 1009289546Scem{ 1010289546Scem struct ntb_vec *nvec = arg; 1011250079Scarl 1012289546Scem ntb_interrupt(nvec->ntb, nvec->num); 1013250079Scarl} 1014250079Scarl 1015250079Scarlstatic void 1016289546Scemndev_irq_isr(void *arg) 1017250079Scarl{ 1018289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1019289546Scem ntb_interrupt(arg, 0); 1020250079Scarl} 1021250079Scarl 1022250079Scarlstatic int 1023289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1024250079Scarl{ 1025289342Scem uint32_t i; 1026250079Scarl 1027289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1028250079Scarl M_ZERO | M_WAITOK); 1029250079Scarl for (i = 0; i < num_vectors; i++) { 1030289546Scem ntb->msix_vec[i].num = i; 1031289546Scem ntb->msix_vec[i].ntb = ntb; 1032250079Scarl } 1033250079Scarl 1034250079Scarl return (0); 1035250079Scarl} 1036250079Scarl 1037250079Scarlstatic void 1038289546Scemntb_free_msix_vec(struct ntb_softc *ntb) 1039250079Scarl{ 1040250079Scarl 1041289546Scem if (ntb->msix_vec == NULL) 1042289539Scem return; 1043289539Scem 1044289546Scem free(ntb->msix_vec, M_NTB); 1045289546Scem ntb->msix_vec = NULL; 1046250079Scarl} 1047250079Scarl 1048250079Scarlstatic struct ntb_hw_info * 1049250079Scarlntb_get_device_info(uint32_t device_id) 1050250079Scarl{ 1051250079Scarl struct ntb_hw_info *ep = pci_ids; 1052250079Scarl 1053250079Scarl while (ep->device_id) { 1054250079Scarl if (ep->device_id == device_id) 1055250079Scarl return (ep); 1056250079Scarl ++ep; 1057250079Scarl } 1058250079Scarl return (NULL); 1059250079Scarl} 1060250079Scarl 1061289272Scemstatic void 1062289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 1063250079Scarl{ 1064250079Scarl 1065289542Scem ntb_link_disable(ntb); 1066250079Scarl} 1067250079Scarl 1068289397Scemstatic void 1069289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 1070289397Scem{ 1071289397Scem 1072289397Scem if (ntb->type == NTB_SOC) { 1073289539Scem ntb->mw_count = SOC_MW_COUNT; 1074289397Scem return; 1075289397Scem } 1076289397Scem 1077289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1078289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1079289397Scem else 1080289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1081289397Scem} 1082289397Scem 1083250079Scarlstatic int 1084289348Scemntb_detect_xeon(struct ntb_softc *ntb) 1085250079Scarl{ 1086289348Scem uint8_t ppd, conn_type; 1087250079Scarl 1088289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1089289348Scem ntb->ppd = ppd; 1090250079Scarl 1091289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1092289257Scem ntb->dev_type = NTB_DEV_USD; 1093289257Scem else 1094289257Scem ntb->dev_type = NTB_DEV_DSD; 1095289257Scem 1096289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1097289397Scem ntb->features |= NTB_SPLIT_BAR; 1098289397Scem 1099289542Scem /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1100289542Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1101289542Scem ntb->features |= NTB_SDOORBELL_LOCKUP; 1102289542Scem 1103289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1104289348Scem switch (conn_type) { 1105289348Scem case NTB_CONN_B2B: 1106289348Scem ntb->conn_type = conn_type; 1107289348Scem break; 1108289348Scem case NTB_CONN_RP: 1109289348Scem case NTB_CONN_TRANSPARENT: 1110289348Scem default: 1111289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1112289348Scem (unsigned)conn_type); 1113289348Scem return (ENXIO); 1114289348Scem } 1115289348Scem return (0); 1116289348Scem} 1117289348Scem 1118289348Scemstatic int 1119289348Scemntb_detect_soc(struct ntb_softc *ntb) 1120289348Scem{ 1121289348Scem uint32_t ppd, conn_type; 1122289348Scem 1123289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1124289348Scem ntb->ppd = ppd; 1125289348Scem 1126289348Scem if ((ppd & SOC_PPD_DEV_TYPE) != 0) 1127289348Scem ntb->dev_type = NTB_DEV_DSD; 1128289348Scem else 1129289348Scem ntb->dev_type = NTB_DEV_USD; 1130289348Scem 1131289348Scem conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8; 1132289348Scem switch (conn_type) { 1133289348Scem case NTB_CONN_B2B: 1134289348Scem ntb->conn_type = conn_type; 1135289348Scem break; 1136289348Scem default: 1137289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1138289348Scem return (ENXIO); 1139289348Scem } 1140289348Scem return (0); 1141289348Scem} 1142289348Scem 1143289348Scemstatic int 1144289542Scemntb_xeon_init_dev(struct ntb_softc *ntb) 1145289348Scem{ 1146289542Scem int rc; 1147289348Scem 1148289257Scem ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; 1149289257Scem ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; 1150289257Scem ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 1151289257Scem ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; 1152289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1153289397Scem ntb->reg_ofs.bar5_xlat = XEON_SBAR5XLAT_OFFSET; 1154289542Scem ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 1155289257Scem 1156289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1157289542Scem ntb->db_count = XEON_DB_COUNT; 1158289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1159289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1160289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1161289257Scem 1162289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1163250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1164289348Scem ntb->conn_type); 1165250079Scarl return (ENXIO); 1166250079Scarl } 1167250079Scarl 1168289542Scem ntb->reg = &xeon_reg; 1169289542Scem ntb->peer_reg = &xeon_b2b_reg; 1170289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1171289542Scem 1172289208Scem /* 1173289208Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1174289208Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1175289208Scem * which may hang the system. To workaround this use the second memory 1176289208Scem * window to access the interrupt and scratch pad registers on the 1177289208Scem * remote system. 1178289208Scem */ 1179289543Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1180289543Scem /* Use the last MW for mapping remote spad */ 1181289542Scem ntb->b2b_mw_idx = ntb->mw_count - 1; 1182289543Scem else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1183289208Scem /* 1184289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1185289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1186289542Scem * since bit 14 is the last bit. 1187289542Scem * 1188289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1189289542Scem * anyway. Nor for non-B2B connection types. 1190289542Scem */ 1191289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1192250079Scarl 1193289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1194250079Scarl 1195289542Scem if (ntb->dev_type == NTB_DEV_USD) 1196289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1197289542Scem &xeon_b2b_usd_addr); 1198289542Scem else 1199289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1200289542Scem &xeon_b2b_dsd_addr); 1201289542Scem if (rc != 0) 1202289542Scem return (rc); 1203289271Scem 1204250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1205289542Scem ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 1206289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1207255279Scarl 1208255269Scarl /* Enable link training */ 1209289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1210250079Scarl 1211250079Scarl return (0); 1212250079Scarl} 1213250079Scarl 1214250079Scarlstatic int 1215289542Scemntb_soc_init_dev(struct ntb_softc *ntb) 1216250079Scarl{ 1217250079Scarl 1218289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1219289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1220250079Scarl 1221289255Scem ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; 1222289255Scem ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; 1223289255Scem ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; 1224250079Scarl ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 1225250079Scarl ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 1226250079Scarl 1227289542Scem ntb->spad_count = SOC_SPAD_COUNT; 1228289539Scem ntb->db_count = SOC_DB_COUNT; 1229289539Scem ntb->db_vec_count = SOC_DB_MSIX_VECTOR_COUNT; 1230289539Scem ntb->db_vec_shift = SOC_DB_MSIX_VECTOR_SHIFT; 1231289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1232250079Scarl 1233289542Scem ntb->reg = &soc_reg; 1234289542Scem ntb->peer_reg = &soc_b2b_reg; 1235289542Scem ntb->xlat_reg = &soc_sec_xlat; 1236289542Scem 1237250079Scarl /* 1238250079Scarl * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 1239250079Scarl * resolved. Mask transaction layer internal parity errors. 1240250079Scarl */ 1241250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1242250079Scarl 1243255279Scarl configure_soc_secondary_side_bars(ntb); 1244250079Scarl 1245250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1246255278Scarl ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 1247250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1248289209Scem 1249289542Scem /* Initiate PCI-E link training */ 1250289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1251250079Scarl 1252289542Scem callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb); 1253289542Scem 1254250079Scarl return (0); 1255250079Scarl} 1256250079Scarl 1257289542Scem/* XXX: Linux driver doesn't seem to do any of this for SoC. */ 1258255279Scarlstatic void 1259255279Scarlconfigure_soc_secondary_side_bars(struct ntb_softc *ntb) 1260255279Scarl{ 1261255279Scarl 1262255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1263289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1264289542Scem XEON_B2B_BAR2_DSD_ADDR64); 1265289542Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1266289542Scem XEON_B2B_BAR4_DSD_ADDR64); 1267289542Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64); 1268289542Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64); 1269255279Scarl } else { 1270289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1271289542Scem XEON_B2B_BAR2_USD_ADDR64); 1272289542Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1273289542Scem XEON_B2B_BAR4_USD_ADDR64); 1274289542Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64); 1275289542Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64); 1276255279Scarl } 1277255279Scarl} 1278255279Scarl 1279289543Scem 1280289543Scem/* 1281289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1282289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1283289543Scem * remains for use by a higher layer. 1284289543Scem * 1285289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1286289543Scem * MW size is sufficiently large. 1287289543Scem */ 1288289543Scemstatic unsigned int ntb_b2b_mw_share; 1289289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1290289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1291289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1292289543Scem "value here."); 1293289543Scem 1294289543Scemstatic void 1295289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1296289543Scem enum ntb_bar regbar) 1297289543Scem{ 1298289543Scem struct ntb_pci_bar_info *bar; 1299289543Scem uint8_t bar_sz; 1300289543Scem 1301289543Scem if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1302289543Scem return; 1303289543Scem 1304289543Scem bar = &ntb->bar_info[idx]; 1305289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1306289543Scem if (idx == regbar) { 1307289543Scem if (ntb->b2b_off != 0) 1308289543Scem bar_sz--; 1309289543Scem else 1310289543Scem bar_sz = 0; 1311289543Scem } 1312289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1313289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1314289543Scem (void)bar_sz; 1315289543Scem} 1316289543Scem 1317289543Scemstatic void 1318289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1319289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1320289543Scem{ 1321289546Scem uint64_t reg_val; 1322289546Scem uint32_t base_reg, lmt_reg; 1323289543Scem 1324289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1325289546Scem if (idx == regbar) 1326289546Scem bar_addr += ntb->b2b_off; 1327289543Scem 1328289546Scem if (!bar_is_64bit(ntb, idx)) { 1329289546Scem ntb_reg_write(4, base_reg, bar_addr); 1330289546Scem reg_val = ntb_reg_read(4, base_reg); 1331289546Scem (void)reg_val; 1332289546Scem 1333289546Scem ntb_reg_write(4, lmt_reg, bar_addr); 1334289546Scem reg_val = ntb_reg_read(4, lmt_reg); 1335289546Scem (void)reg_val; 1336289543Scem } else { 1337289546Scem ntb_reg_write(8, base_reg, bar_addr); 1338289546Scem reg_val = ntb_reg_read(8, base_reg); 1339289546Scem (void)reg_val; 1340289546Scem 1341289546Scem ntb_reg_write(8, lmt_reg, bar_addr); 1342289546Scem reg_val = ntb_reg_read(8, lmt_reg); 1343289546Scem (void)reg_val; 1344289543Scem } 1345289543Scem} 1346289543Scem 1347289543Scemstatic void 1348289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1349289543Scem{ 1350289543Scem struct ntb_pci_bar_info *bar; 1351289543Scem 1352289543Scem bar = &ntb->bar_info[idx]; 1353289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1354289543Scem ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1355289543Scem base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1356289543Scem } else { 1357289543Scem ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1358289543Scem base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1359289543Scem } 1360289543Scem (void)base_addr; 1361289543Scem} 1362289543Scem 1363289542Scemstatic int 1364289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1365289542Scem const struct ntb_b2b_addr *peer_addr) 1366255279Scarl{ 1367289543Scem struct ntb_pci_bar_info *b2b_bar; 1368289543Scem vm_size_t bar_size; 1369289543Scem uint64_t bar_addr; 1370289543Scem enum ntb_bar b2b_bar_num, i; 1371255279Scarl 1372289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1373289543Scem b2b_bar = NULL; 1374289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1375289543Scem ntb->b2b_off = 0; 1376289543Scem } else { 1377289543Scem b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1378289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1379289543Scem ("invalid b2b mw bar")); 1380289543Scem 1381289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1382289543Scem bar_size = b2b_bar->size; 1383289543Scem 1384289543Scem if (ntb_b2b_mw_share != 0 && 1385289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1386289543Scem ntb->b2b_off = bar_size >> 1; 1387289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1388289543Scem ntb->b2b_off = 0; 1389289543Scem ntb->mw_count--; 1390289543Scem } else { 1391289543Scem device_printf(ntb->device, 1392289543Scem "B2B bar size is too small!\n"); 1393289543Scem return (EIO); 1394289543Scem } 1395255279Scarl } 1396289542Scem 1397289543Scem /* 1398289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1399289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1400289543Scem */ 1401289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1402289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1403289543Scem 1404289543Scem bar_addr = 0; 1405289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1406289543Scem bar_addr = addr->bar0_addr; 1407289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1408289543Scem bar_addr = addr->bar2_addr64; 1409289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1410289543Scem bar_addr = addr->bar4_addr64; 1411289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1412289543Scem bar_addr = addr->bar4_addr32; 1413289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1414289543Scem bar_addr = addr->bar5_addr32; 1415289543Scem else 1416289543Scem KASSERT(false, ("invalid bar")); 1417289543Scem 1418289543Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1419289543Scem 1420289543Scem /* 1421289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1422289543Scem * register BAR. The B2B BAR is either disabled above or configured 1423289543Scem * half-size. It starts at PBAR xlat + offset. 1424289543Scem * 1425289543Scem * Also set up incoming BAR limits == base (zero length window). 1426289543Scem */ 1427289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1428289543Scem b2b_bar_num); 1429289542Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1430289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1431289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1432289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1433289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1434289542Scem } else 1435289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1436289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1437289543Scem 1438289543Scem /* Zero incoming translation addrs */ 1439289543Scem ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1440289543Scem ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1441289543Scem 1442289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1443289543Scem ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1444289543Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1445289543Scem 1446289543Scem /* Set outgoing translation offsets */ 1447289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1448289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1449289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1450289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1451289543Scem } else 1452289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1453289543Scem 1454289543Scem /* Set the translation offset for B2B registers */ 1455289543Scem bar_addr = 0; 1456289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1457289543Scem bar_addr = peer_addr->bar0_addr; 1458289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1459289543Scem bar_addr = peer_addr->bar2_addr64; 1460289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1461289543Scem bar_addr = peer_addr->bar4_addr64; 1462289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1463289543Scem bar_addr = peer_addr->bar4_addr32; 1464289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1465289543Scem bar_addr = peer_addr->bar5_addr32; 1466289543Scem else 1467289543Scem KASSERT(false, ("invalid bar")); 1468289543Scem 1469289543Scem /* 1470289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1471289543Scem * at a time. 1472289543Scem */ 1473289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1474289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1475289542Scem return (0); 1476255279Scarl} 1477255279Scarl 1478289546Scemstatic inline bool 1479289546Scemlink_is_up(struct ntb_softc *ntb) 1480289546Scem{ 1481289546Scem 1482289546Scem if (ntb->type == NTB_XEON) 1483289546Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1484289546Scem 1485289546Scem KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1486289546Scem return ((ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0); 1487289546Scem} 1488289546Scem 1489289546Scemstatic inline bool 1490289546Scemsoc_link_is_err(struct ntb_softc *ntb) 1491289546Scem{ 1492289546Scem uint32_t status; 1493289546Scem 1494289546Scem KASSERT(ntb->type == NTB_SOC, ("ntb type")); 1495289546Scem 1496289546Scem status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1497289546Scem if ((status & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1498289546Scem return (true); 1499289546Scem 1500289546Scem status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1501289546Scem return ((status & SOC_IBIST_ERR_OFLOW) != 0); 1502289546Scem} 1503289546Scem 1504255281Scarl/* SOC does not have link status interrupt, poll on that platform */ 1505250079Scarlstatic void 1506289542Scemsoc_link_hb(void *arg) 1507250079Scarl{ 1508250079Scarl struct ntb_softc *ntb = arg; 1509289546Scem sbintime_t timo, poll_ts; 1510250079Scarl 1511289546Scem timo = NTB_HB_TIMEOUT * hz; 1512289546Scem poll_ts = ntb->last_ts + timo; 1513289546Scem 1514289542Scem /* 1515289542Scem * Delay polling the link status if an interrupt was received, unless 1516289542Scem * the cached link status says the link is down. 1517289542Scem */ 1518289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1519289546Scem timo = poll_ts - ticks; 1520289542Scem goto out; 1521289546Scem } 1522289542Scem 1523289546Scem if (ntb_poll_link(ntb)) 1524289546Scem ntb_link_event(ntb); 1525289542Scem 1526289546Scem if (!link_is_up(ntb) && soc_link_is_err(ntb)) { 1527289546Scem /* Link is down with error, proceed with recovery */ 1528289546Scem callout_reset(&ntb->lr_timer, 0, recover_soc_link, ntb); 1529289546Scem return; 1530250079Scarl } 1531250079Scarl 1532289542Scemout: 1533289546Scem callout_reset(&ntb->heartbeat_timer, timo, soc_link_hb, ntb); 1534250079Scarl} 1535250079Scarl 1536250079Scarlstatic void 1537250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb) 1538250079Scarl{ 1539250079Scarl uint32_t status; 1540250079Scarl 1541250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1542255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1543255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1544255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1545255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1546250079Scarl 1547250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1548250079Scarl pause("ModPhy", hz / 10); 1549250079Scarl 1550250079Scarl /* Clear AER Errors, write to clear */ 1551255278Scarl status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1552250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1553255278Scarl ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1554250079Scarl 1555250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1556255278Scarl status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1557250079Scarl status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1558255278Scarl ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1559250079Scarl 1560250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1561255278Scarl status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1562250079Scarl status |= SOC_DESKEWSTS_DBERR; 1563255278Scarl ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1564250079Scarl 1565255278Scarl status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1566250079Scarl status &= SOC_IBIST_ERR_OFLOW; 1567255278Scarl ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1568250079Scarl 1569250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1570255278Scarl status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1571250079Scarl status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1572255278Scarl ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1573250079Scarl} 1574250079Scarl 1575289546Scem/* 1576289546Scem * ntb_set_ctx() - associate a driver context with an ntb device 1577289546Scem * @ntb: NTB device context 1578289546Scem * @ctx: Driver context 1579289546Scem * @ctx_ops: Driver context operations 1580289546Scem * 1581289546Scem * Associate a driver context and operations with a ntb device. The context is 1582289546Scem * provided by the client driver, and the driver may associate a different 1583289546Scem * context with each ntb device. 1584289546Scem * 1585289546Scem * Return: Zero if the context is associated, otherwise an error number. 1586289546Scem */ 1587289546Scemint 1588289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 1589250079Scarl{ 1590250079Scarl 1591289546Scem if (ctx == NULL || ops == NULL) 1592289546Scem return (EINVAL); 1593289546Scem if (ntb->ctx_ops != NULL) 1594289546Scem return (EINVAL); 1595250079Scarl 1596289546Scem CTX_LOCK(ntb); 1597289546Scem if (ntb->ctx_ops != NULL) { 1598289546Scem CTX_UNLOCK(ntb); 1599289546Scem return (EINVAL); 1600250079Scarl } 1601289546Scem ntb->ntb_ctx = ctx; 1602289546Scem ntb->ctx_ops = ops; 1603289546Scem CTX_UNLOCK(ntb); 1604250079Scarl 1605289546Scem return (0); 1606250079Scarl} 1607250079Scarl 1608289546Scem/* 1609289546Scem * It is expected that this will only be used from contexts where the ctx_lock 1610289546Scem * is not needed to protect ntb_ctx lifetime. 1611289546Scem */ 1612289546Scemvoid * 1613289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 1614289546Scem{ 1615289546Scem 1616289546Scem KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 1617289546Scem if (ops != NULL) 1618289546Scem *ops = ntb->ctx_ops; 1619289546Scem return (ntb->ntb_ctx); 1620289546Scem} 1621289546Scem 1622289546Scem/* 1623289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device 1624289546Scem * @ntb: NTB device context 1625289546Scem * 1626289546Scem * Clear any association that may exist between a driver context and the ntb 1627289546Scem * device. 1628289546Scem */ 1629289546Scemvoid 1630289546Scemntb_clear_ctx(struct ntb_softc *ntb) 1631289546Scem{ 1632289546Scem 1633289546Scem CTX_LOCK(ntb); 1634289546Scem ntb->ntb_ctx = NULL; 1635289546Scem ntb->ctx_ops = NULL; 1636289546Scem CTX_UNLOCK(ntb); 1637289546Scem} 1638289546Scem 1639289546Scem/* 1640289546Scem * ntb_link_event() - notify driver context of a change in link status 1641289546Scem * @ntb: NTB device context 1642289546Scem * 1643289546Scem * Notify the driver context that the link status may have changed. The driver 1644289546Scem * should call ntb_link_is_up() to get the current status. 1645289546Scem */ 1646289546Scemvoid 1647289546Scemntb_link_event(struct ntb_softc *ntb) 1648289546Scem{ 1649289546Scem 1650289546Scem CTX_LOCK(ntb); 1651289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 1652289546Scem ntb->ctx_ops->link_event(ntb->ntb_ctx); 1653289546Scem CTX_UNLOCK(ntb); 1654289546Scem} 1655289546Scem 1656289546Scem/* 1657289546Scem * ntb_db_event() - notify driver context of a doorbell event 1658289546Scem * @ntb: NTB device context 1659289546Scem * @vector: Interrupt vector number 1660289546Scem * 1661289546Scem * Notify the driver context of a doorbell event. If hardware supports 1662289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which 1663289546Scem * vector received the interrupt. The vector number is relative to the first 1664289546Scem * vector used for doorbells, starting at zero, and must be less than 1665289546Scem * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 1666289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of 1667289546Scem * those bits are associated with the vector number. 1668289546Scem */ 1669250079Scarlstatic void 1670289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec) 1671289272Scem{ 1672289546Scem 1673289546Scem CTX_LOCK(ntb); 1674289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 1675289546Scem ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 1676289546Scem CTX_UNLOCK(ntb); 1677289546Scem} 1678289546Scem 1679289546Scem/* 1680289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb 1681289546Scem * @ntb: NTB device context 1682289546Scem * @max_speed: The maximum link speed expressed as PCIe generation number[0] 1683289546Scem * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 1684289546Scem * 1685289546Scem * Enable the link on the secondary side of the ntb. This can only be done 1686289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1687289546Scem * should train the link to its maximum speed and width, or the requested speed 1688289546Scem * and width, whichever is smaller, if supported. 1689289546Scem * 1690289546Scem * Return: Zero on success, otherwise an error number. 1691289546Scem * 1692289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 1693289546Scem * and width input will be ignored. 1694289546Scem */ 1695289546Scemint 1696289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 1697289546Scem enum ntb_width w __unused) 1698289546Scem{ 1699289280Scem uint32_t cntl; 1700289272Scem 1701289542Scem if (ntb->type == NTB_SOC) { 1702289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1703289542Scem ntb->ppd | SOC_PPD_INIT_LINK, 4); 1704289546Scem return (0); 1705289542Scem } 1706289542Scem 1707289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1708289546Scem ntb_link_event(ntb); 1709289546Scem return (0); 1710289280Scem } 1711289280Scem 1712289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1713289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1714289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1715289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1716289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1717289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1718289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1719289546Scem return (0); 1720289272Scem} 1721289272Scem 1722289546Scem/* 1723289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb 1724289546Scem * @ntb: NTB device context 1725289546Scem * 1726289546Scem * Disable the link on the secondary side of the ntb. This can only be done 1727289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1728289546Scem * should disable the link. Returning from this call must indicate that a 1729289546Scem * barrier has passed, though with no more writes may pass in either direction 1730289546Scem * across the link, except if this call returns an error number. 1731289546Scem * 1732289546Scem * Return: Zero on success, otherwise an error number. 1733289546Scem */ 1734289546Scemint 1735289542Scemntb_link_disable(struct ntb_softc *ntb) 1736289272Scem{ 1737289272Scem uint32_t cntl; 1738289272Scem 1739289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1740289546Scem ntb_link_event(ntb); 1741289546Scem return (0); 1742289272Scem } 1743289272Scem 1744289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1745289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1746289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1747289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1748289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1749289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1750289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1751289546Scem return (0); 1752289272Scem} 1753289272Scem 1754289272Scemstatic void 1755250079Scarlrecover_soc_link(void *arg) 1756250079Scarl{ 1757250079Scarl struct ntb_softc *ntb = arg; 1758250079Scarl uint8_t speed, width; 1759250079Scarl uint32_t status32; 1760250079Scarl 1761250079Scarl soc_perform_link_restart(ntb); 1762250079Scarl 1763289232Scem /* 1764289232Scem * There is a potential race between the 2 NTB devices recovering at 1765289232Scem * the same time. If the times are the same, the link will not recover 1766289232Scem * and the driver will be stuck in this loop forever. Add a random 1767289232Scem * interval to the recovery time to prevent this race. 1768289232Scem */ 1769289232Scem status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1770289232Scem pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1771289232Scem 1772255278Scarl status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1773250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1774250079Scarl goto retry; 1775250079Scarl 1776255278Scarl status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1777250079Scarl if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1778250079Scarl goto retry; 1779250079Scarl 1780289542Scem status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1781289232Scem if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1782289232Scem goto out; 1783289232Scem 1784289542Scem status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1785289542Scem width = (status32 & NTB_LINK_WIDTH_MASK) >> 4; 1786289542Scem speed = (status32 & NTB_LINK_SPEED_MASK); 1787250079Scarl if (ntb->link_width != width || ntb->link_speed != speed) 1788250079Scarl goto retry; 1789250079Scarl 1790289232Scemout: 1791289542Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, 1792289542Scem ntb); 1793250079Scarl return; 1794250079Scarl 1795250079Scarlretry: 1796250079Scarl callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1797250079Scarl ntb); 1798250079Scarl} 1799250079Scarl 1800289546Scem/* 1801289546Scem * Polls the HW link status register(s); returns true if something has changed. 1802289546Scem */ 1803289546Scemstatic bool 1804289542Scemntb_poll_link(struct ntb_softc *ntb) 1805250079Scarl{ 1806250079Scarl uint32_t ntb_cntl; 1807289546Scem uint16_t reg_val; 1808250079Scarl 1809250079Scarl if (ntb->type == NTB_SOC) { 1810289542Scem ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1811289546Scem if (ntb_cntl == ntb->ntb_ctl) 1812289546Scem return (false); 1813289546Scem 1814289542Scem ntb->ntb_ctl = ntb_cntl; 1815289542Scem ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1816250079Scarl } else { 1817289546Scem db_iowrite(ntb, ntb->reg_ofs.ldb, ntb->db_link_mask); 1818250079Scarl 1819289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 1820289546Scem if (reg_val == ntb->lnk_sta) 1821289546Scem return (false); 1822250079Scarl 1823289546Scem ntb->lnk_sta = reg_val; 1824289542Scem } 1825289546Scem return (true); 1826289542Scem} 1827289542Scem 1828289546Scemstatic inline enum ntb_speed 1829289546Scemntb_link_sta_speed(struct ntb_softc *ntb) 1830250079Scarl{ 1831250079Scarl 1832289546Scem if (!link_is_up(ntb)) 1833289546Scem return (NTB_SPEED_NONE); 1834289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 1835250079Scarl} 1836250079Scarl 1837289546Scemstatic inline enum ntb_width 1838289546Scemntb_link_sta_width(struct ntb_softc *ntb) 1839250079Scarl{ 1840250079Scarl 1841289546Scem if (!link_is_up(ntb)) 1842289546Scem return (NTB_WIDTH_NONE); 1843289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 1844250079Scarl} 1845250079Scarl 1846289546Scem/* 1847289546Scem * Public API to the rest of the OS 1848250079Scarl */ 1849250079Scarl 1850250079Scarl/** 1851250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 1852250079Scarl * @ntb: pointer to ntb_softc instance 1853250079Scarl * 1854250079Scarl * This function returns the max 32bit scratchpad registers usable by the 1855250079Scarl * upper layer. 1856250079Scarl * 1857250079Scarl * RETURNS: total number of scratch pad registers available 1858250079Scarl */ 1859289208Scemuint8_t 1860250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 1861250079Scarl{ 1862250079Scarl 1863289539Scem return (ntb->spad_count); 1864250079Scarl} 1865250079Scarl 1866289396Scemuint8_t 1867289539Scemntb_mw_count(struct ntb_softc *ntb) 1868289396Scem{ 1869289396Scem 1870289539Scem return (ntb->mw_count); 1871289396Scem} 1872289396Scem 1873250079Scarl/** 1874289545Scem * ntb_spad_write() - write to the secondary scratchpad register 1875250079Scarl * @ntb: pointer to ntb_softc instance 1876250079Scarl * @idx: index to the scratchpad register, 0 based 1877250079Scarl * @val: the data value to put into the register 1878250079Scarl * 1879250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1880250079Scarl * register. The register resides on the secondary (external) side. 1881250079Scarl * 1882289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1883250079Scarl */ 1884250079Scarlint 1885289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1886250079Scarl{ 1887250079Scarl 1888289539Scem if (idx >= ntb->spad_count) 1889250079Scarl return (EINVAL); 1890250079Scarl 1891255278Scarl ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1892250079Scarl 1893250079Scarl return (0); 1894250079Scarl} 1895250079Scarl 1896250079Scarl/** 1897289545Scem * ntb_spad_read() - read from the primary scratchpad register 1898250079Scarl * @ntb: pointer to ntb_softc instance 1899250079Scarl * @idx: index to scratchpad register, 0 based 1900250079Scarl * @val: pointer to 32bit integer for storing the register value 1901250079Scarl * 1902250079Scarl * This function allows reading of the 32bit scratchpad register on 1903250079Scarl * the primary (internal) side. 1904250079Scarl * 1905289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1906250079Scarl */ 1907250079Scarlint 1908289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1909250079Scarl{ 1910250079Scarl 1911289539Scem if (idx >= ntb->spad_count) 1912250079Scarl return (EINVAL); 1913250079Scarl 1914255278Scarl *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1915250079Scarl 1916250079Scarl return (0); 1917250079Scarl} 1918250079Scarl 1919250079Scarl/** 1920289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register 1921250079Scarl * @ntb: pointer to ntb_softc instance 1922250079Scarl * @idx: index to the scratchpad register, 0 based 1923250079Scarl * @val: the data value to put into the register 1924250079Scarl * 1925250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1926250079Scarl * register. The register resides on the secondary (external) side. 1927250079Scarl * 1928289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1929250079Scarl */ 1930250079Scarlint 1931289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1932250079Scarl{ 1933250079Scarl 1934289539Scem if (idx >= ntb->spad_count) 1935250079Scarl return (EINVAL); 1936250079Scarl 1937289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1938255279Scarl ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1939255279Scarl else 1940289542Scem ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 1941250079Scarl 1942250079Scarl return (0); 1943250079Scarl} 1944250079Scarl 1945250079Scarl/** 1946289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register 1947250079Scarl * @ntb: pointer to ntb_softc instance 1948250079Scarl * @idx: index to scratchpad register, 0 based 1949250079Scarl * @val: pointer to 32bit integer for storing the register value 1950250079Scarl * 1951250079Scarl * This function allows reading of the 32bit scratchpad register on 1952250079Scarl * the primary (internal) side. 1953250079Scarl * 1954289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1955250079Scarl */ 1956250079Scarlint 1957289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1958250079Scarl{ 1959250079Scarl 1960289539Scem if (idx >= ntb->spad_count) 1961250079Scarl return (EINVAL); 1962250079Scarl 1963289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1964255279Scarl *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1965255279Scarl else 1966289542Scem *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 1967250079Scarl 1968250079Scarl return (0); 1969250079Scarl} 1970250079Scarl 1971289546Scem/* 1972289546Scem * ntb_mw_get_range() - get the range of a memory window 1973289546Scem * @ntb: NTB device context 1974289546Scem * @idx: Memory window number 1975289546Scem * @base: OUT - the base address for mapping the memory window 1976289546Scem * @size: OUT - the size for mapping the memory window 1977289546Scem * @align: OUT - the base alignment for translating the memory window 1978289546Scem * @align_size: OUT - the size alignment for translating the memory window 1979250079Scarl * 1980289546Scem * Get the range of a memory window. NULL may be given for any output 1981289546Scem * parameter if the value is not needed. The base and size may be used for 1982289546Scem * mapping the memory window, to access the peer memory. The alignment and 1983289546Scem * size may be used for translating the memory window, for the peer to access 1984289546Scem * memory on the local system. 1985250079Scarl * 1986289546Scem * Return: Zero on success, otherwise an error number. 1987250079Scarl */ 1988289546Scemint 1989289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 1990289546Scem void **vbase, size_t *size, size_t *align, size_t *align_size) 1991250079Scarl{ 1992289546Scem struct ntb_pci_bar_info *bar; 1993289546Scem size_t bar_b2b_off; 1994250079Scarl 1995289546Scem if (mw_idx >= ntb_mw_count(ntb)) 1996289546Scem return (EINVAL); 1997250079Scarl 1998289546Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)]; 1999289546Scem bar_b2b_off = 0; 2000289546Scem if (mw_idx == ntb->b2b_mw_idx) { 2001289546Scem KASSERT(ntb->b2b_off != 0, 2002289546Scem ("user shouldn't get non-shared b2b mw")); 2003289546Scem bar_b2b_off = ntb->b2b_off; 2004289546Scem } 2005250079Scarl 2006289546Scem if (base != NULL) 2007289546Scem *base = bar->pbase + bar_b2b_off; 2008289546Scem if (vbase != NULL) 2009289546Scem *vbase = (char *)bar->vbase + bar_b2b_off; 2010289546Scem if (size != NULL) 2011289546Scem *size = bar->size - bar_b2b_off; 2012289546Scem if (align != NULL) 2013289546Scem *align = bar->size; 2014289546Scem if (align_size != NULL) 2015289546Scem *align_size = 1; 2016289546Scem return (0); 2017250079Scarl} 2018250079Scarl 2019289546Scem/* 2020289546Scem * ntb_mw_set_trans() - set the translation of a memory window 2021289546Scem * @ntb: NTB device context 2022289546Scem * @idx: Memory window number 2023289546Scem * @addr: The dma address local memory to expose to the peer 2024289546Scem * @size: The size of the local memory to expose to the peer 2025250079Scarl * 2026289546Scem * Set the translation of a memory window. The peer may access local memory 2027289546Scem * through the window starting at the address, up to the size. The address 2028289546Scem * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2029289546Scem * must be aligned to the size alignment specified by ntb_mw_get_range(). 2030250079Scarl * 2031289546Scem * Return: Zero on success, otherwise an error number. 2032250079Scarl */ 2033289546Scemint 2034289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2035289546Scem size_t size) 2036250079Scarl{ 2037289546Scem struct ntb_pci_bar_info *bar; 2038289546Scem uint64_t base, limit, reg_val; 2039289546Scem size_t bar_size, mw_size; 2040289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2041289546Scem enum ntb_bar bar_num; 2042250079Scarl 2043289546Scem if (idx >= ntb_mw_count(ntb)) 2044289546Scem return (EINVAL); 2045250079Scarl 2046289546Scem bar_num = ntb_mw_to_bar(ntb, idx); 2047289546Scem bar = &ntb->bar_info[bar_num]; 2048250079Scarl 2049289546Scem bar_size = bar->size; 2050289546Scem if (idx == ntb->b2b_mw_idx) 2051289546Scem mw_size = bar_size - ntb->b2b_off; 2052289546Scem else 2053289546Scem mw_size = bar_size; 2054250079Scarl 2055289546Scem /* Hardware requires that addr is aligned to bar size */ 2056289546Scem if ((addr & (bar_size - 1)) != 0) 2057289546Scem return (EINVAL); 2058250079Scarl 2059289546Scem if (size > mw_size) 2060289546Scem return (EINVAL); 2061289546Scem 2062289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2063289546Scem 2064289546Scem limit = 0; 2065289546Scem if (bar_is_64bit(ntb, bar_num)) { 2066289546Scem base = ntb_reg_read(8, base_reg); 2067289546Scem 2068289546Scem if (limit_reg != 0 && size != mw_size) 2069289546Scem limit = base + size; 2070289546Scem 2071289546Scem /* Set and verify translation address */ 2072289546Scem ntb_reg_write(8, xlat_reg, addr); 2073289546Scem reg_val = ntb_reg_read(8, xlat_reg); 2074289546Scem if (reg_val != addr) { 2075289546Scem ntb_reg_write(8, xlat_reg, 0); 2076289546Scem return (EIO); 2077289546Scem } 2078289546Scem 2079289546Scem /* Set and verify the limit */ 2080289546Scem ntb_reg_write(8, limit_reg, limit); 2081289546Scem reg_val = ntb_reg_read(8, limit_reg); 2082289546Scem if (reg_val != limit) { 2083289546Scem ntb_reg_write(8, limit_reg, base); 2084289546Scem ntb_reg_write(8, xlat_reg, 0); 2085289546Scem return (EIO); 2086289546Scem } 2087289546Scem } else { 2088289546Scem /* Configure 32-bit (split) BAR MW */ 2089289546Scem 2090289546Scem if ((addr & ~UINT32_MAX) != 0) 2091289546Scem return (EINVAL); 2092289546Scem if (((addr + size) & ~UINT32_MAX) != 0) 2093289546Scem return (EINVAL); 2094289546Scem 2095289546Scem base = ntb_reg_read(4, base_reg); 2096289546Scem 2097289546Scem if (limit_reg != 0 && size != mw_size) 2098289546Scem limit = base + size; 2099289546Scem 2100289546Scem /* Set and verify translation address */ 2101289546Scem ntb_reg_write(4, xlat_reg, addr); 2102289546Scem reg_val = ntb_reg_read(4, xlat_reg); 2103289546Scem if (reg_val != addr) { 2104289546Scem ntb_reg_write(4, xlat_reg, 0); 2105289546Scem return (EIO); 2106289546Scem } 2107289546Scem 2108289546Scem /* Set and verify the limit */ 2109289546Scem ntb_reg_write(4, limit_reg, limit); 2110289546Scem reg_val = ntb_reg_read(4, limit_reg); 2111289546Scem if (reg_val != limit) { 2112289546Scem ntb_reg_write(4, limit_reg, base); 2113289546Scem ntb_reg_write(4, xlat_reg, 0); 2114289546Scem return (EIO); 2115289546Scem } 2116250079Scarl } 2117289546Scem return (0); 2118250079Scarl} 2119250079Scarl 2120289596Scem/* 2121289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window 2122289596Scem * @ntb: NTB device context 2123289596Scem * @idx: Memory window number 2124289596Scem * 2125289596Scem * Clear the translation of a memory window. The peer may no longer access 2126289596Scem * local memory through the window. 2127289596Scem * 2128289596Scem * Return: Zero on success, otherwise an error number. 2129289596Scem */ 2130289596Scemint 2131289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 2132289596Scem{ 2133289596Scem 2134289596Scem return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 2135289596Scem} 2136289596Scem 2137250079Scarl/** 2138289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side 2139250079Scarl * @ntb: pointer to ntb_softc instance 2140289545Scem * @bit: doorbell bits to ring 2141250079Scarl * 2142250079Scarl * This function allows triggering of a doorbell on the secondary/external 2143250079Scarl * side that will initiate an interrupt on the remote host 2144250079Scarl */ 2145250079Scarlvoid 2146289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 2147250079Scarl{ 2148250079Scarl 2149289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2150289347Scem ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit); 2151289347Scem return; 2152289209Scem } 2153289347Scem 2154289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2155250079Scarl} 2156250079Scarl 2157289542Scem/* 2158289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 2159289542Scem * as well as the size of the register (via *sz_out). 2160289542Scem * 2161289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell 2162289542Scem * ring to its memory window write. 2163289542Scem * 2164289542Scem * Note that writing the peer doorbell via a memory window will *not* generate 2165289542Scem * an interrupt on the remote host; that must be done seperately. 2166289542Scem */ 2167289542Scembus_addr_t 2168289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 2169289542Scem{ 2170289542Scem struct ntb_pci_bar_info *bar; 2171289542Scem uint64_t regoff; 2172289542Scem 2173289542Scem KASSERT(sz_out != NULL, ("must be non-NULL")); 2174289542Scem 2175289542Scem if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2176289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 2177289542Scem regoff = ntb->peer_reg->db_bell; 2178289542Scem } else { 2179289542Scem KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) || 2180289542Scem (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1), 2181289542Scem ("mw_count invalid after setup")); 2182289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 2183289543Scem ("invalid b2b idx")); 2184289542Scem 2185289542Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 2186289542Scem regoff = XEON_SHADOW_PDOORBELL_OFFSET; 2187289542Scem } 2188289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 2189289542Scem 2190289542Scem *sz_out = ntb->reg->db_size; 2191289542Scem /* HACK: Specific to current x86 bus implementation. */ 2192289542Scem return ((uint64_t)bar->pci_bus_handle + regoff); 2193289542Scem} 2194289542Scem 2195289597Scem/* 2196289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 2197289597Scem * @ntb: NTB device context 2198289597Scem * 2199289597Scem * Hardware may support different number or arrangement of doorbell bits. 2200289597Scem * 2201289597Scem * Return: A mask of doorbell bits supported by the ntb. 2202289597Scem */ 2203289597Scemuint64_t 2204289597Scemntb_db_valid_mask(struct ntb_softc *ntb) 2205289597Scem{ 2206289597Scem 2207289597Scem return (ntb->db_valid_mask); 2208289597Scem} 2209289597Scem 2210289598Scem/* 2211289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 2212289598Scem * @ntb: NTB device context 2213289598Scem * @vector: Doorbell vector number 2214289598Scem * 2215289598Scem * Each interrupt vector may have a different number or arrangement of bits. 2216289598Scem * 2217289598Scem * Return: A mask of doorbell bits serviced by a vector. 2218289598Scem */ 2219289598Scemuint64_t 2220289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 2221289598Scem{ 2222289598Scem 2223289598Scem if (vector > ntb->db_vec_count) 2224289598Scem return (0); 2225289598Scem return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 2226289598Scem} 2227289598Scem 2228250079Scarl/** 2229289546Scem * ntb_link_is_up() - get the current ntb link state 2230289546Scem * @ntb: NTB device context 2231289546Scem * @speed: OUT - The link speed expressed as PCIe generation number 2232289546Scem * @width: OUT - The link width expressed as the number of PCIe lanes 2233250079Scarl * 2234250079Scarl * RETURNS: true or false based on the hardware link state 2235250079Scarl */ 2236250079Scarlbool 2237289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 2238289546Scem enum ntb_width *width) 2239250079Scarl{ 2240250079Scarl 2241289546Scem if (speed != NULL) 2242289546Scem *speed = ntb_link_sta_speed(ntb); 2243289546Scem if (width != NULL) 2244289546Scem *width = ntb_link_sta_width(ntb); 2245289546Scem return (link_is_up(ntb)); 2246250079Scarl} 2247250079Scarl 2248255272Scarlstatic void 2249255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 2250250079Scarl{ 2251255272Scarl 2252289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2253289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2254289209Scem bar->pbase = rman_get_start(bar->pci_resource); 2255289209Scem bar->size = rman_get_size(bar->pci_resource); 2256289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 2257250079Scarl} 2258255268Scarl 2259289209Scemdevice_t 2260289209Scemntb_get_device(struct ntb_softc *ntb) 2261255268Scarl{ 2262255268Scarl 2263255268Scarl return (ntb->device); 2264255268Scarl} 2265289208Scem 2266289208Scem/* Export HW-specific errata information. */ 2267289208Scembool 2268289208Scemntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 2269289208Scem{ 2270289208Scem 2271289208Scem return (HAS_FEATURE(feature)); 2272289208Scem} 2273