ntb_hw_intel.c revision 302484
1250079Scarl/*- 2302484Smav * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 3250079Scarl * Copyright (C) 2013 Intel Corporation 4289542Scem * Copyright (C) 2015 EMC Corporation 5250079Scarl * All rights reserved. 6250079Scarl * 7250079Scarl * Redistribution and use in source and binary forms, with or without 8250079Scarl * modification, are permitted provided that the following conditions 9250079Scarl * are met: 10250079Scarl * 1. Redistributions of source code must retain the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer. 12250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 13250079Scarl * notice, this list of conditions and the following disclaimer in the 14250079Scarl * documentation and/or other materials provided with the distribution. 15250079Scarl * 16250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26250079Scarl * SUCH DAMAGE. 27250079Scarl */ 28250079Scarl 29302484Smav/* 30302484Smav * The Non-Transparent Bridge (NTB) is a device that allows you to connect 31302484Smav * two or more systems using a PCI-e links, providing remote memory access. 32302484Smav * 33302484Smav * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 34302484Smav * 35302484Smav * NOTE: Much of the code in this module is shared with Linux. Any patches may 36302484Smav * be picked up and redistributed in Linux with a dual GPL/BSD license. 37302484Smav */ 38302484Smav 39250079Scarl#include <sys/cdefs.h> 40250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 302484 2016-07-09 11:20:42Z mav $"); 41250079Scarl 42250079Scarl#include <sys/param.h> 43250079Scarl#include <sys/kernel.h> 44250079Scarl#include <sys/systm.h> 45250079Scarl#include <sys/bus.h> 46289774Scem#include <sys/endian.h> 47250079Scarl#include <sys/malloc.h> 48250079Scarl#include <sys/module.h> 49295618Scem#include <sys/mutex.h> 50295618Scem#include <sys/pciio.h> 51250079Scarl#include <sys/queue.h> 52250079Scarl#include <sys/rman.h> 53289774Scem#include <sys/sbuf.h> 54289207Scem#include <sys/sysctl.h> 55250079Scarl#include <vm/vm.h> 56250079Scarl#include <vm/pmap.h> 57250079Scarl#include <machine/bus.h> 58295618Scem#include <machine/intr_machdep.h> 59250079Scarl#include <machine/resource.h> 60250079Scarl#include <dev/pci/pcireg.h> 61250079Scarl#include <dev/pci/pcivar.h> 62250079Scarl 63250079Scarl#include "ntb_regs.h" 64302484Smav#include "../ntb.h" 65250079Scarl 66289648Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 67250079Scarl 68289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 69289648Scem#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 70291032Scem#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 71250079Scarl 72295618Scem#define NTB_MSIX_VER_GUARD 0xaabbccdd 73295618Scem#define NTB_MSIX_RECEIVED 0xe0f0e0f0 74295618Scem 75295618Scem/* 76295618Scem * PCI constants could be somewhere more generic, but aren't defined/used in 77295618Scem * pci.c. 78295618Scem */ 79295618Scem#define PCI_MSIX_ENTRY_SIZE 16 80295618Scem#define PCI_MSIX_ENTRY_LOWER_ADDR 0 81295618Scem#define PCI_MSIX_ENTRY_UPPER_ADDR 4 82295618Scem#define PCI_MSIX_ENTRY_DATA 8 83295618Scem 84250079Scarlenum ntb_device_type { 85250079Scarl NTB_XEON, 86289648Scem NTB_ATOM 87250079Scarl}; 88250079Scarl 89289610Scem/* ntb_conn_type are hardware numbers, cannot change. */ 90289610Scemenum ntb_conn_type { 91289610Scem NTB_CONN_TRANSPARENT = 0, 92289610Scem NTB_CONN_B2B = 1, 93289610Scem NTB_CONN_RP = 2, 94289610Scem}; 95289610Scem 96289610Scemenum ntb_b2b_direction { 97289610Scem NTB_DEV_USD = 0, 98289610Scem NTB_DEV_DSD = 1, 99289610Scem}; 100289610Scem 101289539Scemenum ntb_bar { 102289539Scem NTB_CONFIG_BAR = 0, 103289539Scem NTB_B2B_BAR_1, 104289539Scem NTB_B2B_BAR_2, 105289539Scem NTB_B2B_BAR_3, 106289539Scem NTB_MAX_BARS 107289539Scem}; 108289539Scem 109295618Scemenum { 110295618Scem NTB_MSIX_GUARD = 0, 111295618Scem NTB_MSIX_DATA0, 112295618Scem NTB_MSIX_DATA1, 113295618Scem NTB_MSIX_DATA2, 114295618Scem NTB_MSIX_OFS0, 115295618Scem NTB_MSIX_OFS1, 116295618Scem NTB_MSIX_OFS2, 117295618Scem NTB_MSIX_DONE, 118295618Scem NTB_MAX_MSIX_SPAD 119295618Scem}; 120295618Scem 121255274Scarl/* Device features and workarounds */ 122302484Smav#define HAS_FEATURE(ntb, feature) \ 123302484Smav (((ntb)->features & (feature)) != 0) 124255274Scarl 125250079Scarlstruct ntb_hw_info { 126250079Scarl uint32_t device_id; 127255274Scarl const char *desc; 128250079Scarl enum ntb_device_type type; 129289397Scem uint32_t features; 130250079Scarl}; 131250079Scarl 132250079Scarlstruct ntb_pci_bar_info { 133250079Scarl bus_space_tag_t pci_bus_tag; 134250079Scarl bus_space_handle_t pci_bus_handle; 135250079Scarl int pci_resource_id; 136250079Scarl struct resource *pci_resource; 137250079Scarl vm_paddr_t pbase; 138290679Scem caddr_t vbase; 139290679Scem vm_size_t size; 140291280Scem vm_memattr_t map_mode; 141289543Scem 142289543Scem /* Configuration register offsets */ 143289543Scem uint32_t psz_off; 144289543Scem uint32_t ssz_off; 145289543Scem uint32_t pbarxlat_off; 146250079Scarl}; 147250079Scarl 148250079Scarlstruct ntb_int_info { 149250079Scarl struct resource *res; 150250079Scarl int rid; 151250079Scarl void *tag; 152250079Scarl}; 153250079Scarl 154289546Scemstruct ntb_vec { 155250079Scarl struct ntb_softc *ntb; 156289546Scem uint32_t num; 157295618Scem unsigned masked; 158250079Scarl}; 159250079Scarl 160289542Scemstruct ntb_reg { 161289542Scem uint32_t ntb_ctl; 162289542Scem uint32_t lnk_sta; 163289542Scem uint8_t db_size; 164289542Scem unsigned mw_bar[NTB_MAX_BARS]; 165289542Scem}; 166289542Scem 167289542Scemstruct ntb_alt_reg { 168289542Scem uint32_t db_bell; 169289542Scem uint32_t db_mask; 170289542Scem uint32_t spad; 171289542Scem}; 172289542Scem 173289542Scemstruct ntb_xlat_reg { 174289546Scem uint32_t bar0_base; 175289546Scem uint32_t bar2_base; 176289546Scem uint32_t bar4_base; 177289546Scem uint32_t bar5_base; 178289546Scem 179289546Scem uint32_t bar2_xlat; 180289546Scem uint32_t bar4_xlat; 181289546Scem uint32_t bar5_xlat; 182289546Scem 183289546Scem uint32_t bar2_limit; 184289546Scem uint32_t bar4_limit; 185289546Scem uint32_t bar5_limit; 186289542Scem}; 187289542Scem 188289542Scemstruct ntb_b2b_addr { 189289542Scem uint64_t bar0_addr; 190289542Scem uint64_t bar2_addr64; 191289542Scem uint64_t bar4_addr64; 192289542Scem uint64_t bar4_addr32; 193289542Scem uint64_t bar5_addr32; 194289542Scem}; 195289542Scem 196295618Scemstruct ntb_msix_data { 197295618Scem uint32_t nmd_ofs; 198295618Scem uint32_t nmd_data; 199295618Scem}; 200295618Scem 201250079Scarlstruct ntb_softc { 202250079Scarl device_t device; 203250079Scarl enum ntb_device_type type; 204289774Scem uint32_t features; 205250079Scarl 206250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 207250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 208250079Scarl uint32_t allocated_interrupts; 209250079Scarl 210295618Scem struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 211295618Scem struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 212295618Scem bool peer_msix_good; 213295618Scem bool peer_msix_done; 214295618Scem struct ntb_pci_bar_info *peer_lapic_bar; 215295618Scem struct callout peer_msix_work; 216295618Scem 217250079Scarl struct callout heartbeat_timer; 218250079Scarl struct callout lr_timer; 219250079Scarl 220289546Scem void *ntb_ctx; 221289546Scem const struct ntb_ctx_ops *ctx_ops; 222289546Scem struct ntb_vec *msix_vec; 223290683Scem#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 224290683Scem#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 225289546Scem#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 226289546Scem struct mtx ctx_lock; 227250079Scarl 228289610Scem uint32_t ppd; 229289610Scem enum ntb_conn_type conn_type; 230289610Scem enum ntb_b2b_direction dev_type; 231289539Scem 232289542Scem /* Offset of peer bar0 in B2B BAR */ 233289542Scem uint64_t b2b_off; 234289542Scem /* Memory window used to access peer bar0 */ 235289543Scem#define B2B_MW_DISABLED UINT8_MAX 236289542Scem uint8_t b2b_mw_idx; 237301293Smav uint32_t msix_xlat; 238295618Scem uint8_t msix_mw_idx; 239289542Scem 240289539Scem uint8_t mw_count; 241289539Scem uint8_t spad_count; 242289539Scem uint8_t db_count; 243289539Scem uint8_t db_vec_count; 244289539Scem uint8_t db_vec_shift; 245289542Scem 246289546Scem /* Protects local db_mask. */ 247289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 248289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 249289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 250289542Scem struct mtx db_mask_lock; 251289542Scem 252290686Scem volatile uint32_t ntb_ctl; 253290686Scem volatile uint32_t lnk_sta; 254289542Scem 255289542Scem uint64_t db_valid_mask; 256289542Scem uint64_t db_link_mask; 257289546Scem uint64_t db_mask; 258289542Scem 259289542Scem int last_ts; /* ticks @ last irq */ 260289542Scem 261289542Scem const struct ntb_reg *reg; 262289542Scem const struct ntb_alt_reg *self_reg; 263289542Scem const struct ntb_alt_reg *peer_reg; 264289542Scem const struct ntb_xlat_reg *xlat_reg; 265250079Scarl}; 266250079Scarl 267289234Scem#ifdef __i386__ 268289234Scemstatic __inline uint64_t 269289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 270289234Scem bus_size_t offset) 271289234Scem{ 272289234Scem 273289234Scem return (bus_space_read_4(tag, handle, offset) | 274289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 275289234Scem} 276289234Scem 277289234Scemstatic __inline void 278289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 279289234Scem bus_size_t offset, uint64_t val) 280289234Scem{ 281289234Scem 282289234Scem bus_space_write_4(tag, handle, offset, val); 283289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 284289234Scem} 285289234Scem#endif 286289234Scem 287255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 288255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 289255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 290255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 291255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 292255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 293255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 294250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 295255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 296289397Scem#define ntb_mw_read(SIZE, offset) \ 297289542Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 298255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 299289542Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 300289397Scem offset, val) 301250079Scarl 302250079Scarlstatic int ntb_probe(device_t device); 303250079Scarlstatic int ntb_attach(device_t device); 304250079Scarlstatic int ntb_detach(device_t device); 305302484Smavstatic uint64_t ntb_db_valid_mask(device_t dev); 306302484Smavstatic void ntb_spad_clear(device_t dev); 307302484Smavstatic uint64_t ntb_db_vector_mask(device_t dev, uint32_t vector); 308302484Smavstatic bool ntb_link_is_up(device_t dev, enum ntb_speed *speed, 309302484Smav enum ntb_width *width); 310302484Smavstatic int ntb_link_enable(device_t dev, enum ntb_speed speed, 311302484Smav enum ntb_width width); 312302484Smavstatic int ntb_link_disable(device_t dev); 313302484Smavstatic int ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 314302484Smavstatic int ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 315302484Smav 316291263Scemstatic unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 317289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 318289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 319289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 320289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 321255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 322291280Scemstatic int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 323291280Scem vm_memattr_t); 324289647Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 325289647Scem const char *); 326255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 327255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 328255272Scarl struct ntb_pci_bar_info *bar); 329250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 330289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 331289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 332289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 333289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 334250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 335289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 336289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 337289546Scemstatic void ndev_vec_isr(void *arg); 338289546Scemstatic void ndev_irq_isr(void *arg); 339289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 340290678Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 341290678Scemstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 342289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 343289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 344300531Scemstatic void ntb_get_msix_info(struct ntb_softc *ntb); 345295618Scemstatic void ntb_exchange_msix(void *); 346250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 347289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 348289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 349289648Scemstatic int ntb_detect_atom(struct ntb_softc *ntb); 350289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 351289648Scemstatic int ntb_atom_init_dev(struct ntb_softc *ntb); 352289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 353289648Scemstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 354289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 355289543Scem enum ntb_bar regbar); 356289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 357289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 358289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 359289543Scem enum ntb_bar idx); 360289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 361289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 362295618Scemstatic int xeon_setup_msix_bar(struct ntb_softc *); 363289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 364295618Scemstatic inline bool _xeon_link_is_up(struct ntb_softc *ntb); 365289648Scemstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 366289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 367289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 368289648Scemstatic void atom_link_hb(void *arg); 369302484Smavstatic void ntb_link_event(device_t dev); 370302484Smavstatic void ntb_db_event(device_t dev, uint32_t vec); 371289648Scemstatic void recover_atom_link(void *arg); 372289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb); 373255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 374289774Scemstatic void ntb_sysctl_init(struct ntb_softc *); 375289774Scemstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 376300100Scemstatic int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 377300100Scemstatic int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 378289774Scemstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 379289774Scemstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 380250079Scarl 381290685Scemstatic unsigned g_ntb_hw_debug_level; 382290685ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 383290685Scem &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 384290685Scem#define ntb_printf(lvl, ...) do { \ 385290685Scem if ((lvl) <= g_ntb_hw_debug_level) { \ 386290685Scem device_printf(ntb->device, __VA_ARGS__); \ 387290685Scem } \ 388290685Scem} while (0) 389290685Scem 390295486Scem#define _NTB_PAT_UC 0 391295486Scem#define _NTB_PAT_WC 1 392295486Scem#define _NTB_PAT_WT 4 393295486Scem#define _NTB_PAT_WP 5 394295486Scem#define _NTB_PAT_WB 6 395295486Scem#define _NTB_PAT_UCM 7 396295486Scemstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 397295486ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 398295486Scem &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 399295486Scem "UC: " __XSTRING(_NTB_PAT_UC) ", " 400295486Scem "WC: " __XSTRING(_NTB_PAT_WC) ", " 401295486Scem "WT: " __XSTRING(_NTB_PAT_WT) ", " 402295486Scem "WP: " __XSTRING(_NTB_PAT_WP) ", " 403295486Scem "WB: " __XSTRING(_NTB_PAT_WB) ", " 404295486Scem "UC-: " __XSTRING(_NTB_PAT_UCM)); 405291030Scem 406295486Scemstatic inline vm_memattr_t 407295486Scemntb_pat_flags(void) 408295486Scem{ 409295486Scem 410295486Scem switch (g_ntb_mw_pat) { 411295486Scem case _NTB_PAT_WC: 412295486Scem return (VM_MEMATTR_WRITE_COMBINING); 413295486Scem case _NTB_PAT_WT: 414295486Scem return (VM_MEMATTR_WRITE_THROUGH); 415295486Scem case _NTB_PAT_WP: 416295486Scem return (VM_MEMATTR_WRITE_PROTECTED); 417295486Scem case _NTB_PAT_WB: 418295486Scem return (VM_MEMATTR_WRITE_BACK); 419295486Scem case _NTB_PAT_UCM: 420295486Scem return (VM_MEMATTR_WEAK_UNCACHEABLE); 421295486Scem case _NTB_PAT_UC: 422295486Scem /* FALLTHROUGH */ 423295486Scem default: 424295486Scem return (VM_MEMATTR_UNCACHEABLE); 425295486Scem } 426295486Scem} 427295486Scem 428295487Scem/* 429295487Scem * Well, this obviously doesn't belong here, but it doesn't seem to exist 430295487Scem * anywhere better yet. 431295487Scem */ 432295487Scemstatic inline const char * 433295487Scemntb_vm_memattr_to_str(vm_memattr_t pat) 434295487Scem{ 435295487Scem 436295487Scem switch (pat) { 437295487Scem case VM_MEMATTR_WRITE_COMBINING: 438295487Scem return ("WRITE_COMBINING"); 439295487Scem case VM_MEMATTR_WRITE_THROUGH: 440295487Scem return ("WRITE_THROUGH"); 441295487Scem case VM_MEMATTR_WRITE_PROTECTED: 442295487Scem return ("WRITE_PROTECTED"); 443295487Scem case VM_MEMATTR_WRITE_BACK: 444295487Scem return ("WRITE_BACK"); 445295487Scem case VM_MEMATTR_WEAK_UNCACHEABLE: 446295487Scem return ("UNCACHED"); 447295487Scem case VM_MEMATTR_UNCACHEABLE: 448295487Scem return ("UNCACHEABLE"); 449295487Scem default: 450295487Scem return ("UNKNOWN"); 451295487Scem } 452295487Scem} 453295487Scem 454295618Scemstatic int g_ntb_msix_idx = 0; 455295618ScemSYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 456295618Scem 0, "Use this memory window to access the peer MSIX message complex on " 457295618Scem "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 458295618Scem "Like b2b_mw_idx, negative values index from the last available memory " 459295618Scem "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 460295618Scem 461291263Scemstatic int g_ntb_mw_idx = -1; 462291263ScemSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 463291263Scem 0, "Use this memory window to access the peer NTB registers. A " 464291263Scem "non-negative value starts from the first MW index; a negative value " 465291263Scem "starts from the last MW index. The default is -1, i.e., the last " 466291263Scem "available memory window. Both sides of the NTB MUST set the same " 467291263Scem "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 468291263Scem 469302484Smav/* Hardware owns the low 16 bits of features. */ 470302484Smav#define NTB_BAR_SIZE_4K (1 << 0) 471302484Smav#define NTB_SDOORBELL_LOCKUP (1 << 1) 472302484Smav#define NTB_SB01BASE_LOCKUP (1 << 2) 473302484Smav#define NTB_B2BDOORBELL_BIT14 (1 << 3) 474302484Smav/* Software/configuration owns the top 16 bits. */ 475302484Smav#define NTB_SPLIT_BAR (1ull << 16) 476302484Smav 477302484Smav#define NTB_FEATURES_STR \ 478302484Smav "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 479302484Smav "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 480302484Smav 481250079Scarlstatic struct ntb_hw_info pci_ids[] = { 482289612Scem /* XXX: PS/SS IDs left out until they are supported. */ 483289612Scem { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 484289648Scem NTB_ATOM, 0 }, 485289233Scem 486289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 487289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 488289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 489289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 490289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 491289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 492289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 493289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 494289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 495289538Scem NTB_SB01BASE_LOCKUP }, 496289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 497289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 498289538Scem NTB_SB01BASE_LOCKUP }, 499289233Scem 500289648Scem { 0x00000000, NULL, NTB_ATOM, 0 } 501250079Scarl}; 502250079Scarl 503289648Scemstatic const struct ntb_reg atom_reg = { 504289648Scem .ntb_ctl = ATOM_NTBCNTL_OFFSET, 505289648Scem .lnk_sta = ATOM_LINK_STATUS_OFFSET, 506289542Scem .db_size = sizeof(uint64_t), 507289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 508289542Scem}; 509289542Scem 510289648Scemstatic const struct ntb_alt_reg atom_pri_reg = { 511289648Scem .db_bell = ATOM_PDOORBELL_OFFSET, 512289648Scem .db_mask = ATOM_PDBMSK_OFFSET, 513289648Scem .spad = ATOM_SPAD_OFFSET, 514289607Scem}; 515289607Scem 516289648Scemstatic const struct ntb_alt_reg atom_b2b_reg = { 517289648Scem .db_bell = ATOM_B2B_DOORBELL_OFFSET, 518289648Scem .spad = ATOM_B2B_SPAD_OFFSET, 519289542Scem}; 520289542Scem 521289648Scemstatic const struct ntb_xlat_reg atom_sec_xlat = { 522289542Scem#if 0 523289542Scem /* "FIXME" says the Linux driver. */ 524289648Scem .bar0_base = ATOM_SBAR0BASE_OFFSET, 525289648Scem .bar2_base = ATOM_SBAR2BASE_OFFSET, 526289648Scem .bar4_base = ATOM_SBAR4BASE_OFFSET, 527289546Scem 528289648Scem .bar2_limit = ATOM_SBAR2LMT_OFFSET, 529289648Scem .bar4_limit = ATOM_SBAR4LMT_OFFSET, 530289542Scem#endif 531289546Scem 532289648Scem .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 533289648Scem .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 534289542Scem}; 535289542Scem 536289542Scemstatic const struct ntb_reg xeon_reg = { 537289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 538289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 539289542Scem .db_size = sizeof(uint16_t), 540289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 541289542Scem}; 542289542Scem 543289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 544289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 545289607Scem .db_mask = XEON_PDBMSK_OFFSET, 546289607Scem .spad = XEON_SPAD_OFFSET, 547289607Scem}; 548289607Scem 549289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 550289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 551289542Scem .spad = XEON_B2B_SPAD_OFFSET, 552289542Scem}; 553289542Scem 554289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 555289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 556289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 557289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 558289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 559289546Scem 560289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 561289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 562289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 563289546Scem 564289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 565289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 566289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 567289542Scem}; 568289542Scem 569289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 570290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 571290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 572290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 573290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 574290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 575289542Scem}; 576289542Scem 577289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 578290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 579290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 580290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 581290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 582290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 583289542Scem}; 584289542Scem 585289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 586289614Scem "B2B MW segment overrides -- MUST be the same on both sides"); 587289614Scem 588289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 589289614Scem &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 590289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 591289614Scem "the window at BAR2, on the upstream side of the link. MUST be the same " 592289614Scem "address on both sides."); 593289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 594289614Scem &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 595289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 596289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 597289614Scem "(split-BAR mode)."); 598289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 599289646Scem &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 600289614Scem "(split-BAR mode)."); 601289614Scem 602289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 603289614Scem &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 604289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 605289614Scem "the window at BAR2, on the downstream side of the link. MUST be the same" 606289614Scem " address on both sides."); 607289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 608289614Scem &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 609289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 610289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 611289614Scem "(split-BAR mode)."); 612289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 613289646Scem &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 614289614Scem "(split-BAR mode)."); 615289614Scem 616250079Scarl/* 617250079Scarl * OS <-> Driver interface structures 618250079Scarl */ 619250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 620250079Scarl 621289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 622289207Scem 623250079Scarl/* 624250079Scarl * OS <-> Driver linkage functions 625250079Scarl */ 626250079Scarlstatic int 627250079Scarlntb_probe(device_t device) 628250079Scarl{ 629289209Scem struct ntb_hw_info *p; 630250079Scarl 631289209Scem p = ntb_get_device_info(pci_get_devid(device)); 632289209Scem if (p == NULL) 633250079Scarl return (ENXIO); 634289209Scem 635289209Scem device_set_desc(device, p->desc); 636289209Scem return (0); 637250079Scarl} 638250079Scarl 639250079Scarlstatic int 640250079Scarlntb_attach(device_t device) 641250079Scarl{ 642289209Scem struct ntb_softc *ntb; 643289209Scem struct ntb_hw_info *p; 644250079Scarl int error; 645250079Scarl 646302484Smav ntb = device_get_softc(device); 647289209Scem p = ntb_get_device_info(pci_get_devid(device)); 648289209Scem 649250079Scarl ntb->device = device; 650250079Scarl ntb->type = p->type; 651255274Scarl ntb->features = p->features; 652289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 653295618Scem ntb->msix_mw_idx = B2B_MW_DISABLED; 654250079Scarl 655289648Scem /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 656283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 657283291Sjkim callout_init(&ntb->lr_timer, 1); 658295618Scem callout_init(&ntb->peer_msix_work, 1); 659289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 660290683Scem mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 661250079Scarl 662289648Scem if (ntb->type == NTB_ATOM) 663289648Scem error = ntb_detect_atom(ntb); 664289348Scem else 665289348Scem error = ntb_detect_xeon(ntb); 666290682Scem if (error != 0) 667289348Scem goto out; 668289348Scem 669289397Scem ntb_detect_max_mw(ntb); 670289396Scem 671290682Scem pci_enable_busmaster(ntb->device); 672290682Scem 673289209Scem error = ntb_map_pci_bars(ntb); 674290682Scem if (error != 0) 675289209Scem goto out; 676289648Scem if (ntb->type == NTB_ATOM) 677289648Scem error = ntb_atom_init_dev(ntb); 678289272Scem else 679289542Scem error = ntb_xeon_init_dev(ntb); 680290682Scem if (error != 0) 681289209Scem goto out; 682290682Scem 683302484Smav ntb_spad_clear(device); 684295618Scem 685290682Scem ntb_poll_link(ntb); 686290682Scem 687289774Scem ntb_sysctl_init(ntb); 688250079Scarl 689302484Smav /* Attach children to this controller */ 690302484Smav device_add_child(device, NULL, -1); 691302484Smav bus_generic_attach(device); 692302484Smav 693289209Scemout: 694289209Scem if (error != 0) 695289209Scem ntb_detach(device); 696250079Scarl return (error); 697250079Scarl} 698250079Scarl 699250079Scarlstatic int 700250079Scarlntb_detach(device_t device) 701250079Scarl{ 702289209Scem struct ntb_softc *ntb; 703250079Scarl 704302484Smav ntb = device_get_softc(device); 705289542Scem 706302484Smav /* Detach & delete all children */ 707302484Smav device_delete_children(device); 708302484Smav 709295618Scem if (ntb->self_reg != NULL) { 710295618Scem DB_MASK_LOCK(ntb); 711295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 712295618Scem DB_MASK_UNLOCK(ntb); 713295618Scem } 714250079Scarl callout_drain(&ntb->heartbeat_timer); 715250079Scarl callout_drain(&ntb->lr_timer); 716295618Scem callout_drain(&ntb->peer_msix_work); 717290682Scem pci_disable_busmaster(ntb->device); 718289272Scem if (ntb->type == NTB_XEON) 719289272Scem ntb_teardown_xeon(ntb); 720250079Scarl ntb_teardown_interrupts(ntb); 721289397Scem 722289542Scem mtx_destroy(&ntb->db_mask_lock); 723289546Scem mtx_destroy(&ntb->ctx_lock); 724289542Scem 725250079Scarl ntb_unmap_pci_bar(ntb); 726250079Scarl 727250079Scarl return (0); 728250079Scarl} 729250079Scarl 730289542Scem/* 731289542Scem * Driver internal routines 732289542Scem */ 733289539Scemstatic inline enum ntb_bar 734289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 735289539Scem{ 736289539Scem 737291263Scem KASSERT(mw < ntb->mw_count, 738289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 739289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 740289539Scem 741289542Scem return (ntb->reg->mw_bar[mw]); 742289539Scem} 743289539Scem 744289546Scemstatic inline bool 745289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 746289546Scem{ 747289546Scem /* XXX This assertion could be stronger. */ 748289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 749302484Smav return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 750289546Scem} 751289546Scem 752289546Scemstatic inline void 753289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 754289546Scem uint32_t *xlat, uint32_t *lmt) 755289546Scem{ 756289546Scem uint32_t basev, lmtv, xlatv; 757289546Scem 758289546Scem switch (bar) { 759289546Scem case NTB_B2B_BAR_1: 760289546Scem basev = ntb->xlat_reg->bar2_base; 761289546Scem lmtv = ntb->xlat_reg->bar2_limit; 762289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 763289546Scem break; 764289546Scem case NTB_B2B_BAR_2: 765289546Scem basev = ntb->xlat_reg->bar4_base; 766289546Scem lmtv = ntb->xlat_reg->bar4_limit; 767289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 768289546Scem break; 769289546Scem case NTB_B2B_BAR_3: 770289546Scem basev = ntb->xlat_reg->bar5_base; 771289546Scem lmtv = ntb->xlat_reg->bar5_limit; 772289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 773289546Scem break; 774289546Scem default: 775289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 776289546Scem ("bad bar")); 777289546Scem basev = lmtv = xlatv = 0; 778289546Scem break; 779289546Scem } 780289546Scem 781289546Scem if (base != NULL) 782289546Scem *base = basev; 783289546Scem if (xlat != NULL) 784289546Scem *xlat = xlatv; 785289546Scem if (lmt != NULL) 786289546Scem *lmt = lmtv; 787289546Scem} 788289546Scem 789250079Scarlstatic int 790255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 791250079Scarl{ 792255272Scarl int rc; 793250079Scarl 794250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 795289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 796255272Scarl if (rc != 0) 797289541Scem goto out; 798255272Scarl 799289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 800289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 801255272Scarl if (rc != 0) 802289541Scem goto out; 803289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 804289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 805289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 806255272Scarl 807289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 808291263Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 809291263Scem if (rc != 0) 810291263Scem goto out; 811289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 812289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 813289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 814289543Scem 815302484Smav if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 816289541Scem goto out; 817289397Scem 818289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 819291263Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 820289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 821289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 822289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 823250079Scarl 824289541Scemout: 825289209Scem if (rc != 0) 826255272Scarl device_printf(ntb->device, 827255272Scarl "unable to allocate pci resource\n"); 828255272Scarl return (rc); 829255272Scarl} 830255272Scarl 831289541Scemstatic void 832289647Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 833289647Scem const char *kind) 834289541Scem{ 835289541Scem 836289647Scem device_printf(ntb->device, 837289647Scem "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 838289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 839289647Scem (char *)bar->vbase + bar->size - 1, 840289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 841289647Scem (uintmax_t)bar->size, kind); 842289541Scem} 843289541Scem 844255272Scarlstatic int 845255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 846255272Scarl{ 847255272Scarl 848255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 849289209Scem &bar->pci_resource_id, RF_ACTIVE); 850255272Scarl if (bar->pci_resource == NULL) 851255272Scarl return (ENXIO); 852289209Scem 853289209Scem save_bar_parameters(bar); 854291280Scem bar->map_mode = VM_MEMATTR_UNCACHEABLE; 855289647Scem print_map_success(ntb, bar, "mmr"); 856289209Scem return (0); 857255272Scarl} 858255272Scarl 859255272Scarlstatic int 860255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 861255272Scarl{ 862255272Scarl int rc; 863291280Scem vm_memattr_t mapmode; 864255276Scarl uint8_t bar_size_bits = 0; 865255272Scarl 866289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 867289209Scem &bar->pci_resource_id, RF_ACTIVE); 868250079Scarl 869255272Scarl if (bar->pci_resource == NULL) 870255272Scarl return (ENXIO); 871255276Scarl 872289209Scem save_bar_parameters(bar); 873289209Scem /* 874289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 875289209Scem * hardware issue. To work around this, query the size it should be 876289209Scem * configured to by the device and modify the resource to correspond to 877289209Scem * this new size. The BIOS on systems with this problem is required to 878289209Scem * provide enough address space to allow the driver to make this change 879289209Scem * safely. 880289209Scem * 881289209Scem * Ideally I could have just specified the size when I allocated the 882289209Scem * resource like: 883289209Scem * bus_alloc_resource(ntb->device, 884289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 885289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 886289209Scem * but the PCI driver does not honor the size in this call, so we have 887289209Scem * to modify it after the fact. 888289209Scem */ 889302484Smav if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 890289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 891289209Scem bar_size_bits = pci_read_config(ntb->device, 892289209Scem XEON_PBAR23SZ_OFFSET, 1); 893289209Scem else 894289209Scem bar_size_bits = pci_read_config(ntb->device, 895289209Scem XEON_PBAR45SZ_OFFSET, 1); 896289209Scem 897289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 898289209Scem bar->pci_resource, bar->pbase, 899289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 900255272Scarl if (rc != 0) { 901289209Scem device_printf(ntb->device, 902289209Scem "unable to resize bar\n"); 903255272Scarl return (rc); 904250079Scarl } 905289209Scem 906289209Scem save_bar_parameters(bar); 907250079Scarl } 908289209Scem 909291280Scem bar->map_mode = VM_MEMATTR_UNCACHEABLE; 910291030Scem print_map_success(ntb, bar, "mw"); 911291280Scem 912295486Scem /* 913295486Scem * Optionally, mark MW BARs as anything other than UC to improve 914295486Scem * performance. 915295486Scem */ 916295486Scem mapmode = ntb_pat_flags(); 917295486Scem if (mapmode == bar->map_mode) 918295486Scem return (0); 919291030Scem 920291280Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 921291031Scem if (rc == 0) { 922291280Scem bar->map_mode = mapmode; 923289209Scem device_printf(ntb->device, 924289647Scem "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 925291280Scem "%s.\n", 926289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 927289647Scem (char *)bar->vbase + bar->size - 1, 928291280Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 929295487Scem ntb_vm_memattr_to_str(mapmode)); 930291031Scem } else 931289647Scem device_printf(ntb->device, 932289647Scem "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 933291280Scem "%s: %d\n", 934289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 935289647Scem (char *)bar->vbase + bar->size - 1, 936289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 937295487Scem ntb_vm_memattr_to_str(mapmode), rc); 938289647Scem /* Proceed anyway */ 939250079Scarl return (0); 940250079Scarl} 941250079Scarl 942250079Scarlstatic void 943250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 944250079Scarl{ 945250079Scarl struct ntb_pci_bar_info *current_bar; 946250079Scarl int i; 947250079Scarl 948289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 949250079Scarl current_bar = &ntb->bar_info[i]; 950250079Scarl if (current_bar->pci_resource != NULL) 951250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 952250079Scarl current_bar->pci_resource_id, 953250079Scarl current_bar->pci_resource); 954250079Scarl } 955250079Scarl} 956250079Scarl 957250079Scarlstatic int 958289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 959250079Scarl{ 960289342Scem uint32_t i; 961289342Scem int rc; 962289342Scem 963289342Scem for (i = 0; i < num_vectors; i++) { 964289342Scem ntb->int_info[i].rid = i + 1; 965289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 966289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 967289342Scem if (ntb->int_info[i].res == NULL) { 968289342Scem device_printf(ntb->device, 969289342Scem "bus_alloc_resource failed\n"); 970289342Scem return (ENOMEM); 971289342Scem } 972289342Scem ntb->int_info[i].tag = NULL; 973289342Scem ntb->allocated_interrupts++; 974289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 975289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 976289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 977289342Scem if (rc != 0) { 978289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 979289342Scem return (ENXIO); 980289342Scem } 981289342Scem } 982289342Scem return (0); 983289342Scem} 984289342Scem 985289344Scem/* 986289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 987289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 988289344Scem * should be okay. This tunable should enable us to test that hypothesis 989289344Scem * when someone gets their hands on some Xeon hardware. 990289344Scem */ 991289344Scemstatic int ntb_force_remap_mode; 992289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 993289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 994289344Scem " to a smaller number of ithreads, even if the desired number are " 995289344Scem "available"); 996289344Scem 997289344Scem/* 998289344Scem * In case it is NOT ok, give consumers an abort button. 999289344Scem */ 1000289344Scemstatic int ntb_prefer_intx; 1001289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 1002289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1003289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 1004289344Scem "behavior)"); 1005289344Scem 1006289344Scem/* 1007289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 1008289344Scem * round-robin fashion. 1009289344Scem */ 1010289342Scemstatic int 1011289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1012289344Scem{ 1013289344Scem u_int *vectors; 1014289344Scem uint32_t i; 1015289344Scem int rc; 1016289344Scem 1017289344Scem if (ntb_prefer_intx != 0) 1018289344Scem return (ENXIO); 1019289344Scem 1020289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1021289344Scem 1022289344Scem for (i = 0; i < desired; i++) 1023289344Scem vectors[i] = (i % avail) + 1; 1024289344Scem 1025289344Scem rc = pci_remap_msix(dev, desired, vectors); 1026289344Scem free(vectors, M_NTB); 1027289344Scem return (rc); 1028289344Scem} 1029289344Scem 1030289344Scemstatic int 1031289540Scemntb_init_isr(struct ntb_softc *ntb) 1032289342Scem{ 1033289344Scem uint32_t desired_vectors, num_vectors; 1034289342Scem int rc; 1035250079Scarl 1036250079Scarl ntb->allocated_interrupts = 0; 1037289542Scem ntb->last_ts = ticks; 1038289347Scem 1039250079Scarl /* 1040295618Scem * Mask all doorbell interrupts. (Except link events!) 1041250079Scarl */ 1042295618Scem DB_MASK_LOCK(ntb); 1043295618Scem ntb->db_mask = ntb->db_valid_mask; 1044295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1045295618Scem DB_MASK_UNLOCK(ntb); 1046250079Scarl 1047289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1048289539Scem ntb->db_count); 1049289344Scem if (desired_vectors >= 1) { 1050289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 1051250079Scarl 1052289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 1053289344Scem num_vectors == desired_vectors) 1054289344Scem num_vectors--; 1055289344Scem 1056289344Scem if (rc == 0 && num_vectors < desired_vectors) { 1057289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 1058289344Scem num_vectors); 1059289344Scem if (rc == 0) 1060289344Scem num_vectors = desired_vectors; 1061289344Scem else 1062289344Scem pci_release_msi(ntb->device); 1063289344Scem } 1064289344Scem if (rc != 0) 1065289344Scem num_vectors = 1; 1066289344Scem } else 1067289344Scem num_vectors = 1; 1068289344Scem 1069289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1070302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1071295618Scem device_printf(ntb->device, 1072295618Scem "Errata workaround does not support MSI or INTX\n"); 1073295618Scem return (EINVAL); 1074295618Scem } 1075295618Scem 1076289539Scem ntb->db_vec_count = 1; 1077290680Scem ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1078289539Scem rc = ntb_setup_legacy_interrupt(ntb); 1079289539Scem } else { 1080300531Scem if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1081302484Smav HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1082300531Scem device_printf(ntb->device, 1083300531Scem "Errata workaround expects %d doorbell bits\n", 1084300531Scem XEON_NONLINK_DB_MSIX_BITS); 1085300531Scem return (EINVAL); 1086300531Scem } 1087300531Scem 1088289546Scem ntb_create_msix_vec(ntb, num_vectors); 1089289540Scem rc = ntb_setup_msix(ntb, num_vectors); 1090302484Smav if (rc == 0 && HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1091300531Scem ntb_get_msix_info(ntb); 1092289539Scem } 1093289539Scem if (rc != 0) { 1094289539Scem device_printf(ntb->device, 1095289539Scem "Error allocating interrupts: %d\n", rc); 1096289546Scem ntb_free_msix_vec(ntb); 1097289396Scem } 1098289396Scem 1099289342Scem return (rc); 1100289342Scem} 1101289342Scem 1102289342Scemstatic int 1103289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1104289342Scem{ 1105289342Scem int rc; 1106289342Scem 1107289342Scem ntb->int_info[0].rid = 0; 1108289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1109289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1110289342Scem if (ntb->int_info[0].res == NULL) { 1111289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 1112289342Scem return (ENOMEM); 1113250079Scarl } 1114250079Scarl 1115289342Scem ntb->int_info[0].tag = NULL; 1116289342Scem ntb->allocated_interrupts = 1; 1117289342Scem 1118289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1119289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1120289342Scem ntb, &ntb->int_info[0].tag); 1121289342Scem if (rc != 0) { 1122289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 1123289342Scem return (ENXIO); 1124289342Scem } 1125289342Scem 1126250079Scarl return (0); 1127250079Scarl} 1128250079Scarl 1129250079Scarlstatic void 1130250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 1131250079Scarl{ 1132250079Scarl struct ntb_int_info *current_int; 1133250079Scarl int i; 1134250079Scarl 1135289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 1136250079Scarl current_int = &ntb->int_info[i]; 1137250079Scarl if (current_int->tag != NULL) 1138250079Scarl bus_teardown_intr(ntb->device, current_int->res, 1139250079Scarl current_int->tag); 1140250079Scarl 1141250079Scarl if (current_int->res != NULL) 1142250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 1143250079Scarl rman_get_rid(current_int->res), current_int->res); 1144250079Scarl } 1145250079Scarl 1146289546Scem ntb_free_msix_vec(ntb); 1147250079Scarl pci_release_msi(ntb->device); 1148250079Scarl} 1149250079Scarl 1150289347Scem/* 1151289648Scem * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1152289347Scem * out to make code clearer. 1153289347Scem */ 1154289539Scemstatic inline uint64_t 1155289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 1156289347Scem{ 1157289347Scem 1158289648Scem if (ntb->type == NTB_ATOM) 1159289347Scem return (ntb_reg_read(8, regoff)); 1160289347Scem 1161289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1162289347Scem 1163289347Scem return (ntb_reg_read(2, regoff)); 1164289347Scem} 1165289347Scem 1166289539Scemstatic inline void 1167289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1168289347Scem{ 1169289347Scem 1170289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 1171289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1172289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 1173289542Scem (uintmax_t)ntb->db_valid_mask)); 1174289542Scem 1175289607Scem if (regoff == ntb->self_reg->db_mask) 1176289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 1177290678Scem db_iowrite_raw(ntb, regoff, val); 1178290678Scem} 1179289542Scem 1180290678Scemstatic inline void 1181290678Scemdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1182290678Scem{ 1183290678Scem 1184289648Scem if (ntb->type == NTB_ATOM) { 1185289347Scem ntb_reg_write(8, regoff, val); 1186289347Scem return; 1187289347Scem } 1188289347Scem 1189289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1190289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 1191289347Scem} 1192289347Scem 1193302484Smavstatic void 1194302484Smavntb_db_set_mask(device_t dev, uint64_t bits) 1195289542Scem{ 1196302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1197289542Scem 1198302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1199295618Scem return; 1200295618Scem 1201289546Scem DB_MASK_LOCK(ntb); 1202289542Scem ntb->db_mask |= bits; 1203289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1204289546Scem DB_MASK_UNLOCK(ntb); 1205289542Scem} 1206289542Scem 1207302484Smavstatic void 1208302484Smavntb_db_clear_mask(device_t dev, uint64_t bits) 1209289542Scem{ 1210302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1211289542Scem 1212289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1213289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1214289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1215289542Scem (uintmax_t)ntb->db_valid_mask)); 1216289542Scem 1217302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1218295618Scem return; 1219295618Scem 1220289546Scem DB_MASK_LOCK(ntb); 1221289542Scem ntb->db_mask &= ~bits; 1222289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1223289546Scem DB_MASK_UNLOCK(ntb); 1224289542Scem} 1225289542Scem 1226302484Smavstatic uint64_t 1227302484Smavntb_db_read(device_t dev) 1228289281Scem{ 1229302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1230289281Scem 1231302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1232295618Scem uint64_t res; 1233295618Scem unsigned i; 1234295618Scem 1235295618Scem res = 0; 1236295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1237295618Scem if (ntb->msix_vec[i].masked != 0) 1238302484Smav res |= ntb_db_vector_mask(dev, i); 1239295618Scem } 1240295618Scem return (res); 1241295618Scem } 1242295618Scem 1243289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 1244289281Scem} 1245289281Scem 1246302484Smavstatic void 1247302484Smavntb_db_clear(device_t dev, uint64_t bits) 1248289281Scem{ 1249302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1250289281Scem 1251289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1252289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1253289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1254289546Scem (uintmax_t)ntb->db_valid_mask)); 1255289546Scem 1256302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1257295618Scem unsigned i; 1258295618Scem 1259295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1260302484Smav if ((bits & ntb_db_vector_mask(dev, i)) != 0) { 1261295618Scem DB_MASK_LOCK(ntb); 1262295618Scem if (ntb->msix_vec[i].masked != 0) { 1263295618Scem /* XXX These need a public API. */ 1264295618Scem#if 0 1265295618Scem pci_unmask_msix(ntb->device, i); 1266295618Scem#endif 1267295618Scem ntb->msix_vec[i].masked = 0; 1268295618Scem } 1269295618Scem DB_MASK_UNLOCK(ntb); 1270295618Scem } 1271295618Scem } 1272295618Scem return; 1273295618Scem } 1274295618Scem 1275289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1276289281Scem} 1277289281Scem 1278289540Scemstatic inline uint64_t 1279289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1280250079Scarl{ 1281289540Scem uint64_t shift, mask; 1282250079Scarl 1283289540Scem shift = ntb->db_vec_shift; 1284289540Scem mask = (1ull << shift) - 1; 1285289540Scem return (mask << (shift * db_vector)); 1286250079Scarl} 1287250079Scarl 1288250079Scarlstatic void 1289289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1290250079Scarl{ 1291289540Scem uint64_t vec_mask; 1292250079Scarl 1293289542Scem ntb->last_ts = ticks; 1294289546Scem vec_mask = ntb_vec_mask(ntb, vec); 1295250079Scarl 1296289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1297289546Scem if (ntb_poll_link(ntb)) 1298302484Smav ntb_link_event(ntb->device); 1299289540Scem } 1300289540Scem 1301302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1302295618Scem (vec_mask & ntb->db_link_mask) == 0) { 1303295618Scem DB_MASK_LOCK(ntb); 1304295618Scem if (ntb->msix_vec[vec].masked == 0) { 1305295618Scem /* XXX These need a public API. */ 1306295618Scem#if 0 1307295618Scem pci_mask_msix(ntb->device, vec); 1308295618Scem#endif 1309295618Scem ntb->msix_vec[vec].masked = 1; 1310295618Scem } 1311295618Scem DB_MASK_UNLOCK(ntb); 1312295618Scem } 1313295618Scem 1314289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1315302484Smav ntb_db_event(ntb->device, vec); 1316289546Scem} 1317250079Scarl 1318289546Scemstatic void 1319289546Scemndev_vec_isr(void *arg) 1320289546Scem{ 1321289546Scem struct ntb_vec *nvec = arg; 1322250079Scarl 1323289546Scem ntb_interrupt(nvec->ntb, nvec->num); 1324250079Scarl} 1325250079Scarl 1326250079Scarlstatic void 1327289546Scemndev_irq_isr(void *arg) 1328250079Scarl{ 1329289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1330289546Scem ntb_interrupt(arg, 0); 1331250079Scarl} 1332250079Scarl 1333250079Scarlstatic int 1334289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1335250079Scarl{ 1336289342Scem uint32_t i; 1337250079Scarl 1338289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1339250079Scarl M_ZERO | M_WAITOK); 1340250079Scarl for (i = 0; i < num_vectors; i++) { 1341289546Scem ntb->msix_vec[i].num = i; 1342289546Scem ntb->msix_vec[i].ntb = ntb; 1343250079Scarl } 1344250079Scarl 1345250079Scarl return (0); 1346250079Scarl} 1347250079Scarl 1348250079Scarlstatic void 1349289546Scemntb_free_msix_vec(struct ntb_softc *ntb) 1350250079Scarl{ 1351250079Scarl 1352289546Scem if (ntb->msix_vec == NULL) 1353289539Scem return; 1354289539Scem 1355289546Scem free(ntb->msix_vec, M_NTB); 1356289546Scem ntb->msix_vec = NULL; 1357250079Scarl} 1358250079Scarl 1359295618Scemstatic void 1360300531Scemntb_get_msix_info(struct ntb_softc *ntb) 1361295618Scem{ 1362295618Scem struct pci_devinfo *dinfo; 1363295618Scem struct pcicfg_msix *msix; 1364295618Scem uint32_t laddr, data, i, offset; 1365295618Scem 1366295618Scem dinfo = device_get_ivars(ntb->device); 1367295618Scem msix = &dinfo->cfg.msix; 1368295618Scem 1369300531Scem CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1370300531Scem 1371300531Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1372295618Scem offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1373295618Scem 1374295618Scem laddr = bus_read_4(msix->msix_table_res, offset + 1375295618Scem PCI_MSIX_ENTRY_LOWER_ADDR); 1376301293Smav ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1377295618Scem 1378295618Scem KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1379295618Scem ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1380295618Scem MSI_INTEL_ADDR_BASE)); 1381301293Smav ntb->msix_data[i].nmd_ofs = laddr; 1382295618Scem 1383295618Scem data = bus_read_4(msix->msix_table_res, offset + 1384295618Scem PCI_MSIX_ENTRY_DATA); 1385295618Scem ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1386295618Scem 1387295618Scem ntb->msix_data[i].nmd_data = data; 1388295618Scem } 1389295618Scem} 1390295618Scem 1391250079Scarlstatic struct ntb_hw_info * 1392250079Scarlntb_get_device_info(uint32_t device_id) 1393250079Scarl{ 1394250079Scarl struct ntb_hw_info *ep = pci_ids; 1395250079Scarl 1396250079Scarl while (ep->device_id) { 1397250079Scarl if (ep->device_id == device_id) 1398250079Scarl return (ep); 1399250079Scarl ++ep; 1400250079Scarl } 1401250079Scarl return (NULL); 1402250079Scarl} 1403250079Scarl 1404289272Scemstatic void 1405289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 1406250079Scarl{ 1407250079Scarl 1408289617Scem if (ntb->reg != NULL) 1409302484Smav ntb_link_disable(ntb->device); 1410250079Scarl} 1411250079Scarl 1412289397Scemstatic void 1413289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 1414289397Scem{ 1415289397Scem 1416289648Scem if (ntb->type == NTB_ATOM) { 1417289648Scem ntb->mw_count = ATOM_MW_COUNT; 1418289397Scem return; 1419289397Scem } 1420289397Scem 1421302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1422289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1423289397Scem else 1424289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1425289397Scem} 1426289397Scem 1427250079Scarlstatic int 1428289348Scemntb_detect_xeon(struct ntb_softc *ntb) 1429250079Scarl{ 1430289348Scem uint8_t ppd, conn_type; 1431250079Scarl 1432289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1433289348Scem ntb->ppd = ppd; 1434250079Scarl 1435289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1436290681Scem ntb->dev_type = NTB_DEV_DSD; 1437290681Scem else 1438289257Scem ntb->dev_type = NTB_DEV_USD; 1439289257Scem 1440289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1441289397Scem ntb->features |= NTB_SPLIT_BAR; 1442289397Scem 1443295618Scem /* 1444295618Scem * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1445295618Scem * errata workaround; only do one at a time. 1446295618Scem */ 1447302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1448295618Scem ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1449289542Scem 1450289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1451289348Scem switch (conn_type) { 1452289348Scem case NTB_CONN_B2B: 1453289348Scem ntb->conn_type = conn_type; 1454289348Scem break; 1455289348Scem case NTB_CONN_RP: 1456289348Scem case NTB_CONN_TRANSPARENT: 1457289348Scem default: 1458289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1459289348Scem (unsigned)conn_type); 1460289348Scem return (ENXIO); 1461289348Scem } 1462289348Scem return (0); 1463289348Scem} 1464289348Scem 1465289348Scemstatic int 1466289648Scemntb_detect_atom(struct ntb_softc *ntb) 1467289348Scem{ 1468289348Scem uint32_t ppd, conn_type; 1469289348Scem 1470289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1471289348Scem ntb->ppd = ppd; 1472289348Scem 1473289648Scem if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1474289348Scem ntb->dev_type = NTB_DEV_DSD; 1475289348Scem else 1476289348Scem ntb->dev_type = NTB_DEV_USD; 1477289348Scem 1478289648Scem conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1479289348Scem switch (conn_type) { 1480289348Scem case NTB_CONN_B2B: 1481289348Scem ntb->conn_type = conn_type; 1482289348Scem break; 1483289348Scem default: 1484289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1485289348Scem return (ENXIO); 1486289348Scem } 1487289348Scem return (0); 1488289348Scem} 1489289348Scem 1490289348Scemstatic int 1491289542Scemntb_xeon_init_dev(struct ntb_softc *ntb) 1492289348Scem{ 1493289542Scem int rc; 1494289348Scem 1495289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1496289542Scem ntb->db_count = XEON_DB_COUNT; 1497289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1498289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1499289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1500289257Scem 1501289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1502250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1503289348Scem ntb->conn_type); 1504250079Scarl return (ENXIO); 1505250079Scarl } 1506250079Scarl 1507289542Scem ntb->reg = &xeon_reg; 1508289607Scem ntb->self_reg = &xeon_pri_reg; 1509289542Scem ntb->peer_reg = &xeon_b2b_reg; 1510289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1511289542Scem 1512302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1513295618Scem ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1514295618Scem ntb->mw_count; 1515295618Scem ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1516295618Scem g_ntb_msix_idx, ntb->msix_mw_idx); 1517295618Scem rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1518295618Scem VM_MEMATTR_UNCACHEABLE); 1519295618Scem KASSERT(rc == 0, ("shouldn't fail")); 1520302484Smav } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1521295618Scem /* 1522295618Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1523295618Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1524295618Scem * which may hang the system. To workaround this, use a memory 1525295618Scem * window to access the interrupt and scratch pad registers on the 1526295618Scem * remote system. 1527295618Scem */ 1528291263Scem ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1529291263Scem ntb->mw_count; 1530291263Scem ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1531291263Scem g_ntb_mw_idx, ntb->b2b_mw_idx); 1532295618Scem rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1533295618Scem VM_MEMATTR_UNCACHEABLE); 1534291263Scem KASSERT(rc == 0, ("shouldn't fail")); 1535302484Smav } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1536289208Scem /* 1537289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1538289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1539289542Scem * since bit 14 is the last bit. 1540289542Scem * 1541289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1542289542Scem * anyway. Nor for non-B2B connection types. 1543289542Scem */ 1544289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1545250079Scarl 1546289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1547250079Scarl 1548289542Scem if (ntb->dev_type == NTB_DEV_USD) 1549289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1550289542Scem &xeon_b2b_usd_addr); 1551289542Scem else 1552289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1553289542Scem &xeon_b2b_dsd_addr); 1554289542Scem if (rc != 0) 1555289542Scem return (rc); 1556289271Scem 1557250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1558290682Scem ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1559289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1560255279Scarl 1561290682Scem /* 1562290682Scem * Mask all doorbell interrupts. 1563290682Scem */ 1564295618Scem DB_MASK_LOCK(ntb); 1565295618Scem ntb->db_mask = ntb->db_valid_mask; 1566295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1567295618Scem DB_MASK_UNLOCK(ntb); 1568250079Scarl 1569295618Scem rc = xeon_setup_msix_bar(ntb); 1570295618Scem if (rc != 0) 1571295618Scem return (rc); 1572295618Scem 1573290682Scem rc = ntb_init_isr(ntb); 1574290682Scem return (rc); 1575250079Scarl} 1576250079Scarl 1577250079Scarlstatic int 1578289648Scemntb_atom_init_dev(struct ntb_softc *ntb) 1579250079Scarl{ 1580290682Scem int error; 1581250079Scarl 1582289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1583289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1584250079Scarl 1585289648Scem ntb->spad_count = ATOM_SPAD_COUNT; 1586289648Scem ntb->db_count = ATOM_DB_COUNT; 1587289648Scem ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1588289648Scem ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1589289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1590250079Scarl 1591289648Scem ntb->reg = &atom_reg; 1592289648Scem ntb->self_reg = &atom_pri_reg; 1593289648Scem ntb->peer_reg = &atom_b2b_reg; 1594289648Scem ntb->xlat_reg = &atom_sec_xlat; 1595289542Scem 1596250079Scarl /* 1597289648Scem * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1598250079Scarl * resolved. Mask transaction layer internal parity errors. 1599250079Scarl */ 1600250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1601250079Scarl 1602289648Scem configure_atom_secondary_side_bars(ntb); 1603250079Scarl 1604250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1605290682Scem ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1606250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1607289209Scem 1608290682Scem error = ntb_init_isr(ntb); 1609290682Scem if (error != 0) 1610290682Scem return (error); 1611290682Scem 1612289542Scem /* Initiate PCI-E link training */ 1613302484Smav ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1614250079Scarl 1615289648Scem callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1616289542Scem 1617250079Scarl return (0); 1618250079Scarl} 1619250079Scarl 1620289648Scem/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1621255279Scarlstatic void 1622289648Scemconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1623255279Scarl{ 1624255279Scarl 1625255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1626289648Scem ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1627290725Scem XEON_B2B_BAR2_ADDR64); 1628289648Scem ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1629290725Scem XEON_B2B_BAR4_ADDR64); 1630290725Scem ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1631290725Scem ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1632255279Scarl } else { 1633289648Scem ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1634290725Scem XEON_B2B_BAR2_ADDR64); 1635289648Scem ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1636290725Scem XEON_B2B_BAR4_ADDR64); 1637290725Scem ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1638290725Scem ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1639255279Scarl } 1640255279Scarl} 1641255279Scarl 1642289543Scem 1643289543Scem/* 1644289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1645289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1646289543Scem * remains for use by a higher layer. 1647289543Scem * 1648289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1649289543Scem * MW size is sufficiently large. 1650289543Scem */ 1651289543Scemstatic unsigned int ntb_b2b_mw_share; 1652289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1653289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1654289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1655289543Scem "value here."); 1656289543Scem 1657289543Scemstatic void 1658289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1659289543Scem enum ntb_bar regbar) 1660289543Scem{ 1661289543Scem struct ntb_pci_bar_info *bar; 1662289543Scem uint8_t bar_sz; 1663289543Scem 1664302484Smav if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1665289543Scem return; 1666289543Scem 1667289543Scem bar = &ntb->bar_info[idx]; 1668289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1669289543Scem if (idx == regbar) { 1670289543Scem if (ntb->b2b_off != 0) 1671289543Scem bar_sz--; 1672289543Scem else 1673289543Scem bar_sz = 0; 1674289543Scem } 1675289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1676289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1677289543Scem (void)bar_sz; 1678289543Scem} 1679289543Scem 1680289543Scemstatic void 1681289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1682289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1683289543Scem{ 1684301293Smav uint64_t reg_val; 1685289546Scem uint32_t base_reg, lmt_reg; 1686289543Scem 1687289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1688302482Smav if (idx == regbar) { 1689302482Smav if (ntb->b2b_off) 1690302482Smav bar_addr += ntb->b2b_off; 1691302482Smav else 1692302482Smav bar_addr = 0; 1693302482Smav } 1694289543Scem 1695295618Scem /* 1696295618Scem * Set limit registers first to avoid an errata where setting the base 1697295618Scem * registers locks the limit registers. 1698295618Scem */ 1699289546Scem if (!bar_is_64bit(ntb, idx)) { 1700301293Smav ntb_reg_write(4, lmt_reg, bar_addr); 1701295618Scem reg_val = ntb_reg_read(4, lmt_reg); 1702295618Scem (void)reg_val; 1703295618Scem 1704289546Scem ntb_reg_write(4, base_reg, bar_addr); 1705289546Scem reg_val = ntb_reg_read(4, base_reg); 1706289546Scem (void)reg_val; 1707295618Scem } else { 1708301293Smav ntb_reg_write(8, lmt_reg, bar_addr); 1709295618Scem reg_val = ntb_reg_read(8, lmt_reg); 1710295618Scem (void)reg_val; 1711289546Scem 1712289546Scem ntb_reg_write(8, base_reg, bar_addr); 1713289546Scem reg_val = ntb_reg_read(8, base_reg); 1714289546Scem (void)reg_val; 1715289543Scem } 1716289543Scem} 1717289543Scem 1718289543Scemstatic void 1719289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1720289543Scem{ 1721289543Scem struct ntb_pci_bar_info *bar; 1722289543Scem 1723289543Scem bar = &ntb->bar_info[idx]; 1724302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1725289543Scem ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1726289543Scem base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1727289543Scem } else { 1728289543Scem ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1729289543Scem base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1730289543Scem } 1731289543Scem (void)base_addr; 1732289543Scem} 1733289543Scem 1734289542Scemstatic int 1735295618Scemxeon_setup_msix_bar(struct ntb_softc *ntb) 1736295618Scem{ 1737295618Scem enum ntb_bar bar_num; 1738295618Scem 1739302484Smav if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1740295618Scem return (0); 1741295618Scem 1742295618Scem bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1743301293Smav ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1744295618Scem return (0); 1745295618Scem} 1746295618Scem 1747295618Scemstatic int 1748289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1749289542Scem const struct ntb_b2b_addr *peer_addr) 1750255279Scarl{ 1751289543Scem struct ntb_pci_bar_info *b2b_bar; 1752289543Scem vm_size_t bar_size; 1753289543Scem uint64_t bar_addr; 1754289543Scem enum ntb_bar b2b_bar_num, i; 1755255279Scarl 1756289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1757289543Scem b2b_bar = NULL; 1758289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1759289543Scem ntb->b2b_off = 0; 1760289543Scem } else { 1761289543Scem b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1762289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1763289543Scem ("invalid b2b mw bar")); 1764289543Scem 1765289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1766289543Scem bar_size = b2b_bar->size; 1767289543Scem 1768289543Scem if (ntb_b2b_mw_share != 0 && 1769289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1770289543Scem ntb->b2b_off = bar_size >> 1; 1771289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1772289543Scem ntb->b2b_off = 0; 1773289543Scem } else { 1774289543Scem device_printf(ntb->device, 1775289543Scem "B2B bar size is too small!\n"); 1776289543Scem return (EIO); 1777289543Scem } 1778255279Scarl } 1779289542Scem 1780289543Scem /* 1781289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1782289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1783289543Scem */ 1784289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1785289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1786289543Scem 1787289543Scem bar_addr = 0; 1788289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1789289543Scem bar_addr = addr->bar0_addr; 1790289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1791289543Scem bar_addr = addr->bar2_addr64; 1792302484Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1793289543Scem bar_addr = addr->bar4_addr64; 1794289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1795289543Scem bar_addr = addr->bar4_addr32; 1796289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1797289543Scem bar_addr = addr->bar5_addr32; 1798289543Scem else 1799289543Scem KASSERT(false, ("invalid bar")); 1800289543Scem 1801289543Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1802289543Scem 1803289543Scem /* 1804289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1805289543Scem * register BAR. The B2B BAR is either disabled above or configured 1806289543Scem * half-size. It starts at PBAR xlat + offset. 1807289543Scem * 1808289543Scem * Also set up incoming BAR limits == base (zero length window). 1809289543Scem */ 1810289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1811289543Scem b2b_bar_num); 1812302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1813289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1814289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1815289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1816289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1817289542Scem } else 1818289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1819289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1820289543Scem 1821289543Scem /* Zero incoming translation addrs */ 1822289543Scem ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1823289543Scem ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1824289543Scem 1825302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1826295618Scem size_t size, xlatoffset; 1827295618Scem 1828295618Scem switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1829295618Scem case NTB_B2B_BAR_1: 1830295618Scem size = 8; 1831295618Scem xlatoffset = XEON_SBAR2XLAT_OFFSET; 1832295618Scem break; 1833295618Scem case NTB_B2B_BAR_2: 1834295618Scem xlatoffset = XEON_SBAR4XLAT_OFFSET; 1835302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1836295618Scem size = 4; 1837295618Scem else 1838295618Scem size = 8; 1839295618Scem break; 1840295618Scem case NTB_B2B_BAR_3: 1841295618Scem xlatoffset = XEON_SBAR5XLAT_OFFSET; 1842295618Scem size = 4; 1843295618Scem break; 1844295618Scem default: 1845295618Scem KASSERT(false, ("Bogus msix mw idx: %u", 1846295618Scem ntb->msix_mw_idx)); 1847295618Scem return (EINVAL); 1848295618Scem } 1849295618Scem 1850295618Scem /* 1851295618Scem * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1852295618Scem * workaround 1853295618Scem */ 1854301293Smav if (size == 4) { 1855295618Scem ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1856301293Smav ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1857301293Smav } else { 1858295618Scem ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1859301293Smav ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1860301293Smav } 1861295618Scem } 1862295618Scem (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1863295618Scem (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1864295618Scem 1865289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1866289543Scem ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1867289543Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1868289543Scem 1869289543Scem /* Set outgoing translation offsets */ 1870289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1871302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1872289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1873289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1874289543Scem } else 1875289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1876289543Scem 1877289543Scem /* Set the translation offset for B2B registers */ 1878289543Scem bar_addr = 0; 1879289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1880289543Scem bar_addr = peer_addr->bar0_addr; 1881289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1882289543Scem bar_addr = peer_addr->bar2_addr64; 1883302484Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1884289543Scem bar_addr = peer_addr->bar4_addr64; 1885289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1886289543Scem bar_addr = peer_addr->bar4_addr32; 1887289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1888289543Scem bar_addr = peer_addr->bar5_addr32; 1889289543Scem else 1890289543Scem KASSERT(false, ("invalid bar")); 1891289543Scem 1892289543Scem /* 1893289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1894289543Scem * at a time. 1895289543Scem */ 1896289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1897289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1898289542Scem return (0); 1899255279Scarl} 1900255279Scarl 1901289546Scemstatic inline bool 1902295618Scem_xeon_link_is_up(struct ntb_softc *ntb) 1903295618Scem{ 1904295618Scem 1905295618Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1906295618Scem return (true); 1907295618Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1908295618Scem} 1909295618Scem 1910295618Scemstatic inline bool 1911289546Scemlink_is_up(struct ntb_softc *ntb) 1912289546Scem{ 1913289546Scem 1914295618Scem if (ntb->type == NTB_XEON) 1915295618Scem return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1916302484Smav !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 1917289546Scem 1918289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1919289648Scem return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1920289546Scem} 1921289546Scem 1922289546Scemstatic inline bool 1923289648Scematom_link_is_err(struct ntb_softc *ntb) 1924289546Scem{ 1925289546Scem uint32_t status; 1926289546Scem 1927289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1928289546Scem 1929289648Scem status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1930289648Scem if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1931289546Scem return (true); 1932289546Scem 1933289648Scem status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1934289648Scem return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1935289546Scem} 1936289546Scem 1937289648Scem/* Atom does not have link status interrupt, poll on that platform */ 1938250079Scarlstatic void 1939289648Scematom_link_hb(void *arg) 1940250079Scarl{ 1941250079Scarl struct ntb_softc *ntb = arg; 1942289546Scem sbintime_t timo, poll_ts; 1943250079Scarl 1944289546Scem timo = NTB_HB_TIMEOUT * hz; 1945289546Scem poll_ts = ntb->last_ts + timo; 1946289546Scem 1947289542Scem /* 1948289542Scem * Delay polling the link status if an interrupt was received, unless 1949289542Scem * the cached link status says the link is down. 1950289542Scem */ 1951289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1952289546Scem timo = poll_ts - ticks; 1953289542Scem goto out; 1954289546Scem } 1955289542Scem 1956289546Scem if (ntb_poll_link(ntb)) 1957302484Smav ntb_link_event(ntb->device); 1958289542Scem 1959289648Scem if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1960289546Scem /* Link is down with error, proceed with recovery */ 1961289648Scem callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1962289546Scem return; 1963250079Scarl } 1964250079Scarl 1965289542Scemout: 1966289648Scem callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1967250079Scarl} 1968250079Scarl 1969250079Scarlstatic void 1970289648Scematom_perform_link_restart(struct ntb_softc *ntb) 1971250079Scarl{ 1972250079Scarl uint32_t status; 1973250079Scarl 1974250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1975289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1976289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1977289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1978289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1979250079Scarl 1980250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1981250079Scarl pause("ModPhy", hz / 10); 1982250079Scarl 1983250079Scarl /* Clear AER Errors, write to clear */ 1984289648Scem status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1985250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1986289648Scem ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1987250079Scarl 1988250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1989289648Scem status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1990289648Scem status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1991289648Scem ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1992250079Scarl 1993250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1994289648Scem status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1995289648Scem status |= ATOM_DESKEWSTS_DBERR; 1996289648Scem ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1997250079Scarl 1998289648Scem status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1999289648Scem status &= ATOM_IBIST_ERR_OFLOW; 2000289648Scem ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2001250079Scarl 2002250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 2003289648Scem status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2004289648Scem status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2005289648Scem ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2006250079Scarl} 2007250079Scarl 2008302484Smavstatic int 2009302484Smavntb_set_ctx(device_t dev, void *ctx, const struct ntb_ctx_ops *ops) 2010250079Scarl{ 2011302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2012250079Scarl 2013289546Scem if (ctx == NULL || ops == NULL) 2014289546Scem return (EINVAL); 2015289546Scem if (ntb->ctx_ops != NULL) 2016289546Scem return (EINVAL); 2017250079Scarl 2018289546Scem CTX_LOCK(ntb); 2019289546Scem if (ntb->ctx_ops != NULL) { 2020289546Scem CTX_UNLOCK(ntb); 2021289546Scem return (EINVAL); 2022250079Scarl } 2023289546Scem ntb->ntb_ctx = ctx; 2024289546Scem ntb->ctx_ops = ops; 2025289546Scem CTX_UNLOCK(ntb); 2026250079Scarl 2027289546Scem return (0); 2028250079Scarl} 2029250079Scarl 2030289546Scem/* 2031289546Scem * It is expected that this will only be used from contexts where the ctx_lock 2032289546Scem * is not needed to protect ntb_ctx lifetime. 2033289546Scem */ 2034302484Smavstatic void * 2035302484Smavntb_get_ctx(device_t dev, const struct ntb_ctx_ops **ops) 2036289546Scem{ 2037302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2038289546Scem 2039289546Scem KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2040289546Scem if (ops != NULL) 2041289546Scem *ops = ntb->ctx_ops; 2042289546Scem return (ntb->ntb_ctx); 2043289546Scem} 2044289546Scem 2045302484Smavstatic void 2046302484Smavntb_clear_ctx(device_t dev) 2047289546Scem{ 2048302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2049289546Scem 2050289546Scem CTX_LOCK(ntb); 2051289546Scem ntb->ntb_ctx = NULL; 2052289546Scem ntb->ctx_ops = NULL; 2053289546Scem CTX_UNLOCK(ntb); 2054289546Scem} 2055289546Scem 2056289546Scem/* 2057289546Scem * ntb_link_event() - notify driver context of a change in link status 2058289546Scem * @ntb: NTB device context 2059289546Scem * 2060289546Scem * Notify the driver context that the link status may have changed. The driver 2061289546Scem * should call ntb_link_is_up() to get the current status. 2062289546Scem */ 2063302484Smavstatic void 2064302484Smavntb_link_event(device_t dev) 2065289546Scem{ 2066302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2067289546Scem 2068289546Scem CTX_LOCK(ntb); 2069289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2070289546Scem ntb->ctx_ops->link_event(ntb->ntb_ctx); 2071289546Scem CTX_UNLOCK(ntb); 2072289546Scem} 2073289546Scem 2074289546Scem/* 2075289546Scem * ntb_db_event() - notify driver context of a doorbell event 2076289546Scem * @ntb: NTB device context 2077289546Scem * @vector: Interrupt vector number 2078289546Scem * 2079289546Scem * Notify the driver context of a doorbell event. If hardware supports 2080289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which 2081289546Scem * vector received the interrupt. The vector number is relative to the first 2082289546Scem * vector used for doorbells, starting at zero, and must be less than 2083289546Scem * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2084289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2085289546Scem * those bits are associated with the vector number. 2086289546Scem */ 2087250079Scarlstatic void 2088302484Smavntb_db_event(device_t dev, uint32_t vec) 2089289272Scem{ 2090302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2091289546Scem 2092289546Scem CTX_LOCK(ntb); 2093289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2094289546Scem ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2095289546Scem CTX_UNLOCK(ntb); 2096289546Scem} 2097289546Scem 2098302484Smavstatic int 2099302484Smavntb_link_enable(device_t dev, enum ntb_speed speed __unused, 2100302484Smav enum ntb_width width __unused) 2101289546Scem{ 2102302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2103289280Scem uint32_t cntl; 2104289272Scem 2105300100Scem ntb_printf(2, "%s\n", __func__); 2106300100Scem 2107289648Scem if (ntb->type == NTB_ATOM) { 2108289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 2109289648Scem ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2110289546Scem return (0); 2111289542Scem } 2112289542Scem 2113289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2114302484Smav ntb_link_event(dev); 2115289546Scem return (0); 2116289280Scem } 2117289280Scem 2118289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2119289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2120289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2121289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2122302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2123289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2124289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2125289546Scem return (0); 2126289272Scem} 2127289272Scem 2128302484Smavstatic int 2129302484Smavntb_link_disable(device_t dev) 2130289272Scem{ 2131302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2132289272Scem uint32_t cntl; 2133289272Scem 2134300100Scem ntb_printf(2, "%s\n", __func__); 2135300100Scem 2136289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2137302484Smav ntb_link_event(dev); 2138289546Scem return (0); 2139289272Scem } 2140289272Scem 2141289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2142289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2143289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2144302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2145289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2146289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2147289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2148289546Scem return (0); 2149289272Scem} 2150289272Scem 2151302484Smavstatic bool 2152302484Smavntb_link_enabled(device_t dev) 2153300100Scem{ 2154302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2155300100Scem uint32_t cntl; 2156300100Scem 2157300100Scem if (ntb->type == NTB_ATOM) { 2158300100Scem cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2159300100Scem return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2160300100Scem } 2161300100Scem 2162300100Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2163300100Scem return (true); 2164300100Scem 2165300100Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2166300100Scem return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2167300100Scem} 2168300100Scem 2169289272Scemstatic void 2170289648Scemrecover_atom_link(void *arg) 2171250079Scarl{ 2172250079Scarl struct ntb_softc *ntb = arg; 2173289608Scem unsigned speed, width, oldspeed, oldwidth; 2174250079Scarl uint32_t status32; 2175250079Scarl 2176289648Scem atom_perform_link_restart(ntb); 2177250079Scarl 2178289232Scem /* 2179289232Scem * There is a potential race between the 2 NTB devices recovering at 2180289232Scem * the same time. If the times are the same, the link will not recover 2181289232Scem * and the driver will be stuck in this loop forever. Add a random 2182289232Scem * interval to the recovery time to prevent this race. 2183289232Scem */ 2184289648Scem status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2185289648Scem pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2186289232Scem 2187289648Scem if (atom_link_is_err(ntb)) 2188250079Scarl goto retry; 2189250079Scarl 2190289542Scem status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2191289648Scem if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2192289232Scem goto out; 2193289232Scem 2194289542Scem status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2195289608Scem width = NTB_LNK_STA_WIDTH(status32); 2196289608Scem speed = status32 & NTB_LINK_SPEED_MASK; 2197289608Scem 2198289608Scem oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2199289608Scem oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2200289608Scem if (oldwidth != width || oldspeed != speed) 2201250079Scarl goto retry; 2202250079Scarl 2203289232Scemout: 2204289648Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2205289542Scem ntb); 2206250079Scarl return; 2207250079Scarl 2208250079Scarlretry: 2209289648Scem callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2210250079Scarl ntb); 2211250079Scarl} 2212250079Scarl 2213289546Scem/* 2214289546Scem * Polls the HW link status register(s); returns true if something has changed. 2215289546Scem */ 2216289546Scemstatic bool 2217289542Scemntb_poll_link(struct ntb_softc *ntb) 2218250079Scarl{ 2219250079Scarl uint32_t ntb_cntl; 2220289546Scem uint16_t reg_val; 2221250079Scarl 2222289648Scem if (ntb->type == NTB_ATOM) { 2223289542Scem ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2224289546Scem if (ntb_cntl == ntb->ntb_ctl) 2225289546Scem return (false); 2226289546Scem 2227289542Scem ntb->ntb_ctl = ntb_cntl; 2228289542Scem ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2229250079Scarl } else { 2230290678Scem db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2231250079Scarl 2232289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2233289546Scem if (reg_val == ntb->lnk_sta) 2234289546Scem return (false); 2235250079Scarl 2236289546Scem ntb->lnk_sta = reg_val; 2237295618Scem 2238302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2239295618Scem if (_xeon_link_is_up(ntb)) { 2240295618Scem if (!ntb->peer_msix_good) { 2241295618Scem callout_reset(&ntb->peer_msix_work, 0, 2242295618Scem ntb_exchange_msix, ntb); 2243295618Scem return (false); 2244295618Scem } 2245295618Scem } else { 2246295618Scem ntb->peer_msix_good = false; 2247295618Scem ntb->peer_msix_done = false; 2248295618Scem } 2249295618Scem } 2250289542Scem } 2251289546Scem return (true); 2252289542Scem} 2253289542Scem 2254289546Scemstatic inline enum ntb_speed 2255289546Scemntb_link_sta_speed(struct ntb_softc *ntb) 2256250079Scarl{ 2257250079Scarl 2258289546Scem if (!link_is_up(ntb)) 2259289546Scem return (NTB_SPEED_NONE); 2260289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2261250079Scarl} 2262250079Scarl 2263289546Scemstatic inline enum ntb_width 2264289546Scemntb_link_sta_width(struct ntb_softc *ntb) 2265250079Scarl{ 2266250079Scarl 2267289546Scem if (!link_is_up(ntb)) 2268289546Scem return (NTB_WIDTH_NONE); 2269289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2270250079Scarl} 2271250079Scarl 2272289774ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2273289774Scem "Driver state, statistics, and HW registers"); 2274289774Scem 2275289774Scem#define NTB_REGSZ_MASK (3ul << 30) 2276289774Scem#define NTB_REG_64 (1ul << 30) 2277289774Scem#define NTB_REG_32 (2ul << 30) 2278289774Scem#define NTB_REG_16 (3ul << 30) 2279289774Scem#define NTB_REG_8 (0ul << 30) 2280289774Scem 2281289774Scem#define NTB_DB_READ (1ul << 29) 2282289774Scem#define NTB_PCI_REG (1ul << 28) 2283289774Scem#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2284289774Scem 2285289774Scemstatic void 2286289774Scemntb_sysctl_init(struct ntb_softc *ntb) 2287289774Scem{ 2288300100Scem struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2289289774Scem struct sysctl_ctx_list *ctx; 2290289774Scem struct sysctl_oid *tree, *tmptree; 2291289774Scem 2292289774Scem ctx = device_get_sysctl_ctx(ntb->device); 2293300100Scem globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2294289774Scem 2295300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2296300100Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2297300100Scem sysctl_handle_link_status_human, "A", 2298300100Scem "Link status (human readable)"); 2299300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2300300100Scem CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2301300100Scem "IU", "Link status (1=active, 0=inactive)"); 2302300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2303300100Scem CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2304300100Scem "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2305300100Scem 2306300100Scem tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2307300100Scem CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2308289774Scem tree_par = SYSCTL_CHILDREN(tree); 2309289774Scem 2310289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2311289774Scem &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2312289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2313289774Scem &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2314290687Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2315290687Scem &ntb->ppd, 0, "Raw PPD register (cached)"); 2316289774Scem 2317289774Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2318289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2319289774Scem &ntb->b2b_mw_idx, 0, 2320289774Scem "Index of the MW used for B2B remote register access"); 2321289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2322289774Scem CTLFLAG_RD, &ntb->b2b_off, 2323289774Scem "If non-zero, offset of B2B register region in shared MW"); 2324289774Scem } 2325289774Scem 2326289774Scem SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2327289774Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2328289774Scem "Features/errata of this NTB device"); 2329289774Scem 2330289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2331290686Scem __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2332290686Scem "NTB CTL register (cached)"); 2333289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2334290686Scem __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2335290686Scem "LNK STA register (cached)"); 2336289774Scem 2337289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2338291263Scem &ntb->mw_count, 0, "MW count"); 2339289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2340289774Scem &ntb->spad_count, 0, "Scratchpad count"); 2341289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2342289774Scem &ntb->db_count, 0, "Doorbell count"); 2343289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2344289774Scem &ntb->db_vec_count, 0, "Doorbell vector count"); 2345289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2346289774Scem &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2347289774Scem 2348289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2349289774Scem &ntb->db_valid_mask, "Doorbell valid mask"); 2350289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2351289774Scem &ntb->db_link_mask, "Doorbell link mask"); 2352289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2353289774Scem &ntb->db_mask, "Doorbell mask (cached)"); 2354289774Scem 2355289774Scem tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2356289774Scem CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2357289774Scem regpar = SYSCTL_CHILDREN(tmptree); 2358289774Scem 2359290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2360290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2361290682Scem ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2362290682Scem "NTB Control register"); 2363290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2364290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2365290682Scem 0x19c, sysctl_handle_register, "IU", 2366290682Scem "NTB Link Capabilities"); 2367290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2368290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2369290682Scem 0x1a0, sysctl_handle_register, "IU", 2370290682Scem "NTB Link Control register"); 2371290682Scem 2372289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2373289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2374289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2375289774Scem sysctl_handle_register, "QU", "Doorbell mask register"); 2376289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2377289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2378289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2379289774Scem sysctl_handle_register, "QU", "Doorbell register"); 2380289774Scem 2381289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2382289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2383289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2384289774Scem sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2385302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2386289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2387289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2388289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2389289774Scem sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2390289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2391289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2392289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2393289774Scem sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2394289774Scem } else { 2395289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2396289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2397289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2398289774Scem sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2399289774Scem } 2400289774Scem 2401289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2402289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2403289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2404289774Scem sysctl_handle_register, "QU", "Incoming LMT23 register"); 2405302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2406289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2407289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2408289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2409289774Scem sysctl_handle_register, "IU", "Incoming LMT4 register"); 2410289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2411289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2412289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2413289774Scem sysctl_handle_register, "IU", "Incoming LMT5 register"); 2414289774Scem } else { 2415289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2416289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2417289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2418289774Scem sysctl_handle_register, "QU", "Incoming LMT45 register"); 2419289774Scem } 2420289774Scem 2421289774Scem if (ntb->type == NTB_ATOM) 2422289774Scem return; 2423289774Scem 2424289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2425289774Scem CTLFLAG_RD, NULL, "Xeon HW statistics"); 2426289774Scem statpar = SYSCTL_CHILDREN(tmptree); 2427289774Scem SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2428289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2429289774Scem NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2430289774Scem sysctl_handle_register, "SU", "Upstream Memory Miss"); 2431289774Scem 2432289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2433289774Scem CTLFLAG_RD, NULL, "Xeon HW errors"); 2434289774Scem errpar = SYSCTL_CHILDREN(tmptree); 2435289774Scem 2436290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2437289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2438290687Scem NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2439290687Scem sysctl_handle_register, "CU", "PPD"); 2440290687Scem 2441290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2442290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2443290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2444290687Scem sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2445290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2446290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2447290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2448290687Scem sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2449290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2450290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2451290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2452290687Scem sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2453290687Scem 2454290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2455290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2456290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2457290687Scem sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2458290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2459290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2460290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2461290687Scem sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2462290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2463290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2464290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2465290687Scem sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2466290687Scem 2467290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2468290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2469289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2470289774Scem sysctl_handle_register, "SU", "DEVSTS"); 2471290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2472289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2473289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2474289774Scem sysctl_handle_register, "SU", "LNKSTS"); 2475290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2476290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2477290687Scem NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2478290687Scem sysctl_handle_register, "SU", "SLNKSTS"); 2479290687Scem 2480289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2481289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2482289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2483289774Scem sysctl_handle_register, "IU", "UNCERRSTS"); 2484289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2485289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2486289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2487289774Scem sysctl_handle_register, "IU", "CORERRSTS"); 2488289774Scem 2489289774Scem if (ntb->conn_type != NTB_CONN_B2B) 2490289774Scem return; 2491289774Scem 2492289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2493289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2494289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2495289774Scem sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2496302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2497289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2498289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2499289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2500289774Scem sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2501289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2502289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2503289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2504289774Scem sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2505289774Scem } else { 2506289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2507289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2508289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2509289774Scem sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2510289774Scem } 2511289774Scem 2512289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2513289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2514289774Scem NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2515289774Scem sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2516302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2517289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2518289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2519289774Scem NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2520289774Scem sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2521289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2522289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2523289774Scem NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2524289774Scem sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2525289774Scem } else { 2526289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2527289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2528289774Scem NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2529289774Scem sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2530289774Scem } 2531289774Scem 2532289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2533289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2534289774Scem NTB_REG_64 | ntb->xlat_reg->bar0_base, 2535289774Scem sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2536289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2537289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2538289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_base, 2539289774Scem sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2540302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2541289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2542289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2543289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_base, 2544289774Scem sysctl_handle_register, "IU", 2545289774Scem "Secondary BAR4 base register"); 2546289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2547289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2548289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_base, 2549289774Scem sysctl_handle_register, "IU", 2550289774Scem "Secondary BAR5 base register"); 2551289774Scem } else { 2552289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2553289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2554289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_base, 2555289774Scem sysctl_handle_register, "QU", 2556289774Scem "Secondary BAR45 base register"); 2557289774Scem } 2558289774Scem} 2559289774Scem 2560289774Scemstatic int 2561289774Scemsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2562289774Scem{ 2563302483Smav struct ntb_softc *ntb = arg1; 2564289774Scem struct sbuf sb; 2565289774Scem int error; 2566289774Scem 2567289774Scem sbuf_new_for_sysctl(&sb, NULL, 256, req); 2568289774Scem 2569289774Scem sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2570289774Scem error = sbuf_finish(&sb); 2571289774Scem sbuf_delete(&sb); 2572289774Scem 2573289774Scem if (error || !req->newptr) 2574289774Scem return (error); 2575289774Scem return (EINVAL); 2576289774Scem} 2577289774Scem 2578289774Scemstatic int 2579300100Scemsysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2580289774Scem{ 2581302483Smav struct ntb_softc *ntb = arg1; 2582300100Scem unsigned old, new; 2583300100Scem int error; 2584300100Scem 2585302484Smav old = ntb_link_enabled(ntb->device); 2586300100Scem 2587300100Scem error = SYSCTL_OUT(req, &old, sizeof(old)); 2588300100Scem if (error != 0 || req->newptr == NULL) 2589300100Scem return (error); 2590300100Scem 2591300100Scem error = SYSCTL_IN(req, &new, sizeof(new)); 2592300100Scem if (error != 0) 2593300100Scem return (error); 2594300100Scem 2595300100Scem ntb_printf(0, "Admin set interface state to '%sabled'\n", 2596300100Scem (new != 0)? "en" : "dis"); 2597300100Scem 2598300100Scem if (new != 0) 2599302484Smav error = ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2600300100Scem else 2601302484Smav error = ntb_link_disable(ntb->device); 2602300100Scem return (error); 2603300100Scem} 2604300100Scem 2605300100Scemstatic int 2606300100Scemsysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2607300100Scem{ 2608302483Smav struct ntb_softc *ntb = arg1; 2609289774Scem struct sbuf sb; 2610289774Scem enum ntb_speed speed; 2611289774Scem enum ntb_width width; 2612289774Scem int error; 2613289774Scem 2614289774Scem sbuf_new_for_sysctl(&sb, NULL, 32, req); 2615289774Scem 2616302484Smav if (ntb_link_is_up(ntb->device, &speed, &width)) 2617289774Scem sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2618289774Scem (unsigned)speed, (unsigned)width); 2619289774Scem else 2620289774Scem sbuf_printf(&sb, "down"); 2621289774Scem 2622289774Scem error = sbuf_finish(&sb); 2623289774Scem sbuf_delete(&sb); 2624289774Scem 2625289774Scem if (error || !req->newptr) 2626289774Scem return (error); 2627289774Scem return (EINVAL); 2628289774Scem} 2629289774Scem 2630289774Scemstatic int 2631300100Scemsysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2632300100Scem{ 2633302483Smav struct ntb_softc *ntb = arg1; 2634300100Scem unsigned res; 2635300100Scem int error; 2636300100Scem 2637302484Smav res = ntb_link_is_up(ntb->device, NULL, NULL); 2638300100Scem 2639300100Scem error = SYSCTL_OUT(req, &res, sizeof(res)); 2640300100Scem if (error || !req->newptr) 2641300100Scem return (error); 2642300100Scem return (EINVAL); 2643300100Scem} 2644300100Scem 2645300100Scemstatic int 2646289774Scemsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2647289774Scem{ 2648289774Scem struct ntb_softc *ntb; 2649289774Scem const void *outp; 2650289774Scem uintptr_t sz; 2651289774Scem uint64_t umv; 2652289774Scem char be[sizeof(umv)]; 2653289774Scem size_t outsz; 2654289774Scem uint32_t reg; 2655289774Scem bool db, pci; 2656289774Scem int error; 2657289774Scem 2658289774Scem ntb = arg1; 2659289774Scem reg = arg2 & ~NTB_REGFLAGS_MASK; 2660289774Scem sz = arg2 & NTB_REGSZ_MASK; 2661289774Scem db = (arg2 & NTB_DB_READ) != 0; 2662289774Scem pci = (arg2 & NTB_PCI_REG) != 0; 2663289774Scem 2664289774Scem KASSERT(!(db && pci), ("bogus")); 2665289774Scem 2666289774Scem if (db) { 2667289774Scem KASSERT(sz == NTB_REG_64, ("bogus")); 2668289774Scem umv = db_ioread(ntb, reg); 2669289774Scem outsz = sizeof(uint64_t); 2670289774Scem } else { 2671289774Scem switch (sz) { 2672289774Scem case NTB_REG_64: 2673289774Scem if (pci) 2674289774Scem umv = pci_read_config(ntb->device, reg, 8); 2675289774Scem else 2676289774Scem umv = ntb_reg_read(8, reg); 2677289774Scem outsz = sizeof(uint64_t); 2678289774Scem break; 2679289774Scem case NTB_REG_32: 2680289774Scem if (pci) 2681289774Scem umv = pci_read_config(ntb->device, reg, 4); 2682289774Scem else 2683289774Scem umv = ntb_reg_read(4, reg); 2684289774Scem outsz = sizeof(uint32_t); 2685289774Scem break; 2686289774Scem case NTB_REG_16: 2687289774Scem if (pci) 2688289774Scem umv = pci_read_config(ntb->device, reg, 2); 2689289774Scem else 2690289774Scem umv = ntb_reg_read(2, reg); 2691289774Scem outsz = sizeof(uint16_t); 2692289774Scem break; 2693289774Scem case NTB_REG_8: 2694289774Scem if (pci) 2695289774Scem umv = pci_read_config(ntb->device, reg, 1); 2696289774Scem else 2697289774Scem umv = ntb_reg_read(1, reg); 2698289774Scem outsz = sizeof(uint8_t); 2699289774Scem break; 2700289774Scem default: 2701289774Scem panic("bogus"); 2702289774Scem break; 2703289774Scem } 2704289774Scem } 2705289774Scem 2706289774Scem /* Encode bigendian so that sysctl -x is legible. */ 2707289774Scem be64enc(be, umv); 2708289774Scem outp = ((char *)be) + sizeof(umv) - outsz; 2709289774Scem 2710289774Scem error = SYSCTL_OUT(req, outp, outsz); 2711289774Scem if (error || !req->newptr) 2712289774Scem return (error); 2713289774Scem return (EINVAL); 2714289774Scem} 2715289774Scem 2716291263Scemstatic unsigned 2717291263Scemntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2718291263Scem{ 2719291263Scem 2720295618Scem if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2721295618Scem uidx >= ntb->b2b_mw_idx) || 2722295618Scem (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2723295618Scem uidx++; 2724295618Scem if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2725295618Scem uidx >= ntb->b2b_mw_idx) && 2726295618Scem (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2727295618Scem uidx++; 2728291263Scem return (uidx); 2729291263Scem} 2730291263Scem 2731295618Scemstatic void 2732295618Scemntb_exchange_msix(void *ctx) 2733295618Scem{ 2734295618Scem struct ntb_softc *ntb; 2735295618Scem uint32_t val; 2736295618Scem unsigned i; 2737295618Scem 2738295618Scem ntb = ctx; 2739295618Scem 2740301292Smav if (ntb->peer_msix_good) 2741301292Smav goto msix_good; 2742295618Scem if (ntb->peer_msix_done) 2743295618Scem goto msix_done; 2744295618Scem 2745295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2746302484Smav ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 2747295618Scem ntb->msix_data[i].nmd_data); 2748302484Smav ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 2749301293Smav ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2750295618Scem } 2751302484Smav ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2752295618Scem 2753302484Smav ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 2754295618Scem if (val != NTB_MSIX_VER_GUARD) 2755295618Scem goto reschedule; 2756295618Scem 2757295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2758302484Smav ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 2759301293Smav ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2760295618Scem ntb->peer_msix_data[i].nmd_data = val; 2761302484Smav ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 2762301293Smav ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2763295618Scem ntb->peer_msix_data[i].nmd_ofs = val; 2764295618Scem } 2765295618Scem 2766295618Scem ntb->peer_msix_done = true; 2767295618Scem 2768295618Scemmsix_done: 2769302484Smav ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2770302484Smav ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 2771295618Scem if (val != NTB_MSIX_RECEIVED) 2772295618Scem goto reschedule; 2773295618Scem 2774295618Scem ntb->peer_msix_good = true; 2775301292Smav /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2776301292Smav goto reschedule; 2777295618Scem 2778301292Smavmsix_good: 2779295618Scem ntb_poll_link(ntb); 2780302484Smav ntb_link_event(ntb->device); 2781295618Scem return; 2782295618Scem 2783295618Scemreschedule: 2784295618Scem ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2785301292Smav if (_xeon_link_is_up(ntb)) { 2786301292Smav callout_reset(&ntb->peer_msix_work, 2787301292Smav hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2788301292Smav ntb_exchange_msix, ntb); 2789301292Smav } else 2790302484Smav ntb_spad_clear(ntb->device); 2791295618Scem} 2792295618Scem 2793289546Scem/* 2794289546Scem * Public API to the rest of the OS 2795250079Scarl */ 2796250079Scarl 2797302484Smavstatic uint8_t 2798302484Smavntb_spad_count(device_t dev) 2799250079Scarl{ 2800302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2801250079Scarl 2802289539Scem return (ntb->spad_count); 2803250079Scarl} 2804250079Scarl 2805302484Smavstatic uint8_t 2806302484Smavntb_mw_count(device_t dev) 2807289396Scem{ 2808302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2809295618Scem uint8_t res; 2810289396Scem 2811295618Scem res = ntb->mw_count; 2812291263Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2813295618Scem res--; 2814295618Scem if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2815295618Scem res--; 2816295618Scem return (res); 2817289396Scem} 2818289396Scem 2819302484Smavstatic int 2820302484Smavntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 2821250079Scarl{ 2822302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2823250079Scarl 2824289539Scem if (idx >= ntb->spad_count) 2825250079Scarl return (EINVAL); 2826250079Scarl 2827289607Scem ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2828250079Scarl 2829250079Scarl return (0); 2830250079Scarl} 2831250079Scarl 2832295618Scem/* 2833295618Scem * Zeros the local scratchpad. 2834295618Scem */ 2835302484Smavstatic void 2836302484Smavntb_spad_clear(device_t dev) 2837295618Scem{ 2838302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2839295618Scem unsigned i; 2840295618Scem 2841295618Scem for (i = 0; i < ntb->spad_count; i++) 2842302484Smav ntb_spad_write(dev, i, 0); 2843295618Scem} 2844295618Scem 2845302484Smavstatic int 2846302484Smavntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2847250079Scarl{ 2848302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2849250079Scarl 2850289539Scem if (idx >= ntb->spad_count) 2851250079Scarl return (EINVAL); 2852250079Scarl 2853289607Scem *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2854250079Scarl 2855250079Scarl return (0); 2856250079Scarl} 2857250079Scarl 2858302484Smavstatic int 2859302484Smavntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 2860250079Scarl{ 2861302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2862250079Scarl 2863289539Scem if (idx >= ntb->spad_count) 2864250079Scarl return (EINVAL); 2865250079Scarl 2866302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2867290682Scem ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2868255279Scarl else 2869289542Scem ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2870250079Scarl 2871250079Scarl return (0); 2872250079Scarl} 2873250079Scarl 2874302484Smavstatic int 2875302484Smavntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2876250079Scarl{ 2877302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2878250079Scarl 2879289539Scem if (idx >= ntb->spad_count) 2880250079Scarl return (EINVAL); 2881250079Scarl 2882302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2883290682Scem *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2884255279Scarl else 2885289542Scem *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2886250079Scarl 2887250079Scarl return (0); 2888250079Scarl} 2889250079Scarl 2890302484Smavstatic int 2891302484Smavntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 2892291033Scem caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2893291033Scem bus_addr_t *plimit) 2894250079Scarl{ 2895302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2896289546Scem struct ntb_pci_bar_info *bar; 2897291033Scem bus_addr_t limit; 2898289546Scem size_t bar_b2b_off; 2899291033Scem enum ntb_bar bar_num; 2900250079Scarl 2901302484Smav if (mw_idx >= ntb_mw_count(dev)) 2902289546Scem return (EINVAL); 2903291263Scem mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 2904250079Scarl 2905291033Scem bar_num = ntb_mw_to_bar(ntb, mw_idx); 2906291033Scem bar = &ntb->bar_info[bar_num]; 2907289546Scem bar_b2b_off = 0; 2908289546Scem if (mw_idx == ntb->b2b_mw_idx) { 2909289546Scem KASSERT(ntb->b2b_off != 0, 2910289546Scem ("user shouldn't get non-shared b2b mw")); 2911289546Scem bar_b2b_off = ntb->b2b_off; 2912289546Scem } 2913250079Scarl 2914291033Scem if (bar_is_64bit(ntb, bar_num)) 2915291033Scem limit = BUS_SPACE_MAXADDR; 2916291033Scem else 2917291033Scem limit = BUS_SPACE_MAXADDR_32BIT; 2918291033Scem 2919289546Scem if (base != NULL) 2920289546Scem *base = bar->pbase + bar_b2b_off; 2921289546Scem if (vbase != NULL) 2922290679Scem *vbase = bar->vbase + bar_b2b_off; 2923289546Scem if (size != NULL) 2924289546Scem *size = bar->size - bar_b2b_off; 2925289546Scem if (align != NULL) 2926289546Scem *align = bar->size; 2927289546Scem if (align_size != NULL) 2928289546Scem *align_size = 1; 2929291033Scem if (plimit != NULL) 2930291033Scem *plimit = limit; 2931289546Scem return (0); 2932250079Scarl} 2933250079Scarl 2934302484Smavstatic int 2935302484Smavntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 2936250079Scarl{ 2937302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2938289546Scem struct ntb_pci_bar_info *bar; 2939289546Scem uint64_t base, limit, reg_val; 2940289546Scem size_t bar_size, mw_size; 2941289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2942289546Scem enum ntb_bar bar_num; 2943250079Scarl 2944302484Smav if (idx >= ntb_mw_count(dev)) 2945289546Scem return (EINVAL); 2946291263Scem idx = ntb_user_mw_to_idx(ntb, idx); 2947250079Scarl 2948289546Scem bar_num = ntb_mw_to_bar(ntb, idx); 2949289546Scem bar = &ntb->bar_info[bar_num]; 2950250079Scarl 2951289546Scem bar_size = bar->size; 2952289546Scem if (idx == ntb->b2b_mw_idx) 2953289546Scem mw_size = bar_size - ntb->b2b_off; 2954289546Scem else 2955289546Scem mw_size = bar_size; 2956250079Scarl 2957289546Scem /* Hardware requires that addr is aligned to bar size */ 2958289546Scem if ((addr & (bar_size - 1)) != 0) 2959289546Scem return (EINVAL); 2960250079Scarl 2961289546Scem if (size > mw_size) 2962289546Scem return (EINVAL); 2963289546Scem 2964289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2965289546Scem 2966289546Scem limit = 0; 2967289546Scem if (bar_is_64bit(ntb, bar_num)) { 2968291032Scem base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2969289546Scem 2970289546Scem if (limit_reg != 0 && size != mw_size) 2971289546Scem limit = base + size; 2972289546Scem 2973289546Scem /* Set and verify translation address */ 2974289546Scem ntb_reg_write(8, xlat_reg, addr); 2975291032Scem reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2976289546Scem if (reg_val != addr) { 2977289546Scem ntb_reg_write(8, xlat_reg, 0); 2978289546Scem return (EIO); 2979289546Scem } 2980289546Scem 2981289546Scem /* Set and verify the limit */ 2982289546Scem ntb_reg_write(8, limit_reg, limit); 2983291032Scem reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 2984289546Scem if (reg_val != limit) { 2985289546Scem ntb_reg_write(8, limit_reg, base); 2986289546Scem ntb_reg_write(8, xlat_reg, 0); 2987289546Scem return (EIO); 2988289546Scem } 2989289546Scem } else { 2990289546Scem /* Configure 32-bit (split) BAR MW */ 2991289546Scem 2992291029Scem if ((addr & UINT32_MAX) != addr) 2993291033Scem return (ERANGE); 2994291029Scem if (((addr + size) & UINT32_MAX) != (addr + size)) 2995291033Scem return (ERANGE); 2996289546Scem 2997291032Scem base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 2998289546Scem 2999289546Scem if (limit_reg != 0 && size != mw_size) 3000289546Scem limit = base + size; 3001289546Scem 3002289546Scem /* Set and verify translation address */ 3003289546Scem ntb_reg_write(4, xlat_reg, addr); 3004291032Scem reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3005289546Scem if (reg_val != addr) { 3006289546Scem ntb_reg_write(4, xlat_reg, 0); 3007289546Scem return (EIO); 3008289546Scem } 3009289546Scem 3010289546Scem /* Set and verify the limit */ 3011289546Scem ntb_reg_write(4, limit_reg, limit); 3012291032Scem reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3013289546Scem if (reg_val != limit) { 3014289546Scem ntb_reg_write(4, limit_reg, base); 3015289546Scem ntb_reg_write(4, xlat_reg, 0); 3016289546Scem return (EIO); 3017289546Scem } 3018250079Scarl } 3019289546Scem return (0); 3020250079Scarl} 3021250079Scarl 3022302484Smavstatic int 3023302484Smavntb_mw_clear_trans(device_t dev, unsigned mw_idx) 3024289596Scem{ 3025289596Scem 3026302484Smav return (ntb_mw_set_trans(dev, mw_idx, 0, 0)); 3027289596Scem} 3028289596Scem 3029302484Smavstatic int 3030302484Smavntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 3031291031Scem{ 3032302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3033291031Scem struct ntb_pci_bar_info *bar; 3034291031Scem 3035302484Smav if (idx >= ntb_mw_count(dev)) 3036291031Scem return (EINVAL); 3037291263Scem idx = ntb_user_mw_to_idx(ntb, idx); 3038291031Scem 3039291031Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3040291280Scem *mode = bar->map_mode; 3041291031Scem return (0); 3042291031Scem} 3043291031Scem 3044302484Smavstatic int 3045302484Smavntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 3046291031Scem{ 3047302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3048291263Scem 3049302484Smav if (idx >= ntb_mw_count(dev)) 3050291263Scem return (EINVAL); 3051291263Scem 3052291263Scem idx = ntb_user_mw_to_idx(ntb, idx); 3053291280Scem return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3054291263Scem} 3055291263Scem 3056291263Scemstatic int 3057291280Scemntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3058291263Scem{ 3059291031Scem struct ntb_pci_bar_info *bar; 3060291031Scem int rc; 3061291031Scem 3062291031Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3063291280Scem if (bar->map_mode == mode) 3064291031Scem return (0); 3065291031Scem 3066291280Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3067291031Scem if (rc == 0) 3068291280Scem bar->map_mode = mode; 3069291031Scem 3070291031Scem return (rc); 3071291031Scem} 3072291031Scem 3073302484Smavstatic void 3074302484Smavntb_peer_db_set(device_t dev, uint64_t bit) 3075250079Scarl{ 3076302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3077250079Scarl 3078302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 3079295618Scem struct ntb_pci_bar_info *lapic; 3080295618Scem unsigned i; 3081295618Scem 3082295618Scem lapic = ntb->peer_lapic_bar; 3083295618Scem 3084295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3085302484Smav if ((bit & ntb_db_vector_mask(dev, i)) != 0) 3086295618Scem bus_space_write_4(lapic->pci_bus_tag, 3087295618Scem lapic->pci_bus_handle, 3088295618Scem ntb->peer_msix_data[i].nmd_ofs, 3089295618Scem ntb->peer_msix_data[i].nmd_data); 3090295618Scem } 3091295618Scem return; 3092295618Scem } 3093295618Scem 3094302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3095290682Scem ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3096289347Scem return; 3097289209Scem } 3098289347Scem 3099289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3100250079Scarl} 3101250079Scarl 3102302484Smavstatic int 3103302484Smavntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3104289542Scem{ 3105302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3106289542Scem struct ntb_pci_bar_info *bar; 3107289542Scem uint64_t regoff; 3108289542Scem 3109302484Smav KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3110289542Scem 3111302484Smav if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3112289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3113289542Scem regoff = ntb->peer_reg->db_bell; 3114289542Scem } else { 3115289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3116289543Scem ("invalid b2b idx")); 3117289542Scem 3118289542Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3119290682Scem regoff = XEON_PDOORBELL_OFFSET; 3120289542Scem } 3121289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3122289542Scem 3123289542Scem /* HACK: Specific to current x86 bus implementation. */ 3124302484Smav *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3125302484Smav *db_size = ntb->reg->db_size; 3126302484Smav return (0); 3127289542Scem} 3128289542Scem 3129302484Smavstatic uint64_t 3130302484Smavntb_db_valid_mask(device_t dev) 3131289597Scem{ 3132302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3133289597Scem 3134289597Scem return (ntb->db_valid_mask); 3135289597Scem} 3136289597Scem 3137302484Smavstatic int 3138302484Smavntb_db_vector_count(device_t dev) 3139289598Scem{ 3140302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3141289598Scem 3142302484Smav return (ntb->db_vec_count); 3143302484Smav} 3144302484Smav 3145302484Smavstatic uint64_t 3146302484Smavntb_db_vector_mask(device_t dev, uint32_t vector) 3147302484Smav{ 3148302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3149302484Smav 3150289598Scem if (vector > ntb->db_vec_count) 3151289598Scem return (0); 3152289598Scem return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3153289598Scem} 3154289598Scem 3155302484Smavstatic bool 3156302484Smavntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3157250079Scarl{ 3158302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3159250079Scarl 3160289546Scem if (speed != NULL) 3161289546Scem *speed = ntb_link_sta_speed(ntb); 3162289546Scem if (width != NULL) 3163289546Scem *width = ntb_link_sta_width(ntb); 3164289546Scem return (link_is_up(ntb)); 3165250079Scarl} 3166250079Scarl 3167255272Scarlstatic void 3168255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 3169250079Scarl{ 3170255272Scarl 3171289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3172289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3173289209Scem bar->pbase = rman_get_start(bar->pci_resource); 3174289209Scem bar->size = rman_get_size(bar->pci_resource); 3175289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 3176250079Scarl} 3177255268Scarl 3178302484Smavstatic device_method_t ntb_intel_methods[] = { 3179302484Smav /* Device interface */ 3180302484Smav DEVMETHOD(device_probe, ntb_probe), 3181302484Smav DEVMETHOD(device_attach, ntb_attach), 3182302484Smav DEVMETHOD(device_detach, ntb_detach), 3183302484Smav /* NTB interface */ 3184302484Smav DEVMETHOD(ntb_link_is_up, ntb_link_is_up), 3185302484Smav DEVMETHOD(ntb_link_enable, ntb_link_enable), 3186302484Smav DEVMETHOD(ntb_link_disable, ntb_link_disable), 3187302484Smav DEVMETHOD(ntb_link_enabled, ntb_link_enabled), 3188302484Smav DEVMETHOD(ntb_set_ctx, ntb_set_ctx), 3189302484Smav DEVMETHOD(ntb_get_ctx, ntb_get_ctx), 3190302484Smav DEVMETHOD(ntb_clear_ctx, ntb_clear_ctx), 3191302484Smav DEVMETHOD(ntb_mw_count, ntb_mw_count), 3192302484Smav DEVMETHOD(ntb_mw_get_range, ntb_mw_get_range), 3193302484Smav DEVMETHOD(ntb_mw_set_trans, ntb_mw_set_trans), 3194302484Smav DEVMETHOD(ntb_mw_clear_trans, ntb_mw_clear_trans), 3195302484Smav DEVMETHOD(ntb_mw_get_wc, ntb_mw_get_wc), 3196302484Smav DEVMETHOD(ntb_mw_set_wc, ntb_mw_set_wc), 3197302484Smav DEVMETHOD(ntb_spad_count, ntb_spad_count), 3198302484Smav DEVMETHOD(ntb_spad_clear, ntb_spad_clear), 3199302484Smav DEVMETHOD(ntb_spad_write, ntb_spad_write), 3200302484Smav DEVMETHOD(ntb_spad_read, ntb_spad_read), 3201302484Smav DEVMETHOD(ntb_peer_spad_write, ntb_peer_spad_write), 3202302484Smav DEVMETHOD(ntb_peer_spad_read, ntb_peer_spad_read), 3203302484Smav DEVMETHOD(ntb_db_valid_mask, ntb_db_valid_mask), 3204302484Smav DEVMETHOD(ntb_db_vector_count, ntb_db_vector_count), 3205302484Smav DEVMETHOD(ntb_db_vector_mask, ntb_db_vector_mask), 3206302484Smav DEVMETHOD(ntb_db_clear, ntb_db_clear), 3207302484Smav DEVMETHOD(ntb_db_clear_mask, ntb_db_clear_mask), 3208302484Smav DEVMETHOD(ntb_db_read, ntb_db_read), 3209302484Smav DEVMETHOD(ntb_db_set_mask, ntb_db_set_mask), 3210302484Smav DEVMETHOD(ntb_peer_db_addr, ntb_peer_db_addr), 3211302484Smav DEVMETHOD(ntb_peer_db_set, ntb_peer_db_set), 3212302484Smav DEVMETHOD_END 3213302484Smav}; 3214255268Scarl 3215302484Smavstatic DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3216302484Smav sizeof(struct ntb_softc)); 3217302484SmavDRIVER_MODULE(ntb_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL); 3218302484SmavMODULE_DEPEND(ntb_intel, ntb, 1, 1, 1); 3219302484SmavMODULE_VERSION(ntb_intel, 1); 3220