ntb_hw_intel.c revision 303561
1250079Scarl/*- 2302484Smav * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 3250079Scarl * Copyright (C) 2013 Intel Corporation 4289542Scem * Copyright (C) 2015 EMC Corporation 5250079Scarl * All rights reserved. 6250079Scarl * 7250079Scarl * Redistribution and use in source and binary forms, with or without 8250079Scarl * modification, are permitted provided that the following conditions 9250079Scarl * are met: 10250079Scarl * 1. Redistributions of source code must retain the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer. 12250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 13250079Scarl * notice, this list of conditions and the following disclaimer in the 14250079Scarl * documentation and/or other materials provided with the distribution. 15250079Scarl * 16250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26250079Scarl * SUCH DAMAGE. 27250079Scarl */ 28250079Scarl 29302484Smav/* 30302484Smav * The Non-Transparent Bridge (NTB) is a device that allows you to connect 31302484Smav * two or more systems using a PCI-e links, providing remote memory access. 32302484Smav * 33302484Smav * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 34302484Smav * 35302484Smav * NOTE: Much of the code in this module is shared with Linux. Any patches may 36302484Smav * be picked up and redistributed in Linux with a dual GPL/BSD license. 37302484Smav */ 38302484Smav 39250079Scarl#include <sys/cdefs.h> 40250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 303561 2016-07-30 21:06:59Z mav $"); 41250079Scarl 42250079Scarl#include <sys/param.h> 43250079Scarl#include <sys/kernel.h> 44250079Scarl#include <sys/systm.h> 45250079Scarl#include <sys/bus.h> 46289774Scem#include <sys/endian.h> 47302493Smav#include <sys/interrupt.h> 48250079Scarl#include <sys/malloc.h> 49250079Scarl#include <sys/module.h> 50295618Scem#include <sys/mutex.h> 51295618Scem#include <sys/pciio.h> 52250079Scarl#include <sys/queue.h> 53250079Scarl#include <sys/rman.h> 54289774Scem#include <sys/sbuf.h> 55289207Scem#include <sys/sysctl.h> 56250079Scarl#include <vm/vm.h> 57250079Scarl#include <vm/pmap.h> 58250079Scarl#include <machine/bus.h> 59295618Scem#include <machine/intr_machdep.h> 60250079Scarl#include <machine/resource.h> 61250079Scarl#include <dev/pci/pcireg.h> 62250079Scarl#include <dev/pci/pcivar.h> 63250079Scarl 64250079Scarl#include "ntb_regs.h" 65302484Smav#include "../ntb.h" 66250079Scarl 67289648Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 68250079Scarl 69289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 70289648Scem#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 71291032Scem#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 72250079Scarl 73295618Scem#define NTB_MSIX_VER_GUARD 0xaabbccdd 74295618Scem#define NTB_MSIX_RECEIVED 0xe0f0e0f0 75295618Scem 76295618Scem/* 77295618Scem * PCI constants could be somewhere more generic, but aren't defined/used in 78295618Scem * pci.c. 79295618Scem */ 80295618Scem#define PCI_MSIX_ENTRY_SIZE 16 81295618Scem#define PCI_MSIX_ENTRY_LOWER_ADDR 0 82295618Scem#define PCI_MSIX_ENTRY_UPPER_ADDR 4 83295618Scem#define PCI_MSIX_ENTRY_DATA 8 84295618Scem 85250079Scarlenum ntb_device_type { 86250079Scarl NTB_XEON, 87289648Scem NTB_ATOM 88250079Scarl}; 89250079Scarl 90289610Scem/* ntb_conn_type are hardware numbers, cannot change. */ 91289610Scemenum ntb_conn_type { 92289610Scem NTB_CONN_TRANSPARENT = 0, 93289610Scem NTB_CONN_B2B = 1, 94289610Scem NTB_CONN_RP = 2, 95289610Scem}; 96289610Scem 97289610Scemenum ntb_b2b_direction { 98289610Scem NTB_DEV_USD = 0, 99289610Scem NTB_DEV_DSD = 1, 100289610Scem}; 101289610Scem 102289539Scemenum ntb_bar { 103289539Scem NTB_CONFIG_BAR = 0, 104289539Scem NTB_B2B_BAR_1, 105289539Scem NTB_B2B_BAR_2, 106289539Scem NTB_B2B_BAR_3, 107289539Scem NTB_MAX_BARS 108289539Scem}; 109289539Scem 110295618Scemenum { 111295618Scem NTB_MSIX_GUARD = 0, 112295618Scem NTB_MSIX_DATA0, 113295618Scem NTB_MSIX_DATA1, 114295618Scem NTB_MSIX_DATA2, 115295618Scem NTB_MSIX_OFS0, 116295618Scem NTB_MSIX_OFS1, 117295618Scem NTB_MSIX_OFS2, 118295618Scem NTB_MSIX_DONE, 119295618Scem NTB_MAX_MSIX_SPAD 120295618Scem}; 121295618Scem 122255274Scarl/* Device features and workarounds */ 123302484Smav#define HAS_FEATURE(ntb, feature) \ 124302484Smav (((ntb)->features & (feature)) != 0) 125255274Scarl 126250079Scarlstruct ntb_hw_info { 127250079Scarl uint32_t device_id; 128255274Scarl const char *desc; 129250079Scarl enum ntb_device_type type; 130289397Scem uint32_t features; 131250079Scarl}; 132250079Scarl 133250079Scarlstruct ntb_pci_bar_info { 134250079Scarl bus_space_tag_t pci_bus_tag; 135250079Scarl bus_space_handle_t pci_bus_handle; 136250079Scarl int pci_resource_id; 137250079Scarl struct resource *pci_resource; 138250079Scarl vm_paddr_t pbase; 139290679Scem caddr_t vbase; 140290679Scem vm_size_t size; 141291280Scem vm_memattr_t map_mode; 142289543Scem 143289543Scem /* Configuration register offsets */ 144289543Scem uint32_t psz_off; 145289543Scem uint32_t ssz_off; 146289543Scem uint32_t pbarxlat_off; 147250079Scarl}; 148250079Scarl 149250079Scarlstruct ntb_int_info { 150250079Scarl struct resource *res; 151250079Scarl int rid; 152250079Scarl void *tag; 153250079Scarl}; 154250079Scarl 155289546Scemstruct ntb_vec { 156250079Scarl struct ntb_softc *ntb; 157289546Scem uint32_t num; 158295618Scem unsigned masked; 159250079Scarl}; 160250079Scarl 161289542Scemstruct ntb_reg { 162289542Scem uint32_t ntb_ctl; 163289542Scem uint32_t lnk_sta; 164289542Scem uint8_t db_size; 165289542Scem unsigned mw_bar[NTB_MAX_BARS]; 166289542Scem}; 167289542Scem 168289542Scemstruct ntb_alt_reg { 169289542Scem uint32_t db_bell; 170289542Scem uint32_t db_mask; 171289542Scem uint32_t spad; 172289542Scem}; 173289542Scem 174289542Scemstruct ntb_xlat_reg { 175289546Scem uint32_t bar0_base; 176289546Scem uint32_t bar2_base; 177289546Scem uint32_t bar4_base; 178289546Scem uint32_t bar5_base; 179289546Scem 180289546Scem uint32_t bar2_xlat; 181289546Scem uint32_t bar4_xlat; 182289546Scem uint32_t bar5_xlat; 183289546Scem 184289546Scem uint32_t bar2_limit; 185289546Scem uint32_t bar4_limit; 186289546Scem uint32_t bar5_limit; 187289542Scem}; 188289542Scem 189289542Scemstruct ntb_b2b_addr { 190289542Scem uint64_t bar0_addr; 191289542Scem uint64_t bar2_addr64; 192289542Scem uint64_t bar4_addr64; 193289542Scem uint64_t bar4_addr32; 194289542Scem uint64_t bar5_addr32; 195289542Scem}; 196289542Scem 197295618Scemstruct ntb_msix_data { 198295618Scem uint32_t nmd_ofs; 199295618Scem uint32_t nmd_data; 200295618Scem}; 201295618Scem 202250079Scarlstruct ntb_softc { 203303429Smav /* ntb.c context. Do not move! Must go first! */ 204303429Smav void *ntb_store; 205303429Smav 206250079Scarl device_t device; 207250079Scarl enum ntb_device_type type; 208289774Scem uint32_t features; 209250079Scarl 210250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 211250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 212250079Scarl uint32_t allocated_interrupts; 213250079Scarl 214295618Scem struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215295618Scem struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216295618Scem bool peer_msix_good; 217295618Scem bool peer_msix_done; 218295618Scem struct ntb_pci_bar_info *peer_lapic_bar; 219295618Scem struct callout peer_msix_work; 220295618Scem 221250079Scarl struct callout heartbeat_timer; 222250079Scarl struct callout lr_timer; 223250079Scarl 224289546Scem struct ntb_vec *msix_vec; 225250079Scarl 226289610Scem uint32_t ppd; 227289610Scem enum ntb_conn_type conn_type; 228289610Scem enum ntb_b2b_direction dev_type; 229289539Scem 230289542Scem /* Offset of peer bar0 in B2B BAR */ 231289542Scem uint64_t b2b_off; 232289542Scem /* Memory window used to access peer bar0 */ 233289543Scem#define B2B_MW_DISABLED UINT8_MAX 234289542Scem uint8_t b2b_mw_idx; 235301293Smav uint32_t msix_xlat; 236295618Scem uint8_t msix_mw_idx; 237289542Scem 238289539Scem uint8_t mw_count; 239289539Scem uint8_t spad_count; 240289539Scem uint8_t db_count; 241289539Scem uint8_t db_vec_count; 242289539Scem uint8_t db_vec_shift; 243289542Scem 244289546Scem /* Protects local db_mask. */ 245289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 246289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 247289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 248289542Scem struct mtx db_mask_lock; 249289542Scem 250290686Scem volatile uint32_t ntb_ctl; 251290686Scem volatile uint32_t lnk_sta; 252289542Scem 253289542Scem uint64_t db_valid_mask; 254289542Scem uint64_t db_link_mask; 255289546Scem uint64_t db_mask; 256302493Smav uint64_t fake_db_bell; /* NTB_SB01BASE_LOCKUP*/ 257289542Scem 258289542Scem int last_ts; /* ticks @ last irq */ 259289542Scem 260289542Scem const struct ntb_reg *reg; 261289542Scem const struct ntb_alt_reg *self_reg; 262289542Scem const struct ntb_alt_reg *peer_reg; 263289542Scem const struct ntb_xlat_reg *xlat_reg; 264250079Scarl}; 265250079Scarl 266289234Scem#ifdef __i386__ 267289234Scemstatic __inline uint64_t 268289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 269289234Scem bus_size_t offset) 270289234Scem{ 271289234Scem 272289234Scem return (bus_space_read_4(tag, handle, offset) | 273289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 274289234Scem} 275289234Scem 276289234Scemstatic __inline void 277289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 278289234Scem bus_size_t offset, uint64_t val) 279289234Scem{ 280289234Scem 281289234Scem bus_space_write_4(tag, handle, offset, val); 282289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 283289234Scem} 284289234Scem#endif 285289234Scem 286303429Smav#define intel_ntb_bar_read(SIZE, bar, offset) \ 287255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 288255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 289303429Smav#define intel_ntb_bar_write(SIZE, bar, offset, val) \ 290255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 291255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 292303429Smav#define intel_ntb_reg_read(SIZE, offset) \ 293303429Smav intel_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 294303429Smav#define intel_ntb_reg_write(SIZE, offset, val) \ 295303429Smav intel_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 296303429Smav#define intel_ntb_mw_read(SIZE, offset) \ 297303429Smav intel_ntb_bar_read(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 298303429Smav offset) 299303429Smav#define intel_ntb_mw_write(SIZE, offset, val) \ 300303429Smav intel_ntb_bar_write(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 301289397Scem offset, val) 302250079Scarl 303303429Smavstatic int intel_ntb_probe(device_t device); 304303429Smavstatic int intel_ntb_attach(device_t device); 305303429Smavstatic int intel_ntb_detach(device_t device); 306303429Smavstatic uint64_t intel_ntb_db_valid_mask(device_t dev); 307303429Smavstatic void intel_ntb_spad_clear(device_t dev); 308303429Smavstatic uint64_t intel_ntb_db_vector_mask(device_t dev, uint32_t vector); 309303429Smavstatic bool intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, 310302484Smav enum ntb_width *width); 311303429Smavstatic int intel_ntb_link_enable(device_t dev, enum ntb_speed speed, 312302484Smav enum ntb_width width); 313303429Smavstatic int intel_ntb_link_disable(device_t dev); 314303429Smavstatic int intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 315303429Smavstatic int intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 316302484Smav 317303429Smavstatic unsigned intel_ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 318303429Smavstatic inline enum ntb_bar intel_ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 319289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 320289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 321289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 322303429Smavstatic int intel_ntb_map_pci_bars(struct ntb_softc *ntb); 323303429Smavstatic int intel_ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 324291280Scem vm_memattr_t); 325289647Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 326289647Scem const char *); 327255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 328255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 329255272Scarl struct ntb_pci_bar_info *bar); 330303429Smavstatic void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb); 331303429Smavstatic int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 332303429Smavstatic int intel_ntb_init_isr(struct ntb_softc *ntb); 333303429Smavstatic int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 334303429Smavstatic int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 335303429Smavstatic void intel_ntb_teardown_interrupts(struct ntb_softc *ntb); 336303429Smavstatic inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 337303429Smavstatic void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec); 338289546Scemstatic void ndev_vec_isr(void *arg); 339289546Scemstatic void ndev_irq_isr(void *arg); 340289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 341290678Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 342290678Scemstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 343303429Smavstatic int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 344303429Smavstatic void intel_ntb_free_msix_vec(struct ntb_softc *ntb); 345303429Smavstatic void intel_ntb_get_msix_info(struct ntb_softc *ntb); 346303429Smavstatic void intel_ntb_exchange_msix(void *); 347303429Smavstatic struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id); 348303429Smavstatic void intel_ntb_detect_max_mw(struct ntb_softc *ntb); 349303429Smavstatic int intel_ntb_detect_xeon(struct ntb_softc *ntb); 350303429Smavstatic int intel_ntb_detect_atom(struct ntb_softc *ntb); 351303429Smavstatic int intel_ntb_xeon_init_dev(struct ntb_softc *ntb); 352303429Smavstatic int intel_ntb_atom_init_dev(struct ntb_softc *ntb); 353303429Smavstatic void intel_ntb_teardown_xeon(struct ntb_softc *ntb); 354289648Scemstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 355289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 356289543Scem enum ntb_bar regbar); 357289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 358289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 359289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 360289543Scem enum ntb_bar idx); 361289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 362289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 363289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 364295618Scemstatic inline bool _xeon_link_is_up(struct ntb_softc *ntb); 365289648Scemstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 366303429Smavstatic inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *); 367303429Smavstatic inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *); 368289648Scemstatic void atom_link_hb(void *arg); 369289648Scemstatic void recover_atom_link(void *arg); 370303429Smavstatic bool intel_ntb_poll_link(struct ntb_softc *ntb); 371255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 372303429Smavstatic void intel_ntb_sysctl_init(struct ntb_softc *); 373289774Scemstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 374300100Scemstatic int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 375300100Scemstatic int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 376289774Scemstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 377289774Scemstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 378250079Scarl 379290685Scemstatic unsigned g_ntb_hw_debug_level; 380290685ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 381290685Scem &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 382303429Smav#define intel_ntb_printf(lvl, ...) do { \ 383290685Scem if ((lvl) <= g_ntb_hw_debug_level) { \ 384290685Scem device_printf(ntb->device, __VA_ARGS__); \ 385290685Scem } \ 386290685Scem} while (0) 387290685Scem 388295486Scem#define _NTB_PAT_UC 0 389295486Scem#define _NTB_PAT_WC 1 390295486Scem#define _NTB_PAT_WT 4 391295486Scem#define _NTB_PAT_WP 5 392295486Scem#define _NTB_PAT_WB 6 393295486Scem#define _NTB_PAT_UCM 7 394295486Scemstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 395295486ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 396295486Scem &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 397295486Scem "UC: " __XSTRING(_NTB_PAT_UC) ", " 398295486Scem "WC: " __XSTRING(_NTB_PAT_WC) ", " 399295486Scem "WT: " __XSTRING(_NTB_PAT_WT) ", " 400295486Scem "WP: " __XSTRING(_NTB_PAT_WP) ", " 401295486Scem "WB: " __XSTRING(_NTB_PAT_WB) ", " 402295486Scem "UC-: " __XSTRING(_NTB_PAT_UCM)); 403291030Scem 404295486Scemstatic inline vm_memattr_t 405303429Smavintel_ntb_pat_flags(void) 406295486Scem{ 407295486Scem 408295486Scem switch (g_ntb_mw_pat) { 409295486Scem case _NTB_PAT_WC: 410295486Scem return (VM_MEMATTR_WRITE_COMBINING); 411295486Scem case _NTB_PAT_WT: 412295486Scem return (VM_MEMATTR_WRITE_THROUGH); 413295486Scem case _NTB_PAT_WP: 414295486Scem return (VM_MEMATTR_WRITE_PROTECTED); 415295486Scem case _NTB_PAT_WB: 416295486Scem return (VM_MEMATTR_WRITE_BACK); 417295486Scem case _NTB_PAT_UCM: 418295486Scem return (VM_MEMATTR_WEAK_UNCACHEABLE); 419295486Scem case _NTB_PAT_UC: 420295486Scem /* FALLTHROUGH */ 421295486Scem default: 422295486Scem return (VM_MEMATTR_UNCACHEABLE); 423295486Scem } 424295486Scem} 425295486Scem 426295487Scem/* 427295487Scem * Well, this obviously doesn't belong here, but it doesn't seem to exist 428295487Scem * anywhere better yet. 429295487Scem */ 430295487Scemstatic inline const char * 431303429Smavintel_ntb_vm_memattr_to_str(vm_memattr_t pat) 432295487Scem{ 433295487Scem 434295487Scem switch (pat) { 435295487Scem case VM_MEMATTR_WRITE_COMBINING: 436295487Scem return ("WRITE_COMBINING"); 437295487Scem case VM_MEMATTR_WRITE_THROUGH: 438295487Scem return ("WRITE_THROUGH"); 439295487Scem case VM_MEMATTR_WRITE_PROTECTED: 440295487Scem return ("WRITE_PROTECTED"); 441295487Scem case VM_MEMATTR_WRITE_BACK: 442295487Scem return ("WRITE_BACK"); 443295487Scem case VM_MEMATTR_WEAK_UNCACHEABLE: 444295487Scem return ("UNCACHED"); 445295487Scem case VM_MEMATTR_UNCACHEABLE: 446295487Scem return ("UNCACHEABLE"); 447295487Scem default: 448295487Scem return ("UNKNOWN"); 449295487Scem } 450295487Scem} 451295487Scem 452302508Smavstatic int g_ntb_msix_idx = 1; 453295618ScemSYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 454295618Scem 0, "Use this memory window to access the peer MSIX message complex on " 455295618Scem "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 456295618Scem "Like b2b_mw_idx, negative values index from the last available memory " 457295618Scem "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 458295618Scem 459291263Scemstatic int g_ntb_mw_idx = -1; 460291263ScemSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 461291263Scem 0, "Use this memory window to access the peer NTB registers. A " 462291263Scem "non-negative value starts from the first MW index; a negative value " 463291263Scem "starts from the last MW index. The default is -1, i.e., the last " 464291263Scem "available memory window. Both sides of the NTB MUST set the same " 465291263Scem "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 466291263Scem 467302484Smav/* Hardware owns the low 16 bits of features. */ 468302484Smav#define NTB_BAR_SIZE_4K (1 << 0) 469302484Smav#define NTB_SDOORBELL_LOCKUP (1 << 1) 470302484Smav#define NTB_SB01BASE_LOCKUP (1 << 2) 471302484Smav#define NTB_B2BDOORBELL_BIT14 (1 << 3) 472302484Smav/* Software/configuration owns the top 16 bits. */ 473302484Smav#define NTB_SPLIT_BAR (1ull << 16) 474302484Smav 475302484Smav#define NTB_FEATURES_STR \ 476302484Smav "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 477302484Smav "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 478302484Smav 479250079Scarlstatic struct ntb_hw_info pci_ids[] = { 480289612Scem /* XXX: PS/SS IDs left out until they are supported. */ 481289612Scem { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 482289648Scem NTB_ATOM, 0 }, 483289233Scem 484289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 485289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 486289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 487289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 488289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 489289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 490289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 491289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 492289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 493289538Scem NTB_SB01BASE_LOCKUP }, 494289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 495289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 496289538Scem NTB_SB01BASE_LOCKUP }, 497289233Scem 498289648Scem { 0x00000000, NULL, NTB_ATOM, 0 } 499250079Scarl}; 500250079Scarl 501289648Scemstatic const struct ntb_reg atom_reg = { 502289648Scem .ntb_ctl = ATOM_NTBCNTL_OFFSET, 503289648Scem .lnk_sta = ATOM_LINK_STATUS_OFFSET, 504289542Scem .db_size = sizeof(uint64_t), 505289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 506289542Scem}; 507289542Scem 508289648Scemstatic const struct ntb_alt_reg atom_pri_reg = { 509289648Scem .db_bell = ATOM_PDOORBELL_OFFSET, 510289648Scem .db_mask = ATOM_PDBMSK_OFFSET, 511289648Scem .spad = ATOM_SPAD_OFFSET, 512289607Scem}; 513289607Scem 514289648Scemstatic const struct ntb_alt_reg atom_b2b_reg = { 515289648Scem .db_bell = ATOM_B2B_DOORBELL_OFFSET, 516289648Scem .spad = ATOM_B2B_SPAD_OFFSET, 517289542Scem}; 518289542Scem 519289648Scemstatic const struct ntb_xlat_reg atom_sec_xlat = { 520289542Scem#if 0 521289542Scem /* "FIXME" says the Linux driver. */ 522289648Scem .bar0_base = ATOM_SBAR0BASE_OFFSET, 523289648Scem .bar2_base = ATOM_SBAR2BASE_OFFSET, 524289648Scem .bar4_base = ATOM_SBAR4BASE_OFFSET, 525289546Scem 526289648Scem .bar2_limit = ATOM_SBAR2LMT_OFFSET, 527289648Scem .bar4_limit = ATOM_SBAR4LMT_OFFSET, 528289542Scem#endif 529289546Scem 530289648Scem .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 531289648Scem .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 532289542Scem}; 533289542Scem 534289542Scemstatic const struct ntb_reg xeon_reg = { 535289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 536289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 537289542Scem .db_size = sizeof(uint16_t), 538289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 539289542Scem}; 540289542Scem 541289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 542289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 543289607Scem .db_mask = XEON_PDBMSK_OFFSET, 544289607Scem .spad = XEON_SPAD_OFFSET, 545289607Scem}; 546289607Scem 547289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 548289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 549289542Scem .spad = XEON_B2B_SPAD_OFFSET, 550289542Scem}; 551289542Scem 552289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 553289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 554289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 555289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 556289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 557289546Scem 558289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 559289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 560289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 561289546Scem 562289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 563289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 564289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 565289542Scem}; 566289542Scem 567289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 568290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 569290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 570290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 571290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 572290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 573289542Scem}; 574289542Scem 575289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 576290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 577290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 578290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 579290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 580290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 581289542Scem}; 582289542Scem 583289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 584289614Scem "B2B MW segment overrides -- MUST be the same on both sides"); 585289614Scem 586289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 587289614Scem &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 588289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 589289614Scem "the window at BAR2, on the upstream side of the link. MUST be the same " 590289614Scem "address on both sides."); 591289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 592289614Scem &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 593289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 594289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 595289614Scem "(split-BAR mode)."); 596289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 597289646Scem &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 598289614Scem "(split-BAR mode)."); 599289614Scem 600289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 601289614Scem &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 602289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 603289614Scem "the window at BAR2, on the downstream side of the link. MUST be the same" 604289614Scem " address on both sides."); 605289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 606289614Scem &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 607289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 608289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 609289614Scem "(split-BAR mode)."); 610289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 611289646Scem &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 612289614Scem "(split-BAR mode)."); 613289614Scem 614250079Scarl/* 615250079Scarl * OS <-> Driver interface structures 616250079Scarl */ 617250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 618250079Scarl 619250079Scarl/* 620250079Scarl * OS <-> Driver linkage functions 621250079Scarl */ 622250079Scarlstatic int 623303429Smavintel_ntb_probe(device_t device) 624250079Scarl{ 625289209Scem struct ntb_hw_info *p; 626250079Scarl 627303429Smav p = intel_ntb_get_device_info(pci_get_devid(device)); 628289209Scem if (p == NULL) 629250079Scarl return (ENXIO); 630289209Scem 631289209Scem device_set_desc(device, p->desc); 632289209Scem return (0); 633250079Scarl} 634250079Scarl 635250079Scarlstatic int 636303429Smavintel_ntb_attach(device_t device) 637250079Scarl{ 638289209Scem struct ntb_softc *ntb; 639289209Scem struct ntb_hw_info *p; 640250079Scarl int error; 641250079Scarl 642302484Smav ntb = device_get_softc(device); 643303429Smav p = intel_ntb_get_device_info(pci_get_devid(device)); 644289209Scem 645250079Scarl ntb->device = device; 646250079Scarl ntb->type = p->type; 647255274Scarl ntb->features = p->features; 648289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 649295618Scem ntb->msix_mw_idx = B2B_MW_DISABLED; 650250079Scarl 651289648Scem /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 652283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 653283291Sjkim callout_init(&ntb->lr_timer, 1); 654295618Scem callout_init(&ntb->peer_msix_work, 1); 655289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 656250079Scarl 657289648Scem if (ntb->type == NTB_ATOM) 658303429Smav error = intel_ntb_detect_atom(ntb); 659289348Scem else 660303429Smav error = intel_ntb_detect_xeon(ntb); 661290682Scem if (error != 0) 662289348Scem goto out; 663289348Scem 664303429Smav intel_ntb_detect_max_mw(ntb); 665289396Scem 666290682Scem pci_enable_busmaster(ntb->device); 667290682Scem 668303429Smav error = intel_ntb_map_pci_bars(ntb); 669290682Scem if (error != 0) 670289209Scem goto out; 671289648Scem if (ntb->type == NTB_ATOM) 672303429Smav error = intel_ntb_atom_init_dev(ntb); 673289272Scem else 674303429Smav error = intel_ntb_xeon_init_dev(ntb); 675290682Scem if (error != 0) 676289209Scem goto out; 677290682Scem 678303429Smav intel_ntb_spad_clear(device); 679295618Scem 680303429Smav intel_ntb_poll_link(ntb); 681290682Scem 682303429Smav intel_ntb_sysctl_init(ntb); 683250079Scarl 684302484Smav /* Attach children to this controller */ 685303429Smav error = ntb_register_device(device); 686302484Smav 687289209Scemout: 688289209Scem if (error != 0) 689303429Smav intel_ntb_detach(device); 690250079Scarl return (error); 691250079Scarl} 692250079Scarl 693250079Scarlstatic int 694303429Smavintel_ntb_detach(device_t device) 695250079Scarl{ 696289209Scem struct ntb_softc *ntb; 697250079Scarl 698302484Smav ntb = device_get_softc(device); 699289542Scem 700302484Smav /* Detach & delete all children */ 701303429Smav ntb_unregister_device(device); 702302484Smav 703295618Scem if (ntb->self_reg != NULL) { 704295618Scem DB_MASK_LOCK(ntb); 705295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 706295618Scem DB_MASK_UNLOCK(ntb); 707295618Scem } 708250079Scarl callout_drain(&ntb->heartbeat_timer); 709250079Scarl callout_drain(&ntb->lr_timer); 710295618Scem callout_drain(&ntb->peer_msix_work); 711290682Scem pci_disable_busmaster(ntb->device); 712289272Scem if (ntb->type == NTB_XEON) 713303429Smav intel_ntb_teardown_xeon(ntb); 714303429Smav intel_ntb_teardown_interrupts(ntb); 715289397Scem 716289542Scem mtx_destroy(&ntb->db_mask_lock); 717289542Scem 718303429Smav intel_ntb_unmap_pci_bar(ntb); 719250079Scarl 720250079Scarl return (0); 721250079Scarl} 722250079Scarl 723289542Scem/* 724289542Scem * Driver internal routines 725289542Scem */ 726289539Scemstatic inline enum ntb_bar 727303429Smavintel_ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 728289539Scem{ 729289539Scem 730291263Scem KASSERT(mw < ntb->mw_count, 731289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 732289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 733289539Scem 734289542Scem return (ntb->reg->mw_bar[mw]); 735289539Scem} 736289539Scem 737289546Scemstatic inline bool 738289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 739289546Scem{ 740289546Scem /* XXX This assertion could be stronger. */ 741289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 742302484Smav return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 743289546Scem} 744289546Scem 745289546Scemstatic inline void 746289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 747289546Scem uint32_t *xlat, uint32_t *lmt) 748289546Scem{ 749289546Scem uint32_t basev, lmtv, xlatv; 750289546Scem 751289546Scem switch (bar) { 752289546Scem case NTB_B2B_BAR_1: 753289546Scem basev = ntb->xlat_reg->bar2_base; 754289546Scem lmtv = ntb->xlat_reg->bar2_limit; 755289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 756289546Scem break; 757289546Scem case NTB_B2B_BAR_2: 758289546Scem basev = ntb->xlat_reg->bar4_base; 759289546Scem lmtv = ntb->xlat_reg->bar4_limit; 760289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 761289546Scem break; 762289546Scem case NTB_B2B_BAR_3: 763289546Scem basev = ntb->xlat_reg->bar5_base; 764289546Scem lmtv = ntb->xlat_reg->bar5_limit; 765289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 766289546Scem break; 767289546Scem default: 768289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 769289546Scem ("bad bar")); 770289546Scem basev = lmtv = xlatv = 0; 771289546Scem break; 772289546Scem } 773289546Scem 774289546Scem if (base != NULL) 775289546Scem *base = basev; 776289546Scem if (xlat != NULL) 777289546Scem *xlat = xlatv; 778289546Scem if (lmt != NULL) 779289546Scem *lmt = lmtv; 780289546Scem} 781289546Scem 782250079Scarlstatic int 783303429Smavintel_ntb_map_pci_bars(struct ntb_softc *ntb) 784250079Scarl{ 785255272Scarl int rc; 786250079Scarl 787250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 788289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 789255272Scarl if (rc != 0) 790289541Scem goto out; 791255272Scarl 792289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 793289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 794255272Scarl if (rc != 0) 795289541Scem goto out; 796289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 797289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 798289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 799255272Scarl 800289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 801291263Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 802291263Scem if (rc != 0) 803291263Scem goto out; 804289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 805289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 806289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 807289543Scem 808302484Smav if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 809289541Scem goto out; 810289397Scem 811289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 812291263Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 813289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 814289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 815289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 816250079Scarl 817289541Scemout: 818289209Scem if (rc != 0) 819255272Scarl device_printf(ntb->device, 820255272Scarl "unable to allocate pci resource\n"); 821255272Scarl return (rc); 822255272Scarl} 823255272Scarl 824289541Scemstatic void 825289647Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 826289647Scem const char *kind) 827289541Scem{ 828289541Scem 829289647Scem device_printf(ntb->device, 830289647Scem "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 831289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 832289647Scem (char *)bar->vbase + bar->size - 1, 833289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 834289647Scem (uintmax_t)bar->size, kind); 835289541Scem} 836289541Scem 837255272Scarlstatic int 838255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 839255272Scarl{ 840255272Scarl 841255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 842289209Scem &bar->pci_resource_id, RF_ACTIVE); 843255272Scarl if (bar->pci_resource == NULL) 844255272Scarl return (ENXIO); 845289209Scem 846289209Scem save_bar_parameters(bar); 847291280Scem bar->map_mode = VM_MEMATTR_UNCACHEABLE; 848289647Scem print_map_success(ntb, bar, "mmr"); 849289209Scem return (0); 850255272Scarl} 851255272Scarl 852255272Scarlstatic int 853255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 854255272Scarl{ 855255272Scarl int rc; 856291280Scem vm_memattr_t mapmode; 857255276Scarl uint8_t bar_size_bits = 0; 858255272Scarl 859289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 860289209Scem &bar->pci_resource_id, RF_ACTIVE); 861250079Scarl 862255272Scarl if (bar->pci_resource == NULL) 863255272Scarl return (ENXIO); 864255276Scarl 865289209Scem save_bar_parameters(bar); 866289209Scem /* 867289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 868289209Scem * hardware issue. To work around this, query the size it should be 869289209Scem * configured to by the device and modify the resource to correspond to 870289209Scem * this new size. The BIOS on systems with this problem is required to 871289209Scem * provide enough address space to allow the driver to make this change 872289209Scem * safely. 873289209Scem * 874289209Scem * Ideally I could have just specified the size when I allocated the 875289209Scem * resource like: 876289209Scem * bus_alloc_resource(ntb->device, 877289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 878289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 879289209Scem * but the PCI driver does not honor the size in this call, so we have 880289209Scem * to modify it after the fact. 881289209Scem */ 882302484Smav if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 883289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 884289209Scem bar_size_bits = pci_read_config(ntb->device, 885289209Scem XEON_PBAR23SZ_OFFSET, 1); 886289209Scem else 887289209Scem bar_size_bits = pci_read_config(ntb->device, 888289209Scem XEON_PBAR45SZ_OFFSET, 1); 889289209Scem 890289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 891289209Scem bar->pci_resource, bar->pbase, 892289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 893255272Scarl if (rc != 0) { 894289209Scem device_printf(ntb->device, 895289209Scem "unable to resize bar\n"); 896255272Scarl return (rc); 897250079Scarl } 898289209Scem 899289209Scem save_bar_parameters(bar); 900250079Scarl } 901289209Scem 902291280Scem bar->map_mode = VM_MEMATTR_UNCACHEABLE; 903291030Scem print_map_success(ntb, bar, "mw"); 904291280Scem 905295486Scem /* 906295486Scem * Optionally, mark MW BARs as anything other than UC to improve 907295486Scem * performance. 908295486Scem */ 909303429Smav mapmode = intel_ntb_pat_flags(); 910295486Scem if (mapmode == bar->map_mode) 911295486Scem return (0); 912291030Scem 913291280Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 914291031Scem if (rc == 0) { 915291280Scem bar->map_mode = mapmode; 916289209Scem device_printf(ntb->device, 917289647Scem "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 918291280Scem "%s.\n", 919289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 920289647Scem (char *)bar->vbase + bar->size - 1, 921291280Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 922303429Smav intel_ntb_vm_memattr_to_str(mapmode)); 923291031Scem } else 924289647Scem device_printf(ntb->device, 925289647Scem "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 926291280Scem "%s: %d\n", 927289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 928289647Scem (char *)bar->vbase + bar->size - 1, 929289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 930303429Smav intel_ntb_vm_memattr_to_str(mapmode), rc); 931289647Scem /* Proceed anyway */ 932250079Scarl return (0); 933250079Scarl} 934250079Scarl 935250079Scarlstatic void 936303429Smavintel_ntb_unmap_pci_bar(struct ntb_softc *ntb) 937250079Scarl{ 938250079Scarl struct ntb_pci_bar_info *current_bar; 939250079Scarl int i; 940250079Scarl 941289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 942250079Scarl current_bar = &ntb->bar_info[i]; 943250079Scarl if (current_bar->pci_resource != NULL) 944250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 945250079Scarl current_bar->pci_resource_id, 946250079Scarl current_bar->pci_resource); 947250079Scarl } 948250079Scarl} 949250079Scarl 950250079Scarlstatic int 951303429Smavintel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 952250079Scarl{ 953289342Scem uint32_t i; 954289342Scem int rc; 955289342Scem 956289342Scem for (i = 0; i < num_vectors; i++) { 957289342Scem ntb->int_info[i].rid = i + 1; 958289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 959289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 960289342Scem if (ntb->int_info[i].res == NULL) { 961289342Scem device_printf(ntb->device, 962289342Scem "bus_alloc_resource failed\n"); 963289342Scem return (ENOMEM); 964289342Scem } 965289342Scem ntb->int_info[i].tag = NULL; 966289342Scem ntb->allocated_interrupts++; 967289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 968289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 969289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 970289342Scem if (rc != 0) { 971289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 972289342Scem return (ENXIO); 973289342Scem } 974289342Scem } 975289342Scem return (0); 976289342Scem} 977289342Scem 978289344Scem/* 979289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 980289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 981289344Scem * should be okay. This tunable should enable us to test that hypothesis 982289344Scem * when someone gets their hands on some Xeon hardware. 983289344Scem */ 984289344Scemstatic int ntb_force_remap_mode; 985289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 986289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 987289344Scem " to a smaller number of ithreads, even if the desired number are " 988289344Scem "available"); 989289344Scem 990289344Scem/* 991289344Scem * In case it is NOT ok, give consumers an abort button. 992289344Scem */ 993289344Scemstatic int ntb_prefer_intx; 994289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 995289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 996289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 997289344Scem "behavior)"); 998289344Scem 999289344Scem/* 1000289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 1001289344Scem * round-robin fashion. 1002289344Scem */ 1003289342Scemstatic int 1004303429Smavintel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1005289344Scem{ 1006289344Scem u_int *vectors; 1007289344Scem uint32_t i; 1008289344Scem int rc; 1009289344Scem 1010289344Scem if (ntb_prefer_intx != 0) 1011289344Scem return (ENXIO); 1012289344Scem 1013289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1014289344Scem 1015289344Scem for (i = 0; i < desired; i++) 1016289344Scem vectors[i] = (i % avail) + 1; 1017289344Scem 1018289344Scem rc = pci_remap_msix(dev, desired, vectors); 1019289344Scem free(vectors, M_NTB); 1020289344Scem return (rc); 1021289344Scem} 1022289344Scem 1023289344Scemstatic int 1024303429Smavintel_ntb_init_isr(struct ntb_softc *ntb) 1025289342Scem{ 1026289344Scem uint32_t desired_vectors, num_vectors; 1027289342Scem int rc; 1028250079Scarl 1029250079Scarl ntb->allocated_interrupts = 0; 1030289542Scem ntb->last_ts = ticks; 1031289347Scem 1032250079Scarl /* 1033295618Scem * Mask all doorbell interrupts. (Except link events!) 1034250079Scarl */ 1035295618Scem DB_MASK_LOCK(ntb); 1036295618Scem ntb->db_mask = ntb->db_valid_mask; 1037295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1038295618Scem DB_MASK_UNLOCK(ntb); 1039250079Scarl 1040289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1041289539Scem ntb->db_count); 1042289344Scem if (desired_vectors >= 1) { 1043289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 1044250079Scarl 1045289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 1046289344Scem num_vectors == desired_vectors) 1047289344Scem num_vectors--; 1048289344Scem 1049289344Scem if (rc == 0 && num_vectors < desired_vectors) { 1050303429Smav rc = intel_ntb_remap_msix(ntb->device, desired_vectors, 1051289344Scem num_vectors); 1052289344Scem if (rc == 0) 1053289344Scem num_vectors = desired_vectors; 1054289344Scem else 1055289344Scem pci_release_msi(ntb->device); 1056289344Scem } 1057289344Scem if (rc != 0) 1058289344Scem num_vectors = 1; 1059289344Scem } else 1060289344Scem num_vectors = 1; 1061289344Scem 1062289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1063302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1064295618Scem device_printf(ntb->device, 1065295618Scem "Errata workaround does not support MSI or INTX\n"); 1066295618Scem return (EINVAL); 1067295618Scem } 1068295618Scem 1069289539Scem ntb->db_vec_count = 1; 1070290680Scem ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1071303429Smav rc = intel_ntb_setup_legacy_interrupt(ntb); 1072289539Scem } else { 1073300531Scem if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1074302484Smav HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1075300531Scem device_printf(ntb->device, 1076300531Scem "Errata workaround expects %d doorbell bits\n", 1077300531Scem XEON_NONLINK_DB_MSIX_BITS); 1078300531Scem return (EINVAL); 1079300531Scem } 1080300531Scem 1081303429Smav intel_ntb_create_msix_vec(ntb, num_vectors); 1082303429Smav rc = intel_ntb_setup_msix(ntb, num_vectors); 1083289539Scem } 1084289539Scem if (rc != 0) { 1085289539Scem device_printf(ntb->device, 1086289539Scem "Error allocating interrupts: %d\n", rc); 1087303429Smav intel_ntb_free_msix_vec(ntb); 1088289396Scem } 1089289396Scem 1090289342Scem return (rc); 1091289342Scem} 1092289342Scem 1093289342Scemstatic int 1094303429Smavintel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1095289342Scem{ 1096289342Scem int rc; 1097289342Scem 1098289342Scem ntb->int_info[0].rid = 0; 1099289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1100289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1101289342Scem if (ntb->int_info[0].res == NULL) { 1102289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 1103289342Scem return (ENOMEM); 1104250079Scarl } 1105250079Scarl 1106289342Scem ntb->int_info[0].tag = NULL; 1107289342Scem ntb->allocated_interrupts = 1; 1108289342Scem 1109289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1110289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1111289342Scem ntb, &ntb->int_info[0].tag); 1112289342Scem if (rc != 0) { 1113289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 1114289342Scem return (ENXIO); 1115289342Scem } 1116289342Scem 1117250079Scarl return (0); 1118250079Scarl} 1119250079Scarl 1120250079Scarlstatic void 1121303429Smavintel_ntb_teardown_interrupts(struct ntb_softc *ntb) 1122250079Scarl{ 1123250079Scarl struct ntb_int_info *current_int; 1124250079Scarl int i; 1125250079Scarl 1126289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 1127250079Scarl current_int = &ntb->int_info[i]; 1128250079Scarl if (current_int->tag != NULL) 1129250079Scarl bus_teardown_intr(ntb->device, current_int->res, 1130250079Scarl current_int->tag); 1131250079Scarl 1132250079Scarl if (current_int->res != NULL) 1133250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 1134250079Scarl rman_get_rid(current_int->res), current_int->res); 1135250079Scarl } 1136250079Scarl 1137303429Smav intel_ntb_free_msix_vec(ntb); 1138250079Scarl pci_release_msi(ntb->device); 1139250079Scarl} 1140250079Scarl 1141289347Scem/* 1142289648Scem * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1143289347Scem * out to make code clearer. 1144289347Scem */ 1145289539Scemstatic inline uint64_t 1146289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 1147289347Scem{ 1148289347Scem 1149289648Scem if (ntb->type == NTB_ATOM) 1150303429Smav return (intel_ntb_reg_read(8, regoff)); 1151289347Scem 1152289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1153289347Scem 1154303429Smav return (intel_ntb_reg_read(2, regoff)); 1155289347Scem} 1156289347Scem 1157289539Scemstatic inline void 1158289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1159289347Scem{ 1160289347Scem 1161289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 1162289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1163289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 1164289542Scem (uintmax_t)ntb->db_valid_mask)); 1165289542Scem 1166289607Scem if (regoff == ntb->self_reg->db_mask) 1167289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 1168290678Scem db_iowrite_raw(ntb, regoff, val); 1169290678Scem} 1170289542Scem 1171290678Scemstatic inline void 1172290678Scemdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1173290678Scem{ 1174290678Scem 1175289648Scem if (ntb->type == NTB_ATOM) { 1176303429Smav intel_ntb_reg_write(8, regoff, val); 1177289347Scem return; 1178289347Scem } 1179289347Scem 1180289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1181303429Smav intel_ntb_reg_write(2, regoff, (uint16_t)val); 1182289347Scem} 1183289347Scem 1184302484Smavstatic void 1185303429Smavintel_ntb_db_set_mask(device_t dev, uint64_t bits) 1186289542Scem{ 1187302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1188289542Scem 1189289546Scem DB_MASK_LOCK(ntb); 1190289542Scem ntb->db_mask |= bits; 1191302493Smav if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1192302493Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1193289546Scem DB_MASK_UNLOCK(ntb); 1194289542Scem} 1195289542Scem 1196302484Smavstatic void 1197303429Smavintel_ntb_db_clear_mask(device_t dev, uint64_t bits) 1198289542Scem{ 1199302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1200302493Smav uint64_t ibits; 1201302493Smav int i; 1202289542Scem 1203289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1204289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1205289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1206289542Scem (uintmax_t)ntb->db_valid_mask)); 1207289542Scem 1208289546Scem DB_MASK_LOCK(ntb); 1209302493Smav ibits = ntb->fake_db_bell & ntb->db_mask & bits; 1210289542Scem ntb->db_mask &= ~bits; 1211302493Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1212302493Smav /* Simulate fake interrupts if unmasked DB bits are set. */ 1213302493Smav for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1214303429Smav if ((ibits & intel_ntb_db_vector_mask(dev, i)) != 0) 1215302493Smav swi_sched(ntb->int_info[i].tag, 0); 1216302493Smav } 1217302493Smav } else { 1218302493Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1219302493Smav } 1220289546Scem DB_MASK_UNLOCK(ntb); 1221289542Scem} 1222289542Scem 1223302484Smavstatic uint64_t 1224303429Smavintel_ntb_db_read(device_t dev) 1225289281Scem{ 1226302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1227289281Scem 1228302493Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1229302493Smav return (ntb->fake_db_bell); 1230295618Scem 1231289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 1232289281Scem} 1233289281Scem 1234302484Smavstatic void 1235303429Smavintel_ntb_db_clear(device_t dev, uint64_t bits) 1236289281Scem{ 1237302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1238289281Scem 1239289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1240289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1241289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1242289546Scem (uintmax_t)ntb->db_valid_mask)); 1243289546Scem 1244302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1245302493Smav DB_MASK_LOCK(ntb); 1246302493Smav ntb->fake_db_bell &= ~bits; 1247302493Smav DB_MASK_UNLOCK(ntb); 1248295618Scem return; 1249295618Scem } 1250295618Scem 1251289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1252289281Scem} 1253289281Scem 1254289540Scemstatic inline uint64_t 1255303429Smavintel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1256250079Scarl{ 1257289540Scem uint64_t shift, mask; 1258250079Scarl 1259302493Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1260302493Smav /* 1261302493Smav * Remap vectors in custom way to make at least first 1262302493Smav * three doorbells to not generate stray events. 1263302493Smav * This breaks Linux compatibility (if one existed) 1264302493Smav * when more then one DB is used (not by if_ntb). 1265302493Smav */ 1266302493Smav if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1) 1267302493Smav return (1 << db_vector); 1268302493Smav if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1) 1269302493Smav return (0x7ffc); 1270302493Smav } 1271302493Smav 1272289540Scem shift = ntb->db_vec_shift; 1273289540Scem mask = (1ull << shift) - 1; 1274289540Scem return (mask << (shift * db_vector)); 1275250079Scarl} 1276250079Scarl 1277250079Scarlstatic void 1278303429Smavintel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1279250079Scarl{ 1280289540Scem uint64_t vec_mask; 1281250079Scarl 1282289542Scem ntb->last_ts = ticks; 1283303429Smav vec_mask = intel_ntb_vec_mask(ntb, vec); 1284250079Scarl 1285289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1286303429Smav if (intel_ntb_poll_link(ntb)) 1287302484Smav ntb_link_event(ntb->device); 1288289540Scem } 1289289540Scem 1290302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1291295618Scem (vec_mask & ntb->db_link_mask) == 0) { 1292295618Scem DB_MASK_LOCK(ntb); 1293302493Smav 1294302493Smav /* Do not report same DB events again if not cleared yet. */ 1295302493Smav vec_mask &= ~ntb->fake_db_bell; 1296302493Smav 1297302493Smav /* Update our internal doorbell register. */ 1298302493Smav ntb->fake_db_bell |= vec_mask; 1299302493Smav 1300302493Smav /* Do not report masked DB events. */ 1301302493Smav vec_mask &= ~ntb->db_mask; 1302302493Smav 1303295618Scem DB_MASK_UNLOCK(ntb); 1304295618Scem } 1305295618Scem 1306289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1307302484Smav ntb_db_event(ntb->device, vec); 1308289546Scem} 1309250079Scarl 1310289546Scemstatic void 1311289546Scemndev_vec_isr(void *arg) 1312289546Scem{ 1313289546Scem struct ntb_vec *nvec = arg; 1314250079Scarl 1315303429Smav intel_ntb_interrupt(nvec->ntb, nvec->num); 1316250079Scarl} 1317250079Scarl 1318250079Scarlstatic void 1319289546Scemndev_irq_isr(void *arg) 1320250079Scarl{ 1321289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1322303429Smav intel_ntb_interrupt(arg, 0); 1323250079Scarl} 1324250079Scarl 1325250079Scarlstatic int 1326303429Smavintel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1327250079Scarl{ 1328289342Scem uint32_t i; 1329250079Scarl 1330289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1331250079Scarl M_ZERO | M_WAITOK); 1332250079Scarl for (i = 0; i < num_vectors; i++) { 1333289546Scem ntb->msix_vec[i].num = i; 1334289546Scem ntb->msix_vec[i].ntb = ntb; 1335250079Scarl } 1336250079Scarl 1337250079Scarl return (0); 1338250079Scarl} 1339250079Scarl 1340250079Scarlstatic void 1341303429Smavintel_ntb_free_msix_vec(struct ntb_softc *ntb) 1342250079Scarl{ 1343250079Scarl 1344289546Scem if (ntb->msix_vec == NULL) 1345289539Scem return; 1346289539Scem 1347289546Scem free(ntb->msix_vec, M_NTB); 1348289546Scem ntb->msix_vec = NULL; 1349250079Scarl} 1350250079Scarl 1351295618Scemstatic void 1352303429Smavintel_ntb_get_msix_info(struct ntb_softc *ntb) 1353295618Scem{ 1354295618Scem struct pci_devinfo *dinfo; 1355295618Scem struct pcicfg_msix *msix; 1356295618Scem uint32_t laddr, data, i, offset; 1357295618Scem 1358295618Scem dinfo = device_get_ivars(ntb->device); 1359295618Scem msix = &dinfo->cfg.msix; 1360295618Scem 1361300531Scem CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1362300531Scem 1363300531Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1364295618Scem offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1365295618Scem 1366295618Scem laddr = bus_read_4(msix->msix_table_res, offset + 1367295618Scem PCI_MSIX_ENTRY_LOWER_ADDR); 1368303429Smav intel_ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1369295618Scem 1370295618Scem KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1371295618Scem ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1372295618Scem MSI_INTEL_ADDR_BASE)); 1373301293Smav ntb->msix_data[i].nmd_ofs = laddr; 1374295618Scem 1375295618Scem data = bus_read_4(msix->msix_table_res, offset + 1376295618Scem PCI_MSIX_ENTRY_DATA); 1377303429Smav intel_ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1378295618Scem 1379295618Scem ntb->msix_data[i].nmd_data = data; 1380295618Scem } 1381295618Scem} 1382295618Scem 1383250079Scarlstatic struct ntb_hw_info * 1384303429Smavintel_ntb_get_device_info(uint32_t device_id) 1385250079Scarl{ 1386250079Scarl struct ntb_hw_info *ep = pci_ids; 1387250079Scarl 1388250079Scarl while (ep->device_id) { 1389250079Scarl if (ep->device_id == device_id) 1390250079Scarl return (ep); 1391250079Scarl ++ep; 1392250079Scarl } 1393250079Scarl return (NULL); 1394250079Scarl} 1395250079Scarl 1396289272Scemstatic void 1397303429Smavintel_ntb_teardown_xeon(struct ntb_softc *ntb) 1398250079Scarl{ 1399250079Scarl 1400289617Scem if (ntb->reg != NULL) 1401303429Smav intel_ntb_link_disable(ntb->device); 1402250079Scarl} 1403250079Scarl 1404289397Scemstatic void 1405303429Smavintel_ntb_detect_max_mw(struct ntb_softc *ntb) 1406289397Scem{ 1407289397Scem 1408289648Scem if (ntb->type == NTB_ATOM) { 1409289648Scem ntb->mw_count = ATOM_MW_COUNT; 1410289397Scem return; 1411289397Scem } 1412289397Scem 1413302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1414289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1415289397Scem else 1416289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1417289397Scem} 1418289397Scem 1419250079Scarlstatic int 1420303429Smavintel_ntb_detect_xeon(struct ntb_softc *ntb) 1421250079Scarl{ 1422289348Scem uint8_t ppd, conn_type; 1423250079Scarl 1424289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1425289348Scem ntb->ppd = ppd; 1426250079Scarl 1427289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1428290681Scem ntb->dev_type = NTB_DEV_DSD; 1429290681Scem else 1430289257Scem ntb->dev_type = NTB_DEV_USD; 1431289257Scem 1432289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1433289397Scem ntb->features |= NTB_SPLIT_BAR; 1434289397Scem 1435302508Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1436302508Smav !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1437302508Smav device_printf(ntb->device, 1438302508Smav "Can not apply SB01BASE_LOCKUP workaround " 1439302508Smav "with split BARs disabled!\n"); 1440302508Smav device_printf(ntb->device, 1441302508Smav "Expect system hangs under heavy NTB traffic!\n"); 1442302508Smav ntb->features &= ~NTB_SB01BASE_LOCKUP; 1443302508Smav } 1444302508Smav 1445295618Scem /* 1446295618Scem * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1447295618Scem * errata workaround; only do one at a time. 1448295618Scem */ 1449302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1450295618Scem ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1451289542Scem 1452289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1453289348Scem switch (conn_type) { 1454289348Scem case NTB_CONN_B2B: 1455289348Scem ntb->conn_type = conn_type; 1456289348Scem break; 1457289348Scem case NTB_CONN_RP: 1458289348Scem case NTB_CONN_TRANSPARENT: 1459289348Scem default: 1460289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1461289348Scem (unsigned)conn_type); 1462289348Scem return (ENXIO); 1463289348Scem } 1464289348Scem return (0); 1465289348Scem} 1466289348Scem 1467289348Scemstatic int 1468303429Smavintel_ntb_detect_atom(struct ntb_softc *ntb) 1469289348Scem{ 1470289348Scem uint32_t ppd, conn_type; 1471289348Scem 1472289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1473289348Scem ntb->ppd = ppd; 1474289348Scem 1475289648Scem if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1476289348Scem ntb->dev_type = NTB_DEV_DSD; 1477289348Scem else 1478289348Scem ntb->dev_type = NTB_DEV_USD; 1479289348Scem 1480289648Scem conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1481289348Scem switch (conn_type) { 1482289348Scem case NTB_CONN_B2B: 1483289348Scem ntb->conn_type = conn_type; 1484289348Scem break; 1485289348Scem default: 1486289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1487289348Scem return (ENXIO); 1488289348Scem } 1489289348Scem return (0); 1490289348Scem} 1491289348Scem 1492289348Scemstatic int 1493303429Smavintel_ntb_xeon_init_dev(struct ntb_softc *ntb) 1494289348Scem{ 1495289542Scem int rc; 1496289348Scem 1497289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1498289542Scem ntb->db_count = XEON_DB_COUNT; 1499289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1500289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1501289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1502289257Scem 1503289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1504250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1505289348Scem ntb->conn_type); 1506250079Scarl return (ENXIO); 1507250079Scarl } 1508250079Scarl 1509289542Scem ntb->reg = &xeon_reg; 1510289607Scem ntb->self_reg = &xeon_pri_reg; 1511289542Scem ntb->peer_reg = &xeon_b2b_reg; 1512289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1513289542Scem 1514302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1515302493Smav ntb->fake_db_bell = 0; 1516295618Scem ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1517295618Scem ntb->mw_count; 1518303429Smav intel_ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1519295618Scem g_ntb_msix_idx, ntb->msix_mw_idx); 1520303429Smav rc = intel_ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1521295618Scem VM_MEMATTR_UNCACHEABLE); 1522295618Scem KASSERT(rc == 0, ("shouldn't fail")); 1523302484Smav } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1524295618Scem /* 1525295618Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1526295618Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1527295618Scem * which may hang the system. To workaround this, use a memory 1528295618Scem * window to access the interrupt and scratch pad registers on the 1529295618Scem * remote system. 1530295618Scem */ 1531291263Scem ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1532291263Scem ntb->mw_count; 1533303429Smav intel_ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1534291263Scem g_ntb_mw_idx, ntb->b2b_mw_idx); 1535303429Smav rc = intel_ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1536295618Scem VM_MEMATTR_UNCACHEABLE); 1537291263Scem KASSERT(rc == 0, ("shouldn't fail")); 1538302484Smav } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1539289208Scem /* 1540289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1541289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1542289542Scem * since bit 14 is the last bit. 1543289542Scem * 1544289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1545289542Scem * anyway. Nor for non-B2B connection types. 1546289542Scem */ 1547289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1548250079Scarl 1549289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1550250079Scarl 1551289542Scem if (ntb->dev_type == NTB_DEV_USD) 1552289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1553289542Scem &xeon_b2b_usd_addr); 1554289542Scem else 1555289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1556289542Scem &xeon_b2b_dsd_addr); 1557289542Scem if (rc != 0) 1558289542Scem return (rc); 1559289271Scem 1560250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1561303429Smav intel_ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1562289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1563255279Scarl 1564290682Scem /* 1565290682Scem * Mask all doorbell interrupts. 1566290682Scem */ 1567295618Scem DB_MASK_LOCK(ntb); 1568295618Scem ntb->db_mask = ntb->db_valid_mask; 1569295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1570295618Scem DB_MASK_UNLOCK(ntb); 1571250079Scarl 1572303429Smav rc = intel_ntb_init_isr(ntb); 1573290682Scem return (rc); 1574250079Scarl} 1575250079Scarl 1576250079Scarlstatic int 1577303429Smavintel_ntb_atom_init_dev(struct ntb_softc *ntb) 1578250079Scarl{ 1579290682Scem int error; 1580250079Scarl 1581289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1582289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1583250079Scarl 1584289648Scem ntb->spad_count = ATOM_SPAD_COUNT; 1585289648Scem ntb->db_count = ATOM_DB_COUNT; 1586289648Scem ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1587289648Scem ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1588289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1589250079Scarl 1590289648Scem ntb->reg = &atom_reg; 1591289648Scem ntb->self_reg = &atom_pri_reg; 1592289648Scem ntb->peer_reg = &atom_b2b_reg; 1593289648Scem ntb->xlat_reg = &atom_sec_xlat; 1594289542Scem 1595250079Scarl /* 1596289648Scem * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1597250079Scarl * resolved. Mask transaction layer internal parity errors. 1598250079Scarl */ 1599250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1600250079Scarl 1601289648Scem configure_atom_secondary_side_bars(ntb); 1602250079Scarl 1603250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1604303429Smav intel_ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1605250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1606289209Scem 1607303429Smav error = intel_ntb_init_isr(ntb); 1608290682Scem if (error != 0) 1609290682Scem return (error); 1610290682Scem 1611289542Scem /* Initiate PCI-E link training */ 1612303429Smav intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1613250079Scarl 1614289648Scem callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1615289542Scem 1616250079Scarl return (0); 1617250079Scarl} 1618250079Scarl 1619289648Scem/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1620255279Scarlstatic void 1621289648Scemconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1622255279Scarl{ 1623255279Scarl 1624255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1625303429Smav intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1626290725Scem XEON_B2B_BAR2_ADDR64); 1627303429Smav intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1628290725Scem XEON_B2B_BAR4_ADDR64); 1629303429Smav intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1630303429Smav intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1631255279Scarl } else { 1632303429Smav intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1633290725Scem XEON_B2B_BAR2_ADDR64); 1634303429Smav intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1635290725Scem XEON_B2B_BAR4_ADDR64); 1636303429Smav intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1637303429Smav intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1638255279Scarl } 1639255279Scarl} 1640255279Scarl 1641289543Scem 1642289543Scem/* 1643289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1644289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1645289543Scem * remains for use by a higher layer. 1646289543Scem * 1647289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1648289543Scem * MW size is sufficiently large. 1649289543Scem */ 1650289543Scemstatic unsigned int ntb_b2b_mw_share; 1651289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1652289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1653289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1654289543Scem "value here."); 1655289543Scem 1656289543Scemstatic void 1657289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1658289543Scem enum ntb_bar regbar) 1659289543Scem{ 1660289543Scem struct ntb_pci_bar_info *bar; 1661289543Scem uint8_t bar_sz; 1662289543Scem 1663302484Smav if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1664289543Scem return; 1665289543Scem 1666289543Scem bar = &ntb->bar_info[idx]; 1667289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1668289543Scem if (idx == regbar) { 1669289543Scem if (ntb->b2b_off != 0) 1670289543Scem bar_sz--; 1671289543Scem else 1672289543Scem bar_sz = 0; 1673289543Scem } 1674289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1675289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1676289543Scem (void)bar_sz; 1677289543Scem} 1678289543Scem 1679289543Scemstatic void 1680289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1681289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1682289543Scem{ 1683301293Smav uint64_t reg_val; 1684289546Scem uint32_t base_reg, lmt_reg; 1685289543Scem 1686289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1687302482Smav if (idx == regbar) { 1688302482Smav if (ntb->b2b_off) 1689302482Smav bar_addr += ntb->b2b_off; 1690302482Smav else 1691302482Smav bar_addr = 0; 1692302482Smav } 1693289543Scem 1694289546Scem if (!bar_is_64bit(ntb, idx)) { 1695303429Smav intel_ntb_reg_write(4, base_reg, bar_addr); 1696303429Smav reg_val = intel_ntb_reg_read(4, base_reg); 1697302531Smav (void)reg_val; 1698302531Smav 1699303429Smav intel_ntb_reg_write(4, lmt_reg, bar_addr); 1700303429Smav reg_val = intel_ntb_reg_read(4, lmt_reg); 1701295618Scem (void)reg_val; 1702302531Smav } else { 1703303429Smav intel_ntb_reg_write(8, base_reg, bar_addr); 1704303429Smav reg_val = intel_ntb_reg_read(8, base_reg); 1705302531Smav (void)reg_val; 1706295618Scem 1707303429Smav intel_ntb_reg_write(8, lmt_reg, bar_addr); 1708303429Smav reg_val = intel_ntb_reg_read(8, lmt_reg); 1709295618Scem (void)reg_val; 1710289543Scem } 1711289543Scem} 1712289543Scem 1713289543Scemstatic void 1714289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1715289543Scem{ 1716289543Scem struct ntb_pci_bar_info *bar; 1717289543Scem 1718289543Scem bar = &ntb->bar_info[idx]; 1719302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1720303429Smav intel_ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1721303429Smav base_addr = intel_ntb_reg_read(4, bar->pbarxlat_off); 1722289543Scem } else { 1723303429Smav intel_ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1724303429Smav base_addr = intel_ntb_reg_read(8, bar->pbarxlat_off); 1725289543Scem } 1726289543Scem (void)base_addr; 1727289543Scem} 1728289543Scem 1729289542Scemstatic int 1730289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1731289542Scem const struct ntb_b2b_addr *peer_addr) 1732255279Scarl{ 1733289543Scem struct ntb_pci_bar_info *b2b_bar; 1734289543Scem vm_size_t bar_size; 1735289543Scem uint64_t bar_addr; 1736289543Scem enum ntb_bar b2b_bar_num, i; 1737255279Scarl 1738289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1739289543Scem b2b_bar = NULL; 1740289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1741289543Scem ntb->b2b_off = 0; 1742289543Scem } else { 1743303429Smav b2b_bar_num = intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1744289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1745289543Scem ("invalid b2b mw bar")); 1746289543Scem 1747289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1748289543Scem bar_size = b2b_bar->size; 1749289543Scem 1750289543Scem if (ntb_b2b_mw_share != 0 && 1751289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1752289543Scem ntb->b2b_off = bar_size >> 1; 1753289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1754289543Scem ntb->b2b_off = 0; 1755289543Scem } else { 1756289543Scem device_printf(ntb->device, 1757289543Scem "B2B bar size is too small!\n"); 1758289543Scem return (EIO); 1759289543Scem } 1760255279Scarl } 1761289542Scem 1762289543Scem /* 1763289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1764289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1765289543Scem */ 1766289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1767289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1768289543Scem 1769289543Scem bar_addr = 0; 1770289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1771289543Scem bar_addr = addr->bar0_addr; 1772289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1773289543Scem bar_addr = addr->bar2_addr64; 1774302484Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1775289543Scem bar_addr = addr->bar4_addr64; 1776289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1777289543Scem bar_addr = addr->bar4_addr32; 1778289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1779289543Scem bar_addr = addr->bar5_addr32; 1780289543Scem else 1781289543Scem KASSERT(false, ("invalid bar")); 1782289543Scem 1783303429Smav intel_ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1784289543Scem 1785289543Scem /* 1786289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1787289543Scem * register BAR. The B2B BAR is either disabled above or configured 1788289543Scem * half-size. It starts at PBAR xlat + offset. 1789289543Scem * 1790289543Scem * Also set up incoming BAR limits == base (zero length window). 1791289543Scem */ 1792289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1793289543Scem b2b_bar_num); 1794302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1795289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1796289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1797289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1798289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1799289542Scem } else 1800289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1801289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1802289543Scem 1803289543Scem /* Zero incoming translation addrs */ 1804303429Smav intel_ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1805303429Smav intel_ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1806289543Scem 1807302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1808302510Smav uint32_t xlat_reg, lmt_reg; 1809302493Smav enum ntb_bar bar_num; 1810295618Scem 1811295618Scem /* 1812295618Scem * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1813295618Scem * workaround 1814295618Scem */ 1815303429Smav bar_num = intel_ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1816302510Smav bar_get_xlat_params(ntb, bar_num, NULL, &xlat_reg, &lmt_reg); 1817302510Smav if (bar_is_64bit(ntb, bar_num)) { 1818303429Smav intel_ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE); 1819303429Smav ntb->msix_xlat = intel_ntb_reg_read(8, xlat_reg); 1820303429Smav intel_ntb_reg_write(8, lmt_reg, 0); 1821301293Smav } else { 1822303429Smav intel_ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE); 1823303429Smav ntb->msix_xlat = intel_ntb_reg_read(4, xlat_reg); 1824303429Smav intel_ntb_reg_write(4, lmt_reg, 0); 1825301293Smav } 1826302493Smav 1827302493Smav ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1828295618Scem } 1829303429Smav (void)intel_ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1830303429Smav (void)intel_ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1831295618Scem 1832289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1833303429Smav intel_ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1834303429Smav intel_ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1835289543Scem 1836289543Scem /* Set outgoing translation offsets */ 1837289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1838302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1839289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1840289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1841289543Scem } else 1842289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1843289543Scem 1844289543Scem /* Set the translation offset for B2B registers */ 1845289543Scem bar_addr = 0; 1846289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1847289543Scem bar_addr = peer_addr->bar0_addr; 1848289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1849289543Scem bar_addr = peer_addr->bar2_addr64; 1850302484Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1851289543Scem bar_addr = peer_addr->bar4_addr64; 1852289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1853289543Scem bar_addr = peer_addr->bar4_addr32; 1854289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1855289543Scem bar_addr = peer_addr->bar5_addr32; 1856289543Scem else 1857289543Scem KASSERT(false, ("invalid bar")); 1858289543Scem 1859289543Scem /* 1860289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1861289543Scem * at a time. 1862289543Scem */ 1863303429Smav intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1864303429Smav intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1865289542Scem return (0); 1866255279Scarl} 1867255279Scarl 1868289546Scemstatic inline bool 1869295618Scem_xeon_link_is_up(struct ntb_softc *ntb) 1870295618Scem{ 1871295618Scem 1872295618Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1873295618Scem return (true); 1874295618Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1875295618Scem} 1876295618Scem 1877295618Scemstatic inline bool 1878289546Scemlink_is_up(struct ntb_softc *ntb) 1879289546Scem{ 1880289546Scem 1881295618Scem if (ntb->type == NTB_XEON) 1882295618Scem return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1883302484Smav !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 1884289546Scem 1885289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1886289648Scem return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1887289546Scem} 1888289546Scem 1889289546Scemstatic inline bool 1890289648Scematom_link_is_err(struct ntb_softc *ntb) 1891289546Scem{ 1892289546Scem uint32_t status; 1893289546Scem 1894289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1895289546Scem 1896303429Smav status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1897289648Scem if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1898289546Scem return (true); 1899289546Scem 1900303429Smav status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1901289648Scem return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1902289546Scem} 1903289546Scem 1904289648Scem/* Atom does not have link status interrupt, poll on that platform */ 1905250079Scarlstatic void 1906289648Scematom_link_hb(void *arg) 1907250079Scarl{ 1908250079Scarl struct ntb_softc *ntb = arg; 1909289546Scem sbintime_t timo, poll_ts; 1910250079Scarl 1911289546Scem timo = NTB_HB_TIMEOUT * hz; 1912289546Scem poll_ts = ntb->last_ts + timo; 1913289546Scem 1914289542Scem /* 1915289542Scem * Delay polling the link status if an interrupt was received, unless 1916289542Scem * the cached link status says the link is down. 1917289542Scem */ 1918289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1919289546Scem timo = poll_ts - ticks; 1920289542Scem goto out; 1921289546Scem } 1922289542Scem 1923303429Smav if (intel_ntb_poll_link(ntb)) 1924302484Smav ntb_link_event(ntb->device); 1925289542Scem 1926289648Scem if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1927289546Scem /* Link is down with error, proceed with recovery */ 1928289648Scem callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1929289546Scem return; 1930250079Scarl } 1931250079Scarl 1932289542Scemout: 1933289648Scem callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1934250079Scarl} 1935250079Scarl 1936250079Scarlstatic void 1937289648Scematom_perform_link_restart(struct ntb_softc *ntb) 1938250079Scarl{ 1939250079Scarl uint32_t status; 1940250079Scarl 1941250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1942303429Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1943303429Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1944303429Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1945303429Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1946250079Scarl 1947250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1948250079Scarl pause("ModPhy", hz / 10); 1949250079Scarl 1950250079Scarl /* Clear AER Errors, write to clear */ 1951303429Smav status = intel_ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1952250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1953303429Smav intel_ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1954250079Scarl 1955250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1956303429Smav status = intel_ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1957289648Scem status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1958303429Smav intel_ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1959250079Scarl 1960250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1961303429Smav status = intel_ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1962289648Scem status |= ATOM_DESKEWSTS_DBERR; 1963303429Smav intel_ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1964250079Scarl 1965303429Smav status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1966289648Scem status &= ATOM_IBIST_ERR_OFLOW; 1967303429Smav intel_ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1968250079Scarl 1969250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1970303429Smav status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1971289648Scem status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1972303429Smav intel_ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1973250079Scarl} 1974250079Scarl 1975302484Smavstatic int 1976303429Smavintel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, 1977302484Smav enum ntb_width width __unused) 1978289546Scem{ 1979302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1980289280Scem uint32_t cntl; 1981289272Scem 1982303429Smav intel_ntb_printf(2, "%s\n", __func__); 1983300100Scem 1984289648Scem if (ntb->type == NTB_ATOM) { 1985289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1986289648Scem ntb->ppd | ATOM_PPD_INIT_LINK, 4); 1987289546Scem return (0); 1988289542Scem } 1989289542Scem 1990289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1991302484Smav ntb_link_event(dev); 1992289546Scem return (0); 1993289280Scem } 1994289280Scem 1995303429Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 1996289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1997289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1998289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1999302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2000289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2001303429Smav intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2002289546Scem return (0); 2003289272Scem} 2004289272Scem 2005302484Smavstatic int 2006303429Smavintel_ntb_link_disable(device_t dev) 2007289272Scem{ 2008302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2009289272Scem uint32_t cntl; 2010289272Scem 2011303429Smav intel_ntb_printf(2, "%s\n", __func__); 2012300100Scem 2013289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2014302484Smav ntb_link_event(dev); 2015289546Scem return (0); 2016289272Scem } 2017289272Scem 2018303429Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2019289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2020289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2021302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2022289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2023289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2024303429Smav intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2025289546Scem return (0); 2026289272Scem} 2027289272Scem 2028302484Smavstatic bool 2029303429Smavintel_ntb_link_enabled(device_t dev) 2030300100Scem{ 2031302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2032300100Scem uint32_t cntl; 2033300100Scem 2034300100Scem if (ntb->type == NTB_ATOM) { 2035300100Scem cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2036300100Scem return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2037300100Scem } 2038300100Scem 2039300100Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2040300100Scem return (true); 2041300100Scem 2042303429Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2043300100Scem return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2044300100Scem} 2045300100Scem 2046289272Scemstatic void 2047289648Scemrecover_atom_link(void *arg) 2048250079Scarl{ 2049250079Scarl struct ntb_softc *ntb = arg; 2050289608Scem unsigned speed, width, oldspeed, oldwidth; 2051250079Scarl uint32_t status32; 2052250079Scarl 2053289648Scem atom_perform_link_restart(ntb); 2054250079Scarl 2055289232Scem /* 2056289232Scem * There is a potential race between the 2 NTB devices recovering at 2057289232Scem * the same time. If the times are the same, the link will not recover 2058289232Scem * and the driver will be stuck in this loop forever. Add a random 2059289232Scem * interval to the recovery time to prevent this race. 2060289232Scem */ 2061289648Scem status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2062289648Scem pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2063289232Scem 2064289648Scem if (atom_link_is_err(ntb)) 2065250079Scarl goto retry; 2066250079Scarl 2067303429Smav status32 = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2068289648Scem if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2069289232Scem goto out; 2070289232Scem 2071303429Smav status32 = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2072289608Scem width = NTB_LNK_STA_WIDTH(status32); 2073289608Scem speed = status32 & NTB_LINK_SPEED_MASK; 2074289608Scem 2075289608Scem oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2076289608Scem oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2077289608Scem if (oldwidth != width || oldspeed != speed) 2078250079Scarl goto retry; 2079250079Scarl 2080289232Scemout: 2081289648Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2082289542Scem ntb); 2083250079Scarl return; 2084250079Scarl 2085250079Scarlretry: 2086289648Scem callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2087250079Scarl ntb); 2088250079Scarl} 2089250079Scarl 2090289546Scem/* 2091289546Scem * Polls the HW link status register(s); returns true if something has changed. 2092289546Scem */ 2093289546Scemstatic bool 2094303429Smavintel_ntb_poll_link(struct ntb_softc *ntb) 2095250079Scarl{ 2096250079Scarl uint32_t ntb_cntl; 2097289546Scem uint16_t reg_val; 2098250079Scarl 2099289648Scem if (ntb->type == NTB_ATOM) { 2100303429Smav ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2101289546Scem if (ntb_cntl == ntb->ntb_ctl) 2102289546Scem return (false); 2103289546Scem 2104289542Scem ntb->ntb_ctl = ntb_cntl; 2105303429Smav ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2106250079Scarl } else { 2107290678Scem db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2108250079Scarl 2109289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2110289546Scem if (reg_val == ntb->lnk_sta) 2111289546Scem return (false); 2112250079Scarl 2113289546Scem ntb->lnk_sta = reg_val; 2114295618Scem 2115302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2116295618Scem if (_xeon_link_is_up(ntb)) { 2117295618Scem if (!ntb->peer_msix_good) { 2118295618Scem callout_reset(&ntb->peer_msix_work, 0, 2119303429Smav intel_ntb_exchange_msix, ntb); 2120295618Scem return (false); 2121295618Scem } 2122295618Scem } else { 2123295618Scem ntb->peer_msix_good = false; 2124295618Scem ntb->peer_msix_done = false; 2125295618Scem } 2126295618Scem } 2127289542Scem } 2128289546Scem return (true); 2129289542Scem} 2130289542Scem 2131289546Scemstatic inline enum ntb_speed 2132303429Smavintel_ntb_link_sta_speed(struct ntb_softc *ntb) 2133250079Scarl{ 2134250079Scarl 2135289546Scem if (!link_is_up(ntb)) 2136289546Scem return (NTB_SPEED_NONE); 2137289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2138250079Scarl} 2139250079Scarl 2140289546Scemstatic inline enum ntb_width 2141303429Smavintel_ntb_link_sta_width(struct ntb_softc *ntb) 2142250079Scarl{ 2143250079Scarl 2144289546Scem if (!link_is_up(ntb)) 2145289546Scem return (NTB_WIDTH_NONE); 2146289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2147250079Scarl} 2148250079Scarl 2149289774ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2150289774Scem "Driver state, statistics, and HW registers"); 2151289774Scem 2152289774Scem#define NTB_REGSZ_MASK (3ul << 30) 2153289774Scem#define NTB_REG_64 (1ul << 30) 2154289774Scem#define NTB_REG_32 (2ul << 30) 2155289774Scem#define NTB_REG_16 (3ul << 30) 2156289774Scem#define NTB_REG_8 (0ul << 30) 2157289774Scem 2158289774Scem#define NTB_DB_READ (1ul << 29) 2159289774Scem#define NTB_PCI_REG (1ul << 28) 2160289774Scem#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2161289774Scem 2162289774Scemstatic void 2163303429Smavintel_ntb_sysctl_init(struct ntb_softc *ntb) 2164289774Scem{ 2165300100Scem struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2166289774Scem struct sysctl_ctx_list *ctx; 2167289774Scem struct sysctl_oid *tree, *tmptree; 2168289774Scem 2169289774Scem ctx = device_get_sysctl_ctx(ntb->device); 2170300100Scem globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2171289774Scem 2172300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2173300100Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2174300100Scem sysctl_handle_link_status_human, "A", 2175300100Scem "Link status (human readable)"); 2176300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2177300100Scem CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2178300100Scem "IU", "Link status (1=active, 0=inactive)"); 2179300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2180300100Scem CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2181300100Scem "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2182300100Scem 2183300100Scem tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2184300100Scem CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2185289774Scem tree_par = SYSCTL_CHILDREN(tree); 2186289774Scem 2187289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2188289774Scem &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2189289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2190289774Scem &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2191290687Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2192290687Scem &ntb->ppd, 0, "Raw PPD register (cached)"); 2193289774Scem 2194289774Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2195289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2196289774Scem &ntb->b2b_mw_idx, 0, 2197289774Scem "Index of the MW used for B2B remote register access"); 2198289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2199289774Scem CTLFLAG_RD, &ntb->b2b_off, 2200289774Scem "If non-zero, offset of B2B register region in shared MW"); 2201289774Scem } 2202289774Scem 2203289774Scem SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2204289774Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2205289774Scem "Features/errata of this NTB device"); 2206289774Scem 2207289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2208290686Scem __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2209290686Scem "NTB CTL register (cached)"); 2210289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2211290686Scem __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2212290686Scem "LNK STA register (cached)"); 2213289774Scem 2214289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2215291263Scem &ntb->mw_count, 0, "MW count"); 2216289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2217289774Scem &ntb->spad_count, 0, "Scratchpad count"); 2218289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2219289774Scem &ntb->db_count, 0, "Doorbell count"); 2220289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2221289774Scem &ntb->db_vec_count, 0, "Doorbell vector count"); 2222289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2223289774Scem &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2224289774Scem 2225289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2226289774Scem &ntb->db_valid_mask, "Doorbell valid mask"); 2227289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2228289774Scem &ntb->db_link_mask, "Doorbell link mask"); 2229289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2230289774Scem &ntb->db_mask, "Doorbell mask (cached)"); 2231289774Scem 2232289774Scem tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2233289774Scem CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2234289774Scem regpar = SYSCTL_CHILDREN(tmptree); 2235289774Scem 2236290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2237290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2238290682Scem ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2239290682Scem "NTB Control register"); 2240290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2241290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2242290682Scem 0x19c, sysctl_handle_register, "IU", 2243290682Scem "NTB Link Capabilities"); 2244290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2245290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2246290682Scem 0x1a0, sysctl_handle_register, "IU", 2247290682Scem "NTB Link Control register"); 2248290682Scem 2249289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2250289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2251289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2252289774Scem sysctl_handle_register, "QU", "Doorbell mask register"); 2253289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2254289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2255289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2256289774Scem sysctl_handle_register, "QU", "Doorbell register"); 2257289774Scem 2258289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2259289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2260289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2261289774Scem sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2262302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2263289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2264289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2265289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2266289774Scem sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2267289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2268289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2269289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2270289774Scem sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2271289774Scem } else { 2272289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2273289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2274289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2275289774Scem sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2276289774Scem } 2277289774Scem 2278289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2279289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2280289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2281289774Scem sysctl_handle_register, "QU", "Incoming LMT23 register"); 2282302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2283289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2284289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2285289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2286289774Scem sysctl_handle_register, "IU", "Incoming LMT4 register"); 2287289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2288289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2289289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2290289774Scem sysctl_handle_register, "IU", "Incoming LMT5 register"); 2291289774Scem } else { 2292289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2293289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2294289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2295289774Scem sysctl_handle_register, "QU", "Incoming LMT45 register"); 2296289774Scem } 2297289774Scem 2298289774Scem if (ntb->type == NTB_ATOM) 2299289774Scem return; 2300289774Scem 2301289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2302289774Scem CTLFLAG_RD, NULL, "Xeon HW statistics"); 2303289774Scem statpar = SYSCTL_CHILDREN(tmptree); 2304289774Scem SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2305289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2306289774Scem NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2307289774Scem sysctl_handle_register, "SU", "Upstream Memory Miss"); 2308289774Scem 2309289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2310289774Scem CTLFLAG_RD, NULL, "Xeon HW errors"); 2311289774Scem errpar = SYSCTL_CHILDREN(tmptree); 2312289774Scem 2313290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2314289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2315290687Scem NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2316290687Scem sysctl_handle_register, "CU", "PPD"); 2317290687Scem 2318290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2319290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2320290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2321290687Scem sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2322290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2323290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2324290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2325290687Scem sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2326290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2327290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2328290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2329290687Scem sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2330290687Scem 2331290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2332290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2333290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2334290687Scem sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2335290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2336290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2337290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2338290687Scem sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2339290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2340290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2341290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2342290687Scem sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2343290687Scem 2344290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2345290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2346289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2347289774Scem sysctl_handle_register, "SU", "DEVSTS"); 2348290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2349289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2350289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2351289774Scem sysctl_handle_register, "SU", "LNKSTS"); 2352290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2353290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2354290687Scem NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2355290687Scem sysctl_handle_register, "SU", "SLNKSTS"); 2356290687Scem 2357289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2358289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2359289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2360289774Scem sysctl_handle_register, "IU", "UNCERRSTS"); 2361289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2362289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2363289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2364289774Scem sysctl_handle_register, "IU", "CORERRSTS"); 2365289774Scem 2366289774Scem if (ntb->conn_type != NTB_CONN_B2B) 2367289774Scem return; 2368289774Scem 2369289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2370289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2371289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2372289774Scem sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2373302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2374289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2375289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2376289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2377289774Scem sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2378289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2379289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2380289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2381289774Scem sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2382289774Scem } else { 2383289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2384289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2385289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2386289774Scem sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2387289774Scem } 2388289774Scem 2389289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2390289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2391289774Scem NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2392289774Scem sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2393302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2394289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2395289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2396289774Scem NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2397289774Scem sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2398289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2399289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2400289774Scem NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2401289774Scem sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2402289774Scem } else { 2403289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2404289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2405289774Scem NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2406289774Scem sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2407289774Scem } 2408289774Scem 2409289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2410289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2411289774Scem NTB_REG_64 | ntb->xlat_reg->bar0_base, 2412289774Scem sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2413289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2414289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2415289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_base, 2416289774Scem sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2417302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2418289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2419289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2420289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_base, 2421289774Scem sysctl_handle_register, "IU", 2422289774Scem "Secondary BAR4 base register"); 2423289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2424289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2425289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_base, 2426289774Scem sysctl_handle_register, "IU", 2427289774Scem "Secondary BAR5 base register"); 2428289774Scem } else { 2429289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2430289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2431289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_base, 2432289774Scem sysctl_handle_register, "QU", 2433289774Scem "Secondary BAR45 base register"); 2434289774Scem } 2435289774Scem} 2436289774Scem 2437289774Scemstatic int 2438289774Scemsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2439289774Scem{ 2440302483Smav struct ntb_softc *ntb = arg1; 2441289774Scem struct sbuf sb; 2442289774Scem int error; 2443289774Scem 2444289774Scem sbuf_new_for_sysctl(&sb, NULL, 256, req); 2445289774Scem 2446289774Scem sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2447289774Scem error = sbuf_finish(&sb); 2448289774Scem sbuf_delete(&sb); 2449289774Scem 2450289774Scem if (error || !req->newptr) 2451289774Scem return (error); 2452289774Scem return (EINVAL); 2453289774Scem} 2454289774Scem 2455289774Scemstatic int 2456300100Scemsysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2457289774Scem{ 2458302483Smav struct ntb_softc *ntb = arg1; 2459300100Scem unsigned old, new; 2460300100Scem int error; 2461300100Scem 2462303429Smav old = intel_ntb_link_enabled(ntb->device); 2463300100Scem 2464300100Scem error = SYSCTL_OUT(req, &old, sizeof(old)); 2465300100Scem if (error != 0 || req->newptr == NULL) 2466300100Scem return (error); 2467300100Scem 2468300100Scem error = SYSCTL_IN(req, &new, sizeof(new)); 2469300100Scem if (error != 0) 2470300100Scem return (error); 2471300100Scem 2472303429Smav intel_ntb_printf(0, "Admin set interface state to '%sabled'\n", 2473300100Scem (new != 0)? "en" : "dis"); 2474300100Scem 2475300100Scem if (new != 0) 2476303429Smav error = intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2477300100Scem else 2478303429Smav error = intel_ntb_link_disable(ntb->device); 2479300100Scem return (error); 2480300100Scem} 2481300100Scem 2482300100Scemstatic int 2483300100Scemsysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2484300100Scem{ 2485302483Smav struct ntb_softc *ntb = arg1; 2486289774Scem struct sbuf sb; 2487289774Scem enum ntb_speed speed; 2488289774Scem enum ntb_width width; 2489289774Scem int error; 2490289774Scem 2491289774Scem sbuf_new_for_sysctl(&sb, NULL, 32, req); 2492289774Scem 2493303429Smav if (intel_ntb_link_is_up(ntb->device, &speed, &width)) 2494289774Scem sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2495289774Scem (unsigned)speed, (unsigned)width); 2496289774Scem else 2497289774Scem sbuf_printf(&sb, "down"); 2498289774Scem 2499289774Scem error = sbuf_finish(&sb); 2500289774Scem sbuf_delete(&sb); 2501289774Scem 2502289774Scem if (error || !req->newptr) 2503289774Scem return (error); 2504289774Scem return (EINVAL); 2505289774Scem} 2506289774Scem 2507289774Scemstatic int 2508300100Scemsysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2509300100Scem{ 2510302483Smav struct ntb_softc *ntb = arg1; 2511300100Scem unsigned res; 2512300100Scem int error; 2513300100Scem 2514303429Smav res = intel_ntb_link_is_up(ntb->device, NULL, NULL); 2515300100Scem 2516300100Scem error = SYSCTL_OUT(req, &res, sizeof(res)); 2517300100Scem if (error || !req->newptr) 2518300100Scem return (error); 2519300100Scem return (EINVAL); 2520300100Scem} 2521300100Scem 2522300100Scemstatic int 2523289774Scemsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2524289774Scem{ 2525289774Scem struct ntb_softc *ntb; 2526289774Scem const void *outp; 2527289774Scem uintptr_t sz; 2528289774Scem uint64_t umv; 2529289774Scem char be[sizeof(umv)]; 2530289774Scem size_t outsz; 2531289774Scem uint32_t reg; 2532289774Scem bool db, pci; 2533289774Scem int error; 2534289774Scem 2535289774Scem ntb = arg1; 2536289774Scem reg = arg2 & ~NTB_REGFLAGS_MASK; 2537289774Scem sz = arg2 & NTB_REGSZ_MASK; 2538289774Scem db = (arg2 & NTB_DB_READ) != 0; 2539289774Scem pci = (arg2 & NTB_PCI_REG) != 0; 2540289774Scem 2541289774Scem KASSERT(!(db && pci), ("bogus")); 2542289774Scem 2543289774Scem if (db) { 2544289774Scem KASSERT(sz == NTB_REG_64, ("bogus")); 2545289774Scem umv = db_ioread(ntb, reg); 2546289774Scem outsz = sizeof(uint64_t); 2547289774Scem } else { 2548289774Scem switch (sz) { 2549289774Scem case NTB_REG_64: 2550289774Scem if (pci) 2551289774Scem umv = pci_read_config(ntb->device, reg, 8); 2552289774Scem else 2553303429Smav umv = intel_ntb_reg_read(8, reg); 2554289774Scem outsz = sizeof(uint64_t); 2555289774Scem break; 2556289774Scem case NTB_REG_32: 2557289774Scem if (pci) 2558289774Scem umv = pci_read_config(ntb->device, reg, 4); 2559289774Scem else 2560303429Smav umv = intel_ntb_reg_read(4, reg); 2561289774Scem outsz = sizeof(uint32_t); 2562289774Scem break; 2563289774Scem case NTB_REG_16: 2564289774Scem if (pci) 2565289774Scem umv = pci_read_config(ntb->device, reg, 2); 2566289774Scem else 2567303429Smav umv = intel_ntb_reg_read(2, reg); 2568289774Scem outsz = sizeof(uint16_t); 2569289774Scem break; 2570289774Scem case NTB_REG_8: 2571289774Scem if (pci) 2572289774Scem umv = pci_read_config(ntb->device, reg, 1); 2573289774Scem else 2574303429Smav umv = intel_ntb_reg_read(1, reg); 2575289774Scem outsz = sizeof(uint8_t); 2576289774Scem break; 2577289774Scem default: 2578289774Scem panic("bogus"); 2579289774Scem break; 2580289774Scem } 2581289774Scem } 2582289774Scem 2583289774Scem /* Encode bigendian so that sysctl -x is legible. */ 2584289774Scem be64enc(be, umv); 2585289774Scem outp = ((char *)be) + sizeof(umv) - outsz; 2586289774Scem 2587289774Scem error = SYSCTL_OUT(req, outp, outsz); 2588289774Scem if (error || !req->newptr) 2589289774Scem return (error); 2590289774Scem return (EINVAL); 2591289774Scem} 2592289774Scem 2593291263Scemstatic unsigned 2594303429Smavintel_ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2595291263Scem{ 2596291263Scem 2597295618Scem if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2598295618Scem uidx >= ntb->b2b_mw_idx) || 2599295618Scem (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2600295618Scem uidx++; 2601295618Scem if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2602295618Scem uidx >= ntb->b2b_mw_idx) && 2603295618Scem (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2604295618Scem uidx++; 2605291263Scem return (uidx); 2606291263Scem} 2607291263Scem 2608303561Smav#ifndef EARLY_AP_STARTUP 2609303554Smavstatic int msix_ready; 2610303554Smav 2611295618Scemstatic void 2612303554Smavintel_ntb_msix_ready(void *arg __unused) 2613303554Smav{ 2614303554Smav 2615303554Smav msix_ready = 1; 2616303554Smav} 2617303554SmavSYSINIT(intel_ntb_msix_ready, SI_SUB_SMP, SI_ORDER_ANY, 2618303554Smav intel_ntb_msix_ready, NULL); 2619303561Smav#endif 2620303554Smav 2621303554Smavstatic void 2622303429Smavintel_ntb_exchange_msix(void *ctx) 2623295618Scem{ 2624295618Scem struct ntb_softc *ntb; 2625295618Scem uint32_t val; 2626295618Scem unsigned i; 2627295618Scem 2628295618Scem ntb = ctx; 2629295618Scem 2630301292Smav if (ntb->peer_msix_good) 2631301292Smav goto msix_good; 2632295618Scem if (ntb->peer_msix_done) 2633295618Scem goto msix_done; 2634295618Scem 2635303561Smav#ifndef EARLY_AP_STARTUP 2636303554Smav /* Block MSIX negotiation until SMP started and IRQ reshuffled. */ 2637303554Smav if (!msix_ready) 2638303554Smav goto reschedule; 2639303561Smav#endif 2640303554Smav 2641303429Smav intel_ntb_get_msix_info(ntb); 2642295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2643303429Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 2644295618Scem ntb->msix_data[i].nmd_data); 2645303429Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 2646301293Smav ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2647295618Scem } 2648303429Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2649295618Scem 2650303429Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 2651295618Scem if (val != NTB_MSIX_VER_GUARD) 2652295618Scem goto reschedule; 2653295618Scem 2654295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2655303429Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 2656303429Smav intel_ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2657295618Scem ntb->peer_msix_data[i].nmd_data = val; 2658303429Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 2659303429Smav intel_ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2660295618Scem ntb->peer_msix_data[i].nmd_ofs = val; 2661295618Scem } 2662295618Scem 2663295618Scem ntb->peer_msix_done = true; 2664295618Scem 2665295618Scemmsix_done: 2666303429Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2667303429Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 2668295618Scem if (val != NTB_MSIX_RECEIVED) 2669295618Scem goto reschedule; 2670295618Scem 2671303510Smav intel_ntb_spad_clear(ntb->device); 2672295618Scem ntb->peer_msix_good = true; 2673301292Smav /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2674301292Smav goto reschedule; 2675295618Scem 2676301292Smavmsix_good: 2677303429Smav intel_ntb_poll_link(ntb); 2678302484Smav ntb_link_event(ntb->device); 2679295618Scem return; 2680295618Scem 2681295618Scemreschedule: 2682295618Scem ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2683301292Smav if (_xeon_link_is_up(ntb)) { 2684301292Smav callout_reset(&ntb->peer_msix_work, 2685301292Smav hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2686303429Smav intel_ntb_exchange_msix, ntb); 2687301292Smav } else 2688303429Smav intel_ntb_spad_clear(ntb->device); 2689295618Scem} 2690295618Scem 2691289546Scem/* 2692289546Scem * Public API to the rest of the OS 2693250079Scarl */ 2694250079Scarl 2695302484Smavstatic uint8_t 2696303429Smavintel_ntb_spad_count(device_t dev) 2697250079Scarl{ 2698302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2699250079Scarl 2700289539Scem return (ntb->spad_count); 2701250079Scarl} 2702250079Scarl 2703302484Smavstatic uint8_t 2704303429Smavintel_ntb_mw_count(device_t dev) 2705289396Scem{ 2706302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2707295618Scem uint8_t res; 2708289396Scem 2709295618Scem res = ntb->mw_count; 2710291263Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2711295618Scem res--; 2712295618Scem if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2713295618Scem res--; 2714295618Scem return (res); 2715289396Scem} 2716289396Scem 2717302484Smavstatic int 2718303429Smavintel_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 2719250079Scarl{ 2720302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2721250079Scarl 2722289539Scem if (idx >= ntb->spad_count) 2723250079Scarl return (EINVAL); 2724250079Scarl 2725303429Smav intel_ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2726250079Scarl 2727250079Scarl return (0); 2728250079Scarl} 2729250079Scarl 2730295618Scem/* 2731295618Scem * Zeros the local scratchpad. 2732295618Scem */ 2733302484Smavstatic void 2734303429Smavintel_ntb_spad_clear(device_t dev) 2735295618Scem{ 2736302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2737295618Scem unsigned i; 2738295618Scem 2739295618Scem for (i = 0; i < ntb->spad_count; i++) 2740303429Smav intel_ntb_spad_write(dev, i, 0); 2741295618Scem} 2742295618Scem 2743302484Smavstatic int 2744303429Smavintel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2745250079Scarl{ 2746302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2747250079Scarl 2748289539Scem if (idx >= ntb->spad_count) 2749250079Scarl return (EINVAL); 2750250079Scarl 2751303429Smav *val = intel_ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2752250079Scarl 2753250079Scarl return (0); 2754250079Scarl} 2755250079Scarl 2756302484Smavstatic int 2757303429Smavintel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 2758250079Scarl{ 2759302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2760250079Scarl 2761289539Scem if (idx >= ntb->spad_count) 2762250079Scarl return (EINVAL); 2763250079Scarl 2764302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2765303429Smav intel_ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2766255279Scarl else 2767303429Smav intel_ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2768250079Scarl 2769250079Scarl return (0); 2770250079Scarl} 2771250079Scarl 2772302484Smavstatic int 2773303429Smavintel_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2774250079Scarl{ 2775302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2776250079Scarl 2777289539Scem if (idx >= ntb->spad_count) 2778250079Scarl return (EINVAL); 2779250079Scarl 2780302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2781303429Smav *val = intel_ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2782255279Scarl else 2783303429Smav *val = intel_ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2784250079Scarl 2785250079Scarl return (0); 2786250079Scarl} 2787250079Scarl 2788302484Smavstatic int 2789303429Smavintel_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 2790291033Scem caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2791291033Scem bus_addr_t *plimit) 2792250079Scarl{ 2793302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2794289546Scem struct ntb_pci_bar_info *bar; 2795291033Scem bus_addr_t limit; 2796289546Scem size_t bar_b2b_off; 2797291033Scem enum ntb_bar bar_num; 2798250079Scarl 2799303429Smav if (mw_idx >= intel_ntb_mw_count(dev)) 2800289546Scem return (EINVAL); 2801303429Smav mw_idx = intel_ntb_user_mw_to_idx(ntb, mw_idx); 2802250079Scarl 2803303429Smav bar_num = intel_ntb_mw_to_bar(ntb, mw_idx); 2804291033Scem bar = &ntb->bar_info[bar_num]; 2805289546Scem bar_b2b_off = 0; 2806289546Scem if (mw_idx == ntb->b2b_mw_idx) { 2807289546Scem KASSERT(ntb->b2b_off != 0, 2808289546Scem ("user shouldn't get non-shared b2b mw")); 2809289546Scem bar_b2b_off = ntb->b2b_off; 2810289546Scem } 2811250079Scarl 2812291033Scem if (bar_is_64bit(ntb, bar_num)) 2813291033Scem limit = BUS_SPACE_MAXADDR; 2814291033Scem else 2815291033Scem limit = BUS_SPACE_MAXADDR_32BIT; 2816291033Scem 2817289546Scem if (base != NULL) 2818289546Scem *base = bar->pbase + bar_b2b_off; 2819289546Scem if (vbase != NULL) 2820290679Scem *vbase = bar->vbase + bar_b2b_off; 2821289546Scem if (size != NULL) 2822289546Scem *size = bar->size - bar_b2b_off; 2823289546Scem if (align != NULL) 2824289546Scem *align = bar->size; 2825289546Scem if (align_size != NULL) 2826289546Scem *align_size = 1; 2827291033Scem if (plimit != NULL) 2828291033Scem *plimit = limit; 2829289546Scem return (0); 2830250079Scarl} 2831250079Scarl 2832302484Smavstatic int 2833303429Smavintel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 2834250079Scarl{ 2835302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2836289546Scem struct ntb_pci_bar_info *bar; 2837289546Scem uint64_t base, limit, reg_val; 2838289546Scem size_t bar_size, mw_size; 2839289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2840289546Scem enum ntb_bar bar_num; 2841250079Scarl 2842303429Smav if (idx >= intel_ntb_mw_count(dev)) 2843289546Scem return (EINVAL); 2844303429Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2845250079Scarl 2846303429Smav bar_num = intel_ntb_mw_to_bar(ntb, idx); 2847289546Scem bar = &ntb->bar_info[bar_num]; 2848250079Scarl 2849289546Scem bar_size = bar->size; 2850289546Scem if (idx == ntb->b2b_mw_idx) 2851289546Scem mw_size = bar_size - ntb->b2b_off; 2852289546Scem else 2853289546Scem mw_size = bar_size; 2854250079Scarl 2855289546Scem /* Hardware requires that addr is aligned to bar size */ 2856289546Scem if ((addr & (bar_size - 1)) != 0) 2857289546Scem return (EINVAL); 2858250079Scarl 2859289546Scem if (size > mw_size) 2860289546Scem return (EINVAL); 2861289546Scem 2862289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2863289546Scem 2864289546Scem limit = 0; 2865289546Scem if (bar_is_64bit(ntb, bar_num)) { 2866303429Smav base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2867289546Scem 2868289546Scem if (limit_reg != 0 && size != mw_size) 2869289546Scem limit = base + size; 2870289546Scem 2871289546Scem /* Set and verify translation address */ 2872303429Smav intel_ntb_reg_write(8, xlat_reg, addr); 2873303429Smav reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2874289546Scem if (reg_val != addr) { 2875303429Smav intel_ntb_reg_write(8, xlat_reg, 0); 2876289546Scem return (EIO); 2877289546Scem } 2878289546Scem 2879289546Scem /* Set and verify the limit */ 2880303429Smav intel_ntb_reg_write(8, limit_reg, limit); 2881303429Smav reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 2882289546Scem if (reg_val != limit) { 2883303429Smav intel_ntb_reg_write(8, limit_reg, base); 2884303429Smav intel_ntb_reg_write(8, xlat_reg, 0); 2885289546Scem return (EIO); 2886289546Scem } 2887289546Scem } else { 2888289546Scem /* Configure 32-bit (split) BAR MW */ 2889289546Scem 2890291029Scem if ((addr & UINT32_MAX) != addr) 2891291033Scem return (ERANGE); 2892291029Scem if (((addr + size) & UINT32_MAX) != (addr + size)) 2893291033Scem return (ERANGE); 2894289546Scem 2895303429Smav base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 2896289546Scem 2897289546Scem if (limit_reg != 0 && size != mw_size) 2898289546Scem limit = base + size; 2899289546Scem 2900289546Scem /* Set and verify translation address */ 2901303429Smav intel_ntb_reg_write(4, xlat_reg, addr); 2902303429Smav reg_val = intel_ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 2903289546Scem if (reg_val != addr) { 2904303429Smav intel_ntb_reg_write(4, xlat_reg, 0); 2905289546Scem return (EIO); 2906289546Scem } 2907289546Scem 2908289546Scem /* Set and verify the limit */ 2909303429Smav intel_ntb_reg_write(4, limit_reg, limit); 2910303429Smav reg_val = intel_ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 2911289546Scem if (reg_val != limit) { 2912303429Smav intel_ntb_reg_write(4, limit_reg, base); 2913303429Smav intel_ntb_reg_write(4, xlat_reg, 0); 2914289546Scem return (EIO); 2915289546Scem } 2916250079Scarl } 2917289546Scem return (0); 2918250079Scarl} 2919250079Scarl 2920302484Smavstatic int 2921303429Smavintel_ntb_mw_clear_trans(device_t dev, unsigned mw_idx) 2922289596Scem{ 2923289596Scem 2924303429Smav return (intel_ntb_mw_set_trans(dev, mw_idx, 0, 0)); 2925289596Scem} 2926289596Scem 2927302484Smavstatic int 2928303429Smavintel_ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 2929291031Scem{ 2930302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2931291031Scem struct ntb_pci_bar_info *bar; 2932291031Scem 2933303429Smav if (idx >= intel_ntb_mw_count(dev)) 2934291031Scem return (EINVAL); 2935303429Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2936291031Scem 2937303429Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 2938291280Scem *mode = bar->map_mode; 2939291031Scem return (0); 2940291031Scem} 2941291031Scem 2942302484Smavstatic int 2943303429Smavintel_ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 2944291031Scem{ 2945302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2946291263Scem 2947303429Smav if (idx >= intel_ntb_mw_count(dev)) 2948291263Scem return (EINVAL); 2949291263Scem 2950303429Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2951303429Smav return (intel_ntb_mw_set_wc_internal(ntb, idx, mode)); 2952291263Scem} 2953291263Scem 2954291263Scemstatic int 2955303429Smavintel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 2956291263Scem{ 2957291031Scem struct ntb_pci_bar_info *bar; 2958291031Scem int rc; 2959291031Scem 2960303429Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 2961291280Scem if (bar->map_mode == mode) 2962291031Scem return (0); 2963291031Scem 2964291280Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 2965291031Scem if (rc == 0) 2966291280Scem bar->map_mode = mode; 2967291031Scem 2968291031Scem return (rc); 2969291031Scem} 2970291031Scem 2971302484Smavstatic void 2972303429Smavintel_ntb_peer_db_set(device_t dev, uint64_t bit) 2973250079Scarl{ 2974302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2975250079Scarl 2976302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2977295618Scem struct ntb_pci_bar_info *lapic; 2978295618Scem unsigned i; 2979295618Scem 2980295618Scem lapic = ntb->peer_lapic_bar; 2981295618Scem 2982295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2983303429Smav if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0) 2984295618Scem bus_space_write_4(lapic->pci_bus_tag, 2985295618Scem lapic->pci_bus_handle, 2986295618Scem ntb->peer_msix_data[i].nmd_ofs, 2987295618Scem ntb->peer_msix_data[i].nmd_data); 2988295618Scem } 2989295618Scem return; 2990295618Scem } 2991295618Scem 2992302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 2993303429Smav intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 2994289347Scem return; 2995289209Scem } 2996289347Scem 2997289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2998250079Scarl} 2999250079Scarl 3000302484Smavstatic int 3001303429Smavintel_ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3002289542Scem{ 3003302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3004289542Scem struct ntb_pci_bar_info *bar; 3005289542Scem uint64_t regoff; 3006289542Scem 3007302484Smav KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3008289542Scem 3009302484Smav if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3010289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3011289542Scem regoff = ntb->peer_reg->db_bell; 3012289542Scem } else { 3013289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3014289543Scem ("invalid b2b idx")); 3015289542Scem 3016303429Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3017290682Scem regoff = XEON_PDOORBELL_OFFSET; 3018289542Scem } 3019289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3020289542Scem 3021289542Scem /* HACK: Specific to current x86 bus implementation. */ 3022302484Smav *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3023302484Smav *db_size = ntb->reg->db_size; 3024302484Smav return (0); 3025289542Scem} 3026289542Scem 3027302484Smavstatic uint64_t 3028303429Smavintel_ntb_db_valid_mask(device_t dev) 3029289597Scem{ 3030302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3031289597Scem 3032289597Scem return (ntb->db_valid_mask); 3033289597Scem} 3034289597Scem 3035302484Smavstatic int 3036303429Smavintel_ntb_db_vector_count(device_t dev) 3037289598Scem{ 3038302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3039289598Scem 3040302484Smav return (ntb->db_vec_count); 3041302484Smav} 3042302484Smav 3043302484Smavstatic uint64_t 3044303429Smavintel_ntb_db_vector_mask(device_t dev, uint32_t vector) 3045302484Smav{ 3046302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3047302484Smav 3048289598Scem if (vector > ntb->db_vec_count) 3049289598Scem return (0); 3050303429Smav return (ntb->db_valid_mask & intel_ntb_vec_mask(ntb, vector)); 3051289598Scem} 3052289598Scem 3053302484Smavstatic bool 3054303429Smavintel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3055250079Scarl{ 3056302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3057250079Scarl 3058289546Scem if (speed != NULL) 3059303429Smav *speed = intel_ntb_link_sta_speed(ntb); 3060289546Scem if (width != NULL) 3061303429Smav *width = intel_ntb_link_sta_width(ntb); 3062289546Scem return (link_is_up(ntb)); 3063250079Scarl} 3064250079Scarl 3065255272Scarlstatic void 3066255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 3067250079Scarl{ 3068255272Scarl 3069289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3070289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3071289209Scem bar->pbase = rman_get_start(bar->pci_resource); 3072289209Scem bar->size = rman_get_size(bar->pci_resource); 3073289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 3074250079Scarl} 3075255268Scarl 3076302484Smavstatic device_method_t ntb_intel_methods[] = { 3077302484Smav /* Device interface */ 3078303429Smav DEVMETHOD(device_probe, intel_ntb_probe), 3079303429Smav DEVMETHOD(device_attach, intel_ntb_attach), 3080303429Smav DEVMETHOD(device_detach, intel_ntb_detach), 3081302484Smav /* NTB interface */ 3082303429Smav DEVMETHOD(ntb_link_is_up, intel_ntb_link_is_up), 3083303429Smav DEVMETHOD(ntb_link_enable, intel_ntb_link_enable), 3084303429Smav DEVMETHOD(ntb_link_disable, intel_ntb_link_disable), 3085303429Smav DEVMETHOD(ntb_link_enabled, intel_ntb_link_enabled), 3086303429Smav DEVMETHOD(ntb_mw_count, intel_ntb_mw_count), 3087303429Smav DEVMETHOD(ntb_mw_get_range, intel_ntb_mw_get_range), 3088303429Smav DEVMETHOD(ntb_mw_set_trans, intel_ntb_mw_set_trans), 3089303429Smav DEVMETHOD(ntb_mw_clear_trans, intel_ntb_mw_clear_trans), 3090303429Smav DEVMETHOD(ntb_mw_get_wc, intel_ntb_mw_get_wc), 3091303429Smav DEVMETHOD(ntb_mw_set_wc, intel_ntb_mw_set_wc), 3092303429Smav DEVMETHOD(ntb_spad_count, intel_ntb_spad_count), 3093303429Smav DEVMETHOD(ntb_spad_clear, intel_ntb_spad_clear), 3094303429Smav DEVMETHOD(ntb_spad_write, intel_ntb_spad_write), 3095303429Smav DEVMETHOD(ntb_spad_read, intel_ntb_spad_read), 3096303429Smav DEVMETHOD(ntb_peer_spad_write, intel_ntb_peer_spad_write), 3097303429Smav DEVMETHOD(ntb_peer_spad_read, intel_ntb_peer_spad_read), 3098303429Smav DEVMETHOD(ntb_db_valid_mask, intel_ntb_db_valid_mask), 3099303429Smav DEVMETHOD(ntb_db_vector_count, intel_ntb_db_vector_count), 3100303429Smav DEVMETHOD(ntb_db_vector_mask, intel_ntb_db_vector_mask), 3101303429Smav DEVMETHOD(ntb_db_clear, intel_ntb_db_clear), 3102303429Smav DEVMETHOD(ntb_db_clear_mask, intel_ntb_db_clear_mask), 3103303429Smav DEVMETHOD(ntb_db_read, intel_ntb_db_read), 3104303429Smav DEVMETHOD(ntb_db_set_mask, intel_ntb_db_set_mask), 3105303429Smav DEVMETHOD(ntb_peer_db_addr, intel_ntb_peer_db_addr), 3106303429Smav DEVMETHOD(ntb_peer_db_set, intel_ntb_peer_db_set), 3107302484Smav DEVMETHOD_END 3108302484Smav}; 3109255268Scarl 3110302484Smavstatic DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3111302484Smav sizeof(struct ntb_softc)); 3112302484SmavDRIVER_MODULE(ntb_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL); 3113302484SmavMODULE_DEPEND(ntb_intel, ntb, 1, 1, 1); 3114302484SmavMODULE_VERSION(ntb_intel, 1); 3115