1250079Scarl/*- 2323455Smav * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org> 3250079Scarl * Copyright (C) 2013 Intel Corporation 4289542Scem * Copyright (C) 2015 EMC Corporation 5250079Scarl * All rights reserved. 6250079Scarl * 7250079Scarl * Redistribution and use in source and binary forms, with or without 8250079Scarl * modification, are permitted provided that the following conditions 9250079Scarl * are met: 10250079Scarl * 1. Redistributions of source code must retain the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer. 12250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 13250079Scarl * notice, this list of conditions and the following disclaimer in the 14250079Scarl * documentation and/or other materials provided with the distribution. 15250079Scarl * 16250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26250079Scarl * SUCH DAMAGE. 27250079Scarl */ 28250079Scarl 29302484Smav/* 30302484Smav * The Non-Transparent Bridge (NTB) is a device that allows you to connect 31302484Smav * two or more systems using a PCI-e links, providing remote memory access. 32302484Smav * 33302484Smav * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 34302484Smav * 35302484Smav * NOTE: Much of the code in this module is shared with Linux. Any patches may 36302484Smav * be picked up and redistributed in Linux with a dual GPL/BSD license. 37302484Smav */ 38302484Smav 39250079Scarl#include <sys/cdefs.h> 40250079Scarl__FBSDID("$FreeBSD: stable/11/sys/dev/ntb/ntb_hw/ntb_hw_intel.c 355152 2019-11-28 00:41:42Z mav $"); 41250079Scarl 42250079Scarl#include <sys/param.h> 43250079Scarl#include <sys/kernel.h> 44250079Scarl#include <sys/systm.h> 45250079Scarl#include <sys/bus.h> 46289774Scem#include <sys/endian.h> 47302493Smav#include <sys/interrupt.h> 48250079Scarl#include <sys/malloc.h> 49250079Scarl#include <sys/module.h> 50295618Scem#include <sys/mutex.h> 51295618Scem#include <sys/pciio.h> 52250079Scarl#include <sys/queue.h> 53250079Scarl#include <sys/rman.h> 54289774Scem#include <sys/sbuf.h> 55289207Scem#include <sys/sysctl.h> 56250079Scarl#include <vm/vm.h> 57250079Scarl#include <vm/pmap.h> 58250079Scarl#include <machine/bus.h> 59295618Scem#include <machine/intr_machdep.h> 60250079Scarl#include <machine/resource.h> 61250079Scarl#include <dev/pci/pcireg.h> 62250079Scarl#include <dev/pci/pcivar.h> 63250079Scarl 64323032Smav#include "ntb_hw_intel.h" 65302484Smav#include "../ntb.h" 66250079Scarl 67289648Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 68250079Scarl 69289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 70289648Scem#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 71291032Scem#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 72250079Scarl 73295618Scem#define NTB_MSIX_VER_GUARD 0xaabbccdd 74295618Scem#define NTB_MSIX_RECEIVED 0xe0f0e0f0 75295618Scem 76295618Scem/* 77295618Scem * PCI constants could be somewhere more generic, but aren't defined/used in 78295618Scem * pci.c. 79295618Scem */ 80295618Scem#define PCI_MSIX_ENTRY_SIZE 16 81295618Scem#define PCI_MSIX_ENTRY_LOWER_ADDR 0 82295618Scem#define PCI_MSIX_ENTRY_UPPER_ADDR 4 83295618Scem#define PCI_MSIX_ENTRY_DATA 8 84295618Scem 85250079Scarlenum ntb_device_type { 86250079Scarl NTB_XEON, 87289648Scem NTB_ATOM 88250079Scarl}; 89250079Scarl 90289610Scem/* ntb_conn_type are hardware numbers, cannot change. */ 91289610Scemenum ntb_conn_type { 92289610Scem NTB_CONN_TRANSPARENT = 0, 93289610Scem NTB_CONN_B2B = 1, 94289610Scem NTB_CONN_RP = 2, 95289610Scem}; 96289610Scem 97289610Scemenum ntb_b2b_direction { 98289610Scem NTB_DEV_USD = 0, 99289610Scem NTB_DEV_DSD = 1, 100289610Scem}; 101289610Scem 102289539Scemenum ntb_bar { 103289539Scem NTB_CONFIG_BAR = 0, 104289539Scem NTB_B2B_BAR_1, 105289539Scem NTB_B2B_BAR_2, 106289539Scem NTB_B2B_BAR_3, 107289539Scem NTB_MAX_BARS 108289539Scem}; 109289539Scem 110295618Scemenum { 111295618Scem NTB_MSIX_GUARD = 0, 112295618Scem NTB_MSIX_DATA0, 113295618Scem NTB_MSIX_DATA1, 114295618Scem NTB_MSIX_DATA2, 115295618Scem NTB_MSIX_OFS0, 116295618Scem NTB_MSIX_OFS1, 117295618Scem NTB_MSIX_OFS2, 118295618Scem NTB_MSIX_DONE, 119295618Scem NTB_MAX_MSIX_SPAD 120295618Scem}; 121295618Scem 122255274Scarl/* Device features and workarounds */ 123302484Smav#define HAS_FEATURE(ntb, feature) \ 124302484Smav (((ntb)->features & (feature)) != 0) 125255274Scarl 126250079Scarlstruct ntb_hw_info { 127250079Scarl uint32_t device_id; 128255274Scarl const char *desc; 129250079Scarl enum ntb_device_type type; 130289397Scem uint32_t features; 131250079Scarl}; 132250079Scarl 133250079Scarlstruct ntb_pci_bar_info { 134250079Scarl bus_space_tag_t pci_bus_tag; 135250079Scarl bus_space_handle_t pci_bus_handle; 136250079Scarl int pci_resource_id; 137250079Scarl struct resource *pci_resource; 138250079Scarl vm_paddr_t pbase; 139290679Scem caddr_t vbase; 140290679Scem vm_size_t size; 141291280Scem vm_memattr_t map_mode; 142289543Scem 143289543Scem /* Configuration register offsets */ 144289543Scem uint32_t psz_off; 145289543Scem uint32_t ssz_off; 146289543Scem uint32_t pbarxlat_off; 147250079Scarl}; 148250079Scarl 149250079Scarlstruct ntb_int_info { 150250079Scarl struct resource *res; 151250079Scarl int rid; 152250079Scarl void *tag; 153250079Scarl}; 154250079Scarl 155289546Scemstruct ntb_vec { 156250079Scarl struct ntb_softc *ntb; 157289546Scem uint32_t num; 158295618Scem unsigned masked; 159250079Scarl}; 160250079Scarl 161289542Scemstruct ntb_reg { 162289542Scem uint32_t ntb_ctl; 163289542Scem uint32_t lnk_sta; 164289542Scem uint8_t db_size; 165289542Scem unsigned mw_bar[NTB_MAX_BARS]; 166289542Scem}; 167289542Scem 168289542Scemstruct ntb_alt_reg { 169289542Scem uint32_t db_bell; 170289542Scem uint32_t db_mask; 171289542Scem uint32_t spad; 172289542Scem}; 173289542Scem 174289542Scemstruct ntb_xlat_reg { 175289546Scem uint32_t bar0_base; 176289546Scem uint32_t bar2_base; 177289546Scem uint32_t bar4_base; 178289546Scem uint32_t bar5_base; 179289546Scem 180289546Scem uint32_t bar2_xlat; 181289546Scem uint32_t bar4_xlat; 182289546Scem uint32_t bar5_xlat; 183289546Scem 184289546Scem uint32_t bar2_limit; 185289546Scem uint32_t bar4_limit; 186289546Scem uint32_t bar5_limit; 187289542Scem}; 188289542Scem 189289542Scemstruct ntb_b2b_addr { 190289542Scem uint64_t bar0_addr; 191289542Scem uint64_t bar2_addr64; 192289542Scem uint64_t bar4_addr64; 193289542Scem uint64_t bar4_addr32; 194289542Scem uint64_t bar5_addr32; 195289542Scem}; 196289542Scem 197295618Scemstruct ntb_msix_data { 198295618Scem uint32_t nmd_ofs; 199295618Scem uint32_t nmd_data; 200295618Scem}; 201295618Scem 202250079Scarlstruct ntb_softc { 203303429Smav /* ntb.c context. Do not move! Must go first! */ 204303429Smav void *ntb_store; 205303429Smav 206250079Scarl device_t device; 207250079Scarl enum ntb_device_type type; 208289774Scem uint32_t features; 209250079Scarl 210250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 211250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 212250079Scarl uint32_t allocated_interrupts; 213250079Scarl 214295618Scem struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215295618Scem struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216295618Scem bool peer_msix_good; 217295618Scem bool peer_msix_done; 218295618Scem struct ntb_pci_bar_info *peer_lapic_bar; 219295618Scem struct callout peer_msix_work; 220295618Scem 221250079Scarl struct callout heartbeat_timer; 222250079Scarl struct callout lr_timer; 223250079Scarl 224289546Scem struct ntb_vec *msix_vec; 225250079Scarl 226289610Scem uint32_t ppd; 227289610Scem enum ntb_conn_type conn_type; 228289610Scem enum ntb_b2b_direction dev_type; 229289539Scem 230289542Scem /* Offset of peer bar0 in B2B BAR */ 231289542Scem uint64_t b2b_off; 232289542Scem /* Memory window used to access peer bar0 */ 233289543Scem#define B2B_MW_DISABLED UINT8_MAX 234289542Scem uint8_t b2b_mw_idx; 235301293Smav uint32_t msix_xlat; 236295618Scem uint8_t msix_mw_idx; 237289542Scem 238289539Scem uint8_t mw_count; 239289539Scem uint8_t spad_count; 240289539Scem uint8_t db_count; 241289539Scem uint8_t db_vec_count; 242289539Scem uint8_t db_vec_shift; 243289542Scem 244289546Scem /* Protects local db_mask. */ 245289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 246289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 247289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 248289542Scem struct mtx db_mask_lock; 249289542Scem 250290686Scem volatile uint32_t ntb_ctl; 251290686Scem volatile uint32_t lnk_sta; 252289542Scem 253289542Scem uint64_t db_valid_mask; 254289542Scem uint64_t db_link_mask; 255289546Scem uint64_t db_mask; 256322980Smav uint64_t fake_db; /* NTB_SB01BASE_LOCKUP*/ 257322980Smav uint64_t force_db; /* NTB_SB01BASE_LOCKUP*/ 258289542Scem 259289542Scem int last_ts; /* ticks @ last irq */ 260289542Scem 261289542Scem const struct ntb_reg *reg; 262289542Scem const struct ntb_alt_reg *self_reg; 263289542Scem const struct ntb_alt_reg *peer_reg; 264289542Scem const struct ntb_xlat_reg *xlat_reg; 265250079Scarl}; 266250079Scarl 267289234Scem#ifdef __i386__ 268289234Scemstatic __inline uint64_t 269289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 270289234Scem bus_size_t offset) 271289234Scem{ 272289234Scem 273289234Scem return (bus_space_read_4(tag, handle, offset) | 274289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 275289234Scem} 276289234Scem 277289234Scemstatic __inline void 278289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 279289234Scem bus_size_t offset, uint64_t val) 280289234Scem{ 281289234Scem 282289234Scem bus_space_write_4(tag, handle, offset, val); 283289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 284289234Scem} 285289234Scem#endif 286289234Scem 287303429Smav#define intel_ntb_bar_read(SIZE, bar, offset) \ 288255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 289255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 290303429Smav#define intel_ntb_bar_write(SIZE, bar, offset, val) \ 291255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 292255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 293303429Smav#define intel_ntb_reg_read(SIZE, offset) \ 294303429Smav intel_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 295303429Smav#define intel_ntb_reg_write(SIZE, offset, val) \ 296303429Smav intel_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 297303429Smav#define intel_ntb_mw_read(SIZE, offset) \ 298303429Smav intel_ntb_bar_read(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 299303429Smav offset) 300303429Smav#define intel_ntb_mw_write(SIZE, offset, val) \ 301303429Smav intel_ntb_bar_write(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 302289397Scem offset, val) 303250079Scarl 304303429Smavstatic int intel_ntb_probe(device_t device); 305303429Smavstatic int intel_ntb_attach(device_t device); 306303429Smavstatic int intel_ntb_detach(device_t device); 307303429Smavstatic uint64_t intel_ntb_db_valid_mask(device_t dev); 308303429Smavstatic void intel_ntb_spad_clear(device_t dev); 309303429Smavstatic uint64_t intel_ntb_db_vector_mask(device_t dev, uint32_t vector); 310303429Smavstatic bool intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, 311302484Smav enum ntb_width *width); 312303429Smavstatic int intel_ntb_link_enable(device_t dev, enum ntb_speed speed, 313302484Smav enum ntb_width width); 314303429Smavstatic int intel_ntb_link_disable(device_t dev); 315303429Smavstatic int intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 316303429Smavstatic int intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 317302484Smav 318303429Smavstatic unsigned intel_ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 319303429Smavstatic inline enum ntb_bar intel_ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 320289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 321289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 322289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 323303429Smavstatic int intel_ntb_map_pci_bars(struct ntb_softc *ntb); 324303429Smavstatic int intel_ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 325291280Scem vm_memattr_t); 326289647Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 327289647Scem const char *); 328255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 329255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 330255272Scarl struct ntb_pci_bar_info *bar); 331303429Smavstatic void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb); 332303429Smavstatic int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 333303429Smavstatic int intel_ntb_init_isr(struct ntb_softc *ntb); 334303429Smavstatic int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 335303429Smavstatic int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 336303429Smavstatic void intel_ntb_teardown_interrupts(struct ntb_softc *ntb); 337303429Smavstatic inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 338303429Smavstatic void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec); 339289546Scemstatic void ndev_vec_isr(void *arg); 340289546Scemstatic void ndev_irq_isr(void *arg); 341289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 342290678Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 343290678Scemstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 344303429Smavstatic int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 345303429Smavstatic void intel_ntb_free_msix_vec(struct ntb_softc *ntb); 346303429Smavstatic void intel_ntb_get_msix_info(struct ntb_softc *ntb); 347303429Smavstatic void intel_ntb_exchange_msix(void *); 348303429Smavstatic struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id); 349303429Smavstatic void intel_ntb_detect_max_mw(struct ntb_softc *ntb); 350303429Smavstatic int intel_ntb_detect_xeon(struct ntb_softc *ntb); 351303429Smavstatic int intel_ntb_detect_atom(struct ntb_softc *ntb); 352303429Smavstatic int intel_ntb_xeon_init_dev(struct ntb_softc *ntb); 353303429Smavstatic int intel_ntb_atom_init_dev(struct ntb_softc *ntb); 354303429Smavstatic void intel_ntb_teardown_xeon(struct ntb_softc *ntb); 355289648Scemstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 356289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 357289543Scem enum ntb_bar regbar); 358289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 359289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 360289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 361289543Scem enum ntb_bar idx); 362289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 363289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 364289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 365295618Scemstatic inline bool _xeon_link_is_up(struct ntb_softc *ntb); 366289648Scemstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 367303429Smavstatic inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *); 368303429Smavstatic inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *); 369289648Scemstatic void atom_link_hb(void *arg); 370289648Scemstatic void recover_atom_link(void *arg); 371303429Smavstatic bool intel_ntb_poll_link(struct ntb_softc *ntb); 372255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 373303429Smavstatic void intel_ntb_sysctl_init(struct ntb_softc *); 374289774Scemstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 375300100Scemstatic int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 376300100Scemstatic int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 377289774Scemstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 378289774Scemstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 379250079Scarl 380290685Scemstatic unsigned g_ntb_hw_debug_level; 381290685ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 382290685Scem &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 383303429Smav#define intel_ntb_printf(lvl, ...) do { \ 384290685Scem if ((lvl) <= g_ntb_hw_debug_level) { \ 385290685Scem device_printf(ntb->device, __VA_ARGS__); \ 386290685Scem } \ 387290685Scem} while (0) 388290685Scem 389295486Scem#define _NTB_PAT_UC 0 390295486Scem#define _NTB_PAT_WC 1 391295486Scem#define _NTB_PAT_WT 4 392295486Scem#define _NTB_PAT_WP 5 393295486Scem#define _NTB_PAT_WB 6 394295486Scem#define _NTB_PAT_UCM 7 395295486Scemstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 396295486ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 397295486Scem &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 398295486Scem "UC: " __XSTRING(_NTB_PAT_UC) ", " 399295486Scem "WC: " __XSTRING(_NTB_PAT_WC) ", " 400295486Scem "WT: " __XSTRING(_NTB_PAT_WT) ", " 401295486Scem "WP: " __XSTRING(_NTB_PAT_WP) ", " 402295486Scem "WB: " __XSTRING(_NTB_PAT_WB) ", " 403295486Scem "UC-: " __XSTRING(_NTB_PAT_UCM)); 404291030Scem 405295486Scemstatic inline vm_memattr_t 406303429Smavintel_ntb_pat_flags(void) 407295486Scem{ 408295486Scem 409295486Scem switch (g_ntb_mw_pat) { 410295486Scem case _NTB_PAT_WC: 411295486Scem return (VM_MEMATTR_WRITE_COMBINING); 412295486Scem case _NTB_PAT_WT: 413295486Scem return (VM_MEMATTR_WRITE_THROUGH); 414295486Scem case _NTB_PAT_WP: 415295486Scem return (VM_MEMATTR_WRITE_PROTECTED); 416295486Scem case _NTB_PAT_WB: 417295486Scem return (VM_MEMATTR_WRITE_BACK); 418295486Scem case _NTB_PAT_UCM: 419295486Scem return (VM_MEMATTR_WEAK_UNCACHEABLE); 420295486Scem case _NTB_PAT_UC: 421295486Scem /* FALLTHROUGH */ 422295486Scem default: 423295486Scem return (VM_MEMATTR_UNCACHEABLE); 424295486Scem } 425295486Scem} 426295486Scem 427295487Scem/* 428295487Scem * Well, this obviously doesn't belong here, but it doesn't seem to exist 429295487Scem * anywhere better yet. 430295487Scem */ 431295487Scemstatic inline const char * 432303429Smavintel_ntb_vm_memattr_to_str(vm_memattr_t pat) 433295487Scem{ 434295487Scem 435295487Scem switch (pat) { 436295487Scem case VM_MEMATTR_WRITE_COMBINING: 437295487Scem return ("WRITE_COMBINING"); 438295487Scem case VM_MEMATTR_WRITE_THROUGH: 439295487Scem return ("WRITE_THROUGH"); 440295487Scem case VM_MEMATTR_WRITE_PROTECTED: 441295487Scem return ("WRITE_PROTECTED"); 442295487Scem case VM_MEMATTR_WRITE_BACK: 443295487Scem return ("WRITE_BACK"); 444295487Scem case VM_MEMATTR_WEAK_UNCACHEABLE: 445295487Scem return ("UNCACHED"); 446295487Scem case VM_MEMATTR_UNCACHEABLE: 447295487Scem return ("UNCACHEABLE"); 448295487Scem default: 449295487Scem return ("UNKNOWN"); 450295487Scem } 451295487Scem} 452295487Scem 453302508Smavstatic int g_ntb_msix_idx = 1; 454295618ScemSYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 455295618Scem 0, "Use this memory window to access the peer MSIX message complex on " 456295618Scem "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 457295618Scem "Like b2b_mw_idx, negative values index from the last available memory " 458295618Scem "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 459295618Scem 460291263Scemstatic int g_ntb_mw_idx = -1; 461291263ScemSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 462291263Scem 0, "Use this memory window to access the peer NTB registers. A " 463291263Scem "non-negative value starts from the first MW index; a negative value " 464291263Scem "starts from the last MW index. The default is -1, i.e., the last " 465291263Scem "available memory window. Both sides of the NTB MUST set the same " 466291263Scem "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 467291263Scem 468302484Smav/* Hardware owns the low 16 bits of features. */ 469302484Smav#define NTB_BAR_SIZE_4K (1 << 0) 470302484Smav#define NTB_SDOORBELL_LOCKUP (1 << 1) 471302484Smav#define NTB_SB01BASE_LOCKUP (1 << 2) 472302484Smav#define NTB_B2BDOORBELL_BIT14 (1 << 3) 473302484Smav/* Software/configuration owns the top 16 bits. */ 474302484Smav#define NTB_SPLIT_BAR (1ull << 16) 475302484Smav 476302484Smav#define NTB_FEATURES_STR \ 477302484Smav "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 478302484Smav "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 479302484Smav 480250079Scarlstatic struct ntb_hw_info pci_ids[] = { 481289612Scem /* XXX: PS/SS IDs left out until they are supported. */ 482289612Scem { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 483289648Scem NTB_ATOM, 0 }, 484289233Scem 485289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 486289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 487289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 488289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 489289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 490289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 491289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 492289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 493289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 494289538Scem NTB_SB01BASE_LOCKUP }, 495289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 496289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 497289538Scem NTB_SB01BASE_LOCKUP }, 498289233Scem 499289648Scem { 0x00000000, NULL, NTB_ATOM, 0 } 500250079Scarl}; 501250079Scarl 502289648Scemstatic const struct ntb_reg atom_reg = { 503289648Scem .ntb_ctl = ATOM_NTBCNTL_OFFSET, 504289648Scem .lnk_sta = ATOM_LINK_STATUS_OFFSET, 505289542Scem .db_size = sizeof(uint64_t), 506289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 507289542Scem}; 508289542Scem 509289648Scemstatic const struct ntb_alt_reg atom_pri_reg = { 510289648Scem .db_bell = ATOM_PDOORBELL_OFFSET, 511289648Scem .db_mask = ATOM_PDBMSK_OFFSET, 512289648Scem .spad = ATOM_SPAD_OFFSET, 513289607Scem}; 514289607Scem 515289648Scemstatic const struct ntb_alt_reg atom_b2b_reg = { 516289648Scem .db_bell = ATOM_B2B_DOORBELL_OFFSET, 517289648Scem .spad = ATOM_B2B_SPAD_OFFSET, 518289542Scem}; 519289542Scem 520289648Scemstatic const struct ntb_xlat_reg atom_sec_xlat = { 521289542Scem#if 0 522289542Scem /* "FIXME" says the Linux driver. */ 523289648Scem .bar0_base = ATOM_SBAR0BASE_OFFSET, 524289648Scem .bar2_base = ATOM_SBAR2BASE_OFFSET, 525289648Scem .bar4_base = ATOM_SBAR4BASE_OFFSET, 526289546Scem 527289648Scem .bar2_limit = ATOM_SBAR2LMT_OFFSET, 528289648Scem .bar4_limit = ATOM_SBAR4LMT_OFFSET, 529289542Scem#endif 530289546Scem 531289648Scem .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 532289648Scem .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 533289542Scem}; 534289542Scem 535289542Scemstatic const struct ntb_reg xeon_reg = { 536289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 537289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 538289542Scem .db_size = sizeof(uint16_t), 539289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 540289542Scem}; 541289542Scem 542289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 543289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 544289607Scem .db_mask = XEON_PDBMSK_OFFSET, 545289607Scem .spad = XEON_SPAD_OFFSET, 546289607Scem}; 547289607Scem 548289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 549289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 550289542Scem .spad = XEON_B2B_SPAD_OFFSET, 551289542Scem}; 552289542Scem 553289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 554289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 555289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 556289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 557289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 558289546Scem 559289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 560289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 561289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 562289546Scem 563289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 564289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 565289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 566289542Scem}; 567289542Scem 568289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 569290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 570290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 571290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 572290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 573290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 574289542Scem}; 575289542Scem 576289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 577290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 578290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 579290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 580290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 581290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 582289542Scem}; 583289542Scem 584289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 585289614Scem "B2B MW segment overrides -- MUST be the same on both sides"); 586289614Scem 587289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 588289614Scem &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 589289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 590289614Scem "the window at BAR2, on the upstream side of the link. MUST be the same " 591289614Scem "address on both sides."); 592289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 593289614Scem &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 594289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 595289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 596289614Scem "(split-BAR mode)."); 597289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 598289646Scem &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 599289614Scem "(split-BAR mode)."); 600289614Scem 601289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 602289614Scem &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 603289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 604289614Scem "the window at BAR2, on the downstream side of the link. MUST be the same" 605289614Scem " address on both sides."); 606289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 607289614Scem &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 608289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 609289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 610289614Scem "(split-BAR mode)."); 611289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 612289646Scem &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 613289614Scem "(split-BAR mode)."); 614289614Scem 615250079Scarl/* 616250079Scarl * OS <-> Driver interface structures 617250079Scarl */ 618250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 619250079Scarl 620250079Scarl/* 621250079Scarl * OS <-> Driver linkage functions 622250079Scarl */ 623250079Scarlstatic int 624303429Smavintel_ntb_probe(device_t device) 625250079Scarl{ 626289209Scem struct ntb_hw_info *p; 627250079Scarl 628303429Smav p = intel_ntb_get_device_info(pci_get_devid(device)); 629289209Scem if (p == NULL) 630250079Scarl return (ENXIO); 631289209Scem 632289209Scem device_set_desc(device, p->desc); 633289209Scem return (0); 634250079Scarl} 635250079Scarl 636250079Scarlstatic int 637303429Smavintel_ntb_attach(device_t device) 638250079Scarl{ 639289209Scem struct ntb_softc *ntb; 640289209Scem struct ntb_hw_info *p; 641250079Scarl int error; 642250079Scarl 643302484Smav ntb = device_get_softc(device); 644303429Smav p = intel_ntb_get_device_info(pci_get_devid(device)); 645289209Scem 646250079Scarl ntb->device = device; 647250079Scarl ntb->type = p->type; 648255274Scarl ntb->features = p->features; 649289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 650295618Scem ntb->msix_mw_idx = B2B_MW_DISABLED; 651250079Scarl 652289648Scem /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 653283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 654283291Sjkim callout_init(&ntb->lr_timer, 1); 655295618Scem callout_init(&ntb->peer_msix_work, 1); 656289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 657250079Scarl 658289648Scem if (ntb->type == NTB_ATOM) 659303429Smav error = intel_ntb_detect_atom(ntb); 660289348Scem else 661303429Smav error = intel_ntb_detect_xeon(ntb); 662290682Scem if (error != 0) 663289348Scem goto out; 664289348Scem 665303429Smav intel_ntb_detect_max_mw(ntb); 666289396Scem 667290682Scem pci_enable_busmaster(ntb->device); 668290682Scem 669303429Smav error = intel_ntb_map_pci_bars(ntb); 670290682Scem if (error != 0) 671289209Scem goto out; 672289648Scem if (ntb->type == NTB_ATOM) 673303429Smav error = intel_ntb_atom_init_dev(ntb); 674289272Scem else 675303429Smav error = intel_ntb_xeon_init_dev(ntb); 676290682Scem if (error != 0) 677289209Scem goto out; 678290682Scem 679303429Smav intel_ntb_spad_clear(device); 680295618Scem 681303429Smav intel_ntb_poll_link(ntb); 682290682Scem 683303429Smav intel_ntb_sysctl_init(ntb); 684250079Scarl 685302484Smav /* Attach children to this controller */ 686303429Smav error = ntb_register_device(device); 687302484Smav 688289209Scemout: 689289209Scem if (error != 0) 690303429Smav intel_ntb_detach(device); 691250079Scarl return (error); 692250079Scarl} 693250079Scarl 694250079Scarlstatic int 695303429Smavintel_ntb_detach(device_t device) 696250079Scarl{ 697289209Scem struct ntb_softc *ntb; 698250079Scarl 699302484Smav ntb = device_get_softc(device); 700289542Scem 701302484Smav /* Detach & delete all children */ 702303429Smav ntb_unregister_device(device); 703302484Smav 704295618Scem if (ntb->self_reg != NULL) { 705295618Scem DB_MASK_LOCK(ntb); 706295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 707295618Scem DB_MASK_UNLOCK(ntb); 708295618Scem } 709250079Scarl callout_drain(&ntb->heartbeat_timer); 710250079Scarl callout_drain(&ntb->lr_timer); 711295618Scem callout_drain(&ntb->peer_msix_work); 712290682Scem pci_disable_busmaster(ntb->device); 713289272Scem if (ntb->type == NTB_XEON) 714303429Smav intel_ntb_teardown_xeon(ntb); 715303429Smav intel_ntb_teardown_interrupts(ntb); 716289397Scem 717289542Scem mtx_destroy(&ntb->db_mask_lock); 718289542Scem 719303429Smav intel_ntb_unmap_pci_bar(ntb); 720250079Scarl 721250079Scarl return (0); 722250079Scarl} 723250079Scarl 724289542Scem/* 725289542Scem * Driver internal routines 726289542Scem */ 727289539Scemstatic inline enum ntb_bar 728303429Smavintel_ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 729289539Scem{ 730289539Scem 731291263Scem KASSERT(mw < ntb->mw_count, 732289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 733289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 734289539Scem 735289542Scem return (ntb->reg->mw_bar[mw]); 736289539Scem} 737289539Scem 738289546Scemstatic inline bool 739289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 740289546Scem{ 741289546Scem /* XXX This assertion could be stronger. */ 742289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 743302484Smav return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 744289546Scem} 745289546Scem 746289546Scemstatic inline void 747289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 748289546Scem uint32_t *xlat, uint32_t *lmt) 749289546Scem{ 750289546Scem uint32_t basev, lmtv, xlatv; 751289546Scem 752289546Scem switch (bar) { 753289546Scem case NTB_B2B_BAR_1: 754289546Scem basev = ntb->xlat_reg->bar2_base; 755289546Scem lmtv = ntb->xlat_reg->bar2_limit; 756289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 757289546Scem break; 758289546Scem case NTB_B2B_BAR_2: 759289546Scem basev = ntb->xlat_reg->bar4_base; 760289546Scem lmtv = ntb->xlat_reg->bar4_limit; 761289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 762289546Scem break; 763289546Scem case NTB_B2B_BAR_3: 764289546Scem basev = ntb->xlat_reg->bar5_base; 765289546Scem lmtv = ntb->xlat_reg->bar5_limit; 766289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 767289546Scem break; 768289546Scem default: 769289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 770289546Scem ("bad bar")); 771289546Scem basev = lmtv = xlatv = 0; 772289546Scem break; 773289546Scem } 774289546Scem 775289546Scem if (base != NULL) 776289546Scem *base = basev; 777289546Scem if (xlat != NULL) 778289546Scem *xlat = xlatv; 779289546Scem if (lmt != NULL) 780289546Scem *lmt = lmtv; 781289546Scem} 782289546Scem 783250079Scarlstatic int 784303429Smavintel_ntb_map_pci_bars(struct ntb_softc *ntb) 785250079Scarl{ 786255272Scarl int rc; 787250079Scarl 788250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 789289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 790255272Scarl if (rc != 0) 791289541Scem goto out; 792255272Scarl 793289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 794289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 795255272Scarl if (rc != 0) 796289541Scem goto out; 797289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 798289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 799289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 800255272Scarl 801289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 802291263Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 803291263Scem if (rc != 0) 804291263Scem goto out; 805289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 806289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 807289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 808289543Scem 809302484Smav if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 810289541Scem goto out; 811289397Scem 812289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 813291263Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 814289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 815289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 816289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 817250079Scarl 818289541Scemout: 819289209Scem if (rc != 0) 820255272Scarl device_printf(ntb->device, 821255272Scarl "unable to allocate pci resource\n"); 822255272Scarl return (rc); 823255272Scarl} 824255272Scarl 825289541Scemstatic void 826289647Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 827289647Scem const char *kind) 828289541Scem{ 829289541Scem 830289647Scem device_printf(ntb->device, 831289647Scem "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 832289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 833289647Scem (char *)bar->vbase + bar->size - 1, 834289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 835289647Scem (uintmax_t)bar->size, kind); 836289541Scem} 837289541Scem 838255272Scarlstatic int 839255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 840255272Scarl{ 841255272Scarl 842255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 843289209Scem &bar->pci_resource_id, RF_ACTIVE); 844255272Scarl if (bar->pci_resource == NULL) 845255272Scarl return (ENXIO); 846289209Scem 847289209Scem save_bar_parameters(bar); 848291280Scem bar->map_mode = VM_MEMATTR_UNCACHEABLE; 849289647Scem print_map_success(ntb, bar, "mmr"); 850289209Scem return (0); 851255272Scarl} 852255272Scarl 853255272Scarlstatic int 854255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 855255272Scarl{ 856255272Scarl int rc; 857291280Scem vm_memattr_t mapmode; 858255276Scarl uint8_t bar_size_bits = 0; 859255272Scarl 860289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 861289209Scem &bar->pci_resource_id, RF_ACTIVE); 862250079Scarl 863255272Scarl if (bar->pci_resource == NULL) 864255272Scarl return (ENXIO); 865255276Scarl 866289209Scem save_bar_parameters(bar); 867289209Scem /* 868289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 869289209Scem * hardware issue. To work around this, query the size it should be 870289209Scem * configured to by the device and modify the resource to correspond to 871289209Scem * this new size. The BIOS on systems with this problem is required to 872289209Scem * provide enough address space to allow the driver to make this change 873289209Scem * safely. 874289209Scem * 875289209Scem * Ideally I could have just specified the size when I allocated the 876289209Scem * resource like: 877289209Scem * bus_alloc_resource(ntb->device, 878289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 879289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 880289209Scem * but the PCI driver does not honor the size in this call, so we have 881289209Scem * to modify it after the fact. 882289209Scem */ 883302484Smav if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 884289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 885289209Scem bar_size_bits = pci_read_config(ntb->device, 886289209Scem XEON_PBAR23SZ_OFFSET, 1); 887289209Scem else 888289209Scem bar_size_bits = pci_read_config(ntb->device, 889289209Scem XEON_PBAR45SZ_OFFSET, 1); 890289209Scem 891289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 892289209Scem bar->pci_resource, bar->pbase, 893289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 894255272Scarl if (rc != 0) { 895289209Scem device_printf(ntb->device, 896289209Scem "unable to resize bar\n"); 897255272Scarl return (rc); 898250079Scarl } 899289209Scem 900289209Scem save_bar_parameters(bar); 901250079Scarl } 902289209Scem 903291280Scem bar->map_mode = VM_MEMATTR_UNCACHEABLE; 904291030Scem print_map_success(ntb, bar, "mw"); 905291280Scem 906295486Scem /* 907295486Scem * Optionally, mark MW BARs as anything other than UC to improve 908295486Scem * performance. 909295486Scem */ 910303429Smav mapmode = intel_ntb_pat_flags(); 911295486Scem if (mapmode == bar->map_mode) 912295486Scem return (0); 913291030Scem 914291280Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 915291031Scem if (rc == 0) { 916291280Scem bar->map_mode = mapmode; 917289209Scem device_printf(ntb->device, 918289647Scem "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 919291280Scem "%s.\n", 920289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 921289647Scem (char *)bar->vbase + bar->size - 1, 922291280Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 923303429Smav intel_ntb_vm_memattr_to_str(mapmode)); 924291031Scem } else 925289647Scem device_printf(ntb->device, 926289647Scem "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 927291280Scem "%s: %d\n", 928289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 929289647Scem (char *)bar->vbase + bar->size - 1, 930289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 931303429Smav intel_ntb_vm_memattr_to_str(mapmode), rc); 932289647Scem /* Proceed anyway */ 933250079Scarl return (0); 934250079Scarl} 935250079Scarl 936250079Scarlstatic void 937303429Smavintel_ntb_unmap_pci_bar(struct ntb_softc *ntb) 938250079Scarl{ 939250079Scarl struct ntb_pci_bar_info *current_bar; 940250079Scarl int i; 941250079Scarl 942289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 943250079Scarl current_bar = &ntb->bar_info[i]; 944250079Scarl if (current_bar->pci_resource != NULL) 945250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 946250079Scarl current_bar->pci_resource_id, 947250079Scarl current_bar->pci_resource); 948250079Scarl } 949250079Scarl} 950250079Scarl 951250079Scarlstatic int 952303429Smavintel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 953250079Scarl{ 954289342Scem uint32_t i; 955289342Scem int rc; 956289342Scem 957289342Scem for (i = 0; i < num_vectors; i++) { 958289342Scem ntb->int_info[i].rid = i + 1; 959289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 960289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 961289342Scem if (ntb->int_info[i].res == NULL) { 962289342Scem device_printf(ntb->device, 963289342Scem "bus_alloc_resource failed\n"); 964289342Scem return (ENOMEM); 965289342Scem } 966289342Scem ntb->int_info[i].tag = NULL; 967289342Scem ntb->allocated_interrupts++; 968289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 969289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 970289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 971289342Scem if (rc != 0) { 972289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 973289342Scem return (ENXIO); 974289342Scem } 975289342Scem } 976289342Scem return (0); 977289342Scem} 978289342Scem 979289344Scem/* 980289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 981289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 982289344Scem * should be okay. This tunable should enable us to test that hypothesis 983289344Scem * when someone gets their hands on some Xeon hardware. 984289344Scem */ 985289344Scemstatic int ntb_force_remap_mode; 986289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 987289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 988289344Scem " to a smaller number of ithreads, even if the desired number are " 989289344Scem "available"); 990289344Scem 991289344Scem/* 992289344Scem * In case it is NOT ok, give consumers an abort button. 993289344Scem */ 994289344Scemstatic int ntb_prefer_intx; 995289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 996289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 997289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 998289344Scem "behavior)"); 999289344Scem 1000289344Scem/* 1001289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 1002289344Scem * round-robin fashion. 1003289344Scem */ 1004289342Scemstatic int 1005303429Smavintel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1006289344Scem{ 1007289344Scem u_int *vectors; 1008289344Scem uint32_t i; 1009289344Scem int rc; 1010289344Scem 1011289344Scem if (ntb_prefer_intx != 0) 1012289344Scem return (ENXIO); 1013289344Scem 1014289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1015289344Scem 1016289344Scem for (i = 0; i < desired; i++) 1017289344Scem vectors[i] = (i % avail) + 1; 1018289344Scem 1019289344Scem rc = pci_remap_msix(dev, desired, vectors); 1020289344Scem free(vectors, M_NTB); 1021289344Scem return (rc); 1022289344Scem} 1023289344Scem 1024289344Scemstatic int 1025303429Smavintel_ntb_init_isr(struct ntb_softc *ntb) 1026289342Scem{ 1027289344Scem uint32_t desired_vectors, num_vectors; 1028289342Scem int rc; 1029250079Scarl 1030250079Scarl ntb->allocated_interrupts = 0; 1031289542Scem ntb->last_ts = ticks; 1032289347Scem 1033250079Scarl /* 1034295618Scem * Mask all doorbell interrupts. (Except link events!) 1035250079Scarl */ 1036295618Scem DB_MASK_LOCK(ntb); 1037295618Scem ntb->db_mask = ntb->db_valid_mask; 1038295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1039295618Scem DB_MASK_UNLOCK(ntb); 1040250079Scarl 1041289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1042289539Scem ntb->db_count); 1043289344Scem if (desired_vectors >= 1) { 1044289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 1045250079Scarl 1046289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 1047289344Scem num_vectors == desired_vectors) 1048289344Scem num_vectors--; 1049289344Scem 1050289344Scem if (rc == 0 && num_vectors < desired_vectors) { 1051303429Smav rc = intel_ntb_remap_msix(ntb->device, desired_vectors, 1052289344Scem num_vectors); 1053289344Scem if (rc == 0) 1054289344Scem num_vectors = desired_vectors; 1055289344Scem else 1056289344Scem pci_release_msi(ntb->device); 1057289344Scem } 1058289344Scem if (rc != 0) 1059289344Scem num_vectors = 1; 1060289344Scem } else 1061289344Scem num_vectors = 1; 1062289344Scem 1063289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1064302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1065295618Scem device_printf(ntb->device, 1066295618Scem "Errata workaround does not support MSI or INTX\n"); 1067295618Scem return (EINVAL); 1068295618Scem } 1069295618Scem 1070289539Scem ntb->db_vec_count = 1; 1071290680Scem ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1072303429Smav rc = intel_ntb_setup_legacy_interrupt(ntb); 1073289539Scem } else { 1074300531Scem if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1075302484Smav HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1076300531Scem device_printf(ntb->device, 1077300531Scem "Errata workaround expects %d doorbell bits\n", 1078300531Scem XEON_NONLINK_DB_MSIX_BITS); 1079300531Scem return (EINVAL); 1080300531Scem } 1081300531Scem 1082303429Smav intel_ntb_create_msix_vec(ntb, num_vectors); 1083303429Smav rc = intel_ntb_setup_msix(ntb, num_vectors); 1084289539Scem } 1085289539Scem if (rc != 0) { 1086289539Scem device_printf(ntb->device, 1087289539Scem "Error allocating interrupts: %d\n", rc); 1088303429Smav intel_ntb_free_msix_vec(ntb); 1089289396Scem } 1090289396Scem 1091289342Scem return (rc); 1092289342Scem} 1093289342Scem 1094289342Scemstatic int 1095303429Smavintel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1096289342Scem{ 1097289342Scem int rc; 1098289342Scem 1099289342Scem ntb->int_info[0].rid = 0; 1100289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1101289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1102289342Scem if (ntb->int_info[0].res == NULL) { 1103289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 1104289342Scem return (ENOMEM); 1105250079Scarl } 1106250079Scarl 1107289342Scem ntb->int_info[0].tag = NULL; 1108289342Scem ntb->allocated_interrupts = 1; 1109289342Scem 1110289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1111289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1112289342Scem ntb, &ntb->int_info[0].tag); 1113289342Scem if (rc != 0) { 1114289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 1115289342Scem return (ENXIO); 1116289342Scem } 1117289342Scem 1118250079Scarl return (0); 1119250079Scarl} 1120250079Scarl 1121250079Scarlstatic void 1122303429Smavintel_ntb_teardown_interrupts(struct ntb_softc *ntb) 1123250079Scarl{ 1124250079Scarl struct ntb_int_info *current_int; 1125250079Scarl int i; 1126250079Scarl 1127289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 1128250079Scarl current_int = &ntb->int_info[i]; 1129250079Scarl if (current_int->tag != NULL) 1130250079Scarl bus_teardown_intr(ntb->device, current_int->res, 1131250079Scarl current_int->tag); 1132250079Scarl 1133250079Scarl if (current_int->res != NULL) 1134250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 1135250079Scarl rman_get_rid(current_int->res), current_int->res); 1136250079Scarl } 1137250079Scarl 1138303429Smav intel_ntb_free_msix_vec(ntb); 1139250079Scarl pci_release_msi(ntb->device); 1140250079Scarl} 1141250079Scarl 1142289347Scem/* 1143289648Scem * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1144289347Scem * out to make code clearer. 1145289347Scem */ 1146289539Scemstatic inline uint64_t 1147289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 1148289347Scem{ 1149289347Scem 1150289648Scem if (ntb->type == NTB_ATOM) 1151303429Smav return (intel_ntb_reg_read(8, regoff)); 1152289347Scem 1153289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1154289347Scem 1155303429Smav return (intel_ntb_reg_read(2, regoff)); 1156289347Scem} 1157289347Scem 1158289539Scemstatic inline void 1159289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1160289347Scem{ 1161289347Scem 1162289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 1163289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1164289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 1165289542Scem (uintmax_t)ntb->db_valid_mask)); 1166289542Scem 1167289607Scem if (regoff == ntb->self_reg->db_mask) 1168289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 1169290678Scem db_iowrite_raw(ntb, regoff, val); 1170290678Scem} 1171289542Scem 1172290678Scemstatic inline void 1173290678Scemdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1174290678Scem{ 1175290678Scem 1176289648Scem if (ntb->type == NTB_ATOM) { 1177303429Smav intel_ntb_reg_write(8, regoff, val); 1178289347Scem return; 1179289347Scem } 1180289347Scem 1181289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1182303429Smav intel_ntb_reg_write(2, regoff, (uint16_t)val); 1183289347Scem} 1184289347Scem 1185302484Smavstatic void 1186303429Smavintel_ntb_db_set_mask(device_t dev, uint64_t bits) 1187289542Scem{ 1188302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1189289542Scem 1190289546Scem DB_MASK_LOCK(ntb); 1191289542Scem ntb->db_mask |= bits; 1192302493Smav if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1193302493Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1194289546Scem DB_MASK_UNLOCK(ntb); 1195289542Scem} 1196289542Scem 1197302484Smavstatic void 1198303429Smavintel_ntb_db_clear_mask(device_t dev, uint64_t bits) 1199289542Scem{ 1200302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1201302493Smav uint64_t ibits; 1202302493Smav int i; 1203289542Scem 1204289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1205289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1206289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1207289542Scem (uintmax_t)ntb->db_valid_mask)); 1208289542Scem 1209289546Scem DB_MASK_LOCK(ntb); 1210322980Smav ibits = ntb->fake_db & ntb->db_mask & bits; 1211289542Scem ntb->db_mask &= ~bits; 1212302493Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1213302493Smav /* Simulate fake interrupts if unmasked DB bits are set. */ 1214322980Smav ntb->force_db |= ibits; 1215302493Smav for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1216303429Smav if ((ibits & intel_ntb_db_vector_mask(dev, i)) != 0) 1217302493Smav swi_sched(ntb->int_info[i].tag, 0); 1218302493Smav } 1219302493Smav } else { 1220302493Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1221302493Smav } 1222289546Scem DB_MASK_UNLOCK(ntb); 1223289542Scem} 1224289542Scem 1225302484Smavstatic uint64_t 1226303429Smavintel_ntb_db_read(device_t dev) 1227289281Scem{ 1228302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1229289281Scem 1230302493Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1231322980Smav return (ntb->fake_db); 1232295618Scem 1233289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 1234289281Scem} 1235289281Scem 1236302484Smavstatic void 1237303429Smavintel_ntb_db_clear(device_t dev, uint64_t bits) 1238289281Scem{ 1239302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1240289281Scem 1241289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1242289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1243289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1244289546Scem (uintmax_t)ntb->db_valid_mask)); 1245289546Scem 1246302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1247302493Smav DB_MASK_LOCK(ntb); 1248322980Smav ntb->fake_db &= ~bits; 1249302493Smav DB_MASK_UNLOCK(ntb); 1250295618Scem return; 1251295618Scem } 1252295618Scem 1253289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1254289281Scem} 1255289281Scem 1256289540Scemstatic inline uint64_t 1257303429Smavintel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1258250079Scarl{ 1259289540Scem uint64_t shift, mask; 1260250079Scarl 1261302493Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1262302493Smav /* 1263302493Smav * Remap vectors in custom way to make at least first 1264302493Smav * three doorbells to not generate stray events. 1265302493Smav * This breaks Linux compatibility (if one existed) 1266302493Smav * when more then one DB is used (not by if_ntb). 1267302493Smav */ 1268302493Smav if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1) 1269302493Smav return (1 << db_vector); 1270302493Smav if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1) 1271302493Smav return (0x7ffc); 1272302493Smav } 1273302493Smav 1274289540Scem shift = ntb->db_vec_shift; 1275289540Scem mask = (1ull << shift) - 1; 1276289540Scem return (mask << (shift * db_vector)); 1277250079Scarl} 1278250079Scarl 1279250079Scarlstatic void 1280303429Smavintel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1281250079Scarl{ 1282289540Scem uint64_t vec_mask; 1283250079Scarl 1284289542Scem ntb->last_ts = ticks; 1285303429Smav vec_mask = intel_ntb_vec_mask(ntb, vec); 1286250079Scarl 1287289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1288303429Smav if (intel_ntb_poll_link(ntb)) 1289302484Smav ntb_link_event(ntb->device); 1290289540Scem } 1291289540Scem 1292302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1293295618Scem (vec_mask & ntb->db_link_mask) == 0) { 1294295618Scem DB_MASK_LOCK(ntb); 1295302493Smav 1296322980Smav /* 1297322980Smav * Do not report same DB events again if not cleared yet, 1298322980Smav * unless the mask was just cleared for them and this 1299322980Smav * interrupt handler call can be the consequence of it. 1300322980Smav */ 1301322980Smav vec_mask &= ~ntb->fake_db | ntb->force_db; 1302322980Smav ntb->force_db &= ~vec_mask; 1303302493Smav 1304302493Smav /* Update our internal doorbell register. */ 1305322980Smav ntb->fake_db |= vec_mask; 1306302493Smav 1307302493Smav /* Do not report masked DB events. */ 1308302493Smav vec_mask &= ~ntb->db_mask; 1309302493Smav 1310295618Scem DB_MASK_UNLOCK(ntb); 1311295618Scem } 1312295618Scem 1313289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1314302484Smav ntb_db_event(ntb->device, vec); 1315289546Scem} 1316250079Scarl 1317289546Scemstatic void 1318289546Scemndev_vec_isr(void *arg) 1319289546Scem{ 1320289546Scem struct ntb_vec *nvec = arg; 1321250079Scarl 1322303429Smav intel_ntb_interrupt(nvec->ntb, nvec->num); 1323250079Scarl} 1324250079Scarl 1325250079Scarlstatic void 1326289546Scemndev_irq_isr(void *arg) 1327250079Scarl{ 1328289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1329303429Smav intel_ntb_interrupt(arg, 0); 1330250079Scarl} 1331250079Scarl 1332250079Scarlstatic int 1333303429Smavintel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1334250079Scarl{ 1335289342Scem uint32_t i; 1336250079Scarl 1337289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1338250079Scarl M_ZERO | M_WAITOK); 1339250079Scarl for (i = 0; i < num_vectors; i++) { 1340289546Scem ntb->msix_vec[i].num = i; 1341289546Scem ntb->msix_vec[i].ntb = ntb; 1342250079Scarl } 1343250079Scarl 1344250079Scarl return (0); 1345250079Scarl} 1346250079Scarl 1347250079Scarlstatic void 1348303429Smavintel_ntb_free_msix_vec(struct ntb_softc *ntb) 1349250079Scarl{ 1350250079Scarl 1351289546Scem if (ntb->msix_vec == NULL) 1352289539Scem return; 1353289539Scem 1354289546Scem free(ntb->msix_vec, M_NTB); 1355289546Scem ntb->msix_vec = NULL; 1356250079Scarl} 1357250079Scarl 1358295618Scemstatic void 1359303429Smavintel_ntb_get_msix_info(struct ntb_softc *ntb) 1360295618Scem{ 1361295618Scem struct pci_devinfo *dinfo; 1362295618Scem struct pcicfg_msix *msix; 1363295618Scem uint32_t laddr, data, i, offset; 1364295618Scem 1365295618Scem dinfo = device_get_ivars(ntb->device); 1366295618Scem msix = &dinfo->cfg.msix; 1367295618Scem 1368300531Scem CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1369300531Scem 1370300531Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1371295618Scem offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1372295618Scem 1373295618Scem laddr = bus_read_4(msix->msix_table_res, offset + 1374295618Scem PCI_MSIX_ENTRY_LOWER_ADDR); 1375303429Smav intel_ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1376295618Scem 1377295618Scem KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1378295618Scem ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1379295618Scem MSI_INTEL_ADDR_BASE)); 1380301293Smav ntb->msix_data[i].nmd_ofs = laddr; 1381295618Scem 1382295618Scem data = bus_read_4(msix->msix_table_res, offset + 1383295618Scem PCI_MSIX_ENTRY_DATA); 1384303429Smav intel_ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1385295618Scem 1386295618Scem ntb->msix_data[i].nmd_data = data; 1387295618Scem } 1388295618Scem} 1389295618Scem 1390250079Scarlstatic struct ntb_hw_info * 1391303429Smavintel_ntb_get_device_info(uint32_t device_id) 1392250079Scarl{ 1393250079Scarl struct ntb_hw_info *ep = pci_ids; 1394250079Scarl 1395250079Scarl while (ep->device_id) { 1396250079Scarl if (ep->device_id == device_id) 1397250079Scarl return (ep); 1398250079Scarl ++ep; 1399250079Scarl } 1400250079Scarl return (NULL); 1401250079Scarl} 1402250079Scarl 1403289272Scemstatic void 1404303429Smavintel_ntb_teardown_xeon(struct ntb_softc *ntb) 1405250079Scarl{ 1406250079Scarl 1407289617Scem if (ntb->reg != NULL) 1408303429Smav intel_ntb_link_disable(ntb->device); 1409250079Scarl} 1410250079Scarl 1411289397Scemstatic void 1412303429Smavintel_ntb_detect_max_mw(struct ntb_softc *ntb) 1413289397Scem{ 1414289397Scem 1415289648Scem if (ntb->type == NTB_ATOM) { 1416289648Scem ntb->mw_count = ATOM_MW_COUNT; 1417289397Scem return; 1418289397Scem } 1419289397Scem 1420302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1421289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1422289397Scem else 1423289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1424289397Scem} 1425289397Scem 1426250079Scarlstatic int 1427303429Smavintel_ntb_detect_xeon(struct ntb_softc *ntb) 1428250079Scarl{ 1429289348Scem uint8_t ppd, conn_type; 1430250079Scarl 1431289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1432289348Scem ntb->ppd = ppd; 1433250079Scarl 1434289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1435290681Scem ntb->dev_type = NTB_DEV_DSD; 1436290681Scem else 1437289257Scem ntb->dev_type = NTB_DEV_USD; 1438289257Scem 1439289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1440289397Scem ntb->features |= NTB_SPLIT_BAR; 1441289397Scem 1442302508Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1443302508Smav !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1444302508Smav device_printf(ntb->device, 1445302508Smav "Can not apply SB01BASE_LOCKUP workaround " 1446302508Smav "with split BARs disabled!\n"); 1447302508Smav device_printf(ntb->device, 1448302508Smav "Expect system hangs under heavy NTB traffic!\n"); 1449302508Smav ntb->features &= ~NTB_SB01BASE_LOCKUP; 1450302508Smav } 1451302508Smav 1452295618Scem /* 1453295618Scem * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1454295618Scem * errata workaround; only do one at a time. 1455295618Scem */ 1456302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1457295618Scem ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1458289542Scem 1459289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1460289348Scem switch (conn_type) { 1461289348Scem case NTB_CONN_B2B: 1462289348Scem ntb->conn_type = conn_type; 1463289348Scem break; 1464289348Scem case NTB_CONN_RP: 1465289348Scem case NTB_CONN_TRANSPARENT: 1466289348Scem default: 1467289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1468289348Scem (unsigned)conn_type); 1469289348Scem return (ENXIO); 1470289348Scem } 1471289348Scem return (0); 1472289348Scem} 1473289348Scem 1474289348Scemstatic int 1475303429Smavintel_ntb_detect_atom(struct ntb_softc *ntb) 1476289348Scem{ 1477289348Scem uint32_t ppd, conn_type; 1478289348Scem 1479289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1480289348Scem ntb->ppd = ppd; 1481289348Scem 1482289648Scem if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1483289348Scem ntb->dev_type = NTB_DEV_DSD; 1484289348Scem else 1485289348Scem ntb->dev_type = NTB_DEV_USD; 1486289348Scem 1487289648Scem conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1488289348Scem switch (conn_type) { 1489289348Scem case NTB_CONN_B2B: 1490289348Scem ntb->conn_type = conn_type; 1491289348Scem break; 1492289348Scem default: 1493289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1494289348Scem return (ENXIO); 1495289348Scem } 1496289348Scem return (0); 1497289348Scem} 1498289348Scem 1499289348Scemstatic int 1500303429Smavintel_ntb_xeon_init_dev(struct ntb_softc *ntb) 1501289348Scem{ 1502289542Scem int rc; 1503289348Scem 1504289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1505289542Scem ntb->db_count = XEON_DB_COUNT; 1506289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1507289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1508289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1509289257Scem 1510289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1511250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1512289348Scem ntb->conn_type); 1513250079Scarl return (ENXIO); 1514250079Scarl } 1515250079Scarl 1516289542Scem ntb->reg = &xeon_reg; 1517289607Scem ntb->self_reg = &xeon_pri_reg; 1518289542Scem ntb->peer_reg = &xeon_b2b_reg; 1519289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1520289542Scem 1521302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1522322980Smav ntb->force_db = ntb->fake_db = 0; 1523295618Scem ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1524295618Scem ntb->mw_count; 1525303429Smav intel_ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1526295618Scem g_ntb_msix_idx, ntb->msix_mw_idx); 1527303429Smav rc = intel_ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1528295618Scem VM_MEMATTR_UNCACHEABLE); 1529295618Scem KASSERT(rc == 0, ("shouldn't fail")); 1530302484Smav } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1531295618Scem /* 1532295618Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1533295618Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1534295618Scem * which may hang the system. To workaround this, use a memory 1535295618Scem * window to access the interrupt and scratch pad registers on the 1536295618Scem * remote system. 1537295618Scem */ 1538291263Scem ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1539291263Scem ntb->mw_count; 1540303429Smav intel_ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1541291263Scem g_ntb_mw_idx, ntb->b2b_mw_idx); 1542303429Smav rc = intel_ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1543295618Scem VM_MEMATTR_UNCACHEABLE); 1544291263Scem KASSERT(rc == 0, ("shouldn't fail")); 1545302484Smav } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1546289208Scem /* 1547289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1548289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1549289542Scem * since bit 14 is the last bit. 1550289542Scem * 1551289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1552289542Scem * anyway. Nor for non-B2B connection types. 1553289542Scem */ 1554289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1555250079Scarl 1556289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1557250079Scarl 1558289542Scem if (ntb->dev_type == NTB_DEV_USD) 1559289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1560289542Scem &xeon_b2b_usd_addr); 1561289542Scem else 1562289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1563289542Scem &xeon_b2b_dsd_addr); 1564289542Scem if (rc != 0) 1565289542Scem return (rc); 1566289271Scem 1567250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1568303429Smav intel_ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1569289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1570255279Scarl 1571290682Scem /* 1572290682Scem * Mask all doorbell interrupts. 1573290682Scem */ 1574295618Scem DB_MASK_LOCK(ntb); 1575295618Scem ntb->db_mask = ntb->db_valid_mask; 1576295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1577295618Scem DB_MASK_UNLOCK(ntb); 1578250079Scarl 1579303429Smav rc = intel_ntb_init_isr(ntb); 1580290682Scem return (rc); 1581250079Scarl} 1582250079Scarl 1583250079Scarlstatic int 1584303429Smavintel_ntb_atom_init_dev(struct ntb_softc *ntb) 1585250079Scarl{ 1586290682Scem int error; 1587250079Scarl 1588289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1589289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1590250079Scarl 1591289648Scem ntb->spad_count = ATOM_SPAD_COUNT; 1592289648Scem ntb->db_count = ATOM_DB_COUNT; 1593289648Scem ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1594289648Scem ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1595289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1596250079Scarl 1597289648Scem ntb->reg = &atom_reg; 1598289648Scem ntb->self_reg = &atom_pri_reg; 1599289648Scem ntb->peer_reg = &atom_b2b_reg; 1600289648Scem ntb->xlat_reg = &atom_sec_xlat; 1601289542Scem 1602250079Scarl /* 1603289648Scem * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1604250079Scarl * resolved. Mask transaction layer internal parity errors. 1605250079Scarl */ 1606250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1607250079Scarl 1608289648Scem configure_atom_secondary_side_bars(ntb); 1609250079Scarl 1610250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1611303429Smav intel_ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1612250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1613289209Scem 1614303429Smav error = intel_ntb_init_isr(ntb); 1615290682Scem if (error != 0) 1616290682Scem return (error); 1617290682Scem 1618289542Scem /* Initiate PCI-E link training */ 1619303429Smav intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1620250079Scarl 1621289648Scem callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1622289542Scem 1623250079Scarl return (0); 1624250079Scarl} 1625250079Scarl 1626289648Scem/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1627255279Scarlstatic void 1628289648Scemconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1629255279Scarl{ 1630255279Scarl 1631255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1632303429Smav intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1633290725Scem XEON_B2B_BAR2_ADDR64); 1634303429Smav intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1635290725Scem XEON_B2B_BAR4_ADDR64); 1636303429Smav intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1637303429Smav intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1638255279Scarl } else { 1639303429Smav intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1640290725Scem XEON_B2B_BAR2_ADDR64); 1641303429Smav intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1642290725Scem XEON_B2B_BAR4_ADDR64); 1643303429Smav intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1644303429Smav intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1645255279Scarl } 1646255279Scarl} 1647255279Scarl 1648289543Scem 1649289543Scem/* 1650289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1651289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1652289543Scem * remains for use by a higher layer. 1653289543Scem * 1654289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1655289543Scem * MW size is sufficiently large. 1656289543Scem */ 1657289543Scemstatic unsigned int ntb_b2b_mw_share; 1658289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1659289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1660289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1661289543Scem "value here."); 1662289543Scem 1663289543Scemstatic void 1664289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1665289543Scem enum ntb_bar regbar) 1666289543Scem{ 1667289543Scem struct ntb_pci_bar_info *bar; 1668289543Scem uint8_t bar_sz; 1669289543Scem 1670302484Smav if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1671289543Scem return; 1672289543Scem 1673289543Scem bar = &ntb->bar_info[idx]; 1674289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1675289543Scem if (idx == regbar) { 1676289543Scem if (ntb->b2b_off != 0) 1677289543Scem bar_sz--; 1678289543Scem else 1679289543Scem bar_sz = 0; 1680289543Scem } 1681289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1682289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1683289543Scem (void)bar_sz; 1684289543Scem} 1685289543Scem 1686289543Scemstatic void 1687289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1688289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1689289543Scem{ 1690301293Smav uint64_t reg_val; 1691289546Scem uint32_t base_reg, lmt_reg; 1692289543Scem 1693289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1694302482Smav if (idx == regbar) { 1695302482Smav if (ntb->b2b_off) 1696302482Smav bar_addr += ntb->b2b_off; 1697302482Smav else 1698302482Smav bar_addr = 0; 1699302482Smav } 1700289543Scem 1701289546Scem if (!bar_is_64bit(ntb, idx)) { 1702303429Smav intel_ntb_reg_write(4, base_reg, bar_addr); 1703303429Smav reg_val = intel_ntb_reg_read(4, base_reg); 1704302531Smav (void)reg_val; 1705302531Smav 1706303429Smav intel_ntb_reg_write(4, lmt_reg, bar_addr); 1707303429Smav reg_val = intel_ntb_reg_read(4, lmt_reg); 1708295618Scem (void)reg_val; 1709302531Smav } else { 1710303429Smav intel_ntb_reg_write(8, base_reg, bar_addr); 1711303429Smav reg_val = intel_ntb_reg_read(8, base_reg); 1712302531Smav (void)reg_val; 1713295618Scem 1714303429Smav intel_ntb_reg_write(8, lmt_reg, bar_addr); 1715303429Smav reg_val = intel_ntb_reg_read(8, lmt_reg); 1716295618Scem (void)reg_val; 1717289543Scem } 1718289543Scem} 1719289543Scem 1720289543Scemstatic void 1721289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1722289543Scem{ 1723289543Scem struct ntb_pci_bar_info *bar; 1724289543Scem 1725289543Scem bar = &ntb->bar_info[idx]; 1726302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1727303429Smav intel_ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1728303429Smav base_addr = intel_ntb_reg_read(4, bar->pbarxlat_off); 1729289543Scem } else { 1730303429Smav intel_ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1731303429Smav base_addr = intel_ntb_reg_read(8, bar->pbarxlat_off); 1732289543Scem } 1733289543Scem (void)base_addr; 1734289543Scem} 1735289543Scem 1736289542Scemstatic int 1737289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1738289542Scem const struct ntb_b2b_addr *peer_addr) 1739255279Scarl{ 1740289543Scem struct ntb_pci_bar_info *b2b_bar; 1741289543Scem vm_size_t bar_size; 1742289543Scem uint64_t bar_addr; 1743289543Scem enum ntb_bar b2b_bar_num, i; 1744255279Scarl 1745289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1746289543Scem b2b_bar = NULL; 1747289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1748289543Scem ntb->b2b_off = 0; 1749289543Scem } else { 1750303429Smav b2b_bar_num = intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1751289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1752289543Scem ("invalid b2b mw bar")); 1753289543Scem 1754289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1755289543Scem bar_size = b2b_bar->size; 1756289543Scem 1757289543Scem if (ntb_b2b_mw_share != 0 && 1758289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1759289543Scem ntb->b2b_off = bar_size >> 1; 1760289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1761289543Scem ntb->b2b_off = 0; 1762289543Scem } else { 1763289543Scem device_printf(ntb->device, 1764289543Scem "B2B bar size is too small!\n"); 1765289543Scem return (EIO); 1766289543Scem } 1767255279Scarl } 1768289542Scem 1769289543Scem /* 1770289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1771289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1772289543Scem */ 1773289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1774289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1775289543Scem 1776289543Scem bar_addr = 0; 1777289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1778289543Scem bar_addr = addr->bar0_addr; 1779289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1780289543Scem bar_addr = addr->bar2_addr64; 1781302484Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1782289543Scem bar_addr = addr->bar4_addr64; 1783289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1784289543Scem bar_addr = addr->bar4_addr32; 1785289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1786289543Scem bar_addr = addr->bar5_addr32; 1787289543Scem else 1788289543Scem KASSERT(false, ("invalid bar")); 1789289543Scem 1790303429Smav intel_ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1791289543Scem 1792289543Scem /* 1793289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1794289543Scem * register BAR. The B2B BAR is either disabled above or configured 1795289543Scem * half-size. It starts at PBAR xlat + offset. 1796289543Scem * 1797289543Scem * Also set up incoming BAR limits == base (zero length window). 1798289543Scem */ 1799289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1800289543Scem b2b_bar_num); 1801302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1802289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1803289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1804289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1805289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1806289542Scem } else 1807289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1808289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1809289543Scem 1810289543Scem /* Zero incoming translation addrs */ 1811303429Smav intel_ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1812303429Smav intel_ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1813289543Scem 1814302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1815302510Smav uint32_t xlat_reg, lmt_reg; 1816302493Smav enum ntb_bar bar_num; 1817295618Scem 1818295618Scem /* 1819295618Scem * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1820295618Scem * workaround 1821295618Scem */ 1822303429Smav bar_num = intel_ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1823302510Smav bar_get_xlat_params(ntb, bar_num, NULL, &xlat_reg, &lmt_reg); 1824302510Smav if (bar_is_64bit(ntb, bar_num)) { 1825303429Smav intel_ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE); 1826303429Smav ntb->msix_xlat = intel_ntb_reg_read(8, xlat_reg); 1827303429Smav intel_ntb_reg_write(8, lmt_reg, 0); 1828301293Smav } else { 1829303429Smav intel_ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE); 1830303429Smav ntb->msix_xlat = intel_ntb_reg_read(4, xlat_reg); 1831303429Smav intel_ntb_reg_write(4, lmt_reg, 0); 1832301293Smav } 1833302493Smav 1834302493Smav ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1835295618Scem } 1836303429Smav (void)intel_ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1837303429Smav (void)intel_ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1838295618Scem 1839289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1840303429Smav intel_ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1841303429Smav intel_ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1842289543Scem 1843289543Scem /* Set outgoing translation offsets */ 1844289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1845302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1846289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1847289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1848289543Scem } else 1849289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1850289543Scem 1851289543Scem /* Set the translation offset for B2B registers */ 1852289543Scem bar_addr = 0; 1853289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1854289543Scem bar_addr = peer_addr->bar0_addr; 1855289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1856289543Scem bar_addr = peer_addr->bar2_addr64; 1857302484Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1858289543Scem bar_addr = peer_addr->bar4_addr64; 1859289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1860289543Scem bar_addr = peer_addr->bar4_addr32; 1861289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1862289543Scem bar_addr = peer_addr->bar5_addr32; 1863289543Scem else 1864289543Scem KASSERT(false, ("invalid bar")); 1865289543Scem 1866289543Scem /* 1867289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1868289543Scem * at a time. 1869289543Scem */ 1870303429Smav intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1871303429Smav intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1872289542Scem return (0); 1873255279Scarl} 1874255279Scarl 1875289546Scemstatic inline bool 1876295618Scem_xeon_link_is_up(struct ntb_softc *ntb) 1877295618Scem{ 1878295618Scem 1879295618Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1880295618Scem return (true); 1881295618Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1882295618Scem} 1883295618Scem 1884295618Scemstatic inline bool 1885289546Scemlink_is_up(struct ntb_softc *ntb) 1886289546Scem{ 1887289546Scem 1888295618Scem if (ntb->type == NTB_XEON) 1889295618Scem return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1890302484Smav !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 1891289546Scem 1892289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1893289648Scem return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1894289546Scem} 1895289546Scem 1896289546Scemstatic inline bool 1897289648Scematom_link_is_err(struct ntb_softc *ntb) 1898289546Scem{ 1899289546Scem uint32_t status; 1900289546Scem 1901289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1902289546Scem 1903303429Smav status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1904289648Scem if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1905289546Scem return (true); 1906289546Scem 1907303429Smav status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1908289648Scem return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1909289546Scem} 1910289546Scem 1911289648Scem/* Atom does not have link status interrupt, poll on that platform */ 1912250079Scarlstatic void 1913289648Scematom_link_hb(void *arg) 1914250079Scarl{ 1915250079Scarl struct ntb_softc *ntb = arg; 1916289546Scem sbintime_t timo, poll_ts; 1917250079Scarl 1918289546Scem timo = NTB_HB_TIMEOUT * hz; 1919289546Scem poll_ts = ntb->last_ts + timo; 1920289546Scem 1921289542Scem /* 1922289542Scem * Delay polling the link status if an interrupt was received, unless 1923289542Scem * the cached link status says the link is down. 1924289542Scem */ 1925289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1926289546Scem timo = poll_ts - ticks; 1927289542Scem goto out; 1928289546Scem } 1929289542Scem 1930303429Smav if (intel_ntb_poll_link(ntb)) 1931302484Smav ntb_link_event(ntb->device); 1932289542Scem 1933289648Scem if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1934289546Scem /* Link is down with error, proceed with recovery */ 1935289648Scem callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1936289546Scem return; 1937250079Scarl } 1938250079Scarl 1939289542Scemout: 1940289648Scem callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1941250079Scarl} 1942250079Scarl 1943250079Scarlstatic void 1944289648Scematom_perform_link_restart(struct ntb_softc *ntb) 1945250079Scarl{ 1946250079Scarl uint32_t status; 1947250079Scarl 1948250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1949303429Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1950303429Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1951303429Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1952303429Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1953250079Scarl 1954250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1955250079Scarl pause("ModPhy", hz / 10); 1956250079Scarl 1957250079Scarl /* Clear AER Errors, write to clear */ 1958303429Smav status = intel_ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1959250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1960303429Smav intel_ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1961250079Scarl 1962250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1963303429Smav status = intel_ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1964289648Scem status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1965303429Smav intel_ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1966250079Scarl 1967250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1968303429Smav status = intel_ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1969289648Scem status |= ATOM_DESKEWSTS_DBERR; 1970303429Smav intel_ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1971250079Scarl 1972303429Smav status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1973289648Scem status &= ATOM_IBIST_ERR_OFLOW; 1974303429Smav intel_ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1975250079Scarl 1976250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1977303429Smav status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1978289648Scem status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1979303429Smav intel_ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1980250079Scarl} 1981250079Scarl 1982302484Smavstatic int 1983303429Smavintel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, 1984302484Smav enum ntb_width width __unused) 1985289546Scem{ 1986302484Smav struct ntb_softc *ntb = device_get_softc(dev); 1987289280Scem uint32_t cntl; 1988289272Scem 1989303429Smav intel_ntb_printf(2, "%s\n", __func__); 1990300100Scem 1991289648Scem if (ntb->type == NTB_ATOM) { 1992289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1993289648Scem ntb->ppd | ATOM_PPD_INIT_LINK, 4); 1994289546Scem return (0); 1995289542Scem } 1996289542Scem 1997289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1998302484Smav ntb_link_event(dev); 1999289546Scem return (0); 2000289280Scem } 2001289280Scem 2002303429Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2003289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2004289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2005289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2006302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2007289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2008303429Smav intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2009289546Scem return (0); 2010289272Scem} 2011289272Scem 2012302484Smavstatic int 2013303429Smavintel_ntb_link_disable(device_t dev) 2014289272Scem{ 2015302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2016289272Scem uint32_t cntl; 2017289272Scem 2018303429Smav intel_ntb_printf(2, "%s\n", __func__); 2019300100Scem 2020289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2021302484Smav ntb_link_event(dev); 2022289546Scem return (0); 2023289272Scem } 2024289272Scem 2025303429Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2026289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2027289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2028302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2029289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2030289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2031303429Smav intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2032289546Scem return (0); 2033289272Scem} 2034289272Scem 2035302484Smavstatic bool 2036303429Smavintel_ntb_link_enabled(device_t dev) 2037300100Scem{ 2038302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2039300100Scem uint32_t cntl; 2040300100Scem 2041300100Scem if (ntb->type == NTB_ATOM) { 2042300100Scem cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2043300100Scem return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2044300100Scem } 2045300100Scem 2046300100Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2047300100Scem return (true); 2048300100Scem 2049303429Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2050300100Scem return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2051300100Scem} 2052300100Scem 2053289272Scemstatic void 2054289648Scemrecover_atom_link(void *arg) 2055250079Scarl{ 2056250079Scarl struct ntb_softc *ntb = arg; 2057289608Scem unsigned speed, width, oldspeed, oldwidth; 2058250079Scarl uint32_t status32; 2059250079Scarl 2060289648Scem atom_perform_link_restart(ntb); 2061250079Scarl 2062289232Scem /* 2063289232Scem * There is a potential race between the 2 NTB devices recovering at 2064289232Scem * the same time. If the times are the same, the link will not recover 2065289232Scem * and the driver will be stuck in this loop forever. Add a random 2066289232Scem * interval to the recovery time to prevent this race. 2067289232Scem */ 2068289648Scem status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2069289648Scem pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2070289232Scem 2071289648Scem if (atom_link_is_err(ntb)) 2072250079Scarl goto retry; 2073250079Scarl 2074303429Smav status32 = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2075289648Scem if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2076289232Scem goto out; 2077289232Scem 2078303429Smav status32 = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2079289608Scem width = NTB_LNK_STA_WIDTH(status32); 2080289608Scem speed = status32 & NTB_LINK_SPEED_MASK; 2081289608Scem 2082289608Scem oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2083289608Scem oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2084289608Scem if (oldwidth != width || oldspeed != speed) 2085250079Scarl goto retry; 2086250079Scarl 2087289232Scemout: 2088289648Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2089289542Scem ntb); 2090250079Scarl return; 2091250079Scarl 2092250079Scarlretry: 2093289648Scem callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2094250079Scarl ntb); 2095250079Scarl} 2096250079Scarl 2097289546Scem/* 2098289546Scem * Polls the HW link status register(s); returns true if something has changed. 2099289546Scem */ 2100289546Scemstatic bool 2101303429Smavintel_ntb_poll_link(struct ntb_softc *ntb) 2102250079Scarl{ 2103250079Scarl uint32_t ntb_cntl; 2104289546Scem uint16_t reg_val; 2105250079Scarl 2106289648Scem if (ntb->type == NTB_ATOM) { 2107303429Smav ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2108289546Scem if (ntb_cntl == ntb->ntb_ctl) 2109289546Scem return (false); 2110289546Scem 2111289542Scem ntb->ntb_ctl = ntb_cntl; 2112303429Smav ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2113250079Scarl } else { 2114290678Scem db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2115250079Scarl 2116289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2117289546Scem if (reg_val == ntb->lnk_sta) 2118289546Scem return (false); 2119250079Scarl 2120289546Scem ntb->lnk_sta = reg_val; 2121295618Scem 2122302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2123295618Scem if (_xeon_link_is_up(ntb)) { 2124295618Scem if (!ntb->peer_msix_good) { 2125295618Scem callout_reset(&ntb->peer_msix_work, 0, 2126303429Smav intel_ntb_exchange_msix, ntb); 2127295618Scem return (false); 2128295618Scem } 2129295618Scem } else { 2130295618Scem ntb->peer_msix_good = false; 2131295618Scem ntb->peer_msix_done = false; 2132295618Scem } 2133295618Scem } 2134289542Scem } 2135289546Scem return (true); 2136289542Scem} 2137289542Scem 2138289546Scemstatic inline enum ntb_speed 2139303429Smavintel_ntb_link_sta_speed(struct ntb_softc *ntb) 2140250079Scarl{ 2141250079Scarl 2142289546Scem if (!link_is_up(ntb)) 2143289546Scem return (NTB_SPEED_NONE); 2144289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2145250079Scarl} 2146250079Scarl 2147289546Scemstatic inline enum ntb_width 2148303429Smavintel_ntb_link_sta_width(struct ntb_softc *ntb) 2149250079Scarl{ 2150250079Scarl 2151289546Scem if (!link_is_up(ntb)) 2152289546Scem return (NTB_WIDTH_NONE); 2153289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2154250079Scarl} 2155250079Scarl 2156289774ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2157289774Scem "Driver state, statistics, and HW registers"); 2158289774Scem 2159289774Scem#define NTB_REGSZ_MASK (3ul << 30) 2160289774Scem#define NTB_REG_64 (1ul << 30) 2161289774Scem#define NTB_REG_32 (2ul << 30) 2162289774Scem#define NTB_REG_16 (3ul << 30) 2163289774Scem#define NTB_REG_8 (0ul << 30) 2164289774Scem 2165289774Scem#define NTB_DB_READ (1ul << 29) 2166289774Scem#define NTB_PCI_REG (1ul << 28) 2167289774Scem#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2168289774Scem 2169289774Scemstatic void 2170303429Smavintel_ntb_sysctl_init(struct ntb_softc *ntb) 2171289774Scem{ 2172300100Scem struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2173289774Scem struct sysctl_ctx_list *ctx; 2174289774Scem struct sysctl_oid *tree, *tmptree; 2175289774Scem 2176289774Scem ctx = device_get_sysctl_ctx(ntb->device); 2177300100Scem globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2178289774Scem 2179300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2180300100Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2181300100Scem sysctl_handle_link_status_human, "A", 2182300100Scem "Link status (human readable)"); 2183300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2184300100Scem CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2185300100Scem "IU", "Link status (1=active, 0=inactive)"); 2186300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2187300100Scem CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2188300100Scem "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2189300100Scem 2190300100Scem tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2191300100Scem CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2192289774Scem tree_par = SYSCTL_CHILDREN(tree); 2193289774Scem 2194289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2195289774Scem &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2196289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2197289774Scem &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2198290687Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2199290687Scem &ntb->ppd, 0, "Raw PPD register (cached)"); 2200289774Scem 2201289774Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2202289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2203289774Scem &ntb->b2b_mw_idx, 0, 2204289774Scem "Index of the MW used for B2B remote register access"); 2205289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2206289774Scem CTLFLAG_RD, &ntb->b2b_off, 2207289774Scem "If non-zero, offset of B2B register region in shared MW"); 2208289774Scem } 2209289774Scem 2210289774Scem SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2211289774Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2212289774Scem "Features/errata of this NTB device"); 2213289774Scem 2214289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2215290686Scem __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2216290686Scem "NTB CTL register (cached)"); 2217289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2218290686Scem __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2219290686Scem "LNK STA register (cached)"); 2220289774Scem 2221289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2222291263Scem &ntb->mw_count, 0, "MW count"); 2223289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2224289774Scem &ntb->spad_count, 0, "Scratchpad count"); 2225289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2226289774Scem &ntb->db_count, 0, "Doorbell count"); 2227289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2228289774Scem &ntb->db_vec_count, 0, "Doorbell vector count"); 2229289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2230289774Scem &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2231289774Scem 2232289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2233289774Scem &ntb->db_valid_mask, "Doorbell valid mask"); 2234289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2235289774Scem &ntb->db_link_mask, "Doorbell link mask"); 2236289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2237289774Scem &ntb->db_mask, "Doorbell mask (cached)"); 2238289774Scem 2239289774Scem tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2240289774Scem CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2241289774Scem regpar = SYSCTL_CHILDREN(tmptree); 2242289774Scem 2243290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2244290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2245290682Scem ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2246290682Scem "NTB Control register"); 2247290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2248290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2249290682Scem 0x19c, sysctl_handle_register, "IU", 2250290682Scem "NTB Link Capabilities"); 2251290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2252290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2253290682Scem 0x1a0, sysctl_handle_register, "IU", 2254290682Scem "NTB Link Control register"); 2255290682Scem 2256289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2257289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2258289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2259289774Scem sysctl_handle_register, "QU", "Doorbell mask register"); 2260289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2261289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2262289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2263289774Scem sysctl_handle_register, "QU", "Doorbell register"); 2264289774Scem 2265289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2266289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2267289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2268289774Scem sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2269302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2270289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2271289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2272289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2273289774Scem sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2274289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2275289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2276289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2277289774Scem sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2278289774Scem } else { 2279289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2280289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2281289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2282289774Scem sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2283289774Scem } 2284289774Scem 2285289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2286289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2287289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2288289774Scem sysctl_handle_register, "QU", "Incoming LMT23 register"); 2289302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2290289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2291289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2292289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2293289774Scem sysctl_handle_register, "IU", "Incoming LMT4 register"); 2294289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2295289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2296289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2297289774Scem sysctl_handle_register, "IU", "Incoming LMT5 register"); 2298289774Scem } else { 2299289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2300289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2301289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2302289774Scem sysctl_handle_register, "QU", "Incoming LMT45 register"); 2303289774Scem } 2304289774Scem 2305289774Scem if (ntb->type == NTB_ATOM) 2306289774Scem return; 2307289774Scem 2308289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2309289774Scem CTLFLAG_RD, NULL, "Xeon HW statistics"); 2310289774Scem statpar = SYSCTL_CHILDREN(tmptree); 2311289774Scem SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2312289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2313289774Scem NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2314289774Scem sysctl_handle_register, "SU", "Upstream Memory Miss"); 2315289774Scem 2316289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2317289774Scem CTLFLAG_RD, NULL, "Xeon HW errors"); 2318289774Scem errpar = SYSCTL_CHILDREN(tmptree); 2319289774Scem 2320290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2321289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2322290687Scem NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2323290687Scem sysctl_handle_register, "CU", "PPD"); 2324290687Scem 2325290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2326290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2327290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2328290687Scem sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2329290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2330290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2331290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2332290687Scem sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2333290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2334290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2335290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2336290687Scem sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2337290687Scem 2338290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2339290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2340290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2341290687Scem sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2342290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2343290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2344290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2345290687Scem sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2346290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2347290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2348290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2349290687Scem sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2350290687Scem 2351290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2352290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2353289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2354289774Scem sysctl_handle_register, "SU", "DEVSTS"); 2355290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2356289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2357289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2358289774Scem sysctl_handle_register, "SU", "LNKSTS"); 2359290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2360290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2361290687Scem NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2362290687Scem sysctl_handle_register, "SU", "SLNKSTS"); 2363290687Scem 2364289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2365289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2366289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2367289774Scem sysctl_handle_register, "IU", "UNCERRSTS"); 2368289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2369289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2370289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2371289774Scem sysctl_handle_register, "IU", "CORERRSTS"); 2372289774Scem 2373289774Scem if (ntb->conn_type != NTB_CONN_B2B) 2374289774Scem return; 2375289774Scem 2376289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2377289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2378289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2379289774Scem sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2380302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2381289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2382289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2383289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2384289774Scem sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2385289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2386289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2387289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2388289774Scem sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2389289774Scem } else { 2390289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2391289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2392289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2393289774Scem sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2394289774Scem } 2395289774Scem 2396289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2397289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2398289774Scem NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2399289774Scem sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2400302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2401289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2402289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2403289774Scem NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2404289774Scem sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2405289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2406289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2407289774Scem NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2408289774Scem sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2409289774Scem } else { 2410289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2411289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2412289774Scem NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2413289774Scem sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2414289774Scem } 2415289774Scem 2416289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2417289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2418289774Scem NTB_REG_64 | ntb->xlat_reg->bar0_base, 2419289774Scem sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2420289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2421289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2422289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_base, 2423289774Scem sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2424302484Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2425289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2426289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2427289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_base, 2428289774Scem sysctl_handle_register, "IU", 2429289774Scem "Secondary BAR4 base register"); 2430289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2431289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2432289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_base, 2433289774Scem sysctl_handle_register, "IU", 2434289774Scem "Secondary BAR5 base register"); 2435289774Scem } else { 2436289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2437289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2438289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_base, 2439289774Scem sysctl_handle_register, "QU", 2440289774Scem "Secondary BAR45 base register"); 2441289774Scem } 2442289774Scem} 2443289774Scem 2444289774Scemstatic int 2445289774Scemsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2446289774Scem{ 2447302483Smav struct ntb_softc *ntb = arg1; 2448289774Scem struct sbuf sb; 2449289774Scem int error; 2450289774Scem 2451289774Scem sbuf_new_for_sysctl(&sb, NULL, 256, req); 2452289774Scem 2453289774Scem sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2454289774Scem error = sbuf_finish(&sb); 2455289774Scem sbuf_delete(&sb); 2456289774Scem 2457289774Scem if (error || !req->newptr) 2458289774Scem return (error); 2459289774Scem return (EINVAL); 2460289774Scem} 2461289774Scem 2462289774Scemstatic int 2463300100Scemsysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2464289774Scem{ 2465302483Smav struct ntb_softc *ntb = arg1; 2466300100Scem unsigned old, new; 2467300100Scem int error; 2468300100Scem 2469303429Smav old = intel_ntb_link_enabled(ntb->device); 2470300100Scem 2471300100Scem error = SYSCTL_OUT(req, &old, sizeof(old)); 2472300100Scem if (error != 0 || req->newptr == NULL) 2473300100Scem return (error); 2474300100Scem 2475300100Scem error = SYSCTL_IN(req, &new, sizeof(new)); 2476300100Scem if (error != 0) 2477300100Scem return (error); 2478300100Scem 2479303429Smav intel_ntb_printf(0, "Admin set interface state to '%sabled'\n", 2480300100Scem (new != 0)? "en" : "dis"); 2481300100Scem 2482300100Scem if (new != 0) 2483303429Smav error = intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2484300100Scem else 2485303429Smav error = intel_ntb_link_disable(ntb->device); 2486300100Scem return (error); 2487300100Scem} 2488300100Scem 2489300100Scemstatic int 2490300100Scemsysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2491300100Scem{ 2492302483Smav struct ntb_softc *ntb = arg1; 2493289774Scem struct sbuf sb; 2494289774Scem enum ntb_speed speed; 2495289774Scem enum ntb_width width; 2496289774Scem int error; 2497289774Scem 2498289774Scem sbuf_new_for_sysctl(&sb, NULL, 32, req); 2499289774Scem 2500303429Smav if (intel_ntb_link_is_up(ntb->device, &speed, &width)) 2501289774Scem sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2502289774Scem (unsigned)speed, (unsigned)width); 2503289774Scem else 2504289774Scem sbuf_printf(&sb, "down"); 2505289774Scem 2506289774Scem error = sbuf_finish(&sb); 2507289774Scem sbuf_delete(&sb); 2508289774Scem 2509289774Scem if (error || !req->newptr) 2510289774Scem return (error); 2511289774Scem return (EINVAL); 2512289774Scem} 2513289774Scem 2514289774Scemstatic int 2515300100Scemsysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2516300100Scem{ 2517302483Smav struct ntb_softc *ntb = arg1; 2518300100Scem unsigned res; 2519300100Scem int error; 2520300100Scem 2521303429Smav res = intel_ntb_link_is_up(ntb->device, NULL, NULL); 2522300100Scem 2523300100Scem error = SYSCTL_OUT(req, &res, sizeof(res)); 2524300100Scem if (error || !req->newptr) 2525300100Scem return (error); 2526300100Scem return (EINVAL); 2527300100Scem} 2528300100Scem 2529300100Scemstatic int 2530289774Scemsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2531289774Scem{ 2532289774Scem struct ntb_softc *ntb; 2533289774Scem const void *outp; 2534289774Scem uintptr_t sz; 2535289774Scem uint64_t umv; 2536289774Scem char be[sizeof(umv)]; 2537289774Scem size_t outsz; 2538289774Scem uint32_t reg; 2539289774Scem bool db, pci; 2540289774Scem int error; 2541289774Scem 2542289774Scem ntb = arg1; 2543289774Scem reg = arg2 & ~NTB_REGFLAGS_MASK; 2544289774Scem sz = arg2 & NTB_REGSZ_MASK; 2545289774Scem db = (arg2 & NTB_DB_READ) != 0; 2546289774Scem pci = (arg2 & NTB_PCI_REG) != 0; 2547289774Scem 2548289774Scem KASSERT(!(db && pci), ("bogus")); 2549289774Scem 2550289774Scem if (db) { 2551289774Scem KASSERT(sz == NTB_REG_64, ("bogus")); 2552289774Scem umv = db_ioread(ntb, reg); 2553289774Scem outsz = sizeof(uint64_t); 2554289774Scem } else { 2555289774Scem switch (sz) { 2556289774Scem case NTB_REG_64: 2557289774Scem if (pci) 2558289774Scem umv = pci_read_config(ntb->device, reg, 8); 2559289774Scem else 2560303429Smav umv = intel_ntb_reg_read(8, reg); 2561289774Scem outsz = sizeof(uint64_t); 2562289774Scem break; 2563289774Scem case NTB_REG_32: 2564289774Scem if (pci) 2565289774Scem umv = pci_read_config(ntb->device, reg, 4); 2566289774Scem else 2567303429Smav umv = intel_ntb_reg_read(4, reg); 2568289774Scem outsz = sizeof(uint32_t); 2569289774Scem break; 2570289774Scem case NTB_REG_16: 2571289774Scem if (pci) 2572289774Scem umv = pci_read_config(ntb->device, reg, 2); 2573289774Scem else 2574303429Smav umv = intel_ntb_reg_read(2, reg); 2575289774Scem outsz = sizeof(uint16_t); 2576289774Scem break; 2577289774Scem case NTB_REG_8: 2578289774Scem if (pci) 2579289774Scem umv = pci_read_config(ntb->device, reg, 1); 2580289774Scem else 2581303429Smav umv = intel_ntb_reg_read(1, reg); 2582289774Scem outsz = sizeof(uint8_t); 2583289774Scem break; 2584289774Scem default: 2585289774Scem panic("bogus"); 2586289774Scem break; 2587289774Scem } 2588289774Scem } 2589289774Scem 2590289774Scem /* Encode bigendian so that sysctl -x is legible. */ 2591289774Scem be64enc(be, umv); 2592289774Scem outp = ((char *)be) + sizeof(umv) - outsz; 2593289774Scem 2594289774Scem error = SYSCTL_OUT(req, outp, outsz); 2595289774Scem if (error || !req->newptr) 2596289774Scem return (error); 2597289774Scem return (EINVAL); 2598289774Scem} 2599289774Scem 2600291263Scemstatic unsigned 2601303429Smavintel_ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2602291263Scem{ 2603291263Scem 2604295618Scem if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2605295618Scem uidx >= ntb->b2b_mw_idx) || 2606295618Scem (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2607295618Scem uidx++; 2608295618Scem if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2609295618Scem uidx >= ntb->b2b_mw_idx) && 2610295618Scem (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2611295618Scem uidx++; 2612291263Scem return (uidx); 2613291263Scem} 2614291263Scem 2615303561Smav#ifndef EARLY_AP_STARTUP 2616303554Smavstatic int msix_ready; 2617303554Smav 2618295618Scemstatic void 2619303554Smavintel_ntb_msix_ready(void *arg __unused) 2620303554Smav{ 2621303554Smav 2622303554Smav msix_ready = 1; 2623303554Smav} 2624303554SmavSYSINIT(intel_ntb_msix_ready, SI_SUB_SMP, SI_ORDER_ANY, 2625303554Smav intel_ntb_msix_ready, NULL); 2626303561Smav#endif 2627303554Smav 2628303554Smavstatic void 2629303429Smavintel_ntb_exchange_msix(void *ctx) 2630295618Scem{ 2631295618Scem struct ntb_softc *ntb; 2632295618Scem uint32_t val; 2633295618Scem unsigned i; 2634295618Scem 2635295618Scem ntb = ctx; 2636295618Scem 2637301292Smav if (ntb->peer_msix_good) 2638301292Smav goto msix_good; 2639295618Scem if (ntb->peer_msix_done) 2640295618Scem goto msix_done; 2641295618Scem 2642303561Smav#ifndef EARLY_AP_STARTUP 2643303554Smav /* Block MSIX negotiation until SMP started and IRQ reshuffled. */ 2644303554Smav if (!msix_ready) 2645303554Smav goto reschedule; 2646303561Smav#endif 2647303554Smav 2648303429Smav intel_ntb_get_msix_info(ntb); 2649295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2650303429Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 2651295618Scem ntb->msix_data[i].nmd_data); 2652303429Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 2653301293Smav ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2654295618Scem } 2655303429Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2656295618Scem 2657303429Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 2658295618Scem if (val != NTB_MSIX_VER_GUARD) 2659295618Scem goto reschedule; 2660295618Scem 2661295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2662303429Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 2663303429Smav intel_ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2664295618Scem ntb->peer_msix_data[i].nmd_data = val; 2665303429Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 2666303429Smav intel_ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2667295618Scem ntb->peer_msix_data[i].nmd_ofs = val; 2668295618Scem } 2669295618Scem 2670295618Scem ntb->peer_msix_done = true; 2671295618Scem 2672295618Scemmsix_done: 2673303429Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2674303429Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 2675295618Scem if (val != NTB_MSIX_RECEIVED) 2676295618Scem goto reschedule; 2677295618Scem 2678303510Smav intel_ntb_spad_clear(ntb->device); 2679295618Scem ntb->peer_msix_good = true; 2680301292Smav /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2681301292Smav goto reschedule; 2682295618Scem 2683301292Smavmsix_good: 2684303429Smav intel_ntb_poll_link(ntb); 2685302484Smav ntb_link_event(ntb->device); 2686295618Scem return; 2687295618Scem 2688295618Scemreschedule: 2689295618Scem ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2690301292Smav if (_xeon_link_is_up(ntb)) { 2691301292Smav callout_reset(&ntb->peer_msix_work, 2692323456Smav hz * (ntb->peer_msix_good ? 2 : 1) / 10, 2693303429Smav intel_ntb_exchange_msix, ntb); 2694301292Smav } else 2695303429Smav intel_ntb_spad_clear(ntb->device); 2696295618Scem} 2697295618Scem 2698289546Scem/* 2699289546Scem * Public API to the rest of the OS 2700250079Scarl */ 2701250079Scarl 2702302484Smavstatic uint8_t 2703303429Smavintel_ntb_spad_count(device_t dev) 2704250079Scarl{ 2705302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2706250079Scarl 2707289539Scem return (ntb->spad_count); 2708250079Scarl} 2709250079Scarl 2710302484Smavstatic uint8_t 2711303429Smavintel_ntb_mw_count(device_t dev) 2712289396Scem{ 2713302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2714295618Scem uint8_t res; 2715289396Scem 2716295618Scem res = ntb->mw_count; 2717291263Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2718295618Scem res--; 2719295618Scem if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2720295618Scem res--; 2721295618Scem return (res); 2722289396Scem} 2723289396Scem 2724302484Smavstatic int 2725303429Smavintel_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 2726250079Scarl{ 2727302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2728250079Scarl 2729289539Scem if (idx >= ntb->spad_count) 2730250079Scarl return (EINVAL); 2731250079Scarl 2732303429Smav intel_ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2733250079Scarl 2734250079Scarl return (0); 2735250079Scarl} 2736250079Scarl 2737295618Scem/* 2738295618Scem * Zeros the local scratchpad. 2739295618Scem */ 2740302484Smavstatic void 2741303429Smavintel_ntb_spad_clear(device_t dev) 2742295618Scem{ 2743302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2744295618Scem unsigned i; 2745295618Scem 2746295618Scem for (i = 0; i < ntb->spad_count; i++) 2747303429Smav intel_ntb_spad_write(dev, i, 0); 2748295618Scem} 2749295618Scem 2750302484Smavstatic int 2751303429Smavintel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2752250079Scarl{ 2753302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2754250079Scarl 2755289539Scem if (idx >= ntb->spad_count) 2756250079Scarl return (EINVAL); 2757250079Scarl 2758303429Smav *val = intel_ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2759250079Scarl 2760250079Scarl return (0); 2761250079Scarl} 2762250079Scarl 2763302484Smavstatic int 2764303429Smavintel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 2765250079Scarl{ 2766302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2767250079Scarl 2768289539Scem if (idx >= ntb->spad_count) 2769250079Scarl return (EINVAL); 2770250079Scarl 2771302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2772303429Smav intel_ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2773255279Scarl else 2774303429Smav intel_ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2775250079Scarl 2776250079Scarl return (0); 2777250079Scarl} 2778250079Scarl 2779302484Smavstatic int 2780303429Smavintel_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2781250079Scarl{ 2782302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2783250079Scarl 2784289539Scem if (idx >= ntb->spad_count) 2785250079Scarl return (EINVAL); 2786250079Scarl 2787302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2788303429Smav *val = intel_ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2789255279Scarl else 2790303429Smav *val = intel_ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2791250079Scarl 2792250079Scarl return (0); 2793250079Scarl} 2794250079Scarl 2795302484Smavstatic int 2796303429Smavintel_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 2797291033Scem caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2798291033Scem bus_addr_t *plimit) 2799250079Scarl{ 2800302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2801289546Scem struct ntb_pci_bar_info *bar; 2802291033Scem bus_addr_t limit; 2803289546Scem size_t bar_b2b_off; 2804291033Scem enum ntb_bar bar_num; 2805250079Scarl 2806303429Smav if (mw_idx >= intel_ntb_mw_count(dev)) 2807289546Scem return (EINVAL); 2808303429Smav mw_idx = intel_ntb_user_mw_to_idx(ntb, mw_idx); 2809250079Scarl 2810303429Smav bar_num = intel_ntb_mw_to_bar(ntb, mw_idx); 2811291033Scem bar = &ntb->bar_info[bar_num]; 2812289546Scem bar_b2b_off = 0; 2813289546Scem if (mw_idx == ntb->b2b_mw_idx) { 2814289546Scem KASSERT(ntb->b2b_off != 0, 2815289546Scem ("user shouldn't get non-shared b2b mw")); 2816289546Scem bar_b2b_off = ntb->b2b_off; 2817289546Scem } 2818250079Scarl 2819291033Scem if (bar_is_64bit(ntb, bar_num)) 2820291033Scem limit = BUS_SPACE_MAXADDR; 2821291033Scem else 2822291033Scem limit = BUS_SPACE_MAXADDR_32BIT; 2823291033Scem 2824289546Scem if (base != NULL) 2825289546Scem *base = bar->pbase + bar_b2b_off; 2826289546Scem if (vbase != NULL) 2827290679Scem *vbase = bar->vbase + bar_b2b_off; 2828289546Scem if (size != NULL) 2829289546Scem *size = bar->size - bar_b2b_off; 2830289546Scem if (align != NULL) 2831289546Scem *align = bar->size; 2832289546Scem if (align_size != NULL) 2833289546Scem *align_size = 1; 2834291033Scem if (plimit != NULL) 2835291033Scem *plimit = limit; 2836289546Scem return (0); 2837250079Scarl} 2838250079Scarl 2839302484Smavstatic int 2840303429Smavintel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 2841250079Scarl{ 2842302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2843289546Scem struct ntb_pci_bar_info *bar; 2844289546Scem uint64_t base, limit, reg_val; 2845289546Scem size_t bar_size, mw_size; 2846289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2847289546Scem enum ntb_bar bar_num; 2848250079Scarl 2849303429Smav if (idx >= intel_ntb_mw_count(dev)) 2850289546Scem return (EINVAL); 2851303429Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2852250079Scarl 2853303429Smav bar_num = intel_ntb_mw_to_bar(ntb, idx); 2854289546Scem bar = &ntb->bar_info[bar_num]; 2855250079Scarl 2856289546Scem bar_size = bar->size; 2857289546Scem if (idx == ntb->b2b_mw_idx) 2858289546Scem mw_size = bar_size - ntb->b2b_off; 2859289546Scem else 2860289546Scem mw_size = bar_size; 2861250079Scarl 2862289546Scem /* Hardware requires that addr is aligned to bar size */ 2863289546Scem if ((addr & (bar_size - 1)) != 0) 2864289546Scem return (EINVAL); 2865250079Scarl 2866289546Scem if (size > mw_size) 2867289546Scem return (EINVAL); 2868289546Scem 2869289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2870289546Scem 2871289546Scem limit = 0; 2872289546Scem if (bar_is_64bit(ntb, bar_num)) { 2873303429Smav base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2874289546Scem 2875289546Scem if (limit_reg != 0 && size != mw_size) 2876289546Scem limit = base + size; 2877289546Scem 2878289546Scem /* Set and verify translation address */ 2879303429Smav intel_ntb_reg_write(8, xlat_reg, addr); 2880303429Smav reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2881289546Scem if (reg_val != addr) { 2882303429Smav intel_ntb_reg_write(8, xlat_reg, 0); 2883289546Scem return (EIO); 2884289546Scem } 2885289546Scem 2886289546Scem /* Set and verify the limit */ 2887303429Smav intel_ntb_reg_write(8, limit_reg, limit); 2888303429Smav reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 2889289546Scem if (reg_val != limit) { 2890303429Smav intel_ntb_reg_write(8, limit_reg, base); 2891303429Smav intel_ntb_reg_write(8, xlat_reg, 0); 2892289546Scem return (EIO); 2893289546Scem } 2894289546Scem } else { 2895289546Scem /* Configure 32-bit (split) BAR MW */ 2896289546Scem 2897291029Scem if ((addr & UINT32_MAX) != addr) 2898291033Scem return (ERANGE); 2899291029Scem if (((addr + size) & UINT32_MAX) != (addr + size)) 2900291033Scem return (ERANGE); 2901289546Scem 2902303429Smav base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 2903289546Scem 2904289546Scem if (limit_reg != 0 && size != mw_size) 2905289546Scem limit = base + size; 2906289546Scem 2907289546Scem /* Set and verify translation address */ 2908303429Smav intel_ntb_reg_write(4, xlat_reg, addr); 2909303429Smav reg_val = intel_ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 2910289546Scem if (reg_val != addr) { 2911303429Smav intel_ntb_reg_write(4, xlat_reg, 0); 2912289546Scem return (EIO); 2913289546Scem } 2914289546Scem 2915289546Scem /* Set and verify the limit */ 2916303429Smav intel_ntb_reg_write(4, limit_reg, limit); 2917303429Smav reg_val = intel_ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 2918289546Scem if (reg_val != limit) { 2919303429Smav intel_ntb_reg_write(4, limit_reg, base); 2920303429Smav intel_ntb_reg_write(4, xlat_reg, 0); 2921289546Scem return (EIO); 2922289546Scem } 2923250079Scarl } 2924289546Scem return (0); 2925250079Scarl} 2926250079Scarl 2927302484Smavstatic int 2928303429Smavintel_ntb_mw_clear_trans(device_t dev, unsigned mw_idx) 2929289596Scem{ 2930289596Scem 2931303429Smav return (intel_ntb_mw_set_trans(dev, mw_idx, 0, 0)); 2932289596Scem} 2933289596Scem 2934302484Smavstatic int 2935303429Smavintel_ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 2936291031Scem{ 2937302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2938291031Scem struct ntb_pci_bar_info *bar; 2939291031Scem 2940303429Smav if (idx >= intel_ntb_mw_count(dev)) 2941291031Scem return (EINVAL); 2942303429Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2943291031Scem 2944303429Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 2945291280Scem *mode = bar->map_mode; 2946291031Scem return (0); 2947291031Scem} 2948291031Scem 2949302484Smavstatic int 2950303429Smavintel_ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 2951291031Scem{ 2952302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2953291263Scem 2954303429Smav if (idx >= intel_ntb_mw_count(dev)) 2955291263Scem return (EINVAL); 2956291263Scem 2957303429Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2958303429Smav return (intel_ntb_mw_set_wc_internal(ntb, idx, mode)); 2959291263Scem} 2960291263Scem 2961291263Scemstatic int 2962303429Smavintel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 2963291263Scem{ 2964291031Scem struct ntb_pci_bar_info *bar; 2965291031Scem int rc; 2966291031Scem 2967303429Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 2968291280Scem if (bar->map_mode == mode) 2969291031Scem return (0); 2970291031Scem 2971291280Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 2972291031Scem if (rc == 0) 2973291280Scem bar->map_mode = mode; 2974291031Scem 2975291031Scem return (rc); 2976291031Scem} 2977291031Scem 2978302484Smavstatic void 2979303429Smavintel_ntb_peer_db_set(device_t dev, uint64_t bit) 2980250079Scarl{ 2981302484Smav struct ntb_softc *ntb = device_get_softc(dev); 2982250079Scarl 2983302484Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2984295618Scem struct ntb_pci_bar_info *lapic; 2985295618Scem unsigned i; 2986295618Scem 2987295618Scem lapic = ntb->peer_lapic_bar; 2988295618Scem 2989295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2990303429Smav if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0) 2991295618Scem bus_space_write_4(lapic->pci_bus_tag, 2992295618Scem lapic->pci_bus_handle, 2993295618Scem ntb->peer_msix_data[i].nmd_ofs, 2994295618Scem ntb->peer_msix_data[i].nmd_data); 2995295618Scem } 2996295618Scem return; 2997295618Scem } 2998295618Scem 2999302484Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3000303429Smav intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3001289347Scem return; 3002289209Scem } 3003289347Scem 3004289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3005250079Scarl} 3006250079Scarl 3007302484Smavstatic int 3008303429Smavintel_ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3009289542Scem{ 3010302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3011289542Scem struct ntb_pci_bar_info *bar; 3012289542Scem uint64_t regoff; 3013289542Scem 3014302484Smav KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3015289542Scem 3016302484Smav if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3017289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3018289542Scem regoff = ntb->peer_reg->db_bell; 3019289542Scem } else { 3020289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3021289543Scem ("invalid b2b idx")); 3022289542Scem 3023303429Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3024290682Scem regoff = XEON_PDOORBELL_OFFSET; 3025289542Scem } 3026289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3027289542Scem 3028289542Scem /* HACK: Specific to current x86 bus implementation. */ 3029302484Smav *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3030302484Smav *db_size = ntb->reg->db_size; 3031302484Smav return (0); 3032289542Scem} 3033289542Scem 3034302484Smavstatic uint64_t 3035303429Smavintel_ntb_db_valid_mask(device_t dev) 3036289597Scem{ 3037302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3038289597Scem 3039289597Scem return (ntb->db_valid_mask); 3040289597Scem} 3041289597Scem 3042302484Smavstatic int 3043303429Smavintel_ntb_db_vector_count(device_t dev) 3044289598Scem{ 3045302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3046289598Scem 3047302484Smav return (ntb->db_vec_count); 3048302484Smav} 3049302484Smav 3050302484Smavstatic uint64_t 3051303429Smavintel_ntb_db_vector_mask(device_t dev, uint32_t vector) 3052302484Smav{ 3053302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3054302484Smav 3055289598Scem if (vector > ntb->db_vec_count) 3056289598Scem return (0); 3057303429Smav return (ntb->db_valid_mask & intel_ntb_vec_mask(ntb, vector)); 3058289598Scem} 3059289598Scem 3060302484Smavstatic bool 3061303429Smavintel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3062250079Scarl{ 3063302484Smav struct ntb_softc *ntb = device_get_softc(dev); 3064250079Scarl 3065289546Scem if (speed != NULL) 3066303429Smav *speed = intel_ntb_link_sta_speed(ntb); 3067289546Scem if (width != NULL) 3068303429Smav *width = intel_ntb_link_sta_width(ntb); 3069289546Scem return (link_is_up(ntb)); 3070250079Scarl} 3071250079Scarl 3072255272Scarlstatic void 3073255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 3074250079Scarl{ 3075255272Scarl 3076289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3077289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3078289209Scem bar->pbase = rman_get_start(bar->pci_resource); 3079289209Scem bar->size = rman_get_size(bar->pci_resource); 3080289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 3081250079Scarl} 3082255268Scarl 3083302484Smavstatic device_method_t ntb_intel_methods[] = { 3084302484Smav /* Device interface */ 3085303429Smav DEVMETHOD(device_probe, intel_ntb_probe), 3086303429Smav DEVMETHOD(device_attach, intel_ntb_attach), 3087303429Smav DEVMETHOD(device_detach, intel_ntb_detach), 3088323455Smav /* Bus interface */ 3089323455Smav DEVMETHOD(bus_child_location_str, ntb_child_location_str), 3090323455Smav DEVMETHOD(bus_print_child, ntb_print_child), 3091355152Smav DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag), 3092302484Smav /* NTB interface */ 3093303429Smav DEVMETHOD(ntb_link_is_up, intel_ntb_link_is_up), 3094303429Smav DEVMETHOD(ntb_link_enable, intel_ntb_link_enable), 3095303429Smav DEVMETHOD(ntb_link_disable, intel_ntb_link_disable), 3096303429Smav DEVMETHOD(ntb_link_enabled, intel_ntb_link_enabled), 3097303429Smav DEVMETHOD(ntb_mw_count, intel_ntb_mw_count), 3098303429Smav DEVMETHOD(ntb_mw_get_range, intel_ntb_mw_get_range), 3099303429Smav DEVMETHOD(ntb_mw_set_trans, intel_ntb_mw_set_trans), 3100303429Smav DEVMETHOD(ntb_mw_clear_trans, intel_ntb_mw_clear_trans), 3101303429Smav DEVMETHOD(ntb_mw_get_wc, intel_ntb_mw_get_wc), 3102303429Smav DEVMETHOD(ntb_mw_set_wc, intel_ntb_mw_set_wc), 3103303429Smav DEVMETHOD(ntb_spad_count, intel_ntb_spad_count), 3104303429Smav DEVMETHOD(ntb_spad_clear, intel_ntb_spad_clear), 3105303429Smav DEVMETHOD(ntb_spad_write, intel_ntb_spad_write), 3106303429Smav DEVMETHOD(ntb_spad_read, intel_ntb_spad_read), 3107303429Smav DEVMETHOD(ntb_peer_spad_write, intel_ntb_peer_spad_write), 3108303429Smav DEVMETHOD(ntb_peer_spad_read, intel_ntb_peer_spad_read), 3109303429Smav DEVMETHOD(ntb_db_valid_mask, intel_ntb_db_valid_mask), 3110303429Smav DEVMETHOD(ntb_db_vector_count, intel_ntb_db_vector_count), 3111303429Smav DEVMETHOD(ntb_db_vector_mask, intel_ntb_db_vector_mask), 3112303429Smav DEVMETHOD(ntb_db_clear, intel_ntb_db_clear), 3113303429Smav DEVMETHOD(ntb_db_clear_mask, intel_ntb_db_clear_mask), 3114303429Smav DEVMETHOD(ntb_db_read, intel_ntb_db_read), 3115303429Smav DEVMETHOD(ntb_db_set_mask, intel_ntb_db_set_mask), 3116303429Smav DEVMETHOD(ntb_peer_db_addr, intel_ntb_peer_db_addr), 3117303429Smav DEVMETHOD(ntb_peer_db_set, intel_ntb_peer_db_set), 3118302484Smav DEVMETHOD_END 3119302484Smav}; 3120255268Scarl 3121302484Smavstatic DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3122302484Smav sizeof(struct ntb_softc)); 3123323032SmavDRIVER_MODULE(ntb_hw_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL); 3124323032SmavMODULE_DEPEND(ntb_hw_intel, ntb, 1, 1, 1); 3125323032SmavMODULE_VERSION(ntb_hw_intel, 1); 3126