ntb_hw_intel.c revision 301293
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3289542Scem * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 301293 2016-06-04 00:18:59Z mav $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35289774Scem#include <sys/endian.h> 36250079Scarl#include <sys/malloc.h> 37250079Scarl#include <sys/module.h> 38295618Scem#include <sys/mutex.h> 39295618Scem#include <sys/pciio.h> 40250079Scarl#include <sys/queue.h> 41250079Scarl#include <sys/rman.h> 42289774Scem#include <sys/sbuf.h> 43289207Scem#include <sys/sysctl.h> 44250079Scarl#include <vm/vm.h> 45250079Scarl#include <vm/pmap.h> 46250079Scarl#include <machine/bus.h> 47295618Scem#include <machine/intr_machdep.h> 48250079Scarl#include <machine/resource.h> 49250079Scarl#include <dev/pci/pcireg.h> 50250079Scarl#include <dev/pci/pcivar.h> 51250079Scarl 52250079Scarl#include "ntb_regs.h" 53250079Scarl#include "ntb_hw.h" 54250079Scarl 55250079Scarl/* 56250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 57250079Scarl * allows you to connect two systems using a PCI-e link. 58250079Scarl * 59250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 60298955Spfg * you to send and receive interrupts, map the memory windows and send and 61250079Scarl * receive messages in the scratch-pad registers. 62250079Scarl * 63250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 64250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 65250079Scarl */ 66250079Scarl 67289648Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 68250079Scarl 69289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 70289648Scem#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 71291032Scem#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 72250079Scarl 73250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74250079Scarl 75295618Scem#define NTB_MSIX_VER_GUARD 0xaabbccdd 76295618Scem#define NTB_MSIX_RECEIVED 0xe0f0e0f0 77295618Scem 78295618Scem/* 79295618Scem * PCI constants could be somewhere more generic, but aren't defined/used in 80295618Scem * pci.c. 81295618Scem */ 82295618Scem#define PCI_MSIX_ENTRY_SIZE 16 83295618Scem#define PCI_MSIX_ENTRY_LOWER_ADDR 0 84295618Scem#define PCI_MSIX_ENTRY_UPPER_ADDR 4 85295618Scem#define PCI_MSIX_ENTRY_DATA 8 86295618Scem 87250079Scarlenum ntb_device_type { 88250079Scarl NTB_XEON, 89289648Scem NTB_ATOM 90250079Scarl}; 91250079Scarl 92289610Scem/* ntb_conn_type are hardware numbers, cannot change. */ 93289610Scemenum ntb_conn_type { 94289610Scem NTB_CONN_TRANSPARENT = 0, 95289610Scem NTB_CONN_B2B = 1, 96289610Scem NTB_CONN_RP = 2, 97289610Scem}; 98289610Scem 99289610Scemenum ntb_b2b_direction { 100289610Scem NTB_DEV_USD = 0, 101289610Scem NTB_DEV_DSD = 1, 102289610Scem}; 103289610Scem 104289539Scemenum ntb_bar { 105289539Scem NTB_CONFIG_BAR = 0, 106289539Scem NTB_B2B_BAR_1, 107289539Scem NTB_B2B_BAR_2, 108289539Scem NTB_B2B_BAR_3, 109289539Scem NTB_MAX_BARS 110289539Scem}; 111289539Scem 112295618Scemenum { 113295618Scem NTB_MSIX_GUARD = 0, 114295618Scem NTB_MSIX_DATA0, 115295618Scem NTB_MSIX_DATA1, 116295618Scem NTB_MSIX_DATA2, 117295618Scem NTB_MSIX_OFS0, 118295618Scem NTB_MSIX_OFS1, 119295618Scem NTB_MSIX_OFS2, 120295618Scem NTB_MSIX_DONE, 121295618Scem NTB_MAX_MSIX_SPAD 122295618Scem}; 123295618Scem 124255274Scarl/* Device features and workarounds */ 125255274Scarl#define HAS_FEATURE(feature) \ 126255274Scarl ((ntb->features & (feature)) != 0) 127255274Scarl 128250079Scarlstruct ntb_hw_info { 129250079Scarl uint32_t device_id; 130255274Scarl const char *desc; 131250079Scarl enum ntb_device_type type; 132289397Scem uint32_t features; 133250079Scarl}; 134250079Scarl 135250079Scarlstruct ntb_pci_bar_info { 136250079Scarl bus_space_tag_t pci_bus_tag; 137250079Scarl bus_space_handle_t pci_bus_handle; 138250079Scarl int pci_resource_id; 139250079Scarl struct resource *pci_resource; 140250079Scarl vm_paddr_t pbase; 141290679Scem caddr_t vbase; 142290679Scem vm_size_t size; 143291280Scem vm_memattr_t map_mode; 144289543Scem 145289543Scem /* Configuration register offsets */ 146289543Scem uint32_t psz_off; 147289543Scem uint32_t ssz_off; 148289543Scem uint32_t pbarxlat_off; 149250079Scarl}; 150250079Scarl 151250079Scarlstruct ntb_int_info { 152250079Scarl struct resource *res; 153250079Scarl int rid; 154250079Scarl void *tag; 155250079Scarl}; 156250079Scarl 157289546Scemstruct ntb_vec { 158250079Scarl struct ntb_softc *ntb; 159289546Scem uint32_t num; 160295618Scem unsigned masked; 161250079Scarl}; 162250079Scarl 163289542Scemstruct ntb_reg { 164289542Scem uint32_t ntb_ctl; 165289542Scem uint32_t lnk_sta; 166289542Scem uint8_t db_size; 167289542Scem unsigned mw_bar[NTB_MAX_BARS]; 168289542Scem}; 169289542Scem 170289542Scemstruct ntb_alt_reg { 171289542Scem uint32_t db_bell; 172289542Scem uint32_t db_mask; 173289542Scem uint32_t spad; 174289542Scem}; 175289542Scem 176289542Scemstruct ntb_xlat_reg { 177289546Scem uint32_t bar0_base; 178289546Scem uint32_t bar2_base; 179289546Scem uint32_t bar4_base; 180289546Scem uint32_t bar5_base; 181289546Scem 182289546Scem uint32_t bar2_xlat; 183289546Scem uint32_t bar4_xlat; 184289546Scem uint32_t bar5_xlat; 185289546Scem 186289546Scem uint32_t bar2_limit; 187289546Scem uint32_t bar4_limit; 188289546Scem uint32_t bar5_limit; 189289542Scem}; 190289542Scem 191289542Scemstruct ntb_b2b_addr { 192289542Scem uint64_t bar0_addr; 193289542Scem uint64_t bar2_addr64; 194289542Scem uint64_t bar4_addr64; 195289542Scem uint64_t bar4_addr32; 196289542Scem uint64_t bar5_addr32; 197289542Scem}; 198289542Scem 199295618Scemstruct ntb_msix_data { 200295618Scem uint32_t nmd_ofs; 201295618Scem uint32_t nmd_data; 202295618Scem}; 203295618Scem 204250079Scarlstruct ntb_softc { 205250079Scarl device_t device; 206250079Scarl enum ntb_device_type type; 207289774Scem uint32_t features; 208250079Scarl 209250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 210250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 211250079Scarl uint32_t allocated_interrupts; 212250079Scarl 213295618Scem struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 214295618Scem struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215295618Scem bool peer_msix_good; 216295618Scem bool peer_msix_done; 217295618Scem struct ntb_pci_bar_info *peer_lapic_bar; 218295618Scem struct callout peer_msix_work; 219295618Scem 220250079Scarl struct callout heartbeat_timer; 221250079Scarl struct callout lr_timer; 222250079Scarl 223289546Scem void *ntb_ctx; 224289546Scem const struct ntb_ctx_ops *ctx_ops; 225289546Scem struct ntb_vec *msix_vec; 226290683Scem#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 227290683Scem#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 228289546Scem#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 229289546Scem struct mtx ctx_lock; 230250079Scarl 231289610Scem uint32_t ppd; 232289610Scem enum ntb_conn_type conn_type; 233289610Scem enum ntb_b2b_direction dev_type; 234289539Scem 235289542Scem /* Offset of peer bar0 in B2B BAR */ 236289542Scem uint64_t b2b_off; 237289542Scem /* Memory window used to access peer bar0 */ 238289543Scem#define B2B_MW_DISABLED UINT8_MAX 239289542Scem uint8_t b2b_mw_idx; 240301293Smav uint32_t msix_xlat; 241295618Scem uint8_t msix_mw_idx; 242289542Scem 243289539Scem uint8_t mw_count; 244289539Scem uint8_t spad_count; 245289539Scem uint8_t db_count; 246289539Scem uint8_t db_vec_count; 247289539Scem uint8_t db_vec_shift; 248289542Scem 249289546Scem /* Protects local db_mask. */ 250289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 251289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 252289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 253289542Scem struct mtx db_mask_lock; 254289542Scem 255290686Scem volatile uint32_t ntb_ctl; 256290686Scem volatile uint32_t lnk_sta; 257289542Scem 258289542Scem uint64_t db_valid_mask; 259289542Scem uint64_t db_link_mask; 260289546Scem uint64_t db_mask; 261289542Scem 262289542Scem int last_ts; /* ticks @ last irq */ 263289542Scem 264289542Scem const struct ntb_reg *reg; 265289542Scem const struct ntb_alt_reg *self_reg; 266289542Scem const struct ntb_alt_reg *peer_reg; 267289542Scem const struct ntb_xlat_reg *xlat_reg; 268250079Scarl}; 269250079Scarl 270289234Scem#ifdef __i386__ 271289234Scemstatic __inline uint64_t 272289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 273289234Scem bus_size_t offset) 274289234Scem{ 275289234Scem 276289234Scem return (bus_space_read_4(tag, handle, offset) | 277289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 278289234Scem} 279289234Scem 280289234Scemstatic __inline void 281289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 282289234Scem bus_size_t offset, uint64_t val) 283289234Scem{ 284289234Scem 285289234Scem bus_space_write_4(tag, handle, offset, val); 286289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 287289234Scem} 288289234Scem#endif 289289234Scem 290255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 291255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 292255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 293255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 294255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 295255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 296255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 297250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 298255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 299289397Scem#define ntb_mw_read(SIZE, offset) \ 300289542Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 301255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 302289542Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 303289397Scem offset, val) 304250079Scarl 305250079Scarlstatic int ntb_probe(device_t device); 306250079Scarlstatic int ntb_attach(device_t device); 307250079Scarlstatic int ntb_detach(device_t device); 308291263Scemstatic unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 309289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 310289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 311289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 312289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 313255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 314291280Scemstatic int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 315291280Scem vm_memattr_t); 316289647Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 317289647Scem const char *); 318255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 319255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 320255272Scarl struct ntb_pci_bar_info *bar); 321250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 322289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 323289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 324289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 325289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 326250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 327289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 328289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 329289546Scemstatic void ndev_vec_isr(void *arg); 330289546Scemstatic void ndev_irq_isr(void *arg); 331289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 332290678Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 333290678Scemstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 334289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 335289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 336300531Scemstatic void ntb_get_msix_info(struct ntb_softc *ntb); 337295618Scemstatic void ntb_exchange_msix(void *); 338250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 339289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 340289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 341289648Scemstatic int ntb_detect_atom(struct ntb_softc *ntb); 342289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 343289648Scemstatic int ntb_atom_init_dev(struct ntb_softc *ntb); 344289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 345289648Scemstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 346289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 347289543Scem enum ntb_bar regbar); 348289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 349289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 350289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 351289543Scem enum ntb_bar idx); 352289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 353289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 354295618Scemstatic int xeon_setup_msix_bar(struct ntb_softc *); 355289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 356295618Scemstatic inline bool _xeon_link_is_up(struct ntb_softc *ntb); 357289648Scemstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 358289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 359289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 360289648Scemstatic void atom_link_hb(void *arg); 361289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 362289648Scemstatic void recover_atom_link(void *arg); 363289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb); 364255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 365289774Scemstatic void ntb_sysctl_init(struct ntb_softc *); 366289774Scemstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 367300100Scemstatic int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 368300100Scemstatic int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 369289774Scemstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 370289774Scemstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 371250079Scarl 372290685Scemstatic unsigned g_ntb_hw_debug_level; 373290685ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 374290685Scem &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 375290685Scem#define ntb_printf(lvl, ...) do { \ 376290685Scem if ((lvl) <= g_ntb_hw_debug_level) { \ 377290685Scem device_printf(ntb->device, __VA_ARGS__); \ 378290685Scem } \ 379290685Scem} while (0) 380290685Scem 381295486Scem#define _NTB_PAT_UC 0 382295486Scem#define _NTB_PAT_WC 1 383295486Scem#define _NTB_PAT_WT 4 384295486Scem#define _NTB_PAT_WP 5 385295486Scem#define _NTB_PAT_WB 6 386295486Scem#define _NTB_PAT_UCM 7 387295486Scemstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 388295486ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 389295486Scem &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 390295486Scem "UC: " __XSTRING(_NTB_PAT_UC) ", " 391295486Scem "WC: " __XSTRING(_NTB_PAT_WC) ", " 392295486Scem "WT: " __XSTRING(_NTB_PAT_WT) ", " 393295486Scem "WP: " __XSTRING(_NTB_PAT_WP) ", " 394295486Scem "WB: " __XSTRING(_NTB_PAT_WB) ", " 395295486Scem "UC-: " __XSTRING(_NTB_PAT_UCM)); 396291030Scem 397295486Scemstatic inline vm_memattr_t 398295486Scemntb_pat_flags(void) 399295486Scem{ 400295486Scem 401295486Scem switch (g_ntb_mw_pat) { 402295486Scem case _NTB_PAT_WC: 403295486Scem return (VM_MEMATTR_WRITE_COMBINING); 404295486Scem case _NTB_PAT_WT: 405295486Scem return (VM_MEMATTR_WRITE_THROUGH); 406295486Scem case _NTB_PAT_WP: 407295486Scem return (VM_MEMATTR_WRITE_PROTECTED); 408295486Scem case _NTB_PAT_WB: 409295486Scem return (VM_MEMATTR_WRITE_BACK); 410295486Scem case _NTB_PAT_UCM: 411295486Scem return (VM_MEMATTR_WEAK_UNCACHEABLE); 412295486Scem case _NTB_PAT_UC: 413295486Scem /* FALLTHROUGH */ 414295486Scem default: 415295486Scem return (VM_MEMATTR_UNCACHEABLE); 416295486Scem } 417295486Scem} 418295486Scem 419295487Scem/* 420295487Scem * Well, this obviously doesn't belong here, but it doesn't seem to exist 421295487Scem * anywhere better yet. 422295487Scem */ 423295487Scemstatic inline const char * 424295487Scemntb_vm_memattr_to_str(vm_memattr_t pat) 425295487Scem{ 426295487Scem 427295487Scem switch (pat) { 428295487Scem case VM_MEMATTR_WRITE_COMBINING: 429295487Scem return ("WRITE_COMBINING"); 430295487Scem case VM_MEMATTR_WRITE_THROUGH: 431295487Scem return ("WRITE_THROUGH"); 432295487Scem case VM_MEMATTR_WRITE_PROTECTED: 433295487Scem return ("WRITE_PROTECTED"); 434295487Scem case VM_MEMATTR_WRITE_BACK: 435295487Scem return ("WRITE_BACK"); 436295487Scem case VM_MEMATTR_WEAK_UNCACHEABLE: 437295487Scem return ("UNCACHED"); 438295487Scem case VM_MEMATTR_UNCACHEABLE: 439295487Scem return ("UNCACHEABLE"); 440295487Scem default: 441295487Scem return ("UNKNOWN"); 442295487Scem } 443295487Scem} 444295487Scem 445295618Scemstatic int g_ntb_msix_idx = 0; 446295618ScemSYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 447295618Scem 0, "Use this memory window to access the peer MSIX message complex on " 448295618Scem "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 449295618Scem "Like b2b_mw_idx, negative values index from the last available memory " 450295618Scem "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 451295618Scem 452291263Scemstatic int g_ntb_mw_idx = -1; 453291263ScemSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 454291263Scem 0, "Use this memory window to access the peer NTB registers. A " 455291263Scem "non-negative value starts from the first MW index; a negative value " 456291263Scem "starts from the last MW index. The default is -1, i.e., the last " 457291263Scem "available memory window. Both sides of the NTB MUST set the same " 458291263Scem "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 459291263Scem 460250079Scarlstatic struct ntb_hw_info pci_ids[] = { 461289612Scem /* XXX: PS/SS IDs left out until they are supported. */ 462289612Scem { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 463289648Scem NTB_ATOM, 0 }, 464289233Scem 465289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 466289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 467289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 468289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 469289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 470289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 471289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 472289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 473289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 474289538Scem NTB_SB01BASE_LOCKUP }, 475289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 476289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 477289538Scem NTB_SB01BASE_LOCKUP }, 478289233Scem 479289648Scem { 0x00000000, NULL, NTB_ATOM, 0 } 480250079Scarl}; 481250079Scarl 482289648Scemstatic const struct ntb_reg atom_reg = { 483289648Scem .ntb_ctl = ATOM_NTBCNTL_OFFSET, 484289648Scem .lnk_sta = ATOM_LINK_STATUS_OFFSET, 485289542Scem .db_size = sizeof(uint64_t), 486289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 487289542Scem}; 488289542Scem 489289648Scemstatic const struct ntb_alt_reg atom_pri_reg = { 490289648Scem .db_bell = ATOM_PDOORBELL_OFFSET, 491289648Scem .db_mask = ATOM_PDBMSK_OFFSET, 492289648Scem .spad = ATOM_SPAD_OFFSET, 493289607Scem}; 494289607Scem 495289648Scemstatic const struct ntb_alt_reg atom_b2b_reg = { 496289648Scem .db_bell = ATOM_B2B_DOORBELL_OFFSET, 497289648Scem .spad = ATOM_B2B_SPAD_OFFSET, 498289542Scem}; 499289542Scem 500289648Scemstatic const struct ntb_xlat_reg atom_sec_xlat = { 501289542Scem#if 0 502289542Scem /* "FIXME" says the Linux driver. */ 503289648Scem .bar0_base = ATOM_SBAR0BASE_OFFSET, 504289648Scem .bar2_base = ATOM_SBAR2BASE_OFFSET, 505289648Scem .bar4_base = ATOM_SBAR4BASE_OFFSET, 506289546Scem 507289648Scem .bar2_limit = ATOM_SBAR2LMT_OFFSET, 508289648Scem .bar4_limit = ATOM_SBAR4LMT_OFFSET, 509289542Scem#endif 510289546Scem 511289648Scem .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 512289648Scem .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 513289542Scem}; 514289542Scem 515289542Scemstatic const struct ntb_reg xeon_reg = { 516289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 517289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 518289542Scem .db_size = sizeof(uint16_t), 519289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 520289542Scem}; 521289542Scem 522289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 523289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 524289607Scem .db_mask = XEON_PDBMSK_OFFSET, 525289607Scem .spad = XEON_SPAD_OFFSET, 526289607Scem}; 527289607Scem 528289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 529289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 530289542Scem .spad = XEON_B2B_SPAD_OFFSET, 531289542Scem}; 532289542Scem 533289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 534289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 535289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 536289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 537289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 538289546Scem 539289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 540289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 541289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 542289546Scem 543289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 544289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 545289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 546289542Scem}; 547289542Scem 548289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 549290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 550290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 551290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 552290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 553290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 554289542Scem}; 555289542Scem 556289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 557290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 558290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 559290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 560290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 561290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 562289542Scem}; 563289542Scem 564289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 565289614Scem "B2B MW segment overrides -- MUST be the same on both sides"); 566289614Scem 567289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 568289614Scem &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 569289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 570289614Scem "the window at BAR2, on the upstream side of the link. MUST be the same " 571289614Scem "address on both sides."); 572289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 573289614Scem &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 574289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 575289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 576289614Scem "(split-BAR mode)."); 577289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 578289646Scem &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 579289614Scem "(split-BAR mode)."); 580289614Scem 581289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 582289614Scem &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 583289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 584289614Scem "the window at BAR2, on the downstream side of the link. MUST be the same" 585289614Scem " address on both sides."); 586289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 587289614Scem &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 588289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 589289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 590289614Scem "(split-BAR mode)."); 591289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 592289646Scem &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 593289614Scem "(split-BAR mode)."); 594289614Scem 595250079Scarl/* 596250079Scarl * OS <-> Driver interface structures 597250079Scarl */ 598250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 599250079Scarl 600250079Scarlstatic device_method_t ntb_pci_methods[] = { 601250079Scarl /* Device interface */ 602250079Scarl DEVMETHOD(device_probe, ntb_probe), 603250079Scarl DEVMETHOD(device_attach, ntb_attach), 604250079Scarl DEVMETHOD(device_detach, ntb_detach), 605250079Scarl DEVMETHOD_END 606250079Scarl}; 607250079Scarl 608250079Scarlstatic driver_t ntb_pci_driver = { 609250079Scarl "ntb_hw", 610250079Scarl ntb_pci_methods, 611250079Scarl sizeof(struct ntb_softc), 612250079Scarl}; 613250079Scarl 614250079Scarlstatic devclass_t ntb_devclass; 615250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 616250079ScarlMODULE_VERSION(ntb_hw, 1); 617250079Scarl 618289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 619289207Scem 620250079Scarl/* 621250079Scarl * OS <-> Driver linkage functions 622250079Scarl */ 623250079Scarlstatic int 624250079Scarlntb_probe(device_t device) 625250079Scarl{ 626289209Scem struct ntb_hw_info *p; 627250079Scarl 628289209Scem p = ntb_get_device_info(pci_get_devid(device)); 629289209Scem if (p == NULL) 630250079Scarl return (ENXIO); 631289209Scem 632289209Scem device_set_desc(device, p->desc); 633289209Scem return (0); 634250079Scarl} 635250079Scarl 636250079Scarlstatic int 637250079Scarlntb_attach(device_t device) 638250079Scarl{ 639289209Scem struct ntb_softc *ntb; 640289209Scem struct ntb_hw_info *p; 641250079Scarl int error; 642250079Scarl 643289209Scem ntb = DEVICE2SOFTC(device); 644289209Scem p = ntb_get_device_info(pci_get_devid(device)); 645289209Scem 646250079Scarl ntb->device = device; 647250079Scarl ntb->type = p->type; 648255274Scarl ntb->features = p->features; 649289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 650295618Scem ntb->msix_mw_idx = B2B_MW_DISABLED; 651250079Scarl 652289648Scem /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 653283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 654283291Sjkim callout_init(&ntb->lr_timer, 1); 655295618Scem callout_init(&ntb->peer_msix_work, 1); 656289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 657290683Scem mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 658250079Scarl 659289648Scem if (ntb->type == NTB_ATOM) 660289648Scem error = ntb_detect_atom(ntb); 661289348Scem else 662289348Scem error = ntb_detect_xeon(ntb); 663290682Scem if (error != 0) 664289348Scem goto out; 665289348Scem 666289397Scem ntb_detect_max_mw(ntb); 667289396Scem 668290682Scem pci_enable_busmaster(ntb->device); 669290682Scem 670289209Scem error = ntb_map_pci_bars(ntb); 671290682Scem if (error != 0) 672289209Scem goto out; 673289648Scem if (ntb->type == NTB_ATOM) 674289648Scem error = ntb_atom_init_dev(ntb); 675289272Scem else 676289542Scem error = ntb_xeon_init_dev(ntb); 677290682Scem if (error != 0) 678289209Scem goto out; 679290682Scem 680295618Scem ntb_spad_clear(ntb); 681295618Scem 682290682Scem ntb_poll_link(ntb); 683290682Scem 684289774Scem ntb_sysctl_init(ntb); 685250079Scarl 686289209Scemout: 687289209Scem if (error != 0) 688289209Scem ntb_detach(device); 689250079Scarl return (error); 690250079Scarl} 691250079Scarl 692250079Scarlstatic int 693250079Scarlntb_detach(device_t device) 694250079Scarl{ 695289209Scem struct ntb_softc *ntb; 696250079Scarl 697289209Scem ntb = DEVICE2SOFTC(device); 698289542Scem 699295618Scem if (ntb->self_reg != NULL) { 700295618Scem DB_MASK_LOCK(ntb); 701295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 702295618Scem DB_MASK_UNLOCK(ntb); 703295618Scem } 704250079Scarl callout_drain(&ntb->heartbeat_timer); 705250079Scarl callout_drain(&ntb->lr_timer); 706295618Scem callout_drain(&ntb->peer_msix_work); 707290682Scem pci_disable_busmaster(ntb->device); 708289272Scem if (ntb->type == NTB_XEON) 709289272Scem ntb_teardown_xeon(ntb); 710250079Scarl ntb_teardown_interrupts(ntb); 711289397Scem 712289542Scem mtx_destroy(&ntb->db_mask_lock); 713289546Scem mtx_destroy(&ntb->ctx_lock); 714289542Scem 715250079Scarl ntb_unmap_pci_bar(ntb); 716250079Scarl 717250079Scarl return (0); 718250079Scarl} 719250079Scarl 720289542Scem/* 721289542Scem * Driver internal routines 722289542Scem */ 723289539Scemstatic inline enum ntb_bar 724289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 725289539Scem{ 726289539Scem 727291263Scem KASSERT(mw < ntb->mw_count, 728289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 729289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 730289539Scem 731289542Scem return (ntb->reg->mw_bar[mw]); 732289539Scem} 733289539Scem 734289546Scemstatic inline bool 735289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 736289546Scem{ 737289546Scem /* XXX This assertion could be stronger. */ 738289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 739289546Scem return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 740289546Scem} 741289546Scem 742289546Scemstatic inline void 743289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 744289546Scem uint32_t *xlat, uint32_t *lmt) 745289546Scem{ 746289546Scem uint32_t basev, lmtv, xlatv; 747289546Scem 748289546Scem switch (bar) { 749289546Scem case NTB_B2B_BAR_1: 750289546Scem basev = ntb->xlat_reg->bar2_base; 751289546Scem lmtv = ntb->xlat_reg->bar2_limit; 752289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 753289546Scem break; 754289546Scem case NTB_B2B_BAR_2: 755289546Scem basev = ntb->xlat_reg->bar4_base; 756289546Scem lmtv = ntb->xlat_reg->bar4_limit; 757289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 758289546Scem break; 759289546Scem case NTB_B2B_BAR_3: 760289546Scem basev = ntb->xlat_reg->bar5_base; 761289546Scem lmtv = ntb->xlat_reg->bar5_limit; 762289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 763289546Scem break; 764289546Scem default: 765289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 766289546Scem ("bad bar")); 767289546Scem basev = lmtv = xlatv = 0; 768289546Scem break; 769289546Scem } 770289546Scem 771289546Scem if (base != NULL) 772289546Scem *base = basev; 773289546Scem if (xlat != NULL) 774289546Scem *xlat = xlatv; 775289546Scem if (lmt != NULL) 776289546Scem *lmt = lmtv; 777289546Scem} 778289546Scem 779250079Scarlstatic int 780255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 781250079Scarl{ 782255272Scarl int rc; 783250079Scarl 784250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 785289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 786255272Scarl if (rc != 0) 787289541Scem goto out; 788255272Scarl 789289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 790289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 791255272Scarl if (rc != 0) 792289541Scem goto out; 793289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 794289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 795289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 796255272Scarl 797289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 798291263Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 799291263Scem if (rc != 0) 800291263Scem goto out; 801289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 802289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 803289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 804289543Scem 805289397Scem if (!HAS_FEATURE(NTB_SPLIT_BAR)) 806289541Scem goto out; 807289397Scem 808289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 809291263Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 810289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 811289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 812289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 813250079Scarl 814289541Scemout: 815289209Scem if (rc != 0) 816255272Scarl device_printf(ntb->device, 817255272Scarl "unable to allocate pci resource\n"); 818255272Scarl return (rc); 819255272Scarl} 820255272Scarl 821289541Scemstatic void 822289647Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 823289647Scem const char *kind) 824289541Scem{ 825289541Scem 826289647Scem device_printf(ntb->device, 827289647Scem "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 828289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 829289647Scem (char *)bar->vbase + bar->size - 1, 830289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 831289647Scem (uintmax_t)bar->size, kind); 832289541Scem} 833289541Scem 834255272Scarlstatic int 835255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 836255272Scarl{ 837255272Scarl 838255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 839289209Scem &bar->pci_resource_id, RF_ACTIVE); 840255272Scarl if (bar->pci_resource == NULL) 841255272Scarl return (ENXIO); 842289209Scem 843289209Scem save_bar_parameters(bar); 844291280Scem bar->map_mode = VM_MEMATTR_UNCACHEABLE; 845289647Scem print_map_success(ntb, bar, "mmr"); 846289209Scem return (0); 847255272Scarl} 848255272Scarl 849255272Scarlstatic int 850255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 851255272Scarl{ 852255272Scarl int rc; 853291280Scem vm_memattr_t mapmode; 854255276Scarl uint8_t bar_size_bits = 0; 855255272Scarl 856289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 857289209Scem &bar->pci_resource_id, RF_ACTIVE); 858250079Scarl 859255272Scarl if (bar->pci_resource == NULL) 860255272Scarl return (ENXIO); 861255276Scarl 862289209Scem save_bar_parameters(bar); 863289209Scem /* 864289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 865289209Scem * hardware issue. To work around this, query the size it should be 866289209Scem * configured to by the device and modify the resource to correspond to 867289209Scem * this new size. The BIOS on systems with this problem is required to 868289209Scem * provide enough address space to allow the driver to make this change 869289209Scem * safely. 870289209Scem * 871289209Scem * Ideally I could have just specified the size when I allocated the 872289209Scem * resource like: 873289209Scem * bus_alloc_resource(ntb->device, 874289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 875289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 876289209Scem * but the PCI driver does not honor the size in this call, so we have 877289209Scem * to modify it after the fact. 878289209Scem */ 879289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 880289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 881289209Scem bar_size_bits = pci_read_config(ntb->device, 882289209Scem XEON_PBAR23SZ_OFFSET, 1); 883289209Scem else 884289209Scem bar_size_bits = pci_read_config(ntb->device, 885289209Scem XEON_PBAR45SZ_OFFSET, 1); 886289209Scem 887289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 888289209Scem bar->pci_resource, bar->pbase, 889289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 890255272Scarl if (rc != 0) { 891289209Scem device_printf(ntb->device, 892289209Scem "unable to resize bar\n"); 893255272Scarl return (rc); 894250079Scarl } 895289209Scem 896289209Scem save_bar_parameters(bar); 897250079Scarl } 898289209Scem 899291280Scem bar->map_mode = VM_MEMATTR_UNCACHEABLE; 900291030Scem print_map_success(ntb, bar, "mw"); 901291280Scem 902295486Scem /* 903295486Scem * Optionally, mark MW BARs as anything other than UC to improve 904295486Scem * performance. 905295486Scem */ 906295486Scem mapmode = ntb_pat_flags(); 907295486Scem if (mapmode == bar->map_mode) 908295486Scem return (0); 909291030Scem 910291280Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 911291031Scem if (rc == 0) { 912291280Scem bar->map_mode = mapmode; 913289209Scem device_printf(ntb->device, 914289647Scem "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 915291280Scem "%s.\n", 916289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 917289647Scem (char *)bar->vbase + bar->size - 1, 918291280Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 919295487Scem ntb_vm_memattr_to_str(mapmode)); 920291031Scem } else 921289647Scem device_printf(ntb->device, 922289647Scem "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 923291280Scem "%s: %d\n", 924289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 925289647Scem (char *)bar->vbase + bar->size - 1, 926289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 927295487Scem ntb_vm_memattr_to_str(mapmode), rc); 928289647Scem /* Proceed anyway */ 929250079Scarl return (0); 930250079Scarl} 931250079Scarl 932250079Scarlstatic void 933250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 934250079Scarl{ 935250079Scarl struct ntb_pci_bar_info *current_bar; 936250079Scarl int i; 937250079Scarl 938289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 939250079Scarl current_bar = &ntb->bar_info[i]; 940250079Scarl if (current_bar->pci_resource != NULL) 941250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 942250079Scarl current_bar->pci_resource_id, 943250079Scarl current_bar->pci_resource); 944250079Scarl } 945250079Scarl} 946250079Scarl 947250079Scarlstatic int 948289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 949250079Scarl{ 950289342Scem uint32_t i; 951289342Scem int rc; 952289342Scem 953289342Scem for (i = 0; i < num_vectors; i++) { 954289342Scem ntb->int_info[i].rid = i + 1; 955289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 956289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 957289342Scem if (ntb->int_info[i].res == NULL) { 958289342Scem device_printf(ntb->device, 959289342Scem "bus_alloc_resource failed\n"); 960289342Scem return (ENOMEM); 961289342Scem } 962289342Scem ntb->int_info[i].tag = NULL; 963289342Scem ntb->allocated_interrupts++; 964289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 965289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 966289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 967289342Scem if (rc != 0) { 968289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 969289342Scem return (ENXIO); 970289342Scem } 971289342Scem } 972289342Scem return (0); 973289342Scem} 974289342Scem 975289344Scem/* 976289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 977289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 978289344Scem * should be okay. This tunable should enable us to test that hypothesis 979289344Scem * when someone gets their hands on some Xeon hardware. 980289344Scem */ 981289344Scemstatic int ntb_force_remap_mode; 982289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 983289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 984289344Scem " to a smaller number of ithreads, even if the desired number are " 985289344Scem "available"); 986289344Scem 987289344Scem/* 988289344Scem * In case it is NOT ok, give consumers an abort button. 989289344Scem */ 990289344Scemstatic int ntb_prefer_intx; 991289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 992289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 993289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 994289344Scem "behavior)"); 995289344Scem 996289344Scem/* 997289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 998289344Scem * round-robin fashion. 999289344Scem */ 1000289342Scemstatic int 1001289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1002289344Scem{ 1003289344Scem u_int *vectors; 1004289344Scem uint32_t i; 1005289344Scem int rc; 1006289344Scem 1007289344Scem if (ntb_prefer_intx != 0) 1008289344Scem return (ENXIO); 1009289344Scem 1010289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1011289344Scem 1012289344Scem for (i = 0; i < desired; i++) 1013289344Scem vectors[i] = (i % avail) + 1; 1014289344Scem 1015289344Scem rc = pci_remap_msix(dev, desired, vectors); 1016289344Scem free(vectors, M_NTB); 1017289344Scem return (rc); 1018289344Scem} 1019289344Scem 1020289344Scemstatic int 1021289540Scemntb_init_isr(struct ntb_softc *ntb) 1022289342Scem{ 1023289344Scem uint32_t desired_vectors, num_vectors; 1024289342Scem int rc; 1025250079Scarl 1026250079Scarl ntb->allocated_interrupts = 0; 1027289542Scem ntb->last_ts = ticks; 1028289347Scem 1029250079Scarl /* 1030295618Scem * Mask all doorbell interrupts. (Except link events!) 1031250079Scarl */ 1032295618Scem DB_MASK_LOCK(ntb); 1033295618Scem ntb->db_mask = ntb->db_valid_mask; 1034295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1035295618Scem DB_MASK_UNLOCK(ntb); 1036250079Scarl 1037289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1038289539Scem ntb->db_count); 1039289344Scem if (desired_vectors >= 1) { 1040289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 1041250079Scarl 1042289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 1043289344Scem num_vectors == desired_vectors) 1044289344Scem num_vectors--; 1045289344Scem 1046289344Scem if (rc == 0 && num_vectors < desired_vectors) { 1047289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 1048289344Scem num_vectors); 1049289344Scem if (rc == 0) 1050289344Scem num_vectors = desired_vectors; 1051289344Scem else 1052289344Scem pci_release_msi(ntb->device); 1053289344Scem } 1054289344Scem if (rc != 0) 1055289344Scem num_vectors = 1; 1056289344Scem } else 1057289344Scem num_vectors = 1; 1058289344Scem 1059289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1060295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1061295618Scem device_printf(ntb->device, 1062295618Scem "Errata workaround does not support MSI or INTX\n"); 1063295618Scem return (EINVAL); 1064295618Scem } 1065295618Scem 1066289539Scem ntb->db_vec_count = 1; 1067290680Scem ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1068289539Scem rc = ntb_setup_legacy_interrupt(ntb); 1069289539Scem } else { 1070300531Scem if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1071300531Scem HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1072300531Scem device_printf(ntb->device, 1073300531Scem "Errata workaround expects %d doorbell bits\n", 1074300531Scem XEON_NONLINK_DB_MSIX_BITS); 1075300531Scem return (EINVAL); 1076300531Scem } 1077300531Scem 1078289546Scem ntb_create_msix_vec(ntb, num_vectors); 1079289540Scem rc = ntb_setup_msix(ntb, num_vectors); 1080295618Scem if (rc == 0 && HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1081300531Scem ntb_get_msix_info(ntb); 1082289539Scem } 1083289539Scem if (rc != 0) { 1084289539Scem device_printf(ntb->device, 1085289539Scem "Error allocating interrupts: %d\n", rc); 1086289546Scem ntb_free_msix_vec(ntb); 1087289396Scem } 1088289396Scem 1089289342Scem return (rc); 1090289342Scem} 1091289342Scem 1092289342Scemstatic int 1093289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1094289342Scem{ 1095289342Scem int rc; 1096289342Scem 1097289342Scem ntb->int_info[0].rid = 0; 1098289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1099289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1100289342Scem if (ntb->int_info[0].res == NULL) { 1101289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 1102289342Scem return (ENOMEM); 1103250079Scarl } 1104250079Scarl 1105289342Scem ntb->int_info[0].tag = NULL; 1106289342Scem ntb->allocated_interrupts = 1; 1107289342Scem 1108289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1109289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1110289342Scem ntb, &ntb->int_info[0].tag); 1111289342Scem if (rc != 0) { 1112289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 1113289342Scem return (ENXIO); 1114289342Scem } 1115289342Scem 1116250079Scarl return (0); 1117250079Scarl} 1118250079Scarl 1119250079Scarlstatic void 1120250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 1121250079Scarl{ 1122250079Scarl struct ntb_int_info *current_int; 1123250079Scarl int i; 1124250079Scarl 1125289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 1126250079Scarl current_int = &ntb->int_info[i]; 1127250079Scarl if (current_int->tag != NULL) 1128250079Scarl bus_teardown_intr(ntb->device, current_int->res, 1129250079Scarl current_int->tag); 1130250079Scarl 1131250079Scarl if (current_int->res != NULL) 1132250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 1133250079Scarl rman_get_rid(current_int->res), current_int->res); 1134250079Scarl } 1135250079Scarl 1136289546Scem ntb_free_msix_vec(ntb); 1137250079Scarl pci_release_msi(ntb->device); 1138250079Scarl} 1139250079Scarl 1140289347Scem/* 1141289648Scem * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1142289347Scem * out to make code clearer. 1143289347Scem */ 1144289539Scemstatic inline uint64_t 1145289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 1146289347Scem{ 1147289347Scem 1148289648Scem if (ntb->type == NTB_ATOM) 1149289347Scem return (ntb_reg_read(8, regoff)); 1150289347Scem 1151289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1152289347Scem 1153289347Scem return (ntb_reg_read(2, regoff)); 1154289347Scem} 1155289347Scem 1156289539Scemstatic inline void 1157289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1158289347Scem{ 1159289347Scem 1160289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 1161289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1162289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 1163289542Scem (uintmax_t)ntb->db_valid_mask)); 1164289542Scem 1165289607Scem if (regoff == ntb->self_reg->db_mask) 1166289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 1167290678Scem db_iowrite_raw(ntb, regoff, val); 1168290678Scem} 1169289542Scem 1170290678Scemstatic inline void 1171290678Scemdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1172290678Scem{ 1173290678Scem 1174289648Scem if (ntb->type == NTB_ATOM) { 1175289347Scem ntb_reg_write(8, regoff, val); 1176289347Scem return; 1177289347Scem } 1178289347Scem 1179289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1180289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 1181289347Scem} 1182289347Scem 1183289546Scemvoid 1184289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1185289542Scem{ 1186289542Scem 1187295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1188295618Scem return; 1189295618Scem 1190289546Scem DB_MASK_LOCK(ntb); 1191289542Scem ntb->db_mask |= bits; 1192289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1193289546Scem DB_MASK_UNLOCK(ntb); 1194289542Scem} 1195289542Scem 1196289546Scemvoid 1197289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1198289542Scem{ 1199289542Scem 1200289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1201289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1202289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1203289542Scem (uintmax_t)ntb->db_valid_mask)); 1204289542Scem 1205295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1206295618Scem return; 1207295618Scem 1208289546Scem DB_MASK_LOCK(ntb); 1209289542Scem ntb->db_mask &= ~bits; 1210289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1211289546Scem DB_MASK_UNLOCK(ntb); 1212289542Scem} 1213289542Scem 1214289546Scemuint64_t 1215289546Scemntb_db_read(struct ntb_softc *ntb) 1216289281Scem{ 1217289281Scem 1218295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1219295618Scem uint64_t res; 1220295618Scem unsigned i; 1221295618Scem 1222295618Scem res = 0; 1223295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1224295618Scem if (ntb->msix_vec[i].masked != 0) 1225295618Scem res |= ntb_db_vector_mask(ntb, i); 1226295618Scem } 1227295618Scem return (res); 1228295618Scem } 1229295618Scem 1230289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 1231289281Scem} 1232289281Scem 1233289546Scemvoid 1234289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1235289281Scem{ 1236289281Scem 1237289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1238289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1239289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1240289546Scem (uintmax_t)ntb->db_valid_mask)); 1241289546Scem 1242295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1243295618Scem unsigned i; 1244295618Scem 1245295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1246295618Scem if ((bits & ntb_db_vector_mask(ntb, i)) != 0) { 1247295618Scem DB_MASK_LOCK(ntb); 1248295618Scem if (ntb->msix_vec[i].masked != 0) { 1249295618Scem /* XXX These need a public API. */ 1250295618Scem#if 0 1251295618Scem pci_unmask_msix(ntb->device, i); 1252295618Scem#endif 1253295618Scem ntb->msix_vec[i].masked = 0; 1254295618Scem } 1255295618Scem DB_MASK_UNLOCK(ntb); 1256295618Scem } 1257295618Scem } 1258295618Scem return; 1259295618Scem } 1260295618Scem 1261289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1262289281Scem} 1263289281Scem 1264289540Scemstatic inline uint64_t 1265289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1266250079Scarl{ 1267289540Scem uint64_t shift, mask; 1268250079Scarl 1269289540Scem shift = ntb->db_vec_shift; 1270289540Scem mask = (1ull << shift) - 1; 1271289540Scem return (mask << (shift * db_vector)); 1272250079Scarl} 1273250079Scarl 1274250079Scarlstatic void 1275289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1276250079Scarl{ 1277289540Scem uint64_t vec_mask; 1278250079Scarl 1279289542Scem ntb->last_ts = ticks; 1280289546Scem vec_mask = ntb_vec_mask(ntb, vec); 1281250079Scarl 1282289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1283289546Scem if (ntb_poll_link(ntb)) 1284289546Scem ntb_link_event(ntb); 1285289540Scem } 1286289540Scem 1287295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1288295618Scem (vec_mask & ntb->db_link_mask) == 0) { 1289295618Scem DB_MASK_LOCK(ntb); 1290295618Scem if (ntb->msix_vec[vec].masked == 0) { 1291295618Scem /* XXX These need a public API. */ 1292295618Scem#if 0 1293295618Scem pci_mask_msix(ntb->device, vec); 1294295618Scem#endif 1295295618Scem ntb->msix_vec[vec].masked = 1; 1296295618Scem } 1297295618Scem DB_MASK_UNLOCK(ntb); 1298295618Scem } 1299295618Scem 1300289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1301289546Scem ntb_db_event(ntb, vec); 1302289546Scem} 1303250079Scarl 1304289546Scemstatic void 1305289546Scemndev_vec_isr(void *arg) 1306289546Scem{ 1307289546Scem struct ntb_vec *nvec = arg; 1308250079Scarl 1309289546Scem ntb_interrupt(nvec->ntb, nvec->num); 1310250079Scarl} 1311250079Scarl 1312250079Scarlstatic void 1313289546Scemndev_irq_isr(void *arg) 1314250079Scarl{ 1315289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1316289546Scem ntb_interrupt(arg, 0); 1317250079Scarl} 1318250079Scarl 1319250079Scarlstatic int 1320289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1321250079Scarl{ 1322289342Scem uint32_t i; 1323250079Scarl 1324289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1325250079Scarl M_ZERO | M_WAITOK); 1326250079Scarl for (i = 0; i < num_vectors; i++) { 1327289546Scem ntb->msix_vec[i].num = i; 1328289546Scem ntb->msix_vec[i].ntb = ntb; 1329250079Scarl } 1330250079Scarl 1331250079Scarl return (0); 1332250079Scarl} 1333250079Scarl 1334250079Scarlstatic void 1335289546Scemntb_free_msix_vec(struct ntb_softc *ntb) 1336250079Scarl{ 1337250079Scarl 1338289546Scem if (ntb->msix_vec == NULL) 1339289539Scem return; 1340289539Scem 1341289546Scem free(ntb->msix_vec, M_NTB); 1342289546Scem ntb->msix_vec = NULL; 1343250079Scarl} 1344250079Scarl 1345295618Scemstatic void 1346300531Scemntb_get_msix_info(struct ntb_softc *ntb) 1347295618Scem{ 1348295618Scem struct pci_devinfo *dinfo; 1349295618Scem struct pcicfg_msix *msix; 1350295618Scem uint32_t laddr, data, i, offset; 1351295618Scem 1352295618Scem dinfo = device_get_ivars(ntb->device); 1353295618Scem msix = &dinfo->cfg.msix; 1354295618Scem 1355295618Scem laddr = data = 0; 1356295618Scem 1357300531Scem CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1358300531Scem 1359300531Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1360295618Scem offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1361295618Scem 1362295618Scem laddr = bus_read_4(msix->msix_table_res, offset + 1363295618Scem PCI_MSIX_ENTRY_LOWER_ADDR); 1364301293Smav ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1365295618Scem 1366295618Scem KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1367295618Scem ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1368295618Scem MSI_INTEL_ADDR_BASE)); 1369301293Smav ntb->msix_data[i].nmd_ofs = laddr; 1370295618Scem 1371295618Scem data = bus_read_4(msix->msix_table_res, offset + 1372295618Scem PCI_MSIX_ENTRY_DATA); 1373295618Scem ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1374295618Scem 1375295618Scem ntb->msix_data[i].nmd_data = data; 1376295618Scem } 1377295618Scem} 1378295618Scem 1379250079Scarlstatic struct ntb_hw_info * 1380250079Scarlntb_get_device_info(uint32_t device_id) 1381250079Scarl{ 1382250079Scarl struct ntb_hw_info *ep = pci_ids; 1383250079Scarl 1384250079Scarl while (ep->device_id) { 1385250079Scarl if (ep->device_id == device_id) 1386250079Scarl return (ep); 1387250079Scarl ++ep; 1388250079Scarl } 1389250079Scarl return (NULL); 1390250079Scarl} 1391250079Scarl 1392289272Scemstatic void 1393289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 1394250079Scarl{ 1395250079Scarl 1396289617Scem if (ntb->reg != NULL) 1397289617Scem ntb_link_disable(ntb); 1398250079Scarl} 1399250079Scarl 1400289397Scemstatic void 1401289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 1402289397Scem{ 1403289397Scem 1404289648Scem if (ntb->type == NTB_ATOM) { 1405289648Scem ntb->mw_count = ATOM_MW_COUNT; 1406289397Scem return; 1407289397Scem } 1408289397Scem 1409289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1410289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1411289397Scem else 1412289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1413289397Scem} 1414289397Scem 1415250079Scarlstatic int 1416289348Scemntb_detect_xeon(struct ntb_softc *ntb) 1417250079Scarl{ 1418289348Scem uint8_t ppd, conn_type; 1419250079Scarl 1420289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1421289348Scem ntb->ppd = ppd; 1422250079Scarl 1423289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1424290681Scem ntb->dev_type = NTB_DEV_DSD; 1425290681Scem else 1426289257Scem ntb->dev_type = NTB_DEV_USD; 1427289257Scem 1428289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1429289397Scem ntb->features |= NTB_SPLIT_BAR; 1430289397Scem 1431295618Scem /* 1432295618Scem * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1433295618Scem * errata workaround; only do one at a time. 1434295618Scem */ 1435289542Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1436295618Scem ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1437289542Scem 1438289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1439289348Scem switch (conn_type) { 1440289348Scem case NTB_CONN_B2B: 1441289348Scem ntb->conn_type = conn_type; 1442289348Scem break; 1443289348Scem case NTB_CONN_RP: 1444289348Scem case NTB_CONN_TRANSPARENT: 1445289348Scem default: 1446289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1447289348Scem (unsigned)conn_type); 1448289348Scem return (ENXIO); 1449289348Scem } 1450289348Scem return (0); 1451289348Scem} 1452289348Scem 1453289348Scemstatic int 1454289648Scemntb_detect_atom(struct ntb_softc *ntb) 1455289348Scem{ 1456289348Scem uint32_t ppd, conn_type; 1457289348Scem 1458289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1459289348Scem ntb->ppd = ppd; 1460289348Scem 1461289648Scem if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1462289348Scem ntb->dev_type = NTB_DEV_DSD; 1463289348Scem else 1464289348Scem ntb->dev_type = NTB_DEV_USD; 1465289348Scem 1466289648Scem conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1467289348Scem switch (conn_type) { 1468289348Scem case NTB_CONN_B2B: 1469289348Scem ntb->conn_type = conn_type; 1470289348Scem break; 1471289348Scem default: 1472289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1473289348Scem return (ENXIO); 1474289348Scem } 1475289348Scem return (0); 1476289348Scem} 1477289348Scem 1478289348Scemstatic int 1479289542Scemntb_xeon_init_dev(struct ntb_softc *ntb) 1480289348Scem{ 1481289542Scem int rc; 1482289348Scem 1483289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1484289542Scem ntb->db_count = XEON_DB_COUNT; 1485289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1486289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1487289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1488289257Scem 1489289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1490250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1491289348Scem ntb->conn_type); 1492250079Scarl return (ENXIO); 1493250079Scarl } 1494250079Scarl 1495289542Scem ntb->reg = &xeon_reg; 1496289607Scem ntb->self_reg = &xeon_pri_reg; 1497289542Scem ntb->peer_reg = &xeon_b2b_reg; 1498289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1499289542Scem 1500295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1501295618Scem ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1502295618Scem ntb->mw_count; 1503295618Scem ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1504295618Scem g_ntb_msix_idx, ntb->msix_mw_idx); 1505295618Scem rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1506295618Scem VM_MEMATTR_UNCACHEABLE); 1507295618Scem KASSERT(rc == 0, ("shouldn't fail")); 1508295618Scem } else if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1509295618Scem /* 1510295618Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1511295618Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1512295618Scem * which may hang the system. To workaround this, use a memory 1513295618Scem * window to access the interrupt and scratch pad registers on the 1514295618Scem * remote system. 1515295618Scem */ 1516291263Scem ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1517291263Scem ntb->mw_count; 1518291263Scem ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1519291263Scem g_ntb_mw_idx, ntb->b2b_mw_idx); 1520295618Scem rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1521295618Scem VM_MEMATTR_UNCACHEABLE); 1522291263Scem KASSERT(rc == 0, ("shouldn't fail")); 1523291263Scem } else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1524289208Scem /* 1525289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1526289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1527289542Scem * since bit 14 is the last bit. 1528289542Scem * 1529289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1530289542Scem * anyway. Nor for non-B2B connection types. 1531289542Scem */ 1532289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1533250079Scarl 1534289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1535250079Scarl 1536289542Scem if (ntb->dev_type == NTB_DEV_USD) 1537289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1538289542Scem &xeon_b2b_usd_addr); 1539289542Scem else 1540289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1541289542Scem &xeon_b2b_dsd_addr); 1542289542Scem if (rc != 0) 1543289542Scem return (rc); 1544289271Scem 1545250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1546290682Scem ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1547289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1548255279Scarl 1549290682Scem /* 1550290682Scem * Mask all doorbell interrupts. 1551290682Scem */ 1552295618Scem DB_MASK_LOCK(ntb); 1553295618Scem ntb->db_mask = ntb->db_valid_mask; 1554295618Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1555295618Scem DB_MASK_UNLOCK(ntb); 1556250079Scarl 1557295618Scem rc = xeon_setup_msix_bar(ntb); 1558295618Scem if (rc != 0) 1559295618Scem return (rc); 1560295618Scem 1561290682Scem rc = ntb_init_isr(ntb); 1562290682Scem return (rc); 1563250079Scarl} 1564250079Scarl 1565250079Scarlstatic int 1566289648Scemntb_atom_init_dev(struct ntb_softc *ntb) 1567250079Scarl{ 1568290682Scem int error; 1569250079Scarl 1570289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1571289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1572250079Scarl 1573289648Scem ntb->spad_count = ATOM_SPAD_COUNT; 1574289648Scem ntb->db_count = ATOM_DB_COUNT; 1575289648Scem ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1576289648Scem ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1577289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1578250079Scarl 1579289648Scem ntb->reg = &atom_reg; 1580289648Scem ntb->self_reg = &atom_pri_reg; 1581289648Scem ntb->peer_reg = &atom_b2b_reg; 1582289648Scem ntb->xlat_reg = &atom_sec_xlat; 1583289542Scem 1584250079Scarl /* 1585289648Scem * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1586250079Scarl * resolved. Mask transaction layer internal parity errors. 1587250079Scarl */ 1588250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1589250079Scarl 1590289648Scem configure_atom_secondary_side_bars(ntb); 1591250079Scarl 1592250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1593290682Scem ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1594250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1595289209Scem 1596290682Scem error = ntb_init_isr(ntb); 1597290682Scem if (error != 0) 1598290682Scem return (error); 1599290682Scem 1600289542Scem /* Initiate PCI-E link training */ 1601289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1602250079Scarl 1603289648Scem callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1604289542Scem 1605250079Scarl return (0); 1606250079Scarl} 1607250079Scarl 1608289648Scem/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1609255279Scarlstatic void 1610289648Scemconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1611255279Scarl{ 1612255279Scarl 1613255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1614289648Scem ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1615290725Scem XEON_B2B_BAR2_ADDR64); 1616289648Scem ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1617290725Scem XEON_B2B_BAR4_ADDR64); 1618290725Scem ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1619290725Scem ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1620255279Scarl } else { 1621289648Scem ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1622290725Scem XEON_B2B_BAR2_ADDR64); 1623289648Scem ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1624290725Scem XEON_B2B_BAR4_ADDR64); 1625290725Scem ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1626290725Scem ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1627255279Scarl } 1628255279Scarl} 1629255279Scarl 1630289543Scem 1631289543Scem/* 1632289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1633289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1634289543Scem * remains for use by a higher layer. 1635289543Scem * 1636289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1637289543Scem * MW size is sufficiently large. 1638289543Scem */ 1639289543Scemstatic unsigned int ntb_b2b_mw_share; 1640289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1641289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1642289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1643289543Scem "value here."); 1644289543Scem 1645289543Scemstatic void 1646289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1647289543Scem enum ntb_bar regbar) 1648289543Scem{ 1649289543Scem struct ntb_pci_bar_info *bar; 1650289543Scem uint8_t bar_sz; 1651289543Scem 1652289543Scem if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1653289543Scem return; 1654289543Scem 1655289543Scem bar = &ntb->bar_info[idx]; 1656289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1657289543Scem if (idx == regbar) { 1658289543Scem if (ntb->b2b_off != 0) 1659289543Scem bar_sz--; 1660289543Scem else 1661289543Scem bar_sz = 0; 1662289543Scem } 1663289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1664289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1665289543Scem (void)bar_sz; 1666289543Scem} 1667289543Scem 1668289543Scemstatic void 1669289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1670289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1671289543Scem{ 1672301293Smav uint64_t reg_val; 1673289546Scem uint32_t base_reg, lmt_reg; 1674289543Scem 1675289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1676289546Scem if (idx == regbar) 1677289546Scem bar_addr += ntb->b2b_off; 1678289543Scem 1679295618Scem /* 1680295618Scem * Set limit registers first to avoid an errata where setting the base 1681295618Scem * registers locks the limit registers. 1682295618Scem */ 1683289546Scem if (!bar_is_64bit(ntb, idx)) { 1684301293Smav ntb_reg_write(4, lmt_reg, bar_addr); 1685295618Scem reg_val = ntb_reg_read(4, lmt_reg); 1686295618Scem (void)reg_val; 1687295618Scem 1688289546Scem ntb_reg_write(4, base_reg, bar_addr); 1689289546Scem reg_val = ntb_reg_read(4, base_reg); 1690289546Scem (void)reg_val; 1691295618Scem } else { 1692301293Smav ntb_reg_write(8, lmt_reg, bar_addr); 1693295618Scem reg_val = ntb_reg_read(8, lmt_reg); 1694295618Scem (void)reg_val; 1695289546Scem 1696289546Scem ntb_reg_write(8, base_reg, bar_addr); 1697289546Scem reg_val = ntb_reg_read(8, base_reg); 1698289546Scem (void)reg_val; 1699289543Scem } 1700289543Scem} 1701289543Scem 1702289543Scemstatic void 1703289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1704289543Scem{ 1705289543Scem struct ntb_pci_bar_info *bar; 1706289543Scem 1707289543Scem bar = &ntb->bar_info[idx]; 1708289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1709289543Scem ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1710289543Scem base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1711289543Scem } else { 1712289543Scem ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1713289543Scem base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1714289543Scem } 1715289543Scem (void)base_addr; 1716289543Scem} 1717289543Scem 1718289542Scemstatic int 1719295618Scemxeon_setup_msix_bar(struct ntb_softc *ntb) 1720295618Scem{ 1721295618Scem enum ntb_bar bar_num; 1722295618Scem 1723295618Scem if (!HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1724295618Scem return (0); 1725295618Scem 1726295618Scem bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1727301293Smav ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1728295618Scem return (0); 1729295618Scem} 1730295618Scem 1731295618Scemstatic int 1732289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1733289542Scem const struct ntb_b2b_addr *peer_addr) 1734255279Scarl{ 1735289543Scem struct ntb_pci_bar_info *b2b_bar; 1736289543Scem vm_size_t bar_size; 1737289543Scem uint64_t bar_addr; 1738289543Scem enum ntb_bar b2b_bar_num, i; 1739255279Scarl 1740289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1741289543Scem b2b_bar = NULL; 1742289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1743289543Scem ntb->b2b_off = 0; 1744289543Scem } else { 1745289543Scem b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1746289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1747289543Scem ("invalid b2b mw bar")); 1748289543Scem 1749289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1750289543Scem bar_size = b2b_bar->size; 1751289543Scem 1752289543Scem if (ntb_b2b_mw_share != 0 && 1753289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1754289543Scem ntb->b2b_off = bar_size >> 1; 1755289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1756289543Scem ntb->b2b_off = 0; 1757289543Scem } else { 1758289543Scem device_printf(ntb->device, 1759289543Scem "B2B bar size is too small!\n"); 1760289543Scem return (EIO); 1761289543Scem } 1762255279Scarl } 1763289542Scem 1764289543Scem /* 1765289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1766289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1767289543Scem */ 1768289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1769289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1770289543Scem 1771289543Scem bar_addr = 0; 1772289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1773289543Scem bar_addr = addr->bar0_addr; 1774289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1775289543Scem bar_addr = addr->bar2_addr64; 1776289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1777289543Scem bar_addr = addr->bar4_addr64; 1778289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1779289543Scem bar_addr = addr->bar4_addr32; 1780289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1781289543Scem bar_addr = addr->bar5_addr32; 1782289543Scem else 1783289543Scem KASSERT(false, ("invalid bar")); 1784289543Scem 1785289543Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1786289543Scem 1787289543Scem /* 1788289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1789289543Scem * register BAR. The B2B BAR is either disabled above or configured 1790289543Scem * half-size. It starts at PBAR xlat + offset. 1791289543Scem * 1792289543Scem * Also set up incoming BAR limits == base (zero length window). 1793289543Scem */ 1794289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1795289543Scem b2b_bar_num); 1796289542Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1797289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1798289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1799289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1800289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1801289542Scem } else 1802289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1803289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1804289543Scem 1805289543Scem /* Zero incoming translation addrs */ 1806289543Scem ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1807289543Scem ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1808289543Scem 1809295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1810295618Scem size_t size, xlatoffset; 1811295618Scem 1812295618Scem switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1813295618Scem case NTB_B2B_BAR_1: 1814295618Scem size = 8; 1815295618Scem xlatoffset = XEON_SBAR2XLAT_OFFSET; 1816295618Scem break; 1817295618Scem case NTB_B2B_BAR_2: 1818295618Scem xlatoffset = XEON_SBAR4XLAT_OFFSET; 1819295618Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1820295618Scem size = 4; 1821295618Scem else 1822295618Scem size = 8; 1823295618Scem break; 1824295618Scem case NTB_B2B_BAR_3: 1825295618Scem xlatoffset = XEON_SBAR5XLAT_OFFSET; 1826295618Scem size = 4; 1827295618Scem break; 1828295618Scem default: 1829295618Scem KASSERT(false, ("Bogus msix mw idx: %u", 1830295618Scem ntb->msix_mw_idx)); 1831295618Scem return (EINVAL); 1832295618Scem } 1833295618Scem 1834295618Scem /* 1835295618Scem * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1836295618Scem * workaround 1837295618Scem */ 1838301293Smav if (size == 4) { 1839295618Scem ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1840301293Smav ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1841301293Smav } else { 1842295618Scem ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1843301293Smav ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1844301293Smav } 1845295618Scem } 1846295618Scem (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1847295618Scem (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1848295618Scem 1849289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1850289543Scem ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1851289543Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1852289543Scem 1853289543Scem /* Set outgoing translation offsets */ 1854289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1855289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1856289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1857289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1858289543Scem } else 1859289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1860289543Scem 1861289543Scem /* Set the translation offset for B2B registers */ 1862289543Scem bar_addr = 0; 1863289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1864289543Scem bar_addr = peer_addr->bar0_addr; 1865289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1866289543Scem bar_addr = peer_addr->bar2_addr64; 1867289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1868289543Scem bar_addr = peer_addr->bar4_addr64; 1869289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1870289543Scem bar_addr = peer_addr->bar4_addr32; 1871289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1872289543Scem bar_addr = peer_addr->bar5_addr32; 1873289543Scem else 1874289543Scem KASSERT(false, ("invalid bar")); 1875289543Scem 1876289543Scem /* 1877289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1878289543Scem * at a time. 1879289543Scem */ 1880289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1881289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1882289542Scem return (0); 1883255279Scarl} 1884255279Scarl 1885289546Scemstatic inline bool 1886295618Scem_xeon_link_is_up(struct ntb_softc *ntb) 1887295618Scem{ 1888295618Scem 1889295618Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1890295618Scem return (true); 1891295618Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1892295618Scem} 1893295618Scem 1894295618Scemstatic inline bool 1895289546Scemlink_is_up(struct ntb_softc *ntb) 1896289546Scem{ 1897289546Scem 1898295618Scem if (ntb->type == NTB_XEON) 1899295618Scem return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1900295618Scem !HAS_FEATURE(NTB_SB01BASE_LOCKUP))); 1901289546Scem 1902289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1903289648Scem return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1904289546Scem} 1905289546Scem 1906289546Scemstatic inline bool 1907289648Scematom_link_is_err(struct ntb_softc *ntb) 1908289546Scem{ 1909289546Scem uint32_t status; 1910289546Scem 1911289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1912289546Scem 1913289648Scem status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1914289648Scem if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1915289546Scem return (true); 1916289546Scem 1917289648Scem status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1918289648Scem return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1919289546Scem} 1920289546Scem 1921289648Scem/* Atom does not have link status interrupt, poll on that platform */ 1922250079Scarlstatic void 1923289648Scematom_link_hb(void *arg) 1924250079Scarl{ 1925250079Scarl struct ntb_softc *ntb = arg; 1926289546Scem sbintime_t timo, poll_ts; 1927250079Scarl 1928289546Scem timo = NTB_HB_TIMEOUT * hz; 1929289546Scem poll_ts = ntb->last_ts + timo; 1930289546Scem 1931289542Scem /* 1932289542Scem * Delay polling the link status if an interrupt was received, unless 1933289542Scem * the cached link status says the link is down. 1934289542Scem */ 1935289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1936289546Scem timo = poll_ts - ticks; 1937289542Scem goto out; 1938289546Scem } 1939289542Scem 1940289546Scem if (ntb_poll_link(ntb)) 1941289546Scem ntb_link_event(ntb); 1942289542Scem 1943289648Scem if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1944289546Scem /* Link is down with error, proceed with recovery */ 1945289648Scem callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1946289546Scem return; 1947250079Scarl } 1948250079Scarl 1949289542Scemout: 1950289648Scem callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1951250079Scarl} 1952250079Scarl 1953250079Scarlstatic void 1954289648Scematom_perform_link_restart(struct ntb_softc *ntb) 1955250079Scarl{ 1956250079Scarl uint32_t status; 1957250079Scarl 1958250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1959289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1960289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1961289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1962289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1963250079Scarl 1964250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1965250079Scarl pause("ModPhy", hz / 10); 1966250079Scarl 1967250079Scarl /* Clear AER Errors, write to clear */ 1968289648Scem status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1969250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1970289648Scem ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1971250079Scarl 1972250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1973289648Scem status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1974289648Scem status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1975289648Scem ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1976250079Scarl 1977250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1978289648Scem status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1979289648Scem status |= ATOM_DESKEWSTS_DBERR; 1980289648Scem ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1981250079Scarl 1982289648Scem status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1983289648Scem status &= ATOM_IBIST_ERR_OFLOW; 1984289648Scem ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1985250079Scarl 1986250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1987289648Scem status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1988289648Scem status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1989289648Scem ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1990250079Scarl} 1991250079Scarl 1992289546Scem/* 1993289546Scem * ntb_set_ctx() - associate a driver context with an ntb device 1994289546Scem * @ntb: NTB device context 1995289546Scem * @ctx: Driver context 1996289546Scem * @ctx_ops: Driver context operations 1997289546Scem * 1998289546Scem * Associate a driver context and operations with a ntb device. The context is 1999289546Scem * provided by the client driver, and the driver may associate a different 2000289546Scem * context with each ntb device. 2001289546Scem * 2002289546Scem * Return: Zero if the context is associated, otherwise an error number. 2003289546Scem */ 2004289546Scemint 2005289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 2006250079Scarl{ 2007250079Scarl 2008289546Scem if (ctx == NULL || ops == NULL) 2009289546Scem return (EINVAL); 2010289546Scem if (ntb->ctx_ops != NULL) 2011289546Scem return (EINVAL); 2012250079Scarl 2013289546Scem CTX_LOCK(ntb); 2014289546Scem if (ntb->ctx_ops != NULL) { 2015289546Scem CTX_UNLOCK(ntb); 2016289546Scem return (EINVAL); 2017250079Scarl } 2018289546Scem ntb->ntb_ctx = ctx; 2019289546Scem ntb->ctx_ops = ops; 2020289546Scem CTX_UNLOCK(ntb); 2021250079Scarl 2022289546Scem return (0); 2023250079Scarl} 2024250079Scarl 2025289546Scem/* 2026289546Scem * It is expected that this will only be used from contexts where the ctx_lock 2027289546Scem * is not needed to protect ntb_ctx lifetime. 2028289546Scem */ 2029289546Scemvoid * 2030289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 2031289546Scem{ 2032289546Scem 2033289546Scem KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2034289546Scem if (ops != NULL) 2035289546Scem *ops = ntb->ctx_ops; 2036289546Scem return (ntb->ntb_ctx); 2037289546Scem} 2038289546Scem 2039289546Scem/* 2040289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device 2041289546Scem * @ntb: NTB device context 2042289546Scem * 2043289546Scem * Clear any association that may exist between a driver context and the ntb 2044289546Scem * device. 2045289546Scem */ 2046289546Scemvoid 2047289546Scemntb_clear_ctx(struct ntb_softc *ntb) 2048289546Scem{ 2049289546Scem 2050289546Scem CTX_LOCK(ntb); 2051289546Scem ntb->ntb_ctx = NULL; 2052289546Scem ntb->ctx_ops = NULL; 2053289546Scem CTX_UNLOCK(ntb); 2054289546Scem} 2055289546Scem 2056289546Scem/* 2057289546Scem * ntb_link_event() - notify driver context of a change in link status 2058289546Scem * @ntb: NTB device context 2059289546Scem * 2060289546Scem * Notify the driver context that the link status may have changed. The driver 2061289546Scem * should call ntb_link_is_up() to get the current status. 2062289546Scem */ 2063289546Scemvoid 2064289546Scemntb_link_event(struct ntb_softc *ntb) 2065289546Scem{ 2066289546Scem 2067289546Scem CTX_LOCK(ntb); 2068289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2069289546Scem ntb->ctx_ops->link_event(ntb->ntb_ctx); 2070289546Scem CTX_UNLOCK(ntb); 2071289546Scem} 2072289546Scem 2073289546Scem/* 2074289546Scem * ntb_db_event() - notify driver context of a doorbell event 2075289546Scem * @ntb: NTB device context 2076289546Scem * @vector: Interrupt vector number 2077289546Scem * 2078289546Scem * Notify the driver context of a doorbell event. If hardware supports 2079289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which 2080289546Scem * vector received the interrupt. The vector number is relative to the first 2081289546Scem * vector used for doorbells, starting at zero, and must be less than 2082289546Scem * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2083289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2084289546Scem * those bits are associated with the vector number. 2085289546Scem */ 2086250079Scarlstatic void 2087289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec) 2088289272Scem{ 2089289546Scem 2090289546Scem CTX_LOCK(ntb); 2091289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2092289546Scem ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2093289546Scem CTX_UNLOCK(ntb); 2094289546Scem} 2095289546Scem 2096289546Scem/* 2097289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb 2098289546Scem * @ntb: NTB device context 2099289546Scem * @max_speed: The maximum link speed expressed as PCIe generation number[0] 2100289546Scem * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 2101289546Scem * 2102289546Scem * Enable the link on the secondary side of the ntb. This can only be done 2103289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 2104289546Scem * should train the link to its maximum speed and width, or the requested speed 2105289546Scem * and width, whichever is smaller, if supported. 2106289546Scem * 2107289546Scem * Return: Zero on success, otherwise an error number. 2108289546Scem * 2109289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 2110289546Scem * and width input will be ignored. 2111289546Scem */ 2112289546Scemint 2113289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 2114289546Scem enum ntb_width w __unused) 2115289546Scem{ 2116289280Scem uint32_t cntl; 2117289272Scem 2118300100Scem ntb_printf(2, "%s\n", __func__); 2119300100Scem 2120289648Scem if (ntb->type == NTB_ATOM) { 2121289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 2122289648Scem ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2123289546Scem return (0); 2124289542Scem } 2125289542Scem 2126289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2127289546Scem ntb_link_event(ntb); 2128289546Scem return (0); 2129289280Scem } 2130289280Scem 2131289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2132289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2133289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2134289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2135289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 2136289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2137289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2138289546Scem return (0); 2139289272Scem} 2140289272Scem 2141289546Scem/* 2142289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb 2143289546Scem * @ntb: NTB device context 2144289546Scem * 2145289546Scem * Disable the link on the secondary side of the ntb. This can only be done 2146289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 2147289546Scem * should disable the link. Returning from this call must indicate that a 2148289546Scem * barrier has passed, though with no more writes may pass in either direction 2149289546Scem * across the link, except if this call returns an error number. 2150289546Scem * 2151289546Scem * Return: Zero on success, otherwise an error number. 2152289546Scem */ 2153289546Scemint 2154289542Scemntb_link_disable(struct ntb_softc *ntb) 2155289272Scem{ 2156289272Scem uint32_t cntl; 2157289272Scem 2158300100Scem ntb_printf(2, "%s\n", __func__); 2159300100Scem 2160289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2161289546Scem ntb_link_event(ntb); 2162289546Scem return (0); 2163289272Scem } 2164289272Scem 2165289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2166289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2167289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2168289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 2169289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2170289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2171289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2172289546Scem return (0); 2173289272Scem} 2174289272Scem 2175300100Scembool 2176300100Scemntb_link_enabled(struct ntb_softc *ntb) 2177300100Scem{ 2178300100Scem uint32_t cntl; 2179300100Scem 2180300100Scem if (ntb->type == NTB_ATOM) { 2181300100Scem cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2182300100Scem return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2183300100Scem } 2184300100Scem 2185300100Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2186300100Scem return (true); 2187300100Scem 2188300100Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2189300100Scem return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2190300100Scem} 2191300100Scem 2192289272Scemstatic void 2193289648Scemrecover_atom_link(void *arg) 2194250079Scarl{ 2195250079Scarl struct ntb_softc *ntb = arg; 2196289608Scem unsigned speed, width, oldspeed, oldwidth; 2197250079Scarl uint32_t status32; 2198250079Scarl 2199289648Scem atom_perform_link_restart(ntb); 2200250079Scarl 2201289232Scem /* 2202289232Scem * There is a potential race between the 2 NTB devices recovering at 2203289232Scem * the same time. If the times are the same, the link will not recover 2204289232Scem * and the driver will be stuck in this loop forever. Add a random 2205289232Scem * interval to the recovery time to prevent this race. 2206289232Scem */ 2207289648Scem status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2208289648Scem pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2209289232Scem 2210289648Scem if (atom_link_is_err(ntb)) 2211250079Scarl goto retry; 2212250079Scarl 2213289542Scem status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2214289648Scem if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2215289232Scem goto out; 2216289232Scem 2217289542Scem status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2218289608Scem width = NTB_LNK_STA_WIDTH(status32); 2219289608Scem speed = status32 & NTB_LINK_SPEED_MASK; 2220289608Scem 2221289608Scem oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2222289608Scem oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2223289608Scem if (oldwidth != width || oldspeed != speed) 2224250079Scarl goto retry; 2225250079Scarl 2226289232Scemout: 2227289648Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2228289542Scem ntb); 2229250079Scarl return; 2230250079Scarl 2231250079Scarlretry: 2232289648Scem callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2233250079Scarl ntb); 2234250079Scarl} 2235250079Scarl 2236289546Scem/* 2237289546Scem * Polls the HW link status register(s); returns true if something has changed. 2238289546Scem */ 2239289546Scemstatic bool 2240289542Scemntb_poll_link(struct ntb_softc *ntb) 2241250079Scarl{ 2242250079Scarl uint32_t ntb_cntl; 2243289546Scem uint16_t reg_val; 2244250079Scarl 2245289648Scem if (ntb->type == NTB_ATOM) { 2246289542Scem ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2247289546Scem if (ntb_cntl == ntb->ntb_ctl) 2248289546Scem return (false); 2249289546Scem 2250289542Scem ntb->ntb_ctl = ntb_cntl; 2251289542Scem ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2252250079Scarl } else { 2253290678Scem db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2254250079Scarl 2255289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2256289546Scem if (reg_val == ntb->lnk_sta) 2257289546Scem return (false); 2258250079Scarl 2259289546Scem ntb->lnk_sta = reg_val; 2260295618Scem 2261295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 2262295618Scem if (_xeon_link_is_up(ntb)) { 2263295618Scem if (!ntb->peer_msix_good) { 2264295618Scem callout_reset(&ntb->peer_msix_work, 0, 2265295618Scem ntb_exchange_msix, ntb); 2266295618Scem return (false); 2267295618Scem } 2268295618Scem } else { 2269295618Scem ntb->peer_msix_good = false; 2270295618Scem ntb->peer_msix_done = false; 2271295618Scem } 2272295618Scem } 2273289542Scem } 2274289546Scem return (true); 2275289542Scem} 2276289542Scem 2277289546Scemstatic inline enum ntb_speed 2278289546Scemntb_link_sta_speed(struct ntb_softc *ntb) 2279250079Scarl{ 2280250079Scarl 2281289546Scem if (!link_is_up(ntb)) 2282289546Scem return (NTB_SPEED_NONE); 2283289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2284250079Scarl} 2285250079Scarl 2286289546Scemstatic inline enum ntb_width 2287289546Scemntb_link_sta_width(struct ntb_softc *ntb) 2288250079Scarl{ 2289250079Scarl 2290289546Scem if (!link_is_up(ntb)) 2291289546Scem return (NTB_WIDTH_NONE); 2292289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2293250079Scarl} 2294250079Scarl 2295289774ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2296289774Scem "Driver state, statistics, and HW registers"); 2297289774Scem 2298289774Scem#define NTB_REGSZ_MASK (3ul << 30) 2299289774Scem#define NTB_REG_64 (1ul << 30) 2300289774Scem#define NTB_REG_32 (2ul << 30) 2301289774Scem#define NTB_REG_16 (3ul << 30) 2302289774Scem#define NTB_REG_8 (0ul << 30) 2303289774Scem 2304289774Scem#define NTB_DB_READ (1ul << 29) 2305289774Scem#define NTB_PCI_REG (1ul << 28) 2306289774Scem#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2307289774Scem 2308289774Scemstatic void 2309289774Scemntb_sysctl_init(struct ntb_softc *ntb) 2310289774Scem{ 2311300100Scem struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2312289774Scem struct sysctl_ctx_list *ctx; 2313289774Scem struct sysctl_oid *tree, *tmptree; 2314289774Scem 2315289774Scem ctx = device_get_sysctl_ctx(ntb->device); 2316300100Scem globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2317289774Scem 2318300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2319300100Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2320300100Scem sysctl_handle_link_status_human, "A", 2321300100Scem "Link status (human readable)"); 2322300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2323300100Scem CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2324300100Scem "IU", "Link status (1=active, 0=inactive)"); 2325300100Scem SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2326300100Scem CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2327300100Scem "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2328300100Scem 2329300100Scem tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2330300100Scem CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2331289774Scem tree_par = SYSCTL_CHILDREN(tree); 2332289774Scem 2333289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2334289774Scem &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2335289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2336289774Scem &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2337290687Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2338290687Scem &ntb->ppd, 0, "Raw PPD register (cached)"); 2339289774Scem 2340289774Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2341289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2342289774Scem &ntb->b2b_mw_idx, 0, 2343289774Scem "Index of the MW used for B2B remote register access"); 2344289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2345289774Scem CTLFLAG_RD, &ntb->b2b_off, 2346289774Scem "If non-zero, offset of B2B register region in shared MW"); 2347289774Scem } 2348289774Scem 2349289774Scem SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2350289774Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2351289774Scem "Features/errata of this NTB device"); 2352289774Scem 2353289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2354290686Scem __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2355290686Scem "NTB CTL register (cached)"); 2356289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2357290686Scem __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2358290686Scem "LNK STA register (cached)"); 2359289774Scem 2360289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2361291263Scem &ntb->mw_count, 0, "MW count"); 2362289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2363289774Scem &ntb->spad_count, 0, "Scratchpad count"); 2364289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2365289774Scem &ntb->db_count, 0, "Doorbell count"); 2366289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2367289774Scem &ntb->db_vec_count, 0, "Doorbell vector count"); 2368289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2369289774Scem &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2370289774Scem 2371289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2372289774Scem &ntb->db_valid_mask, "Doorbell valid mask"); 2373289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2374289774Scem &ntb->db_link_mask, "Doorbell link mask"); 2375289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2376289774Scem &ntb->db_mask, "Doorbell mask (cached)"); 2377289774Scem 2378289774Scem tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2379289774Scem CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2380289774Scem regpar = SYSCTL_CHILDREN(tmptree); 2381289774Scem 2382290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2383290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2384290682Scem ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2385290682Scem "NTB Control register"); 2386290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2387290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2388290682Scem 0x19c, sysctl_handle_register, "IU", 2389290682Scem "NTB Link Capabilities"); 2390290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2391290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2392290682Scem 0x1a0, sysctl_handle_register, "IU", 2393290682Scem "NTB Link Control register"); 2394290682Scem 2395289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2396289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2397289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2398289774Scem sysctl_handle_register, "QU", "Doorbell mask register"); 2399289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2400289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2401289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2402289774Scem sysctl_handle_register, "QU", "Doorbell register"); 2403289774Scem 2404289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2405289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2406289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2407289774Scem sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2408289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2409289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2410289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2411289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2412289774Scem sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2413289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2414289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2415289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2416289774Scem sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2417289774Scem } else { 2418289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2419289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2420289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2421289774Scem sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2422289774Scem } 2423289774Scem 2424289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2425289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2426289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2427289774Scem sysctl_handle_register, "QU", "Incoming LMT23 register"); 2428289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2429289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2430289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2431289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2432289774Scem sysctl_handle_register, "IU", "Incoming LMT4 register"); 2433289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2434289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2435289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2436289774Scem sysctl_handle_register, "IU", "Incoming LMT5 register"); 2437289774Scem } else { 2438289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2439289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2440289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2441289774Scem sysctl_handle_register, "QU", "Incoming LMT45 register"); 2442289774Scem } 2443289774Scem 2444289774Scem if (ntb->type == NTB_ATOM) 2445289774Scem return; 2446289774Scem 2447289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2448289774Scem CTLFLAG_RD, NULL, "Xeon HW statistics"); 2449289774Scem statpar = SYSCTL_CHILDREN(tmptree); 2450289774Scem SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2451289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2452289774Scem NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2453289774Scem sysctl_handle_register, "SU", "Upstream Memory Miss"); 2454289774Scem 2455289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2456289774Scem CTLFLAG_RD, NULL, "Xeon HW errors"); 2457289774Scem errpar = SYSCTL_CHILDREN(tmptree); 2458289774Scem 2459290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2460289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2461290687Scem NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2462290687Scem sysctl_handle_register, "CU", "PPD"); 2463290687Scem 2464290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2465290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2466290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2467290687Scem sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2468290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2469290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2470290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2471290687Scem sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2472290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2473290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2474290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2475290687Scem sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2476290687Scem 2477290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2478290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2479290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2480290687Scem sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2481290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2482290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2483290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2484290687Scem sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2485290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2486290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2487290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2488290687Scem sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2489290687Scem 2490290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2491290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2492289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2493289774Scem sysctl_handle_register, "SU", "DEVSTS"); 2494290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2495289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2496289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2497289774Scem sysctl_handle_register, "SU", "LNKSTS"); 2498290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2499290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2500290687Scem NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2501290687Scem sysctl_handle_register, "SU", "SLNKSTS"); 2502290687Scem 2503289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2504289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2505289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2506289774Scem sysctl_handle_register, "IU", "UNCERRSTS"); 2507289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2508289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2509289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2510289774Scem sysctl_handle_register, "IU", "CORERRSTS"); 2511289774Scem 2512289774Scem if (ntb->conn_type != NTB_CONN_B2B) 2513289774Scem return; 2514289774Scem 2515289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2516289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2517289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2518289774Scem sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2519289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2520289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2521289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2522289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2523289774Scem sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2524289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2525289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2526289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2527289774Scem sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2528289774Scem } else { 2529289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2530289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2531289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2532289774Scem sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2533289774Scem } 2534289774Scem 2535289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2536289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2537289774Scem NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2538289774Scem sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2539289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2540289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2541289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2542289774Scem NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2543289774Scem sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2544289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2545289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2546289774Scem NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2547289774Scem sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2548289774Scem } else { 2549289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2550289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2551289774Scem NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2552289774Scem sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2553289774Scem } 2554289774Scem 2555289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2556289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2557289774Scem NTB_REG_64 | ntb->xlat_reg->bar0_base, 2558289774Scem sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2559289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2560289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2561289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_base, 2562289774Scem sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2563289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2564289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2565289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2566289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_base, 2567289774Scem sysctl_handle_register, "IU", 2568289774Scem "Secondary BAR4 base register"); 2569289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2570289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2571289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_base, 2572289774Scem sysctl_handle_register, "IU", 2573289774Scem "Secondary BAR5 base register"); 2574289774Scem } else { 2575289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2576289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2577289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_base, 2578289774Scem sysctl_handle_register, "QU", 2579289774Scem "Secondary BAR45 base register"); 2580289774Scem } 2581289774Scem} 2582289774Scem 2583289774Scemstatic int 2584289774Scemsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2585289774Scem{ 2586289774Scem struct ntb_softc *ntb; 2587289774Scem struct sbuf sb; 2588289774Scem int error; 2589289774Scem 2590289774Scem error = 0; 2591289774Scem ntb = arg1; 2592289774Scem 2593289774Scem sbuf_new_for_sysctl(&sb, NULL, 256, req); 2594289774Scem 2595289774Scem sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2596289774Scem error = sbuf_finish(&sb); 2597289774Scem sbuf_delete(&sb); 2598289774Scem 2599289774Scem if (error || !req->newptr) 2600289774Scem return (error); 2601289774Scem return (EINVAL); 2602289774Scem} 2603289774Scem 2604289774Scemstatic int 2605300100Scemsysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2606289774Scem{ 2607289774Scem struct ntb_softc *ntb; 2608300100Scem unsigned old, new; 2609300100Scem int error; 2610300100Scem 2611300100Scem error = 0; 2612300100Scem ntb = arg1; 2613300100Scem 2614300100Scem old = ntb_link_enabled(ntb); 2615300100Scem 2616300100Scem error = SYSCTL_OUT(req, &old, sizeof(old)); 2617300100Scem if (error != 0 || req->newptr == NULL) 2618300100Scem return (error); 2619300100Scem 2620300100Scem error = SYSCTL_IN(req, &new, sizeof(new)); 2621300100Scem if (error != 0) 2622300100Scem return (error); 2623300100Scem 2624300100Scem ntb_printf(0, "Admin set interface state to '%sabled'\n", 2625300100Scem (new != 0)? "en" : "dis"); 2626300100Scem 2627300100Scem if (new != 0) 2628300100Scem error = ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2629300100Scem else 2630300100Scem error = ntb_link_disable(ntb); 2631300100Scem return (error); 2632300100Scem} 2633300100Scem 2634300100Scemstatic int 2635300100Scemsysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2636300100Scem{ 2637300100Scem struct ntb_softc *ntb; 2638289774Scem struct sbuf sb; 2639289774Scem enum ntb_speed speed; 2640289774Scem enum ntb_width width; 2641289774Scem int error; 2642289774Scem 2643289774Scem error = 0; 2644289774Scem ntb = arg1; 2645289774Scem 2646289774Scem sbuf_new_for_sysctl(&sb, NULL, 32, req); 2647289774Scem 2648289774Scem if (ntb_link_is_up(ntb, &speed, &width)) 2649289774Scem sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2650289774Scem (unsigned)speed, (unsigned)width); 2651289774Scem else 2652289774Scem sbuf_printf(&sb, "down"); 2653289774Scem 2654289774Scem error = sbuf_finish(&sb); 2655289774Scem sbuf_delete(&sb); 2656289774Scem 2657289774Scem if (error || !req->newptr) 2658289774Scem return (error); 2659289774Scem return (EINVAL); 2660289774Scem} 2661289774Scem 2662289774Scemstatic int 2663300100Scemsysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2664300100Scem{ 2665300100Scem struct ntb_softc *ntb; 2666300100Scem unsigned res; 2667300100Scem int error; 2668300100Scem 2669300100Scem error = 0; 2670300100Scem ntb = arg1; 2671300100Scem 2672300100Scem res = ntb_link_is_up(ntb, NULL, NULL); 2673300100Scem 2674300100Scem error = SYSCTL_OUT(req, &res, sizeof(res)); 2675300100Scem if (error || !req->newptr) 2676300100Scem return (error); 2677300100Scem return (EINVAL); 2678300100Scem} 2679300100Scem 2680300100Scemstatic int 2681289774Scemsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2682289774Scem{ 2683289774Scem struct ntb_softc *ntb; 2684289774Scem const void *outp; 2685289774Scem uintptr_t sz; 2686289774Scem uint64_t umv; 2687289774Scem char be[sizeof(umv)]; 2688289774Scem size_t outsz; 2689289774Scem uint32_t reg; 2690289774Scem bool db, pci; 2691289774Scem int error; 2692289774Scem 2693289774Scem ntb = arg1; 2694289774Scem reg = arg2 & ~NTB_REGFLAGS_MASK; 2695289774Scem sz = arg2 & NTB_REGSZ_MASK; 2696289774Scem db = (arg2 & NTB_DB_READ) != 0; 2697289774Scem pci = (arg2 & NTB_PCI_REG) != 0; 2698289774Scem 2699289774Scem KASSERT(!(db && pci), ("bogus")); 2700289774Scem 2701289774Scem if (db) { 2702289774Scem KASSERT(sz == NTB_REG_64, ("bogus")); 2703289774Scem umv = db_ioread(ntb, reg); 2704289774Scem outsz = sizeof(uint64_t); 2705289774Scem } else { 2706289774Scem switch (sz) { 2707289774Scem case NTB_REG_64: 2708289774Scem if (pci) 2709289774Scem umv = pci_read_config(ntb->device, reg, 8); 2710289774Scem else 2711289774Scem umv = ntb_reg_read(8, reg); 2712289774Scem outsz = sizeof(uint64_t); 2713289774Scem break; 2714289774Scem case NTB_REG_32: 2715289774Scem if (pci) 2716289774Scem umv = pci_read_config(ntb->device, reg, 4); 2717289774Scem else 2718289774Scem umv = ntb_reg_read(4, reg); 2719289774Scem outsz = sizeof(uint32_t); 2720289774Scem break; 2721289774Scem case NTB_REG_16: 2722289774Scem if (pci) 2723289774Scem umv = pci_read_config(ntb->device, reg, 2); 2724289774Scem else 2725289774Scem umv = ntb_reg_read(2, reg); 2726289774Scem outsz = sizeof(uint16_t); 2727289774Scem break; 2728289774Scem case NTB_REG_8: 2729289774Scem if (pci) 2730289774Scem umv = pci_read_config(ntb->device, reg, 1); 2731289774Scem else 2732289774Scem umv = ntb_reg_read(1, reg); 2733289774Scem outsz = sizeof(uint8_t); 2734289774Scem break; 2735289774Scem default: 2736289774Scem panic("bogus"); 2737289774Scem break; 2738289774Scem } 2739289774Scem } 2740289774Scem 2741289774Scem /* Encode bigendian so that sysctl -x is legible. */ 2742289774Scem be64enc(be, umv); 2743289774Scem outp = ((char *)be) + sizeof(umv) - outsz; 2744289774Scem 2745289774Scem error = SYSCTL_OUT(req, outp, outsz); 2746289774Scem if (error || !req->newptr) 2747289774Scem return (error); 2748289774Scem return (EINVAL); 2749289774Scem} 2750289774Scem 2751291263Scemstatic unsigned 2752291263Scemntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2753291263Scem{ 2754291263Scem 2755295618Scem if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2756295618Scem uidx >= ntb->b2b_mw_idx) || 2757295618Scem (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2758295618Scem uidx++; 2759295618Scem if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2760295618Scem uidx >= ntb->b2b_mw_idx) && 2761295618Scem (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2762295618Scem uidx++; 2763291263Scem return (uidx); 2764291263Scem} 2765291263Scem 2766295618Scemstatic void 2767295618Scemntb_exchange_msix(void *ctx) 2768295618Scem{ 2769295618Scem struct ntb_softc *ntb; 2770295618Scem uint32_t val; 2771295618Scem unsigned i; 2772295618Scem 2773295618Scem ntb = ctx; 2774295618Scem 2775301292Smav if (ntb->peer_msix_good) 2776301292Smav goto msix_good; 2777295618Scem if (ntb->peer_msix_done) 2778295618Scem goto msix_done; 2779295618Scem 2780295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2781295618Scem ntb_peer_spad_write(ntb, NTB_MSIX_DATA0 + i, 2782295618Scem ntb->msix_data[i].nmd_data); 2783295618Scem ntb_peer_spad_write(ntb, NTB_MSIX_OFS0 + i, 2784301293Smav ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2785295618Scem } 2786295618Scem ntb_peer_spad_write(ntb, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2787295618Scem 2788295618Scem ntb_spad_read(ntb, NTB_MSIX_GUARD, &val); 2789295618Scem if (val != NTB_MSIX_VER_GUARD) 2790295618Scem goto reschedule; 2791295618Scem 2792295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2793295618Scem ntb_spad_read(ntb, NTB_MSIX_DATA0 + i, &val); 2794301293Smav ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2795295618Scem ntb->peer_msix_data[i].nmd_data = val; 2796295618Scem ntb_spad_read(ntb, NTB_MSIX_OFS0 + i, &val); 2797301293Smav ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2798295618Scem ntb->peer_msix_data[i].nmd_ofs = val; 2799295618Scem } 2800295618Scem 2801295618Scem ntb->peer_msix_done = true; 2802295618Scem 2803295618Scemmsix_done: 2804295618Scem ntb_peer_spad_write(ntb, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2805295618Scem ntb_spad_read(ntb, NTB_MSIX_DONE, &val); 2806295618Scem if (val != NTB_MSIX_RECEIVED) 2807295618Scem goto reschedule; 2808295618Scem 2809295618Scem ntb->peer_msix_good = true; 2810301292Smav /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2811301292Smav goto reschedule; 2812295618Scem 2813301292Smavmsix_good: 2814295618Scem ntb_poll_link(ntb); 2815295618Scem ntb_link_event(ntb); 2816295618Scem return; 2817295618Scem 2818295618Scemreschedule: 2819295618Scem ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2820301292Smav if (_xeon_link_is_up(ntb)) { 2821301292Smav callout_reset(&ntb->peer_msix_work, 2822301292Smav hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2823301292Smav ntb_exchange_msix, ntb); 2824301292Smav } else 2825295618Scem ntb_spad_clear(ntb); 2826295618Scem} 2827295618Scem 2828289546Scem/* 2829289546Scem * Public API to the rest of the OS 2830250079Scarl */ 2831250079Scarl 2832250079Scarl/** 2833250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 2834250079Scarl * @ntb: pointer to ntb_softc instance 2835250079Scarl * 2836250079Scarl * This function returns the max 32bit scratchpad registers usable by the 2837250079Scarl * upper layer. 2838250079Scarl * 2839250079Scarl * RETURNS: total number of scratch pad registers available 2840250079Scarl */ 2841289208Scemuint8_t 2842250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 2843250079Scarl{ 2844250079Scarl 2845289539Scem return (ntb->spad_count); 2846250079Scarl} 2847250079Scarl 2848291263Scem/* 2849291263Scem * ntb_mw_count() - Get the number of memory windows available for KPI 2850291263Scem * consumers. 2851291263Scem * 2852291263Scem * (Excludes any MW wholly reserved for register access.) 2853291263Scem */ 2854289396Scemuint8_t 2855289539Scemntb_mw_count(struct ntb_softc *ntb) 2856289396Scem{ 2857295618Scem uint8_t res; 2858289396Scem 2859295618Scem res = ntb->mw_count; 2860291263Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2861295618Scem res--; 2862295618Scem if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2863295618Scem res--; 2864295618Scem return (res); 2865289396Scem} 2866289396Scem 2867250079Scarl/** 2868289545Scem * ntb_spad_write() - write to the secondary scratchpad register 2869250079Scarl * @ntb: pointer to ntb_softc instance 2870250079Scarl * @idx: index to the scratchpad register, 0 based 2871250079Scarl * @val: the data value to put into the register 2872250079Scarl * 2873250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2874250079Scarl * register. The register resides on the secondary (external) side. 2875250079Scarl * 2876289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2877250079Scarl */ 2878250079Scarlint 2879289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2880250079Scarl{ 2881250079Scarl 2882289539Scem if (idx >= ntb->spad_count) 2883250079Scarl return (EINVAL); 2884250079Scarl 2885289607Scem ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2886250079Scarl 2887250079Scarl return (0); 2888250079Scarl} 2889250079Scarl 2890295618Scem/* 2891295618Scem * Zeros the local scratchpad. 2892295618Scem */ 2893295618Scemvoid 2894295618Scemntb_spad_clear(struct ntb_softc *ntb) 2895295618Scem{ 2896295618Scem unsigned i; 2897295618Scem 2898295618Scem for (i = 0; i < ntb->spad_count; i++) 2899295618Scem ntb_spad_write(ntb, i, 0); 2900295618Scem} 2901295618Scem 2902250079Scarl/** 2903289545Scem * ntb_spad_read() - read from the primary scratchpad register 2904250079Scarl * @ntb: pointer to ntb_softc instance 2905250079Scarl * @idx: index to scratchpad register, 0 based 2906250079Scarl * @val: pointer to 32bit integer for storing the register value 2907250079Scarl * 2908250079Scarl * This function allows reading of the 32bit scratchpad register on 2909250079Scarl * the primary (internal) side. 2910250079Scarl * 2911289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2912250079Scarl */ 2913250079Scarlint 2914289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2915250079Scarl{ 2916250079Scarl 2917289539Scem if (idx >= ntb->spad_count) 2918250079Scarl return (EINVAL); 2919250079Scarl 2920289607Scem *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2921250079Scarl 2922250079Scarl return (0); 2923250079Scarl} 2924250079Scarl 2925250079Scarl/** 2926289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register 2927250079Scarl * @ntb: pointer to ntb_softc instance 2928250079Scarl * @idx: index to the scratchpad register, 0 based 2929250079Scarl * @val: the data value to put into the register 2930250079Scarl * 2931250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2932250079Scarl * register. The register resides on the secondary (external) side. 2933250079Scarl * 2934289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2935250079Scarl */ 2936250079Scarlint 2937289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2938250079Scarl{ 2939250079Scarl 2940289539Scem if (idx >= ntb->spad_count) 2941250079Scarl return (EINVAL); 2942250079Scarl 2943289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2944290682Scem ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2945255279Scarl else 2946289542Scem ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2947250079Scarl 2948250079Scarl return (0); 2949250079Scarl} 2950250079Scarl 2951250079Scarl/** 2952289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register 2953250079Scarl * @ntb: pointer to ntb_softc instance 2954250079Scarl * @idx: index to scratchpad register, 0 based 2955250079Scarl * @val: pointer to 32bit integer for storing the register value 2956250079Scarl * 2957250079Scarl * This function allows reading of the 32bit scratchpad register on 2958250079Scarl * the primary (internal) side. 2959250079Scarl * 2960289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2961250079Scarl */ 2962250079Scarlint 2963289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2964250079Scarl{ 2965250079Scarl 2966289539Scem if (idx >= ntb->spad_count) 2967250079Scarl return (EINVAL); 2968250079Scarl 2969289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2970290682Scem *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2971255279Scarl else 2972289542Scem *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2973250079Scarl 2974250079Scarl return (0); 2975250079Scarl} 2976250079Scarl 2977289546Scem/* 2978289546Scem * ntb_mw_get_range() - get the range of a memory window 2979289546Scem * @ntb: NTB device context 2980289546Scem * @idx: Memory window number 2981289546Scem * @base: OUT - the base address for mapping the memory window 2982289546Scem * @size: OUT - the size for mapping the memory window 2983289546Scem * @align: OUT - the base alignment for translating the memory window 2984289546Scem * @align_size: OUT - the size alignment for translating the memory window 2985250079Scarl * 2986289546Scem * Get the range of a memory window. NULL may be given for any output 2987289546Scem * parameter if the value is not needed. The base and size may be used for 2988289546Scem * mapping the memory window, to access the peer memory. The alignment and 2989289546Scem * size may be used for translating the memory window, for the peer to access 2990289546Scem * memory on the local system. 2991250079Scarl * 2992289546Scem * Return: Zero on success, otherwise an error number. 2993250079Scarl */ 2994289546Scemint 2995289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 2996291033Scem caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2997291033Scem bus_addr_t *plimit) 2998250079Scarl{ 2999289546Scem struct ntb_pci_bar_info *bar; 3000291033Scem bus_addr_t limit; 3001289546Scem size_t bar_b2b_off; 3002291033Scem enum ntb_bar bar_num; 3003250079Scarl 3004289546Scem if (mw_idx >= ntb_mw_count(ntb)) 3005289546Scem return (EINVAL); 3006291263Scem mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 3007250079Scarl 3008291033Scem bar_num = ntb_mw_to_bar(ntb, mw_idx); 3009291033Scem bar = &ntb->bar_info[bar_num]; 3010289546Scem bar_b2b_off = 0; 3011289546Scem if (mw_idx == ntb->b2b_mw_idx) { 3012289546Scem KASSERT(ntb->b2b_off != 0, 3013289546Scem ("user shouldn't get non-shared b2b mw")); 3014289546Scem bar_b2b_off = ntb->b2b_off; 3015289546Scem } 3016250079Scarl 3017291033Scem if (bar_is_64bit(ntb, bar_num)) 3018291033Scem limit = BUS_SPACE_MAXADDR; 3019291033Scem else 3020291033Scem limit = BUS_SPACE_MAXADDR_32BIT; 3021291033Scem 3022289546Scem if (base != NULL) 3023289546Scem *base = bar->pbase + bar_b2b_off; 3024289546Scem if (vbase != NULL) 3025290679Scem *vbase = bar->vbase + bar_b2b_off; 3026289546Scem if (size != NULL) 3027289546Scem *size = bar->size - bar_b2b_off; 3028289546Scem if (align != NULL) 3029289546Scem *align = bar->size; 3030289546Scem if (align_size != NULL) 3031289546Scem *align_size = 1; 3032291033Scem if (plimit != NULL) 3033291033Scem *plimit = limit; 3034289546Scem return (0); 3035250079Scarl} 3036250079Scarl 3037289546Scem/* 3038289546Scem * ntb_mw_set_trans() - set the translation of a memory window 3039289546Scem * @ntb: NTB device context 3040289546Scem * @idx: Memory window number 3041289546Scem * @addr: The dma address local memory to expose to the peer 3042289546Scem * @size: The size of the local memory to expose to the peer 3043250079Scarl * 3044289546Scem * Set the translation of a memory window. The peer may access local memory 3045289546Scem * through the window starting at the address, up to the size. The address 3046289546Scem * must be aligned to the alignment specified by ntb_mw_get_range(). The size 3047291033Scem * must be aligned to the size alignment specified by ntb_mw_get_range(). The 3048291033Scem * address must be below the plimit specified by ntb_mw_get_range() (i.e. for 3049291033Scem * 32-bit BARs). 3050250079Scarl * 3051289546Scem * Return: Zero on success, otherwise an error number. 3052250079Scarl */ 3053289546Scemint 3054289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 3055289546Scem size_t size) 3056250079Scarl{ 3057289546Scem struct ntb_pci_bar_info *bar; 3058289546Scem uint64_t base, limit, reg_val; 3059289546Scem size_t bar_size, mw_size; 3060289546Scem uint32_t base_reg, xlat_reg, limit_reg; 3061289546Scem enum ntb_bar bar_num; 3062250079Scarl 3063289546Scem if (idx >= ntb_mw_count(ntb)) 3064289546Scem return (EINVAL); 3065291263Scem idx = ntb_user_mw_to_idx(ntb, idx); 3066250079Scarl 3067289546Scem bar_num = ntb_mw_to_bar(ntb, idx); 3068289546Scem bar = &ntb->bar_info[bar_num]; 3069250079Scarl 3070289546Scem bar_size = bar->size; 3071289546Scem if (idx == ntb->b2b_mw_idx) 3072289546Scem mw_size = bar_size - ntb->b2b_off; 3073289546Scem else 3074289546Scem mw_size = bar_size; 3075250079Scarl 3076289546Scem /* Hardware requires that addr is aligned to bar size */ 3077289546Scem if ((addr & (bar_size - 1)) != 0) 3078289546Scem return (EINVAL); 3079250079Scarl 3080289546Scem if (size > mw_size) 3081289546Scem return (EINVAL); 3082289546Scem 3083289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 3084289546Scem 3085289546Scem limit = 0; 3086289546Scem if (bar_is_64bit(ntb, bar_num)) { 3087291032Scem base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 3088289546Scem 3089289546Scem if (limit_reg != 0 && size != mw_size) 3090289546Scem limit = base + size; 3091289546Scem 3092289546Scem /* Set and verify translation address */ 3093289546Scem ntb_reg_write(8, xlat_reg, addr); 3094291032Scem reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 3095289546Scem if (reg_val != addr) { 3096289546Scem ntb_reg_write(8, xlat_reg, 0); 3097289546Scem return (EIO); 3098289546Scem } 3099289546Scem 3100289546Scem /* Set and verify the limit */ 3101289546Scem ntb_reg_write(8, limit_reg, limit); 3102291032Scem reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3103289546Scem if (reg_val != limit) { 3104289546Scem ntb_reg_write(8, limit_reg, base); 3105289546Scem ntb_reg_write(8, xlat_reg, 0); 3106289546Scem return (EIO); 3107289546Scem } 3108289546Scem } else { 3109289546Scem /* Configure 32-bit (split) BAR MW */ 3110289546Scem 3111291029Scem if ((addr & UINT32_MAX) != addr) 3112291033Scem return (ERANGE); 3113291029Scem if (((addr + size) & UINT32_MAX) != (addr + size)) 3114291033Scem return (ERANGE); 3115289546Scem 3116291032Scem base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3117289546Scem 3118289546Scem if (limit_reg != 0 && size != mw_size) 3119289546Scem limit = base + size; 3120289546Scem 3121289546Scem /* Set and verify translation address */ 3122289546Scem ntb_reg_write(4, xlat_reg, addr); 3123291032Scem reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3124289546Scem if (reg_val != addr) { 3125289546Scem ntb_reg_write(4, xlat_reg, 0); 3126289546Scem return (EIO); 3127289546Scem } 3128289546Scem 3129289546Scem /* Set and verify the limit */ 3130289546Scem ntb_reg_write(4, limit_reg, limit); 3131291032Scem reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3132289546Scem if (reg_val != limit) { 3133289546Scem ntb_reg_write(4, limit_reg, base); 3134289546Scem ntb_reg_write(4, xlat_reg, 0); 3135289546Scem return (EIO); 3136289546Scem } 3137250079Scarl } 3138289546Scem return (0); 3139250079Scarl} 3140250079Scarl 3141289596Scem/* 3142289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window 3143289596Scem * @ntb: NTB device context 3144289596Scem * @idx: Memory window number 3145289596Scem * 3146289596Scem * Clear the translation of a memory window. The peer may no longer access 3147289596Scem * local memory through the window. 3148289596Scem * 3149289596Scem * Return: Zero on success, otherwise an error number. 3150289596Scem */ 3151289596Scemint 3152289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 3153289596Scem{ 3154289596Scem 3155289596Scem return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 3156289596Scem} 3157289596Scem 3158291031Scem/* 3159291031Scem * ntb_mw_get_wc - Get the write-combine status of a memory window 3160291031Scem * 3161291031Scem * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 3162291031Scem * idx is an invalid memory window). 3163291280Scem * 3164291280Scem * Mode is a VM_MEMATTR_* type. 3165291031Scem */ 3166291031Scemint 3167291280Scemntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t *mode) 3168291031Scem{ 3169291031Scem struct ntb_pci_bar_info *bar; 3170291031Scem 3171291031Scem if (idx >= ntb_mw_count(ntb)) 3172291031Scem return (EINVAL); 3173291263Scem idx = ntb_user_mw_to_idx(ntb, idx); 3174291031Scem 3175291031Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3176291280Scem *mode = bar->map_mode; 3177291031Scem return (0); 3178291031Scem} 3179291031Scem 3180291031Scem/* 3181291031Scem * ntb_mw_set_wc - Set the write-combine status of a memory window 3182291031Scem * 3183291280Scem * If 'mode' matches the current status, this does nothing and succeeds. Mode 3184291280Scem * is a VM_MEMATTR_* type. 3185291031Scem * 3186291031Scem * Returns: Zero on success, setting the caching attribute on the virtual 3187291031Scem * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 3188291031Scem * memory window, or if changing the caching attribute fails). 3189291031Scem */ 3190291031Scemint 3191291280Scemntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3192291031Scem{ 3193291263Scem 3194291263Scem if (idx >= ntb_mw_count(ntb)) 3195291263Scem return (EINVAL); 3196291263Scem 3197291263Scem idx = ntb_user_mw_to_idx(ntb, idx); 3198291280Scem return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3199291263Scem} 3200291263Scem 3201291263Scemstatic int 3202291280Scemntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3203291263Scem{ 3204291031Scem struct ntb_pci_bar_info *bar; 3205291031Scem int rc; 3206291031Scem 3207291031Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3208291280Scem if (bar->map_mode == mode) 3209291031Scem return (0); 3210291031Scem 3211291280Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3212291031Scem if (rc == 0) 3213291280Scem bar->map_mode = mode; 3214291031Scem 3215291031Scem return (rc); 3216291031Scem} 3217291031Scem 3218250079Scarl/** 3219289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side 3220250079Scarl * @ntb: pointer to ntb_softc instance 3221289545Scem * @bit: doorbell bits to ring 3222250079Scarl * 3223250079Scarl * This function allows triggering of a doorbell on the secondary/external 3224250079Scarl * side that will initiate an interrupt on the remote host 3225250079Scarl */ 3226250079Scarlvoid 3227289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 3228250079Scarl{ 3229250079Scarl 3230295618Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 3231295618Scem struct ntb_pci_bar_info *lapic; 3232295618Scem unsigned i; 3233295618Scem 3234295618Scem lapic = ntb->peer_lapic_bar; 3235295618Scem 3236295618Scem for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3237295618Scem if ((bit & ntb_db_vector_mask(ntb, i)) != 0) 3238295618Scem bus_space_write_4(lapic->pci_bus_tag, 3239295618Scem lapic->pci_bus_handle, 3240295618Scem ntb->peer_msix_data[i].nmd_ofs, 3241295618Scem ntb->peer_msix_data[i].nmd_data); 3242295618Scem } 3243295618Scem return; 3244295618Scem } 3245295618Scem 3246289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3247290682Scem ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3248289347Scem return; 3249289209Scem } 3250289347Scem 3251289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3252250079Scarl} 3253250079Scarl 3254289542Scem/* 3255289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 3256289542Scem * as well as the size of the register (via *sz_out). 3257289542Scem * 3258289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell 3259289542Scem * ring to its memory window write. 3260289542Scem * 3261289542Scem * Note that writing the peer doorbell via a memory window will *not* generate 3262298955Spfg * an interrupt on the remote host; that must be done separately. 3263289542Scem */ 3264289542Scembus_addr_t 3265289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 3266289542Scem{ 3267289542Scem struct ntb_pci_bar_info *bar; 3268289542Scem uint64_t regoff; 3269289542Scem 3270289542Scem KASSERT(sz_out != NULL, ("must be non-NULL")); 3271289542Scem 3272289542Scem if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3273289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3274289542Scem regoff = ntb->peer_reg->db_bell; 3275289542Scem } else { 3276289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3277289543Scem ("invalid b2b idx")); 3278289542Scem 3279289542Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3280290682Scem regoff = XEON_PDOORBELL_OFFSET; 3281289542Scem } 3282289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3283289542Scem 3284289542Scem *sz_out = ntb->reg->db_size; 3285289542Scem /* HACK: Specific to current x86 bus implementation. */ 3286289542Scem return ((uint64_t)bar->pci_bus_handle + regoff); 3287289542Scem} 3288289542Scem 3289289597Scem/* 3290289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 3291289597Scem * @ntb: NTB device context 3292289597Scem * 3293289597Scem * Hardware may support different number or arrangement of doorbell bits. 3294289597Scem * 3295289597Scem * Return: A mask of doorbell bits supported by the ntb. 3296289597Scem */ 3297289597Scemuint64_t 3298289597Scemntb_db_valid_mask(struct ntb_softc *ntb) 3299289597Scem{ 3300289597Scem 3301289597Scem return (ntb->db_valid_mask); 3302289597Scem} 3303289597Scem 3304289598Scem/* 3305289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 3306289598Scem * @ntb: NTB device context 3307289598Scem * @vector: Doorbell vector number 3308289598Scem * 3309289598Scem * Each interrupt vector may have a different number or arrangement of bits. 3310289598Scem * 3311289598Scem * Return: A mask of doorbell bits serviced by a vector. 3312289598Scem */ 3313289598Scemuint64_t 3314289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 3315289598Scem{ 3316289598Scem 3317289598Scem if (vector > ntb->db_vec_count) 3318289598Scem return (0); 3319289598Scem return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3320289598Scem} 3321289598Scem 3322250079Scarl/** 3323289546Scem * ntb_link_is_up() - get the current ntb link state 3324289546Scem * @ntb: NTB device context 3325289546Scem * @speed: OUT - The link speed expressed as PCIe generation number 3326289546Scem * @width: OUT - The link width expressed as the number of PCIe lanes 3327250079Scarl * 3328250079Scarl * RETURNS: true or false based on the hardware link state 3329250079Scarl */ 3330250079Scarlbool 3331289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 3332289546Scem enum ntb_width *width) 3333250079Scarl{ 3334250079Scarl 3335289546Scem if (speed != NULL) 3336289546Scem *speed = ntb_link_sta_speed(ntb); 3337289546Scem if (width != NULL) 3338289546Scem *width = ntb_link_sta_width(ntb); 3339289546Scem return (link_is_up(ntb)); 3340250079Scarl} 3341250079Scarl 3342255272Scarlstatic void 3343255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 3344250079Scarl{ 3345255272Scarl 3346289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3347289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3348289209Scem bar->pbase = rman_get_start(bar->pci_resource); 3349289209Scem bar->size = rman_get_size(bar->pci_resource); 3350289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 3351250079Scarl} 3352255268Scarl 3353289209Scemdevice_t 3354289209Scemntb_get_device(struct ntb_softc *ntb) 3355255268Scarl{ 3356255268Scarl 3357255268Scarl return (ntb->device); 3358255268Scarl} 3359289208Scem 3360289208Scem/* Export HW-specific errata information. */ 3361289208Scembool 3362289774Scemntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 3363289208Scem{ 3364289208Scem 3365289208Scem return (HAS_FEATURE(feature)); 3366289208Scem} 3367