ntb_hw.c revision 300516
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3300373Smav * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c 300516 2016-05-23 16:12:11Z mav $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35300373Smav#include <sys/endian.h> 36250079Scarl#include <sys/malloc.h> 37250079Scarl#include <sys/module.h> 38250079Scarl#include <sys/queue.h> 39250079Scarl#include <sys/rman.h> 40300373Smav#include <sys/sbuf.h> 41300373Smav#include <sys/sysctl.h> 42250079Scarl#include <vm/vm.h> 43250079Scarl#include <vm/pmap.h> 44250079Scarl#include <machine/bus.h> 45250079Scarl#include <machine/pmap.h> 46250079Scarl#include <machine/resource.h> 47250079Scarl#include <dev/pci/pcireg.h> 48250079Scarl#include <dev/pci/pcivar.h> 49250079Scarl 50250079Scarl#include "ntb_regs.h" 51250079Scarl#include "ntb_hw.h" 52250079Scarl 53250079Scarl/* 54250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 55250079Scarl * allows you to connect two systems using a PCI-e link. 56250079Scarl * 57250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 58250079Scarl * you to send and recieve interrupts, map the memory windows and send and 59250079Scarl * receive messages in the scratch-pad registers. 60250079Scarl * 61250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 62250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 63250079Scarl */ 64250079Scarl 65300373Smav#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 66250079Scarl 67300373Smav#define NTB_HB_TIMEOUT 1 /* second */ 68300373Smav#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 69300373Smav#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 70250079Scarl 71250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 72250079Scarl 73250079Scarlenum ntb_device_type { 74250079Scarl NTB_XEON, 75300373Smav NTB_ATOM 76250079Scarl}; 77250079Scarl 78300373Smav/* ntb_conn_type are hardware numbers, cannot change. */ 79300373Smavenum ntb_conn_type { 80300373Smav NTB_CONN_TRANSPARENT = 0, 81300373Smav NTB_CONN_B2B = 1, 82300373Smav NTB_CONN_RP = 2, 83300373Smav}; 84300373Smav 85300373Smavenum ntb_b2b_direction { 86300373Smav NTB_DEV_USD = 0, 87300373Smav NTB_DEV_DSD = 1, 88300373Smav}; 89300373Smav 90300373Smavenum ntb_bar { 91300373Smav NTB_CONFIG_BAR = 0, 92300373Smav NTB_B2B_BAR_1, 93300373Smav NTB_B2B_BAR_2, 94300373Smav NTB_B2B_BAR_3, 95300373Smav NTB_MAX_BARS 96300373Smav}; 97300373Smav 98255274Scarl/* Device features and workarounds */ 99255274Scarl#define HAS_FEATURE(feature) \ 100255274Scarl ((ntb->features & (feature)) != 0) 101255274Scarl 102250079Scarlstruct ntb_hw_info { 103250079Scarl uint32_t device_id; 104255274Scarl const char *desc; 105250079Scarl enum ntb_device_type type; 106300373Smav uint32_t features; 107250079Scarl}; 108250079Scarl 109250079Scarlstruct ntb_pci_bar_info { 110250079Scarl bus_space_tag_t pci_bus_tag; 111250079Scarl bus_space_handle_t pci_bus_handle; 112250079Scarl int pci_resource_id; 113250079Scarl struct resource *pci_resource; 114250079Scarl vm_paddr_t pbase; 115300373Smav caddr_t vbase; 116300373Smav vm_size_t size; 117300373Smav vm_memattr_t map_mode; 118300373Smav 119300373Smav /* Configuration register offsets */ 120300373Smav uint32_t psz_off; 121300373Smav uint32_t ssz_off; 122300373Smav uint32_t pbarxlat_off; 123250079Scarl}; 124250079Scarl 125250079Scarlstruct ntb_int_info { 126250079Scarl struct resource *res; 127250079Scarl int rid; 128250079Scarl void *tag; 129250079Scarl}; 130250079Scarl 131300373Smavstruct ntb_vec { 132250079Scarl struct ntb_softc *ntb; 133300373Smav uint32_t num; 134250079Scarl}; 135250079Scarl 136300373Smavstruct ntb_reg { 137300373Smav uint32_t ntb_ctl; 138300373Smav uint32_t lnk_sta; 139300373Smav uint8_t db_size; 140300373Smav unsigned mw_bar[NTB_MAX_BARS]; 141300373Smav}; 142300373Smav 143300373Smavstruct ntb_alt_reg { 144300373Smav uint32_t db_bell; 145300373Smav uint32_t db_mask; 146300373Smav uint32_t spad; 147300373Smav}; 148300373Smav 149300373Smavstruct ntb_xlat_reg { 150300373Smav uint32_t bar0_base; 151300373Smav uint32_t bar2_base; 152300373Smav uint32_t bar4_base; 153300373Smav uint32_t bar5_base; 154300373Smav 155300373Smav uint32_t bar2_xlat; 156300373Smav uint32_t bar4_xlat; 157300373Smav uint32_t bar5_xlat; 158300373Smav 159300373Smav uint32_t bar2_limit; 160300373Smav uint32_t bar4_limit; 161300373Smav uint32_t bar5_limit; 162300373Smav}; 163300373Smav 164300373Smavstruct ntb_b2b_addr { 165300373Smav uint64_t bar0_addr; 166300373Smav uint64_t bar2_addr64; 167300373Smav uint64_t bar4_addr64; 168300373Smav uint64_t bar4_addr32; 169300373Smav uint64_t bar5_addr32; 170300373Smav}; 171300373Smav 172250079Scarlstruct ntb_softc { 173250079Scarl device_t device; 174250079Scarl enum ntb_device_type type; 175300373Smav uint32_t features; 176250079Scarl 177250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 178250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 179250079Scarl uint32_t allocated_interrupts; 180250079Scarl 181250079Scarl struct callout heartbeat_timer; 182250079Scarl struct callout lr_timer; 183250079Scarl 184300373Smav void *ntb_ctx; 185300373Smav const struct ntb_ctx_ops *ctx_ops; 186300373Smav struct ntb_vec *msix_vec; 187300373Smav#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 188300373Smav#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 189300373Smav#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 190300373Smav struct mtx ctx_lock; 191250079Scarl 192300373Smav uint32_t ppd; 193300373Smav enum ntb_conn_type conn_type; 194300373Smav enum ntb_b2b_direction dev_type; 195300373Smav 196300373Smav /* Offset of peer bar0 in B2B BAR */ 197300373Smav uint64_t b2b_off; 198300373Smav /* Memory window used to access peer bar0 */ 199300373Smav#define B2B_MW_DISABLED UINT8_MAX 200300373Smav uint8_t b2b_mw_idx; 201300373Smav 202300373Smav uint8_t mw_count; 203300373Smav uint8_t spad_count; 204300373Smav uint8_t db_count; 205300373Smav uint8_t db_vec_count; 206300373Smav uint8_t db_vec_shift; 207300373Smav 208300373Smav /* Protects local db_mask. */ 209300373Smav#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 210300373Smav#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 211300373Smav#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 212300373Smav struct mtx db_mask_lock; 213300373Smav 214300373Smav volatile uint32_t ntb_ctl; 215300373Smav volatile uint32_t lnk_sta; 216300373Smav 217300373Smav uint64_t db_valid_mask; 218300373Smav uint64_t db_link_mask; 219300373Smav uint64_t db_mask; 220300373Smav 221300373Smav int last_ts; /* ticks @ last irq */ 222300373Smav 223300373Smav const struct ntb_reg *reg; 224300373Smav const struct ntb_alt_reg *self_reg; 225300373Smav const struct ntb_alt_reg *peer_reg; 226300373Smav const struct ntb_xlat_reg *xlat_reg; 227250079Scarl}; 228250079Scarl 229300373Smav#ifdef __i386__ 230300373Smavstatic __inline uint64_t 231300373Smavbus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 232300373Smav bus_size_t offset) 233300373Smav{ 234300373Smav 235300373Smav return (bus_space_read_4(tag, handle, offset) | 236300373Smav ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 237300373Smav} 238300373Smav 239300373Smavstatic __inline void 240300373Smavbus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 241300373Smav bus_size_t offset, uint64_t val) 242300373Smav{ 243300373Smav 244300373Smav bus_space_write_4(tag, handle, offset, val); 245300373Smav bus_space_write_4(tag, handle, offset + 4, val >> 32); 246300373Smav} 247300373Smav#endif 248300373Smav 249255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 250255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 251255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 252255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 253255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 254255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 255255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 256250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 257255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 258300373Smav#define ntb_mw_read(SIZE, offset) \ 259300373Smav ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 260255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 261300373Smav ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 262300373Smav offset, val) 263250079Scarl 264250079Scarlstatic int ntb_probe(device_t device); 265250079Scarlstatic int ntb_attach(device_t device); 266250079Scarlstatic int ntb_detach(device_t device); 267300373Smavstatic unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 268300373Smavstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 269300373Smavstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 270300373Smavstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 271300373Smav uint32_t *base, uint32_t *xlat, uint32_t *lmt); 272255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 273300373Smavstatic int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 274300373Smav vm_memattr_t); 275300373Smavstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 276300373Smav const char *); 277255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 278255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 279255272Scarl struct ntb_pci_bar_info *bar); 280250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 281300373Smavstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 282300373Smavstatic int ntb_init_isr(struct ntb_softc *ntb); 283300373Smavstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 284300373Smavstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 285250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 286300373Smavstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 287300373Smavstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 288300373Smavstatic void ndev_vec_isr(void *arg); 289300373Smavstatic void ndev_irq_isr(void *arg); 290300373Smavstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 291300373Smavstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 292300373Smavstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 293300373Smavstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 294300373Smavstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 295250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 296300373Smavstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 297300373Smavstatic int ntb_detect_xeon(struct ntb_softc *ntb); 298300373Smavstatic int ntb_detect_atom(struct ntb_softc *ntb); 299300373Smavstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 300300373Smavstatic int ntb_atom_init_dev(struct ntb_softc *ntb); 301300373Smavstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 302300373Smavstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 303300373Smavstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 304300373Smav enum ntb_bar regbar); 305300373Smavstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 306300373Smav uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 307300373Smavstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 308300373Smav enum ntb_bar idx); 309300373Smavstatic int xeon_setup_b2b_mw(struct ntb_softc *, 310300373Smav const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 311300373Smavstatic inline bool link_is_up(struct ntb_softc *ntb); 312300373Smavstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 313300373Smavstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 314300373Smavstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 315300373Smavstatic void atom_link_hb(void *arg); 316300373Smavstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 317300373Smavstatic void recover_atom_link(void *arg); 318300373Smavstatic bool ntb_poll_link(struct ntb_softc *ntb); 319255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 320300373Smavstatic void ntb_sysctl_init(struct ntb_softc *); 321300373Smavstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 322300373Smavstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 323300373Smavstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 324250079Scarl 325300373Smavstatic unsigned g_ntb_hw_debug_level; 326300516SmavTUNABLE_INT("hw.ntb.debug_level", &g_ntb_hw_debug_level); 327300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 328300373Smav &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 329300373Smav#define ntb_printf(lvl, ...) do { \ 330300373Smav if ((lvl) <= g_ntb_hw_debug_level) { \ 331300373Smav device_printf(ntb->device, __VA_ARGS__); \ 332300373Smav } \ 333300373Smav} while (0) 334300373Smav 335300373Smav#define _NTB_PAT_UC 0 336300373Smav#define _NTB_PAT_WC 1 337300373Smav#define _NTB_PAT_WT 4 338300373Smav#define _NTB_PAT_WP 5 339300373Smav#define _NTB_PAT_WB 6 340300373Smav#define _NTB_PAT_UCM 7 341300373Smavstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 342300516SmavTUNABLE_INT("hw.ntb.default_mw_pat", &g_ntb_mw_pat); 343300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 344300373Smav &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 345300373Smav "UC: " __XSTRING(_NTB_PAT_UC) ", " 346300373Smav "WC: " __XSTRING(_NTB_PAT_WC) ", " 347300373Smav "WT: " __XSTRING(_NTB_PAT_WT) ", " 348300373Smav "WP: " __XSTRING(_NTB_PAT_WP) ", " 349300373Smav "WB: " __XSTRING(_NTB_PAT_WB) ", " 350300373Smav "UC-: " __XSTRING(_NTB_PAT_UCM)); 351300373Smav 352300373Smavstatic inline vm_memattr_t 353300373Smavntb_pat_flags(void) 354300373Smav{ 355300373Smav 356300373Smav switch (g_ntb_mw_pat) { 357300373Smav case _NTB_PAT_WC: 358300373Smav return (VM_MEMATTR_WRITE_COMBINING); 359300373Smav case _NTB_PAT_WT: 360300373Smav return (VM_MEMATTR_WRITE_THROUGH); 361300373Smav case _NTB_PAT_WP: 362300373Smav return (VM_MEMATTR_WRITE_PROTECTED); 363300373Smav case _NTB_PAT_WB: 364300373Smav return (VM_MEMATTR_WRITE_BACK); 365300373Smav case _NTB_PAT_UCM: 366300373Smav return (VM_MEMATTR_WEAK_UNCACHEABLE); 367300373Smav case _NTB_PAT_UC: 368300373Smav /* FALLTHROUGH */ 369300373Smav default: 370300373Smav return (VM_MEMATTR_UNCACHEABLE); 371300373Smav } 372300373Smav} 373300373Smav 374300373Smav/* 375300373Smav * Well, this obviously doesn't belong here, but it doesn't seem to exist 376300373Smav * anywhere better yet. 377300373Smav */ 378300373Smavstatic inline const char * 379300373Smavntb_vm_memattr_to_str(vm_memattr_t pat) 380300373Smav{ 381300373Smav 382300373Smav switch (pat) { 383300373Smav case VM_MEMATTR_WRITE_COMBINING: 384300373Smav return ("WRITE_COMBINING"); 385300373Smav case VM_MEMATTR_WRITE_THROUGH: 386300373Smav return ("WRITE_THROUGH"); 387300373Smav case VM_MEMATTR_WRITE_PROTECTED: 388300373Smav return ("WRITE_PROTECTED"); 389300373Smav case VM_MEMATTR_WRITE_BACK: 390300373Smav return ("WRITE_BACK"); 391300373Smav case VM_MEMATTR_WEAK_UNCACHEABLE: 392300373Smav return ("UNCACHED"); 393300373Smav case VM_MEMATTR_UNCACHEABLE: 394300373Smav return ("UNCACHEABLE"); 395300373Smav default: 396300373Smav return ("UNKNOWN"); 397300373Smav } 398300373Smav} 399300373Smav 400300373Smavstatic int g_ntb_mw_idx = -1; 401300516SmavTUNABLE_INT("hw.ntb.b2b_mw_idx", &g_ntb_mw_idx); 402300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 403300373Smav 0, "Use this memory window to access the peer NTB registers. A " 404300373Smav "non-negative value starts from the first MW index; a negative value " 405300373Smav "starts from the last MW index. The default is -1, i.e., the last " 406300373Smav "available memory window. Both sides of the NTB MUST set the same " 407300373Smav "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 408300373Smav 409250079Scarlstatic struct ntb_hw_info pci_ids[] = { 410300373Smav /* XXX: PS/SS IDs left out until they are supported. */ 411300373Smav { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 412300373Smav NTB_ATOM, 0 }, 413300373Smav 414300373Smav { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 415300373Smav NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 416300373Smav { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 417300373Smav NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 418300373Smav { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 419300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 420300373Smav NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 421300373Smav { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 422300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 423300373Smav NTB_SB01BASE_LOCKUP }, 424300373Smav { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 425300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 426300373Smav NTB_SB01BASE_LOCKUP }, 427300373Smav 428300373Smav { 0x00000000, NULL, NTB_ATOM, 0 } 429250079Scarl}; 430250079Scarl 431300373Smavstatic const struct ntb_reg atom_reg = { 432300373Smav .ntb_ctl = ATOM_NTBCNTL_OFFSET, 433300373Smav .lnk_sta = ATOM_LINK_STATUS_OFFSET, 434300373Smav .db_size = sizeof(uint64_t), 435300373Smav .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 436300373Smav}; 437300373Smav 438300373Smavstatic const struct ntb_alt_reg atom_pri_reg = { 439300373Smav .db_bell = ATOM_PDOORBELL_OFFSET, 440300373Smav .db_mask = ATOM_PDBMSK_OFFSET, 441300373Smav .spad = ATOM_SPAD_OFFSET, 442300373Smav}; 443300373Smav 444300373Smavstatic const struct ntb_alt_reg atom_b2b_reg = { 445300373Smav .db_bell = ATOM_B2B_DOORBELL_OFFSET, 446300373Smav .spad = ATOM_B2B_SPAD_OFFSET, 447300373Smav}; 448300373Smav 449300373Smavstatic const struct ntb_xlat_reg atom_sec_xlat = { 450300373Smav#if 0 451300373Smav /* "FIXME" says the Linux driver. */ 452300373Smav .bar0_base = ATOM_SBAR0BASE_OFFSET, 453300373Smav .bar2_base = ATOM_SBAR2BASE_OFFSET, 454300373Smav .bar4_base = ATOM_SBAR4BASE_OFFSET, 455300373Smav 456300373Smav .bar2_limit = ATOM_SBAR2LMT_OFFSET, 457300373Smav .bar4_limit = ATOM_SBAR4LMT_OFFSET, 458300373Smav#endif 459300373Smav 460300373Smav .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 461300373Smav .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 462300373Smav}; 463300373Smav 464300373Smavstatic const struct ntb_reg xeon_reg = { 465300373Smav .ntb_ctl = XEON_NTBCNTL_OFFSET, 466300373Smav .lnk_sta = XEON_LINK_STATUS_OFFSET, 467300373Smav .db_size = sizeof(uint16_t), 468300373Smav .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 469300373Smav}; 470300373Smav 471300373Smavstatic const struct ntb_alt_reg xeon_pri_reg = { 472300373Smav .db_bell = XEON_PDOORBELL_OFFSET, 473300373Smav .db_mask = XEON_PDBMSK_OFFSET, 474300373Smav .spad = XEON_SPAD_OFFSET, 475300373Smav}; 476300373Smav 477300373Smavstatic const struct ntb_alt_reg xeon_b2b_reg = { 478300373Smav .db_bell = XEON_B2B_DOORBELL_OFFSET, 479300373Smav .spad = XEON_B2B_SPAD_OFFSET, 480300373Smav}; 481300373Smav 482300373Smavstatic const struct ntb_xlat_reg xeon_sec_xlat = { 483300373Smav .bar0_base = XEON_SBAR0BASE_OFFSET, 484300373Smav .bar2_base = XEON_SBAR2BASE_OFFSET, 485300373Smav .bar4_base = XEON_SBAR4BASE_OFFSET, 486300373Smav .bar5_base = XEON_SBAR5BASE_OFFSET, 487300373Smav 488300373Smav .bar2_limit = XEON_SBAR2LMT_OFFSET, 489300373Smav .bar4_limit = XEON_SBAR4LMT_OFFSET, 490300373Smav .bar5_limit = XEON_SBAR5LMT_OFFSET, 491300373Smav 492300373Smav .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 493300373Smav .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 494300373Smav .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 495300373Smav}; 496300373Smav 497300373Smavstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 498300373Smav .bar0_addr = XEON_B2B_BAR0_ADDR, 499300373Smav .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 500300373Smav .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 501300373Smav .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 502300373Smav .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 503300373Smav}; 504300373Smav 505300373Smavstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 506300373Smav .bar0_addr = XEON_B2B_BAR0_ADDR, 507300373Smav .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 508300373Smav .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 509300373Smav .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 510300373Smav .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 511300373Smav}; 512300373Smav 513300373SmavSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 514300373Smav "B2B MW segment overrides -- MUST be the same on both sides"); 515300373Smav 516300516SmavTUNABLE_QUAD("hw.ntb.usd_bar2_addr64", &xeon_b2b_usd_addr.bar2_addr64); 517300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 518300373Smav &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 519300373Smav "hardware, use this 64-bit address on the bus between the NTB devices for " 520300373Smav "the window at BAR2, on the upstream side of the link. MUST be the same " 521300373Smav "address on both sides."); 522300516SmavTUNABLE_QUAD("hw.ntb.usd_bar4_addr64", &xeon_b2b_usd_addr.bar4_addr64); 523300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 524300373Smav &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 525300516SmavTUNABLE_QUAD("hw.ntb.usd_bar4_addr32", &xeon_b2b_usd_addr.bar4_addr32); 526300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 527300373Smav &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 528300373Smav "(split-BAR mode)."); 529300516SmavTUNABLE_QUAD("hw.ntb.usd_bar5_addr32", &xeon_b2b_usd_addr.bar5_addr32); 530300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 531300373Smav &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 532300373Smav "(split-BAR mode)."); 533300373Smav 534300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar2_addr64", &xeon_b2b_dsd_addr.bar2_addr64); 535300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 536300373Smav &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 537300373Smav "hardware, use this 64-bit address on the bus between the NTB devices for " 538300373Smav "the window at BAR2, on the downstream side of the link. MUST be the same" 539300373Smav " address on both sides."); 540300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar4_addr64", &xeon_b2b_dsd_addr.bar4_addr64); 541300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 542300373Smav &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 543300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar4_addr32", &xeon_b2b_dsd_addr.bar4_addr32); 544300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 545300373Smav &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 546300373Smav "(split-BAR mode)."); 547300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar5_addr32", &xeon_b2b_dsd_addr.bar5_addr32); 548300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 549300373Smav &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 550300373Smav "(split-BAR mode)."); 551300373Smav 552250079Scarl/* 553250079Scarl * OS <-> Driver interface structures 554250079Scarl */ 555250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 556250079Scarl 557250079Scarlstatic device_method_t ntb_pci_methods[] = { 558250079Scarl /* Device interface */ 559250079Scarl DEVMETHOD(device_probe, ntb_probe), 560250079Scarl DEVMETHOD(device_attach, ntb_attach), 561250079Scarl DEVMETHOD(device_detach, ntb_detach), 562250079Scarl DEVMETHOD_END 563250079Scarl}; 564250079Scarl 565250079Scarlstatic driver_t ntb_pci_driver = { 566250079Scarl "ntb_hw", 567250079Scarl ntb_pci_methods, 568250079Scarl sizeof(struct ntb_softc), 569250079Scarl}; 570250079Scarl 571250079Scarlstatic devclass_t ntb_devclass; 572250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 573250079ScarlMODULE_VERSION(ntb_hw, 1); 574250079Scarl 575300373SmavSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 576300373Smav 577250079Scarl/* 578250079Scarl * OS <-> Driver linkage functions 579250079Scarl */ 580250079Scarlstatic int 581250079Scarlntb_probe(device_t device) 582250079Scarl{ 583300373Smav struct ntb_hw_info *p; 584250079Scarl 585300373Smav p = ntb_get_device_info(pci_get_devid(device)); 586300373Smav if (p == NULL) 587250079Scarl return (ENXIO); 588300373Smav 589300373Smav device_set_desc(device, p->desc); 590300373Smav return (0); 591250079Scarl} 592250079Scarl 593250079Scarlstatic int 594250079Scarlntb_attach(device_t device) 595250079Scarl{ 596300373Smav struct ntb_softc *ntb; 597300373Smav struct ntb_hw_info *p; 598250079Scarl int error; 599250079Scarl 600300373Smav ntb = DEVICE2SOFTC(device); 601300373Smav p = ntb_get_device_info(pci_get_devid(device)); 602300373Smav 603250079Scarl ntb->device = device; 604250079Scarl ntb->type = p->type; 605255274Scarl ntb->features = p->features; 606300373Smav ntb->b2b_mw_idx = B2B_MW_DISABLED; 607250079Scarl 608300373Smav /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 609250079Scarl callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE); 610250079Scarl callout_init(&ntb->lr_timer, CALLOUT_MPSAFE); 611300373Smav mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 612300373Smav mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 613250079Scarl 614300373Smav if (ntb->type == NTB_ATOM) 615300373Smav error = ntb_detect_atom(ntb); 616300373Smav else 617300373Smav error = ntb_detect_xeon(ntb); 618300373Smav if (error != 0) 619300373Smav goto out; 620250079Scarl 621300373Smav ntb_detect_max_mw(ntb); 622300373Smav 623250079Scarl pci_enable_busmaster(ntb->device); 624250079Scarl 625300373Smav error = ntb_map_pci_bars(ntb); 626300373Smav if (error != 0) 627300373Smav goto out; 628300373Smav if (ntb->type == NTB_ATOM) 629300373Smav error = ntb_atom_init_dev(ntb); 630300373Smav else 631300373Smav error = ntb_xeon_init_dev(ntb); 632300373Smav if (error != 0) 633300373Smav goto out; 634300373Smav 635300373Smav ntb_poll_link(ntb); 636300373Smav 637300373Smav ntb_sysctl_init(ntb); 638300373Smav 639300373Smavout: 640300373Smav if (error != 0) 641300373Smav ntb_detach(device); 642250079Scarl return (error); 643250079Scarl} 644250079Scarl 645250079Scarlstatic int 646250079Scarlntb_detach(device_t device) 647250079Scarl{ 648300373Smav struct ntb_softc *ntb; 649250079Scarl 650300373Smav ntb = DEVICE2SOFTC(device); 651300373Smav 652300373Smav if (ntb->self_reg != NULL) 653300373Smav ntb_db_set_mask(ntb, ntb->db_valid_mask); 654250079Scarl callout_drain(&ntb->heartbeat_timer); 655250079Scarl callout_drain(&ntb->lr_timer); 656300373Smav pci_disable_busmaster(ntb->device); 657300373Smav if (ntb->type == NTB_XEON) 658300373Smav ntb_teardown_xeon(ntb); 659250079Scarl ntb_teardown_interrupts(ntb); 660300373Smav 661300373Smav mtx_destroy(&ntb->db_mask_lock); 662300373Smav mtx_destroy(&ntb->ctx_lock); 663300373Smav 664250079Scarl ntb_unmap_pci_bar(ntb); 665250079Scarl 666250079Scarl return (0); 667250079Scarl} 668250079Scarl 669300373Smav/* 670300373Smav * Driver internal routines 671300373Smav */ 672300373Smavstatic inline enum ntb_bar 673300373Smavntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 674300373Smav{ 675300373Smav 676300373Smav KASSERT(mw < ntb->mw_count, 677300373Smav ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 678300373Smav KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 679300373Smav 680300373Smav return (ntb->reg->mw_bar[mw]); 681300373Smav} 682300373Smav 683300373Smavstatic inline bool 684300373Smavbar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 685300373Smav{ 686300373Smav /* XXX This assertion could be stronger. */ 687300373Smav KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 688300373Smav return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 689300373Smav} 690300373Smav 691300373Smavstatic inline void 692300373Smavbar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 693300373Smav uint32_t *xlat, uint32_t *lmt) 694300373Smav{ 695300373Smav uint32_t basev, lmtv, xlatv; 696300373Smav 697300373Smav switch (bar) { 698300373Smav case NTB_B2B_BAR_1: 699300373Smav basev = ntb->xlat_reg->bar2_base; 700300373Smav lmtv = ntb->xlat_reg->bar2_limit; 701300373Smav xlatv = ntb->xlat_reg->bar2_xlat; 702300373Smav break; 703300373Smav case NTB_B2B_BAR_2: 704300373Smav basev = ntb->xlat_reg->bar4_base; 705300373Smav lmtv = ntb->xlat_reg->bar4_limit; 706300373Smav xlatv = ntb->xlat_reg->bar4_xlat; 707300373Smav break; 708300373Smav case NTB_B2B_BAR_3: 709300373Smav basev = ntb->xlat_reg->bar5_base; 710300373Smav lmtv = ntb->xlat_reg->bar5_limit; 711300373Smav xlatv = ntb->xlat_reg->bar5_xlat; 712300373Smav break; 713300373Smav default: 714300373Smav KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 715300373Smav ("bad bar")); 716300373Smav basev = lmtv = xlatv = 0; 717300373Smav break; 718300373Smav } 719300373Smav 720300373Smav if (base != NULL) 721300373Smav *base = basev; 722300373Smav if (xlat != NULL) 723300373Smav *xlat = xlatv; 724300373Smav if (lmt != NULL) 725300373Smav *lmt = lmtv; 726300373Smav} 727300373Smav 728250079Scarlstatic int 729255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 730250079Scarl{ 731255272Scarl int rc; 732250079Scarl 733250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 734300373Smav rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 735255272Scarl if (rc != 0) 736300373Smav goto out; 737255272Scarl 738300373Smav ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 739300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 740255272Scarl if (rc != 0) 741300373Smav goto out; 742300373Smav ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 743300373Smav ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 744300373Smav ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 745255272Scarl 746300373Smav ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 747300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 748255272Scarl if (rc != 0) 749300373Smav goto out; 750300373Smav ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 751300373Smav ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 752300373Smav ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 753255274Scarl 754300373Smav if (!HAS_FEATURE(NTB_SPLIT_BAR)) 755300373Smav goto out; 756250079Scarl 757300373Smav ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 758300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 759300373Smav ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 760300373Smav ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 761300373Smav ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 762255272Scarl 763300373Smavout: 764300373Smav if (rc != 0) 765255272Scarl device_printf(ntb->device, 766255272Scarl "unable to allocate pci resource\n"); 767255272Scarl return (rc); 768255272Scarl} 769255272Scarl 770300373Smavstatic void 771300373Smavprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 772300373Smav const char *kind) 773300373Smav{ 774300373Smav 775300373Smav device_printf(ntb->device, 776300373Smav "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 777300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 778300373Smav (char *)bar->vbase + bar->size - 1, 779300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 780300373Smav (uintmax_t)bar->size, kind); 781300373Smav} 782300373Smav 783255272Scarlstatic int 784255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 785255272Scarl{ 786255272Scarl 787255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 788300373Smav &bar->pci_resource_id, RF_ACTIVE); 789255272Scarl if (bar->pci_resource == NULL) 790255272Scarl return (ENXIO); 791300373Smav 792300373Smav save_bar_parameters(bar); 793300373Smav bar->map_mode = VM_MEMATTR_UNCACHEABLE; 794300373Smav print_map_success(ntb, bar, "mmr"); 795300373Smav return (0); 796255272Scarl} 797255272Scarl 798255272Scarlstatic int 799255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 800255272Scarl{ 801255272Scarl int rc; 802300373Smav vm_memattr_t mapmode; 803255276Scarl uint8_t bar_size_bits = 0; 804255272Scarl 805300373Smav bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 806300373Smav &bar->pci_resource_id, RF_ACTIVE); 807250079Scarl 808255272Scarl if (bar->pci_resource == NULL) 809255272Scarl return (ENXIO); 810255276Scarl 811300373Smav save_bar_parameters(bar); 812300373Smav /* 813300373Smav * Ivytown NTB BAR sizes are misreported by the hardware due to a 814300373Smav * hardware issue. To work around this, query the size it should be 815300373Smav * configured to by the device and modify the resource to correspond to 816300373Smav * this new size. The BIOS on systems with this problem is required to 817300373Smav * provide enough address space to allow the driver to make this change 818300373Smav * safely. 819300373Smav * 820300373Smav * Ideally I could have just specified the size when I allocated the 821300373Smav * resource like: 822300373Smav * bus_alloc_resource(ntb->device, 823300373Smav * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 824300373Smav * 1ul << bar_size_bits, RF_ACTIVE); 825300373Smav * but the PCI driver does not honor the size in this call, so we have 826300373Smav * to modify it after the fact. 827300373Smav */ 828300373Smav if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 829300373Smav if (bar->pci_resource_id == PCIR_BAR(2)) 830300373Smav bar_size_bits = pci_read_config(ntb->device, 831300373Smav XEON_PBAR23SZ_OFFSET, 1); 832300373Smav else 833300373Smav bar_size_bits = pci_read_config(ntb->device, 834300373Smav XEON_PBAR45SZ_OFFSET, 1); 835300373Smav 836300373Smav rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 837300373Smav bar->pci_resource, bar->pbase, 838300373Smav bar->pbase + (1ul << bar_size_bits) - 1); 839255272Scarl if (rc != 0) { 840300373Smav device_printf(ntb->device, 841300373Smav "unable to resize bar\n"); 842255272Scarl return (rc); 843250079Scarl } 844300373Smav 845300373Smav save_bar_parameters(bar); 846250079Scarl } 847300373Smav 848300373Smav bar->map_mode = VM_MEMATTR_UNCACHEABLE; 849300373Smav print_map_success(ntb, bar, "mw"); 850300373Smav 851300373Smav /* 852300373Smav * Optionally, mark MW BARs as anything other than UC to improve 853300373Smav * performance. 854300373Smav */ 855300373Smav mapmode = ntb_pat_flags(); 856300373Smav if (mapmode == bar->map_mode) 857300373Smav return (0); 858300373Smav 859300373Smav rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 860300373Smav if (rc == 0) { 861300373Smav bar->map_mode = mapmode; 862300373Smav device_printf(ntb->device, 863300373Smav "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 864300373Smav "%s.\n", 865300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 866300373Smav (char *)bar->vbase + bar->size - 1, 867300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 868300373Smav ntb_vm_memattr_to_str(mapmode)); 869300373Smav } else 870300373Smav device_printf(ntb->device, 871300373Smav "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 872300373Smav "%s: %d\n", 873300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 874300373Smav (char *)bar->vbase + bar->size - 1, 875300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 876300373Smav ntb_vm_memattr_to_str(mapmode), rc); 877300373Smav /* Proceed anyway */ 878250079Scarl return (0); 879250079Scarl} 880250079Scarl 881250079Scarlstatic void 882250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 883250079Scarl{ 884250079Scarl struct ntb_pci_bar_info *current_bar; 885250079Scarl int i; 886250079Scarl 887300373Smav for (i = 0; i < NTB_MAX_BARS; i++) { 888250079Scarl current_bar = &ntb->bar_info[i]; 889250079Scarl if (current_bar->pci_resource != NULL) 890250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 891250079Scarl current_bar->pci_resource_id, 892250079Scarl current_bar->pci_resource); 893250079Scarl } 894250079Scarl} 895250079Scarl 896250079Scarlstatic int 897300373Smavntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 898250079Scarl{ 899300373Smav uint32_t i; 900300373Smav int rc; 901250079Scarl 902300373Smav for (i = 0; i < num_vectors; i++) { 903300373Smav ntb->int_info[i].rid = i + 1; 904300373Smav ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 905300373Smav SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 906300373Smav if (ntb->int_info[i].res == NULL) { 907300373Smav device_printf(ntb->device, 908300373Smav "bus_alloc_resource failed\n"); 909300373Smav return (ENOMEM); 910300373Smav } 911300373Smav ntb->int_info[i].tag = NULL; 912300373Smav ntb->allocated_interrupts++; 913300373Smav rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 914300373Smav INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 915300373Smav &ntb->msix_vec[i], &ntb->int_info[i].tag); 916300373Smav if (rc != 0) { 917300373Smav device_printf(ntb->device, "bus_setup_intr failed\n"); 918300373Smav return (ENXIO); 919300373Smav } 920300373Smav } 921300373Smav return (0); 922300373Smav} 923300373Smav 924300373Smav/* 925300373Smav * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 926300373Smav * cannot be allocated for each MSI-X message. JHB seems to think remapping 927300373Smav * should be okay. This tunable should enable us to test that hypothesis 928300373Smav * when someone gets their hands on some Xeon hardware. 929300373Smav */ 930300373Smavstatic int ntb_force_remap_mode; 931300516SmavTUNABLE_INT("hw.ntb.force_remap_mode", &ntb_force_remap_mode); 932300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 933300373Smav &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 934300373Smav " to a smaller number of ithreads, even if the desired number are " 935300373Smav "available"); 936300373Smav 937300373Smav/* 938300373Smav * In case it is NOT ok, give consumers an abort button. 939300373Smav */ 940300373Smavstatic int ntb_prefer_intx; 941300516SmavTUNABLE_INT("hw.ntb.prefer_intx_to_remap", &ntb_prefer_intx); 942300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 943300373Smav &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 944300373Smav "than remapping MSI-X messages over available slots (match Linux driver " 945300373Smav "behavior)"); 946300373Smav 947300373Smav/* 948300373Smav * Remap the desired number of MSI-X messages to available ithreads in a simple 949300373Smav * round-robin fashion. 950300373Smav */ 951300373Smavstatic int 952300373Smavntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 953300373Smav{ 954300373Smav u_int *vectors; 955300373Smav uint32_t i; 956300373Smav int rc; 957300373Smav 958300373Smav if (ntb_prefer_intx != 0) 959300373Smav return (ENXIO); 960300373Smav 961300373Smav vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 962300373Smav 963300373Smav for (i = 0; i < desired; i++) 964300373Smav vectors[i] = (i % avail) + 1; 965300373Smav 966300373Smav rc = pci_remap_msix(dev, desired, vectors); 967300373Smav free(vectors, M_NTB); 968300373Smav return (rc); 969300373Smav} 970300373Smav 971300373Smavstatic int 972300373Smavntb_init_isr(struct ntb_softc *ntb) 973300373Smav{ 974300373Smav uint32_t desired_vectors, num_vectors; 975300373Smav int rc; 976300373Smav 977250079Scarl ntb->allocated_interrupts = 0; 978300373Smav ntb->last_ts = ticks; 979300373Smav 980250079Scarl /* 981300373Smav * Mask all doorbell interrupts. 982250079Scarl */ 983300373Smav ntb_db_set_mask(ntb, ntb->db_valid_mask); 984250079Scarl 985300373Smav num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 986300373Smav ntb->db_count); 987300373Smav if (desired_vectors >= 1) { 988300373Smav rc = pci_alloc_msix(ntb->device, &num_vectors); 989250079Scarl 990300373Smav if (ntb_force_remap_mode != 0 && rc == 0 && 991300373Smav num_vectors == desired_vectors) 992300373Smav num_vectors--; 993300373Smav 994300373Smav if (rc == 0 && num_vectors < desired_vectors) { 995300373Smav rc = ntb_remap_msix(ntb->device, desired_vectors, 996300373Smav num_vectors); 997300373Smav if (rc == 0) 998300373Smav num_vectors = desired_vectors; 999300373Smav else 1000300373Smav pci_release_msi(ntb->device); 1001250079Scarl } 1002300373Smav if (rc != 0) 1003300373Smav num_vectors = 1; 1004300373Smav } else 1005300373Smav num_vectors = 1; 1006300373Smav 1007300373Smav if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1008300373Smav ntb->db_vec_count = 1; 1009300373Smav ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1010300373Smav rc = ntb_setup_legacy_interrupt(ntb); 1011300373Smav } else { 1012300373Smav ntb_create_msix_vec(ntb, num_vectors); 1013300373Smav rc = ntb_setup_msix(ntb, num_vectors); 1014250079Scarl } 1015300373Smav if (rc != 0) { 1016300373Smav device_printf(ntb->device, 1017300373Smav "Error allocating interrupts: %d\n", rc); 1018300373Smav ntb_free_msix_vec(ntb); 1019300373Smav } 1020250079Scarl 1021300373Smav return (rc); 1022300373Smav} 1023250079Scarl 1024300373Smavstatic int 1025300373Smavntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1026300373Smav{ 1027300373Smav int rc; 1028300373Smav 1029300373Smav ntb->int_info[0].rid = 0; 1030300373Smav ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1031300373Smav &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1032300373Smav if (ntb->int_info[0].res == NULL) { 1033300373Smav device_printf(ntb->device, "bus_alloc_resource failed\n"); 1034300373Smav return (ENOMEM); 1035250079Scarl } 1036250079Scarl 1037300373Smav ntb->int_info[0].tag = NULL; 1038300373Smav ntb->allocated_interrupts = 1; 1039300373Smav 1040300373Smav rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1041300373Smav INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1042300373Smav ntb, &ntb->int_info[0].tag); 1043300373Smav if (rc != 0) { 1044300373Smav device_printf(ntb->device, "bus_setup_intr failed\n"); 1045300373Smav return (ENXIO); 1046300373Smav } 1047300373Smav 1048250079Scarl return (0); 1049250079Scarl} 1050250079Scarl 1051250079Scarlstatic void 1052250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 1053250079Scarl{ 1054250079Scarl struct ntb_int_info *current_int; 1055250079Scarl int i; 1056250079Scarl 1057300373Smav for (i = 0; i < ntb->allocated_interrupts; i++) { 1058250079Scarl current_int = &ntb->int_info[i]; 1059250079Scarl if (current_int->tag != NULL) 1060250079Scarl bus_teardown_intr(ntb->device, current_int->res, 1061250079Scarl current_int->tag); 1062250079Scarl 1063250079Scarl if (current_int->res != NULL) 1064250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 1065250079Scarl rman_get_rid(current_int->res), current_int->res); 1066250079Scarl } 1067250079Scarl 1068300373Smav ntb_free_msix_vec(ntb); 1069250079Scarl pci_release_msi(ntb->device); 1070250079Scarl} 1071250079Scarl 1072300373Smav/* 1073300373Smav * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1074300373Smav * out to make code clearer. 1075300373Smav */ 1076300373Smavstatic inline uint64_t 1077300373Smavdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 1078250079Scarl{ 1079250079Scarl 1080300373Smav if (ntb->type == NTB_ATOM) 1081300373Smav return (ntb_reg_read(8, regoff)); 1082250079Scarl 1083300373Smav KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1084300373Smav 1085300373Smav return (ntb_reg_read(2, regoff)); 1086250079Scarl} 1087250079Scarl 1088300373Smavstatic inline void 1089300373Smavdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1090250079Scarl{ 1091250079Scarl 1092300373Smav KASSERT((val & ~ntb->db_valid_mask) == 0, 1093300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1094300373Smav (uintmax_t)(val & ~ntb->db_valid_mask), 1095300373Smav (uintmax_t)ntb->db_valid_mask)); 1096250079Scarl 1097300373Smav if (regoff == ntb->self_reg->db_mask) 1098300373Smav DB_MASK_ASSERT(ntb, MA_OWNED); 1099300373Smav db_iowrite_raw(ntb, regoff, val); 1100250079Scarl} 1101250079Scarl 1102300373Smavstatic inline void 1103300373Smavdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1104250079Scarl{ 1105250079Scarl 1106300373Smav if (ntb->type == NTB_ATOM) { 1107300373Smav ntb_reg_write(8, regoff, val); 1108300373Smav return; 1109300373Smav } 1110250079Scarl 1111300373Smav KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1112300373Smav ntb_reg_write(2, regoff, (uint16_t)val); 1113250079Scarl} 1114250079Scarl 1115300373Smavvoid 1116300373Smavntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1117250079Scarl{ 1118250079Scarl 1119300373Smav DB_MASK_LOCK(ntb); 1120300373Smav ntb->db_mask |= bits; 1121300373Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1122300373Smav DB_MASK_UNLOCK(ntb); 1123300373Smav} 1124250079Scarl 1125300373Smavvoid 1126300373Smavntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1127300373Smav{ 1128250079Scarl 1129300373Smav KASSERT((bits & ~ntb->db_valid_mask) == 0, 1130300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1131300373Smav (uintmax_t)(bits & ~ntb->db_valid_mask), 1132300373Smav (uintmax_t)ntb->db_valid_mask)); 1133250079Scarl 1134300373Smav DB_MASK_LOCK(ntb); 1135300373Smav ntb->db_mask &= ~bits; 1136300373Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1137300373Smav DB_MASK_UNLOCK(ntb); 1138300373Smav} 1139300373Smav 1140300373Smavuint64_t 1141300373Smavntb_db_read(struct ntb_softc *ntb) 1142300373Smav{ 1143300373Smav 1144300373Smav return (db_ioread(ntb, ntb->self_reg->db_bell)); 1145300373Smav} 1146300373Smav 1147300373Smavvoid 1148300373Smavntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1149300373Smav{ 1150300373Smav 1151300373Smav KASSERT((bits & ~ntb->db_valid_mask) == 0, 1152300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1153300373Smav (uintmax_t)(bits & ~ntb->db_valid_mask), 1154300373Smav (uintmax_t)ntb->db_valid_mask)); 1155300373Smav 1156300373Smav db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1157300373Smav} 1158300373Smav 1159300373Smavstatic inline uint64_t 1160300373Smavntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1161300373Smav{ 1162300373Smav uint64_t shift, mask; 1163300373Smav 1164300373Smav shift = ntb->db_vec_shift; 1165300373Smav mask = (1ull << shift) - 1; 1166300373Smav return (mask << (shift * db_vector)); 1167300373Smav} 1168300373Smav 1169300373Smavstatic void 1170300373Smavntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1171300373Smav{ 1172300373Smav uint64_t vec_mask; 1173300373Smav 1174300373Smav ntb->last_ts = ticks; 1175300373Smav vec_mask = ntb_vec_mask(ntb, vec); 1176300373Smav 1177300373Smav if ((vec_mask & ntb->db_link_mask) != 0) { 1178300373Smav if (ntb_poll_link(ntb)) 1179300373Smav ntb_link_event(ntb); 1180250079Scarl } 1181250079Scarl 1182300373Smav if ((vec_mask & ntb->db_valid_mask) != 0) 1183300373Smav ntb_db_event(ntb, vec); 1184250079Scarl} 1185250079Scarl 1186300373Smavstatic void 1187300373Smavndev_vec_isr(void *arg) 1188300373Smav{ 1189300373Smav struct ntb_vec *nvec = arg; 1190300373Smav 1191300373Smav ntb_interrupt(nvec->ntb, nvec->num); 1192300373Smav} 1193300373Smav 1194300373Smavstatic void 1195300373Smavndev_irq_isr(void *arg) 1196300373Smav{ 1197300373Smav /* If we couldn't set up MSI-X, we only have the one vector. */ 1198300373Smav ntb_interrupt(arg, 0); 1199300373Smav} 1200300373Smav 1201250079Scarlstatic int 1202300373Smavntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1203250079Scarl{ 1204300373Smav uint32_t i; 1205250079Scarl 1206300373Smav ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1207250079Scarl M_ZERO | M_WAITOK); 1208250079Scarl for (i = 0; i < num_vectors; i++) { 1209300373Smav ntb->msix_vec[i].num = i; 1210300373Smav ntb->msix_vec[i].ntb = ntb; 1211250079Scarl } 1212250079Scarl 1213250079Scarl return (0); 1214250079Scarl} 1215250079Scarl 1216250079Scarlstatic void 1217300373Smavntb_free_msix_vec(struct ntb_softc *ntb) 1218250079Scarl{ 1219250079Scarl 1220300373Smav if (ntb->msix_vec == NULL) 1221300373Smav return; 1222250079Scarl 1223300373Smav free(ntb->msix_vec, M_NTB); 1224300373Smav ntb->msix_vec = NULL; 1225250079Scarl} 1226250079Scarl 1227250079Scarlstatic struct ntb_hw_info * 1228250079Scarlntb_get_device_info(uint32_t device_id) 1229250079Scarl{ 1230250079Scarl struct ntb_hw_info *ep = pci_ids; 1231250079Scarl 1232250079Scarl while (ep->device_id) { 1233250079Scarl if (ep->device_id == device_id) 1234250079Scarl return (ep); 1235250079Scarl ++ep; 1236250079Scarl } 1237250079Scarl return (NULL); 1238250079Scarl} 1239250079Scarl 1240300373Smavstatic void 1241300373Smavntb_teardown_xeon(struct ntb_softc *ntb) 1242250079Scarl{ 1243250079Scarl 1244300373Smav if (ntb->reg != NULL) 1245300373Smav ntb_link_disable(ntb); 1246300373Smav} 1247300373Smav 1248300373Smavstatic void 1249300373Smavntb_detect_max_mw(struct ntb_softc *ntb) 1250300373Smav{ 1251300373Smav 1252300373Smav if (ntb->type == NTB_ATOM) { 1253300373Smav ntb->mw_count = ATOM_MW_COUNT; 1254300373Smav return; 1255300373Smav } 1256300373Smav 1257300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 1258300373Smav ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1259250079Scarl else 1260300373Smav ntb->mw_count = XEON_SNB_MW_COUNT; 1261250079Scarl} 1262250079Scarl 1263250079Scarlstatic int 1264300373Smavntb_detect_xeon(struct ntb_softc *ntb) 1265250079Scarl{ 1266300373Smav uint8_t ppd, conn_type; 1267250079Scarl 1268300373Smav ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1269300373Smav ntb->ppd = ppd; 1270250079Scarl 1271300373Smav if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1272300373Smav ntb->dev_type = NTB_DEV_DSD; 1273300373Smav else 1274300373Smav ntb->dev_type = NTB_DEV_USD; 1275300373Smav 1276300373Smav if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1277300373Smav ntb->features |= NTB_SPLIT_BAR; 1278300373Smav 1279300373Smav /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1280300373Smav if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1281300373Smav ntb->features |= NTB_SDOORBELL_LOCKUP; 1282300373Smav 1283300373Smav conn_type = ppd & XEON_PPD_CONN_TYPE; 1284300373Smav switch (conn_type) { 1285250079Scarl case NTB_CONN_B2B: 1286300373Smav ntb->conn_type = conn_type; 1287250079Scarl break; 1288250079Scarl case NTB_CONN_RP: 1289300373Smav case NTB_CONN_TRANSPARENT: 1290250079Scarl default: 1291300373Smav device_printf(ntb->device, "Unsupported connection type: %u\n", 1292300373Smav (unsigned)conn_type); 1293250079Scarl return (ENXIO); 1294250079Scarl } 1295300373Smav return (0); 1296300373Smav} 1297250079Scarl 1298300373Smavstatic int 1299300373Smavntb_detect_atom(struct ntb_softc *ntb) 1300300373Smav{ 1301300373Smav uint32_t ppd, conn_type; 1302300373Smav 1303300373Smav ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1304300373Smav ntb->ppd = ppd; 1305300373Smav 1306300373Smav if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1307250079Scarl ntb->dev_type = NTB_DEV_DSD; 1308250079Scarl else 1309250079Scarl ntb->dev_type = NTB_DEV_USD; 1310250079Scarl 1311300373Smav conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1312300373Smav switch (conn_type) { 1313300373Smav case NTB_CONN_B2B: 1314300373Smav ntb->conn_type = conn_type; 1315300373Smav break; 1316300373Smav default: 1317300373Smav device_printf(ntb->device, "Unsupported NTB configuration\n"); 1318300373Smav return (ENXIO); 1319250079Scarl } 1320250079Scarl return (0); 1321250079Scarl} 1322250079Scarl 1323250079Scarlstatic int 1324300373Smavntb_xeon_init_dev(struct ntb_softc *ntb) 1325250079Scarl{ 1326300373Smav int rc; 1327250079Scarl 1328300373Smav ntb->spad_count = XEON_SPAD_COUNT; 1329300373Smav ntb->db_count = XEON_DB_COUNT; 1330300373Smav ntb->db_link_mask = XEON_DB_LINK_BIT; 1331300373Smav ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1332300373Smav ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1333250079Scarl 1334300373Smav if (ntb->conn_type != NTB_CONN_B2B) { 1335250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1336300373Smav ntb->conn_type); 1337250079Scarl return (ENXIO); 1338250079Scarl } 1339250079Scarl 1340300373Smav ntb->reg = &xeon_reg; 1341300373Smav ntb->self_reg = &xeon_pri_reg; 1342300373Smav ntb->peer_reg = &xeon_b2b_reg; 1343300373Smav ntb->xlat_reg = &xeon_sec_xlat; 1344300373Smav 1345300373Smav /* 1346300373Smav * There is a Xeon hardware errata related to writes to SDOORBELL or 1347300373Smav * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1348300373Smav * which may hang the system. To workaround this, use a memory 1349300373Smav * window to access the interrupt and scratch pad registers on the 1350300373Smav * remote system. 1351300373Smav */ 1352300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1353300373Smav ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1354300373Smav ntb->mw_count; 1355300373Smav ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1356300373Smav g_ntb_mw_idx, ntb->b2b_mw_idx); 1357300373Smav rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, VM_MEMATTR_UNCACHEABLE); 1358300373Smav KASSERT(rc == 0, ("shouldn't fail")); 1359300373Smav } else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1360300373Smav /* 1361300373Smav * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1362300373Smav * mirrored to the remote system. Shrink the number of bits by one, 1363300373Smav * since bit 14 is the last bit. 1364300373Smav * 1365300373Smav * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1366300373Smav * anyway. Nor for non-B2B connection types. 1367300373Smav */ 1368300373Smav ntb->db_count = XEON_DB_COUNT - 1; 1369300373Smav 1370300373Smav ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1371300373Smav 1372300373Smav if (ntb->dev_type == NTB_DEV_USD) 1373300373Smav rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1374300373Smav &xeon_b2b_usd_addr); 1375250079Scarl else 1376300373Smav rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1377300373Smav &xeon_b2b_dsd_addr); 1378300373Smav if (rc != 0) 1379300373Smav return (rc); 1380250079Scarl 1381300373Smav /* Enable Bus Master and Memory Space on the secondary side */ 1382300373Smav ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1383300373Smav PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1384250079Scarl 1385300373Smav /* 1386300373Smav * Mask all doorbell interrupts. 1387300373Smav */ 1388300373Smav ntb_db_set_mask(ntb, ntb->db_valid_mask); 1389250079Scarl 1390300373Smav rc = ntb_init_isr(ntb); 1391300373Smav return (rc); 1392300373Smav} 1393250079Scarl 1394300373Smavstatic int 1395300373Smavntb_atom_init_dev(struct ntb_softc *ntb) 1396300373Smav{ 1397300373Smav int error; 1398250079Scarl 1399300373Smav KASSERT(ntb->conn_type == NTB_CONN_B2B, 1400300373Smav ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1401300373Smav 1402300373Smav ntb->spad_count = ATOM_SPAD_COUNT; 1403300373Smav ntb->db_count = ATOM_DB_COUNT; 1404300373Smav ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1405300373Smav ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1406300373Smav ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1407300373Smav 1408300373Smav ntb->reg = &atom_reg; 1409300373Smav ntb->self_reg = &atom_pri_reg; 1410300373Smav ntb->peer_reg = &atom_b2b_reg; 1411300373Smav ntb->xlat_reg = &atom_sec_xlat; 1412300373Smav 1413250079Scarl /* 1414300373Smav * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1415250079Scarl * resolved. Mask transaction layer internal parity errors. 1416250079Scarl */ 1417250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1418250079Scarl 1419300373Smav configure_atom_secondary_side_bars(ntb); 1420250079Scarl 1421250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1422300373Smav ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1423250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1424250079Scarl 1425300373Smav error = ntb_init_isr(ntb); 1426300373Smav if (error != 0) 1427300373Smav return (error); 1428300373Smav 1429300373Smav /* Initiate PCI-E link training */ 1430300373Smav ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1431300373Smav 1432300373Smav callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1433300373Smav 1434250079Scarl return (0); 1435250079Scarl} 1436250079Scarl 1437300373Smav/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1438255279Scarlstatic void 1439300373Smavconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1440255279Scarl{ 1441255279Scarl 1442255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1443300373Smav ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1444300373Smav XEON_B2B_BAR2_ADDR64); 1445300373Smav ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1446300373Smav XEON_B2B_BAR4_ADDR64); 1447300373Smav ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1448300373Smav ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1449255279Scarl } else { 1450300373Smav ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1451300373Smav XEON_B2B_BAR2_ADDR64); 1452300373Smav ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1453300373Smav XEON_B2B_BAR4_ADDR64); 1454300373Smav ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1455300373Smav ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1456255279Scarl } 1457255279Scarl} 1458255279Scarl 1459300373Smav 1460300373Smav/* 1461300373Smav * When working around Xeon SDOORBELL errata by remapping remote registers in a 1462300373Smav * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1463300373Smav * remains for use by a higher layer. 1464300373Smav * 1465300373Smav * Will only be used if working around SDOORBELL errata and the BIOS-configured 1466300373Smav * MW size is sufficiently large. 1467300373Smav */ 1468300373Smavstatic unsigned int ntb_b2b_mw_share; 1469300516SmavTUNABLE_INT("hw.ntb.b2b_mw_share", &ntb_b2b_mw_share); 1470300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1471300373Smav 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1472300373Smav "MW with higher level consumers. Both sides of the NTB MUST set the same " 1473300373Smav "value here."); 1474300373Smav 1475255279Scarlstatic void 1476300373Smavxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1477300373Smav enum ntb_bar regbar) 1478255279Scarl{ 1479300373Smav struct ntb_pci_bar_info *bar; 1480300373Smav uint8_t bar_sz; 1481255279Scarl 1482300373Smav if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1483300373Smav return; 1484300373Smav 1485300373Smav bar = &ntb->bar_info[idx]; 1486300373Smav bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1487300373Smav if (idx == regbar) { 1488300373Smav if (ntb->b2b_off != 0) 1489300373Smav bar_sz--; 1490255279Scarl else 1491300373Smav bar_sz = 0; 1492300373Smav } 1493300373Smav pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1494300373Smav bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1495300373Smav (void)bar_sz; 1496300373Smav} 1497300373Smav 1498300373Smavstatic void 1499300373Smavxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1500300373Smav enum ntb_bar idx, enum ntb_bar regbar) 1501300373Smav{ 1502300373Smav uint64_t reg_val; 1503300373Smav uint32_t base_reg, lmt_reg; 1504300373Smav 1505300373Smav bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1506300373Smav if (idx == regbar) 1507300373Smav bar_addr += ntb->b2b_off; 1508300373Smav 1509300373Smav if (!bar_is_64bit(ntb, idx)) { 1510300373Smav ntb_reg_write(4, base_reg, bar_addr); 1511300373Smav reg_val = ntb_reg_read(4, base_reg); 1512300373Smav (void)reg_val; 1513300373Smav 1514300373Smav ntb_reg_write(4, lmt_reg, bar_addr); 1515300373Smav reg_val = ntb_reg_read(4, lmt_reg); 1516300373Smav (void)reg_val; 1517255279Scarl } else { 1518300373Smav ntb_reg_write(8, base_reg, bar_addr); 1519300373Smav reg_val = ntb_reg_read(8, base_reg); 1520300373Smav (void)reg_val; 1521300373Smav 1522300373Smav ntb_reg_write(8, lmt_reg, bar_addr); 1523300373Smav reg_val = ntb_reg_read(8, lmt_reg); 1524300373Smav (void)reg_val; 1525255279Scarl } 1526255279Scarl} 1527255279Scarl 1528250079Scarlstatic void 1529300373Smavxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1530250079Scarl{ 1531300373Smav struct ntb_pci_bar_info *bar; 1532250079Scarl 1533300373Smav bar = &ntb->bar_info[idx]; 1534300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1535300373Smav ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1536300373Smav base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1537300373Smav } else { 1538300373Smav ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1539300373Smav base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1540300373Smav } 1541300373Smav (void)base_addr; 1542300373Smav} 1543300373Smav 1544300373Smavstatic int 1545300373Smavxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1546300373Smav const struct ntb_b2b_addr *peer_addr) 1547300373Smav{ 1548300373Smav struct ntb_pci_bar_info *b2b_bar; 1549300373Smav vm_size_t bar_size; 1550300373Smav uint64_t bar_addr; 1551300373Smav enum ntb_bar b2b_bar_num, i; 1552300373Smav 1553300373Smav if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1554300373Smav b2b_bar = NULL; 1555300373Smav b2b_bar_num = NTB_CONFIG_BAR; 1556300373Smav ntb->b2b_off = 0; 1557300373Smav } else { 1558300373Smav b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1559300373Smav KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1560300373Smav ("invalid b2b mw bar")); 1561300373Smav 1562300373Smav b2b_bar = &ntb->bar_info[b2b_bar_num]; 1563300373Smav bar_size = b2b_bar->size; 1564300373Smav 1565300373Smav if (ntb_b2b_mw_share != 0 && 1566300373Smav (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1567300373Smav ntb->b2b_off = bar_size >> 1; 1568300373Smav else if (bar_size >= XEON_B2B_MIN_SIZE) { 1569300373Smav ntb->b2b_off = 0; 1570300373Smav } else { 1571300373Smav device_printf(ntb->device, 1572300373Smav "B2B bar size is too small!\n"); 1573300373Smav return (EIO); 1574250079Scarl } 1575250079Scarl } 1576250079Scarl 1577300373Smav /* 1578300373Smav * Reset the secondary bar sizes to match the primary bar sizes. 1579300373Smav * (Except, disable or halve the size of the B2B secondary bar.) 1580300373Smav */ 1581300373Smav for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1582300373Smav xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1583300373Smav 1584300373Smav bar_addr = 0; 1585300373Smav if (b2b_bar_num == NTB_CONFIG_BAR) 1586300373Smav bar_addr = addr->bar0_addr; 1587300373Smav else if (b2b_bar_num == NTB_B2B_BAR_1) 1588300373Smav bar_addr = addr->bar2_addr64; 1589300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1590300373Smav bar_addr = addr->bar4_addr64; 1591300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2) 1592300373Smav bar_addr = addr->bar4_addr32; 1593300373Smav else if (b2b_bar_num == NTB_B2B_BAR_3) 1594300373Smav bar_addr = addr->bar5_addr32; 1595300373Smav else 1596300373Smav KASSERT(false, ("invalid bar")); 1597300373Smav 1598300373Smav ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1599300373Smav 1600300373Smav /* 1601300373Smav * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1602300373Smav * register BAR. The B2B BAR is either disabled above or configured 1603300373Smav * half-size. It starts at PBAR xlat + offset. 1604300373Smav * 1605300373Smav * Also set up incoming BAR limits == base (zero length window). 1606300373Smav */ 1607300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1608300373Smav b2b_bar_num); 1609300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1610300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1611300373Smav NTB_B2B_BAR_2, b2b_bar_num); 1612300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1613300373Smav NTB_B2B_BAR_3, b2b_bar_num); 1614300373Smav } else 1615300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1616300373Smav NTB_B2B_BAR_2, b2b_bar_num); 1617300373Smav 1618300373Smav /* Zero incoming translation addrs */ 1619300373Smav ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1620300373Smav ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1621300373Smav 1622300373Smav /* Zero outgoing translation limits (whole bar size windows) */ 1623300373Smav ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1624300373Smav ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1625300373Smav 1626300373Smav /* Set outgoing translation offsets */ 1627300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1628300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1629300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1630300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1631300373Smav } else 1632300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1633300373Smav 1634300373Smav /* Set the translation offset for B2B registers */ 1635300373Smav bar_addr = 0; 1636300373Smav if (b2b_bar_num == NTB_CONFIG_BAR) 1637300373Smav bar_addr = peer_addr->bar0_addr; 1638300373Smav else if (b2b_bar_num == NTB_B2B_BAR_1) 1639300373Smav bar_addr = peer_addr->bar2_addr64; 1640300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1641300373Smav bar_addr = peer_addr->bar4_addr64; 1642300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2) 1643300373Smav bar_addr = peer_addr->bar4_addr32; 1644300373Smav else if (b2b_bar_num == NTB_B2B_BAR_3) 1645300373Smav bar_addr = peer_addr->bar5_addr32; 1646300373Smav else 1647300373Smav KASSERT(false, ("invalid bar")); 1648300373Smav 1649300373Smav /* 1650300373Smav * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1651300373Smav * at a time. 1652300373Smav */ 1653300373Smav ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1654300373Smav ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1655300373Smav return (0); 1656250079Scarl} 1657250079Scarl 1658300373Smavstatic inline bool 1659300373Smavlink_is_up(struct ntb_softc *ntb) 1660300373Smav{ 1661300373Smav 1662300373Smav if (ntb->type == NTB_XEON) { 1663300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1664300373Smav return (true); 1665300373Smav return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1666300373Smav } 1667300373Smav 1668300373Smav KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1669300373Smav return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1670300373Smav} 1671300373Smav 1672300373Smavstatic inline bool 1673300373Smavatom_link_is_err(struct ntb_softc *ntb) 1674300373Smav{ 1675300373Smav uint32_t status; 1676300373Smav 1677300373Smav KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1678300373Smav 1679300373Smav status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1680300373Smav if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1681300373Smav return (true); 1682300373Smav 1683300373Smav status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1684300373Smav return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1685300373Smav} 1686300373Smav 1687300373Smav/* Atom does not have link status interrupt, poll on that platform */ 1688250079Scarlstatic void 1689300373Smavatom_link_hb(void *arg) 1690250079Scarl{ 1691300373Smav struct ntb_softc *ntb = arg; 1692300373Smav sbintime_t timo, poll_ts; 1693300373Smav 1694300373Smav timo = NTB_HB_TIMEOUT * hz; 1695300373Smav poll_ts = ntb->last_ts + timo; 1696300373Smav 1697300373Smav /* 1698300373Smav * Delay polling the link status if an interrupt was received, unless 1699300373Smav * the cached link status says the link is down. 1700300373Smav */ 1701300373Smav if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1702300373Smav timo = poll_ts - ticks; 1703300373Smav goto out; 1704300373Smav } 1705300373Smav 1706300373Smav if (ntb_poll_link(ntb)) 1707300373Smav ntb_link_event(ntb); 1708300373Smav 1709300373Smav if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1710300373Smav /* Link is down with error, proceed with recovery */ 1711300373Smav callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1712300373Smav return; 1713300373Smav } 1714300373Smav 1715300373Smavout: 1716300373Smav callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1717300373Smav} 1718300373Smav 1719300373Smavstatic void 1720300373Smavatom_perform_link_restart(struct ntb_softc *ntb) 1721300373Smav{ 1722250079Scarl uint32_t status; 1723250079Scarl 1724250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1725300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1726300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1727300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1728300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1729250079Scarl 1730250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1731250079Scarl pause("ModPhy", hz / 10); 1732250079Scarl 1733250079Scarl /* Clear AER Errors, write to clear */ 1734300373Smav status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1735250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1736300373Smav ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1737250079Scarl 1738250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1739300373Smav status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1740300373Smav status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1741300373Smav ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1742250079Scarl 1743250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1744300373Smav status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1745300373Smav status |= ATOM_DESKEWSTS_DBERR; 1746300373Smav ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1747250079Scarl 1748300373Smav status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1749300373Smav status &= ATOM_IBIST_ERR_OFLOW; 1750300373Smav ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1751250079Scarl 1752250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1753300373Smav status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1754300373Smav status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1755300373Smav ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1756250079Scarl} 1757250079Scarl 1758300373Smav/* 1759300373Smav * ntb_set_ctx() - associate a driver context with an ntb device 1760300373Smav * @ntb: NTB device context 1761300373Smav * @ctx: Driver context 1762300373Smav * @ctx_ops: Driver context operations 1763300373Smav * 1764300373Smav * Associate a driver context and operations with a ntb device. The context is 1765300373Smav * provided by the client driver, and the driver may associate a different 1766300373Smav * context with each ntb device. 1767300373Smav * 1768300373Smav * Return: Zero if the context is associated, otherwise an error number. 1769300373Smav */ 1770300373Smavint 1771300373Smavntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 1772300373Smav{ 1773300373Smav 1774300373Smav if (ctx == NULL || ops == NULL) 1775300373Smav return (EINVAL); 1776300373Smav if (ntb->ctx_ops != NULL) 1777300373Smav return (EINVAL); 1778300373Smav 1779300373Smav CTX_LOCK(ntb); 1780300373Smav if (ntb->ctx_ops != NULL) { 1781300373Smav CTX_UNLOCK(ntb); 1782300373Smav return (EINVAL); 1783300373Smav } 1784300373Smav ntb->ntb_ctx = ctx; 1785300373Smav ntb->ctx_ops = ops; 1786300373Smav CTX_UNLOCK(ntb); 1787300373Smav 1788300373Smav return (0); 1789300373Smav} 1790300373Smav 1791300373Smav/* 1792300373Smav * It is expected that this will only be used from contexts where the ctx_lock 1793300373Smav * is not needed to protect ntb_ctx lifetime. 1794300373Smav */ 1795300373Smavvoid * 1796300373Smavntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 1797300373Smav{ 1798300373Smav 1799300373Smav KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 1800300373Smav if (ops != NULL) 1801300373Smav *ops = ntb->ctx_ops; 1802300373Smav return (ntb->ntb_ctx); 1803300373Smav} 1804300373Smav 1805300373Smav/* 1806300373Smav * ntb_clear_ctx() - disassociate any driver context from an ntb device 1807300373Smav * @ntb: NTB device context 1808300373Smav * 1809300373Smav * Clear any association that may exist between a driver context and the ntb 1810300373Smav * device. 1811300373Smav */ 1812300373Smavvoid 1813300373Smavntb_clear_ctx(struct ntb_softc *ntb) 1814300373Smav{ 1815300373Smav 1816300373Smav CTX_LOCK(ntb); 1817300373Smav ntb->ntb_ctx = NULL; 1818300373Smav ntb->ctx_ops = NULL; 1819300373Smav CTX_UNLOCK(ntb); 1820300373Smav} 1821300373Smav 1822300373Smav/* 1823300373Smav * ntb_link_event() - notify driver context of a change in link status 1824300373Smav * @ntb: NTB device context 1825300373Smav * 1826300373Smav * Notify the driver context that the link status may have changed. The driver 1827300373Smav * should call ntb_link_is_up() to get the current status. 1828300373Smav */ 1829300373Smavvoid 1830300373Smavntb_link_event(struct ntb_softc *ntb) 1831300373Smav{ 1832300373Smav 1833300373Smav CTX_LOCK(ntb); 1834300373Smav if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 1835300373Smav ntb->ctx_ops->link_event(ntb->ntb_ctx); 1836300373Smav CTX_UNLOCK(ntb); 1837300373Smav} 1838300373Smav 1839300373Smav/* 1840300373Smav * ntb_db_event() - notify driver context of a doorbell event 1841300373Smav * @ntb: NTB device context 1842300373Smav * @vector: Interrupt vector number 1843300373Smav * 1844300373Smav * Notify the driver context of a doorbell event. If hardware supports 1845300373Smav * multiple interrupt vectors for doorbells, the vector number indicates which 1846300373Smav * vector received the interrupt. The vector number is relative to the first 1847300373Smav * vector used for doorbells, starting at zero, and must be less than 1848300373Smav * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 1849300373Smav * doorbell bits need service, and ntb_db_vector_mask() to determine which of 1850300373Smav * those bits are associated with the vector number. 1851300373Smav */ 1852250079Scarlstatic void 1853300373Smavntb_db_event(struct ntb_softc *ntb, uint32_t vec) 1854250079Scarl{ 1855250079Scarl 1856300373Smav CTX_LOCK(ntb); 1857300373Smav if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 1858300373Smav ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 1859300373Smav CTX_UNLOCK(ntb); 1860300373Smav} 1861250079Scarl 1862300373Smav/* 1863300373Smav * ntb_link_enable() - enable the link on the secondary side of the ntb 1864300373Smav * @ntb: NTB device context 1865300373Smav * @max_speed: The maximum link speed expressed as PCIe generation number[0] 1866300373Smav * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 1867300373Smav * 1868300373Smav * Enable the link on the secondary side of the ntb. This can only be done 1869300373Smav * from the primary side of the ntb in primary or b2b topology. The ntb device 1870300373Smav * should train the link to its maximum speed and width, or the requested speed 1871300373Smav * and width, whichever is smaller, if supported. 1872300373Smav * 1873300373Smav * Return: Zero on success, otherwise an error number. 1874300373Smav * 1875300373Smav * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 1876300373Smav * and width input will be ignored. 1877300373Smav */ 1878300373Smavint 1879300373Smavntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 1880300373Smav enum ntb_width w __unused) 1881300373Smav{ 1882300373Smav uint32_t cntl; 1883250079Scarl 1884300373Smav if (ntb->type == NTB_ATOM) { 1885300373Smav pci_write_config(ntb->device, NTB_PPD_OFFSET, 1886300373Smav ntb->ppd | ATOM_PPD_INIT_LINK, 4); 1887300373Smav return (0); 1888250079Scarl } 1889250079Scarl 1890300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1891300373Smav ntb_link_event(ntb); 1892300373Smav return (0); 1893300373Smav } 1894300373Smav 1895300373Smav cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1896300373Smav cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1897300373Smav cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1898300373Smav cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1899300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 1900300373Smav cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1901300373Smav ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1902300373Smav return (0); 1903250079Scarl} 1904250079Scarl 1905300373Smav/* 1906300373Smav * ntb_link_disable() - disable the link on the secondary side of the ntb 1907300373Smav * @ntb: NTB device context 1908300373Smav * 1909300373Smav * Disable the link on the secondary side of the ntb. This can only be done 1910300373Smav * from the primary side of the ntb in primary or b2b topology. The ntb device 1911300373Smav * should disable the link. Returning from this call must indicate that a 1912300373Smav * barrier has passed, though with no more writes may pass in either direction 1913300373Smav * across the link, except if this call returns an error number. 1914300373Smav * 1915300373Smav * Return: Zero on success, otherwise an error number. 1916300373Smav */ 1917300373Smavint 1918300373Smavntb_link_disable(struct ntb_softc *ntb) 1919300373Smav{ 1920300373Smav uint32_t cntl; 1921300373Smav 1922300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1923300373Smav ntb_link_event(ntb); 1924300373Smav return (0); 1925300373Smav } 1926300373Smav 1927300373Smav cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1928300373Smav cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1929300373Smav cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1930300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 1931300373Smav cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1932300373Smav cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1933300373Smav ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1934300373Smav return (0); 1935300373Smav} 1936300373Smav 1937250079Scarlstatic void 1938300373Smavrecover_atom_link(void *arg) 1939250079Scarl{ 1940250079Scarl struct ntb_softc *ntb = arg; 1941300373Smav unsigned speed, width, oldspeed, oldwidth; 1942250079Scarl uint32_t status32; 1943250079Scarl 1944300373Smav atom_perform_link_restart(ntb); 1945250079Scarl 1946300373Smav /* 1947300373Smav * There is a potential race between the 2 NTB devices recovering at 1948300373Smav * the same time. If the times are the same, the link will not recover 1949300373Smav * and the driver will be stuck in this loop forever. Add a random 1950300373Smav * interval to the recovery time to prevent this race. 1951300373Smav */ 1952300373Smav status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 1953300373Smav pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 1954250079Scarl 1955300373Smav if (atom_link_is_err(ntb)) 1956250079Scarl goto retry; 1957250079Scarl 1958300373Smav status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1959300373Smav if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 1960300373Smav goto out; 1961300373Smav 1962300373Smav status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1963300373Smav width = NTB_LNK_STA_WIDTH(status32); 1964300373Smav speed = status32 & NTB_LINK_SPEED_MASK; 1965300373Smav 1966300373Smav oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 1967300373Smav oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 1968300373Smav if (oldwidth != width || oldspeed != speed) 1969250079Scarl goto retry; 1970250079Scarl 1971300373Smavout: 1972300373Smav callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 1973300373Smav ntb); 1974250079Scarl return; 1975250079Scarl 1976250079Scarlretry: 1977300373Smav callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 1978250079Scarl ntb); 1979250079Scarl} 1980250079Scarl 1981300373Smav/* 1982300373Smav * Polls the HW link status register(s); returns true if something has changed. 1983300373Smav */ 1984300373Smavstatic bool 1985300373Smavntb_poll_link(struct ntb_softc *ntb) 1986250079Scarl{ 1987250079Scarl uint32_t ntb_cntl; 1988300373Smav uint16_t reg_val; 1989250079Scarl 1990300373Smav if (ntb->type == NTB_ATOM) { 1991300373Smav ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1992300373Smav if (ntb_cntl == ntb->ntb_ctl) 1993300373Smav return (false); 1994300373Smav 1995300373Smav ntb->ntb_ctl = ntb_cntl; 1996300373Smav ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1997250079Scarl } else { 1998300373Smav db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 1999250079Scarl 2000300373Smav reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2001300373Smav if (reg_val == ntb->lnk_sta) 2002300373Smav return (false); 2003300373Smav 2004300373Smav ntb->lnk_sta = reg_val; 2005250079Scarl } 2006300373Smav return (true); 2007300373Smav} 2008250079Scarl 2009300373Smavstatic inline enum ntb_speed 2010300373Smavntb_link_sta_speed(struct ntb_softc *ntb) 2011300373Smav{ 2012250079Scarl 2013300373Smav if (!link_is_up(ntb)) 2014300373Smav return (NTB_SPEED_NONE); 2015300373Smav return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2016250079Scarl} 2017250079Scarl 2018300373Smavstatic inline enum ntb_width 2019300373Smavntb_link_sta_width(struct ntb_softc *ntb) 2020250079Scarl{ 2021250079Scarl 2022300373Smav if (!link_is_up(ntb)) 2023300373Smav return (NTB_WIDTH_NONE); 2024300373Smav return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2025300373Smav} 2026250079Scarl 2027300373SmavSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2028300373Smav "Driver state, statistics, and HW registers"); 2029250079Scarl 2030300373Smav#define NTB_REGSZ_MASK (3ul << 30) 2031300373Smav#define NTB_REG_64 (1ul << 30) 2032300373Smav#define NTB_REG_32 (2ul << 30) 2033300373Smav#define NTB_REG_16 (3ul << 30) 2034300373Smav#define NTB_REG_8 (0ul << 30) 2035250079Scarl 2036300373Smav#define NTB_DB_READ (1ul << 29) 2037300373Smav#define NTB_PCI_REG (1ul << 28) 2038300373Smav#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2039300373Smav 2040300373Smavstatic void 2041300373Smavntb_sysctl_init(struct ntb_softc *ntb) 2042250079Scarl{ 2043300373Smav struct sysctl_oid_list *tree_par, *regpar, *statpar, *errpar; 2044300373Smav struct sysctl_ctx_list *ctx; 2045300373Smav struct sysctl_oid *tree, *tmptree; 2046250079Scarl 2047300373Smav ctx = device_get_sysctl_ctx(ntb->device); 2048250079Scarl 2049300373Smav tree = SYSCTL_ADD_NODE(ctx, 2050300373Smav SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)), OID_AUTO, 2051300373Smav "debug_info", CTLFLAG_RD, NULL, 2052300373Smav "Driver state, statistics, and HW registers"); 2053300373Smav tree_par = SYSCTL_CHILDREN(tree); 2054250079Scarl 2055300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2056300373Smav &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2057300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2058300373Smav &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2059300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2060300373Smav &ntb->ppd, 0, "Raw PPD register (cached)"); 2061300373Smav 2062300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2063300373Smav#ifdef notyet 2064300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2065300373Smav &ntb->b2b_mw_idx, 0, 2066300373Smav "Index of the MW used for B2B remote register access"); 2067300373Smav#endif 2068300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2069300373Smav CTLFLAG_RD, &ntb->b2b_off, 2070300373Smav "If non-zero, offset of B2B register region in shared MW"); 2071250079Scarl } 2072250079Scarl 2073300373Smav SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2074300373Smav CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2075300373Smav "Features/errata of this NTB device"); 2076250079Scarl 2077300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2078300373Smav __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2079300373Smav "NTB CTL register (cached)"); 2080300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2081300373Smav __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2082300373Smav "LNK STA register (cached)"); 2083250079Scarl 2084300373Smav SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "link_status", 2085300373Smav CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_link_status, 2086300373Smav "A", "Link status"); 2087250079Scarl 2088300373Smav#ifdef notyet 2089300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2090300373Smav &ntb->mw_count, 0, "MW count"); 2091300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2092300373Smav &ntb->spad_count, 0, "Scratchpad count"); 2093300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2094300373Smav &ntb->db_count, 0, "Doorbell count"); 2095300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2096300373Smav &ntb->db_vec_count, 0, "Doorbell vector count"); 2097300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2098300373Smav &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2099300373Smav#endif 2100250079Scarl 2101300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2102300373Smav &ntb->db_valid_mask, "Doorbell valid mask"); 2103300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2104300373Smav &ntb->db_link_mask, "Doorbell link mask"); 2105300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2106300373Smav &ntb->db_mask, "Doorbell mask (cached)"); 2107300373Smav 2108300373Smav tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2109300373Smav CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2110300373Smav regpar = SYSCTL_CHILDREN(tmptree); 2111300373Smav 2112300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2113300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2114300373Smav ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2115300373Smav "NTB Control register"); 2116300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2117300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2118300373Smav 0x19c, sysctl_handle_register, "IU", 2119300373Smav "NTB Link Capabilities"); 2120300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2121300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2122300373Smav 0x1a0, sysctl_handle_register, "IU", 2123300373Smav "NTB Link Control register"); 2124300373Smav 2125300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2126300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2127300373Smav NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2128300373Smav sysctl_handle_register, "QU", "Doorbell mask register"); 2129300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2130300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2131300373Smav NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2132300373Smav sysctl_handle_register, "QU", "Doorbell register"); 2133300373Smav 2134300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2135300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2136300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2137300373Smav sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2138300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2139300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2140300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2141300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2142300373Smav sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2143300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2144300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2145300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2146300373Smav sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2147300373Smav } else { 2148300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2149300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2150300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2151300373Smav sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2152300373Smav } 2153300373Smav 2154300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2155300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2156300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2157300373Smav sysctl_handle_register, "QU", "Incoming LMT23 register"); 2158300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2159300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2160300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2161300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2162300373Smav sysctl_handle_register, "IU", "Incoming LMT4 register"); 2163300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2164300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2165300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2166300373Smav sysctl_handle_register, "IU", "Incoming LMT5 register"); 2167300373Smav } else { 2168300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2169300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2170300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2171300373Smav sysctl_handle_register, "QU", "Incoming LMT45 register"); 2172300373Smav } 2173300373Smav 2174300373Smav if (ntb->type == NTB_ATOM) 2175250079Scarl return; 2176250079Scarl 2177300373Smav tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2178300373Smav CTLFLAG_RD, NULL, "Xeon HW statistics"); 2179300373Smav statpar = SYSCTL_CHILDREN(tmptree); 2180300373Smav SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2181300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2182300373Smav NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2183300373Smav sysctl_handle_register, "SU", "Upstream Memory Miss"); 2184250079Scarl 2185300373Smav tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2186300373Smav CTLFLAG_RD, NULL, "Xeon HW errors"); 2187300373Smav errpar = SYSCTL_CHILDREN(tmptree); 2188300373Smav 2189300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2190300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2191300373Smav NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2192300373Smav sysctl_handle_register, "CU", "PPD"); 2193300373Smav 2194300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2195300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2196300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2197300373Smav sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2198300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2199300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2200300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2201300373Smav sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2202300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2203300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2204300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2205300373Smav sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2206300373Smav 2207300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2208300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2209300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2210300373Smav sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2211300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2212300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2213300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2214300373Smav sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2215300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2216300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2217300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2218300373Smav sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2219300373Smav 2220300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2221300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2222300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2223300373Smav sysctl_handle_register, "SU", "DEVSTS"); 2224300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2225300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2226300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2227300373Smav sysctl_handle_register, "SU", "LNKSTS"); 2228300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2229300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2230300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2231300373Smav sysctl_handle_register, "SU", "SLNKSTS"); 2232300373Smav 2233300373Smav SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2234300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2235300373Smav NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2236300373Smav sysctl_handle_register, "IU", "UNCERRSTS"); 2237300373Smav SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2238300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2239300373Smav NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2240300373Smav sysctl_handle_register, "IU", "CORERRSTS"); 2241300373Smav 2242300373Smav if (ntb->conn_type != NTB_CONN_B2B) 2243300373Smav return; 2244300373Smav 2245300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2246300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2247300373Smav NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2248300373Smav sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2249300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2250300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2251300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2252300373Smav NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2253300373Smav sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2254300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2255300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2256300373Smav NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2257300373Smav sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2258300373Smav } else { 2259300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2260300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2261300373Smav NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2262300373Smav sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2263300373Smav } 2264300373Smav 2265300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2266300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2267300373Smav NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2268300373Smav sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2269300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2270300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2271300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2272300373Smav NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2273300373Smav sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2274300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2275300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2276300373Smav NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2277300373Smav sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2278300373Smav } else { 2279300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2280300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2281300373Smav NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2282300373Smav sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2283300373Smav } 2284300373Smav 2285300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2286300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2287300373Smav NTB_REG_64 | ntb->xlat_reg->bar0_base, 2288300373Smav sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2289300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2290300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2291300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_base, 2292300373Smav sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2293300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2294300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2295300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2296300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_base, 2297300373Smav sysctl_handle_register, "IU", 2298300373Smav "Secondary BAR4 base register"); 2299300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2300300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2301300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_base, 2302300373Smav sysctl_handle_register, "IU", 2303300373Smav "Secondary BAR5 base register"); 2304300373Smav } else { 2305300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2306300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2307300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_base, 2308300373Smav sysctl_handle_register, "QU", 2309300373Smav "Secondary BAR45 base register"); 2310300373Smav } 2311250079Scarl} 2312250079Scarl 2313300373Smavstatic int 2314300373Smavsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2315250079Scarl{ 2316300373Smav struct ntb_softc *ntb; 2317300373Smav struct sbuf sb; 2318300373Smav int error; 2319250079Scarl 2320300373Smav error = 0; 2321300373Smav ntb = arg1; 2322300373Smav 2323300373Smav sbuf_new_for_sysctl(&sb, NULL, 256, req); 2324300373Smav 2325300373Smav sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2326300373Smav error = sbuf_finish(&sb); 2327300373Smav sbuf_delete(&sb); 2328300373Smav 2329300373Smav if (error || !req->newptr) 2330300373Smav return (error); 2331300373Smav return (EINVAL); 2332250079Scarl} 2333250079Scarl 2334300373Smavstatic int 2335300373Smavsysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2336250079Scarl{ 2337300373Smav struct ntb_softc *ntb; 2338300373Smav struct sbuf sb; 2339300373Smav enum ntb_speed speed; 2340300373Smav enum ntb_width width; 2341300373Smav int error; 2342250079Scarl 2343300373Smav error = 0; 2344300373Smav ntb = arg1; 2345250079Scarl 2346300373Smav sbuf_new_for_sysctl(&sb, NULL, 32, req); 2347300373Smav 2348300373Smav if (ntb_link_is_up(ntb, &speed, &width)) 2349300373Smav sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2350300373Smav (unsigned)speed, (unsigned)width); 2351300373Smav else 2352300373Smav sbuf_printf(&sb, "down"); 2353300373Smav 2354300373Smav error = sbuf_finish(&sb); 2355300373Smav sbuf_delete(&sb); 2356300373Smav 2357300373Smav if (error || !req->newptr) 2358300373Smav return (error); 2359300373Smav return (EINVAL); 2360250079Scarl} 2361250079Scarl 2362300373Smavstatic int 2363300373Smavsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2364250079Scarl{ 2365300373Smav struct ntb_softc *ntb; 2366300373Smav const void *outp; 2367300373Smav uintptr_t sz; 2368300373Smav uint64_t umv; 2369300373Smav char be[sizeof(umv)]; 2370300373Smav size_t outsz; 2371300373Smav uint32_t reg; 2372300373Smav bool db, pci; 2373300373Smav int error; 2374250079Scarl 2375300373Smav ntb = arg1; 2376300373Smav reg = arg2 & ~NTB_REGFLAGS_MASK; 2377300373Smav sz = arg2 & NTB_REGSZ_MASK; 2378300373Smav db = (arg2 & NTB_DB_READ) != 0; 2379300373Smav pci = (arg2 & NTB_PCI_REG) != 0; 2380250079Scarl 2381300373Smav KASSERT(!(db && pci), ("bogus")); 2382250079Scarl 2383300373Smav if (db) { 2384300373Smav KASSERT(sz == NTB_REG_64, ("bogus")); 2385300373Smav umv = db_ioread(ntb, reg); 2386300373Smav outsz = sizeof(uint64_t); 2387300373Smav } else { 2388300373Smav switch (sz) { 2389300373Smav case NTB_REG_64: 2390300373Smav if (pci) 2391300373Smav umv = pci_read_config(ntb->device, reg, 8); 2392300373Smav else 2393300373Smav umv = ntb_reg_read(8, reg); 2394300373Smav outsz = sizeof(uint64_t); 2395300373Smav break; 2396300373Smav case NTB_REG_32: 2397300373Smav if (pci) 2398300373Smav umv = pci_read_config(ntb->device, reg, 4); 2399300373Smav else 2400300373Smav umv = ntb_reg_read(4, reg); 2401300373Smav outsz = sizeof(uint32_t); 2402300373Smav break; 2403300373Smav case NTB_REG_16: 2404300373Smav if (pci) 2405300373Smav umv = pci_read_config(ntb->device, reg, 2); 2406300373Smav else 2407300373Smav umv = ntb_reg_read(2, reg); 2408300373Smav outsz = sizeof(uint16_t); 2409300373Smav break; 2410300373Smav case NTB_REG_8: 2411300373Smav if (pci) 2412300373Smav umv = pci_read_config(ntb->device, reg, 1); 2413300373Smav else 2414300373Smav umv = ntb_reg_read(1, reg); 2415300373Smav outsz = sizeof(uint8_t); 2416300373Smav break; 2417300373Smav default: 2418300373Smav panic("bogus"); 2419300373Smav break; 2420300373Smav } 2421300373Smav } 2422300373Smav 2423300373Smav /* Encode bigendian so that sysctl -x is legible. */ 2424300373Smav be64enc(be, umv); 2425300373Smav outp = ((char *)be) + sizeof(umv) - outsz; 2426300373Smav 2427300373Smav error = SYSCTL_OUT(req, outp, outsz); 2428300373Smav if (error || !req->newptr) 2429300373Smav return (error); 2430300373Smav return (EINVAL); 2431250079Scarl} 2432250079Scarl 2433300373Smavstatic unsigned 2434300373Smavntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2435300373Smav{ 2436300373Smav 2437300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2438300373Smav uidx >= ntb->b2b_mw_idx) 2439300373Smav return (uidx + 1); 2440300373Smav return (uidx); 2441300373Smav} 2442300373Smav 2443300373Smav/* 2444300373Smav * Public API to the rest of the OS 2445300373Smav */ 2446300373Smav 2447250079Scarl/** 2448250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 2449250079Scarl * @ntb: pointer to ntb_softc instance 2450250079Scarl * 2451250079Scarl * This function returns the max 32bit scratchpad registers usable by the 2452250079Scarl * upper layer. 2453250079Scarl * 2454250079Scarl * RETURNS: total number of scratch pad registers available 2455250079Scarl */ 2456300373Smavuint8_t 2457250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 2458250079Scarl{ 2459250079Scarl 2460300373Smav return (ntb->spad_count); 2461250079Scarl} 2462250079Scarl 2463300373Smav/* 2464300373Smav * ntb_mw_count() - Get the number of memory windows available for KPI 2465300373Smav * consumers. 2466300373Smav * 2467300373Smav * (Excludes any MW wholly reserved for register access.) 2468300373Smav */ 2469300373Smavuint8_t 2470300373Smavntb_mw_count(struct ntb_softc *ntb) 2471300373Smav{ 2472300373Smav 2473300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2474300373Smav return (ntb->mw_count - 1); 2475300373Smav return (ntb->mw_count); 2476300373Smav} 2477300373Smav 2478250079Scarl/** 2479300373Smav * ntb_spad_write() - write to the secondary scratchpad register 2480250079Scarl * @ntb: pointer to ntb_softc instance 2481250079Scarl * @idx: index to the scratchpad register, 0 based 2482250079Scarl * @val: the data value to put into the register 2483250079Scarl * 2484250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2485250079Scarl * register. The register resides on the secondary (external) side. 2486250079Scarl * 2487300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2488250079Scarl */ 2489250079Scarlint 2490300373Smavntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2491250079Scarl{ 2492250079Scarl 2493300373Smav if (idx >= ntb->spad_count) 2494250079Scarl return (EINVAL); 2495250079Scarl 2496300373Smav ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2497250079Scarl 2498250079Scarl return (0); 2499250079Scarl} 2500250079Scarl 2501250079Scarl/** 2502300373Smav * ntb_spad_read() - read from the primary scratchpad register 2503250079Scarl * @ntb: pointer to ntb_softc instance 2504250079Scarl * @idx: index to scratchpad register, 0 based 2505250079Scarl * @val: pointer to 32bit integer for storing the register value 2506250079Scarl * 2507250079Scarl * This function allows reading of the 32bit scratchpad register on 2508250079Scarl * the primary (internal) side. 2509250079Scarl * 2510300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2511250079Scarl */ 2512250079Scarlint 2513300373Smavntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2514250079Scarl{ 2515250079Scarl 2516300373Smav if (idx >= ntb->spad_count) 2517250079Scarl return (EINVAL); 2518250079Scarl 2519300373Smav *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2520250079Scarl 2521250079Scarl return (0); 2522250079Scarl} 2523250079Scarl 2524250079Scarl/** 2525300373Smav * ntb_peer_spad_write() - write to the secondary scratchpad register 2526250079Scarl * @ntb: pointer to ntb_softc instance 2527250079Scarl * @idx: index to the scratchpad register, 0 based 2528250079Scarl * @val: the data value to put into the register 2529250079Scarl * 2530250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2531250079Scarl * register. The register resides on the secondary (external) side. 2532250079Scarl * 2533300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2534250079Scarl */ 2535250079Scarlint 2536300373Smavntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2537250079Scarl{ 2538250079Scarl 2539300373Smav if (idx >= ntb->spad_count) 2540250079Scarl return (EINVAL); 2541250079Scarl 2542300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2543300373Smav ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2544255279Scarl else 2545300373Smav ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2546250079Scarl 2547250079Scarl return (0); 2548250079Scarl} 2549250079Scarl 2550250079Scarl/** 2551300373Smav * ntb_peer_spad_read() - read from the primary scratchpad register 2552250079Scarl * @ntb: pointer to ntb_softc instance 2553250079Scarl * @idx: index to scratchpad register, 0 based 2554250079Scarl * @val: pointer to 32bit integer for storing the register value 2555250079Scarl * 2556250079Scarl * This function allows reading of the 32bit scratchpad register on 2557250079Scarl * the primary (internal) side. 2558250079Scarl * 2559300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2560250079Scarl */ 2561250079Scarlint 2562300373Smavntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2563250079Scarl{ 2564250079Scarl 2565300373Smav if (idx >= ntb->spad_count) 2566250079Scarl return (EINVAL); 2567250079Scarl 2568300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2569300373Smav *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2570255279Scarl else 2571300373Smav *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2572250079Scarl 2573250079Scarl return (0); 2574250079Scarl} 2575250079Scarl 2576300373Smav/* 2577300373Smav * ntb_mw_get_range() - get the range of a memory window 2578300373Smav * @ntb: NTB device context 2579300373Smav * @idx: Memory window number 2580300373Smav * @base: OUT - the base address for mapping the memory window 2581300373Smav * @size: OUT - the size for mapping the memory window 2582300373Smav * @align: OUT - the base alignment for translating the memory window 2583300373Smav * @align_size: OUT - the size alignment for translating the memory window 2584250079Scarl * 2585300373Smav * Get the range of a memory window. NULL may be given for any output 2586300373Smav * parameter if the value is not needed. The base and size may be used for 2587300373Smav * mapping the memory window, to access the peer memory. The alignment and 2588300373Smav * size may be used for translating the memory window, for the peer to access 2589300373Smav * memory on the local system. 2590250079Scarl * 2591300373Smav * Return: Zero on success, otherwise an error number. 2592250079Scarl */ 2593300373Smavint 2594300373Smavntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 2595300373Smav caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2596300373Smav bus_addr_t *plimit) 2597250079Scarl{ 2598300373Smav struct ntb_pci_bar_info *bar; 2599300373Smav bus_addr_t limit; 2600300373Smav size_t bar_b2b_off; 2601300373Smav enum ntb_bar bar_num; 2602250079Scarl 2603300373Smav if (mw_idx >= ntb_mw_count(ntb)) 2604300373Smav return (EINVAL); 2605300373Smav mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 2606250079Scarl 2607300373Smav bar_num = ntb_mw_to_bar(ntb, mw_idx); 2608300373Smav bar = &ntb->bar_info[bar_num]; 2609300373Smav bar_b2b_off = 0; 2610300373Smav if (mw_idx == ntb->b2b_mw_idx) { 2611300373Smav KASSERT(ntb->b2b_off != 0, 2612300373Smav ("user shouldn't get non-shared b2b mw")); 2613300373Smav bar_b2b_off = ntb->b2b_off; 2614300373Smav } 2615300373Smav 2616300373Smav if (bar_is_64bit(ntb, bar_num)) 2617300373Smav limit = BUS_SPACE_MAXADDR; 2618300373Smav else 2619300373Smav limit = BUS_SPACE_MAXADDR_32BIT; 2620300373Smav 2621300373Smav if (base != NULL) 2622300373Smav *base = bar->pbase + bar_b2b_off; 2623300373Smav if (vbase != NULL) 2624300373Smav *vbase = bar->vbase + bar_b2b_off; 2625300373Smav if (size != NULL) 2626300373Smav *size = bar->size - bar_b2b_off; 2627300373Smav if (align != NULL) 2628300373Smav *align = bar->size; 2629300373Smav if (align_size != NULL) 2630300373Smav *align_size = 1; 2631300373Smav if (plimit != NULL) 2632300373Smav *plimit = limit; 2633300373Smav return (0); 2634250079Scarl} 2635250079Scarl 2636300373Smav/* 2637300373Smav * ntb_mw_set_trans() - set the translation of a memory window 2638300373Smav * @ntb: NTB device context 2639300373Smav * @idx: Memory window number 2640300373Smav * @addr: The dma address local memory to expose to the peer 2641300373Smav * @size: The size of the local memory to expose to the peer 2642300373Smav * 2643300373Smav * Set the translation of a memory window. The peer may access local memory 2644300373Smav * through the window starting at the address, up to the size. The address 2645300373Smav * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2646300373Smav * must be aligned to the size alignment specified by ntb_mw_get_range(). The 2647300373Smav * address must be below the plimit specified by ntb_mw_get_range() (i.e. for 2648300373Smav * 32-bit BARs). 2649300373Smav * 2650300373Smav * Return: Zero on success, otherwise an error number. 2651300373Smav */ 2652300373Smavint 2653300373Smavntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2654300373Smav size_t size) 2655250079Scarl{ 2656300373Smav struct ntb_pci_bar_info *bar; 2657300373Smav uint64_t base, limit, reg_val; 2658300373Smav size_t bar_size, mw_size; 2659300373Smav uint32_t base_reg, xlat_reg, limit_reg; 2660300373Smav enum ntb_bar bar_num; 2661250079Scarl 2662300373Smav if (idx >= ntb_mw_count(ntb)) 2663300373Smav return (EINVAL); 2664300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 2665250079Scarl 2666300373Smav bar_num = ntb_mw_to_bar(ntb, idx); 2667300373Smav bar = &ntb->bar_info[bar_num]; 2668300373Smav 2669300373Smav bar_size = bar->size; 2670300373Smav if (idx == ntb->b2b_mw_idx) 2671300373Smav mw_size = bar_size - ntb->b2b_off; 2672300373Smav else 2673300373Smav mw_size = bar_size; 2674300373Smav 2675300373Smav /* Hardware requires that addr is aligned to bar size */ 2676300373Smav if ((addr & (bar_size - 1)) != 0) 2677300373Smav return (EINVAL); 2678300373Smav 2679300373Smav if (size > mw_size) 2680300373Smav return (EINVAL); 2681300373Smav 2682300373Smav bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2683300373Smav 2684300373Smav limit = 0; 2685300373Smav if (bar_is_64bit(ntb, bar_num)) { 2686300373Smav base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2687300373Smav 2688300373Smav if (limit_reg != 0 && size != mw_size) 2689300373Smav limit = base + size; 2690300373Smav 2691300373Smav /* Set and verify translation address */ 2692300373Smav ntb_reg_write(8, xlat_reg, addr); 2693300373Smav reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2694300373Smav if (reg_val != addr) { 2695300373Smav ntb_reg_write(8, xlat_reg, 0); 2696300373Smav return (EIO); 2697300373Smav } 2698300373Smav 2699300373Smav /* Set and verify the limit */ 2700300373Smav ntb_reg_write(8, limit_reg, limit); 2701300373Smav reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 2702300373Smav if (reg_val != limit) { 2703300373Smav ntb_reg_write(8, limit_reg, base); 2704300373Smav ntb_reg_write(8, xlat_reg, 0); 2705300373Smav return (EIO); 2706300373Smav } 2707300373Smav } else { 2708300373Smav /* Configure 32-bit (split) BAR MW */ 2709300373Smav 2710300373Smav if ((addr & UINT32_MAX) != addr) 2711300373Smav return (ERANGE); 2712300373Smav if (((addr + size) & UINT32_MAX) != (addr + size)) 2713300373Smav return (ERANGE); 2714300373Smav 2715300373Smav base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 2716300373Smav 2717300373Smav if (limit_reg != 0 && size != mw_size) 2718300373Smav limit = base + size; 2719300373Smav 2720300373Smav /* Set and verify translation address */ 2721300373Smav ntb_reg_write(4, xlat_reg, addr); 2722300373Smav reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 2723300373Smav if (reg_val != addr) { 2724300373Smav ntb_reg_write(4, xlat_reg, 0); 2725300373Smav return (EIO); 2726300373Smav } 2727300373Smav 2728300373Smav /* Set and verify the limit */ 2729300373Smav ntb_reg_write(4, limit_reg, limit); 2730300373Smav reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 2731300373Smav if (reg_val != limit) { 2732300373Smav ntb_reg_write(4, limit_reg, base); 2733300373Smav ntb_reg_write(4, xlat_reg, 0); 2734300373Smav return (EIO); 2735300373Smav } 2736300373Smav } 2737300373Smav return (0); 2738250079Scarl} 2739250079Scarl 2740300373Smav/* 2741300373Smav * ntb_mw_clear_trans() - clear the translation of a memory window 2742300373Smav * @ntb: NTB device context 2743300373Smav * @idx: Memory window number 2744250079Scarl * 2745300373Smav * Clear the translation of a memory window. The peer may no longer access 2746300373Smav * local memory through the window. 2747250079Scarl * 2748300373Smav * Return: Zero on success, otherwise an error number. 2749250079Scarl */ 2750300373Smavint 2751300373Smavntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 2752250079Scarl{ 2753250079Scarl 2754300373Smav return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 2755300373Smav} 2756300373Smav 2757300373Smav/* 2758300373Smav * ntb_mw_get_wc - Get the write-combine status of a memory window 2759300373Smav * 2760300373Smav * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 2761300373Smav * idx is an invalid memory window). 2762300373Smav * 2763300373Smav * Mode is a VM_MEMATTR_* type. 2764300373Smav */ 2765300373Smavint 2766300373Smavntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t *mode) 2767300373Smav{ 2768300373Smav struct ntb_pci_bar_info *bar; 2769300373Smav 2770300373Smav if (idx >= ntb_mw_count(ntb)) 2771300373Smav return (EINVAL); 2772300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 2773300373Smav 2774300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 2775300373Smav *mode = bar->map_mode; 2776300373Smav return (0); 2777300373Smav} 2778300373Smav 2779300373Smav/* 2780300373Smav * ntb_mw_set_wc - Set the write-combine status of a memory window 2781300373Smav * 2782300373Smav * If 'mode' matches the current status, this does nothing and succeeds. Mode 2783300373Smav * is a VM_MEMATTR_* type. 2784300373Smav * 2785300373Smav * Returns: Zero on success, setting the caching attribute on the virtual 2786300373Smav * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 2787300373Smav * memory window, or if changing the caching attribute fails). 2788300373Smav */ 2789300373Smavint 2790300373Smavntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 2791300373Smav{ 2792300373Smav 2793300373Smav if (idx >= ntb_mw_count(ntb)) 2794300373Smav return (EINVAL); 2795300373Smav 2796300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 2797300373Smav return (ntb_mw_set_wc_internal(ntb, idx, mode)); 2798300373Smav} 2799300373Smav 2800300373Smavstatic int 2801300373Smavntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 2802300373Smav{ 2803300373Smav struct ntb_pci_bar_info *bar; 2804300373Smav int rc; 2805300373Smav 2806300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 2807300373Smav if (bar->map_mode == mode) 2808250079Scarl return (0); 2809250079Scarl 2810300373Smav rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 2811300373Smav if (rc == 0) 2812300373Smav bar->map_mode = mode; 2813300373Smav 2814300373Smav return (rc); 2815250079Scarl} 2816250079Scarl 2817250079Scarl/** 2818300373Smav * ntb_peer_db_set() - Set the doorbell on the secondary/external side 2819250079Scarl * @ntb: pointer to ntb_softc instance 2820300373Smav * @bit: doorbell bits to ring 2821250079Scarl * 2822300373Smav * This function allows triggering of a doorbell on the secondary/external 2823300373Smav * side that will initiate an interrupt on the remote host 2824250079Scarl */ 2825250079Scarlvoid 2826300373Smavntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 2827250079Scarl{ 2828250079Scarl 2829300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2830300373Smav ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 2831250079Scarl return; 2832300373Smav } 2833250079Scarl 2834300373Smav db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2835300373Smav} 2836300373Smav 2837300373Smav/* 2838300373Smav * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 2839300373Smav * as well as the size of the register (via *sz_out). 2840300373Smav * 2841300373Smav * This function allows a caller using I/OAT DMA to chain the remote doorbell 2842300373Smav * ring to its memory window write. 2843300373Smav * 2844300373Smav * Note that writing the peer doorbell via a memory window will *not* generate 2845300373Smav * an interrupt on the remote host; that must be done seperately. 2846300373Smav */ 2847300373Smavbus_addr_t 2848300373Smavntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 2849300373Smav{ 2850300373Smav struct ntb_pci_bar_info *bar; 2851300373Smav uint64_t regoff; 2852300373Smav 2853300373Smav KASSERT(sz_out != NULL, ("must be non-NULL")); 2854300373Smav 2855300373Smav if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2856300373Smav bar = &ntb->bar_info[NTB_CONFIG_BAR]; 2857300373Smav regoff = ntb->peer_reg->db_bell; 2858300373Smav } else { 2859300373Smav KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 2860300373Smav ("invalid b2b idx")); 2861300373Smav 2862300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 2863300373Smav regoff = XEON_PDOORBELL_OFFSET; 2864250079Scarl } 2865300373Smav KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 2866300373Smav 2867300373Smav *sz_out = ntb->reg->db_size; 2868300373Smav /* HACK: Specific to current x86 bus implementation. */ 2869300373Smav return ((uint64_t)bar->pci_bus_handle + regoff); 2870250079Scarl} 2871250079Scarl 2872300373Smav/* 2873300373Smav * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 2874300373Smav * @ntb: NTB device context 2875250079Scarl * 2876300373Smav * Hardware may support different number or arrangement of doorbell bits. 2877250079Scarl * 2878300373Smav * Return: A mask of doorbell bits supported by the ntb. 2879250079Scarl */ 2880300373Smavuint64_t 2881300373Smavntb_db_valid_mask(struct ntb_softc *ntb) 2882250079Scarl{ 2883250079Scarl 2884300373Smav return (ntb->db_valid_mask); 2885250079Scarl} 2886250079Scarl 2887300373Smav/* 2888300373Smav * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 2889300373Smav * @ntb: NTB device context 2890300373Smav * @vector: Doorbell vector number 2891300373Smav * 2892300373Smav * Each interrupt vector may have a different number or arrangement of bits. 2893300373Smav * 2894300373Smav * Return: A mask of doorbell bits serviced by a vector. 2895300373Smav */ 2896300373Smavuint64_t 2897300373Smavntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 2898300373Smav{ 2899300373Smav 2900300373Smav if (vector > ntb->db_vec_count) 2901300373Smav return (0); 2902300373Smav return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 2903300373Smav} 2904300373Smav 2905250079Scarl/** 2906300373Smav * ntb_link_is_up() - get the current ntb link state 2907300373Smav * @ntb: NTB device context 2908300373Smav * @speed: OUT - The link speed expressed as PCIe generation number 2909300373Smav * @width: OUT - The link width expressed as the number of PCIe lanes 2910250079Scarl * 2911250079Scarl * RETURNS: true or false based on the hardware link state 2912250079Scarl */ 2913250079Scarlbool 2914300373Smavntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 2915300373Smav enum ntb_width *width) 2916250079Scarl{ 2917250079Scarl 2918300373Smav if (speed != NULL) 2919300373Smav *speed = ntb_link_sta_speed(ntb); 2920300373Smav if (width != NULL) 2921300373Smav *width = ntb_link_sta_width(ntb); 2922300373Smav return (link_is_up(ntb)); 2923250079Scarl} 2924250079Scarl 2925255272Scarlstatic void 2926255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 2927250079Scarl{ 2928255272Scarl 2929300373Smav bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2930300373Smav bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2931300373Smav bar->pbase = rman_get_start(bar->pci_resource); 2932300373Smav bar->size = rman_get_size(bar->pci_resource); 2933300373Smav bar->vbase = rman_get_virtual(bar->pci_resource); 2934250079Scarl} 2935255268Scarl 2936300373Smavdevice_t 2937300373Smavntb_get_device(struct ntb_softc *ntb) 2938255268Scarl{ 2939255268Scarl 2940255268Scarl return (ntb->device); 2941255268Scarl} 2942300373Smav 2943300373Smav/* Export HW-specific errata information. */ 2944300373Smavbool 2945300373Smavntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 2946300373Smav{ 2947300373Smav 2948300373Smav return (HAS_FEATURE(feature)); 2949300373Smav} 2950