ntb_hw.c revision 304376
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3300373Smav * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c 304376 2016-08-18 10:24:31Z mav $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35300373Smav#include <sys/endian.h> 36250079Scarl#include <sys/malloc.h> 37250079Scarl#include <sys/module.h> 38301811Sngie#include <sys/mutex.h> 39301811Sngie#include <sys/pciio.h> 40250079Scarl#include <sys/queue.h> 41250079Scarl#include <sys/rman.h> 42300373Smav#include <sys/sbuf.h> 43300373Smav#include <sys/sysctl.h> 44250079Scarl#include <vm/vm.h> 45250079Scarl#include <vm/pmap.h> 46250079Scarl#include <machine/bus.h> 47301811Sngie#include <machine/intr_machdep.h> 48250079Scarl#include <machine/pmap.h> 49250079Scarl#include <machine/resource.h> 50250079Scarl#include <dev/pci/pcireg.h> 51250079Scarl#include <dev/pci/pcivar.h> 52250079Scarl 53250079Scarl#include "ntb_regs.h" 54250079Scarl#include "ntb_hw.h" 55250079Scarl 56250079Scarl/* 57250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 58250079Scarl * allows you to connect two systems using a PCI-e link. 59250079Scarl * 60250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 61250079Scarl * you to send and recieve interrupts, map the memory windows and send and 62250079Scarl * receive messages in the scratch-pad registers. 63250079Scarl * 64250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 65250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 66250079Scarl */ 67250079Scarl 68300373Smav#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 69250079Scarl 70300373Smav#define NTB_HB_TIMEOUT 1 /* second */ 71300373Smav#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 72300373Smav#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 73250079Scarl 74250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 75250079Scarl 76301811Sngie#define NTB_MSIX_VER_GUARD 0xaabbccdd 77301811Sngie#define NTB_MSIX_RECEIVED 0xe0f0e0f0 78301811Sngie 79301811Sngie/* 80301811Sngie * PCI constants could be somewhere more generic, but aren't defined/used in 81301811Sngie * pci.c. 82301811Sngie */ 83301811Sngie#define PCI_MSIX_ENTRY_SIZE 16 84301811Sngie#define PCI_MSIX_ENTRY_LOWER_ADDR 0 85301811Sngie#define PCI_MSIX_ENTRY_UPPER_ADDR 4 86301811Sngie#define PCI_MSIX_ENTRY_DATA 8 87301811Sngie 88250079Scarlenum ntb_device_type { 89250079Scarl NTB_XEON, 90300373Smav NTB_ATOM 91250079Scarl}; 92250079Scarl 93300373Smav/* ntb_conn_type are hardware numbers, cannot change. */ 94300373Smavenum ntb_conn_type { 95300373Smav NTB_CONN_TRANSPARENT = 0, 96300373Smav NTB_CONN_B2B = 1, 97300373Smav NTB_CONN_RP = 2, 98300373Smav}; 99300373Smav 100300373Smavenum ntb_b2b_direction { 101300373Smav NTB_DEV_USD = 0, 102300373Smav NTB_DEV_DSD = 1, 103300373Smav}; 104300373Smav 105300373Smavenum ntb_bar { 106300373Smav NTB_CONFIG_BAR = 0, 107300373Smav NTB_B2B_BAR_1, 108300373Smav NTB_B2B_BAR_2, 109300373Smav NTB_B2B_BAR_3, 110300373Smav NTB_MAX_BARS 111300373Smav}; 112300373Smav 113301811Sngieenum { 114301811Sngie NTB_MSIX_GUARD = 0, 115301811Sngie NTB_MSIX_DATA0, 116301811Sngie NTB_MSIX_DATA1, 117301811Sngie NTB_MSIX_DATA2, 118301811Sngie NTB_MSIX_OFS0, 119301811Sngie NTB_MSIX_OFS1, 120301811Sngie NTB_MSIX_OFS2, 121301811Sngie NTB_MSIX_DONE, 122301811Sngie NTB_MAX_MSIX_SPAD 123301811Sngie}; 124301811Sngie 125255274Scarl/* Device features and workarounds */ 126255274Scarl#define HAS_FEATURE(feature) \ 127255274Scarl ((ntb->features & (feature)) != 0) 128255274Scarl 129250079Scarlstruct ntb_hw_info { 130250079Scarl uint32_t device_id; 131255274Scarl const char *desc; 132250079Scarl enum ntb_device_type type; 133300373Smav uint32_t features; 134250079Scarl}; 135250079Scarl 136250079Scarlstruct ntb_pci_bar_info { 137250079Scarl bus_space_tag_t pci_bus_tag; 138250079Scarl bus_space_handle_t pci_bus_handle; 139250079Scarl int pci_resource_id; 140250079Scarl struct resource *pci_resource; 141250079Scarl vm_paddr_t pbase; 142300373Smav caddr_t vbase; 143300373Smav vm_size_t size; 144300373Smav vm_memattr_t map_mode; 145300373Smav 146300373Smav /* Configuration register offsets */ 147300373Smav uint32_t psz_off; 148300373Smav uint32_t ssz_off; 149300373Smav uint32_t pbarxlat_off; 150250079Scarl}; 151250079Scarl 152250079Scarlstruct ntb_int_info { 153250079Scarl struct resource *res; 154250079Scarl int rid; 155250079Scarl void *tag; 156250079Scarl}; 157250079Scarl 158300373Smavstruct ntb_vec { 159250079Scarl struct ntb_softc *ntb; 160300373Smav uint32_t num; 161301811Sngie unsigned masked; 162250079Scarl}; 163250079Scarl 164300373Smavstruct ntb_reg { 165300373Smav uint32_t ntb_ctl; 166300373Smav uint32_t lnk_sta; 167300373Smav uint8_t db_size; 168300373Smav unsigned mw_bar[NTB_MAX_BARS]; 169300373Smav}; 170300373Smav 171300373Smavstruct ntb_alt_reg { 172300373Smav uint32_t db_bell; 173300373Smav uint32_t db_mask; 174300373Smav uint32_t spad; 175300373Smav}; 176300373Smav 177300373Smavstruct ntb_xlat_reg { 178300373Smav uint32_t bar0_base; 179300373Smav uint32_t bar2_base; 180300373Smav uint32_t bar4_base; 181300373Smav uint32_t bar5_base; 182300373Smav 183300373Smav uint32_t bar2_xlat; 184300373Smav uint32_t bar4_xlat; 185300373Smav uint32_t bar5_xlat; 186300373Smav 187300373Smav uint32_t bar2_limit; 188300373Smav uint32_t bar4_limit; 189300373Smav uint32_t bar5_limit; 190300373Smav}; 191300373Smav 192300373Smavstruct ntb_b2b_addr { 193300373Smav uint64_t bar0_addr; 194300373Smav uint64_t bar2_addr64; 195300373Smav uint64_t bar4_addr64; 196300373Smav uint64_t bar4_addr32; 197300373Smav uint64_t bar5_addr32; 198300373Smav}; 199300373Smav 200301811Sngiestruct ntb_msix_data { 201301811Sngie uint32_t nmd_ofs; 202301811Sngie uint32_t nmd_data; 203301811Sngie}; 204301811Sngie 205250079Scarlstruct ntb_softc { 206250079Scarl device_t device; 207250079Scarl enum ntb_device_type type; 208300373Smav uint32_t features; 209250079Scarl 210250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 211250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 212250079Scarl uint32_t allocated_interrupts; 213250079Scarl 214301811Sngie struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215301811Sngie struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216301811Sngie bool peer_msix_good; 217301811Sngie bool peer_msix_done; 218301811Sngie struct ntb_pci_bar_info *peer_lapic_bar; 219301811Sngie struct callout peer_msix_work; 220301811Sngie 221250079Scarl struct callout heartbeat_timer; 222250079Scarl struct callout lr_timer; 223250079Scarl 224300373Smav void *ntb_ctx; 225300373Smav const struct ntb_ctx_ops *ctx_ops; 226300373Smav struct ntb_vec *msix_vec; 227300373Smav#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 228300373Smav#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 229300373Smav#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 230300373Smav struct mtx ctx_lock; 231250079Scarl 232300373Smav uint32_t ppd; 233300373Smav enum ntb_conn_type conn_type; 234300373Smav enum ntb_b2b_direction dev_type; 235300373Smav 236300373Smav /* Offset of peer bar0 in B2B BAR */ 237300373Smav uint64_t b2b_off; 238300373Smav /* Memory window used to access peer bar0 */ 239300373Smav#define B2B_MW_DISABLED UINT8_MAX 240300373Smav uint8_t b2b_mw_idx; 241301904Smav uint32_t msix_xlat; 242301811Sngie uint8_t msix_mw_idx; 243300373Smav 244300373Smav uint8_t mw_count; 245300373Smav uint8_t spad_count; 246300373Smav uint8_t db_count; 247300373Smav uint8_t db_vec_count; 248300373Smav uint8_t db_vec_shift; 249300373Smav 250300373Smav /* Protects local db_mask. */ 251300373Smav#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 252300373Smav#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 253300373Smav#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 254300373Smav struct mtx db_mask_lock; 255300373Smav 256300373Smav volatile uint32_t ntb_ctl; 257300373Smav volatile uint32_t lnk_sta; 258300373Smav 259300373Smav uint64_t db_valid_mask; 260300373Smav uint64_t db_link_mask; 261300373Smav uint64_t db_mask; 262300373Smav 263300373Smav int last_ts; /* ticks @ last irq */ 264300373Smav 265300373Smav const struct ntb_reg *reg; 266300373Smav const struct ntb_alt_reg *self_reg; 267300373Smav const struct ntb_alt_reg *peer_reg; 268300373Smav const struct ntb_xlat_reg *xlat_reg; 269250079Scarl}; 270250079Scarl 271300373Smav#ifdef __i386__ 272300373Smavstatic __inline uint64_t 273300373Smavbus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 274300373Smav bus_size_t offset) 275300373Smav{ 276300373Smav 277300373Smav return (bus_space_read_4(tag, handle, offset) | 278300373Smav ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 279300373Smav} 280300373Smav 281300373Smavstatic __inline void 282300373Smavbus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 283300373Smav bus_size_t offset, uint64_t val) 284300373Smav{ 285300373Smav 286300373Smav bus_space_write_4(tag, handle, offset, val); 287300373Smav bus_space_write_4(tag, handle, offset + 4, val >> 32); 288300373Smav} 289300373Smav#endif 290300373Smav 291255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 292255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 293255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 294255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 295255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 296255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 297255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 298250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 299255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 300300373Smav#define ntb_mw_read(SIZE, offset) \ 301300373Smav ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 302255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 303300373Smav ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 304300373Smav offset, val) 305250079Scarl 306250079Scarlstatic int ntb_probe(device_t device); 307250079Scarlstatic int ntb_attach(device_t device); 308250079Scarlstatic int ntb_detach(device_t device); 309300373Smavstatic unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 310300373Smavstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 311300373Smavstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 312300373Smavstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 313300373Smav uint32_t *base, uint32_t *xlat, uint32_t *lmt); 314255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 315300373Smavstatic int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 316300373Smav vm_memattr_t); 317300373Smavstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 318300373Smav const char *); 319255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 320255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 321255272Scarl struct ntb_pci_bar_info *bar); 322250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 323300373Smavstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 324300373Smavstatic int ntb_init_isr(struct ntb_softc *ntb); 325300373Smavstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 326300373Smavstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 327250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 328300373Smavstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 329300373Smavstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 330300373Smavstatic void ndev_vec_isr(void *arg); 331300373Smavstatic void ndev_irq_isr(void *arg); 332300373Smavstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 333300373Smavstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 334300373Smavstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 335300373Smavstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 336300373Smavstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 337301811Sngiestatic void ntb_get_msix_info(struct ntb_softc *ntb); 338301811Sngiestatic void ntb_exchange_msix(void *); 339250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 340300373Smavstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 341300373Smavstatic int ntb_detect_xeon(struct ntb_softc *ntb); 342300373Smavstatic int ntb_detect_atom(struct ntb_softc *ntb); 343300373Smavstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 344300373Smavstatic int ntb_atom_init_dev(struct ntb_softc *ntb); 345300373Smavstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 346300373Smavstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 347300373Smavstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 348300373Smav enum ntb_bar regbar); 349300373Smavstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 350300373Smav uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 351300373Smavstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 352300373Smav enum ntb_bar idx); 353300373Smavstatic int xeon_setup_b2b_mw(struct ntb_softc *, 354300373Smav const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 355301811Sngiestatic int xeon_setup_msix_bar(struct ntb_softc *); 356300373Smavstatic inline bool link_is_up(struct ntb_softc *ntb); 357301811Sngiestatic inline bool _xeon_link_is_up(struct ntb_softc *ntb); 358300373Smavstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 359300373Smavstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 360300373Smavstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 361300373Smavstatic void atom_link_hb(void *arg); 362300373Smavstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 363300373Smavstatic void recover_atom_link(void *arg); 364300373Smavstatic bool ntb_poll_link(struct ntb_softc *ntb); 365255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 366300373Smavstatic void ntb_sysctl_init(struct ntb_softc *); 367300373Smavstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 368301811Sngiestatic int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 369301811Sngiestatic int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 370300373Smavstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 371300373Smavstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 372250079Scarl 373300373Smavstatic unsigned g_ntb_hw_debug_level; 374300516SmavTUNABLE_INT("hw.ntb.debug_level", &g_ntb_hw_debug_level); 375300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 376300373Smav &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 377300373Smav#define ntb_printf(lvl, ...) do { \ 378300373Smav if ((lvl) <= g_ntb_hw_debug_level) { \ 379300373Smav device_printf(ntb->device, __VA_ARGS__); \ 380300373Smav } \ 381300373Smav} while (0) 382300373Smav 383300373Smav#define _NTB_PAT_UC 0 384300373Smav#define _NTB_PAT_WC 1 385300373Smav#define _NTB_PAT_WT 4 386300373Smav#define _NTB_PAT_WP 5 387300373Smav#define _NTB_PAT_WB 6 388300373Smav#define _NTB_PAT_UCM 7 389300373Smavstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 390300516SmavTUNABLE_INT("hw.ntb.default_mw_pat", &g_ntb_mw_pat); 391300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 392300373Smav &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 393300373Smav "UC: " __XSTRING(_NTB_PAT_UC) ", " 394300373Smav "WC: " __XSTRING(_NTB_PAT_WC) ", " 395300373Smav "WT: " __XSTRING(_NTB_PAT_WT) ", " 396300373Smav "WP: " __XSTRING(_NTB_PAT_WP) ", " 397300373Smav "WB: " __XSTRING(_NTB_PAT_WB) ", " 398300373Smav "UC-: " __XSTRING(_NTB_PAT_UCM)); 399300373Smav 400300373Smavstatic inline vm_memattr_t 401300373Smavntb_pat_flags(void) 402300373Smav{ 403300373Smav 404300373Smav switch (g_ntb_mw_pat) { 405300373Smav case _NTB_PAT_WC: 406300373Smav return (VM_MEMATTR_WRITE_COMBINING); 407300373Smav case _NTB_PAT_WT: 408300373Smav return (VM_MEMATTR_WRITE_THROUGH); 409300373Smav case _NTB_PAT_WP: 410300373Smav return (VM_MEMATTR_WRITE_PROTECTED); 411300373Smav case _NTB_PAT_WB: 412300373Smav return (VM_MEMATTR_WRITE_BACK); 413300373Smav case _NTB_PAT_UCM: 414300373Smav return (VM_MEMATTR_WEAK_UNCACHEABLE); 415300373Smav case _NTB_PAT_UC: 416300373Smav /* FALLTHROUGH */ 417300373Smav default: 418300373Smav return (VM_MEMATTR_UNCACHEABLE); 419300373Smav } 420300373Smav} 421300373Smav 422300373Smav/* 423300373Smav * Well, this obviously doesn't belong here, but it doesn't seem to exist 424300373Smav * anywhere better yet. 425300373Smav */ 426300373Smavstatic inline const char * 427300373Smavntb_vm_memattr_to_str(vm_memattr_t pat) 428300373Smav{ 429300373Smav 430300373Smav switch (pat) { 431300373Smav case VM_MEMATTR_WRITE_COMBINING: 432300373Smav return ("WRITE_COMBINING"); 433300373Smav case VM_MEMATTR_WRITE_THROUGH: 434300373Smav return ("WRITE_THROUGH"); 435300373Smav case VM_MEMATTR_WRITE_PROTECTED: 436300373Smav return ("WRITE_PROTECTED"); 437300373Smav case VM_MEMATTR_WRITE_BACK: 438300373Smav return ("WRITE_BACK"); 439300373Smav case VM_MEMATTR_WEAK_UNCACHEABLE: 440300373Smav return ("UNCACHED"); 441300373Smav case VM_MEMATTR_UNCACHEABLE: 442300373Smav return ("UNCACHEABLE"); 443300373Smav default: 444300373Smav return ("UNKNOWN"); 445300373Smav } 446300373Smav} 447300373Smav 448301811Sngiestatic int g_ntb_msix_idx = 0; 449301811SngieSYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 450301811Sngie 0, "Use this memory window to access the peer MSIX message complex on " 451301811Sngie "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 452301811Sngie "Like b2b_mw_idx, negative values index from the last available memory " 453301811Sngie "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 454301811Sngie 455300373Smavstatic int g_ntb_mw_idx = -1; 456300516SmavTUNABLE_INT("hw.ntb.b2b_mw_idx", &g_ntb_mw_idx); 457300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 458300373Smav 0, "Use this memory window to access the peer NTB registers. A " 459300373Smav "non-negative value starts from the first MW index; a negative value " 460300373Smav "starts from the last MW index. The default is -1, i.e., the last " 461300373Smav "available memory window. Both sides of the NTB MUST set the same " 462300373Smav "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 463300373Smav 464250079Scarlstatic struct ntb_hw_info pci_ids[] = { 465300373Smav /* XXX: PS/SS IDs left out until they are supported. */ 466300373Smav { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 467300373Smav NTB_ATOM, 0 }, 468300373Smav 469300373Smav { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 470300373Smav NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 471300373Smav { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 472300373Smav NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 473300373Smav { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 474300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 475300373Smav NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 476300373Smav { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 477300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 478300373Smav NTB_SB01BASE_LOCKUP }, 479300373Smav { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 480300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 481300373Smav NTB_SB01BASE_LOCKUP }, 482300373Smav 483300373Smav { 0x00000000, NULL, NTB_ATOM, 0 } 484250079Scarl}; 485250079Scarl 486300373Smavstatic const struct ntb_reg atom_reg = { 487300373Smav .ntb_ctl = ATOM_NTBCNTL_OFFSET, 488300373Smav .lnk_sta = ATOM_LINK_STATUS_OFFSET, 489300373Smav .db_size = sizeof(uint64_t), 490300373Smav .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 491300373Smav}; 492300373Smav 493300373Smavstatic const struct ntb_alt_reg atom_pri_reg = { 494300373Smav .db_bell = ATOM_PDOORBELL_OFFSET, 495300373Smav .db_mask = ATOM_PDBMSK_OFFSET, 496300373Smav .spad = ATOM_SPAD_OFFSET, 497300373Smav}; 498300373Smav 499300373Smavstatic const struct ntb_alt_reg atom_b2b_reg = { 500300373Smav .db_bell = ATOM_B2B_DOORBELL_OFFSET, 501300373Smav .spad = ATOM_B2B_SPAD_OFFSET, 502300373Smav}; 503300373Smav 504300373Smavstatic const struct ntb_xlat_reg atom_sec_xlat = { 505300373Smav#if 0 506300373Smav /* "FIXME" says the Linux driver. */ 507300373Smav .bar0_base = ATOM_SBAR0BASE_OFFSET, 508300373Smav .bar2_base = ATOM_SBAR2BASE_OFFSET, 509300373Smav .bar4_base = ATOM_SBAR4BASE_OFFSET, 510300373Smav 511300373Smav .bar2_limit = ATOM_SBAR2LMT_OFFSET, 512300373Smav .bar4_limit = ATOM_SBAR4LMT_OFFSET, 513300373Smav#endif 514300373Smav 515300373Smav .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 516300373Smav .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 517300373Smav}; 518300373Smav 519300373Smavstatic const struct ntb_reg xeon_reg = { 520300373Smav .ntb_ctl = XEON_NTBCNTL_OFFSET, 521300373Smav .lnk_sta = XEON_LINK_STATUS_OFFSET, 522300373Smav .db_size = sizeof(uint16_t), 523300373Smav .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 524300373Smav}; 525300373Smav 526300373Smavstatic const struct ntb_alt_reg xeon_pri_reg = { 527300373Smav .db_bell = XEON_PDOORBELL_OFFSET, 528300373Smav .db_mask = XEON_PDBMSK_OFFSET, 529300373Smav .spad = XEON_SPAD_OFFSET, 530300373Smav}; 531300373Smav 532300373Smavstatic const struct ntb_alt_reg xeon_b2b_reg = { 533300373Smav .db_bell = XEON_B2B_DOORBELL_OFFSET, 534300373Smav .spad = XEON_B2B_SPAD_OFFSET, 535300373Smav}; 536300373Smav 537300373Smavstatic const struct ntb_xlat_reg xeon_sec_xlat = { 538300373Smav .bar0_base = XEON_SBAR0BASE_OFFSET, 539300373Smav .bar2_base = XEON_SBAR2BASE_OFFSET, 540300373Smav .bar4_base = XEON_SBAR4BASE_OFFSET, 541300373Smav .bar5_base = XEON_SBAR5BASE_OFFSET, 542300373Smav 543300373Smav .bar2_limit = XEON_SBAR2LMT_OFFSET, 544300373Smav .bar4_limit = XEON_SBAR4LMT_OFFSET, 545300373Smav .bar5_limit = XEON_SBAR5LMT_OFFSET, 546300373Smav 547300373Smav .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 548300373Smav .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 549300373Smav .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 550300373Smav}; 551300373Smav 552300373Smavstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 553300373Smav .bar0_addr = XEON_B2B_BAR0_ADDR, 554300373Smav .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 555300373Smav .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 556300373Smav .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 557300373Smav .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 558300373Smav}; 559300373Smav 560300373Smavstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 561300373Smav .bar0_addr = XEON_B2B_BAR0_ADDR, 562300373Smav .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 563300373Smav .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 564300373Smav .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 565300373Smav .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 566300373Smav}; 567300373Smav 568300373SmavSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 569300373Smav "B2B MW segment overrides -- MUST be the same on both sides"); 570300373Smav 571300516SmavTUNABLE_QUAD("hw.ntb.usd_bar2_addr64", &xeon_b2b_usd_addr.bar2_addr64); 572300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 573300373Smav &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 574300373Smav "hardware, use this 64-bit address on the bus between the NTB devices for " 575300373Smav "the window at BAR2, on the upstream side of the link. MUST be the same " 576300373Smav "address on both sides."); 577300516SmavTUNABLE_QUAD("hw.ntb.usd_bar4_addr64", &xeon_b2b_usd_addr.bar4_addr64); 578300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 579300373Smav &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 580300516SmavTUNABLE_QUAD("hw.ntb.usd_bar4_addr32", &xeon_b2b_usd_addr.bar4_addr32); 581300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 582300373Smav &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 583300373Smav "(split-BAR mode)."); 584300516SmavTUNABLE_QUAD("hw.ntb.usd_bar5_addr32", &xeon_b2b_usd_addr.bar5_addr32); 585300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 586300373Smav &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 587300373Smav "(split-BAR mode)."); 588300373Smav 589300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar2_addr64", &xeon_b2b_dsd_addr.bar2_addr64); 590300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 591300373Smav &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 592300373Smav "hardware, use this 64-bit address on the bus between the NTB devices for " 593300373Smav "the window at BAR2, on the downstream side of the link. MUST be the same" 594300373Smav " address on both sides."); 595300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar4_addr64", &xeon_b2b_dsd_addr.bar4_addr64); 596300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 597300373Smav &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 598300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar4_addr32", &xeon_b2b_dsd_addr.bar4_addr32); 599300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 600300373Smav &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 601300373Smav "(split-BAR mode)."); 602300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar5_addr32", &xeon_b2b_dsd_addr.bar5_addr32); 603300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 604300373Smav &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 605300373Smav "(split-BAR mode)."); 606300373Smav 607250079Scarl/* 608250079Scarl * OS <-> Driver interface structures 609250079Scarl */ 610250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 611250079Scarl 612250079Scarlstatic device_method_t ntb_pci_methods[] = { 613250079Scarl /* Device interface */ 614250079Scarl DEVMETHOD(device_probe, ntb_probe), 615250079Scarl DEVMETHOD(device_attach, ntb_attach), 616250079Scarl DEVMETHOD(device_detach, ntb_detach), 617250079Scarl DEVMETHOD_END 618250079Scarl}; 619250079Scarl 620250079Scarlstatic driver_t ntb_pci_driver = { 621250079Scarl "ntb_hw", 622250079Scarl ntb_pci_methods, 623250079Scarl sizeof(struct ntb_softc), 624250079Scarl}; 625250079Scarl 626250079Scarlstatic devclass_t ntb_devclass; 627250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 628250079ScarlMODULE_VERSION(ntb_hw, 1); 629250079Scarl 630300373SmavSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 631300373Smav 632250079Scarl/* 633250079Scarl * OS <-> Driver linkage functions 634250079Scarl */ 635250079Scarlstatic int 636250079Scarlntb_probe(device_t device) 637250079Scarl{ 638300373Smav struct ntb_hw_info *p; 639250079Scarl 640300373Smav p = ntb_get_device_info(pci_get_devid(device)); 641300373Smav if (p == NULL) 642250079Scarl return (ENXIO); 643300373Smav 644300373Smav device_set_desc(device, p->desc); 645300373Smav return (0); 646250079Scarl} 647250079Scarl 648250079Scarlstatic int 649250079Scarlntb_attach(device_t device) 650250079Scarl{ 651300373Smav struct ntb_softc *ntb; 652300373Smav struct ntb_hw_info *p; 653250079Scarl int error; 654250079Scarl 655300373Smav ntb = DEVICE2SOFTC(device); 656300373Smav p = ntb_get_device_info(pci_get_devid(device)); 657300373Smav 658250079Scarl ntb->device = device; 659250079Scarl ntb->type = p->type; 660255274Scarl ntb->features = p->features; 661300373Smav ntb->b2b_mw_idx = B2B_MW_DISABLED; 662301811Sngie ntb->msix_mw_idx = B2B_MW_DISABLED; 663250079Scarl 664300373Smav /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 665250079Scarl callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE); 666250079Scarl callout_init(&ntb->lr_timer, CALLOUT_MPSAFE); 667301811Sngie callout_init(&ntb->peer_msix_work, 1); 668300373Smav mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 669300373Smav mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 670250079Scarl 671300373Smav if (ntb->type == NTB_ATOM) 672300373Smav error = ntb_detect_atom(ntb); 673300373Smav else 674300373Smav error = ntb_detect_xeon(ntb); 675300373Smav if (error != 0) 676300373Smav goto out; 677250079Scarl 678300373Smav ntb_detect_max_mw(ntb); 679300373Smav 680250079Scarl pci_enable_busmaster(ntb->device); 681250079Scarl 682300373Smav error = ntb_map_pci_bars(ntb); 683300373Smav if (error != 0) 684300373Smav goto out; 685300373Smav if (ntb->type == NTB_ATOM) 686300373Smav error = ntb_atom_init_dev(ntb); 687300373Smav else 688300373Smav error = ntb_xeon_init_dev(ntb); 689300373Smav if (error != 0) 690300373Smav goto out; 691300373Smav 692301811Sngie ntb_spad_clear(ntb); 693301811Sngie 694300373Smav ntb_poll_link(ntb); 695300373Smav 696300373Smav ntb_sysctl_init(ntb); 697300373Smav 698300373Smavout: 699300373Smav if (error != 0) 700300373Smav ntb_detach(device); 701250079Scarl return (error); 702250079Scarl} 703250079Scarl 704250079Scarlstatic int 705250079Scarlntb_detach(device_t device) 706250079Scarl{ 707300373Smav struct ntb_softc *ntb; 708250079Scarl 709300373Smav ntb = DEVICE2SOFTC(device); 710300373Smav 711301811Sngie if (ntb->self_reg != NULL) { 712301811Sngie DB_MASK_LOCK(ntb); 713301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 714301811Sngie DB_MASK_UNLOCK(ntb); 715301811Sngie } 716250079Scarl callout_drain(&ntb->heartbeat_timer); 717250079Scarl callout_drain(&ntb->lr_timer); 718301811Sngie callout_drain(&ntb->peer_msix_work); 719300373Smav pci_disable_busmaster(ntb->device); 720300373Smav if (ntb->type == NTB_XEON) 721300373Smav ntb_teardown_xeon(ntb); 722250079Scarl ntb_teardown_interrupts(ntb); 723300373Smav 724300373Smav mtx_destroy(&ntb->db_mask_lock); 725300373Smav mtx_destroy(&ntb->ctx_lock); 726300373Smav 727250079Scarl ntb_unmap_pci_bar(ntb); 728250079Scarl 729250079Scarl return (0); 730250079Scarl} 731250079Scarl 732300373Smav/* 733300373Smav * Driver internal routines 734300373Smav */ 735300373Smavstatic inline enum ntb_bar 736300373Smavntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 737300373Smav{ 738300373Smav 739300373Smav KASSERT(mw < ntb->mw_count, 740300373Smav ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 741300373Smav KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 742300373Smav 743300373Smav return (ntb->reg->mw_bar[mw]); 744300373Smav} 745300373Smav 746300373Smavstatic inline bool 747300373Smavbar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 748300373Smav{ 749300373Smav /* XXX This assertion could be stronger. */ 750300373Smav KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 751300373Smav return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 752300373Smav} 753300373Smav 754300373Smavstatic inline void 755300373Smavbar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 756300373Smav uint32_t *xlat, uint32_t *lmt) 757300373Smav{ 758300373Smav uint32_t basev, lmtv, xlatv; 759300373Smav 760300373Smav switch (bar) { 761300373Smav case NTB_B2B_BAR_1: 762300373Smav basev = ntb->xlat_reg->bar2_base; 763300373Smav lmtv = ntb->xlat_reg->bar2_limit; 764300373Smav xlatv = ntb->xlat_reg->bar2_xlat; 765300373Smav break; 766300373Smav case NTB_B2B_BAR_2: 767300373Smav basev = ntb->xlat_reg->bar4_base; 768300373Smav lmtv = ntb->xlat_reg->bar4_limit; 769300373Smav xlatv = ntb->xlat_reg->bar4_xlat; 770300373Smav break; 771300373Smav case NTB_B2B_BAR_3: 772300373Smav basev = ntb->xlat_reg->bar5_base; 773300373Smav lmtv = ntb->xlat_reg->bar5_limit; 774300373Smav xlatv = ntb->xlat_reg->bar5_xlat; 775300373Smav break; 776300373Smav default: 777300373Smav KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 778300373Smav ("bad bar")); 779300373Smav basev = lmtv = xlatv = 0; 780300373Smav break; 781300373Smav } 782300373Smav 783300373Smav if (base != NULL) 784300373Smav *base = basev; 785300373Smav if (xlat != NULL) 786300373Smav *xlat = xlatv; 787300373Smav if (lmt != NULL) 788300373Smav *lmt = lmtv; 789300373Smav} 790300373Smav 791250079Scarlstatic int 792255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 793250079Scarl{ 794255272Scarl int rc; 795250079Scarl 796250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 797300373Smav rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 798255272Scarl if (rc != 0) 799300373Smav goto out; 800255272Scarl 801300373Smav ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 802300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 803255272Scarl if (rc != 0) 804300373Smav goto out; 805300373Smav ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 806300373Smav ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 807300373Smav ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 808255272Scarl 809300373Smav ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 810300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 811255272Scarl if (rc != 0) 812300373Smav goto out; 813300373Smav ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 814300373Smav ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 815300373Smav ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 816255274Scarl 817300373Smav if (!HAS_FEATURE(NTB_SPLIT_BAR)) 818300373Smav goto out; 819250079Scarl 820300373Smav ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 821300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 822300373Smav ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 823300373Smav ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 824300373Smav ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 825255272Scarl 826300373Smavout: 827300373Smav if (rc != 0) 828255272Scarl device_printf(ntb->device, 829255272Scarl "unable to allocate pci resource\n"); 830255272Scarl return (rc); 831255272Scarl} 832255272Scarl 833300373Smavstatic void 834300373Smavprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 835300373Smav const char *kind) 836300373Smav{ 837300373Smav 838300373Smav device_printf(ntb->device, 839300373Smav "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 840300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 841300373Smav (char *)bar->vbase + bar->size - 1, 842300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 843300373Smav (uintmax_t)bar->size, kind); 844300373Smav} 845300373Smav 846255272Scarlstatic int 847255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 848255272Scarl{ 849255272Scarl 850255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 851300373Smav &bar->pci_resource_id, RF_ACTIVE); 852255272Scarl if (bar->pci_resource == NULL) 853255272Scarl return (ENXIO); 854300373Smav 855300373Smav save_bar_parameters(bar); 856300373Smav bar->map_mode = VM_MEMATTR_UNCACHEABLE; 857300373Smav print_map_success(ntb, bar, "mmr"); 858300373Smav return (0); 859255272Scarl} 860255272Scarl 861255272Scarlstatic int 862255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 863255272Scarl{ 864255272Scarl int rc; 865300373Smav vm_memattr_t mapmode; 866255276Scarl uint8_t bar_size_bits = 0; 867255272Scarl 868300373Smav bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 869300373Smav &bar->pci_resource_id, RF_ACTIVE); 870250079Scarl 871255272Scarl if (bar->pci_resource == NULL) 872255272Scarl return (ENXIO); 873255276Scarl 874300373Smav save_bar_parameters(bar); 875300373Smav /* 876300373Smav * Ivytown NTB BAR sizes are misreported by the hardware due to a 877300373Smav * hardware issue. To work around this, query the size it should be 878300373Smav * configured to by the device and modify the resource to correspond to 879300373Smav * this new size. The BIOS on systems with this problem is required to 880300373Smav * provide enough address space to allow the driver to make this change 881300373Smav * safely. 882300373Smav * 883300373Smav * Ideally I could have just specified the size when I allocated the 884300373Smav * resource like: 885300373Smav * bus_alloc_resource(ntb->device, 886300373Smav * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 887300373Smav * 1ul << bar_size_bits, RF_ACTIVE); 888300373Smav * but the PCI driver does not honor the size in this call, so we have 889300373Smav * to modify it after the fact. 890300373Smav */ 891300373Smav if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 892300373Smav if (bar->pci_resource_id == PCIR_BAR(2)) 893300373Smav bar_size_bits = pci_read_config(ntb->device, 894300373Smav XEON_PBAR23SZ_OFFSET, 1); 895300373Smav else 896300373Smav bar_size_bits = pci_read_config(ntb->device, 897300373Smav XEON_PBAR45SZ_OFFSET, 1); 898300373Smav 899300373Smav rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 900300373Smav bar->pci_resource, bar->pbase, 901300373Smav bar->pbase + (1ul << bar_size_bits) - 1); 902255272Scarl if (rc != 0) { 903300373Smav device_printf(ntb->device, 904300373Smav "unable to resize bar\n"); 905255272Scarl return (rc); 906250079Scarl } 907300373Smav 908300373Smav save_bar_parameters(bar); 909250079Scarl } 910300373Smav 911300373Smav bar->map_mode = VM_MEMATTR_UNCACHEABLE; 912300373Smav print_map_success(ntb, bar, "mw"); 913300373Smav 914300373Smav /* 915300373Smav * Optionally, mark MW BARs as anything other than UC to improve 916300373Smav * performance. 917300373Smav */ 918300373Smav mapmode = ntb_pat_flags(); 919300373Smav if (mapmode == bar->map_mode) 920300373Smav return (0); 921300373Smav 922300373Smav rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 923300373Smav if (rc == 0) { 924300373Smav bar->map_mode = mapmode; 925300373Smav device_printf(ntb->device, 926300373Smav "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 927300373Smav "%s.\n", 928300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 929300373Smav (char *)bar->vbase + bar->size - 1, 930300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 931300373Smav ntb_vm_memattr_to_str(mapmode)); 932300373Smav } else 933300373Smav device_printf(ntb->device, 934300373Smav "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 935300373Smav "%s: %d\n", 936300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 937300373Smav (char *)bar->vbase + bar->size - 1, 938300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 939300373Smav ntb_vm_memattr_to_str(mapmode), rc); 940300373Smav /* Proceed anyway */ 941250079Scarl return (0); 942250079Scarl} 943250079Scarl 944250079Scarlstatic void 945250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 946250079Scarl{ 947250079Scarl struct ntb_pci_bar_info *current_bar; 948250079Scarl int i; 949250079Scarl 950300373Smav for (i = 0; i < NTB_MAX_BARS; i++) { 951250079Scarl current_bar = &ntb->bar_info[i]; 952250079Scarl if (current_bar->pci_resource != NULL) 953250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 954250079Scarl current_bar->pci_resource_id, 955250079Scarl current_bar->pci_resource); 956250079Scarl } 957250079Scarl} 958250079Scarl 959250079Scarlstatic int 960300373Smavntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 961250079Scarl{ 962300373Smav uint32_t i; 963300373Smav int rc; 964250079Scarl 965300373Smav for (i = 0; i < num_vectors; i++) { 966300373Smav ntb->int_info[i].rid = i + 1; 967300373Smav ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 968300373Smav SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 969300373Smav if (ntb->int_info[i].res == NULL) { 970300373Smav device_printf(ntb->device, 971300373Smav "bus_alloc_resource failed\n"); 972300373Smav return (ENOMEM); 973300373Smav } 974300373Smav ntb->int_info[i].tag = NULL; 975300373Smav ntb->allocated_interrupts++; 976300373Smav rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 977300373Smav INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 978300373Smav &ntb->msix_vec[i], &ntb->int_info[i].tag); 979300373Smav if (rc != 0) { 980300373Smav device_printf(ntb->device, "bus_setup_intr failed\n"); 981300373Smav return (ENXIO); 982300373Smav } 983300373Smav } 984300373Smav return (0); 985300373Smav} 986300373Smav 987300373Smav/* 988300373Smav * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 989300373Smav * cannot be allocated for each MSI-X message. JHB seems to think remapping 990300373Smav * should be okay. This tunable should enable us to test that hypothesis 991300373Smav * when someone gets their hands on some Xeon hardware. 992300373Smav */ 993300373Smavstatic int ntb_force_remap_mode; 994300516SmavTUNABLE_INT("hw.ntb.force_remap_mode", &ntb_force_remap_mode); 995300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 996300373Smav &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 997300373Smav " to a smaller number of ithreads, even if the desired number are " 998300373Smav "available"); 999300373Smav 1000300373Smav/* 1001300373Smav * In case it is NOT ok, give consumers an abort button. 1002300373Smav */ 1003300373Smavstatic int ntb_prefer_intx; 1004300516SmavTUNABLE_INT("hw.ntb.prefer_intx_to_remap", &ntb_prefer_intx); 1005300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 1006300373Smav &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1007300373Smav "than remapping MSI-X messages over available slots (match Linux driver " 1008300373Smav "behavior)"); 1009300373Smav 1010300373Smav/* 1011300373Smav * Remap the desired number of MSI-X messages to available ithreads in a simple 1012300373Smav * round-robin fashion. 1013300373Smav */ 1014300373Smavstatic int 1015300373Smavntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1016300373Smav{ 1017300373Smav u_int *vectors; 1018300373Smav uint32_t i; 1019300373Smav int rc; 1020300373Smav 1021300373Smav if (ntb_prefer_intx != 0) 1022300373Smav return (ENXIO); 1023300373Smav 1024300373Smav vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1025300373Smav 1026300373Smav for (i = 0; i < desired; i++) 1027300373Smav vectors[i] = (i % avail) + 1; 1028300373Smav 1029300373Smav rc = pci_remap_msix(dev, desired, vectors); 1030300373Smav free(vectors, M_NTB); 1031300373Smav return (rc); 1032300373Smav} 1033300373Smav 1034300373Smavstatic int 1035300373Smavntb_init_isr(struct ntb_softc *ntb) 1036300373Smav{ 1037300373Smav uint32_t desired_vectors, num_vectors; 1038300373Smav int rc; 1039300373Smav 1040250079Scarl ntb->allocated_interrupts = 0; 1041300373Smav ntb->last_ts = ticks; 1042300373Smav 1043250079Scarl /* 1044301811Sngie * Mask all doorbell interrupts. (Except link events!) 1045250079Scarl */ 1046301811Sngie DB_MASK_LOCK(ntb); 1047301811Sngie ntb->db_mask = ntb->db_valid_mask; 1048301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1049301811Sngie DB_MASK_UNLOCK(ntb); 1050250079Scarl 1051300373Smav num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1052300373Smav ntb->db_count); 1053300373Smav if (desired_vectors >= 1) { 1054300373Smav rc = pci_alloc_msix(ntb->device, &num_vectors); 1055250079Scarl 1056300373Smav if (ntb_force_remap_mode != 0 && rc == 0 && 1057300373Smav num_vectors == desired_vectors) 1058300373Smav num_vectors--; 1059300373Smav 1060300373Smav if (rc == 0 && num_vectors < desired_vectors) { 1061300373Smav rc = ntb_remap_msix(ntb->device, desired_vectors, 1062300373Smav num_vectors); 1063300373Smav if (rc == 0) 1064300373Smav num_vectors = desired_vectors; 1065300373Smav else 1066300373Smav pci_release_msi(ntb->device); 1067250079Scarl } 1068300373Smav if (rc != 0) 1069300373Smav num_vectors = 1; 1070300373Smav } else 1071300373Smav num_vectors = 1; 1072300373Smav 1073300373Smav if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1074301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1075301811Sngie device_printf(ntb->device, 1076301811Sngie "Errata workaround does not support MSI or INTX\n"); 1077301811Sngie return (EINVAL); 1078301811Sngie } 1079301811Sngie 1080300373Smav ntb->db_vec_count = 1; 1081300373Smav ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1082300373Smav rc = ntb_setup_legacy_interrupt(ntb); 1083300373Smav } else { 1084301811Sngie if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1085301811Sngie HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1086301811Sngie device_printf(ntb->device, 1087301811Sngie "Errata workaround expects %d doorbell bits\n", 1088301811Sngie XEON_NONLINK_DB_MSIX_BITS); 1089301811Sngie return (EINVAL); 1090301811Sngie } 1091301811Sngie 1092300373Smav ntb_create_msix_vec(ntb, num_vectors); 1093300373Smav rc = ntb_setup_msix(ntb, num_vectors); 1094301811Sngie if (rc == 0 && HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1095301811Sngie ntb_get_msix_info(ntb); 1096250079Scarl } 1097300373Smav if (rc != 0) { 1098300373Smav device_printf(ntb->device, 1099300373Smav "Error allocating interrupts: %d\n", rc); 1100300373Smav ntb_free_msix_vec(ntb); 1101300373Smav } 1102250079Scarl 1103300373Smav return (rc); 1104300373Smav} 1105250079Scarl 1106300373Smavstatic int 1107300373Smavntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1108300373Smav{ 1109300373Smav int rc; 1110300373Smav 1111300373Smav ntb->int_info[0].rid = 0; 1112300373Smav ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1113300373Smav &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1114300373Smav if (ntb->int_info[0].res == NULL) { 1115300373Smav device_printf(ntb->device, "bus_alloc_resource failed\n"); 1116300373Smav return (ENOMEM); 1117250079Scarl } 1118250079Scarl 1119300373Smav ntb->int_info[0].tag = NULL; 1120300373Smav ntb->allocated_interrupts = 1; 1121300373Smav 1122300373Smav rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1123300373Smav INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1124300373Smav ntb, &ntb->int_info[0].tag); 1125300373Smav if (rc != 0) { 1126300373Smav device_printf(ntb->device, "bus_setup_intr failed\n"); 1127300373Smav return (ENXIO); 1128300373Smav } 1129300373Smav 1130250079Scarl return (0); 1131250079Scarl} 1132250079Scarl 1133250079Scarlstatic void 1134250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 1135250079Scarl{ 1136250079Scarl struct ntb_int_info *current_int; 1137250079Scarl int i; 1138250079Scarl 1139300373Smav for (i = 0; i < ntb->allocated_interrupts; i++) { 1140250079Scarl current_int = &ntb->int_info[i]; 1141250079Scarl if (current_int->tag != NULL) 1142250079Scarl bus_teardown_intr(ntb->device, current_int->res, 1143250079Scarl current_int->tag); 1144250079Scarl 1145250079Scarl if (current_int->res != NULL) 1146250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 1147250079Scarl rman_get_rid(current_int->res), current_int->res); 1148250079Scarl } 1149250079Scarl 1150300373Smav ntb_free_msix_vec(ntb); 1151250079Scarl pci_release_msi(ntb->device); 1152250079Scarl} 1153250079Scarl 1154300373Smav/* 1155300373Smav * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1156300373Smav * out to make code clearer. 1157300373Smav */ 1158300373Smavstatic inline uint64_t 1159300373Smavdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 1160250079Scarl{ 1161250079Scarl 1162300373Smav if (ntb->type == NTB_ATOM) 1163300373Smav return (ntb_reg_read(8, regoff)); 1164250079Scarl 1165300373Smav KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1166300373Smav 1167300373Smav return (ntb_reg_read(2, regoff)); 1168250079Scarl} 1169250079Scarl 1170300373Smavstatic inline void 1171300373Smavdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1172250079Scarl{ 1173250079Scarl 1174300373Smav KASSERT((val & ~ntb->db_valid_mask) == 0, 1175300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1176300373Smav (uintmax_t)(val & ~ntb->db_valid_mask), 1177300373Smav (uintmax_t)ntb->db_valid_mask)); 1178250079Scarl 1179300373Smav if (regoff == ntb->self_reg->db_mask) 1180300373Smav DB_MASK_ASSERT(ntb, MA_OWNED); 1181300373Smav db_iowrite_raw(ntb, regoff, val); 1182250079Scarl} 1183250079Scarl 1184300373Smavstatic inline void 1185300373Smavdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1186250079Scarl{ 1187250079Scarl 1188300373Smav if (ntb->type == NTB_ATOM) { 1189300373Smav ntb_reg_write(8, regoff, val); 1190300373Smav return; 1191300373Smav } 1192250079Scarl 1193300373Smav KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1194300373Smav ntb_reg_write(2, regoff, (uint16_t)val); 1195250079Scarl} 1196250079Scarl 1197300373Smavvoid 1198300373Smavntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1199250079Scarl{ 1200250079Scarl 1201301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1202301811Sngie return; 1203301811Sngie 1204300373Smav DB_MASK_LOCK(ntb); 1205300373Smav ntb->db_mask |= bits; 1206300373Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1207300373Smav DB_MASK_UNLOCK(ntb); 1208300373Smav} 1209250079Scarl 1210300373Smavvoid 1211300373Smavntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1212300373Smav{ 1213250079Scarl 1214300373Smav KASSERT((bits & ~ntb->db_valid_mask) == 0, 1215300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1216300373Smav (uintmax_t)(bits & ~ntb->db_valid_mask), 1217300373Smav (uintmax_t)ntb->db_valid_mask)); 1218250079Scarl 1219301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1220301811Sngie return; 1221301811Sngie 1222300373Smav DB_MASK_LOCK(ntb); 1223300373Smav ntb->db_mask &= ~bits; 1224300373Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1225300373Smav DB_MASK_UNLOCK(ntb); 1226300373Smav} 1227300373Smav 1228300373Smavuint64_t 1229300373Smavntb_db_read(struct ntb_softc *ntb) 1230300373Smav{ 1231300373Smav 1232301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1233301811Sngie uint64_t res; 1234301811Sngie unsigned i; 1235301811Sngie 1236301811Sngie res = 0; 1237301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1238301811Sngie if (ntb->msix_vec[i].masked != 0) 1239301811Sngie res |= ntb_db_vector_mask(ntb, i); 1240301811Sngie } 1241301811Sngie return (res); 1242301811Sngie } 1243301811Sngie 1244300373Smav return (db_ioread(ntb, ntb->self_reg->db_bell)); 1245300373Smav} 1246300373Smav 1247300373Smavvoid 1248300373Smavntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1249300373Smav{ 1250300373Smav 1251300373Smav KASSERT((bits & ~ntb->db_valid_mask) == 0, 1252300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1253300373Smav (uintmax_t)(bits & ~ntb->db_valid_mask), 1254300373Smav (uintmax_t)ntb->db_valid_mask)); 1255300373Smav 1256301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1257301811Sngie unsigned i; 1258301811Sngie 1259301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1260301811Sngie if ((bits & ntb_db_vector_mask(ntb, i)) != 0) { 1261301811Sngie DB_MASK_LOCK(ntb); 1262301811Sngie if (ntb->msix_vec[i].masked != 0) { 1263301811Sngie /* XXX These need a public API. */ 1264301811Sngie#if 0 1265301811Sngie pci_unmask_msix(ntb->device, i); 1266301811Sngie#endif 1267301811Sngie ntb->msix_vec[i].masked = 0; 1268301811Sngie } 1269301811Sngie DB_MASK_UNLOCK(ntb); 1270301811Sngie } 1271301811Sngie } 1272301811Sngie return; 1273301811Sngie } 1274301811Sngie 1275300373Smav db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1276300373Smav} 1277300373Smav 1278300373Smavstatic inline uint64_t 1279300373Smavntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1280300373Smav{ 1281300373Smav uint64_t shift, mask; 1282300373Smav 1283300373Smav shift = ntb->db_vec_shift; 1284300373Smav mask = (1ull << shift) - 1; 1285300373Smav return (mask << (shift * db_vector)); 1286300373Smav} 1287300373Smav 1288300373Smavstatic void 1289300373Smavntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1290300373Smav{ 1291300373Smav uint64_t vec_mask; 1292300373Smav 1293300373Smav ntb->last_ts = ticks; 1294300373Smav vec_mask = ntb_vec_mask(ntb, vec); 1295300373Smav 1296300373Smav if ((vec_mask & ntb->db_link_mask) != 0) { 1297300373Smav if (ntb_poll_link(ntb)) 1298300373Smav ntb_link_event(ntb); 1299250079Scarl } 1300250079Scarl 1301301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1302301811Sngie (vec_mask & ntb->db_link_mask) == 0) { 1303301811Sngie DB_MASK_LOCK(ntb); 1304301811Sngie if (ntb->msix_vec[vec].masked == 0) { 1305301811Sngie /* XXX These need a public API. */ 1306301811Sngie#if 0 1307301811Sngie pci_mask_msix(ntb->device, vec); 1308301811Sngie#endif 1309301811Sngie ntb->msix_vec[vec].masked = 1; 1310301811Sngie } 1311301811Sngie DB_MASK_UNLOCK(ntb); 1312301811Sngie } 1313301811Sngie 1314300373Smav if ((vec_mask & ntb->db_valid_mask) != 0) 1315300373Smav ntb_db_event(ntb, vec); 1316250079Scarl} 1317250079Scarl 1318300373Smavstatic void 1319300373Smavndev_vec_isr(void *arg) 1320300373Smav{ 1321300373Smav struct ntb_vec *nvec = arg; 1322300373Smav 1323300373Smav ntb_interrupt(nvec->ntb, nvec->num); 1324300373Smav} 1325300373Smav 1326300373Smavstatic void 1327300373Smavndev_irq_isr(void *arg) 1328300373Smav{ 1329300373Smav /* If we couldn't set up MSI-X, we only have the one vector. */ 1330300373Smav ntb_interrupt(arg, 0); 1331300373Smav} 1332300373Smav 1333250079Scarlstatic int 1334300373Smavntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1335250079Scarl{ 1336300373Smav uint32_t i; 1337250079Scarl 1338300373Smav ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1339250079Scarl M_ZERO | M_WAITOK); 1340250079Scarl for (i = 0; i < num_vectors; i++) { 1341300373Smav ntb->msix_vec[i].num = i; 1342300373Smav ntb->msix_vec[i].ntb = ntb; 1343250079Scarl } 1344250079Scarl 1345250079Scarl return (0); 1346250079Scarl} 1347250079Scarl 1348250079Scarlstatic void 1349300373Smavntb_free_msix_vec(struct ntb_softc *ntb) 1350250079Scarl{ 1351250079Scarl 1352300373Smav if (ntb->msix_vec == NULL) 1353300373Smav return; 1354250079Scarl 1355300373Smav free(ntb->msix_vec, M_NTB); 1356300373Smav ntb->msix_vec = NULL; 1357250079Scarl} 1358250079Scarl 1359301811Sngiestatic void 1360301811Sngientb_get_msix_info(struct ntb_softc *ntb) 1361301811Sngie{ 1362301811Sngie struct pci_devinfo *dinfo; 1363301811Sngie struct pcicfg_msix *msix; 1364301811Sngie uint32_t laddr, data, i, offset; 1365301811Sngie 1366301811Sngie dinfo = device_get_ivars(ntb->device); 1367301811Sngie msix = &dinfo->cfg.msix; 1368301811Sngie 1369301811Sngie laddr = data = 0; 1370301811Sngie 1371301811Sngie CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1372301811Sngie 1373301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1374301811Sngie offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1375301811Sngie 1376301811Sngie laddr = bus_read_4(msix->msix_table_res, offset + 1377301811Sngie PCI_MSIX_ENTRY_LOWER_ADDR); 1378301904Smav ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1379301811Sngie 1380301811Sngie KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1381301811Sngie ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1382301811Sngie MSI_INTEL_ADDR_BASE)); 1383301904Smav ntb->msix_data[i].nmd_ofs = laddr; 1384301811Sngie 1385301811Sngie data = bus_read_4(msix->msix_table_res, offset + 1386301811Sngie PCI_MSIX_ENTRY_DATA); 1387301811Sngie ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1388301811Sngie 1389301811Sngie ntb->msix_data[i].nmd_data = data; 1390301811Sngie } 1391301811Sngie} 1392301811Sngie 1393250079Scarlstatic struct ntb_hw_info * 1394250079Scarlntb_get_device_info(uint32_t device_id) 1395250079Scarl{ 1396250079Scarl struct ntb_hw_info *ep = pci_ids; 1397250079Scarl 1398250079Scarl while (ep->device_id) { 1399250079Scarl if (ep->device_id == device_id) 1400250079Scarl return (ep); 1401250079Scarl ++ep; 1402250079Scarl } 1403250079Scarl return (NULL); 1404250079Scarl} 1405250079Scarl 1406300373Smavstatic void 1407300373Smavntb_teardown_xeon(struct ntb_softc *ntb) 1408250079Scarl{ 1409250079Scarl 1410300373Smav if (ntb->reg != NULL) 1411300373Smav ntb_link_disable(ntb); 1412300373Smav} 1413300373Smav 1414300373Smavstatic void 1415300373Smavntb_detect_max_mw(struct ntb_softc *ntb) 1416300373Smav{ 1417300373Smav 1418300373Smav if (ntb->type == NTB_ATOM) { 1419300373Smav ntb->mw_count = ATOM_MW_COUNT; 1420300373Smav return; 1421300373Smav } 1422300373Smav 1423300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 1424300373Smav ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1425250079Scarl else 1426300373Smav ntb->mw_count = XEON_SNB_MW_COUNT; 1427250079Scarl} 1428250079Scarl 1429250079Scarlstatic int 1430300373Smavntb_detect_xeon(struct ntb_softc *ntb) 1431250079Scarl{ 1432300373Smav uint8_t ppd, conn_type; 1433250079Scarl 1434300373Smav ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1435300373Smav ntb->ppd = ppd; 1436250079Scarl 1437300373Smav if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1438300373Smav ntb->dev_type = NTB_DEV_DSD; 1439300373Smav else 1440300373Smav ntb->dev_type = NTB_DEV_USD; 1441300373Smav 1442300373Smav if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1443300373Smav ntb->features |= NTB_SPLIT_BAR; 1444300373Smav 1445301811Sngie /* 1446301811Sngie * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1447301811Sngie * errata workaround; only do one at a time. 1448301811Sngie */ 1449300373Smav if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1450301811Sngie ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1451300373Smav 1452300373Smav conn_type = ppd & XEON_PPD_CONN_TYPE; 1453300373Smav switch (conn_type) { 1454250079Scarl case NTB_CONN_B2B: 1455300373Smav ntb->conn_type = conn_type; 1456250079Scarl break; 1457250079Scarl case NTB_CONN_RP: 1458300373Smav case NTB_CONN_TRANSPARENT: 1459250079Scarl default: 1460300373Smav device_printf(ntb->device, "Unsupported connection type: %u\n", 1461300373Smav (unsigned)conn_type); 1462250079Scarl return (ENXIO); 1463250079Scarl } 1464300373Smav return (0); 1465300373Smav} 1466250079Scarl 1467300373Smavstatic int 1468300373Smavntb_detect_atom(struct ntb_softc *ntb) 1469300373Smav{ 1470300373Smav uint32_t ppd, conn_type; 1471300373Smav 1472300373Smav ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1473300373Smav ntb->ppd = ppd; 1474300373Smav 1475300373Smav if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1476250079Scarl ntb->dev_type = NTB_DEV_DSD; 1477250079Scarl else 1478250079Scarl ntb->dev_type = NTB_DEV_USD; 1479250079Scarl 1480300373Smav conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1481300373Smav switch (conn_type) { 1482300373Smav case NTB_CONN_B2B: 1483300373Smav ntb->conn_type = conn_type; 1484300373Smav break; 1485300373Smav default: 1486300373Smav device_printf(ntb->device, "Unsupported NTB configuration\n"); 1487300373Smav return (ENXIO); 1488250079Scarl } 1489250079Scarl return (0); 1490250079Scarl} 1491250079Scarl 1492250079Scarlstatic int 1493300373Smavntb_xeon_init_dev(struct ntb_softc *ntb) 1494250079Scarl{ 1495300373Smav int rc; 1496250079Scarl 1497300373Smav ntb->spad_count = XEON_SPAD_COUNT; 1498300373Smav ntb->db_count = XEON_DB_COUNT; 1499300373Smav ntb->db_link_mask = XEON_DB_LINK_BIT; 1500300373Smav ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1501300373Smav ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1502250079Scarl 1503300373Smav if (ntb->conn_type != NTB_CONN_B2B) { 1504250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1505300373Smav ntb->conn_type); 1506250079Scarl return (ENXIO); 1507250079Scarl } 1508250079Scarl 1509300373Smav ntb->reg = &xeon_reg; 1510300373Smav ntb->self_reg = &xeon_pri_reg; 1511300373Smav ntb->peer_reg = &xeon_b2b_reg; 1512300373Smav ntb->xlat_reg = &xeon_sec_xlat; 1513300373Smav 1514301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1515301811Sngie ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1516301811Sngie ntb->mw_count; 1517301811Sngie ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1518301811Sngie g_ntb_msix_idx, ntb->msix_mw_idx); 1519301811Sngie rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1520301811Sngie VM_MEMATTR_UNCACHEABLE); 1521301811Sngie KASSERT(rc == 0, ("shouldn't fail")); 1522301811Sngie } else if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1523301811Sngie /* 1524301811Sngie * There is a Xeon hardware errata related to writes to SDOORBELL or 1525301811Sngie * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1526301811Sngie * which may hang the system. To workaround this, use a memory 1527301811Sngie * window to access the interrupt and scratch pad registers on the 1528301811Sngie * remote system. 1529301811Sngie */ 1530300373Smav ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1531300373Smav ntb->mw_count; 1532300373Smav ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1533300373Smav g_ntb_mw_idx, ntb->b2b_mw_idx); 1534301811Sngie rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1535301811Sngie VM_MEMATTR_UNCACHEABLE); 1536300373Smav KASSERT(rc == 0, ("shouldn't fail")); 1537300373Smav } else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1538300373Smav /* 1539300373Smav * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1540300373Smav * mirrored to the remote system. Shrink the number of bits by one, 1541300373Smav * since bit 14 is the last bit. 1542300373Smav * 1543300373Smav * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1544300373Smav * anyway. Nor for non-B2B connection types. 1545300373Smav */ 1546300373Smav ntb->db_count = XEON_DB_COUNT - 1; 1547300373Smav 1548300373Smav ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1549300373Smav 1550300373Smav if (ntb->dev_type == NTB_DEV_USD) 1551300373Smav rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1552300373Smav &xeon_b2b_usd_addr); 1553250079Scarl else 1554300373Smav rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1555300373Smav &xeon_b2b_dsd_addr); 1556300373Smav if (rc != 0) 1557300373Smav return (rc); 1558250079Scarl 1559300373Smav /* Enable Bus Master and Memory Space on the secondary side */ 1560300373Smav ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1561300373Smav PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1562250079Scarl 1563300373Smav /* 1564300373Smav * Mask all doorbell interrupts. 1565300373Smav */ 1566301811Sngie DB_MASK_LOCK(ntb); 1567301811Sngie ntb->db_mask = ntb->db_valid_mask; 1568301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1569301811Sngie DB_MASK_UNLOCK(ntb); 1570250079Scarl 1571301811Sngie rc = xeon_setup_msix_bar(ntb); 1572301811Sngie if (rc != 0) 1573301811Sngie return (rc); 1574301811Sngie 1575300373Smav rc = ntb_init_isr(ntb); 1576300373Smav return (rc); 1577300373Smav} 1578250079Scarl 1579300373Smavstatic int 1580300373Smavntb_atom_init_dev(struct ntb_softc *ntb) 1581300373Smav{ 1582300373Smav int error; 1583250079Scarl 1584300373Smav KASSERT(ntb->conn_type == NTB_CONN_B2B, 1585300373Smav ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1586300373Smav 1587300373Smav ntb->spad_count = ATOM_SPAD_COUNT; 1588300373Smav ntb->db_count = ATOM_DB_COUNT; 1589300373Smav ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1590300373Smav ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1591300373Smav ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1592300373Smav 1593300373Smav ntb->reg = &atom_reg; 1594300373Smav ntb->self_reg = &atom_pri_reg; 1595300373Smav ntb->peer_reg = &atom_b2b_reg; 1596300373Smav ntb->xlat_reg = &atom_sec_xlat; 1597300373Smav 1598250079Scarl /* 1599300373Smav * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1600250079Scarl * resolved. Mask transaction layer internal parity errors. 1601250079Scarl */ 1602250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1603250079Scarl 1604300373Smav configure_atom_secondary_side_bars(ntb); 1605250079Scarl 1606250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1607300373Smav ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1608250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1609250079Scarl 1610300373Smav error = ntb_init_isr(ntb); 1611300373Smav if (error != 0) 1612300373Smav return (error); 1613300373Smav 1614300373Smav /* Initiate PCI-E link training */ 1615300373Smav ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1616300373Smav 1617300373Smav callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1618300373Smav 1619250079Scarl return (0); 1620250079Scarl} 1621250079Scarl 1622300373Smav/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1623255279Scarlstatic void 1624300373Smavconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1625255279Scarl{ 1626255279Scarl 1627255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1628300373Smav ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1629300373Smav XEON_B2B_BAR2_ADDR64); 1630300373Smav ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1631300373Smav XEON_B2B_BAR4_ADDR64); 1632300373Smav ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1633300373Smav ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1634255279Scarl } else { 1635300373Smav ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1636300373Smav XEON_B2B_BAR2_ADDR64); 1637300373Smav ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1638300373Smav XEON_B2B_BAR4_ADDR64); 1639300373Smav ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1640300373Smav ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1641255279Scarl } 1642255279Scarl} 1643255279Scarl 1644300373Smav 1645300373Smav/* 1646300373Smav * When working around Xeon SDOORBELL errata by remapping remote registers in a 1647300373Smav * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1648300373Smav * remains for use by a higher layer. 1649300373Smav * 1650300373Smav * Will only be used if working around SDOORBELL errata and the BIOS-configured 1651300373Smav * MW size is sufficiently large. 1652300373Smav */ 1653300373Smavstatic unsigned int ntb_b2b_mw_share; 1654300516SmavTUNABLE_INT("hw.ntb.b2b_mw_share", &ntb_b2b_mw_share); 1655300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1656300373Smav 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1657300373Smav "MW with higher level consumers. Both sides of the NTB MUST set the same " 1658300373Smav "value here."); 1659300373Smav 1660255279Scarlstatic void 1661300373Smavxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1662300373Smav enum ntb_bar regbar) 1663255279Scarl{ 1664300373Smav struct ntb_pci_bar_info *bar; 1665300373Smav uint8_t bar_sz; 1666255279Scarl 1667300373Smav if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1668300373Smav return; 1669300373Smav 1670300373Smav bar = &ntb->bar_info[idx]; 1671300373Smav bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1672300373Smav if (idx == regbar) { 1673300373Smav if (ntb->b2b_off != 0) 1674300373Smav bar_sz--; 1675255279Scarl else 1676300373Smav bar_sz = 0; 1677300373Smav } 1678300373Smav pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1679300373Smav bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1680300373Smav (void)bar_sz; 1681300373Smav} 1682300373Smav 1683300373Smavstatic void 1684300373Smavxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1685300373Smav enum ntb_bar idx, enum ntb_bar regbar) 1686300373Smav{ 1687301904Smav uint64_t reg_val; 1688300373Smav uint32_t base_reg, lmt_reg; 1689300373Smav 1690300373Smav bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1691304376Smav if (idx == regbar) { 1692304376Smav if (ntb->b2b_off) 1693304376Smav bar_addr += ntb->b2b_off; 1694304376Smav else 1695304376Smav bar_addr = 0; 1696304376Smav } 1697300373Smav 1698301811Sngie /* 1699301811Sngie * Set limit registers first to avoid an errata where setting the base 1700301811Sngie * registers locks the limit registers. 1701301811Sngie */ 1702300373Smav if (!bar_is_64bit(ntb, idx)) { 1703301904Smav ntb_reg_write(4, lmt_reg, bar_addr); 1704301811Sngie reg_val = ntb_reg_read(4, lmt_reg); 1705301811Sngie (void)reg_val; 1706301811Sngie 1707300373Smav ntb_reg_write(4, base_reg, bar_addr); 1708300373Smav reg_val = ntb_reg_read(4, base_reg); 1709300373Smav (void)reg_val; 1710301811Sngie } else { 1711301904Smav ntb_reg_write(8, lmt_reg, bar_addr); 1712301811Sngie reg_val = ntb_reg_read(8, lmt_reg); 1713301811Sngie (void)reg_val; 1714300373Smav 1715300373Smav ntb_reg_write(8, base_reg, bar_addr); 1716300373Smav reg_val = ntb_reg_read(8, base_reg); 1717300373Smav (void)reg_val; 1718255279Scarl } 1719255279Scarl} 1720255279Scarl 1721250079Scarlstatic void 1722300373Smavxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1723250079Scarl{ 1724300373Smav struct ntb_pci_bar_info *bar; 1725250079Scarl 1726300373Smav bar = &ntb->bar_info[idx]; 1727300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1728300373Smav ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1729300373Smav base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1730300373Smav } else { 1731300373Smav ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1732300373Smav base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1733300373Smav } 1734300373Smav (void)base_addr; 1735300373Smav} 1736300373Smav 1737300373Smavstatic int 1738301811Sngiexeon_setup_msix_bar(struct ntb_softc *ntb) 1739301811Sngie{ 1740301811Sngie enum ntb_bar bar_num; 1741301811Sngie 1742301811Sngie if (!HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1743301811Sngie return (0); 1744301811Sngie 1745301811Sngie bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1746301904Smav ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1747301811Sngie return (0); 1748301811Sngie} 1749301811Sngie 1750301811Sngiestatic int 1751300373Smavxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1752300373Smav const struct ntb_b2b_addr *peer_addr) 1753300373Smav{ 1754300373Smav struct ntb_pci_bar_info *b2b_bar; 1755300373Smav vm_size_t bar_size; 1756300373Smav uint64_t bar_addr; 1757300373Smav enum ntb_bar b2b_bar_num, i; 1758300373Smav 1759300373Smav if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1760300373Smav b2b_bar = NULL; 1761300373Smav b2b_bar_num = NTB_CONFIG_BAR; 1762300373Smav ntb->b2b_off = 0; 1763300373Smav } else { 1764300373Smav b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1765300373Smav KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1766300373Smav ("invalid b2b mw bar")); 1767300373Smav 1768300373Smav b2b_bar = &ntb->bar_info[b2b_bar_num]; 1769300373Smav bar_size = b2b_bar->size; 1770300373Smav 1771300373Smav if (ntb_b2b_mw_share != 0 && 1772300373Smav (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1773300373Smav ntb->b2b_off = bar_size >> 1; 1774300373Smav else if (bar_size >= XEON_B2B_MIN_SIZE) { 1775300373Smav ntb->b2b_off = 0; 1776300373Smav } else { 1777300373Smav device_printf(ntb->device, 1778300373Smav "B2B bar size is too small!\n"); 1779300373Smav return (EIO); 1780250079Scarl } 1781250079Scarl } 1782250079Scarl 1783300373Smav /* 1784300373Smav * Reset the secondary bar sizes to match the primary bar sizes. 1785300373Smav * (Except, disable or halve the size of the B2B secondary bar.) 1786300373Smav */ 1787300373Smav for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1788300373Smav xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1789300373Smav 1790300373Smav bar_addr = 0; 1791300373Smav if (b2b_bar_num == NTB_CONFIG_BAR) 1792300373Smav bar_addr = addr->bar0_addr; 1793300373Smav else if (b2b_bar_num == NTB_B2B_BAR_1) 1794300373Smav bar_addr = addr->bar2_addr64; 1795300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1796300373Smav bar_addr = addr->bar4_addr64; 1797300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2) 1798300373Smav bar_addr = addr->bar4_addr32; 1799300373Smav else if (b2b_bar_num == NTB_B2B_BAR_3) 1800300373Smav bar_addr = addr->bar5_addr32; 1801300373Smav else 1802300373Smav KASSERT(false, ("invalid bar")); 1803300373Smav 1804300373Smav ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1805300373Smav 1806300373Smav /* 1807300373Smav * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1808300373Smav * register BAR. The B2B BAR is either disabled above or configured 1809300373Smav * half-size. It starts at PBAR xlat + offset. 1810300373Smav * 1811300373Smav * Also set up incoming BAR limits == base (zero length window). 1812300373Smav */ 1813300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1814300373Smav b2b_bar_num); 1815300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1816300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1817300373Smav NTB_B2B_BAR_2, b2b_bar_num); 1818300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1819300373Smav NTB_B2B_BAR_3, b2b_bar_num); 1820300373Smav } else 1821300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1822300373Smav NTB_B2B_BAR_2, b2b_bar_num); 1823300373Smav 1824300373Smav /* Zero incoming translation addrs */ 1825300373Smav ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1826300373Smav ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1827300373Smav 1828301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1829301811Sngie size_t size, xlatoffset; 1830301811Sngie 1831301811Sngie switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1832301811Sngie case NTB_B2B_BAR_1: 1833301811Sngie size = 8; 1834301811Sngie xlatoffset = XEON_SBAR2XLAT_OFFSET; 1835301811Sngie break; 1836301811Sngie case NTB_B2B_BAR_2: 1837301811Sngie xlatoffset = XEON_SBAR4XLAT_OFFSET; 1838301811Sngie if (HAS_FEATURE(NTB_SPLIT_BAR)) 1839301811Sngie size = 4; 1840301811Sngie else 1841301811Sngie size = 8; 1842301811Sngie break; 1843301811Sngie case NTB_B2B_BAR_3: 1844301811Sngie xlatoffset = XEON_SBAR5XLAT_OFFSET; 1845301811Sngie size = 4; 1846301811Sngie break; 1847301811Sngie default: 1848301811Sngie KASSERT(false, ("Bogus msix mw idx: %u", 1849301811Sngie ntb->msix_mw_idx)); 1850301811Sngie return (EINVAL); 1851301811Sngie } 1852301811Sngie 1853301811Sngie /* 1854301811Sngie * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1855301811Sngie * workaround 1856301811Sngie */ 1857301904Smav if (size == 4) { 1858301811Sngie ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1859301904Smav ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1860301904Smav } else { 1861301811Sngie ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1862301904Smav ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1863301904Smav } 1864301811Sngie } 1865301811Sngie (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1866301811Sngie (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1867301811Sngie 1868300373Smav /* Zero outgoing translation limits (whole bar size windows) */ 1869300373Smav ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1870300373Smav ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1871300373Smav 1872300373Smav /* Set outgoing translation offsets */ 1873300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1874300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1875300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1876300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1877300373Smav } else 1878300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1879300373Smav 1880300373Smav /* Set the translation offset for B2B registers */ 1881300373Smav bar_addr = 0; 1882300373Smav if (b2b_bar_num == NTB_CONFIG_BAR) 1883300373Smav bar_addr = peer_addr->bar0_addr; 1884300373Smav else if (b2b_bar_num == NTB_B2B_BAR_1) 1885300373Smav bar_addr = peer_addr->bar2_addr64; 1886300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1887300373Smav bar_addr = peer_addr->bar4_addr64; 1888300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2) 1889300373Smav bar_addr = peer_addr->bar4_addr32; 1890300373Smav else if (b2b_bar_num == NTB_B2B_BAR_3) 1891300373Smav bar_addr = peer_addr->bar5_addr32; 1892300373Smav else 1893300373Smav KASSERT(false, ("invalid bar")); 1894300373Smav 1895300373Smav /* 1896300373Smav * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1897300373Smav * at a time. 1898300373Smav */ 1899300373Smav ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1900300373Smav ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1901300373Smav return (0); 1902250079Scarl} 1903250079Scarl 1904300373Smavstatic inline bool 1905301811Sngie_xeon_link_is_up(struct ntb_softc *ntb) 1906301811Sngie{ 1907301811Sngie 1908301811Sngie if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1909301811Sngie return (true); 1910301811Sngie return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1911301811Sngie} 1912301811Sngie 1913301811Sngiestatic inline bool 1914300373Smavlink_is_up(struct ntb_softc *ntb) 1915300373Smav{ 1916300373Smav 1917301811Sngie if (ntb->type == NTB_XEON) 1918301811Sngie return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1919301811Sngie !HAS_FEATURE(NTB_SB01BASE_LOCKUP))); 1920300373Smav 1921300373Smav KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1922300373Smav return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1923300373Smav} 1924300373Smav 1925300373Smavstatic inline bool 1926300373Smavatom_link_is_err(struct ntb_softc *ntb) 1927300373Smav{ 1928300373Smav uint32_t status; 1929300373Smav 1930300373Smav KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1931300373Smav 1932300373Smav status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1933300373Smav if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1934300373Smav return (true); 1935300373Smav 1936300373Smav status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1937300373Smav return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1938300373Smav} 1939300373Smav 1940300373Smav/* Atom does not have link status interrupt, poll on that platform */ 1941250079Scarlstatic void 1942300373Smavatom_link_hb(void *arg) 1943250079Scarl{ 1944300373Smav struct ntb_softc *ntb = arg; 1945300373Smav sbintime_t timo, poll_ts; 1946300373Smav 1947300373Smav timo = NTB_HB_TIMEOUT * hz; 1948300373Smav poll_ts = ntb->last_ts + timo; 1949300373Smav 1950300373Smav /* 1951300373Smav * Delay polling the link status if an interrupt was received, unless 1952300373Smav * the cached link status says the link is down. 1953300373Smav */ 1954300373Smav if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1955300373Smav timo = poll_ts - ticks; 1956300373Smav goto out; 1957300373Smav } 1958300373Smav 1959300373Smav if (ntb_poll_link(ntb)) 1960300373Smav ntb_link_event(ntb); 1961300373Smav 1962300373Smav if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1963300373Smav /* Link is down with error, proceed with recovery */ 1964300373Smav callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1965300373Smav return; 1966300373Smav } 1967300373Smav 1968300373Smavout: 1969300373Smav callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1970300373Smav} 1971300373Smav 1972300373Smavstatic void 1973300373Smavatom_perform_link_restart(struct ntb_softc *ntb) 1974300373Smav{ 1975250079Scarl uint32_t status; 1976250079Scarl 1977250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1978300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1979300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1980300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1981300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1982250079Scarl 1983250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1984250079Scarl pause("ModPhy", hz / 10); 1985250079Scarl 1986250079Scarl /* Clear AER Errors, write to clear */ 1987300373Smav status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1988250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1989300373Smav ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1990250079Scarl 1991250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1992300373Smav status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1993300373Smav status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1994300373Smav ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1995250079Scarl 1996250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1997300373Smav status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1998300373Smav status |= ATOM_DESKEWSTS_DBERR; 1999300373Smav ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 2000250079Scarl 2001300373Smav status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2002300373Smav status &= ATOM_IBIST_ERR_OFLOW; 2003300373Smav ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2004250079Scarl 2005250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 2006300373Smav status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2007300373Smav status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2008300373Smav ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2009250079Scarl} 2010250079Scarl 2011300373Smav/* 2012300373Smav * ntb_set_ctx() - associate a driver context with an ntb device 2013300373Smav * @ntb: NTB device context 2014300373Smav * @ctx: Driver context 2015300373Smav * @ctx_ops: Driver context operations 2016300373Smav * 2017300373Smav * Associate a driver context and operations with a ntb device. The context is 2018300373Smav * provided by the client driver, and the driver may associate a different 2019300373Smav * context with each ntb device. 2020300373Smav * 2021300373Smav * Return: Zero if the context is associated, otherwise an error number. 2022300373Smav */ 2023300373Smavint 2024300373Smavntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 2025300373Smav{ 2026300373Smav 2027300373Smav if (ctx == NULL || ops == NULL) 2028300373Smav return (EINVAL); 2029300373Smav if (ntb->ctx_ops != NULL) 2030300373Smav return (EINVAL); 2031300373Smav 2032300373Smav CTX_LOCK(ntb); 2033300373Smav if (ntb->ctx_ops != NULL) { 2034300373Smav CTX_UNLOCK(ntb); 2035300373Smav return (EINVAL); 2036300373Smav } 2037300373Smav ntb->ntb_ctx = ctx; 2038300373Smav ntb->ctx_ops = ops; 2039300373Smav CTX_UNLOCK(ntb); 2040300373Smav 2041300373Smav return (0); 2042300373Smav} 2043300373Smav 2044300373Smav/* 2045300373Smav * It is expected that this will only be used from contexts where the ctx_lock 2046300373Smav * is not needed to protect ntb_ctx lifetime. 2047300373Smav */ 2048300373Smavvoid * 2049300373Smavntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 2050300373Smav{ 2051300373Smav 2052300373Smav KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2053300373Smav if (ops != NULL) 2054300373Smav *ops = ntb->ctx_ops; 2055300373Smav return (ntb->ntb_ctx); 2056300373Smav} 2057300373Smav 2058300373Smav/* 2059300373Smav * ntb_clear_ctx() - disassociate any driver context from an ntb device 2060300373Smav * @ntb: NTB device context 2061300373Smav * 2062300373Smav * Clear any association that may exist between a driver context and the ntb 2063300373Smav * device. 2064300373Smav */ 2065300373Smavvoid 2066300373Smavntb_clear_ctx(struct ntb_softc *ntb) 2067300373Smav{ 2068300373Smav 2069300373Smav CTX_LOCK(ntb); 2070300373Smav ntb->ntb_ctx = NULL; 2071300373Smav ntb->ctx_ops = NULL; 2072300373Smav CTX_UNLOCK(ntb); 2073300373Smav} 2074300373Smav 2075300373Smav/* 2076300373Smav * ntb_link_event() - notify driver context of a change in link status 2077300373Smav * @ntb: NTB device context 2078300373Smav * 2079300373Smav * Notify the driver context that the link status may have changed. The driver 2080300373Smav * should call ntb_link_is_up() to get the current status. 2081300373Smav */ 2082300373Smavvoid 2083300373Smavntb_link_event(struct ntb_softc *ntb) 2084300373Smav{ 2085300373Smav 2086300373Smav CTX_LOCK(ntb); 2087300373Smav if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2088300373Smav ntb->ctx_ops->link_event(ntb->ntb_ctx); 2089300373Smav CTX_UNLOCK(ntb); 2090300373Smav} 2091300373Smav 2092300373Smav/* 2093300373Smav * ntb_db_event() - notify driver context of a doorbell event 2094300373Smav * @ntb: NTB device context 2095300373Smav * @vector: Interrupt vector number 2096300373Smav * 2097300373Smav * Notify the driver context of a doorbell event. If hardware supports 2098300373Smav * multiple interrupt vectors for doorbells, the vector number indicates which 2099300373Smav * vector received the interrupt. The vector number is relative to the first 2100300373Smav * vector used for doorbells, starting at zero, and must be less than 2101300373Smav * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2102300373Smav * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2103300373Smav * those bits are associated with the vector number. 2104300373Smav */ 2105250079Scarlstatic void 2106300373Smavntb_db_event(struct ntb_softc *ntb, uint32_t vec) 2107250079Scarl{ 2108250079Scarl 2109300373Smav CTX_LOCK(ntb); 2110300373Smav if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2111300373Smav ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2112300373Smav CTX_UNLOCK(ntb); 2113300373Smav} 2114250079Scarl 2115300373Smav/* 2116300373Smav * ntb_link_enable() - enable the link on the secondary side of the ntb 2117300373Smav * @ntb: NTB device context 2118300373Smav * @max_speed: The maximum link speed expressed as PCIe generation number[0] 2119300373Smav * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 2120300373Smav * 2121300373Smav * Enable the link on the secondary side of the ntb. This can only be done 2122300373Smav * from the primary side of the ntb in primary or b2b topology. The ntb device 2123300373Smav * should train the link to its maximum speed and width, or the requested speed 2124300373Smav * and width, whichever is smaller, if supported. 2125300373Smav * 2126300373Smav * Return: Zero on success, otherwise an error number. 2127300373Smav * 2128300373Smav * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 2129300373Smav * and width input will be ignored. 2130300373Smav */ 2131300373Smavint 2132300373Smavntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 2133300373Smav enum ntb_width w __unused) 2134300373Smav{ 2135300373Smav uint32_t cntl; 2136250079Scarl 2137301811Sngie ntb_printf(2, "%s\n", __func__); 2138301811Sngie 2139300373Smav if (ntb->type == NTB_ATOM) { 2140300373Smav pci_write_config(ntb->device, NTB_PPD_OFFSET, 2141300373Smav ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2142300373Smav return (0); 2143250079Scarl } 2144250079Scarl 2145300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2146300373Smav ntb_link_event(ntb); 2147300373Smav return (0); 2148300373Smav } 2149300373Smav 2150300373Smav cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2151300373Smav cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2152300373Smav cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2153300373Smav cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2154300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 2155300373Smav cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2156300373Smav ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2157300373Smav return (0); 2158250079Scarl} 2159250079Scarl 2160300373Smav/* 2161300373Smav * ntb_link_disable() - disable the link on the secondary side of the ntb 2162300373Smav * @ntb: NTB device context 2163300373Smav * 2164300373Smav * Disable the link on the secondary side of the ntb. This can only be done 2165300373Smav * from the primary side of the ntb in primary or b2b topology. The ntb device 2166300373Smav * should disable the link. Returning from this call must indicate that a 2167300373Smav * barrier has passed, though with no more writes may pass in either direction 2168300373Smav * across the link, except if this call returns an error number. 2169300373Smav * 2170300373Smav * Return: Zero on success, otherwise an error number. 2171300373Smav */ 2172300373Smavint 2173300373Smavntb_link_disable(struct ntb_softc *ntb) 2174300373Smav{ 2175300373Smav uint32_t cntl; 2176300373Smav 2177301811Sngie ntb_printf(2, "%s\n", __func__); 2178301811Sngie 2179300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2180300373Smav ntb_link_event(ntb); 2181300373Smav return (0); 2182300373Smav } 2183300373Smav 2184300373Smav cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2185300373Smav cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2186300373Smav cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2187300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 2188300373Smav cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2189300373Smav cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2190300373Smav ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2191300373Smav return (0); 2192300373Smav} 2193300373Smav 2194301811Sngiebool 2195301811Sngientb_link_enabled(struct ntb_softc *ntb) 2196301811Sngie{ 2197301811Sngie uint32_t cntl; 2198301811Sngie 2199301811Sngie if (ntb->type == NTB_ATOM) { 2200301811Sngie cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2201301811Sngie return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2202301811Sngie } 2203301811Sngie 2204301811Sngie if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2205301811Sngie return (true); 2206301811Sngie 2207301811Sngie cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2208301811Sngie return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2209301811Sngie} 2210301811Sngie 2211250079Scarlstatic void 2212300373Smavrecover_atom_link(void *arg) 2213250079Scarl{ 2214250079Scarl struct ntb_softc *ntb = arg; 2215300373Smav unsigned speed, width, oldspeed, oldwidth; 2216250079Scarl uint32_t status32; 2217250079Scarl 2218300373Smav atom_perform_link_restart(ntb); 2219250079Scarl 2220300373Smav /* 2221300373Smav * There is a potential race between the 2 NTB devices recovering at 2222300373Smav * the same time. If the times are the same, the link will not recover 2223300373Smav * and the driver will be stuck in this loop forever. Add a random 2224300373Smav * interval to the recovery time to prevent this race. 2225300373Smav */ 2226300373Smav status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2227300373Smav pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2228250079Scarl 2229300373Smav if (atom_link_is_err(ntb)) 2230250079Scarl goto retry; 2231250079Scarl 2232300373Smav status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2233300373Smav if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2234300373Smav goto out; 2235300373Smav 2236300373Smav status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2237300373Smav width = NTB_LNK_STA_WIDTH(status32); 2238300373Smav speed = status32 & NTB_LINK_SPEED_MASK; 2239300373Smav 2240300373Smav oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2241300373Smav oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2242300373Smav if (oldwidth != width || oldspeed != speed) 2243250079Scarl goto retry; 2244250079Scarl 2245300373Smavout: 2246300373Smav callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2247300373Smav ntb); 2248250079Scarl return; 2249250079Scarl 2250250079Scarlretry: 2251300373Smav callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2252250079Scarl ntb); 2253250079Scarl} 2254250079Scarl 2255300373Smav/* 2256300373Smav * Polls the HW link status register(s); returns true if something has changed. 2257300373Smav */ 2258300373Smavstatic bool 2259300373Smavntb_poll_link(struct ntb_softc *ntb) 2260250079Scarl{ 2261250079Scarl uint32_t ntb_cntl; 2262300373Smav uint16_t reg_val; 2263250079Scarl 2264300373Smav if (ntb->type == NTB_ATOM) { 2265300373Smav ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2266300373Smav if (ntb_cntl == ntb->ntb_ctl) 2267300373Smav return (false); 2268300373Smav 2269300373Smav ntb->ntb_ctl = ntb_cntl; 2270300373Smav ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2271250079Scarl } else { 2272300373Smav db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2273250079Scarl 2274300373Smav reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2275300373Smav if (reg_val == ntb->lnk_sta) 2276300373Smav return (false); 2277300373Smav 2278300373Smav ntb->lnk_sta = reg_val; 2279301811Sngie 2280301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 2281301811Sngie if (_xeon_link_is_up(ntb)) { 2282301811Sngie if (!ntb->peer_msix_good) { 2283301811Sngie callout_reset(&ntb->peer_msix_work, 0, 2284301811Sngie ntb_exchange_msix, ntb); 2285301811Sngie return (false); 2286301811Sngie } 2287301811Sngie } else { 2288301811Sngie ntb->peer_msix_good = false; 2289301811Sngie ntb->peer_msix_done = false; 2290301811Sngie } 2291301811Sngie } 2292250079Scarl } 2293300373Smav return (true); 2294300373Smav} 2295250079Scarl 2296300373Smavstatic inline enum ntb_speed 2297300373Smavntb_link_sta_speed(struct ntb_softc *ntb) 2298300373Smav{ 2299250079Scarl 2300300373Smav if (!link_is_up(ntb)) 2301300373Smav return (NTB_SPEED_NONE); 2302300373Smav return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2303250079Scarl} 2304250079Scarl 2305300373Smavstatic inline enum ntb_width 2306300373Smavntb_link_sta_width(struct ntb_softc *ntb) 2307250079Scarl{ 2308250079Scarl 2309300373Smav if (!link_is_up(ntb)) 2310300373Smav return (NTB_WIDTH_NONE); 2311300373Smav return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2312300373Smav} 2313250079Scarl 2314300373SmavSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2315300373Smav "Driver state, statistics, and HW registers"); 2316250079Scarl 2317300373Smav#define NTB_REGSZ_MASK (3ul << 30) 2318300373Smav#define NTB_REG_64 (1ul << 30) 2319300373Smav#define NTB_REG_32 (2ul << 30) 2320300373Smav#define NTB_REG_16 (3ul << 30) 2321300373Smav#define NTB_REG_8 (0ul << 30) 2322250079Scarl 2323300373Smav#define NTB_DB_READ (1ul << 29) 2324300373Smav#define NTB_PCI_REG (1ul << 28) 2325300373Smav#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2326300373Smav 2327300373Smavstatic void 2328300373Smavntb_sysctl_init(struct ntb_softc *ntb) 2329250079Scarl{ 2330301811Sngie struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2331300373Smav struct sysctl_ctx_list *ctx; 2332300373Smav struct sysctl_oid *tree, *tmptree; 2333250079Scarl 2334300373Smav ctx = device_get_sysctl_ctx(ntb->device); 2335301811Sngie globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2336250079Scarl 2337301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2338301811Sngie CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2339301811Sngie sysctl_handle_link_status_human, "A", 2340301811Sngie "Link status (human readable)"); 2341301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2342301811Sngie CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2343301811Sngie "IU", "Link status (1=active, 0=inactive)"); 2344301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2345301811Sngie CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2346301811Sngie "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2347301811Sngie 2348301811Sngie tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2349301811Sngie CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2350300373Smav tree_par = SYSCTL_CHILDREN(tree); 2351250079Scarl 2352300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2353300373Smav &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2354300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2355300373Smav &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2356300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2357300373Smav &ntb->ppd, 0, "Raw PPD register (cached)"); 2358300373Smav 2359300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2360300373Smav#ifdef notyet 2361300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2362300373Smav &ntb->b2b_mw_idx, 0, 2363300373Smav "Index of the MW used for B2B remote register access"); 2364300373Smav#endif 2365300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2366300373Smav CTLFLAG_RD, &ntb->b2b_off, 2367300373Smav "If non-zero, offset of B2B register region in shared MW"); 2368250079Scarl } 2369250079Scarl 2370300373Smav SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2371300373Smav CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2372300373Smav "Features/errata of this NTB device"); 2373250079Scarl 2374300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2375300373Smav __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2376300373Smav "NTB CTL register (cached)"); 2377300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2378300373Smav __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2379300373Smav "LNK STA register (cached)"); 2380250079Scarl 2381300373Smav#ifdef notyet 2382300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2383300373Smav &ntb->mw_count, 0, "MW count"); 2384300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2385300373Smav &ntb->spad_count, 0, "Scratchpad count"); 2386300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2387300373Smav &ntb->db_count, 0, "Doorbell count"); 2388300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2389300373Smav &ntb->db_vec_count, 0, "Doorbell vector count"); 2390300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2391300373Smav &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2392300373Smav#endif 2393250079Scarl 2394300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2395300373Smav &ntb->db_valid_mask, "Doorbell valid mask"); 2396300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2397300373Smav &ntb->db_link_mask, "Doorbell link mask"); 2398300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2399300373Smav &ntb->db_mask, "Doorbell mask (cached)"); 2400300373Smav 2401300373Smav tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2402300373Smav CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2403300373Smav regpar = SYSCTL_CHILDREN(tmptree); 2404300373Smav 2405300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2406300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2407300373Smav ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2408300373Smav "NTB Control register"); 2409300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2410300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2411300373Smav 0x19c, sysctl_handle_register, "IU", 2412300373Smav "NTB Link Capabilities"); 2413300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2414300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2415300373Smav 0x1a0, sysctl_handle_register, "IU", 2416300373Smav "NTB Link Control register"); 2417300373Smav 2418300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2419300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2420300373Smav NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2421300373Smav sysctl_handle_register, "QU", "Doorbell mask register"); 2422300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2423300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2424300373Smav NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2425300373Smav sysctl_handle_register, "QU", "Doorbell register"); 2426300373Smav 2427300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2428300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2429300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2430300373Smav sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2431300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2432300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2433300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2434300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2435300373Smav sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2436300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2437300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2438300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2439300373Smav sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2440300373Smav } else { 2441300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2442300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2443300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2444300373Smav sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2445300373Smav } 2446300373Smav 2447300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2448300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2449300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2450300373Smav sysctl_handle_register, "QU", "Incoming LMT23 register"); 2451300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2452300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2453300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2454300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2455300373Smav sysctl_handle_register, "IU", "Incoming LMT4 register"); 2456300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2457300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2458300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2459300373Smav sysctl_handle_register, "IU", "Incoming LMT5 register"); 2460300373Smav } else { 2461300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2462300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2463300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2464300373Smav sysctl_handle_register, "QU", "Incoming LMT45 register"); 2465300373Smav } 2466300373Smav 2467300373Smav if (ntb->type == NTB_ATOM) 2468250079Scarl return; 2469250079Scarl 2470300373Smav tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2471300373Smav CTLFLAG_RD, NULL, "Xeon HW statistics"); 2472300373Smav statpar = SYSCTL_CHILDREN(tmptree); 2473300373Smav SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2474300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2475300373Smav NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2476300373Smav sysctl_handle_register, "SU", "Upstream Memory Miss"); 2477250079Scarl 2478300373Smav tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2479300373Smav CTLFLAG_RD, NULL, "Xeon HW errors"); 2480300373Smav errpar = SYSCTL_CHILDREN(tmptree); 2481300373Smav 2482300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2483300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2484300373Smav NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2485300373Smav sysctl_handle_register, "CU", "PPD"); 2486300373Smav 2487300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2488300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2489300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2490300373Smav sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2491300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2492300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2493300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2494300373Smav sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2495300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2496300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2497300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2498300373Smav sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2499300373Smav 2500300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2501300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2502300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2503300373Smav sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2504300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2505300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2506300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2507300373Smav sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2508300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2509300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2510300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2511300373Smav sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2512300373Smav 2513300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2514300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2515300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2516300373Smav sysctl_handle_register, "SU", "DEVSTS"); 2517300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2518300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2519300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2520300373Smav sysctl_handle_register, "SU", "LNKSTS"); 2521300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2522300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2523300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2524300373Smav sysctl_handle_register, "SU", "SLNKSTS"); 2525300373Smav 2526300373Smav SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2527300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2528300373Smav NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2529300373Smav sysctl_handle_register, "IU", "UNCERRSTS"); 2530300373Smav SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2531300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2532300373Smav NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2533300373Smav sysctl_handle_register, "IU", "CORERRSTS"); 2534300373Smav 2535300373Smav if (ntb->conn_type != NTB_CONN_B2B) 2536300373Smav return; 2537300373Smav 2538300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2539300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2540300373Smav NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2541300373Smav sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2542300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2543300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2544300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2545300373Smav NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2546300373Smav sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2547300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2548300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2549300373Smav NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2550300373Smav sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2551300373Smav } else { 2552300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2553300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2554300373Smav NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2555300373Smav sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2556300373Smav } 2557300373Smav 2558300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2559300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2560300373Smav NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2561300373Smav sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2562300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2563300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2564300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2565300373Smav NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2566300373Smav sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2567300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2568300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2569300373Smav NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2570300373Smav sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2571300373Smav } else { 2572300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2573300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2574300373Smav NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2575300373Smav sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2576300373Smav } 2577300373Smav 2578300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2579300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2580300373Smav NTB_REG_64 | ntb->xlat_reg->bar0_base, 2581300373Smav sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2582300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2583300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2584300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_base, 2585300373Smav sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2586300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2587300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2588300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2589300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_base, 2590300373Smav sysctl_handle_register, "IU", 2591300373Smav "Secondary BAR4 base register"); 2592300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2593300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2594300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_base, 2595300373Smav sysctl_handle_register, "IU", 2596300373Smav "Secondary BAR5 base register"); 2597300373Smav } else { 2598300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2599300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2600300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_base, 2601300373Smav sysctl_handle_register, "QU", 2602300373Smav "Secondary BAR45 base register"); 2603300373Smav } 2604250079Scarl} 2605250079Scarl 2606300373Smavstatic int 2607300373Smavsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2608250079Scarl{ 2609300373Smav struct ntb_softc *ntb; 2610300373Smav struct sbuf sb; 2611300373Smav int error; 2612250079Scarl 2613300373Smav error = 0; 2614300373Smav ntb = arg1; 2615300373Smav 2616300373Smav sbuf_new_for_sysctl(&sb, NULL, 256, req); 2617300373Smav 2618300373Smav sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2619300373Smav error = sbuf_finish(&sb); 2620300373Smav sbuf_delete(&sb); 2621300373Smav 2622300373Smav if (error || !req->newptr) 2623300373Smav return (error); 2624300373Smav return (EINVAL); 2625250079Scarl} 2626250079Scarl 2627300373Smavstatic int 2628301811Sngiesysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2629250079Scarl{ 2630300373Smav struct ntb_softc *ntb; 2631301811Sngie unsigned old, new; 2632301811Sngie int error; 2633301811Sngie 2634301811Sngie error = 0; 2635301811Sngie ntb = arg1; 2636301811Sngie 2637301811Sngie old = ntb_link_enabled(ntb); 2638301811Sngie 2639301811Sngie error = SYSCTL_OUT(req, &old, sizeof(old)); 2640301811Sngie if (error != 0 || req->newptr == NULL) 2641301811Sngie return (error); 2642301811Sngie 2643301811Sngie error = SYSCTL_IN(req, &new, sizeof(new)); 2644301811Sngie if (error != 0) 2645301811Sngie return (error); 2646301811Sngie 2647301811Sngie ntb_printf(0, "Admin set interface state to '%sabled'\n", 2648301811Sngie (new != 0)? "en" : "dis"); 2649301811Sngie 2650301811Sngie if (new != 0) 2651301811Sngie error = ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2652301811Sngie else 2653301811Sngie error = ntb_link_disable(ntb); 2654301811Sngie return (error); 2655301811Sngie} 2656301811Sngie 2657301811Sngiestatic int 2658301811Sngiesysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2659301811Sngie{ 2660301811Sngie struct ntb_softc *ntb; 2661300373Smav struct sbuf sb; 2662300373Smav enum ntb_speed speed; 2663300373Smav enum ntb_width width; 2664300373Smav int error; 2665250079Scarl 2666300373Smav error = 0; 2667300373Smav ntb = arg1; 2668250079Scarl 2669300373Smav sbuf_new_for_sysctl(&sb, NULL, 32, req); 2670300373Smav 2671300373Smav if (ntb_link_is_up(ntb, &speed, &width)) 2672300373Smav sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2673300373Smav (unsigned)speed, (unsigned)width); 2674300373Smav else 2675300373Smav sbuf_printf(&sb, "down"); 2676300373Smav 2677300373Smav error = sbuf_finish(&sb); 2678300373Smav sbuf_delete(&sb); 2679300373Smav 2680300373Smav if (error || !req->newptr) 2681300373Smav return (error); 2682300373Smav return (EINVAL); 2683250079Scarl} 2684250079Scarl 2685300373Smavstatic int 2686301811Sngiesysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2687301811Sngie{ 2688301811Sngie struct ntb_softc *ntb; 2689301811Sngie unsigned res; 2690301811Sngie int error; 2691301811Sngie 2692301811Sngie error = 0; 2693301811Sngie ntb = arg1; 2694301811Sngie 2695301811Sngie res = ntb_link_is_up(ntb, NULL, NULL); 2696301811Sngie 2697301811Sngie error = SYSCTL_OUT(req, &res, sizeof(res)); 2698301811Sngie if (error || !req->newptr) 2699301811Sngie return (error); 2700301811Sngie return (EINVAL); 2701301811Sngie} 2702301811Sngie 2703301811Sngiestatic int 2704300373Smavsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2705250079Scarl{ 2706300373Smav struct ntb_softc *ntb; 2707300373Smav const void *outp; 2708300373Smav uintptr_t sz; 2709300373Smav uint64_t umv; 2710300373Smav char be[sizeof(umv)]; 2711300373Smav size_t outsz; 2712300373Smav uint32_t reg; 2713300373Smav bool db, pci; 2714300373Smav int error; 2715250079Scarl 2716300373Smav ntb = arg1; 2717300373Smav reg = arg2 & ~NTB_REGFLAGS_MASK; 2718300373Smav sz = arg2 & NTB_REGSZ_MASK; 2719300373Smav db = (arg2 & NTB_DB_READ) != 0; 2720300373Smav pci = (arg2 & NTB_PCI_REG) != 0; 2721250079Scarl 2722300373Smav KASSERT(!(db && pci), ("bogus")); 2723250079Scarl 2724300373Smav if (db) { 2725300373Smav KASSERT(sz == NTB_REG_64, ("bogus")); 2726300373Smav umv = db_ioread(ntb, reg); 2727300373Smav outsz = sizeof(uint64_t); 2728300373Smav } else { 2729300373Smav switch (sz) { 2730300373Smav case NTB_REG_64: 2731300373Smav if (pci) 2732300373Smav umv = pci_read_config(ntb->device, reg, 8); 2733300373Smav else 2734300373Smav umv = ntb_reg_read(8, reg); 2735300373Smav outsz = sizeof(uint64_t); 2736300373Smav break; 2737300373Smav case NTB_REG_32: 2738300373Smav if (pci) 2739300373Smav umv = pci_read_config(ntb->device, reg, 4); 2740300373Smav else 2741300373Smav umv = ntb_reg_read(4, reg); 2742300373Smav outsz = sizeof(uint32_t); 2743300373Smav break; 2744300373Smav case NTB_REG_16: 2745300373Smav if (pci) 2746300373Smav umv = pci_read_config(ntb->device, reg, 2); 2747300373Smav else 2748300373Smav umv = ntb_reg_read(2, reg); 2749300373Smav outsz = sizeof(uint16_t); 2750300373Smav break; 2751300373Smav case NTB_REG_8: 2752300373Smav if (pci) 2753300373Smav umv = pci_read_config(ntb->device, reg, 1); 2754300373Smav else 2755300373Smav umv = ntb_reg_read(1, reg); 2756300373Smav outsz = sizeof(uint8_t); 2757300373Smav break; 2758300373Smav default: 2759300373Smav panic("bogus"); 2760300373Smav break; 2761300373Smav } 2762300373Smav } 2763300373Smav 2764300373Smav /* Encode bigendian so that sysctl -x is legible. */ 2765300373Smav be64enc(be, umv); 2766300373Smav outp = ((char *)be) + sizeof(umv) - outsz; 2767300373Smav 2768300373Smav error = SYSCTL_OUT(req, outp, outsz); 2769300373Smav if (error || !req->newptr) 2770300373Smav return (error); 2771300373Smav return (EINVAL); 2772250079Scarl} 2773250079Scarl 2774300373Smavstatic unsigned 2775300373Smavntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2776300373Smav{ 2777300373Smav 2778301811Sngie if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2779301811Sngie uidx >= ntb->b2b_mw_idx) || 2780301811Sngie (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2781301811Sngie uidx++; 2782301811Sngie if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2783301811Sngie uidx >= ntb->b2b_mw_idx) && 2784301811Sngie (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2785301811Sngie uidx++; 2786300373Smav return (uidx); 2787300373Smav} 2788300373Smav 2789301811Sngiestatic void 2790301811Sngientb_exchange_msix(void *ctx) 2791301811Sngie{ 2792301811Sngie struct ntb_softc *ntb; 2793301811Sngie uint32_t val; 2794301811Sngie unsigned i; 2795301811Sngie 2796301811Sngie ntb = ctx; 2797301811Sngie 2798301903Smav if (ntb->peer_msix_good) 2799301903Smav goto msix_good; 2800301811Sngie if (ntb->peer_msix_done) 2801301811Sngie goto msix_done; 2802301811Sngie 2803301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2804301811Sngie ntb_peer_spad_write(ntb, NTB_MSIX_DATA0 + i, 2805301811Sngie ntb->msix_data[i].nmd_data); 2806301811Sngie ntb_peer_spad_write(ntb, NTB_MSIX_OFS0 + i, 2807301904Smav ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2808301811Sngie } 2809301811Sngie ntb_peer_spad_write(ntb, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2810301811Sngie 2811301811Sngie ntb_spad_read(ntb, NTB_MSIX_GUARD, &val); 2812301811Sngie if (val != NTB_MSIX_VER_GUARD) 2813301811Sngie goto reschedule; 2814301811Sngie 2815301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2816301811Sngie ntb_spad_read(ntb, NTB_MSIX_DATA0 + i, &val); 2817301904Smav ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2818301811Sngie ntb->peer_msix_data[i].nmd_data = val; 2819301811Sngie ntb_spad_read(ntb, NTB_MSIX_OFS0 + i, &val); 2820301904Smav ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2821301811Sngie ntb->peer_msix_data[i].nmd_ofs = val; 2822301811Sngie } 2823301811Sngie 2824301811Sngie ntb->peer_msix_done = true; 2825301811Sngie 2826301811Sngiemsix_done: 2827301811Sngie ntb_peer_spad_write(ntb, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2828301811Sngie ntb_spad_read(ntb, NTB_MSIX_DONE, &val); 2829301811Sngie if (val != NTB_MSIX_RECEIVED) 2830301811Sngie goto reschedule; 2831301811Sngie 2832301811Sngie ntb->peer_msix_good = true; 2833301903Smav /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2834301903Smav goto reschedule; 2835301811Sngie 2836301903Smavmsix_good: 2837301811Sngie ntb_poll_link(ntb); 2838301811Sngie ntb_link_event(ntb); 2839301811Sngie return; 2840301811Sngie 2841301811Sngiereschedule: 2842301811Sngie ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2843301903Smav if (_xeon_link_is_up(ntb)) { 2844301903Smav callout_reset(&ntb->peer_msix_work, 2845301903Smav hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2846301903Smav ntb_exchange_msix, ntb); 2847301903Smav } else 2848301811Sngie ntb_spad_clear(ntb); 2849301811Sngie} 2850301811Sngie 2851300373Smav/* 2852300373Smav * Public API to the rest of the OS 2853300373Smav */ 2854300373Smav 2855250079Scarl/** 2856250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 2857250079Scarl * @ntb: pointer to ntb_softc instance 2858250079Scarl * 2859250079Scarl * This function returns the max 32bit scratchpad registers usable by the 2860250079Scarl * upper layer. 2861250079Scarl * 2862250079Scarl * RETURNS: total number of scratch pad registers available 2863250079Scarl */ 2864300373Smavuint8_t 2865250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 2866250079Scarl{ 2867250079Scarl 2868300373Smav return (ntb->spad_count); 2869250079Scarl} 2870250079Scarl 2871300373Smav/* 2872300373Smav * ntb_mw_count() - Get the number of memory windows available for KPI 2873300373Smav * consumers. 2874300373Smav * 2875300373Smav * (Excludes any MW wholly reserved for register access.) 2876300373Smav */ 2877300373Smavuint8_t 2878300373Smavntb_mw_count(struct ntb_softc *ntb) 2879300373Smav{ 2880301811Sngie uint8_t res; 2881300373Smav 2882301811Sngie res = ntb->mw_count; 2883300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2884301811Sngie res--; 2885301811Sngie if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2886301811Sngie res--; 2887301811Sngie return (res); 2888300373Smav} 2889300373Smav 2890250079Scarl/** 2891300373Smav * ntb_spad_write() - write to the secondary scratchpad register 2892250079Scarl * @ntb: pointer to ntb_softc instance 2893250079Scarl * @idx: index to the scratchpad register, 0 based 2894250079Scarl * @val: the data value to put into the register 2895250079Scarl * 2896250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2897250079Scarl * register. The register resides on the secondary (external) side. 2898250079Scarl * 2899300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2900250079Scarl */ 2901250079Scarlint 2902300373Smavntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2903250079Scarl{ 2904250079Scarl 2905300373Smav if (idx >= ntb->spad_count) 2906250079Scarl return (EINVAL); 2907250079Scarl 2908300373Smav ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2909250079Scarl 2910250079Scarl return (0); 2911250079Scarl} 2912250079Scarl 2913301811Sngie/* 2914301811Sngie * Zeros the local scratchpad. 2915301811Sngie */ 2916301811Sngievoid 2917301811Sngientb_spad_clear(struct ntb_softc *ntb) 2918301811Sngie{ 2919301811Sngie unsigned i; 2920301811Sngie 2921301811Sngie for (i = 0; i < ntb->spad_count; i++) 2922301811Sngie ntb_spad_write(ntb, i, 0); 2923301811Sngie} 2924301811Sngie 2925250079Scarl/** 2926300373Smav * ntb_spad_read() - read from the primary scratchpad register 2927250079Scarl * @ntb: pointer to ntb_softc instance 2928250079Scarl * @idx: index to scratchpad register, 0 based 2929250079Scarl * @val: pointer to 32bit integer for storing the register value 2930250079Scarl * 2931250079Scarl * This function allows reading of the 32bit scratchpad register on 2932250079Scarl * the primary (internal) side. 2933250079Scarl * 2934300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2935250079Scarl */ 2936250079Scarlint 2937300373Smavntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2938250079Scarl{ 2939250079Scarl 2940300373Smav if (idx >= ntb->spad_count) 2941250079Scarl return (EINVAL); 2942250079Scarl 2943300373Smav *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2944250079Scarl 2945250079Scarl return (0); 2946250079Scarl} 2947250079Scarl 2948250079Scarl/** 2949300373Smav * ntb_peer_spad_write() - write to the secondary scratchpad register 2950250079Scarl * @ntb: pointer to ntb_softc instance 2951250079Scarl * @idx: index to the scratchpad register, 0 based 2952250079Scarl * @val: the data value to put into the register 2953250079Scarl * 2954250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2955250079Scarl * register. The register resides on the secondary (external) side. 2956250079Scarl * 2957300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2958250079Scarl */ 2959250079Scarlint 2960300373Smavntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2961250079Scarl{ 2962250079Scarl 2963300373Smav if (idx >= ntb->spad_count) 2964250079Scarl return (EINVAL); 2965250079Scarl 2966300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2967300373Smav ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2968255279Scarl else 2969300373Smav ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2970250079Scarl 2971250079Scarl return (0); 2972250079Scarl} 2973250079Scarl 2974250079Scarl/** 2975300373Smav * ntb_peer_spad_read() - read from the primary scratchpad register 2976250079Scarl * @ntb: pointer to ntb_softc instance 2977250079Scarl * @idx: index to scratchpad register, 0 based 2978250079Scarl * @val: pointer to 32bit integer for storing the register value 2979250079Scarl * 2980250079Scarl * This function allows reading of the 32bit scratchpad register on 2981250079Scarl * the primary (internal) side. 2982250079Scarl * 2983300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2984250079Scarl */ 2985250079Scarlint 2986300373Smavntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2987250079Scarl{ 2988250079Scarl 2989300373Smav if (idx >= ntb->spad_count) 2990250079Scarl return (EINVAL); 2991250079Scarl 2992300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2993300373Smav *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2994255279Scarl else 2995300373Smav *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2996250079Scarl 2997250079Scarl return (0); 2998250079Scarl} 2999250079Scarl 3000300373Smav/* 3001300373Smav * ntb_mw_get_range() - get the range of a memory window 3002300373Smav * @ntb: NTB device context 3003300373Smav * @idx: Memory window number 3004300373Smav * @base: OUT - the base address for mapping the memory window 3005300373Smav * @size: OUT - the size for mapping the memory window 3006300373Smav * @align: OUT - the base alignment for translating the memory window 3007300373Smav * @align_size: OUT - the size alignment for translating the memory window 3008250079Scarl * 3009300373Smav * Get the range of a memory window. NULL may be given for any output 3010300373Smav * parameter if the value is not needed. The base and size may be used for 3011300373Smav * mapping the memory window, to access the peer memory. The alignment and 3012300373Smav * size may be used for translating the memory window, for the peer to access 3013300373Smav * memory on the local system. 3014250079Scarl * 3015300373Smav * Return: Zero on success, otherwise an error number. 3016250079Scarl */ 3017300373Smavint 3018300373Smavntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 3019300373Smav caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 3020300373Smav bus_addr_t *plimit) 3021250079Scarl{ 3022300373Smav struct ntb_pci_bar_info *bar; 3023300373Smav bus_addr_t limit; 3024300373Smav size_t bar_b2b_off; 3025300373Smav enum ntb_bar bar_num; 3026250079Scarl 3027300373Smav if (mw_idx >= ntb_mw_count(ntb)) 3028300373Smav return (EINVAL); 3029300373Smav mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 3030250079Scarl 3031300373Smav bar_num = ntb_mw_to_bar(ntb, mw_idx); 3032300373Smav bar = &ntb->bar_info[bar_num]; 3033300373Smav bar_b2b_off = 0; 3034300373Smav if (mw_idx == ntb->b2b_mw_idx) { 3035300373Smav KASSERT(ntb->b2b_off != 0, 3036300373Smav ("user shouldn't get non-shared b2b mw")); 3037300373Smav bar_b2b_off = ntb->b2b_off; 3038300373Smav } 3039300373Smav 3040300373Smav if (bar_is_64bit(ntb, bar_num)) 3041300373Smav limit = BUS_SPACE_MAXADDR; 3042300373Smav else 3043300373Smav limit = BUS_SPACE_MAXADDR_32BIT; 3044300373Smav 3045300373Smav if (base != NULL) 3046300373Smav *base = bar->pbase + bar_b2b_off; 3047300373Smav if (vbase != NULL) 3048300373Smav *vbase = bar->vbase + bar_b2b_off; 3049300373Smav if (size != NULL) 3050300373Smav *size = bar->size - bar_b2b_off; 3051300373Smav if (align != NULL) 3052300373Smav *align = bar->size; 3053300373Smav if (align_size != NULL) 3054300373Smav *align_size = 1; 3055300373Smav if (plimit != NULL) 3056300373Smav *plimit = limit; 3057300373Smav return (0); 3058250079Scarl} 3059250079Scarl 3060300373Smav/* 3061300373Smav * ntb_mw_set_trans() - set the translation of a memory window 3062300373Smav * @ntb: NTB device context 3063300373Smav * @idx: Memory window number 3064300373Smav * @addr: The dma address local memory to expose to the peer 3065300373Smav * @size: The size of the local memory to expose to the peer 3066300373Smav * 3067300373Smav * Set the translation of a memory window. The peer may access local memory 3068300373Smav * through the window starting at the address, up to the size. The address 3069300373Smav * must be aligned to the alignment specified by ntb_mw_get_range(). The size 3070300373Smav * must be aligned to the size alignment specified by ntb_mw_get_range(). The 3071300373Smav * address must be below the plimit specified by ntb_mw_get_range() (i.e. for 3072300373Smav * 32-bit BARs). 3073300373Smav * 3074300373Smav * Return: Zero on success, otherwise an error number. 3075300373Smav */ 3076300373Smavint 3077300373Smavntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 3078300373Smav size_t size) 3079250079Scarl{ 3080300373Smav struct ntb_pci_bar_info *bar; 3081300373Smav uint64_t base, limit, reg_val; 3082300373Smav size_t bar_size, mw_size; 3083300373Smav uint32_t base_reg, xlat_reg, limit_reg; 3084300373Smav enum ntb_bar bar_num; 3085250079Scarl 3086300373Smav if (idx >= ntb_mw_count(ntb)) 3087300373Smav return (EINVAL); 3088300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 3089250079Scarl 3090300373Smav bar_num = ntb_mw_to_bar(ntb, idx); 3091300373Smav bar = &ntb->bar_info[bar_num]; 3092300373Smav 3093300373Smav bar_size = bar->size; 3094300373Smav if (idx == ntb->b2b_mw_idx) 3095300373Smav mw_size = bar_size - ntb->b2b_off; 3096300373Smav else 3097300373Smav mw_size = bar_size; 3098300373Smav 3099300373Smav /* Hardware requires that addr is aligned to bar size */ 3100300373Smav if ((addr & (bar_size - 1)) != 0) 3101300373Smav return (EINVAL); 3102300373Smav 3103300373Smav if (size > mw_size) 3104300373Smav return (EINVAL); 3105300373Smav 3106300373Smav bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 3107300373Smav 3108300373Smav limit = 0; 3109300373Smav if (bar_is_64bit(ntb, bar_num)) { 3110300373Smav base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 3111300373Smav 3112300373Smav if (limit_reg != 0 && size != mw_size) 3113300373Smav limit = base + size; 3114300373Smav 3115300373Smav /* Set and verify translation address */ 3116300373Smav ntb_reg_write(8, xlat_reg, addr); 3117300373Smav reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 3118300373Smav if (reg_val != addr) { 3119300373Smav ntb_reg_write(8, xlat_reg, 0); 3120300373Smav return (EIO); 3121300373Smav } 3122300373Smav 3123300373Smav /* Set and verify the limit */ 3124300373Smav ntb_reg_write(8, limit_reg, limit); 3125300373Smav reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3126300373Smav if (reg_val != limit) { 3127300373Smav ntb_reg_write(8, limit_reg, base); 3128300373Smav ntb_reg_write(8, xlat_reg, 0); 3129300373Smav return (EIO); 3130300373Smav } 3131300373Smav } else { 3132300373Smav /* Configure 32-bit (split) BAR MW */ 3133300373Smav 3134300373Smav if ((addr & UINT32_MAX) != addr) 3135300373Smav return (ERANGE); 3136300373Smav if (((addr + size) & UINT32_MAX) != (addr + size)) 3137300373Smav return (ERANGE); 3138300373Smav 3139300373Smav base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3140300373Smav 3141300373Smav if (limit_reg != 0 && size != mw_size) 3142300373Smav limit = base + size; 3143300373Smav 3144300373Smav /* Set and verify translation address */ 3145300373Smav ntb_reg_write(4, xlat_reg, addr); 3146300373Smav reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3147300373Smav if (reg_val != addr) { 3148300373Smav ntb_reg_write(4, xlat_reg, 0); 3149300373Smav return (EIO); 3150300373Smav } 3151300373Smav 3152300373Smav /* Set and verify the limit */ 3153300373Smav ntb_reg_write(4, limit_reg, limit); 3154300373Smav reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3155300373Smav if (reg_val != limit) { 3156300373Smav ntb_reg_write(4, limit_reg, base); 3157300373Smav ntb_reg_write(4, xlat_reg, 0); 3158300373Smav return (EIO); 3159300373Smav } 3160300373Smav } 3161300373Smav return (0); 3162250079Scarl} 3163250079Scarl 3164300373Smav/* 3165300373Smav * ntb_mw_clear_trans() - clear the translation of a memory window 3166300373Smav * @ntb: NTB device context 3167300373Smav * @idx: Memory window number 3168250079Scarl * 3169300373Smav * Clear the translation of a memory window. The peer may no longer access 3170300373Smav * local memory through the window. 3171250079Scarl * 3172300373Smav * Return: Zero on success, otherwise an error number. 3173250079Scarl */ 3174300373Smavint 3175300373Smavntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 3176250079Scarl{ 3177250079Scarl 3178300373Smav return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 3179300373Smav} 3180300373Smav 3181300373Smav/* 3182300373Smav * ntb_mw_get_wc - Get the write-combine status of a memory window 3183300373Smav * 3184300373Smav * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 3185300373Smav * idx is an invalid memory window). 3186300373Smav * 3187300373Smav * Mode is a VM_MEMATTR_* type. 3188300373Smav */ 3189300373Smavint 3190300373Smavntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t *mode) 3191300373Smav{ 3192300373Smav struct ntb_pci_bar_info *bar; 3193300373Smav 3194300373Smav if (idx >= ntb_mw_count(ntb)) 3195300373Smav return (EINVAL); 3196300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 3197300373Smav 3198300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3199300373Smav *mode = bar->map_mode; 3200300373Smav return (0); 3201300373Smav} 3202300373Smav 3203300373Smav/* 3204300373Smav * ntb_mw_set_wc - Set the write-combine status of a memory window 3205300373Smav * 3206300373Smav * If 'mode' matches the current status, this does nothing and succeeds. Mode 3207300373Smav * is a VM_MEMATTR_* type. 3208300373Smav * 3209300373Smav * Returns: Zero on success, setting the caching attribute on the virtual 3210300373Smav * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 3211300373Smav * memory window, or if changing the caching attribute fails). 3212300373Smav */ 3213300373Smavint 3214300373Smavntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3215300373Smav{ 3216300373Smav 3217300373Smav if (idx >= ntb_mw_count(ntb)) 3218300373Smav return (EINVAL); 3219300373Smav 3220300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 3221300373Smav return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3222300373Smav} 3223300373Smav 3224300373Smavstatic int 3225300373Smavntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3226300373Smav{ 3227300373Smav struct ntb_pci_bar_info *bar; 3228300373Smav int rc; 3229300373Smav 3230300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3231300373Smav if (bar->map_mode == mode) 3232250079Scarl return (0); 3233250079Scarl 3234300373Smav rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3235300373Smav if (rc == 0) 3236300373Smav bar->map_mode = mode; 3237300373Smav 3238300373Smav return (rc); 3239250079Scarl} 3240250079Scarl 3241250079Scarl/** 3242300373Smav * ntb_peer_db_set() - Set the doorbell on the secondary/external side 3243250079Scarl * @ntb: pointer to ntb_softc instance 3244300373Smav * @bit: doorbell bits to ring 3245250079Scarl * 3246300373Smav * This function allows triggering of a doorbell on the secondary/external 3247300373Smav * side that will initiate an interrupt on the remote host 3248250079Scarl */ 3249250079Scarlvoid 3250300373Smavntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 3251250079Scarl{ 3252250079Scarl 3253301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 3254301811Sngie struct ntb_pci_bar_info *lapic; 3255301811Sngie unsigned i; 3256301811Sngie 3257301811Sngie lapic = ntb->peer_lapic_bar; 3258301811Sngie 3259301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3260301811Sngie if ((bit & ntb_db_vector_mask(ntb, i)) != 0) 3261301811Sngie bus_space_write_4(lapic->pci_bus_tag, 3262301811Sngie lapic->pci_bus_handle, 3263301811Sngie ntb->peer_msix_data[i].nmd_ofs, 3264301811Sngie ntb->peer_msix_data[i].nmd_data); 3265301811Sngie } 3266301811Sngie return; 3267301811Sngie } 3268301811Sngie 3269300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3270300373Smav ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3271250079Scarl return; 3272300373Smav } 3273250079Scarl 3274300373Smav db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3275300373Smav} 3276300373Smav 3277300373Smav/* 3278300373Smav * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 3279300373Smav * as well as the size of the register (via *sz_out). 3280300373Smav * 3281300373Smav * This function allows a caller using I/OAT DMA to chain the remote doorbell 3282300373Smav * ring to its memory window write. 3283300373Smav * 3284300373Smav * Note that writing the peer doorbell via a memory window will *not* generate 3285300373Smav * an interrupt on the remote host; that must be done seperately. 3286300373Smav */ 3287300373Smavbus_addr_t 3288300373Smavntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 3289300373Smav{ 3290300373Smav struct ntb_pci_bar_info *bar; 3291300373Smav uint64_t regoff; 3292300373Smav 3293300373Smav KASSERT(sz_out != NULL, ("must be non-NULL")); 3294300373Smav 3295300373Smav if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3296300373Smav bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3297300373Smav regoff = ntb->peer_reg->db_bell; 3298300373Smav } else { 3299300373Smav KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3300300373Smav ("invalid b2b idx")); 3301300373Smav 3302300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3303300373Smav regoff = XEON_PDOORBELL_OFFSET; 3304250079Scarl } 3305300373Smav KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3306300373Smav 3307300373Smav *sz_out = ntb->reg->db_size; 3308300373Smav /* HACK: Specific to current x86 bus implementation. */ 3309300373Smav return ((uint64_t)bar->pci_bus_handle + regoff); 3310250079Scarl} 3311250079Scarl 3312300373Smav/* 3313300373Smav * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 3314300373Smav * @ntb: NTB device context 3315250079Scarl * 3316300373Smav * Hardware may support different number or arrangement of doorbell bits. 3317250079Scarl * 3318300373Smav * Return: A mask of doorbell bits supported by the ntb. 3319250079Scarl */ 3320300373Smavuint64_t 3321300373Smavntb_db_valid_mask(struct ntb_softc *ntb) 3322250079Scarl{ 3323250079Scarl 3324300373Smav return (ntb->db_valid_mask); 3325250079Scarl} 3326250079Scarl 3327300373Smav/* 3328300373Smav * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 3329300373Smav * @ntb: NTB device context 3330300373Smav * @vector: Doorbell vector number 3331300373Smav * 3332300373Smav * Each interrupt vector may have a different number or arrangement of bits. 3333300373Smav * 3334300373Smav * Return: A mask of doorbell bits serviced by a vector. 3335300373Smav */ 3336300373Smavuint64_t 3337300373Smavntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 3338300373Smav{ 3339300373Smav 3340300373Smav if (vector > ntb->db_vec_count) 3341300373Smav return (0); 3342300373Smav return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3343300373Smav} 3344300373Smav 3345250079Scarl/** 3346300373Smav * ntb_link_is_up() - get the current ntb link state 3347300373Smav * @ntb: NTB device context 3348300373Smav * @speed: OUT - The link speed expressed as PCIe generation number 3349300373Smav * @width: OUT - The link width expressed as the number of PCIe lanes 3350250079Scarl * 3351250079Scarl * RETURNS: true or false based on the hardware link state 3352250079Scarl */ 3353250079Scarlbool 3354300373Smavntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 3355300373Smav enum ntb_width *width) 3356250079Scarl{ 3357250079Scarl 3358300373Smav if (speed != NULL) 3359300373Smav *speed = ntb_link_sta_speed(ntb); 3360300373Smav if (width != NULL) 3361300373Smav *width = ntb_link_sta_width(ntb); 3362300373Smav return (link_is_up(ntb)); 3363250079Scarl} 3364250079Scarl 3365255272Scarlstatic void 3366255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 3367250079Scarl{ 3368255272Scarl 3369300373Smav bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3370300373Smav bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3371300373Smav bar->pbase = rman_get_start(bar->pci_resource); 3372300373Smav bar->size = rman_get_size(bar->pci_resource); 3373300373Smav bar->vbase = rman_get_virtual(bar->pci_resource); 3374250079Scarl} 3375255268Scarl 3376300373Smavdevice_t 3377300373Smavntb_get_device(struct ntb_softc *ntb) 3378255268Scarl{ 3379255268Scarl 3380255268Scarl return (ntb->device); 3381255268Scarl} 3382300373Smav 3383300373Smav/* Export HW-specific errata information. */ 3384300373Smavbool 3385300373Smavntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 3386300373Smav{ 3387300373Smav 3388300373Smav return (HAS_FEATURE(feature)); 3389300373Smav} 3390