ntb_hw.c revision 304377
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3300373Smav * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c 304377 2016-08-18 10:25:07Z mav $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35300373Smav#include <sys/endian.h> 36250079Scarl#include <sys/malloc.h> 37250079Scarl#include <sys/module.h> 38301811Sngie#include <sys/mutex.h> 39301811Sngie#include <sys/pciio.h> 40250079Scarl#include <sys/queue.h> 41250079Scarl#include <sys/rman.h> 42300373Smav#include <sys/sbuf.h> 43300373Smav#include <sys/sysctl.h> 44250079Scarl#include <vm/vm.h> 45250079Scarl#include <vm/pmap.h> 46250079Scarl#include <machine/bus.h> 47301811Sngie#include <machine/intr_machdep.h> 48250079Scarl#include <machine/pmap.h> 49250079Scarl#include <machine/resource.h> 50250079Scarl#include <dev/pci/pcireg.h> 51250079Scarl#include <dev/pci/pcivar.h> 52250079Scarl 53250079Scarl#include "ntb_regs.h" 54250079Scarl#include "ntb_hw.h" 55250079Scarl 56250079Scarl/* 57250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 58250079Scarl * allows you to connect two systems using a PCI-e link. 59250079Scarl * 60250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 61250079Scarl * you to send and recieve interrupts, map the memory windows and send and 62250079Scarl * receive messages in the scratch-pad registers. 63250079Scarl * 64250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 65250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 66250079Scarl */ 67250079Scarl 68300373Smav#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 69250079Scarl 70300373Smav#define NTB_HB_TIMEOUT 1 /* second */ 71300373Smav#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 72300373Smav#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 73250079Scarl 74250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 75250079Scarl 76301811Sngie#define NTB_MSIX_VER_GUARD 0xaabbccdd 77301811Sngie#define NTB_MSIX_RECEIVED 0xe0f0e0f0 78301811Sngie 79301811Sngie/* 80301811Sngie * PCI constants could be somewhere more generic, but aren't defined/used in 81301811Sngie * pci.c. 82301811Sngie */ 83301811Sngie#define PCI_MSIX_ENTRY_SIZE 16 84301811Sngie#define PCI_MSIX_ENTRY_LOWER_ADDR 0 85301811Sngie#define PCI_MSIX_ENTRY_UPPER_ADDR 4 86301811Sngie#define PCI_MSIX_ENTRY_DATA 8 87301811Sngie 88250079Scarlenum ntb_device_type { 89250079Scarl NTB_XEON, 90300373Smav NTB_ATOM 91250079Scarl}; 92250079Scarl 93300373Smav/* ntb_conn_type are hardware numbers, cannot change. */ 94300373Smavenum ntb_conn_type { 95300373Smav NTB_CONN_TRANSPARENT = 0, 96300373Smav NTB_CONN_B2B = 1, 97300373Smav NTB_CONN_RP = 2, 98300373Smav}; 99300373Smav 100300373Smavenum ntb_b2b_direction { 101300373Smav NTB_DEV_USD = 0, 102300373Smav NTB_DEV_DSD = 1, 103300373Smav}; 104300373Smav 105300373Smavenum ntb_bar { 106300373Smav NTB_CONFIG_BAR = 0, 107300373Smav NTB_B2B_BAR_1, 108300373Smav NTB_B2B_BAR_2, 109300373Smav NTB_B2B_BAR_3, 110300373Smav NTB_MAX_BARS 111300373Smav}; 112300373Smav 113301811Sngieenum { 114301811Sngie NTB_MSIX_GUARD = 0, 115301811Sngie NTB_MSIX_DATA0, 116301811Sngie NTB_MSIX_DATA1, 117301811Sngie NTB_MSIX_DATA2, 118301811Sngie NTB_MSIX_OFS0, 119301811Sngie NTB_MSIX_OFS1, 120301811Sngie NTB_MSIX_OFS2, 121301811Sngie NTB_MSIX_DONE, 122301811Sngie NTB_MAX_MSIX_SPAD 123301811Sngie}; 124301811Sngie 125255274Scarl/* Device features and workarounds */ 126255274Scarl#define HAS_FEATURE(feature) \ 127255274Scarl ((ntb->features & (feature)) != 0) 128255274Scarl 129250079Scarlstruct ntb_hw_info { 130250079Scarl uint32_t device_id; 131255274Scarl const char *desc; 132250079Scarl enum ntb_device_type type; 133300373Smav uint32_t features; 134250079Scarl}; 135250079Scarl 136250079Scarlstruct ntb_pci_bar_info { 137250079Scarl bus_space_tag_t pci_bus_tag; 138250079Scarl bus_space_handle_t pci_bus_handle; 139250079Scarl int pci_resource_id; 140250079Scarl struct resource *pci_resource; 141250079Scarl vm_paddr_t pbase; 142300373Smav caddr_t vbase; 143300373Smav vm_size_t size; 144300373Smav vm_memattr_t map_mode; 145300373Smav 146300373Smav /* Configuration register offsets */ 147300373Smav uint32_t psz_off; 148300373Smav uint32_t ssz_off; 149300373Smav uint32_t pbarxlat_off; 150250079Scarl}; 151250079Scarl 152250079Scarlstruct ntb_int_info { 153250079Scarl struct resource *res; 154250079Scarl int rid; 155250079Scarl void *tag; 156250079Scarl}; 157250079Scarl 158300373Smavstruct ntb_vec { 159250079Scarl struct ntb_softc *ntb; 160300373Smav uint32_t num; 161301811Sngie unsigned masked; 162250079Scarl}; 163250079Scarl 164300373Smavstruct ntb_reg { 165300373Smav uint32_t ntb_ctl; 166300373Smav uint32_t lnk_sta; 167300373Smav uint8_t db_size; 168300373Smav unsigned mw_bar[NTB_MAX_BARS]; 169300373Smav}; 170300373Smav 171300373Smavstruct ntb_alt_reg { 172300373Smav uint32_t db_bell; 173300373Smav uint32_t db_mask; 174300373Smav uint32_t spad; 175300373Smav}; 176300373Smav 177300373Smavstruct ntb_xlat_reg { 178300373Smav uint32_t bar0_base; 179300373Smav uint32_t bar2_base; 180300373Smav uint32_t bar4_base; 181300373Smav uint32_t bar5_base; 182300373Smav 183300373Smav uint32_t bar2_xlat; 184300373Smav uint32_t bar4_xlat; 185300373Smav uint32_t bar5_xlat; 186300373Smav 187300373Smav uint32_t bar2_limit; 188300373Smav uint32_t bar4_limit; 189300373Smav uint32_t bar5_limit; 190300373Smav}; 191300373Smav 192300373Smavstruct ntb_b2b_addr { 193300373Smav uint64_t bar0_addr; 194300373Smav uint64_t bar2_addr64; 195300373Smav uint64_t bar4_addr64; 196300373Smav uint64_t bar4_addr32; 197300373Smav uint64_t bar5_addr32; 198300373Smav}; 199300373Smav 200301811Sngiestruct ntb_msix_data { 201301811Sngie uint32_t nmd_ofs; 202301811Sngie uint32_t nmd_data; 203301811Sngie}; 204301811Sngie 205250079Scarlstruct ntb_softc { 206250079Scarl device_t device; 207250079Scarl enum ntb_device_type type; 208300373Smav uint32_t features; 209250079Scarl 210250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 211250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 212250079Scarl uint32_t allocated_interrupts; 213250079Scarl 214301811Sngie struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215301811Sngie struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216301811Sngie bool peer_msix_good; 217301811Sngie bool peer_msix_done; 218301811Sngie struct ntb_pci_bar_info *peer_lapic_bar; 219301811Sngie struct callout peer_msix_work; 220301811Sngie 221250079Scarl struct callout heartbeat_timer; 222250079Scarl struct callout lr_timer; 223250079Scarl 224300373Smav void *ntb_ctx; 225300373Smav const struct ntb_ctx_ops *ctx_ops; 226300373Smav struct ntb_vec *msix_vec; 227300373Smav#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 228300373Smav#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 229300373Smav#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 230300373Smav struct mtx ctx_lock; 231250079Scarl 232300373Smav uint32_t ppd; 233300373Smav enum ntb_conn_type conn_type; 234300373Smav enum ntb_b2b_direction dev_type; 235300373Smav 236300373Smav /* Offset of peer bar0 in B2B BAR */ 237300373Smav uint64_t b2b_off; 238300373Smav /* Memory window used to access peer bar0 */ 239300373Smav#define B2B_MW_DISABLED UINT8_MAX 240300373Smav uint8_t b2b_mw_idx; 241301904Smav uint32_t msix_xlat; 242301811Sngie uint8_t msix_mw_idx; 243300373Smav 244300373Smav uint8_t mw_count; 245300373Smav uint8_t spad_count; 246300373Smav uint8_t db_count; 247300373Smav uint8_t db_vec_count; 248300373Smav uint8_t db_vec_shift; 249300373Smav 250300373Smav /* Protects local db_mask. */ 251300373Smav#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 252300373Smav#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 253300373Smav#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 254300373Smav struct mtx db_mask_lock; 255300373Smav 256300373Smav volatile uint32_t ntb_ctl; 257300373Smav volatile uint32_t lnk_sta; 258300373Smav 259300373Smav uint64_t db_valid_mask; 260300373Smav uint64_t db_link_mask; 261300373Smav uint64_t db_mask; 262300373Smav 263300373Smav int last_ts; /* ticks @ last irq */ 264300373Smav 265300373Smav const struct ntb_reg *reg; 266300373Smav const struct ntb_alt_reg *self_reg; 267300373Smav const struct ntb_alt_reg *peer_reg; 268300373Smav const struct ntb_xlat_reg *xlat_reg; 269250079Scarl}; 270250079Scarl 271300373Smav#ifdef __i386__ 272300373Smavstatic __inline uint64_t 273300373Smavbus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 274300373Smav bus_size_t offset) 275300373Smav{ 276300373Smav 277300373Smav return (bus_space_read_4(tag, handle, offset) | 278300373Smav ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 279300373Smav} 280300373Smav 281300373Smavstatic __inline void 282300373Smavbus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 283300373Smav bus_size_t offset, uint64_t val) 284300373Smav{ 285300373Smav 286300373Smav bus_space_write_4(tag, handle, offset, val); 287300373Smav bus_space_write_4(tag, handle, offset + 4, val >> 32); 288300373Smav} 289300373Smav#endif 290300373Smav 291255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 292255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 293255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 294255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 295255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 296255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 297255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 298250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 299255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 300300373Smav#define ntb_mw_read(SIZE, offset) \ 301300373Smav ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 302255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 303300373Smav ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 304300373Smav offset, val) 305250079Scarl 306250079Scarlstatic int ntb_probe(device_t device); 307250079Scarlstatic int ntb_attach(device_t device); 308250079Scarlstatic int ntb_detach(device_t device); 309300373Smavstatic unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 310300373Smavstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 311300373Smavstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 312300373Smavstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 313300373Smav uint32_t *base, uint32_t *xlat, uint32_t *lmt); 314255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 315300373Smavstatic int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 316300373Smav vm_memattr_t); 317300373Smavstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 318300373Smav const char *); 319255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 320255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 321255272Scarl struct ntb_pci_bar_info *bar); 322250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 323300373Smavstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 324300373Smavstatic int ntb_init_isr(struct ntb_softc *ntb); 325300373Smavstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 326300373Smavstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 327250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 328300373Smavstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 329300373Smavstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 330300373Smavstatic void ndev_vec_isr(void *arg); 331300373Smavstatic void ndev_irq_isr(void *arg); 332300373Smavstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 333300373Smavstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 334300373Smavstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 335300373Smavstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 336300373Smavstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 337301811Sngiestatic void ntb_get_msix_info(struct ntb_softc *ntb); 338301811Sngiestatic void ntb_exchange_msix(void *); 339250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 340300373Smavstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 341300373Smavstatic int ntb_detect_xeon(struct ntb_softc *ntb); 342300373Smavstatic int ntb_detect_atom(struct ntb_softc *ntb); 343300373Smavstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 344300373Smavstatic int ntb_atom_init_dev(struct ntb_softc *ntb); 345300373Smavstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 346300373Smavstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 347300373Smavstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 348300373Smav enum ntb_bar regbar); 349300373Smavstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 350300373Smav uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 351300373Smavstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 352300373Smav enum ntb_bar idx); 353300373Smavstatic int xeon_setup_b2b_mw(struct ntb_softc *, 354300373Smav const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 355301811Sngiestatic int xeon_setup_msix_bar(struct ntb_softc *); 356300373Smavstatic inline bool link_is_up(struct ntb_softc *ntb); 357301811Sngiestatic inline bool _xeon_link_is_up(struct ntb_softc *ntb); 358300373Smavstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 359300373Smavstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 360300373Smavstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 361300373Smavstatic void atom_link_hb(void *arg); 362300373Smavstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 363300373Smavstatic void recover_atom_link(void *arg); 364300373Smavstatic bool ntb_poll_link(struct ntb_softc *ntb); 365255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 366300373Smavstatic void ntb_sysctl_init(struct ntb_softc *); 367300373Smavstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 368301811Sngiestatic int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 369301811Sngiestatic int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 370300373Smavstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 371300373Smavstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 372250079Scarl 373300373Smavstatic unsigned g_ntb_hw_debug_level; 374300516SmavTUNABLE_INT("hw.ntb.debug_level", &g_ntb_hw_debug_level); 375300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 376300373Smav &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 377300373Smav#define ntb_printf(lvl, ...) do { \ 378300373Smav if ((lvl) <= g_ntb_hw_debug_level) { \ 379300373Smav device_printf(ntb->device, __VA_ARGS__); \ 380300373Smav } \ 381300373Smav} while (0) 382300373Smav 383300373Smav#define _NTB_PAT_UC 0 384300373Smav#define _NTB_PAT_WC 1 385300373Smav#define _NTB_PAT_WT 4 386300373Smav#define _NTB_PAT_WP 5 387300373Smav#define _NTB_PAT_WB 6 388300373Smav#define _NTB_PAT_UCM 7 389300373Smavstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 390300516SmavTUNABLE_INT("hw.ntb.default_mw_pat", &g_ntb_mw_pat); 391300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 392300373Smav &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 393300373Smav "UC: " __XSTRING(_NTB_PAT_UC) ", " 394300373Smav "WC: " __XSTRING(_NTB_PAT_WC) ", " 395300373Smav "WT: " __XSTRING(_NTB_PAT_WT) ", " 396300373Smav "WP: " __XSTRING(_NTB_PAT_WP) ", " 397300373Smav "WB: " __XSTRING(_NTB_PAT_WB) ", " 398300373Smav "UC-: " __XSTRING(_NTB_PAT_UCM)); 399300373Smav 400300373Smavstatic inline vm_memattr_t 401300373Smavntb_pat_flags(void) 402300373Smav{ 403300373Smav 404300373Smav switch (g_ntb_mw_pat) { 405300373Smav case _NTB_PAT_WC: 406300373Smav return (VM_MEMATTR_WRITE_COMBINING); 407300373Smav case _NTB_PAT_WT: 408300373Smav return (VM_MEMATTR_WRITE_THROUGH); 409300373Smav case _NTB_PAT_WP: 410300373Smav return (VM_MEMATTR_WRITE_PROTECTED); 411300373Smav case _NTB_PAT_WB: 412300373Smav return (VM_MEMATTR_WRITE_BACK); 413300373Smav case _NTB_PAT_UCM: 414300373Smav return (VM_MEMATTR_WEAK_UNCACHEABLE); 415300373Smav case _NTB_PAT_UC: 416300373Smav /* FALLTHROUGH */ 417300373Smav default: 418300373Smav return (VM_MEMATTR_UNCACHEABLE); 419300373Smav } 420300373Smav} 421300373Smav 422300373Smav/* 423300373Smav * Well, this obviously doesn't belong here, but it doesn't seem to exist 424300373Smav * anywhere better yet. 425300373Smav */ 426300373Smavstatic inline const char * 427300373Smavntb_vm_memattr_to_str(vm_memattr_t pat) 428300373Smav{ 429300373Smav 430300373Smav switch (pat) { 431300373Smav case VM_MEMATTR_WRITE_COMBINING: 432300373Smav return ("WRITE_COMBINING"); 433300373Smav case VM_MEMATTR_WRITE_THROUGH: 434300373Smav return ("WRITE_THROUGH"); 435300373Smav case VM_MEMATTR_WRITE_PROTECTED: 436300373Smav return ("WRITE_PROTECTED"); 437300373Smav case VM_MEMATTR_WRITE_BACK: 438300373Smav return ("WRITE_BACK"); 439300373Smav case VM_MEMATTR_WEAK_UNCACHEABLE: 440300373Smav return ("UNCACHED"); 441300373Smav case VM_MEMATTR_UNCACHEABLE: 442300373Smav return ("UNCACHEABLE"); 443300373Smav default: 444300373Smav return ("UNKNOWN"); 445300373Smav } 446300373Smav} 447300373Smav 448301811Sngiestatic int g_ntb_msix_idx = 0; 449301811SngieSYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 450301811Sngie 0, "Use this memory window to access the peer MSIX message complex on " 451301811Sngie "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 452301811Sngie "Like b2b_mw_idx, negative values index from the last available memory " 453301811Sngie "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 454301811Sngie 455300373Smavstatic int g_ntb_mw_idx = -1; 456300516SmavTUNABLE_INT("hw.ntb.b2b_mw_idx", &g_ntb_mw_idx); 457300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 458300373Smav 0, "Use this memory window to access the peer NTB registers. A " 459300373Smav "non-negative value starts from the first MW index; a negative value " 460300373Smav "starts from the last MW index. The default is -1, i.e., the last " 461300373Smav "available memory window. Both sides of the NTB MUST set the same " 462300373Smav "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 463300373Smav 464250079Scarlstatic struct ntb_hw_info pci_ids[] = { 465300373Smav /* XXX: PS/SS IDs left out until they are supported. */ 466300373Smav { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 467300373Smav NTB_ATOM, 0 }, 468300373Smav 469300373Smav { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 470300373Smav NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 471300373Smav { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 472300373Smav NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 473300373Smav { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 474300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 475300373Smav NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 476300373Smav { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 477300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 478300373Smav NTB_SB01BASE_LOCKUP }, 479300373Smav { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 480300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 481300373Smav NTB_SB01BASE_LOCKUP }, 482300373Smav 483300373Smav { 0x00000000, NULL, NTB_ATOM, 0 } 484250079Scarl}; 485250079Scarl 486300373Smavstatic const struct ntb_reg atom_reg = { 487300373Smav .ntb_ctl = ATOM_NTBCNTL_OFFSET, 488300373Smav .lnk_sta = ATOM_LINK_STATUS_OFFSET, 489300373Smav .db_size = sizeof(uint64_t), 490300373Smav .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 491300373Smav}; 492300373Smav 493300373Smavstatic const struct ntb_alt_reg atom_pri_reg = { 494300373Smav .db_bell = ATOM_PDOORBELL_OFFSET, 495300373Smav .db_mask = ATOM_PDBMSK_OFFSET, 496300373Smav .spad = ATOM_SPAD_OFFSET, 497300373Smav}; 498300373Smav 499300373Smavstatic const struct ntb_alt_reg atom_b2b_reg = { 500300373Smav .db_bell = ATOM_B2B_DOORBELL_OFFSET, 501300373Smav .spad = ATOM_B2B_SPAD_OFFSET, 502300373Smav}; 503300373Smav 504300373Smavstatic const struct ntb_xlat_reg atom_sec_xlat = { 505300373Smav#if 0 506300373Smav /* "FIXME" says the Linux driver. */ 507300373Smav .bar0_base = ATOM_SBAR0BASE_OFFSET, 508300373Smav .bar2_base = ATOM_SBAR2BASE_OFFSET, 509300373Smav .bar4_base = ATOM_SBAR4BASE_OFFSET, 510300373Smav 511300373Smav .bar2_limit = ATOM_SBAR2LMT_OFFSET, 512300373Smav .bar4_limit = ATOM_SBAR4LMT_OFFSET, 513300373Smav#endif 514300373Smav 515300373Smav .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 516300373Smav .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 517300373Smav}; 518300373Smav 519300373Smavstatic const struct ntb_reg xeon_reg = { 520300373Smav .ntb_ctl = XEON_NTBCNTL_OFFSET, 521300373Smav .lnk_sta = XEON_LINK_STATUS_OFFSET, 522300373Smav .db_size = sizeof(uint16_t), 523300373Smav .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 524300373Smav}; 525300373Smav 526300373Smavstatic const struct ntb_alt_reg xeon_pri_reg = { 527300373Smav .db_bell = XEON_PDOORBELL_OFFSET, 528300373Smav .db_mask = XEON_PDBMSK_OFFSET, 529300373Smav .spad = XEON_SPAD_OFFSET, 530300373Smav}; 531300373Smav 532300373Smavstatic const struct ntb_alt_reg xeon_b2b_reg = { 533300373Smav .db_bell = XEON_B2B_DOORBELL_OFFSET, 534300373Smav .spad = XEON_B2B_SPAD_OFFSET, 535300373Smav}; 536300373Smav 537300373Smavstatic const struct ntb_xlat_reg xeon_sec_xlat = { 538300373Smav .bar0_base = XEON_SBAR0BASE_OFFSET, 539300373Smav .bar2_base = XEON_SBAR2BASE_OFFSET, 540300373Smav .bar4_base = XEON_SBAR4BASE_OFFSET, 541300373Smav .bar5_base = XEON_SBAR5BASE_OFFSET, 542300373Smav 543300373Smav .bar2_limit = XEON_SBAR2LMT_OFFSET, 544300373Smav .bar4_limit = XEON_SBAR4LMT_OFFSET, 545300373Smav .bar5_limit = XEON_SBAR5LMT_OFFSET, 546300373Smav 547300373Smav .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 548300373Smav .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 549300373Smav .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 550300373Smav}; 551300373Smav 552300373Smavstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 553300373Smav .bar0_addr = XEON_B2B_BAR0_ADDR, 554300373Smav .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 555300373Smav .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 556300373Smav .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 557300373Smav .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 558300373Smav}; 559300373Smav 560300373Smavstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 561300373Smav .bar0_addr = XEON_B2B_BAR0_ADDR, 562300373Smav .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 563300373Smav .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 564300373Smav .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 565300373Smav .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 566300373Smav}; 567300373Smav 568300373SmavSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 569300373Smav "B2B MW segment overrides -- MUST be the same on both sides"); 570300373Smav 571300516SmavTUNABLE_QUAD("hw.ntb.usd_bar2_addr64", &xeon_b2b_usd_addr.bar2_addr64); 572300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 573300373Smav &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 574300373Smav "hardware, use this 64-bit address on the bus between the NTB devices for " 575300373Smav "the window at BAR2, on the upstream side of the link. MUST be the same " 576300373Smav "address on both sides."); 577300516SmavTUNABLE_QUAD("hw.ntb.usd_bar4_addr64", &xeon_b2b_usd_addr.bar4_addr64); 578300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 579300373Smav &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 580300516SmavTUNABLE_QUAD("hw.ntb.usd_bar4_addr32", &xeon_b2b_usd_addr.bar4_addr32); 581300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 582300373Smav &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 583300373Smav "(split-BAR mode)."); 584300516SmavTUNABLE_QUAD("hw.ntb.usd_bar5_addr32", &xeon_b2b_usd_addr.bar5_addr32); 585300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 586300373Smav &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 587300373Smav "(split-BAR mode)."); 588300373Smav 589300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar2_addr64", &xeon_b2b_dsd_addr.bar2_addr64); 590300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 591300373Smav &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 592300373Smav "hardware, use this 64-bit address on the bus between the NTB devices for " 593300373Smav "the window at BAR2, on the downstream side of the link. MUST be the same" 594300373Smav " address on both sides."); 595300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar4_addr64", &xeon_b2b_dsd_addr.bar4_addr64); 596300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 597300373Smav &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 598300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar4_addr32", &xeon_b2b_dsd_addr.bar4_addr32); 599300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 600300373Smav &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 601300373Smav "(split-BAR mode)."); 602300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar5_addr32", &xeon_b2b_dsd_addr.bar5_addr32); 603300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 604300373Smav &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 605300373Smav "(split-BAR mode)."); 606300373Smav 607250079Scarl/* 608250079Scarl * OS <-> Driver interface structures 609250079Scarl */ 610250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 611250079Scarl 612250079Scarlstatic device_method_t ntb_pci_methods[] = { 613250079Scarl /* Device interface */ 614250079Scarl DEVMETHOD(device_probe, ntb_probe), 615250079Scarl DEVMETHOD(device_attach, ntb_attach), 616250079Scarl DEVMETHOD(device_detach, ntb_detach), 617250079Scarl DEVMETHOD_END 618250079Scarl}; 619250079Scarl 620250079Scarlstatic driver_t ntb_pci_driver = { 621250079Scarl "ntb_hw", 622250079Scarl ntb_pci_methods, 623250079Scarl sizeof(struct ntb_softc), 624250079Scarl}; 625250079Scarl 626250079Scarlstatic devclass_t ntb_devclass; 627250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 628250079ScarlMODULE_VERSION(ntb_hw, 1); 629250079Scarl 630300373SmavSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 631300373Smav 632250079Scarl/* 633250079Scarl * OS <-> Driver linkage functions 634250079Scarl */ 635250079Scarlstatic int 636250079Scarlntb_probe(device_t device) 637250079Scarl{ 638300373Smav struct ntb_hw_info *p; 639250079Scarl 640300373Smav p = ntb_get_device_info(pci_get_devid(device)); 641300373Smav if (p == NULL) 642250079Scarl return (ENXIO); 643300373Smav 644300373Smav device_set_desc(device, p->desc); 645300373Smav return (0); 646250079Scarl} 647250079Scarl 648250079Scarlstatic int 649250079Scarlntb_attach(device_t device) 650250079Scarl{ 651300373Smav struct ntb_softc *ntb; 652300373Smav struct ntb_hw_info *p; 653250079Scarl int error; 654250079Scarl 655300373Smav ntb = DEVICE2SOFTC(device); 656300373Smav p = ntb_get_device_info(pci_get_devid(device)); 657300373Smav 658250079Scarl ntb->device = device; 659250079Scarl ntb->type = p->type; 660255274Scarl ntb->features = p->features; 661300373Smav ntb->b2b_mw_idx = B2B_MW_DISABLED; 662301811Sngie ntb->msix_mw_idx = B2B_MW_DISABLED; 663250079Scarl 664300373Smav /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 665250079Scarl callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE); 666250079Scarl callout_init(&ntb->lr_timer, CALLOUT_MPSAFE); 667301811Sngie callout_init(&ntb->peer_msix_work, 1); 668300373Smav mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 669300373Smav mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 670250079Scarl 671300373Smav if (ntb->type == NTB_ATOM) 672300373Smav error = ntb_detect_atom(ntb); 673300373Smav else 674300373Smav error = ntb_detect_xeon(ntb); 675300373Smav if (error != 0) 676300373Smav goto out; 677250079Scarl 678300373Smav ntb_detect_max_mw(ntb); 679300373Smav 680250079Scarl pci_enable_busmaster(ntb->device); 681250079Scarl 682300373Smav error = ntb_map_pci_bars(ntb); 683300373Smav if (error != 0) 684300373Smav goto out; 685300373Smav if (ntb->type == NTB_ATOM) 686300373Smav error = ntb_atom_init_dev(ntb); 687300373Smav else 688300373Smav error = ntb_xeon_init_dev(ntb); 689300373Smav if (error != 0) 690300373Smav goto out; 691300373Smav 692301811Sngie ntb_spad_clear(ntb); 693301811Sngie 694300373Smav ntb_poll_link(ntb); 695300373Smav 696300373Smav ntb_sysctl_init(ntb); 697300373Smav 698300373Smavout: 699300373Smav if (error != 0) 700300373Smav ntb_detach(device); 701250079Scarl return (error); 702250079Scarl} 703250079Scarl 704250079Scarlstatic int 705250079Scarlntb_detach(device_t device) 706250079Scarl{ 707300373Smav struct ntb_softc *ntb; 708250079Scarl 709300373Smav ntb = DEVICE2SOFTC(device); 710300373Smav 711301811Sngie if (ntb->self_reg != NULL) { 712301811Sngie DB_MASK_LOCK(ntb); 713301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 714301811Sngie DB_MASK_UNLOCK(ntb); 715301811Sngie } 716250079Scarl callout_drain(&ntb->heartbeat_timer); 717250079Scarl callout_drain(&ntb->lr_timer); 718301811Sngie callout_drain(&ntb->peer_msix_work); 719300373Smav pci_disable_busmaster(ntb->device); 720300373Smav if (ntb->type == NTB_XEON) 721300373Smav ntb_teardown_xeon(ntb); 722250079Scarl ntb_teardown_interrupts(ntb); 723300373Smav 724300373Smav mtx_destroy(&ntb->db_mask_lock); 725300373Smav mtx_destroy(&ntb->ctx_lock); 726300373Smav 727250079Scarl ntb_unmap_pci_bar(ntb); 728250079Scarl 729250079Scarl return (0); 730250079Scarl} 731250079Scarl 732300373Smav/* 733300373Smav * Driver internal routines 734300373Smav */ 735300373Smavstatic inline enum ntb_bar 736300373Smavntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 737300373Smav{ 738300373Smav 739300373Smav KASSERT(mw < ntb->mw_count, 740300373Smav ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 741300373Smav KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 742300373Smav 743300373Smav return (ntb->reg->mw_bar[mw]); 744300373Smav} 745300373Smav 746300373Smavstatic inline bool 747300373Smavbar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 748300373Smav{ 749300373Smav /* XXX This assertion could be stronger. */ 750300373Smav KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 751300373Smav return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 752300373Smav} 753300373Smav 754300373Smavstatic inline void 755300373Smavbar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 756300373Smav uint32_t *xlat, uint32_t *lmt) 757300373Smav{ 758300373Smav uint32_t basev, lmtv, xlatv; 759300373Smav 760300373Smav switch (bar) { 761300373Smav case NTB_B2B_BAR_1: 762300373Smav basev = ntb->xlat_reg->bar2_base; 763300373Smav lmtv = ntb->xlat_reg->bar2_limit; 764300373Smav xlatv = ntb->xlat_reg->bar2_xlat; 765300373Smav break; 766300373Smav case NTB_B2B_BAR_2: 767300373Smav basev = ntb->xlat_reg->bar4_base; 768300373Smav lmtv = ntb->xlat_reg->bar4_limit; 769300373Smav xlatv = ntb->xlat_reg->bar4_xlat; 770300373Smav break; 771300373Smav case NTB_B2B_BAR_3: 772300373Smav basev = ntb->xlat_reg->bar5_base; 773300373Smav lmtv = ntb->xlat_reg->bar5_limit; 774300373Smav xlatv = ntb->xlat_reg->bar5_xlat; 775300373Smav break; 776300373Smav default: 777300373Smav KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 778300373Smav ("bad bar")); 779300373Smav basev = lmtv = xlatv = 0; 780300373Smav break; 781300373Smav } 782300373Smav 783300373Smav if (base != NULL) 784300373Smav *base = basev; 785300373Smav if (xlat != NULL) 786300373Smav *xlat = xlatv; 787300373Smav if (lmt != NULL) 788300373Smav *lmt = lmtv; 789300373Smav} 790300373Smav 791250079Scarlstatic int 792255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 793250079Scarl{ 794255272Scarl int rc; 795250079Scarl 796250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 797300373Smav rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 798255272Scarl if (rc != 0) 799300373Smav goto out; 800255272Scarl 801300373Smav ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 802300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 803255272Scarl if (rc != 0) 804300373Smav goto out; 805300373Smav ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 806300373Smav ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 807300373Smav ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 808255272Scarl 809300373Smav ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 810300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 811255272Scarl if (rc != 0) 812300373Smav goto out; 813300373Smav ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 814300373Smav ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 815300373Smav ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 816255274Scarl 817300373Smav if (!HAS_FEATURE(NTB_SPLIT_BAR)) 818300373Smav goto out; 819250079Scarl 820300373Smav ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 821300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 822300373Smav ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 823300373Smav ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 824300373Smav ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 825255272Scarl 826300373Smavout: 827300373Smav if (rc != 0) 828255272Scarl device_printf(ntb->device, 829255272Scarl "unable to allocate pci resource\n"); 830255272Scarl return (rc); 831255272Scarl} 832255272Scarl 833300373Smavstatic void 834300373Smavprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 835300373Smav const char *kind) 836300373Smav{ 837300373Smav 838300373Smav device_printf(ntb->device, 839300373Smav "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 840300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 841300373Smav (char *)bar->vbase + bar->size - 1, 842300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 843300373Smav (uintmax_t)bar->size, kind); 844300373Smav} 845300373Smav 846255272Scarlstatic int 847255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 848255272Scarl{ 849255272Scarl 850255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 851300373Smav &bar->pci_resource_id, RF_ACTIVE); 852255272Scarl if (bar->pci_resource == NULL) 853255272Scarl return (ENXIO); 854300373Smav 855300373Smav save_bar_parameters(bar); 856300373Smav bar->map_mode = VM_MEMATTR_UNCACHEABLE; 857300373Smav print_map_success(ntb, bar, "mmr"); 858300373Smav return (0); 859255272Scarl} 860255272Scarl 861255272Scarlstatic int 862255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 863255272Scarl{ 864255272Scarl int rc; 865300373Smav vm_memattr_t mapmode; 866255276Scarl uint8_t bar_size_bits = 0; 867255272Scarl 868300373Smav bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 869300373Smav &bar->pci_resource_id, RF_ACTIVE); 870250079Scarl 871255272Scarl if (bar->pci_resource == NULL) 872255272Scarl return (ENXIO); 873255276Scarl 874300373Smav save_bar_parameters(bar); 875300373Smav /* 876300373Smav * Ivytown NTB BAR sizes are misreported by the hardware due to a 877300373Smav * hardware issue. To work around this, query the size it should be 878300373Smav * configured to by the device and modify the resource to correspond to 879300373Smav * this new size. The BIOS on systems with this problem is required to 880300373Smav * provide enough address space to allow the driver to make this change 881300373Smav * safely. 882300373Smav * 883300373Smav * Ideally I could have just specified the size when I allocated the 884300373Smav * resource like: 885300373Smav * bus_alloc_resource(ntb->device, 886300373Smav * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 887300373Smav * 1ul << bar_size_bits, RF_ACTIVE); 888300373Smav * but the PCI driver does not honor the size in this call, so we have 889300373Smav * to modify it after the fact. 890300373Smav */ 891300373Smav if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 892300373Smav if (bar->pci_resource_id == PCIR_BAR(2)) 893300373Smav bar_size_bits = pci_read_config(ntb->device, 894300373Smav XEON_PBAR23SZ_OFFSET, 1); 895300373Smav else 896300373Smav bar_size_bits = pci_read_config(ntb->device, 897300373Smav XEON_PBAR45SZ_OFFSET, 1); 898300373Smav 899300373Smav rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 900300373Smav bar->pci_resource, bar->pbase, 901300373Smav bar->pbase + (1ul << bar_size_bits) - 1); 902255272Scarl if (rc != 0) { 903300373Smav device_printf(ntb->device, 904300373Smav "unable to resize bar\n"); 905255272Scarl return (rc); 906250079Scarl } 907300373Smav 908300373Smav save_bar_parameters(bar); 909250079Scarl } 910300373Smav 911300373Smav bar->map_mode = VM_MEMATTR_UNCACHEABLE; 912300373Smav print_map_success(ntb, bar, "mw"); 913300373Smav 914300373Smav /* 915300373Smav * Optionally, mark MW BARs as anything other than UC to improve 916300373Smav * performance. 917300373Smav */ 918300373Smav mapmode = ntb_pat_flags(); 919300373Smav if (mapmode == bar->map_mode) 920300373Smav return (0); 921300373Smav 922300373Smav rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 923300373Smav if (rc == 0) { 924300373Smav bar->map_mode = mapmode; 925300373Smav device_printf(ntb->device, 926300373Smav "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 927300373Smav "%s.\n", 928300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 929300373Smav (char *)bar->vbase + bar->size - 1, 930300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 931300373Smav ntb_vm_memattr_to_str(mapmode)); 932300373Smav } else 933300373Smav device_printf(ntb->device, 934300373Smav "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 935300373Smav "%s: %d\n", 936300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 937300373Smav (char *)bar->vbase + bar->size - 1, 938300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 939300373Smav ntb_vm_memattr_to_str(mapmode), rc); 940300373Smav /* Proceed anyway */ 941250079Scarl return (0); 942250079Scarl} 943250079Scarl 944250079Scarlstatic void 945250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 946250079Scarl{ 947250079Scarl struct ntb_pci_bar_info *current_bar; 948250079Scarl int i; 949250079Scarl 950300373Smav for (i = 0; i < NTB_MAX_BARS; i++) { 951250079Scarl current_bar = &ntb->bar_info[i]; 952250079Scarl if (current_bar->pci_resource != NULL) 953250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 954250079Scarl current_bar->pci_resource_id, 955250079Scarl current_bar->pci_resource); 956250079Scarl } 957250079Scarl} 958250079Scarl 959250079Scarlstatic int 960300373Smavntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 961250079Scarl{ 962300373Smav uint32_t i; 963300373Smav int rc; 964250079Scarl 965300373Smav for (i = 0; i < num_vectors; i++) { 966300373Smav ntb->int_info[i].rid = i + 1; 967300373Smav ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 968300373Smav SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 969300373Smav if (ntb->int_info[i].res == NULL) { 970300373Smav device_printf(ntb->device, 971300373Smav "bus_alloc_resource failed\n"); 972300373Smav return (ENOMEM); 973300373Smav } 974300373Smav ntb->int_info[i].tag = NULL; 975300373Smav ntb->allocated_interrupts++; 976300373Smav rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 977300373Smav INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 978300373Smav &ntb->msix_vec[i], &ntb->int_info[i].tag); 979300373Smav if (rc != 0) { 980300373Smav device_printf(ntb->device, "bus_setup_intr failed\n"); 981300373Smav return (ENXIO); 982300373Smav } 983300373Smav } 984300373Smav return (0); 985300373Smav} 986300373Smav 987300373Smav/* 988300373Smav * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 989300373Smav * cannot be allocated for each MSI-X message. JHB seems to think remapping 990300373Smav * should be okay. This tunable should enable us to test that hypothesis 991300373Smav * when someone gets their hands on some Xeon hardware. 992300373Smav */ 993300373Smavstatic int ntb_force_remap_mode; 994300516SmavTUNABLE_INT("hw.ntb.force_remap_mode", &ntb_force_remap_mode); 995300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 996300373Smav &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 997300373Smav " to a smaller number of ithreads, even if the desired number are " 998300373Smav "available"); 999300373Smav 1000300373Smav/* 1001300373Smav * In case it is NOT ok, give consumers an abort button. 1002300373Smav */ 1003300373Smavstatic int ntb_prefer_intx; 1004300516SmavTUNABLE_INT("hw.ntb.prefer_intx_to_remap", &ntb_prefer_intx); 1005300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 1006300373Smav &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1007300373Smav "than remapping MSI-X messages over available slots (match Linux driver " 1008300373Smav "behavior)"); 1009300373Smav 1010300373Smav/* 1011300373Smav * Remap the desired number of MSI-X messages to available ithreads in a simple 1012300373Smav * round-robin fashion. 1013300373Smav */ 1014300373Smavstatic int 1015300373Smavntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1016300373Smav{ 1017300373Smav u_int *vectors; 1018300373Smav uint32_t i; 1019300373Smav int rc; 1020300373Smav 1021300373Smav if (ntb_prefer_intx != 0) 1022300373Smav return (ENXIO); 1023300373Smav 1024300373Smav vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1025300373Smav 1026300373Smav for (i = 0; i < desired; i++) 1027300373Smav vectors[i] = (i % avail) + 1; 1028300373Smav 1029300373Smav rc = pci_remap_msix(dev, desired, vectors); 1030300373Smav free(vectors, M_NTB); 1031300373Smav return (rc); 1032300373Smav} 1033300373Smav 1034300373Smavstatic int 1035300373Smavntb_init_isr(struct ntb_softc *ntb) 1036300373Smav{ 1037300373Smav uint32_t desired_vectors, num_vectors; 1038300373Smav int rc; 1039300373Smav 1040250079Scarl ntb->allocated_interrupts = 0; 1041300373Smav ntb->last_ts = ticks; 1042300373Smav 1043250079Scarl /* 1044301811Sngie * Mask all doorbell interrupts. (Except link events!) 1045250079Scarl */ 1046301811Sngie DB_MASK_LOCK(ntb); 1047301811Sngie ntb->db_mask = ntb->db_valid_mask; 1048301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1049301811Sngie DB_MASK_UNLOCK(ntb); 1050250079Scarl 1051300373Smav num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1052300373Smav ntb->db_count); 1053300373Smav if (desired_vectors >= 1) { 1054300373Smav rc = pci_alloc_msix(ntb->device, &num_vectors); 1055250079Scarl 1056300373Smav if (ntb_force_remap_mode != 0 && rc == 0 && 1057300373Smav num_vectors == desired_vectors) 1058300373Smav num_vectors--; 1059300373Smav 1060300373Smav if (rc == 0 && num_vectors < desired_vectors) { 1061300373Smav rc = ntb_remap_msix(ntb->device, desired_vectors, 1062300373Smav num_vectors); 1063300373Smav if (rc == 0) 1064300373Smav num_vectors = desired_vectors; 1065300373Smav else 1066300373Smav pci_release_msi(ntb->device); 1067250079Scarl } 1068300373Smav if (rc != 0) 1069300373Smav num_vectors = 1; 1070300373Smav } else 1071300373Smav num_vectors = 1; 1072300373Smav 1073300373Smav if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1074301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1075301811Sngie device_printf(ntb->device, 1076301811Sngie "Errata workaround does not support MSI or INTX\n"); 1077301811Sngie return (EINVAL); 1078301811Sngie } 1079301811Sngie 1080300373Smav ntb->db_vec_count = 1; 1081300373Smav ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1082300373Smav rc = ntb_setup_legacy_interrupt(ntb); 1083300373Smav } else { 1084301811Sngie if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1085301811Sngie HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1086301811Sngie device_printf(ntb->device, 1087301811Sngie "Errata workaround expects %d doorbell bits\n", 1088301811Sngie XEON_NONLINK_DB_MSIX_BITS); 1089301811Sngie return (EINVAL); 1090301811Sngie } 1091301811Sngie 1092300373Smav ntb_create_msix_vec(ntb, num_vectors); 1093300373Smav rc = ntb_setup_msix(ntb, num_vectors); 1094301811Sngie if (rc == 0 && HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1095301811Sngie ntb_get_msix_info(ntb); 1096250079Scarl } 1097300373Smav if (rc != 0) { 1098300373Smav device_printf(ntb->device, 1099300373Smav "Error allocating interrupts: %d\n", rc); 1100300373Smav ntb_free_msix_vec(ntb); 1101300373Smav } 1102250079Scarl 1103300373Smav return (rc); 1104300373Smav} 1105250079Scarl 1106300373Smavstatic int 1107300373Smavntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1108300373Smav{ 1109300373Smav int rc; 1110300373Smav 1111300373Smav ntb->int_info[0].rid = 0; 1112300373Smav ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1113300373Smav &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1114300373Smav if (ntb->int_info[0].res == NULL) { 1115300373Smav device_printf(ntb->device, "bus_alloc_resource failed\n"); 1116300373Smav return (ENOMEM); 1117250079Scarl } 1118250079Scarl 1119300373Smav ntb->int_info[0].tag = NULL; 1120300373Smav ntb->allocated_interrupts = 1; 1121300373Smav 1122300373Smav rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1123300373Smav INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1124300373Smav ntb, &ntb->int_info[0].tag); 1125300373Smav if (rc != 0) { 1126300373Smav device_printf(ntb->device, "bus_setup_intr failed\n"); 1127300373Smav return (ENXIO); 1128300373Smav } 1129300373Smav 1130250079Scarl return (0); 1131250079Scarl} 1132250079Scarl 1133250079Scarlstatic void 1134250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 1135250079Scarl{ 1136250079Scarl struct ntb_int_info *current_int; 1137250079Scarl int i; 1138250079Scarl 1139300373Smav for (i = 0; i < ntb->allocated_interrupts; i++) { 1140250079Scarl current_int = &ntb->int_info[i]; 1141250079Scarl if (current_int->tag != NULL) 1142250079Scarl bus_teardown_intr(ntb->device, current_int->res, 1143250079Scarl current_int->tag); 1144250079Scarl 1145250079Scarl if (current_int->res != NULL) 1146250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 1147250079Scarl rman_get_rid(current_int->res), current_int->res); 1148250079Scarl } 1149250079Scarl 1150300373Smav ntb_free_msix_vec(ntb); 1151250079Scarl pci_release_msi(ntb->device); 1152250079Scarl} 1153250079Scarl 1154300373Smav/* 1155300373Smav * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1156300373Smav * out to make code clearer. 1157300373Smav */ 1158300373Smavstatic inline uint64_t 1159300373Smavdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 1160250079Scarl{ 1161250079Scarl 1162300373Smav if (ntb->type == NTB_ATOM) 1163300373Smav return (ntb_reg_read(8, regoff)); 1164250079Scarl 1165300373Smav KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1166300373Smav 1167300373Smav return (ntb_reg_read(2, regoff)); 1168250079Scarl} 1169250079Scarl 1170300373Smavstatic inline void 1171300373Smavdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1172250079Scarl{ 1173250079Scarl 1174300373Smav KASSERT((val & ~ntb->db_valid_mask) == 0, 1175300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1176300373Smav (uintmax_t)(val & ~ntb->db_valid_mask), 1177300373Smav (uintmax_t)ntb->db_valid_mask)); 1178250079Scarl 1179300373Smav if (regoff == ntb->self_reg->db_mask) 1180300373Smav DB_MASK_ASSERT(ntb, MA_OWNED); 1181300373Smav db_iowrite_raw(ntb, regoff, val); 1182250079Scarl} 1183250079Scarl 1184300373Smavstatic inline void 1185300373Smavdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1186250079Scarl{ 1187250079Scarl 1188300373Smav if (ntb->type == NTB_ATOM) { 1189300373Smav ntb_reg_write(8, regoff, val); 1190300373Smav return; 1191300373Smav } 1192250079Scarl 1193300373Smav KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1194300373Smav ntb_reg_write(2, regoff, (uint16_t)val); 1195250079Scarl} 1196250079Scarl 1197300373Smavvoid 1198300373Smavntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1199250079Scarl{ 1200250079Scarl 1201301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1202301811Sngie return; 1203301811Sngie 1204300373Smav DB_MASK_LOCK(ntb); 1205300373Smav ntb->db_mask |= bits; 1206300373Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1207300373Smav DB_MASK_UNLOCK(ntb); 1208300373Smav} 1209250079Scarl 1210300373Smavvoid 1211300373Smavntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1212300373Smav{ 1213250079Scarl 1214300373Smav KASSERT((bits & ~ntb->db_valid_mask) == 0, 1215300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1216300373Smav (uintmax_t)(bits & ~ntb->db_valid_mask), 1217300373Smav (uintmax_t)ntb->db_valid_mask)); 1218250079Scarl 1219301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1220301811Sngie return; 1221301811Sngie 1222300373Smav DB_MASK_LOCK(ntb); 1223300373Smav ntb->db_mask &= ~bits; 1224300373Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1225300373Smav DB_MASK_UNLOCK(ntb); 1226300373Smav} 1227300373Smav 1228300373Smavuint64_t 1229300373Smavntb_db_read(struct ntb_softc *ntb) 1230300373Smav{ 1231300373Smav 1232301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1233301811Sngie uint64_t res; 1234301811Sngie unsigned i; 1235301811Sngie 1236301811Sngie res = 0; 1237301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1238301811Sngie if (ntb->msix_vec[i].masked != 0) 1239301811Sngie res |= ntb_db_vector_mask(ntb, i); 1240301811Sngie } 1241301811Sngie return (res); 1242301811Sngie } 1243301811Sngie 1244300373Smav return (db_ioread(ntb, ntb->self_reg->db_bell)); 1245300373Smav} 1246300373Smav 1247300373Smavvoid 1248300373Smavntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1249300373Smav{ 1250300373Smav 1251300373Smav KASSERT((bits & ~ntb->db_valid_mask) == 0, 1252300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1253300373Smav (uintmax_t)(bits & ~ntb->db_valid_mask), 1254300373Smav (uintmax_t)ntb->db_valid_mask)); 1255300373Smav 1256301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1257301811Sngie unsigned i; 1258301811Sngie 1259301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1260301811Sngie if ((bits & ntb_db_vector_mask(ntb, i)) != 0) { 1261301811Sngie DB_MASK_LOCK(ntb); 1262301811Sngie if (ntb->msix_vec[i].masked != 0) { 1263301811Sngie /* XXX These need a public API. */ 1264301811Sngie#if 0 1265301811Sngie pci_unmask_msix(ntb->device, i); 1266301811Sngie#endif 1267301811Sngie ntb->msix_vec[i].masked = 0; 1268301811Sngie } 1269301811Sngie DB_MASK_UNLOCK(ntb); 1270301811Sngie } 1271301811Sngie } 1272301811Sngie return; 1273301811Sngie } 1274301811Sngie 1275300373Smav db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1276300373Smav} 1277300373Smav 1278300373Smavstatic inline uint64_t 1279300373Smavntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1280300373Smav{ 1281300373Smav uint64_t shift, mask; 1282300373Smav 1283300373Smav shift = ntb->db_vec_shift; 1284300373Smav mask = (1ull << shift) - 1; 1285300373Smav return (mask << (shift * db_vector)); 1286300373Smav} 1287300373Smav 1288300373Smavstatic void 1289300373Smavntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1290300373Smav{ 1291300373Smav uint64_t vec_mask; 1292300373Smav 1293300373Smav ntb->last_ts = ticks; 1294300373Smav vec_mask = ntb_vec_mask(ntb, vec); 1295300373Smav 1296300373Smav if ((vec_mask & ntb->db_link_mask) != 0) { 1297300373Smav if (ntb_poll_link(ntb)) 1298300373Smav ntb_link_event(ntb); 1299250079Scarl } 1300250079Scarl 1301301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1302301811Sngie (vec_mask & ntb->db_link_mask) == 0) { 1303301811Sngie DB_MASK_LOCK(ntb); 1304301811Sngie if (ntb->msix_vec[vec].masked == 0) { 1305301811Sngie /* XXX These need a public API. */ 1306301811Sngie#if 0 1307301811Sngie pci_mask_msix(ntb->device, vec); 1308301811Sngie#endif 1309301811Sngie ntb->msix_vec[vec].masked = 1; 1310301811Sngie } 1311301811Sngie DB_MASK_UNLOCK(ntb); 1312301811Sngie } 1313301811Sngie 1314300373Smav if ((vec_mask & ntb->db_valid_mask) != 0) 1315300373Smav ntb_db_event(ntb, vec); 1316250079Scarl} 1317250079Scarl 1318300373Smavstatic void 1319300373Smavndev_vec_isr(void *arg) 1320300373Smav{ 1321300373Smav struct ntb_vec *nvec = arg; 1322300373Smav 1323300373Smav ntb_interrupt(nvec->ntb, nvec->num); 1324300373Smav} 1325300373Smav 1326300373Smavstatic void 1327300373Smavndev_irq_isr(void *arg) 1328300373Smav{ 1329300373Smav /* If we couldn't set up MSI-X, we only have the one vector. */ 1330300373Smav ntb_interrupt(arg, 0); 1331300373Smav} 1332300373Smav 1333250079Scarlstatic int 1334300373Smavntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1335250079Scarl{ 1336300373Smav uint32_t i; 1337250079Scarl 1338300373Smav ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1339250079Scarl M_ZERO | M_WAITOK); 1340250079Scarl for (i = 0; i < num_vectors; i++) { 1341300373Smav ntb->msix_vec[i].num = i; 1342300373Smav ntb->msix_vec[i].ntb = ntb; 1343250079Scarl } 1344250079Scarl 1345250079Scarl return (0); 1346250079Scarl} 1347250079Scarl 1348250079Scarlstatic void 1349300373Smavntb_free_msix_vec(struct ntb_softc *ntb) 1350250079Scarl{ 1351250079Scarl 1352300373Smav if (ntb->msix_vec == NULL) 1353300373Smav return; 1354250079Scarl 1355300373Smav free(ntb->msix_vec, M_NTB); 1356300373Smav ntb->msix_vec = NULL; 1357250079Scarl} 1358250079Scarl 1359301811Sngiestatic void 1360301811Sngientb_get_msix_info(struct ntb_softc *ntb) 1361301811Sngie{ 1362301811Sngie struct pci_devinfo *dinfo; 1363301811Sngie struct pcicfg_msix *msix; 1364301811Sngie uint32_t laddr, data, i, offset; 1365301811Sngie 1366301811Sngie dinfo = device_get_ivars(ntb->device); 1367301811Sngie msix = &dinfo->cfg.msix; 1368301811Sngie 1369301811Sngie CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1370301811Sngie 1371301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1372301811Sngie offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1373301811Sngie 1374301811Sngie laddr = bus_read_4(msix->msix_table_res, offset + 1375301811Sngie PCI_MSIX_ENTRY_LOWER_ADDR); 1376301904Smav ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1377301811Sngie 1378301811Sngie KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1379301811Sngie ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1380301811Sngie MSI_INTEL_ADDR_BASE)); 1381301904Smav ntb->msix_data[i].nmd_ofs = laddr; 1382301811Sngie 1383301811Sngie data = bus_read_4(msix->msix_table_res, offset + 1384301811Sngie PCI_MSIX_ENTRY_DATA); 1385301811Sngie ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1386301811Sngie 1387301811Sngie ntb->msix_data[i].nmd_data = data; 1388301811Sngie } 1389301811Sngie} 1390301811Sngie 1391250079Scarlstatic struct ntb_hw_info * 1392250079Scarlntb_get_device_info(uint32_t device_id) 1393250079Scarl{ 1394250079Scarl struct ntb_hw_info *ep = pci_ids; 1395250079Scarl 1396250079Scarl while (ep->device_id) { 1397250079Scarl if (ep->device_id == device_id) 1398250079Scarl return (ep); 1399250079Scarl ++ep; 1400250079Scarl } 1401250079Scarl return (NULL); 1402250079Scarl} 1403250079Scarl 1404300373Smavstatic void 1405300373Smavntb_teardown_xeon(struct ntb_softc *ntb) 1406250079Scarl{ 1407250079Scarl 1408300373Smav if (ntb->reg != NULL) 1409300373Smav ntb_link_disable(ntb); 1410300373Smav} 1411300373Smav 1412300373Smavstatic void 1413300373Smavntb_detect_max_mw(struct ntb_softc *ntb) 1414300373Smav{ 1415300373Smav 1416300373Smav if (ntb->type == NTB_ATOM) { 1417300373Smav ntb->mw_count = ATOM_MW_COUNT; 1418300373Smav return; 1419300373Smav } 1420300373Smav 1421300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 1422300373Smav ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1423250079Scarl else 1424300373Smav ntb->mw_count = XEON_SNB_MW_COUNT; 1425250079Scarl} 1426250079Scarl 1427250079Scarlstatic int 1428300373Smavntb_detect_xeon(struct ntb_softc *ntb) 1429250079Scarl{ 1430300373Smav uint8_t ppd, conn_type; 1431250079Scarl 1432300373Smav ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1433300373Smav ntb->ppd = ppd; 1434250079Scarl 1435300373Smav if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1436300373Smav ntb->dev_type = NTB_DEV_DSD; 1437300373Smav else 1438300373Smav ntb->dev_type = NTB_DEV_USD; 1439300373Smav 1440300373Smav if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1441300373Smav ntb->features |= NTB_SPLIT_BAR; 1442300373Smav 1443301811Sngie /* 1444301811Sngie * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1445301811Sngie * errata workaround; only do one at a time. 1446301811Sngie */ 1447300373Smav if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1448301811Sngie ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1449300373Smav 1450300373Smav conn_type = ppd & XEON_PPD_CONN_TYPE; 1451300373Smav switch (conn_type) { 1452250079Scarl case NTB_CONN_B2B: 1453300373Smav ntb->conn_type = conn_type; 1454250079Scarl break; 1455250079Scarl case NTB_CONN_RP: 1456300373Smav case NTB_CONN_TRANSPARENT: 1457250079Scarl default: 1458300373Smav device_printf(ntb->device, "Unsupported connection type: %u\n", 1459300373Smav (unsigned)conn_type); 1460250079Scarl return (ENXIO); 1461250079Scarl } 1462300373Smav return (0); 1463300373Smav} 1464250079Scarl 1465300373Smavstatic int 1466300373Smavntb_detect_atom(struct ntb_softc *ntb) 1467300373Smav{ 1468300373Smav uint32_t ppd, conn_type; 1469300373Smav 1470300373Smav ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1471300373Smav ntb->ppd = ppd; 1472300373Smav 1473300373Smav if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1474250079Scarl ntb->dev_type = NTB_DEV_DSD; 1475250079Scarl else 1476250079Scarl ntb->dev_type = NTB_DEV_USD; 1477250079Scarl 1478300373Smav conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1479300373Smav switch (conn_type) { 1480300373Smav case NTB_CONN_B2B: 1481300373Smav ntb->conn_type = conn_type; 1482300373Smav break; 1483300373Smav default: 1484300373Smav device_printf(ntb->device, "Unsupported NTB configuration\n"); 1485300373Smav return (ENXIO); 1486250079Scarl } 1487250079Scarl return (0); 1488250079Scarl} 1489250079Scarl 1490250079Scarlstatic int 1491300373Smavntb_xeon_init_dev(struct ntb_softc *ntb) 1492250079Scarl{ 1493300373Smav int rc; 1494250079Scarl 1495300373Smav ntb->spad_count = XEON_SPAD_COUNT; 1496300373Smav ntb->db_count = XEON_DB_COUNT; 1497300373Smav ntb->db_link_mask = XEON_DB_LINK_BIT; 1498300373Smav ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1499300373Smav ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1500250079Scarl 1501300373Smav if (ntb->conn_type != NTB_CONN_B2B) { 1502250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1503300373Smav ntb->conn_type); 1504250079Scarl return (ENXIO); 1505250079Scarl } 1506250079Scarl 1507300373Smav ntb->reg = &xeon_reg; 1508300373Smav ntb->self_reg = &xeon_pri_reg; 1509300373Smav ntb->peer_reg = &xeon_b2b_reg; 1510300373Smav ntb->xlat_reg = &xeon_sec_xlat; 1511300373Smav 1512301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1513301811Sngie ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1514301811Sngie ntb->mw_count; 1515301811Sngie ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1516301811Sngie g_ntb_msix_idx, ntb->msix_mw_idx); 1517301811Sngie rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1518301811Sngie VM_MEMATTR_UNCACHEABLE); 1519301811Sngie KASSERT(rc == 0, ("shouldn't fail")); 1520301811Sngie } else if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1521301811Sngie /* 1522301811Sngie * There is a Xeon hardware errata related to writes to SDOORBELL or 1523301811Sngie * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1524301811Sngie * which may hang the system. To workaround this, use a memory 1525301811Sngie * window to access the interrupt and scratch pad registers on the 1526301811Sngie * remote system. 1527301811Sngie */ 1528300373Smav ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1529300373Smav ntb->mw_count; 1530300373Smav ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1531300373Smav g_ntb_mw_idx, ntb->b2b_mw_idx); 1532301811Sngie rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1533301811Sngie VM_MEMATTR_UNCACHEABLE); 1534300373Smav KASSERT(rc == 0, ("shouldn't fail")); 1535300373Smav } else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1536300373Smav /* 1537300373Smav * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1538300373Smav * mirrored to the remote system. Shrink the number of bits by one, 1539300373Smav * since bit 14 is the last bit. 1540300373Smav * 1541300373Smav * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1542300373Smav * anyway. Nor for non-B2B connection types. 1543300373Smav */ 1544300373Smav ntb->db_count = XEON_DB_COUNT - 1; 1545300373Smav 1546300373Smav ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1547300373Smav 1548300373Smav if (ntb->dev_type == NTB_DEV_USD) 1549300373Smav rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1550300373Smav &xeon_b2b_usd_addr); 1551250079Scarl else 1552300373Smav rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1553300373Smav &xeon_b2b_dsd_addr); 1554300373Smav if (rc != 0) 1555300373Smav return (rc); 1556250079Scarl 1557300373Smav /* Enable Bus Master and Memory Space on the secondary side */ 1558300373Smav ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1559300373Smav PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1560250079Scarl 1561300373Smav /* 1562300373Smav * Mask all doorbell interrupts. 1563300373Smav */ 1564301811Sngie DB_MASK_LOCK(ntb); 1565301811Sngie ntb->db_mask = ntb->db_valid_mask; 1566301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1567301811Sngie DB_MASK_UNLOCK(ntb); 1568250079Scarl 1569301811Sngie rc = xeon_setup_msix_bar(ntb); 1570301811Sngie if (rc != 0) 1571301811Sngie return (rc); 1572301811Sngie 1573300373Smav rc = ntb_init_isr(ntb); 1574300373Smav return (rc); 1575300373Smav} 1576250079Scarl 1577300373Smavstatic int 1578300373Smavntb_atom_init_dev(struct ntb_softc *ntb) 1579300373Smav{ 1580300373Smav int error; 1581250079Scarl 1582300373Smav KASSERT(ntb->conn_type == NTB_CONN_B2B, 1583300373Smav ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1584300373Smav 1585300373Smav ntb->spad_count = ATOM_SPAD_COUNT; 1586300373Smav ntb->db_count = ATOM_DB_COUNT; 1587300373Smav ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1588300373Smav ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1589300373Smav ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1590300373Smav 1591300373Smav ntb->reg = &atom_reg; 1592300373Smav ntb->self_reg = &atom_pri_reg; 1593300373Smav ntb->peer_reg = &atom_b2b_reg; 1594300373Smav ntb->xlat_reg = &atom_sec_xlat; 1595300373Smav 1596250079Scarl /* 1597300373Smav * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1598250079Scarl * resolved. Mask transaction layer internal parity errors. 1599250079Scarl */ 1600250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1601250079Scarl 1602300373Smav configure_atom_secondary_side_bars(ntb); 1603250079Scarl 1604250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1605300373Smav ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1606250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1607250079Scarl 1608300373Smav error = ntb_init_isr(ntb); 1609300373Smav if (error != 0) 1610300373Smav return (error); 1611300373Smav 1612300373Smav /* Initiate PCI-E link training */ 1613300373Smav ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1614300373Smav 1615300373Smav callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1616300373Smav 1617250079Scarl return (0); 1618250079Scarl} 1619250079Scarl 1620300373Smav/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1621255279Scarlstatic void 1622300373Smavconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1623255279Scarl{ 1624255279Scarl 1625255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1626300373Smav ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1627300373Smav XEON_B2B_BAR2_ADDR64); 1628300373Smav ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1629300373Smav XEON_B2B_BAR4_ADDR64); 1630300373Smav ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1631300373Smav ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1632255279Scarl } else { 1633300373Smav ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1634300373Smav XEON_B2B_BAR2_ADDR64); 1635300373Smav ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1636300373Smav XEON_B2B_BAR4_ADDR64); 1637300373Smav ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1638300373Smav ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1639255279Scarl } 1640255279Scarl} 1641255279Scarl 1642300373Smav 1643300373Smav/* 1644300373Smav * When working around Xeon SDOORBELL errata by remapping remote registers in a 1645300373Smav * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1646300373Smav * remains for use by a higher layer. 1647300373Smav * 1648300373Smav * Will only be used if working around SDOORBELL errata and the BIOS-configured 1649300373Smav * MW size is sufficiently large. 1650300373Smav */ 1651300373Smavstatic unsigned int ntb_b2b_mw_share; 1652300516SmavTUNABLE_INT("hw.ntb.b2b_mw_share", &ntb_b2b_mw_share); 1653300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1654300373Smav 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1655300373Smav "MW with higher level consumers. Both sides of the NTB MUST set the same " 1656300373Smav "value here."); 1657300373Smav 1658255279Scarlstatic void 1659300373Smavxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1660300373Smav enum ntb_bar regbar) 1661255279Scarl{ 1662300373Smav struct ntb_pci_bar_info *bar; 1663300373Smav uint8_t bar_sz; 1664255279Scarl 1665300373Smav if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1666300373Smav return; 1667300373Smav 1668300373Smav bar = &ntb->bar_info[idx]; 1669300373Smav bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1670300373Smav if (idx == regbar) { 1671300373Smav if (ntb->b2b_off != 0) 1672300373Smav bar_sz--; 1673255279Scarl else 1674300373Smav bar_sz = 0; 1675300373Smav } 1676300373Smav pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1677300373Smav bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1678300373Smav (void)bar_sz; 1679300373Smav} 1680300373Smav 1681300373Smavstatic void 1682300373Smavxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1683300373Smav enum ntb_bar idx, enum ntb_bar regbar) 1684300373Smav{ 1685301904Smav uint64_t reg_val; 1686300373Smav uint32_t base_reg, lmt_reg; 1687300373Smav 1688300373Smav bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1689304376Smav if (idx == regbar) { 1690304376Smav if (ntb->b2b_off) 1691304376Smav bar_addr += ntb->b2b_off; 1692304376Smav else 1693304376Smav bar_addr = 0; 1694304376Smav } 1695300373Smav 1696301811Sngie /* 1697301811Sngie * Set limit registers first to avoid an errata where setting the base 1698301811Sngie * registers locks the limit registers. 1699301811Sngie */ 1700300373Smav if (!bar_is_64bit(ntb, idx)) { 1701301904Smav ntb_reg_write(4, lmt_reg, bar_addr); 1702301811Sngie reg_val = ntb_reg_read(4, lmt_reg); 1703301811Sngie (void)reg_val; 1704301811Sngie 1705300373Smav ntb_reg_write(4, base_reg, bar_addr); 1706300373Smav reg_val = ntb_reg_read(4, base_reg); 1707300373Smav (void)reg_val; 1708301811Sngie } else { 1709301904Smav ntb_reg_write(8, lmt_reg, bar_addr); 1710301811Sngie reg_val = ntb_reg_read(8, lmt_reg); 1711301811Sngie (void)reg_val; 1712300373Smav 1713300373Smav ntb_reg_write(8, base_reg, bar_addr); 1714300373Smav reg_val = ntb_reg_read(8, base_reg); 1715300373Smav (void)reg_val; 1716255279Scarl } 1717255279Scarl} 1718255279Scarl 1719250079Scarlstatic void 1720300373Smavxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1721250079Scarl{ 1722300373Smav struct ntb_pci_bar_info *bar; 1723250079Scarl 1724300373Smav bar = &ntb->bar_info[idx]; 1725300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1726300373Smav ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1727300373Smav base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1728300373Smav } else { 1729300373Smav ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1730300373Smav base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1731300373Smav } 1732300373Smav (void)base_addr; 1733300373Smav} 1734300373Smav 1735300373Smavstatic int 1736301811Sngiexeon_setup_msix_bar(struct ntb_softc *ntb) 1737301811Sngie{ 1738301811Sngie enum ntb_bar bar_num; 1739301811Sngie 1740301811Sngie if (!HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1741301811Sngie return (0); 1742301811Sngie 1743301811Sngie bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1744301904Smav ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1745301811Sngie return (0); 1746301811Sngie} 1747301811Sngie 1748301811Sngiestatic int 1749300373Smavxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1750300373Smav const struct ntb_b2b_addr *peer_addr) 1751300373Smav{ 1752300373Smav struct ntb_pci_bar_info *b2b_bar; 1753300373Smav vm_size_t bar_size; 1754300373Smav uint64_t bar_addr; 1755300373Smav enum ntb_bar b2b_bar_num, i; 1756300373Smav 1757300373Smav if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1758300373Smav b2b_bar = NULL; 1759300373Smav b2b_bar_num = NTB_CONFIG_BAR; 1760300373Smav ntb->b2b_off = 0; 1761300373Smav } else { 1762300373Smav b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1763300373Smav KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1764300373Smav ("invalid b2b mw bar")); 1765300373Smav 1766300373Smav b2b_bar = &ntb->bar_info[b2b_bar_num]; 1767300373Smav bar_size = b2b_bar->size; 1768300373Smav 1769300373Smav if (ntb_b2b_mw_share != 0 && 1770300373Smav (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1771300373Smav ntb->b2b_off = bar_size >> 1; 1772300373Smav else if (bar_size >= XEON_B2B_MIN_SIZE) { 1773300373Smav ntb->b2b_off = 0; 1774300373Smav } else { 1775300373Smav device_printf(ntb->device, 1776300373Smav "B2B bar size is too small!\n"); 1777300373Smav return (EIO); 1778250079Scarl } 1779250079Scarl } 1780250079Scarl 1781300373Smav /* 1782300373Smav * Reset the secondary bar sizes to match the primary bar sizes. 1783300373Smav * (Except, disable or halve the size of the B2B secondary bar.) 1784300373Smav */ 1785300373Smav for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1786300373Smav xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1787300373Smav 1788300373Smav bar_addr = 0; 1789300373Smav if (b2b_bar_num == NTB_CONFIG_BAR) 1790300373Smav bar_addr = addr->bar0_addr; 1791300373Smav else if (b2b_bar_num == NTB_B2B_BAR_1) 1792300373Smav bar_addr = addr->bar2_addr64; 1793300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1794300373Smav bar_addr = addr->bar4_addr64; 1795300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2) 1796300373Smav bar_addr = addr->bar4_addr32; 1797300373Smav else if (b2b_bar_num == NTB_B2B_BAR_3) 1798300373Smav bar_addr = addr->bar5_addr32; 1799300373Smav else 1800300373Smav KASSERT(false, ("invalid bar")); 1801300373Smav 1802300373Smav ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1803300373Smav 1804300373Smav /* 1805300373Smav * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1806300373Smav * register BAR. The B2B BAR is either disabled above or configured 1807300373Smav * half-size. It starts at PBAR xlat + offset. 1808300373Smav * 1809300373Smav * Also set up incoming BAR limits == base (zero length window). 1810300373Smav */ 1811300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1812300373Smav b2b_bar_num); 1813300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1814300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1815300373Smav NTB_B2B_BAR_2, b2b_bar_num); 1816300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1817300373Smav NTB_B2B_BAR_3, b2b_bar_num); 1818300373Smav } else 1819300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1820300373Smav NTB_B2B_BAR_2, b2b_bar_num); 1821300373Smav 1822300373Smav /* Zero incoming translation addrs */ 1823300373Smav ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1824300373Smav ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1825300373Smav 1826301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1827301811Sngie size_t size, xlatoffset; 1828301811Sngie 1829301811Sngie switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1830301811Sngie case NTB_B2B_BAR_1: 1831301811Sngie size = 8; 1832301811Sngie xlatoffset = XEON_SBAR2XLAT_OFFSET; 1833301811Sngie break; 1834301811Sngie case NTB_B2B_BAR_2: 1835301811Sngie xlatoffset = XEON_SBAR4XLAT_OFFSET; 1836301811Sngie if (HAS_FEATURE(NTB_SPLIT_BAR)) 1837301811Sngie size = 4; 1838301811Sngie else 1839301811Sngie size = 8; 1840301811Sngie break; 1841301811Sngie case NTB_B2B_BAR_3: 1842301811Sngie xlatoffset = XEON_SBAR5XLAT_OFFSET; 1843301811Sngie size = 4; 1844301811Sngie break; 1845301811Sngie default: 1846301811Sngie KASSERT(false, ("Bogus msix mw idx: %u", 1847301811Sngie ntb->msix_mw_idx)); 1848301811Sngie return (EINVAL); 1849301811Sngie } 1850301811Sngie 1851301811Sngie /* 1852301811Sngie * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1853301811Sngie * workaround 1854301811Sngie */ 1855301904Smav if (size == 4) { 1856301811Sngie ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1857301904Smav ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1858301904Smav } else { 1859301811Sngie ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1860301904Smav ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1861301904Smav } 1862301811Sngie } 1863301811Sngie (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1864301811Sngie (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1865301811Sngie 1866300373Smav /* Zero outgoing translation limits (whole bar size windows) */ 1867300373Smav ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1868300373Smav ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1869300373Smav 1870300373Smav /* Set outgoing translation offsets */ 1871300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1872300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1873300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1874300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1875300373Smav } else 1876300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1877300373Smav 1878300373Smav /* Set the translation offset for B2B registers */ 1879300373Smav bar_addr = 0; 1880300373Smav if (b2b_bar_num == NTB_CONFIG_BAR) 1881300373Smav bar_addr = peer_addr->bar0_addr; 1882300373Smav else if (b2b_bar_num == NTB_B2B_BAR_1) 1883300373Smav bar_addr = peer_addr->bar2_addr64; 1884300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1885300373Smav bar_addr = peer_addr->bar4_addr64; 1886300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2) 1887300373Smav bar_addr = peer_addr->bar4_addr32; 1888300373Smav else if (b2b_bar_num == NTB_B2B_BAR_3) 1889300373Smav bar_addr = peer_addr->bar5_addr32; 1890300373Smav else 1891300373Smav KASSERT(false, ("invalid bar")); 1892300373Smav 1893300373Smav /* 1894300373Smav * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1895300373Smav * at a time. 1896300373Smav */ 1897300373Smav ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1898300373Smav ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1899300373Smav return (0); 1900250079Scarl} 1901250079Scarl 1902300373Smavstatic inline bool 1903301811Sngie_xeon_link_is_up(struct ntb_softc *ntb) 1904301811Sngie{ 1905301811Sngie 1906301811Sngie if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1907301811Sngie return (true); 1908301811Sngie return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1909301811Sngie} 1910301811Sngie 1911301811Sngiestatic inline bool 1912300373Smavlink_is_up(struct ntb_softc *ntb) 1913300373Smav{ 1914300373Smav 1915301811Sngie if (ntb->type == NTB_XEON) 1916301811Sngie return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1917301811Sngie !HAS_FEATURE(NTB_SB01BASE_LOCKUP))); 1918300373Smav 1919300373Smav KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1920300373Smav return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1921300373Smav} 1922300373Smav 1923300373Smavstatic inline bool 1924300373Smavatom_link_is_err(struct ntb_softc *ntb) 1925300373Smav{ 1926300373Smav uint32_t status; 1927300373Smav 1928300373Smav KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1929300373Smav 1930300373Smav status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1931300373Smav if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1932300373Smav return (true); 1933300373Smav 1934300373Smav status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1935300373Smav return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1936300373Smav} 1937300373Smav 1938300373Smav/* Atom does not have link status interrupt, poll on that platform */ 1939250079Scarlstatic void 1940300373Smavatom_link_hb(void *arg) 1941250079Scarl{ 1942300373Smav struct ntb_softc *ntb = arg; 1943300373Smav sbintime_t timo, poll_ts; 1944300373Smav 1945300373Smav timo = NTB_HB_TIMEOUT * hz; 1946300373Smav poll_ts = ntb->last_ts + timo; 1947300373Smav 1948300373Smav /* 1949300373Smav * Delay polling the link status if an interrupt was received, unless 1950300373Smav * the cached link status says the link is down. 1951300373Smav */ 1952300373Smav if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1953300373Smav timo = poll_ts - ticks; 1954300373Smav goto out; 1955300373Smav } 1956300373Smav 1957300373Smav if (ntb_poll_link(ntb)) 1958300373Smav ntb_link_event(ntb); 1959300373Smav 1960300373Smav if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1961300373Smav /* Link is down with error, proceed with recovery */ 1962300373Smav callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1963300373Smav return; 1964300373Smav } 1965300373Smav 1966300373Smavout: 1967300373Smav callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1968300373Smav} 1969300373Smav 1970300373Smavstatic void 1971300373Smavatom_perform_link_restart(struct ntb_softc *ntb) 1972300373Smav{ 1973250079Scarl uint32_t status; 1974250079Scarl 1975250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1976300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1977300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1978300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1979300373Smav ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1980250079Scarl 1981250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1982250079Scarl pause("ModPhy", hz / 10); 1983250079Scarl 1984250079Scarl /* Clear AER Errors, write to clear */ 1985300373Smav status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1986250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1987300373Smav ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1988250079Scarl 1989250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1990300373Smav status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1991300373Smav status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1992300373Smav ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1993250079Scarl 1994250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1995300373Smav status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1996300373Smav status |= ATOM_DESKEWSTS_DBERR; 1997300373Smav ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1998250079Scarl 1999300373Smav status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2000300373Smav status &= ATOM_IBIST_ERR_OFLOW; 2001300373Smav ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2002250079Scarl 2003250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 2004300373Smav status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2005300373Smav status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2006300373Smav ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2007250079Scarl} 2008250079Scarl 2009300373Smav/* 2010300373Smav * ntb_set_ctx() - associate a driver context with an ntb device 2011300373Smav * @ntb: NTB device context 2012300373Smav * @ctx: Driver context 2013300373Smav * @ctx_ops: Driver context operations 2014300373Smav * 2015300373Smav * Associate a driver context and operations with a ntb device. The context is 2016300373Smav * provided by the client driver, and the driver may associate a different 2017300373Smav * context with each ntb device. 2018300373Smav * 2019300373Smav * Return: Zero if the context is associated, otherwise an error number. 2020300373Smav */ 2021300373Smavint 2022300373Smavntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 2023300373Smav{ 2024300373Smav 2025300373Smav if (ctx == NULL || ops == NULL) 2026300373Smav return (EINVAL); 2027300373Smav if (ntb->ctx_ops != NULL) 2028300373Smav return (EINVAL); 2029300373Smav 2030300373Smav CTX_LOCK(ntb); 2031300373Smav if (ntb->ctx_ops != NULL) { 2032300373Smav CTX_UNLOCK(ntb); 2033300373Smav return (EINVAL); 2034300373Smav } 2035300373Smav ntb->ntb_ctx = ctx; 2036300373Smav ntb->ctx_ops = ops; 2037300373Smav CTX_UNLOCK(ntb); 2038300373Smav 2039300373Smav return (0); 2040300373Smav} 2041300373Smav 2042300373Smav/* 2043300373Smav * It is expected that this will only be used from contexts where the ctx_lock 2044300373Smav * is not needed to protect ntb_ctx lifetime. 2045300373Smav */ 2046300373Smavvoid * 2047300373Smavntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 2048300373Smav{ 2049300373Smav 2050300373Smav KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2051300373Smav if (ops != NULL) 2052300373Smav *ops = ntb->ctx_ops; 2053300373Smav return (ntb->ntb_ctx); 2054300373Smav} 2055300373Smav 2056300373Smav/* 2057300373Smav * ntb_clear_ctx() - disassociate any driver context from an ntb device 2058300373Smav * @ntb: NTB device context 2059300373Smav * 2060300373Smav * Clear any association that may exist between a driver context and the ntb 2061300373Smav * device. 2062300373Smav */ 2063300373Smavvoid 2064300373Smavntb_clear_ctx(struct ntb_softc *ntb) 2065300373Smav{ 2066300373Smav 2067300373Smav CTX_LOCK(ntb); 2068300373Smav ntb->ntb_ctx = NULL; 2069300373Smav ntb->ctx_ops = NULL; 2070300373Smav CTX_UNLOCK(ntb); 2071300373Smav} 2072300373Smav 2073300373Smav/* 2074300373Smav * ntb_link_event() - notify driver context of a change in link status 2075300373Smav * @ntb: NTB device context 2076300373Smav * 2077300373Smav * Notify the driver context that the link status may have changed. The driver 2078300373Smav * should call ntb_link_is_up() to get the current status. 2079300373Smav */ 2080300373Smavvoid 2081300373Smavntb_link_event(struct ntb_softc *ntb) 2082300373Smav{ 2083300373Smav 2084300373Smav CTX_LOCK(ntb); 2085300373Smav if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2086300373Smav ntb->ctx_ops->link_event(ntb->ntb_ctx); 2087300373Smav CTX_UNLOCK(ntb); 2088300373Smav} 2089300373Smav 2090300373Smav/* 2091300373Smav * ntb_db_event() - notify driver context of a doorbell event 2092300373Smav * @ntb: NTB device context 2093300373Smav * @vector: Interrupt vector number 2094300373Smav * 2095300373Smav * Notify the driver context of a doorbell event. If hardware supports 2096300373Smav * multiple interrupt vectors for doorbells, the vector number indicates which 2097300373Smav * vector received the interrupt. The vector number is relative to the first 2098300373Smav * vector used for doorbells, starting at zero, and must be less than 2099300373Smav * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2100300373Smav * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2101300373Smav * those bits are associated with the vector number. 2102300373Smav */ 2103250079Scarlstatic void 2104300373Smavntb_db_event(struct ntb_softc *ntb, uint32_t vec) 2105250079Scarl{ 2106250079Scarl 2107300373Smav CTX_LOCK(ntb); 2108300373Smav if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2109300373Smav ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2110300373Smav CTX_UNLOCK(ntb); 2111300373Smav} 2112250079Scarl 2113300373Smav/* 2114300373Smav * ntb_link_enable() - enable the link on the secondary side of the ntb 2115300373Smav * @ntb: NTB device context 2116300373Smav * @max_speed: The maximum link speed expressed as PCIe generation number[0] 2117300373Smav * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 2118300373Smav * 2119300373Smav * Enable the link on the secondary side of the ntb. This can only be done 2120300373Smav * from the primary side of the ntb in primary or b2b topology. The ntb device 2121300373Smav * should train the link to its maximum speed and width, or the requested speed 2122300373Smav * and width, whichever is smaller, if supported. 2123300373Smav * 2124300373Smav * Return: Zero on success, otherwise an error number. 2125300373Smav * 2126300373Smav * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 2127300373Smav * and width input will be ignored. 2128300373Smav */ 2129300373Smavint 2130300373Smavntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 2131300373Smav enum ntb_width w __unused) 2132300373Smav{ 2133300373Smav uint32_t cntl; 2134250079Scarl 2135301811Sngie ntb_printf(2, "%s\n", __func__); 2136301811Sngie 2137300373Smav if (ntb->type == NTB_ATOM) { 2138300373Smav pci_write_config(ntb->device, NTB_PPD_OFFSET, 2139300373Smav ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2140300373Smav return (0); 2141250079Scarl } 2142250079Scarl 2143300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2144300373Smav ntb_link_event(ntb); 2145300373Smav return (0); 2146300373Smav } 2147300373Smav 2148300373Smav cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2149300373Smav cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2150300373Smav cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2151300373Smav cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2152300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 2153300373Smav cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2154300373Smav ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2155300373Smav return (0); 2156250079Scarl} 2157250079Scarl 2158300373Smav/* 2159300373Smav * ntb_link_disable() - disable the link on the secondary side of the ntb 2160300373Smav * @ntb: NTB device context 2161300373Smav * 2162300373Smav * Disable the link on the secondary side of the ntb. This can only be done 2163300373Smav * from the primary side of the ntb in primary or b2b topology. The ntb device 2164300373Smav * should disable the link. Returning from this call must indicate that a 2165300373Smav * barrier has passed, though with no more writes may pass in either direction 2166300373Smav * across the link, except if this call returns an error number. 2167300373Smav * 2168300373Smav * Return: Zero on success, otherwise an error number. 2169300373Smav */ 2170300373Smavint 2171300373Smavntb_link_disable(struct ntb_softc *ntb) 2172300373Smav{ 2173300373Smav uint32_t cntl; 2174300373Smav 2175301811Sngie ntb_printf(2, "%s\n", __func__); 2176301811Sngie 2177300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2178300373Smav ntb_link_event(ntb); 2179300373Smav return (0); 2180300373Smav } 2181300373Smav 2182300373Smav cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2183300373Smav cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2184300373Smav cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2185300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) 2186300373Smav cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2187300373Smav cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2188300373Smav ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2189300373Smav return (0); 2190300373Smav} 2191300373Smav 2192301811Sngiebool 2193301811Sngientb_link_enabled(struct ntb_softc *ntb) 2194301811Sngie{ 2195301811Sngie uint32_t cntl; 2196301811Sngie 2197301811Sngie if (ntb->type == NTB_ATOM) { 2198301811Sngie cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2199301811Sngie return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2200301811Sngie } 2201301811Sngie 2202301811Sngie if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2203301811Sngie return (true); 2204301811Sngie 2205301811Sngie cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2206301811Sngie return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2207301811Sngie} 2208301811Sngie 2209250079Scarlstatic void 2210300373Smavrecover_atom_link(void *arg) 2211250079Scarl{ 2212250079Scarl struct ntb_softc *ntb = arg; 2213300373Smav unsigned speed, width, oldspeed, oldwidth; 2214250079Scarl uint32_t status32; 2215250079Scarl 2216300373Smav atom_perform_link_restart(ntb); 2217250079Scarl 2218300373Smav /* 2219300373Smav * There is a potential race between the 2 NTB devices recovering at 2220300373Smav * the same time. If the times are the same, the link will not recover 2221300373Smav * and the driver will be stuck in this loop forever. Add a random 2222300373Smav * interval to the recovery time to prevent this race. 2223300373Smav */ 2224300373Smav status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2225300373Smav pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2226250079Scarl 2227300373Smav if (atom_link_is_err(ntb)) 2228250079Scarl goto retry; 2229250079Scarl 2230300373Smav status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2231300373Smav if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2232300373Smav goto out; 2233300373Smav 2234300373Smav status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2235300373Smav width = NTB_LNK_STA_WIDTH(status32); 2236300373Smav speed = status32 & NTB_LINK_SPEED_MASK; 2237300373Smav 2238300373Smav oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2239300373Smav oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2240300373Smav if (oldwidth != width || oldspeed != speed) 2241250079Scarl goto retry; 2242250079Scarl 2243300373Smavout: 2244300373Smav callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2245300373Smav ntb); 2246250079Scarl return; 2247250079Scarl 2248250079Scarlretry: 2249300373Smav callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2250250079Scarl ntb); 2251250079Scarl} 2252250079Scarl 2253300373Smav/* 2254300373Smav * Polls the HW link status register(s); returns true if something has changed. 2255300373Smav */ 2256300373Smavstatic bool 2257300373Smavntb_poll_link(struct ntb_softc *ntb) 2258250079Scarl{ 2259250079Scarl uint32_t ntb_cntl; 2260300373Smav uint16_t reg_val; 2261250079Scarl 2262300373Smav if (ntb->type == NTB_ATOM) { 2263300373Smav ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2264300373Smav if (ntb_cntl == ntb->ntb_ctl) 2265300373Smav return (false); 2266300373Smav 2267300373Smav ntb->ntb_ctl = ntb_cntl; 2268300373Smav ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2269250079Scarl } else { 2270300373Smav db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2271250079Scarl 2272300373Smav reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2273300373Smav if (reg_val == ntb->lnk_sta) 2274300373Smav return (false); 2275300373Smav 2276300373Smav ntb->lnk_sta = reg_val; 2277301811Sngie 2278301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 2279301811Sngie if (_xeon_link_is_up(ntb)) { 2280301811Sngie if (!ntb->peer_msix_good) { 2281301811Sngie callout_reset(&ntb->peer_msix_work, 0, 2282301811Sngie ntb_exchange_msix, ntb); 2283301811Sngie return (false); 2284301811Sngie } 2285301811Sngie } else { 2286301811Sngie ntb->peer_msix_good = false; 2287301811Sngie ntb->peer_msix_done = false; 2288301811Sngie } 2289301811Sngie } 2290250079Scarl } 2291300373Smav return (true); 2292300373Smav} 2293250079Scarl 2294300373Smavstatic inline enum ntb_speed 2295300373Smavntb_link_sta_speed(struct ntb_softc *ntb) 2296300373Smav{ 2297250079Scarl 2298300373Smav if (!link_is_up(ntb)) 2299300373Smav return (NTB_SPEED_NONE); 2300300373Smav return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2301250079Scarl} 2302250079Scarl 2303300373Smavstatic inline enum ntb_width 2304300373Smavntb_link_sta_width(struct ntb_softc *ntb) 2305250079Scarl{ 2306250079Scarl 2307300373Smav if (!link_is_up(ntb)) 2308300373Smav return (NTB_WIDTH_NONE); 2309300373Smav return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2310300373Smav} 2311250079Scarl 2312300373SmavSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2313300373Smav "Driver state, statistics, and HW registers"); 2314250079Scarl 2315300373Smav#define NTB_REGSZ_MASK (3ul << 30) 2316300373Smav#define NTB_REG_64 (1ul << 30) 2317300373Smav#define NTB_REG_32 (2ul << 30) 2318300373Smav#define NTB_REG_16 (3ul << 30) 2319300373Smav#define NTB_REG_8 (0ul << 30) 2320250079Scarl 2321300373Smav#define NTB_DB_READ (1ul << 29) 2322300373Smav#define NTB_PCI_REG (1ul << 28) 2323300373Smav#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2324300373Smav 2325300373Smavstatic void 2326300373Smavntb_sysctl_init(struct ntb_softc *ntb) 2327250079Scarl{ 2328301811Sngie struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2329300373Smav struct sysctl_ctx_list *ctx; 2330300373Smav struct sysctl_oid *tree, *tmptree; 2331250079Scarl 2332300373Smav ctx = device_get_sysctl_ctx(ntb->device); 2333301811Sngie globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2334250079Scarl 2335301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2336301811Sngie CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2337301811Sngie sysctl_handle_link_status_human, "A", 2338301811Sngie "Link status (human readable)"); 2339301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2340301811Sngie CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2341301811Sngie "IU", "Link status (1=active, 0=inactive)"); 2342301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2343301811Sngie CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2344301811Sngie "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2345301811Sngie 2346301811Sngie tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2347301811Sngie CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2348300373Smav tree_par = SYSCTL_CHILDREN(tree); 2349250079Scarl 2350300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2351300373Smav &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2352300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2353300373Smav &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2354300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2355300373Smav &ntb->ppd, 0, "Raw PPD register (cached)"); 2356300373Smav 2357300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2358300373Smav#ifdef notyet 2359300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2360300373Smav &ntb->b2b_mw_idx, 0, 2361300373Smav "Index of the MW used for B2B remote register access"); 2362300373Smav#endif 2363300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2364300373Smav CTLFLAG_RD, &ntb->b2b_off, 2365300373Smav "If non-zero, offset of B2B register region in shared MW"); 2366250079Scarl } 2367250079Scarl 2368300373Smav SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2369300373Smav CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2370300373Smav "Features/errata of this NTB device"); 2371250079Scarl 2372300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2373300373Smav __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2374300373Smav "NTB CTL register (cached)"); 2375300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2376300373Smav __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2377300373Smav "LNK STA register (cached)"); 2378250079Scarl 2379300373Smav#ifdef notyet 2380300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2381300373Smav &ntb->mw_count, 0, "MW count"); 2382300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2383300373Smav &ntb->spad_count, 0, "Scratchpad count"); 2384300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2385300373Smav &ntb->db_count, 0, "Doorbell count"); 2386300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2387300373Smav &ntb->db_vec_count, 0, "Doorbell vector count"); 2388300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2389300373Smav &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2390300373Smav#endif 2391250079Scarl 2392300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2393300373Smav &ntb->db_valid_mask, "Doorbell valid mask"); 2394300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2395300373Smav &ntb->db_link_mask, "Doorbell link mask"); 2396300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2397300373Smav &ntb->db_mask, "Doorbell mask (cached)"); 2398300373Smav 2399300373Smav tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2400300373Smav CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2401300373Smav regpar = SYSCTL_CHILDREN(tmptree); 2402300373Smav 2403300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2404300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2405300373Smav ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2406300373Smav "NTB Control register"); 2407300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2408300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2409300373Smav 0x19c, sysctl_handle_register, "IU", 2410300373Smav "NTB Link Capabilities"); 2411300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2412300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2413300373Smav 0x1a0, sysctl_handle_register, "IU", 2414300373Smav "NTB Link Control register"); 2415300373Smav 2416300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2417300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2418300373Smav NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2419300373Smav sysctl_handle_register, "QU", "Doorbell mask register"); 2420300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2421300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2422300373Smav NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2423300373Smav sysctl_handle_register, "QU", "Doorbell register"); 2424300373Smav 2425300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2426300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2427300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2428300373Smav sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2429300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2430300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2431300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2432300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2433300373Smav sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2434300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2435300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2436300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2437300373Smav sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2438300373Smav } else { 2439300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2440300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2441300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2442300373Smav sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2443300373Smav } 2444300373Smav 2445300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2446300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2447300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2448300373Smav sysctl_handle_register, "QU", "Incoming LMT23 register"); 2449300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2450300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2451300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2452300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2453300373Smav sysctl_handle_register, "IU", "Incoming LMT4 register"); 2454300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2455300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2456300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2457300373Smav sysctl_handle_register, "IU", "Incoming LMT5 register"); 2458300373Smav } else { 2459300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2460300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2461300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2462300373Smav sysctl_handle_register, "QU", "Incoming LMT45 register"); 2463300373Smav } 2464300373Smav 2465300373Smav if (ntb->type == NTB_ATOM) 2466250079Scarl return; 2467250079Scarl 2468300373Smav tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2469300373Smav CTLFLAG_RD, NULL, "Xeon HW statistics"); 2470300373Smav statpar = SYSCTL_CHILDREN(tmptree); 2471300373Smav SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2472300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2473300373Smav NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2474300373Smav sysctl_handle_register, "SU", "Upstream Memory Miss"); 2475250079Scarl 2476300373Smav tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2477300373Smav CTLFLAG_RD, NULL, "Xeon HW errors"); 2478300373Smav errpar = SYSCTL_CHILDREN(tmptree); 2479300373Smav 2480300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2481300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2482300373Smav NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2483300373Smav sysctl_handle_register, "CU", "PPD"); 2484300373Smav 2485300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2486300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2487300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2488300373Smav sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2489300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2490300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2491300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2492300373Smav sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2493300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2494300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2495300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2496300373Smav sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2497300373Smav 2498300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2499300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2500300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2501300373Smav sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2502300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2503300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2504300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2505300373Smav sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2506300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2507300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2508300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2509300373Smav sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2510300373Smav 2511300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2512300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2513300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2514300373Smav sysctl_handle_register, "SU", "DEVSTS"); 2515300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2516300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2517300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2518300373Smav sysctl_handle_register, "SU", "LNKSTS"); 2519300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2520300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2521300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2522300373Smav sysctl_handle_register, "SU", "SLNKSTS"); 2523300373Smav 2524300373Smav SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2525300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2526300373Smav NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2527300373Smav sysctl_handle_register, "IU", "UNCERRSTS"); 2528300373Smav SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2529300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2530300373Smav NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2531300373Smav sysctl_handle_register, "IU", "CORERRSTS"); 2532300373Smav 2533300373Smav if (ntb->conn_type != NTB_CONN_B2B) 2534300373Smav return; 2535300373Smav 2536300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2537300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2538300373Smav NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2539300373Smav sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2540300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2541300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2542300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2543300373Smav NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2544300373Smav sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2545300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2546300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2547300373Smav NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2548300373Smav sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2549300373Smav } else { 2550300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2551300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2552300373Smav NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2553300373Smav sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2554300373Smav } 2555300373Smav 2556300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2557300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2558300373Smav NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2559300373Smav sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2560300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2561300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2562300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2563300373Smav NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2564300373Smav sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2565300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2566300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2567300373Smav NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2568300373Smav sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2569300373Smav } else { 2570300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2571300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2572300373Smav NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2573300373Smav sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2574300373Smav } 2575300373Smav 2576300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2577300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2578300373Smav NTB_REG_64 | ntb->xlat_reg->bar0_base, 2579300373Smav sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2580300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2581300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2582300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_base, 2583300373Smav sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2584300373Smav if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2585300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2586300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2587300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_base, 2588300373Smav sysctl_handle_register, "IU", 2589300373Smav "Secondary BAR4 base register"); 2590300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2591300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2592300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_base, 2593300373Smav sysctl_handle_register, "IU", 2594300373Smav "Secondary BAR5 base register"); 2595300373Smav } else { 2596300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2597300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2598300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_base, 2599300373Smav sysctl_handle_register, "QU", 2600300373Smav "Secondary BAR45 base register"); 2601300373Smav } 2602250079Scarl} 2603250079Scarl 2604300373Smavstatic int 2605300373Smavsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2606250079Scarl{ 2607304377Smav struct ntb_softc *ntb = arg1; 2608300373Smav struct sbuf sb; 2609300373Smav int error; 2610250079Scarl 2611300373Smav sbuf_new_for_sysctl(&sb, NULL, 256, req); 2612300373Smav 2613300373Smav sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2614300373Smav error = sbuf_finish(&sb); 2615300373Smav sbuf_delete(&sb); 2616300373Smav 2617300373Smav if (error || !req->newptr) 2618300373Smav return (error); 2619300373Smav return (EINVAL); 2620250079Scarl} 2621250079Scarl 2622300373Smavstatic int 2623301811Sngiesysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2624250079Scarl{ 2625304377Smav struct ntb_softc *ntb = arg1; 2626301811Sngie unsigned old, new; 2627301811Sngie int error; 2628301811Sngie 2629301811Sngie old = ntb_link_enabled(ntb); 2630301811Sngie 2631301811Sngie error = SYSCTL_OUT(req, &old, sizeof(old)); 2632301811Sngie if (error != 0 || req->newptr == NULL) 2633301811Sngie return (error); 2634301811Sngie 2635301811Sngie error = SYSCTL_IN(req, &new, sizeof(new)); 2636301811Sngie if (error != 0) 2637301811Sngie return (error); 2638301811Sngie 2639301811Sngie ntb_printf(0, "Admin set interface state to '%sabled'\n", 2640301811Sngie (new != 0)? "en" : "dis"); 2641301811Sngie 2642301811Sngie if (new != 0) 2643301811Sngie error = ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2644301811Sngie else 2645301811Sngie error = ntb_link_disable(ntb); 2646301811Sngie return (error); 2647301811Sngie} 2648301811Sngie 2649301811Sngiestatic int 2650301811Sngiesysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2651301811Sngie{ 2652304377Smav struct ntb_softc *ntb = arg1; 2653300373Smav struct sbuf sb; 2654300373Smav enum ntb_speed speed; 2655300373Smav enum ntb_width width; 2656300373Smav int error; 2657250079Scarl 2658300373Smav sbuf_new_for_sysctl(&sb, NULL, 32, req); 2659300373Smav 2660300373Smav if (ntb_link_is_up(ntb, &speed, &width)) 2661300373Smav sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2662300373Smav (unsigned)speed, (unsigned)width); 2663300373Smav else 2664300373Smav sbuf_printf(&sb, "down"); 2665300373Smav 2666300373Smav error = sbuf_finish(&sb); 2667300373Smav sbuf_delete(&sb); 2668300373Smav 2669300373Smav if (error || !req->newptr) 2670300373Smav return (error); 2671300373Smav return (EINVAL); 2672250079Scarl} 2673250079Scarl 2674300373Smavstatic int 2675301811Sngiesysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2676301811Sngie{ 2677304377Smav struct ntb_softc *ntb = arg1; 2678301811Sngie unsigned res; 2679301811Sngie int error; 2680301811Sngie 2681301811Sngie res = ntb_link_is_up(ntb, NULL, NULL); 2682301811Sngie 2683301811Sngie error = SYSCTL_OUT(req, &res, sizeof(res)); 2684301811Sngie if (error || !req->newptr) 2685301811Sngie return (error); 2686301811Sngie return (EINVAL); 2687301811Sngie} 2688301811Sngie 2689301811Sngiestatic int 2690300373Smavsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2691250079Scarl{ 2692300373Smav struct ntb_softc *ntb; 2693300373Smav const void *outp; 2694300373Smav uintptr_t sz; 2695300373Smav uint64_t umv; 2696300373Smav char be[sizeof(umv)]; 2697300373Smav size_t outsz; 2698300373Smav uint32_t reg; 2699300373Smav bool db, pci; 2700300373Smav int error; 2701250079Scarl 2702300373Smav ntb = arg1; 2703300373Smav reg = arg2 & ~NTB_REGFLAGS_MASK; 2704300373Smav sz = arg2 & NTB_REGSZ_MASK; 2705300373Smav db = (arg2 & NTB_DB_READ) != 0; 2706300373Smav pci = (arg2 & NTB_PCI_REG) != 0; 2707250079Scarl 2708300373Smav KASSERT(!(db && pci), ("bogus")); 2709250079Scarl 2710300373Smav if (db) { 2711300373Smav KASSERT(sz == NTB_REG_64, ("bogus")); 2712300373Smav umv = db_ioread(ntb, reg); 2713300373Smav outsz = sizeof(uint64_t); 2714300373Smav } else { 2715300373Smav switch (sz) { 2716300373Smav case NTB_REG_64: 2717300373Smav if (pci) 2718300373Smav umv = pci_read_config(ntb->device, reg, 8); 2719300373Smav else 2720300373Smav umv = ntb_reg_read(8, reg); 2721300373Smav outsz = sizeof(uint64_t); 2722300373Smav break; 2723300373Smav case NTB_REG_32: 2724300373Smav if (pci) 2725300373Smav umv = pci_read_config(ntb->device, reg, 4); 2726300373Smav else 2727300373Smav umv = ntb_reg_read(4, reg); 2728300373Smav outsz = sizeof(uint32_t); 2729300373Smav break; 2730300373Smav case NTB_REG_16: 2731300373Smav if (pci) 2732300373Smav umv = pci_read_config(ntb->device, reg, 2); 2733300373Smav else 2734300373Smav umv = ntb_reg_read(2, reg); 2735300373Smav outsz = sizeof(uint16_t); 2736300373Smav break; 2737300373Smav case NTB_REG_8: 2738300373Smav if (pci) 2739300373Smav umv = pci_read_config(ntb->device, reg, 1); 2740300373Smav else 2741300373Smav umv = ntb_reg_read(1, reg); 2742300373Smav outsz = sizeof(uint8_t); 2743300373Smav break; 2744300373Smav default: 2745300373Smav panic("bogus"); 2746300373Smav break; 2747300373Smav } 2748300373Smav } 2749300373Smav 2750300373Smav /* Encode bigendian so that sysctl -x is legible. */ 2751300373Smav be64enc(be, umv); 2752300373Smav outp = ((char *)be) + sizeof(umv) - outsz; 2753300373Smav 2754300373Smav error = SYSCTL_OUT(req, outp, outsz); 2755300373Smav if (error || !req->newptr) 2756300373Smav return (error); 2757300373Smav return (EINVAL); 2758250079Scarl} 2759250079Scarl 2760300373Smavstatic unsigned 2761300373Smavntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2762300373Smav{ 2763300373Smav 2764301811Sngie if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2765301811Sngie uidx >= ntb->b2b_mw_idx) || 2766301811Sngie (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2767301811Sngie uidx++; 2768301811Sngie if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2769301811Sngie uidx >= ntb->b2b_mw_idx) && 2770301811Sngie (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2771301811Sngie uidx++; 2772300373Smav return (uidx); 2773300373Smav} 2774300373Smav 2775301811Sngiestatic void 2776301811Sngientb_exchange_msix(void *ctx) 2777301811Sngie{ 2778301811Sngie struct ntb_softc *ntb; 2779301811Sngie uint32_t val; 2780301811Sngie unsigned i; 2781301811Sngie 2782301811Sngie ntb = ctx; 2783301811Sngie 2784301903Smav if (ntb->peer_msix_good) 2785301903Smav goto msix_good; 2786301811Sngie if (ntb->peer_msix_done) 2787301811Sngie goto msix_done; 2788301811Sngie 2789301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2790301811Sngie ntb_peer_spad_write(ntb, NTB_MSIX_DATA0 + i, 2791301811Sngie ntb->msix_data[i].nmd_data); 2792301811Sngie ntb_peer_spad_write(ntb, NTB_MSIX_OFS0 + i, 2793301904Smav ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2794301811Sngie } 2795301811Sngie ntb_peer_spad_write(ntb, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2796301811Sngie 2797301811Sngie ntb_spad_read(ntb, NTB_MSIX_GUARD, &val); 2798301811Sngie if (val != NTB_MSIX_VER_GUARD) 2799301811Sngie goto reschedule; 2800301811Sngie 2801301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2802301811Sngie ntb_spad_read(ntb, NTB_MSIX_DATA0 + i, &val); 2803301904Smav ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2804301811Sngie ntb->peer_msix_data[i].nmd_data = val; 2805301811Sngie ntb_spad_read(ntb, NTB_MSIX_OFS0 + i, &val); 2806301904Smav ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2807301811Sngie ntb->peer_msix_data[i].nmd_ofs = val; 2808301811Sngie } 2809301811Sngie 2810301811Sngie ntb->peer_msix_done = true; 2811301811Sngie 2812301811Sngiemsix_done: 2813301811Sngie ntb_peer_spad_write(ntb, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2814301811Sngie ntb_spad_read(ntb, NTB_MSIX_DONE, &val); 2815301811Sngie if (val != NTB_MSIX_RECEIVED) 2816301811Sngie goto reschedule; 2817301811Sngie 2818301811Sngie ntb->peer_msix_good = true; 2819301903Smav /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2820301903Smav goto reschedule; 2821301811Sngie 2822301903Smavmsix_good: 2823301811Sngie ntb_poll_link(ntb); 2824301811Sngie ntb_link_event(ntb); 2825301811Sngie return; 2826301811Sngie 2827301811Sngiereschedule: 2828301811Sngie ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2829301903Smav if (_xeon_link_is_up(ntb)) { 2830301903Smav callout_reset(&ntb->peer_msix_work, 2831301903Smav hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2832301903Smav ntb_exchange_msix, ntb); 2833301903Smav } else 2834301811Sngie ntb_spad_clear(ntb); 2835301811Sngie} 2836301811Sngie 2837300373Smav/* 2838300373Smav * Public API to the rest of the OS 2839300373Smav */ 2840300373Smav 2841250079Scarl/** 2842250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 2843250079Scarl * @ntb: pointer to ntb_softc instance 2844250079Scarl * 2845250079Scarl * This function returns the max 32bit scratchpad registers usable by the 2846250079Scarl * upper layer. 2847250079Scarl * 2848250079Scarl * RETURNS: total number of scratch pad registers available 2849250079Scarl */ 2850300373Smavuint8_t 2851250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 2852250079Scarl{ 2853250079Scarl 2854300373Smav return (ntb->spad_count); 2855250079Scarl} 2856250079Scarl 2857300373Smav/* 2858300373Smav * ntb_mw_count() - Get the number of memory windows available for KPI 2859300373Smav * consumers. 2860300373Smav * 2861300373Smav * (Excludes any MW wholly reserved for register access.) 2862300373Smav */ 2863300373Smavuint8_t 2864300373Smavntb_mw_count(struct ntb_softc *ntb) 2865300373Smav{ 2866301811Sngie uint8_t res; 2867300373Smav 2868301811Sngie res = ntb->mw_count; 2869300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2870301811Sngie res--; 2871301811Sngie if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2872301811Sngie res--; 2873301811Sngie return (res); 2874300373Smav} 2875300373Smav 2876250079Scarl/** 2877300373Smav * ntb_spad_write() - write to the secondary scratchpad register 2878250079Scarl * @ntb: pointer to ntb_softc instance 2879250079Scarl * @idx: index to the scratchpad register, 0 based 2880250079Scarl * @val: the data value to put into the register 2881250079Scarl * 2882250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2883250079Scarl * register. The register resides on the secondary (external) side. 2884250079Scarl * 2885300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2886250079Scarl */ 2887250079Scarlint 2888300373Smavntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2889250079Scarl{ 2890250079Scarl 2891300373Smav if (idx >= ntb->spad_count) 2892250079Scarl return (EINVAL); 2893250079Scarl 2894300373Smav ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2895250079Scarl 2896250079Scarl return (0); 2897250079Scarl} 2898250079Scarl 2899301811Sngie/* 2900301811Sngie * Zeros the local scratchpad. 2901301811Sngie */ 2902301811Sngievoid 2903301811Sngientb_spad_clear(struct ntb_softc *ntb) 2904301811Sngie{ 2905301811Sngie unsigned i; 2906301811Sngie 2907301811Sngie for (i = 0; i < ntb->spad_count; i++) 2908301811Sngie ntb_spad_write(ntb, i, 0); 2909301811Sngie} 2910301811Sngie 2911250079Scarl/** 2912300373Smav * ntb_spad_read() - read from the primary scratchpad register 2913250079Scarl * @ntb: pointer to ntb_softc instance 2914250079Scarl * @idx: index to scratchpad register, 0 based 2915250079Scarl * @val: pointer to 32bit integer for storing the register value 2916250079Scarl * 2917250079Scarl * This function allows reading of the 32bit scratchpad register on 2918250079Scarl * the primary (internal) side. 2919250079Scarl * 2920300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2921250079Scarl */ 2922250079Scarlint 2923300373Smavntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2924250079Scarl{ 2925250079Scarl 2926300373Smav if (idx >= ntb->spad_count) 2927250079Scarl return (EINVAL); 2928250079Scarl 2929300373Smav *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2930250079Scarl 2931250079Scarl return (0); 2932250079Scarl} 2933250079Scarl 2934250079Scarl/** 2935300373Smav * ntb_peer_spad_write() - write to the secondary scratchpad register 2936250079Scarl * @ntb: pointer to ntb_softc instance 2937250079Scarl * @idx: index to the scratchpad register, 0 based 2938250079Scarl * @val: the data value to put into the register 2939250079Scarl * 2940250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2941250079Scarl * register. The register resides on the secondary (external) side. 2942250079Scarl * 2943300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2944250079Scarl */ 2945250079Scarlint 2946300373Smavntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2947250079Scarl{ 2948250079Scarl 2949300373Smav if (idx >= ntb->spad_count) 2950250079Scarl return (EINVAL); 2951250079Scarl 2952300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2953300373Smav ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2954255279Scarl else 2955300373Smav ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2956250079Scarl 2957250079Scarl return (0); 2958250079Scarl} 2959250079Scarl 2960250079Scarl/** 2961300373Smav * ntb_peer_spad_read() - read from the primary scratchpad register 2962250079Scarl * @ntb: pointer to ntb_softc instance 2963250079Scarl * @idx: index to scratchpad register, 0 based 2964250079Scarl * @val: pointer to 32bit integer for storing the register value 2965250079Scarl * 2966250079Scarl * This function allows reading of the 32bit scratchpad register on 2967250079Scarl * the primary (internal) side. 2968250079Scarl * 2969300373Smav * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2970250079Scarl */ 2971250079Scarlint 2972300373Smavntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2973250079Scarl{ 2974250079Scarl 2975300373Smav if (idx >= ntb->spad_count) 2976250079Scarl return (EINVAL); 2977250079Scarl 2978300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2979300373Smav *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2980255279Scarl else 2981300373Smav *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2982250079Scarl 2983250079Scarl return (0); 2984250079Scarl} 2985250079Scarl 2986300373Smav/* 2987300373Smav * ntb_mw_get_range() - get the range of a memory window 2988300373Smav * @ntb: NTB device context 2989300373Smav * @idx: Memory window number 2990300373Smav * @base: OUT - the base address for mapping the memory window 2991300373Smav * @size: OUT - the size for mapping the memory window 2992300373Smav * @align: OUT - the base alignment for translating the memory window 2993300373Smav * @align_size: OUT - the size alignment for translating the memory window 2994250079Scarl * 2995300373Smav * Get the range of a memory window. NULL may be given for any output 2996300373Smav * parameter if the value is not needed. The base and size may be used for 2997300373Smav * mapping the memory window, to access the peer memory. The alignment and 2998300373Smav * size may be used for translating the memory window, for the peer to access 2999300373Smav * memory on the local system. 3000250079Scarl * 3001300373Smav * Return: Zero on success, otherwise an error number. 3002250079Scarl */ 3003300373Smavint 3004300373Smavntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 3005300373Smav caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 3006300373Smav bus_addr_t *plimit) 3007250079Scarl{ 3008300373Smav struct ntb_pci_bar_info *bar; 3009300373Smav bus_addr_t limit; 3010300373Smav size_t bar_b2b_off; 3011300373Smav enum ntb_bar bar_num; 3012250079Scarl 3013300373Smav if (mw_idx >= ntb_mw_count(ntb)) 3014300373Smav return (EINVAL); 3015300373Smav mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 3016250079Scarl 3017300373Smav bar_num = ntb_mw_to_bar(ntb, mw_idx); 3018300373Smav bar = &ntb->bar_info[bar_num]; 3019300373Smav bar_b2b_off = 0; 3020300373Smav if (mw_idx == ntb->b2b_mw_idx) { 3021300373Smav KASSERT(ntb->b2b_off != 0, 3022300373Smav ("user shouldn't get non-shared b2b mw")); 3023300373Smav bar_b2b_off = ntb->b2b_off; 3024300373Smav } 3025300373Smav 3026300373Smav if (bar_is_64bit(ntb, bar_num)) 3027300373Smav limit = BUS_SPACE_MAXADDR; 3028300373Smav else 3029300373Smav limit = BUS_SPACE_MAXADDR_32BIT; 3030300373Smav 3031300373Smav if (base != NULL) 3032300373Smav *base = bar->pbase + bar_b2b_off; 3033300373Smav if (vbase != NULL) 3034300373Smav *vbase = bar->vbase + bar_b2b_off; 3035300373Smav if (size != NULL) 3036300373Smav *size = bar->size - bar_b2b_off; 3037300373Smav if (align != NULL) 3038300373Smav *align = bar->size; 3039300373Smav if (align_size != NULL) 3040300373Smav *align_size = 1; 3041300373Smav if (plimit != NULL) 3042300373Smav *plimit = limit; 3043300373Smav return (0); 3044250079Scarl} 3045250079Scarl 3046300373Smav/* 3047300373Smav * ntb_mw_set_trans() - set the translation of a memory window 3048300373Smav * @ntb: NTB device context 3049300373Smav * @idx: Memory window number 3050300373Smav * @addr: The dma address local memory to expose to the peer 3051300373Smav * @size: The size of the local memory to expose to the peer 3052300373Smav * 3053300373Smav * Set the translation of a memory window. The peer may access local memory 3054300373Smav * through the window starting at the address, up to the size. The address 3055300373Smav * must be aligned to the alignment specified by ntb_mw_get_range(). The size 3056300373Smav * must be aligned to the size alignment specified by ntb_mw_get_range(). The 3057300373Smav * address must be below the plimit specified by ntb_mw_get_range() (i.e. for 3058300373Smav * 32-bit BARs). 3059300373Smav * 3060300373Smav * Return: Zero on success, otherwise an error number. 3061300373Smav */ 3062300373Smavint 3063300373Smavntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 3064300373Smav size_t size) 3065250079Scarl{ 3066300373Smav struct ntb_pci_bar_info *bar; 3067300373Smav uint64_t base, limit, reg_val; 3068300373Smav size_t bar_size, mw_size; 3069300373Smav uint32_t base_reg, xlat_reg, limit_reg; 3070300373Smav enum ntb_bar bar_num; 3071250079Scarl 3072300373Smav if (idx >= ntb_mw_count(ntb)) 3073300373Smav return (EINVAL); 3074300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 3075250079Scarl 3076300373Smav bar_num = ntb_mw_to_bar(ntb, idx); 3077300373Smav bar = &ntb->bar_info[bar_num]; 3078300373Smav 3079300373Smav bar_size = bar->size; 3080300373Smav if (idx == ntb->b2b_mw_idx) 3081300373Smav mw_size = bar_size - ntb->b2b_off; 3082300373Smav else 3083300373Smav mw_size = bar_size; 3084300373Smav 3085300373Smav /* Hardware requires that addr is aligned to bar size */ 3086300373Smav if ((addr & (bar_size - 1)) != 0) 3087300373Smav return (EINVAL); 3088300373Smav 3089300373Smav if (size > mw_size) 3090300373Smav return (EINVAL); 3091300373Smav 3092300373Smav bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 3093300373Smav 3094300373Smav limit = 0; 3095300373Smav if (bar_is_64bit(ntb, bar_num)) { 3096300373Smav base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 3097300373Smav 3098300373Smav if (limit_reg != 0 && size != mw_size) 3099300373Smav limit = base + size; 3100300373Smav 3101300373Smav /* Set and verify translation address */ 3102300373Smav ntb_reg_write(8, xlat_reg, addr); 3103300373Smav reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 3104300373Smav if (reg_val != addr) { 3105300373Smav ntb_reg_write(8, xlat_reg, 0); 3106300373Smav return (EIO); 3107300373Smav } 3108300373Smav 3109300373Smav /* Set and verify the limit */ 3110300373Smav ntb_reg_write(8, limit_reg, limit); 3111300373Smav reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3112300373Smav if (reg_val != limit) { 3113300373Smav ntb_reg_write(8, limit_reg, base); 3114300373Smav ntb_reg_write(8, xlat_reg, 0); 3115300373Smav return (EIO); 3116300373Smav } 3117300373Smav } else { 3118300373Smav /* Configure 32-bit (split) BAR MW */ 3119300373Smav 3120300373Smav if ((addr & UINT32_MAX) != addr) 3121300373Smav return (ERANGE); 3122300373Smav if (((addr + size) & UINT32_MAX) != (addr + size)) 3123300373Smav return (ERANGE); 3124300373Smav 3125300373Smav base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3126300373Smav 3127300373Smav if (limit_reg != 0 && size != mw_size) 3128300373Smav limit = base + size; 3129300373Smav 3130300373Smav /* Set and verify translation address */ 3131300373Smav ntb_reg_write(4, xlat_reg, addr); 3132300373Smav reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3133300373Smav if (reg_val != addr) { 3134300373Smav ntb_reg_write(4, xlat_reg, 0); 3135300373Smav return (EIO); 3136300373Smav } 3137300373Smav 3138300373Smav /* Set and verify the limit */ 3139300373Smav ntb_reg_write(4, limit_reg, limit); 3140300373Smav reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3141300373Smav if (reg_val != limit) { 3142300373Smav ntb_reg_write(4, limit_reg, base); 3143300373Smav ntb_reg_write(4, xlat_reg, 0); 3144300373Smav return (EIO); 3145300373Smav } 3146300373Smav } 3147300373Smav return (0); 3148250079Scarl} 3149250079Scarl 3150300373Smav/* 3151300373Smav * ntb_mw_clear_trans() - clear the translation of a memory window 3152300373Smav * @ntb: NTB device context 3153300373Smav * @idx: Memory window number 3154250079Scarl * 3155300373Smav * Clear the translation of a memory window. The peer may no longer access 3156300373Smav * local memory through the window. 3157250079Scarl * 3158300373Smav * Return: Zero on success, otherwise an error number. 3159250079Scarl */ 3160300373Smavint 3161300373Smavntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 3162250079Scarl{ 3163250079Scarl 3164300373Smav return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 3165300373Smav} 3166300373Smav 3167300373Smav/* 3168300373Smav * ntb_mw_get_wc - Get the write-combine status of a memory window 3169300373Smav * 3170300373Smav * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 3171300373Smav * idx is an invalid memory window). 3172300373Smav * 3173300373Smav * Mode is a VM_MEMATTR_* type. 3174300373Smav */ 3175300373Smavint 3176300373Smavntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t *mode) 3177300373Smav{ 3178300373Smav struct ntb_pci_bar_info *bar; 3179300373Smav 3180300373Smav if (idx >= ntb_mw_count(ntb)) 3181300373Smav return (EINVAL); 3182300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 3183300373Smav 3184300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3185300373Smav *mode = bar->map_mode; 3186300373Smav return (0); 3187300373Smav} 3188300373Smav 3189300373Smav/* 3190300373Smav * ntb_mw_set_wc - Set the write-combine status of a memory window 3191300373Smav * 3192300373Smav * If 'mode' matches the current status, this does nothing and succeeds. Mode 3193300373Smav * is a VM_MEMATTR_* type. 3194300373Smav * 3195300373Smav * Returns: Zero on success, setting the caching attribute on the virtual 3196300373Smav * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 3197300373Smav * memory window, or if changing the caching attribute fails). 3198300373Smav */ 3199300373Smavint 3200300373Smavntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3201300373Smav{ 3202300373Smav 3203300373Smav if (idx >= ntb_mw_count(ntb)) 3204300373Smav return (EINVAL); 3205300373Smav 3206300373Smav idx = ntb_user_mw_to_idx(ntb, idx); 3207300373Smav return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3208300373Smav} 3209300373Smav 3210300373Smavstatic int 3211300373Smavntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3212300373Smav{ 3213300373Smav struct ntb_pci_bar_info *bar; 3214300373Smav int rc; 3215300373Smav 3216300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3217300373Smav if (bar->map_mode == mode) 3218250079Scarl return (0); 3219250079Scarl 3220300373Smav rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3221300373Smav if (rc == 0) 3222300373Smav bar->map_mode = mode; 3223300373Smav 3224300373Smav return (rc); 3225250079Scarl} 3226250079Scarl 3227250079Scarl/** 3228300373Smav * ntb_peer_db_set() - Set the doorbell on the secondary/external side 3229250079Scarl * @ntb: pointer to ntb_softc instance 3230300373Smav * @bit: doorbell bits to ring 3231250079Scarl * 3232300373Smav * This function allows triggering of a doorbell on the secondary/external 3233300373Smav * side that will initiate an interrupt on the remote host 3234250079Scarl */ 3235250079Scarlvoid 3236300373Smavntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 3237250079Scarl{ 3238250079Scarl 3239301811Sngie if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 3240301811Sngie struct ntb_pci_bar_info *lapic; 3241301811Sngie unsigned i; 3242301811Sngie 3243301811Sngie lapic = ntb->peer_lapic_bar; 3244301811Sngie 3245301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3246301811Sngie if ((bit & ntb_db_vector_mask(ntb, i)) != 0) 3247301811Sngie bus_space_write_4(lapic->pci_bus_tag, 3248301811Sngie lapic->pci_bus_handle, 3249301811Sngie ntb->peer_msix_data[i].nmd_ofs, 3250301811Sngie ntb->peer_msix_data[i].nmd_data); 3251301811Sngie } 3252301811Sngie return; 3253301811Sngie } 3254301811Sngie 3255300373Smav if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3256300373Smav ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3257250079Scarl return; 3258300373Smav } 3259250079Scarl 3260300373Smav db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3261300373Smav} 3262300373Smav 3263300373Smav/* 3264300373Smav * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 3265300373Smav * as well as the size of the register (via *sz_out). 3266300373Smav * 3267300373Smav * This function allows a caller using I/OAT DMA to chain the remote doorbell 3268300373Smav * ring to its memory window write. 3269300373Smav * 3270300373Smav * Note that writing the peer doorbell via a memory window will *not* generate 3271300373Smav * an interrupt on the remote host; that must be done seperately. 3272300373Smav */ 3273300373Smavbus_addr_t 3274300373Smavntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 3275300373Smav{ 3276300373Smav struct ntb_pci_bar_info *bar; 3277300373Smav uint64_t regoff; 3278300373Smav 3279300373Smav KASSERT(sz_out != NULL, ("must be non-NULL")); 3280300373Smav 3281300373Smav if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3282300373Smav bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3283300373Smav regoff = ntb->peer_reg->db_bell; 3284300373Smav } else { 3285300373Smav KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3286300373Smav ("invalid b2b idx")); 3287300373Smav 3288300373Smav bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3289300373Smav regoff = XEON_PDOORBELL_OFFSET; 3290250079Scarl } 3291300373Smav KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3292300373Smav 3293300373Smav *sz_out = ntb->reg->db_size; 3294300373Smav /* HACK: Specific to current x86 bus implementation. */ 3295300373Smav return ((uint64_t)bar->pci_bus_handle + regoff); 3296250079Scarl} 3297250079Scarl 3298300373Smav/* 3299300373Smav * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 3300300373Smav * @ntb: NTB device context 3301250079Scarl * 3302300373Smav * Hardware may support different number or arrangement of doorbell bits. 3303250079Scarl * 3304300373Smav * Return: A mask of doorbell bits supported by the ntb. 3305250079Scarl */ 3306300373Smavuint64_t 3307300373Smavntb_db_valid_mask(struct ntb_softc *ntb) 3308250079Scarl{ 3309250079Scarl 3310300373Smav return (ntb->db_valid_mask); 3311250079Scarl} 3312250079Scarl 3313300373Smav/* 3314300373Smav * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 3315300373Smav * @ntb: NTB device context 3316300373Smav * @vector: Doorbell vector number 3317300373Smav * 3318300373Smav * Each interrupt vector may have a different number or arrangement of bits. 3319300373Smav * 3320300373Smav * Return: A mask of doorbell bits serviced by a vector. 3321300373Smav */ 3322300373Smavuint64_t 3323300373Smavntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 3324300373Smav{ 3325300373Smav 3326300373Smav if (vector > ntb->db_vec_count) 3327300373Smav return (0); 3328300373Smav return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3329300373Smav} 3330300373Smav 3331250079Scarl/** 3332300373Smav * ntb_link_is_up() - get the current ntb link state 3333300373Smav * @ntb: NTB device context 3334300373Smav * @speed: OUT - The link speed expressed as PCIe generation number 3335300373Smav * @width: OUT - The link width expressed as the number of PCIe lanes 3336250079Scarl * 3337250079Scarl * RETURNS: true or false based on the hardware link state 3338250079Scarl */ 3339250079Scarlbool 3340300373Smavntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 3341300373Smav enum ntb_width *width) 3342250079Scarl{ 3343250079Scarl 3344300373Smav if (speed != NULL) 3345300373Smav *speed = ntb_link_sta_speed(ntb); 3346300373Smav if (width != NULL) 3347300373Smav *width = ntb_link_sta_width(ntb); 3348300373Smav return (link_is_up(ntb)); 3349250079Scarl} 3350250079Scarl 3351255272Scarlstatic void 3352255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 3353250079Scarl{ 3354255272Scarl 3355300373Smav bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3356300373Smav bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3357300373Smav bar->pbase = rman_get_start(bar->pci_resource); 3358300373Smav bar->size = rman_get_size(bar->pci_resource); 3359300373Smav bar->vbase = rman_get_virtual(bar->pci_resource); 3360250079Scarl} 3361255268Scarl 3362300373Smavdevice_t 3363300373Smavntb_get_device(struct ntb_softc *ntb) 3364255268Scarl{ 3365255268Scarl 3366255268Scarl return (ntb->device); 3367255268Scarl} 3368300373Smav 3369300373Smav/* Export HW-specific errata information. */ 3370300373Smavbool 3371300373Smavntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 3372300373Smav{ 3373300373Smav 3374300373Smav return (HAS_FEATURE(feature)); 3375300373Smav} 3376