1250079Scarl/*- 2304380Smav * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 3250079Scarl * Copyright (C) 2013 Intel Corporation 4300373Smav * Copyright (C) 2015 EMC Corporation 5250079Scarl * All rights reserved. 6250079Scarl * 7250079Scarl * Redistribution and use in source and binary forms, with or without 8250079Scarl * modification, are permitted provided that the following conditions 9250079Scarl * are met: 10250079Scarl * 1. Redistributions of source code must retain the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer. 12250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 13250079Scarl * notice, this list of conditions and the following disclaimer in the 14250079Scarl * documentation and/or other materials provided with the distribution. 15250079Scarl * 16250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26250079Scarl * SUCH DAMAGE. 27250079Scarl */ 28250079Scarl 29304380Smav/* 30304380Smav * The Non-Transparent Bridge (NTB) is a device that allows you to connect 31304380Smav * two or more systems using a PCI-e links, providing remote memory access. 32304380Smav * 33304380Smav * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 34304380Smav * 35304380Smav * NOTE: Much of the code in this module is shared with Linux. Any patches may 36304380Smav * be picked up and redistributed in Linux with a dual GPL/BSD license. 37304380Smav */ 38304380Smav 39250079Scarl#include <sys/cdefs.h> 40250079Scarl__FBSDID("$FreeBSD: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c 314667 2017-03-04 13:03:31Z avg $"); 41250079Scarl 42250079Scarl#include <sys/param.h> 43250079Scarl#include <sys/kernel.h> 44250079Scarl#include <sys/systm.h> 45250079Scarl#include <sys/bus.h> 46300373Smav#include <sys/endian.h> 47304388Smav#include <sys/interrupt.h> 48250079Scarl#include <sys/malloc.h> 49250079Scarl#include <sys/module.h> 50301811Sngie#include <sys/mutex.h> 51301811Sngie#include <sys/pciio.h> 52250079Scarl#include <sys/queue.h> 53250079Scarl#include <sys/rman.h> 54300373Smav#include <sys/sbuf.h> 55300373Smav#include <sys/sysctl.h> 56250079Scarl#include <vm/vm.h> 57250079Scarl#include <vm/pmap.h> 58250079Scarl#include <machine/bus.h> 59301811Sngie#include <machine/intr_machdep.h> 60250079Scarl#include <machine/pmap.h> 61250079Scarl#include <machine/resource.h> 62250079Scarl#include <dev/pci/pcireg.h> 63250079Scarl#include <dev/pci/pcivar.h> 64250079Scarl 65250079Scarl#include "ntb_regs.h" 66304380Smav#include "../ntb.h" 67250079Scarl 68300373Smav#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 69250079Scarl 70300373Smav#define NTB_HB_TIMEOUT 1 /* second */ 71300373Smav#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 72300373Smav#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 73250079Scarl 74301811Sngie#define NTB_MSIX_VER_GUARD 0xaabbccdd 75301811Sngie#define NTB_MSIX_RECEIVED 0xe0f0e0f0 76301811Sngie 77301811Sngie/* 78301811Sngie * PCI constants could be somewhere more generic, but aren't defined/used in 79301811Sngie * pci.c. 80301811Sngie */ 81301811Sngie#define PCI_MSIX_ENTRY_SIZE 16 82301811Sngie#define PCI_MSIX_ENTRY_LOWER_ADDR 0 83301811Sngie#define PCI_MSIX_ENTRY_UPPER_ADDR 4 84301811Sngie#define PCI_MSIX_ENTRY_DATA 8 85301811Sngie 86250079Scarlenum ntb_device_type { 87250079Scarl NTB_XEON, 88300373Smav NTB_ATOM 89250079Scarl}; 90250079Scarl 91300373Smav/* ntb_conn_type are hardware numbers, cannot change. */ 92300373Smavenum ntb_conn_type { 93300373Smav NTB_CONN_TRANSPARENT = 0, 94300373Smav NTB_CONN_B2B = 1, 95300373Smav NTB_CONN_RP = 2, 96300373Smav}; 97300373Smav 98300373Smavenum ntb_b2b_direction { 99300373Smav NTB_DEV_USD = 0, 100300373Smav NTB_DEV_DSD = 1, 101300373Smav}; 102300373Smav 103300373Smavenum ntb_bar { 104300373Smav NTB_CONFIG_BAR = 0, 105300373Smav NTB_B2B_BAR_1, 106300373Smav NTB_B2B_BAR_2, 107300373Smav NTB_B2B_BAR_3, 108300373Smav NTB_MAX_BARS 109300373Smav}; 110300373Smav 111301811Sngieenum { 112301811Sngie NTB_MSIX_GUARD = 0, 113301811Sngie NTB_MSIX_DATA0, 114301811Sngie NTB_MSIX_DATA1, 115301811Sngie NTB_MSIX_DATA2, 116301811Sngie NTB_MSIX_OFS0, 117301811Sngie NTB_MSIX_OFS1, 118301811Sngie NTB_MSIX_OFS2, 119301811Sngie NTB_MSIX_DONE, 120301811Sngie NTB_MAX_MSIX_SPAD 121301811Sngie}; 122301811Sngie 123255274Scarl/* Device features and workarounds */ 124304380Smav#define HAS_FEATURE(ntb, feature) \ 125304380Smav (((ntb)->features & (feature)) != 0) 126255274Scarl 127250079Scarlstruct ntb_hw_info { 128250079Scarl uint32_t device_id; 129255274Scarl const char *desc; 130250079Scarl enum ntb_device_type type; 131300373Smav uint32_t features; 132250079Scarl}; 133250079Scarl 134250079Scarlstruct ntb_pci_bar_info { 135250079Scarl bus_space_tag_t pci_bus_tag; 136250079Scarl bus_space_handle_t pci_bus_handle; 137250079Scarl int pci_resource_id; 138250079Scarl struct resource *pci_resource; 139250079Scarl vm_paddr_t pbase; 140300373Smav caddr_t vbase; 141300373Smav vm_size_t size; 142300373Smav vm_memattr_t map_mode; 143300373Smav 144300373Smav /* Configuration register offsets */ 145300373Smav uint32_t psz_off; 146300373Smav uint32_t ssz_off; 147300373Smav uint32_t pbarxlat_off; 148250079Scarl}; 149250079Scarl 150250079Scarlstruct ntb_int_info { 151250079Scarl struct resource *res; 152250079Scarl int rid; 153250079Scarl void *tag; 154250079Scarl}; 155250079Scarl 156300373Smavstruct ntb_vec { 157250079Scarl struct ntb_softc *ntb; 158300373Smav uint32_t num; 159301811Sngie unsigned masked; 160250079Scarl}; 161250079Scarl 162300373Smavstruct ntb_reg { 163300373Smav uint32_t ntb_ctl; 164300373Smav uint32_t lnk_sta; 165300373Smav uint8_t db_size; 166300373Smav unsigned mw_bar[NTB_MAX_BARS]; 167300373Smav}; 168300373Smav 169300373Smavstruct ntb_alt_reg { 170300373Smav uint32_t db_bell; 171300373Smav uint32_t db_mask; 172300373Smav uint32_t spad; 173300373Smav}; 174300373Smav 175300373Smavstruct ntb_xlat_reg { 176300373Smav uint32_t bar0_base; 177300373Smav uint32_t bar2_base; 178300373Smav uint32_t bar4_base; 179300373Smav uint32_t bar5_base; 180300373Smav 181300373Smav uint32_t bar2_xlat; 182300373Smav uint32_t bar4_xlat; 183300373Smav uint32_t bar5_xlat; 184300373Smav 185300373Smav uint32_t bar2_limit; 186300373Smav uint32_t bar4_limit; 187300373Smav uint32_t bar5_limit; 188300373Smav}; 189300373Smav 190300373Smavstruct ntb_b2b_addr { 191300373Smav uint64_t bar0_addr; 192300373Smav uint64_t bar2_addr64; 193300373Smav uint64_t bar4_addr64; 194300373Smav uint64_t bar4_addr32; 195300373Smav uint64_t bar5_addr32; 196300373Smav}; 197300373Smav 198301811Sngiestruct ntb_msix_data { 199301811Sngie uint32_t nmd_ofs; 200301811Sngie uint32_t nmd_data; 201301811Sngie}; 202301811Sngie 203250079Scarlstruct ntb_softc { 204304404Smav /* ntb.c context. Do not move! Must go first! */ 205304404Smav void *ntb_store; 206304404Smav 207250079Scarl device_t device; 208250079Scarl enum ntb_device_type type; 209300373Smav uint32_t features; 210250079Scarl 211250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 212250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 213250079Scarl uint32_t allocated_interrupts; 214250079Scarl 215301811Sngie struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216301811Sngie struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 217301811Sngie bool peer_msix_good; 218301811Sngie bool peer_msix_done; 219301811Sngie struct ntb_pci_bar_info *peer_lapic_bar; 220301811Sngie struct callout peer_msix_work; 221301811Sngie 222250079Scarl struct callout heartbeat_timer; 223250079Scarl struct callout lr_timer; 224250079Scarl 225300373Smav struct ntb_vec *msix_vec; 226250079Scarl 227300373Smav uint32_t ppd; 228300373Smav enum ntb_conn_type conn_type; 229300373Smav enum ntb_b2b_direction dev_type; 230300373Smav 231300373Smav /* Offset of peer bar0 in B2B BAR */ 232300373Smav uint64_t b2b_off; 233300373Smav /* Memory window used to access peer bar0 */ 234300373Smav#define B2B_MW_DISABLED UINT8_MAX 235300373Smav uint8_t b2b_mw_idx; 236301904Smav uint32_t msix_xlat; 237301811Sngie uint8_t msix_mw_idx; 238300373Smav 239300373Smav uint8_t mw_count; 240300373Smav uint8_t spad_count; 241300373Smav uint8_t db_count; 242300373Smav uint8_t db_vec_count; 243300373Smav uint8_t db_vec_shift; 244300373Smav 245300373Smav /* Protects local db_mask. */ 246300373Smav#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 247300373Smav#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 248300373Smav#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 249300373Smav struct mtx db_mask_lock; 250300373Smav 251300373Smav volatile uint32_t ntb_ctl; 252300373Smav volatile uint32_t lnk_sta; 253300373Smav 254300373Smav uint64_t db_valid_mask; 255300373Smav uint64_t db_link_mask; 256300373Smav uint64_t db_mask; 257304388Smav uint64_t fake_db_bell; /* NTB_SB01BASE_LOCKUP*/ 258300373Smav 259300373Smav int last_ts; /* ticks @ last irq */ 260300373Smav 261300373Smav const struct ntb_reg *reg; 262300373Smav const struct ntb_alt_reg *self_reg; 263300373Smav const struct ntb_alt_reg *peer_reg; 264300373Smav const struct ntb_xlat_reg *xlat_reg; 265250079Scarl}; 266250079Scarl 267300373Smav#ifdef __i386__ 268300373Smavstatic __inline uint64_t 269300373Smavbus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 270300373Smav bus_size_t offset) 271300373Smav{ 272300373Smav 273300373Smav return (bus_space_read_4(tag, handle, offset) | 274300373Smav ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 275300373Smav} 276300373Smav 277300373Smavstatic __inline void 278300373Smavbus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 279300373Smav bus_size_t offset, uint64_t val) 280300373Smav{ 281300373Smav 282300373Smav bus_space_write_4(tag, handle, offset, val); 283300373Smav bus_space_write_4(tag, handle, offset + 4, val >> 32); 284300373Smav} 285300373Smav#endif 286300373Smav 287304404Smav#define intel_ntb_bar_read(SIZE, bar, offset) \ 288255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 289255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 290304404Smav#define intel_ntb_bar_write(SIZE, bar, offset, val) \ 291255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 292255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 293304404Smav#define intel_ntb_reg_read(SIZE, offset) \ 294304404Smav intel_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 295304404Smav#define intel_ntb_reg_write(SIZE, offset, val) \ 296304404Smav intel_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 297304404Smav#define intel_ntb_mw_read(SIZE, offset) \ 298304404Smav intel_ntb_bar_read(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 299304404Smav offset) 300304404Smav#define intel_ntb_mw_write(SIZE, offset, val) \ 301304404Smav intel_ntb_bar_write(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 302300373Smav offset, val) 303250079Scarl 304304404Smavstatic int intel_ntb_probe(device_t device); 305304404Smavstatic int intel_ntb_attach(device_t device); 306304404Smavstatic int intel_ntb_detach(device_t device); 307304404Smavstatic uint64_t intel_ntb_db_valid_mask(device_t dev); 308304404Smavstatic void intel_ntb_spad_clear(device_t dev); 309304404Smavstatic uint64_t intel_ntb_db_vector_mask(device_t dev, uint32_t vector); 310304404Smavstatic bool intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, 311304380Smav enum ntb_width *width); 312304404Smavstatic int intel_ntb_link_enable(device_t dev, enum ntb_speed speed, 313304380Smav enum ntb_width width); 314304404Smavstatic int intel_ntb_link_disable(device_t dev); 315304404Smavstatic int intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 316304404Smavstatic int intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 317304380Smav 318304404Smavstatic unsigned intel_ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 319304404Smavstatic inline enum ntb_bar intel_ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 320300373Smavstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 321300373Smavstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 322300373Smav uint32_t *base, uint32_t *xlat, uint32_t *lmt); 323304404Smavstatic int intel_ntb_map_pci_bars(struct ntb_softc *ntb); 324304404Smavstatic int intel_ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 325300373Smav vm_memattr_t); 326300373Smavstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 327300373Smav const char *); 328255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 329255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 330255272Scarl struct ntb_pci_bar_info *bar); 331304404Smavstatic void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb); 332304404Smavstatic int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 333304404Smavstatic int intel_ntb_init_isr(struct ntb_softc *ntb); 334304404Smavstatic int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 335304404Smavstatic int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 336304404Smavstatic void intel_ntb_teardown_interrupts(struct ntb_softc *ntb); 337304404Smavstatic inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 338304404Smavstatic void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec); 339300373Smavstatic void ndev_vec_isr(void *arg); 340300373Smavstatic void ndev_irq_isr(void *arg); 341300373Smavstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 342300373Smavstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 343300373Smavstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 344304404Smavstatic int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 345304404Smavstatic void intel_ntb_free_msix_vec(struct ntb_softc *ntb); 346304404Smavstatic void intel_ntb_get_msix_info(struct ntb_softc *ntb); 347304404Smavstatic void intel_ntb_exchange_msix(void *); 348304404Smavstatic struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id); 349304404Smavstatic void intel_ntb_detect_max_mw(struct ntb_softc *ntb); 350304404Smavstatic int intel_ntb_detect_xeon(struct ntb_softc *ntb); 351304404Smavstatic int intel_ntb_detect_atom(struct ntb_softc *ntb); 352304404Smavstatic int intel_ntb_xeon_init_dev(struct ntb_softc *ntb); 353304404Smavstatic int intel_ntb_atom_init_dev(struct ntb_softc *ntb); 354304404Smavstatic void intel_ntb_teardown_xeon(struct ntb_softc *ntb); 355300373Smavstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 356300373Smavstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 357300373Smav enum ntb_bar regbar); 358300373Smavstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 359300373Smav uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 360300373Smavstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 361300373Smav enum ntb_bar idx); 362300373Smavstatic int xeon_setup_b2b_mw(struct ntb_softc *, 363300373Smav const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 364300373Smavstatic inline bool link_is_up(struct ntb_softc *ntb); 365301811Sngiestatic inline bool _xeon_link_is_up(struct ntb_softc *ntb); 366300373Smavstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 367304404Smavstatic inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *); 368304404Smavstatic inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *); 369300373Smavstatic void atom_link_hb(void *arg); 370300373Smavstatic void recover_atom_link(void *arg); 371304404Smavstatic bool intel_ntb_poll_link(struct ntb_softc *ntb); 372255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 373304404Smavstatic void intel_ntb_sysctl_init(struct ntb_softc *); 374300373Smavstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 375301811Sngiestatic int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 376301811Sngiestatic int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 377300373Smavstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 378300373Smavstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 379250079Scarl 380300373Smavstatic unsigned g_ntb_hw_debug_level; 381300516SmavTUNABLE_INT("hw.ntb.debug_level", &g_ntb_hw_debug_level); 382300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 383300373Smav &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 384304404Smav#define intel_ntb_printf(lvl, ...) do { \ 385300373Smav if ((lvl) <= g_ntb_hw_debug_level) { \ 386300373Smav device_printf(ntb->device, __VA_ARGS__); \ 387300373Smav } \ 388300373Smav} while (0) 389300373Smav 390300373Smav#define _NTB_PAT_UC 0 391300373Smav#define _NTB_PAT_WC 1 392300373Smav#define _NTB_PAT_WT 4 393300373Smav#define _NTB_PAT_WP 5 394300373Smav#define _NTB_PAT_WB 6 395300373Smav#define _NTB_PAT_UCM 7 396300373Smavstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 397300516SmavTUNABLE_INT("hw.ntb.default_mw_pat", &g_ntb_mw_pat); 398300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 399300373Smav &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 400300373Smav "UC: " __XSTRING(_NTB_PAT_UC) ", " 401300373Smav "WC: " __XSTRING(_NTB_PAT_WC) ", " 402300373Smav "WT: " __XSTRING(_NTB_PAT_WT) ", " 403300373Smav "WP: " __XSTRING(_NTB_PAT_WP) ", " 404300373Smav "WB: " __XSTRING(_NTB_PAT_WB) ", " 405300373Smav "UC-: " __XSTRING(_NTB_PAT_UCM)); 406300373Smav 407300373Smavstatic inline vm_memattr_t 408304404Smavintel_ntb_pat_flags(void) 409300373Smav{ 410300373Smav 411300373Smav switch (g_ntb_mw_pat) { 412300373Smav case _NTB_PAT_WC: 413300373Smav return (VM_MEMATTR_WRITE_COMBINING); 414300373Smav case _NTB_PAT_WT: 415300373Smav return (VM_MEMATTR_WRITE_THROUGH); 416300373Smav case _NTB_PAT_WP: 417300373Smav return (VM_MEMATTR_WRITE_PROTECTED); 418300373Smav case _NTB_PAT_WB: 419300373Smav return (VM_MEMATTR_WRITE_BACK); 420300373Smav case _NTB_PAT_UCM: 421300373Smav return (VM_MEMATTR_WEAK_UNCACHEABLE); 422300373Smav case _NTB_PAT_UC: 423300373Smav /* FALLTHROUGH */ 424300373Smav default: 425300373Smav return (VM_MEMATTR_UNCACHEABLE); 426300373Smav } 427300373Smav} 428300373Smav 429300373Smav/* 430300373Smav * Well, this obviously doesn't belong here, but it doesn't seem to exist 431300373Smav * anywhere better yet. 432300373Smav */ 433300373Smavstatic inline const char * 434304404Smavintel_ntb_vm_memattr_to_str(vm_memattr_t pat) 435300373Smav{ 436300373Smav 437300373Smav switch (pat) { 438300373Smav case VM_MEMATTR_WRITE_COMBINING: 439300373Smav return ("WRITE_COMBINING"); 440300373Smav case VM_MEMATTR_WRITE_THROUGH: 441300373Smav return ("WRITE_THROUGH"); 442300373Smav case VM_MEMATTR_WRITE_PROTECTED: 443300373Smav return ("WRITE_PROTECTED"); 444300373Smav case VM_MEMATTR_WRITE_BACK: 445300373Smav return ("WRITE_BACK"); 446300373Smav case VM_MEMATTR_WEAK_UNCACHEABLE: 447300373Smav return ("UNCACHED"); 448300373Smav case VM_MEMATTR_UNCACHEABLE: 449300373Smav return ("UNCACHEABLE"); 450300373Smav default: 451300373Smav return ("UNKNOWN"); 452300373Smav } 453300373Smav} 454300373Smav 455304394Smavstatic int g_ntb_msix_idx = 1; 456304380SmavTUNABLE_INT("hw.ntb.msix_mw_idx", &g_ntb_msix_idx); 457301811SngieSYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 458301811Sngie 0, "Use this memory window to access the peer MSIX message complex on " 459301811Sngie "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 460301811Sngie "Like b2b_mw_idx, negative values index from the last available memory " 461301811Sngie "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 462301811Sngie 463300373Smavstatic int g_ntb_mw_idx = -1; 464300516SmavTUNABLE_INT("hw.ntb.b2b_mw_idx", &g_ntb_mw_idx); 465300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 466300373Smav 0, "Use this memory window to access the peer NTB registers. A " 467300373Smav "non-negative value starts from the first MW index; a negative value " 468300373Smav "starts from the last MW index. The default is -1, i.e., the last " 469300373Smav "available memory window. Both sides of the NTB MUST set the same " 470300373Smav "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 471300373Smav 472304380Smav/* Hardware owns the low 16 bits of features. */ 473304380Smav#define NTB_BAR_SIZE_4K (1 << 0) 474304380Smav#define NTB_SDOORBELL_LOCKUP (1 << 1) 475304380Smav#define NTB_SB01BASE_LOCKUP (1 << 2) 476304380Smav#define NTB_B2BDOORBELL_BIT14 (1 << 3) 477304380Smav/* Software/configuration owns the top 16 bits. */ 478304380Smav#define NTB_SPLIT_BAR (1ull << 16) 479304380Smav 480304380Smav#define NTB_FEATURES_STR \ 481304380Smav "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 482304380Smav "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 483304380Smav 484250079Scarlstatic struct ntb_hw_info pci_ids[] = { 485300373Smav /* XXX: PS/SS IDs left out until they are supported. */ 486300373Smav { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 487300373Smav NTB_ATOM, 0 }, 488300373Smav 489300373Smav { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 490300373Smav NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 491300373Smav { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 492300373Smav NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 493300373Smav { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 494300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 495300373Smav NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 496300373Smav { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 497300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 498300373Smav NTB_SB01BASE_LOCKUP }, 499300373Smav { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 500300373Smav NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 501300373Smav NTB_SB01BASE_LOCKUP }, 502300373Smav 503300373Smav { 0x00000000, NULL, NTB_ATOM, 0 } 504250079Scarl}; 505250079Scarl 506300373Smavstatic const struct ntb_reg atom_reg = { 507300373Smav .ntb_ctl = ATOM_NTBCNTL_OFFSET, 508300373Smav .lnk_sta = ATOM_LINK_STATUS_OFFSET, 509300373Smav .db_size = sizeof(uint64_t), 510300373Smav .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 511300373Smav}; 512300373Smav 513300373Smavstatic const struct ntb_alt_reg atom_pri_reg = { 514300373Smav .db_bell = ATOM_PDOORBELL_OFFSET, 515300373Smav .db_mask = ATOM_PDBMSK_OFFSET, 516300373Smav .spad = ATOM_SPAD_OFFSET, 517300373Smav}; 518300373Smav 519300373Smavstatic const struct ntb_alt_reg atom_b2b_reg = { 520300373Smav .db_bell = ATOM_B2B_DOORBELL_OFFSET, 521300373Smav .spad = ATOM_B2B_SPAD_OFFSET, 522300373Smav}; 523300373Smav 524300373Smavstatic const struct ntb_xlat_reg atom_sec_xlat = { 525300373Smav#if 0 526300373Smav /* "FIXME" says the Linux driver. */ 527300373Smav .bar0_base = ATOM_SBAR0BASE_OFFSET, 528300373Smav .bar2_base = ATOM_SBAR2BASE_OFFSET, 529300373Smav .bar4_base = ATOM_SBAR4BASE_OFFSET, 530300373Smav 531300373Smav .bar2_limit = ATOM_SBAR2LMT_OFFSET, 532300373Smav .bar4_limit = ATOM_SBAR4LMT_OFFSET, 533300373Smav#endif 534300373Smav 535300373Smav .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 536300373Smav .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 537300373Smav}; 538300373Smav 539300373Smavstatic const struct ntb_reg xeon_reg = { 540300373Smav .ntb_ctl = XEON_NTBCNTL_OFFSET, 541300373Smav .lnk_sta = XEON_LINK_STATUS_OFFSET, 542300373Smav .db_size = sizeof(uint16_t), 543300373Smav .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 544300373Smav}; 545300373Smav 546300373Smavstatic const struct ntb_alt_reg xeon_pri_reg = { 547300373Smav .db_bell = XEON_PDOORBELL_OFFSET, 548300373Smav .db_mask = XEON_PDBMSK_OFFSET, 549300373Smav .spad = XEON_SPAD_OFFSET, 550300373Smav}; 551300373Smav 552300373Smavstatic const struct ntb_alt_reg xeon_b2b_reg = { 553300373Smav .db_bell = XEON_B2B_DOORBELL_OFFSET, 554300373Smav .spad = XEON_B2B_SPAD_OFFSET, 555300373Smav}; 556300373Smav 557300373Smavstatic const struct ntb_xlat_reg xeon_sec_xlat = { 558300373Smav .bar0_base = XEON_SBAR0BASE_OFFSET, 559300373Smav .bar2_base = XEON_SBAR2BASE_OFFSET, 560300373Smav .bar4_base = XEON_SBAR4BASE_OFFSET, 561300373Smav .bar5_base = XEON_SBAR5BASE_OFFSET, 562300373Smav 563300373Smav .bar2_limit = XEON_SBAR2LMT_OFFSET, 564300373Smav .bar4_limit = XEON_SBAR4LMT_OFFSET, 565300373Smav .bar5_limit = XEON_SBAR5LMT_OFFSET, 566300373Smav 567300373Smav .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 568300373Smav .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 569300373Smav .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 570300373Smav}; 571300373Smav 572300373Smavstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 573300373Smav .bar0_addr = XEON_B2B_BAR0_ADDR, 574300373Smav .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 575300373Smav .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 576300373Smav .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 577300373Smav .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 578300373Smav}; 579300373Smav 580300373Smavstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 581300373Smav .bar0_addr = XEON_B2B_BAR0_ADDR, 582300373Smav .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 583300373Smav .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 584300373Smav .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 585300373Smav .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 586300373Smav}; 587300373Smav 588300373SmavSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 589300373Smav "B2B MW segment overrides -- MUST be the same on both sides"); 590300373Smav 591300516SmavTUNABLE_QUAD("hw.ntb.usd_bar2_addr64", &xeon_b2b_usd_addr.bar2_addr64); 592300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 593300373Smav &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 594300373Smav "hardware, use this 64-bit address on the bus between the NTB devices for " 595300373Smav "the window at BAR2, on the upstream side of the link. MUST be the same " 596300373Smav "address on both sides."); 597300516SmavTUNABLE_QUAD("hw.ntb.usd_bar4_addr64", &xeon_b2b_usd_addr.bar4_addr64); 598300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 599300373Smav &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 600300516SmavTUNABLE_QUAD("hw.ntb.usd_bar4_addr32", &xeon_b2b_usd_addr.bar4_addr32); 601300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 602300373Smav &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 603300373Smav "(split-BAR mode)."); 604300516SmavTUNABLE_QUAD("hw.ntb.usd_bar5_addr32", &xeon_b2b_usd_addr.bar5_addr32); 605300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 606300373Smav &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 607300373Smav "(split-BAR mode)."); 608300373Smav 609300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar2_addr64", &xeon_b2b_dsd_addr.bar2_addr64); 610300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 611300373Smav &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 612300373Smav "hardware, use this 64-bit address on the bus between the NTB devices for " 613300373Smav "the window at BAR2, on the downstream side of the link. MUST be the same" 614300373Smav " address on both sides."); 615300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar4_addr64", &xeon_b2b_dsd_addr.bar4_addr64); 616300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 617300373Smav &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 618300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar4_addr32", &xeon_b2b_dsd_addr.bar4_addr32); 619300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 620300373Smav &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 621300373Smav "(split-BAR mode)."); 622300516SmavTUNABLE_QUAD("hw.ntb.dsd_bar5_addr32", &xeon_b2b_dsd_addr.bar5_addr32); 623300373SmavSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 624300373Smav &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 625300373Smav "(split-BAR mode)."); 626300373Smav 627250079Scarl/* 628250079Scarl * OS <-> Driver interface structures 629250079Scarl */ 630250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 631250079Scarl 632250079Scarl/* 633250079Scarl * OS <-> Driver linkage functions 634250079Scarl */ 635250079Scarlstatic int 636304404Smavintel_ntb_probe(device_t device) 637250079Scarl{ 638300373Smav struct ntb_hw_info *p; 639250079Scarl 640304404Smav p = intel_ntb_get_device_info(pci_get_devid(device)); 641300373Smav if (p == NULL) 642250079Scarl return (ENXIO); 643300373Smav 644300373Smav device_set_desc(device, p->desc); 645300373Smav return (0); 646250079Scarl} 647250079Scarl 648250079Scarlstatic int 649304404Smavintel_ntb_attach(device_t device) 650250079Scarl{ 651300373Smav struct ntb_softc *ntb; 652300373Smav struct ntb_hw_info *p; 653250079Scarl int error; 654250079Scarl 655304380Smav ntb = device_get_softc(device); 656304404Smav p = intel_ntb_get_device_info(pci_get_devid(device)); 657300373Smav 658250079Scarl ntb->device = device; 659250079Scarl ntb->type = p->type; 660255274Scarl ntb->features = p->features; 661300373Smav ntb->b2b_mw_idx = B2B_MW_DISABLED; 662301811Sngie ntb->msix_mw_idx = B2B_MW_DISABLED; 663250079Scarl 664300373Smav /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 665314667Savg callout_init(&ntb->heartbeat_timer, 1); 666314667Savg callout_init(&ntb->lr_timer, 1); 667301811Sngie callout_init(&ntb->peer_msix_work, 1); 668300373Smav mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 669250079Scarl 670300373Smav if (ntb->type == NTB_ATOM) 671304404Smav error = intel_ntb_detect_atom(ntb); 672300373Smav else 673304404Smav error = intel_ntb_detect_xeon(ntb); 674300373Smav if (error != 0) 675300373Smav goto out; 676250079Scarl 677304404Smav intel_ntb_detect_max_mw(ntb); 678300373Smav 679250079Scarl pci_enable_busmaster(ntb->device); 680250079Scarl 681304404Smav error = intel_ntb_map_pci_bars(ntb); 682300373Smav if (error != 0) 683300373Smav goto out; 684300373Smav if (ntb->type == NTB_ATOM) 685304404Smav error = intel_ntb_atom_init_dev(ntb); 686300373Smav else 687304404Smav error = intel_ntb_xeon_init_dev(ntb); 688300373Smav if (error != 0) 689300373Smav goto out; 690300373Smav 691304404Smav intel_ntb_spad_clear(device); 692301811Sngie 693304404Smav intel_ntb_poll_link(ntb); 694300373Smav 695304404Smav intel_ntb_sysctl_init(ntb); 696300373Smav 697304380Smav /* Attach children to this controller */ 698304404Smav error = ntb_register_device(device); 699304380Smav 700300373Smavout: 701300373Smav if (error != 0) 702304404Smav intel_ntb_detach(device); 703250079Scarl return (error); 704250079Scarl} 705250079Scarl 706250079Scarlstatic int 707304404Smavintel_ntb_detach(device_t device) 708250079Scarl{ 709300373Smav struct ntb_softc *ntb; 710250079Scarl 711304380Smav ntb = device_get_softc(device); 712300373Smav 713304380Smav /* Detach & delete all children */ 714304404Smav ntb_unregister_device(device); 715304380Smav 716301811Sngie if (ntb->self_reg != NULL) { 717301811Sngie DB_MASK_LOCK(ntb); 718301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 719301811Sngie DB_MASK_UNLOCK(ntb); 720301811Sngie } 721250079Scarl callout_drain(&ntb->heartbeat_timer); 722250079Scarl callout_drain(&ntb->lr_timer); 723301811Sngie callout_drain(&ntb->peer_msix_work); 724300373Smav pci_disable_busmaster(ntb->device); 725300373Smav if (ntb->type == NTB_XEON) 726304404Smav intel_ntb_teardown_xeon(ntb); 727304404Smav intel_ntb_teardown_interrupts(ntb); 728300373Smav 729300373Smav mtx_destroy(&ntb->db_mask_lock); 730300373Smav 731304404Smav intel_ntb_unmap_pci_bar(ntb); 732250079Scarl 733250079Scarl return (0); 734250079Scarl} 735250079Scarl 736300373Smav/* 737300373Smav * Driver internal routines 738300373Smav */ 739300373Smavstatic inline enum ntb_bar 740304404Smavintel_ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 741300373Smav{ 742300373Smav 743300373Smav KASSERT(mw < ntb->mw_count, 744300373Smav ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 745300373Smav KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 746300373Smav 747300373Smav return (ntb->reg->mw_bar[mw]); 748300373Smav} 749300373Smav 750300373Smavstatic inline bool 751300373Smavbar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 752300373Smav{ 753300373Smav /* XXX This assertion could be stronger. */ 754300373Smav KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 755304380Smav return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 756300373Smav} 757300373Smav 758300373Smavstatic inline void 759300373Smavbar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 760300373Smav uint32_t *xlat, uint32_t *lmt) 761300373Smav{ 762300373Smav uint32_t basev, lmtv, xlatv; 763300373Smav 764300373Smav switch (bar) { 765300373Smav case NTB_B2B_BAR_1: 766300373Smav basev = ntb->xlat_reg->bar2_base; 767300373Smav lmtv = ntb->xlat_reg->bar2_limit; 768300373Smav xlatv = ntb->xlat_reg->bar2_xlat; 769300373Smav break; 770300373Smav case NTB_B2B_BAR_2: 771300373Smav basev = ntb->xlat_reg->bar4_base; 772300373Smav lmtv = ntb->xlat_reg->bar4_limit; 773300373Smav xlatv = ntb->xlat_reg->bar4_xlat; 774300373Smav break; 775300373Smav case NTB_B2B_BAR_3: 776300373Smav basev = ntb->xlat_reg->bar5_base; 777300373Smav lmtv = ntb->xlat_reg->bar5_limit; 778300373Smav xlatv = ntb->xlat_reg->bar5_xlat; 779300373Smav break; 780300373Smav default: 781300373Smav KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 782300373Smav ("bad bar")); 783300373Smav basev = lmtv = xlatv = 0; 784300373Smav break; 785300373Smav } 786300373Smav 787300373Smav if (base != NULL) 788300373Smav *base = basev; 789300373Smav if (xlat != NULL) 790300373Smav *xlat = xlatv; 791300373Smav if (lmt != NULL) 792300373Smav *lmt = lmtv; 793300373Smav} 794300373Smav 795250079Scarlstatic int 796304404Smavintel_ntb_map_pci_bars(struct ntb_softc *ntb) 797250079Scarl{ 798255272Scarl int rc; 799250079Scarl 800250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 801300373Smav rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 802255272Scarl if (rc != 0) 803300373Smav goto out; 804255272Scarl 805300373Smav ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 806300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 807255272Scarl if (rc != 0) 808300373Smav goto out; 809300373Smav ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 810300373Smav ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 811300373Smav ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 812255272Scarl 813300373Smav ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 814300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 815255272Scarl if (rc != 0) 816300373Smav goto out; 817300373Smav ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 818300373Smav ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 819300373Smav ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 820255274Scarl 821304380Smav if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 822300373Smav goto out; 823250079Scarl 824300373Smav ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 825300373Smav rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 826300373Smav ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 827300373Smav ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 828300373Smav ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 829255272Scarl 830300373Smavout: 831300373Smav if (rc != 0) 832255272Scarl device_printf(ntb->device, 833255272Scarl "unable to allocate pci resource\n"); 834255272Scarl return (rc); 835255272Scarl} 836255272Scarl 837300373Smavstatic void 838300373Smavprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 839300373Smav const char *kind) 840300373Smav{ 841300373Smav 842300373Smav device_printf(ntb->device, 843300373Smav "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 844300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 845300373Smav (char *)bar->vbase + bar->size - 1, 846300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 847300373Smav (uintmax_t)bar->size, kind); 848300373Smav} 849300373Smav 850255272Scarlstatic int 851255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 852255272Scarl{ 853255272Scarl 854255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 855300373Smav &bar->pci_resource_id, RF_ACTIVE); 856255272Scarl if (bar->pci_resource == NULL) 857255272Scarl return (ENXIO); 858300373Smav 859300373Smav save_bar_parameters(bar); 860300373Smav bar->map_mode = VM_MEMATTR_UNCACHEABLE; 861300373Smav print_map_success(ntb, bar, "mmr"); 862300373Smav return (0); 863255272Scarl} 864255272Scarl 865255272Scarlstatic int 866255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 867255272Scarl{ 868255272Scarl int rc; 869300373Smav vm_memattr_t mapmode; 870255276Scarl uint8_t bar_size_bits = 0; 871255272Scarl 872300373Smav bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 873300373Smav &bar->pci_resource_id, RF_ACTIVE); 874250079Scarl 875255272Scarl if (bar->pci_resource == NULL) 876255272Scarl return (ENXIO); 877255276Scarl 878300373Smav save_bar_parameters(bar); 879300373Smav /* 880300373Smav * Ivytown NTB BAR sizes are misreported by the hardware due to a 881300373Smav * hardware issue. To work around this, query the size it should be 882300373Smav * configured to by the device and modify the resource to correspond to 883300373Smav * this new size. The BIOS on systems with this problem is required to 884300373Smav * provide enough address space to allow the driver to make this change 885300373Smav * safely. 886300373Smav * 887300373Smav * Ideally I could have just specified the size when I allocated the 888300373Smav * resource like: 889300373Smav * bus_alloc_resource(ntb->device, 890300373Smav * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 891300373Smav * 1ul << bar_size_bits, RF_ACTIVE); 892300373Smav * but the PCI driver does not honor the size in this call, so we have 893300373Smav * to modify it after the fact. 894300373Smav */ 895304380Smav if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 896300373Smav if (bar->pci_resource_id == PCIR_BAR(2)) 897300373Smav bar_size_bits = pci_read_config(ntb->device, 898300373Smav XEON_PBAR23SZ_OFFSET, 1); 899300373Smav else 900300373Smav bar_size_bits = pci_read_config(ntb->device, 901300373Smav XEON_PBAR45SZ_OFFSET, 1); 902300373Smav 903300373Smav rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 904300373Smav bar->pci_resource, bar->pbase, 905300373Smav bar->pbase + (1ul << bar_size_bits) - 1); 906255272Scarl if (rc != 0) { 907300373Smav device_printf(ntb->device, 908300373Smav "unable to resize bar\n"); 909255272Scarl return (rc); 910250079Scarl } 911300373Smav 912300373Smav save_bar_parameters(bar); 913250079Scarl } 914300373Smav 915300373Smav bar->map_mode = VM_MEMATTR_UNCACHEABLE; 916300373Smav print_map_success(ntb, bar, "mw"); 917300373Smav 918300373Smav /* 919300373Smav * Optionally, mark MW BARs as anything other than UC to improve 920300373Smav * performance. 921300373Smav */ 922304404Smav mapmode = intel_ntb_pat_flags(); 923300373Smav if (mapmode == bar->map_mode) 924300373Smav return (0); 925300373Smav 926300373Smav rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 927300373Smav if (rc == 0) { 928300373Smav bar->map_mode = mapmode; 929300373Smav device_printf(ntb->device, 930300373Smav "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 931300373Smav "%s.\n", 932300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 933300373Smav (char *)bar->vbase + bar->size - 1, 934300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 935304404Smav intel_ntb_vm_memattr_to_str(mapmode)); 936300373Smav } else 937300373Smav device_printf(ntb->device, 938300373Smav "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 939300373Smav "%s: %d\n", 940300373Smav PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 941300373Smav (char *)bar->vbase + bar->size - 1, 942300373Smav (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 943304404Smav intel_ntb_vm_memattr_to_str(mapmode), rc); 944300373Smav /* Proceed anyway */ 945250079Scarl return (0); 946250079Scarl} 947250079Scarl 948250079Scarlstatic void 949304404Smavintel_ntb_unmap_pci_bar(struct ntb_softc *ntb) 950250079Scarl{ 951250079Scarl struct ntb_pci_bar_info *current_bar; 952250079Scarl int i; 953250079Scarl 954300373Smav for (i = 0; i < NTB_MAX_BARS; i++) { 955250079Scarl current_bar = &ntb->bar_info[i]; 956250079Scarl if (current_bar->pci_resource != NULL) 957250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 958250079Scarl current_bar->pci_resource_id, 959250079Scarl current_bar->pci_resource); 960250079Scarl } 961250079Scarl} 962250079Scarl 963250079Scarlstatic int 964304404Smavintel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 965250079Scarl{ 966300373Smav uint32_t i; 967300373Smav int rc; 968250079Scarl 969300373Smav for (i = 0; i < num_vectors; i++) { 970300373Smav ntb->int_info[i].rid = i + 1; 971300373Smav ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 972300373Smav SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 973300373Smav if (ntb->int_info[i].res == NULL) { 974300373Smav device_printf(ntb->device, 975300373Smav "bus_alloc_resource failed\n"); 976300373Smav return (ENOMEM); 977300373Smav } 978300373Smav ntb->int_info[i].tag = NULL; 979300373Smav ntb->allocated_interrupts++; 980300373Smav rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 981300373Smav INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 982300373Smav &ntb->msix_vec[i], &ntb->int_info[i].tag); 983300373Smav if (rc != 0) { 984300373Smav device_printf(ntb->device, "bus_setup_intr failed\n"); 985300373Smav return (ENXIO); 986300373Smav } 987300373Smav } 988300373Smav return (0); 989300373Smav} 990300373Smav 991300373Smav/* 992300373Smav * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 993300373Smav * cannot be allocated for each MSI-X message. JHB seems to think remapping 994300373Smav * should be okay. This tunable should enable us to test that hypothesis 995300373Smav * when someone gets their hands on some Xeon hardware. 996300373Smav */ 997300373Smavstatic int ntb_force_remap_mode; 998300516SmavTUNABLE_INT("hw.ntb.force_remap_mode", &ntb_force_remap_mode); 999300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 1000300373Smav &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 1001300373Smav " to a smaller number of ithreads, even if the desired number are " 1002300373Smav "available"); 1003300373Smav 1004300373Smav/* 1005300373Smav * In case it is NOT ok, give consumers an abort button. 1006300373Smav */ 1007300373Smavstatic int ntb_prefer_intx; 1008300516SmavTUNABLE_INT("hw.ntb.prefer_intx_to_remap", &ntb_prefer_intx); 1009300373SmavSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 1010300373Smav &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1011300373Smav "than remapping MSI-X messages over available slots (match Linux driver " 1012300373Smav "behavior)"); 1013300373Smav 1014300373Smav/* 1015300373Smav * Remap the desired number of MSI-X messages to available ithreads in a simple 1016300373Smav * round-robin fashion. 1017300373Smav */ 1018300373Smavstatic int 1019304404Smavintel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1020300373Smav{ 1021300373Smav u_int *vectors; 1022300373Smav uint32_t i; 1023300373Smav int rc; 1024300373Smav 1025300373Smav if (ntb_prefer_intx != 0) 1026300373Smav return (ENXIO); 1027300373Smav 1028300373Smav vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1029300373Smav 1030300373Smav for (i = 0; i < desired; i++) 1031300373Smav vectors[i] = (i % avail) + 1; 1032300373Smav 1033300373Smav rc = pci_remap_msix(dev, desired, vectors); 1034300373Smav free(vectors, M_NTB); 1035300373Smav return (rc); 1036300373Smav} 1037300373Smav 1038300373Smavstatic int 1039304404Smavintel_ntb_init_isr(struct ntb_softc *ntb) 1040300373Smav{ 1041300373Smav uint32_t desired_vectors, num_vectors; 1042300373Smav int rc; 1043300373Smav 1044250079Scarl ntb->allocated_interrupts = 0; 1045300373Smav ntb->last_ts = ticks; 1046300373Smav 1047250079Scarl /* 1048301811Sngie * Mask all doorbell interrupts. (Except link events!) 1049250079Scarl */ 1050301811Sngie DB_MASK_LOCK(ntb); 1051301811Sngie ntb->db_mask = ntb->db_valid_mask; 1052301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1053301811Sngie DB_MASK_UNLOCK(ntb); 1054250079Scarl 1055300373Smav num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1056300373Smav ntb->db_count); 1057300373Smav if (desired_vectors >= 1) { 1058300373Smav rc = pci_alloc_msix(ntb->device, &num_vectors); 1059250079Scarl 1060300373Smav if (ntb_force_remap_mode != 0 && rc == 0 && 1061300373Smav num_vectors == desired_vectors) 1062300373Smav num_vectors--; 1063300373Smav 1064300373Smav if (rc == 0 && num_vectors < desired_vectors) { 1065304404Smav rc = intel_ntb_remap_msix(ntb->device, desired_vectors, 1066300373Smav num_vectors); 1067300373Smav if (rc == 0) 1068300373Smav num_vectors = desired_vectors; 1069300373Smav else 1070300373Smav pci_release_msi(ntb->device); 1071250079Scarl } 1072300373Smav if (rc != 0) 1073300373Smav num_vectors = 1; 1074300373Smav } else 1075300373Smav num_vectors = 1; 1076300373Smav 1077300373Smav if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1078304380Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1079301811Sngie device_printf(ntb->device, 1080301811Sngie "Errata workaround does not support MSI or INTX\n"); 1081301811Sngie return (EINVAL); 1082301811Sngie } 1083301811Sngie 1084300373Smav ntb->db_vec_count = 1; 1085300373Smav ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1086304404Smav rc = intel_ntb_setup_legacy_interrupt(ntb); 1087300373Smav } else { 1088301811Sngie if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1089304380Smav HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1090301811Sngie device_printf(ntb->device, 1091301811Sngie "Errata workaround expects %d doorbell bits\n", 1092301811Sngie XEON_NONLINK_DB_MSIX_BITS); 1093301811Sngie return (EINVAL); 1094301811Sngie } 1095301811Sngie 1096304404Smav intel_ntb_create_msix_vec(ntb, num_vectors); 1097304404Smav rc = intel_ntb_setup_msix(ntb, num_vectors); 1098250079Scarl } 1099300373Smav if (rc != 0) { 1100300373Smav device_printf(ntb->device, 1101300373Smav "Error allocating interrupts: %d\n", rc); 1102304404Smav intel_ntb_free_msix_vec(ntb); 1103300373Smav } 1104250079Scarl 1105300373Smav return (rc); 1106300373Smav} 1107250079Scarl 1108300373Smavstatic int 1109304404Smavintel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1110300373Smav{ 1111300373Smav int rc; 1112300373Smav 1113300373Smav ntb->int_info[0].rid = 0; 1114300373Smav ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1115300373Smav &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1116300373Smav if (ntb->int_info[0].res == NULL) { 1117300373Smav device_printf(ntb->device, "bus_alloc_resource failed\n"); 1118300373Smav return (ENOMEM); 1119250079Scarl } 1120250079Scarl 1121300373Smav ntb->int_info[0].tag = NULL; 1122300373Smav ntb->allocated_interrupts = 1; 1123300373Smav 1124300373Smav rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1125300373Smav INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1126300373Smav ntb, &ntb->int_info[0].tag); 1127300373Smav if (rc != 0) { 1128300373Smav device_printf(ntb->device, "bus_setup_intr failed\n"); 1129300373Smav return (ENXIO); 1130300373Smav } 1131300373Smav 1132250079Scarl return (0); 1133250079Scarl} 1134250079Scarl 1135250079Scarlstatic void 1136304404Smavintel_ntb_teardown_interrupts(struct ntb_softc *ntb) 1137250079Scarl{ 1138250079Scarl struct ntb_int_info *current_int; 1139250079Scarl int i; 1140250079Scarl 1141300373Smav for (i = 0; i < ntb->allocated_interrupts; i++) { 1142250079Scarl current_int = &ntb->int_info[i]; 1143250079Scarl if (current_int->tag != NULL) 1144250079Scarl bus_teardown_intr(ntb->device, current_int->res, 1145250079Scarl current_int->tag); 1146250079Scarl 1147250079Scarl if (current_int->res != NULL) 1148250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 1149250079Scarl rman_get_rid(current_int->res), current_int->res); 1150250079Scarl } 1151250079Scarl 1152304404Smav intel_ntb_free_msix_vec(ntb); 1153250079Scarl pci_release_msi(ntb->device); 1154250079Scarl} 1155250079Scarl 1156300373Smav/* 1157300373Smav * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1158300373Smav * out to make code clearer. 1159300373Smav */ 1160300373Smavstatic inline uint64_t 1161300373Smavdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 1162250079Scarl{ 1163250079Scarl 1164300373Smav if (ntb->type == NTB_ATOM) 1165304404Smav return (intel_ntb_reg_read(8, regoff)); 1166250079Scarl 1167300373Smav KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1168300373Smav 1169304404Smav return (intel_ntb_reg_read(2, regoff)); 1170250079Scarl} 1171250079Scarl 1172300373Smavstatic inline void 1173300373Smavdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1174250079Scarl{ 1175250079Scarl 1176300373Smav KASSERT((val & ~ntb->db_valid_mask) == 0, 1177300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1178300373Smav (uintmax_t)(val & ~ntb->db_valid_mask), 1179300373Smav (uintmax_t)ntb->db_valid_mask)); 1180250079Scarl 1181300373Smav if (regoff == ntb->self_reg->db_mask) 1182300373Smav DB_MASK_ASSERT(ntb, MA_OWNED); 1183300373Smav db_iowrite_raw(ntb, regoff, val); 1184250079Scarl} 1185250079Scarl 1186300373Smavstatic inline void 1187300373Smavdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1188250079Scarl{ 1189250079Scarl 1190300373Smav if (ntb->type == NTB_ATOM) { 1191304404Smav intel_ntb_reg_write(8, regoff, val); 1192300373Smav return; 1193300373Smav } 1194250079Scarl 1195300373Smav KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1196304404Smav intel_ntb_reg_write(2, regoff, (uint16_t)val); 1197250079Scarl} 1198250079Scarl 1199304380Smavstatic void 1200304404Smavintel_ntb_db_set_mask(device_t dev, uint64_t bits) 1201250079Scarl{ 1202304380Smav struct ntb_softc *ntb = device_get_softc(dev); 1203250079Scarl 1204300373Smav DB_MASK_LOCK(ntb); 1205300373Smav ntb->db_mask |= bits; 1206304388Smav if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1207304388Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1208300373Smav DB_MASK_UNLOCK(ntb); 1209300373Smav} 1210250079Scarl 1211304380Smavstatic void 1212304404Smavintel_ntb_db_clear_mask(device_t dev, uint64_t bits) 1213300373Smav{ 1214304380Smav struct ntb_softc *ntb = device_get_softc(dev); 1215304388Smav uint64_t ibits; 1216304388Smav int i; 1217250079Scarl 1218300373Smav KASSERT((bits & ~ntb->db_valid_mask) == 0, 1219300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1220300373Smav (uintmax_t)(bits & ~ntb->db_valid_mask), 1221300373Smav (uintmax_t)ntb->db_valid_mask)); 1222250079Scarl 1223300373Smav DB_MASK_LOCK(ntb); 1224304388Smav ibits = ntb->fake_db_bell & ntb->db_mask & bits; 1225300373Smav ntb->db_mask &= ~bits; 1226304388Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1227304388Smav /* Simulate fake interrupts if unmasked DB bits are set. */ 1228304388Smav for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1229304404Smav if ((ibits & intel_ntb_db_vector_mask(dev, i)) != 0) 1230304388Smav swi_sched(ntb->int_info[i].tag, 0); 1231304388Smav } 1232304388Smav } else { 1233304388Smav db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1234304388Smav } 1235300373Smav DB_MASK_UNLOCK(ntb); 1236300373Smav} 1237300373Smav 1238304380Smavstatic uint64_t 1239304404Smavintel_ntb_db_read(device_t dev) 1240300373Smav{ 1241304380Smav struct ntb_softc *ntb = device_get_softc(dev); 1242300373Smav 1243304388Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1244304388Smav return (ntb->fake_db_bell); 1245301811Sngie 1246300373Smav return (db_ioread(ntb, ntb->self_reg->db_bell)); 1247300373Smav} 1248300373Smav 1249304380Smavstatic void 1250304404Smavintel_ntb_db_clear(device_t dev, uint64_t bits) 1251300373Smav{ 1252304380Smav struct ntb_softc *ntb = device_get_softc(dev); 1253300373Smav 1254300373Smav KASSERT((bits & ~ntb->db_valid_mask) == 0, 1255300373Smav ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1256300373Smav (uintmax_t)(bits & ~ntb->db_valid_mask), 1257300373Smav (uintmax_t)ntb->db_valid_mask)); 1258300373Smav 1259304380Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1260304388Smav DB_MASK_LOCK(ntb); 1261304388Smav ntb->fake_db_bell &= ~bits; 1262304388Smav DB_MASK_UNLOCK(ntb); 1263301811Sngie return; 1264301811Sngie } 1265301811Sngie 1266300373Smav db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1267300373Smav} 1268300373Smav 1269300373Smavstatic inline uint64_t 1270304404Smavintel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1271300373Smav{ 1272300373Smav uint64_t shift, mask; 1273300373Smav 1274304388Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1275304388Smav /* 1276304388Smav * Remap vectors in custom way to make at least first 1277304388Smav * three doorbells to not generate stray events. 1278304388Smav * This breaks Linux compatibility (if one existed) 1279304388Smav * when more then one DB is used (not by if_ntb). 1280304388Smav */ 1281304388Smav if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1) 1282304388Smav return (1 << db_vector); 1283304388Smav if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1) 1284304388Smav return (0x7ffc); 1285304388Smav } 1286304388Smav 1287300373Smav shift = ntb->db_vec_shift; 1288300373Smav mask = (1ull << shift) - 1; 1289300373Smav return (mask << (shift * db_vector)); 1290300373Smav} 1291300373Smav 1292300373Smavstatic void 1293304404Smavintel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1294300373Smav{ 1295300373Smav uint64_t vec_mask; 1296300373Smav 1297300373Smav ntb->last_ts = ticks; 1298304404Smav vec_mask = intel_ntb_vec_mask(ntb, vec); 1299300373Smav 1300300373Smav if ((vec_mask & ntb->db_link_mask) != 0) { 1301304404Smav if (intel_ntb_poll_link(ntb)) 1302304380Smav ntb_link_event(ntb->device); 1303250079Scarl } 1304250079Scarl 1305304380Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1306301811Sngie (vec_mask & ntb->db_link_mask) == 0) { 1307301811Sngie DB_MASK_LOCK(ntb); 1308304388Smav 1309304388Smav /* Do not report same DB events again if not cleared yet. */ 1310304388Smav vec_mask &= ~ntb->fake_db_bell; 1311304388Smav 1312304388Smav /* Update our internal doorbell register. */ 1313304388Smav ntb->fake_db_bell |= vec_mask; 1314304388Smav 1315304388Smav /* Do not report masked DB events. */ 1316304388Smav vec_mask &= ~ntb->db_mask; 1317304388Smav 1318301811Sngie DB_MASK_UNLOCK(ntb); 1319301811Sngie } 1320301811Sngie 1321300373Smav if ((vec_mask & ntb->db_valid_mask) != 0) 1322304380Smav ntb_db_event(ntb->device, vec); 1323250079Scarl} 1324250079Scarl 1325300373Smavstatic void 1326300373Smavndev_vec_isr(void *arg) 1327300373Smav{ 1328300373Smav struct ntb_vec *nvec = arg; 1329300373Smav 1330304404Smav intel_ntb_interrupt(nvec->ntb, nvec->num); 1331300373Smav} 1332300373Smav 1333300373Smavstatic void 1334300373Smavndev_irq_isr(void *arg) 1335300373Smav{ 1336300373Smav /* If we couldn't set up MSI-X, we only have the one vector. */ 1337304404Smav intel_ntb_interrupt(arg, 0); 1338300373Smav} 1339300373Smav 1340250079Scarlstatic int 1341304404Smavintel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1342250079Scarl{ 1343300373Smav uint32_t i; 1344250079Scarl 1345300373Smav ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1346250079Scarl M_ZERO | M_WAITOK); 1347250079Scarl for (i = 0; i < num_vectors; i++) { 1348300373Smav ntb->msix_vec[i].num = i; 1349300373Smav ntb->msix_vec[i].ntb = ntb; 1350250079Scarl } 1351250079Scarl 1352250079Scarl return (0); 1353250079Scarl} 1354250079Scarl 1355250079Scarlstatic void 1356304404Smavintel_ntb_free_msix_vec(struct ntb_softc *ntb) 1357250079Scarl{ 1358250079Scarl 1359300373Smav if (ntb->msix_vec == NULL) 1360300373Smav return; 1361250079Scarl 1362300373Smav free(ntb->msix_vec, M_NTB); 1363300373Smav ntb->msix_vec = NULL; 1364250079Scarl} 1365250079Scarl 1366301811Sngiestatic void 1367304404Smavintel_ntb_get_msix_info(struct ntb_softc *ntb) 1368301811Sngie{ 1369301811Sngie struct pci_devinfo *dinfo; 1370301811Sngie struct pcicfg_msix *msix; 1371301811Sngie uint32_t laddr, data, i, offset; 1372301811Sngie 1373301811Sngie dinfo = device_get_ivars(ntb->device); 1374301811Sngie msix = &dinfo->cfg.msix; 1375301811Sngie 1376301811Sngie CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1377301811Sngie 1378301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1379301811Sngie offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1380301811Sngie 1381301811Sngie laddr = bus_read_4(msix->msix_table_res, offset + 1382301811Sngie PCI_MSIX_ENTRY_LOWER_ADDR); 1383304404Smav intel_ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1384301811Sngie 1385301811Sngie KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1386301811Sngie ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1387301811Sngie MSI_INTEL_ADDR_BASE)); 1388301904Smav ntb->msix_data[i].nmd_ofs = laddr; 1389301811Sngie 1390301811Sngie data = bus_read_4(msix->msix_table_res, offset + 1391301811Sngie PCI_MSIX_ENTRY_DATA); 1392304404Smav intel_ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1393301811Sngie 1394301811Sngie ntb->msix_data[i].nmd_data = data; 1395301811Sngie } 1396301811Sngie} 1397301811Sngie 1398250079Scarlstatic struct ntb_hw_info * 1399304404Smavintel_ntb_get_device_info(uint32_t device_id) 1400250079Scarl{ 1401250079Scarl struct ntb_hw_info *ep = pci_ids; 1402250079Scarl 1403250079Scarl while (ep->device_id) { 1404250079Scarl if (ep->device_id == device_id) 1405250079Scarl return (ep); 1406250079Scarl ++ep; 1407250079Scarl } 1408250079Scarl return (NULL); 1409250079Scarl} 1410250079Scarl 1411300373Smavstatic void 1412304404Smavintel_ntb_teardown_xeon(struct ntb_softc *ntb) 1413250079Scarl{ 1414250079Scarl 1415300373Smav if (ntb->reg != NULL) 1416304404Smav intel_ntb_link_disable(ntb->device); 1417300373Smav} 1418300373Smav 1419300373Smavstatic void 1420304404Smavintel_ntb_detect_max_mw(struct ntb_softc *ntb) 1421300373Smav{ 1422300373Smav 1423300373Smav if (ntb->type == NTB_ATOM) { 1424300373Smav ntb->mw_count = ATOM_MW_COUNT; 1425300373Smav return; 1426300373Smav } 1427300373Smav 1428304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1429300373Smav ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1430250079Scarl else 1431300373Smav ntb->mw_count = XEON_SNB_MW_COUNT; 1432250079Scarl} 1433250079Scarl 1434250079Scarlstatic int 1435304404Smavintel_ntb_detect_xeon(struct ntb_softc *ntb) 1436250079Scarl{ 1437300373Smav uint8_t ppd, conn_type; 1438250079Scarl 1439300373Smav ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1440300373Smav ntb->ppd = ppd; 1441250079Scarl 1442300373Smav if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1443300373Smav ntb->dev_type = NTB_DEV_DSD; 1444300373Smav else 1445300373Smav ntb->dev_type = NTB_DEV_USD; 1446300373Smav 1447300373Smav if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1448300373Smav ntb->features |= NTB_SPLIT_BAR; 1449300373Smav 1450304394Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1451304394Smav !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1452304394Smav device_printf(ntb->device, 1453304394Smav "Can not apply SB01BASE_LOCKUP workaround " 1454304394Smav "with split BARs disabled!\n"); 1455304394Smav device_printf(ntb->device, 1456304394Smav "Expect system hangs under heavy NTB traffic!\n"); 1457304394Smav ntb->features &= ~NTB_SB01BASE_LOCKUP; 1458304394Smav } 1459304394Smav 1460301811Sngie /* 1461301811Sngie * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1462301811Sngie * errata workaround; only do one at a time. 1463301811Sngie */ 1464304380Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1465301811Sngie ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1466300373Smav 1467300373Smav conn_type = ppd & XEON_PPD_CONN_TYPE; 1468300373Smav switch (conn_type) { 1469250079Scarl case NTB_CONN_B2B: 1470300373Smav ntb->conn_type = conn_type; 1471250079Scarl break; 1472250079Scarl case NTB_CONN_RP: 1473300373Smav case NTB_CONN_TRANSPARENT: 1474250079Scarl default: 1475300373Smav device_printf(ntb->device, "Unsupported connection type: %u\n", 1476300373Smav (unsigned)conn_type); 1477250079Scarl return (ENXIO); 1478250079Scarl } 1479300373Smav return (0); 1480300373Smav} 1481250079Scarl 1482300373Smavstatic int 1483304404Smavintel_ntb_detect_atom(struct ntb_softc *ntb) 1484300373Smav{ 1485300373Smav uint32_t ppd, conn_type; 1486300373Smav 1487300373Smav ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1488300373Smav ntb->ppd = ppd; 1489300373Smav 1490300373Smav if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1491250079Scarl ntb->dev_type = NTB_DEV_DSD; 1492250079Scarl else 1493250079Scarl ntb->dev_type = NTB_DEV_USD; 1494250079Scarl 1495300373Smav conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1496300373Smav switch (conn_type) { 1497300373Smav case NTB_CONN_B2B: 1498300373Smav ntb->conn_type = conn_type; 1499300373Smav break; 1500300373Smav default: 1501300373Smav device_printf(ntb->device, "Unsupported NTB configuration\n"); 1502300373Smav return (ENXIO); 1503250079Scarl } 1504250079Scarl return (0); 1505250079Scarl} 1506250079Scarl 1507250079Scarlstatic int 1508304404Smavintel_ntb_xeon_init_dev(struct ntb_softc *ntb) 1509250079Scarl{ 1510300373Smav int rc; 1511250079Scarl 1512300373Smav ntb->spad_count = XEON_SPAD_COUNT; 1513300373Smav ntb->db_count = XEON_DB_COUNT; 1514300373Smav ntb->db_link_mask = XEON_DB_LINK_BIT; 1515300373Smav ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1516300373Smav ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1517250079Scarl 1518300373Smav if (ntb->conn_type != NTB_CONN_B2B) { 1519250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1520300373Smav ntb->conn_type); 1521250079Scarl return (ENXIO); 1522250079Scarl } 1523250079Scarl 1524300373Smav ntb->reg = &xeon_reg; 1525300373Smav ntb->self_reg = &xeon_pri_reg; 1526300373Smav ntb->peer_reg = &xeon_b2b_reg; 1527300373Smav ntb->xlat_reg = &xeon_sec_xlat; 1528300373Smav 1529304380Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1530304388Smav ntb->fake_db_bell = 0; 1531301811Sngie ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1532301811Sngie ntb->mw_count; 1533304404Smav intel_ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1534301811Sngie g_ntb_msix_idx, ntb->msix_mw_idx); 1535304404Smav rc = intel_ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1536301811Sngie VM_MEMATTR_UNCACHEABLE); 1537301811Sngie KASSERT(rc == 0, ("shouldn't fail")); 1538304380Smav } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1539301811Sngie /* 1540301811Sngie * There is a Xeon hardware errata related to writes to SDOORBELL or 1541301811Sngie * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1542301811Sngie * which may hang the system. To workaround this, use a memory 1543301811Sngie * window to access the interrupt and scratch pad registers on the 1544301811Sngie * remote system. 1545301811Sngie */ 1546300373Smav ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1547300373Smav ntb->mw_count; 1548304404Smav intel_ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1549300373Smav g_ntb_mw_idx, ntb->b2b_mw_idx); 1550304404Smav rc = intel_ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1551301811Sngie VM_MEMATTR_UNCACHEABLE); 1552300373Smav KASSERT(rc == 0, ("shouldn't fail")); 1553304380Smav } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1554300373Smav /* 1555300373Smav * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1556300373Smav * mirrored to the remote system. Shrink the number of bits by one, 1557300373Smav * since bit 14 is the last bit. 1558300373Smav * 1559300373Smav * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1560300373Smav * anyway. Nor for non-B2B connection types. 1561300373Smav */ 1562300373Smav ntb->db_count = XEON_DB_COUNT - 1; 1563300373Smav 1564300373Smav ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1565300373Smav 1566300373Smav if (ntb->dev_type == NTB_DEV_USD) 1567300373Smav rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1568300373Smav &xeon_b2b_usd_addr); 1569250079Scarl else 1570300373Smav rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1571300373Smav &xeon_b2b_dsd_addr); 1572300373Smav if (rc != 0) 1573300373Smav return (rc); 1574250079Scarl 1575300373Smav /* Enable Bus Master and Memory Space on the secondary side */ 1576304404Smav intel_ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1577300373Smav PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1578250079Scarl 1579300373Smav /* 1580300373Smav * Mask all doorbell interrupts. 1581300373Smav */ 1582301811Sngie DB_MASK_LOCK(ntb); 1583301811Sngie ntb->db_mask = ntb->db_valid_mask; 1584301811Sngie db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1585301811Sngie DB_MASK_UNLOCK(ntb); 1586250079Scarl 1587304404Smav rc = intel_ntb_init_isr(ntb); 1588300373Smav return (rc); 1589300373Smav} 1590250079Scarl 1591300373Smavstatic int 1592304404Smavintel_ntb_atom_init_dev(struct ntb_softc *ntb) 1593300373Smav{ 1594300373Smav int error; 1595250079Scarl 1596300373Smav KASSERT(ntb->conn_type == NTB_CONN_B2B, 1597300373Smav ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1598300373Smav 1599300373Smav ntb->spad_count = ATOM_SPAD_COUNT; 1600300373Smav ntb->db_count = ATOM_DB_COUNT; 1601300373Smav ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1602300373Smav ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1603300373Smav ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1604300373Smav 1605300373Smav ntb->reg = &atom_reg; 1606300373Smav ntb->self_reg = &atom_pri_reg; 1607300373Smav ntb->peer_reg = &atom_b2b_reg; 1608300373Smav ntb->xlat_reg = &atom_sec_xlat; 1609300373Smav 1610250079Scarl /* 1611300373Smav * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1612250079Scarl * resolved. Mask transaction layer internal parity errors. 1613250079Scarl */ 1614250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1615250079Scarl 1616300373Smav configure_atom_secondary_side_bars(ntb); 1617250079Scarl 1618250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1619304404Smav intel_ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1620250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1621250079Scarl 1622304404Smav error = intel_ntb_init_isr(ntb); 1623300373Smav if (error != 0) 1624300373Smav return (error); 1625300373Smav 1626300373Smav /* Initiate PCI-E link training */ 1627304404Smav intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1628300373Smav 1629300373Smav callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1630300373Smav 1631250079Scarl return (0); 1632250079Scarl} 1633250079Scarl 1634300373Smav/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1635255279Scarlstatic void 1636300373Smavconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1637255279Scarl{ 1638255279Scarl 1639255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1640304404Smav intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1641300373Smav XEON_B2B_BAR2_ADDR64); 1642304404Smav intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1643300373Smav XEON_B2B_BAR4_ADDR64); 1644304404Smav intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1645304404Smav intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1646255279Scarl } else { 1647304404Smav intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1648300373Smav XEON_B2B_BAR2_ADDR64); 1649304404Smav intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1650300373Smav XEON_B2B_BAR4_ADDR64); 1651304404Smav intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1652304404Smav intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1653255279Scarl } 1654255279Scarl} 1655255279Scarl 1656300373Smav 1657300373Smav/* 1658300373Smav * When working around Xeon SDOORBELL errata by remapping remote registers in a 1659300373Smav * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1660300373Smav * remains for use by a higher layer. 1661300373Smav * 1662300373Smav * Will only be used if working around SDOORBELL errata and the BIOS-configured 1663300373Smav * MW size is sufficiently large. 1664300373Smav */ 1665300373Smavstatic unsigned int ntb_b2b_mw_share; 1666300516SmavTUNABLE_INT("hw.ntb.b2b_mw_share", &ntb_b2b_mw_share); 1667300373SmavSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1668300373Smav 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1669300373Smav "MW with higher level consumers. Both sides of the NTB MUST set the same " 1670300373Smav "value here."); 1671300373Smav 1672255279Scarlstatic void 1673300373Smavxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1674300373Smav enum ntb_bar regbar) 1675255279Scarl{ 1676300373Smav struct ntb_pci_bar_info *bar; 1677300373Smav uint8_t bar_sz; 1678255279Scarl 1679304380Smav if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1680300373Smav return; 1681300373Smav 1682300373Smav bar = &ntb->bar_info[idx]; 1683300373Smav bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1684300373Smav if (idx == regbar) { 1685300373Smav if (ntb->b2b_off != 0) 1686300373Smav bar_sz--; 1687255279Scarl else 1688300373Smav bar_sz = 0; 1689300373Smav } 1690300373Smav pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1691300373Smav bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1692300373Smav (void)bar_sz; 1693300373Smav} 1694300373Smav 1695300373Smavstatic void 1696300373Smavxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1697300373Smav enum ntb_bar idx, enum ntb_bar regbar) 1698300373Smav{ 1699301904Smav uint64_t reg_val; 1700300373Smav uint32_t base_reg, lmt_reg; 1701300373Smav 1702300373Smav bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1703304376Smav if (idx == regbar) { 1704304376Smav if (ntb->b2b_off) 1705304376Smav bar_addr += ntb->b2b_off; 1706304376Smav else 1707304376Smav bar_addr = 0; 1708304376Smav } 1709300373Smav 1710300373Smav if (!bar_is_64bit(ntb, idx)) { 1711304404Smav intel_ntb_reg_write(4, base_reg, bar_addr); 1712304404Smav reg_val = intel_ntb_reg_read(4, base_reg); 1713304399Smav (void)reg_val; 1714304399Smav 1715304404Smav intel_ntb_reg_write(4, lmt_reg, bar_addr); 1716304404Smav reg_val = intel_ntb_reg_read(4, lmt_reg); 1717301811Sngie (void)reg_val; 1718304399Smav } else { 1719304404Smav intel_ntb_reg_write(8, base_reg, bar_addr); 1720304404Smav reg_val = intel_ntb_reg_read(8, base_reg); 1721304399Smav (void)reg_val; 1722301811Sngie 1723304404Smav intel_ntb_reg_write(8, lmt_reg, bar_addr); 1724304404Smav reg_val = intel_ntb_reg_read(8, lmt_reg); 1725301811Sngie (void)reg_val; 1726255279Scarl } 1727255279Scarl} 1728255279Scarl 1729250079Scarlstatic void 1730300373Smavxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1731250079Scarl{ 1732300373Smav struct ntb_pci_bar_info *bar; 1733250079Scarl 1734300373Smav bar = &ntb->bar_info[idx]; 1735304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1736304404Smav intel_ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1737304404Smav base_addr = intel_ntb_reg_read(4, bar->pbarxlat_off); 1738300373Smav } else { 1739304404Smav intel_ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1740304404Smav base_addr = intel_ntb_reg_read(8, bar->pbarxlat_off); 1741300373Smav } 1742300373Smav (void)base_addr; 1743300373Smav} 1744300373Smav 1745300373Smavstatic int 1746300373Smavxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1747300373Smav const struct ntb_b2b_addr *peer_addr) 1748300373Smav{ 1749300373Smav struct ntb_pci_bar_info *b2b_bar; 1750300373Smav vm_size_t bar_size; 1751300373Smav uint64_t bar_addr; 1752300373Smav enum ntb_bar b2b_bar_num, i; 1753300373Smav 1754300373Smav if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1755300373Smav b2b_bar = NULL; 1756300373Smav b2b_bar_num = NTB_CONFIG_BAR; 1757300373Smav ntb->b2b_off = 0; 1758300373Smav } else { 1759304404Smav b2b_bar_num = intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1760300373Smav KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1761300373Smav ("invalid b2b mw bar")); 1762300373Smav 1763300373Smav b2b_bar = &ntb->bar_info[b2b_bar_num]; 1764300373Smav bar_size = b2b_bar->size; 1765300373Smav 1766300373Smav if (ntb_b2b_mw_share != 0 && 1767300373Smav (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1768300373Smav ntb->b2b_off = bar_size >> 1; 1769300373Smav else if (bar_size >= XEON_B2B_MIN_SIZE) { 1770300373Smav ntb->b2b_off = 0; 1771300373Smav } else { 1772300373Smav device_printf(ntb->device, 1773300373Smav "B2B bar size is too small!\n"); 1774300373Smav return (EIO); 1775250079Scarl } 1776250079Scarl } 1777250079Scarl 1778300373Smav /* 1779300373Smav * Reset the secondary bar sizes to match the primary bar sizes. 1780300373Smav * (Except, disable or halve the size of the B2B secondary bar.) 1781300373Smav */ 1782300373Smav for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1783300373Smav xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1784300373Smav 1785300373Smav bar_addr = 0; 1786300373Smav if (b2b_bar_num == NTB_CONFIG_BAR) 1787300373Smav bar_addr = addr->bar0_addr; 1788300373Smav else if (b2b_bar_num == NTB_B2B_BAR_1) 1789300373Smav bar_addr = addr->bar2_addr64; 1790304380Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1791300373Smav bar_addr = addr->bar4_addr64; 1792300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2) 1793300373Smav bar_addr = addr->bar4_addr32; 1794300373Smav else if (b2b_bar_num == NTB_B2B_BAR_3) 1795300373Smav bar_addr = addr->bar5_addr32; 1796300373Smav else 1797300373Smav KASSERT(false, ("invalid bar")); 1798300373Smav 1799304404Smav intel_ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1800300373Smav 1801300373Smav /* 1802300373Smav * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1803300373Smav * register BAR. The B2B BAR is either disabled above or configured 1804300373Smav * half-size. It starts at PBAR xlat + offset. 1805300373Smav * 1806300373Smav * Also set up incoming BAR limits == base (zero length window). 1807300373Smav */ 1808300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1809300373Smav b2b_bar_num); 1810304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1811300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1812300373Smav NTB_B2B_BAR_2, b2b_bar_num); 1813300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1814300373Smav NTB_B2B_BAR_3, b2b_bar_num); 1815300373Smav } else 1816300373Smav xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1817300373Smav NTB_B2B_BAR_2, b2b_bar_num); 1818300373Smav 1819300373Smav /* Zero incoming translation addrs */ 1820304404Smav intel_ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1821304404Smav intel_ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1822300373Smav 1823304380Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1824304396Smav uint32_t xlat_reg, lmt_reg; 1825304388Smav enum ntb_bar bar_num; 1826301811Sngie 1827301811Sngie /* 1828301811Sngie * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1829301811Sngie * workaround 1830301811Sngie */ 1831304404Smav bar_num = intel_ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1832304396Smav bar_get_xlat_params(ntb, bar_num, NULL, &xlat_reg, &lmt_reg); 1833304396Smav if (bar_is_64bit(ntb, bar_num)) { 1834304404Smav intel_ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE); 1835304404Smav ntb->msix_xlat = intel_ntb_reg_read(8, xlat_reg); 1836304404Smav intel_ntb_reg_write(8, lmt_reg, 0); 1837301904Smav } else { 1838304404Smav intel_ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE); 1839304404Smav ntb->msix_xlat = intel_ntb_reg_read(4, xlat_reg); 1840304404Smav intel_ntb_reg_write(4, lmt_reg, 0); 1841301904Smav } 1842304388Smav 1843304388Smav ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1844301811Sngie } 1845304404Smav (void)intel_ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1846304404Smav (void)intel_ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1847301811Sngie 1848300373Smav /* Zero outgoing translation limits (whole bar size windows) */ 1849304404Smav intel_ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1850304404Smav intel_ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1851300373Smav 1852300373Smav /* Set outgoing translation offsets */ 1853300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1854304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1855300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1856300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1857300373Smav } else 1858300373Smav xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1859300373Smav 1860300373Smav /* Set the translation offset for B2B registers */ 1861300373Smav bar_addr = 0; 1862300373Smav if (b2b_bar_num == NTB_CONFIG_BAR) 1863300373Smav bar_addr = peer_addr->bar0_addr; 1864300373Smav else if (b2b_bar_num == NTB_B2B_BAR_1) 1865300373Smav bar_addr = peer_addr->bar2_addr64; 1866304380Smav else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1867300373Smav bar_addr = peer_addr->bar4_addr64; 1868300373Smav else if (b2b_bar_num == NTB_B2B_BAR_2) 1869300373Smav bar_addr = peer_addr->bar4_addr32; 1870300373Smav else if (b2b_bar_num == NTB_B2B_BAR_3) 1871300373Smav bar_addr = peer_addr->bar5_addr32; 1872300373Smav else 1873300373Smav KASSERT(false, ("invalid bar")); 1874300373Smav 1875300373Smav /* 1876300373Smav * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1877300373Smav * at a time. 1878300373Smav */ 1879304404Smav intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1880304404Smav intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1881300373Smav return (0); 1882250079Scarl} 1883250079Scarl 1884300373Smavstatic inline bool 1885301811Sngie_xeon_link_is_up(struct ntb_softc *ntb) 1886301811Sngie{ 1887301811Sngie 1888301811Sngie if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1889301811Sngie return (true); 1890301811Sngie return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1891301811Sngie} 1892301811Sngie 1893301811Sngiestatic inline bool 1894300373Smavlink_is_up(struct ntb_softc *ntb) 1895300373Smav{ 1896300373Smav 1897301811Sngie if (ntb->type == NTB_XEON) 1898301811Sngie return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1899304380Smav !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 1900300373Smav 1901300373Smav KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1902300373Smav return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1903300373Smav} 1904300373Smav 1905300373Smavstatic inline bool 1906300373Smavatom_link_is_err(struct ntb_softc *ntb) 1907300373Smav{ 1908300373Smav uint32_t status; 1909300373Smav 1910300373Smav KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1911300373Smav 1912304404Smav status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1913300373Smav if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1914300373Smav return (true); 1915300373Smav 1916304404Smav status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1917300373Smav return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1918300373Smav} 1919300373Smav 1920300373Smav/* Atom does not have link status interrupt, poll on that platform */ 1921250079Scarlstatic void 1922300373Smavatom_link_hb(void *arg) 1923250079Scarl{ 1924300373Smav struct ntb_softc *ntb = arg; 1925300373Smav sbintime_t timo, poll_ts; 1926300373Smav 1927300373Smav timo = NTB_HB_TIMEOUT * hz; 1928300373Smav poll_ts = ntb->last_ts + timo; 1929300373Smav 1930300373Smav /* 1931300373Smav * Delay polling the link status if an interrupt was received, unless 1932300373Smav * the cached link status says the link is down. 1933300373Smav */ 1934300373Smav if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1935300373Smav timo = poll_ts - ticks; 1936300373Smav goto out; 1937300373Smav } 1938300373Smav 1939304404Smav if (intel_ntb_poll_link(ntb)) 1940304380Smav ntb_link_event(ntb->device); 1941300373Smav 1942300373Smav if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1943300373Smav /* Link is down with error, proceed with recovery */ 1944300373Smav callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1945300373Smav return; 1946300373Smav } 1947300373Smav 1948300373Smavout: 1949300373Smav callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1950300373Smav} 1951300373Smav 1952300373Smavstatic void 1953300373Smavatom_perform_link_restart(struct ntb_softc *ntb) 1954300373Smav{ 1955250079Scarl uint32_t status; 1956250079Scarl 1957250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1958304404Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1959304404Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1960304404Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1961304404Smav intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1962250079Scarl 1963250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1964250079Scarl pause("ModPhy", hz / 10); 1965250079Scarl 1966250079Scarl /* Clear AER Errors, write to clear */ 1967304404Smav status = intel_ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1968250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1969304404Smav intel_ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1970250079Scarl 1971250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1972304404Smav status = intel_ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1973300373Smav status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1974304404Smav intel_ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1975250079Scarl 1976250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1977304404Smav status = intel_ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1978300373Smav status |= ATOM_DESKEWSTS_DBERR; 1979304404Smav intel_ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1980250079Scarl 1981304404Smav status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1982300373Smav status &= ATOM_IBIST_ERR_OFLOW; 1983304404Smav intel_ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1984250079Scarl 1985250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1986304404Smav status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1987300373Smav status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1988304404Smav intel_ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1989250079Scarl} 1990250079Scarl 1991304380Smavstatic int 1992304404Smavintel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, 1993304380Smav enum ntb_width width __unused) 1994300373Smav{ 1995304380Smav struct ntb_softc *ntb = device_get_softc(dev); 1996300373Smav uint32_t cntl; 1997250079Scarl 1998304404Smav intel_ntb_printf(2, "%s\n", __func__); 1999301811Sngie 2000300373Smav if (ntb->type == NTB_ATOM) { 2001300373Smav pci_write_config(ntb->device, NTB_PPD_OFFSET, 2002300373Smav ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2003300373Smav return (0); 2004250079Scarl } 2005250079Scarl 2006300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2007304380Smav ntb_link_event(dev); 2008300373Smav return (0); 2009300373Smav } 2010300373Smav 2011304404Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2012300373Smav cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2013300373Smav cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2014300373Smav cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2015304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2016300373Smav cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2017304404Smav intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2018300373Smav return (0); 2019250079Scarl} 2020250079Scarl 2021304380Smavstatic int 2022304404Smavintel_ntb_link_disable(device_t dev) 2023300373Smav{ 2024304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2025300373Smav uint32_t cntl; 2026300373Smav 2027304404Smav intel_ntb_printf(2, "%s\n", __func__); 2028301811Sngie 2029300373Smav if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2030304380Smav ntb_link_event(dev); 2031300373Smav return (0); 2032300373Smav } 2033300373Smav 2034304404Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2035300373Smav cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2036300373Smav cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2037304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2038300373Smav cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2039300373Smav cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2040304404Smav intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2041300373Smav return (0); 2042300373Smav} 2043300373Smav 2044304380Smavstatic bool 2045304404Smavintel_ntb_link_enabled(device_t dev) 2046301811Sngie{ 2047304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2048301811Sngie uint32_t cntl; 2049301811Sngie 2050301811Sngie if (ntb->type == NTB_ATOM) { 2051301811Sngie cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2052301811Sngie return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2053301811Sngie } 2054301811Sngie 2055301811Sngie if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2056301811Sngie return (true); 2057301811Sngie 2058304404Smav cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2059301811Sngie return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2060301811Sngie} 2061301811Sngie 2062250079Scarlstatic void 2063300373Smavrecover_atom_link(void *arg) 2064250079Scarl{ 2065250079Scarl struct ntb_softc *ntb = arg; 2066300373Smav unsigned speed, width, oldspeed, oldwidth; 2067250079Scarl uint32_t status32; 2068250079Scarl 2069300373Smav atom_perform_link_restart(ntb); 2070250079Scarl 2071300373Smav /* 2072300373Smav * There is a potential race between the 2 NTB devices recovering at 2073300373Smav * the same time. If the times are the same, the link will not recover 2074300373Smav * and the driver will be stuck in this loop forever. Add a random 2075300373Smav * interval to the recovery time to prevent this race. 2076300373Smav */ 2077300373Smav status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2078300373Smav pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2079250079Scarl 2080300373Smav if (atom_link_is_err(ntb)) 2081250079Scarl goto retry; 2082250079Scarl 2083304404Smav status32 = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2084300373Smav if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2085300373Smav goto out; 2086300373Smav 2087304404Smav status32 = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2088300373Smav width = NTB_LNK_STA_WIDTH(status32); 2089300373Smav speed = status32 & NTB_LINK_SPEED_MASK; 2090300373Smav 2091300373Smav oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2092300373Smav oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2093300373Smav if (oldwidth != width || oldspeed != speed) 2094250079Scarl goto retry; 2095250079Scarl 2096300373Smavout: 2097300373Smav callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2098300373Smav ntb); 2099250079Scarl return; 2100250079Scarl 2101250079Scarlretry: 2102300373Smav callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2103250079Scarl ntb); 2104250079Scarl} 2105250079Scarl 2106300373Smav/* 2107300373Smav * Polls the HW link status register(s); returns true if something has changed. 2108300373Smav */ 2109300373Smavstatic bool 2110304404Smavintel_ntb_poll_link(struct ntb_softc *ntb) 2111250079Scarl{ 2112250079Scarl uint32_t ntb_cntl; 2113300373Smav uint16_t reg_val; 2114250079Scarl 2115300373Smav if (ntb->type == NTB_ATOM) { 2116304404Smav ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2117300373Smav if (ntb_cntl == ntb->ntb_ctl) 2118300373Smav return (false); 2119300373Smav 2120300373Smav ntb->ntb_ctl = ntb_cntl; 2121304404Smav ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2122250079Scarl } else { 2123300373Smav db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2124250079Scarl 2125300373Smav reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2126300373Smav if (reg_val == ntb->lnk_sta) 2127300373Smav return (false); 2128300373Smav 2129300373Smav ntb->lnk_sta = reg_val; 2130301811Sngie 2131304380Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2132301811Sngie if (_xeon_link_is_up(ntb)) { 2133301811Sngie if (!ntb->peer_msix_good) { 2134301811Sngie callout_reset(&ntb->peer_msix_work, 0, 2135304404Smav intel_ntb_exchange_msix, ntb); 2136301811Sngie return (false); 2137301811Sngie } 2138301811Sngie } else { 2139301811Sngie ntb->peer_msix_good = false; 2140301811Sngie ntb->peer_msix_done = false; 2141301811Sngie } 2142301811Sngie } 2143250079Scarl } 2144300373Smav return (true); 2145300373Smav} 2146250079Scarl 2147300373Smavstatic inline enum ntb_speed 2148304404Smavintel_ntb_link_sta_speed(struct ntb_softc *ntb) 2149300373Smav{ 2150250079Scarl 2151300373Smav if (!link_is_up(ntb)) 2152300373Smav return (NTB_SPEED_NONE); 2153300373Smav return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2154250079Scarl} 2155250079Scarl 2156300373Smavstatic inline enum ntb_width 2157304404Smavintel_ntb_link_sta_width(struct ntb_softc *ntb) 2158250079Scarl{ 2159250079Scarl 2160300373Smav if (!link_is_up(ntb)) 2161300373Smav return (NTB_WIDTH_NONE); 2162300373Smav return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2163300373Smav} 2164250079Scarl 2165300373SmavSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2166300373Smav "Driver state, statistics, and HW registers"); 2167250079Scarl 2168300373Smav#define NTB_REGSZ_MASK (3ul << 30) 2169300373Smav#define NTB_REG_64 (1ul << 30) 2170300373Smav#define NTB_REG_32 (2ul << 30) 2171300373Smav#define NTB_REG_16 (3ul << 30) 2172300373Smav#define NTB_REG_8 (0ul << 30) 2173250079Scarl 2174300373Smav#define NTB_DB_READ (1ul << 29) 2175300373Smav#define NTB_PCI_REG (1ul << 28) 2176300373Smav#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2177300373Smav 2178300373Smavstatic void 2179304404Smavintel_ntb_sysctl_init(struct ntb_softc *ntb) 2180250079Scarl{ 2181301811Sngie struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2182300373Smav struct sysctl_ctx_list *ctx; 2183300373Smav struct sysctl_oid *tree, *tmptree; 2184250079Scarl 2185300373Smav ctx = device_get_sysctl_ctx(ntb->device); 2186301811Sngie globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2187250079Scarl 2188301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2189301811Sngie CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2190301811Sngie sysctl_handle_link_status_human, "A", 2191301811Sngie "Link status (human readable)"); 2192301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2193301811Sngie CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2194301811Sngie "IU", "Link status (1=active, 0=inactive)"); 2195301811Sngie SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2196301811Sngie CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2197301811Sngie "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2198301811Sngie 2199301811Sngie tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2200301811Sngie CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2201300373Smav tree_par = SYSCTL_CHILDREN(tree); 2202250079Scarl 2203300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2204300373Smav &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2205300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2206300373Smav &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2207300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2208300373Smav &ntb->ppd, 0, "Raw PPD register (cached)"); 2209300373Smav 2210300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2211300373Smav#ifdef notyet 2212300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2213300373Smav &ntb->b2b_mw_idx, 0, 2214300373Smav "Index of the MW used for B2B remote register access"); 2215300373Smav#endif 2216300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2217300373Smav CTLFLAG_RD, &ntb->b2b_off, 2218300373Smav "If non-zero, offset of B2B register region in shared MW"); 2219250079Scarl } 2220250079Scarl 2221300373Smav SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2222300373Smav CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2223300373Smav "Features/errata of this NTB device"); 2224250079Scarl 2225300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2226300373Smav __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2227300373Smav "NTB CTL register (cached)"); 2228300373Smav SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2229300373Smav __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2230300373Smav "LNK STA register (cached)"); 2231250079Scarl 2232300373Smav#ifdef notyet 2233300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2234300373Smav &ntb->mw_count, 0, "MW count"); 2235300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2236300373Smav &ntb->spad_count, 0, "Scratchpad count"); 2237300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2238300373Smav &ntb->db_count, 0, "Doorbell count"); 2239300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2240300373Smav &ntb->db_vec_count, 0, "Doorbell vector count"); 2241300373Smav SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2242300373Smav &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2243300373Smav#endif 2244250079Scarl 2245300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2246300373Smav &ntb->db_valid_mask, "Doorbell valid mask"); 2247300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2248300373Smav &ntb->db_link_mask, "Doorbell link mask"); 2249300373Smav SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2250300373Smav &ntb->db_mask, "Doorbell mask (cached)"); 2251300373Smav 2252300373Smav tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2253300373Smav CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2254300373Smav regpar = SYSCTL_CHILDREN(tmptree); 2255300373Smav 2256300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2257300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2258300373Smav ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2259300373Smav "NTB Control register"); 2260300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2261300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2262300373Smav 0x19c, sysctl_handle_register, "IU", 2263300373Smav "NTB Link Capabilities"); 2264300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2265300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2266300373Smav 0x1a0, sysctl_handle_register, "IU", 2267300373Smav "NTB Link Control register"); 2268300373Smav 2269300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2270300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2271300373Smav NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2272300373Smav sysctl_handle_register, "QU", "Doorbell mask register"); 2273300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2274300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2275300373Smav NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2276300373Smav sysctl_handle_register, "QU", "Doorbell register"); 2277300373Smav 2278300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2279300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2280300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2281300373Smav sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2282304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2283300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2284300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2285300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2286300373Smav sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2287300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2288300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2289300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2290300373Smav sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2291300373Smav } else { 2292300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2293300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2294300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2295300373Smav sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2296300373Smav } 2297300373Smav 2298300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2299300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2300300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2301300373Smav sysctl_handle_register, "QU", "Incoming LMT23 register"); 2302304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2303300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2304300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2305300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2306300373Smav sysctl_handle_register, "IU", "Incoming LMT4 register"); 2307300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2308300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2309300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2310300373Smav sysctl_handle_register, "IU", "Incoming LMT5 register"); 2311300373Smav } else { 2312300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2313300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2314300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2315300373Smav sysctl_handle_register, "QU", "Incoming LMT45 register"); 2316300373Smav } 2317300373Smav 2318300373Smav if (ntb->type == NTB_ATOM) 2319250079Scarl return; 2320250079Scarl 2321300373Smav tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2322300373Smav CTLFLAG_RD, NULL, "Xeon HW statistics"); 2323300373Smav statpar = SYSCTL_CHILDREN(tmptree); 2324300373Smav SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2325300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2326300373Smav NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2327300373Smav sysctl_handle_register, "SU", "Upstream Memory Miss"); 2328250079Scarl 2329300373Smav tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2330300373Smav CTLFLAG_RD, NULL, "Xeon HW errors"); 2331300373Smav errpar = SYSCTL_CHILDREN(tmptree); 2332300373Smav 2333300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2334300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2335300373Smav NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2336300373Smav sysctl_handle_register, "CU", "PPD"); 2337300373Smav 2338300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2339300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2340300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2341300373Smav sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2342300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2343300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2344300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2345300373Smav sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2346300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2347300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2348300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2349300373Smav sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2350300373Smav 2351300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2352300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2353300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2354300373Smav sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2355300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2356300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2357300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2358300373Smav sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2359300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2360300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2361300373Smav NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2362300373Smav sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2363300373Smav 2364300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2365300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2366300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2367300373Smav sysctl_handle_register, "SU", "DEVSTS"); 2368300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2369300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2370300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2371300373Smav sysctl_handle_register, "SU", "LNKSTS"); 2372300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2373300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2374300373Smav NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2375300373Smav sysctl_handle_register, "SU", "SLNKSTS"); 2376300373Smav 2377300373Smav SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2378300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2379300373Smav NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2380300373Smav sysctl_handle_register, "IU", "UNCERRSTS"); 2381300373Smav SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2382300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2383300373Smav NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2384300373Smav sysctl_handle_register, "IU", "CORERRSTS"); 2385300373Smav 2386300373Smav if (ntb->conn_type != NTB_CONN_B2B) 2387300373Smav return; 2388300373Smav 2389300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2390300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2391300373Smav NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2392300373Smav sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2393304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2394300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2395300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2396300373Smav NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2397300373Smav sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2398300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2399300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2400300373Smav NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2401300373Smav sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2402300373Smav } else { 2403300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2404300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2405300373Smav NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2406300373Smav sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2407300373Smav } 2408300373Smav 2409300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2410300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2411300373Smav NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2412300373Smav sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2413304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2414300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2415300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2416300373Smav NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2417300373Smav sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2418300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2419300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2420300373Smav NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2421300373Smav sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2422300373Smav } else { 2423300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2424300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2425300373Smav NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2426300373Smav sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2427300373Smav } 2428300373Smav 2429300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2430300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2431300373Smav NTB_REG_64 | ntb->xlat_reg->bar0_base, 2432300373Smav sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2433300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2434300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2435300373Smav NTB_REG_64 | ntb->xlat_reg->bar2_base, 2436300373Smav sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2437304380Smav if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2438300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2439300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2440300373Smav NTB_REG_32 | ntb->xlat_reg->bar4_base, 2441300373Smav sysctl_handle_register, "IU", 2442300373Smav "Secondary BAR4 base register"); 2443300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2444300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2445300373Smav NTB_REG_32 | ntb->xlat_reg->bar5_base, 2446300373Smav sysctl_handle_register, "IU", 2447300373Smav "Secondary BAR5 base register"); 2448300373Smav } else { 2449300373Smav SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2450300373Smav CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2451300373Smav NTB_REG_64 | ntb->xlat_reg->bar4_base, 2452300373Smav sysctl_handle_register, "QU", 2453300373Smav "Secondary BAR45 base register"); 2454300373Smav } 2455250079Scarl} 2456250079Scarl 2457300373Smavstatic int 2458300373Smavsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2459250079Scarl{ 2460304377Smav struct ntb_softc *ntb = arg1; 2461300373Smav struct sbuf sb; 2462300373Smav int error; 2463250079Scarl 2464300373Smav sbuf_new_for_sysctl(&sb, NULL, 256, req); 2465300373Smav 2466300373Smav sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2467300373Smav error = sbuf_finish(&sb); 2468300373Smav sbuf_delete(&sb); 2469300373Smav 2470300373Smav if (error || !req->newptr) 2471300373Smav return (error); 2472300373Smav return (EINVAL); 2473250079Scarl} 2474250079Scarl 2475300373Smavstatic int 2476301811Sngiesysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2477250079Scarl{ 2478304377Smav struct ntb_softc *ntb = arg1; 2479301811Sngie unsigned old, new; 2480301811Sngie int error; 2481301811Sngie 2482304404Smav old = intel_ntb_link_enabled(ntb->device); 2483301811Sngie 2484301811Sngie error = SYSCTL_OUT(req, &old, sizeof(old)); 2485301811Sngie if (error != 0 || req->newptr == NULL) 2486301811Sngie return (error); 2487301811Sngie 2488301811Sngie error = SYSCTL_IN(req, &new, sizeof(new)); 2489301811Sngie if (error != 0) 2490301811Sngie return (error); 2491301811Sngie 2492304404Smav intel_ntb_printf(0, "Admin set interface state to '%sabled'\n", 2493301811Sngie (new != 0)? "en" : "dis"); 2494301811Sngie 2495301811Sngie if (new != 0) 2496304404Smav error = intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2497301811Sngie else 2498304404Smav error = intel_ntb_link_disable(ntb->device); 2499301811Sngie return (error); 2500301811Sngie} 2501301811Sngie 2502301811Sngiestatic int 2503301811Sngiesysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2504301811Sngie{ 2505304377Smav struct ntb_softc *ntb = arg1; 2506300373Smav struct sbuf sb; 2507300373Smav enum ntb_speed speed; 2508300373Smav enum ntb_width width; 2509300373Smav int error; 2510250079Scarl 2511300373Smav sbuf_new_for_sysctl(&sb, NULL, 32, req); 2512300373Smav 2513304404Smav if (intel_ntb_link_is_up(ntb->device, &speed, &width)) 2514300373Smav sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2515300373Smav (unsigned)speed, (unsigned)width); 2516300373Smav else 2517300373Smav sbuf_printf(&sb, "down"); 2518300373Smav 2519300373Smav error = sbuf_finish(&sb); 2520300373Smav sbuf_delete(&sb); 2521300373Smav 2522300373Smav if (error || !req->newptr) 2523300373Smav return (error); 2524300373Smav return (EINVAL); 2525250079Scarl} 2526250079Scarl 2527300373Smavstatic int 2528301811Sngiesysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2529301811Sngie{ 2530304377Smav struct ntb_softc *ntb = arg1; 2531301811Sngie unsigned res; 2532301811Sngie int error; 2533301811Sngie 2534304404Smav res = intel_ntb_link_is_up(ntb->device, NULL, NULL); 2535301811Sngie 2536301811Sngie error = SYSCTL_OUT(req, &res, sizeof(res)); 2537301811Sngie if (error || !req->newptr) 2538301811Sngie return (error); 2539301811Sngie return (EINVAL); 2540301811Sngie} 2541301811Sngie 2542301811Sngiestatic int 2543300373Smavsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2544250079Scarl{ 2545300373Smav struct ntb_softc *ntb; 2546300373Smav const void *outp; 2547300373Smav uintptr_t sz; 2548300373Smav uint64_t umv; 2549300373Smav char be[sizeof(umv)]; 2550300373Smav size_t outsz; 2551300373Smav uint32_t reg; 2552300373Smav bool db, pci; 2553300373Smav int error; 2554250079Scarl 2555300373Smav ntb = arg1; 2556300373Smav reg = arg2 & ~NTB_REGFLAGS_MASK; 2557300373Smav sz = arg2 & NTB_REGSZ_MASK; 2558300373Smav db = (arg2 & NTB_DB_READ) != 0; 2559300373Smav pci = (arg2 & NTB_PCI_REG) != 0; 2560250079Scarl 2561300373Smav KASSERT(!(db && pci), ("bogus")); 2562250079Scarl 2563300373Smav if (db) { 2564300373Smav KASSERT(sz == NTB_REG_64, ("bogus")); 2565300373Smav umv = db_ioread(ntb, reg); 2566300373Smav outsz = sizeof(uint64_t); 2567300373Smav } else { 2568300373Smav switch (sz) { 2569300373Smav case NTB_REG_64: 2570300373Smav if (pci) 2571300373Smav umv = pci_read_config(ntb->device, reg, 8); 2572300373Smav else 2573304404Smav umv = intel_ntb_reg_read(8, reg); 2574300373Smav outsz = sizeof(uint64_t); 2575300373Smav break; 2576300373Smav case NTB_REG_32: 2577300373Smav if (pci) 2578300373Smav umv = pci_read_config(ntb->device, reg, 4); 2579300373Smav else 2580304404Smav umv = intel_ntb_reg_read(4, reg); 2581300373Smav outsz = sizeof(uint32_t); 2582300373Smav break; 2583300373Smav case NTB_REG_16: 2584300373Smav if (pci) 2585300373Smav umv = pci_read_config(ntb->device, reg, 2); 2586300373Smav else 2587304404Smav umv = intel_ntb_reg_read(2, reg); 2588300373Smav outsz = sizeof(uint16_t); 2589300373Smav break; 2590300373Smav case NTB_REG_8: 2591300373Smav if (pci) 2592300373Smav umv = pci_read_config(ntb->device, reg, 1); 2593300373Smav else 2594304404Smav umv = intel_ntb_reg_read(1, reg); 2595300373Smav outsz = sizeof(uint8_t); 2596300373Smav break; 2597300373Smav default: 2598300373Smav panic("bogus"); 2599300373Smav break; 2600300373Smav } 2601300373Smav } 2602300373Smav 2603300373Smav /* Encode bigendian so that sysctl -x is legible. */ 2604300373Smav be64enc(be, umv); 2605300373Smav outp = ((char *)be) + sizeof(umv) - outsz; 2606300373Smav 2607300373Smav error = SYSCTL_OUT(req, outp, outsz); 2608300373Smav if (error || !req->newptr) 2609300373Smav return (error); 2610300373Smav return (EINVAL); 2611250079Scarl} 2612250079Scarl 2613300373Smavstatic unsigned 2614304404Smavintel_ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2615300373Smav{ 2616300373Smav 2617301811Sngie if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2618301811Sngie uidx >= ntb->b2b_mw_idx) || 2619301811Sngie (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2620301811Sngie uidx++; 2621301811Sngie if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2622301811Sngie uidx >= ntb->b2b_mw_idx) && 2623301811Sngie (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2624301811Sngie uidx++; 2625300373Smav return (uidx); 2626300373Smav} 2627300373Smav 2628304410Smav#ifndef EARLY_AP_STARTUP 2629304409Smavstatic int msix_ready; 2630304409Smav 2631301811Sngiestatic void 2632304409Smavintel_ntb_msix_ready(void *arg __unused) 2633304409Smav{ 2634304409Smav 2635304409Smav msix_ready = 1; 2636304409Smav} 2637304409SmavSYSINIT(intel_ntb_msix_ready, SI_SUB_SMP, SI_ORDER_ANY, 2638304409Smav intel_ntb_msix_ready, NULL); 2639304410Smav#endif 2640304409Smav 2641304409Smavstatic void 2642304404Smavintel_ntb_exchange_msix(void *ctx) 2643301811Sngie{ 2644301811Sngie struct ntb_softc *ntb; 2645301811Sngie uint32_t val; 2646301811Sngie unsigned i; 2647301811Sngie 2648301811Sngie ntb = ctx; 2649301811Sngie 2650301903Smav if (ntb->peer_msix_good) 2651301903Smav goto msix_good; 2652301811Sngie if (ntb->peer_msix_done) 2653301811Sngie goto msix_done; 2654301811Sngie 2655304410Smav#ifndef EARLY_AP_STARTUP 2656304409Smav /* Block MSIX negotiation until SMP started and IRQ reshuffled. */ 2657304409Smav if (!msix_ready) 2658304409Smav goto reschedule; 2659304410Smav#endif 2660304409Smav 2661304404Smav intel_ntb_get_msix_info(ntb); 2662301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2663304404Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 2664301811Sngie ntb->msix_data[i].nmd_data); 2665304404Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 2666301904Smav ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2667301811Sngie } 2668304404Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2669301811Sngie 2670304404Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 2671301811Sngie if (val != NTB_MSIX_VER_GUARD) 2672301811Sngie goto reschedule; 2673301811Sngie 2674301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2675304404Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 2676304404Smav intel_ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2677301811Sngie ntb->peer_msix_data[i].nmd_data = val; 2678304404Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 2679304404Smav intel_ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2680301811Sngie ntb->peer_msix_data[i].nmd_ofs = val; 2681301811Sngie } 2682301811Sngie 2683301811Sngie ntb->peer_msix_done = true; 2684301811Sngie 2685301811Sngiemsix_done: 2686304404Smav intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2687304404Smav intel_ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 2688301811Sngie if (val != NTB_MSIX_RECEIVED) 2689301811Sngie goto reschedule; 2690301811Sngie 2691304406Smav intel_ntb_spad_clear(ntb->device); 2692301811Sngie ntb->peer_msix_good = true; 2693301903Smav /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2694301903Smav goto reschedule; 2695301811Sngie 2696301903Smavmsix_good: 2697304404Smav intel_ntb_poll_link(ntb); 2698304380Smav ntb_link_event(ntb->device); 2699301811Sngie return; 2700301811Sngie 2701301811Sngiereschedule: 2702301811Sngie ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2703301903Smav if (_xeon_link_is_up(ntb)) { 2704301903Smav callout_reset(&ntb->peer_msix_work, 2705301903Smav hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2706304404Smav intel_ntb_exchange_msix, ntb); 2707301903Smav } else 2708304404Smav intel_ntb_spad_clear(ntb->device); 2709301811Sngie} 2710301811Sngie 2711300373Smav/* 2712300373Smav * Public API to the rest of the OS 2713300373Smav */ 2714300373Smav 2715304380Smavstatic uint8_t 2716304404Smavintel_ntb_spad_count(device_t dev) 2717250079Scarl{ 2718304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2719250079Scarl 2720300373Smav return (ntb->spad_count); 2721250079Scarl} 2722250079Scarl 2723304380Smavstatic uint8_t 2724304404Smavintel_ntb_mw_count(device_t dev) 2725300373Smav{ 2726304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2727301811Sngie uint8_t res; 2728300373Smav 2729301811Sngie res = ntb->mw_count; 2730300373Smav if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2731301811Sngie res--; 2732301811Sngie if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2733301811Sngie res--; 2734301811Sngie return (res); 2735300373Smav} 2736300373Smav 2737304380Smavstatic int 2738304404Smavintel_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 2739250079Scarl{ 2740304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2741250079Scarl 2742300373Smav if (idx >= ntb->spad_count) 2743250079Scarl return (EINVAL); 2744250079Scarl 2745304404Smav intel_ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2746250079Scarl 2747250079Scarl return (0); 2748250079Scarl} 2749250079Scarl 2750301811Sngie/* 2751301811Sngie * Zeros the local scratchpad. 2752301811Sngie */ 2753304380Smavstatic void 2754304404Smavintel_ntb_spad_clear(device_t dev) 2755301811Sngie{ 2756304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2757301811Sngie unsigned i; 2758301811Sngie 2759301811Sngie for (i = 0; i < ntb->spad_count; i++) 2760304404Smav intel_ntb_spad_write(dev, i, 0); 2761301811Sngie} 2762301811Sngie 2763304380Smavstatic int 2764304404Smavintel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2765250079Scarl{ 2766304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2767250079Scarl 2768300373Smav if (idx >= ntb->spad_count) 2769250079Scarl return (EINVAL); 2770250079Scarl 2771304404Smav *val = intel_ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2772250079Scarl 2773250079Scarl return (0); 2774250079Scarl} 2775250079Scarl 2776304380Smavstatic int 2777304404Smavintel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 2778250079Scarl{ 2779304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2780250079Scarl 2781300373Smav if (idx >= ntb->spad_count) 2782250079Scarl return (EINVAL); 2783250079Scarl 2784304380Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2785304404Smav intel_ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2786255279Scarl else 2787304404Smav intel_ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2788250079Scarl 2789250079Scarl return (0); 2790250079Scarl} 2791250079Scarl 2792304380Smavstatic int 2793304404Smavintel_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2794250079Scarl{ 2795304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2796250079Scarl 2797300373Smav if (idx >= ntb->spad_count) 2798250079Scarl return (EINVAL); 2799250079Scarl 2800304380Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2801304404Smav *val = intel_ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2802255279Scarl else 2803304404Smav *val = intel_ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2804250079Scarl 2805250079Scarl return (0); 2806250079Scarl} 2807250079Scarl 2808304380Smavstatic int 2809304404Smavintel_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 2810300373Smav caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2811300373Smav bus_addr_t *plimit) 2812250079Scarl{ 2813304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2814300373Smav struct ntb_pci_bar_info *bar; 2815300373Smav bus_addr_t limit; 2816300373Smav size_t bar_b2b_off; 2817300373Smav enum ntb_bar bar_num; 2818250079Scarl 2819304404Smav if (mw_idx >= intel_ntb_mw_count(dev)) 2820300373Smav return (EINVAL); 2821304404Smav mw_idx = intel_ntb_user_mw_to_idx(ntb, mw_idx); 2822250079Scarl 2823304404Smav bar_num = intel_ntb_mw_to_bar(ntb, mw_idx); 2824300373Smav bar = &ntb->bar_info[bar_num]; 2825300373Smav bar_b2b_off = 0; 2826300373Smav if (mw_idx == ntb->b2b_mw_idx) { 2827300373Smav KASSERT(ntb->b2b_off != 0, 2828300373Smav ("user shouldn't get non-shared b2b mw")); 2829300373Smav bar_b2b_off = ntb->b2b_off; 2830300373Smav } 2831300373Smav 2832300373Smav if (bar_is_64bit(ntb, bar_num)) 2833300373Smav limit = BUS_SPACE_MAXADDR; 2834300373Smav else 2835300373Smav limit = BUS_SPACE_MAXADDR_32BIT; 2836300373Smav 2837300373Smav if (base != NULL) 2838300373Smav *base = bar->pbase + bar_b2b_off; 2839300373Smav if (vbase != NULL) 2840300373Smav *vbase = bar->vbase + bar_b2b_off; 2841300373Smav if (size != NULL) 2842300373Smav *size = bar->size - bar_b2b_off; 2843300373Smav if (align != NULL) 2844300373Smav *align = bar->size; 2845300373Smav if (align_size != NULL) 2846300373Smav *align_size = 1; 2847300373Smav if (plimit != NULL) 2848300373Smav *plimit = limit; 2849300373Smav return (0); 2850250079Scarl} 2851250079Scarl 2852304380Smavstatic int 2853304404Smavintel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 2854250079Scarl{ 2855304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2856300373Smav struct ntb_pci_bar_info *bar; 2857300373Smav uint64_t base, limit, reg_val; 2858300373Smav size_t bar_size, mw_size; 2859300373Smav uint32_t base_reg, xlat_reg, limit_reg; 2860300373Smav enum ntb_bar bar_num; 2861250079Scarl 2862304404Smav if (idx >= intel_ntb_mw_count(dev)) 2863300373Smav return (EINVAL); 2864304404Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2865250079Scarl 2866304404Smav bar_num = intel_ntb_mw_to_bar(ntb, idx); 2867300373Smav bar = &ntb->bar_info[bar_num]; 2868300373Smav 2869300373Smav bar_size = bar->size; 2870300373Smav if (idx == ntb->b2b_mw_idx) 2871300373Smav mw_size = bar_size - ntb->b2b_off; 2872300373Smav else 2873300373Smav mw_size = bar_size; 2874300373Smav 2875300373Smav /* Hardware requires that addr is aligned to bar size */ 2876300373Smav if ((addr & (bar_size - 1)) != 0) 2877300373Smav return (EINVAL); 2878300373Smav 2879300373Smav if (size > mw_size) 2880300373Smav return (EINVAL); 2881300373Smav 2882300373Smav bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2883300373Smav 2884300373Smav limit = 0; 2885300373Smav if (bar_is_64bit(ntb, bar_num)) { 2886304404Smav base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2887300373Smav 2888300373Smav if (limit_reg != 0 && size != mw_size) 2889300373Smav limit = base + size; 2890300373Smav 2891300373Smav /* Set and verify translation address */ 2892304404Smav intel_ntb_reg_write(8, xlat_reg, addr); 2893304404Smav reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2894300373Smav if (reg_val != addr) { 2895304404Smav intel_ntb_reg_write(8, xlat_reg, 0); 2896300373Smav return (EIO); 2897300373Smav } 2898300373Smav 2899300373Smav /* Set and verify the limit */ 2900304404Smav intel_ntb_reg_write(8, limit_reg, limit); 2901304404Smav reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 2902300373Smav if (reg_val != limit) { 2903304404Smav intel_ntb_reg_write(8, limit_reg, base); 2904304404Smav intel_ntb_reg_write(8, xlat_reg, 0); 2905300373Smav return (EIO); 2906300373Smav } 2907300373Smav } else { 2908300373Smav /* Configure 32-bit (split) BAR MW */ 2909300373Smav 2910300373Smav if ((addr & UINT32_MAX) != addr) 2911300373Smav return (ERANGE); 2912300373Smav if (((addr + size) & UINT32_MAX) != (addr + size)) 2913300373Smav return (ERANGE); 2914300373Smav 2915304404Smav base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 2916300373Smav 2917300373Smav if (limit_reg != 0 && size != mw_size) 2918300373Smav limit = base + size; 2919300373Smav 2920300373Smav /* Set and verify translation address */ 2921304404Smav intel_ntb_reg_write(4, xlat_reg, addr); 2922304404Smav reg_val = intel_ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 2923300373Smav if (reg_val != addr) { 2924304404Smav intel_ntb_reg_write(4, xlat_reg, 0); 2925300373Smav return (EIO); 2926300373Smav } 2927300373Smav 2928300373Smav /* Set and verify the limit */ 2929304404Smav intel_ntb_reg_write(4, limit_reg, limit); 2930304404Smav reg_val = intel_ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 2931300373Smav if (reg_val != limit) { 2932304404Smav intel_ntb_reg_write(4, limit_reg, base); 2933304404Smav intel_ntb_reg_write(4, xlat_reg, 0); 2934300373Smav return (EIO); 2935300373Smav } 2936300373Smav } 2937300373Smav return (0); 2938250079Scarl} 2939250079Scarl 2940304380Smavstatic int 2941304404Smavintel_ntb_mw_clear_trans(device_t dev, unsigned mw_idx) 2942250079Scarl{ 2943250079Scarl 2944304404Smav return (intel_ntb_mw_set_trans(dev, mw_idx, 0, 0)); 2945300373Smav} 2946300373Smav 2947304380Smavstatic int 2948304404Smavintel_ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 2949300373Smav{ 2950304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2951300373Smav struct ntb_pci_bar_info *bar; 2952300373Smav 2953304404Smav if (idx >= intel_ntb_mw_count(dev)) 2954300373Smav return (EINVAL); 2955304404Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2956300373Smav 2957304404Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 2958300373Smav *mode = bar->map_mode; 2959300373Smav return (0); 2960300373Smav} 2961300373Smav 2962304380Smavstatic int 2963304404Smavintel_ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 2964300373Smav{ 2965304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2966300373Smav 2967304404Smav if (idx >= intel_ntb_mw_count(dev)) 2968300373Smav return (EINVAL); 2969300373Smav 2970304404Smav idx = intel_ntb_user_mw_to_idx(ntb, idx); 2971304404Smav return (intel_ntb_mw_set_wc_internal(ntb, idx, mode)); 2972300373Smav} 2973300373Smav 2974300373Smavstatic int 2975304404Smavintel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 2976300373Smav{ 2977300373Smav struct ntb_pci_bar_info *bar; 2978300373Smav int rc; 2979300373Smav 2980304404Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 2981300373Smav if (bar->map_mode == mode) 2982250079Scarl return (0); 2983250079Scarl 2984300373Smav rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 2985300373Smav if (rc == 0) 2986300373Smav bar->map_mode = mode; 2987300373Smav 2988300373Smav return (rc); 2989250079Scarl} 2990250079Scarl 2991304380Smavstatic void 2992304404Smavintel_ntb_peer_db_set(device_t dev, uint64_t bit) 2993250079Scarl{ 2994304380Smav struct ntb_softc *ntb = device_get_softc(dev); 2995250079Scarl 2996304380Smav if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2997301811Sngie struct ntb_pci_bar_info *lapic; 2998301811Sngie unsigned i; 2999301811Sngie 3000301811Sngie lapic = ntb->peer_lapic_bar; 3001301811Sngie 3002301811Sngie for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3003304404Smav if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0) 3004301811Sngie bus_space_write_4(lapic->pci_bus_tag, 3005301811Sngie lapic->pci_bus_handle, 3006301811Sngie ntb->peer_msix_data[i].nmd_ofs, 3007301811Sngie ntb->peer_msix_data[i].nmd_data); 3008301811Sngie } 3009301811Sngie return; 3010301811Sngie } 3011301811Sngie 3012304380Smav if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3013304404Smav intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3014250079Scarl return; 3015300373Smav } 3016250079Scarl 3017300373Smav db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3018300373Smav} 3019300373Smav 3020304380Smavstatic int 3021304404Smavintel_ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3022300373Smav{ 3023304380Smav struct ntb_softc *ntb = device_get_softc(dev); 3024300373Smav struct ntb_pci_bar_info *bar; 3025300373Smav uint64_t regoff; 3026300373Smav 3027304380Smav KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3028300373Smav 3029304380Smav if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3030300373Smav bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3031300373Smav regoff = ntb->peer_reg->db_bell; 3032300373Smav } else { 3033300373Smav KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3034300373Smav ("invalid b2b idx")); 3035300373Smav 3036304404Smav bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3037300373Smav regoff = XEON_PDOORBELL_OFFSET; 3038250079Scarl } 3039300373Smav KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3040300373Smav 3041300373Smav /* HACK: Specific to current x86 bus implementation. */ 3042304380Smav *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3043304380Smav *db_size = ntb->reg->db_size; 3044304380Smav return (0); 3045250079Scarl} 3046250079Scarl 3047304380Smavstatic uint64_t 3048304404Smavintel_ntb_db_valid_mask(device_t dev) 3049250079Scarl{ 3050304380Smav struct ntb_softc *ntb = device_get_softc(dev); 3051250079Scarl 3052300373Smav return (ntb->db_valid_mask); 3053250079Scarl} 3054250079Scarl 3055304380Smavstatic int 3056304404Smavintel_ntb_db_vector_count(device_t dev) 3057300373Smav{ 3058304380Smav struct ntb_softc *ntb = device_get_softc(dev); 3059300373Smav 3060304380Smav return (ntb->db_vec_count); 3061304380Smav} 3062304380Smav 3063304380Smavstatic uint64_t 3064304404Smavintel_ntb_db_vector_mask(device_t dev, uint32_t vector) 3065304380Smav{ 3066304380Smav struct ntb_softc *ntb = device_get_softc(dev); 3067304380Smav 3068300373Smav if (vector > ntb->db_vec_count) 3069300373Smav return (0); 3070304404Smav return (ntb->db_valid_mask & intel_ntb_vec_mask(ntb, vector)); 3071300373Smav} 3072300373Smav 3073304380Smavstatic bool 3074304404Smavintel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3075250079Scarl{ 3076304380Smav struct ntb_softc *ntb = device_get_softc(dev); 3077250079Scarl 3078300373Smav if (speed != NULL) 3079304404Smav *speed = intel_ntb_link_sta_speed(ntb); 3080300373Smav if (width != NULL) 3081304404Smav *width = intel_ntb_link_sta_width(ntb); 3082300373Smav return (link_is_up(ntb)); 3083250079Scarl} 3084250079Scarl 3085255272Scarlstatic void 3086255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 3087250079Scarl{ 3088255272Scarl 3089300373Smav bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3090300373Smav bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3091300373Smav bar->pbase = rman_get_start(bar->pci_resource); 3092300373Smav bar->size = rman_get_size(bar->pci_resource); 3093300373Smav bar->vbase = rman_get_virtual(bar->pci_resource); 3094250079Scarl} 3095255268Scarl 3096304380Smavstatic device_method_t ntb_intel_methods[] = { 3097304380Smav /* Device interface */ 3098304404Smav DEVMETHOD(device_probe, intel_ntb_probe), 3099304404Smav DEVMETHOD(device_attach, intel_ntb_attach), 3100304404Smav DEVMETHOD(device_detach, intel_ntb_detach), 3101304380Smav /* NTB interface */ 3102304404Smav DEVMETHOD(ntb_link_is_up, intel_ntb_link_is_up), 3103304404Smav DEVMETHOD(ntb_link_enable, intel_ntb_link_enable), 3104304404Smav DEVMETHOD(ntb_link_disable, intel_ntb_link_disable), 3105304404Smav DEVMETHOD(ntb_link_enabled, intel_ntb_link_enabled), 3106304404Smav DEVMETHOD(ntb_mw_count, intel_ntb_mw_count), 3107304404Smav DEVMETHOD(ntb_mw_get_range, intel_ntb_mw_get_range), 3108304404Smav DEVMETHOD(ntb_mw_set_trans, intel_ntb_mw_set_trans), 3109304404Smav DEVMETHOD(ntb_mw_clear_trans, intel_ntb_mw_clear_trans), 3110304404Smav DEVMETHOD(ntb_mw_get_wc, intel_ntb_mw_get_wc), 3111304404Smav DEVMETHOD(ntb_mw_set_wc, intel_ntb_mw_set_wc), 3112304404Smav DEVMETHOD(ntb_spad_count, intel_ntb_spad_count), 3113304404Smav DEVMETHOD(ntb_spad_clear, intel_ntb_spad_clear), 3114304404Smav DEVMETHOD(ntb_spad_write, intel_ntb_spad_write), 3115304404Smav DEVMETHOD(ntb_spad_read, intel_ntb_spad_read), 3116304404Smav DEVMETHOD(ntb_peer_spad_write, intel_ntb_peer_spad_write), 3117304404Smav DEVMETHOD(ntb_peer_spad_read, intel_ntb_peer_spad_read), 3118304404Smav DEVMETHOD(ntb_db_valid_mask, intel_ntb_db_valid_mask), 3119304404Smav DEVMETHOD(ntb_db_vector_count, intel_ntb_db_vector_count), 3120304404Smav DEVMETHOD(ntb_db_vector_mask, intel_ntb_db_vector_mask), 3121304404Smav DEVMETHOD(ntb_db_clear, intel_ntb_db_clear), 3122304404Smav DEVMETHOD(ntb_db_clear_mask, intel_ntb_db_clear_mask), 3123304404Smav DEVMETHOD(ntb_db_read, intel_ntb_db_read), 3124304404Smav DEVMETHOD(ntb_db_set_mask, intel_ntb_db_set_mask), 3125304404Smav DEVMETHOD(ntb_peer_db_addr, intel_ntb_peer_db_addr), 3126304404Smav DEVMETHOD(ntb_peer_db_set, intel_ntb_peer_db_set), 3127304380Smav DEVMETHOD_END 3128304380Smav}; 3129255268Scarl 3130304380Smavstatic DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3131304380Smav sizeof(struct ntb_softc)); 3132304380SmavDRIVER_MODULE(ntb_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL); 3133304380SmavMODULE_DEPEND(ntb_intel, ntb, 1, 1, 1); 3134304380SmavMODULE_VERSION(ntb_intel, 1); 3135