ntb_hw_intel.c revision 291031
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3289542Scem * Copyright (C) 2015 EMC Corporation 4250079Scarl * All rights reserved. 5250079Scarl * 6250079Scarl * Redistribution and use in source and binary forms, with or without 7250079Scarl * modification, are permitted provided that the following conditions 8250079Scarl * are met: 9250079Scarl * 1. Redistributions of source code must retain the above copyright 10250079Scarl * notice, this list of conditions and the following disclaimer. 11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 12250079Scarl * notice, this list of conditions and the following disclaimer in the 13250079Scarl * documentation and/or other materials provided with the distribution. 14250079Scarl * 15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25250079Scarl * SUCH DAMAGE. 26250079Scarl */ 27250079Scarl 28250079Scarl#include <sys/cdefs.h> 29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 291031 2015-11-18 22:20:21Z cem $"); 30250079Scarl 31250079Scarl#include <sys/param.h> 32250079Scarl#include <sys/kernel.h> 33250079Scarl#include <sys/systm.h> 34250079Scarl#include <sys/bus.h> 35289774Scem#include <sys/endian.h> 36250079Scarl#include <sys/malloc.h> 37250079Scarl#include <sys/module.h> 38250079Scarl#include <sys/queue.h> 39250079Scarl#include <sys/rman.h> 40289774Scem#include <sys/sbuf.h> 41289207Scem#include <sys/sysctl.h> 42250079Scarl#include <vm/vm.h> 43250079Scarl#include <vm/pmap.h> 44250079Scarl#include <machine/bus.h> 45250079Scarl#include <machine/pmap.h> 46250079Scarl#include <machine/resource.h> 47250079Scarl#include <dev/pci/pcireg.h> 48250079Scarl#include <dev/pci/pcivar.h> 49250079Scarl 50250079Scarl#include "ntb_regs.h" 51250079Scarl#include "ntb_hw.h" 52250079Scarl 53250079Scarl/* 54250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 55250079Scarl * allows you to connect two systems using a PCI-e link. 56250079Scarl * 57250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 58250079Scarl * you to send and recieve interrupts, map the memory windows and send and 59250079Scarl * receive messages in the scratch-pad registers. 60250079Scarl * 61250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 62250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 63250079Scarl */ 64250079Scarl 65289648Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 66250079Scarl 67289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 68289648Scem#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 69250079Scarl 70250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 71250079Scarl 72250079Scarlenum ntb_device_type { 73250079Scarl NTB_XEON, 74289648Scem NTB_ATOM 75250079Scarl}; 76250079Scarl 77289610Scem/* ntb_conn_type are hardware numbers, cannot change. */ 78289610Scemenum ntb_conn_type { 79289610Scem NTB_CONN_TRANSPARENT = 0, 80289610Scem NTB_CONN_B2B = 1, 81289610Scem NTB_CONN_RP = 2, 82289610Scem}; 83289610Scem 84289610Scemenum ntb_b2b_direction { 85289610Scem NTB_DEV_USD = 0, 86289610Scem NTB_DEV_DSD = 1, 87289610Scem}; 88289610Scem 89289539Scemenum ntb_bar { 90289539Scem NTB_CONFIG_BAR = 0, 91289539Scem NTB_B2B_BAR_1, 92289539Scem NTB_B2B_BAR_2, 93289539Scem NTB_B2B_BAR_3, 94289539Scem NTB_MAX_BARS 95289539Scem}; 96289539Scem 97255274Scarl/* Device features and workarounds */ 98255274Scarl#define HAS_FEATURE(feature) \ 99255274Scarl ((ntb->features & (feature)) != 0) 100255274Scarl 101250079Scarlstruct ntb_hw_info { 102250079Scarl uint32_t device_id; 103255274Scarl const char *desc; 104250079Scarl enum ntb_device_type type; 105289397Scem uint32_t features; 106250079Scarl}; 107250079Scarl 108250079Scarlstruct ntb_pci_bar_info { 109250079Scarl bus_space_tag_t pci_bus_tag; 110250079Scarl bus_space_handle_t pci_bus_handle; 111250079Scarl int pci_resource_id; 112250079Scarl struct resource *pci_resource; 113250079Scarl vm_paddr_t pbase; 114290679Scem caddr_t vbase; 115290679Scem vm_size_t size; 116291031Scem bool mapped_wc : 1; 117289543Scem 118289543Scem /* Configuration register offsets */ 119289543Scem uint32_t psz_off; 120289543Scem uint32_t ssz_off; 121289543Scem uint32_t pbarxlat_off; 122250079Scarl}; 123250079Scarl 124250079Scarlstruct ntb_int_info { 125250079Scarl struct resource *res; 126250079Scarl int rid; 127250079Scarl void *tag; 128250079Scarl}; 129250079Scarl 130289546Scemstruct ntb_vec { 131250079Scarl struct ntb_softc *ntb; 132289546Scem uint32_t num; 133250079Scarl}; 134250079Scarl 135289542Scemstruct ntb_reg { 136289542Scem uint32_t ntb_ctl; 137289542Scem uint32_t lnk_sta; 138289542Scem uint8_t db_size; 139289542Scem unsigned mw_bar[NTB_MAX_BARS]; 140289542Scem}; 141289542Scem 142289542Scemstruct ntb_alt_reg { 143289542Scem uint32_t db_bell; 144289542Scem uint32_t db_mask; 145289542Scem uint32_t spad; 146289542Scem}; 147289542Scem 148289542Scemstruct ntb_xlat_reg { 149289546Scem uint32_t bar0_base; 150289546Scem uint32_t bar2_base; 151289546Scem uint32_t bar4_base; 152289546Scem uint32_t bar5_base; 153289546Scem 154289546Scem uint32_t bar2_xlat; 155289546Scem uint32_t bar4_xlat; 156289546Scem uint32_t bar5_xlat; 157289546Scem 158289546Scem uint32_t bar2_limit; 159289546Scem uint32_t bar4_limit; 160289546Scem uint32_t bar5_limit; 161289542Scem}; 162289542Scem 163289542Scemstruct ntb_b2b_addr { 164289542Scem uint64_t bar0_addr; 165289542Scem uint64_t bar2_addr64; 166289542Scem uint64_t bar4_addr64; 167289542Scem uint64_t bar4_addr32; 168289542Scem uint64_t bar5_addr32; 169289542Scem}; 170289542Scem 171250079Scarlstruct ntb_softc { 172250079Scarl device_t device; 173250079Scarl enum ntb_device_type type; 174289774Scem uint32_t features; 175250079Scarl 176250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 177250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 178250079Scarl uint32_t allocated_interrupts; 179250079Scarl 180250079Scarl struct callout heartbeat_timer; 181250079Scarl struct callout lr_timer; 182250079Scarl 183289546Scem void *ntb_ctx; 184289546Scem const struct ntb_ctx_ops *ctx_ops; 185289546Scem struct ntb_vec *msix_vec; 186290683Scem#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 187290683Scem#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 188289546Scem#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 189289546Scem struct mtx ctx_lock; 190250079Scarl 191289610Scem uint32_t ppd; 192289610Scem enum ntb_conn_type conn_type; 193289610Scem enum ntb_b2b_direction dev_type; 194289539Scem 195289542Scem /* Offset of peer bar0 in B2B BAR */ 196289542Scem uint64_t b2b_off; 197289542Scem /* Memory window used to access peer bar0 */ 198289543Scem#define B2B_MW_DISABLED UINT8_MAX 199289542Scem uint8_t b2b_mw_idx; 200289542Scem 201289539Scem uint8_t mw_count; 202289539Scem uint8_t spad_count; 203289539Scem uint8_t db_count; 204289539Scem uint8_t db_vec_count; 205289539Scem uint8_t db_vec_shift; 206289542Scem 207289546Scem /* Protects local db_mask. */ 208289546Scem#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 209289546Scem#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 210289546Scem#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 211289542Scem struct mtx db_mask_lock; 212289542Scem 213290686Scem volatile uint32_t ntb_ctl; 214290686Scem volatile uint32_t lnk_sta; 215289542Scem 216289542Scem uint64_t db_valid_mask; 217289542Scem uint64_t db_link_mask; 218289546Scem uint64_t db_mask; 219289542Scem 220289542Scem int last_ts; /* ticks @ last irq */ 221289542Scem 222289542Scem const struct ntb_reg *reg; 223289542Scem const struct ntb_alt_reg *self_reg; 224289542Scem const struct ntb_alt_reg *peer_reg; 225289542Scem const struct ntb_xlat_reg *xlat_reg; 226250079Scarl}; 227250079Scarl 228289234Scem#ifdef __i386__ 229289234Scemstatic __inline uint64_t 230289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 231289234Scem bus_size_t offset) 232289234Scem{ 233289234Scem 234289234Scem return (bus_space_read_4(tag, handle, offset) | 235289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 236289234Scem} 237289234Scem 238289234Scemstatic __inline void 239289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 240289234Scem bus_size_t offset, uint64_t val) 241289234Scem{ 242289234Scem 243289234Scem bus_space_write_4(tag, handle, offset, val); 244289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 245289234Scem} 246289234Scem#endif 247289234Scem 248255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 249255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 250255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 251255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 252255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 253255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 254255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 255250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 256255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 257289397Scem#define ntb_mw_read(SIZE, offset) \ 258289542Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 259255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 260289542Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 261289397Scem offset, val) 262250079Scarl 263250079Scarlstatic int ntb_probe(device_t device); 264250079Scarlstatic int ntb_attach(device_t device); 265250079Scarlstatic int ntb_detach(device_t device); 266289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 267289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 268289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 269289546Scem uint32_t *base, uint32_t *xlat, uint32_t *lmt); 270255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 271289647Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 272289647Scem const char *); 273255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 274255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 275255272Scarl struct ntb_pci_bar_info *bar); 276250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 277289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 278289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 279289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 280289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 281250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 282289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 283289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 284289546Scemstatic void ndev_vec_isr(void *arg); 285289546Scemstatic void ndev_irq_isr(void *arg); 286289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 287290678Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 288290678Scemstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 289289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 290289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 291250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 292289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 293289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 294289648Scemstatic int ntb_detect_atom(struct ntb_softc *ntb); 295289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 296289648Scemstatic int ntb_atom_init_dev(struct ntb_softc *ntb); 297289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 298289648Scemstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 299289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 300289543Scem enum ntb_bar regbar); 301289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 302289543Scem uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 303289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 304289543Scem enum ntb_bar idx); 305289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *, 306289542Scem const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 307289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb); 308289648Scemstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 309289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 310289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 311289648Scemstatic void atom_link_hb(void *arg); 312289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 313289648Scemstatic void recover_atom_link(void *arg); 314289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb); 315255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 316289774Scemstatic void ntb_sysctl_init(struct ntb_softc *); 317289774Scemstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 318289774Scemstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 319289774Scemstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 320250079Scarl 321290685Scemstatic unsigned g_ntb_hw_debug_level; 322290685ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 323290685Scem &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 324290685Scem#define ntb_printf(lvl, ...) do { \ 325290685Scem if ((lvl) <= g_ntb_hw_debug_level) { \ 326290685Scem device_printf(ntb->device, __VA_ARGS__); \ 327290685Scem } \ 328290685Scem} while (0) 329290685Scem 330291030Scemstatic unsigned g_ntb_enable_wc = 1; 331291030ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, enable_writecombine, CTLFLAG_RDTUN, 332291030Scem &g_ntb_enable_wc, 0, "Set to 1 to map memory windows write combining"); 333291030Scem 334250079Scarlstatic struct ntb_hw_info pci_ids[] = { 335289612Scem /* XXX: PS/SS IDs left out until they are supported. */ 336289612Scem { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 337289648Scem NTB_ATOM, 0 }, 338289233Scem 339289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 340289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 341289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 342289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 343289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 344289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 345289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 346289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 347289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 348289538Scem NTB_SB01BASE_LOCKUP }, 349289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 350289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 351289538Scem NTB_SB01BASE_LOCKUP }, 352289233Scem 353289648Scem { 0x00000000, NULL, NTB_ATOM, 0 } 354250079Scarl}; 355250079Scarl 356289648Scemstatic const struct ntb_reg atom_reg = { 357289648Scem .ntb_ctl = ATOM_NTBCNTL_OFFSET, 358289648Scem .lnk_sta = ATOM_LINK_STATUS_OFFSET, 359289542Scem .db_size = sizeof(uint64_t), 360289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 361289542Scem}; 362289542Scem 363289648Scemstatic const struct ntb_alt_reg atom_pri_reg = { 364289648Scem .db_bell = ATOM_PDOORBELL_OFFSET, 365289648Scem .db_mask = ATOM_PDBMSK_OFFSET, 366289648Scem .spad = ATOM_SPAD_OFFSET, 367289607Scem}; 368289607Scem 369289648Scemstatic const struct ntb_alt_reg atom_b2b_reg = { 370289648Scem .db_bell = ATOM_B2B_DOORBELL_OFFSET, 371289648Scem .spad = ATOM_B2B_SPAD_OFFSET, 372289542Scem}; 373289542Scem 374289648Scemstatic const struct ntb_xlat_reg atom_sec_xlat = { 375289542Scem#if 0 376289542Scem /* "FIXME" says the Linux driver. */ 377289648Scem .bar0_base = ATOM_SBAR0BASE_OFFSET, 378289648Scem .bar2_base = ATOM_SBAR2BASE_OFFSET, 379289648Scem .bar4_base = ATOM_SBAR4BASE_OFFSET, 380289546Scem 381289648Scem .bar2_limit = ATOM_SBAR2LMT_OFFSET, 382289648Scem .bar4_limit = ATOM_SBAR4LMT_OFFSET, 383289542Scem#endif 384289546Scem 385289648Scem .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 386289648Scem .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 387289542Scem}; 388289542Scem 389289542Scemstatic const struct ntb_reg xeon_reg = { 390289542Scem .ntb_ctl = XEON_NTBCNTL_OFFSET, 391289542Scem .lnk_sta = XEON_LINK_STATUS_OFFSET, 392289542Scem .db_size = sizeof(uint16_t), 393289542Scem .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 394289542Scem}; 395289542Scem 396289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = { 397289607Scem .db_bell = XEON_PDOORBELL_OFFSET, 398289607Scem .db_mask = XEON_PDBMSK_OFFSET, 399289607Scem .spad = XEON_SPAD_OFFSET, 400289607Scem}; 401289607Scem 402289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = { 403289542Scem .db_bell = XEON_B2B_DOORBELL_OFFSET, 404289542Scem .spad = XEON_B2B_SPAD_OFFSET, 405289542Scem}; 406289542Scem 407289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = { 408289542Scem .bar0_base = XEON_SBAR0BASE_OFFSET, 409289546Scem .bar2_base = XEON_SBAR2BASE_OFFSET, 410289546Scem .bar4_base = XEON_SBAR4BASE_OFFSET, 411289546Scem .bar5_base = XEON_SBAR5BASE_OFFSET, 412289546Scem 413289542Scem .bar2_limit = XEON_SBAR2LMT_OFFSET, 414289546Scem .bar4_limit = XEON_SBAR4LMT_OFFSET, 415289546Scem .bar5_limit = XEON_SBAR5LMT_OFFSET, 416289546Scem 417289542Scem .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 418289546Scem .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 419289546Scem .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 420289542Scem}; 421289542Scem 422289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 423290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 424290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 425290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 426290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 427290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 428289542Scem}; 429289542Scem 430289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 431290725Scem .bar0_addr = XEON_B2B_BAR0_ADDR, 432290725Scem .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 433290725Scem .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 434290725Scem .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 435290725Scem .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 436289542Scem}; 437289542Scem 438289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 439289614Scem "B2B MW segment overrides -- MUST be the same on both sides"); 440289614Scem 441289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 442289614Scem &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 443289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 444289614Scem "the window at BAR2, on the upstream side of the link. MUST be the same " 445289614Scem "address on both sides."); 446289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 447289614Scem &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 448289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 449289614Scem &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 450289614Scem "(split-BAR mode)."); 451289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 452289646Scem &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 453289614Scem "(split-BAR mode)."); 454289614Scem 455289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 456289614Scem &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 457289614Scem "hardware, use this 64-bit address on the bus between the NTB devices for " 458289614Scem "the window at BAR2, on the downstream side of the link. MUST be the same" 459289614Scem " address on both sides."); 460289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 461289614Scem &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 462289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 463289614Scem &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 464289614Scem "(split-BAR mode)."); 465289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 466289646Scem &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 467289614Scem "(split-BAR mode)."); 468289614Scem 469250079Scarl/* 470250079Scarl * OS <-> Driver interface structures 471250079Scarl */ 472250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 473250079Scarl 474250079Scarlstatic device_method_t ntb_pci_methods[] = { 475250079Scarl /* Device interface */ 476250079Scarl DEVMETHOD(device_probe, ntb_probe), 477250079Scarl DEVMETHOD(device_attach, ntb_attach), 478250079Scarl DEVMETHOD(device_detach, ntb_detach), 479250079Scarl DEVMETHOD_END 480250079Scarl}; 481250079Scarl 482250079Scarlstatic driver_t ntb_pci_driver = { 483250079Scarl "ntb_hw", 484250079Scarl ntb_pci_methods, 485250079Scarl sizeof(struct ntb_softc), 486250079Scarl}; 487250079Scarl 488250079Scarlstatic devclass_t ntb_devclass; 489250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 490250079ScarlMODULE_VERSION(ntb_hw, 1); 491250079Scarl 492289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 493289207Scem 494250079Scarl/* 495250079Scarl * OS <-> Driver linkage functions 496250079Scarl */ 497250079Scarlstatic int 498250079Scarlntb_probe(device_t device) 499250079Scarl{ 500289209Scem struct ntb_hw_info *p; 501250079Scarl 502289209Scem p = ntb_get_device_info(pci_get_devid(device)); 503289209Scem if (p == NULL) 504250079Scarl return (ENXIO); 505289209Scem 506289209Scem device_set_desc(device, p->desc); 507289209Scem return (0); 508250079Scarl} 509250079Scarl 510250079Scarlstatic int 511250079Scarlntb_attach(device_t device) 512250079Scarl{ 513289209Scem struct ntb_softc *ntb; 514289209Scem struct ntb_hw_info *p; 515250079Scarl int error; 516250079Scarl 517289209Scem ntb = DEVICE2SOFTC(device); 518289209Scem p = ntb_get_device_info(pci_get_devid(device)); 519289209Scem 520250079Scarl ntb->device = device; 521250079Scarl ntb->type = p->type; 522255274Scarl ntb->features = p->features; 523289543Scem ntb->b2b_mw_idx = B2B_MW_DISABLED; 524250079Scarl 525289648Scem /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 526283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 527283291Sjkim callout_init(&ntb->lr_timer, 1); 528289542Scem mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 529290683Scem mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 530250079Scarl 531289648Scem if (ntb->type == NTB_ATOM) 532289648Scem error = ntb_detect_atom(ntb); 533289348Scem else 534289348Scem error = ntb_detect_xeon(ntb); 535290682Scem if (error != 0) 536289348Scem goto out; 537289348Scem 538289397Scem ntb_detect_max_mw(ntb); 539289396Scem 540290682Scem pci_enable_busmaster(ntb->device); 541290682Scem 542289209Scem error = ntb_map_pci_bars(ntb); 543290682Scem if (error != 0) 544289209Scem goto out; 545289648Scem if (ntb->type == NTB_ATOM) 546289648Scem error = ntb_atom_init_dev(ntb); 547289272Scem else 548289542Scem error = ntb_xeon_init_dev(ntb); 549290682Scem if (error != 0) 550289209Scem goto out; 551290682Scem 552290682Scem ntb_poll_link(ntb); 553290682Scem 554289774Scem ntb_sysctl_init(ntb); 555250079Scarl 556289209Scemout: 557289209Scem if (error != 0) 558289209Scem ntb_detach(device); 559250079Scarl return (error); 560250079Scarl} 561250079Scarl 562250079Scarlstatic int 563250079Scarlntb_detach(device_t device) 564250079Scarl{ 565289209Scem struct ntb_softc *ntb; 566250079Scarl 567289209Scem ntb = DEVICE2SOFTC(device); 568289542Scem 569289617Scem if (ntb->self_reg != NULL) 570289617Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 571250079Scarl callout_drain(&ntb->heartbeat_timer); 572250079Scarl callout_drain(&ntb->lr_timer); 573290682Scem pci_disable_busmaster(ntb->device); 574289272Scem if (ntb->type == NTB_XEON) 575289272Scem ntb_teardown_xeon(ntb); 576250079Scarl ntb_teardown_interrupts(ntb); 577289397Scem 578289542Scem mtx_destroy(&ntb->db_mask_lock); 579289546Scem mtx_destroy(&ntb->ctx_lock); 580289542Scem 581289397Scem /* 582289397Scem * Redetect total MWs so we unmap properly -- in case we lowered the 583289397Scem * maximum to work around Xeon errata. 584289397Scem */ 585289397Scem ntb_detect_max_mw(ntb); 586250079Scarl ntb_unmap_pci_bar(ntb); 587250079Scarl 588250079Scarl return (0); 589250079Scarl} 590250079Scarl 591289542Scem/* 592289542Scem * Driver internal routines 593289542Scem */ 594289539Scemstatic inline enum ntb_bar 595289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 596289539Scem{ 597289539Scem 598289543Scem KASSERT(mw < ntb->mw_count || 599289543Scem (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx), 600289542Scem ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 601289546Scem KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 602289539Scem 603289542Scem return (ntb->reg->mw_bar[mw]); 604289539Scem} 605289539Scem 606289546Scemstatic inline bool 607289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 608289546Scem{ 609289546Scem /* XXX This assertion could be stronger. */ 610289546Scem KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 611289546Scem return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 612289546Scem} 613289546Scem 614289546Scemstatic inline void 615289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 616289546Scem uint32_t *xlat, uint32_t *lmt) 617289546Scem{ 618289546Scem uint32_t basev, lmtv, xlatv; 619289546Scem 620289546Scem switch (bar) { 621289546Scem case NTB_B2B_BAR_1: 622289546Scem basev = ntb->xlat_reg->bar2_base; 623289546Scem lmtv = ntb->xlat_reg->bar2_limit; 624289546Scem xlatv = ntb->xlat_reg->bar2_xlat; 625289546Scem break; 626289546Scem case NTB_B2B_BAR_2: 627289546Scem basev = ntb->xlat_reg->bar4_base; 628289546Scem lmtv = ntb->xlat_reg->bar4_limit; 629289546Scem xlatv = ntb->xlat_reg->bar4_xlat; 630289546Scem break; 631289546Scem case NTB_B2B_BAR_3: 632289546Scem basev = ntb->xlat_reg->bar5_base; 633289546Scem lmtv = ntb->xlat_reg->bar5_limit; 634289546Scem xlatv = ntb->xlat_reg->bar5_xlat; 635289546Scem break; 636289546Scem default: 637289546Scem KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 638289546Scem ("bad bar")); 639289546Scem basev = lmtv = xlatv = 0; 640289546Scem break; 641289546Scem } 642289546Scem 643289546Scem if (base != NULL) 644289546Scem *base = basev; 645289546Scem if (xlat != NULL) 646289546Scem *xlat = xlatv; 647289546Scem if (lmt != NULL) 648289546Scem *lmt = lmtv; 649289546Scem} 650289546Scem 651250079Scarlstatic int 652255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 653250079Scarl{ 654255272Scarl int rc; 655250079Scarl 656250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 657289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 658255272Scarl if (rc != 0) 659289541Scem goto out; 660255272Scarl 661289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 662289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 663255272Scarl if (rc != 0) 664289541Scem goto out; 665289543Scem ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 666289543Scem ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 667289543Scem ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 668255272Scarl 669289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 670289543Scem /* XXX Are shared MW B2Bs write-combining? */ 671289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 672289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 673255279Scarl else 674289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 675289543Scem ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 676289543Scem ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 677289543Scem ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 678289543Scem 679289397Scem if (!HAS_FEATURE(NTB_SPLIT_BAR)) 680289541Scem goto out; 681289397Scem 682289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 683289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 684289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 685289397Scem else 686289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 687289543Scem ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 688289543Scem ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 689289543Scem ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 690250079Scarl 691289541Scemout: 692289209Scem if (rc != 0) 693255272Scarl device_printf(ntb->device, 694255272Scarl "unable to allocate pci resource\n"); 695255272Scarl return (rc); 696255272Scarl} 697255272Scarl 698289541Scemstatic void 699289647Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 700289647Scem const char *kind) 701289541Scem{ 702289541Scem 703289647Scem device_printf(ntb->device, 704289647Scem "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 705289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 706289647Scem (char *)bar->vbase + bar->size - 1, 707289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 708289647Scem (uintmax_t)bar->size, kind); 709289541Scem} 710289541Scem 711255272Scarlstatic int 712255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 713255272Scarl{ 714255272Scarl 715255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 716289209Scem &bar->pci_resource_id, RF_ACTIVE); 717255272Scarl if (bar->pci_resource == NULL) 718255272Scarl return (ENXIO); 719289209Scem 720289209Scem save_bar_parameters(bar); 721289647Scem print_map_success(ntb, bar, "mmr"); 722289209Scem return (0); 723255272Scarl} 724255272Scarl 725255272Scarlstatic int 726255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 727255272Scarl{ 728255272Scarl int rc; 729255276Scarl uint8_t bar_size_bits = 0; 730255272Scarl 731289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 732289209Scem &bar->pci_resource_id, RF_ACTIVE); 733250079Scarl 734255272Scarl if (bar->pci_resource == NULL) 735255272Scarl return (ENXIO); 736255276Scarl 737289209Scem save_bar_parameters(bar); 738289209Scem /* 739289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 740289209Scem * hardware issue. To work around this, query the size it should be 741289209Scem * configured to by the device and modify the resource to correspond to 742289209Scem * this new size. The BIOS on systems with this problem is required to 743289209Scem * provide enough address space to allow the driver to make this change 744289209Scem * safely. 745289209Scem * 746289209Scem * Ideally I could have just specified the size when I allocated the 747289209Scem * resource like: 748289209Scem * bus_alloc_resource(ntb->device, 749289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 750289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 751289209Scem * but the PCI driver does not honor the size in this call, so we have 752289209Scem * to modify it after the fact. 753289209Scem */ 754289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 755289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 756289209Scem bar_size_bits = pci_read_config(ntb->device, 757289209Scem XEON_PBAR23SZ_OFFSET, 1); 758289209Scem else 759289209Scem bar_size_bits = pci_read_config(ntb->device, 760289209Scem XEON_PBAR45SZ_OFFSET, 1); 761289209Scem 762289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 763289209Scem bar->pci_resource, bar->pbase, 764289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 765255272Scarl if (rc != 0) { 766289209Scem device_printf(ntb->device, 767289209Scem "unable to resize bar\n"); 768255272Scarl return (rc); 769250079Scarl } 770289209Scem 771289209Scem save_bar_parameters(bar); 772250079Scarl } 773289209Scem 774291030Scem print_map_success(ntb, bar, "mw"); 775291030Scem if (g_ntb_enable_wc == 0) 776291030Scem return (0); 777291030Scem 778289209Scem /* Mark bar region as write combining to improve performance. */ 779289209Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 780289209Scem VM_MEMATTR_WRITE_COMBINING); 781291031Scem if (rc == 0) { 782291031Scem bar->mapped_wc = true; 783289209Scem device_printf(ntb->device, 784289647Scem "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 785289647Scem "WRITE_COMBINING.\n", 786289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 787289647Scem (char *)bar->vbase + bar->size - 1, 788289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1)); 789291031Scem } else 790289647Scem device_printf(ntb->device, 791289647Scem "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 792289647Scem "WRITE_COMBINING: %d\n", 793289647Scem PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 794289647Scem (char *)bar->vbase + bar->size - 1, 795289647Scem (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 796289647Scem rc); 797289647Scem /* Proceed anyway */ 798250079Scarl return (0); 799250079Scarl} 800250079Scarl 801250079Scarlstatic void 802250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 803250079Scarl{ 804250079Scarl struct ntb_pci_bar_info *current_bar; 805250079Scarl int i; 806250079Scarl 807289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 808250079Scarl current_bar = &ntb->bar_info[i]; 809250079Scarl if (current_bar->pci_resource != NULL) 810250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 811250079Scarl current_bar->pci_resource_id, 812250079Scarl current_bar->pci_resource); 813250079Scarl } 814250079Scarl} 815250079Scarl 816250079Scarlstatic int 817289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 818250079Scarl{ 819289342Scem uint32_t i; 820289342Scem int rc; 821289342Scem 822289342Scem for (i = 0; i < num_vectors; i++) { 823289342Scem ntb->int_info[i].rid = i + 1; 824289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 825289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 826289342Scem if (ntb->int_info[i].res == NULL) { 827289342Scem device_printf(ntb->device, 828289342Scem "bus_alloc_resource failed\n"); 829289342Scem return (ENOMEM); 830289342Scem } 831289342Scem ntb->int_info[i].tag = NULL; 832289342Scem ntb->allocated_interrupts++; 833289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 834289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 835289546Scem &ntb->msix_vec[i], &ntb->int_info[i].tag); 836289342Scem if (rc != 0) { 837289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 838289342Scem return (ENXIO); 839289342Scem } 840289342Scem } 841289342Scem return (0); 842289342Scem} 843289342Scem 844289344Scem/* 845289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 846289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 847289344Scem * should be okay. This tunable should enable us to test that hypothesis 848289344Scem * when someone gets their hands on some Xeon hardware. 849289344Scem */ 850289344Scemstatic int ntb_force_remap_mode; 851289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 852289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 853289344Scem " to a smaller number of ithreads, even if the desired number are " 854289344Scem "available"); 855289344Scem 856289344Scem/* 857289344Scem * In case it is NOT ok, give consumers an abort button. 858289344Scem */ 859289344Scemstatic int ntb_prefer_intx; 860289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 861289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 862289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 863289344Scem "behavior)"); 864289344Scem 865289344Scem/* 866289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 867289344Scem * round-robin fashion. 868289344Scem */ 869289342Scemstatic int 870289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 871289344Scem{ 872289344Scem u_int *vectors; 873289344Scem uint32_t i; 874289344Scem int rc; 875289344Scem 876289344Scem if (ntb_prefer_intx != 0) 877289344Scem return (ENXIO); 878289344Scem 879289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 880289344Scem 881289344Scem for (i = 0; i < desired; i++) 882289344Scem vectors[i] = (i % avail) + 1; 883289344Scem 884289344Scem rc = pci_remap_msix(dev, desired, vectors); 885289344Scem free(vectors, M_NTB); 886289344Scem return (rc); 887289344Scem} 888289344Scem 889289344Scemstatic int 890289540Scemntb_init_isr(struct ntb_softc *ntb) 891289342Scem{ 892289344Scem uint32_t desired_vectors, num_vectors; 893289342Scem int rc; 894250079Scarl 895250079Scarl ntb->allocated_interrupts = 0; 896289542Scem ntb->last_ts = ticks; 897289347Scem 898250079Scarl /* 899289546Scem * Mask all doorbell interrupts. 900250079Scarl */ 901289546Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 902250079Scarl 903289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 904289539Scem ntb->db_count); 905289344Scem if (desired_vectors >= 1) { 906289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 907250079Scarl 908289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 909289344Scem num_vectors == desired_vectors) 910289344Scem num_vectors--; 911289344Scem 912289344Scem if (rc == 0 && num_vectors < desired_vectors) { 913289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 914289344Scem num_vectors); 915289344Scem if (rc == 0) 916289344Scem num_vectors = desired_vectors; 917289344Scem else 918289344Scem pci_release_msi(ntb->device); 919289344Scem } 920289344Scem if (rc != 0) 921289344Scem num_vectors = 1; 922289344Scem } else 923289344Scem num_vectors = 1; 924289344Scem 925289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 926289539Scem ntb->db_vec_count = 1; 927290680Scem ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 928289539Scem rc = ntb_setup_legacy_interrupt(ntb); 929289539Scem } else { 930289546Scem ntb_create_msix_vec(ntb, num_vectors); 931289540Scem rc = ntb_setup_msix(ntb, num_vectors); 932289539Scem } 933289539Scem if (rc != 0) { 934289539Scem device_printf(ntb->device, 935289539Scem "Error allocating interrupts: %d\n", rc); 936289546Scem ntb_free_msix_vec(ntb); 937289396Scem } 938289396Scem 939289342Scem return (rc); 940289342Scem} 941289342Scem 942289342Scemstatic int 943289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 944289342Scem{ 945289342Scem int rc; 946289342Scem 947289342Scem ntb->int_info[0].rid = 0; 948289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 949289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 950289342Scem if (ntb->int_info[0].res == NULL) { 951289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 952289342Scem return (ENOMEM); 953250079Scarl } 954250079Scarl 955289342Scem ntb->int_info[0].tag = NULL; 956289342Scem ntb->allocated_interrupts = 1; 957289342Scem 958289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 959289546Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 960289342Scem ntb, &ntb->int_info[0].tag); 961289342Scem if (rc != 0) { 962289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 963289342Scem return (ENXIO); 964289342Scem } 965289342Scem 966250079Scarl return (0); 967250079Scarl} 968250079Scarl 969250079Scarlstatic void 970250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 971250079Scarl{ 972250079Scarl struct ntb_int_info *current_int; 973250079Scarl int i; 974250079Scarl 975289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 976250079Scarl current_int = &ntb->int_info[i]; 977250079Scarl if (current_int->tag != NULL) 978250079Scarl bus_teardown_intr(ntb->device, current_int->res, 979250079Scarl current_int->tag); 980250079Scarl 981250079Scarl if (current_int->res != NULL) 982250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 983250079Scarl rman_get_rid(current_int->res), current_int->res); 984250079Scarl } 985250079Scarl 986289546Scem ntb_free_msix_vec(ntb); 987250079Scarl pci_release_msi(ntb->device); 988250079Scarl} 989250079Scarl 990289347Scem/* 991289648Scem * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 992289347Scem * out to make code clearer. 993289347Scem */ 994289539Scemstatic inline uint64_t 995289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff) 996289347Scem{ 997289347Scem 998289648Scem if (ntb->type == NTB_ATOM) 999289347Scem return (ntb_reg_read(8, regoff)); 1000289347Scem 1001289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1002289347Scem 1003289347Scem return (ntb_reg_read(2, regoff)); 1004289347Scem} 1005289347Scem 1006289539Scemstatic inline void 1007289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1008289347Scem{ 1009289347Scem 1010289542Scem KASSERT((val & ~ntb->db_valid_mask) == 0, 1011289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1012289542Scem (uintmax_t)(val & ~ntb->db_valid_mask), 1013289542Scem (uintmax_t)ntb->db_valid_mask)); 1014289542Scem 1015289607Scem if (regoff == ntb->self_reg->db_mask) 1016289546Scem DB_MASK_ASSERT(ntb, MA_OWNED); 1017290678Scem db_iowrite_raw(ntb, regoff, val); 1018290678Scem} 1019289542Scem 1020290678Scemstatic inline void 1021290678Scemdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1022290678Scem{ 1023290678Scem 1024289648Scem if (ntb->type == NTB_ATOM) { 1025289347Scem ntb_reg_write(8, regoff, val); 1026289347Scem return; 1027289347Scem } 1028289347Scem 1029289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1030289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 1031289347Scem} 1032289347Scem 1033289546Scemvoid 1034289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1035289542Scem{ 1036289542Scem 1037289546Scem DB_MASK_LOCK(ntb); 1038289542Scem ntb->db_mask |= bits; 1039289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1040289546Scem DB_MASK_UNLOCK(ntb); 1041289542Scem} 1042289542Scem 1043289546Scemvoid 1044289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1045289542Scem{ 1046289542Scem 1047289542Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1048289542Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1049289542Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1050289542Scem (uintmax_t)ntb->db_valid_mask)); 1051289542Scem 1052289546Scem DB_MASK_LOCK(ntb); 1053289542Scem ntb->db_mask &= ~bits; 1054289607Scem db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1055289546Scem DB_MASK_UNLOCK(ntb); 1056289542Scem} 1057289542Scem 1058289546Scemuint64_t 1059289546Scemntb_db_read(struct ntb_softc *ntb) 1060289281Scem{ 1061289281Scem 1062289607Scem return (db_ioread(ntb, ntb->self_reg->db_bell)); 1063289281Scem} 1064289281Scem 1065289546Scemvoid 1066289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1067289281Scem{ 1068289281Scem 1069289546Scem KASSERT((bits & ~ntb->db_valid_mask) == 0, 1070289546Scem ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1071289546Scem (uintmax_t)(bits & ~ntb->db_valid_mask), 1072289546Scem (uintmax_t)ntb->db_valid_mask)); 1073289546Scem 1074289607Scem db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1075289281Scem} 1076289281Scem 1077289540Scemstatic inline uint64_t 1078289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1079250079Scarl{ 1080289540Scem uint64_t shift, mask; 1081250079Scarl 1082289540Scem shift = ntb->db_vec_shift; 1083289540Scem mask = (1ull << shift) - 1; 1084289540Scem return (mask << (shift * db_vector)); 1085250079Scarl} 1086250079Scarl 1087250079Scarlstatic void 1088289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1089250079Scarl{ 1090289540Scem uint64_t vec_mask; 1091250079Scarl 1092289542Scem ntb->last_ts = ticks; 1093289546Scem vec_mask = ntb_vec_mask(ntb, vec); 1094250079Scarl 1095289542Scem if ((vec_mask & ntb->db_link_mask) != 0) { 1096289546Scem if (ntb_poll_link(ntb)) 1097289546Scem ntb_link_event(ntb); 1098289540Scem } 1099289540Scem 1100289546Scem if ((vec_mask & ntb->db_valid_mask) != 0) 1101289546Scem ntb_db_event(ntb, vec); 1102289546Scem} 1103250079Scarl 1104289546Scemstatic void 1105289546Scemndev_vec_isr(void *arg) 1106289546Scem{ 1107289546Scem struct ntb_vec *nvec = arg; 1108250079Scarl 1109289546Scem ntb_interrupt(nvec->ntb, nvec->num); 1110250079Scarl} 1111250079Scarl 1112250079Scarlstatic void 1113289546Scemndev_irq_isr(void *arg) 1114250079Scarl{ 1115289546Scem /* If we couldn't set up MSI-X, we only have the one vector. */ 1116289546Scem ntb_interrupt(arg, 0); 1117250079Scarl} 1118250079Scarl 1119250079Scarlstatic int 1120289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1121250079Scarl{ 1122289342Scem uint32_t i; 1123250079Scarl 1124289546Scem ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1125250079Scarl M_ZERO | M_WAITOK); 1126250079Scarl for (i = 0; i < num_vectors; i++) { 1127289546Scem ntb->msix_vec[i].num = i; 1128289546Scem ntb->msix_vec[i].ntb = ntb; 1129250079Scarl } 1130250079Scarl 1131250079Scarl return (0); 1132250079Scarl} 1133250079Scarl 1134250079Scarlstatic void 1135289546Scemntb_free_msix_vec(struct ntb_softc *ntb) 1136250079Scarl{ 1137250079Scarl 1138289546Scem if (ntb->msix_vec == NULL) 1139289539Scem return; 1140289539Scem 1141289546Scem free(ntb->msix_vec, M_NTB); 1142289546Scem ntb->msix_vec = NULL; 1143250079Scarl} 1144250079Scarl 1145250079Scarlstatic struct ntb_hw_info * 1146250079Scarlntb_get_device_info(uint32_t device_id) 1147250079Scarl{ 1148250079Scarl struct ntb_hw_info *ep = pci_ids; 1149250079Scarl 1150250079Scarl while (ep->device_id) { 1151250079Scarl if (ep->device_id == device_id) 1152250079Scarl return (ep); 1153250079Scarl ++ep; 1154250079Scarl } 1155250079Scarl return (NULL); 1156250079Scarl} 1157250079Scarl 1158289272Scemstatic void 1159289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 1160250079Scarl{ 1161250079Scarl 1162289617Scem if (ntb->reg != NULL) 1163289617Scem ntb_link_disable(ntb); 1164250079Scarl} 1165250079Scarl 1166289397Scemstatic void 1167289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 1168289397Scem{ 1169289397Scem 1170289648Scem if (ntb->type == NTB_ATOM) { 1171289648Scem ntb->mw_count = ATOM_MW_COUNT; 1172289397Scem return; 1173289397Scem } 1174289397Scem 1175289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1176289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1177289397Scem else 1178289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 1179289397Scem} 1180289397Scem 1181250079Scarlstatic int 1182289348Scemntb_detect_xeon(struct ntb_softc *ntb) 1183250079Scarl{ 1184289348Scem uint8_t ppd, conn_type; 1185250079Scarl 1186289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1187289348Scem ntb->ppd = ppd; 1188250079Scarl 1189289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1190290681Scem ntb->dev_type = NTB_DEV_DSD; 1191290681Scem else 1192289257Scem ntb->dev_type = NTB_DEV_USD; 1193289257Scem 1194289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1195289397Scem ntb->features |= NTB_SPLIT_BAR; 1196289397Scem 1197289542Scem /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1198289542Scem if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1199289542Scem ntb->features |= NTB_SDOORBELL_LOCKUP; 1200289542Scem 1201289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 1202289348Scem switch (conn_type) { 1203289348Scem case NTB_CONN_B2B: 1204289348Scem ntb->conn_type = conn_type; 1205289348Scem break; 1206289348Scem case NTB_CONN_RP: 1207289348Scem case NTB_CONN_TRANSPARENT: 1208289348Scem default: 1209289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 1210289348Scem (unsigned)conn_type); 1211289348Scem return (ENXIO); 1212289348Scem } 1213289348Scem return (0); 1214289348Scem} 1215289348Scem 1216289348Scemstatic int 1217289648Scemntb_detect_atom(struct ntb_softc *ntb) 1218289348Scem{ 1219289348Scem uint32_t ppd, conn_type; 1220289348Scem 1221289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1222289348Scem ntb->ppd = ppd; 1223289348Scem 1224289648Scem if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1225289348Scem ntb->dev_type = NTB_DEV_DSD; 1226289348Scem else 1227289348Scem ntb->dev_type = NTB_DEV_USD; 1228289348Scem 1229289648Scem conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1230289348Scem switch (conn_type) { 1231289348Scem case NTB_CONN_B2B: 1232289348Scem ntb->conn_type = conn_type; 1233289348Scem break; 1234289348Scem default: 1235289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 1236289348Scem return (ENXIO); 1237289348Scem } 1238289348Scem return (0); 1239289348Scem} 1240289348Scem 1241289348Scemstatic int 1242289542Scemntb_xeon_init_dev(struct ntb_softc *ntb) 1243289348Scem{ 1244289542Scem int rc; 1245289348Scem 1246289542Scem ntb->spad_count = XEON_SPAD_COUNT; 1247289542Scem ntb->db_count = XEON_DB_COUNT; 1248289542Scem ntb->db_link_mask = XEON_DB_LINK_BIT; 1249289542Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1250289542Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1251289257Scem 1252289542Scem if (ntb->conn_type != NTB_CONN_B2B) { 1253250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 1254289348Scem ntb->conn_type); 1255250079Scarl return (ENXIO); 1256250079Scarl } 1257250079Scarl 1258289542Scem ntb->reg = &xeon_reg; 1259289607Scem ntb->self_reg = &xeon_pri_reg; 1260289542Scem ntb->peer_reg = &xeon_b2b_reg; 1261289542Scem ntb->xlat_reg = &xeon_sec_xlat; 1262289542Scem 1263289208Scem /* 1264289208Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 1265289208Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1266289208Scem * which may hang the system. To workaround this use the second memory 1267289208Scem * window to access the interrupt and scratch pad registers on the 1268289208Scem * remote system. 1269289208Scem */ 1270289543Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1271289543Scem /* Use the last MW for mapping remote spad */ 1272289542Scem ntb->b2b_mw_idx = ntb->mw_count - 1; 1273289543Scem else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1274289208Scem /* 1275289542Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1276289542Scem * mirrored to the remote system. Shrink the number of bits by one, 1277289542Scem * since bit 14 is the last bit. 1278289542Scem * 1279289542Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1280289542Scem * anyway. Nor for non-B2B connection types. 1281289542Scem */ 1282289543Scem ntb->db_count = XEON_DB_COUNT - 1; 1283250079Scarl 1284289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1285250079Scarl 1286289542Scem if (ntb->dev_type == NTB_DEV_USD) 1287289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1288289542Scem &xeon_b2b_usd_addr); 1289289542Scem else 1290289542Scem rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1291289542Scem &xeon_b2b_dsd_addr); 1292289542Scem if (rc != 0) 1293289542Scem return (rc); 1294289271Scem 1295250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1296290682Scem ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1297289542Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1298255279Scarl 1299290682Scem /* 1300290682Scem * Mask all doorbell interrupts. 1301290682Scem */ 1302290682Scem ntb_db_set_mask(ntb, ntb->db_valid_mask); 1303250079Scarl 1304290682Scem rc = ntb_init_isr(ntb); 1305290682Scem return (rc); 1306250079Scarl} 1307250079Scarl 1308250079Scarlstatic int 1309289648Scemntb_atom_init_dev(struct ntb_softc *ntb) 1310250079Scarl{ 1311290682Scem int error; 1312250079Scarl 1313289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1314289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1315250079Scarl 1316289648Scem ntb->spad_count = ATOM_SPAD_COUNT; 1317289648Scem ntb->db_count = ATOM_DB_COUNT; 1318289648Scem ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1319289648Scem ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1320289542Scem ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1321250079Scarl 1322289648Scem ntb->reg = &atom_reg; 1323289648Scem ntb->self_reg = &atom_pri_reg; 1324289648Scem ntb->peer_reg = &atom_b2b_reg; 1325289648Scem ntb->xlat_reg = &atom_sec_xlat; 1326289542Scem 1327250079Scarl /* 1328289648Scem * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1329250079Scarl * resolved. Mask transaction layer internal parity errors. 1330250079Scarl */ 1331250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1332250079Scarl 1333289648Scem configure_atom_secondary_side_bars(ntb); 1334250079Scarl 1335250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1336290682Scem ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1337250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1338289209Scem 1339290682Scem error = ntb_init_isr(ntb); 1340290682Scem if (error != 0) 1341290682Scem return (error); 1342290682Scem 1343289542Scem /* Initiate PCI-E link training */ 1344289546Scem ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1345250079Scarl 1346289648Scem callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1347289542Scem 1348250079Scarl return (0); 1349250079Scarl} 1350250079Scarl 1351289648Scem/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1352255279Scarlstatic void 1353289648Scemconfigure_atom_secondary_side_bars(struct ntb_softc *ntb) 1354255279Scarl{ 1355255279Scarl 1356255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1357289648Scem ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1358290725Scem XEON_B2B_BAR2_ADDR64); 1359289648Scem ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1360290725Scem XEON_B2B_BAR4_ADDR64); 1361290725Scem ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1362290725Scem ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1363255279Scarl } else { 1364289648Scem ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1365290725Scem XEON_B2B_BAR2_ADDR64); 1366289648Scem ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1367290725Scem XEON_B2B_BAR4_ADDR64); 1368290725Scem ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1369290725Scem ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1370255279Scarl } 1371255279Scarl} 1372255279Scarl 1373289543Scem 1374289543Scem/* 1375289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a 1376289543Scem * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1377289543Scem * remains for use by a higher layer. 1378289543Scem * 1379289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured 1380289543Scem * MW size is sufficiently large. 1381289543Scem */ 1382289543Scemstatic unsigned int ntb_b2b_mw_share; 1383289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1384289543Scem 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1385289543Scem "MW with higher level consumers. Both sides of the NTB MUST set the same " 1386289543Scem "value here."); 1387289543Scem 1388289543Scemstatic void 1389289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1390289543Scem enum ntb_bar regbar) 1391289543Scem{ 1392289543Scem struct ntb_pci_bar_info *bar; 1393289543Scem uint8_t bar_sz; 1394289543Scem 1395289543Scem if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1396289543Scem return; 1397289543Scem 1398289543Scem bar = &ntb->bar_info[idx]; 1399289543Scem bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1400289543Scem if (idx == regbar) { 1401289543Scem if (ntb->b2b_off != 0) 1402289543Scem bar_sz--; 1403289543Scem else 1404289543Scem bar_sz = 0; 1405289543Scem } 1406289543Scem pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1407289543Scem bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1408289543Scem (void)bar_sz; 1409289543Scem} 1410289543Scem 1411289543Scemstatic void 1412289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1413289543Scem enum ntb_bar idx, enum ntb_bar regbar) 1414289543Scem{ 1415289546Scem uint64_t reg_val; 1416289546Scem uint32_t base_reg, lmt_reg; 1417289543Scem 1418289546Scem bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1419289546Scem if (idx == regbar) 1420289546Scem bar_addr += ntb->b2b_off; 1421289543Scem 1422289546Scem if (!bar_is_64bit(ntb, idx)) { 1423289546Scem ntb_reg_write(4, base_reg, bar_addr); 1424289546Scem reg_val = ntb_reg_read(4, base_reg); 1425289546Scem (void)reg_val; 1426289546Scem 1427289546Scem ntb_reg_write(4, lmt_reg, bar_addr); 1428289546Scem reg_val = ntb_reg_read(4, lmt_reg); 1429289546Scem (void)reg_val; 1430289543Scem } else { 1431289546Scem ntb_reg_write(8, base_reg, bar_addr); 1432289546Scem reg_val = ntb_reg_read(8, base_reg); 1433289546Scem (void)reg_val; 1434289546Scem 1435289546Scem ntb_reg_write(8, lmt_reg, bar_addr); 1436289546Scem reg_val = ntb_reg_read(8, lmt_reg); 1437289546Scem (void)reg_val; 1438289543Scem } 1439289543Scem} 1440289543Scem 1441289543Scemstatic void 1442289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1443289543Scem{ 1444289543Scem struct ntb_pci_bar_info *bar; 1445289543Scem 1446289543Scem bar = &ntb->bar_info[idx]; 1447289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1448289543Scem ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1449289543Scem base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1450289543Scem } else { 1451289543Scem ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1452289543Scem base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1453289543Scem } 1454289543Scem (void)base_addr; 1455289543Scem} 1456289543Scem 1457289542Scemstatic int 1458289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1459289542Scem const struct ntb_b2b_addr *peer_addr) 1460255279Scarl{ 1461289543Scem struct ntb_pci_bar_info *b2b_bar; 1462289543Scem vm_size_t bar_size; 1463289543Scem uint64_t bar_addr; 1464289543Scem enum ntb_bar b2b_bar_num, i; 1465255279Scarl 1466289543Scem if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1467289543Scem b2b_bar = NULL; 1468289543Scem b2b_bar_num = NTB_CONFIG_BAR; 1469289543Scem ntb->b2b_off = 0; 1470289543Scem } else { 1471289543Scem b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1472289543Scem KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1473289543Scem ("invalid b2b mw bar")); 1474289543Scem 1475289543Scem b2b_bar = &ntb->bar_info[b2b_bar_num]; 1476289543Scem bar_size = b2b_bar->size; 1477289543Scem 1478289543Scem if (ntb_b2b_mw_share != 0 && 1479289543Scem (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1480289543Scem ntb->b2b_off = bar_size >> 1; 1481289543Scem else if (bar_size >= XEON_B2B_MIN_SIZE) { 1482289543Scem ntb->b2b_off = 0; 1483289543Scem ntb->mw_count--; 1484289543Scem } else { 1485289543Scem device_printf(ntb->device, 1486289543Scem "B2B bar size is too small!\n"); 1487289543Scem return (EIO); 1488289543Scem } 1489255279Scarl } 1490289542Scem 1491289543Scem /* 1492289543Scem * Reset the secondary bar sizes to match the primary bar sizes. 1493289543Scem * (Except, disable or halve the size of the B2B secondary bar.) 1494289543Scem */ 1495289543Scem for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1496289543Scem xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1497289543Scem 1498289543Scem bar_addr = 0; 1499289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1500289543Scem bar_addr = addr->bar0_addr; 1501289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1502289543Scem bar_addr = addr->bar2_addr64; 1503289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1504289543Scem bar_addr = addr->bar4_addr64; 1505289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1506289543Scem bar_addr = addr->bar4_addr32; 1507289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1508289543Scem bar_addr = addr->bar5_addr32; 1509289543Scem else 1510289543Scem KASSERT(false, ("invalid bar")); 1511289543Scem 1512289543Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1513289543Scem 1514289543Scem /* 1515289543Scem * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1516289543Scem * register BAR. The B2B BAR is either disabled above or configured 1517289543Scem * half-size. It starts at PBAR xlat + offset. 1518289543Scem * 1519289543Scem * Also set up incoming BAR limits == base (zero length window). 1520289543Scem */ 1521289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1522289543Scem b2b_bar_num); 1523289542Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1524289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1525289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1526289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1527289543Scem NTB_B2B_BAR_3, b2b_bar_num); 1528289542Scem } else 1529289543Scem xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1530289543Scem NTB_B2B_BAR_2, b2b_bar_num); 1531289543Scem 1532289543Scem /* Zero incoming translation addrs */ 1533289543Scem ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1534289543Scem ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1535289543Scem 1536289543Scem /* Zero outgoing translation limits (whole bar size windows) */ 1537289543Scem ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1538289543Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1539289543Scem 1540289543Scem /* Set outgoing translation offsets */ 1541289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1542289543Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1543289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1544289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1545289543Scem } else 1546289543Scem xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1547289543Scem 1548289543Scem /* Set the translation offset for B2B registers */ 1549289543Scem bar_addr = 0; 1550289543Scem if (b2b_bar_num == NTB_CONFIG_BAR) 1551289543Scem bar_addr = peer_addr->bar0_addr; 1552289543Scem else if (b2b_bar_num == NTB_B2B_BAR_1) 1553289543Scem bar_addr = peer_addr->bar2_addr64; 1554289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1555289543Scem bar_addr = peer_addr->bar4_addr64; 1556289543Scem else if (b2b_bar_num == NTB_B2B_BAR_2) 1557289543Scem bar_addr = peer_addr->bar4_addr32; 1558289543Scem else if (b2b_bar_num == NTB_B2B_BAR_3) 1559289543Scem bar_addr = peer_addr->bar5_addr32; 1560289543Scem else 1561289543Scem KASSERT(false, ("invalid bar")); 1562289543Scem 1563289543Scem /* 1564289543Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1565289543Scem * at a time. 1566289543Scem */ 1567289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1568289543Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1569289542Scem return (0); 1570255279Scarl} 1571255279Scarl 1572289546Scemstatic inline bool 1573289546Scemlink_is_up(struct ntb_softc *ntb) 1574289546Scem{ 1575289546Scem 1576289611Scem if (ntb->type == NTB_XEON) { 1577289611Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1578289611Scem return (true); 1579289546Scem return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1580289611Scem } 1581289546Scem 1582289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1583289648Scem return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1584289546Scem} 1585289546Scem 1586289546Scemstatic inline bool 1587289648Scematom_link_is_err(struct ntb_softc *ntb) 1588289546Scem{ 1589289546Scem uint32_t status; 1590289546Scem 1591289648Scem KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1592289546Scem 1593289648Scem status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1594289648Scem if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1595289546Scem return (true); 1596289546Scem 1597289648Scem status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1598289648Scem return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1599289546Scem} 1600289546Scem 1601289648Scem/* Atom does not have link status interrupt, poll on that platform */ 1602250079Scarlstatic void 1603289648Scematom_link_hb(void *arg) 1604250079Scarl{ 1605250079Scarl struct ntb_softc *ntb = arg; 1606289546Scem sbintime_t timo, poll_ts; 1607250079Scarl 1608289546Scem timo = NTB_HB_TIMEOUT * hz; 1609289546Scem poll_ts = ntb->last_ts + timo; 1610289546Scem 1611289542Scem /* 1612289542Scem * Delay polling the link status if an interrupt was received, unless 1613289542Scem * the cached link status says the link is down. 1614289542Scem */ 1615289546Scem if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1616289546Scem timo = poll_ts - ticks; 1617289542Scem goto out; 1618289546Scem } 1619289542Scem 1620289546Scem if (ntb_poll_link(ntb)) 1621289546Scem ntb_link_event(ntb); 1622289542Scem 1623289648Scem if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1624289546Scem /* Link is down with error, proceed with recovery */ 1625289648Scem callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1626289546Scem return; 1627250079Scarl } 1628250079Scarl 1629289542Scemout: 1630289648Scem callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1631250079Scarl} 1632250079Scarl 1633250079Scarlstatic void 1634289648Scematom_perform_link_restart(struct ntb_softc *ntb) 1635250079Scarl{ 1636250079Scarl uint32_t status; 1637250079Scarl 1638250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1639289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1640289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1641289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1642289648Scem ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1643250079Scarl 1644250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1645250079Scarl pause("ModPhy", hz / 10); 1646250079Scarl 1647250079Scarl /* Clear AER Errors, write to clear */ 1648289648Scem status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1649250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1650289648Scem ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1651250079Scarl 1652250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1653289648Scem status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1654289648Scem status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1655289648Scem ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1656250079Scarl 1657250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1658289648Scem status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1659289648Scem status |= ATOM_DESKEWSTS_DBERR; 1660289648Scem ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1661250079Scarl 1662289648Scem status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1663289648Scem status &= ATOM_IBIST_ERR_OFLOW; 1664289648Scem ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1665250079Scarl 1666250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1667289648Scem status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1668289648Scem status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1669289648Scem ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1670250079Scarl} 1671250079Scarl 1672289546Scem/* 1673289546Scem * ntb_set_ctx() - associate a driver context with an ntb device 1674289546Scem * @ntb: NTB device context 1675289546Scem * @ctx: Driver context 1676289546Scem * @ctx_ops: Driver context operations 1677289546Scem * 1678289546Scem * Associate a driver context and operations with a ntb device. The context is 1679289546Scem * provided by the client driver, and the driver may associate a different 1680289546Scem * context with each ntb device. 1681289546Scem * 1682289546Scem * Return: Zero if the context is associated, otherwise an error number. 1683289546Scem */ 1684289546Scemint 1685289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 1686250079Scarl{ 1687250079Scarl 1688289546Scem if (ctx == NULL || ops == NULL) 1689289546Scem return (EINVAL); 1690289546Scem if (ntb->ctx_ops != NULL) 1691289546Scem return (EINVAL); 1692250079Scarl 1693289546Scem CTX_LOCK(ntb); 1694289546Scem if (ntb->ctx_ops != NULL) { 1695289546Scem CTX_UNLOCK(ntb); 1696289546Scem return (EINVAL); 1697250079Scarl } 1698289546Scem ntb->ntb_ctx = ctx; 1699289546Scem ntb->ctx_ops = ops; 1700289546Scem CTX_UNLOCK(ntb); 1701250079Scarl 1702289546Scem return (0); 1703250079Scarl} 1704250079Scarl 1705289546Scem/* 1706289546Scem * It is expected that this will only be used from contexts where the ctx_lock 1707289546Scem * is not needed to protect ntb_ctx lifetime. 1708289546Scem */ 1709289546Scemvoid * 1710289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 1711289546Scem{ 1712289546Scem 1713289546Scem KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 1714289546Scem if (ops != NULL) 1715289546Scem *ops = ntb->ctx_ops; 1716289546Scem return (ntb->ntb_ctx); 1717289546Scem} 1718289546Scem 1719289546Scem/* 1720289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device 1721289546Scem * @ntb: NTB device context 1722289546Scem * 1723289546Scem * Clear any association that may exist between a driver context and the ntb 1724289546Scem * device. 1725289546Scem */ 1726289546Scemvoid 1727289546Scemntb_clear_ctx(struct ntb_softc *ntb) 1728289546Scem{ 1729289546Scem 1730289546Scem CTX_LOCK(ntb); 1731289546Scem ntb->ntb_ctx = NULL; 1732289546Scem ntb->ctx_ops = NULL; 1733289546Scem CTX_UNLOCK(ntb); 1734289546Scem} 1735289546Scem 1736289546Scem/* 1737289546Scem * ntb_link_event() - notify driver context of a change in link status 1738289546Scem * @ntb: NTB device context 1739289546Scem * 1740289546Scem * Notify the driver context that the link status may have changed. The driver 1741289546Scem * should call ntb_link_is_up() to get the current status. 1742289546Scem */ 1743289546Scemvoid 1744289546Scemntb_link_event(struct ntb_softc *ntb) 1745289546Scem{ 1746289546Scem 1747289546Scem CTX_LOCK(ntb); 1748289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 1749289546Scem ntb->ctx_ops->link_event(ntb->ntb_ctx); 1750289546Scem CTX_UNLOCK(ntb); 1751289546Scem} 1752289546Scem 1753289546Scem/* 1754289546Scem * ntb_db_event() - notify driver context of a doorbell event 1755289546Scem * @ntb: NTB device context 1756289546Scem * @vector: Interrupt vector number 1757289546Scem * 1758289546Scem * Notify the driver context of a doorbell event. If hardware supports 1759289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which 1760289546Scem * vector received the interrupt. The vector number is relative to the first 1761289546Scem * vector used for doorbells, starting at zero, and must be less than 1762289546Scem * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 1763289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of 1764289546Scem * those bits are associated with the vector number. 1765289546Scem */ 1766250079Scarlstatic void 1767289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec) 1768289272Scem{ 1769289546Scem 1770289546Scem CTX_LOCK(ntb); 1771289546Scem if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 1772289546Scem ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 1773289546Scem CTX_UNLOCK(ntb); 1774289546Scem} 1775289546Scem 1776289546Scem/* 1777289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb 1778289546Scem * @ntb: NTB device context 1779289546Scem * @max_speed: The maximum link speed expressed as PCIe generation number[0] 1780289546Scem * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 1781289546Scem * 1782289546Scem * Enable the link on the secondary side of the ntb. This can only be done 1783289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1784289546Scem * should train the link to its maximum speed and width, or the requested speed 1785289546Scem * and width, whichever is smaller, if supported. 1786289546Scem * 1787289546Scem * Return: Zero on success, otherwise an error number. 1788289546Scem * 1789289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 1790289546Scem * and width input will be ignored. 1791289546Scem */ 1792289546Scemint 1793289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 1794289546Scem enum ntb_width w __unused) 1795289546Scem{ 1796289280Scem uint32_t cntl; 1797289272Scem 1798289648Scem if (ntb->type == NTB_ATOM) { 1799289542Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1800289648Scem ntb->ppd | ATOM_PPD_INIT_LINK, 4); 1801289546Scem return (0); 1802289542Scem } 1803289542Scem 1804289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1805289546Scem ntb_link_event(ntb); 1806289546Scem return (0); 1807289280Scem } 1808289280Scem 1809289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1810289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1811289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1812289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1813289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1814289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1815289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1816289546Scem return (0); 1817289272Scem} 1818289272Scem 1819289546Scem/* 1820289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb 1821289546Scem * @ntb: NTB device context 1822289546Scem * 1823289546Scem * Disable the link on the secondary side of the ntb. This can only be done 1824289546Scem * from the primary side of the ntb in primary or b2b topology. The ntb device 1825289546Scem * should disable the link. Returning from this call must indicate that a 1826289546Scem * barrier has passed, though with no more writes may pass in either direction 1827289546Scem * across the link, except if this call returns an error number. 1828289546Scem * 1829289546Scem * Return: Zero on success, otherwise an error number. 1830289546Scem */ 1831289546Scemint 1832289542Scemntb_link_disable(struct ntb_softc *ntb) 1833289272Scem{ 1834289272Scem uint32_t cntl; 1835289272Scem 1836289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1837289546Scem ntb_link_event(ntb); 1838289546Scem return (0); 1839289272Scem } 1840289272Scem 1841289542Scem cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1842289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1843289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1844289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1845289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1846289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1847289542Scem ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1848289546Scem return (0); 1849289272Scem} 1850289272Scem 1851289272Scemstatic void 1852289648Scemrecover_atom_link(void *arg) 1853250079Scarl{ 1854250079Scarl struct ntb_softc *ntb = arg; 1855289608Scem unsigned speed, width, oldspeed, oldwidth; 1856250079Scarl uint32_t status32; 1857250079Scarl 1858289648Scem atom_perform_link_restart(ntb); 1859250079Scarl 1860289232Scem /* 1861289232Scem * There is a potential race between the 2 NTB devices recovering at 1862289232Scem * the same time. If the times are the same, the link will not recover 1863289232Scem * and the driver will be stuck in this loop forever. Add a random 1864289232Scem * interval to the recovery time to prevent this race. 1865289232Scem */ 1866289648Scem status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 1867289648Scem pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 1868289232Scem 1869289648Scem if (atom_link_is_err(ntb)) 1870250079Scarl goto retry; 1871250079Scarl 1872289542Scem status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1873289648Scem if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 1874289232Scem goto out; 1875289232Scem 1876289542Scem status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1877289608Scem width = NTB_LNK_STA_WIDTH(status32); 1878289608Scem speed = status32 & NTB_LINK_SPEED_MASK; 1879289608Scem 1880289608Scem oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 1881289608Scem oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 1882289608Scem if (oldwidth != width || oldspeed != speed) 1883250079Scarl goto retry; 1884250079Scarl 1885289232Scemout: 1886289648Scem callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 1887289542Scem ntb); 1888250079Scarl return; 1889250079Scarl 1890250079Scarlretry: 1891289648Scem callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 1892250079Scarl ntb); 1893250079Scarl} 1894250079Scarl 1895289546Scem/* 1896289546Scem * Polls the HW link status register(s); returns true if something has changed. 1897289546Scem */ 1898289546Scemstatic bool 1899289542Scemntb_poll_link(struct ntb_softc *ntb) 1900250079Scarl{ 1901250079Scarl uint32_t ntb_cntl; 1902289546Scem uint16_t reg_val; 1903250079Scarl 1904289648Scem if (ntb->type == NTB_ATOM) { 1905289542Scem ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1906289546Scem if (ntb_cntl == ntb->ntb_ctl) 1907289546Scem return (false); 1908289546Scem 1909289542Scem ntb->ntb_ctl = ntb_cntl; 1910289542Scem ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1911250079Scarl } else { 1912290678Scem db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 1913250079Scarl 1914289546Scem reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 1915289546Scem if (reg_val == ntb->lnk_sta) 1916289546Scem return (false); 1917250079Scarl 1918289546Scem ntb->lnk_sta = reg_val; 1919289542Scem } 1920289546Scem return (true); 1921289542Scem} 1922289542Scem 1923289546Scemstatic inline enum ntb_speed 1924289546Scemntb_link_sta_speed(struct ntb_softc *ntb) 1925250079Scarl{ 1926250079Scarl 1927289546Scem if (!link_is_up(ntb)) 1928289546Scem return (NTB_SPEED_NONE); 1929289546Scem return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 1930250079Scarl} 1931250079Scarl 1932289546Scemstatic inline enum ntb_width 1933289546Scemntb_link_sta_width(struct ntb_softc *ntb) 1934250079Scarl{ 1935250079Scarl 1936289546Scem if (!link_is_up(ntb)) 1937289546Scem return (NTB_WIDTH_NONE); 1938289546Scem return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 1939250079Scarl} 1940250079Scarl 1941289774ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 1942289774Scem "Driver state, statistics, and HW registers"); 1943289774Scem 1944289774Scem#define NTB_REGSZ_MASK (3ul << 30) 1945289774Scem#define NTB_REG_64 (1ul << 30) 1946289774Scem#define NTB_REG_32 (2ul << 30) 1947289774Scem#define NTB_REG_16 (3ul << 30) 1948289774Scem#define NTB_REG_8 (0ul << 30) 1949289774Scem 1950289774Scem#define NTB_DB_READ (1ul << 29) 1951289774Scem#define NTB_PCI_REG (1ul << 28) 1952289774Scem#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 1953289774Scem 1954289774Scemstatic void 1955289774Scemntb_sysctl_init(struct ntb_softc *ntb) 1956289774Scem{ 1957289774Scem struct sysctl_oid_list *tree_par, *regpar, *statpar, *errpar; 1958289774Scem struct sysctl_ctx_list *ctx; 1959289774Scem struct sysctl_oid *tree, *tmptree; 1960289774Scem 1961289774Scem ctx = device_get_sysctl_ctx(ntb->device); 1962289774Scem 1963289774Scem tree = SYSCTL_ADD_NODE(ctx, 1964289774Scem SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)), OID_AUTO, 1965289774Scem "debug_info", CTLFLAG_RD, NULL, 1966289774Scem "Driver state, statistics, and HW registers"); 1967289774Scem tree_par = SYSCTL_CHILDREN(tree); 1968289774Scem 1969289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 1970289774Scem &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 1971289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 1972289774Scem &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 1973290687Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 1974290687Scem &ntb->ppd, 0, "Raw PPD register (cached)"); 1975289774Scem 1976289774Scem if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 1977289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 1978289774Scem &ntb->b2b_mw_idx, 0, 1979289774Scem "Index of the MW used for B2B remote register access"); 1980289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 1981289774Scem CTLFLAG_RD, &ntb->b2b_off, 1982289774Scem "If non-zero, offset of B2B register region in shared MW"); 1983289774Scem } 1984289774Scem 1985289774Scem SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 1986289774Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 1987289774Scem "Features/errata of this NTB device"); 1988289774Scem 1989289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 1990290686Scem __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 1991290686Scem "NTB CTL register (cached)"); 1992289774Scem SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 1993290686Scem __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 1994290686Scem "LNK STA register (cached)"); 1995289774Scem 1996289774Scem SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "link_status", 1997289774Scem CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_link_status, 1998289774Scem "A", "Link status"); 1999289774Scem 2000289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2001289774Scem &ntb->mw_count, 0, "MW count (excl. non-shared B2B register BAR)"); 2002289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2003289774Scem &ntb->spad_count, 0, "Scratchpad count"); 2004289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2005289774Scem &ntb->db_count, 0, "Doorbell count"); 2006289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2007289774Scem &ntb->db_vec_count, 0, "Doorbell vector count"); 2008289774Scem SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2009289774Scem &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2010289774Scem 2011289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2012289774Scem &ntb->db_valid_mask, "Doorbell valid mask"); 2013289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2014289774Scem &ntb->db_link_mask, "Doorbell link mask"); 2015289774Scem SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2016289774Scem &ntb->db_mask, "Doorbell mask (cached)"); 2017289774Scem 2018289774Scem tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2019289774Scem CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2020289774Scem regpar = SYSCTL_CHILDREN(tmptree); 2021289774Scem 2022290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2023290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2024290682Scem ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2025290682Scem "NTB Control register"); 2026290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2027290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2028290682Scem 0x19c, sysctl_handle_register, "IU", 2029290682Scem "NTB Link Capabilities"); 2030290682Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2031290682Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2032290682Scem 0x1a0, sysctl_handle_register, "IU", 2033290682Scem "NTB Link Control register"); 2034290682Scem 2035289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2036289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2037289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2038289774Scem sysctl_handle_register, "QU", "Doorbell mask register"); 2039289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2040289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2041289774Scem NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2042289774Scem sysctl_handle_register, "QU", "Doorbell register"); 2043289774Scem 2044289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2045289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2046289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2047289774Scem sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2048289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2049289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2050289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2051289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2052289774Scem sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2053289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2054289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2055289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2056289774Scem sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2057289774Scem } else { 2058289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2059289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2060289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2061289774Scem sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2062289774Scem } 2063289774Scem 2064289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2065289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2066289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2067289774Scem sysctl_handle_register, "QU", "Incoming LMT23 register"); 2068289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2069289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2070289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2071289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2072289774Scem sysctl_handle_register, "IU", "Incoming LMT4 register"); 2073289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2074289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2075289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2076289774Scem sysctl_handle_register, "IU", "Incoming LMT5 register"); 2077289774Scem } else { 2078289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2079289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2080289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2081289774Scem sysctl_handle_register, "QU", "Incoming LMT45 register"); 2082289774Scem } 2083289774Scem 2084289774Scem if (ntb->type == NTB_ATOM) 2085289774Scem return; 2086289774Scem 2087289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2088289774Scem CTLFLAG_RD, NULL, "Xeon HW statistics"); 2089289774Scem statpar = SYSCTL_CHILDREN(tmptree); 2090289774Scem SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2091289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2092289774Scem NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2093289774Scem sysctl_handle_register, "SU", "Upstream Memory Miss"); 2094289774Scem 2095289774Scem tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2096289774Scem CTLFLAG_RD, NULL, "Xeon HW errors"); 2097289774Scem errpar = SYSCTL_CHILDREN(tmptree); 2098289774Scem 2099290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2100289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2101290687Scem NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2102290687Scem sysctl_handle_register, "CU", "PPD"); 2103290687Scem 2104290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2105290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2106290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2107290687Scem sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2108290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2109290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2110290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2111290687Scem sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2112290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2113290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2114290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2115290687Scem sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2116290687Scem 2117290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2118290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2119290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2120290687Scem sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2121290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2122290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2123290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2124290687Scem sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2125290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2126290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2127290687Scem NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2128290687Scem sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2129290687Scem 2130290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2131290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2132289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2133289774Scem sysctl_handle_register, "SU", "DEVSTS"); 2134290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2135289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2136289774Scem NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2137289774Scem sysctl_handle_register, "SU", "LNKSTS"); 2138290687Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2139290687Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2140290687Scem NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2141290687Scem sysctl_handle_register, "SU", "SLNKSTS"); 2142290687Scem 2143289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2144289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2145289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2146289774Scem sysctl_handle_register, "IU", "UNCERRSTS"); 2147289774Scem SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2148289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2149289774Scem NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2150289774Scem sysctl_handle_register, "IU", "CORERRSTS"); 2151289774Scem 2152289774Scem if (ntb->conn_type != NTB_CONN_B2B) 2153289774Scem return; 2154289774Scem 2155289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2156289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2157289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2158289774Scem sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2159289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2160289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2161289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2162289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2163289774Scem sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2164289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2165289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2166289774Scem NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2167289774Scem sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2168289774Scem } else { 2169289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2170289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2171289774Scem NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2172289774Scem sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2173289774Scem } 2174289774Scem 2175289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2176289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2177289774Scem NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2178289774Scem sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2179289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2180289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2181289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2182289774Scem NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2183289774Scem sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2184289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2185289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2186289774Scem NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2187289774Scem sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2188289774Scem } else { 2189289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2190289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2191289774Scem NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2192289774Scem sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2193289774Scem } 2194289774Scem 2195289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2196289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2197289774Scem NTB_REG_64 | ntb->xlat_reg->bar0_base, 2198289774Scem sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2199289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2200289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2201289774Scem NTB_REG_64 | ntb->xlat_reg->bar2_base, 2202289774Scem sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2203289774Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2204289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2205289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2206289774Scem NTB_REG_32 | ntb->xlat_reg->bar4_base, 2207289774Scem sysctl_handle_register, "IU", 2208289774Scem "Secondary BAR4 base register"); 2209289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2210289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2211289774Scem NTB_REG_32 | ntb->xlat_reg->bar5_base, 2212289774Scem sysctl_handle_register, "IU", 2213289774Scem "Secondary BAR5 base register"); 2214289774Scem } else { 2215289774Scem SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2216289774Scem CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2217289774Scem NTB_REG_64 | ntb->xlat_reg->bar4_base, 2218289774Scem sysctl_handle_register, "QU", 2219289774Scem "Secondary BAR45 base register"); 2220289774Scem } 2221289774Scem} 2222289774Scem 2223289774Scemstatic int 2224289774Scemsysctl_handle_features(SYSCTL_HANDLER_ARGS) 2225289774Scem{ 2226289774Scem struct ntb_softc *ntb; 2227289774Scem struct sbuf sb; 2228289774Scem int error; 2229289774Scem 2230289774Scem error = 0; 2231289774Scem ntb = arg1; 2232289774Scem 2233289774Scem sbuf_new_for_sysctl(&sb, NULL, 256, req); 2234289774Scem 2235289774Scem sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2236289774Scem error = sbuf_finish(&sb); 2237289774Scem sbuf_delete(&sb); 2238289774Scem 2239289774Scem if (error || !req->newptr) 2240289774Scem return (error); 2241289774Scem return (EINVAL); 2242289774Scem} 2243289774Scem 2244289774Scemstatic int 2245289774Scemsysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2246289774Scem{ 2247289774Scem struct ntb_softc *ntb; 2248289774Scem struct sbuf sb; 2249289774Scem enum ntb_speed speed; 2250289774Scem enum ntb_width width; 2251289774Scem int error; 2252289774Scem 2253289774Scem error = 0; 2254289774Scem ntb = arg1; 2255289774Scem 2256289774Scem sbuf_new_for_sysctl(&sb, NULL, 32, req); 2257289774Scem 2258289774Scem if (ntb_link_is_up(ntb, &speed, &width)) 2259289774Scem sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2260289774Scem (unsigned)speed, (unsigned)width); 2261289774Scem else 2262289774Scem sbuf_printf(&sb, "down"); 2263289774Scem 2264289774Scem error = sbuf_finish(&sb); 2265289774Scem sbuf_delete(&sb); 2266289774Scem 2267289774Scem if (error || !req->newptr) 2268289774Scem return (error); 2269289774Scem return (EINVAL); 2270289774Scem} 2271289774Scem 2272289774Scemstatic int 2273289774Scemsysctl_handle_register(SYSCTL_HANDLER_ARGS) 2274289774Scem{ 2275289774Scem struct ntb_softc *ntb; 2276289774Scem const void *outp; 2277289774Scem uintptr_t sz; 2278289774Scem uint64_t umv; 2279289774Scem char be[sizeof(umv)]; 2280289774Scem size_t outsz; 2281289774Scem uint32_t reg; 2282289774Scem bool db, pci; 2283289774Scem int error; 2284289774Scem 2285289774Scem ntb = arg1; 2286289774Scem reg = arg2 & ~NTB_REGFLAGS_MASK; 2287289774Scem sz = arg2 & NTB_REGSZ_MASK; 2288289774Scem db = (arg2 & NTB_DB_READ) != 0; 2289289774Scem pci = (arg2 & NTB_PCI_REG) != 0; 2290289774Scem 2291289774Scem KASSERT(!(db && pci), ("bogus")); 2292289774Scem 2293289774Scem if (db) { 2294289774Scem KASSERT(sz == NTB_REG_64, ("bogus")); 2295289774Scem umv = db_ioread(ntb, reg); 2296289774Scem outsz = sizeof(uint64_t); 2297289774Scem } else { 2298289774Scem switch (sz) { 2299289774Scem case NTB_REG_64: 2300289774Scem if (pci) 2301289774Scem umv = pci_read_config(ntb->device, reg, 8); 2302289774Scem else 2303289774Scem umv = ntb_reg_read(8, reg); 2304289774Scem outsz = sizeof(uint64_t); 2305289774Scem break; 2306289774Scem case NTB_REG_32: 2307289774Scem if (pci) 2308289774Scem umv = pci_read_config(ntb->device, reg, 4); 2309289774Scem else 2310289774Scem umv = ntb_reg_read(4, reg); 2311289774Scem outsz = sizeof(uint32_t); 2312289774Scem break; 2313289774Scem case NTB_REG_16: 2314289774Scem if (pci) 2315289774Scem umv = pci_read_config(ntb->device, reg, 2); 2316289774Scem else 2317289774Scem umv = ntb_reg_read(2, reg); 2318289774Scem outsz = sizeof(uint16_t); 2319289774Scem break; 2320289774Scem case NTB_REG_8: 2321289774Scem if (pci) 2322289774Scem umv = pci_read_config(ntb->device, reg, 1); 2323289774Scem else 2324289774Scem umv = ntb_reg_read(1, reg); 2325289774Scem outsz = sizeof(uint8_t); 2326289774Scem break; 2327289774Scem default: 2328289774Scem panic("bogus"); 2329289774Scem break; 2330289774Scem } 2331289774Scem } 2332289774Scem 2333289774Scem /* Encode bigendian so that sysctl -x is legible. */ 2334289774Scem be64enc(be, umv); 2335289774Scem outp = ((char *)be) + sizeof(umv) - outsz; 2336289774Scem 2337289774Scem error = SYSCTL_OUT(req, outp, outsz); 2338289774Scem if (error || !req->newptr) 2339289774Scem return (error); 2340289774Scem return (EINVAL); 2341289774Scem} 2342289774Scem 2343289546Scem/* 2344289546Scem * Public API to the rest of the OS 2345250079Scarl */ 2346250079Scarl 2347250079Scarl/** 2348250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 2349250079Scarl * @ntb: pointer to ntb_softc instance 2350250079Scarl * 2351250079Scarl * This function returns the max 32bit scratchpad registers usable by the 2352250079Scarl * upper layer. 2353250079Scarl * 2354250079Scarl * RETURNS: total number of scratch pad registers available 2355250079Scarl */ 2356289208Scemuint8_t 2357250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 2358250079Scarl{ 2359250079Scarl 2360289539Scem return (ntb->spad_count); 2361250079Scarl} 2362250079Scarl 2363289396Scemuint8_t 2364289539Scemntb_mw_count(struct ntb_softc *ntb) 2365289396Scem{ 2366289396Scem 2367289539Scem return (ntb->mw_count); 2368289396Scem} 2369289396Scem 2370250079Scarl/** 2371289545Scem * ntb_spad_write() - write to the secondary scratchpad register 2372250079Scarl * @ntb: pointer to ntb_softc instance 2373250079Scarl * @idx: index to the scratchpad register, 0 based 2374250079Scarl * @val: the data value to put into the register 2375250079Scarl * 2376250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2377250079Scarl * register. The register resides on the secondary (external) side. 2378250079Scarl * 2379289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2380250079Scarl */ 2381250079Scarlint 2382289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2383250079Scarl{ 2384250079Scarl 2385289539Scem if (idx >= ntb->spad_count) 2386250079Scarl return (EINVAL); 2387250079Scarl 2388289607Scem ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2389250079Scarl 2390250079Scarl return (0); 2391250079Scarl} 2392250079Scarl 2393250079Scarl/** 2394289545Scem * ntb_spad_read() - read from the primary scratchpad register 2395250079Scarl * @ntb: pointer to ntb_softc instance 2396250079Scarl * @idx: index to scratchpad register, 0 based 2397250079Scarl * @val: pointer to 32bit integer for storing the register value 2398250079Scarl * 2399250079Scarl * This function allows reading of the 32bit scratchpad register on 2400250079Scarl * the primary (internal) side. 2401250079Scarl * 2402289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2403250079Scarl */ 2404250079Scarlint 2405289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2406250079Scarl{ 2407250079Scarl 2408289539Scem if (idx >= ntb->spad_count) 2409250079Scarl return (EINVAL); 2410250079Scarl 2411289607Scem *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2412250079Scarl 2413250079Scarl return (0); 2414250079Scarl} 2415250079Scarl 2416250079Scarl/** 2417289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register 2418250079Scarl * @ntb: pointer to ntb_softc instance 2419250079Scarl * @idx: index to the scratchpad register, 0 based 2420250079Scarl * @val: the data value to put into the register 2421250079Scarl * 2422250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 2423250079Scarl * register. The register resides on the secondary (external) side. 2424250079Scarl * 2425289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2426250079Scarl */ 2427250079Scarlint 2428289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2429250079Scarl{ 2430250079Scarl 2431289539Scem if (idx >= ntb->spad_count) 2432250079Scarl return (EINVAL); 2433250079Scarl 2434289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2435290682Scem ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2436255279Scarl else 2437289542Scem ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2438250079Scarl 2439250079Scarl return (0); 2440250079Scarl} 2441250079Scarl 2442250079Scarl/** 2443289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register 2444250079Scarl * @ntb: pointer to ntb_softc instance 2445250079Scarl * @idx: index to scratchpad register, 0 based 2446250079Scarl * @val: pointer to 32bit integer for storing the register value 2447250079Scarl * 2448250079Scarl * This function allows reading of the 32bit scratchpad register on 2449250079Scarl * the primary (internal) side. 2450250079Scarl * 2451289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2452250079Scarl */ 2453250079Scarlint 2454289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2455250079Scarl{ 2456250079Scarl 2457289539Scem if (idx >= ntb->spad_count) 2458250079Scarl return (EINVAL); 2459250079Scarl 2460289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2461290682Scem *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2462255279Scarl else 2463289542Scem *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2464250079Scarl 2465250079Scarl return (0); 2466250079Scarl} 2467250079Scarl 2468289546Scem/* 2469289546Scem * ntb_mw_get_range() - get the range of a memory window 2470289546Scem * @ntb: NTB device context 2471289546Scem * @idx: Memory window number 2472289546Scem * @base: OUT - the base address for mapping the memory window 2473289546Scem * @size: OUT - the size for mapping the memory window 2474289546Scem * @align: OUT - the base alignment for translating the memory window 2475289546Scem * @align_size: OUT - the size alignment for translating the memory window 2476250079Scarl * 2477289546Scem * Get the range of a memory window. NULL may be given for any output 2478289546Scem * parameter if the value is not needed. The base and size may be used for 2479289546Scem * mapping the memory window, to access the peer memory. The alignment and 2480289546Scem * size may be used for translating the memory window, for the peer to access 2481289546Scem * memory on the local system. 2482250079Scarl * 2483289546Scem * Return: Zero on success, otherwise an error number. 2484250079Scarl */ 2485289546Scemint 2486289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 2487290679Scem caddr_t *vbase, size_t *size, size_t *align, size_t *align_size) 2488250079Scarl{ 2489289546Scem struct ntb_pci_bar_info *bar; 2490289546Scem size_t bar_b2b_off; 2491250079Scarl 2492289546Scem if (mw_idx >= ntb_mw_count(ntb)) 2493289546Scem return (EINVAL); 2494250079Scarl 2495289546Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)]; 2496289546Scem bar_b2b_off = 0; 2497289546Scem if (mw_idx == ntb->b2b_mw_idx) { 2498289546Scem KASSERT(ntb->b2b_off != 0, 2499289546Scem ("user shouldn't get non-shared b2b mw")); 2500289546Scem bar_b2b_off = ntb->b2b_off; 2501289546Scem } 2502250079Scarl 2503289546Scem if (base != NULL) 2504289546Scem *base = bar->pbase + bar_b2b_off; 2505289546Scem if (vbase != NULL) 2506290679Scem *vbase = bar->vbase + bar_b2b_off; 2507289546Scem if (size != NULL) 2508289546Scem *size = bar->size - bar_b2b_off; 2509289546Scem if (align != NULL) 2510289546Scem *align = bar->size; 2511289546Scem if (align_size != NULL) 2512289546Scem *align_size = 1; 2513289546Scem return (0); 2514250079Scarl} 2515250079Scarl 2516289546Scem/* 2517289546Scem * ntb_mw_set_trans() - set the translation of a memory window 2518289546Scem * @ntb: NTB device context 2519289546Scem * @idx: Memory window number 2520289546Scem * @addr: The dma address local memory to expose to the peer 2521289546Scem * @size: The size of the local memory to expose to the peer 2522250079Scarl * 2523289546Scem * Set the translation of a memory window. The peer may access local memory 2524289546Scem * through the window starting at the address, up to the size. The address 2525289546Scem * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2526289546Scem * must be aligned to the size alignment specified by ntb_mw_get_range(). 2527250079Scarl * 2528289546Scem * Return: Zero on success, otherwise an error number. 2529250079Scarl */ 2530289546Scemint 2531289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2532289546Scem size_t size) 2533250079Scarl{ 2534289546Scem struct ntb_pci_bar_info *bar; 2535289546Scem uint64_t base, limit, reg_val; 2536289546Scem size_t bar_size, mw_size; 2537289546Scem uint32_t base_reg, xlat_reg, limit_reg; 2538289546Scem enum ntb_bar bar_num; 2539250079Scarl 2540289546Scem if (idx >= ntb_mw_count(ntb)) 2541289546Scem return (EINVAL); 2542250079Scarl 2543289546Scem bar_num = ntb_mw_to_bar(ntb, idx); 2544289546Scem bar = &ntb->bar_info[bar_num]; 2545250079Scarl 2546289546Scem bar_size = bar->size; 2547289546Scem if (idx == ntb->b2b_mw_idx) 2548289546Scem mw_size = bar_size - ntb->b2b_off; 2549289546Scem else 2550289546Scem mw_size = bar_size; 2551250079Scarl 2552289546Scem /* Hardware requires that addr is aligned to bar size */ 2553289546Scem if ((addr & (bar_size - 1)) != 0) 2554289546Scem return (EINVAL); 2555250079Scarl 2556289546Scem if (size > mw_size) 2557289546Scem return (EINVAL); 2558289546Scem 2559289546Scem bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2560289546Scem 2561289546Scem limit = 0; 2562289546Scem if (bar_is_64bit(ntb, bar_num)) { 2563289546Scem base = ntb_reg_read(8, base_reg); 2564289546Scem 2565289546Scem if (limit_reg != 0 && size != mw_size) 2566289546Scem limit = base + size; 2567289546Scem 2568289546Scem /* Set and verify translation address */ 2569289546Scem ntb_reg_write(8, xlat_reg, addr); 2570289546Scem reg_val = ntb_reg_read(8, xlat_reg); 2571289546Scem if (reg_val != addr) { 2572289546Scem ntb_reg_write(8, xlat_reg, 0); 2573289546Scem return (EIO); 2574289546Scem } 2575289546Scem 2576289546Scem /* Set and verify the limit */ 2577289546Scem ntb_reg_write(8, limit_reg, limit); 2578289546Scem reg_val = ntb_reg_read(8, limit_reg); 2579289546Scem if (reg_val != limit) { 2580289546Scem ntb_reg_write(8, limit_reg, base); 2581289546Scem ntb_reg_write(8, xlat_reg, 0); 2582289546Scem return (EIO); 2583289546Scem } 2584289546Scem } else { 2585289546Scem /* Configure 32-bit (split) BAR MW */ 2586289546Scem 2587291029Scem if ((addr & UINT32_MAX) != addr) 2588289546Scem return (EINVAL); 2589291029Scem if (((addr + size) & UINT32_MAX) != (addr + size)) 2590289546Scem return (EINVAL); 2591289546Scem 2592289546Scem base = ntb_reg_read(4, base_reg); 2593289546Scem 2594289546Scem if (limit_reg != 0 && size != mw_size) 2595289546Scem limit = base + size; 2596289546Scem 2597289546Scem /* Set and verify translation address */ 2598289546Scem ntb_reg_write(4, xlat_reg, addr); 2599289546Scem reg_val = ntb_reg_read(4, xlat_reg); 2600289546Scem if (reg_val != addr) { 2601289546Scem ntb_reg_write(4, xlat_reg, 0); 2602289546Scem return (EIO); 2603289546Scem } 2604289546Scem 2605289546Scem /* Set and verify the limit */ 2606289546Scem ntb_reg_write(4, limit_reg, limit); 2607289546Scem reg_val = ntb_reg_read(4, limit_reg); 2608289546Scem if (reg_val != limit) { 2609289546Scem ntb_reg_write(4, limit_reg, base); 2610289546Scem ntb_reg_write(4, xlat_reg, 0); 2611289546Scem return (EIO); 2612289546Scem } 2613250079Scarl } 2614289546Scem return (0); 2615250079Scarl} 2616250079Scarl 2617289596Scem/* 2618289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window 2619289596Scem * @ntb: NTB device context 2620289596Scem * @idx: Memory window number 2621289596Scem * 2622289596Scem * Clear the translation of a memory window. The peer may no longer access 2623289596Scem * local memory through the window. 2624289596Scem * 2625289596Scem * Return: Zero on success, otherwise an error number. 2626289596Scem */ 2627289596Scemint 2628289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 2629289596Scem{ 2630289596Scem 2631289596Scem return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 2632289596Scem} 2633289596Scem 2634291031Scem/* 2635291031Scem * ntb_mw_get_wc - Get the write-combine status of a memory window 2636291031Scem * 2637291031Scem * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 2638291031Scem * idx is an invalid memory window). 2639291031Scem */ 2640291031Scemint 2641291031Scemntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, bool *wc) 2642291031Scem{ 2643291031Scem struct ntb_pci_bar_info *bar; 2644291031Scem 2645291031Scem if (idx >= ntb_mw_count(ntb)) 2646291031Scem return (EINVAL); 2647291031Scem 2648291031Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 2649291031Scem *wc = bar->mapped_wc; 2650291031Scem return (0); 2651291031Scem} 2652291031Scem 2653291031Scem/* 2654291031Scem * ntb_mw_set_wc - Set the write-combine status of a memory window 2655291031Scem * 2656291031Scem * If 'wc' matches the current status, this does nothing and succeeds. 2657291031Scem * 2658291031Scem * Returns: Zero on success, setting the caching attribute on the virtual 2659291031Scem * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 2660291031Scem * memory window, or if changing the caching attribute fails). 2661291031Scem */ 2662291031Scemint 2663291031Scemntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, bool wc) 2664291031Scem{ 2665291031Scem struct ntb_pci_bar_info *bar; 2666291031Scem vm_memattr_t attr; 2667291031Scem int rc; 2668291031Scem 2669291031Scem if (idx >= ntb_mw_count(ntb)) 2670291031Scem return (EINVAL); 2671291031Scem 2672291031Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 2673291031Scem if (bar->mapped_wc == wc) 2674291031Scem return (0); 2675291031Scem 2676291031Scem if (wc) 2677291031Scem attr = VM_MEMATTR_WRITE_COMBINING; 2678291031Scem else 2679291031Scem attr = VM_MEMATTR_DEFAULT; 2680291031Scem 2681291031Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, attr); 2682291031Scem if (rc == 0) 2683291031Scem bar->mapped_wc = wc; 2684291031Scem 2685291031Scem return (rc); 2686291031Scem} 2687291031Scem 2688250079Scarl/** 2689289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side 2690250079Scarl * @ntb: pointer to ntb_softc instance 2691289545Scem * @bit: doorbell bits to ring 2692250079Scarl * 2693250079Scarl * This function allows triggering of a doorbell on the secondary/external 2694250079Scarl * side that will initiate an interrupt on the remote host 2695250079Scarl */ 2696250079Scarlvoid 2697289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 2698250079Scarl{ 2699250079Scarl 2700289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2701290682Scem ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 2702289347Scem return; 2703289209Scem } 2704289347Scem 2705289546Scem db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 2706250079Scarl} 2707250079Scarl 2708289542Scem/* 2709289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 2710289542Scem * as well as the size of the register (via *sz_out). 2711289542Scem * 2712289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell 2713289542Scem * ring to its memory window write. 2714289542Scem * 2715289542Scem * Note that writing the peer doorbell via a memory window will *not* generate 2716289542Scem * an interrupt on the remote host; that must be done seperately. 2717289542Scem */ 2718289542Scembus_addr_t 2719289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 2720289542Scem{ 2721289542Scem struct ntb_pci_bar_info *bar; 2722289542Scem uint64_t regoff; 2723289542Scem 2724289542Scem KASSERT(sz_out != NULL, ("must be non-NULL")); 2725289542Scem 2726289542Scem if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 2727289542Scem bar = &ntb->bar_info[NTB_CONFIG_BAR]; 2728289542Scem regoff = ntb->peer_reg->db_bell; 2729289542Scem } else { 2730289542Scem KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) || 2731289542Scem (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1), 2732289542Scem ("mw_count invalid after setup")); 2733289543Scem KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 2734289543Scem ("invalid b2b idx")); 2735289542Scem 2736289542Scem bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 2737290682Scem regoff = XEON_PDOORBELL_OFFSET; 2738289542Scem } 2739289542Scem KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 2740289542Scem 2741289542Scem *sz_out = ntb->reg->db_size; 2742289542Scem /* HACK: Specific to current x86 bus implementation. */ 2743289542Scem return ((uint64_t)bar->pci_bus_handle + regoff); 2744289542Scem} 2745289542Scem 2746289597Scem/* 2747289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 2748289597Scem * @ntb: NTB device context 2749289597Scem * 2750289597Scem * Hardware may support different number or arrangement of doorbell bits. 2751289597Scem * 2752289597Scem * Return: A mask of doorbell bits supported by the ntb. 2753289597Scem */ 2754289597Scemuint64_t 2755289597Scemntb_db_valid_mask(struct ntb_softc *ntb) 2756289597Scem{ 2757289597Scem 2758289597Scem return (ntb->db_valid_mask); 2759289597Scem} 2760289597Scem 2761289598Scem/* 2762289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 2763289598Scem * @ntb: NTB device context 2764289598Scem * @vector: Doorbell vector number 2765289598Scem * 2766289598Scem * Each interrupt vector may have a different number or arrangement of bits. 2767289598Scem * 2768289598Scem * Return: A mask of doorbell bits serviced by a vector. 2769289598Scem */ 2770289598Scemuint64_t 2771289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 2772289598Scem{ 2773289598Scem 2774289598Scem if (vector > ntb->db_vec_count) 2775289598Scem return (0); 2776289598Scem return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 2777289598Scem} 2778289598Scem 2779250079Scarl/** 2780289546Scem * ntb_link_is_up() - get the current ntb link state 2781289546Scem * @ntb: NTB device context 2782289546Scem * @speed: OUT - The link speed expressed as PCIe generation number 2783289546Scem * @width: OUT - The link width expressed as the number of PCIe lanes 2784250079Scarl * 2785250079Scarl * RETURNS: true or false based on the hardware link state 2786250079Scarl */ 2787250079Scarlbool 2788289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 2789289546Scem enum ntb_width *width) 2790250079Scarl{ 2791250079Scarl 2792289546Scem if (speed != NULL) 2793289546Scem *speed = ntb_link_sta_speed(ntb); 2794289546Scem if (width != NULL) 2795289546Scem *width = ntb_link_sta_width(ntb); 2796289546Scem return (link_is_up(ntb)); 2797250079Scarl} 2798250079Scarl 2799255272Scarlstatic void 2800255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 2801250079Scarl{ 2802255272Scarl 2803289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2804289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2805289209Scem bar->pbase = rman_get_start(bar->pci_resource); 2806289209Scem bar->size = rman_get_size(bar->pci_resource); 2807289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 2808250079Scarl} 2809255268Scarl 2810289209Scemdevice_t 2811289209Scemntb_get_device(struct ntb_softc *ntb) 2812255268Scarl{ 2813255268Scarl 2814255268Scarl return (ntb->device); 2815255268Scarl} 2816289208Scem 2817289208Scem/* Export HW-specific errata information. */ 2818289208Scembool 2819289774Scemntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 2820289208Scem{ 2821289208Scem 2822289208Scem return (HAS_FEATURE(feature)); 2823289208Scem} 2824