ntb_hw.c revision 304380
1139778Simp/*- 212115Sdyson * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 312115Sdyson * Copyright (C) 2013 Intel Corporation 412115Sdyson * Copyright (C) 2015 EMC Corporation 512115Sdyson * All rights reserved. 612115Sdyson * 7139778Simp * Redistribution and use in source and binary forms, with or without 812115Sdyson * modification, are permitted provided that the following conditions 912115Sdyson * are met: 1012115Sdyson * 1. Redistributions of source code must retain the above copyright 1112115Sdyson * notice, this list of conditions and the following disclaimer. 1212115Sdyson * 2. Redistributions in binary form must reproduce the above copyright 1312115Sdyson * notice, this list of conditions and the following disclaimer in the 1412115Sdyson * documentation and/or other materials provided with the distribution. 1512115Sdyson * 1612115Sdyson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1712115Sdyson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1812115Sdyson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1912115Sdyson * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2012115Sdyson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2112115Sdyson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2212115Sdyson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2312115Sdyson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2412115Sdyson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2512115Sdyson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2612115Sdyson * SUCH DAMAGE. 2712115Sdyson */ 2812115Sdyson 2912115Sdyson/* 3012115Sdyson * The Non-Transparent Bridge (NTB) is a device that allows you to connect 3112115Sdyson * two or more systems using a PCI-e links, providing remote memory access. 3212115Sdyson * 3312115Sdyson * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 3412115Sdyson * 3512115Sdyson * NOTE: Much of the code in this module is shared with Linux. Any patches may 3612115Sdyson * be picked up and redistributed in Linux with a dual GPL/BSD license. 3712115Sdyson */ 3812115Sdyson 3912115Sdyson#include <sys/cdefs.h> 4012115Sdyson__FBSDID("$FreeBSD: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c 304380 2016-08-18 10:39:00Z mav $"); 4193016Sbde 4212115Sdyson#include <sys/param.h> 4312115Sdyson#include <sys/kernel.h> 4412115Sdyson#include <sys/systm.h> 4512159Sbde#include <sys/bus.h> 4612115Sdyson#include <sys/endian.h> 4760041Sphk#include <sys/malloc.h> 4812115Sdyson#include <sys/module.h> 49193377Sstas#include <sys/mutex.h> 5012115Sdyson#include <sys/pciio.h> 5112115Sdyson#include <sys/queue.h> 5212115Sdyson#include <sys/rman.h> 5312115Sdyson#include <sys/sbuf.h> 5496753Siedowse#include <sys/sysctl.h> 5512115Sdyson#include <vm/vm.h> 5612115Sdyson#include <vm/pmap.h> 5712115Sdyson#include <machine/bus.h> 58202283Slulf#include <machine/intr_machdep.h> 59202283Slulf#include <machine/pmap.h> 60202283Slulf#include <machine/resource.h> 61221128Sjhb#include <dev/pci/pcireg.h> 62202283Slulf#include <dev/pci/pcivar.h> 63221128Sjhb 6412115Sdyson#include "ntb_regs.h" 6596753Siedowse#include "../ntb.h" 6696753Siedowse 6796753Siedowse#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 6896753Siedowse 6996753Siedowse#define NTB_HB_TIMEOUT 1 /* second */ 7096753Siedowse#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 71141633Sphk#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 7296753Siedowse 7396753Siedowse#define NTB_MSIX_VER_GUARD 0xaabbccdd 74111742Sdes#define NTB_MSIX_RECEIVED 0xe0f0e0f0 7512115Sdyson 7612115Sdyson/* 7712115Sdyson * PCI constants could be somewhere more generic, but aren't defined/used in 7812115Sdyson * pci.c. 7912115Sdyson */ 8012115Sdyson#define PCI_MSIX_ENTRY_SIZE 16 8155477Sbde#define PCI_MSIX_ENTRY_LOWER_ADDR 0 8255477Sbde#define PCI_MSIX_ENTRY_UPPER_ADDR 4 8355477Sbde#define PCI_MSIX_ENTRY_DATA 8 8455477Sbde 8555477Sbdeenum ntb_device_type { 8655477Sbde NTB_XEON, 8755477Sbde NTB_ATOM 8855477Sbde}; 8955477Sbde 9055477Sbde/* ntb_conn_type are hardware numbers, cannot change. */ 9155477Sbdeenum ntb_conn_type { 9255477Sbde NTB_CONN_TRANSPARENT = 0, 9355477Sbde NTB_CONN_B2B = 1, 9455477Sbde NTB_CONN_RP = 2, 9555477Sbde}; 9655477Sbde 9755477Sbdeenum ntb_b2b_direction { 9855477Sbde NTB_DEV_USD = 0, 9955477Sbde NTB_DEV_DSD = 1, 10055477Sbde}; 10155477Sbde 10255477Sbdeenum ntb_bar { 10355477Sbde NTB_CONFIG_BAR = 0, 10455477Sbde NTB_B2B_BAR_1, 10555477Sbde NTB_B2B_BAR_2, 10655477Sbde NTB_B2B_BAR_3, 10755477Sbde NTB_MAX_BARS 10855477Sbde}; 10955477Sbde 11055477Sbdeenum { 11155477Sbde NTB_MSIX_GUARD = 0, 11255477Sbde NTB_MSIX_DATA0, 11355477Sbde NTB_MSIX_DATA1, 11455477Sbde NTB_MSIX_DATA2, 11555477Sbde NTB_MSIX_OFS0, 116202283Slulf NTB_MSIX_OFS1, 11793014Sbde NTB_MSIX_OFS2, 11812159Sbde NTB_MSIX_DONE, 11912115Sdyson NTB_MAX_MSIX_SPAD 12012115Sdyson}; 12112115Sdyson 122221867Sjhb/* Device features and workarounds */ 123221867Sjhb#define HAS_FEATURE(ntb, feature) \ 124221867Sjhb (((ntb)->features & (feature)) != 0) 125221867Sjhb 126221867Sjhbstruct ntb_hw_info { 12712115Sdyson uint32_t device_id; 12812115Sdyson const char *desc; 12912115Sdyson enum ntb_device_type type; 13012115Sdyson uint32_t features; 13112115Sdyson}; 13212115Sdyson 133111742Sdesstruct ntb_pci_bar_info { 134111742Sdes bus_space_tag_t pci_bus_tag; 135111742Sdes bus_space_handle_t pci_bus_handle; 136111742Sdes int pci_resource_id; 137111742Sdes struct resource *pci_resource; 13812115Sdyson vm_paddr_t pbase; 139111742Sdes caddr_t vbase; 140111742Sdes vm_size_t size; 14112115Sdyson vm_memattr_t map_mode; 142202283Slulf 14324649Sdfr /* Configuration register offsets */ 14412115Sdyson uint32_t psz_off; 14512115Sdyson uint32_t ssz_off; 14612115Sdyson uint32_t pbarxlat_off; 14712115Sdyson}; 148202283Slulf 14912115Sdysonstruct ntb_int_info { 15065780Sbde struct resource *res; 15112115Sdyson int rid; 15265780Sbde void *tag; 15365780Sbde}; 15465780Sbde 15565780Sbdestruct ntb_vec { 15665780Sbde struct ntb_softc *ntb; 15765780Sbde uint32_t num; 15865780Sbde unsigned masked; 15965780Sbde}; 16065780Sbde 16165780Sbdestruct ntb_reg { 16265780Sbde uint32_t ntb_ctl; 16365780Sbde uint32_t lnk_sta; 16412115Sdyson uint8_t db_size; 16512115Sdyson unsigned mw_bar[NTB_MAX_BARS]; 16612115Sdyson}; 16765780Sbde 16812115Sdysonstruct ntb_alt_reg { 16912115Sdyson uint32_t db_bell; 170184205Sdes uint32_t db_mask; 17112115Sdyson uint32_t spad; 17212115Sdyson}; 17312115Sdyson 17412115Sdysonstruct ntb_xlat_reg { 175202283Slulf uint32_t bar0_base; 17624649Sdfr uint32_t bar2_base; 17755477Sbde uint32_t bar4_base; 178202283Slulf uint32_t bar5_base; 17955477Sbde 18055477Sbde uint32_t bar2_xlat; 18155477Sbde uint32_t bar4_xlat; 18255477Sbde uint32_t bar5_xlat; 18355477Sbde 18455477Sbde uint32_t bar2_limit; 185125843Sbde uint32_t bar4_limit; 18655477Sbde uint32_t bar5_limit; 18755477Sbde}; 18855477Sbde 189125843Sbdestruct ntb_b2b_addr { 190125843Sbde uint64_t bar0_addr; 19155477Sbde uint64_t bar2_addr64; 19255477Sbde uint64_t bar4_addr64; 193125843Sbde uint64_t bar4_addr32; 19455477Sbde uint64_t bar5_addr32; 19555477Sbde}; 196202283Slulf 197202283Slulfstruct ntb_msix_data { 198202283Slulf uint32_t nmd_ofs; 19955477Sbde uint32_t nmd_data; 200202283Slulf}; 20155477Sbde 20255477Sbdestruct ntb_softc { 20355477Sbde device_t device; 20455477Sbde enum ntb_device_type type; 205202283Slulf uint32_t features; 20612115Sdyson 20712115Sdyson struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 208202283Slulf struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 209202283Slulf uint32_t allocated_interrupts; 210111742Sdes 211111741Sdes struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 21224649Sdfr struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 21324649Sdfr bool peer_msix_good; 21412115Sdyson bool peer_msix_done; 21512115Sdyson struct ntb_pci_bar_info *peer_lapic_bar; 21612115Sdyson struct callout peer_msix_work; 21712115Sdyson 21812115Sdyson struct callout heartbeat_timer; 21912115Sdyson struct callout lr_timer; 22012115Sdyson 22112115Sdyson void *ntb_ctx; 22212115Sdyson const struct ntb_ctx_ops *ctx_ops; 22324649Sdfr struct ntb_vec *msix_vec; 22424649Sdfr#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 22565780Sbde#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 22624649Sdfr#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 22724649Sdfr struct mtx ctx_lock; 22824649Sdfr 229125843Sbde uint32_t ppd; 230184205Sdes enum ntb_conn_type conn_type; 231111119Simp enum ntb_b2b_direction dev_type; 23224649Sdfr 233202283Slulf /* Offset of peer bar0 in B2B BAR */ 23465780Sbde uint64_t b2b_off; 23565780Sbde /* Memory window used to access peer bar0 */ 236202283Slulf#define B2B_MW_DISABLED UINT8_MAX 237202283Slulf uint8_t b2b_mw_idx; 23824649Sdfr uint32_t msix_xlat; 23924649Sdfr uint8_t msix_mw_idx; 24024649Sdfr 24124649Sdfr uint8_t mw_count; 24224649Sdfr uint8_t spad_count; 24312115Sdyson uint8_t db_count; 244184205Sdes uint8_t db_vec_count; 24524649Sdfr uint8_t db_vec_shift; 24624649Sdfr 247111742Sdes /* Protects local db_mask. */ 24812115Sdyson#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 24912115Sdyson#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 25012115Sdyson#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 25112115Sdyson struct mtx db_mask_lock; 25212115Sdyson 25312115Sdyson volatile uint32_t ntb_ctl; 25412115Sdyson volatile uint32_t lnk_sta; 25512115Sdyson 25612115Sdyson uint64_t db_valid_mask; 25712115Sdyson uint64_t db_link_mask; 25812115Sdyson uint64_t db_mask; 25912115Sdyson 26012115Sdyson int last_ts; /* ticks @ last irq */ 26112115Sdyson 26212115Sdyson const struct ntb_reg *reg; 26312115Sdyson const struct ntb_alt_reg *self_reg; 26412115Sdyson const struct ntb_alt_reg *peer_reg; 26512115Sdyson const struct ntb_xlat_reg *xlat_reg; 26612115Sdyson}; 267125843Sbde 26812115Sdyson#ifdef __i386__ 26912115Sdysonstatic __inline uint64_t 27012115Sdysonbus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 27112115Sdyson bus_size_t offset) 27212115Sdyson{ 27312115Sdyson 27412115Sdyson return (bus_space_read_4(tag, handle, offset) | 27512115Sdyson ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 27612115Sdyson} 27712115Sdyson 27812115Sdysonstatic __inline void 27912115Sdysonbus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 28012115Sdyson bus_size_t offset, uint64_t val) 28112115Sdyson{ 28228787Sphk 28312115Sdyson bus_space_write_4(tag, handle, offset, val); 28412115Sdyson bus_space_write_4(tag, handle, offset + 4, val >> 32); 28512115Sdyson} 28612115Sdyson#endif 28712115Sdyson 28896752Siedowse#define ntb_bar_read(SIZE, bar, offset) \ 28996752Siedowse bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 29012115Sdyson ntb->bar_info[(bar)].pci_bus_handle, (offset)) 291202283Slulf#define ntb_bar_write(SIZE, bar, offset, val) \ 29212115Sdyson bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 29312115Sdyson ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 29412115Sdyson#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 29512115Sdyson#define ntb_reg_write(SIZE, offset, val) \ 296202283Slulf ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 297202283Slulf#define ntb_mw_read(SIZE, offset) \ 29812115Sdyson ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 29912115Sdyson#define ntb_mw_write(SIZE, offset, val) \ 30012115Sdyson ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 30112115Sdyson offset, val) 30212115Sdyson 30312115Sdysonstatic int ntb_probe(device_t device); 30412115Sdysonstatic int ntb_attach(device_t device); 30512115Sdysonstatic int ntb_detach(device_t device); 30612115Sdysonstatic uint64_t ntb_db_valid_mask(device_t dev); 30712115Sdysonstatic void ntb_spad_clear(device_t dev); 30812115Sdysonstatic uint64_t ntb_db_vector_mask(device_t dev, uint32_t vector); 30912115Sdysonstatic bool ntb_link_is_up(device_t dev, enum ntb_speed *speed, 31012115Sdyson enum ntb_width *width); 31112115Sdysonstatic int ntb_link_enable(device_t dev, enum ntb_speed speed, 31212115Sdyson enum ntb_width width); 313202283Slulfstatic int ntb_link_disable(device_t dev); 314202283Slulfstatic int ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 31512115Sdysonstatic int ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 316202283Slulf 31712115Sdysonstatic unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 31812115Sdysonstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 31912115Sdysonstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 32012115Sdysonstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 32112115Sdyson uint32_t *base, uint32_t *xlat, uint32_t *lmt); 32212115Sdysonstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 323217584Sjhbstatic int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 324217584Sjhb vm_memattr_t); 32512115Sdysonstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 32612115Sdyson const char *); 32712115Sdysonstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 32812115Sdysonstatic int map_memory_window_bar(struct ntb_softc *ntb, 32912115Sdyson struct ntb_pci_bar_info *bar); 33012115Sdysonstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 33112115Sdysonstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 33212115Sdysonstatic int ntb_init_isr(struct ntb_softc *ntb); 33312115Sdysonstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 33412115Sdysonstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 335202283Slulfstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 336202283Slulfstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 33712115Sdysonstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec); 33812115Sdysonstatic void ndev_vec_isr(void *arg); 33912115Sdysonstatic void ndev_irq_isr(void *arg); 34012115Sdysonstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 34112115Sdysonstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 342111742Sdesstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 34312115Sdysonstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 34412115Sdysonstatic void ntb_free_msix_vec(struct ntb_softc *ntb); 34512115Sdysonstatic void ntb_get_msix_info(struct ntb_softc *ntb); 34612115Sdysonstatic void ntb_exchange_msix(void *); 34712115Sdysonstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 34812115Sdysonstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 34912115Sdysonstatic int ntb_detect_xeon(struct ntb_softc *ntb); 35012115Sdysonstatic int ntb_detect_atom(struct ntb_softc *ntb); 35112115Sdysonstatic int ntb_xeon_init_dev(struct ntb_softc *ntb); 35212115Sdysonstatic int ntb_atom_init_dev(struct ntb_softc *ntb); 35312115Sdysonstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 35412115Sdysonstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 35512115Sdysonstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 35612115Sdyson enum ntb_bar regbar); 35712115Sdysonstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *, 35812115Sdyson uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 359202283Slulfstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 360202283Slulf enum ntb_bar idx); 36112115Sdysonstatic int xeon_setup_b2b_mw(struct ntb_softc *, 362202283Slulf const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 36312115Sdysonstatic int xeon_setup_msix_bar(struct ntb_softc *); 36412115Sdysonstatic inline bool link_is_up(struct ntb_softc *ntb); 365202283Slulfstatic inline bool _xeon_link_is_up(struct ntb_softc *ntb); 366202283Slulfstatic inline bool atom_link_is_err(struct ntb_softc *ntb); 367202283Slulfstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 36896749Siedowsestatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 36912115Sdysonstatic void atom_link_hb(void *arg); 37012115Sdysonstatic void ntb_link_event(device_t dev); 37112115Sdysonstatic void ntb_db_event(device_t dev, uint32_t vec); 37212115Sdysonstatic void recover_atom_link(void *arg); 373202283Slulfstatic bool ntb_poll_link(struct ntb_softc *ntb); 374202283Slulfstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 37512115Sdysonstatic void ntb_sysctl_init(struct ntb_softc *); 37612115Sdysonstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 37712115Sdysonstatic int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 378202283Slulfstatic int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 37912115Sdysonstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 38012115Sdysonstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 38112115Sdyson 382202283Slulfstatic unsigned g_ntb_hw_debug_level; 38312115SdysonTUNABLE_INT("hw.ntb.debug_level", &g_ntb_hw_debug_level); 38412115SdysonSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 38543301Sdillon &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 386202283Slulf#define ntb_printf(lvl, ...) do { \ 38796749Siedowse if ((lvl) <= g_ntb_hw_debug_level) { \ 38812115Sdyson device_printf(ntb->device, __VA_ARGS__); \ 38912115Sdyson } \ 39012115Sdyson} while (0) 39112115Sdyson 39212115Sdyson#define _NTB_PAT_UC 0 39312115Sdyson#define _NTB_PAT_WC 1 39412115Sdyson#define _NTB_PAT_WT 4 39512115Sdyson#define _NTB_PAT_WP 5 39612115Sdyson#define _NTB_PAT_WB 6 39712115Sdyson#define _NTB_PAT_UCM 7 39812115Sdysonstatic unsigned g_ntb_mw_pat = _NTB_PAT_UC; 39912115SdysonTUNABLE_INT("hw.ntb.default_mw_pat", &g_ntb_mw_pat); 40012115SdysonSYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 40112115Sdyson &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 40212115Sdyson "UC: " __XSTRING(_NTB_PAT_UC) ", " 40312115Sdyson "WC: " __XSTRING(_NTB_PAT_WC) ", " 40496753Siedowse "WT: " __XSTRING(_NTB_PAT_WT) ", " 40596753Siedowse "WP: " __XSTRING(_NTB_PAT_WP) ", " 40612115Sdyson "WB: " __XSTRING(_NTB_PAT_WB) ", " 407202283Slulf "UC-: " __XSTRING(_NTB_PAT_UCM)); 40812115Sdyson 409202283Slulfstatic inline vm_memattr_t 41012147Sdysonntb_pat_flags(void) 41112115Sdyson{ 412202283Slulf 41312115Sdyson switch (g_ntb_mw_pat) { 414202283Slulf case _NTB_PAT_WC: 41512115Sdyson return (VM_MEMATTR_WRITE_COMBINING); 41612115Sdyson case _NTB_PAT_WT: 41712115Sdyson return (VM_MEMATTR_WRITE_THROUGH); 41812115Sdyson case _NTB_PAT_WP: 41912115Sdyson return (VM_MEMATTR_WRITE_PROTECTED); 42012115Sdyson case _NTB_PAT_WB: 42112115Sdyson return (VM_MEMATTR_WRITE_BACK); 42212115Sdyson case _NTB_PAT_UCM: 42312115Sdyson return (VM_MEMATTR_WEAK_UNCACHEABLE); 42412115Sdyson case _NTB_PAT_UC: 42512115Sdyson /* FALLTHROUGH */ 426202283Slulf default: 42712115Sdyson return (VM_MEMATTR_UNCACHEABLE); 428202283Slulf } 429202283Slulf} 43012115Sdyson 43112115Sdyson/* 43212115Sdyson * Well, this obviously doesn't belong here, but it doesn't seem to exist 433202283Slulf * anywhere better yet. 434202283Slulf */ 43512115Sdysonstatic inline const char * 43612115Sdysonntb_vm_memattr_to_str(vm_memattr_t pat) 43712115Sdyson{ 438202283Slulf 43912115Sdyson switch (pat) { 44012115Sdyson case VM_MEMATTR_WRITE_COMBINING: 441202283Slulf return ("WRITE_COMBINING"); 442202283Slulf case VM_MEMATTR_WRITE_THROUGH: 44312115Sdyson return ("WRITE_THROUGH"); 44412115Sdyson case VM_MEMATTR_WRITE_PROTECTED: 44512115Sdyson return ("WRITE_PROTECTED"); 44612115Sdyson case VM_MEMATTR_WRITE_BACK: 44712115Sdyson return ("WRITE_BACK"); 44812115Sdyson case VM_MEMATTR_WEAK_UNCACHEABLE: 44912115Sdyson return ("UNCACHED"); 45012115Sdyson case VM_MEMATTR_UNCACHEABLE: 451202283Slulf return ("UNCACHEABLE"); 452202283Slulf default: 45312115Sdyson return ("UNKNOWN"); 454202283Slulf } 45512115Sdyson} 45612115Sdyson 45712115Sdysonstatic int g_ntb_msix_idx = 0; 45812115SdysonTUNABLE_INT("hw.ntb.msix_mw_idx", &g_ntb_msix_idx); 45912115SdysonSYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 46012115Sdyson 0, "Use this memory window to access the peer MSIX message complex on " 461202283Slulf "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 46212115Sdyson "Like b2b_mw_idx, negative values index from the last available memory " 46312115Sdyson "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 46412115Sdyson 465202283Slulfstatic int g_ntb_mw_idx = -1; 466202283SlulfTUNABLE_INT("hw.ntb.b2b_mw_idx", &g_ntb_mw_idx); 467202283SlulfSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 468202283Slulf 0, "Use this memory window to access the peer NTB registers. A " 469202283Slulf "non-negative value starts from the first MW index; a negative value " 47012115Sdyson "starts from the last MW index. The default is -1, i.e., the last " 47112115Sdyson "available memory window. Both sides of the NTB MUST set the same " 47212115Sdyson "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 47312115Sdyson 47412115Sdyson/* Hardware owns the low 16 bits of features. */ 47512115Sdyson#define NTB_BAR_SIZE_4K (1 << 0) 47612115Sdyson#define NTB_SDOORBELL_LOCKUP (1 << 1) 47712115Sdyson#define NTB_SB01BASE_LOCKUP (1 << 2) 478202283Slulf#define NTB_B2BDOORBELL_BIT14 (1 << 3) 479202283Slulf/* Software/configuration owns the top 16 bits. */ 48012115Sdyson#define NTB_SPLIT_BAR (1ull << 16) 48112115Sdyson 482202283Slulf#define NTB_FEATURES_STR \ 48312115Sdyson "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 48412115Sdyson "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 48512115Sdyson 48612115Sdysonstatic struct ntb_hw_info pci_ids[] = { 48712115Sdyson /* XXX: PS/SS IDs left out until they are supported. */ 48812115Sdyson { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 48912115Sdyson NTB_ATOM, 0 }, 49012115Sdyson 49112115Sdyson { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 49212115Sdyson NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 49312115Sdyson { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 49412115Sdyson NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 49512115Sdyson { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 49683366Sjulian NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 49712115Sdyson NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 49812115Sdyson { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 49912115Sdyson NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 50012115Sdyson NTB_SB01BASE_LOCKUP }, 50112115Sdyson { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 50212115Sdyson NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 50312115Sdyson NTB_SB01BASE_LOCKUP }, 50412115Sdyson 50512115Sdyson { 0x00000000, NULL, NTB_ATOM, 0 } 50612115Sdyson}; 50712115Sdyson 508202283Slulfstatic const struct ntb_reg atom_reg = { 50912115Sdyson .ntb_ctl = ATOM_NTBCNTL_OFFSET, 51012115Sdyson .lnk_sta = ATOM_LINK_STATUS_OFFSET, 51112115Sdyson .db_size = sizeof(uint64_t), 51212115Sdyson .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 51312115Sdyson}; 51412115Sdyson 51512115Sdysonstatic const struct ntb_alt_reg atom_pri_reg = { 51612115Sdyson .db_bell = ATOM_PDOORBELL_OFFSET, 517202283Slulf .db_mask = ATOM_PDBMSK_OFFSET, 51812115Sdyson .spad = ATOM_SPAD_OFFSET, 51912115Sdyson}; 52012115Sdyson 52112115Sdysonstatic const struct ntb_alt_reg atom_b2b_reg = { 52212115Sdyson .db_bell = ATOM_B2B_DOORBELL_OFFSET, 52312115Sdyson .spad = ATOM_B2B_SPAD_OFFSET, 52412115Sdyson}; 52512115Sdyson 52612115Sdysonstatic const struct ntb_xlat_reg atom_sec_xlat = { 52712115Sdyson#if 0 52812115Sdyson /* "FIXME" says the Linux driver. */ 52912115Sdyson .bar0_base = ATOM_SBAR0BASE_OFFSET, 53012115Sdyson .bar2_base = ATOM_SBAR2BASE_OFFSET, 53112115Sdyson .bar4_base = ATOM_SBAR4BASE_OFFSET, 53212115Sdyson 53312115Sdyson .bar2_limit = ATOM_SBAR2LMT_OFFSET, 53412115Sdyson .bar4_limit = ATOM_SBAR4LMT_OFFSET, 53512115Sdyson#endif 53612115Sdyson 53712115Sdyson .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 53812115Sdyson .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 53912115Sdyson}; 54012115Sdyson 54112115Sdysonstatic const struct ntb_reg xeon_reg = { 54212115Sdyson .ntb_ctl = XEON_NTBCNTL_OFFSET, 54312115Sdyson .lnk_sta = XEON_LINK_STATUS_OFFSET, 54412115Sdyson .db_size = sizeof(uint16_t), 54512115Sdyson .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 54612115Sdyson}; 54712115Sdyson 54812115Sdysonstatic const struct ntb_alt_reg xeon_pri_reg = { 549202283Slulf .db_bell = XEON_PDOORBELL_OFFSET, 55012115Sdyson .db_mask = XEON_PDBMSK_OFFSET, 551202283Slulf .spad = XEON_SPAD_OFFSET, 552202283Slulf}; 55312115Sdyson 55412115Sdysonstatic const struct ntb_alt_reg xeon_b2b_reg = { 555105420Sbde .db_bell = XEON_B2B_DOORBELL_OFFSET, 55612115Sdyson .spad = XEON_B2B_SPAD_OFFSET, 55712115Sdyson}; 55812115Sdyson 55912115Sdysonstatic const struct ntb_xlat_reg xeon_sec_xlat = { 56012115Sdyson .bar0_base = XEON_SBAR0BASE_OFFSET, 56112115Sdyson .bar2_base = XEON_SBAR2BASE_OFFSET, 56212115Sdyson .bar4_base = XEON_SBAR4BASE_OFFSET, 563202283Slulf .bar5_base = XEON_SBAR5BASE_OFFSET, 564202283Slulf 56512115Sdyson .bar2_limit = XEON_SBAR2LMT_OFFSET, 56612115Sdyson .bar4_limit = XEON_SBAR4LMT_OFFSET, 56712115Sdyson .bar5_limit = XEON_SBAR5LMT_OFFSET, 56812115Sdyson 56912115Sdyson .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 57012115Sdyson .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 57112115Sdyson .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 57212115Sdyson}; 57383366Sjulian 57412115Sdysonstatic struct ntb_b2b_addr xeon_b2b_usd_addr = { 57512115Sdyson .bar0_addr = XEON_B2B_BAR0_ADDR, 57612115Sdyson .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 57712115Sdyson .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 57812115Sdyson .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 57912115Sdyson .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 58012115Sdyson}; 58112115Sdyson 58212115Sdysonstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = { 58312115Sdyson .bar0_addr = XEON_B2B_BAR0_ADDR, 58412115Sdyson .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 585202283Slulf .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 58612115Sdyson .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 58712115Sdyson .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 58812115Sdyson}; 58912115Sdyson 590202283SlulfSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 59192462Smckusick "B2B MW segment overrides -- MUST be the same on both sides"); 59212115Sdyson 59312115SdysonTUNABLE_QUAD("hw.ntb.usd_bar2_addr64", &xeon_b2b_usd_addr.bar2_addr64); 59412115SdysonSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 59512115Sdyson &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 59612115Sdyson "hardware, use this 64-bit address on the bus between the NTB devices for " 59712115Sdyson "the window at BAR2, on the upstream side of the link. MUST be the same " 59812115Sdyson "address on both sides."); 59912115SdysonTUNABLE_QUAD("hw.ntb.usd_bar4_addr64", &xeon_b2b_usd_addr.bar4_addr64); 60012115SdysonSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 60112115Sdyson &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 60212115SdysonTUNABLE_QUAD("hw.ntb.usd_bar4_addr32", &xeon_b2b_usd_addr.bar4_addr32); 60312115SdysonSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 60412115Sdyson &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 60512115Sdyson "(split-BAR mode)."); 60612115SdysonTUNABLE_QUAD("hw.ntb.usd_bar5_addr32", &xeon_b2b_usd_addr.bar5_addr32); 60712115SdysonSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 60812115Sdyson &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 60912115Sdyson "(split-BAR mode)."); 61012115Sdyson 61112115SdysonTUNABLE_QUAD("hw.ntb.dsd_bar2_addr64", &xeon_b2b_dsd_addr.bar2_addr64); 61212115SdysonSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 61312115Sdyson &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 61412115Sdyson "hardware, use this 64-bit address on the bus between the NTB devices for " 61512115Sdyson "the window at BAR2, on the downstream side of the link. MUST be the same" 616144299Sjeff " address on both sides."); 61783366SjulianTUNABLE_QUAD("hw.ntb.dsd_bar4_addr64", &xeon_b2b_dsd_addr.bar4_addr64); 61812115SdysonSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 61912115Sdyson &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 62012115SdysonTUNABLE_QUAD("hw.ntb.dsd_bar4_addr32", &xeon_b2b_dsd_addr.bar4_addr32); 62112115SdysonSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 62212115Sdyson &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 623202283Slulf "(split-BAR mode)."); 62412115SdysonTUNABLE_QUAD("hw.ntb.dsd_bar5_addr32", &xeon_b2b_dsd_addr.bar5_addr32); 625202283SlulfSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 62692462Smckusick &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 62712115Sdyson "(split-BAR mode)."); 62812115Sdyson 62912115Sdyson/* 63012115Sdyson * OS <-> Driver interface structures 63112115Sdyson */ 63212115SdysonMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 63312115Sdyson 63412115SdysonSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 63512115Sdyson 63612115Sdyson/* 63712115Sdyson * OS <-> Driver linkage functions 63812115Sdyson */ 63912115Sdysonstatic int 64012115Sdysonntb_probe(device_t device) 64112115Sdyson{ 64212115Sdyson struct ntb_hw_info *p; 64312115Sdyson 64412115Sdyson p = ntb_get_device_info(pci_get_devid(device)); 64512115Sdyson if (p == NULL) 64612115Sdyson return (ENXIO); 64712115Sdyson 64812115Sdyson device_set_desc(device, p->desc); 64912115Sdyson return (0); 65012115Sdyson} 65112115Sdyson 65212115Sdysonstatic int 65312115Sdysonntb_attach(device_t device) 654202283Slulf{ 655175294Sattilio struct ntb_softc *ntb; 656202283Slulf struct ntb_hw_info *p; 657202283Slulf int error; 658145006Sjeff 65912115Sdyson ntb = device_get_softc(device); 66012115Sdyson p = ntb_get_device_info(pci_get_devid(device)); 661202283Slulf 66212115Sdyson ntb->device = device; 663202283Slulf ntb->type = p->type; 664202283Slulf ntb->features = p->features; 665202283Slulf ntb->b2b_mw_idx = B2B_MW_DISABLED; 666202283Slulf ntb->msix_mw_idx = B2B_MW_DISABLED; 667202283Slulf 668202283Slulf /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 669202283Slulf callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE); 670202283Slulf callout_init(&ntb->lr_timer, CALLOUT_MPSAFE); 671202283Slulf callout_init(&ntb->peer_msix_work, 1); 672202283Slulf mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 673202283Slulf mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 67412115Sdyson 67512115Sdyson if (ntb->type == NTB_ATOM) 676202283Slulf error = ntb_detect_atom(ntb); 67792462Smckusick else 67812115Sdyson error = ntb_detect_xeon(ntb); 67912115Sdyson if (error != 0) 68012115Sdyson goto out; 68112115Sdyson 68212115Sdyson ntb_detect_max_mw(ntb); 68312115Sdyson 68412115Sdyson pci_enable_busmaster(ntb->device); 68512115Sdyson 68612115Sdyson error = ntb_map_pci_bars(ntb); 68712115Sdyson if (error != 0) 68812115Sdyson goto out; 68912115Sdyson if (ntb->type == NTB_ATOM) 69096749Siedowse error = ntb_atom_init_dev(ntb); 69196749Siedowse else 69296749Siedowse error = ntb_xeon_init_dev(ntb); 69396749Siedowse if (error != 0) 69496749Siedowse goto out; 69596749Siedowse 69696749Siedowse ntb_spad_clear(device); 69796749Siedowse 69896749Siedowse ntb_poll_link(ntb); 699202283Slulf 700202283Slulf ntb_sysctl_init(ntb); 701202283Slulf 702202283Slulf /* Attach children to this controller */ 70396749Siedowse device_add_child(device, NULL, -1); 704202283Slulf bus_generic_attach(device); 705202283Slulf 70696749Siedowseout: 70796749Siedowse if (error != 0) 70812115Sdyson ntb_detach(device); 70912115Sdyson return (error); 71012115Sdyson} 71112115Sdyson 71212115Sdysonstatic int 71312115Sdysonntb_detach(device_t device) 71412115Sdyson{ 71512115Sdyson struct ntb_softc *ntb; 71612115Sdyson 71712115Sdyson ntb = device_get_softc(device); 71812115Sdyson 71912159Sbde /* Detach & delete all children */ 72012115Sdyson device_delete_children(device); 72112115Sdyson 722202283Slulf if (ntb->self_reg != NULL) { 72312115Sdyson DB_MASK_LOCK(ntb); 72412115Sdyson db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 725202283Slulf DB_MASK_UNLOCK(ntb); 72612115Sdyson } 727111742Sdes callout_drain(&ntb->heartbeat_timer); 72812115Sdyson callout_drain(&ntb->lr_timer); 729202283Slulf callout_drain(&ntb->peer_msix_work); 730111742Sdes pci_disable_busmaster(ntb->device); 731202283Slulf if (ntb->type == NTB_XEON) 732111742Sdes ntb_teardown_xeon(ntb); 733202283Slulf ntb_teardown_interrupts(ntb); 734111742Sdes 735202283Slulf mtx_destroy(&ntb->db_mask_lock); 736111742Sdes mtx_destroy(&ntb->ctx_lock); 737111742Sdes 73812115Sdyson ntb_unmap_pci_bar(ntb); 739111742Sdes 74012115Sdyson return (0); 74112115Sdyson} 742111742Sdes 743111742Sdes/* 744111742Sdes * Driver internal routines 745202283Slulf */ 746202283Slulfstatic inline enum ntb_bar 747111742Sdesntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 748111742Sdes{ 74912115Sdyson 75012115Sdyson KASSERT(mw < ntb->mw_count, 75112115Sdyson ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 75212115Sdyson KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 75312115Sdyson 75412115Sdyson return (ntb->reg->mw_bar[mw]); 75512115Sdyson} 75612115Sdyson 75712115Sdysonstatic inline bool 75812115Sdysonbar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 75912115Sdyson{ 76012115Sdyson /* XXX This assertion could be stronger. */ 76112115Sdyson KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 76212115Sdyson return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 76396752Siedowse} 76412115Sdyson 765202283Slulfstatic inline void 76696752Siedowsebar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 76712115Sdyson uint32_t *xlat, uint32_t *lmt) 768202283Slulf{ 76912115Sdyson uint32_t basev, lmtv, xlatv; 77012115Sdyson 77112115Sdyson switch (bar) { 77212115Sdyson case NTB_B2B_BAR_1: 77312115Sdyson basev = ntb->xlat_reg->bar2_base; 774202283Slulf lmtv = ntb->xlat_reg->bar2_limit; 77512115Sdyson xlatv = ntb->xlat_reg->bar2_xlat; 77612115Sdyson break; 777153110Sru case NTB_B2B_BAR_2: 77812115Sdyson basev = ntb->xlat_reg->bar4_base; 77912115Sdyson lmtv = ntb->xlat_reg->bar4_limit; 78012115Sdyson xlatv = ntb->xlat_reg->bar4_xlat; 78112115Sdyson break; 782202283Slulf case NTB_B2B_BAR_3: 783202283Slulf basev = ntb->xlat_reg->bar5_base; 784193377Sstas lmtv = ntb->xlat_reg->bar5_limit; 785202283Slulf xlatv = ntb->xlat_reg->bar5_xlat; 786202283Slulf break; 78755477Sbde default: 788202283Slulf KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 789202283Slulf ("bad bar")); 790202283Slulf basev = lmtv = xlatv = 0; 79112115Sdyson break; 79212115Sdyson } 79312115Sdyson 79412115Sdyson if (base != NULL) 79512115Sdyson *base = basev; 79612115Sdyson if (xlat != NULL) 79712115Sdyson *xlat = xlatv; 79812115Sdyson if (lmt != NULL) 79912115Sdyson *lmt = lmtv; 80012115Sdyson} 801202283Slulf 80212115Sdysonstatic int 80312115Sdysonntb_map_pci_bars(struct ntb_softc *ntb) 80412115Sdyson{ 80512115Sdyson int rc; 80612115Sdyson 80712115Sdyson ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 80812115Sdyson rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 80983366Sjulian if (rc != 0) 81012115Sdyson goto out; 81112115Sdyson 81296749Siedowse ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 81312115Sdyson rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 81412115Sdyson if (rc != 0) 81512115Sdyson goto out; 816202283Slulf ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 81712115Sdyson ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 81812115Sdyson ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 81912115Sdyson 82012115Sdyson ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 82112115Sdyson rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 82212115Sdyson if (rc != 0) 82312115Sdyson goto out; 82412115Sdyson ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 82512115Sdyson ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 82612115Sdyson ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 82712115Sdyson 82812115Sdyson if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 82912115Sdyson goto out; 83012115Sdyson 83112115Sdyson ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 83212115Sdyson rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 83312115Sdyson ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 83412115Sdyson ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 83512115Sdyson ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 83612115Sdyson 83712115Sdysonout: 83812115Sdyson if (rc != 0) 83912115Sdyson device_printf(ntb->device, 84012115Sdyson "unable to allocate pci resource\n"); 84112115Sdyson return (rc); 84212115Sdyson} 84396749Siedowse 84496749Siedowsestatic void 84512115Sdysonprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 84612115Sdyson const char *kind) 84712115Sdyson{ 84812115Sdyson 84912115Sdyson device_printf(ntb->device, 85012115Sdyson "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 85112115Sdyson PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 85212115Sdyson (char *)bar->vbase + bar->size - 1, 853202283Slulf (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 854202283Slulf (uintmax_t)bar->size, kind); 855202283Slulf} 856202283Slulf 857202283Slulfstatic int 858202283Slulfmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 85912115Sdyson{ 860202283Slulf 861202283Slulf bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 86212115Sdyson &bar->pci_resource_id, RF_ACTIVE); 86312115Sdyson if (bar->pci_resource == NULL) 86412115Sdyson return (ENXIO); 86512115Sdyson 866202283Slulf save_bar_parameters(bar); 867202283Slulf bar->map_mode = VM_MEMATTR_UNCACHEABLE; 868202283Slulf print_map_success(ntb, bar, "mmr"); 86912115Sdyson return (0); 87012115Sdyson} 87112115Sdyson 87212115Sdysonstatic int 87312115Sdysonmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 87412115Sdyson{ 875202283Slulf int rc; 87612115Sdyson vm_memattr_t mapmode; 87712115Sdyson uint8_t bar_size_bits = 0; 878202283Slulf 87912115Sdyson bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 88012115Sdyson &bar->pci_resource_id, RF_ACTIVE); 88112115Sdyson 882202283Slulf if (bar->pci_resource == NULL) 883202283Slulf return (ENXIO); 884202283Slulf 88512115Sdyson save_bar_parameters(bar); 88612115Sdyson /* 887221166Sjhb * Ivytown NTB BAR sizes are misreported by the hardware due to a 888221166Sjhb * hardware issue. To work around this, query the size it should be 889221166Sjhb * configured to by the device and modify the resource to correspond to 890221166Sjhb * this new size. The BIOS on systems with this problem is required to 891221166Sjhb * provide enough address space to allow the driver to make this change 892221166Sjhb * safely. 89312115Sdyson * 89412115Sdyson * Ideally I could have just specified the size when I allocated the 89596749Siedowse * resource like: 89683366Sjulian * bus_alloc_resource(ntb->device, 89712115Sdyson * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 89812115Sdyson * 1ul << bar_size_bits, RF_ACTIVE); 89912115Sdyson * but the PCI driver does not honor the size in this call, so we have 90012115Sdyson * to modify it after the fact. 90112115Sdyson */ 90212115Sdyson if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 90312115Sdyson if (bar->pci_resource_id == PCIR_BAR(2)) 90412115Sdyson bar_size_bits = pci_read_config(ntb->device, 90512115Sdyson XEON_PBAR23SZ_OFFSET, 1); 90612115Sdyson else 90712115Sdyson bar_size_bits = pci_read_config(ntb->device, 90812115Sdyson XEON_PBAR45SZ_OFFSET, 1); 90912115Sdyson 91012115Sdyson rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 91112115Sdyson bar->pci_resource, bar->pbase, 91212115Sdyson bar->pbase + (1ul << bar_size_bits) - 1); 91312115Sdyson if (rc != 0) { 91412115Sdyson device_printf(ntb->device, 91512115Sdyson "unable to resize bar\n"); 91612115Sdyson return (rc); 91796752Siedowse } 918202283Slulf 91912115Sdyson save_bar_parameters(bar); 92012115Sdyson } 921111742Sdes 92212115Sdyson bar->map_mode = VM_MEMATTR_UNCACHEABLE; 92312115Sdyson print_map_success(ntb, bar, "mw"); 92412115Sdyson 92512115Sdyson /* 92612115Sdyson * Optionally, mark MW BARs as anything other than UC to improve 92743301Sdillon * performance. 92896749Siedowse */ 92996749Siedowse mapmode = ntb_pat_flags(); 93012115Sdyson if (mapmode == bar->map_mode) 931202283Slulf return (0); 932126853Sphk 93312115Sdyson rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 93412115Sdyson if (rc == 0) { 93512115Sdyson bar->map_mode = mapmode; 93612115Sdyson device_printf(ntb->device, 93712115Sdyson "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 93812115Sdyson "%s.\n", 93996749Siedowse PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 94043301Sdillon (char *)bar->vbase + bar->size - 1, 94112115Sdyson (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 942202283Slulf ntb_vm_memattr_to_str(mapmode)); 943202283Slulf } else 944202283Slulf device_printf(ntb->device, 945202283Slulf "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 946202283Slulf "%s: %d\n", 947202283Slulf PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 948202283Slulf (char *)bar->vbase + bar->size - 1, 949221166Sjhb (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 950221166Sjhb ntb_vm_memattr_to_str(mapmode), rc); 951221166Sjhb /* Proceed anyway */ 952221166Sjhb return (0); 95312115Sdyson} 95412115Sdyson 95512115Sdysonstatic void 95612115Sdysonntb_unmap_pci_bar(struct ntb_softc *ntb) 95712115Sdyson{ 95812115Sdyson struct ntb_pci_bar_info *current_bar; 95912115Sdyson int i; 96012115Sdyson 96112115Sdyson for (i = 0; i < NTB_MAX_BARS; i++) { 96212115Sdyson current_bar = &ntb->bar_info[i]; 96312115Sdyson if (current_bar->pci_resource != NULL) 96412115Sdyson bus_release_resource(ntb->device, SYS_RES_MEMORY, 96512115Sdyson current_bar->pci_resource_id, 96612115Sdyson current_bar->pci_resource); 96712115Sdyson } 968202283Slulf} 96912115Sdyson 97012115Sdysonstatic int 97112115Sdysonntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 97296749Siedowse{ 97396749Siedowse uint32_t i; 97412115Sdyson int rc; 975202283Slulf 976193377Sstas for (i = 0; i < num_vectors; i++) { 977202283Slulf ntb->int_info[i].rid = i + 1; 978202283Slulf ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 97955477Sbde SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 980202283Slulf if (ntb->int_info[i].res == NULL) { 981126853Sphk device_printf(ntb->device, 98212115Sdyson "bus_alloc_resource failed\n"); 98312115Sdyson return (ENOMEM); 98412115Sdyson } 98512115Sdyson ntb->int_info[i].tag = NULL; 98612115Sdyson ntb->allocated_interrupts++; 98712115Sdyson rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 98812115Sdyson INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 98912115Sdyson &ntb->msix_vec[i], &ntb->int_info[i].tag); 99012115Sdyson if (rc != 0) { 99112115Sdyson device_printf(ntb->device, "bus_setup_intr failed\n"); 99212115Sdyson return (ENXIO); 99312115Sdyson } 99412115Sdyson } 99512115Sdyson return (0); 99612115Sdyson} 99796752Siedowse 99812115Sdyson/* 99912115Sdyson * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 100012115Sdyson * cannot be allocated for each MSI-X message. JHB seems to think remapping 100196752Siedowse * should be okay. This tunable should enable us to test that hypothesis 100212115Sdyson * when someone gets their hands on some Xeon hardware. 1003202283Slulf */ 100412115Sdysonstatic int ntb_force_remap_mode; 1005229549SpfgTUNABLE_INT("hw.ntb.force_remap_mode", &ntb_force_remap_mode); 100612115SdysonSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 1007202283Slulf &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 1008101744Srwatson " to a smaller number of ithreads, even if the desired number are " 1009101744Srwatson "available"); 1010101941Srwatson 101112115Sdyson/* 101212115Sdyson * In case it is NOT ok, give consumers an abort button. 101312115Sdyson */ 101412115Sdysonstatic int ntb_prefer_intx; 101512115SdysonTUNABLE_INT("hw.ntb.prefer_intx_to_remap", &ntb_prefer_intx); 101612115SdysonSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 101712115Sdyson &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1018202283Slulf "than remapping MSI-X messages over available slots (match Linux driver " 101912115Sdyson "behavior)"); 102012115Sdyson 1021202283Slulf/* 102212115Sdyson * Remap the desired number of MSI-X messages to available ithreads in a simple 102312115Sdyson * round-robin fashion. 1024202283Slulf */ 102512115Sdysonstatic int 102612115Sdysonntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1027202283Slulf{ 102812115Sdyson u_int *vectors; 102912115Sdyson uint32_t i; 103012115Sdyson int rc; 103112115Sdyson 103212115Sdyson if (ntb_prefer_intx != 0) 103312115Sdyson return (ENXIO); 103412115Sdyson 103512115Sdyson vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1036202283Slulf 103712115Sdyson for (i = 0; i < desired; i++) 103812115Sdyson vectors[i] = (i % avail) + 1; 103912115Sdyson 104012115Sdyson rc = pci_remap_msix(dev, desired, vectors); 104112115Sdyson free(vectors, M_NTB); 104212115Sdyson return (rc); 104312115Sdyson} 104412115Sdyson 104512115Sdysonstatic int 104612115Sdysonntb_init_isr(struct ntb_softc *ntb) 104712115Sdyson{ 104812115Sdyson uint32_t desired_vectors, num_vectors; 104912115Sdyson int rc; 105012115Sdyson 105112115Sdyson ntb->allocated_interrupts = 0; 105212115Sdyson ntb->last_ts = ticks; 105312115Sdyson 105412115Sdyson /* 105512115Sdyson * Mask all doorbell interrupts. (Except link events!) 105612115Sdyson */ 105712115Sdyson DB_MASK_LOCK(ntb); 105812115Sdyson ntb->db_mask = ntb->db_valid_mask; 105912115Sdyson db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 106012115Sdyson DB_MASK_UNLOCK(ntb); 106112115Sdyson 1062221128Sjhb num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 106312115Sdyson ntb->db_count); 106412115Sdyson if (desired_vectors >= 1) { 106512115Sdyson rc = pci_alloc_msix(ntb->device, &num_vectors); 106612115Sdyson 106712115Sdyson if (ntb_force_remap_mode != 0 && rc == 0 && 106812115Sdyson num_vectors == desired_vectors) 106912115Sdyson num_vectors--; 107012115Sdyson 107112115Sdyson if (rc == 0 && num_vectors < desired_vectors) { 107212115Sdyson rc = ntb_remap_msix(ntb->device, desired_vectors, 1073229549Spfg num_vectors); 1074194296Skib if (rc == 0) 1075194296Skib num_vectors = desired_vectors; 107612115Sdyson else 107712115Sdyson pci_release_msi(ntb->device); 107857710Sbde } 107912115Sdyson if (rc != 0) 108012115Sdyson num_vectors = 1; 108112115Sdyson } else 108212115Sdyson num_vectors = 1; 108312115Sdyson 108412115Sdyson if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 108512115Sdyson if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 108612115Sdyson device_printf(ntb->device, 108712115Sdyson "Errata workaround does not support MSI or INTX\n"); 108812115Sdyson return (EINVAL); 108912115Sdyson } 109012115Sdyson 109112115Sdyson ntb->db_vec_count = 1; 109292462Smckusick ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 109392462Smckusick rc = ntb_setup_legacy_interrupt(ntb); 109412115Sdyson } else { 109512115Sdyson if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 109612115Sdyson HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 109712115Sdyson device_printf(ntb->device, 109812115Sdyson "Errata workaround expects %d doorbell bits\n", 109912115Sdyson XEON_NONLINK_DB_MSIX_BITS); 110012115Sdyson return (EINVAL); 110112115Sdyson } 110212115Sdyson 110312115Sdyson ntb_create_msix_vec(ntb, num_vectors); 110412115Sdyson rc = ntb_setup_msix(ntb, num_vectors); 110512115Sdyson if (rc == 0 && HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1106 ntb_get_msix_info(ntb); 1107 } 1108 if (rc != 0) { 1109 device_printf(ntb->device, 1110 "Error allocating interrupts: %d\n", rc); 1111 ntb_free_msix_vec(ntb); 1112 } 1113 1114 return (rc); 1115} 1116 1117static int 1118ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1119{ 1120 int rc; 1121 1122 ntb->int_info[0].rid = 0; 1123 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1124 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1125 if (ntb->int_info[0].res == NULL) { 1126 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1127 return (ENOMEM); 1128 } 1129 1130 ntb->int_info[0].tag = NULL; 1131 ntb->allocated_interrupts = 1; 1132 1133 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1134 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1135 ntb, &ntb->int_info[0].tag); 1136 if (rc != 0) { 1137 device_printf(ntb->device, "bus_setup_intr failed\n"); 1138 return (ENXIO); 1139 } 1140 1141 return (0); 1142} 1143 1144static void 1145ntb_teardown_interrupts(struct ntb_softc *ntb) 1146{ 1147 struct ntb_int_info *current_int; 1148 int i; 1149 1150 for (i = 0; i < ntb->allocated_interrupts; i++) { 1151 current_int = &ntb->int_info[i]; 1152 if (current_int->tag != NULL) 1153 bus_teardown_intr(ntb->device, current_int->res, 1154 current_int->tag); 1155 1156 if (current_int->res != NULL) 1157 bus_release_resource(ntb->device, SYS_RES_IRQ, 1158 rman_get_rid(current_int->res), current_int->res); 1159 } 1160 1161 ntb_free_msix_vec(ntb); 1162 pci_release_msi(ntb->device); 1163} 1164 1165/* 1166 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1167 * out to make code clearer. 1168 */ 1169static inline uint64_t 1170db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1171{ 1172 1173 if (ntb->type == NTB_ATOM) 1174 return (ntb_reg_read(8, regoff)); 1175 1176 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1177 1178 return (ntb_reg_read(2, regoff)); 1179} 1180 1181static inline void 1182db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1183{ 1184 1185 KASSERT((val & ~ntb->db_valid_mask) == 0, 1186 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1187 (uintmax_t)(val & ~ntb->db_valid_mask), 1188 (uintmax_t)ntb->db_valid_mask)); 1189 1190 if (regoff == ntb->self_reg->db_mask) 1191 DB_MASK_ASSERT(ntb, MA_OWNED); 1192 db_iowrite_raw(ntb, regoff, val); 1193} 1194 1195static inline void 1196db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1197{ 1198 1199 if (ntb->type == NTB_ATOM) { 1200 ntb_reg_write(8, regoff, val); 1201 return; 1202 } 1203 1204 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1205 ntb_reg_write(2, regoff, (uint16_t)val); 1206} 1207 1208static void 1209ntb_db_set_mask(device_t dev, uint64_t bits) 1210{ 1211 struct ntb_softc *ntb = device_get_softc(dev); 1212 1213 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1214 return; 1215 1216 DB_MASK_LOCK(ntb); 1217 ntb->db_mask |= bits; 1218 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1219 DB_MASK_UNLOCK(ntb); 1220} 1221 1222static void 1223ntb_db_clear_mask(device_t dev, uint64_t bits) 1224{ 1225 struct ntb_softc *ntb = device_get_softc(dev); 1226 1227 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1228 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1229 (uintmax_t)(bits & ~ntb->db_valid_mask), 1230 (uintmax_t)ntb->db_valid_mask)); 1231 1232 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1233 return; 1234 1235 DB_MASK_LOCK(ntb); 1236 ntb->db_mask &= ~bits; 1237 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1238 DB_MASK_UNLOCK(ntb); 1239} 1240 1241static uint64_t 1242ntb_db_read(device_t dev) 1243{ 1244 struct ntb_softc *ntb = device_get_softc(dev); 1245 1246 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1247 uint64_t res; 1248 unsigned i; 1249 1250 res = 0; 1251 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1252 if (ntb->msix_vec[i].masked != 0) 1253 res |= ntb_db_vector_mask(dev, i); 1254 } 1255 return (res); 1256 } 1257 1258 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1259} 1260 1261static void 1262ntb_db_clear(device_t dev, uint64_t bits) 1263{ 1264 struct ntb_softc *ntb = device_get_softc(dev); 1265 1266 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1267 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1268 (uintmax_t)(bits & ~ntb->db_valid_mask), 1269 (uintmax_t)ntb->db_valid_mask)); 1270 1271 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1272 unsigned i; 1273 1274 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1275 if ((bits & ntb_db_vector_mask(dev, i)) != 0) { 1276 DB_MASK_LOCK(ntb); 1277 if (ntb->msix_vec[i].masked != 0) { 1278 /* XXX These need a public API. */ 1279#if 0 1280 pci_unmask_msix(ntb->device, i); 1281#endif 1282 ntb->msix_vec[i].masked = 0; 1283 } 1284 DB_MASK_UNLOCK(ntb); 1285 } 1286 } 1287 return; 1288 } 1289 1290 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1291} 1292 1293static inline uint64_t 1294ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1295{ 1296 uint64_t shift, mask; 1297 1298 shift = ntb->db_vec_shift; 1299 mask = (1ull << shift) - 1; 1300 return (mask << (shift * db_vector)); 1301} 1302 1303static void 1304ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1305{ 1306 uint64_t vec_mask; 1307 1308 ntb->last_ts = ticks; 1309 vec_mask = ntb_vec_mask(ntb, vec); 1310 1311 if ((vec_mask & ntb->db_link_mask) != 0) { 1312 if (ntb_poll_link(ntb)) 1313 ntb_link_event(ntb->device); 1314 } 1315 1316 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1317 (vec_mask & ntb->db_link_mask) == 0) { 1318 DB_MASK_LOCK(ntb); 1319 if (ntb->msix_vec[vec].masked == 0) { 1320 /* XXX These need a public API. */ 1321#if 0 1322 pci_mask_msix(ntb->device, vec); 1323#endif 1324 ntb->msix_vec[vec].masked = 1; 1325 } 1326 DB_MASK_UNLOCK(ntb); 1327 } 1328 1329 if ((vec_mask & ntb->db_valid_mask) != 0) 1330 ntb_db_event(ntb->device, vec); 1331} 1332 1333static void 1334ndev_vec_isr(void *arg) 1335{ 1336 struct ntb_vec *nvec = arg; 1337 1338 ntb_interrupt(nvec->ntb, nvec->num); 1339} 1340 1341static void 1342ndev_irq_isr(void *arg) 1343{ 1344 /* If we couldn't set up MSI-X, we only have the one vector. */ 1345 ntb_interrupt(arg, 0); 1346} 1347 1348static int 1349ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1350{ 1351 uint32_t i; 1352 1353 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1354 M_ZERO | M_WAITOK); 1355 for (i = 0; i < num_vectors; i++) { 1356 ntb->msix_vec[i].num = i; 1357 ntb->msix_vec[i].ntb = ntb; 1358 } 1359 1360 return (0); 1361} 1362 1363static void 1364ntb_free_msix_vec(struct ntb_softc *ntb) 1365{ 1366 1367 if (ntb->msix_vec == NULL) 1368 return; 1369 1370 free(ntb->msix_vec, M_NTB); 1371 ntb->msix_vec = NULL; 1372} 1373 1374static void 1375ntb_get_msix_info(struct ntb_softc *ntb) 1376{ 1377 struct pci_devinfo *dinfo; 1378 struct pcicfg_msix *msix; 1379 uint32_t laddr, data, i, offset; 1380 1381 dinfo = device_get_ivars(ntb->device); 1382 msix = &dinfo->cfg.msix; 1383 1384 CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1385 1386 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1387 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1388 1389 laddr = bus_read_4(msix->msix_table_res, offset + 1390 PCI_MSIX_ENTRY_LOWER_ADDR); 1391 ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1392 1393 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1394 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1395 MSI_INTEL_ADDR_BASE)); 1396 ntb->msix_data[i].nmd_ofs = laddr; 1397 1398 data = bus_read_4(msix->msix_table_res, offset + 1399 PCI_MSIX_ENTRY_DATA); 1400 ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1401 1402 ntb->msix_data[i].nmd_data = data; 1403 } 1404} 1405 1406static struct ntb_hw_info * 1407ntb_get_device_info(uint32_t device_id) 1408{ 1409 struct ntb_hw_info *ep = pci_ids; 1410 1411 while (ep->device_id) { 1412 if (ep->device_id == device_id) 1413 return (ep); 1414 ++ep; 1415 } 1416 return (NULL); 1417} 1418 1419static void 1420ntb_teardown_xeon(struct ntb_softc *ntb) 1421{ 1422 1423 if (ntb->reg != NULL) 1424 ntb_link_disable(ntb->device); 1425} 1426 1427static void 1428ntb_detect_max_mw(struct ntb_softc *ntb) 1429{ 1430 1431 if (ntb->type == NTB_ATOM) { 1432 ntb->mw_count = ATOM_MW_COUNT; 1433 return; 1434 } 1435 1436 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1437 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1438 else 1439 ntb->mw_count = XEON_SNB_MW_COUNT; 1440} 1441 1442static int 1443ntb_detect_xeon(struct ntb_softc *ntb) 1444{ 1445 uint8_t ppd, conn_type; 1446 1447 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1448 ntb->ppd = ppd; 1449 1450 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1451 ntb->dev_type = NTB_DEV_DSD; 1452 else 1453 ntb->dev_type = NTB_DEV_USD; 1454 1455 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1456 ntb->features |= NTB_SPLIT_BAR; 1457 1458 /* 1459 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1460 * errata workaround; only do one at a time. 1461 */ 1462 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1463 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1464 1465 conn_type = ppd & XEON_PPD_CONN_TYPE; 1466 switch (conn_type) { 1467 case NTB_CONN_B2B: 1468 ntb->conn_type = conn_type; 1469 break; 1470 case NTB_CONN_RP: 1471 case NTB_CONN_TRANSPARENT: 1472 default: 1473 device_printf(ntb->device, "Unsupported connection type: %u\n", 1474 (unsigned)conn_type); 1475 return (ENXIO); 1476 } 1477 return (0); 1478} 1479 1480static int 1481ntb_detect_atom(struct ntb_softc *ntb) 1482{ 1483 uint32_t ppd, conn_type; 1484 1485 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1486 ntb->ppd = ppd; 1487 1488 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1489 ntb->dev_type = NTB_DEV_DSD; 1490 else 1491 ntb->dev_type = NTB_DEV_USD; 1492 1493 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1494 switch (conn_type) { 1495 case NTB_CONN_B2B: 1496 ntb->conn_type = conn_type; 1497 break; 1498 default: 1499 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1500 return (ENXIO); 1501 } 1502 return (0); 1503} 1504 1505static int 1506ntb_xeon_init_dev(struct ntb_softc *ntb) 1507{ 1508 int rc; 1509 1510 ntb->spad_count = XEON_SPAD_COUNT; 1511 ntb->db_count = XEON_DB_COUNT; 1512 ntb->db_link_mask = XEON_DB_LINK_BIT; 1513 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1514 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1515 1516 if (ntb->conn_type != NTB_CONN_B2B) { 1517 device_printf(ntb->device, "Connection type %d not supported\n", 1518 ntb->conn_type); 1519 return (ENXIO); 1520 } 1521 1522 ntb->reg = &xeon_reg; 1523 ntb->self_reg = &xeon_pri_reg; 1524 ntb->peer_reg = &xeon_b2b_reg; 1525 ntb->xlat_reg = &xeon_sec_xlat; 1526 1527 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1528 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1529 ntb->mw_count; 1530 ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1531 g_ntb_msix_idx, ntb->msix_mw_idx); 1532 rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1533 VM_MEMATTR_UNCACHEABLE); 1534 KASSERT(rc == 0, ("shouldn't fail")); 1535 } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1536 /* 1537 * There is a Xeon hardware errata related to writes to SDOORBELL or 1538 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1539 * which may hang the system. To workaround this, use a memory 1540 * window to access the interrupt and scratch pad registers on the 1541 * remote system. 1542 */ 1543 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1544 ntb->mw_count; 1545 ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1546 g_ntb_mw_idx, ntb->b2b_mw_idx); 1547 rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1548 VM_MEMATTR_UNCACHEABLE); 1549 KASSERT(rc == 0, ("shouldn't fail")); 1550 } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1551 /* 1552 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1553 * mirrored to the remote system. Shrink the number of bits by one, 1554 * since bit 14 is the last bit. 1555 * 1556 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1557 * anyway. Nor for non-B2B connection types. 1558 */ 1559 ntb->db_count = XEON_DB_COUNT - 1; 1560 1561 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1562 1563 if (ntb->dev_type == NTB_DEV_USD) 1564 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1565 &xeon_b2b_usd_addr); 1566 else 1567 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1568 &xeon_b2b_dsd_addr); 1569 if (rc != 0) 1570 return (rc); 1571 1572 /* Enable Bus Master and Memory Space on the secondary side */ 1573 ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1574 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1575 1576 /* 1577 * Mask all doorbell interrupts. 1578 */ 1579 DB_MASK_LOCK(ntb); 1580 ntb->db_mask = ntb->db_valid_mask; 1581 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1582 DB_MASK_UNLOCK(ntb); 1583 1584 rc = xeon_setup_msix_bar(ntb); 1585 if (rc != 0) 1586 return (rc); 1587 1588 rc = ntb_init_isr(ntb); 1589 return (rc); 1590} 1591 1592static int 1593ntb_atom_init_dev(struct ntb_softc *ntb) 1594{ 1595 int error; 1596 1597 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1598 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1599 1600 ntb->spad_count = ATOM_SPAD_COUNT; 1601 ntb->db_count = ATOM_DB_COUNT; 1602 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1603 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1604 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1605 1606 ntb->reg = &atom_reg; 1607 ntb->self_reg = &atom_pri_reg; 1608 ntb->peer_reg = &atom_b2b_reg; 1609 ntb->xlat_reg = &atom_sec_xlat; 1610 1611 /* 1612 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1613 * resolved. Mask transaction layer internal parity errors. 1614 */ 1615 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1616 1617 configure_atom_secondary_side_bars(ntb); 1618 1619 /* Enable Bus Master and Memory Space on the secondary side */ 1620 ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1621 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1622 1623 error = ntb_init_isr(ntb); 1624 if (error != 0) 1625 return (error); 1626 1627 /* Initiate PCI-E link training */ 1628 ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1629 1630 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1631 1632 return (0); 1633} 1634 1635/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1636static void 1637configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1638{ 1639 1640 if (ntb->dev_type == NTB_DEV_USD) { 1641 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1642 XEON_B2B_BAR2_ADDR64); 1643 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1644 XEON_B2B_BAR4_ADDR64); 1645 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1646 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1647 } else { 1648 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1649 XEON_B2B_BAR2_ADDR64); 1650 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1651 XEON_B2B_BAR4_ADDR64); 1652 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1653 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1654 } 1655} 1656 1657 1658/* 1659 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1660 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1661 * remains for use by a higher layer. 1662 * 1663 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1664 * MW size is sufficiently large. 1665 */ 1666static unsigned int ntb_b2b_mw_share; 1667TUNABLE_INT("hw.ntb.b2b_mw_share", &ntb_b2b_mw_share); 1668SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1669 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1670 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1671 "value here."); 1672 1673static void 1674xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1675 enum ntb_bar regbar) 1676{ 1677 struct ntb_pci_bar_info *bar; 1678 uint8_t bar_sz; 1679 1680 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1681 return; 1682 1683 bar = &ntb->bar_info[idx]; 1684 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1685 if (idx == regbar) { 1686 if (ntb->b2b_off != 0) 1687 bar_sz--; 1688 else 1689 bar_sz = 0; 1690 } 1691 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1692 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1693 (void)bar_sz; 1694} 1695 1696static void 1697xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1698 enum ntb_bar idx, enum ntb_bar regbar) 1699{ 1700 uint64_t reg_val; 1701 uint32_t base_reg, lmt_reg; 1702 1703 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1704 if (idx == regbar) { 1705 if (ntb->b2b_off) 1706 bar_addr += ntb->b2b_off; 1707 else 1708 bar_addr = 0; 1709 } 1710 1711 /* 1712 * Set limit registers first to avoid an errata where setting the base 1713 * registers locks the limit registers. 1714 */ 1715 if (!bar_is_64bit(ntb, idx)) { 1716 ntb_reg_write(4, lmt_reg, bar_addr); 1717 reg_val = ntb_reg_read(4, lmt_reg); 1718 (void)reg_val; 1719 1720 ntb_reg_write(4, base_reg, bar_addr); 1721 reg_val = ntb_reg_read(4, base_reg); 1722 (void)reg_val; 1723 } else { 1724 ntb_reg_write(8, lmt_reg, bar_addr); 1725 reg_val = ntb_reg_read(8, lmt_reg); 1726 (void)reg_val; 1727 1728 ntb_reg_write(8, base_reg, bar_addr); 1729 reg_val = ntb_reg_read(8, base_reg); 1730 (void)reg_val; 1731 } 1732} 1733 1734static void 1735xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1736{ 1737 struct ntb_pci_bar_info *bar; 1738 1739 bar = &ntb->bar_info[idx]; 1740 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1741 ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1742 base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1743 } else { 1744 ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1745 base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1746 } 1747 (void)base_addr; 1748} 1749 1750static int 1751xeon_setup_msix_bar(struct ntb_softc *ntb) 1752{ 1753 enum ntb_bar bar_num; 1754 1755 if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1756 return (0); 1757 1758 bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1759 ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1760 return (0); 1761} 1762 1763static int 1764xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1765 const struct ntb_b2b_addr *peer_addr) 1766{ 1767 struct ntb_pci_bar_info *b2b_bar; 1768 vm_size_t bar_size; 1769 uint64_t bar_addr; 1770 enum ntb_bar b2b_bar_num, i; 1771 1772 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1773 b2b_bar = NULL; 1774 b2b_bar_num = NTB_CONFIG_BAR; 1775 ntb->b2b_off = 0; 1776 } else { 1777 b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1778 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1779 ("invalid b2b mw bar")); 1780 1781 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1782 bar_size = b2b_bar->size; 1783 1784 if (ntb_b2b_mw_share != 0 && 1785 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1786 ntb->b2b_off = bar_size >> 1; 1787 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1788 ntb->b2b_off = 0; 1789 } else { 1790 device_printf(ntb->device, 1791 "B2B bar size is too small!\n"); 1792 return (EIO); 1793 } 1794 } 1795 1796 /* 1797 * Reset the secondary bar sizes to match the primary bar sizes. 1798 * (Except, disable or halve the size of the B2B secondary bar.) 1799 */ 1800 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1801 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1802 1803 bar_addr = 0; 1804 if (b2b_bar_num == NTB_CONFIG_BAR) 1805 bar_addr = addr->bar0_addr; 1806 else if (b2b_bar_num == NTB_B2B_BAR_1) 1807 bar_addr = addr->bar2_addr64; 1808 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1809 bar_addr = addr->bar4_addr64; 1810 else if (b2b_bar_num == NTB_B2B_BAR_2) 1811 bar_addr = addr->bar4_addr32; 1812 else if (b2b_bar_num == NTB_B2B_BAR_3) 1813 bar_addr = addr->bar5_addr32; 1814 else 1815 KASSERT(false, ("invalid bar")); 1816 1817 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1818 1819 /* 1820 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1821 * register BAR. The B2B BAR is either disabled above or configured 1822 * half-size. It starts at PBAR xlat + offset. 1823 * 1824 * Also set up incoming BAR limits == base (zero length window). 1825 */ 1826 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1827 b2b_bar_num); 1828 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1829 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1830 NTB_B2B_BAR_2, b2b_bar_num); 1831 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1832 NTB_B2B_BAR_3, b2b_bar_num); 1833 } else 1834 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1835 NTB_B2B_BAR_2, b2b_bar_num); 1836 1837 /* Zero incoming translation addrs */ 1838 ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1839 ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1840 1841 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1842 size_t size, xlatoffset; 1843 1844 switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1845 case NTB_B2B_BAR_1: 1846 size = 8; 1847 xlatoffset = XEON_SBAR2XLAT_OFFSET; 1848 break; 1849 case NTB_B2B_BAR_2: 1850 xlatoffset = XEON_SBAR4XLAT_OFFSET; 1851 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1852 size = 4; 1853 else 1854 size = 8; 1855 break; 1856 case NTB_B2B_BAR_3: 1857 xlatoffset = XEON_SBAR5XLAT_OFFSET; 1858 size = 4; 1859 break; 1860 default: 1861 KASSERT(false, ("Bogus msix mw idx: %u", 1862 ntb->msix_mw_idx)); 1863 return (EINVAL); 1864 } 1865 1866 /* 1867 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1868 * workaround 1869 */ 1870 if (size == 4) { 1871 ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1872 ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1873 } else { 1874 ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1875 ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1876 } 1877 } 1878 (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1879 (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1880 1881 /* Zero outgoing translation limits (whole bar size windows) */ 1882 ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1883 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1884 1885 /* Set outgoing translation offsets */ 1886 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1887 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1888 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1889 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1890 } else 1891 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1892 1893 /* Set the translation offset for B2B registers */ 1894 bar_addr = 0; 1895 if (b2b_bar_num == NTB_CONFIG_BAR) 1896 bar_addr = peer_addr->bar0_addr; 1897 else if (b2b_bar_num == NTB_B2B_BAR_1) 1898 bar_addr = peer_addr->bar2_addr64; 1899 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1900 bar_addr = peer_addr->bar4_addr64; 1901 else if (b2b_bar_num == NTB_B2B_BAR_2) 1902 bar_addr = peer_addr->bar4_addr32; 1903 else if (b2b_bar_num == NTB_B2B_BAR_3) 1904 bar_addr = peer_addr->bar5_addr32; 1905 else 1906 KASSERT(false, ("invalid bar")); 1907 1908 /* 1909 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1910 * at a time. 1911 */ 1912 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1913 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1914 return (0); 1915} 1916 1917static inline bool 1918_xeon_link_is_up(struct ntb_softc *ntb) 1919{ 1920 1921 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1922 return (true); 1923 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1924} 1925 1926static inline bool 1927link_is_up(struct ntb_softc *ntb) 1928{ 1929 1930 if (ntb->type == NTB_XEON) 1931 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1932 !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 1933 1934 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1935 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1936} 1937 1938static inline bool 1939atom_link_is_err(struct ntb_softc *ntb) 1940{ 1941 uint32_t status; 1942 1943 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1944 1945 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1946 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1947 return (true); 1948 1949 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1950 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1951} 1952 1953/* Atom does not have link status interrupt, poll on that platform */ 1954static void 1955atom_link_hb(void *arg) 1956{ 1957 struct ntb_softc *ntb = arg; 1958 sbintime_t timo, poll_ts; 1959 1960 timo = NTB_HB_TIMEOUT * hz; 1961 poll_ts = ntb->last_ts + timo; 1962 1963 /* 1964 * Delay polling the link status if an interrupt was received, unless 1965 * the cached link status says the link is down. 1966 */ 1967 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1968 timo = poll_ts - ticks; 1969 goto out; 1970 } 1971 1972 if (ntb_poll_link(ntb)) 1973 ntb_link_event(ntb->device); 1974 1975 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1976 /* Link is down with error, proceed with recovery */ 1977 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1978 return; 1979 } 1980 1981out: 1982 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1983} 1984 1985static void 1986atom_perform_link_restart(struct ntb_softc *ntb) 1987{ 1988 uint32_t status; 1989 1990 /* Driver resets the NTB ModPhy lanes - magic! */ 1991 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1992 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1993 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1994 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1995 1996 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1997 pause("ModPhy", hz / 10); 1998 1999 /* Clear AER Errors, write to clear */ 2000 status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 2001 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 2002 ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 2003 2004 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 2005 status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 2006 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 2007 ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 2008 2009 /* Clear DeSkew Buffer error, write to clear */ 2010 status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 2011 status |= ATOM_DESKEWSTS_DBERR; 2012 ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 2013 2014 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2015 status &= ATOM_IBIST_ERR_OFLOW; 2016 ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2017 2018 /* Releases the NTB state machine to allow the link to retrain */ 2019 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2020 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2021 ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2022} 2023 2024static int 2025ntb_set_ctx(device_t dev, void *ctx, const struct ntb_ctx_ops *ops) 2026{ 2027 struct ntb_softc *ntb = device_get_softc(dev); 2028 2029 if (ctx == NULL || ops == NULL) 2030 return (EINVAL); 2031 if (ntb->ctx_ops != NULL) 2032 return (EINVAL); 2033 2034 CTX_LOCK(ntb); 2035 if (ntb->ctx_ops != NULL) { 2036 CTX_UNLOCK(ntb); 2037 return (EINVAL); 2038 } 2039 ntb->ntb_ctx = ctx; 2040 ntb->ctx_ops = ops; 2041 CTX_UNLOCK(ntb); 2042 2043 return (0); 2044} 2045 2046/* 2047 * It is expected that this will only be used from contexts where the ctx_lock 2048 * is not needed to protect ntb_ctx lifetime. 2049 */ 2050static void * 2051ntb_get_ctx(device_t dev, const struct ntb_ctx_ops **ops) 2052{ 2053 struct ntb_softc *ntb = device_get_softc(dev); 2054 2055 KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2056 if (ops != NULL) 2057 *ops = ntb->ctx_ops; 2058 return (ntb->ntb_ctx); 2059} 2060 2061static void 2062ntb_clear_ctx(device_t dev) 2063{ 2064 struct ntb_softc *ntb = device_get_softc(dev); 2065 2066 CTX_LOCK(ntb); 2067 ntb->ntb_ctx = NULL; 2068 ntb->ctx_ops = NULL; 2069 CTX_UNLOCK(ntb); 2070} 2071 2072/* 2073 * ntb_link_event() - notify driver context of a change in link status 2074 * @ntb: NTB device context 2075 * 2076 * Notify the driver context that the link status may have changed. The driver 2077 * should call ntb_link_is_up() to get the current status. 2078 */ 2079static void 2080ntb_link_event(device_t dev) 2081{ 2082 struct ntb_softc *ntb = device_get_softc(dev); 2083 2084 CTX_LOCK(ntb); 2085 if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2086 ntb->ctx_ops->link_event(ntb->ntb_ctx); 2087 CTX_UNLOCK(ntb); 2088} 2089 2090/* 2091 * ntb_db_event() - notify driver context of a doorbell event 2092 * @ntb: NTB device context 2093 * @vector: Interrupt vector number 2094 * 2095 * Notify the driver context of a doorbell event. If hardware supports 2096 * multiple interrupt vectors for doorbells, the vector number indicates which 2097 * vector received the interrupt. The vector number is relative to the first 2098 * vector used for doorbells, starting at zero, and must be less than 2099 * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2100 * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2101 * those bits are associated with the vector number. 2102 */ 2103static void 2104ntb_db_event(device_t dev, uint32_t vec) 2105{ 2106 struct ntb_softc *ntb = device_get_softc(dev); 2107 2108 CTX_LOCK(ntb); 2109 if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2110 ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2111 CTX_UNLOCK(ntb); 2112} 2113 2114static int 2115ntb_link_enable(device_t dev, enum ntb_speed speed __unused, 2116 enum ntb_width width __unused) 2117{ 2118 struct ntb_softc *ntb = device_get_softc(dev); 2119 uint32_t cntl; 2120 2121 ntb_printf(2, "%s\n", __func__); 2122 2123 if (ntb->type == NTB_ATOM) { 2124 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2125 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2126 return (0); 2127 } 2128 2129 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2130 ntb_link_event(dev); 2131 return (0); 2132 } 2133 2134 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2135 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2136 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2137 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2138 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2139 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2140 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2141 return (0); 2142} 2143 2144static int 2145ntb_link_disable(device_t dev) 2146{ 2147 struct ntb_softc *ntb = device_get_softc(dev); 2148 uint32_t cntl; 2149 2150 ntb_printf(2, "%s\n", __func__); 2151 2152 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2153 ntb_link_event(dev); 2154 return (0); 2155 } 2156 2157 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2158 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2159 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2160 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2161 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2162 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2163 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2164 return (0); 2165} 2166 2167static bool 2168ntb_link_enabled(device_t dev) 2169{ 2170 struct ntb_softc *ntb = device_get_softc(dev); 2171 uint32_t cntl; 2172 2173 if (ntb->type == NTB_ATOM) { 2174 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2175 return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2176 } 2177 2178 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2179 return (true); 2180 2181 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2182 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2183} 2184 2185static void 2186recover_atom_link(void *arg) 2187{ 2188 struct ntb_softc *ntb = arg; 2189 unsigned speed, width, oldspeed, oldwidth; 2190 uint32_t status32; 2191 2192 atom_perform_link_restart(ntb); 2193 2194 /* 2195 * There is a potential race between the 2 NTB devices recovering at 2196 * the same time. If the times are the same, the link will not recover 2197 * and the driver will be stuck in this loop forever. Add a random 2198 * interval to the recovery time to prevent this race. 2199 */ 2200 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2201 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2202 2203 if (atom_link_is_err(ntb)) 2204 goto retry; 2205 2206 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2207 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2208 goto out; 2209 2210 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2211 width = NTB_LNK_STA_WIDTH(status32); 2212 speed = status32 & NTB_LINK_SPEED_MASK; 2213 2214 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2215 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2216 if (oldwidth != width || oldspeed != speed) 2217 goto retry; 2218 2219out: 2220 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2221 ntb); 2222 return; 2223 2224retry: 2225 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2226 ntb); 2227} 2228 2229/* 2230 * Polls the HW link status register(s); returns true if something has changed. 2231 */ 2232static bool 2233ntb_poll_link(struct ntb_softc *ntb) 2234{ 2235 uint32_t ntb_cntl; 2236 uint16_t reg_val; 2237 2238 if (ntb->type == NTB_ATOM) { 2239 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2240 if (ntb_cntl == ntb->ntb_ctl) 2241 return (false); 2242 2243 ntb->ntb_ctl = ntb_cntl; 2244 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2245 } else { 2246 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2247 2248 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2249 if (reg_val == ntb->lnk_sta) 2250 return (false); 2251 2252 ntb->lnk_sta = reg_val; 2253 2254 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2255 if (_xeon_link_is_up(ntb)) { 2256 if (!ntb->peer_msix_good) { 2257 callout_reset(&ntb->peer_msix_work, 0, 2258 ntb_exchange_msix, ntb); 2259 return (false); 2260 } 2261 } else { 2262 ntb->peer_msix_good = false; 2263 ntb->peer_msix_done = false; 2264 } 2265 } 2266 } 2267 return (true); 2268} 2269 2270static inline enum ntb_speed 2271ntb_link_sta_speed(struct ntb_softc *ntb) 2272{ 2273 2274 if (!link_is_up(ntb)) 2275 return (NTB_SPEED_NONE); 2276 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2277} 2278 2279static inline enum ntb_width 2280ntb_link_sta_width(struct ntb_softc *ntb) 2281{ 2282 2283 if (!link_is_up(ntb)) 2284 return (NTB_WIDTH_NONE); 2285 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2286} 2287 2288SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2289 "Driver state, statistics, and HW registers"); 2290 2291#define NTB_REGSZ_MASK (3ul << 30) 2292#define NTB_REG_64 (1ul << 30) 2293#define NTB_REG_32 (2ul << 30) 2294#define NTB_REG_16 (3ul << 30) 2295#define NTB_REG_8 (0ul << 30) 2296 2297#define NTB_DB_READ (1ul << 29) 2298#define NTB_PCI_REG (1ul << 28) 2299#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2300 2301static void 2302ntb_sysctl_init(struct ntb_softc *ntb) 2303{ 2304 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2305 struct sysctl_ctx_list *ctx; 2306 struct sysctl_oid *tree, *tmptree; 2307 2308 ctx = device_get_sysctl_ctx(ntb->device); 2309 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2310 2311 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2312 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2313 sysctl_handle_link_status_human, "A", 2314 "Link status (human readable)"); 2315 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2316 CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2317 "IU", "Link status (1=active, 0=inactive)"); 2318 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2319 CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2320 "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2321 2322 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2323 CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2324 tree_par = SYSCTL_CHILDREN(tree); 2325 2326 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2327 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2328 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2329 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2330 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2331 &ntb->ppd, 0, "Raw PPD register (cached)"); 2332 2333 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2334#ifdef notyet 2335 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2336 &ntb->b2b_mw_idx, 0, 2337 "Index of the MW used for B2B remote register access"); 2338#endif 2339 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2340 CTLFLAG_RD, &ntb->b2b_off, 2341 "If non-zero, offset of B2B register region in shared MW"); 2342 } 2343 2344 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2345 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2346 "Features/errata of this NTB device"); 2347 2348 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2349 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2350 "NTB CTL register (cached)"); 2351 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2352 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2353 "LNK STA register (cached)"); 2354 2355#ifdef notyet 2356 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2357 &ntb->mw_count, 0, "MW count"); 2358 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2359 &ntb->spad_count, 0, "Scratchpad count"); 2360 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2361 &ntb->db_count, 0, "Doorbell count"); 2362 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2363 &ntb->db_vec_count, 0, "Doorbell vector count"); 2364 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2365 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2366#endif 2367 2368 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2369 &ntb->db_valid_mask, "Doorbell valid mask"); 2370 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2371 &ntb->db_link_mask, "Doorbell link mask"); 2372 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2373 &ntb->db_mask, "Doorbell mask (cached)"); 2374 2375 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2376 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2377 regpar = SYSCTL_CHILDREN(tmptree); 2378 2379 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2380 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2381 ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2382 "NTB Control register"); 2383 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2384 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2385 0x19c, sysctl_handle_register, "IU", 2386 "NTB Link Capabilities"); 2387 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2388 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2389 0x1a0, sysctl_handle_register, "IU", 2390 "NTB Link Control register"); 2391 2392 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2393 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2394 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2395 sysctl_handle_register, "QU", "Doorbell mask register"); 2396 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2397 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2398 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2399 sysctl_handle_register, "QU", "Doorbell register"); 2400 2401 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2402 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2403 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2404 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2405 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2406 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2407 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2408 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2409 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2410 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2411 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2412 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2413 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2414 } else { 2415 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2416 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2417 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2418 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2419 } 2420 2421 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2422 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2423 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2424 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2425 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2426 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2427 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2428 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2429 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2430 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2431 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2432 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2433 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2434 } else { 2435 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2436 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2437 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2438 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2439 } 2440 2441 if (ntb->type == NTB_ATOM) 2442 return; 2443 2444 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2445 CTLFLAG_RD, NULL, "Xeon HW statistics"); 2446 statpar = SYSCTL_CHILDREN(tmptree); 2447 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2448 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2449 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2450 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2451 2452 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2453 CTLFLAG_RD, NULL, "Xeon HW errors"); 2454 errpar = SYSCTL_CHILDREN(tmptree); 2455 2456 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2457 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2458 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2459 sysctl_handle_register, "CU", "PPD"); 2460 2461 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2462 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2463 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2464 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2465 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2466 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2467 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2468 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2469 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2470 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2471 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2472 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2473 2474 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2475 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2476 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2477 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2478 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2479 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2480 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2481 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2482 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2483 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2484 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2485 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2486 2487 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2488 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2489 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2490 sysctl_handle_register, "SU", "DEVSTS"); 2491 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2492 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2493 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2494 sysctl_handle_register, "SU", "LNKSTS"); 2495 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2496 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2497 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2498 sysctl_handle_register, "SU", "SLNKSTS"); 2499 2500 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2501 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2502 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2503 sysctl_handle_register, "IU", "UNCERRSTS"); 2504 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2505 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2506 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2507 sysctl_handle_register, "IU", "CORERRSTS"); 2508 2509 if (ntb->conn_type != NTB_CONN_B2B) 2510 return; 2511 2512 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2513 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2514 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2515 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2516 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2517 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2518 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2519 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2520 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2521 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2522 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2523 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2524 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2525 } else { 2526 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2527 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2528 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2529 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2530 } 2531 2532 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2533 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2534 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2535 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2536 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2537 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2538 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2539 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2540 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2541 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2542 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2543 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2544 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2545 } else { 2546 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2547 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2548 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2549 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2550 } 2551 2552 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2553 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2554 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2555 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2556 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2557 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2558 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2559 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2560 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2561 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2562 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2563 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2564 sysctl_handle_register, "IU", 2565 "Secondary BAR4 base register"); 2566 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2567 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2568 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2569 sysctl_handle_register, "IU", 2570 "Secondary BAR5 base register"); 2571 } else { 2572 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2573 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2574 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2575 sysctl_handle_register, "QU", 2576 "Secondary BAR45 base register"); 2577 } 2578} 2579 2580static int 2581sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2582{ 2583 struct ntb_softc *ntb = arg1; 2584 struct sbuf sb; 2585 int error; 2586 2587 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2588 2589 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2590 error = sbuf_finish(&sb); 2591 sbuf_delete(&sb); 2592 2593 if (error || !req->newptr) 2594 return (error); 2595 return (EINVAL); 2596} 2597 2598static int 2599sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2600{ 2601 struct ntb_softc *ntb = arg1; 2602 unsigned old, new; 2603 int error; 2604 2605 old = ntb_link_enabled(ntb->device); 2606 2607 error = SYSCTL_OUT(req, &old, sizeof(old)); 2608 if (error != 0 || req->newptr == NULL) 2609 return (error); 2610 2611 error = SYSCTL_IN(req, &new, sizeof(new)); 2612 if (error != 0) 2613 return (error); 2614 2615 ntb_printf(0, "Admin set interface state to '%sabled'\n", 2616 (new != 0)? "en" : "dis"); 2617 2618 if (new != 0) 2619 error = ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2620 else 2621 error = ntb_link_disable(ntb->device); 2622 return (error); 2623} 2624 2625static int 2626sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2627{ 2628 struct ntb_softc *ntb = arg1; 2629 struct sbuf sb; 2630 enum ntb_speed speed; 2631 enum ntb_width width; 2632 int error; 2633 2634 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2635 2636 if (ntb_link_is_up(ntb->device, &speed, &width)) 2637 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2638 (unsigned)speed, (unsigned)width); 2639 else 2640 sbuf_printf(&sb, "down"); 2641 2642 error = sbuf_finish(&sb); 2643 sbuf_delete(&sb); 2644 2645 if (error || !req->newptr) 2646 return (error); 2647 return (EINVAL); 2648} 2649 2650static int 2651sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2652{ 2653 struct ntb_softc *ntb = arg1; 2654 unsigned res; 2655 int error; 2656 2657 res = ntb_link_is_up(ntb->device, NULL, NULL); 2658 2659 error = SYSCTL_OUT(req, &res, sizeof(res)); 2660 if (error || !req->newptr) 2661 return (error); 2662 return (EINVAL); 2663} 2664 2665static int 2666sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2667{ 2668 struct ntb_softc *ntb; 2669 const void *outp; 2670 uintptr_t sz; 2671 uint64_t umv; 2672 char be[sizeof(umv)]; 2673 size_t outsz; 2674 uint32_t reg; 2675 bool db, pci; 2676 int error; 2677 2678 ntb = arg1; 2679 reg = arg2 & ~NTB_REGFLAGS_MASK; 2680 sz = arg2 & NTB_REGSZ_MASK; 2681 db = (arg2 & NTB_DB_READ) != 0; 2682 pci = (arg2 & NTB_PCI_REG) != 0; 2683 2684 KASSERT(!(db && pci), ("bogus")); 2685 2686 if (db) { 2687 KASSERT(sz == NTB_REG_64, ("bogus")); 2688 umv = db_ioread(ntb, reg); 2689 outsz = sizeof(uint64_t); 2690 } else { 2691 switch (sz) { 2692 case NTB_REG_64: 2693 if (pci) 2694 umv = pci_read_config(ntb->device, reg, 8); 2695 else 2696 umv = ntb_reg_read(8, reg); 2697 outsz = sizeof(uint64_t); 2698 break; 2699 case NTB_REG_32: 2700 if (pci) 2701 umv = pci_read_config(ntb->device, reg, 4); 2702 else 2703 umv = ntb_reg_read(4, reg); 2704 outsz = sizeof(uint32_t); 2705 break; 2706 case NTB_REG_16: 2707 if (pci) 2708 umv = pci_read_config(ntb->device, reg, 2); 2709 else 2710 umv = ntb_reg_read(2, reg); 2711 outsz = sizeof(uint16_t); 2712 break; 2713 case NTB_REG_8: 2714 if (pci) 2715 umv = pci_read_config(ntb->device, reg, 1); 2716 else 2717 umv = ntb_reg_read(1, reg); 2718 outsz = sizeof(uint8_t); 2719 break; 2720 default: 2721 panic("bogus"); 2722 break; 2723 } 2724 } 2725 2726 /* Encode bigendian so that sysctl -x is legible. */ 2727 be64enc(be, umv); 2728 outp = ((char *)be) + sizeof(umv) - outsz; 2729 2730 error = SYSCTL_OUT(req, outp, outsz); 2731 if (error || !req->newptr) 2732 return (error); 2733 return (EINVAL); 2734} 2735 2736static unsigned 2737ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2738{ 2739 2740 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2741 uidx >= ntb->b2b_mw_idx) || 2742 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2743 uidx++; 2744 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2745 uidx >= ntb->b2b_mw_idx) && 2746 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2747 uidx++; 2748 return (uidx); 2749} 2750 2751static void 2752ntb_exchange_msix(void *ctx) 2753{ 2754 struct ntb_softc *ntb; 2755 uint32_t val; 2756 unsigned i; 2757 2758 ntb = ctx; 2759 2760 if (ntb->peer_msix_good) 2761 goto msix_good; 2762 if (ntb->peer_msix_done) 2763 goto msix_done; 2764 2765 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2766 ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 2767 ntb->msix_data[i].nmd_data); 2768 ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 2769 ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2770 } 2771 ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2772 2773 ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 2774 if (val != NTB_MSIX_VER_GUARD) 2775 goto reschedule; 2776 2777 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2778 ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 2779 ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2780 ntb->peer_msix_data[i].nmd_data = val; 2781 ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 2782 ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2783 ntb->peer_msix_data[i].nmd_ofs = val; 2784 } 2785 2786 ntb->peer_msix_done = true; 2787 2788msix_done: 2789 ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2790 ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 2791 if (val != NTB_MSIX_RECEIVED) 2792 goto reschedule; 2793 2794 ntb->peer_msix_good = true; 2795 /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2796 goto reschedule; 2797 2798msix_good: 2799 ntb_poll_link(ntb); 2800 ntb_link_event(ntb->device); 2801 return; 2802 2803reschedule: 2804 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2805 if (_xeon_link_is_up(ntb)) { 2806 callout_reset(&ntb->peer_msix_work, 2807 hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2808 ntb_exchange_msix, ntb); 2809 } else 2810 ntb_spad_clear(ntb->device); 2811} 2812 2813/* 2814 * Public API to the rest of the OS 2815 */ 2816 2817static uint8_t 2818ntb_spad_count(device_t dev) 2819{ 2820 struct ntb_softc *ntb = device_get_softc(dev); 2821 2822 return (ntb->spad_count); 2823} 2824 2825static uint8_t 2826ntb_mw_count(device_t dev) 2827{ 2828 struct ntb_softc *ntb = device_get_softc(dev); 2829 uint8_t res; 2830 2831 res = ntb->mw_count; 2832 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2833 res--; 2834 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2835 res--; 2836 return (res); 2837} 2838 2839static int 2840ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 2841{ 2842 struct ntb_softc *ntb = device_get_softc(dev); 2843 2844 if (idx >= ntb->spad_count) 2845 return (EINVAL); 2846 2847 ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2848 2849 return (0); 2850} 2851 2852/* 2853 * Zeros the local scratchpad. 2854 */ 2855static void 2856ntb_spad_clear(device_t dev) 2857{ 2858 struct ntb_softc *ntb = device_get_softc(dev); 2859 unsigned i; 2860 2861 for (i = 0; i < ntb->spad_count; i++) 2862 ntb_spad_write(dev, i, 0); 2863} 2864 2865static int 2866ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2867{ 2868 struct ntb_softc *ntb = device_get_softc(dev); 2869 2870 if (idx >= ntb->spad_count) 2871 return (EINVAL); 2872 2873 *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2874 2875 return (0); 2876} 2877 2878static int 2879ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 2880{ 2881 struct ntb_softc *ntb = device_get_softc(dev); 2882 2883 if (idx >= ntb->spad_count) 2884 return (EINVAL); 2885 2886 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2887 ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2888 else 2889 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2890 2891 return (0); 2892} 2893 2894static int 2895ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2896{ 2897 struct ntb_softc *ntb = device_get_softc(dev); 2898 2899 if (idx >= ntb->spad_count) 2900 return (EINVAL); 2901 2902 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2903 *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2904 else 2905 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2906 2907 return (0); 2908} 2909 2910static int 2911ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 2912 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2913 bus_addr_t *plimit) 2914{ 2915 struct ntb_softc *ntb = device_get_softc(dev); 2916 struct ntb_pci_bar_info *bar; 2917 bus_addr_t limit; 2918 size_t bar_b2b_off; 2919 enum ntb_bar bar_num; 2920 2921 if (mw_idx >= ntb_mw_count(dev)) 2922 return (EINVAL); 2923 mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 2924 2925 bar_num = ntb_mw_to_bar(ntb, mw_idx); 2926 bar = &ntb->bar_info[bar_num]; 2927 bar_b2b_off = 0; 2928 if (mw_idx == ntb->b2b_mw_idx) { 2929 KASSERT(ntb->b2b_off != 0, 2930 ("user shouldn't get non-shared b2b mw")); 2931 bar_b2b_off = ntb->b2b_off; 2932 } 2933 2934 if (bar_is_64bit(ntb, bar_num)) 2935 limit = BUS_SPACE_MAXADDR; 2936 else 2937 limit = BUS_SPACE_MAXADDR_32BIT; 2938 2939 if (base != NULL) 2940 *base = bar->pbase + bar_b2b_off; 2941 if (vbase != NULL) 2942 *vbase = bar->vbase + bar_b2b_off; 2943 if (size != NULL) 2944 *size = bar->size - bar_b2b_off; 2945 if (align != NULL) 2946 *align = bar->size; 2947 if (align_size != NULL) 2948 *align_size = 1; 2949 if (plimit != NULL) 2950 *plimit = limit; 2951 return (0); 2952} 2953 2954static int 2955ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 2956{ 2957 struct ntb_softc *ntb = device_get_softc(dev); 2958 struct ntb_pci_bar_info *bar; 2959 uint64_t base, limit, reg_val; 2960 size_t bar_size, mw_size; 2961 uint32_t base_reg, xlat_reg, limit_reg; 2962 enum ntb_bar bar_num; 2963 2964 if (idx >= ntb_mw_count(dev)) 2965 return (EINVAL); 2966 idx = ntb_user_mw_to_idx(ntb, idx); 2967 2968 bar_num = ntb_mw_to_bar(ntb, idx); 2969 bar = &ntb->bar_info[bar_num]; 2970 2971 bar_size = bar->size; 2972 if (idx == ntb->b2b_mw_idx) 2973 mw_size = bar_size - ntb->b2b_off; 2974 else 2975 mw_size = bar_size; 2976 2977 /* Hardware requires that addr is aligned to bar size */ 2978 if ((addr & (bar_size - 1)) != 0) 2979 return (EINVAL); 2980 2981 if (size > mw_size) 2982 return (EINVAL); 2983 2984 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2985 2986 limit = 0; 2987 if (bar_is_64bit(ntb, bar_num)) { 2988 base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2989 2990 if (limit_reg != 0 && size != mw_size) 2991 limit = base + size; 2992 2993 /* Set and verify translation address */ 2994 ntb_reg_write(8, xlat_reg, addr); 2995 reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2996 if (reg_val != addr) { 2997 ntb_reg_write(8, xlat_reg, 0); 2998 return (EIO); 2999 } 3000 3001 /* Set and verify the limit */ 3002 ntb_reg_write(8, limit_reg, limit); 3003 reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3004 if (reg_val != limit) { 3005 ntb_reg_write(8, limit_reg, base); 3006 ntb_reg_write(8, xlat_reg, 0); 3007 return (EIO); 3008 } 3009 } else { 3010 /* Configure 32-bit (split) BAR MW */ 3011 3012 if ((addr & UINT32_MAX) != addr) 3013 return (ERANGE); 3014 if (((addr + size) & UINT32_MAX) != (addr + size)) 3015 return (ERANGE); 3016 3017 base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3018 3019 if (limit_reg != 0 && size != mw_size) 3020 limit = base + size; 3021 3022 /* Set and verify translation address */ 3023 ntb_reg_write(4, xlat_reg, addr); 3024 reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3025 if (reg_val != addr) { 3026 ntb_reg_write(4, xlat_reg, 0); 3027 return (EIO); 3028 } 3029 3030 /* Set and verify the limit */ 3031 ntb_reg_write(4, limit_reg, limit); 3032 reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3033 if (reg_val != limit) { 3034 ntb_reg_write(4, limit_reg, base); 3035 ntb_reg_write(4, xlat_reg, 0); 3036 return (EIO); 3037 } 3038 } 3039 return (0); 3040} 3041 3042static int 3043ntb_mw_clear_trans(device_t dev, unsigned mw_idx) 3044{ 3045 3046 return (ntb_mw_set_trans(dev, mw_idx, 0, 0)); 3047} 3048 3049static int 3050ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 3051{ 3052 struct ntb_softc *ntb = device_get_softc(dev); 3053 struct ntb_pci_bar_info *bar; 3054 3055 if (idx >= ntb_mw_count(dev)) 3056 return (EINVAL); 3057 idx = ntb_user_mw_to_idx(ntb, idx); 3058 3059 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3060 *mode = bar->map_mode; 3061 return (0); 3062} 3063 3064static int 3065ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 3066{ 3067 struct ntb_softc *ntb = device_get_softc(dev); 3068 3069 if (idx >= ntb_mw_count(dev)) 3070 return (EINVAL); 3071 3072 idx = ntb_user_mw_to_idx(ntb, idx); 3073 return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3074} 3075 3076static int 3077ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3078{ 3079 struct ntb_pci_bar_info *bar; 3080 int rc; 3081 3082 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3083 if (bar->map_mode == mode) 3084 return (0); 3085 3086 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3087 if (rc == 0) 3088 bar->map_mode = mode; 3089 3090 return (rc); 3091} 3092 3093static void 3094ntb_peer_db_set(device_t dev, uint64_t bit) 3095{ 3096 struct ntb_softc *ntb = device_get_softc(dev); 3097 3098 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 3099 struct ntb_pci_bar_info *lapic; 3100 unsigned i; 3101 3102 lapic = ntb->peer_lapic_bar; 3103 3104 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3105 if ((bit & ntb_db_vector_mask(dev, i)) != 0) 3106 bus_space_write_4(lapic->pci_bus_tag, 3107 lapic->pci_bus_handle, 3108 ntb->peer_msix_data[i].nmd_ofs, 3109 ntb->peer_msix_data[i].nmd_data); 3110 } 3111 return; 3112 } 3113 3114 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3115 ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3116 return; 3117 } 3118 3119 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3120} 3121 3122static int 3123ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3124{ 3125 struct ntb_softc *ntb = device_get_softc(dev); 3126 struct ntb_pci_bar_info *bar; 3127 uint64_t regoff; 3128 3129 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3130 3131 if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3132 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3133 regoff = ntb->peer_reg->db_bell; 3134 } else { 3135 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3136 ("invalid b2b idx")); 3137 3138 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3139 regoff = XEON_PDOORBELL_OFFSET; 3140 } 3141 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3142 3143 /* HACK: Specific to current x86 bus implementation. */ 3144 *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3145 *db_size = ntb->reg->db_size; 3146 return (0); 3147} 3148 3149static uint64_t 3150ntb_db_valid_mask(device_t dev) 3151{ 3152 struct ntb_softc *ntb = device_get_softc(dev); 3153 3154 return (ntb->db_valid_mask); 3155} 3156 3157static int 3158ntb_db_vector_count(device_t dev) 3159{ 3160 struct ntb_softc *ntb = device_get_softc(dev); 3161 3162 return (ntb->db_vec_count); 3163} 3164 3165static uint64_t 3166ntb_db_vector_mask(device_t dev, uint32_t vector) 3167{ 3168 struct ntb_softc *ntb = device_get_softc(dev); 3169 3170 if (vector > ntb->db_vec_count) 3171 return (0); 3172 return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3173} 3174 3175static bool 3176ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3177{ 3178 struct ntb_softc *ntb = device_get_softc(dev); 3179 3180 if (speed != NULL) 3181 *speed = ntb_link_sta_speed(ntb); 3182 if (width != NULL) 3183 *width = ntb_link_sta_width(ntb); 3184 return (link_is_up(ntb)); 3185} 3186 3187static void 3188save_bar_parameters(struct ntb_pci_bar_info *bar) 3189{ 3190 3191 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3192 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3193 bar->pbase = rman_get_start(bar->pci_resource); 3194 bar->size = rman_get_size(bar->pci_resource); 3195 bar->vbase = rman_get_virtual(bar->pci_resource); 3196} 3197 3198static device_method_t ntb_intel_methods[] = { 3199 /* Device interface */ 3200 DEVMETHOD(device_probe, ntb_probe), 3201 DEVMETHOD(device_attach, ntb_attach), 3202 DEVMETHOD(device_detach, ntb_detach), 3203 /* NTB interface */ 3204 DEVMETHOD(ntb_link_is_up, ntb_link_is_up), 3205 DEVMETHOD(ntb_link_enable, ntb_link_enable), 3206 DEVMETHOD(ntb_link_disable, ntb_link_disable), 3207 DEVMETHOD(ntb_link_enabled, ntb_link_enabled), 3208 DEVMETHOD(ntb_set_ctx, ntb_set_ctx), 3209 DEVMETHOD(ntb_get_ctx, ntb_get_ctx), 3210 DEVMETHOD(ntb_clear_ctx, ntb_clear_ctx), 3211 DEVMETHOD(ntb_mw_count, ntb_mw_count), 3212 DEVMETHOD(ntb_mw_get_range, ntb_mw_get_range), 3213 DEVMETHOD(ntb_mw_set_trans, ntb_mw_set_trans), 3214 DEVMETHOD(ntb_mw_clear_trans, ntb_mw_clear_trans), 3215 DEVMETHOD(ntb_mw_get_wc, ntb_mw_get_wc), 3216 DEVMETHOD(ntb_mw_set_wc, ntb_mw_set_wc), 3217 DEVMETHOD(ntb_spad_count, ntb_spad_count), 3218 DEVMETHOD(ntb_spad_clear, ntb_spad_clear), 3219 DEVMETHOD(ntb_spad_write, ntb_spad_write), 3220 DEVMETHOD(ntb_spad_read, ntb_spad_read), 3221 DEVMETHOD(ntb_peer_spad_write, ntb_peer_spad_write), 3222 DEVMETHOD(ntb_peer_spad_read, ntb_peer_spad_read), 3223 DEVMETHOD(ntb_db_valid_mask, ntb_db_valid_mask), 3224 DEVMETHOD(ntb_db_vector_count, ntb_db_vector_count), 3225 DEVMETHOD(ntb_db_vector_mask, ntb_db_vector_mask), 3226 DEVMETHOD(ntb_db_clear, ntb_db_clear), 3227 DEVMETHOD(ntb_db_clear_mask, ntb_db_clear_mask), 3228 DEVMETHOD(ntb_db_read, ntb_db_read), 3229 DEVMETHOD(ntb_db_set_mask, ntb_db_set_mask), 3230 DEVMETHOD(ntb_peer_db_addr, ntb_peer_db_addr), 3231 DEVMETHOD(ntb_peer_db_set, ntb_peer_db_set), 3232 DEVMETHOD_END 3233}; 3234 3235static DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3236 sizeof(struct ntb_softc)); 3237DRIVER_MODULE(ntb_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL); 3238MODULE_DEPEND(ntb_intel, ntb, 1, 1, 1); 3239MODULE_VERSION(ntb_intel, 1); 3240