ntb_hw_intel.c revision 289541
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3250079Scarl * All rights reserved. 4250079Scarl * 5250079Scarl * Redistribution and use in source and binary forms, with or without 6250079Scarl * modification, are permitted provided that the following conditions 7250079Scarl * are met: 8250079Scarl * 1. Redistributions of source code must retain the above copyright 9250079Scarl * notice, this list of conditions and the following disclaimer. 10250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer in the 12250079Scarl * documentation and/or other materials provided with the distribution. 13250079Scarl * 14250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24250079Scarl * SUCH DAMAGE. 25250079Scarl */ 26250079Scarl 27250079Scarl#include <sys/cdefs.h> 28250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289541 2015-10-18 20:20:11Z cem $"); 29250079Scarl 30250079Scarl#include <sys/param.h> 31250079Scarl#include <sys/kernel.h> 32250079Scarl#include <sys/systm.h> 33250079Scarl#include <sys/bus.h> 34250079Scarl#include <sys/malloc.h> 35250079Scarl#include <sys/module.h> 36250079Scarl#include <sys/queue.h> 37250079Scarl#include <sys/rman.h> 38289207Scem#include <sys/sysctl.h> 39250079Scarl#include <vm/vm.h> 40250079Scarl#include <vm/pmap.h> 41250079Scarl#include <machine/bus.h> 42250079Scarl#include <machine/pmap.h> 43250079Scarl#include <machine/resource.h> 44250079Scarl#include <dev/pci/pcireg.h> 45250079Scarl#include <dev/pci/pcivar.h> 46250079Scarl 47250079Scarl#include "ntb_regs.h" 48250079Scarl#include "ntb_hw.h" 49250079Scarl 50250079Scarl/* 51250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52250079Scarl * allows you to connect two systems using a PCI-e link. 53250079Scarl * 54250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 55250079Scarl * you to send and recieve interrupts, map the memory windows and send and 56250079Scarl * receive messages in the scratch-pad registers. 57250079Scarl * 58250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 59250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 60250079Scarl */ 61250079Scarl 62289538Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT) 63250079Scarl 64289539Scem#define NTB_HB_TIMEOUT 1 /* second */ 65289539Scem#define SOC_LINK_RECOVERY_TIME 500 /* ms */ 66250079Scarl 67250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 68250079Scarl 69250079Scarlenum ntb_device_type { 70250079Scarl NTB_XEON, 71250079Scarl NTB_SOC 72250079Scarl}; 73250079Scarl 74289539Scemenum ntb_bar { 75289539Scem NTB_CONFIG_BAR = 0, 76289539Scem NTB_B2B_BAR_1, 77289539Scem NTB_B2B_BAR_2, 78289539Scem NTB_B2B_BAR_3, 79289539Scem NTB_MAX_BARS 80289539Scem}; 81289539Scem 82255274Scarl/* Device features and workarounds */ 83255274Scarl#define HAS_FEATURE(feature) \ 84255274Scarl ((ntb->features & (feature)) != 0) 85255274Scarl 86250079Scarlstruct ntb_hw_info { 87250079Scarl uint32_t device_id; 88255274Scarl const char *desc; 89250079Scarl enum ntb_device_type type; 90289397Scem uint32_t features; 91250079Scarl}; 92250079Scarl 93250079Scarlstruct ntb_pci_bar_info { 94250079Scarl bus_space_tag_t pci_bus_tag; 95250079Scarl bus_space_handle_t pci_bus_handle; 96250079Scarl int pci_resource_id; 97250079Scarl struct resource *pci_resource; 98250079Scarl vm_paddr_t pbase; 99250079Scarl void *vbase; 100250079Scarl u_long size; 101250079Scarl}; 102250079Scarl 103250079Scarlstruct ntb_int_info { 104250079Scarl struct resource *res; 105250079Scarl int rid; 106250079Scarl void *tag; 107250079Scarl}; 108250079Scarl 109250079Scarlstruct ntb_db_cb { 110250079Scarl ntb_db_callback callback; 111250079Scarl unsigned int db_num; 112250079Scarl void *data; 113250079Scarl struct ntb_softc *ntb; 114289281Scem struct callout irq_work; 115289343Scem bool reserved; 116250079Scarl}; 117250079Scarl 118250079Scarlstruct ntb_softc { 119250079Scarl device_t device; 120250079Scarl enum ntb_device_type type; 121255274Scarl uint64_t features; 122250079Scarl 123250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 124250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 125250079Scarl uint32_t allocated_interrupts; 126250079Scarl 127250079Scarl struct callout heartbeat_timer; 128250079Scarl struct callout lr_timer; 129250079Scarl 130250079Scarl void *ntb_transport; 131250079Scarl ntb_event_callback event_cb; 132289396Scem struct ntb_db_cb *db_cb; 133289396Scem uint8_t max_cbs; 134250079Scarl 135250079Scarl struct { 136289255Scem uint32_t ldb; 137289255Scem uint32_t ldb_mask; 138289255Scem uint32_t rdb; 139289255Scem uint32_t bar2_xlat; 140289255Scem uint32_t bar4_xlat; 141289397Scem uint32_t bar5_xlat; 142250079Scarl uint32_t spad_remote; 143250079Scarl uint32_t spad_local; 144250079Scarl uint32_t lnk_cntl; 145250079Scarl uint32_t lnk_stat; 146250079Scarl uint32_t spci_cmd; 147250079Scarl } reg_ofs; 148289348Scem uint32_t ppd; 149250079Scarl uint8_t conn_type; 150250079Scarl uint8_t dev_type; 151250079Scarl uint8_t link_status; 152250079Scarl uint8_t link_width; 153250079Scarl uint8_t link_speed; 154289539Scem 155289539Scem uint8_t mw_count; 156289539Scem uint8_t spad_count; 157289539Scem uint8_t db_count; 158289539Scem uint8_t db_vec_count; 159289539Scem uint8_t db_vec_shift; 160250079Scarl}; 161250079Scarl 162289234Scem#ifdef __i386__ 163289234Scemstatic __inline uint64_t 164289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 165289234Scem bus_size_t offset) 166289234Scem{ 167289234Scem 168289234Scem return (bus_space_read_4(tag, handle, offset) | 169289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 170289234Scem} 171289234Scem 172289234Scemstatic __inline void 173289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 174289234Scem bus_size_t offset, uint64_t val) 175289234Scem{ 176289234Scem 177289234Scem bus_space_write_4(tag, handle, offset, val); 178289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 179289234Scem} 180289234Scem#endif 181289234Scem 182255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 183255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 184255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 185255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 186255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 187255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 188255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 189250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 190255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 191289397Scem#define ntb_mw_read(SIZE, offset) \ 192289539Scem ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), offset) 193255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 194289539Scem ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), \ 195289397Scem offset, val) 196250079Scarl 197250079Scarlstatic int ntb_probe(device_t device); 198250079Scarlstatic int ntb_attach(device_t device); 199250079Scarlstatic int ntb_detach(device_t device); 200289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 201255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 202289541Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *); 203255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 204255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 205255272Scarl struct ntb_pci_bar_info *bar); 206250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 207289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 208289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb); 209289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 210289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 211250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 212289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 213289540Scemstatic void handle_irq(void *arg); 214250079Scarlstatic void ntb_handle_legacy_interrupt(void *arg); 215289281Scemstatic void ntb_irq_work(void *arg); 216289539Scemstatic inline uint64_t ntb_db_read(struct ntb_softc *, uint64_t regoff); 217289539Scemstatic inline void ntb_db_write(struct ntb_softc *, uint64_t regoff, uint64_t val); 218289539Scemstatic inline void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 219289539Scemstatic inline void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 220289342Scemstatic int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors); 221250079Scarlstatic void ntb_free_callbacks(struct ntb_softc *ntb); 222250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 223289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb); 224289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb); 225289348Scemstatic int ntb_detect_soc(struct ntb_softc *ntb); 226250079Scarlstatic int ntb_setup_xeon(struct ntb_softc *ntb); 227250079Scarlstatic int ntb_setup_soc(struct ntb_softc *ntb); 228289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 229255279Scarlstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 230255279Scarlstatic void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 231250079Scarlstatic void ntb_handle_heartbeat(void *arg); 232250079Scarlstatic void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 233289272Scemstatic void ntb_hw_link_down(struct ntb_softc *ntb); 234289272Scemstatic void ntb_hw_link_up(struct ntb_softc *ntb); 235250079Scarlstatic void recover_soc_link(void *arg); 236250079Scarlstatic int ntb_check_link_status(struct ntb_softc *ntb); 237255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 238250079Scarl 239250079Scarlstatic struct ntb_hw_info pci_ids[] = { 240255274Scarl { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 241289233Scem 242289233Scem /* XXX: PS/SS IDs left out until they are supported. */ 243289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 244289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 245289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 246289538Scem NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 247289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 248289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 249289538Scem NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 250289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 251289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 252289538Scem NTB_SB01BASE_LOCKUP }, 253289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 254289538Scem NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 255289538Scem NTB_SB01BASE_LOCKUP }, 256289233Scem 257255274Scarl { 0x00000000, NULL, NTB_SOC, 0 } 258250079Scarl}; 259250079Scarl 260250079Scarl/* 261250079Scarl * OS <-> Driver interface structures 262250079Scarl */ 263250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 264250079Scarl 265250079Scarlstatic device_method_t ntb_pci_methods[] = { 266250079Scarl /* Device interface */ 267250079Scarl DEVMETHOD(device_probe, ntb_probe), 268250079Scarl DEVMETHOD(device_attach, ntb_attach), 269250079Scarl DEVMETHOD(device_detach, ntb_detach), 270250079Scarl DEVMETHOD_END 271250079Scarl}; 272250079Scarl 273250079Scarlstatic driver_t ntb_pci_driver = { 274250079Scarl "ntb_hw", 275250079Scarl ntb_pci_methods, 276250079Scarl sizeof(struct ntb_softc), 277250079Scarl}; 278250079Scarl 279250079Scarlstatic devclass_t ntb_devclass; 280250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 281250079ScarlMODULE_VERSION(ntb_hw, 1); 282250079Scarl 283289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 284289207Scem 285250079Scarl/* 286250079Scarl * OS <-> Driver linkage functions 287250079Scarl */ 288250079Scarlstatic int 289250079Scarlntb_probe(device_t device) 290250079Scarl{ 291289209Scem struct ntb_hw_info *p; 292250079Scarl 293289209Scem p = ntb_get_device_info(pci_get_devid(device)); 294289209Scem if (p == NULL) 295250079Scarl return (ENXIO); 296289209Scem 297289209Scem device_set_desc(device, p->desc); 298289209Scem return (0); 299250079Scarl} 300250079Scarl 301250079Scarlstatic int 302250079Scarlntb_attach(device_t device) 303250079Scarl{ 304289209Scem struct ntb_softc *ntb; 305289209Scem struct ntb_hw_info *p; 306250079Scarl int error; 307250079Scarl 308289209Scem ntb = DEVICE2SOFTC(device); 309289209Scem p = ntb_get_device_info(pci_get_devid(device)); 310289209Scem 311250079Scarl ntb->device = device; 312250079Scarl ntb->type = p->type; 313255274Scarl ntb->features = p->features; 314250079Scarl 315250079Scarl /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 316283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 317283291Sjkim callout_init(&ntb->lr_timer, 1); 318250079Scarl 319289348Scem if (ntb->type == NTB_SOC) 320289348Scem error = ntb_detect_soc(ntb); 321289348Scem else 322289348Scem error = ntb_detect_xeon(ntb); 323289348Scem if (error) 324289348Scem goto out; 325289348Scem 326289397Scem ntb_detect_max_mw(ntb); 327289396Scem 328289209Scem error = ntb_map_pci_bars(ntb); 329289209Scem if (error) 330289209Scem goto out; 331289272Scem if (ntb->type == NTB_SOC) 332289272Scem error = ntb_setup_soc(ntb); 333289272Scem else 334289272Scem error = ntb_setup_xeon(ntb); 335289209Scem if (error) 336289209Scem goto out; 337289540Scem error = ntb_init_isr(ntb); 338289209Scem if (error) 339289209Scem goto out; 340250079Scarl 341250079Scarl pci_enable_busmaster(ntb->device); 342250079Scarl 343289209Scemout: 344289209Scem if (error != 0) 345289209Scem ntb_detach(device); 346250079Scarl return (error); 347250079Scarl} 348250079Scarl 349250079Scarlstatic int 350250079Scarlntb_detach(device_t device) 351250079Scarl{ 352289209Scem struct ntb_softc *ntb; 353250079Scarl 354289209Scem ntb = DEVICE2SOFTC(device); 355250079Scarl callout_drain(&ntb->heartbeat_timer); 356250079Scarl callout_drain(&ntb->lr_timer); 357289272Scem if (ntb->type == NTB_XEON) 358289272Scem ntb_teardown_xeon(ntb); 359250079Scarl ntb_teardown_interrupts(ntb); 360289397Scem 361289397Scem /* 362289397Scem * Redetect total MWs so we unmap properly -- in case we lowered the 363289397Scem * maximum to work around Xeon errata. 364289397Scem */ 365289397Scem ntb_detect_max_mw(ntb); 366250079Scarl ntb_unmap_pci_bar(ntb); 367250079Scarl 368250079Scarl return (0); 369250079Scarl} 370250079Scarl 371289539Scemstatic inline enum ntb_bar 372289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 373289539Scem{ 374289539Scem 375289539Scem KASSERT(mw < ntb->mw_count, ("%s: mw:%u > count:%u", __func__, mw, 376289539Scem (unsigned)ntb->mw_count)); 377289539Scem 378289539Scem return (NTB_B2B_BAR_1 + mw); 379289539Scem} 380289539Scem 381250079Scarlstatic int 382255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 383250079Scarl{ 384255272Scarl int rc; 385250079Scarl 386250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 387289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 388255272Scarl if (rc != 0) 389289541Scem goto out; 390255272Scarl 391289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 392289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 393255272Scarl if (rc != 0) 394289541Scem goto out; 395255272Scarl 396289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 397289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 398289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 399255279Scarl else 400289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 401289397Scem if (!HAS_FEATURE(NTB_SPLIT_BAR)) 402289541Scem goto out; 403289397Scem 404289397Scem ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 405289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 406289541Scem rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 407289397Scem else 408289541Scem rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 409250079Scarl 410289541Scemout: 411289209Scem if (rc != 0) 412255272Scarl device_printf(ntb->device, 413255272Scarl "unable to allocate pci resource\n"); 414255272Scarl return (rc); 415255272Scarl} 416255272Scarl 417289541Scemstatic void 418289541Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 419289541Scem{ 420289541Scem 421289541Scem device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n", 422289541Scem bar->size, bar->vbase, (void *)(bar->pbase)); 423289541Scem} 424289541Scem 425255272Scarlstatic int 426255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 427255272Scarl{ 428255272Scarl 429255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 430289209Scem &bar->pci_resource_id, RF_ACTIVE); 431255272Scarl if (bar->pci_resource == NULL) 432255272Scarl return (ENXIO); 433289209Scem 434289209Scem save_bar_parameters(bar); 435289541Scem print_map_success(ntb, bar); 436289209Scem return (0); 437255272Scarl} 438255272Scarl 439255272Scarlstatic int 440255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 441255272Scarl{ 442255272Scarl int rc; 443255276Scarl uint8_t bar_size_bits = 0; 444255272Scarl 445289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 446289209Scem &bar->pci_resource_id, RF_ACTIVE); 447250079Scarl 448255272Scarl if (bar->pci_resource == NULL) 449255272Scarl return (ENXIO); 450255276Scarl 451289209Scem save_bar_parameters(bar); 452289209Scem /* 453289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 454289209Scem * hardware issue. To work around this, query the size it should be 455289209Scem * configured to by the device and modify the resource to correspond to 456289209Scem * this new size. The BIOS on systems with this problem is required to 457289209Scem * provide enough address space to allow the driver to make this change 458289209Scem * safely. 459289209Scem * 460289209Scem * Ideally I could have just specified the size when I allocated the 461289209Scem * resource like: 462289209Scem * bus_alloc_resource(ntb->device, 463289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 464289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 465289209Scem * but the PCI driver does not honor the size in this call, so we have 466289209Scem * to modify it after the fact. 467289209Scem */ 468289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 469289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 470289209Scem bar_size_bits = pci_read_config(ntb->device, 471289209Scem XEON_PBAR23SZ_OFFSET, 1); 472289209Scem else 473289209Scem bar_size_bits = pci_read_config(ntb->device, 474289209Scem XEON_PBAR45SZ_OFFSET, 1); 475289209Scem 476289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 477289209Scem bar->pci_resource, bar->pbase, 478289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 479255272Scarl if (rc != 0) { 480289209Scem device_printf(ntb->device, 481289209Scem "unable to resize bar\n"); 482255272Scarl return (rc); 483250079Scarl } 484289209Scem 485289209Scem save_bar_parameters(bar); 486250079Scarl } 487289209Scem 488289209Scem /* Mark bar region as write combining to improve performance. */ 489289209Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 490289209Scem VM_MEMATTR_WRITE_COMBINING); 491289209Scem if (rc != 0) { 492289209Scem device_printf(ntb->device, 493289209Scem "unable to mark bar as WRITE_COMBINING\n"); 494289209Scem return (rc); 495289209Scem } 496289541Scem print_map_success(ntb, bar); 497250079Scarl return (0); 498250079Scarl} 499250079Scarl 500250079Scarlstatic void 501250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 502250079Scarl{ 503250079Scarl struct ntb_pci_bar_info *current_bar; 504250079Scarl int i; 505250079Scarl 506289397Scem for (i = 0; i < NTB_MAX_BARS; i++) { 507250079Scarl current_bar = &ntb->bar_info[i]; 508250079Scarl if (current_bar->pci_resource != NULL) 509250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 510250079Scarl current_bar->pci_resource_id, 511250079Scarl current_bar->pci_resource); 512250079Scarl } 513250079Scarl} 514250079Scarl 515250079Scarlstatic int 516289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 517250079Scarl{ 518289342Scem uint32_t i; 519289342Scem int rc; 520289342Scem 521289342Scem for (i = 0; i < num_vectors; i++) { 522289342Scem ntb->int_info[i].rid = i + 1; 523289342Scem ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 524289342Scem SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 525289342Scem if (ntb->int_info[i].res == NULL) { 526289342Scem device_printf(ntb->device, 527289342Scem "bus_alloc_resource failed\n"); 528289342Scem return (ENOMEM); 529289342Scem } 530289342Scem ntb->int_info[i].tag = NULL; 531289342Scem ntb->allocated_interrupts++; 532289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 533289540Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, handle_irq, 534289342Scem &ntb->db_cb[i], &ntb->int_info[i].tag); 535289342Scem if (rc != 0) { 536289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 537289342Scem return (ENXIO); 538289342Scem } 539289342Scem } 540289342Scem return (0); 541289342Scem} 542289342Scem 543289344Scem/* 544289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 545289344Scem * cannot be allocated for each MSI-X message. JHB seems to think remapping 546289344Scem * should be okay. This tunable should enable us to test that hypothesis 547289344Scem * when someone gets their hands on some Xeon hardware. 548289344Scem */ 549289344Scemstatic int ntb_force_remap_mode; 550289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 551289344Scem &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 552289344Scem " to a smaller number of ithreads, even if the desired number are " 553289344Scem "available"); 554289344Scem 555289344Scem/* 556289344Scem * In case it is NOT ok, give consumers an abort button. 557289344Scem */ 558289344Scemstatic int ntb_prefer_intx; 559289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 560289344Scem &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 561289344Scem "than remapping MSI-X messages over available slots (match Linux driver " 562289344Scem "behavior)"); 563289344Scem 564289344Scem/* 565289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple 566289344Scem * round-robin fashion. 567289344Scem */ 568289342Scemstatic int 569289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 570289344Scem{ 571289344Scem u_int *vectors; 572289344Scem uint32_t i; 573289344Scem int rc; 574289344Scem 575289344Scem if (ntb_prefer_intx != 0) 576289344Scem return (ENXIO); 577289344Scem 578289344Scem vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 579289344Scem 580289344Scem for (i = 0; i < desired; i++) 581289344Scem vectors[i] = (i % avail) + 1; 582289344Scem 583289344Scem rc = pci_remap_msix(dev, desired, vectors); 584289344Scem free(vectors, M_NTB); 585289344Scem return (rc); 586289344Scem} 587289344Scem 588289344Scemstatic int 589289540Scemntb_init_isr(struct ntb_softc *ntb) 590289342Scem{ 591289344Scem uint32_t desired_vectors, num_vectors; 592289347Scem uint64_t mask; 593289342Scem int rc; 594250079Scarl 595250079Scarl ntb->allocated_interrupts = 0; 596289347Scem 597250079Scarl /* 598250079Scarl * On SOC, disable all interrupts. On XEON, disable all but Link 599250079Scarl * Interrupt. The rest will be unmasked as callbacks are registered. 600250079Scarl */ 601289347Scem mask = 0; 602289347Scem if (ntb->type == NTB_XEON) 603289538Scem mask = (1 << XEON_DB_LINK); 604289539Scem ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ~mask); 605250079Scarl 606289344Scem num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 607289539Scem ntb->db_count); 608289344Scem if (desired_vectors >= 1) { 609289344Scem rc = pci_alloc_msix(ntb->device, &num_vectors); 610250079Scarl 611289344Scem if (ntb_force_remap_mode != 0 && rc == 0 && 612289344Scem num_vectors == desired_vectors) 613289344Scem num_vectors--; 614289344Scem 615289344Scem if (rc == 0 && num_vectors < desired_vectors) { 616289344Scem rc = ntb_remap_msix(ntb->device, desired_vectors, 617289344Scem num_vectors); 618289344Scem if (rc == 0) 619289344Scem num_vectors = desired_vectors; 620289344Scem else 621289344Scem pci_release_msi(ntb->device); 622289344Scem } 623289344Scem if (rc != 0) 624289344Scem num_vectors = 1; 625289344Scem } else 626289344Scem num_vectors = 1; 627289344Scem 628289539Scem if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 629289396Scem /* 630289396Scem * If allocating MSI-X interrupts failed and we're forced to 631289396Scem * use legacy INTx anyway, the only limit on individual 632289396Scem * callbacks is the number of doorbell bits. 633289396Scem */ 634289539Scem ntb->db_vec_count = 1; 635289539Scem ntb->db_vec_shift = ntb->db_count; 636289539Scem ntb_create_callbacks(ntb, ntb->db_count); 637289539Scem rc = ntb_setup_legacy_interrupt(ntb); 638289539Scem } else { 639289539Scem ntb_create_callbacks(ntb, num_vectors); 640289540Scem rc = ntb_setup_msix(ntb, num_vectors); 641289539Scem if (rc == 0 && ntb->type == NTB_XEON) { 642289539Scem /* 643289539Scem * Prevent consumers from registering callbacks on the link event irq 644289539Scem * slot, from which they will never be called back. 645289539Scem */ 646289539Scem ntb->db_cb[num_vectors - 1].reserved = true; 647289539Scem ntb->max_cbs--; 648289539Scem } 649289539Scem } 650289539Scem if (rc != 0) { 651289539Scem device_printf(ntb->device, 652289539Scem "Error allocating interrupts: %d\n", rc); 653289396Scem ntb_free_callbacks(ntb); 654289396Scem } 655289396Scem 656289342Scem return (rc); 657289342Scem} 658289342Scem 659289342Scemstatic int 660289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb) 661289342Scem{ 662289342Scem int rc; 663289342Scem 664289342Scem ntb->int_info[0].rid = 0; 665289342Scem ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 666289342Scem &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 667289342Scem if (ntb->int_info[0].res == NULL) { 668289342Scem device_printf(ntb->device, "bus_alloc_resource failed\n"); 669289342Scem return (ENOMEM); 670250079Scarl } 671250079Scarl 672289342Scem ntb->int_info[0].tag = NULL; 673289342Scem ntb->allocated_interrupts = 1; 674289342Scem 675289342Scem rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 676289342Scem INTR_MPSAFE | INTR_TYPE_MISC, NULL, ntb_handle_legacy_interrupt, 677289342Scem ntb, &ntb->int_info[0].tag); 678289342Scem if (rc != 0) { 679289342Scem device_printf(ntb->device, "bus_setup_intr failed\n"); 680289342Scem return (ENXIO); 681289342Scem } 682289342Scem 683250079Scarl return (0); 684250079Scarl} 685250079Scarl 686250079Scarlstatic void 687250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 688250079Scarl{ 689250079Scarl struct ntb_int_info *current_int; 690250079Scarl int i; 691250079Scarl 692289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 693250079Scarl current_int = &ntb->int_info[i]; 694250079Scarl if (current_int->tag != NULL) 695250079Scarl bus_teardown_intr(ntb->device, current_int->res, 696250079Scarl current_int->tag); 697250079Scarl 698250079Scarl if (current_int->res != NULL) 699250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 700250079Scarl rman_get_rid(current_int->res), current_int->res); 701250079Scarl } 702250079Scarl 703250079Scarl ntb_free_callbacks(ntb); 704250079Scarl pci_release_msi(ntb->device); 705250079Scarl} 706250079Scarl 707289347Scem/* 708289347Scem * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon. Abstract it 709289347Scem * out to make code clearer. 710289347Scem */ 711289539Scemstatic inline uint64_t 712289539Scemntb_db_read(struct ntb_softc *ntb, uint64_t regoff) 713289347Scem{ 714289347Scem 715289347Scem if (ntb->type == NTB_SOC) 716289347Scem return (ntb_reg_read(8, regoff)); 717289347Scem 718289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 719289347Scem 720289347Scem return (ntb_reg_read(2, regoff)); 721289347Scem} 722289347Scem 723289539Scemstatic inline void 724289539Scemntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 725289347Scem{ 726289347Scem 727289347Scem if (ntb->type == NTB_SOC) { 728289347Scem ntb_reg_write(8, regoff, val); 729289347Scem return; 730289347Scem } 731289347Scem 732289347Scem KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 733289347Scem ntb_reg_write(2, regoff, (uint16_t)val); 734289347Scem} 735289347Scem 736289347Scemstatic void 737289281Scemmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 738289281Scem{ 739289347Scem uint64_t mask; 740289281Scem 741289539Scem mask = ntb_db_read(ntb, ntb->reg_ofs.ldb_mask); 742289539Scem mask |= 1 << (idx * ntb->db_vec_shift); 743289539Scem ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask); 744289281Scem} 745289281Scem 746289540Scemstatic inline void 747289281Scemunmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 748289281Scem{ 749289347Scem uint64_t mask; 750289281Scem 751289539Scem mask = ntb_db_read(ntb, ntb->reg_ofs.ldb_mask); 752289539Scem mask &= ~(1 << (idx * ntb->db_vec_shift)); 753289539Scem ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask); 754289281Scem} 755289281Scem 756289540Scemstatic inline uint64_t 757289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 758250079Scarl{ 759289540Scem uint64_t shift, mask; 760250079Scarl 761289540Scem shift = ntb->db_vec_shift; 762289540Scem mask = (1ull << shift) - 1; 763289540Scem return (mask << (shift * db_vector)); 764250079Scarl} 765250079Scarl 766250079Scarlstatic void 767289540Scemhandle_irq(void *arg) 768250079Scarl{ 769250079Scarl struct ntb_db_cb *db_cb = arg; 770250079Scarl struct ntb_softc *ntb = db_cb->ntb; 771289540Scem uint64_t vec_mask; 772289540Scem int rc; 773250079Scarl 774289540Scem vec_mask = ntb_vec_mask(ntb, db_cb->db_num); 775250079Scarl 776289540Scem if (ntb->type == NTB_XEON && (vec_mask & XEON_DB_LINK_BIT) != 0) { 777289540Scem rc = ntb_check_link_status(ntb); 778289540Scem if (rc != 0) 779289540Scem device_printf(ntb->device, 780289540Scem "Error determining link status\n"); 781289540Scem } 782289540Scem 783289281Scem if (db_cb->callback != NULL) { 784289540Scem KASSERT(!db_cb->reserved, ("user callback on link event cb")); 785289281Scem mask_ldb_interrupt(ntb, db_cb->db_num); 786289281Scem } 787250079Scarl 788289540Scem ntb_db_write(ntb, ntb->reg_ofs.ldb, vec_mask); 789250079Scarl 790289540Scem if (db_cb->callback != NULL) 791289540Scem callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 792250079Scarl} 793250079Scarl 794250079Scarlstatic void 795250079Scarlntb_handle_legacy_interrupt(void *arg) 796250079Scarl{ 797250079Scarl struct ntb_softc *ntb = arg; 798289347Scem unsigned int i; 799289347Scem uint64_t ldb; 800250079Scarl 801289539Scem ldb = ntb_db_read(ntb, ntb->reg_ofs.ldb); 802289347Scem while (ldb != 0) { 803289347Scem i = ffs(ldb); 804289347Scem ldb &= ldb - 1; 805289540Scem handle_irq(&ntb->db_cb[i]); 806250079Scarl } 807250079Scarl} 808250079Scarl 809250079Scarlstatic int 810289342Scemntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors) 811250079Scarl{ 812289342Scem uint32_t i; 813250079Scarl 814289396Scem ntb->max_cbs = num_vectors; 815289209Scem ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 816250079Scarl M_ZERO | M_WAITOK); 817250079Scarl for (i = 0; i < num_vectors; i++) { 818250079Scarl ntb->db_cb[i].db_num = i; 819250079Scarl ntb->db_cb[i].ntb = ntb; 820250079Scarl } 821250079Scarl 822250079Scarl return (0); 823250079Scarl} 824250079Scarl 825250079Scarlstatic void 826250079Scarlntb_free_callbacks(struct ntb_softc *ntb) 827250079Scarl{ 828289342Scem uint8_t i; 829250079Scarl 830289539Scem if (ntb->db_cb == NULL) 831289539Scem return; 832289539Scem 833289396Scem for (i = 0; i < ntb->max_cbs; i++) 834250079Scarl ntb_unregister_db_callback(ntb, i); 835250079Scarl 836250079Scarl free(ntb->db_cb, M_NTB); 837289539Scem ntb->db_cb = NULL; 838289396Scem ntb->max_cbs = 0; 839250079Scarl} 840250079Scarl 841250079Scarlstatic struct ntb_hw_info * 842250079Scarlntb_get_device_info(uint32_t device_id) 843250079Scarl{ 844250079Scarl struct ntb_hw_info *ep = pci_ids; 845250079Scarl 846250079Scarl while (ep->device_id) { 847250079Scarl if (ep->device_id == device_id) 848250079Scarl return (ep); 849250079Scarl ++ep; 850250079Scarl } 851250079Scarl return (NULL); 852250079Scarl} 853250079Scarl 854289272Scemstatic void 855289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 856250079Scarl{ 857250079Scarl 858289272Scem ntb_hw_link_down(ntb); 859250079Scarl} 860250079Scarl 861289397Scemstatic void 862289397Scemntb_detect_max_mw(struct ntb_softc *ntb) 863289397Scem{ 864289397Scem 865289397Scem if (ntb->type == NTB_SOC) { 866289539Scem ntb->mw_count = SOC_MW_COUNT; 867289397Scem return; 868289397Scem } 869289397Scem 870289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 871289539Scem ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 872289397Scem else 873289539Scem ntb->mw_count = XEON_SNB_MW_COUNT; 874289397Scem} 875289397Scem 876250079Scarlstatic int 877289348Scemntb_detect_xeon(struct ntb_softc *ntb) 878250079Scarl{ 879289348Scem uint8_t ppd, conn_type; 880250079Scarl 881289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 882289348Scem ntb->ppd = ppd; 883250079Scarl 884289348Scem if ((ppd & XEON_PPD_DEV_TYPE) != 0) 885289257Scem ntb->dev_type = NTB_DEV_USD; 886289257Scem else 887289257Scem ntb->dev_type = NTB_DEV_DSD; 888289257Scem 889289397Scem if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 890289397Scem ntb->features |= NTB_SPLIT_BAR; 891289397Scem 892289348Scem conn_type = ppd & XEON_PPD_CONN_TYPE; 893289348Scem switch (conn_type) { 894289348Scem case NTB_CONN_B2B: 895289348Scem ntb->conn_type = conn_type; 896289348Scem break; 897289348Scem case NTB_CONN_RP: 898289348Scem case NTB_CONN_TRANSPARENT: 899289348Scem default: 900289348Scem device_printf(ntb->device, "Unsupported connection type: %u\n", 901289348Scem (unsigned)conn_type); 902289348Scem return (ENXIO); 903289348Scem } 904289348Scem return (0); 905289348Scem} 906289348Scem 907289348Scemstatic int 908289348Scemntb_detect_soc(struct ntb_softc *ntb) 909289348Scem{ 910289348Scem uint32_t ppd, conn_type; 911289348Scem 912289348Scem ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 913289348Scem ntb->ppd = ppd; 914289348Scem 915289348Scem if ((ppd & SOC_PPD_DEV_TYPE) != 0) 916289348Scem ntb->dev_type = NTB_DEV_DSD; 917289348Scem else 918289348Scem ntb->dev_type = NTB_DEV_USD; 919289348Scem 920289348Scem conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8; 921289348Scem switch (conn_type) { 922289348Scem case NTB_CONN_B2B: 923289348Scem ntb->conn_type = conn_type; 924289348Scem break; 925289348Scem default: 926289348Scem device_printf(ntb->device, "Unsupported NTB configuration\n"); 927289348Scem return (ENXIO); 928289348Scem } 929289348Scem return (0); 930289348Scem} 931289348Scem 932289348Scemstatic int 933289348Scemntb_setup_xeon(struct ntb_softc *ntb) 934289348Scem{ 935289348Scem 936289257Scem ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; 937289257Scem ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; 938289257Scem ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 939289257Scem ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET; 940289257Scem ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; 941289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 942289397Scem ntb->reg_ofs.bar5_xlat = XEON_SBAR5XLAT_OFFSET; 943289257Scem 944289348Scem switch (ntb->conn_type) { 945250079Scarl case NTB_CONN_B2B: 946289257Scem /* 947289257Scem * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored 948289538Scem * with the NTB_SDOORBELL_LOCKUP errata mode enabled. (See 949289257Scem * ntb_ring_doorbell() and ntb_read/write_remote_spad().) 950289257Scem */ 951289257Scem ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET; 952289257Scem ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 953289257Scem 954289539Scem ntb->spad_count = XEON_SPAD_COUNT; 955250079Scarl break; 956289257Scem 957250079Scarl case NTB_CONN_RP: 958289257Scem /* 959289538Scem * Every Xeon today needs NTB_SDOORBELL_LOCKUP, so punt on RP for 960289257Scem * now. 961289257Scem */ 962289538Scem KASSERT(HAS_FEATURE(NTB_SDOORBELL_LOCKUP), 963289257Scem ("Xeon without MW errata unimplemented")); 964289257Scem device_printf(ntb->device, 965289257Scem "NTB-RP disabled to due hardware errata.\n"); 966289257Scem return (ENXIO); 967289257Scem 968289257Scem case NTB_CONN_TRANSPARENT: 969250079Scarl default: 970250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 971289348Scem ntb->conn_type); 972250079Scarl return (ENXIO); 973250079Scarl } 974250079Scarl 975289208Scem /* 976289208Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 977289208Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 978289208Scem * which may hang the system. To workaround this use the second memory 979289208Scem * window to access the interrupt and scratch pad registers on the 980289208Scem * remote system. 981289274Scem * 982289274Scem * There is another HW errata on the limit registers -- they can only 983289274Scem * be written when the base register is (?)4GB aligned and < 32-bit. 984289274Scem * This should already be the case based on the driver defaults, but 985289274Scem * write the limit registers first just in case. 986289208Scem */ 987289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 988289208Scem /* 989289208Scem * Set the Limit register to 4k, the minimum size, to prevent 990289208Scem * an illegal access. 991289397Scem * 992289397Scem * XXX: Should this be PBAR5LMT / get_mw_size(, max_mw - 1)? 993289208Scem */ 994289208Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 995289208Scem ntb_get_mw_size(ntb, 1) + 0x1000); 996289397Scem /* Reserve the last MW for mapping remote spad */ 997289539Scem ntb->mw_count--; 998289396Scem } else 999289208Scem /* 1000289208Scem * Disable the limit register, just in case it is set to 1001289397Scem * something silly. A 64-bit write will also clear PBAR5LMT in 1002289397Scem * split-bar mode, and this is desired. 1003289208Scem */ 1004289208Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1005289208Scem 1006289257Scem ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 1007289257Scem ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 1008289257Scem ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 1009250079Scarl 1010289539Scem ntb->db_count = XEON_DB_COUNT; 1011289539Scem ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1012289539Scem ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1013250079Scarl 1014289271Scem /* 1015289271Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1016289271Scem * mirrored to the remote system. Shrink the number of bits by one, 1017289271Scem * since bit 14 is the last bit. 1018289271Scem * 1019289271Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1020289271Scem * anyway. Nor for non-B2B connection types. 1021289271Scem */ 1022289271Scem if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) && 1023289538Scem !HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && 1024289348Scem ntb->conn_type == NTB_CONN_B2B) 1025289539Scem ntb->db_count = XEON_DB_COUNT - 1; 1026289271Scem 1027255279Scarl configure_xeon_secondary_side_bars(ntb); 1028289209Scem 1029250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1030289257Scem if (ntb->conn_type == NTB_CONN_B2B) 1031289257Scem ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 1032289257Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1033255279Scarl 1034255269Scarl /* Enable link training */ 1035289272Scem ntb_hw_link_up(ntb); 1036250079Scarl 1037250079Scarl return (0); 1038250079Scarl} 1039250079Scarl 1040250079Scarlstatic int 1041250079Scarlntb_setup_soc(struct ntb_softc *ntb) 1042250079Scarl{ 1043250079Scarl 1044289348Scem KASSERT(ntb->conn_type == NTB_CONN_B2B, 1045289348Scem ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1046250079Scarl 1047250079Scarl /* Initiate PCI-E link training */ 1048289348Scem pci_write_config(ntb->device, NTB_PPD_OFFSET, 1049289348Scem ntb->ppd | SOC_PPD_INIT_LINK, 4); 1050250079Scarl 1051289255Scem ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; 1052289255Scem ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; 1053289265Scem ntb->reg_ofs.rdb = SOC_B2B_DOORBELL_OFFSET; 1054289255Scem ntb->reg_ofs.bar2_xlat = SOC_SBAR2XLAT_OFFSET; 1055289255Scem ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; 1056250079Scarl ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 1057250079Scarl ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 1058250079Scarl ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 1059289265Scem ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 1060250079Scarl ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 1061250079Scarl 1062289539Scem ntb->spad_count = SOC_SPAD_COUNT; 1063289539Scem ntb->db_count = SOC_DB_COUNT; 1064289539Scem ntb->db_vec_count = SOC_DB_MSIX_VECTOR_COUNT; 1065289539Scem ntb->db_vec_shift = SOC_DB_MSIX_VECTOR_SHIFT; 1066250079Scarl 1067250079Scarl /* 1068250079Scarl * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 1069250079Scarl * resolved. Mask transaction layer internal parity errors. 1070250079Scarl */ 1071250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 1072250079Scarl 1073255279Scarl configure_soc_secondary_side_bars(ntb); 1074250079Scarl 1075250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 1076255278Scarl ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 1077250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1078289209Scem 1079250079Scarl callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 1080250079Scarl 1081250079Scarl return (0); 1082250079Scarl} 1083250079Scarl 1084255279Scarlstatic void 1085255279Scarlconfigure_soc_secondary_side_bars(struct ntb_softc *ntb) 1086255279Scarl{ 1087255279Scarl 1088255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1089289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1090289538Scem XEON_B2B_BAR2_DSD_ADDR); 1091289538Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_DSD_ADDR); 1092289538Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR); 1093289538Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR); 1094255279Scarl } else { 1095289538Scem ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1096289538Scem XEON_B2B_BAR2_USD_ADDR); 1097289538Scem ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_USD_ADDR); 1098289538Scem ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR); 1099289538Scem ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR); 1100255279Scarl } 1101255279Scarl} 1102255279Scarl 1103255279Scarlstatic void 1104255279Scarlconfigure_xeon_secondary_side_bars(struct ntb_softc *ntb) 1105255279Scarl{ 1106255279Scarl 1107255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 1108289538Scem ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, 1109289538Scem XEON_B2B_BAR2_DSD_ADDR); 1110289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1111255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 1112289538Scem XEON_B2B_BAR0_DSD_ADDR); 1113289208Scem else { 1114289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1115289397Scem ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET, 1116289538Scem XEON_B2B_BAR4_DSD_ADDR); 1117289397Scem ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET, 1118289538Scem XEON_B2B_BAR5_DSD_ADDR); 1119289397Scem } else 1120289397Scem ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 1121289538Scem XEON_B2B_BAR4_DSD_ADDR); 1122289208Scem /* 1123289208Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be 1124289208Scem * written 32 bits at a time. 1125289208Scem */ 1126289208Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 1127289538Scem XEON_B2B_BAR0_DSD_ADDR & 0xffffffff); 1128289208Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 1129289538Scem XEON_B2B_BAR0_DSD_ADDR >> 32); 1130289208Scem } 1131289538Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, 1132289538Scem XEON_B2B_BAR0_USD_ADDR); 1133289538Scem ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, 1134289538Scem XEON_B2B_BAR2_USD_ADDR); 1135289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1136289538Scem ntb_reg_write(4, XEON_SBAR4BASE_OFFSET, 1137289538Scem XEON_B2B_BAR4_USD_ADDR); 1138289538Scem ntb_reg_write(4, XEON_SBAR5BASE_OFFSET, 1139289538Scem XEON_B2B_BAR5_USD_ADDR); 1140289397Scem } else 1141289538Scem ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, 1142289538Scem XEON_B2B_BAR4_USD_ADDR); 1143255279Scarl } else { 1144289538Scem ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, 1145289538Scem XEON_B2B_BAR2_USD_ADDR); 1146289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1147255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 1148289538Scem XEON_B2B_BAR0_USD_ADDR); 1149289208Scem else { 1150289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1151289397Scem ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET, 1152289538Scem XEON_B2B_BAR4_USD_ADDR); 1153289397Scem ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET, 1154289538Scem XEON_B2B_BAR5_USD_ADDR); 1155289397Scem } else 1156289397Scem ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 1157289538Scem XEON_B2B_BAR4_USD_ADDR); 1158289208Scem /* 1159289208Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be 1160289208Scem * written 32 bits at a time. 1161289208Scem */ 1162289208Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 1163289538Scem XEON_B2B_BAR0_USD_ADDR & 0xffffffff); 1164289208Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 1165289538Scem XEON_B2B_BAR0_USD_ADDR >> 32); 1166289208Scem } 1167289538Scem ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, 1168289538Scem XEON_B2B_BAR0_DSD_ADDR); 1169289538Scem ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, 1170289538Scem XEON_B2B_BAR2_DSD_ADDR); 1171289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1172289397Scem ntb_reg_write(4, XEON_SBAR4BASE_OFFSET, 1173289538Scem XEON_B2B_BAR4_DSD_ADDR); 1174289397Scem ntb_reg_write(4, XEON_SBAR5BASE_OFFSET, 1175289538Scem XEON_B2B_BAR5_DSD_ADDR); 1176289397Scem } else 1177289397Scem ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, 1178289538Scem XEON_B2B_BAR4_DSD_ADDR); 1179255279Scarl } 1180255279Scarl} 1181255279Scarl 1182255281Scarl/* SOC does not have link status interrupt, poll on that platform */ 1183250079Scarlstatic void 1184250079Scarlntb_handle_heartbeat(void *arg) 1185250079Scarl{ 1186250079Scarl struct ntb_softc *ntb = arg; 1187250079Scarl uint32_t status32; 1188289209Scem int rc; 1189250079Scarl 1190289209Scem rc = ntb_check_link_status(ntb); 1191250079Scarl if (rc != 0) 1192250079Scarl device_printf(ntb->device, 1193250079Scarl "Error determining link status\n"); 1194289232Scem 1195250079Scarl /* Check to see if a link error is the cause of the link down */ 1196250079Scarl if (ntb->link_status == NTB_LINK_DOWN) { 1197255278Scarl status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1198250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 1199250079Scarl callout_reset(&ntb->lr_timer, 0, recover_soc_link, 1200250079Scarl ntb); 1201250079Scarl return; 1202250079Scarl } 1203250079Scarl } 1204250079Scarl 1205250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1206250079Scarl ntb_handle_heartbeat, ntb); 1207250079Scarl} 1208250079Scarl 1209250079Scarlstatic void 1210250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb) 1211250079Scarl{ 1212250079Scarl uint32_t status; 1213250079Scarl 1214250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1215255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1216255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1217255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1218255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1219250079Scarl 1220250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1221250079Scarl pause("ModPhy", hz / 10); 1222250079Scarl 1223250079Scarl /* Clear AER Errors, write to clear */ 1224255278Scarl status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1225250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1226255278Scarl ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1227250079Scarl 1228250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1229255278Scarl status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1230250079Scarl status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1231255278Scarl ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1232250079Scarl 1233250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1234255278Scarl status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1235250079Scarl status |= SOC_DESKEWSTS_DBERR; 1236255278Scarl ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1237250079Scarl 1238255278Scarl status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1239250079Scarl status &= SOC_IBIST_ERR_OFLOW; 1240255278Scarl ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1241250079Scarl 1242250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1243255278Scarl status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1244250079Scarl status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1245255278Scarl ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1246250079Scarl} 1247250079Scarl 1248250079Scarlstatic void 1249250079Scarlntb_handle_link_event(struct ntb_softc *ntb, int link_state) 1250250079Scarl{ 1251250079Scarl enum ntb_hw_event event; 1252250079Scarl uint16_t status; 1253250079Scarl 1254250079Scarl if (ntb->link_status == link_state) 1255250079Scarl return; 1256250079Scarl 1257250079Scarl if (link_state == NTB_LINK_UP) { 1258250079Scarl device_printf(ntb->device, "Link Up\n"); 1259250079Scarl ntb->link_status = NTB_LINK_UP; 1260250079Scarl event = NTB_EVENT_HW_LINK_UP; 1261250079Scarl 1262289257Scem if (ntb->type == NTB_SOC || 1263289257Scem ntb->conn_type == NTB_CONN_TRANSPARENT) 1264255278Scarl status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1265250079Scarl else 1266250079Scarl status = pci_read_config(ntb->device, 1267250079Scarl XEON_LINK_STATUS_OFFSET, 2); 1268250079Scarl ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 1269250079Scarl ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 1270250079Scarl device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 1271250079Scarl ntb->link_width, ntb->link_speed); 1272250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1273250079Scarl ntb_handle_heartbeat, ntb); 1274250079Scarl } else { 1275250079Scarl device_printf(ntb->device, "Link Down\n"); 1276250079Scarl ntb->link_status = NTB_LINK_DOWN; 1277250079Scarl event = NTB_EVENT_HW_LINK_DOWN; 1278255281Scarl /* Do not modify link width/speed, we need it in link recovery */ 1279250079Scarl } 1280250079Scarl 1281250079Scarl /* notify the upper layer if we have an event change */ 1282250079Scarl if (ntb->event_cb != NULL) 1283250079Scarl ntb->event_cb(ntb->ntb_transport, event); 1284250079Scarl} 1285250079Scarl 1286250079Scarlstatic void 1287289272Scemntb_hw_link_up(struct ntb_softc *ntb) 1288289272Scem{ 1289289280Scem uint32_t cntl; 1290289272Scem 1291289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1292289272Scem ntb_handle_link_event(ntb, NTB_LINK_UP); 1293289280Scem return; 1294289280Scem } 1295289280Scem 1296289280Scem cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1297289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1298289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1299289397Scem cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1300289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1301289397Scem cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1302289280Scem ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1303289272Scem} 1304289272Scem 1305289272Scemstatic void 1306289272Scemntb_hw_link_down(struct ntb_softc *ntb) 1307289272Scem{ 1308289272Scem uint32_t cntl; 1309289272Scem 1310289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1311289272Scem ntb_handle_link_event(ntb, NTB_LINK_DOWN); 1312289272Scem return; 1313289272Scem } 1314289272Scem 1315289272Scem cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1316289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1317289397Scem cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1318289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1319289397Scem cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1320289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1321289272Scem ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1322289272Scem} 1323289272Scem 1324289272Scemstatic void 1325250079Scarlrecover_soc_link(void *arg) 1326250079Scarl{ 1327250079Scarl struct ntb_softc *ntb = arg; 1328250079Scarl uint8_t speed, width; 1329250079Scarl uint32_t status32; 1330250079Scarl uint16_t status16; 1331250079Scarl 1332250079Scarl soc_perform_link_restart(ntb); 1333250079Scarl 1334289232Scem /* 1335289232Scem * There is a potential race between the 2 NTB devices recovering at 1336289232Scem * the same time. If the times are the same, the link will not recover 1337289232Scem * and the driver will be stuck in this loop forever. Add a random 1338289232Scem * interval to the recovery time to prevent this race. 1339289232Scem */ 1340289232Scem status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1341289232Scem pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1342289232Scem 1343255278Scarl status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1344250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1345250079Scarl goto retry; 1346250079Scarl 1347255278Scarl status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1348250079Scarl if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1349250079Scarl goto retry; 1350250079Scarl 1351289232Scem status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1352289232Scem if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1353289232Scem goto out; 1354289232Scem 1355255278Scarl status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1356250079Scarl width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 1357250079Scarl speed = (status16 & NTB_LINK_SPEED_MASK); 1358250079Scarl if (ntb->link_width != width || ntb->link_speed != speed) 1359250079Scarl goto retry; 1360250079Scarl 1361289232Scemout: 1362250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1363250079Scarl ntb_handle_heartbeat, ntb); 1364250079Scarl return; 1365250079Scarl 1366250079Scarlretry: 1367250079Scarl callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1368250079Scarl ntb); 1369250079Scarl} 1370250079Scarl 1371250079Scarlstatic int 1372250079Scarlntb_check_link_status(struct ntb_softc *ntb) 1373250079Scarl{ 1374250079Scarl int link_state; 1375250079Scarl uint32_t ntb_cntl; 1376250079Scarl uint16_t status; 1377250079Scarl 1378250079Scarl if (ntb->type == NTB_SOC) { 1379255278Scarl ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1380250079Scarl if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1381250079Scarl link_state = NTB_LINK_DOWN; 1382250079Scarl else 1383250079Scarl link_state = NTB_LINK_UP; 1384250079Scarl } else { 1385250079Scarl status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 1386250079Scarl 2); 1387250079Scarl 1388250079Scarl if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1389250079Scarl link_state = NTB_LINK_UP; 1390250079Scarl else 1391250079Scarl link_state = NTB_LINK_DOWN; 1392250079Scarl } 1393250079Scarl 1394250079Scarl ntb_handle_link_event(ntb, link_state); 1395250079Scarl 1396250079Scarl return (0); 1397250079Scarl} 1398250079Scarl 1399250079Scarl/** 1400250079Scarl * ntb_register_event_callback() - register event callback 1401250079Scarl * @ntb: pointer to ntb_softc instance 1402250079Scarl * @func: callback function to register 1403250079Scarl * 1404250079Scarl * This function registers a callback for any HW driver events such as link 1405250079Scarl * up/down, power management notices and etc. 1406250079Scarl * 1407289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1408250079Scarl */ 1409250079Scarlint 1410250079Scarlntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1411250079Scarl{ 1412250079Scarl 1413250079Scarl if (ntb->event_cb != NULL) 1414250079Scarl return (EINVAL); 1415250079Scarl 1416250079Scarl ntb->event_cb = func; 1417250079Scarl 1418250079Scarl return (0); 1419250079Scarl} 1420250079Scarl 1421250079Scarl/** 1422250079Scarl * ntb_unregister_event_callback() - unregisters the event callback 1423250079Scarl * @ntb: pointer to ntb_softc instance 1424250079Scarl * 1425250079Scarl * This function unregisters the existing callback from transport 1426250079Scarl */ 1427250079Scarlvoid 1428250079Scarlntb_unregister_event_callback(struct ntb_softc *ntb) 1429250079Scarl{ 1430250079Scarl 1431250079Scarl ntb->event_cb = NULL; 1432250079Scarl} 1433250079Scarl 1434289281Scemstatic void 1435289281Scemntb_irq_work(void *arg) 1436289281Scem{ 1437289281Scem struct ntb_db_cb *db_cb = arg; 1438289281Scem struct ntb_softc *ntb; 1439289281Scem int rc; 1440289281Scem 1441289281Scem rc = db_cb->callback(db_cb->data, db_cb->db_num); 1442289281Scem /* Poll if forward progress was made. */ 1443289281Scem if (rc != 0) { 1444289281Scem callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 1445289281Scem return; 1446289281Scem } 1447289281Scem 1448289281Scem /* Unmask interrupt if no progress was made. */ 1449289281Scem ntb = db_cb->ntb; 1450289281Scem unmask_ldb_interrupt(ntb, db_cb->db_num); 1451289281Scem} 1452289281Scem 1453250079Scarl/** 1454250079Scarl * ntb_register_db_callback() - register a callback for doorbell interrupt 1455250079Scarl * @ntb: pointer to ntb_softc instance 1456250079Scarl * @idx: doorbell index to register callback, zero based 1457289266Scem * @data: pointer to be returned to caller with every callback 1458250079Scarl * @func: callback function to register 1459250079Scarl * 1460250079Scarl * This function registers a callback function for the doorbell interrupt 1461250079Scarl * on the primary side. The function will unmask the doorbell as well to 1462250079Scarl * allow interrupt. 1463250079Scarl * 1464289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1465250079Scarl */ 1466250079Scarlint 1467250079Scarlntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1468250079Scarl ntb_db_callback func) 1469250079Scarl{ 1470289343Scem struct ntb_db_cb *db_cb = &ntb->db_cb[idx]; 1471250079Scarl 1472289396Scem if (idx >= ntb->max_cbs || db_cb->callback != NULL || db_cb->reserved) { 1473250079Scarl device_printf(ntb->device, "Invalid Index.\n"); 1474250079Scarl return (EINVAL); 1475250079Scarl } 1476250079Scarl 1477289343Scem db_cb->callback = func; 1478289343Scem db_cb->data = data; 1479289343Scem callout_init(&db_cb->irq_work, 1); 1480250079Scarl 1481289281Scem unmask_ldb_interrupt(ntb, idx); 1482250079Scarl 1483250079Scarl return (0); 1484250079Scarl} 1485250079Scarl 1486250079Scarl/** 1487250079Scarl * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1488250079Scarl * @ntb: pointer to ntb_softc instance 1489250079Scarl * @idx: doorbell index to register callback, zero based 1490250079Scarl * 1491250079Scarl * This function unregisters a callback function for the doorbell interrupt 1492250079Scarl * on the primary side. The function will also mask the said doorbell. 1493250079Scarl */ 1494250079Scarlvoid 1495250079Scarlntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1496250079Scarl{ 1497250079Scarl 1498289396Scem if (idx >= ntb->max_cbs || ntb->db_cb[idx].callback == NULL) 1499250079Scarl return; 1500250079Scarl 1501289281Scem mask_ldb_interrupt(ntb, idx); 1502250079Scarl 1503289281Scem callout_drain(&ntb->db_cb[idx].irq_work); 1504250079Scarl ntb->db_cb[idx].callback = NULL; 1505250079Scarl} 1506250079Scarl 1507250079Scarl/** 1508250079Scarl * ntb_find_transport() - find the transport pointer 1509250079Scarl * @transport: pointer to pci device 1510250079Scarl * 1511250079Scarl * Given the pci device pointer, return the transport pointer passed in when 1512250079Scarl * the transport attached when it was inited. 1513250079Scarl * 1514250079Scarl * RETURNS: pointer to transport. 1515250079Scarl */ 1516250079Scarlvoid * 1517250079Scarlntb_find_transport(struct ntb_softc *ntb) 1518250079Scarl{ 1519250079Scarl 1520250079Scarl return (ntb->ntb_transport); 1521250079Scarl} 1522250079Scarl 1523250079Scarl/** 1524250079Scarl * ntb_register_transport() - Register NTB transport with NTB HW driver 1525250079Scarl * @transport: transport identifier 1526250079Scarl * 1527250079Scarl * This function allows a transport to reserve the hardware driver for 1528250079Scarl * NTB usage. 1529250079Scarl * 1530250079Scarl * RETURNS: pointer to ntb_softc, NULL on error. 1531250079Scarl */ 1532250079Scarlstruct ntb_softc * 1533250079Scarlntb_register_transport(struct ntb_softc *ntb, void *transport) 1534250079Scarl{ 1535250079Scarl 1536250079Scarl /* 1537250079Scarl * TODO: when we have more than one transport, we will need to rewrite 1538250079Scarl * this to prevent race conditions 1539250079Scarl */ 1540250079Scarl if (ntb->ntb_transport != NULL) 1541250079Scarl return (NULL); 1542250079Scarl 1543250079Scarl ntb->ntb_transport = transport; 1544250079Scarl return (ntb); 1545250079Scarl} 1546250079Scarl 1547250079Scarl/** 1548250079Scarl * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1549250079Scarl * @ntb - ntb_softc of the transport to be freed 1550250079Scarl * 1551250079Scarl * This function unregisters the transport from the HW driver and performs any 1552250079Scarl * necessary cleanups. 1553250079Scarl */ 1554250079Scarlvoid 1555250079Scarlntb_unregister_transport(struct ntb_softc *ntb) 1556250079Scarl{ 1557289396Scem uint8_t i; 1558250079Scarl 1559250079Scarl if (ntb->ntb_transport == NULL) 1560250079Scarl return; 1561250079Scarl 1562289396Scem for (i = 0; i < ntb->max_cbs; i++) 1563250079Scarl ntb_unregister_db_callback(ntb, i); 1564250079Scarl 1565250079Scarl ntb_unregister_event_callback(ntb); 1566250079Scarl ntb->ntb_transport = NULL; 1567250079Scarl} 1568250079Scarl 1569250079Scarl/** 1570250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 1571250079Scarl * @ntb: pointer to ntb_softc instance 1572250079Scarl * 1573250079Scarl * This function returns the max 32bit scratchpad registers usable by the 1574250079Scarl * upper layer. 1575250079Scarl * 1576250079Scarl * RETURNS: total number of scratch pad registers available 1577250079Scarl */ 1578289208Scemuint8_t 1579250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 1580250079Scarl{ 1581250079Scarl 1582289539Scem return (ntb->spad_count); 1583250079Scarl} 1584250079Scarl 1585289396Scemuint8_t 1586289396Scemntb_get_max_cbs(struct ntb_softc *ntb) 1587289396Scem{ 1588289396Scem 1589289396Scem return (ntb->max_cbs); 1590289396Scem} 1591289396Scem 1592289396Scemuint8_t 1593289539Scemntb_mw_count(struct ntb_softc *ntb) 1594289396Scem{ 1595289396Scem 1596289539Scem return (ntb->mw_count); 1597289396Scem} 1598289396Scem 1599250079Scarl/** 1600250079Scarl * ntb_write_local_spad() - write to the secondary scratchpad register 1601250079Scarl * @ntb: pointer to ntb_softc instance 1602250079Scarl * @idx: index to the scratchpad register, 0 based 1603250079Scarl * @val: the data value to put into the register 1604250079Scarl * 1605250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1606250079Scarl * register. The register resides on the secondary (external) side. 1607250079Scarl * 1608289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1609250079Scarl */ 1610250079Scarlint 1611250079Scarlntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1612250079Scarl{ 1613250079Scarl 1614289539Scem if (idx >= ntb->spad_count) 1615250079Scarl return (EINVAL); 1616250079Scarl 1617255278Scarl ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1618250079Scarl 1619250079Scarl return (0); 1620250079Scarl} 1621250079Scarl 1622250079Scarl/** 1623250079Scarl * ntb_read_local_spad() - read from the primary scratchpad register 1624250079Scarl * @ntb: pointer to ntb_softc instance 1625250079Scarl * @idx: index to scratchpad register, 0 based 1626250079Scarl * @val: pointer to 32bit integer for storing the register value 1627250079Scarl * 1628250079Scarl * This function allows reading of the 32bit scratchpad register on 1629250079Scarl * the primary (internal) side. 1630250079Scarl * 1631289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1632250079Scarl */ 1633250079Scarlint 1634250079Scarlntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1635250079Scarl{ 1636250079Scarl 1637289539Scem if (idx >= ntb->spad_count) 1638250079Scarl return (EINVAL); 1639250079Scarl 1640255278Scarl *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1641250079Scarl 1642250079Scarl return (0); 1643250079Scarl} 1644250079Scarl 1645250079Scarl/** 1646250079Scarl * ntb_write_remote_spad() - write to the secondary scratchpad register 1647250079Scarl * @ntb: pointer to ntb_softc instance 1648250079Scarl * @idx: index to the scratchpad register, 0 based 1649250079Scarl * @val: the data value to put into the register 1650250079Scarl * 1651250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1652250079Scarl * register. The register resides on the secondary (external) side. 1653250079Scarl * 1654289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1655250079Scarl */ 1656250079Scarlint 1657250079Scarlntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1658250079Scarl{ 1659250079Scarl 1660289539Scem if (idx >= ntb->spad_count) 1661250079Scarl return (EINVAL); 1662250079Scarl 1663289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1664255279Scarl ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1665255279Scarl else 1666255279Scarl ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1667250079Scarl 1668250079Scarl return (0); 1669250079Scarl} 1670250079Scarl 1671250079Scarl/** 1672250079Scarl * ntb_read_remote_spad() - read from the primary scratchpad register 1673250079Scarl * @ntb: pointer to ntb_softc instance 1674250079Scarl * @idx: index to scratchpad register, 0 based 1675250079Scarl * @val: pointer to 32bit integer for storing the register value 1676250079Scarl * 1677250079Scarl * This function allows reading of the 32bit scratchpad register on 1678250079Scarl * the primary (internal) side. 1679250079Scarl * 1680289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1681250079Scarl */ 1682250079Scarlint 1683250079Scarlntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1684250079Scarl{ 1685250079Scarl 1686289539Scem if (idx >= ntb->spad_count) 1687250079Scarl return (EINVAL); 1688250079Scarl 1689289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1690255279Scarl *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1691255279Scarl else 1692255279Scarl *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1693250079Scarl 1694250079Scarl return (0); 1695250079Scarl} 1696250079Scarl 1697250079Scarl/** 1698250079Scarl * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1699250079Scarl * @ntb: pointer to ntb_softc instance 1700250079Scarl * @mw: memory window number 1701250079Scarl * 1702250079Scarl * This function provides the base virtual address of the memory window 1703250079Scarl * specified. 1704250079Scarl * 1705250079Scarl * RETURNS: pointer to virtual address, or NULL on error. 1706250079Scarl */ 1707250079Scarlvoid * 1708250079Scarlntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1709250079Scarl{ 1710250079Scarl 1711289539Scem if (mw >= ntb_mw_count(ntb)) 1712250079Scarl return (NULL); 1713250079Scarl 1714289539Scem return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].vbase); 1715250079Scarl} 1716250079Scarl 1717250079Scarlvm_paddr_t 1718250079Scarlntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1719250079Scarl{ 1720250079Scarl 1721289539Scem if (mw >= ntb_mw_count(ntb)) 1722250079Scarl return (0); 1723250079Scarl 1724289539Scem return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].pbase); 1725250079Scarl} 1726250079Scarl 1727250079Scarl/** 1728250079Scarl * ntb_get_mw_size() - return size of NTB memory window 1729250079Scarl * @ntb: pointer to ntb_softc instance 1730250079Scarl * @mw: memory window number 1731250079Scarl * 1732250079Scarl * This function provides the physical size of the memory window specified 1733250079Scarl * 1734250079Scarl * RETURNS: the size of the memory window or zero on error 1735250079Scarl */ 1736250079Scarlu_long 1737250079Scarlntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1738250079Scarl{ 1739250079Scarl 1740289539Scem if (mw >= ntb_mw_count(ntb)) 1741250079Scarl return (0); 1742250079Scarl 1743289539Scem return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].size); 1744250079Scarl} 1745250079Scarl 1746250079Scarl/** 1747250079Scarl * ntb_set_mw_addr - set the memory window address 1748250079Scarl * @ntb: pointer to ntb_softc instance 1749250079Scarl * @mw: memory window number 1750250079Scarl * @addr: base address for data 1751250079Scarl * 1752250079Scarl * This function sets the base physical address of the memory window. This 1753250079Scarl * memory address is where data from the remote system will be transfered into 1754250079Scarl * or out of depending on how the transport is configured. 1755250079Scarl */ 1756250079Scarlvoid 1757250079Scarlntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1758250079Scarl{ 1759250079Scarl 1760289539Scem if (mw >= ntb_mw_count(ntb)) 1761250079Scarl return; 1762250079Scarl 1763289539Scem switch (ntb_mw_to_bar(ntb, mw)) { 1764250079Scarl case NTB_B2B_BAR_1: 1765289255Scem ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr); 1766250079Scarl break; 1767250079Scarl case NTB_B2B_BAR_2: 1768289397Scem if (HAS_FEATURE(NTB_SPLIT_BAR)) 1769289397Scem ntb_reg_write(4, ntb->reg_ofs.bar4_xlat, addr); 1770289397Scem else 1771289397Scem ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr); 1772250079Scarl break; 1773289397Scem case NTB_B2B_BAR_3: 1774289397Scem ntb_reg_write(4, ntb->reg_ofs.bar5_xlat, addr); 1775289397Scem break; 1776289539Scem default: 1777289539Scem KASSERT(false, ("invalid BAR")); 1778289539Scem break; 1779250079Scarl } 1780250079Scarl} 1781250079Scarl 1782250079Scarl/** 1783289255Scem * ntb_ring_doorbell() - Set the doorbell on the secondary/external side 1784250079Scarl * @ntb: pointer to ntb_softc instance 1785250079Scarl * @db: doorbell to ring 1786250079Scarl * 1787250079Scarl * This function allows triggering of a doorbell on the secondary/external 1788250079Scarl * side that will initiate an interrupt on the remote host 1789250079Scarl */ 1790250079Scarlvoid 1791289255Scemntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db) 1792250079Scarl{ 1793289347Scem uint64_t bit; 1794250079Scarl 1795250079Scarl if (ntb->type == NTB_SOC) 1796289347Scem bit = 1 << db; 1797289347Scem else 1798289539Scem bit = ((1 << ntb->db_vec_shift) - 1) << 1799289539Scem (db * ntb->db_vec_shift); 1800289347Scem 1801289538Scem if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1802289347Scem ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit); 1803289347Scem return; 1804289209Scem } 1805289347Scem 1806289539Scem ntb_db_write(ntb, ntb->reg_ofs.rdb, bit); 1807250079Scarl} 1808250079Scarl 1809250079Scarl/** 1810250079Scarl * ntb_query_link_status() - return the hardware link status 1811250079Scarl * @ndev: pointer to ntb_device instance 1812250079Scarl * 1813250079Scarl * Returns true if the hardware is connected to the remote system 1814250079Scarl * 1815250079Scarl * RETURNS: true or false based on the hardware link state 1816250079Scarl */ 1817250079Scarlbool 1818250079Scarlntb_query_link_status(struct ntb_softc *ntb) 1819250079Scarl{ 1820250079Scarl 1821250079Scarl return (ntb->link_status == NTB_LINK_UP); 1822250079Scarl} 1823250079Scarl 1824255272Scarlstatic void 1825255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 1826250079Scarl{ 1827255272Scarl 1828289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 1829289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 1830289209Scem bar->pbase = rman_get_start(bar->pci_resource); 1831289209Scem bar->size = rman_get_size(bar->pci_resource); 1832289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 1833250079Scarl} 1834255268Scarl 1835289209Scemdevice_t 1836289209Scemntb_get_device(struct ntb_softc *ntb) 1837255268Scarl{ 1838255268Scarl 1839255268Scarl return (ntb->device); 1840255268Scarl} 1841289208Scem 1842289208Scem/* Export HW-specific errata information. */ 1843289208Scembool 1844289208Scemntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 1845289208Scem{ 1846289208Scem 1847289208Scem return (HAS_FEATURE(feature)); 1848289208Scem} 1849