ntb_hw_intel.c revision 289281
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3250079Scarl * All rights reserved. 4250079Scarl * 5250079Scarl * Redistribution and use in source and binary forms, with or without 6250079Scarl * modification, are permitted provided that the following conditions 7250079Scarl * are met: 8250079Scarl * 1. Redistributions of source code must retain the above copyright 9250079Scarl * notice, this list of conditions and the following disclaimer. 10250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer in the 12250079Scarl * documentation and/or other materials provided with the distribution. 13250079Scarl * 14250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24250079Scarl * SUCH DAMAGE. 25250079Scarl */ 26250079Scarl 27250079Scarl#include <sys/cdefs.h> 28250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289281 2015-10-14 02:14:45Z cem $"); 29250079Scarl 30250079Scarl#include <sys/param.h> 31250079Scarl#include <sys/kernel.h> 32250079Scarl#include <sys/systm.h> 33250079Scarl#include <sys/bus.h> 34250079Scarl#include <sys/malloc.h> 35250079Scarl#include <sys/module.h> 36250079Scarl#include <sys/queue.h> 37250079Scarl#include <sys/rman.h> 38289207Scem#include <sys/sysctl.h> 39250079Scarl#include <vm/vm.h> 40250079Scarl#include <vm/pmap.h> 41250079Scarl#include <machine/bus.h> 42250079Scarl#include <machine/pmap.h> 43250079Scarl#include <machine/resource.h> 44250079Scarl#include <dev/pci/pcireg.h> 45250079Scarl#include <dev/pci/pcivar.h> 46250079Scarl 47250079Scarl#include "ntb_regs.h" 48250079Scarl#include "ntb_hw.h" 49250079Scarl 50250079Scarl/* 51250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52250079Scarl * allows you to connect two systems using a PCI-e link. 53250079Scarl * 54250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 55250079Scarl * you to send and recieve interrupts, map the memory windows and send and 56250079Scarl * receive messages in the scratch-pad registers. 57250079Scarl * 58250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 59250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 60250079Scarl */ 61250079Scarl 62250079Scarl#define NTB_CONFIG_BAR 0 63250079Scarl#define NTB_B2B_BAR_1 1 64250079Scarl#define NTB_B2B_BAR_2 2 65250079Scarl#define NTB_MAX_BARS 3 66250079Scarl#define NTB_MW_TO_BAR(mw) ((mw) + 1) 67250079Scarl 68250079Scarl#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 69250079Scarl 70250079Scarl#define NTB_HB_TIMEOUT 1 /* second */ 71250079Scarl#define SOC_LINK_RECOVERY_TIME 500 72250079Scarl 73250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74250079Scarl 75250079Scarlenum ntb_device_type { 76250079Scarl NTB_XEON, 77250079Scarl NTB_SOC 78250079Scarl}; 79250079Scarl 80255274Scarl/* Device features and workarounds */ 81255274Scarl#define HAS_FEATURE(feature) \ 82255274Scarl ((ntb->features & (feature)) != 0) 83255274Scarl 84250079Scarlstruct ntb_hw_info { 85250079Scarl uint32_t device_id; 86255274Scarl const char *desc; 87250079Scarl enum ntb_device_type type; 88255274Scarl uint64_t features; 89250079Scarl}; 90250079Scarl 91250079Scarlstruct ntb_pci_bar_info { 92250079Scarl bus_space_tag_t pci_bus_tag; 93250079Scarl bus_space_handle_t pci_bus_handle; 94250079Scarl int pci_resource_id; 95250079Scarl struct resource *pci_resource; 96250079Scarl vm_paddr_t pbase; 97250079Scarl void *vbase; 98250079Scarl u_long size; 99250079Scarl}; 100250079Scarl 101250079Scarlstruct ntb_int_info { 102250079Scarl struct resource *res; 103250079Scarl int rid; 104250079Scarl void *tag; 105250079Scarl}; 106250079Scarl 107250079Scarlstruct ntb_db_cb { 108250079Scarl ntb_db_callback callback; 109250079Scarl unsigned int db_num; 110250079Scarl void *data; 111250079Scarl struct ntb_softc *ntb; 112289281Scem struct callout irq_work; 113250079Scarl}; 114250079Scarl 115250079Scarlstruct ntb_softc { 116250079Scarl device_t device; 117250079Scarl enum ntb_device_type type; 118255274Scarl uint64_t features; 119250079Scarl 120250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 121250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 122250079Scarl uint32_t allocated_interrupts; 123250079Scarl 124250079Scarl struct callout heartbeat_timer; 125250079Scarl struct callout lr_timer; 126250079Scarl 127250079Scarl void *ntb_transport; 128250079Scarl ntb_event_callback event_cb; 129250079Scarl struct ntb_db_cb *db_cb; 130250079Scarl 131250079Scarl struct { 132289208Scem uint8_t max_spads; 133289208Scem uint8_t max_db_bits; 134289208Scem uint8_t msix_cnt; 135250079Scarl } limits; 136250079Scarl struct { 137289255Scem uint32_t ldb; 138289255Scem uint32_t ldb_mask; 139289255Scem uint32_t rdb; 140289255Scem uint32_t bar2_xlat; 141289255Scem uint32_t bar4_xlat; 142250079Scarl uint32_t spad_remote; 143250079Scarl uint32_t spad_local; 144250079Scarl uint32_t lnk_cntl; 145250079Scarl uint32_t lnk_stat; 146250079Scarl uint32_t spci_cmd; 147250079Scarl } reg_ofs; 148250079Scarl uint8_t conn_type; 149250079Scarl uint8_t dev_type; 150250079Scarl uint8_t bits_per_vector; 151250079Scarl uint8_t link_status; 152250079Scarl uint8_t link_width; 153250079Scarl uint8_t link_speed; 154250079Scarl}; 155250079Scarl 156289234Scem#ifdef __i386__ 157289234Scemstatic __inline uint64_t 158289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 159289234Scem bus_size_t offset) 160289234Scem{ 161289234Scem 162289234Scem return (bus_space_read_4(tag, handle, offset) | 163289234Scem ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 164289234Scem} 165289234Scem 166289234Scemstatic __inline void 167289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 168289234Scem bus_size_t offset, uint64_t val) 169289234Scem{ 170289234Scem 171289234Scem bus_space_write_4(tag, handle, offset, val); 172289234Scem bus_space_write_4(tag, handle, offset + 4, val >> 32); 173289234Scem} 174289234Scem#endif 175289234Scem 176255279Scarl#define ntb_bar_read(SIZE, bar, offset) \ 177255279Scarl bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 178255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset)) 179255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \ 180255279Scarl bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 181255279Scarl ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 182255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 183250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 184255279Scarl ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 185255279Scarl#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 186255279Scarl#define ntb_mw_write(SIZE, offset, val) \ 187255279Scarl ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 188250079Scarl 189255272Scarltypedef int (*bar_map_strategy)(struct ntb_softc *ntb, 190255272Scarl struct ntb_pci_bar_info *bar); 191255272Scarl 192250079Scarlstatic int ntb_probe(device_t device); 193250079Scarlstatic int ntb_attach(device_t device); 194250079Scarlstatic int ntb_detach(device_t device); 195255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb); 196255272Scarlstatic int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 197255272Scarl struct ntb_pci_bar_info *bar); 198255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 199255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb, 200255272Scarl struct ntb_pci_bar_info *bar); 201250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 202250079Scarlstatic int ntb_setup_interrupts(struct ntb_softc *ntb); 203250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 204250079Scarlstatic void handle_soc_irq(void *arg); 205250079Scarlstatic void handle_xeon_irq(void *arg); 206250079Scarlstatic void handle_xeon_event_irq(void *arg); 207250079Scarlstatic void ntb_handle_legacy_interrupt(void *arg); 208289281Scemstatic void ntb_irq_work(void *arg); 209289281Scemstatic void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 210289281Scemstatic void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 211250079Scarlstatic int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 212250079Scarlstatic void ntb_free_callbacks(struct ntb_softc *ntb); 213250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 214250079Scarlstatic int ntb_setup_xeon(struct ntb_softc *ntb); 215250079Scarlstatic int ntb_setup_soc(struct ntb_softc *ntb); 216289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb); 217255279Scarlstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 218255279Scarlstatic void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 219250079Scarlstatic void ntb_handle_heartbeat(void *arg); 220250079Scarlstatic void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 221289272Scemstatic void ntb_hw_link_down(struct ntb_softc *ntb); 222289272Scemstatic void ntb_hw_link_up(struct ntb_softc *ntb); 223250079Scarlstatic void recover_soc_link(void *arg); 224250079Scarlstatic int ntb_check_link_status(struct ntb_softc *ntb); 225255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar); 226250079Scarl 227250079Scarlstatic struct ntb_hw_info pci_ids[] = { 228255274Scarl { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 229289233Scem 230289233Scem /* XXX: PS/SS IDs left out until they are supported. */ 231289233Scem { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 232289233Scem NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 233289233Scem { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 234289233Scem NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 235289233Scem { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 236289233Scem NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 237289233Scem | NTB_BAR_SIZE_4K }, 238289233Scem { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 239289233Scem NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 240289233Scem }, 241289233Scem { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 242289233Scem NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 243289233Scem }, 244289233Scem 245255274Scarl { 0x00000000, NULL, NTB_SOC, 0 } 246250079Scarl}; 247250079Scarl 248250079Scarl/* 249250079Scarl * OS <-> Driver interface structures 250250079Scarl */ 251250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 252250079Scarl 253250079Scarlstatic device_method_t ntb_pci_methods[] = { 254250079Scarl /* Device interface */ 255250079Scarl DEVMETHOD(device_probe, ntb_probe), 256250079Scarl DEVMETHOD(device_attach, ntb_attach), 257250079Scarl DEVMETHOD(device_detach, ntb_detach), 258250079Scarl DEVMETHOD_END 259250079Scarl}; 260250079Scarl 261250079Scarlstatic driver_t ntb_pci_driver = { 262250079Scarl "ntb_hw", 263250079Scarl ntb_pci_methods, 264250079Scarl sizeof(struct ntb_softc), 265250079Scarl}; 266250079Scarl 267250079Scarlstatic devclass_t ntb_devclass; 268250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 269250079ScarlMODULE_VERSION(ntb_hw, 1); 270250079Scarl 271289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 272289207Scem 273250079Scarl/* 274250079Scarl * OS <-> Driver linkage functions 275250079Scarl */ 276250079Scarlstatic int 277250079Scarlntb_probe(device_t device) 278250079Scarl{ 279289209Scem struct ntb_hw_info *p; 280250079Scarl 281289209Scem p = ntb_get_device_info(pci_get_devid(device)); 282289209Scem if (p == NULL) 283250079Scarl return (ENXIO); 284289209Scem 285289209Scem device_set_desc(device, p->desc); 286289209Scem return (0); 287250079Scarl} 288250079Scarl 289250079Scarlstatic int 290250079Scarlntb_attach(device_t device) 291250079Scarl{ 292289209Scem struct ntb_softc *ntb; 293289209Scem struct ntb_hw_info *p; 294250079Scarl int error; 295250079Scarl 296289209Scem ntb = DEVICE2SOFTC(device); 297289209Scem p = ntb_get_device_info(pci_get_devid(device)); 298289209Scem 299250079Scarl ntb->device = device; 300250079Scarl ntb->type = p->type; 301255274Scarl ntb->features = p->features; 302250079Scarl 303250079Scarl /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 304283291Sjkim callout_init(&ntb->heartbeat_timer, 1); 305283291Sjkim callout_init(&ntb->lr_timer, 1); 306250079Scarl 307289209Scem error = ntb_map_pci_bars(ntb); 308289209Scem if (error) 309289209Scem goto out; 310289272Scem if (ntb->type == NTB_SOC) 311289272Scem error = ntb_setup_soc(ntb); 312289272Scem else 313289272Scem error = ntb_setup_xeon(ntb); 314289209Scem if (error) 315289209Scem goto out; 316289209Scem error = ntb_setup_interrupts(ntb); 317289209Scem if (error) 318289209Scem goto out; 319250079Scarl 320250079Scarl pci_enable_busmaster(ntb->device); 321250079Scarl 322289209Scemout: 323289209Scem if (error != 0) 324289209Scem ntb_detach(device); 325250079Scarl return (error); 326250079Scarl} 327250079Scarl 328250079Scarlstatic int 329250079Scarlntb_detach(device_t device) 330250079Scarl{ 331289209Scem struct ntb_softc *ntb; 332250079Scarl 333289209Scem ntb = DEVICE2SOFTC(device); 334250079Scarl callout_drain(&ntb->heartbeat_timer); 335250079Scarl callout_drain(&ntb->lr_timer); 336289272Scem if (ntb->type == NTB_XEON) 337289272Scem ntb_teardown_xeon(ntb); 338250079Scarl ntb_teardown_interrupts(ntb); 339250079Scarl ntb_unmap_pci_bar(ntb); 340250079Scarl 341250079Scarl return (0); 342250079Scarl} 343250079Scarl 344250079Scarlstatic int 345255272Scarlntb_map_pci_bars(struct ntb_softc *ntb) 346250079Scarl{ 347255272Scarl int rc; 348250079Scarl 349250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 350255272Scarl rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 351255272Scarl if (rc != 0) 352289209Scem return (rc); 353255272Scarl 354289209Scem ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 355255272Scarl rc = map_pci_bar(ntb, map_memory_window_bar, 356255272Scarl &ntb->bar_info[NTB_B2B_BAR_1]); 357255272Scarl if (rc != 0) 358289209Scem return (rc); 359255272Scarl 360289209Scem ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 361255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 362255279Scarl rc = map_pci_bar(ntb, map_mmr_bar, 363255279Scarl &ntb->bar_info[NTB_B2B_BAR_2]); 364255279Scarl else 365255279Scarl rc = map_pci_bar(ntb, map_memory_window_bar, 366255279Scarl &ntb->bar_info[NTB_B2B_BAR_2]); 367289209Scem return (rc); 368255272Scarl} 369250079Scarl 370255272Scarlstatic int 371255272Scarlmap_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 372255272Scarl struct ntb_pci_bar_info *bar) 373255272Scarl{ 374255272Scarl int rc; 375255272Scarl 376255272Scarl rc = strategy(ntb, bar); 377289209Scem if (rc != 0) 378255272Scarl device_printf(ntb->device, 379255272Scarl "unable to allocate pci resource\n"); 380289209Scem else 381255279Scarl device_printf(ntb->device, 382255272Scarl "Bar size = %lx, v %p, p %p\n", 383289209Scem bar->size, bar->vbase, (void *)(bar->pbase)); 384255272Scarl return (rc); 385255272Scarl} 386255272Scarl 387255272Scarlstatic int 388255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 389255272Scarl{ 390255272Scarl 391255275Scarl bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 392289209Scem &bar->pci_resource_id, RF_ACTIVE); 393255272Scarl if (bar->pci_resource == NULL) 394255272Scarl return (ENXIO); 395289209Scem 396289209Scem save_bar_parameters(bar); 397289209Scem return (0); 398255272Scarl} 399255272Scarl 400255272Scarlstatic int 401255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 402255272Scarl{ 403255272Scarl int rc; 404255276Scarl uint8_t bar_size_bits = 0; 405255272Scarl 406289209Scem bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 407289209Scem &bar->pci_resource_id, RF_ACTIVE); 408250079Scarl 409255272Scarl if (bar->pci_resource == NULL) 410255272Scarl return (ENXIO); 411255276Scarl 412289209Scem save_bar_parameters(bar); 413289209Scem /* 414289209Scem * Ivytown NTB BAR sizes are misreported by the hardware due to a 415289209Scem * hardware issue. To work around this, query the size it should be 416289209Scem * configured to by the device and modify the resource to correspond to 417289209Scem * this new size. The BIOS on systems with this problem is required to 418289209Scem * provide enough address space to allow the driver to make this change 419289209Scem * safely. 420289209Scem * 421289209Scem * Ideally I could have just specified the size when I allocated the 422289209Scem * resource like: 423289209Scem * bus_alloc_resource(ntb->device, 424289209Scem * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 425289209Scem * 1ul << bar_size_bits, RF_ACTIVE); 426289209Scem * but the PCI driver does not honor the size in this call, so we have 427289209Scem * to modify it after the fact. 428289209Scem */ 429289209Scem if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 430289209Scem if (bar->pci_resource_id == PCIR_BAR(2)) 431289209Scem bar_size_bits = pci_read_config(ntb->device, 432289209Scem XEON_PBAR23SZ_OFFSET, 1); 433289209Scem else 434289209Scem bar_size_bits = pci_read_config(ntb->device, 435289209Scem XEON_PBAR45SZ_OFFSET, 1); 436289209Scem 437289209Scem rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 438289209Scem bar->pci_resource, bar->pbase, 439289209Scem bar->pbase + (1ul << bar_size_bits) - 1); 440255272Scarl if (rc != 0) { 441289209Scem device_printf(ntb->device, 442289209Scem "unable to resize bar\n"); 443255272Scarl return (rc); 444250079Scarl } 445289209Scem 446289209Scem save_bar_parameters(bar); 447250079Scarl } 448289209Scem 449289209Scem /* Mark bar region as write combining to improve performance. */ 450289209Scem rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 451289209Scem VM_MEMATTR_WRITE_COMBINING); 452289209Scem if (rc != 0) { 453289209Scem device_printf(ntb->device, 454289209Scem "unable to mark bar as WRITE_COMBINING\n"); 455289209Scem return (rc); 456289209Scem } 457250079Scarl return (0); 458250079Scarl} 459250079Scarl 460250079Scarlstatic void 461250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 462250079Scarl{ 463250079Scarl struct ntb_pci_bar_info *current_bar; 464250079Scarl int i; 465250079Scarl 466250079Scarl for (i = 0; i< NTB_MAX_BARS; i++) { 467250079Scarl current_bar = &ntb->bar_info[i]; 468250079Scarl if (current_bar->pci_resource != NULL) 469250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 470250079Scarl current_bar->pci_resource_id, 471250079Scarl current_bar->pci_resource); 472250079Scarl } 473250079Scarl} 474250079Scarl 475250079Scarlstatic int 476250079Scarlntb_setup_interrupts(struct ntb_softc *ntb) 477250079Scarl{ 478250079Scarl void (*interrupt_handler)(void *); 479250079Scarl void *int_arg; 480289209Scem bool use_msix = false; 481250079Scarl uint32_t num_vectors; 482250079Scarl int i; 483250079Scarl 484250079Scarl ntb->allocated_interrupts = 0; 485250079Scarl /* 486250079Scarl * On SOC, disable all interrupts. On XEON, disable all but Link 487250079Scarl * Interrupt. The rest will be unmasked as callbacks are registered. 488250079Scarl */ 489250079Scarl if (ntb->type == NTB_SOC) 490289255Scem ntb_reg_write(8, ntb->reg_ofs.ldb_mask, ~0); 491250079Scarl else 492289255Scem ntb_reg_write(2, ntb->reg_ofs.ldb_mask, 493289271Scem (uint16_t) ~(1 << XEON_LINK_DB)); 494250079Scarl 495250079Scarl num_vectors = MIN(pci_msix_count(ntb->device), 496250079Scarl ntb->limits.max_db_bits); 497250079Scarl if (num_vectors >= 1) { 498250079Scarl pci_alloc_msix(ntb->device, &num_vectors); 499250079Scarl if (num_vectors >= 4) 500289209Scem use_msix = true; 501250079Scarl } 502250079Scarl 503250079Scarl ntb_create_callbacks(ntb, num_vectors); 504289209Scem if (use_msix == true) { 505250079Scarl for (i = 0; i < num_vectors; i++) { 506250079Scarl ntb->int_info[i].rid = i + 1; 507250079Scarl ntb->int_info[i].res = bus_alloc_resource_any( 508250079Scarl ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 509250079Scarl RF_ACTIVE); 510250079Scarl if (ntb->int_info[i].res == NULL) { 511250079Scarl device_printf(ntb->device, 512250079Scarl "bus_alloc_resource failed\n"); 513289209Scem return (ENOMEM); 514250079Scarl } 515250079Scarl ntb->int_info[i].tag = NULL; 516250079Scarl ntb->allocated_interrupts++; 517250079Scarl if (ntb->type == NTB_SOC) { 518250079Scarl interrupt_handler = handle_soc_irq; 519250079Scarl int_arg = &ntb->db_cb[i]; 520250079Scarl } else { 521250079Scarl if (i == num_vectors - 1) { 522255279Scarl interrupt_handler = 523255279Scarl handle_xeon_event_irq; 524250079Scarl int_arg = ntb; 525250079Scarl } else { 526250079Scarl interrupt_handler = 527250079Scarl handle_xeon_irq; 528250079Scarl int_arg = &ntb->db_cb[i]; 529250079Scarl } 530250079Scarl } 531250079Scarl if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 532250079Scarl INTR_MPSAFE | INTR_TYPE_MISC, NULL, 533250079Scarl interrupt_handler, int_arg, 534250079Scarl &ntb->int_info[i].tag) != 0) { 535250079Scarl device_printf(ntb->device, 536250079Scarl "bus_setup_intr failed\n"); 537250079Scarl return (ENXIO); 538250079Scarl } 539250079Scarl } 540289209Scem } else { 541250079Scarl ntb->int_info[0].rid = 0; 542255279Scarl ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, 543255279Scarl SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 544250079Scarl interrupt_handler = ntb_handle_legacy_interrupt; 545250079Scarl if (ntb->int_info[0].res == NULL) { 546250079Scarl device_printf(ntb->device, 547250079Scarl "bus_alloc_resource failed\n"); 548289209Scem return (ENOMEM); 549250079Scarl } 550250079Scarl ntb->int_info[0].tag = NULL; 551250079Scarl ntb->allocated_interrupts = 1; 552250079Scarl 553250079Scarl if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 554250079Scarl INTR_MPSAFE | INTR_TYPE_MISC, NULL, 555250079Scarl interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 556250079Scarl 557250079Scarl device_printf(ntb->device, "bus_setup_intr failed\n"); 558250079Scarl return (ENXIO); 559250079Scarl } 560250079Scarl } 561250079Scarl 562250079Scarl return (0); 563250079Scarl} 564250079Scarl 565250079Scarlstatic void 566250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 567250079Scarl{ 568250079Scarl struct ntb_int_info *current_int; 569250079Scarl int i; 570250079Scarl 571289209Scem for (i = 0; i < ntb->allocated_interrupts; i++) { 572250079Scarl current_int = &ntb->int_info[i]; 573250079Scarl if (current_int->tag != NULL) 574250079Scarl bus_teardown_intr(ntb->device, current_int->res, 575250079Scarl current_int->tag); 576250079Scarl 577250079Scarl if (current_int->res != NULL) 578250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 579250079Scarl rman_get_rid(current_int->res), current_int->res); 580250079Scarl } 581250079Scarl 582250079Scarl ntb_free_callbacks(ntb); 583250079Scarl pci_release_msi(ntb->device); 584250079Scarl} 585250079Scarl 586250079Scarlstatic void 587289281Scemmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 588289281Scem{ 589289281Scem unsigned long mask; 590289281Scem 591289281Scem mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 592289281Scem mask |= 1 << (idx * ntb->bits_per_vector); 593289281Scem ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 594289281Scem} 595289281Scem 596289281Scemstatic void 597289281Scemunmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 598289281Scem{ 599289281Scem unsigned long mask; 600289281Scem 601289281Scem mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 602289281Scem mask &= ~(1 << (idx * ntb->bits_per_vector)); 603289281Scem ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 604289281Scem} 605289281Scem 606289281Scemstatic void 607250079Scarlhandle_soc_irq(void *arg) 608250079Scarl{ 609250079Scarl struct ntb_db_cb *db_cb = arg; 610250079Scarl struct ntb_softc *ntb = db_cb->ntb; 611250079Scarl 612289255Scem ntb_reg_write(8, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num); 613250079Scarl 614289281Scem if (db_cb->callback != NULL) { 615289281Scem mask_ldb_interrupt(ntb, db_cb->db_num); 616289281Scem callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 617289281Scem } 618250079Scarl} 619250079Scarl 620250079Scarlstatic void 621250079Scarlhandle_xeon_irq(void *arg) 622250079Scarl{ 623250079Scarl struct ntb_db_cb *db_cb = arg; 624250079Scarl struct ntb_softc *ntb = db_cb->ntb; 625250079Scarl 626250079Scarl /* 627250079Scarl * On Xeon, there are 16 bits in the interrupt register 628250079Scarl * but only 4 vectors. So, 5 bits are assigned to the first 3 629250079Scarl * vectors, with the 4th having a single bit for link 630250079Scarl * interrupts. 631250079Scarl */ 632289255Scem ntb_reg_write(2, ntb->reg_ofs.ldb, 633250079Scarl ((1 << ntb->bits_per_vector) - 1) << 634250079Scarl (db_cb->db_num * ntb->bits_per_vector)); 635250079Scarl 636289281Scem if (db_cb->callback != NULL) { 637289281Scem mask_ldb_interrupt(ntb, db_cb->db_num); 638289281Scem callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 639289281Scem } 640250079Scarl} 641250079Scarl 642250079Scarl/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 643250079Scarlstatic void 644250079Scarlhandle_xeon_event_irq(void *arg) 645250079Scarl{ 646250079Scarl struct ntb_softc *ntb = arg; 647250079Scarl int rc; 648250079Scarl 649250079Scarl rc = ntb_check_link_status(ntb); 650250079Scarl if (rc != 0) 651250079Scarl device_printf(ntb->device, "Error determining link status\n"); 652250079Scarl 653250079Scarl /* bit 15 is always the link bit */ 654289271Scem ntb_reg_write(2, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB); 655250079Scarl} 656250079Scarl 657250079Scarlstatic void 658250079Scarlntb_handle_legacy_interrupt(void *arg) 659250079Scarl{ 660250079Scarl struct ntb_softc *ntb = arg; 661250079Scarl unsigned int i = 0; 662289255Scem uint64_t ldb64; 663289255Scem uint16_t ldb16; 664250079Scarl 665250079Scarl if (ntb->type == NTB_SOC) { 666289255Scem ldb64 = ntb_reg_read(8, ntb->reg_ofs.ldb); 667250079Scarl 668289255Scem while (ldb64) { 669289255Scem i = ffs(ldb64); 670289255Scem ldb64 &= ldb64 - 1; 671250079Scarl handle_soc_irq(&ntb->db_cb[i]); 672250079Scarl } 673250079Scarl } else { 674289255Scem ldb16 = ntb_reg_read(2, ntb->reg_ofs.ldb); 675250079Scarl 676289255Scem if ((ldb16 & XEON_DB_HW_LINK) != 0) { 677250079Scarl handle_xeon_event_irq(ntb); 678289255Scem ldb16 &= ~XEON_DB_HW_LINK; 679250079Scarl } 680250079Scarl 681289255Scem while (ldb16 != 0) { 682289255Scem i = ffs(ldb16); 683289255Scem ldb16 &= ldb16 - 1; 684250079Scarl handle_xeon_irq(&ntb->db_cb[i]); 685250079Scarl } 686250079Scarl } 687250079Scarl 688250079Scarl} 689250079Scarl 690250079Scarlstatic int 691250079Scarlntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 692250079Scarl{ 693250079Scarl int i; 694250079Scarl 695289209Scem ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 696250079Scarl M_ZERO | M_WAITOK); 697250079Scarl for (i = 0; i < num_vectors; i++) { 698250079Scarl ntb->db_cb[i].db_num = i; 699250079Scarl ntb->db_cb[i].ntb = ntb; 700250079Scarl } 701250079Scarl 702250079Scarl return (0); 703250079Scarl} 704250079Scarl 705250079Scarlstatic void 706250079Scarlntb_free_callbacks(struct ntb_softc *ntb) 707250079Scarl{ 708250079Scarl int i; 709250079Scarl 710250079Scarl for (i = 0; i < ntb->limits.max_db_bits; i++) 711250079Scarl ntb_unregister_db_callback(ntb, i); 712250079Scarl 713250079Scarl free(ntb->db_cb, M_NTB); 714250079Scarl} 715250079Scarl 716250079Scarlstatic struct ntb_hw_info * 717250079Scarlntb_get_device_info(uint32_t device_id) 718250079Scarl{ 719250079Scarl struct ntb_hw_info *ep = pci_ids; 720250079Scarl 721250079Scarl while (ep->device_id) { 722250079Scarl if (ep->device_id == device_id) 723250079Scarl return (ep); 724250079Scarl ++ep; 725250079Scarl } 726250079Scarl return (NULL); 727250079Scarl} 728250079Scarl 729289272Scemstatic void 730289272Scemntb_teardown_xeon(struct ntb_softc *ntb) 731250079Scarl{ 732250079Scarl 733289272Scem ntb_hw_link_down(ntb); 734250079Scarl} 735250079Scarl 736250079Scarlstatic int 737250079Scarlntb_setup_xeon(struct ntb_softc *ntb) 738250079Scarl{ 739250079Scarl uint8_t val, connection_type; 740250079Scarl 741250079Scarl val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 742250079Scarl 743250079Scarl connection_type = val & XEON_PPD_CONN_TYPE; 744289257Scem 745289257Scem if ((val & XEON_PPD_DEV_TYPE) != 0) 746289257Scem ntb->dev_type = NTB_DEV_USD; 747289257Scem else 748289257Scem ntb->dev_type = NTB_DEV_DSD; 749289257Scem 750289257Scem ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; 751289257Scem ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; 752289257Scem ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 753289257Scem ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET; 754289257Scem ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; 755289257Scem 756250079Scarl switch (connection_type) { 757250079Scarl case NTB_CONN_B2B: 758250079Scarl ntb->conn_type = NTB_CONN_B2B; 759289257Scem 760289257Scem /* 761289257Scem * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored 762289257Scem * with the NTB_REGS_THRU_MW errata mode enabled. (See 763289257Scem * ntb_ring_doorbell() and ntb_read/write_remote_spad().) 764289257Scem */ 765289257Scem ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET; 766289257Scem ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 767289257Scem 768289257Scem ntb->limits.max_spads = XEON_MAX_SPADS; 769250079Scarl break; 770289257Scem 771250079Scarl case NTB_CONN_RP: 772289257Scem /* 773289257Scem * Every Xeon today needs NTB_REGS_THRU_MW, so punt on RP for 774289257Scem * now. 775289257Scem */ 776289257Scem KASSERT(HAS_FEATURE(NTB_REGS_THRU_MW), 777289257Scem ("Xeon without MW errata unimplemented")); 778289257Scem device_printf(ntb->device, 779289257Scem "NTB-RP disabled to due hardware errata.\n"); 780289257Scem return (ENXIO); 781289257Scem 782289257Scem case NTB_CONN_TRANSPARENT: 783250079Scarl default: 784250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 785250079Scarl connection_type); 786250079Scarl return (ENXIO); 787250079Scarl } 788250079Scarl 789289208Scem /* 790289208Scem * There is a Xeon hardware errata related to writes to SDOORBELL or 791289208Scem * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 792289208Scem * which may hang the system. To workaround this use the second memory 793289208Scem * window to access the interrupt and scratch pad registers on the 794289208Scem * remote system. 795289274Scem * 796289274Scem * There is another HW errata on the limit registers -- they can only 797289274Scem * be written when the base register is (?)4GB aligned and < 32-bit. 798289274Scem * This should already be the case based on the driver defaults, but 799289274Scem * write the limit registers first just in case. 800289208Scem */ 801289208Scem if (HAS_FEATURE(NTB_REGS_THRU_MW)) 802289208Scem /* 803289208Scem * Set the Limit register to 4k, the minimum size, to prevent 804289208Scem * an illegal access. 805289208Scem */ 806289208Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 807289208Scem ntb_get_mw_size(ntb, 1) + 0x1000); 808289208Scem else 809289208Scem /* 810289208Scem * Disable the limit register, just in case it is set to 811289208Scem * something silly. 812289208Scem */ 813289208Scem ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 814289208Scem 815289208Scem 816289257Scem ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 817289257Scem ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 818289257Scem ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 819250079Scarl 820289255Scem ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 821250079Scarl ntb->limits.msix_cnt = XEON_MSIX_CNT; 822250079Scarl ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 823250079Scarl 824289271Scem /* 825289271Scem * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 826289271Scem * mirrored to the remote system. Shrink the number of bits by one, 827289271Scem * since bit 14 is the last bit. 828289271Scem * 829289271Scem * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 830289271Scem * anyway. Nor for non-B2B connection types. 831289271Scem */ 832289271Scem if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) && 833289271Scem !HAS_FEATURE(NTB_REGS_THRU_MW) && 834289271Scem connection_type == NTB_CONN_B2B) 835289271Scem ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1; 836289271Scem 837255279Scarl configure_xeon_secondary_side_bars(ntb); 838289209Scem 839250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 840289257Scem if (ntb->conn_type == NTB_CONN_B2B) 841289257Scem ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 842289257Scem PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 843255279Scarl 844255269Scarl /* Enable link training */ 845289272Scem ntb_hw_link_up(ntb); 846250079Scarl 847250079Scarl return (0); 848250079Scarl} 849250079Scarl 850250079Scarlstatic int 851250079Scarlntb_setup_soc(struct ntb_softc *ntb) 852250079Scarl{ 853250079Scarl uint32_t val, connection_type; 854250079Scarl 855250079Scarl val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 856250079Scarl 857250079Scarl connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 858250079Scarl switch (connection_type) { 859250079Scarl case NTB_CONN_B2B: 860250079Scarl ntb->conn_type = NTB_CONN_B2B; 861250079Scarl break; 862250079Scarl default: 863289265Scem device_printf(ntb->device, 864289265Scem "Unsupported NTB configuration (%d)\n", connection_type); 865250079Scarl return (ENXIO); 866250079Scarl } 867250079Scarl 868250079Scarl if ((val & SOC_PPD_DEV_TYPE) != 0) 869250079Scarl ntb->dev_type = NTB_DEV_DSD; 870250079Scarl else 871250079Scarl ntb->dev_type = NTB_DEV_USD; 872250079Scarl 873250079Scarl /* Initiate PCI-E link training */ 874250079Scarl pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 875250079Scarl 4); 876250079Scarl 877289255Scem ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; 878289255Scem ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; 879289265Scem ntb->reg_ofs.rdb = SOC_B2B_DOORBELL_OFFSET; 880289255Scem ntb->reg_ofs.bar2_xlat = SOC_SBAR2XLAT_OFFSET; 881289255Scem ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; 882250079Scarl ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 883250079Scarl ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 884250079Scarl ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 885289265Scem ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 886250079Scarl ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 887250079Scarl 888289265Scem ntb->limits.max_spads = SOC_MAX_SPADS; 889289255Scem ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 890250079Scarl ntb->limits.msix_cnt = SOC_MSIX_CNT; 891250079Scarl ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 892250079Scarl 893250079Scarl /* 894250079Scarl * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 895250079Scarl * resolved. Mask transaction layer internal parity errors. 896250079Scarl */ 897250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 898250079Scarl 899255279Scarl configure_soc_secondary_side_bars(ntb); 900250079Scarl 901250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 902255278Scarl ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 903250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 904289209Scem 905250079Scarl callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 906250079Scarl 907250079Scarl return (0); 908250079Scarl} 909250079Scarl 910255279Scarlstatic void 911255279Scarlconfigure_soc_secondary_side_bars(struct ntb_softc *ntb) 912255279Scarl{ 913255279Scarl 914255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 915255279Scarl ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 916255279Scarl ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 917255279Scarl ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 918255279Scarl ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 919255279Scarl } else { 920255279Scarl ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 921255279Scarl ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 922255279Scarl ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 923255279Scarl ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 924255279Scarl } 925255279Scarl} 926255279Scarl 927255279Scarlstatic void 928255279Scarlconfigure_xeon_secondary_side_bars(struct ntb_softc *ntb) 929255279Scarl{ 930255279Scarl 931255279Scarl if (ntb->dev_type == NTB_DEV_USD) { 932255279Scarl ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 933255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 934255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 935255279Scarl MBAR01_DSD_ADDR); 936289208Scem else { 937255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 938255279Scarl PBAR4XLAT_USD_ADDR); 939289208Scem /* 940289208Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be 941289208Scem * written 32 bits at a time. 942289208Scem */ 943289208Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 944289208Scem MBAR01_DSD_ADDR & 0xffffffff); 945289208Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 946289208Scem MBAR01_DSD_ADDR >> 32); 947289208Scem } 948255279Scarl ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 949255279Scarl ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 950255279Scarl ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 951255279Scarl } else { 952255279Scarl ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 953255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 954255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 955255279Scarl MBAR01_USD_ADDR); 956289208Scem else { 957255279Scarl ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 958255279Scarl PBAR4XLAT_DSD_ADDR); 959289208Scem /* 960289208Scem * B2B_XLAT_OFFSET is a 64-bit register but can only be 961289208Scem * written 32 bits at a time. 962289208Scem */ 963289208Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 964289208Scem MBAR01_USD_ADDR & 0xffffffff); 965289208Scem ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 966289208Scem MBAR01_USD_ADDR >> 32); 967289208Scem } 968255279Scarl ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 969255279Scarl ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 970255279Scarl ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 971255279Scarl } 972255279Scarl} 973255279Scarl 974255281Scarl/* SOC does not have link status interrupt, poll on that platform */ 975250079Scarlstatic void 976250079Scarlntb_handle_heartbeat(void *arg) 977250079Scarl{ 978250079Scarl struct ntb_softc *ntb = arg; 979250079Scarl uint32_t status32; 980289209Scem int rc; 981250079Scarl 982289209Scem rc = ntb_check_link_status(ntb); 983250079Scarl if (rc != 0) 984250079Scarl device_printf(ntb->device, 985250079Scarl "Error determining link status\n"); 986289232Scem 987250079Scarl /* Check to see if a link error is the cause of the link down */ 988250079Scarl if (ntb->link_status == NTB_LINK_DOWN) { 989255278Scarl status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 990250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 991250079Scarl callout_reset(&ntb->lr_timer, 0, recover_soc_link, 992250079Scarl ntb); 993250079Scarl return; 994250079Scarl } 995250079Scarl } 996250079Scarl 997250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 998250079Scarl ntb_handle_heartbeat, ntb); 999250079Scarl} 1000250079Scarl 1001250079Scarlstatic void 1002250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb) 1003250079Scarl{ 1004250079Scarl uint32_t status; 1005250079Scarl 1006250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 1007255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1008255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1009255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1010255278Scarl ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1011250079Scarl 1012250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1013250079Scarl pause("ModPhy", hz / 10); 1014250079Scarl 1015250079Scarl /* Clear AER Errors, write to clear */ 1016255278Scarl status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1017250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1018255278Scarl ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1019250079Scarl 1020250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1021255278Scarl status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1022250079Scarl status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1023255278Scarl ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1024250079Scarl 1025250079Scarl /* Clear DeSkew Buffer error, write to clear */ 1026255278Scarl status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1027250079Scarl status |= SOC_DESKEWSTS_DBERR; 1028255278Scarl ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1029250079Scarl 1030255278Scarl status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1031250079Scarl status &= SOC_IBIST_ERR_OFLOW; 1032255278Scarl ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1033250079Scarl 1034250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 1035255278Scarl status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1036250079Scarl status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1037255278Scarl ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1038250079Scarl} 1039250079Scarl 1040250079Scarlstatic void 1041250079Scarlntb_handle_link_event(struct ntb_softc *ntb, int link_state) 1042250079Scarl{ 1043250079Scarl enum ntb_hw_event event; 1044250079Scarl uint16_t status; 1045250079Scarl 1046250079Scarl if (ntb->link_status == link_state) 1047250079Scarl return; 1048250079Scarl 1049250079Scarl if (link_state == NTB_LINK_UP) { 1050250079Scarl device_printf(ntb->device, "Link Up\n"); 1051250079Scarl ntb->link_status = NTB_LINK_UP; 1052250079Scarl event = NTB_EVENT_HW_LINK_UP; 1053250079Scarl 1054289257Scem if (ntb->type == NTB_SOC || 1055289257Scem ntb->conn_type == NTB_CONN_TRANSPARENT) 1056255278Scarl status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1057250079Scarl else 1058250079Scarl status = pci_read_config(ntb->device, 1059250079Scarl XEON_LINK_STATUS_OFFSET, 2); 1060250079Scarl ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 1061250079Scarl ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 1062250079Scarl device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 1063250079Scarl ntb->link_width, ntb->link_speed); 1064250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1065250079Scarl ntb_handle_heartbeat, ntb); 1066250079Scarl } else { 1067250079Scarl device_printf(ntb->device, "Link Down\n"); 1068250079Scarl ntb->link_status = NTB_LINK_DOWN; 1069250079Scarl event = NTB_EVENT_HW_LINK_DOWN; 1070255281Scarl /* Do not modify link width/speed, we need it in link recovery */ 1071250079Scarl } 1072250079Scarl 1073250079Scarl /* notify the upper layer if we have an event change */ 1074250079Scarl if (ntb->event_cb != NULL) 1075250079Scarl ntb->event_cb(ntb->ntb_transport, event); 1076250079Scarl} 1077250079Scarl 1078250079Scarlstatic void 1079289272Scemntb_hw_link_up(struct ntb_softc *ntb) 1080289272Scem{ 1081289280Scem uint32_t cntl; 1082289272Scem 1083289280Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1084289272Scem ntb_handle_link_event(ntb, NTB_LINK_UP); 1085289280Scem return; 1086289280Scem } 1087289280Scem 1088289280Scem cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1089289280Scem cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1090289280Scem cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1091289280Scem cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP; 1092289280Scem ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1093289272Scem} 1094289272Scem 1095289272Scemstatic void 1096289272Scemntb_hw_link_down(struct ntb_softc *ntb) 1097289272Scem{ 1098289272Scem uint32_t cntl; 1099289272Scem 1100289272Scem if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1101289272Scem ntb_handle_link_event(ntb, NTB_LINK_DOWN); 1102289272Scem return; 1103289272Scem } 1104289272Scem 1105289272Scem cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1106289280Scem cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1107289280Scem cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP); 1108289280Scem cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1109289272Scem ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1110289272Scem} 1111289272Scem 1112289272Scemstatic void 1113250079Scarlrecover_soc_link(void *arg) 1114250079Scarl{ 1115250079Scarl struct ntb_softc *ntb = arg; 1116250079Scarl uint8_t speed, width; 1117250079Scarl uint32_t status32; 1118250079Scarl uint16_t status16; 1119250079Scarl 1120250079Scarl soc_perform_link_restart(ntb); 1121250079Scarl 1122289232Scem /* 1123289232Scem * There is a potential race between the 2 NTB devices recovering at 1124289232Scem * the same time. If the times are the same, the link will not recover 1125289232Scem * and the driver will be stuck in this loop forever. Add a random 1126289232Scem * interval to the recovery time to prevent this race. 1127289232Scem */ 1128289232Scem status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1129289232Scem pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1130289232Scem 1131255278Scarl status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1132250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1133250079Scarl goto retry; 1134250079Scarl 1135255278Scarl status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1136250079Scarl if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1137250079Scarl goto retry; 1138250079Scarl 1139289232Scem status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1140289232Scem if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1141289232Scem goto out; 1142289232Scem 1143255278Scarl status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1144250079Scarl width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 1145250079Scarl speed = (status16 & NTB_LINK_SPEED_MASK); 1146250079Scarl if (ntb->link_width != width || ntb->link_speed != speed) 1147250079Scarl goto retry; 1148250079Scarl 1149289232Scemout: 1150250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1151250079Scarl ntb_handle_heartbeat, ntb); 1152250079Scarl return; 1153250079Scarl 1154250079Scarlretry: 1155250079Scarl callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1156250079Scarl ntb); 1157250079Scarl} 1158250079Scarl 1159250079Scarlstatic int 1160250079Scarlntb_check_link_status(struct ntb_softc *ntb) 1161250079Scarl{ 1162250079Scarl int link_state; 1163250079Scarl uint32_t ntb_cntl; 1164250079Scarl uint16_t status; 1165250079Scarl 1166250079Scarl if (ntb->type == NTB_SOC) { 1167255278Scarl ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1168250079Scarl if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1169250079Scarl link_state = NTB_LINK_DOWN; 1170250079Scarl else 1171250079Scarl link_state = NTB_LINK_UP; 1172250079Scarl } else { 1173250079Scarl status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 1174250079Scarl 2); 1175250079Scarl 1176250079Scarl if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1177250079Scarl link_state = NTB_LINK_UP; 1178250079Scarl else 1179250079Scarl link_state = NTB_LINK_DOWN; 1180250079Scarl } 1181250079Scarl 1182250079Scarl ntb_handle_link_event(ntb, link_state); 1183250079Scarl 1184250079Scarl return (0); 1185250079Scarl} 1186250079Scarl 1187250079Scarl/** 1188250079Scarl * ntb_register_event_callback() - register event callback 1189250079Scarl * @ntb: pointer to ntb_softc instance 1190250079Scarl * @func: callback function to register 1191250079Scarl * 1192250079Scarl * This function registers a callback for any HW driver events such as link 1193250079Scarl * up/down, power management notices and etc. 1194250079Scarl * 1195289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1196250079Scarl */ 1197250079Scarlint 1198250079Scarlntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1199250079Scarl{ 1200250079Scarl 1201250079Scarl if (ntb->event_cb != NULL) 1202250079Scarl return (EINVAL); 1203250079Scarl 1204250079Scarl ntb->event_cb = func; 1205250079Scarl 1206250079Scarl return (0); 1207250079Scarl} 1208250079Scarl 1209250079Scarl/** 1210250079Scarl * ntb_unregister_event_callback() - unregisters the event callback 1211250079Scarl * @ntb: pointer to ntb_softc instance 1212250079Scarl * 1213250079Scarl * This function unregisters the existing callback from transport 1214250079Scarl */ 1215250079Scarlvoid 1216250079Scarlntb_unregister_event_callback(struct ntb_softc *ntb) 1217250079Scarl{ 1218250079Scarl 1219250079Scarl ntb->event_cb = NULL; 1220250079Scarl} 1221250079Scarl 1222289281Scemstatic void 1223289281Scemntb_irq_work(void *arg) 1224289281Scem{ 1225289281Scem struct ntb_db_cb *db_cb = arg; 1226289281Scem struct ntb_softc *ntb; 1227289281Scem int rc; 1228289281Scem 1229289281Scem rc = db_cb->callback(db_cb->data, db_cb->db_num); 1230289281Scem /* Poll if forward progress was made. */ 1231289281Scem if (rc != 0) { 1232289281Scem callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 1233289281Scem return; 1234289281Scem } 1235289281Scem 1236289281Scem /* Unmask interrupt if no progress was made. */ 1237289281Scem ntb = db_cb->ntb; 1238289281Scem unmask_ldb_interrupt(ntb, db_cb->db_num); 1239289281Scem} 1240289281Scem 1241250079Scarl/** 1242250079Scarl * ntb_register_db_callback() - register a callback for doorbell interrupt 1243250079Scarl * @ntb: pointer to ntb_softc instance 1244250079Scarl * @idx: doorbell index to register callback, zero based 1245289266Scem * @data: pointer to be returned to caller with every callback 1246250079Scarl * @func: callback function to register 1247250079Scarl * 1248250079Scarl * This function registers a callback function for the doorbell interrupt 1249250079Scarl * on the primary side. The function will unmask the doorbell as well to 1250250079Scarl * allow interrupt. 1251250079Scarl * 1252289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1253250079Scarl */ 1254250079Scarlint 1255250079Scarlntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1256250079Scarl ntb_db_callback func) 1257250079Scarl{ 1258250079Scarl 1259250079Scarl if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1260250079Scarl device_printf(ntb->device, "Invalid Index.\n"); 1261250079Scarl return (EINVAL); 1262250079Scarl } 1263250079Scarl 1264250079Scarl ntb->db_cb[idx].callback = func; 1265250079Scarl ntb->db_cb[idx].data = data; 1266289281Scem callout_init(&ntb->db_cb[idx].irq_work, 1); 1267250079Scarl 1268289281Scem unmask_ldb_interrupt(ntb, idx); 1269250079Scarl 1270250079Scarl return (0); 1271250079Scarl} 1272250079Scarl 1273250079Scarl/** 1274250079Scarl * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1275250079Scarl * @ntb: pointer to ntb_softc instance 1276250079Scarl * @idx: doorbell index to register callback, zero based 1277250079Scarl * 1278250079Scarl * This function unregisters a callback function for the doorbell interrupt 1279250079Scarl * on the primary side. The function will also mask the said doorbell. 1280250079Scarl */ 1281250079Scarlvoid 1282250079Scarlntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1283250079Scarl{ 1284250079Scarl 1285250079Scarl if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1286250079Scarl return; 1287250079Scarl 1288289281Scem mask_ldb_interrupt(ntb, idx); 1289250079Scarl 1290289281Scem callout_drain(&ntb->db_cb[idx].irq_work); 1291250079Scarl ntb->db_cb[idx].callback = NULL; 1292250079Scarl} 1293250079Scarl 1294250079Scarl/** 1295250079Scarl * ntb_find_transport() - find the transport pointer 1296250079Scarl * @transport: pointer to pci device 1297250079Scarl * 1298250079Scarl * Given the pci device pointer, return the transport pointer passed in when 1299250079Scarl * the transport attached when it was inited. 1300250079Scarl * 1301250079Scarl * RETURNS: pointer to transport. 1302250079Scarl */ 1303250079Scarlvoid * 1304250079Scarlntb_find_transport(struct ntb_softc *ntb) 1305250079Scarl{ 1306250079Scarl 1307250079Scarl return (ntb->ntb_transport); 1308250079Scarl} 1309250079Scarl 1310250079Scarl/** 1311250079Scarl * ntb_register_transport() - Register NTB transport with NTB HW driver 1312250079Scarl * @transport: transport identifier 1313250079Scarl * 1314250079Scarl * This function allows a transport to reserve the hardware driver for 1315250079Scarl * NTB usage. 1316250079Scarl * 1317250079Scarl * RETURNS: pointer to ntb_softc, NULL on error. 1318250079Scarl */ 1319250079Scarlstruct ntb_softc * 1320250079Scarlntb_register_transport(struct ntb_softc *ntb, void *transport) 1321250079Scarl{ 1322250079Scarl 1323250079Scarl /* 1324250079Scarl * TODO: when we have more than one transport, we will need to rewrite 1325250079Scarl * this to prevent race conditions 1326250079Scarl */ 1327250079Scarl if (ntb->ntb_transport != NULL) 1328250079Scarl return (NULL); 1329250079Scarl 1330250079Scarl ntb->ntb_transport = transport; 1331250079Scarl return (ntb); 1332250079Scarl} 1333250079Scarl 1334250079Scarl/** 1335250079Scarl * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1336250079Scarl * @ntb - ntb_softc of the transport to be freed 1337250079Scarl * 1338250079Scarl * This function unregisters the transport from the HW driver and performs any 1339250079Scarl * necessary cleanups. 1340250079Scarl */ 1341250079Scarlvoid 1342250079Scarlntb_unregister_transport(struct ntb_softc *ntb) 1343250079Scarl{ 1344250079Scarl int i; 1345250079Scarl 1346250079Scarl if (ntb->ntb_transport == NULL) 1347250079Scarl return; 1348250079Scarl 1349250079Scarl for (i = 0; i < ntb->allocated_interrupts; i++) 1350250079Scarl ntb_unregister_db_callback(ntb, i); 1351250079Scarl 1352250079Scarl ntb_unregister_event_callback(ntb); 1353250079Scarl ntb->ntb_transport = NULL; 1354250079Scarl} 1355250079Scarl 1356250079Scarl/** 1357250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 1358250079Scarl * @ntb: pointer to ntb_softc instance 1359250079Scarl * 1360250079Scarl * This function returns the max 32bit scratchpad registers usable by the 1361250079Scarl * upper layer. 1362250079Scarl * 1363250079Scarl * RETURNS: total number of scratch pad registers available 1364250079Scarl */ 1365289208Scemuint8_t 1366250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 1367250079Scarl{ 1368250079Scarl 1369250079Scarl return (ntb->limits.max_spads); 1370250079Scarl} 1371250079Scarl 1372250079Scarl/** 1373250079Scarl * ntb_write_local_spad() - write to the secondary scratchpad register 1374250079Scarl * @ntb: pointer to ntb_softc instance 1375250079Scarl * @idx: index to the scratchpad register, 0 based 1376250079Scarl * @val: the data value to put into the register 1377250079Scarl * 1378250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1379250079Scarl * register. The register resides on the secondary (external) side. 1380250079Scarl * 1381289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1382250079Scarl */ 1383250079Scarlint 1384250079Scarlntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1385250079Scarl{ 1386250079Scarl 1387250079Scarl if (idx >= ntb->limits.max_spads) 1388250079Scarl return (EINVAL); 1389250079Scarl 1390255278Scarl ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1391250079Scarl 1392250079Scarl return (0); 1393250079Scarl} 1394250079Scarl 1395250079Scarl/** 1396250079Scarl * ntb_read_local_spad() - read from the primary scratchpad register 1397250079Scarl * @ntb: pointer to ntb_softc instance 1398250079Scarl * @idx: index to scratchpad register, 0 based 1399250079Scarl * @val: pointer to 32bit integer for storing the register value 1400250079Scarl * 1401250079Scarl * This function allows reading of the 32bit scratchpad register on 1402250079Scarl * the primary (internal) side. 1403250079Scarl * 1404289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1405250079Scarl */ 1406250079Scarlint 1407250079Scarlntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1408250079Scarl{ 1409250079Scarl 1410250079Scarl if (idx >= ntb->limits.max_spads) 1411250079Scarl return (EINVAL); 1412250079Scarl 1413255278Scarl *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1414250079Scarl 1415250079Scarl return (0); 1416250079Scarl} 1417250079Scarl 1418250079Scarl/** 1419250079Scarl * ntb_write_remote_spad() - write to the secondary scratchpad register 1420250079Scarl * @ntb: pointer to ntb_softc instance 1421250079Scarl * @idx: index to the scratchpad register, 0 based 1422250079Scarl * @val: the data value to put into the register 1423250079Scarl * 1424250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1425250079Scarl * register. The register resides on the secondary (external) side. 1426250079Scarl * 1427289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1428250079Scarl */ 1429250079Scarlint 1430250079Scarlntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1431250079Scarl{ 1432250079Scarl 1433250079Scarl if (idx >= ntb->limits.max_spads) 1434250079Scarl return (EINVAL); 1435250079Scarl 1436255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1437255279Scarl ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1438255279Scarl else 1439255279Scarl ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1440250079Scarl 1441250079Scarl return (0); 1442250079Scarl} 1443250079Scarl 1444250079Scarl/** 1445250079Scarl * ntb_read_remote_spad() - read from the primary scratchpad register 1446250079Scarl * @ntb: pointer to ntb_softc instance 1447250079Scarl * @idx: index to scratchpad register, 0 based 1448250079Scarl * @val: pointer to 32bit integer for storing the register value 1449250079Scarl * 1450250079Scarl * This function allows reading of the 32bit scratchpad register on 1451250079Scarl * the primary (internal) side. 1452250079Scarl * 1453289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1454250079Scarl */ 1455250079Scarlint 1456250079Scarlntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1457250079Scarl{ 1458250079Scarl 1459250079Scarl if (idx >= ntb->limits.max_spads) 1460250079Scarl return (EINVAL); 1461250079Scarl 1462255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1463255279Scarl *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1464255279Scarl else 1465255279Scarl *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1466250079Scarl 1467250079Scarl return (0); 1468250079Scarl} 1469250079Scarl 1470250079Scarl/** 1471250079Scarl * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1472250079Scarl * @ntb: pointer to ntb_softc instance 1473250079Scarl * @mw: memory window number 1474250079Scarl * 1475250079Scarl * This function provides the base virtual address of the memory window 1476250079Scarl * specified. 1477250079Scarl * 1478250079Scarl * RETURNS: pointer to virtual address, or NULL on error. 1479250079Scarl */ 1480250079Scarlvoid * 1481250079Scarlntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1482250079Scarl{ 1483250079Scarl 1484250079Scarl if (mw >= NTB_NUM_MW) 1485250079Scarl return (NULL); 1486250079Scarl 1487250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1488250079Scarl} 1489250079Scarl 1490250079Scarlvm_paddr_t 1491250079Scarlntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1492250079Scarl{ 1493250079Scarl 1494250079Scarl if (mw >= NTB_NUM_MW) 1495250079Scarl return (0); 1496250079Scarl 1497250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1498250079Scarl} 1499250079Scarl 1500250079Scarl/** 1501250079Scarl * ntb_get_mw_size() - return size of NTB memory window 1502250079Scarl * @ntb: pointer to ntb_softc instance 1503250079Scarl * @mw: memory window number 1504250079Scarl * 1505250079Scarl * This function provides the physical size of the memory window specified 1506250079Scarl * 1507250079Scarl * RETURNS: the size of the memory window or zero on error 1508250079Scarl */ 1509250079Scarlu_long 1510250079Scarlntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1511250079Scarl{ 1512250079Scarl 1513250079Scarl if (mw >= NTB_NUM_MW) 1514250079Scarl return (0); 1515250079Scarl 1516250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1517250079Scarl} 1518250079Scarl 1519250079Scarl/** 1520250079Scarl * ntb_set_mw_addr - set the memory window address 1521250079Scarl * @ntb: pointer to ntb_softc instance 1522250079Scarl * @mw: memory window number 1523250079Scarl * @addr: base address for data 1524250079Scarl * 1525250079Scarl * This function sets the base physical address of the memory window. This 1526250079Scarl * memory address is where data from the remote system will be transfered into 1527250079Scarl * or out of depending on how the transport is configured. 1528250079Scarl */ 1529250079Scarlvoid 1530250079Scarlntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1531250079Scarl{ 1532250079Scarl 1533250079Scarl if (mw >= NTB_NUM_MW) 1534250079Scarl return; 1535250079Scarl 1536250079Scarl switch (NTB_MW_TO_BAR(mw)) { 1537250079Scarl case NTB_B2B_BAR_1: 1538289255Scem ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr); 1539250079Scarl break; 1540250079Scarl case NTB_B2B_BAR_2: 1541289255Scem ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr); 1542250079Scarl break; 1543250079Scarl } 1544250079Scarl} 1545250079Scarl 1546250079Scarl/** 1547289255Scem * ntb_ring_doorbell() - Set the doorbell on the secondary/external side 1548250079Scarl * @ntb: pointer to ntb_softc instance 1549250079Scarl * @db: doorbell to ring 1550250079Scarl * 1551250079Scarl * This function allows triggering of a doorbell on the secondary/external 1552250079Scarl * side that will initiate an interrupt on the remote host 1553250079Scarl * 1554289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1555250079Scarl */ 1556250079Scarlvoid 1557289255Scemntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db) 1558250079Scarl{ 1559250079Scarl 1560250079Scarl if (ntb->type == NTB_SOC) 1561289255Scem ntb_reg_write(8, ntb->reg_ofs.rdb, (uint64_t) 1 << db); 1562289209Scem else { 1563255279Scarl if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1564255279Scarl ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1565255279Scarl ((1 << ntb->bits_per_vector) - 1) << 1566255279Scarl (db * ntb->bits_per_vector)); 1567255279Scarl else 1568289255Scem ntb_reg_write(2, ntb->reg_ofs.rdb, 1569255279Scarl ((1 << ntb->bits_per_vector) - 1) << 1570255279Scarl (db * ntb->bits_per_vector)); 1571289209Scem } 1572250079Scarl} 1573250079Scarl 1574250079Scarl/** 1575250079Scarl * ntb_query_link_status() - return the hardware link status 1576250079Scarl * @ndev: pointer to ntb_device instance 1577250079Scarl * 1578250079Scarl * Returns true if the hardware is connected to the remote system 1579250079Scarl * 1580250079Scarl * RETURNS: true or false based on the hardware link state 1581250079Scarl */ 1582250079Scarlbool 1583250079Scarlntb_query_link_status(struct ntb_softc *ntb) 1584250079Scarl{ 1585250079Scarl 1586250079Scarl return (ntb->link_status == NTB_LINK_UP); 1587250079Scarl} 1588250079Scarl 1589255272Scarlstatic void 1590255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar) 1591250079Scarl{ 1592255272Scarl 1593289209Scem bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 1594289209Scem bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 1595289209Scem bar->pbase = rman_get_start(bar->pci_resource); 1596289209Scem bar->size = rman_get_size(bar->pci_resource); 1597289209Scem bar->vbase = rman_get_virtual(bar->pci_resource); 1598250079Scarl} 1599255268Scarl 1600289209Scemdevice_t 1601289209Scemntb_get_device(struct ntb_softc *ntb) 1602255268Scarl{ 1603255268Scarl 1604255268Scarl return (ntb->device); 1605255268Scarl} 1606289208Scem 1607289208Scem/* Export HW-specific errata information. */ 1608289208Scembool 1609289208Scemntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 1610289208Scem{ 1611289208Scem 1612289208Scem return (HAS_FEATURE(feature)); 1613289208Scem} 1614