ntb_hw_intel.c revision 250079
1250079Scarl/*- 2250079Scarl * Copyright (C) 2013 Intel Corporation 3250079Scarl * All rights reserved. 4250079Scarl * 5250079Scarl * Redistribution and use in source and binary forms, with or without 6250079Scarl * modification, are permitted provided that the following conditions 7250079Scarl * are met: 8250079Scarl * 1. Redistributions of source code must retain the above copyright 9250079Scarl * notice, this list of conditions and the following disclaimer. 10250079Scarl * 2. Redistributions in binary form must reproduce the above copyright 11250079Scarl * notice, this list of conditions and the following disclaimer in the 12250079Scarl * documentation and/or other materials provided with the distribution. 13250079Scarl * 14250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17250079Scarl * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24250079Scarl * SUCH DAMAGE. 25250079Scarl */ 26250079Scarl 27250079Scarl#include <sys/cdefs.h> 28250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 250079 2013-04-29 22:48:53Z carl $"); 29250079Scarl 30250079Scarl#include <sys/param.h> 31250079Scarl#include <sys/kernel.h> 32250079Scarl#include <sys/systm.h> 33250079Scarl#include <sys/bus.h> 34250079Scarl#include <sys/malloc.h> 35250079Scarl#include <sys/module.h> 36250079Scarl#include <sys/queue.h> 37250079Scarl#include <sys/rman.h> 38250079Scarl#include <vm/vm.h> 39250079Scarl#include <vm/pmap.h> 40250079Scarl#include <machine/bus.h> 41250079Scarl#include <machine/pmap.h> 42250079Scarl#include <machine/resource.h> 43250079Scarl#include <dev/pci/pcireg.h> 44250079Scarl#include <dev/pci/pcivar.h> 45250079Scarl 46250079Scarl#include "ntb_regs.h" 47250079Scarl#include "ntb_hw.h" 48250079Scarl 49250079Scarl/* 50250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 51250079Scarl * allows you to connect two systems using a PCI-e link. 52250079Scarl * 53250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows 54250079Scarl * you to send and recieve interrupts, map the memory windows and send and 55250079Scarl * receive messages in the scratch-pad registers. 56250079Scarl * 57250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may 58250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license. 59250079Scarl */ 60250079Scarl 61250079Scarl#define NTB_CONFIG_BAR 0 62250079Scarl#define NTB_B2B_BAR_1 1 63250079Scarl#define NTB_B2B_BAR_2 2 64250079Scarl#define NTB_MAX_BARS 3 65250079Scarl#define NTB_MW_TO_BAR(mw) ((mw) + 1) 66250079Scarl 67250079Scarl#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 68250079Scarl 69250079Scarl#define NTB_HB_TIMEOUT 1 /* second */ 70250079Scarl#define SOC_LINK_RECOVERY_TIME 500 71250079Scarl 72250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 73250079Scarl 74250079Scarlenum ntb_device_type { 75250079Scarl NTB_XEON, 76250079Scarl NTB_SOC 77250079Scarl}; 78250079Scarl 79250079Scarlstruct ntb_hw_info { 80250079Scarl uint32_t device_id; 81250079Scarl enum ntb_device_type type; 82250079Scarl const char *desc; 83250079Scarl}; 84250079Scarl 85250079Scarlstruct ntb_pci_bar_info { 86250079Scarl bus_space_tag_t pci_bus_tag; 87250079Scarl bus_space_handle_t pci_bus_handle; 88250079Scarl int pci_resource_id; 89250079Scarl struct resource *pci_resource; 90250079Scarl vm_paddr_t pbase; 91250079Scarl void *vbase; 92250079Scarl u_long size; 93250079Scarl}; 94250079Scarl 95250079Scarlstruct ntb_int_info { 96250079Scarl struct resource *res; 97250079Scarl int rid; 98250079Scarl void *tag; 99250079Scarl}; 100250079Scarl 101250079Scarlstruct ntb_db_cb { 102250079Scarl ntb_db_callback callback; 103250079Scarl unsigned int db_num; 104250079Scarl void *data; 105250079Scarl struct ntb_softc *ntb; 106250079Scarl}; 107250079Scarl 108250079Scarlstruct ntb_softc { 109250079Scarl device_t device; 110250079Scarl enum ntb_device_type type; 111250079Scarl 112250079Scarl struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 113250079Scarl struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 114250079Scarl uint32_t allocated_interrupts; 115250079Scarl 116250079Scarl struct callout heartbeat_timer; 117250079Scarl struct callout lr_timer; 118250079Scarl 119250079Scarl void *ntb_transport; 120250079Scarl ntb_event_callback event_cb; 121250079Scarl struct ntb_db_cb *db_cb; 122250079Scarl 123250079Scarl struct { 124250079Scarl uint32_t max_spads; 125250079Scarl uint32_t max_db_bits; 126250079Scarl uint32_t msix_cnt; 127250079Scarl } limits; 128250079Scarl struct { 129250079Scarl uint32_t pdb; 130250079Scarl uint32_t pdb_mask; 131250079Scarl uint32_t sdb; 132250079Scarl uint32_t sbar2_xlat; 133250079Scarl uint32_t sbar4_xlat; 134250079Scarl uint32_t spad_remote; 135250079Scarl uint32_t spad_local; 136250079Scarl uint32_t lnk_cntl; 137250079Scarl uint32_t lnk_stat; 138250079Scarl uint32_t spci_cmd; 139250079Scarl } reg_ofs; 140250079Scarl uint8_t conn_type; 141250079Scarl uint8_t dev_type; 142250079Scarl uint8_t bits_per_vector; 143250079Scarl uint8_t link_status; 144250079Scarl uint8_t link_width; 145250079Scarl uint8_t link_speed; 146250079Scarl}; 147250079Scarl 148250079Scarl#define ntb_reg_read(SIZE, offset) \ 149250079Scarl bus_space_read_ ## SIZE (ntb->bar_info[NTB_CONFIG_BAR].pci_bus_tag, \ 150250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_bus_handle, (offset)) 151250079Scarl#define ntb_reg_write(SIZE, offset, val) \ 152250079Scarl bus_space_write_ ## SIZE (ntb->bar_info[NTB_CONFIG_BAR].pci_bus_tag, \ 153250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_bus_handle, (offset), (val)) 154250079Scarl 155250079Scarl#define ntb_read_1(offset) ntb_reg_read(1, (offset)) 156250079Scarl#define ntb_read_2(offset) ntb_reg_read(2, (offset)) 157250079Scarl#define ntb_read_4(offset) ntb_reg_read(4, (offset)) 158250079Scarl#define ntb_read_8(offset) ntb_reg_read(8, (offset)) 159250079Scarl#define ntb_write_1(offset, val) ntb_reg_write(1, (offset), (val)) 160250079Scarl#define ntb_write_2(offset, val) ntb_reg_write(2, (offset), (val)) 161250079Scarl#define ntb_write_4(offset, val) ntb_reg_write(4, (offset), (val)) 162250079Scarl#define ntb_write_8(offset, val) ntb_reg_write(8, (offset), (val)) 163250079Scarl 164250079Scarlstatic int ntb_probe(device_t device); 165250079Scarlstatic int ntb_attach(device_t device); 166250079Scarlstatic int ntb_detach(device_t device); 167250079Scarlstatic int ntb_map_pci_bar(struct ntb_softc *ntb); 168250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb); 169250079Scarlstatic int ntb_setup_interrupts(struct ntb_softc *ntb); 170250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb); 171250079Scarlstatic void handle_soc_irq(void *arg); 172250079Scarlstatic void handle_xeon_irq(void *arg); 173250079Scarlstatic void handle_xeon_event_irq(void *arg); 174250079Scarlstatic void ntb_handle_legacy_interrupt(void *arg); 175250079Scarlstatic int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 176250079Scarlstatic void ntb_free_callbacks(struct ntb_softc *ntb); 177250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 178250079Scarlstatic int ntb_initialize_hw(struct ntb_softc *ntb); 179250079Scarlstatic int ntb_setup_xeon(struct ntb_softc *ntb); 180250079Scarlstatic int ntb_setup_soc(struct ntb_softc *ntb); 181250079Scarlstatic void ntb_handle_heartbeat(void *arg); 182250079Scarlstatic void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 183250079Scarlstatic void recover_soc_link(void *arg); 184250079Scarlstatic int ntb_check_link_status(struct ntb_softc *ntb); 185250079Scarlstatic bool is_bar_for_data_transfer(int bar_num); 186250079Scarl 187250079Scarlstatic struct ntb_hw_info pci_ids[] = { 188250079Scarl { 0x3C0D8086, NTB_XEON, "Xeon E5/Core i7 Non-Transparent Bridge B2B" }, 189250079Scarl { 0x0C4E8086, NTB_SOC, "Atom Processor S1200 NTB Primary B2B" }, 190250079Scarl { 0x0E0D8086, NTB_XEON, "Xeon E5 V2 Non-Transparent Bridge B2B" }, 191250079Scarl { 0x00000000, NTB_SOC, NULL } 192250079Scarl}; 193250079Scarl 194250079Scarl/* 195250079Scarl * OS <-> Driver interface structures 196250079Scarl */ 197250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 198250079Scarl 199250079Scarlstatic device_method_t ntb_pci_methods[] = { 200250079Scarl /* Device interface */ 201250079Scarl DEVMETHOD(device_probe, ntb_probe), 202250079Scarl DEVMETHOD(device_attach, ntb_attach), 203250079Scarl DEVMETHOD(device_detach, ntb_detach), 204250079Scarl DEVMETHOD_END 205250079Scarl}; 206250079Scarl 207250079Scarlstatic driver_t ntb_pci_driver = { 208250079Scarl "ntb_hw", 209250079Scarl ntb_pci_methods, 210250079Scarl sizeof(struct ntb_softc), 211250079Scarl}; 212250079Scarl 213250079Scarlstatic devclass_t ntb_devclass; 214250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 215250079ScarlMODULE_VERSION(ntb_hw, 1); 216250079Scarl 217250079Scarl/* 218250079Scarl * OS <-> Driver linkage functions 219250079Scarl */ 220250079Scarlstatic int 221250079Scarlntb_probe(device_t device) 222250079Scarl{ 223250079Scarl struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device)); 224250079Scarl 225250079Scarl if (p != NULL) { 226250079Scarl device_set_desc(device, p->desc); 227250079Scarl return (0); 228250079Scarl } else 229250079Scarl return (ENXIO); 230250079Scarl} 231250079Scarl 232250079Scarl#define DETACH_ON_ERROR(func) \ 233250079Scarl error = func; \ 234250079Scarl if (error < 0) { \ 235250079Scarl ntb_detach(device); \ 236250079Scarl return (error); \ 237250079Scarl } 238250079Scarl 239250079Scarlstatic int 240250079Scarlntb_attach(device_t device) 241250079Scarl{ 242250079Scarl struct ntb_softc *ntb = DEVICE2SOFTC(device); 243250079Scarl struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device)); 244250079Scarl int error; 245250079Scarl 246250079Scarl ntb->device = device; 247250079Scarl ntb->type = p->type; 248250079Scarl 249250079Scarl /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 250250079Scarl callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE); 251250079Scarl callout_init(&ntb->lr_timer, CALLOUT_MPSAFE); 252250079Scarl 253250079Scarl DETACH_ON_ERROR(ntb_map_pci_bar(ntb)); 254250079Scarl DETACH_ON_ERROR(ntb_initialize_hw(ntb)); 255250079Scarl DETACH_ON_ERROR(ntb_setup_interrupts(ntb)); 256250079Scarl 257250079Scarl pci_enable_busmaster(ntb->device); 258250079Scarl 259250079Scarl return (error); 260250079Scarl} 261250079Scarl 262250079Scarlstatic int 263250079Scarlntb_detach(device_t device) 264250079Scarl{ 265250079Scarl struct ntb_softc *ntb = DEVICE2SOFTC(device); 266250079Scarl 267250079Scarl callout_drain(&ntb->heartbeat_timer); 268250079Scarl callout_drain(&ntb->lr_timer); 269250079Scarl ntb_teardown_interrupts(ntb); 270250079Scarl ntb_unmap_pci_bar(ntb); 271250079Scarl 272250079Scarl return (0); 273250079Scarl} 274250079Scarl 275250079Scarlstatic int 276250079Scarlntb_map_pci_bar(struct ntb_softc *ntb) 277250079Scarl{ 278250079Scarl struct ntb_pci_bar_info *current_bar; 279250079Scarl int rc, i; 280250079Scarl 281250079Scarl ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 282250079Scarl ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 283250079Scarl ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 284250079Scarl 285250079Scarl for (i = 0; i< NTB_MAX_BARS; i++) { 286250079Scarl current_bar = &ntb->bar_info[i]; 287250079Scarl current_bar->pci_resource = 288250079Scarl bus_alloc_resource(ntb->device, 289250079Scarl SYS_RES_MEMORY, 290250079Scarl ¤t_bar->pci_resource_id, 0, ~0, 1, 291250079Scarl RF_ACTIVE); 292250079Scarl 293250079Scarl if (current_bar->pci_resource == NULL) { 294250079Scarl device_printf(ntb->device, 295250079Scarl "unable to allocate pci resource\n"); 296250079Scarl return (ENXIO); 297250079Scarl } 298250079Scarl else { 299250079Scarl current_bar->pci_bus_tag = 300250079Scarl rman_get_bustag(current_bar->pci_resource); 301250079Scarl current_bar->pci_bus_handle = 302250079Scarl rman_get_bushandle(current_bar->pci_resource); 303250079Scarl current_bar->pbase = 304250079Scarl rman_get_start(current_bar->pci_resource); 305250079Scarl current_bar->size = 306250079Scarl rman_get_size(current_bar->pci_resource); 307250079Scarl current_bar->vbase = 308250079Scarl rman_get_virtual(current_bar->pci_resource); 309250079Scarl if (is_bar_for_data_transfer(i)) { 310250079Scarl /* 311250079Scarl * Mark bar region as write combining to improve 312250079Scarl * performance. 313250079Scarl */ 314250079Scarl rc = pmap_change_attr( 315250079Scarl (vm_offset_t)current_bar->vbase, 316250079Scarl current_bar->size, 317250079Scarl VM_MEMATTR_WRITE_COMBINING); 318250079Scarl if (rc != 0) { 319250079Scarl device_printf(ntb->device, 320250079Scarl "Couldn't mark bar as" 321250079Scarl " WRITE_COMBINING\n"); 322250079Scarl return (rc); 323250079Scarl } 324250079Scarl } 325250079Scarl device_printf(ntb->device, 326250079Scarl "Bar size = %lx, v %p, p %p\n", 327250079Scarl current_bar->size, current_bar->vbase, 328250079Scarl (void *)(current_bar->pbase)); 329250079Scarl } 330250079Scarl } 331250079Scarl return (0); 332250079Scarl} 333250079Scarl 334250079Scarlstatic void 335250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb) 336250079Scarl{ 337250079Scarl struct ntb_pci_bar_info *current_bar; 338250079Scarl int i; 339250079Scarl 340250079Scarl for (i = 0; i< NTB_MAX_BARS; i++) { 341250079Scarl current_bar = &ntb->bar_info[i]; 342250079Scarl if (current_bar->pci_resource != NULL) 343250079Scarl bus_release_resource(ntb->device, SYS_RES_MEMORY, 344250079Scarl current_bar->pci_resource_id, 345250079Scarl current_bar->pci_resource); 346250079Scarl } 347250079Scarl} 348250079Scarl 349250079Scarlstatic int 350250079Scarlntb_setup_interrupts(struct ntb_softc *ntb) 351250079Scarl{ 352250079Scarl void (*interrupt_handler)(void *); 353250079Scarl void *int_arg; 354250079Scarl bool use_msix = 0; 355250079Scarl uint32_t num_vectors; 356250079Scarl int i; 357250079Scarl 358250079Scarl ntb->allocated_interrupts = 0; 359250079Scarl /* 360250079Scarl * On SOC, disable all interrupts. On XEON, disable all but Link 361250079Scarl * Interrupt. The rest will be unmasked as callbacks are registered. 362250079Scarl */ 363250079Scarl if (ntb->type == NTB_SOC) 364250079Scarl ntb_write_8(ntb->reg_ofs.pdb_mask, ~0); 365250079Scarl else 366250079Scarl ntb_write_2(ntb->reg_ofs.pdb_mask, 367250079Scarl ~(1 << ntb->limits.max_db_bits)); 368250079Scarl 369250079Scarl num_vectors = MIN(pci_msix_count(ntb->device), 370250079Scarl ntb->limits.max_db_bits); 371250079Scarl if (num_vectors >= 1) { 372250079Scarl pci_alloc_msix(ntb->device, &num_vectors); 373250079Scarl if (num_vectors >= 4) 374250079Scarl use_msix = TRUE; 375250079Scarl } 376250079Scarl 377250079Scarl ntb_create_callbacks(ntb, num_vectors); 378250079Scarl if (use_msix == TRUE) { 379250079Scarl for (i = 0; i < num_vectors; i++) { 380250079Scarl ntb->int_info[i].rid = i + 1; 381250079Scarl ntb->int_info[i].res = bus_alloc_resource_any( 382250079Scarl ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 383250079Scarl RF_ACTIVE); 384250079Scarl if (ntb->int_info[i].res == NULL) { 385250079Scarl device_printf(ntb->device, 386250079Scarl "bus_alloc_resource failed\n"); 387250079Scarl return (-1); 388250079Scarl } 389250079Scarl ntb->int_info[i].tag = NULL; 390250079Scarl ntb->allocated_interrupts++; 391250079Scarl if (ntb->type == NTB_SOC) { 392250079Scarl interrupt_handler = handle_soc_irq; 393250079Scarl int_arg = &ntb->db_cb[i]; 394250079Scarl } else { 395250079Scarl if (i == num_vectors - 1) { 396250079Scarl interrupt_handler = handle_xeon_event_irq; 397250079Scarl int_arg = ntb; 398250079Scarl } else { 399250079Scarl interrupt_handler = 400250079Scarl handle_xeon_irq; 401250079Scarl int_arg = &ntb->db_cb[i]; 402250079Scarl } 403250079Scarl } 404250079Scarl if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 405250079Scarl INTR_MPSAFE | INTR_TYPE_MISC, NULL, 406250079Scarl interrupt_handler, int_arg, 407250079Scarl &ntb->int_info[i].tag) != 0) { 408250079Scarl device_printf(ntb->device, 409250079Scarl "bus_setup_intr failed\n"); 410250079Scarl return (ENXIO); 411250079Scarl } 412250079Scarl } 413250079Scarl } 414250079Scarl else { 415250079Scarl ntb->int_info[0].rid = 0; 416250079Scarl ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 417250079Scarl &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 418250079Scarl interrupt_handler = ntb_handle_legacy_interrupt; 419250079Scarl if (ntb->int_info[0].res == NULL) { 420250079Scarl device_printf(ntb->device, 421250079Scarl "bus_alloc_resource failed\n"); 422250079Scarl return (-1); 423250079Scarl } 424250079Scarl ntb->int_info[0].tag = NULL; 425250079Scarl ntb->allocated_interrupts = 1; 426250079Scarl 427250079Scarl if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 428250079Scarl INTR_MPSAFE | INTR_TYPE_MISC, NULL, 429250079Scarl interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 430250079Scarl 431250079Scarl device_printf(ntb->device, "bus_setup_intr failed\n"); 432250079Scarl return (ENXIO); 433250079Scarl } 434250079Scarl } 435250079Scarl 436250079Scarl return (0); 437250079Scarl} 438250079Scarl 439250079Scarlstatic void 440250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb) 441250079Scarl{ 442250079Scarl struct ntb_int_info *current_int; 443250079Scarl int i; 444250079Scarl 445250079Scarl for (i=0; i<ntb->allocated_interrupts; i++) { 446250079Scarl current_int = &ntb->int_info[i]; 447250079Scarl if (current_int->tag != NULL) 448250079Scarl bus_teardown_intr(ntb->device, current_int->res, 449250079Scarl current_int->tag); 450250079Scarl 451250079Scarl if (current_int->res != NULL) 452250079Scarl bus_release_resource(ntb->device, SYS_RES_IRQ, 453250079Scarl rman_get_rid(current_int->res), current_int->res); 454250079Scarl } 455250079Scarl 456250079Scarl ntb_free_callbacks(ntb); 457250079Scarl pci_release_msi(ntb->device); 458250079Scarl} 459250079Scarl 460250079Scarlstatic void 461250079Scarlhandle_soc_irq(void *arg) 462250079Scarl{ 463250079Scarl struct ntb_db_cb *db_cb = arg; 464250079Scarl struct ntb_softc *ntb = db_cb->ntb; 465250079Scarl 466250079Scarl ntb_write_8(ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num); 467250079Scarl 468250079Scarl if (db_cb->callback != NULL) 469250079Scarl db_cb->callback(db_cb->data, db_cb->db_num); 470250079Scarl} 471250079Scarl 472250079Scarlstatic void 473250079Scarlhandle_xeon_irq(void *arg) 474250079Scarl{ 475250079Scarl struct ntb_db_cb *db_cb = arg; 476250079Scarl struct ntb_softc *ntb = db_cb->ntb; 477250079Scarl 478250079Scarl /* 479250079Scarl * On Xeon, there are 16 bits in the interrupt register 480250079Scarl * but only 4 vectors. So, 5 bits are assigned to the first 3 481250079Scarl * vectors, with the 4th having a single bit for link 482250079Scarl * interrupts. 483250079Scarl */ 484250079Scarl ntb_write_2(ntb->reg_ofs.pdb, 485250079Scarl ((1 << ntb->bits_per_vector) - 1) << 486250079Scarl (db_cb->db_num * ntb->bits_per_vector)); 487250079Scarl 488250079Scarl if (db_cb->callback != NULL) 489250079Scarl db_cb->callback(db_cb->data, db_cb->db_num); 490250079Scarl} 491250079Scarl 492250079Scarl/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 493250079Scarlstatic void 494250079Scarlhandle_xeon_event_irq(void *arg) 495250079Scarl{ 496250079Scarl struct ntb_softc *ntb = arg; 497250079Scarl int rc; 498250079Scarl 499250079Scarl rc = ntb_check_link_status(ntb); 500250079Scarl if (rc != 0) 501250079Scarl device_printf(ntb->device, "Error determining link status\n"); 502250079Scarl 503250079Scarl /* bit 15 is always the link bit */ 504250079Scarl ntb_write_2(ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits); 505250079Scarl} 506250079Scarl 507250079Scarlstatic void 508250079Scarlntb_handle_legacy_interrupt(void *arg) 509250079Scarl{ 510250079Scarl struct ntb_softc *ntb = arg; 511250079Scarl unsigned int i = 0; 512250079Scarl uint64_t pdb64; 513250079Scarl uint16_t pdb16; 514250079Scarl 515250079Scarl if (ntb->type == NTB_SOC) { 516250079Scarl pdb64 = ntb_read_8(ntb->reg_ofs.pdb); 517250079Scarl 518250079Scarl while (pdb64) { 519250079Scarl i = ffs(pdb64); 520250079Scarl pdb64 &= pdb64 - 1; 521250079Scarl handle_soc_irq(&ntb->db_cb[i]); 522250079Scarl } 523250079Scarl } else { 524250079Scarl pdb16 = ntb_read_2(ntb->reg_ofs.pdb); 525250079Scarl 526250079Scarl if ((pdb16 & XEON_DB_HW_LINK) != 0) { 527250079Scarl handle_xeon_event_irq(ntb); 528250079Scarl pdb16 &= ~XEON_DB_HW_LINK; 529250079Scarl } 530250079Scarl 531250079Scarl while (pdb16 != 0) { 532250079Scarl i = ffs(pdb16); 533250079Scarl pdb16 &= pdb16 - 1; 534250079Scarl handle_xeon_irq(&ntb->db_cb[i]); 535250079Scarl } 536250079Scarl } 537250079Scarl 538250079Scarl} 539250079Scarl 540250079Scarlstatic int 541250079Scarlntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 542250079Scarl{ 543250079Scarl int i; 544250079Scarl 545250079Scarl ntb->db_cb = malloc(num_vectors * sizeof(struct ntb_db_cb), M_NTB, 546250079Scarl M_ZERO | M_WAITOK); 547250079Scarl for (i = 0; i < num_vectors; i++) { 548250079Scarl ntb->db_cb[i].db_num = i; 549250079Scarl ntb->db_cb[i].ntb = ntb; 550250079Scarl } 551250079Scarl 552250079Scarl return (0); 553250079Scarl} 554250079Scarl 555250079Scarlstatic void 556250079Scarlntb_free_callbacks(struct ntb_softc *ntb) 557250079Scarl{ 558250079Scarl int i; 559250079Scarl 560250079Scarl for (i = 0; i < ntb->limits.max_db_bits; i++) 561250079Scarl ntb_unregister_db_callback(ntb, i); 562250079Scarl 563250079Scarl free(ntb->db_cb, M_NTB); 564250079Scarl} 565250079Scarl 566250079Scarlstatic struct ntb_hw_info * 567250079Scarlntb_get_device_info(uint32_t device_id) 568250079Scarl{ 569250079Scarl struct ntb_hw_info *ep = pci_ids; 570250079Scarl 571250079Scarl while (ep->device_id) { 572250079Scarl if (ep->device_id == device_id) 573250079Scarl return (ep); 574250079Scarl ++ep; 575250079Scarl } 576250079Scarl return (NULL); 577250079Scarl} 578250079Scarl 579250079Scarlstatic int 580250079Scarlntb_initialize_hw(struct ntb_softc *ntb) 581250079Scarl{ 582250079Scarl 583250079Scarl if (ntb->type == NTB_SOC) 584250079Scarl return (ntb_setup_soc(ntb)); 585250079Scarl else 586250079Scarl return (ntb_setup_xeon(ntb)); 587250079Scarl} 588250079Scarl 589250079Scarlstatic int 590250079Scarlntb_setup_xeon(struct ntb_softc *ntb) 591250079Scarl{ 592250079Scarl uint8_t val, connection_type; 593250079Scarl 594250079Scarl val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 595250079Scarl 596250079Scarl connection_type = val & XEON_PPD_CONN_TYPE; 597250079Scarl switch (connection_type) { 598250079Scarl case NTB_CONN_B2B: 599250079Scarl ntb->conn_type = NTB_CONN_B2B; 600250079Scarl break; 601250079Scarl case NTB_CONN_CLASSIC: 602250079Scarl case NTB_CONN_RP: 603250079Scarl default: 604250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 605250079Scarl connection_type); 606250079Scarl return (ENXIO); 607250079Scarl } 608250079Scarl 609250079Scarl if ((val & XEON_PPD_DEV_TYPE) != 0) 610250079Scarl ntb->dev_type = NTB_DEV_DSD; 611250079Scarl else 612250079Scarl ntb->dev_type = NTB_DEV_USD; 613250079Scarl 614250079Scarl ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET; 615250079Scarl ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET; 616250079Scarl ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET; 617250079Scarl ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET; 618250079Scarl ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 619250079Scarl ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 620250079Scarl ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 621250079Scarl ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 622250079Scarl 623250079Scarl if (ntb->conn_type == NTB_CONN_B2B) { 624250079Scarl ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET; 625250079Scarl ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 626250079Scarl ntb->limits.max_spads = XEON_MAX_SPADS; 627250079Scarl } else { 628250079Scarl ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET; 629250079Scarl ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET; 630250079Scarl ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS; 631250079Scarl } 632250079Scarl 633250079Scarl ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 634250079Scarl ntb->limits.msix_cnt = XEON_MSIX_CNT; 635250079Scarl ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 636250079Scarl 637250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 638250079Scarl ntb_write_2(ntb->reg_ofs.spci_cmd, 639250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 640250079Scarl 641250079Scarl return (0); 642250079Scarl} 643250079Scarl 644250079Scarlstatic int 645250079Scarlntb_setup_soc(struct ntb_softc *ntb) 646250079Scarl{ 647250079Scarl uint32_t val, connection_type; 648250079Scarl 649250079Scarl val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 650250079Scarl 651250079Scarl connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 652250079Scarl switch (connection_type) { 653250079Scarl case NTB_CONN_B2B: 654250079Scarl ntb->conn_type = NTB_CONN_B2B; 655250079Scarl break; 656250079Scarl case NTB_CONN_RP: 657250079Scarl default: 658250079Scarl device_printf(ntb->device, "Connection type %d not supported\n", 659250079Scarl connection_type); 660250079Scarl return (ENXIO); 661250079Scarl } 662250079Scarl 663250079Scarl if ((val & SOC_PPD_DEV_TYPE) != 0) 664250079Scarl ntb->dev_type = NTB_DEV_DSD; 665250079Scarl else 666250079Scarl ntb->dev_type = NTB_DEV_USD; 667250079Scarl 668250079Scarl /* Initiate PCI-E link training */ 669250079Scarl pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 670250079Scarl 4); 671250079Scarl 672250079Scarl ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET; 673250079Scarl ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET; 674250079Scarl ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET; 675250079Scarl ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET; 676250079Scarl ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 677250079Scarl ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 678250079Scarl ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 679250079Scarl ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 680250079Scarl 681250079Scarl if (ntb->conn_type == NTB_CONN_B2B) { 682250079Scarl ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET; 683250079Scarl ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 684250079Scarl ntb->limits.max_spads = SOC_MAX_SPADS; 685250079Scarl } else { 686250079Scarl ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET; 687250079Scarl ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET; 688250079Scarl ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS; 689250079Scarl } 690250079Scarl 691250079Scarl ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 692250079Scarl ntb->limits.msix_cnt = SOC_MSIX_CNT; 693250079Scarl ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 694250079Scarl 695250079Scarl /* 696250079Scarl * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 697250079Scarl * resolved. Mask transaction layer internal parity errors. 698250079Scarl */ 699250079Scarl pci_write_config(ntb->device, 0xFC, 0x4, 4); 700250079Scarl 701250079Scarl /* 702250079Scarl * Some BIOSes aren't filling out the XLAT offsets. 703250079Scarl * Check and correct the issue. 704250079Scarl */ 705250079Scarl if (ntb->dev_type == NTB_DEV_USD) { 706250079Scarl if (ntb_read_8(SOC_PBAR2XLAT_OFFSET) == 0) 707250079Scarl ntb_write_8(SOC_PBAR2XLAT_OFFSET, 708250079Scarl SOC_PBAR2XLAT_USD_ADDR); 709250079Scarl 710250079Scarl if (ntb_read_8(SOC_PBAR4XLAT_OFFSET) == 0) 711250079Scarl ntb_write_8(SOC_PBAR4XLAT_OFFSET, 712250079Scarl SOC_PBAR4XLAT_USD_ADDR); 713250079Scarl 714250079Scarl if (ntb_read_8(SOC_MBAR23_OFFSET) == 0xC) 715250079Scarl ntb_write_8(SOC_MBAR23_OFFSET, SOC_MBAR23_USD_ADDR); 716250079Scarl 717250079Scarl if (ntb_read_8(SOC_MBAR45_OFFSET) == 0xC) 718250079Scarl ntb_write_8(SOC_MBAR45_OFFSET, SOC_MBAR45_USD_ADDR); 719250079Scarl } else { 720250079Scarl if (ntb_read_8(SOC_PBAR2XLAT_OFFSET) == 0) 721250079Scarl ntb_write_8(SOC_PBAR2XLAT_OFFSET, 722250079Scarl SOC_PBAR2XLAT_DSD_ADDR); 723250079Scarl 724250079Scarl if (ntb_read_8(SOC_PBAR4XLAT_OFFSET) == 0) 725250079Scarl ntb_write_8(SOC_PBAR4XLAT_OFFSET, 726250079Scarl SOC_PBAR4XLAT_DSD_ADDR); 727250079Scarl 728250079Scarl if (ntb_read_8(SOC_MBAR23_OFFSET) == 0xC) 729250079Scarl ntb_write_8(SOC_MBAR23_OFFSET, SOC_MBAR23_DSD_ADDR); 730250079Scarl 731250079Scarl if (ntb_read_8(SOC_MBAR45_OFFSET) == 0xC) 732250079Scarl ntb_write_8(SOC_MBAR45_OFFSET, SOC_MBAR45_DSD_ADDR); 733250079Scarl } 734250079Scarl 735250079Scarl /* Enable Bus Master and Memory Space on the secondary side */ 736250079Scarl ntb_write_2(ntb->reg_ofs.spci_cmd, 737250079Scarl PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 738250079Scarl callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 739250079Scarl 740250079Scarl return (0); 741250079Scarl} 742250079Scarl 743250079Scarl/* SOC doesn't have link status interrupt, poll on that platform */ 744250079Scarlstatic void 745250079Scarlntb_handle_heartbeat(void *arg) 746250079Scarl{ 747250079Scarl struct ntb_softc *ntb = arg; 748250079Scarl uint32_t status32; 749250079Scarl int rc = ntb_check_link_status(ntb); 750250079Scarl 751250079Scarl if (rc != 0) 752250079Scarl device_printf(ntb->device, 753250079Scarl "Error determining link status\n"); 754250079Scarl /* Check to see if a link error is the cause of the link down */ 755250079Scarl if (ntb->link_status == NTB_LINK_DOWN) { 756250079Scarl status32 = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET); 757250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 758250079Scarl callout_reset(&ntb->lr_timer, 0, recover_soc_link, 759250079Scarl ntb); 760250079Scarl return; 761250079Scarl } 762250079Scarl } 763250079Scarl 764250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 765250079Scarl ntb_handle_heartbeat, ntb); 766250079Scarl} 767250079Scarl 768250079Scarlstatic void 769250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb) 770250079Scarl{ 771250079Scarl uint32_t status; 772250079Scarl 773250079Scarl /* Driver resets the NTB ModPhy lanes - magic! */ 774250079Scarl ntb_write_1(SOC_MODPHY_PCSREG6, 0xe0); 775250079Scarl ntb_write_1(SOC_MODPHY_PCSREG4, 0x40); 776250079Scarl ntb_write_1(SOC_MODPHY_PCSREG4, 0x60); 777250079Scarl ntb_write_1(SOC_MODPHY_PCSREG6, 0x60); 778250079Scarl 779250079Scarl /* Driver waits 100ms to allow the NTB ModPhy to settle */ 780250079Scarl pause("ModPhy", hz / 10); 781250079Scarl 782250079Scarl /* Clear AER Errors, write to clear */ 783250079Scarl status = ntb_read_4(SOC_ERRCORSTS_OFFSET); 784250079Scarl status &= PCIM_AER_COR_REPLAY_ROLLOVER; 785250079Scarl ntb_write_4(SOC_ERRCORSTS_OFFSET, status); 786250079Scarl 787250079Scarl /* Clear unexpected electrical idle event in LTSSM, write to clear */ 788250079Scarl status = ntb_read_4(SOC_LTSSMERRSTS0_OFFSET); 789250079Scarl status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 790250079Scarl ntb_write_4(SOC_LTSSMERRSTS0_OFFSET, status); 791250079Scarl 792250079Scarl /* Clear DeSkew Buffer error, write to clear */ 793250079Scarl status = ntb_read_4(SOC_DESKEWSTS_OFFSET); 794250079Scarl status |= SOC_DESKEWSTS_DBERR; 795250079Scarl ntb_write_4(SOC_DESKEWSTS_OFFSET, status); 796250079Scarl 797250079Scarl status = ntb_read_4(SOC_IBSTERRRCRVSTS0_OFFSET); 798250079Scarl status &= SOC_IBIST_ERR_OFLOW; 799250079Scarl ntb_write_4(SOC_IBSTERRRCRVSTS0_OFFSET, status); 800250079Scarl 801250079Scarl /* Releases the NTB state machine to allow the link to retrain */ 802250079Scarl status = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET); 803250079Scarl status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 804250079Scarl ntb_write_4(SOC_LTSSMSTATEJMP_OFFSET, status); 805250079Scarl} 806250079Scarl 807250079Scarlstatic void 808250079Scarlntb_handle_link_event(struct ntb_softc *ntb, int link_state) 809250079Scarl{ 810250079Scarl enum ntb_hw_event event; 811250079Scarl uint16_t status; 812250079Scarl 813250079Scarl if (ntb->link_status == link_state) 814250079Scarl return; 815250079Scarl 816250079Scarl if (link_state == NTB_LINK_UP) { 817250079Scarl device_printf(ntb->device, "Link Up\n"); 818250079Scarl ntb->link_status = NTB_LINK_UP; 819250079Scarl event = NTB_EVENT_HW_LINK_UP; 820250079Scarl 821250079Scarl if (ntb->type == NTB_SOC) 822250079Scarl status = ntb_read_2(ntb->reg_ofs.lnk_stat); 823250079Scarl else 824250079Scarl status = pci_read_config(ntb->device, 825250079Scarl XEON_LINK_STATUS_OFFSET, 2); 826250079Scarl ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 827250079Scarl ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 828250079Scarl device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 829250079Scarl ntb->link_width, ntb->link_speed); 830250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 831250079Scarl ntb_handle_heartbeat, ntb); 832250079Scarl } else { 833250079Scarl device_printf(ntb->device, "Link Down\n"); 834250079Scarl ntb->link_status = NTB_LINK_DOWN; 835250079Scarl event = NTB_EVENT_HW_LINK_DOWN; 836250079Scarl /* Don't modify link width/speed, we need it in link recovery */ 837250079Scarl } 838250079Scarl 839250079Scarl /* notify the upper layer if we have an event change */ 840250079Scarl if (ntb->event_cb != NULL) 841250079Scarl ntb->event_cb(ntb->ntb_transport, event); 842250079Scarl} 843250079Scarl 844250079Scarlstatic void 845250079Scarlrecover_soc_link(void *arg) 846250079Scarl{ 847250079Scarl struct ntb_softc *ntb = arg; 848250079Scarl uint8_t speed, width; 849250079Scarl uint32_t status32; 850250079Scarl uint16_t status16; 851250079Scarl 852250079Scarl soc_perform_link_restart(ntb); 853250079Scarl pause("Link", SOC_LINK_RECOVERY_TIME * hz / 1000); 854250079Scarl 855250079Scarl status32 = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET); 856250079Scarl if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 857250079Scarl goto retry; 858250079Scarl 859250079Scarl status32 = ntb_read_4(SOC_IBSTERRRCRVSTS0_OFFSET); 860250079Scarl if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 861250079Scarl goto retry; 862250079Scarl 863250079Scarl status16 = ntb_read_2(ntb->reg_ofs.lnk_stat); 864250079Scarl width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 865250079Scarl speed = (status16 & NTB_LINK_SPEED_MASK); 866250079Scarl if (ntb->link_width != width || ntb->link_speed != speed) 867250079Scarl goto retry; 868250079Scarl 869250079Scarl callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 870250079Scarl ntb_handle_heartbeat, ntb); 871250079Scarl return; 872250079Scarl 873250079Scarlretry: 874250079Scarl callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 875250079Scarl ntb); 876250079Scarl} 877250079Scarl 878250079Scarlstatic int 879250079Scarlntb_check_link_status(struct ntb_softc *ntb) 880250079Scarl{ 881250079Scarl int link_state; 882250079Scarl uint32_t ntb_cntl; 883250079Scarl uint16_t status; 884250079Scarl 885250079Scarl if (ntb->type == NTB_SOC) { 886250079Scarl ntb_cntl = ntb_read_4(ntb->reg_ofs.lnk_cntl); 887250079Scarl if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 888250079Scarl link_state = NTB_LINK_DOWN; 889250079Scarl else 890250079Scarl link_state = NTB_LINK_UP; 891250079Scarl } else { 892250079Scarl status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 893250079Scarl 2); 894250079Scarl 895250079Scarl if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 896250079Scarl link_state = NTB_LINK_UP; 897250079Scarl else 898250079Scarl link_state = NTB_LINK_DOWN; 899250079Scarl } 900250079Scarl 901250079Scarl ntb_handle_link_event(ntb, link_state); 902250079Scarl 903250079Scarl return (0); 904250079Scarl} 905250079Scarl 906250079Scarl/** 907250079Scarl * ntb_register_event_callback() - register event callback 908250079Scarl * @ntb: pointer to ntb_softc instance 909250079Scarl * @func: callback function to register 910250079Scarl * 911250079Scarl * This function registers a callback for any HW driver events such as link 912250079Scarl * up/down, power management notices and etc. 913250079Scarl * 914250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 915250079Scarl */ 916250079Scarlint 917250079Scarlntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 918250079Scarl{ 919250079Scarl 920250079Scarl if (ntb->event_cb != NULL) 921250079Scarl return (EINVAL); 922250079Scarl 923250079Scarl ntb->event_cb = func; 924250079Scarl 925250079Scarl return (0); 926250079Scarl} 927250079Scarl 928250079Scarl/** 929250079Scarl * ntb_unregister_event_callback() - unregisters the event callback 930250079Scarl * @ntb: pointer to ntb_softc instance 931250079Scarl * 932250079Scarl * This function unregisters the existing callback from transport 933250079Scarl */ 934250079Scarlvoid 935250079Scarlntb_unregister_event_callback(struct ntb_softc *ntb) 936250079Scarl{ 937250079Scarl 938250079Scarl ntb->event_cb = NULL; 939250079Scarl} 940250079Scarl 941250079Scarl/** 942250079Scarl * ntb_register_db_callback() - register a callback for doorbell interrupt 943250079Scarl * @ntb: pointer to ntb_softc instance 944250079Scarl * @idx: doorbell index to register callback, zero based 945250079Scarl * @func: callback function to register 946250079Scarl * 947250079Scarl * This function registers a callback function for the doorbell interrupt 948250079Scarl * on the primary side. The function will unmask the doorbell as well to 949250079Scarl * allow interrupt. 950250079Scarl * 951250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 952250079Scarl */ 953250079Scarlint 954250079Scarlntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 955250079Scarl ntb_db_callback func) 956250079Scarl{ 957250079Scarl uint16_t mask; 958250079Scarl 959250079Scarl if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 960250079Scarl device_printf(ntb->device, "Invalid Index.\n"); 961250079Scarl return (EINVAL); 962250079Scarl } 963250079Scarl 964250079Scarl ntb->db_cb[idx].callback = func; 965250079Scarl ntb->db_cb[idx].data = data; 966250079Scarl 967250079Scarl /* unmask interrupt */ 968250079Scarl mask = ntb_read_2(ntb->reg_ofs.pdb_mask); 969250079Scarl mask &= ~(1 << (idx * ntb->bits_per_vector)); 970250079Scarl ntb_write_2(ntb->reg_ofs.pdb_mask, mask); 971250079Scarl 972250079Scarl return (0); 973250079Scarl} 974250079Scarl 975250079Scarl/** 976250079Scarl * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 977250079Scarl * @ntb: pointer to ntb_softc instance 978250079Scarl * @idx: doorbell index to register callback, zero based 979250079Scarl * 980250079Scarl * This function unregisters a callback function for the doorbell interrupt 981250079Scarl * on the primary side. The function will also mask the said doorbell. 982250079Scarl */ 983250079Scarlvoid 984250079Scarlntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 985250079Scarl{ 986250079Scarl unsigned long mask; 987250079Scarl 988250079Scarl if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 989250079Scarl return; 990250079Scarl 991250079Scarl mask = ntb_read_2(ntb->reg_ofs.pdb_mask); 992250079Scarl mask |= 1 << (idx * ntb->bits_per_vector); 993250079Scarl ntb_write_2(ntb->reg_ofs.pdb_mask, mask); 994250079Scarl 995250079Scarl ntb->db_cb[idx].callback = NULL; 996250079Scarl} 997250079Scarl 998250079Scarl/** 999250079Scarl * ntb_find_transport() - find the transport pointer 1000250079Scarl * @transport: pointer to pci device 1001250079Scarl * 1002250079Scarl * Given the pci device pointer, return the transport pointer passed in when 1003250079Scarl * the transport attached when it was inited. 1004250079Scarl * 1005250079Scarl * RETURNS: pointer to transport. 1006250079Scarl */ 1007250079Scarlvoid * 1008250079Scarlntb_find_transport(struct ntb_softc *ntb) 1009250079Scarl{ 1010250079Scarl 1011250079Scarl return (ntb->ntb_transport); 1012250079Scarl} 1013250079Scarl 1014250079Scarl/** 1015250079Scarl * ntb_register_transport() - Register NTB transport with NTB HW driver 1016250079Scarl * @transport: transport identifier 1017250079Scarl * 1018250079Scarl * This function allows a transport to reserve the hardware driver for 1019250079Scarl * NTB usage. 1020250079Scarl * 1021250079Scarl * RETURNS: pointer to ntb_softc, NULL on error. 1022250079Scarl */ 1023250079Scarlstruct ntb_softc * 1024250079Scarlntb_register_transport(struct ntb_softc *ntb, void *transport) 1025250079Scarl{ 1026250079Scarl 1027250079Scarl /* 1028250079Scarl * TODO: when we have more than one transport, we will need to rewrite 1029250079Scarl * this to prevent race conditions 1030250079Scarl */ 1031250079Scarl if (ntb->ntb_transport != NULL) 1032250079Scarl return (NULL); 1033250079Scarl 1034250079Scarl ntb->ntb_transport = transport; 1035250079Scarl return (ntb); 1036250079Scarl} 1037250079Scarl 1038250079Scarl/** 1039250079Scarl * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1040250079Scarl * @ntb - ntb_softc of the transport to be freed 1041250079Scarl * 1042250079Scarl * This function unregisters the transport from the HW driver and performs any 1043250079Scarl * necessary cleanups. 1044250079Scarl */ 1045250079Scarlvoid 1046250079Scarlntb_unregister_transport(struct ntb_softc *ntb) 1047250079Scarl{ 1048250079Scarl int i; 1049250079Scarl 1050250079Scarl if (ntb->ntb_transport == NULL) 1051250079Scarl return; 1052250079Scarl 1053250079Scarl for (i = 0; i < ntb->allocated_interrupts; i++) 1054250079Scarl ntb_unregister_db_callback(ntb, i); 1055250079Scarl 1056250079Scarl ntb_unregister_event_callback(ntb); 1057250079Scarl ntb->ntb_transport = NULL; 1058250079Scarl} 1059250079Scarl 1060250079Scarl/** 1061250079Scarl * ntb_get_max_spads() - get the total scratch regs usable 1062250079Scarl * @ntb: pointer to ntb_softc instance 1063250079Scarl * 1064250079Scarl * This function returns the max 32bit scratchpad registers usable by the 1065250079Scarl * upper layer. 1066250079Scarl * 1067250079Scarl * RETURNS: total number of scratch pad registers available 1068250079Scarl */ 1069250079Scarlint 1070250079Scarlntb_get_max_spads(struct ntb_softc *ntb) 1071250079Scarl{ 1072250079Scarl 1073250079Scarl return (ntb->limits.max_spads); 1074250079Scarl} 1075250079Scarl 1076250079Scarl/** 1077250079Scarl * ntb_write_local_spad() - write to the secondary scratchpad register 1078250079Scarl * @ntb: pointer to ntb_softc instance 1079250079Scarl * @idx: index to the scratchpad register, 0 based 1080250079Scarl * @val: the data value to put into the register 1081250079Scarl * 1082250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1083250079Scarl * register. The register resides on the secondary (external) side. 1084250079Scarl * 1085250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1086250079Scarl */ 1087250079Scarlint 1088250079Scarlntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1089250079Scarl{ 1090250079Scarl 1091250079Scarl if (idx >= ntb->limits.max_spads) 1092250079Scarl return (EINVAL); 1093250079Scarl 1094250079Scarl ntb_write_4(ntb->reg_ofs.spad_local + idx * 4, val); 1095250079Scarl 1096250079Scarl return (0); 1097250079Scarl} 1098250079Scarl 1099250079Scarl/** 1100250079Scarl * ntb_read_local_spad() - read from the primary scratchpad register 1101250079Scarl * @ntb: pointer to ntb_softc instance 1102250079Scarl * @idx: index to scratchpad register, 0 based 1103250079Scarl * @val: pointer to 32bit integer for storing the register value 1104250079Scarl * 1105250079Scarl * This function allows reading of the 32bit scratchpad register on 1106250079Scarl * the primary (internal) side. 1107250079Scarl * 1108250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1109250079Scarl */ 1110250079Scarlint 1111250079Scarlntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1112250079Scarl{ 1113250079Scarl 1114250079Scarl if (idx >= ntb->limits.max_spads) 1115250079Scarl return (EINVAL); 1116250079Scarl 1117250079Scarl *val = ntb_read_4(ntb->reg_ofs.spad_local + idx * 4); 1118250079Scarl 1119250079Scarl return (0); 1120250079Scarl} 1121250079Scarl 1122250079Scarl/** 1123250079Scarl * ntb_write_remote_spad() - write to the secondary scratchpad register 1124250079Scarl * @ntb: pointer to ntb_softc instance 1125250079Scarl * @idx: index to the scratchpad register, 0 based 1126250079Scarl * @val: the data value to put into the register 1127250079Scarl * 1128250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad 1129250079Scarl * register. The register resides on the secondary (external) side. 1130250079Scarl * 1131250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1132250079Scarl */ 1133250079Scarlint 1134250079Scarlntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1135250079Scarl{ 1136250079Scarl 1137250079Scarl if (idx >= ntb->limits.max_spads) 1138250079Scarl return (EINVAL); 1139250079Scarl 1140250079Scarl ntb_write_4(ntb->reg_ofs.spad_remote + idx * 4, val); 1141250079Scarl 1142250079Scarl return (0); 1143250079Scarl} 1144250079Scarl 1145250079Scarl/** 1146250079Scarl * ntb_read_remote_spad() - read from the primary scratchpad register 1147250079Scarl * @ntb: pointer to ntb_softc instance 1148250079Scarl * @idx: index to scratchpad register, 0 based 1149250079Scarl * @val: pointer to 32bit integer for storing the register value 1150250079Scarl * 1151250079Scarl * This function allows reading of the 32bit scratchpad register on 1152250079Scarl * the primary (internal) side. 1153250079Scarl * 1154250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1155250079Scarl */ 1156250079Scarlint 1157250079Scarlntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1158250079Scarl{ 1159250079Scarl 1160250079Scarl if (idx >= ntb->limits.max_spads) 1161250079Scarl return (EINVAL); 1162250079Scarl 1163250079Scarl *val = ntb_read_4(ntb->reg_ofs.spad_remote + idx * 4); 1164250079Scarl 1165250079Scarl return (0); 1166250079Scarl} 1167250079Scarl 1168250079Scarl/** 1169250079Scarl * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1170250079Scarl * @ntb: pointer to ntb_softc instance 1171250079Scarl * @mw: memory window number 1172250079Scarl * 1173250079Scarl * This function provides the base virtual address of the memory window 1174250079Scarl * specified. 1175250079Scarl * 1176250079Scarl * RETURNS: pointer to virtual address, or NULL on error. 1177250079Scarl */ 1178250079Scarlvoid * 1179250079Scarlntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1180250079Scarl{ 1181250079Scarl 1182250079Scarl if (mw >= NTB_NUM_MW) 1183250079Scarl return (NULL); 1184250079Scarl 1185250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1186250079Scarl} 1187250079Scarl 1188250079Scarlvm_paddr_t 1189250079Scarlntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1190250079Scarl{ 1191250079Scarl 1192250079Scarl if (mw >= NTB_NUM_MW) 1193250079Scarl return (0); 1194250079Scarl 1195250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1196250079Scarl} 1197250079Scarl 1198250079Scarl/** 1199250079Scarl * ntb_get_mw_size() - return size of NTB memory window 1200250079Scarl * @ntb: pointer to ntb_softc instance 1201250079Scarl * @mw: memory window number 1202250079Scarl * 1203250079Scarl * This function provides the physical size of the memory window specified 1204250079Scarl * 1205250079Scarl * RETURNS: the size of the memory window or zero on error 1206250079Scarl */ 1207250079Scarlu_long 1208250079Scarlntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1209250079Scarl{ 1210250079Scarl 1211250079Scarl if (mw >= NTB_NUM_MW) 1212250079Scarl return (0); 1213250079Scarl 1214250079Scarl return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1215250079Scarl} 1216250079Scarl 1217250079Scarl/** 1218250079Scarl * ntb_set_mw_addr - set the memory window address 1219250079Scarl * @ntb: pointer to ntb_softc instance 1220250079Scarl * @mw: memory window number 1221250079Scarl * @addr: base address for data 1222250079Scarl * 1223250079Scarl * This function sets the base physical address of the memory window. This 1224250079Scarl * memory address is where data from the remote system will be transfered into 1225250079Scarl * or out of depending on how the transport is configured. 1226250079Scarl */ 1227250079Scarlvoid 1228250079Scarlntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1229250079Scarl{ 1230250079Scarl 1231250079Scarl if (mw >= NTB_NUM_MW) 1232250079Scarl return; 1233250079Scarl 1234250079Scarl switch (NTB_MW_TO_BAR(mw)) { 1235250079Scarl case NTB_B2B_BAR_1: 1236250079Scarl ntb_write_8(ntb->reg_ofs.sbar2_xlat, addr); 1237250079Scarl break; 1238250079Scarl case NTB_B2B_BAR_2: 1239250079Scarl ntb_write_8(ntb->reg_ofs.sbar4_xlat, addr); 1240250079Scarl break; 1241250079Scarl } 1242250079Scarl} 1243250079Scarl 1244250079Scarl/** 1245250079Scarl * ntb_ring_sdb() - Set the doorbell on the secondary/external side 1246250079Scarl * @ntb: pointer to ntb_softc instance 1247250079Scarl * @db: doorbell to ring 1248250079Scarl * 1249250079Scarl * This function allows triggering of a doorbell on the secondary/external 1250250079Scarl * side that will initiate an interrupt on the remote host 1251250079Scarl * 1252250079Scarl * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1253250079Scarl */ 1254250079Scarlvoid 1255250079Scarlntb_ring_sdb(struct ntb_softc *ntb, unsigned int db) 1256250079Scarl{ 1257250079Scarl 1258250079Scarl if (ntb->type == NTB_SOC) 1259250079Scarl ntb_write_8(ntb->reg_ofs.sdb, (uint64_t) 1 << db); 1260250079Scarl else 1261250079Scarl ntb_write_2(ntb->reg_ofs.sdb, 1262250079Scarl ((1 << ntb->bits_per_vector) - 1) << 1263250079Scarl (db * ntb->bits_per_vector)); 1264250079Scarl} 1265250079Scarl 1266250079Scarl/** 1267250079Scarl * ntb_query_link_status() - return the hardware link status 1268250079Scarl * @ndev: pointer to ntb_device instance 1269250079Scarl * 1270250079Scarl * Returns true if the hardware is connected to the remote system 1271250079Scarl * 1272250079Scarl * RETURNS: true or false based on the hardware link state 1273250079Scarl */ 1274250079Scarlbool 1275250079Scarlntb_query_link_status(struct ntb_softc *ntb) 1276250079Scarl{ 1277250079Scarl 1278250079Scarl return (ntb->link_status == NTB_LINK_UP); 1279250079Scarl} 1280250079Scarl 1281250079Scarlstatic bool 1282250079Scarlis_bar_for_data_transfer(int bar_num) 1283250079Scarl{ 1284250079Scarl if ((bar_num > NTB_CONFIG_BAR) && (bar_num < NTB_MAX_BARS)) 1285250079Scarl return true; 1286250079Scarl else 1287250079Scarl return false; 1288250079Scarl} 1289