ntb_hw_intel.c revision 289207
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289207 2015-10-13 03:11:21Z cem $"); 29 30#include <sys/param.h> 31#include <sys/kernel.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/malloc.h> 35#include <sys/module.h> 36#include <sys/queue.h> 37#include <sys/rman.h> 38#include <sys/sysctl.h> 39#include <vm/vm.h> 40#include <vm/pmap.h> 41#include <machine/bus.h> 42#include <machine/pmap.h> 43#include <machine/resource.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46 47#include "ntb_regs.h" 48#include "ntb_hw.h" 49 50/* 51 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52 * allows you to connect two systems using a PCI-e link. 53 * 54 * This module contains the hardware abstraction layer for the NTB. It allows 55 * you to send and recieve interrupts, map the memory windows and send and 56 * receive messages in the scratch-pad registers. 57 * 58 * NOTE: Much of the code in this module is shared with Linux. Any patches may 59 * be picked up and redistributed in Linux with a dual GPL/BSD license. 60 */ 61 62#define NTB_CONFIG_BAR 0 63#define NTB_B2B_BAR_1 1 64#define NTB_B2B_BAR_2 2 65#define NTB_MAX_BARS 3 66#define NTB_MW_TO_BAR(mw) ((mw) + 1) 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define SOC_LINK_RECOVERY_TIME 500 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75enum ntb_device_type { 76 NTB_XEON, 77 NTB_SOC 78}; 79 80/* Device features and workarounds */ 81#define HAS_FEATURE(feature) \ 82 ((ntb->features & (feature)) != 0) 83 84#define NTB_BAR_SIZE_4K (1 << 0) 85#define NTB_REGS_THRU_MW (1 << 1) 86 87struct ntb_hw_info { 88 uint32_t device_id; 89 const char *desc; 90 enum ntb_device_type type; 91 uint64_t features; 92}; 93 94struct ntb_pci_bar_info { 95 bus_space_tag_t pci_bus_tag; 96 bus_space_handle_t pci_bus_handle; 97 int pci_resource_id; 98 struct resource *pci_resource; 99 vm_paddr_t pbase; 100 void *vbase; 101 u_long size; 102}; 103 104struct ntb_int_info { 105 struct resource *res; 106 int rid; 107 void *tag; 108}; 109 110struct ntb_db_cb { 111 ntb_db_callback callback; 112 unsigned int db_num; 113 void *data; 114 struct ntb_softc *ntb; 115}; 116 117struct ntb_softc { 118 device_t device; 119 enum ntb_device_type type; 120 uint64_t features; 121 122 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 123 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 124 uint32_t allocated_interrupts; 125 126 struct callout heartbeat_timer; 127 struct callout lr_timer; 128 129 void *ntb_transport; 130 ntb_event_callback event_cb; 131 struct ntb_db_cb *db_cb; 132 133 struct { 134 uint32_t max_spads; 135 uint32_t max_db_bits; 136 uint32_t msix_cnt; 137 } limits; 138 struct { 139 uint32_t pdb; 140 uint32_t pdb_mask; 141 uint32_t sdb; 142 uint32_t sbar2_xlat; 143 uint32_t sbar4_xlat; 144 uint32_t spad_remote; 145 uint32_t spad_local; 146 uint32_t lnk_cntl; 147 uint32_t lnk_stat; 148 uint32_t spci_cmd; 149 } reg_ofs; 150 uint8_t conn_type; 151 uint8_t dev_type; 152 uint8_t bits_per_vector; 153 uint8_t link_status; 154 uint8_t link_width; 155 uint8_t link_speed; 156}; 157 158#define ntb_bar_read(SIZE, bar, offset) \ 159 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 160 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 161#define ntb_bar_write(SIZE, bar, offset, val) \ 162 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 163 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 164#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 165#define ntb_reg_write(SIZE, offset, val) \ 166 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 167#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 168#define ntb_mw_write(SIZE, offset, val) \ 169 ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 170 171typedef int (*bar_map_strategy)(struct ntb_softc *ntb, 172 struct ntb_pci_bar_info *bar); 173 174static int ntb_probe(device_t device); 175static int ntb_attach(device_t device); 176static int ntb_detach(device_t device); 177static int ntb_map_pci_bars(struct ntb_softc *ntb); 178static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 179 struct ntb_pci_bar_info *bar); 180static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 181static int map_memory_window_bar(struct ntb_softc *ntb, 182 struct ntb_pci_bar_info *bar); 183static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 184static int ntb_setup_interrupts(struct ntb_softc *ntb); 185static void ntb_teardown_interrupts(struct ntb_softc *ntb); 186static void handle_soc_irq(void *arg); 187static void handle_xeon_irq(void *arg); 188static void handle_xeon_event_irq(void *arg); 189static void ntb_handle_legacy_interrupt(void *arg); 190static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 191static void ntb_free_callbacks(struct ntb_softc *ntb); 192static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 193static int ntb_initialize_hw(struct ntb_softc *ntb); 194static int ntb_setup_xeon(struct ntb_softc *ntb); 195static int ntb_setup_soc(struct ntb_softc *ntb); 196static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 197static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 198static void ntb_handle_heartbeat(void *arg); 199static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 200static void recover_soc_link(void *arg); 201static int ntb_check_link_status(struct ntb_softc *ntb); 202static void save_bar_parameters(struct ntb_pci_bar_info *bar); 203 204static struct ntb_hw_info pci_ids[] = { 205 { 0x3C0D8086, "Xeon E5/Core i7 Non-Transparent Bridge B2B", NTB_XEON, 206 NTB_REGS_THRU_MW }, 207 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 208 { 0x0E0D8086, "Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 209 NTB_REGS_THRU_MW | NTB_BAR_SIZE_4K }, 210 { 0x00000000, NULL, NTB_SOC, 0 } 211}; 212 213/* 214 * OS <-> Driver interface structures 215 */ 216MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 217 218static device_method_t ntb_pci_methods[] = { 219 /* Device interface */ 220 DEVMETHOD(device_probe, ntb_probe), 221 DEVMETHOD(device_attach, ntb_attach), 222 DEVMETHOD(device_detach, ntb_detach), 223 DEVMETHOD_END 224}; 225 226static driver_t ntb_pci_driver = { 227 "ntb_hw", 228 ntb_pci_methods, 229 sizeof(struct ntb_softc), 230}; 231 232static devclass_t ntb_devclass; 233DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 234MODULE_VERSION(ntb_hw, 1); 235 236SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 237 238/* 239 * OS <-> Driver linkage functions 240 */ 241static int 242ntb_probe(device_t device) 243{ 244 struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device)); 245 246 if (p != NULL) { 247 device_set_desc(device, p->desc); 248 return (0); 249 } else 250 return (ENXIO); 251} 252 253#define DETACH_ON_ERROR(func) \ 254 error = func; \ 255 if (error < 0) { \ 256 ntb_detach(device); \ 257 return (error); \ 258 } 259 260static int 261ntb_attach(device_t device) 262{ 263 struct ntb_softc *ntb = DEVICE2SOFTC(device); 264 struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device)); 265 int error; 266 267 ntb->device = device; 268 ntb->type = p->type; 269 ntb->features = p->features; 270 271 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 272 callout_init(&ntb->heartbeat_timer, 1); 273 callout_init(&ntb->lr_timer, 1); 274 275 DETACH_ON_ERROR(ntb_map_pci_bars(ntb)); 276 DETACH_ON_ERROR(ntb_initialize_hw(ntb)); 277 DETACH_ON_ERROR(ntb_setup_interrupts(ntb)); 278 279 pci_enable_busmaster(ntb->device); 280 281 return (error); 282} 283 284static int 285ntb_detach(device_t device) 286{ 287 struct ntb_softc *ntb = DEVICE2SOFTC(device); 288 289 callout_drain(&ntb->heartbeat_timer); 290 callout_drain(&ntb->lr_timer); 291 ntb_teardown_interrupts(ntb); 292 ntb_unmap_pci_bar(ntb); 293 294 return (0); 295} 296 297static int 298ntb_map_pci_bars(struct ntb_softc *ntb) 299{ 300 int rc; 301 302 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 303 rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 304 if (rc != 0) 305 return rc; 306 307 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 308 rc = map_pci_bar(ntb, map_memory_window_bar, 309 &ntb->bar_info[NTB_B2B_BAR_1]); 310 if (rc != 0) 311 return rc; 312 313 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 314 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 315 rc = map_pci_bar(ntb, map_mmr_bar, 316 &ntb->bar_info[NTB_B2B_BAR_2]); 317 else 318 rc = map_pci_bar(ntb, map_memory_window_bar, 319 &ntb->bar_info[NTB_B2B_BAR_2]); 320 if (rc != 0) 321 return rc; 322 323 return (0); 324} 325 326static int 327map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 328 struct ntb_pci_bar_info *bar) 329{ 330 int rc; 331 332 rc = strategy(ntb, bar); 333 if (rc != 0) { 334 device_printf(ntb->device, 335 "unable to allocate pci resource\n"); 336 } else { 337 device_printf(ntb->device, 338 "Bar size = %lx, v %p, p %p\n", 339 bar->size, bar->vbase, 340 (void *)(bar->pbase)); 341 } 342 return (rc); 343} 344 345static int 346map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 347{ 348 349 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 350 &bar->pci_resource_id, RF_ACTIVE); 351 352 if (bar->pci_resource == NULL) 353 return (ENXIO); 354 else { 355 save_bar_parameters(bar); 356 return (0); 357 } 358} 359 360static int 361map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 362{ 363 int rc; 364 uint8_t bar_size_bits = 0; 365 366 bar->pci_resource = bus_alloc_resource_any(ntb->device, 367 SYS_RES_MEMORY, &bar->pci_resource_id, RF_ACTIVE); 368 369 if (bar->pci_resource == NULL) 370 return (ENXIO); 371 else { 372 save_bar_parameters(bar); 373 /* 374 * Ivytown NTB BAR sizes are misreported by the hardware due to 375 * a hardware issue. To work around this, query the size it 376 * should be configured to by the device and modify the resource 377 * to correspond to this new size. The BIOS on systems with this 378 * problem is required to provide enough address space to allow 379 * the driver to make this change safely. 380 * 381 * Ideally I could have just specified the size when I allocated 382 * the resource like: 383 * bus_alloc_resource(ntb->device, 384 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 385 * 1ul << bar_size_bits, RF_ACTIVE); 386 * but the PCI driver does not honor the size in this call, so 387 * we have to modify it after the fact. 388 */ 389 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 390 if (bar->pci_resource_id == PCIR_BAR(2)) 391 bar_size_bits = pci_read_config(ntb->device, 392 XEON_PBAR23SZ_OFFSET, 1); 393 else 394 bar_size_bits = pci_read_config(ntb->device, 395 XEON_PBAR45SZ_OFFSET, 1); 396 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 397 bar->pci_resource, bar->pbase, 398 bar->pbase + (1ul << bar_size_bits) - 1); 399 if (rc != 0 ) { 400 device_printf(ntb->device, 401 "unable to resize bar\n"); 402 return (rc); 403 } else 404 save_bar_parameters(bar); 405 } 406 407 /* Mark bar region as write combining to improve performance. */ 408 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 409 VM_MEMATTR_WRITE_COMBINING); 410 if (rc != 0) { 411 device_printf(ntb->device, "unable to mark bar as" 412 " WRITE_COMBINING\n"); 413 return (rc); 414 } 415 } 416 return (0); 417} 418 419static void 420ntb_unmap_pci_bar(struct ntb_softc *ntb) 421{ 422 struct ntb_pci_bar_info *current_bar; 423 int i; 424 425 for (i = 0; i< NTB_MAX_BARS; i++) { 426 current_bar = &ntb->bar_info[i]; 427 if (current_bar->pci_resource != NULL) 428 bus_release_resource(ntb->device, SYS_RES_MEMORY, 429 current_bar->pci_resource_id, 430 current_bar->pci_resource); 431 } 432} 433 434static int 435ntb_setup_interrupts(struct ntb_softc *ntb) 436{ 437 void (*interrupt_handler)(void *); 438 void *int_arg; 439 bool use_msix = 0; 440 uint32_t num_vectors; 441 int i; 442 443 ntb->allocated_interrupts = 0; 444 /* 445 * On SOC, disable all interrupts. On XEON, disable all but Link 446 * Interrupt. The rest will be unmasked as callbacks are registered. 447 */ 448 if (ntb->type == NTB_SOC) 449 ntb_reg_write(8, ntb->reg_ofs.pdb_mask, ~0); 450 else 451 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, 452 ~(1 << ntb->limits.max_db_bits)); 453 454 num_vectors = MIN(pci_msix_count(ntb->device), 455 ntb->limits.max_db_bits); 456 if (num_vectors >= 1) { 457 pci_alloc_msix(ntb->device, &num_vectors); 458 if (num_vectors >= 4) 459 use_msix = TRUE; 460 } 461 462 ntb_create_callbacks(ntb, num_vectors); 463 if (use_msix == TRUE) { 464 for (i = 0; i < num_vectors; i++) { 465 ntb->int_info[i].rid = i + 1; 466 ntb->int_info[i].res = bus_alloc_resource_any( 467 ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 468 RF_ACTIVE); 469 if (ntb->int_info[i].res == NULL) { 470 device_printf(ntb->device, 471 "bus_alloc_resource failed\n"); 472 return (-1); 473 } 474 ntb->int_info[i].tag = NULL; 475 ntb->allocated_interrupts++; 476 if (ntb->type == NTB_SOC) { 477 interrupt_handler = handle_soc_irq; 478 int_arg = &ntb->db_cb[i]; 479 } else { 480 if (i == num_vectors - 1) { 481 interrupt_handler = 482 handle_xeon_event_irq; 483 int_arg = ntb; 484 } else { 485 interrupt_handler = 486 handle_xeon_irq; 487 int_arg = &ntb->db_cb[i]; 488 } 489 } 490 if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 491 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 492 interrupt_handler, int_arg, 493 &ntb->int_info[i].tag) != 0) { 494 device_printf(ntb->device, 495 "bus_setup_intr failed\n"); 496 return (ENXIO); 497 } 498 } 499 } 500 else { 501 ntb->int_info[0].rid = 0; 502 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, 503 SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 504 interrupt_handler = ntb_handle_legacy_interrupt; 505 if (ntb->int_info[0].res == NULL) { 506 device_printf(ntb->device, 507 "bus_alloc_resource failed\n"); 508 return (-1); 509 } 510 ntb->int_info[0].tag = NULL; 511 ntb->allocated_interrupts = 1; 512 513 if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 514 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 515 interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 516 517 device_printf(ntb->device, "bus_setup_intr failed\n"); 518 return (ENXIO); 519 } 520 } 521 522 return (0); 523} 524 525static void 526ntb_teardown_interrupts(struct ntb_softc *ntb) 527{ 528 struct ntb_int_info *current_int; 529 int i; 530 531 for (i=0; i<ntb->allocated_interrupts; i++) { 532 current_int = &ntb->int_info[i]; 533 if (current_int->tag != NULL) 534 bus_teardown_intr(ntb->device, current_int->res, 535 current_int->tag); 536 537 if (current_int->res != NULL) 538 bus_release_resource(ntb->device, SYS_RES_IRQ, 539 rman_get_rid(current_int->res), current_int->res); 540 } 541 542 ntb_free_callbacks(ntb); 543 pci_release_msi(ntb->device); 544} 545 546static void 547handle_soc_irq(void *arg) 548{ 549 struct ntb_db_cb *db_cb = arg; 550 struct ntb_softc *ntb = db_cb->ntb; 551 552 ntb_reg_write(8, ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num); 553 554 if (db_cb->callback != NULL) 555 db_cb->callback(db_cb->data, db_cb->db_num); 556} 557 558static void 559handle_xeon_irq(void *arg) 560{ 561 struct ntb_db_cb *db_cb = arg; 562 struct ntb_softc *ntb = db_cb->ntb; 563 564 /* 565 * On Xeon, there are 16 bits in the interrupt register 566 * but only 4 vectors. So, 5 bits are assigned to the first 3 567 * vectors, with the 4th having a single bit for link 568 * interrupts. 569 */ 570 ntb_reg_write(2, ntb->reg_ofs.pdb, 571 ((1 << ntb->bits_per_vector) - 1) << 572 (db_cb->db_num * ntb->bits_per_vector)); 573 574 if (db_cb->callback != NULL) 575 db_cb->callback(db_cb->data, db_cb->db_num); 576} 577 578/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 579static void 580handle_xeon_event_irq(void *arg) 581{ 582 struct ntb_softc *ntb = arg; 583 int rc; 584 585 rc = ntb_check_link_status(ntb); 586 if (rc != 0) 587 device_printf(ntb->device, "Error determining link status\n"); 588 589 /* bit 15 is always the link bit */ 590 ntb_reg_write(2, ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits); 591} 592 593static void 594ntb_handle_legacy_interrupt(void *arg) 595{ 596 struct ntb_softc *ntb = arg; 597 unsigned int i = 0; 598 uint64_t pdb64; 599 uint16_t pdb16; 600 601 if (ntb->type == NTB_SOC) { 602 pdb64 = ntb_reg_read(8, ntb->reg_ofs.pdb); 603 604 while (pdb64) { 605 i = ffs(pdb64); 606 pdb64 &= pdb64 - 1; 607 handle_soc_irq(&ntb->db_cb[i]); 608 } 609 } else { 610 pdb16 = ntb_reg_read(2, ntb->reg_ofs.pdb); 611 612 if ((pdb16 & XEON_DB_HW_LINK) != 0) { 613 handle_xeon_event_irq(ntb); 614 pdb16 &= ~XEON_DB_HW_LINK; 615 } 616 617 while (pdb16 != 0) { 618 i = ffs(pdb16); 619 pdb16 &= pdb16 - 1; 620 handle_xeon_irq(&ntb->db_cb[i]); 621 } 622 } 623 624} 625 626static int 627ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 628{ 629 int i; 630 631 ntb->db_cb = malloc(num_vectors * sizeof(struct ntb_db_cb), M_NTB, 632 M_ZERO | M_WAITOK); 633 for (i = 0; i < num_vectors; i++) { 634 ntb->db_cb[i].db_num = i; 635 ntb->db_cb[i].ntb = ntb; 636 } 637 638 return (0); 639} 640 641static void 642ntb_free_callbacks(struct ntb_softc *ntb) 643{ 644 int i; 645 646 for (i = 0; i < ntb->limits.max_db_bits; i++) 647 ntb_unregister_db_callback(ntb, i); 648 649 free(ntb->db_cb, M_NTB); 650} 651 652static struct ntb_hw_info * 653ntb_get_device_info(uint32_t device_id) 654{ 655 struct ntb_hw_info *ep = pci_ids; 656 657 while (ep->device_id) { 658 if (ep->device_id == device_id) 659 return (ep); 660 ++ep; 661 } 662 return (NULL); 663} 664 665static int 666ntb_initialize_hw(struct ntb_softc *ntb) 667{ 668 669 if (ntb->type == NTB_SOC) 670 return (ntb_setup_soc(ntb)); 671 else 672 return (ntb_setup_xeon(ntb)); 673} 674 675static int 676ntb_setup_xeon(struct ntb_softc *ntb) 677{ 678 uint8_t val, connection_type; 679 680 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 681 682 connection_type = val & XEON_PPD_CONN_TYPE; 683 switch (connection_type) { 684 case NTB_CONN_B2B: 685 ntb->conn_type = NTB_CONN_B2B; 686 break; 687 case NTB_CONN_CLASSIC: 688 case NTB_CONN_RP: 689 default: 690 device_printf(ntb->device, "Connection type %d not supported\n", 691 connection_type); 692 return (ENXIO); 693 } 694 695 if ((val & XEON_PPD_DEV_TYPE) != 0) 696 ntb->dev_type = NTB_DEV_USD; 697 else 698 ntb->dev_type = NTB_DEV_DSD; 699 700 ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET; 701 ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET; 702 ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET; 703 ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET; 704 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 705 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 706 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 707 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 708 709 if (ntb->conn_type == NTB_CONN_B2B) { 710 ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET; 711 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 712 ntb->limits.max_spads = XEON_MAX_SPADS; 713 } else { 714 ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET; 715 ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET; 716 ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS; 717 } 718 719 ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 720 ntb->limits.msix_cnt = XEON_MSIX_CNT; 721 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 722 723 configure_xeon_secondary_side_bars(ntb); 724 /* Enable Bus Master and Memory Space on the secondary side */ 725 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 726 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 727 728 /* Enable link training */ 729 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, 730 NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP); 731 732 return (0); 733} 734 735static int 736ntb_setup_soc(struct ntb_softc *ntb) 737{ 738 uint32_t val, connection_type; 739 740 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 741 742 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 743 switch (connection_type) { 744 case NTB_CONN_B2B: 745 ntb->conn_type = NTB_CONN_B2B; 746 break; 747 case NTB_CONN_RP: 748 default: 749 device_printf(ntb->device, "Connection type %d not supported\n", 750 connection_type); 751 return (ENXIO); 752 } 753 754 if ((val & SOC_PPD_DEV_TYPE) != 0) 755 ntb->dev_type = NTB_DEV_DSD; 756 else 757 ntb->dev_type = NTB_DEV_USD; 758 759 /* Initiate PCI-E link training */ 760 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 761 4); 762 763 ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET; 764 ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET; 765 ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET; 766 ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET; 767 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 768 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 769 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 770 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 771 772 if (ntb->conn_type == NTB_CONN_B2B) { 773 ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET; 774 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 775 ntb->limits.max_spads = SOC_MAX_SPADS; 776 } else { 777 ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET; 778 ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET; 779 ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS; 780 } 781 782 ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 783 ntb->limits.msix_cnt = SOC_MSIX_CNT; 784 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 785 786 /* 787 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 788 * resolved. Mask transaction layer internal parity errors. 789 */ 790 pci_write_config(ntb->device, 0xFC, 0x4, 4); 791 792 configure_soc_secondary_side_bars(ntb); 793 794 /* Enable Bus Master and Memory Space on the secondary side */ 795 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 796 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 797 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 798 799 return (0); 800} 801 802static void 803configure_soc_secondary_side_bars(struct ntb_softc *ntb) 804{ 805 806 if (ntb->dev_type == NTB_DEV_USD) { 807 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 808 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 809 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 810 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 811 } else { 812 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 813 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 814 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 815 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 816 } 817} 818 819static void 820configure_xeon_secondary_side_bars(struct ntb_softc *ntb) 821{ 822 823 if (ntb->dev_type == NTB_DEV_USD) { 824 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 825 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 826 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 827 MBAR01_DSD_ADDR); 828 else 829 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 830 PBAR4XLAT_USD_ADDR); 831 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 832 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 833 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 834 } else { 835 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 836 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 837 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 838 MBAR01_USD_ADDR); 839 else 840 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 841 PBAR4XLAT_DSD_ADDR); 842 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 843 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 844 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 845 } 846} 847 848/* SOC does not have link status interrupt, poll on that platform */ 849static void 850ntb_handle_heartbeat(void *arg) 851{ 852 struct ntb_softc *ntb = arg; 853 uint32_t status32; 854 int rc = ntb_check_link_status(ntb); 855 856 if (rc != 0) 857 device_printf(ntb->device, 858 "Error determining link status\n"); 859 /* Check to see if a link error is the cause of the link down */ 860 if (ntb->link_status == NTB_LINK_DOWN) { 861 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 862 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 863 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 864 ntb); 865 return; 866 } 867 } 868 869 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 870 ntb_handle_heartbeat, ntb); 871} 872 873static void 874soc_perform_link_restart(struct ntb_softc *ntb) 875{ 876 uint32_t status; 877 878 /* Driver resets the NTB ModPhy lanes - magic! */ 879 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 880 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 881 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 882 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 883 884 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 885 pause("ModPhy", hz / 10); 886 887 /* Clear AER Errors, write to clear */ 888 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 889 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 890 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 891 892 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 893 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 894 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 895 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 896 897 /* Clear DeSkew Buffer error, write to clear */ 898 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 899 status |= SOC_DESKEWSTS_DBERR; 900 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 901 902 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 903 status &= SOC_IBIST_ERR_OFLOW; 904 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 905 906 /* Releases the NTB state machine to allow the link to retrain */ 907 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 908 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 909 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 910} 911 912static void 913ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 914{ 915 enum ntb_hw_event event; 916 uint16_t status; 917 918 if (ntb->link_status == link_state) 919 return; 920 921 if (link_state == NTB_LINK_UP) { 922 device_printf(ntb->device, "Link Up\n"); 923 ntb->link_status = NTB_LINK_UP; 924 event = NTB_EVENT_HW_LINK_UP; 925 926 if (ntb->type == NTB_SOC) 927 status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 928 else 929 status = pci_read_config(ntb->device, 930 XEON_LINK_STATUS_OFFSET, 2); 931 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 932 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 933 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 934 ntb->link_width, ntb->link_speed); 935 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 936 ntb_handle_heartbeat, ntb); 937 } else { 938 device_printf(ntb->device, "Link Down\n"); 939 ntb->link_status = NTB_LINK_DOWN; 940 event = NTB_EVENT_HW_LINK_DOWN; 941 /* Do not modify link width/speed, we need it in link recovery */ 942 } 943 944 /* notify the upper layer if we have an event change */ 945 if (ntb->event_cb != NULL) 946 ntb->event_cb(ntb->ntb_transport, event); 947} 948 949static void 950recover_soc_link(void *arg) 951{ 952 struct ntb_softc *ntb = arg; 953 uint8_t speed, width; 954 uint32_t status32; 955 uint16_t status16; 956 957 soc_perform_link_restart(ntb); 958 pause("Link", SOC_LINK_RECOVERY_TIME * hz / 1000); 959 960 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 961 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 962 goto retry; 963 964 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 965 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 966 goto retry; 967 968 status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 969 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 970 speed = (status16 & NTB_LINK_SPEED_MASK); 971 if (ntb->link_width != width || ntb->link_speed != speed) 972 goto retry; 973 974 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 975 ntb_handle_heartbeat, ntb); 976 return; 977 978retry: 979 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 980 ntb); 981} 982 983static int 984ntb_check_link_status(struct ntb_softc *ntb) 985{ 986 int link_state; 987 uint32_t ntb_cntl; 988 uint16_t status; 989 990 if (ntb->type == NTB_SOC) { 991 ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 992 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 993 link_state = NTB_LINK_DOWN; 994 else 995 link_state = NTB_LINK_UP; 996 } else { 997 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 998 2); 999 1000 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1001 link_state = NTB_LINK_UP; 1002 else 1003 link_state = NTB_LINK_DOWN; 1004 } 1005 1006 ntb_handle_link_event(ntb, link_state); 1007 1008 return (0); 1009} 1010 1011/** 1012 * ntb_register_event_callback() - register event callback 1013 * @ntb: pointer to ntb_softc instance 1014 * @func: callback function to register 1015 * 1016 * This function registers a callback for any HW driver events such as link 1017 * up/down, power management notices and etc. 1018 * 1019 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1020 */ 1021int 1022ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1023{ 1024 1025 if (ntb->event_cb != NULL) 1026 return (EINVAL); 1027 1028 ntb->event_cb = func; 1029 1030 return (0); 1031} 1032 1033/** 1034 * ntb_unregister_event_callback() - unregisters the event callback 1035 * @ntb: pointer to ntb_softc instance 1036 * 1037 * This function unregisters the existing callback from transport 1038 */ 1039void 1040ntb_unregister_event_callback(struct ntb_softc *ntb) 1041{ 1042 1043 ntb->event_cb = NULL; 1044} 1045 1046/** 1047 * ntb_register_db_callback() - register a callback for doorbell interrupt 1048 * @ntb: pointer to ntb_softc instance 1049 * @idx: doorbell index to register callback, zero based 1050 * @func: callback function to register 1051 * 1052 * This function registers a callback function for the doorbell interrupt 1053 * on the primary side. The function will unmask the doorbell as well to 1054 * allow interrupt. 1055 * 1056 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1057 */ 1058int 1059ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1060 ntb_db_callback func) 1061{ 1062 uint16_t mask; 1063 1064 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1065 device_printf(ntb->device, "Invalid Index.\n"); 1066 return (EINVAL); 1067 } 1068 1069 ntb->db_cb[idx].callback = func; 1070 ntb->db_cb[idx].data = data; 1071 1072 /* unmask interrupt */ 1073 mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask); 1074 mask &= ~(1 << (idx * ntb->bits_per_vector)); 1075 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask); 1076 1077 return (0); 1078} 1079 1080/** 1081 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1082 * @ntb: pointer to ntb_softc instance 1083 * @idx: doorbell index to register callback, zero based 1084 * 1085 * This function unregisters a callback function for the doorbell interrupt 1086 * on the primary side. The function will also mask the said doorbell. 1087 */ 1088void 1089ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1090{ 1091 unsigned long mask; 1092 1093 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1094 return; 1095 1096 mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask); 1097 mask |= 1 << (idx * ntb->bits_per_vector); 1098 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask); 1099 1100 ntb->db_cb[idx].callback = NULL; 1101} 1102 1103/** 1104 * ntb_find_transport() - find the transport pointer 1105 * @transport: pointer to pci device 1106 * 1107 * Given the pci device pointer, return the transport pointer passed in when 1108 * the transport attached when it was inited. 1109 * 1110 * RETURNS: pointer to transport. 1111 */ 1112void * 1113ntb_find_transport(struct ntb_softc *ntb) 1114{ 1115 1116 return (ntb->ntb_transport); 1117} 1118 1119/** 1120 * ntb_register_transport() - Register NTB transport with NTB HW driver 1121 * @transport: transport identifier 1122 * 1123 * This function allows a transport to reserve the hardware driver for 1124 * NTB usage. 1125 * 1126 * RETURNS: pointer to ntb_softc, NULL on error. 1127 */ 1128struct ntb_softc * 1129ntb_register_transport(struct ntb_softc *ntb, void *transport) 1130{ 1131 1132 /* 1133 * TODO: when we have more than one transport, we will need to rewrite 1134 * this to prevent race conditions 1135 */ 1136 if (ntb->ntb_transport != NULL) 1137 return (NULL); 1138 1139 ntb->ntb_transport = transport; 1140 return (ntb); 1141} 1142 1143/** 1144 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1145 * @ntb - ntb_softc of the transport to be freed 1146 * 1147 * This function unregisters the transport from the HW driver and performs any 1148 * necessary cleanups. 1149 */ 1150void 1151ntb_unregister_transport(struct ntb_softc *ntb) 1152{ 1153 int i; 1154 1155 if (ntb->ntb_transport == NULL) 1156 return; 1157 1158 for (i = 0; i < ntb->allocated_interrupts; i++) 1159 ntb_unregister_db_callback(ntb, i); 1160 1161 ntb_unregister_event_callback(ntb); 1162 ntb->ntb_transport = NULL; 1163} 1164 1165/** 1166 * ntb_get_max_spads() - get the total scratch regs usable 1167 * @ntb: pointer to ntb_softc instance 1168 * 1169 * This function returns the max 32bit scratchpad registers usable by the 1170 * upper layer. 1171 * 1172 * RETURNS: total number of scratch pad registers available 1173 */ 1174int 1175ntb_get_max_spads(struct ntb_softc *ntb) 1176{ 1177 1178 return (ntb->limits.max_spads); 1179} 1180 1181/** 1182 * ntb_write_local_spad() - write to the secondary scratchpad register 1183 * @ntb: pointer to ntb_softc instance 1184 * @idx: index to the scratchpad register, 0 based 1185 * @val: the data value to put into the register 1186 * 1187 * This function allows writing of a 32bit value to the indexed scratchpad 1188 * register. The register resides on the secondary (external) side. 1189 * 1190 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1191 */ 1192int 1193ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1194{ 1195 1196 if (idx >= ntb->limits.max_spads) 1197 return (EINVAL); 1198 1199 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1200 1201 return (0); 1202} 1203 1204/** 1205 * ntb_read_local_spad() - read from the primary scratchpad register 1206 * @ntb: pointer to ntb_softc instance 1207 * @idx: index to scratchpad register, 0 based 1208 * @val: pointer to 32bit integer for storing the register value 1209 * 1210 * This function allows reading of the 32bit scratchpad register on 1211 * the primary (internal) side. 1212 * 1213 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1214 */ 1215int 1216ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1217{ 1218 1219 if (idx >= ntb->limits.max_spads) 1220 return (EINVAL); 1221 1222 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1223 1224 return (0); 1225} 1226 1227/** 1228 * ntb_write_remote_spad() - write to the secondary scratchpad register 1229 * @ntb: pointer to ntb_softc instance 1230 * @idx: index to the scratchpad register, 0 based 1231 * @val: the data value to put into the register 1232 * 1233 * This function allows writing of a 32bit value to the indexed scratchpad 1234 * register. The register resides on the secondary (external) side. 1235 * 1236 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1237 */ 1238int 1239ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1240{ 1241 1242 if (idx >= ntb->limits.max_spads) 1243 return (EINVAL); 1244 1245 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1246 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1247 else 1248 ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1249 1250 return (0); 1251} 1252 1253/** 1254 * ntb_read_remote_spad() - read from the primary scratchpad register 1255 * @ntb: pointer to ntb_softc instance 1256 * @idx: index to scratchpad register, 0 based 1257 * @val: pointer to 32bit integer for storing the register value 1258 * 1259 * This function allows reading of the 32bit scratchpad register on 1260 * the primary (internal) side. 1261 * 1262 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1263 */ 1264int 1265ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1266{ 1267 1268 if (idx >= ntb->limits.max_spads) 1269 return (EINVAL); 1270 1271 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1272 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1273 else 1274 *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1275 1276 return (0); 1277} 1278 1279/** 1280 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1281 * @ntb: pointer to ntb_softc instance 1282 * @mw: memory window number 1283 * 1284 * This function provides the base virtual address of the memory window 1285 * specified. 1286 * 1287 * RETURNS: pointer to virtual address, or NULL on error. 1288 */ 1289void * 1290ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1291{ 1292 1293 if (mw >= NTB_NUM_MW) 1294 return (NULL); 1295 1296 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1297} 1298 1299vm_paddr_t 1300ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1301{ 1302 1303 if (mw >= NTB_NUM_MW) 1304 return (0); 1305 1306 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1307} 1308 1309/** 1310 * ntb_get_mw_size() - return size of NTB memory window 1311 * @ntb: pointer to ntb_softc instance 1312 * @mw: memory window number 1313 * 1314 * This function provides the physical size of the memory window specified 1315 * 1316 * RETURNS: the size of the memory window or zero on error 1317 */ 1318u_long 1319ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1320{ 1321 1322 if (mw >= NTB_NUM_MW) 1323 return (0); 1324 1325 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1326} 1327 1328/** 1329 * ntb_set_mw_addr - set the memory window address 1330 * @ntb: pointer to ntb_softc instance 1331 * @mw: memory window number 1332 * @addr: base address for data 1333 * 1334 * This function sets the base physical address of the memory window. This 1335 * memory address is where data from the remote system will be transfered into 1336 * or out of depending on how the transport is configured. 1337 */ 1338void 1339ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1340{ 1341 1342 if (mw >= NTB_NUM_MW) 1343 return; 1344 1345 switch (NTB_MW_TO_BAR(mw)) { 1346 case NTB_B2B_BAR_1: 1347 ntb_reg_write(8, ntb->reg_ofs.sbar2_xlat, addr); 1348 break; 1349 case NTB_B2B_BAR_2: 1350 ntb_reg_write(8, ntb->reg_ofs.sbar4_xlat, addr); 1351 break; 1352 } 1353} 1354 1355/** 1356 * ntb_ring_sdb() - Set the doorbell on the secondary/external side 1357 * @ntb: pointer to ntb_softc instance 1358 * @db: doorbell to ring 1359 * 1360 * This function allows triggering of a doorbell on the secondary/external 1361 * side that will initiate an interrupt on the remote host 1362 * 1363 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1364 */ 1365void 1366ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db) 1367{ 1368 1369 if (ntb->type == NTB_SOC) 1370 ntb_reg_write(8, ntb->reg_ofs.sdb, (uint64_t) 1 << db); 1371 else 1372 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1373 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1374 ((1 << ntb->bits_per_vector) - 1) << 1375 (db * ntb->bits_per_vector)); 1376 else 1377 ntb_reg_write(2, ntb->reg_ofs.sdb, 1378 ((1 << ntb->bits_per_vector) - 1) << 1379 (db * ntb->bits_per_vector)); 1380} 1381 1382/** 1383 * ntb_query_link_status() - return the hardware link status 1384 * @ndev: pointer to ntb_device instance 1385 * 1386 * Returns true if the hardware is connected to the remote system 1387 * 1388 * RETURNS: true or false based on the hardware link state 1389 */ 1390bool 1391ntb_query_link_status(struct ntb_softc *ntb) 1392{ 1393 1394 return (ntb->link_status == NTB_LINK_UP); 1395} 1396 1397static void 1398save_bar_parameters(struct ntb_pci_bar_info *bar) 1399{ 1400 bar->pci_bus_tag = 1401 rman_get_bustag(bar->pci_resource); 1402 bar->pci_bus_handle = 1403 rman_get_bushandle(bar->pci_resource); 1404 bar->pbase = 1405 rman_get_start(bar->pci_resource); 1406 bar->size = 1407 rman_get_size(bar->pci_resource); 1408 bar->vbase = 1409 rman_get_virtual(bar->pci_resource); 1410 1411} 1412 1413device_t ntb_get_device(struct ntb_softc *ntb) 1414{ 1415 1416 return (ntb->device); 1417} 1418