ntb_hw_intel.c revision 289232
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289232 2015-10-13 17:20:47Z cem $"); 29 30#include <sys/param.h> 31#include <sys/kernel.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/malloc.h> 35#include <sys/module.h> 36#include <sys/queue.h> 37#include <sys/rman.h> 38#include <sys/sysctl.h> 39#include <vm/vm.h> 40#include <vm/pmap.h> 41#include <machine/bus.h> 42#include <machine/pmap.h> 43#include <machine/resource.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46 47#include "ntb_regs.h" 48#include "ntb_hw.h" 49 50/* 51 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52 * allows you to connect two systems using a PCI-e link. 53 * 54 * This module contains the hardware abstraction layer for the NTB. It allows 55 * you to send and recieve interrupts, map the memory windows and send and 56 * receive messages in the scratch-pad registers. 57 * 58 * NOTE: Much of the code in this module is shared with Linux. Any patches may 59 * be picked up and redistributed in Linux with a dual GPL/BSD license. 60 */ 61 62#define NTB_CONFIG_BAR 0 63#define NTB_B2B_BAR_1 1 64#define NTB_B2B_BAR_2 2 65#define NTB_MAX_BARS 3 66#define NTB_MW_TO_BAR(mw) ((mw) + 1) 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define SOC_LINK_RECOVERY_TIME 500 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75enum ntb_device_type { 76 NTB_XEON, 77 NTB_SOC 78}; 79 80/* Device features and workarounds */ 81#define HAS_FEATURE(feature) \ 82 ((ntb->features & (feature)) != 0) 83 84struct ntb_hw_info { 85 uint32_t device_id; 86 const char *desc; 87 enum ntb_device_type type; 88 uint64_t features; 89}; 90 91struct ntb_pci_bar_info { 92 bus_space_tag_t pci_bus_tag; 93 bus_space_handle_t pci_bus_handle; 94 int pci_resource_id; 95 struct resource *pci_resource; 96 vm_paddr_t pbase; 97 void *vbase; 98 u_long size; 99}; 100 101struct ntb_int_info { 102 struct resource *res; 103 int rid; 104 void *tag; 105}; 106 107struct ntb_db_cb { 108 ntb_db_callback callback; 109 unsigned int db_num; 110 void *data; 111 struct ntb_softc *ntb; 112}; 113 114struct ntb_softc { 115 device_t device; 116 enum ntb_device_type type; 117 uint64_t features; 118 119 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 120 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 121 uint32_t allocated_interrupts; 122 123 struct callout heartbeat_timer; 124 struct callout lr_timer; 125 126 void *ntb_transport; 127 ntb_event_callback event_cb; 128 struct ntb_db_cb *db_cb; 129 130 struct { 131 uint8_t max_spads; 132 uint8_t max_db_bits; 133 uint8_t msix_cnt; 134 } limits; 135 struct { 136 uint32_t pdb; 137 uint32_t pdb_mask; 138 uint32_t sdb; 139 uint32_t sbar2_xlat; 140 uint32_t sbar4_xlat; 141 uint32_t spad_remote; 142 uint32_t spad_local; 143 uint32_t lnk_cntl; 144 uint32_t lnk_stat; 145 uint32_t spci_cmd; 146 } reg_ofs; 147 uint8_t conn_type; 148 uint8_t dev_type; 149 uint8_t bits_per_vector; 150 uint8_t link_status; 151 uint8_t link_width; 152 uint8_t link_speed; 153}; 154 155#define ntb_bar_read(SIZE, bar, offset) \ 156 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 157 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 158#define ntb_bar_write(SIZE, bar, offset, val) \ 159 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 160 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 161#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 162#define ntb_reg_write(SIZE, offset, val) \ 163 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 164#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 165#define ntb_mw_write(SIZE, offset, val) \ 166 ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 167 168typedef int (*bar_map_strategy)(struct ntb_softc *ntb, 169 struct ntb_pci_bar_info *bar); 170 171static int ntb_probe(device_t device); 172static int ntb_attach(device_t device); 173static int ntb_detach(device_t device); 174static int ntb_map_pci_bars(struct ntb_softc *ntb); 175static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 176 struct ntb_pci_bar_info *bar); 177static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 178static int map_memory_window_bar(struct ntb_softc *ntb, 179 struct ntb_pci_bar_info *bar); 180static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 181static int ntb_setup_interrupts(struct ntb_softc *ntb); 182static void ntb_teardown_interrupts(struct ntb_softc *ntb); 183static void handle_soc_irq(void *arg); 184static void handle_xeon_irq(void *arg); 185static void handle_xeon_event_irq(void *arg); 186static void ntb_handle_legacy_interrupt(void *arg); 187static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 188static void ntb_free_callbacks(struct ntb_softc *ntb); 189static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 190static int ntb_initialize_hw(struct ntb_softc *ntb); 191static int ntb_setup_xeon(struct ntb_softc *ntb); 192static int ntb_setup_soc(struct ntb_softc *ntb); 193static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 194static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 195static void ntb_handle_heartbeat(void *arg); 196static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 197static void recover_soc_link(void *arg); 198static int ntb_check_link_status(struct ntb_softc *ntb); 199static void save_bar_parameters(struct ntb_pci_bar_info *bar); 200 201static struct ntb_hw_info pci_ids[] = { 202 { 0x3C0D8086, "Xeon E5/Core i7 Non-Transparent Bridge B2B", NTB_XEON, 203 NTB_REGS_THRU_MW }, 204 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 205 { 0x0E0D8086, "Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 206 NTB_REGS_THRU_MW | NTB_BAR_SIZE_4K }, 207 { 0x00000000, NULL, NTB_SOC, 0 } 208}; 209 210/* 211 * OS <-> Driver interface structures 212 */ 213MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 214 215static device_method_t ntb_pci_methods[] = { 216 /* Device interface */ 217 DEVMETHOD(device_probe, ntb_probe), 218 DEVMETHOD(device_attach, ntb_attach), 219 DEVMETHOD(device_detach, ntb_detach), 220 DEVMETHOD_END 221}; 222 223static driver_t ntb_pci_driver = { 224 "ntb_hw", 225 ntb_pci_methods, 226 sizeof(struct ntb_softc), 227}; 228 229static devclass_t ntb_devclass; 230DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 231MODULE_VERSION(ntb_hw, 1); 232 233SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 234 235/* 236 * OS <-> Driver linkage functions 237 */ 238static int 239ntb_probe(device_t device) 240{ 241 struct ntb_hw_info *p; 242 243 p = ntb_get_device_info(pci_get_devid(device)); 244 if (p == NULL) 245 return (ENXIO); 246 247 device_set_desc(device, p->desc); 248 return (0); 249} 250 251static int 252ntb_attach(device_t device) 253{ 254 struct ntb_softc *ntb; 255 struct ntb_hw_info *p; 256 int error; 257 258 ntb = DEVICE2SOFTC(device); 259 p = ntb_get_device_info(pci_get_devid(device)); 260 261 ntb->device = device; 262 ntb->type = p->type; 263 ntb->features = p->features; 264 265 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 266 callout_init(&ntb->heartbeat_timer, 1); 267 callout_init(&ntb->lr_timer, 1); 268 269 error = ntb_map_pci_bars(ntb); 270 if (error) 271 goto out; 272 error = ntb_initialize_hw(ntb); 273 if (error) 274 goto out; 275 error = ntb_setup_interrupts(ntb); 276 if (error) 277 goto out; 278 279 pci_enable_busmaster(ntb->device); 280 281out: 282 if (error != 0) 283 ntb_detach(device); 284 return (error); 285} 286 287static int 288ntb_detach(device_t device) 289{ 290 struct ntb_softc *ntb; 291 292 ntb = DEVICE2SOFTC(device); 293 callout_drain(&ntb->heartbeat_timer); 294 callout_drain(&ntb->lr_timer); 295 ntb_teardown_interrupts(ntb); 296 ntb_unmap_pci_bar(ntb); 297 298 return (0); 299} 300 301static int 302ntb_map_pci_bars(struct ntb_softc *ntb) 303{ 304 int rc; 305 306 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 307 rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 308 if (rc != 0) 309 return (rc); 310 311 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 312 rc = map_pci_bar(ntb, map_memory_window_bar, 313 &ntb->bar_info[NTB_B2B_BAR_1]); 314 if (rc != 0) 315 return (rc); 316 317 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 318 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 319 rc = map_pci_bar(ntb, map_mmr_bar, 320 &ntb->bar_info[NTB_B2B_BAR_2]); 321 else 322 rc = map_pci_bar(ntb, map_memory_window_bar, 323 &ntb->bar_info[NTB_B2B_BAR_2]); 324 return (rc); 325} 326 327static int 328map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 329 struct ntb_pci_bar_info *bar) 330{ 331 int rc; 332 333 rc = strategy(ntb, bar); 334 if (rc != 0) 335 device_printf(ntb->device, 336 "unable to allocate pci resource\n"); 337 else 338 device_printf(ntb->device, 339 "Bar size = %lx, v %p, p %p\n", 340 bar->size, bar->vbase, (void *)(bar->pbase)); 341 return (rc); 342} 343 344static int 345map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 346{ 347 348 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 349 &bar->pci_resource_id, RF_ACTIVE); 350 if (bar->pci_resource == NULL) 351 return (ENXIO); 352 353 save_bar_parameters(bar); 354 return (0); 355} 356 357static int 358map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 359{ 360 int rc; 361 uint8_t bar_size_bits = 0; 362 363 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 364 &bar->pci_resource_id, RF_ACTIVE); 365 366 if (bar->pci_resource == NULL) 367 return (ENXIO); 368 369 save_bar_parameters(bar); 370 /* 371 * Ivytown NTB BAR sizes are misreported by the hardware due to a 372 * hardware issue. To work around this, query the size it should be 373 * configured to by the device and modify the resource to correspond to 374 * this new size. The BIOS on systems with this problem is required to 375 * provide enough address space to allow the driver to make this change 376 * safely. 377 * 378 * Ideally I could have just specified the size when I allocated the 379 * resource like: 380 * bus_alloc_resource(ntb->device, 381 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 382 * 1ul << bar_size_bits, RF_ACTIVE); 383 * but the PCI driver does not honor the size in this call, so we have 384 * to modify it after the fact. 385 */ 386 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 387 if (bar->pci_resource_id == PCIR_BAR(2)) 388 bar_size_bits = pci_read_config(ntb->device, 389 XEON_PBAR23SZ_OFFSET, 1); 390 else 391 bar_size_bits = pci_read_config(ntb->device, 392 XEON_PBAR45SZ_OFFSET, 1); 393 394 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 395 bar->pci_resource, bar->pbase, 396 bar->pbase + (1ul << bar_size_bits) - 1); 397 if (rc != 0) { 398 device_printf(ntb->device, 399 "unable to resize bar\n"); 400 return (rc); 401 } 402 403 save_bar_parameters(bar); 404 } 405 406 /* Mark bar region as write combining to improve performance. */ 407 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 408 VM_MEMATTR_WRITE_COMBINING); 409 if (rc != 0) { 410 device_printf(ntb->device, 411 "unable to mark bar as WRITE_COMBINING\n"); 412 return (rc); 413 } 414 return (0); 415} 416 417static void 418ntb_unmap_pci_bar(struct ntb_softc *ntb) 419{ 420 struct ntb_pci_bar_info *current_bar; 421 int i; 422 423 for (i = 0; i< NTB_MAX_BARS; i++) { 424 current_bar = &ntb->bar_info[i]; 425 if (current_bar->pci_resource != NULL) 426 bus_release_resource(ntb->device, SYS_RES_MEMORY, 427 current_bar->pci_resource_id, 428 current_bar->pci_resource); 429 } 430} 431 432static int 433ntb_setup_interrupts(struct ntb_softc *ntb) 434{ 435 void (*interrupt_handler)(void *); 436 void *int_arg; 437 bool use_msix = false; 438 uint32_t num_vectors; 439 int i; 440 441 ntb->allocated_interrupts = 0; 442 /* 443 * On SOC, disable all interrupts. On XEON, disable all but Link 444 * Interrupt. The rest will be unmasked as callbacks are registered. 445 */ 446 if (ntb->type == NTB_SOC) 447 ntb_reg_write(8, ntb->reg_ofs.pdb_mask, ~0); 448 else 449 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, 450 ~(1 << ntb->limits.max_db_bits)); 451 452 num_vectors = MIN(pci_msix_count(ntb->device), 453 ntb->limits.max_db_bits); 454 if (num_vectors >= 1) { 455 pci_alloc_msix(ntb->device, &num_vectors); 456 if (num_vectors >= 4) 457 use_msix = true; 458 } 459 460 ntb_create_callbacks(ntb, num_vectors); 461 if (use_msix == true) { 462 for (i = 0; i < num_vectors; i++) { 463 ntb->int_info[i].rid = i + 1; 464 ntb->int_info[i].res = bus_alloc_resource_any( 465 ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 466 RF_ACTIVE); 467 if (ntb->int_info[i].res == NULL) { 468 device_printf(ntb->device, 469 "bus_alloc_resource failed\n"); 470 return (ENOMEM); 471 } 472 ntb->int_info[i].tag = NULL; 473 ntb->allocated_interrupts++; 474 if (ntb->type == NTB_SOC) { 475 interrupt_handler = handle_soc_irq; 476 int_arg = &ntb->db_cb[i]; 477 } else { 478 if (i == num_vectors - 1) { 479 interrupt_handler = 480 handle_xeon_event_irq; 481 int_arg = ntb; 482 } else { 483 interrupt_handler = 484 handle_xeon_irq; 485 int_arg = &ntb->db_cb[i]; 486 } 487 } 488 if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 489 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 490 interrupt_handler, int_arg, 491 &ntb->int_info[i].tag) != 0) { 492 device_printf(ntb->device, 493 "bus_setup_intr failed\n"); 494 return (ENXIO); 495 } 496 } 497 } else { 498 ntb->int_info[0].rid = 0; 499 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, 500 SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 501 interrupt_handler = ntb_handle_legacy_interrupt; 502 if (ntb->int_info[0].res == NULL) { 503 device_printf(ntb->device, 504 "bus_alloc_resource failed\n"); 505 return (ENOMEM); 506 } 507 ntb->int_info[0].tag = NULL; 508 ntb->allocated_interrupts = 1; 509 510 if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 511 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 512 interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 513 514 device_printf(ntb->device, "bus_setup_intr failed\n"); 515 return (ENXIO); 516 } 517 } 518 519 return (0); 520} 521 522static void 523ntb_teardown_interrupts(struct ntb_softc *ntb) 524{ 525 struct ntb_int_info *current_int; 526 int i; 527 528 for (i = 0; i < ntb->allocated_interrupts; i++) { 529 current_int = &ntb->int_info[i]; 530 if (current_int->tag != NULL) 531 bus_teardown_intr(ntb->device, current_int->res, 532 current_int->tag); 533 534 if (current_int->res != NULL) 535 bus_release_resource(ntb->device, SYS_RES_IRQ, 536 rman_get_rid(current_int->res), current_int->res); 537 } 538 539 ntb_free_callbacks(ntb); 540 pci_release_msi(ntb->device); 541} 542 543static void 544handle_soc_irq(void *arg) 545{ 546 struct ntb_db_cb *db_cb = arg; 547 struct ntb_softc *ntb = db_cb->ntb; 548 549 ntb_reg_write(8, ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num); 550 551 if (db_cb->callback != NULL) 552 db_cb->callback(db_cb->data, db_cb->db_num); 553} 554 555static void 556handle_xeon_irq(void *arg) 557{ 558 struct ntb_db_cb *db_cb = arg; 559 struct ntb_softc *ntb = db_cb->ntb; 560 561 /* 562 * On Xeon, there are 16 bits in the interrupt register 563 * but only 4 vectors. So, 5 bits are assigned to the first 3 564 * vectors, with the 4th having a single bit for link 565 * interrupts. 566 */ 567 ntb_reg_write(2, ntb->reg_ofs.pdb, 568 ((1 << ntb->bits_per_vector) - 1) << 569 (db_cb->db_num * ntb->bits_per_vector)); 570 571 if (db_cb->callback != NULL) 572 db_cb->callback(db_cb->data, db_cb->db_num); 573} 574 575/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 576static void 577handle_xeon_event_irq(void *arg) 578{ 579 struct ntb_softc *ntb = arg; 580 int rc; 581 582 rc = ntb_check_link_status(ntb); 583 if (rc != 0) 584 device_printf(ntb->device, "Error determining link status\n"); 585 586 /* bit 15 is always the link bit */ 587 ntb_reg_write(2, ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits); 588} 589 590static void 591ntb_handle_legacy_interrupt(void *arg) 592{ 593 struct ntb_softc *ntb = arg; 594 unsigned int i = 0; 595 uint64_t pdb64; 596 uint16_t pdb16; 597 598 if (ntb->type == NTB_SOC) { 599 pdb64 = ntb_reg_read(8, ntb->reg_ofs.pdb); 600 601 while (pdb64) { 602 i = ffs(pdb64); 603 pdb64 &= pdb64 - 1; 604 handle_soc_irq(&ntb->db_cb[i]); 605 } 606 } else { 607 pdb16 = ntb_reg_read(2, ntb->reg_ofs.pdb); 608 609 if ((pdb16 & XEON_DB_HW_LINK) != 0) { 610 handle_xeon_event_irq(ntb); 611 pdb16 &= ~XEON_DB_HW_LINK; 612 } 613 614 while (pdb16 != 0) { 615 i = ffs(pdb16); 616 pdb16 &= pdb16 - 1; 617 handle_xeon_irq(&ntb->db_cb[i]); 618 } 619 } 620 621} 622 623static int 624ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 625{ 626 int i; 627 628 ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 629 M_ZERO | M_WAITOK); 630 for (i = 0; i < num_vectors; i++) { 631 ntb->db_cb[i].db_num = i; 632 ntb->db_cb[i].ntb = ntb; 633 } 634 635 return (0); 636} 637 638static void 639ntb_free_callbacks(struct ntb_softc *ntb) 640{ 641 int i; 642 643 for (i = 0; i < ntb->limits.max_db_bits; i++) 644 ntb_unregister_db_callback(ntb, i); 645 646 free(ntb->db_cb, M_NTB); 647} 648 649static struct ntb_hw_info * 650ntb_get_device_info(uint32_t device_id) 651{ 652 struct ntb_hw_info *ep = pci_ids; 653 654 while (ep->device_id) { 655 if (ep->device_id == device_id) 656 return (ep); 657 ++ep; 658 } 659 return (NULL); 660} 661 662static int 663ntb_initialize_hw(struct ntb_softc *ntb) 664{ 665 666 if (ntb->type == NTB_SOC) 667 return (ntb_setup_soc(ntb)); 668 else 669 return (ntb_setup_xeon(ntb)); 670} 671 672static int 673ntb_setup_xeon(struct ntb_softc *ntb) 674{ 675 uint8_t val, connection_type; 676 677 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 678 679 connection_type = val & XEON_PPD_CONN_TYPE; 680 switch (connection_type) { 681 case NTB_CONN_B2B: 682 ntb->conn_type = NTB_CONN_B2B; 683 break; 684 case NTB_CONN_CLASSIC: 685 case NTB_CONN_RP: 686 default: 687 device_printf(ntb->device, "Connection type %d not supported\n", 688 connection_type); 689 return (ENXIO); 690 } 691 692 if ((val & XEON_PPD_DEV_TYPE) != 0) 693 ntb->dev_type = NTB_DEV_USD; 694 else 695 ntb->dev_type = NTB_DEV_DSD; 696 697 ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET; 698 ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET; 699 ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET; 700 ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET; 701 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 702 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 703 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 704 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 705 706 /* 707 * There is a Xeon hardware errata related to writes to SDOORBELL or 708 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 709 * which may hang the system. To workaround this use the second memory 710 * window to access the interrupt and scratch pad registers on the 711 * remote system. 712 */ 713 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 714 /* 715 * Set the Limit register to 4k, the minimum size, to prevent 716 * an illegal access. 717 */ 718 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 719 ntb_get_mw_size(ntb, 1) + 0x1000); 720 else 721 /* 722 * Disable the limit register, just in case it is set to 723 * something silly. 724 */ 725 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 726 727 728 if (ntb->conn_type == NTB_CONN_B2B) { 729 ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET; 730 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 731 ntb->limits.max_spads = XEON_MAX_SPADS; 732 } else { 733 ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET; 734 ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET; 735 ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS; 736 } 737 738 ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 739 ntb->limits.msix_cnt = XEON_MSIX_CNT; 740 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 741 742 configure_xeon_secondary_side_bars(ntb); 743 744 /* Enable Bus Master and Memory Space on the secondary side */ 745 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 746 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 747 748 /* Enable link training */ 749 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, 750 NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP); 751 752 return (0); 753} 754 755static int 756ntb_setup_soc(struct ntb_softc *ntb) 757{ 758 uint32_t val, connection_type; 759 760 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 761 762 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 763 switch (connection_type) { 764 case NTB_CONN_B2B: 765 ntb->conn_type = NTB_CONN_B2B; 766 break; 767 case NTB_CONN_RP: 768 default: 769 device_printf(ntb->device, "Connection type %d not supported\n", 770 connection_type); 771 return (ENXIO); 772 } 773 774 if ((val & SOC_PPD_DEV_TYPE) != 0) 775 ntb->dev_type = NTB_DEV_DSD; 776 else 777 ntb->dev_type = NTB_DEV_USD; 778 779 /* Initiate PCI-E link training */ 780 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 781 4); 782 783 ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET; 784 ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET; 785 ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET; 786 ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET; 787 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 788 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 789 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 790 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 791 792 if (ntb->conn_type == NTB_CONN_B2B) { 793 ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET; 794 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 795 ntb->limits.max_spads = SOC_MAX_SPADS; 796 } else { 797 ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET; 798 ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET; 799 ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS; 800 } 801 802 ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 803 ntb->limits.msix_cnt = SOC_MSIX_CNT; 804 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 805 806 /* 807 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 808 * resolved. Mask transaction layer internal parity errors. 809 */ 810 pci_write_config(ntb->device, 0xFC, 0x4, 4); 811 812 configure_soc_secondary_side_bars(ntb); 813 814 /* Enable Bus Master and Memory Space on the secondary side */ 815 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 816 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 817 818 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 819 820 return (0); 821} 822 823static void 824configure_soc_secondary_side_bars(struct ntb_softc *ntb) 825{ 826 827 if (ntb->dev_type == NTB_DEV_USD) { 828 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 829 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 830 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 831 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 832 } else { 833 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 834 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 835 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 836 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 837 } 838} 839 840static void 841configure_xeon_secondary_side_bars(struct ntb_softc *ntb) 842{ 843 844 if (ntb->dev_type == NTB_DEV_USD) { 845 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 846 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 847 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 848 MBAR01_DSD_ADDR); 849 else { 850 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 851 PBAR4XLAT_USD_ADDR); 852 /* 853 * B2B_XLAT_OFFSET is a 64-bit register but can only be 854 * written 32 bits at a time. 855 */ 856 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 857 MBAR01_DSD_ADDR & 0xffffffff); 858 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 859 MBAR01_DSD_ADDR >> 32); 860 } 861 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 862 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 863 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 864 } else { 865 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 866 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 867 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 868 MBAR01_USD_ADDR); 869 else { 870 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 871 PBAR4XLAT_DSD_ADDR); 872 /* 873 * B2B_XLAT_OFFSET is a 64-bit register but can only be 874 * written 32 bits at a time. 875 */ 876 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 877 MBAR01_USD_ADDR & 0xffffffff); 878 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 879 MBAR01_USD_ADDR >> 32); 880 } 881 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 882 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 883 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 884 } 885} 886 887/* SOC does not have link status interrupt, poll on that platform */ 888static void 889ntb_handle_heartbeat(void *arg) 890{ 891 struct ntb_softc *ntb = arg; 892 uint32_t status32; 893 int rc; 894 895 rc = ntb_check_link_status(ntb); 896 if (rc != 0) 897 device_printf(ntb->device, 898 "Error determining link status\n"); 899 900 /* Check to see if a link error is the cause of the link down */ 901 if (ntb->link_status == NTB_LINK_DOWN) { 902 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 903 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 904 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 905 ntb); 906 return; 907 } 908 } 909 910 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 911 ntb_handle_heartbeat, ntb); 912} 913 914static void 915soc_perform_link_restart(struct ntb_softc *ntb) 916{ 917 uint32_t status; 918 919 /* Driver resets the NTB ModPhy lanes - magic! */ 920 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 921 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 922 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 923 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 924 925 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 926 pause("ModPhy", hz / 10); 927 928 /* Clear AER Errors, write to clear */ 929 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 930 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 931 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 932 933 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 934 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 935 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 936 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 937 938 /* Clear DeSkew Buffer error, write to clear */ 939 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 940 status |= SOC_DESKEWSTS_DBERR; 941 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 942 943 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 944 status &= SOC_IBIST_ERR_OFLOW; 945 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 946 947 /* Releases the NTB state machine to allow the link to retrain */ 948 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 949 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 950 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 951} 952 953static void 954ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 955{ 956 enum ntb_hw_event event; 957 uint16_t status; 958 959 if (ntb->link_status == link_state) 960 return; 961 962 if (link_state == NTB_LINK_UP) { 963 device_printf(ntb->device, "Link Up\n"); 964 ntb->link_status = NTB_LINK_UP; 965 event = NTB_EVENT_HW_LINK_UP; 966 967 if (ntb->type == NTB_SOC) 968 status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 969 else 970 status = pci_read_config(ntb->device, 971 XEON_LINK_STATUS_OFFSET, 2); 972 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 973 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 974 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 975 ntb->link_width, ntb->link_speed); 976 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 977 ntb_handle_heartbeat, ntb); 978 } else { 979 device_printf(ntb->device, "Link Down\n"); 980 ntb->link_status = NTB_LINK_DOWN; 981 event = NTB_EVENT_HW_LINK_DOWN; 982 /* Do not modify link width/speed, we need it in link recovery */ 983 } 984 985 /* notify the upper layer if we have an event change */ 986 if (ntb->event_cb != NULL) 987 ntb->event_cb(ntb->ntb_transport, event); 988} 989 990static void 991recover_soc_link(void *arg) 992{ 993 struct ntb_softc *ntb = arg; 994 uint8_t speed, width; 995 uint32_t status32; 996 uint16_t status16; 997 998 soc_perform_link_restart(ntb); 999 1000 /* 1001 * There is a potential race between the 2 NTB devices recovering at 1002 * the same time. If the times are the same, the link will not recover 1003 * and the driver will be stuck in this loop forever. Add a random 1004 * interval to the recovery time to prevent this race. 1005 */ 1006 status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1007 pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1008 1009 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1010 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1011 goto retry; 1012 1013 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1014 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1015 goto retry; 1016 1017 status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1018 if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1019 goto out; 1020 1021 status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1022 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 1023 speed = (status16 & NTB_LINK_SPEED_MASK); 1024 if (ntb->link_width != width || ntb->link_speed != speed) 1025 goto retry; 1026 1027out: 1028 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1029 ntb_handle_heartbeat, ntb); 1030 return; 1031 1032retry: 1033 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1034 ntb); 1035} 1036 1037static int 1038ntb_check_link_status(struct ntb_softc *ntb) 1039{ 1040 int link_state; 1041 uint32_t ntb_cntl; 1042 uint16_t status; 1043 1044 if (ntb->type == NTB_SOC) { 1045 ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1046 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1047 link_state = NTB_LINK_DOWN; 1048 else 1049 link_state = NTB_LINK_UP; 1050 } else { 1051 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 1052 2); 1053 1054 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1055 link_state = NTB_LINK_UP; 1056 else 1057 link_state = NTB_LINK_DOWN; 1058 } 1059 1060 ntb_handle_link_event(ntb, link_state); 1061 1062 return (0); 1063} 1064 1065/** 1066 * ntb_register_event_callback() - register event callback 1067 * @ntb: pointer to ntb_softc instance 1068 * @func: callback function to register 1069 * 1070 * This function registers a callback for any HW driver events such as link 1071 * up/down, power management notices and etc. 1072 * 1073 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1074 */ 1075int 1076ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1077{ 1078 1079 if (ntb->event_cb != NULL) 1080 return (EINVAL); 1081 1082 ntb->event_cb = func; 1083 1084 return (0); 1085} 1086 1087/** 1088 * ntb_unregister_event_callback() - unregisters the event callback 1089 * @ntb: pointer to ntb_softc instance 1090 * 1091 * This function unregisters the existing callback from transport 1092 */ 1093void 1094ntb_unregister_event_callback(struct ntb_softc *ntb) 1095{ 1096 1097 ntb->event_cb = NULL; 1098} 1099 1100/** 1101 * ntb_register_db_callback() - register a callback for doorbell interrupt 1102 * @ntb: pointer to ntb_softc instance 1103 * @idx: doorbell index to register callback, zero based 1104 * @func: callback function to register 1105 * 1106 * This function registers a callback function for the doorbell interrupt 1107 * on the primary side. The function will unmask the doorbell as well to 1108 * allow interrupt. 1109 * 1110 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1111 */ 1112int 1113ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1114 ntb_db_callback func) 1115{ 1116 uint16_t mask; 1117 1118 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1119 device_printf(ntb->device, "Invalid Index.\n"); 1120 return (EINVAL); 1121 } 1122 1123 ntb->db_cb[idx].callback = func; 1124 ntb->db_cb[idx].data = data; 1125 1126 /* unmask interrupt */ 1127 mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask); 1128 mask &= ~(1 << (idx * ntb->bits_per_vector)); 1129 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask); 1130 1131 return (0); 1132} 1133 1134/** 1135 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1136 * @ntb: pointer to ntb_softc instance 1137 * @idx: doorbell index to register callback, zero based 1138 * 1139 * This function unregisters a callback function for the doorbell interrupt 1140 * on the primary side. The function will also mask the said doorbell. 1141 */ 1142void 1143ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1144{ 1145 unsigned long mask; 1146 1147 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1148 return; 1149 1150 mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask); 1151 mask |= 1 << (idx * ntb->bits_per_vector); 1152 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask); 1153 1154 ntb->db_cb[idx].callback = NULL; 1155} 1156 1157/** 1158 * ntb_find_transport() - find the transport pointer 1159 * @transport: pointer to pci device 1160 * 1161 * Given the pci device pointer, return the transport pointer passed in when 1162 * the transport attached when it was inited. 1163 * 1164 * RETURNS: pointer to transport. 1165 */ 1166void * 1167ntb_find_transport(struct ntb_softc *ntb) 1168{ 1169 1170 return (ntb->ntb_transport); 1171} 1172 1173/** 1174 * ntb_register_transport() - Register NTB transport with NTB HW driver 1175 * @transport: transport identifier 1176 * 1177 * This function allows a transport to reserve the hardware driver for 1178 * NTB usage. 1179 * 1180 * RETURNS: pointer to ntb_softc, NULL on error. 1181 */ 1182struct ntb_softc * 1183ntb_register_transport(struct ntb_softc *ntb, void *transport) 1184{ 1185 1186 /* 1187 * TODO: when we have more than one transport, we will need to rewrite 1188 * this to prevent race conditions 1189 */ 1190 if (ntb->ntb_transport != NULL) 1191 return (NULL); 1192 1193 ntb->ntb_transport = transport; 1194 return (ntb); 1195} 1196 1197/** 1198 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1199 * @ntb - ntb_softc of the transport to be freed 1200 * 1201 * This function unregisters the transport from the HW driver and performs any 1202 * necessary cleanups. 1203 */ 1204void 1205ntb_unregister_transport(struct ntb_softc *ntb) 1206{ 1207 int i; 1208 1209 if (ntb->ntb_transport == NULL) 1210 return; 1211 1212 for (i = 0; i < ntb->allocated_interrupts; i++) 1213 ntb_unregister_db_callback(ntb, i); 1214 1215 ntb_unregister_event_callback(ntb); 1216 ntb->ntb_transport = NULL; 1217} 1218 1219/** 1220 * ntb_get_max_spads() - get the total scratch regs usable 1221 * @ntb: pointer to ntb_softc instance 1222 * 1223 * This function returns the max 32bit scratchpad registers usable by the 1224 * upper layer. 1225 * 1226 * RETURNS: total number of scratch pad registers available 1227 */ 1228uint8_t 1229ntb_get_max_spads(struct ntb_softc *ntb) 1230{ 1231 1232 return (ntb->limits.max_spads); 1233} 1234 1235/** 1236 * ntb_write_local_spad() - write to the secondary scratchpad register 1237 * @ntb: pointer to ntb_softc instance 1238 * @idx: index to the scratchpad register, 0 based 1239 * @val: the data value to put into the register 1240 * 1241 * This function allows writing of a 32bit value to the indexed scratchpad 1242 * register. The register resides on the secondary (external) side. 1243 * 1244 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1245 */ 1246int 1247ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1248{ 1249 1250 if (idx >= ntb->limits.max_spads) 1251 return (EINVAL); 1252 1253 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1254 1255 return (0); 1256} 1257 1258/** 1259 * ntb_read_local_spad() - read from the primary scratchpad register 1260 * @ntb: pointer to ntb_softc instance 1261 * @idx: index to scratchpad register, 0 based 1262 * @val: pointer to 32bit integer for storing the register value 1263 * 1264 * This function allows reading of the 32bit scratchpad register on 1265 * the primary (internal) side. 1266 * 1267 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1268 */ 1269int 1270ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1271{ 1272 1273 if (idx >= ntb->limits.max_spads) 1274 return (EINVAL); 1275 1276 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1277 1278 return (0); 1279} 1280 1281/** 1282 * ntb_write_remote_spad() - write to the secondary scratchpad register 1283 * @ntb: pointer to ntb_softc instance 1284 * @idx: index to the scratchpad register, 0 based 1285 * @val: the data value to put into the register 1286 * 1287 * This function allows writing of a 32bit value to the indexed scratchpad 1288 * register. The register resides on the secondary (external) side. 1289 * 1290 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1291 */ 1292int 1293ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1294{ 1295 1296 if (idx >= ntb->limits.max_spads) 1297 return (EINVAL); 1298 1299 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1300 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1301 else 1302 ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1303 1304 return (0); 1305} 1306 1307/** 1308 * ntb_read_remote_spad() - read from the primary scratchpad register 1309 * @ntb: pointer to ntb_softc instance 1310 * @idx: index to scratchpad register, 0 based 1311 * @val: pointer to 32bit integer for storing the register value 1312 * 1313 * This function allows reading of the 32bit scratchpad register on 1314 * the primary (internal) side. 1315 * 1316 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1317 */ 1318int 1319ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1320{ 1321 1322 if (idx >= ntb->limits.max_spads) 1323 return (EINVAL); 1324 1325 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1326 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1327 else 1328 *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1329 1330 return (0); 1331} 1332 1333/** 1334 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1335 * @ntb: pointer to ntb_softc instance 1336 * @mw: memory window number 1337 * 1338 * This function provides the base virtual address of the memory window 1339 * specified. 1340 * 1341 * RETURNS: pointer to virtual address, or NULL on error. 1342 */ 1343void * 1344ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1345{ 1346 1347 if (mw >= NTB_NUM_MW) 1348 return (NULL); 1349 1350 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1351} 1352 1353vm_paddr_t 1354ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1355{ 1356 1357 if (mw >= NTB_NUM_MW) 1358 return (0); 1359 1360 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1361} 1362 1363/** 1364 * ntb_get_mw_size() - return size of NTB memory window 1365 * @ntb: pointer to ntb_softc instance 1366 * @mw: memory window number 1367 * 1368 * This function provides the physical size of the memory window specified 1369 * 1370 * RETURNS: the size of the memory window or zero on error 1371 */ 1372u_long 1373ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1374{ 1375 1376 if (mw >= NTB_NUM_MW) 1377 return (0); 1378 1379 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1380} 1381 1382/** 1383 * ntb_set_mw_addr - set the memory window address 1384 * @ntb: pointer to ntb_softc instance 1385 * @mw: memory window number 1386 * @addr: base address for data 1387 * 1388 * This function sets the base physical address of the memory window. This 1389 * memory address is where data from the remote system will be transfered into 1390 * or out of depending on how the transport is configured. 1391 */ 1392void 1393ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1394{ 1395 1396 if (mw >= NTB_NUM_MW) 1397 return; 1398 1399 switch (NTB_MW_TO_BAR(mw)) { 1400 case NTB_B2B_BAR_1: 1401 ntb_reg_write(8, ntb->reg_ofs.sbar2_xlat, addr); 1402 break; 1403 case NTB_B2B_BAR_2: 1404 ntb_reg_write(8, ntb->reg_ofs.sbar4_xlat, addr); 1405 break; 1406 } 1407} 1408 1409/** 1410 * ntb_ring_sdb() - Set the doorbell on the secondary/external side 1411 * @ntb: pointer to ntb_softc instance 1412 * @db: doorbell to ring 1413 * 1414 * This function allows triggering of a doorbell on the secondary/external 1415 * side that will initiate an interrupt on the remote host 1416 * 1417 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1418 */ 1419void 1420ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db) 1421{ 1422 1423 if (ntb->type == NTB_SOC) 1424 ntb_reg_write(8, ntb->reg_ofs.sdb, (uint64_t) 1 << db); 1425 else { 1426 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1427 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1428 ((1 << ntb->bits_per_vector) - 1) << 1429 (db * ntb->bits_per_vector)); 1430 else 1431 ntb_reg_write(2, ntb->reg_ofs.sdb, 1432 ((1 << ntb->bits_per_vector) - 1) << 1433 (db * ntb->bits_per_vector)); 1434 } 1435} 1436 1437/** 1438 * ntb_query_link_status() - return the hardware link status 1439 * @ndev: pointer to ntb_device instance 1440 * 1441 * Returns true if the hardware is connected to the remote system 1442 * 1443 * RETURNS: true or false based on the hardware link state 1444 */ 1445bool 1446ntb_query_link_status(struct ntb_softc *ntb) 1447{ 1448 1449 return (ntb->link_status == NTB_LINK_UP); 1450} 1451 1452static void 1453save_bar_parameters(struct ntb_pci_bar_info *bar) 1454{ 1455 1456 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 1457 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 1458 bar->pbase = rman_get_start(bar->pci_resource); 1459 bar->size = rman_get_size(bar->pci_resource); 1460 bar->vbase = rman_get_virtual(bar->pci_resource); 1461} 1462 1463device_t 1464ntb_get_device(struct ntb_softc *ntb) 1465{ 1466 1467 return (ntb->device); 1468} 1469 1470/* Export HW-specific errata information. */ 1471bool 1472ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 1473{ 1474 1475 return (HAS_FEATURE(feature)); 1476} 1477