ntb_hw_intel.c revision 289233
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289233 2015-10-13 17:21:38Z cem $"); 29 30#include <sys/param.h> 31#include <sys/kernel.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/malloc.h> 35#include <sys/module.h> 36#include <sys/queue.h> 37#include <sys/rman.h> 38#include <sys/sysctl.h> 39#include <vm/vm.h> 40#include <vm/pmap.h> 41#include <machine/bus.h> 42#include <machine/pmap.h> 43#include <machine/resource.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46 47#include "ntb_regs.h" 48#include "ntb_hw.h" 49 50/* 51 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52 * allows you to connect two systems using a PCI-e link. 53 * 54 * This module contains the hardware abstraction layer for the NTB. It allows 55 * you to send and recieve interrupts, map the memory windows and send and 56 * receive messages in the scratch-pad registers. 57 * 58 * NOTE: Much of the code in this module is shared with Linux. Any patches may 59 * be picked up and redistributed in Linux with a dual GPL/BSD license. 60 */ 61 62#define NTB_CONFIG_BAR 0 63#define NTB_B2B_BAR_1 1 64#define NTB_B2B_BAR_2 2 65#define NTB_MAX_BARS 3 66#define NTB_MW_TO_BAR(mw) ((mw) + 1) 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define SOC_LINK_RECOVERY_TIME 500 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75enum ntb_device_type { 76 NTB_XEON, 77 NTB_SOC 78}; 79 80/* Device features and workarounds */ 81#define HAS_FEATURE(feature) \ 82 ((ntb->features & (feature)) != 0) 83 84struct ntb_hw_info { 85 uint32_t device_id; 86 const char *desc; 87 enum ntb_device_type type; 88 uint64_t features; 89}; 90 91struct ntb_pci_bar_info { 92 bus_space_tag_t pci_bus_tag; 93 bus_space_handle_t pci_bus_handle; 94 int pci_resource_id; 95 struct resource *pci_resource; 96 vm_paddr_t pbase; 97 void *vbase; 98 u_long size; 99}; 100 101struct ntb_int_info { 102 struct resource *res; 103 int rid; 104 void *tag; 105}; 106 107struct ntb_db_cb { 108 ntb_db_callback callback; 109 unsigned int db_num; 110 void *data; 111 struct ntb_softc *ntb; 112}; 113 114struct ntb_softc { 115 device_t device; 116 enum ntb_device_type type; 117 uint64_t features; 118 119 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 120 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 121 uint32_t allocated_interrupts; 122 123 struct callout heartbeat_timer; 124 struct callout lr_timer; 125 126 void *ntb_transport; 127 ntb_event_callback event_cb; 128 struct ntb_db_cb *db_cb; 129 130 struct { 131 uint8_t max_spads; 132 uint8_t max_db_bits; 133 uint8_t msix_cnt; 134 } limits; 135 struct { 136 uint32_t pdb; 137 uint32_t pdb_mask; 138 uint32_t sdb; 139 uint32_t sbar2_xlat; 140 uint32_t sbar4_xlat; 141 uint32_t spad_remote; 142 uint32_t spad_local; 143 uint32_t lnk_cntl; 144 uint32_t lnk_stat; 145 uint32_t spci_cmd; 146 } reg_ofs; 147 uint8_t conn_type; 148 uint8_t dev_type; 149 uint8_t bits_per_vector; 150 uint8_t link_status; 151 uint8_t link_width; 152 uint8_t link_speed; 153}; 154 155#define ntb_bar_read(SIZE, bar, offset) \ 156 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 157 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 158#define ntb_bar_write(SIZE, bar, offset, val) \ 159 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 160 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 161#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 162#define ntb_reg_write(SIZE, offset, val) \ 163 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 164#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 165#define ntb_mw_write(SIZE, offset, val) \ 166 ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 167 168typedef int (*bar_map_strategy)(struct ntb_softc *ntb, 169 struct ntb_pci_bar_info *bar); 170 171static int ntb_probe(device_t device); 172static int ntb_attach(device_t device); 173static int ntb_detach(device_t device); 174static int ntb_map_pci_bars(struct ntb_softc *ntb); 175static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 176 struct ntb_pci_bar_info *bar); 177static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 178static int map_memory_window_bar(struct ntb_softc *ntb, 179 struct ntb_pci_bar_info *bar); 180static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 181static int ntb_setup_interrupts(struct ntb_softc *ntb); 182static void ntb_teardown_interrupts(struct ntb_softc *ntb); 183static void handle_soc_irq(void *arg); 184static void handle_xeon_irq(void *arg); 185static void handle_xeon_event_irq(void *arg); 186static void ntb_handle_legacy_interrupt(void *arg); 187static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 188static void ntb_free_callbacks(struct ntb_softc *ntb); 189static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 190static int ntb_initialize_hw(struct ntb_softc *ntb); 191static int ntb_setup_xeon(struct ntb_softc *ntb); 192static int ntb_setup_soc(struct ntb_softc *ntb); 193static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 194static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 195static void ntb_handle_heartbeat(void *arg); 196static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 197static void recover_soc_link(void *arg); 198static int ntb_check_link_status(struct ntb_softc *ntb); 199static void save_bar_parameters(struct ntb_pci_bar_info *bar); 200 201static struct ntb_hw_info pci_ids[] = { 202 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 203 204 /* XXX: PS/SS IDs left out until they are supported. */ 205 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 206 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 207 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 208 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 209 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 210 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 211 | NTB_BAR_SIZE_4K }, 212 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 213 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 214 }, 215 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 216 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 217 }, 218 219 { 0x00000000, NULL, NTB_SOC, 0 } 220}; 221 222/* 223 * OS <-> Driver interface structures 224 */ 225MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 226 227static device_method_t ntb_pci_methods[] = { 228 /* Device interface */ 229 DEVMETHOD(device_probe, ntb_probe), 230 DEVMETHOD(device_attach, ntb_attach), 231 DEVMETHOD(device_detach, ntb_detach), 232 DEVMETHOD_END 233}; 234 235static driver_t ntb_pci_driver = { 236 "ntb_hw", 237 ntb_pci_methods, 238 sizeof(struct ntb_softc), 239}; 240 241static devclass_t ntb_devclass; 242DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 243MODULE_VERSION(ntb_hw, 1); 244 245SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 246 247/* 248 * OS <-> Driver linkage functions 249 */ 250static int 251ntb_probe(device_t device) 252{ 253 struct ntb_hw_info *p; 254 255 p = ntb_get_device_info(pci_get_devid(device)); 256 if (p == NULL) 257 return (ENXIO); 258 259 device_set_desc(device, p->desc); 260 return (0); 261} 262 263static int 264ntb_attach(device_t device) 265{ 266 struct ntb_softc *ntb; 267 struct ntb_hw_info *p; 268 int error; 269 270 ntb = DEVICE2SOFTC(device); 271 p = ntb_get_device_info(pci_get_devid(device)); 272 273 ntb->device = device; 274 ntb->type = p->type; 275 ntb->features = p->features; 276 277 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 278 callout_init(&ntb->heartbeat_timer, 1); 279 callout_init(&ntb->lr_timer, 1); 280 281 error = ntb_map_pci_bars(ntb); 282 if (error) 283 goto out; 284 error = ntb_initialize_hw(ntb); 285 if (error) 286 goto out; 287 error = ntb_setup_interrupts(ntb); 288 if (error) 289 goto out; 290 291 pci_enable_busmaster(ntb->device); 292 293out: 294 if (error != 0) 295 ntb_detach(device); 296 return (error); 297} 298 299static int 300ntb_detach(device_t device) 301{ 302 struct ntb_softc *ntb; 303 304 ntb = DEVICE2SOFTC(device); 305 callout_drain(&ntb->heartbeat_timer); 306 callout_drain(&ntb->lr_timer); 307 ntb_teardown_interrupts(ntb); 308 ntb_unmap_pci_bar(ntb); 309 310 return (0); 311} 312 313static int 314ntb_map_pci_bars(struct ntb_softc *ntb) 315{ 316 int rc; 317 318 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 319 rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 320 if (rc != 0) 321 return (rc); 322 323 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 324 rc = map_pci_bar(ntb, map_memory_window_bar, 325 &ntb->bar_info[NTB_B2B_BAR_1]); 326 if (rc != 0) 327 return (rc); 328 329 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 330 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 331 rc = map_pci_bar(ntb, map_mmr_bar, 332 &ntb->bar_info[NTB_B2B_BAR_2]); 333 else 334 rc = map_pci_bar(ntb, map_memory_window_bar, 335 &ntb->bar_info[NTB_B2B_BAR_2]); 336 return (rc); 337} 338 339static int 340map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 341 struct ntb_pci_bar_info *bar) 342{ 343 int rc; 344 345 rc = strategy(ntb, bar); 346 if (rc != 0) 347 device_printf(ntb->device, 348 "unable to allocate pci resource\n"); 349 else 350 device_printf(ntb->device, 351 "Bar size = %lx, v %p, p %p\n", 352 bar->size, bar->vbase, (void *)(bar->pbase)); 353 return (rc); 354} 355 356static int 357map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 358{ 359 360 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 361 &bar->pci_resource_id, RF_ACTIVE); 362 if (bar->pci_resource == NULL) 363 return (ENXIO); 364 365 save_bar_parameters(bar); 366 return (0); 367} 368 369static int 370map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 371{ 372 int rc; 373 uint8_t bar_size_bits = 0; 374 375 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 376 &bar->pci_resource_id, RF_ACTIVE); 377 378 if (bar->pci_resource == NULL) 379 return (ENXIO); 380 381 save_bar_parameters(bar); 382 /* 383 * Ivytown NTB BAR sizes are misreported by the hardware due to a 384 * hardware issue. To work around this, query the size it should be 385 * configured to by the device and modify the resource to correspond to 386 * this new size. The BIOS on systems with this problem is required to 387 * provide enough address space to allow the driver to make this change 388 * safely. 389 * 390 * Ideally I could have just specified the size when I allocated the 391 * resource like: 392 * bus_alloc_resource(ntb->device, 393 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 394 * 1ul << bar_size_bits, RF_ACTIVE); 395 * but the PCI driver does not honor the size in this call, so we have 396 * to modify it after the fact. 397 */ 398 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 399 if (bar->pci_resource_id == PCIR_BAR(2)) 400 bar_size_bits = pci_read_config(ntb->device, 401 XEON_PBAR23SZ_OFFSET, 1); 402 else 403 bar_size_bits = pci_read_config(ntb->device, 404 XEON_PBAR45SZ_OFFSET, 1); 405 406 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 407 bar->pci_resource, bar->pbase, 408 bar->pbase + (1ul << bar_size_bits) - 1); 409 if (rc != 0) { 410 device_printf(ntb->device, 411 "unable to resize bar\n"); 412 return (rc); 413 } 414 415 save_bar_parameters(bar); 416 } 417 418 /* Mark bar region as write combining to improve performance. */ 419 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 420 VM_MEMATTR_WRITE_COMBINING); 421 if (rc != 0) { 422 device_printf(ntb->device, 423 "unable to mark bar as WRITE_COMBINING\n"); 424 return (rc); 425 } 426 return (0); 427} 428 429static void 430ntb_unmap_pci_bar(struct ntb_softc *ntb) 431{ 432 struct ntb_pci_bar_info *current_bar; 433 int i; 434 435 for (i = 0; i< NTB_MAX_BARS; i++) { 436 current_bar = &ntb->bar_info[i]; 437 if (current_bar->pci_resource != NULL) 438 bus_release_resource(ntb->device, SYS_RES_MEMORY, 439 current_bar->pci_resource_id, 440 current_bar->pci_resource); 441 } 442} 443 444static int 445ntb_setup_interrupts(struct ntb_softc *ntb) 446{ 447 void (*interrupt_handler)(void *); 448 void *int_arg; 449 bool use_msix = false; 450 uint32_t num_vectors; 451 int i; 452 453 ntb->allocated_interrupts = 0; 454 /* 455 * On SOC, disable all interrupts. On XEON, disable all but Link 456 * Interrupt. The rest will be unmasked as callbacks are registered. 457 */ 458 if (ntb->type == NTB_SOC) 459 ntb_reg_write(8, ntb->reg_ofs.pdb_mask, ~0); 460 else 461 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, 462 ~(1 << ntb->limits.max_db_bits)); 463 464 num_vectors = MIN(pci_msix_count(ntb->device), 465 ntb->limits.max_db_bits); 466 if (num_vectors >= 1) { 467 pci_alloc_msix(ntb->device, &num_vectors); 468 if (num_vectors >= 4) 469 use_msix = true; 470 } 471 472 ntb_create_callbacks(ntb, num_vectors); 473 if (use_msix == true) { 474 for (i = 0; i < num_vectors; i++) { 475 ntb->int_info[i].rid = i + 1; 476 ntb->int_info[i].res = bus_alloc_resource_any( 477 ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 478 RF_ACTIVE); 479 if (ntb->int_info[i].res == NULL) { 480 device_printf(ntb->device, 481 "bus_alloc_resource failed\n"); 482 return (ENOMEM); 483 } 484 ntb->int_info[i].tag = NULL; 485 ntb->allocated_interrupts++; 486 if (ntb->type == NTB_SOC) { 487 interrupt_handler = handle_soc_irq; 488 int_arg = &ntb->db_cb[i]; 489 } else { 490 if (i == num_vectors - 1) { 491 interrupt_handler = 492 handle_xeon_event_irq; 493 int_arg = ntb; 494 } else { 495 interrupt_handler = 496 handle_xeon_irq; 497 int_arg = &ntb->db_cb[i]; 498 } 499 } 500 if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 501 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 502 interrupt_handler, int_arg, 503 &ntb->int_info[i].tag) != 0) { 504 device_printf(ntb->device, 505 "bus_setup_intr failed\n"); 506 return (ENXIO); 507 } 508 } 509 } else { 510 ntb->int_info[0].rid = 0; 511 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, 512 SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 513 interrupt_handler = ntb_handle_legacy_interrupt; 514 if (ntb->int_info[0].res == NULL) { 515 device_printf(ntb->device, 516 "bus_alloc_resource failed\n"); 517 return (ENOMEM); 518 } 519 ntb->int_info[0].tag = NULL; 520 ntb->allocated_interrupts = 1; 521 522 if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 523 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 524 interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 525 526 device_printf(ntb->device, "bus_setup_intr failed\n"); 527 return (ENXIO); 528 } 529 } 530 531 return (0); 532} 533 534static void 535ntb_teardown_interrupts(struct ntb_softc *ntb) 536{ 537 struct ntb_int_info *current_int; 538 int i; 539 540 for (i = 0; i < ntb->allocated_interrupts; i++) { 541 current_int = &ntb->int_info[i]; 542 if (current_int->tag != NULL) 543 bus_teardown_intr(ntb->device, current_int->res, 544 current_int->tag); 545 546 if (current_int->res != NULL) 547 bus_release_resource(ntb->device, SYS_RES_IRQ, 548 rman_get_rid(current_int->res), current_int->res); 549 } 550 551 ntb_free_callbacks(ntb); 552 pci_release_msi(ntb->device); 553} 554 555static void 556handle_soc_irq(void *arg) 557{ 558 struct ntb_db_cb *db_cb = arg; 559 struct ntb_softc *ntb = db_cb->ntb; 560 561 ntb_reg_write(8, ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num); 562 563 if (db_cb->callback != NULL) 564 db_cb->callback(db_cb->data, db_cb->db_num); 565} 566 567static void 568handle_xeon_irq(void *arg) 569{ 570 struct ntb_db_cb *db_cb = arg; 571 struct ntb_softc *ntb = db_cb->ntb; 572 573 /* 574 * On Xeon, there are 16 bits in the interrupt register 575 * but only 4 vectors. So, 5 bits are assigned to the first 3 576 * vectors, with the 4th having a single bit for link 577 * interrupts. 578 */ 579 ntb_reg_write(2, ntb->reg_ofs.pdb, 580 ((1 << ntb->bits_per_vector) - 1) << 581 (db_cb->db_num * ntb->bits_per_vector)); 582 583 if (db_cb->callback != NULL) 584 db_cb->callback(db_cb->data, db_cb->db_num); 585} 586 587/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 588static void 589handle_xeon_event_irq(void *arg) 590{ 591 struct ntb_softc *ntb = arg; 592 int rc; 593 594 rc = ntb_check_link_status(ntb); 595 if (rc != 0) 596 device_printf(ntb->device, "Error determining link status\n"); 597 598 /* bit 15 is always the link bit */ 599 ntb_reg_write(2, ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits); 600} 601 602static void 603ntb_handle_legacy_interrupt(void *arg) 604{ 605 struct ntb_softc *ntb = arg; 606 unsigned int i = 0; 607 uint64_t pdb64; 608 uint16_t pdb16; 609 610 if (ntb->type == NTB_SOC) { 611 pdb64 = ntb_reg_read(8, ntb->reg_ofs.pdb); 612 613 while (pdb64) { 614 i = ffs(pdb64); 615 pdb64 &= pdb64 - 1; 616 handle_soc_irq(&ntb->db_cb[i]); 617 } 618 } else { 619 pdb16 = ntb_reg_read(2, ntb->reg_ofs.pdb); 620 621 if ((pdb16 & XEON_DB_HW_LINK) != 0) { 622 handle_xeon_event_irq(ntb); 623 pdb16 &= ~XEON_DB_HW_LINK; 624 } 625 626 while (pdb16 != 0) { 627 i = ffs(pdb16); 628 pdb16 &= pdb16 - 1; 629 handle_xeon_irq(&ntb->db_cb[i]); 630 } 631 } 632 633} 634 635static int 636ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 637{ 638 int i; 639 640 ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 641 M_ZERO | M_WAITOK); 642 for (i = 0; i < num_vectors; i++) { 643 ntb->db_cb[i].db_num = i; 644 ntb->db_cb[i].ntb = ntb; 645 } 646 647 return (0); 648} 649 650static void 651ntb_free_callbacks(struct ntb_softc *ntb) 652{ 653 int i; 654 655 for (i = 0; i < ntb->limits.max_db_bits; i++) 656 ntb_unregister_db_callback(ntb, i); 657 658 free(ntb->db_cb, M_NTB); 659} 660 661static struct ntb_hw_info * 662ntb_get_device_info(uint32_t device_id) 663{ 664 struct ntb_hw_info *ep = pci_ids; 665 666 while (ep->device_id) { 667 if (ep->device_id == device_id) 668 return (ep); 669 ++ep; 670 } 671 return (NULL); 672} 673 674static int 675ntb_initialize_hw(struct ntb_softc *ntb) 676{ 677 678 if (ntb->type == NTB_SOC) 679 return (ntb_setup_soc(ntb)); 680 else 681 return (ntb_setup_xeon(ntb)); 682} 683 684static int 685ntb_setup_xeon(struct ntb_softc *ntb) 686{ 687 uint8_t val, connection_type; 688 689 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 690 691 connection_type = val & XEON_PPD_CONN_TYPE; 692 switch (connection_type) { 693 case NTB_CONN_B2B: 694 ntb->conn_type = NTB_CONN_B2B; 695 break; 696 case NTB_CONN_CLASSIC: 697 case NTB_CONN_RP: 698 default: 699 device_printf(ntb->device, "Connection type %d not supported\n", 700 connection_type); 701 return (ENXIO); 702 } 703 704 if ((val & XEON_PPD_DEV_TYPE) != 0) 705 ntb->dev_type = NTB_DEV_USD; 706 else 707 ntb->dev_type = NTB_DEV_DSD; 708 709 ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET; 710 ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET; 711 ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET; 712 ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET; 713 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 714 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 715 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 716 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 717 718 /* 719 * There is a Xeon hardware errata related to writes to SDOORBELL or 720 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 721 * which may hang the system. To workaround this use the second memory 722 * window to access the interrupt and scratch pad registers on the 723 * remote system. 724 */ 725 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 726 /* 727 * Set the Limit register to 4k, the minimum size, to prevent 728 * an illegal access. 729 */ 730 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 731 ntb_get_mw_size(ntb, 1) + 0x1000); 732 else 733 /* 734 * Disable the limit register, just in case it is set to 735 * something silly. 736 */ 737 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 738 739 740 if (ntb->conn_type == NTB_CONN_B2B) { 741 ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET; 742 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 743 ntb->limits.max_spads = XEON_MAX_SPADS; 744 } else { 745 ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET; 746 ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET; 747 ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS; 748 } 749 750 ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 751 ntb->limits.msix_cnt = XEON_MSIX_CNT; 752 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 753 754 configure_xeon_secondary_side_bars(ntb); 755 756 /* Enable Bus Master and Memory Space on the secondary side */ 757 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 758 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 759 760 /* Enable link training */ 761 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, 762 NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP); 763 764 return (0); 765} 766 767static int 768ntb_setup_soc(struct ntb_softc *ntb) 769{ 770 uint32_t val, connection_type; 771 772 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 773 774 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 775 switch (connection_type) { 776 case NTB_CONN_B2B: 777 ntb->conn_type = NTB_CONN_B2B; 778 break; 779 case NTB_CONN_RP: 780 default: 781 device_printf(ntb->device, "Connection type %d not supported\n", 782 connection_type); 783 return (ENXIO); 784 } 785 786 if ((val & SOC_PPD_DEV_TYPE) != 0) 787 ntb->dev_type = NTB_DEV_DSD; 788 else 789 ntb->dev_type = NTB_DEV_USD; 790 791 /* Initiate PCI-E link training */ 792 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 793 4); 794 795 ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET; 796 ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET; 797 ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET; 798 ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET; 799 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 800 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 801 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 802 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 803 804 if (ntb->conn_type == NTB_CONN_B2B) { 805 ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET; 806 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 807 ntb->limits.max_spads = SOC_MAX_SPADS; 808 } else { 809 ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET; 810 ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET; 811 ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS; 812 } 813 814 ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 815 ntb->limits.msix_cnt = SOC_MSIX_CNT; 816 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 817 818 /* 819 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 820 * resolved. Mask transaction layer internal parity errors. 821 */ 822 pci_write_config(ntb->device, 0xFC, 0x4, 4); 823 824 configure_soc_secondary_side_bars(ntb); 825 826 /* Enable Bus Master and Memory Space on the secondary side */ 827 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 828 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 829 830 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 831 832 return (0); 833} 834 835static void 836configure_soc_secondary_side_bars(struct ntb_softc *ntb) 837{ 838 839 if (ntb->dev_type == NTB_DEV_USD) { 840 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 841 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 842 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 843 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 844 } else { 845 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 846 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 847 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 848 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 849 } 850} 851 852static void 853configure_xeon_secondary_side_bars(struct ntb_softc *ntb) 854{ 855 856 if (ntb->dev_type == NTB_DEV_USD) { 857 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 858 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 859 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 860 MBAR01_DSD_ADDR); 861 else { 862 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 863 PBAR4XLAT_USD_ADDR); 864 /* 865 * B2B_XLAT_OFFSET is a 64-bit register but can only be 866 * written 32 bits at a time. 867 */ 868 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 869 MBAR01_DSD_ADDR & 0xffffffff); 870 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 871 MBAR01_DSD_ADDR >> 32); 872 } 873 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 874 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 875 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 876 } else { 877 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 878 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 879 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 880 MBAR01_USD_ADDR); 881 else { 882 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 883 PBAR4XLAT_DSD_ADDR); 884 /* 885 * B2B_XLAT_OFFSET is a 64-bit register but can only be 886 * written 32 bits at a time. 887 */ 888 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 889 MBAR01_USD_ADDR & 0xffffffff); 890 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 891 MBAR01_USD_ADDR >> 32); 892 } 893 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 894 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 895 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 896 } 897} 898 899/* SOC does not have link status interrupt, poll on that platform */ 900static void 901ntb_handle_heartbeat(void *arg) 902{ 903 struct ntb_softc *ntb = arg; 904 uint32_t status32; 905 int rc; 906 907 rc = ntb_check_link_status(ntb); 908 if (rc != 0) 909 device_printf(ntb->device, 910 "Error determining link status\n"); 911 912 /* Check to see if a link error is the cause of the link down */ 913 if (ntb->link_status == NTB_LINK_DOWN) { 914 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 915 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 916 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 917 ntb); 918 return; 919 } 920 } 921 922 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 923 ntb_handle_heartbeat, ntb); 924} 925 926static void 927soc_perform_link_restart(struct ntb_softc *ntb) 928{ 929 uint32_t status; 930 931 /* Driver resets the NTB ModPhy lanes - magic! */ 932 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 933 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 934 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 935 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 936 937 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 938 pause("ModPhy", hz / 10); 939 940 /* Clear AER Errors, write to clear */ 941 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 942 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 943 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 944 945 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 946 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 947 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 948 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 949 950 /* Clear DeSkew Buffer error, write to clear */ 951 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 952 status |= SOC_DESKEWSTS_DBERR; 953 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 954 955 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 956 status &= SOC_IBIST_ERR_OFLOW; 957 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 958 959 /* Releases the NTB state machine to allow the link to retrain */ 960 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 961 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 962 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 963} 964 965static void 966ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 967{ 968 enum ntb_hw_event event; 969 uint16_t status; 970 971 if (ntb->link_status == link_state) 972 return; 973 974 if (link_state == NTB_LINK_UP) { 975 device_printf(ntb->device, "Link Up\n"); 976 ntb->link_status = NTB_LINK_UP; 977 event = NTB_EVENT_HW_LINK_UP; 978 979 if (ntb->type == NTB_SOC) 980 status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 981 else 982 status = pci_read_config(ntb->device, 983 XEON_LINK_STATUS_OFFSET, 2); 984 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 985 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 986 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 987 ntb->link_width, ntb->link_speed); 988 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 989 ntb_handle_heartbeat, ntb); 990 } else { 991 device_printf(ntb->device, "Link Down\n"); 992 ntb->link_status = NTB_LINK_DOWN; 993 event = NTB_EVENT_HW_LINK_DOWN; 994 /* Do not modify link width/speed, we need it in link recovery */ 995 } 996 997 /* notify the upper layer if we have an event change */ 998 if (ntb->event_cb != NULL) 999 ntb->event_cb(ntb->ntb_transport, event); 1000} 1001 1002static void 1003recover_soc_link(void *arg) 1004{ 1005 struct ntb_softc *ntb = arg; 1006 uint8_t speed, width; 1007 uint32_t status32; 1008 uint16_t status16; 1009 1010 soc_perform_link_restart(ntb); 1011 1012 /* 1013 * There is a potential race between the 2 NTB devices recovering at 1014 * the same time. If the times are the same, the link will not recover 1015 * and the driver will be stuck in this loop forever. Add a random 1016 * interval to the recovery time to prevent this race. 1017 */ 1018 status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1019 pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1020 1021 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1022 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1023 goto retry; 1024 1025 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1026 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1027 goto retry; 1028 1029 status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1030 if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1031 goto out; 1032 1033 status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1034 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 1035 speed = (status16 & NTB_LINK_SPEED_MASK); 1036 if (ntb->link_width != width || ntb->link_speed != speed) 1037 goto retry; 1038 1039out: 1040 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1041 ntb_handle_heartbeat, ntb); 1042 return; 1043 1044retry: 1045 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1046 ntb); 1047} 1048 1049static int 1050ntb_check_link_status(struct ntb_softc *ntb) 1051{ 1052 int link_state; 1053 uint32_t ntb_cntl; 1054 uint16_t status; 1055 1056 if (ntb->type == NTB_SOC) { 1057 ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1058 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1059 link_state = NTB_LINK_DOWN; 1060 else 1061 link_state = NTB_LINK_UP; 1062 } else { 1063 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 1064 2); 1065 1066 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1067 link_state = NTB_LINK_UP; 1068 else 1069 link_state = NTB_LINK_DOWN; 1070 } 1071 1072 ntb_handle_link_event(ntb, link_state); 1073 1074 return (0); 1075} 1076 1077/** 1078 * ntb_register_event_callback() - register event callback 1079 * @ntb: pointer to ntb_softc instance 1080 * @func: callback function to register 1081 * 1082 * This function registers a callback for any HW driver events such as link 1083 * up/down, power management notices and etc. 1084 * 1085 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1086 */ 1087int 1088ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1089{ 1090 1091 if (ntb->event_cb != NULL) 1092 return (EINVAL); 1093 1094 ntb->event_cb = func; 1095 1096 return (0); 1097} 1098 1099/** 1100 * ntb_unregister_event_callback() - unregisters the event callback 1101 * @ntb: pointer to ntb_softc instance 1102 * 1103 * This function unregisters the existing callback from transport 1104 */ 1105void 1106ntb_unregister_event_callback(struct ntb_softc *ntb) 1107{ 1108 1109 ntb->event_cb = NULL; 1110} 1111 1112/** 1113 * ntb_register_db_callback() - register a callback for doorbell interrupt 1114 * @ntb: pointer to ntb_softc instance 1115 * @idx: doorbell index to register callback, zero based 1116 * @func: callback function to register 1117 * 1118 * This function registers a callback function for the doorbell interrupt 1119 * on the primary side. The function will unmask the doorbell as well to 1120 * allow interrupt. 1121 * 1122 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1123 */ 1124int 1125ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1126 ntb_db_callback func) 1127{ 1128 uint16_t mask; 1129 1130 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1131 device_printf(ntb->device, "Invalid Index.\n"); 1132 return (EINVAL); 1133 } 1134 1135 ntb->db_cb[idx].callback = func; 1136 ntb->db_cb[idx].data = data; 1137 1138 /* unmask interrupt */ 1139 mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask); 1140 mask &= ~(1 << (idx * ntb->bits_per_vector)); 1141 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask); 1142 1143 return (0); 1144} 1145 1146/** 1147 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1148 * @ntb: pointer to ntb_softc instance 1149 * @idx: doorbell index to register callback, zero based 1150 * 1151 * This function unregisters a callback function for the doorbell interrupt 1152 * on the primary side. The function will also mask the said doorbell. 1153 */ 1154void 1155ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1156{ 1157 unsigned long mask; 1158 1159 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1160 return; 1161 1162 mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask); 1163 mask |= 1 << (idx * ntb->bits_per_vector); 1164 ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask); 1165 1166 ntb->db_cb[idx].callback = NULL; 1167} 1168 1169/** 1170 * ntb_find_transport() - find the transport pointer 1171 * @transport: pointer to pci device 1172 * 1173 * Given the pci device pointer, return the transport pointer passed in when 1174 * the transport attached when it was inited. 1175 * 1176 * RETURNS: pointer to transport. 1177 */ 1178void * 1179ntb_find_transport(struct ntb_softc *ntb) 1180{ 1181 1182 return (ntb->ntb_transport); 1183} 1184 1185/** 1186 * ntb_register_transport() - Register NTB transport with NTB HW driver 1187 * @transport: transport identifier 1188 * 1189 * This function allows a transport to reserve the hardware driver for 1190 * NTB usage. 1191 * 1192 * RETURNS: pointer to ntb_softc, NULL on error. 1193 */ 1194struct ntb_softc * 1195ntb_register_transport(struct ntb_softc *ntb, void *transport) 1196{ 1197 1198 /* 1199 * TODO: when we have more than one transport, we will need to rewrite 1200 * this to prevent race conditions 1201 */ 1202 if (ntb->ntb_transport != NULL) 1203 return (NULL); 1204 1205 ntb->ntb_transport = transport; 1206 return (ntb); 1207} 1208 1209/** 1210 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1211 * @ntb - ntb_softc of the transport to be freed 1212 * 1213 * This function unregisters the transport from the HW driver and performs any 1214 * necessary cleanups. 1215 */ 1216void 1217ntb_unregister_transport(struct ntb_softc *ntb) 1218{ 1219 int i; 1220 1221 if (ntb->ntb_transport == NULL) 1222 return; 1223 1224 for (i = 0; i < ntb->allocated_interrupts; i++) 1225 ntb_unregister_db_callback(ntb, i); 1226 1227 ntb_unregister_event_callback(ntb); 1228 ntb->ntb_transport = NULL; 1229} 1230 1231/** 1232 * ntb_get_max_spads() - get the total scratch regs usable 1233 * @ntb: pointer to ntb_softc instance 1234 * 1235 * This function returns the max 32bit scratchpad registers usable by the 1236 * upper layer. 1237 * 1238 * RETURNS: total number of scratch pad registers available 1239 */ 1240uint8_t 1241ntb_get_max_spads(struct ntb_softc *ntb) 1242{ 1243 1244 return (ntb->limits.max_spads); 1245} 1246 1247/** 1248 * ntb_write_local_spad() - write to the secondary scratchpad register 1249 * @ntb: pointer to ntb_softc instance 1250 * @idx: index to the scratchpad register, 0 based 1251 * @val: the data value to put into the register 1252 * 1253 * This function allows writing of a 32bit value to the indexed scratchpad 1254 * register. The register resides on the secondary (external) side. 1255 * 1256 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1257 */ 1258int 1259ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1260{ 1261 1262 if (idx >= ntb->limits.max_spads) 1263 return (EINVAL); 1264 1265 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1266 1267 return (0); 1268} 1269 1270/** 1271 * ntb_read_local_spad() - read from the primary scratchpad register 1272 * @ntb: pointer to ntb_softc instance 1273 * @idx: index to scratchpad register, 0 based 1274 * @val: pointer to 32bit integer for storing the register value 1275 * 1276 * This function allows reading of the 32bit scratchpad register on 1277 * the primary (internal) side. 1278 * 1279 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1280 */ 1281int 1282ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1283{ 1284 1285 if (idx >= ntb->limits.max_spads) 1286 return (EINVAL); 1287 1288 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1289 1290 return (0); 1291} 1292 1293/** 1294 * ntb_write_remote_spad() - write to the secondary scratchpad register 1295 * @ntb: pointer to ntb_softc instance 1296 * @idx: index to the scratchpad register, 0 based 1297 * @val: the data value to put into the register 1298 * 1299 * This function allows writing of a 32bit value to the indexed scratchpad 1300 * register. The register resides on the secondary (external) side. 1301 * 1302 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1303 */ 1304int 1305ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1306{ 1307 1308 if (idx >= ntb->limits.max_spads) 1309 return (EINVAL); 1310 1311 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1312 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1313 else 1314 ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1315 1316 return (0); 1317} 1318 1319/** 1320 * ntb_read_remote_spad() - read from the primary scratchpad register 1321 * @ntb: pointer to ntb_softc instance 1322 * @idx: index to scratchpad register, 0 based 1323 * @val: pointer to 32bit integer for storing the register value 1324 * 1325 * This function allows reading of the 32bit scratchpad register on 1326 * the primary (internal) side. 1327 * 1328 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1329 */ 1330int 1331ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1332{ 1333 1334 if (idx >= ntb->limits.max_spads) 1335 return (EINVAL); 1336 1337 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1338 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1339 else 1340 *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1341 1342 return (0); 1343} 1344 1345/** 1346 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1347 * @ntb: pointer to ntb_softc instance 1348 * @mw: memory window number 1349 * 1350 * This function provides the base virtual address of the memory window 1351 * specified. 1352 * 1353 * RETURNS: pointer to virtual address, or NULL on error. 1354 */ 1355void * 1356ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1357{ 1358 1359 if (mw >= NTB_NUM_MW) 1360 return (NULL); 1361 1362 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1363} 1364 1365vm_paddr_t 1366ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1367{ 1368 1369 if (mw >= NTB_NUM_MW) 1370 return (0); 1371 1372 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1373} 1374 1375/** 1376 * ntb_get_mw_size() - return size of NTB memory window 1377 * @ntb: pointer to ntb_softc instance 1378 * @mw: memory window number 1379 * 1380 * This function provides the physical size of the memory window specified 1381 * 1382 * RETURNS: the size of the memory window or zero on error 1383 */ 1384u_long 1385ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1386{ 1387 1388 if (mw >= NTB_NUM_MW) 1389 return (0); 1390 1391 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1392} 1393 1394/** 1395 * ntb_set_mw_addr - set the memory window address 1396 * @ntb: pointer to ntb_softc instance 1397 * @mw: memory window number 1398 * @addr: base address for data 1399 * 1400 * This function sets the base physical address of the memory window. This 1401 * memory address is where data from the remote system will be transfered into 1402 * or out of depending on how the transport is configured. 1403 */ 1404void 1405ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1406{ 1407 1408 if (mw >= NTB_NUM_MW) 1409 return; 1410 1411 switch (NTB_MW_TO_BAR(mw)) { 1412 case NTB_B2B_BAR_1: 1413 ntb_reg_write(8, ntb->reg_ofs.sbar2_xlat, addr); 1414 break; 1415 case NTB_B2B_BAR_2: 1416 ntb_reg_write(8, ntb->reg_ofs.sbar4_xlat, addr); 1417 break; 1418 } 1419} 1420 1421/** 1422 * ntb_ring_sdb() - Set the doorbell on the secondary/external side 1423 * @ntb: pointer to ntb_softc instance 1424 * @db: doorbell to ring 1425 * 1426 * This function allows triggering of a doorbell on the secondary/external 1427 * side that will initiate an interrupt on the remote host 1428 * 1429 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1430 */ 1431void 1432ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db) 1433{ 1434 1435 if (ntb->type == NTB_SOC) 1436 ntb_reg_write(8, ntb->reg_ofs.sdb, (uint64_t) 1 << db); 1437 else { 1438 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1439 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1440 ((1 << ntb->bits_per_vector) - 1) << 1441 (db * ntb->bits_per_vector)); 1442 else 1443 ntb_reg_write(2, ntb->reg_ofs.sdb, 1444 ((1 << ntb->bits_per_vector) - 1) << 1445 (db * ntb->bits_per_vector)); 1446 } 1447} 1448 1449/** 1450 * ntb_query_link_status() - return the hardware link status 1451 * @ndev: pointer to ntb_device instance 1452 * 1453 * Returns true if the hardware is connected to the remote system 1454 * 1455 * RETURNS: true or false based on the hardware link state 1456 */ 1457bool 1458ntb_query_link_status(struct ntb_softc *ntb) 1459{ 1460 1461 return (ntb->link_status == NTB_LINK_UP); 1462} 1463 1464static void 1465save_bar_parameters(struct ntb_pci_bar_info *bar) 1466{ 1467 1468 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 1469 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 1470 bar->pbase = rman_get_start(bar->pci_resource); 1471 bar->size = rman_get_size(bar->pci_resource); 1472 bar->vbase = rman_get_virtual(bar->pci_resource); 1473} 1474 1475device_t 1476ntb_get_device(struct ntb_softc *ntb) 1477{ 1478 1479 return (ntb->device); 1480} 1481 1482/* Export HW-specific errata information. */ 1483bool 1484ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 1485{ 1486 1487 return (HAS_FEATURE(feature)); 1488} 1489