ntb_hw_intel.c revision 289342
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289342 2015-10-14 23:46:15Z cem $"); 29 30#include <sys/param.h> 31#include <sys/kernel.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/malloc.h> 35#include <sys/module.h> 36#include <sys/queue.h> 37#include <sys/rman.h> 38#include <sys/sysctl.h> 39#include <vm/vm.h> 40#include <vm/pmap.h> 41#include <machine/bus.h> 42#include <machine/pmap.h> 43#include <machine/resource.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46 47#include "ntb_regs.h" 48#include "ntb_hw.h" 49 50/* 51 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52 * allows you to connect two systems using a PCI-e link. 53 * 54 * This module contains the hardware abstraction layer for the NTB. It allows 55 * you to send and recieve interrupts, map the memory windows and send and 56 * receive messages in the scratch-pad registers. 57 * 58 * NOTE: Much of the code in this module is shared with Linux. Any patches may 59 * be picked up and redistributed in Linux with a dual GPL/BSD license. 60 */ 61 62#define NTB_CONFIG_BAR 0 63#define NTB_B2B_BAR_1 1 64#define NTB_B2B_BAR_2 2 65#define NTB_MAX_BARS 3 66#define NTB_MW_TO_BAR(mw) ((mw) + 1) 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define SOC_LINK_RECOVERY_TIME 500 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75enum ntb_device_type { 76 NTB_XEON, 77 NTB_SOC 78}; 79 80/* Device features and workarounds */ 81#define HAS_FEATURE(feature) \ 82 ((ntb->features & (feature)) != 0) 83 84struct ntb_hw_info { 85 uint32_t device_id; 86 const char *desc; 87 enum ntb_device_type type; 88 uint64_t features; 89}; 90 91struct ntb_pci_bar_info { 92 bus_space_tag_t pci_bus_tag; 93 bus_space_handle_t pci_bus_handle; 94 int pci_resource_id; 95 struct resource *pci_resource; 96 vm_paddr_t pbase; 97 void *vbase; 98 u_long size; 99}; 100 101struct ntb_int_info { 102 struct resource *res; 103 int rid; 104 void *tag; 105}; 106 107struct ntb_db_cb { 108 ntb_db_callback callback; 109 unsigned int db_num; 110 void *data; 111 struct ntb_softc *ntb; 112 struct callout irq_work; 113}; 114 115struct ntb_softc { 116 device_t device; 117 enum ntb_device_type type; 118 uint64_t features; 119 120 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 121 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 122 uint32_t allocated_interrupts; 123 124 struct callout heartbeat_timer; 125 struct callout lr_timer; 126 127 void *ntb_transport; 128 ntb_event_callback event_cb; 129 struct ntb_db_cb *db_cb; 130 131 struct { 132 uint8_t max_spads; 133 uint8_t max_db_bits; 134 uint8_t msix_cnt; 135 } limits; 136 struct { 137 uint32_t ldb; 138 uint32_t ldb_mask; 139 uint32_t rdb; 140 uint32_t bar2_xlat; 141 uint32_t bar4_xlat; 142 uint32_t spad_remote; 143 uint32_t spad_local; 144 uint32_t lnk_cntl; 145 uint32_t lnk_stat; 146 uint32_t spci_cmd; 147 } reg_ofs; 148 uint8_t conn_type; 149 uint8_t dev_type; 150 uint8_t bits_per_vector; 151 uint8_t link_status; 152 uint8_t link_width; 153 uint8_t link_speed; 154}; 155 156#ifdef __i386__ 157static __inline uint64_t 158bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 159 bus_size_t offset) 160{ 161 162 return (bus_space_read_4(tag, handle, offset) | 163 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 164} 165 166static __inline void 167bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 168 bus_size_t offset, uint64_t val) 169{ 170 171 bus_space_write_4(tag, handle, offset, val); 172 bus_space_write_4(tag, handle, offset + 4, val >> 32); 173} 174#endif 175 176#define ntb_bar_read(SIZE, bar, offset) \ 177 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 178 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 179#define ntb_bar_write(SIZE, bar, offset, val) \ 180 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 181 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 182#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 183#define ntb_reg_write(SIZE, offset, val) \ 184 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 185#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 186#define ntb_mw_write(SIZE, offset, val) \ 187 ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 188 189typedef int (*bar_map_strategy)(struct ntb_softc *ntb, 190 struct ntb_pci_bar_info *bar); 191 192static int ntb_probe(device_t device); 193static int ntb_attach(device_t device); 194static int ntb_detach(device_t device); 195static int ntb_map_pci_bars(struct ntb_softc *ntb); 196static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 197 struct ntb_pci_bar_info *bar); 198static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 199static int map_memory_window_bar(struct ntb_softc *ntb, 200 struct ntb_pci_bar_info *bar); 201static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 202static int ntb_setup_interrupts(struct ntb_softc *ntb); 203static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 204static int ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors); 205static int ntb_setup_soc_msix(struct ntb_softc *ntb, uint32_t num_vectors); 206static void ntb_teardown_interrupts(struct ntb_softc *ntb); 207static void handle_soc_irq(void *arg); 208static void handle_xeon_irq(void *arg); 209static void handle_xeon_event_irq(void *arg); 210static void ntb_handle_legacy_interrupt(void *arg); 211static void ntb_irq_work(void *arg); 212static void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 213static void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 214static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors); 215static void ntb_free_callbacks(struct ntb_softc *ntb); 216static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 217static int ntb_setup_xeon(struct ntb_softc *ntb); 218static int ntb_setup_soc(struct ntb_softc *ntb); 219static void ntb_teardown_xeon(struct ntb_softc *ntb); 220static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 221static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 222static void ntb_handle_heartbeat(void *arg); 223static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 224static void ntb_hw_link_down(struct ntb_softc *ntb); 225static void ntb_hw_link_up(struct ntb_softc *ntb); 226static void recover_soc_link(void *arg); 227static int ntb_check_link_status(struct ntb_softc *ntb); 228static void save_bar_parameters(struct ntb_pci_bar_info *bar); 229 230static struct ntb_hw_info pci_ids[] = { 231 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 232 233 /* XXX: PS/SS IDs left out until they are supported. */ 234 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 235 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 236 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 237 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 238 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 239 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 240 | NTB_BAR_SIZE_4K }, 241 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 242 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 243 }, 244 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 245 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 246 }, 247 248 { 0x00000000, NULL, NTB_SOC, 0 } 249}; 250 251/* 252 * OS <-> Driver interface structures 253 */ 254MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 255 256static device_method_t ntb_pci_methods[] = { 257 /* Device interface */ 258 DEVMETHOD(device_probe, ntb_probe), 259 DEVMETHOD(device_attach, ntb_attach), 260 DEVMETHOD(device_detach, ntb_detach), 261 DEVMETHOD_END 262}; 263 264static driver_t ntb_pci_driver = { 265 "ntb_hw", 266 ntb_pci_methods, 267 sizeof(struct ntb_softc), 268}; 269 270static devclass_t ntb_devclass; 271DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 272MODULE_VERSION(ntb_hw, 1); 273 274SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 275 276/* 277 * OS <-> Driver linkage functions 278 */ 279static int 280ntb_probe(device_t device) 281{ 282 struct ntb_hw_info *p; 283 284 p = ntb_get_device_info(pci_get_devid(device)); 285 if (p == NULL) 286 return (ENXIO); 287 288 device_set_desc(device, p->desc); 289 return (0); 290} 291 292static int 293ntb_attach(device_t device) 294{ 295 struct ntb_softc *ntb; 296 struct ntb_hw_info *p; 297 int error; 298 299 ntb = DEVICE2SOFTC(device); 300 p = ntb_get_device_info(pci_get_devid(device)); 301 302 ntb->device = device; 303 ntb->type = p->type; 304 ntb->features = p->features; 305 306 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 307 callout_init(&ntb->heartbeat_timer, 1); 308 callout_init(&ntb->lr_timer, 1); 309 310 error = ntb_map_pci_bars(ntb); 311 if (error) 312 goto out; 313 if (ntb->type == NTB_SOC) 314 error = ntb_setup_soc(ntb); 315 else 316 error = ntb_setup_xeon(ntb); 317 if (error) 318 goto out; 319 error = ntb_setup_interrupts(ntb); 320 if (error) 321 goto out; 322 323 pci_enable_busmaster(ntb->device); 324 325out: 326 if (error != 0) 327 ntb_detach(device); 328 return (error); 329} 330 331static int 332ntb_detach(device_t device) 333{ 334 struct ntb_softc *ntb; 335 336 ntb = DEVICE2SOFTC(device); 337 callout_drain(&ntb->heartbeat_timer); 338 callout_drain(&ntb->lr_timer); 339 if (ntb->type == NTB_XEON) 340 ntb_teardown_xeon(ntb); 341 ntb_teardown_interrupts(ntb); 342 ntb_unmap_pci_bar(ntb); 343 344 return (0); 345} 346 347static int 348ntb_map_pci_bars(struct ntb_softc *ntb) 349{ 350 int rc; 351 352 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 353 rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 354 if (rc != 0) 355 return (rc); 356 357 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 358 rc = map_pci_bar(ntb, map_memory_window_bar, 359 &ntb->bar_info[NTB_B2B_BAR_1]); 360 if (rc != 0) 361 return (rc); 362 363 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 364 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 365 rc = map_pci_bar(ntb, map_mmr_bar, 366 &ntb->bar_info[NTB_B2B_BAR_2]); 367 else 368 rc = map_pci_bar(ntb, map_memory_window_bar, 369 &ntb->bar_info[NTB_B2B_BAR_2]); 370 return (rc); 371} 372 373static int 374map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 375 struct ntb_pci_bar_info *bar) 376{ 377 int rc; 378 379 rc = strategy(ntb, bar); 380 if (rc != 0) 381 device_printf(ntb->device, 382 "unable to allocate pci resource\n"); 383 else 384 device_printf(ntb->device, 385 "Bar size = %lx, v %p, p %p\n", 386 bar->size, bar->vbase, (void *)(bar->pbase)); 387 return (rc); 388} 389 390static int 391map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 392{ 393 394 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 395 &bar->pci_resource_id, RF_ACTIVE); 396 if (bar->pci_resource == NULL) 397 return (ENXIO); 398 399 save_bar_parameters(bar); 400 return (0); 401} 402 403static int 404map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 405{ 406 int rc; 407 uint8_t bar_size_bits = 0; 408 409 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 410 &bar->pci_resource_id, RF_ACTIVE); 411 412 if (bar->pci_resource == NULL) 413 return (ENXIO); 414 415 save_bar_parameters(bar); 416 /* 417 * Ivytown NTB BAR sizes are misreported by the hardware due to a 418 * hardware issue. To work around this, query the size it should be 419 * configured to by the device and modify the resource to correspond to 420 * this new size. The BIOS on systems with this problem is required to 421 * provide enough address space to allow the driver to make this change 422 * safely. 423 * 424 * Ideally I could have just specified the size when I allocated the 425 * resource like: 426 * bus_alloc_resource(ntb->device, 427 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 428 * 1ul << bar_size_bits, RF_ACTIVE); 429 * but the PCI driver does not honor the size in this call, so we have 430 * to modify it after the fact. 431 */ 432 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 433 if (bar->pci_resource_id == PCIR_BAR(2)) 434 bar_size_bits = pci_read_config(ntb->device, 435 XEON_PBAR23SZ_OFFSET, 1); 436 else 437 bar_size_bits = pci_read_config(ntb->device, 438 XEON_PBAR45SZ_OFFSET, 1); 439 440 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 441 bar->pci_resource, bar->pbase, 442 bar->pbase + (1ul << bar_size_bits) - 1); 443 if (rc != 0) { 444 device_printf(ntb->device, 445 "unable to resize bar\n"); 446 return (rc); 447 } 448 449 save_bar_parameters(bar); 450 } 451 452 /* Mark bar region as write combining to improve performance. */ 453 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 454 VM_MEMATTR_WRITE_COMBINING); 455 if (rc != 0) { 456 device_printf(ntb->device, 457 "unable to mark bar as WRITE_COMBINING\n"); 458 return (rc); 459 } 460 return (0); 461} 462 463static void 464ntb_unmap_pci_bar(struct ntb_softc *ntb) 465{ 466 struct ntb_pci_bar_info *current_bar; 467 int i; 468 469 for (i = 0; i< NTB_MAX_BARS; i++) { 470 current_bar = &ntb->bar_info[i]; 471 if (current_bar->pci_resource != NULL) 472 bus_release_resource(ntb->device, SYS_RES_MEMORY, 473 current_bar->pci_resource_id, 474 current_bar->pci_resource); 475 } 476} 477 478static int 479ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors) 480{ 481 void (*interrupt_handler)(void *); 482 void *int_arg; 483 uint32_t i; 484 int rc; 485 486 if (num_vectors < 4) 487 return (ENOSPC); 488 489 for (i = 0; i < num_vectors; i++) { 490 ntb->int_info[i].rid = i + 1; 491 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 492 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 493 if (ntb->int_info[i].res == NULL) { 494 device_printf(ntb->device, 495 "bus_alloc_resource failed\n"); 496 return (ENOMEM); 497 } 498 ntb->int_info[i].tag = NULL; 499 ntb->allocated_interrupts++; 500 if (i == num_vectors - 1) { 501 interrupt_handler = handle_xeon_event_irq; 502 int_arg = ntb; 503 } else { 504 interrupt_handler = handle_xeon_irq; 505 int_arg = &ntb->db_cb[i]; 506 } 507 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 508 INTR_MPSAFE | INTR_TYPE_MISC, NULL, interrupt_handler, 509 int_arg, &ntb->int_info[i].tag); 510 if (rc != 0) { 511 device_printf(ntb->device, 512 "bus_setup_intr failed\n"); 513 return (ENXIO); 514 } 515 } 516 return (0); 517} 518 519static int 520ntb_setup_soc_msix(struct ntb_softc *ntb, uint32_t num_vectors) 521{ 522 uint32_t i; 523 int rc; 524 525 for (i = 0; i < num_vectors; i++) { 526 ntb->int_info[i].rid = i + 1; 527 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 528 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 529 if (ntb->int_info[i].res == NULL) { 530 device_printf(ntb->device, 531 "bus_alloc_resource failed\n"); 532 return (ENOMEM); 533 } 534 ntb->int_info[i].tag = NULL; 535 ntb->allocated_interrupts++; 536 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 537 INTR_MPSAFE | INTR_TYPE_MISC, NULL, handle_soc_irq, 538 &ntb->db_cb[i], &ntb->int_info[i].tag); 539 if (rc != 0) { 540 device_printf(ntb->device, "bus_setup_intr failed\n"); 541 return (ENXIO); 542 } 543 } 544 return (0); 545} 546 547static int 548ntb_setup_interrupts(struct ntb_softc *ntb) 549{ 550 uint32_t num_vectors; 551 int rc; 552 553 ntb->allocated_interrupts = 0; 554 /* 555 * On SOC, disable all interrupts. On XEON, disable all but Link 556 * Interrupt. The rest will be unmasked as callbacks are registered. 557 */ 558 if (ntb->type == NTB_SOC) 559 ntb_reg_write(8, ntb->reg_ofs.ldb_mask, ~0); 560 else 561 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, 562 (uint16_t) ~(1 << XEON_LINK_DB)); 563 564 num_vectors = MIN(pci_msix_count(ntb->device), 565 ntb->limits.max_db_bits); 566 if (num_vectors >= 1) 567 pci_alloc_msix(ntb->device, &num_vectors); 568 569 ntb_create_callbacks(ntb, num_vectors); 570 571 if (ntb->type == NTB_XEON) 572 rc = ntb_setup_xeon_msix(ntb, num_vectors); 573 else 574 rc = ntb_setup_soc_msix(ntb, num_vectors); 575 if (rc != 0) 576 device_printf(ntb->device, 577 "Error allocating MSI-X interrupts: %d\n", rc); 578 579 if (ntb->type == NTB_XEON && rc == ENOSPC) 580 rc = ntb_setup_legacy_interrupt(ntb); 581 582 return (rc); 583} 584 585static int 586ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 587{ 588 int rc; 589 590 ntb->int_info[0].rid = 0; 591 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 592 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 593 if (ntb->int_info[0].res == NULL) { 594 device_printf(ntb->device, "bus_alloc_resource failed\n"); 595 return (ENOMEM); 596 } 597 598 ntb->int_info[0].tag = NULL; 599 ntb->allocated_interrupts = 1; 600 601 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 602 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ntb_handle_legacy_interrupt, 603 ntb, &ntb->int_info[0].tag); 604 if (rc != 0) { 605 device_printf(ntb->device, "bus_setup_intr failed\n"); 606 return (ENXIO); 607 } 608 609 return (0); 610} 611 612static void 613ntb_teardown_interrupts(struct ntb_softc *ntb) 614{ 615 struct ntb_int_info *current_int; 616 int i; 617 618 for (i = 0; i < ntb->allocated_interrupts; i++) { 619 current_int = &ntb->int_info[i]; 620 if (current_int->tag != NULL) 621 bus_teardown_intr(ntb->device, current_int->res, 622 current_int->tag); 623 624 if (current_int->res != NULL) 625 bus_release_resource(ntb->device, SYS_RES_IRQ, 626 rman_get_rid(current_int->res), current_int->res); 627 } 628 629 ntb_free_callbacks(ntb); 630 pci_release_msi(ntb->device); 631} 632 633static void 634mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 635{ 636 unsigned long mask; 637 638 mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 639 mask |= 1 << (idx * ntb->bits_per_vector); 640 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 641} 642 643static void 644unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 645{ 646 unsigned long mask; 647 648 mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 649 mask &= ~(1 << (idx * ntb->bits_per_vector)); 650 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 651} 652 653static void 654handle_soc_irq(void *arg) 655{ 656 struct ntb_db_cb *db_cb = arg; 657 struct ntb_softc *ntb = db_cb->ntb; 658 659 ntb_reg_write(8, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num); 660 661 if (db_cb->callback != NULL) { 662 mask_ldb_interrupt(ntb, db_cb->db_num); 663 callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 664 } 665} 666 667static void 668handle_xeon_irq(void *arg) 669{ 670 struct ntb_db_cb *db_cb = arg; 671 struct ntb_softc *ntb = db_cb->ntb; 672 673 /* 674 * On Xeon, there are 16 bits in the interrupt register 675 * but only 4 vectors. So, 5 bits are assigned to the first 3 676 * vectors, with the 4th having a single bit for link 677 * interrupts. 678 */ 679 ntb_reg_write(2, ntb->reg_ofs.ldb, 680 ((1 << ntb->bits_per_vector) - 1) << 681 (db_cb->db_num * ntb->bits_per_vector)); 682 683 if (db_cb->callback != NULL) { 684 mask_ldb_interrupt(ntb, db_cb->db_num); 685 callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 686 } 687} 688 689/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 690static void 691handle_xeon_event_irq(void *arg) 692{ 693 struct ntb_softc *ntb = arg; 694 int rc; 695 696 rc = ntb_check_link_status(ntb); 697 if (rc != 0) 698 device_printf(ntb->device, "Error determining link status\n"); 699 700 /* bit 15 is always the link bit */ 701 ntb_reg_write(2, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB); 702} 703 704static void 705ntb_handle_legacy_interrupt(void *arg) 706{ 707 struct ntb_softc *ntb = arg; 708 unsigned int i = 0; 709 uint64_t ldb64; 710 uint16_t ldb16; 711 712 if (ntb->type == NTB_SOC) { 713 ldb64 = ntb_reg_read(8, ntb->reg_ofs.ldb); 714 715 while (ldb64) { 716 i = ffs(ldb64); 717 ldb64 &= ldb64 - 1; 718 handle_soc_irq(&ntb->db_cb[i]); 719 } 720 } else { 721 ldb16 = ntb_reg_read(2, ntb->reg_ofs.ldb); 722 723 if ((ldb16 & XEON_DB_HW_LINK) != 0) { 724 handle_xeon_event_irq(ntb); 725 ldb16 &= ~XEON_DB_HW_LINK; 726 } 727 728 while (ldb16 != 0) { 729 i = ffs(ldb16); 730 ldb16 &= ldb16 - 1; 731 handle_xeon_irq(&ntb->db_cb[i]); 732 } 733 } 734 735} 736 737static int 738ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors) 739{ 740 uint32_t i; 741 742 ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 743 M_ZERO | M_WAITOK); 744 for (i = 0; i < num_vectors; i++) { 745 ntb->db_cb[i].db_num = i; 746 ntb->db_cb[i].ntb = ntb; 747 } 748 749 return (0); 750} 751 752static void 753ntb_free_callbacks(struct ntb_softc *ntb) 754{ 755 uint8_t i; 756 757 for (i = 0; i < ntb->limits.max_db_bits; i++) 758 ntb_unregister_db_callback(ntb, i); 759 760 free(ntb->db_cb, M_NTB); 761} 762 763static struct ntb_hw_info * 764ntb_get_device_info(uint32_t device_id) 765{ 766 struct ntb_hw_info *ep = pci_ids; 767 768 while (ep->device_id) { 769 if (ep->device_id == device_id) 770 return (ep); 771 ++ep; 772 } 773 return (NULL); 774} 775 776static void 777ntb_teardown_xeon(struct ntb_softc *ntb) 778{ 779 780 ntb_hw_link_down(ntb); 781} 782 783static int 784ntb_setup_xeon(struct ntb_softc *ntb) 785{ 786 uint8_t val, connection_type; 787 788 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 789 790 connection_type = val & XEON_PPD_CONN_TYPE; 791 792 if ((val & XEON_PPD_DEV_TYPE) != 0) 793 ntb->dev_type = NTB_DEV_USD; 794 else 795 ntb->dev_type = NTB_DEV_DSD; 796 797 ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; 798 ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; 799 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 800 ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET; 801 ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; 802 803 switch (connection_type) { 804 case NTB_CONN_B2B: 805 ntb->conn_type = NTB_CONN_B2B; 806 807 /* 808 * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored 809 * with the NTB_REGS_THRU_MW errata mode enabled. (See 810 * ntb_ring_doorbell() and ntb_read/write_remote_spad().) 811 */ 812 ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET; 813 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 814 815 ntb->limits.max_spads = XEON_MAX_SPADS; 816 break; 817 818 case NTB_CONN_RP: 819 /* 820 * Every Xeon today needs NTB_REGS_THRU_MW, so punt on RP for 821 * now. 822 */ 823 KASSERT(HAS_FEATURE(NTB_REGS_THRU_MW), 824 ("Xeon without MW errata unimplemented")); 825 device_printf(ntb->device, 826 "NTB-RP disabled to due hardware errata.\n"); 827 return (ENXIO); 828 829 case NTB_CONN_TRANSPARENT: 830 default: 831 device_printf(ntb->device, "Connection type %d not supported\n", 832 connection_type); 833 return (ENXIO); 834 } 835 836 /* 837 * There is a Xeon hardware errata related to writes to SDOORBELL or 838 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 839 * which may hang the system. To workaround this use the second memory 840 * window to access the interrupt and scratch pad registers on the 841 * remote system. 842 * 843 * There is another HW errata on the limit registers -- they can only 844 * be written when the base register is (?)4GB aligned and < 32-bit. 845 * This should already be the case based on the driver defaults, but 846 * write the limit registers first just in case. 847 */ 848 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 849 /* 850 * Set the Limit register to 4k, the minimum size, to prevent 851 * an illegal access. 852 */ 853 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 854 ntb_get_mw_size(ntb, 1) + 0x1000); 855 else 856 /* 857 * Disable the limit register, just in case it is set to 858 * something silly. 859 */ 860 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 861 862 863 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 864 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 865 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 866 867 ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 868 ntb->limits.msix_cnt = XEON_MSIX_CNT; 869 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 870 871 /* 872 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 873 * mirrored to the remote system. Shrink the number of bits by one, 874 * since bit 14 is the last bit. 875 * 876 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 877 * anyway. Nor for non-B2B connection types. 878 */ 879 if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) && 880 !HAS_FEATURE(NTB_REGS_THRU_MW) && 881 connection_type == NTB_CONN_B2B) 882 ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1; 883 884 configure_xeon_secondary_side_bars(ntb); 885 886 /* Enable Bus Master and Memory Space on the secondary side */ 887 if (ntb->conn_type == NTB_CONN_B2B) 888 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 889 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 890 891 /* Enable link training */ 892 ntb_hw_link_up(ntb); 893 894 return (0); 895} 896 897static int 898ntb_setup_soc(struct ntb_softc *ntb) 899{ 900 uint32_t val, connection_type; 901 902 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 903 904 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 905 switch (connection_type) { 906 case NTB_CONN_B2B: 907 ntb->conn_type = NTB_CONN_B2B; 908 break; 909 default: 910 device_printf(ntb->device, 911 "Unsupported NTB configuration (%d)\n", connection_type); 912 return (ENXIO); 913 } 914 915 if ((val & SOC_PPD_DEV_TYPE) != 0) 916 ntb->dev_type = NTB_DEV_DSD; 917 else 918 ntb->dev_type = NTB_DEV_USD; 919 920 /* Initiate PCI-E link training */ 921 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 922 4); 923 924 ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; 925 ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; 926 ntb->reg_ofs.rdb = SOC_B2B_DOORBELL_OFFSET; 927 ntb->reg_ofs.bar2_xlat = SOC_SBAR2XLAT_OFFSET; 928 ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; 929 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 930 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 931 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 932 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 933 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 934 935 ntb->limits.max_spads = SOC_MAX_SPADS; 936 ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 937 ntb->limits.msix_cnt = SOC_MSIX_CNT; 938 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 939 940 /* 941 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 942 * resolved. Mask transaction layer internal parity errors. 943 */ 944 pci_write_config(ntb->device, 0xFC, 0x4, 4); 945 946 configure_soc_secondary_side_bars(ntb); 947 948 /* Enable Bus Master and Memory Space on the secondary side */ 949 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 950 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 951 952 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 953 954 return (0); 955} 956 957static void 958configure_soc_secondary_side_bars(struct ntb_softc *ntb) 959{ 960 961 if (ntb->dev_type == NTB_DEV_USD) { 962 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 963 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 964 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 965 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 966 } else { 967 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 968 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 969 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 970 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 971 } 972} 973 974static void 975configure_xeon_secondary_side_bars(struct ntb_softc *ntb) 976{ 977 978 if (ntb->dev_type == NTB_DEV_USD) { 979 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 980 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 981 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 982 MBAR01_DSD_ADDR); 983 else { 984 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 985 PBAR4XLAT_USD_ADDR); 986 /* 987 * B2B_XLAT_OFFSET is a 64-bit register but can only be 988 * written 32 bits at a time. 989 */ 990 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 991 MBAR01_DSD_ADDR & 0xffffffff); 992 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 993 MBAR01_DSD_ADDR >> 32); 994 } 995 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 996 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 997 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 998 } else { 999 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 1000 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1001 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 1002 MBAR01_USD_ADDR); 1003 else { 1004 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 1005 PBAR4XLAT_DSD_ADDR); 1006 /* 1007 * B2B_XLAT_OFFSET is a 64-bit register but can only be 1008 * written 32 bits at a time. 1009 */ 1010 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 1011 MBAR01_USD_ADDR & 0xffffffff); 1012 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 1013 MBAR01_USD_ADDR >> 32); 1014 } 1015 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 1016 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 1017 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 1018 } 1019} 1020 1021/* SOC does not have link status interrupt, poll on that platform */ 1022static void 1023ntb_handle_heartbeat(void *arg) 1024{ 1025 struct ntb_softc *ntb = arg; 1026 uint32_t status32; 1027 int rc; 1028 1029 rc = ntb_check_link_status(ntb); 1030 if (rc != 0) 1031 device_printf(ntb->device, 1032 "Error determining link status\n"); 1033 1034 /* Check to see if a link error is the cause of the link down */ 1035 if (ntb->link_status == NTB_LINK_DOWN) { 1036 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1037 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 1038 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 1039 ntb); 1040 return; 1041 } 1042 } 1043 1044 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1045 ntb_handle_heartbeat, ntb); 1046} 1047 1048static void 1049soc_perform_link_restart(struct ntb_softc *ntb) 1050{ 1051 uint32_t status; 1052 1053 /* Driver resets the NTB ModPhy lanes - magic! */ 1054 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1055 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1056 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1057 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1058 1059 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1060 pause("ModPhy", hz / 10); 1061 1062 /* Clear AER Errors, write to clear */ 1063 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1064 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1065 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1066 1067 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1068 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1069 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1070 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1071 1072 /* Clear DeSkew Buffer error, write to clear */ 1073 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1074 status |= SOC_DESKEWSTS_DBERR; 1075 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1076 1077 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1078 status &= SOC_IBIST_ERR_OFLOW; 1079 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1080 1081 /* Releases the NTB state machine to allow the link to retrain */ 1082 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1083 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1084 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1085} 1086 1087static void 1088ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 1089{ 1090 enum ntb_hw_event event; 1091 uint16_t status; 1092 1093 if (ntb->link_status == link_state) 1094 return; 1095 1096 if (link_state == NTB_LINK_UP) { 1097 device_printf(ntb->device, "Link Up\n"); 1098 ntb->link_status = NTB_LINK_UP; 1099 event = NTB_EVENT_HW_LINK_UP; 1100 1101 if (ntb->type == NTB_SOC || 1102 ntb->conn_type == NTB_CONN_TRANSPARENT) 1103 status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1104 else 1105 status = pci_read_config(ntb->device, 1106 XEON_LINK_STATUS_OFFSET, 2); 1107 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 1108 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 1109 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 1110 ntb->link_width, ntb->link_speed); 1111 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1112 ntb_handle_heartbeat, ntb); 1113 } else { 1114 device_printf(ntb->device, "Link Down\n"); 1115 ntb->link_status = NTB_LINK_DOWN; 1116 event = NTB_EVENT_HW_LINK_DOWN; 1117 /* Do not modify link width/speed, we need it in link recovery */ 1118 } 1119 1120 /* notify the upper layer if we have an event change */ 1121 if (ntb->event_cb != NULL) 1122 ntb->event_cb(ntb->ntb_transport, event); 1123} 1124 1125static void 1126ntb_hw_link_up(struct ntb_softc *ntb) 1127{ 1128 uint32_t cntl; 1129 1130 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1131 ntb_handle_link_event(ntb, NTB_LINK_UP); 1132 return; 1133 } 1134 1135 cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1136 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1137 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1138 cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP; 1139 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1140} 1141 1142static void 1143ntb_hw_link_down(struct ntb_softc *ntb) 1144{ 1145 uint32_t cntl; 1146 1147 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1148 ntb_handle_link_event(ntb, NTB_LINK_DOWN); 1149 return; 1150 } 1151 1152 cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1153 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1154 cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP); 1155 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1156 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1157} 1158 1159static void 1160recover_soc_link(void *arg) 1161{ 1162 struct ntb_softc *ntb = arg; 1163 uint8_t speed, width; 1164 uint32_t status32; 1165 uint16_t status16; 1166 1167 soc_perform_link_restart(ntb); 1168 1169 /* 1170 * There is a potential race between the 2 NTB devices recovering at 1171 * the same time. If the times are the same, the link will not recover 1172 * and the driver will be stuck in this loop forever. Add a random 1173 * interval to the recovery time to prevent this race. 1174 */ 1175 status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1176 pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1177 1178 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1179 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1180 goto retry; 1181 1182 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1183 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1184 goto retry; 1185 1186 status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1187 if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1188 goto out; 1189 1190 status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1191 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 1192 speed = (status16 & NTB_LINK_SPEED_MASK); 1193 if (ntb->link_width != width || ntb->link_speed != speed) 1194 goto retry; 1195 1196out: 1197 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1198 ntb_handle_heartbeat, ntb); 1199 return; 1200 1201retry: 1202 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1203 ntb); 1204} 1205 1206static int 1207ntb_check_link_status(struct ntb_softc *ntb) 1208{ 1209 int link_state; 1210 uint32_t ntb_cntl; 1211 uint16_t status; 1212 1213 if (ntb->type == NTB_SOC) { 1214 ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1215 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1216 link_state = NTB_LINK_DOWN; 1217 else 1218 link_state = NTB_LINK_UP; 1219 } else { 1220 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 1221 2); 1222 1223 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1224 link_state = NTB_LINK_UP; 1225 else 1226 link_state = NTB_LINK_DOWN; 1227 } 1228 1229 ntb_handle_link_event(ntb, link_state); 1230 1231 return (0); 1232} 1233 1234/** 1235 * ntb_register_event_callback() - register event callback 1236 * @ntb: pointer to ntb_softc instance 1237 * @func: callback function to register 1238 * 1239 * This function registers a callback for any HW driver events such as link 1240 * up/down, power management notices and etc. 1241 * 1242 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1243 */ 1244int 1245ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1246{ 1247 1248 if (ntb->event_cb != NULL) 1249 return (EINVAL); 1250 1251 ntb->event_cb = func; 1252 1253 return (0); 1254} 1255 1256/** 1257 * ntb_unregister_event_callback() - unregisters the event callback 1258 * @ntb: pointer to ntb_softc instance 1259 * 1260 * This function unregisters the existing callback from transport 1261 */ 1262void 1263ntb_unregister_event_callback(struct ntb_softc *ntb) 1264{ 1265 1266 ntb->event_cb = NULL; 1267} 1268 1269static void 1270ntb_irq_work(void *arg) 1271{ 1272 struct ntb_db_cb *db_cb = arg; 1273 struct ntb_softc *ntb; 1274 int rc; 1275 1276 rc = db_cb->callback(db_cb->data, db_cb->db_num); 1277 /* Poll if forward progress was made. */ 1278 if (rc != 0) { 1279 callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 1280 return; 1281 } 1282 1283 /* Unmask interrupt if no progress was made. */ 1284 ntb = db_cb->ntb; 1285 unmask_ldb_interrupt(ntb, db_cb->db_num); 1286} 1287 1288/** 1289 * ntb_register_db_callback() - register a callback for doorbell interrupt 1290 * @ntb: pointer to ntb_softc instance 1291 * @idx: doorbell index to register callback, zero based 1292 * @data: pointer to be returned to caller with every callback 1293 * @func: callback function to register 1294 * 1295 * This function registers a callback function for the doorbell interrupt 1296 * on the primary side. The function will unmask the doorbell as well to 1297 * allow interrupt. 1298 * 1299 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1300 */ 1301int 1302ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1303 ntb_db_callback func) 1304{ 1305 1306 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1307 device_printf(ntb->device, "Invalid Index.\n"); 1308 return (EINVAL); 1309 } 1310 1311 ntb->db_cb[idx].callback = func; 1312 ntb->db_cb[idx].data = data; 1313 callout_init(&ntb->db_cb[idx].irq_work, 1); 1314 1315 unmask_ldb_interrupt(ntb, idx); 1316 1317 return (0); 1318} 1319 1320/** 1321 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1322 * @ntb: pointer to ntb_softc instance 1323 * @idx: doorbell index to register callback, zero based 1324 * 1325 * This function unregisters a callback function for the doorbell interrupt 1326 * on the primary side. The function will also mask the said doorbell. 1327 */ 1328void 1329ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1330{ 1331 1332 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1333 return; 1334 1335 mask_ldb_interrupt(ntb, idx); 1336 1337 callout_drain(&ntb->db_cb[idx].irq_work); 1338 ntb->db_cb[idx].callback = NULL; 1339} 1340 1341/** 1342 * ntb_find_transport() - find the transport pointer 1343 * @transport: pointer to pci device 1344 * 1345 * Given the pci device pointer, return the transport pointer passed in when 1346 * the transport attached when it was inited. 1347 * 1348 * RETURNS: pointer to transport. 1349 */ 1350void * 1351ntb_find_transport(struct ntb_softc *ntb) 1352{ 1353 1354 return (ntb->ntb_transport); 1355} 1356 1357/** 1358 * ntb_register_transport() - Register NTB transport with NTB HW driver 1359 * @transport: transport identifier 1360 * 1361 * This function allows a transport to reserve the hardware driver for 1362 * NTB usage. 1363 * 1364 * RETURNS: pointer to ntb_softc, NULL on error. 1365 */ 1366struct ntb_softc * 1367ntb_register_transport(struct ntb_softc *ntb, void *transport) 1368{ 1369 1370 /* 1371 * TODO: when we have more than one transport, we will need to rewrite 1372 * this to prevent race conditions 1373 */ 1374 if (ntb->ntb_transport != NULL) 1375 return (NULL); 1376 1377 ntb->ntb_transport = transport; 1378 return (ntb); 1379} 1380 1381/** 1382 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1383 * @ntb - ntb_softc of the transport to be freed 1384 * 1385 * This function unregisters the transport from the HW driver and performs any 1386 * necessary cleanups. 1387 */ 1388void 1389ntb_unregister_transport(struct ntb_softc *ntb) 1390{ 1391 int i; 1392 1393 if (ntb->ntb_transport == NULL) 1394 return; 1395 1396 for (i = 0; i < ntb->allocated_interrupts; i++) 1397 ntb_unregister_db_callback(ntb, i); 1398 1399 ntb_unregister_event_callback(ntb); 1400 ntb->ntb_transport = NULL; 1401} 1402 1403/** 1404 * ntb_get_max_spads() - get the total scratch regs usable 1405 * @ntb: pointer to ntb_softc instance 1406 * 1407 * This function returns the max 32bit scratchpad registers usable by the 1408 * upper layer. 1409 * 1410 * RETURNS: total number of scratch pad registers available 1411 */ 1412uint8_t 1413ntb_get_max_spads(struct ntb_softc *ntb) 1414{ 1415 1416 return (ntb->limits.max_spads); 1417} 1418 1419/** 1420 * ntb_write_local_spad() - write to the secondary scratchpad register 1421 * @ntb: pointer to ntb_softc instance 1422 * @idx: index to the scratchpad register, 0 based 1423 * @val: the data value to put into the register 1424 * 1425 * This function allows writing of a 32bit value to the indexed scratchpad 1426 * register. The register resides on the secondary (external) side. 1427 * 1428 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1429 */ 1430int 1431ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1432{ 1433 1434 if (idx >= ntb->limits.max_spads) 1435 return (EINVAL); 1436 1437 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1438 1439 return (0); 1440} 1441 1442/** 1443 * ntb_read_local_spad() - read from the primary scratchpad register 1444 * @ntb: pointer to ntb_softc instance 1445 * @idx: index to scratchpad register, 0 based 1446 * @val: pointer to 32bit integer for storing the register value 1447 * 1448 * This function allows reading of the 32bit scratchpad register on 1449 * the primary (internal) side. 1450 * 1451 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1452 */ 1453int 1454ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1455{ 1456 1457 if (idx >= ntb->limits.max_spads) 1458 return (EINVAL); 1459 1460 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1461 1462 return (0); 1463} 1464 1465/** 1466 * ntb_write_remote_spad() - write to the secondary scratchpad register 1467 * @ntb: pointer to ntb_softc instance 1468 * @idx: index to the scratchpad register, 0 based 1469 * @val: the data value to put into the register 1470 * 1471 * This function allows writing of a 32bit value to the indexed scratchpad 1472 * register. The register resides on the secondary (external) side. 1473 * 1474 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1475 */ 1476int 1477ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1478{ 1479 1480 if (idx >= ntb->limits.max_spads) 1481 return (EINVAL); 1482 1483 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1484 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1485 else 1486 ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1487 1488 return (0); 1489} 1490 1491/** 1492 * ntb_read_remote_spad() - read from the primary scratchpad register 1493 * @ntb: pointer to ntb_softc instance 1494 * @idx: index to scratchpad register, 0 based 1495 * @val: pointer to 32bit integer for storing the register value 1496 * 1497 * This function allows reading of the 32bit scratchpad register on 1498 * the primary (internal) side. 1499 * 1500 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1501 */ 1502int 1503ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1504{ 1505 1506 if (idx >= ntb->limits.max_spads) 1507 return (EINVAL); 1508 1509 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1510 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1511 else 1512 *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1513 1514 return (0); 1515} 1516 1517/** 1518 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1519 * @ntb: pointer to ntb_softc instance 1520 * @mw: memory window number 1521 * 1522 * This function provides the base virtual address of the memory window 1523 * specified. 1524 * 1525 * RETURNS: pointer to virtual address, or NULL on error. 1526 */ 1527void * 1528ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1529{ 1530 1531 if (mw >= NTB_NUM_MW) 1532 return (NULL); 1533 1534 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1535} 1536 1537vm_paddr_t 1538ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1539{ 1540 1541 if (mw >= NTB_NUM_MW) 1542 return (0); 1543 1544 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1545} 1546 1547/** 1548 * ntb_get_mw_size() - return size of NTB memory window 1549 * @ntb: pointer to ntb_softc instance 1550 * @mw: memory window number 1551 * 1552 * This function provides the physical size of the memory window specified 1553 * 1554 * RETURNS: the size of the memory window or zero on error 1555 */ 1556u_long 1557ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1558{ 1559 1560 if (mw >= NTB_NUM_MW) 1561 return (0); 1562 1563 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1564} 1565 1566/** 1567 * ntb_set_mw_addr - set the memory window address 1568 * @ntb: pointer to ntb_softc instance 1569 * @mw: memory window number 1570 * @addr: base address for data 1571 * 1572 * This function sets the base physical address of the memory window. This 1573 * memory address is where data from the remote system will be transfered into 1574 * or out of depending on how the transport is configured. 1575 */ 1576void 1577ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1578{ 1579 1580 if (mw >= NTB_NUM_MW) 1581 return; 1582 1583 switch (NTB_MW_TO_BAR(mw)) { 1584 case NTB_B2B_BAR_1: 1585 ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr); 1586 break; 1587 case NTB_B2B_BAR_2: 1588 ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr); 1589 break; 1590 } 1591} 1592 1593/** 1594 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side 1595 * @ntb: pointer to ntb_softc instance 1596 * @db: doorbell to ring 1597 * 1598 * This function allows triggering of a doorbell on the secondary/external 1599 * side that will initiate an interrupt on the remote host 1600 * 1601 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1602 */ 1603void 1604ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db) 1605{ 1606 1607 if (ntb->type == NTB_SOC) 1608 ntb_reg_write(8, ntb->reg_ofs.rdb, (uint64_t) 1 << db); 1609 else { 1610 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1611 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1612 ((1 << ntb->bits_per_vector) - 1) << 1613 (db * ntb->bits_per_vector)); 1614 else 1615 ntb_reg_write(2, ntb->reg_ofs.rdb, 1616 ((1 << ntb->bits_per_vector) - 1) << 1617 (db * ntb->bits_per_vector)); 1618 } 1619} 1620 1621/** 1622 * ntb_query_link_status() - return the hardware link status 1623 * @ndev: pointer to ntb_device instance 1624 * 1625 * Returns true if the hardware is connected to the remote system 1626 * 1627 * RETURNS: true or false based on the hardware link state 1628 */ 1629bool 1630ntb_query_link_status(struct ntb_softc *ntb) 1631{ 1632 1633 return (ntb->link_status == NTB_LINK_UP); 1634} 1635 1636static void 1637save_bar_parameters(struct ntb_pci_bar_info *bar) 1638{ 1639 1640 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 1641 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 1642 bar->pbase = rman_get_start(bar->pci_resource); 1643 bar->size = rman_get_size(bar->pci_resource); 1644 bar->vbase = rman_get_virtual(bar->pci_resource); 1645} 1646 1647device_t 1648ntb_get_device(struct ntb_softc *ntb) 1649{ 1650 1651 return (ntb->device); 1652} 1653 1654/* Export HW-specific errata information. */ 1655bool 1656ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 1657{ 1658 1659 return (HAS_FEATURE(feature)); 1660} 1661