ntb_hw_intel.c revision 289281
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289281 2015-10-14 02:14:45Z cem $"); 29 30#include <sys/param.h> 31#include <sys/kernel.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/malloc.h> 35#include <sys/module.h> 36#include <sys/queue.h> 37#include <sys/rman.h> 38#include <sys/sysctl.h> 39#include <vm/vm.h> 40#include <vm/pmap.h> 41#include <machine/bus.h> 42#include <machine/pmap.h> 43#include <machine/resource.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46 47#include "ntb_regs.h" 48#include "ntb_hw.h" 49 50/* 51 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52 * allows you to connect two systems using a PCI-e link. 53 * 54 * This module contains the hardware abstraction layer for the NTB. It allows 55 * you to send and recieve interrupts, map the memory windows and send and 56 * receive messages in the scratch-pad registers. 57 * 58 * NOTE: Much of the code in this module is shared with Linux. Any patches may 59 * be picked up and redistributed in Linux with a dual GPL/BSD license. 60 */ 61 62#define NTB_CONFIG_BAR 0 63#define NTB_B2B_BAR_1 1 64#define NTB_B2B_BAR_2 2 65#define NTB_MAX_BARS 3 66#define NTB_MW_TO_BAR(mw) ((mw) + 1) 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define SOC_LINK_RECOVERY_TIME 500 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75enum ntb_device_type { 76 NTB_XEON, 77 NTB_SOC 78}; 79 80/* Device features and workarounds */ 81#define HAS_FEATURE(feature) \ 82 ((ntb->features & (feature)) != 0) 83 84struct ntb_hw_info { 85 uint32_t device_id; 86 const char *desc; 87 enum ntb_device_type type; 88 uint64_t features; 89}; 90 91struct ntb_pci_bar_info { 92 bus_space_tag_t pci_bus_tag; 93 bus_space_handle_t pci_bus_handle; 94 int pci_resource_id; 95 struct resource *pci_resource; 96 vm_paddr_t pbase; 97 void *vbase; 98 u_long size; 99}; 100 101struct ntb_int_info { 102 struct resource *res; 103 int rid; 104 void *tag; 105}; 106 107struct ntb_db_cb { 108 ntb_db_callback callback; 109 unsigned int db_num; 110 void *data; 111 struct ntb_softc *ntb; 112 struct callout irq_work; 113}; 114 115struct ntb_softc { 116 device_t device; 117 enum ntb_device_type type; 118 uint64_t features; 119 120 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 121 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 122 uint32_t allocated_interrupts; 123 124 struct callout heartbeat_timer; 125 struct callout lr_timer; 126 127 void *ntb_transport; 128 ntb_event_callback event_cb; 129 struct ntb_db_cb *db_cb; 130 131 struct { 132 uint8_t max_spads; 133 uint8_t max_db_bits; 134 uint8_t msix_cnt; 135 } limits; 136 struct { 137 uint32_t ldb; 138 uint32_t ldb_mask; 139 uint32_t rdb; 140 uint32_t bar2_xlat; 141 uint32_t bar4_xlat; 142 uint32_t spad_remote; 143 uint32_t spad_local; 144 uint32_t lnk_cntl; 145 uint32_t lnk_stat; 146 uint32_t spci_cmd; 147 } reg_ofs; 148 uint8_t conn_type; 149 uint8_t dev_type; 150 uint8_t bits_per_vector; 151 uint8_t link_status; 152 uint8_t link_width; 153 uint8_t link_speed; 154}; 155 156#ifdef __i386__ 157static __inline uint64_t 158bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 159 bus_size_t offset) 160{ 161 162 return (bus_space_read_4(tag, handle, offset) | 163 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 164} 165 166static __inline void 167bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 168 bus_size_t offset, uint64_t val) 169{ 170 171 bus_space_write_4(tag, handle, offset, val); 172 bus_space_write_4(tag, handle, offset + 4, val >> 32); 173} 174#endif 175 176#define ntb_bar_read(SIZE, bar, offset) \ 177 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 178 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 179#define ntb_bar_write(SIZE, bar, offset, val) \ 180 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 181 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 182#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 183#define ntb_reg_write(SIZE, offset, val) \ 184 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 185#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 186#define ntb_mw_write(SIZE, offset, val) \ 187 ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 188 189typedef int (*bar_map_strategy)(struct ntb_softc *ntb, 190 struct ntb_pci_bar_info *bar); 191 192static int ntb_probe(device_t device); 193static int ntb_attach(device_t device); 194static int ntb_detach(device_t device); 195static int ntb_map_pci_bars(struct ntb_softc *ntb); 196static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 197 struct ntb_pci_bar_info *bar); 198static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 199static int map_memory_window_bar(struct ntb_softc *ntb, 200 struct ntb_pci_bar_info *bar); 201static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 202static int ntb_setup_interrupts(struct ntb_softc *ntb); 203static void ntb_teardown_interrupts(struct ntb_softc *ntb); 204static void handle_soc_irq(void *arg); 205static void handle_xeon_irq(void *arg); 206static void handle_xeon_event_irq(void *arg); 207static void ntb_handle_legacy_interrupt(void *arg); 208static void ntb_irq_work(void *arg); 209static void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 210static void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 211static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 212static void ntb_free_callbacks(struct ntb_softc *ntb); 213static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 214static int ntb_setup_xeon(struct ntb_softc *ntb); 215static int ntb_setup_soc(struct ntb_softc *ntb); 216static void ntb_teardown_xeon(struct ntb_softc *ntb); 217static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 218static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 219static void ntb_handle_heartbeat(void *arg); 220static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 221static void ntb_hw_link_down(struct ntb_softc *ntb); 222static void ntb_hw_link_up(struct ntb_softc *ntb); 223static void recover_soc_link(void *arg); 224static int ntb_check_link_status(struct ntb_softc *ntb); 225static void save_bar_parameters(struct ntb_pci_bar_info *bar); 226 227static struct ntb_hw_info pci_ids[] = { 228 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 229 230 /* XXX: PS/SS IDs left out until they are supported. */ 231 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 232 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 233 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 234 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 235 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 236 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 237 | NTB_BAR_SIZE_4K }, 238 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 239 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 240 }, 241 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 242 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 243 }, 244 245 { 0x00000000, NULL, NTB_SOC, 0 } 246}; 247 248/* 249 * OS <-> Driver interface structures 250 */ 251MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 252 253static device_method_t ntb_pci_methods[] = { 254 /* Device interface */ 255 DEVMETHOD(device_probe, ntb_probe), 256 DEVMETHOD(device_attach, ntb_attach), 257 DEVMETHOD(device_detach, ntb_detach), 258 DEVMETHOD_END 259}; 260 261static driver_t ntb_pci_driver = { 262 "ntb_hw", 263 ntb_pci_methods, 264 sizeof(struct ntb_softc), 265}; 266 267static devclass_t ntb_devclass; 268DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 269MODULE_VERSION(ntb_hw, 1); 270 271SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 272 273/* 274 * OS <-> Driver linkage functions 275 */ 276static int 277ntb_probe(device_t device) 278{ 279 struct ntb_hw_info *p; 280 281 p = ntb_get_device_info(pci_get_devid(device)); 282 if (p == NULL) 283 return (ENXIO); 284 285 device_set_desc(device, p->desc); 286 return (0); 287} 288 289static int 290ntb_attach(device_t device) 291{ 292 struct ntb_softc *ntb; 293 struct ntb_hw_info *p; 294 int error; 295 296 ntb = DEVICE2SOFTC(device); 297 p = ntb_get_device_info(pci_get_devid(device)); 298 299 ntb->device = device; 300 ntb->type = p->type; 301 ntb->features = p->features; 302 303 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 304 callout_init(&ntb->heartbeat_timer, 1); 305 callout_init(&ntb->lr_timer, 1); 306 307 error = ntb_map_pci_bars(ntb); 308 if (error) 309 goto out; 310 if (ntb->type == NTB_SOC) 311 error = ntb_setup_soc(ntb); 312 else 313 error = ntb_setup_xeon(ntb); 314 if (error) 315 goto out; 316 error = ntb_setup_interrupts(ntb); 317 if (error) 318 goto out; 319 320 pci_enable_busmaster(ntb->device); 321 322out: 323 if (error != 0) 324 ntb_detach(device); 325 return (error); 326} 327 328static int 329ntb_detach(device_t device) 330{ 331 struct ntb_softc *ntb; 332 333 ntb = DEVICE2SOFTC(device); 334 callout_drain(&ntb->heartbeat_timer); 335 callout_drain(&ntb->lr_timer); 336 if (ntb->type == NTB_XEON) 337 ntb_teardown_xeon(ntb); 338 ntb_teardown_interrupts(ntb); 339 ntb_unmap_pci_bar(ntb); 340 341 return (0); 342} 343 344static int 345ntb_map_pci_bars(struct ntb_softc *ntb) 346{ 347 int rc; 348 349 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 350 rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 351 if (rc != 0) 352 return (rc); 353 354 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 355 rc = map_pci_bar(ntb, map_memory_window_bar, 356 &ntb->bar_info[NTB_B2B_BAR_1]); 357 if (rc != 0) 358 return (rc); 359 360 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 361 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 362 rc = map_pci_bar(ntb, map_mmr_bar, 363 &ntb->bar_info[NTB_B2B_BAR_2]); 364 else 365 rc = map_pci_bar(ntb, map_memory_window_bar, 366 &ntb->bar_info[NTB_B2B_BAR_2]); 367 return (rc); 368} 369 370static int 371map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 372 struct ntb_pci_bar_info *bar) 373{ 374 int rc; 375 376 rc = strategy(ntb, bar); 377 if (rc != 0) 378 device_printf(ntb->device, 379 "unable to allocate pci resource\n"); 380 else 381 device_printf(ntb->device, 382 "Bar size = %lx, v %p, p %p\n", 383 bar->size, bar->vbase, (void *)(bar->pbase)); 384 return (rc); 385} 386 387static int 388map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 389{ 390 391 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 392 &bar->pci_resource_id, RF_ACTIVE); 393 if (bar->pci_resource == NULL) 394 return (ENXIO); 395 396 save_bar_parameters(bar); 397 return (0); 398} 399 400static int 401map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 402{ 403 int rc; 404 uint8_t bar_size_bits = 0; 405 406 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 407 &bar->pci_resource_id, RF_ACTIVE); 408 409 if (bar->pci_resource == NULL) 410 return (ENXIO); 411 412 save_bar_parameters(bar); 413 /* 414 * Ivytown NTB BAR sizes are misreported by the hardware due to a 415 * hardware issue. To work around this, query the size it should be 416 * configured to by the device and modify the resource to correspond to 417 * this new size. The BIOS on systems with this problem is required to 418 * provide enough address space to allow the driver to make this change 419 * safely. 420 * 421 * Ideally I could have just specified the size when I allocated the 422 * resource like: 423 * bus_alloc_resource(ntb->device, 424 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 425 * 1ul << bar_size_bits, RF_ACTIVE); 426 * but the PCI driver does not honor the size in this call, so we have 427 * to modify it after the fact. 428 */ 429 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 430 if (bar->pci_resource_id == PCIR_BAR(2)) 431 bar_size_bits = pci_read_config(ntb->device, 432 XEON_PBAR23SZ_OFFSET, 1); 433 else 434 bar_size_bits = pci_read_config(ntb->device, 435 XEON_PBAR45SZ_OFFSET, 1); 436 437 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 438 bar->pci_resource, bar->pbase, 439 bar->pbase + (1ul << bar_size_bits) - 1); 440 if (rc != 0) { 441 device_printf(ntb->device, 442 "unable to resize bar\n"); 443 return (rc); 444 } 445 446 save_bar_parameters(bar); 447 } 448 449 /* Mark bar region as write combining to improve performance. */ 450 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 451 VM_MEMATTR_WRITE_COMBINING); 452 if (rc != 0) { 453 device_printf(ntb->device, 454 "unable to mark bar as WRITE_COMBINING\n"); 455 return (rc); 456 } 457 return (0); 458} 459 460static void 461ntb_unmap_pci_bar(struct ntb_softc *ntb) 462{ 463 struct ntb_pci_bar_info *current_bar; 464 int i; 465 466 for (i = 0; i< NTB_MAX_BARS; i++) { 467 current_bar = &ntb->bar_info[i]; 468 if (current_bar->pci_resource != NULL) 469 bus_release_resource(ntb->device, SYS_RES_MEMORY, 470 current_bar->pci_resource_id, 471 current_bar->pci_resource); 472 } 473} 474 475static int 476ntb_setup_interrupts(struct ntb_softc *ntb) 477{ 478 void (*interrupt_handler)(void *); 479 void *int_arg; 480 bool use_msix = false; 481 uint32_t num_vectors; 482 int i; 483 484 ntb->allocated_interrupts = 0; 485 /* 486 * On SOC, disable all interrupts. On XEON, disable all but Link 487 * Interrupt. The rest will be unmasked as callbacks are registered. 488 */ 489 if (ntb->type == NTB_SOC) 490 ntb_reg_write(8, ntb->reg_ofs.ldb_mask, ~0); 491 else 492 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, 493 (uint16_t) ~(1 << XEON_LINK_DB)); 494 495 num_vectors = MIN(pci_msix_count(ntb->device), 496 ntb->limits.max_db_bits); 497 if (num_vectors >= 1) { 498 pci_alloc_msix(ntb->device, &num_vectors); 499 if (num_vectors >= 4) 500 use_msix = true; 501 } 502 503 ntb_create_callbacks(ntb, num_vectors); 504 if (use_msix == true) { 505 for (i = 0; i < num_vectors; i++) { 506 ntb->int_info[i].rid = i + 1; 507 ntb->int_info[i].res = bus_alloc_resource_any( 508 ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 509 RF_ACTIVE); 510 if (ntb->int_info[i].res == NULL) { 511 device_printf(ntb->device, 512 "bus_alloc_resource failed\n"); 513 return (ENOMEM); 514 } 515 ntb->int_info[i].tag = NULL; 516 ntb->allocated_interrupts++; 517 if (ntb->type == NTB_SOC) { 518 interrupt_handler = handle_soc_irq; 519 int_arg = &ntb->db_cb[i]; 520 } else { 521 if (i == num_vectors - 1) { 522 interrupt_handler = 523 handle_xeon_event_irq; 524 int_arg = ntb; 525 } else { 526 interrupt_handler = 527 handle_xeon_irq; 528 int_arg = &ntb->db_cb[i]; 529 } 530 } 531 if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 532 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 533 interrupt_handler, int_arg, 534 &ntb->int_info[i].tag) != 0) { 535 device_printf(ntb->device, 536 "bus_setup_intr failed\n"); 537 return (ENXIO); 538 } 539 } 540 } else { 541 ntb->int_info[0].rid = 0; 542 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, 543 SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 544 interrupt_handler = ntb_handle_legacy_interrupt; 545 if (ntb->int_info[0].res == NULL) { 546 device_printf(ntb->device, 547 "bus_alloc_resource failed\n"); 548 return (ENOMEM); 549 } 550 ntb->int_info[0].tag = NULL; 551 ntb->allocated_interrupts = 1; 552 553 if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 554 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 555 interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 556 557 device_printf(ntb->device, "bus_setup_intr failed\n"); 558 return (ENXIO); 559 } 560 } 561 562 return (0); 563} 564 565static void 566ntb_teardown_interrupts(struct ntb_softc *ntb) 567{ 568 struct ntb_int_info *current_int; 569 int i; 570 571 for (i = 0; i < ntb->allocated_interrupts; i++) { 572 current_int = &ntb->int_info[i]; 573 if (current_int->tag != NULL) 574 bus_teardown_intr(ntb->device, current_int->res, 575 current_int->tag); 576 577 if (current_int->res != NULL) 578 bus_release_resource(ntb->device, SYS_RES_IRQ, 579 rman_get_rid(current_int->res), current_int->res); 580 } 581 582 ntb_free_callbacks(ntb); 583 pci_release_msi(ntb->device); 584} 585 586static void 587mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 588{ 589 unsigned long mask; 590 591 mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 592 mask |= 1 << (idx * ntb->bits_per_vector); 593 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 594} 595 596static void 597unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 598{ 599 unsigned long mask; 600 601 mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 602 mask &= ~(1 << (idx * ntb->bits_per_vector)); 603 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 604} 605 606static void 607handle_soc_irq(void *arg) 608{ 609 struct ntb_db_cb *db_cb = arg; 610 struct ntb_softc *ntb = db_cb->ntb; 611 612 ntb_reg_write(8, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num); 613 614 if (db_cb->callback != NULL) { 615 mask_ldb_interrupt(ntb, db_cb->db_num); 616 callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 617 } 618} 619 620static void 621handle_xeon_irq(void *arg) 622{ 623 struct ntb_db_cb *db_cb = arg; 624 struct ntb_softc *ntb = db_cb->ntb; 625 626 /* 627 * On Xeon, there are 16 bits in the interrupt register 628 * but only 4 vectors. So, 5 bits are assigned to the first 3 629 * vectors, with the 4th having a single bit for link 630 * interrupts. 631 */ 632 ntb_reg_write(2, ntb->reg_ofs.ldb, 633 ((1 << ntb->bits_per_vector) - 1) << 634 (db_cb->db_num * ntb->bits_per_vector)); 635 636 if (db_cb->callback != NULL) { 637 mask_ldb_interrupt(ntb, db_cb->db_num); 638 callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 639 } 640} 641 642/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 643static void 644handle_xeon_event_irq(void *arg) 645{ 646 struct ntb_softc *ntb = arg; 647 int rc; 648 649 rc = ntb_check_link_status(ntb); 650 if (rc != 0) 651 device_printf(ntb->device, "Error determining link status\n"); 652 653 /* bit 15 is always the link bit */ 654 ntb_reg_write(2, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB); 655} 656 657static void 658ntb_handle_legacy_interrupt(void *arg) 659{ 660 struct ntb_softc *ntb = arg; 661 unsigned int i = 0; 662 uint64_t ldb64; 663 uint16_t ldb16; 664 665 if (ntb->type == NTB_SOC) { 666 ldb64 = ntb_reg_read(8, ntb->reg_ofs.ldb); 667 668 while (ldb64) { 669 i = ffs(ldb64); 670 ldb64 &= ldb64 - 1; 671 handle_soc_irq(&ntb->db_cb[i]); 672 } 673 } else { 674 ldb16 = ntb_reg_read(2, ntb->reg_ofs.ldb); 675 676 if ((ldb16 & XEON_DB_HW_LINK) != 0) { 677 handle_xeon_event_irq(ntb); 678 ldb16 &= ~XEON_DB_HW_LINK; 679 } 680 681 while (ldb16 != 0) { 682 i = ffs(ldb16); 683 ldb16 &= ldb16 - 1; 684 handle_xeon_irq(&ntb->db_cb[i]); 685 } 686 } 687 688} 689 690static int 691ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 692{ 693 int i; 694 695 ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 696 M_ZERO | M_WAITOK); 697 for (i = 0; i < num_vectors; i++) { 698 ntb->db_cb[i].db_num = i; 699 ntb->db_cb[i].ntb = ntb; 700 } 701 702 return (0); 703} 704 705static void 706ntb_free_callbacks(struct ntb_softc *ntb) 707{ 708 int i; 709 710 for (i = 0; i < ntb->limits.max_db_bits; i++) 711 ntb_unregister_db_callback(ntb, i); 712 713 free(ntb->db_cb, M_NTB); 714} 715 716static struct ntb_hw_info * 717ntb_get_device_info(uint32_t device_id) 718{ 719 struct ntb_hw_info *ep = pci_ids; 720 721 while (ep->device_id) { 722 if (ep->device_id == device_id) 723 return (ep); 724 ++ep; 725 } 726 return (NULL); 727} 728 729static void 730ntb_teardown_xeon(struct ntb_softc *ntb) 731{ 732 733 ntb_hw_link_down(ntb); 734} 735 736static int 737ntb_setup_xeon(struct ntb_softc *ntb) 738{ 739 uint8_t val, connection_type; 740 741 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 742 743 connection_type = val & XEON_PPD_CONN_TYPE; 744 745 if ((val & XEON_PPD_DEV_TYPE) != 0) 746 ntb->dev_type = NTB_DEV_USD; 747 else 748 ntb->dev_type = NTB_DEV_DSD; 749 750 ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; 751 ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; 752 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 753 ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET; 754 ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; 755 756 switch (connection_type) { 757 case NTB_CONN_B2B: 758 ntb->conn_type = NTB_CONN_B2B; 759 760 /* 761 * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored 762 * with the NTB_REGS_THRU_MW errata mode enabled. (See 763 * ntb_ring_doorbell() and ntb_read/write_remote_spad().) 764 */ 765 ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET; 766 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 767 768 ntb->limits.max_spads = XEON_MAX_SPADS; 769 break; 770 771 case NTB_CONN_RP: 772 /* 773 * Every Xeon today needs NTB_REGS_THRU_MW, so punt on RP for 774 * now. 775 */ 776 KASSERT(HAS_FEATURE(NTB_REGS_THRU_MW), 777 ("Xeon without MW errata unimplemented")); 778 device_printf(ntb->device, 779 "NTB-RP disabled to due hardware errata.\n"); 780 return (ENXIO); 781 782 case NTB_CONN_TRANSPARENT: 783 default: 784 device_printf(ntb->device, "Connection type %d not supported\n", 785 connection_type); 786 return (ENXIO); 787 } 788 789 /* 790 * There is a Xeon hardware errata related to writes to SDOORBELL or 791 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 792 * which may hang the system. To workaround this use the second memory 793 * window to access the interrupt and scratch pad registers on the 794 * remote system. 795 * 796 * There is another HW errata on the limit registers -- they can only 797 * be written when the base register is (?)4GB aligned and < 32-bit. 798 * This should already be the case based on the driver defaults, but 799 * write the limit registers first just in case. 800 */ 801 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 802 /* 803 * Set the Limit register to 4k, the minimum size, to prevent 804 * an illegal access. 805 */ 806 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 807 ntb_get_mw_size(ntb, 1) + 0x1000); 808 else 809 /* 810 * Disable the limit register, just in case it is set to 811 * something silly. 812 */ 813 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 814 815 816 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 817 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 818 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 819 820 ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 821 ntb->limits.msix_cnt = XEON_MSIX_CNT; 822 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 823 824 /* 825 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 826 * mirrored to the remote system. Shrink the number of bits by one, 827 * since bit 14 is the last bit. 828 * 829 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 830 * anyway. Nor for non-B2B connection types. 831 */ 832 if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) && 833 !HAS_FEATURE(NTB_REGS_THRU_MW) && 834 connection_type == NTB_CONN_B2B) 835 ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1; 836 837 configure_xeon_secondary_side_bars(ntb); 838 839 /* Enable Bus Master and Memory Space on the secondary side */ 840 if (ntb->conn_type == NTB_CONN_B2B) 841 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 842 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 843 844 /* Enable link training */ 845 ntb_hw_link_up(ntb); 846 847 return (0); 848} 849 850static int 851ntb_setup_soc(struct ntb_softc *ntb) 852{ 853 uint32_t val, connection_type; 854 855 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 856 857 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 858 switch (connection_type) { 859 case NTB_CONN_B2B: 860 ntb->conn_type = NTB_CONN_B2B; 861 break; 862 default: 863 device_printf(ntb->device, 864 "Unsupported NTB configuration (%d)\n", connection_type); 865 return (ENXIO); 866 } 867 868 if ((val & SOC_PPD_DEV_TYPE) != 0) 869 ntb->dev_type = NTB_DEV_DSD; 870 else 871 ntb->dev_type = NTB_DEV_USD; 872 873 /* Initiate PCI-E link training */ 874 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 875 4); 876 877 ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; 878 ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; 879 ntb->reg_ofs.rdb = SOC_B2B_DOORBELL_OFFSET; 880 ntb->reg_ofs.bar2_xlat = SOC_SBAR2XLAT_OFFSET; 881 ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; 882 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 883 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 884 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 885 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 886 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 887 888 ntb->limits.max_spads = SOC_MAX_SPADS; 889 ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 890 ntb->limits.msix_cnt = SOC_MSIX_CNT; 891 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 892 893 /* 894 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 895 * resolved. Mask transaction layer internal parity errors. 896 */ 897 pci_write_config(ntb->device, 0xFC, 0x4, 4); 898 899 configure_soc_secondary_side_bars(ntb); 900 901 /* Enable Bus Master and Memory Space on the secondary side */ 902 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 903 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 904 905 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 906 907 return (0); 908} 909 910static void 911configure_soc_secondary_side_bars(struct ntb_softc *ntb) 912{ 913 914 if (ntb->dev_type == NTB_DEV_USD) { 915 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 916 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 917 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 918 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 919 } else { 920 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 921 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 922 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 923 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 924 } 925} 926 927static void 928configure_xeon_secondary_side_bars(struct ntb_softc *ntb) 929{ 930 931 if (ntb->dev_type == NTB_DEV_USD) { 932 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 933 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 934 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 935 MBAR01_DSD_ADDR); 936 else { 937 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 938 PBAR4XLAT_USD_ADDR); 939 /* 940 * B2B_XLAT_OFFSET is a 64-bit register but can only be 941 * written 32 bits at a time. 942 */ 943 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 944 MBAR01_DSD_ADDR & 0xffffffff); 945 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 946 MBAR01_DSD_ADDR >> 32); 947 } 948 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 949 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 950 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 951 } else { 952 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 953 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 954 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 955 MBAR01_USD_ADDR); 956 else { 957 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 958 PBAR4XLAT_DSD_ADDR); 959 /* 960 * B2B_XLAT_OFFSET is a 64-bit register but can only be 961 * written 32 bits at a time. 962 */ 963 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 964 MBAR01_USD_ADDR & 0xffffffff); 965 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 966 MBAR01_USD_ADDR >> 32); 967 } 968 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 969 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 970 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 971 } 972} 973 974/* SOC does not have link status interrupt, poll on that platform */ 975static void 976ntb_handle_heartbeat(void *arg) 977{ 978 struct ntb_softc *ntb = arg; 979 uint32_t status32; 980 int rc; 981 982 rc = ntb_check_link_status(ntb); 983 if (rc != 0) 984 device_printf(ntb->device, 985 "Error determining link status\n"); 986 987 /* Check to see if a link error is the cause of the link down */ 988 if (ntb->link_status == NTB_LINK_DOWN) { 989 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 990 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 991 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 992 ntb); 993 return; 994 } 995 } 996 997 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 998 ntb_handle_heartbeat, ntb); 999} 1000 1001static void 1002soc_perform_link_restart(struct ntb_softc *ntb) 1003{ 1004 uint32_t status; 1005 1006 /* Driver resets the NTB ModPhy lanes - magic! */ 1007 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1008 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1009 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1010 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1011 1012 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1013 pause("ModPhy", hz / 10); 1014 1015 /* Clear AER Errors, write to clear */ 1016 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1017 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1018 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1019 1020 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1021 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1022 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1023 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1024 1025 /* Clear DeSkew Buffer error, write to clear */ 1026 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1027 status |= SOC_DESKEWSTS_DBERR; 1028 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1029 1030 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1031 status &= SOC_IBIST_ERR_OFLOW; 1032 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1033 1034 /* Releases the NTB state machine to allow the link to retrain */ 1035 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1036 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1037 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1038} 1039 1040static void 1041ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 1042{ 1043 enum ntb_hw_event event; 1044 uint16_t status; 1045 1046 if (ntb->link_status == link_state) 1047 return; 1048 1049 if (link_state == NTB_LINK_UP) { 1050 device_printf(ntb->device, "Link Up\n"); 1051 ntb->link_status = NTB_LINK_UP; 1052 event = NTB_EVENT_HW_LINK_UP; 1053 1054 if (ntb->type == NTB_SOC || 1055 ntb->conn_type == NTB_CONN_TRANSPARENT) 1056 status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1057 else 1058 status = pci_read_config(ntb->device, 1059 XEON_LINK_STATUS_OFFSET, 2); 1060 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 1061 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 1062 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 1063 ntb->link_width, ntb->link_speed); 1064 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1065 ntb_handle_heartbeat, ntb); 1066 } else { 1067 device_printf(ntb->device, "Link Down\n"); 1068 ntb->link_status = NTB_LINK_DOWN; 1069 event = NTB_EVENT_HW_LINK_DOWN; 1070 /* Do not modify link width/speed, we need it in link recovery */ 1071 } 1072 1073 /* notify the upper layer if we have an event change */ 1074 if (ntb->event_cb != NULL) 1075 ntb->event_cb(ntb->ntb_transport, event); 1076} 1077 1078static void 1079ntb_hw_link_up(struct ntb_softc *ntb) 1080{ 1081 uint32_t cntl; 1082 1083 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1084 ntb_handle_link_event(ntb, NTB_LINK_UP); 1085 return; 1086 } 1087 1088 cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1089 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1090 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1091 cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP; 1092 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1093} 1094 1095static void 1096ntb_hw_link_down(struct ntb_softc *ntb) 1097{ 1098 uint32_t cntl; 1099 1100 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1101 ntb_handle_link_event(ntb, NTB_LINK_DOWN); 1102 return; 1103 } 1104 1105 cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1106 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1107 cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP); 1108 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1109 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1110} 1111 1112static void 1113recover_soc_link(void *arg) 1114{ 1115 struct ntb_softc *ntb = arg; 1116 uint8_t speed, width; 1117 uint32_t status32; 1118 uint16_t status16; 1119 1120 soc_perform_link_restart(ntb); 1121 1122 /* 1123 * There is a potential race between the 2 NTB devices recovering at 1124 * the same time. If the times are the same, the link will not recover 1125 * and the driver will be stuck in this loop forever. Add a random 1126 * interval to the recovery time to prevent this race. 1127 */ 1128 status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1129 pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1130 1131 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1132 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1133 goto retry; 1134 1135 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1136 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1137 goto retry; 1138 1139 status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1140 if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1141 goto out; 1142 1143 status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1144 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 1145 speed = (status16 & NTB_LINK_SPEED_MASK); 1146 if (ntb->link_width != width || ntb->link_speed != speed) 1147 goto retry; 1148 1149out: 1150 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1151 ntb_handle_heartbeat, ntb); 1152 return; 1153 1154retry: 1155 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1156 ntb); 1157} 1158 1159static int 1160ntb_check_link_status(struct ntb_softc *ntb) 1161{ 1162 int link_state; 1163 uint32_t ntb_cntl; 1164 uint16_t status; 1165 1166 if (ntb->type == NTB_SOC) { 1167 ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1168 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1169 link_state = NTB_LINK_DOWN; 1170 else 1171 link_state = NTB_LINK_UP; 1172 } else { 1173 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 1174 2); 1175 1176 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1177 link_state = NTB_LINK_UP; 1178 else 1179 link_state = NTB_LINK_DOWN; 1180 } 1181 1182 ntb_handle_link_event(ntb, link_state); 1183 1184 return (0); 1185} 1186 1187/** 1188 * ntb_register_event_callback() - register event callback 1189 * @ntb: pointer to ntb_softc instance 1190 * @func: callback function to register 1191 * 1192 * This function registers a callback for any HW driver events such as link 1193 * up/down, power management notices and etc. 1194 * 1195 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1196 */ 1197int 1198ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1199{ 1200 1201 if (ntb->event_cb != NULL) 1202 return (EINVAL); 1203 1204 ntb->event_cb = func; 1205 1206 return (0); 1207} 1208 1209/** 1210 * ntb_unregister_event_callback() - unregisters the event callback 1211 * @ntb: pointer to ntb_softc instance 1212 * 1213 * This function unregisters the existing callback from transport 1214 */ 1215void 1216ntb_unregister_event_callback(struct ntb_softc *ntb) 1217{ 1218 1219 ntb->event_cb = NULL; 1220} 1221 1222static void 1223ntb_irq_work(void *arg) 1224{ 1225 struct ntb_db_cb *db_cb = arg; 1226 struct ntb_softc *ntb; 1227 int rc; 1228 1229 rc = db_cb->callback(db_cb->data, db_cb->db_num); 1230 /* Poll if forward progress was made. */ 1231 if (rc != 0) { 1232 callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 1233 return; 1234 } 1235 1236 /* Unmask interrupt if no progress was made. */ 1237 ntb = db_cb->ntb; 1238 unmask_ldb_interrupt(ntb, db_cb->db_num); 1239} 1240 1241/** 1242 * ntb_register_db_callback() - register a callback for doorbell interrupt 1243 * @ntb: pointer to ntb_softc instance 1244 * @idx: doorbell index to register callback, zero based 1245 * @data: pointer to be returned to caller with every callback 1246 * @func: callback function to register 1247 * 1248 * This function registers a callback function for the doorbell interrupt 1249 * on the primary side. The function will unmask the doorbell as well to 1250 * allow interrupt. 1251 * 1252 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1253 */ 1254int 1255ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1256 ntb_db_callback func) 1257{ 1258 1259 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1260 device_printf(ntb->device, "Invalid Index.\n"); 1261 return (EINVAL); 1262 } 1263 1264 ntb->db_cb[idx].callback = func; 1265 ntb->db_cb[idx].data = data; 1266 callout_init(&ntb->db_cb[idx].irq_work, 1); 1267 1268 unmask_ldb_interrupt(ntb, idx); 1269 1270 return (0); 1271} 1272 1273/** 1274 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1275 * @ntb: pointer to ntb_softc instance 1276 * @idx: doorbell index to register callback, zero based 1277 * 1278 * This function unregisters a callback function for the doorbell interrupt 1279 * on the primary side. The function will also mask the said doorbell. 1280 */ 1281void 1282ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1283{ 1284 1285 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1286 return; 1287 1288 mask_ldb_interrupt(ntb, idx); 1289 1290 callout_drain(&ntb->db_cb[idx].irq_work); 1291 ntb->db_cb[idx].callback = NULL; 1292} 1293 1294/** 1295 * ntb_find_transport() - find the transport pointer 1296 * @transport: pointer to pci device 1297 * 1298 * Given the pci device pointer, return the transport pointer passed in when 1299 * the transport attached when it was inited. 1300 * 1301 * RETURNS: pointer to transport. 1302 */ 1303void * 1304ntb_find_transport(struct ntb_softc *ntb) 1305{ 1306 1307 return (ntb->ntb_transport); 1308} 1309 1310/** 1311 * ntb_register_transport() - Register NTB transport with NTB HW driver 1312 * @transport: transport identifier 1313 * 1314 * This function allows a transport to reserve the hardware driver for 1315 * NTB usage. 1316 * 1317 * RETURNS: pointer to ntb_softc, NULL on error. 1318 */ 1319struct ntb_softc * 1320ntb_register_transport(struct ntb_softc *ntb, void *transport) 1321{ 1322 1323 /* 1324 * TODO: when we have more than one transport, we will need to rewrite 1325 * this to prevent race conditions 1326 */ 1327 if (ntb->ntb_transport != NULL) 1328 return (NULL); 1329 1330 ntb->ntb_transport = transport; 1331 return (ntb); 1332} 1333 1334/** 1335 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1336 * @ntb - ntb_softc of the transport to be freed 1337 * 1338 * This function unregisters the transport from the HW driver and performs any 1339 * necessary cleanups. 1340 */ 1341void 1342ntb_unregister_transport(struct ntb_softc *ntb) 1343{ 1344 int i; 1345 1346 if (ntb->ntb_transport == NULL) 1347 return; 1348 1349 for (i = 0; i < ntb->allocated_interrupts; i++) 1350 ntb_unregister_db_callback(ntb, i); 1351 1352 ntb_unregister_event_callback(ntb); 1353 ntb->ntb_transport = NULL; 1354} 1355 1356/** 1357 * ntb_get_max_spads() - get the total scratch regs usable 1358 * @ntb: pointer to ntb_softc instance 1359 * 1360 * This function returns the max 32bit scratchpad registers usable by the 1361 * upper layer. 1362 * 1363 * RETURNS: total number of scratch pad registers available 1364 */ 1365uint8_t 1366ntb_get_max_spads(struct ntb_softc *ntb) 1367{ 1368 1369 return (ntb->limits.max_spads); 1370} 1371 1372/** 1373 * ntb_write_local_spad() - write to the secondary scratchpad register 1374 * @ntb: pointer to ntb_softc instance 1375 * @idx: index to the scratchpad register, 0 based 1376 * @val: the data value to put into the register 1377 * 1378 * This function allows writing of a 32bit value to the indexed scratchpad 1379 * register. The register resides on the secondary (external) side. 1380 * 1381 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1382 */ 1383int 1384ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1385{ 1386 1387 if (idx >= ntb->limits.max_spads) 1388 return (EINVAL); 1389 1390 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1391 1392 return (0); 1393} 1394 1395/** 1396 * ntb_read_local_spad() - read from the primary scratchpad register 1397 * @ntb: pointer to ntb_softc instance 1398 * @idx: index to scratchpad register, 0 based 1399 * @val: pointer to 32bit integer for storing the register value 1400 * 1401 * This function allows reading of the 32bit scratchpad register on 1402 * the primary (internal) side. 1403 * 1404 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1405 */ 1406int 1407ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1408{ 1409 1410 if (idx >= ntb->limits.max_spads) 1411 return (EINVAL); 1412 1413 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1414 1415 return (0); 1416} 1417 1418/** 1419 * ntb_write_remote_spad() - write to the secondary scratchpad register 1420 * @ntb: pointer to ntb_softc instance 1421 * @idx: index to the scratchpad register, 0 based 1422 * @val: the data value to put into the register 1423 * 1424 * This function allows writing of a 32bit value to the indexed scratchpad 1425 * register. The register resides on the secondary (external) side. 1426 * 1427 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1428 */ 1429int 1430ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1431{ 1432 1433 if (idx >= ntb->limits.max_spads) 1434 return (EINVAL); 1435 1436 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1437 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1438 else 1439 ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1440 1441 return (0); 1442} 1443 1444/** 1445 * ntb_read_remote_spad() - read from the primary scratchpad register 1446 * @ntb: pointer to ntb_softc instance 1447 * @idx: index to scratchpad register, 0 based 1448 * @val: pointer to 32bit integer for storing the register value 1449 * 1450 * This function allows reading of the 32bit scratchpad register on 1451 * the primary (internal) side. 1452 * 1453 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1454 */ 1455int 1456ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1457{ 1458 1459 if (idx >= ntb->limits.max_spads) 1460 return (EINVAL); 1461 1462 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1463 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1464 else 1465 *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1466 1467 return (0); 1468} 1469 1470/** 1471 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1472 * @ntb: pointer to ntb_softc instance 1473 * @mw: memory window number 1474 * 1475 * This function provides the base virtual address of the memory window 1476 * specified. 1477 * 1478 * RETURNS: pointer to virtual address, or NULL on error. 1479 */ 1480void * 1481ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1482{ 1483 1484 if (mw >= NTB_NUM_MW) 1485 return (NULL); 1486 1487 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1488} 1489 1490vm_paddr_t 1491ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1492{ 1493 1494 if (mw >= NTB_NUM_MW) 1495 return (0); 1496 1497 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1498} 1499 1500/** 1501 * ntb_get_mw_size() - return size of NTB memory window 1502 * @ntb: pointer to ntb_softc instance 1503 * @mw: memory window number 1504 * 1505 * This function provides the physical size of the memory window specified 1506 * 1507 * RETURNS: the size of the memory window or zero on error 1508 */ 1509u_long 1510ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1511{ 1512 1513 if (mw >= NTB_NUM_MW) 1514 return (0); 1515 1516 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1517} 1518 1519/** 1520 * ntb_set_mw_addr - set the memory window address 1521 * @ntb: pointer to ntb_softc instance 1522 * @mw: memory window number 1523 * @addr: base address for data 1524 * 1525 * This function sets the base physical address of the memory window. This 1526 * memory address is where data from the remote system will be transfered into 1527 * or out of depending on how the transport is configured. 1528 */ 1529void 1530ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1531{ 1532 1533 if (mw >= NTB_NUM_MW) 1534 return; 1535 1536 switch (NTB_MW_TO_BAR(mw)) { 1537 case NTB_B2B_BAR_1: 1538 ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr); 1539 break; 1540 case NTB_B2B_BAR_2: 1541 ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr); 1542 break; 1543 } 1544} 1545 1546/** 1547 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side 1548 * @ntb: pointer to ntb_softc instance 1549 * @db: doorbell to ring 1550 * 1551 * This function allows triggering of a doorbell on the secondary/external 1552 * side that will initiate an interrupt on the remote host 1553 * 1554 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1555 */ 1556void 1557ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db) 1558{ 1559 1560 if (ntb->type == NTB_SOC) 1561 ntb_reg_write(8, ntb->reg_ofs.rdb, (uint64_t) 1 << db); 1562 else { 1563 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1564 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1565 ((1 << ntb->bits_per_vector) - 1) << 1566 (db * ntb->bits_per_vector)); 1567 else 1568 ntb_reg_write(2, ntb->reg_ofs.rdb, 1569 ((1 << ntb->bits_per_vector) - 1) << 1570 (db * ntb->bits_per_vector)); 1571 } 1572} 1573 1574/** 1575 * ntb_query_link_status() - return the hardware link status 1576 * @ndev: pointer to ntb_device instance 1577 * 1578 * Returns true if the hardware is connected to the remote system 1579 * 1580 * RETURNS: true or false based on the hardware link state 1581 */ 1582bool 1583ntb_query_link_status(struct ntb_softc *ntb) 1584{ 1585 1586 return (ntb->link_status == NTB_LINK_UP); 1587} 1588 1589static void 1590save_bar_parameters(struct ntb_pci_bar_info *bar) 1591{ 1592 1593 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 1594 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 1595 bar->pbase = rman_get_start(bar->pci_resource); 1596 bar->size = rman_get_size(bar->pci_resource); 1597 bar->vbase = rman_get_virtual(bar->pci_resource); 1598} 1599 1600device_t 1601ntb_get_device(struct ntb_softc *ntb) 1602{ 1603 1604 return (ntb->device); 1605} 1606 1607/* Export HW-specific errata information. */ 1608bool 1609ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 1610{ 1611 1612 return (HAS_FEATURE(feature)); 1613} 1614