ntb_hw_intel.c revision 289271
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289271 2015-10-13 23:41:06Z cem $"); 29 30#include <sys/param.h> 31#include <sys/kernel.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/malloc.h> 35#include <sys/module.h> 36#include <sys/queue.h> 37#include <sys/rman.h> 38#include <sys/sysctl.h> 39#include <vm/vm.h> 40#include <vm/pmap.h> 41#include <machine/bus.h> 42#include <machine/pmap.h> 43#include <machine/resource.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46 47#include "ntb_regs.h" 48#include "ntb_hw.h" 49 50/* 51 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52 * allows you to connect two systems using a PCI-e link. 53 * 54 * This module contains the hardware abstraction layer for the NTB. It allows 55 * you to send and recieve interrupts, map the memory windows and send and 56 * receive messages in the scratch-pad registers. 57 * 58 * NOTE: Much of the code in this module is shared with Linux. Any patches may 59 * be picked up and redistributed in Linux with a dual GPL/BSD license. 60 */ 61 62#define NTB_CONFIG_BAR 0 63#define NTB_B2B_BAR_1 1 64#define NTB_B2B_BAR_2 2 65#define NTB_MAX_BARS 3 66#define NTB_MW_TO_BAR(mw) ((mw) + 1) 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define SOC_LINK_RECOVERY_TIME 500 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75enum ntb_device_type { 76 NTB_XEON, 77 NTB_SOC 78}; 79 80/* Device features and workarounds */ 81#define HAS_FEATURE(feature) \ 82 ((ntb->features & (feature)) != 0) 83 84struct ntb_hw_info { 85 uint32_t device_id; 86 const char *desc; 87 enum ntb_device_type type; 88 uint64_t features; 89}; 90 91struct ntb_pci_bar_info { 92 bus_space_tag_t pci_bus_tag; 93 bus_space_handle_t pci_bus_handle; 94 int pci_resource_id; 95 struct resource *pci_resource; 96 vm_paddr_t pbase; 97 void *vbase; 98 u_long size; 99}; 100 101struct ntb_int_info { 102 struct resource *res; 103 int rid; 104 void *tag; 105}; 106 107struct ntb_db_cb { 108 ntb_db_callback callback; 109 unsigned int db_num; 110 void *data; 111 struct ntb_softc *ntb; 112}; 113 114struct ntb_softc { 115 device_t device; 116 enum ntb_device_type type; 117 uint64_t features; 118 119 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 120 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 121 uint32_t allocated_interrupts; 122 123 struct callout heartbeat_timer; 124 struct callout lr_timer; 125 126 void *ntb_transport; 127 ntb_event_callback event_cb; 128 struct ntb_db_cb *db_cb; 129 130 struct { 131 uint8_t max_spads; 132 uint8_t max_db_bits; 133 uint8_t msix_cnt; 134 } limits; 135 struct { 136 uint32_t ldb; 137 uint32_t ldb_mask; 138 uint32_t rdb; 139 uint32_t bar2_xlat; 140 uint32_t bar4_xlat; 141 uint32_t spad_remote; 142 uint32_t spad_local; 143 uint32_t lnk_cntl; 144 uint32_t lnk_stat; 145 uint32_t spci_cmd; 146 } reg_ofs; 147 uint8_t conn_type; 148 uint8_t dev_type; 149 uint8_t bits_per_vector; 150 uint8_t link_status; 151 uint8_t link_width; 152 uint8_t link_speed; 153}; 154 155#ifdef __i386__ 156static __inline uint64_t 157bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 158 bus_size_t offset) 159{ 160 161 return (bus_space_read_4(tag, handle, offset) | 162 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 163} 164 165static __inline void 166bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 167 bus_size_t offset, uint64_t val) 168{ 169 170 bus_space_write_4(tag, handle, offset, val); 171 bus_space_write_4(tag, handle, offset + 4, val >> 32); 172} 173#endif 174 175#define ntb_bar_read(SIZE, bar, offset) \ 176 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 177 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 178#define ntb_bar_write(SIZE, bar, offset, val) \ 179 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 180 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 181#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 182#define ntb_reg_write(SIZE, offset, val) \ 183 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 184#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 185#define ntb_mw_write(SIZE, offset, val) \ 186 ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 187 188typedef int (*bar_map_strategy)(struct ntb_softc *ntb, 189 struct ntb_pci_bar_info *bar); 190 191static int ntb_probe(device_t device); 192static int ntb_attach(device_t device); 193static int ntb_detach(device_t device); 194static int ntb_map_pci_bars(struct ntb_softc *ntb); 195static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 196 struct ntb_pci_bar_info *bar); 197static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 198static int map_memory_window_bar(struct ntb_softc *ntb, 199 struct ntb_pci_bar_info *bar); 200static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 201static int ntb_setup_interrupts(struct ntb_softc *ntb); 202static void ntb_teardown_interrupts(struct ntb_softc *ntb); 203static void handle_soc_irq(void *arg); 204static void handle_xeon_irq(void *arg); 205static void handle_xeon_event_irq(void *arg); 206static void ntb_handle_legacy_interrupt(void *arg); 207static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 208static void ntb_free_callbacks(struct ntb_softc *ntb); 209static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 210static int ntb_initialize_hw(struct ntb_softc *ntb); 211static int ntb_setup_xeon(struct ntb_softc *ntb); 212static int ntb_setup_soc(struct ntb_softc *ntb); 213static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 214static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 215static void ntb_handle_heartbeat(void *arg); 216static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 217static void recover_soc_link(void *arg); 218static int ntb_check_link_status(struct ntb_softc *ntb); 219static void save_bar_parameters(struct ntb_pci_bar_info *bar); 220 221static struct ntb_hw_info pci_ids[] = { 222 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 223 224 /* XXX: PS/SS IDs left out until they are supported. */ 225 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 226 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 227 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 228 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 229 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 230 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 231 | NTB_BAR_SIZE_4K }, 232 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 233 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 234 }, 235 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 236 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 237 }, 238 239 { 0x00000000, NULL, NTB_SOC, 0 } 240}; 241 242/* 243 * OS <-> Driver interface structures 244 */ 245MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 246 247static device_method_t ntb_pci_methods[] = { 248 /* Device interface */ 249 DEVMETHOD(device_probe, ntb_probe), 250 DEVMETHOD(device_attach, ntb_attach), 251 DEVMETHOD(device_detach, ntb_detach), 252 DEVMETHOD_END 253}; 254 255static driver_t ntb_pci_driver = { 256 "ntb_hw", 257 ntb_pci_methods, 258 sizeof(struct ntb_softc), 259}; 260 261static devclass_t ntb_devclass; 262DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 263MODULE_VERSION(ntb_hw, 1); 264 265SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 266 267/* 268 * OS <-> Driver linkage functions 269 */ 270static int 271ntb_probe(device_t device) 272{ 273 struct ntb_hw_info *p; 274 275 p = ntb_get_device_info(pci_get_devid(device)); 276 if (p == NULL) 277 return (ENXIO); 278 279 device_set_desc(device, p->desc); 280 return (0); 281} 282 283static int 284ntb_attach(device_t device) 285{ 286 struct ntb_softc *ntb; 287 struct ntb_hw_info *p; 288 int error; 289 290 ntb = DEVICE2SOFTC(device); 291 p = ntb_get_device_info(pci_get_devid(device)); 292 293 ntb->device = device; 294 ntb->type = p->type; 295 ntb->features = p->features; 296 297 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 298 callout_init(&ntb->heartbeat_timer, 1); 299 callout_init(&ntb->lr_timer, 1); 300 301 error = ntb_map_pci_bars(ntb); 302 if (error) 303 goto out; 304 error = ntb_initialize_hw(ntb); 305 if (error) 306 goto out; 307 error = ntb_setup_interrupts(ntb); 308 if (error) 309 goto out; 310 311 pci_enable_busmaster(ntb->device); 312 313out: 314 if (error != 0) 315 ntb_detach(device); 316 return (error); 317} 318 319static int 320ntb_detach(device_t device) 321{ 322 struct ntb_softc *ntb; 323 324 ntb = DEVICE2SOFTC(device); 325 callout_drain(&ntb->heartbeat_timer); 326 callout_drain(&ntb->lr_timer); 327 ntb_teardown_interrupts(ntb); 328 ntb_unmap_pci_bar(ntb); 329 330 return (0); 331} 332 333static int 334ntb_map_pci_bars(struct ntb_softc *ntb) 335{ 336 int rc; 337 338 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 339 rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 340 if (rc != 0) 341 return (rc); 342 343 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 344 rc = map_pci_bar(ntb, map_memory_window_bar, 345 &ntb->bar_info[NTB_B2B_BAR_1]); 346 if (rc != 0) 347 return (rc); 348 349 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 350 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 351 rc = map_pci_bar(ntb, map_mmr_bar, 352 &ntb->bar_info[NTB_B2B_BAR_2]); 353 else 354 rc = map_pci_bar(ntb, map_memory_window_bar, 355 &ntb->bar_info[NTB_B2B_BAR_2]); 356 return (rc); 357} 358 359static int 360map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 361 struct ntb_pci_bar_info *bar) 362{ 363 int rc; 364 365 rc = strategy(ntb, bar); 366 if (rc != 0) 367 device_printf(ntb->device, 368 "unable to allocate pci resource\n"); 369 else 370 device_printf(ntb->device, 371 "Bar size = %lx, v %p, p %p\n", 372 bar->size, bar->vbase, (void *)(bar->pbase)); 373 return (rc); 374} 375 376static int 377map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 378{ 379 380 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 381 &bar->pci_resource_id, RF_ACTIVE); 382 if (bar->pci_resource == NULL) 383 return (ENXIO); 384 385 save_bar_parameters(bar); 386 return (0); 387} 388 389static int 390map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 391{ 392 int rc; 393 uint8_t bar_size_bits = 0; 394 395 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 396 &bar->pci_resource_id, RF_ACTIVE); 397 398 if (bar->pci_resource == NULL) 399 return (ENXIO); 400 401 save_bar_parameters(bar); 402 /* 403 * Ivytown NTB BAR sizes are misreported by the hardware due to a 404 * hardware issue. To work around this, query the size it should be 405 * configured to by the device and modify the resource to correspond to 406 * this new size. The BIOS on systems with this problem is required to 407 * provide enough address space to allow the driver to make this change 408 * safely. 409 * 410 * Ideally I could have just specified the size when I allocated the 411 * resource like: 412 * bus_alloc_resource(ntb->device, 413 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 414 * 1ul << bar_size_bits, RF_ACTIVE); 415 * but the PCI driver does not honor the size in this call, so we have 416 * to modify it after the fact. 417 */ 418 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 419 if (bar->pci_resource_id == PCIR_BAR(2)) 420 bar_size_bits = pci_read_config(ntb->device, 421 XEON_PBAR23SZ_OFFSET, 1); 422 else 423 bar_size_bits = pci_read_config(ntb->device, 424 XEON_PBAR45SZ_OFFSET, 1); 425 426 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 427 bar->pci_resource, bar->pbase, 428 bar->pbase + (1ul << bar_size_bits) - 1); 429 if (rc != 0) { 430 device_printf(ntb->device, 431 "unable to resize bar\n"); 432 return (rc); 433 } 434 435 save_bar_parameters(bar); 436 } 437 438 /* Mark bar region as write combining to improve performance. */ 439 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 440 VM_MEMATTR_WRITE_COMBINING); 441 if (rc != 0) { 442 device_printf(ntb->device, 443 "unable to mark bar as WRITE_COMBINING\n"); 444 return (rc); 445 } 446 return (0); 447} 448 449static void 450ntb_unmap_pci_bar(struct ntb_softc *ntb) 451{ 452 struct ntb_pci_bar_info *current_bar; 453 int i; 454 455 for (i = 0; i< NTB_MAX_BARS; i++) { 456 current_bar = &ntb->bar_info[i]; 457 if (current_bar->pci_resource != NULL) 458 bus_release_resource(ntb->device, SYS_RES_MEMORY, 459 current_bar->pci_resource_id, 460 current_bar->pci_resource); 461 } 462} 463 464static int 465ntb_setup_interrupts(struct ntb_softc *ntb) 466{ 467 void (*interrupt_handler)(void *); 468 void *int_arg; 469 bool use_msix = false; 470 uint32_t num_vectors; 471 int i; 472 473 ntb->allocated_interrupts = 0; 474 /* 475 * On SOC, disable all interrupts. On XEON, disable all but Link 476 * Interrupt. The rest will be unmasked as callbacks are registered. 477 */ 478 if (ntb->type == NTB_SOC) 479 ntb_reg_write(8, ntb->reg_ofs.ldb_mask, ~0); 480 else 481 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, 482 (uint16_t) ~(1 << XEON_LINK_DB)); 483 484 num_vectors = MIN(pci_msix_count(ntb->device), 485 ntb->limits.max_db_bits); 486 if (num_vectors >= 1) { 487 pci_alloc_msix(ntb->device, &num_vectors); 488 if (num_vectors >= 4) 489 use_msix = true; 490 } 491 492 ntb_create_callbacks(ntb, num_vectors); 493 if (use_msix == true) { 494 for (i = 0; i < num_vectors; i++) { 495 ntb->int_info[i].rid = i + 1; 496 ntb->int_info[i].res = bus_alloc_resource_any( 497 ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 498 RF_ACTIVE); 499 if (ntb->int_info[i].res == NULL) { 500 device_printf(ntb->device, 501 "bus_alloc_resource failed\n"); 502 return (ENOMEM); 503 } 504 ntb->int_info[i].tag = NULL; 505 ntb->allocated_interrupts++; 506 if (ntb->type == NTB_SOC) { 507 interrupt_handler = handle_soc_irq; 508 int_arg = &ntb->db_cb[i]; 509 } else { 510 if (i == num_vectors - 1) { 511 interrupt_handler = 512 handle_xeon_event_irq; 513 int_arg = ntb; 514 } else { 515 interrupt_handler = 516 handle_xeon_irq; 517 int_arg = &ntb->db_cb[i]; 518 } 519 } 520 if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 521 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 522 interrupt_handler, int_arg, 523 &ntb->int_info[i].tag) != 0) { 524 device_printf(ntb->device, 525 "bus_setup_intr failed\n"); 526 return (ENXIO); 527 } 528 } 529 } else { 530 ntb->int_info[0].rid = 0; 531 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, 532 SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 533 interrupt_handler = ntb_handle_legacy_interrupt; 534 if (ntb->int_info[0].res == NULL) { 535 device_printf(ntb->device, 536 "bus_alloc_resource failed\n"); 537 return (ENOMEM); 538 } 539 ntb->int_info[0].tag = NULL; 540 ntb->allocated_interrupts = 1; 541 542 if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 543 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 544 interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 545 546 device_printf(ntb->device, "bus_setup_intr failed\n"); 547 return (ENXIO); 548 } 549 } 550 551 return (0); 552} 553 554static void 555ntb_teardown_interrupts(struct ntb_softc *ntb) 556{ 557 struct ntb_int_info *current_int; 558 int i; 559 560 for (i = 0; i < ntb->allocated_interrupts; i++) { 561 current_int = &ntb->int_info[i]; 562 if (current_int->tag != NULL) 563 bus_teardown_intr(ntb->device, current_int->res, 564 current_int->tag); 565 566 if (current_int->res != NULL) 567 bus_release_resource(ntb->device, SYS_RES_IRQ, 568 rman_get_rid(current_int->res), current_int->res); 569 } 570 571 ntb_free_callbacks(ntb); 572 pci_release_msi(ntb->device); 573} 574 575static void 576handle_soc_irq(void *arg) 577{ 578 struct ntb_db_cb *db_cb = arg; 579 struct ntb_softc *ntb = db_cb->ntb; 580 581 ntb_reg_write(8, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num); 582 583 if (db_cb->callback != NULL) 584 db_cb->callback(db_cb->data, db_cb->db_num); 585} 586 587static void 588handle_xeon_irq(void *arg) 589{ 590 struct ntb_db_cb *db_cb = arg; 591 struct ntb_softc *ntb = db_cb->ntb; 592 593 /* 594 * On Xeon, there are 16 bits in the interrupt register 595 * but only 4 vectors. So, 5 bits are assigned to the first 3 596 * vectors, with the 4th having a single bit for link 597 * interrupts. 598 */ 599 ntb_reg_write(2, ntb->reg_ofs.ldb, 600 ((1 << ntb->bits_per_vector) - 1) << 601 (db_cb->db_num * ntb->bits_per_vector)); 602 603 if (db_cb->callback != NULL) 604 db_cb->callback(db_cb->data, db_cb->db_num); 605} 606 607/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 608static void 609handle_xeon_event_irq(void *arg) 610{ 611 struct ntb_softc *ntb = arg; 612 int rc; 613 614 rc = ntb_check_link_status(ntb); 615 if (rc != 0) 616 device_printf(ntb->device, "Error determining link status\n"); 617 618 /* bit 15 is always the link bit */ 619 ntb_reg_write(2, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB); 620} 621 622static void 623ntb_handle_legacy_interrupt(void *arg) 624{ 625 struct ntb_softc *ntb = arg; 626 unsigned int i = 0; 627 uint64_t ldb64; 628 uint16_t ldb16; 629 630 if (ntb->type == NTB_SOC) { 631 ldb64 = ntb_reg_read(8, ntb->reg_ofs.ldb); 632 633 while (ldb64) { 634 i = ffs(ldb64); 635 ldb64 &= ldb64 - 1; 636 handle_soc_irq(&ntb->db_cb[i]); 637 } 638 } else { 639 ldb16 = ntb_reg_read(2, ntb->reg_ofs.ldb); 640 641 if ((ldb16 & XEON_DB_HW_LINK) != 0) { 642 handle_xeon_event_irq(ntb); 643 ldb16 &= ~XEON_DB_HW_LINK; 644 } 645 646 while (ldb16 != 0) { 647 i = ffs(ldb16); 648 ldb16 &= ldb16 - 1; 649 handle_xeon_irq(&ntb->db_cb[i]); 650 } 651 } 652 653} 654 655static int 656ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 657{ 658 int i; 659 660 ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 661 M_ZERO | M_WAITOK); 662 for (i = 0; i < num_vectors; i++) { 663 ntb->db_cb[i].db_num = i; 664 ntb->db_cb[i].ntb = ntb; 665 } 666 667 return (0); 668} 669 670static void 671ntb_free_callbacks(struct ntb_softc *ntb) 672{ 673 int i; 674 675 for (i = 0; i < ntb->limits.max_db_bits; i++) 676 ntb_unregister_db_callback(ntb, i); 677 678 free(ntb->db_cb, M_NTB); 679} 680 681static struct ntb_hw_info * 682ntb_get_device_info(uint32_t device_id) 683{ 684 struct ntb_hw_info *ep = pci_ids; 685 686 while (ep->device_id) { 687 if (ep->device_id == device_id) 688 return (ep); 689 ++ep; 690 } 691 return (NULL); 692} 693 694static int 695ntb_initialize_hw(struct ntb_softc *ntb) 696{ 697 698 if (ntb->type == NTB_SOC) 699 return (ntb_setup_soc(ntb)); 700 else 701 return (ntb_setup_xeon(ntb)); 702} 703 704static int 705ntb_setup_xeon(struct ntb_softc *ntb) 706{ 707 uint8_t val, connection_type; 708 709 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 710 711 connection_type = val & XEON_PPD_CONN_TYPE; 712 713 if ((val & XEON_PPD_DEV_TYPE) != 0) 714 ntb->dev_type = NTB_DEV_USD; 715 else 716 ntb->dev_type = NTB_DEV_DSD; 717 718 ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; 719 ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; 720 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 721 ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET; 722 ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; 723 724 switch (connection_type) { 725 case NTB_CONN_B2B: 726 ntb->conn_type = NTB_CONN_B2B; 727 728 /* 729 * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored 730 * with the NTB_REGS_THRU_MW errata mode enabled. (See 731 * ntb_ring_doorbell() and ntb_read/write_remote_spad().) 732 */ 733 ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET; 734 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 735 736 ntb->limits.max_spads = XEON_MAX_SPADS; 737 break; 738 739 case NTB_CONN_RP: 740 /* 741 * Every Xeon today needs NTB_REGS_THRU_MW, so punt on RP for 742 * now. 743 */ 744 KASSERT(HAS_FEATURE(NTB_REGS_THRU_MW), 745 ("Xeon without MW errata unimplemented")); 746 device_printf(ntb->device, 747 "NTB-RP disabled to due hardware errata.\n"); 748 return (ENXIO); 749 750 case NTB_CONN_TRANSPARENT: 751 default: 752 device_printf(ntb->device, "Connection type %d not supported\n", 753 connection_type); 754 return (ENXIO); 755 } 756 757 /* 758 * There is a Xeon hardware errata related to writes to SDOORBELL or 759 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 760 * which may hang the system. To workaround this use the second memory 761 * window to access the interrupt and scratch pad registers on the 762 * remote system. 763 */ 764 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 765 /* 766 * Set the Limit register to 4k, the minimum size, to prevent 767 * an illegal access. 768 */ 769 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 770 ntb_get_mw_size(ntb, 1) + 0x1000); 771 else 772 /* 773 * Disable the limit register, just in case it is set to 774 * something silly. 775 */ 776 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 777 778 779 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 780 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 781 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 782 783 ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 784 ntb->limits.msix_cnt = XEON_MSIX_CNT; 785 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 786 787 /* 788 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 789 * mirrored to the remote system. Shrink the number of bits by one, 790 * since bit 14 is the last bit. 791 * 792 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 793 * anyway. Nor for non-B2B connection types. 794 */ 795 if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) && 796 !HAS_FEATURE(NTB_REGS_THRU_MW) && 797 connection_type == NTB_CONN_B2B) 798 ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1; 799 800 configure_xeon_secondary_side_bars(ntb); 801 802 /* Enable Bus Master and Memory Space on the secondary side */ 803 if (ntb->conn_type == NTB_CONN_B2B) 804 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 805 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 806 807 /* Enable link training */ 808 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, 809 NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP); 810 811 return (0); 812} 813 814static int 815ntb_setup_soc(struct ntb_softc *ntb) 816{ 817 uint32_t val, connection_type; 818 819 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 820 821 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 822 switch (connection_type) { 823 case NTB_CONN_B2B: 824 ntb->conn_type = NTB_CONN_B2B; 825 break; 826 default: 827 device_printf(ntb->device, 828 "Unsupported NTB configuration (%d)\n", connection_type); 829 return (ENXIO); 830 } 831 832 if ((val & SOC_PPD_DEV_TYPE) != 0) 833 ntb->dev_type = NTB_DEV_DSD; 834 else 835 ntb->dev_type = NTB_DEV_USD; 836 837 /* Initiate PCI-E link training */ 838 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 839 4); 840 841 ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; 842 ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; 843 ntb->reg_ofs.rdb = SOC_B2B_DOORBELL_OFFSET; 844 ntb->reg_ofs.bar2_xlat = SOC_SBAR2XLAT_OFFSET; 845 ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; 846 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 847 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 848 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 849 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 850 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 851 852 ntb->limits.max_spads = SOC_MAX_SPADS; 853 ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 854 ntb->limits.msix_cnt = SOC_MSIX_CNT; 855 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 856 857 /* 858 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 859 * resolved. Mask transaction layer internal parity errors. 860 */ 861 pci_write_config(ntb->device, 0xFC, 0x4, 4); 862 863 configure_soc_secondary_side_bars(ntb); 864 865 /* Enable Bus Master and Memory Space on the secondary side */ 866 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 867 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 868 869 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 870 871 return (0); 872} 873 874static void 875configure_soc_secondary_side_bars(struct ntb_softc *ntb) 876{ 877 878 if (ntb->dev_type == NTB_DEV_USD) { 879 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 880 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 881 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 882 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 883 } else { 884 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 885 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 886 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 887 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 888 } 889} 890 891static void 892configure_xeon_secondary_side_bars(struct ntb_softc *ntb) 893{ 894 895 if (ntb->dev_type == NTB_DEV_USD) { 896 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 897 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 898 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 899 MBAR01_DSD_ADDR); 900 else { 901 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 902 PBAR4XLAT_USD_ADDR); 903 /* 904 * B2B_XLAT_OFFSET is a 64-bit register but can only be 905 * written 32 bits at a time. 906 */ 907 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 908 MBAR01_DSD_ADDR & 0xffffffff); 909 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 910 MBAR01_DSD_ADDR >> 32); 911 } 912 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 913 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 914 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 915 } else { 916 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 917 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 918 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 919 MBAR01_USD_ADDR); 920 else { 921 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 922 PBAR4XLAT_DSD_ADDR); 923 /* 924 * B2B_XLAT_OFFSET is a 64-bit register but can only be 925 * written 32 bits at a time. 926 */ 927 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 928 MBAR01_USD_ADDR & 0xffffffff); 929 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 930 MBAR01_USD_ADDR >> 32); 931 } 932 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 933 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 934 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 935 } 936} 937 938/* SOC does not have link status interrupt, poll on that platform */ 939static void 940ntb_handle_heartbeat(void *arg) 941{ 942 struct ntb_softc *ntb = arg; 943 uint32_t status32; 944 int rc; 945 946 rc = ntb_check_link_status(ntb); 947 if (rc != 0) 948 device_printf(ntb->device, 949 "Error determining link status\n"); 950 951 /* Check to see if a link error is the cause of the link down */ 952 if (ntb->link_status == NTB_LINK_DOWN) { 953 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 954 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 955 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 956 ntb); 957 return; 958 } 959 } 960 961 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 962 ntb_handle_heartbeat, ntb); 963} 964 965static void 966soc_perform_link_restart(struct ntb_softc *ntb) 967{ 968 uint32_t status; 969 970 /* Driver resets the NTB ModPhy lanes - magic! */ 971 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 972 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 973 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 974 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 975 976 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 977 pause("ModPhy", hz / 10); 978 979 /* Clear AER Errors, write to clear */ 980 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 981 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 982 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 983 984 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 985 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 986 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 987 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 988 989 /* Clear DeSkew Buffer error, write to clear */ 990 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 991 status |= SOC_DESKEWSTS_DBERR; 992 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 993 994 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 995 status &= SOC_IBIST_ERR_OFLOW; 996 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 997 998 /* Releases the NTB state machine to allow the link to retrain */ 999 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1000 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1001 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1002} 1003 1004static void 1005ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 1006{ 1007 enum ntb_hw_event event; 1008 uint16_t status; 1009 1010 if (ntb->link_status == link_state) 1011 return; 1012 1013 if (link_state == NTB_LINK_UP) { 1014 device_printf(ntb->device, "Link Up\n"); 1015 ntb->link_status = NTB_LINK_UP; 1016 event = NTB_EVENT_HW_LINK_UP; 1017 1018 if (ntb->type == NTB_SOC || 1019 ntb->conn_type == NTB_CONN_TRANSPARENT) 1020 status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1021 else 1022 status = pci_read_config(ntb->device, 1023 XEON_LINK_STATUS_OFFSET, 2); 1024 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 1025 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 1026 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 1027 ntb->link_width, ntb->link_speed); 1028 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1029 ntb_handle_heartbeat, ntb); 1030 } else { 1031 device_printf(ntb->device, "Link Down\n"); 1032 ntb->link_status = NTB_LINK_DOWN; 1033 event = NTB_EVENT_HW_LINK_DOWN; 1034 /* Do not modify link width/speed, we need it in link recovery */ 1035 } 1036 1037 /* notify the upper layer if we have an event change */ 1038 if (ntb->event_cb != NULL) 1039 ntb->event_cb(ntb->ntb_transport, event); 1040} 1041 1042static void 1043recover_soc_link(void *arg) 1044{ 1045 struct ntb_softc *ntb = arg; 1046 uint8_t speed, width; 1047 uint32_t status32; 1048 uint16_t status16; 1049 1050 soc_perform_link_restart(ntb); 1051 1052 /* 1053 * There is a potential race between the 2 NTB devices recovering at 1054 * the same time. If the times are the same, the link will not recover 1055 * and the driver will be stuck in this loop forever. Add a random 1056 * interval to the recovery time to prevent this race. 1057 */ 1058 status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1059 pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1060 1061 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1062 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1063 goto retry; 1064 1065 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1066 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1067 goto retry; 1068 1069 status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1070 if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1071 goto out; 1072 1073 status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1074 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 1075 speed = (status16 & NTB_LINK_SPEED_MASK); 1076 if (ntb->link_width != width || ntb->link_speed != speed) 1077 goto retry; 1078 1079out: 1080 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1081 ntb_handle_heartbeat, ntb); 1082 return; 1083 1084retry: 1085 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1086 ntb); 1087} 1088 1089static int 1090ntb_check_link_status(struct ntb_softc *ntb) 1091{ 1092 int link_state; 1093 uint32_t ntb_cntl; 1094 uint16_t status; 1095 1096 if (ntb->type == NTB_SOC) { 1097 ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1098 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1099 link_state = NTB_LINK_DOWN; 1100 else 1101 link_state = NTB_LINK_UP; 1102 } else { 1103 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 1104 2); 1105 1106 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1107 link_state = NTB_LINK_UP; 1108 else 1109 link_state = NTB_LINK_DOWN; 1110 } 1111 1112 ntb_handle_link_event(ntb, link_state); 1113 1114 return (0); 1115} 1116 1117/** 1118 * ntb_register_event_callback() - register event callback 1119 * @ntb: pointer to ntb_softc instance 1120 * @func: callback function to register 1121 * 1122 * This function registers a callback for any HW driver events such as link 1123 * up/down, power management notices and etc. 1124 * 1125 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1126 */ 1127int 1128ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1129{ 1130 1131 if (ntb->event_cb != NULL) 1132 return (EINVAL); 1133 1134 ntb->event_cb = func; 1135 1136 return (0); 1137} 1138 1139/** 1140 * ntb_unregister_event_callback() - unregisters the event callback 1141 * @ntb: pointer to ntb_softc instance 1142 * 1143 * This function unregisters the existing callback from transport 1144 */ 1145void 1146ntb_unregister_event_callback(struct ntb_softc *ntb) 1147{ 1148 1149 ntb->event_cb = NULL; 1150} 1151 1152/** 1153 * ntb_register_db_callback() - register a callback for doorbell interrupt 1154 * @ntb: pointer to ntb_softc instance 1155 * @idx: doorbell index to register callback, zero based 1156 * @data: pointer to be returned to caller with every callback 1157 * @func: callback function to register 1158 * 1159 * This function registers a callback function for the doorbell interrupt 1160 * on the primary side. The function will unmask the doorbell as well to 1161 * allow interrupt. 1162 * 1163 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1164 */ 1165int 1166ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1167 ntb_db_callback func) 1168{ 1169 uint16_t mask; 1170 1171 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1172 device_printf(ntb->device, "Invalid Index.\n"); 1173 return (EINVAL); 1174 } 1175 1176 ntb->db_cb[idx].callback = func; 1177 ntb->db_cb[idx].data = data; 1178 1179 /* unmask interrupt */ 1180 mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 1181 mask &= ~(1 << (idx * ntb->bits_per_vector)); 1182 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 1183 1184 return (0); 1185} 1186 1187/** 1188 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1189 * @ntb: pointer to ntb_softc instance 1190 * @idx: doorbell index to register callback, zero based 1191 * 1192 * This function unregisters a callback function for the doorbell interrupt 1193 * on the primary side. The function will also mask the said doorbell. 1194 */ 1195void 1196ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1197{ 1198 unsigned long mask; 1199 1200 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1201 return; 1202 1203 mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 1204 mask |= 1 << (idx * ntb->bits_per_vector); 1205 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 1206 1207 ntb->db_cb[idx].callback = NULL; 1208} 1209 1210/** 1211 * ntb_find_transport() - find the transport pointer 1212 * @transport: pointer to pci device 1213 * 1214 * Given the pci device pointer, return the transport pointer passed in when 1215 * the transport attached when it was inited. 1216 * 1217 * RETURNS: pointer to transport. 1218 */ 1219void * 1220ntb_find_transport(struct ntb_softc *ntb) 1221{ 1222 1223 return (ntb->ntb_transport); 1224} 1225 1226/** 1227 * ntb_register_transport() - Register NTB transport with NTB HW driver 1228 * @transport: transport identifier 1229 * 1230 * This function allows a transport to reserve the hardware driver for 1231 * NTB usage. 1232 * 1233 * RETURNS: pointer to ntb_softc, NULL on error. 1234 */ 1235struct ntb_softc * 1236ntb_register_transport(struct ntb_softc *ntb, void *transport) 1237{ 1238 1239 /* 1240 * TODO: when we have more than one transport, we will need to rewrite 1241 * this to prevent race conditions 1242 */ 1243 if (ntb->ntb_transport != NULL) 1244 return (NULL); 1245 1246 ntb->ntb_transport = transport; 1247 return (ntb); 1248} 1249 1250/** 1251 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1252 * @ntb - ntb_softc of the transport to be freed 1253 * 1254 * This function unregisters the transport from the HW driver and performs any 1255 * necessary cleanups. 1256 */ 1257void 1258ntb_unregister_transport(struct ntb_softc *ntb) 1259{ 1260 int i; 1261 1262 if (ntb->ntb_transport == NULL) 1263 return; 1264 1265 for (i = 0; i < ntb->allocated_interrupts; i++) 1266 ntb_unregister_db_callback(ntb, i); 1267 1268 ntb_unregister_event_callback(ntb); 1269 ntb->ntb_transport = NULL; 1270} 1271 1272/** 1273 * ntb_get_max_spads() - get the total scratch regs usable 1274 * @ntb: pointer to ntb_softc instance 1275 * 1276 * This function returns the max 32bit scratchpad registers usable by the 1277 * upper layer. 1278 * 1279 * RETURNS: total number of scratch pad registers available 1280 */ 1281uint8_t 1282ntb_get_max_spads(struct ntb_softc *ntb) 1283{ 1284 1285 return (ntb->limits.max_spads); 1286} 1287 1288/** 1289 * ntb_write_local_spad() - write to the secondary scratchpad register 1290 * @ntb: pointer to ntb_softc instance 1291 * @idx: index to the scratchpad register, 0 based 1292 * @val: the data value to put into the register 1293 * 1294 * This function allows writing of a 32bit value to the indexed scratchpad 1295 * register. The register resides on the secondary (external) side. 1296 * 1297 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1298 */ 1299int 1300ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1301{ 1302 1303 if (idx >= ntb->limits.max_spads) 1304 return (EINVAL); 1305 1306 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1307 1308 return (0); 1309} 1310 1311/** 1312 * ntb_read_local_spad() - read from the primary scratchpad register 1313 * @ntb: pointer to ntb_softc instance 1314 * @idx: index to scratchpad register, 0 based 1315 * @val: pointer to 32bit integer for storing the register value 1316 * 1317 * This function allows reading of the 32bit scratchpad register on 1318 * the primary (internal) side. 1319 * 1320 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1321 */ 1322int 1323ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1324{ 1325 1326 if (idx >= ntb->limits.max_spads) 1327 return (EINVAL); 1328 1329 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1330 1331 return (0); 1332} 1333 1334/** 1335 * ntb_write_remote_spad() - write to the secondary scratchpad register 1336 * @ntb: pointer to ntb_softc instance 1337 * @idx: index to the scratchpad register, 0 based 1338 * @val: the data value to put into the register 1339 * 1340 * This function allows writing of a 32bit value to the indexed scratchpad 1341 * register. The register resides on the secondary (external) side. 1342 * 1343 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1344 */ 1345int 1346ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1347{ 1348 1349 if (idx >= ntb->limits.max_spads) 1350 return (EINVAL); 1351 1352 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1353 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1354 else 1355 ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1356 1357 return (0); 1358} 1359 1360/** 1361 * ntb_read_remote_spad() - read from the primary scratchpad register 1362 * @ntb: pointer to ntb_softc instance 1363 * @idx: index to scratchpad register, 0 based 1364 * @val: pointer to 32bit integer for storing the register value 1365 * 1366 * This function allows reading of the 32bit scratchpad register on 1367 * the primary (internal) side. 1368 * 1369 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1370 */ 1371int 1372ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1373{ 1374 1375 if (idx >= ntb->limits.max_spads) 1376 return (EINVAL); 1377 1378 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1379 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1380 else 1381 *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1382 1383 return (0); 1384} 1385 1386/** 1387 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1388 * @ntb: pointer to ntb_softc instance 1389 * @mw: memory window number 1390 * 1391 * This function provides the base virtual address of the memory window 1392 * specified. 1393 * 1394 * RETURNS: pointer to virtual address, or NULL on error. 1395 */ 1396void * 1397ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1398{ 1399 1400 if (mw >= NTB_NUM_MW) 1401 return (NULL); 1402 1403 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1404} 1405 1406vm_paddr_t 1407ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1408{ 1409 1410 if (mw >= NTB_NUM_MW) 1411 return (0); 1412 1413 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1414} 1415 1416/** 1417 * ntb_get_mw_size() - return size of NTB memory window 1418 * @ntb: pointer to ntb_softc instance 1419 * @mw: memory window number 1420 * 1421 * This function provides the physical size of the memory window specified 1422 * 1423 * RETURNS: the size of the memory window or zero on error 1424 */ 1425u_long 1426ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1427{ 1428 1429 if (mw >= NTB_NUM_MW) 1430 return (0); 1431 1432 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1433} 1434 1435/** 1436 * ntb_set_mw_addr - set the memory window address 1437 * @ntb: pointer to ntb_softc instance 1438 * @mw: memory window number 1439 * @addr: base address for data 1440 * 1441 * This function sets the base physical address of the memory window. This 1442 * memory address is where data from the remote system will be transfered into 1443 * or out of depending on how the transport is configured. 1444 */ 1445void 1446ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1447{ 1448 1449 if (mw >= NTB_NUM_MW) 1450 return; 1451 1452 switch (NTB_MW_TO_BAR(mw)) { 1453 case NTB_B2B_BAR_1: 1454 ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr); 1455 break; 1456 case NTB_B2B_BAR_2: 1457 ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr); 1458 break; 1459 } 1460} 1461 1462/** 1463 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side 1464 * @ntb: pointer to ntb_softc instance 1465 * @db: doorbell to ring 1466 * 1467 * This function allows triggering of a doorbell on the secondary/external 1468 * side that will initiate an interrupt on the remote host 1469 * 1470 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1471 */ 1472void 1473ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db) 1474{ 1475 1476 if (ntb->type == NTB_SOC) 1477 ntb_reg_write(8, ntb->reg_ofs.rdb, (uint64_t) 1 << db); 1478 else { 1479 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1480 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1481 ((1 << ntb->bits_per_vector) - 1) << 1482 (db * ntb->bits_per_vector)); 1483 else 1484 ntb_reg_write(2, ntb->reg_ofs.rdb, 1485 ((1 << ntb->bits_per_vector) - 1) << 1486 (db * ntb->bits_per_vector)); 1487 } 1488} 1489 1490/** 1491 * ntb_query_link_status() - return the hardware link status 1492 * @ndev: pointer to ntb_device instance 1493 * 1494 * Returns true if the hardware is connected to the remote system 1495 * 1496 * RETURNS: true or false based on the hardware link state 1497 */ 1498bool 1499ntb_query_link_status(struct ntb_softc *ntb) 1500{ 1501 1502 return (ntb->link_status == NTB_LINK_UP); 1503} 1504 1505static void 1506save_bar_parameters(struct ntb_pci_bar_info *bar) 1507{ 1508 1509 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 1510 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 1511 bar->pbase = rman_get_start(bar->pci_resource); 1512 bar->size = rman_get_size(bar->pci_resource); 1513 bar->vbase = rman_get_virtual(bar->pci_resource); 1514} 1515 1516device_t 1517ntb_get_device(struct ntb_softc *ntb) 1518{ 1519 1520 return (ntb->device); 1521} 1522 1523/* Export HW-specific errata information. */ 1524bool 1525ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 1526{ 1527 1528 return (HAS_FEATURE(feature)); 1529} 1530