ntb_hw_intel.c revision 289280
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289280 2015-10-14 02:14:15Z cem $"); 29 30#include <sys/param.h> 31#include <sys/kernel.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/malloc.h> 35#include <sys/module.h> 36#include <sys/queue.h> 37#include <sys/rman.h> 38#include <sys/sysctl.h> 39#include <vm/vm.h> 40#include <vm/pmap.h> 41#include <machine/bus.h> 42#include <machine/pmap.h> 43#include <machine/resource.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46 47#include "ntb_regs.h" 48#include "ntb_hw.h" 49 50/* 51 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 52 * allows you to connect two systems using a PCI-e link. 53 * 54 * This module contains the hardware abstraction layer for the NTB. It allows 55 * you to send and recieve interrupts, map the memory windows and send and 56 * receive messages in the scratch-pad registers. 57 * 58 * NOTE: Much of the code in this module is shared with Linux. Any patches may 59 * be picked up and redistributed in Linux with a dual GPL/BSD license. 60 */ 61 62#define NTB_CONFIG_BAR 0 63#define NTB_B2B_BAR_1 1 64#define NTB_B2B_BAR_2 2 65#define NTB_MAX_BARS 3 66#define NTB_MW_TO_BAR(mw) ((mw) + 1) 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define SOC_LINK_RECOVERY_TIME 500 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75enum ntb_device_type { 76 NTB_XEON, 77 NTB_SOC 78}; 79 80/* Device features and workarounds */ 81#define HAS_FEATURE(feature) \ 82 ((ntb->features & (feature)) != 0) 83 84struct ntb_hw_info { 85 uint32_t device_id; 86 const char *desc; 87 enum ntb_device_type type; 88 uint64_t features; 89}; 90 91struct ntb_pci_bar_info { 92 bus_space_tag_t pci_bus_tag; 93 bus_space_handle_t pci_bus_handle; 94 int pci_resource_id; 95 struct resource *pci_resource; 96 vm_paddr_t pbase; 97 void *vbase; 98 u_long size; 99}; 100 101struct ntb_int_info { 102 struct resource *res; 103 int rid; 104 void *tag; 105}; 106 107struct ntb_db_cb { 108 ntb_db_callback callback; 109 unsigned int db_num; 110 void *data; 111 struct ntb_softc *ntb; 112}; 113 114struct ntb_softc { 115 device_t device; 116 enum ntb_device_type type; 117 uint64_t features; 118 119 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 120 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 121 uint32_t allocated_interrupts; 122 123 struct callout heartbeat_timer; 124 struct callout lr_timer; 125 126 void *ntb_transport; 127 ntb_event_callback event_cb; 128 struct ntb_db_cb *db_cb; 129 130 struct { 131 uint8_t max_spads; 132 uint8_t max_db_bits; 133 uint8_t msix_cnt; 134 } limits; 135 struct { 136 uint32_t ldb; 137 uint32_t ldb_mask; 138 uint32_t rdb; 139 uint32_t bar2_xlat; 140 uint32_t bar4_xlat; 141 uint32_t spad_remote; 142 uint32_t spad_local; 143 uint32_t lnk_cntl; 144 uint32_t lnk_stat; 145 uint32_t spci_cmd; 146 } reg_ofs; 147 uint8_t conn_type; 148 uint8_t dev_type; 149 uint8_t bits_per_vector; 150 uint8_t link_status; 151 uint8_t link_width; 152 uint8_t link_speed; 153}; 154 155#ifdef __i386__ 156static __inline uint64_t 157bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 158 bus_size_t offset) 159{ 160 161 return (bus_space_read_4(tag, handle, offset) | 162 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 163} 164 165static __inline void 166bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 167 bus_size_t offset, uint64_t val) 168{ 169 170 bus_space_write_4(tag, handle, offset, val); 171 bus_space_write_4(tag, handle, offset + 4, val >> 32); 172} 173#endif 174 175#define ntb_bar_read(SIZE, bar, offset) \ 176 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 177 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 178#define ntb_bar_write(SIZE, bar, offset, val) \ 179 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 180 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 181#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 182#define ntb_reg_write(SIZE, offset, val) \ 183 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 184#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset) 185#define ntb_mw_write(SIZE, offset, val) \ 186 ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val) 187 188typedef int (*bar_map_strategy)(struct ntb_softc *ntb, 189 struct ntb_pci_bar_info *bar); 190 191static int ntb_probe(device_t device); 192static int ntb_attach(device_t device); 193static int ntb_detach(device_t device); 194static int ntb_map_pci_bars(struct ntb_softc *ntb); 195static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 196 struct ntb_pci_bar_info *bar); 197static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 198static int map_memory_window_bar(struct ntb_softc *ntb, 199 struct ntb_pci_bar_info *bar); 200static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 201static int ntb_setup_interrupts(struct ntb_softc *ntb); 202static void ntb_teardown_interrupts(struct ntb_softc *ntb); 203static void handle_soc_irq(void *arg); 204static void handle_xeon_irq(void *arg); 205static void handle_xeon_event_irq(void *arg); 206static void ntb_handle_legacy_interrupt(void *arg); 207static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors); 208static void ntb_free_callbacks(struct ntb_softc *ntb); 209static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 210static int ntb_setup_xeon(struct ntb_softc *ntb); 211static int ntb_setup_soc(struct ntb_softc *ntb); 212static void ntb_teardown_xeon(struct ntb_softc *ntb); 213static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 214static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 215static void ntb_handle_heartbeat(void *arg); 216static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 217static void ntb_hw_link_down(struct ntb_softc *ntb); 218static void ntb_hw_link_up(struct ntb_softc *ntb); 219static void recover_soc_link(void *arg); 220static int ntb_check_link_status(struct ntb_softc *ntb); 221static void save_bar_parameters(struct ntb_pci_bar_info *bar); 222 223static struct ntb_hw_info pci_ids[] = { 224 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 225 226 /* XXX: PS/SS IDs left out until they are supported. */ 227 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 228 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 229 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 230 NTB_XEON, NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 }, 231 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 232 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 233 | NTB_BAR_SIZE_4K }, 234 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 235 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 236 }, 237 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 238 NTB_REGS_THRU_MW | NTB_B2BDOORBELL_BIT14 | NTB_SB01BASE_LOCKUP 239 }, 240 241 { 0x00000000, NULL, NTB_SOC, 0 } 242}; 243 244/* 245 * OS <-> Driver interface structures 246 */ 247MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 248 249static device_method_t ntb_pci_methods[] = { 250 /* Device interface */ 251 DEVMETHOD(device_probe, ntb_probe), 252 DEVMETHOD(device_attach, ntb_attach), 253 DEVMETHOD(device_detach, ntb_detach), 254 DEVMETHOD_END 255}; 256 257static driver_t ntb_pci_driver = { 258 "ntb_hw", 259 ntb_pci_methods, 260 sizeof(struct ntb_softc), 261}; 262 263static devclass_t ntb_devclass; 264DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 265MODULE_VERSION(ntb_hw, 1); 266 267SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 268 269/* 270 * OS <-> Driver linkage functions 271 */ 272static int 273ntb_probe(device_t device) 274{ 275 struct ntb_hw_info *p; 276 277 p = ntb_get_device_info(pci_get_devid(device)); 278 if (p == NULL) 279 return (ENXIO); 280 281 device_set_desc(device, p->desc); 282 return (0); 283} 284 285static int 286ntb_attach(device_t device) 287{ 288 struct ntb_softc *ntb; 289 struct ntb_hw_info *p; 290 int error; 291 292 ntb = DEVICE2SOFTC(device); 293 p = ntb_get_device_info(pci_get_devid(device)); 294 295 ntb->device = device; 296 ntb->type = p->type; 297 ntb->features = p->features; 298 299 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 300 callout_init(&ntb->heartbeat_timer, 1); 301 callout_init(&ntb->lr_timer, 1); 302 303 error = ntb_map_pci_bars(ntb); 304 if (error) 305 goto out; 306 if (ntb->type == NTB_SOC) 307 error = ntb_setup_soc(ntb); 308 else 309 error = ntb_setup_xeon(ntb); 310 if (error) 311 goto out; 312 error = ntb_setup_interrupts(ntb); 313 if (error) 314 goto out; 315 316 pci_enable_busmaster(ntb->device); 317 318out: 319 if (error != 0) 320 ntb_detach(device); 321 return (error); 322} 323 324static int 325ntb_detach(device_t device) 326{ 327 struct ntb_softc *ntb; 328 329 ntb = DEVICE2SOFTC(device); 330 callout_drain(&ntb->heartbeat_timer); 331 callout_drain(&ntb->lr_timer); 332 if (ntb->type == NTB_XEON) 333 ntb_teardown_xeon(ntb); 334 ntb_teardown_interrupts(ntb); 335 ntb_unmap_pci_bar(ntb); 336 337 return (0); 338} 339 340static int 341ntb_map_pci_bars(struct ntb_softc *ntb) 342{ 343 int rc; 344 345 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 346 rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]); 347 if (rc != 0) 348 return (rc); 349 350 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 351 rc = map_pci_bar(ntb, map_memory_window_bar, 352 &ntb->bar_info[NTB_B2B_BAR_1]); 353 if (rc != 0) 354 return (rc); 355 356 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 357 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 358 rc = map_pci_bar(ntb, map_mmr_bar, 359 &ntb->bar_info[NTB_B2B_BAR_2]); 360 else 361 rc = map_pci_bar(ntb, map_memory_window_bar, 362 &ntb->bar_info[NTB_B2B_BAR_2]); 363 return (rc); 364} 365 366static int 367map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy, 368 struct ntb_pci_bar_info *bar) 369{ 370 int rc; 371 372 rc = strategy(ntb, bar); 373 if (rc != 0) 374 device_printf(ntb->device, 375 "unable to allocate pci resource\n"); 376 else 377 device_printf(ntb->device, 378 "Bar size = %lx, v %p, p %p\n", 379 bar->size, bar->vbase, (void *)(bar->pbase)); 380 return (rc); 381} 382 383static int 384map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 385{ 386 387 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 388 &bar->pci_resource_id, RF_ACTIVE); 389 if (bar->pci_resource == NULL) 390 return (ENXIO); 391 392 save_bar_parameters(bar); 393 return (0); 394} 395 396static int 397map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 398{ 399 int rc; 400 uint8_t bar_size_bits = 0; 401 402 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 403 &bar->pci_resource_id, RF_ACTIVE); 404 405 if (bar->pci_resource == NULL) 406 return (ENXIO); 407 408 save_bar_parameters(bar); 409 /* 410 * Ivytown NTB BAR sizes are misreported by the hardware due to a 411 * hardware issue. To work around this, query the size it should be 412 * configured to by the device and modify the resource to correspond to 413 * this new size. The BIOS on systems with this problem is required to 414 * provide enough address space to allow the driver to make this change 415 * safely. 416 * 417 * Ideally I could have just specified the size when I allocated the 418 * resource like: 419 * bus_alloc_resource(ntb->device, 420 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 421 * 1ul << bar_size_bits, RF_ACTIVE); 422 * but the PCI driver does not honor the size in this call, so we have 423 * to modify it after the fact. 424 */ 425 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 426 if (bar->pci_resource_id == PCIR_BAR(2)) 427 bar_size_bits = pci_read_config(ntb->device, 428 XEON_PBAR23SZ_OFFSET, 1); 429 else 430 bar_size_bits = pci_read_config(ntb->device, 431 XEON_PBAR45SZ_OFFSET, 1); 432 433 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 434 bar->pci_resource, bar->pbase, 435 bar->pbase + (1ul << bar_size_bits) - 1); 436 if (rc != 0) { 437 device_printf(ntb->device, 438 "unable to resize bar\n"); 439 return (rc); 440 } 441 442 save_bar_parameters(bar); 443 } 444 445 /* Mark bar region as write combining to improve performance. */ 446 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 447 VM_MEMATTR_WRITE_COMBINING); 448 if (rc != 0) { 449 device_printf(ntb->device, 450 "unable to mark bar as WRITE_COMBINING\n"); 451 return (rc); 452 } 453 return (0); 454} 455 456static void 457ntb_unmap_pci_bar(struct ntb_softc *ntb) 458{ 459 struct ntb_pci_bar_info *current_bar; 460 int i; 461 462 for (i = 0; i< NTB_MAX_BARS; i++) { 463 current_bar = &ntb->bar_info[i]; 464 if (current_bar->pci_resource != NULL) 465 bus_release_resource(ntb->device, SYS_RES_MEMORY, 466 current_bar->pci_resource_id, 467 current_bar->pci_resource); 468 } 469} 470 471static int 472ntb_setup_interrupts(struct ntb_softc *ntb) 473{ 474 void (*interrupt_handler)(void *); 475 void *int_arg; 476 bool use_msix = false; 477 uint32_t num_vectors; 478 int i; 479 480 ntb->allocated_interrupts = 0; 481 /* 482 * On SOC, disable all interrupts. On XEON, disable all but Link 483 * Interrupt. The rest will be unmasked as callbacks are registered. 484 */ 485 if (ntb->type == NTB_SOC) 486 ntb_reg_write(8, ntb->reg_ofs.ldb_mask, ~0); 487 else 488 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, 489 (uint16_t) ~(1 << XEON_LINK_DB)); 490 491 num_vectors = MIN(pci_msix_count(ntb->device), 492 ntb->limits.max_db_bits); 493 if (num_vectors >= 1) { 494 pci_alloc_msix(ntb->device, &num_vectors); 495 if (num_vectors >= 4) 496 use_msix = true; 497 } 498 499 ntb_create_callbacks(ntb, num_vectors); 500 if (use_msix == true) { 501 for (i = 0; i < num_vectors; i++) { 502 ntb->int_info[i].rid = i + 1; 503 ntb->int_info[i].res = bus_alloc_resource_any( 504 ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid, 505 RF_ACTIVE); 506 if (ntb->int_info[i].res == NULL) { 507 device_printf(ntb->device, 508 "bus_alloc_resource failed\n"); 509 return (ENOMEM); 510 } 511 ntb->int_info[i].tag = NULL; 512 ntb->allocated_interrupts++; 513 if (ntb->type == NTB_SOC) { 514 interrupt_handler = handle_soc_irq; 515 int_arg = &ntb->db_cb[i]; 516 } else { 517 if (i == num_vectors - 1) { 518 interrupt_handler = 519 handle_xeon_event_irq; 520 int_arg = ntb; 521 } else { 522 interrupt_handler = 523 handle_xeon_irq; 524 int_arg = &ntb->db_cb[i]; 525 } 526 } 527 if (bus_setup_intr(ntb->device, ntb->int_info[i].res, 528 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 529 interrupt_handler, int_arg, 530 &ntb->int_info[i].tag) != 0) { 531 device_printf(ntb->device, 532 "bus_setup_intr failed\n"); 533 return (ENXIO); 534 } 535 } 536 } else { 537 ntb->int_info[0].rid = 0; 538 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, 539 SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 540 interrupt_handler = ntb_handle_legacy_interrupt; 541 if (ntb->int_info[0].res == NULL) { 542 device_printf(ntb->device, 543 "bus_alloc_resource failed\n"); 544 return (ENOMEM); 545 } 546 ntb->int_info[0].tag = NULL; 547 ntb->allocated_interrupts = 1; 548 549 if (bus_setup_intr(ntb->device, ntb->int_info[0].res, 550 INTR_MPSAFE | INTR_TYPE_MISC, NULL, 551 interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) { 552 553 device_printf(ntb->device, "bus_setup_intr failed\n"); 554 return (ENXIO); 555 } 556 } 557 558 return (0); 559} 560 561static void 562ntb_teardown_interrupts(struct ntb_softc *ntb) 563{ 564 struct ntb_int_info *current_int; 565 int i; 566 567 for (i = 0; i < ntb->allocated_interrupts; i++) { 568 current_int = &ntb->int_info[i]; 569 if (current_int->tag != NULL) 570 bus_teardown_intr(ntb->device, current_int->res, 571 current_int->tag); 572 573 if (current_int->res != NULL) 574 bus_release_resource(ntb->device, SYS_RES_IRQ, 575 rman_get_rid(current_int->res), current_int->res); 576 } 577 578 ntb_free_callbacks(ntb); 579 pci_release_msi(ntb->device); 580} 581 582static void 583handle_soc_irq(void *arg) 584{ 585 struct ntb_db_cb *db_cb = arg; 586 struct ntb_softc *ntb = db_cb->ntb; 587 588 ntb_reg_write(8, ntb->reg_ofs.ldb, (uint64_t) 1 << db_cb->db_num); 589 590 if (db_cb->callback != NULL) 591 db_cb->callback(db_cb->data, db_cb->db_num); 592} 593 594static void 595handle_xeon_irq(void *arg) 596{ 597 struct ntb_db_cb *db_cb = arg; 598 struct ntb_softc *ntb = db_cb->ntb; 599 600 /* 601 * On Xeon, there are 16 bits in the interrupt register 602 * but only 4 vectors. So, 5 bits are assigned to the first 3 603 * vectors, with the 4th having a single bit for link 604 * interrupts. 605 */ 606 ntb_reg_write(2, ntb->reg_ofs.ldb, 607 ((1 << ntb->bits_per_vector) - 1) << 608 (db_cb->db_num * ntb->bits_per_vector)); 609 610 if (db_cb->callback != NULL) 611 db_cb->callback(db_cb->data, db_cb->db_num); 612} 613 614/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */ 615static void 616handle_xeon_event_irq(void *arg) 617{ 618 struct ntb_softc *ntb = arg; 619 int rc; 620 621 rc = ntb_check_link_status(ntb); 622 if (rc != 0) 623 device_printf(ntb->device, "Error determining link status\n"); 624 625 /* bit 15 is always the link bit */ 626 ntb_reg_write(2, ntb->reg_ofs.ldb, 1 << XEON_LINK_DB); 627} 628 629static void 630ntb_handle_legacy_interrupt(void *arg) 631{ 632 struct ntb_softc *ntb = arg; 633 unsigned int i = 0; 634 uint64_t ldb64; 635 uint16_t ldb16; 636 637 if (ntb->type == NTB_SOC) { 638 ldb64 = ntb_reg_read(8, ntb->reg_ofs.ldb); 639 640 while (ldb64) { 641 i = ffs(ldb64); 642 ldb64 &= ldb64 - 1; 643 handle_soc_irq(&ntb->db_cb[i]); 644 } 645 } else { 646 ldb16 = ntb_reg_read(2, ntb->reg_ofs.ldb); 647 648 if ((ldb16 & XEON_DB_HW_LINK) != 0) { 649 handle_xeon_event_irq(ntb); 650 ldb16 &= ~XEON_DB_HW_LINK; 651 } 652 653 while (ldb16 != 0) { 654 i = ffs(ldb16); 655 ldb16 &= ldb16 - 1; 656 handle_xeon_irq(&ntb->db_cb[i]); 657 } 658 } 659 660} 661 662static int 663ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors) 664{ 665 int i; 666 667 ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 668 M_ZERO | M_WAITOK); 669 for (i = 0; i < num_vectors; i++) { 670 ntb->db_cb[i].db_num = i; 671 ntb->db_cb[i].ntb = ntb; 672 } 673 674 return (0); 675} 676 677static void 678ntb_free_callbacks(struct ntb_softc *ntb) 679{ 680 int i; 681 682 for (i = 0; i < ntb->limits.max_db_bits; i++) 683 ntb_unregister_db_callback(ntb, i); 684 685 free(ntb->db_cb, M_NTB); 686} 687 688static struct ntb_hw_info * 689ntb_get_device_info(uint32_t device_id) 690{ 691 struct ntb_hw_info *ep = pci_ids; 692 693 while (ep->device_id) { 694 if (ep->device_id == device_id) 695 return (ep); 696 ++ep; 697 } 698 return (NULL); 699} 700 701static void 702ntb_teardown_xeon(struct ntb_softc *ntb) 703{ 704 705 ntb_hw_link_down(ntb); 706} 707 708static int 709ntb_setup_xeon(struct ntb_softc *ntb) 710{ 711 uint8_t val, connection_type; 712 713 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 714 715 connection_type = val & XEON_PPD_CONN_TYPE; 716 717 if ((val & XEON_PPD_DEV_TYPE) != 0) 718 ntb->dev_type = NTB_DEV_USD; 719 else 720 ntb->dev_type = NTB_DEV_DSD; 721 722 ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; 723 ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; 724 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 725 ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET; 726 ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; 727 728 switch (connection_type) { 729 case NTB_CONN_B2B: 730 ntb->conn_type = NTB_CONN_B2B; 731 732 /* 733 * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored 734 * with the NTB_REGS_THRU_MW errata mode enabled. (See 735 * ntb_ring_doorbell() and ntb_read/write_remote_spad().) 736 */ 737 ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET; 738 ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; 739 740 ntb->limits.max_spads = XEON_MAX_SPADS; 741 break; 742 743 case NTB_CONN_RP: 744 /* 745 * Every Xeon today needs NTB_REGS_THRU_MW, so punt on RP for 746 * now. 747 */ 748 KASSERT(HAS_FEATURE(NTB_REGS_THRU_MW), 749 ("Xeon without MW errata unimplemented")); 750 device_printf(ntb->device, 751 "NTB-RP disabled to due hardware errata.\n"); 752 return (ENXIO); 753 754 case NTB_CONN_TRANSPARENT: 755 default: 756 device_printf(ntb->device, "Connection type %d not supported\n", 757 connection_type); 758 return (ENXIO); 759 } 760 761 /* 762 * There is a Xeon hardware errata related to writes to SDOORBELL or 763 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 764 * which may hang the system. To workaround this use the second memory 765 * window to access the interrupt and scratch pad registers on the 766 * remote system. 767 * 768 * There is another HW errata on the limit registers -- they can only 769 * be written when the base register is (?)4GB aligned and < 32-bit. 770 * This should already be the case based on the driver defaults, but 771 * write the limit registers first just in case. 772 */ 773 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 774 /* 775 * Set the Limit register to 4k, the minimum size, to prevent 776 * an illegal access. 777 */ 778 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 779 ntb_get_mw_size(ntb, 1) + 0x1000); 780 else 781 /* 782 * Disable the limit register, just in case it is set to 783 * something silly. 784 */ 785 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 786 787 788 ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; 789 ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; 790 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 791 792 ntb->limits.max_db_bits = XEON_MAX_DB_BITS; 793 ntb->limits.msix_cnt = XEON_MSIX_CNT; 794 ntb->bits_per_vector = XEON_DB_BITS_PER_VEC; 795 796 /* 797 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 798 * mirrored to the remote system. Shrink the number of bits by one, 799 * since bit 14 is the last bit. 800 * 801 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 802 * anyway. Nor for non-B2B connection types. 803 */ 804 if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) && 805 !HAS_FEATURE(NTB_REGS_THRU_MW) && 806 connection_type == NTB_CONN_B2B) 807 ntb->limits.max_db_bits = XEON_MAX_DB_BITS - 1; 808 809 configure_xeon_secondary_side_bars(ntb); 810 811 /* Enable Bus Master and Memory Space on the secondary side */ 812 if (ntb->conn_type == NTB_CONN_B2B) 813 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 814 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 815 816 /* Enable link training */ 817 ntb_hw_link_up(ntb); 818 819 return (0); 820} 821 822static int 823ntb_setup_soc(struct ntb_softc *ntb) 824{ 825 uint32_t val, connection_type; 826 827 val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 828 829 connection_type = (val & SOC_PPD_CONN_TYPE) >> 8; 830 switch (connection_type) { 831 case NTB_CONN_B2B: 832 ntb->conn_type = NTB_CONN_B2B; 833 break; 834 default: 835 device_printf(ntb->device, 836 "Unsupported NTB configuration (%d)\n", connection_type); 837 return (ENXIO); 838 } 839 840 if ((val & SOC_PPD_DEV_TYPE) != 0) 841 ntb->dev_type = NTB_DEV_DSD; 842 else 843 ntb->dev_type = NTB_DEV_USD; 844 845 /* Initiate PCI-E link training */ 846 pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK, 847 4); 848 849 ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; 850 ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; 851 ntb->reg_ofs.rdb = SOC_B2B_DOORBELL_OFFSET; 852 ntb->reg_ofs.bar2_xlat = SOC_SBAR2XLAT_OFFSET; 853 ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; 854 ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; 855 ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; 856 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 857 ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; 858 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 859 860 ntb->limits.max_spads = SOC_MAX_SPADS; 861 ntb->limits.max_db_bits = SOC_MAX_DB_BITS; 862 ntb->limits.msix_cnt = SOC_MSIX_CNT; 863 ntb->bits_per_vector = SOC_DB_BITS_PER_VEC; 864 865 /* 866 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 867 * resolved. Mask transaction layer internal parity errors. 868 */ 869 pci_write_config(ntb->device, 0xFC, 0x4, 4); 870 871 configure_soc_secondary_side_bars(ntb); 872 873 /* Enable Bus Master and Memory Space on the secondary side */ 874 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 875 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 876 877 callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); 878 879 return (0); 880} 881 882static void 883configure_soc_secondary_side_bars(struct ntb_softc *ntb) 884{ 885 886 if (ntb->dev_type == NTB_DEV_USD) { 887 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 888 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); 889 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR); 890 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR); 891 } else { 892 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 893 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); 894 ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR); 895 ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR); 896 } 897} 898 899static void 900configure_xeon_secondary_side_bars(struct ntb_softc *ntb) 901{ 902 903 if (ntb->dev_type == NTB_DEV_USD) { 904 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR); 905 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 906 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 907 MBAR01_DSD_ADDR); 908 else { 909 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 910 PBAR4XLAT_USD_ADDR); 911 /* 912 * B2B_XLAT_OFFSET is a 64-bit register but can only be 913 * written 32 bits at a time. 914 */ 915 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 916 MBAR01_DSD_ADDR & 0xffffffff); 917 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 918 MBAR01_DSD_ADDR >> 32); 919 } 920 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); 921 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); 922 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); 923 } else { 924 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR); 925 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 926 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 927 MBAR01_USD_ADDR); 928 else { 929 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 930 PBAR4XLAT_DSD_ADDR); 931 /* 932 * B2B_XLAT_OFFSET is a 64-bit register but can only be 933 * written 32 bits at a time. 934 */ 935 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 936 MBAR01_USD_ADDR & 0xffffffff); 937 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, 938 MBAR01_USD_ADDR >> 32); 939 } 940 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); 941 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); 942 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); 943 } 944} 945 946/* SOC does not have link status interrupt, poll on that platform */ 947static void 948ntb_handle_heartbeat(void *arg) 949{ 950 struct ntb_softc *ntb = arg; 951 uint32_t status32; 952 int rc; 953 954 rc = ntb_check_link_status(ntb); 955 if (rc != 0) 956 device_printf(ntb->device, 957 "Error determining link status\n"); 958 959 /* Check to see if a link error is the cause of the link down */ 960 if (ntb->link_status == NTB_LINK_DOWN) { 961 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 962 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 963 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 964 ntb); 965 return; 966 } 967 } 968 969 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 970 ntb_handle_heartbeat, ntb); 971} 972 973static void 974soc_perform_link_restart(struct ntb_softc *ntb) 975{ 976 uint32_t status; 977 978 /* Driver resets the NTB ModPhy lanes - magic! */ 979 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 980 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 981 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 982 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 983 984 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 985 pause("ModPhy", hz / 10); 986 987 /* Clear AER Errors, write to clear */ 988 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 989 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 990 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 991 992 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 993 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 994 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 995 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 996 997 /* Clear DeSkew Buffer error, write to clear */ 998 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 999 status |= SOC_DESKEWSTS_DBERR; 1000 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1001 1002 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1003 status &= SOC_IBIST_ERR_OFLOW; 1004 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1005 1006 /* Releases the NTB state machine to allow the link to retrain */ 1007 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1008 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1009 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1010} 1011 1012static void 1013ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 1014{ 1015 enum ntb_hw_event event; 1016 uint16_t status; 1017 1018 if (ntb->link_status == link_state) 1019 return; 1020 1021 if (link_state == NTB_LINK_UP) { 1022 device_printf(ntb->device, "Link Up\n"); 1023 ntb->link_status = NTB_LINK_UP; 1024 event = NTB_EVENT_HW_LINK_UP; 1025 1026 if (ntb->type == NTB_SOC || 1027 ntb->conn_type == NTB_CONN_TRANSPARENT) 1028 status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1029 else 1030 status = pci_read_config(ntb->device, 1031 XEON_LINK_STATUS_OFFSET, 2); 1032 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 1033 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 1034 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 1035 ntb->link_width, ntb->link_speed); 1036 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1037 ntb_handle_heartbeat, ntb); 1038 } else { 1039 device_printf(ntb->device, "Link Down\n"); 1040 ntb->link_status = NTB_LINK_DOWN; 1041 event = NTB_EVENT_HW_LINK_DOWN; 1042 /* Do not modify link width/speed, we need it in link recovery */ 1043 } 1044 1045 /* notify the upper layer if we have an event change */ 1046 if (ntb->event_cb != NULL) 1047 ntb->event_cb(ntb->ntb_transport, event); 1048} 1049 1050static void 1051ntb_hw_link_up(struct ntb_softc *ntb) 1052{ 1053 uint32_t cntl; 1054 1055 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1056 ntb_handle_link_event(ntb, NTB_LINK_UP); 1057 return; 1058 } 1059 1060 cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1061 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1062 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1063 cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP; 1064 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1065} 1066 1067static void 1068ntb_hw_link_down(struct ntb_softc *ntb) 1069{ 1070 uint32_t cntl; 1071 1072 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1073 ntb_handle_link_event(ntb, NTB_LINK_DOWN); 1074 return; 1075 } 1076 1077 cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1078 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1079 cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP); 1080 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1081 ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); 1082} 1083 1084static void 1085recover_soc_link(void *arg) 1086{ 1087 struct ntb_softc *ntb = arg; 1088 uint8_t speed, width; 1089 uint32_t status32; 1090 uint16_t status16; 1091 1092 soc_perform_link_restart(ntb); 1093 1094 /* 1095 * There is a potential race between the 2 NTB devices recovering at 1096 * the same time. If the times are the same, the link will not recover 1097 * and the driver will be stuck in this loop forever. Add a random 1098 * interval to the recovery time to prevent this race. 1099 */ 1100 status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1101 pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1102 1103 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1104 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1105 goto retry; 1106 1107 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1108 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1109 goto retry; 1110 1111 status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1112 if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1113 goto out; 1114 1115 status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); 1116 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; 1117 speed = (status16 & NTB_LINK_SPEED_MASK); 1118 if (ntb->link_width != width || ntb->link_speed != speed) 1119 goto retry; 1120 1121out: 1122 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1123 ntb_handle_heartbeat, ntb); 1124 return; 1125 1126retry: 1127 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1128 ntb); 1129} 1130 1131static int 1132ntb_check_link_status(struct ntb_softc *ntb) 1133{ 1134 int link_state; 1135 uint32_t ntb_cntl; 1136 uint16_t status; 1137 1138 if (ntb->type == NTB_SOC) { 1139 ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); 1140 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1141 link_state = NTB_LINK_DOWN; 1142 else 1143 link_state = NTB_LINK_UP; 1144 } else { 1145 status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 1146 2); 1147 1148 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1149 link_state = NTB_LINK_UP; 1150 else 1151 link_state = NTB_LINK_DOWN; 1152 } 1153 1154 ntb_handle_link_event(ntb, link_state); 1155 1156 return (0); 1157} 1158 1159/** 1160 * ntb_register_event_callback() - register event callback 1161 * @ntb: pointer to ntb_softc instance 1162 * @func: callback function to register 1163 * 1164 * This function registers a callback for any HW driver events such as link 1165 * up/down, power management notices and etc. 1166 * 1167 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1168 */ 1169int 1170ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1171{ 1172 1173 if (ntb->event_cb != NULL) 1174 return (EINVAL); 1175 1176 ntb->event_cb = func; 1177 1178 return (0); 1179} 1180 1181/** 1182 * ntb_unregister_event_callback() - unregisters the event callback 1183 * @ntb: pointer to ntb_softc instance 1184 * 1185 * This function unregisters the existing callback from transport 1186 */ 1187void 1188ntb_unregister_event_callback(struct ntb_softc *ntb) 1189{ 1190 1191 ntb->event_cb = NULL; 1192} 1193 1194/** 1195 * ntb_register_db_callback() - register a callback for doorbell interrupt 1196 * @ntb: pointer to ntb_softc instance 1197 * @idx: doorbell index to register callback, zero based 1198 * @data: pointer to be returned to caller with every callback 1199 * @func: callback function to register 1200 * 1201 * This function registers a callback function for the doorbell interrupt 1202 * on the primary side. The function will unmask the doorbell as well to 1203 * allow interrupt. 1204 * 1205 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1206 */ 1207int 1208ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1209 ntb_db_callback func) 1210{ 1211 uint16_t mask; 1212 1213 if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) { 1214 device_printf(ntb->device, "Invalid Index.\n"); 1215 return (EINVAL); 1216 } 1217 1218 ntb->db_cb[idx].callback = func; 1219 ntb->db_cb[idx].data = data; 1220 1221 /* unmask interrupt */ 1222 mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 1223 mask &= ~(1 << (idx * ntb->bits_per_vector)); 1224 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 1225 1226 return (0); 1227} 1228 1229/** 1230 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1231 * @ntb: pointer to ntb_softc instance 1232 * @idx: doorbell index to register callback, zero based 1233 * 1234 * This function unregisters a callback function for the doorbell interrupt 1235 * on the primary side. The function will also mask the said doorbell. 1236 */ 1237void 1238ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1239{ 1240 unsigned long mask; 1241 1242 if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback) 1243 return; 1244 1245 mask = ntb_reg_read(2, ntb->reg_ofs.ldb_mask); 1246 mask |= 1 << (idx * ntb->bits_per_vector); 1247 ntb_reg_write(2, ntb->reg_ofs.ldb_mask, mask); 1248 1249 ntb->db_cb[idx].callback = NULL; 1250} 1251 1252/** 1253 * ntb_find_transport() - find the transport pointer 1254 * @transport: pointer to pci device 1255 * 1256 * Given the pci device pointer, return the transport pointer passed in when 1257 * the transport attached when it was inited. 1258 * 1259 * RETURNS: pointer to transport. 1260 */ 1261void * 1262ntb_find_transport(struct ntb_softc *ntb) 1263{ 1264 1265 return (ntb->ntb_transport); 1266} 1267 1268/** 1269 * ntb_register_transport() - Register NTB transport with NTB HW driver 1270 * @transport: transport identifier 1271 * 1272 * This function allows a transport to reserve the hardware driver for 1273 * NTB usage. 1274 * 1275 * RETURNS: pointer to ntb_softc, NULL on error. 1276 */ 1277struct ntb_softc * 1278ntb_register_transport(struct ntb_softc *ntb, void *transport) 1279{ 1280 1281 /* 1282 * TODO: when we have more than one transport, we will need to rewrite 1283 * this to prevent race conditions 1284 */ 1285 if (ntb->ntb_transport != NULL) 1286 return (NULL); 1287 1288 ntb->ntb_transport = transport; 1289 return (ntb); 1290} 1291 1292/** 1293 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1294 * @ntb - ntb_softc of the transport to be freed 1295 * 1296 * This function unregisters the transport from the HW driver and performs any 1297 * necessary cleanups. 1298 */ 1299void 1300ntb_unregister_transport(struct ntb_softc *ntb) 1301{ 1302 int i; 1303 1304 if (ntb->ntb_transport == NULL) 1305 return; 1306 1307 for (i = 0; i < ntb->allocated_interrupts; i++) 1308 ntb_unregister_db_callback(ntb, i); 1309 1310 ntb_unregister_event_callback(ntb); 1311 ntb->ntb_transport = NULL; 1312} 1313 1314/** 1315 * ntb_get_max_spads() - get the total scratch regs usable 1316 * @ntb: pointer to ntb_softc instance 1317 * 1318 * This function returns the max 32bit scratchpad registers usable by the 1319 * upper layer. 1320 * 1321 * RETURNS: total number of scratch pad registers available 1322 */ 1323uint8_t 1324ntb_get_max_spads(struct ntb_softc *ntb) 1325{ 1326 1327 return (ntb->limits.max_spads); 1328} 1329 1330/** 1331 * ntb_write_local_spad() - write to the secondary scratchpad register 1332 * @ntb: pointer to ntb_softc instance 1333 * @idx: index to the scratchpad register, 0 based 1334 * @val: the data value to put into the register 1335 * 1336 * This function allows writing of a 32bit value to the indexed scratchpad 1337 * register. The register resides on the secondary (external) side. 1338 * 1339 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1340 */ 1341int 1342ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1343{ 1344 1345 if (idx >= ntb->limits.max_spads) 1346 return (EINVAL); 1347 1348 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1349 1350 return (0); 1351} 1352 1353/** 1354 * ntb_read_local_spad() - read from the primary scratchpad register 1355 * @ntb: pointer to ntb_softc instance 1356 * @idx: index to scratchpad register, 0 based 1357 * @val: pointer to 32bit integer for storing the register value 1358 * 1359 * This function allows reading of the 32bit scratchpad register on 1360 * the primary (internal) side. 1361 * 1362 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1363 */ 1364int 1365ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1366{ 1367 1368 if (idx >= ntb->limits.max_spads) 1369 return (EINVAL); 1370 1371 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1372 1373 return (0); 1374} 1375 1376/** 1377 * ntb_write_remote_spad() - write to the secondary scratchpad register 1378 * @ntb: pointer to ntb_softc instance 1379 * @idx: index to the scratchpad register, 0 based 1380 * @val: the data value to put into the register 1381 * 1382 * This function allows writing of a 32bit value to the indexed scratchpad 1383 * register. The register resides on the secondary (external) side. 1384 * 1385 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1386 */ 1387int 1388ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1389{ 1390 1391 if (idx >= ntb->limits.max_spads) 1392 return (EINVAL); 1393 1394 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1395 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1396 else 1397 ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); 1398 1399 return (0); 1400} 1401 1402/** 1403 * ntb_read_remote_spad() - read from the primary scratchpad register 1404 * @ntb: pointer to ntb_softc instance 1405 * @idx: index to scratchpad register, 0 based 1406 * @val: pointer to 32bit integer for storing the register value 1407 * 1408 * This function allows reading of the 32bit scratchpad register on 1409 * the primary (internal) side. 1410 * 1411 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1412 */ 1413int 1414ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1415{ 1416 1417 if (idx >= ntb->limits.max_spads) 1418 return (EINVAL); 1419 1420 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1421 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1422 else 1423 *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); 1424 1425 return (0); 1426} 1427 1428/** 1429 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1430 * @ntb: pointer to ntb_softc instance 1431 * @mw: memory window number 1432 * 1433 * This function provides the base virtual address of the memory window 1434 * specified. 1435 * 1436 * RETURNS: pointer to virtual address, or NULL on error. 1437 */ 1438void * 1439ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1440{ 1441 1442 if (mw >= NTB_NUM_MW) 1443 return (NULL); 1444 1445 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase); 1446} 1447 1448vm_paddr_t 1449ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1450{ 1451 1452 if (mw >= NTB_NUM_MW) 1453 return (0); 1454 1455 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase); 1456} 1457 1458/** 1459 * ntb_get_mw_size() - return size of NTB memory window 1460 * @ntb: pointer to ntb_softc instance 1461 * @mw: memory window number 1462 * 1463 * This function provides the physical size of the memory window specified 1464 * 1465 * RETURNS: the size of the memory window or zero on error 1466 */ 1467u_long 1468ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1469{ 1470 1471 if (mw >= NTB_NUM_MW) 1472 return (0); 1473 1474 return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size); 1475} 1476 1477/** 1478 * ntb_set_mw_addr - set the memory window address 1479 * @ntb: pointer to ntb_softc instance 1480 * @mw: memory window number 1481 * @addr: base address for data 1482 * 1483 * This function sets the base physical address of the memory window. This 1484 * memory address is where data from the remote system will be transfered into 1485 * or out of depending on how the transport is configured. 1486 */ 1487void 1488ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1489{ 1490 1491 if (mw >= NTB_NUM_MW) 1492 return; 1493 1494 switch (NTB_MW_TO_BAR(mw)) { 1495 case NTB_B2B_BAR_1: 1496 ntb_reg_write(8, ntb->reg_ofs.bar2_xlat, addr); 1497 break; 1498 case NTB_B2B_BAR_2: 1499 ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr); 1500 break; 1501 } 1502} 1503 1504/** 1505 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side 1506 * @ntb: pointer to ntb_softc instance 1507 * @db: doorbell to ring 1508 * 1509 * This function allows triggering of a doorbell on the secondary/external 1510 * side that will initiate an interrupt on the remote host 1511 * 1512 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1513 */ 1514void 1515ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db) 1516{ 1517 1518 if (ntb->type == NTB_SOC) 1519 ntb_reg_write(8, ntb->reg_ofs.rdb, (uint64_t) 1 << db); 1520 else { 1521 if (HAS_FEATURE(NTB_REGS_THRU_MW)) 1522 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, 1523 ((1 << ntb->bits_per_vector) - 1) << 1524 (db * ntb->bits_per_vector)); 1525 else 1526 ntb_reg_write(2, ntb->reg_ofs.rdb, 1527 ((1 << ntb->bits_per_vector) - 1) << 1528 (db * ntb->bits_per_vector)); 1529 } 1530} 1531 1532/** 1533 * ntb_query_link_status() - return the hardware link status 1534 * @ndev: pointer to ntb_device instance 1535 * 1536 * Returns true if the hardware is connected to the remote system 1537 * 1538 * RETURNS: true or false based on the hardware link state 1539 */ 1540bool 1541ntb_query_link_status(struct ntb_softc *ntb) 1542{ 1543 1544 return (ntb->link_status == NTB_LINK_UP); 1545} 1546 1547static void 1548save_bar_parameters(struct ntb_pci_bar_info *bar) 1549{ 1550 1551 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 1552 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 1553 bar->pbase = rman_get_start(bar->pci_resource); 1554 bar->size = rman_get_size(bar->pci_resource); 1555 bar->vbase = rman_get_virtual(bar->pci_resource); 1556} 1557 1558device_t 1559ntb_get_device(struct ntb_softc *ntb) 1560{ 1561 1562 return (ntb->device); 1563} 1564 1565/* Export HW-specific errata information. */ 1566bool 1567ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 1568{ 1569 1570 return (HAS_FEATURE(feature)); 1571} 1572