ntb_hw_intel.c revision 289542
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * Copyright (C) 2015 EMC Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289542 2015-10-18 20:20:20Z cem $"); 30 31#include <sys/param.h> 32#include <sys/kernel.h> 33#include <sys/systm.h> 34#include <sys/bus.h> 35#include <sys/malloc.h> 36#include <sys/module.h> 37#include <sys/queue.h> 38#include <sys/rman.h> 39#include <sys/sysctl.h> 40#include <vm/vm.h> 41#include <vm/pmap.h> 42#include <machine/bus.h> 43#include <machine/pmap.h> 44#include <machine/resource.h> 45#include <dev/pci/pcireg.h> 46#include <dev/pci/pcivar.h> 47 48#include "ntb_regs.h" 49#include "ntb_hw.h" 50 51/* 52 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 53 * allows you to connect two systems using a PCI-e link. 54 * 55 * This module contains the hardware abstraction layer for the NTB. It allows 56 * you to send and recieve interrupts, map the memory windows and send and 57 * receive messages in the scratch-pad registers. 58 * 59 * NOTE: Much of the code in this module is shared with Linux. Any patches may 60 * be picked up and redistributed in Linux with a dual GPL/BSD license. 61 */ 62 63#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT) 64 65#define NTB_HB_TIMEOUT 1 /* second */ 66#define SOC_LINK_RECOVERY_TIME 500 /* ms */ 67 68#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 69 70enum ntb_device_type { 71 NTB_XEON, 72 NTB_SOC 73}; 74 75enum ntb_bar { 76 NTB_CONFIG_BAR = 0, 77 NTB_B2B_BAR_1, 78 NTB_B2B_BAR_2, 79 NTB_B2B_BAR_3, 80 NTB_MAX_BARS 81}; 82 83/* Device features and workarounds */ 84#define HAS_FEATURE(feature) \ 85 ((ntb->features & (feature)) != 0) 86 87struct ntb_hw_info { 88 uint32_t device_id; 89 const char *desc; 90 enum ntb_device_type type; 91 uint32_t features; 92}; 93 94struct ntb_pci_bar_info { 95 bus_space_tag_t pci_bus_tag; 96 bus_space_handle_t pci_bus_handle; 97 int pci_resource_id; 98 struct resource *pci_resource; 99 vm_paddr_t pbase; 100 void *vbase; 101 u_long size; 102}; 103 104struct ntb_int_info { 105 struct resource *res; 106 int rid; 107 void *tag; 108}; 109 110struct ntb_db_cb { 111 ntb_db_callback callback; 112 unsigned int db_num; 113 void *data; 114 struct ntb_softc *ntb; 115 struct callout irq_work; 116 bool reserved; 117}; 118 119struct ntb_reg { 120 uint32_t ntb_ctl; 121 uint32_t lnk_sta; 122 uint8_t db_size; 123 unsigned mw_bar[NTB_MAX_BARS]; 124}; 125 126struct ntb_alt_reg { 127 uint32_t db_bell; 128 uint32_t db_mask; 129 uint32_t spad; 130}; 131 132struct ntb_xlat_reg { 133 uint64_t bar0_base; 134 uint64_t bar2_xlat; 135 uint64_t bar2_limit; 136}; 137 138struct ntb_b2b_addr { 139 uint64_t bar0_addr; 140 uint64_t bar2_addr64; 141 uint64_t bar4_addr64; 142 uint64_t bar4_addr32; 143 uint64_t bar5_addr32; 144}; 145 146struct ntb_softc { 147 device_t device; 148 enum ntb_device_type type; 149 uint64_t features; 150 151 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 152 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 153 uint32_t allocated_interrupts; 154 155 struct callout heartbeat_timer; 156 struct callout lr_timer; 157 158 void *ntb_transport; 159 ntb_event_callback event_cb; 160 struct ntb_db_cb *db_cb; 161 uint8_t max_cbs; 162 163 struct { 164 uint32_t ldb; 165 uint32_t ldb_mask; 166 uint32_t bar4_xlat; 167 uint32_t bar5_xlat; 168 uint32_t spad_local; 169 uint32_t spci_cmd; 170 } reg_ofs; 171 uint32_t ppd; 172 uint8_t conn_type; 173 uint8_t dev_type; 174 uint8_t link_status; 175 uint8_t link_width; 176 uint8_t link_speed; 177 178 /* Offset of peer bar0 in B2B BAR */ 179 uint64_t b2b_off; 180 /* Memory window used to access peer bar0 */ 181 uint8_t b2b_mw_idx; 182 183 uint8_t mw_count; 184 uint8_t spad_count; 185 uint8_t db_count; 186 uint8_t db_vec_count; 187 uint8_t db_vec_shift; 188 189 /* Protects local DB mask and (h). */ 190#define HW_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 191#define HW_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 192#define HW_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 193 struct mtx db_mask_lock; 194 195 uint32_t ntb_ctl; /* (h) - SOC only */ 196 uint32_t lnk_sta; /* (h) - SOC only */ 197 198 uint64_t db_valid_mask; 199 uint64_t db_link_mask; 200 uint64_t db_mask; /* (h) */ 201 202 int last_ts; /* ticks @ last irq */ 203 204 const struct ntb_reg *reg; 205 const struct ntb_alt_reg *self_reg; 206 const struct ntb_alt_reg *peer_reg; 207 const struct ntb_xlat_reg *xlat_reg; 208}; 209 210#ifdef __i386__ 211static __inline uint64_t 212bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 213 bus_size_t offset) 214{ 215 216 return (bus_space_read_4(tag, handle, offset) | 217 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 218} 219 220static __inline void 221bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 222 bus_size_t offset, uint64_t val) 223{ 224 225 bus_space_write_4(tag, handle, offset, val); 226 bus_space_write_4(tag, handle, offset + 4, val >> 32); 227} 228#endif 229 230#define ntb_bar_read(SIZE, bar, offset) \ 231 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 232 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 233#define ntb_bar_write(SIZE, bar, offset, val) \ 234 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 235 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 236#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 237#define ntb_reg_write(SIZE, offset, val) \ 238 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 239#define ntb_mw_read(SIZE, offset) \ 240 ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 241#define ntb_mw_write(SIZE, offset, val) \ 242 ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 243 offset, val) 244 245static int ntb_probe(device_t device); 246static int ntb_attach(device_t device); 247static int ntb_detach(device_t device); 248static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 249static int ntb_map_pci_bars(struct ntb_softc *ntb); 250static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *); 251static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 252static int map_memory_window_bar(struct ntb_softc *ntb, 253 struct ntb_pci_bar_info *bar); 254static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 255static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 256static int ntb_init_isr(struct ntb_softc *ntb); 257static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 258static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 259static void ntb_teardown_interrupts(struct ntb_softc *ntb); 260static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 261static void handle_irq(void *arg); 262static void ntb_handle_legacy_interrupt(void *arg); 263static void ntb_irq_work(void *arg); 264static inline uint64_t ntb_db_read(struct ntb_softc *, uint64_t regoff); 265static inline void ntb_db_write(struct ntb_softc *, uint64_t regoff, uint64_t val); 266static inline void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 267static inline void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); 268static inline void ntb_db_set_mask(struct ntb_softc *, uint64_t bits); 269static inline void ntb_db_clear_mask(struct ntb_softc *, uint64_t bits); 270static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors); 271static void ntb_free_callbacks(struct ntb_softc *ntb); 272static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 273static void ntb_detect_max_mw(struct ntb_softc *ntb); 274static int ntb_detect_xeon(struct ntb_softc *ntb); 275static int ntb_detect_soc(struct ntb_softc *ntb); 276static int ntb_xeon_init_dev(struct ntb_softc *ntb); 277static int ntb_soc_init_dev(struct ntb_softc *ntb); 278static void ntb_teardown_xeon(struct ntb_softc *ntb); 279static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); 280static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); 281static int xeon_setup_b2b_mw(struct ntb_softc *, 282 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 283static void soc_link_hb(void *arg); 284static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); 285static void ntb_link_disable(struct ntb_softc *ntb); 286static void ntb_link_enable(struct ntb_softc *ntb); 287static void recover_soc_link(void *arg); 288static int ntb_poll_link(struct ntb_softc *ntb); 289static void save_bar_parameters(struct ntb_pci_bar_info *bar); 290 291static struct ntb_hw_info pci_ids[] = { 292 { 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 }, 293 294 /* XXX: PS/SS IDs left out until they are supported. */ 295 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 296 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 297 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 298 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 299 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 300 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 301 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 302 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 303 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 304 NTB_SB01BASE_LOCKUP }, 305 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 306 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 307 NTB_SB01BASE_LOCKUP }, 308 309 { 0x00000000, NULL, NTB_SOC, 0 } 310}; 311 312static const struct ntb_reg soc_reg = { 313 .ntb_ctl = SOC_NTBCNTL_OFFSET, 314 .lnk_sta = SOC_LINK_STATUS_OFFSET, 315 .db_size = sizeof(uint64_t), 316 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 317}; 318 319static const struct ntb_alt_reg soc_b2b_reg = { 320 .db_bell = SOC_B2B_DOORBELL_OFFSET, 321 .spad = SOC_B2B_SPAD_OFFSET, 322}; 323 324static const struct ntb_xlat_reg soc_sec_xlat = { 325#if 0 326 /* "FIXME" says the Linux driver. */ 327 .bar0_base = SOC_SBAR0BASE_OFFSET, 328 .bar2_limit = SOC_SBAR2LMT_OFFSET, 329#endif 330 .bar2_xlat = SOC_SBAR2XLAT_OFFSET, 331}; 332 333static const struct ntb_reg xeon_reg = { 334 .ntb_ctl = XEON_NTBCNTL_OFFSET, 335 .lnk_sta = XEON_LINK_STATUS_OFFSET, 336 .db_size = sizeof(uint16_t), 337 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 338}; 339 340static const struct ntb_alt_reg xeon_b2b_reg = { 341 .db_bell = XEON_B2B_DOORBELL_OFFSET, 342 .spad = XEON_B2B_SPAD_OFFSET, 343}; 344 345static const struct ntb_xlat_reg xeon_sec_xlat = { 346 .bar0_base = XEON_SBAR0BASE_OFFSET, 347 .bar2_limit = XEON_SBAR2LMT_OFFSET, 348 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 349}; 350 351static const struct ntb_b2b_addr xeon_b2b_usd_addr = { 352 .bar0_addr = XEON_B2B_BAR0_USD_ADDR, 353 .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, 354 .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, 355 .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, 356 .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, 357}; 358 359static const struct ntb_b2b_addr xeon_b2b_dsd_addr = { 360 .bar0_addr = XEON_B2B_BAR0_DSD_ADDR, 361 .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, 362 .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, 363 .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, 364 .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, 365}; 366 367/* 368 * OS <-> Driver interface structures 369 */ 370MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 371 372static device_method_t ntb_pci_methods[] = { 373 /* Device interface */ 374 DEVMETHOD(device_probe, ntb_probe), 375 DEVMETHOD(device_attach, ntb_attach), 376 DEVMETHOD(device_detach, ntb_detach), 377 DEVMETHOD_END 378}; 379 380static driver_t ntb_pci_driver = { 381 "ntb_hw", 382 ntb_pci_methods, 383 sizeof(struct ntb_softc), 384}; 385 386static devclass_t ntb_devclass; 387DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 388MODULE_VERSION(ntb_hw, 1); 389 390SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 391 392/* 393 * OS <-> Driver linkage functions 394 */ 395static int 396ntb_probe(device_t device) 397{ 398 struct ntb_hw_info *p; 399 400 p = ntb_get_device_info(pci_get_devid(device)); 401 if (p == NULL) 402 return (ENXIO); 403 404 device_set_desc(device, p->desc); 405 return (0); 406} 407 408static int 409ntb_attach(device_t device) 410{ 411 struct ntb_softc *ntb; 412 struct ntb_hw_info *p; 413 int error; 414 415 ntb = DEVICE2SOFTC(device); 416 p = ntb_get_device_info(pci_get_devid(device)); 417 418 ntb->device = device; 419 ntb->type = p->type; 420 ntb->features = p->features; 421 ntb->b2b_mw_idx = UINT8_MAX; 422 423 /* Heartbeat timer for NTB_SOC since there is no link interrupt */ 424 callout_init(&ntb->heartbeat_timer, 1); 425 callout_init(&ntb->lr_timer, 1); 426 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 427 428 if (ntb->type == NTB_SOC) 429 error = ntb_detect_soc(ntb); 430 else 431 error = ntb_detect_xeon(ntb); 432 if (error) 433 goto out; 434 435 ntb_detect_max_mw(ntb); 436 437 error = ntb_map_pci_bars(ntb); 438 if (error) 439 goto out; 440 if (ntb->type == NTB_SOC) 441 error = ntb_soc_init_dev(ntb); 442 else 443 error = ntb_xeon_init_dev(ntb); 444 if (error) 445 goto out; 446 error = ntb_init_isr(ntb); 447 if (error) 448 goto out; 449 450 pci_enable_busmaster(ntb->device); 451 452out: 453 if (error != 0) 454 ntb_detach(device); 455 return (error); 456} 457 458static int 459ntb_detach(device_t device) 460{ 461 struct ntb_softc *ntb; 462 463 ntb = DEVICE2SOFTC(device); 464 465 ntb_db_set_mask(ntb, ntb->db_valid_mask); 466 callout_drain(&ntb->heartbeat_timer); 467 callout_drain(&ntb->lr_timer); 468 if (ntb->type == NTB_XEON) 469 ntb_teardown_xeon(ntb); 470 ntb_teardown_interrupts(ntb); 471 472 mtx_destroy(&ntb->db_mask_lock); 473 474 /* 475 * Redetect total MWs so we unmap properly -- in case we lowered the 476 * maximum to work around Xeon errata. 477 */ 478 ntb_detect_max_mw(ntb); 479 ntb_unmap_pci_bar(ntb); 480 481 return (0); 482} 483 484/* 485 * Driver internal routines 486 */ 487static inline enum ntb_bar 488ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 489{ 490 491 KASSERT(mw < ntb->mw_count || (mw != UINT8_MAX && mw == ntb->b2b_mw_idx), 492 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 493 494 return (ntb->reg->mw_bar[mw]); 495} 496 497static int 498ntb_map_pci_bars(struct ntb_softc *ntb) 499{ 500 int rc; 501 502 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 503 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 504 if (rc != 0) 505 goto out; 506 507 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 508 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 509 if (rc != 0) 510 goto out; 511 512 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 513 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR)) 514 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 515 else 516 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 517 if (!HAS_FEATURE(NTB_SPLIT_BAR)) 518 goto out; 519 520 ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 521 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 522 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 523 else 524 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 525 526out: 527 if (rc != 0) 528 device_printf(ntb->device, 529 "unable to allocate pci resource\n"); 530 return (rc); 531} 532 533static void 534print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 535{ 536 537 device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n", 538 bar->size, bar->vbase, (void *)(bar->pbase)); 539} 540 541static int 542map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 543{ 544 545 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 546 &bar->pci_resource_id, RF_ACTIVE); 547 if (bar->pci_resource == NULL) 548 return (ENXIO); 549 550 save_bar_parameters(bar); 551 print_map_success(ntb, bar); 552 return (0); 553} 554 555static int 556map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 557{ 558 int rc; 559 uint8_t bar_size_bits = 0; 560 561 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 562 &bar->pci_resource_id, RF_ACTIVE); 563 564 if (bar->pci_resource == NULL) 565 return (ENXIO); 566 567 save_bar_parameters(bar); 568 /* 569 * Ivytown NTB BAR sizes are misreported by the hardware due to a 570 * hardware issue. To work around this, query the size it should be 571 * configured to by the device and modify the resource to correspond to 572 * this new size. The BIOS on systems with this problem is required to 573 * provide enough address space to allow the driver to make this change 574 * safely. 575 * 576 * Ideally I could have just specified the size when I allocated the 577 * resource like: 578 * bus_alloc_resource(ntb->device, 579 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 580 * 1ul << bar_size_bits, RF_ACTIVE); 581 * but the PCI driver does not honor the size in this call, so we have 582 * to modify it after the fact. 583 */ 584 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 585 if (bar->pci_resource_id == PCIR_BAR(2)) 586 bar_size_bits = pci_read_config(ntb->device, 587 XEON_PBAR23SZ_OFFSET, 1); 588 else 589 bar_size_bits = pci_read_config(ntb->device, 590 XEON_PBAR45SZ_OFFSET, 1); 591 592 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 593 bar->pci_resource, bar->pbase, 594 bar->pbase + (1ul << bar_size_bits) - 1); 595 if (rc != 0) { 596 device_printf(ntb->device, 597 "unable to resize bar\n"); 598 return (rc); 599 } 600 601 save_bar_parameters(bar); 602 } 603 604 /* Mark bar region as write combining to improve performance. */ 605 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, 606 VM_MEMATTR_WRITE_COMBINING); 607 if (rc != 0) { 608 device_printf(ntb->device, 609 "unable to mark bar as WRITE_COMBINING\n"); 610 return (rc); 611 } 612 print_map_success(ntb, bar); 613 return (0); 614} 615 616static void 617ntb_unmap_pci_bar(struct ntb_softc *ntb) 618{ 619 struct ntb_pci_bar_info *current_bar; 620 int i; 621 622 for (i = 0; i < NTB_MAX_BARS; i++) { 623 current_bar = &ntb->bar_info[i]; 624 if (current_bar->pci_resource != NULL) 625 bus_release_resource(ntb->device, SYS_RES_MEMORY, 626 current_bar->pci_resource_id, 627 current_bar->pci_resource); 628 } 629} 630 631static int 632ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 633{ 634 uint32_t i; 635 int rc; 636 637 for (i = 0; i < num_vectors; i++) { 638 ntb->int_info[i].rid = i + 1; 639 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 640 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 641 if (ntb->int_info[i].res == NULL) { 642 device_printf(ntb->device, 643 "bus_alloc_resource failed\n"); 644 return (ENOMEM); 645 } 646 ntb->int_info[i].tag = NULL; 647 ntb->allocated_interrupts++; 648 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 649 INTR_MPSAFE | INTR_TYPE_MISC, NULL, handle_irq, 650 &ntb->db_cb[i], &ntb->int_info[i].tag); 651 if (rc != 0) { 652 device_printf(ntb->device, "bus_setup_intr failed\n"); 653 return (ENXIO); 654 } 655 } 656 return (0); 657} 658 659/* 660 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 661 * cannot be allocated for each MSI-X message. JHB seems to think remapping 662 * should be okay. This tunable should enable us to test that hypothesis 663 * when someone gets their hands on some Xeon hardware. 664 */ 665static int ntb_force_remap_mode; 666SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 667 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 668 " to a smaller number of ithreads, even if the desired number are " 669 "available"); 670 671/* 672 * In case it is NOT ok, give consumers an abort button. 673 */ 674static int ntb_prefer_intx; 675SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 676 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 677 "than remapping MSI-X messages over available slots (match Linux driver " 678 "behavior)"); 679 680/* 681 * Remap the desired number of MSI-X messages to available ithreads in a simple 682 * round-robin fashion. 683 */ 684static int 685ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 686{ 687 u_int *vectors; 688 uint32_t i; 689 int rc; 690 691 if (ntb_prefer_intx != 0) 692 return (ENXIO); 693 694 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 695 696 for (i = 0; i < desired; i++) 697 vectors[i] = (i % avail) + 1; 698 699 rc = pci_remap_msix(dev, desired, vectors); 700 free(vectors, M_NTB); 701 return (rc); 702} 703 704static int 705ntb_init_isr(struct ntb_softc *ntb) 706{ 707 uint32_t desired_vectors, num_vectors; 708 uint64_t mask; 709 int rc; 710 711 ntb->allocated_interrupts = 0; 712 ntb->last_ts = ticks; 713 714 /* 715 * On SOC, disable all interrupts. On XEON, disable all but Link 716 * Interrupt. The rest will be unmasked as callbacks are registered. 717 */ 718 mask = ntb->db_valid_mask; 719 if (ntb->type == NTB_XEON) 720 mask &= ~ntb->db_link_mask; 721 ntb_db_set_mask(ntb, mask); 722 723 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 724 ntb->db_count); 725 if (desired_vectors >= 1) { 726 rc = pci_alloc_msix(ntb->device, &num_vectors); 727 728 if (ntb_force_remap_mode != 0 && rc == 0 && 729 num_vectors == desired_vectors) 730 num_vectors--; 731 732 if (rc == 0 && num_vectors < desired_vectors) { 733 rc = ntb_remap_msix(ntb->device, desired_vectors, 734 num_vectors); 735 if (rc == 0) 736 num_vectors = desired_vectors; 737 else 738 pci_release_msi(ntb->device); 739 } 740 if (rc != 0) 741 num_vectors = 1; 742 } else 743 num_vectors = 1; 744 745 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 746 /* 747 * If allocating MSI-X interrupts failed and we're forced to 748 * use legacy INTx anyway, the only limit on individual 749 * callbacks is the number of doorbell bits. 750 */ 751 ntb->db_vec_count = 1; 752 ntb->db_vec_shift = ntb->db_count; 753 ntb_create_callbacks(ntb, ntb->db_count); 754 rc = ntb_setup_legacy_interrupt(ntb); 755 } else { 756 ntb_create_callbacks(ntb, num_vectors); 757 rc = ntb_setup_msix(ntb, num_vectors); 758 if (rc == 0 && ntb->type == NTB_XEON) { 759 /* 760 * Prevent consumers from registering callbacks on the link event irq 761 * slot, from which they will never be called back. 762 */ 763 ntb->db_cb[num_vectors - 1].reserved = true; 764 ntb->max_cbs--; 765 } 766 } 767 if (rc != 0) { 768 device_printf(ntb->device, 769 "Error allocating interrupts: %d\n", rc); 770 ntb_free_callbacks(ntb); 771 } 772 773 return (rc); 774} 775 776static int 777ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 778{ 779 int rc; 780 781 ntb->int_info[0].rid = 0; 782 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 783 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 784 if (ntb->int_info[0].res == NULL) { 785 device_printf(ntb->device, "bus_alloc_resource failed\n"); 786 return (ENOMEM); 787 } 788 789 ntb->int_info[0].tag = NULL; 790 ntb->allocated_interrupts = 1; 791 792 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 793 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ntb_handle_legacy_interrupt, 794 ntb, &ntb->int_info[0].tag); 795 if (rc != 0) { 796 device_printf(ntb->device, "bus_setup_intr failed\n"); 797 return (ENXIO); 798 } 799 800 return (0); 801} 802 803static void 804ntb_teardown_interrupts(struct ntb_softc *ntb) 805{ 806 struct ntb_int_info *current_int; 807 int i; 808 809 for (i = 0; i < ntb->allocated_interrupts; i++) { 810 current_int = &ntb->int_info[i]; 811 if (current_int->tag != NULL) 812 bus_teardown_intr(ntb->device, current_int->res, 813 current_int->tag); 814 815 if (current_int->res != NULL) 816 bus_release_resource(ntb->device, SYS_RES_IRQ, 817 rman_get_rid(current_int->res), current_int->res); 818 } 819 820 ntb_free_callbacks(ntb); 821 pci_release_msi(ntb->device); 822} 823 824/* 825 * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon. Abstract it 826 * out to make code clearer. 827 */ 828static inline uint64_t 829ntb_db_read(struct ntb_softc *ntb, uint64_t regoff) 830{ 831 832 if (ntb->type == NTB_SOC) 833 return (ntb_reg_read(8, regoff)); 834 835 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 836 837 return (ntb_reg_read(2, regoff)); 838} 839 840static inline void 841ntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 842{ 843 844 KASSERT((val & ~ntb->db_valid_mask) == 0, 845 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 846 (uintmax_t)(val & ~ntb->db_valid_mask), 847 (uintmax_t)ntb->db_valid_mask)); 848 849 if (regoff == ntb->reg_ofs.ldb_mask) 850 HW_ASSERT(ntb, MA_OWNED); 851 852 if (ntb->type == NTB_SOC) { 853 ntb_reg_write(8, regoff, val); 854 return; 855 } 856 857 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 858 ntb_reg_write(2, regoff, (uint16_t)val); 859} 860 861static inline void 862ntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 863{ 864 865 HW_LOCK(ntb); 866 ntb->db_mask |= bits; 867 ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask); 868 HW_UNLOCK(ntb); 869} 870 871static inline void 872ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 873{ 874 875 KASSERT((bits & ~ntb->db_valid_mask) == 0, 876 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 877 (uintmax_t)(bits & ~ntb->db_valid_mask), 878 (uintmax_t)ntb->db_valid_mask)); 879 880 HW_LOCK(ntb); 881 ntb->db_mask &= ~bits; 882 ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask); 883 HW_UNLOCK(ntb); 884} 885 886static inline void 887mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 888{ 889 uint64_t mask; 890 891 mask = 1ull << (idx * ntb->db_vec_shift); 892 ntb_db_set_mask(ntb, mask); 893} 894 895static inline void 896unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) 897{ 898 uint64_t mask; 899 900 mask = 1ull << (idx * ntb->db_vec_shift); 901 ntb_db_clear_mask(ntb, mask); 902} 903 904static inline uint64_t 905ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 906{ 907 uint64_t shift, mask; 908 909 shift = ntb->db_vec_shift; 910 mask = (1ull << shift) - 1; 911 return (mask << (shift * db_vector)); 912} 913 914static void 915handle_irq(void *arg) 916{ 917 struct ntb_db_cb *db_cb = arg; 918 struct ntb_softc *ntb = db_cb->ntb; 919 uint64_t vec_mask; 920 int rc; 921 922 ntb->last_ts = ticks; 923 vec_mask = ntb_vec_mask(ntb, db_cb->db_num); 924 925 if ((vec_mask & ntb->db_link_mask) != 0) { 926 rc = ntb_poll_link(ntb); 927 if (rc != 0) 928 device_printf(ntb->device, 929 "Error determining link status\n"); 930 } 931 932 if (db_cb->callback != NULL) { 933 KASSERT(!db_cb->reserved, ("user callback on link event cb")); 934 mask_ldb_interrupt(ntb, db_cb->db_num); 935 } 936 937 ntb_db_write(ntb, ntb->reg_ofs.ldb, vec_mask); 938 939 if (db_cb->callback != NULL) 940 callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 941} 942 943static void 944ntb_handle_legacy_interrupt(void *arg) 945{ 946 struct ntb_softc *ntb = arg; 947 unsigned int i; 948 uint64_t ldb; 949 950 ldb = ntb_db_read(ntb, ntb->reg_ofs.ldb); 951 while (ldb != 0) { 952 i = ffs(ldb); 953 ldb &= ldb - 1; 954 handle_irq(&ntb->db_cb[i]); 955 } 956} 957 958static int 959ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors) 960{ 961 uint32_t i; 962 963 ntb->max_cbs = num_vectors; 964 ntb->db_cb = malloc(num_vectors * sizeof(*ntb->db_cb), M_NTB, 965 M_ZERO | M_WAITOK); 966 for (i = 0; i < num_vectors; i++) { 967 ntb->db_cb[i].db_num = i; 968 ntb->db_cb[i].ntb = ntb; 969 } 970 971 return (0); 972} 973 974static void 975ntb_free_callbacks(struct ntb_softc *ntb) 976{ 977 uint8_t i; 978 979 if (ntb->db_cb == NULL) 980 return; 981 982 for (i = 0; i < ntb->max_cbs; i++) 983 ntb_unregister_db_callback(ntb, i); 984 985 free(ntb->db_cb, M_NTB); 986 ntb->db_cb = NULL; 987 ntb->max_cbs = 0; 988} 989 990static struct ntb_hw_info * 991ntb_get_device_info(uint32_t device_id) 992{ 993 struct ntb_hw_info *ep = pci_ids; 994 995 while (ep->device_id) { 996 if (ep->device_id == device_id) 997 return (ep); 998 ++ep; 999 } 1000 return (NULL); 1001} 1002 1003static void 1004ntb_teardown_xeon(struct ntb_softc *ntb) 1005{ 1006 1007 ntb_link_disable(ntb); 1008} 1009 1010static void 1011ntb_detect_max_mw(struct ntb_softc *ntb) 1012{ 1013 1014 if (ntb->type == NTB_SOC) { 1015 ntb->mw_count = SOC_MW_COUNT; 1016 return; 1017 } 1018 1019 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1020 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1021 else 1022 ntb->mw_count = XEON_SNB_MW_COUNT; 1023} 1024 1025static int 1026ntb_detect_xeon(struct ntb_softc *ntb) 1027{ 1028 uint8_t ppd, conn_type; 1029 1030 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1031 ntb->ppd = ppd; 1032 1033 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1034 ntb->dev_type = NTB_DEV_USD; 1035 else 1036 ntb->dev_type = NTB_DEV_DSD; 1037 1038 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1039 ntb->features |= NTB_SPLIT_BAR; 1040 1041 /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ 1042 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1043 ntb->features |= NTB_SDOORBELL_LOCKUP; 1044 1045 conn_type = ppd & XEON_PPD_CONN_TYPE; 1046 switch (conn_type) { 1047 case NTB_CONN_B2B: 1048 ntb->conn_type = conn_type; 1049 break; 1050 case NTB_CONN_RP: 1051 case NTB_CONN_TRANSPARENT: 1052 default: 1053 device_printf(ntb->device, "Unsupported connection type: %u\n", 1054 (unsigned)conn_type); 1055 return (ENXIO); 1056 } 1057 return (0); 1058} 1059 1060static int 1061ntb_detect_soc(struct ntb_softc *ntb) 1062{ 1063 uint32_t ppd, conn_type; 1064 1065 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1066 ntb->ppd = ppd; 1067 1068 if ((ppd & SOC_PPD_DEV_TYPE) != 0) 1069 ntb->dev_type = NTB_DEV_DSD; 1070 else 1071 ntb->dev_type = NTB_DEV_USD; 1072 1073 conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8; 1074 switch (conn_type) { 1075 case NTB_CONN_B2B: 1076 ntb->conn_type = conn_type; 1077 break; 1078 default: 1079 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1080 return (ENXIO); 1081 } 1082 return (0); 1083} 1084 1085static int 1086ntb_xeon_init_dev(struct ntb_softc *ntb) 1087{ 1088 int rc; 1089 1090 ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; 1091 ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; 1092 ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; 1093 ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; 1094 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1095 ntb->reg_ofs.bar5_xlat = XEON_SBAR5XLAT_OFFSET; 1096 ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; 1097 1098 ntb->spad_count = XEON_SPAD_COUNT; 1099 ntb->db_count = XEON_DB_COUNT; 1100 ntb->db_link_mask = XEON_DB_LINK_BIT; 1101 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1102 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1103 1104 if (ntb->conn_type != NTB_CONN_B2B) { 1105 device_printf(ntb->device, "Connection type %d not supported\n", 1106 ntb->conn_type); 1107 return (ENXIO); 1108 } 1109 1110 ntb->reg = &xeon_reg; 1111 ntb->peer_reg = &xeon_b2b_reg; 1112 ntb->xlat_reg = &xeon_sec_xlat; 1113 1114 /* 1115 * There is a Xeon hardware errata related to writes to SDOORBELL or 1116 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1117 * which may hang the system. To workaround this use the second memory 1118 * window to access the interrupt and scratch pad registers on the 1119 * remote system. 1120 * 1121 * There is another HW errata on the limit registers -- they can only 1122 * be written when the base register is (?)4GB aligned and < 32-bit. 1123 * This should already be the case based on the driver defaults, but 1124 * write the limit registers first just in case. 1125 */ 1126 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1127 /* Reserve the last MW for mapping remote spad */ 1128 ntb->b2b_mw_idx = ntb->mw_count - 1; 1129 ntb->mw_count--; 1130 /* 1131 * Set the Limit register to 4k, the minimum size, to prevent 1132 * an illegal access. 1133 */ 1134 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1135 ntb_reg_write(4, XEON_PBAR4LMT_OFFSET, 0); 1136 ntb_reg_write(4, XEON_PBAR5LMT_OFFSET, 1137 ntb_get_mw_size(ntb, ntb->b2b_mw_idx) + 0x1000); 1138 } else 1139 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 1140 ntb_get_mw_size(ntb, ntb->b2b_mw_idx) + 0x1000); 1141 } else { 1142 /* 1143 * Disable the limit register, just in case it is set to 1144 * something silly. A 64-bit write will also clear PBAR5LMT in 1145 * split-bar mode, and this is desired. 1146 */ 1147 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1148 1149 /* 1150 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1151 * mirrored to the remote system. Shrink the number of bits by one, 1152 * since bit 14 is the last bit. 1153 * 1154 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1155 * anyway. Nor for non-B2B connection types. 1156 */ 1157 if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1158 ntb->db_count = XEON_DB_COUNT - 1; 1159 } 1160 1161 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1162 1163 if (ntb->dev_type == NTB_DEV_USD) 1164 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1165 &xeon_b2b_usd_addr); 1166 else 1167 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1168 &xeon_b2b_dsd_addr); 1169 if (rc != 0) 1170 return (rc); 1171 1172 /* Enable Bus Master and Memory Space on the secondary side */ 1173 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 1174 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1175 1176 /* Enable link training */ 1177 ntb_link_enable(ntb); 1178 1179 return (0); 1180} 1181 1182static int 1183ntb_soc_init_dev(struct ntb_softc *ntb) 1184{ 1185 1186 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1187 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1188 1189 ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; 1190 ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; 1191 ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; 1192 ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; 1193 ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; 1194 1195 ntb->spad_count = SOC_SPAD_COUNT; 1196 ntb->db_count = SOC_DB_COUNT; 1197 ntb->db_vec_count = SOC_DB_MSIX_VECTOR_COUNT; 1198 ntb->db_vec_shift = SOC_DB_MSIX_VECTOR_SHIFT; 1199 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1200 1201 ntb->reg = &soc_reg; 1202 ntb->peer_reg = &soc_b2b_reg; 1203 ntb->xlat_reg = &soc_sec_xlat; 1204 1205 /* 1206 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is 1207 * resolved. Mask transaction layer internal parity errors. 1208 */ 1209 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1210 1211 configure_soc_secondary_side_bars(ntb); 1212 1213 /* Enable Bus Master and Memory Space on the secondary side */ 1214 ntb_reg_write(2, ntb->reg_ofs.spci_cmd, 1215 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1216 1217 /* Initiate PCI-E link training */ 1218 ntb_link_enable(ntb); 1219 1220 callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb); 1221 1222 return (0); 1223} 1224 1225/* XXX: Linux driver doesn't seem to do any of this for SoC. */ 1226static void 1227configure_soc_secondary_side_bars(struct ntb_softc *ntb) 1228{ 1229 1230 if (ntb->dev_type == NTB_DEV_USD) { 1231 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1232 XEON_B2B_BAR2_DSD_ADDR64); 1233 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1234 XEON_B2B_BAR4_DSD_ADDR64); 1235 ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64); 1236 ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64); 1237 } else { 1238 ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, 1239 XEON_B2B_BAR2_USD_ADDR64); 1240 ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, 1241 XEON_B2B_BAR4_USD_ADDR64); 1242 ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64); 1243 ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64); 1244 } 1245} 1246 1247static int 1248xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1249 const struct ntb_b2b_addr *peer_addr) 1250{ 1251 1252 /* Local addresses */ 1253 ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, addr->bar2_addr64); 1254 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1255 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, addr->bar0_addr); 1256 else { 1257 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1258 ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET, 1259 addr->bar4_addr32); 1260 ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET, 1261 addr->bar5_addr32); 1262 } else 1263 ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, 1264 addr->bar4_addr64); 1265 /* 1266 * B2B_XLAT_OFFSET is a 64-bit register but can only be 1267 * written 32 bits at a time. 1268 */ 1269 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, 1270 addr->bar0_addr & 0xffffffff); 1271 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, addr->bar0_addr >> 32); 1272 } 1273 1274 /* Peer addresses */ 1275 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, peer_addr->bar0_addr); 1276 ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, peer_addr->bar2_addr64); 1277 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1278 ntb_reg_write(4, XEON_SBAR4BASE_OFFSET, 1279 peer_addr->bar4_addr32); 1280 ntb_reg_write(4, XEON_SBAR5BASE_OFFSET, 1281 peer_addr->bar5_addr32); 1282 } else 1283 ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, 1284 peer_addr->bar4_addr64); 1285 return (0); 1286} 1287 1288/* SOC does not have link status interrupt, poll on that platform */ 1289static void 1290soc_link_hb(void *arg) 1291{ 1292 struct ntb_softc *ntb = arg; 1293 uint32_t status32; 1294 int rc; 1295 1296 /* 1297 * Delay polling the link status if an interrupt was received, unless 1298 * the cached link status says the link is down. 1299 */ 1300 if ((long)ticks - ((long)ntb->last_ts + NTB_HB_TIMEOUT * hz) < 0 && 1301 (ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0) 1302 goto out; 1303 1304 1305 rc = ntb_poll_link(ntb); 1306 if (rc != 0) 1307 device_printf(ntb->device, 1308 "Error determining link status\n"); 1309 1310 /* Check to see if a link error is the cause of the link down */ 1311 if (ntb->link_status == NTB_LINK_DOWN) { 1312 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1313 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) { 1314 callout_reset(&ntb->lr_timer, 0, recover_soc_link, 1315 ntb); 1316 return; 1317 } 1318 } 1319 1320out: 1321 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, 1322 ntb); 1323} 1324 1325static void 1326soc_perform_link_restart(struct ntb_softc *ntb) 1327{ 1328 uint32_t status; 1329 1330 /* Driver resets the NTB ModPhy lanes - magic! */ 1331 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0); 1332 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40); 1333 ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60); 1334 ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60); 1335 1336 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1337 pause("ModPhy", hz / 10); 1338 1339 /* Clear AER Errors, write to clear */ 1340 status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET); 1341 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1342 ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status); 1343 1344 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1345 status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET); 1346 status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI; 1347 ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status); 1348 1349 /* Clear DeSkew Buffer error, write to clear */ 1350 status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET); 1351 status |= SOC_DESKEWSTS_DBERR; 1352 ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status); 1353 1354 status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1355 status &= SOC_IBIST_ERR_OFLOW; 1356 ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status); 1357 1358 /* Releases the NTB state machine to allow the link to retrain */ 1359 status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1360 status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT; 1361 ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status); 1362} 1363 1364static void 1365ntb_handle_link_event(struct ntb_softc *ntb, int link_state) 1366{ 1367 enum ntb_hw_event event; 1368 uint16_t status; 1369 1370 if (ntb->link_status == link_state) 1371 return; 1372 1373 if (link_state == NTB_LINK_UP) { 1374 device_printf(ntb->device, "Link Up\n"); 1375 ntb->link_status = NTB_LINK_UP; 1376 event = NTB_EVENT_HW_LINK_UP; 1377 1378 if (ntb->type == NTB_SOC || 1379 ntb->conn_type == NTB_CONN_TRANSPARENT) 1380 status = ntb_reg_read(2, ntb->reg->lnk_sta); 1381 else 1382 status = pci_read_config(ntb->device, 1383 XEON_LINK_STATUS_OFFSET, 2); 1384 ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 1385 ntb->link_speed = (status & NTB_LINK_SPEED_MASK); 1386 device_printf(ntb->device, "Link Width %d, Link Speed %d\n", 1387 ntb->link_width, ntb->link_speed); 1388 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, 1389 soc_link_hb, ntb); 1390 } else { 1391 device_printf(ntb->device, "Link Down\n"); 1392 ntb->link_status = NTB_LINK_DOWN; 1393 event = NTB_EVENT_HW_LINK_DOWN; 1394 /* Do not modify link width/speed, we need it in link recovery */ 1395 } 1396 1397 /* notify the upper layer if we have an event change */ 1398 if (ntb->event_cb != NULL) 1399 ntb->event_cb(ntb->ntb_transport, event); 1400} 1401 1402static void 1403ntb_link_enable(struct ntb_softc *ntb) 1404{ 1405 uint32_t cntl; 1406 1407 if (ntb->type == NTB_SOC) { 1408 pci_write_config(ntb->device, NTB_PPD_OFFSET, 1409 ntb->ppd | SOC_PPD_INIT_LINK, 4); 1410 return; 1411 } 1412 1413 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1414 ntb_handle_link_event(ntb, NTB_LINK_UP); 1415 return; 1416 } 1417 1418 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1419 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1420 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1421 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1422 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1423 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 1424 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1425} 1426 1427static void 1428ntb_link_disable(struct ntb_softc *ntb) 1429{ 1430 uint32_t cntl; 1431 1432 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 1433 ntb_handle_link_event(ntb, NTB_LINK_DOWN); 1434 return; 1435 } 1436 1437 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1438 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 1439 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 1440 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1441 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 1442 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 1443 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 1444} 1445 1446static void 1447recover_soc_link(void *arg) 1448{ 1449 struct ntb_softc *ntb = arg; 1450 uint8_t speed, width; 1451 uint32_t status32; 1452 1453 soc_perform_link_restart(ntb); 1454 1455 /* 1456 * There is a potential race between the 2 NTB devices recovering at 1457 * the same time. If the times are the same, the link will not recover 1458 * and the driver will be stuck in this loop forever. Add a random 1459 * interval to the recovery time to prevent this race. 1460 */ 1461 status32 = arc4random() % SOC_LINK_RECOVERY_TIME; 1462 pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000); 1463 1464 status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET); 1465 if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) 1466 goto retry; 1467 1468 status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET); 1469 if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) 1470 goto retry; 1471 1472 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 1473 if ((status32 & SOC_CNTL_LINK_DOWN) != 0) 1474 goto out; 1475 1476 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 1477 width = (status32 & NTB_LINK_WIDTH_MASK) >> 4; 1478 speed = (status32 & NTB_LINK_SPEED_MASK); 1479 if (ntb->link_width != width || ntb->link_speed != speed) 1480 goto retry; 1481 1482out: 1483 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, 1484 ntb); 1485 return; 1486 1487retry: 1488 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link, 1489 ntb); 1490} 1491 1492static int 1493ntb_poll_link(struct ntb_softc *ntb) 1494{ 1495 int link_state; 1496 uint32_t ntb_cntl; 1497 uint16_t status; 1498 1499 if (ntb->type == NTB_SOC) { 1500 HW_LOCK(ntb); 1501 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 1502 if (ntb_cntl == ntb->ntb_ctl) { 1503 HW_UNLOCK(ntb); 1504 return (0); 1505 } 1506 ntb->ntb_ctl = ntb_cntl; 1507 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 1508 HW_UNLOCK(ntb); 1509 1510 if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) 1511 link_state = NTB_LINK_DOWN; 1512 else 1513 link_state = NTB_LINK_UP; 1514 } else { 1515 status = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 1516 if (status == ntb->lnk_sta) 1517 return (0); 1518 ntb->lnk_sta = status; 1519 1520 if ((status & NTB_LINK_STATUS_ACTIVE) != 0) 1521 link_state = NTB_LINK_UP; 1522 else 1523 link_state = NTB_LINK_DOWN; 1524 } 1525 1526 ntb_handle_link_event(ntb, link_state); 1527 return (0); 1528} 1529 1530static void 1531ntb_irq_work(void *arg) 1532{ 1533 struct ntb_db_cb *db_cb = arg; 1534 struct ntb_softc *ntb; 1535 int rc; 1536 1537 rc = db_cb->callback(db_cb->data, db_cb->db_num); 1538 /* Poll if forward progress was made. */ 1539 if (rc != 0) { 1540 callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); 1541 return; 1542 } 1543 1544 /* Unmask interrupt if no progress was made. */ 1545 ntb = db_cb->ntb; 1546 unmask_ldb_interrupt(ntb, db_cb->db_num); 1547} 1548 1549/* 1550 * Public API to the rest of the OS 1551 */ 1552 1553/** 1554 * ntb_register_event_callback() - register event callback 1555 * @ntb: pointer to ntb_softc instance 1556 * @func: callback function to register 1557 * 1558 * This function registers a callback for any HW driver events such as link 1559 * up/down, power management notices and etc. 1560 * 1561 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1562 */ 1563int 1564ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func) 1565{ 1566 1567 if (ntb->event_cb != NULL) 1568 return (EINVAL); 1569 1570 ntb->event_cb = func; 1571 1572 return (0); 1573} 1574 1575/** 1576 * ntb_unregister_event_callback() - unregisters the event callback 1577 * @ntb: pointer to ntb_softc instance 1578 * 1579 * This function unregisters the existing callback from transport 1580 */ 1581void 1582ntb_unregister_event_callback(struct ntb_softc *ntb) 1583{ 1584 1585 ntb->event_cb = NULL; 1586} 1587 1588/** 1589 * ntb_register_db_callback() - register a callback for doorbell interrupt 1590 * @ntb: pointer to ntb_softc instance 1591 * @idx: doorbell index to register callback, zero based 1592 * @data: pointer to be returned to caller with every callback 1593 * @func: callback function to register 1594 * 1595 * This function registers a callback function for the doorbell interrupt 1596 * on the primary side. The function will unmask the doorbell as well to 1597 * allow interrupt. 1598 * 1599 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1600 */ 1601int 1602ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data, 1603 ntb_db_callback func) 1604{ 1605 struct ntb_db_cb *db_cb = &ntb->db_cb[idx]; 1606 1607 if (idx >= ntb->max_cbs || db_cb->callback != NULL || db_cb->reserved) { 1608 device_printf(ntb->device, "Invalid Index.\n"); 1609 return (EINVAL); 1610 } 1611 1612 db_cb->callback = func; 1613 db_cb->data = data; 1614 callout_init(&db_cb->irq_work, 1); 1615 1616 unmask_ldb_interrupt(ntb, idx); 1617 1618 return (0); 1619} 1620 1621/** 1622 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 1623 * @ntb: pointer to ntb_softc instance 1624 * @idx: doorbell index to register callback, zero based 1625 * 1626 * This function unregisters a callback function for the doorbell interrupt 1627 * on the primary side. The function will also mask the said doorbell. 1628 */ 1629void 1630ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx) 1631{ 1632 1633 if (idx >= ntb->max_cbs || ntb->db_cb[idx].callback == NULL) 1634 return; 1635 1636 mask_ldb_interrupt(ntb, idx); 1637 1638 callout_drain(&ntb->db_cb[idx].irq_work); 1639 ntb->db_cb[idx].callback = NULL; 1640} 1641 1642/** 1643 * ntb_find_transport() - find the transport pointer 1644 * @transport: pointer to pci device 1645 * 1646 * Given the pci device pointer, return the transport pointer passed in when 1647 * the transport attached when it was inited. 1648 * 1649 * RETURNS: pointer to transport. 1650 */ 1651void * 1652ntb_find_transport(struct ntb_softc *ntb) 1653{ 1654 1655 return (ntb->ntb_transport); 1656} 1657 1658/** 1659 * ntb_register_transport() - Register NTB transport with NTB HW driver 1660 * @transport: transport identifier 1661 * 1662 * This function allows a transport to reserve the hardware driver for 1663 * NTB usage. 1664 * 1665 * RETURNS: pointer to ntb_softc, NULL on error. 1666 */ 1667struct ntb_softc * 1668ntb_register_transport(struct ntb_softc *ntb, void *transport) 1669{ 1670 1671 /* 1672 * TODO: when we have more than one transport, we will need to rewrite 1673 * this to prevent race conditions 1674 */ 1675 if (ntb->ntb_transport != NULL) 1676 return (NULL); 1677 1678 ntb->ntb_transport = transport; 1679 return (ntb); 1680} 1681 1682/** 1683 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 1684 * @ntb - ntb_softc of the transport to be freed 1685 * 1686 * This function unregisters the transport from the HW driver and performs any 1687 * necessary cleanups. 1688 */ 1689void 1690ntb_unregister_transport(struct ntb_softc *ntb) 1691{ 1692 uint8_t i; 1693 1694 if (ntb->ntb_transport == NULL) 1695 return; 1696 1697 for (i = 0; i < ntb->max_cbs; i++) 1698 ntb_unregister_db_callback(ntb, i); 1699 1700 ntb_unregister_event_callback(ntb); 1701 ntb->ntb_transport = NULL; 1702} 1703 1704/** 1705 * ntb_get_max_spads() - get the total scratch regs usable 1706 * @ntb: pointer to ntb_softc instance 1707 * 1708 * This function returns the max 32bit scratchpad registers usable by the 1709 * upper layer. 1710 * 1711 * RETURNS: total number of scratch pad registers available 1712 */ 1713uint8_t 1714ntb_get_max_spads(struct ntb_softc *ntb) 1715{ 1716 1717 return (ntb->spad_count); 1718} 1719 1720uint8_t 1721ntb_get_max_cbs(struct ntb_softc *ntb) 1722{ 1723 1724 return (ntb->max_cbs); 1725} 1726 1727uint8_t 1728ntb_mw_count(struct ntb_softc *ntb) 1729{ 1730 1731 return (ntb->mw_count); 1732} 1733 1734/** 1735 * ntb_write_local_spad() - write to the secondary scratchpad register 1736 * @ntb: pointer to ntb_softc instance 1737 * @idx: index to the scratchpad register, 0 based 1738 * @val: the data value to put into the register 1739 * 1740 * This function allows writing of a 32bit value to the indexed scratchpad 1741 * register. The register resides on the secondary (external) side. 1742 * 1743 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1744 */ 1745int 1746ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1747{ 1748 1749 if (idx >= ntb->spad_count) 1750 return (EINVAL); 1751 1752 ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val); 1753 1754 return (0); 1755} 1756 1757/** 1758 * ntb_read_local_spad() - read from the primary scratchpad register 1759 * @ntb: pointer to ntb_softc instance 1760 * @idx: index to scratchpad register, 0 based 1761 * @val: pointer to 32bit integer for storing the register value 1762 * 1763 * This function allows reading of the 32bit scratchpad register on 1764 * the primary (internal) side. 1765 * 1766 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1767 */ 1768int 1769ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1770{ 1771 1772 if (idx >= ntb->spad_count) 1773 return (EINVAL); 1774 1775 *val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4); 1776 1777 return (0); 1778} 1779 1780/** 1781 * ntb_write_remote_spad() - write to the secondary scratchpad register 1782 * @ntb: pointer to ntb_softc instance 1783 * @idx: index to the scratchpad register, 0 based 1784 * @val: the data value to put into the register 1785 * 1786 * This function allows writing of a 32bit value to the indexed scratchpad 1787 * register. The register resides on the secondary (external) side. 1788 * 1789 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1790 */ 1791int 1792ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 1793{ 1794 1795 if (idx >= ntb->spad_count) 1796 return (EINVAL); 1797 1798 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1799 ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); 1800 else 1801 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 1802 1803 return (0); 1804} 1805 1806/** 1807 * ntb_read_remote_spad() - read from the primary scratchpad register 1808 * @ntb: pointer to ntb_softc instance 1809 * @idx: index to scratchpad register, 0 based 1810 * @val: pointer to 32bit integer for storing the register value 1811 * 1812 * This function allows reading of the 32bit scratchpad register on 1813 * the primary (internal) side. 1814 * 1815 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 1816 */ 1817int 1818ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 1819{ 1820 1821 if (idx >= ntb->spad_count) 1822 return (EINVAL); 1823 1824 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 1825 *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); 1826 else 1827 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 1828 1829 return (0); 1830} 1831 1832/** 1833 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window 1834 * @ntb: pointer to ntb_softc instance 1835 * @mw: memory window number 1836 * 1837 * This function provides the base virtual address of the memory window 1838 * specified. 1839 * 1840 * RETURNS: pointer to virtual address, or NULL on error. 1841 */ 1842void * 1843ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw) 1844{ 1845 1846 if (mw >= ntb_mw_count(ntb)) 1847 return (NULL); 1848 1849 return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].vbase); 1850} 1851 1852bus_addr_t 1853ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) 1854{ 1855 1856 if (mw >= ntb_mw_count(ntb)) 1857 return (0); 1858 1859 return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].pbase); 1860} 1861 1862/** 1863 * ntb_get_mw_size() - return size of NTB memory window 1864 * @ntb: pointer to ntb_softc instance 1865 * @mw: memory window number 1866 * 1867 * This function provides the physical size of the memory window specified 1868 * 1869 * RETURNS: the size of the memory window or zero on error 1870 */ 1871u_long 1872ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw) 1873{ 1874 1875 if (mw >= ntb_mw_count(ntb)) 1876 return (0); 1877 1878 return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].size); 1879} 1880 1881/** 1882 * ntb_set_mw_addr - set the memory window address 1883 * @ntb: pointer to ntb_softc instance 1884 * @mw: memory window number 1885 * @addr: base address for data 1886 * 1887 * This function sets the base physical address of the memory window. This 1888 * memory address is where data from the remote system will be transfered into 1889 * or out of depending on how the transport is configured. 1890 */ 1891void 1892ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr) 1893{ 1894 1895 if (mw >= ntb_mw_count(ntb)) 1896 return; 1897 1898 switch (ntb_mw_to_bar(ntb, mw)) { 1899 case NTB_B2B_BAR_1: 1900 ntb_reg_write(8, ntb->xlat_reg->bar2_xlat, addr); 1901 break; 1902 case NTB_B2B_BAR_2: 1903 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1904 ntb_reg_write(4, ntb->reg_ofs.bar4_xlat, addr); 1905 else 1906 ntb_reg_write(8, ntb->reg_ofs.bar4_xlat, addr); 1907 break; 1908 case NTB_B2B_BAR_3: 1909 ntb_reg_write(4, ntb->reg_ofs.bar5_xlat, addr); 1910 break; 1911 default: 1912 KASSERT(false, ("invalid BAR")); 1913 break; 1914 } 1915} 1916 1917/** 1918 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side 1919 * @ntb: pointer to ntb_softc instance 1920 * @db: doorbell to ring 1921 * 1922 * This function allows triggering of a doorbell on the secondary/external 1923 * side that will initiate an interrupt on the remote host 1924 */ 1925void 1926ntb_ring_doorbell(struct ntb_softc *ntb, unsigned int db) 1927{ 1928 uint64_t bit; 1929 1930 if (ntb->type == NTB_SOC) 1931 bit = 1 << db; 1932 else 1933 bit = ((1 << ntb->db_vec_shift) - 1) << 1934 (db * ntb->db_vec_shift); 1935 1936 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1937 ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit); 1938 return; 1939 } 1940 1941 ntb_db_write(ntb, ntb->peer_reg->db_bell, bit); 1942} 1943 1944/* 1945 * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 1946 * as well as the size of the register (via *sz_out). 1947 * 1948 * This function allows a caller using I/OAT DMA to chain the remote doorbell 1949 * ring to its memory window write. 1950 * 1951 * Note that writing the peer doorbell via a memory window will *not* generate 1952 * an interrupt on the remote host; that must be done seperately. 1953 */ 1954bus_addr_t 1955ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 1956{ 1957 struct ntb_pci_bar_info *bar; 1958 uint64_t regoff; 1959 1960 KASSERT(sz_out != NULL, ("must be non-NULL")); 1961 1962 if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1963 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 1964 regoff = ntb->peer_reg->db_bell; 1965 } else { 1966 KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) || 1967 (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1), 1968 ("mw_count invalid after setup")); 1969 KASSERT(ntb->b2b_mw_idx != UINT8_MAX, ("invalid b2b idx")); 1970 1971 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 1972 regoff = XEON_SHADOW_PDOORBELL_OFFSET; 1973 } 1974 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 1975 1976 *sz_out = ntb->reg->db_size; 1977 /* HACK: Specific to current x86 bus implementation. */ 1978 return ((uint64_t)bar->pci_bus_handle + regoff); 1979} 1980 1981/** 1982 * ntb_query_link_status() - return the hardware link status 1983 * @ndev: pointer to ntb_device instance 1984 * 1985 * Returns true if the hardware is connected to the remote system 1986 * 1987 * RETURNS: true or false based on the hardware link state 1988 */ 1989bool 1990ntb_query_link_status(struct ntb_softc *ntb) 1991{ 1992 1993 return (ntb->link_status == NTB_LINK_UP); 1994} 1995 1996static void 1997save_bar_parameters(struct ntb_pci_bar_info *bar) 1998{ 1999 2000 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 2001 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 2002 bar->pbase = rman_get_start(bar->pci_resource); 2003 bar->size = rman_get_size(bar->pci_resource); 2004 bar->vbase = rman_get_virtual(bar->pci_resource); 2005} 2006 2007device_t 2008ntb_get_device(struct ntb_softc *ntb) 2009{ 2010 2011 return (ntb->device); 2012} 2013 2014/* Export HW-specific errata information. */ 2015bool 2016ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) 2017{ 2018 2019 return (HAS_FEATURE(feature)); 2020} 2021