ntb_hw.c revision 300100
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * Copyright (C) 2015 EMC Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 300100 2016-05-18 02:10:05Z cem $"); 30 31#include <sys/param.h> 32#include <sys/kernel.h> 33#include <sys/systm.h> 34#include <sys/bus.h> 35#include <sys/endian.h> 36#include <sys/malloc.h> 37#include <sys/module.h> 38#include <sys/mutex.h> 39#include <sys/pciio.h> 40#include <sys/queue.h> 41#include <sys/rman.h> 42#include <sys/sbuf.h> 43#include <sys/sysctl.h> 44#include <vm/vm.h> 45#include <vm/pmap.h> 46#include <machine/bus.h> 47#include <machine/intr_machdep.h> 48#include <machine/resource.h> 49#include <dev/pci/pcireg.h> 50#include <dev/pci/pcivar.h> 51 52#include "ntb_regs.h" 53#include "ntb_hw.h" 54 55/* 56 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 57 * allows you to connect two systems using a PCI-e link. 58 * 59 * This module contains the hardware abstraction layer for the NTB. It allows 60 * you to send and receive interrupts, map the memory windows and send and 61 * receive messages in the scratch-pad registers. 62 * 63 * NOTE: Much of the code in this module is shared with Linux. Any patches may 64 * be picked up and redistributed in Linux with a dual GPL/BSD license. 65 */ 66 67#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 68 69#define NTB_HB_TIMEOUT 1 /* second */ 70#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 71#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75#define NTB_MSIX_VER_GUARD 0xaabbccdd 76#define NTB_MSIX_RECEIVED 0xe0f0e0f0 77#define ONE_MB (1024u * 1024) 78 79/* 80 * PCI constants could be somewhere more generic, but aren't defined/used in 81 * pci.c. 82 */ 83#define PCI_MSIX_ENTRY_SIZE 16 84#define PCI_MSIX_ENTRY_LOWER_ADDR 0 85#define PCI_MSIX_ENTRY_UPPER_ADDR 4 86#define PCI_MSIX_ENTRY_DATA 8 87 88enum ntb_device_type { 89 NTB_XEON, 90 NTB_ATOM 91}; 92 93/* ntb_conn_type are hardware numbers, cannot change. */ 94enum ntb_conn_type { 95 NTB_CONN_TRANSPARENT = 0, 96 NTB_CONN_B2B = 1, 97 NTB_CONN_RP = 2, 98}; 99 100enum ntb_b2b_direction { 101 NTB_DEV_USD = 0, 102 NTB_DEV_DSD = 1, 103}; 104 105enum ntb_bar { 106 NTB_CONFIG_BAR = 0, 107 NTB_B2B_BAR_1, 108 NTB_B2B_BAR_2, 109 NTB_B2B_BAR_3, 110 NTB_MAX_BARS 111}; 112 113enum { 114 NTB_MSIX_GUARD = 0, 115 NTB_MSIX_DATA0, 116 NTB_MSIX_DATA1, 117 NTB_MSIX_DATA2, 118 NTB_MSIX_OFS0, 119 NTB_MSIX_OFS1, 120 NTB_MSIX_OFS2, 121 NTB_MSIX_DONE, 122 NTB_MAX_MSIX_SPAD 123}; 124 125/* Device features and workarounds */ 126#define HAS_FEATURE(feature) \ 127 ((ntb->features & (feature)) != 0) 128 129struct ntb_hw_info { 130 uint32_t device_id; 131 const char *desc; 132 enum ntb_device_type type; 133 uint32_t features; 134}; 135 136struct ntb_pci_bar_info { 137 bus_space_tag_t pci_bus_tag; 138 bus_space_handle_t pci_bus_handle; 139 int pci_resource_id; 140 struct resource *pci_resource; 141 vm_paddr_t pbase; 142 caddr_t vbase; 143 vm_size_t size; 144 vm_memattr_t map_mode; 145 146 /* Configuration register offsets */ 147 uint32_t psz_off; 148 uint32_t ssz_off; 149 uint32_t pbarxlat_off; 150}; 151 152struct ntb_int_info { 153 struct resource *res; 154 int rid; 155 void *tag; 156}; 157 158struct ntb_vec { 159 struct ntb_softc *ntb; 160 uint32_t num; 161 unsigned masked; 162}; 163 164struct ntb_reg { 165 uint32_t ntb_ctl; 166 uint32_t lnk_sta; 167 uint8_t db_size; 168 unsigned mw_bar[NTB_MAX_BARS]; 169}; 170 171struct ntb_alt_reg { 172 uint32_t db_bell; 173 uint32_t db_mask; 174 uint32_t spad; 175}; 176 177struct ntb_xlat_reg { 178 uint32_t bar0_base; 179 uint32_t bar2_base; 180 uint32_t bar4_base; 181 uint32_t bar5_base; 182 183 uint32_t bar2_xlat; 184 uint32_t bar4_xlat; 185 uint32_t bar5_xlat; 186 187 uint32_t bar2_limit; 188 uint32_t bar4_limit; 189 uint32_t bar5_limit; 190}; 191 192struct ntb_b2b_addr { 193 uint64_t bar0_addr; 194 uint64_t bar2_addr64; 195 uint64_t bar4_addr64; 196 uint64_t bar4_addr32; 197 uint64_t bar5_addr32; 198}; 199 200struct ntb_msix_data { 201 uint32_t nmd_ofs; 202 uint32_t nmd_data; 203}; 204 205struct ntb_softc { 206 device_t device; 207 enum ntb_device_type type; 208 uint32_t features; 209 210 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 211 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 212 uint32_t allocated_interrupts; 213 214 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216 bool peer_msix_good; 217 bool peer_msix_done; 218 struct ntb_pci_bar_info *peer_lapic_bar; 219 struct callout peer_msix_work; 220 221 struct callout heartbeat_timer; 222 struct callout lr_timer; 223 224 void *ntb_ctx; 225 const struct ntb_ctx_ops *ctx_ops; 226 struct ntb_vec *msix_vec; 227#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 228#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 229#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 230 struct mtx ctx_lock; 231 232 uint32_t ppd; 233 enum ntb_conn_type conn_type; 234 enum ntb_b2b_direction dev_type; 235 236 /* Offset of peer bar0 in B2B BAR */ 237 uint64_t b2b_off; 238 /* Memory window used to access peer bar0 */ 239#define B2B_MW_DISABLED UINT8_MAX 240 uint8_t b2b_mw_idx; 241 uint8_t msix_mw_idx; 242 243 uint8_t mw_count; 244 uint8_t spad_count; 245 uint8_t db_count; 246 uint8_t db_vec_count; 247 uint8_t db_vec_shift; 248 249 /* Protects local db_mask. */ 250#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 251#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 252#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 253 struct mtx db_mask_lock; 254 255 volatile uint32_t ntb_ctl; 256 volatile uint32_t lnk_sta; 257 258 uint64_t db_valid_mask; 259 uint64_t db_link_mask; 260 uint64_t db_mask; 261 262 int last_ts; /* ticks @ last irq */ 263 264 const struct ntb_reg *reg; 265 const struct ntb_alt_reg *self_reg; 266 const struct ntb_alt_reg *peer_reg; 267 const struct ntb_xlat_reg *xlat_reg; 268}; 269 270#ifdef __i386__ 271static __inline uint64_t 272bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 273 bus_size_t offset) 274{ 275 276 return (bus_space_read_4(tag, handle, offset) | 277 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 278} 279 280static __inline void 281bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 282 bus_size_t offset, uint64_t val) 283{ 284 285 bus_space_write_4(tag, handle, offset, val); 286 bus_space_write_4(tag, handle, offset + 4, val >> 32); 287} 288#endif 289 290#define ntb_bar_read(SIZE, bar, offset) \ 291 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 292 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 293#define ntb_bar_write(SIZE, bar, offset, val) \ 294 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 295 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 296#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 297#define ntb_reg_write(SIZE, offset, val) \ 298 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 299#define ntb_mw_read(SIZE, offset) \ 300 ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 301#define ntb_mw_write(SIZE, offset, val) \ 302 ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 303 offset, val) 304 305static int ntb_probe(device_t device); 306static int ntb_attach(device_t device); 307static int ntb_detach(device_t device); 308static unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 309static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 310static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 311static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 312 uint32_t *base, uint32_t *xlat, uint32_t *lmt); 313static int ntb_map_pci_bars(struct ntb_softc *ntb); 314static int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 315 vm_memattr_t); 316static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 317 const char *); 318static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 319static int map_memory_window_bar(struct ntb_softc *ntb, 320 struct ntb_pci_bar_info *bar); 321static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 322static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 323static int ntb_init_isr(struct ntb_softc *ntb); 324static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 325static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 326static void ntb_teardown_interrupts(struct ntb_softc *ntb); 327static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 328static void ntb_interrupt(struct ntb_softc *, uint32_t vec); 329static void ndev_vec_isr(void *arg); 330static void ndev_irq_isr(void *arg); 331static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 332static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 333static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 334static int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 335static void ntb_free_msix_vec(struct ntb_softc *ntb); 336static void ntb_get_msix_info(struct ntb_softc *ntb, uint32_t num_vectors); 337static void ntb_exchange_msix(void *); 338static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 339static void ntb_detect_max_mw(struct ntb_softc *ntb); 340static int ntb_detect_xeon(struct ntb_softc *ntb); 341static int ntb_detect_atom(struct ntb_softc *ntb); 342static int ntb_xeon_init_dev(struct ntb_softc *ntb); 343static int ntb_atom_init_dev(struct ntb_softc *ntb); 344static void ntb_teardown_xeon(struct ntb_softc *ntb); 345static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 346static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 347 enum ntb_bar regbar); 348static void xeon_set_sbar_base_and_limit(struct ntb_softc *, 349 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 350static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 351 enum ntb_bar idx); 352static int xeon_setup_b2b_mw(struct ntb_softc *, 353 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 354static int xeon_setup_msix_bar(struct ntb_softc *); 355static inline bool link_is_up(struct ntb_softc *ntb); 356static inline bool _xeon_link_is_up(struct ntb_softc *ntb); 357static inline bool atom_link_is_err(struct ntb_softc *ntb); 358static inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 359static inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 360static void atom_link_hb(void *arg); 361static void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 362static void recover_atom_link(void *arg); 363static bool ntb_poll_link(struct ntb_softc *ntb); 364static void save_bar_parameters(struct ntb_pci_bar_info *bar); 365static void ntb_sysctl_init(struct ntb_softc *); 366static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 367static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 368static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 369static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 370static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 371 372static unsigned g_ntb_hw_debug_level; 373SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 374 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 375#define ntb_printf(lvl, ...) do { \ 376 if ((lvl) <= g_ntb_hw_debug_level) { \ 377 device_printf(ntb->device, __VA_ARGS__); \ 378 } \ 379} while (0) 380 381#define _NTB_PAT_UC 0 382#define _NTB_PAT_WC 1 383#define _NTB_PAT_WT 4 384#define _NTB_PAT_WP 5 385#define _NTB_PAT_WB 6 386#define _NTB_PAT_UCM 7 387static unsigned g_ntb_mw_pat = _NTB_PAT_UC; 388SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 389 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 390 "UC: " __XSTRING(_NTB_PAT_UC) ", " 391 "WC: " __XSTRING(_NTB_PAT_WC) ", " 392 "WT: " __XSTRING(_NTB_PAT_WT) ", " 393 "WP: " __XSTRING(_NTB_PAT_WP) ", " 394 "WB: " __XSTRING(_NTB_PAT_WB) ", " 395 "UC-: " __XSTRING(_NTB_PAT_UCM)); 396 397static inline vm_memattr_t 398ntb_pat_flags(void) 399{ 400 401 switch (g_ntb_mw_pat) { 402 case _NTB_PAT_WC: 403 return (VM_MEMATTR_WRITE_COMBINING); 404 case _NTB_PAT_WT: 405 return (VM_MEMATTR_WRITE_THROUGH); 406 case _NTB_PAT_WP: 407 return (VM_MEMATTR_WRITE_PROTECTED); 408 case _NTB_PAT_WB: 409 return (VM_MEMATTR_WRITE_BACK); 410 case _NTB_PAT_UCM: 411 return (VM_MEMATTR_WEAK_UNCACHEABLE); 412 case _NTB_PAT_UC: 413 /* FALLTHROUGH */ 414 default: 415 return (VM_MEMATTR_UNCACHEABLE); 416 } 417} 418 419/* 420 * Well, this obviously doesn't belong here, but it doesn't seem to exist 421 * anywhere better yet. 422 */ 423static inline const char * 424ntb_vm_memattr_to_str(vm_memattr_t pat) 425{ 426 427 switch (pat) { 428 case VM_MEMATTR_WRITE_COMBINING: 429 return ("WRITE_COMBINING"); 430 case VM_MEMATTR_WRITE_THROUGH: 431 return ("WRITE_THROUGH"); 432 case VM_MEMATTR_WRITE_PROTECTED: 433 return ("WRITE_PROTECTED"); 434 case VM_MEMATTR_WRITE_BACK: 435 return ("WRITE_BACK"); 436 case VM_MEMATTR_WEAK_UNCACHEABLE: 437 return ("UNCACHED"); 438 case VM_MEMATTR_UNCACHEABLE: 439 return ("UNCACHEABLE"); 440 default: 441 return ("UNKNOWN"); 442 } 443} 444 445static int g_ntb_msix_idx = 0; 446SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 447 0, "Use this memory window to access the peer MSIX message complex on " 448 "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 449 "Like b2b_mw_idx, negative values index from the last available memory " 450 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 451 452static int g_ntb_mw_idx = -1; 453SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 454 0, "Use this memory window to access the peer NTB registers. A " 455 "non-negative value starts from the first MW index; a negative value " 456 "starts from the last MW index. The default is -1, i.e., the last " 457 "available memory window. Both sides of the NTB MUST set the same " 458 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 459 460static struct ntb_hw_info pci_ids[] = { 461 /* XXX: PS/SS IDs left out until they are supported. */ 462 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 463 NTB_ATOM, 0 }, 464 465 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 466 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 467 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 468 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 469 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 470 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 471 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 472 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 473 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 474 NTB_SB01BASE_LOCKUP }, 475 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 476 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 477 NTB_SB01BASE_LOCKUP }, 478 479 { 0x00000000, NULL, NTB_ATOM, 0 } 480}; 481 482static const struct ntb_reg atom_reg = { 483 .ntb_ctl = ATOM_NTBCNTL_OFFSET, 484 .lnk_sta = ATOM_LINK_STATUS_OFFSET, 485 .db_size = sizeof(uint64_t), 486 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 487}; 488 489static const struct ntb_alt_reg atom_pri_reg = { 490 .db_bell = ATOM_PDOORBELL_OFFSET, 491 .db_mask = ATOM_PDBMSK_OFFSET, 492 .spad = ATOM_SPAD_OFFSET, 493}; 494 495static const struct ntb_alt_reg atom_b2b_reg = { 496 .db_bell = ATOM_B2B_DOORBELL_OFFSET, 497 .spad = ATOM_B2B_SPAD_OFFSET, 498}; 499 500static const struct ntb_xlat_reg atom_sec_xlat = { 501#if 0 502 /* "FIXME" says the Linux driver. */ 503 .bar0_base = ATOM_SBAR0BASE_OFFSET, 504 .bar2_base = ATOM_SBAR2BASE_OFFSET, 505 .bar4_base = ATOM_SBAR4BASE_OFFSET, 506 507 .bar2_limit = ATOM_SBAR2LMT_OFFSET, 508 .bar4_limit = ATOM_SBAR4LMT_OFFSET, 509#endif 510 511 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 512 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 513}; 514 515static const struct ntb_reg xeon_reg = { 516 .ntb_ctl = XEON_NTBCNTL_OFFSET, 517 .lnk_sta = XEON_LINK_STATUS_OFFSET, 518 .db_size = sizeof(uint16_t), 519 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 520}; 521 522static const struct ntb_alt_reg xeon_pri_reg = { 523 .db_bell = XEON_PDOORBELL_OFFSET, 524 .db_mask = XEON_PDBMSK_OFFSET, 525 .spad = XEON_SPAD_OFFSET, 526}; 527 528static const struct ntb_alt_reg xeon_b2b_reg = { 529 .db_bell = XEON_B2B_DOORBELL_OFFSET, 530 .spad = XEON_B2B_SPAD_OFFSET, 531}; 532 533static const struct ntb_xlat_reg xeon_sec_xlat = { 534 .bar0_base = XEON_SBAR0BASE_OFFSET, 535 .bar2_base = XEON_SBAR2BASE_OFFSET, 536 .bar4_base = XEON_SBAR4BASE_OFFSET, 537 .bar5_base = XEON_SBAR5BASE_OFFSET, 538 539 .bar2_limit = XEON_SBAR2LMT_OFFSET, 540 .bar4_limit = XEON_SBAR4LMT_OFFSET, 541 .bar5_limit = XEON_SBAR5LMT_OFFSET, 542 543 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 544 .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 545 .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 546}; 547 548static struct ntb_b2b_addr xeon_b2b_usd_addr = { 549 .bar0_addr = XEON_B2B_BAR0_ADDR, 550 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 551 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 552 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 553 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 554}; 555 556static struct ntb_b2b_addr xeon_b2b_dsd_addr = { 557 .bar0_addr = XEON_B2B_BAR0_ADDR, 558 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 559 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 560 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 561 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 562}; 563 564SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 565 "B2B MW segment overrides -- MUST be the same on both sides"); 566 567SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 568 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 569 "hardware, use this 64-bit address on the bus between the NTB devices for " 570 "the window at BAR2, on the upstream side of the link. MUST be the same " 571 "address on both sides."); 572SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 573 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 574SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 575 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 576 "(split-BAR mode)."); 577SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 578 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 579 "(split-BAR mode)."); 580 581SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 582 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 583 "hardware, use this 64-bit address on the bus between the NTB devices for " 584 "the window at BAR2, on the downstream side of the link. MUST be the same" 585 " address on both sides."); 586SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 587 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 588SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 589 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 590 "(split-BAR mode)."); 591SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 592 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 593 "(split-BAR mode)."); 594 595/* 596 * OS <-> Driver interface structures 597 */ 598MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 599 600static device_method_t ntb_pci_methods[] = { 601 /* Device interface */ 602 DEVMETHOD(device_probe, ntb_probe), 603 DEVMETHOD(device_attach, ntb_attach), 604 DEVMETHOD(device_detach, ntb_detach), 605 DEVMETHOD_END 606}; 607 608static driver_t ntb_pci_driver = { 609 "ntb_hw", 610 ntb_pci_methods, 611 sizeof(struct ntb_softc), 612}; 613 614static devclass_t ntb_devclass; 615DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 616MODULE_VERSION(ntb_hw, 1); 617 618SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 619 620/* 621 * OS <-> Driver linkage functions 622 */ 623static int 624ntb_probe(device_t device) 625{ 626 struct ntb_hw_info *p; 627 628 p = ntb_get_device_info(pci_get_devid(device)); 629 if (p == NULL) 630 return (ENXIO); 631 632 device_set_desc(device, p->desc); 633 return (0); 634} 635 636static int 637ntb_attach(device_t device) 638{ 639 struct ntb_softc *ntb; 640 struct ntb_hw_info *p; 641 int error; 642 643 ntb = DEVICE2SOFTC(device); 644 p = ntb_get_device_info(pci_get_devid(device)); 645 646 ntb->device = device; 647 ntb->type = p->type; 648 ntb->features = p->features; 649 ntb->b2b_mw_idx = B2B_MW_DISABLED; 650 ntb->msix_mw_idx = B2B_MW_DISABLED; 651 652 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 653 callout_init(&ntb->heartbeat_timer, 1); 654 callout_init(&ntb->lr_timer, 1); 655 callout_init(&ntb->peer_msix_work, 1); 656 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 657 mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 658 659 if (ntb->type == NTB_ATOM) 660 error = ntb_detect_atom(ntb); 661 else 662 error = ntb_detect_xeon(ntb); 663 if (error != 0) 664 goto out; 665 666 ntb_detect_max_mw(ntb); 667 668 pci_enable_busmaster(ntb->device); 669 670 error = ntb_map_pci_bars(ntb); 671 if (error != 0) 672 goto out; 673 if (ntb->type == NTB_ATOM) 674 error = ntb_atom_init_dev(ntb); 675 else 676 error = ntb_xeon_init_dev(ntb); 677 if (error != 0) 678 goto out; 679 680 ntb_spad_clear(ntb); 681 682 ntb_poll_link(ntb); 683 684 ntb_sysctl_init(ntb); 685 686out: 687 if (error != 0) 688 ntb_detach(device); 689 return (error); 690} 691 692static int 693ntb_detach(device_t device) 694{ 695 struct ntb_softc *ntb; 696 697 ntb = DEVICE2SOFTC(device); 698 699 if (ntb->self_reg != NULL) { 700 DB_MASK_LOCK(ntb); 701 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 702 DB_MASK_UNLOCK(ntb); 703 } 704 callout_drain(&ntb->heartbeat_timer); 705 callout_drain(&ntb->lr_timer); 706 callout_drain(&ntb->peer_msix_work); 707 pci_disable_busmaster(ntb->device); 708 if (ntb->type == NTB_XEON) 709 ntb_teardown_xeon(ntb); 710 ntb_teardown_interrupts(ntb); 711 712 mtx_destroy(&ntb->db_mask_lock); 713 mtx_destroy(&ntb->ctx_lock); 714 715 ntb_unmap_pci_bar(ntb); 716 717 return (0); 718} 719 720/* 721 * Driver internal routines 722 */ 723static inline enum ntb_bar 724ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 725{ 726 727 KASSERT(mw < ntb->mw_count, 728 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 729 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 730 731 return (ntb->reg->mw_bar[mw]); 732} 733 734static inline bool 735bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 736{ 737 /* XXX This assertion could be stronger. */ 738 KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 739 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 740} 741 742static inline void 743bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 744 uint32_t *xlat, uint32_t *lmt) 745{ 746 uint32_t basev, lmtv, xlatv; 747 748 switch (bar) { 749 case NTB_B2B_BAR_1: 750 basev = ntb->xlat_reg->bar2_base; 751 lmtv = ntb->xlat_reg->bar2_limit; 752 xlatv = ntb->xlat_reg->bar2_xlat; 753 break; 754 case NTB_B2B_BAR_2: 755 basev = ntb->xlat_reg->bar4_base; 756 lmtv = ntb->xlat_reg->bar4_limit; 757 xlatv = ntb->xlat_reg->bar4_xlat; 758 break; 759 case NTB_B2B_BAR_3: 760 basev = ntb->xlat_reg->bar5_base; 761 lmtv = ntb->xlat_reg->bar5_limit; 762 xlatv = ntb->xlat_reg->bar5_xlat; 763 break; 764 default: 765 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 766 ("bad bar")); 767 basev = lmtv = xlatv = 0; 768 break; 769 } 770 771 if (base != NULL) 772 *base = basev; 773 if (xlat != NULL) 774 *xlat = xlatv; 775 if (lmt != NULL) 776 *lmt = lmtv; 777} 778 779static int 780ntb_map_pci_bars(struct ntb_softc *ntb) 781{ 782 int rc; 783 784 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 785 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 786 if (rc != 0) 787 goto out; 788 789 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 790 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 791 if (rc != 0) 792 goto out; 793 ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 794 ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 795 ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 796 797 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 798 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 799 if (rc != 0) 800 goto out; 801 ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 802 ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 803 ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 804 805 if (!HAS_FEATURE(NTB_SPLIT_BAR)) 806 goto out; 807 808 ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 809 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 810 ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 811 ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 812 ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 813 814out: 815 if (rc != 0) 816 device_printf(ntb->device, 817 "unable to allocate pci resource\n"); 818 return (rc); 819} 820 821static void 822print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 823 const char *kind) 824{ 825 826 device_printf(ntb->device, 827 "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 828 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 829 (char *)bar->vbase + bar->size - 1, 830 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 831 (uintmax_t)bar->size, kind); 832} 833 834static int 835map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 836{ 837 838 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 839 &bar->pci_resource_id, RF_ACTIVE); 840 if (bar->pci_resource == NULL) 841 return (ENXIO); 842 843 save_bar_parameters(bar); 844 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 845 print_map_success(ntb, bar, "mmr"); 846 return (0); 847} 848 849static int 850map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 851{ 852 int rc; 853 vm_memattr_t mapmode; 854 uint8_t bar_size_bits = 0; 855 856 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 857 &bar->pci_resource_id, RF_ACTIVE); 858 859 if (bar->pci_resource == NULL) 860 return (ENXIO); 861 862 save_bar_parameters(bar); 863 /* 864 * Ivytown NTB BAR sizes are misreported by the hardware due to a 865 * hardware issue. To work around this, query the size it should be 866 * configured to by the device and modify the resource to correspond to 867 * this new size. The BIOS on systems with this problem is required to 868 * provide enough address space to allow the driver to make this change 869 * safely. 870 * 871 * Ideally I could have just specified the size when I allocated the 872 * resource like: 873 * bus_alloc_resource(ntb->device, 874 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 875 * 1ul << bar_size_bits, RF_ACTIVE); 876 * but the PCI driver does not honor the size in this call, so we have 877 * to modify it after the fact. 878 */ 879 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 880 if (bar->pci_resource_id == PCIR_BAR(2)) 881 bar_size_bits = pci_read_config(ntb->device, 882 XEON_PBAR23SZ_OFFSET, 1); 883 else 884 bar_size_bits = pci_read_config(ntb->device, 885 XEON_PBAR45SZ_OFFSET, 1); 886 887 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 888 bar->pci_resource, bar->pbase, 889 bar->pbase + (1ul << bar_size_bits) - 1); 890 if (rc != 0) { 891 device_printf(ntb->device, 892 "unable to resize bar\n"); 893 return (rc); 894 } 895 896 save_bar_parameters(bar); 897 } 898 899 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 900 print_map_success(ntb, bar, "mw"); 901 902 /* 903 * Optionally, mark MW BARs as anything other than UC to improve 904 * performance. 905 */ 906 mapmode = ntb_pat_flags(); 907 if (mapmode == bar->map_mode) 908 return (0); 909 910 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 911 if (rc == 0) { 912 bar->map_mode = mapmode; 913 device_printf(ntb->device, 914 "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 915 "%s.\n", 916 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 917 (char *)bar->vbase + bar->size - 1, 918 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 919 ntb_vm_memattr_to_str(mapmode)); 920 } else 921 device_printf(ntb->device, 922 "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 923 "%s: %d\n", 924 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 925 (char *)bar->vbase + bar->size - 1, 926 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 927 ntb_vm_memattr_to_str(mapmode), rc); 928 /* Proceed anyway */ 929 return (0); 930} 931 932static void 933ntb_unmap_pci_bar(struct ntb_softc *ntb) 934{ 935 struct ntb_pci_bar_info *current_bar; 936 int i; 937 938 for (i = 0; i < NTB_MAX_BARS; i++) { 939 current_bar = &ntb->bar_info[i]; 940 if (current_bar->pci_resource != NULL) 941 bus_release_resource(ntb->device, SYS_RES_MEMORY, 942 current_bar->pci_resource_id, 943 current_bar->pci_resource); 944 } 945} 946 947static int 948ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 949{ 950 uint32_t i; 951 int rc; 952 953 for (i = 0; i < num_vectors; i++) { 954 ntb->int_info[i].rid = i + 1; 955 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 956 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 957 if (ntb->int_info[i].res == NULL) { 958 device_printf(ntb->device, 959 "bus_alloc_resource failed\n"); 960 return (ENOMEM); 961 } 962 ntb->int_info[i].tag = NULL; 963 ntb->allocated_interrupts++; 964 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 965 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 966 &ntb->msix_vec[i], &ntb->int_info[i].tag); 967 if (rc != 0) { 968 device_printf(ntb->device, "bus_setup_intr failed\n"); 969 return (ENXIO); 970 } 971 } 972 return (0); 973} 974 975/* 976 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 977 * cannot be allocated for each MSI-X message. JHB seems to think remapping 978 * should be okay. This tunable should enable us to test that hypothesis 979 * when someone gets their hands on some Xeon hardware. 980 */ 981static int ntb_force_remap_mode; 982SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 983 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 984 " to a smaller number of ithreads, even if the desired number are " 985 "available"); 986 987/* 988 * In case it is NOT ok, give consumers an abort button. 989 */ 990static int ntb_prefer_intx; 991SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 992 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 993 "than remapping MSI-X messages over available slots (match Linux driver " 994 "behavior)"); 995 996/* 997 * Remap the desired number of MSI-X messages to available ithreads in a simple 998 * round-robin fashion. 999 */ 1000static int 1001ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1002{ 1003 u_int *vectors; 1004 uint32_t i; 1005 int rc; 1006 1007 if (ntb_prefer_intx != 0) 1008 return (ENXIO); 1009 1010 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1011 1012 for (i = 0; i < desired; i++) 1013 vectors[i] = (i % avail) + 1; 1014 1015 rc = pci_remap_msix(dev, desired, vectors); 1016 free(vectors, M_NTB); 1017 return (rc); 1018} 1019 1020static int 1021ntb_init_isr(struct ntb_softc *ntb) 1022{ 1023 uint32_t desired_vectors, num_vectors; 1024 int rc; 1025 1026 ntb->allocated_interrupts = 0; 1027 ntb->last_ts = ticks; 1028 1029 /* 1030 * Mask all doorbell interrupts. (Except link events!) 1031 */ 1032 DB_MASK_LOCK(ntb); 1033 ntb->db_mask = ntb->db_valid_mask; 1034 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1035 DB_MASK_UNLOCK(ntb); 1036 1037 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1038 ntb->db_count); 1039 if (desired_vectors >= 1) { 1040 rc = pci_alloc_msix(ntb->device, &num_vectors); 1041 1042 if (ntb_force_remap_mode != 0 && rc == 0 && 1043 num_vectors == desired_vectors) 1044 num_vectors--; 1045 1046 if (rc == 0 && num_vectors < desired_vectors) { 1047 rc = ntb_remap_msix(ntb->device, desired_vectors, 1048 num_vectors); 1049 if (rc == 0) 1050 num_vectors = desired_vectors; 1051 else 1052 pci_release_msi(ntb->device); 1053 } 1054 if (rc != 0) 1055 num_vectors = 1; 1056 } else 1057 num_vectors = 1; 1058 1059 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1060 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1061 device_printf(ntb->device, 1062 "Errata workaround does not support MSI or INTX\n"); 1063 return (EINVAL); 1064 } 1065 1066 ntb->db_vec_count = 1; 1067 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1068 rc = ntb_setup_legacy_interrupt(ntb); 1069 } else { 1070 ntb_create_msix_vec(ntb, num_vectors); 1071 rc = ntb_setup_msix(ntb, num_vectors); 1072 if (rc == 0 && HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1073 ntb_get_msix_info(ntb, num_vectors); 1074 } 1075 if (rc != 0) { 1076 device_printf(ntb->device, 1077 "Error allocating interrupts: %d\n", rc); 1078 ntb_free_msix_vec(ntb); 1079 } 1080 1081 return (rc); 1082} 1083 1084static int 1085ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1086{ 1087 int rc; 1088 1089 ntb->int_info[0].rid = 0; 1090 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1091 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1092 if (ntb->int_info[0].res == NULL) { 1093 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1094 return (ENOMEM); 1095 } 1096 1097 ntb->int_info[0].tag = NULL; 1098 ntb->allocated_interrupts = 1; 1099 1100 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1101 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1102 ntb, &ntb->int_info[0].tag); 1103 if (rc != 0) { 1104 device_printf(ntb->device, "bus_setup_intr failed\n"); 1105 return (ENXIO); 1106 } 1107 1108 return (0); 1109} 1110 1111static void 1112ntb_teardown_interrupts(struct ntb_softc *ntb) 1113{ 1114 struct ntb_int_info *current_int; 1115 int i; 1116 1117 for (i = 0; i < ntb->allocated_interrupts; i++) { 1118 current_int = &ntb->int_info[i]; 1119 if (current_int->tag != NULL) 1120 bus_teardown_intr(ntb->device, current_int->res, 1121 current_int->tag); 1122 1123 if (current_int->res != NULL) 1124 bus_release_resource(ntb->device, SYS_RES_IRQ, 1125 rman_get_rid(current_int->res), current_int->res); 1126 } 1127 1128 ntb_free_msix_vec(ntb); 1129 pci_release_msi(ntb->device); 1130} 1131 1132/* 1133 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1134 * out to make code clearer. 1135 */ 1136static inline uint64_t 1137db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1138{ 1139 1140 if (ntb->type == NTB_ATOM) 1141 return (ntb_reg_read(8, regoff)); 1142 1143 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1144 1145 return (ntb_reg_read(2, regoff)); 1146} 1147 1148static inline void 1149db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1150{ 1151 1152 KASSERT((val & ~ntb->db_valid_mask) == 0, 1153 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1154 (uintmax_t)(val & ~ntb->db_valid_mask), 1155 (uintmax_t)ntb->db_valid_mask)); 1156 1157 if (regoff == ntb->self_reg->db_mask) 1158 DB_MASK_ASSERT(ntb, MA_OWNED); 1159 db_iowrite_raw(ntb, regoff, val); 1160} 1161 1162static inline void 1163db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1164{ 1165 1166 if (ntb->type == NTB_ATOM) { 1167 ntb_reg_write(8, regoff, val); 1168 return; 1169 } 1170 1171 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1172 ntb_reg_write(2, regoff, (uint16_t)val); 1173} 1174 1175void 1176ntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1177{ 1178 1179 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1180 return; 1181 1182 DB_MASK_LOCK(ntb); 1183 ntb->db_mask |= bits; 1184 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1185 DB_MASK_UNLOCK(ntb); 1186} 1187 1188void 1189ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1190{ 1191 1192 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1193 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1194 (uintmax_t)(bits & ~ntb->db_valid_mask), 1195 (uintmax_t)ntb->db_valid_mask)); 1196 1197 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1198 return; 1199 1200 DB_MASK_LOCK(ntb); 1201 ntb->db_mask &= ~bits; 1202 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1203 DB_MASK_UNLOCK(ntb); 1204} 1205 1206uint64_t 1207ntb_db_read(struct ntb_softc *ntb) 1208{ 1209 1210 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1211 uint64_t res; 1212 unsigned i; 1213 1214 res = 0; 1215 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1216 if (ntb->msix_vec[i].masked != 0) 1217 res |= ntb_db_vector_mask(ntb, i); 1218 } 1219 return (res); 1220 } 1221 1222 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1223} 1224 1225void 1226ntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1227{ 1228 1229 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1230 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1231 (uintmax_t)(bits & ~ntb->db_valid_mask), 1232 (uintmax_t)ntb->db_valid_mask)); 1233 1234 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1235 unsigned i; 1236 1237 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1238 if ((bits & ntb_db_vector_mask(ntb, i)) != 0) { 1239 DB_MASK_LOCK(ntb); 1240 if (ntb->msix_vec[i].masked != 0) { 1241 /* XXX These need a public API. */ 1242#if 0 1243 pci_unmask_msix(ntb->device, i); 1244#endif 1245 ntb->msix_vec[i].masked = 0; 1246 } 1247 DB_MASK_UNLOCK(ntb); 1248 } 1249 } 1250 return; 1251 } 1252 1253 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1254} 1255 1256static inline uint64_t 1257ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1258{ 1259 uint64_t shift, mask; 1260 1261 shift = ntb->db_vec_shift; 1262 mask = (1ull << shift) - 1; 1263 return (mask << (shift * db_vector)); 1264} 1265 1266static void 1267ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1268{ 1269 uint64_t vec_mask; 1270 1271 ntb->last_ts = ticks; 1272 vec_mask = ntb_vec_mask(ntb, vec); 1273 1274 if ((vec_mask & ntb->db_link_mask) != 0) { 1275 if (ntb_poll_link(ntb)) 1276 ntb_link_event(ntb); 1277 } 1278 1279 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1280 (vec_mask & ntb->db_link_mask) == 0) { 1281 DB_MASK_LOCK(ntb); 1282 if (ntb->msix_vec[vec].masked == 0) { 1283 /* XXX These need a public API. */ 1284#if 0 1285 pci_mask_msix(ntb->device, vec); 1286#endif 1287 ntb->msix_vec[vec].masked = 1; 1288 } 1289 DB_MASK_UNLOCK(ntb); 1290 } 1291 1292 if ((vec_mask & ntb->db_valid_mask) != 0) 1293 ntb_db_event(ntb, vec); 1294} 1295 1296static void 1297ndev_vec_isr(void *arg) 1298{ 1299 struct ntb_vec *nvec = arg; 1300 1301 ntb_interrupt(nvec->ntb, nvec->num); 1302} 1303 1304static void 1305ndev_irq_isr(void *arg) 1306{ 1307 /* If we couldn't set up MSI-X, we only have the one vector. */ 1308 ntb_interrupt(arg, 0); 1309} 1310 1311static int 1312ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1313{ 1314 uint32_t i; 1315 1316 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1317 M_ZERO | M_WAITOK); 1318 for (i = 0; i < num_vectors; i++) { 1319 ntb->msix_vec[i].num = i; 1320 ntb->msix_vec[i].ntb = ntb; 1321 } 1322 1323 return (0); 1324} 1325 1326static void 1327ntb_free_msix_vec(struct ntb_softc *ntb) 1328{ 1329 1330 if (ntb->msix_vec == NULL) 1331 return; 1332 1333 free(ntb->msix_vec, M_NTB); 1334 ntb->msix_vec = NULL; 1335} 1336 1337static void 1338ntb_get_msix_info(struct ntb_softc *ntb, uint32_t num_vectors) 1339{ 1340 struct pci_devinfo *dinfo; 1341 struct pcicfg_msix *msix; 1342 uint32_t laddr, data, i, offset; 1343 1344 dinfo = device_get_ivars(ntb->device); 1345 msix = &dinfo->cfg.msix; 1346 1347 laddr = data = 0; 1348 1349 for (i = 0; i < num_vectors; i++) { 1350 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1351 1352 laddr = bus_read_4(msix->msix_table_res, offset + 1353 PCI_MSIX_ENTRY_LOWER_ADDR); 1354 ntb_printf(2, "local lower MSIX addr(%u): 0x%x\n", i, laddr); 1355 1356 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1357 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1358 MSI_INTEL_ADDR_BASE)); 1359 ntb->msix_data[i].nmd_ofs = laddr & ~MSI_INTEL_ADDR_BASE; 1360 1361 data = bus_read_4(msix->msix_table_res, offset + 1362 PCI_MSIX_ENTRY_DATA); 1363 ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1364 1365 ntb->msix_data[i].nmd_data = data; 1366 } 1367} 1368 1369static struct ntb_hw_info * 1370ntb_get_device_info(uint32_t device_id) 1371{ 1372 struct ntb_hw_info *ep = pci_ids; 1373 1374 while (ep->device_id) { 1375 if (ep->device_id == device_id) 1376 return (ep); 1377 ++ep; 1378 } 1379 return (NULL); 1380} 1381 1382static void 1383ntb_teardown_xeon(struct ntb_softc *ntb) 1384{ 1385 1386 if (ntb->reg != NULL) 1387 ntb_link_disable(ntb); 1388} 1389 1390static void 1391ntb_detect_max_mw(struct ntb_softc *ntb) 1392{ 1393 1394 if (ntb->type == NTB_ATOM) { 1395 ntb->mw_count = ATOM_MW_COUNT; 1396 return; 1397 } 1398 1399 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1400 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1401 else 1402 ntb->mw_count = XEON_SNB_MW_COUNT; 1403} 1404 1405static int 1406ntb_detect_xeon(struct ntb_softc *ntb) 1407{ 1408 uint8_t ppd, conn_type; 1409 1410 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1411 ntb->ppd = ppd; 1412 1413 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1414 ntb->dev_type = NTB_DEV_DSD; 1415 else 1416 ntb->dev_type = NTB_DEV_USD; 1417 1418 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1419 ntb->features |= NTB_SPLIT_BAR; 1420 1421 /* 1422 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1423 * errata workaround; only do one at a time. 1424 */ 1425 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1426 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1427 1428 conn_type = ppd & XEON_PPD_CONN_TYPE; 1429 switch (conn_type) { 1430 case NTB_CONN_B2B: 1431 ntb->conn_type = conn_type; 1432 break; 1433 case NTB_CONN_RP: 1434 case NTB_CONN_TRANSPARENT: 1435 default: 1436 device_printf(ntb->device, "Unsupported connection type: %u\n", 1437 (unsigned)conn_type); 1438 return (ENXIO); 1439 } 1440 return (0); 1441} 1442 1443static int 1444ntb_detect_atom(struct ntb_softc *ntb) 1445{ 1446 uint32_t ppd, conn_type; 1447 1448 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1449 ntb->ppd = ppd; 1450 1451 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1452 ntb->dev_type = NTB_DEV_DSD; 1453 else 1454 ntb->dev_type = NTB_DEV_USD; 1455 1456 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1457 switch (conn_type) { 1458 case NTB_CONN_B2B: 1459 ntb->conn_type = conn_type; 1460 break; 1461 default: 1462 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1463 return (ENXIO); 1464 } 1465 return (0); 1466} 1467 1468static int 1469ntb_xeon_init_dev(struct ntb_softc *ntb) 1470{ 1471 int rc; 1472 1473 ntb->spad_count = XEON_SPAD_COUNT; 1474 ntb->db_count = XEON_DB_COUNT; 1475 ntb->db_link_mask = XEON_DB_LINK_BIT; 1476 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1477 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1478 1479 if (ntb->conn_type != NTB_CONN_B2B) { 1480 device_printf(ntb->device, "Connection type %d not supported\n", 1481 ntb->conn_type); 1482 return (ENXIO); 1483 } 1484 1485 ntb->reg = &xeon_reg; 1486 ntb->self_reg = &xeon_pri_reg; 1487 ntb->peer_reg = &xeon_b2b_reg; 1488 ntb->xlat_reg = &xeon_sec_xlat; 1489 1490 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1491 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1492 ntb->mw_count; 1493 ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1494 g_ntb_msix_idx, ntb->msix_mw_idx); 1495 rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1496 VM_MEMATTR_UNCACHEABLE); 1497 KASSERT(rc == 0, ("shouldn't fail")); 1498 } else if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1499 /* 1500 * There is a Xeon hardware errata related to writes to SDOORBELL or 1501 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1502 * which may hang the system. To workaround this, use a memory 1503 * window to access the interrupt and scratch pad registers on the 1504 * remote system. 1505 */ 1506 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1507 ntb->mw_count; 1508 ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1509 g_ntb_mw_idx, ntb->b2b_mw_idx); 1510 rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1511 VM_MEMATTR_UNCACHEABLE); 1512 KASSERT(rc == 0, ("shouldn't fail")); 1513 } else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1514 /* 1515 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1516 * mirrored to the remote system. Shrink the number of bits by one, 1517 * since bit 14 is the last bit. 1518 * 1519 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1520 * anyway. Nor for non-B2B connection types. 1521 */ 1522 ntb->db_count = XEON_DB_COUNT - 1; 1523 1524 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1525 1526 if (ntb->dev_type == NTB_DEV_USD) 1527 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1528 &xeon_b2b_usd_addr); 1529 else 1530 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1531 &xeon_b2b_dsd_addr); 1532 if (rc != 0) 1533 return (rc); 1534 1535 /* Enable Bus Master and Memory Space on the secondary side */ 1536 ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1537 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1538 1539 /* 1540 * Mask all doorbell interrupts. 1541 */ 1542 DB_MASK_LOCK(ntb); 1543 ntb->db_mask = ntb->db_valid_mask; 1544 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1545 DB_MASK_UNLOCK(ntb); 1546 1547 rc = xeon_setup_msix_bar(ntb); 1548 if (rc != 0) 1549 return (rc); 1550 1551 rc = ntb_init_isr(ntb); 1552 return (rc); 1553} 1554 1555static int 1556ntb_atom_init_dev(struct ntb_softc *ntb) 1557{ 1558 int error; 1559 1560 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1561 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1562 1563 ntb->spad_count = ATOM_SPAD_COUNT; 1564 ntb->db_count = ATOM_DB_COUNT; 1565 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1566 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1567 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1568 1569 ntb->reg = &atom_reg; 1570 ntb->self_reg = &atom_pri_reg; 1571 ntb->peer_reg = &atom_b2b_reg; 1572 ntb->xlat_reg = &atom_sec_xlat; 1573 1574 /* 1575 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1576 * resolved. Mask transaction layer internal parity errors. 1577 */ 1578 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1579 1580 configure_atom_secondary_side_bars(ntb); 1581 1582 /* Enable Bus Master and Memory Space on the secondary side */ 1583 ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1584 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1585 1586 error = ntb_init_isr(ntb); 1587 if (error != 0) 1588 return (error); 1589 1590 /* Initiate PCI-E link training */ 1591 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1592 1593 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1594 1595 return (0); 1596} 1597 1598/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1599static void 1600configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1601{ 1602 1603 if (ntb->dev_type == NTB_DEV_USD) { 1604 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1605 XEON_B2B_BAR2_ADDR64); 1606 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1607 XEON_B2B_BAR4_ADDR64); 1608 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1609 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1610 } else { 1611 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1612 XEON_B2B_BAR2_ADDR64); 1613 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1614 XEON_B2B_BAR4_ADDR64); 1615 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1616 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1617 } 1618} 1619 1620 1621/* 1622 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1623 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1624 * remains for use by a higher layer. 1625 * 1626 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1627 * MW size is sufficiently large. 1628 */ 1629static unsigned int ntb_b2b_mw_share; 1630SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1631 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1632 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1633 "value here."); 1634 1635static void 1636xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1637 enum ntb_bar regbar) 1638{ 1639 struct ntb_pci_bar_info *bar; 1640 uint8_t bar_sz; 1641 1642 if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1643 return; 1644 1645 bar = &ntb->bar_info[idx]; 1646 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1647 if (idx == regbar) { 1648 if (ntb->b2b_off != 0) 1649 bar_sz--; 1650 else 1651 bar_sz = 0; 1652 } else if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1653 ntb_mw_to_bar(ntb, ntb->msix_mw_idx) == idx) { 1654 /* Restrict LAPIC BAR to 1MB */ 1655 pci_write_config(ntb->device, bar->psz_off, 20, 1); 1656 pci_write_config(ntb->device, bar->ssz_off, 20, 1); 1657 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1658 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1659 (void)bar_sz; 1660 return; 1661 } 1662 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1663 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1664 (void)bar_sz; 1665} 1666 1667static void 1668xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1669 enum ntb_bar idx, enum ntb_bar regbar) 1670{ 1671 uint64_t reg_val, lmt_addr; 1672 uint32_t base_reg, lmt_reg; 1673 1674 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1675 if (idx == regbar) 1676 bar_addr += ntb->b2b_off; 1677 lmt_addr = bar_addr; 1678 1679 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1680 ntb_mw_to_bar(ntb, ntb->msix_mw_idx) == idx) 1681 lmt_addr += ONE_MB; 1682 1683 /* 1684 * Set limit registers first to avoid an errata where setting the base 1685 * registers locks the limit registers. 1686 */ 1687 if (!bar_is_64bit(ntb, idx)) { 1688 ntb_reg_write(4, lmt_reg, lmt_addr); 1689 reg_val = ntb_reg_read(4, lmt_reg); 1690 (void)reg_val; 1691 1692 ntb_reg_write(4, base_reg, bar_addr); 1693 reg_val = ntb_reg_read(4, base_reg); 1694 (void)reg_val; 1695 } else { 1696 ntb_reg_write(8, lmt_reg, lmt_addr); 1697 reg_val = ntb_reg_read(8, lmt_reg); 1698 (void)reg_val; 1699 1700 ntb_reg_write(8, base_reg, bar_addr); 1701 reg_val = ntb_reg_read(8, base_reg); 1702 (void)reg_val; 1703 } 1704} 1705 1706static void 1707xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1708{ 1709 struct ntb_pci_bar_info *bar; 1710 1711 bar = &ntb->bar_info[idx]; 1712 if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1713 ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1714 base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1715 } else { 1716 ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1717 base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1718 } 1719 (void)base_addr; 1720} 1721 1722static int 1723xeon_setup_msix_bar(struct ntb_softc *ntb) 1724{ 1725 struct ntb_pci_bar_info *lapic_bar; 1726 enum ntb_bar bar_num; 1727 int rc; 1728 1729 if (!HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1730 return (0); 1731 1732 bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1733 lapic_bar = &ntb->bar_info[bar_num]; 1734 1735 /* Restrict LAPIC BAR to 1MB */ 1736 if (lapic_bar->size > ONE_MB) { 1737 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 1738 lapic_bar->pci_resource, lapic_bar->pbase, 1739 lapic_bar->pbase + ONE_MB - 1); 1740 if (rc == 0) 1741 lapic_bar->size = ONE_MB; 1742 else { 1743 ntb_printf(0, "Failed to shrink LAPIC BAR resource to " 1744 "1 MB: %d\n", rc); 1745 /* Ignore error */ 1746 } 1747 } 1748 1749 ntb->peer_lapic_bar = lapic_bar; 1750 return (0); 1751} 1752 1753static int 1754xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1755 const struct ntb_b2b_addr *peer_addr) 1756{ 1757 struct ntb_pci_bar_info *b2b_bar; 1758 vm_size_t bar_size; 1759 uint64_t bar_addr; 1760 enum ntb_bar b2b_bar_num, i; 1761 1762 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1763 b2b_bar = NULL; 1764 b2b_bar_num = NTB_CONFIG_BAR; 1765 ntb->b2b_off = 0; 1766 } else { 1767 b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1768 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1769 ("invalid b2b mw bar")); 1770 1771 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1772 bar_size = b2b_bar->size; 1773 1774 if (ntb_b2b_mw_share != 0 && 1775 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1776 ntb->b2b_off = bar_size >> 1; 1777 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1778 ntb->b2b_off = 0; 1779 } else { 1780 device_printf(ntb->device, 1781 "B2B bar size is too small!\n"); 1782 return (EIO); 1783 } 1784 } 1785 1786 /* 1787 * Reset the secondary bar sizes to match the primary bar sizes. 1788 * (Except, disable or halve the size of the B2B secondary bar.) 1789 */ 1790 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1791 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1792 1793 bar_addr = 0; 1794 if (b2b_bar_num == NTB_CONFIG_BAR) 1795 bar_addr = addr->bar0_addr; 1796 else if (b2b_bar_num == NTB_B2B_BAR_1) 1797 bar_addr = addr->bar2_addr64; 1798 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1799 bar_addr = addr->bar4_addr64; 1800 else if (b2b_bar_num == NTB_B2B_BAR_2) 1801 bar_addr = addr->bar4_addr32; 1802 else if (b2b_bar_num == NTB_B2B_BAR_3) 1803 bar_addr = addr->bar5_addr32; 1804 else 1805 KASSERT(false, ("invalid bar")); 1806 1807 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1808 1809 /* 1810 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1811 * register BAR. The B2B BAR is either disabled above or configured 1812 * half-size. It starts at PBAR xlat + offset. 1813 * 1814 * Also set up incoming BAR limits == base (zero length window). 1815 */ 1816 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1817 b2b_bar_num); 1818 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1819 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1820 NTB_B2B_BAR_2, b2b_bar_num); 1821 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1822 NTB_B2B_BAR_3, b2b_bar_num); 1823 } else 1824 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1825 NTB_B2B_BAR_2, b2b_bar_num); 1826 1827 /* Zero incoming translation addrs */ 1828 ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1829 ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1830 1831 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1832 size_t size, xlatoffset; 1833 1834 switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1835 case NTB_B2B_BAR_1: 1836 size = 8; 1837 xlatoffset = XEON_SBAR2XLAT_OFFSET; 1838 break; 1839 case NTB_B2B_BAR_2: 1840 xlatoffset = XEON_SBAR4XLAT_OFFSET; 1841 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1842 size = 4; 1843 else 1844 size = 8; 1845 break; 1846 case NTB_B2B_BAR_3: 1847 xlatoffset = XEON_SBAR5XLAT_OFFSET; 1848 size = 4; 1849 break; 1850 default: 1851 KASSERT(false, ("Bogus msix mw idx: %u", 1852 ntb->msix_mw_idx)); 1853 return (EINVAL); 1854 } 1855 1856 /* 1857 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1858 * workaround 1859 */ 1860 if (size == 4) 1861 ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1862 else 1863 ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1864 } 1865 (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1866 (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1867 1868 /* Zero outgoing translation limits (whole bar size windows) */ 1869 ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1870 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1871 1872 /* Set outgoing translation offsets */ 1873 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1874 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1875 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1876 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1877 } else 1878 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1879 1880 /* Set the translation offset for B2B registers */ 1881 bar_addr = 0; 1882 if (b2b_bar_num == NTB_CONFIG_BAR) 1883 bar_addr = peer_addr->bar0_addr; 1884 else if (b2b_bar_num == NTB_B2B_BAR_1) 1885 bar_addr = peer_addr->bar2_addr64; 1886 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1887 bar_addr = peer_addr->bar4_addr64; 1888 else if (b2b_bar_num == NTB_B2B_BAR_2) 1889 bar_addr = peer_addr->bar4_addr32; 1890 else if (b2b_bar_num == NTB_B2B_BAR_3) 1891 bar_addr = peer_addr->bar5_addr32; 1892 else 1893 KASSERT(false, ("invalid bar")); 1894 1895 /* 1896 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1897 * at a time. 1898 */ 1899 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1900 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1901 return (0); 1902} 1903 1904static inline bool 1905_xeon_link_is_up(struct ntb_softc *ntb) 1906{ 1907 1908 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1909 return (true); 1910 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1911} 1912 1913static inline bool 1914link_is_up(struct ntb_softc *ntb) 1915{ 1916 1917 if (ntb->type == NTB_XEON) 1918 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1919 !HAS_FEATURE(NTB_SB01BASE_LOCKUP))); 1920 1921 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1922 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1923} 1924 1925static inline bool 1926atom_link_is_err(struct ntb_softc *ntb) 1927{ 1928 uint32_t status; 1929 1930 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1931 1932 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1933 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1934 return (true); 1935 1936 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1937 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1938} 1939 1940/* Atom does not have link status interrupt, poll on that platform */ 1941static void 1942atom_link_hb(void *arg) 1943{ 1944 struct ntb_softc *ntb = arg; 1945 sbintime_t timo, poll_ts; 1946 1947 timo = NTB_HB_TIMEOUT * hz; 1948 poll_ts = ntb->last_ts + timo; 1949 1950 /* 1951 * Delay polling the link status if an interrupt was received, unless 1952 * the cached link status says the link is down. 1953 */ 1954 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1955 timo = poll_ts - ticks; 1956 goto out; 1957 } 1958 1959 if (ntb_poll_link(ntb)) 1960 ntb_link_event(ntb); 1961 1962 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1963 /* Link is down with error, proceed with recovery */ 1964 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1965 return; 1966 } 1967 1968out: 1969 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1970} 1971 1972static void 1973atom_perform_link_restart(struct ntb_softc *ntb) 1974{ 1975 uint32_t status; 1976 1977 /* Driver resets the NTB ModPhy lanes - magic! */ 1978 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1979 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1980 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1981 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1982 1983 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1984 pause("ModPhy", hz / 10); 1985 1986 /* Clear AER Errors, write to clear */ 1987 status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1988 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1989 ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1990 1991 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1992 status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1993 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1994 ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1995 1996 /* Clear DeSkew Buffer error, write to clear */ 1997 status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1998 status |= ATOM_DESKEWSTS_DBERR; 1999 ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 2000 2001 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2002 status &= ATOM_IBIST_ERR_OFLOW; 2003 ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2004 2005 /* Releases the NTB state machine to allow the link to retrain */ 2006 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2007 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2008 ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2009} 2010 2011/* 2012 * ntb_set_ctx() - associate a driver context with an ntb device 2013 * @ntb: NTB device context 2014 * @ctx: Driver context 2015 * @ctx_ops: Driver context operations 2016 * 2017 * Associate a driver context and operations with a ntb device. The context is 2018 * provided by the client driver, and the driver may associate a different 2019 * context with each ntb device. 2020 * 2021 * Return: Zero if the context is associated, otherwise an error number. 2022 */ 2023int 2024ntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 2025{ 2026 2027 if (ctx == NULL || ops == NULL) 2028 return (EINVAL); 2029 if (ntb->ctx_ops != NULL) 2030 return (EINVAL); 2031 2032 CTX_LOCK(ntb); 2033 if (ntb->ctx_ops != NULL) { 2034 CTX_UNLOCK(ntb); 2035 return (EINVAL); 2036 } 2037 ntb->ntb_ctx = ctx; 2038 ntb->ctx_ops = ops; 2039 CTX_UNLOCK(ntb); 2040 2041 return (0); 2042} 2043 2044/* 2045 * It is expected that this will only be used from contexts where the ctx_lock 2046 * is not needed to protect ntb_ctx lifetime. 2047 */ 2048void * 2049ntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 2050{ 2051 2052 KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2053 if (ops != NULL) 2054 *ops = ntb->ctx_ops; 2055 return (ntb->ntb_ctx); 2056} 2057 2058/* 2059 * ntb_clear_ctx() - disassociate any driver context from an ntb device 2060 * @ntb: NTB device context 2061 * 2062 * Clear any association that may exist between a driver context and the ntb 2063 * device. 2064 */ 2065void 2066ntb_clear_ctx(struct ntb_softc *ntb) 2067{ 2068 2069 CTX_LOCK(ntb); 2070 ntb->ntb_ctx = NULL; 2071 ntb->ctx_ops = NULL; 2072 CTX_UNLOCK(ntb); 2073} 2074 2075/* 2076 * ntb_link_event() - notify driver context of a change in link status 2077 * @ntb: NTB device context 2078 * 2079 * Notify the driver context that the link status may have changed. The driver 2080 * should call ntb_link_is_up() to get the current status. 2081 */ 2082void 2083ntb_link_event(struct ntb_softc *ntb) 2084{ 2085 2086 CTX_LOCK(ntb); 2087 if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2088 ntb->ctx_ops->link_event(ntb->ntb_ctx); 2089 CTX_UNLOCK(ntb); 2090} 2091 2092/* 2093 * ntb_db_event() - notify driver context of a doorbell event 2094 * @ntb: NTB device context 2095 * @vector: Interrupt vector number 2096 * 2097 * Notify the driver context of a doorbell event. If hardware supports 2098 * multiple interrupt vectors for doorbells, the vector number indicates which 2099 * vector received the interrupt. The vector number is relative to the first 2100 * vector used for doorbells, starting at zero, and must be less than 2101 * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2102 * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2103 * those bits are associated with the vector number. 2104 */ 2105static void 2106ntb_db_event(struct ntb_softc *ntb, uint32_t vec) 2107{ 2108 2109 CTX_LOCK(ntb); 2110 if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2111 ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2112 CTX_UNLOCK(ntb); 2113} 2114 2115/* 2116 * ntb_link_enable() - enable the link on the secondary side of the ntb 2117 * @ntb: NTB device context 2118 * @max_speed: The maximum link speed expressed as PCIe generation number[0] 2119 * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 2120 * 2121 * Enable the link on the secondary side of the ntb. This can only be done 2122 * from the primary side of the ntb in primary or b2b topology. The ntb device 2123 * should train the link to its maximum speed and width, or the requested speed 2124 * and width, whichever is smaller, if supported. 2125 * 2126 * Return: Zero on success, otherwise an error number. 2127 * 2128 * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 2129 * and width input will be ignored. 2130 */ 2131int 2132ntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 2133 enum ntb_width w __unused) 2134{ 2135 uint32_t cntl; 2136 2137 ntb_printf(2, "%s\n", __func__); 2138 2139 if (ntb->type == NTB_ATOM) { 2140 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2141 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2142 return (0); 2143 } 2144 2145 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2146 ntb_link_event(ntb); 2147 return (0); 2148 } 2149 2150 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2151 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2152 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2153 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2154 if (HAS_FEATURE(NTB_SPLIT_BAR)) 2155 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2156 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2157 return (0); 2158} 2159 2160/* 2161 * ntb_link_disable() - disable the link on the secondary side of the ntb 2162 * @ntb: NTB device context 2163 * 2164 * Disable the link on the secondary side of the ntb. This can only be done 2165 * from the primary side of the ntb in primary or b2b topology. The ntb device 2166 * should disable the link. Returning from this call must indicate that a 2167 * barrier has passed, though with no more writes may pass in either direction 2168 * across the link, except if this call returns an error number. 2169 * 2170 * Return: Zero on success, otherwise an error number. 2171 */ 2172int 2173ntb_link_disable(struct ntb_softc *ntb) 2174{ 2175 uint32_t cntl; 2176 2177 ntb_printf(2, "%s\n", __func__); 2178 2179 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2180 ntb_link_event(ntb); 2181 return (0); 2182 } 2183 2184 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2185 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2186 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2187 if (HAS_FEATURE(NTB_SPLIT_BAR)) 2188 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2189 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2190 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2191 return (0); 2192} 2193 2194bool 2195ntb_link_enabled(struct ntb_softc *ntb) 2196{ 2197 uint32_t cntl; 2198 2199 if (ntb->type == NTB_ATOM) { 2200 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2201 return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2202 } 2203 2204 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2205 return (true); 2206 2207 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2208 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2209} 2210 2211static void 2212recover_atom_link(void *arg) 2213{ 2214 struct ntb_softc *ntb = arg; 2215 unsigned speed, width, oldspeed, oldwidth; 2216 uint32_t status32; 2217 2218 atom_perform_link_restart(ntb); 2219 2220 /* 2221 * There is a potential race between the 2 NTB devices recovering at 2222 * the same time. If the times are the same, the link will not recover 2223 * and the driver will be stuck in this loop forever. Add a random 2224 * interval to the recovery time to prevent this race. 2225 */ 2226 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2227 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2228 2229 if (atom_link_is_err(ntb)) 2230 goto retry; 2231 2232 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2233 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2234 goto out; 2235 2236 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2237 width = NTB_LNK_STA_WIDTH(status32); 2238 speed = status32 & NTB_LINK_SPEED_MASK; 2239 2240 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2241 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2242 if (oldwidth != width || oldspeed != speed) 2243 goto retry; 2244 2245out: 2246 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2247 ntb); 2248 return; 2249 2250retry: 2251 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2252 ntb); 2253} 2254 2255/* 2256 * Polls the HW link status register(s); returns true if something has changed. 2257 */ 2258static bool 2259ntb_poll_link(struct ntb_softc *ntb) 2260{ 2261 uint32_t ntb_cntl; 2262 uint16_t reg_val; 2263 2264 if (ntb->type == NTB_ATOM) { 2265 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2266 if (ntb_cntl == ntb->ntb_ctl) 2267 return (false); 2268 2269 ntb->ntb_ctl = ntb_cntl; 2270 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2271 } else { 2272 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2273 2274 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2275 if (reg_val == ntb->lnk_sta) 2276 return (false); 2277 2278 ntb->lnk_sta = reg_val; 2279 2280 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 2281 if (_xeon_link_is_up(ntb)) { 2282 if (!ntb->peer_msix_good) { 2283 callout_reset(&ntb->peer_msix_work, 0, 2284 ntb_exchange_msix, ntb); 2285 return (false); 2286 } 2287 } else { 2288 ntb->peer_msix_good = false; 2289 ntb->peer_msix_done = false; 2290 } 2291 } 2292 } 2293 return (true); 2294} 2295 2296static inline enum ntb_speed 2297ntb_link_sta_speed(struct ntb_softc *ntb) 2298{ 2299 2300 if (!link_is_up(ntb)) 2301 return (NTB_SPEED_NONE); 2302 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2303} 2304 2305static inline enum ntb_width 2306ntb_link_sta_width(struct ntb_softc *ntb) 2307{ 2308 2309 if (!link_is_up(ntb)) 2310 return (NTB_WIDTH_NONE); 2311 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2312} 2313 2314SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2315 "Driver state, statistics, and HW registers"); 2316 2317#define NTB_REGSZ_MASK (3ul << 30) 2318#define NTB_REG_64 (1ul << 30) 2319#define NTB_REG_32 (2ul << 30) 2320#define NTB_REG_16 (3ul << 30) 2321#define NTB_REG_8 (0ul << 30) 2322 2323#define NTB_DB_READ (1ul << 29) 2324#define NTB_PCI_REG (1ul << 28) 2325#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2326 2327static void 2328ntb_sysctl_init(struct ntb_softc *ntb) 2329{ 2330 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2331 struct sysctl_ctx_list *ctx; 2332 struct sysctl_oid *tree, *tmptree; 2333 2334 ctx = device_get_sysctl_ctx(ntb->device); 2335 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2336 2337 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2338 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2339 sysctl_handle_link_status_human, "A", 2340 "Link status (human readable)"); 2341 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2342 CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2343 "IU", "Link status (1=active, 0=inactive)"); 2344 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2345 CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2346 "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2347 2348 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2349 CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2350 tree_par = SYSCTL_CHILDREN(tree); 2351 2352 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2353 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2354 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2355 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2356 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2357 &ntb->ppd, 0, "Raw PPD register (cached)"); 2358 2359 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2360 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2361 &ntb->b2b_mw_idx, 0, 2362 "Index of the MW used for B2B remote register access"); 2363 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2364 CTLFLAG_RD, &ntb->b2b_off, 2365 "If non-zero, offset of B2B register region in shared MW"); 2366 } 2367 2368 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2369 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2370 "Features/errata of this NTB device"); 2371 2372 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2373 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2374 "NTB CTL register (cached)"); 2375 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2376 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2377 "LNK STA register (cached)"); 2378 2379 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2380 &ntb->mw_count, 0, "MW count"); 2381 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2382 &ntb->spad_count, 0, "Scratchpad count"); 2383 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2384 &ntb->db_count, 0, "Doorbell count"); 2385 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2386 &ntb->db_vec_count, 0, "Doorbell vector count"); 2387 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2388 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2389 2390 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2391 &ntb->db_valid_mask, "Doorbell valid mask"); 2392 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2393 &ntb->db_link_mask, "Doorbell link mask"); 2394 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2395 &ntb->db_mask, "Doorbell mask (cached)"); 2396 2397 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2398 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2399 regpar = SYSCTL_CHILDREN(tmptree); 2400 2401 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2402 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2403 ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2404 "NTB Control register"); 2405 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2406 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2407 0x19c, sysctl_handle_register, "IU", 2408 "NTB Link Capabilities"); 2409 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2410 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2411 0x1a0, sysctl_handle_register, "IU", 2412 "NTB Link Control register"); 2413 2414 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2415 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2416 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2417 sysctl_handle_register, "QU", "Doorbell mask register"); 2418 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2419 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2420 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2421 sysctl_handle_register, "QU", "Doorbell register"); 2422 2423 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2424 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2425 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2426 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2427 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2428 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2429 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2430 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2431 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2432 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2433 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2434 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2435 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2436 } else { 2437 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2438 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2439 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2440 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2441 } 2442 2443 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2444 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2445 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2446 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2447 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2448 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2449 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2450 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2451 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2452 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2453 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2454 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2455 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2456 } else { 2457 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2458 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2459 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2460 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2461 } 2462 2463 if (ntb->type == NTB_ATOM) 2464 return; 2465 2466 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2467 CTLFLAG_RD, NULL, "Xeon HW statistics"); 2468 statpar = SYSCTL_CHILDREN(tmptree); 2469 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2470 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2471 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2472 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2473 2474 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2475 CTLFLAG_RD, NULL, "Xeon HW errors"); 2476 errpar = SYSCTL_CHILDREN(tmptree); 2477 2478 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2479 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2480 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2481 sysctl_handle_register, "CU", "PPD"); 2482 2483 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2484 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2485 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2486 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2487 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2488 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2489 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2490 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2491 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2492 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2493 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2494 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2495 2496 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2497 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2498 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2499 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2500 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2501 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2502 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2503 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2504 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2505 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2506 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2507 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2508 2509 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2510 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2511 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2512 sysctl_handle_register, "SU", "DEVSTS"); 2513 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2514 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2515 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2516 sysctl_handle_register, "SU", "LNKSTS"); 2517 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2518 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2519 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2520 sysctl_handle_register, "SU", "SLNKSTS"); 2521 2522 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2523 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2524 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2525 sysctl_handle_register, "IU", "UNCERRSTS"); 2526 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2527 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2528 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2529 sysctl_handle_register, "IU", "CORERRSTS"); 2530 2531 if (ntb->conn_type != NTB_CONN_B2B) 2532 return; 2533 2534 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2535 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2536 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2537 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2538 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2539 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2540 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2541 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2542 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2543 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2544 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2545 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2546 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2547 } else { 2548 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2549 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2550 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2551 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2552 } 2553 2554 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2555 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2556 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2557 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2558 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2559 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2560 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2561 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2562 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2563 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2564 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2565 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2566 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2567 } else { 2568 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2569 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2570 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2571 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2572 } 2573 2574 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2575 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2576 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2577 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2578 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2579 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2580 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2581 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2582 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2583 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2584 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2585 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2586 sysctl_handle_register, "IU", 2587 "Secondary BAR4 base register"); 2588 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2589 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2590 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2591 sysctl_handle_register, "IU", 2592 "Secondary BAR5 base register"); 2593 } else { 2594 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2595 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2596 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2597 sysctl_handle_register, "QU", 2598 "Secondary BAR45 base register"); 2599 } 2600} 2601 2602static int 2603sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2604{ 2605 struct ntb_softc *ntb; 2606 struct sbuf sb; 2607 int error; 2608 2609 error = 0; 2610 ntb = arg1; 2611 2612 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2613 2614 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2615 error = sbuf_finish(&sb); 2616 sbuf_delete(&sb); 2617 2618 if (error || !req->newptr) 2619 return (error); 2620 return (EINVAL); 2621} 2622 2623static int 2624sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2625{ 2626 struct ntb_softc *ntb; 2627 unsigned old, new; 2628 int error; 2629 2630 error = 0; 2631 ntb = arg1; 2632 2633 old = ntb_link_enabled(ntb); 2634 2635 error = SYSCTL_OUT(req, &old, sizeof(old)); 2636 if (error != 0 || req->newptr == NULL) 2637 return (error); 2638 2639 error = SYSCTL_IN(req, &new, sizeof(new)); 2640 if (error != 0) 2641 return (error); 2642 2643 ntb_printf(0, "Admin set interface state to '%sabled'\n", 2644 (new != 0)? "en" : "dis"); 2645 2646 if (new != 0) 2647 error = ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2648 else 2649 error = ntb_link_disable(ntb); 2650 return (error); 2651} 2652 2653static int 2654sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2655{ 2656 struct ntb_softc *ntb; 2657 struct sbuf sb; 2658 enum ntb_speed speed; 2659 enum ntb_width width; 2660 int error; 2661 2662 error = 0; 2663 ntb = arg1; 2664 2665 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2666 2667 if (ntb_link_is_up(ntb, &speed, &width)) 2668 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2669 (unsigned)speed, (unsigned)width); 2670 else 2671 sbuf_printf(&sb, "down"); 2672 2673 error = sbuf_finish(&sb); 2674 sbuf_delete(&sb); 2675 2676 if (error || !req->newptr) 2677 return (error); 2678 return (EINVAL); 2679} 2680 2681static int 2682sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2683{ 2684 struct ntb_softc *ntb; 2685 unsigned res; 2686 int error; 2687 2688 error = 0; 2689 ntb = arg1; 2690 2691 res = ntb_link_is_up(ntb, NULL, NULL); 2692 2693 error = SYSCTL_OUT(req, &res, sizeof(res)); 2694 if (error || !req->newptr) 2695 return (error); 2696 return (EINVAL); 2697} 2698 2699static int 2700sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2701{ 2702 struct ntb_softc *ntb; 2703 const void *outp; 2704 uintptr_t sz; 2705 uint64_t umv; 2706 char be[sizeof(umv)]; 2707 size_t outsz; 2708 uint32_t reg; 2709 bool db, pci; 2710 int error; 2711 2712 ntb = arg1; 2713 reg = arg2 & ~NTB_REGFLAGS_MASK; 2714 sz = arg2 & NTB_REGSZ_MASK; 2715 db = (arg2 & NTB_DB_READ) != 0; 2716 pci = (arg2 & NTB_PCI_REG) != 0; 2717 2718 KASSERT(!(db && pci), ("bogus")); 2719 2720 if (db) { 2721 KASSERT(sz == NTB_REG_64, ("bogus")); 2722 umv = db_ioread(ntb, reg); 2723 outsz = sizeof(uint64_t); 2724 } else { 2725 switch (sz) { 2726 case NTB_REG_64: 2727 if (pci) 2728 umv = pci_read_config(ntb->device, reg, 8); 2729 else 2730 umv = ntb_reg_read(8, reg); 2731 outsz = sizeof(uint64_t); 2732 break; 2733 case NTB_REG_32: 2734 if (pci) 2735 umv = pci_read_config(ntb->device, reg, 4); 2736 else 2737 umv = ntb_reg_read(4, reg); 2738 outsz = sizeof(uint32_t); 2739 break; 2740 case NTB_REG_16: 2741 if (pci) 2742 umv = pci_read_config(ntb->device, reg, 2); 2743 else 2744 umv = ntb_reg_read(2, reg); 2745 outsz = sizeof(uint16_t); 2746 break; 2747 case NTB_REG_8: 2748 if (pci) 2749 umv = pci_read_config(ntb->device, reg, 1); 2750 else 2751 umv = ntb_reg_read(1, reg); 2752 outsz = sizeof(uint8_t); 2753 break; 2754 default: 2755 panic("bogus"); 2756 break; 2757 } 2758 } 2759 2760 /* Encode bigendian so that sysctl -x is legible. */ 2761 be64enc(be, umv); 2762 outp = ((char *)be) + sizeof(umv) - outsz; 2763 2764 error = SYSCTL_OUT(req, outp, outsz); 2765 if (error || !req->newptr) 2766 return (error); 2767 return (EINVAL); 2768} 2769 2770static unsigned 2771ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2772{ 2773 2774 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2775 uidx >= ntb->b2b_mw_idx) || 2776 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2777 uidx++; 2778 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2779 uidx >= ntb->b2b_mw_idx) && 2780 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2781 uidx++; 2782 return (uidx); 2783} 2784 2785static void 2786ntb_exchange_msix(void *ctx) 2787{ 2788 struct ntb_softc *ntb; 2789 uint32_t val; 2790 unsigned i; 2791 2792 ntb = ctx; 2793 2794 if (ntb->peer_msix_done) 2795 goto msix_done; 2796 2797 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2798 ntb_peer_spad_write(ntb, NTB_MSIX_DATA0 + i, 2799 ntb->msix_data[i].nmd_data); 2800 ntb_peer_spad_write(ntb, NTB_MSIX_OFS0 + i, 2801 ntb->msix_data[i].nmd_ofs); 2802 } 2803 ntb_peer_spad_write(ntb, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2804 2805 ntb_spad_read(ntb, NTB_MSIX_GUARD, &val); 2806 if (val != NTB_MSIX_VER_GUARD) 2807 goto reschedule; 2808 2809 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2810 ntb_spad_read(ntb, NTB_MSIX_DATA0 + i, &val); 2811 ntb->peer_msix_data[i].nmd_data = val; 2812 ntb_spad_read(ntb, NTB_MSIX_OFS0 + i, &val); 2813 ntb->peer_msix_data[i].nmd_ofs = val; 2814 } 2815 2816 ntb->peer_msix_done = true; 2817 2818msix_done: 2819 ntb_peer_spad_write(ntb, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2820 ntb_spad_read(ntb, NTB_MSIX_DONE, &val); 2821 if (val != NTB_MSIX_RECEIVED) 2822 goto reschedule; 2823 2824 ntb->peer_msix_good = true; 2825 2826 ntb_poll_link(ntb); 2827 ntb_link_event(ntb); 2828 return; 2829 2830reschedule: 2831 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2832 if (_xeon_link_is_up(ntb)) 2833 callout_reset(&ntb->peer_msix_work, hz / 100, ntb_exchange_msix, ntb); 2834 else 2835 ntb_spad_clear(ntb); 2836} 2837 2838/* 2839 * Public API to the rest of the OS 2840 */ 2841 2842/** 2843 * ntb_get_max_spads() - get the total scratch regs usable 2844 * @ntb: pointer to ntb_softc instance 2845 * 2846 * This function returns the max 32bit scratchpad registers usable by the 2847 * upper layer. 2848 * 2849 * RETURNS: total number of scratch pad registers available 2850 */ 2851uint8_t 2852ntb_get_max_spads(struct ntb_softc *ntb) 2853{ 2854 2855 return (ntb->spad_count); 2856} 2857 2858/* 2859 * ntb_mw_count() - Get the number of memory windows available for KPI 2860 * consumers. 2861 * 2862 * (Excludes any MW wholly reserved for register access.) 2863 */ 2864uint8_t 2865ntb_mw_count(struct ntb_softc *ntb) 2866{ 2867 uint8_t res; 2868 2869 res = ntb->mw_count; 2870 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2871 res--; 2872 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2873 res--; 2874 return (res); 2875} 2876 2877/** 2878 * ntb_spad_write() - write to the secondary scratchpad register 2879 * @ntb: pointer to ntb_softc instance 2880 * @idx: index to the scratchpad register, 0 based 2881 * @val: the data value to put into the register 2882 * 2883 * This function allows writing of a 32bit value to the indexed scratchpad 2884 * register. The register resides on the secondary (external) side. 2885 * 2886 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2887 */ 2888int 2889ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2890{ 2891 2892 if (idx >= ntb->spad_count) 2893 return (EINVAL); 2894 2895 ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2896 2897 return (0); 2898} 2899 2900/* 2901 * Zeros the local scratchpad. 2902 */ 2903void 2904ntb_spad_clear(struct ntb_softc *ntb) 2905{ 2906 unsigned i; 2907 2908 for (i = 0; i < ntb->spad_count; i++) 2909 ntb_spad_write(ntb, i, 0); 2910} 2911 2912/** 2913 * ntb_spad_read() - read from the primary scratchpad register 2914 * @ntb: pointer to ntb_softc instance 2915 * @idx: index to scratchpad register, 0 based 2916 * @val: pointer to 32bit integer for storing the register value 2917 * 2918 * This function allows reading of the 32bit scratchpad register on 2919 * the primary (internal) side. 2920 * 2921 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2922 */ 2923int 2924ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2925{ 2926 2927 if (idx >= ntb->spad_count) 2928 return (EINVAL); 2929 2930 *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2931 2932 return (0); 2933} 2934 2935/** 2936 * ntb_peer_spad_write() - write to the secondary scratchpad register 2937 * @ntb: pointer to ntb_softc instance 2938 * @idx: index to the scratchpad register, 0 based 2939 * @val: the data value to put into the register 2940 * 2941 * This function allows writing of a 32bit value to the indexed scratchpad 2942 * register. The register resides on the secondary (external) side. 2943 * 2944 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2945 */ 2946int 2947ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2948{ 2949 2950 if (idx >= ntb->spad_count) 2951 return (EINVAL); 2952 2953 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2954 ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2955 else 2956 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2957 2958 return (0); 2959} 2960 2961/** 2962 * ntb_peer_spad_read() - read from the primary scratchpad register 2963 * @ntb: pointer to ntb_softc instance 2964 * @idx: index to scratchpad register, 0 based 2965 * @val: pointer to 32bit integer for storing the register value 2966 * 2967 * This function allows reading of the 32bit scratchpad register on 2968 * the primary (internal) side. 2969 * 2970 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2971 */ 2972int 2973ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2974{ 2975 2976 if (idx >= ntb->spad_count) 2977 return (EINVAL); 2978 2979 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2980 *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2981 else 2982 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2983 2984 return (0); 2985} 2986 2987/* 2988 * ntb_mw_get_range() - get the range of a memory window 2989 * @ntb: NTB device context 2990 * @idx: Memory window number 2991 * @base: OUT - the base address for mapping the memory window 2992 * @size: OUT - the size for mapping the memory window 2993 * @align: OUT - the base alignment for translating the memory window 2994 * @align_size: OUT - the size alignment for translating the memory window 2995 * 2996 * Get the range of a memory window. NULL may be given for any output 2997 * parameter if the value is not needed. The base and size may be used for 2998 * mapping the memory window, to access the peer memory. The alignment and 2999 * size may be used for translating the memory window, for the peer to access 3000 * memory on the local system. 3001 * 3002 * Return: Zero on success, otherwise an error number. 3003 */ 3004int 3005ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 3006 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 3007 bus_addr_t *plimit) 3008{ 3009 struct ntb_pci_bar_info *bar; 3010 bus_addr_t limit; 3011 size_t bar_b2b_off; 3012 enum ntb_bar bar_num; 3013 3014 if (mw_idx >= ntb_mw_count(ntb)) 3015 return (EINVAL); 3016 mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 3017 3018 bar_num = ntb_mw_to_bar(ntb, mw_idx); 3019 bar = &ntb->bar_info[bar_num]; 3020 bar_b2b_off = 0; 3021 if (mw_idx == ntb->b2b_mw_idx) { 3022 KASSERT(ntb->b2b_off != 0, 3023 ("user shouldn't get non-shared b2b mw")); 3024 bar_b2b_off = ntb->b2b_off; 3025 } 3026 3027 if (bar_is_64bit(ntb, bar_num)) 3028 limit = BUS_SPACE_MAXADDR; 3029 else 3030 limit = BUS_SPACE_MAXADDR_32BIT; 3031 3032 if (base != NULL) 3033 *base = bar->pbase + bar_b2b_off; 3034 if (vbase != NULL) 3035 *vbase = bar->vbase + bar_b2b_off; 3036 if (size != NULL) 3037 *size = bar->size - bar_b2b_off; 3038 if (align != NULL) 3039 *align = bar->size; 3040 if (align_size != NULL) 3041 *align_size = 1; 3042 if (plimit != NULL) 3043 *plimit = limit; 3044 return (0); 3045} 3046 3047/* 3048 * ntb_mw_set_trans() - set the translation of a memory window 3049 * @ntb: NTB device context 3050 * @idx: Memory window number 3051 * @addr: The dma address local memory to expose to the peer 3052 * @size: The size of the local memory to expose to the peer 3053 * 3054 * Set the translation of a memory window. The peer may access local memory 3055 * through the window starting at the address, up to the size. The address 3056 * must be aligned to the alignment specified by ntb_mw_get_range(). The size 3057 * must be aligned to the size alignment specified by ntb_mw_get_range(). The 3058 * address must be below the plimit specified by ntb_mw_get_range() (i.e. for 3059 * 32-bit BARs). 3060 * 3061 * Return: Zero on success, otherwise an error number. 3062 */ 3063int 3064ntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 3065 size_t size) 3066{ 3067 struct ntb_pci_bar_info *bar; 3068 uint64_t base, limit, reg_val; 3069 size_t bar_size, mw_size; 3070 uint32_t base_reg, xlat_reg, limit_reg; 3071 enum ntb_bar bar_num; 3072 3073 if (idx >= ntb_mw_count(ntb)) 3074 return (EINVAL); 3075 idx = ntb_user_mw_to_idx(ntb, idx); 3076 3077 bar_num = ntb_mw_to_bar(ntb, idx); 3078 bar = &ntb->bar_info[bar_num]; 3079 3080 bar_size = bar->size; 3081 if (idx == ntb->b2b_mw_idx) 3082 mw_size = bar_size - ntb->b2b_off; 3083 else 3084 mw_size = bar_size; 3085 3086 /* Hardware requires that addr is aligned to bar size */ 3087 if ((addr & (bar_size - 1)) != 0) 3088 return (EINVAL); 3089 3090 if (size > mw_size) 3091 return (EINVAL); 3092 3093 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 3094 3095 limit = 0; 3096 if (bar_is_64bit(ntb, bar_num)) { 3097 base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 3098 3099 if (limit_reg != 0 && size != mw_size) 3100 limit = base + size; 3101 3102 /* Set and verify translation address */ 3103 ntb_reg_write(8, xlat_reg, addr); 3104 reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 3105 if (reg_val != addr) { 3106 ntb_reg_write(8, xlat_reg, 0); 3107 return (EIO); 3108 } 3109 3110 /* Set and verify the limit */ 3111 ntb_reg_write(8, limit_reg, limit); 3112 reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3113 if (reg_val != limit) { 3114 ntb_reg_write(8, limit_reg, base); 3115 ntb_reg_write(8, xlat_reg, 0); 3116 return (EIO); 3117 } 3118 } else { 3119 /* Configure 32-bit (split) BAR MW */ 3120 3121 if ((addr & UINT32_MAX) != addr) 3122 return (ERANGE); 3123 if (((addr + size) & UINT32_MAX) != (addr + size)) 3124 return (ERANGE); 3125 3126 base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3127 3128 if (limit_reg != 0 && size != mw_size) 3129 limit = base + size; 3130 3131 /* Set and verify translation address */ 3132 ntb_reg_write(4, xlat_reg, addr); 3133 reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3134 if (reg_val != addr) { 3135 ntb_reg_write(4, xlat_reg, 0); 3136 return (EIO); 3137 } 3138 3139 /* Set and verify the limit */ 3140 ntb_reg_write(4, limit_reg, limit); 3141 reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3142 if (reg_val != limit) { 3143 ntb_reg_write(4, limit_reg, base); 3144 ntb_reg_write(4, xlat_reg, 0); 3145 return (EIO); 3146 } 3147 } 3148 return (0); 3149} 3150 3151/* 3152 * ntb_mw_clear_trans() - clear the translation of a memory window 3153 * @ntb: NTB device context 3154 * @idx: Memory window number 3155 * 3156 * Clear the translation of a memory window. The peer may no longer access 3157 * local memory through the window. 3158 * 3159 * Return: Zero on success, otherwise an error number. 3160 */ 3161int 3162ntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 3163{ 3164 3165 return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 3166} 3167 3168/* 3169 * ntb_mw_get_wc - Get the write-combine status of a memory window 3170 * 3171 * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 3172 * idx is an invalid memory window). 3173 * 3174 * Mode is a VM_MEMATTR_* type. 3175 */ 3176int 3177ntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t *mode) 3178{ 3179 struct ntb_pci_bar_info *bar; 3180 3181 if (idx >= ntb_mw_count(ntb)) 3182 return (EINVAL); 3183 idx = ntb_user_mw_to_idx(ntb, idx); 3184 3185 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3186 *mode = bar->map_mode; 3187 return (0); 3188} 3189 3190/* 3191 * ntb_mw_set_wc - Set the write-combine status of a memory window 3192 * 3193 * If 'mode' matches the current status, this does nothing and succeeds. Mode 3194 * is a VM_MEMATTR_* type. 3195 * 3196 * Returns: Zero on success, setting the caching attribute on the virtual 3197 * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 3198 * memory window, or if changing the caching attribute fails). 3199 */ 3200int 3201ntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3202{ 3203 3204 if (idx >= ntb_mw_count(ntb)) 3205 return (EINVAL); 3206 3207 idx = ntb_user_mw_to_idx(ntb, idx); 3208 return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3209} 3210 3211static int 3212ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3213{ 3214 struct ntb_pci_bar_info *bar; 3215 int rc; 3216 3217 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3218 if (bar->map_mode == mode) 3219 return (0); 3220 3221 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3222 if (rc == 0) 3223 bar->map_mode = mode; 3224 3225 return (rc); 3226} 3227 3228/** 3229 * ntb_peer_db_set() - Set the doorbell on the secondary/external side 3230 * @ntb: pointer to ntb_softc instance 3231 * @bit: doorbell bits to ring 3232 * 3233 * This function allows triggering of a doorbell on the secondary/external 3234 * side that will initiate an interrupt on the remote host 3235 */ 3236void 3237ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 3238{ 3239 3240 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 3241 struct ntb_pci_bar_info *lapic; 3242 unsigned i; 3243 3244 lapic = ntb->peer_lapic_bar; 3245 3246 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3247 if ((bit & ntb_db_vector_mask(ntb, i)) != 0) 3248 bus_space_write_4(lapic->pci_bus_tag, 3249 lapic->pci_bus_handle, 3250 ntb->peer_msix_data[i].nmd_ofs, 3251 ntb->peer_msix_data[i].nmd_data); 3252 } 3253 return; 3254 } 3255 3256 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3257 ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3258 return; 3259 } 3260 3261 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3262} 3263 3264/* 3265 * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 3266 * as well as the size of the register (via *sz_out). 3267 * 3268 * This function allows a caller using I/OAT DMA to chain the remote doorbell 3269 * ring to its memory window write. 3270 * 3271 * Note that writing the peer doorbell via a memory window will *not* generate 3272 * an interrupt on the remote host; that must be done separately. 3273 */ 3274bus_addr_t 3275ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 3276{ 3277 struct ntb_pci_bar_info *bar; 3278 uint64_t regoff; 3279 3280 KASSERT(sz_out != NULL, ("must be non-NULL")); 3281 3282 if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3283 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3284 regoff = ntb->peer_reg->db_bell; 3285 } else { 3286 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3287 ("invalid b2b idx")); 3288 3289 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3290 regoff = XEON_PDOORBELL_OFFSET; 3291 } 3292 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3293 3294 *sz_out = ntb->reg->db_size; 3295 /* HACK: Specific to current x86 bus implementation. */ 3296 return ((uint64_t)bar->pci_bus_handle + regoff); 3297} 3298 3299/* 3300 * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 3301 * @ntb: NTB device context 3302 * 3303 * Hardware may support different number or arrangement of doorbell bits. 3304 * 3305 * Return: A mask of doorbell bits supported by the ntb. 3306 */ 3307uint64_t 3308ntb_db_valid_mask(struct ntb_softc *ntb) 3309{ 3310 3311 return (ntb->db_valid_mask); 3312} 3313 3314/* 3315 * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 3316 * @ntb: NTB device context 3317 * @vector: Doorbell vector number 3318 * 3319 * Each interrupt vector may have a different number or arrangement of bits. 3320 * 3321 * Return: A mask of doorbell bits serviced by a vector. 3322 */ 3323uint64_t 3324ntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 3325{ 3326 3327 if (vector > ntb->db_vec_count) 3328 return (0); 3329 return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3330} 3331 3332/** 3333 * ntb_link_is_up() - get the current ntb link state 3334 * @ntb: NTB device context 3335 * @speed: OUT - The link speed expressed as PCIe generation number 3336 * @width: OUT - The link width expressed as the number of PCIe lanes 3337 * 3338 * RETURNS: true or false based on the hardware link state 3339 */ 3340bool 3341ntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 3342 enum ntb_width *width) 3343{ 3344 3345 if (speed != NULL) 3346 *speed = ntb_link_sta_speed(ntb); 3347 if (width != NULL) 3348 *width = ntb_link_sta_width(ntb); 3349 return (link_is_up(ntb)); 3350} 3351 3352static void 3353save_bar_parameters(struct ntb_pci_bar_info *bar) 3354{ 3355 3356 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3357 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3358 bar->pbase = rman_get_start(bar->pci_resource); 3359 bar->size = rman_get_size(bar->pci_resource); 3360 bar->vbase = rman_get_virtual(bar->pci_resource); 3361} 3362 3363device_t 3364ntb_get_device(struct ntb_softc *ntb) 3365{ 3366 3367 return (ntb->device); 3368} 3369 3370/* Export HW-specific errata information. */ 3371bool 3372ntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 3373{ 3374 3375 return (HAS_FEATURE(feature)); 3376} 3377