ntb_hw_intel.c revision 302483
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * Copyright (C) 2015 EMC Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 302483 2016-07-09 09:47:11Z mav $"); 30 31#include <sys/param.h> 32#include <sys/kernel.h> 33#include <sys/systm.h> 34#include <sys/bus.h> 35#include <sys/endian.h> 36#include <sys/malloc.h> 37#include <sys/module.h> 38#include <sys/mutex.h> 39#include <sys/pciio.h> 40#include <sys/queue.h> 41#include <sys/rman.h> 42#include <sys/sbuf.h> 43#include <sys/sysctl.h> 44#include <vm/vm.h> 45#include <vm/pmap.h> 46#include <machine/bus.h> 47#include <machine/intr_machdep.h> 48#include <machine/resource.h> 49#include <dev/pci/pcireg.h> 50#include <dev/pci/pcivar.h> 51 52#include "ntb_regs.h" 53#include "ntb_hw.h" 54 55/* 56 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 57 * allows you to connect two systems using a PCI-e link. 58 * 59 * This module contains the hardware abstraction layer for the NTB. It allows 60 * you to send and receive interrupts, map the memory windows and send and 61 * receive messages in the scratch-pad registers. 62 * 63 * NOTE: Much of the code in this module is shared with Linux. Any patches may 64 * be picked up and redistributed in Linux with a dual GPL/BSD license. 65 */ 66 67#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 68 69#define NTB_HB_TIMEOUT 1 /* second */ 70#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 71#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 72 73#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 74 75#define NTB_MSIX_VER_GUARD 0xaabbccdd 76#define NTB_MSIX_RECEIVED 0xe0f0e0f0 77 78/* 79 * PCI constants could be somewhere more generic, but aren't defined/used in 80 * pci.c. 81 */ 82#define PCI_MSIX_ENTRY_SIZE 16 83#define PCI_MSIX_ENTRY_LOWER_ADDR 0 84#define PCI_MSIX_ENTRY_UPPER_ADDR 4 85#define PCI_MSIX_ENTRY_DATA 8 86 87enum ntb_device_type { 88 NTB_XEON, 89 NTB_ATOM 90}; 91 92/* ntb_conn_type are hardware numbers, cannot change. */ 93enum ntb_conn_type { 94 NTB_CONN_TRANSPARENT = 0, 95 NTB_CONN_B2B = 1, 96 NTB_CONN_RP = 2, 97}; 98 99enum ntb_b2b_direction { 100 NTB_DEV_USD = 0, 101 NTB_DEV_DSD = 1, 102}; 103 104enum ntb_bar { 105 NTB_CONFIG_BAR = 0, 106 NTB_B2B_BAR_1, 107 NTB_B2B_BAR_2, 108 NTB_B2B_BAR_3, 109 NTB_MAX_BARS 110}; 111 112enum { 113 NTB_MSIX_GUARD = 0, 114 NTB_MSIX_DATA0, 115 NTB_MSIX_DATA1, 116 NTB_MSIX_DATA2, 117 NTB_MSIX_OFS0, 118 NTB_MSIX_OFS1, 119 NTB_MSIX_OFS2, 120 NTB_MSIX_DONE, 121 NTB_MAX_MSIX_SPAD 122}; 123 124/* Device features and workarounds */ 125#define HAS_FEATURE(feature) \ 126 ((ntb->features & (feature)) != 0) 127 128struct ntb_hw_info { 129 uint32_t device_id; 130 const char *desc; 131 enum ntb_device_type type; 132 uint32_t features; 133}; 134 135struct ntb_pci_bar_info { 136 bus_space_tag_t pci_bus_tag; 137 bus_space_handle_t pci_bus_handle; 138 int pci_resource_id; 139 struct resource *pci_resource; 140 vm_paddr_t pbase; 141 caddr_t vbase; 142 vm_size_t size; 143 vm_memattr_t map_mode; 144 145 /* Configuration register offsets */ 146 uint32_t psz_off; 147 uint32_t ssz_off; 148 uint32_t pbarxlat_off; 149}; 150 151struct ntb_int_info { 152 struct resource *res; 153 int rid; 154 void *tag; 155}; 156 157struct ntb_vec { 158 struct ntb_softc *ntb; 159 uint32_t num; 160 unsigned masked; 161}; 162 163struct ntb_reg { 164 uint32_t ntb_ctl; 165 uint32_t lnk_sta; 166 uint8_t db_size; 167 unsigned mw_bar[NTB_MAX_BARS]; 168}; 169 170struct ntb_alt_reg { 171 uint32_t db_bell; 172 uint32_t db_mask; 173 uint32_t spad; 174}; 175 176struct ntb_xlat_reg { 177 uint32_t bar0_base; 178 uint32_t bar2_base; 179 uint32_t bar4_base; 180 uint32_t bar5_base; 181 182 uint32_t bar2_xlat; 183 uint32_t bar4_xlat; 184 uint32_t bar5_xlat; 185 186 uint32_t bar2_limit; 187 uint32_t bar4_limit; 188 uint32_t bar5_limit; 189}; 190 191struct ntb_b2b_addr { 192 uint64_t bar0_addr; 193 uint64_t bar2_addr64; 194 uint64_t bar4_addr64; 195 uint64_t bar4_addr32; 196 uint64_t bar5_addr32; 197}; 198 199struct ntb_msix_data { 200 uint32_t nmd_ofs; 201 uint32_t nmd_data; 202}; 203 204struct ntb_softc { 205 device_t device; 206 enum ntb_device_type type; 207 uint32_t features; 208 209 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 210 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 211 uint32_t allocated_interrupts; 212 213 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 214 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215 bool peer_msix_good; 216 bool peer_msix_done; 217 struct ntb_pci_bar_info *peer_lapic_bar; 218 struct callout peer_msix_work; 219 220 struct callout heartbeat_timer; 221 struct callout lr_timer; 222 223 void *ntb_ctx; 224 const struct ntb_ctx_ops *ctx_ops; 225 struct ntb_vec *msix_vec; 226#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 227#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 228#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 229 struct mtx ctx_lock; 230 231 uint32_t ppd; 232 enum ntb_conn_type conn_type; 233 enum ntb_b2b_direction dev_type; 234 235 /* Offset of peer bar0 in B2B BAR */ 236 uint64_t b2b_off; 237 /* Memory window used to access peer bar0 */ 238#define B2B_MW_DISABLED UINT8_MAX 239 uint8_t b2b_mw_idx; 240 uint32_t msix_xlat; 241 uint8_t msix_mw_idx; 242 243 uint8_t mw_count; 244 uint8_t spad_count; 245 uint8_t db_count; 246 uint8_t db_vec_count; 247 uint8_t db_vec_shift; 248 249 /* Protects local db_mask. */ 250#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 251#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 252#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 253 struct mtx db_mask_lock; 254 255 volatile uint32_t ntb_ctl; 256 volatile uint32_t lnk_sta; 257 258 uint64_t db_valid_mask; 259 uint64_t db_link_mask; 260 uint64_t db_mask; 261 262 int last_ts; /* ticks @ last irq */ 263 264 const struct ntb_reg *reg; 265 const struct ntb_alt_reg *self_reg; 266 const struct ntb_alt_reg *peer_reg; 267 const struct ntb_xlat_reg *xlat_reg; 268}; 269 270#ifdef __i386__ 271static __inline uint64_t 272bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 273 bus_size_t offset) 274{ 275 276 return (bus_space_read_4(tag, handle, offset) | 277 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 278} 279 280static __inline void 281bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 282 bus_size_t offset, uint64_t val) 283{ 284 285 bus_space_write_4(tag, handle, offset, val); 286 bus_space_write_4(tag, handle, offset + 4, val >> 32); 287} 288#endif 289 290#define ntb_bar_read(SIZE, bar, offset) \ 291 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 292 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 293#define ntb_bar_write(SIZE, bar, offset, val) \ 294 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 295 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 296#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 297#define ntb_reg_write(SIZE, offset, val) \ 298 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 299#define ntb_mw_read(SIZE, offset) \ 300 ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 301#define ntb_mw_write(SIZE, offset, val) \ 302 ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 303 offset, val) 304 305static int ntb_probe(device_t device); 306static int ntb_attach(device_t device); 307static int ntb_detach(device_t device); 308static unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 309static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 310static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 311static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 312 uint32_t *base, uint32_t *xlat, uint32_t *lmt); 313static int ntb_map_pci_bars(struct ntb_softc *ntb); 314static int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 315 vm_memattr_t); 316static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 317 const char *); 318static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 319static int map_memory_window_bar(struct ntb_softc *ntb, 320 struct ntb_pci_bar_info *bar); 321static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 322static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 323static int ntb_init_isr(struct ntb_softc *ntb); 324static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 325static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 326static void ntb_teardown_interrupts(struct ntb_softc *ntb); 327static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 328static void ntb_interrupt(struct ntb_softc *, uint32_t vec); 329static void ndev_vec_isr(void *arg); 330static void ndev_irq_isr(void *arg); 331static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 332static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 333static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 334static int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 335static void ntb_free_msix_vec(struct ntb_softc *ntb); 336static void ntb_get_msix_info(struct ntb_softc *ntb); 337static void ntb_exchange_msix(void *); 338static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 339static void ntb_detect_max_mw(struct ntb_softc *ntb); 340static int ntb_detect_xeon(struct ntb_softc *ntb); 341static int ntb_detect_atom(struct ntb_softc *ntb); 342static int ntb_xeon_init_dev(struct ntb_softc *ntb); 343static int ntb_atom_init_dev(struct ntb_softc *ntb); 344static void ntb_teardown_xeon(struct ntb_softc *ntb); 345static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 346static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 347 enum ntb_bar regbar); 348static void xeon_set_sbar_base_and_limit(struct ntb_softc *, 349 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 350static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 351 enum ntb_bar idx); 352static int xeon_setup_b2b_mw(struct ntb_softc *, 353 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 354static int xeon_setup_msix_bar(struct ntb_softc *); 355static inline bool link_is_up(struct ntb_softc *ntb); 356static inline bool _xeon_link_is_up(struct ntb_softc *ntb); 357static inline bool atom_link_is_err(struct ntb_softc *ntb); 358static inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 359static inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 360static void atom_link_hb(void *arg); 361static void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 362static void recover_atom_link(void *arg); 363static bool ntb_poll_link(struct ntb_softc *ntb); 364static void save_bar_parameters(struct ntb_pci_bar_info *bar); 365static void ntb_sysctl_init(struct ntb_softc *); 366static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 367static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 368static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 369static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 370static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 371 372static unsigned g_ntb_hw_debug_level; 373SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 374 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 375#define ntb_printf(lvl, ...) do { \ 376 if ((lvl) <= g_ntb_hw_debug_level) { \ 377 device_printf(ntb->device, __VA_ARGS__); \ 378 } \ 379} while (0) 380 381#define _NTB_PAT_UC 0 382#define _NTB_PAT_WC 1 383#define _NTB_PAT_WT 4 384#define _NTB_PAT_WP 5 385#define _NTB_PAT_WB 6 386#define _NTB_PAT_UCM 7 387static unsigned g_ntb_mw_pat = _NTB_PAT_UC; 388SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 389 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 390 "UC: " __XSTRING(_NTB_PAT_UC) ", " 391 "WC: " __XSTRING(_NTB_PAT_WC) ", " 392 "WT: " __XSTRING(_NTB_PAT_WT) ", " 393 "WP: " __XSTRING(_NTB_PAT_WP) ", " 394 "WB: " __XSTRING(_NTB_PAT_WB) ", " 395 "UC-: " __XSTRING(_NTB_PAT_UCM)); 396 397static inline vm_memattr_t 398ntb_pat_flags(void) 399{ 400 401 switch (g_ntb_mw_pat) { 402 case _NTB_PAT_WC: 403 return (VM_MEMATTR_WRITE_COMBINING); 404 case _NTB_PAT_WT: 405 return (VM_MEMATTR_WRITE_THROUGH); 406 case _NTB_PAT_WP: 407 return (VM_MEMATTR_WRITE_PROTECTED); 408 case _NTB_PAT_WB: 409 return (VM_MEMATTR_WRITE_BACK); 410 case _NTB_PAT_UCM: 411 return (VM_MEMATTR_WEAK_UNCACHEABLE); 412 case _NTB_PAT_UC: 413 /* FALLTHROUGH */ 414 default: 415 return (VM_MEMATTR_UNCACHEABLE); 416 } 417} 418 419/* 420 * Well, this obviously doesn't belong here, but it doesn't seem to exist 421 * anywhere better yet. 422 */ 423static inline const char * 424ntb_vm_memattr_to_str(vm_memattr_t pat) 425{ 426 427 switch (pat) { 428 case VM_MEMATTR_WRITE_COMBINING: 429 return ("WRITE_COMBINING"); 430 case VM_MEMATTR_WRITE_THROUGH: 431 return ("WRITE_THROUGH"); 432 case VM_MEMATTR_WRITE_PROTECTED: 433 return ("WRITE_PROTECTED"); 434 case VM_MEMATTR_WRITE_BACK: 435 return ("WRITE_BACK"); 436 case VM_MEMATTR_WEAK_UNCACHEABLE: 437 return ("UNCACHED"); 438 case VM_MEMATTR_UNCACHEABLE: 439 return ("UNCACHEABLE"); 440 default: 441 return ("UNKNOWN"); 442 } 443} 444 445static int g_ntb_msix_idx = 0; 446SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 447 0, "Use this memory window to access the peer MSIX message complex on " 448 "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 449 "Like b2b_mw_idx, negative values index from the last available memory " 450 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 451 452static int g_ntb_mw_idx = -1; 453SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 454 0, "Use this memory window to access the peer NTB registers. A " 455 "non-negative value starts from the first MW index; a negative value " 456 "starts from the last MW index. The default is -1, i.e., the last " 457 "available memory window. Both sides of the NTB MUST set the same " 458 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 459 460static struct ntb_hw_info pci_ids[] = { 461 /* XXX: PS/SS IDs left out until they are supported. */ 462 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 463 NTB_ATOM, 0 }, 464 465 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 466 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 467 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 468 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 469 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 470 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 471 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 472 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 473 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 474 NTB_SB01BASE_LOCKUP }, 475 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 476 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 477 NTB_SB01BASE_LOCKUP }, 478 479 { 0x00000000, NULL, NTB_ATOM, 0 } 480}; 481 482static const struct ntb_reg atom_reg = { 483 .ntb_ctl = ATOM_NTBCNTL_OFFSET, 484 .lnk_sta = ATOM_LINK_STATUS_OFFSET, 485 .db_size = sizeof(uint64_t), 486 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 487}; 488 489static const struct ntb_alt_reg atom_pri_reg = { 490 .db_bell = ATOM_PDOORBELL_OFFSET, 491 .db_mask = ATOM_PDBMSK_OFFSET, 492 .spad = ATOM_SPAD_OFFSET, 493}; 494 495static const struct ntb_alt_reg atom_b2b_reg = { 496 .db_bell = ATOM_B2B_DOORBELL_OFFSET, 497 .spad = ATOM_B2B_SPAD_OFFSET, 498}; 499 500static const struct ntb_xlat_reg atom_sec_xlat = { 501#if 0 502 /* "FIXME" says the Linux driver. */ 503 .bar0_base = ATOM_SBAR0BASE_OFFSET, 504 .bar2_base = ATOM_SBAR2BASE_OFFSET, 505 .bar4_base = ATOM_SBAR4BASE_OFFSET, 506 507 .bar2_limit = ATOM_SBAR2LMT_OFFSET, 508 .bar4_limit = ATOM_SBAR4LMT_OFFSET, 509#endif 510 511 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 512 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 513}; 514 515static const struct ntb_reg xeon_reg = { 516 .ntb_ctl = XEON_NTBCNTL_OFFSET, 517 .lnk_sta = XEON_LINK_STATUS_OFFSET, 518 .db_size = sizeof(uint16_t), 519 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 520}; 521 522static const struct ntb_alt_reg xeon_pri_reg = { 523 .db_bell = XEON_PDOORBELL_OFFSET, 524 .db_mask = XEON_PDBMSK_OFFSET, 525 .spad = XEON_SPAD_OFFSET, 526}; 527 528static const struct ntb_alt_reg xeon_b2b_reg = { 529 .db_bell = XEON_B2B_DOORBELL_OFFSET, 530 .spad = XEON_B2B_SPAD_OFFSET, 531}; 532 533static const struct ntb_xlat_reg xeon_sec_xlat = { 534 .bar0_base = XEON_SBAR0BASE_OFFSET, 535 .bar2_base = XEON_SBAR2BASE_OFFSET, 536 .bar4_base = XEON_SBAR4BASE_OFFSET, 537 .bar5_base = XEON_SBAR5BASE_OFFSET, 538 539 .bar2_limit = XEON_SBAR2LMT_OFFSET, 540 .bar4_limit = XEON_SBAR4LMT_OFFSET, 541 .bar5_limit = XEON_SBAR5LMT_OFFSET, 542 543 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 544 .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 545 .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 546}; 547 548static struct ntb_b2b_addr xeon_b2b_usd_addr = { 549 .bar0_addr = XEON_B2B_BAR0_ADDR, 550 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 551 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 552 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 553 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 554}; 555 556static struct ntb_b2b_addr xeon_b2b_dsd_addr = { 557 .bar0_addr = XEON_B2B_BAR0_ADDR, 558 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 559 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 560 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 561 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 562}; 563 564SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 565 "B2B MW segment overrides -- MUST be the same on both sides"); 566 567SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 568 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 569 "hardware, use this 64-bit address on the bus between the NTB devices for " 570 "the window at BAR2, on the upstream side of the link. MUST be the same " 571 "address on both sides."); 572SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 573 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 574SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 575 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 576 "(split-BAR mode)."); 577SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 578 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 579 "(split-BAR mode)."); 580 581SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 582 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 583 "hardware, use this 64-bit address on the bus between the NTB devices for " 584 "the window at BAR2, on the downstream side of the link. MUST be the same" 585 " address on both sides."); 586SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 587 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 588SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 589 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 590 "(split-BAR mode)."); 591SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 592 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 593 "(split-BAR mode)."); 594 595/* 596 * OS <-> Driver interface structures 597 */ 598MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 599 600static device_method_t ntb_pci_methods[] = { 601 /* Device interface */ 602 DEVMETHOD(device_probe, ntb_probe), 603 DEVMETHOD(device_attach, ntb_attach), 604 DEVMETHOD(device_detach, ntb_detach), 605 DEVMETHOD_END 606}; 607 608static driver_t ntb_pci_driver = { 609 "ntb_hw", 610 ntb_pci_methods, 611 sizeof(struct ntb_softc), 612}; 613 614static devclass_t ntb_devclass; 615DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 616MODULE_VERSION(ntb_hw, 1); 617 618SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 619 620/* 621 * OS <-> Driver linkage functions 622 */ 623static int 624ntb_probe(device_t device) 625{ 626 struct ntb_hw_info *p; 627 628 p = ntb_get_device_info(pci_get_devid(device)); 629 if (p == NULL) 630 return (ENXIO); 631 632 device_set_desc(device, p->desc); 633 return (0); 634} 635 636static int 637ntb_attach(device_t device) 638{ 639 struct ntb_softc *ntb; 640 struct ntb_hw_info *p; 641 int error; 642 643 ntb = DEVICE2SOFTC(device); 644 p = ntb_get_device_info(pci_get_devid(device)); 645 646 ntb->device = device; 647 ntb->type = p->type; 648 ntb->features = p->features; 649 ntb->b2b_mw_idx = B2B_MW_DISABLED; 650 ntb->msix_mw_idx = B2B_MW_DISABLED; 651 652 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 653 callout_init(&ntb->heartbeat_timer, 1); 654 callout_init(&ntb->lr_timer, 1); 655 callout_init(&ntb->peer_msix_work, 1); 656 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 657 mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 658 659 if (ntb->type == NTB_ATOM) 660 error = ntb_detect_atom(ntb); 661 else 662 error = ntb_detect_xeon(ntb); 663 if (error != 0) 664 goto out; 665 666 ntb_detect_max_mw(ntb); 667 668 pci_enable_busmaster(ntb->device); 669 670 error = ntb_map_pci_bars(ntb); 671 if (error != 0) 672 goto out; 673 if (ntb->type == NTB_ATOM) 674 error = ntb_atom_init_dev(ntb); 675 else 676 error = ntb_xeon_init_dev(ntb); 677 if (error != 0) 678 goto out; 679 680 ntb_spad_clear(ntb); 681 682 ntb_poll_link(ntb); 683 684 ntb_sysctl_init(ntb); 685 686out: 687 if (error != 0) 688 ntb_detach(device); 689 return (error); 690} 691 692static int 693ntb_detach(device_t device) 694{ 695 struct ntb_softc *ntb; 696 697 ntb = DEVICE2SOFTC(device); 698 699 if (ntb->self_reg != NULL) { 700 DB_MASK_LOCK(ntb); 701 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 702 DB_MASK_UNLOCK(ntb); 703 } 704 callout_drain(&ntb->heartbeat_timer); 705 callout_drain(&ntb->lr_timer); 706 callout_drain(&ntb->peer_msix_work); 707 pci_disable_busmaster(ntb->device); 708 if (ntb->type == NTB_XEON) 709 ntb_teardown_xeon(ntb); 710 ntb_teardown_interrupts(ntb); 711 712 mtx_destroy(&ntb->db_mask_lock); 713 mtx_destroy(&ntb->ctx_lock); 714 715 ntb_unmap_pci_bar(ntb); 716 717 return (0); 718} 719 720/* 721 * Driver internal routines 722 */ 723static inline enum ntb_bar 724ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 725{ 726 727 KASSERT(mw < ntb->mw_count, 728 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 729 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 730 731 return (ntb->reg->mw_bar[mw]); 732} 733 734static inline bool 735bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 736{ 737 /* XXX This assertion could be stronger. */ 738 KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 739 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 740} 741 742static inline void 743bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 744 uint32_t *xlat, uint32_t *lmt) 745{ 746 uint32_t basev, lmtv, xlatv; 747 748 switch (bar) { 749 case NTB_B2B_BAR_1: 750 basev = ntb->xlat_reg->bar2_base; 751 lmtv = ntb->xlat_reg->bar2_limit; 752 xlatv = ntb->xlat_reg->bar2_xlat; 753 break; 754 case NTB_B2B_BAR_2: 755 basev = ntb->xlat_reg->bar4_base; 756 lmtv = ntb->xlat_reg->bar4_limit; 757 xlatv = ntb->xlat_reg->bar4_xlat; 758 break; 759 case NTB_B2B_BAR_3: 760 basev = ntb->xlat_reg->bar5_base; 761 lmtv = ntb->xlat_reg->bar5_limit; 762 xlatv = ntb->xlat_reg->bar5_xlat; 763 break; 764 default: 765 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 766 ("bad bar")); 767 basev = lmtv = xlatv = 0; 768 break; 769 } 770 771 if (base != NULL) 772 *base = basev; 773 if (xlat != NULL) 774 *xlat = xlatv; 775 if (lmt != NULL) 776 *lmt = lmtv; 777} 778 779static int 780ntb_map_pci_bars(struct ntb_softc *ntb) 781{ 782 int rc; 783 784 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 785 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 786 if (rc != 0) 787 goto out; 788 789 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 790 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 791 if (rc != 0) 792 goto out; 793 ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 794 ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 795 ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 796 797 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 798 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 799 if (rc != 0) 800 goto out; 801 ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 802 ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 803 ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 804 805 if (!HAS_FEATURE(NTB_SPLIT_BAR)) 806 goto out; 807 808 ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 809 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 810 ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 811 ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 812 ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 813 814out: 815 if (rc != 0) 816 device_printf(ntb->device, 817 "unable to allocate pci resource\n"); 818 return (rc); 819} 820 821static void 822print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 823 const char *kind) 824{ 825 826 device_printf(ntb->device, 827 "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 828 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 829 (char *)bar->vbase + bar->size - 1, 830 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 831 (uintmax_t)bar->size, kind); 832} 833 834static int 835map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 836{ 837 838 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 839 &bar->pci_resource_id, RF_ACTIVE); 840 if (bar->pci_resource == NULL) 841 return (ENXIO); 842 843 save_bar_parameters(bar); 844 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 845 print_map_success(ntb, bar, "mmr"); 846 return (0); 847} 848 849static int 850map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 851{ 852 int rc; 853 vm_memattr_t mapmode; 854 uint8_t bar_size_bits = 0; 855 856 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 857 &bar->pci_resource_id, RF_ACTIVE); 858 859 if (bar->pci_resource == NULL) 860 return (ENXIO); 861 862 save_bar_parameters(bar); 863 /* 864 * Ivytown NTB BAR sizes are misreported by the hardware due to a 865 * hardware issue. To work around this, query the size it should be 866 * configured to by the device and modify the resource to correspond to 867 * this new size. The BIOS on systems with this problem is required to 868 * provide enough address space to allow the driver to make this change 869 * safely. 870 * 871 * Ideally I could have just specified the size when I allocated the 872 * resource like: 873 * bus_alloc_resource(ntb->device, 874 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 875 * 1ul << bar_size_bits, RF_ACTIVE); 876 * but the PCI driver does not honor the size in this call, so we have 877 * to modify it after the fact. 878 */ 879 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 880 if (bar->pci_resource_id == PCIR_BAR(2)) 881 bar_size_bits = pci_read_config(ntb->device, 882 XEON_PBAR23SZ_OFFSET, 1); 883 else 884 bar_size_bits = pci_read_config(ntb->device, 885 XEON_PBAR45SZ_OFFSET, 1); 886 887 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 888 bar->pci_resource, bar->pbase, 889 bar->pbase + (1ul << bar_size_bits) - 1); 890 if (rc != 0) { 891 device_printf(ntb->device, 892 "unable to resize bar\n"); 893 return (rc); 894 } 895 896 save_bar_parameters(bar); 897 } 898 899 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 900 print_map_success(ntb, bar, "mw"); 901 902 /* 903 * Optionally, mark MW BARs as anything other than UC to improve 904 * performance. 905 */ 906 mapmode = ntb_pat_flags(); 907 if (mapmode == bar->map_mode) 908 return (0); 909 910 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 911 if (rc == 0) { 912 bar->map_mode = mapmode; 913 device_printf(ntb->device, 914 "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 915 "%s.\n", 916 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 917 (char *)bar->vbase + bar->size - 1, 918 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 919 ntb_vm_memattr_to_str(mapmode)); 920 } else 921 device_printf(ntb->device, 922 "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 923 "%s: %d\n", 924 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 925 (char *)bar->vbase + bar->size - 1, 926 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 927 ntb_vm_memattr_to_str(mapmode), rc); 928 /* Proceed anyway */ 929 return (0); 930} 931 932static void 933ntb_unmap_pci_bar(struct ntb_softc *ntb) 934{ 935 struct ntb_pci_bar_info *current_bar; 936 int i; 937 938 for (i = 0; i < NTB_MAX_BARS; i++) { 939 current_bar = &ntb->bar_info[i]; 940 if (current_bar->pci_resource != NULL) 941 bus_release_resource(ntb->device, SYS_RES_MEMORY, 942 current_bar->pci_resource_id, 943 current_bar->pci_resource); 944 } 945} 946 947static int 948ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 949{ 950 uint32_t i; 951 int rc; 952 953 for (i = 0; i < num_vectors; i++) { 954 ntb->int_info[i].rid = i + 1; 955 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 956 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 957 if (ntb->int_info[i].res == NULL) { 958 device_printf(ntb->device, 959 "bus_alloc_resource failed\n"); 960 return (ENOMEM); 961 } 962 ntb->int_info[i].tag = NULL; 963 ntb->allocated_interrupts++; 964 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 965 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 966 &ntb->msix_vec[i], &ntb->int_info[i].tag); 967 if (rc != 0) { 968 device_printf(ntb->device, "bus_setup_intr failed\n"); 969 return (ENXIO); 970 } 971 } 972 return (0); 973} 974 975/* 976 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 977 * cannot be allocated for each MSI-X message. JHB seems to think remapping 978 * should be okay. This tunable should enable us to test that hypothesis 979 * when someone gets their hands on some Xeon hardware. 980 */ 981static int ntb_force_remap_mode; 982SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 983 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 984 " to a smaller number of ithreads, even if the desired number are " 985 "available"); 986 987/* 988 * In case it is NOT ok, give consumers an abort button. 989 */ 990static int ntb_prefer_intx; 991SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 992 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 993 "than remapping MSI-X messages over available slots (match Linux driver " 994 "behavior)"); 995 996/* 997 * Remap the desired number of MSI-X messages to available ithreads in a simple 998 * round-robin fashion. 999 */ 1000static int 1001ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1002{ 1003 u_int *vectors; 1004 uint32_t i; 1005 int rc; 1006 1007 if (ntb_prefer_intx != 0) 1008 return (ENXIO); 1009 1010 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1011 1012 for (i = 0; i < desired; i++) 1013 vectors[i] = (i % avail) + 1; 1014 1015 rc = pci_remap_msix(dev, desired, vectors); 1016 free(vectors, M_NTB); 1017 return (rc); 1018} 1019 1020static int 1021ntb_init_isr(struct ntb_softc *ntb) 1022{ 1023 uint32_t desired_vectors, num_vectors; 1024 int rc; 1025 1026 ntb->allocated_interrupts = 0; 1027 ntb->last_ts = ticks; 1028 1029 /* 1030 * Mask all doorbell interrupts. (Except link events!) 1031 */ 1032 DB_MASK_LOCK(ntb); 1033 ntb->db_mask = ntb->db_valid_mask; 1034 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1035 DB_MASK_UNLOCK(ntb); 1036 1037 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1038 ntb->db_count); 1039 if (desired_vectors >= 1) { 1040 rc = pci_alloc_msix(ntb->device, &num_vectors); 1041 1042 if (ntb_force_remap_mode != 0 && rc == 0 && 1043 num_vectors == desired_vectors) 1044 num_vectors--; 1045 1046 if (rc == 0 && num_vectors < desired_vectors) { 1047 rc = ntb_remap_msix(ntb->device, desired_vectors, 1048 num_vectors); 1049 if (rc == 0) 1050 num_vectors = desired_vectors; 1051 else 1052 pci_release_msi(ntb->device); 1053 } 1054 if (rc != 0) 1055 num_vectors = 1; 1056 } else 1057 num_vectors = 1; 1058 1059 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1060 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1061 device_printf(ntb->device, 1062 "Errata workaround does not support MSI or INTX\n"); 1063 return (EINVAL); 1064 } 1065 1066 ntb->db_vec_count = 1; 1067 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1068 rc = ntb_setup_legacy_interrupt(ntb); 1069 } else { 1070 if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1071 HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1072 device_printf(ntb->device, 1073 "Errata workaround expects %d doorbell bits\n", 1074 XEON_NONLINK_DB_MSIX_BITS); 1075 return (EINVAL); 1076 } 1077 1078 ntb_create_msix_vec(ntb, num_vectors); 1079 rc = ntb_setup_msix(ntb, num_vectors); 1080 if (rc == 0 && HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1081 ntb_get_msix_info(ntb); 1082 } 1083 if (rc != 0) { 1084 device_printf(ntb->device, 1085 "Error allocating interrupts: %d\n", rc); 1086 ntb_free_msix_vec(ntb); 1087 } 1088 1089 return (rc); 1090} 1091 1092static int 1093ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1094{ 1095 int rc; 1096 1097 ntb->int_info[0].rid = 0; 1098 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1099 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1100 if (ntb->int_info[0].res == NULL) { 1101 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1102 return (ENOMEM); 1103 } 1104 1105 ntb->int_info[0].tag = NULL; 1106 ntb->allocated_interrupts = 1; 1107 1108 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1109 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1110 ntb, &ntb->int_info[0].tag); 1111 if (rc != 0) { 1112 device_printf(ntb->device, "bus_setup_intr failed\n"); 1113 return (ENXIO); 1114 } 1115 1116 return (0); 1117} 1118 1119static void 1120ntb_teardown_interrupts(struct ntb_softc *ntb) 1121{ 1122 struct ntb_int_info *current_int; 1123 int i; 1124 1125 for (i = 0; i < ntb->allocated_interrupts; i++) { 1126 current_int = &ntb->int_info[i]; 1127 if (current_int->tag != NULL) 1128 bus_teardown_intr(ntb->device, current_int->res, 1129 current_int->tag); 1130 1131 if (current_int->res != NULL) 1132 bus_release_resource(ntb->device, SYS_RES_IRQ, 1133 rman_get_rid(current_int->res), current_int->res); 1134 } 1135 1136 ntb_free_msix_vec(ntb); 1137 pci_release_msi(ntb->device); 1138} 1139 1140/* 1141 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1142 * out to make code clearer. 1143 */ 1144static inline uint64_t 1145db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1146{ 1147 1148 if (ntb->type == NTB_ATOM) 1149 return (ntb_reg_read(8, regoff)); 1150 1151 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1152 1153 return (ntb_reg_read(2, regoff)); 1154} 1155 1156static inline void 1157db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1158{ 1159 1160 KASSERT((val & ~ntb->db_valid_mask) == 0, 1161 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1162 (uintmax_t)(val & ~ntb->db_valid_mask), 1163 (uintmax_t)ntb->db_valid_mask)); 1164 1165 if (regoff == ntb->self_reg->db_mask) 1166 DB_MASK_ASSERT(ntb, MA_OWNED); 1167 db_iowrite_raw(ntb, regoff, val); 1168} 1169 1170static inline void 1171db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1172{ 1173 1174 if (ntb->type == NTB_ATOM) { 1175 ntb_reg_write(8, regoff, val); 1176 return; 1177 } 1178 1179 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1180 ntb_reg_write(2, regoff, (uint16_t)val); 1181} 1182 1183void 1184ntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1185{ 1186 1187 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1188 return; 1189 1190 DB_MASK_LOCK(ntb); 1191 ntb->db_mask |= bits; 1192 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1193 DB_MASK_UNLOCK(ntb); 1194} 1195 1196void 1197ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1198{ 1199 1200 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1201 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1202 (uintmax_t)(bits & ~ntb->db_valid_mask), 1203 (uintmax_t)ntb->db_valid_mask)); 1204 1205 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1206 return; 1207 1208 DB_MASK_LOCK(ntb); 1209 ntb->db_mask &= ~bits; 1210 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1211 DB_MASK_UNLOCK(ntb); 1212} 1213 1214uint64_t 1215ntb_db_read(struct ntb_softc *ntb) 1216{ 1217 1218 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1219 uint64_t res; 1220 unsigned i; 1221 1222 res = 0; 1223 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1224 if (ntb->msix_vec[i].masked != 0) 1225 res |= ntb_db_vector_mask(ntb, i); 1226 } 1227 return (res); 1228 } 1229 1230 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1231} 1232 1233void 1234ntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1235{ 1236 1237 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1238 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1239 (uintmax_t)(bits & ~ntb->db_valid_mask), 1240 (uintmax_t)ntb->db_valid_mask)); 1241 1242 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1243 unsigned i; 1244 1245 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1246 if ((bits & ntb_db_vector_mask(ntb, i)) != 0) { 1247 DB_MASK_LOCK(ntb); 1248 if (ntb->msix_vec[i].masked != 0) { 1249 /* XXX These need a public API. */ 1250#if 0 1251 pci_unmask_msix(ntb->device, i); 1252#endif 1253 ntb->msix_vec[i].masked = 0; 1254 } 1255 DB_MASK_UNLOCK(ntb); 1256 } 1257 } 1258 return; 1259 } 1260 1261 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1262} 1263 1264static inline uint64_t 1265ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1266{ 1267 uint64_t shift, mask; 1268 1269 shift = ntb->db_vec_shift; 1270 mask = (1ull << shift) - 1; 1271 return (mask << (shift * db_vector)); 1272} 1273 1274static void 1275ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1276{ 1277 uint64_t vec_mask; 1278 1279 ntb->last_ts = ticks; 1280 vec_mask = ntb_vec_mask(ntb, vec); 1281 1282 if ((vec_mask & ntb->db_link_mask) != 0) { 1283 if (ntb_poll_link(ntb)) 1284 ntb_link_event(ntb); 1285 } 1286 1287 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1288 (vec_mask & ntb->db_link_mask) == 0) { 1289 DB_MASK_LOCK(ntb); 1290 if (ntb->msix_vec[vec].masked == 0) { 1291 /* XXX These need a public API. */ 1292#if 0 1293 pci_mask_msix(ntb->device, vec); 1294#endif 1295 ntb->msix_vec[vec].masked = 1; 1296 } 1297 DB_MASK_UNLOCK(ntb); 1298 } 1299 1300 if ((vec_mask & ntb->db_valid_mask) != 0) 1301 ntb_db_event(ntb, vec); 1302} 1303 1304static void 1305ndev_vec_isr(void *arg) 1306{ 1307 struct ntb_vec *nvec = arg; 1308 1309 ntb_interrupt(nvec->ntb, nvec->num); 1310} 1311 1312static void 1313ndev_irq_isr(void *arg) 1314{ 1315 /* If we couldn't set up MSI-X, we only have the one vector. */ 1316 ntb_interrupt(arg, 0); 1317} 1318 1319static int 1320ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1321{ 1322 uint32_t i; 1323 1324 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1325 M_ZERO | M_WAITOK); 1326 for (i = 0; i < num_vectors; i++) { 1327 ntb->msix_vec[i].num = i; 1328 ntb->msix_vec[i].ntb = ntb; 1329 } 1330 1331 return (0); 1332} 1333 1334static void 1335ntb_free_msix_vec(struct ntb_softc *ntb) 1336{ 1337 1338 if (ntb->msix_vec == NULL) 1339 return; 1340 1341 free(ntb->msix_vec, M_NTB); 1342 ntb->msix_vec = NULL; 1343} 1344 1345static void 1346ntb_get_msix_info(struct ntb_softc *ntb) 1347{ 1348 struct pci_devinfo *dinfo; 1349 struct pcicfg_msix *msix; 1350 uint32_t laddr, data, i, offset; 1351 1352 dinfo = device_get_ivars(ntb->device); 1353 msix = &dinfo->cfg.msix; 1354 1355 CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1356 1357 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1358 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1359 1360 laddr = bus_read_4(msix->msix_table_res, offset + 1361 PCI_MSIX_ENTRY_LOWER_ADDR); 1362 ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1363 1364 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1365 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1366 MSI_INTEL_ADDR_BASE)); 1367 ntb->msix_data[i].nmd_ofs = laddr; 1368 1369 data = bus_read_4(msix->msix_table_res, offset + 1370 PCI_MSIX_ENTRY_DATA); 1371 ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1372 1373 ntb->msix_data[i].nmd_data = data; 1374 } 1375} 1376 1377static struct ntb_hw_info * 1378ntb_get_device_info(uint32_t device_id) 1379{ 1380 struct ntb_hw_info *ep = pci_ids; 1381 1382 while (ep->device_id) { 1383 if (ep->device_id == device_id) 1384 return (ep); 1385 ++ep; 1386 } 1387 return (NULL); 1388} 1389 1390static void 1391ntb_teardown_xeon(struct ntb_softc *ntb) 1392{ 1393 1394 if (ntb->reg != NULL) 1395 ntb_link_disable(ntb); 1396} 1397 1398static void 1399ntb_detect_max_mw(struct ntb_softc *ntb) 1400{ 1401 1402 if (ntb->type == NTB_ATOM) { 1403 ntb->mw_count = ATOM_MW_COUNT; 1404 return; 1405 } 1406 1407 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1408 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1409 else 1410 ntb->mw_count = XEON_SNB_MW_COUNT; 1411} 1412 1413static int 1414ntb_detect_xeon(struct ntb_softc *ntb) 1415{ 1416 uint8_t ppd, conn_type; 1417 1418 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1419 ntb->ppd = ppd; 1420 1421 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1422 ntb->dev_type = NTB_DEV_DSD; 1423 else 1424 ntb->dev_type = NTB_DEV_USD; 1425 1426 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1427 ntb->features |= NTB_SPLIT_BAR; 1428 1429 /* 1430 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1431 * errata workaround; only do one at a time. 1432 */ 1433 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1434 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1435 1436 conn_type = ppd & XEON_PPD_CONN_TYPE; 1437 switch (conn_type) { 1438 case NTB_CONN_B2B: 1439 ntb->conn_type = conn_type; 1440 break; 1441 case NTB_CONN_RP: 1442 case NTB_CONN_TRANSPARENT: 1443 default: 1444 device_printf(ntb->device, "Unsupported connection type: %u\n", 1445 (unsigned)conn_type); 1446 return (ENXIO); 1447 } 1448 return (0); 1449} 1450 1451static int 1452ntb_detect_atom(struct ntb_softc *ntb) 1453{ 1454 uint32_t ppd, conn_type; 1455 1456 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1457 ntb->ppd = ppd; 1458 1459 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1460 ntb->dev_type = NTB_DEV_DSD; 1461 else 1462 ntb->dev_type = NTB_DEV_USD; 1463 1464 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1465 switch (conn_type) { 1466 case NTB_CONN_B2B: 1467 ntb->conn_type = conn_type; 1468 break; 1469 default: 1470 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1471 return (ENXIO); 1472 } 1473 return (0); 1474} 1475 1476static int 1477ntb_xeon_init_dev(struct ntb_softc *ntb) 1478{ 1479 int rc; 1480 1481 ntb->spad_count = XEON_SPAD_COUNT; 1482 ntb->db_count = XEON_DB_COUNT; 1483 ntb->db_link_mask = XEON_DB_LINK_BIT; 1484 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1485 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1486 1487 if (ntb->conn_type != NTB_CONN_B2B) { 1488 device_printf(ntb->device, "Connection type %d not supported\n", 1489 ntb->conn_type); 1490 return (ENXIO); 1491 } 1492 1493 ntb->reg = &xeon_reg; 1494 ntb->self_reg = &xeon_pri_reg; 1495 ntb->peer_reg = &xeon_b2b_reg; 1496 ntb->xlat_reg = &xeon_sec_xlat; 1497 1498 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1499 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1500 ntb->mw_count; 1501 ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1502 g_ntb_msix_idx, ntb->msix_mw_idx); 1503 rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1504 VM_MEMATTR_UNCACHEABLE); 1505 KASSERT(rc == 0, ("shouldn't fail")); 1506 } else if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1507 /* 1508 * There is a Xeon hardware errata related to writes to SDOORBELL or 1509 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1510 * which may hang the system. To workaround this, use a memory 1511 * window to access the interrupt and scratch pad registers on the 1512 * remote system. 1513 */ 1514 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1515 ntb->mw_count; 1516 ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1517 g_ntb_mw_idx, ntb->b2b_mw_idx); 1518 rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1519 VM_MEMATTR_UNCACHEABLE); 1520 KASSERT(rc == 0, ("shouldn't fail")); 1521 } else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1522 /* 1523 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1524 * mirrored to the remote system. Shrink the number of bits by one, 1525 * since bit 14 is the last bit. 1526 * 1527 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1528 * anyway. Nor for non-B2B connection types. 1529 */ 1530 ntb->db_count = XEON_DB_COUNT - 1; 1531 1532 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1533 1534 if (ntb->dev_type == NTB_DEV_USD) 1535 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1536 &xeon_b2b_usd_addr); 1537 else 1538 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1539 &xeon_b2b_dsd_addr); 1540 if (rc != 0) 1541 return (rc); 1542 1543 /* Enable Bus Master and Memory Space on the secondary side */ 1544 ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1545 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1546 1547 /* 1548 * Mask all doorbell interrupts. 1549 */ 1550 DB_MASK_LOCK(ntb); 1551 ntb->db_mask = ntb->db_valid_mask; 1552 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1553 DB_MASK_UNLOCK(ntb); 1554 1555 rc = xeon_setup_msix_bar(ntb); 1556 if (rc != 0) 1557 return (rc); 1558 1559 rc = ntb_init_isr(ntb); 1560 return (rc); 1561} 1562 1563static int 1564ntb_atom_init_dev(struct ntb_softc *ntb) 1565{ 1566 int error; 1567 1568 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1569 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1570 1571 ntb->spad_count = ATOM_SPAD_COUNT; 1572 ntb->db_count = ATOM_DB_COUNT; 1573 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1574 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1575 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1576 1577 ntb->reg = &atom_reg; 1578 ntb->self_reg = &atom_pri_reg; 1579 ntb->peer_reg = &atom_b2b_reg; 1580 ntb->xlat_reg = &atom_sec_xlat; 1581 1582 /* 1583 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1584 * resolved. Mask transaction layer internal parity errors. 1585 */ 1586 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1587 1588 configure_atom_secondary_side_bars(ntb); 1589 1590 /* Enable Bus Master and Memory Space on the secondary side */ 1591 ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1592 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1593 1594 error = ntb_init_isr(ntb); 1595 if (error != 0) 1596 return (error); 1597 1598 /* Initiate PCI-E link training */ 1599 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1600 1601 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1602 1603 return (0); 1604} 1605 1606/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1607static void 1608configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1609{ 1610 1611 if (ntb->dev_type == NTB_DEV_USD) { 1612 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1613 XEON_B2B_BAR2_ADDR64); 1614 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1615 XEON_B2B_BAR4_ADDR64); 1616 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1617 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1618 } else { 1619 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1620 XEON_B2B_BAR2_ADDR64); 1621 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1622 XEON_B2B_BAR4_ADDR64); 1623 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1624 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1625 } 1626} 1627 1628 1629/* 1630 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1631 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1632 * remains for use by a higher layer. 1633 * 1634 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1635 * MW size is sufficiently large. 1636 */ 1637static unsigned int ntb_b2b_mw_share; 1638SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1639 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1640 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1641 "value here."); 1642 1643static void 1644xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1645 enum ntb_bar regbar) 1646{ 1647 struct ntb_pci_bar_info *bar; 1648 uint8_t bar_sz; 1649 1650 if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1651 return; 1652 1653 bar = &ntb->bar_info[idx]; 1654 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1655 if (idx == regbar) { 1656 if (ntb->b2b_off != 0) 1657 bar_sz--; 1658 else 1659 bar_sz = 0; 1660 } 1661 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1662 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1663 (void)bar_sz; 1664} 1665 1666static void 1667xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1668 enum ntb_bar idx, enum ntb_bar regbar) 1669{ 1670 uint64_t reg_val; 1671 uint32_t base_reg, lmt_reg; 1672 1673 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1674 if (idx == regbar) { 1675 if (ntb->b2b_off) 1676 bar_addr += ntb->b2b_off; 1677 else 1678 bar_addr = 0; 1679 } 1680 1681 /* 1682 * Set limit registers first to avoid an errata where setting the base 1683 * registers locks the limit registers. 1684 */ 1685 if (!bar_is_64bit(ntb, idx)) { 1686 ntb_reg_write(4, lmt_reg, bar_addr); 1687 reg_val = ntb_reg_read(4, lmt_reg); 1688 (void)reg_val; 1689 1690 ntb_reg_write(4, base_reg, bar_addr); 1691 reg_val = ntb_reg_read(4, base_reg); 1692 (void)reg_val; 1693 } else { 1694 ntb_reg_write(8, lmt_reg, bar_addr); 1695 reg_val = ntb_reg_read(8, lmt_reg); 1696 (void)reg_val; 1697 1698 ntb_reg_write(8, base_reg, bar_addr); 1699 reg_val = ntb_reg_read(8, base_reg); 1700 (void)reg_val; 1701 } 1702} 1703 1704static void 1705xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1706{ 1707 struct ntb_pci_bar_info *bar; 1708 1709 bar = &ntb->bar_info[idx]; 1710 if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1711 ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1712 base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1713 } else { 1714 ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1715 base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1716 } 1717 (void)base_addr; 1718} 1719 1720static int 1721xeon_setup_msix_bar(struct ntb_softc *ntb) 1722{ 1723 enum ntb_bar bar_num; 1724 1725 if (!HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1726 return (0); 1727 1728 bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1729 ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1730 return (0); 1731} 1732 1733static int 1734xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1735 const struct ntb_b2b_addr *peer_addr) 1736{ 1737 struct ntb_pci_bar_info *b2b_bar; 1738 vm_size_t bar_size; 1739 uint64_t bar_addr; 1740 enum ntb_bar b2b_bar_num, i; 1741 1742 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1743 b2b_bar = NULL; 1744 b2b_bar_num = NTB_CONFIG_BAR; 1745 ntb->b2b_off = 0; 1746 } else { 1747 b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1748 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1749 ("invalid b2b mw bar")); 1750 1751 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1752 bar_size = b2b_bar->size; 1753 1754 if (ntb_b2b_mw_share != 0 && 1755 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1756 ntb->b2b_off = bar_size >> 1; 1757 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1758 ntb->b2b_off = 0; 1759 } else { 1760 device_printf(ntb->device, 1761 "B2B bar size is too small!\n"); 1762 return (EIO); 1763 } 1764 } 1765 1766 /* 1767 * Reset the secondary bar sizes to match the primary bar sizes. 1768 * (Except, disable or halve the size of the B2B secondary bar.) 1769 */ 1770 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1771 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1772 1773 bar_addr = 0; 1774 if (b2b_bar_num == NTB_CONFIG_BAR) 1775 bar_addr = addr->bar0_addr; 1776 else if (b2b_bar_num == NTB_B2B_BAR_1) 1777 bar_addr = addr->bar2_addr64; 1778 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1779 bar_addr = addr->bar4_addr64; 1780 else if (b2b_bar_num == NTB_B2B_BAR_2) 1781 bar_addr = addr->bar4_addr32; 1782 else if (b2b_bar_num == NTB_B2B_BAR_3) 1783 bar_addr = addr->bar5_addr32; 1784 else 1785 KASSERT(false, ("invalid bar")); 1786 1787 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1788 1789 /* 1790 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1791 * register BAR. The B2B BAR is either disabled above or configured 1792 * half-size. It starts at PBAR xlat + offset. 1793 * 1794 * Also set up incoming BAR limits == base (zero length window). 1795 */ 1796 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1797 b2b_bar_num); 1798 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1799 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1800 NTB_B2B_BAR_2, b2b_bar_num); 1801 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1802 NTB_B2B_BAR_3, b2b_bar_num); 1803 } else 1804 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1805 NTB_B2B_BAR_2, b2b_bar_num); 1806 1807 /* Zero incoming translation addrs */ 1808 ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1809 ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1810 1811 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1812 size_t size, xlatoffset; 1813 1814 switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1815 case NTB_B2B_BAR_1: 1816 size = 8; 1817 xlatoffset = XEON_SBAR2XLAT_OFFSET; 1818 break; 1819 case NTB_B2B_BAR_2: 1820 xlatoffset = XEON_SBAR4XLAT_OFFSET; 1821 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1822 size = 4; 1823 else 1824 size = 8; 1825 break; 1826 case NTB_B2B_BAR_3: 1827 xlatoffset = XEON_SBAR5XLAT_OFFSET; 1828 size = 4; 1829 break; 1830 default: 1831 KASSERT(false, ("Bogus msix mw idx: %u", 1832 ntb->msix_mw_idx)); 1833 return (EINVAL); 1834 } 1835 1836 /* 1837 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1838 * workaround 1839 */ 1840 if (size == 4) { 1841 ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1842 ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1843 } else { 1844 ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1845 ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1846 } 1847 } 1848 (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1849 (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1850 1851 /* Zero outgoing translation limits (whole bar size windows) */ 1852 ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1853 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1854 1855 /* Set outgoing translation offsets */ 1856 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1857 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1858 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1859 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1860 } else 1861 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1862 1863 /* Set the translation offset for B2B registers */ 1864 bar_addr = 0; 1865 if (b2b_bar_num == NTB_CONFIG_BAR) 1866 bar_addr = peer_addr->bar0_addr; 1867 else if (b2b_bar_num == NTB_B2B_BAR_1) 1868 bar_addr = peer_addr->bar2_addr64; 1869 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1870 bar_addr = peer_addr->bar4_addr64; 1871 else if (b2b_bar_num == NTB_B2B_BAR_2) 1872 bar_addr = peer_addr->bar4_addr32; 1873 else if (b2b_bar_num == NTB_B2B_BAR_3) 1874 bar_addr = peer_addr->bar5_addr32; 1875 else 1876 KASSERT(false, ("invalid bar")); 1877 1878 /* 1879 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1880 * at a time. 1881 */ 1882 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1883 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1884 return (0); 1885} 1886 1887static inline bool 1888_xeon_link_is_up(struct ntb_softc *ntb) 1889{ 1890 1891 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1892 return (true); 1893 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1894} 1895 1896static inline bool 1897link_is_up(struct ntb_softc *ntb) 1898{ 1899 1900 if (ntb->type == NTB_XEON) 1901 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1902 !HAS_FEATURE(NTB_SB01BASE_LOCKUP))); 1903 1904 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1905 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1906} 1907 1908static inline bool 1909atom_link_is_err(struct ntb_softc *ntb) 1910{ 1911 uint32_t status; 1912 1913 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1914 1915 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1916 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1917 return (true); 1918 1919 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1920 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1921} 1922 1923/* Atom does not have link status interrupt, poll on that platform */ 1924static void 1925atom_link_hb(void *arg) 1926{ 1927 struct ntb_softc *ntb = arg; 1928 sbintime_t timo, poll_ts; 1929 1930 timo = NTB_HB_TIMEOUT * hz; 1931 poll_ts = ntb->last_ts + timo; 1932 1933 /* 1934 * Delay polling the link status if an interrupt was received, unless 1935 * the cached link status says the link is down. 1936 */ 1937 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1938 timo = poll_ts - ticks; 1939 goto out; 1940 } 1941 1942 if (ntb_poll_link(ntb)) 1943 ntb_link_event(ntb); 1944 1945 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1946 /* Link is down with error, proceed with recovery */ 1947 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1948 return; 1949 } 1950 1951out: 1952 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1953} 1954 1955static void 1956atom_perform_link_restart(struct ntb_softc *ntb) 1957{ 1958 uint32_t status; 1959 1960 /* Driver resets the NTB ModPhy lanes - magic! */ 1961 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1962 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1963 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1964 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1965 1966 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1967 pause("ModPhy", hz / 10); 1968 1969 /* Clear AER Errors, write to clear */ 1970 status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1971 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1972 ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1973 1974 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1975 status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1976 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1977 ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1978 1979 /* Clear DeSkew Buffer error, write to clear */ 1980 status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1981 status |= ATOM_DESKEWSTS_DBERR; 1982 ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1983 1984 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1985 status &= ATOM_IBIST_ERR_OFLOW; 1986 ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1987 1988 /* Releases the NTB state machine to allow the link to retrain */ 1989 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1990 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1991 ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1992} 1993 1994/* 1995 * ntb_set_ctx() - associate a driver context with an ntb device 1996 * @ntb: NTB device context 1997 * @ctx: Driver context 1998 * @ctx_ops: Driver context operations 1999 * 2000 * Associate a driver context and operations with a ntb device. The context is 2001 * provided by the client driver, and the driver may associate a different 2002 * context with each ntb device. 2003 * 2004 * Return: Zero if the context is associated, otherwise an error number. 2005 */ 2006int 2007ntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 2008{ 2009 2010 if (ctx == NULL || ops == NULL) 2011 return (EINVAL); 2012 if (ntb->ctx_ops != NULL) 2013 return (EINVAL); 2014 2015 CTX_LOCK(ntb); 2016 if (ntb->ctx_ops != NULL) { 2017 CTX_UNLOCK(ntb); 2018 return (EINVAL); 2019 } 2020 ntb->ntb_ctx = ctx; 2021 ntb->ctx_ops = ops; 2022 CTX_UNLOCK(ntb); 2023 2024 return (0); 2025} 2026 2027/* 2028 * It is expected that this will only be used from contexts where the ctx_lock 2029 * is not needed to protect ntb_ctx lifetime. 2030 */ 2031void * 2032ntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 2033{ 2034 2035 KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2036 if (ops != NULL) 2037 *ops = ntb->ctx_ops; 2038 return (ntb->ntb_ctx); 2039} 2040 2041/* 2042 * ntb_clear_ctx() - disassociate any driver context from an ntb device 2043 * @ntb: NTB device context 2044 * 2045 * Clear any association that may exist between a driver context and the ntb 2046 * device. 2047 */ 2048void 2049ntb_clear_ctx(struct ntb_softc *ntb) 2050{ 2051 2052 CTX_LOCK(ntb); 2053 ntb->ntb_ctx = NULL; 2054 ntb->ctx_ops = NULL; 2055 CTX_UNLOCK(ntb); 2056} 2057 2058/* 2059 * ntb_link_event() - notify driver context of a change in link status 2060 * @ntb: NTB device context 2061 * 2062 * Notify the driver context that the link status may have changed. The driver 2063 * should call ntb_link_is_up() to get the current status. 2064 */ 2065void 2066ntb_link_event(struct ntb_softc *ntb) 2067{ 2068 2069 CTX_LOCK(ntb); 2070 if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2071 ntb->ctx_ops->link_event(ntb->ntb_ctx); 2072 CTX_UNLOCK(ntb); 2073} 2074 2075/* 2076 * ntb_db_event() - notify driver context of a doorbell event 2077 * @ntb: NTB device context 2078 * @vector: Interrupt vector number 2079 * 2080 * Notify the driver context of a doorbell event. If hardware supports 2081 * multiple interrupt vectors for doorbells, the vector number indicates which 2082 * vector received the interrupt. The vector number is relative to the first 2083 * vector used for doorbells, starting at zero, and must be less than 2084 * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2085 * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2086 * those bits are associated with the vector number. 2087 */ 2088static void 2089ntb_db_event(struct ntb_softc *ntb, uint32_t vec) 2090{ 2091 2092 CTX_LOCK(ntb); 2093 if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2094 ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2095 CTX_UNLOCK(ntb); 2096} 2097 2098/* 2099 * ntb_link_enable() - enable the link on the secondary side of the ntb 2100 * @ntb: NTB device context 2101 * @max_speed: The maximum link speed expressed as PCIe generation number[0] 2102 * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 2103 * 2104 * Enable the link on the secondary side of the ntb. This can only be done 2105 * from the primary side of the ntb in primary or b2b topology. The ntb device 2106 * should train the link to its maximum speed and width, or the requested speed 2107 * and width, whichever is smaller, if supported. 2108 * 2109 * Return: Zero on success, otherwise an error number. 2110 * 2111 * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 2112 * and width input will be ignored. 2113 */ 2114int 2115ntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 2116 enum ntb_width w __unused) 2117{ 2118 uint32_t cntl; 2119 2120 ntb_printf(2, "%s\n", __func__); 2121 2122 if (ntb->type == NTB_ATOM) { 2123 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2124 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2125 return (0); 2126 } 2127 2128 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2129 ntb_link_event(ntb); 2130 return (0); 2131 } 2132 2133 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2134 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2135 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2136 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2137 if (HAS_FEATURE(NTB_SPLIT_BAR)) 2138 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2139 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2140 return (0); 2141} 2142 2143/* 2144 * ntb_link_disable() - disable the link on the secondary side of the ntb 2145 * @ntb: NTB device context 2146 * 2147 * Disable the link on the secondary side of the ntb. This can only be done 2148 * from the primary side of the ntb in primary or b2b topology. The ntb device 2149 * should disable the link. Returning from this call must indicate that a 2150 * barrier has passed, though with no more writes may pass in either direction 2151 * across the link, except if this call returns an error number. 2152 * 2153 * Return: Zero on success, otherwise an error number. 2154 */ 2155int 2156ntb_link_disable(struct ntb_softc *ntb) 2157{ 2158 uint32_t cntl; 2159 2160 ntb_printf(2, "%s\n", __func__); 2161 2162 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2163 ntb_link_event(ntb); 2164 return (0); 2165 } 2166 2167 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2168 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2169 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2170 if (HAS_FEATURE(NTB_SPLIT_BAR)) 2171 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2172 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2173 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2174 return (0); 2175} 2176 2177bool 2178ntb_link_enabled(struct ntb_softc *ntb) 2179{ 2180 uint32_t cntl; 2181 2182 if (ntb->type == NTB_ATOM) { 2183 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2184 return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2185 } 2186 2187 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2188 return (true); 2189 2190 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2191 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2192} 2193 2194static void 2195recover_atom_link(void *arg) 2196{ 2197 struct ntb_softc *ntb = arg; 2198 unsigned speed, width, oldspeed, oldwidth; 2199 uint32_t status32; 2200 2201 atom_perform_link_restart(ntb); 2202 2203 /* 2204 * There is a potential race between the 2 NTB devices recovering at 2205 * the same time. If the times are the same, the link will not recover 2206 * and the driver will be stuck in this loop forever. Add a random 2207 * interval to the recovery time to prevent this race. 2208 */ 2209 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2210 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2211 2212 if (atom_link_is_err(ntb)) 2213 goto retry; 2214 2215 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2216 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2217 goto out; 2218 2219 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2220 width = NTB_LNK_STA_WIDTH(status32); 2221 speed = status32 & NTB_LINK_SPEED_MASK; 2222 2223 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2224 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2225 if (oldwidth != width || oldspeed != speed) 2226 goto retry; 2227 2228out: 2229 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2230 ntb); 2231 return; 2232 2233retry: 2234 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2235 ntb); 2236} 2237 2238/* 2239 * Polls the HW link status register(s); returns true if something has changed. 2240 */ 2241static bool 2242ntb_poll_link(struct ntb_softc *ntb) 2243{ 2244 uint32_t ntb_cntl; 2245 uint16_t reg_val; 2246 2247 if (ntb->type == NTB_ATOM) { 2248 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2249 if (ntb_cntl == ntb->ntb_ctl) 2250 return (false); 2251 2252 ntb->ntb_ctl = ntb_cntl; 2253 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2254 } else { 2255 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2256 2257 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2258 if (reg_val == ntb->lnk_sta) 2259 return (false); 2260 2261 ntb->lnk_sta = reg_val; 2262 2263 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 2264 if (_xeon_link_is_up(ntb)) { 2265 if (!ntb->peer_msix_good) { 2266 callout_reset(&ntb->peer_msix_work, 0, 2267 ntb_exchange_msix, ntb); 2268 return (false); 2269 } 2270 } else { 2271 ntb->peer_msix_good = false; 2272 ntb->peer_msix_done = false; 2273 } 2274 } 2275 } 2276 return (true); 2277} 2278 2279static inline enum ntb_speed 2280ntb_link_sta_speed(struct ntb_softc *ntb) 2281{ 2282 2283 if (!link_is_up(ntb)) 2284 return (NTB_SPEED_NONE); 2285 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2286} 2287 2288static inline enum ntb_width 2289ntb_link_sta_width(struct ntb_softc *ntb) 2290{ 2291 2292 if (!link_is_up(ntb)) 2293 return (NTB_WIDTH_NONE); 2294 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2295} 2296 2297SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2298 "Driver state, statistics, and HW registers"); 2299 2300#define NTB_REGSZ_MASK (3ul << 30) 2301#define NTB_REG_64 (1ul << 30) 2302#define NTB_REG_32 (2ul << 30) 2303#define NTB_REG_16 (3ul << 30) 2304#define NTB_REG_8 (0ul << 30) 2305 2306#define NTB_DB_READ (1ul << 29) 2307#define NTB_PCI_REG (1ul << 28) 2308#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2309 2310static void 2311ntb_sysctl_init(struct ntb_softc *ntb) 2312{ 2313 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2314 struct sysctl_ctx_list *ctx; 2315 struct sysctl_oid *tree, *tmptree; 2316 2317 ctx = device_get_sysctl_ctx(ntb->device); 2318 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2319 2320 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2321 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2322 sysctl_handle_link_status_human, "A", 2323 "Link status (human readable)"); 2324 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2325 CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2326 "IU", "Link status (1=active, 0=inactive)"); 2327 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2328 CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2329 "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2330 2331 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2332 CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2333 tree_par = SYSCTL_CHILDREN(tree); 2334 2335 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2336 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2337 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2338 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2339 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2340 &ntb->ppd, 0, "Raw PPD register (cached)"); 2341 2342 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2343 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2344 &ntb->b2b_mw_idx, 0, 2345 "Index of the MW used for B2B remote register access"); 2346 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2347 CTLFLAG_RD, &ntb->b2b_off, 2348 "If non-zero, offset of B2B register region in shared MW"); 2349 } 2350 2351 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2352 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2353 "Features/errata of this NTB device"); 2354 2355 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2356 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2357 "NTB CTL register (cached)"); 2358 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2359 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2360 "LNK STA register (cached)"); 2361 2362 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2363 &ntb->mw_count, 0, "MW count"); 2364 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2365 &ntb->spad_count, 0, "Scratchpad count"); 2366 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2367 &ntb->db_count, 0, "Doorbell count"); 2368 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2369 &ntb->db_vec_count, 0, "Doorbell vector count"); 2370 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2371 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2372 2373 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2374 &ntb->db_valid_mask, "Doorbell valid mask"); 2375 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2376 &ntb->db_link_mask, "Doorbell link mask"); 2377 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2378 &ntb->db_mask, "Doorbell mask (cached)"); 2379 2380 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2381 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2382 regpar = SYSCTL_CHILDREN(tmptree); 2383 2384 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2385 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2386 ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2387 "NTB Control register"); 2388 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2389 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2390 0x19c, sysctl_handle_register, "IU", 2391 "NTB Link Capabilities"); 2392 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2393 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2394 0x1a0, sysctl_handle_register, "IU", 2395 "NTB Link Control register"); 2396 2397 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2398 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2399 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2400 sysctl_handle_register, "QU", "Doorbell mask register"); 2401 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2402 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2403 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2404 sysctl_handle_register, "QU", "Doorbell register"); 2405 2406 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2407 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2408 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2409 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2410 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2411 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2412 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2413 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2414 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2415 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2416 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2417 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2418 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2419 } else { 2420 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2421 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2422 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2423 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2424 } 2425 2426 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2427 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2428 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2429 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2430 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2431 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2432 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2433 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2434 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2435 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2436 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2437 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2438 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2439 } else { 2440 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2441 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2442 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2443 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2444 } 2445 2446 if (ntb->type == NTB_ATOM) 2447 return; 2448 2449 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2450 CTLFLAG_RD, NULL, "Xeon HW statistics"); 2451 statpar = SYSCTL_CHILDREN(tmptree); 2452 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2453 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2454 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2455 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2456 2457 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2458 CTLFLAG_RD, NULL, "Xeon HW errors"); 2459 errpar = SYSCTL_CHILDREN(tmptree); 2460 2461 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2462 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2463 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2464 sysctl_handle_register, "CU", "PPD"); 2465 2466 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2467 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2468 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2469 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2470 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2471 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2472 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2473 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2474 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2475 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2476 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2477 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2478 2479 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2480 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2481 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2482 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2483 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2484 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2485 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2486 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2487 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2488 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2489 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2490 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2491 2492 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2493 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2494 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2495 sysctl_handle_register, "SU", "DEVSTS"); 2496 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2497 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2498 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2499 sysctl_handle_register, "SU", "LNKSTS"); 2500 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2501 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2502 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2503 sysctl_handle_register, "SU", "SLNKSTS"); 2504 2505 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2506 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2507 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2508 sysctl_handle_register, "IU", "UNCERRSTS"); 2509 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2510 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2511 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2512 sysctl_handle_register, "IU", "CORERRSTS"); 2513 2514 if (ntb->conn_type != NTB_CONN_B2B) 2515 return; 2516 2517 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2518 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2519 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2520 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2521 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2522 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2523 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2524 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2525 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2526 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2527 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2528 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2529 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2530 } else { 2531 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2532 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2533 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2534 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2535 } 2536 2537 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2538 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2539 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2540 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2541 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2542 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2543 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2544 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2545 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2546 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2547 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2548 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2549 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2550 } else { 2551 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2552 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2553 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2554 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2555 } 2556 2557 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2558 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2559 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2560 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2561 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2562 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2563 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2564 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2565 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2566 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2567 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2568 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2569 sysctl_handle_register, "IU", 2570 "Secondary BAR4 base register"); 2571 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2572 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2573 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2574 sysctl_handle_register, "IU", 2575 "Secondary BAR5 base register"); 2576 } else { 2577 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2578 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2579 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2580 sysctl_handle_register, "QU", 2581 "Secondary BAR45 base register"); 2582 } 2583} 2584 2585static int 2586sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2587{ 2588 struct ntb_softc *ntb = arg1; 2589 struct sbuf sb; 2590 int error; 2591 2592 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2593 2594 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2595 error = sbuf_finish(&sb); 2596 sbuf_delete(&sb); 2597 2598 if (error || !req->newptr) 2599 return (error); 2600 return (EINVAL); 2601} 2602 2603static int 2604sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2605{ 2606 struct ntb_softc *ntb = arg1; 2607 unsigned old, new; 2608 int error; 2609 2610 old = ntb_link_enabled(ntb); 2611 2612 error = SYSCTL_OUT(req, &old, sizeof(old)); 2613 if (error != 0 || req->newptr == NULL) 2614 return (error); 2615 2616 error = SYSCTL_IN(req, &new, sizeof(new)); 2617 if (error != 0) 2618 return (error); 2619 2620 ntb_printf(0, "Admin set interface state to '%sabled'\n", 2621 (new != 0)? "en" : "dis"); 2622 2623 if (new != 0) 2624 error = ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2625 else 2626 error = ntb_link_disable(ntb); 2627 return (error); 2628} 2629 2630static int 2631sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2632{ 2633 struct ntb_softc *ntb = arg1; 2634 struct sbuf sb; 2635 enum ntb_speed speed; 2636 enum ntb_width width; 2637 int error; 2638 2639 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2640 2641 if (ntb_link_is_up(ntb, &speed, &width)) 2642 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2643 (unsigned)speed, (unsigned)width); 2644 else 2645 sbuf_printf(&sb, "down"); 2646 2647 error = sbuf_finish(&sb); 2648 sbuf_delete(&sb); 2649 2650 if (error || !req->newptr) 2651 return (error); 2652 return (EINVAL); 2653} 2654 2655static int 2656sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2657{ 2658 struct ntb_softc *ntb = arg1; 2659 unsigned res; 2660 int error; 2661 2662 res = ntb_link_is_up(ntb, NULL, NULL); 2663 2664 error = SYSCTL_OUT(req, &res, sizeof(res)); 2665 if (error || !req->newptr) 2666 return (error); 2667 return (EINVAL); 2668} 2669 2670static int 2671sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2672{ 2673 struct ntb_softc *ntb; 2674 const void *outp; 2675 uintptr_t sz; 2676 uint64_t umv; 2677 char be[sizeof(umv)]; 2678 size_t outsz; 2679 uint32_t reg; 2680 bool db, pci; 2681 int error; 2682 2683 ntb = arg1; 2684 reg = arg2 & ~NTB_REGFLAGS_MASK; 2685 sz = arg2 & NTB_REGSZ_MASK; 2686 db = (arg2 & NTB_DB_READ) != 0; 2687 pci = (arg2 & NTB_PCI_REG) != 0; 2688 2689 KASSERT(!(db && pci), ("bogus")); 2690 2691 if (db) { 2692 KASSERT(sz == NTB_REG_64, ("bogus")); 2693 umv = db_ioread(ntb, reg); 2694 outsz = sizeof(uint64_t); 2695 } else { 2696 switch (sz) { 2697 case NTB_REG_64: 2698 if (pci) 2699 umv = pci_read_config(ntb->device, reg, 8); 2700 else 2701 umv = ntb_reg_read(8, reg); 2702 outsz = sizeof(uint64_t); 2703 break; 2704 case NTB_REG_32: 2705 if (pci) 2706 umv = pci_read_config(ntb->device, reg, 4); 2707 else 2708 umv = ntb_reg_read(4, reg); 2709 outsz = sizeof(uint32_t); 2710 break; 2711 case NTB_REG_16: 2712 if (pci) 2713 umv = pci_read_config(ntb->device, reg, 2); 2714 else 2715 umv = ntb_reg_read(2, reg); 2716 outsz = sizeof(uint16_t); 2717 break; 2718 case NTB_REG_8: 2719 if (pci) 2720 umv = pci_read_config(ntb->device, reg, 1); 2721 else 2722 umv = ntb_reg_read(1, reg); 2723 outsz = sizeof(uint8_t); 2724 break; 2725 default: 2726 panic("bogus"); 2727 break; 2728 } 2729 } 2730 2731 /* Encode bigendian so that sysctl -x is legible. */ 2732 be64enc(be, umv); 2733 outp = ((char *)be) + sizeof(umv) - outsz; 2734 2735 error = SYSCTL_OUT(req, outp, outsz); 2736 if (error || !req->newptr) 2737 return (error); 2738 return (EINVAL); 2739} 2740 2741static unsigned 2742ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2743{ 2744 2745 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2746 uidx >= ntb->b2b_mw_idx) || 2747 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2748 uidx++; 2749 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2750 uidx >= ntb->b2b_mw_idx) && 2751 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2752 uidx++; 2753 return (uidx); 2754} 2755 2756static void 2757ntb_exchange_msix(void *ctx) 2758{ 2759 struct ntb_softc *ntb; 2760 uint32_t val; 2761 unsigned i; 2762 2763 ntb = ctx; 2764 2765 if (ntb->peer_msix_good) 2766 goto msix_good; 2767 if (ntb->peer_msix_done) 2768 goto msix_done; 2769 2770 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2771 ntb_peer_spad_write(ntb, NTB_MSIX_DATA0 + i, 2772 ntb->msix_data[i].nmd_data); 2773 ntb_peer_spad_write(ntb, NTB_MSIX_OFS0 + i, 2774 ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2775 } 2776 ntb_peer_spad_write(ntb, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2777 2778 ntb_spad_read(ntb, NTB_MSIX_GUARD, &val); 2779 if (val != NTB_MSIX_VER_GUARD) 2780 goto reschedule; 2781 2782 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2783 ntb_spad_read(ntb, NTB_MSIX_DATA0 + i, &val); 2784 ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2785 ntb->peer_msix_data[i].nmd_data = val; 2786 ntb_spad_read(ntb, NTB_MSIX_OFS0 + i, &val); 2787 ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2788 ntb->peer_msix_data[i].nmd_ofs = val; 2789 } 2790 2791 ntb->peer_msix_done = true; 2792 2793msix_done: 2794 ntb_peer_spad_write(ntb, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2795 ntb_spad_read(ntb, NTB_MSIX_DONE, &val); 2796 if (val != NTB_MSIX_RECEIVED) 2797 goto reschedule; 2798 2799 ntb->peer_msix_good = true; 2800 /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2801 goto reschedule; 2802 2803msix_good: 2804 ntb_poll_link(ntb); 2805 ntb_link_event(ntb); 2806 return; 2807 2808reschedule: 2809 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2810 if (_xeon_link_is_up(ntb)) { 2811 callout_reset(&ntb->peer_msix_work, 2812 hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2813 ntb_exchange_msix, ntb); 2814 } else 2815 ntb_spad_clear(ntb); 2816} 2817 2818/* 2819 * Public API to the rest of the OS 2820 */ 2821 2822/** 2823 * ntb_get_max_spads() - get the total scratch regs usable 2824 * @ntb: pointer to ntb_softc instance 2825 * 2826 * This function returns the max 32bit scratchpad registers usable by the 2827 * upper layer. 2828 * 2829 * RETURNS: total number of scratch pad registers available 2830 */ 2831uint8_t 2832ntb_get_max_spads(struct ntb_softc *ntb) 2833{ 2834 2835 return (ntb->spad_count); 2836} 2837 2838/* 2839 * ntb_mw_count() - Get the number of memory windows available for KPI 2840 * consumers. 2841 * 2842 * (Excludes any MW wholly reserved for register access.) 2843 */ 2844uint8_t 2845ntb_mw_count(struct ntb_softc *ntb) 2846{ 2847 uint8_t res; 2848 2849 res = ntb->mw_count; 2850 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2851 res--; 2852 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2853 res--; 2854 return (res); 2855} 2856 2857/** 2858 * ntb_spad_write() - write to the secondary scratchpad register 2859 * @ntb: pointer to ntb_softc instance 2860 * @idx: index to the scratchpad register, 0 based 2861 * @val: the data value to put into the register 2862 * 2863 * This function allows writing of a 32bit value to the indexed scratchpad 2864 * register. The register resides on the secondary (external) side. 2865 * 2866 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2867 */ 2868int 2869ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2870{ 2871 2872 if (idx >= ntb->spad_count) 2873 return (EINVAL); 2874 2875 ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2876 2877 return (0); 2878} 2879 2880/* 2881 * Zeros the local scratchpad. 2882 */ 2883void 2884ntb_spad_clear(struct ntb_softc *ntb) 2885{ 2886 unsigned i; 2887 2888 for (i = 0; i < ntb->spad_count; i++) 2889 ntb_spad_write(ntb, i, 0); 2890} 2891 2892/** 2893 * ntb_spad_read() - read from the primary scratchpad register 2894 * @ntb: pointer to ntb_softc instance 2895 * @idx: index to scratchpad register, 0 based 2896 * @val: pointer to 32bit integer for storing the register value 2897 * 2898 * This function allows reading of the 32bit scratchpad register on 2899 * the primary (internal) side. 2900 * 2901 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2902 */ 2903int 2904ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2905{ 2906 2907 if (idx >= ntb->spad_count) 2908 return (EINVAL); 2909 2910 *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2911 2912 return (0); 2913} 2914 2915/** 2916 * ntb_peer_spad_write() - write to the secondary scratchpad register 2917 * @ntb: pointer to ntb_softc instance 2918 * @idx: index to the scratchpad register, 0 based 2919 * @val: the data value to put into the register 2920 * 2921 * This function allows writing of a 32bit value to the indexed scratchpad 2922 * register. The register resides on the secondary (external) side. 2923 * 2924 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2925 */ 2926int 2927ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2928{ 2929 2930 if (idx >= ntb->spad_count) 2931 return (EINVAL); 2932 2933 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2934 ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2935 else 2936 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2937 2938 return (0); 2939} 2940 2941/** 2942 * ntb_peer_spad_read() - read from the primary scratchpad register 2943 * @ntb: pointer to ntb_softc instance 2944 * @idx: index to scratchpad register, 0 based 2945 * @val: pointer to 32bit integer for storing the register value 2946 * 2947 * This function allows reading of the 32bit scratchpad register on 2948 * the primary (internal) side. 2949 * 2950 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2951 */ 2952int 2953ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2954{ 2955 2956 if (idx >= ntb->spad_count) 2957 return (EINVAL); 2958 2959 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2960 *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2961 else 2962 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2963 2964 return (0); 2965} 2966 2967/* 2968 * ntb_mw_get_range() - get the range of a memory window 2969 * @ntb: NTB device context 2970 * @idx: Memory window number 2971 * @base: OUT - the base address for mapping the memory window 2972 * @size: OUT - the size for mapping the memory window 2973 * @align: OUT - the base alignment for translating the memory window 2974 * @align_size: OUT - the size alignment for translating the memory window 2975 * 2976 * Get the range of a memory window. NULL may be given for any output 2977 * parameter if the value is not needed. The base and size may be used for 2978 * mapping the memory window, to access the peer memory. The alignment and 2979 * size may be used for translating the memory window, for the peer to access 2980 * memory on the local system. 2981 * 2982 * Return: Zero on success, otherwise an error number. 2983 */ 2984int 2985ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 2986 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2987 bus_addr_t *plimit) 2988{ 2989 struct ntb_pci_bar_info *bar; 2990 bus_addr_t limit; 2991 size_t bar_b2b_off; 2992 enum ntb_bar bar_num; 2993 2994 if (mw_idx >= ntb_mw_count(ntb)) 2995 return (EINVAL); 2996 mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 2997 2998 bar_num = ntb_mw_to_bar(ntb, mw_idx); 2999 bar = &ntb->bar_info[bar_num]; 3000 bar_b2b_off = 0; 3001 if (mw_idx == ntb->b2b_mw_idx) { 3002 KASSERT(ntb->b2b_off != 0, 3003 ("user shouldn't get non-shared b2b mw")); 3004 bar_b2b_off = ntb->b2b_off; 3005 } 3006 3007 if (bar_is_64bit(ntb, bar_num)) 3008 limit = BUS_SPACE_MAXADDR; 3009 else 3010 limit = BUS_SPACE_MAXADDR_32BIT; 3011 3012 if (base != NULL) 3013 *base = bar->pbase + bar_b2b_off; 3014 if (vbase != NULL) 3015 *vbase = bar->vbase + bar_b2b_off; 3016 if (size != NULL) 3017 *size = bar->size - bar_b2b_off; 3018 if (align != NULL) 3019 *align = bar->size; 3020 if (align_size != NULL) 3021 *align_size = 1; 3022 if (plimit != NULL) 3023 *plimit = limit; 3024 return (0); 3025} 3026 3027/* 3028 * ntb_mw_set_trans() - set the translation of a memory window 3029 * @ntb: NTB device context 3030 * @idx: Memory window number 3031 * @addr: The dma address local memory to expose to the peer 3032 * @size: The size of the local memory to expose to the peer 3033 * 3034 * Set the translation of a memory window. The peer may access local memory 3035 * through the window starting at the address, up to the size. The address 3036 * must be aligned to the alignment specified by ntb_mw_get_range(). The size 3037 * must be aligned to the size alignment specified by ntb_mw_get_range(). The 3038 * address must be below the plimit specified by ntb_mw_get_range() (i.e. for 3039 * 32-bit BARs). 3040 * 3041 * Return: Zero on success, otherwise an error number. 3042 */ 3043int 3044ntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 3045 size_t size) 3046{ 3047 struct ntb_pci_bar_info *bar; 3048 uint64_t base, limit, reg_val; 3049 size_t bar_size, mw_size; 3050 uint32_t base_reg, xlat_reg, limit_reg; 3051 enum ntb_bar bar_num; 3052 3053 if (idx >= ntb_mw_count(ntb)) 3054 return (EINVAL); 3055 idx = ntb_user_mw_to_idx(ntb, idx); 3056 3057 bar_num = ntb_mw_to_bar(ntb, idx); 3058 bar = &ntb->bar_info[bar_num]; 3059 3060 bar_size = bar->size; 3061 if (idx == ntb->b2b_mw_idx) 3062 mw_size = bar_size - ntb->b2b_off; 3063 else 3064 mw_size = bar_size; 3065 3066 /* Hardware requires that addr is aligned to bar size */ 3067 if ((addr & (bar_size - 1)) != 0) 3068 return (EINVAL); 3069 3070 if (size > mw_size) 3071 return (EINVAL); 3072 3073 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 3074 3075 limit = 0; 3076 if (bar_is_64bit(ntb, bar_num)) { 3077 base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 3078 3079 if (limit_reg != 0 && size != mw_size) 3080 limit = base + size; 3081 3082 /* Set and verify translation address */ 3083 ntb_reg_write(8, xlat_reg, addr); 3084 reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 3085 if (reg_val != addr) { 3086 ntb_reg_write(8, xlat_reg, 0); 3087 return (EIO); 3088 } 3089 3090 /* Set and verify the limit */ 3091 ntb_reg_write(8, limit_reg, limit); 3092 reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3093 if (reg_val != limit) { 3094 ntb_reg_write(8, limit_reg, base); 3095 ntb_reg_write(8, xlat_reg, 0); 3096 return (EIO); 3097 } 3098 } else { 3099 /* Configure 32-bit (split) BAR MW */ 3100 3101 if ((addr & UINT32_MAX) != addr) 3102 return (ERANGE); 3103 if (((addr + size) & UINT32_MAX) != (addr + size)) 3104 return (ERANGE); 3105 3106 base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3107 3108 if (limit_reg != 0 && size != mw_size) 3109 limit = base + size; 3110 3111 /* Set and verify translation address */ 3112 ntb_reg_write(4, xlat_reg, addr); 3113 reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3114 if (reg_val != addr) { 3115 ntb_reg_write(4, xlat_reg, 0); 3116 return (EIO); 3117 } 3118 3119 /* Set and verify the limit */ 3120 ntb_reg_write(4, limit_reg, limit); 3121 reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3122 if (reg_val != limit) { 3123 ntb_reg_write(4, limit_reg, base); 3124 ntb_reg_write(4, xlat_reg, 0); 3125 return (EIO); 3126 } 3127 } 3128 return (0); 3129} 3130 3131/* 3132 * ntb_mw_clear_trans() - clear the translation of a memory window 3133 * @ntb: NTB device context 3134 * @idx: Memory window number 3135 * 3136 * Clear the translation of a memory window. The peer may no longer access 3137 * local memory through the window. 3138 * 3139 * Return: Zero on success, otherwise an error number. 3140 */ 3141int 3142ntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 3143{ 3144 3145 return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 3146} 3147 3148/* 3149 * ntb_mw_get_wc - Get the write-combine status of a memory window 3150 * 3151 * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 3152 * idx is an invalid memory window). 3153 * 3154 * Mode is a VM_MEMATTR_* type. 3155 */ 3156int 3157ntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t *mode) 3158{ 3159 struct ntb_pci_bar_info *bar; 3160 3161 if (idx >= ntb_mw_count(ntb)) 3162 return (EINVAL); 3163 idx = ntb_user_mw_to_idx(ntb, idx); 3164 3165 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3166 *mode = bar->map_mode; 3167 return (0); 3168} 3169 3170/* 3171 * ntb_mw_set_wc - Set the write-combine status of a memory window 3172 * 3173 * If 'mode' matches the current status, this does nothing and succeeds. Mode 3174 * is a VM_MEMATTR_* type. 3175 * 3176 * Returns: Zero on success, setting the caching attribute on the virtual 3177 * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 3178 * memory window, or if changing the caching attribute fails). 3179 */ 3180int 3181ntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3182{ 3183 3184 if (idx >= ntb_mw_count(ntb)) 3185 return (EINVAL); 3186 3187 idx = ntb_user_mw_to_idx(ntb, idx); 3188 return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3189} 3190 3191static int 3192ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3193{ 3194 struct ntb_pci_bar_info *bar; 3195 int rc; 3196 3197 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3198 if (bar->map_mode == mode) 3199 return (0); 3200 3201 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3202 if (rc == 0) 3203 bar->map_mode = mode; 3204 3205 return (rc); 3206} 3207 3208/** 3209 * ntb_peer_db_set() - Set the doorbell on the secondary/external side 3210 * @ntb: pointer to ntb_softc instance 3211 * @bit: doorbell bits to ring 3212 * 3213 * This function allows triggering of a doorbell on the secondary/external 3214 * side that will initiate an interrupt on the remote host 3215 */ 3216void 3217ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 3218{ 3219 3220 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 3221 struct ntb_pci_bar_info *lapic; 3222 unsigned i; 3223 3224 lapic = ntb->peer_lapic_bar; 3225 3226 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3227 if ((bit & ntb_db_vector_mask(ntb, i)) != 0) 3228 bus_space_write_4(lapic->pci_bus_tag, 3229 lapic->pci_bus_handle, 3230 ntb->peer_msix_data[i].nmd_ofs, 3231 ntb->peer_msix_data[i].nmd_data); 3232 } 3233 return; 3234 } 3235 3236 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3237 ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3238 return; 3239 } 3240 3241 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3242} 3243 3244/* 3245 * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 3246 * as well as the size of the register (via *sz_out). 3247 * 3248 * This function allows a caller using I/OAT DMA to chain the remote doorbell 3249 * ring to its memory window write. 3250 * 3251 * Note that writing the peer doorbell via a memory window will *not* generate 3252 * an interrupt on the remote host; that must be done separately. 3253 */ 3254bus_addr_t 3255ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 3256{ 3257 struct ntb_pci_bar_info *bar; 3258 uint64_t regoff; 3259 3260 KASSERT(sz_out != NULL, ("must be non-NULL")); 3261 3262 if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3263 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3264 regoff = ntb->peer_reg->db_bell; 3265 } else { 3266 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3267 ("invalid b2b idx")); 3268 3269 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3270 regoff = XEON_PDOORBELL_OFFSET; 3271 } 3272 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3273 3274 *sz_out = ntb->reg->db_size; 3275 /* HACK: Specific to current x86 bus implementation. */ 3276 return ((uint64_t)bar->pci_bus_handle + regoff); 3277} 3278 3279/* 3280 * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 3281 * @ntb: NTB device context 3282 * 3283 * Hardware may support different number or arrangement of doorbell bits. 3284 * 3285 * Return: A mask of doorbell bits supported by the ntb. 3286 */ 3287uint64_t 3288ntb_db_valid_mask(struct ntb_softc *ntb) 3289{ 3290 3291 return (ntb->db_valid_mask); 3292} 3293 3294/* 3295 * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 3296 * @ntb: NTB device context 3297 * @vector: Doorbell vector number 3298 * 3299 * Each interrupt vector may have a different number or arrangement of bits. 3300 * 3301 * Return: A mask of doorbell bits serviced by a vector. 3302 */ 3303uint64_t 3304ntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 3305{ 3306 3307 if (vector > ntb->db_vec_count) 3308 return (0); 3309 return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3310} 3311 3312/** 3313 * ntb_link_is_up() - get the current ntb link state 3314 * @ntb: NTB device context 3315 * @speed: OUT - The link speed expressed as PCIe generation number 3316 * @width: OUT - The link width expressed as the number of PCIe lanes 3317 * 3318 * RETURNS: true or false based on the hardware link state 3319 */ 3320bool 3321ntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 3322 enum ntb_width *width) 3323{ 3324 3325 if (speed != NULL) 3326 *speed = ntb_link_sta_speed(ntb); 3327 if (width != NULL) 3328 *width = ntb_link_sta_width(ntb); 3329 return (link_is_up(ntb)); 3330} 3331 3332static void 3333save_bar_parameters(struct ntb_pci_bar_info *bar) 3334{ 3335 3336 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3337 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3338 bar->pbase = rman_get_start(bar->pci_resource); 3339 bar->size = rman_get_size(bar->pci_resource); 3340 bar->vbase = rman_get_virtual(bar->pci_resource); 3341} 3342 3343device_t 3344ntb_get_device(struct ntb_softc *ntb) 3345{ 3346 3347 return (ntb->device); 3348} 3349 3350/* Export HW-specific errata information. */ 3351bool 3352ntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 3353{ 3354 3355 return (HAS_FEATURE(feature)); 3356} 3357