ntb_hw.c revision 304376
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * Copyright (C) 2015 EMC Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c 304376 2016-08-18 10:24:31Z mav $"); 30 31#include <sys/param.h> 32#include <sys/kernel.h> 33#include <sys/systm.h> 34#include <sys/bus.h> 35#include <sys/endian.h> 36#include <sys/malloc.h> 37#include <sys/module.h> 38#include <sys/mutex.h> 39#include <sys/pciio.h> 40#include <sys/queue.h> 41#include <sys/rman.h> 42#include <sys/sbuf.h> 43#include <sys/sysctl.h> 44#include <vm/vm.h> 45#include <vm/pmap.h> 46#include <machine/bus.h> 47#include <machine/intr_machdep.h> 48#include <machine/pmap.h> 49#include <machine/resource.h> 50#include <dev/pci/pcireg.h> 51#include <dev/pci/pcivar.h> 52 53#include "ntb_regs.h" 54#include "ntb_hw.h" 55 56/* 57 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 58 * allows you to connect two systems using a PCI-e link. 59 * 60 * This module contains the hardware abstraction layer for the NTB. It allows 61 * you to send and recieve interrupts, map the memory windows and send and 62 * receive messages in the scratch-pad registers. 63 * 64 * NOTE: Much of the code in this module is shared with Linux. Any patches may 65 * be picked up and redistributed in Linux with a dual GPL/BSD license. 66 */ 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 72#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 73 74#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 75 76#define NTB_MSIX_VER_GUARD 0xaabbccdd 77#define NTB_MSIX_RECEIVED 0xe0f0e0f0 78 79/* 80 * PCI constants could be somewhere more generic, but aren't defined/used in 81 * pci.c. 82 */ 83#define PCI_MSIX_ENTRY_SIZE 16 84#define PCI_MSIX_ENTRY_LOWER_ADDR 0 85#define PCI_MSIX_ENTRY_UPPER_ADDR 4 86#define PCI_MSIX_ENTRY_DATA 8 87 88enum ntb_device_type { 89 NTB_XEON, 90 NTB_ATOM 91}; 92 93/* ntb_conn_type are hardware numbers, cannot change. */ 94enum ntb_conn_type { 95 NTB_CONN_TRANSPARENT = 0, 96 NTB_CONN_B2B = 1, 97 NTB_CONN_RP = 2, 98}; 99 100enum ntb_b2b_direction { 101 NTB_DEV_USD = 0, 102 NTB_DEV_DSD = 1, 103}; 104 105enum ntb_bar { 106 NTB_CONFIG_BAR = 0, 107 NTB_B2B_BAR_1, 108 NTB_B2B_BAR_2, 109 NTB_B2B_BAR_3, 110 NTB_MAX_BARS 111}; 112 113enum { 114 NTB_MSIX_GUARD = 0, 115 NTB_MSIX_DATA0, 116 NTB_MSIX_DATA1, 117 NTB_MSIX_DATA2, 118 NTB_MSIX_OFS0, 119 NTB_MSIX_OFS1, 120 NTB_MSIX_OFS2, 121 NTB_MSIX_DONE, 122 NTB_MAX_MSIX_SPAD 123}; 124 125/* Device features and workarounds */ 126#define HAS_FEATURE(feature) \ 127 ((ntb->features & (feature)) != 0) 128 129struct ntb_hw_info { 130 uint32_t device_id; 131 const char *desc; 132 enum ntb_device_type type; 133 uint32_t features; 134}; 135 136struct ntb_pci_bar_info { 137 bus_space_tag_t pci_bus_tag; 138 bus_space_handle_t pci_bus_handle; 139 int pci_resource_id; 140 struct resource *pci_resource; 141 vm_paddr_t pbase; 142 caddr_t vbase; 143 vm_size_t size; 144 vm_memattr_t map_mode; 145 146 /* Configuration register offsets */ 147 uint32_t psz_off; 148 uint32_t ssz_off; 149 uint32_t pbarxlat_off; 150}; 151 152struct ntb_int_info { 153 struct resource *res; 154 int rid; 155 void *tag; 156}; 157 158struct ntb_vec { 159 struct ntb_softc *ntb; 160 uint32_t num; 161 unsigned masked; 162}; 163 164struct ntb_reg { 165 uint32_t ntb_ctl; 166 uint32_t lnk_sta; 167 uint8_t db_size; 168 unsigned mw_bar[NTB_MAX_BARS]; 169}; 170 171struct ntb_alt_reg { 172 uint32_t db_bell; 173 uint32_t db_mask; 174 uint32_t spad; 175}; 176 177struct ntb_xlat_reg { 178 uint32_t bar0_base; 179 uint32_t bar2_base; 180 uint32_t bar4_base; 181 uint32_t bar5_base; 182 183 uint32_t bar2_xlat; 184 uint32_t bar4_xlat; 185 uint32_t bar5_xlat; 186 187 uint32_t bar2_limit; 188 uint32_t bar4_limit; 189 uint32_t bar5_limit; 190}; 191 192struct ntb_b2b_addr { 193 uint64_t bar0_addr; 194 uint64_t bar2_addr64; 195 uint64_t bar4_addr64; 196 uint64_t bar4_addr32; 197 uint64_t bar5_addr32; 198}; 199 200struct ntb_msix_data { 201 uint32_t nmd_ofs; 202 uint32_t nmd_data; 203}; 204 205struct ntb_softc { 206 device_t device; 207 enum ntb_device_type type; 208 uint32_t features; 209 210 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 211 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 212 uint32_t allocated_interrupts; 213 214 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216 bool peer_msix_good; 217 bool peer_msix_done; 218 struct ntb_pci_bar_info *peer_lapic_bar; 219 struct callout peer_msix_work; 220 221 struct callout heartbeat_timer; 222 struct callout lr_timer; 223 224 void *ntb_ctx; 225 const struct ntb_ctx_ops *ctx_ops; 226 struct ntb_vec *msix_vec; 227#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 228#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 229#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 230 struct mtx ctx_lock; 231 232 uint32_t ppd; 233 enum ntb_conn_type conn_type; 234 enum ntb_b2b_direction dev_type; 235 236 /* Offset of peer bar0 in B2B BAR */ 237 uint64_t b2b_off; 238 /* Memory window used to access peer bar0 */ 239#define B2B_MW_DISABLED UINT8_MAX 240 uint8_t b2b_mw_idx; 241 uint32_t msix_xlat; 242 uint8_t msix_mw_idx; 243 244 uint8_t mw_count; 245 uint8_t spad_count; 246 uint8_t db_count; 247 uint8_t db_vec_count; 248 uint8_t db_vec_shift; 249 250 /* Protects local db_mask. */ 251#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 252#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 253#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 254 struct mtx db_mask_lock; 255 256 volatile uint32_t ntb_ctl; 257 volatile uint32_t lnk_sta; 258 259 uint64_t db_valid_mask; 260 uint64_t db_link_mask; 261 uint64_t db_mask; 262 263 int last_ts; /* ticks @ last irq */ 264 265 const struct ntb_reg *reg; 266 const struct ntb_alt_reg *self_reg; 267 const struct ntb_alt_reg *peer_reg; 268 const struct ntb_xlat_reg *xlat_reg; 269}; 270 271#ifdef __i386__ 272static __inline uint64_t 273bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 274 bus_size_t offset) 275{ 276 277 return (bus_space_read_4(tag, handle, offset) | 278 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 279} 280 281static __inline void 282bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 283 bus_size_t offset, uint64_t val) 284{ 285 286 bus_space_write_4(tag, handle, offset, val); 287 bus_space_write_4(tag, handle, offset + 4, val >> 32); 288} 289#endif 290 291#define ntb_bar_read(SIZE, bar, offset) \ 292 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 293 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 294#define ntb_bar_write(SIZE, bar, offset, val) \ 295 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 296 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 297#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 298#define ntb_reg_write(SIZE, offset, val) \ 299 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 300#define ntb_mw_read(SIZE, offset) \ 301 ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 302#define ntb_mw_write(SIZE, offset, val) \ 303 ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 304 offset, val) 305 306static int ntb_probe(device_t device); 307static int ntb_attach(device_t device); 308static int ntb_detach(device_t device); 309static unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 310static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 311static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 312static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 313 uint32_t *base, uint32_t *xlat, uint32_t *lmt); 314static int ntb_map_pci_bars(struct ntb_softc *ntb); 315static int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 316 vm_memattr_t); 317static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 318 const char *); 319static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 320static int map_memory_window_bar(struct ntb_softc *ntb, 321 struct ntb_pci_bar_info *bar); 322static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 323static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 324static int ntb_init_isr(struct ntb_softc *ntb); 325static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 326static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 327static void ntb_teardown_interrupts(struct ntb_softc *ntb); 328static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 329static void ntb_interrupt(struct ntb_softc *, uint32_t vec); 330static void ndev_vec_isr(void *arg); 331static void ndev_irq_isr(void *arg); 332static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 333static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 334static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 335static int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 336static void ntb_free_msix_vec(struct ntb_softc *ntb); 337static void ntb_get_msix_info(struct ntb_softc *ntb); 338static void ntb_exchange_msix(void *); 339static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 340static void ntb_detect_max_mw(struct ntb_softc *ntb); 341static int ntb_detect_xeon(struct ntb_softc *ntb); 342static int ntb_detect_atom(struct ntb_softc *ntb); 343static int ntb_xeon_init_dev(struct ntb_softc *ntb); 344static int ntb_atom_init_dev(struct ntb_softc *ntb); 345static void ntb_teardown_xeon(struct ntb_softc *ntb); 346static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 347static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 348 enum ntb_bar regbar); 349static void xeon_set_sbar_base_and_limit(struct ntb_softc *, 350 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 351static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 352 enum ntb_bar idx); 353static int xeon_setup_b2b_mw(struct ntb_softc *, 354 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 355static int xeon_setup_msix_bar(struct ntb_softc *); 356static inline bool link_is_up(struct ntb_softc *ntb); 357static inline bool _xeon_link_is_up(struct ntb_softc *ntb); 358static inline bool atom_link_is_err(struct ntb_softc *ntb); 359static inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 360static inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 361static void atom_link_hb(void *arg); 362static void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 363static void recover_atom_link(void *arg); 364static bool ntb_poll_link(struct ntb_softc *ntb); 365static void save_bar_parameters(struct ntb_pci_bar_info *bar); 366static void ntb_sysctl_init(struct ntb_softc *); 367static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 368static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 369static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 370static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 371static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 372 373static unsigned g_ntb_hw_debug_level; 374TUNABLE_INT("hw.ntb.debug_level", &g_ntb_hw_debug_level); 375SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 376 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 377#define ntb_printf(lvl, ...) do { \ 378 if ((lvl) <= g_ntb_hw_debug_level) { \ 379 device_printf(ntb->device, __VA_ARGS__); \ 380 } \ 381} while (0) 382 383#define _NTB_PAT_UC 0 384#define _NTB_PAT_WC 1 385#define _NTB_PAT_WT 4 386#define _NTB_PAT_WP 5 387#define _NTB_PAT_WB 6 388#define _NTB_PAT_UCM 7 389static unsigned g_ntb_mw_pat = _NTB_PAT_UC; 390TUNABLE_INT("hw.ntb.default_mw_pat", &g_ntb_mw_pat); 391SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 392 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 393 "UC: " __XSTRING(_NTB_PAT_UC) ", " 394 "WC: " __XSTRING(_NTB_PAT_WC) ", " 395 "WT: " __XSTRING(_NTB_PAT_WT) ", " 396 "WP: " __XSTRING(_NTB_PAT_WP) ", " 397 "WB: " __XSTRING(_NTB_PAT_WB) ", " 398 "UC-: " __XSTRING(_NTB_PAT_UCM)); 399 400static inline vm_memattr_t 401ntb_pat_flags(void) 402{ 403 404 switch (g_ntb_mw_pat) { 405 case _NTB_PAT_WC: 406 return (VM_MEMATTR_WRITE_COMBINING); 407 case _NTB_PAT_WT: 408 return (VM_MEMATTR_WRITE_THROUGH); 409 case _NTB_PAT_WP: 410 return (VM_MEMATTR_WRITE_PROTECTED); 411 case _NTB_PAT_WB: 412 return (VM_MEMATTR_WRITE_BACK); 413 case _NTB_PAT_UCM: 414 return (VM_MEMATTR_WEAK_UNCACHEABLE); 415 case _NTB_PAT_UC: 416 /* FALLTHROUGH */ 417 default: 418 return (VM_MEMATTR_UNCACHEABLE); 419 } 420} 421 422/* 423 * Well, this obviously doesn't belong here, but it doesn't seem to exist 424 * anywhere better yet. 425 */ 426static inline const char * 427ntb_vm_memattr_to_str(vm_memattr_t pat) 428{ 429 430 switch (pat) { 431 case VM_MEMATTR_WRITE_COMBINING: 432 return ("WRITE_COMBINING"); 433 case VM_MEMATTR_WRITE_THROUGH: 434 return ("WRITE_THROUGH"); 435 case VM_MEMATTR_WRITE_PROTECTED: 436 return ("WRITE_PROTECTED"); 437 case VM_MEMATTR_WRITE_BACK: 438 return ("WRITE_BACK"); 439 case VM_MEMATTR_WEAK_UNCACHEABLE: 440 return ("UNCACHED"); 441 case VM_MEMATTR_UNCACHEABLE: 442 return ("UNCACHEABLE"); 443 default: 444 return ("UNKNOWN"); 445 } 446} 447 448static int g_ntb_msix_idx = 0; 449SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 450 0, "Use this memory window to access the peer MSIX message complex on " 451 "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 452 "Like b2b_mw_idx, negative values index from the last available memory " 453 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 454 455static int g_ntb_mw_idx = -1; 456TUNABLE_INT("hw.ntb.b2b_mw_idx", &g_ntb_mw_idx); 457SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 458 0, "Use this memory window to access the peer NTB registers. A " 459 "non-negative value starts from the first MW index; a negative value " 460 "starts from the last MW index. The default is -1, i.e., the last " 461 "available memory window. Both sides of the NTB MUST set the same " 462 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 463 464static struct ntb_hw_info pci_ids[] = { 465 /* XXX: PS/SS IDs left out until they are supported. */ 466 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 467 NTB_ATOM, 0 }, 468 469 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 470 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 471 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 472 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 473 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 474 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 475 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 476 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 477 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 478 NTB_SB01BASE_LOCKUP }, 479 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 480 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 481 NTB_SB01BASE_LOCKUP }, 482 483 { 0x00000000, NULL, NTB_ATOM, 0 } 484}; 485 486static const struct ntb_reg atom_reg = { 487 .ntb_ctl = ATOM_NTBCNTL_OFFSET, 488 .lnk_sta = ATOM_LINK_STATUS_OFFSET, 489 .db_size = sizeof(uint64_t), 490 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 491}; 492 493static const struct ntb_alt_reg atom_pri_reg = { 494 .db_bell = ATOM_PDOORBELL_OFFSET, 495 .db_mask = ATOM_PDBMSK_OFFSET, 496 .spad = ATOM_SPAD_OFFSET, 497}; 498 499static const struct ntb_alt_reg atom_b2b_reg = { 500 .db_bell = ATOM_B2B_DOORBELL_OFFSET, 501 .spad = ATOM_B2B_SPAD_OFFSET, 502}; 503 504static const struct ntb_xlat_reg atom_sec_xlat = { 505#if 0 506 /* "FIXME" says the Linux driver. */ 507 .bar0_base = ATOM_SBAR0BASE_OFFSET, 508 .bar2_base = ATOM_SBAR2BASE_OFFSET, 509 .bar4_base = ATOM_SBAR4BASE_OFFSET, 510 511 .bar2_limit = ATOM_SBAR2LMT_OFFSET, 512 .bar4_limit = ATOM_SBAR4LMT_OFFSET, 513#endif 514 515 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 516 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 517}; 518 519static const struct ntb_reg xeon_reg = { 520 .ntb_ctl = XEON_NTBCNTL_OFFSET, 521 .lnk_sta = XEON_LINK_STATUS_OFFSET, 522 .db_size = sizeof(uint16_t), 523 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 524}; 525 526static const struct ntb_alt_reg xeon_pri_reg = { 527 .db_bell = XEON_PDOORBELL_OFFSET, 528 .db_mask = XEON_PDBMSK_OFFSET, 529 .spad = XEON_SPAD_OFFSET, 530}; 531 532static const struct ntb_alt_reg xeon_b2b_reg = { 533 .db_bell = XEON_B2B_DOORBELL_OFFSET, 534 .spad = XEON_B2B_SPAD_OFFSET, 535}; 536 537static const struct ntb_xlat_reg xeon_sec_xlat = { 538 .bar0_base = XEON_SBAR0BASE_OFFSET, 539 .bar2_base = XEON_SBAR2BASE_OFFSET, 540 .bar4_base = XEON_SBAR4BASE_OFFSET, 541 .bar5_base = XEON_SBAR5BASE_OFFSET, 542 543 .bar2_limit = XEON_SBAR2LMT_OFFSET, 544 .bar4_limit = XEON_SBAR4LMT_OFFSET, 545 .bar5_limit = XEON_SBAR5LMT_OFFSET, 546 547 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 548 .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 549 .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 550}; 551 552static struct ntb_b2b_addr xeon_b2b_usd_addr = { 553 .bar0_addr = XEON_B2B_BAR0_ADDR, 554 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 555 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 556 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 557 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 558}; 559 560static struct ntb_b2b_addr xeon_b2b_dsd_addr = { 561 .bar0_addr = XEON_B2B_BAR0_ADDR, 562 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 563 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 564 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 565 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 566}; 567 568SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 569 "B2B MW segment overrides -- MUST be the same on both sides"); 570 571TUNABLE_QUAD("hw.ntb.usd_bar2_addr64", &xeon_b2b_usd_addr.bar2_addr64); 572SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 573 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 574 "hardware, use this 64-bit address on the bus between the NTB devices for " 575 "the window at BAR2, on the upstream side of the link. MUST be the same " 576 "address on both sides."); 577TUNABLE_QUAD("hw.ntb.usd_bar4_addr64", &xeon_b2b_usd_addr.bar4_addr64); 578SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 579 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 580TUNABLE_QUAD("hw.ntb.usd_bar4_addr32", &xeon_b2b_usd_addr.bar4_addr32); 581SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 582 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 583 "(split-BAR mode)."); 584TUNABLE_QUAD("hw.ntb.usd_bar5_addr32", &xeon_b2b_usd_addr.bar5_addr32); 585SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 586 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 587 "(split-BAR mode)."); 588 589TUNABLE_QUAD("hw.ntb.dsd_bar2_addr64", &xeon_b2b_dsd_addr.bar2_addr64); 590SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 591 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 592 "hardware, use this 64-bit address on the bus between the NTB devices for " 593 "the window at BAR2, on the downstream side of the link. MUST be the same" 594 " address on both sides."); 595TUNABLE_QUAD("hw.ntb.dsd_bar4_addr64", &xeon_b2b_dsd_addr.bar4_addr64); 596SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 597 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 598TUNABLE_QUAD("hw.ntb.dsd_bar4_addr32", &xeon_b2b_dsd_addr.bar4_addr32); 599SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 600 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 601 "(split-BAR mode)."); 602TUNABLE_QUAD("hw.ntb.dsd_bar5_addr32", &xeon_b2b_dsd_addr.bar5_addr32); 603SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 604 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 605 "(split-BAR mode)."); 606 607/* 608 * OS <-> Driver interface structures 609 */ 610MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 611 612static device_method_t ntb_pci_methods[] = { 613 /* Device interface */ 614 DEVMETHOD(device_probe, ntb_probe), 615 DEVMETHOD(device_attach, ntb_attach), 616 DEVMETHOD(device_detach, ntb_detach), 617 DEVMETHOD_END 618}; 619 620static driver_t ntb_pci_driver = { 621 "ntb_hw", 622 ntb_pci_methods, 623 sizeof(struct ntb_softc), 624}; 625 626static devclass_t ntb_devclass; 627DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 628MODULE_VERSION(ntb_hw, 1); 629 630SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 631 632/* 633 * OS <-> Driver linkage functions 634 */ 635static int 636ntb_probe(device_t device) 637{ 638 struct ntb_hw_info *p; 639 640 p = ntb_get_device_info(pci_get_devid(device)); 641 if (p == NULL) 642 return (ENXIO); 643 644 device_set_desc(device, p->desc); 645 return (0); 646} 647 648static int 649ntb_attach(device_t device) 650{ 651 struct ntb_softc *ntb; 652 struct ntb_hw_info *p; 653 int error; 654 655 ntb = DEVICE2SOFTC(device); 656 p = ntb_get_device_info(pci_get_devid(device)); 657 658 ntb->device = device; 659 ntb->type = p->type; 660 ntb->features = p->features; 661 ntb->b2b_mw_idx = B2B_MW_DISABLED; 662 ntb->msix_mw_idx = B2B_MW_DISABLED; 663 664 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 665 callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE); 666 callout_init(&ntb->lr_timer, CALLOUT_MPSAFE); 667 callout_init(&ntb->peer_msix_work, 1); 668 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 669 mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 670 671 if (ntb->type == NTB_ATOM) 672 error = ntb_detect_atom(ntb); 673 else 674 error = ntb_detect_xeon(ntb); 675 if (error != 0) 676 goto out; 677 678 ntb_detect_max_mw(ntb); 679 680 pci_enable_busmaster(ntb->device); 681 682 error = ntb_map_pci_bars(ntb); 683 if (error != 0) 684 goto out; 685 if (ntb->type == NTB_ATOM) 686 error = ntb_atom_init_dev(ntb); 687 else 688 error = ntb_xeon_init_dev(ntb); 689 if (error != 0) 690 goto out; 691 692 ntb_spad_clear(ntb); 693 694 ntb_poll_link(ntb); 695 696 ntb_sysctl_init(ntb); 697 698out: 699 if (error != 0) 700 ntb_detach(device); 701 return (error); 702} 703 704static int 705ntb_detach(device_t device) 706{ 707 struct ntb_softc *ntb; 708 709 ntb = DEVICE2SOFTC(device); 710 711 if (ntb->self_reg != NULL) { 712 DB_MASK_LOCK(ntb); 713 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 714 DB_MASK_UNLOCK(ntb); 715 } 716 callout_drain(&ntb->heartbeat_timer); 717 callout_drain(&ntb->lr_timer); 718 callout_drain(&ntb->peer_msix_work); 719 pci_disable_busmaster(ntb->device); 720 if (ntb->type == NTB_XEON) 721 ntb_teardown_xeon(ntb); 722 ntb_teardown_interrupts(ntb); 723 724 mtx_destroy(&ntb->db_mask_lock); 725 mtx_destroy(&ntb->ctx_lock); 726 727 ntb_unmap_pci_bar(ntb); 728 729 return (0); 730} 731 732/* 733 * Driver internal routines 734 */ 735static inline enum ntb_bar 736ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 737{ 738 739 KASSERT(mw < ntb->mw_count, 740 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 741 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 742 743 return (ntb->reg->mw_bar[mw]); 744} 745 746static inline bool 747bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 748{ 749 /* XXX This assertion could be stronger. */ 750 KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 751 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 752} 753 754static inline void 755bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 756 uint32_t *xlat, uint32_t *lmt) 757{ 758 uint32_t basev, lmtv, xlatv; 759 760 switch (bar) { 761 case NTB_B2B_BAR_1: 762 basev = ntb->xlat_reg->bar2_base; 763 lmtv = ntb->xlat_reg->bar2_limit; 764 xlatv = ntb->xlat_reg->bar2_xlat; 765 break; 766 case NTB_B2B_BAR_2: 767 basev = ntb->xlat_reg->bar4_base; 768 lmtv = ntb->xlat_reg->bar4_limit; 769 xlatv = ntb->xlat_reg->bar4_xlat; 770 break; 771 case NTB_B2B_BAR_3: 772 basev = ntb->xlat_reg->bar5_base; 773 lmtv = ntb->xlat_reg->bar5_limit; 774 xlatv = ntb->xlat_reg->bar5_xlat; 775 break; 776 default: 777 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 778 ("bad bar")); 779 basev = lmtv = xlatv = 0; 780 break; 781 } 782 783 if (base != NULL) 784 *base = basev; 785 if (xlat != NULL) 786 *xlat = xlatv; 787 if (lmt != NULL) 788 *lmt = lmtv; 789} 790 791static int 792ntb_map_pci_bars(struct ntb_softc *ntb) 793{ 794 int rc; 795 796 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 797 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 798 if (rc != 0) 799 goto out; 800 801 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 802 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 803 if (rc != 0) 804 goto out; 805 ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 806 ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 807 ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 808 809 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 810 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 811 if (rc != 0) 812 goto out; 813 ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 814 ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 815 ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 816 817 if (!HAS_FEATURE(NTB_SPLIT_BAR)) 818 goto out; 819 820 ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 821 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 822 ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 823 ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 824 ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 825 826out: 827 if (rc != 0) 828 device_printf(ntb->device, 829 "unable to allocate pci resource\n"); 830 return (rc); 831} 832 833static void 834print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 835 const char *kind) 836{ 837 838 device_printf(ntb->device, 839 "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 840 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 841 (char *)bar->vbase + bar->size - 1, 842 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 843 (uintmax_t)bar->size, kind); 844} 845 846static int 847map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 848{ 849 850 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 851 &bar->pci_resource_id, RF_ACTIVE); 852 if (bar->pci_resource == NULL) 853 return (ENXIO); 854 855 save_bar_parameters(bar); 856 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 857 print_map_success(ntb, bar, "mmr"); 858 return (0); 859} 860 861static int 862map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 863{ 864 int rc; 865 vm_memattr_t mapmode; 866 uint8_t bar_size_bits = 0; 867 868 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 869 &bar->pci_resource_id, RF_ACTIVE); 870 871 if (bar->pci_resource == NULL) 872 return (ENXIO); 873 874 save_bar_parameters(bar); 875 /* 876 * Ivytown NTB BAR sizes are misreported by the hardware due to a 877 * hardware issue. To work around this, query the size it should be 878 * configured to by the device and modify the resource to correspond to 879 * this new size. The BIOS on systems with this problem is required to 880 * provide enough address space to allow the driver to make this change 881 * safely. 882 * 883 * Ideally I could have just specified the size when I allocated the 884 * resource like: 885 * bus_alloc_resource(ntb->device, 886 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 887 * 1ul << bar_size_bits, RF_ACTIVE); 888 * but the PCI driver does not honor the size in this call, so we have 889 * to modify it after the fact. 890 */ 891 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 892 if (bar->pci_resource_id == PCIR_BAR(2)) 893 bar_size_bits = pci_read_config(ntb->device, 894 XEON_PBAR23SZ_OFFSET, 1); 895 else 896 bar_size_bits = pci_read_config(ntb->device, 897 XEON_PBAR45SZ_OFFSET, 1); 898 899 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 900 bar->pci_resource, bar->pbase, 901 bar->pbase + (1ul << bar_size_bits) - 1); 902 if (rc != 0) { 903 device_printf(ntb->device, 904 "unable to resize bar\n"); 905 return (rc); 906 } 907 908 save_bar_parameters(bar); 909 } 910 911 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 912 print_map_success(ntb, bar, "mw"); 913 914 /* 915 * Optionally, mark MW BARs as anything other than UC to improve 916 * performance. 917 */ 918 mapmode = ntb_pat_flags(); 919 if (mapmode == bar->map_mode) 920 return (0); 921 922 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 923 if (rc == 0) { 924 bar->map_mode = mapmode; 925 device_printf(ntb->device, 926 "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 927 "%s.\n", 928 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 929 (char *)bar->vbase + bar->size - 1, 930 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 931 ntb_vm_memattr_to_str(mapmode)); 932 } else 933 device_printf(ntb->device, 934 "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 935 "%s: %d\n", 936 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 937 (char *)bar->vbase + bar->size - 1, 938 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 939 ntb_vm_memattr_to_str(mapmode), rc); 940 /* Proceed anyway */ 941 return (0); 942} 943 944static void 945ntb_unmap_pci_bar(struct ntb_softc *ntb) 946{ 947 struct ntb_pci_bar_info *current_bar; 948 int i; 949 950 for (i = 0; i < NTB_MAX_BARS; i++) { 951 current_bar = &ntb->bar_info[i]; 952 if (current_bar->pci_resource != NULL) 953 bus_release_resource(ntb->device, SYS_RES_MEMORY, 954 current_bar->pci_resource_id, 955 current_bar->pci_resource); 956 } 957} 958 959static int 960ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 961{ 962 uint32_t i; 963 int rc; 964 965 for (i = 0; i < num_vectors; i++) { 966 ntb->int_info[i].rid = i + 1; 967 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 968 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 969 if (ntb->int_info[i].res == NULL) { 970 device_printf(ntb->device, 971 "bus_alloc_resource failed\n"); 972 return (ENOMEM); 973 } 974 ntb->int_info[i].tag = NULL; 975 ntb->allocated_interrupts++; 976 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 977 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 978 &ntb->msix_vec[i], &ntb->int_info[i].tag); 979 if (rc != 0) { 980 device_printf(ntb->device, "bus_setup_intr failed\n"); 981 return (ENXIO); 982 } 983 } 984 return (0); 985} 986 987/* 988 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 989 * cannot be allocated for each MSI-X message. JHB seems to think remapping 990 * should be okay. This tunable should enable us to test that hypothesis 991 * when someone gets their hands on some Xeon hardware. 992 */ 993static int ntb_force_remap_mode; 994TUNABLE_INT("hw.ntb.force_remap_mode", &ntb_force_remap_mode); 995SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 996 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 997 " to a smaller number of ithreads, even if the desired number are " 998 "available"); 999 1000/* 1001 * In case it is NOT ok, give consumers an abort button. 1002 */ 1003static int ntb_prefer_intx; 1004TUNABLE_INT("hw.ntb.prefer_intx_to_remap", &ntb_prefer_intx); 1005SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 1006 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1007 "than remapping MSI-X messages over available slots (match Linux driver " 1008 "behavior)"); 1009 1010/* 1011 * Remap the desired number of MSI-X messages to available ithreads in a simple 1012 * round-robin fashion. 1013 */ 1014static int 1015ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1016{ 1017 u_int *vectors; 1018 uint32_t i; 1019 int rc; 1020 1021 if (ntb_prefer_intx != 0) 1022 return (ENXIO); 1023 1024 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1025 1026 for (i = 0; i < desired; i++) 1027 vectors[i] = (i % avail) + 1; 1028 1029 rc = pci_remap_msix(dev, desired, vectors); 1030 free(vectors, M_NTB); 1031 return (rc); 1032} 1033 1034static int 1035ntb_init_isr(struct ntb_softc *ntb) 1036{ 1037 uint32_t desired_vectors, num_vectors; 1038 int rc; 1039 1040 ntb->allocated_interrupts = 0; 1041 ntb->last_ts = ticks; 1042 1043 /* 1044 * Mask all doorbell interrupts. (Except link events!) 1045 */ 1046 DB_MASK_LOCK(ntb); 1047 ntb->db_mask = ntb->db_valid_mask; 1048 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1049 DB_MASK_UNLOCK(ntb); 1050 1051 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1052 ntb->db_count); 1053 if (desired_vectors >= 1) { 1054 rc = pci_alloc_msix(ntb->device, &num_vectors); 1055 1056 if (ntb_force_remap_mode != 0 && rc == 0 && 1057 num_vectors == desired_vectors) 1058 num_vectors--; 1059 1060 if (rc == 0 && num_vectors < desired_vectors) { 1061 rc = ntb_remap_msix(ntb->device, desired_vectors, 1062 num_vectors); 1063 if (rc == 0) 1064 num_vectors = desired_vectors; 1065 else 1066 pci_release_msi(ntb->device); 1067 } 1068 if (rc != 0) 1069 num_vectors = 1; 1070 } else 1071 num_vectors = 1; 1072 1073 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1074 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1075 device_printf(ntb->device, 1076 "Errata workaround does not support MSI or INTX\n"); 1077 return (EINVAL); 1078 } 1079 1080 ntb->db_vec_count = 1; 1081 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1082 rc = ntb_setup_legacy_interrupt(ntb); 1083 } else { 1084 if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1085 HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1086 device_printf(ntb->device, 1087 "Errata workaround expects %d doorbell bits\n", 1088 XEON_NONLINK_DB_MSIX_BITS); 1089 return (EINVAL); 1090 } 1091 1092 ntb_create_msix_vec(ntb, num_vectors); 1093 rc = ntb_setup_msix(ntb, num_vectors); 1094 if (rc == 0 && HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1095 ntb_get_msix_info(ntb); 1096 } 1097 if (rc != 0) { 1098 device_printf(ntb->device, 1099 "Error allocating interrupts: %d\n", rc); 1100 ntb_free_msix_vec(ntb); 1101 } 1102 1103 return (rc); 1104} 1105 1106static int 1107ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1108{ 1109 int rc; 1110 1111 ntb->int_info[0].rid = 0; 1112 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1113 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1114 if (ntb->int_info[0].res == NULL) { 1115 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1116 return (ENOMEM); 1117 } 1118 1119 ntb->int_info[0].tag = NULL; 1120 ntb->allocated_interrupts = 1; 1121 1122 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1123 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1124 ntb, &ntb->int_info[0].tag); 1125 if (rc != 0) { 1126 device_printf(ntb->device, "bus_setup_intr failed\n"); 1127 return (ENXIO); 1128 } 1129 1130 return (0); 1131} 1132 1133static void 1134ntb_teardown_interrupts(struct ntb_softc *ntb) 1135{ 1136 struct ntb_int_info *current_int; 1137 int i; 1138 1139 for (i = 0; i < ntb->allocated_interrupts; i++) { 1140 current_int = &ntb->int_info[i]; 1141 if (current_int->tag != NULL) 1142 bus_teardown_intr(ntb->device, current_int->res, 1143 current_int->tag); 1144 1145 if (current_int->res != NULL) 1146 bus_release_resource(ntb->device, SYS_RES_IRQ, 1147 rman_get_rid(current_int->res), current_int->res); 1148 } 1149 1150 ntb_free_msix_vec(ntb); 1151 pci_release_msi(ntb->device); 1152} 1153 1154/* 1155 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1156 * out to make code clearer. 1157 */ 1158static inline uint64_t 1159db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1160{ 1161 1162 if (ntb->type == NTB_ATOM) 1163 return (ntb_reg_read(8, regoff)); 1164 1165 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1166 1167 return (ntb_reg_read(2, regoff)); 1168} 1169 1170static inline void 1171db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1172{ 1173 1174 KASSERT((val & ~ntb->db_valid_mask) == 0, 1175 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1176 (uintmax_t)(val & ~ntb->db_valid_mask), 1177 (uintmax_t)ntb->db_valid_mask)); 1178 1179 if (regoff == ntb->self_reg->db_mask) 1180 DB_MASK_ASSERT(ntb, MA_OWNED); 1181 db_iowrite_raw(ntb, regoff, val); 1182} 1183 1184static inline void 1185db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1186{ 1187 1188 if (ntb->type == NTB_ATOM) { 1189 ntb_reg_write(8, regoff, val); 1190 return; 1191 } 1192 1193 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1194 ntb_reg_write(2, regoff, (uint16_t)val); 1195} 1196 1197void 1198ntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1199{ 1200 1201 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1202 return; 1203 1204 DB_MASK_LOCK(ntb); 1205 ntb->db_mask |= bits; 1206 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1207 DB_MASK_UNLOCK(ntb); 1208} 1209 1210void 1211ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1212{ 1213 1214 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1215 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1216 (uintmax_t)(bits & ~ntb->db_valid_mask), 1217 (uintmax_t)ntb->db_valid_mask)); 1218 1219 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1220 return; 1221 1222 DB_MASK_LOCK(ntb); 1223 ntb->db_mask &= ~bits; 1224 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1225 DB_MASK_UNLOCK(ntb); 1226} 1227 1228uint64_t 1229ntb_db_read(struct ntb_softc *ntb) 1230{ 1231 1232 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1233 uint64_t res; 1234 unsigned i; 1235 1236 res = 0; 1237 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1238 if (ntb->msix_vec[i].masked != 0) 1239 res |= ntb_db_vector_mask(ntb, i); 1240 } 1241 return (res); 1242 } 1243 1244 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1245} 1246 1247void 1248ntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1249{ 1250 1251 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1252 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1253 (uintmax_t)(bits & ~ntb->db_valid_mask), 1254 (uintmax_t)ntb->db_valid_mask)); 1255 1256 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1257 unsigned i; 1258 1259 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1260 if ((bits & ntb_db_vector_mask(ntb, i)) != 0) { 1261 DB_MASK_LOCK(ntb); 1262 if (ntb->msix_vec[i].masked != 0) { 1263 /* XXX These need a public API. */ 1264#if 0 1265 pci_unmask_msix(ntb->device, i); 1266#endif 1267 ntb->msix_vec[i].masked = 0; 1268 } 1269 DB_MASK_UNLOCK(ntb); 1270 } 1271 } 1272 return; 1273 } 1274 1275 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1276} 1277 1278static inline uint64_t 1279ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1280{ 1281 uint64_t shift, mask; 1282 1283 shift = ntb->db_vec_shift; 1284 mask = (1ull << shift) - 1; 1285 return (mask << (shift * db_vector)); 1286} 1287 1288static void 1289ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1290{ 1291 uint64_t vec_mask; 1292 1293 ntb->last_ts = ticks; 1294 vec_mask = ntb_vec_mask(ntb, vec); 1295 1296 if ((vec_mask & ntb->db_link_mask) != 0) { 1297 if (ntb_poll_link(ntb)) 1298 ntb_link_event(ntb); 1299 } 1300 1301 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1302 (vec_mask & ntb->db_link_mask) == 0) { 1303 DB_MASK_LOCK(ntb); 1304 if (ntb->msix_vec[vec].masked == 0) { 1305 /* XXX These need a public API. */ 1306#if 0 1307 pci_mask_msix(ntb->device, vec); 1308#endif 1309 ntb->msix_vec[vec].masked = 1; 1310 } 1311 DB_MASK_UNLOCK(ntb); 1312 } 1313 1314 if ((vec_mask & ntb->db_valid_mask) != 0) 1315 ntb_db_event(ntb, vec); 1316} 1317 1318static void 1319ndev_vec_isr(void *arg) 1320{ 1321 struct ntb_vec *nvec = arg; 1322 1323 ntb_interrupt(nvec->ntb, nvec->num); 1324} 1325 1326static void 1327ndev_irq_isr(void *arg) 1328{ 1329 /* If we couldn't set up MSI-X, we only have the one vector. */ 1330 ntb_interrupt(arg, 0); 1331} 1332 1333static int 1334ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1335{ 1336 uint32_t i; 1337 1338 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1339 M_ZERO | M_WAITOK); 1340 for (i = 0; i < num_vectors; i++) { 1341 ntb->msix_vec[i].num = i; 1342 ntb->msix_vec[i].ntb = ntb; 1343 } 1344 1345 return (0); 1346} 1347 1348static void 1349ntb_free_msix_vec(struct ntb_softc *ntb) 1350{ 1351 1352 if (ntb->msix_vec == NULL) 1353 return; 1354 1355 free(ntb->msix_vec, M_NTB); 1356 ntb->msix_vec = NULL; 1357} 1358 1359static void 1360ntb_get_msix_info(struct ntb_softc *ntb) 1361{ 1362 struct pci_devinfo *dinfo; 1363 struct pcicfg_msix *msix; 1364 uint32_t laddr, data, i, offset; 1365 1366 dinfo = device_get_ivars(ntb->device); 1367 msix = &dinfo->cfg.msix; 1368 1369 laddr = data = 0; 1370 1371 CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1372 1373 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1374 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1375 1376 laddr = bus_read_4(msix->msix_table_res, offset + 1377 PCI_MSIX_ENTRY_LOWER_ADDR); 1378 ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1379 1380 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1381 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1382 MSI_INTEL_ADDR_BASE)); 1383 ntb->msix_data[i].nmd_ofs = laddr; 1384 1385 data = bus_read_4(msix->msix_table_res, offset + 1386 PCI_MSIX_ENTRY_DATA); 1387 ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1388 1389 ntb->msix_data[i].nmd_data = data; 1390 } 1391} 1392 1393static struct ntb_hw_info * 1394ntb_get_device_info(uint32_t device_id) 1395{ 1396 struct ntb_hw_info *ep = pci_ids; 1397 1398 while (ep->device_id) { 1399 if (ep->device_id == device_id) 1400 return (ep); 1401 ++ep; 1402 } 1403 return (NULL); 1404} 1405 1406static void 1407ntb_teardown_xeon(struct ntb_softc *ntb) 1408{ 1409 1410 if (ntb->reg != NULL) 1411 ntb_link_disable(ntb); 1412} 1413 1414static void 1415ntb_detect_max_mw(struct ntb_softc *ntb) 1416{ 1417 1418 if (ntb->type == NTB_ATOM) { 1419 ntb->mw_count = ATOM_MW_COUNT; 1420 return; 1421 } 1422 1423 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1424 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1425 else 1426 ntb->mw_count = XEON_SNB_MW_COUNT; 1427} 1428 1429static int 1430ntb_detect_xeon(struct ntb_softc *ntb) 1431{ 1432 uint8_t ppd, conn_type; 1433 1434 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1435 ntb->ppd = ppd; 1436 1437 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1438 ntb->dev_type = NTB_DEV_DSD; 1439 else 1440 ntb->dev_type = NTB_DEV_USD; 1441 1442 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1443 ntb->features |= NTB_SPLIT_BAR; 1444 1445 /* 1446 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1447 * errata workaround; only do one at a time. 1448 */ 1449 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1450 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1451 1452 conn_type = ppd & XEON_PPD_CONN_TYPE; 1453 switch (conn_type) { 1454 case NTB_CONN_B2B: 1455 ntb->conn_type = conn_type; 1456 break; 1457 case NTB_CONN_RP: 1458 case NTB_CONN_TRANSPARENT: 1459 default: 1460 device_printf(ntb->device, "Unsupported connection type: %u\n", 1461 (unsigned)conn_type); 1462 return (ENXIO); 1463 } 1464 return (0); 1465} 1466 1467static int 1468ntb_detect_atom(struct ntb_softc *ntb) 1469{ 1470 uint32_t ppd, conn_type; 1471 1472 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1473 ntb->ppd = ppd; 1474 1475 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1476 ntb->dev_type = NTB_DEV_DSD; 1477 else 1478 ntb->dev_type = NTB_DEV_USD; 1479 1480 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1481 switch (conn_type) { 1482 case NTB_CONN_B2B: 1483 ntb->conn_type = conn_type; 1484 break; 1485 default: 1486 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1487 return (ENXIO); 1488 } 1489 return (0); 1490} 1491 1492static int 1493ntb_xeon_init_dev(struct ntb_softc *ntb) 1494{ 1495 int rc; 1496 1497 ntb->spad_count = XEON_SPAD_COUNT; 1498 ntb->db_count = XEON_DB_COUNT; 1499 ntb->db_link_mask = XEON_DB_LINK_BIT; 1500 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1501 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1502 1503 if (ntb->conn_type != NTB_CONN_B2B) { 1504 device_printf(ntb->device, "Connection type %d not supported\n", 1505 ntb->conn_type); 1506 return (ENXIO); 1507 } 1508 1509 ntb->reg = &xeon_reg; 1510 ntb->self_reg = &xeon_pri_reg; 1511 ntb->peer_reg = &xeon_b2b_reg; 1512 ntb->xlat_reg = &xeon_sec_xlat; 1513 1514 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1515 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1516 ntb->mw_count; 1517 ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1518 g_ntb_msix_idx, ntb->msix_mw_idx); 1519 rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1520 VM_MEMATTR_UNCACHEABLE); 1521 KASSERT(rc == 0, ("shouldn't fail")); 1522 } else if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1523 /* 1524 * There is a Xeon hardware errata related to writes to SDOORBELL or 1525 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1526 * which may hang the system. To workaround this, use a memory 1527 * window to access the interrupt and scratch pad registers on the 1528 * remote system. 1529 */ 1530 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1531 ntb->mw_count; 1532 ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1533 g_ntb_mw_idx, ntb->b2b_mw_idx); 1534 rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1535 VM_MEMATTR_UNCACHEABLE); 1536 KASSERT(rc == 0, ("shouldn't fail")); 1537 } else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1538 /* 1539 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1540 * mirrored to the remote system. Shrink the number of bits by one, 1541 * since bit 14 is the last bit. 1542 * 1543 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1544 * anyway. Nor for non-B2B connection types. 1545 */ 1546 ntb->db_count = XEON_DB_COUNT - 1; 1547 1548 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1549 1550 if (ntb->dev_type == NTB_DEV_USD) 1551 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1552 &xeon_b2b_usd_addr); 1553 else 1554 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1555 &xeon_b2b_dsd_addr); 1556 if (rc != 0) 1557 return (rc); 1558 1559 /* Enable Bus Master and Memory Space on the secondary side */ 1560 ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1561 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1562 1563 /* 1564 * Mask all doorbell interrupts. 1565 */ 1566 DB_MASK_LOCK(ntb); 1567 ntb->db_mask = ntb->db_valid_mask; 1568 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1569 DB_MASK_UNLOCK(ntb); 1570 1571 rc = xeon_setup_msix_bar(ntb); 1572 if (rc != 0) 1573 return (rc); 1574 1575 rc = ntb_init_isr(ntb); 1576 return (rc); 1577} 1578 1579static int 1580ntb_atom_init_dev(struct ntb_softc *ntb) 1581{ 1582 int error; 1583 1584 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1585 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1586 1587 ntb->spad_count = ATOM_SPAD_COUNT; 1588 ntb->db_count = ATOM_DB_COUNT; 1589 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1590 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1591 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1592 1593 ntb->reg = &atom_reg; 1594 ntb->self_reg = &atom_pri_reg; 1595 ntb->peer_reg = &atom_b2b_reg; 1596 ntb->xlat_reg = &atom_sec_xlat; 1597 1598 /* 1599 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1600 * resolved. Mask transaction layer internal parity errors. 1601 */ 1602 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1603 1604 configure_atom_secondary_side_bars(ntb); 1605 1606 /* Enable Bus Master and Memory Space on the secondary side */ 1607 ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1608 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1609 1610 error = ntb_init_isr(ntb); 1611 if (error != 0) 1612 return (error); 1613 1614 /* Initiate PCI-E link training */ 1615 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1616 1617 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1618 1619 return (0); 1620} 1621 1622/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1623static void 1624configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1625{ 1626 1627 if (ntb->dev_type == NTB_DEV_USD) { 1628 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1629 XEON_B2B_BAR2_ADDR64); 1630 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1631 XEON_B2B_BAR4_ADDR64); 1632 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1633 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1634 } else { 1635 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1636 XEON_B2B_BAR2_ADDR64); 1637 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1638 XEON_B2B_BAR4_ADDR64); 1639 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1640 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1641 } 1642} 1643 1644 1645/* 1646 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1647 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1648 * remains for use by a higher layer. 1649 * 1650 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1651 * MW size is sufficiently large. 1652 */ 1653static unsigned int ntb_b2b_mw_share; 1654TUNABLE_INT("hw.ntb.b2b_mw_share", &ntb_b2b_mw_share); 1655SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1656 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1657 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1658 "value here."); 1659 1660static void 1661xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1662 enum ntb_bar regbar) 1663{ 1664 struct ntb_pci_bar_info *bar; 1665 uint8_t bar_sz; 1666 1667 if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1668 return; 1669 1670 bar = &ntb->bar_info[idx]; 1671 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1672 if (idx == regbar) { 1673 if (ntb->b2b_off != 0) 1674 bar_sz--; 1675 else 1676 bar_sz = 0; 1677 } 1678 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1679 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1680 (void)bar_sz; 1681} 1682 1683static void 1684xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1685 enum ntb_bar idx, enum ntb_bar regbar) 1686{ 1687 uint64_t reg_val; 1688 uint32_t base_reg, lmt_reg; 1689 1690 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1691 if (idx == regbar) { 1692 if (ntb->b2b_off) 1693 bar_addr += ntb->b2b_off; 1694 else 1695 bar_addr = 0; 1696 } 1697 1698 /* 1699 * Set limit registers first to avoid an errata where setting the base 1700 * registers locks the limit registers. 1701 */ 1702 if (!bar_is_64bit(ntb, idx)) { 1703 ntb_reg_write(4, lmt_reg, bar_addr); 1704 reg_val = ntb_reg_read(4, lmt_reg); 1705 (void)reg_val; 1706 1707 ntb_reg_write(4, base_reg, bar_addr); 1708 reg_val = ntb_reg_read(4, base_reg); 1709 (void)reg_val; 1710 } else { 1711 ntb_reg_write(8, lmt_reg, bar_addr); 1712 reg_val = ntb_reg_read(8, lmt_reg); 1713 (void)reg_val; 1714 1715 ntb_reg_write(8, base_reg, bar_addr); 1716 reg_val = ntb_reg_read(8, base_reg); 1717 (void)reg_val; 1718 } 1719} 1720 1721static void 1722xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1723{ 1724 struct ntb_pci_bar_info *bar; 1725 1726 bar = &ntb->bar_info[idx]; 1727 if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1728 ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1729 base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1730 } else { 1731 ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1732 base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1733 } 1734 (void)base_addr; 1735} 1736 1737static int 1738xeon_setup_msix_bar(struct ntb_softc *ntb) 1739{ 1740 enum ntb_bar bar_num; 1741 1742 if (!HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1743 return (0); 1744 1745 bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1746 ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1747 return (0); 1748} 1749 1750static int 1751xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1752 const struct ntb_b2b_addr *peer_addr) 1753{ 1754 struct ntb_pci_bar_info *b2b_bar; 1755 vm_size_t bar_size; 1756 uint64_t bar_addr; 1757 enum ntb_bar b2b_bar_num, i; 1758 1759 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1760 b2b_bar = NULL; 1761 b2b_bar_num = NTB_CONFIG_BAR; 1762 ntb->b2b_off = 0; 1763 } else { 1764 b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1765 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1766 ("invalid b2b mw bar")); 1767 1768 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1769 bar_size = b2b_bar->size; 1770 1771 if (ntb_b2b_mw_share != 0 && 1772 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1773 ntb->b2b_off = bar_size >> 1; 1774 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1775 ntb->b2b_off = 0; 1776 } else { 1777 device_printf(ntb->device, 1778 "B2B bar size is too small!\n"); 1779 return (EIO); 1780 } 1781 } 1782 1783 /* 1784 * Reset the secondary bar sizes to match the primary bar sizes. 1785 * (Except, disable or halve the size of the B2B secondary bar.) 1786 */ 1787 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1788 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1789 1790 bar_addr = 0; 1791 if (b2b_bar_num == NTB_CONFIG_BAR) 1792 bar_addr = addr->bar0_addr; 1793 else if (b2b_bar_num == NTB_B2B_BAR_1) 1794 bar_addr = addr->bar2_addr64; 1795 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1796 bar_addr = addr->bar4_addr64; 1797 else if (b2b_bar_num == NTB_B2B_BAR_2) 1798 bar_addr = addr->bar4_addr32; 1799 else if (b2b_bar_num == NTB_B2B_BAR_3) 1800 bar_addr = addr->bar5_addr32; 1801 else 1802 KASSERT(false, ("invalid bar")); 1803 1804 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1805 1806 /* 1807 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1808 * register BAR. The B2B BAR is either disabled above or configured 1809 * half-size. It starts at PBAR xlat + offset. 1810 * 1811 * Also set up incoming BAR limits == base (zero length window). 1812 */ 1813 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1814 b2b_bar_num); 1815 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1816 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1817 NTB_B2B_BAR_2, b2b_bar_num); 1818 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1819 NTB_B2B_BAR_3, b2b_bar_num); 1820 } else 1821 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1822 NTB_B2B_BAR_2, b2b_bar_num); 1823 1824 /* Zero incoming translation addrs */ 1825 ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1826 ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1827 1828 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1829 size_t size, xlatoffset; 1830 1831 switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1832 case NTB_B2B_BAR_1: 1833 size = 8; 1834 xlatoffset = XEON_SBAR2XLAT_OFFSET; 1835 break; 1836 case NTB_B2B_BAR_2: 1837 xlatoffset = XEON_SBAR4XLAT_OFFSET; 1838 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1839 size = 4; 1840 else 1841 size = 8; 1842 break; 1843 case NTB_B2B_BAR_3: 1844 xlatoffset = XEON_SBAR5XLAT_OFFSET; 1845 size = 4; 1846 break; 1847 default: 1848 KASSERT(false, ("Bogus msix mw idx: %u", 1849 ntb->msix_mw_idx)); 1850 return (EINVAL); 1851 } 1852 1853 /* 1854 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1855 * workaround 1856 */ 1857 if (size == 4) { 1858 ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1859 ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1860 } else { 1861 ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1862 ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1863 } 1864 } 1865 (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1866 (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1867 1868 /* Zero outgoing translation limits (whole bar size windows) */ 1869 ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1870 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1871 1872 /* Set outgoing translation offsets */ 1873 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1874 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1875 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1876 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1877 } else 1878 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1879 1880 /* Set the translation offset for B2B registers */ 1881 bar_addr = 0; 1882 if (b2b_bar_num == NTB_CONFIG_BAR) 1883 bar_addr = peer_addr->bar0_addr; 1884 else if (b2b_bar_num == NTB_B2B_BAR_1) 1885 bar_addr = peer_addr->bar2_addr64; 1886 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1887 bar_addr = peer_addr->bar4_addr64; 1888 else if (b2b_bar_num == NTB_B2B_BAR_2) 1889 bar_addr = peer_addr->bar4_addr32; 1890 else if (b2b_bar_num == NTB_B2B_BAR_3) 1891 bar_addr = peer_addr->bar5_addr32; 1892 else 1893 KASSERT(false, ("invalid bar")); 1894 1895 /* 1896 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1897 * at a time. 1898 */ 1899 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1900 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1901 return (0); 1902} 1903 1904static inline bool 1905_xeon_link_is_up(struct ntb_softc *ntb) 1906{ 1907 1908 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1909 return (true); 1910 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1911} 1912 1913static inline bool 1914link_is_up(struct ntb_softc *ntb) 1915{ 1916 1917 if (ntb->type == NTB_XEON) 1918 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1919 !HAS_FEATURE(NTB_SB01BASE_LOCKUP))); 1920 1921 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1922 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1923} 1924 1925static inline bool 1926atom_link_is_err(struct ntb_softc *ntb) 1927{ 1928 uint32_t status; 1929 1930 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1931 1932 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1933 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1934 return (true); 1935 1936 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1937 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1938} 1939 1940/* Atom does not have link status interrupt, poll on that platform */ 1941static void 1942atom_link_hb(void *arg) 1943{ 1944 struct ntb_softc *ntb = arg; 1945 sbintime_t timo, poll_ts; 1946 1947 timo = NTB_HB_TIMEOUT * hz; 1948 poll_ts = ntb->last_ts + timo; 1949 1950 /* 1951 * Delay polling the link status if an interrupt was received, unless 1952 * the cached link status says the link is down. 1953 */ 1954 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1955 timo = poll_ts - ticks; 1956 goto out; 1957 } 1958 1959 if (ntb_poll_link(ntb)) 1960 ntb_link_event(ntb); 1961 1962 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1963 /* Link is down with error, proceed with recovery */ 1964 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1965 return; 1966 } 1967 1968out: 1969 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1970} 1971 1972static void 1973atom_perform_link_restart(struct ntb_softc *ntb) 1974{ 1975 uint32_t status; 1976 1977 /* Driver resets the NTB ModPhy lanes - magic! */ 1978 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1979 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1980 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1981 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1982 1983 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1984 pause("ModPhy", hz / 10); 1985 1986 /* Clear AER Errors, write to clear */ 1987 status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1988 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1989 ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1990 1991 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1992 status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1993 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1994 ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1995 1996 /* Clear DeSkew Buffer error, write to clear */ 1997 status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1998 status |= ATOM_DESKEWSTS_DBERR; 1999 ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 2000 2001 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2002 status &= ATOM_IBIST_ERR_OFLOW; 2003 ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2004 2005 /* Releases the NTB state machine to allow the link to retrain */ 2006 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2007 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2008 ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2009} 2010 2011/* 2012 * ntb_set_ctx() - associate a driver context with an ntb device 2013 * @ntb: NTB device context 2014 * @ctx: Driver context 2015 * @ctx_ops: Driver context operations 2016 * 2017 * Associate a driver context and operations with a ntb device. The context is 2018 * provided by the client driver, and the driver may associate a different 2019 * context with each ntb device. 2020 * 2021 * Return: Zero if the context is associated, otherwise an error number. 2022 */ 2023int 2024ntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 2025{ 2026 2027 if (ctx == NULL || ops == NULL) 2028 return (EINVAL); 2029 if (ntb->ctx_ops != NULL) 2030 return (EINVAL); 2031 2032 CTX_LOCK(ntb); 2033 if (ntb->ctx_ops != NULL) { 2034 CTX_UNLOCK(ntb); 2035 return (EINVAL); 2036 } 2037 ntb->ntb_ctx = ctx; 2038 ntb->ctx_ops = ops; 2039 CTX_UNLOCK(ntb); 2040 2041 return (0); 2042} 2043 2044/* 2045 * It is expected that this will only be used from contexts where the ctx_lock 2046 * is not needed to protect ntb_ctx lifetime. 2047 */ 2048void * 2049ntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 2050{ 2051 2052 KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2053 if (ops != NULL) 2054 *ops = ntb->ctx_ops; 2055 return (ntb->ntb_ctx); 2056} 2057 2058/* 2059 * ntb_clear_ctx() - disassociate any driver context from an ntb device 2060 * @ntb: NTB device context 2061 * 2062 * Clear any association that may exist between a driver context and the ntb 2063 * device. 2064 */ 2065void 2066ntb_clear_ctx(struct ntb_softc *ntb) 2067{ 2068 2069 CTX_LOCK(ntb); 2070 ntb->ntb_ctx = NULL; 2071 ntb->ctx_ops = NULL; 2072 CTX_UNLOCK(ntb); 2073} 2074 2075/* 2076 * ntb_link_event() - notify driver context of a change in link status 2077 * @ntb: NTB device context 2078 * 2079 * Notify the driver context that the link status may have changed. The driver 2080 * should call ntb_link_is_up() to get the current status. 2081 */ 2082void 2083ntb_link_event(struct ntb_softc *ntb) 2084{ 2085 2086 CTX_LOCK(ntb); 2087 if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2088 ntb->ctx_ops->link_event(ntb->ntb_ctx); 2089 CTX_UNLOCK(ntb); 2090} 2091 2092/* 2093 * ntb_db_event() - notify driver context of a doorbell event 2094 * @ntb: NTB device context 2095 * @vector: Interrupt vector number 2096 * 2097 * Notify the driver context of a doorbell event. If hardware supports 2098 * multiple interrupt vectors for doorbells, the vector number indicates which 2099 * vector received the interrupt. The vector number is relative to the first 2100 * vector used for doorbells, starting at zero, and must be less than 2101 * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2102 * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2103 * those bits are associated with the vector number. 2104 */ 2105static void 2106ntb_db_event(struct ntb_softc *ntb, uint32_t vec) 2107{ 2108 2109 CTX_LOCK(ntb); 2110 if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2111 ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2112 CTX_UNLOCK(ntb); 2113} 2114 2115/* 2116 * ntb_link_enable() - enable the link on the secondary side of the ntb 2117 * @ntb: NTB device context 2118 * @max_speed: The maximum link speed expressed as PCIe generation number[0] 2119 * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 2120 * 2121 * Enable the link on the secondary side of the ntb. This can only be done 2122 * from the primary side of the ntb in primary or b2b topology. The ntb device 2123 * should train the link to its maximum speed and width, or the requested speed 2124 * and width, whichever is smaller, if supported. 2125 * 2126 * Return: Zero on success, otherwise an error number. 2127 * 2128 * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 2129 * and width input will be ignored. 2130 */ 2131int 2132ntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 2133 enum ntb_width w __unused) 2134{ 2135 uint32_t cntl; 2136 2137 ntb_printf(2, "%s\n", __func__); 2138 2139 if (ntb->type == NTB_ATOM) { 2140 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2141 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2142 return (0); 2143 } 2144 2145 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2146 ntb_link_event(ntb); 2147 return (0); 2148 } 2149 2150 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2151 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2152 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2153 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2154 if (HAS_FEATURE(NTB_SPLIT_BAR)) 2155 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2156 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2157 return (0); 2158} 2159 2160/* 2161 * ntb_link_disable() - disable the link on the secondary side of the ntb 2162 * @ntb: NTB device context 2163 * 2164 * Disable the link on the secondary side of the ntb. This can only be done 2165 * from the primary side of the ntb in primary or b2b topology. The ntb device 2166 * should disable the link. Returning from this call must indicate that a 2167 * barrier has passed, though with no more writes may pass in either direction 2168 * across the link, except if this call returns an error number. 2169 * 2170 * Return: Zero on success, otherwise an error number. 2171 */ 2172int 2173ntb_link_disable(struct ntb_softc *ntb) 2174{ 2175 uint32_t cntl; 2176 2177 ntb_printf(2, "%s\n", __func__); 2178 2179 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2180 ntb_link_event(ntb); 2181 return (0); 2182 } 2183 2184 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2185 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2186 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2187 if (HAS_FEATURE(NTB_SPLIT_BAR)) 2188 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2189 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2190 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2191 return (0); 2192} 2193 2194bool 2195ntb_link_enabled(struct ntb_softc *ntb) 2196{ 2197 uint32_t cntl; 2198 2199 if (ntb->type == NTB_ATOM) { 2200 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2201 return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2202 } 2203 2204 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2205 return (true); 2206 2207 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2208 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2209} 2210 2211static void 2212recover_atom_link(void *arg) 2213{ 2214 struct ntb_softc *ntb = arg; 2215 unsigned speed, width, oldspeed, oldwidth; 2216 uint32_t status32; 2217 2218 atom_perform_link_restart(ntb); 2219 2220 /* 2221 * There is a potential race between the 2 NTB devices recovering at 2222 * the same time. If the times are the same, the link will not recover 2223 * and the driver will be stuck in this loop forever. Add a random 2224 * interval to the recovery time to prevent this race. 2225 */ 2226 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2227 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2228 2229 if (atom_link_is_err(ntb)) 2230 goto retry; 2231 2232 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2233 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2234 goto out; 2235 2236 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2237 width = NTB_LNK_STA_WIDTH(status32); 2238 speed = status32 & NTB_LINK_SPEED_MASK; 2239 2240 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2241 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2242 if (oldwidth != width || oldspeed != speed) 2243 goto retry; 2244 2245out: 2246 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2247 ntb); 2248 return; 2249 2250retry: 2251 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2252 ntb); 2253} 2254 2255/* 2256 * Polls the HW link status register(s); returns true if something has changed. 2257 */ 2258static bool 2259ntb_poll_link(struct ntb_softc *ntb) 2260{ 2261 uint32_t ntb_cntl; 2262 uint16_t reg_val; 2263 2264 if (ntb->type == NTB_ATOM) { 2265 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2266 if (ntb_cntl == ntb->ntb_ctl) 2267 return (false); 2268 2269 ntb->ntb_ctl = ntb_cntl; 2270 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2271 } else { 2272 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2273 2274 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2275 if (reg_val == ntb->lnk_sta) 2276 return (false); 2277 2278 ntb->lnk_sta = reg_val; 2279 2280 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 2281 if (_xeon_link_is_up(ntb)) { 2282 if (!ntb->peer_msix_good) { 2283 callout_reset(&ntb->peer_msix_work, 0, 2284 ntb_exchange_msix, ntb); 2285 return (false); 2286 } 2287 } else { 2288 ntb->peer_msix_good = false; 2289 ntb->peer_msix_done = false; 2290 } 2291 } 2292 } 2293 return (true); 2294} 2295 2296static inline enum ntb_speed 2297ntb_link_sta_speed(struct ntb_softc *ntb) 2298{ 2299 2300 if (!link_is_up(ntb)) 2301 return (NTB_SPEED_NONE); 2302 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2303} 2304 2305static inline enum ntb_width 2306ntb_link_sta_width(struct ntb_softc *ntb) 2307{ 2308 2309 if (!link_is_up(ntb)) 2310 return (NTB_WIDTH_NONE); 2311 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2312} 2313 2314SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2315 "Driver state, statistics, and HW registers"); 2316 2317#define NTB_REGSZ_MASK (3ul << 30) 2318#define NTB_REG_64 (1ul << 30) 2319#define NTB_REG_32 (2ul << 30) 2320#define NTB_REG_16 (3ul << 30) 2321#define NTB_REG_8 (0ul << 30) 2322 2323#define NTB_DB_READ (1ul << 29) 2324#define NTB_PCI_REG (1ul << 28) 2325#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2326 2327static void 2328ntb_sysctl_init(struct ntb_softc *ntb) 2329{ 2330 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2331 struct sysctl_ctx_list *ctx; 2332 struct sysctl_oid *tree, *tmptree; 2333 2334 ctx = device_get_sysctl_ctx(ntb->device); 2335 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2336 2337 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2338 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2339 sysctl_handle_link_status_human, "A", 2340 "Link status (human readable)"); 2341 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2342 CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2343 "IU", "Link status (1=active, 0=inactive)"); 2344 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2345 CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2346 "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2347 2348 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2349 CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2350 tree_par = SYSCTL_CHILDREN(tree); 2351 2352 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2353 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2354 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2355 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2356 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2357 &ntb->ppd, 0, "Raw PPD register (cached)"); 2358 2359 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2360#ifdef notyet 2361 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2362 &ntb->b2b_mw_idx, 0, 2363 "Index of the MW used for B2B remote register access"); 2364#endif 2365 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2366 CTLFLAG_RD, &ntb->b2b_off, 2367 "If non-zero, offset of B2B register region in shared MW"); 2368 } 2369 2370 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2371 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2372 "Features/errata of this NTB device"); 2373 2374 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2375 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2376 "NTB CTL register (cached)"); 2377 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2378 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2379 "LNK STA register (cached)"); 2380 2381#ifdef notyet 2382 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2383 &ntb->mw_count, 0, "MW count"); 2384 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2385 &ntb->spad_count, 0, "Scratchpad count"); 2386 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2387 &ntb->db_count, 0, "Doorbell count"); 2388 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2389 &ntb->db_vec_count, 0, "Doorbell vector count"); 2390 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2391 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2392#endif 2393 2394 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2395 &ntb->db_valid_mask, "Doorbell valid mask"); 2396 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2397 &ntb->db_link_mask, "Doorbell link mask"); 2398 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2399 &ntb->db_mask, "Doorbell mask (cached)"); 2400 2401 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2402 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2403 regpar = SYSCTL_CHILDREN(tmptree); 2404 2405 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2406 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2407 ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2408 "NTB Control register"); 2409 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2410 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2411 0x19c, sysctl_handle_register, "IU", 2412 "NTB Link Capabilities"); 2413 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2414 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2415 0x1a0, sysctl_handle_register, "IU", 2416 "NTB Link Control register"); 2417 2418 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2419 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2420 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2421 sysctl_handle_register, "QU", "Doorbell mask register"); 2422 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2423 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2424 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2425 sysctl_handle_register, "QU", "Doorbell register"); 2426 2427 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2428 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2429 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2430 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2431 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2432 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2433 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2434 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2435 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2436 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2437 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2438 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2439 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2440 } else { 2441 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2442 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2443 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2444 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2445 } 2446 2447 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2448 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2449 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2450 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2451 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2452 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2453 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2454 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2455 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2456 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2457 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2458 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2459 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2460 } else { 2461 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2462 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2463 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2464 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2465 } 2466 2467 if (ntb->type == NTB_ATOM) 2468 return; 2469 2470 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2471 CTLFLAG_RD, NULL, "Xeon HW statistics"); 2472 statpar = SYSCTL_CHILDREN(tmptree); 2473 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2474 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2475 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2476 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2477 2478 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2479 CTLFLAG_RD, NULL, "Xeon HW errors"); 2480 errpar = SYSCTL_CHILDREN(tmptree); 2481 2482 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2483 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2484 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2485 sysctl_handle_register, "CU", "PPD"); 2486 2487 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2488 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2489 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2490 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2491 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2492 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2493 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2494 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2495 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2496 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2497 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2498 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2499 2500 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2501 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2502 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2503 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2504 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2505 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2506 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2507 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2508 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2509 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2510 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2511 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2512 2513 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2514 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2515 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2516 sysctl_handle_register, "SU", "DEVSTS"); 2517 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2518 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2519 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2520 sysctl_handle_register, "SU", "LNKSTS"); 2521 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2522 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2523 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2524 sysctl_handle_register, "SU", "SLNKSTS"); 2525 2526 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2527 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2528 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2529 sysctl_handle_register, "IU", "UNCERRSTS"); 2530 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2531 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2532 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2533 sysctl_handle_register, "IU", "CORERRSTS"); 2534 2535 if (ntb->conn_type != NTB_CONN_B2B) 2536 return; 2537 2538 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2539 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2540 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2541 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2542 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2543 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2544 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2545 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2546 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2547 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2548 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2549 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2550 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2551 } else { 2552 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2553 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2554 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2555 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2556 } 2557 2558 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2559 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2560 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2561 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2562 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2563 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2564 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2565 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2566 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2567 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2568 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2569 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2570 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2571 } else { 2572 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2573 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2574 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2575 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2576 } 2577 2578 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2579 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2580 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2581 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2582 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2583 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2584 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2585 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2586 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2587 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2588 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2589 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2590 sysctl_handle_register, "IU", 2591 "Secondary BAR4 base register"); 2592 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2593 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2594 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2595 sysctl_handle_register, "IU", 2596 "Secondary BAR5 base register"); 2597 } else { 2598 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2599 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2600 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2601 sysctl_handle_register, "QU", 2602 "Secondary BAR45 base register"); 2603 } 2604} 2605 2606static int 2607sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2608{ 2609 struct ntb_softc *ntb; 2610 struct sbuf sb; 2611 int error; 2612 2613 error = 0; 2614 ntb = arg1; 2615 2616 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2617 2618 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2619 error = sbuf_finish(&sb); 2620 sbuf_delete(&sb); 2621 2622 if (error || !req->newptr) 2623 return (error); 2624 return (EINVAL); 2625} 2626 2627static int 2628sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2629{ 2630 struct ntb_softc *ntb; 2631 unsigned old, new; 2632 int error; 2633 2634 error = 0; 2635 ntb = arg1; 2636 2637 old = ntb_link_enabled(ntb); 2638 2639 error = SYSCTL_OUT(req, &old, sizeof(old)); 2640 if (error != 0 || req->newptr == NULL) 2641 return (error); 2642 2643 error = SYSCTL_IN(req, &new, sizeof(new)); 2644 if (error != 0) 2645 return (error); 2646 2647 ntb_printf(0, "Admin set interface state to '%sabled'\n", 2648 (new != 0)? "en" : "dis"); 2649 2650 if (new != 0) 2651 error = ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2652 else 2653 error = ntb_link_disable(ntb); 2654 return (error); 2655} 2656 2657static int 2658sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2659{ 2660 struct ntb_softc *ntb; 2661 struct sbuf sb; 2662 enum ntb_speed speed; 2663 enum ntb_width width; 2664 int error; 2665 2666 error = 0; 2667 ntb = arg1; 2668 2669 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2670 2671 if (ntb_link_is_up(ntb, &speed, &width)) 2672 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2673 (unsigned)speed, (unsigned)width); 2674 else 2675 sbuf_printf(&sb, "down"); 2676 2677 error = sbuf_finish(&sb); 2678 sbuf_delete(&sb); 2679 2680 if (error || !req->newptr) 2681 return (error); 2682 return (EINVAL); 2683} 2684 2685static int 2686sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2687{ 2688 struct ntb_softc *ntb; 2689 unsigned res; 2690 int error; 2691 2692 error = 0; 2693 ntb = arg1; 2694 2695 res = ntb_link_is_up(ntb, NULL, NULL); 2696 2697 error = SYSCTL_OUT(req, &res, sizeof(res)); 2698 if (error || !req->newptr) 2699 return (error); 2700 return (EINVAL); 2701} 2702 2703static int 2704sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2705{ 2706 struct ntb_softc *ntb; 2707 const void *outp; 2708 uintptr_t sz; 2709 uint64_t umv; 2710 char be[sizeof(umv)]; 2711 size_t outsz; 2712 uint32_t reg; 2713 bool db, pci; 2714 int error; 2715 2716 ntb = arg1; 2717 reg = arg2 & ~NTB_REGFLAGS_MASK; 2718 sz = arg2 & NTB_REGSZ_MASK; 2719 db = (arg2 & NTB_DB_READ) != 0; 2720 pci = (arg2 & NTB_PCI_REG) != 0; 2721 2722 KASSERT(!(db && pci), ("bogus")); 2723 2724 if (db) { 2725 KASSERT(sz == NTB_REG_64, ("bogus")); 2726 umv = db_ioread(ntb, reg); 2727 outsz = sizeof(uint64_t); 2728 } else { 2729 switch (sz) { 2730 case NTB_REG_64: 2731 if (pci) 2732 umv = pci_read_config(ntb->device, reg, 8); 2733 else 2734 umv = ntb_reg_read(8, reg); 2735 outsz = sizeof(uint64_t); 2736 break; 2737 case NTB_REG_32: 2738 if (pci) 2739 umv = pci_read_config(ntb->device, reg, 4); 2740 else 2741 umv = ntb_reg_read(4, reg); 2742 outsz = sizeof(uint32_t); 2743 break; 2744 case NTB_REG_16: 2745 if (pci) 2746 umv = pci_read_config(ntb->device, reg, 2); 2747 else 2748 umv = ntb_reg_read(2, reg); 2749 outsz = sizeof(uint16_t); 2750 break; 2751 case NTB_REG_8: 2752 if (pci) 2753 umv = pci_read_config(ntb->device, reg, 1); 2754 else 2755 umv = ntb_reg_read(1, reg); 2756 outsz = sizeof(uint8_t); 2757 break; 2758 default: 2759 panic("bogus"); 2760 break; 2761 } 2762 } 2763 2764 /* Encode bigendian so that sysctl -x is legible. */ 2765 be64enc(be, umv); 2766 outp = ((char *)be) + sizeof(umv) - outsz; 2767 2768 error = SYSCTL_OUT(req, outp, outsz); 2769 if (error || !req->newptr) 2770 return (error); 2771 return (EINVAL); 2772} 2773 2774static unsigned 2775ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2776{ 2777 2778 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2779 uidx >= ntb->b2b_mw_idx) || 2780 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2781 uidx++; 2782 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2783 uidx >= ntb->b2b_mw_idx) && 2784 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2785 uidx++; 2786 return (uidx); 2787} 2788 2789static void 2790ntb_exchange_msix(void *ctx) 2791{ 2792 struct ntb_softc *ntb; 2793 uint32_t val; 2794 unsigned i; 2795 2796 ntb = ctx; 2797 2798 if (ntb->peer_msix_good) 2799 goto msix_good; 2800 if (ntb->peer_msix_done) 2801 goto msix_done; 2802 2803 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2804 ntb_peer_spad_write(ntb, NTB_MSIX_DATA0 + i, 2805 ntb->msix_data[i].nmd_data); 2806 ntb_peer_spad_write(ntb, NTB_MSIX_OFS0 + i, 2807 ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2808 } 2809 ntb_peer_spad_write(ntb, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2810 2811 ntb_spad_read(ntb, NTB_MSIX_GUARD, &val); 2812 if (val != NTB_MSIX_VER_GUARD) 2813 goto reschedule; 2814 2815 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2816 ntb_spad_read(ntb, NTB_MSIX_DATA0 + i, &val); 2817 ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2818 ntb->peer_msix_data[i].nmd_data = val; 2819 ntb_spad_read(ntb, NTB_MSIX_OFS0 + i, &val); 2820 ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2821 ntb->peer_msix_data[i].nmd_ofs = val; 2822 } 2823 2824 ntb->peer_msix_done = true; 2825 2826msix_done: 2827 ntb_peer_spad_write(ntb, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2828 ntb_spad_read(ntb, NTB_MSIX_DONE, &val); 2829 if (val != NTB_MSIX_RECEIVED) 2830 goto reschedule; 2831 2832 ntb->peer_msix_good = true; 2833 /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2834 goto reschedule; 2835 2836msix_good: 2837 ntb_poll_link(ntb); 2838 ntb_link_event(ntb); 2839 return; 2840 2841reschedule: 2842 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2843 if (_xeon_link_is_up(ntb)) { 2844 callout_reset(&ntb->peer_msix_work, 2845 hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2846 ntb_exchange_msix, ntb); 2847 } else 2848 ntb_spad_clear(ntb); 2849} 2850 2851/* 2852 * Public API to the rest of the OS 2853 */ 2854 2855/** 2856 * ntb_get_max_spads() - get the total scratch regs usable 2857 * @ntb: pointer to ntb_softc instance 2858 * 2859 * This function returns the max 32bit scratchpad registers usable by the 2860 * upper layer. 2861 * 2862 * RETURNS: total number of scratch pad registers available 2863 */ 2864uint8_t 2865ntb_get_max_spads(struct ntb_softc *ntb) 2866{ 2867 2868 return (ntb->spad_count); 2869} 2870 2871/* 2872 * ntb_mw_count() - Get the number of memory windows available for KPI 2873 * consumers. 2874 * 2875 * (Excludes any MW wholly reserved for register access.) 2876 */ 2877uint8_t 2878ntb_mw_count(struct ntb_softc *ntb) 2879{ 2880 uint8_t res; 2881 2882 res = ntb->mw_count; 2883 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2884 res--; 2885 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2886 res--; 2887 return (res); 2888} 2889 2890/** 2891 * ntb_spad_write() - write to the secondary scratchpad register 2892 * @ntb: pointer to ntb_softc instance 2893 * @idx: index to the scratchpad register, 0 based 2894 * @val: the data value to put into the register 2895 * 2896 * This function allows writing of a 32bit value to the indexed scratchpad 2897 * register. The register resides on the secondary (external) side. 2898 * 2899 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2900 */ 2901int 2902ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2903{ 2904 2905 if (idx >= ntb->spad_count) 2906 return (EINVAL); 2907 2908 ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2909 2910 return (0); 2911} 2912 2913/* 2914 * Zeros the local scratchpad. 2915 */ 2916void 2917ntb_spad_clear(struct ntb_softc *ntb) 2918{ 2919 unsigned i; 2920 2921 for (i = 0; i < ntb->spad_count; i++) 2922 ntb_spad_write(ntb, i, 0); 2923} 2924 2925/** 2926 * ntb_spad_read() - read from the primary scratchpad register 2927 * @ntb: pointer to ntb_softc instance 2928 * @idx: index to scratchpad register, 0 based 2929 * @val: pointer to 32bit integer for storing the register value 2930 * 2931 * This function allows reading of the 32bit scratchpad register on 2932 * the primary (internal) side. 2933 * 2934 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2935 */ 2936int 2937ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2938{ 2939 2940 if (idx >= ntb->spad_count) 2941 return (EINVAL); 2942 2943 *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2944 2945 return (0); 2946} 2947 2948/** 2949 * ntb_peer_spad_write() - write to the secondary scratchpad register 2950 * @ntb: pointer to ntb_softc instance 2951 * @idx: index to the scratchpad register, 0 based 2952 * @val: the data value to put into the register 2953 * 2954 * This function allows writing of a 32bit value to the indexed scratchpad 2955 * register. The register resides on the secondary (external) side. 2956 * 2957 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2958 */ 2959int 2960ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2961{ 2962 2963 if (idx >= ntb->spad_count) 2964 return (EINVAL); 2965 2966 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2967 ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2968 else 2969 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2970 2971 return (0); 2972} 2973 2974/** 2975 * ntb_peer_spad_read() - read from the primary scratchpad register 2976 * @ntb: pointer to ntb_softc instance 2977 * @idx: index to scratchpad register, 0 based 2978 * @val: pointer to 32bit integer for storing the register value 2979 * 2980 * This function allows reading of the 32bit scratchpad register on 2981 * the primary (internal) side. 2982 * 2983 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2984 */ 2985int 2986ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2987{ 2988 2989 if (idx >= ntb->spad_count) 2990 return (EINVAL); 2991 2992 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2993 *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2994 else 2995 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2996 2997 return (0); 2998} 2999 3000/* 3001 * ntb_mw_get_range() - get the range of a memory window 3002 * @ntb: NTB device context 3003 * @idx: Memory window number 3004 * @base: OUT - the base address for mapping the memory window 3005 * @size: OUT - the size for mapping the memory window 3006 * @align: OUT - the base alignment for translating the memory window 3007 * @align_size: OUT - the size alignment for translating the memory window 3008 * 3009 * Get the range of a memory window. NULL may be given for any output 3010 * parameter if the value is not needed. The base and size may be used for 3011 * mapping the memory window, to access the peer memory. The alignment and 3012 * size may be used for translating the memory window, for the peer to access 3013 * memory on the local system. 3014 * 3015 * Return: Zero on success, otherwise an error number. 3016 */ 3017int 3018ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 3019 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 3020 bus_addr_t *plimit) 3021{ 3022 struct ntb_pci_bar_info *bar; 3023 bus_addr_t limit; 3024 size_t bar_b2b_off; 3025 enum ntb_bar bar_num; 3026 3027 if (mw_idx >= ntb_mw_count(ntb)) 3028 return (EINVAL); 3029 mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 3030 3031 bar_num = ntb_mw_to_bar(ntb, mw_idx); 3032 bar = &ntb->bar_info[bar_num]; 3033 bar_b2b_off = 0; 3034 if (mw_idx == ntb->b2b_mw_idx) { 3035 KASSERT(ntb->b2b_off != 0, 3036 ("user shouldn't get non-shared b2b mw")); 3037 bar_b2b_off = ntb->b2b_off; 3038 } 3039 3040 if (bar_is_64bit(ntb, bar_num)) 3041 limit = BUS_SPACE_MAXADDR; 3042 else 3043 limit = BUS_SPACE_MAXADDR_32BIT; 3044 3045 if (base != NULL) 3046 *base = bar->pbase + bar_b2b_off; 3047 if (vbase != NULL) 3048 *vbase = bar->vbase + bar_b2b_off; 3049 if (size != NULL) 3050 *size = bar->size - bar_b2b_off; 3051 if (align != NULL) 3052 *align = bar->size; 3053 if (align_size != NULL) 3054 *align_size = 1; 3055 if (plimit != NULL) 3056 *plimit = limit; 3057 return (0); 3058} 3059 3060/* 3061 * ntb_mw_set_trans() - set the translation of a memory window 3062 * @ntb: NTB device context 3063 * @idx: Memory window number 3064 * @addr: The dma address local memory to expose to the peer 3065 * @size: The size of the local memory to expose to the peer 3066 * 3067 * Set the translation of a memory window. The peer may access local memory 3068 * through the window starting at the address, up to the size. The address 3069 * must be aligned to the alignment specified by ntb_mw_get_range(). The size 3070 * must be aligned to the size alignment specified by ntb_mw_get_range(). The 3071 * address must be below the plimit specified by ntb_mw_get_range() (i.e. for 3072 * 32-bit BARs). 3073 * 3074 * Return: Zero on success, otherwise an error number. 3075 */ 3076int 3077ntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 3078 size_t size) 3079{ 3080 struct ntb_pci_bar_info *bar; 3081 uint64_t base, limit, reg_val; 3082 size_t bar_size, mw_size; 3083 uint32_t base_reg, xlat_reg, limit_reg; 3084 enum ntb_bar bar_num; 3085 3086 if (idx >= ntb_mw_count(ntb)) 3087 return (EINVAL); 3088 idx = ntb_user_mw_to_idx(ntb, idx); 3089 3090 bar_num = ntb_mw_to_bar(ntb, idx); 3091 bar = &ntb->bar_info[bar_num]; 3092 3093 bar_size = bar->size; 3094 if (idx == ntb->b2b_mw_idx) 3095 mw_size = bar_size - ntb->b2b_off; 3096 else 3097 mw_size = bar_size; 3098 3099 /* Hardware requires that addr is aligned to bar size */ 3100 if ((addr & (bar_size - 1)) != 0) 3101 return (EINVAL); 3102 3103 if (size > mw_size) 3104 return (EINVAL); 3105 3106 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 3107 3108 limit = 0; 3109 if (bar_is_64bit(ntb, bar_num)) { 3110 base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 3111 3112 if (limit_reg != 0 && size != mw_size) 3113 limit = base + size; 3114 3115 /* Set and verify translation address */ 3116 ntb_reg_write(8, xlat_reg, addr); 3117 reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 3118 if (reg_val != addr) { 3119 ntb_reg_write(8, xlat_reg, 0); 3120 return (EIO); 3121 } 3122 3123 /* Set and verify the limit */ 3124 ntb_reg_write(8, limit_reg, limit); 3125 reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3126 if (reg_val != limit) { 3127 ntb_reg_write(8, limit_reg, base); 3128 ntb_reg_write(8, xlat_reg, 0); 3129 return (EIO); 3130 } 3131 } else { 3132 /* Configure 32-bit (split) BAR MW */ 3133 3134 if ((addr & UINT32_MAX) != addr) 3135 return (ERANGE); 3136 if (((addr + size) & UINT32_MAX) != (addr + size)) 3137 return (ERANGE); 3138 3139 base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3140 3141 if (limit_reg != 0 && size != mw_size) 3142 limit = base + size; 3143 3144 /* Set and verify translation address */ 3145 ntb_reg_write(4, xlat_reg, addr); 3146 reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3147 if (reg_val != addr) { 3148 ntb_reg_write(4, xlat_reg, 0); 3149 return (EIO); 3150 } 3151 3152 /* Set and verify the limit */ 3153 ntb_reg_write(4, limit_reg, limit); 3154 reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3155 if (reg_val != limit) { 3156 ntb_reg_write(4, limit_reg, base); 3157 ntb_reg_write(4, xlat_reg, 0); 3158 return (EIO); 3159 } 3160 } 3161 return (0); 3162} 3163 3164/* 3165 * ntb_mw_clear_trans() - clear the translation of a memory window 3166 * @ntb: NTB device context 3167 * @idx: Memory window number 3168 * 3169 * Clear the translation of a memory window. The peer may no longer access 3170 * local memory through the window. 3171 * 3172 * Return: Zero on success, otherwise an error number. 3173 */ 3174int 3175ntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 3176{ 3177 3178 return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 3179} 3180 3181/* 3182 * ntb_mw_get_wc - Get the write-combine status of a memory window 3183 * 3184 * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 3185 * idx is an invalid memory window). 3186 * 3187 * Mode is a VM_MEMATTR_* type. 3188 */ 3189int 3190ntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t *mode) 3191{ 3192 struct ntb_pci_bar_info *bar; 3193 3194 if (idx >= ntb_mw_count(ntb)) 3195 return (EINVAL); 3196 idx = ntb_user_mw_to_idx(ntb, idx); 3197 3198 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3199 *mode = bar->map_mode; 3200 return (0); 3201} 3202 3203/* 3204 * ntb_mw_set_wc - Set the write-combine status of a memory window 3205 * 3206 * If 'mode' matches the current status, this does nothing and succeeds. Mode 3207 * is a VM_MEMATTR_* type. 3208 * 3209 * Returns: Zero on success, setting the caching attribute on the virtual 3210 * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 3211 * memory window, or if changing the caching attribute fails). 3212 */ 3213int 3214ntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3215{ 3216 3217 if (idx >= ntb_mw_count(ntb)) 3218 return (EINVAL); 3219 3220 idx = ntb_user_mw_to_idx(ntb, idx); 3221 return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3222} 3223 3224static int 3225ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3226{ 3227 struct ntb_pci_bar_info *bar; 3228 int rc; 3229 3230 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3231 if (bar->map_mode == mode) 3232 return (0); 3233 3234 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3235 if (rc == 0) 3236 bar->map_mode = mode; 3237 3238 return (rc); 3239} 3240 3241/** 3242 * ntb_peer_db_set() - Set the doorbell on the secondary/external side 3243 * @ntb: pointer to ntb_softc instance 3244 * @bit: doorbell bits to ring 3245 * 3246 * This function allows triggering of a doorbell on the secondary/external 3247 * side that will initiate an interrupt on the remote host 3248 */ 3249void 3250ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 3251{ 3252 3253 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 3254 struct ntb_pci_bar_info *lapic; 3255 unsigned i; 3256 3257 lapic = ntb->peer_lapic_bar; 3258 3259 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3260 if ((bit & ntb_db_vector_mask(ntb, i)) != 0) 3261 bus_space_write_4(lapic->pci_bus_tag, 3262 lapic->pci_bus_handle, 3263 ntb->peer_msix_data[i].nmd_ofs, 3264 ntb->peer_msix_data[i].nmd_data); 3265 } 3266 return; 3267 } 3268 3269 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3270 ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3271 return; 3272 } 3273 3274 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3275} 3276 3277/* 3278 * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 3279 * as well as the size of the register (via *sz_out). 3280 * 3281 * This function allows a caller using I/OAT DMA to chain the remote doorbell 3282 * ring to its memory window write. 3283 * 3284 * Note that writing the peer doorbell via a memory window will *not* generate 3285 * an interrupt on the remote host; that must be done seperately. 3286 */ 3287bus_addr_t 3288ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 3289{ 3290 struct ntb_pci_bar_info *bar; 3291 uint64_t regoff; 3292 3293 KASSERT(sz_out != NULL, ("must be non-NULL")); 3294 3295 if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3296 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3297 regoff = ntb->peer_reg->db_bell; 3298 } else { 3299 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3300 ("invalid b2b idx")); 3301 3302 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3303 regoff = XEON_PDOORBELL_OFFSET; 3304 } 3305 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3306 3307 *sz_out = ntb->reg->db_size; 3308 /* HACK: Specific to current x86 bus implementation. */ 3309 return ((uint64_t)bar->pci_bus_handle + regoff); 3310} 3311 3312/* 3313 * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 3314 * @ntb: NTB device context 3315 * 3316 * Hardware may support different number or arrangement of doorbell bits. 3317 * 3318 * Return: A mask of doorbell bits supported by the ntb. 3319 */ 3320uint64_t 3321ntb_db_valid_mask(struct ntb_softc *ntb) 3322{ 3323 3324 return (ntb->db_valid_mask); 3325} 3326 3327/* 3328 * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 3329 * @ntb: NTB device context 3330 * @vector: Doorbell vector number 3331 * 3332 * Each interrupt vector may have a different number or arrangement of bits. 3333 * 3334 * Return: A mask of doorbell bits serviced by a vector. 3335 */ 3336uint64_t 3337ntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 3338{ 3339 3340 if (vector > ntb->db_vec_count) 3341 return (0); 3342 return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3343} 3344 3345/** 3346 * ntb_link_is_up() - get the current ntb link state 3347 * @ntb: NTB device context 3348 * @speed: OUT - The link speed expressed as PCIe generation number 3349 * @width: OUT - The link width expressed as the number of PCIe lanes 3350 * 3351 * RETURNS: true or false based on the hardware link state 3352 */ 3353bool 3354ntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 3355 enum ntb_width *width) 3356{ 3357 3358 if (speed != NULL) 3359 *speed = ntb_link_sta_speed(ntb); 3360 if (width != NULL) 3361 *width = ntb_link_sta_width(ntb); 3362 return (link_is_up(ntb)); 3363} 3364 3365static void 3366save_bar_parameters(struct ntb_pci_bar_info *bar) 3367{ 3368 3369 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3370 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3371 bar->pbase = rman_get_start(bar->pci_resource); 3372 bar->size = rman_get_size(bar->pci_resource); 3373 bar->vbase = rman_get_virtual(bar->pci_resource); 3374} 3375 3376device_t 3377ntb_get_device(struct ntb_softc *ntb) 3378{ 3379 3380 return (ntb->device); 3381} 3382 3383/* Export HW-specific errata information. */ 3384bool 3385ntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 3386{ 3387 3388 return (HAS_FEATURE(feature)); 3389} 3390