ntb_hw.c revision 295618
1/*- 2 * Copyright (C) 2013 Intel Corporation 3 * Copyright (C) 2015 EMC Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 295618 2016-02-14 22:37:28Z cem $"); 30 31#include <sys/param.h> 32#include <sys/kernel.h> 33#include <sys/systm.h> 34#include <sys/bus.h> 35#include <sys/endian.h> 36#include <sys/malloc.h> 37#include <sys/module.h> 38#include <sys/mutex.h> 39#include <sys/pciio.h> 40#include <sys/queue.h> 41#include <sys/rman.h> 42#include <sys/sbuf.h> 43#include <sys/sysctl.h> 44#include <vm/vm.h> 45#include <vm/pmap.h> 46#include <machine/bus.h> 47#include <machine/intr_machdep.h> 48#include <machine/pmap.h> 49#include <machine/resource.h> 50#include <dev/pci/pcireg.h> 51#include <dev/pci/pcivar.h> 52 53#include "ntb_regs.h" 54#include "ntb_hw.h" 55 56/* 57 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 58 * allows you to connect two systems using a PCI-e link. 59 * 60 * This module contains the hardware abstraction layer for the NTB. It allows 61 * you to send and recieve interrupts, map the memory windows and send and 62 * receive messages in the scratch-pad registers. 63 * 64 * NOTE: Much of the code in this module is shared with Linux. Any patches may 65 * be picked up and redistributed in Linux with a dual GPL/BSD license. 66 */ 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 72#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 73 74#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev)) 75 76#define NTB_MSIX_VER_GUARD 0xaabbccdd 77#define NTB_MSIX_RECEIVED 0xe0f0e0f0 78#define ONE_MB (1024u * 1024) 79 80/* 81 * PCI constants could be somewhere more generic, but aren't defined/used in 82 * pci.c. 83 */ 84#define PCI_MSIX_ENTRY_SIZE 16 85#define PCI_MSIX_ENTRY_LOWER_ADDR 0 86#define PCI_MSIX_ENTRY_UPPER_ADDR 4 87#define PCI_MSIX_ENTRY_DATA 8 88 89enum ntb_device_type { 90 NTB_XEON, 91 NTB_ATOM 92}; 93 94/* ntb_conn_type are hardware numbers, cannot change. */ 95enum ntb_conn_type { 96 NTB_CONN_TRANSPARENT = 0, 97 NTB_CONN_B2B = 1, 98 NTB_CONN_RP = 2, 99}; 100 101enum ntb_b2b_direction { 102 NTB_DEV_USD = 0, 103 NTB_DEV_DSD = 1, 104}; 105 106enum ntb_bar { 107 NTB_CONFIG_BAR = 0, 108 NTB_B2B_BAR_1, 109 NTB_B2B_BAR_2, 110 NTB_B2B_BAR_3, 111 NTB_MAX_BARS 112}; 113 114enum { 115 NTB_MSIX_GUARD = 0, 116 NTB_MSIX_DATA0, 117 NTB_MSIX_DATA1, 118 NTB_MSIX_DATA2, 119 NTB_MSIX_OFS0, 120 NTB_MSIX_OFS1, 121 NTB_MSIX_OFS2, 122 NTB_MSIX_DONE, 123 NTB_MAX_MSIX_SPAD 124}; 125 126/* Device features and workarounds */ 127#define HAS_FEATURE(feature) \ 128 ((ntb->features & (feature)) != 0) 129 130struct ntb_hw_info { 131 uint32_t device_id; 132 const char *desc; 133 enum ntb_device_type type; 134 uint32_t features; 135}; 136 137struct ntb_pci_bar_info { 138 bus_space_tag_t pci_bus_tag; 139 bus_space_handle_t pci_bus_handle; 140 int pci_resource_id; 141 struct resource *pci_resource; 142 vm_paddr_t pbase; 143 caddr_t vbase; 144 vm_size_t size; 145 vm_memattr_t map_mode; 146 147 /* Configuration register offsets */ 148 uint32_t psz_off; 149 uint32_t ssz_off; 150 uint32_t pbarxlat_off; 151}; 152 153struct ntb_int_info { 154 struct resource *res; 155 int rid; 156 void *tag; 157}; 158 159struct ntb_vec { 160 struct ntb_softc *ntb; 161 uint32_t num; 162 unsigned masked; 163}; 164 165struct ntb_reg { 166 uint32_t ntb_ctl; 167 uint32_t lnk_sta; 168 uint8_t db_size; 169 unsigned mw_bar[NTB_MAX_BARS]; 170}; 171 172struct ntb_alt_reg { 173 uint32_t db_bell; 174 uint32_t db_mask; 175 uint32_t spad; 176}; 177 178struct ntb_xlat_reg { 179 uint32_t bar0_base; 180 uint32_t bar2_base; 181 uint32_t bar4_base; 182 uint32_t bar5_base; 183 184 uint32_t bar2_xlat; 185 uint32_t bar4_xlat; 186 uint32_t bar5_xlat; 187 188 uint32_t bar2_limit; 189 uint32_t bar4_limit; 190 uint32_t bar5_limit; 191}; 192 193struct ntb_b2b_addr { 194 uint64_t bar0_addr; 195 uint64_t bar2_addr64; 196 uint64_t bar4_addr64; 197 uint64_t bar4_addr32; 198 uint64_t bar5_addr32; 199}; 200 201struct ntb_msix_data { 202 uint32_t nmd_ofs; 203 uint32_t nmd_data; 204}; 205 206struct ntb_softc { 207 device_t device; 208 enum ntb_device_type type; 209 uint32_t features; 210 211 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 212 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 213 uint32_t allocated_interrupts; 214 215 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 217 bool peer_msix_good; 218 bool peer_msix_done; 219 struct ntb_pci_bar_info *peer_lapic_bar; 220 struct callout peer_msix_work; 221 222 struct callout heartbeat_timer; 223 struct callout lr_timer; 224 225 void *ntb_ctx; 226 const struct ntb_ctx_ops *ctx_ops; 227 struct ntb_vec *msix_vec; 228#define CTX_LOCK(sc) mtx_lock(&(sc)->ctx_lock) 229#define CTX_UNLOCK(sc) mtx_unlock(&(sc)->ctx_lock) 230#define CTX_ASSERT(sc,f) mtx_assert(&(sc)->ctx_lock, (f)) 231 struct mtx ctx_lock; 232 233 uint32_t ppd; 234 enum ntb_conn_type conn_type; 235 enum ntb_b2b_direction dev_type; 236 237 /* Offset of peer bar0 in B2B BAR */ 238 uint64_t b2b_off; 239 /* Memory window used to access peer bar0 */ 240#define B2B_MW_DISABLED UINT8_MAX 241 uint8_t b2b_mw_idx; 242 uint8_t msix_mw_idx; 243 244 uint8_t mw_count; 245 uint8_t spad_count; 246 uint8_t db_count; 247 uint8_t db_vec_count; 248 uint8_t db_vec_shift; 249 250 /* Protects local db_mask. */ 251#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 252#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 253#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 254 struct mtx db_mask_lock; 255 256 volatile uint32_t ntb_ctl; 257 volatile uint32_t lnk_sta; 258 259 uint64_t db_valid_mask; 260 uint64_t db_link_mask; 261 uint64_t db_mask; 262 263 int last_ts; /* ticks @ last irq */ 264 265 const struct ntb_reg *reg; 266 const struct ntb_alt_reg *self_reg; 267 const struct ntb_alt_reg *peer_reg; 268 const struct ntb_xlat_reg *xlat_reg; 269}; 270 271#ifdef __i386__ 272static __inline uint64_t 273bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 274 bus_size_t offset) 275{ 276 277 return (bus_space_read_4(tag, handle, offset) | 278 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 279} 280 281static __inline void 282bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 283 bus_size_t offset, uint64_t val) 284{ 285 286 bus_space_write_4(tag, handle, offset, val); 287 bus_space_write_4(tag, handle, offset + 4, val >> 32); 288} 289#endif 290 291#define ntb_bar_read(SIZE, bar, offset) \ 292 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 293 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 294#define ntb_bar_write(SIZE, bar, offset, val) \ 295 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 296 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 297#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 298#define ntb_reg_write(SIZE, offset, val) \ 299 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 300#define ntb_mw_read(SIZE, offset) \ 301 ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 302#define ntb_mw_write(SIZE, offset, val) \ 303 ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 304 offset, val) 305 306static int ntb_probe(device_t device); 307static int ntb_attach(device_t device); 308static int ntb_detach(device_t device); 309static unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 310static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 311static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 312static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 313 uint32_t *base, uint32_t *xlat, uint32_t *lmt); 314static int ntb_map_pci_bars(struct ntb_softc *ntb); 315static int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 316 vm_memattr_t); 317static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 318 const char *); 319static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 320static int map_memory_window_bar(struct ntb_softc *ntb, 321 struct ntb_pci_bar_info *bar); 322static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 323static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 324static int ntb_init_isr(struct ntb_softc *ntb); 325static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 326static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 327static void ntb_teardown_interrupts(struct ntb_softc *ntb); 328static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 329static void ntb_interrupt(struct ntb_softc *, uint32_t vec); 330static void ndev_vec_isr(void *arg); 331static void ndev_irq_isr(void *arg); 332static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 333static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 334static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 335static int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 336static void ntb_free_msix_vec(struct ntb_softc *ntb); 337static void ntb_get_msix_info(struct ntb_softc *ntb, uint32_t num_vectors); 338static void ntb_exchange_msix(void *); 339static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 340static void ntb_detect_max_mw(struct ntb_softc *ntb); 341static int ntb_detect_xeon(struct ntb_softc *ntb); 342static int ntb_detect_atom(struct ntb_softc *ntb); 343static int ntb_xeon_init_dev(struct ntb_softc *ntb); 344static int ntb_atom_init_dev(struct ntb_softc *ntb); 345static void ntb_teardown_xeon(struct ntb_softc *ntb); 346static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 347static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 348 enum ntb_bar regbar); 349static void xeon_set_sbar_base_and_limit(struct ntb_softc *, 350 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 351static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 352 enum ntb_bar idx); 353static int xeon_setup_b2b_mw(struct ntb_softc *, 354 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 355static int xeon_setup_msix_bar(struct ntb_softc *); 356static inline bool link_is_up(struct ntb_softc *ntb); 357static inline bool _xeon_link_is_up(struct ntb_softc *ntb); 358static inline bool atom_link_is_err(struct ntb_softc *ntb); 359static inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 360static inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 361static void atom_link_hb(void *arg); 362static void ntb_db_event(struct ntb_softc *ntb, uint32_t vec); 363static void recover_atom_link(void *arg); 364static bool ntb_poll_link(struct ntb_softc *ntb); 365static void save_bar_parameters(struct ntb_pci_bar_info *bar); 366static void ntb_sysctl_init(struct ntb_softc *); 367static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 368static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 369static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 370 371static unsigned g_ntb_hw_debug_level; 372SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 373 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 374#define ntb_printf(lvl, ...) do { \ 375 if ((lvl) <= g_ntb_hw_debug_level) { \ 376 device_printf(ntb->device, __VA_ARGS__); \ 377 } \ 378} while (0) 379 380#define _NTB_PAT_UC 0 381#define _NTB_PAT_WC 1 382#define _NTB_PAT_WT 4 383#define _NTB_PAT_WP 5 384#define _NTB_PAT_WB 6 385#define _NTB_PAT_UCM 7 386static unsigned g_ntb_mw_pat = _NTB_PAT_UC; 387SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 388 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 389 "UC: " __XSTRING(_NTB_PAT_UC) ", " 390 "WC: " __XSTRING(_NTB_PAT_WC) ", " 391 "WT: " __XSTRING(_NTB_PAT_WT) ", " 392 "WP: " __XSTRING(_NTB_PAT_WP) ", " 393 "WB: " __XSTRING(_NTB_PAT_WB) ", " 394 "UC-: " __XSTRING(_NTB_PAT_UCM)); 395 396static inline vm_memattr_t 397ntb_pat_flags(void) 398{ 399 400 switch (g_ntb_mw_pat) { 401 case _NTB_PAT_WC: 402 return (VM_MEMATTR_WRITE_COMBINING); 403 case _NTB_PAT_WT: 404 return (VM_MEMATTR_WRITE_THROUGH); 405 case _NTB_PAT_WP: 406 return (VM_MEMATTR_WRITE_PROTECTED); 407 case _NTB_PAT_WB: 408 return (VM_MEMATTR_WRITE_BACK); 409 case _NTB_PAT_UCM: 410 return (VM_MEMATTR_WEAK_UNCACHEABLE); 411 case _NTB_PAT_UC: 412 /* FALLTHROUGH */ 413 default: 414 return (VM_MEMATTR_UNCACHEABLE); 415 } 416} 417 418/* 419 * Well, this obviously doesn't belong here, but it doesn't seem to exist 420 * anywhere better yet. 421 */ 422static inline const char * 423ntb_vm_memattr_to_str(vm_memattr_t pat) 424{ 425 426 switch (pat) { 427 case VM_MEMATTR_WRITE_COMBINING: 428 return ("WRITE_COMBINING"); 429 case VM_MEMATTR_WRITE_THROUGH: 430 return ("WRITE_THROUGH"); 431 case VM_MEMATTR_WRITE_PROTECTED: 432 return ("WRITE_PROTECTED"); 433 case VM_MEMATTR_WRITE_BACK: 434 return ("WRITE_BACK"); 435 case VM_MEMATTR_WEAK_UNCACHEABLE: 436 return ("UNCACHED"); 437 case VM_MEMATTR_UNCACHEABLE: 438 return ("UNCACHEABLE"); 439 default: 440 return ("UNKNOWN"); 441 } 442} 443 444static int g_ntb_msix_idx = 0; 445SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 446 0, "Use this memory window to access the peer MSIX message complex on " 447 "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 448 "Like b2b_mw_idx, negative values index from the last available memory " 449 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 450 451static int g_ntb_mw_idx = -1; 452SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 453 0, "Use this memory window to access the peer NTB registers. A " 454 "non-negative value starts from the first MW index; a negative value " 455 "starts from the last MW index. The default is -1, i.e., the last " 456 "available memory window. Both sides of the NTB MUST set the same " 457 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 458 459static struct ntb_hw_info pci_ids[] = { 460 /* XXX: PS/SS IDs left out until they are supported. */ 461 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 462 NTB_ATOM, 0 }, 463 464 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 465 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 466 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 467 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 468 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 469 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 470 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 471 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 472 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 473 NTB_SB01BASE_LOCKUP }, 474 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 475 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 476 NTB_SB01BASE_LOCKUP }, 477 478 { 0x00000000, NULL, NTB_ATOM, 0 } 479}; 480 481static const struct ntb_reg atom_reg = { 482 .ntb_ctl = ATOM_NTBCNTL_OFFSET, 483 .lnk_sta = ATOM_LINK_STATUS_OFFSET, 484 .db_size = sizeof(uint64_t), 485 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 486}; 487 488static const struct ntb_alt_reg atom_pri_reg = { 489 .db_bell = ATOM_PDOORBELL_OFFSET, 490 .db_mask = ATOM_PDBMSK_OFFSET, 491 .spad = ATOM_SPAD_OFFSET, 492}; 493 494static const struct ntb_alt_reg atom_b2b_reg = { 495 .db_bell = ATOM_B2B_DOORBELL_OFFSET, 496 .spad = ATOM_B2B_SPAD_OFFSET, 497}; 498 499static const struct ntb_xlat_reg atom_sec_xlat = { 500#if 0 501 /* "FIXME" says the Linux driver. */ 502 .bar0_base = ATOM_SBAR0BASE_OFFSET, 503 .bar2_base = ATOM_SBAR2BASE_OFFSET, 504 .bar4_base = ATOM_SBAR4BASE_OFFSET, 505 506 .bar2_limit = ATOM_SBAR2LMT_OFFSET, 507 .bar4_limit = ATOM_SBAR4LMT_OFFSET, 508#endif 509 510 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 511 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 512}; 513 514static const struct ntb_reg xeon_reg = { 515 .ntb_ctl = XEON_NTBCNTL_OFFSET, 516 .lnk_sta = XEON_LINK_STATUS_OFFSET, 517 .db_size = sizeof(uint16_t), 518 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 519}; 520 521static const struct ntb_alt_reg xeon_pri_reg = { 522 .db_bell = XEON_PDOORBELL_OFFSET, 523 .db_mask = XEON_PDBMSK_OFFSET, 524 .spad = XEON_SPAD_OFFSET, 525}; 526 527static const struct ntb_alt_reg xeon_b2b_reg = { 528 .db_bell = XEON_B2B_DOORBELL_OFFSET, 529 .spad = XEON_B2B_SPAD_OFFSET, 530}; 531 532static const struct ntb_xlat_reg xeon_sec_xlat = { 533 .bar0_base = XEON_SBAR0BASE_OFFSET, 534 .bar2_base = XEON_SBAR2BASE_OFFSET, 535 .bar4_base = XEON_SBAR4BASE_OFFSET, 536 .bar5_base = XEON_SBAR5BASE_OFFSET, 537 538 .bar2_limit = XEON_SBAR2LMT_OFFSET, 539 .bar4_limit = XEON_SBAR4LMT_OFFSET, 540 .bar5_limit = XEON_SBAR5LMT_OFFSET, 541 542 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 543 .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 544 .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 545}; 546 547static struct ntb_b2b_addr xeon_b2b_usd_addr = { 548 .bar0_addr = XEON_B2B_BAR0_ADDR, 549 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 550 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 551 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 552 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 553}; 554 555static struct ntb_b2b_addr xeon_b2b_dsd_addr = { 556 .bar0_addr = XEON_B2B_BAR0_ADDR, 557 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 558 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 559 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 560 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 561}; 562 563SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 564 "B2B MW segment overrides -- MUST be the same on both sides"); 565 566SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 567 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 568 "hardware, use this 64-bit address on the bus between the NTB devices for " 569 "the window at BAR2, on the upstream side of the link. MUST be the same " 570 "address on both sides."); 571SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 572 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 573SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 574 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 575 "(split-BAR mode)."); 576SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 577 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 578 "(split-BAR mode)."); 579 580SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 581 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 582 "hardware, use this 64-bit address on the bus between the NTB devices for " 583 "the window at BAR2, on the downstream side of the link. MUST be the same" 584 " address on both sides."); 585SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 586 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 587SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 588 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 589 "(split-BAR mode)."); 590SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 591 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 592 "(split-BAR mode)."); 593 594/* 595 * OS <-> Driver interface structures 596 */ 597MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 598 599static device_method_t ntb_pci_methods[] = { 600 /* Device interface */ 601 DEVMETHOD(device_probe, ntb_probe), 602 DEVMETHOD(device_attach, ntb_attach), 603 DEVMETHOD(device_detach, ntb_detach), 604 DEVMETHOD_END 605}; 606 607static driver_t ntb_pci_driver = { 608 "ntb_hw", 609 ntb_pci_methods, 610 sizeof(struct ntb_softc), 611}; 612 613static devclass_t ntb_devclass; 614DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL); 615MODULE_VERSION(ntb_hw, 1); 616 617SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 618 619/* 620 * OS <-> Driver linkage functions 621 */ 622static int 623ntb_probe(device_t device) 624{ 625 struct ntb_hw_info *p; 626 627 p = ntb_get_device_info(pci_get_devid(device)); 628 if (p == NULL) 629 return (ENXIO); 630 631 device_set_desc(device, p->desc); 632 return (0); 633} 634 635static int 636ntb_attach(device_t device) 637{ 638 struct ntb_softc *ntb; 639 struct ntb_hw_info *p; 640 int error; 641 642 ntb = DEVICE2SOFTC(device); 643 p = ntb_get_device_info(pci_get_devid(device)); 644 645 ntb->device = device; 646 ntb->type = p->type; 647 ntb->features = p->features; 648 ntb->b2b_mw_idx = B2B_MW_DISABLED; 649 ntb->msix_mw_idx = B2B_MW_DISABLED; 650 651 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 652 callout_init(&ntb->heartbeat_timer, 1); 653 callout_init(&ntb->lr_timer, 1); 654 callout_init(&ntb->peer_msix_work, 1); 655 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 656 mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF); 657 658 if (ntb->type == NTB_ATOM) 659 error = ntb_detect_atom(ntb); 660 else 661 error = ntb_detect_xeon(ntb); 662 if (error != 0) 663 goto out; 664 665 ntb_detect_max_mw(ntb); 666 667 pci_enable_busmaster(ntb->device); 668 669 error = ntb_map_pci_bars(ntb); 670 if (error != 0) 671 goto out; 672 if (ntb->type == NTB_ATOM) 673 error = ntb_atom_init_dev(ntb); 674 else 675 error = ntb_xeon_init_dev(ntb); 676 if (error != 0) 677 goto out; 678 679 ntb_spad_clear(ntb); 680 681 ntb_poll_link(ntb); 682 683 ntb_sysctl_init(ntb); 684 685out: 686 if (error != 0) 687 ntb_detach(device); 688 return (error); 689} 690 691static int 692ntb_detach(device_t device) 693{ 694 struct ntb_softc *ntb; 695 696 ntb = DEVICE2SOFTC(device); 697 698 if (ntb->self_reg != NULL) { 699 DB_MASK_LOCK(ntb); 700 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 701 DB_MASK_UNLOCK(ntb); 702 } 703 callout_drain(&ntb->heartbeat_timer); 704 callout_drain(&ntb->lr_timer); 705 callout_drain(&ntb->peer_msix_work); 706 pci_disable_busmaster(ntb->device); 707 if (ntb->type == NTB_XEON) 708 ntb_teardown_xeon(ntb); 709 ntb_teardown_interrupts(ntb); 710 711 mtx_destroy(&ntb->db_mask_lock); 712 mtx_destroy(&ntb->ctx_lock); 713 714 ntb_unmap_pci_bar(ntb); 715 716 return (0); 717} 718 719/* 720 * Driver internal routines 721 */ 722static inline enum ntb_bar 723ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 724{ 725 726 KASSERT(mw < ntb->mw_count, 727 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 728 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 729 730 return (ntb->reg->mw_bar[mw]); 731} 732 733static inline bool 734bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 735{ 736 /* XXX This assertion could be stronger. */ 737 KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 738 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR)); 739} 740 741static inline void 742bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 743 uint32_t *xlat, uint32_t *lmt) 744{ 745 uint32_t basev, lmtv, xlatv; 746 747 switch (bar) { 748 case NTB_B2B_BAR_1: 749 basev = ntb->xlat_reg->bar2_base; 750 lmtv = ntb->xlat_reg->bar2_limit; 751 xlatv = ntb->xlat_reg->bar2_xlat; 752 break; 753 case NTB_B2B_BAR_2: 754 basev = ntb->xlat_reg->bar4_base; 755 lmtv = ntb->xlat_reg->bar4_limit; 756 xlatv = ntb->xlat_reg->bar4_xlat; 757 break; 758 case NTB_B2B_BAR_3: 759 basev = ntb->xlat_reg->bar5_base; 760 lmtv = ntb->xlat_reg->bar5_limit; 761 xlatv = ntb->xlat_reg->bar5_xlat; 762 break; 763 default: 764 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 765 ("bad bar")); 766 basev = lmtv = xlatv = 0; 767 break; 768 } 769 770 if (base != NULL) 771 *base = basev; 772 if (xlat != NULL) 773 *xlat = xlatv; 774 if (lmt != NULL) 775 *lmt = lmtv; 776} 777 778static int 779ntb_map_pci_bars(struct ntb_softc *ntb) 780{ 781 int rc; 782 783 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 784 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 785 if (rc != 0) 786 goto out; 787 788 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 789 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 790 if (rc != 0) 791 goto out; 792 ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 793 ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 794 ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 795 796 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 797 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 798 if (rc != 0) 799 goto out; 800 ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 801 ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 802 ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 803 804 if (!HAS_FEATURE(NTB_SPLIT_BAR)) 805 goto out; 806 807 ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 808 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 809 ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 810 ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 811 ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 812 813out: 814 if (rc != 0) 815 device_printf(ntb->device, 816 "unable to allocate pci resource\n"); 817 return (rc); 818} 819 820static void 821print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 822 const char *kind) 823{ 824 825 device_printf(ntb->device, 826 "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 827 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 828 (char *)bar->vbase + bar->size - 1, 829 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 830 (uintmax_t)bar->size, kind); 831} 832 833static int 834map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 835{ 836 837 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 838 &bar->pci_resource_id, RF_ACTIVE); 839 if (bar->pci_resource == NULL) 840 return (ENXIO); 841 842 save_bar_parameters(bar); 843 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 844 print_map_success(ntb, bar, "mmr"); 845 return (0); 846} 847 848static int 849map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 850{ 851 int rc; 852 vm_memattr_t mapmode; 853 uint8_t bar_size_bits = 0; 854 855 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 856 &bar->pci_resource_id, RF_ACTIVE); 857 858 if (bar->pci_resource == NULL) 859 return (ENXIO); 860 861 save_bar_parameters(bar); 862 /* 863 * Ivytown NTB BAR sizes are misreported by the hardware due to a 864 * hardware issue. To work around this, query the size it should be 865 * configured to by the device and modify the resource to correspond to 866 * this new size. The BIOS on systems with this problem is required to 867 * provide enough address space to allow the driver to make this change 868 * safely. 869 * 870 * Ideally I could have just specified the size when I allocated the 871 * resource like: 872 * bus_alloc_resource(ntb->device, 873 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 874 * 1ul << bar_size_bits, RF_ACTIVE); 875 * but the PCI driver does not honor the size in this call, so we have 876 * to modify it after the fact. 877 */ 878 if (HAS_FEATURE(NTB_BAR_SIZE_4K)) { 879 if (bar->pci_resource_id == PCIR_BAR(2)) 880 bar_size_bits = pci_read_config(ntb->device, 881 XEON_PBAR23SZ_OFFSET, 1); 882 else 883 bar_size_bits = pci_read_config(ntb->device, 884 XEON_PBAR45SZ_OFFSET, 1); 885 886 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 887 bar->pci_resource, bar->pbase, 888 bar->pbase + (1ul << bar_size_bits) - 1); 889 if (rc != 0) { 890 device_printf(ntb->device, 891 "unable to resize bar\n"); 892 return (rc); 893 } 894 895 save_bar_parameters(bar); 896 } 897 898 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 899 print_map_success(ntb, bar, "mw"); 900 901 /* 902 * Optionally, mark MW BARs as anything other than UC to improve 903 * performance. 904 */ 905 mapmode = ntb_pat_flags(); 906 if (mapmode == bar->map_mode) 907 return (0); 908 909 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 910 if (rc == 0) { 911 bar->map_mode = mapmode; 912 device_printf(ntb->device, 913 "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 914 "%s.\n", 915 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 916 (char *)bar->vbase + bar->size - 1, 917 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 918 ntb_vm_memattr_to_str(mapmode)); 919 } else 920 device_printf(ntb->device, 921 "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 922 "%s: %d\n", 923 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 924 (char *)bar->vbase + bar->size - 1, 925 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 926 ntb_vm_memattr_to_str(mapmode), rc); 927 /* Proceed anyway */ 928 return (0); 929} 930 931static void 932ntb_unmap_pci_bar(struct ntb_softc *ntb) 933{ 934 struct ntb_pci_bar_info *current_bar; 935 int i; 936 937 for (i = 0; i < NTB_MAX_BARS; i++) { 938 current_bar = &ntb->bar_info[i]; 939 if (current_bar->pci_resource != NULL) 940 bus_release_resource(ntb->device, SYS_RES_MEMORY, 941 current_bar->pci_resource_id, 942 current_bar->pci_resource); 943 } 944} 945 946static int 947ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 948{ 949 uint32_t i; 950 int rc; 951 952 for (i = 0; i < num_vectors; i++) { 953 ntb->int_info[i].rid = i + 1; 954 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 955 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 956 if (ntb->int_info[i].res == NULL) { 957 device_printf(ntb->device, 958 "bus_alloc_resource failed\n"); 959 return (ENOMEM); 960 } 961 ntb->int_info[i].tag = NULL; 962 ntb->allocated_interrupts++; 963 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 964 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 965 &ntb->msix_vec[i], &ntb->int_info[i].tag); 966 if (rc != 0) { 967 device_printf(ntb->device, "bus_setup_intr failed\n"); 968 return (ENXIO); 969 } 970 } 971 return (0); 972} 973 974/* 975 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 976 * cannot be allocated for each MSI-X message. JHB seems to think remapping 977 * should be okay. This tunable should enable us to test that hypothesis 978 * when someone gets their hands on some Xeon hardware. 979 */ 980static int ntb_force_remap_mode; 981SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 982 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 983 " to a smaller number of ithreads, even if the desired number are " 984 "available"); 985 986/* 987 * In case it is NOT ok, give consumers an abort button. 988 */ 989static int ntb_prefer_intx; 990SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 991 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 992 "than remapping MSI-X messages over available slots (match Linux driver " 993 "behavior)"); 994 995/* 996 * Remap the desired number of MSI-X messages to available ithreads in a simple 997 * round-robin fashion. 998 */ 999static int 1000ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1001{ 1002 u_int *vectors; 1003 uint32_t i; 1004 int rc; 1005 1006 if (ntb_prefer_intx != 0) 1007 return (ENXIO); 1008 1009 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1010 1011 for (i = 0; i < desired; i++) 1012 vectors[i] = (i % avail) + 1; 1013 1014 rc = pci_remap_msix(dev, desired, vectors); 1015 free(vectors, M_NTB); 1016 return (rc); 1017} 1018 1019static int 1020ntb_init_isr(struct ntb_softc *ntb) 1021{ 1022 uint32_t desired_vectors, num_vectors; 1023 int rc; 1024 1025 ntb->allocated_interrupts = 0; 1026 ntb->last_ts = ticks; 1027 1028 /* 1029 * Mask all doorbell interrupts. (Except link events!) 1030 */ 1031 DB_MASK_LOCK(ntb); 1032 ntb->db_mask = ntb->db_valid_mask; 1033 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1034 DB_MASK_UNLOCK(ntb); 1035 1036 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1037 ntb->db_count); 1038 if (desired_vectors >= 1) { 1039 rc = pci_alloc_msix(ntb->device, &num_vectors); 1040 1041 if (ntb_force_remap_mode != 0 && rc == 0 && 1042 num_vectors == desired_vectors) 1043 num_vectors--; 1044 1045 if (rc == 0 && num_vectors < desired_vectors) { 1046 rc = ntb_remap_msix(ntb->device, desired_vectors, 1047 num_vectors); 1048 if (rc == 0) 1049 num_vectors = desired_vectors; 1050 else 1051 pci_release_msi(ntb->device); 1052 } 1053 if (rc != 0) 1054 num_vectors = 1; 1055 } else 1056 num_vectors = 1; 1057 1058 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1059 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1060 device_printf(ntb->device, 1061 "Errata workaround does not support MSI or INTX\n"); 1062 return (EINVAL); 1063 } 1064 1065 ntb->db_vec_count = 1; 1066 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1067 rc = ntb_setup_legacy_interrupt(ntb); 1068 } else { 1069 ntb_create_msix_vec(ntb, num_vectors); 1070 rc = ntb_setup_msix(ntb, num_vectors); 1071 if (rc == 0 && HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1072 ntb_get_msix_info(ntb, num_vectors); 1073 } 1074 if (rc != 0) { 1075 device_printf(ntb->device, 1076 "Error allocating interrupts: %d\n", rc); 1077 ntb_free_msix_vec(ntb); 1078 } 1079 1080 return (rc); 1081} 1082 1083static int 1084ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1085{ 1086 int rc; 1087 1088 ntb->int_info[0].rid = 0; 1089 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1090 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1091 if (ntb->int_info[0].res == NULL) { 1092 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1093 return (ENOMEM); 1094 } 1095 1096 ntb->int_info[0].tag = NULL; 1097 ntb->allocated_interrupts = 1; 1098 1099 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1100 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1101 ntb, &ntb->int_info[0].tag); 1102 if (rc != 0) { 1103 device_printf(ntb->device, "bus_setup_intr failed\n"); 1104 return (ENXIO); 1105 } 1106 1107 return (0); 1108} 1109 1110static void 1111ntb_teardown_interrupts(struct ntb_softc *ntb) 1112{ 1113 struct ntb_int_info *current_int; 1114 int i; 1115 1116 for (i = 0; i < ntb->allocated_interrupts; i++) { 1117 current_int = &ntb->int_info[i]; 1118 if (current_int->tag != NULL) 1119 bus_teardown_intr(ntb->device, current_int->res, 1120 current_int->tag); 1121 1122 if (current_int->res != NULL) 1123 bus_release_resource(ntb->device, SYS_RES_IRQ, 1124 rman_get_rid(current_int->res), current_int->res); 1125 } 1126 1127 ntb_free_msix_vec(ntb); 1128 pci_release_msi(ntb->device); 1129} 1130 1131/* 1132 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1133 * out to make code clearer. 1134 */ 1135static inline uint64_t 1136db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1137{ 1138 1139 if (ntb->type == NTB_ATOM) 1140 return (ntb_reg_read(8, regoff)); 1141 1142 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1143 1144 return (ntb_reg_read(2, regoff)); 1145} 1146 1147static inline void 1148db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1149{ 1150 1151 KASSERT((val & ~ntb->db_valid_mask) == 0, 1152 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1153 (uintmax_t)(val & ~ntb->db_valid_mask), 1154 (uintmax_t)ntb->db_valid_mask)); 1155 1156 if (regoff == ntb->self_reg->db_mask) 1157 DB_MASK_ASSERT(ntb, MA_OWNED); 1158 db_iowrite_raw(ntb, regoff, val); 1159} 1160 1161static inline void 1162db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1163{ 1164 1165 if (ntb->type == NTB_ATOM) { 1166 ntb_reg_write(8, regoff, val); 1167 return; 1168 } 1169 1170 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1171 ntb_reg_write(2, regoff, (uint16_t)val); 1172} 1173 1174void 1175ntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) 1176{ 1177 1178 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1179 return; 1180 1181 DB_MASK_LOCK(ntb); 1182 ntb->db_mask |= bits; 1183 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1184 DB_MASK_UNLOCK(ntb); 1185} 1186 1187void 1188ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) 1189{ 1190 1191 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1192 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1193 (uintmax_t)(bits & ~ntb->db_valid_mask), 1194 (uintmax_t)ntb->db_valid_mask)); 1195 1196 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1197 return; 1198 1199 DB_MASK_LOCK(ntb); 1200 ntb->db_mask &= ~bits; 1201 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1202 DB_MASK_UNLOCK(ntb); 1203} 1204 1205uint64_t 1206ntb_db_read(struct ntb_softc *ntb) 1207{ 1208 1209 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1210 uint64_t res; 1211 unsigned i; 1212 1213 res = 0; 1214 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1215 if (ntb->msix_vec[i].masked != 0) 1216 res |= ntb_db_vector_mask(ntb, i); 1217 } 1218 return (res); 1219 } 1220 1221 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1222} 1223 1224void 1225ntb_db_clear(struct ntb_softc *ntb, uint64_t bits) 1226{ 1227 1228 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1229 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1230 (uintmax_t)(bits & ~ntb->db_valid_mask), 1231 (uintmax_t)ntb->db_valid_mask)); 1232 1233 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1234 unsigned i; 1235 1236 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1237 if ((bits & ntb_db_vector_mask(ntb, i)) != 0) { 1238 DB_MASK_LOCK(ntb); 1239 if (ntb->msix_vec[i].masked != 0) { 1240 /* XXX These need a public API. */ 1241#if 0 1242 pci_unmask_msix(ntb->device, i); 1243#endif 1244 ntb->msix_vec[i].masked = 0; 1245 } 1246 DB_MASK_UNLOCK(ntb); 1247 } 1248 } 1249 return; 1250 } 1251 1252 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1253} 1254 1255static inline uint64_t 1256ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1257{ 1258 uint64_t shift, mask; 1259 1260 shift = ntb->db_vec_shift; 1261 mask = (1ull << shift) - 1; 1262 return (mask << (shift * db_vector)); 1263} 1264 1265static void 1266ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1267{ 1268 uint64_t vec_mask; 1269 1270 ntb->last_ts = ticks; 1271 vec_mask = ntb_vec_mask(ntb, vec); 1272 1273 if ((vec_mask & ntb->db_link_mask) != 0) { 1274 if (ntb_poll_link(ntb)) 1275 ntb_link_event(ntb); 1276 } 1277 1278 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1279 (vec_mask & ntb->db_link_mask) == 0) { 1280 DB_MASK_LOCK(ntb); 1281 if (ntb->msix_vec[vec].masked == 0) { 1282 /* XXX These need a public API. */ 1283#if 0 1284 pci_mask_msix(ntb->device, vec); 1285#endif 1286 ntb->msix_vec[vec].masked = 1; 1287 } 1288 DB_MASK_UNLOCK(ntb); 1289 } 1290 1291 if ((vec_mask & ntb->db_valid_mask) != 0) 1292 ntb_db_event(ntb, vec); 1293} 1294 1295static void 1296ndev_vec_isr(void *arg) 1297{ 1298 struct ntb_vec *nvec = arg; 1299 1300 ntb_interrupt(nvec->ntb, nvec->num); 1301} 1302 1303static void 1304ndev_irq_isr(void *arg) 1305{ 1306 /* If we couldn't set up MSI-X, we only have the one vector. */ 1307 ntb_interrupt(arg, 0); 1308} 1309 1310static int 1311ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1312{ 1313 uint32_t i; 1314 1315 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1316 M_ZERO | M_WAITOK); 1317 for (i = 0; i < num_vectors; i++) { 1318 ntb->msix_vec[i].num = i; 1319 ntb->msix_vec[i].ntb = ntb; 1320 } 1321 1322 return (0); 1323} 1324 1325static void 1326ntb_free_msix_vec(struct ntb_softc *ntb) 1327{ 1328 1329 if (ntb->msix_vec == NULL) 1330 return; 1331 1332 free(ntb->msix_vec, M_NTB); 1333 ntb->msix_vec = NULL; 1334} 1335 1336static void 1337ntb_get_msix_info(struct ntb_softc *ntb, uint32_t num_vectors) 1338{ 1339 struct pci_devinfo *dinfo; 1340 struct pcicfg_msix *msix; 1341 uint32_t laddr, data, i, offset; 1342 1343 dinfo = device_get_ivars(ntb->device); 1344 msix = &dinfo->cfg.msix; 1345 1346 laddr = data = 0; 1347 1348 for (i = 0; i < num_vectors; i++) { 1349 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1350 1351 laddr = bus_read_4(msix->msix_table_res, offset + 1352 PCI_MSIX_ENTRY_LOWER_ADDR); 1353 ntb_printf(2, "local lower MSIX addr(%u): 0x%x\n", i, laddr); 1354 1355 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1356 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1357 MSI_INTEL_ADDR_BASE)); 1358 ntb->msix_data[i].nmd_ofs = laddr & ~MSI_INTEL_ADDR_BASE; 1359 1360 data = bus_read_4(msix->msix_table_res, offset + 1361 PCI_MSIX_ENTRY_DATA); 1362 ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1363 1364 ntb->msix_data[i].nmd_data = data; 1365 } 1366} 1367 1368static struct ntb_hw_info * 1369ntb_get_device_info(uint32_t device_id) 1370{ 1371 struct ntb_hw_info *ep = pci_ids; 1372 1373 while (ep->device_id) { 1374 if (ep->device_id == device_id) 1375 return (ep); 1376 ++ep; 1377 } 1378 return (NULL); 1379} 1380 1381static void 1382ntb_teardown_xeon(struct ntb_softc *ntb) 1383{ 1384 1385 if (ntb->reg != NULL) 1386 ntb_link_disable(ntb); 1387} 1388 1389static void 1390ntb_detect_max_mw(struct ntb_softc *ntb) 1391{ 1392 1393 if (ntb->type == NTB_ATOM) { 1394 ntb->mw_count = ATOM_MW_COUNT; 1395 return; 1396 } 1397 1398 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1399 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1400 else 1401 ntb->mw_count = XEON_SNB_MW_COUNT; 1402} 1403 1404static int 1405ntb_detect_xeon(struct ntb_softc *ntb) 1406{ 1407 uint8_t ppd, conn_type; 1408 1409 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1410 ntb->ppd = ppd; 1411 1412 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1413 ntb->dev_type = NTB_DEV_DSD; 1414 else 1415 ntb->dev_type = NTB_DEV_USD; 1416 1417 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1418 ntb->features |= NTB_SPLIT_BAR; 1419 1420 /* 1421 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1422 * errata workaround; only do one at a time. 1423 */ 1424 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1425 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1426 1427 conn_type = ppd & XEON_PPD_CONN_TYPE; 1428 switch (conn_type) { 1429 case NTB_CONN_B2B: 1430 ntb->conn_type = conn_type; 1431 break; 1432 case NTB_CONN_RP: 1433 case NTB_CONN_TRANSPARENT: 1434 default: 1435 device_printf(ntb->device, "Unsupported connection type: %u\n", 1436 (unsigned)conn_type); 1437 return (ENXIO); 1438 } 1439 return (0); 1440} 1441 1442static int 1443ntb_detect_atom(struct ntb_softc *ntb) 1444{ 1445 uint32_t ppd, conn_type; 1446 1447 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1448 ntb->ppd = ppd; 1449 1450 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1451 ntb->dev_type = NTB_DEV_DSD; 1452 else 1453 ntb->dev_type = NTB_DEV_USD; 1454 1455 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1456 switch (conn_type) { 1457 case NTB_CONN_B2B: 1458 ntb->conn_type = conn_type; 1459 break; 1460 default: 1461 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1462 return (ENXIO); 1463 } 1464 return (0); 1465} 1466 1467static int 1468ntb_xeon_init_dev(struct ntb_softc *ntb) 1469{ 1470 int rc; 1471 1472 ntb->spad_count = XEON_SPAD_COUNT; 1473 ntb->db_count = XEON_DB_COUNT; 1474 ntb->db_link_mask = XEON_DB_LINK_BIT; 1475 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1476 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1477 1478 if (ntb->conn_type != NTB_CONN_B2B) { 1479 device_printf(ntb->device, "Connection type %d not supported\n", 1480 ntb->conn_type); 1481 return (ENXIO); 1482 } 1483 1484 ntb->reg = &xeon_reg; 1485 ntb->self_reg = &xeon_pri_reg; 1486 ntb->peer_reg = &xeon_b2b_reg; 1487 ntb->xlat_reg = &xeon_sec_xlat; 1488 1489 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1490 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1491 ntb->mw_count; 1492 ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1493 g_ntb_msix_idx, ntb->msix_mw_idx); 1494 rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1495 VM_MEMATTR_UNCACHEABLE); 1496 KASSERT(rc == 0, ("shouldn't fail")); 1497 } else if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 1498 /* 1499 * There is a Xeon hardware errata related to writes to SDOORBELL or 1500 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1501 * which may hang the system. To workaround this, use a memory 1502 * window to access the interrupt and scratch pad registers on the 1503 * remote system. 1504 */ 1505 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1506 ntb->mw_count; 1507 ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1508 g_ntb_mw_idx, ntb->b2b_mw_idx); 1509 rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1510 VM_MEMATTR_UNCACHEABLE); 1511 KASSERT(rc == 0, ("shouldn't fail")); 1512 } else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) 1513 /* 1514 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1515 * mirrored to the remote system. Shrink the number of bits by one, 1516 * since bit 14 is the last bit. 1517 * 1518 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1519 * anyway. Nor for non-B2B connection types. 1520 */ 1521 ntb->db_count = XEON_DB_COUNT - 1; 1522 1523 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1524 1525 if (ntb->dev_type == NTB_DEV_USD) 1526 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1527 &xeon_b2b_usd_addr); 1528 else 1529 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1530 &xeon_b2b_dsd_addr); 1531 if (rc != 0) 1532 return (rc); 1533 1534 /* Enable Bus Master and Memory Space on the secondary side */ 1535 ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1536 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1537 1538 /* 1539 * Mask all doorbell interrupts. 1540 */ 1541 DB_MASK_LOCK(ntb); 1542 ntb->db_mask = ntb->db_valid_mask; 1543 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1544 DB_MASK_UNLOCK(ntb); 1545 1546 rc = xeon_setup_msix_bar(ntb); 1547 if (rc != 0) 1548 return (rc); 1549 1550 rc = ntb_init_isr(ntb); 1551 return (rc); 1552} 1553 1554static int 1555ntb_atom_init_dev(struct ntb_softc *ntb) 1556{ 1557 int error; 1558 1559 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1560 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1561 1562 ntb->spad_count = ATOM_SPAD_COUNT; 1563 ntb->db_count = ATOM_DB_COUNT; 1564 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1565 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1566 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1567 1568 ntb->reg = &atom_reg; 1569 ntb->self_reg = &atom_pri_reg; 1570 ntb->peer_reg = &atom_b2b_reg; 1571 ntb->xlat_reg = &atom_sec_xlat; 1572 1573 /* 1574 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1575 * resolved. Mask transaction layer internal parity errors. 1576 */ 1577 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1578 1579 configure_atom_secondary_side_bars(ntb); 1580 1581 /* Enable Bus Master and Memory Space on the secondary side */ 1582 ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1583 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1584 1585 error = ntb_init_isr(ntb); 1586 if (error != 0) 1587 return (error); 1588 1589 /* Initiate PCI-E link training */ 1590 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1591 1592 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1593 1594 return (0); 1595} 1596 1597/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1598static void 1599configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1600{ 1601 1602 if (ntb->dev_type == NTB_DEV_USD) { 1603 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1604 XEON_B2B_BAR2_ADDR64); 1605 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1606 XEON_B2B_BAR4_ADDR64); 1607 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1608 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1609 } else { 1610 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1611 XEON_B2B_BAR2_ADDR64); 1612 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1613 XEON_B2B_BAR4_ADDR64); 1614 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1615 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1616 } 1617} 1618 1619 1620/* 1621 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1622 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1623 * remains for use by a higher layer. 1624 * 1625 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1626 * MW size is sufficiently large. 1627 */ 1628static unsigned int ntb_b2b_mw_share; 1629SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1630 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1631 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1632 "value here."); 1633 1634static void 1635xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1636 enum ntb_bar regbar) 1637{ 1638 struct ntb_pci_bar_info *bar; 1639 uint8_t bar_sz; 1640 1641 if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1642 return; 1643 1644 bar = &ntb->bar_info[idx]; 1645 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1646 if (idx == regbar) { 1647 if (ntb->b2b_off != 0) 1648 bar_sz--; 1649 else 1650 bar_sz = 0; 1651 } else if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1652 ntb_mw_to_bar(ntb, ntb->msix_mw_idx) == idx) { 1653 /* Restrict LAPIC BAR to 1MB */ 1654 pci_write_config(ntb->device, bar->psz_off, 20, 1); 1655 pci_write_config(ntb->device, bar->ssz_off, 20, 1); 1656 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1657 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1658 (void)bar_sz; 1659 return; 1660 } 1661 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1662 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1663 (void)bar_sz; 1664} 1665 1666static void 1667xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1668 enum ntb_bar idx, enum ntb_bar regbar) 1669{ 1670 uint64_t reg_val, lmt_addr; 1671 uint32_t base_reg, lmt_reg; 1672 1673 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1674 if (idx == regbar) 1675 bar_addr += ntb->b2b_off; 1676 lmt_addr = bar_addr; 1677 1678 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && 1679 ntb_mw_to_bar(ntb, ntb->msix_mw_idx) == idx) 1680 lmt_addr += ONE_MB; 1681 1682 /* 1683 * Set limit registers first to avoid an errata where setting the base 1684 * registers locks the limit registers. 1685 */ 1686 if (!bar_is_64bit(ntb, idx)) { 1687 ntb_reg_write(4, lmt_reg, lmt_addr); 1688 reg_val = ntb_reg_read(4, lmt_reg); 1689 (void)reg_val; 1690 1691 ntb_reg_write(4, base_reg, bar_addr); 1692 reg_val = ntb_reg_read(4, base_reg); 1693 (void)reg_val; 1694 } else { 1695 ntb_reg_write(8, lmt_reg, lmt_addr); 1696 reg_val = ntb_reg_read(8, lmt_reg); 1697 (void)reg_val; 1698 1699 ntb_reg_write(8, base_reg, bar_addr); 1700 reg_val = ntb_reg_read(8, base_reg); 1701 (void)reg_val; 1702 } 1703} 1704 1705static void 1706xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1707{ 1708 struct ntb_pci_bar_info *bar; 1709 1710 bar = &ntb->bar_info[idx]; 1711 if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1712 ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1713 base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1714 } else { 1715 ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1716 base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1717 } 1718 (void)base_addr; 1719} 1720 1721static int 1722xeon_setup_msix_bar(struct ntb_softc *ntb) 1723{ 1724 struct ntb_pci_bar_info *lapic_bar; 1725 enum ntb_bar bar_num; 1726 int rc; 1727 1728 if (!HAS_FEATURE(NTB_SB01BASE_LOCKUP)) 1729 return (0); 1730 1731 bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1732 lapic_bar = &ntb->bar_info[bar_num]; 1733 1734 /* Restrict LAPIC BAR to 1MB */ 1735 if (lapic_bar->size > ONE_MB) { 1736 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 1737 lapic_bar->pci_resource, lapic_bar->pbase, 1738 lapic_bar->pbase + ONE_MB - 1); 1739 if (rc == 0) 1740 lapic_bar->size = ONE_MB; 1741 else { 1742 ntb_printf(0, "Failed to shrink LAPIC BAR resource to " 1743 "1 MB: %d\n", rc); 1744 /* Ignore error */ 1745 } 1746 } 1747 1748 ntb->peer_lapic_bar = lapic_bar; 1749 return (0); 1750} 1751 1752static int 1753xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1754 const struct ntb_b2b_addr *peer_addr) 1755{ 1756 struct ntb_pci_bar_info *b2b_bar; 1757 vm_size_t bar_size; 1758 uint64_t bar_addr; 1759 enum ntb_bar b2b_bar_num, i; 1760 1761 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1762 b2b_bar = NULL; 1763 b2b_bar_num = NTB_CONFIG_BAR; 1764 ntb->b2b_off = 0; 1765 } else { 1766 b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1767 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1768 ("invalid b2b mw bar")); 1769 1770 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1771 bar_size = b2b_bar->size; 1772 1773 if (ntb_b2b_mw_share != 0 && 1774 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1775 ntb->b2b_off = bar_size >> 1; 1776 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1777 ntb->b2b_off = 0; 1778 } else { 1779 device_printf(ntb->device, 1780 "B2B bar size is too small!\n"); 1781 return (EIO); 1782 } 1783 } 1784 1785 /* 1786 * Reset the secondary bar sizes to match the primary bar sizes. 1787 * (Except, disable or halve the size of the B2B secondary bar.) 1788 */ 1789 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1790 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1791 1792 bar_addr = 0; 1793 if (b2b_bar_num == NTB_CONFIG_BAR) 1794 bar_addr = addr->bar0_addr; 1795 else if (b2b_bar_num == NTB_B2B_BAR_1) 1796 bar_addr = addr->bar2_addr64; 1797 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1798 bar_addr = addr->bar4_addr64; 1799 else if (b2b_bar_num == NTB_B2B_BAR_2) 1800 bar_addr = addr->bar4_addr32; 1801 else if (b2b_bar_num == NTB_B2B_BAR_3) 1802 bar_addr = addr->bar5_addr32; 1803 else 1804 KASSERT(false, ("invalid bar")); 1805 1806 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1807 1808 /* 1809 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1810 * register BAR. The B2B BAR is either disabled above or configured 1811 * half-size. It starts at PBAR xlat + offset. 1812 * 1813 * Also set up incoming BAR limits == base (zero length window). 1814 */ 1815 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1816 b2b_bar_num); 1817 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1818 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1819 NTB_B2B_BAR_2, b2b_bar_num); 1820 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1821 NTB_B2B_BAR_3, b2b_bar_num); 1822 } else 1823 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1824 NTB_B2B_BAR_2, b2b_bar_num); 1825 1826 /* Zero incoming translation addrs */ 1827 ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1828 ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1829 1830 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 1831 size_t size, xlatoffset; 1832 1833 switch (ntb_mw_to_bar(ntb, ntb->msix_mw_idx)) { 1834 case NTB_B2B_BAR_1: 1835 size = 8; 1836 xlatoffset = XEON_SBAR2XLAT_OFFSET; 1837 break; 1838 case NTB_B2B_BAR_2: 1839 xlatoffset = XEON_SBAR4XLAT_OFFSET; 1840 if (HAS_FEATURE(NTB_SPLIT_BAR)) 1841 size = 4; 1842 else 1843 size = 8; 1844 break; 1845 case NTB_B2B_BAR_3: 1846 xlatoffset = XEON_SBAR5XLAT_OFFSET; 1847 size = 4; 1848 break; 1849 default: 1850 KASSERT(false, ("Bogus msix mw idx: %u", 1851 ntb->msix_mw_idx)); 1852 return (EINVAL); 1853 } 1854 1855 /* 1856 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1857 * workaround 1858 */ 1859 if (size == 4) 1860 ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1861 else 1862 ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1863 } 1864 (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1865 (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1866 1867 /* Zero outgoing translation limits (whole bar size windows) */ 1868 ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1869 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1870 1871 /* Set outgoing translation offsets */ 1872 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1873 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 1874 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1875 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1876 } else 1877 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1878 1879 /* Set the translation offset for B2B registers */ 1880 bar_addr = 0; 1881 if (b2b_bar_num == NTB_CONFIG_BAR) 1882 bar_addr = peer_addr->bar0_addr; 1883 else if (b2b_bar_num == NTB_B2B_BAR_1) 1884 bar_addr = peer_addr->bar2_addr64; 1885 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR)) 1886 bar_addr = peer_addr->bar4_addr64; 1887 else if (b2b_bar_num == NTB_B2B_BAR_2) 1888 bar_addr = peer_addr->bar4_addr32; 1889 else if (b2b_bar_num == NTB_B2B_BAR_3) 1890 bar_addr = peer_addr->bar5_addr32; 1891 else 1892 KASSERT(false, ("invalid bar")); 1893 1894 /* 1895 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1896 * at a time. 1897 */ 1898 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1899 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1900 return (0); 1901} 1902 1903static inline bool 1904_xeon_link_is_up(struct ntb_softc *ntb) 1905{ 1906 1907 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1908 return (true); 1909 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1910} 1911 1912static inline bool 1913link_is_up(struct ntb_softc *ntb) 1914{ 1915 1916 if (ntb->type == NTB_XEON) 1917 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1918 !HAS_FEATURE(NTB_SB01BASE_LOCKUP))); 1919 1920 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1921 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1922} 1923 1924static inline bool 1925atom_link_is_err(struct ntb_softc *ntb) 1926{ 1927 uint32_t status; 1928 1929 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1930 1931 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1932 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1933 return (true); 1934 1935 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1936 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1937} 1938 1939/* Atom does not have link status interrupt, poll on that platform */ 1940static void 1941atom_link_hb(void *arg) 1942{ 1943 struct ntb_softc *ntb = arg; 1944 sbintime_t timo, poll_ts; 1945 1946 timo = NTB_HB_TIMEOUT * hz; 1947 poll_ts = ntb->last_ts + timo; 1948 1949 /* 1950 * Delay polling the link status if an interrupt was received, unless 1951 * the cached link status says the link is down. 1952 */ 1953 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1954 timo = poll_ts - ticks; 1955 goto out; 1956 } 1957 1958 if (ntb_poll_link(ntb)) 1959 ntb_link_event(ntb); 1960 1961 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1962 /* Link is down with error, proceed with recovery */ 1963 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1964 return; 1965 } 1966 1967out: 1968 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1969} 1970 1971static void 1972atom_perform_link_restart(struct ntb_softc *ntb) 1973{ 1974 uint32_t status; 1975 1976 /* Driver resets the NTB ModPhy lanes - magic! */ 1977 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1978 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1979 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1980 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1981 1982 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1983 pause("ModPhy", hz / 10); 1984 1985 /* Clear AER Errors, write to clear */ 1986 status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1987 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1988 ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1989 1990 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1991 status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1992 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1993 ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1994 1995 /* Clear DeSkew Buffer error, write to clear */ 1996 status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1997 status |= ATOM_DESKEWSTS_DBERR; 1998 ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1999 2000 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2001 status &= ATOM_IBIST_ERR_OFLOW; 2002 ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2003 2004 /* Releases the NTB state machine to allow the link to retrain */ 2005 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2006 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2007 ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2008} 2009 2010/* 2011 * ntb_set_ctx() - associate a driver context with an ntb device 2012 * @ntb: NTB device context 2013 * @ctx: Driver context 2014 * @ctx_ops: Driver context operations 2015 * 2016 * Associate a driver context and operations with a ntb device. The context is 2017 * provided by the client driver, and the driver may associate a different 2018 * context with each ntb device. 2019 * 2020 * Return: Zero if the context is associated, otherwise an error number. 2021 */ 2022int 2023ntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops) 2024{ 2025 2026 if (ctx == NULL || ops == NULL) 2027 return (EINVAL); 2028 if (ntb->ctx_ops != NULL) 2029 return (EINVAL); 2030 2031 CTX_LOCK(ntb); 2032 if (ntb->ctx_ops != NULL) { 2033 CTX_UNLOCK(ntb); 2034 return (EINVAL); 2035 } 2036 ntb->ntb_ctx = ctx; 2037 ntb->ctx_ops = ops; 2038 CTX_UNLOCK(ntb); 2039 2040 return (0); 2041} 2042 2043/* 2044 * It is expected that this will only be used from contexts where the ctx_lock 2045 * is not needed to protect ntb_ctx lifetime. 2046 */ 2047void * 2048ntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops) 2049{ 2050 2051 KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2052 if (ops != NULL) 2053 *ops = ntb->ctx_ops; 2054 return (ntb->ntb_ctx); 2055} 2056 2057/* 2058 * ntb_clear_ctx() - disassociate any driver context from an ntb device 2059 * @ntb: NTB device context 2060 * 2061 * Clear any association that may exist between a driver context and the ntb 2062 * device. 2063 */ 2064void 2065ntb_clear_ctx(struct ntb_softc *ntb) 2066{ 2067 2068 CTX_LOCK(ntb); 2069 ntb->ntb_ctx = NULL; 2070 ntb->ctx_ops = NULL; 2071 CTX_UNLOCK(ntb); 2072} 2073 2074/* 2075 * ntb_link_event() - notify driver context of a change in link status 2076 * @ntb: NTB device context 2077 * 2078 * Notify the driver context that the link status may have changed. The driver 2079 * should call ntb_link_is_up() to get the current status. 2080 */ 2081void 2082ntb_link_event(struct ntb_softc *ntb) 2083{ 2084 2085 CTX_LOCK(ntb); 2086 if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2087 ntb->ctx_ops->link_event(ntb->ntb_ctx); 2088 CTX_UNLOCK(ntb); 2089} 2090 2091/* 2092 * ntb_db_event() - notify driver context of a doorbell event 2093 * @ntb: NTB device context 2094 * @vector: Interrupt vector number 2095 * 2096 * Notify the driver context of a doorbell event. If hardware supports 2097 * multiple interrupt vectors for doorbells, the vector number indicates which 2098 * vector received the interrupt. The vector number is relative to the first 2099 * vector used for doorbells, starting at zero, and must be less than 2100 * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2101 * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2102 * those bits are associated with the vector number. 2103 */ 2104static void 2105ntb_db_event(struct ntb_softc *ntb, uint32_t vec) 2106{ 2107 2108 CTX_LOCK(ntb); 2109 if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2110 ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2111 CTX_UNLOCK(ntb); 2112} 2113 2114/* 2115 * ntb_link_enable() - enable the link on the secondary side of the ntb 2116 * @ntb: NTB device context 2117 * @max_speed: The maximum link speed expressed as PCIe generation number[0] 2118 * @max_width: The maximum link width expressed as the number of PCIe lanes[0] 2119 * 2120 * Enable the link on the secondary side of the ntb. This can only be done 2121 * from the primary side of the ntb in primary or b2b topology. The ntb device 2122 * should train the link to its maximum speed and width, or the requested speed 2123 * and width, whichever is smaller, if supported. 2124 * 2125 * Return: Zero on success, otherwise an error number. 2126 * 2127 * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed 2128 * and width input will be ignored. 2129 */ 2130int 2131ntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused, 2132 enum ntb_width w __unused) 2133{ 2134 uint32_t cntl; 2135 2136 if (ntb->type == NTB_ATOM) { 2137 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2138 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2139 return (0); 2140 } 2141 2142 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2143 ntb_link_event(ntb); 2144 return (0); 2145 } 2146 2147 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2148 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2149 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2150 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2151 if (HAS_FEATURE(NTB_SPLIT_BAR)) 2152 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2153 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2154 return (0); 2155} 2156 2157/* 2158 * ntb_link_disable() - disable the link on the secondary side of the ntb 2159 * @ntb: NTB device context 2160 * 2161 * Disable the link on the secondary side of the ntb. This can only be done 2162 * from the primary side of the ntb in primary or b2b topology. The ntb device 2163 * should disable the link. Returning from this call must indicate that a 2164 * barrier has passed, though with no more writes may pass in either direction 2165 * across the link, except if this call returns an error number. 2166 * 2167 * Return: Zero on success, otherwise an error number. 2168 */ 2169int 2170ntb_link_disable(struct ntb_softc *ntb) 2171{ 2172 uint32_t cntl; 2173 2174 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2175 ntb_link_event(ntb); 2176 return (0); 2177 } 2178 2179 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2180 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2181 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2182 if (HAS_FEATURE(NTB_SPLIT_BAR)) 2183 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2184 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2185 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2186 return (0); 2187} 2188 2189static void 2190recover_atom_link(void *arg) 2191{ 2192 struct ntb_softc *ntb = arg; 2193 unsigned speed, width, oldspeed, oldwidth; 2194 uint32_t status32; 2195 2196 atom_perform_link_restart(ntb); 2197 2198 /* 2199 * There is a potential race between the 2 NTB devices recovering at 2200 * the same time. If the times are the same, the link will not recover 2201 * and the driver will be stuck in this loop forever. Add a random 2202 * interval to the recovery time to prevent this race. 2203 */ 2204 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2205 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2206 2207 if (atom_link_is_err(ntb)) 2208 goto retry; 2209 2210 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2211 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2212 goto out; 2213 2214 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2215 width = NTB_LNK_STA_WIDTH(status32); 2216 speed = status32 & NTB_LINK_SPEED_MASK; 2217 2218 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2219 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2220 if (oldwidth != width || oldspeed != speed) 2221 goto retry; 2222 2223out: 2224 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2225 ntb); 2226 return; 2227 2228retry: 2229 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2230 ntb); 2231} 2232 2233/* 2234 * Polls the HW link status register(s); returns true if something has changed. 2235 */ 2236static bool 2237ntb_poll_link(struct ntb_softc *ntb) 2238{ 2239 uint32_t ntb_cntl; 2240 uint16_t reg_val; 2241 2242 if (ntb->type == NTB_ATOM) { 2243 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2244 if (ntb_cntl == ntb->ntb_ctl) 2245 return (false); 2246 2247 ntb->ntb_ctl = ntb_cntl; 2248 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2249 } else { 2250 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2251 2252 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2253 if (reg_val == ntb->lnk_sta) 2254 return (false); 2255 2256 ntb->lnk_sta = reg_val; 2257 2258 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 2259 if (_xeon_link_is_up(ntb)) { 2260 if (!ntb->peer_msix_good) { 2261 callout_reset(&ntb->peer_msix_work, 0, 2262 ntb_exchange_msix, ntb); 2263 return (false); 2264 } 2265 } else { 2266 ntb->peer_msix_good = false; 2267 ntb->peer_msix_done = false; 2268 } 2269 } 2270 } 2271 return (true); 2272} 2273 2274static inline enum ntb_speed 2275ntb_link_sta_speed(struct ntb_softc *ntb) 2276{ 2277 2278 if (!link_is_up(ntb)) 2279 return (NTB_SPEED_NONE); 2280 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2281} 2282 2283static inline enum ntb_width 2284ntb_link_sta_width(struct ntb_softc *ntb) 2285{ 2286 2287 if (!link_is_up(ntb)) 2288 return (NTB_WIDTH_NONE); 2289 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2290} 2291 2292SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2293 "Driver state, statistics, and HW registers"); 2294 2295#define NTB_REGSZ_MASK (3ul << 30) 2296#define NTB_REG_64 (1ul << 30) 2297#define NTB_REG_32 (2ul << 30) 2298#define NTB_REG_16 (3ul << 30) 2299#define NTB_REG_8 (0ul << 30) 2300 2301#define NTB_DB_READ (1ul << 29) 2302#define NTB_PCI_REG (1ul << 28) 2303#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2304 2305static void 2306ntb_sysctl_init(struct ntb_softc *ntb) 2307{ 2308 struct sysctl_oid_list *tree_par, *regpar, *statpar, *errpar; 2309 struct sysctl_ctx_list *ctx; 2310 struct sysctl_oid *tree, *tmptree; 2311 2312 ctx = device_get_sysctl_ctx(ntb->device); 2313 2314 tree = SYSCTL_ADD_NODE(ctx, 2315 SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)), OID_AUTO, 2316 "debug_info", CTLFLAG_RD, NULL, 2317 "Driver state, statistics, and HW registers"); 2318 tree_par = SYSCTL_CHILDREN(tree); 2319 2320 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2321 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2322 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2323 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2324 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2325 &ntb->ppd, 0, "Raw PPD register (cached)"); 2326 2327 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2328 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2329 &ntb->b2b_mw_idx, 0, 2330 "Index of the MW used for B2B remote register access"); 2331 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2332 CTLFLAG_RD, &ntb->b2b_off, 2333 "If non-zero, offset of B2B register region in shared MW"); 2334 } 2335 2336 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2337 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2338 "Features/errata of this NTB device"); 2339 2340 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2341 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2342 "NTB CTL register (cached)"); 2343 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2344 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2345 "LNK STA register (cached)"); 2346 2347 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "link_status", 2348 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_link_status, 2349 "A", "Link status"); 2350 2351 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2352 &ntb->mw_count, 0, "MW count"); 2353 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2354 &ntb->spad_count, 0, "Scratchpad count"); 2355 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2356 &ntb->db_count, 0, "Doorbell count"); 2357 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2358 &ntb->db_vec_count, 0, "Doorbell vector count"); 2359 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2360 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2361 2362 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2363 &ntb->db_valid_mask, "Doorbell valid mask"); 2364 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2365 &ntb->db_link_mask, "Doorbell link mask"); 2366 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2367 &ntb->db_mask, "Doorbell mask (cached)"); 2368 2369 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2370 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2371 regpar = SYSCTL_CHILDREN(tmptree); 2372 2373 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2374 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2375 ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2376 "NTB Control register"); 2377 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2378 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2379 0x19c, sysctl_handle_register, "IU", 2380 "NTB Link Capabilities"); 2381 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2382 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2383 0x1a0, sysctl_handle_register, "IU", 2384 "NTB Link Control register"); 2385 2386 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2387 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2388 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2389 sysctl_handle_register, "QU", "Doorbell mask register"); 2390 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2391 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2392 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2393 sysctl_handle_register, "QU", "Doorbell register"); 2394 2395 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2396 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2397 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2398 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2399 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2400 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2401 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2402 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2403 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2404 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2405 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2406 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2407 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2408 } else { 2409 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2410 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2411 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2412 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2413 } 2414 2415 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2416 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2417 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2418 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2419 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2420 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2421 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2422 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2423 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2424 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2425 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2426 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2427 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2428 } else { 2429 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2430 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2431 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2432 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2433 } 2434 2435 if (ntb->type == NTB_ATOM) 2436 return; 2437 2438 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2439 CTLFLAG_RD, NULL, "Xeon HW statistics"); 2440 statpar = SYSCTL_CHILDREN(tmptree); 2441 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2442 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2443 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2444 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2445 2446 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2447 CTLFLAG_RD, NULL, "Xeon HW errors"); 2448 errpar = SYSCTL_CHILDREN(tmptree); 2449 2450 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2451 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2452 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2453 sysctl_handle_register, "CU", "PPD"); 2454 2455 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2456 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2457 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2458 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2459 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2460 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2461 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2462 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2463 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2464 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2465 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2466 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2467 2468 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2469 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2470 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2471 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2472 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2473 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2474 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2475 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2476 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2477 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2478 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2479 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2480 2481 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2482 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2483 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2484 sysctl_handle_register, "SU", "DEVSTS"); 2485 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2486 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2487 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2488 sysctl_handle_register, "SU", "LNKSTS"); 2489 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2490 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2491 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2492 sysctl_handle_register, "SU", "SLNKSTS"); 2493 2494 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2495 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2496 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2497 sysctl_handle_register, "IU", "UNCERRSTS"); 2498 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2499 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2500 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2501 sysctl_handle_register, "IU", "CORERRSTS"); 2502 2503 if (ntb->conn_type != NTB_CONN_B2B) 2504 return; 2505 2506 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2507 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2508 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2509 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2510 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2511 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2512 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2513 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2514 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2515 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2516 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2517 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2518 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2519 } else { 2520 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2521 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2522 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2523 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2524 } 2525 2526 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2527 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2528 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2529 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2530 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2531 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2532 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2533 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2534 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2535 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2536 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2537 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2538 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2539 } else { 2540 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2541 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2542 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2543 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2544 } 2545 2546 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2547 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2548 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2549 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2550 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2551 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2552 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2553 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2554 if (HAS_FEATURE(NTB_SPLIT_BAR)) { 2555 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2556 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2557 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2558 sysctl_handle_register, "IU", 2559 "Secondary BAR4 base register"); 2560 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2561 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2562 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2563 sysctl_handle_register, "IU", 2564 "Secondary BAR5 base register"); 2565 } else { 2566 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2567 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2568 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2569 sysctl_handle_register, "QU", 2570 "Secondary BAR45 base register"); 2571 } 2572} 2573 2574static int 2575sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2576{ 2577 struct ntb_softc *ntb; 2578 struct sbuf sb; 2579 int error; 2580 2581 error = 0; 2582 ntb = arg1; 2583 2584 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2585 2586 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2587 error = sbuf_finish(&sb); 2588 sbuf_delete(&sb); 2589 2590 if (error || !req->newptr) 2591 return (error); 2592 return (EINVAL); 2593} 2594 2595static int 2596sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2597{ 2598 struct ntb_softc *ntb; 2599 struct sbuf sb; 2600 enum ntb_speed speed; 2601 enum ntb_width width; 2602 int error; 2603 2604 error = 0; 2605 ntb = arg1; 2606 2607 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2608 2609 if (ntb_link_is_up(ntb, &speed, &width)) 2610 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2611 (unsigned)speed, (unsigned)width); 2612 else 2613 sbuf_printf(&sb, "down"); 2614 2615 error = sbuf_finish(&sb); 2616 sbuf_delete(&sb); 2617 2618 if (error || !req->newptr) 2619 return (error); 2620 return (EINVAL); 2621} 2622 2623static int 2624sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2625{ 2626 struct ntb_softc *ntb; 2627 const void *outp; 2628 uintptr_t sz; 2629 uint64_t umv; 2630 char be[sizeof(umv)]; 2631 size_t outsz; 2632 uint32_t reg; 2633 bool db, pci; 2634 int error; 2635 2636 ntb = arg1; 2637 reg = arg2 & ~NTB_REGFLAGS_MASK; 2638 sz = arg2 & NTB_REGSZ_MASK; 2639 db = (arg2 & NTB_DB_READ) != 0; 2640 pci = (arg2 & NTB_PCI_REG) != 0; 2641 2642 KASSERT(!(db && pci), ("bogus")); 2643 2644 if (db) { 2645 KASSERT(sz == NTB_REG_64, ("bogus")); 2646 umv = db_ioread(ntb, reg); 2647 outsz = sizeof(uint64_t); 2648 } else { 2649 switch (sz) { 2650 case NTB_REG_64: 2651 if (pci) 2652 umv = pci_read_config(ntb->device, reg, 8); 2653 else 2654 umv = ntb_reg_read(8, reg); 2655 outsz = sizeof(uint64_t); 2656 break; 2657 case NTB_REG_32: 2658 if (pci) 2659 umv = pci_read_config(ntb->device, reg, 4); 2660 else 2661 umv = ntb_reg_read(4, reg); 2662 outsz = sizeof(uint32_t); 2663 break; 2664 case NTB_REG_16: 2665 if (pci) 2666 umv = pci_read_config(ntb->device, reg, 2); 2667 else 2668 umv = ntb_reg_read(2, reg); 2669 outsz = sizeof(uint16_t); 2670 break; 2671 case NTB_REG_8: 2672 if (pci) 2673 umv = pci_read_config(ntb->device, reg, 1); 2674 else 2675 umv = ntb_reg_read(1, reg); 2676 outsz = sizeof(uint8_t); 2677 break; 2678 default: 2679 panic("bogus"); 2680 break; 2681 } 2682 } 2683 2684 /* Encode bigendian so that sysctl -x is legible. */ 2685 be64enc(be, umv); 2686 outp = ((char *)be) + sizeof(umv) - outsz; 2687 2688 error = SYSCTL_OUT(req, outp, outsz); 2689 if (error || !req->newptr) 2690 return (error); 2691 return (EINVAL); 2692} 2693 2694static unsigned 2695ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2696{ 2697 2698 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2699 uidx >= ntb->b2b_mw_idx) || 2700 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2701 uidx++; 2702 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2703 uidx >= ntb->b2b_mw_idx) && 2704 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2705 uidx++; 2706 return (uidx); 2707} 2708 2709static void 2710ntb_exchange_msix(void *ctx) 2711{ 2712 struct ntb_softc *ntb; 2713 uint32_t val; 2714 unsigned i; 2715 2716 ntb = ctx; 2717 2718 if (ntb->peer_msix_done) 2719 goto msix_done; 2720 2721 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2722 ntb_peer_spad_write(ntb, NTB_MSIX_DATA0 + i, 2723 ntb->msix_data[i].nmd_data); 2724 ntb_peer_spad_write(ntb, NTB_MSIX_OFS0 + i, 2725 ntb->msix_data[i].nmd_ofs); 2726 } 2727 ntb_peer_spad_write(ntb, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2728 2729 ntb_spad_read(ntb, NTB_MSIX_GUARD, &val); 2730 if (val != NTB_MSIX_VER_GUARD) 2731 goto reschedule; 2732 2733 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2734 ntb_spad_read(ntb, NTB_MSIX_DATA0 + i, &val); 2735 ntb->peer_msix_data[i].nmd_data = val; 2736 ntb_spad_read(ntb, NTB_MSIX_OFS0 + i, &val); 2737 ntb->peer_msix_data[i].nmd_ofs = val; 2738 } 2739 2740 ntb->peer_msix_done = true; 2741 2742msix_done: 2743 ntb_peer_spad_write(ntb, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2744 ntb_spad_read(ntb, NTB_MSIX_DONE, &val); 2745 if (val != NTB_MSIX_RECEIVED) 2746 goto reschedule; 2747 2748 ntb->peer_msix_good = true; 2749 2750 ntb_poll_link(ntb); 2751 ntb_link_event(ntb); 2752 return; 2753 2754reschedule: 2755 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2756 if (_xeon_link_is_up(ntb)) 2757 callout_reset(&ntb->peer_msix_work, hz / 100, ntb_exchange_msix, ntb); 2758 else 2759 ntb_spad_clear(ntb); 2760} 2761 2762/* 2763 * Public API to the rest of the OS 2764 */ 2765 2766/** 2767 * ntb_get_max_spads() - get the total scratch regs usable 2768 * @ntb: pointer to ntb_softc instance 2769 * 2770 * This function returns the max 32bit scratchpad registers usable by the 2771 * upper layer. 2772 * 2773 * RETURNS: total number of scratch pad registers available 2774 */ 2775uint8_t 2776ntb_get_max_spads(struct ntb_softc *ntb) 2777{ 2778 2779 return (ntb->spad_count); 2780} 2781 2782/* 2783 * ntb_mw_count() - Get the number of memory windows available for KPI 2784 * consumers. 2785 * 2786 * (Excludes any MW wholly reserved for register access.) 2787 */ 2788uint8_t 2789ntb_mw_count(struct ntb_softc *ntb) 2790{ 2791 uint8_t res; 2792 2793 res = ntb->mw_count; 2794 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2795 res--; 2796 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2797 res--; 2798 return (res); 2799} 2800 2801/** 2802 * ntb_spad_write() - write to the secondary scratchpad register 2803 * @ntb: pointer to ntb_softc instance 2804 * @idx: index to the scratchpad register, 0 based 2805 * @val: the data value to put into the register 2806 * 2807 * This function allows writing of a 32bit value to the indexed scratchpad 2808 * register. The register resides on the secondary (external) side. 2809 * 2810 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2811 */ 2812int 2813ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2814{ 2815 2816 if (idx >= ntb->spad_count) 2817 return (EINVAL); 2818 2819 ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2820 2821 return (0); 2822} 2823 2824/* 2825 * Zeros the local scratchpad. 2826 */ 2827void 2828ntb_spad_clear(struct ntb_softc *ntb) 2829{ 2830 unsigned i; 2831 2832 for (i = 0; i < ntb->spad_count; i++) 2833 ntb_spad_write(ntb, i, 0); 2834} 2835 2836/** 2837 * ntb_spad_read() - read from the primary scratchpad register 2838 * @ntb: pointer to ntb_softc instance 2839 * @idx: index to scratchpad register, 0 based 2840 * @val: pointer to 32bit integer for storing the register value 2841 * 2842 * This function allows reading of the 32bit scratchpad register on 2843 * the primary (internal) side. 2844 * 2845 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2846 */ 2847int 2848ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2849{ 2850 2851 if (idx >= ntb->spad_count) 2852 return (EINVAL); 2853 2854 *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2855 2856 return (0); 2857} 2858 2859/** 2860 * ntb_peer_spad_write() - write to the secondary scratchpad register 2861 * @ntb: pointer to ntb_softc instance 2862 * @idx: index to the scratchpad register, 0 based 2863 * @val: the data value to put into the register 2864 * 2865 * This function allows writing of a 32bit value to the indexed scratchpad 2866 * register. The register resides on the secondary (external) side. 2867 * 2868 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2869 */ 2870int 2871ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val) 2872{ 2873 2874 if (idx >= ntb->spad_count) 2875 return (EINVAL); 2876 2877 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2878 ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2879 else 2880 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2881 2882 return (0); 2883} 2884 2885/** 2886 * ntb_peer_spad_read() - read from the primary scratchpad register 2887 * @ntb: pointer to ntb_softc instance 2888 * @idx: index to scratchpad register, 0 based 2889 * @val: pointer to 32bit integer for storing the register value 2890 * 2891 * This function allows reading of the 32bit scratchpad register on 2892 * the primary (internal) side. 2893 * 2894 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 2895 */ 2896int 2897ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val) 2898{ 2899 2900 if (idx >= ntb->spad_count) 2901 return (EINVAL); 2902 2903 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) 2904 *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2905 else 2906 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2907 2908 return (0); 2909} 2910 2911/* 2912 * ntb_mw_get_range() - get the range of a memory window 2913 * @ntb: NTB device context 2914 * @idx: Memory window number 2915 * @base: OUT - the base address for mapping the memory window 2916 * @size: OUT - the size for mapping the memory window 2917 * @align: OUT - the base alignment for translating the memory window 2918 * @align_size: OUT - the size alignment for translating the memory window 2919 * 2920 * Get the range of a memory window. NULL may be given for any output 2921 * parameter if the value is not needed. The base and size may be used for 2922 * mapping the memory window, to access the peer memory. The alignment and 2923 * size may be used for translating the memory window, for the peer to access 2924 * memory on the local system. 2925 * 2926 * Return: Zero on success, otherwise an error number. 2927 */ 2928int 2929ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base, 2930 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2931 bus_addr_t *plimit) 2932{ 2933 struct ntb_pci_bar_info *bar; 2934 bus_addr_t limit; 2935 size_t bar_b2b_off; 2936 enum ntb_bar bar_num; 2937 2938 if (mw_idx >= ntb_mw_count(ntb)) 2939 return (EINVAL); 2940 mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 2941 2942 bar_num = ntb_mw_to_bar(ntb, mw_idx); 2943 bar = &ntb->bar_info[bar_num]; 2944 bar_b2b_off = 0; 2945 if (mw_idx == ntb->b2b_mw_idx) { 2946 KASSERT(ntb->b2b_off != 0, 2947 ("user shouldn't get non-shared b2b mw")); 2948 bar_b2b_off = ntb->b2b_off; 2949 } 2950 2951 if (bar_is_64bit(ntb, bar_num)) 2952 limit = BUS_SPACE_MAXADDR; 2953 else 2954 limit = BUS_SPACE_MAXADDR_32BIT; 2955 2956 if (base != NULL) 2957 *base = bar->pbase + bar_b2b_off; 2958 if (vbase != NULL) 2959 *vbase = bar->vbase + bar_b2b_off; 2960 if (size != NULL) 2961 *size = bar->size - bar_b2b_off; 2962 if (align != NULL) 2963 *align = bar->size; 2964 if (align_size != NULL) 2965 *align_size = 1; 2966 if (plimit != NULL) 2967 *plimit = limit; 2968 return (0); 2969} 2970 2971/* 2972 * ntb_mw_set_trans() - set the translation of a memory window 2973 * @ntb: NTB device context 2974 * @idx: Memory window number 2975 * @addr: The dma address local memory to expose to the peer 2976 * @size: The size of the local memory to expose to the peer 2977 * 2978 * Set the translation of a memory window. The peer may access local memory 2979 * through the window starting at the address, up to the size. The address 2980 * must be aligned to the alignment specified by ntb_mw_get_range(). The size 2981 * must be aligned to the size alignment specified by ntb_mw_get_range(). The 2982 * address must be below the plimit specified by ntb_mw_get_range() (i.e. for 2983 * 32-bit BARs). 2984 * 2985 * Return: Zero on success, otherwise an error number. 2986 */ 2987int 2988ntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr, 2989 size_t size) 2990{ 2991 struct ntb_pci_bar_info *bar; 2992 uint64_t base, limit, reg_val; 2993 size_t bar_size, mw_size; 2994 uint32_t base_reg, xlat_reg, limit_reg; 2995 enum ntb_bar bar_num; 2996 2997 if (idx >= ntb_mw_count(ntb)) 2998 return (EINVAL); 2999 idx = ntb_user_mw_to_idx(ntb, idx); 3000 3001 bar_num = ntb_mw_to_bar(ntb, idx); 3002 bar = &ntb->bar_info[bar_num]; 3003 3004 bar_size = bar->size; 3005 if (idx == ntb->b2b_mw_idx) 3006 mw_size = bar_size - ntb->b2b_off; 3007 else 3008 mw_size = bar_size; 3009 3010 /* Hardware requires that addr is aligned to bar size */ 3011 if ((addr & (bar_size - 1)) != 0) 3012 return (EINVAL); 3013 3014 if (size > mw_size) 3015 return (EINVAL); 3016 3017 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 3018 3019 limit = 0; 3020 if (bar_is_64bit(ntb, bar_num)) { 3021 base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 3022 3023 if (limit_reg != 0 && size != mw_size) 3024 limit = base + size; 3025 3026 /* Set and verify translation address */ 3027 ntb_reg_write(8, xlat_reg, addr); 3028 reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 3029 if (reg_val != addr) { 3030 ntb_reg_write(8, xlat_reg, 0); 3031 return (EIO); 3032 } 3033 3034 /* Set and verify the limit */ 3035 ntb_reg_write(8, limit_reg, limit); 3036 reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3037 if (reg_val != limit) { 3038 ntb_reg_write(8, limit_reg, base); 3039 ntb_reg_write(8, xlat_reg, 0); 3040 return (EIO); 3041 } 3042 } else { 3043 /* Configure 32-bit (split) BAR MW */ 3044 3045 if ((addr & UINT32_MAX) != addr) 3046 return (ERANGE); 3047 if (((addr + size) & UINT32_MAX) != (addr + size)) 3048 return (ERANGE); 3049 3050 base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3051 3052 if (limit_reg != 0 && size != mw_size) 3053 limit = base + size; 3054 3055 /* Set and verify translation address */ 3056 ntb_reg_write(4, xlat_reg, addr); 3057 reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3058 if (reg_val != addr) { 3059 ntb_reg_write(4, xlat_reg, 0); 3060 return (EIO); 3061 } 3062 3063 /* Set and verify the limit */ 3064 ntb_reg_write(4, limit_reg, limit); 3065 reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3066 if (reg_val != limit) { 3067 ntb_reg_write(4, limit_reg, base); 3068 ntb_reg_write(4, xlat_reg, 0); 3069 return (EIO); 3070 } 3071 } 3072 return (0); 3073} 3074 3075/* 3076 * ntb_mw_clear_trans() - clear the translation of a memory window 3077 * @ntb: NTB device context 3078 * @idx: Memory window number 3079 * 3080 * Clear the translation of a memory window. The peer may no longer access 3081 * local memory through the window. 3082 * 3083 * Return: Zero on success, otherwise an error number. 3084 */ 3085int 3086ntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx) 3087{ 3088 3089 return (ntb_mw_set_trans(ntb, mw_idx, 0, 0)); 3090} 3091 3092/* 3093 * ntb_mw_get_wc - Get the write-combine status of a memory window 3094 * 3095 * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if 3096 * idx is an invalid memory window). 3097 * 3098 * Mode is a VM_MEMATTR_* type. 3099 */ 3100int 3101ntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t *mode) 3102{ 3103 struct ntb_pci_bar_info *bar; 3104 3105 if (idx >= ntb_mw_count(ntb)) 3106 return (EINVAL); 3107 idx = ntb_user_mw_to_idx(ntb, idx); 3108 3109 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3110 *mode = bar->map_mode; 3111 return (0); 3112} 3113 3114/* 3115 * ntb_mw_set_wc - Set the write-combine status of a memory window 3116 * 3117 * If 'mode' matches the current status, this does nothing and succeeds. Mode 3118 * is a VM_MEMATTR_* type. 3119 * 3120 * Returns: Zero on success, setting the caching attribute on the virtual 3121 * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid 3122 * memory window, or if changing the caching attribute fails). 3123 */ 3124int 3125ntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3126{ 3127 3128 if (idx >= ntb_mw_count(ntb)) 3129 return (EINVAL); 3130 3131 idx = ntb_user_mw_to_idx(ntb, idx); 3132 return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3133} 3134 3135static int 3136ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3137{ 3138 struct ntb_pci_bar_info *bar; 3139 int rc; 3140 3141 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3142 if (bar->map_mode == mode) 3143 return (0); 3144 3145 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3146 if (rc == 0) 3147 bar->map_mode = mode; 3148 3149 return (rc); 3150} 3151 3152/** 3153 * ntb_peer_db_set() - Set the doorbell on the secondary/external side 3154 * @ntb: pointer to ntb_softc instance 3155 * @bit: doorbell bits to ring 3156 * 3157 * This function allows triggering of a doorbell on the secondary/external 3158 * side that will initiate an interrupt on the remote host 3159 */ 3160void 3161ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit) 3162{ 3163 3164 if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) { 3165 struct ntb_pci_bar_info *lapic; 3166 unsigned i; 3167 3168 lapic = ntb->peer_lapic_bar; 3169 3170 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3171 if ((bit & ntb_db_vector_mask(ntb, i)) != 0) 3172 bus_space_write_4(lapic->pci_bus_tag, 3173 lapic->pci_bus_handle, 3174 ntb->peer_msix_data[i].nmd_ofs, 3175 ntb->peer_msix_data[i].nmd_data); 3176 } 3177 return; 3178 } 3179 3180 if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3181 ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3182 return; 3183 } 3184 3185 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3186} 3187 3188/* 3189 * ntb_get_peer_db_addr() - Return the address of the remote doorbell register, 3190 * as well as the size of the register (via *sz_out). 3191 * 3192 * This function allows a caller using I/OAT DMA to chain the remote doorbell 3193 * ring to its memory window write. 3194 * 3195 * Note that writing the peer doorbell via a memory window will *not* generate 3196 * an interrupt on the remote host; that must be done seperately. 3197 */ 3198bus_addr_t 3199ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out) 3200{ 3201 struct ntb_pci_bar_info *bar; 3202 uint64_t regoff; 3203 3204 KASSERT(sz_out != NULL, ("must be non-NULL")); 3205 3206 if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { 3207 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3208 regoff = ntb->peer_reg->db_bell; 3209 } else { 3210 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3211 ("invalid b2b idx")); 3212 3213 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3214 regoff = XEON_PDOORBELL_OFFSET; 3215 } 3216 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3217 3218 *sz_out = ntb->reg->db_size; 3219 /* HACK: Specific to current x86 bus implementation. */ 3220 return ((uint64_t)bar->pci_bus_handle + regoff); 3221} 3222 3223/* 3224 * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 3225 * @ntb: NTB device context 3226 * 3227 * Hardware may support different number or arrangement of doorbell bits. 3228 * 3229 * Return: A mask of doorbell bits supported by the ntb. 3230 */ 3231uint64_t 3232ntb_db_valid_mask(struct ntb_softc *ntb) 3233{ 3234 3235 return (ntb->db_valid_mask); 3236} 3237 3238/* 3239 * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector 3240 * @ntb: NTB device context 3241 * @vector: Doorbell vector number 3242 * 3243 * Each interrupt vector may have a different number or arrangement of bits. 3244 * 3245 * Return: A mask of doorbell bits serviced by a vector. 3246 */ 3247uint64_t 3248ntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector) 3249{ 3250 3251 if (vector > ntb->db_vec_count) 3252 return (0); 3253 return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3254} 3255 3256/** 3257 * ntb_link_is_up() - get the current ntb link state 3258 * @ntb: NTB device context 3259 * @speed: OUT - The link speed expressed as PCIe generation number 3260 * @width: OUT - The link width expressed as the number of PCIe lanes 3261 * 3262 * RETURNS: true or false based on the hardware link state 3263 */ 3264bool 3265ntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed, 3266 enum ntb_width *width) 3267{ 3268 3269 if (speed != NULL) 3270 *speed = ntb_link_sta_speed(ntb); 3271 if (width != NULL) 3272 *width = ntb_link_sta_width(ntb); 3273 return (link_is_up(ntb)); 3274} 3275 3276static void 3277save_bar_parameters(struct ntb_pci_bar_info *bar) 3278{ 3279 3280 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3281 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3282 bar->pbase = rman_get_start(bar->pci_resource); 3283 bar->size = rman_get_size(bar->pci_resource); 3284 bar->vbase = rman_get_virtual(bar->pci_resource); 3285} 3286 3287device_t 3288ntb_get_device(struct ntb_softc *ntb) 3289{ 3290 3291 return (ntb->device); 3292} 3293 3294/* Export HW-specific errata information. */ 3295bool 3296ntb_has_feature(struct ntb_softc *ntb, uint32_t feature) 3297{ 3298 3299 return (HAS_FEATURE(feature)); 3300} 3301