ntb_hw_intel.c revision 302508
1/*- 2 * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 3 * Copyright (C) 2013 Intel Corporation 4 * Copyright (C) 2015 EMC Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29/* 30 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 31 * two or more systems using a PCI-e links, providing remote memory access. 32 * 33 * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 34 * 35 * NOTE: Much of the code in this module is shared with Linux. Any patches may 36 * be picked up and redistributed in Linux with a dual GPL/BSD license. 37 */ 38 39#include <sys/cdefs.h> 40__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 302508 2016-07-09 23:22:44Z mav $"); 41 42#include <sys/param.h> 43#include <sys/kernel.h> 44#include <sys/systm.h> 45#include <sys/bus.h> 46#include <sys/endian.h> 47#include <sys/interrupt.h> 48#include <sys/malloc.h> 49#include <sys/module.h> 50#include <sys/mutex.h> 51#include <sys/pciio.h> 52#include <sys/queue.h> 53#include <sys/rman.h> 54#include <sys/rmlock.h> 55#include <sys/sbuf.h> 56#include <sys/sysctl.h> 57#include <vm/vm.h> 58#include <vm/pmap.h> 59#include <machine/bus.h> 60#include <machine/intr_machdep.h> 61#include <machine/resource.h> 62#include <dev/pci/pcireg.h> 63#include <dev/pci/pcivar.h> 64 65#include "ntb_regs.h" 66#include "../ntb.h" 67 68#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 69 70#define NTB_HB_TIMEOUT 1 /* second */ 71#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 72#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 73 74#define NTB_MSIX_VER_GUARD 0xaabbccdd 75#define NTB_MSIX_RECEIVED 0xe0f0e0f0 76 77/* 78 * PCI constants could be somewhere more generic, but aren't defined/used in 79 * pci.c. 80 */ 81#define PCI_MSIX_ENTRY_SIZE 16 82#define PCI_MSIX_ENTRY_LOWER_ADDR 0 83#define PCI_MSIX_ENTRY_UPPER_ADDR 4 84#define PCI_MSIX_ENTRY_DATA 8 85 86enum ntb_device_type { 87 NTB_XEON, 88 NTB_ATOM 89}; 90 91/* ntb_conn_type are hardware numbers, cannot change. */ 92enum ntb_conn_type { 93 NTB_CONN_TRANSPARENT = 0, 94 NTB_CONN_B2B = 1, 95 NTB_CONN_RP = 2, 96}; 97 98enum ntb_b2b_direction { 99 NTB_DEV_USD = 0, 100 NTB_DEV_DSD = 1, 101}; 102 103enum ntb_bar { 104 NTB_CONFIG_BAR = 0, 105 NTB_B2B_BAR_1, 106 NTB_B2B_BAR_2, 107 NTB_B2B_BAR_3, 108 NTB_MAX_BARS 109}; 110 111enum { 112 NTB_MSIX_GUARD = 0, 113 NTB_MSIX_DATA0, 114 NTB_MSIX_DATA1, 115 NTB_MSIX_DATA2, 116 NTB_MSIX_OFS0, 117 NTB_MSIX_OFS1, 118 NTB_MSIX_OFS2, 119 NTB_MSIX_DONE, 120 NTB_MAX_MSIX_SPAD 121}; 122 123/* Device features and workarounds */ 124#define HAS_FEATURE(ntb, feature) \ 125 (((ntb)->features & (feature)) != 0) 126 127struct ntb_hw_info { 128 uint32_t device_id; 129 const char *desc; 130 enum ntb_device_type type; 131 uint32_t features; 132}; 133 134struct ntb_pci_bar_info { 135 bus_space_tag_t pci_bus_tag; 136 bus_space_handle_t pci_bus_handle; 137 int pci_resource_id; 138 struct resource *pci_resource; 139 vm_paddr_t pbase; 140 caddr_t vbase; 141 vm_size_t size; 142 vm_memattr_t map_mode; 143 144 /* Configuration register offsets */ 145 uint32_t psz_off; 146 uint32_t ssz_off; 147 uint32_t pbarxlat_off; 148}; 149 150struct ntb_int_info { 151 struct resource *res; 152 int rid; 153 void *tag; 154}; 155 156struct ntb_vec { 157 struct ntb_softc *ntb; 158 uint32_t num; 159 unsigned masked; 160}; 161 162struct ntb_reg { 163 uint32_t ntb_ctl; 164 uint32_t lnk_sta; 165 uint8_t db_size; 166 unsigned mw_bar[NTB_MAX_BARS]; 167}; 168 169struct ntb_alt_reg { 170 uint32_t db_bell; 171 uint32_t db_mask; 172 uint32_t spad; 173}; 174 175struct ntb_xlat_reg { 176 uint32_t bar0_base; 177 uint32_t bar2_base; 178 uint32_t bar4_base; 179 uint32_t bar5_base; 180 181 uint32_t bar2_xlat; 182 uint32_t bar4_xlat; 183 uint32_t bar5_xlat; 184 185 uint32_t bar2_limit; 186 uint32_t bar4_limit; 187 uint32_t bar5_limit; 188}; 189 190struct ntb_b2b_addr { 191 uint64_t bar0_addr; 192 uint64_t bar2_addr64; 193 uint64_t bar4_addr64; 194 uint64_t bar4_addr32; 195 uint64_t bar5_addr32; 196}; 197 198struct ntb_msix_data { 199 uint32_t nmd_ofs; 200 uint32_t nmd_data; 201}; 202 203struct ntb_softc { 204 device_t device; 205 enum ntb_device_type type; 206 uint32_t features; 207 208 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 209 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 210 uint32_t allocated_interrupts; 211 212 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 213 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 214 bool peer_msix_good; 215 bool peer_msix_done; 216 struct ntb_pci_bar_info *peer_lapic_bar; 217 struct callout peer_msix_work; 218 219 struct callout heartbeat_timer; 220 struct callout lr_timer; 221 222 void *ntb_ctx; 223 const struct ntb_ctx_ops *ctx_ops; 224 struct ntb_vec *msix_vec; 225 struct rmlock ctx_lock; 226 227 uint32_t ppd; 228 enum ntb_conn_type conn_type; 229 enum ntb_b2b_direction dev_type; 230 231 /* Offset of peer bar0 in B2B BAR */ 232 uint64_t b2b_off; 233 /* Memory window used to access peer bar0 */ 234#define B2B_MW_DISABLED UINT8_MAX 235 uint8_t b2b_mw_idx; 236 uint32_t msix_xlat; 237 uint8_t msix_mw_idx; 238 239 uint8_t mw_count; 240 uint8_t spad_count; 241 uint8_t db_count; 242 uint8_t db_vec_count; 243 uint8_t db_vec_shift; 244 245 /* Protects local db_mask. */ 246#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 247#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 248#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 249 struct mtx db_mask_lock; 250 251 volatile uint32_t ntb_ctl; 252 volatile uint32_t lnk_sta; 253 254 uint64_t db_valid_mask; 255 uint64_t db_link_mask; 256 uint64_t db_mask; 257 uint64_t fake_db_bell; /* NTB_SB01BASE_LOCKUP*/ 258 259 int last_ts; /* ticks @ last irq */ 260 261 const struct ntb_reg *reg; 262 const struct ntb_alt_reg *self_reg; 263 const struct ntb_alt_reg *peer_reg; 264 const struct ntb_xlat_reg *xlat_reg; 265}; 266 267#ifdef __i386__ 268static __inline uint64_t 269bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 270 bus_size_t offset) 271{ 272 273 return (bus_space_read_4(tag, handle, offset) | 274 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 275} 276 277static __inline void 278bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 279 bus_size_t offset, uint64_t val) 280{ 281 282 bus_space_write_4(tag, handle, offset, val); 283 bus_space_write_4(tag, handle, offset + 4, val >> 32); 284} 285#endif 286 287#define ntb_bar_read(SIZE, bar, offset) \ 288 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 289 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 290#define ntb_bar_write(SIZE, bar, offset, val) \ 291 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 292 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 293#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 294#define ntb_reg_write(SIZE, offset, val) \ 295 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 296#define ntb_mw_read(SIZE, offset) \ 297 ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 298#define ntb_mw_write(SIZE, offset, val) \ 299 ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 300 offset, val) 301 302static int ntb_probe(device_t device); 303static int ntb_attach(device_t device); 304static int ntb_detach(device_t device); 305static uint64_t ntb_db_valid_mask(device_t dev); 306static void ntb_spad_clear(device_t dev); 307static uint64_t ntb_db_vector_mask(device_t dev, uint32_t vector); 308static bool ntb_link_is_up(device_t dev, enum ntb_speed *speed, 309 enum ntb_width *width); 310static int ntb_link_enable(device_t dev, enum ntb_speed speed, 311 enum ntb_width width); 312static int ntb_link_disable(device_t dev); 313static int ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 314static int ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 315 316static unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 317static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 318static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 319static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 320 uint32_t *base, uint32_t *xlat, uint32_t *lmt); 321static int ntb_map_pci_bars(struct ntb_softc *ntb); 322static int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 323 vm_memattr_t); 324static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 325 const char *); 326static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 327static int map_memory_window_bar(struct ntb_softc *ntb, 328 struct ntb_pci_bar_info *bar); 329static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 330static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 331static int ntb_init_isr(struct ntb_softc *ntb); 332static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 333static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 334static void ntb_teardown_interrupts(struct ntb_softc *ntb); 335static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 336static void ntb_interrupt(struct ntb_softc *, uint32_t vec); 337static void ndev_vec_isr(void *arg); 338static void ndev_irq_isr(void *arg); 339static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 340static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 341static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 342static int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 343static void ntb_free_msix_vec(struct ntb_softc *ntb); 344static void ntb_get_msix_info(struct ntb_softc *ntb); 345static void ntb_exchange_msix(void *); 346static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 347static void ntb_detect_max_mw(struct ntb_softc *ntb); 348static int ntb_detect_xeon(struct ntb_softc *ntb); 349static int ntb_detect_atom(struct ntb_softc *ntb); 350static int ntb_xeon_init_dev(struct ntb_softc *ntb); 351static int ntb_atom_init_dev(struct ntb_softc *ntb); 352static void ntb_teardown_xeon(struct ntb_softc *ntb); 353static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 354static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 355 enum ntb_bar regbar); 356static void xeon_set_sbar_base_and_limit(struct ntb_softc *, 357 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 358static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 359 enum ntb_bar idx); 360static int xeon_setup_b2b_mw(struct ntb_softc *, 361 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 362static inline bool link_is_up(struct ntb_softc *ntb); 363static inline bool _xeon_link_is_up(struct ntb_softc *ntb); 364static inline bool atom_link_is_err(struct ntb_softc *ntb); 365static inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 366static inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 367static void atom_link_hb(void *arg); 368static void ntb_link_event(device_t dev); 369static void ntb_db_event(device_t dev, uint32_t vec); 370static void recover_atom_link(void *arg); 371static bool ntb_poll_link(struct ntb_softc *ntb); 372static void save_bar_parameters(struct ntb_pci_bar_info *bar); 373static void ntb_sysctl_init(struct ntb_softc *); 374static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 375static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 376static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 377static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 378static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 379 380static unsigned g_ntb_hw_debug_level; 381SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 382 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 383#define ntb_printf(lvl, ...) do { \ 384 if ((lvl) <= g_ntb_hw_debug_level) { \ 385 device_printf(ntb->device, __VA_ARGS__); \ 386 } \ 387} while (0) 388 389#define _NTB_PAT_UC 0 390#define _NTB_PAT_WC 1 391#define _NTB_PAT_WT 4 392#define _NTB_PAT_WP 5 393#define _NTB_PAT_WB 6 394#define _NTB_PAT_UCM 7 395static unsigned g_ntb_mw_pat = _NTB_PAT_UC; 396SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 397 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 398 "UC: " __XSTRING(_NTB_PAT_UC) ", " 399 "WC: " __XSTRING(_NTB_PAT_WC) ", " 400 "WT: " __XSTRING(_NTB_PAT_WT) ", " 401 "WP: " __XSTRING(_NTB_PAT_WP) ", " 402 "WB: " __XSTRING(_NTB_PAT_WB) ", " 403 "UC-: " __XSTRING(_NTB_PAT_UCM)); 404 405static inline vm_memattr_t 406ntb_pat_flags(void) 407{ 408 409 switch (g_ntb_mw_pat) { 410 case _NTB_PAT_WC: 411 return (VM_MEMATTR_WRITE_COMBINING); 412 case _NTB_PAT_WT: 413 return (VM_MEMATTR_WRITE_THROUGH); 414 case _NTB_PAT_WP: 415 return (VM_MEMATTR_WRITE_PROTECTED); 416 case _NTB_PAT_WB: 417 return (VM_MEMATTR_WRITE_BACK); 418 case _NTB_PAT_UCM: 419 return (VM_MEMATTR_WEAK_UNCACHEABLE); 420 case _NTB_PAT_UC: 421 /* FALLTHROUGH */ 422 default: 423 return (VM_MEMATTR_UNCACHEABLE); 424 } 425} 426 427/* 428 * Well, this obviously doesn't belong here, but it doesn't seem to exist 429 * anywhere better yet. 430 */ 431static inline const char * 432ntb_vm_memattr_to_str(vm_memattr_t pat) 433{ 434 435 switch (pat) { 436 case VM_MEMATTR_WRITE_COMBINING: 437 return ("WRITE_COMBINING"); 438 case VM_MEMATTR_WRITE_THROUGH: 439 return ("WRITE_THROUGH"); 440 case VM_MEMATTR_WRITE_PROTECTED: 441 return ("WRITE_PROTECTED"); 442 case VM_MEMATTR_WRITE_BACK: 443 return ("WRITE_BACK"); 444 case VM_MEMATTR_WEAK_UNCACHEABLE: 445 return ("UNCACHED"); 446 case VM_MEMATTR_UNCACHEABLE: 447 return ("UNCACHEABLE"); 448 default: 449 return ("UNKNOWN"); 450 } 451} 452 453static int g_ntb_msix_idx = 1; 454SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 455 0, "Use this memory window to access the peer MSIX message complex on " 456 "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 457 "Like b2b_mw_idx, negative values index from the last available memory " 458 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 459 460static int g_ntb_mw_idx = -1; 461SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 462 0, "Use this memory window to access the peer NTB registers. A " 463 "non-negative value starts from the first MW index; a negative value " 464 "starts from the last MW index. The default is -1, i.e., the last " 465 "available memory window. Both sides of the NTB MUST set the same " 466 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 467 468/* Hardware owns the low 16 bits of features. */ 469#define NTB_BAR_SIZE_4K (1 << 0) 470#define NTB_SDOORBELL_LOCKUP (1 << 1) 471#define NTB_SB01BASE_LOCKUP (1 << 2) 472#define NTB_B2BDOORBELL_BIT14 (1 << 3) 473/* Software/configuration owns the top 16 bits. */ 474#define NTB_SPLIT_BAR (1ull << 16) 475 476#define NTB_FEATURES_STR \ 477 "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 478 "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 479 480static struct ntb_hw_info pci_ids[] = { 481 /* XXX: PS/SS IDs left out until they are supported. */ 482 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 483 NTB_ATOM, 0 }, 484 485 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 486 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 487 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 488 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 489 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 490 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 491 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 492 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 493 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 494 NTB_SB01BASE_LOCKUP }, 495 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 496 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 497 NTB_SB01BASE_LOCKUP }, 498 499 { 0x00000000, NULL, NTB_ATOM, 0 } 500}; 501 502static const struct ntb_reg atom_reg = { 503 .ntb_ctl = ATOM_NTBCNTL_OFFSET, 504 .lnk_sta = ATOM_LINK_STATUS_OFFSET, 505 .db_size = sizeof(uint64_t), 506 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 507}; 508 509static const struct ntb_alt_reg atom_pri_reg = { 510 .db_bell = ATOM_PDOORBELL_OFFSET, 511 .db_mask = ATOM_PDBMSK_OFFSET, 512 .spad = ATOM_SPAD_OFFSET, 513}; 514 515static const struct ntb_alt_reg atom_b2b_reg = { 516 .db_bell = ATOM_B2B_DOORBELL_OFFSET, 517 .spad = ATOM_B2B_SPAD_OFFSET, 518}; 519 520static const struct ntb_xlat_reg atom_sec_xlat = { 521#if 0 522 /* "FIXME" says the Linux driver. */ 523 .bar0_base = ATOM_SBAR0BASE_OFFSET, 524 .bar2_base = ATOM_SBAR2BASE_OFFSET, 525 .bar4_base = ATOM_SBAR4BASE_OFFSET, 526 527 .bar2_limit = ATOM_SBAR2LMT_OFFSET, 528 .bar4_limit = ATOM_SBAR4LMT_OFFSET, 529#endif 530 531 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 532 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 533}; 534 535static const struct ntb_reg xeon_reg = { 536 .ntb_ctl = XEON_NTBCNTL_OFFSET, 537 .lnk_sta = XEON_LINK_STATUS_OFFSET, 538 .db_size = sizeof(uint16_t), 539 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 540}; 541 542static const struct ntb_alt_reg xeon_pri_reg = { 543 .db_bell = XEON_PDOORBELL_OFFSET, 544 .db_mask = XEON_PDBMSK_OFFSET, 545 .spad = XEON_SPAD_OFFSET, 546}; 547 548static const struct ntb_alt_reg xeon_b2b_reg = { 549 .db_bell = XEON_B2B_DOORBELL_OFFSET, 550 .spad = XEON_B2B_SPAD_OFFSET, 551}; 552 553static const struct ntb_xlat_reg xeon_sec_xlat = { 554 .bar0_base = XEON_SBAR0BASE_OFFSET, 555 .bar2_base = XEON_SBAR2BASE_OFFSET, 556 .bar4_base = XEON_SBAR4BASE_OFFSET, 557 .bar5_base = XEON_SBAR5BASE_OFFSET, 558 559 .bar2_limit = XEON_SBAR2LMT_OFFSET, 560 .bar4_limit = XEON_SBAR4LMT_OFFSET, 561 .bar5_limit = XEON_SBAR5LMT_OFFSET, 562 563 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 564 .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 565 .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 566}; 567 568static struct ntb_b2b_addr xeon_b2b_usd_addr = { 569 .bar0_addr = XEON_B2B_BAR0_ADDR, 570 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 571 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 572 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 573 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 574}; 575 576static struct ntb_b2b_addr xeon_b2b_dsd_addr = { 577 .bar0_addr = XEON_B2B_BAR0_ADDR, 578 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 579 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 580 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 581 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 582}; 583 584SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 585 "B2B MW segment overrides -- MUST be the same on both sides"); 586 587SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 588 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 589 "hardware, use this 64-bit address on the bus between the NTB devices for " 590 "the window at BAR2, on the upstream side of the link. MUST be the same " 591 "address on both sides."); 592SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 593 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 594SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 595 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 596 "(split-BAR mode)."); 597SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 598 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 599 "(split-BAR mode)."); 600 601SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 602 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 603 "hardware, use this 64-bit address on the bus between the NTB devices for " 604 "the window at BAR2, on the downstream side of the link. MUST be the same" 605 " address on both sides."); 606SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 607 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 608SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 609 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 610 "(split-BAR mode)."); 611SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 612 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 613 "(split-BAR mode)."); 614 615/* 616 * OS <-> Driver interface structures 617 */ 618MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 619 620SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 621 622/* 623 * OS <-> Driver linkage functions 624 */ 625static int 626ntb_probe(device_t device) 627{ 628 struct ntb_hw_info *p; 629 630 p = ntb_get_device_info(pci_get_devid(device)); 631 if (p == NULL) 632 return (ENXIO); 633 634 device_set_desc(device, p->desc); 635 return (0); 636} 637 638static int 639ntb_attach(device_t device) 640{ 641 struct ntb_softc *ntb; 642 struct ntb_hw_info *p; 643 int error; 644 645 ntb = device_get_softc(device); 646 p = ntb_get_device_info(pci_get_devid(device)); 647 648 ntb->device = device; 649 ntb->type = p->type; 650 ntb->features = p->features; 651 ntb->b2b_mw_idx = B2B_MW_DISABLED; 652 ntb->msix_mw_idx = B2B_MW_DISABLED; 653 654 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 655 callout_init(&ntb->heartbeat_timer, 1); 656 callout_init(&ntb->lr_timer, 1); 657 callout_init(&ntb->peer_msix_work, 1); 658 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 659 rm_init(&ntb->ctx_lock, "ntb ctx"); 660 661 if (ntb->type == NTB_ATOM) 662 error = ntb_detect_atom(ntb); 663 else 664 error = ntb_detect_xeon(ntb); 665 if (error != 0) 666 goto out; 667 668 ntb_detect_max_mw(ntb); 669 670 pci_enable_busmaster(ntb->device); 671 672 error = ntb_map_pci_bars(ntb); 673 if (error != 0) 674 goto out; 675 if (ntb->type == NTB_ATOM) 676 error = ntb_atom_init_dev(ntb); 677 else 678 error = ntb_xeon_init_dev(ntb); 679 if (error != 0) 680 goto out; 681 682 ntb_spad_clear(device); 683 684 ntb_poll_link(ntb); 685 686 ntb_sysctl_init(ntb); 687 688 /* Attach children to this controller */ 689 device_add_child(device, NULL, -1); 690 bus_generic_attach(device); 691 692out: 693 if (error != 0) 694 ntb_detach(device); 695 return (error); 696} 697 698static int 699ntb_detach(device_t device) 700{ 701 struct ntb_softc *ntb; 702 703 ntb = device_get_softc(device); 704 705 /* Detach & delete all children */ 706 device_delete_children(device); 707 708 if (ntb->self_reg != NULL) { 709 DB_MASK_LOCK(ntb); 710 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 711 DB_MASK_UNLOCK(ntb); 712 } 713 callout_drain(&ntb->heartbeat_timer); 714 callout_drain(&ntb->lr_timer); 715 callout_drain(&ntb->peer_msix_work); 716 pci_disable_busmaster(ntb->device); 717 if (ntb->type == NTB_XEON) 718 ntb_teardown_xeon(ntb); 719 ntb_teardown_interrupts(ntb); 720 721 mtx_destroy(&ntb->db_mask_lock); 722 rm_destroy(&ntb->ctx_lock); 723 724 ntb_unmap_pci_bar(ntb); 725 726 return (0); 727} 728 729/* 730 * Driver internal routines 731 */ 732static inline enum ntb_bar 733ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 734{ 735 736 KASSERT(mw < ntb->mw_count, 737 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 738 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 739 740 return (ntb->reg->mw_bar[mw]); 741} 742 743static inline bool 744bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 745{ 746 /* XXX This assertion could be stronger. */ 747 KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 748 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 749} 750 751static inline void 752bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 753 uint32_t *xlat, uint32_t *lmt) 754{ 755 uint32_t basev, lmtv, xlatv; 756 757 switch (bar) { 758 case NTB_B2B_BAR_1: 759 basev = ntb->xlat_reg->bar2_base; 760 lmtv = ntb->xlat_reg->bar2_limit; 761 xlatv = ntb->xlat_reg->bar2_xlat; 762 break; 763 case NTB_B2B_BAR_2: 764 basev = ntb->xlat_reg->bar4_base; 765 lmtv = ntb->xlat_reg->bar4_limit; 766 xlatv = ntb->xlat_reg->bar4_xlat; 767 break; 768 case NTB_B2B_BAR_3: 769 basev = ntb->xlat_reg->bar5_base; 770 lmtv = ntb->xlat_reg->bar5_limit; 771 xlatv = ntb->xlat_reg->bar5_xlat; 772 break; 773 default: 774 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 775 ("bad bar")); 776 basev = lmtv = xlatv = 0; 777 break; 778 } 779 780 if (base != NULL) 781 *base = basev; 782 if (xlat != NULL) 783 *xlat = xlatv; 784 if (lmt != NULL) 785 *lmt = lmtv; 786} 787 788static int 789ntb_map_pci_bars(struct ntb_softc *ntb) 790{ 791 int rc; 792 793 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 794 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 795 if (rc != 0) 796 goto out; 797 798 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 799 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 800 if (rc != 0) 801 goto out; 802 ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 803 ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 804 ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 805 806 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 807 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 808 if (rc != 0) 809 goto out; 810 ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 811 ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 812 ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 813 814 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 815 goto out; 816 817 ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 818 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 819 ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 820 ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 821 ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 822 823out: 824 if (rc != 0) 825 device_printf(ntb->device, 826 "unable to allocate pci resource\n"); 827 return (rc); 828} 829 830static void 831print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 832 const char *kind) 833{ 834 835 device_printf(ntb->device, 836 "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 837 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 838 (char *)bar->vbase + bar->size - 1, 839 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 840 (uintmax_t)bar->size, kind); 841} 842 843static int 844map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 845{ 846 847 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 848 &bar->pci_resource_id, RF_ACTIVE); 849 if (bar->pci_resource == NULL) 850 return (ENXIO); 851 852 save_bar_parameters(bar); 853 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 854 print_map_success(ntb, bar, "mmr"); 855 return (0); 856} 857 858static int 859map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 860{ 861 int rc; 862 vm_memattr_t mapmode; 863 uint8_t bar_size_bits = 0; 864 865 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 866 &bar->pci_resource_id, RF_ACTIVE); 867 868 if (bar->pci_resource == NULL) 869 return (ENXIO); 870 871 save_bar_parameters(bar); 872 /* 873 * Ivytown NTB BAR sizes are misreported by the hardware due to a 874 * hardware issue. To work around this, query the size it should be 875 * configured to by the device and modify the resource to correspond to 876 * this new size. The BIOS on systems with this problem is required to 877 * provide enough address space to allow the driver to make this change 878 * safely. 879 * 880 * Ideally I could have just specified the size when I allocated the 881 * resource like: 882 * bus_alloc_resource(ntb->device, 883 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 884 * 1ul << bar_size_bits, RF_ACTIVE); 885 * but the PCI driver does not honor the size in this call, so we have 886 * to modify it after the fact. 887 */ 888 if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 889 if (bar->pci_resource_id == PCIR_BAR(2)) 890 bar_size_bits = pci_read_config(ntb->device, 891 XEON_PBAR23SZ_OFFSET, 1); 892 else 893 bar_size_bits = pci_read_config(ntb->device, 894 XEON_PBAR45SZ_OFFSET, 1); 895 896 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 897 bar->pci_resource, bar->pbase, 898 bar->pbase + (1ul << bar_size_bits) - 1); 899 if (rc != 0) { 900 device_printf(ntb->device, 901 "unable to resize bar\n"); 902 return (rc); 903 } 904 905 save_bar_parameters(bar); 906 } 907 908 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 909 print_map_success(ntb, bar, "mw"); 910 911 /* 912 * Optionally, mark MW BARs as anything other than UC to improve 913 * performance. 914 */ 915 mapmode = ntb_pat_flags(); 916 if (mapmode == bar->map_mode) 917 return (0); 918 919 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 920 if (rc == 0) { 921 bar->map_mode = mapmode; 922 device_printf(ntb->device, 923 "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 924 "%s.\n", 925 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 926 (char *)bar->vbase + bar->size - 1, 927 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 928 ntb_vm_memattr_to_str(mapmode)); 929 } else 930 device_printf(ntb->device, 931 "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 932 "%s: %d\n", 933 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 934 (char *)bar->vbase + bar->size - 1, 935 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 936 ntb_vm_memattr_to_str(mapmode), rc); 937 /* Proceed anyway */ 938 return (0); 939} 940 941static void 942ntb_unmap_pci_bar(struct ntb_softc *ntb) 943{ 944 struct ntb_pci_bar_info *current_bar; 945 int i; 946 947 for (i = 0; i < NTB_MAX_BARS; i++) { 948 current_bar = &ntb->bar_info[i]; 949 if (current_bar->pci_resource != NULL) 950 bus_release_resource(ntb->device, SYS_RES_MEMORY, 951 current_bar->pci_resource_id, 952 current_bar->pci_resource); 953 } 954} 955 956static int 957ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 958{ 959 uint32_t i; 960 int rc; 961 962 for (i = 0; i < num_vectors; i++) { 963 ntb->int_info[i].rid = i + 1; 964 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 965 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 966 if (ntb->int_info[i].res == NULL) { 967 device_printf(ntb->device, 968 "bus_alloc_resource failed\n"); 969 return (ENOMEM); 970 } 971 ntb->int_info[i].tag = NULL; 972 ntb->allocated_interrupts++; 973 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 974 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 975 &ntb->msix_vec[i], &ntb->int_info[i].tag); 976 if (rc != 0) { 977 device_printf(ntb->device, "bus_setup_intr failed\n"); 978 return (ENXIO); 979 } 980 } 981 return (0); 982} 983 984/* 985 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 986 * cannot be allocated for each MSI-X message. JHB seems to think remapping 987 * should be okay. This tunable should enable us to test that hypothesis 988 * when someone gets their hands on some Xeon hardware. 989 */ 990static int ntb_force_remap_mode; 991SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 992 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 993 " to a smaller number of ithreads, even if the desired number are " 994 "available"); 995 996/* 997 * In case it is NOT ok, give consumers an abort button. 998 */ 999static int ntb_prefer_intx; 1000SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 1001 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1002 "than remapping MSI-X messages over available slots (match Linux driver " 1003 "behavior)"); 1004 1005/* 1006 * Remap the desired number of MSI-X messages to available ithreads in a simple 1007 * round-robin fashion. 1008 */ 1009static int 1010ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1011{ 1012 u_int *vectors; 1013 uint32_t i; 1014 int rc; 1015 1016 if (ntb_prefer_intx != 0) 1017 return (ENXIO); 1018 1019 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1020 1021 for (i = 0; i < desired; i++) 1022 vectors[i] = (i % avail) + 1; 1023 1024 rc = pci_remap_msix(dev, desired, vectors); 1025 free(vectors, M_NTB); 1026 return (rc); 1027} 1028 1029static int 1030ntb_init_isr(struct ntb_softc *ntb) 1031{ 1032 uint32_t desired_vectors, num_vectors; 1033 int rc; 1034 1035 ntb->allocated_interrupts = 0; 1036 ntb->last_ts = ticks; 1037 1038 /* 1039 * Mask all doorbell interrupts. (Except link events!) 1040 */ 1041 DB_MASK_LOCK(ntb); 1042 ntb->db_mask = ntb->db_valid_mask; 1043 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1044 DB_MASK_UNLOCK(ntb); 1045 1046 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1047 ntb->db_count); 1048 if (desired_vectors >= 1) { 1049 rc = pci_alloc_msix(ntb->device, &num_vectors); 1050 1051 if (ntb_force_remap_mode != 0 && rc == 0 && 1052 num_vectors == desired_vectors) 1053 num_vectors--; 1054 1055 if (rc == 0 && num_vectors < desired_vectors) { 1056 rc = ntb_remap_msix(ntb->device, desired_vectors, 1057 num_vectors); 1058 if (rc == 0) 1059 num_vectors = desired_vectors; 1060 else 1061 pci_release_msi(ntb->device); 1062 } 1063 if (rc != 0) 1064 num_vectors = 1; 1065 } else 1066 num_vectors = 1; 1067 1068 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1069 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1070 device_printf(ntb->device, 1071 "Errata workaround does not support MSI or INTX\n"); 1072 return (EINVAL); 1073 } 1074 1075 ntb->db_vec_count = 1; 1076 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1077 rc = ntb_setup_legacy_interrupt(ntb); 1078 } else { 1079 if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1080 HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1081 device_printf(ntb->device, 1082 "Errata workaround expects %d doorbell bits\n", 1083 XEON_NONLINK_DB_MSIX_BITS); 1084 return (EINVAL); 1085 } 1086 1087 ntb_create_msix_vec(ntb, num_vectors); 1088 rc = ntb_setup_msix(ntb, num_vectors); 1089 if (rc == 0 && HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1090 ntb_get_msix_info(ntb); 1091 } 1092 if (rc != 0) { 1093 device_printf(ntb->device, 1094 "Error allocating interrupts: %d\n", rc); 1095 ntb_free_msix_vec(ntb); 1096 } 1097 1098 return (rc); 1099} 1100 1101static int 1102ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1103{ 1104 int rc; 1105 1106 ntb->int_info[0].rid = 0; 1107 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1108 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1109 if (ntb->int_info[0].res == NULL) { 1110 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1111 return (ENOMEM); 1112 } 1113 1114 ntb->int_info[0].tag = NULL; 1115 ntb->allocated_interrupts = 1; 1116 1117 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1118 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1119 ntb, &ntb->int_info[0].tag); 1120 if (rc != 0) { 1121 device_printf(ntb->device, "bus_setup_intr failed\n"); 1122 return (ENXIO); 1123 } 1124 1125 return (0); 1126} 1127 1128static void 1129ntb_teardown_interrupts(struct ntb_softc *ntb) 1130{ 1131 struct ntb_int_info *current_int; 1132 int i; 1133 1134 for (i = 0; i < ntb->allocated_interrupts; i++) { 1135 current_int = &ntb->int_info[i]; 1136 if (current_int->tag != NULL) 1137 bus_teardown_intr(ntb->device, current_int->res, 1138 current_int->tag); 1139 1140 if (current_int->res != NULL) 1141 bus_release_resource(ntb->device, SYS_RES_IRQ, 1142 rman_get_rid(current_int->res), current_int->res); 1143 } 1144 1145 ntb_free_msix_vec(ntb); 1146 pci_release_msi(ntb->device); 1147} 1148 1149/* 1150 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1151 * out to make code clearer. 1152 */ 1153static inline uint64_t 1154db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1155{ 1156 1157 if (ntb->type == NTB_ATOM) 1158 return (ntb_reg_read(8, regoff)); 1159 1160 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1161 1162 return (ntb_reg_read(2, regoff)); 1163} 1164 1165static inline void 1166db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1167{ 1168 1169 KASSERT((val & ~ntb->db_valid_mask) == 0, 1170 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1171 (uintmax_t)(val & ~ntb->db_valid_mask), 1172 (uintmax_t)ntb->db_valid_mask)); 1173 1174 if (regoff == ntb->self_reg->db_mask) 1175 DB_MASK_ASSERT(ntb, MA_OWNED); 1176 db_iowrite_raw(ntb, regoff, val); 1177} 1178 1179static inline void 1180db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1181{ 1182 1183 if (ntb->type == NTB_ATOM) { 1184 ntb_reg_write(8, regoff, val); 1185 return; 1186 } 1187 1188 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1189 ntb_reg_write(2, regoff, (uint16_t)val); 1190} 1191 1192static void 1193ntb_db_set_mask(device_t dev, uint64_t bits) 1194{ 1195 struct ntb_softc *ntb = device_get_softc(dev); 1196 1197 DB_MASK_LOCK(ntb); 1198 ntb->db_mask |= bits; 1199 if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1200 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1201 DB_MASK_UNLOCK(ntb); 1202} 1203 1204static void 1205ntb_db_clear_mask(device_t dev, uint64_t bits) 1206{ 1207 struct ntb_softc *ntb = device_get_softc(dev); 1208 uint64_t ibits; 1209 int i; 1210 1211 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1212 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1213 (uintmax_t)(bits & ~ntb->db_valid_mask), 1214 (uintmax_t)ntb->db_valid_mask)); 1215 1216 DB_MASK_LOCK(ntb); 1217 ibits = ntb->fake_db_bell & ntb->db_mask & bits; 1218 ntb->db_mask &= ~bits; 1219 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1220 /* Simulate fake interrupts if unmasked DB bits are set. */ 1221 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1222 if ((ibits & ntb_db_vector_mask(dev, i)) != 0) 1223 swi_sched(ntb->int_info[i].tag, 0); 1224 } 1225 } else { 1226 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1227 } 1228 DB_MASK_UNLOCK(ntb); 1229} 1230 1231static uint64_t 1232ntb_db_read(device_t dev) 1233{ 1234 struct ntb_softc *ntb = device_get_softc(dev); 1235 1236 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1237 return (ntb->fake_db_bell); 1238 1239 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1240} 1241 1242static void 1243ntb_db_clear(device_t dev, uint64_t bits) 1244{ 1245 struct ntb_softc *ntb = device_get_softc(dev); 1246 1247 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1248 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1249 (uintmax_t)(bits & ~ntb->db_valid_mask), 1250 (uintmax_t)ntb->db_valid_mask)); 1251 1252 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1253 DB_MASK_LOCK(ntb); 1254 ntb->fake_db_bell &= ~bits; 1255 DB_MASK_UNLOCK(ntb); 1256 return; 1257 } 1258 1259 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1260} 1261 1262static inline uint64_t 1263ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1264{ 1265 uint64_t shift, mask; 1266 1267 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1268 /* 1269 * Remap vectors in custom way to make at least first 1270 * three doorbells to not generate stray events. 1271 * This breaks Linux compatibility (if one existed) 1272 * when more then one DB is used (not by if_ntb). 1273 */ 1274 if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1) 1275 return (1 << db_vector); 1276 if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1) 1277 return (0x7ffc); 1278 } 1279 1280 shift = ntb->db_vec_shift; 1281 mask = (1ull << shift) - 1; 1282 return (mask << (shift * db_vector)); 1283} 1284 1285static void 1286ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1287{ 1288 uint64_t vec_mask; 1289 1290 ntb->last_ts = ticks; 1291 vec_mask = ntb_vec_mask(ntb, vec); 1292 1293 if ((vec_mask & ntb->db_link_mask) != 0) { 1294 if (ntb_poll_link(ntb)) 1295 ntb_link_event(ntb->device); 1296 } 1297 1298 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1299 (vec_mask & ntb->db_link_mask) == 0) { 1300 DB_MASK_LOCK(ntb); 1301 1302 /* Do not report same DB events again if not cleared yet. */ 1303 vec_mask &= ~ntb->fake_db_bell; 1304 1305 /* Update our internal doorbell register. */ 1306 ntb->fake_db_bell |= vec_mask; 1307 1308 /* Do not report masked DB events. */ 1309 vec_mask &= ~ntb->db_mask; 1310 1311 DB_MASK_UNLOCK(ntb); 1312 } 1313 1314 if ((vec_mask & ntb->db_valid_mask) != 0) 1315 ntb_db_event(ntb->device, vec); 1316} 1317 1318static void 1319ndev_vec_isr(void *arg) 1320{ 1321 struct ntb_vec *nvec = arg; 1322 1323 ntb_interrupt(nvec->ntb, nvec->num); 1324} 1325 1326static void 1327ndev_irq_isr(void *arg) 1328{ 1329 /* If we couldn't set up MSI-X, we only have the one vector. */ 1330 ntb_interrupt(arg, 0); 1331} 1332 1333static int 1334ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1335{ 1336 uint32_t i; 1337 1338 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1339 M_ZERO | M_WAITOK); 1340 for (i = 0; i < num_vectors; i++) { 1341 ntb->msix_vec[i].num = i; 1342 ntb->msix_vec[i].ntb = ntb; 1343 } 1344 1345 return (0); 1346} 1347 1348static void 1349ntb_free_msix_vec(struct ntb_softc *ntb) 1350{ 1351 1352 if (ntb->msix_vec == NULL) 1353 return; 1354 1355 free(ntb->msix_vec, M_NTB); 1356 ntb->msix_vec = NULL; 1357} 1358 1359static void 1360ntb_get_msix_info(struct ntb_softc *ntb) 1361{ 1362 struct pci_devinfo *dinfo; 1363 struct pcicfg_msix *msix; 1364 uint32_t laddr, data, i, offset; 1365 1366 dinfo = device_get_ivars(ntb->device); 1367 msix = &dinfo->cfg.msix; 1368 1369 CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1370 1371 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1372 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1373 1374 laddr = bus_read_4(msix->msix_table_res, offset + 1375 PCI_MSIX_ENTRY_LOWER_ADDR); 1376 ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1377 1378 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1379 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1380 MSI_INTEL_ADDR_BASE)); 1381 ntb->msix_data[i].nmd_ofs = laddr; 1382 1383 data = bus_read_4(msix->msix_table_res, offset + 1384 PCI_MSIX_ENTRY_DATA); 1385 ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1386 1387 ntb->msix_data[i].nmd_data = data; 1388 } 1389} 1390 1391static struct ntb_hw_info * 1392ntb_get_device_info(uint32_t device_id) 1393{ 1394 struct ntb_hw_info *ep = pci_ids; 1395 1396 while (ep->device_id) { 1397 if (ep->device_id == device_id) 1398 return (ep); 1399 ++ep; 1400 } 1401 return (NULL); 1402} 1403 1404static void 1405ntb_teardown_xeon(struct ntb_softc *ntb) 1406{ 1407 1408 if (ntb->reg != NULL) 1409 ntb_link_disable(ntb->device); 1410} 1411 1412static void 1413ntb_detect_max_mw(struct ntb_softc *ntb) 1414{ 1415 1416 if (ntb->type == NTB_ATOM) { 1417 ntb->mw_count = ATOM_MW_COUNT; 1418 return; 1419 } 1420 1421 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1422 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1423 else 1424 ntb->mw_count = XEON_SNB_MW_COUNT; 1425} 1426 1427static int 1428ntb_detect_xeon(struct ntb_softc *ntb) 1429{ 1430 uint8_t ppd, conn_type; 1431 1432 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1433 ntb->ppd = ppd; 1434 1435 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1436 ntb->dev_type = NTB_DEV_DSD; 1437 else 1438 ntb->dev_type = NTB_DEV_USD; 1439 1440 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1441 ntb->features |= NTB_SPLIT_BAR; 1442 1443 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1444 !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1445 device_printf(ntb->device, 1446 "Can not apply SB01BASE_LOCKUP workaround " 1447 "with split BARs disabled!\n"); 1448 device_printf(ntb->device, 1449 "Expect system hangs under heavy NTB traffic!\n"); 1450 ntb->features &= ~NTB_SB01BASE_LOCKUP; 1451 } 1452 1453 /* 1454 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1455 * errata workaround; only do one at a time. 1456 */ 1457 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1458 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1459 1460 conn_type = ppd & XEON_PPD_CONN_TYPE; 1461 switch (conn_type) { 1462 case NTB_CONN_B2B: 1463 ntb->conn_type = conn_type; 1464 break; 1465 case NTB_CONN_RP: 1466 case NTB_CONN_TRANSPARENT: 1467 default: 1468 device_printf(ntb->device, "Unsupported connection type: %u\n", 1469 (unsigned)conn_type); 1470 return (ENXIO); 1471 } 1472 return (0); 1473} 1474 1475static int 1476ntb_detect_atom(struct ntb_softc *ntb) 1477{ 1478 uint32_t ppd, conn_type; 1479 1480 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1481 ntb->ppd = ppd; 1482 1483 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1484 ntb->dev_type = NTB_DEV_DSD; 1485 else 1486 ntb->dev_type = NTB_DEV_USD; 1487 1488 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1489 switch (conn_type) { 1490 case NTB_CONN_B2B: 1491 ntb->conn_type = conn_type; 1492 break; 1493 default: 1494 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1495 return (ENXIO); 1496 } 1497 return (0); 1498} 1499 1500static int 1501ntb_xeon_init_dev(struct ntb_softc *ntb) 1502{ 1503 int rc; 1504 1505 ntb->spad_count = XEON_SPAD_COUNT; 1506 ntb->db_count = XEON_DB_COUNT; 1507 ntb->db_link_mask = XEON_DB_LINK_BIT; 1508 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1509 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1510 1511 if (ntb->conn_type != NTB_CONN_B2B) { 1512 device_printf(ntb->device, "Connection type %d not supported\n", 1513 ntb->conn_type); 1514 return (ENXIO); 1515 } 1516 1517 ntb->reg = &xeon_reg; 1518 ntb->self_reg = &xeon_pri_reg; 1519 ntb->peer_reg = &xeon_b2b_reg; 1520 ntb->xlat_reg = &xeon_sec_xlat; 1521 1522 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1523 ntb->fake_db_bell = 0; 1524 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1525 ntb->mw_count; 1526 ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1527 g_ntb_msix_idx, ntb->msix_mw_idx); 1528 rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1529 VM_MEMATTR_UNCACHEABLE); 1530 KASSERT(rc == 0, ("shouldn't fail")); 1531 } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1532 /* 1533 * There is a Xeon hardware errata related to writes to SDOORBELL or 1534 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1535 * which may hang the system. To workaround this, use a memory 1536 * window to access the interrupt and scratch pad registers on the 1537 * remote system. 1538 */ 1539 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1540 ntb->mw_count; 1541 ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1542 g_ntb_mw_idx, ntb->b2b_mw_idx); 1543 rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1544 VM_MEMATTR_UNCACHEABLE); 1545 KASSERT(rc == 0, ("shouldn't fail")); 1546 } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1547 /* 1548 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1549 * mirrored to the remote system. Shrink the number of bits by one, 1550 * since bit 14 is the last bit. 1551 * 1552 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1553 * anyway. Nor for non-B2B connection types. 1554 */ 1555 ntb->db_count = XEON_DB_COUNT - 1; 1556 1557 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1558 1559 if (ntb->dev_type == NTB_DEV_USD) 1560 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1561 &xeon_b2b_usd_addr); 1562 else 1563 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1564 &xeon_b2b_dsd_addr); 1565 if (rc != 0) 1566 return (rc); 1567 1568 /* Enable Bus Master and Memory Space on the secondary side */ 1569 ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1570 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1571 1572 /* 1573 * Mask all doorbell interrupts. 1574 */ 1575 DB_MASK_LOCK(ntb); 1576 ntb->db_mask = ntb->db_valid_mask; 1577 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1578 DB_MASK_UNLOCK(ntb); 1579 1580 rc = ntb_init_isr(ntb); 1581 return (rc); 1582} 1583 1584static int 1585ntb_atom_init_dev(struct ntb_softc *ntb) 1586{ 1587 int error; 1588 1589 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1590 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1591 1592 ntb->spad_count = ATOM_SPAD_COUNT; 1593 ntb->db_count = ATOM_DB_COUNT; 1594 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1595 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1596 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1597 1598 ntb->reg = &atom_reg; 1599 ntb->self_reg = &atom_pri_reg; 1600 ntb->peer_reg = &atom_b2b_reg; 1601 ntb->xlat_reg = &atom_sec_xlat; 1602 1603 /* 1604 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1605 * resolved. Mask transaction layer internal parity errors. 1606 */ 1607 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1608 1609 configure_atom_secondary_side_bars(ntb); 1610 1611 /* Enable Bus Master and Memory Space on the secondary side */ 1612 ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1613 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1614 1615 error = ntb_init_isr(ntb); 1616 if (error != 0) 1617 return (error); 1618 1619 /* Initiate PCI-E link training */ 1620 ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1621 1622 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1623 1624 return (0); 1625} 1626 1627/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1628static void 1629configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1630{ 1631 1632 if (ntb->dev_type == NTB_DEV_USD) { 1633 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1634 XEON_B2B_BAR2_ADDR64); 1635 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1636 XEON_B2B_BAR4_ADDR64); 1637 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1638 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1639 } else { 1640 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1641 XEON_B2B_BAR2_ADDR64); 1642 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1643 XEON_B2B_BAR4_ADDR64); 1644 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1645 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1646 } 1647} 1648 1649 1650/* 1651 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1652 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1653 * remains for use by a higher layer. 1654 * 1655 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1656 * MW size is sufficiently large. 1657 */ 1658static unsigned int ntb_b2b_mw_share; 1659SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1660 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1661 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1662 "value here."); 1663 1664static void 1665xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1666 enum ntb_bar regbar) 1667{ 1668 struct ntb_pci_bar_info *bar; 1669 uint8_t bar_sz; 1670 1671 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1672 return; 1673 1674 bar = &ntb->bar_info[idx]; 1675 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1676 if (idx == regbar) { 1677 if (ntb->b2b_off != 0) 1678 bar_sz--; 1679 else 1680 bar_sz = 0; 1681 } 1682 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1683 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1684 (void)bar_sz; 1685} 1686 1687static void 1688xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1689 enum ntb_bar idx, enum ntb_bar regbar) 1690{ 1691 uint64_t reg_val; 1692 uint32_t base_reg, lmt_reg; 1693 1694 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1695 if (idx == regbar) { 1696 if (ntb->b2b_off) 1697 bar_addr += ntb->b2b_off; 1698 else 1699 bar_addr = 0; 1700 } 1701 1702 /* 1703 * Set limit registers first to avoid an errata where setting the base 1704 * registers locks the limit registers. 1705 */ 1706 if (!bar_is_64bit(ntb, idx)) { 1707 ntb_reg_write(4, lmt_reg, bar_addr); 1708 reg_val = ntb_reg_read(4, lmt_reg); 1709 (void)reg_val; 1710 1711 ntb_reg_write(4, base_reg, bar_addr); 1712 reg_val = ntb_reg_read(4, base_reg); 1713 (void)reg_val; 1714 } else { 1715 ntb_reg_write(8, lmt_reg, bar_addr); 1716 reg_val = ntb_reg_read(8, lmt_reg); 1717 (void)reg_val; 1718 1719 ntb_reg_write(8, base_reg, bar_addr); 1720 reg_val = ntb_reg_read(8, base_reg); 1721 (void)reg_val; 1722 } 1723} 1724 1725static void 1726xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1727{ 1728 struct ntb_pci_bar_info *bar; 1729 1730 bar = &ntb->bar_info[idx]; 1731 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1732 ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1733 base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1734 } else { 1735 ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1736 base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1737 } 1738 (void)base_addr; 1739} 1740 1741static int 1742xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1743 const struct ntb_b2b_addr *peer_addr) 1744{ 1745 struct ntb_pci_bar_info *b2b_bar; 1746 vm_size_t bar_size; 1747 uint64_t bar_addr; 1748 enum ntb_bar b2b_bar_num, i; 1749 1750 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1751 b2b_bar = NULL; 1752 b2b_bar_num = NTB_CONFIG_BAR; 1753 ntb->b2b_off = 0; 1754 } else { 1755 b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1756 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1757 ("invalid b2b mw bar")); 1758 1759 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1760 bar_size = b2b_bar->size; 1761 1762 if (ntb_b2b_mw_share != 0 && 1763 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1764 ntb->b2b_off = bar_size >> 1; 1765 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1766 ntb->b2b_off = 0; 1767 } else { 1768 device_printf(ntb->device, 1769 "B2B bar size is too small!\n"); 1770 return (EIO); 1771 } 1772 } 1773 1774 /* 1775 * Reset the secondary bar sizes to match the primary bar sizes. 1776 * (Except, disable or halve the size of the B2B secondary bar.) 1777 */ 1778 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1779 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1780 1781 bar_addr = 0; 1782 if (b2b_bar_num == NTB_CONFIG_BAR) 1783 bar_addr = addr->bar0_addr; 1784 else if (b2b_bar_num == NTB_B2B_BAR_1) 1785 bar_addr = addr->bar2_addr64; 1786 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1787 bar_addr = addr->bar4_addr64; 1788 else if (b2b_bar_num == NTB_B2B_BAR_2) 1789 bar_addr = addr->bar4_addr32; 1790 else if (b2b_bar_num == NTB_B2B_BAR_3) 1791 bar_addr = addr->bar5_addr32; 1792 else 1793 KASSERT(false, ("invalid bar")); 1794 1795 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1796 1797 /* 1798 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1799 * register BAR. The B2B BAR is either disabled above or configured 1800 * half-size. It starts at PBAR xlat + offset. 1801 * 1802 * Also set up incoming BAR limits == base (zero length window). 1803 */ 1804 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1805 b2b_bar_num); 1806 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1807 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1808 NTB_B2B_BAR_2, b2b_bar_num); 1809 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1810 NTB_B2B_BAR_3, b2b_bar_num); 1811 } else 1812 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1813 NTB_B2B_BAR_2, b2b_bar_num); 1814 1815 /* Zero incoming translation addrs */ 1816 ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1817 ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1818 1819 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1820 size_t size, xlatoffset; 1821 enum ntb_bar bar_num; 1822 1823 bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1824 switch (bar_num) { 1825 case NTB_B2B_BAR_1: 1826 size = 8; 1827 xlatoffset = XEON_SBAR2XLAT_OFFSET; 1828 break; 1829 case NTB_B2B_BAR_2: 1830 xlatoffset = XEON_SBAR4XLAT_OFFSET; 1831 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1832 size = 4; 1833 else 1834 size = 8; 1835 break; 1836 case NTB_B2B_BAR_3: 1837 xlatoffset = XEON_SBAR5XLAT_OFFSET; 1838 size = 4; 1839 break; 1840 default: 1841 KASSERT(false, ("Bogus msix mw idx: %u", 1842 ntb->msix_mw_idx)); 1843 return (EINVAL); 1844 } 1845 1846 /* 1847 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1848 * workaround 1849 */ 1850 if (size == 4) { 1851 ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1852 ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1853 } else { 1854 ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1855 ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1856 } 1857 1858 ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1859 } 1860 (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1861 (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1862 1863 /* Zero outgoing translation limits (whole bar size windows) */ 1864 ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1865 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1866 1867 /* Set outgoing translation offsets */ 1868 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1869 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1870 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1871 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1872 } else 1873 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1874 1875 /* Set the translation offset for B2B registers */ 1876 bar_addr = 0; 1877 if (b2b_bar_num == NTB_CONFIG_BAR) 1878 bar_addr = peer_addr->bar0_addr; 1879 else if (b2b_bar_num == NTB_B2B_BAR_1) 1880 bar_addr = peer_addr->bar2_addr64; 1881 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1882 bar_addr = peer_addr->bar4_addr64; 1883 else if (b2b_bar_num == NTB_B2B_BAR_2) 1884 bar_addr = peer_addr->bar4_addr32; 1885 else if (b2b_bar_num == NTB_B2B_BAR_3) 1886 bar_addr = peer_addr->bar5_addr32; 1887 else 1888 KASSERT(false, ("invalid bar")); 1889 1890 /* 1891 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1892 * at a time. 1893 */ 1894 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1895 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1896 return (0); 1897} 1898 1899static inline bool 1900_xeon_link_is_up(struct ntb_softc *ntb) 1901{ 1902 1903 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1904 return (true); 1905 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1906} 1907 1908static inline bool 1909link_is_up(struct ntb_softc *ntb) 1910{ 1911 1912 if (ntb->type == NTB_XEON) 1913 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1914 !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 1915 1916 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1917 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1918} 1919 1920static inline bool 1921atom_link_is_err(struct ntb_softc *ntb) 1922{ 1923 uint32_t status; 1924 1925 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1926 1927 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1928 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1929 return (true); 1930 1931 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1932 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1933} 1934 1935/* Atom does not have link status interrupt, poll on that platform */ 1936static void 1937atom_link_hb(void *arg) 1938{ 1939 struct ntb_softc *ntb = arg; 1940 sbintime_t timo, poll_ts; 1941 1942 timo = NTB_HB_TIMEOUT * hz; 1943 poll_ts = ntb->last_ts + timo; 1944 1945 /* 1946 * Delay polling the link status if an interrupt was received, unless 1947 * the cached link status says the link is down. 1948 */ 1949 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1950 timo = poll_ts - ticks; 1951 goto out; 1952 } 1953 1954 if (ntb_poll_link(ntb)) 1955 ntb_link_event(ntb->device); 1956 1957 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1958 /* Link is down with error, proceed with recovery */ 1959 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1960 return; 1961 } 1962 1963out: 1964 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1965} 1966 1967static void 1968atom_perform_link_restart(struct ntb_softc *ntb) 1969{ 1970 uint32_t status; 1971 1972 /* Driver resets the NTB ModPhy lanes - magic! */ 1973 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1974 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1975 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1976 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1977 1978 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1979 pause("ModPhy", hz / 10); 1980 1981 /* Clear AER Errors, write to clear */ 1982 status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1983 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1984 ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1985 1986 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1987 status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1988 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1989 ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1990 1991 /* Clear DeSkew Buffer error, write to clear */ 1992 status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1993 status |= ATOM_DESKEWSTS_DBERR; 1994 ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1995 1996 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1997 status &= ATOM_IBIST_ERR_OFLOW; 1998 ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1999 2000 /* Releases the NTB state machine to allow the link to retrain */ 2001 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2002 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2003 ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2004} 2005 2006static int 2007ntb_set_ctx(device_t dev, void *ctx, const struct ntb_ctx_ops *ops) 2008{ 2009 struct ntb_softc *ntb = device_get_softc(dev); 2010 2011 if (ctx == NULL || ops == NULL) 2012 return (EINVAL); 2013 2014 rm_wlock(&ntb->ctx_lock); 2015 if (ntb->ctx_ops != NULL) { 2016 rm_wunlock(&ntb->ctx_lock); 2017 return (EINVAL); 2018 } 2019 ntb->ntb_ctx = ctx; 2020 ntb->ctx_ops = ops; 2021 rm_wunlock(&ntb->ctx_lock); 2022 2023 return (0); 2024} 2025 2026/* 2027 * It is expected that this will only be used from contexts where the ctx_lock 2028 * is not needed to protect ntb_ctx lifetime. 2029 */ 2030static void * 2031ntb_get_ctx(device_t dev, const struct ntb_ctx_ops **ops) 2032{ 2033 struct ntb_softc *ntb = device_get_softc(dev); 2034 2035 KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2036 if (ops != NULL) 2037 *ops = ntb->ctx_ops; 2038 return (ntb->ntb_ctx); 2039} 2040 2041static void 2042ntb_clear_ctx(device_t dev) 2043{ 2044 struct ntb_softc *ntb = device_get_softc(dev); 2045 2046 rm_wlock(&ntb->ctx_lock); 2047 ntb->ntb_ctx = NULL; 2048 ntb->ctx_ops = NULL; 2049 rm_wunlock(&ntb->ctx_lock); 2050} 2051 2052/* 2053 * ntb_link_event() - notify driver context of a change in link status 2054 * @ntb: NTB device context 2055 * 2056 * Notify the driver context that the link status may have changed. The driver 2057 * should call ntb_link_is_up() to get the current status. 2058 */ 2059static void 2060ntb_link_event(device_t dev) 2061{ 2062 struct ntb_softc *ntb = device_get_softc(dev); 2063 struct rm_priotracker ctx_tracker; 2064 2065 rm_rlock(&ntb->ctx_lock, &ctx_tracker); 2066 if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2067 ntb->ctx_ops->link_event(ntb->ntb_ctx); 2068 rm_runlock(&ntb->ctx_lock, &ctx_tracker); 2069} 2070 2071/* 2072 * ntb_db_event() - notify driver context of a doorbell event 2073 * @ntb: NTB device context 2074 * @vector: Interrupt vector number 2075 * 2076 * Notify the driver context of a doorbell event. If hardware supports 2077 * multiple interrupt vectors for doorbells, the vector number indicates which 2078 * vector received the interrupt. The vector number is relative to the first 2079 * vector used for doorbells, starting at zero, and must be less than 2080 * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2081 * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2082 * those bits are associated with the vector number. 2083 */ 2084static void 2085ntb_db_event(device_t dev, uint32_t vec) 2086{ 2087 struct ntb_softc *ntb = device_get_softc(dev); 2088 struct rm_priotracker ctx_tracker; 2089 2090 rm_rlock(&ntb->ctx_lock, &ctx_tracker); 2091 if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2092 ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2093 rm_runlock(&ntb->ctx_lock, &ctx_tracker); 2094} 2095 2096static int 2097ntb_link_enable(device_t dev, enum ntb_speed speed __unused, 2098 enum ntb_width width __unused) 2099{ 2100 struct ntb_softc *ntb = device_get_softc(dev); 2101 uint32_t cntl; 2102 2103 ntb_printf(2, "%s\n", __func__); 2104 2105 if (ntb->type == NTB_ATOM) { 2106 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2107 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2108 return (0); 2109 } 2110 2111 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2112 ntb_link_event(dev); 2113 return (0); 2114 } 2115 2116 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2117 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2118 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2119 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2120 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2121 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2122 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2123 return (0); 2124} 2125 2126static int 2127ntb_link_disable(device_t dev) 2128{ 2129 struct ntb_softc *ntb = device_get_softc(dev); 2130 uint32_t cntl; 2131 2132 ntb_printf(2, "%s\n", __func__); 2133 2134 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2135 ntb_link_event(dev); 2136 return (0); 2137 } 2138 2139 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2140 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2141 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2142 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2143 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2144 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2145 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2146 return (0); 2147} 2148 2149static bool 2150ntb_link_enabled(device_t dev) 2151{ 2152 struct ntb_softc *ntb = device_get_softc(dev); 2153 uint32_t cntl; 2154 2155 if (ntb->type == NTB_ATOM) { 2156 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2157 return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2158 } 2159 2160 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2161 return (true); 2162 2163 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2164 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2165} 2166 2167static void 2168recover_atom_link(void *arg) 2169{ 2170 struct ntb_softc *ntb = arg; 2171 unsigned speed, width, oldspeed, oldwidth; 2172 uint32_t status32; 2173 2174 atom_perform_link_restart(ntb); 2175 2176 /* 2177 * There is a potential race between the 2 NTB devices recovering at 2178 * the same time. If the times are the same, the link will not recover 2179 * and the driver will be stuck in this loop forever. Add a random 2180 * interval to the recovery time to prevent this race. 2181 */ 2182 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2183 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2184 2185 if (atom_link_is_err(ntb)) 2186 goto retry; 2187 2188 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2189 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2190 goto out; 2191 2192 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2193 width = NTB_LNK_STA_WIDTH(status32); 2194 speed = status32 & NTB_LINK_SPEED_MASK; 2195 2196 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2197 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2198 if (oldwidth != width || oldspeed != speed) 2199 goto retry; 2200 2201out: 2202 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2203 ntb); 2204 return; 2205 2206retry: 2207 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2208 ntb); 2209} 2210 2211/* 2212 * Polls the HW link status register(s); returns true if something has changed. 2213 */ 2214static bool 2215ntb_poll_link(struct ntb_softc *ntb) 2216{ 2217 uint32_t ntb_cntl; 2218 uint16_t reg_val; 2219 2220 if (ntb->type == NTB_ATOM) { 2221 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2222 if (ntb_cntl == ntb->ntb_ctl) 2223 return (false); 2224 2225 ntb->ntb_ctl = ntb_cntl; 2226 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2227 } else { 2228 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2229 2230 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2231 if (reg_val == ntb->lnk_sta) 2232 return (false); 2233 2234 ntb->lnk_sta = reg_val; 2235 2236 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2237 if (_xeon_link_is_up(ntb)) { 2238 if (!ntb->peer_msix_good) { 2239 callout_reset(&ntb->peer_msix_work, 0, 2240 ntb_exchange_msix, ntb); 2241 return (false); 2242 } 2243 } else { 2244 ntb->peer_msix_good = false; 2245 ntb->peer_msix_done = false; 2246 } 2247 } 2248 } 2249 return (true); 2250} 2251 2252static inline enum ntb_speed 2253ntb_link_sta_speed(struct ntb_softc *ntb) 2254{ 2255 2256 if (!link_is_up(ntb)) 2257 return (NTB_SPEED_NONE); 2258 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2259} 2260 2261static inline enum ntb_width 2262ntb_link_sta_width(struct ntb_softc *ntb) 2263{ 2264 2265 if (!link_is_up(ntb)) 2266 return (NTB_WIDTH_NONE); 2267 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2268} 2269 2270SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2271 "Driver state, statistics, and HW registers"); 2272 2273#define NTB_REGSZ_MASK (3ul << 30) 2274#define NTB_REG_64 (1ul << 30) 2275#define NTB_REG_32 (2ul << 30) 2276#define NTB_REG_16 (3ul << 30) 2277#define NTB_REG_8 (0ul << 30) 2278 2279#define NTB_DB_READ (1ul << 29) 2280#define NTB_PCI_REG (1ul << 28) 2281#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2282 2283static void 2284ntb_sysctl_init(struct ntb_softc *ntb) 2285{ 2286 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2287 struct sysctl_ctx_list *ctx; 2288 struct sysctl_oid *tree, *tmptree; 2289 2290 ctx = device_get_sysctl_ctx(ntb->device); 2291 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2292 2293 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2294 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2295 sysctl_handle_link_status_human, "A", 2296 "Link status (human readable)"); 2297 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2298 CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2299 "IU", "Link status (1=active, 0=inactive)"); 2300 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2301 CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2302 "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2303 2304 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2305 CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2306 tree_par = SYSCTL_CHILDREN(tree); 2307 2308 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2309 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2310 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2311 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2312 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2313 &ntb->ppd, 0, "Raw PPD register (cached)"); 2314 2315 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2316 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2317 &ntb->b2b_mw_idx, 0, 2318 "Index of the MW used for B2B remote register access"); 2319 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2320 CTLFLAG_RD, &ntb->b2b_off, 2321 "If non-zero, offset of B2B register region in shared MW"); 2322 } 2323 2324 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2325 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2326 "Features/errata of this NTB device"); 2327 2328 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2329 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2330 "NTB CTL register (cached)"); 2331 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2332 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2333 "LNK STA register (cached)"); 2334 2335 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2336 &ntb->mw_count, 0, "MW count"); 2337 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2338 &ntb->spad_count, 0, "Scratchpad count"); 2339 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2340 &ntb->db_count, 0, "Doorbell count"); 2341 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2342 &ntb->db_vec_count, 0, "Doorbell vector count"); 2343 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2344 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2345 2346 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2347 &ntb->db_valid_mask, "Doorbell valid mask"); 2348 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2349 &ntb->db_link_mask, "Doorbell link mask"); 2350 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2351 &ntb->db_mask, "Doorbell mask (cached)"); 2352 2353 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2354 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2355 regpar = SYSCTL_CHILDREN(tmptree); 2356 2357 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2358 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2359 ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2360 "NTB Control register"); 2361 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2362 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2363 0x19c, sysctl_handle_register, "IU", 2364 "NTB Link Capabilities"); 2365 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2366 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2367 0x1a0, sysctl_handle_register, "IU", 2368 "NTB Link Control register"); 2369 2370 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2371 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2372 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2373 sysctl_handle_register, "QU", "Doorbell mask register"); 2374 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2375 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2376 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2377 sysctl_handle_register, "QU", "Doorbell register"); 2378 2379 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2380 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2381 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2382 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2383 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2384 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2385 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2386 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2387 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2388 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2389 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2390 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2391 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2392 } else { 2393 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2394 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2395 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2396 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2397 } 2398 2399 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2400 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2401 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2402 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2403 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2404 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2405 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2406 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2407 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2408 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2409 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2410 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2411 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2412 } else { 2413 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2414 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2415 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2416 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2417 } 2418 2419 if (ntb->type == NTB_ATOM) 2420 return; 2421 2422 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2423 CTLFLAG_RD, NULL, "Xeon HW statistics"); 2424 statpar = SYSCTL_CHILDREN(tmptree); 2425 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2426 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2427 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2428 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2429 2430 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2431 CTLFLAG_RD, NULL, "Xeon HW errors"); 2432 errpar = SYSCTL_CHILDREN(tmptree); 2433 2434 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2435 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2436 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2437 sysctl_handle_register, "CU", "PPD"); 2438 2439 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2440 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2441 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2442 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2443 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2444 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2445 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2446 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2447 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2448 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2449 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2450 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2451 2452 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2453 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2454 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2455 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2456 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2457 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2458 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2459 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2460 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2461 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2462 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2463 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2464 2465 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2466 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2467 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2468 sysctl_handle_register, "SU", "DEVSTS"); 2469 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2470 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2471 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2472 sysctl_handle_register, "SU", "LNKSTS"); 2473 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2474 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2475 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2476 sysctl_handle_register, "SU", "SLNKSTS"); 2477 2478 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2479 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2480 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2481 sysctl_handle_register, "IU", "UNCERRSTS"); 2482 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2483 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2484 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2485 sysctl_handle_register, "IU", "CORERRSTS"); 2486 2487 if (ntb->conn_type != NTB_CONN_B2B) 2488 return; 2489 2490 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2491 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2492 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2493 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2494 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2495 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2496 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2497 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2498 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2499 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2500 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2501 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2502 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2503 } else { 2504 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2505 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2506 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2507 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2508 } 2509 2510 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2511 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2512 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2513 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2514 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2515 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2516 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2517 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2518 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2519 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2520 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2521 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2522 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2523 } else { 2524 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2525 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2526 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2527 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2528 } 2529 2530 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2531 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2532 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2533 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2534 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2535 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2536 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2537 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2538 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2539 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2540 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2541 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2542 sysctl_handle_register, "IU", 2543 "Secondary BAR4 base register"); 2544 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2545 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2546 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2547 sysctl_handle_register, "IU", 2548 "Secondary BAR5 base register"); 2549 } else { 2550 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2551 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2552 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2553 sysctl_handle_register, "QU", 2554 "Secondary BAR45 base register"); 2555 } 2556} 2557 2558static int 2559sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2560{ 2561 struct ntb_softc *ntb = arg1; 2562 struct sbuf sb; 2563 int error; 2564 2565 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2566 2567 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2568 error = sbuf_finish(&sb); 2569 sbuf_delete(&sb); 2570 2571 if (error || !req->newptr) 2572 return (error); 2573 return (EINVAL); 2574} 2575 2576static int 2577sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2578{ 2579 struct ntb_softc *ntb = arg1; 2580 unsigned old, new; 2581 int error; 2582 2583 old = ntb_link_enabled(ntb->device); 2584 2585 error = SYSCTL_OUT(req, &old, sizeof(old)); 2586 if (error != 0 || req->newptr == NULL) 2587 return (error); 2588 2589 error = SYSCTL_IN(req, &new, sizeof(new)); 2590 if (error != 0) 2591 return (error); 2592 2593 ntb_printf(0, "Admin set interface state to '%sabled'\n", 2594 (new != 0)? "en" : "dis"); 2595 2596 if (new != 0) 2597 error = ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2598 else 2599 error = ntb_link_disable(ntb->device); 2600 return (error); 2601} 2602 2603static int 2604sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2605{ 2606 struct ntb_softc *ntb = arg1; 2607 struct sbuf sb; 2608 enum ntb_speed speed; 2609 enum ntb_width width; 2610 int error; 2611 2612 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2613 2614 if (ntb_link_is_up(ntb->device, &speed, &width)) 2615 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2616 (unsigned)speed, (unsigned)width); 2617 else 2618 sbuf_printf(&sb, "down"); 2619 2620 error = sbuf_finish(&sb); 2621 sbuf_delete(&sb); 2622 2623 if (error || !req->newptr) 2624 return (error); 2625 return (EINVAL); 2626} 2627 2628static int 2629sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2630{ 2631 struct ntb_softc *ntb = arg1; 2632 unsigned res; 2633 int error; 2634 2635 res = ntb_link_is_up(ntb->device, NULL, NULL); 2636 2637 error = SYSCTL_OUT(req, &res, sizeof(res)); 2638 if (error || !req->newptr) 2639 return (error); 2640 return (EINVAL); 2641} 2642 2643static int 2644sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2645{ 2646 struct ntb_softc *ntb; 2647 const void *outp; 2648 uintptr_t sz; 2649 uint64_t umv; 2650 char be[sizeof(umv)]; 2651 size_t outsz; 2652 uint32_t reg; 2653 bool db, pci; 2654 int error; 2655 2656 ntb = arg1; 2657 reg = arg2 & ~NTB_REGFLAGS_MASK; 2658 sz = arg2 & NTB_REGSZ_MASK; 2659 db = (arg2 & NTB_DB_READ) != 0; 2660 pci = (arg2 & NTB_PCI_REG) != 0; 2661 2662 KASSERT(!(db && pci), ("bogus")); 2663 2664 if (db) { 2665 KASSERT(sz == NTB_REG_64, ("bogus")); 2666 umv = db_ioread(ntb, reg); 2667 outsz = sizeof(uint64_t); 2668 } else { 2669 switch (sz) { 2670 case NTB_REG_64: 2671 if (pci) 2672 umv = pci_read_config(ntb->device, reg, 8); 2673 else 2674 umv = ntb_reg_read(8, reg); 2675 outsz = sizeof(uint64_t); 2676 break; 2677 case NTB_REG_32: 2678 if (pci) 2679 umv = pci_read_config(ntb->device, reg, 4); 2680 else 2681 umv = ntb_reg_read(4, reg); 2682 outsz = sizeof(uint32_t); 2683 break; 2684 case NTB_REG_16: 2685 if (pci) 2686 umv = pci_read_config(ntb->device, reg, 2); 2687 else 2688 umv = ntb_reg_read(2, reg); 2689 outsz = sizeof(uint16_t); 2690 break; 2691 case NTB_REG_8: 2692 if (pci) 2693 umv = pci_read_config(ntb->device, reg, 1); 2694 else 2695 umv = ntb_reg_read(1, reg); 2696 outsz = sizeof(uint8_t); 2697 break; 2698 default: 2699 panic("bogus"); 2700 break; 2701 } 2702 } 2703 2704 /* Encode bigendian so that sysctl -x is legible. */ 2705 be64enc(be, umv); 2706 outp = ((char *)be) + sizeof(umv) - outsz; 2707 2708 error = SYSCTL_OUT(req, outp, outsz); 2709 if (error || !req->newptr) 2710 return (error); 2711 return (EINVAL); 2712} 2713 2714static unsigned 2715ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2716{ 2717 2718 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2719 uidx >= ntb->b2b_mw_idx) || 2720 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2721 uidx++; 2722 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2723 uidx >= ntb->b2b_mw_idx) && 2724 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2725 uidx++; 2726 return (uidx); 2727} 2728 2729static void 2730ntb_exchange_msix(void *ctx) 2731{ 2732 struct ntb_softc *ntb; 2733 uint32_t val; 2734 unsigned i; 2735 2736 ntb = ctx; 2737 2738 if (ntb->peer_msix_good) 2739 goto msix_good; 2740 if (ntb->peer_msix_done) 2741 goto msix_done; 2742 2743 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2744 ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 2745 ntb->msix_data[i].nmd_data); 2746 ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 2747 ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2748 } 2749 ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2750 2751 ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 2752 if (val != NTB_MSIX_VER_GUARD) 2753 goto reschedule; 2754 2755 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2756 ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 2757 ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2758 ntb->peer_msix_data[i].nmd_data = val; 2759 ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 2760 ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2761 ntb->peer_msix_data[i].nmd_ofs = val; 2762 } 2763 2764 ntb->peer_msix_done = true; 2765 2766msix_done: 2767 ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2768 ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 2769 if (val != NTB_MSIX_RECEIVED) 2770 goto reschedule; 2771 2772 ntb->peer_msix_good = true; 2773 /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2774 goto reschedule; 2775 2776msix_good: 2777 ntb_poll_link(ntb); 2778 ntb_link_event(ntb->device); 2779 return; 2780 2781reschedule: 2782 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2783 if (_xeon_link_is_up(ntb)) { 2784 callout_reset(&ntb->peer_msix_work, 2785 hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2786 ntb_exchange_msix, ntb); 2787 } else 2788 ntb_spad_clear(ntb->device); 2789} 2790 2791/* 2792 * Public API to the rest of the OS 2793 */ 2794 2795static uint8_t 2796ntb_spad_count(device_t dev) 2797{ 2798 struct ntb_softc *ntb = device_get_softc(dev); 2799 2800 return (ntb->spad_count); 2801} 2802 2803static uint8_t 2804ntb_mw_count(device_t dev) 2805{ 2806 struct ntb_softc *ntb = device_get_softc(dev); 2807 uint8_t res; 2808 2809 res = ntb->mw_count; 2810 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2811 res--; 2812 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2813 res--; 2814 return (res); 2815} 2816 2817static int 2818ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 2819{ 2820 struct ntb_softc *ntb = device_get_softc(dev); 2821 2822 if (idx >= ntb->spad_count) 2823 return (EINVAL); 2824 2825 ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2826 2827 return (0); 2828} 2829 2830/* 2831 * Zeros the local scratchpad. 2832 */ 2833static void 2834ntb_spad_clear(device_t dev) 2835{ 2836 struct ntb_softc *ntb = device_get_softc(dev); 2837 unsigned i; 2838 2839 for (i = 0; i < ntb->spad_count; i++) 2840 ntb_spad_write(dev, i, 0); 2841} 2842 2843static int 2844ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2845{ 2846 struct ntb_softc *ntb = device_get_softc(dev); 2847 2848 if (idx >= ntb->spad_count) 2849 return (EINVAL); 2850 2851 *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2852 2853 return (0); 2854} 2855 2856static int 2857ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 2858{ 2859 struct ntb_softc *ntb = device_get_softc(dev); 2860 2861 if (idx >= ntb->spad_count) 2862 return (EINVAL); 2863 2864 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2865 ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2866 else 2867 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2868 2869 return (0); 2870} 2871 2872static int 2873ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2874{ 2875 struct ntb_softc *ntb = device_get_softc(dev); 2876 2877 if (idx >= ntb->spad_count) 2878 return (EINVAL); 2879 2880 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2881 *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2882 else 2883 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2884 2885 return (0); 2886} 2887 2888static int 2889ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 2890 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2891 bus_addr_t *plimit) 2892{ 2893 struct ntb_softc *ntb = device_get_softc(dev); 2894 struct ntb_pci_bar_info *bar; 2895 bus_addr_t limit; 2896 size_t bar_b2b_off; 2897 enum ntb_bar bar_num; 2898 2899 if (mw_idx >= ntb_mw_count(dev)) 2900 return (EINVAL); 2901 mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 2902 2903 bar_num = ntb_mw_to_bar(ntb, mw_idx); 2904 bar = &ntb->bar_info[bar_num]; 2905 bar_b2b_off = 0; 2906 if (mw_idx == ntb->b2b_mw_idx) { 2907 KASSERT(ntb->b2b_off != 0, 2908 ("user shouldn't get non-shared b2b mw")); 2909 bar_b2b_off = ntb->b2b_off; 2910 } 2911 2912 if (bar_is_64bit(ntb, bar_num)) 2913 limit = BUS_SPACE_MAXADDR; 2914 else 2915 limit = BUS_SPACE_MAXADDR_32BIT; 2916 2917 if (base != NULL) 2918 *base = bar->pbase + bar_b2b_off; 2919 if (vbase != NULL) 2920 *vbase = bar->vbase + bar_b2b_off; 2921 if (size != NULL) 2922 *size = bar->size - bar_b2b_off; 2923 if (align != NULL) 2924 *align = bar->size; 2925 if (align_size != NULL) 2926 *align_size = 1; 2927 if (plimit != NULL) 2928 *plimit = limit; 2929 return (0); 2930} 2931 2932static int 2933ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 2934{ 2935 struct ntb_softc *ntb = device_get_softc(dev); 2936 struct ntb_pci_bar_info *bar; 2937 uint64_t base, limit, reg_val; 2938 size_t bar_size, mw_size; 2939 uint32_t base_reg, xlat_reg, limit_reg; 2940 enum ntb_bar bar_num; 2941 2942 if (idx >= ntb_mw_count(dev)) 2943 return (EINVAL); 2944 idx = ntb_user_mw_to_idx(ntb, idx); 2945 2946 bar_num = ntb_mw_to_bar(ntb, idx); 2947 bar = &ntb->bar_info[bar_num]; 2948 2949 bar_size = bar->size; 2950 if (idx == ntb->b2b_mw_idx) 2951 mw_size = bar_size - ntb->b2b_off; 2952 else 2953 mw_size = bar_size; 2954 2955 /* Hardware requires that addr is aligned to bar size */ 2956 if ((addr & (bar_size - 1)) != 0) 2957 return (EINVAL); 2958 2959 if (size > mw_size) 2960 return (EINVAL); 2961 2962 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2963 2964 limit = 0; 2965 if (bar_is_64bit(ntb, bar_num)) { 2966 base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2967 2968 if (limit_reg != 0 && size != mw_size) 2969 limit = base + size; 2970 2971 /* Set and verify translation address */ 2972 ntb_reg_write(8, xlat_reg, addr); 2973 reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2974 if (reg_val != addr) { 2975 ntb_reg_write(8, xlat_reg, 0); 2976 return (EIO); 2977 } 2978 2979 /* Set and verify the limit */ 2980 ntb_reg_write(8, limit_reg, limit); 2981 reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 2982 if (reg_val != limit) { 2983 ntb_reg_write(8, limit_reg, base); 2984 ntb_reg_write(8, xlat_reg, 0); 2985 return (EIO); 2986 } 2987 } else { 2988 /* Configure 32-bit (split) BAR MW */ 2989 2990 if ((addr & UINT32_MAX) != addr) 2991 return (ERANGE); 2992 if (((addr + size) & UINT32_MAX) != (addr + size)) 2993 return (ERANGE); 2994 2995 base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 2996 2997 if (limit_reg != 0 && size != mw_size) 2998 limit = base + size; 2999 3000 /* Set and verify translation address */ 3001 ntb_reg_write(4, xlat_reg, addr); 3002 reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3003 if (reg_val != addr) { 3004 ntb_reg_write(4, xlat_reg, 0); 3005 return (EIO); 3006 } 3007 3008 /* Set and verify the limit */ 3009 ntb_reg_write(4, limit_reg, limit); 3010 reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3011 if (reg_val != limit) { 3012 ntb_reg_write(4, limit_reg, base); 3013 ntb_reg_write(4, xlat_reg, 0); 3014 return (EIO); 3015 } 3016 } 3017 return (0); 3018} 3019 3020static int 3021ntb_mw_clear_trans(device_t dev, unsigned mw_idx) 3022{ 3023 3024 return (ntb_mw_set_trans(dev, mw_idx, 0, 0)); 3025} 3026 3027static int 3028ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 3029{ 3030 struct ntb_softc *ntb = device_get_softc(dev); 3031 struct ntb_pci_bar_info *bar; 3032 3033 if (idx >= ntb_mw_count(dev)) 3034 return (EINVAL); 3035 idx = ntb_user_mw_to_idx(ntb, idx); 3036 3037 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3038 *mode = bar->map_mode; 3039 return (0); 3040} 3041 3042static int 3043ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 3044{ 3045 struct ntb_softc *ntb = device_get_softc(dev); 3046 3047 if (idx >= ntb_mw_count(dev)) 3048 return (EINVAL); 3049 3050 idx = ntb_user_mw_to_idx(ntb, idx); 3051 return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3052} 3053 3054static int 3055ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3056{ 3057 struct ntb_pci_bar_info *bar; 3058 int rc; 3059 3060 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3061 if (bar->map_mode == mode) 3062 return (0); 3063 3064 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3065 if (rc == 0) 3066 bar->map_mode = mode; 3067 3068 return (rc); 3069} 3070 3071static void 3072ntb_peer_db_set(device_t dev, uint64_t bit) 3073{ 3074 struct ntb_softc *ntb = device_get_softc(dev); 3075 3076 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 3077 struct ntb_pci_bar_info *lapic; 3078 unsigned i; 3079 3080 lapic = ntb->peer_lapic_bar; 3081 3082 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3083 if ((bit & ntb_db_vector_mask(dev, i)) != 0) 3084 bus_space_write_4(lapic->pci_bus_tag, 3085 lapic->pci_bus_handle, 3086 ntb->peer_msix_data[i].nmd_ofs, 3087 ntb->peer_msix_data[i].nmd_data); 3088 } 3089 return; 3090 } 3091 3092 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3093 ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3094 return; 3095 } 3096 3097 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3098} 3099 3100static int 3101ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3102{ 3103 struct ntb_softc *ntb = device_get_softc(dev); 3104 struct ntb_pci_bar_info *bar; 3105 uint64_t regoff; 3106 3107 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3108 3109 if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3110 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3111 regoff = ntb->peer_reg->db_bell; 3112 } else { 3113 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3114 ("invalid b2b idx")); 3115 3116 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3117 regoff = XEON_PDOORBELL_OFFSET; 3118 } 3119 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3120 3121 /* HACK: Specific to current x86 bus implementation. */ 3122 *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3123 *db_size = ntb->reg->db_size; 3124 return (0); 3125} 3126 3127static uint64_t 3128ntb_db_valid_mask(device_t dev) 3129{ 3130 struct ntb_softc *ntb = device_get_softc(dev); 3131 3132 return (ntb->db_valid_mask); 3133} 3134 3135static int 3136ntb_db_vector_count(device_t dev) 3137{ 3138 struct ntb_softc *ntb = device_get_softc(dev); 3139 3140 return (ntb->db_vec_count); 3141} 3142 3143static uint64_t 3144ntb_db_vector_mask(device_t dev, uint32_t vector) 3145{ 3146 struct ntb_softc *ntb = device_get_softc(dev); 3147 3148 if (vector > ntb->db_vec_count) 3149 return (0); 3150 return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3151} 3152 3153static bool 3154ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3155{ 3156 struct ntb_softc *ntb = device_get_softc(dev); 3157 3158 if (speed != NULL) 3159 *speed = ntb_link_sta_speed(ntb); 3160 if (width != NULL) 3161 *width = ntb_link_sta_width(ntb); 3162 return (link_is_up(ntb)); 3163} 3164 3165static void 3166save_bar_parameters(struct ntb_pci_bar_info *bar) 3167{ 3168 3169 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3170 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3171 bar->pbase = rman_get_start(bar->pci_resource); 3172 bar->size = rman_get_size(bar->pci_resource); 3173 bar->vbase = rman_get_virtual(bar->pci_resource); 3174} 3175 3176static device_method_t ntb_intel_methods[] = { 3177 /* Device interface */ 3178 DEVMETHOD(device_probe, ntb_probe), 3179 DEVMETHOD(device_attach, ntb_attach), 3180 DEVMETHOD(device_detach, ntb_detach), 3181 /* NTB interface */ 3182 DEVMETHOD(ntb_link_is_up, ntb_link_is_up), 3183 DEVMETHOD(ntb_link_enable, ntb_link_enable), 3184 DEVMETHOD(ntb_link_disable, ntb_link_disable), 3185 DEVMETHOD(ntb_link_enabled, ntb_link_enabled), 3186 DEVMETHOD(ntb_set_ctx, ntb_set_ctx), 3187 DEVMETHOD(ntb_get_ctx, ntb_get_ctx), 3188 DEVMETHOD(ntb_clear_ctx, ntb_clear_ctx), 3189 DEVMETHOD(ntb_mw_count, ntb_mw_count), 3190 DEVMETHOD(ntb_mw_get_range, ntb_mw_get_range), 3191 DEVMETHOD(ntb_mw_set_trans, ntb_mw_set_trans), 3192 DEVMETHOD(ntb_mw_clear_trans, ntb_mw_clear_trans), 3193 DEVMETHOD(ntb_mw_get_wc, ntb_mw_get_wc), 3194 DEVMETHOD(ntb_mw_set_wc, ntb_mw_set_wc), 3195 DEVMETHOD(ntb_spad_count, ntb_spad_count), 3196 DEVMETHOD(ntb_spad_clear, ntb_spad_clear), 3197 DEVMETHOD(ntb_spad_write, ntb_spad_write), 3198 DEVMETHOD(ntb_spad_read, ntb_spad_read), 3199 DEVMETHOD(ntb_peer_spad_write, ntb_peer_spad_write), 3200 DEVMETHOD(ntb_peer_spad_read, ntb_peer_spad_read), 3201 DEVMETHOD(ntb_db_valid_mask, ntb_db_valid_mask), 3202 DEVMETHOD(ntb_db_vector_count, ntb_db_vector_count), 3203 DEVMETHOD(ntb_db_vector_mask, ntb_db_vector_mask), 3204 DEVMETHOD(ntb_db_clear, ntb_db_clear), 3205 DEVMETHOD(ntb_db_clear_mask, ntb_db_clear_mask), 3206 DEVMETHOD(ntb_db_read, ntb_db_read), 3207 DEVMETHOD(ntb_db_set_mask, ntb_db_set_mask), 3208 DEVMETHOD(ntb_peer_db_addr, ntb_peer_db_addr), 3209 DEVMETHOD(ntb_peer_db_set, ntb_peer_db_set), 3210 DEVMETHOD_END 3211}; 3212 3213static DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3214 sizeof(struct ntb_softc)); 3215DRIVER_MODULE(ntb_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL); 3216MODULE_DEPEND(ntb_intel, ntb, 1, 1, 1); 3217MODULE_VERSION(ntb_intel, 1); 3218