ntb_hw.c revision 304388
1/*- 2 * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 3 * Copyright (C) 2013 Intel Corporation 4 * Copyright (C) 2015 EMC Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29/* 30 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 31 * two or more systems using a PCI-e links, providing remote memory access. 32 * 33 * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 34 * 35 * NOTE: Much of the code in this module is shared with Linux. Any patches may 36 * be picked up and redistributed in Linux with a dual GPL/BSD license. 37 */ 38 39#include <sys/cdefs.h> 40__FBSDID("$FreeBSD: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c 304388 2016-08-18 10:43:59Z mav $"); 41 42#include <sys/param.h> 43#include <sys/kernel.h> 44#include <sys/systm.h> 45#include <sys/bus.h> 46#include <sys/endian.h> 47#include <sys/interrupt.h> 48#include <sys/malloc.h> 49#include <sys/module.h> 50#include <sys/mutex.h> 51#include <sys/pciio.h> 52#include <sys/queue.h> 53#include <sys/rman.h> 54#include <sys/rmlock.h> 55#include <sys/sbuf.h> 56#include <sys/sysctl.h> 57#include <vm/vm.h> 58#include <vm/pmap.h> 59#include <machine/bus.h> 60#include <machine/intr_machdep.h> 61#include <machine/pmap.h> 62#include <machine/resource.h> 63#include <dev/pci/pcireg.h> 64#include <dev/pci/pcivar.h> 65 66#include "ntb_regs.h" 67#include "../ntb.h" 68 69#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 70 71#define NTB_HB_TIMEOUT 1 /* second */ 72#define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 73#define BAR_HIGH_MASK (~((1ull << 12) - 1)) 74 75#define NTB_MSIX_VER_GUARD 0xaabbccdd 76#define NTB_MSIX_RECEIVED 0xe0f0e0f0 77 78/* 79 * PCI constants could be somewhere more generic, but aren't defined/used in 80 * pci.c. 81 */ 82#define PCI_MSIX_ENTRY_SIZE 16 83#define PCI_MSIX_ENTRY_LOWER_ADDR 0 84#define PCI_MSIX_ENTRY_UPPER_ADDR 4 85#define PCI_MSIX_ENTRY_DATA 8 86 87enum ntb_device_type { 88 NTB_XEON, 89 NTB_ATOM 90}; 91 92/* ntb_conn_type are hardware numbers, cannot change. */ 93enum ntb_conn_type { 94 NTB_CONN_TRANSPARENT = 0, 95 NTB_CONN_B2B = 1, 96 NTB_CONN_RP = 2, 97}; 98 99enum ntb_b2b_direction { 100 NTB_DEV_USD = 0, 101 NTB_DEV_DSD = 1, 102}; 103 104enum ntb_bar { 105 NTB_CONFIG_BAR = 0, 106 NTB_B2B_BAR_1, 107 NTB_B2B_BAR_2, 108 NTB_B2B_BAR_3, 109 NTB_MAX_BARS 110}; 111 112enum { 113 NTB_MSIX_GUARD = 0, 114 NTB_MSIX_DATA0, 115 NTB_MSIX_DATA1, 116 NTB_MSIX_DATA2, 117 NTB_MSIX_OFS0, 118 NTB_MSIX_OFS1, 119 NTB_MSIX_OFS2, 120 NTB_MSIX_DONE, 121 NTB_MAX_MSIX_SPAD 122}; 123 124/* Device features and workarounds */ 125#define HAS_FEATURE(ntb, feature) \ 126 (((ntb)->features & (feature)) != 0) 127 128struct ntb_hw_info { 129 uint32_t device_id; 130 const char *desc; 131 enum ntb_device_type type; 132 uint32_t features; 133}; 134 135struct ntb_pci_bar_info { 136 bus_space_tag_t pci_bus_tag; 137 bus_space_handle_t pci_bus_handle; 138 int pci_resource_id; 139 struct resource *pci_resource; 140 vm_paddr_t pbase; 141 caddr_t vbase; 142 vm_size_t size; 143 vm_memattr_t map_mode; 144 145 /* Configuration register offsets */ 146 uint32_t psz_off; 147 uint32_t ssz_off; 148 uint32_t pbarxlat_off; 149}; 150 151struct ntb_int_info { 152 struct resource *res; 153 int rid; 154 void *tag; 155}; 156 157struct ntb_vec { 158 struct ntb_softc *ntb; 159 uint32_t num; 160 unsigned masked; 161}; 162 163struct ntb_reg { 164 uint32_t ntb_ctl; 165 uint32_t lnk_sta; 166 uint8_t db_size; 167 unsigned mw_bar[NTB_MAX_BARS]; 168}; 169 170struct ntb_alt_reg { 171 uint32_t db_bell; 172 uint32_t db_mask; 173 uint32_t spad; 174}; 175 176struct ntb_xlat_reg { 177 uint32_t bar0_base; 178 uint32_t bar2_base; 179 uint32_t bar4_base; 180 uint32_t bar5_base; 181 182 uint32_t bar2_xlat; 183 uint32_t bar4_xlat; 184 uint32_t bar5_xlat; 185 186 uint32_t bar2_limit; 187 uint32_t bar4_limit; 188 uint32_t bar5_limit; 189}; 190 191struct ntb_b2b_addr { 192 uint64_t bar0_addr; 193 uint64_t bar2_addr64; 194 uint64_t bar4_addr64; 195 uint64_t bar4_addr32; 196 uint64_t bar5_addr32; 197}; 198 199struct ntb_msix_data { 200 uint32_t nmd_ofs; 201 uint32_t nmd_data; 202}; 203 204struct ntb_softc { 205 device_t device; 206 enum ntb_device_type type; 207 uint32_t features; 208 209 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 210 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 211 uint32_t allocated_interrupts; 212 213 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 214 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 215 bool peer_msix_good; 216 bool peer_msix_done; 217 struct ntb_pci_bar_info *peer_lapic_bar; 218 struct callout peer_msix_work; 219 220 struct callout heartbeat_timer; 221 struct callout lr_timer; 222 223 void *ntb_ctx; 224 const struct ntb_ctx_ops *ctx_ops; 225 struct ntb_vec *msix_vec; 226 struct rmlock ctx_lock; 227 228 uint32_t ppd; 229 enum ntb_conn_type conn_type; 230 enum ntb_b2b_direction dev_type; 231 232 /* Offset of peer bar0 in B2B BAR */ 233 uint64_t b2b_off; 234 /* Memory window used to access peer bar0 */ 235#define B2B_MW_DISABLED UINT8_MAX 236 uint8_t b2b_mw_idx; 237 uint32_t msix_xlat; 238 uint8_t msix_mw_idx; 239 240 uint8_t mw_count; 241 uint8_t spad_count; 242 uint8_t db_count; 243 uint8_t db_vec_count; 244 uint8_t db_vec_shift; 245 246 /* Protects local db_mask. */ 247#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 248#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 249#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 250 struct mtx db_mask_lock; 251 252 volatile uint32_t ntb_ctl; 253 volatile uint32_t lnk_sta; 254 255 uint64_t db_valid_mask; 256 uint64_t db_link_mask; 257 uint64_t db_mask; 258 uint64_t fake_db_bell; /* NTB_SB01BASE_LOCKUP*/ 259 260 int last_ts; /* ticks @ last irq */ 261 262 const struct ntb_reg *reg; 263 const struct ntb_alt_reg *self_reg; 264 const struct ntb_alt_reg *peer_reg; 265 const struct ntb_xlat_reg *xlat_reg; 266}; 267 268#ifdef __i386__ 269static __inline uint64_t 270bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 271 bus_size_t offset) 272{ 273 274 return (bus_space_read_4(tag, handle, offset) | 275 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 276} 277 278static __inline void 279bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 280 bus_size_t offset, uint64_t val) 281{ 282 283 bus_space_write_4(tag, handle, offset, val); 284 bus_space_write_4(tag, handle, offset + 4, val >> 32); 285} 286#endif 287 288#define ntb_bar_read(SIZE, bar, offset) \ 289 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 290 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 291#define ntb_bar_write(SIZE, bar, offset, val) \ 292 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 293 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 294#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 295#define ntb_reg_write(SIZE, offset, val) \ 296 ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 297#define ntb_mw_read(SIZE, offset) \ 298 ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) 299#define ntb_mw_write(SIZE, offset, val) \ 300 ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 301 offset, val) 302 303static int ntb_probe(device_t device); 304static int ntb_attach(device_t device); 305static int ntb_detach(device_t device); 306static uint64_t ntb_db_valid_mask(device_t dev); 307static void ntb_spad_clear(device_t dev); 308static uint64_t ntb_db_vector_mask(device_t dev, uint32_t vector); 309static bool ntb_link_is_up(device_t dev, enum ntb_speed *speed, 310 enum ntb_width *width); 311static int ntb_link_enable(device_t dev, enum ntb_speed speed, 312 enum ntb_width width); 313static int ntb_link_disable(device_t dev); 314static int ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 315static int ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 316 317static unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 318static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 319static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 320static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 321 uint32_t *base, uint32_t *xlat, uint32_t *lmt); 322static int ntb_map_pci_bars(struct ntb_softc *ntb); 323static int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 324 vm_memattr_t); 325static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 326 const char *); 327static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 328static int map_memory_window_bar(struct ntb_softc *ntb, 329 struct ntb_pci_bar_info *bar); 330static void ntb_unmap_pci_bar(struct ntb_softc *ntb); 331static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 332static int ntb_init_isr(struct ntb_softc *ntb); 333static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 334static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 335static void ntb_teardown_interrupts(struct ntb_softc *ntb); 336static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 337static void ntb_interrupt(struct ntb_softc *, uint32_t vec); 338static void ndev_vec_isr(void *arg); 339static void ndev_irq_isr(void *arg); 340static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 341static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 342static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 343static int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 344static void ntb_free_msix_vec(struct ntb_softc *ntb); 345static void ntb_get_msix_info(struct ntb_softc *ntb); 346static void ntb_exchange_msix(void *); 347static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); 348static void ntb_detect_max_mw(struct ntb_softc *ntb); 349static int ntb_detect_xeon(struct ntb_softc *ntb); 350static int ntb_detect_atom(struct ntb_softc *ntb); 351static int ntb_xeon_init_dev(struct ntb_softc *ntb); 352static int ntb_atom_init_dev(struct ntb_softc *ntb); 353static void ntb_teardown_xeon(struct ntb_softc *ntb); 354static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 355static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 356 enum ntb_bar regbar); 357static void xeon_set_sbar_base_and_limit(struct ntb_softc *, 358 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 359static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 360 enum ntb_bar idx); 361static int xeon_setup_b2b_mw(struct ntb_softc *, 362 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 363static inline bool link_is_up(struct ntb_softc *ntb); 364static inline bool _xeon_link_is_up(struct ntb_softc *ntb); 365static inline bool atom_link_is_err(struct ntb_softc *ntb); 366static inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); 367static inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); 368static void atom_link_hb(void *arg); 369static void ntb_link_event(device_t dev); 370static void ntb_db_event(device_t dev, uint32_t vec); 371static void recover_atom_link(void *arg); 372static bool ntb_poll_link(struct ntb_softc *ntb); 373static void save_bar_parameters(struct ntb_pci_bar_info *bar); 374static void ntb_sysctl_init(struct ntb_softc *); 375static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 376static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 377static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 378static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 379static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 380 381static unsigned g_ntb_hw_debug_level; 382TUNABLE_INT("hw.ntb.debug_level", &g_ntb_hw_debug_level); 383SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 384 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 385#define ntb_printf(lvl, ...) do { \ 386 if ((lvl) <= g_ntb_hw_debug_level) { \ 387 device_printf(ntb->device, __VA_ARGS__); \ 388 } \ 389} while (0) 390 391#define _NTB_PAT_UC 0 392#define _NTB_PAT_WC 1 393#define _NTB_PAT_WT 4 394#define _NTB_PAT_WP 5 395#define _NTB_PAT_WB 6 396#define _NTB_PAT_UCM 7 397static unsigned g_ntb_mw_pat = _NTB_PAT_UC; 398TUNABLE_INT("hw.ntb.default_mw_pat", &g_ntb_mw_pat); 399SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 400 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 401 "UC: " __XSTRING(_NTB_PAT_UC) ", " 402 "WC: " __XSTRING(_NTB_PAT_WC) ", " 403 "WT: " __XSTRING(_NTB_PAT_WT) ", " 404 "WP: " __XSTRING(_NTB_PAT_WP) ", " 405 "WB: " __XSTRING(_NTB_PAT_WB) ", " 406 "UC-: " __XSTRING(_NTB_PAT_UCM)); 407 408static inline vm_memattr_t 409ntb_pat_flags(void) 410{ 411 412 switch (g_ntb_mw_pat) { 413 case _NTB_PAT_WC: 414 return (VM_MEMATTR_WRITE_COMBINING); 415 case _NTB_PAT_WT: 416 return (VM_MEMATTR_WRITE_THROUGH); 417 case _NTB_PAT_WP: 418 return (VM_MEMATTR_WRITE_PROTECTED); 419 case _NTB_PAT_WB: 420 return (VM_MEMATTR_WRITE_BACK); 421 case _NTB_PAT_UCM: 422 return (VM_MEMATTR_WEAK_UNCACHEABLE); 423 case _NTB_PAT_UC: 424 /* FALLTHROUGH */ 425 default: 426 return (VM_MEMATTR_UNCACHEABLE); 427 } 428} 429 430/* 431 * Well, this obviously doesn't belong here, but it doesn't seem to exist 432 * anywhere better yet. 433 */ 434static inline const char * 435ntb_vm_memattr_to_str(vm_memattr_t pat) 436{ 437 438 switch (pat) { 439 case VM_MEMATTR_WRITE_COMBINING: 440 return ("WRITE_COMBINING"); 441 case VM_MEMATTR_WRITE_THROUGH: 442 return ("WRITE_THROUGH"); 443 case VM_MEMATTR_WRITE_PROTECTED: 444 return ("WRITE_PROTECTED"); 445 case VM_MEMATTR_WRITE_BACK: 446 return ("WRITE_BACK"); 447 case VM_MEMATTR_WEAK_UNCACHEABLE: 448 return ("UNCACHED"); 449 case VM_MEMATTR_UNCACHEABLE: 450 return ("UNCACHEABLE"); 451 default: 452 return ("UNKNOWN"); 453 } 454} 455 456static int g_ntb_msix_idx = 0; 457TUNABLE_INT("hw.ntb.msix_mw_idx", &g_ntb_msix_idx); 458SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 459 0, "Use this memory window to access the peer MSIX message complex on " 460 "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 461 "Like b2b_mw_idx, negative values index from the last available memory " 462 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 463 464static int g_ntb_mw_idx = -1; 465TUNABLE_INT("hw.ntb.b2b_mw_idx", &g_ntb_mw_idx); 466SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 467 0, "Use this memory window to access the peer NTB registers. A " 468 "non-negative value starts from the first MW index; a negative value " 469 "starts from the last MW index. The default is -1, i.e., the last " 470 "available memory window. Both sides of the NTB MUST set the same " 471 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 472 473/* Hardware owns the low 16 bits of features. */ 474#define NTB_BAR_SIZE_4K (1 << 0) 475#define NTB_SDOORBELL_LOCKUP (1 << 1) 476#define NTB_SB01BASE_LOCKUP (1 << 2) 477#define NTB_B2BDOORBELL_BIT14 (1 << 3) 478/* Software/configuration owns the top 16 bits. */ 479#define NTB_SPLIT_BAR (1ull << 16) 480 481#define NTB_FEATURES_STR \ 482 "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 483 "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 484 485static struct ntb_hw_info pci_ids[] = { 486 /* XXX: PS/SS IDs left out until they are supported. */ 487 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 488 NTB_ATOM, 0 }, 489 490 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 491 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 492 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 493 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 494 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 495 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 496 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 497 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 498 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 499 NTB_SB01BASE_LOCKUP }, 500 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 501 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 502 NTB_SB01BASE_LOCKUP }, 503 504 { 0x00000000, NULL, NTB_ATOM, 0 } 505}; 506 507static const struct ntb_reg atom_reg = { 508 .ntb_ctl = ATOM_NTBCNTL_OFFSET, 509 .lnk_sta = ATOM_LINK_STATUS_OFFSET, 510 .db_size = sizeof(uint64_t), 511 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 512}; 513 514static const struct ntb_alt_reg atom_pri_reg = { 515 .db_bell = ATOM_PDOORBELL_OFFSET, 516 .db_mask = ATOM_PDBMSK_OFFSET, 517 .spad = ATOM_SPAD_OFFSET, 518}; 519 520static const struct ntb_alt_reg atom_b2b_reg = { 521 .db_bell = ATOM_B2B_DOORBELL_OFFSET, 522 .spad = ATOM_B2B_SPAD_OFFSET, 523}; 524 525static const struct ntb_xlat_reg atom_sec_xlat = { 526#if 0 527 /* "FIXME" says the Linux driver. */ 528 .bar0_base = ATOM_SBAR0BASE_OFFSET, 529 .bar2_base = ATOM_SBAR2BASE_OFFSET, 530 .bar4_base = ATOM_SBAR4BASE_OFFSET, 531 532 .bar2_limit = ATOM_SBAR2LMT_OFFSET, 533 .bar4_limit = ATOM_SBAR4LMT_OFFSET, 534#endif 535 536 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 537 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 538}; 539 540static const struct ntb_reg xeon_reg = { 541 .ntb_ctl = XEON_NTBCNTL_OFFSET, 542 .lnk_sta = XEON_LINK_STATUS_OFFSET, 543 .db_size = sizeof(uint16_t), 544 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 545}; 546 547static const struct ntb_alt_reg xeon_pri_reg = { 548 .db_bell = XEON_PDOORBELL_OFFSET, 549 .db_mask = XEON_PDBMSK_OFFSET, 550 .spad = XEON_SPAD_OFFSET, 551}; 552 553static const struct ntb_alt_reg xeon_b2b_reg = { 554 .db_bell = XEON_B2B_DOORBELL_OFFSET, 555 .spad = XEON_B2B_SPAD_OFFSET, 556}; 557 558static const struct ntb_xlat_reg xeon_sec_xlat = { 559 .bar0_base = XEON_SBAR0BASE_OFFSET, 560 .bar2_base = XEON_SBAR2BASE_OFFSET, 561 .bar4_base = XEON_SBAR4BASE_OFFSET, 562 .bar5_base = XEON_SBAR5BASE_OFFSET, 563 564 .bar2_limit = XEON_SBAR2LMT_OFFSET, 565 .bar4_limit = XEON_SBAR4LMT_OFFSET, 566 .bar5_limit = XEON_SBAR5LMT_OFFSET, 567 568 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 569 .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 570 .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 571}; 572 573static struct ntb_b2b_addr xeon_b2b_usd_addr = { 574 .bar0_addr = XEON_B2B_BAR0_ADDR, 575 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 576 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 577 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 578 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 579}; 580 581static struct ntb_b2b_addr xeon_b2b_dsd_addr = { 582 .bar0_addr = XEON_B2B_BAR0_ADDR, 583 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 584 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 585 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 586 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 587}; 588 589SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 590 "B2B MW segment overrides -- MUST be the same on both sides"); 591 592TUNABLE_QUAD("hw.ntb.usd_bar2_addr64", &xeon_b2b_usd_addr.bar2_addr64); 593SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 594 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 595 "hardware, use this 64-bit address on the bus between the NTB devices for " 596 "the window at BAR2, on the upstream side of the link. MUST be the same " 597 "address on both sides."); 598TUNABLE_QUAD("hw.ntb.usd_bar4_addr64", &xeon_b2b_usd_addr.bar4_addr64); 599SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 600 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 601TUNABLE_QUAD("hw.ntb.usd_bar4_addr32", &xeon_b2b_usd_addr.bar4_addr32); 602SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 603 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 604 "(split-BAR mode)."); 605TUNABLE_QUAD("hw.ntb.usd_bar5_addr32", &xeon_b2b_usd_addr.bar5_addr32); 606SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 607 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 608 "(split-BAR mode)."); 609 610TUNABLE_QUAD("hw.ntb.dsd_bar2_addr64", &xeon_b2b_dsd_addr.bar2_addr64); 611SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 612 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 613 "hardware, use this 64-bit address on the bus between the NTB devices for " 614 "the window at BAR2, on the downstream side of the link. MUST be the same" 615 " address on both sides."); 616TUNABLE_QUAD("hw.ntb.dsd_bar4_addr64", &xeon_b2b_dsd_addr.bar4_addr64); 617SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 618 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 619TUNABLE_QUAD("hw.ntb.dsd_bar4_addr32", &xeon_b2b_dsd_addr.bar4_addr32); 620SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 621 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 622 "(split-BAR mode)."); 623TUNABLE_QUAD("hw.ntb.dsd_bar5_addr32", &xeon_b2b_dsd_addr.bar5_addr32); 624SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 625 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 626 "(split-BAR mode)."); 627 628/* 629 * OS <-> Driver interface structures 630 */ 631MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 632 633SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 634 635/* 636 * OS <-> Driver linkage functions 637 */ 638static int 639ntb_probe(device_t device) 640{ 641 struct ntb_hw_info *p; 642 643 p = ntb_get_device_info(pci_get_devid(device)); 644 if (p == NULL) 645 return (ENXIO); 646 647 device_set_desc(device, p->desc); 648 return (0); 649} 650 651static int 652ntb_attach(device_t device) 653{ 654 struct ntb_softc *ntb; 655 struct ntb_hw_info *p; 656 int error; 657 658 ntb = device_get_softc(device); 659 p = ntb_get_device_info(pci_get_devid(device)); 660 661 ntb->device = device; 662 ntb->type = p->type; 663 ntb->features = p->features; 664 ntb->b2b_mw_idx = B2B_MW_DISABLED; 665 ntb->msix_mw_idx = B2B_MW_DISABLED; 666 667 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 668 callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE); 669 callout_init(&ntb->lr_timer, CALLOUT_MPSAFE); 670 callout_init(&ntb->peer_msix_work, 1); 671 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 672 rm_init(&ntb->ctx_lock, "ntb ctx"); 673 674 if (ntb->type == NTB_ATOM) 675 error = ntb_detect_atom(ntb); 676 else 677 error = ntb_detect_xeon(ntb); 678 if (error != 0) 679 goto out; 680 681 ntb_detect_max_mw(ntb); 682 683 pci_enable_busmaster(ntb->device); 684 685 error = ntb_map_pci_bars(ntb); 686 if (error != 0) 687 goto out; 688 if (ntb->type == NTB_ATOM) 689 error = ntb_atom_init_dev(ntb); 690 else 691 error = ntb_xeon_init_dev(ntb); 692 if (error != 0) 693 goto out; 694 695 ntb_spad_clear(device); 696 697 ntb_poll_link(ntb); 698 699 ntb_sysctl_init(ntb); 700 701 /* Attach children to this controller */ 702 device_add_child(device, NULL, -1); 703 bus_generic_attach(device); 704 705out: 706 if (error != 0) 707 ntb_detach(device); 708 return (error); 709} 710 711static int 712ntb_detach(device_t device) 713{ 714 struct ntb_softc *ntb; 715 716 ntb = device_get_softc(device); 717 718 /* Detach & delete all children */ 719 device_delete_children(device); 720 721 if (ntb->self_reg != NULL) { 722 DB_MASK_LOCK(ntb); 723 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 724 DB_MASK_UNLOCK(ntb); 725 } 726 callout_drain(&ntb->heartbeat_timer); 727 callout_drain(&ntb->lr_timer); 728 callout_drain(&ntb->peer_msix_work); 729 pci_disable_busmaster(ntb->device); 730 if (ntb->type == NTB_XEON) 731 ntb_teardown_xeon(ntb); 732 ntb_teardown_interrupts(ntb); 733 734 mtx_destroy(&ntb->db_mask_lock); 735 rm_destroy(&ntb->ctx_lock); 736 737 ntb_unmap_pci_bar(ntb); 738 739 return (0); 740} 741 742/* 743 * Driver internal routines 744 */ 745static inline enum ntb_bar 746ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 747{ 748 749 KASSERT(mw < ntb->mw_count, 750 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 751 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 752 753 return (ntb->reg->mw_bar[mw]); 754} 755 756static inline bool 757bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 758{ 759 /* XXX This assertion could be stronger. */ 760 KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 761 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 762} 763 764static inline void 765bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 766 uint32_t *xlat, uint32_t *lmt) 767{ 768 uint32_t basev, lmtv, xlatv; 769 770 switch (bar) { 771 case NTB_B2B_BAR_1: 772 basev = ntb->xlat_reg->bar2_base; 773 lmtv = ntb->xlat_reg->bar2_limit; 774 xlatv = ntb->xlat_reg->bar2_xlat; 775 break; 776 case NTB_B2B_BAR_2: 777 basev = ntb->xlat_reg->bar4_base; 778 lmtv = ntb->xlat_reg->bar4_limit; 779 xlatv = ntb->xlat_reg->bar4_xlat; 780 break; 781 case NTB_B2B_BAR_3: 782 basev = ntb->xlat_reg->bar5_base; 783 lmtv = ntb->xlat_reg->bar5_limit; 784 xlatv = ntb->xlat_reg->bar5_xlat; 785 break; 786 default: 787 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 788 ("bad bar")); 789 basev = lmtv = xlatv = 0; 790 break; 791 } 792 793 if (base != NULL) 794 *base = basev; 795 if (xlat != NULL) 796 *xlat = xlatv; 797 if (lmt != NULL) 798 *lmt = lmtv; 799} 800 801static int 802ntb_map_pci_bars(struct ntb_softc *ntb) 803{ 804 int rc; 805 806 ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0); 807 rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]); 808 if (rc != 0) 809 goto out; 810 811 ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2); 812 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]); 813 if (rc != 0) 814 goto out; 815 ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET; 816 ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET; 817 ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 818 819 ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4); 820 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]); 821 if (rc != 0) 822 goto out; 823 ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET; 824 ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET; 825 ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 826 827 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 828 goto out; 829 830 ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5); 831 rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]); 832 ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET; 833 ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET; 834 ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 835 836out: 837 if (rc != 0) 838 device_printf(ntb->device, 839 "unable to allocate pci resource\n"); 840 return (rc); 841} 842 843static void 844print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 845 const char *kind) 846{ 847 848 device_printf(ntb->device, 849 "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 850 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 851 (char *)bar->vbase + bar->size - 1, 852 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 853 (uintmax_t)bar->size, kind); 854} 855 856static int 857map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 858{ 859 860 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 861 &bar->pci_resource_id, RF_ACTIVE); 862 if (bar->pci_resource == NULL) 863 return (ENXIO); 864 865 save_bar_parameters(bar); 866 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 867 print_map_success(ntb, bar, "mmr"); 868 return (0); 869} 870 871static int 872map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 873{ 874 int rc; 875 vm_memattr_t mapmode; 876 uint8_t bar_size_bits = 0; 877 878 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 879 &bar->pci_resource_id, RF_ACTIVE); 880 881 if (bar->pci_resource == NULL) 882 return (ENXIO); 883 884 save_bar_parameters(bar); 885 /* 886 * Ivytown NTB BAR sizes are misreported by the hardware due to a 887 * hardware issue. To work around this, query the size it should be 888 * configured to by the device and modify the resource to correspond to 889 * this new size. The BIOS on systems with this problem is required to 890 * provide enough address space to allow the driver to make this change 891 * safely. 892 * 893 * Ideally I could have just specified the size when I allocated the 894 * resource like: 895 * bus_alloc_resource(ntb->device, 896 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 897 * 1ul << bar_size_bits, RF_ACTIVE); 898 * but the PCI driver does not honor the size in this call, so we have 899 * to modify it after the fact. 900 */ 901 if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 902 if (bar->pci_resource_id == PCIR_BAR(2)) 903 bar_size_bits = pci_read_config(ntb->device, 904 XEON_PBAR23SZ_OFFSET, 1); 905 else 906 bar_size_bits = pci_read_config(ntb->device, 907 XEON_PBAR45SZ_OFFSET, 1); 908 909 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 910 bar->pci_resource, bar->pbase, 911 bar->pbase + (1ul << bar_size_bits) - 1); 912 if (rc != 0) { 913 device_printf(ntb->device, 914 "unable to resize bar\n"); 915 return (rc); 916 } 917 918 save_bar_parameters(bar); 919 } 920 921 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 922 print_map_success(ntb, bar, "mw"); 923 924 /* 925 * Optionally, mark MW BARs as anything other than UC to improve 926 * performance. 927 */ 928 mapmode = ntb_pat_flags(); 929 if (mapmode == bar->map_mode) 930 return (0); 931 932 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 933 if (rc == 0) { 934 bar->map_mode = mapmode; 935 device_printf(ntb->device, 936 "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 937 "%s.\n", 938 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 939 (char *)bar->vbase + bar->size - 1, 940 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 941 ntb_vm_memattr_to_str(mapmode)); 942 } else 943 device_printf(ntb->device, 944 "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 945 "%s: %d\n", 946 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 947 (char *)bar->vbase + bar->size - 1, 948 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 949 ntb_vm_memattr_to_str(mapmode), rc); 950 /* Proceed anyway */ 951 return (0); 952} 953 954static void 955ntb_unmap_pci_bar(struct ntb_softc *ntb) 956{ 957 struct ntb_pci_bar_info *current_bar; 958 int i; 959 960 for (i = 0; i < NTB_MAX_BARS; i++) { 961 current_bar = &ntb->bar_info[i]; 962 if (current_bar->pci_resource != NULL) 963 bus_release_resource(ntb->device, SYS_RES_MEMORY, 964 current_bar->pci_resource_id, 965 current_bar->pci_resource); 966 } 967} 968 969static int 970ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 971{ 972 uint32_t i; 973 int rc; 974 975 for (i = 0; i < num_vectors; i++) { 976 ntb->int_info[i].rid = i + 1; 977 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 978 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 979 if (ntb->int_info[i].res == NULL) { 980 device_printf(ntb->device, 981 "bus_alloc_resource failed\n"); 982 return (ENOMEM); 983 } 984 ntb->int_info[i].tag = NULL; 985 ntb->allocated_interrupts++; 986 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 987 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 988 &ntb->msix_vec[i], &ntb->int_info[i].tag); 989 if (rc != 0) { 990 device_printf(ntb->device, "bus_setup_intr failed\n"); 991 return (ENXIO); 992 } 993 } 994 return (0); 995} 996 997/* 998 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 999 * cannot be allocated for each MSI-X message. JHB seems to think remapping 1000 * should be okay. This tunable should enable us to test that hypothesis 1001 * when someone gets their hands on some Xeon hardware. 1002 */ 1003static int ntb_force_remap_mode; 1004TUNABLE_INT("hw.ntb.force_remap_mode", &ntb_force_remap_mode); 1005SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 1006 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 1007 " to a smaller number of ithreads, even if the desired number are " 1008 "available"); 1009 1010/* 1011 * In case it is NOT ok, give consumers an abort button. 1012 */ 1013static int ntb_prefer_intx; 1014TUNABLE_INT("hw.ntb.prefer_intx_to_remap", &ntb_prefer_intx); 1015SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 1016 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1017 "than remapping MSI-X messages over available slots (match Linux driver " 1018 "behavior)"); 1019 1020/* 1021 * Remap the desired number of MSI-X messages to available ithreads in a simple 1022 * round-robin fashion. 1023 */ 1024static int 1025ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1026{ 1027 u_int *vectors; 1028 uint32_t i; 1029 int rc; 1030 1031 if (ntb_prefer_intx != 0) 1032 return (ENXIO); 1033 1034 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1035 1036 for (i = 0; i < desired; i++) 1037 vectors[i] = (i % avail) + 1; 1038 1039 rc = pci_remap_msix(dev, desired, vectors); 1040 free(vectors, M_NTB); 1041 return (rc); 1042} 1043 1044static int 1045ntb_init_isr(struct ntb_softc *ntb) 1046{ 1047 uint32_t desired_vectors, num_vectors; 1048 int rc; 1049 1050 ntb->allocated_interrupts = 0; 1051 ntb->last_ts = ticks; 1052 1053 /* 1054 * Mask all doorbell interrupts. (Except link events!) 1055 */ 1056 DB_MASK_LOCK(ntb); 1057 ntb->db_mask = ntb->db_valid_mask; 1058 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1059 DB_MASK_UNLOCK(ntb); 1060 1061 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1062 ntb->db_count); 1063 if (desired_vectors >= 1) { 1064 rc = pci_alloc_msix(ntb->device, &num_vectors); 1065 1066 if (ntb_force_remap_mode != 0 && rc == 0 && 1067 num_vectors == desired_vectors) 1068 num_vectors--; 1069 1070 if (rc == 0 && num_vectors < desired_vectors) { 1071 rc = ntb_remap_msix(ntb->device, desired_vectors, 1072 num_vectors); 1073 if (rc == 0) 1074 num_vectors = desired_vectors; 1075 else 1076 pci_release_msi(ntb->device); 1077 } 1078 if (rc != 0) 1079 num_vectors = 1; 1080 } else 1081 num_vectors = 1; 1082 1083 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1084 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1085 device_printf(ntb->device, 1086 "Errata workaround does not support MSI or INTX\n"); 1087 return (EINVAL); 1088 } 1089 1090 ntb->db_vec_count = 1; 1091 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1092 rc = ntb_setup_legacy_interrupt(ntb); 1093 } else { 1094 if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1095 HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1096 device_printf(ntb->device, 1097 "Errata workaround expects %d doorbell bits\n", 1098 XEON_NONLINK_DB_MSIX_BITS); 1099 return (EINVAL); 1100 } 1101 1102 ntb_create_msix_vec(ntb, num_vectors); 1103 rc = ntb_setup_msix(ntb, num_vectors); 1104 if (rc == 0 && HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1105 ntb_get_msix_info(ntb); 1106 } 1107 if (rc != 0) { 1108 device_printf(ntb->device, 1109 "Error allocating interrupts: %d\n", rc); 1110 ntb_free_msix_vec(ntb); 1111 } 1112 1113 return (rc); 1114} 1115 1116static int 1117ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1118{ 1119 int rc; 1120 1121 ntb->int_info[0].rid = 0; 1122 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1123 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1124 if (ntb->int_info[0].res == NULL) { 1125 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1126 return (ENOMEM); 1127 } 1128 1129 ntb->int_info[0].tag = NULL; 1130 ntb->allocated_interrupts = 1; 1131 1132 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1133 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1134 ntb, &ntb->int_info[0].tag); 1135 if (rc != 0) { 1136 device_printf(ntb->device, "bus_setup_intr failed\n"); 1137 return (ENXIO); 1138 } 1139 1140 return (0); 1141} 1142 1143static void 1144ntb_teardown_interrupts(struct ntb_softc *ntb) 1145{ 1146 struct ntb_int_info *current_int; 1147 int i; 1148 1149 for (i = 0; i < ntb->allocated_interrupts; i++) { 1150 current_int = &ntb->int_info[i]; 1151 if (current_int->tag != NULL) 1152 bus_teardown_intr(ntb->device, current_int->res, 1153 current_int->tag); 1154 1155 if (current_int->res != NULL) 1156 bus_release_resource(ntb->device, SYS_RES_IRQ, 1157 rman_get_rid(current_int->res), current_int->res); 1158 } 1159 1160 ntb_free_msix_vec(ntb); 1161 pci_release_msi(ntb->device); 1162} 1163 1164/* 1165 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1166 * out to make code clearer. 1167 */ 1168static inline uint64_t 1169db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1170{ 1171 1172 if (ntb->type == NTB_ATOM) 1173 return (ntb_reg_read(8, regoff)); 1174 1175 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1176 1177 return (ntb_reg_read(2, regoff)); 1178} 1179 1180static inline void 1181db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1182{ 1183 1184 KASSERT((val & ~ntb->db_valid_mask) == 0, 1185 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1186 (uintmax_t)(val & ~ntb->db_valid_mask), 1187 (uintmax_t)ntb->db_valid_mask)); 1188 1189 if (regoff == ntb->self_reg->db_mask) 1190 DB_MASK_ASSERT(ntb, MA_OWNED); 1191 db_iowrite_raw(ntb, regoff, val); 1192} 1193 1194static inline void 1195db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1196{ 1197 1198 if (ntb->type == NTB_ATOM) { 1199 ntb_reg_write(8, regoff, val); 1200 return; 1201 } 1202 1203 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1204 ntb_reg_write(2, regoff, (uint16_t)val); 1205} 1206 1207static void 1208ntb_db_set_mask(device_t dev, uint64_t bits) 1209{ 1210 struct ntb_softc *ntb = device_get_softc(dev); 1211 1212 DB_MASK_LOCK(ntb); 1213 ntb->db_mask |= bits; 1214 if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1215 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1216 DB_MASK_UNLOCK(ntb); 1217} 1218 1219static void 1220ntb_db_clear_mask(device_t dev, uint64_t bits) 1221{ 1222 struct ntb_softc *ntb = device_get_softc(dev); 1223 uint64_t ibits; 1224 int i; 1225 1226 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1227 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1228 (uintmax_t)(bits & ~ntb->db_valid_mask), 1229 (uintmax_t)ntb->db_valid_mask)); 1230 1231 DB_MASK_LOCK(ntb); 1232 ibits = ntb->fake_db_bell & ntb->db_mask & bits; 1233 ntb->db_mask &= ~bits; 1234 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1235 /* Simulate fake interrupts if unmasked DB bits are set. */ 1236 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1237 if ((ibits & ntb_db_vector_mask(dev, i)) != 0) 1238 swi_sched(ntb->int_info[i].tag, 0); 1239 } 1240 } else { 1241 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1242 } 1243 DB_MASK_UNLOCK(ntb); 1244} 1245 1246static uint64_t 1247ntb_db_read(device_t dev) 1248{ 1249 struct ntb_softc *ntb = device_get_softc(dev); 1250 1251 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1252 return (ntb->fake_db_bell); 1253 1254 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1255} 1256 1257static void 1258ntb_db_clear(device_t dev, uint64_t bits) 1259{ 1260 struct ntb_softc *ntb = device_get_softc(dev); 1261 1262 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1263 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1264 (uintmax_t)(bits & ~ntb->db_valid_mask), 1265 (uintmax_t)ntb->db_valid_mask)); 1266 1267 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1268 DB_MASK_LOCK(ntb); 1269 ntb->fake_db_bell &= ~bits; 1270 DB_MASK_UNLOCK(ntb); 1271 return; 1272 } 1273 1274 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1275} 1276 1277static inline uint64_t 1278ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1279{ 1280 uint64_t shift, mask; 1281 1282 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1283 /* 1284 * Remap vectors in custom way to make at least first 1285 * three doorbells to not generate stray events. 1286 * This breaks Linux compatibility (if one existed) 1287 * when more then one DB is used (not by if_ntb). 1288 */ 1289 if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1) 1290 return (1 << db_vector); 1291 if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1) 1292 return (0x7ffc); 1293 } 1294 1295 shift = ntb->db_vec_shift; 1296 mask = (1ull << shift) - 1; 1297 return (mask << (shift * db_vector)); 1298} 1299 1300static void 1301ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1302{ 1303 uint64_t vec_mask; 1304 1305 ntb->last_ts = ticks; 1306 vec_mask = ntb_vec_mask(ntb, vec); 1307 1308 if ((vec_mask & ntb->db_link_mask) != 0) { 1309 if (ntb_poll_link(ntb)) 1310 ntb_link_event(ntb->device); 1311 } 1312 1313 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1314 (vec_mask & ntb->db_link_mask) == 0) { 1315 DB_MASK_LOCK(ntb); 1316 1317 /* Do not report same DB events again if not cleared yet. */ 1318 vec_mask &= ~ntb->fake_db_bell; 1319 1320 /* Update our internal doorbell register. */ 1321 ntb->fake_db_bell |= vec_mask; 1322 1323 /* Do not report masked DB events. */ 1324 vec_mask &= ~ntb->db_mask; 1325 1326 DB_MASK_UNLOCK(ntb); 1327 } 1328 1329 if ((vec_mask & ntb->db_valid_mask) != 0) 1330 ntb_db_event(ntb->device, vec); 1331} 1332 1333static void 1334ndev_vec_isr(void *arg) 1335{ 1336 struct ntb_vec *nvec = arg; 1337 1338 ntb_interrupt(nvec->ntb, nvec->num); 1339} 1340 1341static void 1342ndev_irq_isr(void *arg) 1343{ 1344 /* If we couldn't set up MSI-X, we only have the one vector. */ 1345 ntb_interrupt(arg, 0); 1346} 1347 1348static int 1349ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1350{ 1351 uint32_t i; 1352 1353 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1354 M_ZERO | M_WAITOK); 1355 for (i = 0; i < num_vectors; i++) { 1356 ntb->msix_vec[i].num = i; 1357 ntb->msix_vec[i].ntb = ntb; 1358 } 1359 1360 return (0); 1361} 1362 1363static void 1364ntb_free_msix_vec(struct ntb_softc *ntb) 1365{ 1366 1367 if (ntb->msix_vec == NULL) 1368 return; 1369 1370 free(ntb->msix_vec, M_NTB); 1371 ntb->msix_vec = NULL; 1372} 1373 1374static void 1375ntb_get_msix_info(struct ntb_softc *ntb) 1376{ 1377 struct pci_devinfo *dinfo; 1378 struct pcicfg_msix *msix; 1379 uint32_t laddr, data, i, offset; 1380 1381 dinfo = device_get_ivars(ntb->device); 1382 msix = &dinfo->cfg.msix; 1383 1384 CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1385 1386 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1387 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1388 1389 laddr = bus_read_4(msix->msix_table_res, offset + 1390 PCI_MSIX_ENTRY_LOWER_ADDR); 1391 ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1392 1393 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1394 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1395 MSI_INTEL_ADDR_BASE)); 1396 ntb->msix_data[i].nmd_ofs = laddr; 1397 1398 data = bus_read_4(msix->msix_table_res, offset + 1399 PCI_MSIX_ENTRY_DATA); 1400 ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1401 1402 ntb->msix_data[i].nmd_data = data; 1403 } 1404} 1405 1406static struct ntb_hw_info * 1407ntb_get_device_info(uint32_t device_id) 1408{ 1409 struct ntb_hw_info *ep = pci_ids; 1410 1411 while (ep->device_id) { 1412 if (ep->device_id == device_id) 1413 return (ep); 1414 ++ep; 1415 } 1416 return (NULL); 1417} 1418 1419static void 1420ntb_teardown_xeon(struct ntb_softc *ntb) 1421{ 1422 1423 if (ntb->reg != NULL) 1424 ntb_link_disable(ntb->device); 1425} 1426 1427static void 1428ntb_detect_max_mw(struct ntb_softc *ntb) 1429{ 1430 1431 if (ntb->type == NTB_ATOM) { 1432 ntb->mw_count = ATOM_MW_COUNT; 1433 return; 1434 } 1435 1436 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1437 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1438 else 1439 ntb->mw_count = XEON_SNB_MW_COUNT; 1440} 1441 1442static int 1443ntb_detect_xeon(struct ntb_softc *ntb) 1444{ 1445 uint8_t ppd, conn_type; 1446 1447 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1448 ntb->ppd = ppd; 1449 1450 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1451 ntb->dev_type = NTB_DEV_DSD; 1452 else 1453 ntb->dev_type = NTB_DEV_USD; 1454 1455 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1456 ntb->features |= NTB_SPLIT_BAR; 1457 1458 /* 1459 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1460 * errata workaround; only do one at a time. 1461 */ 1462 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1463 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1464 1465 conn_type = ppd & XEON_PPD_CONN_TYPE; 1466 switch (conn_type) { 1467 case NTB_CONN_B2B: 1468 ntb->conn_type = conn_type; 1469 break; 1470 case NTB_CONN_RP: 1471 case NTB_CONN_TRANSPARENT: 1472 default: 1473 device_printf(ntb->device, "Unsupported connection type: %u\n", 1474 (unsigned)conn_type); 1475 return (ENXIO); 1476 } 1477 return (0); 1478} 1479 1480static int 1481ntb_detect_atom(struct ntb_softc *ntb) 1482{ 1483 uint32_t ppd, conn_type; 1484 1485 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1486 ntb->ppd = ppd; 1487 1488 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1489 ntb->dev_type = NTB_DEV_DSD; 1490 else 1491 ntb->dev_type = NTB_DEV_USD; 1492 1493 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1494 switch (conn_type) { 1495 case NTB_CONN_B2B: 1496 ntb->conn_type = conn_type; 1497 break; 1498 default: 1499 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1500 return (ENXIO); 1501 } 1502 return (0); 1503} 1504 1505static int 1506ntb_xeon_init_dev(struct ntb_softc *ntb) 1507{ 1508 int rc; 1509 1510 ntb->spad_count = XEON_SPAD_COUNT; 1511 ntb->db_count = XEON_DB_COUNT; 1512 ntb->db_link_mask = XEON_DB_LINK_BIT; 1513 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1514 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1515 1516 if (ntb->conn_type != NTB_CONN_B2B) { 1517 device_printf(ntb->device, "Connection type %d not supported\n", 1518 ntb->conn_type); 1519 return (ENXIO); 1520 } 1521 1522 ntb->reg = &xeon_reg; 1523 ntb->self_reg = &xeon_pri_reg; 1524 ntb->peer_reg = &xeon_b2b_reg; 1525 ntb->xlat_reg = &xeon_sec_xlat; 1526 1527 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1528 ntb->fake_db_bell = 0; 1529 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1530 ntb->mw_count; 1531 ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1532 g_ntb_msix_idx, ntb->msix_mw_idx); 1533 rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1534 VM_MEMATTR_UNCACHEABLE); 1535 KASSERT(rc == 0, ("shouldn't fail")); 1536 } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1537 /* 1538 * There is a Xeon hardware errata related to writes to SDOORBELL or 1539 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1540 * which may hang the system. To workaround this, use a memory 1541 * window to access the interrupt and scratch pad registers on the 1542 * remote system. 1543 */ 1544 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1545 ntb->mw_count; 1546 ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1547 g_ntb_mw_idx, ntb->b2b_mw_idx); 1548 rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1549 VM_MEMATTR_UNCACHEABLE); 1550 KASSERT(rc == 0, ("shouldn't fail")); 1551 } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1552 /* 1553 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1554 * mirrored to the remote system. Shrink the number of bits by one, 1555 * since bit 14 is the last bit. 1556 * 1557 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1558 * anyway. Nor for non-B2B connection types. 1559 */ 1560 ntb->db_count = XEON_DB_COUNT - 1; 1561 1562 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1563 1564 if (ntb->dev_type == NTB_DEV_USD) 1565 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1566 &xeon_b2b_usd_addr); 1567 else 1568 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1569 &xeon_b2b_dsd_addr); 1570 if (rc != 0) 1571 return (rc); 1572 1573 /* Enable Bus Master and Memory Space on the secondary side */ 1574 ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1575 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1576 1577 /* 1578 * Mask all doorbell interrupts. 1579 */ 1580 DB_MASK_LOCK(ntb); 1581 ntb->db_mask = ntb->db_valid_mask; 1582 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1583 DB_MASK_UNLOCK(ntb); 1584 1585 rc = ntb_init_isr(ntb); 1586 return (rc); 1587} 1588 1589static int 1590ntb_atom_init_dev(struct ntb_softc *ntb) 1591{ 1592 int error; 1593 1594 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1595 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1596 1597 ntb->spad_count = ATOM_SPAD_COUNT; 1598 ntb->db_count = ATOM_DB_COUNT; 1599 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1600 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1601 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1602 1603 ntb->reg = &atom_reg; 1604 ntb->self_reg = &atom_pri_reg; 1605 ntb->peer_reg = &atom_b2b_reg; 1606 ntb->xlat_reg = &atom_sec_xlat; 1607 1608 /* 1609 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1610 * resolved. Mask transaction layer internal parity errors. 1611 */ 1612 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1613 1614 configure_atom_secondary_side_bars(ntb); 1615 1616 /* Enable Bus Master and Memory Space on the secondary side */ 1617 ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1618 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1619 1620 error = ntb_init_isr(ntb); 1621 if (error != 0) 1622 return (error); 1623 1624 /* Initiate PCI-E link training */ 1625 ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1626 1627 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1628 1629 return (0); 1630} 1631 1632/* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1633static void 1634configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1635{ 1636 1637 if (ntb->dev_type == NTB_DEV_USD) { 1638 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1639 XEON_B2B_BAR2_ADDR64); 1640 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1641 XEON_B2B_BAR4_ADDR64); 1642 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1643 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1644 } else { 1645 ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1646 XEON_B2B_BAR2_ADDR64); 1647 ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1648 XEON_B2B_BAR4_ADDR64); 1649 ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1650 ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1651 } 1652} 1653 1654 1655/* 1656 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1657 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1658 * remains for use by a higher layer. 1659 * 1660 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1661 * MW size is sufficiently large. 1662 */ 1663static unsigned int ntb_b2b_mw_share; 1664TUNABLE_INT("hw.ntb.b2b_mw_share", &ntb_b2b_mw_share); 1665SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1666 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1667 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1668 "value here."); 1669 1670static void 1671xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1672 enum ntb_bar regbar) 1673{ 1674 struct ntb_pci_bar_info *bar; 1675 uint8_t bar_sz; 1676 1677 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1678 return; 1679 1680 bar = &ntb->bar_info[idx]; 1681 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1682 if (idx == regbar) { 1683 if (ntb->b2b_off != 0) 1684 bar_sz--; 1685 else 1686 bar_sz = 0; 1687 } 1688 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1689 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1690 (void)bar_sz; 1691} 1692 1693static void 1694xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1695 enum ntb_bar idx, enum ntb_bar regbar) 1696{ 1697 uint64_t reg_val; 1698 uint32_t base_reg, lmt_reg; 1699 1700 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1701 if (idx == regbar) { 1702 if (ntb->b2b_off) 1703 bar_addr += ntb->b2b_off; 1704 else 1705 bar_addr = 0; 1706 } 1707 1708 /* 1709 * Set limit registers first to avoid an errata where setting the base 1710 * registers locks the limit registers. 1711 */ 1712 if (!bar_is_64bit(ntb, idx)) { 1713 ntb_reg_write(4, lmt_reg, bar_addr); 1714 reg_val = ntb_reg_read(4, lmt_reg); 1715 (void)reg_val; 1716 1717 ntb_reg_write(4, base_reg, bar_addr); 1718 reg_val = ntb_reg_read(4, base_reg); 1719 (void)reg_val; 1720 } else { 1721 ntb_reg_write(8, lmt_reg, bar_addr); 1722 reg_val = ntb_reg_read(8, lmt_reg); 1723 (void)reg_val; 1724 1725 ntb_reg_write(8, base_reg, bar_addr); 1726 reg_val = ntb_reg_read(8, base_reg); 1727 (void)reg_val; 1728 } 1729} 1730 1731static void 1732xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1733{ 1734 struct ntb_pci_bar_info *bar; 1735 1736 bar = &ntb->bar_info[idx]; 1737 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1738 ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1739 base_addr = ntb_reg_read(4, bar->pbarxlat_off); 1740 } else { 1741 ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1742 base_addr = ntb_reg_read(8, bar->pbarxlat_off); 1743 } 1744 (void)base_addr; 1745} 1746 1747static int 1748xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1749 const struct ntb_b2b_addr *peer_addr) 1750{ 1751 struct ntb_pci_bar_info *b2b_bar; 1752 vm_size_t bar_size; 1753 uint64_t bar_addr; 1754 enum ntb_bar b2b_bar_num, i; 1755 1756 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1757 b2b_bar = NULL; 1758 b2b_bar_num = NTB_CONFIG_BAR; 1759 ntb->b2b_off = 0; 1760 } else { 1761 b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1762 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1763 ("invalid b2b mw bar")); 1764 1765 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1766 bar_size = b2b_bar->size; 1767 1768 if (ntb_b2b_mw_share != 0 && 1769 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1770 ntb->b2b_off = bar_size >> 1; 1771 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1772 ntb->b2b_off = 0; 1773 } else { 1774 device_printf(ntb->device, 1775 "B2B bar size is too small!\n"); 1776 return (EIO); 1777 } 1778 } 1779 1780 /* 1781 * Reset the secondary bar sizes to match the primary bar sizes. 1782 * (Except, disable or halve the size of the B2B secondary bar.) 1783 */ 1784 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1785 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1786 1787 bar_addr = 0; 1788 if (b2b_bar_num == NTB_CONFIG_BAR) 1789 bar_addr = addr->bar0_addr; 1790 else if (b2b_bar_num == NTB_B2B_BAR_1) 1791 bar_addr = addr->bar2_addr64; 1792 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1793 bar_addr = addr->bar4_addr64; 1794 else if (b2b_bar_num == NTB_B2B_BAR_2) 1795 bar_addr = addr->bar4_addr32; 1796 else if (b2b_bar_num == NTB_B2B_BAR_3) 1797 bar_addr = addr->bar5_addr32; 1798 else 1799 KASSERT(false, ("invalid bar")); 1800 1801 ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1802 1803 /* 1804 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1805 * register BAR. The B2B BAR is either disabled above or configured 1806 * half-size. It starts at PBAR xlat + offset. 1807 * 1808 * Also set up incoming BAR limits == base (zero length window). 1809 */ 1810 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1811 b2b_bar_num); 1812 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1813 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1814 NTB_B2B_BAR_2, b2b_bar_num); 1815 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1816 NTB_B2B_BAR_3, b2b_bar_num); 1817 } else 1818 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1819 NTB_B2B_BAR_2, b2b_bar_num); 1820 1821 /* Zero incoming translation addrs */ 1822 ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1823 ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1824 1825 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1826 size_t size, xlatoffset; 1827 enum ntb_bar bar_num; 1828 1829 bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1830 switch (bar_num) { 1831 case NTB_B2B_BAR_1: 1832 size = 8; 1833 xlatoffset = XEON_SBAR2XLAT_OFFSET; 1834 break; 1835 case NTB_B2B_BAR_2: 1836 xlatoffset = XEON_SBAR4XLAT_OFFSET; 1837 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1838 size = 4; 1839 else 1840 size = 8; 1841 break; 1842 case NTB_B2B_BAR_3: 1843 xlatoffset = XEON_SBAR5XLAT_OFFSET; 1844 size = 4; 1845 break; 1846 default: 1847 KASSERT(false, ("Bogus msix mw idx: %u", 1848 ntb->msix_mw_idx)); 1849 return (EINVAL); 1850 } 1851 1852 /* 1853 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1854 * workaround 1855 */ 1856 if (size == 4) { 1857 ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); 1858 ntb->msix_xlat = ntb_reg_read(4, xlatoffset); 1859 } else { 1860 ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); 1861 ntb->msix_xlat = ntb_reg_read(8, xlatoffset); 1862 } 1863 1864 ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1865 } 1866 (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1867 (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1868 1869 /* Zero outgoing translation limits (whole bar size windows) */ 1870 ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1871 ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1872 1873 /* Set outgoing translation offsets */ 1874 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1875 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1876 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1877 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1878 } else 1879 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1880 1881 /* Set the translation offset for B2B registers */ 1882 bar_addr = 0; 1883 if (b2b_bar_num == NTB_CONFIG_BAR) 1884 bar_addr = peer_addr->bar0_addr; 1885 else if (b2b_bar_num == NTB_B2B_BAR_1) 1886 bar_addr = peer_addr->bar2_addr64; 1887 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1888 bar_addr = peer_addr->bar4_addr64; 1889 else if (b2b_bar_num == NTB_B2B_BAR_2) 1890 bar_addr = peer_addr->bar4_addr32; 1891 else if (b2b_bar_num == NTB_B2B_BAR_3) 1892 bar_addr = peer_addr->bar5_addr32; 1893 else 1894 KASSERT(false, ("invalid bar")); 1895 1896 /* 1897 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1898 * at a time. 1899 */ 1900 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1901 ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1902 return (0); 1903} 1904 1905static inline bool 1906_xeon_link_is_up(struct ntb_softc *ntb) 1907{ 1908 1909 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1910 return (true); 1911 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1912} 1913 1914static inline bool 1915link_is_up(struct ntb_softc *ntb) 1916{ 1917 1918 if (ntb->type == NTB_XEON) 1919 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1920 !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 1921 1922 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1923 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1924} 1925 1926static inline bool 1927atom_link_is_err(struct ntb_softc *ntb) 1928{ 1929 uint32_t status; 1930 1931 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1932 1933 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1934 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1935 return (true); 1936 1937 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1938 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1939} 1940 1941/* Atom does not have link status interrupt, poll on that platform */ 1942static void 1943atom_link_hb(void *arg) 1944{ 1945 struct ntb_softc *ntb = arg; 1946 sbintime_t timo, poll_ts; 1947 1948 timo = NTB_HB_TIMEOUT * hz; 1949 poll_ts = ntb->last_ts + timo; 1950 1951 /* 1952 * Delay polling the link status if an interrupt was received, unless 1953 * the cached link status says the link is down. 1954 */ 1955 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1956 timo = poll_ts - ticks; 1957 goto out; 1958 } 1959 1960 if (ntb_poll_link(ntb)) 1961 ntb_link_event(ntb->device); 1962 1963 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1964 /* Link is down with error, proceed with recovery */ 1965 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1966 return; 1967 } 1968 1969out: 1970 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1971} 1972 1973static void 1974atom_perform_link_restart(struct ntb_softc *ntb) 1975{ 1976 uint32_t status; 1977 1978 /* Driver resets the NTB ModPhy lanes - magic! */ 1979 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1980 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1981 ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1982 ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1983 1984 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1985 pause("ModPhy", hz / 10); 1986 1987 /* Clear AER Errors, write to clear */ 1988 status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1989 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1990 ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1991 1992 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1993 status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1994 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1995 ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1996 1997 /* Clear DeSkew Buffer error, write to clear */ 1998 status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1999 status |= ATOM_DESKEWSTS_DBERR; 2000 ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 2001 2002 status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2003 status &= ATOM_IBIST_ERR_OFLOW; 2004 ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2005 2006 /* Releases the NTB state machine to allow the link to retrain */ 2007 status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2008 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2009 ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2010} 2011 2012static int 2013ntb_set_ctx(device_t dev, void *ctx, const struct ntb_ctx_ops *ops) 2014{ 2015 struct ntb_softc *ntb = device_get_softc(dev); 2016 2017 if (ctx == NULL || ops == NULL) 2018 return (EINVAL); 2019 2020 rm_wlock(&ntb->ctx_lock); 2021 if (ntb->ctx_ops != NULL) { 2022 rm_wunlock(&ntb->ctx_lock); 2023 return (EINVAL); 2024 } 2025 ntb->ntb_ctx = ctx; 2026 ntb->ctx_ops = ops; 2027 rm_wunlock(&ntb->ctx_lock); 2028 2029 return (0); 2030} 2031 2032/* 2033 * It is expected that this will only be used from contexts where the ctx_lock 2034 * is not needed to protect ntb_ctx lifetime. 2035 */ 2036static void * 2037ntb_get_ctx(device_t dev, const struct ntb_ctx_ops **ops) 2038{ 2039 struct ntb_softc *ntb = device_get_softc(dev); 2040 2041 KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); 2042 if (ops != NULL) 2043 *ops = ntb->ctx_ops; 2044 return (ntb->ntb_ctx); 2045} 2046 2047static void 2048ntb_clear_ctx(device_t dev) 2049{ 2050 struct ntb_softc *ntb = device_get_softc(dev); 2051 2052 rm_wlock(&ntb->ctx_lock); 2053 ntb->ntb_ctx = NULL; 2054 ntb->ctx_ops = NULL; 2055 rm_wunlock(&ntb->ctx_lock); 2056} 2057 2058/* 2059 * ntb_link_event() - notify driver context of a change in link status 2060 * @ntb: NTB device context 2061 * 2062 * Notify the driver context that the link status may have changed. The driver 2063 * should call ntb_link_is_up() to get the current status. 2064 */ 2065static void 2066ntb_link_event(device_t dev) 2067{ 2068 struct ntb_softc *ntb = device_get_softc(dev); 2069 struct rm_priotracker ctx_tracker; 2070 2071 rm_rlock(&ntb->ctx_lock, &ctx_tracker); 2072 if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) 2073 ntb->ctx_ops->link_event(ntb->ntb_ctx); 2074 rm_runlock(&ntb->ctx_lock, &ctx_tracker); 2075} 2076 2077/* 2078 * ntb_db_event() - notify driver context of a doorbell event 2079 * @ntb: NTB device context 2080 * @vector: Interrupt vector number 2081 * 2082 * Notify the driver context of a doorbell event. If hardware supports 2083 * multiple interrupt vectors for doorbells, the vector number indicates which 2084 * vector received the interrupt. The vector number is relative to the first 2085 * vector used for doorbells, starting at zero, and must be less than 2086 * ntb_db_vector_count(). The driver may call ntb_db_read() to check which 2087 * doorbell bits need service, and ntb_db_vector_mask() to determine which of 2088 * those bits are associated with the vector number. 2089 */ 2090static void 2091ntb_db_event(device_t dev, uint32_t vec) 2092{ 2093 struct ntb_softc *ntb = device_get_softc(dev); 2094 struct rm_priotracker ctx_tracker; 2095 2096 rm_rlock(&ntb->ctx_lock, &ctx_tracker); 2097 if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) 2098 ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); 2099 rm_runlock(&ntb->ctx_lock, &ctx_tracker); 2100} 2101 2102static int 2103ntb_link_enable(device_t dev, enum ntb_speed speed __unused, 2104 enum ntb_width width __unused) 2105{ 2106 struct ntb_softc *ntb = device_get_softc(dev); 2107 uint32_t cntl; 2108 2109 ntb_printf(2, "%s\n", __func__); 2110 2111 if (ntb->type == NTB_ATOM) { 2112 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2113 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2114 return (0); 2115 } 2116 2117 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2118 ntb_link_event(dev); 2119 return (0); 2120 } 2121 2122 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2123 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2124 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2125 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2126 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2127 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2128 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2129 return (0); 2130} 2131 2132static int 2133ntb_link_disable(device_t dev) 2134{ 2135 struct ntb_softc *ntb = device_get_softc(dev); 2136 uint32_t cntl; 2137 2138 ntb_printf(2, "%s\n", __func__); 2139 2140 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2141 ntb_link_event(dev); 2142 return (0); 2143 } 2144 2145 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2146 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2147 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2148 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2149 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2150 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2151 ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2152 return (0); 2153} 2154 2155static bool 2156ntb_link_enabled(device_t dev) 2157{ 2158 struct ntb_softc *ntb = device_get_softc(dev); 2159 uint32_t cntl; 2160 2161 if (ntb->type == NTB_ATOM) { 2162 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2163 return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2164 } 2165 2166 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2167 return (true); 2168 2169 cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2170 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2171} 2172 2173static void 2174recover_atom_link(void *arg) 2175{ 2176 struct ntb_softc *ntb = arg; 2177 unsigned speed, width, oldspeed, oldwidth; 2178 uint32_t status32; 2179 2180 atom_perform_link_restart(ntb); 2181 2182 /* 2183 * There is a potential race between the 2 NTB devices recovering at 2184 * the same time. If the times are the same, the link will not recover 2185 * and the driver will be stuck in this loop forever. Add a random 2186 * interval to the recovery time to prevent this race. 2187 */ 2188 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2189 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2190 2191 if (atom_link_is_err(ntb)) 2192 goto retry; 2193 2194 status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); 2195 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2196 goto out; 2197 2198 status32 = ntb_reg_read(4, ntb->reg->lnk_sta); 2199 width = NTB_LNK_STA_WIDTH(status32); 2200 speed = status32 & NTB_LINK_SPEED_MASK; 2201 2202 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2203 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2204 if (oldwidth != width || oldspeed != speed) 2205 goto retry; 2206 2207out: 2208 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2209 ntb); 2210 return; 2211 2212retry: 2213 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2214 ntb); 2215} 2216 2217/* 2218 * Polls the HW link status register(s); returns true if something has changed. 2219 */ 2220static bool 2221ntb_poll_link(struct ntb_softc *ntb) 2222{ 2223 uint32_t ntb_cntl; 2224 uint16_t reg_val; 2225 2226 if (ntb->type == NTB_ATOM) { 2227 ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); 2228 if (ntb_cntl == ntb->ntb_ctl) 2229 return (false); 2230 2231 ntb->ntb_ctl = ntb_cntl; 2232 ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); 2233 } else { 2234 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2235 2236 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2237 if (reg_val == ntb->lnk_sta) 2238 return (false); 2239 2240 ntb->lnk_sta = reg_val; 2241 2242 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2243 if (_xeon_link_is_up(ntb)) { 2244 if (!ntb->peer_msix_good) { 2245 callout_reset(&ntb->peer_msix_work, 0, 2246 ntb_exchange_msix, ntb); 2247 return (false); 2248 } 2249 } else { 2250 ntb->peer_msix_good = false; 2251 ntb->peer_msix_done = false; 2252 } 2253 } 2254 } 2255 return (true); 2256} 2257 2258static inline enum ntb_speed 2259ntb_link_sta_speed(struct ntb_softc *ntb) 2260{ 2261 2262 if (!link_is_up(ntb)) 2263 return (NTB_SPEED_NONE); 2264 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2265} 2266 2267static inline enum ntb_width 2268ntb_link_sta_width(struct ntb_softc *ntb) 2269{ 2270 2271 if (!link_is_up(ntb)) 2272 return (NTB_WIDTH_NONE); 2273 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2274} 2275 2276SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2277 "Driver state, statistics, and HW registers"); 2278 2279#define NTB_REGSZ_MASK (3ul << 30) 2280#define NTB_REG_64 (1ul << 30) 2281#define NTB_REG_32 (2ul << 30) 2282#define NTB_REG_16 (3ul << 30) 2283#define NTB_REG_8 (0ul << 30) 2284 2285#define NTB_DB_READ (1ul << 29) 2286#define NTB_PCI_REG (1ul << 28) 2287#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2288 2289static void 2290ntb_sysctl_init(struct ntb_softc *ntb) 2291{ 2292 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2293 struct sysctl_ctx_list *ctx; 2294 struct sysctl_oid *tree, *tmptree; 2295 2296 ctx = device_get_sysctl_ctx(ntb->device); 2297 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2298 2299 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2300 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2301 sysctl_handle_link_status_human, "A", 2302 "Link status (human readable)"); 2303 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2304 CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2305 "IU", "Link status (1=active, 0=inactive)"); 2306 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2307 CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2308 "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2309 2310 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2311 CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2312 tree_par = SYSCTL_CHILDREN(tree); 2313 2314 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2315 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2316 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2317 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2318 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2319 &ntb->ppd, 0, "Raw PPD register (cached)"); 2320 2321 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2322#ifdef notyet 2323 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2324 &ntb->b2b_mw_idx, 0, 2325 "Index of the MW used for B2B remote register access"); 2326#endif 2327 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2328 CTLFLAG_RD, &ntb->b2b_off, 2329 "If non-zero, offset of B2B register region in shared MW"); 2330 } 2331 2332 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2333 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2334 "Features/errata of this NTB device"); 2335 2336 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2337 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2338 "NTB CTL register (cached)"); 2339 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2340 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2341 "LNK STA register (cached)"); 2342 2343#ifdef notyet 2344 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2345 &ntb->mw_count, 0, "MW count"); 2346 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2347 &ntb->spad_count, 0, "Scratchpad count"); 2348 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2349 &ntb->db_count, 0, "Doorbell count"); 2350 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2351 &ntb->db_vec_count, 0, "Doorbell vector count"); 2352 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2353 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2354#endif 2355 2356 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2357 &ntb->db_valid_mask, "Doorbell valid mask"); 2358 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2359 &ntb->db_link_mask, "Doorbell link mask"); 2360 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2361 &ntb->db_mask, "Doorbell mask (cached)"); 2362 2363 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2364 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2365 regpar = SYSCTL_CHILDREN(tmptree); 2366 2367 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2368 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2369 ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2370 "NTB Control register"); 2371 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2372 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2373 0x19c, sysctl_handle_register, "IU", 2374 "NTB Link Capabilities"); 2375 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2376 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2377 0x1a0, sysctl_handle_register, "IU", 2378 "NTB Link Control register"); 2379 2380 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2381 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2382 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2383 sysctl_handle_register, "QU", "Doorbell mask register"); 2384 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2385 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2386 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2387 sysctl_handle_register, "QU", "Doorbell register"); 2388 2389 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2390 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2391 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2392 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2393 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2394 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2395 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2396 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2397 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2398 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2399 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2400 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2401 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2402 } else { 2403 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2404 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2405 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2406 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2407 } 2408 2409 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2410 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2411 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2412 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2413 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2414 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2415 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2416 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2417 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2418 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2419 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2420 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2421 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2422 } else { 2423 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2424 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2425 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2426 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2427 } 2428 2429 if (ntb->type == NTB_ATOM) 2430 return; 2431 2432 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2433 CTLFLAG_RD, NULL, "Xeon HW statistics"); 2434 statpar = SYSCTL_CHILDREN(tmptree); 2435 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2436 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2437 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2438 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2439 2440 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2441 CTLFLAG_RD, NULL, "Xeon HW errors"); 2442 errpar = SYSCTL_CHILDREN(tmptree); 2443 2444 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2445 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2446 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2447 sysctl_handle_register, "CU", "PPD"); 2448 2449 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2450 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2451 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2452 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2453 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2454 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2455 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2456 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2457 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2458 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2459 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2460 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2461 2462 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2463 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2464 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2465 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2466 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2467 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2468 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2469 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2470 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2471 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2472 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2473 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2474 2475 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2476 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2477 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2478 sysctl_handle_register, "SU", "DEVSTS"); 2479 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2480 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2481 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2482 sysctl_handle_register, "SU", "LNKSTS"); 2483 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2484 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2485 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2486 sysctl_handle_register, "SU", "SLNKSTS"); 2487 2488 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2489 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2490 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2491 sysctl_handle_register, "IU", "UNCERRSTS"); 2492 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2493 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2494 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2495 sysctl_handle_register, "IU", "CORERRSTS"); 2496 2497 if (ntb->conn_type != NTB_CONN_B2B) 2498 return; 2499 2500 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2501 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2502 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2503 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2504 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2505 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2506 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2507 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2508 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2509 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2510 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2511 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2512 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2513 } else { 2514 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2515 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2516 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2517 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2518 } 2519 2520 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2521 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2522 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2523 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2524 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2525 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2526 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2527 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2528 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2529 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2530 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2531 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2532 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2533 } else { 2534 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2535 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2536 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2537 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2538 } 2539 2540 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2541 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2542 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2543 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2544 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2545 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2546 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2547 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2548 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2549 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2550 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2551 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2552 sysctl_handle_register, "IU", 2553 "Secondary BAR4 base register"); 2554 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2555 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2556 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2557 sysctl_handle_register, "IU", 2558 "Secondary BAR5 base register"); 2559 } else { 2560 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2561 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2562 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2563 sysctl_handle_register, "QU", 2564 "Secondary BAR45 base register"); 2565 } 2566} 2567 2568static int 2569sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2570{ 2571 struct ntb_softc *ntb = arg1; 2572 struct sbuf sb; 2573 int error; 2574 2575 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2576 2577 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2578 error = sbuf_finish(&sb); 2579 sbuf_delete(&sb); 2580 2581 if (error || !req->newptr) 2582 return (error); 2583 return (EINVAL); 2584} 2585 2586static int 2587sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2588{ 2589 struct ntb_softc *ntb = arg1; 2590 unsigned old, new; 2591 int error; 2592 2593 old = ntb_link_enabled(ntb->device); 2594 2595 error = SYSCTL_OUT(req, &old, sizeof(old)); 2596 if (error != 0 || req->newptr == NULL) 2597 return (error); 2598 2599 error = SYSCTL_IN(req, &new, sizeof(new)); 2600 if (error != 0) 2601 return (error); 2602 2603 ntb_printf(0, "Admin set interface state to '%sabled'\n", 2604 (new != 0)? "en" : "dis"); 2605 2606 if (new != 0) 2607 error = ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2608 else 2609 error = ntb_link_disable(ntb->device); 2610 return (error); 2611} 2612 2613static int 2614sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2615{ 2616 struct ntb_softc *ntb = arg1; 2617 struct sbuf sb; 2618 enum ntb_speed speed; 2619 enum ntb_width width; 2620 int error; 2621 2622 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2623 2624 if (ntb_link_is_up(ntb->device, &speed, &width)) 2625 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2626 (unsigned)speed, (unsigned)width); 2627 else 2628 sbuf_printf(&sb, "down"); 2629 2630 error = sbuf_finish(&sb); 2631 sbuf_delete(&sb); 2632 2633 if (error || !req->newptr) 2634 return (error); 2635 return (EINVAL); 2636} 2637 2638static int 2639sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2640{ 2641 struct ntb_softc *ntb = arg1; 2642 unsigned res; 2643 int error; 2644 2645 res = ntb_link_is_up(ntb->device, NULL, NULL); 2646 2647 error = SYSCTL_OUT(req, &res, sizeof(res)); 2648 if (error || !req->newptr) 2649 return (error); 2650 return (EINVAL); 2651} 2652 2653static int 2654sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2655{ 2656 struct ntb_softc *ntb; 2657 const void *outp; 2658 uintptr_t sz; 2659 uint64_t umv; 2660 char be[sizeof(umv)]; 2661 size_t outsz; 2662 uint32_t reg; 2663 bool db, pci; 2664 int error; 2665 2666 ntb = arg1; 2667 reg = arg2 & ~NTB_REGFLAGS_MASK; 2668 sz = arg2 & NTB_REGSZ_MASK; 2669 db = (arg2 & NTB_DB_READ) != 0; 2670 pci = (arg2 & NTB_PCI_REG) != 0; 2671 2672 KASSERT(!(db && pci), ("bogus")); 2673 2674 if (db) { 2675 KASSERT(sz == NTB_REG_64, ("bogus")); 2676 umv = db_ioread(ntb, reg); 2677 outsz = sizeof(uint64_t); 2678 } else { 2679 switch (sz) { 2680 case NTB_REG_64: 2681 if (pci) 2682 umv = pci_read_config(ntb->device, reg, 8); 2683 else 2684 umv = ntb_reg_read(8, reg); 2685 outsz = sizeof(uint64_t); 2686 break; 2687 case NTB_REG_32: 2688 if (pci) 2689 umv = pci_read_config(ntb->device, reg, 4); 2690 else 2691 umv = ntb_reg_read(4, reg); 2692 outsz = sizeof(uint32_t); 2693 break; 2694 case NTB_REG_16: 2695 if (pci) 2696 umv = pci_read_config(ntb->device, reg, 2); 2697 else 2698 umv = ntb_reg_read(2, reg); 2699 outsz = sizeof(uint16_t); 2700 break; 2701 case NTB_REG_8: 2702 if (pci) 2703 umv = pci_read_config(ntb->device, reg, 1); 2704 else 2705 umv = ntb_reg_read(1, reg); 2706 outsz = sizeof(uint8_t); 2707 break; 2708 default: 2709 panic("bogus"); 2710 break; 2711 } 2712 } 2713 2714 /* Encode bigendian so that sysctl -x is legible. */ 2715 be64enc(be, umv); 2716 outp = ((char *)be) + sizeof(umv) - outsz; 2717 2718 error = SYSCTL_OUT(req, outp, outsz); 2719 if (error || !req->newptr) 2720 return (error); 2721 return (EINVAL); 2722} 2723 2724static unsigned 2725ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2726{ 2727 2728 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2729 uidx >= ntb->b2b_mw_idx) || 2730 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2731 uidx++; 2732 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2733 uidx >= ntb->b2b_mw_idx) && 2734 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2735 uidx++; 2736 return (uidx); 2737} 2738 2739static void 2740ntb_exchange_msix(void *ctx) 2741{ 2742 struct ntb_softc *ntb; 2743 uint32_t val; 2744 unsigned i; 2745 2746 ntb = ctx; 2747 2748 if (ntb->peer_msix_good) 2749 goto msix_good; 2750 if (ntb->peer_msix_done) 2751 goto msix_done; 2752 2753 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2754 ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 2755 ntb->msix_data[i].nmd_data); 2756 ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 2757 ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2758 } 2759 ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2760 2761 ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 2762 if (val != NTB_MSIX_VER_GUARD) 2763 goto reschedule; 2764 2765 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2766 ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 2767 ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2768 ntb->peer_msix_data[i].nmd_data = val; 2769 ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 2770 ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2771 ntb->peer_msix_data[i].nmd_ofs = val; 2772 } 2773 2774 ntb->peer_msix_done = true; 2775 2776msix_done: 2777 ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2778 ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 2779 if (val != NTB_MSIX_RECEIVED) 2780 goto reschedule; 2781 2782 ntb->peer_msix_good = true; 2783 /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2784 goto reschedule; 2785 2786msix_good: 2787 ntb_poll_link(ntb); 2788 ntb_link_event(ntb->device); 2789 return; 2790 2791reschedule: 2792 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2793 if (_xeon_link_is_up(ntb)) { 2794 callout_reset(&ntb->peer_msix_work, 2795 hz * (ntb->peer_msix_good ? 2 : 1) / 100, 2796 ntb_exchange_msix, ntb); 2797 } else 2798 ntb_spad_clear(ntb->device); 2799} 2800 2801/* 2802 * Public API to the rest of the OS 2803 */ 2804 2805static uint8_t 2806ntb_spad_count(device_t dev) 2807{ 2808 struct ntb_softc *ntb = device_get_softc(dev); 2809 2810 return (ntb->spad_count); 2811} 2812 2813static uint8_t 2814ntb_mw_count(device_t dev) 2815{ 2816 struct ntb_softc *ntb = device_get_softc(dev); 2817 uint8_t res; 2818 2819 res = ntb->mw_count; 2820 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2821 res--; 2822 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2823 res--; 2824 return (res); 2825} 2826 2827static int 2828ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 2829{ 2830 struct ntb_softc *ntb = device_get_softc(dev); 2831 2832 if (idx >= ntb->spad_count) 2833 return (EINVAL); 2834 2835 ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2836 2837 return (0); 2838} 2839 2840/* 2841 * Zeros the local scratchpad. 2842 */ 2843static void 2844ntb_spad_clear(device_t dev) 2845{ 2846 struct ntb_softc *ntb = device_get_softc(dev); 2847 unsigned i; 2848 2849 for (i = 0; i < ntb->spad_count; i++) 2850 ntb_spad_write(dev, i, 0); 2851} 2852 2853static int 2854ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2855{ 2856 struct ntb_softc *ntb = device_get_softc(dev); 2857 2858 if (idx >= ntb->spad_count) 2859 return (EINVAL); 2860 2861 *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2862 2863 return (0); 2864} 2865 2866static int 2867ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 2868{ 2869 struct ntb_softc *ntb = device_get_softc(dev); 2870 2871 if (idx >= ntb->spad_count) 2872 return (EINVAL); 2873 2874 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2875 ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2876 else 2877 ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2878 2879 return (0); 2880} 2881 2882static int 2883ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2884{ 2885 struct ntb_softc *ntb = device_get_softc(dev); 2886 2887 if (idx >= ntb->spad_count) 2888 return (EINVAL); 2889 2890 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2891 *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2892 else 2893 *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2894 2895 return (0); 2896} 2897 2898static int 2899ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 2900 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2901 bus_addr_t *plimit) 2902{ 2903 struct ntb_softc *ntb = device_get_softc(dev); 2904 struct ntb_pci_bar_info *bar; 2905 bus_addr_t limit; 2906 size_t bar_b2b_off; 2907 enum ntb_bar bar_num; 2908 2909 if (mw_idx >= ntb_mw_count(dev)) 2910 return (EINVAL); 2911 mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); 2912 2913 bar_num = ntb_mw_to_bar(ntb, mw_idx); 2914 bar = &ntb->bar_info[bar_num]; 2915 bar_b2b_off = 0; 2916 if (mw_idx == ntb->b2b_mw_idx) { 2917 KASSERT(ntb->b2b_off != 0, 2918 ("user shouldn't get non-shared b2b mw")); 2919 bar_b2b_off = ntb->b2b_off; 2920 } 2921 2922 if (bar_is_64bit(ntb, bar_num)) 2923 limit = BUS_SPACE_MAXADDR; 2924 else 2925 limit = BUS_SPACE_MAXADDR_32BIT; 2926 2927 if (base != NULL) 2928 *base = bar->pbase + bar_b2b_off; 2929 if (vbase != NULL) 2930 *vbase = bar->vbase + bar_b2b_off; 2931 if (size != NULL) 2932 *size = bar->size - bar_b2b_off; 2933 if (align != NULL) 2934 *align = bar->size; 2935 if (align_size != NULL) 2936 *align_size = 1; 2937 if (plimit != NULL) 2938 *plimit = limit; 2939 return (0); 2940} 2941 2942static int 2943ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 2944{ 2945 struct ntb_softc *ntb = device_get_softc(dev); 2946 struct ntb_pci_bar_info *bar; 2947 uint64_t base, limit, reg_val; 2948 size_t bar_size, mw_size; 2949 uint32_t base_reg, xlat_reg, limit_reg; 2950 enum ntb_bar bar_num; 2951 2952 if (idx >= ntb_mw_count(dev)) 2953 return (EINVAL); 2954 idx = ntb_user_mw_to_idx(ntb, idx); 2955 2956 bar_num = ntb_mw_to_bar(ntb, idx); 2957 bar = &ntb->bar_info[bar_num]; 2958 2959 bar_size = bar->size; 2960 if (idx == ntb->b2b_mw_idx) 2961 mw_size = bar_size - ntb->b2b_off; 2962 else 2963 mw_size = bar_size; 2964 2965 /* Hardware requires that addr is aligned to bar size */ 2966 if ((addr & (bar_size - 1)) != 0) 2967 return (EINVAL); 2968 2969 if (size > mw_size) 2970 return (EINVAL); 2971 2972 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2973 2974 limit = 0; 2975 if (bar_is_64bit(ntb, bar_num)) { 2976 base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2977 2978 if (limit_reg != 0 && size != mw_size) 2979 limit = base + size; 2980 2981 /* Set and verify translation address */ 2982 ntb_reg_write(8, xlat_reg, addr); 2983 reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2984 if (reg_val != addr) { 2985 ntb_reg_write(8, xlat_reg, 0); 2986 return (EIO); 2987 } 2988 2989 /* Set and verify the limit */ 2990 ntb_reg_write(8, limit_reg, limit); 2991 reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 2992 if (reg_val != limit) { 2993 ntb_reg_write(8, limit_reg, base); 2994 ntb_reg_write(8, xlat_reg, 0); 2995 return (EIO); 2996 } 2997 } else { 2998 /* Configure 32-bit (split) BAR MW */ 2999 3000 if ((addr & UINT32_MAX) != addr) 3001 return (ERANGE); 3002 if (((addr + size) & UINT32_MAX) != (addr + size)) 3003 return (ERANGE); 3004 3005 base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3006 3007 if (limit_reg != 0 && size != mw_size) 3008 limit = base + size; 3009 3010 /* Set and verify translation address */ 3011 ntb_reg_write(4, xlat_reg, addr); 3012 reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3013 if (reg_val != addr) { 3014 ntb_reg_write(4, xlat_reg, 0); 3015 return (EIO); 3016 } 3017 3018 /* Set and verify the limit */ 3019 ntb_reg_write(4, limit_reg, limit); 3020 reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3021 if (reg_val != limit) { 3022 ntb_reg_write(4, limit_reg, base); 3023 ntb_reg_write(4, xlat_reg, 0); 3024 return (EIO); 3025 } 3026 } 3027 return (0); 3028} 3029 3030static int 3031ntb_mw_clear_trans(device_t dev, unsigned mw_idx) 3032{ 3033 3034 return (ntb_mw_set_trans(dev, mw_idx, 0, 0)); 3035} 3036 3037static int 3038ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 3039{ 3040 struct ntb_softc *ntb = device_get_softc(dev); 3041 struct ntb_pci_bar_info *bar; 3042 3043 if (idx >= ntb_mw_count(dev)) 3044 return (EINVAL); 3045 idx = ntb_user_mw_to_idx(ntb, idx); 3046 3047 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3048 *mode = bar->map_mode; 3049 return (0); 3050} 3051 3052static int 3053ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 3054{ 3055 struct ntb_softc *ntb = device_get_softc(dev); 3056 3057 if (idx >= ntb_mw_count(dev)) 3058 return (EINVAL); 3059 3060 idx = ntb_user_mw_to_idx(ntb, idx); 3061 return (ntb_mw_set_wc_internal(ntb, idx, mode)); 3062} 3063 3064static int 3065ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3066{ 3067 struct ntb_pci_bar_info *bar; 3068 int rc; 3069 3070 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; 3071 if (bar->map_mode == mode) 3072 return (0); 3073 3074 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3075 if (rc == 0) 3076 bar->map_mode = mode; 3077 3078 return (rc); 3079} 3080 3081static void 3082ntb_peer_db_set(device_t dev, uint64_t bit) 3083{ 3084 struct ntb_softc *ntb = device_get_softc(dev); 3085 3086 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 3087 struct ntb_pci_bar_info *lapic; 3088 unsigned i; 3089 3090 lapic = ntb->peer_lapic_bar; 3091 3092 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3093 if ((bit & ntb_db_vector_mask(dev, i)) != 0) 3094 bus_space_write_4(lapic->pci_bus_tag, 3095 lapic->pci_bus_handle, 3096 ntb->peer_msix_data[i].nmd_ofs, 3097 ntb->peer_msix_data[i].nmd_data); 3098 } 3099 return; 3100 } 3101 3102 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3103 ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3104 return; 3105 } 3106 3107 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3108} 3109 3110static int 3111ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3112{ 3113 struct ntb_softc *ntb = device_get_softc(dev); 3114 struct ntb_pci_bar_info *bar; 3115 uint64_t regoff; 3116 3117 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3118 3119 if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3120 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3121 regoff = ntb->peer_reg->db_bell; 3122 } else { 3123 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3124 ("invalid b2b idx")); 3125 3126 bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3127 regoff = XEON_PDOORBELL_OFFSET; 3128 } 3129 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3130 3131 /* HACK: Specific to current x86 bus implementation. */ 3132 *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3133 *db_size = ntb->reg->db_size; 3134 return (0); 3135} 3136 3137static uint64_t 3138ntb_db_valid_mask(device_t dev) 3139{ 3140 struct ntb_softc *ntb = device_get_softc(dev); 3141 3142 return (ntb->db_valid_mask); 3143} 3144 3145static int 3146ntb_db_vector_count(device_t dev) 3147{ 3148 struct ntb_softc *ntb = device_get_softc(dev); 3149 3150 return (ntb->db_vec_count); 3151} 3152 3153static uint64_t 3154ntb_db_vector_mask(device_t dev, uint32_t vector) 3155{ 3156 struct ntb_softc *ntb = device_get_softc(dev); 3157 3158 if (vector > ntb->db_vec_count) 3159 return (0); 3160 return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); 3161} 3162 3163static bool 3164ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3165{ 3166 struct ntb_softc *ntb = device_get_softc(dev); 3167 3168 if (speed != NULL) 3169 *speed = ntb_link_sta_speed(ntb); 3170 if (width != NULL) 3171 *width = ntb_link_sta_width(ntb); 3172 return (link_is_up(ntb)); 3173} 3174 3175static void 3176save_bar_parameters(struct ntb_pci_bar_info *bar) 3177{ 3178 3179 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3180 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3181 bar->pbase = rman_get_start(bar->pci_resource); 3182 bar->size = rman_get_size(bar->pci_resource); 3183 bar->vbase = rman_get_virtual(bar->pci_resource); 3184} 3185 3186static device_method_t ntb_intel_methods[] = { 3187 /* Device interface */ 3188 DEVMETHOD(device_probe, ntb_probe), 3189 DEVMETHOD(device_attach, ntb_attach), 3190 DEVMETHOD(device_detach, ntb_detach), 3191 /* NTB interface */ 3192 DEVMETHOD(ntb_link_is_up, ntb_link_is_up), 3193 DEVMETHOD(ntb_link_enable, ntb_link_enable), 3194 DEVMETHOD(ntb_link_disable, ntb_link_disable), 3195 DEVMETHOD(ntb_link_enabled, ntb_link_enabled), 3196 DEVMETHOD(ntb_set_ctx, ntb_set_ctx), 3197 DEVMETHOD(ntb_get_ctx, ntb_get_ctx), 3198 DEVMETHOD(ntb_clear_ctx, ntb_clear_ctx), 3199 DEVMETHOD(ntb_mw_count, ntb_mw_count), 3200 DEVMETHOD(ntb_mw_get_range, ntb_mw_get_range), 3201 DEVMETHOD(ntb_mw_set_trans, ntb_mw_set_trans), 3202 DEVMETHOD(ntb_mw_clear_trans, ntb_mw_clear_trans), 3203 DEVMETHOD(ntb_mw_get_wc, ntb_mw_get_wc), 3204 DEVMETHOD(ntb_mw_set_wc, ntb_mw_set_wc), 3205 DEVMETHOD(ntb_spad_count, ntb_spad_count), 3206 DEVMETHOD(ntb_spad_clear, ntb_spad_clear), 3207 DEVMETHOD(ntb_spad_write, ntb_spad_write), 3208 DEVMETHOD(ntb_spad_read, ntb_spad_read), 3209 DEVMETHOD(ntb_peer_spad_write, ntb_peer_spad_write), 3210 DEVMETHOD(ntb_peer_spad_read, ntb_peer_spad_read), 3211 DEVMETHOD(ntb_db_valid_mask, ntb_db_valid_mask), 3212 DEVMETHOD(ntb_db_vector_count, ntb_db_vector_count), 3213 DEVMETHOD(ntb_db_vector_mask, ntb_db_vector_mask), 3214 DEVMETHOD(ntb_db_clear, ntb_db_clear), 3215 DEVMETHOD(ntb_db_clear_mask, ntb_db_clear_mask), 3216 DEVMETHOD(ntb_db_read, ntb_db_read), 3217 DEVMETHOD(ntb_db_set_mask, ntb_db_set_mask), 3218 DEVMETHOD(ntb_peer_db_addr, ntb_peer_db_addr), 3219 DEVMETHOD(ntb_peer_db_set, ntb_peer_db_set), 3220 DEVMETHOD_END 3221}; 3222 3223static DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3224 sizeof(struct ntb_softc)); 3225DRIVER_MODULE(ntb_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL); 3226MODULE_DEPEND(ntb_intel, ntb, 1, 1, 1); 3227MODULE_VERSION(ntb_intel, 1); 3228