ntb_hw_intel.c revision 291263
1250079Scarl/*-
2250079Scarl * Copyright (C) 2013 Intel Corporation
3289542Scem * Copyright (C) 2015 EMC Corporation
4250079Scarl * All rights reserved.
5250079Scarl *
6250079Scarl * Redistribution and use in source and binary forms, with or without
7250079Scarl * modification, are permitted provided that the following conditions
8250079Scarl * are met:
9250079Scarl * 1. Redistributions of source code must retain the above copyright
10250079Scarl *    notice, this list of conditions and the following disclaimer.
11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright
12250079Scarl *    notice, this list of conditions and the following disclaimer in the
13250079Scarl *    documentation and/or other materials provided with the distribution.
14250079Scarl *
15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18250079Scarl * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25250079Scarl * SUCH DAMAGE.
26250079Scarl */
27250079Scarl
28250079Scarl#include <sys/cdefs.h>
29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 291263 2015-11-24 18:51:17Z cem $");
30250079Scarl
31250079Scarl#include <sys/param.h>
32250079Scarl#include <sys/kernel.h>
33250079Scarl#include <sys/systm.h>
34250079Scarl#include <sys/bus.h>
35289774Scem#include <sys/endian.h>
36250079Scarl#include <sys/malloc.h>
37250079Scarl#include <sys/module.h>
38250079Scarl#include <sys/queue.h>
39250079Scarl#include <sys/rman.h>
40289774Scem#include <sys/sbuf.h>
41289207Scem#include <sys/sysctl.h>
42250079Scarl#include <vm/vm.h>
43250079Scarl#include <vm/pmap.h>
44250079Scarl#include <machine/bus.h>
45250079Scarl#include <machine/pmap.h>
46250079Scarl#include <machine/resource.h>
47250079Scarl#include <dev/pci/pcireg.h>
48250079Scarl#include <dev/pci/pcivar.h>
49250079Scarl
50250079Scarl#include "ntb_regs.h"
51250079Scarl#include "ntb_hw.h"
52250079Scarl
53250079Scarl/*
54250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that
55250079Scarl * allows you to connect two systems using a PCI-e link.
56250079Scarl *
57250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows
58250079Scarl * you to send and recieve interrupts, map the memory windows and send and
59250079Scarl * receive messages in the scratch-pad registers.
60250079Scarl *
61250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may
62250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license.
63250079Scarl */
64250079Scarl
65289648Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT)
66250079Scarl
67289539Scem#define NTB_HB_TIMEOUT		1 /* second */
68289648Scem#define ATOM_LINK_RECOVERY_TIME	500 /* ms */
69291032Scem#define BAR_HIGH_MASK		(~((1ull << 12) - 1))
70250079Scarl
71250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev))
72250079Scarl
73250079Scarlenum ntb_device_type {
74250079Scarl	NTB_XEON,
75289648Scem	NTB_ATOM
76250079Scarl};
77250079Scarl
78289610Scem/* ntb_conn_type are hardware numbers, cannot change. */
79289610Scemenum ntb_conn_type {
80289610Scem	NTB_CONN_TRANSPARENT = 0,
81289610Scem	NTB_CONN_B2B = 1,
82289610Scem	NTB_CONN_RP = 2,
83289610Scem};
84289610Scem
85289610Scemenum ntb_b2b_direction {
86289610Scem	NTB_DEV_USD = 0,
87289610Scem	NTB_DEV_DSD = 1,
88289610Scem};
89289610Scem
90289539Scemenum ntb_bar {
91289539Scem	NTB_CONFIG_BAR = 0,
92289539Scem	NTB_B2B_BAR_1,
93289539Scem	NTB_B2B_BAR_2,
94289539Scem	NTB_B2B_BAR_3,
95289539Scem	NTB_MAX_BARS
96289539Scem};
97289539Scem
98255274Scarl/* Device features and workarounds */
99255274Scarl#define HAS_FEATURE(feature)	\
100255274Scarl	((ntb->features & (feature)) != 0)
101255274Scarl
102250079Scarlstruct ntb_hw_info {
103250079Scarl	uint32_t		device_id;
104255274Scarl	const char		*desc;
105250079Scarl	enum ntb_device_type	type;
106289397Scem	uint32_t		features;
107250079Scarl};
108250079Scarl
109250079Scarlstruct ntb_pci_bar_info {
110250079Scarl	bus_space_tag_t		pci_bus_tag;
111250079Scarl	bus_space_handle_t	pci_bus_handle;
112250079Scarl	int			pci_resource_id;
113250079Scarl	struct resource		*pci_resource;
114250079Scarl	vm_paddr_t		pbase;
115290679Scem	caddr_t			vbase;
116290679Scem	vm_size_t		size;
117291031Scem	bool			mapped_wc : 1;
118289543Scem
119289543Scem	/* Configuration register offsets */
120289543Scem	uint32_t		psz_off;
121289543Scem	uint32_t		ssz_off;
122289543Scem	uint32_t		pbarxlat_off;
123250079Scarl};
124250079Scarl
125250079Scarlstruct ntb_int_info {
126250079Scarl	struct resource	*res;
127250079Scarl	int		rid;
128250079Scarl	void		*tag;
129250079Scarl};
130250079Scarl
131289546Scemstruct ntb_vec {
132250079Scarl	struct ntb_softc	*ntb;
133289546Scem	uint32_t		num;
134250079Scarl};
135250079Scarl
136289542Scemstruct ntb_reg {
137289542Scem	uint32_t	ntb_ctl;
138289542Scem	uint32_t	lnk_sta;
139289542Scem	uint8_t		db_size;
140289542Scem	unsigned	mw_bar[NTB_MAX_BARS];
141289542Scem};
142289542Scem
143289542Scemstruct ntb_alt_reg {
144289542Scem	uint32_t	db_bell;
145289542Scem	uint32_t	db_mask;
146289542Scem	uint32_t	spad;
147289542Scem};
148289542Scem
149289542Scemstruct ntb_xlat_reg {
150289546Scem	uint32_t	bar0_base;
151289546Scem	uint32_t	bar2_base;
152289546Scem	uint32_t	bar4_base;
153289546Scem	uint32_t	bar5_base;
154289546Scem
155289546Scem	uint32_t	bar2_xlat;
156289546Scem	uint32_t	bar4_xlat;
157289546Scem	uint32_t	bar5_xlat;
158289546Scem
159289546Scem	uint32_t	bar2_limit;
160289546Scem	uint32_t	bar4_limit;
161289546Scem	uint32_t	bar5_limit;
162289542Scem};
163289542Scem
164289542Scemstruct ntb_b2b_addr {
165289542Scem	uint64_t	bar0_addr;
166289542Scem	uint64_t	bar2_addr64;
167289542Scem	uint64_t	bar4_addr64;
168289542Scem	uint64_t	bar4_addr32;
169289542Scem	uint64_t	bar5_addr32;
170289542Scem};
171289542Scem
172250079Scarlstruct ntb_softc {
173250079Scarl	device_t		device;
174250079Scarl	enum ntb_device_type	type;
175289774Scem	uint32_t		features;
176250079Scarl
177250079Scarl	struct ntb_pci_bar_info	bar_info[NTB_MAX_BARS];
178250079Scarl	struct ntb_int_info	int_info[MAX_MSIX_INTERRUPTS];
179250079Scarl	uint32_t		allocated_interrupts;
180250079Scarl
181250079Scarl	struct callout		heartbeat_timer;
182250079Scarl	struct callout		lr_timer;
183250079Scarl
184289546Scem	void			*ntb_ctx;
185289546Scem	const struct ntb_ctx_ops *ctx_ops;
186289546Scem	struct ntb_vec		*msix_vec;
187290683Scem#define CTX_LOCK(sc)		mtx_lock(&(sc)->ctx_lock)
188290683Scem#define CTX_UNLOCK(sc)		mtx_unlock(&(sc)->ctx_lock)
189289546Scem#define CTX_ASSERT(sc,f)	mtx_assert(&(sc)->ctx_lock, (f))
190289546Scem	struct mtx		ctx_lock;
191250079Scarl
192289610Scem	uint32_t		ppd;
193289610Scem	enum ntb_conn_type	conn_type;
194289610Scem	enum ntb_b2b_direction	dev_type;
195289539Scem
196289542Scem	/* Offset of peer bar0 in B2B BAR */
197289542Scem	uint64_t			b2b_off;
198289542Scem	/* Memory window used to access peer bar0 */
199289543Scem#define B2B_MW_DISABLED			UINT8_MAX
200289542Scem	uint8_t				b2b_mw_idx;
201289542Scem
202289539Scem	uint8_t				mw_count;
203289539Scem	uint8_t				spad_count;
204289539Scem	uint8_t				db_count;
205289539Scem	uint8_t				db_vec_count;
206289539Scem	uint8_t				db_vec_shift;
207289542Scem
208289546Scem	/* Protects local db_mask. */
209289546Scem#define DB_MASK_LOCK(sc)	mtx_lock_spin(&(sc)->db_mask_lock)
210289546Scem#define DB_MASK_UNLOCK(sc)	mtx_unlock_spin(&(sc)->db_mask_lock)
211289546Scem#define DB_MASK_ASSERT(sc,f)	mtx_assert(&(sc)->db_mask_lock, (f))
212289542Scem	struct mtx			db_mask_lock;
213289542Scem
214290686Scem	volatile uint32_t		ntb_ctl;
215290686Scem	volatile uint32_t		lnk_sta;
216289542Scem
217289542Scem	uint64_t			db_valid_mask;
218289542Scem	uint64_t			db_link_mask;
219289546Scem	uint64_t			db_mask;
220289542Scem
221289542Scem	int				last_ts;	/* ticks @ last irq */
222289542Scem
223289542Scem	const struct ntb_reg		*reg;
224289542Scem	const struct ntb_alt_reg	*self_reg;
225289542Scem	const struct ntb_alt_reg	*peer_reg;
226289542Scem	const struct ntb_xlat_reg	*xlat_reg;
227250079Scarl};
228250079Scarl
229289234Scem#ifdef __i386__
230289234Scemstatic __inline uint64_t
231289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
232289234Scem    bus_size_t offset)
233289234Scem{
234289234Scem
235289234Scem	return (bus_space_read_4(tag, handle, offset) |
236289234Scem	    ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32);
237289234Scem}
238289234Scem
239289234Scemstatic __inline void
240289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
241289234Scem    bus_size_t offset, uint64_t val)
242289234Scem{
243289234Scem
244289234Scem	bus_space_write_4(tag, handle, offset, val);
245289234Scem	bus_space_write_4(tag, handle, offset + 4, val >> 32);
246289234Scem}
247289234Scem#endif
248289234Scem
249255279Scarl#define ntb_bar_read(SIZE, bar, offset) \
250255279Scarl	    bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
251255279Scarl	    ntb->bar_info[(bar)].pci_bus_handle, (offset))
252255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \
253255279Scarl	    bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
254255279Scarl	    ntb->bar_info[(bar)].pci_bus_handle, (offset), (val))
255255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
256250079Scarl#define ntb_reg_write(SIZE, offset, val) \
257255279Scarl	    ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
258289397Scem#define ntb_mw_read(SIZE, offset) \
259289542Scem	    ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset)
260255279Scarl#define ntb_mw_write(SIZE, offset, val) \
261289542Scem	    ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \
262289397Scem		offset, val)
263250079Scarl
264250079Scarlstatic int ntb_probe(device_t device);
265250079Scarlstatic int ntb_attach(device_t device);
266250079Scarlstatic int ntb_detach(device_t device);
267291263Scemstatic unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx);
268289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw);
269289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar);
270289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar,
271289546Scem    uint32_t *base, uint32_t *xlat, uint32_t *lmt);
272255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb);
273291263Scemstatic int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, bool wc);
274289647Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *,
275289647Scem    const char *);
276255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
277255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb,
278255272Scarl    struct ntb_pci_bar_info *bar);
279250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb);
280289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
281289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb);
282289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
283289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
284250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb);
285289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);
286289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec);
287289546Scemstatic void ndev_vec_isr(void *arg);
288289546Scemstatic void ndev_irq_isr(void *arg);
289289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff);
290290678Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t);
291290678Scemstatic inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t);
292289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors);
293289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb);
294250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
295289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb);
296289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb);
297289648Scemstatic int ntb_detect_atom(struct ntb_softc *ntb);
298289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb);
299289648Scemstatic int ntb_atom_init_dev(struct ntb_softc *ntb);
300289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb);
301289648Scemstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
302289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx,
303289543Scem    enum ntb_bar regbar);
304289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *,
305289543Scem    uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar);
306289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
307289543Scem    enum ntb_bar idx);
308289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *,
309289542Scem    const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
310289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb);
311289648Scemstatic inline bool atom_link_is_err(struct ntb_softc *ntb);
312289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *);
313289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *);
314289648Scemstatic void atom_link_hb(void *arg);
315289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec);
316289648Scemstatic void recover_atom_link(void *arg);
317289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb);
318255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar);
319289774Scemstatic void ntb_sysctl_init(struct ntb_softc *);
320289774Scemstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS);
321289774Scemstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS);
322289774Scemstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS);
323250079Scarl
324290685Scemstatic unsigned g_ntb_hw_debug_level;
325290685ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN,
326290685Scem    &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose");
327290685Scem#define ntb_printf(lvl, ...) do {				\
328290685Scem	if ((lvl) <= g_ntb_hw_debug_level) {			\
329290685Scem		device_printf(ntb->device, __VA_ARGS__);	\
330290685Scem	}							\
331290685Scem} while (0)
332290685Scem
333291030Scemstatic unsigned g_ntb_enable_wc = 1;
334291030ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, enable_writecombine, CTLFLAG_RDTUN,
335291030Scem    &g_ntb_enable_wc, 0, "Set to 1 to map memory windows write combining");
336291030Scem
337291263Scemstatic int g_ntb_mw_idx = -1;
338291263ScemSYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx,
339291263Scem    0, "Use this memory window to access the peer NTB registers.  A "
340291263Scem    "non-negative value starts from the first MW index; a negative value "
341291263Scem    "starts from the last MW index.  The default is -1, i.e., the last "
342291263Scem    "available memory window.  Both sides of the NTB MUST set the same "
343291263Scem    "value here!  (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)");
344291263Scem
345250079Scarlstatic struct ntb_hw_info pci_ids[] = {
346289612Scem	/* XXX: PS/SS IDs left out until they are supported. */
347289612Scem	{ 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B",
348289648Scem		NTB_ATOM, 0 },
349289233Scem
350289233Scem	{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
351289538Scem		NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
352289233Scem	{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
353289538Scem		NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
354289233Scem	{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
355289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
356289538Scem		    NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K },
357289233Scem	{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
358289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
359289538Scem		    NTB_SB01BASE_LOCKUP },
360289233Scem	{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
361289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
362289538Scem		    NTB_SB01BASE_LOCKUP },
363289233Scem
364289648Scem	{ 0x00000000, NULL, NTB_ATOM, 0 }
365250079Scarl};
366250079Scarl
367289648Scemstatic const struct ntb_reg atom_reg = {
368289648Scem	.ntb_ctl = ATOM_NTBCNTL_OFFSET,
369289648Scem	.lnk_sta = ATOM_LINK_STATUS_OFFSET,
370289542Scem	.db_size = sizeof(uint64_t),
371289542Scem	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
372289542Scem};
373289542Scem
374289648Scemstatic const struct ntb_alt_reg atom_pri_reg = {
375289648Scem	.db_bell = ATOM_PDOORBELL_OFFSET,
376289648Scem	.db_mask = ATOM_PDBMSK_OFFSET,
377289648Scem	.spad = ATOM_SPAD_OFFSET,
378289607Scem};
379289607Scem
380289648Scemstatic const struct ntb_alt_reg atom_b2b_reg = {
381289648Scem	.db_bell = ATOM_B2B_DOORBELL_OFFSET,
382289648Scem	.spad = ATOM_B2B_SPAD_OFFSET,
383289542Scem};
384289542Scem
385289648Scemstatic const struct ntb_xlat_reg atom_sec_xlat = {
386289542Scem#if 0
387289542Scem	/* "FIXME" says the Linux driver. */
388289648Scem	.bar0_base = ATOM_SBAR0BASE_OFFSET,
389289648Scem	.bar2_base = ATOM_SBAR2BASE_OFFSET,
390289648Scem	.bar4_base = ATOM_SBAR4BASE_OFFSET,
391289546Scem
392289648Scem	.bar2_limit = ATOM_SBAR2LMT_OFFSET,
393289648Scem	.bar4_limit = ATOM_SBAR4LMT_OFFSET,
394289542Scem#endif
395289546Scem
396289648Scem	.bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
397289648Scem	.bar4_xlat = ATOM_SBAR4XLAT_OFFSET,
398289542Scem};
399289542Scem
400289542Scemstatic const struct ntb_reg xeon_reg = {
401289542Scem	.ntb_ctl = XEON_NTBCNTL_OFFSET,
402289542Scem	.lnk_sta = XEON_LINK_STATUS_OFFSET,
403289542Scem	.db_size = sizeof(uint16_t),
404289542Scem	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 },
405289542Scem};
406289542Scem
407289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = {
408289607Scem	.db_bell = XEON_PDOORBELL_OFFSET,
409289607Scem	.db_mask = XEON_PDBMSK_OFFSET,
410289607Scem	.spad = XEON_SPAD_OFFSET,
411289607Scem};
412289607Scem
413289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = {
414289542Scem	.db_bell = XEON_B2B_DOORBELL_OFFSET,
415289542Scem	.spad = XEON_B2B_SPAD_OFFSET,
416289542Scem};
417289542Scem
418289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = {
419289542Scem	.bar0_base = XEON_SBAR0BASE_OFFSET,
420289546Scem	.bar2_base = XEON_SBAR2BASE_OFFSET,
421289546Scem	.bar4_base = XEON_SBAR4BASE_OFFSET,
422289546Scem	.bar5_base = XEON_SBAR5BASE_OFFSET,
423289546Scem
424289542Scem	.bar2_limit = XEON_SBAR2LMT_OFFSET,
425289546Scem	.bar4_limit = XEON_SBAR4LMT_OFFSET,
426289546Scem	.bar5_limit = XEON_SBAR5LMT_OFFSET,
427289546Scem
428289542Scem	.bar2_xlat = XEON_SBAR2XLAT_OFFSET,
429289546Scem	.bar4_xlat = XEON_SBAR4XLAT_OFFSET,
430289546Scem	.bar5_xlat = XEON_SBAR5XLAT_OFFSET,
431289542Scem};
432289542Scem
433289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = {
434290725Scem	.bar0_addr = XEON_B2B_BAR0_ADDR,
435290725Scem	.bar2_addr64 = XEON_B2B_BAR2_ADDR64,
436290725Scem	.bar4_addr64 = XEON_B2B_BAR4_ADDR64,
437290725Scem	.bar4_addr32 = XEON_B2B_BAR4_ADDR32,
438290725Scem	.bar5_addr32 = XEON_B2B_BAR5_ADDR32,
439289542Scem};
440289542Scem
441289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = {
442290725Scem	.bar0_addr = XEON_B2B_BAR0_ADDR,
443290725Scem	.bar2_addr64 = XEON_B2B_BAR2_ADDR64,
444290725Scem	.bar4_addr64 = XEON_B2B_BAR4_ADDR64,
445290725Scem	.bar4_addr32 = XEON_B2B_BAR4_ADDR32,
446290725Scem	.bar5_addr32 = XEON_B2B_BAR5_ADDR32,
447289542Scem};
448289542Scem
449289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0,
450289614Scem    "B2B MW segment overrides -- MUST be the same on both sides");
451289614Scem
452289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN,
453289614Scem    &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
454289614Scem    "hardware, use this 64-bit address on the bus between the NTB devices for "
455289614Scem    "the window at BAR2, on the upstream side of the link.  MUST be the same "
456289614Scem    "address on both sides.");
457289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN,
458289614Scem    &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4.");
459289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN,
460289614Scem    &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 "
461289614Scem    "(split-BAR mode).");
462289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN,
463289646Scem    &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 "
464289614Scem    "(split-BAR mode).");
465289614Scem
466289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN,
467289614Scem    &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
468289614Scem    "hardware, use this 64-bit address on the bus between the NTB devices for "
469289614Scem    "the window at BAR2, on the downstream side of the link.  MUST be the same"
470289614Scem    " address on both sides.");
471289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN,
472289614Scem    &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4.");
473289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN,
474289614Scem    &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 "
475289614Scem    "(split-BAR mode).");
476289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN,
477289646Scem    &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 "
478289614Scem    "(split-BAR mode).");
479289614Scem
480250079Scarl/*
481250079Scarl * OS <-> Driver interface structures
482250079Scarl */
483250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");
484250079Scarl
485250079Scarlstatic device_method_t ntb_pci_methods[] = {
486250079Scarl	/* Device interface */
487250079Scarl	DEVMETHOD(device_probe,     ntb_probe),
488250079Scarl	DEVMETHOD(device_attach,    ntb_attach),
489250079Scarl	DEVMETHOD(device_detach,    ntb_detach),
490250079Scarl	DEVMETHOD_END
491250079Scarl};
492250079Scarl
493250079Scarlstatic driver_t ntb_pci_driver = {
494250079Scarl	"ntb_hw",
495250079Scarl	ntb_pci_methods,
496250079Scarl	sizeof(struct ntb_softc),
497250079Scarl};
498250079Scarl
499250079Scarlstatic devclass_t ntb_devclass;
500250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL);
501250079ScarlMODULE_VERSION(ntb_hw, 1);
502250079Scarl
503289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls");
504289207Scem
505250079Scarl/*
506250079Scarl * OS <-> Driver linkage functions
507250079Scarl */
508250079Scarlstatic int
509250079Scarlntb_probe(device_t device)
510250079Scarl{
511289209Scem	struct ntb_hw_info *p;
512250079Scarl
513289209Scem	p = ntb_get_device_info(pci_get_devid(device));
514289209Scem	if (p == NULL)
515250079Scarl		return (ENXIO);
516289209Scem
517289209Scem	device_set_desc(device, p->desc);
518289209Scem	return (0);
519250079Scarl}
520250079Scarl
521250079Scarlstatic int
522250079Scarlntb_attach(device_t device)
523250079Scarl{
524289209Scem	struct ntb_softc *ntb;
525289209Scem	struct ntb_hw_info *p;
526250079Scarl	int error;
527250079Scarl
528289209Scem	ntb = DEVICE2SOFTC(device);
529289209Scem	p = ntb_get_device_info(pci_get_devid(device));
530289209Scem
531250079Scarl	ntb->device = device;
532250079Scarl	ntb->type = p->type;
533255274Scarl	ntb->features = p->features;
534289543Scem	ntb->b2b_mw_idx = B2B_MW_DISABLED;
535250079Scarl
536289648Scem	/* Heartbeat timer for NTB_ATOM since there is no link interrupt */
537283291Sjkim	callout_init(&ntb->heartbeat_timer, 1);
538283291Sjkim	callout_init(&ntb->lr_timer, 1);
539289542Scem	mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);
540290683Scem	mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF);
541250079Scarl
542289648Scem	if (ntb->type == NTB_ATOM)
543289648Scem		error = ntb_detect_atom(ntb);
544289348Scem	else
545289348Scem		error = ntb_detect_xeon(ntb);
546290682Scem	if (error != 0)
547289348Scem		goto out;
548289348Scem
549289397Scem	ntb_detect_max_mw(ntb);
550289396Scem
551290682Scem	pci_enable_busmaster(ntb->device);
552290682Scem
553289209Scem	error = ntb_map_pci_bars(ntb);
554290682Scem	if (error != 0)
555289209Scem		goto out;
556289648Scem	if (ntb->type == NTB_ATOM)
557289648Scem		error = ntb_atom_init_dev(ntb);
558289272Scem	else
559289542Scem		error = ntb_xeon_init_dev(ntb);
560290682Scem	if (error != 0)
561289209Scem		goto out;
562290682Scem
563290682Scem	ntb_poll_link(ntb);
564290682Scem
565289774Scem	ntb_sysctl_init(ntb);
566250079Scarl
567289209Scemout:
568289209Scem	if (error != 0)
569289209Scem		ntb_detach(device);
570250079Scarl	return (error);
571250079Scarl}
572250079Scarl
573250079Scarlstatic int
574250079Scarlntb_detach(device_t device)
575250079Scarl{
576289209Scem	struct ntb_softc *ntb;
577250079Scarl
578289209Scem	ntb = DEVICE2SOFTC(device);
579289542Scem
580289617Scem	if (ntb->self_reg != NULL)
581289617Scem		ntb_db_set_mask(ntb, ntb->db_valid_mask);
582250079Scarl	callout_drain(&ntb->heartbeat_timer);
583250079Scarl	callout_drain(&ntb->lr_timer);
584290682Scem	pci_disable_busmaster(ntb->device);
585289272Scem	if (ntb->type == NTB_XEON)
586289272Scem		ntb_teardown_xeon(ntb);
587250079Scarl	ntb_teardown_interrupts(ntb);
588289397Scem
589289542Scem	mtx_destroy(&ntb->db_mask_lock);
590289546Scem	mtx_destroy(&ntb->ctx_lock);
591289542Scem
592250079Scarl	ntb_unmap_pci_bar(ntb);
593250079Scarl
594250079Scarl	return (0);
595250079Scarl}
596250079Scarl
597289542Scem/*
598289542Scem * Driver internal routines
599289542Scem */
600289539Scemstatic inline enum ntb_bar
601289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
602289539Scem{
603289539Scem
604291263Scem	KASSERT(mw < ntb->mw_count,
605289542Scem	    ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count));
606289546Scem	KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw"));
607289539Scem
608289542Scem	return (ntb->reg->mw_bar[mw]);
609289539Scem}
610289539Scem
611289546Scemstatic inline bool
612289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar)
613289546Scem{
614289546Scem	/* XXX This assertion could be stronger. */
615289546Scem	KASSERT(bar < NTB_MAX_BARS, ("bogus bar"));
616289546Scem	return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR));
617289546Scem}
618289546Scem
619289546Scemstatic inline void
620289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base,
621289546Scem    uint32_t *xlat, uint32_t *lmt)
622289546Scem{
623289546Scem	uint32_t basev, lmtv, xlatv;
624289546Scem
625289546Scem	switch (bar) {
626289546Scem	case NTB_B2B_BAR_1:
627289546Scem		basev = ntb->xlat_reg->bar2_base;
628289546Scem		lmtv = ntb->xlat_reg->bar2_limit;
629289546Scem		xlatv = ntb->xlat_reg->bar2_xlat;
630289546Scem		break;
631289546Scem	case NTB_B2B_BAR_2:
632289546Scem		basev = ntb->xlat_reg->bar4_base;
633289546Scem		lmtv = ntb->xlat_reg->bar4_limit;
634289546Scem		xlatv = ntb->xlat_reg->bar4_xlat;
635289546Scem		break;
636289546Scem	case NTB_B2B_BAR_3:
637289546Scem		basev = ntb->xlat_reg->bar5_base;
638289546Scem		lmtv = ntb->xlat_reg->bar5_limit;
639289546Scem		xlatv = ntb->xlat_reg->bar5_xlat;
640289546Scem		break;
641289546Scem	default:
642289546Scem		KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS,
643289546Scem		    ("bad bar"));
644289546Scem		basev = lmtv = xlatv = 0;
645289546Scem		break;
646289546Scem	}
647289546Scem
648289546Scem	if (base != NULL)
649289546Scem		*base = basev;
650289546Scem	if (xlat != NULL)
651289546Scem		*xlat = xlatv;
652289546Scem	if (lmt != NULL)
653289546Scem		*lmt = lmtv;
654289546Scem}
655289546Scem
656250079Scarlstatic int
657255272Scarlntb_map_pci_bars(struct ntb_softc *ntb)
658250079Scarl{
659255272Scarl	int rc;
660250079Scarl
661250079Scarl	ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0);
662289541Scem	rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]);
663255272Scarl	if (rc != 0)
664289541Scem		goto out;
665255272Scarl
666289209Scem	ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2);
667289541Scem	rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]);
668255272Scarl	if (rc != 0)
669289541Scem		goto out;
670289543Scem	ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET;
671289543Scem	ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET;
672289543Scem	ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
673255272Scarl
674289209Scem	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
675291263Scem	rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
676291263Scem	if (rc != 0)
677291263Scem		goto out;
678289543Scem	ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET;
679289543Scem	ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET;
680289543Scem	ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
681289543Scem
682289397Scem	if (!HAS_FEATURE(NTB_SPLIT_BAR))
683289541Scem		goto out;
684289397Scem
685289397Scem	ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5);
686291263Scem	rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
687289543Scem	ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET;
688289543Scem	ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET;
689289543Scem	ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET;
690250079Scarl
691289541Scemout:
692289209Scem	if (rc != 0)
693255272Scarl		device_printf(ntb->device,
694255272Scarl		    "unable to allocate pci resource\n");
695255272Scarl	return (rc);
696255272Scarl}
697255272Scarl
698289541Scemstatic void
699289647Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar,
700289647Scem    const char *kind)
701289541Scem{
702289541Scem
703289647Scem	device_printf(ntb->device,
704289647Scem	    "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n",
705289647Scem	    PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
706289647Scem	    (char *)bar->vbase + bar->size - 1,
707289647Scem	    (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1),
708289647Scem	    (uintmax_t)bar->size, kind);
709289541Scem}
710289541Scem
711255272Scarlstatic int
712255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
713255272Scarl{
714255272Scarl
715255275Scarl	bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
716289209Scem	    &bar->pci_resource_id, RF_ACTIVE);
717255272Scarl	if (bar->pci_resource == NULL)
718255272Scarl		return (ENXIO);
719289209Scem
720289209Scem	save_bar_parameters(bar);
721289647Scem	print_map_success(ntb, bar, "mmr");
722289209Scem	return (0);
723255272Scarl}
724255272Scarl
725255272Scarlstatic int
726255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
727255272Scarl{
728255272Scarl	int rc;
729255276Scarl	uint8_t bar_size_bits = 0;
730255272Scarl
731289209Scem	bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
732289209Scem	    &bar->pci_resource_id, RF_ACTIVE);
733250079Scarl
734255272Scarl	if (bar->pci_resource == NULL)
735255272Scarl		return (ENXIO);
736255276Scarl
737289209Scem	save_bar_parameters(bar);
738289209Scem	/*
739289209Scem	 * Ivytown NTB BAR sizes are misreported by the hardware due to a
740289209Scem	 * hardware issue. To work around this, query the size it should be
741289209Scem	 * configured to by the device and modify the resource to correspond to
742289209Scem	 * this new size. The BIOS on systems with this problem is required to
743289209Scem	 * provide enough address space to allow the driver to make this change
744289209Scem	 * safely.
745289209Scem	 *
746289209Scem	 * Ideally I could have just specified the size when I allocated the
747289209Scem	 * resource like:
748289209Scem	 *  bus_alloc_resource(ntb->device,
749289209Scem	 *	SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul,
750289209Scem	 *	1ul << bar_size_bits, RF_ACTIVE);
751289209Scem	 * but the PCI driver does not honor the size in this call, so we have
752289209Scem	 * to modify it after the fact.
753289209Scem	 */
754289209Scem	if (HAS_FEATURE(NTB_BAR_SIZE_4K)) {
755289209Scem		if (bar->pci_resource_id == PCIR_BAR(2))
756289209Scem			bar_size_bits = pci_read_config(ntb->device,
757289209Scem			    XEON_PBAR23SZ_OFFSET, 1);
758289209Scem		else
759289209Scem			bar_size_bits = pci_read_config(ntb->device,
760289209Scem			    XEON_PBAR45SZ_OFFSET, 1);
761289209Scem
762289209Scem		rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY,
763289209Scem		    bar->pci_resource, bar->pbase,
764289209Scem		    bar->pbase + (1ul << bar_size_bits) - 1);
765255272Scarl		if (rc != 0) {
766289209Scem			device_printf(ntb->device,
767289209Scem			    "unable to resize bar\n");
768255272Scarl			return (rc);
769250079Scarl		}
770289209Scem
771289209Scem		save_bar_parameters(bar);
772250079Scarl	}
773289209Scem
774291030Scem	print_map_success(ntb, bar, "mw");
775291030Scem	if (g_ntb_enable_wc == 0)
776291030Scem		return (0);
777291030Scem
778289209Scem	/* Mark bar region as write combining to improve performance. */
779289209Scem	rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size,
780289209Scem	    VM_MEMATTR_WRITE_COMBINING);
781291031Scem	if (rc == 0) {
782291031Scem		bar->mapped_wc = true;
783289209Scem		device_printf(ntb->device,
784289647Scem		    "Marked BAR%d v:[%p-%p] p:[%p-%p] as "
785289647Scem		    "WRITE_COMBINING.\n",
786289647Scem		    PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
787289647Scem		    (char *)bar->vbase + bar->size - 1,
788289647Scem		    (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1));
789291031Scem	} else
790289647Scem		device_printf(ntb->device,
791289647Scem		    "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as "
792289647Scem		    "WRITE_COMBINING: %d\n",
793289647Scem		    PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
794289647Scem		    (char *)bar->vbase + bar->size - 1,
795289647Scem		    (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1),
796289647Scem		    rc);
797289647Scem		/* Proceed anyway */
798250079Scarl	return (0);
799250079Scarl}
800250079Scarl
801250079Scarlstatic void
802250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb)
803250079Scarl{
804250079Scarl	struct ntb_pci_bar_info *current_bar;
805250079Scarl	int i;
806250079Scarl
807289397Scem	for (i = 0; i < NTB_MAX_BARS; i++) {
808250079Scarl		current_bar = &ntb->bar_info[i];
809250079Scarl		if (current_bar->pci_resource != NULL)
810250079Scarl			bus_release_resource(ntb->device, SYS_RES_MEMORY,
811250079Scarl			    current_bar->pci_resource_id,
812250079Scarl			    current_bar->pci_resource);
813250079Scarl	}
814250079Scarl}
815250079Scarl
816250079Scarlstatic int
817289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors)
818250079Scarl{
819289342Scem	uint32_t i;
820289342Scem	int rc;
821289342Scem
822289342Scem	for (i = 0; i < num_vectors; i++) {
823289342Scem		ntb->int_info[i].rid = i + 1;
824289342Scem		ntb->int_info[i].res = bus_alloc_resource_any(ntb->device,
825289342Scem		    SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE);
826289342Scem		if (ntb->int_info[i].res == NULL) {
827289342Scem			device_printf(ntb->device,
828289342Scem			    "bus_alloc_resource failed\n");
829289342Scem			return (ENOMEM);
830289342Scem		}
831289342Scem		ntb->int_info[i].tag = NULL;
832289342Scem		ntb->allocated_interrupts++;
833289342Scem		rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
834289546Scem		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr,
835289546Scem		    &ntb->msix_vec[i], &ntb->int_info[i].tag);
836289342Scem		if (rc != 0) {
837289342Scem			device_printf(ntb->device, "bus_setup_intr failed\n");
838289342Scem			return (ENXIO);
839289342Scem		}
840289342Scem	}
841289342Scem	return (0);
842289342Scem}
843289342Scem
844289344Scem/*
845289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector
846289344Scem * cannot be allocated for each MSI-X message.  JHB seems to think remapping
847289344Scem * should be okay.  This tunable should enable us to test that hypothesis
848289344Scem * when someone gets their hands on some Xeon hardware.
849289344Scem */
850289344Scemstatic int ntb_force_remap_mode;
851289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN,
852289344Scem    &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped"
853289344Scem    " to a smaller number of ithreads, even if the desired number are "
854289344Scem    "available");
855289344Scem
856289344Scem/*
857289344Scem * In case it is NOT ok, give consumers an abort button.
858289344Scem */
859289344Scemstatic int ntb_prefer_intx;
860289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN,
861289344Scem    &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather "
862289344Scem    "than remapping MSI-X messages over available slots (match Linux driver "
863289344Scem    "behavior)");
864289344Scem
865289344Scem/*
866289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple
867289344Scem * round-robin fashion.
868289344Scem */
869289342Scemstatic int
870289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
871289344Scem{
872289344Scem	u_int *vectors;
873289344Scem	uint32_t i;
874289344Scem	int rc;
875289344Scem
876289344Scem	if (ntb_prefer_intx != 0)
877289344Scem		return (ENXIO);
878289344Scem
879289344Scem	vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK);
880289344Scem
881289344Scem	for (i = 0; i < desired; i++)
882289344Scem		vectors[i] = (i % avail) + 1;
883289344Scem
884289344Scem	rc = pci_remap_msix(dev, desired, vectors);
885289344Scem	free(vectors, M_NTB);
886289344Scem	return (rc);
887289344Scem}
888289344Scem
889289344Scemstatic int
890289540Scemntb_init_isr(struct ntb_softc *ntb)
891289342Scem{
892289344Scem	uint32_t desired_vectors, num_vectors;
893289342Scem	int rc;
894250079Scarl
895250079Scarl	ntb->allocated_interrupts = 0;
896289542Scem	ntb->last_ts = ticks;
897289347Scem
898250079Scarl	/*
899289546Scem	 * Mask all doorbell interrupts.
900250079Scarl	 */
901289546Scem	ntb_db_set_mask(ntb, ntb->db_valid_mask);
902250079Scarl
903289344Scem	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
904289539Scem	    ntb->db_count);
905289344Scem	if (desired_vectors >= 1) {
906289344Scem		rc = pci_alloc_msix(ntb->device, &num_vectors);
907250079Scarl
908289344Scem		if (ntb_force_remap_mode != 0 && rc == 0 &&
909289344Scem		    num_vectors == desired_vectors)
910289344Scem			num_vectors--;
911289344Scem
912289344Scem		if (rc == 0 && num_vectors < desired_vectors) {
913289344Scem			rc = ntb_remap_msix(ntb->device, desired_vectors,
914289344Scem			    num_vectors);
915289344Scem			if (rc == 0)
916289344Scem				num_vectors = desired_vectors;
917289344Scem			else
918289344Scem				pci_release_msi(ntb->device);
919289344Scem		}
920289344Scem		if (rc != 0)
921289344Scem			num_vectors = 1;
922289344Scem	} else
923289344Scem		num_vectors = 1;
924289344Scem
925289539Scem	if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
926289539Scem		ntb->db_vec_count = 1;
927290680Scem		ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT;
928289539Scem		rc = ntb_setup_legacy_interrupt(ntb);
929289539Scem	} else {
930289546Scem		ntb_create_msix_vec(ntb, num_vectors);
931289540Scem		rc = ntb_setup_msix(ntb, num_vectors);
932289539Scem	}
933289539Scem	if (rc != 0) {
934289539Scem		device_printf(ntb->device,
935289539Scem		    "Error allocating interrupts: %d\n", rc);
936289546Scem		ntb_free_msix_vec(ntb);
937289396Scem	}
938289396Scem
939289342Scem	return (rc);
940289342Scem}
941289342Scem
942289342Scemstatic int
943289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb)
944289342Scem{
945289342Scem	int rc;
946289342Scem
947289342Scem	ntb->int_info[0].rid = 0;
948289342Scem	ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ,
949289342Scem	    &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
950289342Scem	if (ntb->int_info[0].res == NULL) {
951289342Scem		device_printf(ntb->device, "bus_alloc_resource failed\n");
952289342Scem		return (ENOMEM);
953250079Scarl	}
954250079Scarl
955289342Scem	ntb->int_info[0].tag = NULL;
956289342Scem	ntb->allocated_interrupts = 1;
957289342Scem
958289342Scem	rc = bus_setup_intr(ntb->device, ntb->int_info[0].res,
959289546Scem	    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr,
960289342Scem	    ntb, &ntb->int_info[0].tag);
961289342Scem	if (rc != 0) {
962289342Scem		device_printf(ntb->device, "bus_setup_intr failed\n");
963289342Scem		return (ENXIO);
964289342Scem	}
965289342Scem
966250079Scarl	return (0);
967250079Scarl}
968250079Scarl
969250079Scarlstatic void
970250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb)
971250079Scarl{
972250079Scarl	struct ntb_int_info *current_int;
973250079Scarl	int i;
974250079Scarl
975289209Scem	for (i = 0; i < ntb->allocated_interrupts; i++) {
976250079Scarl		current_int = &ntb->int_info[i];
977250079Scarl		if (current_int->tag != NULL)
978250079Scarl			bus_teardown_intr(ntb->device, current_int->res,
979250079Scarl			    current_int->tag);
980250079Scarl
981250079Scarl		if (current_int->res != NULL)
982250079Scarl			bus_release_resource(ntb->device, SYS_RES_IRQ,
983250079Scarl			    rman_get_rid(current_int->res), current_int->res);
984250079Scarl	}
985250079Scarl
986289546Scem	ntb_free_msix_vec(ntb);
987250079Scarl	pci_release_msi(ntb->device);
988250079Scarl}
989250079Scarl
990289347Scem/*
991289648Scem * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon.  Abstract it
992289347Scem * out to make code clearer.
993289347Scem */
994289539Scemstatic inline uint64_t
995289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff)
996289347Scem{
997289347Scem
998289648Scem	if (ntb->type == NTB_ATOM)
999289347Scem		return (ntb_reg_read(8, regoff));
1000289347Scem
1001289347Scem	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
1002289347Scem
1003289347Scem	return (ntb_reg_read(2, regoff));
1004289347Scem}
1005289347Scem
1006289539Scemstatic inline void
1007289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
1008289347Scem{
1009289347Scem
1010289542Scem	KASSERT((val & ~ntb->db_valid_mask) == 0,
1011289542Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
1012289542Scem	     (uintmax_t)(val & ~ntb->db_valid_mask),
1013289542Scem	     (uintmax_t)ntb->db_valid_mask));
1014289542Scem
1015289607Scem	if (regoff == ntb->self_reg->db_mask)
1016289546Scem		DB_MASK_ASSERT(ntb, MA_OWNED);
1017290678Scem	db_iowrite_raw(ntb, regoff, val);
1018290678Scem}
1019289542Scem
1020290678Scemstatic inline void
1021290678Scemdb_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
1022290678Scem{
1023290678Scem
1024289648Scem	if (ntb->type == NTB_ATOM) {
1025289347Scem		ntb_reg_write(8, regoff, val);
1026289347Scem		return;
1027289347Scem	}
1028289347Scem
1029289347Scem	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
1030289347Scem	ntb_reg_write(2, regoff, (uint16_t)val);
1031289347Scem}
1032289347Scem
1033289546Scemvoid
1034289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits)
1035289542Scem{
1036289542Scem
1037289546Scem	DB_MASK_LOCK(ntb);
1038289542Scem	ntb->db_mask |= bits;
1039289607Scem	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
1040289546Scem	DB_MASK_UNLOCK(ntb);
1041289542Scem}
1042289542Scem
1043289546Scemvoid
1044289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits)
1045289542Scem{
1046289542Scem
1047289542Scem	KASSERT((bits & ~ntb->db_valid_mask) == 0,
1048289542Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
1049289542Scem	     (uintmax_t)(bits & ~ntb->db_valid_mask),
1050289542Scem	     (uintmax_t)ntb->db_valid_mask));
1051289542Scem
1052289546Scem	DB_MASK_LOCK(ntb);
1053289542Scem	ntb->db_mask &= ~bits;
1054289607Scem	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
1055289546Scem	DB_MASK_UNLOCK(ntb);
1056289542Scem}
1057289542Scem
1058289546Scemuint64_t
1059289546Scemntb_db_read(struct ntb_softc *ntb)
1060289281Scem{
1061289281Scem
1062289607Scem	return (db_ioread(ntb, ntb->self_reg->db_bell));
1063289281Scem}
1064289281Scem
1065289546Scemvoid
1066289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits)
1067289281Scem{
1068289281Scem
1069289546Scem	KASSERT((bits & ~ntb->db_valid_mask) == 0,
1070289546Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
1071289546Scem	     (uintmax_t)(bits & ~ntb->db_valid_mask),
1072289546Scem	     (uintmax_t)ntb->db_valid_mask));
1073289546Scem
1074289607Scem	db_iowrite(ntb, ntb->self_reg->db_bell, bits);
1075289281Scem}
1076289281Scem
1077289540Scemstatic inline uint64_t
1078289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)
1079250079Scarl{
1080289540Scem	uint64_t shift, mask;
1081250079Scarl
1082289540Scem	shift = ntb->db_vec_shift;
1083289540Scem	mask = (1ull << shift) - 1;
1084289540Scem	return (mask << (shift * db_vector));
1085250079Scarl}
1086250079Scarl
1087250079Scarlstatic void
1088289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
1089250079Scarl{
1090289540Scem	uint64_t vec_mask;
1091250079Scarl
1092289542Scem	ntb->last_ts = ticks;
1093289546Scem	vec_mask = ntb_vec_mask(ntb, vec);
1094250079Scarl
1095289542Scem	if ((vec_mask & ntb->db_link_mask) != 0) {
1096289546Scem		if (ntb_poll_link(ntb))
1097289546Scem			ntb_link_event(ntb);
1098289540Scem	}
1099289540Scem
1100289546Scem	if ((vec_mask & ntb->db_valid_mask) != 0)
1101289546Scem		ntb_db_event(ntb, vec);
1102289546Scem}
1103250079Scarl
1104289546Scemstatic void
1105289546Scemndev_vec_isr(void *arg)
1106289546Scem{
1107289546Scem	struct ntb_vec *nvec = arg;
1108250079Scarl
1109289546Scem	ntb_interrupt(nvec->ntb, nvec->num);
1110250079Scarl}
1111250079Scarl
1112250079Scarlstatic void
1113289546Scemndev_irq_isr(void *arg)
1114250079Scarl{
1115289546Scem	/* If we couldn't set up MSI-X, we only have the one vector. */
1116289546Scem	ntb_interrupt(arg, 0);
1117250079Scarl}
1118250079Scarl
1119250079Scarlstatic int
1120289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors)
1121250079Scarl{
1122289342Scem	uint32_t i;
1123250079Scarl
1124289546Scem	ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB,
1125250079Scarl	    M_ZERO | M_WAITOK);
1126250079Scarl	for (i = 0; i < num_vectors; i++) {
1127289546Scem		ntb->msix_vec[i].num = i;
1128289546Scem		ntb->msix_vec[i].ntb = ntb;
1129250079Scarl	}
1130250079Scarl
1131250079Scarl	return (0);
1132250079Scarl}
1133250079Scarl
1134250079Scarlstatic void
1135289546Scemntb_free_msix_vec(struct ntb_softc *ntb)
1136250079Scarl{
1137250079Scarl
1138289546Scem	if (ntb->msix_vec == NULL)
1139289539Scem		return;
1140289539Scem
1141289546Scem	free(ntb->msix_vec, M_NTB);
1142289546Scem	ntb->msix_vec = NULL;
1143250079Scarl}
1144250079Scarl
1145250079Scarlstatic struct ntb_hw_info *
1146250079Scarlntb_get_device_info(uint32_t device_id)
1147250079Scarl{
1148250079Scarl	struct ntb_hw_info *ep = pci_ids;
1149250079Scarl
1150250079Scarl	while (ep->device_id) {
1151250079Scarl		if (ep->device_id == device_id)
1152250079Scarl			return (ep);
1153250079Scarl		++ep;
1154250079Scarl	}
1155250079Scarl	return (NULL);
1156250079Scarl}
1157250079Scarl
1158289272Scemstatic void
1159289272Scemntb_teardown_xeon(struct ntb_softc *ntb)
1160250079Scarl{
1161250079Scarl
1162289617Scem	if (ntb->reg != NULL)
1163289617Scem		ntb_link_disable(ntb);
1164250079Scarl}
1165250079Scarl
1166289397Scemstatic void
1167289397Scemntb_detect_max_mw(struct ntb_softc *ntb)
1168289397Scem{
1169289397Scem
1170289648Scem	if (ntb->type == NTB_ATOM) {
1171289648Scem		ntb->mw_count = ATOM_MW_COUNT;
1172289397Scem		return;
1173289397Scem	}
1174289397Scem
1175289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1176289539Scem		ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
1177289397Scem	else
1178289539Scem		ntb->mw_count = XEON_SNB_MW_COUNT;
1179289397Scem}
1180289397Scem
1181250079Scarlstatic int
1182289348Scemntb_detect_xeon(struct ntb_softc *ntb)
1183250079Scarl{
1184289348Scem	uint8_t ppd, conn_type;
1185250079Scarl
1186289348Scem	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
1187289348Scem	ntb->ppd = ppd;
1188250079Scarl
1189289348Scem	if ((ppd & XEON_PPD_DEV_TYPE) != 0)
1190290681Scem		ntb->dev_type = NTB_DEV_DSD;
1191290681Scem	else
1192289257Scem		ntb->dev_type = NTB_DEV_USD;
1193289257Scem
1194289397Scem	if ((ppd & XEON_PPD_SPLIT_BAR) != 0)
1195289397Scem		ntb->features |= NTB_SPLIT_BAR;
1196289397Scem
1197289542Scem	/* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */
1198289542Scem	if (HAS_FEATURE(NTB_SB01BASE_LOCKUP))
1199289542Scem		ntb->features |= NTB_SDOORBELL_LOCKUP;
1200289542Scem
1201289348Scem	conn_type = ppd & XEON_PPD_CONN_TYPE;
1202289348Scem	switch (conn_type) {
1203289348Scem	case NTB_CONN_B2B:
1204289348Scem		ntb->conn_type = conn_type;
1205289348Scem		break;
1206289348Scem	case NTB_CONN_RP:
1207289348Scem	case NTB_CONN_TRANSPARENT:
1208289348Scem	default:
1209289348Scem		device_printf(ntb->device, "Unsupported connection type: %u\n",
1210289348Scem		    (unsigned)conn_type);
1211289348Scem		return (ENXIO);
1212289348Scem	}
1213289348Scem	return (0);
1214289348Scem}
1215289348Scem
1216289348Scemstatic int
1217289648Scemntb_detect_atom(struct ntb_softc *ntb)
1218289348Scem{
1219289348Scem	uint32_t ppd, conn_type;
1220289348Scem
1221289348Scem	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
1222289348Scem	ntb->ppd = ppd;
1223289348Scem
1224289648Scem	if ((ppd & ATOM_PPD_DEV_TYPE) != 0)
1225289348Scem		ntb->dev_type = NTB_DEV_DSD;
1226289348Scem	else
1227289348Scem		ntb->dev_type = NTB_DEV_USD;
1228289348Scem
1229289648Scem	conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8;
1230289348Scem	switch (conn_type) {
1231289348Scem	case NTB_CONN_B2B:
1232289348Scem		ntb->conn_type = conn_type;
1233289348Scem		break;
1234289348Scem	default:
1235289348Scem		device_printf(ntb->device, "Unsupported NTB configuration\n");
1236289348Scem		return (ENXIO);
1237289348Scem	}
1238289348Scem	return (0);
1239289348Scem}
1240289348Scem
1241289348Scemstatic int
1242289542Scemntb_xeon_init_dev(struct ntb_softc *ntb)
1243289348Scem{
1244289542Scem	int rc;
1245289348Scem
1246289542Scem	ntb->spad_count		= XEON_SPAD_COUNT;
1247289542Scem	ntb->db_count		= XEON_DB_COUNT;
1248289542Scem	ntb->db_link_mask	= XEON_DB_LINK_BIT;
1249289542Scem	ntb->db_vec_count	= XEON_DB_MSIX_VECTOR_COUNT;
1250289542Scem	ntb->db_vec_shift	= XEON_DB_MSIX_VECTOR_SHIFT;
1251289257Scem
1252289542Scem	if (ntb->conn_type != NTB_CONN_B2B) {
1253250079Scarl		device_printf(ntb->device, "Connection type %d not supported\n",
1254289348Scem		    ntb->conn_type);
1255250079Scarl		return (ENXIO);
1256250079Scarl	}
1257250079Scarl
1258289542Scem	ntb->reg = &xeon_reg;
1259289607Scem	ntb->self_reg = &xeon_pri_reg;
1260289542Scem	ntb->peer_reg = &xeon_b2b_reg;
1261289542Scem	ntb->xlat_reg = &xeon_sec_xlat;
1262289542Scem
1263289208Scem	/*
1264289208Scem	 * There is a Xeon hardware errata related to writes to SDOORBELL or
1265289208Scem	 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space,
1266291263Scem	 * which may hang the system.  To workaround this, use a memory
1267289208Scem	 * window to access the interrupt and scratch pad registers on the
1268289208Scem	 * remote system.
1269289208Scem	 */
1270291263Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
1271291263Scem		ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) %
1272291263Scem		    ntb->mw_count;
1273291263Scem		ntb_printf(2, "Setting up b2b mw idx %d means %u\n",
1274291263Scem		    g_ntb_mw_idx, ntb->b2b_mw_idx);
1275291263Scem		rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, false);
1276291263Scem		KASSERT(rc == 0, ("shouldn't fail"));
1277291263Scem	} else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14))
1278289208Scem		/*
1279289542Scem		 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
1280289542Scem		 * mirrored to the remote system.  Shrink the number of bits by one,
1281289542Scem		 * since bit 14 is the last bit.
1282289542Scem		 *
1283289542Scem		 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
1284289542Scem		 * anyway.  Nor for non-B2B connection types.
1285289542Scem		 */
1286289543Scem		ntb->db_count = XEON_DB_COUNT - 1;
1287250079Scarl
1288289542Scem	ntb->db_valid_mask = (1ull << ntb->db_count) - 1;
1289250079Scarl
1290289542Scem	if (ntb->dev_type == NTB_DEV_USD)
1291289542Scem		rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr,
1292289542Scem		    &xeon_b2b_usd_addr);
1293289542Scem	else
1294289542Scem		rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr,
1295289542Scem		    &xeon_b2b_dsd_addr);
1296289542Scem	if (rc != 0)
1297289542Scem		return (rc);
1298289271Scem
1299250079Scarl	/* Enable Bus Master and Memory Space on the secondary side */
1300290682Scem	ntb_reg_write(2, XEON_SPCICMD_OFFSET,
1301289542Scem	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
1302255279Scarl
1303290682Scem	/*
1304290682Scem	 * Mask all doorbell interrupts.
1305290682Scem	 */
1306290682Scem	ntb_db_set_mask(ntb, ntb->db_valid_mask);
1307250079Scarl
1308290682Scem	rc = ntb_init_isr(ntb);
1309290682Scem	return (rc);
1310250079Scarl}
1311250079Scarl
1312250079Scarlstatic int
1313289648Scemntb_atom_init_dev(struct ntb_softc *ntb)
1314250079Scarl{
1315290682Scem	int error;
1316250079Scarl
1317289348Scem	KASSERT(ntb->conn_type == NTB_CONN_B2B,
1318289348Scem	    ("Unsupported NTB configuration (%d)\n", ntb->conn_type));
1319250079Scarl
1320289648Scem	ntb->spad_count		 = ATOM_SPAD_COUNT;
1321289648Scem	ntb->db_count		 = ATOM_DB_COUNT;
1322289648Scem	ntb->db_vec_count	 = ATOM_DB_MSIX_VECTOR_COUNT;
1323289648Scem	ntb->db_vec_shift	 = ATOM_DB_MSIX_VECTOR_SHIFT;
1324289542Scem	ntb->db_valid_mask	 = (1ull << ntb->db_count) - 1;
1325250079Scarl
1326289648Scem	ntb->reg = &atom_reg;
1327289648Scem	ntb->self_reg = &atom_pri_reg;
1328289648Scem	ntb->peer_reg = &atom_b2b_reg;
1329289648Scem	ntb->xlat_reg = &atom_sec_xlat;
1330289542Scem
1331250079Scarl	/*
1332289648Scem	 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is
1333250079Scarl	 * resolved.  Mask transaction layer internal parity errors.
1334250079Scarl	 */
1335250079Scarl	pci_write_config(ntb->device, 0xFC, 0x4, 4);
1336250079Scarl
1337289648Scem	configure_atom_secondary_side_bars(ntb);
1338250079Scarl
1339250079Scarl	/* Enable Bus Master and Memory Space on the secondary side */
1340290682Scem	ntb_reg_write(2, ATOM_SPCICMD_OFFSET,
1341250079Scarl	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
1342289209Scem
1343290682Scem	error = ntb_init_isr(ntb);
1344290682Scem	if (error != 0)
1345290682Scem		return (error);
1346290682Scem
1347289542Scem	/* Initiate PCI-E link training */
1348289546Scem	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
1349250079Scarl
1350289648Scem	callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb);
1351289542Scem
1352250079Scarl	return (0);
1353250079Scarl}
1354250079Scarl
1355289648Scem/* XXX: Linux driver doesn't seem to do any of this for Atom. */
1356255279Scarlstatic void
1357289648Scemconfigure_atom_secondary_side_bars(struct ntb_softc *ntb)
1358255279Scarl{
1359255279Scarl
1360255279Scarl	if (ntb->dev_type == NTB_DEV_USD) {
1361289648Scem		ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET,
1362290725Scem		    XEON_B2B_BAR2_ADDR64);
1363289648Scem		ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET,
1364290725Scem		    XEON_B2B_BAR4_ADDR64);
1365290725Scem		ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64);
1366290725Scem		ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64);
1367255279Scarl	} else {
1368289648Scem		ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET,
1369290725Scem		    XEON_B2B_BAR2_ADDR64);
1370289648Scem		ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET,
1371290725Scem		    XEON_B2B_BAR4_ADDR64);
1372290725Scem		ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64);
1373290725Scem		ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64);
1374255279Scarl	}
1375255279Scarl}
1376255279Scarl
1377289543Scem
1378289543Scem/*
1379289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a
1380289543Scem * MW, limit the B2B MW to half a MW.  By sharing a MW, half the shared MW
1381289543Scem * remains for use by a higher layer.
1382289543Scem *
1383289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured
1384289543Scem * MW size is sufficiently large.
1385289543Scem */
1386289543Scemstatic unsigned int ntb_b2b_mw_share;
1387289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share,
1388289543Scem    0, "If enabled (non-zero), prefer to share half of the B2B peer register "
1389289543Scem    "MW with higher level consumers.  Both sides of the NTB MUST set the same "
1390289543Scem    "value here.");
1391289543Scem
1392289543Scemstatic void
1393289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx,
1394289543Scem    enum ntb_bar regbar)
1395289543Scem{
1396289543Scem	struct ntb_pci_bar_info *bar;
1397289543Scem	uint8_t bar_sz;
1398289543Scem
1399289543Scem	if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3)
1400289543Scem		return;
1401289543Scem
1402289543Scem	bar = &ntb->bar_info[idx];
1403289543Scem	bar_sz = pci_read_config(ntb->device, bar->psz_off, 1);
1404289543Scem	if (idx == regbar) {
1405289543Scem		if (ntb->b2b_off != 0)
1406289543Scem			bar_sz--;
1407289543Scem		else
1408289543Scem			bar_sz = 0;
1409289543Scem	}
1410289543Scem	pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1);
1411289543Scem	bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1);
1412289543Scem	(void)bar_sz;
1413289543Scem}
1414289543Scem
1415289543Scemstatic void
1416289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr,
1417289543Scem    enum ntb_bar idx, enum ntb_bar regbar)
1418289543Scem{
1419289546Scem	uint64_t reg_val;
1420289546Scem	uint32_t base_reg, lmt_reg;
1421289543Scem
1422289546Scem	bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg);
1423289546Scem	if (idx == regbar)
1424289546Scem		bar_addr += ntb->b2b_off;
1425289543Scem
1426289546Scem	if (!bar_is_64bit(ntb, idx)) {
1427289546Scem		ntb_reg_write(4, base_reg, bar_addr);
1428289546Scem		reg_val = ntb_reg_read(4, base_reg);
1429289546Scem		(void)reg_val;
1430289546Scem
1431289546Scem		ntb_reg_write(4, lmt_reg, bar_addr);
1432289546Scem		reg_val = ntb_reg_read(4, lmt_reg);
1433289546Scem		(void)reg_val;
1434289543Scem	} else {
1435289546Scem		ntb_reg_write(8, base_reg, bar_addr);
1436289546Scem		reg_val = ntb_reg_read(8, base_reg);
1437289546Scem		(void)reg_val;
1438289546Scem
1439289546Scem		ntb_reg_write(8, lmt_reg, bar_addr);
1440289546Scem		reg_val = ntb_reg_read(8, lmt_reg);
1441289546Scem		(void)reg_val;
1442289543Scem	}
1443289543Scem}
1444289543Scem
1445289543Scemstatic void
1446289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx)
1447289543Scem{
1448289543Scem	struct ntb_pci_bar_info *bar;
1449289543Scem
1450289543Scem	bar = &ntb->bar_info[idx];
1451289543Scem	if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) {
1452289543Scem		ntb_reg_write(4, bar->pbarxlat_off, base_addr);
1453289543Scem		base_addr = ntb_reg_read(4, bar->pbarxlat_off);
1454289543Scem	} else {
1455289543Scem		ntb_reg_write(8, bar->pbarxlat_off, base_addr);
1456289543Scem		base_addr = ntb_reg_read(8, bar->pbarxlat_off);
1457289543Scem	}
1458289543Scem	(void)base_addr;
1459289543Scem}
1460289543Scem
1461289542Scemstatic int
1462289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
1463289542Scem    const struct ntb_b2b_addr *peer_addr)
1464255279Scarl{
1465289543Scem	struct ntb_pci_bar_info *b2b_bar;
1466289543Scem	vm_size_t bar_size;
1467289543Scem	uint64_t bar_addr;
1468289543Scem	enum ntb_bar b2b_bar_num, i;
1469255279Scarl
1470289543Scem	if (ntb->b2b_mw_idx == B2B_MW_DISABLED) {
1471289543Scem		b2b_bar = NULL;
1472289543Scem		b2b_bar_num = NTB_CONFIG_BAR;
1473289543Scem		ntb->b2b_off = 0;
1474289543Scem	} else {
1475289543Scem		b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx);
1476289543Scem		KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS,
1477289543Scem		    ("invalid b2b mw bar"));
1478289543Scem
1479289543Scem		b2b_bar = &ntb->bar_info[b2b_bar_num];
1480289543Scem		bar_size = b2b_bar->size;
1481289543Scem
1482289543Scem		if (ntb_b2b_mw_share != 0 &&
1483289543Scem		    (bar_size >> 1) >= XEON_B2B_MIN_SIZE)
1484289543Scem			ntb->b2b_off = bar_size >> 1;
1485289543Scem		else if (bar_size >= XEON_B2B_MIN_SIZE) {
1486289543Scem			ntb->b2b_off = 0;
1487289543Scem		} else {
1488289543Scem			device_printf(ntb->device,
1489289543Scem			    "B2B bar size is too small!\n");
1490289543Scem			return (EIO);
1491289543Scem		}
1492255279Scarl	}
1493289542Scem
1494289543Scem	/*
1495289543Scem	 * Reset the secondary bar sizes to match the primary bar sizes.
1496289543Scem	 * (Except, disable or halve the size of the B2B secondary bar.)
1497289543Scem	 */
1498289543Scem	for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++)
1499289543Scem		xeon_reset_sbar_size(ntb, i, b2b_bar_num);
1500289543Scem
1501289543Scem	bar_addr = 0;
1502289543Scem	if (b2b_bar_num == NTB_CONFIG_BAR)
1503289543Scem		bar_addr = addr->bar0_addr;
1504289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_1)
1505289543Scem		bar_addr = addr->bar2_addr64;
1506289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR))
1507289543Scem		bar_addr = addr->bar4_addr64;
1508289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2)
1509289543Scem		bar_addr = addr->bar4_addr32;
1510289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_3)
1511289543Scem		bar_addr = addr->bar5_addr32;
1512289543Scem	else
1513289543Scem		KASSERT(false, ("invalid bar"));
1514289543Scem
1515289543Scem	ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr);
1516289543Scem
1517289543Scem	/*
1518289543Scem	 * Other SBARs are normally hit by the PBAR xlat, except for the b2b
1519289543Scem	 * register BAR.  The B2B BAR is either disabled above or configured
1520289543Scem	 * half-size.  It starts at PBAR xlat + offset.
1521289543Scem	 *
1522289543Scem	 * Also set up incoming BAR limits == base (zero length window).
1523289543Scem	 */
1524289543Scem	xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1,
1525289543Scem	    b2b_bar_num);
1526289542Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
1527289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32,
1528289543Scem		    NTB_B2B_BAR_2, b2b_bar_num);
1529289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32,
1530289543Scem		    NTB_B2B_BAR_3, b2b_bar_num);
1531289542Scem	} else
1532289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64,
1533289543Scem		    NTB_B2B_BAR_2, b2b_bar_num);
1534289543Scem
1535289543Scem	/* Zero incoming translation addrs */
1536289543Scem	ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0);
1537289543Scem	ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0);
1538289543Scem
1539289543Scem	/* Zero outgoing translation limits (whole bar size windows) */
1540289543Scem	ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0);
1541289543Scem	ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
1542289543Scem
1543289543Scem	/* Set outgoing translation offsets */
1544289543Scem	xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1);
1545289543Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
1546289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2);
1547289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3);
1548289543Scem	} else
1549289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2);
1550289543Scem
1551289543Scem	/* Set the translation offset for B2B registers */
1552289543Scem	bar_addr = 0;
1553289543Scem	if (b2b_bar_num == NTB_CONFIG_BAR)
1554289543Scem		bar_addr = peer_addr->bar0_addr;
1555289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_1)
1556289543Scem		bar_addr = peer_addr->bar2_addr64;
1557289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR))
1558289543Scem		bar_addr = peer_addr->bar4_addr64;
1559289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2)
1560289543Scem		bar_addr = peer_addr->bar4_addr32;
1561289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_3)
1562289543Scem		bar_addr = peer_addr->bar5_addr32;
1563289543Scem	else
1564289543Scem		KASSERT(false, ("invalid bar"));
1565289543Scem
1566289543Scem	/*
1567289543Scem	 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits
1568289543Scem	 * at a time.
1569289543Scem	 */
1570289543Scem	ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff);
1571289543Scem	ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32);
1572289542Scem	return (0);
1573255279Scarl}
1574255279Scarl
1575289546Scemstatic inline bool
1576289546Scemlink_is_up(struct ntb_softc *ntb)
1577289546Scem{
1578289546Scem
1579289611Scem	if (ntb->type == NTB_XEON) {
1580289611Scem		if (ntb->conn_type == NTB_CONN_TRANSPARENT)
1581289611Scem			return (true);
1582289546Scem		return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0);
1583289611Scem	}
1584289546Scem
1585289648Scem	KASSERT(ntb->type == NTB_ATOM, ("ntb type"));
1586289648Scem	return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0);
1587289546Scem}
1588289546Scem
1589289546Scemstatic inline bool
1590289648Scematom_link_is_err(struct ntb_softc *ntb)
1591289546Scem{
1592289546Scem	uint32_t status;
1593289546Scem
1594289648Scem	KASSERT(ntb->type == NTB_ATOM, ("ntb type"));
1595289546Scem
1596289648Scem	status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET);
1597289648Scem	if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0)
1598289546Scem		return (true);
1599289546Scem
1600289648Scem	status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET);
1601289648Scem	return ((status & ATOM_IBIST_ERR_OFLOW) != 0);
1602289546Scem}
1603289546Scem
1604289648Scem/* Atom does not have link status interrupt, poll on that platform */
1605250079Scarlstatic void
1606289648Scematom_link_hb(void *arg)
1607250079Scarl{
1608250079Scarl	struct ntb_softc *ntb = arg;
1609289546Scem	sbintime_t timo, poll_ts;
1610250079Scarl
1611289546Scem	timo = NTB_HB_TIMEOUT * hz;
1612289546Scem	poll_ts = ntb->last_ts + timo;
1613289546Scem
1614289542Scem	/*
1615289542Scem	 * Delay polling the link status if an interrupt was received, unless
1616289542Scem	 * the cached link status says the link is down.
1617289542Scem	 */
1618289546Scem	if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) {
1619289546Scem		timo = poll_ts - ticks;
1620289542Scem		goto out;
1621289546Scem	}
1622289542Scem
1623289546Scem	if (ntb_poll_link(ntb))
1624289546Scem		ntb_link_event(ntb);
1625289542Scem
1626289648Scem	if (!link_is_up(ntb) && atom_link_is_err(ntb)) {
1627289546Scem		/* Link is down with error, proceed with recovery */
1628289648Scem		callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb);
1629289546Scem		return;
1630250079Scarl	}
1631250079Scarl
1632289542Scemout:
1633289648Scem	callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb);
1634250079Scarl}
1635250079Scarl
1636250079Scarlstatic void
1637289648Scematom_perform_link_restart(struct ntb_softc *ntb)
1638250079Scarl{
1639250079Scarl	uint32_t status;
1640250079Scarl
1641250079Scarl	/* Driver resets the NTB ModPhy lanes - magic! */
1642289648Scem	ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0);
1643289648Scem	ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40);
1644289648Scem	ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60);
1645289648Scem	ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60);
1646250079Scarl
1647250079Scarl	/* Driver waits 100ms to allow the NTB ModPhy to settle */
1648250079Scarl	pause("ModPhy", hz / 10);
1649250079Scarl
1650250079Scarl	/* Clear AER Errors, write to clear */
1651289648Scem	status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET);
1652250079Scarl	status &= PCIM_AER_COR_REPLAY_ROLLOVER;
1653289648Scem	ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status);
1654250079Scarl
1655250079Scarl	/* Clear unexpected electrical idle event in LTSSM, write to clear */
1656289648Scem	status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET);
1657289648Scem	status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1658289648Scem	ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status);
1659250079Scarl
1660250079Scarl	/* Clear DeSkew Buffer error, write to clear */
1661289648Scem	status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET);
1662289648Scem	status |= ATOM_DESKEWSTS_DBERR;
1663289648Scem	ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status);
1664250079Scarl
1665289648Scem	status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET);
1666289648Scem	status &= ATOM_IBIST_ERR_OFLOW;
1667289648Scem	ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status);
1668250079Scarl
1669250079Scarl	/* Releases the NTB state machine to allow the link to retrain */
1670289648Scem	status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET);
1671289648Scem	status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1672289648Scem	ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status);
1673250079Scarl}
1674250079Scarl
1675289546Scem/*
1676289546Scem * ntb_set_ctx() - associate a driver context with an ntb device
1677289546Scem * @ntb:        NTB device context
1678289546Scem * @ctx:        Driver context
1679289546Scem * @ctx_ops:    Driver context operations
1680289546Scem *
1681289546Scem * Associate a driver context and operations with a ntb device.  The context is
1682289546Scem * provided by the client driver, and the driver may associate a different
1683289546Scem * context with each ntb device.
1684289546Scem *
1685289546Scem * Return: Zero if the context is associated, otherwise an error number.
1686289546Scem */
1687289546Scemint
1688289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops)
1689250079Scarl{
1690250079Scarl
1691289546Scem	if (ctx == NULL || ops == NULL)
1692289546Scem		return (EINVAL);
1693289546Scem	if (ntb->ctx_ops != NULL)
1694289546Scem		return (EINVAL);
1695250079Scarl
1696289546Scem	CTX_LOCK(ntb);
1697289546Scem	if (ntb->ctx_ops != NULL) {
1698289546Scem		CTX_UNLOCK(ntb);
1699289546Scem		return (EINVAL);
1700250079Scarl	}
1701289546Scem	ntb->ntb_ctx = ctx;
1702289546Scem	ntb->ctx_ops = ops;
1703289546Scem	CTX_UNLOCK(ntb);
1704250079Scarl
1705289546Scem	return (0);
1706250079Scarl}
1707250079Scarl
1708289546Scem/*
1709289546Scem * It is expected that this will only be used from contexts where the ctx_lock
1710289546Scem * is not needed to protect ntb_ctx lifetime.
1711289546Scem */
1712289546Scemvoid *
1713289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops)
1714289546Scem{
1715289546Scem
1716289546Scem	KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus"));
1717289546Scem	if (ops != NULL)
1718289546Scem		*ops = ntb->ctx_ops;
1719289546Scem	return (ntb->ntb_ctx);
1720289546Scem}
1721289546Scem
1722289546Scem/*
1723289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device
1724289546Scem * @ntb:        NTB device context
1725289546Scem *
1726289546Scem * Clear any association that may exist between a driver context and the ntb
1727289546Scem * device.
1728289546Scem */
1729289546Scemvoid
1730289546Scemntb_clear_ctx(struct ntb_softc *ntb)
1731289546Scem{
1732289546Scem
1733289546Scem	CTX_LOCK(ntb);
1734289546Scem	ntb->ntb_ctx = NULL;
1735289546Scem	ntb->ctx_ops = NULL;
1736289546Scem	CTX_UNLOCK(ntb);
1737289546Scem}
1738289546Scem
1739289546Scem/*
1740289546Scem * ntb_link_event() - notify driver context of a change in link status
1741289546Scem * @ntb:        NTB device context
1742289546Scem *
1743289546Scem * Notify the driver context that the link status may have changed.  The driver
1744289546Scem * should call ntb_link_is_up() to get the current status.
1745289546Scem */
1746289546Scemvoid
1747289546Scemntb_link_event(struct ntb_softc *ntb)
1748289546Scem{
1749289546Scem
1750289546Scem	CTX_LOCK(ntb);
1751289546Scem	if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL)
1752289546Scem		ntb->ctx_ops->link_event(ntb->ntb_ctx);
1753289546Scem	CTX_UNLOCK(ntb);
1754289546Scem}
1755289546Scem
1756289546Scem/*
1757289546Scem * ntb_db_event() - notify driver context of a doorbell event
1758289546Scem * @ntb:        NTB device context
1759289546Scem * @vector:     Interrupt vector number
1760289546Scem *
1761289546Scem * Notify the driver context of a doorbell event.  If hardware supports
1762289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which
1763289546Scem * vector received the interrupt.  The vector number is relative to the first
1764289546Scem * vector used for doorbells, starting at zero, and must be less than
1765289546Scem * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
1766289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of
1767289546Scem * those bits are associated with the vector number.
1768289546Scem */
1769250079Scarlstatic void
1770289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec)
1771289272Scem{
1772289546Scem
1773289546Scem	CTX_LOCK(ntb);
1774289546Scem	if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL)
1775289546Scem		ntb->ctx_ops->db_event(ntb->ntb_ctx, vec);
1776289546Scem	CTX_UNLOCK(ntb);
1777289546Scem}
1778289546Scem
1779289546Scem/*
1780289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb
1781289546Scem * @ntb:        NTB device context
1782289546Scem * @max_speed:  The maximum link speed expressed as PCIe generation number[0]
1783289546Scem * @max_width:  The maximum link width expressed as the number of PCIe lanes[0]
1784289546Scem *
1785289546Scem * Enable the link on the secondary side of the ntb.  This can only be done
1786289546Scem * from the primary side of the ntb in primary or b2b topology.  The ntb device
1787289546Scem * should train the link to its maximum speed and width, or the requested speed
1788289546Scem * and width, whichever is smaller, if supported.
1789289546Scem *
1790289546Scem * Return: Zero on success, otherwise an error number.
1791289546Scem *
1792289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed
1793289546Scem *      and width input will be ignored.
1794289546Scem */
1795289546Scemint
1796289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused,
1797289546Scem    enum ntb_width w __unused)
1798289546Scem{
1799289280Scem	uint32_t cntl;
1800289272Scem
1801289648Scem	if (ntb->type == NTB_ATOM) {
1802289542Scem		pci_write_config(ntb->device, NTB_PPD_OFFSET,
1803289648Scem		    ntb->ppd | ATOM_PPD_INIT_LINK, 4);
1804289546Scem		return (0);
1805289542Scem	}
1806289542Scem
1807289280Scem	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
1808289546Scem		ntb_link_event(ntb);
1809289546Scem		return (0);
1810289280Scem	}
1811289280Scem
1812289542Scem	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1813289280Scem	cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
1814289280Scem	cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
1815289397Scem	cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
1816289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1817289397Scem		cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP;
1818289542Scem	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
1819289546Scem	return (0);
1820289272Scem}
1821289272Scem
1822289546Scem/*
1823289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb
1824289546Scem * @ntb:        NTB device context
1825289546Scem *
1826289546Scem * Disable the link on the secondary side of the ntb.  This can only be done
1827289546Scem * from the primary side of the ntb in primary or b2b topology.  The ntb device
1828289546Scem * should disable the link.  Returning from this call must indicate that a
1829289546Scem * barrier has passed, though with no more writes may pass in either direction
1830289546Scem * across the link, except if this call returns an error number.
1831289546Scem *
1832289546Scem * Return: Zero on success, otherwise an error number.
1833289546Scem */
1834289546Scemint
1835289542Scemntb_link_disable(struct ntb_softc *ntb)
1836289272Scem{
1837289272Scem	uint32_t cntl;
1838289272Scem
1839289272Scem	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
1840289546Scem		ntb_link_event(ntb);
1841289546Scem		return (0);
1842289272Scem	}
1843289272Scem
1844289542Scem	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1845289280Scem	cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
1846289397Scem	cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
1847289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1848289397Scem		cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP);
1849289280Scem	cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
1850289542Scem	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
1851289546Scem	return (0);
1852289272Scem}
1853289272Scem
1854289272Scemstatic void
1855289648Scemrecover_atom_link(void *arg)
1856250079Scarl{
1857250079Scarl	struct ntb_softc *ntb = arg;
1858289608Scem	unsigned speed, width, oldspeed, oldwidth;
1859250079Scarl	uint32_t status32;
1860250079Scarl
1861289648Scem	atom_perform_link_restart(ntb);
1862250079Scarl
1863289232Scem	/*
1864289232Scem	 * There is a potential race between the 2 NTB devices recovering at
1865289232Scem	 * the same time.  If the times are the same, the link will not recover
1866289232Scem	 * and the driver will be stuck in this loop forever.  Add a random
1867289232Scem	 * interval to the recovery time to prevent this race.
1868289232Scem	 */
1869289648Scem	status32 = arc4random() % ATOM_LINK_RECOVERY_TIME;
1870289648Scem	pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000);
1871289232Scem
1872289648Scem	if (atom_link_is_err(ntb))
1873250079Scarl		goto retry;
1874250079Scarl
1875289542Scem	status32 = ntb_reg_read(4, ntb->reg->ntb_ctl);
1876289648Scem	if ((status32 & ATOM_CNTL_LINK_DOWN) != 0)
1877289232Scem		goto out;
1878289232Scem
1879289542Scem	status32 = ntb_reg_read(4, ntb->reg->lnk_sta);
1880289608Scem	width = NTB_LNK_STA_WIDTH(status32);
1881289608Scem	speed = status32 & NTB_LINK_SPEED_MASK;
1882289608Scem
1883289608Scem	oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta);
1884289608Scem	oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK;
1885289608Scem	if (oldwidth != width || oldspeed != speed)
1886250079Scarl		goto retry;
1887250079Scarl
1888289232Scemout:
1889289648Scem	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb,
1890289542Scem	    ntb);
1891250079Scarl	return;
1892250079Scarl
1893250079Scarlretry:
1894289648Scem	callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link,
1895250079Scarl	    ntb);
1896250079Scarl}
1897250079Scarl
1898289546Scem/*
1899289546Scem * Polls the HW link status register(s); returns true if something has changed.
1900289546Scem */
1901289546Scemstatic bool
1902289542Scemntb_poll_link(struct ntb_softc *ntb)
1903250079Scarl{
1904250079Scarl	uint32_t ntb_cntl;
1905289546Scem	uint16_t reg_val;
1906250079Scarl
1907289648Scem	if (ntb->type == NTB_ATOM) {
1908289542Scem		ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1909289546Scem		if (ntb_cntl == ntb->ntb_ctl)
1910289546Scem			return (false);
1911289546Scem
1912289542Scem		ntb->ntb_ctl = ntb_cntl;
1913289542Scem		ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta);
1914250079Scarl	} else {
1915290678Scem		db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask);
1916250079Scarl
1917289546Scem		reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
1918289546Scem		if (reg_val == ntb->lnk_sta)
1919289546Scem			return (false);
1920250079Scarl
1921289546Scem		ntb->lnk_sta = reg_val;
1922289542Scem	}
1923289546Scem	return (true);
1924289542Scem}
1925289542Scem
1926289546Scemstatic inline enum ntb_speed
1927289546Scemntb_link_sta_speed(struct ntb_softc *ntb)
1928250079Scarl{
1929250079Scarl
1930289546Scem	if (!link_is_up(ntb))
1931289546Scem		return (NTB_SPEED_NONE);
1932289546Scem	return (ntb->lnk_sta & NTB_LINK_SPEED_MASK);
1933250079Scarl}
1934250079Scarl
1935289546Scemstatic inline enum ntb_width
1936289546Scemntb_link_sta_width(struct ntb_softc *ntb)
1937250079Scarl{
1938250079Scarl
1939289546Scem	if (!link_is_up(ntb))
1940289546Scem		return (NTB_WIDTH_NONE);
1941289546Scem	return (NTB_LNK_STA_WIDTH(ntb->lnk_sta));
1942250079Scarl}
1943250079Scarl
1944289774ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0,
1945289774Scem    "Driver state, statistics, and HW registers");
1946289774Scem
1947289774Scem#define NTB_REGSZ_MASK	(3ul << 30)
1948289774Scem#define NTB_REG_64	(1ul << 30)
1949289774Scem#define NTB_REG_32	(2ul << 30)
1950289774Scem#define NTB_REG_16	(3ul << 30)
1951289774Scem#define NTB_REG_8	(0ul << 30)
1952289774Scem
1953289774Scem#define NTB_DB_READ	(1ul << 29)
1954289774Scem#define NTB_PCI_REG	(1ul << 28)
1955289774Scem#define NTB_REGFLAGS_MASK	(NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG)
1956289774Scem
1957289774Scemstatic void
1958289774Scemntb_sysctl_init(struct ntb_softc *ntb)
1959289774Scem{
1960289774Scem	struct sysctl_oid_list *tree_par, *regpar, *statpar, *errpar;
1961289774Scem	struct sysctl_ctx_list *ctx;
1962289774Scem	struct sysctl_oid *tree, *tmptree;
1963289774Scem
1964289774Scem	ctx = device_get_sysctl_ctx(ntb->device);
1965289774Scem
1966289774Scem	tree = SYSCTL_ADD_NODE(ctx,
1967289774Scem	    SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)), OID_AUTO,
1968289774Scem	    "debug_info", CTLFLAG_RD, NULL,
1969289774Scem	    "Driver state, statistics, and HW registers");
1970289774Scem	tree_par = SYSCTL_CHILDREN(tree);
1971289774Scem
1972289774Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD,
1973289774Scem	    &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port");
1974289774Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD,
1975289774Scem	    &ntb->dev_type, 0, "0 - USD; 1 - DSD");
1976290687Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD,
1977290687Scem	    &ntb->ppd, 0, "Raw PPD register (cached)");
1978289774Scem
1979289774Scem	if (ntb->b2b_mw_idx != B2B_MW_DISABLED) {
1980289774Scem		SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD,
1981289774Scem		    &ntb->b2b_mw_idx, 0,
1982289774Scem		    "Index of the MW used for B2B remote register access");
1983289774Scem		SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off",
1984289774Scem		    CTLFLAG_RD, &ntb->b2b_off,
1985289774Scem		    "If non-zero, offset of B2B register region in shared MW");
1986289774Scem	}
1987289774Scem
1988289774Scem	SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features",
1989289774Scem	    CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A",
1990289774Scem	    "Features/errata of this NTB device");
1991289774Scem
1992289774Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD,
1993290686Scem	    __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0,
1994290686Scem	    "NTB CTL register (cached)");
1995289774Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD,
1996290686Scem	    __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0,
1997290686Scem	    "LNK STA register (cached)");
1998289774Scem
1999289774Scem	SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "link_status",
2000289774Scem	    CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_link_status,
2001289774Scem	    "A", "Link status");
2002289774Scem
2003289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD,
2004291263Scem	    &ntb->mw_count, 0, "MW count");
2005289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD,
2006289774Scem	    &ntb->spad_count, 0, "Scratchpad count");
2007289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD,
2008289774Scem	    &ntb->db_count, 0, "Doorbell count");
2009289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD,
2010289774Scem	    &ntb->db_vec_count, 0, "Doorbell vector count");
2011289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD,
2012289774Scem	    &ntb->db_vec_shift, 0, "Doorbell vector shift");
2013289774Scem
2014289774Scem	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD,
2015289774Scem	    &ntb->db_valid_mask, "Doorbell valid mask");
2016289774Scem	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD,
2017289774Scem	    &ntb->db_link_mask, "Doorbell link mask");
2018289774Scem	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD,
2019289774Scem	    &ntb->db_mask, "Doorbell mask (cached)");
2020289774Scem
2021289774Scem	tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers",
2022289774Scem	    CTLFLAG_RD, NULL, "Raw HW registers (big-endian)");
2023289774Scem	regpar = SYSCTL_CHILDREN(tmptree);
2024289774Scem
2025290682Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl",
2026290682Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 |
2027290682Scem	    ntb->reg->ntb_ctl, sysctl_handle_register, "IU",
2028290682Scem	    "NTB Control register");
2029290682Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap",
2030290682Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 |
2031290682Scem	    0x19c, sysctl_handle_register, "IU",
2032290682Scem	    "NTB Link Capabilities");
2033290682Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon",
2034290682Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 |
2035290682Scem	    0x1a0, sysctl_handle_register, "IU",
2036290682Scem	    "NTB Link Control register");
2037290682Scem
2038289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask",
2039289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2040289774Scem	    NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask,
2041289774Scem	    sysctl_handle_register, "QU", "Doorbell mask register");
2042289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell",
2043289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2044289774Scem	    NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell,
2045289774Scem	    sysctl_handle_register, "QU", "Doorbell register");
2046289774Scem
2047289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23",
2048289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2049289774Scem	    NTB_REG_64 | ntb->xlat_reg->bar2_xlat,
2050289774Scem	    sysctl_handle_register, "QU", "Incoming XLAT23 register");
2051289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2052289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4",
2053289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2054289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar4_xlat,
2055289774Scem		    sysctl_handle_register, "IU", "Incoming XLAT4 register");
2056289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5",
2057289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2058289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar5_xlat,
2059289774Scem		    sysctl_handle_register, "IU", "Incoming XLAT5 register");
2060289774Scem	} else {
2061289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45",
2062289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2063289774Scem		    NTB_REG_64 | ntb->xlat_reg->bar4_xlat,
2064289774Scem		    sysctl_handle_register, "QU", "Incoming XLAT45 register");
2065289774Scem	}
2066289774Scem
2067289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23",
2068289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2069289774Scem	    NTB_REG_64 | ntb->xlat_reg->bar2_limit,
2070289774Scem	    sysctl_handle_register, "QU", "Incoming LMT23 register");
2071289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2072289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4",
2073289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2074289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar4_limit,
2075289774Scem		    sysctl_handle_register, "IU", "Incoming LMT4 register");
2076289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5",
2077289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2078289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar5_limit,
2079289774Scem		    sysctl_handle_register, "IU", "Incoming LMT5 register");
2080289774Scem	} else {
2081289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45",
2082289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2083289774Scem		    NTB_REG_64 | ntb->xlat_reg->bar4_limit,
2084289774Scem		    sysctl_handle_register, "QU", "Incoming LMT45 register");
2085289774Scem	}
2086289774Scem
2087289774Scem	if (ntb->type == NTB_ATOM)
2088289774Scem		return;
2089289774Scem
2090289774Scem	tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats",
2091289774Scem	    CTLFLAG_RD, NULL, "Xeon HW statistics");
2092289774Scem	statpar = SYSCTL_CHILDREN(tmptree);
2093289774Scem	SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss",
2094289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2095289774Scem	    NTB_REG_16 | XEON_USMEMMISS_OFFSET,
2096289774Scem	    sysctl_handle_register, "SU", "Upstream Memory Miss");
2097289774Scem
2098289774Scem	tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err",
2099289774Scem	    CTLFLAG_RD, NULL, "Xeon HW errors");
2100289774Scem	errpar = SYSCTL_CHILDREN(tmptree);
2101289774Scem
2102290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd",
2103289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2104290687Scem	    NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET,
2105290687Scem	    sysctl_handle_register, "CU", "PPD");
2106290687Scem
2107290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz",
2108290687Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2109290687Scem	    NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET,
2110290687Scem	    sysctl_handle_register, "CU", "PBAR23 SZ (log2)");
2111290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz",
2112290687Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2113290687Scem	    NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET,
2114290687Scem	    sysctl_handle_register, "CU", "PBAR4 SZ (log2)");
2115290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz",
2116290687Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2117290687Scem	    NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET,
2118290687Scem	    sysctl_handle_register, "CU", "PBAR5 SZ (log2)");
2119290687Scem
2120290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz",
2121290687Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2122290687Scem	    NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET,
2123290687Scem	    sysctl_handle_register, "CU", "SBAR23 SZ (log2)");
2124290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz",
2125290687Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2126290687Scem	    NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET,
2127290687Scem	    sysctl_handle_register, "CU", "SBAR4 SZ (log2)");
2128290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz",
2129290687Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2130290687Scem	    NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET,
2131290687Scem	    sysctl_handle_register, "CU", "SBAR5 SZ (log2)");
2132290687Scem
2133290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts",
2134290687Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2135289774Scem	    NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET,
2136289774Scem	    sysctl_handle_register, "SU", "DEVSTS");
2137290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts",
2138289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2139289774Scem	    NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET,
2140289774Scem	    sysctl_handle_register, "SU", "LNKSTS");
2141290687Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts",
2142290687Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2143290687Scem	    NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET,
2144290687Scem	    sysctl_handle_register, "SU", "SLNKSTS");
2145290687Scem
2146289774Scem	SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts",
2147289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2148289774Scem	    NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET,
2149289774Scem	    sysctl_handle_register, "IU", "UNCERRSTS");
2150289774Scem	SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts",
2151289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2152289774Scem	    NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET,
2153289774Scem	    sysctl_handle_register, "IU", "CORERRSTS");
2154289774Scem
2155289774Scem	if (ntb->conn_type != NTB_CONN_B2B)
2156289774Scem		return;
2157289774Scem
2158289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23",
2159289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2160289774Scem	    NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off,
2161289774Scem	    sysctl_handle_register, "QU", "Outgoing XLAT23 register");
2162289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2163289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4",
2164289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2165289774Scem		    NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off,
2166289774Scem		    sysctl_handle_register, "IU", "Outgoing XLAT4 register");
2167289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5",
2168289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2169289774Scem		    NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off,
2170289774Scem		    sysctl_handle_register, "IU", "Outgoing XLAT5 register");
2171289774Scem	} else {
2172289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45",
2173289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2174289774Scem		    NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off,
2175289774Scem		    sysctl_handle_register, "QU", "Outgoing XLAT45 register");
2176289774Scem	}
2177289774Scem
2178289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23",
2179289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2180289774Scem	    NTB_REG_64 | XEON_PBAR2LMT_OFFSET,
2181289774Scem	    sysctl_handle_register, "QU", "Outgoing LMT23 register");
2182289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2183289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4",
2184289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2185289774Scem		    NTB_REG_32 | XEON_PBAR4LMT_OFFSET,
2186289774Scem		    sysctl_handle_register, "IU", "Outgoing LMT4 register");
2187289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5",
2188289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2189289774Scem		    NTB_REG_32 | XEON_PBAR5LMT_OFFSET,
2190289774Scem		    sysctl_handle_register, "IU", "Outgoing LMT5 register");
2191289774Scem	} else {
2192289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45",
2193289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2194289774Scem		    NTB_REG_64 | XEON_PBAR4LMT_OFFSET,
2195289774Scem		    sysctl_handle_register, "QU", "Outgoing LMT45 register");
2196289774Scem	}
2197289774Scem
2198289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base",
2199289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2200289774Scem	    NTB_REG_64 | ntb->xlat_reg->bar0_base,
2201289774Scem	    sysctl_handle_register, "QU", "Secondary BAR01 base register");
2202289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base",
2203289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2204289774Scem	    NTB_REG_64 | ntb->xlat_reg->bar2_base,
2205289774Scem	    sysctl_handle_register, "QU", "Secondary BAR23 base register");
2206289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2207289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base",
2208289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2209289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar4_base,
2210289774Scem		    sysctl_handle_register, "IU",
2211289774Scem		    "Secondary BAR4 base register");
2212289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base",
2213289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2214289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar5_base,
2215289774Scem		    sysctl_handle_register, "IU",
2216289774Scem		    "Secondary BAR5 base register");
2217289774Scem	} else {
2218289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base",
2219289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2220289774Scem		    NTB_REG_64 | ntb->xlat_reg->bar4_base,
2221289774Scem		    sysctl_handle_register, "QU",
2222289774Scem		    "Secondary BAR45 base register");
2223289774Scem	}
2224289774Scem}
2225289774Scem
2226289774Scemstatic int
2227289774Scemsysctl_handle_features(SYSCTL_HANDLER_ARGS)
2228289774Scem{
2229289774Scem	struct ntb_softc *ntb;
2230289774Scem	struct sbuf sb;
2231289774Scem	int error;
2232289774Scem
2233289774Scem	error = 0;
2234289774Scem	ntb = arg1;
2235289774Scem
2236289774Scem	sbuf_new_for_sysctl(&sb, NULL, 256, req);
2237289774Scem
2238289774Scem	sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR);
2239289774Scem	error = sbuf_finish(&sb);
2240289774Scem	sbuf_delete(&sb);
2241289774Scem
2242289774Scem	if (error || !req->newptr)
2243289774Scem		return (error);
2244289774Scem	return (EINVAL);
2245289774Scem}
2246289774Scem
2247289774Scemstatic int
2248289774Scemsysctl_handle_link_status(SYSCTL_HANDLER_ARGS)
2249289774Scem{
2250289774Scem	struct ntb_softc *ntb;
2251289774Scem	struct sbuf sb;
2252289774Scem	enum ntb_speed speed;
2253289774Scem	enum ntb_width width;
2254289774Scem	int error;
2255289774Scem
2256289774Scem	error = 0;
2257289774Scem	ntb = arg1;
2258289774Scem
2259289774Scem	sbuf_new_for_sysctl(&sb, NULL, 32, req);
2260289774Scem
2261289774Scem	if (ntb_link_is_up(ntb, &speed, &width))
2262289774Scem		sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u",
2263289774Scem		    (unsigned)speed, (unsigned)width);
2264289774Scem	else
2265289774Scem		sbuf_printf(&sb, "down");
2266289774Scem
2267289774Scem	error = sbuf_finish(&sb);
2268289774Scem	sbuf_delete(&sb);
2269289774Scem
2270289774Scem	if (error || !req->newptr)
2271289774Scem		return (error);
2272289774Scem	return (EINVAL);
2273289774Scem}
2274289774Scem
2275289774Scemstatic int
2276289774Scemsysctl_handle_register(SYSCTL_HANDLER_ARGS)
2277289774Scem{
2278289774Scem	struct ntb_softc *ntb;
2279289774Scem	const void *outp;
2280289774Scem	uintptr_t sz;
2281289774Scem	uint64_t umv;
2282289774Scem	char be[sizeof(umv)];
2283289774Scem	size_t outsz;
2284289774Scem	uint32_t reg;
2285289774Scem	bool db, pci;
2286289774Scem	int error;
2287289774Scem
2288289774Scem	ntb = arg1;
2289289774Scem	reg = arg2 & ~NTB_REGFLAGS_MASK;
2290289774Scem	sz = arg2 & NTB_REGSZ_MASK;
2291289774Scem	db = (arg2 & NTB_DB_READ) != 0;
2292289774Scem	pci = (arg2 & NTB_PCI_REG) != 0;
2293289774Scem
2294289774Scem	KASSERT(!(db && pci), ("bogus"));
2295289774Scem
2296289774Scem	if (db) {
2297289774Scem		KASSERT(sz == NTB_REG_64, ("bogus"));
2298289774Scem		umv = db_ioread(ntb, reg);
2299289774Scem		outsz = sizeof(uint64_t);
2300289774Scem	} else {
2301289774Scem		switch (sz) {
2302289774Scem		case NTB_REG_64:
2303289774Scem			if (pci)
2304289774Scem				umv = pci_read_config(ntb->device, reg, 8);
2305289774Scem			else
2306289774Scem				umv = ntb_reg_read(8, reg);
2307289774Scem			outsz = sizeof(uint64_t);
2308289774Scem			break;
2309289774Scem		case NTB_REG_32:
2310289774Scem			if (pci)
2311289774Scem				umv = pci_read_config(ntb->device, reg, 4);
2312289774Scem			else
2313289774Scem				umv = ntb_reg_read(4, reg);
2314289774Scem			outsz = sizeof(uint32_t);
2315289774Scem			break;
2316289774Scem		case NTB_REG_16:
2317289774Scem			if (pci)
2318289774Scem				umv = pci_read_config(ntb->device, reg, 2);
2319289774Scem			else
2320289774Scem				umv = ntb_reg_read(2, reg);
2321289774Scem			outsz = sizeof(uint16_t);
2322289774Scem			break;
2323289774Scem		case NTB_REG_8:
2324289774Scem			if (pci)
2325289774Scem				umv = pci_read_config(ntb->device, reg, 1);
2326289774Scem			else
2327289774Scem				umv = ntb_reg_read(1, reg);
2328289774Scem			outsz = sizeof(uint8_t);
2329289774Scem			break;
2330289774Scem		default:
2331289774Scem			panic("bogus");
2332289774Scem			break;
2333289774Scem		}
2334289774Scem	}
2335289774Scem
2336289774Scem	/* Encode bigendian so that sysctl -x is legible. */
2337289774Scem	be64enc(be, umv);
2338289774Scem	outp = ((char *)be) + sizeof(umv) - outsz;
2339289774Scem
2340289774Scem	error = SYSCTL_OUT(req, outp, outsz);
2341289774Scem	if (error || !req->newptr)
2342289774Scem		return (error);
2343289774Scem	return (EINVAL);
2344289774Scem}
2345289774Scem
2346291263Scemstatic unsigned
2347291263Scemntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx)
2348291263Scem{
2349291263Scem
2350291263Scem	if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 &&
2351291263Scem	    uidx >= ntb->b2b_mw_idx)
2352291263Scem		return (uidx + 1);
2353291263Scem	return (uidx);
2354291263Scem}
2355291263Scem
2356289546Scem/*
2357289546Scem * Public API to the rest of the OS
2358250079Scarl */
2359250079Scarl
2360250079Scarl/**
2361250079Scarl * ntb_get_max_spads() - get the total scratch regs usable
2362250079Scarl * @ntb: pointer to ntb_softc instance
2363250079Scarl *
2364250079Scarl * This function returns the max 32bit scratchpad registers usable by the
2365250079Scarl * upper layer.
2366250079Scarl *
2367250079Scarl * RETURNS: total number of scratch pad registers available
2368250079Scarl */
2369289208Scemuint8_t
2370250079Scarlntb_get_max_spads(struct ntb_softc *ntb)
2371250079Scarl{
2372250079Scarl
2373289539Scem	return (ntb->spad_count);
2374250079Scarl}
2375250079Scarl
2376291263Scem/*
2377291263Scem * ntb_mw_count() - Get the number of memory windows available for KPI
2378291263Scem * consumers.
2379291263Scem *
2380291263Scem * (Excludes any MW wholly reserved for register access.)
2381291263Scem */
2382289396Scemuint8_t
2383289539Scemntb_mw_count(struct ntb_softc *ntb)
2384289396Scem{
2385289396Scem
2386291263Scem	if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0)
2387291263Scem		return (ntb->mw_count - 1);
2388289539Scem	return (ntb->mw_count);
2389289396Scem}
2390289396Scem
2391250079Scarl/**
2392289545Scem * ntb_spad_write() - write to the secondary scratchpad register
2393250079Scarl * @ntb: pointer to ntb_softc instance
2394250079Scarl * @idx: index to the scratchpad register, 0 based
2395250079Scarl * @val: the data value to put into the register
2396250079Scarl *
2397250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad
2398250079Scarl * register. The register resides on the secondary (external) side.
2399250079Scarl *
2400289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
2401250079Scarl */
2402250079Scarlint
2403289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
2404250079Scarl{
2405250079Scarl
2406289539Scem	if (idx >= ntb->spad_count)
2407250079Scarl		return (EINVAL);
2408250079Scarl
2409289607Scem	ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val);
2410250079Scarl
2411250079Scarl	return (0);
2412250079Scarl}
2413250079Scarl
2414250079Scarl/**
2415289545Scem * ntb_spad_read() - read from the primary scratchpad register
2416250079Scarl * @ntb: pointer to ntb_softc instance
2417250079Scarl * @idx: index to scratchpad register, 0 based
2418250079Scarl * @val: pointer to 32bit integer for storing the register value
2419250079Scarl *
2420250079Scarl * This function allows reading of the 32bit scratchpad register on
2421250079Scarl * the primary (internal) side.
2422250079Scarl *
2423289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
2424250079Scarl */
2425250079Scarlint
2426289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
2427250079Scarl{
2428250079Scarl
2429289539Scem	if (idx >= ntb->spad_count)
2430250079Scarl		return (EINVAL);
2431250079Scarl
2432289607Scem	*val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4);
2433250079Scarl
2434250079Scarl	return (0);
2435250079Scarl}
2436250079Scarl
2437250079Scarl/**
2438289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register
2439250079Scarl * @ntb: pointer to ntb_softc instance
2440250079Scarl * @idx: index to the scratchpad register, 0 based
2441250079Scarl * @val: the data value to put into the register
2442250079Scarl *
2443250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad
2444250079Scarl * register. The register resides on the secondary (external) side.
2445250079Scarl *
2446289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
2447250079Scarl */
2448250079Scarlint
2449289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
2450250079Scarl{
2451250079Scarl
2452289539Scem	if (idx >= ntb->spad_count)
2453250079Scarl		return (EINVAL);
2454250079Scarl
2455289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
2456290682Scem		ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val);
2457255279Scarl	else
2458289542Scem		ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val);
2459250079Scarl
2460250079Scarl	return (0);
2461250079Scarl}
2462250079Scarl
2463250079Scarl/**
2464289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register
2465250079Scarl * @ntb: pointer to ntb_softc instance
2466250079Scarl * @idx: index to scratchpad register, 0 based
2467250079Scarl * @val: pointer to 32bit integer for storing the register value
2468250079Scarl *
2469250079Scarl * This function allows reading of the 32bit scratchpad register on
2470250079Scarl * the primary (internal) side.
2471250079Scarl *
2472289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
2473250079Scarl */
2474250079Scarlint
2475289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
2476250079Scarl{
2477250079Scarl
2478289539Scem	if (idx >= ntb->spad_count)
2479250079Scarl		return (EINVAL);
2480250079Scarl
2481289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
2482290682Scem		*val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4);
2483255279Scarl	else
2484289542Scem		*val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4);
2485250079Scarl
2486250079Scarl	return (0);
2487250079Scarl}
2488250079Scarl
2489289546Scem/*
2490289546Scem * ntb_mw_get_range() - get the range of a memory window
2491289546Scem * @ntb:        NTB device context
2492289546Scem * @idx:        Memory window number
2493289546Scem * @base:       OUT - the base address for mapping the memory window
2494289546Scem * @size:       OUT - the size for mapping the memory window
2495289546Scem * @align:      OUT - the base alignment for translating the memory window
2496289546Scem * @align_size: OUT - the size alignment for translating the memory window
2497250079Scarl *
2498289546Scem * Get the range of a memory window.  NULL may be given for any output
2499289546Scem * parameter if the value is not needed.  The base and size may be used for
2500289546Scem * mapping the memory window, to access the peer memory.  The alignment and
2501289546Scem * size may be used for translating the memory window, for the peer to access
2502289546Scem * memory on the local system.
2503250079Scarl *
2504289546Scem * Return: Zero on success, otherwise an error number.
2505250079Scarl */
2506289546Scemint
2507289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base,
2508291033Scem    caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
2509291033Scem    bus_addr_t *plimit)
2510250079Scarl{
2511289546Scem	struct ntb_pci_bar_info *bar;
2512291033Scem	bus_addr_t limit;
2513289546Scem	size_t bar_b2b_off;
2514291033Scem	enum ntb_bar bar_num;
2515250079Scarl
2516289546Scem	if (mw_idx >= ntb_mw_count(ntb))
2517289546Scem		return (EINVAL);
2518291263Scem	mw_idx = ntb_user_mw_to_idx(ntb, mw_idx);
2519250079Scarl
2520291033Scem	bar_num = ntb_mw_to_bar(ntb, mw_idx);
2521291033Scem	bar = &ntb->bar_info[bar_num];
2522289546Scem	bar_b2b_off = 0;
2523289546Scem	if (mw_idx == ntb->b2b_mw_idx) {
2524289546Scem		KASSERT(ntb->b2b_off != 0,
2525289546Scem		    ("user shouldn't get non-shared b2b mw"));
2526289546Scem		bar_b2b_off = ntb->b2b_off;
2527289546Scem	}
2528250079Scarl
2529291033Scem	if (bar_is_64bit(ntb, bar_num))
2530291033Scem		limit = BUS_SPACE_MAXADDR;
2531291033Scem	else
2532291033Scem		limit = BUS_SPACE_MAXADDR_32BIT;
2533291033Scem
2534289546Scem	if (base != NULL)
2535289546Scem		*base = bar->pbase + bar_b2b_off;
2536289546Scem	if (vbase != NULL)
2537290679Scem		*vbase = bar->vbase + bar_b2b_off;
2538289546Scem	if (size != NULL)
2539289546Scem		*size = bar->size - bar_b2b_off;
2540289546Scem	if (align != NULL)
2541289546Scem		*align = bar->size;
2542289546Scem	if (align_size != NULL)
2543289546Scem		*align_size = 1;
2544291033Scem	if (plimit != NULL)
2545291033Scem		*plimit = limit;
2546289546Scem	return (0);
2547250079Scarl}
2548250079Scarl
2549289546Scem/*
2550289546Scem * ntb_mw_set_trans() - set the translation of a memory window
2551289546Scem * @ntb:        NTB device context
2552289546Scem * @idx:        Memory window number
2553289546Scem * @addr:       The dma address local memory to expose to the peer
2554289546Scem * @size:       The size of the local memory to expose to the peer
2555250079Scarl *
2556289546Scem * Set the translation of a memory window.  The peer may access local memory
2557289546Scem * through the window starting at the address, up to the size.  The address
2558289546Scem * must be aligned to the alignment specified by ntb_mw_get_range().  The size
2559291033Scem * must be aligned to the size alignment specified by ntb_mw_get_range().  The
2560291033Scem * address must be below the plimit specified by ntb_mw_get_range() (i.e. for
2561291033Scem * 32-bit BARs).
2562250079Scarl *
2563289546Scem * Return: Zero on success, otherwise an error number.
2564250079Scarl */
2565289546Scemint
2566289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr,
2567289546Scem    size_t size)
2568250079Scarl{
2569289546Scem	struct ntb_pci_bar_info *bar;
2570289546Scem	uint64_t base, limit, reg_val;
2571289546Scem	size_t bar_size, mw_size;
2572289546Scem	uint32_t base_reg, xlat_reg, limit_reg;
2573289546Scem	enum ntb_bar bar_num;
2574250079Scarl
2575289546Scem	if (idx >= ntb_mw_count(ntb))
2576289546Scem		return (EINVAL);
2577291263Scem	idx = ntb_user_mw_to_idx(ntb, idx);
2578250079Scarl
2579289546Scem	bar_num = ntb_mw_to_bar(ntb, idx);
2580289546Scem	bar = &ntb->bar_info[bar_num];
2581250079Scarl
2582289546Scem	bar_size = bar->size;
2583289546Scem	if (idx == ntb->b2b_mw_idx)
2584289546Scem		mw_size = bar_size - ntb->b2b_off;
2585289546Scem	else
2586289546Scem		mw_size = bar_size;
2587250079Scarl
2588289546Scem	/* Hardware requires that addr is aligned to bar size */
2589289546Scem	if ((addr & (bar_size - 1)) != 0)
2590289546Scem		return (EINVAL);
2591250079Scarl
2592289546Scem	if (size > mw_size)
2593289546Scem		return (EINVAL);
2594289546Scem
2595289546Scem	bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg);
2596289546Scem
2597289546Scem	limit = 0;
2598289546Scem	if (bar_is_64bit(ntb, bar_num)) {
2599291032Scem		base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK;
2600289546Scem
2601289546Scem		if (limit_reg != 0 && size != mw_size)
2602289546Scem			limit = base + size;
2603289546Scem
2604289546Scem		/* Set and verify translation address */
2605289546Scem		ntb_reg_write(8, xlat_reg, addr);
2606291032Scem		reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK;
2607289546Scem		if (reg_val != addr) {
2608289546Scem			ntb_reg_write(8, xlat_reg, 0);
2609289546Scem			return (EIO);
2610289546Scem		}
2611289546Scem
2612289546Scem		/* Set and verify the limit */
2613289546Scem		ntb_reg_write(8, limit_reg, limit);
2614291032Scem		reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK;
2615289546Scem		if (reg_val != limit) {
2616289546Scem			ntb_reg_write(8, limit_reg, base);
2617289546Scem			ntb_reg_write(8, xlat_reg, 0);
2618289546Scem			return (EIO);
2619289546Scem		}
2620289546Scem	} else {
2621289546Scem		/* Configure 32-bit (split) BAR MW */
2622289546Scem
2623291029Scem		if ((addr & UINT32_MAX) != addr)
2624291033Scem			return (ERANGE);
2625291029Scem		if (((addr + size) & UINT32_MAX) != (addr + size))
2626291033Scem			return (ERANGE);
2627289546Scem
2628291032Scem		base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK;
2629289546Scem
2630289546Scem		if (limit_reg != 0 && size != mw_size)
2631289546Scem			limit = base + size;
2632289546Scem
2633289546Scem		/* Set and verify translation address */
2634289546Scem		ntb_reg_write(4, xlat_reg, addr);
2635291032Scem		reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK;
2636289546Scem		if (reg_val != addr) {
2637289546Scem			ntb_reg_write(4, xlat_reg, 0);
2638289546Scem			return (EIO);
2639289546Scem		}
2640289546Scem
2641289546Scem		/* Set and verify the limit */
2642289546Scem		ntb_reg_write(4, limit_reg, limit);
2643291032Scem		reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK;
2644289546Scem		if (reg_val != limit) {
2645289546Scem			ntb_reg_write(4, limit_reg, base);
2646289546Scem			ntb_reg_write(4, xlat_reg, 0);
2647289546Scem			return (EIO);
2648289546Scem		}
2649250079Scarl	}
2650289546Scem	return (0);
2651250079Scarl}
2652250079Scarl
2653289596Scem/*
2654289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window
2655289596Scem * @ntb:	NTB device context
2656289596Scem * @idx:	Memory window number
2657289596Scem *
2658289596Scem * Clear the translation of a memory window.  The peer may no longer access
2659289596Scem * local memory through the window.
2660289596Scem *
2661289596Scem * Return: Zero on success, otherwise an error number.
2662289596Scem */
2663289596Scemint
2664289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx)
2665289596Scem{
2666289596Scem
2667289596Scem	return (ntb_mw_set_trans(ntb, mw_idx, 0, 0));
2668289596Scem}
2669289596Scem
2670291031Scem/*
2671291031Scem * ntb_mw_get_wc - Get the write-combine status of a memory window
2672291031Scem *
2673291031Scem * Returns:  Zero on success, setting *wc; otherwise an error number (e.g. if
2674291031Scem * idx is an invalid memory window).
2675291031Scem */
2676291031Scemint
2677291031Scemntb_mw_get_wc(struct ntb_softc *ntb, unsigned idx, bool *wc)
2678291031Scem{
2679291031Scem	struct ntb_pci_bar_info *bar;
2680291031Scem
2681291031Scem	if (idx >= ntb_mw_count(ntb))
2682291031Scem		return (EINVAL);
2683291263Scem	idx = ntb_user_mw_to_idx(ntb, idx);
2684291031Scem
2685291031Scem	bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)];
2686291031Scem	*wc = bar->mapped_wc;
2687291031Scem	return (0);
2688291031Scem}
2689291031Scem
2690291031Scem/*
2691291031Scem * ntb_mw_set_wc - Set the write-combine status of a memory window
2692291031Scem *
2693291031Scem * If 'wc' matches the current status, this does nothing and succeeds.
2694291031Scem *
2695291031Scem * Returns:  Zero on success, setting the caching attribute on the virtual
2696291031Scem * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid
2697291031Scem * memory window, or if changing the caching attribute fails).
2698291031Scem */
2699291031Scemint
2700291031Scemntb_mw_set_wc(struct ntb_softc *ntb, unsigned idx, bool wc)
2701291031Scem{
2702291263Scem
2703291263Scem	if (idx >= ntb_mw_count(ntb))
2704291263Scem		return (EINVAL);
2705291263Scem
2706291263Scem	idx = ntb_user_mw_to_idx(ntb, idx);
2707291263Scem	return (ntb_mw_set_wc_internal(ntb, idx, wc));
2708291263Scem}
2709291263Scem
2710291263Scemstatic int
2711291263Scemntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, bool wc)
2712291263Scem{
2713291031Scem	struct ntb_pci_bar_info *bar;
2714291031Scem	vm_memattr_t attr;
2715291031Scem	int rc;
2716291031Scem
2717291031Scem	bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)];
2718291031Scem	if (bar->mapped_wc == wc)
2719291031Scem		return (0);
2720291031Scem
2721291031Scem	if (wc)
2722291031Scem		attr = VM_MEMATTR_WRITE_COMBINING;
2723291031Scem	else
2724291031Scem		attr = VM_MEMATTR_DEFAULT;
2725291031Scem
2726291031Scem	rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, attr);
2727291031Scem	if (rc == 0)
2728291031Scem		bar->mapped_wc = wc;
2729291031Scem
2730291031Scem	return (rc);
2731291031Scem}
2732291031Scem
2733250079Scarl/**
2734289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side
2735250079Scarl * @ntb: pointer to ntb_softc instance
2736289545Scem * @bit: doorbell bits to ring
2737250079Scarl *
2738250079Scarl * This function allows triggering of a doorbell on the secondary/external
2739250079Scarl * side that will initiate an interrupt on the remote host
2740250079Scarl */
2741250079Scarlvoid
2742289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit)
2743250079Scarl{
2744250079Scarl
2745289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
2746290682Scem		ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit);
2747289347Scem		return;
2748289209Scem	}
2749289347Scem
2750289546Scem	db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
2751250079Scarl}
2752250079Scarl
2753289542Scem/*
2754289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register,
2755289542Scem * as well as the size of the register (via *sz_out).
2756289542Scem *
2757289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell
2758289542Scem * ring to its memory window write.
2759289542Scem *
2760289542Scem * Note that writing the peer doorbell via a memory window will *not* generate
2761289542Scem * an interrupt on the remote host; that must be done seperately.
2762289542Scem */
2763289542Scembus_addr_t
2764289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out)
2765289542Scem{
2766289542Scem	struct ntb_pci_bar_info *bar;
2767289542Scem	uint64_t regoff;
2768289542Scem
2769289542Scem	KASSERT(sz_out != NULL, ("must be non-NULL"));
2770289542Scem
2771289542Scem	if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
2772289542Scem		bar = &ntb->bar_info[NTB_CONFIG_BAR];
2773289542Scem		regoff = ntb->peer_reg->db_bell;
2774289542Scem	} else {
2775289543Scem		KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED,
2776289543Scem		    ("invalid b2b idx"));
2777289542Scem
2778289542Scem		bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)];
2779290682Scem		regoff = XEON_PDOORBELL_OFFSET;
2780289542Scem	}
2781289542Scem	KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh"));
2782289542Scem
2783289542Scem	*sz_out = ntb->reg->db_size;
2784289542Scem	/* HACK: Specific to current x86 bus implementation. */
2785289542Scem	return ((uint64_t)bar->pci_bus_handle + regoff);
2786289542Scem}
2787289542Scem
2788289597Scem/*
2789289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb
2790289597Scem * @ntb:	NTB device context
2791289597Scem *
2792289597Scem * Hardware may support different number or arrangement of doorbell bits.
2793289597Scem *
2794289597Scem * Return: A mask of doorbell bits supported by the ntb.
2795289597Scem */
2796289597Scemuint64_t
2797289597Scemntb_db_valid_mask(struct ntb_softc *ntb)
2798289597Scem{
2799289597Scem
2800289597Scem	return (ntb->db_valid_mask);
2801289597Scem}
2802289597Scem
2803289598Scem/*
2804289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector
2805289598Scem * @ntb:	NTB device context
2806289598Scem * @vector:	Doorbell vector number
2807289598Scem *
2808289598Scem * Each interrupt vector may have a different number or arrangement of bits.
2809289598Scem *
2810289598Scem * Return: A mask of doorbell bits serviced by a vector.
2811289598Scem */
2812289598Scemuint64_t
2813289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector)
2814289598Scem{
2815289598Scem
2816289598Scem	if (vector > ntb->db_vec_count)
2817289598Scem		return (0);
2818289598Scem	return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector));
2819289598Scem}
2820289598Scem
2821250079Scarl/**
2822289546Scem * ntb_link_is_up() - get the current ntb link state
2823289546Scem * @ntb:        NTB device context
2824289546Scem * @speed:      OUT - The link speed expressed as PCIe generation number
2825289546Scem * @width:      OUT - The link width expressed as the number of PCIe lanes
2826250079Scarl *
2827250079Scarl * RETURNS: true or false based on the hardware link state
2828250079Scarl */
2829250079Scarlbool
2830289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed,
2831289546Scem    enum ntb_width *width)
2832250079Scarl{
2833250079Scarl
2834289546Scem	if (speed != NULL)
2835289546Scem		*speed = ntb_link_sta_speed(ntb);
2836289546Scem	if (width != NULL)
2837289546Scem		*width = ntb_link_sta_width(ntb);
2838289546Scem	return (link_is_up(ntb));
2839250079Scarl}
2840250079Scarl
2841255272Scarlstatic void
2842255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar)
2843250079Scarl{
2844255272Scarl
2845289209Scem	bar->pci_bus_tag = rman_get_bustag(bar->pci_resource);
2846289209Scem	bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource);
2847289209Scem	bar->pbase = rman_get_start(bar->pci_resource);
2848289209Scem	bar->size = rman_get_size(bar->pci_resource);
2849289209Scem	bar->vbase = rman_get_virtual(bar->pci_resource);
2850250079Scarl}
2851255268Scarl
2852289209Scemdevice_t
2853289209Scemntb_get_device(struct ntb_softc *ntb)
2854255268Scarl{
2855255268Scarl
2856255268Scarl	return (ntb->device);
2857255268Scarl}
2858289208Scem
2859289208Scem/* Export HW-specific errata information. */
2860289208Scembool
2861289774Scemntb_has_feature(struct ntb_softc *ntb, uint32_t feature)
2862289208Scem{
2863289208Scem
2864289208Scem	return (HAS_FEATURE(feature));
2865289208Scem}
2866