ntb_hw_intel.c revision 289774
1250079Scarl/*-
2250079Scarl * Copyright (C) 2013 Intel Corporation
3289542Scem * Copyright (C) 2015 EMC Corporation
4250079Scarl * All rights reserved.
5250079Scarl *
6250079Scarl * Redistribution and use in source and binary forms, with or without
7250079Scarl * modification, are permitted provided that the following conditions
8250079Scarl * are met:
9250079Scarl * 1. Redistributions of source code must retain the above copyright
10250079Scarl *    notice, this list of conditions and the following disclaimer.
11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright
12250079Scarl *    notice, this list of conditions and the following disclaimer in the
13250079Scarl *    documentation and/or other materials provided with the distribution.
14250079Scarl *
15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18250079Scarl * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25250079Scarl * SUCH DAMAGE.
26250079Scarl */
27250079Scarl
28250079Scarl#include <sys/cdefs.h>
29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289774 2015-10-22 23:03:15Z cem $");
30250079Scarl
31250079Scarl#include <sys/param.h>
32250079Scarl#include <sys/kernel.h>
33250079Scarl#include <sys/systm.h>
34250079Scarl#include <sys/bus.h>
35289774Scem#include <sys/endian.h>
36250079Scarl#include <sys/malloc.h>
37250079Scarl#include <sys/module.h>
38250079Scarl#include <sys/queue.h>
39250079Scarl#include <sys/rman.h>
40289774Scem#include <sys/sbuf.h>
41289207Scem#include <sys/sysctl.h>
42250079Scarl#include <vm/vm.h>
43250079Scarl#include <vm/pmap.h>
44250079Scarl#include <machine/bus.h>
45250079Scarl#include <machine/pmap.h>
46250079Scarl#include <machine/resource.h>
47250079Scarl#include <dev/pci/pcireg.h>
48250079Scarl#include <dev/pci/pcivar.h>
49250079Scarl
50250079Scarl#include "ntb_regs.h"
51250079Scarl#include "ntb_hw.h"
52250079Scarl
53250079Scarl/*
54250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that
55250079Scarl * allows you to connect two systems using a PCI-e link.
56250079Scarl *
57250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows
58250079Scarl * you to send and recieve interrupts, map the memory windows and send and
59250079Scarl * receive messages in the scratch-pad registers.
60250079Scarl *
61250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may
62250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license.
63250079Scarl */
64250079Scarl
65289648Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT)
66250079Scarl
67289539Scem#define NTB_HB_TIMEOUT		1 /* second */
68289648Scem#define ATOM_LINK_RECOVERY_TIME	500 /* ms */
69250079Scarl
70250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev))
71250079Scarl
72250079Scarlenum ntb_device_type {
73250079Scarl	NTB_XEON,
74289648Scem	NTB_ATOM
75250079Scarl};
76250079Scarl
77289610Scem/* ntb_conn_type are hardware numbers, cannot change. */
78289610Scemenum ntb_conn_type {
79289610Scem	NTB_CONN_TRANSPARENT = 0,
80289610Scem	NTB_CONN_B2B = 1,
81289610Scem	NTB_CONN_RP = 2,
82289610Scem};
83289610Scem
84289610Scemenum ntb_b2b_direction {
85289610Scem	NTB_DEV_USD = 0,
86289610Scem	NTB_DEV_DSD = 1,
87289610Scem};
88289610Scem
89289539Scemenum ntb_bar {
90289539Scem	NTB_CONFIG_BAR = 0,
91289539Scem	NTB_B2B_BAR_1,
92289539Scem	NTB_B2B_BAR_2,
93289539Scem	NTB_B2B_BAR_3,
94289539Scem	NTB_MAX_BARS
95289539Scem};
96289539Scem
97255274Scarl/* Device features and workarounds */
98255274Scarl#define HAS_FEATURE(feature)	\
99255274Scarl	((ntb->features & (feature)) != 0)
100255274Scarl
101250079Scarlstruct ntb_hw_info {
102250079Scarl	uint32_t		device_id;
103255274Scarl	const char		*desc;
104250079Scarl	enum ntb_device_type	type;
105289397Scem	uint32_t		features;
106250079Scarl};
107250079Scarl
108250079Scarlstruct ntb_pci_bar_info {
109250079Scarl	bus_space_tag_t		pci_bus_tag;
110250079Scarl	bus_space_handle_t	pci_bus_handle;
111250079Scarl	int			pci_resource_id;
112250079Scarl	struct resource		*pci_resource;
113250079Scarl	vm_paddr_t		pbase;
114250079Scarl	void			*vbase;
115250079Scarl	u_long			size;
116289543Scem
117289543Scem	/* Configuration register offsets */
118289543Scem	uint32_t		psz_off;
119289543Scem	uint32_t		ssz_off;
120289543Scem	uint32_t		pbarxlat_off;
121250079Scarl};
122250079Scarl
123250079Scarlstruct ntb_int_info {
124250079Scarl	struct resource	*res;
125250079Scarl	int		rid;
126250079Scarl	void		*tag;
127250079Scarl};
128250079Scarl
129289546Scemstruct ntb_vec {
130250079Scarl	struct ntb_softc	*ntb;
131289546Scem	uint32_t		num;
132250079Scarl};
133250079Scarl
134289542Scemstruct ntb_reg {
135289542Scem	uint32_t	ntb_ctl;
136289542Scem	uint32_t	lnk_sta;
137289542Scem	uint8_t		db_size;
138289542Scem	unsigned	mw_bar[NTB_MAX_BARS];
139289542Scem};
140289542Scem
141289542Scemstruct ntb_alt_reg {
142289542Scem	uint32_t	db_bell;
143289542Scem	uint32_t	db_mask;
144289542Scem	uint32_t	spad;
145289542Scem};
146289542Scem
147289542Scemstruct ntb_xlat_reg {
148289546Scem	uint32_t	bar0_base;
149289546Scem	uint32_t	bar2_base;
150289546Scem	uint32_t	bar4_base;
151289546Scem	uint32_t	bar5_base;
152289546Scem
153289546Scem	uint32_t	bar2_xlat;
154289546Scem	uint32_t	bar4_xlat;
155289546Scem	uint32_t	bar5_xlat;
156289546Scem
157289546Scem	uint32_t	bar2_limit;
158289546Scem	uint32_t	bar4_limit;
159289546Scem	uint32_t	bar5_limit;
160289542Scem};
161289542Scem
162289542Scemstruct ntb_b2b_addr {
163289542Scem	uint64_t	bar0_addr;
164289542Scem	uint64_t	bar2_addr64;
165289542Scem	uint64_t	bar4_addr64;
166289542Scem	uint64_t	bar4_addr32;
167289542Scem	uint64_t	bar5_addr32;
168289542Scem};
169289542Scem
170250079Scarlstruct ntb_softc {
171250079Scarl	device_t		device;
172250079Scarl	enum ntb_device_type	type;
173289774Scem	uint32_t		features;
174250079Scarl
175250079Scarl	struct ntb_pci_bar_info	bar_info[NTB_MAX_BARS];
176250079Scarl	struct ntb_int_info	int_info[MAX_MSIX_INTERRUPTS];
177250079Scarl	uint32_t		allocated_interrupts;
178250079Scarl
179250079Scarl	struct callout		heartbeat_timer;
180250079Scarl	struct callout		lr_timer;
181250079Scarl
182289546Scem	void			*ntb_ctx;
183289546Scem	const struct ntb_ctx_ops *ctx_ops;
184289546Scem	struct ntb_vec		*msix_vec;
185289546Scem#define CTX_LOCK(sc)		mtx_lock_spin(&(sc)->ctx_lock)
186289546Scem#define CTX_UNLOCK(sc)		mtx_unlock_spin(&(sc)->ctx_lock)
187289546Scem#define CTX_ASSERT(sc,f)	mtx_assert(&(sc)->ctx_lock, (f))
188289546Scem	struct mtx		ctx_lock;
189250079Scarl
190289610Scem	uint32_t		ppd;
191289610Scem	enum ntb_conn_type	conn_type;
192289610Scem	enum ntb_b2b_direction	dev_type;
193289539Scem
194289542Scem	/* Offset of peer bar0 in B2B BAR */
195289542Scem	uint64_t			b2b_off;
196289542Scem	/* Memory window used to access peer bar0 */
197289543Scem#define B2B_MW_DISABLED			UINT8_MAX
198289542Scem	uint8_t				b2b_mw_idx;
199289542Scem
200289539Scem	uint8_t				mw_count;
201289539Scem	uint8_t				spad_count;
202289539Scem	uint8_t				db_count;
203289539Scem	uint8_t				db_vec_count;
204289539Scem	uint8_t				db_vec_shift;
205289542Scem
206289546Scem	/* Protects local db_mask. */
207289546Scem#define DB_MASK_LOCK(sc)	mtx_lock_spin(&(sc)->db_mask_lock)
208289546Scem#define DB_MASK_UNLOCK(sc)	mtx_unlock_spin(&(sc)->db_mask_lock)
209289546Scem#define DB_MASK_ASSERT(sc,f)	mtx_assert(&(sc)->db_mask_lock, (f))
210289542Scem	struct mtx			db_mask_lock;
211289542Scem
212289546Scem	uint32_t			ntb_ctl;
213289546Scem	uint32_t			lnk_sta;
214289542Scem
215289542Scem	uint64_t			db_valid_mask;
216289542Scem	uint64_t			db_link_mask;
217289546Scem	uint64_t			db_mask;
218289542Scem
219289542Scem	int				last_ts;	/* ticks @ last irq */
220289542Scem
221289542Scem	const struct ntb_reg		*reg;
222289542Scem	const struct ntb_alt_reg	*self_reg;
223289542Scem	const struct ntb_alt_reg	*peer_reg;
224289542Scem	const struct ntb_xlat_reg	*xlat_reg;
225250079Scarl};
226250079Scarl
227289234Scem#ifdef __i386__
228289234Scemstatic __inline uint64_t
229289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
230289234Scem    bus_size_t offset)
231289234Scem{
232289234Scem
233289234Scem	return (bus_space_read_4(tag, handle, offset) |
234289234Scem	    ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32);
235289234Scem}
236289234Scem
237289234Scemstatic __inline void
238289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
239289234Scem    bus_size_t offset, uint64_t val)
240289234Scem{
241289234Scem
242289234Scem	bus_space_write_4(tag, handle, offset, val);
243289234Scem	bus_space_write_4(tag, handle, offset + 4, val >> 32);
244289234Scem}
245289234Scem#endif
246289234Scem
247255279Scarl#define ntb_bar_read(SIZE, bar, offset) \
248255279Scarl	    bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
249255279Scarl	    ntb->bar_info[(bar)].pci_bus_handle, (offset))
250255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \
251255279Scarl	    bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
252255279Scarl	    ntb->bar_info[(bar)].pci_bus_handle, (offset), (val))
253255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
254250079Scarl#define ntb_reg_write(SIZE, offset, val) \
255255279Scarl	    ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
256289397Scem#define ntb_mw_read(SIZE, offset) \
257289542Scem	    ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset)
258255279Scarl#define ntb_mw_write(SIZE, offset, val) \
259289542Scem	    ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \
260289397Scem		offset, val)
261250079Scarl
262250079Scarlstatic int ntb_probe(device_t device);
263250079Scarlstatic int ntb_attach(device_t device);
264250079Scarlstatic int ntb_detach(device_t device);
265289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw);
266289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar);
267289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar,
268289546Scem    uint32_t *base, uint32_t *xlat, uint32_t *lmt);
269255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb);
270289647Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *,
271289647Scem    const char *);
272255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
273255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb,
274255272Scarl    struct ntb_pci_bar_info *bar);
275250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb);
276289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
277289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb);
278289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
279289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
280250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb);
281289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);
282289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec);
283289546Scemstatic void ndev_vec_isr(void *arg);
284289546Scemstatic void ndev_irq_isr(void *arg);
285289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff);
286289546Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t val);
287289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors);
288289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb);
289250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
290289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb);
291289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb);
292289648Scemstatic int ntb_detect_atom(struct ntb_softc *ntb);
293289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb);
294289648Scemstatic int ntb_atom_init_dev(struct ntb_softc *ntb);
295289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb);
296289648Scemstatic void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
297289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx,
298289543Scem    enum ntb_bar regbar);
299289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *,
300289543Scem    uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar);
301289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
302289543Scem    enum ntb_bar idx);
303289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *,
304289542Scem    const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
305289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb);
306289648Scemstatic inline bool atom_link_is_err(struct ntb_softc *ntb);
307289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *);
308289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *);
309289648Scemstatic void atom_link_hb(void *arg);
310289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec);
311289648Scemstatic void recover_atom_link(void *arg);
312289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb);
313255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar);
314289774Scemstatic void ntb_sysctl_init(struct ntb_softc *);
315289774Scemstatic int sysctl_handle_features(SYSCTL_HANDLER_ARGS);
316289774Scemstatic int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS);
317289774Scemstatic int sysctl_handle_register(SYSCTL_HANDLER_ARGS);
318250079Scarl
319250079Scarlstatic struct ntb_hw_info pci_ids[] = {
320289612Scem	/* XXX: PS/SS IDs left out until they are supported. */
321289612Scem	{ 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B",
322289648Scem		NTB_ATOM, 0 },
323289233Scem
324289233Scem	{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
325289538Scem		NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
326289233Scem	{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
327289538Scem		NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
328289233Scem	{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
329289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
330289538Scem		    NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K },
331289233Scem	{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
332289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
333289538Scem		    NTB_SB01BASE_LOCKUP },
334289233Scem	{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
335289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
336289538Scem		    NTB_SB01BASE_LOCKUP },
337289233Scem
338289648Scem	{ 0x00000000, NULL, NTB_ATOM, 0 }
339250079Scarl};
340250079Scarl
341289648Scemstatic const struct ntb_reg atom_reg = {
342289648Scem	.ntb_ctl = ATOM_NTBCNTL_OFFSET,
343289648Scem	.lnk_sta = ATOM_LINK_STATUS_OFFSET,
344289542Scem	.db_size = sizeof(uint64_t),
345289542Scem	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
346289542Scem};
347289542Scem
348289648Scemstatic const struct ntb_alt_reg atom_pri_reg = {
349289648Scem	.db_bell = ATOM_PDOORBELL_OFFSET,
350289648Scem	.db_mask = ATOM_PDBMSK_OFFSET,
351289648Scem	.spad = ATOM_SPAD_OFFSET,
352289607Scem};
353289607Scem
354289648Scemstatic const struct ntb_alt_reg atom_b2b_reg = {
355289648Scem	.db_bell = ATOM_B2B_DOORBELL_OFFSET,
356289648Scem	.spad = ATOM_B2B_SPAD_OFFSET,
357289542Scem};
358289542Scem
359289648Scemstatic const struct ntb_xlat_reg atom_sec_xlat = {
360289542Scem#if 0
361289542Scem	/* "FIXME" says the Linux driver. */
362289648Scem	.bar0_base = ATOM_SBAR0BASE_OFFSET,
363289648Scem	.bar2_base = ATOM_SBAR2BASE_OFFSET,
364289648Scem	.bar4_base = ATOM_SBAR4BASE_OFFSET,
365289546Scem
366289648Scem	.bar2_limit = ATOM_SBAR2LMT_OFFSET,
367289648Scem	.bar4_limit = ATOM_SBAR4LMT_OFFSET,
368289542Scem#endif
369289546Scem
370289648Scem	.bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
371289648Scem	.bar4_xlat = ATOM_SBAR4XLAT_OFFSET,
372289542Scem};
373289542Scem
374289542Scemstatic const struct ntb_reg xeon_reg = {
375289542Scem	.ntb_ctl = XEON_NTBCNTL_OFFSET,
376289542Scem	.lnk_sta = XEON_LINK_STATUS_OFFSET,
377289542Scem	.db_size = sizeof(uint16_t),
378289542Scem	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 },
379289542Scem};
380289542Scem
381289607Scemstatic const struct ntb_alt_reg xeon_pri_reg = {
382289607Scem	.db_bell = XEON_PDOORBELL_OFFSET,
383289607Scem	.db_mask = XEON_PDBMSK_OFFSET,
384289607Scem	.spad = XEON_SPAD_OFFSET,
385289607Scem};
386289607Scem
387289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = {
388289542Scem	.db_bell = XEON_B2B_DOORBELL_OFFSET,
389289542Scem	.spad = XEON_B2B_SPAD_OFFSET,
390289542Scem};
391289542Scem
392289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = {
393289542Scem	.bar0_base = XEON_SBAR0BASE_OFFSET,
394289546Scem	.bar2_base = XEON_SBAR2BASE_OFFSET,
395289546Scem	.bar4_base = XEON_SBAR4BASE_OFFSET,
396289546Scem	.bar5_base = XEON_SBAR5BASE_OFFSET,
397289546Scem
398289542Scem	.bar2_limit = XEON_SBAR2LMT_OFFSET,
399289546Scem	.bar4_limit = XEON_SBAR4LMT_OFFSET,
400289546Scem	.bar5_limit = XEON_SBAR5LMT_OFFSET,
401289546Scem
402289542Scem	.bar2_xlat = XEON_SBAR2XLAT_OFFSET,
403289546Scem	.bar4_xlat = XEON_SBAR4XLAT_OFFSET,
404289546Scem	.bar5_xlat = XEON_SBAR5XLAT_OFFSET,
405289542Scem};
406289542Scem
407289614Scemstatic struct ntb_b2b_addr xeon_b2b_usd_addr = {
408289542Scem	.bar0_addr = XEON_B2B_BAR0_USD_ADDR,
409289542Scem	.bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64,
410289542Scem	.bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64,
411289542Scem	.bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32,
412289542Scem	.bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32,
413289542Scem};
414289542Scem
415289614Scemstatic struct ntb_b2b_addr xeon_b2b_dsd_addr = {
416289542Scem	.bar0_addr = XEON_B2B_BAR0_DSD_ADDR,
417289542Scem	.bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64,
418289542Scem	.bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64,
419289542Scem	.bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32,
420289542Scem	.bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32,
421289542Scem};
422289542Scem
423289614ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0,
424289614Scem    "B2B MW segment overrides -- MUST be the same on both sides");
425289614Scem
426289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN,
427289614Scem    &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
428289614Scem    "hardware, use this 64-bit address on the bus between the NTB devices for "
429289614Scem    "the window at BAR2, on the upstream side of the link.  MUST be the same "
430289614Scem    "address on both sides.");
431289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN,
432289614Scem    &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4.");
433289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN,
434289614Scem    &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 "
435289614Scem    "(split-BAR mode).");
436289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN,
437289646Scem    &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 "
438289614Scem    "(split-BAR mode).");
439289614Scem
440289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN,
441289614Scem    &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
442289614Scem    "hardware, use this 64-bit address on the bus between the NTB devices for "
443289614Scem    "the window at BAR2, on the downstream side of the link.  MUST be the same"
444289614Scem    " address on both sides.");
445289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN,
446289614Scem    &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4.");
447289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN,
448289614Scem    &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 "
449289614Scem    "(split-BAR mode).");
450289614ScemSYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN,
451289646Scem    &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 "
452289614Scem    "(split-BAR mode).");
453289614Scem
454250079Scarl/*
455250079Scarl * OS <-> Driver interface structures
456250079Scarl */
457250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");
458250079Scarl
459250079Scarlstatic device_method_t ntb_pci_methods[] = {
460250079Scarl	/* Device interface */
461250079Scarl	DEVMETHOD(device_probe,     ntb_probe),
462250079Scarl	DEVMETHOD(device_attach,    ntb_attach),
463250079Scarl	DEVMETHOD(device_detach,    ntb_detach),
464250079Scarl	DEVMETHOD_END
465250079Scarl};
466250079Scarl
467250079Scarlstatic driver_t ntb_pci_driver = {
468250079Scarl	"ntb_hw",
469250079Scarl	ntb_pci_methods,
470250079Scarl	sizeof(struct ntb_softc),
471250079Scarl};
472250079Scarl
473250079Scarlstatic devclass_t ntb_devclass;
474250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL);
475250079ScarlMODULE_VERSION(ntb_hw, 1);
476250079Scarl
477289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls");
478289207Scem
479250079Scarl/*
480250079Scarl * OS <-> Driver linkage functions
481250079Scarl */
482250079Scarlstatic int
483250079Scarlntb_probe(device_t device)
484250079Scarl{
485289209Scem	struct ntb_hw_info *p;
486250079Scarl
487289209Scem	p = ntb_get_device_info(pci_get_devid(device));
488289209Scem	if (p == NULL)
489250079Scarl		return (ENXIO);
490289209Scem
491289209Scem	device_set_desc(device, p->desc);
492289209Scem	return (0);
493250079Scarl}
494250079Scarl
495250079Scarlstatic int
496250079Scarlntb_attach(device_t device)
497250079Scarl{
498289209Scem	struct ntb_softc *ntb;
499289209Scem	struct ntb_hw_info *p;
500250079Scarl	int error;
501250079Scarl
502289209Scem	ntb = DEVICE2SOFTC(device);
503289209Scem	p = ntb_get_device_info(pci_get_devid(device));
504289209Scem
505250079Scarl	ntb->device = device;
506250079Scarl	ntb->type = p->type;
507255274Scarl	ntb->features = p->features;
508289543Scem	ntb->b2b_mw_idx = B2B_MW_DISABLED;
509250079Scarl
510289648Scem	/* Heartbeat timer for NTB_ATOM since there is no link interrupt */
511283291Sjkim	callout_init(&ntb->heartbeat_timer, 1);
512283291Sjkim	callout_init(&ntb->lr_timer, 1);
513289542Scem	mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);
514289546Scem	mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN);
515250079Scarl
516289648Scem	if (ntb->type == NTB_ATOM)
517289648Scem		error = ntb_detect_atom(ntb);
518289348Scem	else
519289348Scem		error = ntb_detect_xeon(ntb);
520289348Scem	if (error)
521289348Scem		goto out;
522289348Scem
523289397Scem	ntb_detect_max_mw(ntb);
524289396Scem
525289209Scem	error = ntb_map_pci_bars(ntb);
526289209Scem	if (error)
527289209Scem		goto out;
528289648Scem	if (ntb->type == NTB_ATOM)
529289648Scem		error = ntb_atom_init_dev(ntb);
530289272Scem	else
531289542Scem		error = ntb_xeon_init_dev(ntb);
532289209Scem	if (error)
533289209Scem		goto out;
534289540Scem	error = ntb_init_isr(ntb);
535289209Scem	if (error)
536289209Scem		goto out;
537289774Scem	ntb_sysctl_init(ntb);
538250079Scarl
539250079Scarl	pci_enable_busmaster(ntb->device);
540250079Scarl
541289209Scemout:
542289209Scem	if (error != 0)
543289209Scem		ntb_detach(device);
544250079Scarl	return (error);
545250079Scarl}
546250079Scarl
547250079Scarlstatic int
548250079Scarlntb_detach(device_t device)
549250079Scarl{
550289209Scem	struct ntb_softc *ntb;
551250079Scarl
552289209Scem	ntb = DEVICE2SOFTC(device);
553289542Scem
554289617Scem	if (ntb->self_reg != NULL)
555289617Scem		ntb_db_set_mask(ntb, ntb->db_valid_mask);
556250079Scarl	callout_drain(&ntb->heartbeat_timer);
557250079Scarl	callout_drain(&ntb->lr_timer);
558289272Scem	if (ntb->type == NTB_XEON)
559289272Scem		ntb_teardown_xeon(ntb);
560250079Scarl	ntb_teardown_interrupts(ntb);
561289397Scem
562289542Scem	mtx_destroy(&ntb->db_mask_lock);
563289546Scem	mtx_destroy(&ntb->ctx_lock);
564289542Scem
565289397Scem	/*
566289397Scem	 * Redetect total MWs so we unmap properly -- in case we lowered the
567289397Scem	 * maximum to work around Xeon errata.
568289397Scem	 */
569289397Scem	ntb_detect_max_mw(ntb);
570250079Scarl	ntb_unmap_pci_bar(ntb);
571250079Scarl
572250079Scarl	return (0);
573250079Scarl}
574250079Scarl
575289542Scem/*
576289542Scem * Driver internal routines
577289542Scem */
578289539Scemstatic inline enum ntb_bar
579289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
580289539Scem{
581289539Scem
582289543Scem	KASSERT(mw < ntb->mw_count ||
583289543Scem	    (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx),
584289542Scem	    ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count));
585289546Scem	KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw"));
586289539Scem
587289542Scem	return (ntb->reg->mw_bar[mw]);
588289539Scem}
589289539Scem
590289546Scemstatic inline bool
591289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar)
592289546Scem{
593289546Scem	/* XXX This assertion could be stronger. */
594289546Scem	KASSERT(bar < NTB_MAX_BARS, ("bogus bar"));
595289546Scem	return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR));
596289546Scem}
597289546Scem
598289546Scemstatic inline void
599289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base,
600289546Scem    uint32_t *xlat, uint32_t *lmt)
601289546Scem{
602289546Scem	uint32_t basev, lmtv, xlatv;
603289546Scem
604289546Scem	switch (bar) {
605289546Scem	case NTB_B2B_BAR_1:
606289546Scem		basev = ntb->xlat_reg->bar2_base;
607289546Scem		lmtv = ntb->xlat_reg->bar2_limit;
608289546Scem		xlatv = ntb->xlat_reg->bar2_xlat;
609289546Scem		break;
610289546Scem	case NTB_B2B_BAR_2:
611289546Scem		basev = ntb->xlat_reg->bar4_base;
612289546Scem		lmtv = ntb->xlat_reg->bar4_limit;
613289546Scem		xlatv = ntb->xlat_reg->bar4_xlat;
614289546Scem		break;
615289546Scem	case NTB_B2B_BAR_3:
616289546Scem		basev = ntb->xlat_reg->bar5_base;
617289546Scem		lmtv = ntb->xlat_reg->bar5_limit;
618289546Scem		xlatv = ntb->xlat_reg->bar5_xlat;
619289546Scem		break;
620289546Scem	default:
621289546Scem		KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS,
622289546Scem		    ("bad bar"));
623289546Scem		basev = lmtv = xlatv = 0;
624289546Scem		break;
625289546Scem	}
626289546Scem
627289546Scem	if (base != NULL)
628289546Scem		*base = basev;
629289546Scem	if (xlat != NULL)
630289546Scem		*xlat = xlatv;
631289546Scem	if (lmt != NULL)
632289546Scem		*lmt = lmtv;
633289546Scem}
634289546Scem
635250079Scarlstatic int
636255272Scarlntb_map_pci_bars(struct ntb_softc *ntb)
637250079Scarl{
638255272Scarl	int rc;
639250079Scarl
640250079Scarl	ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0);
641289541Scem	rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]);
642255272Scarl	if (rc != 0)
643289541Scem		goto out;
644255272Scarl
645289209Scem	ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2);
646289541Scem	rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]);
647255272Scarl	if (rc != 0)
648289541Scem		goto out;
649289543Scem	ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET;
650289543Scem	ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET;
651289543Scem	ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
652255272Scarl
653289209Scem	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
654289543Scem	/* XXX Are shared MW B2Bs write-combining? */
655289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR))
656289541Scem		rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
657255279Scarl	else
658289541Scem		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
659289543Scem	ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET;
660289543Scem	ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET;
661289543Scem	ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
662289543Scem
663289397Scem	if (!HAS_FEATURE(NTB_SPLIT_BAR))
664289541Scem		goto out;
665289397Scem
666289397Scem	ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5);
667289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
668289541Scem		rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
669289397Scem	else
670289541Scem		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
671289543Scem	ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET;
672289543Scem	ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET;
673289543Scem	ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET;
674250079Scarl
675289541Scemout:
676289209Scem	if (rc != 0)
677255272Scarl		device_printf(ntb->device,
678255272Scarl		    "unable to allocate pci resource\n");
679255272Scarl	return (rc);
680255272Scarl}
681255272Scarl
682289541Scemstatic void
683289647Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar,
684289647Scem    const char *kind)
685289541Scem{
686289541Scem
687289647Scem	device_printf(ntb->device,
688289647Scem	    "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n",
689289647Scem	    PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
690289647Scem	    (char *)bar->vbase + bar->size - 1,
691289647Scem	    (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1),
692289647Scem	    (uintmax_t)bar->size, kind);
693289541Scem}
694289541Scem
695255272Scarlstatic int
696255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
697255272Scarl{
698255272Scarl
699255275Scarl	bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
700289209Scem	    &bar->pci_resource_id, RF_ACTIVE);
701255272Scarl	if (bar->pci_resource == NULL)
702255272Scarl		return (ENXIO);
703289209Scem
704289209Scem	save_bar_parameters(bar);
705289647Scem	print_map_success(ntb, bar, "mmr");
706289209Scem	return (0);
707255272Scarl}
708255272Scarl
709255272Scarlstatic int
710255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
711255272Scarl{
712255272Scarl	int rc;
713255276Scarl	uint8_t bar_size_bits = 0;
714255272Scarl
715289209Scem	bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
716289209Scem	    &bar->pci_resource_id, RF_ACTIVE);
717250079Scarl
718255272Scarl	if (bar->pci_resource == NULL)
719255272Scarl		return (ENXIO);
720255276Scarl
721289209Scem	save_bar_parameters(bar);
722289209Scem	/*
723289209Scem	 * Ivytown NTB BAR sizes are misreported by the hardware due to a
724289209Scem	 * hardware issue. To work around this, query the size it should be
725289209Scem	 * configured to by the device and modify the resource to correspond to
726289209Scem	 * this new size. The BIOS on systems with this problem is required to
727289209Scem	 * provide enough address space to allow the driver to make this change
728289209Scem	 * safely.
729289209Scem	 *
730289209Scem	 * Ideally I could have just specified the size when I allocated the
731289209Scem	 * resource like:
732289209Scem	 *  bus_alloc_resource(ntb->device,
733289209Scem	 *	SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul,
734289209Scem	 *	1ul << bar_size_bits, RF_ACTIVE);
735289209Scem	 * but the PCI driver does not honor the size in this call, so we have
736289209Scem	 * to modify it after the fact.
737289209Scem	 */
738289209Scem	if (HAS_FEATURE(NTB_BAR_SIZE_4K)) {
739289209Scem		if (bar->pci_resource_id == PCIR_BAR(2))
740289209Scem			bar_size_bits = pci_read_config(ntb->device,
741289209Scem			    XEON_PBAR23SZ_OFFSET, 1);
742289209Scem		else
743289209Scem			bar_size_bits = pci_read_config(ntb->device,
744289209Scem			    XEON_PBAR45SZ_OFFSET, 1);
745289209Scem
746289209Scem		rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY,
747289209Scem		    bar->pci_resource, bar->pbase,
748289209Scem		    bar->pbase + (1ul << bar_size_bits) - 1);
749255272Scarl		if (rc != 0) {
750289209Scem			device_printf(ntb->device,
751289209Scem			    "unable to resize bar\n");
752255272Scarl			return (rc);
753250079Scarl		}
754289209Scem
755289209Scem		save_bar_parameters(bar);
756250079Scarl	}
757289209Scem
758289209Scem	/* Mark bar region as write combining to improve performance. */
759289209Scem	rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size,
760289209Scem	    VM_MEMATTR_WRITE_COMBINING);
761289647Scem	print_map_success(ntb, bar, "mw");
762289647Scem	if (rc == 0)
763289209Scem		device_printf(ntb->device,
764289647Scem		    "Marked BAR%d v:[%p-%p] p:[%p-%p] as "
765289647Scem		    "WRITE_COMBINING.\n",
766289647Scem		    PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
767289647Scem		    (char *)bar->vbase + bar->size - 1,
768289647Scem		    (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1));
769289647Scem	else
770289647Scem		device_printf(ntb->device,
771289647Scem		    "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as "
772289647Scem		    "WRITE_COMBINING: %d\n",
773289647Scem		    PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
774289647Scem		    (char *)bar->vbase + bar->size - 1,
775289647Scem		    (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1),
776289647Scem		    rc);
777289647Scem		/* Proceed anyway */
778250079Scarl	return (0);
779250079Scarl}
780250079Scarl
781250079Scarlstatic void
782250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb)
783250079Scarl{
784250079Scarl	struct ntb_pci_bar_info *current_bar;
785250079Scarl	int i;
786250079Scarl
787289397Scem	for (i = 0; i < NTB_MAX_BARS; i++) {
788250079Scarl		current_bar = &ntb->bar_info[i];
789250079Scarl		if (current_bar->pci_resource != NULL)
790250079Scarl			bus_release_resource(ntb->device, SYS_RES_MEMORY,
791250079Scarl			    current_bar->pci_resource_id,
792250079Scarl			    current_bar->pci_resource);
793250079Scarl	}
794250079Scarl}
795250079Scarl
796250079Scarlstatic int
797289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors)
798250079Scarl{
799289342Scem	uint32_t i;
800289342Scem	int rc;
801289342Scem
802289342Scem	for (i = 0; i < num_vectors; i++) {
803289342Scem		ntb->int_info[i].rid = i + 1;
804289342Scem		ntb->int_info[i].res = bus_alloc_resource_any(ntb->device,
805289342Scem		    SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE);
806289342Scem		if (ntb->int_info[i].res == NULL) {
807289342Scem			device_printf(ntb->device,
808289342Scem			    "bus_alloc_resource failed\n");
809289342Scem			return (ENOMEM);
810289342Scem		}
811289342Scem		ntb->int_info[i].tag = NULL;
812289342Scem		ntb->allocated_interrupts++;
813289342Scem		rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
814289546Scem		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr,
815289546Scem		    &ntb->msix_vec[i], &ntb->int_info[i].tag);
816289342Scem		if (rc != 0) {
817289342Scem			device_printf(ntb->device, "bus_setup_intr failed\n");
818289342Scem			return (ENXIO);
819289342Scem		}
820289342Scem	}
821289342Scem	return (0);
822289342Scem}
823289342Scem
824289344Scem/*
825289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector
826289344Scem * cannot be allocated for each MSI-X message.  JHB seems to think remapping
827289344Scem * should be okay.  This tunable should enable us to test that hypothesis
828289344Scem * when someone gets their hands on some Xeon hardware.
829289344Scem */
830289344Scemstatic int ntb_force_remap_mode;
831289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN,
832289344Scem    &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped"
833289344Scem    " to a smaller number of ithreads, even if the desired number are "
834289344Scem    "available");
835289344Scem
836289344Scem/*
837289344Scem * In case it is NOT ok, give consumers an abort button.
838289344Scem */
839289344Scemstatic int ntb_prefer_intx;
840289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN,
841289344Scem    &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather "
842289344Scem    "than remapping MSI-X messages over available slots (match Linux driver "
843289344Scem    "behavior)");
844289344Scem
845289344Scem/*
846289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple
847289344Scem * round-robin fashion.
848289344Scem */
849289342Scemstatic int
850289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
851289344Scem{
852289344Scem	u_int *vectors;
853289344Scem	uint32_t i;
854289344Scem	int rc;
855289344Scem
856289344Scem	if (ntb_prefer_intx != 0)
857289344Scem		return (ENXIO);
858289344Scem
859289344Scem	vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK);
860289344Scem
861289344Scem	for (i = 0; i < desired; i++)
862289344Scem		vectors[i] = (i % avail) + 1;
863289344Scem
864289344Scem	rc = pci_remap_msix(dev, desired, vectors);
865289344Scem	free(vectors, M_NTB);
866289344Scem	return (rc);
867289344Scem}
868289344Scem
869289344Scemstatic int
870289540Scemntb_init_isr(struct ntb_softc *ntb)
871289342Scem{
872289344Scem	uint32_t desired_vectors, num_vectors;
873289342Scem	int rc;
874250079Scarl
875250079Scarl	ntb->allocated_interrupts = 0;
876289542Scem	ntb->last_ts = ticks;
877289347Scem
878250079Scarl	/*
879289546Scem	 * Mask all doorbell interrupts.
880250079Scarl	 */
881289546Scem	ntb_db_set_mask(ntb, ntb->db_valid_mask);
882250079Scarl
883289344Scem	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
884289539Scem	    ntb->db_count);
885289344Scem	if (desired_vectors >= 1) {
886289344Scem		rc = pci_alloc_msix(ntb->device, &num_vectors);
887250079Scarl
888289344Scem		if (ntb_force_remap_mode != 0 && rc == 0 &&
889289344Scem		    num_vectors == desired_vectors)
890289344Scem			num_vectors--;
891289344Scem
892289344Scem		if (rc == 0 && num_vectors < desired_vectors) {
893289344Scem			rc = ntb_remap_msix(ntb->device, desired_vectors,
894289344Scem			    num_vectors);
895289344Scem			if (rc == 0)
896289344Scem				num_vectors = desired_vectors;
897289344Scem			else
898289344Scem				pci_release_msi(ntb->device);
899289344Scem		}
900289344Scem		if (rc != 0)
901289344Scem			num_vectors = 1;
902289344Scem	} else
903289344Scem		num_vectors = 1;
904289344Scem
905289539Scem	if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
906289539Scem		ntb->db_vec_count = 1;
907289539Scem		ntb->db_vec_shift = ntb->db_count;
908289539Scem		rc = ntb_setup_legacy_interrupt(ntb);
909289539Scem	} else {
910289546Scem		ntb_create_msix_vec(ntb, num_vectors);
911289540Scem		rc = ntb_setup_msix(ntb, num_vectors);
912289539Scem	}
913289539Scem	if (rc != 0) {
914289539Scem		device_printf(ntb->device,
915289539Scem		    "Error allocating interrupts: %d\n", rc);
916289546Scem		ntb_free_msix_vec(ntb);
917289396Scem	}
918289396Scem
919289342Scem	return (rc);
920289342Scem}
921289342Scem
922289342Scemstatic int
923289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb)
924289342Scem{
925289342Scem	int rc;
926289342Scem
927289342Scem	ntb->int_info[0].rid = 0;
928289342Scem	ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ,
929289342Scem	    &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
930289342Scem	if (ntb->int_info[0].res == NULL) {
931289342Scem		device_printf(ntb->device, "bus_alloc_resource failed\n");
932289342Scem		return (ENOMEM);
933250079Scarl	}
934250079Scarl
935289342Scem	ntb->int_info[0].tag = NULL;
936289342Scem	ntb->allocated_interrupts = 1;
937289342Scem
938289342Scem	rc = bus_setup_intr(ntb->device, ntb->int_info[0].res,
939289546Scem	    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr,
940289342Scem	    ntb, &ntb->int_info[0].tag);
941289342Scem	if (rc != 0) {
942289342Scem		device_printf(ntb->device, "bus_setup_intr failed\n");
943289342Scem		return (ENXIO);
944289342Scem	}
945289342Scem
946250079Scarl	return (0);
947250079Scarl}
948250079Scarl
949250079Scarlstatic void
950250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb)
951250079Scarl{
952250079Scarl	struct ntb_int_info *current_int;
953250079Scarl	int i;
954250079Scarl
955289209Scem	for (i = 0; i < ntb->allocated_interrupts; i++) {
956250079Scarl		current_int = &ntb->int_info[i];
957250079Scarl		if (current_int->tag != NULL)
958250079Scarl			bus_teardown_intr(ntb->device, current_int->res,
959250079Scarl			    current_int->tag);
960250079Scarl
961250079Scarl		if (current_int->res != NULL)
962250079Scarl			bus_release_resource(ntb->device, SYS_RES_IRQ,
963250079Scarl			    rman_get_rid(current_int->res), current_int->res);
964250079Scarl	}
965250079Scarl
966289546Scem	ntb_free_msix_vec(ntb);
967250079Scarl	pci_release_msi(ntb->device);
968250079Scarl}
969250079Scarl
970289347Scem/*
971289648Scem * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon.  Abstract it
972289347Scem * out to make code clearer.
973289347Scem */
974289539Scemstatic inline uint64_t
975289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff)
976289347Scem{
977289347Scem
978289648Scem	if (ntb->type == NTB_ATOM)
979289347Scem		return (ntb_reg_read(8, regoff));
980289347Scem
981289347Scem	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
982289347Scem
983289347Scem	return (ntb_reg_read(2, regoff));
984289347Scem}
985289347Scem
986289539Scemstatic inline void
987289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
988289347Scem{
989289347Scem
990289542Scem	KASSERT((val & ~ntb->db_valid_mask) == 0,
991289542Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
992289542Scem	     (uintmax_t)(val & ~ntb->db_valid_mask),
993289542Scem	     (uintmax_t)ntb->db_valid_mask));
994289542Scem
995289607Scem	if (regoff == ntb->self_reg->db_mask)
996289546Scem		DB_MASK_ASSERT(ntb, MA_OWNED);
997289542Scem
998289648Scem	if (ntb->type == NTB_ATOM) {
999289347Scem		ntb_reg_write(8, regoff, val);
1000289347Scem		return;
1001289347Scem	}
1002289347Scem
1003289347Scem	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
1004289347Scem	ntb_reg_write(2, regoff, (uint16_t)val);
1005289347Scem}
1006289347Scem
1007289546Scemvoid
1008289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits)
1009289542Scem{
1010289542Scem
1011289546Scem	DB_MASK_LOCK(ntb);
1012289542Scem	ntb->db_mask |= bits;
1013289607Scem	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
1014289546Scem	DB_MASK_UNLOCK(ntb);
1015289542Scem}
1016289542Scem
1017289546Scemvoid
1018289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits)
1019289542Scem{
1020289542Scem
1021289542Scem	KASSERT((bits & ~ntb->db_valid_mask) == 0,
1022289542Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
1023289542Scem	     (uintmax_t)(bits & ~ntb->db_valid_mask),
1024289542Scem	     (uintmax_t)ntb->db_valid_mask));
1025289542Scem
1026289546Scem	DB_MASK_LOCK(ntb);
1027289542Scem	ntb->db_mask &= ~bits;
1028289607Scem	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
1029289546Scem	DB_MASK_UNLOCK(ntb);
1030289542Scem}
1031289542Scem
1032289546Scemuint64_t
1033289546Scemntb_db_read(struct ntb_softc *ntb)
1034289281Scem{
1035289281Scem
1036289607Scem	return (db_ioread(ntb, ntb->self_reg->db_bell));
1037289281Scem}
1038289281Scem
1039289546Scemvoid
1040289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits)
1041289281Scem{
1042289281Scem
1043289546Scem	KASSERT((bits & ~ntb->db_valid_mask) == 0,
1044289546Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
1045289546Scem	     (uintmax_t)(bits & ~ntb->db_valid_mask),
1046289546Scem	     (uintmax_t)ntb->db_valid_mask));
1047289546Scem
1048289607Scem	db_iowrite(ntb, ntb->self_reg->db_bell, bits);
1049289281Scem}
1050289281Scem
1051289540Scemstatic inline uint64_t
1052289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)
1053250079Scarl{
1054289540Scem	uint64_t shift, mask;
1055250079Scarl
1056289540Scem	shift = ntb->db_vec_shift;
1057289540Scem	mask = (1ull << shift) - 1;
1058289540Scem	return (mask << (shift * db_vector));
1059250079Scarl}
1060250079Scarl
1061250079Scarlstatic void
1062289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
1063250079Scarl{
1064289540Scem	uint64_t vec_mask;
1065250079Scarl
1066289542Scem	ntb->last_ts = ticks;
1067289546Scem	vec_mask = ntb_vec_mask(ntb, vec);
1068250079Scarl
1069289542Scem	if ((vec_mask & ntb->db_link_mask) != 0) {
1070289546Scem		if (ntb_poll_link(ntb))
1071289546Scem			ntb_link_event(ntb);
1072289540Scem	}
1073289540Scem
1074289546Scem	if ((vec_mask & ntb->db_valid_mask) != 0)
1075289546Scem		ntb_db_event(ntb, vec);
1076289546Scem}
1077250079Scarl
1078289546Scemstatic void
1079289546Scemndev_vec_isr(void *arg)
1080289546Scem{
1081289546Scem	struct ntb_vec *nvec = arg;
1082250079Scarl
1083289546Scem	ntb_interrupt(nvec->ntb, nvec->num);
1084250079Scarl}
1085250079Scarl
1086250079Scarlstatic void
1087289546Scemndev_irq_isr(void *arg)
1088250079Scarl{
1089289546Scem	/* If we couldn't set up MSI-X, we only have the one vector. */
1090289546Scem	ntb_interrupt(arg, 0);
1091250079Scarl}
1092250079Scarl
1093250079Scarlstatic int
1094289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors)
1095250079Scarl{
1096289342Scem	uint32_t i;
1097250079Scarl
1098289546Scem	ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB,
1099250079Scarl	    M_ZERO | M_WAITOK);
1100250079Scarl	for (i = 0; i < num_vectors; i++) {
1101289546Scem		ntb->msix_vec[i].num = i;
1102289546Scem		ntb->msix_vec[i].ntb = ntb;
1103250079Scarl	}
1104250079Scarl
1105250079Scarl	return (0);
1106250079Scarl}
1107250079Scarl
1108250079Scarlstatic void
1109289546Scemntb_free_msix_vec(struct ntb_softc *ntb)
1110250079Scarl{
1111250079Scarl
1112289546Scem	if (ntb->msix_vec == NULL)
1113289539Scem		return;
1114289539Scem
1115289546Scem	free(ntb->msix_vec, M_NTB);
1116289546Scem	ntb->msix_vec = NULL;
1117250079Scarl}
1118250079Scarl
1119250079Scarlstatic struct ntb_hw_info *
1120250079Scarlntb_get_device_info(uint32_t device_id)
1121250079Scarl{
1122250079Scarl	struct ntb_hw_info *ep = pci_ids;
1123250079Scarl
1124250079Scarl	while (ep->device_id) {
1125250079Scarl		if (ep->device_id == device_id)
1126250079Scarl			return (ep);
1127250079Scarl		++ep;
1128250079Scarl	}
1129250079Scarl	return (NULL);
1130250079Scarl}
1131250079Scarl
1132289272Scemstatic void
1133289272Scemntb_teardown_xeon(struct ntb_softc *ntb)
1134250079Scarl{
1135250079Scarl
1136289617Scem	if (ntb->reg != NULL)
1137289617Scem		ntb_link_disable(ntb);
1138250079Scarl}
1139250079Scarl
1140289397Scemstatic void
1141289397Scemntb_detect_max_mw(struct ntb_softc *ntb)
1142289397Scem{
1143289397Scem
1144289648Scem	if (ntb->type == NTB_ATOM) {
1145289648Scem		ntb->mw_count = ATOM_MW_COUNT;
1146289397Scem		return;
1147289397Scem	}
1148289397Scem
1149289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1150289539Scem		ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
1151289397Scem	else
1152289539Scem		ntb->mw_count = XEON_SNB_MW_COUNT;
1153289397Scem}
1154289397Scem
1155250079Scarlstatic int
1156289348Scemntb_detect_xeon(struct ntb_softc *ntb)
1157250079Scarl{
1158289348Scem	uint8_t ppd, conn_type;
1159250079Scarl
1160289348Scem	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
1161289348Scem	ntb->ppd = ppd;
1162250079Scarl
1163289348Scem	if ((ppd & XEON_PPD_DEV_TYPE) != 0)
1164289257Scem		ntb->dev_type = NTB_DEV_USD;
1165289257Scem	else
1166289257Scem		ntb->dev_type = NTB_DEV_DSD;
1167289257Scem
1168289397Scem	if ((ppd & XEON_PPD_SPLIT_BAR) != 0)
1169289397Scem		ntb->features |= NTB_SPLIT_BAR;
1170289397Scem
1171289542Scem	/* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */
1172289542Scem	if (HAS_FEATURE(NTB_SB01BASE_LOCKUP))
1173289542Scem		ntb->features |= NTB_SDOORBELL_LOCKUP;
1174289542Scem
1175289348Scem	conn_type = ppd & XEON_PPD_CONN_TYPE;
1176289348Scem	switch (conn_type) {
1177289348Scem	case NTB_CONN_B2B:
1178289348Scem		ntb->conn_type = conn_type;
1179289348Scem		break;
1180289348Scem	case NTB_CONN_RP:
1181289348Scem	case NTB_CONN_TRANSPARENT:
1182289348Scem	default:
1183289348Scem		device_printf(ntb->device, "Unsupported connection type: %u\n",
1184289348Scem		    (unsigned)conn_type);
1185289348Scem		return (ENXIO);
1186289348Scem	}
1187289348Scem	return (0);
1188289348Scem}
1189289348Scem
1190289348Scemstatic int
1191289648Scemntb_detect_atom(struct ntb_softc *ntb)
1192289348Scem{
1193289348Scem	uint32_t ppd, conn_type;
1194289348Scem
1195289348Scem	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
1196289348Scem	ntb->ppd = ppd;
1197289348Scem
1198289648Scem	if ((ppd & ATOM_PPD_DEV_TYPE) != 0)
1199289348Scem		ntb->dev_type = NTB_DEV_DSD;
1200289348Scem	else
1201289348Scem		ntb->dev_type = NTB_DEV_USD;
1202289348Scem
1203289648Scem	conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8;
1204289348Scem	switch (conn_type) {
1205289348Scem	case NTB_CONN_B2B:
1206289348Scem		ntb->conn_type = conn_type;
1207289348Scem		break;
1208289348Scem	default:
1209289348Scem		device_printf(ntb->device, "Unsupported NTB configuration\n");
1210289348Scem		return (ENXIO);
1211289348Scem	}
1212289348Scem	return (0);
1213289348Scem}
1214289348Scem
1215289348Scemstatic int
1216289542Scemntb_xeon_init_dev(struct ntb_softc *ntb)
1217289348Scem{
1218289542Scem	int rc;
1219289348Scem
1220289542Scem	ntb->spad_count		= XEON_SPAD_COUNT;
1221289542Scem	ntb->db_count		= XEON_DB_COUNT;
1222289542Scem	ntb->db_link_mask	= XEON_DB_LINK_BIT;
1223289542Scem	ntb->db_vec_count	= XEON_DB_MSIX_VECTOR_COUNT;
1224289542Scem	ntb->db_vec_shift	= XEON_DB_MSIX_VECTOR_SHIFT;
1225289257Scem
1226289542Scem	if (ntb->conn_type != NTB_CONN_B2B) {
1227250079Scarl		device_printf(ntb->device, "Connection type %d not supported\n",
1228289348Scem		    ntb->conn_type);
1229250079Scarl		return (ENXIO);
1230250079Scarl	}
1231250079Scarl
1232289542Scem	ntb->reg = &xeon_reg;
1233289607Scem	ntb->self_reg = &xeon_pri_reg;
1234289542Scem	ntb->peer_reg = &xeon_b2b_reg;
1235289542Scem	ntb->xlat_reg = &xeon_sec_xlat;
1236289542Scem
1237289208Scem	/*
1238289208Scem	 * There is a Xeon hardware errata related to writes to SDOORBELL or
1239289208Scem	 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space,
1240289208Scem	 * which may hang the system.  To workaround this use the second memory
1241289208Scem	 * window to access the interrupt and scratch pad registers on the
1242289208Scem	 * remote system.
1243289208Scem	 */
1244289543Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
1245289543Scem		/* Use the last MW for mapping remote spad */
1246289542Scem		ntb->b2b_mw_idx = ntb->mw_count - 1;
1247289543Scem	else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14))
1248289208Scem		/*
1249289542Scem		 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
1250289542Scem		 * mirrored to the remote system.  Shrink the number of bits by one,
1251289542Scem		 * since bit 14 is the last bit.
1252289542Scem		 *
1253289542Scem		 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
1254289542Scem		 * anyway.  Nor for non-B2B connection types.
1255289542Scem		 */
1256289543Scem		ntb->db_count = XEON_DB_COUNT - 1;
1257250079Scarl
1258289542Scem	ntb->db_valid_mask = (1ull << ntb->db_count) - 1;
1259250079Scarl
1260289542Scem	if (ntb->dev_type == NTB_DEV_USD)
1261289542Scem		rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr,
1262289542Scem		    &xeon_b2b_usd_addr);
1263289542Scem	else
1264289542Scem		rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr,
1265289542Scem		    &xeon_b2b_dsd_addr);
1266289542Scem	if (rc != 0)
1267289542Scem		return (rc);
1268289271Scem
1269250079Scarl	/* Enable Bus Master and Memory Space on the secondary side */
1270289607Scem	ntb_reg_write(2, XEON_PCICMD_OFFSET,
1271289542Scem	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
1272255279Scarl
1273255269Scarl	/* Enable link training */
1274289546Scem	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
1275250079Scarl
1276250079Scarl	return (0);
1277250079Scarl}
1278250079Scarl
1279250079Scarlstatic int
1280289648Scemntb_atom_init_dev(struct ntb_softc *ntb)
1281250079Scarl{
1282250079Scarl
1283289348Scem	KASSERT(ntb->conn_type == NTB_CONN_B2B,
1284289348Scem	    ("Unsupported NTB configuration (%d)\n", ntb->conn_type));
1285250079Scarl
1286289648Scem	ntb->spad_count		 = ATOM_SPAD_COUNT;
1287289648Scem	ntb->db_count		 = ATOM_DB_COUNT;
1288289648Scem	ntb->db_vec_count	 = ATOM_DB_MSIX_VECTOR_COUNT;
1289289648Scem	ntb->db_vec_shift	 = ATOM_DB_MSIX_VECTOR_SHIFT;
1290289542Scem	ntb->db_valid_mask	 = (1ull << ntb->db_count) - 1;
1291250079Scarl
1292289648Scem	ntb->reg = &atom_reg;
1293289648Scem	ntb->self_reg = &atom_pri_reg;
1294289648Scem	ntb->peer_reg = &atom_b2b_reg;
1295289648Scem	ntb->xlat_reg = &atom_sec_xlat;
1296289542Scem
1297250079Scarl	/*
1298289648Scem	 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is
1299250079Scarl	 * resolved.  Mask transaction layer internal parity errors.
1300250079Scarl	 */
1301250079Scarl	pci_write_config(ntb->device, 0xFC, 0x4, 4);
1302250079Scarl
1303289648Scem	configure_atom_secondary_side_bars(ntb);
1304250079Scarl
1305250079Scarl	/* Enable Bus Master and Memory Space on the secondary side */
1306289648Scem	ntb_reg_write(2, ATOM_PCICMD_OFFSET,
1307250079Scarl	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
1308289209Scem
1309289542Scem	/* Initiate PCI-E link training */
1310289546Scem	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
1311250079Scarl
1312289648Scem	callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb);
1313289542Scem
1314250079Scarl	return (0);
1315250079Scarl}
1316250079Scarl
1317289648Scem/* XXX: Linux driver doesn't seem to do any of this for Atom. */
1318255279Scarlstatic void
1319289648Scemconfigure_atom_secondary_side_bars(struct ntb_softc *ntb)
1320255279Scarl{
1321255279Scarl
1322255279Scarl	if (ntb->dev_type == NTB_DEV_USD) {
1323289648Scem		ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET,
1324289542Scem		    XEON_B2B_BAR2_DSD_ADDR64);
1325289648Scem		ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET,
1326289542Scem		    XEON_B2B_BAR4_DSD_ADDR64);
1327289648Scem		ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64);
1328289648Scem		ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64);
1329255279Scarl	} else {
1330289648Scem		ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET,
1331289542Scem		    XEON_B2B_BAR2_USD_ADDR64);
1332289648Scem		ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET,
1333289542Scem		    XEON_B2B_BAR4_USD_ADDR64);
1334289648Scem		ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64);
1335289648Scem		ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64);
1336255279Scarl	}
1337255279Scarl}
1338255279Scarl
1339289543Scem
1340289543Scem/*
1341289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a
1342289543Scem * MW, limit the B2B MW to half a MW.  By sharing a MW, half the shared MW
1343289543Scem * remains for use by a higher layer.
1344289543Scem *
1345289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured
1346289543Scem * MW size is sufficiently large.
1347289543Scem */
1348289543Scemstatic unsigned int ntb_b2b_mw_share;
1349289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share,
1350289543Scem    0, "If enabled (non-zero), prefer to share half of the B2B peer register "
1351289543Scem    "MW with higher level consumers.  Both sides of the NTB MUST set the same "
1352289543Scem    "value here.");
1353289543Scem
1354289543Scemstatic void
1355289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx,
1356289543Scem    enum ntb_bar regbar)
1357289543Scem{
1358289543Scem	struct ntb_pci_bar_info *bar;
1359289543Scem	uint8_t bar_sz;
1360289543Scem
1361289543Scem	if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3)
1362289543Scem		return;
1363289543Scem
1364289543Scem	bar = &ntb->bar_info[idx];
1365289543Scem	bar_sz = pci_read_config(ntb->device, bar->psz_off, 1);
1366289543Scem	if (idx == regbar) {
1367289543Scem		if (ntb->b2b_off != 0)
1368289543Scem			bar_sz--;
1369289543Scem		else
1370289543Scem			bar_sz = 0;
1371289543Scem	}
1372289543Scem	pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1);
1373289543Scem	bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1);
1374289543Scem	(void)bar_sz;
1375289543Scem}
1376289543Scem
1377289543Scemstatic void
1378289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr,
1379289543Scem    enum ntb_bar idx, enum ntb_bar regbar)
1380289543Scem{
1381289546Scem	uint64_t reg_val;
1382289546Scem	uint32_t base_reg, lmt_reg;
1383289543Scem
1384289546Scem	bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg);
1385289546Scem	if (idx == regbar)
1386289546Scem		bar_addr += ntb->b2b_off;
1387289543Scem
1388289546Scem	if (!bar_is_64bit(ntb, idx)) {
1389289546Scem		ntb_reg_write(4, base_reg, bar_addr);
1390289546Scem		reg_val = ntb_reg_read(4, base_reg);
1391289546Scem		(void)reg_val;
1392289546Scem
1393289546Scem		ntb_reg_write(4, lmt_reg, bar_addr);
1394289546Scem		reg_val = ntb_reg_read(4, lmt_reg);
1395289546Scem		(void)reg_val;
1396289543Scem	} else {
1397289546Scem		ntb_reg_write(8, base_reg, bar_addr);
1398289546Scem		reg_val = ntb_reg_read(8, base_reg);
1399289546Scem		(void)reg_val;
1400289546Scem
1401289546Scem		ntb_reg_write(8, lmt_reg, bar_addr);
1402289546Scem		reg_val = ntb_reg_read(8, lmt_reg);
1403289546Scem		(void)reg_val;
1404289543Scem	}
1405289543Scem}
1406289543Scem
1407289543Scemstatic void
1408289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx)
1409289543Scem{
1410289543Scem	struct ntb_pci_bar_info *bar;
1411289543Scem
1412289543Scem	bar = &ntb->bar_info[idx];
1413289543Scem	if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) {
1414289543Scem		ntb_reg_write(4, bar->pbarxlat_off, base_addr);
1415289543Scem		base_addr = ntb_reg_read(4, bar->pbarxlat_off);
1416289543Scem	} else {
1417289543Scem		ntb_reg_write(8, bar->pbarxlat_off, base_addr);
1418289543Scem		base_addr = ntb_reg_read(8, bar->pbarxlat_off);
1419289543Scem	}
1420289543Scem	(void)base_addr;
1421289543Scem}
1422289543Scem
1423289542Scemstatic int
1424289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
1425289542Scem    const struct ntb_b2b_addr *peer_addr)
1426255279Scarl{
1427289543Scem	struct ntb_pci_bar_info *b2b_bar;
1428289543Scem	vm_size_t bar_size;
1429289543Scem	uint64_t bar_addr;
1430289543Scem	enum ntb_bar b2b_bar_num, i;
1431255279Scarl
1432289543Scem	if (ntb->b2b_mw_idx == B2B_MW_DISABLED) {
1433289543Scem		b2b_bar = NULL;
1434289543Scem		b2b_bar_num = NTB_CONFIG_BAR;
1435289543Scem		ntb->b2b_off = 0;
1436289543Scem	} else {
1437289543Scem		b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx);
1438289543Scem		KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS,
1439289543Scem		    ("invalid b2b mw bar"));
1440289543Scem
1441289543Scem		b2b_bar = &ntb->bar_info[b2b_bar_num];
1442289543Scem		bar_size = b2b_bar->size;
1443289543Scem
1444289543Scem		if (ntb_b2b_mw_share != 0 &&
1445289543Scem		    (bar_size >> 1) >= XEON_B2B_MIN_SIZE)
1446289543Scem			ntb->b2b_off = bar_size >> 1;
1447289543Scem		else if (bar_size >= XEON_B2B_MIN_SIZE) {
1448289543Scem			ntb->b2b_off = 0;
1449289543Scem			ntb->mw_count--;
1450289543Scem		} else {
1451289543Scem			device_printf(ntb->device,
1452289543Scem			    "B2B bar size is too small!\n");
1453289543Scem			return (EIO);
1454289543Scem		}
1455255279Scarl	}
1456289542Scem
1457289543Scem	/*
1458289543Scem	 * Reset the secondary bar sizes to match the primary bar sizes.
1459289543Scem	 * (Except, disable or halve the size of the B2B secondary bar.)
1460289543Scem	 */
1461289543Scem	for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++)
1462289543Scem		xeon_reset_sbar_size(ntb, i, b2b_bar_num);
1463289543Scem
1464289543Scem	bar_addr = 0;
1465289543Scem	if (b2b_bar_num == NTB_CONFIG_BAR)
1466289543Scem		bar_addr = addr->bar0_addr;
1467289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_1)
1468289543Scem		bar_addr = addr->bar2_addr64;
1469289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR))
1470289543Scem		bar_addr = addr->bar4_addr64;
1471289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2)
1472289543Scem		bar_addr = addr->bar4_addr32;
1473289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_3)
1474289543Scem		bar_addr = addr->bar5_addr32;
1475289543Scem	else
1476289543Scem		KASSERT(false, ("invalid bar"));
1477289543Scem
1478289543Scem	ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr);
1479289543Scem
1480289543Scem	/*
1481289543Scem	 * Other SBARs are normally hit by the PBAR xlat, except for the b2b
1482289543Scem	 * register BAR.  The B2B BAR is either disabled above or configured
1483289543Scem	 * half-size.  It starts at PBAR xlat + offset.
1484289543Scem	 *
1485289543Scem	 * Also set up incoming BAR limits == base (zero length window).
1486289543Scem	 */
1487289543Scem	xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1,
1488289543Scem	    b2b_bar_num);
1489289542Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
1490289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32,
1491289543Scem		    NTB_B2B_BAR_2, b2b_bar_num);
1492289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32,
1493289543Scem		    NTB_B2B_BAR_3, b2b_bar_num);
1494289542Scem	} else
1495289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64,
1496289543Scem		    NTB_B2B_BAR_2, b2b_bar_num);
1497289543Scem
1498289543Scem	/* Zero incoming translation addrs */
1499289543Scem	ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0);
1500289543Scem	ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0);
1501289543Scem
1502289543Scem	/* Zero outgoing translation limits (whole bar size windows) */
1503289543Scem	ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0);
1504289543Scem	ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
1505289543Scem
1506289543Scem	/* Set outgoing translation offsets */
1507289543Scem	xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1);
1508289543Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
1509289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2);
1510289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3);
1511289543Scem	} else
1512289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2);
1513289543Scem
1514289543Scem	/* Set the translation offset for B2B registers */
1515289543Scem	bar_addr = 0;
1516289543Scem	if (b2b_bar_num == NTB_CONFIG_BAR)
1517289543Scem		bar_addr = peer_addr->bar0_addr;
1518289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_1)
1519289543Scem		bar_addr = peer_addr->bar2_addr64;
1520289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR))
1521289543Scem		bar_addr = peer_addr->bar4_addr64;
1522289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2)
1523289543Scem		bar_addr = peer_addr->bar4_addr32;
1524289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_3)
1525289543Scem		bar_addr = peer_addr->bar5_addr32;
1526289543Scem	else
1527289543Scem		KASSERT(false, ("invalid bar"));
1528289543Scem
1529289543Scem	/*
1530289543Scem	 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits
1531289543Scem	 * at a time.
1532289543Scem	 */
1533289543Scem	ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff);
1534289543Scem	ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32);
1535289542Scem	return (0);
1536255279Scarl}
1537255279Scarl
1538289546Scemstatic inline bool
1539289546Scemlink_is_up(struct ntb_softc *ntb)
1540289546Scem{
1541289546Scem
1542289611Scem	if (ntb->type == NTB_XEON) {
1543289611Scem		if (ntb->conn_type == NTB_CONN_TRANSPARENT)
1544289611Scem			return (true);
1545289546Scem		return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0);
1546289611Scem	}
1547289546Scem
1548289648Scem	KASSERT(ntb->type == NTB_ATOM, ("ntb type"));
1549289648Scem	return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0);
1550289546Scem}
1551289546Scem
1552289546Scemstatic inline bool
1553289648Scematom_link_is_err(struct ntb_softc *ntb)
1554289546Scem{
1555289546Scem	uint32_t status;
1556289546Scem
1557289648Scem	KASSERT(ntb->type == NTB_ATOM, ("ntb type"));
1558289546Scem
1559289648Scem	status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET);
1560289648Scem	if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0)
1561289546Scem		return (true);
1562289546Scem
1563289648Scem	status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET);
1564289648Scem	return ((status & ATOM_IBIST_ERR_OFLOW) != 0);
1565289546Scem}
1566289546Scem
1567289648Scem/* Atom does not have link status interrupt, poll on that platform */
1568250079Scarlstatic void
1569289648Scematom_link_hb(void *arg)
1570250079Scarl{
1571250079Scarl	struct ntb_softc *ntb = arg;
1572289546Scem	sbintime_t timo, poll_ts;
1573250079Scarl
1574289546Scem	timo = NTB_HB_TIMEOUT * hz;
1575289546Scem	poll_ts = ntb->last_ts + timo;
1576289546Scem
1577289542Scem	/*
1578289542Scem	 * Delay polling the link status if an interrupt was received, unless
1579289542Scem	 * the cached link status says the link is down.
1580289542Scem	 */
1581289546Scem	if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) {
1582289546Scem		timo = poll_ts - ticks;
1583289542Scem		goto out;
1584289546Scem	}
1585289542Scem
1586289546Scem	if (ntb_poll_link(ntb))
1587289546Scem		ntb_link_event(ntb);
1588289542Scem
1589289648Scem	if (!link_is_up(ntb) && atom_link_is_err(ntb)) {
1590289546Scem		/* Link is down with error, proceed with recovery */
1591289648Scem		callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb);
1592289546Scem		return;
1593250079Scarl	}
1594250079Scarl
1595289542Scemout:
1596289648Scem	callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb);
1597250079Scarl}
1598250079Scarl
1599250079Scarlstatic void
1600289648Scematom_perform_link_restart(struct ntb_softc *ntb)
1601250079Scarl{
1602250079Scarl	uint32_t status;
1603250079Scarl
1604250079Scarl	/* Driver resets the NTB ModPhy lanes - magic! */
1605289648Scem	ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0);
1606289648Scem	ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40);
1607289648Scem	ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60);
1608289648Scem	ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60);
1609250079Scarl
1610250079Scarl	/* Driver waits 100ms to allow the NTB ModPhy to settle */
1611250079Scarl	pause("ModPhy", hz / 10);
1612250079Scarl
1613250079Scarl	/* Clear AER Errors, write to clear */
1614289648Scem	status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET);
1615250079Scarl	status &= PCIM_AER_COR_REPLAY_ROLLOVER;
1616289648Scem	ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status);
1617250079Scarl
1618250079Scarl	/* Clear unexpected electrical idle event in LTSSM, write to clear */
1619289648Scem	status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET);
1620289648Scem	status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1621289648Scem	ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status);
1622250079Scarl
1623250079Scarl	/* Clear DeSkew Buffer error, write to clear */
1624289648Scem	status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET);
1625289648Scem	status |= ATOM_DESKEWSTS_DBERR;
1626289648Scem	ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status);
1627250079Scarl
1628289648Scem	status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET);
1629289648Scem	status &= ATOM_IBIST_ERR_OFLOW;
1630289648Scem	ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status);
1631250079Scarl
1632250079Scarl	/* Releases the NTB state machine to allow the link to retrain */
1633289648Scem	status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET);
1634289648Scem	status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1635289648Scem	ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status);
1636250079Scarl}
1637250079Scarl
1638289546Scem/*
1639289546Scem * ntb_set_ctx() - associate a driver context with an ntb device
1640289546Scem * @ntb:        NTB device context
1641289546Scem * @ctx:        Driver context
1642289546Scem * @ctx_ops:    Driver context operations
1643289546Scem *
1644289546Scem * Associate a driver context and operations with a ntb device.  The context is
1645289546Scem * provided by the client driver, and the driver may associate a different
1646289546Scem * context with each ntb device.
1647289546Scem *
1648289546Scem * Return: Zero if the context is associated, otherwise an error number.
1649289546Scem */
1650289546Scemint
1651289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops)
1652250079Scarl{
1653250079Scarl
1654289546Scem	if (ctx == NULL || ops == NULL)
1655289546Scem		return (EINVAL);
1656289546Scem	if (ntb->ctx_ops != NULL)
1657289546Scem		return (EINVAL);
1658250079Scarl
1659289546Scem	CTX_LOCK(ntb);
1660289546Scem	if (ntb->ctx_ops != NULL) {
1661289546Scem		CTX_UNLOCK(ntb);
1662289546Scem		return (EINVAL);
1663250079Scarl	}
1664289546Scem	ntb->ntb_ctx = ctx;
1665289546Scem	ntb->ctx_ops = ops;
1666289546Scem	CTX_UNLOCK(ntb);
1667250079Scarl
1668289546Scem	return (0);
1669250079Scarl}
1670250079Scarl
1671289546Scem/*
1672289546Scem * It is expected that this will only be used from contexts where the ctx_lock
1673289546Scem * is not needed to protect ntb_ctx lifetime.
1674289546Scem */
1675289546Scemvoid *
1676289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops)
1677289546Scem{
1678289546Scem
1679289546Scem	KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus"));
1680289546Scem	if (ops != NULL)
1681289546Scem		*ops = ntb->ctx_ops;
1682289546Scem	return (ntb->ntb_ctx);
1683289546Scem}
1684289546Scem
1685289546Scem/*
1686289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device
1687289546Scem * @ntb:        NTB device context
1688289546Scem *
1689289546Scem * Clear any association that may exist between a driver context and the ntb
1690289546Scem * device.
1691289546Scem */
1692289546Scemvoid
1693289546Scemntb_clear_ctx(struct ntb_softc *ntb)
1694289546Scem{
1695289546Scem
1696289546Scem	CTX_LOCK(ntb);
1697289546Scem	ntb->ntb_ctx = NULL;
1698289546Scem	ntb->ctx_ops = NULL;
1699289546Scem	CTX_UNLOCK(ntb);
1700289546Scem}
1701289546Scem
1702289546Scem/*
1703289546Scem * ntb_link_event() - notify driver context of a change in link status
1704289546Scem * @ntb:        NTB device context
1705289546Scem *
1706289546Scem * Notify the driver context that the link status may have changed.  The driver
1707289546Scem * should call ntb_link_is_up() to get the current status.
1708289546Scem */
1709289546Scemvoid
1710289546Scemntb_link_event(struct ntb_softc *ntb)
1711289546Scem{
1712289546Scem
1713289546Scem	CTX_LOCK(ntb);
1714289546Scem	if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL)
1715289546Scem		ntb->ctx_ops->link_event(ntb->ntb_ctx);
1716289546Scem	CTX_UNLOCK(ntb);
1717289546Scem}
1718289546Scem
1719289546Scem/*
1720289546Scem * ntb_db_event() - notify driver context of a doorbell event
1721289546Scem * @ntb:        NTB device context
1722289546Scem * @vector:     Interrupt vector number
1723289546Scem *
1724289546Scem * Notify the driver context of a doorbell event.  If hardware supports
1725289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which
1726289546Scem * vector received the interrupt.  The vector number is relative to the first
1727289546Scem * vector used for doorbells, starting at zero, and must be less than
1728289546Scem * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
1729289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of
1730289546Scem * those bits are associated with the vector number.
1731289546Scem */
1732250079Scarlstatic void
1733289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec)
1734289272Scem{
1735289546Scem
1736289546Scem	CTX_LOCK(ntb);
1737289546Scem	if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL)
1738289546Scem		ntb->ctx_ops->db_event(ntb->ntb_ctx, vec);
1739289546Scem	CTX_UNLOCK(ntb);
1740289546Scem}
1741289546Scem
1742289546Scem/*
1743289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb
1744289546Scem * @ntb:        NTB device context
1745289546Scem * @max_speed:  The maximum link speed expressed as PCIe generation number[0]
1746289546Scem * @max_width:  The maximum link width expressed as the number of PCIe lanes[0]
1747289546Scem *
1748289546Scem * Enable the link on the secondary side of the ntb.  This can only be done
1749289546Scem * from the primary side of the ntb in primary or b2b topology.  The ntb device
1750289546Scem * should train the link to its maximum speed and width, or the requested speed
1751289546Scem * and width, whichever is smaller, if supported.
1752289546Scem *
1753289546Scem * Return: Zero on success, otherwise an error number.
1754289546Scem *
1755289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed
1756289546Scem *      and width input will be ignored.
1757289546Scem */
1758289546Scemint
1759289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused,
1760289546Scem    enum ntb_width w __unused)
1761289546Scem{
1762289280Scem	uint32_t cntl;
1763289272Scem
1764289648Scem	if (ntb->type == NTB_ATOM) {
1765289542Scem		pci_write_config(ntb->device, NTB_PPD_OFFSET,
1766289648Scem		    ntb->ppd | ATOM_PPD_INIT_LINK, 4);
1767289546Scem		return (0);
1768289542Scem	}
1769289542Scem
1770289280Scem	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
1771289546Scem		ntb_link_event(ntb);
1772289546Scem		return (0);
1773289280Scem	}
1774289280Scem
1775289542Scem	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1776289280Scem	cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
1777289280Scem	cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
1778289397Scem	cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
1779289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1780289397Scem		cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP;
1781289542Scem	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
1782289546Scem	return (0);
1783289272Scem}
1784289272Scem
1785289546Scem/*
1786289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb
1787289546Scem * @ntb:        NTB device context
1788289546Scem *
1789289546Scem * Disable the link on the secondary side of the ntb.  This can only be done
1790289546Scem * from the primary side of the ntb in primary or b2b topology.  The ntb device
1791289546Scem * should disable the link.  Returning from this call must indicate that a
1792289546Scem * barrier has passed, though with no more writes may pass in either direction
1793289546Scem * across the link, except if this call returns an error number.
1794289546Scem *
1795289546Scem * Return: Zero on success, otherwise an error number.
1796289546Scem */
1797289546Scemint
1798289542Scemntb_link_disable(struct ntb_softc *ntb)
1799289272Scem{
1800289272Scem	uint32_t cntl;
1801289272Scem
1802289272Scem	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
1803289546Scem		ntb_link_event(ntb);
1804289546Scem		return (0);
1805289272Scem	}
1806289272Scem
1807289542Scem	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1808289280Scem	cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
1809289397Scem	cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
1810289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1811289397Scem		cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP);
1812289280Scem	cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
1813289542Scem	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
1814289546Scem	return (0);
1815289272Scem}
1816289272Scem
1817289272Scemstatic void
1818289648Scemrecover_atom_link(void *arg)
1819250079Scarl{
1820250079Scarl	struct ntb_softc *ntb = arg;
1821289608Scem	unsigned speed, width, oldspeed, oldwidth;
1822250079Scarl	uint32_t status32;
1823250079Scarl
1824289648Scem	atom_perform_link_restart(ntb);
1825250079Scarl
1826289232Scem	/*
1827289232Scem	 * There is a potential race between the 2 NTB devices recovering at
1828289232Scem	 * the same time.  If the times are the same, the link will not recover
1829289232Scem	 * and the driver will be stuck in this loop forever.  Add a random
1830289232Scem	 * interval to the recovery time to prevent this race.
1831289232Scem	 */
1832289648Scem	status32 = arc4random() % ATOM_LINK_RECOVERY_TIME;
1833289648Scem	pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000);
1834289232Scem
1835289648Scem	if (atom_link_is_err(ntb))
1836250079Scarl		goto retry;
1837250079Scarl
1838289542Scem	status32 = ntb_reg_read(4, ntb->reg->ntb_ctl);
1839289648Scem	if ((status32 & ATOM_CNTL_LINK_DOWN) != 0)
1840289232Scem		goto out;
1841289232Scem
1842289542Scem	status32 = ntb_reg_read(4, ntb->reg->lnk_sta);
1843289608Scem	width = NTB_LNK_STA_WIDTH(status32);
1844289608Scem	speed = status32 & NTB_LINK_SPEED_MASK;
1845289608Scem
1846289608Scem	oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta);
1847289608Scem	oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK;
1848289608Scem	if (oldwidth != width || oldspeed != speed)
1849250079Scarl		goto retry;
1850250079Scarl
1851289232Scemout:
1852289648Scem	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb,
1853289542Scem	    ntb);
1854250079Scarl	return;
1855250079Scarl
1856250079Scarlretry:
1857289648Scem	callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link,
1858250079Scarl	    ntb);
1859250079Scarl}
1860250079Scarl
1861289546Scem/*
1862289546Scem * Polls the HW link status register(s); returns true if something has changed.
1863289546Scem */
1864289546Scemstatic bool
1865289542Scemntb_poll_link(struct ntb_softc *ntb)
1866250079Scarl{
1867250079Scarl	uint32_t ntb_cntl;
1868289546Scem	uint16_t reg_val;
1869250079Scarl
1870289648Scem	if (ntb->type == NTB_ATOM) {
1871289542Scem		ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1872289546Scem		if (ntb_cntl == ntb->ntb_ctl)
1873289546Scem			return (false);
1874289546Scem
1875289542Scem		ntb->ntb_ctl = ntb_cntl;
1876289542Scem		ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta);
1877250079Scarl	} else {
1878289607Scem		db_iowrite(ntb, ntb->self_reg->db_bell, ntb->db_link_mask);
1879250079Scarl
1880289546Scem		reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
1881289546Scem		if (reg_val == ntb->lnk_sta)
1882289546Scem			return (false);
1883250079Scarl
1884289546Scem		ntb->lnk_sta = reg_val;
1885289542Scem	}
1886289546Scem	return (true);
1887289542Scem}
1888289542Scem
1889289546Scemstatic inline enum ntb_speed
1890289546Scemntb_link_sta_speed(struct ntb_softc *ntb)
1891250079Scarl{
1892250079Scarl
1893289546Scem	if (!link_is_up(ntb))
1894289546Scem		return (NTB_SPEED_NONE);
1895289546Scem	return (ntb->lnk_sta & NTB_LINK_SPEED_MASK);
1896250079Scarl}
1897250079Scarl
1898289546Scemstatic inline enum ntb_width
1899289546Scemntb_link_sta_width(struct ntb_softc *ntb)
1900250079Scarl{
1901250079Scarl
1902289546Scem	if (!link_is_up(ntb))
1903289546Scem		return (NTB_WIDTH_NONE);
1904289546Scem	return (NTB_LNK_STA_WIDTH(ntb->lnk_sta));
1905250079Scarl}
1906250079Scarl
1907289774ScemSYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0,
1908289774Scem    "Driver state, statistics, and HW registers");
1909289774Scem
1910289774Scem#define NTB_REGSZ_MASK	(3ul << 30)
1911289774Scem#define NTB_REG_64	(1ul << 30)
1912289774Scem#define NTB_REG_32	(2ul << 30)
1913289774Scem#define NTB_REG_16	(3ul << 30)
1914289774Scem#define NTB_REG_8	(0ul << 30)
1915289774Scem
1916289774Scem#define NTB_DB_READ	(1ul << 29)
1917289774Scem#define NTB_PCI_REG	(1ul << 28)
1918289774Scem#define NTB_REGFLAGS_MASK	(NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG)
1919289774Scem
1920289774Scemstatic void
1921289774Scemntb_sysctl_init(struct ntb_softc *ntb)
1922289774Scem{
1923289774Scem	struct sysctl_oid_list *tree_par, *regpar, *statpar, *errpar;
1924289774Scem	struct sysctl_ctx_list *ctx;
1925289774Scem	struct sysctl_oid *tree, *tmptree;
1926289774Scem
1927289774Scem	ctx = device_get_sysctl_ctx(ntb->device);
1928289774Scem
1929289774Scem	tree = SYSCTL_ADD_NODE(ctx,
1930289774Scem	    SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)), OID_AUTO,
1931289774Scem	    "debug_info", CTLFLAG_RD, NULL,
1932289774Scem	    "Driver state, statistics, and HW registers");
1933289774Scem	tree_par = SYSCTL_CHILDREN(tree);
1934289774Scem
1935289774Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD,
1936289774Scem	    &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port");
1937289774Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD,
1938289774Scem	    &ntb->dev_type, 0, "0 - USD; 1 - DSD");
1939289774Scem
1940289774Scem	if (ntb->b2b_mw_idx != B2B_MW_DISABLED) {
1941289774Scem		SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD,
1942289774Scem		    &ntb->b2b_mw_idx, 0,
1943289774Scem		    "Index of the MW used for B2B remote register access");
1944289774Scem		SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off",
1945289774Scem		    CTLFLAG_RD, &ntb->b2b_off,
1946289774Scem		    "If non-zero, offset of B2B register region in shared MW");
1947289774Scem	}
1948289774Scem
1949289774Scem	SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features",
1950289774Scem	    CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A",
1951289774Scem	    "Features/errata of this NTB device");
1952289774Scem
1953289774Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD,
1954289774Scem	    &ntb->ntb_ctl, 0, "NTB CTL register (cached)");
1955289774Scem	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD,
1956289774Scem	    &ntb->lnk_sta, 0, "LNK STA register (cached)");
1957289774Scem
1958289774Scem	SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "link_status",
1959289774Scem	    CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_link_status,
1960289774Scem	    "A", "Link status");
1961289774Scem
1962289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD,
1963289774Scem	    &ntb->mw_count, 0, "MW count (excl. non-shared B2B register BAR)");
1964289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD,
1965289774Scem	    &ntb->spad_count, 0, "Scratchpad count");
1966289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD,
1967289774Scem	    &ntb->db_count, 0, "Doorbell count");
1968289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD,
1969289774Scem	    &ntb->db_vec_count, 0, "Doorbell vector count");
1970289774Scem	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD,
1971289774Scem	    &ntb->db_vec_shift, 0, "Doorbell vector shift");
1972289774Scem
1973289774Scem	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD,
1974289774Scem	    &ntb->db_valid_mask, "Doorbell valid mask");
1975289774Scem	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD,
1976289774Scem	    &ntb->db_link_mask, "Doorbell link mask");
1977289774Scem	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD,
1978289774Scem	    &ntb->db_mask, "Doorbell mask (cached)");
1979289774Scem
1980289774Scem	tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers",
1981289774Scem	    CTLFLAG_RD, NULL, "Raw HW registers (big-endian)");
1982289774Scem	regpar = SYSCTL_CHILDREN(tmptree);
1983289774Scem
1984289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask",
1985289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
1986289774Scem	    NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask,
1987289774Scem	    sysctl_handle_register, "QU", "Doorbell mask register");
1988289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell",
1989289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
1990289774Scem	    NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell,
1991289774Scem	    sysctl_handle_register, "QU", "Doorbell register");
1992289774Scem
1993289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23",
1994289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
1995289774Scem	    NTB_REG_64 | ntb->xlat_reg->bar2_xlat,
1996289774Scem	    sysctl_handle_register, "QU", "Incoming XLAT23 register");
1997289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
1998289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4",
1999289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2000289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar4_xlat,
2001289774Scem		    sysctl_handle_register, "IU", "Incoming XLAT4 register");
2002289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5",
2003289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2004289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar5_xlat,
2005289774Scem		    sysctl_handle_register, "IU", "Incoming XLAT5 register");
2006289774Scem	} else {
2007289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45",
2008289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2009289774Scem		    NTB_REG_64 | ntb->xlat_reg->bar4_xlat,
2010289774Scem		    sysctl_handle_register, "QU", "Incoming XLAT45 register");
2011289774Scem	}
2012289774Scem
2013289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23",
2014289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2015289774Scem	    NTB_REG_64 | ntb->xlat_reg->bar2_limit,
2016289774Scem	    sysctl_handle_register, "QU", "Incoming LMT23 register");
2017289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2018289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4",
2019289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2020289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar4_limit,
2021289774Scem		    sysctl_handle_register, "IU", "Incoming LMT4 register");
2022289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5",
2023289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2024289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar5_limit,
2025289774Scem		    sysctl_handle_register, "IU", "Incoming LMT5 register");
2026289774Scem	} else {
2027289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45",
2028289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2029289774Scem		    NTB_REG_64 | ntb->xlat_reg->bar4_limit,
2030289774Scem		    sysctl_handle_register, "QU", "Incoming LMT45 register");
2031289774Scem	}
2032289774Scem
2033289774Scem	if (ntb->type == NTB_ATOM)
2034289774Scem		return;
2035289774Scem
2036289774Scem	tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats",
2037289774Scem	    CTLFLAG_RD, NULL, "Xeon HW statistics");
2038289774Scem	statpar = SYSCTL_CHILDREN(tmptree);
2039289774Scem	SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss",
2040289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2041289774Scem	    NTB_REG_16 | XEON_USMEMMISS_OFFSET,
2042289774Scem	    sysctl_handle_register, "SU", "Upstream Memory Miss");
2043289774Scem
2044289774Scem	tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err",
2045289774Scem	    CTLFLAG_RD, NULL, "Xeon HW errors");
2046289774Scem	errpar = SYSCTL_CHILDREN(tmptree);
2047289774Scem
2048289774Scem	SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "devsts",
2049289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2050289774Scem	    NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET,
2051289774Scem	    sysctl_handle_register, "SU", "DEVSTS");
2052289774Scem	SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "lnksts",
2053289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2054289774Scem	    NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET,
2055289774Scem	    sysctl_handle_register, "SU", "LNKSTS");
2056289774Scem	SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts",
2057289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2058289774Scem	    NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET,
2059289774Scem	    sysctl_handle_register, "IU", "UNCERRSTS");
2060289774Scem	SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts",
2061289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2062289774Scem	    NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET,
2063289774Scem	    sysctl_handle_register, "IU", "CORERRSTS");
2064289774Scem
2065289774Scem	if (ntb->conn_type != NTB_CONN_B2B)
2066289774Scem		return;
2067289774Scem
2068289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23",
2069289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2070289774Scem	    NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off,
2071289774Scem	    sysctl_handle_register, "QU", "Outgoing XLAT23 register");
2072289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2073289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4",
2074289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2075289774Scem		    NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off,
2076289774Scem		    sysctl_handle_register, "IU", "Outgoing XLAT4 register");
2077289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5",
2078289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2079289774Scem		    NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off,
2080289774Scem		    sysctl_handle_register, "IU", "Outgoing XLAT5 register");
2081289774Scem	} else {
2082289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45",
2083289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2084289774Scem		    NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off,
2085289774Scem		    sysctl_handle_register, "QU", "Outgoing XLAT45 register");
2086289774Scem	}
2087289774Scem
2088289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23",
2089289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2090289774Scem	    NTB_REG_64 | XEON_PBAR2LMT_OFFSET,
2091289774Scem	    sysctl_handle_register, "QU", "Outgoing LMT23 register");
2092289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2093289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4",
2094289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2095289774Scem		    NTB_REG_32 | XEON_PBAR4LMT_OFFSET,
2096289774Scem		    sysctl_handle_register, "IU", "Outgoing LMT4 register");
2097289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5",
2098289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2099289774Scem		    NTB_REG_32 | XEON_PBAR5LMT_OFFSET,
2100289774Scem		    sysctl_handle_register, "IU", "Outgoing LMT5 register");
2101289774Scem	} else {
2102289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45",
2103289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2104289774Scem		    NTB_REG_64 | XEON_PBAR4LMT_OFFSET,
2105289774Scem		    sysctl_handle_register, "QU", "Outgoing LMT45 register");
2106289774Scem	}
2107289774Scem
2108289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base",
2109289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2110289774Scem	    NTB_REG_64 | ntb->xlat_reg->bar0_base,
2111289774Scem	    sysctl_handle_register, "QU", "Secondary BAR01 base register");
2112289774Scem	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base",
2113289774Scem	    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2114289774Scem	    NTB_REG_64 | ntb->xlat_reg->bar2_base,
2115289774Scem	    sysctl_handle_register, "QU", "Secondary BAR23 base register");
2116289774Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
2117289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base",
2118289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2119289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar4_base,
2120289774Scem		    sysctl_handle_register, "IU",
2121289774Scem		    "Secondary BAR4 base register");
2122289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base",
2123289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2124289774Scem		    NTB_REG_32 | ntb->xlat_reg->bar5_base,
2125289774Scem		    sysctl_handle_register, "IU",
2126289774Scem		    "Secondary BAR5 base register");
2127289774Scem	} else {
2128289774Scem		SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base",
2129289774Scem		    CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2130289774Scem		    NTB_REG_64 | ntb->xlat_reg->bar4_base,
2131289774Scem		    sysctl_handle_register, "QU",
2132289774Scem		    "Secondary BAR45 base register");
2133289774Scem	}
2134289774Scem}
2135289774Scem
2136289774Scemstatic int
2137289774Scemsysctl_handle_features(SYSCTL_HANDLER_ARGS)
2138289774Scem{
2139289774Scem	struct ntb_softc *ntb;
2140289774Scem	struct sbuf sb;
2141289774Scem	int error;
2142289774Scem
2143289774Scem	error = 0;
2144289774Scem	ntb = arg1;
2145289774Scem
2146289774Scem	sbuf_new_for_sysctl(&sb, NULL, 256, req);
2147289774Scem
2148289774Scem	sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR);
2149289774Scem	error = sbuf_finish(&sb);
2150289774Scem	sbuf_delete(&sb);
2151289774Scem
2152289774Scem	if (error || !req->newptr)
2153289774Scem		return (error);
2154289774Scem	return (EINVAL);
2155289774Scem}
2156289774Scem
2157289774Scemstatic int
2158289774Scemsysctl_handle_link_status(SYSCTL_HANDLER_ARGS)
2159289774Scem{
2160289774Scem	struct ntb_softc *ntb;
2161289774Scem	struct sbuf sb;
2162289774Scem	enum ntb_speed speed;
2163289774Scem	enum ntb_width width;
2164289774Scem	int error;
2165289774Scem
2166289774Scem	error = 0;
2167289774Scem	ntb = arg1;
2168289774Scem
2169289774Scem	sbuf_new_for_sysctl(&sb, NULL, 32, req);
2170289774Scem
2171289774Scem	if (ntb_link_is_up(ntb, &speed, &width))
2172289774Scem		sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u",
2173289774Scem		    (unsigned)speed, (unsigned)width);
2174289774Scem	else
2175289774Scem		sbuf_printf(&sb, "down");
2176289774Scem
2177289774Scem	error = sbuf_finish(&sb);
2178289774Scem	sbuf_delete(&sb);
2179289774Scem
2180289774Scem	if (error || !req->newptr)
2181289774Scem		return (error);
2182289774Scem	return (EINVAL);
2183289774Scem}
2184289774Scem
2185289774Scemstatic int
2186289774Scemsysctl_handle_register(SYSCTL_HANDLER_ARGS)
2187289774Scem{
2188289774Scem	struct ntb_softc *ntb;
2189289774Scem	const void *outp;
2190289774Scem	uintptr_t sz;
2191289774Scem	uint64_t umv;
2192289774Scem	char be[sizeof(umv)];
2193289774Scem	size_t outsz;
2194289774Scem	uint32_t reg;
2195289774Scem	bool db, pci;
2196289774Scem	int error;
2197289774Scem
2198289774Scem	ntb = arg1;
2199289774Scem	reg = arg2 & ~NTB_REGFLAGS_MASK;
2200289774Scem	sz = arg2 & NTB_REGSZ_MASK;
2201289774Scem	db = (arg2 & NTB_DB_READ) != 0;
2202289774Scem	pci = (arg2 & NTB_PCI_REG) != 0;
2203289774Scem
2204289774Scem	KASSERT(!(db && pci), ("bogus"));
2205289774Scem
2206289774Scem	if (db) {
2207289774Scem		KASSERT(sz == NTB_REG_64, ("bogus"));
2208289774Scem		umv = db_ioread(ntb, reg);
2209289774Scem		outsz = sizeof(uint64_t);
2210289774Scem	} else {
2211289774Scem		switch (sz) {
2212289774Scem		case NTB_REG_64:
2213289774Scem			if (pci)
2214289774Scem				umv = pci_read_config(ntb->device, reg, 8);
2215289774Scem			else
2216289774Scem				umv = ntb_reg_read(8, reg);
2217289774Scem			outsz = sizeof(uint64_t);
2218289774Scem			break;
2219289774Scem		case NTB_REG_32:
2220289774Scem			if (pci)
2221289774Scem				umv = pci_read_config(ntb->device, reg, 4);
2222289774Scem			else
2223289774Scem				umv = ntb_reg_read(4, reg);
2224289774Scem			outsz = sizeof(uint32_t);
2225289774Scem			break;
2226289774Scem		case NTB_REG_16:
2227289774Scem			if (pci)
2228289774Scem				umv = pci_read_config(ntb->device, reg, 2);
2229289774Scem			else
2230289774Scem				umv = ntb_reg_read(2, reg);
2231289774Scem			outsz = sizeof(uint16_t);
2232289774Scem			break;
2233289774Scem		case NTB_REG_8:
2234289774Scem			if (pci)
2235289774Scem				umv = pci_read_config(ntb->device, reg, 1);
2236289774Scem			else
2237289774Scem				umv = ntb_reg_read(1, reg);
2238289774Scem			outsz = sizeof(uint8_t);
2239289774Scem			break;
2240289774Scem		default:
2241289774Scem			panic("bogus");
2242289774Scem			break;
2243289774Scem		}
2244289774Scem	}
2245289774Scem
2246289774Scem	/* Encode bigendian so that sysctl -x is legible. */
2247289774Scem	be64enc(be, umv);
2248289774Scem	outp = ((char *)be) + sizeof(umv) - outsz;
2249289774Scem
2250289774Scem	error = SYSCTL_OUT(req, outp, outsz);
2251289774Scem	if (error || !req->newptr)
2252289774Scem		return (error);
2253289774Scem	return (EINVAL);
2254289774Scem}
2255289774Scem
2256289546Scem/*
2257289546Scem * Public API to the rest of the OS
2258250079Scarl */
2259250079Scarl
2260250079Scarl/**
2261250079Scarl * ntb_get_max_spads() - get the total scratch regs usable
2262250079Scarl * @ntb: pointer to ntb_softc instance
2263250079Scarl *
2264250079Scarl * This function returns the max 32bit scratchpad registers usable by the
2265250079Scarl * upper layer.
2266250079Scarl *
2267250079Scarl * RETURNS: total number of scratch pad registers available
2268250079Scarl */
2269289208Scemuint8_t
2270250079Scarlntb_get_max_spads(struct ntb_softc *ntb)
2271250079Scarl{
2272250079Scarl
2273289539Scem	return (ntb->spad_count);
2274250079Scarl}
2275250079Scarl
2276289396Scemuint8_t
2277289539Scemntb_mw_count(struct ntb_softc *ntb)
2278289396Scem{
2279289396Scem
2280289539Scem	return (ntb->mw_count);
2281289396Scem}
2282289396Scem
2283250079Scarl/**
2284289545Scem * ntb_spad_write() - write to the secondary scratchpad register
2285250079Scarl * @ntb: pointer to ntb_softc instance
2286250079Scarl * @idx: index to the scratchpad register, 0 based
2287250079Scarl * @val: the data value to put into the register
2288250079Scarl *
2289250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad
2290250079Scarl * register. The register resides on the secondary (external) side.
2291250079Scarl *
2292289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
2293250079Scarl */
2294250079Scarlint
2295289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
2296250079Scarl{
2297250079Scarl
2298289539Scem	if (idx >= ntb->spad_count)
2299250079Scarl		return (EINVAL);
2300250079Scarl
2301289607Scem	ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val);
2302250079Scarl
2303250079Scarl	return (0);
2304250079Scarl}
2305250079Scarl
2306250079Scarl/**
2307289545Scem * ntb_spad_read() - read from the primary scratchpad register
2308250079Scarl * @ntb: pointer to ntb_softc instance
2309250079Scarl * @idx: index to scratchpad register, 0 based
2310250079Scarl * @val: pointer to 32bit integer for storing the register value
2311250079Scarl *
2312250079Scarl * This function allows reading of the 32bit scratchpad register on
2313250079Scarl * the primary (internal) side.
2314250079Scarl *
2315289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
2316250079Scarl */
2317250079Scarlint
2318289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
2319250079Scarl{
2320250079Scarl
2321289539Scem	if (idx >= ntb->spad_count)
2322250079Scarl		return (EINVAL);
2323250079Scarl
2324289607Scem	*val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4);
2325250079Scarl
2326250079Scarl	return (0);
2327250079Scarl}
2328250079Scarl
2329250079Scarl/**
2330289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register
2331250079Scarl * @ntb: pointer to ntb_softc instance
2332250079Scarl * @idx: index to the scratchpad register, 0 based
2333250079Scarl * @val: the data value to put into the register
2334250079Scarl *
2335250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad
2336250079Scarl * register. The register resides on the secondary (external) side.
2337250079Scarl *
2338289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
2339250079Scarl */
2340250079Scarlint
2341289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
2342250079Scarl{
2343250079Scarl
2344289539Scem	if (idx >= ntb->spad_count)
2345250079Scarl		return (EINVAL);
2346250079Scarl
2347289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
2348255279Scarl		ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val);
2349255279Scarl	else
2350289542Scem		ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val);
2351250079Scarl
2352250079Scarl	return (0);
2353250079Scarl}
2354250079Scarl
2355250079Scarl/**
2356289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register
2357250079Scarl * @ntb: pointer to ntb_softc instance
2358250079Scarl * @idx: index to scratchpad register, 0 based
2359250079Scarl * @val: pointer to 32bit integer for storing the register value
2360250079Scarl *
2361250079Scarl * This function allows reading of the 32bit scratchpad register on
2362250079Scarl * the primary (internal) side.
2363250079Scarl *
2364289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
2365250079Scarl */
2366250079Scarlint
2367289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
2368250079Scarl{
2369250079Scarl
2370289539Scem	if (idx >= ntb->spad_count)
2371250079Scarl		return (EINVAL);
2372250079Scarl
2373289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
2374255279Scarl		*val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4);
2375255279Scarl	else
2376289542Scem		*val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4);
2377250079Scarl
2378250079Scarl	return (0);
2379250079Scarl}
2380250079Scarl
2381289546Scem/*
2382289546Scem * ntb_mw_get_range() - get the range of a memory window
2383289546Scem * @ntb:        NTB device context
2384289546Scem * @idx:        Memory window number
2385289546Scem * @base:       OUT - the base address for mapping the memory window
2386289546Scem * @size:       OUT - the size for mapping the memory window
2387289546Scem * @align:      OUT - the base alignment for translating the memory window
2388289546Scem * @align_size: OUT - the size alignment for translating the memory window
2389250079Scarl *
2390289546Scem * Get the range of a memory window.  NULL may be given for any output
2391289546Scem * parameter if the value is not needed.  The base and size may be used for
2392289546Scem * mapping the memory window, to access the peer memory.  The alignment and
2393289546Scem * size may be used for translating the memory window, for the peer to access
2394289546Scem * memory on the local system.
2395250079Scarl *
2396289546Scem * Return: Zero on success, otherwise an error number.
2397250079Scarl */
2398289546Scemint
2399289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base,
2400289546Scem    void **vbase, size_t *size, size_t *align, size_t *align_size)
2401250079Scarl{
2402289546Scem	struct ntb_pci_bar_info *bar;
2403289546Scem	size_t bar_b2b_off;
2404250079Scarl
2405289546Scem	if (mw_idx >= ntb_mw_count(ntb))
2406289546Scem		return (EINVAL);
2407250079Scarl
2408289546Scem	bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)];
2409289546Scem	bar_b2b_off = 0;
2410289546Scem	if (mw_idx == ntb->b2b_mw_idx) {
2411289546Scem		KASSERT(ntb->b2b_off != 0,
2412289546Scem		    ("user shouldn't get non-shared b2b mw"));
2413289546Scem		bar_b2b_off = ntb->b2b_off;
2414289546Scem	}
2415250079Scarl
2416289546Scem	if (base != NULL)
2417289546Scem		*base = bar->pbase + bar_b2b_off;
2418289546Scem	if (vbase != NULL)
2419289546Scem		*vbase = (char *)bar->vbase + bar_b2b_off;
2420289546Scem	if (size != NULL)
2421289546Scem		*size = bar->size - bar_b2b_off;
2422289546Scem	if (align != NULL)
2423289546Scem		*align = bar->size;
2424289546Scem	if (align_size != NULL)
2425289546Scem		*align_size = 1;
2426289546Scem	return (0);
2427250079Scarl}
2428250079Scarl
2429289546Scem/*
2430289546Scem * ntb_mw_set_trans() - set the translation of a memory window
2431289546Scem * @ntb:        NTB device context
2432289546Scem * @idx:        Memory window number
2433289546Scem * @addr:       The dma address local memory to expose to the peer
2434289546Scem * @size:       The size of the local memory to expose to the peer
2435250079Scarl *
2436289546Scem * Set the translation of a memory window.  The peer may access local memory
2437289546Scem * through the window starting at the address, up to the size.  The address
2438289546Scem * must be aligned to the alignment specified by ntb_mw_get_range().  The size
2439289546Scem * must be aligned to the size alignment specified by ntb_mw_get_range().
2440250079Scarl *
2441289546Scem * Return: Zero on success, otherwise an error number.
2442250079Scarl */
2443289546Scemint
2444289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr,
2445289546Scem    size_t size)
2446250079Scarl{
2447289546Scem	struct ntb_pci_bar_info *bar;
2448289546Scem	uint64_t base, limit, reg_val;
2449289546Scem	size_t bar_size, mw_size;
2450289546Scem	uint32_t base_reg, xlat_reg, limit_reg;
2451289546Scem	enum ntb_bar bar_num;
2452250079Scarl
2453289546Scem	if (idx >= ntb_mw_count(ntb))
2454289546Scem		return (EINVAL);
2455250079Scarl
2456289546Scem	bar_num = ntb_mw_to_bar(ntb, idx);
2457289546Scem	bar = &ntb->bar_info[bar_num];
2458250079Scarl
2459289546Scem	bar_size = bar->size;
2460289546Scem	if (idx == ntb->b2b_mw_idx)
2461289546Scem		mw_size = bar_size - ntb->b2b_off;
2462289546Scem	else
2463289546Scem		mw_size = bar_size;
2464250079Scarl
2465289546Scem	/* Hardware requires that addr is aligned to bar size */
2466289546Scem	if ((addr & (bar_size - 1)) != 0)
2467289546Scem		return (EINVAL);
2468250079Scarl
2469289546Scem	if (size > mw_size)
2470289546Scem		return (EINVAL);
2471289546Scem
2472289546Scem	bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg);
2473289546Scem
2474289546Scem	limit = 0;
2475289546Scem	if (bar_is_64bit(ntb, bar_num)) {
2476289546Scem		base = ntb_reg_read(8, base_reg);
2477289546Scem
2478289546Scem		if (limit_reg != 0 && size != mw_size)
2479289546Scem			limit = base + size;
2480289546Scem
2481289546Scem		/* Set and verify translation address */
2482289546Scem		ntb_reg_write(8, xlat_reg, addr);
2483289546Scem		reg_val = ntb_reg_read(8, xlat_reg);
2484289546Scem		if (reg_val != addr) {
2485289546Scem			ntb_reg_write(8, xlat_reg, 0);
2486289546Scem			return (EIO);
2487289546Scem		}
2488289546Scem
2489289546Scem		/* Set and verify the limit */
2490289546Scem		ntb_reg_write(8, limit_reg, limit);
2491289546Scem		reg_val = ntb_reg_read(8, limit_reg);
2492289546Scem		if (reg_val != limit) {
2493289546Scem			ntb_reg_write(8, limit_reg, base);
2494289546Scem			ntb_reg_write(8, xlat_reg, 0);
2495289546Scem			return (EIO);
2496289546Scem		}
2497289546Scem	} else {
2498289546Scem		/* Configure 32-bit (split) BAR MW */
2499289546Scem
2500289546Scem		if ((addr & ~UINT32_MAX) != 0)
2501289546Scem			return (EINVAL);
2502289546Scem		if (((addr + size) & ~UINT32_MAX) != 0)
2503289546Scem			return (EINVAL);
2504289546Scem
2505289546Scem		base = ntb_reg_read(4, base_reg);
2506289546Scem
2507289546Scem		if (limit_reg != 0 && size != mw_size)
2508289546Scem			limit = base + size;
2509289546Scem
2510289546Scem		/* Set and verify translation address */
2511289546Scem		ntb_reg_write(4, xlat_reg, addr);
2512289546Scem		reg_val = ntb_reg_read(4, xlat_reg);
2513289546Scem		if (reg_val != addr) {
2514289546Scem			ntb_reg_write(4, xlat_reg, 0);
2515289546Scem			return (EIO);
2516289546Scem		}
2517289546Scem
2518289546Scem		/* Set and verify the limit */
2519289546Scem		ntb_reg_write(4, limit_reg, limit);
2520289546Scem		reg_val = ntb_reg_read(4, limit_reg);
2521289546Scem		if (reg_val != limit) {
2522289546Scem			ntb_reg_write(4, limit_reg, base);
2523289546Scem			ntb_reg_write(4, xlat_reg, 0);
2524289546Scem			return (EIO);
2525289546Scem		}
2526250079Scarl	}
2527289546Scem	return (0);
2528250079Scarl}
2529250079Scarl
2530289596Scem/*
2531289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window
2532289596Scem * @ntb:	NTB device context
2533289596Scem * @idx:	Memory window number
2534289596Scem *
2535289596Scem * Clear the translation of a memory window.  The peer may no longer access
2536289596Scem * local memory through the window.
2537289596Scem *
2538289596Scem * Return: Zero on success, otherwise an error number.
2539289596Scem */
2540289596Scemint
2541289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx)
2542289596Scem{
2543289596Scem
2544289596Scem	return (ntb_mw_set_trans(ntb, mw_idx, 0, 0));
2545289596Scem}
2546289596Scem
2547250079Scarl/**
2548289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side
2549250079Scarl * @ntb: pointer to ntb_softc instance
2550289545Scem * @bit: doorbell bits to ring
2551250079Scarl *
2552250079Scarl * This function allows triggering of a doorbell on the secondary/external
2553250079Scarl * side that will initiate an interrupt on the remote host
2554250079Scarl */
2555250079Scarlvoid
2556289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit)
2557250079Scarl{
2558250079Scarl
2559289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
2560289347Scem		ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit);
2561289347Scem		return;
2562289209Scem	}
2563289347Scem
2564289546Scem	db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
2565250079Scarl}
2566250079Scarl
2567289542Scem/*
2568289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register,
2569289542Scem * as well as the size of the register (via *sz_out).
2570289542Scem *
2571289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell
2572289542Scem * ring to its memory window write.
2573289542Scem *
2574289542Scem * Note that writing the peer doorbell via a memory window will *not* generate
2575289542Scem * an interrupt on the remote host; that must be done seperately.
2576289542Scem */
2577289542Scembus_addr_t
2578289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out)
2579289542Scem{
2580289542Scem	struct ntb_pci_bar_info *bar;
2581289542Scem	uint64_t regoff;
2582289542Scem
2583289542Scem	KASSERT(sz_out != NULL, ("must be non-NULL"));
2584289542Scem
2585289542Scem	if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
2586289542Scem		bar = &ntb->bar_info[NTB_CONFIG_BAR];
2587289542Scem		regoff = ntb->peer_reg->db_bell;
2588289542Scem	} else {
2589289542Scem		KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) ||
2590289542Scem		    (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1),
2591289542Scem		    ("mw_count invalid after setup"));
2592289543Scem		KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED,
2593289543Scem		    ("invalid b2b idx"));
2594289542Scem
2595289542Scem		bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)];
2596289542Scem		regoff = XEON_SHADOW_PDOORBELL_OFFSET;
2597289542Scem	}
2598289542Scem	KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh"));
2599289542Scem
2600289542Scem	*sz_out = ntb->reg->db_size;
2601289542Scem	/* HACK: Specific to current x86 bus implementation. */
2602289542Scem	return ((uint64_t)bar->pci_bus_handle + regoff);
2603289542Scem}
2604289542Scem
2605289597Scem/*
2606289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb
2607289597Scem * @ntb:	NTB device context
2608289597Scem *
2609289597Scem * Hardware may support different number or arrangement of doorbell bits.
2610289597Scem *
2611289597Scem * Return: A mask of doorbell bits supported by the ntb.
2612289597Scem */
2613289597Scemuint64_t
2614289597Scemntb_db_valid_mask(struct ntb_softc *ntb)
2615289597Scem{
2616289597Scem
2617289597Scem	return (ntb->db_valid_mask);
2618289597Scem}
2619289597Scem
2620289598Scem/*
2621289598Scem * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector
2622289598Scem * @ntb:	NTB device context
2623289598Scem * @vector:	Doorbell vector number
2624289598Scem *
2625289598Scem * Each interrupt vector may have a different number or arrangement of bits.
2626289598Scem *
2627289598Scem * Return: A mask of doorbell bits serviced by a vector.
2628289598Scem */
2629289598Scemuint64_t
2630289598Scemntb_db_vector_mask(struct ntb_softc *ntb, uint32_t vector)
2631289598Scem{
2632289598Scem
2633289598Scem	if (vector > ntb->db_vec_count)
2634289598Scem		return (0);
2635289598Scem	return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector));
2636289598Scem}
2637289598Scem
2638250079Scarl/**
2639289546Scem * ntb_link_is_up() - get the current ntb link state
2640289546Scem * @ntb:        NTB device context
2641289546Scem * @speed:      OUT - The link speed expressed as PCIe generation number
2642289546Scem * @width:      OUT - The link width expressed as the number of PCIe lanes
2643250079Scarl *
2644250079Scarl * RETURNS: true or false based on the hardware link state
2645250079Scarl */
2646250079Scarlbool
2647289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed,
2648289546Scem    enum ntb_width *width)
2649250079Scarl{
2650250079Scarl
2651289546Scem	if (speed != NULL)
2652289546Scem		*speed = ntb_link_sta_speed(ntb);
2653289546Scem	if (width != NULL)
2654289546Scem		*width = ntb_link_sta_width(ntb);
2655289546Scem	return (link_is_up(ntb));
2656250079Scarl}
2657250079Scarl
2658255272Scarlstatic void
2659255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar)
2660250079Scarl{
2661255272Scarl
2662289209Scem	bar->pci_bus_tag = rman_get_bustag(bar->pci_resource);
2663289209Scem	bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource);
2664289209Scem	bar->pbase = rman_get_start(bar->pci_resource);
2665289209Scem	bar->size = rman_get_size(bar->pci_resource);
2666289209Scem	bar->vbase = rman_get_virtual(bar->pci_resource);
2667250079Scarl}
2668255268Scarl
2669289209Scemdevice_t
2670289209Scemntb_get_device(struct ntb_softc *ntb)
2671255268Scarl{
2672255268Scarl
2673255268Scarl	return (ntb->device);
2674255268Scarl}
2675289208Scem
2676289208Scem/* Export HW-specific errata information. */
2677289208Scembool
2678289774Scemntb_has_feature(struct ntb_softc *ntb, uint32_t feature)
2679289208Scem{
2680289208Scem
2681289208Scem	return (HAS_FEATURE(feature));
2682289208Scem}
2683