ntb_hw_intel.c revision 289597
1250079Scarl/*-
2250079Scarl * Copyright (C) 2013 Intel Corporation
3289542Scem * Copyright (C) 2015 EMC Corporation
4250079Scarl * All rights reserved.
5250079Scarl *
6250079Scarl * Redistribution and use in source and binary forms, with or without
7250079Scarl * modification, are permitted provided that the following conditions
8250079Scarl * are met:
9250079Scarl * 1. Redistributions of source code must retain the above copyright
10250079Scarl *    notice, this list of conditions and the following disclaimer.
11250079Scarl * 2. Redistributions in binary form must reproduce the above copyright
12250079Scarl *    notice, this list of conditions and the following disclaimer in the
13250079Scarl *    documentation and/or other materials provided with the distribution.
14250079Scarl *
15250079Scarl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16250079Scarl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17250079Scarl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18250079Scarl * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19250079Scarl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20250079Scarl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21250079Scarl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22250079Scarl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23250079Scarl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24250079Scarl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25250079Scarl * SUCH DAMAGE.
26250079Scarl */
27250079Scarl
28250079Scarl#include <sys/cdefs.h>
29250079Scarl__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 289597 2015-10-19 17:53:20Z cem $");
30250079Scarl
31250079Scarl#include <sys/param.h>
32250079Scarl#include <sys/kernel.h>
33250079Scarl#include <sys/systm.h>
34250079Scarl#include <sys/bus.h>
35250079Scarl#include <sys/malloc.h>
36250079Scarl#include <sys/module.h>
37250079Scarl#include <sys/queue.h>
38250079Scarl#include <sys/rman.h>
39289207Scem#include <sys/sysctl.h>
40250079Scarl#include <vm/vm.h>
41250079Scarl#include <vm/pmap.h>
42250079Scarl#include <machine/bus.h>
43250079Scarl#include <machine/pmap.h>
44250079Scarl#include <machine/resource.h>
45250079Scarl#include <dev/pci/pcireg.h>
46250079Scarl#include <dev/pci/pcivar.h>
47250079Scarl
48250079Scarl#include "ntb_regs.h"
49250079Scarl#include "ntb_hw.h"
50250079Scarl
51250079Scarl/*
52250079Scarl * The Non-Transparent Bridge (NTB) is a device on some Intel processors that
53250079Scarl * allows you to connect two systems using a PCI-e link.
54250079Scarl *
55250079Scarl * This module contains the hardware abstraction layer for the NTB. It allows
56250079Scarl * you to send and recieve interrupts, map the memory windows and send and
57250079Scarl * receive messages in the scratch-pad registers.
58250079Scarl *
59250079Scarl * NOTE: Much of the code in this module is shared with Linux. Any patches may
60250079Scarl * be picked up and redistributed in Linux with a dual GPL/BSD license.
61250079Scarl */
62250079Scarl
63289538Scem#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, SOC_DB_COUNT)
64250079Scarl
65289539Scem#define NTB_HB_TIMEOUT		1 /* second */
66289539Scem#define SOC_LINK_RECOVERY_TIME	500 /* ms */
67250079Scarl
68250079Scarl#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev))
69250079Scarl
70250079Scarlenum ntb_device_type {
71250079Scarl	NTB_XEON,
72250079Scarl	NTB_SOC
73250079Scarl};
74250079Scarl
75289539Scemenum ntb_bar {
76289539Scem	NTB_CONFIG_BAR = 0,
77289539Scem	NTB_B2B_BAR_1,
78289539Scem	NTB_B2B_BAR_2,
79289539Scem	NTB_B2B_BAR_3,
80289539Scem	NTB_MAX_BARS
81289539Scem};
82289539Scem
83255274Scarl/* Device features and workarounds */
84255274Scarl#define HAS_FEATURE(feature)	\
85255274Scarl	((ntb->features & (feature)) != 0)
86255274Scarl
87250079Scarlstruct ntb_hw_info {
88250079Scarl	uint32_t		device_id;
89255274Scarl	const char		*desc;
90250079Scarl	enum ntb_device_type	type;
91289397Scem	uint32_t		features;
92250079Scarl};
93250079Scarl
94250079Scarlstruct ntb_pci_bar_info {
95250079Scarl	bus_space_tag_t		pci_bus_tag;
96250079Scarl	bus_space_handle_t	pci_bus_handle;
97250079Scarl	int			pci_resource_id;
98250079Scarl	struct resource		*pci_resource;
99250079Scarl	vm_paddr_t		pbase;
100250079Scarl	void			*vbase;
101250079Scarl	u_long			size;
102289543Scem
103289543Scem	/* Configuration register offsets */
104289543Scem	uint32_t		psz_off;
105289543Scem	uint32_t		ssz_off;
106289543Scem	uint32_t		pbarxlat_off;
107250079Scarl};
108250079Scarl
109250079Scarlstruct ntb_int_info {
110250079Scarl	struct resource	*res;
111250079Scarl	int		rid;
112250079Scarl	void		*tag;
113250079Scarl};
114250079Scarl
115289546Scemstruct ntb_vec {
116250079Scarl	struct ntb_softc	*ntb;
117289546Scem	uint32_t		num;
118250079Scarl};
119250079Scarl
120289542Scemstruct ntb_reg {
121289542Scem	uint32_t	ntb_ctl;
122289542Scem	uint32_t	lnk_sta;
123289542Scem	uint8_t		db_size;
124289542Scem	unsigned	mw_bar[NTB_MAX_BARS];
125289542Scem};
126289542Scem
127289542Scemstruct ntb_alt_reg {
128289542Scem	uint32_t	db_bell;
129289542Scem	uint32_t	db_mask;
130289542Scem	uint32_t	spad;
131289542Scem};
132289542Scem
133289542Scemstruct ntb_xlat_reg {
134289546Scem	uint32_t	bar0_base;
135289546Scem	uint32_t	bar2_base;
136289546Scem	uint32_t	bar4_base;
137289546Scem	uint32_t	bar5_base;
138289546Scem
139289546Scem	uint32_t	bar2_xlat;
140289546Scem	uint32_t	bar4_xlat;
141289546Scem	uint32_t	bar5_xlat;
142289546Scem
143289546Scem	uint32_t	bar2_limit;
144289546Scem	uint32_t	bar4_limit;
145289546Scem	uint32_t	bar5_limit;
146289542Scem};
147289542Scem
148289542Scemstruct ntb_b2b_addr {
149289542Scem	uint64_t	bar0_addr;
150289542Scem	uint64_t	bar2_addr64;
151289542Scem	uint64_t	bar4_addr64;
152289542Scem	uint64_t	bar4_addr32;
153289542Scem	uint64_t	bar5_addr32;
154289542Scem};
155289542Scem
156250079Scarlstruct ntb_softc {
157250079Scarl	device_t		device;
158250079Scarl	enum ntb_device_type	type;
159255274Scarl	uint64_t		features;
160250079Scarl
161250079Scarl	struct ntb_pci_bar_info	bar_info[NTB_MAX_BARS];
162250079Scarl	struct ntb_int_info	int_info[MAX_MSIX_INTERRUPTS];
163250079Scarl	uint32_t		allocated_interrupts;
164250079Scarl
165250079Scarl	struct callout		heartbeat_timer;
166250079Scarl	struct callout		lr_timer;
167250079Scarl
168289546Scem	void			*ntb_ctx;
169289546Scem	const struct ntb_ctx_ops *ctx_ops;
170289546Scem	struct ntb_vec		*msix_vec;
171289546Scem#define CTX_LOCK(sc)		mtx_lock_spin(&(sc)->ctx_lock)
172289546Scem#define CTX_UNLOCK(sc)		mtx_unlock_spin(&(sc)->ctx_lock)
173289546Scem#define CTX_ASSERT(sc,f)	mtx_assert(&(sc)->ctx_lock, (f))
174289546Scem	struct mtx		ctx_lock;
175250079Scarl
176250079Scarl	struct {
177289255Scem		uint32_t ldb;
178289255Scem		uint32_t ldb_mask;
179289255Scem		uint32_t bar4_xlat;
180289397Scem		uint32_t bar5_xlat;
181250079Scarl		uint32_t spad_local;
182250079Scarl		uint32_t spci_cmd;
183250079Scarl	} reg_ofs;
184289348Scem	uint32_t ppd;
185250079Scarl	uint8_t conn_type;
186250079Scarl	uint8_t dev_type;
187250079Scarl	uint8_t link_width;
188250079Scarl	uint8_t link_speed;
189289539Scem
190289542Scem	/* Offset of peer bar0 in B2B BAR */
191289542Scem	uint64_t			b2b_off;
192289542Scem	/* Memory window used to access peer bar0 */
193289543Scem#define B2B_MW_DISABLED			UINT8_MAX
194289542Scem	uint8_t				b2b_mw_idx;
195289542Scem
196289539Scem	uint8_t				mw_count;
197289539Scem	uint8_t				spad_count;
198289539Scem	uint8_t				db_count;
199289539Scem	uint8_t				db_vec_count;
200289539Scem	uint8_t				db_vec_shift;
201289542Scem
202289546Scem	/* Protects local db_mask. */
203289546Scem#define DB_MASK_LOCK(sc)	mtx_lock_spin(&(sc)->db_mask_lock)
204289546Scem#define DB_MASK_UNLOCK(sc)	mtx_unlock_spin(&(sc)->db_mask_lock)
205289546Scem#define DB_MASK_ASSERT(sc,f)	mtx_assert(&(sc)->db_mask_lock, (f))
206289542Scem	struct mtx			db_mask_lock;
207289542Scem
208289546Scem	uint32_t			ntb_ctl;
209289546Scem	uint32_t			lnk_sta;
210289542Scem
211289542Scem	uint64_t			db_valid_mask;
212289542Scem	uint64_t			db_link_mask;
213289546Scem	uint64_t			db_mask;
214289542Scem
215289542Scem	int				last_ts;	/* ticks @ last irq */
216289542Scem
217289542Scem	const struct ntb_reg		*reg;
218289542Scem	const struct ntb_alt_reg	*self_reg;
219289542Scem	const struct ntb_alt_reg	*peer_reg;
220289542Scem	const struct ntb_xlat_reg	*xlat_reg;
221250079Scarl};
222250079Scarl
223289234Scem#ifdef __i386__
224289234Scemstatic __inline uint64_t
225289234Scembus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
226289234Scem    bus_size_t offset)
227289234Scem{
228289234Scem
229289234Scem	return (bus_space_read_4(tag, handle, offset) |
230289234Scem	    ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32);
231289234Scem}
232289234Scem
233289234Scemstatic __inline void
234289234Scembus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
235289234Scem    bus_size_t offset, uint64_t val)
236289234Scem{
237289234Scem
238289234Scem	bus_space_write_4(tag, handle, offset, val);
239289234Scem	bus_space_write_4(tag, handle, offset + 4, val >> 32);
240289234Scem}
241289234Scem#endif
242289234Scem
243255279Scarl#define ntb_bar_read(SIZE, bar, offset) \
244255279Scarl	    bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
245255279Scarl	    ntb->bar_info[(bar)].pci_bus_handle, (offset))
246255279Scarl#define ntb_bar_write(SIZE, bar, offset, val) \
247255279Scarl	    bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
248255279Scarl	    ntb->bar_info[(bar)].pci_bus_handle, (offset), (val))
249255279Scarl#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
250250079Scarl#define ntb_reg_write(SIZE, offset, val) \
251255279Scarl	    ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
252289397Scem#define ntb_mw_read(SIZE, offset) \
253289542Scem	    ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset)
254255279Scarl#define ntb_mw_write(SIZE, offset, val) \
255289542Scem	    ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \
256289397Scem		offset, val)
257250079Scarl
258250079Scarlstatic int ntb_probe(device_t device);
259250079Scarlstatic int ntb_attach(device_t device);
260250079Scarlstatic int ntb_detach(device_t device);
261289539Scemstatic inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw);
262289546Scemstatic inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar);
263289546Scemstatic inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar,
264289546Scem    uint32_t *base, uint32_t *xlat, uint32_t *lmt);
265255272Scarlstatic int ntb_map_pci_bars(struct ntb_softc *ntb);
266289541Scemstatic void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *);
267255272Scarlstatic int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
268255272Scarlstatic int map_memory_window_bar(struct ntb_softc *ntb,
269255272Scarl    struct ntb_pci_bar_info *bar);
270250079Scarlstatic void ntb_unmap_pci_bar(struct ntb_softc *ntb);
271289344Scemstatic int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
272289540Scemstatic int ntb_init_isr(struct ntb_softc *ntb);
273289342Scemstatic int ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
274289540Scemstatic int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
275250079Scarlstatic void ntb_teardown_interrupts(struct ntb_softc *ntb);
276289540Scemstatic inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);
277289546Scemstatic void ntb_interrupt(struct ntb_softc *, uint32_t vec);
278289546Scemstatic void ndev_vec_isr(void *arg);
279289546Scemstatic void ndev_irq_isr(void *arg);
280289546Scemstatic inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff);
281289546Scemstatic inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t val);
282289546Scemstatic int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors);
283289546Scemstatic void ntb_free_msix_vec(struct ntb_softc *ntb);
284250079Scarlstatic struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
285289397Scemstatic void ntb_detect_max_mw(struct ntb_softc *ntb);
286289348Scemstatic int ntb_detect_xeon(struct ntb_softc *ntb);
287289348Scemstatic int ntb_detect_soc(struct ntb_softc *ntb);
288289542Scemstatic int ntb_xeon_init_dev(struct ntb_softc *ntb);
289289542Scemstatic int ntb_soc_init_dev(struct ntb_softc *ntb);
290289272Scemstatic void ntb_teardown_xeon(struct ntb_softc *ntb);
291255279Scarlstatic void configure_soc_secondary_side_bars(struct ntb_softc *ntb);
292289543Scemstatic void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx,
293289543Scem    enum ntb_bar regbar);
294289543Scemstatic void xeon_set_sbar_base_and_limit(struct ntb_softc *,
295289543Scem    uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar);
296289543Scemstatic void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
297289543Scem    enum ntb_bar idx);
298289542Scemstatic int xeon_setup_b2b_mw(struct ntb_softc *,
299289542Scem    const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
300289546Scemstatic inline bool link_is_up(struct ntb_softc *ntb);
301289546Scemstatic inline bool soc_link_is_err(struct ntb_softc *ntb);
302289546Scemstatic inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *);
303289546Scemstatic inline enum ntb_width ntb_link_sta_width(struct ntb_softc *);
304289542Scemstatic void soc_link_hb(void *arg);
305289546Scemstatic void ntb_db_event(struct ntb_softc *ntb, uint32_t vec);
306250079Scarlstatic void recover_soc_link(void *arg);
307289546Scemstatic bool ntb_poll_link(struct ntb_softc *ntb);
308255274Scarlstatic void save_bar_parameters(struct ntb_pci_bar_info *bar);
309250079Scarl
310250079Scarlstatic struct ntb_hw_info pci_ids[] = {
311255274Scarl	{ 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 },
312289233Scem
313289233Scem	/* XXX: PS/SS IDs left out until they are supported. */
314289233Scem	{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
315289538Scem		NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
316289233Scem	{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
317289538Scem		NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
318289233Scem	{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
319289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
320289538Scem		    NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K },
321289233Scem	{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
322289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
323289538Scem		    NTB_SB01BASE_LOCKUP },
324289233Scem	{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
325289538Scem		NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
326289538Scem		    NTB_SB01BASE_LOCKUP },
327289233Scem
328255274Scarl	{ 0x00000000, NULL, NTB_SOC, 0 }
329250079Scarl};
330250079Scarl
331289542Scemstatic const struct ntb_reg soc_reg = {
332289542Scem	.ntb_ctl = SOC_NTBCNTL_OFFSET,
333289542Scem	.lnk_sta = SOC_LINK_STATUS_OFFSET,
334289542Scem	.db_size = sizeof(uint64_t),
335289542Scem	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
336289542Scem};
337289542Scem
338289542Scemstatic const struct ntb_alt_reg soc_b2b_reg = {
339289542Scem	.db_bell = SOC_B2B_DOORBELL_OFFSET,
340289542Scem	.spad = SOC_B2B_SPAD_OFFSET,
341289542Scem};
342289542Scem
343289542Scemstatic const struct ntb_xlat_reg soc_sec_xlat = {
344289542Scem#if 0
345289542Scem	/* "FIXME" says the Linux driver. */
346289542Scem	.bar0_base = SOC_SBAR0BASE_OFFSET,
347289546Scem	.bar2_base = SOC_SBAR2BASE_OFFSET,
348289546Scem	.bar4_base = SOC_SBAR4BASE_OFFSET,
349289546Scem
350289542Scem	.bar2_limit = SOC_SBAR2LMT_OFFSET,
351289546Scem	.bar4_limit = SOC_SBAR4LMT_OFFSET,
352289542Scem#endif
353289546Scem
354289542Scem	.bar2_xlat = SOC_SBAR2XLAT_OFFSET,
355289546Scem	.bar4_xlat = SOC_SBAR4XLAT_OFFSET,
356289542Scem};
357289542Scem
358289542Scemstatic const struct ntb_reg xeon_reg = {
359289542Scem	.ntb_ctl = XEON_NTBCNTL_OFFSET,
360289542Scem	.lnk_sta = XEON_LINK_STATUS_OFFSET,
361289542Scem	.db_size = sizeof(uint16_t),
362289542Scem	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 },
363289542Scem};
364289542Scem
365289542Scemstatic const struct ntb_alt_reg xeon_b2b_reg = {
366289542Scem	.db_bell = XEON_B2B_DOORBELL_OFFSET,
367289542Scem	.spad = XEON_B2B_SPAD_OFFSET,
368289542Scem};
369289542Scem
370289542Scemstatic const struct ntb_xlat_reg xeon_sec_xlat = {
371289542Scem	.bar0_base = XEON_SBAR0BASE_OFFSET,
372289546Scem	.bar2_base = XEON_SBAR2BASE_OFFSET,
373289546Scem	.bar4_base = XEON_SBAR4BASE_OFFSET,
374289546Scem	.bar5_base = XEON_SBAR5BASE_OFFSET,
375289546Scem
376289542Scem	.bar2_limit = XEON_SBAR2LMT_OFFSET,
377289546Scem	.bar4_limit = XEON_SBAR4LMT_OFFSET,
378289546Scem	.bar5_limit = XEON_SBAR5LMT_OFFSET,
379289546Scem
380289542Scem	.bar2_xlat = XEON_SBAR2XLAT_OFFSET,
381289546Scem	.bar4_xlat = XEON_SBAR4XLAT_OFFSET,
382289546Scem	.bar5_xlat = XEON_SBAR5XLAT_OFFSET,
383289542Scem};
384289542Scem
385289542Scemstatic const struct ntb_b2b_addr xeon_b2b_usd_addr = {
386289542Scem	.bar0_addr = XEON_B2B_BAR0_USD_ADDR,
387289542Scem	.bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64,
388289542Scem	.bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64,
389289542Scem	.bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32,
390289542Scem	.bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32,
391289542Scem};
392289542Scem
393289542Scemstatic const struct ntb_b2b_addr xeon_b2b_dsd_addr = {
394289542Scem	.bar0_addr = XEON_B2B_BAR0_DSD_ADDR,
395289542Scem	.bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64,
396289542Scem	.bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64,
397289542Scem	.bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32,
398289542Scem	.bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32,
399289542Scem};
400289542Scem
401250079Scarl/*
402250079Scarl * OS <-> Driver interface structures
403250079Scarl */
404250079ScarlMALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");
405250079Scarl
406250079Scarlstatic device_method_t ntb_pci_methods[] = {
407250079Scarl	/* Device interface */
408250079Scarl	DEVMETHOD(device_probe,     ntb_probe),
409250079Scarl	DEVMETHOD(device_attach,    ntb_attach),
410250079Scarl	DEVMETHOD(device_detach,    ntb_detach),
411250079Scarl	DEVMETHOD_END
412250079Scarl};
413250079Scarl
414250079Scarlstatic driver_t ntb_pci_driver = {
415250079Scarl	"ntb_hw",
416250079Scarl	ntb_pci_methods,
417250079Scarl	sizeof(struct ntb_softc),
418250079Scarl};
419250079Scarl
420250079Scarlstatic devclass_t ntb_devclass;
421250079ScarlDRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL);
422250079ScarlMODULE_VERSION(ntb_hw, 1);
423250079Scarl
424289207ScemSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls");
425289207Scem
426250079Scarl/*
427250079Scarl * OS <-> Driver linkage functions
428250079Scarl */
429250079Scarlstatic int
430250079Scarlntb_probe(device_t device)
431250079Scarl{
432289209Scem	struct ntb_hw_info *p;
433250079Scarl
434289209Scem	p = ntb_get_device_info(pci_get_devid(device));
435289209Scem	if (p == NULL)
436250079Scarl		return (ENXIO);
437289209Scem
438289209Scem	device_set_desc(device, p->desc);
439289209Scem	return (0);
440250079Scarl}
441250079Scarl
442250079Scarlstatic int
443250079Scarlntb_attach(device_t device)
444250079Scarl{
445289209Scem	struct ntb_softc *ntb;
446289209Scem	struct ntb_hw_info *p;
447250079Scarl	int error;
448250079Scarl
449289209Scem	ntb = DEVICE2SOFTC(device);
450289209Scem	p = ntb_get_device_info(pci_get_devid(device));
451289209Scem
452250079Scarl	ntb->device = device;
453250079Scarl	ntb->type = p->type;
454255274Scarl	ntb->features = p->features;
455289543Scem	ntb->b2b_mw_idx = B2B_MW_DISABLED;
456250079Scarl
457250079Scarl	/* Heartbeat timer for NTB_SOC since there is no link interrupt */
458283291Sjkim	callout_init(&ntb->heartbeat_timer, 1);
459283291Sjkim	callout_init(&ntb->lr_timer, 1);
460289542Scem	mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);
461289546Scem	mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN);
462250079Scarl
463289348Scem	if (ntb->type == NTB_SOC)
464289348Scem		error = ntb_detect_soc(ntb);
465289348Scem	else
466289348Scem		error = ntb_detect_xeon(ntb);
467289348Scem	if (error)
468289348Scem		goto out;
469289348Scem
470289397Scem	ntb_detect_max_mw(ntb);
471289396Scem
472289209Scem	error = ntb_map_pci_bars(ntb);
473289209Scem	if (error)
474289209Scem		goto out;
475289272Scem	if (ntb->type == NTB_SOC)
476289542Scem		error = ntb_soc_init_dev(ntb);
477289272Scem	else
478289542Scem		error = ntb_xeon_init_dev(ntb);
479289209Scem	if (error)
480289209Scem		goto out;
481289540Scem	error = ntb_init_isr(ntb);
482289209Scem	if (error)
483289209Scem		goto out;
484250079Scarl
485250079Scarl	pci_enable_busmaster(ntb->device);
486250079Scarl
487289209Scemout:
488289209Scem	if (error != 0)
489289209Scem		ntb_detach(device);
490250079Scarl	return (error);
491250079Scarl}
492250079Scarl
493250079Scarlstatic int
494250079Scarlntb_detach(device_t device)
495250079Scarl{
496289209Scem	struct ntb_softc *ntb;
497250079Scarl
498289209Scem	ntb = DEVICE2SOFTC(device);
499289542Scem
500289542Scem	ntb_db_set_mask(ntb, ntb->db_valid_mask);
501250079Scarl	callout_drain(&ntb->heartbeat_timer);
502250079Scarl	callout_drain(&ntb->lr_timer);
503289272Scem	if (ntb->type == NTB_XEON)
504289272Scem		ntb_teardown_xeon(ntb);
505250079Scarl	ntb_teardown_interrupts(ntb);
506289397Scem
507289542Scem	mtx_destroy(&ntb->db_mask_lock);
508289546Scem	mtx_destroy(&ntb->ctx_lock);
509289542Scem
510289397Scem	/*
511289397Scem	 * Redetect total MWs so we unmap properly -- in case we lowered the
512289397Scem	 * maximum to work around Xeon errata.
513289397Scem	 */
514289397Scem	ntb_detect_max_mw(ntb);
515250079Scarl	ntb_unmap_pci_bar(ntb);
516250079Scarl
517250079Scarl	return (0);
518250079Scarl}
519250079Scarl
520289542Scem/*
521289542Scem * Driver internal routines
522289542Scem */
523289539Scemstatic inline enum ntb_bar
524289539Scemntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
525289539Scem{
526289539Scem
527289543Scem	KASSERT(mw < ntb->mw_count ||
528289543Scem	    (mw != B2B_MW_DISABLED && mw == ntb->b2b_mw_idx),
529289542Scem	    ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count));
530289546Scem	KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw"));
531289539Scem
532289542Scem	return (ntb->reg->mw_bar[mw]);
533289539Scem}
534289539Scem
535289546Scemstatic inline bool
536289546Scembar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar)
537289546Scem{
538289546Scem	/* XXX This assertion could be stronger. */
539289546Scem	KASSERT(bar < NTB_MAX_BARS, ("bogus bar"));
540289546Scem	return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(NTB_SPLIT_BAR));
541289546Scem}
542289546Scem
543289546Scemstatic inline void
544289546Scembar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base,
545289546Scem    uint32_t *xlat, uint32_t *lmt)
546289546Scem{
547289546Scem	uint32_t basev, lmtv, xlatv;
548289546Scem
549289546Scem	switch (bar) {
550289546Scem	case NTB_B2B_BAR_1:
551289546Scem		basev = ntb->xlat_reg->bar2_base;
552289546Scem		lmtv = ntb->xlat_reg->bar2_limit;
553289546Scem		xlatv = ntb->xlat_reg->bar2_xlat;
554289546Scem		break;
555289546Scem	case NTB_B2B_BAR_2:
556289546Scem		basev = ntb->xlat_reg->bar4_base;
557289546Scem		lmtv = ntb->xlat_reg->bar4_limit;
558289546Scem		xlatv = ntb->xlat_reg->bar4_xlat;
559289546Scem		break;
560289546Scem	case NTB_B2B_BAR_3:
561289546Scem		basev = ntb->xlat_reg->bar5_base;
562289546Scem		lmtv = ntb->xlat_reg->bar5_limit;
563289546Scem		xlatv = ntb->xlat_reg->bar5_xlat;
564289546Scem		break;
565289546Scem	default:
566289546Scem		KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS,
567289546Scem		    ("bad bar"));
568289546Scem		basev = lmtv = xlatv = 0;
569289546Scem		break;
570289546Scem	}
571289546Scem
572289546Scem	if (base != NULL)
573289546Scem		*base = basev;
574289546Scem	if (xlat != NULL)
575289546Scem		*xlat = xlatv;
576289546Scem	if (lmt != NULL)
577289546Scem		*lmt = lmtv;
578289546Scem}
579289546Scem
580250079Scarlstatic int
581255272Scarlntb_map_pci_bars(struct ntb_softc *ntb)
582250079Scarl{
583255272Scarl	int rc;
584250079Scarl
585250079Scarl	ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0);
586289541Scem	rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_CONFIG_BAR]);
587255272Scarl	if (rc != 0)
588289541Scem		goto out;
589255272Scarl
590289209Scem	ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2);
591289541Scem	rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_1]);
592255272Scarl	if (rc != 0)
593289541Scem		goto out;
594289543Scem	ntb->bar_info[NTB_B2B_BAR_1].psz_off = XEON_PBAR23SZ_OFFSET;
595289543Scem	ntb->bar_info[NTB_B2B_BAR_1].ssz_off = XEON_SBAR23SZ_OFFSET;
596289543Scem	ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
597255272Scarl
598289209Scem	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
599289543Scem	/* XXX Are shared MW B2Bs write-combining? */
600289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && !HAS_FEATURE(NTB_SPLIT_BAR))
601289541Scem		rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
602255279Scarl	else
603289541Scem		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_2]);
604289543Scem	ntb->bar_info[NTB_B2B_BAR_2].psz_off = XEON_PBAR4SZ_OFFSET;
605289543Scem	ntb->bar_info[NTB_B2B_BAR_2].ssz_off = XEON_SBAR4SZ_OFFSET;
606289543Scem	ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
607289543Scem
608289397Scem	if (!HAS_FEATURE(NTB_SPLIT_BAR))
609289541Scem		goto out;
610289397Scem
611289397Scem	ntb->bar_info[NTB_B2B_BAR_3].pci_resource_id = PCIR_BAR(5);
612289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
613289541Scem		rc = map_mmr_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
614289397Scem	else
615289541Scem		rc = map_memory_window_bar(ntb, &ntb->bar_info[NTB_B2B_BAR_3]);
616289543Scem	ntb->bar_info[NTB_B2B_BAR_3].psz_off = XEON_PBAR5SZ_OFFSET;
617289543Scem	ntb->bar_info[NTB_B2B_BAR_3].ssz_off = XEON_SBAR5SZ_OFFSET;
618289543Scem	ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off = XEON_PBAR5XLAT_OFFSET;
619250079Scarl
620289541Scemout:
621289209Scem	if (rc != 0)
622255272Scarl		device_printf(ntb->device,
623255272Scarl		    "unable to allocate pci resource\n");
624255272Scarl	return (rc);
625255272Scarl}
626255272Scarl
627289541Scemstatic void
628289541Scemprint_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
629289541Scem{
630289541Scem
631289541Scem	device_printf(ntb->device, "Bar size = %lx, v %p, p %p\n",
632289541Scem	    bar->size, bar->vbase, (void *)(bar->pbase));
633289541Scem}
634289541Scem
635255272Scarlstatic int
636255272Scarlmap_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
637255272Scarl{
638255272Scarl
639255275Scarl	bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
640289209Scem	    &bar->pci_resource_id, RF_ACTIVE);
641255272Scarl	if (bar->pci_resource == NULL)
642255272Scarl		return (ENXIO);
643289209Scem
644289209Scem	save_bar_parameters(bar);
645289541Scem	print_map_success(ntb, bar);
646289209Scem	return (0);
647255272Scarl}
648255272Scarl
649255272Scarlstatic int
650255272Scarlmap_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
651255272Scarl{
652255272Scarl	int rc;
653255276Scarl	uint8_t bar_size_bits = 0;
654255272Scarl
655289209Scem	bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
656289209Scem	    &bar->pci_resource_id, RF_ACTIVE);
657250079Scarl
658255272Scarl	if (bar->pci_resource == NULL)
659255272Scarl		return (ENXIO);
660255276Scarl
661289209Scem	save_bar_parameters(bar);
662289209Scem	/*
663289209Scem	 * Ivytown NTB BAR sizes are misreported by the hardware due to a
664289209Scem	 * hardware issue. To work around this, query the size it should be
665289209Scem	 * configured to by the device and modify the resource to correspond to
666289209Scem	 * this new size. The BIOS on systems with this problem is required to
667289209Scem	 * provide enough address space to allow the driver to make this change
668289209Scem	 * safely.
669289209Scem	 *
670289209Scem	 * Ideally I could have just specified the size when I allocated the
671289209Scem	 * resource like:
672289209Scem	 *  bus_alloc_resource(ntb->device,
673289209Scem	 *	SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul,
674289209Scem	 *	1ul << bar_size_bits, RF_ACTIVE);
675289209Scem	 * but the PCI driver does not honor the size in this call, so we have
676289209Scem	 * to modify it after the fact.
677289209Scem	 */
678289209Scem	if (HAS_FEATURE(NTB_BAR_SIZE_4K)) {
679289209Scem		if (bar->pci_resource_id == PCIR_BAR(2))
680289209Scem			bar_size_bits = pci_read_config(ntb->device,
681289209Scem			    XEON_PBAR23SZ_OFFSET, 1);
682289209Scem		else
683289209Scem			bar_size_bits = pci_read_config(ntb->device,
684289209Scem			    XEON_PBAR45SZ_OFFSET, 1);
685289209Scem
686289209Scem		rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY,
687289209Scem		    bar->pci_resource, bar->pbase,
688289209Scem		    bar->pbase + (1ul << bar_size_bits) - 1);
689255272Scarl		if (rc != 0) {
690289209Scem			device_printf(ntb->device,
691289209Scem			    "unable to resize bar\n");
692255272Scarl			return (rc);
693250079Scarl		}
694289209Scem
695289209Scem		save_bar_parameters(bar);
696250079Scarl	}
697289209Scem
698289209Scem	/* Mark bar region as write combining to improve performance. */
699289209Scem	rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size,
700289209Scem	    VM_MEMATTR_WRITE_COMBINING);
701289209Scem	if (rc != 0) {
702289209Scem		device_printf(ntb->device,
703289209Scem		    "unable to mark bar as WRITE_COMBINING\n");
704289209Scem		return (rc);
705289209Scem	}
706289541Scem	print_map_success(ntb, bar);
707250079Scarl	return (0);
708250079Scarl}
709250079Scarl
710250079Scarlstatic void
711250079Scarlntb_unmap_pci_bar(struct ntb_softc *ntb)
712250079Scarl{
713250079Scarl	struct ntb_pci_bar_info *current_bar;
714250079Scarl	int i;
715250079Scarl
716289397Scem	for (i = 0; i < NTB_MAX_BARS; i++) {
717250079Scarl		current_bar = &ntb->bar_info[i];
718250079Scarl		if (current_bar->pci_resource != NULL)
719250079Scarl			bus_release_resource(ntb->device, SYS_RES_MEMORY,
720250079Scarl			    current_bar->pci_resource_id,
721250079Scarl			    current_bar->pci_resource);
722250079Scarl	}
723250079Scarl}
724250079Scarl
725250079Scarlstatic int
726289540Scemntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors)
727250079Scarl{
728289342Scem	uint32_t i;
729289342Scem	int rc;
730289342Scem
731289342Scem	for (i = 0; i < num_vectors; i++) {
732289342Scem		ntb->int_info[i].rid = i + 1;
733289342Scem		ntb->int_info[i].res = bus_alloc_resource_any(ntb->device,
734289342Scem		    SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE);
735289342Scem		if (ntb->int_info[i].res == NULL) {
736289342Scem			device_printf(ntb->device,
737289342Scem			    "bus_alloc_resource failed\n");
738289342Scem			return (ENOMEM);
739289342Scem		}
740289342Scem		ntb->int_info[i].tag = NULL;
741289342Scem		ntb->allocated_interrupts++;
742289342Scem		rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
743289546Scem		    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr,
744289546Scem		    &ntb->msix_vec[i], &ntb->int_info[i].tag);
745289342Scem		if (rc != 0) {
746289342Scem			device_printf(ntb->device, "bus_setup_intr failed\n");
747289342Scem			return (ENXIO);
748289342Scem		}
749289342Scem	}
750289342Scem	return (0);
751289342Scem}
752289342Scem
753289344Scem/*
754289344Scem * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector
755289344Scem * cannot be allocated for each MSI-X message.  JHB seems to think remapping
756289344Scem * should be okay.  This tunable should enable us to test that hypothesis
757289344Scem * when someone gets their hands on some Xeon hardware.
758289344Scem */
759289344Scemstatic int ntb_force_remap_mode;
760289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN,
761289344Scem    &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped"
762289344Scem    " to a smaller number of ithreads, even if the desired number are "
763289344Scem    "available");
764289344Scem
765289344Scem/*
766289344Scem * In case it is NOT ok, give consumers an abort button.
767289344Scem */
768289344Scemstatic int ntb_prefer_intx;
769289344ScemSYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN,
770289344Scem    &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather "
771289344Scem    "than remapping MSI-X messages over available slots (match Linux driver "
772289344Scem    "behavior)");
773289344Scem
774289344Scem/*
775289344Scem * Remap the desired number of MSI-X messages to available ithreads in a simple
776289344Scem * round-robin fashion.
777289344Scem */
778289342Scemstatic int
779289344Scemntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
780289344Scem{
781289344Scem	u_int *vectors;
782289344Scem	uint32_t i;
783289344Scem	int rc;
784289344Scem
785289344Scem	if (ntb_prefer_intx != 0)
786289344Scem		return (ENXIO);
787289344Scem
788289344Scem	vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK);
789289344Scem
790289344Scem	for (i = 0; i < desired; i++)
791289344Scem		vectors[i] = (i % avail) + 1;
792289344Scem
793289344Scem	rc = pci_remap_msix(dev, desired, vectors);
794289344Scem	free(vectors, M_NTB);
795289344Scem	return (rc);
796289344Scem}
797289344Scem
798289344Scemstatic int
799289540Scemntb_init_isr(struct ntb_softc *ntb)
800289342Scem{
801289344Scem	uint32_t desired_vectors, num_vectors;
802289342Scem	int rc;
803250079Scarl
804250079Scarl	ntb->allocated_interrupts = 0;
805289542Scem	ntb->last_ts = ticks;
806289347Scem
807250079Scarl	/*
808289546Scem	 * Mask all doorbell interrupts.
809250079Scarl	 */
810289546Scem	ntb_db_set_mask(ntb, ntb->db_valid_mask);
811250079Scarl
812289344Scem	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
813289539Scem	    ntb->db_count);
814289344Scem	if (desired_vectors >= 1) {
815289344Scem		rc = pci_alloc_msix(ntb->device, &num_vectors);
816250079Scarl
817289344Scem		if (ntb_force_remap_mode != 0 && rc == 0 &&
818289344Scem		    num_vectors == desired_vectors)
819289344Scem			num_vectors--;
820289344Scem
821289344Scem		if (rc == 0 && num_vectors < desired_vectors) {
822289344Scem			rc = ntb_remap_msix(ntb->device, desired_vectors,
823289344Scem			    num_vectors);
824289344Scem			if (rc == 0)
825289344Scem				num_vectors = desired_vectors;
826289344Scem			else
827289344Scem				pci_release_msi(ntb->device);
828289344Scem		}
829289344Scem		if (rc != 0)
830289344Scem			num_vectors = 1;
831289344Scem	} else
832289344Scem		num_vectors = 1;
833289344Scem
834289539Scem	if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
835289539Scem		ntb->db_vec_count = 1;
836289539Scem		ntb->db_vec_shift = ntb->db_count;
837289539Scem		rc = ntb_setup_legacy_interrupt(ntb);
838289539Scem	} else {
839289546Scem		ntb_create_msix_vec(ntb, num_vectors);
840289540Scem		rc = ntb_setup_msix(ntb, num_vectors);
841289539Scem	}
842289539Scem	if (rc != 0) {
843289539Scem		device_printf(ntb->device,
844289539Scem		    "Error allocating interrupts: %d\n", rc);
845289546Scem		ntb_free_msix_vec(ntb);
846289396Scem	}
847289396Scem
848289342Scem	return (rc);
849289342Scem}
850289342Scem
851289342Scemstatic int
852289342Scemntb_setup_legacy_interrupt(struct ntb_softc *ntb)
853289342Scem{
854289342Scem	int rc;
855289342Scem
856289342Scem	ntb->int_info[0].rid = 0;
857289342Scem	ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ,
858289342Scem	    &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
859289342Scem	if (ntb->int_info[0].res == NULL) {
860289342Scem		device_printf(ntb->device, "bus_alloc_resource failed\n");
861289342Scem		return (ENOMEM);
862250079Scarl	}
863250079Scarl
864289342Scem	ntb->int_info[0].tag = NULL;
865289342Scem	ntb->allocated_interrupts = 1;
866289342Scem
867289342Scem	rc = bus_setup_intr(ntb->device, ntb->int_info[0].res,
868289546Scem	    INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr,
869289342Scem	    ntb, &ntb->int_info[0].tag);
870289342Scem	if (rc != 0) {
871289342Scem		device_printf(ntb->device, "bus_setup_intr failed\n");
872289342Scem		return (ENXIO);
873289342Scem	}
874289342Scem
875250079Scarl	return (0);
876250079Scarl}
877250079Scarl
878250079Scarlstatic void
879250079Scarlntb_teardown_interrupts(struct ntb_softc *ntb)
880250079Scarl{
881250079Scarl	struct ntb_int_info *current_int;
882250079Scarl	int i;
883250079Scarl
884289209Scem	for (i = 0; i < ntb->allocated_interrupts; i++) {
885250079Scarl		current_int = &ntb->int_info[i];
886250079Scarl		if (current_int->tag != NULL)
887250079Scarl			bus_teardown_intr(ntb->device, current_int->res,
888250079Scarl			    current_int->tag);
889250079Scarl
890250079Scarl		if (current_int->res != NULL)
891250079Scarl			bus_release_resource(ntb->device, SYS_RES_IRQ,
892250079Scarl			    rman_get_rid(current_int->res), current_int->res);
893250079Scarl	}
894250079Scarl
895289546Scem	ntb_free_msix_vec(ntb);
896250079Scarl	pci_release_msi(ntb->device);
897250079Scarl}
898250079Scarl
899289347Scem/*
900289347Scem * Doorbell register and mask are 64-bit on SoC, 16-bit on Xeon.  Abstract it
901289347Scem * out to make code clearer.
902289347Scem */
903289539Scemstatic inline uint64_t
904289546Scemdb_ioread(struct ntb_softc *ntb, uint64_t regoff)
905289347Scem{
906289347Scem
907289347Scem	if (ntb->type == NTB_SOC)
908289347Scem		return (ntb_reg_read(8, regoff));
909289347Scem
910289347Scem	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
911289347Scem
912289347Scem	return (ntb_reg_read(2, regoff));
913289347Scem}
914289347Scem
915289539Scemstatic inline void
916289546Scemdb_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
917289347Scem{
918289347Scem
919289542Scem	KASSERT((val & ~ntb->db_valid_mask) == 0,
920289542Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
921289542Scem	     (uintmax_t)(val & ~ntb->db_valid_mask),
922289542Scem	     (uintmax_t)ntb->db_valid_mask));
923289542Scem
924289542Scem	if (regoff == ntb->reg_ofs.ldb_mask)
925289546Scem		DB_MASK_ASSERT(ntb, MA_OWNED);
926289542Scem
927289347Scem	if (ntb->type == NTB_SOC) {
928289347Scem		ntb_reg_write(8, regoff, val);
929289347Scem		return;
930289347Scem	}
931289347Scem
932289347Scem	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
933289347Scem	ntb_reg_write(2, regoff, (uint16_t)val);
934289347Scem}
935289347Scem
936289546Scemvoid
937289542Scemntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits)
938289542Scem{
939289542Scem
940289546Scem	DB_MASK_LOCK(ntb);
941289542Scem	ntb->db_mask |= bits;
942289546Scem	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask);
943289546Scem	DB_MASK_UNLOCK(ntb);
944289542Scem}
945289542Scem
946289546Scemvoid
947289542Scemntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits)
948289542Scem{
949289542Scem
950289542Scem	KASSERT((bits & ~ntb->db_valid_mask) == 0,
951289542Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
952289542Scem	     (uintmax_t)(bits & ~ntb->db_valid_mask),
953289542Scem	     (uintmax_t)ntb->db_valid_mask));
954289542Scem
955289546Scem	DB_MASK_LOCK(ntb);
956289542Scem	ntb->db_mask &= ~bits;
957289546Scem	db_iowrite(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask);
958289546Scem	DB_MASK_UNLOCK(ntb);
959289542Scem}
960289542Scem
961289546Scemuint64_t
962289546Scemntb_db_read(struct ntb_softc *ntb)
963289281Scem{
964289281Scem
965289546Scem	return (db_ioread(ntb, ntb->reg_ofs.ldb));
966289281Scem}
967289281Scem
968289546Scemvoid
969289546Scemntb_db_clear(struct ntb_softc *ntb, uint64_t bits)
970289281Scem{
971289281Scem
972289546Scem	KASSERT((bits & ~ntb->db_valid_mask) == 0,
973289546Scem	    ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
974289546Scem	     (uintmax_t)(bits & ~ntb->db_valid_mask),
975289546Scem	     (uintmax_t)ntb->db_valid_mask));
976289546Scem
977289546Scem	db_iowrite(ntb, ntb->reg_ofs.ldb, bits);
978289281Scem}
979289281Scem
980289540Scemstatic inline uint64_t
981289540Scemntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)
982250079Scarl{
983289540Scem	uint64_t shift, mask;
984250079Scarl
985289540Scem	shift = ntb->db_vec_shift;
986289540Scem	mask = (1ull << shift) - 1;
987289540Scem	return (mask << (shift * db_vector));
988250079Scarl}
989250079Scarl
990250079Scarlstatic void
991289546Scemntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
992250079Scarl{
993289540Scem	uint64_t vec_mask;
994250079Scarl
995289542Scem	ntb->last_ts = ticks;
996289546Scem	vec_mask = ntb_vec_mask(ntb, vec);
997250079Scarl
998289542Scem	if ((vec_mask & ntb->db_link_mask) != 0) {
999289546Scem		if (ntb_poll_link(ntb))
1000289546Scem			ntb_link_event(ntb);
1001289540Scem	}
1002289540Scem
1003289546Scem	if ((vec_mask & ntb->db_valid_mask) != 0)
1004289546Scem		ntb_db_event(ntb, vec);
1005289546Scem}
1006250079Scarl
1007289546Scemstatic void
1008289546Scemndev_vec_isr(void *arg)
1009289546Scem{
1010289546Scem	struct ntb_vec *nvec = arg;
1011250079Scarl
1012289546Scem	ntb_interrupt(nvec->ntb, nvec->num);
1013250079Scarl}
1014250079Scarl
1015250079Scarlstatic void
1016289546Scemndev_irq_isr(void *arg)
1017250079Scarl{
1018289546Scem	/* If we couldn't set up MSI-X, we only have the one vector. */
1019289546Scem	ntb_interrupt(arg, 0);
1020250079Scarl}
1021250079Scarl
1022250079Scarlstatic int
1023289546Scemntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors)
1024250079Scarl{
1025289342Scem	uint32_t i;
1026250079Scarl
1027289546Scem	ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB,
1028250079Scarl	    M_ZERO | M_WAITOK);
1029250079Scarl	for (i = 0; i < num_vectors; i++) {
1030289546Scem		ntb->msix_vec[i].num = i;
1031289546Scem		ntb->msix_vec[i].ntb = ntb;
1032250079Scarl	}
1033250079Scarl
1034250079Scarl	return (0);
1035250079Scarl}
1036250079Scarl
1037250079Scarlstatic void
1038289546Scemntb_free_msix_vec(struct ntb_softc *ntb)
1039250079Scarl{
1040250079Scarl
1041289546Scem	if (ntb->msix_vec == NULL)
1042289539Scem		return;
1043289539Scem
1044289546Scem	free(ntb->msix_vec, M_NTB);
1045289546Scem	ntb->msix_vec = NULL;
1046250079Scarl}
1047250079Scarl
1048250079Scarlstatic struct ntb_hw_info *
1049250079Scarlntb_get_device_info(uint32_t device_id)
1050250079Scarl{
1051250079Scarl	struct ntb_hw_info *ep = pci_ids;
1052250079Scarl
1053250079Scarl	while (ep->device_id) {
1054250079Scarl		if (ep->device_id == device_id)
1055250079Scarl			return (ep);
1056250079Scarl		++ep;
1057250079Scarl	}
1058250079Scarl	return (NULL);
1059250079Scarl}
1060250079Scarl
1061289272Scemstatic void
1062289272Scemntb_teardown_xeon(struct ntb_softc *ntb)
1063250079Scarl{
1064250079Scarl
1065289542Scem	ntb_link_disable(ntb);
1066250079Scarl}
1067250079Scarl
1068289397Scemstatic void
1069289397Scemntb_detect_max_mw(struct ntb_softc *ntb)
1070289397Scem{
1071289397Scem
1072289397Scem	if (ntb->type == NTB_SOC) {
1073289539Scem		ntb->mw_count = SOC_MW_COUNT;
1074289397Scem		return;
1075289397Scem	}
1076289397Scem
1077289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1078289539Scem		ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
1079289397Scem	else
1080289539Scem		ntb->mw_count = XEON_SNB_MW_COUNT;
1081289397Scem}
1082289397Scem
1083250079Scarlstatic int
1084289348Scemntb_detect_xeon(struct ntb_softc *ntb)
1085250079Scarl{
1086289348Scem	uint8_t ppd, conn_type;
1087250079Scarl
1088289348Scem	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
1089289348Scem	ntb->ppd = ppd;
1090250079Scarl
1091289348Scem	if ((ppd & XEON_PPD_DEV_TYPE) != 0)
1092289257Scem		ntb->dev_type = NTB_DEV_USD;
1093289257Scem	else
1094289257Scem		ntb->dev_type = NTB_DEV_DSD;
1095289257Scem
1096289397Scem	if ((ppd & XEON_PPD_SPLIT_BAR) != 0)
1097289397Scem		ntb->features |= NTB_SPLIT_BAR;
1098289397Scem
1099289542Scem	/* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */
1100289542Scem	if (HAS_FEATURE(NTB_SB01BASE_LOCKUP))
1101289542Scem		ntb->features |= NTB_SDOORBELL_LOCKUP;
1102289542Scem
1103289348Scem	conn_type = ppd & XEON_PPD_CONN_TYPE;
1104289348Scem	switch (conn_type) {
1105289348Scem	case NTB_CONN_B2B:
1106289348Scem		ntb->conn_type = conn_type;
1107289348Scem		break;
1108289348Scem	case NTB_CONN_RP:
1109289348Scem	case NTB_CONN_TRANSPARENT:
1110289348Scem	default:
1111289348Scem		device_printf(ntb->device, "Unsupported connection type: %u\n",
1112289348Scem		    (unsigned)conn_type);
1113289348Scem		return (ENXIO);
1114289348Scem	}
1115289348Scem	return (0);
1116289348Scem}
1117289348Scem
1118289348Scemstatic int
1119289348Scemntb_detect_soc(struct ntb_softc *ntb)
1120289348Scem{
1121289348Scem	uint32_t ppd, conn_type;
1122289348Scem
1123289348Scem	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
1124289348Scem	ntb->ppd = ppd;
1125289348Scem
1126289348Scem	if ((ppd & SOC_PPD_DEV_TYPE) != 0)
1127289348Scem		ntb->dev_type = NTB_DEV_DSD;
1128289348Scem	else
1129289348Scem		ntb->dev_type = NTB_DEV_USD;
1130289348Scem
1131289348Scem	conn_type = (ppd & SOC_PPD_CONN_TYPE) >> 8;
1132289348Scem	switch (conn_type) {
1133289348Scem	case NTB_CONN_B2B:
1134289348Scem		ntb->conn_type = conn_type;
1135289348Scem		break;
1136289348Scem	default:
1137289348Scem		device_printf(ntb->device, "Unsupported NTB configuration\n");
1138289348Scem		return (ENXIO);
1139289348Scem	}
1140289348Scem	return (0);
1141289348Scem}
1142289348Scem
1143289348Scemstatic int
1144289542Scemntb_xeon_init_dev(struct ntb_softc *ntb)
1145289348Scem{
1146289542Scem	int rc;
1147289348Scem
1148289257Scem	ntb->reg_ofs.ldb	= XEON_PDOORBELL_OFFSET;
1149289257Scem	ntb->reg_ofs.ldb_mask	= XEON_PDBMSK_OFFSET;
1150289257Scem	ntb->reg_ofs.spad_local	= XEON_SPAD_OFFSET;
1151289257Scem	ntb->reg_ofs.bar4_xlat	= XEON_SBAR4XLAT_OFFSET;
1152289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1153289397Scem		ntb->reg_ofs.bar5_xlat = XEON_SBAR5XLAT_OFFSET;
1154289542Scem	ntb->reg_ofs.spci_cmd	= XEON_PCICMD_OFFSET;
1155289257Scem
1156289542Scem	ntb->spad_count		= XEON_SPAD_COUNT;
1157289542Scem	ntb->db_count		= XEON_DB_COUNT;
1158289542Scem	ntb->db_link_mask	= XEON_DB_LINK_BIT;
1159289542Scem	ntb->db_vec_count	= XEON_DB_MSIX_VECTOR_COUNT;
1160289542Scem	ntb->db_vec_shift	= XEON_DB_MSIX_VECTOR_SHIFT;
1161289257Scem
1162289542Scem	if (ntb->conn_type != NTB_CONN_B2B) {
1163250079Scarl		device_printf(ntb->device, "Connection type %d not supported\n",
1164289348Scem		    ntb->conn_type);
1165250079Scarl		return (ENXIO);
1166250079Scarl	}
1167250079Scarl
1168289542Scem	ntb->reg = &xeon_reg;
1169289542Scem	ntb->peer_reg = &xeon_b2b_reg;
1170289542Scem	ntb->xlat_reg = &xeon_sec_xlat;
1171289542Scem
1172289208Scem	/*
1173289208Scem	 * There is a Xeon hardware errata related to writes to SDOORBELL or
1174289208Scem	 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space,
1175289208Scem	 * which may hang the system.  To workaround this use the second memory
1176289208Scem	 * window to access the interrupt and scratch pad registers on the
1177289208Scem	 * remote system.
1178289208Scem	 */
1179289543Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
1180289543Scem		/* Use the last MW for mapping remote spad */
1181289542Scem		ntb->b2b_mw_idx = ntb->mw_count - 1;
1182289543Scem	else if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14))
1183289208Scem		/*
1184289542Scem		 * HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
1185289542Scem		 * mirrored to the remote system.  Shrink the number of bits by one,
1186289542Scem		 * since bit 14 is the last bit.
1187289542Scem		 *
1188289542Scem		 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
1189289542Scem		 * anyway.  Nor for non-B2B connection types.
1190289542Scem		 */
1191289543Scem		ntb->db_count = XEON_DB_COUNT - 1;
1192250079Scarl
1193289542Scem	ntb->db_valid_mask = (1ull << ntb->db_count) - 1;
1194250079Scarl
1195289542Scem	if (ntb->dev_type == NTB_DEV_USD)
1196289542Scem		rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr,
1197289542Scem		    &xeon_b2b_usd_addr);
1198289542Scem	else
1199289542Scem		rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr,
1200289542Scem		    &xeon_b2b_dsd_addr);
1201289542Scem	if (rc != 0)
1202289542Scem		return (rc);
1203289271Scem
1204250079Scarl	/* Enable Bus Master and Memory Space on the secondary side */
1205289542Scem	ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
1206289542Scem	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
1207255279Scarl
1208255269Scarl	/* Enable link training */
1209289546Scem	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
1210250079Scarl
1211250079Scarl	return (0);
1212250079Scarl}
1213250079Scarl
1214250079Scarlstatic int
1215289542Scemntb_soc_init_dev(struct ntb_softc *ntb)
1216250079Scarl{
1217250079Scarl
1218289348Scem	KASSERT(ntb->conn_type == NTB_CONN_B2B,
1219289348Scem	    ("Unsupported NTB configuration (%d)\n", ntb->conn_type));
1220250079Scarl
1221289255Scem	ntb->reg_ofs.ldb	 = SOC_PDOORBELL_OFFSET;
1222289255Scem	ntb->reg_ofs.ldb_mask	 = SOC_PDBMSK_OFFSET;
1223289255Scem	ntb->reg_ofs.bar4_xlat	 = SOC_SBAR4XLAT_OFFSET;
1224250079Scarl	ntb->reg_ofs.spad_local	 = SOC_SPAD_OFFSET;
1225250079Scarl	ntb->reg_ofs.spci_cmd	 = SOC_PCICMD_OFFSET;
1226250079Scarl
1227289542Scem	ntb->spad_count		 = SOC_SPAD_COUNT;
1228289539Scem	ntb->db_count		 = SOC_DB_COUNT;
1229289539Scem	ntb->db_vec_count	 = SOC_DB_MSIX_VECTOR_COUNT;
1230289539Scem	ntb->db_vec_shift	 = SOC_DB_MSIX_VECTOR_SHIFT;
1231289542Scem	ntb->db_valid_mask	 = (1ull << ntb->db_count) - 1;
1232250079Scarl
1233289542Scem	ntb->reg = &soc_reg;
1234289542Scem	ntb->peer_reg = &soc_b2b_reg;
1235289542Scem	ntb->xlat_reg = &soc_sec_xlat;
1236289542Scem
1237250079Scarl	/*
1238250079Scarl	 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is
1239250079Scarl	 * resolved.  Mask transaction layer internal parity errors.
1240250079Scarl	 */
1241250079Scarl	pci_write_config(ntb->device, 0xFC, 0x4, 4);
1242250079Scarl
1243255279Scarl	configure_soc_secondary_side_bars(ntb);
1244250079Scarl
1245250079Scarl	/* Enable Bus Master and Memory Space on the secondary side */
1246255278Scarl	ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
1247250079Scarl	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
1248289209Scem
1249289542Scem	/* Initiate PCI-E link training */
1250289546Scem	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
1251250079Scarl
1252289542Scem	callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb);
1253289542Scem
1254250079Scarl	return (0);
1255250079Scarl}
1256250079Scarl
1257289542Scem/* XXX: Linux driver doesn't seem to do any of this for SoC. */
1258255279Scarlstatic void
1259255279Scarlconfigure_soc_secondary_side_bars(struct ntb_softc *ntb)
1260255279Scarl{
1261255279Scarl
1262255279Scarl	if (ntb->dev_type == NTB_DEV_USD) {
1263289538Scem		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET,
1264289542Scem		    XEON_B2B_BAR2_DSD_ADDR64);
1265289542Scem		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET,
1266289542Scem		    XEON_B2B_BAR4_DSD_ADDR64);
1267289542Scem		ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64);
1268289542Scem		ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64);
1269255279Scarl	} else {
1270289538Scem		ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET,
1271289542Scem		    XEON_B2B_BAR2_USD_ADDR64);
1272289542Scem		ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET,
1273289542Scem		    XEON_B2B_BAR4_USD_ADDR64);
1274289542Scem		ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64);
1275289542Scem		ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64);
1276255279Scarl	}
1277255279Scarl}
1278255279Scarl
1279289543Scem
1280289543Scem/*
1281289543Scem * When working around Xeon SDOORBELL errata by remapping remote registers in a
1282289543Scem * MW, limit the B2B MW to half a MW.  By sharing a MW, half the shared MW
1283289543Scem * remains for use by a higher layer.
1284289543Scem *
1285289543Scem * Will only be used if working around SDOORBELL errata and the BIOS-configured
1286289543Scem * MW size is sufficiently large.
1287289543Scem */
1288289543Scemstatic unsigned int ntb_b2b_mw_share;
1289289543ScemSYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share,
1290289543Scem    0, "If enabled (non-zero), prefer to share half of the B2B peer register "
1291289543Scem    "MW with higher level consumers.  Both sides of the NTB MUST set the same "
1292289543Scem    "value here.");
1293289543Scem
1294289543Scemstatic void
1295289543Scemxeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx,
1296289543Scem    enum ntb_bar regbar)
1297289543Scem{
1298289543Scem	struct ntb_pci_bar_info *bar;
1299289543Scem	uint8_t bar_sz;
1300289543Scem
1301289543Scem	if (!HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3)
1302289543Scem		return;
1303289543Scem
1304289543Scem	bar = &ntb->bar_info[idx];
1305289543Scem	bar_sz = pci_read_config(ntb->device, bar->psz_off, 1);
1306289543Scem	if (idx == regbar) {
1307289543Scem		if (ntb->b2b_off != 0)
1308289543Scem			bar_sz--;
1309289543Scem		else
1310289543Scem			bar_sz = 0;
1311289543Scem	}
1312289543Scem	pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1);
1313289543Scem	bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1);
1314289543Scem	(void)bar_sz;
1315289543Scem}
1316289543Scem
1317289543Scemstatic void
1318289546Scemxeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr,
1319289543Scem    enum ntb_bar idx, enum ntb_bar regbar)
1320289543Scem{
1321289546Scem	uint64_t reg_val;
1322289546Scem	uint32_t base_reg, lmt_reg;
1323289543Scem
1324289546Scem	bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg);
1325289546Scem	if (idx == regbar)
1326289546Scem		bar_addr += ntb->b2b_off;
1327289543Scem
1328289546Scem	if (!bar_is_64bit(ntb, idx)) {
1329289546Scem		ntb_reg_write(4, base_reg, bar_addr);
1330289546Scem		reg_val = ntb_reg_read(4, base_reg);
1331289546Scem		(void)reg_val;
1332289546Scem
1333289546Scem		ntb_reg_write(4, lmt_reg, bar_addr);
1334289546Scem		reg_val = ntb_reg_read(4, lmt_reg);
1335289546Scem		(void)reg_val;
1336289543Scem	} else {
1337289546Scem		ntb_reg_write(8, base_reg, bar_addr);
1338289546Scem		reg_val = ntb_reg_read(8, base_reg);
1339289546Scem		(void)reg_val;
1340289546Scem
1341289546Scem		ntb_reg_write(8, lmt_reg, bar_addr);
1342289546Scem		reg_val = ntb_reg_read(8, lmt_reg);
1343289546Scem		(void)reg_val;
1344289543Scem	}
1345289543Scem}
1346289543Scem
1347289543Scemstatic void
1348289543Scemxeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx)
1349289543Scem{
1350289543Scem	struct ntb_pci_bar_info *bar;
1351289543Scem
1352289543Scem	bar = &ntb->bar_info[idx];
1353289543Scem	if (HAS_FEATURE(NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) {
1354289543Scem		ntb_reg_write(4, bar->pbarxlat_off, base_addr);
1355289543Scem		base_addr = ntb_reg_read(4, bar->pbarxlat_off);
1356289543Scem	} else {
1357289543Scem		ntb_reg_write(8, bar->pbarxlat_off, base_addr);
1358289543Scem		base_addr = ntb_reg_read(8, bar->pbarxlat_off);
1359289543Scem	}
1360289543Scem	(void)base_addr;
1361289543Scem}
1362289543Scem
1363289542Scemstatic int
1364289542Scemxeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
1365289542Scem    const struct ntb_b2b_addr *peer_addr)
1366255279Scarl{
1367289543Scem	struct ntb_pci_bar_info *b2b_bar;
1368289543Scem	vm_size_t bar_size;
1369289543Scem	uint64_t bar_addr;
1370289543Scem	enum ntb_bar b2b_bar_num, i;
1371255279Scarl
1372289543Scem	if (ntb->b2b_mw_idx == B2B_MW_DISABLED) {
1373289543Scem		b2b_bar = NULL;
1374289543Scem		b2b_bar_num = NTB_CONFIG_BAR;
1375289543Scem		ntb->b2b_off = 0;
1376289543Scem	} else {
1377289543Scem		b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx);
1378289543Scem		KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS,
1379289543Scem		    ("invalid b2b mw bar"));
1380289543Scem
1381289543Scem		b2b_bar = &ntb->bar_info[b2b_bar_num];
1382289543Scem		bar_size = b2b_bar->size;
1383289543Scem
1384289543Scem		if (ntb_b2b_mw_share != 0 &&
1385289543Scem		    (bar_size >> 1) >= XEON_B2B_MIN_SIZE)
1386289543Scem			ntb->b2b_off = bar_size >> 1;
1387289543Scem		else if (bar_size >= XEON_B2B_MIN_SIZE) {
1388289543Scem			ntb->b2b_off = 0;
1389289543Scem			ntb->mw_count--;
1390289543Scem		} else {
1391289543Scem			device_printf(ntb->device,
1392289543Scem			    "B2B bar size is too small!\n");
1393289543Scem			return (EIO);
1394289543Scem		}
1395255279Scarl	}
1396289542Scem
1397289543Scem	/*
1398289543Scem	 * Reset the secondary bar sizes to match the primary bar sizes.
1399289543Scem	 * (Except, disable or halve the size of the B2B secondary bar.)
1400289543Scem	 */
1401289543Scem	for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++)
1402289543Scem		xeon_reset_sbar_size(ntb, i, b2b_bar_num);
1403289543Scem
1404289543Scem	bar_addr = 0;
1405289543Scem	if (b2b_bar_num == NTB_CONFIG_BAR)
1406289543Scem		bar_addr = addr->bar0_addr;
1407289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_1)
1408289543Scem		bar_addr = addr->bar2_addr64;
1409289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR))
1410289543Scem		bar_addr = addr->bar4_addr64;
1411289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2)
1412289543Scem		bar_addr = addr->bar4_addr32;
1413289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_3)
1414289543Scem		bar_addr = addr->bar5_addr32;
1415289543Scem	else
1416289543Scem		KASSERT(false, ("invalid bar"));
1417289543Scem
1418289543Scem	ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr);
1419289543Scem
1420289543Scem	/*
1421289543Scem	 * Other SBARs are normally hit by the PBAR xlat, except for the b2b
1422289543Scem	 * register BAR.  The B2B BAR is either disabled above or configured
1423289543Scem	 * half-size.  It starts at PBAR xlat + offset.
1424289543Scem	 *
1425289543Scem	 * Also set up incoming BAR limits == base (zero length window).
1426289543Scem	 */
1427289543Scem	xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1,
1428289543Scem	    b2b_bar_num);
1429289542Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
1430289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32,
1431289543Scem		    NTB_B2B_BAR_2, b2b_bar_num);
1432289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32,
1433289543Scem		    NTB_B2B_BAR_3, b2b_bar_num);
1434289542Scem	} else
1435289543Scem		xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64,
1436289543Scem		    NTB_B2B_BAR_2, b2b_bar_num);
1437289543Scem
1438289543Scem	/* Zero incoming translation addrs */
1439289543Scem	ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0);
1440289543Scem	ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0);
1441289543Scem
1442289543Scem	/* Zero outgoing translation limits (whole bar size windows) */
1443289543Scem	ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0);
1444289543Scem	ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
1445289543Scem
1446289543Scem	/* Set outgoing translation offsets */
1447289543Scem	xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1);
1448289543Scem	if (HAS_FEATURE(NTB_SPLIT_BAR)) {
1449289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2);
1450289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3);
1451289543Scem	} else
1452289543Scem		xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2);
1453289543Scem
1454289543Scem	/* Set the translation offset for B2B registers */
1455289543Scem	bar_addr = 0;
1456289543Scem	if (b2b_bar_num == NTB_CONFIG_BAR)
1457289543Scem		bar_addr = peer_addr->bar0_addr;
1458289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_1)
1459289543Scem		bar_addr = peer_addr->bar2_addr64;
1460289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(NTB_SPLIT_BAR))
1461289543Scem		bar_addr = peer_addr->bar4_addr64;
1462289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_2)
1463289543Scem		bar_addr = peer_addr->bar4_addr32;
1464289543Scem	else if (b2b_bar_num == NTB_B2B_BAR_3)
1465289543Scem		bar_addr = peer_addr->bar5_addr32;
1466289543Scem	else
1467289543Scem		KASSERT(false, ("invalid bar"));
1468289543Scem
1469289543Scem	/*
1470289543Scem	 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits
1471289543Scem	 * at a time.
1472289543Scem	 */
1473289543Scem	ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff);
1474289543Scem	ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32);
1475289542Scem	return (0);
1476255279Scarl}
1477255279Scarl
1478289546Scemstatic inline bool
1479289546Scemlink_is_up(struct ntb_softc *ntb)
1480289546Scem{
1481289546Scem
1482289546Scem	if (ntb->type == NTB_XEON)
1483289546Scem		return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0);
1484289546Scem
1485289546Scem	KASSERT(ntb->type == NTB_SOC, ("ntb type"));
1486289546Scem	return ((ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0);
1487289546Scem}
1488289546Scem
1489289546Scemstatic inline bool
1490289546Scemsoc_link_is_err(struct ntb_softc *ntb)
1491289546Scem{
1492289546Scem	uint32_t status;
1493289546Scem
1494289546Scem	KASSERT(ntb->type == NTB_SOC, ("ntb type"));
1495289546Scem
1496289546Scem	status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
1497289546Scem	if ((status & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0)
1498289546Scem		return (true);
1499289546Scem
1500289546Scem	status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
1501289546Scem	return ((status & SOC_IBIST_ERR_OFLOW) != 0);
1502289546Scem}
1503289546Scem
1504255281Scarl/* SOC does not have link status interrupt, poll on that platform */
1505250079Scarlstatic void
1506289542Scemsoc_link_hb(void *arg)
1507250079Scarl{
1508250079Scarl	struct ntb_softc *ntb = arg;
1509289546Scem	sbintime_t timo, poll_ts;
1510250079Scarl
1511289546Scem	timo = NTB_HB_TIMEOUT * hz;
1512289546Scem	poll_ts = ntb->last_ts + timo;
1513289546Scem
1514289542Scem	/*
1515289542Scem	 * Delay polling the link status if an interrupt was received, unless
1516289542Scem	 * the cached link status says the link is down.
1517289542Scem	 */
1518289546Scem	if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) {
1519289546Scem		timo = poll_ts - ticks;
1520289542Scem		goto out;
1521289546Scem	}
1522289542Scem
1523289546Scem	if (ntb_poll_link(ntb))
1524289546Scem		ntb_link_event(ntb);
1525289542Scem
1526289546Scem	if (!link_is_up(ntb) && soc_link_is_err(ntb)) {
1527289546Scem		/* Link is down with error, proceed with recovery */
1528289546Scem		callout_reset(&ntb->lr_timer, 0, recover_soc_link, ntb);
1529289546Scem		return;
1530250079Scarl	}
1531250079Scarl
1532289542Scemout:
1533289546Scem	callout_reset(&ntb->heartbeat_timer, timo, soc_link_hb, ntb);
1534250079Scarl}
1535250079Scarl
1536250079Scarlstatic void
1537250079Scarlsoc_perform_link_restart(struct ntb_softc *ntb)
1538250079Scarl{
1539250079Scarl	uint32_t status;
1540250079Scarl
1541250079Scarl	/* Driver resets the NTB ModPhy lanes - magic! */
1542255278Scarl	ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0);
1543255278Scarl	ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40);
1544255278Scarl	ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60);
1545255278Scarl	ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60);
1546250079Scarl
1547250079Scarl	/* Driver waits 100ms to allow the NTB ModPhy to settle */
1548250079Scarl	pause("ModPhy", hz / 10);
1549250079Scarl
1550250079Scarl	/* Clear AER Errors, write to clear */
1551255278Scarl	status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET);
1552250079Scarl	status &= PCIM_AER_COR_REPLAY_ROLLOVER;
1553255278Scarl	ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status);
1554250079Scarl
1555250079Scarl	/* Clear unexpected electrical idle event in LTSSM, write to clear */
1556255278Scarl	status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET);
1557250079Scarl	status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI;
1558255278Scarl	ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status);
1559250079Scarl
1560250079Scarl	/* Clear DeSkew Buffer error, write to clear */
1561255278Scarl	status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET);
1562250079Scarl	status |= SOC_DESKEWSTS_DBERR;
1563255278Scarl	ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status);
1564250079Scarl
1565255278Scarl	status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
1566250079Scarl	status &= SOC_IBIST_ERR_OFLOW;
1567255278Scarl	ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status);
1568250079Scarl
1569250079Scarl	/* Releases the NTB state machine to allow the link to retrain */
1570255278Scarl	status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
1571250079Scarl	status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT;
1572255278Scarl	ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status);
1573250079Scarl}
1574250079Scarl
1575289546Scem/*
1576289546Scem * ntb_set_ctx() - associate a driver context with an ntb device
1577289546Scem * @ntb:        NTB device context
1578289546Scem * @ctx:        Driver context
1579289546Scem * @ctx_ops:    Driver context operations
1580289546Scem *
1581289546Scem * Associate a driver context and operations with a ntb device.  The context is
1582289546Scem * provided by the client driver, and the driver may associate a different
1583289546Scem * context with each ntb device.
1584289546Scem *
1585289546Scem * Return: Zero if the context is associated, otherwise an error number.
1586289546Scem */
1587289546Scemint
1588289546Scemntb_set_ctx(struct ntb_softc *ntb, void *ctx, const struct ntb_ctx_ops *ops)
1589250079Scarl{
1590250079Scarl
1591289546Scem	if (ctx == NULL || ops == NULL)
1592289546Scem		return (EINVAL);
1593289546Scem	if (ntb->ctx_ops != NULL)
1594289546Scem		return (EINVAL);
1595250079Scarl
1596289546Scem	CTX_LOCK(ntb);
1597289546Scem	if (ntb->ctx_ops != NULL) {
1598289546Scem		CTX_UNLOCK(ntb);
1599289546Scem		return (EINVAL);
1600250079Scarl	}
1601289546Scem	ntb->ntb_ctx = ctx;
1602289546Scem	ntb->ctx_ops = ops;
1603289546Scem	CTX_UNLOCK(ntb);
1604250079Scarl
1605289546Scem	return (0);
1606250079Scarl}
1607250079Scarl
1608289546Scem/*
1609289546Scem * It is expected that this will only be used from contexts where the ctx_lock
1610289546Scem * is not needed to protect ntb_ctx lifetime.
1611289546Scem */
1612289546Scemvoid *
1613289546Scemntb_get_ctx(struct ntb_softc *ntb, const struct ntb_ctx_ops **ops)
1614289546Scem{
1615289546Scem
1616289546Scem	KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus"));
1617289546Scem	if (ops != NULL)
1618289546Scem		*ops = ntb->ctx_ops;
1619289546Scem	return (ntb->ntb_ctx);
1620289546Scem}
1621289546Scem
1622289546Scem/*
1623289546Scem * ntb_clear_ctx() - disassociate any driver context from an ntb device
1624289546Scem * @ntb:        NTB device context
1625289546Scem *
1626289546Scem * Clear any association that may exist between a driver context and the ntb
1627289546Scem * device.
1628289546Scem */
1629289546Scemvoid
1630289546Scemntb_clear_ctx(struct ntb_softc *ntb)
1631289546Scem{
1632289546Scem
1633289546Scem	CTX_LOCK(ntb);
1634289546Scem	ntb->ntb_ctx = NULL;
1635289546Scem	ntb->ctx_ops = NULL;
1636289546Scem	CTX_UNLOCK(ntb);
1637289546Scem}
1638289546Scem
1639289546Scem/*
1640289546Scem * ntb_link_event() - notify driver context of a change in link status
1641289546Scem * @ntb:        NTB device context
1642289546Scem *
1643289546Scem * Notify the driver context that the link status may have changed.  The driver
1644289546Scem * should call ntb_link_is_up() to get the current status.
1645289546Scem */
1646289546Scemvoid
1647289546Scemntb_link_event(struct ntb_softc *ntb)
1648289546Scem{
1649289546Scem
1650289546Scem	CTX_LOCK(ntb);
1651289546Scem	if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL)
1652289546Scem		ntb->ctx_ops->link_event(ntb->ntb_ctx);
1653289546Scem	CTX_UNLOCK(ntb);
1654289546Scem}
1655289546Scem
1656289546Scem/*
1657289546Scem * ntb_db_event() - notify driver context of a doorbell event
1658289546Scem * @ntb:        NTB device context
1659289546Scem * @vector:     Interrupt vector number
1660289546Scem *
1661289546Scem * Notify the driver context of a doorbell event.  If hardware supports
1662289546Scem * multiple interrupt vectors for doorbells, the vector number indicates which
1663289546Scem * vector received the interrupt.  The vector number is relative to the first
1664289546Scem * vector used for doorbells, starting at zero, and must be less than
1665289546Scem * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
1666289546Scem * doorbell bits need service, and ntb_db_vector_mask() to determine which of
1667289546Scem * those bits are associated with the vector number.
1668289546Scem */
1669250079Scarlstatic void
1670289546Scemntb_db_event(struct ntb_softc *ntb, uint32_t vec)
1671289272Scem{
1672289546Scem
1673289546Scem	CTX_LOCK(ntb);
1674289546Scem	if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL)
1675289546Scem		ntb->ctx_ops->db_event(ntb->ntb_ctx, vec);
1676289546Scem	CTX_UNLOCK(ntb);
1677289546Scem}
1678289546Scem
1679289546Scem/*
1680289546Scem * ntb_link_enable() - enable the link on the secondary side of the ntb
1681289546Scem * @ntb:        NTB device context
1682289546Scem * @max_speed:  The maximum link speed expressed as PCIe generation number[0]
1683289546Scem * @max_width:  The maximum link width expressed as the number of PCIe lanes[0]
1684289546Scem *
1685289546Scem * Enable the link on the secondary side of the ntb.  This can only be done
1686289546Scem * from the primary side of the ntb in primary or b2b topology.  The ntb device
1687289546Scem * should train the link to its maximum speed and width, or the requested speed
1688289546Scem * and width, whichever is smaller, if supported.
1689289546Scem *
1690289546Scem * Return: Zero on success, otherwise an error number.
1691289546Scem *
1692289546Scem * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed
1693289546Scem *      and width input will be ignored.
1694289546Scem */
1695289546Scemint
1696289546Scemntb_link_enable(struct ntb_softc *ntb, enum ntb_speed s __unused,
1697289546Scem    enum ntb_width w __unused)
1698289546Scem{
1699289280Scem	uint32_t cntl;
1700289272Scem
1701289542Scem	if (ntb->type == NTB_SOC) {
1702289542Scem		pci_write_config(ntb->device, NTB_PPD_OFFSET,
1703289542Scem		    ntb->ppd | SOC_PPD_INIT_LINK, 4);
1704289546Scem		return (0);
1705289542Scem	}
1706289542Scem
1707289280Scem	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
1708289546Scem		ntb_link_event(ntb);
1709289546Scem		return (0);
1710289280Scem	}
1711289280Scem
1712289542Scem	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1713289280Scem	cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
1714289280Scem	cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
1715289397Scem	cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
1716289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1717289397Scem		cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP;
1718289542Scem	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
1719289546Scem	return (0);
1720289272Scem}
1721289272Scem
1722289546Scem/*
1723289546Scem * ntb_link_disable() - disable the link on the secondary side of the ntb
1724289546Scem * @ntb:        NTB device context
1725289546Scem *
1726289546Scem * Disable the link on the secondary side of the ntb.  This can only be done
1727289546Scem * from the primary side of the ntb in primary or b2b topology.  The ntb device
1728289546Scem * should disable the link.  Returning from this call must indicate that a
1729289546Scem * barrier has passed, though with no more writes may pass in either direction
1730289546Scem * across the link, except if this call returns an error number.
1731289546Scem *
1732289546Scem * Return: Zero on success, otherwise an error number.
1733289546Scem */
1734289546Scemint
1735289542Scemntb_link_disable(struct ntb_softc *ntb)
1736289272Scem{
1737289272Scem	uint32_t cntl;
1738289272Scem
1739289272Scem	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
1740289546Scem		ntb_link_event(ntb);
1741289546Scem		return (0);
1742289272Scem	}
1743289272Scem
1744289542Scem	cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1745289280Scem	cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
1746289397Scem	cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
1747289397Scem	if (HAS_FEATURE(NTB_SPLIT_BAR))
1748289397Scem		cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP);
1749289280Scem	cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
1750289542Scem	ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
1751289546Scem	return (0);
1752289272Scem}
1753289272Scem
1754289272Scemstatic void
1755250079Scarlrecover_soc_link(void *arg)
1756250079Scarl{
1757250079Scarl	struct ntb_softc *ntb = arg;
1758250079Scarl	uint8_t speed, width;
1759250079Scarl	uint32_t status32;
1760250079Scarl
1761250079Scarl	soc_perform_link_restart(ntb);
1762250079Scarl
1763289232Scem	/*
1764289232Scem	 * There is a potential race between the 2 NTB devices recovering at
1765289232Scem	 * the same time.  If the times are the same, the link will not recover
1766289232Scem	 * and the driver will be stuck in this loop forever.  Add a random
1767289232Scem	 * interval to the recovery time to prevent this race.
1768289232Scem	 */
1769289232Scem	status32 = arc4random() % SOC_LINK_RECOVERY_TIME;
1770289232Scem	pause("Link", (SOC_LINK_RECOVERY_TIME + status32) * hz / 1000);
1771289232Scem
1772255278Scarl	status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
1773250079Scarl	if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0)
1774250079Scarl		goto retry;
1775250079Scarl
1776255278Scarl	status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
1777250079Scarl	if ((status32 & SOC_IBIST_ERR_OFLOW) != 0)
1778250079Scarl		goto retry;
1779250079Scarl
1780289542Scem	status32 = ntb_reg_read(4, ntb->reg->ntb_ctl);
1781289232Scem	if ((status32 & SOC_CNTL_LINK_DOWN) != 0)
1782289232Scem		goto out;
1783289232Scem
1784289542Scem	status32 = ntb_reg_read(4, ntb->reg->lnk_sta);
1785289542Scem	width = (status32 & NTB_LINK_WIDTH_MASK) >> 4;
1786289542Scem	speed = (status32 & NTB_LINK_SPEED_MASK);
1787250079Scarl	if (ntb->link_width != width || ntb->link_speed != speed)
1788250079Scarl		goto retry;
1789250079Scarl
1790289232Scemout:
1791289542Scem	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb,
1792289542Scem	    ntb);
1793250079Scarl	return;
1794250079Scarl
1795250079Scarlretry:
1796250079Scarl	callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link,
1797250079Scarl	    ntb);
1798250079Scarl}
1799250079Scarl
1800289546Scem/*
1801289546Scem * Polls the HW link status register(s); returns true if something has changed.
1802289546Scem */
1803289546Scemstatic bool
1804289542Scemntb_poll_link(struct ntb_softc *ntb)
1805250079Scarl{
1806250079Scarl	uint32_t ntb_cntl;
1807289546Scem	uint16_t reg_val;
1808250079Scarl
1809250079Scarl	if (ntb->type == NTB_SOC) {
1810289542Scem		ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl);
1811289546Scem		if (ntb_cntl == ntb->ntb_ctl)
1812289546Scem			return (false);
1813289546Scem
1814289542Scem		ntb->ntb_ctl = ntb_cntl;
1815289542Scem		ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta);
1816250079Scarl	} else {
1817289546Scem		db_iowrite(ntb, ntb->reg_ofs.ldb, ntb->db_link_mask);
1818250079Scarl
1819289546Scem		reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
1820289546Scem		if (reg_val == ntb->lnk_sta)
1821289546Scem			return (false);
1822250079Scarl
1823289546Scem		ntb->lnk_sta = reg_val;
1824289542Scem	}
1825289546Scem	return (true);
1826289542Scem}
1827289542Scem
1828289546Scemstatic inline enum ntb_speed
1829289546Scemntb_link_sta_speed(struct ntb_softc *ntb)
1830250079Scarl{
1831250079Scarl
1832289546Scem	if (!link_is_up(ntb))
1833289546Scem		return (NTB_SPEED_NONE);
1834289546Scem	return (ntb->lnk_sta & NTB_LINK_SPEED_MASK);
1835250079Scarl}
1836250079Scarl
1837289546Scemstatic inline enum ntb_width
1838289546Scemntb_link_sta_width(struct ntb_softc *ntb)
1839250079Scarl{
1840250079Scarl
1841289546Scem	if (!link_is_up(ntb))
1842289546Scem		return (NTB_WIDTH_NONE);
1843289546Scem	return (NTB_LNK_STA_WIDTH(ntb->lnk_sta));
1844250079Scarl}
1845250079Scarl
1846289546Scem/*
1847289546Scem * Public API to the rest of the OS
1848250079Scarl */
1849250079Scarl
1850250079Scarl/**
1851250079Scarl * ntb_get_max_spads() - get the total scratch regs usable
1852250079Scarl * @ntb: pointer to ntb_softc instance
1853250079Scarl *
1854250079Scarl * This function returns the max 32bit scratchpad registers usable by the
1855250079Scarl * upper layer.
1856250079Scarl *
1857250079Scarl * RETURNS: total number of scratch pad registers available
1858250079Scarl */
1859289208Scemuint8_t
1860250079Scarlntb_get_max_spads(struct ntb_softc *ntb)
1861250079Scarl{
1862250079Scarl
1863289539Scem	return (ntb->spad_count);
1864250079Scarl}
1865250079Scarl
1866289396Scemuint8_t
1867289539Scemntb_mw_count(struct ntb_softc *ntb)
1868289396Scem{
1869289396Scem
1870289539Scem	return (ntb->mw_count);
1871289396Scem}
1872289396Scem
1873250079Scarl/**
1874289545Scem * ntb_spad_write() - write to the secondary scratchpad register
1875250079Scarl * @ntb: pointer to ntb_softc instance
1876250079Scarl * @idx: index to the scratchpad register, 0 based
1877250079Scarl * @val: the data value to put into the register
1878250079Scarl *
1879250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad
1880250079Scarl * register. The register resides on the secondary (external) side.
1881250079Scarl *
1882289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
1883250079Scarl */
1884250079Scarlint
1885289545Scemntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
1886250079Scarl{
1887250079Scarl
1888289539Scem	if (idx >= ntb->spad_count)
1889250079Scarl		return (EINVAL);
1890250079Scarl
1891255278Scarl	ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val);
1892250079Scarl
1893250079Scarl	return (0);
1894250079Scarl}
1895250079Scarl
1896250079Scarl/**
1897289545Scem * ntb_spad_read() - read from the primary scratchpad register
1898250079Scarl * @ntb: pointer to ntb_softc instance
1899250079Scarl * @idx: index to scratchpad register, 0 based
1900250079Scarl * @val: pointer to 32bit integer for storing the register value
1901250079Scarl *
1902250079Scarl * This function allows reading of the 32bit scratchpad register on
1903250079Scarl * the primary (internal) side.
1904250079Scarl *
1905289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
1906250079Scarl */
1907250079Scarlint
1908289545Scemntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
1909250079Scarl{
1910250079Scarl
1911289539Scem	if (idx >= ntb->spad_count)
1912250079Scarl		return (EINVAL);
1913250079Scarl
1914255278Scarl	*val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4);
1915250079Scarl
1916250079Scarl	return (0);
1917250079Scarl}
1918250079Scarl
1919250079Scarl/**
1920289545Scem * ntb_peer_spad_write() - write to the secondary scratchpad register
1921250079Scarl * @ntb: pointer to ntb_softc instance
1922250079Scarl * @idx: index to the scratchpad register, 0 based
1923250079Scarl * @val: the data value to put into the register
1924250079Scarl *
1925250079Scarl * This function allows writing of a 32bit value to the indexed scratchpad
1926250079Scarl * register. The register resides on the secondary (external) side.
1927250079Scarl *
1928289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
1929250079Scarl */
1930250079Scarlint
1931289545Scemntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
1932250079Scarl{
1933250079Scarl
1934289539Scem	if (idx >= ntb->spad_count)
1935250079Scarl		return (EINVAL);
1936250079Scarl
1937289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
1938255279Scarl		ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val);
1939255279Scarl	else
1940289542Scem		ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val);
1941250079Scarl
1942250079Scarl	return (0);
1943250079Scarl}
1944250079Scarl
1945250079Scarl/**
1946289545Scem * ntb_peer_spad_read() - read from the primary scratchpad register
1947250079Scarl * @ntb: pointer to ntb_softc instance
1948250079Scarl * @idx: index to scratchpad register, 0 based
1949250079Scarl * @val: pointer to 32bit integer for storing the register value
1950250079Scarl *
1951250079Scarl * This function allows reading of the 32bit scratchpad register on
1952250079Scarl * the primary (internal) side.
1953250079Scarl *
1954289209Scem * RETURNS: An appropriate ERRNO error value on error, or zero for success.
1955250079Scarl */
1956250079Scarlint
1957289545Scemntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
1958250079Scarl{
1959250079Scarl
1960289539Scem	if (idx >= ntb->spad_count)
1961250079Scarl		return (EINVAL);
1962250079Scarl
1963289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP))
1964255279Scarl		*val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4);
1965255279Scarl	else
1966289542Scem		*val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4);
1967250079Scarl
1968250079Scarl	return (0);
1969250079Scarl}
1970250079Scarl
1971289546Scem/*
1972289546Scem * ntb_mw_get_range() - get the range of a memory window
1973289546Scem * @ntb:        NTB device context
1974289546Scem * @idx:        Memory window number
1975289546Scem * @base:       OUT - the base address for mapping the memory window
1976289546Scem * @size:       OUT - the size for mapping the memory window
1977289546Scem * @align:      OUT - the base alignment for translating the memory window
1978289546Scem * @align_size: OUT - the size alignment for translating the memory window
1979250079Scarl *
1980289546Scem * Get the range of a memory window.  NULL may be given for any output
1981289546Scem * parameter if the value is not needed.  The base and size may be used for
1982289546Scem * mapping the memory window, to access the peer memory.  The alignment and
1983289546Scem * size may be used for translating the memory window, for the peer to access
1984289546Scem * memory on the local system.
1985250079Scarl *
1986289546Scem * Return: Zero on success, otherwise an error number.
1987250079Scarl */
1988289546Scemint
1989289546Scemntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base,
1990289546Scem    void **vbase, size_t *size, size_t *align, size_t *align_size)
1991250079Scarl{
1992289546Scem	struct ntb_pci_bar_info *bar;
1993289546Scem	size_t bar_b2b_off;
1994250079Scarl
1995289546Scem	if (mw_idx >= ntb_mw_count(ntb))
1996289546Scem		return (EINVAL);
1997250079Scarl
1998289546Scem	bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)];
1999289546Scem	bar_b2b_off = 0;
2000289546Scem	if (mw_idx == ntb->b2b_mw_idx) {
2001289546Scem		KASSERT(ntb->b2b_off != 0,
2002289546Scem		    ("user shouldn't get non-shared b2b mw"));
2003289546Scem		bar_b2b_off = ntb->b2b_off;
2004289546Scem	}
2005250079Scarl
2006289546Scem	if (base != NULL)
2007289546Scem		*base = bar->pbase + bar_b2b_off;
2008289546Scem	if (vbase != NULL)
2009289546Scem		*vbase = (char *)bar->vbase + bar_b2b_off;
2010289546Scem	if (size != NULL)
2011289546Scem		*size = bar->size - bar_b2b_off;
2012289546Scem	if (align != NULL)
2013289546Scem		*align = bar->size;
2014289546Scem	if (align_size != NULL)
2015289546Scem		*align_size = 1;
2016289546Scem	return (0);
2017250079Scarl}
2018250079Scarl
2019289546Scem/*
2020289546Scem * ntb_mw_set_trans() - set the translation of a memory window
2021289546Scem * @ntb:        NTB device context
2022289546Scem * @idx:        Memory window number
2023289546Scem * @addr:       The dma address local memory to expose to the peer
2024289546Scem * @size:       The size of the local memory to expose to the peer
2025250079Scarl *
2026289546Scem * Set the translation of a memory window.  The peer may access local memory
2027289546Scem * through the window starting at the address, up to the size.  The address
2028289546Scem * must be aligned to the alignment specified by ntb_mw_get_range().  The size
2029289546Scem * must be aligned to the size alignment specified by ntb_mw_get_range().
2030250079Scarl *
2031289546Scem * Return: Zero on success, otherwise an error number.
2032250079Scarl */
2033289546Scemint
2034289546Scemntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr,
2035289546Scem    size_t size)
2036250079Scarl{
2037289546Scem	struct ntb_pci_bar_info *bar;
2038289546Scem	uint64_t base, limit, reg_val;
2039289546Scem	size_t bar_size, mw_size;
2040289546Scem	uint32_t base_reg, xlat_reg, limit_reg;
2041289546Scem	enum ntb_bar bar_num;
2042250079Scarl
2043289546Scem	if (idx >= ntb_mw_count(ntb))
2044289546Scem		return (EINVAL);
2045250079Scarl
2046289546Scem	bar_num = ntb_mw_to_bar(ntb, idx);
2047289546Scem	bar = &ntb->bar_info[bar_num];
2048250079Scarl
2049289546Scem	bar_size = bar->size;
2050289546Scem	if (idx == ntb->b2b_mw_idx)
2051289546Scem		mw_size = bar_size - ntb->b2b_off;
2052289546Scem	else
2053289546Scem		mw_size = bar_size;
2054250079Scarl
2055289546Scem	/* Hardware requires that addr is aligned to bar size */
2056289546Scem	if ((addr & (bar_size - 1)) != 0)
2057289546Scem		return (EINVAL);
2058250079Scarl
2059289546Scem	if (size > mw_size)
2060289546Scem		return (EINVAL);
2061289546Scem
2062289546Scem	bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg);
2063289546Scem
2064289546Scem	limit = 0;
2065289546Scem	if (bar_is_64bit(ntb, bar_num)) {
2066289546Scem		base = ntb_reg_read(8, base_reg);
2067289546Scem
2068289546Scem		if (limit_reg != 0 && size != mw_size)
2069289546Scem			limit = base + size;
2070289546Scem
2071289546Scem		/* Set and verify translation address */
2072289546Scem		ntb_reg_write(8, xlat_reg, addr);
2073289546Scem		reg_val = ntb_reg_read(8, xlat_reg);
2074289546Scem		if (reg_val != addr) {
2075289546Scem			ntb_reg_write(8, xlat_reg, 0);
2076289546Scem			return (EIO);
2077289546Scem		}
2078289546Scem
2079289546Scem		/* Set and verify the limit */
2080289546Scem		ntb_reg_write(8, limit_reg, limit);
2081289546Scem		reg_val = ntb_reg_read(8, limit_reg);
2082289546Scem		if (reg_val != limit) {
2083289546Scem			ntb_reg_write(8, limit_reg, base);
2084289546Scem			ntb_reg_write(8, xlat_reg, 0);
2085289546Scem			return (EIO);
2086289546Scem		}
2087289546Scem	} else {
2088289546Scem		/* Configure 32-bit (split) BAR MW */
2089289546Scem
2090289546Scem		if ((addr & ~UINT32_MAX) != 0)
2091289546Scem			return (EINVAL);
2092289546Scem		if (((addr + size) & ~UINT32_MAX) != 0)
2093289546Scem			return (EINVAL);
2094289546Scem
2095289546Scem		base = ntb_reg_read(4, base_reg);
2096289546Scem
2097289546Scem		if (limit_reg != 0 && size != mw_size)
2098289546Scem			limit = base + size;
2099289546Scem
2100289546Scem		/* Set and verify translation address */
2101289546Scem		ntb_reg_write(4, xlat_reg, addr);
2102289546Scem		reg_val = ntb_reg_read(4, xlat_reg);
2103289546Scem		if (reg_val != addr) {
2104289546Scem			ntb_reg_write(4, xlat_reg, 0);
2105289546Scem			return (EIO);
2106289546Scem		}
2107289546Scem
2108289546Scem		/* Set and verify the limit */
2109289546Scem		ntb_reg_write(4, limit_reg, limit);
2110289546Scem		reg_val = ntb_reg_read(4, limit_reg);
2111289546Scem		if (reg_val != limit) {
2112289546Scem			ntb_reg_write(4, limit_reg, base);
2113289546Scem			ntb_reg_write(4, xlat_reg, 0);
2114289546Scem			return (EIO);
2115289546Scem		}
2116250079Scarl	}
2117289546Scem	return (0);
2118250079Scarl}
2119250079Scarl
2120289596Scem/*
2121289596Scem * ntb_mw_clear_trans() - clear the translation of a memory window
2122289596Scem * @ntb:	NTB device context
2123289596Scem * @idx:	Memory window number
2124289596Scem *
2125289596Scem * Clear the translation of a memory window.  The peer may no longer access
2126289596Scem * local memory through the window.
2127289596Scem *
2128289596Scem * Return: Zero on success, otherwise an error number.
2129289596Scem */
2130289596Scemint
2131289596Scemntb_mw_clear_trans(struct ntb_softc *ntb, unsigned mw_idx)
2132289596Scem{
2133289596Scem
2134289596Scem	return (ntb_mw_set_trans(ntb, mw_idx, 0, 0));
2135289596Scem}
2136289596Scem
2137250079Scarl/**
2138289545Scem * ntb_peer_db_set() - Set the doorbell on the secondary/external side
2139250079Scarl * @ntb: pointer to ntb_softc instance
2140289545Scem * @bit: doorbell bits to ring
2141250079Scarl *
2142250079Scarl * This function allows triggering of a doorbell on the secondary/external
2143250079Scarl * side that will initiate an interrupt on the remote host
2144250079Scarl */
2145250079Scarlvoid
2146289545Scemntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit)
2147250079Scarl{
2148250079Scarl
2149289538Scem	if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
2150289347Scem		ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET, bit);
2151289347Scem		return;
2152289209Scem	}
2153289347Scem
2154289546Scem	db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
2155250079Scarl}
2156250079Scarl
2157289542Scem/*
2158289542Scem * ntb_get_peer_db_addr() - Return the address of the remote doorbell register,
2159289542Scem * as well as the size of the register (via *sz_out).
2160289542Scem *
2161289542Scem * This function allows a caller using I/OAT DMA to chain the remote doorbell
2162289542Scem * ring to its memory window write.
2163289542Scem *
2164289542Scem * Note that writing the peer doorbell via a memory window will *not* generate
2165289542Scem * an interrupt on the remote host; that must be done seperately.
2166289542Scem */
2167289542Scembus_addr_t
2168289542Scemntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out)
2169289542Scem{
2170289542Scem	struct ntb_pci_bar_info *bar;
2171289542Scem	uint64_t regoff;
2172289542Scem
2173289542Scem	KASSERT(sz_out != NULL, ("must be non-NULL"));
2174289542Scem
2175289542Scem	if (!HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) {
2176289542Scem		bar = &ntb->bar_info[NTB_CONFIG_BAR];
2177289542Scem		regoff = ntb->peer_reg->db_bell;
2178289542Scem	} else {
2179289542Scem		KASSERT((HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 2) ||
2180289542Scem		    (!HAS_FEATURE(NTB_SPLIT_BAR) && ntb->mw_count == 1),
2181289542Scem		    ("mw_count invalid after setup"));
2182289543Scem		KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED,
2183289543Scem		    ("invalid b2b idx"));
2184289542Scem
2185289542Scem		bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)];
2186289542Scem		regoff = XEON_SHADOW_PDOORBELL_OFFSET;
2187289542Scem	}
2188289542Scem	KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh"));
2189289542Scem
2190289542Scem	*sz_out = ntb->reg->db_size;
2191289542Scem	/* HACK: Specific to current x86 bus implementation. */
2192289542Scem	return ((uint64_t)bar->pci_bus_handle + regoff);
2193289542Scem}
2194289542Scem
2195289597Scem/*
2196289597Scem * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb
2197289597Scem * @ntb:	NTB device context
2198289597Scem *
2199289597Scem * Hardware may support different number or arrangement of doorbell bits.
2200289597Scem *
2201289597Scem * Return: A mask of doorbell bits supported by the ntb.
2202289597Scem */
2203289597Scemuint64_t
2204289597Scemntb_db_valid_mask(struct ntb_softc *ntb)
2205289597Scem{
2206289597Scem
2207289597Scem	return (ntb->db_valid_mask);
2208289597Scem}
2209289597Scem
2210250079Scarl/**
2211289546Scem * ntb_link_is_up() - get the current ntb link state
2212289546Scem * @ntb:        NTB device context
2213289546Scem * @speed:      OUT - The link speed expressed as PCIe generation number
2214289546Scem * @width:      OUT - The link width expressed as the number of PCIe lanes
2215250079Scarl *
2216250079Scarl * RETURNS: true or false based on the hardware link state
2217250079Scarl */
2218250079Scarlbool
2219289546Scemntb_link_is_up(struct ntb_softc *ntb, enum ntb_speed *speed,
2220289546Scem    enum ntb_width *width)
2221250079Scarl{
2222250079Scarl
2223289546Scem	if (speed != NULL)
2224289546Scem		*speed = ntb_link_sta_speed(ntb);
2225289546Scem	if (width != NULL)
2226289546Scem		*width = ntb_link_sta_width(ntb);
2227289546Scem	return (link_is_up(ntb));
2228250079Scarl}
2229250079Scarl
2230255272Scarlstatic void
2231255272Scarlsave_bar_parameters(struct ntb_pci_bar_info *bar)
2232250079Scarl{
2233255272Scarl
2234289209Scem	bar->pci_bus_tag = rman_get_bustag(bar->pci_resource);
2235289209Scem	bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource);
2236289209Scem	bar->pbase = rman_get_start(bar->pci_resource);
2237289209Scem	bar->size = rman_get_size(bar->pci_resource);
2238289209Scem	bar->vbase = rman_get_virtual(bar->pci_resource);
2239250079Scarl}
2240255268Scarl
2241289209Scemdevice_t
2242289209Scemntb_get_device(struct ntb_softc *ntb)
2243255268Scarl{
2244255268Scarl
2245255268Scarl	return (ntb->device);
2246255268Scarl}
2247289208Scem
2248289208Scem/* Export HW-specific errata information. */
2249289208Scembool
2250289208Scemntb_has_feature(struct ntb_softc *ntb, uint64_t feature)
2251289208Scem{
2252289208Scem
2253289208Scem	return (HAS_FEATURE(feature));
2254289208Scem}
2255