xref: /freebsd-11-stable/sys/dev/ntb/ntb_hw/ntb_hw_intel.c

/*-
 * Copyright (C) 2013 Intel Corporation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD: head/sys/dev/ntb/ntb_hw/ntb_hw.c 255274 2013-09-05 23:00:59Z carl $");

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/queue.h>
#include <sys/rman.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/bus.h>
#include <machine/pmap.h>
#include <machine/resource.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#include "ntb_regs.h"
#include "ntb_hw.h"

/*
 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that
 * allows you to connect two systems using a PCI-e link.
 *
 * This module contains the hardware abstraction layer for the NTB. It allows
 * you to send and recieve interrupts, map the memory windows and send and
 * receive messages in the scratch-pad registers.
 *
 * NOTE: Much of the code in this module is shared with Linux. Any patches may
 * be picked up and redistributed in Linux with a dual GPL/BSD license.
 */

#define NTB_CONFIG_BAR	0
#define NTB_B2B_BAR_1	1
#define NTB_B2B_BAR_2	2
#define NTB_MAX_BARS	3
#define NTB_MW_TO_BAR(mw) ((mw) + 1)

#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS)

#define NTB_HB_TIMEOUT	1 /* second */
#define SOC_LINK_RECOVERY_TIME	500

#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev))

enum ntb_device_type {
	NTB_XEON,
	NTB_SOC
};

/* Device features and workarounds */
#define HAS_FEATURE(feature)	\
	((ntb->features & (feature)) != 0)

#define NTB_BAR_SIZE_4K		(1 << 0)
#define NTB_REGS_THRU_MW	(1 << 1)

struct ntb_hw_info {
	uint32_t		device_id;
	const char		*desc;
	enum ntb_device_type	type;
	uint64_t		features;
};

struct ntb_pci_bar_info {
	bus_space_tag_t		pci_bus_tag;
	bus_space_handle_t	pci_bus_handle;
	int			pci_resource_id;
	struct resource		*pci_resource;
	vm_paddr_t		pbase;
	void			*vbase;
	u_long			size;
};

struct ntb_int_info {
	struct resource	*res;
	int		rid;
	void		*tag;
};

struct ntb_db_cb {
	ntb_db_callback		callback;
	unsigned int		db_num;
	void			*data;
	struct ntb_softc	*ntb;
};

struct ntb_softc {
	device_t		device;
	enum ntb_device_type	type;
	uint64_t		features;

	struct ntb_pci_bar_info	bar_info[NTB_MAX_BARS];
	struct ntb_int_info	int_info[MAX_MSIX_INTERRUPTS];
	uint32_t		allocated_interrupts;

	struct callout		heartbeat_timer;
	struct callout		lr_timer;

	void			*ntb_transport;
	ntb_event_callback	event_cb;
	struct ntb_db_cb 	*db_cb;

	struct {
		uint32_t max_spads;
		uint32_t max_db_bits;
		uint32_t msix_cnt;
	} limits;
	struct {
		uint32_t pdb;
		uint32_t pdb_mask;
		uint32_t sdb;
		uint32_t sbar2_xlat;
		uint32_t sbar4_xlat;
		uint32_t spad_remote;
		uint32_t spad_local;
		uint32_t lnk_cntl;
		uint32_t lnk_stat;
		uint32_t spci_cmd;
	} reg_ofs;
	uint8_t conn_type;
	uint8_t dev_type;
	uint8_t bits_per_vector;
	uint8_t link_status;
	uint8_t link_width;
	uint8_t link_speed;
};

#define ntb_reg_read(SIZE, offset) \
	bus_space_read_ ## SIZE (ntb->bar_info[NTB_CONFIG_BAR].pci_bus_tag, \
	    ntb->bar_info[NTB_CONFIG_BAR].pci_bus_handle, (offset))
#define ntb_reg_write(SIZE, offset, val) \
	bus_space_write_ ## SIZE (ntb->bar_info[NTB_CONFIG_BAR].pci_bus_tag, \
	    ntb->bar_info[NTB_CONFIG_BAR].pci_bus_handle, (offset), (val))

#define ntb_read_1(offset) ntb_reg_read(1, (offset))
#define ntb_read_2(offset) ntb_reg_read(2, (offset))
#define ntb_read_4(offset) ntb_reg_read(4, (offset))
#define ntb_read_8(offset) ntb_reg_read(8, (offset))
#define ntb_write_1(offset, val) ntb_reg_write(1, (offset), (val))
#define ntb_write_2(offset, val) ntb_reg_write(2, (offset), (val))
#define ntb_write_4(offset, val) ntb_reg_write(4, (offset), (val))
#define ntb_write_8(offset, val) ntb_reg_write(8, (offset), (val))

typedef int (*bar_map_strategy)(struct ntb_softc *ntb,
    struct ntb_pci_bar_info *bar);

static int ntb_probe(device_t device);
static int ntb_attach(device_t device);
static int ntb_detach(device_t device);
static int ntb_map_pci_bars(struct ntb_softc *ntb);
static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
    struct ntb_pci_bar_info *bar);
static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
static int map_memory_window_bar(struct ntb_softc *ntb,
    struct ntb_pci_bar_info *bar);
static void ntb_unmap_pci_bar(struct ntb_softc *ntb);
static int ntb_setup_interrupts(struct ntb_softc *ntb);
static void ntb_teardown_interrupts(struct ntb_softc *ntb);
static void handle_soc_irq(void *arg);
static void handle_xeon_irq(void *arg);
static void handle_xeon_event_irq(void *arg);
static void ntb_handle_legacy_interrupt(void *arg);
static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors);
static void ntb_free_callbacks(struct ntb_softc *ntb);
static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
static int ntb_initialize_hw(struct ntb_softc *ntb);
static int ntb_setup_xeon(struct ntb_softc *ntb);
static int ntb_setup_soc(struct ntb_softc *ntb);
static void ntb_handle_heartbeat(void *arg);
static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state);
static void recover_soc_link(void *arg);
static int ntb_check_link_status(struct ntb_softc *ntb);
static void save_bar_parameters(struct ntb_pci_bar_info *bar);

static struct ntb_hw_info pci_ids[] = {
	{ 0x3C0D8086, "Xeon E5/Core i7 Non-Transparent Bridge B2B", NTB_XEON,
	    NTB_REGS_THRU_MW },
	{ 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 },
	{ 0x0E0D8086, "Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
	    NTB_REGS_THRU_MW | NTB_BAR_SIZE_4K },
	{ 0x00000000, NULL, NTB_SOC, 0 }
};

/*
 * OS <-> Driver interface structures
 */
MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");

static device_method_t ntb_pci_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe,     ntb_probe),
	DEVMETHOD(device_attach,    ntb_attach),
	DEVMETHOD(device_detach,    ntb_detach),
	DEVMETHOD_END
};

static driver_t ntb_pci_driver = {
	"ntb_hw",
	ntb_pci_methods,
	sizeof(struct ntb_softc),
};

static devclass_t ntb_devclass;
DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL);
MODULE_VERSION(ntb_hw, 1);

/*
 * OS <-> Driver linkage functions
 */
static int
ntb_probe(device_t device)
{
	struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device));

	if (p != NULL) {
		device_set_desc(device, p->desc);
		return (0);
	} else
		return (ENXIO);
}

#define DETACH_ON_ERROR(func)           \
	error = func;		        \
	if (error < 0) {		\
		ntb_detach(device);	\
		return (error);		\
	}

static int
ntb_attach(device_t device)
{
	struct ntb_softc *ntb = DEVICE2SOFTC(device);
	struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device));
	int error;

	ntb->device = device;
	ntb->type = p->type;
	ntb->features = p->features;

	/* Heartbeat timer for NTB_SOC since there is no link interrupt */
	callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE);
	callout_init(&ntb->lr_timer, CALLOUT_MPSAFE);

	DETACH_ON_ERROR(ntb_map_pci_bars(ntb));
	DETACH_ON_ERROR(ntb_initialize_hw(ntb));
	DETACH_ON_ERROR(ntb_setup_interrupts(ntb));

	pci_enable_busmaster(ntb->device);

	return (error);
}

static int
ntb_detach(device_t device)
{
	struct ntb_softc *ntb = DEVICE2SOFTC(device);

	callout_drain(&ntb->heartbeat_timer);
	callout_drain(&ntb->lr_timer);
	ntb_teardown_interrupts(ntb);
	ntb_unmap_pci_bar(ntb);

	return (0);
}

static int
ntb_map_pci_bars(struct ntb_softc *ntb)
{
	int rc;

	ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0);
	rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]);
	if (rc != 0)
		return rc;

	ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id  = PCIR_BAR(2);
	rc = map_pci_bar(ntb, map_memory_window_bar,
	    &ntb->bar_info[NTB_B2B_BAR_1]);
	if (rc != 0)
		return rc;

	ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id  = PCIR_BAR(4);
	rc = map_pci_bar(ntb, map_memory_window_bar,
	    &ntb->bar_info[NTB_B2B_BAR_2]);
	if (rc != 0)
		return rc;

	return (0);
}

static int
map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
    struct ntb_pci_bar_info *bar)
{
	int rc;

	rc = strategy(ntb, bar);
	if (rc != 0) {
		device_printf(ntb->device,
		    "unable to allocate pci resource\n");
	} else {
		device_printf(ntb->device,
		    "Bar size = %lx, v %p, p %p\n",
		    bar->size, bar->vbase,
		    (void *)(bar->pbase));
	}
	return (rc);
}

static int
map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
{

	bar->pci_resource = bus_alloc_resource(ntb->device, SYS_RES_MEMORY,
		&bar->pci_resource_id, 0, ~0, 1, RF_ACTIVE);

	if (bar->pci_resource == NULL)
		return (ENXIO);
	else {
		save_bar_parameters(bar);
		return (0);
	}
}

static int
map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
{
	int rc;

	bar->pci_resource = bus_alloc_resource(ntb->device,
		SYS_RES_MEMORY, &bar->pci_resource_id, 0, ~0, 1,
			RF_ACTIVE);

	if (bar->pci_resource == NULL)
		return (ENXIO);
	else {
		save_bar_parameters(bar);
		/* Mark bar region as write combining to improve performance. */
		rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size,
		    VM_MEMATTR_WRITE_COMBINING);
		if (rc != 0) {
			device_printf(ntb->device, "unable to mark bar as"
			    " WRITE_COMBINING\n");
			return (rc);
		}
	}
	return (0);
}

static void
ntb_unmap_pci_bar(struct ntb_softc *ntb)
{
	struct ntb_pci_bar_info *current_bar;
	int i;

	for (i = 0; i< NTB_MAX_BARS; i++) {
		current_bar = &ntb->bar_info[i];
		if (current_bar->pci_resource != NULL)
			bus_release_resource(ntb->device, SYS_RES_MEMORY,
			    current_bar->pci_resource_id,
			    current_bar->pci_resource);
	}
}

static int
ntb_setup_interrupts(struct ntb_softc *ntb)
{
	void (*interrupt_handler)(void *);
	void *int_arg;
	bool use_msix = 0;
	uint32_t num_vectors;
	int i;

	ntb->allocated_interrupts = 0;
	/*
	 * On SOC, disable all interrupts.  On XEON, disable all but Link
	 * Interrupt.  The rest will be unmasked as callbacks are registered.
	 */
	if (ntb->type == NTB_SOC)
		ntb_write_8(ntb->reg_ofs.pdb_mask, ~0);
	else
		ntb_write_2(ntb->reg_ofs.pdb_mask,
		    ~(1 << ntb->limits.max_db_bits));

	num_vectors = MIN(pci_msix_count(ntb->device),
	    ntb->limits.max_db_bits);
	if (num_vectors >= 1) {
		pci_alloc_msix(ntb->device, &num_vectors);
		if (num_vectors >= 4)
			use_msix = TRUE;
	}

	ntb_create_callbacks(ntb, num_vectors);
	if (use_msix == TRUE) {
		for (i = 0; i < num_vectors; i++) {
			ntb->int_info[i].rid = i + 1;
			ntb->int_info[i].res = bus_alloc_resource_any(
			    ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid,
			    RF_ACTIVE);
			if (ntb->int_info[i].res == NULL) {
				device_printf(ntb->device,
				    "bus_alloc_resource failed\n");
				return (-1);
			}
			ntb->int_info[i].tag = NULL;
			ntb->allocated_interrupts++;
			if (ntb->type == NTB_SOC) {
				interrupt_handler = handle_soc_irq;
				int_arg = &ntb->db_cb[i];
			} else {
				if (i == num_vectors - 1) {
					interrupt_handler = handle_xeon_event_irq;
					int_arg = ntb;
				} else {
					interrupt_handler =
					    handle_xeon_irq;
					int_arg = &ntb->db_cb[i];
				}
			}
			if (bus_setup_intr(ntb->device, ntb->int_info[i].res,
			    INTR_MPSAFE | INTR_TYPE_MISC, NULL,
			    interrupt_handler, int_arg,
			    &ntb->int_info[i].tag) != 0) {
				device_printf(ntb->device,
				    "bus_setup_intr failed\n");
				return (ENXIO);
			}
		}
	}
	else {
		ntb->int_info[0].rid = 0;
		ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ,
		    &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
		interrupt_handler = ntb_handle_legacy_interrupt;
		if (ntb->int_info[0].res == NULL) {
			device_printf(ntb->device,
			    "bus_alloc_resource failed\n");
			return (-1);
		}
		ntb->int_info[0].tag = NULL;
		ntb->allocated_interrupts = 1;

		if (bus_setup_intr(ntb->device, ntb->int_info[0].res,
			INTR_MPSAFE | INTR_TYPE_MISC, NULL,
			interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) {

			device_printf(ntb->device, "bus_setup_intr failed\n");
			return (ENXIO);
		}
	}

	return (0);
}

static void
ntb_teardown_interrupts(struct ntb_softc *ntb)
{
	struct ntb_int_info *current_int;
	int i;

	for (i=0; i<ntb->allocated_interrupts; i++) {
		current_int = &ntb->int_info[i];
		if (current_int->tag != NULL)
			bus_teardown_intr(ntb->device, current_int->res,
			    current_int->tag);

		if (current_int->res != NULL)
			bus_release_resource(ntb->device, SYS_RES_IRQ,
			    rman_get_rid(current_int->res), current_int->res);
	}

	ntb_free_callbacks(ntb);
	pci_release_msi(ntb->device);
}

static void
handle_soc_irq(void *arg)
{
	struct ntb_db_cb *db_cb = arg;
	struct ntb_softc *ntb = db_cb->ntb;

	ntb_write_8(ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num);

	if (db_cb->callback != NULL)
		db_cb->callback(db_cb->data, db_cb->db_num);
}

static void
handle_xeon_irq(void *arg)
{
	struct ntb_db_cb *db_cb = arg;
	struct ntb_softc *ntb = db_cb->ntb;

	/*
	 * On Xeon, there are 16 bits in the interrupt register
	 * but only 4 vectors.  So, 5 bits are assigned to the first 3
	 * vectors, with the 4th having a single bit for link
	 * interrupts.
	 */
	ntb_write_2(ntb->reg_ofs.pdb,
	    ((1 << ntb->bits_per_vector) - 1) <<
	    (db_cb->db_num * ntb->bits_per_vector));

	if (db_cb->callback != NULL)
		db_cb->callback(db_cb->data, db_cb->db_num);
}

/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */
static void
handle_xeon_event_irq(void *arg)
{
	struct ntb_softc *ntb = arg;
	int rc;

	rc = ntb_check_link_status(ntb);
	if (rc != 0)
		device_printf(ntb->device, "Error determining link status\n");

	/* bit 15 is always the link bit */
	ntb_write_2(ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits);
}

static void
ntb_handle_legacy_interrupt(void *arg)
{
	struct ntb_softc *ntb = arg;
	unsigned int i = 0;
	uint64_t pdb64;
	uint16_t pdb16;

	if (ntb->type == NTB_SOC) {
		pdb64 = ntb_read_8(ntb->reg_ofs.pdb);

		while (pdb64) {
			i = ffs(pdb64);
			pdb64 &= pdb64 - 1;
			handle_soc_irq(&ntb->db_cb[i]);
		}
	} else {
		pdb16 = ntb_read_2(ntb->reg_ofs.pdb);

		if ((pdb16 & XEON_DB_HW_LINK) != 0) {
			handle_xeon_event_irq(ntb);
			pdb16 &= ~XEON_DB_HW_LINK;
		}

		while (pdb16 != 0) {
			i = ffs(pdb16);
			pdb16 &= pdb16 - 1;
			handle_xeon_irq(&ntb->db_cb[i]);
		}
	}

}

static int
ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors)
{
	int i;

	ntb->db_cb = malloc(num_vectors * sizeof(struct ntb_db_cb), M_NTB,
	    M_ZERO | M_WAITOK);
	for (i = 0; i < num_vectors; i++) {
		ntb->db_cb[i].db_num = i;
		ntb->db_cb[i].ntb = ntb;
	}

	return (0);
}

static void
ntb_free_callbacks(struct ntb_softc *ntb)
{
	int i;

	for (i = 0; i < ntb->limits.max_db_bits; i++)
		ntb_unregister_db_callback(ntb, i);

	free(ntb->db_cb, M_NTB);
}

static struct ntb_hw_info *
ntb_get_device_info(uint32_t device_id)
{
	struct ntb_hw_info *ep = pci_ids;

	while (ep->device_id) {
		if (ep->device_id == device_id)
			return (ep);
		++ep;
	}
	return (NULL);
}

static int
ntb_initialize_hw(struct ntb_softc *ntb)
{

	if (ntb->type == NTB_SOC)
		return (ntb_setup_soc(ntb));
	else
		return (ntb_setup_xeon(ntb));
}

static int
ntb_setup_xeon(struct ntb_softc *ntb)
{
	uint8_t val, connection_type;

	val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);

	connection_type = val & XEON_PPD_CONN_TYPE;
	switch (connection_type) {
	case NTB_CONN_B2B:
		ntb->conn_type = NTB_CONN_B2B;
		break;
	case NTB_CONN_CLASSIC:
	case NTB_CONN_RP:
	default:
		device_printf(ntb->device, "Connection type %d not supported\n",
		    connection_type);
		return (ENXIO);
	}

	if ((val & XEON_PPD_DEV_TYPE) != 0)
		ntb->dev_type = NTB_DEV_DSD;
	else
		ntb->dev_type = NTB_DEV_USD;

	ntb->reg_ofs.pdb	= XEON_PDOORBELL_OFFSET;
	ntb->reg_ofs.pdb_mask	= XEON_PDBMSK_OFFSET;
	ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET;
	ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET;
	ntb->reg_ofs.lnk_cntl	= XEON_NTBCNTL_OFFSET;
	ntb->reg_ofs.lnk_stat	= XEON_LINK_STATUS_OFFSET;
	ntb->reg_ofs.spad_local	= XEON_SPAD_OFFSET;
	ntb->reg_ofs.spci_cmd	= XEON_PCICMD_OFFSET;

	if (ntb->conn_type == NTB_CONN_B2B) {
		ntb->reg_ofs.sdb	 = XEON_B2B_DOORBELL_OFFSET;
		ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
		ntb->limits.max_spads	 = XEON_MAX_SPADS;
	} else {
		ntb->reg_ofs.sdb	 = XEON_SDOORBELL_OFFSET;
		ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET;
		ntb->limits.max_spads	 = XEON_MAX_COMPAT_SPADS;
	}

	ntb->limits.max_db_bits  = XEON_MAX_DB_BITS;
	ntb->limits.msix_cnt	 = XEON_MSIX_CNT;
	ntb->bits_per_vector	 = XEON_DB_BITS_PER_VEC;

	/* Enable Bus Master and Memory Space on the secondary side */
	ntb_write_2(ntb->reg_ofs.spci_cmd,
	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);

	/* Enable link training */
	ntb_write_4(ntb->reg_ofs.lnk_cntl,
	    NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);

	return (0);
}

static int
ntb_setup_soc(struct ntb_softc *ntb)
{
	uint32_t val, connection_type;

	val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);

	connection_type = (val & SOC_PPD_CONN_TYPE) >> 8;
	switch (connection_type) {
	case NTB_CONN_B2B:
		ntb->conn_type = NTB_CONN_B2B;
		break;
	case NTB_CONN_RP:
	default:
		device_printf(ntb->device, "Connection type %d not supported\n",
		    connection_type);
		return (ENXIO);
	}

	if ((val & SOC_PPD_DEV_TYPE) != 0)
		ntb->dev_type = NTB_DEV_DSD;
	else
		ntb->dev_type = NTB_DEV_USD;

	/* Initiate PCI-E link training */
	pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK,
	    4);

	ntb->reg_ofs.pdb	 = SOC_PDOORBELL_OFFSET;
	ntb->reg_ofs.pdb_mask	 = SOC_PDBMSK_OFFSET;
	ntb->reg_ofs.sbar2_xlat  = SOC_SBAR2XLAT_OFFSET;
	ntb->reg_ofs.sbar4_xlat  = SOC_SBAR4XLAT_OFFSET;
	ntb->reg_ofs.lnk_cntl	 = SOC_NTBCNTL_OFFSET;
	ntb->reg_ofs.lnk_stat	 = SOC_LINK_STATUS_OFFSET;
	ntb->reg_ofs.spad_local	 = SOC_SPAD_OFFSET;
	ntb->reg_ofs.spci_cmd	 = SOC_PCICMD_OFFSET;

	if (ntb->conn_type == NTB_CONN_B2B) {
		ntb->reg_ofs.sdb	 = SOC_B2B_DOORBELL_OFFSET;
		ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
		ntb->limits.max_spads	 = SOC_MAX_SPADS;
	} else {
		ntb->reg_ofs.sdb	 = SOC_PDOORBELL_OFFSET;
		ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET;
		ntb->limits.max_spads	 = SOC_MAX_COMPAT_SPADS;
	}

	ntb->limits.max_db_bits  = SOC_MAX_DB_BITS;
	ntb->limits.msix_cnt	 = SOC_MSIX_CNT;
	ntb->bits_per_vector	 = SOC_DB_BITS_PER_VEC;

	/*
	 * FIXME - MSI-X bug on early SOC HW, remove once internal issue is
	 * resolved.  Mask transaction layer internal parity errors.
	 */
	pci_write_config(ntb->device, 0xFC, 0x4, 4);

	/*
	 * Some BIOSes aren't filling out the XLAT offsets.
	 * Check and correct the issue.
	 */
	if (ntb->dev_type == NTB_DEV_USD) {
		if (ntb_read_8(SOC_PBAR2XLAT_OFFSET) == 0)
			ntb_write_8(SOC_PBAR2XLAT_OFFSET,
			    SOC_PBAR2XLAT_USD_ADDR);

		if (ntb_read_8(SOC_PBAR4XLAT_OFFSET) == 0)
			ntb_write_8(SOC_PBAR4XLAT_OFFSET,
			    SOC_PBAR4XLAT_USD_ADDR);

		if (ntb_read_8(SOC_MBAR23_OFFSET) == 0xC)
			ntb_write_8(SOC_MBAR23_OFFSET, SOC_MBAR23_USD_ADDR);

		if (ntb_read_8(SOC_MBAR45_OFFSET) == 0xC)
			ntb_write_8(SOC_MBAR45_OFFSET, SOC_MBAR45_USD_ADDR);
	} else {
		if (ntb_read_8(SOC_PBAR2XLAT_OFFSET) == 0)
			ntb_write_8(SOC_PBAR2XLAT_OFFSET,
			    SOC_PBAR2XLAT_DSD_ADDR);

		if (ntb_read_8(SOC_PBAR4XLAT_OFFSET) == 0)
			ntb_write_8(SOC_PBAR4XLAT_OFFSET,
			    SOC_PBAR4XLAT_DSD_ADDR);

		if (ntb_read_8(SOC_MBAR23_OFFSET) == 0xC)
			ntb_write_8(SOC_MBAR23_OFFSET, SOC_MBAR23_DSD_ADDR);

		if (ntb_read_8(SOC_MBAR45_OFFSET) == 0xC)
			ntb_write_8(SOC_MBAR45_OFFSET, SOC_MBAR45_DSD_ADDR);
	}

	/* Enable Bus Master and Memory Space on the secondary side */
	ntb_write_2(ntb->reg_ofs.spci_cmd,
	    PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
	callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb);

	return (0);
}

/* SOC doesn't have link status interrupt, poll on that platform */
static void
ntb_handle_heartbeat(void *arg)
{
	struct ntb_softc *ntb = arg;
	uint32_t status32;
	int rc = ntb_check_link_status(ntb);

	if (rc != 0)
		device_printf(ntb->device,
		    "Error determining link status\n");
	/* Check to see if a link error is the cause of the link down */
	if (ntb->link_status == NTB_LINK_DOWN) {
		status32 = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET);
		if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) {
			callout_reset(&ntb->lr_timer, 0, recover_soc_link,
			    ntb);
			return;
		}
	}

	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
	    ntb_handle_heartbeat, ntb);
}

static void
soc_perform_link_restart(struct ntb_softc *ntb)
{
	uint32_t status;

	/* Driver resets the NTB ModPhy lanes - magic! */
	ntb_write_1(SOC_MODPHY_PCSREG6, 0xe0);
	ntb_write_1(SOC_MODPHY_PCSREG4, 0x40);
	ntb_write_1(SOC_MODPHY_PCSREG4, 0x60);
	ntb_write_1(SOC_MODPHY_PCSREG6, 0x60);

	/* Driver waits 100ms to allow the NTB ModPhy to settle */
	pause("ModPhy", hz / 10);

	/* Clear AER Errors, write to clear */
	status = ntb_read_4(SOC_ERRCORSTS_OFFSET);
	status &= PCIM_AER_COR_REPLAY_ROLLOVER;
	ntb_write_4(SOC_ERRCORSTS_OFFSET, status);

	/* Clear unexpected electrical idle event in LTSSM, write to clear */
	status = ntb_read_4(SOC_LTSSMERRSTS0_OFFSET);
	status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI;
	ntb_write_4(SOC_LTSSMERRSTS0_OFFSET, status);

	/* Clear DeSkew Buffer error, write to clear */
	status = ntb_read_4(SOC_DESKEWSTS_OFFSET);
	status |= SOC_DESKEWSTS_DBERR;
	ntb_write_4(SOC_DESKEWSTS_OFFSET, status);

	status = ntb_read_4(SOC_IBSTERRRCRVSTS0_OFFSET);
	status &= SOC_IBIST_ERR_OFLOW;
	ntb_write_4(SOC_IBSTERRRCRVSTS0_OFFSET, status);

	/* Releases the NTB state machine to allow the link to retrain */
	status = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET);
	status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT;
	ntb_write_4(SOC_LTSSMSTATEJMP_OFFSET, status);
}

static void
ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
{
	enum ntb_hw_event event;
	uint16_t status;

	if (ntb->link_status == link_state)
		return;

	if (link_state == NTB_LINK_UP) {
		device_printf(ntb->device, "Link Up\n");
		ntb->link_status = NTB_LINK_UP;
		event = NTB_EVENT_HW_LINK_UP;

		if (ntb->type == NTB_SOC)
			status = ntb_read_2(ntb->reg_ofs.lnk_stat);
		else
			status = pci_read_config(ntb->device,
			    XEON_LINK_STATUS_OFFSET, 2);
		ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
		ntb->link_speed = (status & NTB_LINK_SPEED_MASK);
		device_printf(ntb->device, "Link Width %d, Link Speed %d\n",
		    ntb->link_width, ntb->link_speed);
		callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
		    ntb_handle_heartbeat, ntb);
	} else {
		device_printf(ntb->device, "Link Down\n");
		ntb->link_status = NTB_LINK_DOWN;
		event = NTB_EVENT_HW_LINK_DOWN;
		/* Don't modify link width/speed, we need it in link recovery */
	}

	/* notify the upper layer if we have an event change */
	if (ntb->event_cb != NULL)
		ntb->event_cb(ntb->ntb_transport, event);
}

static void
recover_soc_link(void *arg)
{
	struct ntb_softc *ntb = arg;
	uint8_t speed, width;
	uint32_t status32;
	uint16_t status16;

	soc_perform_link_restart(ntb);
	pause("Link", SOC_LINK_RECOVERY_TIME * hz / 1000);

	status32 = ntb_read_4(SOC_LTSSMSTATEJMP_OFFSET);
	if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0)
		goto retry;

	status32 = ntb_read_4(SOC_IBSTERRRCRVSTS0_OFFSET);
	if ((status32 & SOC_IBIST_ERR_OFLOW) != 0)
		goto retry;

	status16 = ntb_read_2(ntb->reg_ofs.lnk_stat);
	width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
	speed = (status16 & NTB_LINK_SPEED_MASK);
	if (ntb->link_width != width || ntb->link_speed != speed)
		goto retry;

	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
	    ntb_handle_heartbeat, ntb);
	return;

retry:
	callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link,
	    ntb);
}

static int
ntb_check_link_status(struct ntb_softc *ntb)
{
	int link_state;
	uint32_t ntb_cntl;
	uint16_t status;

	if (ntb->type == NTB_SOC) {
		ntb_cntl = ntb_read_4(ntb->reg_ofs.lnk_cntl);
		if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0)
			link_state = NTB_LINK_DOWN;
		else
			link_state = NTB_LINK_UP;
	} else {
		status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET,
		    2);

		if ((status & NTB_LINK_STATUS_ACTIVE) != 0)
			link_state = NTB_LINK_UP;
		else
			link_state = NTB_LINK_DOWN;
	}

	ntb_handle_link_event(ntb, link_state);

	return (0);
}

/**
 * ntb_register_event_callback() - register event callback
 * @ntb: pointer to ntb_softc instance
 * @func: callback function to register
 *
 * This function registers a callback for any HW driver events such as link
 * up/down, power management notices and etc.
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
int
ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func)
{

	if (ntb->event_cb != NULL)
		return (EINVAL);

	ntb->event_cb = func;

	return (0);
}

/**
 * ntb_unregister_event_callback() - unregisters the event callback
 * @ntb: pointer to ntb_softc instance
 *
 * This function unregisters the existing callback from transport
 */
void
ntb_unregister_event_callback(struct ntb_softc *ntb)
{

	ntb->event_cb = NULL;
}

/**
 * ntb_register_db_callback() - register a callback for doorbell interrupt
 * @ntb: pointer to ntb_softc instance
 * @idx: doorbell index to register callback, zero based
 * @func: callback function to register
 *
 * This function registers a callback function for the doorbell interrupt
 * on the primary side. The function will unmask the doorbell as well to
 * allow interrupt.
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
int
ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
    ntb_db_callback func)
{
	uint16_t mask;

	if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) {
		device_printf(ntb->device, "Invalid Index.\n");
		return (EINVAL);
	}

	ntb->db_cb[idx].callback = func;
	ntb->db_cb[idx].data = data;

	/* unmask interrupt */
	mask = ntb_read_2(ntb->reg_ofs.pdb_mask);
	mask &= ~(1 << (idx * ntb->bits_per_vector));
	ntb_write_2(ntb->reg_ofs.pdb_mask, mask);

	return (0);
}

/**
 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
 * @ntb: pointer to ntb_softc instance
 * @idx: doorbell index to register callback, zero based
 *
 * This function unregisters a callback function for the doorbell interrupt
 * on the primary side. The function will also mask the said doorbell.
 */
void
ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx)
{
	unsigned long mask;

	if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback)
		return;

	mask = ntb_read_2(ntb->reg_ofs.pdb_mask);
	mask |= 1 << (idx * ntb->bits_per_vector);
	ntb_write_2(ntb->reg_ofs.pdb_mask, mask);

	ntb->db_cb[idx].callback = NULL;
}

/**
 * ntb_find_transport() - find the transport pointer
 * @transport: pointer to pci device
 *
 * Given the pci device pointer, return the transport pointer passed in when
 * the transport attached when it was inited.
 *
 * RETURNS: pointer to transport.
 */
void *
ntb_find_transport(struct ntb_softc *ntb)
{

	return (ntb->ntb_transport);
}

/**
 * ntb_register_transport() - Register NTB transport with NTB HW driver
 * @transport: transport identifier
 *
 * This function allows a transport to reserve the hardware driver for
 * NTB usage.
 *
 * RETURNS: pointer to ntb_softc, NULL on error.
 */
struct ntb_softc *
ntb_register_transport(struct ntb_softc *ntb, void *transport)
{

	/*
	 * TODO: when we have more than one transport, we will need to rewrite
	 * this to prevent race conditions
	 */
	if (ntb->ntb_transport != NULL)
		return (NULL);

	ntb->ntb_transport = transport;
	return (ntb);
}

/**
 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
 * @ntb - ntb_softc of the transport to be freed
 *
 * This function unregisters the transport from the HW driver and performs any
 * necessary cleanups.
 */
void
ntb_unregister_transport(struct ntb_softc *ntb)
{
	int i;

	if (ntb->ntb_transport == NULL)
		return;

	for (i = 0; i < ntb->allocated_interrupts; i++)
		ntb_unregister_db_callback(ntb, i);

	ntb_unregister_event_callback(ntb);
	ntb->ntb_transport = NULL;
}

/**
 * ntb_get_max_spads() - get the total scratch regs usable
 * @ntb: pointer to ntb_softc instance
 *
 * This function returns the max 32bit scratchpad registers usable by the
 * upper layer.
 *
 * RETURNS: total number of scratch pad registers available
 */
int
ntb_get_max_spads(struct ntb_softc *ntb)
{

	return (ntb->limits.max_spads);
}

/**
 * ntb_write_local_spad() - write to the secondary scratchpad register
 * @ntb: pointer to ntb_softc instance
 * @idx: index to the scratchpad register, 0 based
 * @val: the data value to put into the register
 *
 * This function allows writing of a 32bit value to the indexed scratchpad
 * register. The register resides on the secondary (external) side.
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
int
ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
{

	if (idx >= ntb->limits.max_spads)
		return (EINVAL);

	ntb_write_4(ntb->reg_ofs.spad_local + idx * 4, val);

	return (0);
}

/**
 * ntb_read_local_spad() - read from the primary scratchpad register
 * @ntb: pointer to ntb_softc instance
 * @idx: index to scratchpad register, 0 based
 * @val: pointer to 32bit integer for storing the register value
 *
 * This function allows reading of the 32bit scratchpad register on
 * the primary (internal) side.
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
int
ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
{

	if (idx >= ntb->limits.max_spads)
		return (EINVAL);

	*val = ntb_read_4(ntb->reg_ofs.spad_local + idx * 4);

	return (0);
}

/**
 * ntb_write_remote_spad() - write to the secondary scratchpad register
 * @ntb: pointer to ntb_softc instance
 * @idx: index to the scratchpad register, 0 based
 * @val: the data value to put into the register
 *
 * This function allows writing of a 32bit value to the indexed scratchpad
 * register. The register resides on the secondary (external) side.
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
int
ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
{

	if (idx >= ntb->limits.max_spads)
		return (EINVAL);

	ntb_write_4(ntb->reg_ofs.spad_remote + idx * 4, val);

	return (0);
}

/**
 * ntb_read_remote_spad() - read from the primary scratchpad register
 * @ntb: pointer to ntb_softc instance
 * @idx: index to scratchpad register, 0 based
 * @val: pointer to 32bit integer for storing the register value
 *
 * This function allows reading of the 32bit scratchpad register on
 * the primary (internal) side.
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
int
ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
{

	if (idx >= ntb->limits.max_spads)
		return (EINVAL);

	*val = ntb_read_4(ntb->reg_ofs.spad_remote + idx * 4);

	return (0);
}

/**
 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
 * @ntb: pointer to ntb_softc instance
 * @mw: memory window number
 *
 * This function provides the base virtual address of the memory window
 * specified.
 *
 * RETURNS: pointer to virtual address, or NULL on error.
 */
void *
ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw)
{

	if (mw >= NTB_NUM_MW)
		return (NULL);

	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase);
}

vm_paddr_t
ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw)
{

	if (mw >= NTB_NUM_MW)
		return (0);

	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase);
}

/**
 * ntb_get_mw_size() - return size of NTB memory window
 * @ntb: pointer to ntb_softc instance
 * @mw: memory window number
 *
 * This function provides the physical size of the memory window specified
 *
 * RETURNS: the size of the memory window or zero on error
 */
u_long
ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw)
{

	if (mw >= NTB_NUM_MW)
		return (0);

	return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size);
}

/**
 * ntb_set_mw_addr - set the memory window address
 * @ntb: pointer to ntb_softc instance
 * @mw: memory window number
 * @addr: base address for data
 *
 * This function sets the base physical address of the memory window.  This
 * memory address is where data from the remote system will be transfered into
 * or out of depending on how the transport is configured.
 */
void
ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
{

	if (mw >= NTB_NUM_MW)
		return;

	switch (NTB_MW_TO_BAR(mw)) {
	case NTB_B2B_BAR_1:
		ntb_write_8(ntb->reg_ofs.sbar2_xlat, addr);
		break;
	case NTB_B2B_BAR_2:
		ntb_write_8(ntb->reg_ofs.sbar4_xlat, addr);
		break;
	}
}

/**
 * ntb_ring_sdb() - Set the doorbell on the secondary/external side
 * @ntb: pointer to ntb_softc instance
 * @db: doorbell to ring
 *
 * This function allows triggering of a doorbell on the secondary/external
 * side that will initiate an interrupt on the remote host
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
void
ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db)
{

	if (ntb->type == NTB_SOC)
		ntb_write_8(ntb->reg_ofs.sdb, (uint64_t) 1 << db);
	else
		ntb_write_2(ntb->reg_ofs.sdb,
		    ((1 << ntb->bits_per_vector) - 1) <<
		    (db * ntb->bits_per_vector));
}

/**
 * ntb_query_link_status() - return the hardware link status
 * @ndev: pointer to ntb_device instance
 *
 * Returns true if the hardware is connected to the remote system
 *
 * RETURNS: true or false based on the hardware link state
 */
bool
ntb_query_link_status(struct ntb_softc *ntb)
{

	return (ntb->link_status == NTB_LINK_UP);
}

static void
save_bar_parameters(struct ntb_pci_bar_info *bar)
{
	bar->pci_bus_tag =
	    rman_get_bustag(bar->pci_resource);
	bar->pci_bus_handle =
	    rman_get_bushandle(bar->pci_resource);
	bar->pbase =
	    rman_get_start(bar->pci_resource);
	bar->size =
	    rman_get_size(bar->pci_resource);
	bar->vbase =
	    rman_get_virtual(bar->pci_resource);

}

device_t ntb_get_device(struct ntb_softc *ntb)
{

	return (ntb->device);
}