1/*
2 * Copyright (c) 2014 ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#ifndef XEON_PHI_H_
11#define XEON_PHI_H_
12
13#include <xeon_phi/xeon_phi.h>
14#include <collections/hash_table.h>
15#include <dev/xeon_phi/xeon_phi_serial_dev.h>
16#include "debug.h"
17
18struct iommu_client;
19
20/*
21 * Common setting values
22 */
23
24/// the name of the Xeon Phi bootloader image
25#define XEON_PHI_BOOTLOADER "/weever"
26
27/// name of the file containing the modules to be loaded
28#define XEON_PHI_MOD_LIST "menu.lst.k1om"
29
30/// the name fo the nfs path to mount
31#define XEON_PHI_NFS_PATH NULL
32
33/// path where the NFS directory will be mounted
34#define XEON_PHI_NFS_MNT "/xeon_phi"
35
36/// if we use MSIX interrupts or legacy interrupts
37#define XEON_PHI_MSIX_ENABLED 1
38
39/// the number of MSIX interrupts we use
40#define XEON_PHI_MSIX_NUM     1
41
42/// the initial aperture size mapped to boot the card
43#define XEON_PHI_APERTURE_INIT_SIZE (1024*1024*1024)
44
45
46#define XEON_PHI_IS_CLIENT 0x0
47#define XEON_PHI_IS_HOST   0x1
48
49/*
50 * This defines are used to reference the MMIO registers on the host side.
51 *
52 * The Mackerel Specifications use the SBOX or DBOX baseaddress as their
53 * register base. however the SBOX or DBOX have a certain offset into the
54 * MMIO range.
55 */
56#define XEON_PHI_HOST_DBOX_OFFSET      0x00000000
57#define XEON_PHI_HOST_SBOX_OFFSET      0x00010000
58
59#define XEON_PHI_MMIO_TO_SBOX(phi) \
60    ((void *)((lvaddr_t)(phi->mmio.vbase)+XEON_PHI_HOST_SBOX_OFFSET))
61#define XEON_PHI_MMIO_TO_DBOX(phi) \
62    ((void *)((lvaddr_t)(phi->mmio.vbase)+XEON_PHI_HOST_DBOX_OFFSET))
63
64/*
65 * --------------------------------------------------------------------------
66 * Xeon Phi Management structure
67 */
68
69
70/// represents the state of the Xeon Phi
71typedef enum xeon_phi_state
72{
73    XEON_PHI_STATE_NULL,    ///< The card has not yet been initialized
74    XEON_PHI_STATE_PCI_OK,  ///< The card has been registered with PCI
75    XEON_PHI_STATE_RESET,   ///< The card has been reset
76    XEON_PHI_STATE_READY,   ///< The card is ready to load the os
77    XEON_PHI_STATE_BOOTING,  ///< The card is in the booting state
78    XEON_PHI_STATE_ONLINE   ///< the card has booted and is online
79} xeon_phi_state_t;
80
81typedef enum xnode_state
82{
83    XNODE_STATE_NONE,
84    XNODE_STATE_REGISTERING,
85    XNODE_STATE_WAIT_CONNECTION,
86    XNODE_STATE_READY,
87    XNODE_STATE_FAILURE
88} xnode_state_t;
89
90/// represents the memory ranges occupied by the Xeon Phi card
91struct mbar
92{
93    lvaddr_t vbase;     ///< virtual address of the mbar if mapped
94    lpaddr_t pbase;     ///< physical address of the mbar
95    struct capref cap;  ///< capability of the mbar
96    size_t  bytes;      ///< size of the region in bytes
97};
98
99struct xnode
100{
101    struct xeon_phi_driver_binding *binding;
102    iref_t iref;
103    xnode_state_t state;
104    uint8_t bootstrap_done;
105    errval_t err;
106    uint8_t id;
107    lpaddr_t apt_base;
108    size_t apt_size;
109    struct msg_info *msg;
110    struct xeon_phi *local;
111};
112
113#define XEON_PHI_BUFFER_LENGTH 0x400
114
115struct xeon_phi
116{
117    struct xeon_phi *next;
118
119    xeon_phi_state_t state;
120    struct mbar mmio;       ///< pointer to the MMIO address range
121    struct mbar apt;        ///< pointer to the aperture address range
122
123    lvaddr_t os_offset;     ///< offset of the OS image into the aperture
124    uint32_t os_size;       ///< the size of the OS image
125    char *cmdline;          ///< pointer to the bootloader cmdline
126    uint32_t cmdlen;        ///< the length of the cmd line
127
128    uint8_t id;             ///< card id for identifying the card
129    iref_t iref;
130    uint32_t apicid;        ///< APIC id used for sending the boot interrupt
131
132    uint8_t connected;      ///< number of connected xphis
133    struct xnode topology[XEON_PHI_NUM_MAX];
134
135    domainid_t current_key;
136    collections_hash_table* did_to_cap; ///< hash table to translate caps to dids
137
138    uint8_t is_client;
139
140    iref_t xphi_svc_iref;
141
142    struct iommu_client *iommu_client;
143
144    xeon_phi_serial_t serial_base;
145    char serial_buffer[XEON_PHI_BUFFER_LENGTH + 1];
146    uint32_t serial_buffer_idx;
147
148    struct smpt_info *smpt;  ///< pointer to the SMPT information struct
149    struct irq_info *irq;   ///< pointer to the IRQ information struct
150    struct dma_device *dma;   ///< pointer to the DMA information struct
151    struct msg_info *msg;   ///< pointer to the Messaging information struct
152
153    int32_t nodeid;             ///< model nodeid of the xeon phi
154};
155
156/**
157 * \brief starts the serial receive thread
158 *
159 * \param phi   pointer to the card information
160 */
161errval_t xeon_phi_serial_start_recv_thread(struct xeon_phi *phi);
162
163/**
164 * \brief initializes the serial receiver
165 *
166 * \param phi   pointer to the card information
167 */
168errval_t xeon_phi_serial_init(struct xeon_phi *phi);
169
170/**
171 * \brief checks if there is a message waiting on the serial and reads
172 *        it into the buffer.
173 *
174 * \return 0: There was no message
175 *         1: There was a message waiting and it porocessed.
176 */
177uint32_t xeon_phi_serial_handle_recv(struct xeon_phi *phi);
178
179/**
180 * \brief boots the card with the given loader and multiboot image
181 *
182 * \param phi           pointer to the card information
183 * \param xloader_img   pointer to the card bootloader image
184 * \param multiboot_img pointer to the card multiboot image
185 */
186errval_t xeon_phi_boot(struct xeon_phi *phi,
187                       char *xloader_img,
188                       char *multiboot_img);
189
190/**
191 * \brief performs a soft reset of the card
192 *
193 * \param phi   pointer to the card information
194 */
195errval_t xeon_phi_reset(struct xeon_phi *phi);
196
197/**
198 * \brief initializes the coprocessor card
199 *
200 * \param phi pointer to the information structure
201 */
202errval_t xeon_phi_init(struct xeon_phi *phi, struct capref mmio, struct capref apt);
203
204/**
205 * \brief maps the aperture memory range of the Xeon Phi into the drivers
206 *        vspace to be able to load the coprocessor OS onto the card
207 *
208 * \param phi   pointer to the Xeon Phi structure holding aperture information
209 * \param range how much bytes to map
210 *
211 * \returns SYS_ERR_OK on success
212 */
213errval_t xeon_phi_map_aperture(struct xeon_phi *phi,
214                               size_t range);
215
216/**
217 * \brief unmaps the previously mapped aperture range when the programming
218 *        completes.
219 *
220 * \param phi pointer to the Xeon Phi structure holiding mapping information
221 *
222 * \return SYS_ERR_OK on success
223 */
224errval_t xeon_phi_unmap_aperture(struct xeon_phi *phi);
225
226/**
227 * \brief handles events on the waitset and polls for completed DMA transfers
228 *        and new data on the serial line (host only)
229 *
230 * \param do_yield if set, yield thread if no event was discovered
231 *
232 * \return SYS_ERR_OK if an event was handled
233 *         LIB_ERR_NO_EVENT if there was no evetn
234 */
235errval_t xeon_phi_event_poll(struct xeon_phi *phi, uint8_t do_yield);
236
237/**
238 * \brief Given the PCI node id (as returned from driverkit lib),
239 *        return other xeon phi node ids.
240 *
241 */
242errval_t xeon_phi_hw_model_lookup_nodeids(int32_t pci_nodeid,
243        int32_t *knc_socket,
244        int32_t *smpt,
245        int32_t *iommu,
246        int32_t *dma,
247        int32_t *k1om_core, int32_t *gddr
248        );
249
250/**
251 * \brief  Make the capability mem visible to the Xeon Phi (retaddr) and the
252 *         local process (local_retaddr).
253 *
254 */
255errval_t xeon_phi_hw_model_query_and_config(void *arg,
256                                            struct capref mem,
257                                            genpaddr_t *retaddr,
258                                            genvaddr_t *local_retaddr);
259
260#endif /* XEON_PHI_H_ */
261