1/*
2 * Copyright (c) 2014, ETH Zurich. All rights reserved.
3 *
4 * This file is distributed under the terms in the attached LICENSE file.
5 * If you do not find this file, copies can be found by writing to:
6 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
7 */
8
9#include <string.h>
10#include <barrelfish/barrelfish.h>
11#include <bench/bench.h>
12#include <driverkit/iommu.h>
13#include <driverkit/hwmodel.h>
14#include <dev/xeon_phi/xeon_phi_dma_dev.h>
15
16#include <dma_mem_utils.h>
17
18#include <xeon_phi/xeon_phi_dma_internal.h>
19#include <xeon_phi/xeon_phi_dma_device_internal.h>
20#include <xeon_phi/xeon_phi_dma_channel_internal.h>
21
22#include <debug.h>
23
24/**
25 * Xeon Phi DMA device representation
26 */
27struct xeon_phi_dma_device
28{
29    struct dma_device common;
30    struct iommu_client *iommu;
31    xeon_phi_dma_t device;          ///< mackerel device base
32    struct dmem dstat;     ///< memory region for channels dstat_wb
33    uint32_t flags;
34};
35
36/// counter for device ID enumeration
37static dma_dev_id_t device_id = 1;
38
39/*
40 * ===========================================================================
41 * Library Internal Interface
42 * ===========================================================================
43 */
44
45/**
46 * \brief fills in the memory information structure for the channel's dstat
47 *        address
48 *
49 * \param dev   Xeon Phi DMA device
50 * \param mem   Memory structure to fill in
51 */
52void xeon_phi_dma_device_get_dstat_addr(struct xeon_phi_dma_device *dev,
53                                        struct dmem *mem)
54{
55    assert(dev->dstat.vbase);
56
57    *mem = dev->dstat;
58    mem->size = XEON_PHI_DMA_CHANNEL_DSTAT_SIZE;
59    mem->devaddr += (XEON_PHI_DMA_CHANNEL_DSTAT_SIZE * dev->common.channels.next);
60    mem->mem = NULL_CAP;
61    mem->vbase += (XEON_PHI_DMA_CHANNEL_DSTAT_SIZE * dev->common.channels.next++);
62}
63
64/**
65 * \brief globally enables the interrupts for the given device
66 *
67 * \param dev   Xeon Phi DMA device
68 * \param type  the interrupt type to enable
69 */
70errval_t xeon_phi_dma_device_irq_setup(struct xeon_phi_dma_device *dev,
71                                       dma_irq_t type)
72{
73    assert(!"NYI");
74
75    return SYS_ERR_OK;
76}
77
78/**
79 * \brief gets the Xeon Phi virtual base address of the DMA channel with
80 *        the given id
81 *
82 * \param dev   Xeon Phi DMA device
83 * \param idx   DMA channel index
84 *
85 * \returns virtual address of MMIO registers for the channel
86 */
87void *xeon_phi_dma_device_get_channel_vbase(struct xeon_phi_dma_device *dev,
88                                            uint8_t idx)
89{
90    XPHIDEV_DEBUG("getting channel vbase for %u, offset=%x\n", dev->common.id,
91                  idx, (idx * 0x40) + XEON_PHI_DMA_OFFSET);
92    return (void *) (dev->common.mmio.vaddr + (idx * 0x40) + XEON_PHI_DMA_OFFSET);
93}
94
95/**
96 * \brief sets the channel owner register of the Xeon Phi DMA device
97 *
98 * \param dev   Xeon Phi DMA device
99 * \param idx   channel index
100 * \param owner owner of the channel
101 */
102void xeon_phi_dma_device_set_channel_owner(struct xeon_phi_dma_device *dev,
103                                           uint8_t idx,
104                                           xeon_phi_dma_owner_t owner)
105{
106    uint8_t owner_val;
107    if (owner == XEON_PHI_DMA_OWNER_CARD) {
108        XPHIDEV_DEBUG("settings owner of channel [%u] to card.\n", dev->common.id, idx);
109        owner_val = 0;
110    } else {
111        XPHIDEV_DEBUG("settings owner of channel [%u] to host.\n", dev->common.id, idx);
112        owner_val = 1;
113    }
114
115    xeon_phi_dma_dcr_t dcr = xeon_phi_dma_dcr_rd(&dev->device);
116
117    switch (idx) {
118        case 0x0:
119            dcr = xeon_phi_dma_dcr_co0_insert(dcr, owner_val);
120            break;
121        case 0x1:
122            dcr = xeon_phi_dma_dcr_co1_insert(dcr, owner_val);
123            break;
124        case 0x2:
125            dcr = xeon_phi_dma_dcr_co2_insert(dcr, owner_val);
126            break;
127        case 0x3:
128            dcr = xeon_phi_dma_dcr_co3_insert(dcr, owner_val);
129            break;
130        case 0x4:
131            dcr = xeon_phi_dma_dcr_co4_insert(dcr, owner_val);
132            break;
133        case 0x5:
134            dcr = xeon_phi_dma_dcr_co5_insert(dcr, owner_val);
135            break;
136        case 0x6:
137            dcr = xeon_phi_dma_dcr_co6_insert(dcr, owner_val);
138            break;
139        case 0x7:
140            dcr = xeon_phi_dma_dcr_co7_insert(dcr, owner_val);
141            break;
142    }
143
144    xeon_phi_dma_dcr_wr(&dev->device, dcr);
145}
146
147/**
148 * \brief Enables / Disables the Xeon Phi DMA channel
149 *
150 * \param chan      Xeon Phi DMA channel
151 * \param idx       channel index
152 * \param enabled   flag to set the channel enabled
153 */
154void xeon_phi_dma_device_set_channel_state(struct xeon_phi_dma_device *dev,
155                                           uint8_t idx,
156                                           uint8_t enabled)
157{
158    uint8_t id = idx - XEON_PHI_DMA_DEVICE_CHAN_OFFSET;
159
160    xeon_phi_dma_dcr_t dcr = xeon_phi_dma_dcr_rd(&dev->device);
161
162    uint8_t enabled_val;
163    if (enabled) {
164        XPHIDEV_DEBUG("Enabling channel. [%u]\n", dev->common.id, idx);
165        if (dev->common.channels.c[id]) {
166            dev->common.channels.c[id]->state = DMA_CHAN_ST_RUNNING;
167        }
168        enabled_val = 0x1;
169    } else {
170        XPHIDEV_DEBUG("Disabling channel. [%u]\n", dev->common.id, idx);
171        if (dev->common.channels.c[id]) {
172            dev->common.channels.c[id]->state = DMA_CHAN_ST_SUSPENDED;
173        }
174        enabled_val = 0x0;
175    }
176
177    switch (idx) {
178        case 0x0:
179            dcr = xeon_phi_dma_dcr_ce0_insert(dcr, enabled_val);
180            break;
181        case 0x1:
182            dcr = xeon_phi_dma_dcr_ce1_insert(dcr, enabled_val);
183            break;
184        case 0x2:
185            dcr = xeon_phi_dma_dcr_ce2_insert(dcr, enabled_val);
186            break;
187        case 0x3:
188            dcr = xeon_phi_dma_dcr_ce3_insert(dcr, enabled_val);
189            break;
190        case 0x4:
191            dcr = xeon_phi_dma_dcr_ce4_insert(dcr, enabled_val);
192            break;
193        case 0x5:
194            dcr = xeon_phi_dma_dcr_ce5_insert(dcr, enabled_val);
195            break;
196        case 0x6:
197            dcr = xeon_phi_dma_dcr_ce6_insert(dcr, enabled_val);
198            break;
199        case 0x7:
200            dcr = xeon_phi_dma_dcr_ce7_insert(dcr, enabled_val);
201            break;
202    }
203
204    xeon_phi_dma_dcr_wr(&dev->device, dcr);
205}
206
207/*
208 * ===========================================================================
209 * Public Interface
210 * ===========================================================================
211 */
212
213/*
214 * ----------------------------------------------------------------------------
215 * device initialization / termination
216 * ----------------------------------------------------------------------------
217 */
218
219#include <xeon_phi/xeon_phi.h>
220
221/**
222 * \brief initializes a Xeon Phi DMA device with the giving capability
223 *
224 * \param mmio capability representing the device's MMIO registers
225 * \param iommu
226 * \param convert memory conversion function. In a simple case, returns the
227 *                physical address of the cap.
228 * \param nodeid HW model node id where xphi requests originate from, necessary
229 *               for model supported frame allocation.
230 * \param dev  returns a pointer to the device structure
231 *
232 *
233 * \returns SYS_ERR_OK on success
234 *          errval on error
235 */
236errval_t xeon_phi_dma_device_init(void *mmio_base, struct iommu_client *iommu,
237                                  dma_mem_convert_fn convert, void *convert_arg,
238                                  int32_t nodeid,
239                                  struct xeon_phi_dma_device **dev)
240{
241    errval_t err;
242
243    struct xeon_phi_dma_device *xdev = calloc(1, sizeof(*xdev));
244    if (xdev == NULL) {
245        return LIB_ERR_MALLOC_FAIL;
246    }
247
248#if DMA_BENCH_ENABLED
249     bench_init();
250#endif
251
252    struct dma_device *dma_dev = (struct dma_device *) xdev;
253
254    XPHIDEV_DEBUG("initializing Xeon Phi DMA device @ %p\n", device_id,
255                  mmio_base);
256
257
258#if defined(XEON_PHI_USE_HW_MODEL)
259    int32_t nodes[3];
260    nodes[0] = nodeid;
261    nodes[1] = driverkit_hwmodel_get_my_node_id();
262    nodes[2] = 0;
263    int32_t dest_nodeid = driverkit_hwmodel_lookup_dram_node_id();
264
265    err =  driverkit_hwmodel_frame_alloc(&xdev->dstat.mem, LARGE_PAGE_SIZE,
266                                      dest_nodeid, nodes);
267    if (err_is_fail(err)) {
268        free(xdev);
269        return err;
270    }
271
272    err = convert(convert_arg, xdev->dstat.mem, &xdev->dstat.devaddr, &xdev->dstat.vbase);
273    if (err_is_fail(err)) {
274        DEBUG_ERR(err, "convert mem");
275        free(xdev);
276        return err;
277    }
278#else
279    // Alloc cap slota
280    err = dma_mem_alloc(LARGE_PAGE_SIZE, VREGION_FLAGS_READ_WRITE, NULL, &xdev->dstat);
281    if (err_is_fail(err)) {
282        free(xdev);
283        return err;
284    }
285#endif
286
287    //debug_printf("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx\n");
288    //debug_printf("HACK: setting address!\n");
289    //xdev->dstat.devaddr = xdev->dstat.devaddr + XEON_PHI_SYSMEM_BASE -  2 * (512UL << 30);
290    //debug_printf("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx\n");R
291
292    dma_dev->id = device_id++;
293    dma_dev->irq_type = DMA_IRQ_DISABLED;
294    dma_dev->type = DMA_DEV_TYPE_XEON_PHI;
295    dma_dev->mmio.vaddr = (lvaddr_t) mmio_base;
296    dma_dev->f.poll = xeon_phi_dma_device_poll_channels;
297    dma_dev->iommu = iommu;
298    dma_dev->convert = convert;
299    dma_dev->convert_arg = convert_arg;
300    dma_dev->nodeid = nodeid;
301
302    xeon_phi_dma_initialize(&xdev->device, mmio_base);
303
304    XPHIDEV_DEBUG("initializing %u channels\n", device_id,
305                  XEON_PHI_DMA_DEVICE_CHANNELS);
306
307    dma_dev->channels.count = XEON_PHI_DMA_DEVICE_CHANNELS;
308    dma_dev->channels.c = calloc(XEON_PHI_DMA_DEVICE_CHANNELS,
309                                 sizeof(void *));
310
311    if (dma_dev->channels.c == NULL) {
312        device_id--;
313        free(xdev);
314        return LIB_ERR_MALLOC_FAIL;
315    }
316
317    for (uint8_t i = 0; i < XEON_PHI_DMA_DEVICE_CHANNELS; ++i) {
318        struct dma_channel **chan = &dma_dev->channels.c[i];
319        err = xeon_phi_dma_channel_init(xdev, i, XEON_PHI_DMA_DEVICE_MAX_XFER,
320                                        (struct xeon_phi_dma_channel **) chan);
321        if (err_is_fail(err)) {
322            free(dma_dev->channels.c);
323            device_id--;
324            free(xdev);
325            return err;
326        }
327    }
328
329    *dev = xdev;
330
331    XPHIDEV_DEBUG("Xeon Phi DMA device initialized\n", dma_dev->id);
332
333    return err;
334}
335
336/**
337 * \brief terminates the device operation and frees up the allocated resources
338 *
339 * \param dev Xeon Phi DMA device to shutdown
340 *
341 * \returns SYS_ERR_OK on success
342 *          errval on error
343 */
344errval_t xeon_phi_dma_device_shutdown(struct xeon_phi_dma_device *dev)
345{
346    assert(!"NYI");
347    return SYS_ERR_OK;
348}
349
350/*
351 * ----------------------------------------------------------------------------
352 * Interrupt management
353 * ----------------------------------------------------------------------------
354 */
355
356/**
357 * \brief enables the interrupts for the device
358 *
359 * \param dev   Xeon Phi DMA device
360 * \param type  interrupt type
361 * \param fn    interrupt handler function
362 * \param arg   argument supplied to the handler function
363 */
364errval_t xeon_phi_dma_device_intr_enable(struct xeon_phi_dma_device *dev,
365                                         dma_irq_t type,
366                                         dma_irq_fn_t fn,
367                                         void *arg)
368{
369    assert(!"NYI");
370    return SYS_ERR_OK;
371}
372
373/**
374 * \brief disables the interrupts for the device
375 *
376 * \param dev   Xeon Phi DMA device
377 */
378void xeon_phi_dma_device_intr_disable(struct xeon_phi_dma_device *dev)
379{
380    assert(!"NYI");
381}
382
383/*
384 * ----------------------------------------------------------------------------
385 * Device Operation Functions
386 * ----------------------------------------------------------------------------
387 */
388
389/**
390 * \brief polls the channels of the Xeon Phi DMA device
391 *
392 * \param dev   Xeon Phi DMA device
393 *
394 * \returns SYS_ERR_OK on success
395 *          DMA_ERR_DEVICE_IDLE if there is nothing completed on the channels
396 *          errval on error
397 */
398errval_t xeon_phi_dma_device_poll_channels(struct dma_device *dev)
399{
400    errval_t err;
401
402    uint8_t idle = 0x1;
403
404    for (uint8_t i = 0; i < dev->channels.count; ++i) {
405        err = xeon_phi_dma_channel_poll(dev->channels.c[i]);
406        switch (err_no(err)) {
407            case DMA_ERR_CHAN_IDLE:
408                /* no op */
409                break;
410            case SYS_ERR_OK:
411                idle = 0;
412                break;
413            default:
414                return err;
415        }
416    }
417
418    if (idle) {
419        return DMA_ERR_DEVICE_IDLE;
420    }
421
422    return SYS_ERR_OK;
423}
424
425