1/*
2 * Copyright (c) 2014, ETH Zurich. All rights reserved.
3 *
4 * This file is distributed under the terms in the attached LICENSE file.
5 * If you do not find this file, copies can be found by writing to:
6 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
7 */
8#include <string.h>
9#include <barrelfish/barrelfish.h>
10#include <barrelfish/waitset.h>
11#include <bench/bench.h>
12#include <dev/ioat_dma_dev.h>
13
14#include <dma_mem_utils.h>
15
16#include <ioat/ioat_dma_internal.h>
17#include <ioat/ioat_dma_dca_internal.h>
18#include <ioat/ioat_dma_device_internal.h>
19#include <ioat/ioat_dma_channel_internal.h>
20
21#include <debug.h>
22
23/**
24 * IOAT DMA device representation
25 */
26struct ioat_dma_device
27{
28    struct dma_device common;
29
30    ioat_dma_t device;                  ///< mackerel device base
31    ioat_dma_cbver_t version;           ///< Crystal Beach version number
32
33    struct dma_mem complstatus;         ///< memory region for channels CHANSTS
34    struct pci_addr pci_addr;        ///< the PCI address of the device
35
36    uint8_t irq_msix_vector;
37    uint16_t irq_msix_count;
38
39    uint32_t flags;
40};
41
42/// counter for device ID enumeration
43static dma_dev_id_t device_id = 1;
44
45/*
46 * ----------------------------------------------------------------------------
47 * device initialization functions
48 * ----------------------------------------------------------------------------
49 */
50
51static errval_t device_init_ioat_v1(struct ioat_dma_device *dev)
52{
53    IOATDEV_DEBUG("devices of Crystal Beach Version 1.xx are currently not supported.\n",
54                  dev->common.id);
55    return DMA_ERR_DEVICE_UNSUPPORTED;
56}
57
58static errval_t device_init_ioat_v2(struct ioat_dma_device *dev)
59{
60    IOATDEV_DEBUG("devices of Crystal Beach Version 2.xx are currently not supported.\n",
61                  dev->common.id);
62    return DMA_ERR_DEVICE_UNSUPPORTED;
63}
64
65static errval_t device_init_ioat_v3(struct ioat_dma_device *dev)
66{
67    errval_t err;
68
69    IOATDEV_DEBUG("initialize Crystal Beach 3 DMA device\n", dev->common.id);
70
71    ioat_dma_dmacapability_t cap = ioat_dma_dmacapability_rd(&dev->device);
72
73    if (ioat_dma_cbver_minor_extract(dev->version) == 2) {
74        IOATDEV_DEBUG("disabling XOR and PQ opcodes for Crystal Beach 3.2\n",
75                      dev->common.id);
76        cap = ioat_dma_dmacapability_xor_insert(cap, 0x0);
77        cap = ioat_dma_dmacapability_pq_insert(cap, 0x0);
78    } else if (ioat_dma_cbver_minor_extract(dev->version) == 3) {
79        IOATDEV_DEBUG("devices of Crystal Beach Version 3.3 are not supported.\n",
80                      dev->common.id);
81        return DMA_ERR_DEVICE_UNSUPPORTED;
82    }
83
84    /* if DCA is enabled, we cannot support the RAID functions */
85    if (ioat_dma_dca_is_enabled()) {
86        IOATDEV_DEBUG("Disabling XOR and PQ while DCA is enabled\n", dev->common.id);
87        cap = ioat_dma_dmacapability_xor_insert(cap, 0x0);
88        cap = ioat_dma_dmacapability_pq_insert(cap, 0x0);
89    }
90
91    if (ioat_dma_dmacapability_xor_extract(cap)) {
92        IOATDEV_DEBUG("device supports XOR RAID.\n", dev->common.id);
93
94        dev->flags |= IOAT_DMA_DEV_F_RAID;
95
96        /*
97         * this may need some additional functions to prepare
98         * the specific transfers...
99         *
100         * max_xor = 8;
101         * prepare_xor, prepare_xor_val
102         */
103    }
104
105    if (ioat_dma_dmacapability_pq_extract(cap)) {
106        IOATDEV_DEBUG("device supports PQ RAID.\n", dev->common.id);
107
108        dev->flags |= IOAT_DMA_DEV_F_RAID;
109
110        /*
111         * this may need some additional functions to prepare the
112         * DMA descriptors
113         *
114         * max_xor = 8;
115         * max_pq = 8;
116         * prepare_pq, perpare_pq_val
117         *
118         * also set the prepare_xor pointers...
119         *
120         */
121    }
122
123    /* set the interrupt type to disabled*/
124    dev->common.irq_type = DMA_IRQ_DISABLED;
125    dev->common.type = DMA_DEV_TYPE_IOAT;
126
127    /* allocate memory for completion status writeback */
128    err = dma_mem_alloc(IOAT_DMA_COMPLSTATUS_SIZE, IOAT_DMA_COMPLSTATUS_FLAGS,
129                        &dev->complstatus);
130    if (err_is_fail(err)) {
131        return err;
132    }
133
134    dev->common.channels.count = ioat_dma_chancnt_num_rdf(&dev->device);
135
136    dev->common.channels.c = calloc(dev->common.channels.count,
137                                    sizeof(*dev->common.channels.c));
138    if (dev->common.channels.c == NULL) {
139        dma_mem_free(&dev->complstatus);
140        return LIB_ERR_MALLOC_FAIL;
141    }
142
143    /* channel enumeration */
144
145    IOATDEV_DEBUG("channel enumeration. discovered %u channels\n", dev->common.id,
146                  dev->common.channels.count);
147
148    uint32_t max_xfer_size = (1 << ioat_dma_xfercap_max_rdf(&dev->device));
149
150    for (uint8_t i = 0; i < dev->common.channels.count; ++i) {
151        struct dma_channel **chan = &dev->common.channels.c[i];
152        err = ioat_dma_channel_init(dev, i, max_xfer_size,
153                                    (struct ioat_dma_channel **) chan);
154    }
155
156    if (dev->flags & IOAT_DMA_DEV_F_DCA) {
157        /*
158         * TODO: DCA initialization
159         * device->dca = ioat3_dca_init(pdev, device->reg_base);
160         */
161    }
162
163    err = ioat_dma_device_irq_setup(dev, DMA_IRQ_MSIX);
164    if (err_is_fail(err)) {
165        return err;
166    }
167
168    return SYS_ERR_OK;
169}
170
171/*
172 * ===========================================================================
173 * Library Internal Interface
174 * ===========================================================================
175 */
176
177void ioat_dma_device_get_complsts_addr(struct ioat_dma_device *dev,
178                                       struct dma_mem *mem)
179{
180    if (dev->common.state != DMA_DEV_ST_CHAN_ENUM) {
181        memset(mem, 0, sizeof(*mem));
182    }
183
184    assert(dev->complstatus.vaddr);
185
186    *mem = dev->complstatus;
187    mem->bytes = IOAT_DMA_COMPLSTATUS_SIZE;
188    mem->paddr += (IOAT_DMA_COMPLSTATUS_SIZE * dev->common.channels.next);
189    mem->frame = NULL_CAP;
190    mem->vaddr += (IOAT_DMA_COMPLSTATUS_SIZE * dev->common.channels.next++);
191}
192
193#if IOAT_DEBUG_INTR_ENABLED
194///< flag indicating that the interrupt has happened for debugging purposes
195static uint32_t msix_intr_happened = 0;
196#include <dma/ioat/ioat_dma_request.h>
197#endif
198
199static void ioat_dma_device_irq_handler(void* arg)
200{
201    errval_t err;
202    struct dma_device *dev = arg;
203
204    IOATDEV_DEBUG("############ MSIX INTERRUPT HAPPENED.\n", dev->id);
205
206#if IOAT_DEBUG_INTR_ENABLED
207    msix_intr_happened=1;
208#endif
209
210    err = ioat_dma_device_poll_channels(dev);
211    if (err_is_fail(err)) {
212        if (err_no(err) == DMA_ERR_DEVICE_IDLE) {
213            IOATDEV_DEBUG("WARNING: MSI-X interrupt on idle device\n", dev->id);
214            return;
215        }
216        USER_PANIC_ERR(err, "dma poll device returned an error\n");
217    }
218}
219
220/**
221 * \brief gets the local apic ID from the CPU id
222 *
223 * \return local apic ID
224 */
225static inline uint8_t get_local_apic_id(void)
226{
227    uint32_t eax, ebx;
228
229    cpuid(1, &eax, &ebx, NULL, NULL);
230    return  ebx >> 24;
231}
232
233/**
234 * \brief globally enables the interrupts for the given device
235 *
236 * \param dev   IOAT DMA device
237 * \param type  the interrupt type to enable
238 */
239errval_t ioat_dma_device_irq_setup(struct ioat_dma_device *dev,
240                                   dma_irq_t type)
241{
242    errval_t err;
243
244    ioat_dma_intrctrl_t intcrtl = 0;
245    intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
246
247    dev->common.irq_type = type;
248    switch (type) {
249        case DMA_IRQ_MSIX:
250            /* The number of MSI-X vectors should equal the number of channels */
251            IOATDEV_DEBUG("MSI-X interrupt setup for device (%u, %u, %u)\n",
252                          dev->common.id, dev->pci_addr.bus, dev->pci_addr.device,
253                          dev->pci_addr.function);
254
255            err = pci_msix_enable_addr(&dev->pci_addr, &dev->irq_msix_count);
256            if (err_is_fail(err)) {
257                return err;
258            }
259
260            assert(dev->irq_msix_count > 0);
261
262            IOATDEV_DEBUG("MSI-X enabled #vecs=%d\n", dev->common.id,
263                          dev->irq_msix_count);
264
265            err = pci_setup_inthandler(ioat_dma_device_irq_handler, dev,
266                                       &dev->irq_msix_vector);
267            assert(err_is_ok(err));
268
269            uint8_t dest = get_local_apic_id();
270
271            IOATDEV_DEBUG("MSI-X routing to apic=%u\n", dev->common.id,
272                          dest);
273
274            err = pci_msix_vector_init_addr(&dev->pci_addr, 0, dest,
275                                            dev->irq_msix_vector);
276            assert(err_is_ok(err));
277
278            /* enable the interrupts */
279            intcrtl = ioat_dma_intrctrl_msix_vec_insert(intcrtl, 1);
280            intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
281            break;
282        case DMA_IRQ_MSI:
283            IOATDEV_DEBUG("Initializing MSI interrupts \n", dev->common.id);
284            assert(!"NYI");
285            break;
286        case DMA_IRQ_INTX:
287            IOATDEV_DEBUG("Initializing INTx interrupts \n", dev->common.id);
288            assert(!"NYI");
289            break;
290        default:
291            /* disabled */
292            intcrtl = 0;
293            IOATDEV_DEBUG("Disabling interrupts \n", dev->common.id);
294            break;
295    }
296
297    ioat_dma_intrctrl_wr(&dev->device, intcrtl);
298
299
300#if IOAT_DEBUG_INTR_ENABLED
301    /*
302     * check if interrupts are working.
303     */
304    msix_intr_happened = 0;
305
306    struct ioat_dma_channel *chan;
307    chan = (struct ioat_dma_channel *)dev->common.channels.c[0];
308
309    ioat_dma_request_nop_chan(chan);
310    err = ioat_dma_channel_issue_pending(chan);
311    if (err_is_fail(err)) {
312        return err;
313    }
314
315    while(msix_intr_happened == 0) {
316        uint64_t status = ioat_dma_channel_get_status(chan);
317        err = event_dispatch_non_block(get_default_waitset());
318
319        if (!ioat_dma_channel_is_active(status) && !ioat_dma_channel_is_idle(status)) {
320            USER_PANIC("DMA request turned channel into erroneous state.")
321        }
322
323        switch(err_no(err)) {
324            case LIB_ERR_NO_EVENT:
325                thread_yield();
326                break;
327            case SYS_ERR_OK:
328                continue;
329            default:
330                USER_PANIC_ERR(err, "dispatching event");
331        }
332    }
333#endif
334
335    return SYS_ERR_OK;
336}
337
338/*
339 * ===========================================================================
340 * Public Interface
341 * ===========================================================================
342 */
343
344/*
345 * ----------------------------------------------------------------------------
346 * device initialization / termination
347 * ----------------------------------------------------------------------------
348 */
349
350/**
351 * \brief initializes a IOAT DMA device with the giving capability
352 *
353 * \param mmio     capability representing the device's MMIO registers
354 * \param pci_addr the PCI address of this device
355 * \param dev      returns a pointer to the device structure
356 *
357 * \returns SYS_ERR_OK on success
358 *          errval on error
359 */
360errval_t ioat_dma_device_init(struct capref mmio,
361                              struct pci_addr *pci_addr,
362                              struct ioat_dma_device **dev)
363{
364    errval_t err;
365
366    struct ioat_dma_device *ioat_device = calloc(1, sizeof(*ioat_device));
367    if (ioat_device == NULL) {
368        return LIB_ERR_MALLOC_FAIL;
369    }
370
371#if DMA_BENCH_ENABLED
372     bench_init();
373#endif
374
375    struct dma_device *dma_dev = &ioat_device->common;
376
377    struct frame_identity mmio_id;
378    err = frame_identify(mmio, &mmio_id);
379    if (err_is_fail(err)) {
380        free(ioat_device);
381        return err;
382    }
383
384    dma_dev->id = device_id++;
385    dma_dev->mmio.paddr = mmio_id.base;
386    dma_dev->mmio.bytes = mmio_id.bytes;
387    dma_dev->mmio.frame = mmio;
388    ioat_device->pci_addr = *pci_addr;
389
390    IOATDEV_DEBUG("init device with mmio range: {paddr=0x%016lx, size=%u kB}\n",
391                  dma_dev->id, mmio_id.base, mmio_id.bytes / 1024);
392
393    err = vspace_map_one_frame_attr((void**) &dma_dev->mmio.vaddr,
394                                    dma_dev->mmio.bytes, dma_dev->mmio.frame,
395                                    VREGION_FLAGS_READ_WRITE_NOCACHE,
396                                    NULL, NULL);
397    if (err_is_fail(err)) {
398        free(ioat_device);
399        return err;
400    }
401
402    ioat_dma_initialize(&ioat_device->device, NULL, (void *) dma_dev->mmio.vaddr);
403
404    ioat_device->version = ioat_dma_cbver_rd(&ioat_device->device);
405
406    IOATDEV_DEBUG("device registers mapped at 0x%016lx. IOAT version: %u.%u\n",
407                  dma_dev->id, dma_dev->mmio.vaddr,
408                  ioat_dma_cbver_major_extract(ioat_device->version),
409                  ioat_dma_cbver_minor_extract(ioat_device->version));
410
411    switch (ioat_dma_cbver_major_extract(ioat_device->version)) {
412        case ioat_dma_cbver_1x:
413            err = device_init_ioat_v1(ioat_device);
414            break;
415        case ioat_dma_cbver_2x:
416            err = device_init_ioat_v2(ioat_device);
417            break;
418        case ioat_dma_cbver_3x:
419            err = device_init_ioat_v3(ioat_device);
420            break;
421        default:
422            err = DMA_ERR_DEVICE_UNSUPPORTED;
423    }
424
425    if (err_is_fail(err)) {
426        vspace_unmap((void*) dma_dev->mmio.vaddr);
427        free(ioat_device);
428        return err;
429    }
430
431    dma_dev->f.deregister_memory = NULL;
432    dma_dev->f.register_memory = NULL;
433    dma_dev->f.poll = ioat_dma_device_poll_channels;
434
435    *dev = ioat_device;
436
437    return err;
438}
439
440/**
441 * \brief terminates the device operation and frees up the allocated resources
442 *
443 * \param dev IOAT DMA device to shutdown
444 *
445 * \returns SYS_ERR_OK on success
446 *          errval on error
447 */
448errval_t ioat_dma_device_shutdown(struct ioat_dma_device *dev)
449{
450    assert(!"NYI");
451    return SYS_ERR_OK;
452}
453
454/**
455 * \brief requests access to a IOAT DMA device from the IOAT device manager
456 *        and initializes the device.
457 *
458 * \param dev  returns a pointer to the device structure
459 *
460 * \returns SYS_ERR_OK on success
461 *          errval on error
462 */
463errval_t ioat_dma_device_acquire(struct ioat_dma_device **dev)
464{
465    errval_t err;
466
467    struct ioat_dma_device *ioat_device = calloc(1, sizeof(*ioat_device));
468    if (ioat_device == NULL) {
469        return LIB_ERR_MALLOC_FAIL;
470    }
471    assert(!"NYI");
472    err = SYS_ERR_OK;
473    return err;
474}
475
476/**
477 * \brief terminates the device operation and frees up the allocated resources
478 *        and releases the device and returns it to the IOAT device manager.
479 *
480 * \param dev IOAT DMA device to be released
481 *
482 * \returns SYS_ERR_OK on success
483 *          errval on error
484 */
485errval_t ioat_dma_device_release(struct ioat_dma_device *dev)
486{
487    assert(!"NYI");
488    return SYS_ERR_OK;
489}
490
491/*
492 * ----------------------------------------------------------------------------
493 * Interrupt management
494 * ----------------------------------------------------------------------------
495 */
496
497/**
498 * \brief enables the interrupts for the device
499 *
500 * \param dev   IOAT DMA device
501 * \param type  interrupt type
502 * \param fn    interrupt handler function
503 * \param arg   argument supplied to the handler function
504 */
505errval_t ioat_dma_device_intr_enable(struct ioat_dma_device *dev,
506                                     dma_irq_t type,
507                                     dma_irq_fn_t fn,
508                                     void *arg)
509{
510    assert(!"NYI");
511    return SYS_ERR_OK;
512}
513
514/**
515 * \brief disables the interrupts for the device
516 *
517 * \param dev   IOAT DMA device
518 */
519void ioat_dma_device_intr_disable(struct ioat_dma_device *dev)
520{
521    assert(!"NYI");
522}
523
524/**
525 * \brief sets the interrupt delay for the device
526 *
527 * \param dev   IOAT DMA device
528 * \param usec  interrupt delay in microseconds
529 */
530void ioat_dma_device_set_intr_delay(struct ioat_dma_device *dev,
531                                    uint16_t usec)
532{
533    ioat_dma_intrdelay_delay_us_wrf(&dev->device, usec);
534}
535
536/*
537 * ----------------------------------------------------------------------------
538 * Device Operation Functions
539 * ----------------------------------------------------------------------------
540 */
541
542/**
543 * \brief polls the channels of the IOAT DMA device
544 *
545 * \param dev   IOAT DMA device
546 *
547 * \returns SYS_ERR_OK on success
548 *          DMA_ERR_DEVICE_IDLE if there is nothing completed on the channels
549 *          errval on error
550 */
551errval_t ioat_dma_device_poll_channels(struct dma_device *dev)
552{
553    errval_t err;
554
555    uint8_t idle = 0x1;
556
557    for (uint8_t i = 0; i < dev->channels.count; ++i) {
558        err = ioat_dma_channel_poll(dev->channels.c[i]);
559        switch (err_no(err)) {
560            case DMA_ERR_CHAN_IDLE:
561                break;
562            case SYS_ERR_OK:
563                idle = 0;
564                break;
565            default:
566                return err;
567        }
568    }
569
570    if (idle) {
571        return DMA_ERR_DEVICE_IDLE;
572    }
573
574    return SYS_ERR_OK;
575}
576
577