1/**
2 * \file
3 * \brief Driver module example template.
4 *
5 * In summary, every driver (struct bfdriver) shall implement the five
6 * life-cycle functions init/set_sleep_level/attach/detach/destroy
7 * (along with additional IRQ handler functions etc.).
8 *
9 * A driver module is linked with a driver domain (see main.c in this directory).
10 * At runtime, a driver domain will instantiate a driver instance (struct bfdriver_instance)
11 * of a given module (or class if you want) using the `init` function. During the lifetime
12 * of a driver instance it may be `detached` from and re-`attach`-ed to the
13 * device, set in different sleep levels, and finally it can be `destroy`-ed.
14 *
15 * For every driver instance (i.e., struct bfdriver_instance), a corresponding
16 * control interface created and exported. The interface is defined in dcontrol.if,
17 * the corresponding code is located in the driverkit library (dcontrol_service.c).
18 *
19 * \note For more information about driver domains check the main.c file in this
20 * directory.
21 */
22/*
23 * Copyright (c) 2016, ETH Zurich.
24 * All rights reserved.
25 *
26 * This file is distributed under the terms in the attached LICENSE file.
27 * If you do not find this file, copies can be found by writing to:
28 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group.
29 */
30
31#include <stdlib.h>
32#include <stdio.h>
33#include <string.h>
34#include <assert.h>
35
36#include <barrelfish/barrelfish.h>
37#include <driverkit/driverkit.h>
38#include <driverkit/iommu.h>
39
40#include <pci/pci.h>
41
42#include <dma/dma.h>
43#include <dma/dma_device.h>
44#include <dma/dma_request.h>
45#include <dma/ioat/ioat_dma.h>
46#include <dma/ioat/ioat_dma_device.h>
47#include <dma/ioat/ioat_dma_request.h>
48
49#include "device.h"
50#include "ioat_mgr_service.h"
51#include "debug.h"
52
53static uint8_t device_next = 0;
54static uint8_t device_count = 0;
55struct ioat_dma_device **devices = NULL;
56
57#if 0
58static void handle_device_interrupt(void *arg)
59{
60
61//    struct ioat_dma_device *dev = *((struct ioat_dma_device **) arg);
62//    struct dma_device *dma_dev = (struct dma_device *) dev;
63
64    INTR_DEBUG("interrupt! device %u", dma_device_get_id(arg));
65
66}
67#endif
68
69struct ioat_dma_device *ioat_device_get_next(void)
70{
71    if (device_next >= device_count) {
72        device_next = 0;
73    }
74    return devices[device_next++];
75}
76
77errval_t ioat_device_poll(void)
78{
79    errval_t err;
80
81    uint8_t idle = 0x1;
82    for (uint8_t i = 0; i < device_count; ++i) {
83        err = ioat_dma_device_poll_channels((struct dma_device *)devices[i]);
84        switch (err_no(err)) {
85            case SYS_ERR_OK:
86                idle = 0;
87                break;
88            case DMA_ERR_DEVICE_IDLE:
89                break;
90            default:
91                return err;
92        }
93    }
94    if (idle) {
95        return DMA_ERR_DEVICE_IDLE;
96    }
97    return SYS_ERR_OK;
98}
99
100
101#define OSDI18_RUN_BENCHMARK 1
102#include <dma/dma_bench.h>
103#include <skb/skb.h>
104#include <bench/bench.h>
105
106#define BUFFER_SIZE (1UL << 20)
107#if OSDI18_RUN_BENCHMARK
108
109static void cpumemcpy(genvaddr_t to, genvaddr_t from, size_t bytes)
110{
111    cycles_t t = bench_tsc();
112    memcpy((void *)to, (void *)from, bytes);
113    t = bench_tsc() - t;
114
115    debug_printf("cpumemcpy Elapsed: %lu\n", t);
116}
117
118
119static volatile bool done = false;
120
121static void impl_test_cb(errval_t err, dma_req_id_t id, void *arg)
122{
123    debug_printf("impl_test_cb\n");
124  //  assert(memcmp(arg, arg + BUFFER_SIZE, BUFFER_SIZE) == 0);
125    debug_printf("test ok\n");
126
127    done = 1;
128}
129
130static struct dma_device *dma_dev;
131
132static void dmamemcpy(genvaddr_t to, genvaddr_t from, size_t bytes)
133{
134    errval_t err;
135
136    struct dma_req_setup setup = {
137            .args.memcpy = {
138                    .src = (genvaddr_t) from,
139                    .dst = (genvaddr_t) to,
140                    .bytes = bytes,
141            },
142            .type = DMA_REQ_TYPE_MEMCPY,
143            .done_cb = impl_test_cb,
144            .cb_arg = (void *)to
145    };
146    done = false;
147    cycles_t t = bench_tsc();
148
149    dma_req_id_t rid;
150    err = ioat_dma_request_memcpy(dma_dev, &setup, &rid);
151    if (err_is_fail(err)) {
152        USER_PANIC_ERR(err, "failed to setup transfer\n");
153    }
154
155    while(!done) {
156        while(done == 0) {
157            err = ioat_dma_device_poll_channels(dma_dev);
158            switch (err_no(err)) {
159                case DMA_ERR_DEVICE_IDLE :
160                case DMA_ERR_CHAN_IDLE:
161                case SYS_ERR_OK:
162                    break;
163                default:
164                    USER_PANIC_ERR(err, "failed to poll the channel!\n");
165            }
166        }
167    }
168
169    t = bench_tsc() - t;
170    debug_printf("dmamemcpy Elapsed: %lu\n", t);
171}
172
173static void osdi18_benchmark(struct ioat_dma_device *dev, struct iommu_client *cl)
174{
175    errval_t err;
176
177    debug_printf("========================================================\n");
178    debug_printf("BENCHMARK FOR OSDI18\n");
179    debug_printf("========================================================\n");
180
181
182    bench_init();
183
184    dma_dev = (struct dma_device *)dev;
185
186
187    struct dmem mem;
188    err = driverkit_iommu_mmap_cl(cl, 2 * BUFFER_SIZE, VREGION_FLAGS_READ_WRITE,
189                                  &mem);
190    if (err_is_fail(err)) {
191        USER_PANIC_ERR(err, "failed to get memory");
192    }
193
194    cpumemcpy(mem.vbase, mem.vbase + BUFFER_SIZE, BUFFER_SIZE);
195
196    dmamemcpy(mem.devaddr, mem.devaddr + BUFFER_SIZE, BUFFER_SIZE);
197
198}
199#endif
200
201//#define TEST_IMPLEMENTATION 1
202#if TEST_IMPLEMENTATION
203#include <dma/dma_bench.h>
204#include <skb/skb.h>
205
206
207#define BUFFER_SIZE (1<<20)
208
209uint32_t done = 0;
210
211static void impl_test_cb(errval_t err, dma_req_id_t id, void *arg)
212{
213    debug_printf("impl_test_cb\n");
214    assert(memcmp(arg, arg + BUFFER_SIZE, BUFFER_SIZE) == 0);
215    debug_printf("test ok\n");
216
217    done = 1;
218}
219
220static void impl_test(struct ioat_dma_device *dev, struct iommu_client *cl)
221{
222    errval_t err;
223
224    debug_printf("Doing an implementation test\n");
225
226    struct dmem mem;
227    err = driverkit_iommu_mmap_cl(cl, 2 * BUFFER_SIZE, VREGION_FLAGS_READ_WRITE,
228                                  &mem);
229    if (err_is_fail(err)) {
230        DEBUG_ERR(err, "failed to get memory");
231    }
232
233    void *buf = (void *)mem.vbase;
234    memset(buf, 0, mem.size);
235    memset(buf, 0xA5, BUFFER_SIZE);
236
237    struct dma_req_setup setup = {
238            .args.memcpy = {
239                .src = mem.devaddr,
240                .dst = mem.devaddr + BUFFER_SIZE,
241                .bytes = BUFFER_SIZE,
242            },
243        .type = DMA_REQ_TYPE_MEMCPY,
244        .done_cb = impl_test_cb,
245        .cb_arg = buf
246    };
247    int reps = 10;
248    do {
249        memset(buf, 0, mem.size);
250        memset(buf, reps + 2, BUFFER_SIZE);
251        assert(memcmp(buf, buf + BUFFER_SIZE, BUFFER_SIZE));
252
253        debug_printf("!!!!!! NEW ROUND\n");
254        dma_req_id_t rid;
255        err = ioat_dma_request_memcpy((struct dma_device *)dev, &setup, &rid);
256        assert(err_is_ok(err));
257
258        done = 0;
259        while(done == 0) {
260            err = ioat_dma_device_poll_channels((struct dma_device *)dev);
261            switch (err_no(err)) {
262                case DMA_ERR_DEVICE_IDLE :
263                case DMA_ERR_CHAN_IDLE:
264                case SYS_ERR_OK:
265                    break;
266                default:
267                    DEBUG_ERR(err, "failed to poll the channel!\n");
268            }
269        }
270#if 0
271        if (reps == 1) {
272            debug_printf("using phys addr!\n");
273            setup.args.memcpy.src = id.base;
274            setup.args.memcpy.dst = id.base + BUFFER_SIZE;
275        }
276#endif
277
278    } while(reps--);
279
280
281}
282#endif
283
284/**
285 * Driver initialization function. This function is called by the driver domain
286 * (see also 'create_handler' in ddomain_service.c).
287 * Typically through a request from the device manager.
288 *
289 * The init function is supposed to set `dev` to the exported service iref.
290 * The init function may use `bfi->dstate` to store additional state about the device.
291 *
292 * \param[in]   bfi   The instance of this driver.
293 * \param[in]   name  The name of this driver instance.
294 * \param[in]   flags Additional flags (The exact flags supported is device/driver specific).
295 * \param[in]   c     Capabilities (for registers etc.) as provided by the device manager.
296 *                    The exact layout of the `c` is device specific.
297 * \param[out]  dev   The service iref over which the device can be contacted.
298 *
299 * \retval SYS_ERR_OK Device initialized successfully.
300 * \retval LIB_ERR_MALLOC_FAIL Unable to allocate memory for the driver.
301 */
302static errval_t init(struct bfdriver_instance *bfi, uint64_t flags, iref_t* dev) {
303
304    errval_t err;
305    // 1. Initialize the device:
306
307    debug_printf("[ioat]: attaching device '%s'\n", bfi->name);
308
309    struct ioat_dma_device **devices_new;
310    devices_new = realloc(devices, (device_count + 1) * sizeof(void *));
311    if (devices_new == NULL) {
312        return LIB_ERR_MALLOC_FAIL;
313    }
314    devices = devices_new;
315
316    struct capref iommuep;
317    err = driverkit_get_iommu_cap(bfi, &iommuep);
318    if (err_is_fail(err)) {
319        return err;
320    }
321
322    //driverkit_iommu_vspace_set_default_policy(IOMMU_VSPACE_POLICY_SHARED);
323
324    struct iommu_client *cl;
325    err = driverkit_iommu_client_init_cl(iommuep, &cl);
326    if (err_is_fail(err)) {
327        USER_PANIC_ERR(err, "Failed to initialize the IOMMU library");
328    }
329    assert(cl);
330
331    debug_printf("IOMMU PRESENT: %u\n", driverkit_iommu_present(cl));
332
333
334    struct capref regs;
335    err = driverkit_get_bar_cap(bfi, 0, &regs);
336    if (err_is_fail(err)) {
337        return err;
338    }
339
340    /* initialize the device */
341    //err = ioat_dma_device_init(regs, &pc1, false, &devices[device_count]);
342    err = ioat_dma_device_init(regs, cl, &devices[device_count]);
343    if (err_is_fail(err)) {
344
345        DEV_ERR("Could not initialize the device: %s\n", err_getstring(err));
346        return SYS_ERR_OK;
347    }
348
349    #if TEST_IMPLEMENTATION
350    impl_test(devices[device_count], cl);
351    #endif
352
353    #if OSDI18_RUN_BENCHMARK
354    osdi18_benchmark(devices[device_count], cl);
355    #endif
356
357    device_count++;
358
359
360    // 2. Export service to talk to the device:
361
362    // 3. Set iref of your exported service (this is reported back to Kaluga)
363    *dev = 0x00;
364
365
366    return SYS_ERR_OK;
367}
368
369/**
370 * Instructs driver to attach to the device.
371 * This function is only called if the driver has previously detached
372 * from the device (see also detach).
373 *
374 * \note After detachment the driver can not assume anything about the
375 * configuration of the device.
376 *
377 * \param[in]   bfi   The instance of this driver.
378 * \retval SYS_ERR_OK Device initialized successfully.
379 */
380static errval_t attach(struct bfdriver_instance* bfi) {
381
382    return SYS_ERR_OK;
383}
384
385/**
386 * Instructs driver to detach from the device.
387 * The driver must yield any control over to the device after this function returns.
388 * The device may be left in any state.
389 *
390 * \param[in]   bfi   The instance of this driver.
391 * \retval SYS_ERR_OK Device initialized successfully.
392 */
393static errval_t detach(struct bfdriver_instance* bfi) {
394
395    return SYS_ERR_OK;
396}
397
398/**
399 * Instructs the driver to go in a particular sleep state.
400 * Supported states are platform/device specific.
401 *
402 * \param[in]   bfi   The instance of this driver.
403 * \retval SYS_ERR_OK Device initialized successfully.
404 */
405static errval_t set_sleep_level(struct bfdriver_instance* bfi, uint32_t level) {
406
407    return SYS_ERR_OK;
408}
409
410/**
411 * Destroys this driver instance. The driver will yield any
412 * control over the device and free any state allocated.
413 *
414 * \param[in]   bfi   The instance of this driver.
415 * \retval SYS_ERR_OK Device initialized successfully.
416 */
417static errval_t destroy(struct bfdriver_instance* bfi) {
418
419    // XXX: Tear-down the service
420    bfi->device = 0x0;
421
422    return SYS_ERR_OK;
423}
424
425
426
427static errval_t get_ep(struct bfdriver_instance* bfi, bool lmp, struct capref* ret_cap)
428{
429    return SYS_ERR_OK;
430}
431/**
432 * Registers the driver module with the system.
433 *
434 * To link this particular module in your driver domain,
435 * add it to the addModules list in the Hakefile.
436 */
437DEFINE_MODULE(ioat_dma_module, init, attach, detach, set_sleep_level, destroy, get_ep);
438