1/*
2 * Copyright (c) 2014 ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
8 */
9#include <stdio.h>
10#include <barrelfish/barrelfish.h>
11#include <dma/xeon_phi/xeon_phi_dma.h>
12#include <dma/xeon_phi/xeon_phi_dma_device.h>
13#include <dma/xeon_phi/xeon_phi_dma_request.h>
14#include <dma/dma.h>
15#include <dma/dma_service.h>
16#include <dma/dma_mem_mgr.h>
17#include <dma/dma_manager_client.h>
18#include <dma/dma_bench.h>
19#include <xeon_phi/xeon_phi.h>
20
21#include "xeon_phi_internal.h"
22#include "dma_service.h"
23
24struct user_st
25{
26    struct dma_mem_mgr *mem_mgr;
27    struct xeon_phi *phi;
28    uint64_t num_req;
29};
30
31static void memcpy_req_cb(errval_t err,
32                          dma_req_id_t id,
33                          void *st)
34{
35    XDMA_DEBUG("memcpy_req_cb %lx, %s\n", id, err_getstring(err));
36    dma_service_send_done(st, err, id);
37}
38
39static lpaddr_t translate_address(void *arg,
40                                  lpaddr_t addr,
41                                  size_t size)
42{
43#ifdef __k1om__
44    if ((addr + size) > (1UL << 40)) {
45        return 0;
46    }
47    return addr;
48#else
49    struct xeon_phi *phi = ((struct user_st *)arg)->phi;
50    lpaddr_t apt_lo = phi->apt.pbase;
51    lpaddr_t apt_hi = phi->apt.pbase + phi->apt.bytes;
52    if ((addr >= apt_lo) && ((addr + size) <= apt_hi)) {
53        /* we are within the GDDR range */
54        return addr - apt_lo;
55    } else if ((addr + size)< XEON_PHI_SYSMEM_SIZE) {
56        /*
57         * Xeon Phi does not support more host memory,
58         * assume host memory starts at 0x0
59         */
60        return addr + XEON_PHI_SYSMEM_BASE;
61    } else {
62        return 0;
63    }
64#endif
65}
66
67/*
68 * ---------------------------------------------------------------------------
69 * Callbacks for the service events
70 * ---------------------------------------------------------------------------
71 */
72
73/*
74 * \brief callback handler for new connect requests
75 *
76 * \param arg       argument passed on the init function
77 * \param user_st   pointer to store the user state
78 *
79 * \returns SYS_ERR_OK if the connection is accepted
80 *          errval if connection is rejected
81 */
82static errval_t dma_svc_connect_cb(void *arg,
83                                   void **user_st)
84{
85    errval_t err;
86
87    struct user_st *st = calloc(1, sizeof(*st));
88    if (st == NULL) {
89        return LIB_ERR_MALLOC_FAIL;
90    }
91
92    XDMA_DEBUG("dma_svc_connect_cb user_st = %p\n", st);
93
94    err = dma_mem_mgr_init(&st->mem_mgr, 0x0, (1UL << 48) - 1);
95    if (err_is_fail(err)) {
96        free(st);
97        return err;
98    }
99
100    st->phi = arg;
101
102    dma_mem_mgr_set_convert_fn(st->mem_mgr, translate_address, st);
103
104    *user_st = st;
105
106    return SYS_ERR_OK;
107}
108
109/**
110 * \brief registers a memory region with the client such that it can be used later
111 *
112 * \param user_st   pointer to stored user state
113 * \param cap       the capability to register
114 *
115 * \returns SYS_ERR_OK if the memory region was accepted
116 *          errval if the memory region was rejected
117 */
118static errval_t dma_svc_addregion_cb(dma_svc_handle_t svc_handle,
119                                     struct capref cap)
120{
121    struct user_st *user_st = dma_service_get_user_state(svc_handle);
122
123    XDMA_DEBUG("dma_svc_addregion_cb user_st = %p\n", user_st);
124
125    return dma_mem_register(user_st->mem_mgr, cap);
126}
127
128/**
129 * \brief deregisters a memory region with the client
130 *
131 * \param user_st   pointer to stored user state
132 * \param cap       the capability to deregister
133 *
134 * \returns SYS_ERR_OK if the memory region was removed
135 *          errval if the memory region removal was rejected
136 */
137static errval_t dma_svc_removeregion_cb(dma_svc_handle_t svc_handle,
138                                        struct capref cap)
139{
140    struct user_st *user_st = dma_service_get_user_state(svc_handle);
141
142    XDMA_DEBUG("dma_svc_removeregion_cb user_st = %p\n", user_st);
143
144    return dma_mem_deregister(user_st->mem_mgr, cap);
145}
146
147/**
148 * \brief executes a DMA memcpy
149 *
150 * \param user_st   pointer to stored user state
151 * \param dst       the physical destination address
152 * \param src       the physical source address
153 * \param bytes     size of the transfer in bytes
154 * \param id        returns the DMA request ID of the transfer
155 *
156 * \returns SYS_ERR_OK if the memory region was removed
157 *          errval if the memory region removal was rejected
158 */
159static errval_t dma_svc_memcpy_cb(dma_svc_handle_t svc_handle,
160                                  lpaddr_t dst,
161                                  lpaddr_t src,
162                                  size_t bytes,
163                                  dma_req_id_t *id)
164{
165    errval_t err;
166
167    struct user_st *st = dma_service_get_user_state(svc_handle);
168
169    XDMA_DEBUG("dma_svc_memcpy_cb user_st = %p\n", st);
170
171    lpaddr_t dma_dst, dma_src;
172    err = dma_mem_verify(st->mem_mgr, dst, bytes, &dma_dst);
173    if (err_is_fail(err)) {
174        return err;
175    }
176    err = dma_mem_verify(st->mem_mgr, src, bytes, &dma_src);
177    if (err_is_fail(err)) {
178        return err;
179    }
180
181    XDMA_DEBUG("[%016lx]->[%016lx] of %lu bytes\n", dma_src, dma_dst, bytes);
182
183    /* both addresses are valid and have been translated now */
184    struct dma_device *dev = st->phi->dma;
185    assert(dev);
186
187    struct dma_req_setup setup = {
188        .type = DMA_REQ_TYPE_MEMCPY,
189        .done_cb = memcpy_req_cb,
190        .cb_arg = svc_handle,
191        .args = {
192            .memcpy = {
193                .src = dma_src,
194                .dst = dma_dst,
195                .bytes = bytes
196            }
197        }
198    };
199
200    return dma_request_memcpy(dev, &setup, id);
201}
202
203static struct dma_service_cb dma_svc_cb = {
204    .connect = dma_svc_connect_cb,
205    .addregion = dma_svc_addregion_cb,
206    .removeregion = dma_svc_removeregion_cb,
207    .memcpy = dma_svc_memcpy_cb
208};
209
210/**
211 * \brief initializes the Xeon Phi DMA devices and the service
212 *
213 * \param phi   Xeon Phi handle
214 *
215 * \returns SYS_ERR_OK on success
216 */
217errval_t xdma_service_init(struct xeon_phi *phi)
218{
219    errval_t err;
220
221    void *mmio_base = NULL;
222#ifdef __k1om__
223    mmio_base = (void *) (phi->mmio.vbase);
224#else
225    mmio_base = (void *) XEON_PHI_MMIO_TO_SBOX(phi);
226#endif
227
228    struct xeon_phi_dma_device *dev;
229    err = xeon_phi_dma_device_init(mmio_base, &dev);
230    if (err_is_fail(err)) {
231        return err;
232    }
233
234    phi->dma = (struct dma_device *)dev;
235
236#if DMA_BENCH_RUN_BENCHMARK
237    dma_bench_run_default_xphi((struct dma_device *)dev);
238#endif
239
240    iref_t svc_iref;
241    char svc_name[30];
242#ifdef __k1om__
243    snprintf(svc_name, 30, "%s", XEON_PHI_DMA_SERVICE_NAME);
244#else
245    snprintf(svc_name, 30, "%s.%u", XEON_PHI_DMA_SERVICE_NAME, phi->id);
246#endif
247    err = dma_service_init_with_name(svc_name, &dma_svc_cb, phi, &svc_iref);
248    if (err_is_fail(err)) {
249        USER_PANIC_ERR(err, "Failed to start the DMA service");
250    }
251
252    err = dma_manager_register_driver(0, 1ULL << 40, DMA_DEV_TYPE_XEON_PHI,
253                                      svc_iref);
254    if (err_is_fail(err)) {
255        USER_PANIC_ERR(err, "Failed to register with the DMA manager\n");
256    }
257
258    return SYS_ERR_OK;
259}
260
261