1/*
2 * Copyright (c) 2014, ETH Zurich. All rights reserved.
3 *
4 * This file is distributed under the terms in the attached LICENSE file.
5 * If you do not find this file, copies can be found by writing to:
6 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
7 */
8
9#include <string.h>
10#include <barrelfish/barrelfish.h>
11#include <xeon_phi/xeon_phi.h>
12#include <driverkit/iommu.h>
13#include <driverkit/hwmodel.h>
14
15#include <dma_mem_utils.h>
16
17#include <dma_internal.h>
18#include <dma_descriptor_internal.h>
19
20#include <debug.h>
21#include <dma_device_internal.h>
22
23/* helper macros */
24#define DMA_ALIGN(val, align) (((val) + (align)-1) & ~((align)-1))
25
26/* DMA Descriptor flags */
27
28/// this descriptor is valid
29#define DMA_DESC_FLAG_VALID     0x01
30
31/// this descriptor is currently used in a request
32#define DMA_DESC_FLAG_USED      0x02
33
34/// this is the first descriptor of a request
35#define DMA_DESC_FLAG_FIRST     0x04
36
37/// this is the last descriptor in a request
38#define DMA_DESC_FLAG_LAST      0x08
39
40/// the descriptor has not been processed yet
41#define DMA_DESC_FLAG_PROGRESS  0x10
42
43/// this descriptor is the head of the allocation unit (only this can be freed)
44#define DMA_DESC_FLAG_HEAD      0x20
45
46/// the descriptor has been executed
47#define DMA_DESC_FLAG_DONE      0x40
48
49/// there was an error during the execution of this descriptor
50#define DMA_DESC_FLAG_ERR       0x80
51
52/**
53 * DMA Descriptor Meta structure. This wraps around the hardware descriptor
54 * and is used to keep track of the descriptors.
55 */
56struct dma_descriptor
57{
58    lpaddr_t paddr;               ///< physical address of the descriptor
59    uint8_t *desc;                ///< virtual address of the descriptor
60    uint32_t flags;               ///< descriptor flags
61    struct dma_descriptor *next;  ///< next descriptor in the list
62    struct dma_request *req;      ///< pointer to the DMA request
63    struct dmem *mem;             ///< the dma memory information
64};
65
66/*
67 * ============================================================================
68 * Library Internal Interface
69 * ============================================================================
70 */
71
72/**
73 * \brief allocates a number of hardware DMA descriptors and fills them into the
74 *        array of descriptor pointers
75 *
76 * \param size  size of a signle descriptor in bytes
77 * \param align alignment constraints of the descriptors
78 * \param count number of descriptors to allocate in bits
79 * \param desc  pointer to the array of descriptor pointers
80 *
81 * \returns SYS_ERR_OK on success
82 *          errval on error
83 *
84 * NOTE: The descriptors are linked in a ring on a software level,
85 *       the corresonding hardware descriptors are not linked.
86 */
87errval_t dma_desc_alloc(uint32_t size,
88                        uint16_t align,
89                        uint8_t count,
90                        struct dma_device *dev,
91                        struct dma_descriptor **desc)
92{
93    errval_t err;
94
95    assert(desc);
96
97    size_t ndesc = (1 << count);
98
99    size = DMA_ALIGN(size, align);
100
101    struct dma_descriptor *dma_desc = calloc(ndesc, sizeof(*dma_desc));
102    if (dma_desc == NULL) {
103        return LIB_ERR_MALLOC_FAIL;
104    }
105
106    struct dmem *mem = calloc(1, sizeof(*mem));
107    if (mem == NULL) {
108        free(dma_desc);
109        return LIB_ERR_MALLOC_FAIL;
110    }
111
112#ifndef __k1om__
113    /*
114     * we set the ram affinity to the maximum range mapped by the system memory
115     * page tables when being on the host. Otherwise the card cannot access it.
116     */
117    uint64_t minbase, maxlimit;
118    ram_get_affinity(&minbase, &maxlimit);
119    ram_set_affinity(0, XEON_PHI_SYSMEM_SIZE-8*XEON_PHI_SYSMEM_PAGE_SIZE);
120#endif
121
122
123#ifdef XEON_PHI_USE_HW_MODEL
124    if (dev->convert) {
125        debug_printf("USING THE CONVERT FUNCTION\n");
126
127        int32_t nodes[3];
128        nodes[0] = dev->nodeid;
129        nodes[1] = driverkit_hwmodel_get_my_node_id();
130        nodes[2] = 0;
131        int32_t dest_nodeid = driverkit_hwmodel_lookup_dram_node_id();
132        err =  driverkit_hwmodel_frame_alloc(&mem->mem, ndesc * size,
133                                      dest_nodeid, nodes);
134
135        if (err_is_fail(err)) {
136            DEBUG_ERR(err, "failed");
137            free(dma_desc);
138            free(mem);
139            return err;
140        }
141
142
143        err = dev->convert(dev->convert_arg, mem->mem, &mem->devaddr, &mem->vbase);
144        if (err_is_fail(err)) {
145            DEBUG_ERR(err, "failed");
146            free(dma_desc);
147            free(mem);
148            return err;
149        }
150
151
152    } else {
153        err = driverkit_iommu_mmap_cl(dev->iommu, ndesc * size, DMA_DESC_MAP_FLAGS, mem);
154        if (err_is_fail(err)) {
155            free(dma_desc);
156            return err;
157        }
158    }
159#else
160    err = dma_mem_alloc(ndesc*size, DMA_DESC_MAP_FLAGS, dev->iommu, mem);
161    if (err_is_fail(err)) {
162        free(dma_desc);
163        free(mem);
164        return err;
165    }
166#endif
167
168
169#ifndef __k1om__
170    ram_set_affinity(minbase, maxlimit);
171#endif
172
173    memset((void*) mem->vbase, 0, ndesc * size);
174
175
176    DMADESC_DEBUG("Allocated frame of size %lu bytes @ [%016lx]\n",
177                  (uint64_t ) mem->size, mem->devaddr);
178
179    lpaddr_t desc_paddr = mem->devaddr;
180    uint8_t *desc_vaddr = (uint8_t*) mem->vbase;
181
182    /* set the last virtual address pointer */
183    dma_desc[ndesc - 1].desc = desc_vaddr + ((ndesc - 1) * size);
184
185    for (uint32_t i = 0; i < ndesc; ++i) {
186        /* initialize the fields */
187        dma_desc[i].desc = desc_vaddr;
188        dma_desc[i].paddr = desc_paddr;
189        dma_desc[i].req = NULL;
190        dma_desc[i].flags = DMA_DESC_FLAG_VALID;
191        dma_desc[i].mem = mem;
192
193        /* mark the first one */
194        if (i == 0) {
195            dma_desc[i].flags |= DMA_DESC_FLAG_HEAD;
196        }
197
198        /* do the linkage */
199        dma_desc[(i - 1) & (ndesc - 1)].next = &dma_desc[i];
200
201        /* set the entry in the array */
202        desc[i] = &dma_desc[i];
203
204        desc_vaddr += size;
205        desc_paddr += size;
206    }
207
208    DMADESC_DEBUG("Allocated %zu desc of size %u\n", ndesc, size);
209
210    return SYS_ERR_OK;
211}
212
213/**
214 * \brief brief frees up the array of previously allocated descriptors
215 *        and releases the resources
216 *
217 * \param desc  the descriptors to be freed
218 *
219 * \returns SYS_ERR_OK on success
220 *          errval on failure
221 */
222errval_t dma_desc_free(struct dma_descriptor *desc)
223{
224    errval_t err;
225
226    if (desc->flags & DMA_DESC_FLAG_HEAD) {
227        return DMA_ERR_ARG_INVALID;
228    }
229
230    struct dmem *mem = desc->mem;
231
232    err = driverkit_iommu_munmap(mem);
233    if (err_is_fail(err)) {
234        return err;
235    }
236
237    free(desc);
238    free(mem);
239
240    return SYS_ERR_OK;
241}
242
243/*
244 * ----------------------------------------------------------------------------
245 * Descriptor getters / setters
246 * ----------------------------------------------------------------------------
247 */
248
249/**
250 * \brief returns a virtual address pointer to the location where the descriptor
251 *        is mapped
252 *
253 * \param desc DMA descriptor
254 */
255inline uint8_t *dma_desc_get_desc_handle(struct dma_descriptor *desc)
256{
257    return desc->desc;
258}
259
260/**
261 * \brief returns the physical address of the descriptor
262 *
263 * \param desc DMA descriptor
264 *
265 * \returns physical address of the descriptor
266 */
267inline lpaddr_t dma_desc_get_paddr(struct dma_descriptor *desc)
268{
269    return desc->paddr;
270}
271
272/**
273 * \brief sets the corresponding request
274 *
275 * \param desc  DMA descriptor
276 */
277inline void dma_desc_set_request(struct dma_descriptor *desc,
278                                 struct dma_request *req)
279{
280    desc->req = req;
281}
282
283/*
284 * ============================================================================
285 * Public Interface
286 * ============================================================================
287 */
288
289/*
290 * ----------------------------------------------------------------------------
291 * Descriptor getters / setters
292 * ----------------------------------------------------------------------------
293 */
294
295/**
296 * \brief returns the corresponding DMA request this descriptor belongs
297 *
298 * \param desc DMA descriptor
299 *
300 * \brief pointer to the request
301 *        NULL if there is none
302 */
303inline struct dma_request *dma_desc_get_request(struct dma_descriptor *desc)
304{
305    return desc->req;
306}
307
308/**
309 * \brief returns a pointer to the next descriptor in a chain
310 *
311 * \param desc  DMA descriptor
312 *
313 * \returns next descriptor
314 *          NULL if the end of chain
315 */
316struct dma_descriptor *dma_desc_get_next(struct dma_descriptor *desc)
317{
318    return desc->next;
319}
320