1/* 2 * Copyright (c) 2014, ETH Zurich. All rights reserved. 3 * 4 * This file is distributed under the terms in the attached LICENSE file. 5 * If you do not find this file, copies can be found by writing to: 6 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group. 7 */ 8 9#include <string.h> 10#include <barrelfish/barrelfish.h> 11#include <xeon_phi/xeon_phi.h> 12#include <driverkit/iommu.h> 13#include <driverkit/hwmodel.h> 14 15#include <dma_mem_utils.h> 16 17#include <dma_internal.h> 18#include <dma_descriptor_internal.h> 19 20#include <debug.h> 21#include <dma_device_internal.h> 22 23/* helper macros */ 24#define DMA_ALIGN(val, align) (((val) + (align)-1) & ~((align)-1)) 25 26/* DMA Descriptor flags */ 27 28/// this descriptor is valid 29#define DMA_DESC_FLAG_VALID 0x01 30 31/// this descriptor is currently used in a request 32#define DMA_DESC_FLAG_USED 0x02 33 34/// this is the first descriptor of a request 35#define DMA_DESC_FLAG_FIRST 0x04 36 37/// this is the last descriptor in a request 38#define DMA_DESC_FLAG_LAST 0x08 39 40/// the descriptor has not been processed yet 41#define DMA_DESC_FLAG_PROGRESS 0x10 42 43/// this descriptor is the head of the allocation unit (only this can be freed) 44#define DMA_DESC_FLAG_HEAD 0x20 45 46/// the descriptor has been executed 47#define DMA_DESC_FLAG_DONE 0x40 48 49/// there was an error during the execution of this descriptor 50#define DMA_DESC_FLAG_ERR 0x80 51 52/** 53 * DMA Descriptor Meta structure. This wraps around the hardware descriptor 54 * and is used to keep track of the descriptors. 55 */ 56struct dma_descriptor 57{ 58 lpaddr_t paddr; ///< physical address of the descriptor 59 uint8_t *desc; ///< virtual address of the descriptor 60 uint32_t flags; ///< descriptor flags 61 struct dma_descriptor *next; ///< next descriptor in the list 62 struct dma_request *req; ///< pointer to the DMA request 63 struct dmem *mem; ///< the dma memory information 64}; 65 66/* 67 * ============================================================================ 68 * Library Internal Interface 69 * ============================================================================ 70 */ 71 72/** 73 * \brief allocates a number of hardware DMA descriptors and fills them into the 74 * array of descriptor pointers 75 * 76 * \param size size of a signle descriptor in bytes 77 * \param align alignment constraints of the descriptors 78 * \param count number of descriptors to allocate in bits 79 * \param desc pointer to the array of descriptor pointers 80 * 81 * \returns SYS_ERR_OK on success 82 * errval on error 83 * 84 * NOTE: The descriptors are linked in a ring on a software level, 85 * the corresonding hardware descriptors are not linked. 86 */ 87errval_t dma_desc_alloc(uint32_t size, 88 uint16_t align, 89 uint8_t count, 90 struct dma_device *dev, 91 struct dma_descriptor **desc) 92{ 93 errval_t err; 94 95 assert(desc); 96 97 size_t ndesc = (1 << count); 98 99 size = DMA_ALIGN(size, align); 100 101 struct dma_descriptor *dma_desc = calloc(ndesc, sizeof(*dma_desc)); 102 if (dma_desc == NULL) { 103 return LIB_ERR_MALLOC_FAIL; 104 } 105 106 struct dmem *mem = calloc(1, sizeof(*mem)); 107 if (mem == NULL) { 108 free(dma_desc); 109 return LIB_ERR_MALLOC_FAIL; 110 } 111 112#ifndef __k1om__ 113 /* 114 * we set the ram affinity to the maximum range mapped by the system memory 115 * page tables when being on the host. Otherwise the card cannot access it. 116 */ 117 uint64_t minbase, maxlimit; 118 ram_get_affinity(&minbase, &maxlimit); 119 ram_set_affinity(0, XEON_PHI_SYSMEM_SIZE-8*XEON_PHI_SYSMEM_PAGE_SIZE); 120#endif 121 122 123#ifdef XEON_PHI_USE_HW_MODEL 124 if (dev->convert) { 125 debug_printf("USING THE CONVERT FUNCTION\n"); 126 127 int32_t nodes[3]; 128 nodes[0] = dev->nodeid; 129 nodes[1] = driverkit_hwmodel_get_my_node_id(); 130 nodes[2] = 0; 131 int32_t dest_nodeid = driverkit_hwmodel_lookup_dram_node_id(); 132 err = driverkit_hwmodel_frame_alloc(&mem->mem, ndesc * size, 133 dest_nodeid, nodes); 134 135 if (err_is_fail(err)) { 136 DEBUG_ERR(err, "failed"); 137 free(dma_desc); 138 free(mem); 139 return err; 140 } 141 142 143 err = dev->convert(dev->convert_arg, mem->mem, &mem->devaddr, &mem->vbase); 144 if (err_is_fail(err)) { 145 DEBUG_ERR(err, "failed"); 146 free(dma_desc); 147 free(mem); 148 return err; 149 } 150 151 152 } else { 153 err = driverkit_iommu_mmap_cl(dev->iommu, ndesc * size, DMA_DESC_MAP_FLAGS, mem); 154 if (err_is_fail(err)) { 155 free(dma_desc); 156 return err; 157 } 158 } 159#else 160 err = dma_mem_alloc(ndesc*size, DMA_DESC_MAP_FLAGS, dev->iommu, mem); 161 if (err_is_fail(err)) { 162 free(dma_desc); 163 free(mem); 164 return err; 165 } 166#endif 167 168 169#ifndef __k1om__ 170 ram_set_affinity(minbase, maxlimit); 171#endif 172 173 memset((void*) mem->vbase, 0, ndesc * size); 174 175 176 DMADESC_DEBUG("Allocated frame of size %lu bytes @ [%016lx]\n", 177 (uint64_t ) mem->size, mem->devaddr); 178 179 lpaddr_t desc_paddr = mem->devaddr; 180 uint8_t *desc_vaddr = (uint8_t*) mem->vbase; 181 182 /* set the last virtual address pointer */ 183 dma_desc[ndesc - 1].desc = desc_vaddr + ((ndesc - 1) * size); 184 185 for (uint32_t i = 0; i < ndesc; ++i) { 186 /* initialize the fields */ 187 dma_desc[i].desc = desc_vaddr; 188 dma_desc[i].paddr = desc_paddr; 189 dma_desc[i].req = NULL; 190 dma_desc[i].flags = DMA_DESC_FLAG_VALID; 191 dma_desc[i].mem = mem; 192 193 /* mark the first one */ 194 if (i == 0) { 195 dma_desc[i].flags |= DMA_DESC_FLAG_HEAD; 196 } 197 198 /* do the linkage */ 199 dma_desc[(i - 1) & (ndesc - 1)].next = &dma_desc[i]; 200 201 /* set the entry in the array */ 202 desc[i] = &dma_desc[i]; 203 204 desc_vaddr += size; 205 desc_paddr += size; 206 } 207 208 DMADESC_DEBUG("Allocated %zu desc of size %u\n", ndesc, size); 209 210 return SYS_ERR_OK; 211} 212 213/** 214 * \brief brief frees up the array of previously allocated descriptors 215 * and releases the resources 216 * 217 * \param desc the descriptors to be freed 218 * 219 * \returns SYS_ERR_OK on success 220 * errval on failure 221 */ 222errval_t dma_desc_free(struct dma_descriptor *desc) 223{ 224 errval_t err; 225 226 if (desc->flags & DMA_DESC_FLAG_HEAD) { 227 return DMA_ERR_ARG_INVALID; 228 } 229 230 struct dmem *mem = desc->mem; 231 232 err = driverkit_iommu_munmap(mem); 233 if (err_is_fail(err)) { 234 return err; 235 } 236 237 free(desc); 238 free(mem); 239 240 return SYS_ERR_OK; 241} 242 243/* 244 * ---------------------------------------------------------------------------- 245 * Descriptor getters / setters 246 * ---------------------------------------------------------------------------- 247 */ 248 249/** 250 * \brief returns a virtual address pointer to the location where the descriptor 251 * is mapped 252 * 253 * \param desc DMA descriptor 254 */ 255inline uint8_t *dma_desc_get_desc_handle(struct dma_descriptor *desc) 256{ 257 return desc->desc; 258} 259 260/** 261 * \brief returns the physical address of the descriptor 262 * 263 * \param desc DMA descriptor 264 * 265 * \returns physical address of the descriptor 266 */ 267inline lpaddr_t dma_desc_get_paddr(struct dma_descriptor *desc) 268{ 269 return desc->paddr; 270} 271 272/** 273 * \brief sets the corresponding request 274 * 275 * \param desc DMA descriptor 276 */ 277inline void dma_desc_set_request(struct dma_descriptor *desc, 278 struct dma_request *req) 279{ 280 desc->req = req; 281} 282 283/* 284 * ============================================================================ 285 * Public Interface 286 * ============================================================================ 287 */ 288 289/* 290 * ---------------------------------------------------------------------------- 291 * Descriptor getters / setters 292 * ---------------------------------------------------------------------------- 293 */ 294 295/** 296 * \brief returns the corresponding DMA request this descriptor belongs 297 * 298 * \param desc DMA descriptor 299 * 300 * \brief pointer to the request 301 * NULL if there is none 302 */ 303inline struct dma_request *dma_desc_get_request(struct dma_descriptor *desc) 304{ 305 return desc->req; 306} 307 308/** 309 * \brief returns a pointer to the next descriptor in a chain 310 * 311 * \param desc DMA descriptor 312 * 313 * \returns next descriptor 314 * NULL if the end of chain 315 */ 316struct dma_descriptor *dma_desc_get_next(struct dma_descriptor *desc) 317{ 318 return desc->next; 319} 320