1/* 2 * Copyright (c) 2014 ETH Zurich. 3 * All rights reserved. 4 * 5 * This file is distributed under the terms in the attached LICENSE file. 6 * If you do not find this file, copies can be found by writing to: 7 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group. 8 */ 9#include <stdio.h> 10#include <barrelfish/barrelfish.h> 11#include <dma/xeon_phi/xeon_phi_dma.h> 12#include <dma/xeon_phi/xeon_phi_dma_device.h> 13#include <dma/xeon_phi/xeon_phi_dma_request.h> 14#include <dma/dma.h> 15#include <dma/dma_service.h> 16#include <dma/dma_mem_mgr.h> 17#include <dma/dma_manager_client.h> 18#include <dma/dma_bench.h> 19#include <xeon_phi/xeon_phi.h> 20 21#include "xeon_phi_internal.h" 22#include "dma_service.h" 23 24struct user_st 25{ 26 struct dma_mem_mgr *mem_mgr; 27 struct xeon_phi *phi; 28 uint64_t num_req; 29}; 30 31static void memcpy_req_cb(errval_t err, 32 dma_req_id_t id, 33 void *st) 34{ 35 XDMA_DEBUG("memcpy_req_cb %lx, %s\n", id, err_getstring(err)); 36 dma_service_send_done(st, err, id); 37} 38 39static lpaddr_t translate_address(void *arg, 40 lpaddr_t addr, 41 size_t size) 42{ 43#ifdef __k1om__ 44 if ((addr + size) > (1UL << 40)) { 45 return 0; 46 } 47 return addr; 48#else 49 struct xeon_phi *phi = ((struct user_st *)arg)->phi; 50 lpaddr_t apt_lo = phi->apt.pbase; 51 lpaddr_t apt_hi = phi->apt.pbase + phi->apt.bytes; 52 if ((addr >= apt_lo) && ((addr + size) <= apt_hi)) { 53 /* we are within the GDDR range */ 54 return addr - apt_lo; 55 } else if ((addr + size)< XEON_PHI_SYSMEM_SIZE) { 56 /* 57 * Xeon Phi does not support more host memory, 58 * assume host memory starts at 0x0 59 */ 60 return addr + XEON_PHI_SYSMEM_BASE; 61 } else { 62 return 0; 63 } 64#endif 65} 66 67/* 68 * --------------------------------------------------------------------------- 69 * Callbacks for the service events 70 * --------------------------------------------------------------------------- 71 */ 72 73/* 74 * \brief callback handler for new connect requests 75 * 76 * \param arg argument passed on the init function 77 * \param user_st pointer to store the user state 78 * 79 * \returns SYS_ERR_OK if the connection is accepted 80 * errval if connection is rejected 81 */ 82static errval_t dma_svc_connect_cb(void *arg, 83 void **user_st) 84{ 85 errval_t err; 86 87 struct user_st *st = calloc(1, sizeof(*st)); 88 if (st == NULL) { 89 return LIB_ERR_MALLOC_FAIL; 90 } 91 92 XDMA_DEBUG("dma_svc_connect_cb user_st = %p\n", st); 93 94 err = dma_mem_mgr_init(&st->mem_mgr, 0x0, (1UL << 48) - 1); 95 if (err_is_fail(err)) { 96 free(st); 97 return err; 98 } 99 100 st->phi = arg; 101 102 dma_mem_mgr_set_convert_fn(st->mem_mgr, translate_address, st); 103 104 *user_st = st; 105 106 return SYS_ERR_OK; 107} 108 109/** 110 * \brief registers a memory region with the client such that it can be used later 111 * 112 * \param user_st pointer to stored user state 113 * \param cap the capability to register 114 * 115 * \returns SYS_ERR_OK if the memory region was accepted 116 * errval if the memory region was rejected 117 */ 118static errval_t dma_svc_addregion_cb(dma_svc_handle_t svc_handle, 119 struct capref cap) 120{ 121 struct user_st *user_st = dma_service_get_user_state(svc_handle); 122 123 XDMA_DEBUG("dma_svc_addregion_cb user_st = %p\n", user_st); 124 125 return dma_mem_register(user_st->mem_mgr, cap); 126} 127 128/** 129 * \brief deregisters a memory region with the client 130 * 131 * \param user_st pointer to stored user state 132 * \param cap the capability to deregister 133 * 134 * \returns SYS_ERR_OK if the memory region was removed 135 * errval if the memory region removal was rejected 136 */ 137static errval_t dma_svc_removeregion_cb(dma_svc_handle_t svc_handle, 138 struct capref cap) 139{ 140 struct user_st *user_st = dma_service_get_user_state(svc_handle); 141 142 XDMA_DEBUG("dma_svc_removeregion_cb user_st = %p\n", user_st); 143 144 return dma_mem_deregister(user_st->mem_mgr, cap); 145} 146 147/** 148 * \brief executes a DMA memcpy 149 * 150 * \param user_st pointer to stored user state 151 * \param dst the physical destination address 152 * \param src the physical source address 153 * \param bytes size of the transfer in bytes 154 * \param id returns the DMA request ID of the transfer 155 * 156 * \returns SYS_ERR_OK if the memory region was removed 157 * errval if the memory region removal was rejected 158 */ 159static errval_t dma_svc_memcpy_cb(dma_svc_handle_t svc_handle, 160 lpaddr_t dst, 161 lpaddr_t src, 162 size_t bytes, 163 dma_req_id_t *id) 164{ 165 errval_t err; 166 167 struct user_st *st = dma_service_get_user_state(svc_handle); 168 169 XDMA_DEBUG("dma_svc_memcpy_cb user_st = %p\n", st); 170 171 lpaddr_t dma_dst, dma_src; 172 err = dma_mem_verify(st->mem_mgr, dst, bytes, &dma_dst); 173 if (err_is_fail(err)) { 174 return err; 175 } 176 err = dma_mem_verify(st->mem_mgr, src, bytes, &dma_src); 177 if (err_is_fail(err)) { 178 return err; 179 } 180 181 XDMA_DEBUG("[%016lx]->[%016lx] of %lu bytes\n", dma_src, dma_dst, bytes); 182 183 /* both addresses are valid and have been translated now */ 184 struct dma_device *dev = st->phi->dma; 185 assert(dev); 186 187 struct dma_req_setup setup = { 188 .type = DMA_REQ_TYPE_MEMCPY, 189 .done_cb = memcpy_req_cb, 190 .cb_arg = svc_handle, 191 .args = { 192 .memcpy = { 193 .src = dma_src, 194 .dst = dma_dst, 195 .bytes = bytes 196 } 197 } 198 }; 199 200 return dma_request_memcpy(dev, &setup, id); 201} 202 203static struct dma_service_cb dma_svc_cb = { 204 .connect = dma_svc_connect_cb, 205 .addregion = dma_svc_addregion_cb, 206 .removeregion = dma_svc_removeregion_cb, 207 .memcpy = dma_svc_memcpy_cb 208}; 209 210/** 211 * \brief initializes the Xeon Phi DMA devices and the service 212 * 213 * \param phi Xeon Phi handle 214 * 215 * \returns SYS_ERR_OK on success 216 */ 217errval_t xdma_service_init(struct xeon_phi *phi) 218{ 219 errval_t err; 220 221 void *mmio_base = NULL; 222#ifdef __k1om__ 223 mmio_base = (void *) (phi->mmio.vbase); 224#else 225 mmio_base = (void *) XEON_PHI_MMIO_TO_SBOX(phi); 226#endif 227 228 struct xeon_phi_dma_device *dev; 229 err = xeon_phi_dma_device_init(mmio_base, &dev); 230 if (err_is_fail(err)) { 231 return err; 232 } 233 234 phi->dma = (struct dma_device *)dev; 235 236#if DMA_BENCH_RUN_BENCHMARK 237 dma_bench_run_default_xphi((struct dma_device *)dev); 238#endif 239 240 iref_t svc_iref; 241 char svc_name[30]; 242#ifdef __k1om__ 243 snprintf(svc_name, 30, "%s", XEON_PHI_DMA_SERVICE_NAME); 244#else 245 snprintf(svc_name, 30, "%s.%u", XEON_PHI_DMA_SERVICE_NAME, phi->id); 246#endif 247 err = dma_service_init_with_name(svc_name, &dma_svc_cb, phi, &svc_iref); 248 if (err_is_fail(err)) { 249 USER_PANIC_ERR(err, "Failed to start the DMA service"); 250 } 251 252 err = dma_manager_register_driver(0, 1ULL << 40, DMA_DEV_TYPE_XEON_PHI, 253 svc_iref); 254 if (err_is_fail(err)) { 255 USER_PANIC_ERR(err, "Failed to register with the DMA manager\n"); 256 } 257 258 return SYS_ERR_OK; 259} 260 261