1/** 2 * \file 3 * \brief Driver module example template. 4 * 5 * In summary, every driver (struct bfdriver) shall implement the five 6 * life-cycle functions init/set_sleep_level/attach/detach/destroy 7 * (along with additional IRQ handler functions etc.). 8 * 9 * A driver module is linked with a driver domain (see main.c in this directory). 10 * At runtime, a driver domain will instantiate a driver instance (struct bfdriver_instance) 11 * of a given module (or class if you want) using the `init` function. During the lifetime 12 * of a driver instance it may be `detached` from and re-`attach`-ed to the 13 * device, set in different sleep levels, and finally it can be `destroy`-ed. 14 * 15 * For every driver instance (i.e., struct bfdriver_instance), a corresponding 16 * control interface created and exported. The interface is defined in dcontrol.if, 17 * the corresponding code is located in the driverkit library (dcontrol_service.c). 18 * 19 * \note For more information about driver domains check the main.c file in this 20 * directory. 21 */ 22/* 23 * Copyright (c) 2016, ETH Zurich. 24 * All rights reserved. 25 * 26 * This file is distributed under the terms in the attached LICENSE file. 27 * If you do not find this file, copies can be found by writing to: 28 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group. 29 */ 30 31#include <stdlib.h> 32#include <stdio.h> 33#include <string.h> 34#include <assert.h> 35 36#include <barrelfish/barrelfish.h> 37#include <driverkit/driverkit.h> 38#include <driverkit/iommu.h> 39 40#include <pci/pci.h> 41 42#include <dma/dma.h> 43#include <dma/dma_device.h> 44#include <dma/dma_request.h> 45#include <dma/ioat/ioat_dma.h> 46#include <dma/ioat/ioat_dma_device.h> 47#include <dma/ioat/ioat_dma_request.h> 48 49#include "device.h" 50#include "ioat_mgr_service.h" 51#include "debug.h" 52 53static uint8_t device_next = 0; 54static uint8_t device_count = 0; 55struct ioat_dma_device **devices = NULL; 56 57#if 0 58static void handle_device_interrupt(void *arg) 59{ 60 61// struct ioat_dma_device *dev = *((struct ioat_dma_device **) arg); 62// struct dma_device *dma_dev = (struct dma_device *) dev; 63 64 INTR_DEBUG("interrupt! device %u", dma_device_get_id(arg)); 65 66} 67#endif 68 69struct ioat_dma_device *ioat_device_get_next(void) 70{ 71 if (device_next >= device_count) { 72 device_next = 0; 73 } 74 return devices[device_next++]; 75} 76 77errval_t ioat_device_poll(void) 78{ 79 errval_t err; 80 81 uint8_t idle = 0x1; 82 for (uint8_t i = 0; i < device_count; ++i) { 83 err = ioat_dma_device_poll_channels((struct dma_device *)devices[i]); 84 switch (err_no(err)) { 85 case SYS_ERR_OK: 86 idle = 0; 87 break; 88 case DMA_ERR_DEVICE_IDLE: 89 break; 90 default: 91 return err; 92 } 93 } 94 if (idle) { 95 return DMA_ERR_DEVICE_IDLE; 96 } 97 return SYS_ERR_OK; 98} 99 100 101#define OSDI18_RUN_BENCHMARK 1 102#include <dma/dma_bench.h> 103#include <skb/skb.h> 104#include <bench/bench.h> 105 106#define BUFFER_SIZE (1UL << 20) 107#if OSDI18_RUN_BENCHMARK 108 109static void cpumemcpy(genvaddr_t to, genvaddr_t from, size_t bytes) 110{ 111 cycles_t t = bench_tsc(); 112 memcpy((void *)to, (void *)from, bytes); 113 t = bench_tsc() - t; 114 115 debug_printf("cpumemcpy Elapsed: %lu\n", t); 116} 117 118 119static volatile bool done = false; 120 121static void impl_test_cb(errval_t err, dma_req_id_t id, void *arg) 122{ 123 debug_printf("impl_test_cb\n"); 124 // assert(memcmp(arg, arg + BUFFER_SIZE, BUFFER_SIZE) == 0); 125 debug_printf("test ok\n"); 126 127 done = 1; 128} 129 130static struct dma_device *dma_dev; 131 132static void dmamemcpy(genvaddr_t to, genvaddr_t from, size_t bytes) 133{ 134 errval_t err; 135 136 struct dma_req_setup setup = { 137 .args.memcpy = { 138 .src = (genvaddr_t) from, 139 .dst = (genvaddr_t) to, 140 .bytes = bytes, 141 }, 142 .type = DMA_REQ_TYPE_MEMCPY, 143 .done_cb = impl_test_cb, 144 .cb_arg = (void *)to 145 }; 146 done = false; 147 cycles_t t = bench_tsc(); 148 149 dma_req_id_t rid; 150 err = ioat_dma_request_memcpy(dma_dev, &setup, &rid); 151 if (err_is_fail(err)) { 152 USER_PANIC_ERR(err, "failed to setup transfer\n"); 153 } 154 155 while(!done) { 156 while(done == 0) { 157 err = ioat_dma_device_poll_channels(dma_dev); 158 switch (err_no(err)) { 159 case DMA_ERR_DEVICE_IDLE : 160 case DMA_ERR_CHAN_IDLE: 161 case SYS_ERR_OK: 162 break; 163 default: 164 USER_PANIC_ERR(err, "failed to poll the channel!\n"); 165 } 166 } 167 } 168 169 t = bench_tsc() - t; 170 debug_printf("dmamemcpy Elapsed: %lu\n", t); 171} 172 173static void osdi18_benchmark(struct ioat_dma_device *dev, struct iommu_client *cl) 174{ 175 errval_t err; 176 177 debug_printf("========================================================\n"); 178 debug_printf("BENCHMARK FOR OSDI18\n"); 179 debug_printf("========================================================\n"); 180 181 182 bench_init(); 183 184 dma_dev = (struct dma_device *)dev; 185 186 187 struct dmem mem; 188 err = driverkit_iommu_mmap_cl(cl, 2 * BUFFER_SIZE, VREGION_FLAGS_READ_WRITE, 189 &mem); 190 if (err_is_fail(err)) { 191 USER_PANIC_ERR(err, "failed to get memory"); 192 } 193 194 cpumemcpy(mem.vbase, mem.vbase + BUFFER_SIZE, BUFFER_SIZE); 195 196 dmamemcpy(mem.devaddr, mem.devaddr + BUFFER_SIZE, BUFFER_SIZE); 197 198} 199#endif 200 201//#define TEST_IMPLEMENTATION 1 202#if TEST_IMPLEMENTATION 203#include <dma/dma_bench.h> 204#include <skb/skb.h> 205 206 207#define BUFFER_SIZE (1<<20) 208 209uint32_t done = 0; 210 211static void impl_test_cb(errval_t err, dma_req_id_t id, void *arg) 212{ 213 debug_printf("impl_test_cb\n"); 214 assert(memcmp(arg, arg + BUFFER_SIZE, BUFFER_SIZE) == 0); 215 debug_printf("test ok\n"); 216 217 done = 1; 218} 219 220static void impl_test(struct ioat_dma_device *dev, struct iommu_client *cl) 221{ 222 errval_t err; 223 224 debug_printf("Doing an implementation test\n"); 225 226 struct dmem mem; 227 err = driverkit_iommu_mmap_cl(cl, 2 * BUFFER_SIZE, VREGION_FLAGS_READ_WRITE, 228 &mem); 229 if (err_is_fail(err)) { 230 DEBUG_ERR(err, "failed to get memory"); 231 } 232 233 void *buf = (void *)mem.vbase; 234 memset(buf, 0, mem.size); 235 memset(buf, 0xA5, BUFFER_SIZE); 236 237 struct dma_req_setup setup = { 238 .args.memcpy = { 239 .src = mem.devaddr, 240 .dst = mem.devaddr + BUFFER_SIZE, 241 .bytes = BUFFER_SIZE, 242 }, 243 .type = DMA_REQ_TYPE_MEMCPY, 244 .done_cb = impl_test_cb, 245 .cb_arg = buf 246 }; 247 int reps = 10; 248 do { 249 memset(buf, 0, mem.size); 250 memset(buf, reps + 2, BUFFER_SIZE); 251 assert(memcmp(buf, buf + BUFFER_SIZE, BUFFER_SIZE)); 252 253 debug_printf("!!!!!! NEW ROUND\n"); 254 dma_req_id_t rid; 255 err = ioat_dma_request_memcpy((struct dma_device *)dev, &setup, &rid); 256 assert(err_is_ok(err)); 257 258 done = 0; 259 while(done == 0) { 260 err = ioat_dma_device_poll_channels((struct dma_device *)dev); 261 switch (err_no(err)) { 262 case DMA_ERR_DEVICE_IDLE : 263 case DMA_ERR_CHAN_IDLE: 264 case SYS_ERR_OK: 265 break; 266 default: 267 DEBUG_ERR(err, "failed to poll the channel!\n"); 268 } 269 } 270#if 0 271 if (reps == 1) { 272 debug_printf("using phys addr!\n"); 273 setup.args.memcpy.src = id.base; 274 setup.args.memcpy.dst = id.base + BUFFER_SIZE; 275 } 276#endif 277 278 } while(reps--); 279 280 281} 282#endif 283 284/** 285 * Driver initialization function. This function is called by the driver domain 286 * (see also 'create_handler' in ddomain_service.c). 287 * Typically through a request from the device manager. 288 * 289 * The init function is supposed to set `dev` to the exported service iref. 290 * The init function may use `bfi->dstate` to store additional state about the device. 291 * 292 * \param[in] bfi The instance of this driver. 293 * \param[in] name The name of this driver instance. 294 * \param[in] flags Additional flags (The exact flags supported is device/driver specific). 295 * \param[in] c Capabilities (for registers etc.) as provided by the device manager. 296 * The exact layout of the `c` is device specific. 297 * \param[out] dev The service iref over which the device can be contacted. 298 * 299 * \retval SYS_ERR_OK Device initialized successfully. 300 * \retval LIB_ERR_MALLOC_FAIL Unable to allocate memory for the driver. 301 */ 302static errval_t init(struct bfdriver_instance *bfi, uint64_t flags, iref_t* dev) { 303 304 errval_t err; 305 // 1. Initialize the device: 306 307 debug_printf("[ioat]: attaching device '%s'\n", bfi->name); 308 309 struct ioat_dma_device **devices_new; 310 devices_new = realloc(devices, (device_count + 1) * sizeof(void *)); 311 if (devices_new == NULL) { 312 return LIB_ERR_MALLOC_FAIL; 313 } 314 devices = devices_new; 315 316 struct capref iommuep; 317 err = driverkit_get_iommu_cap(bfi, &iommuep); 318 if (err_is_fail(err)) { 319 return err; 320 } 321 322 //driverkit_iommu_vspace_set_default_policy(IOMMU_VSPACE_POLICY_SHARED); 323 324 struct iommu_client *cl; 325 err = driverkit_iommu_client_init_cl(iommuep, &cl); 326 if (err_is_fail(err)) { 327 USER_PANIC_ERR(err, "Failed to initialize the IOMMU library"); 328 } 329 assert(cl); 330 331 debug_printf("IOMMU PRESENT: %u\n", driverkit_iommu_present(cl)); 332 333 334 struct capref regs; 335 err = driverkit_get_bar_cap(bfi, 0, ®s); 336 if (err_is_fail(err)) { 337 return err; 338 } 339 340 /* initialize the device */ 341 //err = ioat_dma_device_init(regs, &pc1, false, &devices[device_count]); 342 err = ioat_dma_device_init(regs, cl, &devices[device_count]); 343 if (err_is_fail(err)) { 344 345 DEV_ERR("Could not initialize the device: %s\n", err_getstring(err)); 346 return SYS_ERR_OK; 347 } 348 349 #if TEST_IMPLEMENTATION 350 impl_test(devices[device_count], cl); 351 #endif 352 353 #if OSDI18_RUN_BENCHMARK 354 osdi18_benchmark(devices[device_count], cl); 355 #endif 356 357 device_count++; 358 359 360 // 2. Export service to talk to the device: 361 362 // 3. Set iref of your exported service (this is reported back to Kaluga) 363 *dev = 0x00; 364 365 366 return SYS_ERR_OK; 367} 368 369/** 370 * Instructs driver to attach to the device. 371 * This function is only called if the driver has previously detached 372 * from the device (see also detach). 373 * 374 * \note After detachment the driver can not assume anything about the 375 * configuration of the device. 376 * 377 * \param[in] bfi The instance of this driver. 378 * \retval SYS_ERR_OK Device initialized successfully. 379 */ 380static errval_t attach(struct bfdriver_instance* bfi) { 381 382 return SYS_ERR_OK; 383} 384 385/** 386 * Instructs driver to detach from the device. 387 * The driver must yield any control over to the device after this function returns. 388 * The device may be left in any state. 389 * 390 * \param[in] bfi The instance of this driver. 391 * \retval SYS_ERR_OK Device initialized successfully. 392 */ 393static errval_t detach(struct bfdriver_instance* bfi) { 394 395 return SYS_ERR_OK; 396} 397 398/** 399 * Instructs the driver to go in a particular sleep state. 400 * Supported states are platform/device specific. 401 * 402 * \param[in] bfi The instance of this driver. 403 * \retval SYS_ERR_OK Device initialized successfully. 404 */ 405static errval_t set_sleep_level(struct bfdriver_instance* bfi, uint32_t level) { 406 407 return SYS_ERR_OK; 408} 409 410/** 411 * Destroys this driver instance. The driver will yield any 412 * control over the device and free any state allocated. 413 * 414 * \param[in] bfi The instance of this driver. 415 * \retval SYS_ERR_OK Device initialized successfully. 416 */ 417static errval_t destroy(struct bfdriver_instance* bfi) { 418 419 // XXX: Tear-down the service 420 bfi->device = 0x0; 421 422 return SYS_ERR_OK; 423} 424 425 426 427static errval_t get_ep(struct bfdriver_instance* bfi, bool lmp, struct capref* ret_cap) 428{ 429 return SYS_ERR_OK; 430} 431/** 432 * Registers the driver module with the system. 433 * 434 * To link this particular module in your driver domain, 435 * add it to the addModules list in the Hakefile. 436 */ 437DEFINE_MODULE(ioat_dma_module, init, attach, detach, set_sleep_level, destroy, get_ep); 438