1/*
2 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#include <stdlib.h>
11#include <stdio.h>
12//#include <time.h>
13#include <barrelfish/barrelfish.h>
14#include <barrelfish/sys_debug.h>
15#include <barrelfish/deferred.h>
16#include <devif/queue_interface.h>
17#include <devif/backends/loopback_devif.h>
18#include <devif/backends/debug.h>
19#include <devif/backends/null.h>
20#include <bench/bench.h>
21#include <vfs/vfs.h>
22
23//#define DEBUG(x...) printf("devif_test: " x)
24#define DEBUG(x...) do {} while (0)
25
26#define BUF_SIZE 2048
27#define NUM_BUFS 128
28#define MEMORY_SIZE BUF_SIZE*NUM_BUFS
29
30#define NUM_REGIONS 128
31#define NUM_ROUNDS 100000
32
33static struct capref memory;
34static regionid_t regid;
35static struct frame_identity id;
36static lpaddr_t phys;
37
38static void* va;
39
40struct direct_state {
41    struct list_ele* first;
42    struct list_ele* last;
43};
44
45struct list_ele{
46    regionid_t rid;
47    bufferid_t bid;
48    lpaddr_t addr;
49    size_t len;
50    uint64_t flags;
51
52    struct list_ele* next;
53};
54
55static struct loopback_queue* queue;
56static struct debug_q* debug_q;
57static struct null_q* null_q;
58//static struct null_q* null_q2;
59static struct null_q* all_q;
60static struct devq* que;
61
62static cycles_t tot_deq = 0;
63static cycles_t tot_enq = 0;
64static cycles_t tot_reg = 0;
65static cycles_t tot_dereg = 0;
66static cycles_t start_enq = 0, end_enq = 0;
67static cycles_t start_deq = 0, end_deq = 0;
68static cycles_t start_reg = 0, end_reg = 0;
69static cycles_t start_dereg = 0, end_dereg = 0;
70
71static double avg_deq, avg_enq;
72static double avg_reg, avg_dereg;
73static double avg_deq_d, avg_enq_d;
74static double avg_reg_d, avg_dereg_d;
75static double avg_deq_n, avg_enq_n;
76static double avg_reg_n, avg_dereg_n;
77static double avg_deq_tot, avg_enq_tot;
78static double avg_reg_tot, avg_dereg_tot;
79
80static bench_ctl_t *ctl_tmp_en;
81static bench_ctl_t *ctl_tmp_de;
82static bench_ctl_t *ctl_tmp_reg;
83static bench_ctl_t *ctl_tmp_dereg;
84
85static cycles_t tscperus;
86static char* machine_name;
87
88/*
89static void dump_results_nfs(char* filename, bool destroy)
90{
91    errval_t err;
92    char buffer[256];
93    vfs_handle_t handle;
94#ifdef BENCH_DEVQ
95    //sprintf(buffer, "/nfs/%s/%s_devq.csv", machine_name, filename);
96    sprintf(buffer, "/nfs/%s_devq.csv", filename);
97#else
98    //sprintf(buffer, "/nfs/%s/%s.csv", machine_name, filename);
99    sprintf(buffer, "/nfs/%s.csv", filename);
100#endif
101
102    debug_printf("%s \n", buffer);
103    err = vfs_open(buffer, &handle);
104    if (err_is_fail(err)) {
105        err = vfs_create(buffer, &handle);
106        assert(err_is_ok(err));
107    }
108
109    size_t bytes;
110    // first 10 % is warmup
111    for (int i = NUM_ROUNDS/10 ; i < NUM_ROUNDS; i++) {
112        sprintf(buffer, "enqueue,%lu \ndequeue,%lu \nregister,%lu \nderegister,%lu \n",
113                ctl_tmp_en->data[i], ctl_tmp_de->data[i], ctl_tmp_reg->data[i],
114                ctl_tmp_dereg->data[i]);
115        err = vfs_write(handle, buffer, strlen(buffer), &bytes);
116        assert(err_is_ok(err));
117        assert(bytes == strlen(buffer));
118    }
119
120    err = vfs_close(handle);
121    assert(err_is_ok(err));
122
123    if (destroy) {
124        bench_ctl_destroy(ctl_tmp_dereg);
125        bench_ctl_destroy(ctl_tmp_reg);
126        bench_ctl_destroy(ctl_tmp_de);
127        bench_ctl_destroy(ctl_tmp_en);
128    }
129}
130*/
131
132static void dump_results_nfs(char* prefix, bool destroy)
133//static void dump_results_console(char* prefix, bool destroy)
134{
135    char buffer[512];
136
137    // first 10 % is warmup
138    if (ctl_tmp_en == NULL) {
139        return;
140    }
141
142    for (int i = NUM_ROUNDS/10 ; i < NUM_ROUNDS; i++) {
143#ifdef BENCH_DEVQ
144        sprintf(buffer, ";%s_devq;enqueue;%lu;dequeue;%lu;register;%lu;deregister;%lu \n",
145#else
146        sprintf(buffer, ";%s;enqueue;%lu;dequeue;%lu;register;%lu;deregister;%lu \n",
147#endif
148                prefix,
149                ctl_tmp_en->data[i], ctl_tmp_de->data[i], ctl_tmp_reg->data[i],
150                ctl_tmp_dereg->data[i]);
151        printf("%s", buffer);
152    }
153
154    if (destroy) {
155        bench_ctl_destroy(ctl_tmp_dereg);
156        bench_ctl_destroy(ctl_tmp_reg);
157        bench_ctl_destroy(ctl_tmp_de);
158        bench_ctl_destroy(ctl_tmp_en);
159    }
160}
161
162static void test_register(void)
163{
164    errval_t err;
165    struct capref regions[NUM_REGIONS];
166    regionid_t rids[NUM_REGIONS];
167
168    tot_reg = 0;
169    tot_dereg = 0;
170
171    ctl_tmp_reg = calloc(1, sizeof(*ctl_tmp_reg));
172    ctl_tmp_reg->mode = BENCH_MODE_FIXEDRUNS;
173    ctl_tmp_reg->result_dimensions = 1;
174    ctl_tmp_reg->min_runs = NUM_ROUNDS;
175    ctl_tmp_reg->data = calloc(ctl_tmp_reg->min_runs * ctl_tmp_reg->result_dimensions,
176                       sizeof(*ctl_tmp_de->data));
177
178    ctl_tmp_dereg = calloc(1, sizeof(*ctl_tmp_dereg));
179    ctl_tmp_dereg->mode = BENCH_MODE_FIXEDRUNS;
180    ctl_tmp_dereg->result_dimensions = 1;
181    ctl_tmp_dereg->min_runs = NUM_ROUNDS;
182    ctl_tmp_dereg->data = calloc(ctl_tmp_dereg->min_runs * ctl_tmp_dereg->result_dimensions,
183                       sizeof(*ctl_tmp_dereg->data));
184
185    for (int i = 0; i < NUM_REGIONS; i++) {
186        err = frame_alloc(&regions[i], BASE_PAGE_SIZE, NULL);
187        if (err_is_fail(err)){
188            USER_PANIC("Allocating cap failed \n");
189        }
190    }
191
192    srand(rdtscp());
193    int idx = 0;
194    struct capref ret;
195    cycles_t res1;
196    cycles_t res2;
197    for (int i = 0; i < NUM_ROUNDS; i++) {
198        idx = i % NUM_REGIONS;
199        start_reg = rdtscp();
200        err = devq_register(que, regions[idx], &rids[idx]);
201        end_reg = rdtscp();
202        if (err_is_fail(err)){
203            USER_PANIC("Registering memory to devq failed: %s \n",
204                        err_getstring(err));
205        } else {
206            tot_reg += end_reg - start_reg;
207            res1 = end_reg - start_reg;
208            bench_ctl_add_run(ctl_tmp_reg, &res1);
209        }
210
211        start_dereg = rdtscp();
212        err = devq_deregister(que, rids[idx], &ret);
213        end_dereg = rdtscp();
214        if (err_is_fail(err)){
215            USER_PANIC("Registering memory to devq failed: %s\n",
216                       err_getstring(err));
217        } else {
218            tot_dereg += end_dereg - start_dereg;
219            res2 = end_dereg - start_dereg;
220            bench_ctl_add_run(ctl_tmp_dereg, &res2);
221        }
222    }
223
224    for (int i = 0; i < NUM_REGIONS; i++) {
225        err = cap_destroy(regions[i]);
226        if (err_is_fail(err)){
227            USER_PANIC("Destroy region failed: %s\n",
228                       err_getstring(err));
229        }
230    }
231
232    //bench_ctl_dump_analysis(ctl_tmp_dereg, 0, "deregister", tscperus);
233    //bench_ctl_dump_analysis(ctl_tmp_reg, 0, "register", tscperus);
234
235}
236
237static void test_randomized_test(void)
238{
239    errval_t err;
240    regionid_t rid;
241    genoffset_t offset;
242    genoffset_t length;
243    genoffset_t valid_data;
244    genoffset_t valid_length;
245    uint64_t flags;
246
247    tot_enq = 0;
248    tot_deq = 0;
249
250    ctl_tmp_en = calloc(1, sizeof(*ctl_tmp_en));
251    ctl_tmp_en->mode = BENCH_MODE_FIXEDRUNS;
252    ctl_tmp_en->result_dimensions = 1;
253    ctl_tmp_en->min_runs = NUM_ROUNDS;
254    ctl_tmp_en->data = calloc(ctl_tmp_en->min_runs * ctl_tmp_en->result_dimensions,
255                       sizeof(*ctl_tmp_de->data));
256
257    ctl_tmp_de = calloc(1, sizeof(*ctl_tmp_en));
258    ctl_tmp_de->mode = BENCH_MODE_FIXEDRUNS;
259    ctl_tmp_de->result_dimensions = 1;
260    ctl_tmp_de->min_runs = NUM_ROUNDS;
261    ctl_tmp_de->data = calloc(ctl_tmp_de->min_runs * ctl_tmp_de->result_dimensions,
262                       sizeof(*ctl_tmp_de->data));
263
264    srand(rdtscp());
265    int idx = 0;
266    cycles_t res;
267    // enqueue from the beginning of the region
268    for (int i = 0; i < NUM_ROUNDS; i++) {
269
270        idx = i % NUM_BUFS;
271        start_enq = rdtscp();
272
273        err = devq_enqueue(que, regid, idx*BUF_SIZE, BUF_SIZE,
274                           0, BUF_SIZE, 0);
275
276        end_enq = rdtscp();
277        if (err_is_fail(err)){
278            USER_PANIC("Enqueue failed: %s \n", err_getstring(err));
279        } else {
280            tot_enq += end_enq - start_enq;
281            res = end_enq - start_enq;
282            bench_ctl_add_run(ctl_tmp_de, &res);
283        }
284
285        start_deq = rdtscp();
286        err = devq_dequeue(que, &rid, &offset, &length, &valid_data,
287                           &valid_length, &flags);
288        end_deq = rdtscp();
289        if (err_is_ok(err)){
290            tot_deq += end_deq - start_deq;
291            res = end_deq - start_deq;
292            bench_ctl_add_run(ctl_tmp_en, &res);
293        } else {
294            USER_PANIC("Dequeue failed: %s \n", err_getstring(err));
295        }
296    }
297
298    //bench_ctl_dump_analysis(ctl_tmp_de, 0, "enqueue", tscperus);
299    //bench_ctl_dump_analysis(ctl_tmp_en, 0, "dequeue", tscperus);
300}
301
302int main(int argc, char *argv[])
303{
304
305    if (argc > 1) {
306        machine_name = argv[1];
307    } else {
308        machine_name = "default";
309    }
310
311    errval_t err;
312
313    // mount_vfs
314    vfs_init();
315
316    char fname[256];
317    err = vfs_mount("/nfs", "nfs://10.110.4.4/mnt/local/nfs/haeckir");
318    if(err_is_fail(err)) {
319        USER_PANIC("vfs_mount: %s \n", err_getstring(err));
320    }
321
322    sprintf(fname, "/nfs/%s", machine_name);
323    err = vfs_mkdir(fname);
324    if (err_is_fail(err)) {
325        printf("Folder %s already exists \n", fname);
326    } else {
327        printf("Creating folder %s \n", fname);
328    }
329
330    // Allocate memory
331    err = frame_alloc(&memory, MEMORY_SIZE, NULL);
332    if (err_is_fail(err)){
333        USER_PANIC("Allocating cap failed \n");
334    }
335
336    // RX frame
337    err = frame_identify(memory, &id);
338    if (err_is_fail(err)) {
339        USER_PANIC("Frame identify failed \n");
340    }
341
342    err = vspace_map_one_frame_attr(&va, id.bytes, memory,
343                                    VREGION_FLAGS_READ, NULL, NULL);
344    if (err_is_fail(err)) {
345        USER_PANIC("Frame mapping failed \n");
346    }
347
348    phys = id.base;
349
350    debug_printf("Descriptor queue test started \n");
351    err = loopback_queue_create(&queue);
352    if (err_is_fail(err)){
353        USER_PANIC("Allocating devq failed \n");
354    }
355
356    // stack debug queue on top
357    err = debug_create(&debug_q, (struct devq*) queue);
358    if (err_is_fail(err)) {
359        USER_PANIC("Allocating debug q failed \n");
360    }
361
362    // stack null queue on top
363    err = null_create(&null_q, (struct devq*) queue);
364    if (err_is_fail(err)) {
365        USER_PANIC("Allocating null q failed \n");
366    }
367
368    // stack null queue on top of debug queue
369    err = null_create(&all_q, (struct devq*) debug_q);
370    if (err_is_fail(err)) {
371        USER_PANIC("Allocating null q failed \n");
372    }
373
374
375    que = (struct devq*) queue;
376
377    err = devq_register(que, memory, &regid);
378    if (err_is_fail(err)){
379        USER_PANIC("Registering memory to devq failed \n");
380    }
381
382    err = sys_debug_get_tsc_per_ms(&tscperus);
383    assert(err_is_ok(err));
384    tscperus /= 1000;
385
386    printf("Starting randomized queue\n");
387    que = (struct devq*) queue;
388
389    test_register();
390
391    test_randomized_test();
392
393    dump_results_nfs("Loopback", true);
394    avg_enq = ((double) tot_deq)/NUM_ROUNDS;
395    avg_deq = ((double) tot_enq)/NUM_ROUNDS;
396    avg_dereg = ((double) tot_dereg)/NUM_ROUNDS;
397    avg_reg = ((double) tot_reg)/NUM_ROUNDS;
398
399    /*
400    ctl_tmp_en = devq_get_benchmark_data(que, 0);
401    ctl_tmp_de = devq_get_benchmark_data(que, 1);
402    ctl_tmp_reg = devq_get_benchmark_data(que, 2);
403    ctl_tmp_dereg = devq_get_benchmark_data(que, 3);
404    dump_results_nfs("Loopback_bcalls", false);
405    */
406/*
407    printf("AVG enq %f \n", avg_enq);
408    printf("AVG deq %f \n", avg_deq);
409    printf("AVG reg %f \n", avg_reg);
410    printf("AVG dereg %f \n", avg_dereg);
411*/
412    printf("############################################################ \n");
413
414    err = devq_deregister(que, regid, &memory);
415    if (err_is_fail(err)){
416        USER_PANIC("Deregistering memory from devq failed: %s \n",
417                   err_getstring(err));
418    }
419
420    printf("Starting randomized test debug\n");
421    que = (struct devq*) debug_q;
422
423    test_register();
424
425    err = devq_register(que, memory, &regid);
426    if (err_is_fail(err)){
427        USER_PANIC("Registering memory to devq failed \n");
428    }
429
430    test_randomized_test();
431
432    dump_results_nfs("Overhead Debug", true);
433    avg_enq_d = ((double) tot_deq)/NUM_ROUNDS;
434    avg_deq_d = ((double) tot_enq)/NUM_ROUNDS;
435    avg_dereg_d  = ((double) tot_dereg)/NUM_ROUNDS;
436    avg_reg_d = ((double) tot_reg)/NUM_ROUNDS;
437/*
438    printf("AVG enq debug %f \n", avg_enq_d);
439    printf("AVG deq debug %f \n", avg_deq_d);
440    printf("AVG reg debug %f \n", avg_reg_d);
441    printf("AVG dereg debug %f \n", avg_dereg_d);
442*/
443    printf("############################################################ \n");
444
445    err = devq_deregister(que, regid, &memory);
446    if (err_is_fail(err)){
447        USER_PANIC("Deregistering memory from devq failed: %s \n",
448                   err_getstring(err));
449    }
450
451    printf("Starting randomized test null\n");
452    que = (struct devq*) null_q;
453
454    err = devq_register(que, memory, &regid);
455    if (err_is_fail(err)){
456        USER_PANIC("Registering memory to devq failed \n");
457    }
458
459    test_register();
460
461    test_randomized_test();
462
463    dump_results_nfs("Overhead Null", true);
464    avg_enq_n = ((double) tot_enq)/NUM_ROUNDS;
465    avg_deq_n = ((double) tot_deq)/NUM_ROUNDS;
466    avg_reg_n = ((double) tot_reg)/NUM_ROUNDS;
467    avg_dereg_n = ((double) tot_dereg)/NUM_ROUNDS;
468
469    printf("############################################################ \n");
470
471
472    err = devq_deregister(que, regid, &memory);
473    if (err_is_fail(err)){
474        USER_PANIC("Deregistering memory from devq failed: %s \n",
475                   err_getstring(err));
476    }
477
478    printf("Starting randomized test all queues\n");
479    que = (struct devq*) all_q;
480
481    test_register();
482
483    err = devq_register(que, memory, &regid);
484    if (err_is_fail(err)){
485        USER_PANIC("Registering memory to devq failed \n");
486    }
487
488    test_randomized_test();
489
490    dump_results_nfs("Overhead Null and Debug", true);
491    avg_enq_tot = ((double) tot_enq)/NUM_ROUNDS;
492    avg_deq_tot = ((double) tot_deq)/NUM_ROUNDS;
493    avg_reg_tot = ((double) tot_reg)/NUM_ROUNDS;
494    avg_dereg_tot = ((double) tot_dereg)/NUM_ROUNDS;
495
496/*
497    printf("AVG enq null %f \n", avg_enq_n);
498    printf("AVG deq null %f \n", avg_deq_n);
499    printf("AVG reg null %f \n", avg_reg_n);
500    printf("AVG dereg null %f \n", avg_dereg_n);
501    printf("############################################################ \n");
502
503    printf("AVG enq overhead null %f \n", avg_enq_n - avg_enq);
504    printf("AVG deq overhead null %f \n", avg_deq_n - avg_deq);
505    printf("AVG reg overhead null %f \n", avg_reg_n - avg_reg);
506    printf("AVG dereg overhead null %f \n", avg_dereg_n - avg_dereg);
507
508    printf("############################################################ \n");
509
510    printf("AVG enq overhead debug %f \n", avg_enq_d - avg_enq);
511    printf("AVG deq overhead debug %f \n", avg_deq_d - avg_deq);
512    printf("AVG reg overhead debug %f \n", avg_reg_d - avg_reg);
513    printf("AVG dereg overhead debug %f \n", avg_dereg_d - avg_dereg);
514*/
515    err = devq_deregister(que, regid, &memory);
516    if (err_is_fail(err)){
517        USER_PANIC("Deregistering memory from devq failed: %s \n",
518                   err_getstring(err));
519    }
520    printf("SUCCESS! \n");;
521
522}
523
524