1/*
2 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#include <stdlib.h>
11#include <stdio.h>
12//#include <time.h>
13#include <barrelfish/barrelfish.h>
14#include <barrelfish/sys_debug.h>
15#include <barrelfish/deferred.h>
16#include <devif/queue_interface.h>
17#include <devif/backends/loopback_devif.h>
18#include <devif/backends/debug.h>
19#include <devif/backends/null.h>
20#include <bench/bench.h>
21#include <vfs/vfs.h>
22
23//#define DEBUG(x...) printf("devif_test: " x)
24#define DEBUG(x...) do {} while (0)
25
26#define BUF_SIZE 2048
27#define NUM_BUFS 128
28#define MEMORY_SIZE BUF_SIZE*NUM_BUFS
29
30#define NUM_REGIONS 128
31#define NUM_ROUNDS 100000
32
33static struct capref memory;
34static regionid_t regid;
35static struct frame_identity id;
36static lpaddr_t phys;
37
38static void* va;
39
40struct direct_state {
41    struct list_ele* first;
42    struct list_ele* last;
43};
44
45struct list_ele{
46    regionid_t rid;
47    bufferid_t bid;
48    lpaddr_t addr;
49    size_t len;
50    uint64_t flags;
51
52    struct list_ele* next;
53};
54
55static struct loopback_queue* queue;
56static struct debug_q* debug_q;
57static struct null_q* null_q;
58static struct null_q* all_q;
59static struct devq* que;
60
61static cycles_t tot_deq = 0;
62static cycles_t tot_enq = 0;
63static cycles_t tot_reg = 0;
64static cycles_t tot_dereg = 0;
65static cycles_t start_enq = 0, end_enq = 0;
66static cycles_t start_deq = 0, end_deq = 0;
67static cycles_t start_reg = 0, end_reg = 0;
68static cycles_t start_dereg = 0, end_dereg = 0;
69
70static double avg_deq, avg_enq;
71static double avg_reg, avg_dereg;
72static double avg_deq_d, avg_enq_d;
73static double avg_reg_d, avg_dereg_d;
74static double avg_deq_n, avg_enq_n;
75static double avg_reg_n, avg_dereg_n;
76static double avg_deq_tot, avg_enq_tot;
77static double avg_reg_tot, avg_dereg_tot;
78
79static bench_ctl_t *ctl_tmp_en;
80static bench_ctl_t *ctl_tmp_de;
81static bench_ctl_t *ctl_tmp_reg;
82static bench_ctl_t *ctl_tmp_dereg;
83
84static uint64_t tscperus;
85static char* machine_name;
86
87static void dump_results_nfs(char* filename)
88{
89    errval_t err;
90    char buffer[256];
91    vfs_handle_t handle;
92    sprintf(buffer, "/nfs/%s/%s.csv", machine_name, filename);
93    printf("%s \n", buffer);
94    err = vfs_open(buffer, &handle);
95    assert(err_is_ok(err));
96
97    size_t bytes;
98    // first 10 % is warmup
99    for (int i = NUM_ROUNDS/10 ; i < NUM_ROUNDS; i++) {
100        sprintf(buffer, "enqueue,%lu \ndequeue,%lu \nregister,%lu \nderegister,%lu \n",
101                ctl_tmp_en->data[i], ctl_tmp_de->data[i], ctl_tmp_reg->data[i],
102                ctl_tmp_dereg->data[i]);
103        err = vfs_write(handle, buffer, strlen(buffer), &bytes);
104        assert(err_is_ok(err));
105        assert(bytes == strlen(buffer));
106    }
107
108    err = vfs_close(handle);
109    assert(err_is_ok(err));
110
111    bench_ctl_destroy(ctl_tmp_dereg);
112    bench_ctl_destroy(ctl_tmp_reg);
113    bench_ctl_destroy(ctl_tmp_de);
114    bench_ctl_destroy(ctl_tmp_en);
115}
116
117static void test_register(void)
118{
119    errval_t err;
120    struct capref regions[NUM_REGIONS];
121    regionid_t rids[NUM_REGIONS];
122
123    tot_reg = 0;
124    tot_dereg = 0;
125
126    ctl_tmp_reg = calloc(1, sizeof(*ctl_tmp_reg));
127    ctl_tmp_reg->mode = BENCH_MODE_FIXEDRUNS;
128    ctl_tmp_reg->result_dimensions = 1;
129    ctl_tmp_reg->min_runs = NUM_ROUNDS;
130    ctl_tmp_reg->data = calloc(ctl_tmp_reg->min_runs * ctl_tmp_reg->result_dimensions,
131                       sizeof(*ctl_tmp_de->data));
132
133    ctl_tmp_dereg = calloc(1, sizeof(*ctl_tmp_dereg));
134    ctl_tmp_dereg->mode = BENCH_MODE_FIXEDRUNS;
135    ctl_tmp_dereg->result_dimensions = 1;
136    ctl_tmp_dereg->min_runs = NUM_ROUNDS;
137    ctl_tmp_dereg->data = calloc(ctl_tmp_dereg->min_runs * ctl_tmp_dereg->result_dimensions,
138                       sizeof(*ctl_tmp_dereg->data));
139
140    for (int i = 0; i < NUM_REGIONS; i++) {
141        err = frame_alloc(&regions[i], BASE_PAGE_SIZE, NULL);
142        if (err_is_fail(err)){
143            USER_PANIC("Allocating cap failed \n");
144        }
145    }
146
147    srand(rdtsc());
148    int idx = 0;
149    struct capref ret;
150    cycles_t res1;
151    cycles_t res2;
152    for (int i = 0; i < NUM_ROUNDS; i++) {
153        idx = i % NUM_REGIONS;
154        start_reg = bench_tsc();
155        err = devq_register(que, regions[idx], &rids[idx]);
156        if (err_is_fail(err)){
157            USER_PANIC("Registering memory to devq failed: %s \n",
158                        err_getstring(err));
159        }
160        end_reg = bench_tsc();
161        tot_reg += end_reg - start_reg;
162        res1 = end_reg - start_reg;
163
164        start_dereg = bench_tsc();
165        err = devq_deregister(que, rids[idx], &ret);
166        if (err_is_fail(err)){
167            USER_PANIC("Registering memory to devq failed: %s\n",
168                       err_getstring(err));
169        }
170
171        end_dereg = bench_tsc();
172        tot_dereg += end_dereg - start_dereg;
173        res2 = end_dereg - start_dereg;
174        bench_ctl_add_run(ctl_tmp_reg, &res1);
175        bench_ctl_add_run(ctl_tmp_dereg, &res2);
176    }
177
178    for (int i = 0; i < NUM_REGIONS; i++) {
179        err = cap_destroy(regions[i]);
180        if (err_is_fail(err)){
181            USER_PANIC("Destroy region failed: %s\n",
182                       err_getstring(err));
183        }
184    }
185
186    bench_ctl_dump_analysis(ctl_tmp_dereg, 0, "deregister", tscperus);
187    bench_ctl_dump_analysis(ctl_tmp_reg, 0, "register", tscperus);
188
189}
190
191static void test_randomized_test(void)
192{
193    errval_t err;
194    regionid_t rid;
195    genoffset_t offset;
196    genoffset_t length;
197    genoffset_t valid_data;
198    genoffset_t valid_length;
199    uint64_t flags;
200
201    tot_enq = 0;
202    tot_deq = 0;
203
204    ctl_tmp_en = calloc(1, sizeof(*ctl_tmp_en));
205    ctl_tmp_en->mode = BENCH_MODE_FIXEDRUNS;
206    ctl_tmp_en->result_dimensions = 1;
207    ctl_tmp_en->min_runs = NUM_ROUNDS;
208    ctl_tmp_en->data = calloc(ctl_tmp_en->min_runs * ctl_tmp_en->result_dimensions,
209                       sizeof(*ctl_tmp_de->data));
210
211    ctl_tmp_de = calloc(1, sizeof(*ctl_tmp_en));
212    ctl_tmp_de->mode = BENCH_MODE_FIXEDRUNS;
213    ctl_tmp_de->result_dimensions = 1;
214    ctl_tmp_de->min_runs = NUM_ROUNDS;
215    ctl_tmp_de->data = calloc(ctl_tmp_de->min_runs * ctl_tmp_de->result_dimensions,
216                       sizeof(*ctl_tmp_de->data));
217
218    srand(rdtsc());
219    int idx = 0;
220    cycles_t res;
221    // enqueue from the beginning of the region
222    for (int i = 0; i < NUM_ROUNDS; i++) {
223
224        idx = i % NUM_BUFS;
225        start_enq = rdtsc();
226
227        err = devq_enqueue(que, regid, idx*BUF_SIZE, BUF_SIZE,
228                           0, BUF_SIZE, 0);
229        if (err_is_fail(err)){
230            USER_PANIC("Enqueue failed: %s \n", err_getstring(err));
231        } else {
232            end_enq = rdtsc();
233            tot_enq += end_enq - start_enq;
234            res = end_enq - start_enq;
235            bench_ctl_add_run(ctl_tmp_de, &res);
236        }
237
238        start_deq = rdtsc();
239        err = devq_dequeue(que, &rid, &offset, &length, &valid_data,
240                           &valid_length, &flags);
241        if (err_is_ok(err)){
242            end_deq = rdtsc();
243            tot_deq += end_deq - start_deq;
244            res = end_deq - start_deq;
245            bench_ctl_add_run(ctl_tmp_en, &res);
246        } else {
247            USER_PANIC("Dequeue failed: %s \n", err_getstring(err));
248        }
249    }
250
251    bench_ctl_dump_analysis(ctl_tmp_de, 0, "enqueue", tscperus);
252    bench_ctl_dump_analysis(ctl_tmp_en, 0, "dequeue", tscperus);
253}
254
255int main(int argc, char *argv[])
256{
257
258    if (argc > 1) {
259        machine_name = argv[1];
260    } else {
261        machine_name = "default";
262    }
263
264    errval_t err;
265
266    // mount_vfs
267    vfs_init();
268
269    char fname[256];
270    err = vfs_mount("/nfs", "nfs://10.110.4.4/mnt/local/nfs/haeckir");
271    if(err_is_fail(err)) {
272        USER_PANIC("vfs_mount: %s \n", err_getstring(err));
273    }
274
275    vfs_handle_t handle;
276    sprintf(fname, "/nfs/%s", machine_name);
277    err = vfs_mkdir(fname);
278    if (err_is_fail(err)) {
279        printf("Folder %s already exists \n", fname);
280    } else {
281        printf("Creating folder %s \n", fname);
282    }
283
284    sprintf(fname, "/nfs/%s/debug.csv", machine_name);
285    err = vfs_create(fname, &handle);
286    assert(err_is_ok(err));
287
288    sprintf(fname, "/nfs/%s/loopback.csv", machine_name);
289    err = vfs_create(fname, &handle);
290    assert(err_is_ok(err));
291
292
293    sprintf(fname, "/nfs/%s/null.csv", machine_name);
294    err = vfs_create(fname, &handle);
295    assert(err_is_ok(err));
296
297
298    sprintf(fname, "/nfs/%s/null_debug.csv", machine_name);
299    err = vfs_create(fname, &handle);
300    assert(err_is_ok(err));
301
302    // Allocate memory
303    err = frame_alloc(&memory, MEMORY_SIZE, NULL);
304    if (err_is_fail(err)){
305        USER_PANIC("Allocating cap failed \n");
306    }
307
308    // RX frame
309    err = invoke_frame_identify(memory, &id);
310    if (err_is_fail(err)) {
311        USER_PANIC("Frame identify failed \n");
312    }
313
314    err = vspace_map_one_frame_attr(&va, id.bytes, memory,
315                                    VREGION_FLAGS_READ, NULL, NULL);
316    if (err_is_fail(err)) {
317        USER_PANIC("Frame mapping failed \n");
318    }
319
320    phys = id.base;
321
322    debug_printf("Descriptor queue test started \n");
323    err = loopback_queue_create(&queue);
324    if (err_is_fail(err)){
325        USER_PANIC("Allocating devq failed \n");
326    }
327
328    // stack debug queue on top
329    err = debug_create(&debug_q, (struct devq*) queue);
330    if (err_is_fail(err)) {
331        USER_PANIC("Allocating debug q failed \n");
332    }
333
334    // stack null queue on top
335    err = null_create(&null_q, (struct devq*) queue);
336    if (err_is_fail(err)) {
337        USER_PANIC("Allocating null q failed \n");
338    }
339
340    // stack null queue on top of debug queue
341    err = null_create(&all_q, (struct devq*) debug_q);
342    if (err_is_fail(err)) {
343        USER_PANIC("Allocating null q failed \n");
344    }
345
346
347    que = (struct devq*) queue;
348
349    err = devq_register(que, memory, &regid);
350    if (err_is_fail(err)){
351        USER_PANIC("Registering memory to devq failed \n");
352    }
353
354    err = sys_debug_get_tsc_per_ms(&tscperus);
355    assert(err_is_ok(err));
356    tscperus /= 1000;
357
358    printf("Starting randomized queue\n");
359    que = (struct devq*) queue;
360
361    test_register();
362
363    test_randomized_test();
364
365    dump_results_nfs("loopback");
366    avg_enq = ((double) tot_deq)/NUM_ROUNDS;
367    avg_deq = ((double) tot_enq)/NUM_ROUNDS;
368    avg_dereg = ((double) tot_dereg)/NUM_ROUNDS;
369    avg_reg = ((double) tot_reg)/NUM_ROUNDS;
370/*
371    printf("AVG enq %f \n", avg_enq);
372    printf("AVG deq %f \n", avg_deq);
373    printf("AVG reg %f \n", avg_reg);
374    printf("AVG dereg %f \n", avg_dereg);
375*/
376    printf("############################################################ \n");
377
378    err = devq_deregister(que, regid, &memory);
379    if (err_is_fail(err)){
380        USER_PANIC("Deregistering memory from devq failed: %s \n",
381                   err_getstring(err));
382    }
383
384    printf("Starting randomized test debug\n");
385    que = (struct devq*) debug_q;
386
387    test_register();
388
389    err = devq_register(que, memory, &regid);
390    if (err_is_fail(err)){
391        USER_PANIC("Registering memory to devq failed \n");
392    }
393
394    test_randomized_test();
395
396    dump_results_nfs("debug");
397    avg_enq_d = ((double) tot_deq)/NUM_ROUNDS;
398    avg_deq_d = ((double) tot_enq)/NUM_ROUNDS;
399    avg_dereg_d  = ((double) tot_dereg)/NUM_ROUNDS;
400    avg_reg_d = ((double) tot_reg)/NUM_ROUNDS;
401/*
402    printf("AVG enq debug %f \n", avg_enq_d);
403    printf("AVG deq debug %f \n", avg_deq_d);
404    printf("AVG reg debug %f \n", avg_reg_d);
405    printf("AVG dereg debug %f \n", avg_dereg_d);
406*/
407    printf("############################################################ \n");
408
409    err = devq_deregister(que, regid, &memory);
410    if (err_is_fail(err)){
411        USER_PANIC("Deregistering memory from devq failed: %s \n",
412                   err_getstring(err));
413    }
414
415    printf("Starting randomized test null\n");
416    que = (struct devq*) null_q;
417
418    err = devq_register(que, memory, &regid);
419    if (err_is_fail(err)){
420        USER_PANIC("Registering memory to devq failed \n");
421    }
422
423    test_register();
424
425    test_randomized_test();
426
427    dump_results_nfs("null");
428    avg_enq_n = ((double) tot_enq)/NUM_ROUNDS;
429    avg_deq_n = ((double) tot_deq)/NUM_ROUNDS;
430    avg_reg_n = ((double) tot_reg)/NUM_ROUNDS;
431    avg_dereg_n = ((double) tot_dereg)/NUM_ROUNDS;
432
433    printf("############################################################ \n");
434
435    err = devq_deregister(que, regid, &memory);
436    if (err_is_fail(err)){
437        USER_PANIC("Deregistering memory from devq failed: %s \n",
438                   err_getstring(err));
439    }
440
441    printf("Starting randomized test all queues\n");
442    que = (struct devq*) all_q;
443
444    test_register();
445
446    err = devq_register(que, memory, &regid);
447    if (err_is_fail(err)){
448        USER_PANIC("Registering memory to devq failed \n");
449    }
450
451    test_randomized_test();
452
453    dump_results_nfs("null_debug");
454    avg_enq_tot = ((double) tot_enq)/NUM_ROUNDS;
455    avg_deq_tot = ((double) tot_deq)/NUM_ROUNDS;
456    avg_reg_tot = ((double) tot_reg)/NUM_ROUNDS;
457    avg_dereg_tot = ((double) tot_dereg)/NUM_ROUNDS;
458
459/*
460    printf("AVG enq null %f \n", avg_enq_n);
461    printf("AVG deq null %f \n", avg_deq_n);
462    printf("AVG reg null %f \n", avg_reg_n);
463    printf("AVG dereg null %f \n", avg_dereg_n);
464    printf("############################################################ \n");
465
466    printf("AVG enq overhead null %f \n", avg_enq_n - avg_enq);
467    printf("AVG deq overhead null %f \n", avg_deq_n - avg_deq);
468    printf("AVG reg overhead null %f \n", avg_reg_n - avg_reg);
469    printf("AVG dereg overhead null %f \n", avg_dereg_n - avg_dereg);
470
471    printf("############################################################ \n");
472
473    printf("AVG enq overhead debug %f \n", avg_enq_d - avg_enq);
474    printf("AVG deq overhead debug %f \n", avg_deq_d - avg_deq);
475    printf("AVG reg overhead debug %f \n", avg_reg_d - avg_reg);
476    printf("AVG dereg overhead debug %f \n", avg_dereg_d - avg_dereg);
477*/
478    err = devq_deregister(que, regid, &memory);
479    if (err_is_fail(err)){
480        USER_PANIC("Deregistering memory from devq failed: %s \n",
481                   err_getstring(err));
482    }
483    printf("SUCCESS! \n");;
484
485}
486
487