1/*
2 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#include <stdlib.h>
11#include <stdio.h>
12#include <time.h>
13#include <barrelfish/barrelfish.h>
14#include <barrelfish/waitset.h>
15#include <barrelfish/waitset_chan.h>
16#include <barrelfish/nameservice_client.h>
17#include <barrelfish/deferred.h>
18#include <devif/queue_interface.h>
19#include <devif/backends/net/sfn5122f_devif.h>
20#include <devif/backends/net/e10k_devif.h>
21#include <devif/backends/descq.h>
22#include <bench/bench.h>
23#include <net_interfaces/flags.h>
24#include <net/net_filter.h>
25#include <if/devif_test_defs.h>
26
27//#define DEBUG(x...) printf("devif_test: " x)
28#define DEBUG(x...) do {} while (0)
29
30#define TX_BUF_SIZE 2048
31#define RX_BUF_SIZE 2048
32#define NUM_ENQ 512
33#define NUM_RX_BUF 1024
34#define NUM_ROUNDS_TX 16384
35#define NUM_ROUNDS_RX 128
36#define MEMORY_SIZE BASE_PAGE_SIZE*512
37
38static char* card;
39static uint32_t ip_dst;
40static uint32_t ip_src;
41
42static struct capref memory_rx;
43static struct capref memory_tx;
44static regionid_t regid_rx;
45static regionid_t regid_tx;
46static struct frame_identity id;
47static lpaddr_t phys_rx;
48static lpaddr_t phys_tx;
49
50
51static volatile uint32_t num_tx = 0;
52static volatile uint32_t num_rx = 0;
53static uint64_t enq_total = 0;
54static uint64_t deq_total = 0;
55static uint64_t qid;
56
57static void* va_rx;
58static void* va_tx;
59
60struct direct_state {
61    struct list_ele* first;
62    struct list_ele* last;
63};
64
65struct list_ele{
66    regionid_t rid;
67    bufferid_t bid;
68    lpaddr_t addr;
69    size_t len;
70    uint64_t flags;
71
72    struct list_ele* next;
73};
74
75
76static struct devif_test_binding* binding;
77
78#ifndef __ARM_ARCH_7A__
79static struct net_filter_state* filter;
80
81static struct waitset_chanstate *chan = NULL;
82static struct waitset card_ws;
83
84static uint8_t udp_header[8] = {
85    0x07, 0xD0, 0x07, 0xD0,
86    0x00, 0x80, 0x00, 0x00,
87};
88
89static void print_buffer(size_t len, bufferid_t bid)
90{
91   /*
92    uint8_t* buf = (uint8_t*) va_rx+bid;
93    printf("Packet in region %p at address %p len %zu \n",
94           va_rx, buf, len);
95    for (int i = 0; i < len; i++) {
96        if (((i % 10) == 0) && i > 0) {
97            printf("\n");
98        }
99        printf("%2X ", buf[i]);
100    }
101    printf("\n");
102    */
103}
104
105static void wait_for_interrupt(void)
106{
107    errval_t err = event_dispatch(&card_ws);
108    if (err_is_fail(err)) {
109        USER_PANIC_ERR(err, "error in event_dispatch for wait_for_interrupt");
110    }
111}
112
113static void event_cb(void* queue)
114{
115    struct devq* q = (struct devq*) queue;
116
117    errval_t err;
118
119    regionid_t rid;
120    genoffset_t offset;
121    genoffset_t length;
122    genoffset_t valid_data;
123    genoffset_t valid_length;
124    uint64_t flags;
125
126    err = SYS_ERR_OK;
127    uint64_t start, end;
128
129    while (err == SYS_ERR_OK) {
130        start = rdtscp();
131        err = devq_dequeue(q, &rid, &offset, &length, &valid_data,
132                           &valid_length, &flags);
133        end = rdtscp();
134        if (err_is_fail(err)) {
135            break;
136        }
137
138        deq_total += end - start;
139
140        if (flags & NETIF_TXFLAG) {
141            DEBUG("Received TX buffer back \n");
142            num_tx++;
143        } else if (flags & NETIF_RXFLAG) {
144            num_rx++;
145            DEBUG("Received RX buffer \n");
146            print_buffer(valid_length, offset);
147        } else {
148            printf("Unknown flags %lx \n", flags);
149        }
150    }
151
152    // MSIX is not working on so we have to "simulate interrupts"
153    err = waitset_chan_register(&card_ws, chan,
154                                MKCLOSURE(event_cb, queue));
155    if (err_is_fail(err) && err_no(err) == LIB_ERR_CHAN_ALREADY_REGISTERED) {
156        printf("Got actual interrupt?\n");
157    }
158    else if (err_is_fail(err)) {
159        USER_PANIC_ERR(err, "Can't register our dummy channel.");
160    }
161    err = waitset_chan_trigger(chan);
162    if (err_is_fail(err)) {
163        USER_PANIC_ERR(err, "trigger failed.");
164    }
165}
166
167static struct devq* create_net_queue(char* card_name)
168{
169    errval_t err;
170    struct capref ep = NULL_CAP;
171
172    if (strncmp(card_name, "sfn5122f", 8) == 0) {
173        debug_printf("Creating sfn5122f queue \n");
174        struct sfn5122f_queue* q;
175
176        err = sfn5122f_queue_create(&q, event_cb, &ep, /* userlevel*/ true,
177                                    /*interrupts*/ false,
178                                    /*default queue*/ false);
179        if (err_is_fail(err)){
180            USER_PANIC("Allocating devq failed \n");
181        }
182
183        return (struct devq*) q;
184    }
185
186    if (strncmp(card_name, "e10k", 4) == 0) {
187        struct e10k_queue* q;
188
189        debug_printf("Creating e10k queue \n");
190        err = e10k_queue_create(&q, event_cb, &ep, /*VFs */ false,
191                                6, 0, 0, 0,
192                                /*MSIX interrupts*/ false, false);
193        if (err_is_fail(err)){
194            USER_PANIC("Allocating devq failed \n");
195        }
196        return (struct devq*) q;
197    }
198
199    USER_PANIC("Unknown card name\n");
200
201    return NULL;
202}
203
204static void test_net_tx(void)
205{
206    num_tx = 0;
207    num_rx = 0;
208
209    errval_t err;
210    struct devq* q;
211
212
213    q = create_net_queue(card);
214    assert(q != NULL);
215
216
217    debug_printf("Creating net queue done\n");
218    waitset_init(&card_ws);
219
220    // MSIX is not working on sfn5122f yet so we have to "simulate interrupts"
221    chan = malloc(sizeof(struct waitset_chanstate));
222    waitset_chanstate_init(chan, CHANTYPE_AHCI);
223
224    err = waitset_chan_register(&card_ws, chan, MKCLOSURE(event_cb, q));
225    if (err_is_fail(err)) {
226        USER_PANIC_ERR(err, "waitset_chan_regster failed.");
227    }
228
229    err = waitset_chan_trigger(chan);
230    if (err_is_fail(err)) {
231        USER_PANIC_ERR(err, "trigger failed.");
232    }
233
234    err = devq_register(q, memory_tx, &regid_tx);
235    if (err_is_fail(err)){
236        USER_PANIC("Registering memory to devq failed \n");
237    }
238
239
240    // write something into the buffers
241    char* write = NULL;
242
243    for (int i = 0; i < NUM_ENQ; i++) {
244        write = va_tx + i*(TX_BUF_SIZE);
245        for (int j = 0; j < 8; j++) {
246            write[j] = udp_header[j];
247        }
248        for (int j = 8; j < TX_BUF_SIZE; j++) {
249            write[j] = 'a';
250        }
251    }
252
253    // Send something
254    cycles_t t1 = bench_tsc();
255
256    for (int z = 0; z < NUM_ROUNDS_TX; z++) {
257        for (int i = 0; i < NUM_ENQ; i++) {
258            err = devq_enqueue(q, regid_tx, i*(TX_BUF_SIZE), TX_BUF_SIZE,
259                               0, TX_BUF_SIZE,
260                               NETIF_TXFLAG | NETIF_TXFLAG_LAST);
261            if (err_is_fail(err)){
262                USER_PANIC("Devq enqueue failed \n");
263            }
264        }
265
266        while(true) {
267            if ((num_tx < NUM_ENQ)) {
268                wait_for_interrupt();
269            } else {
270                break;
271            }
272        }
273        num_tx = 0;
274    }
275
276    cycles_t t2 = bench_tsc();
277    cycles_t result = (t2 -t1 - bench_tscoverhead());
278
279    uint64_t sent_bytes = (uint64_t) TX_BUF_SIZE*NUM_ENQ*NUM_ROUNDS_TX;
280    double result_ms = (double) bench_tsc_to_ms(result);
281    double bw = sent_bytes / result_ms / 1000;
282
283    printf("Write throughput %.2f [MB/s] for %.2f ms \n", bw, result_ms);
284
285
286    err = devq_control(q, 1, 1, &sent_bytes);
287    if (err_is_fail(err)){
288        printf("%s \n", err_getstring(err));
289        USER_PANIC("Devq control failed \n");
290    }
291
292    err = devq_deregister(q, regid_tx, &memory_tx);
293    if (err_is_fail(err)){
294        printf("%s \n", err_getstring(err));
295        USER_PANIC("Devq deregister tx failed \n");
296    }
297
298    err = devq_destroy(q);
299    if (err_is_fail(err)){
300        printf("%s \n", err_getstring(err));
301        USER_PANIC("Destroying %s queue failed \n", card);
302    }
303
304    printf("SUCCESS: %s tx test ended\n", card);
305}
306
307
308static void test_net_rx(void)
309{
310
311    num_tx = 0;
312    num_rx = 0;
313
314    errval_t err;
315    struct devq* q;
316
317    q = create_net_queue(card);
318    assert(q != NULL);
319
320    waitset_init(&card_ws);
321
322    // MSIX is not working on sfn5122f yet so we have to "simulate interrupts"
323    chan = malloc(sizeof(struct waitset_chanstate));
324    waitset_chanstate_init(chan, CHANTYPE_AHCI);
325
326    err = waitset_chan_register(&card_ws, chan, MKCLOSURE(event_cb, q));
327    if (err_is_fail(err)) {
328        USER_PANIC_ERR(err, "waitset_chan_regster failed.");
329    }
330
331    err = waitset_chan_trigger(chan);
332    if (err_is_fail(err)) {
333        USER_PANIC_ERR(err, "trigger failed.");
334    }
335
336    err = net_filter_init(&filter, card);
337    if (err_is_fail(err)) {
338        USER_PANIC("Installing filter failed \n");
339    }
340
341    struct net_filter_ip ip_filt ={
342        .qid = 1,
343        .ip_dst = ip_dst,
344        .ip_src = ip_src,
345        .port_src = 0,
346        .port_dst = 7,
347        .type = NET_FILTER_UDP,
348    };
349
350    err = net_filter_ip_install(filter, &ip_filt);
351    if (err_is_fail(err)){
352        USER_PANIC("Allocating devq failed \n");
353    }
354
355    err = devq_register(q, memory_rx, &regid_rx);
356    if (err_is_fail(err)){
357        USER_PANIC("Registering memory to devq failed \n");
358    }
359
360    // Enqueue RX buffers to receive into
361    for (int i = 0; i < NUM_ROUNDS_RX; i++){
362        err = devq_enqueue(q, regid_rx, i*RX_BUF_SIZE, RX_BUF_SIZE,
363                           0, RX_BUF_SIZE,
364                           NETIF_RXFLAG);
365        if (err_is_fail(err)){
366            USER_PANIC("Devq enqueue failed: %s\n", err_getstring(err));
367        }
368
369    }
370
371    while (true) {
372        if ((num_rx < NUM_ROUNDS_RX)) {
373            wait_for_interrupt();
374        } else {
375            break;
376        }
377    }
378
379    uint64_t ret;
380    err = devq_control(q, 1, 1, &ret);
381    if (err_is_fail(err)){
382        printf("%s \n", err_getstring(err));
383        USER_PANIC("Devq control failed \n");
384    }
385
386    err = devq_deregister(q, regid_rx, &memory_rx);
387    if (err_is_fail(err)){
388        printf("%s \n", err_getstring(err));
389        USER_PANIC("Devq deregister rx failed \n");
390    }
391
392    err = devq_destroy(q);
393    if (err_is_fail(err)){
394        printf("%s \n", err_getstring(err));
395        USER_PANIC("Destroying %s queue failed \n", card);
396    }
397
398    printf("SUCCESS: %s rx test ended\n", card);
399}
400#endif
401
402static errval_t descq_notify(struct descq* q)
403{
404    errval_t err = SYS_ERR_OK;
405    struct devq* queue = (struct devq*) q;
406
407    regionid_t rid;
408    genoffset_t offset;
409    genoffset_t length;
410    genoffset_t valid_data;
411    genoffset_t valid_length;
412    uint64_t flags;
413    uint64_t start, end;
414
415    while(err_is_ok(err)) {
416        start = rdtscp();
417        err = devq_dequeue(queue, &rid, &offset, &length, &valid_data,
418                           &valid_length, &flags);
419        end = rdtscp();
420        if (err_is_ok(err)){
421            num_rx++;
422            deq_total += end - start;
423        }
424    }
425    return SYS_ERR_OK;
426}
427
428
429static void bind_cb(void *st, errval_t err, struct devif_test_binding *b)
430{
431    uint64_t* bound = (uint64_t*) st;
432    assert(err_is_ok(err));
433    devif_test_rpc_client_init(b);
434    binding = b;
435    *bound = 1;
436}
437
438static errval_t get_descq_ep(struct capref* ep)
439{
440    errval_t err;
441    iref_t iref;
442    uint64_t state = 0;
443
444    err = slot_alloc(ep);
445    if (err_is_fail(err)) {
446        return err;
447    }
448
449    err = nameservice_blocking_lookup("devif_test_ep", &iref);
450    if (err_is_fail(err)) {
451        goto out;
452    }
453
454    err = devif_test_bind(iref, bind_cb, (void*) &state, get_default_waitset(),
455                          IDC_BIND_FLAGS_DEFAULT);
456    if (err_is_fail(err)) {
457        goto out;
458    }
459
460    while (state == 0) {
461        event_dispatch(get_default_waitset());
462    }
463
464    errval_t err2;
465    err = binding->rpc_tx_vtbl.request_ep(binding, disp_get_core_id(),
466                                          &err2, ep);
467    if (err_is_fail(err) || err_is_fail(err2)) {
468        err = err_is_fail(err) ? err : err2;
469        goto out;
470    }
471
472    debug_printf("Connection setup done \n");
473    return SYS_ERR_OK;
474
475out:
476    slot_free(*ep);
477    return err;
478}
479
480
481static void test_idc_queue(bool use_ep)
482{
483    num_tx = 0;
484    num_rx = 0;
485    enq_total = 0;
486    deq_total = 0;
487
488
489    errval_t err;
490    struct devq* q;
491    struct descq* queue;
492    struct descq_func_pointer f;
493    f.notify = descq_notify;
494
495    debug_printf("Descriptor queue test started \n");
496    if (use_ep) {
497        printf("Descriptor queue use endpoint for setup\n");
498        struct capref ep;
499
500        printf("Getting endpoint \n");
501        err = get_descq_ep(&ep);
502        if (err_is_fail(err)){
503            USER_PANIC("Allocating devq failed \n");
504        }
505
506        printf("Creating descq with ep \n");
507        err = descq_create_with_ep(&queue, DESCQ_DEFAULT_SIZE, ep,
508                                   &qid, &f);
509        if (err_is_fail(err)){
510            USER_PANIC("Allocating devq failed \n");
511        }
512
513    } else {
514        printf("Descriptor queue use name service for setup\n");
515        err = descq_create(&queue, DESCQ_DEFAULT_SIZE, "test_queue",
516                           false, &qid, &f);
517        if (err_is_fail(err)){
518            USER_PANIC("Allocating devq failed \n");
519        }
520    }
521
522    q = (struct devq*) queue;
523
524    printf("Registering RX\n");
525    err = devq_register(q, memory_rx, &regid_rx);
526    if (err_is_fail(err)){
527        USER_PANIC("Registering memory to devq failed \n");
528    }
529
530    printf("Registering TX\n");
531    err = devq_register(q, memory_tx, &regid_tx);
532    if (err_is_fail(err)){
533        USER_PANIC("Registering memory to devq failed \n");
534    }
535
536    printf("Sending messages\n");
537    // Enqueue RX buffers to receive into
538    uint64_t start, end, total;
539    total = 0;
540    for (int j = 0; j < 1000000; j++){
541        for (int i = 0; i < 32; i++){
542            start = rdtscp();
543            err = devq_enqueue(q, regid_rx, i*2048, 2048,
544                               0, 2048, 0);
545            end = rdtscp();
546            if (err_is_fail(err)){
547                // retry
548                i--;
549            } else {
550                enq_total += end - start;
551                num_tx++;
552            }
553        }
554
555        err = devq_notify(q);
556        if (err_is_fail(err)) {
557                USER_PANIC("Devq notify failed: %s\n", err_getstring(err));
558        }
559        event_dispatch(get_default_waitset());
560        if ((j % 100000) == 0) {
561            debug_printf("Round %d \n", j);
562        }
563    }
564
565    while(num_tx != num_rx) {
566        event_dispatch(get_default_waitset());
567    }
568
569    err = devq_control(q, 1, 1, NULL);
570    if (err_is_fail(err)){
571        printf("%s \n", err_getstring(err));
572        USER_PANIC("Devq control failed \n");
573    }
574
575    err = devq_deregister(q, regid_rx, &memory_rx);
576    if (err_is_fail(err)){
577        printf("%s \n", err_getstring(err));
578        USER_PANIC("Devq deregister rx failed \n");
579    }
580
581    err = devq_deregister(q, regid_tx, &memory_tx);
582    if (err_is_fail(err)){
583        printf("%s \n", err_getstring(err));
584        USER_PANIC("Devq deregister tx failed \n");
585    }
586
587    printf("AVG enqueue %f num_enq %d \n", (double) enq_total/num_tx, num_tx);
588    printf("AVG dequeue %f num_deq %d\n", (double) deq_total/num_rx, num_rx);
589}
590
591int main(int argc, char *argv[])
592{
593    errval_t err;
594    // Allocate memory
595    err = frame_alloc(&memory_rx, MEMORY_SIZE, NULL);
596    if (err_is_fail(err)){
597        USER_PANIC("Allocating cap failed \n");
598    }
599
600    err = frame_alloc(&memory_tx, MEMORY_SIZE, NULL);
601    if (err_is_fail(err)){
602        USER_PANIC("Allocating cap failed \n");
603    }
604
605    // RX frame
606    err = frame_identify(memory_rx, &id);
607    if (err_is_fail(err)) {
608        USER_PANIC("Frame identify failed \n");
609    }
610
611    err = vspace_map_one_frame_attr(&va_rx, id.bytes, memory_rx,
612                                    VREGION_FLAGS_READ, NULL, NULL);
613    if (err_is_fail(err)) {
614        USER_PANIC("Frame mapping failed \n");
615    }
616
617    phys_rx = id.base;
618
619    // TX Frame
620    err = frame_identify(memory_tx, &id);
621    if (err_is_fail(err)) {
622        USER_PANIC("Frame identify failed \n");
623    }
624
625    err = vspace_map_one_frame_attr(&va_tx, id.bytes, memory_tx,
626                                    VREGION_FLAGS_WRITE, NULL, NULL);
627    if (err_is_fail(err)) {
628        USER_PANIC("Frame mapping failed \n");
629    }
630
631    phys_tx = id.base;
632
633    if (argc > 3) {
634        ip_src = atoi(argv[2]);
635        ip_dst = atoi(argv[3]);
636    } else {
637        USER_PANIC("NO src or dst IP given \n");
638    }
639
640    if (argc > 4) {
641        card = argv[4];
642        printf("Card =%s \n", card);
643    } else {
644        card = "e10k";
645    }
646
647    #ifndef __ARM_ARCH_7A__
648    if (strcmp(argv[1], "net_tx") == 0) {
649        test_net_tx();
650    }
651
652    if (strcmp(argv[1], "net_rx") == 0) {
653        test_net_rx();
654    }
655    #endif
656
657    if (strcmp(argv[1], "idc") == 0) {
658        test_idc_queue(true);
659        test_idc_queue(false);
660        printf("SUCCESS: IDC queue\n");
661    }
662
663    barrelfish_usleep(1000*1000*5);
664}
665
666