1/*
2 * Copyright (c) 2007-2011, ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#include <barrelfish/barrelfish.h>
11#include <barrelfish/nameservice_client.h>
12#include <devif/queue_interface.h>
13#include <driverkit/iommu.h>
14
15#include <dev/e10k_q_dev.h>
16#include <dev/e10k_dev.h>
17#include <dev/e10k_vf_dev.h>
18#include <if/e10k_vf_defs.h>
19#include <if/e10k_vf_rpcclient_defs.h>
20
21#include "e10k_devif_vf.h"
22#include "e10k_queue.h"
23#include "debug.h"
24
25#define NUM_TX_DESC 2048
26#define NUM_RX_DESC 2048
27
28#define ETHHDR_LEN 14
29#define IPHDR_LEN 20
30#define UDPHDR_LEN 8
31
32
33// TODO only required for legacy interrupts
34struct e10k_queue* queues[128];
35
36/*
37#define prnonz(x, d)                                               \
38    uint32_t x = e10k_vf_vf##x##_rd(d);                           \
39    snprintf(str[cnt++], 32, #x "=%x \n", x);                      \
40
41static void stats_dump(e10k_vf_t* d)
42{
43  char str[256][32];
44  int cnt = 0;
45  memset(str, 0, 256 * 32);
46
47    prnonz(ctrl, d);
48    prnonz(status, d);
49    prnonz(links, d);
50    prnonz(rxmemwrap, d);
51    prnonz(eicr, d);
52    prnonz(eics, d);
53    prnonz(eims, d);
54    prnonz(gprc, d);
55    prnonz(gptc, d);
56
57    if(cnt > 0) {
58      for(int i = 0; i < cnt; i++) {
59	    printf("PF: %s ", str[i]);
60      }
61      printf("\n");
62    }
63}
64
65static void print_packet(void* buf, size_t len)
66{
67    printf("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ \n");
68    uint8_t* b = (uint8_t*) buf;
69    for (int i = 0; i < len; i++) {
70        printf("%2X ", b[i]);
71        if ((i % 16) == 0) {
72            printf("\n 0x");
73        }
74    }
75
76    printf(" \n @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ \n");
77}
78*/
79
80/******************************************************************************/
81/* Misc functions */
82
83static inline bool buf_use_tcpxsm(uint64_t flags)
84{
85    return (flags & NETIF_TXFLAG_TCPCHECKSUM);
86}
87
88static inline bool buf_use_udpxsm(uint64_t flags)
89{
90    return (flags & NETIF_TXFLAG_UDPCHECKSUM);
91}
92
93static inline bool buf_use_ipxsm(uint64_t flags)
94{
95    return (flags & NETIF_TXFLAG_IPCHECKSUM) ||
96        buf_use_tcpxsm(flags) || buf_use_udpxsm(flags);
97}
98
99static inline uint8_t buf_tcphdrlen(uint64_t flags)
100{
101    return ((flags & NETIF_TXFLAG_TCPHDRLEN_MASK) >>
102        NETIF_TXFLAG_TCPHDRLEN_SHIFT) * 4;
103}
104
105static errval_t update_txtail(struct e10k_queue* q, size_t tail)
106{
107    assert(q->d != NULL);
108
109    if (q->use_vf) {
110        e10k_vf_vftdt_wr(q->d, q->id, tail);
111    } else {
112        e10k_tdt_wr(q->d, q->id, tail);
113    }
114    return SYS_ERR_OK;
115}
116
117static errval_t update_rxtail(struct e10k_queue* q, size_t tail)
118{
119    assert(q->d != NULL);
120
121    if (q->use_vf) {
122        e10k_vf_vfrdt_wr(q->d, q->id, tail);
123    } else {
124        e10k_rdt_1_wr(q->d, q->id, tail);
125    }
126    return SYS_ERR_OK;
127}
128
129
130static struct region_entry* get_region(struct e10k_queue* q, regionid_t rid)
131{
132    struct region_entry* entry = q->regions;
133    while (entry != NULL) {
134        if (entry->rid == rid) {
135            return entry;
136        }
137        entry = entry->next;
138    }
139    return NULL;
140}
141
142static errval_t enqueue_tx_buf(struct e10k_queue* q, regionid_t rid,
143                               genoffset_t offset,
144                               genoffset_t length,
145                               genoffset_t valid_data,
146                               genoffset_t valid_length,
147                               uint64_t flags)
148{
149    if (e10k_queue_free_txslots(q) == 0) {
150        DEBUG_QUEUE("e10k_%d: Not enough space in TX ring, not adding buffer\n",
151                q->id);
152        // TODO better error
153        return NIC_ERR_ENQUEUE;
154    }
155
156    // Prepare checksum offload
157    //
158    struct region_entry* entry = get_region(q, rid);
159    assert(entry != NULL);
160
161    e10k_q_l4_type_t l4t = 0;
162    uint8_t l4len = 0;
163
164
165    DEBUG_QUEUE("TX Enqueuing offset=%lu valid_data=%lu phys=%lx txhwb=%d tx_tail=%zu tx_head=%zu"
166                " flags =%lx valid_length=%lu length=%lu\n",
167           offset, valid_data, entry->mem.devaddr + offset + valid_data,
168           (q->tx_hwb == NULL) ? 0 : *((uint32_t*)q->tx_hwb), q->tx_tail,
169           q->tx_head, flags, valid_length, length);
170
171    if (buf_use_ipxsm(flags)) {
172
173        if (buf_use_tcpxsm(flags)) {
174            l4t = e10k_q_tcp;
175            l4len = buf_tcphdrlen(flags);
176        } else if (buf_use_udpxsm(flags)) {
177            l4t = e10k_q_udp;
178            l4len = UDPHDR_LEN;
179        }
180
181        e10k_queue_add_txcontext(q, 0, ETHHDR_LEN, IPHDR_LEN, l4len, l4t);
182
183        lpaddr_t addr = 0;
184        addr = (lpaddr_t) entry->mem.devaddr + offset + valid_data;
185        e10k_queue_add_txbuf_ctx(q, addr, rid, offset, length,
186                                 valid_data, valid_length, flags,
187                                 valid_length, 0, true, l4len != 0);
188    } else {
189        lpaddr_t addr;
190        addr = (lpaddr_t) entry->mem.devaddr + offset + valid_data;
191
192        e10k_queue_add_txbuf(q, addr, rid, offset, length, valid_data,
193                             valid_length, flags,
194                             valid_length);
195    }
196    e10k_queue_bump_txtail(q);
197    return SYS_ERR_OK;
198}
199
200
201static errval_t enqueue_rx_buf(struct e10k_queue* q, regionid_t rid,
202                               genoffset_t offset,
203                               genoffset_t length,
204                               genoffset_t valid_data,
205                               genoffset_t valid_length,
206                               uint64_t flags)
207{
208    //DEBUG_QUEUE("Enqueueing RX buf \n");
209    // check if there is space
210    if (e10k_queue_free_rxslots(q) == 0) {
211        DEBUG_QUEUE("e10k_%d: Not enough space in RX ring, not adding buffer\n",
212                q->id);
213        // TODO better error
214        return NIC_ERR_ENQUEUE;
215    }
216
217    // get virtual address of buffer
218    struct region_entry* entry = get_region(q, rid);
219    assert(entry != NULL);
220
221    lpaddr_t addr = 0;
222    addr = (lpaddr_t) entry->mem.devaddr + offset;
223    e10k_queue_add_rxbuf(q, addr, rid, offset, length, valid_data,
224                         valid_length, flags);
225
226    e10k_queue_bump_rxtail(q);
227    return SYS_ERR_OK;
228}
229
230/******************************************************************************/
231/* Queue functions */
232
233static errval_t e10k_enqueue(struct devq* q, regionid_t rid, genoffset_t offset,
234                             genoffset_t length, genoffset_t valid_data,
235                             genoffset_t valid_length, uint64_t flags)
236{
237    errval_t err;
238
239
240    struct e10k_queue* queue = (struct e10k_queue*) q;
241    if (flags & NETIF_RXFLAG) {
242        /* can not enqueue receive buffer larger than 2048 bytes */
243        assert(length <= 2048);
244
245        err = enqueue_rx_buf(queue, rid, offset, length, valid_data,
246                             valid_length, flags);
247        if (err_is_fail(err)) {
248            return err;
249        }
250    } else if (flags & NETIF_TXFLAG) {
251
252        assert(length <= 2048);
253
254        err = enqueue_tx_buf(queue, rid, offset, length, valid_data,
255                             valid_length, flags);
256        if (err_is_fail(err)) {
257            return err;
258        }
259    }
260
261    return SYS_ERR_OK;
262}
263
264
265static errval_t e10k_dequeue(struct devq* q, regionid_t* rid,
266                             genoffset_t* offset, genoffset_t* length,
267                             genoffset_t* valid_data,
268                             genoffset_t* valid_length, uint64_t* flags)
269{
270    struct e10k_queue* que = (struct e10k_queue*) q;
271    int last;
272    errval_t err = SYS_ERR_OK;
273
274    if (!e10k_queue_get_txbuf(que, rid, offset, length, valid_data,
275                              valid_length, flags)) {
276        err = DEVQ_ERR_QUEUE_EMPTY;
277    }  else {
278        DEBUG_QUEUE("Queue %d sent offset=%lu valid_length=%lu \n",
279               que->id, *offset, *valid_length);
280        return SYS_ERR_OK;
281    }
282
283    if (!e10k_queue_get_rxbuf(que, rid, offset, length, valid_data,
284                             valid_length, flags, &last)) {
285        err = DEVQ_ERR_QUEUE_EMPTY;
286    } else {
287        DEBUG_QUEUE("Queue %d received offset=%lu valid_length=%lu \n",
288               que->id, *offset, *valid_length);
289        return SYS_ERR_OK;
290    }
291
292
293    return err;
294}
295
296static errval_t e10k_register(struct devq* q, struct capref cap, regionid_t rid)
297{
298    errval_t err;
299    struct e10k_queue* queue = (struct e10k_queue*) q;
300
301    struct frame_identity id;
302    err = frame_identify(cap, &id);
303    if (err_is_fail(err)) {
304        return err;
305    }
306
307    /*
308    void* va;
309    err = vspace_map_one_frame_attr(&va, id.bytes, cap,
310                                    VREGION_FLAGS_READ_WRITE,
311                                    NULL, NULL);
312    if (err_is_fail(err)) {
313        return err;
314    }
315
316    entry->mem.devaddr = id.bytes;
317    entry->mem.vbase = (lvaddr_t) va;
318    entry->mem.mem = cap;
319    */
320    // keep track of regions since we need the virtual address ...
321    struct region_entry* entry = malloc(sizeof(struct region_entry));
322    entry->rid = rid;
323    entry->next = NULL;
324
325    struct iommu_client* cl = e10k_vf_get_iommu_client();
326
327    err = driverkit_iommu_vspace_map_cl(cl, cap,
328                                        VREGION_FLAGS_READ_WRITE,
329                                        &entry->mem);
330    if (err_is_fail(err)) {
331        free(entry);
332        return err;
333    }
334
335    DEBUG_QUEUE("register region id %d base=%lx \n", rid, entry->mem.devaddr);
336    // linked list of regions
337    struct region_entry* cur = queue->regions;
338    if (cur == NULL) {
339        queue->regions = entry;
340        return SYS_ERR_OK;
341    }
342
343    while (cur->next != NULL) {
344        cur = cur->next;
345    }
346
347    cur->next = entry;
348
349    DEBUG_QUEUE("registerd region id %d base=%p len=%ld \n", rid,
350                (void*) entry->mem.vbase, entry->mem.size);
351
352    return SYS_ERR_OK;
353}
354
355static errval_t e10k_deregister(struct devq* q, regionid_t rid)
356{
357    return SYS_ERR_OK;
358}
359
360static errval_t e10k_control(struct devq* q, uint64_t cmd, uint64_t value, uint64_t *result)
361{
362    struct e10k_queue* queue = (struct e10k_queue*) q;
363    *result = queue->mac;
364    return SYS_ERR_OK;
365}
366
367
368static errval_t e10k_notify(struct devq* q)
369{
370    return SYS_ERR_OK;
371}
372
373static errval_t e10k_destroy(struct devq* queue)
374{
375    struct e10k_queue* q = (struct e10k_queue*) queue;
376    free(q);
377    //TODO: rest of the cleanup
378    return SYS_ERR_OK;
379}
380
381/******************************************************************
382 * Management functions
383 *
384 */
385
386static void interrupt_cb(struct e10k_vf_binding *b, uint16_t qid)
387{
388    struct e10k_queue* q = queues[qid];
389
390    DEBUG_QUEUE("Interrupt on queue %d \n", qid);
391    if (q != b->st) {
392        debug_printf("STATE MISMATCH!\n %p %p\n", q, b->st);
393        q = b->st;
394    }
395
396    q->cb(q);
397}
398
399static struct e10k_vf_rx_vtbl rx_vtbl = {
400    .interrupt = interrupt_cb,
401};
402
403static void bind_cb(void *st, errval_t err, struct e10k_vf_binding *b)
404{
405    struct e10k_queue* q = (struct e10k_queue*) st;
406    assert(err_is_ok(err));
407
408    DEBUG_QUEUE("Sucessfully connected to management interface\n");
409
410    b->st = q;
411    q->binding = b;
412    b->rx_vtbl = rx_vtbl;
413    e10k_vf_rpc_client_init(q->binding);
414    q->bound = true;
415}
416
417/** Connect to the management interface */
418static void connect_to_mngif(struct e10k_queue* q)
419{
420    errval_t r;
421    iref_t iref;
422
423    q->bound = false;
424    char name[strlen("e10k_vf") + 2];
425
426    // Build label for interal management service
427    sprintf(name, "%s%u", "e10k_vf", q->pci_function);
428
429    // Connect to service
430    r = nameservice_blocking_lookup(name, &iref);
431    assert(err_is_ok(r));
432
433    r = e10k_vf_bind(iref, bind_cb, q, get_default_waitset(),
434                     IDC_BIND_FLAGS_DEFAULT);
435    assert(err_is_ok(r));
436
437    while (!q->bound) {
438        event_dispatch(get_default_waitset());
439    }
440}
441
442/** Connect to the management interface */
443static errval_t connect_to_mngif_with_ep(struct e10k_queue* q, struct capref ep)
444{
445    errval_t err;
446
447    q->bound = false;
448
449    err = e10k_vf_bind_to_endpoint(ep, bind_cb, q, get_default_waitset(),
450                                 IDC_BIND_FLAGS_DEFAULT);
451    if (err_is_fail(err)) {
452        return err;
453    }
454
455    while (!q->bound) {
456        event_dispatch(get_default_waitset());
457    }
458    return SYS_ERR_OK;
459}
460
461/*********************************************************
462 * Queue creation
463 */
464
465static errval_t map_device_memory(struct e10k_queue* q,
466                                  struct capref regs)
467{
468
469    struct frame_identity id = {.base = 0, .bytes = 0};
470    errval_t err;
471
472    err = frame_identify(regs, &id);
473    if (err_is_fail(err)) {
474        return err;
475    }
476
477    void* va;
478    err = vspace_map_one_frame_attr(&va, id.bytes, regs, VREGION_FLAGS_READ_WRITE_NOCACHE,
479                                    NULL, NULL);
480    if (err_is_fail(err)) {
481        return err;
482    }
483
484    DEBUG_QUEUE("mapped %zu bytes at address %p\n", id.bytes,
485                va);
486    q->d = malloc(sizeof(e10k_t));
487    assert(q->d != NULL);
488    e10k_initialize(q->d, (void*) va);
489    return SYS_ERR_OK;
490}
491
492// TODO mostly cleanup when fail
493errval_t e10k_queue_create(struct e10k_queue** queue, e10k_event_cb_t cb, struct capref* ep,
494                           uint32_t bus, uint32_t function, uint32_t devid, uint32_t dev,
495                           bool use_vf, bool interrupts, bool qzero)
496{
497
498    errval_t err;
499    struct e10k_queue* q;
500    struct iommu_client* cl = NULL;
501    uint8_t vf_num;
502    // start VF driver
503
504    q = malloc(sizeof(struct e10k_queue));
505    assert(q);
506    q->pci_function = 0; // TODO allow also function 1
507    q->use_msix = false; // TODO init MSI-X
508
509    // txhwb
510    if (use_vf) {
511        q->use_txhwb = false;
512    } else {
513        // TODO revert to true
514        q->use_txhwb = true;
515    }
516    q->cb = cb;
517
518    if (use_vf) {
519        //USER_PANIC("NOT YET WORKING \n");
520        // Start VF
521        if (!e10k_vf_started()) {
522            err = e10k_init_vf_driver(ep, 0, 0, bus+1, dev+16, interrupts, &vf_num);
523            if (err_is_fail(err)) {
524                free(q);
525                *queue = NULL;
526                return err;
527            }
528        }
529
530        // If i can not create any more queues -> start new VF
531        if (!e10k_vf_can_create_queue()) {
532            err = e10k_init_vf_driver(ep, 0, 0, bus+1, dev+17, interrupts, &vf_num);
533            if (err_is_fail(err)) {
534                free(q);
535                return err;
536            }
537        }
538
539        q->use_vtd = true;
540        cl = e10k_vf_get_iommu_client();
541        DEBUG_QUEUE("Assume enabled VT-d \n");
542    } else {
543        q->use_vtd = false;
544        // need to set up communicaton to PF
545        if (ep == NULL || capref_is_null(*ep)) {
546            DEBUG_QUEUE("Connect to PF \n");
547            connect_to_mngif(q);
548        } else {
549            DEBUG_QUEUE("Connect to PF using EP\n");
550            err = connect_to_mngif_with_ep(q, *ep);
551            if (err_is_fail(err)) {
552                free(q);
553                return err;
554            }
555        }
556    }
557
558
559    // allocate memory for RX/TX rings
560    q->tx_ring_size = e10k_q_tdesc_adv_wb_size*NUM_TX_DESC;
561
562    DEBUG_QUEUE("Allocating TX queue memory\n");
563    err = driverkit_iommu_mmap_cl(cl, q->tx_ring_size, VREGION_FLAGS_READ_WRITE,
564                                  &q->tx);
565    if (err_is_fail(err)) {
566        // TODO cleanup
567        return DEVQ_ERR_INIT_QUEUE;
568    }
569
570
571    DEBUG_QUEUE("Allocated TX queue memory is=%lu requested=%lu \n", q->tx.size, q->tx_ring_size);
572    DEBUG_QUEUE("Allocating RX queue memory\n");
573    q->rx_ring_size = e10k_q_rdesc_adv_wb_size*NUM_RX_DESC;
574    err = driverkit_iommu_mmap_cl(cl, q->rx_ring_size, VREGION_FLAGS_READ_WRITE,
575                          &q->rx);
576    if (err_is_fail(err)) {
577        // TODO cleanup
578        return DEVQ_ERR_INIT_QUEUE;
579    }
580
581    DEBUG_QUEUE("Allocated RX queue memory is=%lu requested=%lu \n", q->rx.size, q->rx_ring_size);
582    struct e10k_queue_ops ops = {
583        .update_txtail = update_txtail,
584        .update_rxtail = update_rxtail,
585    };
586
587    if (q->use_txhwb) {
588        DEBUG_QUEUE("Allocating TX HWB queue memory\n");
589        err = driverkit_iommu_mmap_cl(cl, BASE_PAGE_SIZE, VREGION_FLAGS_READ_WRITE,
590                                       &q->txhwb);
591        if (err_is_fail(err)) {
592            // TODO cleanup
593            return DEVQ_ERR_INIT_QUEUE;
594        }
595        memset((void*) q->txhwb.vbase, 0, sizeof(uint32_t));
596    }
597
598    DEBUG_QUEUE("Init queue struct\n");
599    e10k_queue_init(q, (void*) q->tx.vbase, NUM_TX_DESC, (void*) q->txhwb.vbase,
600                    (void*) q->rx.vbase, NUM_RX_DESC, &ops);
601
602    DEBUG_QUEUE("Local queue init done\n");
603
604    q->use_vf = use_vf;
605    q->use_irq = interrupts;
606    q->rx_frame = q->rx.mem;
607    q->tx_frame = q->tx.mem;
608    q->txhwb_frame = q->txhwb.mem;
609
610    // XXX:disable by default for now
611    q->use_rsc = false;
612
613    if (q->use_vf) {
614        err = e10k_vf_init_queue_hw(q, vf_num);
615        if (err_is_fail(err)) {
616            return err;
617        }
618    } else {
619
620        int qid;
621        errval_t err2;
622        struct capref regs;
623
624        if (q->use_irq) {
625            /*
626            err = pci_setup_inthandler(interrupt_handler, NULL, &vector);
627            assert(err_is_ok(err));
628            core = disp_get_core_id();
629            */
630            // TODO setup MSI-X interrupts
631        }
632
633        // Inform card driver about new queue and get the registers/queue id
634        err = slot_alloc(&regs);
635        if (err_is_fail(err)) {
636            return err;
637        }
638
639        err = slot_alloc(&(q->filter_ep));
640        if (err_is_fail(err)) {
641            slot_free(regs);
642            return err;
643        }
644
645        err = q->binding->rpc_tx_vtbl.create_queue(q->binding, q->tx.mem, q->txhwb.mem,
646                                                   q->rx.mem, 2048, disp_get_core_id(),
647                                                   q->use_irq, q->use_msix, false, qzero,
648                                                   &q->mac, &qid,
649                                                   &regs, &q->filter_ep, &err2);
650
651        if (err_is_fail(err) || err_is_fail(err2)) {
652            DEBUG_QUEUE("e10k rpc error\n");
653            return err_is_fail(err)? err: err2;
654        }
655
656        assert(qid >= 0);
657        q->id = (uint16_t)qid;
658
659        err = map_device_memory(q, regs);
660        if (err_is_fail(err)) {
661            DEBUG_QUEUE("e10k map device error\n");
662            return err;
663        }
664
665        update_txtail(q, 0);
666        update_rxtail(q, 0);
667
668    }
669
670    err = devq_init(&q->q, false);
671    if (err_is_fail(err)) {
672        DEBUG_QUEUE("e10k devq_init error\n");
673        return err;
674    }
675
676    q->q.f.enq = e10k_enqueue;
677    q->q.f.deq = e10k_dequeue;
678    q->q.f.reg = e10k_register;
679    q->q.f.dereg = e10k_deregister;
680    q->q.f.ctrl = e10k_control;
681    q->q.f.notify = e10k_notify;
682    q->q.f.destroy = e10k_destroy;
683    q->q.iommu = cl;
684
685    *queue = q;
686    queues[q->id] = q;
687
688    DEBUG_QUEUE("e10k queue init done\n");
689    return SYS_ERR_OK;
690}
691
692
693void e10k_queue_get_netfilter_ep(struct e10k_queue* q, struct capref* ep)
694{
695    *ep = q->filter_ep;
696}
697
698uint64_t e10k_queue_get_id(struct e10k_queue* q)
699{
700    return q->id;
701}
702
703struct bench_ctl* e10k_get_benchmark_data(struct devq* q, uint8_t type)
704{
705    return e10k_queue_get_benchmark_data((struct e10k_queue*) q, type);
706}
707
708