1/*
2 * Copyright (c) 2007-2011, ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#include <barrelfish/barrelfish.h>
11#include <barrelfish/nameservice_client.h>
12#include <devif/queue_interface.h>
13#include <dev/e10k_q_dev.h>
14#include <dev/e10k_dev.h>
15#include <dev/e10k_vf_dev.h>
16#include <skb/skb.h>
17
18#include <if/e10k_vf_defs.h>
19#include <if/e10k_vf_rpcclient_defs.h>
20
21#include "e10k_devif_vf.h"
22#include "helper.h"
23#include "e10k_queue.h"
24#include "debug.h"
25
26#define NUM_TX_DESC 2048
27#define NUM_RX_DESC 2048
28
29#define ETHHDR_LEN 14
30#define IPHDR_LEN 20
31#define UDPHDR_LEN 8
32
33// for debugging
34static e10k_t* d;
35
36// TODO only required for legacy interrupts
37struct e10k_queue* queues[128];
38
39/******************************************************************************/
40/* Misc functions */
41
42
43static inline bool buf_use_tcpxsm(uint64_t flags)
44{
45    return (flags & NETIF_TXFLAG_TCPCHECKSUM);
46}
47
48static inline bool buf_use_udpxsm(uint64_t flags)
49{
50    return (flags & NETIF_TXFLAG_UDPCHECKSUM);
51}
52
53static inline bool buf_use_ipxsm(uint64_t flags)
54{
55    return (flags & NETIF_TXFLAG_IPCHECKSUM) ||
56        buf_use_tcpxsm(flags) || buf_use_udpxsm(flags);
57}
58
59static inline uint8_t buf_tcphdrlen(uint64_t flags)
60{
61    return ((flags & NETIF_TXFLAG_TCPHDRLEN_MASK) >>
62        NETIF_TXFLAG_TCPHDRLEN_SHIFT) * 4;
63}
64
65static errval_t update_txtail(struct e10k_queue* q, size_t tail)
66{
67    assert(q->d != NULL);
68
69    if (q->use_vf) {
70        e10k_vf_vftdt_wr(q->d, q->id, tail);
71    } else {
72        e10k_tdt_wr(q->d, q->id, tail);
73    }
74    return SYS_ERR_OK;
75}
76
77static errval_t update_rxtail(struct e10k_queue* q, size_t tail)
78{
79    assert(q->d != NULL);
80
81    if (q->use_vf) {
82        e10k_vf_vfrdt_wr(q->d, q->id, tail);
83    } else {
84        e10k_rdt_1_wr(q->d, q->id, tail);
85    }
86    return SYS_ERR_OK;
87}
88
89
90static struct region_entry* get_region(struct e10k_queue* q, regionid_t rid)
91{
92    struct region_entry* entry = q->regions;
93    while (entry != NULL) {
94        if (entry->rid == rid) {
95            return entry;
96        }
97        entry = entry->next;
98    }
99    return NULL;
100}
101
102static errval_t enqueue_tx_buf(struct e10k_queue* q, regionid_t rid,
103                               genoffset_t offset,
104                               genoffset_t length,
105                               genoffset_t valid_data,
106                               genoffset_t valid_length,
107                               uint64_t flags)
108{
109    DEBUG_QUEUE("Enqueueing TX buf \n");
110
111    if (e10k_queue_free_txslots(q) == 0) {
112        DEBUG_QUEUE("e10k_%d: Not enough space in TX ring, not adding buffer\n",
113                q->id);
114        // TODO better error
115        return NIC_ERR_ENQUEUE;
116    }
117
118    // Prepare checksum offload
119    //
120    struct region_entry* entry = get_region(q, rid);
121    assert(entry != NULL);
122
123    e10k_q_l4_type_t l4t = 0;
124    uint8_t l4len = 0;
125
126    if (buf_use_ipxsm(flags)) {
127
128        if (buf_use_tcpxsm(flags)) {
129            l4t = e10k_q_tcp;
130            l4len = buf_tcphdrlen(flags);
131        } else if (buf_use_udpxsm(flags)) {
132            l4t = e10k_q_udp;
133            l4len = UDPHDR_LEN;
134        }
135
136        e10k_queue_add_txcontext(q, 0, ETHHDR_LEN, IPHDR_LEN, l4len, l4t);
137
138	    if (q->use_vtd) {
139            lpaddr_t addr = 0;
140            addr = (lpaddr_t) entry->virt + offset + valid_data;
141            e10k_queue_add_txbuf_ctx(q, addr, rid, offset, length,
142                                     valid_data, valid_length, flags,
143                                     valid_length, 0, true, l4len !=0);
144	    } else {
145            lpaddr_t addr = 0;
146            addr = (lpaddr_t) entry->phys + offset + valid_data;
147            e10k_queue_add_txbuf_ctx(q, addr, rid, offset, length,
148                                     valid_data, valid_length, flags,
149                                     valid_length, 0, true, l4len != 0);
150        }
151    } else {
152        if (q->use_vtd) {
153            lvaddr_t addr = 0;
154            addr = (lvaddr_t) entry->virt + offset + valid_data;
155            e10k_queue_add_txbuf(q, addr, rid, offset, length, valid_data,
156                                 valid_length, flags,
157                                 valid_length);
158        } else {
159            lpaddr_t addr;
160            addr = (lpaddr_t) entry->phys + offset + valid_data;
161
162            e10k_queue_add_txbuf(q, addr, rid, offset, length, valid_data,
163                                 valid_length, flags,
164                                 valid_length);
165        }
166    }
167    e10k_queue_bump_txtail(q);
168    return SYS_ERR_OK;
169}
170
171
172static errval_t enqueue_rx_buf(struct e10k_queue* q, regionid_t rid,
173                               genoffset_t offset,
174                               genoffset_t length,
175                               genoffset_t valid_data,
176                               genoffset_t valid_length,
177                               uint64_t flags)
178{
179    //DEBUG_QUEUE("Enqueueing RX buf \n");
180    // check if there is space
181    if (e10k_queue_free_rxslots(q) == 0) {
182        DEBUG_QUEUE("e10k_%d: Not enough space in RX ring, not adding buffer\n",
183                q->id);
184        // TODO better error
185        return NIC_ERR_ENQUEUE;
186    }
187
188    if (q->use_vtd) {
189        // get virtual address of buffer
190        struct region_entry* entry = get_region(q, rid);
191        assert(entry != NULL);
192
193        lpaddr_t addr = 0;
194        addr = (lpaddr_t) entry->virt + offset;
195        e10k_queue_add_rxbuf(q, addr, rid, offset, length, valid_data,
196                             valid_length, flags);
197    } else {
198        // get virtual address of buffer
199        struct region_entry* entry = get_region(q, rid);
200        assert(entry != NULL);
201
202        lpaddr_t addr = 0;
203        addr = (lpaddr_t) entry->phys + offset;
204        e10k_queue_add_rxbuf(q, addr, rid, offset, length, valid_data,
205                             valid_length, flags);
206    }
207
208    e10k_queue_bump_rxtail(q);
209    return SYS_ERR_OK;
210}
211
212/******************************************************************************/
213/* Queue functions */
214
215static errval_t e10k_enqueue(struct devq* q, regionid_t rid, genoffset_t offset,
216                             genoffset_t length, genoffset_t valid_data,
217                             genoffset_t valid_length, uint64_t flags)
218{
219    errval_t err;
220
221
222    struct e10k_queue* queue = (struct e10k_queue*) q;
223    if (flags & NETIF_RXFLAG) {
224        /* can not enqueue receive buffer larger than 2048 bytes */
225        assert(length <= 2048);
226
227        err = enqueue_rx_buf(queue, rid, offset, length, valid_data,
228                             valid_length, flags);
229        if (err_is_fail(err)) {
230            return err;
231        }
232    } else if (flags & NETIF_TXFLAG) {
233
234        assert(length <= 2048);
235
236        DEBUG_QUEUE("Enqueuing offset=%lu valid_data=%lu txhwb=%d tx_tail=%zu tx_head=%zu \n",
237               offset, valid_data, (queue->tx_hwb == NULL) ? 0 : *((uint32_t*)queue->tx_hwb), queue->tx_tail,
238               queue->tx_head);
239
240        err = enqueue_tx_buf(queue, rid, offset, length, valid_data,
241                             valid_length, flags);
242        if (err_is_fail(err)) {
243            return err;
244        }
245    }
246
247    return SYS_ERR_OK;
248}
249
250
251static errval_t e10k_dequeue(struct devq* q, regionid_t* rid,
252                             genoffset_t* offset, genoffset_t* length,
253                             genoffset_t* valid_data,
254                             genoffset_t* valid_length, uint64_t* flags)
255{
256    struct e10k_queue* que = (struct e10k_queue*) q;
257    int last;
258    errval_t err = SYS_ERR_OK;
259
260    if (!e10k_queue_get_txbuf(que, rid, offset, length, valid_data,
261                              valid_length, flags)) {
262        err = DEVQ_ERR_QUEUE_EMPTY;
263    }  else {
264        DEBUG_QUEUE("Queue %d sent offset=%lu valid_length=%lu transmit count %d\n",
265               que->id, *offset, *valid_length, e10k_vf_vfgptc_rd(que->d));
266        return SYS_ERR_OK;
267    }
268
269    if (!e10k_queue_get_rxbuf(que, rid, offset, length, valid_data,
270                             valid_length, flags, &last)) {
271        err = DEVQ_ERR_QUEUE_EMPTY;
272    } else {
273        DEBUG_QUEUE("Queue %d received offset=%lu valid_length=%lu \n",
274               que->id, *offset, *valid_length);
275        return SYS_ERR_OK;
276    }
277
278
279    return err;
280}
281
282static errval_t e10k_register(struct devq* q, struct capref cap, regionid_t rid)
283{
284    errval_t err;
285    struct e10k_queue* queue = (struct e10k_queue*) q;
286
287    struct frame_identity id;
288    err = invoke_frame_identify(cap, &id);
289    if (err_is_fail(err)) {
290        return err;
291    }
292
293    void* va;
294    err = vspace_map_one_frame_attr(&va, id.bytes, cap,
295                                    VREGION_FLAGS_READ_WRITE,
296                                    NULL, NULL);
297    if (err_is_fail(err)) {
298        return err;
299    }
300
301    // keep track of regions since we need the virtual address ...
302    struct region_entry* entry = malloc(sizeof(struct region_entry));
303    entry->rid = rid;
304    entry->cap = cap;
305    entry->phys = id.base;
306    entry->virt = (lvaddr_t)va;
307    entry->size = id.bytes;
308    entry->next = NULL;
309
310    // linked list of regions
311    struct region_entry* cur = queue->regions;
312    if (cur == NULL) {
313        queue->regions = entry;
314        return SYS_ERR_OK;
315    }
316
317    while (cur->next != NULL) {
318        cur = cur->next;
319    }
320
321    cur->next = entry;
322
323    return SYS_ERR_OK;
324}
325
326static errval_t e10k_deregister(struct devq* q, regionid_t rid)
327{
328    return SYS_ERR_OK;
329}
330
331static errval_t e10k_control(struct devq* q, uint64_t cmd, uint64_t value, uint64_t *result)
332{
333    struct e10k_queue* queue = (struct e10k_queue*) q;
334    *result = queue->mac;
335    return SYS_ERR_OK;
336}
337
338
339static errval_t e10k_notify(struct devq* q)
340{
341    return SYS_ERR_OK;
342}
343
344static errval_t e10k_destroy(struct devq* queue)
345{
346    struct e10k_queue* q = (struct e10k_queue*) queue;
347    free(q);
348    //TODO: rest of the cleanup
349    return SYS_ERR_OK;
350}
351
352/******************************************************************
353 * Management functions
354 *
355 */
356
357static void interrupt_cb(struct e10k_vf_binding *b, uint16_t qid)
358{
359    struct e10k_queue* q = queues[qid];
360
361    if (q != b->st) {
362        debug_printf("STATE MISMATCH!\n %p %p\n", q, b->st);
363        q = b->st;
364    }
365
366    q->cb(q);
367}
368
369static struct e10k_vf_rx_vtbl rx_vtbl = {
370    .interrupt = interrupt_cb,
371};
372
373static void bind_cb(void *st, errval_t err, struct e10k_vf_binding *b)
374{
375    struct e10k_queue* q = (struct e10k_queue*) st;
376    assert(err_is_ok(err));
377
378    DEBUG_QUEUE("Sucessfully connected to management interface\n");
379
380    b->st = q;
381    q->binding = b;
382    b->rx_vtbl = rx_vtbl;
383    e10k_vf_rpc_client_init(q->binding);
384    q->bound = true;
385}
386
387/** Connect to the management interface */
388static void connect_to_mngif(struct e10k_queue* q)
389{
390    errval_t r;
391    iref_t iref;
392
393    q->bound = false;
394    char name[strlen("e10k_vf") + 2];
395
396    // Build label for interal management service
397    sprintf(name, "%s%u", "e10k_vf", q->pci_function);
398
399    // Connect to service
400    DEBUG_QUEUE("Looking up management interface (%s)\n", name);
401    r = nameservice_blocking_lookup(name, &iref);
402    assert(err_is_ok(r));
403
404    DEBUG_QUEUE("Binding to management interface\n");
405    r = e10k_vf_bind(iref, bind_cb, q, get_default_waitset(),
406                     IDC_BIND_FLAGS_DEFAULT);
407    assert(err_is_ok(r));
408
409    while (!q->bound) {
410        event_dispatch(get_default_waitset());
411    }
412}
413
414/*********************************************************
415 * Queue creation
416 */
417
418static errval_t map_device_memory(struct e10k_queue* q,
419                                  struct capref regs)
420{
421
422    struct frame_identity id = {.base = 0, .bytes = 0};
423    errval_t err;
424
425    err = invoke_frame_identify(regs, &id);
426    if (err_is_fail(err)) {
427        return err;
428    }
429
430    void* va;
431    err = vspace_map_one_frame_attr(&va, id.bytes, regs,
432                                    VREGION_FLAGS_READ_WRITE_NOCACHE,
433                                    NULL, NULL);
434    if (err_is_fail(err)) {
435        return err;
436    }
437
438
439    DEBUG_QUEUE("mapped %zu bytes at address %p\n", id.bytes, va);
440    q->d = malloc(sizeof(e10k_t));
441    assert(q->d != NULL);
442    e10k_initialize(q->d, va);
443    d = q->d;
444    return SYS_ERR_OK;
445}
446
447// TODO mostly cleanup when fail
448errval_t e10k_queue_create(struct e10k_queue** queue, e10k_event_cb_t cb,
449                           uint32_t bus, uint32_t function, uint32_t devid, uint32_t dev,
450                           bool use_vf, bool interrupts, bool qzero)
451{
452
453    errval_t err;
454    struct e10k_queue* q;
455    // start VF driver
456
457    q = malloc(sizeof(struct e10k_queue));
458    q->pci_function = 0; // TODO allow also function 1
459
460    // txhwb
461    if (use_vf) {
462        q->use_txhwb = false;
463    } else {
464        q->use_txhwb = true;
465    }
466    q->cb = cb;
467
468    if (use_vf) {
469        //USER_PANIC("NOT YET WORKING \n");
470        // Start VF
471        if (!e10k_vf_started()) {
472            err = e10k_init_vf_driver(0, 0, bus+1, dev+16, devid, interrupts);
473            if (err_is_fail(err)) {
474                return err;
475            }
476        }
477
478        // If i can not create any more queues -> start new VF
479        if (!e10k_vf_can_create_queue()) {
480            err = e10k_init_vf_driver(0, 0, bus+1, dev+16, devid, interrupts);
481            if (err_is_fail(err)) {
482                return err;
483            }
484        }
485
486        err = skb_client_connect();
487        assert(err_is_ok(err));
488        err = skb_execute_query("vtd_enabled(0,_).");
489        if (err_is_fail(err)) {
490            DEBUG_QUEUE("Assume disabled VT-d \n");
491            q->use_vtd = false;
492        } else {
493            DEBUG_QUEUE("Assume enabled VT-d \n");
494            q->use_vtd = true;
495        }
496
497    } else {
498        q->use_vtd = false;
499        // need to set up communicaton to PF
500        connect_to_mngif(q);
501    }
502
503    // allocate memory for RX/TX rings
504    struct capref tx_frame;
505    size_t tx_size = e10k_q_tdesc_adv_wb_size*NUM_TX_DESC;
506    void* tx_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, tx_size, &tx_frame);
507    if (tx_virt == NULL) {
508        return DEVQ_ERR_INIT_QUEUE;
509    }
510
511    struct capref rx_frame;
512    size_t rx_size = e10k_q_rdesc_adv_wb_size*NUM_RX_DESC;
513    void* rx_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, rx_size, &rx_frame);
514    if (rx_virt == NULL) {
515        return DEVQ_ERR_INIT_QUEUE;
516    }
517
518    struct e10k_queue_ops ops = {
519        .update_txtail = update_txtail,
520        .update_rxtail = update_rxtail,
521    };
522
523    struct capref txhwb_frame;
524    void* txhwb_virt = NULL;
525
526    if (q->use_txhwb) {
527        txhwb_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, BASE_PAGE_SIZE,
528        &txhwb_frame);
529        if (txhwb_virt == NULL) {
530            return DEVQ_ERR_INIT_QUEUE;
531        }
532        memset(txhwb_virt, 0, sizeof(uint32_t));
533    }
534
535    e10k_queue_init(q, tx_virt, NUM_TX_DESC, txhwb_virt,
536                    rx_virt, NUM_RX_DESC, &ops);
537
538    DEBUG_QUEUE("Local queue init done\n");
539
540    q->use_vf = use_vf;
541    q->rx_frame = rx_frame;
542    q->tx_frame = tx_frame;
543    q->txhwb_frame = txhwb_frame;
544    q->use_irq = interrupts;
545
546    // XXX:disable by default for now
547    q->use_rsc = false;
548
549    if (q->use_vf) {
550        err = e10k_vf_init_queue_hw(q);
551        if (err_is_fail(err)) {
552            return err;
553        }
554    } else {
555
556        int qid;
557        errval_t err2;
558        struct capref regs;
559
560        if (q->use_irq) {
561            /*
562            err = pci_setup_inthandler(interrupt_handler, NULL, &vector);
563            assert(err_is_ok(err));
564            core = disp_get_core_id();
565            */
566            // TODO setup MSI-X interrupts
567        }
568
569        // Inform card driver about new queue and get the registers/queue id
570        err = slot_alloc(&regs);
571        if (err_is_fail(err)) {
572            return err;
573        }
574
575        err = q->binding->rpc_tx_vtbl.create_queue(q->binding, tx_frame, txhwb_frame,
576                                            rx_frame, 2048, q->msix_intvec,
577                                            q->msix_intdest, q->use_irq, false, qzero,
578                                            &q->mac, &qid,
579                                            &regs, &err2);
580        if (err_is_fail(err) || err_is_fail(err2)) {
581            DEBUG_QUEUE("e10k rpc error\n");
582            return err_is_fail(err)? err: err2;
583        }
584
585        assert(qid >= 0);
586        q->id = (uint16_t)qid;
587
588        err = map_device_memory(q, regs);
589        if (err_is_fail(err)) {
590            DEBUG_QUEUE("e10k map device error\n");
591            return err;
592        }
593
594        update_txtail(q, 0);
595        update_rxtail(q, 0);
596
597    }
598
599    err = devq_init(&q->q, false);
600    if (err_is_fail(err)) {
601        DEBUG_QUEUE("e10k devq_init error\n");
602        return err;
603    }
604
605    q->q.f.enq = e10k_enqueue;
606    q->q.f.deq = e10k_dequeue;
607    q->q.f.reg = e10k_register;
608    q->q.f.dereg = e10k_deregister;
609    q->q.f.ctrl = e10k_control;
610    q->q.f.notify = e10k_notify;
611    q->q.f.destroy = e10k_destroy;
612
613
614    *queue = q;
615    queues[q->id] = q;
616
617    DEBUG_QUEUE("e10k queue init done\n");
618    return SYS_ERR_OK;
619}
620
621uint64_t e10k_queue_get_id(struct e10k_queue* q)
622{
623    return q->id;
624}
625