4a5
> * Copyright (c) 2016 Mellanox Technologies.
33c34
< __FBSDID("$FreeBSD: head/sys/netinet/tcp_lro.c 284961 2015-06-30 17:19:58Z np $");
---
> __FBSDID("$FreeBSD: head/sys/netinet/tcp_lro.c 294327 2016-01-19 15:33:28Z hselasky $");
61,63c62
< #ifndef LRO_ENTRIES
< #define LRO_ENTRIES 8 /* # of LRO entries per RX queue. */
< #endif
---
> static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures");
72a72,78
> return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0));
> }
>
> int
> tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp,
> unsigned lro_entries, unsigned lro_mbufs)
> {
74c80,81
< int error, i;
---
> size_t size;
> unsigned i;
79a87,90
> lc->lro_mbuf_count = 0;
> lc->lro_mbuf_max = lro_mbufs;
> lc->lro_cnt = lro_entries;
> lc->ifp = ifp;
83,94c94,98
< error = 0;
< for (i = 0; i < LRO_ENTRIES; i++) {
< le = (struct lro_entry *)malloc(sizeof(*le), M_DEVBUF,
< M_NOWAIT | M_ZERO);
< if (le == NULL) {
< if (i == 0)
< error = ENOMEM;
< break;
< }
< lc->lro_cnt = i + 1;
< SLIST_INSERT_HEAD(&lc->lro_free, le, next);
< }
---
> /* compute size to allocate */
> size = (lro_mbufs * sizeof(struct mbuf *)) +
> (lro_entries * sizeof(*le));
> lc->lro_mbuf_data = (struct mbuf **)
> malloc(size, M_LRO, M_NOWAIT | M_ZERO);
96c100,113
< return (error);
---
> /* check for out of memory */
> if (lc->lro_mbuf_data == NULL) {
> memset(lc, 0, sizeof(*lc));
> return (ENOMEM);
> }
> /* compute offset for LRO entries */
> le = (struct lro_entry *)
> (lc->lro_mbuf_data + lro_mbufs);
>
> /* setup linked list */
> for (i = 0; i != lro_entries; i++)
> SLIST_INSERT_HEAD(&lc->lro_free, le + i, next);
>
> return (0);
102a120
> unsigned x;
104,107c122,128
< while (!SLIST_EMPTY(&lc->lro_free)) {
< le = SLIST_FIRST(&lc->lro_free);
< SLIST_REMOVE_HEAD(&lc->lro_free, next);
< free(le, M_DEVBUF);
---
> /* reset LRO free list */
> SLIST_INIT(&lc->lro_free);
>
> /* free active mbufs, if any */
> while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) {
> SLIST_REMOVE_HEAD(&lc->lro_active, next);
> m_freem(le->m_head);
108a130,138
>
> /* free mbuf array, if any */
> for (x = 0; x != lc->lro_mbuf_count; x++)
> m_freem(lc->lro_mbuf_data[x]);
> lc->lro_mbuf_count = 0;
>
> /* free allocated memory, if any */
> free(lc->lro_mbuf_data, M_LRO);
> lc->lro_mbuf_data = NULL;
307a338,414
> static int
> tcp_lro_mbuf_compare_header(const void *ppa, const void *ppb)
> {
> const struct mbuf *ma = *((const struct mbuf * const *)ppa);
> const struct mbuf *mb = *((const struct mbuf * const *)ppb);
> int ret;
>
> ret = M_HASHTYPE_GET(ma) - M_HASHTYPE_GET(mb);
> if (ret != 0)
> goto done;
>
> ret = ma->m_pkthdr.flowid - mb->m_pkthdr.flowid;
> if (ret != 0)
> goto done;
>
> ret = TCP_LRO_SEQUENCE(ma) - TCP_LRO_SEQUENCE(mb);
> done:
> return (ret);
> }
>
> void
> tcp_lro_flush_all(struct lro_ctrl *lc)
> {
> struct lro_entry *le;
> uint32_t hashtype;
> uint32_t flowid;
> unsigned x;
>
> /* check if no mbufs to flush */
> if (__predict_false(lc->lro_mbuf_count == 0))
> goto done;
>
> /* sort all mbufs according to stream */
> qsort(lc->lro_mbuf_data, lc->lro_mbuf_count, sizeof(struct mbuf *),
> &tcp_lro_mbuf_compare_header);
>
> /* input data into LRO engine, stream by stream */
> flowid = 0;
> hashtype = M_HASHTYPE_NONE;
> for (x = 0; x != lc->lro_mbuf_count; x++) {
> struct mbuf *mb;
>
> mb = lc->lro_mbuf_data[x];
>
> /* check for new stream */
> if (mb->m_pkthdr.flowid != flowid ||
> M_HASHTYPE_GET(mb) != hashtype) {
> flowid = mb->m_pkthdr.flowid;
> hashtype = M_HASHTYPE_GET(mb);
>
> /* flush active streams */
> while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) {
> SLIST_REMOVE_HEAD(&lc->lro_active, next);
> tcp_lro_flush(lc, le);
> }
> }
> #ifdef TCP_LRO_RESET_SEQUENCE
> /* reset sequence number */
> TCP_LRO_SEQUENCE(mb) = 0;
> #endif
> /* add packet to LRO engine */
> if (tcp_lro_rx(lc, mb, 0) != 0) {
> /* input packet to network layer */
> (*lc->ifp->if_input)(lc->ifp, mb);
> lc->lro_queued++;
> lc->lro_flushed++;
> }
> }
> done:
> /* flush active streams */
> while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) {
> SLIST_REMOVE_HEAD(&lc->lro_active, next);
> tcp_lro_flush(lc, le);
> }
> lc->lro_mbuf_count = 0;
> }
>
635a743,775
> void
> tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)
> {
> /* sanity checks */
> if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL ||
> lc->lro_mbuf_max == 0)) {
> /* packet drop */
> m_freem(mb);
> return;
> }
>
> /* check if packet is not LRO capable */
> if (__predict_false(mb->m_pkthdr.csum_flags == 0 ||
> (lc->ifp->if_capenable & IFCAP_LRO) == 0)) {
> lc->lro_flushed++;
> lc->lro_queued++;
>
> /* input packet to network layer */
> (*lc->ifp->if_input) (lc->ifp, mb);
> return;
> }
>
> /* check if array is full */
> if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max))
> tcp_lro_flush_all(lc);
>
> /* store sequence number */
> TCP_LRO_SEQUENCE(mb) = lc->lro_mbuf_count;
>
> /* enter mbuf */
> lc->lro_mbuf_data[lc->lro_mbuf_count++] = mb;
> }
>