sys/netinet/tcp_lro.c

84 tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_entry *le)
87 	LIST_INSERT_HEAD(&lc->lro_active, le, next);
91 tcp_lro_active_remove(struct lro_entry *le)
94 	LIST_REMOVE(le, next);
107 	struct lro_entry *le;
125 	    (lro_entries * sizeof(*le));
135 	le = (struct lro_entry *)
140 		LIST_INSERT_HEAD(&lc->lro_free, le + i, next);
148 	struct lro_entry *le;
155 	while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
156 		tcp_lro_active_remove(le);
157 		m_freem(le->m_head);
194 tcp_lro_rx_csum_fixup(struct lro_entry *le, void *l3hdr, struct tcphdr *th,
203 	switch (le->eh_type) {
210 		if (le->append_cnt == 0)
227 		if (le->append_cnt == 0)
258 	struct lro_entry *le;
260 	while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
261 		tcp_lro_active_remove(le);
262 		tcp_lro_flush(lc, le);
269 	struct lro_entry *le, *le_tmp;
277 	LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) {
278 		if (timevalcmp(&tv, &le->mtime, >=)) {
279 			tcp_lro_active_remove(le);
280 			tcp_lro_flush(lc, le);
286 tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le)
289 	if (le->append_cnt > 0) {
293 		p_len = htons(le->p_len);
294 		switch (le->eh_type) {
300 			ip6 = le->le_ip6;
303 			le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |
305 			le->p_len += ETHER_HDR_LEN + sizeof(*ip6);
318 			ip4 = le->le_ip4;
334 			le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |
336 			le->p_len += ETHER_HDR_LEN;
343 		le->m_head->m_pkthdr.csum_data = 0xffff;
344 		le->m_head->m_pkthdr.len = le->p_len;
347 		th->th_ack = le->ack_seq;
348 		th->th_win = le->window;
350 		if (le->timestamp != 0) {
354 			ts_ptr[1] = htonl(le->tsval);
355 			ts_ptr[2] = le->tsecr;
359 		le->ulp_csum += p_len;
360 		le->ulp_csum += tcp_lro_csum_th(th);
361 		while (le->ulp_csum > 0xffff)
362 			le->ulp_csum = (le->ulp_csum >> 16) +
363 			    (le->ulp_csum & 0xffff);
364 		th->th_sum = (le->ulp_csum & 0xffff);
371 	(*lc->ifp->if_input)(lc->ifp, le->m_head);
372 	lc->lro_queued += le->append_cnt + 1;
374 	bzero(le, sizeof(*le));
375 	LIST_INSERT_HEAD(&lc->lro_free, le, next);
577 	struct lro_entry *le;
697 	LIST_FOREACH(le, &lc->lro_active, next) {
698 		if (le->eh_type != eh_type)
700 		if (le->source_port != th->th_sport ||
701 		    le->dest_port != th->th_dport)
706 			if (bcmp(&le->source_ip6, &ip6->ip6_src,
708 			    bcmp(&le->dest_ip6, &ip6->ip6_dst,
715 			if (le->source_ip4 != ip4->ip_src.s_addr ||
716 			    le->dest_ip4 != ip4->ip_dst.s_addr)
724 			tcp_lro_active_remove(le);
725 			tcp_lro_flush(lc, le);
730 		if (le->p_len > (lc->lro_length_lim - tcp_data_len)) {
731 			tcp_lro_active_remove(le);
732 			tcp_lro_flush(lc, le);
737 		if (__predict_false(seq != le->next_seq ||
739 		    le->ack_seq == th->th_ack &&
740 		    le->window == th->th_win))) {
742 			tcp_lro_active_remove(le);
743 			tcp_lro_flush(lc, le);
751 			if (__predict_false(le->tsval > tsval ||
754 			le->tsval = tsval;
755 			le->tsecr = *(ts_ptr + 2);
757 		if (tcp_data_len || SEQ_GT(ntohl(th->th_ack), ntohl(le->ack_seq))) {
758 			le->next_seq += tcp_data_len;
759 			le->ack_seq = th->th_ack;
760 			le->window = th->th_win;
761 			le->append_cnt++;
762 		} else if (th->th_ack == le->ack_seq) {
763 			le->window = WIN_MAX(le->window, th->th_win);
764 			le->append_cnt++;
767 			le->append_cnt++;
772 		le->ulp_csum += tcp_lro_rx_csum_fixup(le, l3hdr, th,
782 			if (le->append_cnt >= lc->lro_ackcnt_lim) {
783 				tcp_lro_active_remove(le);
784 				tcp_lro_flush(lc, le);
789 		le->p_len += tcp_data_len;
799 		le->m_tail->m_next = m;
800 		le->m_tail = m_last(m);
806 		if (le->p_len > (lc->lro_length_lim - lc->ifp->if_mtu)) {
807 			tcp_lro_active_remove(le);
808 			tcp_lro_flush(lc, le);
810 			getmicrotime(&le->mtime);
828 	le = LIST_FIRST(&lc->lro_free);
829 	LIST_REMOVE(le, next);
830 	tcp_lro_active_insert(lc, le);
831 	getmicrotime(&le->mtime);
837 		le->le_ip6 = ip6;
838 		le->source_ip6 = ip6->ip6_src;
839 		le->dest_ip6 = ip6->ip6_dst;
840 		le->eh_type = eh_type;
841 		le->p_len = m->m_pkthdr.len - ETHER_HDR_LEN - sizeof(*ip6);
846 		le->le_ip4 = ip4;
847 		le->source_ip4 = ip4->ip_src.s_addr;
848 		le->dest_ip4 = ip4->ip_dst.s_addr;
849 		le->eh_type = eh_type;
850 		le->p_len = m->m_pkthdr.len - ETHER_HDR_LEN;
854 	le->source_port = th->th_sport;
855 	le->dest_port = th->th_dport;
857 	le->next_seq = seq + tcp_data_len;
858 	le->ack_seq = th->th_ack;
859 	le->window = th->th_win;
861 		le->timestamp = 1;
862 		le->tsval = ntohl(*(ts_ptr + 1));
863 		le->tsecr = *(ts_ptr + 2);
874 	KASSERT(le->ulp_csum == 0, ("%s: le=%p le->ulp_csum=0x%04x\n",
875 	    __func__, le, le->ulp_csum));
877 	le->ulp_csum = tcp_lro_rx_csum_fixup(le, l3hdr, th, tcp_data_len,
882 	le->m_head = m;
883 	le->m_tail = m_last(m);