dev/sfxge/sfxge_tx.c

972 static const struct ip *tso_iph(const struct sfxge_tso_state *tso)
974 	KASSERT(tso->protocol == htons(ETHERTYPE_IP),
976 	return (const struct ip *)(tso->mbuf->m_data + tso->nh_off);
979 static __unused const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso)
981 	KASSERT(tso->protocol == htons(ETHERTYPE_IPV6),
983 	return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off);
986 static const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso)
988 	return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off);
1044 static void tso_start(struct sfxge_txq *txq, struct sfxge_tso_state *tso,
1055 	tso->fw_assisted = txq->tso_fw_assisted;
1056 	tso->mbuf = mbuf;
1060 	tso->protocol = eh->ether_type;
1061 	if (tso->protocol == htons(ETHERTYPE_VLAN)) {
1064 		tso->protocol = veh->evl_proto;
1065 		tso->nh_off = sizeof(*veh);
1067 		tso->nh_off = sizeof(*eh);
1070 	tso->protocol = TSO_MBUF_PROTO(mbuf);
1071 	tso->nh_off = mbuf->m_pkthdr.l2hlen;
1072 	tso->tcph_off = mbuf->m_pkthdr.l3hlen;
1073 	tso->packet_id = ntohs(TSO_MBUF_PACKETID(mbuf));
1078 	if (tso->protocol == htons(ETHERTYPE_IP)) {
1079 		KASSERT(tso_iph(tso)->ip_p == IPPROTO_TCP,
1081 		tso->tcph_off = tso->nh_off + 4 * tso_iph(tso)->ip_hl;
1082 		tso->packet_id = ntohs(tso_iph(tso)->ip_id);
1084 		KASSERT(tso->protocol == htons(ETHERTYPE_IPV6),
1086 		KASSERT(tso_ip6h(tso)->ip6_nxt == IPPROTO_TCP,
1088 		tso->tcph_off = tso->nh_off + sizeof(struct ip6_hdr);
1089 		tso->packet_id = 0;
1094 	if (tso->fw_assisted &&
1095 	    __predict_false(tso->tcph_off >
1097 		tso->fw_assisted = 0;
1102 	KASSERT(mbuf->m_len >= tso->tcph_off,
1105 	if (mbuf->m_len < tso->tcph_off + offsetof(struct tcphdr, th_win)) {
1106 		m_copydata(tso->mbuf, tso->tcph_off, sizeof(th_copy),
1110 		th = tso_tcph(tso);
1112 	tso->header_len = tso->tcph_off + 4 * th->th_off;
1114 	tso->header_len = mbuf->m_pkthdr.l4hlen;
1116 	tso->seg_size = mbuf->m_pkthdr.tso_segsz;
1119 	tso->seqnum = ntohl(th->th_seq);
1130 	tso->tcp_flags = th->th_flags;
1132 	tso->seqnum = TSO_MBUF_SEQNUM(mbuf);
1133 	tso->tcp_flags = TSO_MBUF_FLAGS(mbuf);
1136 	tso->out_len = mbuf->m_pkthdr.len - tso->header_len;
1138 	if (tso->fw_assisted) {
1139 		if (hdr_dma_seg->ds_len >= tso->header_len)
1142 						tso->header_len,
1144 						&tso->header_desc);
1146 			tso->fw_assisted = 0;
1158 					  struct sfxge_tso_state *tso)
1162 	uint64_t dma_addr = tso->dma_addr;
1165 	if (tso->in_len == 0 || tso->packet_space == 0)
1168 	KASSERT(tso->in_len > 0, ("TSO input length went negative"));
1169 	KASSERT(tso->packet_space > 0, ("TSO packet space went negative"));
1171 	if (tso->fw_assisted & SFXGE_FATSOV2) {
1172 		n = tso->in_len;
1173 		tso->out_len -= n;
1174 		tso->seqnum += n;
1175 		tso->in_len = 0;
1176 		if (n < tso->packet_space) {
1177 			tso->packet_space -= n;
1178 			tso->segs_space--;
1180 			tso->packet_space = tso->seg_size -
1181 			    (n - tso->packet_space) % tso->seg_size;
1182 			tso->segs_space =
1184 			    (tso->packet_space != tso->seg_size);
1187 		n = min(tso->in_len, tso->packet_space);
1188 		tso->packet_space -= n;
1189 		tso->out_len -= n;
1190 		tso->dma_addr += n;
1191 		tso->in_len -= n;
1198 	eop = (tso->out_len == 0) | (tso->packet_space == 0) |
1199 	    (tso->segs_space == 0);
1222 				struct sfxge_tso_state *tso,
1234 	if (tso->fw_assisted) {
1235 		if (tso->fw_assisted & SFXGE_FATSOV2) {
1239 						 tso->packet_id,
1240 						 tso->seqnum,
1241 						 tso->seg_size,
1249 			tso->segs_space =
1252 			uint8_t tcp_flags = tso->tcp_flags;
1254 			if (tso->out_len > tso->seg_size)
1260 						tso->packet_id,
1261 						tso->seqnum,
1267 			tso->seqnum += tso->seg_size;
1268 			tso->segs_space = UINT_MAX;
1272 		*desc = tso->header_desc;
1278 		if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) {
1294 			header = malloc(tso->header_len, M_SFXGE, M_NOWAIT);
1298 					     header, tso->header_len,
1318 		tsoh_th = (struct tcphdr *)(header + tso->tcph_off);
1321 		m_copydata(tso->mbuf, 0, tso->header_len, header);
1323 		tsoh_th->th_seq = htonl(tso->seqnum);
1324 		tso->seqnum += tso->seg_size;
1325 		if (tso->out_len > tso->seg_size) {
1327 			ip_length = tso->header_len - tso->nh_off + tso->seg_size;
1331 			ip_length = tso->header_len - tso->nh_off + tso->out_len;
1334 		if (tso->protocol == htons(ETHERTYPE_IP)) {
1335 			struct ip *tsoh_iph = (struct ip *)(header + tso->nh_off);
1342 				(struct ip6_hdr *)(header + tso->nh_off);
1353 					tso->header_len,
1358 		tso->segs_space = UINT_MAX;
1360 	tso->packet_space = tso->seg_size;
1372 	struct sfxge_tso_state tso;
1376 	tso_start(txq, &tso, dma_seg, mbuf);
1378 	while (dma_seg->ds_len + skipped <= tso.header_len) {
1384 	tso.in_len = dma_seg->ds_len - (tso.header_len - skipped);
1385 	tso.dma_addr = dma_seg->ds_addr + (tso.header_len - skipped);
1388 	if (__predict_false(tso_start_new_packet(txq, &tso, &id)))
1392 		tso_fill_packet_with_fragment(txq, &tso);
1398 		if (tso.in_len == 0) {
1403 			tso.in_len = dma_seg->ds_len;
1404 			tso.dma_addr = dma_seg->ds_addr;
1408 		if ((tso.packet_space == 0) | (tso.segs_space == 0)) {
1410 			    (tso.fw_assisted & SFXGE_FATSOV2) ?
1412 			    (tso.fw_assisted & SFXGE_FATSOV1) ? 1 : 0;
1424 			if (__predict_false(tso_start_new_packet(txq, &tso,