Deleted Added
sdiff udiff text old ( 291924 ) new ( 294077 )
full compact
sfxge_tx.c (291924) sfxge_tx.c (294077)
1/*-
2 * Copyright (c) 2010-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 33 unchanged lines hidden (view full) ---

42 *
43 * So, event queue plus label mapping to Tx queue index is:
44 * if event queue index is 0, TxQ-index = TxQ-label * [0..SFXGE_TXQ_NTYPES)
45 * else TxQ-index = SFXGE_TXQ_NTYPES + EvQ-index - 1
46 * See sfxge_get_txq_by_label() sfxge_ev.c
47 */
48
49#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2010-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 33 unchanged lines hidden (view full) ---

42 *
43 * So, event queue plus label mapping to Tx queue index is:
44 * if event queue index is 0, TxQ-index = TxQ-label * [0..SFXGE_TXQ_NTYPES)
45 * else TxQ-index = SFXGE_TXQ_NTYPES + EvQ-index - 1
46 * See sfxge_get_txq_by_label() sfxge_ev.c
47 */
48
49#include <sys/cdefs.h>
50__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_tx.c 291924 2015-12-07 06:04:24Z arybchik $");
50__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_tx.c 294077 2016-01-15 06:25:26Z arybchik $");
51
52#include <sys/types.h>
53#include <sys/mbuf.h>
54#include <sys/smp.h>
55#include <sys/socket.h>
56#include <sys/sysctl.h>
57#include <sys/syslog.h>
51
52#include <sys/types.h>
53#include <sys/mbuf.h>
54#include <sys/smp.h>
55#include <sys/socket.h>
56#include <sys/sysctl.h>
57#include <sys/syslog.h>
58#include <sys/limits.h>
58
59#include <net/bpf.h>
60#include <net/ethernet.h>
61#include <net/if.h>
62#include <net/if_vlan_var.h>
63
64#include <netinet/in.h>
65#include <netinet/ip.h>

--- 25 unchanged lines hidden (view full) ---

91#define SFXGE_PARAM_TX_DPL_PUT_MAX SFXGE_PARAM(tx_dpl_put_max)
92static int sfxge_tx_dpl_put_max = SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT;
93TUNABLE_INT(SFXGE_PARAM_TX_DPL_PUT_MAX, &sfxge_tx_dpl_put_max);
94SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN,
95 &sfxge_tx_dpl_put_max, 0,
96 "Maximum number of any packets in deferred packet put-list");
97
98#define SFXGE_PARAM_TSO_FW_ASSISTED SFXGE_PARAM(tso_fw_assisted)
59
60#include <net/bpf.h>
61#include <net/ethernet.h>
62#include <net/if.h>
63#include <net/if_vlan_var.h>
64
65#include <netinet/in.h>
66#include <netinet/ip.h>

--- 25 unchanged lines hidden (view full) ---

92#define SFXGE_PARAM_TX_DPL_PUT_MAX SFXGE_PARAM(tx_dpl_put_max)
93static int sfxge_tx_dpl_put_max = SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT;
94TUNABLE_INT(SFXGE_PARAM_TX_DPL_PUT_MAX, &sfxge_tx_dpl_put_max);
95SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN,
96 &sfxge_tx_dpl_put_max, 0,
97 "Maximum number of any packets in deferred packet put-list");
98
99#define SFXGE_PARAM_TSO_FW_ASSISTED SFXGE_PARAM(tso_fw_assisted)
99static int sfxge_tso_fw_assisted = 1;
100static int sfxge_tso_fw_assisted = (SFXGE_FATSOV1 | SFXGE_FATSOV2);
100TUNABLE_INT(SFXGE_PARAM_TSO_FW_ASSISTED, &sfxge_tso_fw_assisted);
101SYSCTL_INT(_hw_sfxge, OID_AUTO, tso_fw_assisted, CTLFLAG_RDTUN,
102 &sfxge_tso_fw_assisted, 0,
101TUNABLE_INT(SFXGE_PARAM_TSO_FW_ASSISTED, &sfxge_tso_fw_assisted);
102SYSCTL_INT(_hw_sfxge, OID_AUTO, tso_fw_assisted, CTLFLAG_RDTUN,
103 &sfxge_tso_fw_assisted, 0,
103 "Use FW-assisted TSO if supported by NIC firmware");
104 "Bitmask of FW-assisted TSO allowed to use if supported by NIC firmware");
104
105
106static const struct {
107 const char *name;
108 size_t offset;
109} sfxge_tx_stats[] = {
110#define SFXGE_TX_STAT(name, member) \
111 { #name, offsetof(struct sfxge_txq, member) }

--- 733 unchanged lines hidden (view full) ---

845 * still faster than segmenting in the stack.
846 */
847
848struct sfxge_tso_state {
849 /* Output position */
850 unsigned out_len; /* Remaining length in current segment */
851 unsigned seqnum; /* Current sequence number */
852 unsigned packet_space; /* Remaining space in current packet */
105
106
107static const struct {
108 const char *name;
109 size_t offset;
110} sfxge_tx_stats[] = {
111#define SFXGE_TX_STAT(name, member) \
112 { #name, offsetof(struct sfxge_txq, member) }

--- 733 unchanged lines hidden (view full) ---

846 * still faster than segmenting in the stack.
847 */
848
849struct sfxge_tso_state {
850 /* Output position */
851 unsigned out_len; /* Remaining length in current segment */
852 unsigned seqnum; /* Current sequence number */
853 unsigned packet_space; /* Remaining space in current packet */
854 unsigned segs_space; /* Remaining number of DMA segments
855 for the packet (FATSOv2 only) */
853
854 /* Input position */
855 uint64_t dma_addr; /* DMA address of current position */
856 unsigned in_len; /* Remaining length in current mbuf */
857
858 const struct mbuf *mbuf; /* Input mbuf (head of chain) */
859 u_short protocol; /* Network protocol (after VLAN decap) */
860 ssize_t nh_off; /* Offset of network header */

--- 86 unchanged lines hidden (view full) ---

947{
948 const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->sc->enp);
949#if !SFXGE_TX_PARSE_EARLY
950 struct ether_header *eh = mtod(mbuf, struct ether_header *);
951 const struct tcphdr *th;
952 struct tcphdr th_copy;
953#endif
954
856
857 /* Input position */
858 uint64_t dma_addr; /* DMA address of current position */
859 unsigned in_len; /* Remaining length in current mbuf */
860
861 const struct mbuf *mbuf; /* Input mbuf (head of chain) */
862 u_short protocol; /* Network protocol (after VLAN decap) */
863 ssize_t nh_off; /* Offset of network header */

--- 86 unchanged lines hidden (view full) ---

950{
951 const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->sc->enp);
952#if !SFXGE_TX_PARSE_EARLY
953 struct ether_header *eh = mtod(mbuf, struct ether_header *);
954 const struct tcphdr *th;
955 struct tcphdr th_copy;
956#endif
957
955 tso->fw_assisted = txq->sc->tso_fw_assisted;
958 tso->fw_assisted = txq->tso_fw_assisted;
956 tso->mbuf = mbuf;
957
958 /* Find network protocol and header */
959#if !SFXGE_TX_PARSE_EARLY
960 tso->protocol = eh->ether_type;
961 if (tso->protocol == htons(ETHERTYPE_VLAN)) {
962 struct ether_vlan_header *veh =
963 mtod(mbuf, struct ether_vlan_header *);

--- 90 unchanged lines hidden (view full) ---

1054 * of fragment or end-of-packet. Return 0 on success, 1 if not enough
1055 * space.
1056 */
1057static void tso_fill_packet_with_fragment(struct sfxge_txq *txq,
1058 struct sfxge_tso_state *tso)
1059{
1060 efx_desc_t *desc;
1061 int n;
959 tso->mbuf = mbuf;
960
961 /* Find network protocol and header */
962#if !SFXGE_TX_PARSE_EARLY
963 tso->protocol = eh->ether_type;
964 if (tso->protocol == htons(ETHERTYPE_VLAN)) {
965 struct ether_vlan_header *veh =
966 mtod(mbuf, struct ether_vlan_header *);

--- 90 unchanged lines hidden (view full) ---

1057 * of fragment or end-of-packet. Return 0 on success, 1 if not enough
1058 * space.
1059 */
1060static void tso_fill_packet_with_fragment(struct sfxge_txq *txq,
1061 struct sfxge_tso_state *tso)
1062{
1063 efx_desc_t *desc;
1064 int n;
1065 uint64_t dma_addr = tso->dma_addr;
1066 boolean_t eop;
1062
1063 if (tso->in_len == 0 || tso->packet_space == 0)
1064 return;
1065
1066 KASSERT(tso->in_len > 0, ("TSO input length went negative"));
1067 KASSERT(tso->packet_space > 0, ("TSO packet space went negative"));
1068
1067
1068 if (tso->in_len == 0 || tso->packet_space == 0)
1069 return;
1070
1071 KASSERT(tso->in_len > 0, ("TSO input length went negative"));
1072 KASSERT(tso->packet_space > 0, ("TSO packet space went negative"));
1073
1069 n = min(tso->in_len, tso->packet_space);
1074 if (tso->fw_assisted & SFXGE_FATSOV2) {
1075 n = tso->in_len;
1076 tso->out_len -= n;
1077 tso->seqnum += n;
1078 tso->in_len = 0;
1079 if (n < tso->packet_space) {
1080 tso->packet_space -= n;
1081 tso->segs_space--;
1082 } else {
1083 tso->packet_space = tso->seg_size -
1084 (n - tso->packet_space) % tso->seg_size;
1085 tso->segs_space =
1086 EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1 -
1087 (tso->packet_space != tso->seg_size);
1088 }
1089 } else {
1090 n = min(tso->in_len, tso->packet_space);
1091 tso->packet_space -= n;
1092 tso->out_len -= n;
1093 tso->dma_addr += n;
1094 tso->in_len -= n;
1095 }
1070
1096
1071 tso->packet_space -= n;
1072 tso->out_len -= n;
1073 tso->in_len -= n;
1097 /*
1098 * It is OK to use binary OR below to avoid extra branching
1099 * since all conditions may always be checked.
1100 */
1101 eop = (tso->out_len == 0) | (tso->packet_space == 0) |
1102 (tso->segs_space == 0);
1074
1075 desc = &txq->pend_desc[txq->n_pend_desc++];
1103
1104 desc = &txq->pend_desc[txq->n_pend_desc++];
1076 efx_tx_qdesc_dma_create(txq->common,
1077 tso->dma_addr,
1078 n,
1079 tso->out_len == 0 || tso->packet_space == 0,
1080 desc);
1081
1082 tso->dma_addr += n;
1105 efx_tx_qdesc_dma_create(txq->common, dma_addr, n, eop, desc);
1083}
1084
1085/* Callback from bus_dmamap_load() for long TSO headers. */
1086static void tso_map_long_header(void *dma_addr_ret,
1087 bus_dma_segment_t *segs, int nseg,
1088 int error)
1089{
1090 *(uint64_t *)dma_addr_ret = ((__predict_true(error == 0) &&

--- 16 unchanged lines hidden (view full) ---

1107 unsigned ip_length;
1108 caddr_t header;
1109 uint64_t dma_addr;
1110 bus_dmamap_t map;
1111 efx_desc_t *desc;
1112 int rc;
1113
1114 if (tso->fw_assisted) {
1106}
1107
1108/* Callback from bus_dmamap_load() for long TSO headers. */
1109static void tso_map_long_header(void *dma_addr_ret,
1110 bus_dma_segment_t *segs, int nseg,
1111 int error)
1112{
1113 *(uint64_t *)dma_addr_ret = ((__predict_true(error == 0) &&

--- 16 unchanged lines hidden (view full) ---

1130 unsigned ip_length;
1131 caddr_t header;
1132 uint64_t dma_addr;
1133 bus_dmamap_t map;
1134 efx_desc_t *desc;
1135 int rc;
1136
1137 if (tso->fw_assisted) {
1115 uint8_t tcp_flags = tso->tcp_flags;
1138 if (tso->fw_assisted & SFXGE_FATSOV2) {
1139 /* Add 2 FATSOv2 option descriptors */
1140 desc = &txq->pend_desc[txq->n_pend_desc];
1141 efx_tx_qdesc_tso2_create(txq->common,
1142 tso->packet_id,
1143 tso->seqnum,
1144 tso->seg_size,
1145 desc,
1146 EFX_TX_FATSOV2_OPT_NDESCS);
1147 desc += EFX_TX_FATSOV2_OPT_NDESCS;
1148 txq->n_pend_desc += EFX_TX_FATSOV2_OPT_NDESCS;
1149 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1150 id = (id + EFX_TX_FATSOV2_OPT_NDESCS) & txq->ptr_mask;
1116
1151
1117 if (tso->out_len > tso->seg_size)
1118 tcp_flags &= ~(TH_FIN | TH_PUSH);
1152 tso->segs_space =
1153 EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1;
1154 } else {
1155 uint8_t tcp_flags = tso->tcp_flags;
1119
1156
1120 /* TSO option descriptor */
1121 desc = &txq->pend_desc[txq->n_pend_desc++];
1122 efx_tx_qdesc_tso_create(txq->common,
1123 tso->packet_id,
1124 tso->seqnum,
1125 tcp_flags,
1126 desc++);
1127 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1128 id = (id + 1) & txq->ptr_mask;
1157 if (tso->out_len > tso->seg_size)
1158 tcp_flags &= ~(TH_FIN | TH_PUSH);
1129
1159
1160 /* Add FATSOv1 option descriptor */
1161 desc = &txq->pend_desc[txq->n_pend_desc++];
1162 efx_tx_qdesc_tso_create(txq->common,
1163 tso->packet_id,
1164 tso->seqnum,
1165 tcp_flags,
1166 desc++);
1167 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1168 id = (id + 1) & txq->ptr_mask;
1169
1170 tso->seqnum += tso->seg_size;
1171 tso->segs_space = UINT_MAX;
1172 }
1173
1130 /* Header DMA descriptor */
1131 *desc = tso->header_desc;
1132 txq->n_pend_desc++;
1133 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1134 id = (id + 1) & txq->ptr_mask;
1174 /* Header DMA descriptor */
1175 *desc = tso->header_desc;
1176 txq->n_pend_desc++;
1177 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1178 id = (id + 1) & txq->ptr_mask;
1135
1136 tso->seqnum += tso->seg_size;
1137 } else {
1138 /* Allocate a DMA-mapped header buffer. */
1139 if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) {
1140 unsigned int page_index = (id / 2) / TSOH_PER_PAGE;
1141 unsigned int buf_index = (id / 2) % TSOH_PER_PAGE;
1142
1143 header = (txq->tsoh_buffer[page_index].esm_base +
1144 buf_index * TSOH_STD_SIZE);

--- 65 unchanged lines hidden (view full) ---

1210 /* Form a descriptor for this header. */
1211 desc = &txq->pend_desc[txq->n_pend_desc++];
1212 efx_tx_qdesc_dma_create(txq->common,
1213 dma_addr,
1214 tso->header_len,
1215 0,
1216 desc);
1217 id = (id + 1) & txq->ptr_mask;
1179 } else {
1180 /* Allocate a DMA-mapped header buffer. */
1181 if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) {
1182 unsigned int page_index = (id / 2) / TSOH_PER_PAGE;
1183 unsigned int buf_index = (id / 2) % TSOH_PER_PAGE;
1184
1185 header = (txq->tsoh_buffer[page_index].esm_base +
1186 buf_index * TSOH_STD_SIZE);

--- 65 unchanged lines hidden (view full) ---

1252 /* Form a descriptor for this header. */
1253 desc = &txq->pend_desc[txq->n_pend_desc++];
1254 efx_tx_qdesc_dma_create(txq->common,
1255 dma_addr,
1256 tso->header_len,
1257 0,
1258 desc);
1259 id = (id + 1) & txq->ptr_mask;
1260
1261 tso->segs_space = UINT_MAX;
1218 }
1219 tso->packet_space = tso->seg_size;
1220 txq->tso_packets++;
1221 *idp = id;
1222
1223 return (0);
1224}
1225

--- 33 unchanged lines hidden (view full) ---

1259 if (n_dma_seg == 0)
1260 break;
1261 ++dma_seg;
1262 tso.in_len = dma_seg->ds_len;
1263 tso.dma_addr = dma_seg->ds_addr;
1264 }
1265
1266 /* End of packet? */
1262 }
1263 tso->packet_space = tso->seg_size;
1264 txq->tso_packets++;
1265 *idp = id;
1266
1267 return (0);
1268}
1269

--- 33 unchanged lines hidden (view full) ---

1303 if (n_dma_seg == 0)
1304 break;
1305 ++dma_seg;
1306 tso.in_len = dma_seg->ds_len;
1307 tso.dma_addr = dma_seg->ds_addr;
1308 }
1309
1310 /* End of packet? */
1267 if (tso.packet_space == 0) {
1311 if ((tso.packet_space == 0) | (tso.segs_space == 0)) {
1312 unsigned int n_fatso_opt_desc =
1313 (tso.fw_assisted & SFXGE_FATSOV2) ?
1314 EFX_TX_FATSOV2_OPT_NDESCS :
1315 (tso.fw_assisted & SFXGE_FATSOV1) ? 1 : 0;
1316
1268 /* If the queue is now full due to tiny MSS,
1269 * or we can't create another header, discard
1270 * the remainder of the input mbuf but do not
1271 * roll back the work we have done.
1272 */
1317 /* If the queue is now full due to tiny MSS,
1318 * or we can't create another header, discard
1319 * the remainder of the input mbuf but do not
1320 * roll back the work we have done.
1321 */
1273 if (txq->n_pend_desc + tso.fw_assisted +
1274 1 /* header */ + n_dma_seg >
1275 txq->max_pkt_desc) {
1322 if (txq->n_pend_desc + n_fatso_opt_desc +
1323 1 /* header */ + n_dma_seg > txq->max_pkt_desc) {
1276 txq->tso_pdrop_too_many++;
1277 break;
1278 }
1279 if (__predict_false(tso_start_new_packet(txq, &tso,
1280 &id))) {
1281 txq->tso_pdrop_no_rsrc++;
1282 break;
1283 }

--- 118 unchanged lines hidden (view full) ---

1402
1403 efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id,
1404 EFX_TXQ_NBUFS(sc->txq_entries));
1405
1406 SFXGE_EVQ_UNLOCK(evq);
1407 SFXGE_TXQ_UNLOCK(txq);
1408}
1409
1324 txq->tso_pdrop_too_many++;
1325 break;
1326 }
1327 if (__predict_false(tso_start_new_packet(txq, &tso,
1328 &id))) {
1329 txq->tso_pdrop_no_rsrc++;
1330 break;
1331 }

--- 118 unchanged lines hidden (view full) ---

1450
1451 efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id,
1452 EFX_TXQ_NBUFS(sc->txq_entries));
1453
1454 SFXGE_EVQ_UNLOCK(evq);
1455 SFXGE_TXQ_UNLOCK(txq);
1456}
1457
1458/*
1459 * Estimate maximum number of Tx descriptors required for TSO packet.
1460 * With minimum MSS and maximum mbuf length we might need more (even
1461 * than a ring-ful of descriptors), but this should not happen in
1462 * practice except due to deliberate attack. In that case we will
1463 * truncate the output at a packet boundary.
1464 */
1465static unsigned int
1466sfxge_tx_max_pkt_desc(const struct sfxge_softc *sc, enum sfxge_txq_type type,
1467 unsigned int tso_fw_assisted)
1468{
1469 /* One descriptor for every input fragment */
1470 unsigned int max_descs = SFXGE_TX_MAPPING_MAX_SEG;
1471 unsigned int sw_tso_max_descs;
1472 unsigned int fa_tso_v1_max_descs = 0;
1473 unsigned int fa_tso_v2_max_descs = 0;
1474
1475 /* VLAN tagging Tx option descriptor may be required */
1476 if (efx_nic_cfg_get(sc->enp)->enc_hw_tx_insert_vlan_enabled)
1477 max_descs++;
1478
1479 if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) {
1480 /*
1481 * Plus header and payload descriptor for each output segment.
1482 * Minus one since header fragment is already counted.
1483 * Even if FATSO is used, we should be ready to fallback
1484 * to do it in the driver.
1485 */
1486 sw_tso_max_descs = SFXGE_TSO_MAX_SEGS * 2 - 1;
1487
1488 /* FW assisted TSOv1 requires one more descriptor per segment
1489 * in comparison to SW TSO */
1490 if (tso_fw_assisted & SFXGE_FATSOV1)
1491 fa_tso_v1_max_descs =
1492 sw_tso_max_descs + SFXGE_TSO_MAX_SEGS;
1493
1494 /* FW assisted TSOv2 requires 3 (2 FATSO plus header) extra
1495 * descriptors per superframe limited by number of DMA fetches
1496 * per packet. The first packet header is already counted.
1497 */
1498 if (tso_fw_assisted & SFXGE_FATSOV2) {
1499 fa_tso_v2_max_descs =
1500 howmany(SFXGE_TX_MAPPING_MAX_SEG,
1501 EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1) *
1502 (EFX_TX_FATSOV2_OPT_NDESCS + 1) - 1;
1503 }
1504
1505 max_descs += MAX(sw_tso_max_descs,
1506 MAX(fa_tso_v1_max_descs, fa_tso_v2_max_descs));
1507 }
1508
1509 return (max_descs);
1510}
1511
1410static int
1411sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
1412{
1413 struct sfxge_txq *txq;
1414 efsys_mem_t *esmp;
1415 uint16_t flags;
1512static int
1513sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
1514{
1515 struct sfxge_txq *txq;
1516 efsys_mem_t *esmp;
1517 uint16_t flags;
1518 unsigned int tso_fw_assisted;
1416 struct sfxge_evq *evq;
1417 unsigned int desc_index;
1418 int rc;
1419
1420 SFXGE_ADAPTER_LOCK_ASSERT_OWNED(sc);
1421
1422 txq = sc->txq[index];
1423 esmp = &txq->mem;

--- 5 unchanged lines hidden (view full) ---

1429 ("evq->init_state != SFXGE_EVQ_STARTED"));
1430
1431 /* Program the buffer table. */
1432 if ((rc = efx_sram_buf_tbl_set(sc->enp, txq->buf_base_id, esmp,
1433 EFX_TXQ_NBUFS(sc->txq_entries))) != 0)
1434 return (rc);
1435
1436 /* Determine the kind of queue we are creating. */
1519 struct sfxge_evq *evq;
1520 unsigned int desc_index;
1521 int rc;
1522
1523 SFXGE_ADAPTER_LOCK_ASSERT_OWNED(sc);
1524
1525 txq = sc->txq[index];
1526 esmp = &txq->mem;

--- 5 unchanged lines hidden (view full) ---

1532 ("evq->init_state != SFXGE_EVQ_STARTED"));
1533
1534 /* Program the buffer table. */
1535 if ((rc = efx_sram_buf_tbl_set(sc->enp, txq->buf_base_id, esmp,
1536 EFX_TXQ_NBUFS(sc->txq_entries))) != 0)
1537 return (rc);
1538
1539 /* Determine the kind of queue we are creating. */
1540 tso_fw_assisted = 0;
1437 switch (txq->type) {
1438 case SFXGE_TXQ_NON_CKSUM:
1439 flags = 0;
1440 break;
1441 case SFXGE_TXQ_IP_CKSUM:
1442 flags = EFX_TXQ_CKSUM_IPV4;
1443 break;
1444 case SFXGE_TXQ_IP_TCP_UDP_CKSUM:
1445 flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP;
1541 switch (txq->type) {
1542 case SFXGE_TXQ_NON_CKSUM:
1543 flags = 0;
1544 break;
1545 case SFXGE_TXQ_IP_CKSUM:
1546 flags = EFX_TXQ_CKSUM_IPV4;
1547 break;
1548 case SFXGE_TXQ_IP_TCP_UDP_CKSUM:
1549 flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP;
1550 tso_fw_assisted = sc->tso_fw_assisted;
1551 if (tso_fw_assisted & SFXGE_FATSOV2)
1552 flags |= EFX_TXQ_FATSOV2;
1446 break;
1447 default:
1448 KASSERT(0, ("Impossible TX queue"));
1449 flags = 0;
1450 break;
1451 }
1452
1453 /* Create the common code transmit queue. */
1454 if ((rc = efx_tx_qcreate(sc->enp, index, txq->type, esmp,
1455 sc->txq_entries, txq->buf_base_id, flags, evq->common,
1553 break;
1554 default:
1555 KASSERT(0, ("Impossible TX queue"));
1556 flags = 0;
1557 break;
1558 }
1559
1560 /* Create the common code transmit queue. */
1561 if ((rc = efx_tx_qcreate(sc->enp, index, txq->type, esmp,
1562 sc->txq_entries, txq->buf_base_id, flags, evq->common,
1456 &txq->common, &desc_index)) != 0)
1457 goto fail;
1563 &txq->common, &desc_index)) != 0) {
1564 /* Retry if no FATSOv2 resources, otherwise fail */
1565 if ((rc != ENOSPC) || (~flags & EFX_TXQ_FATSOV2))
1566 goto fail;
1458
1567
1568 /* Looks like all FATSOv2 contexts are used */
1569 flags &= ~EFX_TXQ_FATSOV2;
1570 tso_fw_assisted &= ~SFXGE_FATSOV2;
1571 if ((rc = efx_tx_qcreate(sc->enp, index, txq->type, esmp,
1572 sc->txq_entries, txq->buf_base_id, flags, evq->common,
1573 &txq->common, &desc_index)) != 0)
1574 goto fail;
1575 }
1576
1459 /* Initialise queue descriptor indexes */
1460 txq->added = txq->pending = txq->completed = txq->reaped = desc_index;
1461
1462 SFXGE_TXQ_LOCK(txq);
1463
1464 /* Enable the transmit queue. */
1465 efx_tx_qenable(txq->common);
1466
1467 txq->init_state = SFXGE_TXQ_STARTED;
1468 txq->flush_state = SFXGE_FLUSH_REQUIRED;
1577 /* Initialise queue descriptor indexes */
1578 txq->added = txq->pending = txq->completed = txq->reaped = desc_index;
1579
1580 SFXGE_TXQ_LOCK(txq);
1581
1582 /* Enable the transmit queue. */
1583 efx_tx_qenable(txq->common);
1584
1585 txq->init_state = SFXGE_TXQ_STARTED;
1586 txq->flush_state = SFXGE_FLUSH_REQUIRED;
1587 txq->tso_fw_assisted = tso_fw_assisted;
1469
1588
1589 txq->max_pkt_desc = sfxge_tx_max_pkt_desc(sc, txq->type,
1590 tso_fw_assisted);
1591
1470 SFXGE_TXQ_UNLOCK(txq);
1471
1472 return (0);
1473
1474fail:
1475 efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id,
1476 EFX_TXQ_NBUFS(sc->txq_entries));
1477 return (rc);

--- 91 unchanged lines hidden (view full) ---

1569
1570 sc->txq[index] = NULL;
1571
1572 SFXGE_TXQ_LOCK_DESTROY(txq);
1573
1574 free(txq, M_SFXGE);
1575}
1576
1592 SFXGE_TXQ_UNLOCK(txq);
1593
1594 return (0);
1595
1596fail:
1597 efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id,
1598 EFX_TXQ_NBUFS(sc->txq_entries));
1599 return (rc);

--- 91 unchanged lines hidden (view full) ---

1691
1692 sc->txq[index] = NULL;
1693
1694 SFXGE_TXQ_LOCK_DESTROY(txq);
1695
1696 free(txq, M_SFXGE);
1697}
1698
1577/*
1578 * Estimate maximum number of Tx descriptors required for TSO packet.
1579 * With minimum MSS and maximum mbuf length we might need more (even
1580 * than a ring-ful of descriptors), but this should not happen in
1581 * practice except due to deliberate attack. In that case we will
1582 * truncate the output at a packet boundary.
1583 */
1584static unsigned int
1585sfxge_tx_max_pkt_desc(const struct sfxge_softc *sc, enum sfxge_txq_type type)
1586{
1587 /* One descriptor for every input fragment */
1588 unsigned int max_descs = SFXGE_TX_MAPPING_MAX_SEG;
1589
1590 /* VLAN tagging Tx option descriptor may be required */
1591 if (efx_nic_cfg_get(sc->enp)->enc_hw_tx_insert_vlan_enabled)
1592 max_descs++;
1593
1594 if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) {
1595 /*
1596 * Plus header and payload descriptor for each output segment.
1597 * Minus one since header fragment is already counted.
1598 */
1599 max_descs += SFXGE_TSO_MAX_SEGS * 2 - 1;
1600
1601 /* FW assisted TSO requires one more descriptor per segment */
1602 if (sc->tso_fw_assisted)
1603 max_descs += SFXGE_TSO_MAX_SEGS;
1604 }
1605
1606 return (max_descs);
1607}
1608
1609static int
1610sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index,
1611 enum sfxge_txq_type type, unsigned int evq_index)
1612{
1613 char name[16];
1614 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
1615 struct sysctl_oid *txq_node;
1616 struct sfxge_txq *txq;

--- 113 unchanged lines hidden (view full) ---

1730 goto fail_txq_stat_init;
1731
1732 txq->type = type;
1733 txq->evq_index = evq_index;
1734 txq->txq_index = txq_index;
1735 txq->init_state = SFXGE_TXQ_INITIALIZED;
1736 txq->hw_vlan_tci = 0;
1737
1699static int
1700sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index,
1701 enum sfxge_txq_type type, unsigned int evq_index)
1702{
1703 char name[16];
1704 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
1705 struct sysctl_oid *txq_node;
1706 struct sfxge_txq *txq;

--- 113 unchanged lines hidden (view full) ---

1820 goto fail_txq_stat_init;
1821
1822 txq->type = type;
1823 txq->evq_index = evq_index;
1824 txq->txq_index = txq_index;
1825 txq->init_state = SFXGE_TXQ_INITIALIZED;
1826 txq->hw_vlan_tci = 0;
1827
1738 txq->max_pkt_desc = sfxge_tx_max_pkt_desc(sc, type);
1739
1740 return (0);
1741
1742fail_txq_stat_init:
1743fail_dpl_node:
1744fail_tx_dpl_put_max:
1745fail_tx_dpl_get_max:
1746fail3:
1747fail_txq_node:

--- 93 unchanged lines hidden (view full) ---

1841 intr = &sc->intr;
1842
1843 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
1844 ("intr->state != SFXGE_INTR_INITIALIZED"));
1845
1846 sc->txq_count = SFXGE_TXQ_NTYPES - 1 + sc->intr.n_alloc;
1847
1848 sc->tso_fw_assisted = sfxge_tso_fw_assisted;
1828 return (0);
1829
1830fail_txq_stat_init:
1831fail_dpl_node:
1832fail_tx_dpl_put_max:
1833fail_tx_dpl_get_max:
1834fail3:
1835fail_txq_node:

--- 93 unchanged lines hidden (view full) ---

1929 intr = &sc->intr;
1930
1931 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
1932 ("intr->state != SFXGE_INTR_INITIALIZED"));
1933
1934 sc->txq_count = SFXGE_TXQ_NTYPES - 1 + sc->intr.n_alloc;
1935
1936 sc->tso_fw_assisted = sfxge_tso_fw_assisted;
1849 if (sc->tso_fw_assisted)
1850 sc->tso_fw_assisted =
1851 (encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO) &&
1852 (encp->enc_fw_assisted_tso_enabled);
1937 if ((~encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO) ||
1938 (!encp->enc_fw_assisted_tso_enabled))
1939 sc->tso_fw_assisted &= ~SFXGE_FATSOV1;
1940 if ((~encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO_V2) ||
1941 (!encp->enc_fw_assisted_tso_v2_enabled))
1942 sc->tso_fw_assisted &= ~SFXGE_FATSOV2;
1853
1854 sc->txqs_node = SYSCTL_ADD_NODE(
1855 device_get_sysctl_ctx(sc->dev),
1856 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)),
1857 OID_AUTO, "txq", CTLFLAG_RD, NULL, "Tx queues");
1858 if (sc->txqs_node == NULL) {
1859 rc = ENOMEM;
1860 goto fail_txq_node;

--- 37 unchanged lines hidden --- 		1943
1944 sc->txqs_node = SYSCTL_ADD_NODE(
1945 device_get_sysctl_ctx(sc->dev),
1946 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)),
1947 OID_AUTO, "txq", CTLFLAG_RD, NULL, "Tx queues");
1948 if (sc->txqs_node == NULL) {
1949 rc = ENOMEM;
1950 goto fail_txq_node;

--- 37 unchanged lines hidden ---