ethernet-tx.c revision 217244
1/************************************************************************* 2Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights 3reserved. 4 5 6Redistribution and use in source and binary forms, with or without 7modification, are permitted provided that the following conditions are 8met: 9 10 * Redistributions of source code must retain the above copyright 11 notice, this list of conditions and the following disclaimer. 12 13 * Redistributions in binary form must reproduce the above 14 copyright notice, this list of conditions and the following 15 disclaimer in the documentation and/or other materials provided 16 with the distribution. 17 18 * Neither the name of Cavium Networks nor the names of 19 its contributors may be used to endorse or promote products 20 derived from this software without specific prior written 21 permission. 22 23This Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries. 24 25TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 26AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 27 28*************************************************************************/ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: head/sys/mips/cavium/octe/ethernet-tx.c 217244 2011-01-10 22:14:30Z jmallett $"); 32 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/bus.h> 36#include <sys/endian.h> 37#include <sys/kernel.h> 38#include <sys/mbuf.h> 39#include <sys/socket.h> 40 41#include <net/bpf.h> 42#include <net/ethernet.h> 43#include <net/if.h> 44 45#include "wrapper-cvmx-includes.h" 46#include "ethernet-headers.h" 47 48/* You can define GET_MBUF_QOS() to override how the mbuf output function 49 determines which output queue is used. The default implementation 50 always uses the base queue for the port. If, for example, you wanted 51 to use the m->priority fieid, define GET_MBUF_QOS as: 52 #define GET_MBUF_QOS(m) ((m)->priority) */ 53#ifndef GET_MBUF_QOS 54 #define GET_MBUF_QOS(m) 0 55#endif 56 57 58/** 59 * Packet transmit 60 * 61 * @param m Packet to send 62 * @param dev Device info structure 63 * @return Always returns zero 64 */ 65int cvm_oct_xmit(struct mbuf *m, struct ifnet *ifp) 66{ 67 cvmx_pko_command_word0_t pko_command; 68 cvmx_buf_ptr_t hw_buffer; 69 uint64_t old_scratch; 70 uint64_t old_scratch2; 71 int dropped; 72 int qos; 73 cvm_oct_private_t *priv = (cvm_oct_private_t *)ifp->if_softc; 74 int32_t in_use; 75 int32_t buffers_to_free; 76 cvmx_wqe_t *work; 77 78 /* Prefetch the private data structure. 79 It is larger that one cache line */ 80 CVMX_PREFETCH(priv, 0); 81 82 /* Start off assuming no drop */ 83 dropped = 0; 84 85 /* The check on CVMX_PKO_QUEUES_PER_PORT_* is designed to completely 86 remove "qos" in the event neither interface supports multiple queues 87 per port */ 88 if ((CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 > 1) || 89 (CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 > 1)) { 90 qos = GET_MBUF_QOS(m); 91 if (qos <= 0) 92 qos = 0; 93 else if (qos >= cvmx_pko_get_num_queues(priv->port)) 94 qos = 0; 95 } else 96 qos = 0; 97 98 if (USE_ASYNC_IOBDMA) { 99 /* Save scratch in case userspace is using it */ 100 CVMX_SYNCIOBDMA; 101 old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH); 102 old_scratch2 = cvmx_scratch_read64(CVMX_SCR_SCRATCH+8); 103 104 /* Assume we're going to be able t osend this packet. Fetch and increment 105 the number of pending packets for output */ 106 cvmx_fau_async_fetch_and_add32(CVMX_SCR_SCRATCH+8, FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); 107 cvmx_fau_async_fetch_and_add32(CVMX_SCR_SCRATCH, priv->fau+qos*4, 1); 108 } 109 110 /* The CN3XXX series of parts has an errata (GMX-401) which causes the 111 GMX block to hang if a collision occurs towards the end of a 112 <68 byte packet. As a workaround for this, we pad packets to be 113 68 bytes whenever we are in half duplex mode. We don't handle 114 the case of having a small packet but no room to add the padding. 115 The kernel should always give us at least a cache line */ 116 if (__predict_false(m->m_pkthdr.len < 64) && OCTEON_IS_MODEL(OCTEON_CN3XXX)) { 117 cvmx_gmxx_prtx_cfg_t gmx_prt_cfg; 118 int interface = INTERFACE(priv->port); 119 int index = INDEX(priv->port); 120 121 if (interface < 2) { 122 /* We only need to pad packet in half duplex mode */ 123 gmx_prt_cfg.u64 = cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface)); 124 if (gmx_prt_cfg.s.duplex == 0) { 125 static uint8_t pad[64]; 126 127 if (!m_append(m, sizeof pad - m->m_pkthdr.len, pad)) 128 printf("%s: unable to padd small packet.", __func__); 129 } 130 } 131 } 132 133 /* 134 * If the packet is not fragmented. 135 */ 136 if (m->m_pkthdr.len == m->m_len) { 137 /* Build the PKO buffer pointer */ 138 hw_buffer.u64 = 0; 139 hw_buffer.s.addr = cvmx_ptr_to_phys(m->m_data); 140 hw_buffer.s.pool = 0; 141 hw_buffer.s.size = m->m_len; 142 143 /* Build the PKO command */ 144 pko_command.u64 = 0; 145 pko_command.s.segs = 1; 146 pko_command.s.dontfree = 1; /* Do not put this buffer into the FPA. */ 147 148 work = NULL; 149 } else { 150 struct mbuf *n; 151 unsigned segs; 152 uint64_t *gp; 153 154 /* 155 * The packet is fragmented, we need to send a list of segments 156 * in memory we borrow from the WQE pool. 157 */ 158 work = cvmx_fpa_alloc(CVMX_FPA_WQE_POOL); 159 gp = (uint64_t *)work; 160 161 segs = 0; 162 for (n = m; n != NULL; n = n->m_next) { 163 if (segs == CVMX_FPA_WQE_POOL_SIZE / sizeof (uint64_t)) 164 panic("%s: too many segments in packet; call m_collapse().", __func__); 165 166 /* Build the PKO buffer pointer */ 167 hw_buffer.u64 = 0; 168 hw_buffer.s.i = 1; /* Do not put this buffer into the FPA. */ 169 hw_buffer.s.addr = cvmx_ptr_to_phys(n->m_data); 170 hw_buffer.s.pool = 0; 171 hw_buffer.s.size = n->m_len; 172 173 *gp++ = hw_buffer.u64; 174 segs++; 175 } 176 177 /* Build the PKO buffer gather list pointer */ 178 hw_buffer.u64 = 0; 179 hw_buffer.s.addr = cvmx_ptr_to_phys(work); 180 hw_buffer.s.pool = CVMX_FPA_WQE_POOL; 181 hw_buffer.s.size = segs; 182 183 /* Build the PKO command */ 184 pko_command.u64 = 0; 185 pko_command.s.segs = segs; 186 pko_command.s.gather = 1; 187 pko_command.s.dontfree = 0; /* Put the WQE above back into the FPA. */ 188 } 189 190 /* Finish building the PKO command */ 191 pko_command.s.n2 = 1; /* Don't pollute L2 with the outgoing packet */ 192 pko_command.s.reg0 = priv->fau+qos*4; 193 pko_command.s.total_bytes = m->m_pkthdr.len; 194 pko_command.s.size0 = CVMX_FAU_OP_SIZE_32; 195 pko_command.s.subone0 = 1; 196 197 /* Check if we can use the hardware checksumming */ 198 if (USE_HW_TCPUDP_CHECKSUM && 199 (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) != 0) { 200 /* Use hardware checksum calc */ 201 pko_command.s.ipoffp1 = ETHER_HDR_LEN + 1; 202 } 203 204 /* 205 * XXX 206 * Could use a different free queue (and different FAU address) per 207 * core instead of per QoS, to reduce contention here. 208 */ 209 IF_LOCK(&priv->tx_free_queue[qos]); 210 if (USE_ASYNC_IOBDMA) { 211 /* Get the number of mbufs in use by the hardware */ 212 CVMX_SYNCIOBDMA; 213 in_use = cvmx_scratch_read64(CVMX_SCR_SCRATCH); 214 buffers_to_free = cvmx_scratch_read64(CVMX_SCR_SCRATCH+8); 215 } else { 216 /* Get the number of mbufs in use by the hardware */ 217 in_use = cvmx_fau_fetch_and_add32(priv->fau+qos*4, 1); 218 buffers_to_free = cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); 219 } 220 221 cvmx_pko_send_packet_prepare(priv->port, priv->queue + qos, CVMX_PKO_LOCK_CMD_QUEUE); 222 223 /* Drop this packet if we have too many already queued to the HW */ 224 if (_IF_QFULL(&priv->tx_free_queue[qos])) { 225 dropped = 1; 226 } 227 /* Send the packet to the output queue */ 228 else 229 if (__predict_false(cvmx_pko_send_packet_finish(priv->port, priv->queue + qos, pko_command, hw_buffer, CVMX_PKO_LOCK_CMD_QUEUE))) { 230 DEBUGPRINT("%s: Failed to send the packet\n", if_name(ifp)); 231 dropped = 1; 232 } 233 234 if (USE_ASYNC_IOBDMA) { 235 /* Restore the scratch area */ 236 cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch); 237 cvmx_scratch_write64(CVMX_SCR_SCRATCH+8, old_scratch2); 238 } 239 240 if (__predict_false(dropped)) { 241 m_freem(m); 242 cvmx_fau_atomic_add32(priv->fau+qos*4, -1); 243 ifp->if_oerrors++; 244 } else { 245 /* Put this packet on the queue to be freed later */ 246 _IF_ENQUEUE(&priv->tx_free_queue[qos], m); 247 248 /* Pass it to any BPF listeners. */ 249 ETHER_BPF_MTAP(ifp, m); 250 251 ifp->if_opackets++; 252 ifp->if_obytes += m->m_pkthdr.len; 253 } 254 255 /* Free mbufs not in use by the hardware */ 256 if (_IF_QLEN(&priv->tx_free_queue[qos]) > in_use) { 257 while (_IF_QLEN(&priv->tx_free_queue[qos]) > in_use) { 258 _IF_DEQUEUE(&priv->tx_free_queue[qos], m); 259 m_freem(m); 260 } 261 } 262 IF_UNLOCK(&priv->tx_free_queue[qos]); 263 264 return dropped; 265} 266 267 268/** 269 * This function frees all mbufs that are currenty queued for TX. 270 * 271 * @param dev Device being shutdown 272 */ 273void cvm_oct_tx_shutdown(struct ifnet *ifp) 274{ 275 cvm_oct_private_t *priv = (cvm_oct_private_t *)ifp->if_softc; 276 int qos; 277 278 for (qos = 0; qos < 16; qos++) { 279 IF_DRAIN(&priv->tx_free_queue[qos]); 280 } 281} 282