1210311Sjmallett/************************************************************************* 2210311SjmallettCopyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights 3210311Sjmallettreserved. 4210311Sjmallett 5210311Sjmallett 6210311SjmallettRedistribution and use in source and binary forms, with or without 7210311Sjmallettmodification, are permitted provided that the following conditions are 8210311Sjmallettmet: 9210311Sjmallett 10210311Sjmallett * Redistributions of source code must retain the above copyright 11210311Sjmallett notice, this list of conditions and the following disclaimer. 12210311Sjmallett 13210311Sjmallett * Redistributions in binary form must reproduce the above 14210311Sjmallett copyright notice, this list of conditions and the following 15210311Sjmallett disclaimer in the documentation and/or other materials provided 16210311Sjmallett with the distribution. 17210311Sjmallett 18210311Sjmallett * Neither the name of Cavium Networks nor the names of 19210311Sjmallett its contributors may be used to endorse or promote products 20210311Sjmallett derived from this software without specific prior written 21210311Sjmallett permission. 22210311Sjmallett 23210311SjmallettThis Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries. 24210311Sjmallett 25210311SjmallettTO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 26210311SjmallettAND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 27210311Sjmallett 28210311Sjmallett*************************************************************************/ 29210311Sjmallett 30210311Sjmallett#include <sys/cdefs.h> 31210311Sjmallett__FBSDID("$FreeBSD: releng/10.3/sys/mips/cavium/octe/ethernet-tx.c 243264 2012-11-19 08:30:29Z jmallett $"); 32210311Sjmallett 33210311Sjmallett#include <sys/param.h> 34210311Sjmallett#include <sys/systm.h> 35210311Sjmallett#include <sys/bus.h> 36210311Sjmallett#include <sys/endian.h> 37210311Sjmallett#include <sys/kernel.h> 38210311Sjmallett#include <sys/mbuf.h> 39210311Sjmallett#include <sys/socket.h> 40210311Sjmallett 41213156Sjmallett#include <net/bpf.h> 42210311Sjmallett#include <net/ethernet.h> 43210311Sjmallett#include <net/if.h> 44210311Sjmallett 45210311Sjmallett#include "wrapper-cvmx-includes.h" 46210311Sjmallett#include "ethernet-headers.h" 47210311Sjmallett 48210311Sjmallett/* You can define GET_MBUF_QOS() to override how the mbuf output function 49210311Sjmallett determines which output queue is used. The default implementation 50210311Sjmallett always uses the base queue for the port. If, for example, you wanted 51210311Sjmallett to use the m->priority fieid, define GET_MBUF_QOS as: 52210311Sjmallett #define GET_MBUF_QOS(m) ((m)->priority) */ 53210311Sjmallett#ifndef GET_MBUF_QOS 54210311Sjmallett #define GET_MBUF_QOS(m) 0 55210311Sjmallett#endif 56210311Sjmallett 57210311Sjmallett 58210311Sjmallett/** 59210311Sjmallett * Packet transmit 60210311Sjmallett * 61210311Sjmallett * @param m Packet to send 62210311Sjmallett * @param dev Device info structure 63210311Sjmallett * @return Always returns zero 64210311Sjmallett */ 65210311Sjmallettint cvm_oct_xmit(struct mbuf *m, struct ifnet *ifp) 66210311Sjmallett{ 67210311Sjmallett cvmx_pko_command_word0_t pko_command; 68210311Sjmallett cvmx_buf_ptr_t hw_buffer; 69210311Sjmallett int dropped; 70210311Sjmallett int qos; 71210311Sjmallett cvm_oct_private_t *priv = (cvm_oct_private_t *)ifp->if_softc; 72210311Sjmallett int32_t in_use; 73210311Sjmallett int32_t buffers_to_free; 74210311Sjmallett cvmx_wqe_t *work; 75210311Sjmallett 76210311Sjmallett /* Prefetch the private data structure. 77210311Sjmallett It is larger that one cache line */ 78210311Sjmallett CVMX_PREFETCH(priv, 0); 79210311Sjmallett 80210311Sjmallett /* Start off assuming no drop */ 81210311Sjmallett dropped = 0; 82210311Sjmallett 83210311Sjmallett /* The check on CVMX_PKO_QUEUES_PER_PORT_* is designed to completely 84210311Sjmallett remove "qos" in the event neither interface supports multiple queues 85210311Sjmallett per port */ 86210311Sjmallett if ((CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 > 1) || 87210311Sjmallett (CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 > 1)) { 88210311Sjmallett qos = GET_MBUF_QOS(m); 89210311Sjmallett if (qos <= 0) 90210311Sjmallett qos = 0; 91210311Sjmallett else if (qos >= cvmx_pko_get_num_queues(priv->port)) 92210311Sjmallett qos = 0; 93210311Sjmallett } else 94210311Sjmallett qos = 0; 95210311Sjmallett 96210311Sjmallett /* The CN3XXX series of parts has an errata (GMX-401) which causes the 97210311Sjmallett GMX block to hang if a collision occurs towards the end of a 98210311Sjmallett <68 byte packet. As a workaround for this, we pad packets to be 99210311Sjmallett 68 bytes whenever we are in half duplex mode. We don't handle 100210311Sjmallett the case of having a small packet but no room to add the padding. 101210311Sjmallett The kernel should always give us at least a cache line */ 102210311Sjmallett if (__predict_false(m->m_pkthdr.len < 64) && OCTEON_IS_MODEL(OCTEON_CN3XXX)) { 103210311Sjmallett cvmx_gmxx_prtx_cfg_t gmx_prt_cfg; 104210311Sjmallett int interface = INTERFACE(priv->port); 105210311Sjmallett int index = INDEX(priv->port); 106210311Sjmallett 107210311Sjmallett if (interface < 2) { 108210311Sjmallett /* We only need to pad packet in half duplex mode */ 109210311Sjmallett gmx_prt_cfg.u64 = cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface)); 110210311Sjmallett if (gmx_prt_cfg.s.duplex == 0) { 111210311Sjmallett static uint8_t pad[64]; 112210311Sjmallett 113210311Sjmallett if (!m_append(m, sizeof pad - m->m_pkthdr.len, pad)) 114210311Sjmallett printf("%s: unable to padd small packet.", __func__); 115210311Sjmallett } 116210311Sjmallett } 117210311Sjmallett } 118210311Sjmallett 119243264Sjmallett#ifdef OCTEON_VENDOR_RADISYS 120210311Sjmallett /* 121243264Sjmallett * The RSYS4GBE will hang if asked to transmit a packet less than 60 bytes. 122243264Sjmallett */ 123243264Sjmallett if (__predict_false(m->m_pkthdr.len < 60) && 124243264Sjmallett cvmx_sysinfo_get()->board_type == CVMX_BOARD_TYPE_CUST_RADISYS_RSYS4GBE) { 125243264Sjmallett static uint8_t pad[60]; 126243264Sjmallett 127243264Sjmallett if (!m_append(m, sizeof pad - m->m_pkthdr.len, pad)) 128243264Sjmallett printf("%s: unable to pad small packet.", __func__); 129243264Sjmallett } 130243264Sjmallett#endif 131243264Sjmallett 132243264Sjmallett /* 133210311Sjmallett * If the packet is not fragmented. 134210311Sjmallett */ 135210311Sjmallett if (m->m_pkthdr.len == m->m_len) { 136210311Sjmallett /* Build the PKO buffer pointer */ 137210311Sjmallett hw_buffer.u64 = 0; 138210311Sjmallett hw_buffer.s.addr = cvmx_ptr_to_phys(m->m_data); 139210311Sjmallett hw_buffer.s.pool = 0; 140210311Sjmallett hw_buffer.s.size = m->m_len; 141210311Sjmallett 142210311Sjmallett /* Build the PKO command */ 143210311Sjmallett pko_command.u64 = 0; 144210311Sjmallett pko_command.s.segs = 1; 145216064Sjmallett pko_command.s.dontfree = 1; /* Do not put this buffer into the FPA. */ 146210311Sjmallett 147210311Sjmallett work = NULL; 148210311Sjmallett } else { 149210311Sjmallett struct mbuf *n; 150210311Sjmallett unsigned segs; 151210311Sjmallett uint64_t *gp; 152210311Sjmallett 153210311Sjmallett /* 154210311Sjmallett * The packet is fragmented, we need to send a list of segments 155210311Sjmallett * in memory we borrow from the WQE pool. 156210311Sjmallett */ 157210311Sjmallett work = cvmx_fpa_alloc(CVMX_FPA_WQE_POOL); 158217665Sjmallett if (work == NULL) { 159217665Sjmallett m_freem(m); 160217665Sjmallett ifp->if_oerrors++; 161217665Sjmallett return 1; 162217665Sjmallett } 163210311Sjmallett 164210311Sjmallett segs = 0; 165217665Sjmallett gp = (uint64_t *)work; 166210311Sjmallett for (n = m; n != NULL; n = n->m_next) { 167210311Sjmallett if (segs == CVMX_FPA_WQE_POOL_SIZE / sizeof (uint64_t)) 168210311Sjmallett panic("%s: too many segments in packet; call m_collapse().", __func__); 169210311Sjmallett 170210311Sjmallett /* Build the PKO buffer pointer */ 171210311Sjmallett hw_buffer.u64 = 0; 172216064Sjmallett hw_buffer.s.i = 1; /* Do not put this buffer into the FPA. */ 173210311Sjmallett hw_buffer.s.addr = cvmx_ptr_to_phys(n->m_data); 174210311Sjmallett hw_buffer.s.pool = 0; 175210311Sjmallett hw_buffer.s.size = n->m_len; 176210311Sjmallett 177210311Sjmallett *gp++ = hw_buffer.u64; 178210311Sjmallett segs++; 179210311Sjmallett } 180210311Sjmallett 181210311Sjmallett /* Build the PKO buffer gather list pointer */ 182210311Sjmallett hw_buffer.u64 = 0; 183210311Sjmallett hw_buffer.s.addr = cvmx_ptr_to_phys(work); 184210311Sjmallett hw_buffer.s.pool = CVMX_FPA_WQE_POOL; 185210311Sjmallett hw_buffer.s.size = segs; 186210311Sjmallett 187210311Sjmallett /* Build the PKO command */ 188210311Sjmallett pko_command.u64 = 0; 189210311Sjmallett pko_command.s.segs = segs; 190210311Sjmallett pko_command.s.gather = 1; 191216064Sjmallett pko_command.s.dontfree = 0; /* Put the WQE above back into the FPA. */ 192210311Sjmallett } 193210311Sjmallett 194210311Sjmallett /* Finish building the PKO command */ 195210311Sjmallett pko_command.s.n2 = 1; /* Don't pollute L2 with the outgoing packet */ 196210311Sjmallett pko_command.s.reg0 = priv->fau+qos*4; 197210311Sjmallett pko_command.s.total_bytes = m->m_pkthdr.len; 198210311Sjmallett pko_command.s.size0 = CVMX_FAU_OP_SIZE_32; 199210311Sjmallett pko_command.s.subone0 = 1; 200210311Sjmallett 201210311Sjmallett /* Check if we can use the hardware checksumming */ 202217664Sjmallett if ((m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) != 0) { 203210311Sjmallett /* Use hardware checksum calc */ 204210311Sjmallett pko_command.s.ipoffp1 = ETHER_HDR_LEN + 1; 205210311Sjmallett } 206210311Sjmallett 207216064Sjmallett /* 208216064Sjmallett * XXX 209216064Sjmallett * Could use a different free queue (and different FAU address) per 210216064Sjmallett * core instead of per QoS, to reduce contention here. 211216064Sjmallett */ 212210311Sjmallett IF_LOCK(&priv->tx_free_queue[qos]); 213217664Sjmallett /* Get the number of mbufs in use by the hardware */ 214217664Sjmallett in_use = cvmx_fau_fetch_and_add32(priv->fau+qos*4, 1); 215217664Sjmallett buffers_to_free = cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); 216210311Sjmallett 217210311Sjmallett cvmx_pko_send_packet_prepare(priv->port, priv->queue + qos, CVMX_PKO_LOCK_CMD_QUEUE); 218210311Sjmallett 219210311Sjmallett /* Drop this packet if we have too many already queued to the HW */ 220210311Sjmallett if (_IF_QFULL(&priv->tx_free_queue[qos])) { 221210311Sjmallett dropped = 1; 222210311Sjmallett } 223210311Sjmallett /* Send the packet to the output queue */ 224210311Sjmallett else 225210311Sjmallett if (__predict_false(cvmx_pko_send_packet_finish(priv->port, priv->queue + qos, pko_command, hw_buffer, CVMX_PKO_LOCK_CMD_QUEUE))) { 226210311Sjmallett DEBUGPRINT("%s: Failed to send the packet\n", if_name(ifp)); 227210311Sjmallett dropped = 1; 228210311Sjmallett } 229210311Sjmallett 230210311Sjmallett if (__predict_false(dropped)) { 231210311Sjmallett m_freem(m); 232210311Sjmallett cvmx_fau_atomic_add32(priv->fau+qos*4, -1); 233210311Sjmallett ifp->if_oerrors++; 234210311Sjmallett } else { 235210311Sjmallett /* Put this packet on the queue to be freed later */ 236210311Sjmallett _IF_ENQUEUE(&priv->tx_free_queue[qos], m); 237213156Sjmallett 238213156Sjmallett /* Pass it to any BPF listeners. */ 239213156Sjmallett ETHER_BPF_MTAP(ifp, m); 240217244Sjmallett 241217244Sjmallett ifp->if_opackets++; 242217244Sjmallett ifp->if_obytes += m->m_pkthdr.len; 243210311Sjmallett } 244210311Sjmallett 245210311Sjmallett /* Free mbufs not in use by the hardware */ 246210311Sjmallett if (_IF_QLEN(&priv->tx_free_queue[qos]) > in_use) { 247210311Sjmallett while (_IF_QLEN(&priv->tx_free_queue[qos]) > in_use) { 248210311Sjmallett _IF_DEQUEUE(&priv->tx_free_queue[qos], m); 249210311Sjmallett m_freem(m); 250210311Sjmallett } 251210311Sjmallett } 252210311Sjmallett IF_UNLOCK(&priv->tx_free_queue[qos]); 253210311Sjmallett 254210311Sjmallett return dropped; 255210311Sjmallett} 256210311Sjmallett 257210311Sjmallett 258210311Sjmallett/** 259210311Sjmallett * This function frees all mbufs that are currenty queued for TX. 260210311Sjmallett * 261210311Sjmallett * @param dev Device being shutdown 262210311Sjmallett */ 263210311Sjmallettvoid cvm_oct_tx_shutdown(struct ifnet *ifp) 264210311Sjmallett{ 265210311Sjmallett cvm_oct_private_t *priv = (cvm_oct_private_t *)ifp->if_softc; 266210311Sjmallett int qos; 267210311Sjmallett 268210311Sjmallett for (qos = 0; qos < 16; qos++) { 269210311Sjmallett IF_DRAIN(&priv->tx_free_queue[qos]); 270210311Sjmallett } 271210311Sjmallett} 272