1210311Sjmallett/************************************************************************* 2210311SjmallettCopyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights 3210311Sjmallettreserved. 4210311Sjmallett 5210311Sjmallett 6210311SjmallettRedistribution and use in source and binary forms, with or without 7210311Sjmallettmodification, are permitted provided that the following conditions are 8210311Sjmallettmet: 9210311Sjmallett 10210311Sjmallett * Redistributions of source code must retain the above copyright 11210311Sjmallett notice, this list of conditions and the following disclaimer. 12210311Sjmallett 13210311Sjmallett * Redistributions in binary form must reproduce the above 14210311Sjmallett copyright notice, this list of conditions and the following 15210311Sjmallett disclaimer in the documentation and/or other materials provided 16210311Sjmallett with the distribution. 17210311Sjmallett 18210311Sjmallett * Neither the name of Cavium Networks nor the names of 19210311Sjmallett its contributors may be used to endorse or promote products 20210311Sjmallett derived from this software without specific prior written 21210311Sjmallett permission. 22210311Sjmallett 23210311SjmallettThis Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries. 24210311Sjmallett 25210311SjmallettTO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 26210311SjmallettAND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 27210311Sjmallett 28210311Sjmallett*************************************************************************/ 29210311Sjmallett 30210311Sjmallett#include <sys/cdefs.h> 31210311Sjmallett__FBSDID("$FreeBSD$"); 32210311Sjmallett 33210311Sjmallett#include <sys/param.h> 34210311Sjmallett#include <sys/systm.h> 35210311Sjmallett#include <sys/bus.h> 36210311Sjmallett#include <sys/endian.h> 37210311Sjmallett#include <sys/kernel.h> 38210311Sjmallett#include <sys/mbuf.h> 39210311Sjmallett#include <sys/socket.h> 40213156Sjmallett#include <sys/proc.h> 41213156Sjmallett#include <sys/sched.h> 42210311Sjmallett#include <sys/smp.h> 43210311Sjmallett#include <sys/taskqueue.h> 44210311Sjmallett 45210311Sjmallett#include <net/ethernet.h> 46210311Sjmallett#include <net/if.h> 47257324Sglebius#include <net/if_var.h> 48210311Sjmallett 49210311Sjmallett#include "wrapper-cvmx-includes.h" 50210311Sjmallett#include "ethernet-headers.h" 51210311Sjmallett 52210311Sjmallettextern int pow_receive_group; 53210311Sjmallettextern struct ifnet *cvm_oct_device[]; 54210311Sjmallett 55210311Sjmallettstatic struct task cvm_oct_task; 56210311Sjmallettstatic struct taskqueue *cvm_oct_taskq; 57210311Sjmallett 58217212Sjmallettstatic int cvm_oct_rx_active; 59217212Sjmallett 60210311Sjmallett/** 61210311Sjmallett * Interrupt handler. The interrupt occurs whenever the POW 62210311Sjmallett * transitions from 0->1 packets in our group. 63210311Sjmallett * 64210311Sjmallett * @param cpl 65210311Sjmallett * @param dev_id 66210311Sjmallett * @param regs 67210311Sjmallett * @return 68210311Sjmallett */ 69210311Sjmallettint cvm_oct_do_interrupt(void *dev_id) 70210311Sjmallett{ 71210311Sjmallett /* Acknowledge the interrupt */ 72210311Sjmallett if (INTERRUPT_LIMIT) 73210311Sjmallett cvmx_write_csr(CVMX_POW_WQ_INT, 1<<pow_receive_group); 74210311Sjmallett else 75210311Sjmallett cvmx_write_csr(CVMX_POW_WQ_INT, 0x10001<<pow_receive_group); 76217212Sjmallett 77217212Sjmallett /* 78217212Sjmallett * Schedule task if there isn't one running. 79217212Sjmallett */ 80217212Sjmallett if (atomic_cmpset_int(&cvm_oct_rx_active, 0, 1)) 81217212Sjmallett taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task); 82217212Sjmallett 83210311Sjmallett return FILTER_HANDLED; 84210311Sjmallett} 85210311Sjmallett 86210311Sjmallett 87210311Sjmallett/** 88210311Sjmallett * This is called on receive errors, and determines if the packet 89210311Sjmallett * can be dropped early-on in cvm_oct_tasklet_rx(). 90210311Sjmallett * 91210311Sjmallett * @param work Work queue entry pointing to the packet. 92210311Sjmallett * @return Non-zero if the packet can be dropped, zero otherwise. 93210311Sjmallett */ 94210311Sjmallettstatic inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work) 95210311Sjmallett{ 96232812Sjmallett if ((work->word2.snoip.err_code == 10) && (work->word1.s.len <= 64)) { 97210311Sjmallett /* Ignore length errors on min size packets. Some equipment 98210311Sjmallett incorrectly pads packets to 64+4FCS instead of 60+4FCS. 99210311Sjmallett Note these packets still get counted as frame errors. */ 100210311Sjmallett } else 101210311Sjmallett if (USE_10MBPS_PREAMBLE_WORKAROUND && ((work->word2.snoip.err_code == 5) || (work->word2.snoip.err_code == 7))) { 102210311Sjmallett 103210311Sjmallett /* We received a packet with either an alignment error or a 104210311Sjmallett FCS error. This may be signalling that we are running 105210311Sjmallett 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK} off. If this is the 106210311Sjmallett case we need to parse the packet to determine if we can 107210311Sjmallett remove a non spec preamble and generate a correct packet */ 108232812Sjmallett int interface = cvmx_helper_get_interface_num(work->word1.cn38xx.ipprt); 109232812Sjmallett int index = cvmx_helper_get_interface_index_num(work->word1.cn38xx.ipprt); 110210311Sjmallett cvmx_gmxx_rxx_frm_ctl_t gmxx_rxx_frm_ctl; 111210311Sjmallett gmxx_rxx_frm_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface)); 112210311Sjmallett if (gmxx_rxx_frm_ctl.s.pre_chk == 0) { 113210311Sjmallett 114210311Sjmallett uint8_t *ptr = cvmx_phys_to_ptr(work->packet_ptr.s.addr); 115210311Sjmallett int i = 0; 116210311Sjmallett 117232812Sjmallett while (i < work->word1.s.len-1) { 118210311Sjmallett if (*ptr != 0x55) 119210311Sjmallett break; 120210311Sjmallett ptr++; 121210311Sjmallett i++; 122210311Sjmallett } 123210311Sjmallett 124210311Sjmallett if (*ptr == 0xd5) { 125210311Sjmallett /* 126232812Sjmallett DEBUGPRINT("Port %d received 0xd5 preamble\n", work->word1.cn38xx.ipprt); 127210311Sjmallett */ 128210311Sjmallett work->packet_ptr.s.addr += i+1; 129232812Sjmallett work->word1.s.len -= i+5; 130210311Sjmallett } else 131210311Sjmallett if ((*ptr & 0xf) == 0xd) { 132210311Sjmallett /* 133232812Sjmallett DEBUGPRINT("Port %d received 0x?d preamble\n", work->word1.cn38xx.ipprt); 134210311Sjmallett */ 135210311Sjmallett work->packet_ptr.s.addr += i; 136232812Sjmallett work->word1.s.len -= i+4; 137232812Sjmallett for (i = 0; i < work->word1.s.len; i++) { 138210311Sjmallett *ptr = ((*ptr&0xf0)>>4) | ((*(ptr+1)&0xf)<<4); 139210311Sjmallett ptr++; 140210311Sjmallett } 141210311Sjmallett } else { 142232812Sjmallett DEBUGPRINT("Port %d unknown preamble, packet dropped\n", work->word1.cn38xx.ipprt); 143210311Sjmallett /* 144210311Sjmallett cvmx_helper_dump_packet(work); 145210311Sjmallett */ 146210311Sjmallett cvm_oct_free_work(work); 147210311Sjmallett return 1; 148210311Sjmallett } 149210311Sjmallett } 150210311Sjmallett } else { 151232812Sjmallett DEBUGPRINT("Port %d receive error code %d, packet dropped\n", work->word1.cn38xx.ipprt, work->word2.snoip.err_code); 152210311Sjmallett cvm_oct_free_work(work); 153210311Sjmallett return 1; 154210311Sjmallett } 155210311Sjmallett 156210311Sjmallett return 0; 157210311Sjmallett} 158210311Sjmallett 159210311Sjmallett/** 160210311Sjmallett * Tasklet function that is scheduled on a core when an interrupt occurs. 161210311Sjmallett * 162210311Sjmallett * @param unused 163210311Sjmallett */ 164210311Sjmallettvoid cvm_oct_tasklet_rx(void *context, int pending) 165210311Sjmallett{ 166213156Sjmallett int coreid; 167210311Sjmallett uint64_t old_group_mask; 168210311Sjmallett int rx_count = 0; 169210311Sjmallett int number_to_free; 170210311Sjmallett int num_freed; 171210311Sjmallett int packet_not_copied; 172210311Sjmallett 173213156Sjmallett sched_pin(); 174213156Sjmallett coreid = cvmx_get_core_num(); 175213156Sjmallett 176210311Sjmallett /* Prefetch cvm_oct_device since we know we need it soon */ 177210311Sjmallett CVMX_PREFETCH(cvm_oct_device, 0); 178210311Sjmallett 179210311Sjmallett /* Only allow work for our group (and preserve priorities) */ 180210311Sjmallett old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid)); 181210311Sjmallett cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), 182210311Sjmallett (old_group_mask & ~0xFFFFull) | 1<<pow_receive_group); 183210311Sjmallett 184210311Sjmallett while (1) { 185210311Sjmallett struct mbuf *m = NULL; 186210311Sjmallett int mbuf_in_hw; 187210311Sjmallett cvmx_wqe_t *work; 188210311Sjmallett 189217664Sjmallett if ((INTERRUPT_LIMIT == 0) || (rx_count < MAX_RX_PACKETS)) 190217664Sjmallett work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT); 191217664Sjmallett else 192217664Sjmallett work = NULL; 193210311Sjmallett CVMX_PREFETCH(work, 0); 194210311Sjmallett if (work == NULL) 195210311Sjmallett break; 196210311Sjmallett 197215974Sjmallett mbuf_in_hw = work->word2.s.bufs == 1; 198210311Sjmallett if ((mbuf_in_hw)) { 199210311Sjmallett m = *(struct mbuf **)(cvm_oct_get_buffer_ptr(work->packet_ptr) - sizeof(void *)); 200210311Sjmallett CVMX_PREFETCH(m, offsetof(struct mbuf, m_data)); 201210311Sjmallett CVMX_PREFETCH(m, offsetof(struct mbuf, m_pkthdr)); 202210311Sjmallett } 203232812Sjmallett CVMX_PREFETCH(cvm_oct_device[work->word1.cn38xx.ipprt], 0); 204210311Sjmallett //CVMX_PREFETCH(m, 0); 205210311Sjmallett 206210311Sjmallett 207210311Sjmallett rx_count++; 208210311Sjmallett /* Immediately throw away all packets with receive errors */ 209210311Sjmallett if ((work->word2.snoip.rcv_error)) { 210210311Sjmallett if (cvm_oct_check_rcv_error(work)) 211210311Sjmallett continue; 212210311Sjmallett } 213210311Sjmallett 214210311Sjmallett /* We can only use the zero copy path if mbufs are in the FPA pool 215210311Sjmallett and the packet fits in a single buffer */ 216210311Sjmallett if ((mbuf_in_hw)) { 217210311Sjmallett CVMX_PREFETCH(m->m_data, 0); 218210311Sjmallett 219232812Sjmallett m->m_pkthdr.len = m->m_len = work->word1.s.len; 220210311Sjmallett 221210311Sjmallett packet_not_copied = 1; 222210311Sjmallett 223210311Sjmallett /* 224210311Sjmallett * Adjust the data pointer based on the offset 225210311Sjmallett * of the packet within the buffer. 226210311Sjmallett */ 227210311Sjmallett m->m_data += (work->packet_ptr.s.back << 7) + (work->packet_ptr.s.addr & 0x7f); 228210311Sjmallett } else { 229210311Sjmallett 230210311Sjmallett /* We have to copy the packet. First allocate an 231210311Sjmallett mbuf for it */ 232243882Sglebius MGETHDR(m, M_NOWAIT, MT_DATA); 233210311Sjmallett if (m == NULL) { 234232812Sjmallett DEBUGPRINT("Port %d failed to allocate mbuf, packet dropped\n", work->word1.cn38xx.ipprt); 235210311Sjmallett cvm_oct_free_work(work); 236210311Sjmallett continue; 237210311Sjmallett } 238210311Sjmallett 239210311Sjmallett /* Check if we've received a packet that was entirely 240210311Sjmallett stored in the work entry. This is untested */ 241210311Sjmallett if ((work->word2.s.bufs == 0)) { 242210311Sjmallett uint8_t *ptr = work->packet_data; 243210311Sjmallett 244210311Sjmallett if (cvmx_likely(!work->word2.s.not_IP)) { 245210311Sjmallett /* The beginning of the packet moves 246210311Sjmallett for IP packets */ 247210311Sjmallett if (work->word2.s.is_v6) 248210311Sjmallett ptr += 2; 249210311Sjmallett else 250210311Sjmallett ptr += 6; 251210311Sjmallett } 252210311Sjmallett panic("%s: not yet implemented; copy in small packet.", __func__); 253210311Sjmallett /* No packet buffers to free */ 254210311Sjmallett } else { 255210311Sjmallett int segments = work->word2.s.bufs; 256210311Sjmallett cvmx_buf_ptr_t segment_ptr = work->packet_ptr; 257232812Sjmallett int len = work->word1.s.len; 258210311Sjmallett 259210311Sjmallett while (segments--) { 260210311Sjmallett cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8); 261210311Sjmallett /* Octeon Errata PKI-100: The segment 262210311Sjmallett size is wrong. Until it is fixed, 263210311Sjmallett calculate the segment size based on 264210311Sjmallett the packet pool buffer size. When 265210311Sjmallett it is fixed, the following line 266210311Sjmallett should be replaced with this one: 267210311Sjmallett int segment_size = segment_ptr.s.size; */ 268210311Sjmallett int segment_size = CVMX_FPA_PACKET_POOL_SIZE - (segment_ptr.s.addr - (((segment_ptr.s.addr >> 7) - segment_ptr.s.back) << 7)); 269210311Sjmallett /* Don't copy more than what is left 270210311Sjmallett in the packet */ 271210311Sjmallett if (segment_size > len) 272210311Sjmallett segment_size = len; 273210311Sjmallett /* Copy the data into the packet */ 274210311Sjmallett panic("%s: not yet implemented; copy in packet segments.", __func__); 275210311Sjmallett#if 0 276210311Sjmallett memcpy(m_put(m, segment_size), cvmx_phys_to_ptr(segment_ptr.s.addr), segment_size); 277210311Sjmallett#endif 278210311Sjmallett /* Reduce the amount of bytes left 279210311Sjmallett to copy */ 280210311Sjmallett len -= segment_size; 281210311Sjmallett segment_ptr = next_ptr; 282210311Sjmallett } 283210311Sjmallett } 284210311Sjmallett packet_not_copied = 0; 285210311Sjmallett } 286210311Sjmallett 287232812Sjmallett if (((work->word1.cn38xx.ipprt < TOTAL_NUMBER_OF_PORTS) && 288232812Sjmallett cvm_oct_device[work->word1.cn38xx.ipprt])) { 289232812Sjmallett struct ifnet *ifp = cvm_oct_device[work->word1.cn38xx.ipprt]; 290210311Sjmallett 291210311Sjmallett /* Only accept packets for devices 292210311Sjmallett that are currently up */ 293210311Sjmallett if ((ifp->if_flags & IFF_UP)) { 294210311Sjmallett m->m_pkthdr.rcvif = ifp; 295210311Sjmallett 296210311Sjmallett if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 297210311Sjmallett if ((work->word2.s.not_IP || work->word2.s.IP_exc || work->word2.s.L4_error)) 298210311Sjmallett m->m_pkthdr.csum_flags = 0; /* XXX */ 299210311Sjmallett else { 300210311Sjmallett m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 301210311Sjmallett m->m_pkthdr.csum_data = 0xffff; 302210311Sjmallett } 303210311Sjmallett } else { 304210311Sjmallett m->m_pkthdr.csum_flags = 0; /* XXX */ 305210311Sjmallett } 306210311Sjmallett 307271858Sglebius if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 308210311Sjmallett 309210311Sjmallett (*ifp->if_input)(ifp, m); 310210311Sjmallett } else { 311210311Sjmallett /* Drop any packet received for a device that isn't up */ 312210311Sjmallett /* 313210311Sjmallett DEBUGPRINT("%s: Device not up, packet dropped\n", 314210311Sjmallett if_name(ifp)); 315210311Sjmallett */ 316210311Sjmallett m_freem(m); 317210311Sjmallett } 318210311Sjmallett } else { 319210311Sjmallett /* Drop any packet received for a device that 320210311Sjmallett doesn't exist */ 321242390Sjmallett DEBUGPRINT("Port %d not controlled by FreeBSD, packet dropped\n", work->word1.cn38xx.ipprt); 322210311Sjmallett m_freem(m); 323210311Sjmallett } 324210311Sjmallett 325210311Sjmallett /* Check to see if the mbuf and work share 326210311Sjmallett the same packet buffer */ 327215974Sjmallett if ((packet_not_copied)) { 328210311Sjmallett /* This buffer needs to be replaced, increment 329210311Sjmallett the number of buffers we need to free by one */ 330210311Sjmallett cvmx_fau_atomic_add32( 331210311Sjmallett FAU_NUM_PACKET_BUFFERS_TO_FREE, 1); 332210311Sjmallett 333210311Sjmallett cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 334210311Sjmallett DONT_WRITEBACK(1)); 335210311Sjmallett } else 336210311Sjmallett cvm_oct_free_work(work); 337210311Sjmallett } 338210311Sjmallett 339217212Sjmallett /* 340217212Sjmallett * If we hit our limit, schedule another task while we clean up. 341217212Sjmallett */ 342217212Sjmallett if (INTERRUPT_LIMIT != 0 && rx_count == MAX_RX_PACKETS) { 343217212Sjmallett taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task); 344217212Sjmallett } else { 345217212Sjmallett /* 346217212Sjmallett * No more packets, all done. 347217212Sjmallett */ 348217212Sjmallett if (!atomic_cmpset_int(&cvm_oct_rx_active, 1, 0)) 349217212Sjmallett panic("%s: inconsistent rx active state.", __func__); 350217212Sjmallett } 351217212Sjmallett 352210311Sjmallett /* Restore the original POW group mask */ 353210311Sjmallett cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask); 354210311Sjmallett 355215974Sjmallett /* Refill the packet buffer pool */ 356215974Sjmallett number_to_free = 357215974Sjmallett cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); 358210311Sjmallett 359215974Sjmallett if (number_to_free > 0) { 360215974Sjmallett cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 361215974Sjmallett -number_to_free); 362215974Sjmallett num_freed = 363215974Sjmallett cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, 364215974Sjmallett CVMX_FPA_PACKET_POOL_SIZE, 365215974Sjmallett number_to_free); 366215974Sjmallett if (num_freed != number_to_free) { 367210311Sjmallett cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 368215974Sjmallett number_to_free - num_freed); 369210311Sjmallett } 370210311Sjmallett } 371213156Sjmallett sched_unpin(); 372210311Sjmallett} 373210311Sjmallett 374210311Sjmallett 375210311Sjmallett 376210311Sjmallettvoid cvm_oct_rx_initialize(void) 377210311Sjmallett{ 378210311Sjmallett TASK_INIT(&cvm_oct_task, 0, cvm_oct_tasklet_rx, NULL); 379210311Sjmallett 380210311Sjmallett cvm_oct_taskq = taskqueue_create_fast("oct_rx", M_NOWAIT, 381210311Sjmallett taskqueue_thread_enqueue, 382210311Sjmallett &cvm_oct_taskq); 383210311Sjmallett taskqueue_start_threads(&cvm_oct_taskq, min(mp_ncpus, MAXCPU), 384210311Sjmallett PI_NET, "octe taskq"); 385210311Sjmallett} 386210311Sjmallett 387210311Sjmallettvoid cvm_oct_rx_shutdown(void) 388210311Sjmallett{ 389210311Sjmallett panic("%s: not yet implemented.", __func__); 390210311Sjmallett} 391210311Sjmallett 392