1210311Sjmallett/************************************************************************* 2210311SjmallettCopyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights 3210311Sjmallettreserved. 4210311Sjmallett 5210311Sjmallett 6210311SjmallettRedistribution and use in source and binary forms, with or without 7210311Sjmallettmodification, are permitted provided that the following conditions are 8210311Sjmallettmet: 9210311Sjmallett 10210311Sjmallett * Redistributions of source code must retain the above copyright 11210311Sjmallett notice, this list of conditions and the following disclaimer. 12210311Sjmallett 13210311Sjmallett * Redistributions in binary form must reproduce the above 14210311Sjmallett copyright notice, this list of conditions and the following 15210311Sjmallett disclaimer in the documentation and/or other materials provided 16210311Sjmallett with the distribution. 17210311Sjmallett 18210311Sjmallett * Neither the name of Cavium Networks nor the names of 19210311Sjmallett its contributors may be used to endorse or promote products 20210311Sjmallett derived from this software without specific prior written 21210311Sjmallett permission. 22210311Sjmallett 23210311SjmallettThis Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries. 24210311Sjmallett 25210311SjmallettTO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 26210311SjmallettAND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 27210311Sjmallett 28210311Sjmallett*************************************************************************/ 29210311Sjmallett 30210311Sjmallett#include <sys/cdefs.h> 31210311Sjmallett__FBSDID("$FreeBSD$"); 32210311Sjmallett 33210311Sjmallett#include <sys/param.h> 34210311Sjmallett#include <sys/systm.h> 35210311Sjmallett#include <sys/bus.h> 36210311Sjmallett#include <sys/endian.h> 37210311Sjmallett#include <sys/kernel.h> 38210311Sjmallett#include <sys/mbuf.h> 39210311Sjmallett#include <sys/socket.h> 40213156Sjmallett#include <sys/proc.h> 41213156Sjmallett#include <sys/sched.h> 42210311Sjmallett#include <sys/smp.h> 43210311Sjmallett#include <sys/taskqueue.h> 44210311Sjmallett 45210311Sjmallett#include <net/ethernet.h> 46210311Sjmallett#include <net/if.h> 47210311Sjmallett 48210311Sjmallett#include "wrapper-cvmx-includes.h" 49210311Sjmallett#include "ethernet-headers.h" 50210311Sjmallett 51210311Sjmallettextern int pow_receive_group; 52210311Sjmallettextern struct ifnet *cvm_oct_device[]; 53210311Sjmallett 54210311Sjmallettstatic struct task cvm_oct_task; 55210311Sjmallettstatic struct taskqueue *cvm_oct_taskq; 56210311Sjmallett 57217212Sjmallettstatic int cvm_oct_rx_active; 58217212Sjmallett 59210311Sjmallett/** 60210311Sjmallett * Interrupt handler. The interrupt occurs whenever the POW 61210311Sjmallett * transitions from 0->1 packets in our group. 62210311Sjmallett * 63210311Sjmallett * @param cpl 64210311Sjmallett * @param dev_id 65210311Sjmallett * @param regs 66210311Sjmallett * @return 67210311Sjmallett */ 68210311Sjmallettint cvm_oct_do_interrupt(void *dev_id) 69210311Sjmallett{ 70210311Sjmallett /* Acknowledge the interrupt */ 71210311Sjmallett if (INTERRUPT_LIMIT) 72210311Sjmallett cvmx_write_csr(CVMX_POW_WQ_INT, 1<<pow_receive_group); 73210311Sjmallett else 74210311Sjmallett cvmx_write_csr(CVMX_POW_WQ_INT, 0x10001<<pow_receive_group); 75217212Sjmallett 76217212Sjmallett /* 77217212Sjmallett * Schedule task if there isn't one running. 78217212Sjmallett */ 79217212Sjmallett if (atomic_cmpset_int(&cvm_oct_rx_active, 0, 1)) 80217212Sjmallett taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task); 81217212Sjmallett 82210311Sjmallett return FILTER_HANDLED; 83210311Sjmallett} 84210311Sjmallett 85210311Sjmallett 86210311Sjmallett/** 87210311Sjmallett * This is called on receive errors, and determines if the packet 88210311Sjmallett * can be dropped early-on in cvm_oct_tasklet_rx(). 89210311Sjmallett * 90210311Sjmallett * @param work Work queue entry pointing to the packet. 91210311Sjmallett * @return Non-zero if the packet can be dropped, zero otherwise. 92210311Sjmallett */ 93210311Sjmallettstatic inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work) 94210311Sjmallett{ 95232812Sjmallett if ((work->word2.snoip.err_code == 10) && (work->word1.s.len <= 64)) { 96210311Sjmallett /* Ignore length errors on min size packets. Some equipment 97210311Sjmallett incorrectly pads packets to 64+4FCS instead of 60+4FCS. 98210311Sjmallett Note these packets still get counted as frame errors. */ 99210311Sjmallett } else 100210311Sjmallett if (USE_10MBPS_PREAMBLE_WORKAROUND && ((work->word2.snoip.err_code == 5) || (work->word2.snoip.err_code == 7))) { 101210311Sjmallett 102210311Sjmallett /* We received a packet with either an alignment error or a 103210311Sjmallett FCS error. This may be signalling that we are running 104210311Sjmallett 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK} off. If this is the 105210311Sjmallett case we need to parse the packet to determine if we can 106210311Sjmallett remove a non spec preamble and generate a correct packet */ 107232812Sjmallett int interface = cvmx_helper_get_interface_num(work->word1.cn38xx.ipprt); 108232812Sjmallett int index = cvmx_helper_get_interface_index_num(work->word1.cn38xx.ipprt); 109210311Sjmallett cvmx_gmxx_rxx_frm_ctl_t gmxx_rxx_frm_ctl; 110210311Sjmallett gmxx_rxx_frm_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface)); 111210311Sjmallett if (gmxx_rxx_frm_ctl.s.pre_chk == 0) { 112210311Sjmallett 113210311Sjmallett uint8_t *ptr = cvmx_phys_to_ptr(work->packet_ptr.s.addr); 114210311Sjmallett int i = 0; 115210311Sjmallett 116232812Sjmallett while (i < work->word1.s.len-1) { 117210311Sjmallett if (*ptr != 0x55) 118210311Sjmallett break; 119210311Sjmallett ptr++; 120210311Sjmallett i++; 121210311Sjmallett } 122210311Sjmallett 123210311Sjmallett if (*ptr == 0xd5) { 124210311Sjmallett /* 125232812Sjmallett DEBUGPRINT("Port %d received 0xd5 preamble\n", work->word1.cn38xx.ipprt); 126210311Sjmallett */ 127210311Sjmallett work->packet_ptr.s.addr += i+1; 128232812Sjmallett work->word1.s.len -= i+5; 129210311Sjmallett } else 130210311Sjmallett if ((*ptr & 0xf) == 0xd) { 131210311Sjmallett /* 132232812Sjmallett DEBUGPRINT("Port %d received 0x?d preamble\n", work->word1.cn38xx.ipprt); 133210311Sjmallett */ 134210311Sjmallett work->packet_ptr.s.addr += i; 135232812Sjmallett work->word1.s.len -= i+4; 136232812Sjmallett for (i = 0; i < work->word1.s.len; i++) { 137210311Sjmallett *ptr = ((*ptr&0xf0)>>4) | ((*(ptr+1)&0xf)<<4); 138210311Sjmallett ptr++; 139210311Sjmallett } 140210311Sjmallett } else { 141232812Sjmallett DEBUGPRINT("Port %d unknown preamble, packet dropped\n", work->word1.cn38xx.ipprt); 142210311Sjmallett /* 143210311Sjmallett cvmx_helper_dump_packet(work); 144210311Sjmallett */ 145210311Sjmallett cvm_oct_free_work(work); 146210311Sjmallett return 1; 147210311Sjmallett } 148210311Sjmallett } 149210311Sjmallett } else { 150232812Sjmallett DEBUGPRINT("Port %d receive error code %d, packet dropped\n", work->word1.cn38xx.ipprt, work->word2.snoip.err_code); 151210311Sjmallett cvm_oct_free_work(work); 152210311Sjmallett return 1; 153210311Sjmallett } 154210311Sjmallett 155210311Sjmallett return 0; 156210311Sjmallett} 157210311Sjmallett 158210311Sjmallett/** 159210311Sjmallett * Tasklet function that is scheduled on a core when an interrupt occurs. 160210311Sjmallett * 161210311Sjmallett * @param unused 162210311Sjmallett */ 163210311Sjmallettvoid cvm_oct_tasklet_rx(void *context, int pending) 164210311Sjmallett{ 165213156Sjmallett int coreid; 166210311Sjmallett uint64_t old_group_mask; 167210311Sjmallett int rx_count = 0; 168210311Sjmallett int number_to_free; 169210311Sjmallett int num_freed; 170210311Sjmallett int packet_not_copied; 171210311Sjmallett 172213156Sjmallett sched_pin(); 173213156Sjmallett coreid = cvmx_get_core_num(); 174213156Sjmallett 175210311Sjmallett /* Prefetch cvm_oct_device since we know we need it soon */ 176210311Sjmallett CVMX_PREFETCH(cvm_oct_device, 0); 177210311Sjmallett 178210311Sjmallett /* Only allow work for our group (and preserve priorities) */ 179210311Sjmallett old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid)); 180210311Sjmallett cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), 181210311Sjmallett (old_group_mask & ~0xFFFFull) | 1<<pow_receive_group); 182210311Sjmallett 183210311Sjmallett while (1) { 184210311Sjmallett struct mbuf *m = NULL; 185210311Sjmallett int mbuf_in_hw; 186210311Sjmallett cvmx_wqe_t *work; 187210311Sjmallett 188217664Sjmallett if ((INTERRUPT_LIMIT == 0) || (rx_count < MAX_RX_PACKETS)) 189217664Sjmallett work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT); 190217664Sjmallett else 191217664Sjmallett work = NULL; 192210311Sjmallett CVMX_PREFETCH(work, 0); 193210311Sjmallett if (work == NULL) 194210311Sjmallett break; 195210311Sjmallett 196215974Sjmallett mbuf_in_hw = work->word2.s.bufs == 1; 197210311Sjmallett if ((mbuf_in_hw)) { 198210311Sjmallett m = *(struct mbuf **)(cvm_oct_get_buffer_ptr(work->packet_ptr) - sizeof(void *)); 199210311Sjmallett CVMX_PREFETCH(m, offsetof(struct mbuf, m_data)); 200210311Sjmallett CVMX_PREFETCH(m, offsetof(struct mbuf, m_pkthdr)); 201210311Sjmallett } 202232812Sjmallett CVMX_PREFETCH(cvm_oct_device[work->word1.cn38xx.ipprt], 0); 203210311Sjmallett //CVMX_PREFETCH(m, 0); 204210311Sjmallett 205210311Sjmallett 206210311Sjmallett rx_count++; 207210311Sjmallett /* Immediately throw away all packets with receive errors */ 208210311Sjmallett if ((work->word2.snoip.rcv_error)) { 209210311Sjmallett if (cvm_oct_check_rcv_error(work)) 210210311Sjmallett continue; 211210311Sjmallett } 212210311Sjmallett 213210311Sjmallett /* We can only use the zero copy path if mbufs are in the FPA pool 214210311Sjmallett and the packet fits in a single buffer */ 215210311Sjmallett if ((mbuf_in_hw)) { 216210311Sjmallett CVMX_PREFETCH(m->m_data, 0); 217210311Sjmallett 218232812Sjmallett m->m_pkthdr.len = m->m_len = work->word1.s.len; 219210311Sjmallett 220210311Sjmallett packet_not_copied = 1; 221210311Sjmallett 222210311Sjmallett /* 223210311Sjmallett * Adjust the data pointer based on the offset 224210311Sjmallett * of the packet within the buffer. 225210311Sjmallett */ 226210311Sjmallett m->m_data += (work->packet_ptr.s.back << 7) + (work->packet_ptr.s.addr & 0x7f); 227210311Sjmallett } else { 228210311Sjmallett 229210311Sjmallett /* We have to copy the packet. First allocate an 230210311Sjmallett mbuf for it */ 231243882Sglebius MGETHDR(m, M_NOWAIT, MT_DATA); 232210311Sjmallett if (m == NULL) { 233232812Sjmallett DEBUGPRINT("Port %d failed to allocate mbuf, packet dropped\n", work->word1.cn38xx.ipprt); 234210311Sjmallett cvm_oct_free_work(work); 235210311Sjmallett continue; 236210311Sjmallett } 237210311Sjmallett 238210311Sjmallett /* Check if we've received a packet that was entirely 239210311Sjmallett stored in the work entry. This is untested */ 240210311Sjmallett if ((work->word2.s.bufs == 0)) { 241210311Sjmallett uint8_t *ptr = work->packet_data; 242210311Sjmallett 243210311Sjmallett if (cvmx_likely(!work->word2.s.not_IP)) { 244210311Sjmallett /* The beginning of the packet moves 245210311Sjmallett for IP packets */ 246210311Sjmallett if (work->word2.s.is_v6) 247210311Sjmallett ptr += 2; 248210311Sjmallett else 249210311Sjmallett ptr += 6; 250210311Sjmallett } 251210311Sjmallett panic("%s: not yet implemented; copy in small packet.", __func__); 252210311Sjmallett /* No packet buffers to free */ 253210311Sjmallett } else { 254210311Sjmallett int segments = work->word2.s.bufs; 255210311Sjmallett cvmx_buf_ptr_t segment_ptr = work->packet_ptr; 256232812Sjmallett int len = work->word1.s.len; 257210311Sjmallett 258210311Sjmallett while (segments--) { 259210311Sjmallett cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8); 260210311Sjmallett /* Octeon Errata PKI-100: The segment 261210311Sjmallett size is wrong. Until it is fixed, 262210311Sjmallett calculate the segment size based on 263210311Sjmallett the packet pool buffer size. When 264210311Sjmallett it is fixed, the following line 265210311Sjmallett should be replaced with this one: 266210311Sjmallett int segment_size = segment_ptr.s.size; */ 267210311Sjmallett int segment_size = CVMX_FPA_PACKET_POOL_SIZE - (segment_ptr.s.addr - (((segment_ptr.s.addr >> 7) - segment_ptr.s.back) << 7)); 268210311Sjmallett /* Don't copy more than what is left 269210311Sjmallett in the packet */ 270210311Sjmallett if (segment_size > len) 271210311Sjmallett segment_size = len; 272210311Sjmallett /* Copy the data into the packet */ 273210311Sjmallett panic("%s: not yet implemented; copy in packet segments.", __func__); 274210311Sjmallett#if 0 275210311Sjmallett memcpy(m_put(m, segment_size), cvmx_phys_to_ptr(segment_ptr.s.addr), segment_size); 276210311Sjmallett#endif 277210311Sjmallett /* Reduce the amount of bytes left 278210311Sjmallett to copy */ 279210311Sjmallett len -= segment_size; 280210311Sjmallett segment_ptr = next_ptr; 281210311Sjmallett } 282210311Sjmallett } 283210311Sjmallett packet_not_copied = 0; 284210311Sjmallett } 285210311Sjmallett 286232812Sjmallett if (((work->word1.cn38xx.ipprt < TOTAL_NUMBER_OF_PORTS) && 287232812Sjmallett cvm_oct_device[work->word1.cn38xx.ipprt])) { 288232812Sjmallett struct ifnet *ifp = cvm_oct_device[work->word1.cn38xx.ipprt]; 289210311Sjmallett 290210311Sjmallett /* Only accept packets for devices 291210311Sjmallett that are currently up */ 292210311Sjmallett if ((ifp->if_flags & IFF_UP)) { 293210311Sjmallett m->m_pkthdr.rcvif = ifp; 294210311Sjmallett 295210311Sjmallett if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 296210311Sjmallett if ((work->word2.s.not_IP || work->word2.s.IP_exc || work->word2.s.L4_error)) 297210311Sjmallett m->m_pkthdr.csum_flags = 0; /* XXX */ 298210311Sjmallett else { 299210311Sjmallett m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 300210311Sjmallett m->m_pkthdr.csum_data = 0xffff; 301210311Sjmallett } 302210311Sjmallett } else { 303210311Sjmallett m->m_pkthdr.csum_flags = 0; /* XXX */ 304210311Sjmallett } 305210311Sjmallett 306210311Sjmallett ifp->if_ipackets++; 307210311Sjmallett 308210311Sjmallett (*ifp->if_input)(ifp, m); 309210311Sjmallett } else { 310210311Sjmallett /* Drop any packet received for a device that isn't up */ 311210311Sjmallett /* 312210311Sjmallett DEBUGPRINT("%s: Device not up, packet dropped\n", 313210311Sjmallett if_name(ifp)); 314210311Sjmallett */ 315210311Sjmallett m_freem(m); 316210311Sjmallett } 317210311Sjmallett } else { 318210311Sjmallett /* Drop any packet received for a device that 319210311Sjmallett doesn't exist */ 320242390Sjmallett DEBUGPRINT("Port %d not controlled by FreeBSD, packet dropped\n", work->word1.cn38xx.ipprt); 321210311Sjmallett m_freem(m); 322210311Sjmallett } 323210311Sjmallett 324210311Sjmallett /* Check to see if the mbuf and work share 325210311Sjmallett the same packet buffer */ 326215974Sjmallett if ((packet_not_copied)) { 327210311Sjmallett /* This buffer needs to be replaced, increment 328210311Sjmallett the number of buffers we need to free by one */ 329210311Sjmallett cvmx_fau_atomic_add32( 330210311Sjmallett FAU_NUM_PACKET_BUFFERS_TO_FREE, 1); 331210311Sjmallett 332210311Sjmallett cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 333210311Sjmallett DONT_WRITEBACK(1)); 334210311Sjmallett } else 335210311Sjmallett cvm_oct_free_work(work); 336210311Sjmallett } 337210311Sjmallett 338217212Sjmallett /* 339217212Sjmallett * If we hit our limit, schedule another task while we clean up. 340217212Sjmallett */ 341217212Sjmallett if (INTERRUPT_LIMIT != 0 && rx_count == MAX_RX_PACKETS) { 342217212Sjmallett taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task); 343217212Sjmallett } else { 344217212Sjmallett /* 345217212Sjmallett * No more packets, all done. 346217212Sjmallett */ 347217212Sjmallett if (!atomic_cmpset_int(&cvm_oct_rx_active, 1, 0)) 348217212Sjmallett panic("%s: inconsistent rx active state.", __func__); 349217212Sjmallett } 350217212Sjmallett 351210311Sjmallett /* Restore the original POW group mask */ 352210311Sjmallett cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask); 353210311Sjmallett 354215974Sjmallett /* Refill the packet buffer pool */ 355215974Sjmallett number_to_free = 356215974Sjmallett cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); 357210311Sjmallett 358215974Sjmallett if (number_to_free > 0) { 359215974Sjmallett cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 360215974Sjmallett -number_to_free); 361215974Sjmallett num_freed = 362215974Sjmallett cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, 363215974Sjmallett CVMX_FPA_PACKET_POOL_SIZE, 364215974Sjmallett number_to_free); 365215974Sjmallett if (num_freed != number_to_free) { 366210311Sjmallett cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 367215974Sjmallett number_to_free - num_freed); 368210311Sjmallett } 369210311Sjmallett } 370213156Sjmallett sched_unpin(); 371210311Sjmallett} 372210311Sjmallett 373210311Sjmallett 374210311Sjmallett 375210311Sjmallettvoid cvm_oct_rx_initialize(void) 376210311Sjmallett{ 377210311Sjmallett TASK_INIT(&cvm_oct_task, 0, cvm_oct_tasklet_rx, NULL); 378210311Sjmallett 379210311Sjmallett cvm_oct_taskq = taskqueue_create_fast("oct_rx", M_NOWAIT, 380210311Sjmallett taskqueue_thread_enqueue, 381210311Sjmallett &cvm_oct_taskq); 382210311Sjmallett taskqueue_start_threads(&cvm_oct_taskq, min(mp_ncpus, MAXCPU), 383210311Sjmallett PI_NET, "octe taskq"); 384210311Sjmallett} 385210311Sjmallett 386210311Sjmallettvoid cvm_oct_rx_shutdown(void) 387210311Sjmallett{ 388210311Sjmallett panic("%s: not yet implemented.", __func__); 389210311Sjmallett} 390210311Sjmallett 391