1/*************************************************************************
2Copyright (c) 2003-2007  Cavium Networks (support@cavium.com). All rights
3reserved.
4
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are
8met:
9
10    * Redistributions of source code must retain the above copyright
11      notice, this list of conditions and the following disclaimer.
12
13    * Redistributions in binary form must reproduce the above
14      copyright notice, this list of conditions and the following
15      disclaimer in the documentation and/or other materials provided
16      with the distribution.
17
18    * Neither the name of Cavium Networks nor the names of
19      its contributors may be used to endorse or promote products
20      derived from this software without specific prior written
21      permission.
22
23This Software, including technical data, may be subject to U.S. export  control laws, including the U.S. Export Administration Act and its  associated regulations, and may be subject to export or import  regulations in other countries.
24
25TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
26AND WITH ALL FAULTS AND CAVIUM  NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
27
28*************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/bus.h>
36#include <sys/endian.h>
37#include <sys/kernel.h>
38#include <sys/mbuf.h>
39#include <sys/socket.h>
40#include <sys/proc.h>
41#include <sys/sched.h>
42#include <sys/smp.h>
43#include <sys/taskqueue.h>
44
45#include <net/ethernet.h>
46#include <net/if.h>
47#include <net/if_var.h>
48
49#include "wrapper-cvmx-includes.h"
50#include "ethernet-headers.h"
51
52extern int pow_receive_group;
53extern struct ifnet *cvm_oct_device[];
54
55static struct task cvm_oct_task;
56static struct taskqueue *cvm_oct_taskq;
57
58static int cvm_oct_rx_active;
59
60/**
61 * Interrupt handler. The interrupt occurs whenever the POW
62 * transitions from 0->1 packets in our group.
63 *
64 * @param cpl
65 * @param dev_id
66 * @param regs
67 * @return
68 */
69int cvm_oct_do_interrupt(void *dev_id)
70{
71	/* Acknowledge the interrupt */
72	if (INTERRUPT_LIMIT)
73		cvmx_write_csr(CVMX_POW_WQ_INT, 1<<pow_receive_group);
74	else
75		cvmx_write_csr(CVMX_POW_WQ_INT, 0x10001<<pow_receive_group);
76
77	/*
78	 * Schedule task if there isn't one running.
79	 */
80	if (atomic_cmpset_int(&cvm_oct_rx_active, 0, 1))
81		taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task);
82
83	return FILTER_HANDLED;
84}
85
86
87/**
88 * This is called on receive errors, and determines if the packet
89 * can be dropped early-on in cvm_oct_tasklet_rx().
90 *
91 * @param work Work queue entry pointing to the packet.
92 * @return Non-zero if the packet can be dropped, zero otherwise.
93 */
94static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work)
95{
96	if ((work->word2.snoip.err_code == 10) && (work->word1.s.len <= 64)) {
97		/* Ignore length errors on min size packets. Some equipment
98		   incorrectly pads packets to 64+4FCS instead of 60+4FCS.
99		   Note these packets still get counted as frame errors. */
100	} else
101	if (USE_10MBPS_PREAMBLE_WORKAROUND && ((work->word2.snoip.err_code == 5) || (work->word2.snoip.err_code == 7))) {
102
103		/* We received a packet with either an alignment error or a
104		   FCS error. This may be signalling that we are running
105		   10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK} off. If this is the
106		   case we need to parse the packet to determine if we can
107		   remove a non spec preamble and generate a correct packet */
108		int interface = cvmx_helper_get_interface_num(work->word1.cn38xx.ipprt);
109		int index = cvmx_helper_get_interface_index_num(work->word1.cn38xx.ipprt);
110		cvmx_gmxx_rxx_frm_ctl_t gmxx_rxx_frm_ctl;
111		gmxx_rxx_frm_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface));
112		if (gmxx_rxx_frm_ctl.s.pre_chk == 0) {
113
114			uint8_t *ptr = cvmx_phys_to_ptr(work->packet_ptr.s.addr);
115			int i = 0;
116
117			while (i < work->word1.s.len-1) {
118				if (*ptr != 0x55)
119					break;
120				ptr++;
121				i++;
122			}
123
124			if (*ptr == 0xd5) {
125				/*
126				DEBUGPRINT("Port %d received 0xd5 preamble\n", work->word1.cn38xx.ipprt);
127				*/
128				work->packet_ptr.s.addr += i+1;
129				work->word1.s.len -= i+5;
130			} else
131			if ((*ptr & 0xf) == 0xd) {
132				/*
133				DEBUGPRINT("Port %d received 0x?d preamble\n", work->word1.cn38xx.ipprt);
134				*/
135				work->packet_ptr.s.addr += i;
136				work->word1.s.len -= i+4;
137				for (i = 0; i < work->word1.s.len; i++) {
138					*ptr = ((*ptr&0xf0)>>4) | ((*(ptr+1)&0xf)<<4);
139					ptr++;
140				}
141			} else {
142				DEBUGPRINT("Port %d unknown preamble, packet dropped\n", work->word1.cn38xx.ipprt);
143				/*
144				cvmx_helper_dump_packet(work);
145				*/
146				cvm_oct_free_work(work);
147				return 1;
148			}
149		}
150	} else {
151		DEBUGPRINT("Port %d receive error code %d, packet dropped\n", work->word1.cn38xx.ipprt, work->word2.snoip.err_code);
152		cvm_oct_free_work(work);
153		return 1;
154	}
155
156	return 0;
157}
158
159/**
160 * Tasklet function that is scheduled on a core when an interrupt occurs.
161 *
162 * @param unused
163 */
164void cvm_oct_tasklet_rx(void *context, int pending)
165{
166	int                 coreid;
167	uint64_t            old_group_mask;
168	int                 rx_count = 0;
169	int                 number_to_free;
170	int                 num_freed;
171	int                 packet_not_copied;
172
173	sched_pin();
174	coreid = cvmx_get_core_num();
175
176	/* Prefetch cvm_oct_device since we know we need it soon */
177	CVMX_PREFETCH(cvm_oct_device, 0);
178
179	/* Only allow work for our group (and preserve priorities) */
180	old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid));
181	cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid),
182		       (old_group_mask & ~0xFFFFull) | 1<<pow_receive_group);
183
184	while (1) {
185		struct mbuf *m = NULL;
186		int mbuf_in_hw;
187		cvmx_wqe_t *work;
188
189		if ((INTERRUPT_LIMIT == 0) || (rx_count < MAX_RX_PACKETS))
190			work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT);
191		else
192			work = NULL;
193		CVMX_PREFETCH(work, 0);
194		if (work == NULL)
195			break;
196
197		mbuf_in_hw = work->word2.s.bufs == 1;
198		if ((mbuf_in_hw)) {
199			m = *(struct mbuf **)(cvm_oct_get_buffer_ptr(work->packet_ptr) - sizeof(void *));
200			CVMX_PREFETCH(m, offsetof(struct mbuf, m_data));
201			CVMX_PREFETCH(m, offsetof(struct mbuf, m_pkthdr));
202		}
203		CVMX_PREFETCH(cvm_oct_device[work->word1.cn38xx.ipprt], 0);
204		//CVMX_PREFETCH(m, 0);
205
206
207		rx_count++;
208		/* Immediately throw away all packets with receive errors */
209		if ((work->word2.snoip.rcv_error)) {
210			if (cvm_oct_check_rcv_error(work))
211				continue;
212		}
213
214		/* We can only use the zero copy path if mbufs are in the FPA pool
215		   and the packet fits in a single buffer */
216		if ((mbuf_in_hw)) {
217			CVMX_PREFETCH(m->m_data, 0);
218
219			m->m_pkthdr.len = m->m_len = work->word1.s.len;
220
221			packet_not_copied = 1;
222
223			/*
224			 * Adjust the data pointer based on the offset
225			 * of the packet within the buffer.
226			 */
227			m->m_data += (work->packet_ptr.s.back << 7) + (work->packet_ptr.s.addr & 0x7f);
228		} else {
229
230			/* We have to copy the packet. First allocate an
231			   mbuf for it */
232			MGETHDR(m, M_NOWAIT, MT_DATA);
233			if (m == NULL) {
234				DEBUGPRINT("Port %d failed to allocate mbuf, packet dropped\n", work->word1.cn38xx.ipprt);
235				cvm_oct_free_work(work);
236				continue;
237			}
238
239			/* Check if we've received a packet that was entirely
240			   stored in the work entry. This is untested */
241			if ((work->word2.s.bufs == 0)) {
242				uint8_t *ptr = work->packet_data;
243
244				if (cvmx_likely(!work->word2.s.not_IP)) {
245					/* The beginning of the packet moves
246					   for IP packets */
247					if (work->word2.s.is_v6)
248						ptr += 2;
249					else
250						ptr += 6;
251				}
252				panic("%s: not yet implemented; copy in small packet.", __func__);
253				/* No packet buffers to free */
254			} else {
255				int segments = work->word2.s.bufs;
256				cvmx_buf_ptr_t segment_ptr = work->packet_ptr;
257				int len = work->word1.s.len;
258
259				while (segments--) {
260					cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8);
261					/* Octeon Errata PKI-100: The segment
262					   size is wrong. Until it is fixed,
263					   calculate the segment size based on
264					   the packet pool buffer size. When
265					   it is fixed, the following line
266					   should be replaced with this one:
267					int segment_size = segment_ptr.s.size; */
268					int segment_size = CVMX_FPA_PACKET_POOL_SIZE - (segment_ptr.s.addr - (((segment_ptr.s.addr >> 7) - segment_ptr.s.back) << 7));
269					/* Don't copy more than what is left
270					   in the packet */
271					if (segment_size > len)
272						segment_size = len;
273					/* Copy the data into the packet */
274					panic("%s: not yet implemented; copy in packet segments.", __func__);
275#if 0
276					memcpy(m_put(m, segment_size), cvmx_phys_to_ptr(segment_ptr.s.addr), segment_size);
277#endif
278					/* Reduce the amount of bytes left
279					   to copy */
280					len -= segment_size;
281					segment_ptr = next_ptr;
282				}
283			}
284			packet_not_copied = 0;
285		}
286
287		if (((work->word1.cn38xx.ipprt < TOTAL_NUMBER_OF_PORTS) &&
288		    cvm_oct_device[work->word1.cn38xx.ipprt])) {
289			struct ifnet *ifp = cvm_oct_device[work->word1.cn38xx.ipprt];
290
291			/* Only accept packets for devices
292			   that are currently up */
293			if ((ifp->if_flags & IFF_UP)) {
294				m->m_pkthdr.rcvif = ifp;
295
296				if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
297					if ((work->word2.s.not_IP || work->word2.s.IP_exc || work->word2.s.L4_error))
298						m->m_pkthdr.csum_flags = 0; /* XXX */
299					else {
300						m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
301						m->m_pkthdr.csum_data = 0xffff;
302					}
303				} else {
304					m->m_pkthdr.csum_flags = 0; /* XXX */
305				}
306
307				if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
308
309				(*ifp->if_input)(ifp, m);
310			} else {
311				/* Drop any packet received for a device that isn't up */
312				/*
313				DEBUGPRINT("%s: Device not up, packet dropped\n",
314					   if_name(ifp));
315				*/
316				m_freem(m);
317			}
318		} else {
319			/* Drop any packet received for a device that
320			   doesn't exist */
321			DEBUGPRINT("Port %d not controlled by FreeBSD, packet dropped\n", work->word1.cn38xx.ipprt);
322			m_freem(m);
323		}
324
325		/* Check to see if the mbuf and work share
326		   the same packet buffer */
327		if ((packet_not_copied)) {
328			/* This buffer needs to be replaced, increment
329			the number of buffers we need to free by one */
330			cvmx_fau_atomic_add32(
331				FAU_NUM_PACKET_BUFFERS_TO_FREE, 1);
332
333			cvmx_fpa_free(work, CVMX_FPA_WQE_POOL,
334				      DONT_WRITEBACK(1));
335		} else
336			cvm_oct_free_work(work);
337	}
338
339	/*
340	 * If we hit our limit, schedule another task while we clean up.
341	 */
342	if (INTERRUPT_LIMIT != 0 && rx_count == MAX_RX_PACKETS) {
343		taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task);
344	} else {
345		/*
346		 * No more packets, all done.
347		 */
348		if (!atomic_cmpset_int(&cvm_oct_rx_active, 1, 0))
349			panic("%s: inconsistent rx active state.", __func__);
350	}
351
352	/* Restore the original POW group mask */
353	cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask);
354
355	/* Refill the packet buffer pool */
356	number_to_free =
357	  cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
358
359	if (number_to_free > 0) {
360		cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
361				      -number_to_free);
362		num_freed =
363			cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL,
364					     CVMX_FPA_PACKET_POOL_SIZE,
365					     number_to_free);
366		if (num_freed != number_to_free) {
367			cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
368					      number_to_free - num_freed);
369		}
370	}
371	sched_unpin();
372}
373
374
375
376void cvm_oct_rx_initialize(void)
377{
378	TASK_INIT(&cvm_oct_task, 0, cvm_oct_tasklet_rx, NULL);
379
380	cvm_oct_taskq = taskqueue_create_fast("oct_rx", M_NOWAIT,
381					      taskqueue_thread_enqueue,
382					      &cvm_oct_taskq);
383	taskqueue_start_threads(&cvm_oct_taskq, min(mp_ncpus, MAXCPU),
384				PI_NET, "octe taskq");
385}
386
387void cvm_oct_rx_shutdown(void)
388{
389	panic("%s: not yet implemented.", __func__);
390}
391
392