1/*-
2 * Copyright (c) 2012-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 *    this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 *    this list of conditions and the following disclaimer in the documentation
12 *    and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: releng/11.0/sys/dev/sfxge/common/ef10_rx.c 300847 2016-05-27 11:49:05Z arybchik $");
33
34#include "efx.h"
35#include "efx_impl.h"
36
37
38#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
39
40
41static	__checkReturn	efx_rc_t
42efx_mcdi_init_rxq(
43	__in		efx_nic_t *enp,
44	__in		uint32_t size,
45	__in		uint32_t target_evq,
46	__in		uint32_t label,
47	__in		uint32_t instance,
48	__in		efsys_mem_t *esmp,
49	__in		boolean_t disable_scatter)
50{
51	efx_mcdi_req_t req;
52	uint8_t payload[
53	    MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS))];
54	int npages = EFX_RXQ_NBUFS(size);
55	int i;
56	efx_qword_t *dma_addr;
57	uint64_t addr;
58	efx_rc_t rc;
59
60	/* If this changes, then the payload size might need to change. */
61	EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0);
62	EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
63
64	(void) memset(payload, 0, sizeof (payload));
65	req.emr_cmd = MC_CMD_INIT_RXQ;
66	req.emr_in_buf = payload;
67	req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages);
68	req.emr_out_buf = payload;
69	req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN;
70
71	MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size);
72	MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq);
73	MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label);
74	MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance);
75	MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS,
76			    INIT_RXQ_IN_FLAG_BUFF_MODE, 0,
77			    INIT_RXQ_IN_FLAG_HDR_SPLIT, 0,
78			    INIT_RXQ_IN_FLAG_TIMESTAMP, 0,
79			    INIT_RXQ_IN_CRC_MODE, 0,
80			    INIT_RXQ_IN_FLAG_PREFIX, 1,
81			    INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter);
82	MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0);
83	MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
84
85	dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
86	addr = EFSYS_MEM_ADDR(esmp);
87
88	for (i = 0; i < npages; i++) {
89		EFX_POPULATE_QWORD_2(*dma_addr,
90		    EFX_DWORD_1, (uint32_t)(addr >> 32),
91		    EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
92
93		dma_addr++;
94		addr += EFX_BUF_SIZE;
95	}
96
97	efx_mcdi_execute(enp, &req);
98
99	if (req.emr_rc != 0) {
100		rc = req.emr_rc;
101		goto fail1;
102	}
103
104	return (0);
105
106fail1:
107	EFSYS_PROBE1(fail1, efx_rc_t, rc);
108
109	return (rc);
110}
111
112static	__checkReturn	efx_rc_t
113efx_mcdi_fini_rxq(
114	__in		efx_nic_t *enp,
115	__in		uint32_t instance)
116{
117	efx_mcdi_req_t req;
118	uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
119			    MC_CMD_FINI_RXQ_OUT_LEN)];
120	efx_rc_t rc;
121
122	(void) memset(payload, 0, sizeof (payload));
123	req.emr_cmd = MC_CMD_FINI_RXQ;
124	req.emr_in_buf = payload;
125	req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
126	req.emr_out_buf = payload;
127	req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
128
129	MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
130
131	efx_mcdi_execute_quiet(enp, &req);
132
133	if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) {
134		rc = req.emr_rc;
135		goto fail1;
136	}
137
138	return (0);
139
140fail1:
141	EFSYS_PROBE1(fail1, efx_rc_t, rc);
142
143	return (rc);
144}
145
146#if EFSYS_OPT_RX_SCALE
147static	__checkReturn	efx_rc_t
148efx_mcdi_rss_context_alloc(
149	__in		efx_nic_t *enp,
150	__in		efx_rx_scale_support_t scale_support,
151	__in		uint32_t num_queues,
152	__out		uint32_t *rss_contextp)
153{
154	efx_mcdi_req_t req;
155	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
156			    MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
157	uint32_t rss_context;
158	uint32_t context_type;
159	efx_rc_t rc;
160
161	if (num_queues > EFX_MAXRSS) {
162		rc = EINVAL;
163		goto fail1;
164	}
165
166	switch (scale_support) {
167	case EFX_RX_SCALE_EXCLUSIVE:
168		context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
169		break;
170	case EFX_RX_SCALE_SHARED:
171		context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
172		break;
173	default:
174		rc = EINVAL;
175		goto fail2;
176	}
177
178	(void) memset(payload, 0, sizeof (payload));
179	req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
180	req.emr_in_buf = payload;
181	req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
182	req.emr_out_buf = payload;
183	req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
184
185	MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
186	    EVB_PORT_ID_ASSIGNED);
187	MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
188	/* NUM_QUEUES is only used to validate indirection table offsets */
189	MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
190
191	efx_mcdi_execute(enp, &req);
192
193	if (req.emr_rc != 0) {
194		rc = req.emr_rc;
195		goto fail3;
196	}
197
198	if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
199		rc = EMSGSIZE;
200		goto fail4;
201	}
202
203	rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
204	if (rss_context == EF10_RSS_CONTEXT_INVALID) {
205		rc = ENOENT;
206		goto fail5;
207	}
208
209	*rss_contextp = rss_context;
210
211	return (0);
212
213fail5:
214	EFSYS_PROBE(fail5);
215fail4:
216	EFSYS_PROBE(fail4);
217fail3:
218	EFSYS_PROBE(fail3);
219fail2:
220	EFSYS_PROBE(fail2);
221fail1:
222	EFSYS_PROBE1(fail1, efx_rc_t, rc);
223
224	return (rc);
225}
226#endif /* EFSYS_OPT_RX_SCALE */
227
228#if EFSYS_OPT_RX_SCALE
229static			efx_rc_t
230efx_mcdi_rss_context_free(
231	__in		efx_nic_t *enp,
232	__in		uint32_t rss_context)
233{
234	efx_mcdi_req_t req;
235	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
236			    MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
237	efx_rc_t rc;
238
239	if (rss_context == EF10_RSS_CONTEXT_INVALID) {
240		rc = EINVAL;
241		goto fail1;
242	}
243
244	(void) memset(payload, 0, sizeof (payload));
245	req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
246	req.emr_in_buf = payload;
247	req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
248	req.emr_out_buf = payload;
249	req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
250
251	MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
252
253	efx_mcdi_execute_quiet(enp, &req);
254
255	if (req.emr_rc != 0) {
256		rc = req.emr_rc;
257		goto fail2;
258	}
259
260	return (0);
261
262fail2:
263	EFSYS_PROBE(fail2);
264fail1:
265	EFSYS_PROBE1(fail1, efx_rc_t, rc);
266
267	return (rc);
268}
269#endif /* EFSYS_OPT_RX_SCALE */
270
271#if EFSYS_OPT_RX_SCALE
272static			efx_rc_t
273efx_mcdi_rss_context_set_flags(
274	__in		efx_nic_t *enp,
275	__in		uint32_t rss_context,
276	__in		efx_rx_hash_type_t type)
277{
278	efx_mcdi_req_t req;
279	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
280			    MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
281	efx_rc_t rc;
282
283	if (rss_context == EF10_RSS_CONTEXT_INVALID) {
284		rc = EINVAL;
285		goto fail1;
286	}
287
288	(void) memset(payload, 0, sizeof (payload));
289	req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
290	req.emr_in_buf = payload;
291	req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
292	req.emr_out_buf = payload;
293	req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
294
295	MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
296	    rss_context);
297
298	MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
299	    RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
300	    (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0,
301	    RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
302	    (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0,
303	    RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
304	    (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0,
305	    RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
306	    (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0);
307
308	efx_mcdi_execute(enp, &req);
309
310	if (req.emr_rc != 0) {
311		rc = req.emr_rc;
312		goto fail2;
313	}
314
315	return (0);
316
317fail2:
318	EFSYS_PROBE(fail2);
319fail1:
320	EFSYS_PROBE1(fail1, efx_rc_t, rc);
321
322	return (rc);
323}
324#endif /* EFSYS_OPT_RX_SCALE */
325
326#if EFSYS_OPT_RX_SCALE
327static			efx_rc_t
328efx_mcdi_rss_context_set_key(
329	__in		efx_nic_t *enp,
330	__in		uint32_t rss_context,
331	__in_ecount(n)	uint8_t *key,
332	__in		size_t n)
333{
334	efx_mcdi_req_t req;
335	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
336			    MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
337	efx_rc_t rc;
338
339	if (rss_context == EF10_RSS_CONTEXT_INVALID) {
340		rc = EINVAL;
341		goto fail1;
342	}
343
344	(void) memset(payload, 0, sizeof (payload));
345	req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
346	req.emr_in_buf = payload;
347	req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
348	req.emr_out_buf = payload;
349	req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
350
351	MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
352	    rss_context);
353
354	EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
355	if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
356		rc = EINVAL;
357		goto fail2;
358	}
359
360	memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
361	    key, n);
362
363	efx_mcdi_execute(enp, &req);
364
365	if (req.emr_rc != 0) {
366		rc = req.emr_rc;
367		goto fail3;
368	}
369
370	return (0);
371
372fail3:
373	EFSYS_PROBE(fail3);
374fail2:
375	EFSYS_PROBE(fail2);
376fail1:
377	EFSYS_PROBE1(fail1, efx_rc_t, rc);
378
379	return (rc);
380}
381#endif /* EFSYS_OPT_RX_SCALE */
382
383#if EFSYS_OPT_RX_SCALE
384static			efx_rc_t
385efx_mcdi_rss_context_set_table(
386	__in		efx_nic_t *enp,
387	__in		uint32_t rss_context,
388	__in_ecount(n)	unsigned int *table,
389	__in		size_t n)
390{
391	efx_mcdi_req_t req;
392	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
393			    MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
394	uint8_t *req_table;
395	int i, rc;
396
397	if (rss_context == EF10_RSS_CONTEXT_INVALID) {
398		rc = EINVAL;
399		goto fail1;
400	}
401
402	(void) memset(payload, 0, sizeof (payload));
403	req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
404	req.emr_in_buf = payload;
405	req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
406	req.emr_out_buf = payload;
407	req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
408
409	MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
410	    rss_context);
411
412	req_table =
413	    MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
414
415	for (i = 0;
416	    i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
417	    i++) {
418		req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
419	}
420
421	efx_mcdi_execute(enp, &req);
422
423	if (req.emr_rc != 0) {
424		rc = req.emr_rc;
425		goto fail2;
426	}
427
428	return (0);
429
430fail2:
431	EFSYS_PROBE(fail2);
432fail1:
433	EFSYS_PROBE1(fail1, efx_rc_t, rc);
434
435	return (rc);
436}
437#endif /* EFSYS_OPT_RX_SCALE */
438
439
440	__checkReturn	efx_rc_t
441ef10_rx_init(
442	__in		efx_nic_t *enp)
443{
444#if EFSYS_OPT_RX_SCALE
445
446	if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
447		&enp->en_rss_context) == 0) {
448		/*
449		 * Allocated an exclusive RSS context, which allows both the
450		 * indirection table and key to be modified.
451		 */
452		enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
453		enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
454	} else {
455		/*
456		 * Failed to allocate an exclusive RSS context. Continue
457		 * operation without support for RSS. The pseudo-header in
458		 * received packets will not contain a Toeplitz hash value.
459		 */
460		enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
461		enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
462	}
463
464#endif /* EFSYS_OPT_RX_SCALE */
465
466	return (0);
467}
468
469#if EFSYS_OPT_RX_SCATTER
470	__checkReturn	efx_rc_t
471ef10_rx_scatter_enable(
472	__in		efx_nic_t *enp,
473	__in		unsigned int buf_size)
474{
475	_NOTE(ARGUNUSED(enp, buf_size))
476	return (0);
477}
478#endif	/* EFSYS_OPT_RX_SCATTER */
479
480#if EFSYS_OPT_RX_SCALE
481	__checkReturn	efx_rc_t
482ef10_rx_scale_mode_set(
483	__in		efx_nic_t *enp,
484	__in		efx_rx_hash_alg_t alg,
485	__in		efx_rx_hash_type_t type,
486	__in		boolean_t insert)
487{
488	efx_rc_t rc;
489
490	EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
491	EFSYS_ASSERT3U(insert, ==, B_TRUE);
492
493	if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
494		rc = EINVAL;
495		goto fail1;
496	}
497
498	if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
499		rc = ENOTSUP;
500		goto fail2;
501	}
502
503	if ((rc = efx_mcdi_rss_context_set_flags(enp,
504		    enp->en_rss_context, type)) != 0)
505		goto fail3;
506
507	return (0);
508
509fail3:
510	EFSYS_PROBE(fail3);
511fail2:
512	EFSYS_PROBE(fail2);
513fail1:
514	EFSYS_PROBE1(fail1, efx_rc_t, rc);
515
516	return (rc);
517}
518#endif /* EFSYS_OPT_RX_SCALE */
519
520#if EFSYS_OPT_RX_SCALE
521	__checkReturn	efx_rc_t
522ef10_rx_scale_key_set(
523	__in		efx_nic_t *enp,
524	__in_ecount(n)	uint8_t *key,
525	__in		size_t n)
526{
527	efx_rc_t rc;
528
529	if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
530		rc = ENOTSUP;
531		goto fail1;
532	}
533
534	if ((rc = efx_mcdi_rss_context_set_key(enp,
535	    enp->en_rss_context, key, n)) != 0)
536		goto fail2;
537
538	return (0);
539
540fail2:
541	EFSYS_PROBE(fail2);
542fail1:
543	EFSYS_PROBE1(fail1, efx_rc_t, rc);
544
545	return (rc);
546}
547#endif /* EFSYS_OPT_RX_SCALE */
548
549#if EFSYS_OPT_RX_SCALE
550	__checkReturn	efx_rc_t
551ef10_rx_scale_tbl_set(
552	__in		efx_nic_t *enp,
553	__in_ecount(n)	unsigned int *table,
554	__in		size_t n)
555{
556	efx_rc_t rc;
557
558	if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
559		rc = ENOTSUP;
560		goto fail1;
561	}
562
563	if ((rc = efx_mcdi_rss_context_set_table(enp,
564	    enp->en_rss_context, table, n)) != 0)
565		goto fail2;
566
567	return (0);
568
569fail2:
570	EFSYS_PROBE(fail2);
571fail1:
572	EFSYS_PROBE1(fail1, efx_rc_t, rc);
573
574	return (rc);
575}
576#endif /* EFSYS_OPT_RX_SCALE */
577
578
579/*
580 * EF10 RX pseudo-header
581 * ---------------------
582 *
583 * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
584 *
585 *  +00: Toeplitz hash value.
586 *       (32bit little-endian)
587 *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
588 *       (16bit big-endian)
589 *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
590 *       (16bit big-endian)
591 *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
592 *       (16bit little-endian)
593 *  +10: MAC timestamp. Zero if timestamping is not enabled.
594 *       (32bit little-endian)
595 *
596 * See "The RX Pseudo-header" in SF-109306-TC.
597 */
598
599	__checkReturn	efx_rc_t
600ef10_rx_prefix_pktlen(
601	__in		efx_nic_t *enp,
602	__in		uint8_t *buffer,
603	__out		uint16_t *lengthp)
604{
605	_NOTE(ARGUNUSED(enp))
606
607	/*
608	 * The RX pseudo-header contains the packet length, excluding the
609	 * pseudo-header. If the hardware receive datapath was operating in
610	 * cut-through mode then the length in the RX pseudo-header will be
611	 * zero, and the packet length must be obtained from the DMA length
612	 * reported in the RX event.
613	 */
614	*lengthp = buffer[8] | (buffer[9] << 8);
615	return (0);
616}
617
618#if EFSYS_OPT_RX_SCALE
619	__checkReturn	uint32_t
620ef10_rx_prefix_hash(
621	__in		efx_nic_t *enp,
622	__in		efx_rx_hash_alg_t func,
623	__in		uint8_t *buffer)
624{
625	_NOTE(ARGUNUSED(enp))
626
627	switch (func) {
628	case EFX_RX_HASHALG_TOEPLITZ:
629		return (buffer[0] |
630		    (buffer[1] << 8) |
631		    (buffer[2] << 16) |
632		    (buffer[3] << 24));
633
634	default:
635		EFSYS_ASSERT(0);
636		return (0);
637	}
638}
639#endif /* EFSYS_OPT_RX_SCALE */
640
641			void
642ef10_rx_qpost(
643	__in		efx_rxq_t *erp,
644	__in_ecount(n)	efsys_dma_addr_t *addrp,
645	__in		size_t size,
646	__in		unsigned int n,
647	__in		unsigned int completed,
648	__in		unsigned int added)
649{
650	efx_qword_t qword;
651	unsigned int i;
652	unsigned int offset;
653	unsigned int id;
654
655	/* The client driver must not overfill the queue */
656	EFSYS_ASSERT3U(added - completed + n, <=,
657	    EFX_RXQ_LIMIT(erp->er_mask + 1));
658
659	id = added & (erp->er_mask);
660	for (i = 0; i < n; i++) {
661		EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
662		    unsigned int, id, efsys_dma_addr_t, addrp[i],
663		    size_t, size);
664
665		EFX_POPULATE_QWORD_3(qword,
666		    ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
667		    ESF_DZ_RX_KER_BUF_ADDR_DW0,
668		    (uint32_t)(addrp[i] & 0xffffffff),
669		    ESF_DZ_RX_KER_BUF_ADDR_DW1,
670		    (uint32_t)(addrp[i] >> 32));
671
672		offset = id * sizeof (efx_qword_t);
673		EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
674
675		id = (id + 1) & (erp->er_mask);
676	}
677}
678
679			void
680ef10_rx_qpush(
681	__in	efx_rxq_t *erp,
682	__in	unsigned int added,
683	__inout	unsigned int *pushedp)
684{
685	efx_nic_t *enp = erp->er_enp;
686	unsigned int pushed = *pushedp;
687	uint32_t wptr;
688	efx_dword_t dword;
689
690	/* Hardware has alignment restriction for WPTR */
691	wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
692	if (pushed == wptr)
693		return;
694
695	*pushedp = wptr;
696
697	/* Push the populated descriptors out */
698	wptr &= erp->er_mask;
699
700	EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
701
702	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
703	EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
704	    wptr, pushed & erp->er_mask);
705	EFSYS_PIO_WRITE_BARRIER();
706	EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
707			    erp->er_index, &dword, B_FALSE);
708}
709
710	__checkReturn	efx_rc_t
711ef10_rx_qflush(
712	__in	efx_rxq_t *erp)
713{
714	efx_nic_t *enp = erp->er_enp;
715	efx_rc_t rc;
716
717	if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
718		goto fail1;
719
720	return (0);
721
722fail1:
723	EFSYS_PROBE1(fail1, efx_rc_t, rc);
724
725	return (rc);
726}
727
728		void
729ef10_rx_qenable(
730	__in	efx_rxq_t *erp)
731{
732	/* FIXME */
733	_NOTE(ARGUNUSED(erp))
734	/* FIXME */
735}
736
737	__checkReturn	efx_rc_t
738ef10_rx_qcreate(
739	__in		efx_nic_t *enp,
740	__in		unsigned int index,
741	__in		unsigned int label,
742	__in		efx_rxq_type_t type,
743	__in		efsys_mem_t *esmp,
744	__in		size_t n,
745	__in		uint32_t id,
746	__in		efx_evq_t *eep,
747	__in		efx_rxq_t *erp)
748{
749	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
750	efx_rc_t rc;
751	boolean_t disable_scatter;
752
753	_NOTE(ARGUNUSED(id, erp))
754
755	EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
756	EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
757	EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
758
759	EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
760	EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
761
762	if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
763		rc = EINVAL;
764		goto fail1;
765	}
766	if (index >= encp->enc_rxq_limit) {
767		rc = EINVAL;
768		goto fail2;
769	}
770
771	/* Scatter can only be disabled if the firmware supports doing so */
772	if ((type != EFX_RXQ_TYPE_SCATTER) &&
773	    enp->en_nic_cfg.enc_rx_disable_scatter_supported) {
774		disable_scatter = B_TRUE;
775	} else {
776		disable_scatter = B_FALSE;
777	}
778
779	if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
780	    esmp, disable_scatter)) != 0)
781		goto fail3;
782
783	erp->er_eep = eep;
784	erp->er_label = label;
785
786	ef10_ev_rxlabel_init(eep, erp, label);
787
788	return (0);
789
790fail3:
791	EFSYS_PROBE(fail3);
792fail2:
793	EFSYS_PROBE(fail2);
794fail1:
795	EFSYS_PROBE1(fail1, efx_rc_t, rc);
796
797	return (rc);
798}
799
800		void
801ef10_rx_qdestroy(
802	__in	efx_rxq_t *erp)
803{
804	efx_nic_t *enp = erp->er_enp;
805	efx_evq_t *eep = erp->er_eep;
806	unsigned int label = erp->er_label;
807
808	ef10_ev_rxlabel_fini(eep, label);
809
810	EFSYS_ASSERT(enp->en_rx_qcount != 0);
811	--enp->en_rx_qcount;
812
813	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
814}
815
816		void
817ef10_rx_fini(
818	__in	efx_nic_t *enp)
819{
820#if EFSYS_OPT_RX_SCALE
821	if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) {
822		(void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
823	}
824	enp->en_rss_context = 0;
825	enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
826#else
827	_NOTE(ARGUNUSED(enp))
828#endif /* EFSYS_OPT_RX_SCALE */
829}
830
831#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
832