1227569Sphilip/*-
2301388Sarybchik * Copyright (c) 2007-2016 Solarflare Communications Inc.
3284555Sarybchik * All rights reserved.
4227569Sphilip *
5227569Sphilip * Redistribution and use in source and binary forms, with or without
6284555Sarybchik * modification, are permitted provided that the following conditions are met:
7227569Sphilip *
8284555Sarybchik * 1. Redistributions of source code must retain the above copyright notice,
9284555Sarybchik *    this list of conditions and the following disclaimer.
10284555Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice,
11284555Sarybchik *    this list of conditions and the following disclaimer in the documentation
12284555Sarybchik *    and/or other materials provided with the distribution.
13284555Sarybchik *
14284555Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15284555Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16284555Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17284555Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18284555Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19284555Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20284555Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21284555Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22284555Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23284555Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24284555Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25284555Sarybchik *
26284555Sarybchik * The views and conclusions contained in the software and documentation are
27284555Sarybchik * those of the authors and should not be interpreted as representing official
28284555Sarybchik * policies, either expressed or implied, of the FreeBSD Project.
29227569Sphilip */
30227569Sphilip
31228078Sphilip#include <sys/cdefs.h>
32228078Sphilip__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/common/efx_ev.c 342521 2018-12-26 10:28:03Z arybchik $");
33228078Sphilip
34227569Sphilip#include "efx.h"
35227569Sphilip#include "efx_impl.h"
36294016Sarybchik#if EFSYS_OPT_MON_MCDI
37284555Sarybchik#include "mcdi_mon.h"
38294016Sarybchik#endif
39227569Sphilip
40227569Sphilip#if EFSYS_OPT_QSTATS
41227569Sphilip#define	EFX_EV_QSTAT_INCR(_eep, _stat)					\
42227569Sphilip	do {								\
43227569Sphilip		(_eep)->ee_stat[_stat]++;				\
44227569Sphilip	_NOTE(CONSTANTCONDITION)					\
45227569Sphilip	} while (B_FALSE)
46227569Sphilip#else
47227569Sphilip#define	EFX_EV_QSTAT_INCR(_eep, _stat)
48227569Sphilip#endif
49227569Sphilip
50284555Sarybchik#define	EFX_EV_PRESENT(_qword)						\
51284555Sarybchik	(EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&	\
52284555Sarybchik	EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
53284555Sarybchik
54284555Sarybchik
55284555Sarybchik
56301324Sarybchik#if EFSYS_OPT_SIENA
57284555Sarybchik
58293927Sarybchikstatic	__checkReturn	efx_rc_t
59301345Sarybchiksiena_ev_init(
60284555Sarybchik	__in		efx_nic_t *enp);
61284555Sarybchik
62284555Sarybchikstatic			void
63301345Sarybchiksiena_ev_fini(
64284555Sarybchik	__in		efx_nic_t *enp);
65284555Sarybchik
66293927Sarybchikstatic	__checkReturn	efx_rc_t
67301345Sarybchiksiena_ev_qcreate(
68284555Sarybchik	__in		efx_nic_t *enp,
69284555Sarybchik	__in		unsigned int index,
70284555Sarybchik	__in		efsys_mem_t *esmp,
71284555Sarybchik	__in		size_t n,
72284555Sarybchik	__in		uint32_t id,
73301980Sarybchik	__in		uint32_t us,
74311070Sarybchik	__in		uint32_t flags,
75284555Sarybchik	__in		efx_evq_t *eep);
76284555Sarybchik
77284555Sarybchikstatic			void
78301345Sarybchiksiena_ev_qdestroy(
79284555Sarybchik	__in		efx_evq_t *eep);
80284555Sarybchik
81293927Sarybchikstatic	__checkReturn	efx_rc_t
82301345Sarybchiksiena_ev_qprime(
83284555Sarybchik	__in		efx_evq_t *eep,
84284555Sarybchik	__in		unsigned int count);
85284555Sarybchik
86284555Sarybchikstatic			void
87301345Sarybchiksiena_ev_qpost(
88284555Sarybchik	__in	efx_evq_t *eep,
89284555Sarybchik	__in	uint16_t data);
90284555Sarybchik
91293927Sarybchikstatic	__checkReturn	efx_rc_t
92301345Sarybchiksiena_ev_qmoderate(
93284555Sarybchik	__in		efx_evq_t *eep,
94284555Sarybchik	__in		unsigned int us);
95284555Sarybchik
96284555Sarybchik#if EFSYS_OPT_QSTATS
97284555Sarybchikstatic			void
98301345Sarybchiksiena_ev_qstats_update(
99284555Sarybchik	__in				efx_evq_t *eep,
100284555Sarybchik	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat);
101284555Sarybchik
102284555Sarybchik#endif
103284555Sarybchik
104301324Sarybchik#endif /* EFSYS_OPT_SIENA */
105284555Sarybchik
106284555Sarybchik#if EFSYS_OPT_SIENA
107301340Sarybchikstatic const efx_ev_ops_t	__efx_ev_siena_ops = {
108301345Sarybchik	siena_ev_init,				/* eevo_init */
109301345Sarybchik	siena_ev_fini,				/* eevo_fini */
110301345Sarybchik	siena_ev_qcreate,			/* eevo_qcreate */
111301345Sarybchik	siena_ev_qdestroy,			/* eevo_qdestroy */
112301345Sarybchik	siena_ev_qprime,			/* eevo_qprime */
113301345Sarybchik	siena_ev_qpost,				/* eevo_qpost */
114301345Sarybchik	siena_ev_qmoderate,			/* eevo_qmoderate */
115284555Sarybchik#if EFSYS_OPT_QSTATS
116301345Sarybchik	siena_ev_qstats_update,			/* eevo_qstats_update */
117284555Sarybchik#endif
118284555Sarybchik};
119284555Sarybchik#endif /* EFSYS_OPT_SIENA */
120284555Sarybchik
121293986Sarybchik#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
122301340Sarybchikstatic const efx_ev_ops_t	__efx_ev_ef10_ops = {
123293986Sarybchik	ef10_ev_init,				/* eevo_init */
124293986Sarybchik	ef10_ev_fini,				/* eevo_fini */
125293986Sarybchik	ef10_ev_qcreate,			/* eevo_qcreate */
126293986Sarybchik	ef10_ev_qdestroy,			/* eevo_qdestroy */
127293986Sarybchik	ef10_ev_qprime,				/* eevo_qprime */
128293986Sarybchik	ef10_ev_qpost,				/* eevo_qpost */
129293986Sarybchik	ef10_ev_qmoderate,			/* eevo_qmoderate */
130284555Sarybchik#if EFSYS_OPT_QSTATS
131293986Sarybchik	ef10_ev_qstats_update,			/* eevo_qstats_update */
132284555Sarybchik#endif
133284555Sarybchik};
134293986Sarybchik#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
135284555Sarybchik
136284555Sarybchik
137293927Sarybchik	__checkReturn	efx_rc_t
138227569Sphilipefx_ev_init(
139227569Sphilip	__in		efx_nic_t *enp)
140227569Sphilip{
141301340Sarybchik	const efx_ev_ops_t *eevop;
142293927Sarybchik	efx_rc_t rc;
143227569Sphilip
144227569Sphilip	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
145227569Sphilip	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
146227569Sphilip
147227569Sphilip	if (enp->en_mod_flags & EFX_MOD_EV) {
148227569Sphilip		rc = EINVAL;
149227569Sphilip		goto fail1;
150227569Sphilip	}
151227569Sphilip
152284555Sarybchik	switch (enp->en_family) {
153284555Sarybchik#if EFSYS_OPT_SIENA
154284555Sarybchik	case EFX_FAMILY_SIENA:
155301340Sarybchik		eevop = &__efx_ev_siena_ops;
156284555Sarybchik		break;
157284555Sarybchik#endif /* EFSYS_OPT_SIENA */
158284555Sarybchik
159284555Sarybchik#if EFSYS_OPT_HUNTINGTON
160284555Sarybchik	case EFX_FAMILY_HUNTINGTON:
161301340Sarybchik		eevop = &__efx_ev_ef10_ops;
162284555Sarybchik		break;
163284555Sarybchik#endif /* EFSYS_OPT_HUNTINGTON */
164284555Sarybchik
165293986Sarybchik#if EFSYS_OPT_MEDFORD
166293986Sarybchik	case EFX_FAMILY_MEDFORD:
167301340Sarybchik		eevop = &__efx_ev_ef10_ops;
168293986Sarybchik		break;
169293986Sarybchik#endif /* EFSYS_OPT_MEDFORD */
170293986Sarybchik
171284555Sarybchik	default:
172284555Sarybchik		EFSYS_ASSERT(0);
173284555Sarybchik		rc = ENOTSUP;
174284555Sarybchik		goto fail1;
175284555Sarybchik	}
176284555Sarybchik
177227569Sphilip	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
178227569Sphilip
179284555Sarybchik	if ((rc = eevop->eevo_init(enp)) != 0)
180284555Sarybchik		goto fail2;
181284555Sarybchik
182284555Sarybchik	enp->en_eevop = eevop;
183284555Sarybchik	enp->en_mod_flags |= EFX_MOD_EV;
184284555Sarybchik	return (0);
185284555Sarybchik
186284555Sarybchikfail2:
187284555Sarybchik	EFSYS_PROBE(fail2);
188284555Sarybchik
189284555Sarybchikfail1:
190293927Sarybchik	EFSYS_PROBE1(fail1, efx_rc_t, rc);
191284555Sarybchik
192284555Sarybchik	enp->en_eevop = NULL;
193284555Sarybchik	enp->en_mod_flags &= ~EFX_MOD_EV;
194284555Sarybchik	return (rc);
195284555Sarybchik}
196284555Sarybchik
197284555Sarybchik		void
198284555Sarybchikefx_ev_fini(
199284555Sarybchik	__in	efx_nic_t *enp)
200284555Sarybchik{
201301340Sarybchik	const efx_ev_ops_t *eevop = enp->en_eevop;
202284555Sarybchik
203284555Sarybchik	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
204284555Sarybchik	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
205284555Sarybchik	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
206284555Sarybchik	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
207284555Sarybchik	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
208284555Sarybchik	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
209284555Sarybchik
210284555Sarybchik	eevop->eevo_fini(enp);
211284555Sarybchik
212284555Sarybchik	enp->en_eevop = NULL;
213284555Sarybchik	enp->en_mod_flags &= ~EFX_MOD_EV;
214284555Sarybchik}
215284555Sarybchik
216284555Sarybchik
217293927Sarybchik	__checkReturn	efx_rc_t
218284555Sarybchikefx_ev_qcreate(
219284555Sarybchik	__in		efx_nic_t *enp,
220284555Sarybchik	__in		unsigned int index,
221284555Sarybchik	__in		efsys_mem_t *esmp,
222284555Sarybchik	__in		size_t n,
223284555Sarybchik	__in		uint32_t id,
224301980Sarybchik	__in		uint32_t us,
225311070Sarybchik	__in		uint32_t flags,
226284555Sarybchik	__deref_out	efx_evq_t **eepp)
227284555Sarybchik{
228301340Sarybchik	const efx_ev_ops_t *eevop = enp->en_eevop;
229284555Sarybchik	efx_evq_t *eep;
230293927Sarybchik	efx_rc_t rc;
231284555Sarybchik
232284555Sarybchik	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
233284555Sarybchik	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
234284555Sarybchik
235342502Sarybchik	EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <,
236342502Sarybchik	    enp->en_nic_cfg.enc_evq_limit);
237284555Sarybchik
238311094Sarybchik	switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) {
239311094Sarybchik	case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT:
240311094Sarybchik		break;
241311094Sarybchik	case EFX_EVQ_FLAGS_NOTIFY_DISABLED:
242311094Sarybchik		if (us != 0) {
243311094Sarybchik			rc = EINVAL;
244311094Sarybchik			goto fail1;
245311094Sarybchik		}
246311094Sarybchik		break;
247311094Sarybchik	default:
248311094Sarybchik		rc = EINVAL;
249311094Sarybchik		goto fail2;
250311094Sarybchik	}
251311094Sarybchik
252284555Sarybchik	/* Allocate an EVQ object */
253284555Sarybchik	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
254284555Sarybchik	if (eep == NULL) {
255284555Sarybchik		rc = ENOMEM;
256311094Sarybchik		goto fail3;
257284555Sarybchik	}
258284555Sarybchik
259284555Sarybchik	eep->ee_magic = EFX_EVQ_MAGIC;
260284555Sarybchik	eep->ee_enp = enp;
261284555Sarybchik	eep->ee_index = index;
262284555Sarybchik	eep->ee_mask = n - 1;
263311094Sarybchik	eep->ee_flags = flags;
264284555Sarybchik	eep->ee_esmp = esmp;
265284555Sarybchik
266301386Sarybchik	/*
267301386Sarybchik	 * Set outputs before the queue is created because interrupts may be
268301386Sarybchik	 * raised for events immediately after the queue is created, before the
269301386Sarybchik	 * function call below returns. See bug58606.
270301386Sarybchik	 *
271301386Sarybchik	 * The eepp pointer passed in by the client must therefore point to data
272301386Sarybchik	 * shared with the client's event processing context.
273301386Sarybchik	 */
274301386Sarybchik	enp->en_ev_qcount++;
275301386Sarybchik	*eepp = eep;
276301386Sarybchik
277311070Sarybchik	if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, us, flags,
278311070Sarybchik	    eep)) != 0)
279311094Sarybchik		goto fail4;
280284555Sarybchik
281284555Sarybchik	return (0);
282284555Sarybchik
283311094Sarybchikfail4:
284311094Sarybchik	EFSYS_PROBE(fail4);
285301386Sarybchik
286301386Sarybchik	*eepp = NULL;
287301386Sarybchik	enp->en_ev_qcount--;
288284555Sarybchik	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
289311094Sarybchikfail3:
290311094Sarybchik	EFSYS_PROBE(fail3);
291311094Sarybchikfail2:
292311094Sarybchik	EFSYS_PROBE(fail2);
293284555Sarybchikfail1:
294293927Sarybchik	EFSYS_PROBE1(fail1, efx_rc_t, rc);
295284555Sarybchik	return (rc);
296284555Sarybchik}
297284555Sarybchik
298284555Sarybchik		void
299284555Sarybchikefx_ev_qdestroy(
300284555Sarybchik	__in	efx_evq_t *eep)
301284555Sarybchik{
302284555Sarybchik	efx_nic_t *enp = eep->ee_enp;
303301340Sarybchik	const efx_ev_ops_t *eevop = enp->en_eevop;
304284555Sarybchik
305284555Sarybchik	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
306284555Sarybchik
307284555Sarybchik	EFSYS_ASSERT(enp->en_ev_qcount != 0);
308284555Sarybchik	--enp->en_ev_qcount;
309284555Sarybchik
310284555Sarybchik	eevop->eevo_qdestroy(eep);
311284555Sarybchik
312284555Sarybchik	/* Free the EVQ object */
313284555Sarybchik	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
314284555Sarybchik}
315284555Sarybchik
316293927Sarybchik	__checkReturn	efx_rc_t
317284555Sarybchikefx_ev_qprime(
318284555Sarybchik	__in		efx_evq_t *eep,
319284555Sarybchik	__in		unsigned int count)
320284555Sarybchik{
321284555Sarybchik	efx_nic_t *enp = eep->ee_enp;
322301340Sarybchik	const efx_ev_ops_t *eevop = enp->en_eevop;
323293927Sarybchik	efx_rc_t rc;
324284555Sarybchik
325284555Sarybchik	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
326284555Sarybchik
327284555Sarybchik	if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
328284555Sarybchik		rc = EINVAL;
329284555Sarybchik		goto fail1;
330284555Sarybchik	}
331284555Sarybchik
332284555Sarybchik	if ((rc = eevop->eevo_qprime(eep, count)) != 0)
333284555Sarybchik		goto fail2;
334284555Sarybchik
335284555Sarybchik	return (0);
336284555Sarybchik
337284555Sarybchikfail2:
338284555Sarybchik	EFSYS_PROBE(fail2);
339284555Sarybchikfail1:
340293927Sarybchik	EFSYS_PROBE1(fail1, efx_rc_t, rc);
341284555Sarybchik	return (rc);
342284555Sarybchik}
343284555Sarybchik
344284555Sarybchik	__checkReturn	boolean_t
345284555Sarybchikefx_ev_qpending(
346284555Sarybchik	__in		efx_evq_t *eep,
347284555Sarybchik	__in		unsigned int count)
348284555Sarybchik{
349284555Sarybchik	size_t offset;
350284555Sarybchik	efx_qword_t qword;
351284555Sarybchik
352284555Sarybchik	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
353284555Sarybchik
354284555Sarybchik	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
355284555Sarybchik	EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
356284555Sarybchik
357284555Sarybchik	return (EFX_EV_PRESENT(qword));
358284555Sarybchik}
359284555Sarybchik
360284555Sarybchik#if EFSYS_OPT_EV_PREFETCH
361284555Sarybchik
362284555Sarybchik			void
363284555Sarybchikefx_ev_qprefetch(
364284555Sarybchik	__in		efx_evq_t *eep,
365284555Sarybchik	__in		unsigned int count)
366284555Sarybchik{
367284555Sarybchik	unsigned int offset;
368284555Sarybchik
369284555Sarybchik	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
370284555Sarybchik
371284555Sarybchik	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
372284555Sarybchik	EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
373284555Sarybchik}
374284555Sarybchik
375284555Sarybchik#endif	/* EFSYS_OPT_EV_PREFETCH */
376284555Sarybchik
377311491Sarybchik#define	EFX_EV_BATCH	8
378311491Sarybchik
379284555Sarybchik			void
380284555Sarybchikefx_ev_qpoll(
381284555Sarybchik	__in		efx_evq_t *eep,
382284555Sarybchik	__inout		unsigned int *countp,
383284555Sarybchik	__in		const efx_ev_callbacks_t *eecp,
384284555Sarybchik	__in_opt	void *arg)
385284555Sarybchik{
386311491Sarybchik	efx_qword_t ev[EFX_EV_BATCH];
387311491Sarybchik	unsigned int batch;
388311491Sarybchik	unsigned int total;
389311491Sarybchik	unsigned int count;
390311491Sarybchik	unsigned int index;
391311491Sarybchik	size_t offset;
392284555Sarybchik
393311491Sarybchik	/* Ensure events codes match for EF10 (Huntington/Medford) and Siena */
394284555Sarybchik	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
395284555Sarybchik	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
396284555Sarybchik
397284555Sarybchik	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
398284555Sarybchik	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
399284555Sarybchik	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
400284555Sarybchik	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
401284555Sarybchik	    FSE_AZ_EV_CODE_DRV_GEN_EV);
402284555Sarybchik#if EFSYS_OPT_MCDI
403284555Sarybchik	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
404284555Sarybchik	    FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
405284555Sarybchik#endif
406311491Sarybchik
407311491Sarybchik	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
408311491Sarybchik	EFSYS_ASSERT(countp != NULL);
409311491Sarybchik	EFSYS_ASSERT(eecp != NULL);
410311491Sarybchik
411311491Sarybchik	count = *countp;
412311491Sarybchik	do {
413311491Sarybchik		/* Read up until the end of the batch period */
414311491Sarybchik		batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
415311491Sarybchik		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
416311491Sarybchik		for (total = 0; total < batch; ++total) {
417311491Sarybchik			EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
418311491Sarybchik
419311491Sarybchik			if (!EFX_EV_PRESENT(ev[total]))
420311491Sarybchik				break;
421311491Sarybchik
422311491Sarybchik			EFSYS_PROBE3(event, unsigned int, eep->ee_index,
423311491Sarybchik			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
424311491Sarybchik			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
425311491Sarybchik
426311491Sarybchik			offset += sizeof (efx_qword_t);
427311491Sarybchik		}
428311491Sarybchik
429311491Sarybchik#if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
430311491Sarybchik		/*
431311491Sarybchik		 * Prefetch the next batch when we get within PREFETCH_PERIOD
432311491Sarybchik		 * of a completed batch. If the batch is smaller, then prefetch
433311491Sarybchik		 * immediately.
434311491Sarybchik		 */
435311491Sarybchik		if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
436311491Sarybchik			EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
437311491Sarybchik#endif	/* EFSYS_OPT_EV_PREFETCH */
438311491Sarybchik
439311491Sarybchik		/* Process the batch of events */
440311491Sarybchik		for (index = 0; index < total; ++index) {
441311491Sarybchik			boolean_t should_abort;
442311491Sarybchik			uint32_t code;
443311491Sarybchik
444311491Sarybchik#if EFSYS_OPT_EV_PREFETCH
445311491Sarybchik			/* Prefetch if we've now reached the batch period */
446311491Sarybchik			if (total == batch &&
447311491Sarybchik			    index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
448311491Sarybchik				offset = (count + batch) & eep->ee_mask;
449311491Sarybchik				offset *= sizeof (efx_qword_t);
450311491Sarybchik
451311491Sarybchik				EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
452311491Sarybchik			}
453311491Sarybchik#endif	/* EFSYS_OPT_EV_PREFETCH */
454311491Sarybchik
455311491Sarybchik			EFX_EV_QSTAT_INCR(eep, EV_ALL);
456311491Sarybchik
457311491Sarybchik			code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
458311491Sarybchik			switch (code) {
459311491Sarybchik			case FSE_AZ_EV_CODE_RX_EV:
460311491Sarybchik				should_abort = eep->ee_rx(eep,
461311491Sarybchik				    &(ev[index]), eecp, arg);
462311491Sarybchik				break;
463311491Sarybchik			case FSE_AZ_EV_CODE_TX_EV:
464311491Sarybchik				should_abort = eep->ee_tx(eep,
465311491Sarybchik				    &(ev[index]), eecp, arg);
466311491Sarybchik				break;
467311491Sarybchik			case FSE_AZ_EV_CODE_DRIVER_EV:
468311491Sarybchik				should_abort = eep->ee_driver(eep,
469311491Sarybchik				    &(ev[index]), eecp, arg);
470311491Sarybchik				break;
471311491Sarybchik			case FSE_AZ_EV_CODE_DRV_GEN_EV:
472311491Sarybchik				should_abort = eep->ee_drv_gen(eep,
473311491Sarybchik				    &(ev[index]), eecp, arg);
474311491Sarybchik				break;
475311491Sarybchik#if EFSYS_OPT_MCDI
476311491Sarybchik			case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
477311491Sarybchik				should_abort = eep->ee_mcdi(eep,
478311491Sarybchik				    &(ev[index]), eecp, arg);
479311491Sarybchik				break;
480311491Sarybchik#endif
481311491Sarybchik			case FSE_AZ_EV_CODE_GLOBAL_EV:
482311491Sarybchik				if (eep->ee_global) {
483311491Sarybchik					should_abort = eep->ee_global(eep,
484311491Sarybchik					    &(ev[index]), eecp, arg);
485311491Sarybchik					break;
486311491Sarybchik				}
487311491Sarybchik				/* else fallthrough */
488311491Sarybchik			default:
489311491Sarybchik				EFSYS_PROBE3(bad_event,
490311491Sarybchik				    unsigned int, eep->ee_index,
491311491Sarybchik				    uint32_t,
492311491Sarybchik				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
493311491Sarybchik				    uint32_t,
494311491Sarybchik				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
495311491Sarybchik
496311491Sarybchik				EFSYS_ASSERT(eecp->eec_exception != NULL);
497311491Sarybchik				(void) eecp->eec_exception(arg,
498311491Sarybchik					EFX_EXCEPTION_EV_ERROR, code);
499311491Sarybchik				should_abort = B_TRUE;
500311491Sarybchik			}
501311491Sarybchik			if (should_abort) {
502311491Sarybchik				/* Ignore subsequent events */
503311491Sarybchik				total = index + 1;
504342521Sarybchik
505342521Sarybchik				/*
506342521Sarybchik				 * Poison batch to ensure the outer
507342521Sarybchik				 * loop is broken out of.
508342521Sarybchik				 */
509342521Sarybchik				EFSYS_ASSERT(batch <= EFX_EV_BATCH);
510342521Sarybchik				batch += (EFX_EV_BATCH << 1);
511342521Sarybchik				EFSYS_ASSERT(total != batch);
512311491Sarybchik				break;
513311491Sarybchik			}
514311491Sarybchik		}
515311491Sarybchik
516311491Sarybchik		/*
517311491Sarybchik		 * Now that the hardware has most likely moved onto dma'ing
518311491Sarybchik		 * into the next cache line, clear the processed events. Take
519311491Sarybchik		 * care to only clear out events that we've processed
520311491Sarybchik		 */
521311491Sarybchik		EFX_SET_QWORD(ev[0]);
522311491Sarybchik		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
523311491Sarybchik		for (index = 0; index < total; ++index) {
524311491Sarybchik			EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
525311491Sarybchik			offset += sizeof (efx_qword_t);
526311491Sarybchik		}
527311491Sarybchik
528311491Sarybchik		count += total;
529311491Sarybchik
530311491Sarybchik	} while (total == batch);
531311491Sarybchik
532311491Sarybchik	*countp = count;
533284555Sarybchik}
534284555Sarybchik
535284555Sarybchik			void
536284555Sarybchikefx_ev_qpost(
537284555Sarybchik	__in	efx_evq_t *eep,
538284555Sarybchik	__in	uint16_t data)
539284555Sarybchik{
540284555Sarybchik	efx_nic_t *enp = eep->ee_enp;
541301340Sarybchik	const efx_ev_ops_t *eevop = enp->en_eevop;
542284555Sarybchik
543284555Sarybchik	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
544284555Sarybchik
545284555Sarybchik	EFSYS_ASSERT(eevop != NULL &&
546284555Sarybchik	    eevop->eevo_qpost != NULL);
547284555Sarybchik
548284555Sarybchik	eevop->eevo_qpost(eep, data);
549284555Sarybchik}
550284555Sarybchik
551293927Sarybchik	__checkReturn	efx_rc_t
552301984Sarybchikefx_ev_usecs_to_ticks(
553301984Sarybchik	__in		efx_nic_t *enp,
554301984Sarybchik	__in		unsigned int us,
555301984Sarybchik	__out		unsigned int *ticksp)
556301984Sarybchik{
557301984Sarybchik	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
558301984Sarybchik	unsigned int ticks;
559301984Sarybchik
560301984Sarybchik	/* Convert microseconds to a timer tick count */
561301984Sarybchik	if (us == 0)
562301984Sarybchik		ticks = 0;
563301984Sarybchik	else if (us * 1000 < encp->enc_evq_timer_quantum_ns)
564301984Sarybchik		ticks = 1;	/* Never round down to zero */
565301984Sarybchik	else
566301984Sarybchik		ticks = us * 1000 / encp->enc_evq_timer_quantum_ns;
567301984Sarybchik
568301984Sarybchik	*ticksp = ticks;
569301984Sarybchik	return (0);
570301984Sarybchik}
571301984Sarybchik
572301984Sarybchik	__checkReturn	efx_rc_t
573284555Sarybchikefx_ev_qmoderate(
574284555Sarybchik	__in		efx_evq_t *eep,
575284555Sarybchik	__in		unsigned int us)
576284555Sarybchik{
577284555Sarybchik	efx_nic_t *enp = eep->ee_enp;
578301340Sarybchik	const efx_ev_ops_t *eevop = enp->en_eevop;
579293927Sarybchik	efx_rc_t rc;
580284555Sarybchik
581284555Sarybchik	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
582284555Sarybchik
583311094Sarybchik	if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) ==
584311094Sarybchik	    EFX_EVQ_FLAGS_NOTIFY_DISABLED) {
585311094Sarybchik		rc = EINVAL;
586284555Sarybchik		goto fail1;
587311094Sarybchik	}
588284555Sarybchik
589311094Sarybchik	if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
590311094Sarybchik		goto fail2;
591311094Sarybchik
592284555Sarybchik	return (0);
593284555Sarybchik
594311094Sarybchikfail2:
595311094Sarybchik	EFSYS_PROBE(fail2);
596284555Sarybchikfail1:
597293927Sarybchik	EFSYS_PROBE1(fail1, efx_rc_t, rc);
598284555Sarybchik	return (rc);
599284555Sarybchik}
600284555Sarybchik
601284555Sarybchik#if EFSYS_OPT_QSTATS
602284555Sarybchik					void
603284555Sarybchikefx_ev_qstats_update(
604284555Sarybchik	__in				efx_evq_t *eep,
605284555Sarybchik	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
606284555Sarybchik
607284555Sarybchik{	efx_nic_t *enp = eep->ee_enp;
608301340Sarybchik	const efx_ev_ops_t *eevop = enp->en_eevop;
609284555Sarybchik
610284555Sarybchik	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
611284555Sarybchik
612284555Sarybchik	eevop->eevo_qstats_update(eep, stat);
613284555Sarybchik}
614284555Sarybchik
615284555Sarybchik#endif	/* EFSYS_OPT_QSTATS */
616284555Sarybchik
617301324Sarybchik#if EFSYS_OPT_SIENA
618284555Sarybchik
619293927Sarybchikstatic	__checkReturn	efx_rc_t
620301345Sarybchiksiena_ev_init(
621284555Sarybchik	__in		efx_nic_t *enp)
622284555Sarybchik{
623284555Sarybchik	efx_oword_t oword;
624284555Sarybchik
625284555Sarybchik	/*
626227569Sphilip	 * Program the event queue for receive and transmit queue
627227569Sphilip	 * flush events.
628227569Sphilip	 */
629227569Sphilip	EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
630227569Sphilip	EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
631227569Sphilip	EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
632227569Sphilip
633227569Sphilip	return (0);
634227569Sphilip
635227569Sphilip}
636227569Sphilip
637227569Sphilipstatic  __checkReturn   boolean_t
638301345Sarybchiksiena_ev_rx_not_ok(
639227569Sphilip	__in		efx_evq_t *eep,
640227569Sphilip	__in		efx_qword_t *eqp,
641227569Sphilip	__in		uint32_t label,
642227569Sphilip	__in		uint32_t id,
643227569Sphilip	__inout		uint16_t *flagsp)
644227569Sphilip{
645227569Sphilip	boolean_t ignore = B_FALSE;
646227569Sphilip
647227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
648227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
649227569Sphilip		EFSYS_PROBE(tobe_disc);
650280535Sarybchik		/*
651280535Sarybchik		 * Assume this is a unicast address mismatch, unless below
652227569Sphilip		 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
653227569Sphilip		 * EV_RX_PAUSE_FRM_ERR is set.
654227569Sphilip		 */
655227569Sphilip		(*flagsp) |= EFX_ADDR_MISMATCH;
656227569Sphilip	}
657227569Sphilip
658227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
659227569Sphilip		EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
660227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
661227569Sphilip		(*flagsp) |= EFX_DISCARD;
662227569Sphilip
663294007Sarybchik#if EFSYS_OPT_RX_SCATTER
664280535Sarybchik		/*
665280535Sarybchik		 * Lookout for payload queue ran dry errors and ignore them.
666227569Sphilip		 *
667227569Sphilip		 * Sadly for the header/data split cases, the descriptor
668227569Sphilip		 * pointer in this event refers to the header queue and
669227569Sphilip		 * therefore cannot be easily detected as duplicate.
670227569Sphilip		 * So we drop these and rely on the receive processing seeing
671227569Sphilip		 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
672227569Sphilip		 * the partially received packet.
673227569Sphilip		 */
674227569Sphilip		if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
675227569Sphilip		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
676227569Sphilip		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
677227569Sphilip			ignore = B_TRUE;
678294007Sarybchik#endif	/* EFSYS_OPT_RX_SCATTER */
679227569Sphilip	}
680227569Sphilip
681227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
682227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
683227569Sphilip		EFSYS_PROBE(crc_err);
684227569Sphilip		(*flagsp) &= ~EFX_ADDR_MISMATCH;
685227569Sphilip		(*flagsp) |= EFX_DISCARD;
686227569Sphilip	}
687227569Sphilip
688227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
689227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
690227569Sphilip		EFSYS_PROBE(pause_frm_err);
691227569Sphilip		(*flagsp) &= ~EFX_ADDR_MISMATCH;
692227569Sphilip		(*flagsp) |= EFX_DISCARD;
693227569Sphilip	}
694227569Sphilip
695227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
696227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
697227569Sphilip		EFSYS_PROBE(owner_id_err);
698227569Sphilip		(*flagsp) |= EFX_DISCARD;
699227569Sphilip	}
700227569Sphilip
701227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
702227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
703227569Sphilip		EFSYS_PROBE(ipv4_err);
704227569Sphilip		(*flagsp) &= ~EFX_CKSUM_IPV4;
705227569Sphilip	}
706227569Sphilip
707227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
708227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
709227569Sphilip		EFSYS_PROBE(udp_chk_err);
710227569Sphilip		(*flagsp) &= ~EFX_CKSUM_TCPUDP;
711227569Sphilip	}
712227569Sphilip
713227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
714227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
715227569Sphilip
716227569Sphilip		/*
717227569Sphilip		 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
718227569Sphilip		 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
719227569Sphilip		 * condition.
720227569Sphilip		 */
721227569Sphilip		(*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
722227569Sphilip	}
723227569Sphilip
724227569Sphilip	return (ignore);
725227569Sphilip}
726227569Sphilip
727227569Sphilipstatic	__checkReturn	boolean_t
728301345Sarybchiksiena_ev_rx(
729227569Sphilip	__in		efx_evq_t *eep,
730227569Sphilip	__in		efx_qword_t *eqp,
731227569Sphilip	__in		const efx_ev_callbacks_t *eecp,
732227569Sphilip	__in_opt	void *arg)
733227569Sphilip{
734227569Sphilip	uint32_t id;
735227569Sphilip	uint32_t size;
736227569Sphilip	uint32_t label;
737227569Sphilip	boolean_t ok;
738294007Sarybchik#if EFSYS_OPT_RX_SCATTER
739227569Sphilip	boolean_t sop;
740227569Sphilip	boolean_t jumbo_cont;
741294007Sarybchik#endif	/* EFSYS_OPT_RX_SCATTER */
742227569Sphilip	uint32_t hdr_type;
743227569Sphilip	boolean_t is_v6;
744227569Sphilip	uint16_t flags;
745227569Sphilip	boolean_t ignore;
746227569Sphilip	boolean_t should_abort;
747227569Sphilip
748227569Sphilip	EFX_EV_QSTAT_INCR(eep, EV_RX);
749227569Sphilip
750227569Sphilip	/* Basic packet information */
751227569Sphilip	id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
752227569Sphilip	size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
753227569Sphilip	label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
754227569Sphilip	ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
755227569Sphilip
756294007Sarybchik#if EFSYS_OPT_RX_SCATTER
757227569Sphilip	sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
758227569Sphilip	jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
759294007Sarybchik#endif	/* EFSYS_OPT_RX_SCATTER */
760227569Sphilip
761227569Sphilip	hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
762227569Sphilip
763301330Sarybchik	is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
764227569Sphilip
765227569Sphilip	/*
766227569Sphilip	 * If packet is marked as OK and packet type is TCP/IP or
767227569Sphilip	 * UDP/IP or other IP, then we can rely on the hardware checksums.
768227569Sphilip	 */
769227569Sphilip	switch (hdr_type) {
770227569Sphilip	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
771227569Sphilip		flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
772227569Sphilip		if (is_v6) {
773227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
774227569Sphilip			flags |= EFX_PKT_IPV6;
775227569Sphilip		} else {
776227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
777227569Sphilip			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
778227569Sphilip		}
779227569Sphilip		break;
780227569Sphilip
781227569Sphilip	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
782227569Sphilip		flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
783227569Sphilip		if (is_v6) {
784227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
785227569Sphilip			flags |= EFX_PKT_IPV6;
786227569Sphilip		} else {
787227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
788227569Sphilip			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
789227569Sphilip		}
790227569Sphilip		break;
791227569Sphilip
792227569Sphilip	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
793227569Sphilip		if (is_v6) {
794227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
795227569Sphilip			flags = EFX_PKT_IPV6;
796227569Sphilip		} else {
797227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
798227569Sphilip			flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
799227569Sphilip		}
800227569Sphilip		break;
801227569Sphilip
802227569Sphilip	case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
803227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
804227569Sphilip		flags = 0;
805227569Sphilip		break;
806227569Sphilip
807227569Sphilip	default:
808227569Sphilip		EFSYS_ASSERT(B_FALSE);
809227569Sphilip		flags = 0;
810227569Sphilip		break;
811227569Sphilip	}
812227569Sphilip
813294007Sarybchik#if EFSYS_OPT_RX_SCATTER
814227569Sphilip	/* Report scatter and header/lookahead split buffer flags */
815227569Sphilip	if (sop)
816227569Sphilip		flags |= EFX_PKT_START;
817227569Sphilip	if (jumbo_cont)
818227569Sphilip		flags |= EFX_PKT_CONT;
819294007Sarybchik#endif	/* EFSYS_OPT_RX_SCATTER */
820227569Sphilip
821227569Sphilip	/* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
822227569Sphilip	if (!ok) {
823301345Sarybchik		ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags);
824227569Sphilip		if (ignore) {
825227569Sphilip			EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
826227569Sphilip			    uint32_t, size, uint16_t, flags);
827227569Sphilip
828227569Sphilip			return (B_FALSE);
829227569Sphilip		}
830227569Sphilip	}
831227569Sphilip
832227569Sphilip	/* If we're not discarding the packet then it is ok */
833227569Sphilip	if (~flags & EFX_DISCARD)
834227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
835227569Sphilip
836227569Sphilip	/* Detect multicast packets that didn't match the filter */
837227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
838227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
839227569Sphilip
840227569Sphilip		if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
841227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
842227569Sphilip		} else {
843227569Sphilip			EFSYS_PROBE(mcast_mismatch);
844227569Sphilip			flags |= EFX_ADDR_MISMATCH;
845227569Sphilip		}
846227569Sphilip	} else {
847227569Sphilip		flags |= EFX_PKT_UNICAST;
848227569Sphilip	}
849227569Sphilip
850227569Sphilip	/*
851227569Sphilip	 * The packet parser in Siena can abort parsing packets under
852227569Sphilip	 * certain error conditions, setting the PKT_NOT_PARSED bit
853227569Sphilip	 * (which clears PKT_OK). If this is set, then don't trust
854227569Sphilip	 * the PKT_TYPE field.
855227569Sphilip	 */
856301330Sarybchik	if (!ok) {
857227569Sphilip		uint32_t parse_err;
858227569Sphilip
859227569Sphilip		parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
860227569Sphilip		if (parse_err != 0)
861227569Sphilip			flags |= EFX_CHECK_VLAN;
862227569Sphilip	}
863227569Sphilip
864227569Sphilip	if (~flags & EFX_CHECK_VLAN) {
865227569Sphilip		uint32_t pkt_type;
866227569Sphilip
867227569Sphilip		pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
868227569Sphilip		if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
869227569Sphilip			flags |= EFX_PKT_VLAN_TAGGED;
870227569Sphilip	}
871227569Sphilip
872227569Sphilip	EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
873227569Sphilip	    uint32_t, size, uint16_t, flags);
874227569Sphilip
875227569Sphilip	EFSYS_ASSERT(eecp->eec_rx != NULL);
876227569Sphilip	should_abort = eecp->eec_rx(arg, label, id, size, flags);
877227569Sphilip
878227569Sphilip	return (should_abort);
879227569Sphilip}
880227569Sphilip
881227569Sphilipstatic	__checkReturn	boolean_t
882301345Sarybchiksiena_ev_tx(
883227569Sphilip	__in		efx_evq_t *eep,
884227569Sphilip	__in		efx_qword_t *eqp,
885227569Sphilip	__in		const efx_ev_callbacks_t *eecp,
886227569Sphilip	__in_opt	void *arg)
887227569Sphilip{
888227569Sphilip	uint32_t id;
889227569Sphilip	uint32_t label;
890227569Sphilip	boolean_t should_abort;
891227569Sphilip
892227569Sphilip	EFX_EV_QSTAT_INCR(eep, EV_TX);
893227569Sphilip
894227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
895227569Sphilip	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
896227569Sphilip	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
897227569Sphilip	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
898227569Sphilip
899227569Sphilip		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
900227569Sphilip		label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
901227569Sphilip
902227569Sphilip		EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
903227569Sphilip
904227569Sphilip		EFSYS_ASSERT(eecp->eec_tx != NULL);
905227569Sphilip		should_abort = eecp->eec_tx(arg, label, id);
906227569Sphilip
907227569Sphilip		return (should_abort);
908227569Sphilip	}
909227569Sphilip
910227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
911227569Sphilip		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
912227569Sphilip			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
913227569Sphilip			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
914227569Sphilip
915227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
916227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
917227569Sphilip
918227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
919227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
920227569Sphilip
921227569Sphilip	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
922227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
923227569Sphilip
924227569Sphilip	EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
925227569Sphilip	return (B_FALSE);
926227569Sphilip}
927227569Sphilip
928227569Sphilipstatic	__checkReturn	boolean_t
929301345Sarybchiksiena_ev_global(
930227569Sphilip	__in		efx_evq_t *eep,
931227569Sphilip	__in		efx_qword_t *eqp,
932227569Sphilip	__in		const efx_ev_callbacks_t *eecp,
933227569Sphilip	__in_opt	void *arg)
934227569Sphilip{
935301331Sarybchik	_NOTE(ARGUNUSED(eqp, eecp, arg))
936227569Sphilip
937227569Sphilip	EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
938227569Sphilip
939301331Sarybchik	return (B_FALSE);
940227569Sphilip}
941227569Sphilip
942227569Sphilipstatic	__checkReturn	boolean_t
943301345Sarybchiksiena_ev_driver(
944227569Sphilip	__in		efx_evq_t *eep,
945227569Sphilip	__in		efx_qword_t *eqp,
946227569Sphilip	__in		const efx_ev_callbacks_t *eecp,
947227569Sphilip	__in_opt	void *arg)
948227569Sphilip{
949227569Sphilip	boolean_t should_abort;
950227569Sphilip
951227569Sphilip	EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
952227569Sphilip	should_abort = B_FALSE;
953227569Sphilip
954227569Sphilip	switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
955227569Sphilip	case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
956265884Sgnn		uint32_t txq_index;
957227569Sphilip
958227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
959227569Sphilip
960265884Sgnn		txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
961227569Sphilip
962265884Sgnn		EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
963227569Sphilip
964227569Sphilip		EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
965265884Sgnn		should_abort = eecp->eec_txq_flush_done(arg, txq_index);
966227569Sphilip
967227569Sphilip		break;
968227569Sphilip	}
969227569Sphilip	case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
970265884Sgnn		uint32_t rxq_index;
971227569Sphilip		uint32_t failed;
972227569Sphilip
973265884Sgnn		rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
974227569Sphilip		failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
975227569Sphilip
976227569Sphilip		EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
977227569Sphilip		EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
978227569Sphilip
979227569Sphilip		if (failed) {
980227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
981227569Sphilip
982265884Sgnn			EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
983227569Sphilip
984284555Sarybchik			should_abort = eecp->eec_rxq_flush_failed(arg,
985284555Sarybchik								    rxq_index);
986227569Sphilip		} else {
987227569Sphilip			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
988227569Sphilip
989265884Sgnn			EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
990227569Sphilip
991265884Sgnn			should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
992227569Sphilip		}
993227569Sphilip
994227569Sphilip		break;
995227569Sphilip	}
996227569Sphilip	case FSE_AZ_EVQ_INIT_DONE_EV:
997227569Sphilip		EFSYS_ASSERT(eecp->eec_initialized != NULL);
998227569Sphilip		should_abort = eecp->eec_initialized(arg);
999227569Sphilip
1000227569Sphilip		break;
1001227569Sphilip
1002227569Sphilip	case FSE_AZ_EVQ_NOT_EN_EV:
1003227569Sphilip		EFSYS_PROBE(evq_not_en);
1004227569Sphilip		break;
1005227569Sphilip
1006227569Sphilip	case FSE_AZ_SRM_UPD_DONE_EV: {
1007227569Sphilip		uint32_t code;
1008227569Sphilip
1009227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
1010227569Sphilip
1011227569Sphilip		code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1012227569Sphilip
1013227569Sphilip		EFSYS_ASSERT(eecp->eec_sram != NULL);
1014227569Sphilip		should_abort = eecp->eec_sram(arg, code);
1015227569Sphilip
1016227569Sphilip		break;
1017227569Sphilip	}
1018227569Sphilip	case FSE_AZ_WAKE_UP_EV: {
1019227569Sphilip		uint32_t id;
1020227569Sphilip
1021227569Sphilip		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1022227569Sphilip
1023227569Sphilip		EFSYS_ASSERT(eecp->eec_wake_up != NULL);
1024227569Sphilip		should_abort = eecp->eec_wake_up(arg, id);
1025227569Sphilip
1026227569Sphilip		break;
1027227569Sphilip	}
1028227569Sphilip	case FSE_AZ_TX_PKT_NON_TCP_UDP:
1029227569Sphilip		EFSYS_PROBE(tx_pkt_non_tcp_udp);
1030227569Sphilip		break;
1031227569Sphilip
1032227569Sphilip	case FSE_AZ_TIMER_EV: {
1033227569Sphilip		uint32_t id;
1034227569Sphilip
1035227569Sphilip		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1036227569Sphilip
1037227569Sphilip		EFSYS_ASSERT(eecp->eec_timer != NULL);
1038227569Sphilip		should_abort = eecp->eec_timer(arg, id);
1039227569Sphilip
1040227569Sphilip		break;
1041227569Sphilip	}
1042227569Sphilip	case FSE_AZ_RX_DSC_ERROR_EV:
1043227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
1044227569Sphilip
1045227569Sphilip		EFSYS_PROBE(rx_dsc_error);
1046227569Sphilip
1047227569Sphilip		EFSYS_ASSERT(eecp->eec_exception != NULL);
1048227569Sphilip		should_abort = eecp->eec_exception(arg,
1049227569Sphilip			EFX_EXCEPTION_RX_DSC_ERROR, 0);
1050227569Sphilip
1051227569Sphilip		break;
1052227569Sphilip
1053227569Sphilip	case FSE_AZ_TX_DSC_ERROR_EV:
1054227569Sphilip		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
1055227569Sphilip
1056227569Sphilip		EFSYS_PROBE(tx_dsc_error);
1057227569Sphilip
1058227569Sphilip		EFSYS_ASSERT(eecp->eec_exception != NULL);
1059227569Sphilip		should_abort = eecp->eec_exception(arg,
1060227569Sphilip			EFX_EXCEPTION_TX_DSC_ERROR, 0);
1061227569Sphilip
1062227569Sphilip		break;
1063227569Sphilip
1064227569Sphilip	default:
1065227569Sphilip		break;
1066227569Sphilip	}
1067227569Sphilip
1068227569Sphilip	return (should_abort);
1069227569Sphilip}
1070227569Sphilip
1071227569Sphilipstatic	__checkReturn	boolean_t
1072301345Sarybchiksiena_ev_drv_gen(
1073227569Sphilip	__in		efx_evq_t *eep,
1074227569Sphilip	__in		efx_qword_t *eqp,
1075227569Sphilip	__in		const efx_ev_callbacks_t *eecp,
1076227569Sphilip	__in_opt	void *arg)
1077227569Sphilip{
1078227569Sphilip	uint32_t data;
1079227569Sphilip	boolean_t should_abort;
1080227569Sphilip
1081227569Sphilip	EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
1082227569Sphilip
1083227569Sphilip	data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
1084227569Sphilip	if (data >= ((uint32_t)1 << 16)) {
1085227569Sphilip		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
1086227569Sphilip			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
1087227569Sphilip			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
1088227569Sphilip		return (B_TRUE);
1089227569Sphilip	}
1090227569Sphilip
1091227569Sphilip	EFSYS_ASSERT(eecp->eec_software != NULL);
1092227569Sphilip	should_abort = eecp->eec_software(arg, (uint16_t)data);
1093227569Sphilip
1094227569Sphilip	return (should_abort);
1095227569Sphilip}
1096227569Sphilip
1097227569Sphilip#if EFSYS_OPT_MCDI
1098227569Sphilip
1099227569Sphilipstatic	__checkReturn	boolean_t
1100301345Sarybchiksiena_ev_mcdi(
1101227569Sphilip	__in		efx_evq_t *eep,
1102227569Sphilip	__in		efx_qword_t *eqp,
1103227569Sphilip	__in		const efx_ev_callbacks_t *eecp,
1104227569Sphilip	__in_opt	void *arg)
1105227569Sphilip{
1106227569Sphilip	efx_nic_t *enp = eep->ee_enp;
1107311062Sarybchik	unsigned int code;
1108227569Sphilip	boolean_t should_abort = B_FALSE;
1109227569Sphilip
1110227569Sphilip	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
1111227569Sphilip
1112227569Sphilip	if (enp->en_family != EFX_FAMILY_SIENA)
1113227569Sphilip		goto out;
1114227569Sphilip
1115280555Sarybchik	EFSYS_ASSERT(eecp->eec_link_change != NULL);
1116280555Sarybchik	EFSYS_ASSERT(eecp->eec_exception != NULL);
1117280555Sarybchik#if EFSYS_OPT_MON_STATS
1118280555Sarybchik	EFSYS_ASSERT(eecp->eec_monitor != NULL);
1119280555Sarybchik#endif
1120280555Sarybchik
1121227569Sphilip	EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
1122227569Sphilip
1123227569Sphilip	code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
1124227569Sphilip	switch (code) {
1125227569Sphilip	case MCDI_EVENT_CODE_BADSSERT:
1126227569Sphilip		efx_mcdi_ev_death(enp, EINTR);
1127227569Sphilip		break;
1128227569Sphilip
1129227569Sphilip	case MCDI_EVENT_CODE_CMDDONE:
1130227569Sphilip		efx_mcdi_ev_cpl(enp,
1131284555Sarybchik		    MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
1132284555Sarybchik		    MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
1133284555Sarybchik		    MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
1134227569Sphilip		break;
1135227569Sphilip
1136227569Sphilip	case MCDI_EVENT_CODE_LINKCHANGE: {
1137227569Sphilip		efx_link_mode_t link_mode;
1138227569Sphilip
1139227569Sphilip		siena_phy_link_ev(enp, eqp, &link_mode);
1140227569Sphilip		should_abort = eecp->eec_link_change(arg, link_mode);
1141227569Sphilip		break;
1142227569Sphilip	}
1143227569Sphilip	case MCDI_EVENT_CODE_SENSOREVT: {
1144227569Sphilip#if EFSYS_OPT_MON_STATS
1145227569Sphilip		efx_mon_stat_t id;
1146227569Sphilip		efx_mon_stat_value_t value;
1147293927Sarybchik		efx_rc_t rc;
1148227569Sphilip
1149284555Sarybchik		if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
1150227569Sphilip			should_abort = eecp->eec_monitor(arg, id, value);
1151227569Sphilip		else if (rc == ENOTSUP) {
1152227569Sphilip			should_abort = eecp->eec_exception(arg,
1153227569Sphilip				EFX_EXCEPTION_UNKNOWN_SENSOREVT,
1154227569Sphilip				MCDI_EV_FIELD(eqp, DATA));
1155227569Sphilip		} else
1156227569Sphilip			EFSYS_ASSERT(rc == ENODEV);	/* Wrong port */
1157227569Sphilip#else
1158227569Sphilip		should_abort = B_FALSE;
1159227569Sphilip#endif
1160227569Sphilip		break;
1161227569Sphilip	}
1162227569Sphilip	case MCDI_EVENT_CODE_SCHEDERR:
1163227569Sphilip		/* Informational only */
1164227569Sphilip		break;
1165227569Sphilip
1166227569Sphilip	case MCDI_EVENT_CODE_REBOOT:
1167227569Sphilip		efx_mcdi_ev_death(enp, EIO);
1168227569Sphilip		break;
1169227569Sphilip
1170227569Sphilip	case MCDI_EVENT_CODE_MAC_STATS_DMA:
1171227569Sphilip#if EFSYS_OPT_MAC_STATS
1172227569Sphilip		if (eecp->eec_mac_stats != NULL) {
1173227569Sphilip			eecp->eec_mac_stats(arg,
1174227569Sphilip			    MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
1175227569Sphilip		}
1176227569Sphilip#endif
1177227569Sphilip		break;
1178227569Sphilip
1179227569Sphilip	case MCDI_EVENT_CODE_FWALERT: {
1180227569Sphilip		uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
1181227569Sphilip
1182227569Sphilip		if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
1183227569Sphilip			should_abort = eecp->eec_exception(arg,
1184227569Sphilip				EFX_EXCEPTION_FWALERT_SRAM,
1185227569Sphilip				MCDI_EV_FIELD(eqp, FWALERT_DATA));
1186227569Sphilip		else
1187227569Sphilip			should_abort = eecp->eec_exception(arg,
1188227569Sphilip				EFX_EXCEPTION_UNKNOWN_FWALERT,
1189227569Sphilip				MCDI_EV_FIELD(eqp, DATA));
1190227569Sphilip		break;
1191227569Sphilip	}
1192227569Sphilip
1193227569Sphilip	default:
1194227569Sphilip		EFSYS_PROBE1(mc_pcol_error, int, code);
1195227569Sphilip		break;
1196227569Sphilip	}
1197227569Sphilip
1198227569Sphilipout:
1199227569Sphilip	return (should_abort);
1200227569Sphilip}
1201227569Sphilip
1202280563Sarybchik#endif	/* EFSYS_OPT_MCDI */
1203227569Sphilip
1204293927Sarybchikstatic	__checkReturn	efx_rc_t
1205301345Sarybchiksiena_ev_qprime(
1206227569Sphilip	__in		efx_evq_t *eep,
1207227569Sphilip	__in		unsigned int count)
1208227569Sphilip{
1209227569Sphilip	efx_nic_t *enp = eep->ee_enp;
1210227569Sphilip	uint32_t rptr;
1211227569Sphilip	efx_dword_t dword;
1212227569Sphilip
1213227569Sphilip	rptr = count & eep->ee_mask;
1214227569Sphilip
1215227569Sphilip	EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
1216227569Sphilip
1217227569Sphilip	EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
1218227569Sphilip			    &dword, B_FALSE);
1219227569Sphilip
1220227569Sphilip	return (0);
1221227569Sphilip}
1222227569Sphilip
1223284555Sarybchikstatic		void
1224301345Sarybchiksiena_ev_qpost(
1225227569Sphilip	__in	efx_evq_t *eep,
1226227569Sphilip	__in	uint16_t data)
1227227569Sphilip{
1228227569Sphilip	efx_nic_t *enp = eep->ee_enp;
1229227569Sphilip	efx_qword_t ev;
1230227569Sphilip	efx_oword_t oword;
1231227569Sphilip
1232227569Sphilip	EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
1233227569Sphilip	    FSF_AZ_EV_DATA_DW0, (uint32_t)data);
1234227569Sphilip
1235227569Sphilip	EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
1236227569Sphilip	    EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
1237227569Sphilip	    EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
1238227569Sphilip
1239227569Sphilip	EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
1240227569Sphilip}
1241227569Sphilip
1242293927Sarybchikstatic	__checkReturn	efx_rc_t
1243301345Sarybchiksiena_ev_qmoderate(
1244227569Sphilip	__in		efx_evq_t *eep,
1245227569Sphilip	__in		unsigned int us)
1246227569Sphilip{
1247227569Sphilip	efx_nic_t *enp = eep->ee_enp;
1248280550Sarybchik	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1249227569Sphilip	unsigned int locked;
1250227569Sphilip	efx_dword_t dword;
1251293927Sarybchik	efx_rc_t rc;
1252227569Sphilip
1253280588Sarybchik	if (us > encp->enc_evq_timer_max_us) {
1254227569Sphilip		rc = EINVAL;
1255227569Sphilip		goto fail1;
1256227569Sphilip	}
1257227569Sphilip
1258227569Sphilip	/* If the value is zero then disable the timer */
1259227569Sphilip	if (us == 0) {
1260301330Sarybchik		EFX_POPULATE_DWORD_2(dword,
1261301330Sarybchik		    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
1262301330Sarybchik		    FRF_CZ_TC_TIMER_VAL, 0);
1263227569Sphilip	} else {
1264301984Sarybchik		unsigned int ticks;
1265227569Sphilip
1266301984Sarybchik		if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0)
1267301984Sarybchik			goto fail2;
1268227569Sphilip
1269301984Sarybchik		EFSYS_ASSERT(ticks > 0);
1270301330Sarybchik		EFX_POPULATE_DWORD_2(dword,
1271301330Sarybchik		    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
1272301984Sarybchik		    FRF_CZ_TC_TIMER_VAL, ticks - 1);
1273227569Sphilip	}
1274227569Sphilip
1275227569Sphilip	locked = (eep->ee_index == 0) ? 1 : 0;
1276227569Sphilip
1277227569Sphilip	EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
1278227569Sphilip	    eep->ee_index, &dword, locked);
1279227569Sphilip
1280227569Sphilip	return (0);
1281227569Sphilip
1282301984Sarybchikfail2:
1283301984Sarybchik	EFSYS_PROBE(fail2);
1284227569Sphilipfail1:
1285293927Sarybchik	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1286227569Sphilip
1287227569Sphilip	return (rc);
1288227569Sphilip}
1289227569Sphilip
1290293927Sarybchikstatic	__checkReturn	efx_rc_t
1291301345Sarybchiksiena_ev_qcreate(
1292227569Sphilip	__in		efx_nic_t *enp,
1293227569Sphilip	__in		unsigned int index,
1294227569Sphilip	__in		efsys_mem_t *esmp,
1295227569Sphilip	__in		size_t n,
1296227569Sphilip	__in		uint32_t id,
1297301980Sarybchik	__in		uint32_t us,
1298311070Sarybchik	__in		uint32_t flags,
1299284555Sarybchik	__in		efx_evq_t *eep)
1300227569Sphilip{
1301227569Sphilip	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1302227569Sphilip	uint32_t size;
1303227569Sphilip	efx_oword_t oword;
1304293927Sarybchik	efx_rc_t rc;
1305311094Sarybchik	boolean_t notify_mode;
1306227569Sphilip
1307301392Sarybchik	_NOTE(ARGUNUSED(esmp))
1308301392Sarybchik
1309284555Sarybchik	EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS));
1310284555Sarybchik	EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS));
1311227569Sphilip
1312284555Sarybchik	if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) {
1313227569Sphilip		rc = EINVAL;
1314227569Sphilip		goto fail1;
1315227569Sphilip	}
1316227569Sphilip	if (index >= encp->enc_evq_limit) {
1317227569Sphilip		rc = EINVAL;
1318227569Sphilip		goto fail2;
1319227569Sphilip	}
1320227569Sphilip#if EFSYS_OPT_RX_SCALE
1321227569Sphilip	if (enp->en_intr.ei_type == EFX_INTR_LINE &&
1322227569Sphilip	    index >= EFX_MAXRSS_LEGACY) {
1323227569Sphilip		rc = EINVAL;
1324227569Sphilip		goto fail3;
1325227569Sphilip	}
1326227569Sphilip#endif
1327227569Sphilip	for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
1328227569Sphilip	    size++)
1329227569Sphilip		if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS))
1330227569Sphilip			break;
1331227569Sphilip	if (id + (1 << size) >= encp->enc_buftbl_limit) {
1332227569Sphilip		rc = EINVAL;
1333227569Sphilip		goto fail4;
1334227569Sphilip	}
1335227569Sphilip
1336227569Sphilip	/* Set up the handler table */
1337301345Sarybchik	eep->ee_rx	= siena_ev_rx;
1338301345Sarybchik	eep->ee_tx	= siena_ev_tx;
1339301345Sarybchik	eep->ee_driver	= siena_ev_driver;
1340301345Sarybchik	eep->ee_global	= siena_ev_global;
1341301345Sarybchik	eep->ee_drv_gen	= siena_ev_drv_gen;
1342227569Sphilip#if EFSYS_OPT_MCDI
1343301345Sarybchik	eep->ee_mcdi	= siena_ev_mcdi;
1344280563Sarybchik#endif	/* EFSYS_OPT_MCDI */
1345227569Sphilip
1346311094Sarybchik	notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) !=
1347311094Sarybchik	    EFX_EVQ_FLAGS_NOTIFY_INTERRUPT);
1348311094Sarybchik
1349227569Sphilip	/* Set up the new event queue */
1350311094Sarybchik	EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1,
1351311094Sarybchik	    FRF_CZ_HOST_NOTIFY_MODE, notify_mode,
1352311094Sarybchik	    FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
1353301330Sarybchik	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
1354227569Sphilip
1355227569Sphilip	EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
1356227569Sphilip	    FRF_AZ_EVQ_BUF_BASE_ID, id);
1357227569Sphilip
1358284555Sarybchik	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
1359227569Sphilip
1360301980Sarybchik	/* Set initial interrupt moderation */
1361301980Sarybchik	siena_ev_qmoderate(eep, us);
1362301980Sarybchik
1363227569Sphilip	return (0);
1364227569Sphilip
1365227569Sphilipfail4:
1366227569Sphilip	EFSYS_PROBE(fail4);
1367227569Sphilip#if EFSYS_OPT_RX_SCALE
1368227569Sphilipfail3:
1369227569Sphilip	EFSYS_PROBE(fail3);
1370227569Sphilip#endif
1371227569Sphilipfail2:
1372227569Sphilip	EFSYS_PROBE(fail2);
1373227569Sphilipfail1:
1374293927Sarybchik	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1375227569Sphilip
1376227569Sphilip	return (rc);
1377227569Sphilip}
1378227569Sphilip
1379301324Sarybchik#endif /* EFSYS_OPT_SIENA */
1380284555Sarybchik
1381280510Sarybchik#if EFSYS_OPT_QSTATS
1382227569Sphilip#if EFSYS_OPT_NAMES
1383311059Sarybchik/* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock c0f3bc5083b40532 */
1384311059Sarybchikstatic const char * const __efx_ev_qstat_name[] = {
1385227569Sphilip	"all",
1386227569Sphilip	"rx",
1387227569Sphilip	"rx_ok",
1388227569Sphilip	"rx_frm_trunc",
1389227569Sphilip	"rx_tobe_disc",
1390227569Sphilip	"rx_pause_frm_err",
1391227569Sphilip	"rx_buf_owner_id_err",
1392227569Sphilip	"rx_ipv4_hdr_chksum_err",
1393227569Sphilip	"rx_tcp_udp_chksum_err",
1394227569Sphilip	"rx_eth_crc_err",
1395227569Sphilip	"rx_ip_frag_err",
1396227569Sphilip	"rx_mcast_pkt",
1397227569Sphilip	"rx_mcast_hash_match",
1398227569Sphilip	"rx_tcp_ipv4",
1399227569Sphilip	"rx_tcp_ipv6",
1400227569Sphilip	"rx_udp_ipv4",
1401227569Sphilip	"rx_udp_ipv6",
1402227569Sphilip	"rx_other_ipv4",
1403227569Sphilip	"rx_other_ipv6",
1404227569Sphilip	"rx_non_ip",
1405284555Sarybchik	"rx_batch",
1406227569Sphilip	"tx",
1407227569Sphilip	"tx_wq_ff_full",
1408227569Sphilip	"tx_pkt_err",
1409227569Sphilip	"tx_pkt_too_big",
1410227569Sphilip	"tx_unexpected",
1411227569Sphilip	"global",
1412227569Sphilip	"global_mnt",
1413227569Sphilip	"driver",
1414227569Sphilip	"driver_srm_upd_done",
1415227569Sphilip	"driver_tx_descq_fls_done",
1416227569Sphilip	"driver_rx_descq_fls_done",
1417227569Sphilip	"driver_rx_descq_fls_failed",
1418227569Sphilip	"driver_rx_dsc_error",
1419227569Sphilip	"driver_tx_dsc_error",
1420227569Sphilip	"drv_gen",
1421227569Sphilip	"mcdi_response",
1422227569Sphilip};
1423227569Sphilip/* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1424227569Sphilip
1425284555Sarybchik		const char *
1426227569Sphilipefx_ev_qstat_name(
1427227569Sphilip	__in	efx_nic_t *enp,
1428227569Sphilip	__in	unsigned int id)
1429227569Sphilip{
1430342501Sarybchik	_NOTE(ARGUNUSED(enp))
1431342501Sarybchik
1432227569Sphilip	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1433227569Sphilip	EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1434227569Sphilip
1435227569Sphilip	return (__efx_ev_qstat_name[id]);
1436227569Sphilip}
1437227569Sphilip#endif	/* EFSYS_OPT_NAMES */
1438280510Sarybchik#endif	/* EFSYS_OPT_QSTATS */
1439227569Sphilip
1440301324Sarybchik#if EFSYS_OPT_SIENA
1441284555Sarybchik
1442227569Sphilip#if EFSYS_OPT_QSTATS
1443284555Sarybchikstatic					void
1444301345Sarybchiksiena_ev_qstats_update(
1445227569Sphilip	__in				efx_evq_t *eep,
1446227569Sphilip	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
1447227569Sphilip{
1448227569Sphilip	unsigned int id;
1449227569Sphilip
1450227569Sphilip	for (id = 0; id < EV_NQSTATS; id++) {
1451227569Sphilip		efsys_stat_t *essp = &stat[id];
1452227569Sphilip
1453227569Sphilip		EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1454227569Sphilip		eep->ee_stat[id] = 0;
1455227569Sphilip	}
1456227569Sphilip}
1457227569Sphilip#endif	/* EFSYS_OPT_QSTATS */
1458227569Sphilip
1459284555Sarybchikstatic		void
1460301345Sarybchiksiena_ev_qdestroy(
1461227569Sphilip	__in	efx_evq_t *eep)
1462227569Sphilip{
1463227569Sphilip	efx_nic_t *enp = eep->ee_enp;
1464227569Sphilip	efx_oword_t oword;
1465227569Sphilip
1466227569Sphilip	/* Purge event queue */
1467227569Sphilip	EFX_ZERO_OWORD(oword);
1468227569Sphilip
1469227569Sphilip	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
1470284555Sarybchik	    eep->ee_index, &oword, B_TRUE);
1471227569Sphilip
1472301330Sarybchik	EFX_ZERO_OWORD(oword);
1473301330Sarybchik	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE);
1474227569Sphilip}
1475227569Sphilip
1476284555Sarybchikstatic		void
1477301345Sarybchiksiena_ev_fini(
1478227569Sphilip	__in	efx_nic_t *enp)
1479227569Sphilip{
1480284555Sarybchik	_NOTE(ARGUNUSED(enp))
1481284555Sarybchik}
1482227569Sphilip
1483301324Sarybchik#endif /* EFSYS_OPT_SIENA */
1484