1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2010-2016 Solarflare Communications Inc.
5 * All rights reserved.
6 *
7 * This software was developed in part by Philip Paeps under contract for
8 * Solarflare Communications, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright notice,
14 *    this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 *    this list of conditions and the following disclaimer in the documentation
17 *    and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * The views and conclusions contained in the software and documentation are
32 * those of the authors and should not be interpreted as representing official
33 * policies, either expressed or implied, of the FreeBSD Project.
34 */
35
36#include <sys/cdefs.h>
37__FBSDID("$FreeBSD$");
38
39#include "opt_rss.h"
40
41#include <sys/param.h>
42#include <sys/bus.h>
43#include <sys/kernel.h>
44#include <sys/malloc.h>
45#include <sys/queue.h>
46#include <sys/rman.h>
47#include <sys/syslog.h>
48#include <sys/taskqueue.h>
49
50#include <machine/bus.h>
51#include <machine/resource.h>
52
53#include <dev/pci/pcireg.h>
54#include <dev/pci/pcivar.h>
55
56#ifdef RSS
57#include <net/rss_config.h>
58#endif
59
60#include "common/efx.h"
61
62#include "sfxge.h"
63
64static int
65sfxge_intr_line_filter(void *arg)
66{
67	struct sfxge_evq *evq;
68	struct sfxge_softc *sc;
69	efx_nic_t *enp;
70	struct sfxge_intr *intr;
71	boolean_t fatal;
72	uint32_t qmask;
73
74	evq = (struct sfxge_evq *)arg;
75	sc = evq->sc;
76	enp = sc->enp;
77	intr = &sc->intr;
78
79	KASSERT(intr != NULL, ("intr == NULL"));
80	KASSERT(intr->type == EFX_INTR_LINE,
81	    ("intr->type != EFX_INTR_LINE"));
82
83	if (intr->state != SFXGE_INTR_STARTED)
84		return (FILTER_STRAY);
85
86	(void)efx_intr_status_line(enp, &fatal, &qmask);
87
88	if (fatal) {
89		(void) efx_intr_disable(enp);
90		(void) efx_intr_fatal(enp);
91		return (FILTER_HANDLED);
92	}
93
94	if (qmask != 0) {
95		intr->zero_count = 0;
96		return (FILTER_SCHEDULE_THREAD);
97	}
98
99	/* SF bug 15783: If the function is not asserting its IRQ and
100	 * we read the queue mask on the cycle before a flag is added
101	 * to the mask, this inhibits the function from asserting the
102	 * IRQ even though we don't see the flag set.  To work around
103	 * this, we must re-prime all event queues and report the IRQ
104	 * as handled when we see a mask of zero.  To allow for shared
105	 * IRQs, we don't repeat this if we see a mask of zero twice
106	 * or more in a row.
107	 */
108	if (intr->zero_count++ == 0) {
109		if (evq->init_state == SFXGE_EVQ_STARTED) {
110			if (efx_ev_qpending(evq->common, evq->read_ptr))
111				return (FILTER_SCHEDULE_THREAD);
112			efx_ev_qprime(evq->common, evq->read_ptr);
113			return (FILTER_HANDLED);
114		}
115	}
116
117	return (FILTER_STRAY);
118}
119
120static void
121sfxge_intr_line(void *arg)
122{
123	struct sfxge_evq *evq = arg;
124
125	(void)sfxge_ev_qpoll(evq);
126}
127
128static void
129sfxge_intr_message(void *arg)
130{
131	struct sfxge_evq *evq;
132	struct sfxge_softc *sc;
133	efx_nic_t *enp;
134	struct sfxge_intr *intr;
135	unsigned int index;
136	boolean_t fatal;
137
138	evq = (struct sfxge_evq *)arg;
139	sc = evq->sc;
140	enp = sc->enp;
141	intr = &sc->intr;
142	index = evq->index;
143
144	KASSERT(intr != NULL, ("intr == NULL"));
145	KASSERT(intr->type == EFX_INTR_MESSAGE,
146	    ("intr->type != EFX_INTR_MESSAGE"));
147
148	if (__predict_false(intr->state != SFXGE_INTR_STARTED))
149		return;
150
151	(void)efx_intr_status_message(enp, index, &fatal);
152
153	if (fatal) {
154		(void)efx_intr_disable(enp);
155		(void)efx_intr_fatal(enp);
156		return;
157	}
158
159	(void)sfxge_ev_qpoll(evq);
160}
161
162static int
163sfxge_intr_bus_enable(struct sfxge_softc *sc)
164{
165	struct sfxge_intr *intr;
166	struct sfxge_intr_hdl *table;
167	driver_filter_t *filter;
168	driver_intr_t *handler;
169	int index;
170	int err;
171
172	intr = &sc->intr;
173	table = intr->table;
174
175	switch (intr->type) {
176	case EFX_INTR_MESSAGE:
177		filter = NULL; /* not shared */
178		handler = sfxge_intr_message;
179		break;
180
181	case EFX_INTR_LINE:
182		filter = sfxge_intr_line_filter;
183		handler = sfxge_intr_line;
184		break;
185
186	default:
187		KASSERT(0, ("Invalid interrupt type"));
188		return (EINVAL);
189	}
190
191	/* Try to add the handlers */
192	for (index = 0; index < intr->n_alloc; index++) {
193		if ((err = bus_setup_intr(sc->dev, table[index].eih_res,
194			    INTR_MPSAFE|INTR_TYPE_NET, filter, handler,
195			    sc->evq[index], &table[index].eih_tag)) != 0) {
196			goto fail;
197		}
198#ifdef SFXGE_HAVE_DESCRIBE_INTR
199		if (intr->n_alloc > 1)
200			bus_describe_intr(sc->dev, table[index].eih_res,
201			    table[index].eih_tag, "%d", index);
202#endif
203#ifdef RSS
204		bus_bind_intr(sc->dev, table[index].eih_res,
205			      rss_getcpu(index));
206#else
207		bus_bind_intr(sc->dev, table[index].eih_res, index);
208#endif
209
210	}
211
212	return (0);
213
214fail:
215	/* Remove remaining handlers */
216	while (--index >= 0)
217		bus_teardown_intr(sc->dev, table[index].eih_res,
218		    table[index].eih_tag);
219
220	return (err);
221}
222
223static void
224sfxge_intr_bus_disable(struct sfxge_softc *sc)
225{
226	struct sfxge_intr *intr;
227	struct sfxge_intr_hdl *table;
228	int i;
229
230	intr = &sc->intr;
231	table = intr->table;
232
233	/* Remove all handlers */
234	for (i = 0; i < intr->n_alloc; i++)
235		bus_teardown_intr(sc->dev, table[i].eih_res,
236		    table[i].eih_tag);
237}
238
239static int
240sfxge_intr_alloc(struct sfxge_softc *sc, int count)
241{
242	device_t dev;
243	struct sfxge_intr_hdl *table;
244	struct sfxge_intr *intr;
245	struct resource *res;
246	int rid;
247	int error;
248	int i;
249
250	dev = sc->dev;
251	intr = &sc->intr;
252	error = 0;
253
254	table = malloc(count * sizeof(struct sfxge_intr_hdl),
255	    M_SFXGE, M_WAITOK);
256	intr->table = table;
257
258	for (i = 0; i < count; i++) {
259		rid = i + 1;
260		res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
261		    RF_SHAREABLE | RF_ACTIVE);
262		if (res == NULL) {
263			device_printf(dev, "Couldn't allocate interrupts for "
264			    "message %d\n", rid);
265			error = ENOMEM;
266			break;
267		}
268		table[i].eih_rid = rid;
269		table[i].eih_res = res;
270	}
271
272	if (error != 0) {
273		count = i - 1;
274		for (i = 0; i < count; i++)
275			bus_release_resource(dev, SYS_RES_IRQ,
276			    table[i].eih_rid, table[i].eih_res);
277	}
278
279	return (error);
280}
281
282static void
283sfxge_intr_teardown_msix(struct sfxge_softc *sc)
284{
285	device_t dev;
286	struct resource *resp;
287	int rid;
288
289	dev = sc->dev;
290	resp = sc->intr.msix_res;
291
292	rid = rman_get_rid(resp);
293	bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
294}
295
296static int
297sfxge_intr_setup_msix(struct sfxge_softc *sc)
298{
299	struct sfxge_intr *intr;
300	struct resource *resp;
301	device_t dev;
302	int count;
303	int rid;
304
305	dev = sc->dev;
306	intr = &sc->intr;
307
308	/* Check if MSI-X is available. */
309	count = pci_msix_count(dev);
310	if (count == 0)
311		return (EINVAL);
312
313	/* Do not try to allocate more than already estimated EVQ maximum */
314	KASSERT(sc->evq_max > 0, ("evq_max is zero"));
315	count = MIN(count, sc->evq_max);
316
317	rid = PCIR_BAR(4);
318	resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
319	if (resp == NULL)
320		return (ENOMEM);
321
322	if (pci_alloc_msix(dev, &count) != 0) {
323		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
324		return (ENOMEM);
325	}
326
327	/* Allocate interrupt handlers. */
328	if (sfxge_intr_alloc(sc, count) != 0) {
329		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
330		pci_release_msi(dev);
331		return (ENOMEM);
332	}
333
334	intr->type = EFX_INTR_MESSAGE;
335	intr->n_alloc = count;
336	intr->msix_res = resp;
337
338	return (0);
339}
340
341static int
342sfxge_intr_setup_msi(struct sfxge_softc *sc)
343{
344	struct sfxge_intr_hdl *table;
345	struct sfxge_intr *intr;
346	device_t dev;
347	int count;
348	int error;
349
350	dev = sc->dev;
351	intr = &sc->intr;
352	table = intr->table;
353
354	/*
355	 * Check if MSI is available.  All messages must be written to
356	 * the same address and on x86 this means the IRQs have the
357	 * same CPU affinity.  So we only ever allocate 1.
358	 */
359	count = pci_msi_count(dev) ? 1 : 0;
360	if (count == 0)
361		return (EINVAL);
362
363	if ((error = pci_alloc_msi(dev, &count)) != 0)
364		return (ENOMEM);
365
366	/* Allocate interrupt handler. */
367	if (sfxge_intr_alloc(sc, count) != 0) {
368		pci_release_msi(dev);
369		return (ENOMEM);
370	}
371
372	intr->type = EFX_INTR_MESSAGE;
373	intr->n_alloc = count;
374
375	return (0);
376}
377
378static int
379sfxge_intr_setup_fixed(struct sfxge_softc *sc)
380{
381	struct sfxge_intr_hdl *table;
382	struct sfxge_intr *intr;
383	struct resource *res;
384	device_t dev;
385	int rid;
386
387	dev = sc->dev;
388	intr = &sc->intr;
389
390	rid = 0;
391	res =  bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
392	    RF_SHAREABLE | RF_ACTIVE);
393	if (res == NULL)
394		return (ENOMEM);
395
396	table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK);
397	table[0].eih_rid = rid;
398	table[0].eih_res = res;
399
400	intr->type = EFX_INTR_LINE;
401	intr->n_alloc = 1;
402	intr->table = table;
403
404	return (0);
405}
406
407static const char *const __sfxge_err[] = {
408	"",
409	"SRAM out-of-bounds",
410	"Buffer ID out-of-bounds",
411	"Internal memory parity",
412	"Receive buffer ownership",
413	"Transmit buffer ownership",
414	"Receive descriptor ownership",
415	"Transmit descriptor ownership",
416	"Event queue ownership",
417	"Event queue FIFO overflow",
418	"Illegal address",
419	"SRAM parity"
420};
421
422void
423sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0,
424	  uint32_t dword1)
425{
426	struct sfxge_softc *sc = (struct sfxge_softc *)arg;
427	device_t dev = sc->dev;
428
429	log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)",
430	    device_get_name(dev), device_get_unit(dev),
431		__sfxge_err[code], dword1, dword0);
432}
433
434void
435sfxge_intr_stop(struct sfxge_softc *sc)
436{
437	struct sfxge_intr *intr;
438
439	intr = &sc->intr;
440
441	KASSERT(intr->state == SFXGE_INTR_STARTED,
442	    ("Interrupts not started"));
443
444	intr->state = SFXGE_INTR_INITIALIZED;
445
446	/* Disable interrupts at the NIC */
447	efx_intr_disable(sc->enp);
448
449	/* Disable interrupts at the bus */
450	sfxge_intr_bus_disable(sc);
451
452	/* Tear down common code interrupt bits. */
453	efx_intr_fini(sc->enp);
454}
455
456int
457sfxge_intr_start(struct sfxge_softc *sc)
458{
459	struct sfxge_intr *intr;
460	efsys_mem_t *esmp;
461	int rc;
462
463	intr = &sc->intr;
464	esmp = &intr->status;
465
466	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
467	    ("Interrupts not initialized"));
468
469	/* Zero the memory. */
470	(void)memset(esmp->esm_base, 0, EFX_INTR_SIZE);
471
472	/* Initialize common code interrupt bits. */
473	(void)efx_intr_init(sc->enp, intr->type, esmp);
474
475	/* Enable interrupts at the bus */
476	if ((rc = sfxge_intr_bus_enable(sc)) != 0)
477		goto fail;
478
479	intr->state = SFXGE_INTR_STARTED;
480
481	/* Enable interrupts at the NIC */
482	efx_intr_enable(sc->enp);
483
484	return (0);
485
486fail:
487	/* Tear down common code interrupt bits. */
488	efx_intr_fini(sc->enp);
489
490	intr->state = SFXGE_INTR_INITIALIZED;
491
492	return (rc);
493}
494
495void
496sfxge_intr_fini(struct sfxge_softc *sc)
497{
498	struct sfxge_intr_hdl *table;
499	struct sfxge_intr *intr;
500	efsys_mem_t *esmp;
501	device_t dev;
502	int i;
503
504	dev = sc->dev;
505	intr = &sc->intr;
506	esmp = &intr->status;
507	table = intr->table;
508
509	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
510	    ("intr->state != SFXGE_INTR_INITIALIZED"));
511
512	/* Free DMA memory. */
513	sfxge_dma_free(esmp);
514
515	/* Free interrupt handles. */
516	for (i = 0; i < intr->n_alloc; i++)
517		bus_release_resource(dev, SYS_RES_IRQ,
518		    table[i].eih_rid, table[i].eih_res);
519
520	if (table[0].eih_rid != 0)
521		pci_release_msi(dev);
522
523	if (intr->msix_res != NULL)
524		sfxge_intr_teardown_msix(sc);
525
526	/* Free the handle table */
527	free(table, M_SFXGE);
528	intr->table = NULL;
529	intr->n_alloc = 0;
530
531	/* Clear the interrupt type */
532	intr->type = EFX_INTR_INVALID;
533
534	intr->state = SFXGE_INTR_UNINITIALIZED;
535}
536
537int
538sfxge_intr_init(struct sfxge_softc *sc)
539{
540	device_t dev;
541	struct sfxge_intr *intr;
542	efsys_mem_t *esmp;
543	int rc;
544
545	dev = sc->dev;
546	intr = &sc->intr;
547	esmp = &intr->status;
548
549	KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED,
550	    ("Interrupts already initialized"));
551
552	/* Try to setup MSI-X or MSI interrupts if available. */
553	if ((rc = sfxge_intr_setup_msix(sc)) == 0)
554		device_printf(dev, "Using MSI-X interrupts\n");
555	else if ((rc = sfxge_intr_setup_msi(sc)) == 0)
556		device_printf(dev, "Using MSI interrupts\n");
557	else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) {
558		device_printf(dev, "Using fixed interrupts\n");
559	} else {
560		device_printf(dev, "Couldn't setup interrupts\n");
561		return (ENOMEM);
562	}
563
564	/* Set up DMA for interrupts. */
565	if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0)
566		return (ENOMEM);
567
568	intr->state = SFXGE_INTR_INITIALIZED;
569
570	return (0);
571}
572