1/*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include <sys/param.h>
34#include <sys/bus.h>
35#include <sys/rman.h>
36#include <sys/smp.h>
37#include <sys/syslog.h>
38
39#include <machine/bus.h>
40#include <machine/resource.h>
41
42#include <dev/pci/pcireg.h>
43#include <dev/pci/pcivar.h>
44
45#include "common/efx.h"
46
47#include "sfxge.h"
48
49static int
50sfxge_intr_line_filter(void *arg)
51{
52	struct sfxge_evq *evq;
53	struct sfxge_softc *sc;
54	efx_nic_t *enp;
55	struct sfxge_intr *intr;
56	boolean_t fatal;
57	uint32_t qmask;
58
59	evq = (struct sfxge_evq *)arg;
60	sc = evq->sc;
61	enp = sc->enp;
62	intr = &sc->intr;
63
64	KASSERT(intr != NULL, ("intr == NULL"));
65	KASSERT(intr->type == EFX_INTR_LINE,
66	    ("intr->type != EFX_INTR_LINE"));
67
68	if (intr->state != SFXGE_INTR_STARTED)
69		return FILTER_STRAY;
70
71	(void)efx_intr_status_line(enp, &fatal, &qmask);
72
73	if (fatal) {
74		(void) efx_intr_disable(enp);
75		(void) efx_intr_fatal(enp);
76		return FILTER_HANDLED;
77	}
78
79	if (qmask != 0) {
80		intr->zero_count = 0;
81		return FILTER_SCHEDULE_THREAD;
82	}
83
84	/* SF bug 15783: If the function is not asserting its IRQ and
85	 * we read the queue mask on the cycle before a flag is added
86	 * to the mask, this inhibits the function from asserting the
87	 * IRQ even though we don't see the flag set.  To work around
88	 * this, we must re-prime all event queues and report the IRQ
89	 * as handled when we see a mask of zero.  To allow for shared
90	 * IRQs, we don't repeat this if we see a mask of zero twice
91	 * or more in a row.
92	 */
93	if (intr->zero_count++ == 0) {
94		if (evq->init_state == SFXGE_EVQ_STARTED) {
95			if (efx_ev_qpending(evq->common, evq->read_ptr))
96				return FILTER_SCHEDULE_THREAD;
97			efx_ev_qprime(evq->common, evq->read_ptr);
98			return FILTER_HANDLED;
99		}
100	}
101
102	return FILTER_STRAY;
103}
104
105static void
106sfxge_intr_line(void *arg)
107{
108	struct sfxge_evq *evq = arg;
109	struct sfxge_softc *sc = evq->sc;
110
111	(void)sfxge_ev_qpoll(sc, 0);
112}
113
114static void
115sfxge_intr_message(void *arg)
116{
117	struct sfxge_evq *evq;
118	struct sfxge_softc *sc;
119	efx_nic_t *enp;
120	struct sfxge_intr *intr;
121	unsigned int index;
122	boolean_t fatal;
123
124	evq = (struct sfxge_evq *)arg;
125	sc = evq->sc;
126	enp = sc->enp;
127	intr = &sc->intr;
128	index = evq->index;
129
130	KASSERT(intr != NULL, ("intr == NULL"));
131	KASSERT(intr->type == EFX_INTR_MESSAGE,
132	    ("intr->type != EFX_INTR_MESSAGE"));
133
134	if (intr->state != SFXGE_INTR_STARTED)
135		return;
136
137	(void)efx_intr_status_message(enp, index, &fatal);
138
139	if (fatal) {
140		(void)efx_intr_disable(enp);
141		(void)efx_intr_fatal(enp);
142		return;
143	}
144
145	(void)sfxge_ev_qpoll(sc, index);
146}
147
148static int
149sfxge_intr_bus_enable(struct sfxge_softc *sc)
150{
151	struct sfxge_intr *intr;
152	struct sfxge_intr_hdl *table;
153	driver_filter_t *filter;
154	driver_intr_t *handler;
155	int index;
156	int err;
157
158	intr = &sc->intr;
159	table = intr->table;
160
161	switch (intr->type) {
162	case EFX_INTR_MESSAGE:
163		filter = NULL; /* not shared */
164		handler = sfxge_intr_message;
165		break;
166
167	case EFX_INTR_LINE:
168		filter = sfxge_intr_line_filter;
169		handler = sfxge_intr_line;
170		break;
171
172	default:
173		KASSERT(0, ("Invalid interrupt type"));
174		return EINVAL;
175	}
176
177	/* Try to add the handlers */
178	for (index = 0; index < intr->n_alloc; index++) {
179		if ((err = bus_setup_intr(sc->dev, table[index].eih_res,
180			    INTR_MPSAFE|INTR_TYPE_NET, filter, handler,
181			    sc->evq[index], &table[index].eih_tag)) != 0) {
182			goto fail;
183		}
184#ifdef SFXGE_HAVE_DESCRIBE_INTR
185		if (intr->n_alloc > 1)
186			bus_describe_intr(sc->dev, table[index].eih_res,
187			    table[index].eih_tag, "%d", index);
188#endif
189		bus_bind_intr(sc->dev, table[index].eih_res, index);
190
191	}
192
193	return (0);
194
195fail:
196	/* Remove remaining handlers */
197	while (--index >= 0)
198		bus_teardown_intr(sc->dev, table[index].eih_res,
199		    table[index].eih_tag);
200
201	return (err);
202}
203
204static void
205sfxge_intr_bus_disable(struct sfxge_softc *sc)
206{
207	struct sfxge_intr *intr;
208	struct sfxge_intr_hdl *table;
209	int i;
210
211	intr = &sc->intr;
212	table = intr->table;
213
214	/* Remove all handlers */
215	for (i = 0; i < intr->n_alloc; i++)
216		bus_teardown_intr(sc->dev, table[i].eih_res,
217		    table[i].eih_tag);
218}
219
220static int
221sfxge_intr_alloc(struct sfxge_softc *sc, int count)
222{
223	device_t dev;
224	struct sfxge_intr_hdl *table;
225	struct sfxge_intr *intr;
226	struct resource *res;
227	int rid;
228	int error;
229	int i;
230
231	dev = sc->dev;
232	intr = &sc->intr;
233	error = 0;
234
235	table = malloc(count * sizeof(struct sfxge_intr_hdl),
236	    M_SFXGE, M_WAITOK);
237	intr->table = table;
238
239	for (i = 0; i < count; i++) {
240		rid = i + 1;
241		res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
242		    RF_SHAREABLE | RF_ACTIVE);
243		if (res == NULL) {
244			device_printf(dev, "Couldn't allocate interrupts for "
245			    "message %d\n", rid);
246			error = ENOMEM;
247			break;
248		}
249		table[i].eih_rid = rid;
250		table[i].eih_res = res;
251	}
252
253	if (error) {
254		count = i - 1;
255		for (i = 0; i < count; i++)
256			bus_release_resource(dev, SYS_RES_IRQ,
257			    table[i].eih_rid, table[i].eih_res);
258	}
259
260	return (error);
261}
262
263static void
264sfxge_intr_teardown_msix(struct sfxge_softc *sc)
265{
266	device_t dev;
267	struct resource *resp;
268	int rid;
269
270	dev = sc->dev;
271	resp = sc->intr.msix_res;
272
273	rid = rman_get_rid(resp);
274	bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
275}
276
277static int
278sfxge_intr_setup_msix(struct sfxge_softc *sc)
279{
280	struct sfxge_intr *intr;
281	struct resource *resp;
282	device_t dev;
283	int count;
284	int rid;
285
286	dev = sc->dev;
287	intr = &sc->intr;
288
289	/* Check if MSI-X is available. */
290	count = pci_msix_count(dev);
291	if (count == 0)
292		return (EINVAL);
293
294	/* Limit the number of interrupts to the number of CPUs. */
295	if (count > mp_ncpus)
296		count = mp_ncpus;
297
298	/* Not very likely these days... */
299	if (count > EFX_MAXRSS)
300		count = EFX_MAXRSS;
301
302	rid = PCIR_BAR(4);
303	resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
304	if (resp == NULL)
305		return (ENOMEM);
306
307	if (pci_alloc_msix(dev, &count) != 0) {
308		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
309		return (ENOMEM);
310	}
311
312	/* Allocate interrupt handlers. */
313	if (sfxge_intr_alloc(sc, count) != 0) {
314		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
315		pci_release_msi(dev);
316		return (ENOMEM);
317	}
318
319	intr->type = EFX_INTR_MESSAGE;
320	intr->n_alloc = count;
321	intr->msix_res = resp;
322
323	return (0);
324}
325
326static int
327sfxge_intr_setup_msi(struct sfxge_softc *sc)
328{
329	struct sfxge_intr_hdl *table;
330	struct sfxge_intr *intr;
331	device_t dev;
332	int count;
333	int error;
334
335	dev = sc->dev;
336	intr = &sc->intr;
337	table = intr->table;
338
339	/*
340	 * Check if MSI is available.  All messages must be written to
341	 * the same address and on x86 this means the IRQs have the
342	 * same CPU affinity.  So we only ever allocate 1.
343	 */
344	count = pci_msi_count(dev) ? 1 : 0;
345	if (count == 0)
346		return (EINVAL);
347
348	if ((error = pci_alloc_msi(dev, &count)) != 0)
349		return (ENOMEM);
350
351	/* Allocate interrupt handler. */
352	if (sfxge_intr_alloc(sc, count) != 0) {
353		pci_release_msi(dev);
354		return (ENOMEM);
355	}
356
357	intr->type = EFX_INTR_MESSAGE;
358	intr->n_alloc = count;
359
360	return (0);
361}
362
363static int
364sfxge_intr_setup_fixed(struct sfxge_softc *sc)
365{
366	struct sfxge_intr_hdl *table;
367	struct sfxge_intr *intr;
368	struct resource *res;
369	device_t dev;
370	int rid;
371
372	dev = sc->dev;
373	intr = &sc->intr;
374
375	rid = 0;
376	res =  bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
377	    RF_SHAREABLE | RF_ACTIVE);
378	if (res == NULL)
379		return (ENOMEM);
380
381	table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK);
382	table[0].eih_rid = rid;
383	table[0].eih_res = res;
384
385	intr->type = EFX_INTR_LINE;
386	intr->n_alloc = 1;
387	intr->table = table;
388
389	return (0);
390}
391
392static const char *const __sfxge_err[] = {
393	"",
394	"SRAM out-of-bounds",
395	"Buffer ID out-of-bounds",
396	"Internal memory parity",
397	"Receive buffer ownership",
398	"Transmit buffer ownership",
399	"Receive descriptor ownership",
400	"Transmit descriptor ownership",
401	"Event queue ownership",
402	"Event queue FIFO overflow",
403	"Illegal address",
404	"SRAM parity"
405};
406
407void
408sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0,
409    uint32_t dword1)
410{
411	struct sfxge_softc *sc = (struct sfxge_softc *)arg;
412	device_t dev = sc->dev;
413
414	log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)",
415	    device_get_name(dev), device_get_unit(dev),
416		__sfxge_err[code], dword1, dword0);
417}
418
419void
420sfxge_intr_stop(struct sfxge_softc *sc)
421{
422	struct sfxge_intr *intr;
423
424	intr = &sc->intr;
425
426	KASSERT(intr->state == SFXGE_INTR_STARTED,
427	    ("Interrupts not started"));
428
429	intr->state = SFXGE_INTR_INITIALIZED;
430
431	/* Disable interrupts at the NIC */
432	efx_intr_disable(sc->enp);
433
434	/* Disable interrupts at the bus */
435	sfxge_intr_bus_disable(sc);
436
437	/* Tear down common code interrupt bits. */
438	efx_intr_fini(sc->enp);
439}
440
441int
442sfxge_intr_start(struct sfxge_softc *sc)
443{
444	struct sfxge_intr *intr;
445	efsys_mem_t *esmp;
446	int rc;
447
448	intr = &sc->intr;
449	esmp = &intr->status;
450
451	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
452	    ("Interrupts not initialized"));
453
454	/* Zero the memory. */
455	(void)memset(esmp->esm_base, 0, EFX_INTR_SIZE);
456
457	/* Initialize common code interrupt bits. */
458	(void)efx_intr_init(sc->enp, intr->type, esmp);
459
460	/* Enable interrupts at the bus */
461	if ((rc = sfxge_intr_bus_enable(sc)) != 0)
462		goto fail;
463
464	intr->state = SFXGE_INTR_STARTED;
465
466	/* Enable interrupts at the NIC */
467	efx_intr_enable(sc->enp);
468
469	return (0);
470
471fail:
472	/* Tear down common code interrupt bits. */
473	efx_intr_fini(sc->enp);
474
475	intr->state = SFXGE_INTR_INITIALIZED;
476
477	return (rc);
478}
479
480void
481sfxge_intr_fini(struct sfxge_softc *sc)
482{
483	struct sfxge_intr_hdl *table;
484	struct sfxge_intr *intr;
485	efsys_mem_t *esmp;
486	device_t dev;
487	int i;
488
489	dev = sc->dev;
490	intr = &sc->intr;
491	esmp = &intr->status;
492	table = intr->table;
493
494	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
495	    ("intr->state != SFXGE_INTR_INITIALIZED"));
496
497	/* Free DMA memory. */
498	sfxge_dma_free(esmp);
499
500	/* Free interrupt handles. */
501	for (i = 0; i < intr->n_alloc; i++)
502		bus_release_resource(dev, SYS_RES_IRQ,
503		    table[i].eih_rid, table[i].eih_res);
504
505	if (table[0].eih_rid != 0)
506		pci_release_msi(dev);
507
508	if (intr->msix_res != NULL)
509		sfxge_intr_teardown_msix(sc);
510
511	/* Free the handle table */
512	free(table, M_SFXGE);
513	intr->table = NULL;
514	intr->n_alloc = 0;
515
516	/* Clear the interrupt type */
517	intr->type = EFX_INTR_INVALID;
518
519	intr->state = SFXGE_INTR_UNINITIALIZED;
520}
521
522int
523sfxge_intr_init(struct sfxge_softc *sc)
524{
525	device_t dev;
526	struct sfxge_intr *intr;
527	efsys_mem_t *esmp;
528	int rc;
529
530	dev = sc->dev;
531	intr = &sc->intr;
532	esmp = &intr->status;
533
534	KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED,
535	    ("Interrupts already initialized"));
536
537	/* Try to setup MSI-X or MSI interrupts if available. */
538	if ((rc = sfxge_intr_setup_msix(sc)) == 0)
539		device_printf(dev, "Using MSI-X interrupts\n");
540	else if ((rc = sfxge_intr_setup_msi(sc)) == 0)
541		device_printf(dev, "Using MSI interrupts\n");
542	else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) {
543		device_printf(dev, "Using fixed interrupts\n");
544	} else {
545		device_printf(dev, "Couldn't setup interrupts\n");
546		return (ENOMEM);
547	}
548
549	/* Set up DMA for interrupts. */
550	if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0)
551		return (ENOMEM);
552
553	intr->state = SFXGE_INTR_INITIALIZED;
554
555	return (0);
556}
557