1/*-
2 * Copyright (c) 2010-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright notice,
12 *    this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 *    this list of conditions and the following disclaimer in the documentation
15 *    and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * The views and conclusions contained in the software and documentation are
30 * those of the authors and should not be interpreted as representing official
31 * policies, either expressed or implied, of the FreeBSD Project.
32 */
33
34#include <sys/cdefs.h>
35__FBSDID("$FreeBSD$");
36
37#include "opt_rss.h"
38
39#include <sys/param.h>
40#include <sys/bus.h>
41#include <sys/kernel.h>
42#include <sys/malloc.h>
43#include <sys/queue.h>
44#include <sys/rman.h>
45#include <sys/syslog.h>
46#include <sys/taskqueue.h>
47
48#include <machine/bus.h>
49#include <machine/resource.h>
50
51#include <dev/pci/pcireg.h>
52#include <dev/pci/pcivar.h>
53
54#ifdef RSS
55#include <net/rss_config.h>
56#endif
57
58#include "common/efx.h"
59
60#include "sfxge.h"
61
62static int
63sfxge_intr_line_filter(void *arg)
64{
65	struct sfxge_evq *evq;
66	struct sfxge_softc *sc;
67	efx_nic_t *enp;
68	struct sfxge_intr *intr;
69	boolean_t fatal;
70	uint32_t qmask;
71
72	evq = (struct sfxge_evq *)arg;
73	sc = evq->sc;
74	enp = sc->enp;
75	intr = &sc->intr;
76
77	KASSERT(intr != NULL, ("intr == NULL"));
78	KASSERT(intr->type == EFX_INTR_LINE,
79	    ("intr->type != EFX_INTR_LINE"));
80
81	if (intr->state != SFXGE_INTR_STARTED)
82		return (FILTER_STRAY);
83
84	(void)efx_intr_status_line(enp, &fatal, &qmask);
85
86	if (fatal) {
87		(void) efx_intr_disable(enp);
88		(void) efx_intr_fatal(enp);
89		return (FILTER_HANDLED);
90	}
91
92	if (qmask != 0) {
93		intr->zero_count = 0;
94		return (FILTER_SCHEDULE_THREAD);
95	}
96
97	/* SF bug 15783: If the function is not asserting its IRQ and
98	 * we read the queue mask on the cycle before a flag is added
99	 * to the mask, this inhibits the function from asserting the
100	 * IRQ even though we don't see the flag set.  To work around
101	 * this, we must re-prime all event queues and report the IRQ
102	 * as handled when we see a mask of zero.  To allow for shared
103	 * IRQs, we don't repeat this if we see a mask of zero twice
104	 * or more in a row.
105	 */
106	if (intr->zero_count++ == 0) {
107		if (evq->init_state == SFXGE_EVQ_STARTED) {
108			if (efx_ev_qpending(evq->common, evq->read_ptr))
109				return (FILTER_SCHEDULE_THREAD);
110			efx_ev_qprime(evq->common, evq->read_ptr);
111			return (FILTER_HANDLED);
112		}
113	}
114
115	return (FILTER_STRAY);
116}
117
118static void
119sfxge_intr_line(void *arg)
120{
121	struct sfxge_evq *evq = arg;
122
123	(void)sfxge_ev_qpoll(evq);
124}
125
126static void
127sfxge_intr_message(void *arg)
128{
129	struct sfxge_evq *evq;
130	struct sfxge_softc *sc;
131	efx_nic_t *enp;
132	struct sfxge_intr *intr;
133	unsigned int index;
134	boolean_t fatal;
135
136	evq = (struct sfxge_evq *)arg;
137	sc = evq->sc;
138	enp = sc->enp;
139	intr = &sc->intr;
140	index = evq->index;
141
142	KASSERT(intr != NULL, ("intr == NULL"));
143	KASSERT(intr->type == EFX_INTR_MESSAGE,
144	    ("intr->type != EFX_INTR_MESSAGE"));
145
146	if (__predict_false(intr->state != SFXGE_INTR_STARTED))
147		return;
148
149	(void)efx_intr_status_message(enp, index, &fatal);
150
151	if (fatal) {
152		(void)efx_intr_disable(enp);
153		(void)efx_intr_fatal(enp);
154		return;
155	}
156
157	(void)sfxge_ev_qpoll(evq);
158}
159
160static int
161sfxge_intr_bus_enable(struct sfxge_softc *sc)
162{
163	struct sfxge_intr *intr;
164	struct sfxge_intr_hdl *table;
165	driver_filter_t *filter;
166	driver_intr_t *handler;
167	int index;
168	int err;
169
170	intr = &sc->intr;
171	table = intr->table;
172
173	switch (intr->type) {
174	case EFX_INTR_MESSAGE:
175		filter = NULL; /* not shared */
176		handler = sfxge_intr_message;
177		break;
178
179	case EFX_INTR_LINE:
180		filter = sfxge_intr_line_filter;
181		handler = sfxge_intr_line;
182		break;
183
184	default:
185		KASSERT(0, ("Invalid interrupt type"));
186		return (EINVAL);
187	}
188
189	/* Try to add the handlers */
190	for (index = 0; index < intr->n_alloc; index++) {
191		if ((err = bus_setup_intr(sc->dev, table[index].eih_res,
192			    INTR_MPSAFE|INTR_TYPE_NET, filter, handler,
193			    sc->evq[index], &table[index].eih_tag)) != 0) {
194			goto fail;
195		}
196#ifdef SFXGE_HAVE_DESCRIBE_INTR
197		if (intr->n_alloc > 1)
198			bus_describe_intr(sc->dev, table[index].eih_res,
199			    table[index].eih_tag, "%d", index);
200#endif
201#ifdef RSS
202		bus_bind_intr(sc->dev, table[index].eih_res,
203			      rss_getcpu(index));
204#else
205		bus_bind_intr(sc->dev, table[index].eih_res, index);
206#endif
207
208	}
209
210	return (0);
211
212fail:
213	/* Remove remaining handlers */
214	while (--index >= 0)
215		bus_teardown_intr(sc->dev, table[index].eih_res,
216		    table[index].eih_tag);
217
218	return (err);
219}
220
221static void
222sfxge_intr_bus_disable(struct sfxge_softc *sc)
223{
224	struct sfxge_intr *intr;
225	struct sfxge_intr_hdl *table;
226	int i;
227
228	intr = &sc->intr;
229	table = intr->table;
230
231	/* Remove all handlers */
232	for (i = 0; i < intr->n_alloc; i++)
233		bus_teardown_intr(sc->dev, table[i].eih_res,
234		    table[i].eih_tag);
235}
236
237static int
238sfxge_intr_alloc(struct sfxge_softc *sc, int count)
239{
240	device_t dev;
241	struct sfxge_intr_hdl *table;
242	struct sfxge_intr *intr;
243	struct resource *res;
244	int rid;
245	int error;
246	int i;
247
248	dev = sc->dev;
249	intr = &sc->intr;
250	error = 0;
251
252	table = malloc(count * sizeof(struct sfxge_intr_hdl),
253	    M_SFXGE, M_WAITOK);
254	intr->table = table;
255
256	for (i = 0; i < count; i++) {
257		rid = i + 1;
258		res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
259		    RF_SHAREABLE | RF_ACTIVE);
260		if (res == NULL) {
261			device_printf(dev, "Couldn't allocate interrupts for "
262			    "message %d\n", rid);
263			error = ENOMEM;
264			break;
265		}
266		table[i].eih_rid = rid;
267		table[i].eih_res = res;
268	}
269
270	if (error != 0) {
271		count = i - 1;
272		for (i = 0; i < count; i++)
273			bus_release_resource(dev, SYS_RES_IRQ,
274			    table[i].eih_rid, table[i].eih_res);
275	}
276
277	return (error);
278}
279
280static void
281sfxge_intr_teardown_msix(struct sfxge_softc *sc)
282{
283	device_t dev;
284	struct resource *resp;
285	int rid;
286
287	dev = sc->dev;
288	resp = sc->intr.msix_res;
289
290	rid = rman_get_rid(resp);
291	bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
292}
293
294static int
295sfxge_intr_setup_msix(struct sfxge_softc *sc)
296{
297	struct sfxge_intr *intr;
298	struct resource *resp;
299	device_t dev;
300	int count;
301	int rid;
302
303	dev = sc->dev;
304	intr = &sc->intr;
305
306	/* Check if MSI-X is available. */
307	count = pci_msix_count(dev);
308	if (count == 0)
309		return (EINVAL);
310
311	/* Do not try to allocate more than already estimated EVQ maximum */
312	KASSERT(sc->evq_max > 0, ("evq_max is zero"));
313	count = MIN(count, sc->evq_max);
314
315	rid = PCIR_BAR(4);
316	resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
317	if (resp == NULL)
318		return (ENOMEM);
319
320	if (pci_alloc_msix(dev, &count) != 0) {
321		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
322		return (ENOMEM);
323	}
324
325	/* Allocate interrupt handlers. */
326	if (sfxge_intr_alloc(sc, count) != 0) {
327		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
328		pci_release_msi(dev);
329		return (ENOMEM);
330	}
331
332	intr->type = EFX_INTR_MESSAGE;
333	intr->n_alloc = count;
334	intr->msix_res = resp;
335
336	return (0);
337}
338
339static int
340sfxge_intr_setup_msi(struct sfxge_softc *sc)
341{
342	struct sfxge_intr_hdl *table;
343	struct sfxge_intr *intr;
344	device_t dev;
345	int count;
346	int error;
347
348	dev = sc->dev;
349	intr = &sc->intr;
350	table = intr->table;
351
352	/*
353	 * Check if MSI is available.  All messages must be written to
354	 * the same address and on x86 this means the IRQs have the
355	 * same CPU affinity.  So we only ever allocate 1.
356	 */
357	count = pci_msi_count(dev) ? 1 : 0;
358	if (count == 0)
359		return (EINVAL);
360
361	if ((error = pci_alloc_msi(dev, &count)) != 0)
362		return (ENOMEM);
363
364	/* Allocate interrupt handler. */
365	if (sfxge_intr_alloc(sc, count) != 0) {
366		pci_release_msi(dev);
367		return (ENOMEM);
368	}
369
370	intr->type = EFX_INTR_MESSAGE;
371	intr->n_alloc = count;
372
373	return (0);
374}
375
376static int
377sfxge_intr_setup_fixed(struct sfxge_softc *sc)
378{
379	struct sfxge_intr_hdl *table;
380	struct sfxge_intr *intr;
381	struct resource *res;
382	device_t dev;
383	int rid;
384
385	dev = sc->dev;
386	intr = &sc->intr;
387
388	rid = 0;
389	res =  bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
390	    RF_SHAREABLE | RF_ACTIVE);
391	if (res == NULL)
392		return (ENOMEM);
393
394	table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK);
395	table[0].eih_rid = rid;
396	table[0].eih_res = res;
397
398	intr->type = EFX_INTR_LINE;
399	intr->n_alloc = 1;
400	intr->table = table;
401
402	return (0);
403}
404
405static const char *const __sfxge_err[] = {
406	"",
407	"SRAM out-of-bounds",
408	"Buffer ID out-of-bounds",
409	"Internal memory parity",
410	"Receive buffer ownership",
411	"Transmit buffer ownership",
412	"Receive descriptor ownership",
413	"Transmit descriptor ownership",
414	"Event queue ownership",
415	"Event queue FIFO overflow",
416	"Illegal address",
417	"SRAM parity"
418};
419
420void
421sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0,
422	  uint32_t dword1)
423{
424	struct sfxge_softc *sc = (struct sfxge_softc *)arg;
425	device_t dev = sc->dev;
426
427	log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)",
428	    device_get_name(dev), device_get_unit(dev),
429		__sfxge_err[code], dword1, dword0);
430}
431
432void
433sfxge_intr_stop(struct sfxge_softc *sc)
434{
435	struct sfxge_intr *intr;
436
437	intr = &sc->intr;
438
439	KASSERT(intr->state == SFXGE_INTR_STARTED,
440	    ("Interrupts not started"));
441
442	intr->state = SFXGE_INTR_INITIALIZED;
443
444	/* Disable interrupts at the NIC */
445	efx_intr_disable(sc->enp);
446
447	/* Disable interrupts at the bus */
448	sfxge_intr_bus_disable(sc);
449
450	/* Tear down common code interrupt bits. */
451	efx_intr_fini(sc->enp);
452}
453
454int
455sfxge_intr_start(struct sfxge_softc *sc)
456{
457	struct sfxge_intr *intr;
458	efsys_mem_t *esmp;
459	int rc;
460
461	intr = &sc->intr;
462	esmp = &intr->status;
463
464	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
465	    ("Interrupts not initialized"));
466
467	/* Zero the memory. */
468	(void)memset(esmp->esm_base, 0, EFX_INTR_SIZE);
469
470	/* Initialize common code interrupt bits. */
471	(void)efx_intr_init(sc->enp, intr->type, esmp);
472
473	/* Enable interrupts at the bus */
474	if ((rc = sfxge_intr_bus_enable(sc)) != 0)
475		goto fail;
476
477	intr->state = SFXGE_INTR_STARTED;
478
479	/* Enable interrupts at the NIC */
480	efx_intr_enable(sc->enp);
481
482	return (0);
483
484fail:
485	/* Tear down common code interrupt bits. */
486	efx_intr_fini(sc->enp);
487
488	intr->state = SFXGE_INTR_INITIALIZED;
489
490	return (rc);
491}
492
493void
494sfxge_intr_fini(struct sfxge_softc *sc)
495{
496	struct sfxge_intr_hdl *table;
497	struct sfxge_intr *intr;
498	efsys_mem_t *esmp;
499	device_t dev;
500	int i;
501
502	dev = sc->dev;
503	intr = &sc->intr;
504	esmp = &intr->status;
505	table = intr->table;
506
507	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
508	    ("intr->state != SFXGE_INTR_INITIALIZED"));
509
510	/* Free DMA memory. */
511	sfxge_dma_free(esmp);
512
513	/* Free interrupt handles. */
514	for (i = 0; i < intr->n_alloc; i++)
515		bus_release_resource(dev, SYS_RES_IRQ,
516		    table[i].eih_rid, table[i].eih_res);
517
518	if (table[0].eih_rid != 0)
519		pci_release_msi(dev);
520
521	if (intr->msix_res != NULL)
522		sfxge_intr_teardown_msix(sc);
523
524	/* Free the handle table */
525	free(table, M_SFXGE);
526	intr->table = NULL;
527	intr->n_alloc = 0;
528
529	/* Clear the interrupt type */
530	intr->type = EFX_INTR_INVALID;
531
532	intr->state = SFXGE_INTR_UNINITIALIZED;
533}
534
535int
536sfxge_intr_init(struct sfxge_softc *sc)
537{
538	device_t dev;
539	struct sfxge_intr *intr;
540	efsys_mem_t *esmp;
541	int rc;
542
543	dev = sc->dev;
544	intr = &sc->intr;
545	esmp = &intr->status;
546
547	KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED,
548	    ("Interrupts already initialized"));
549
550	/* Try to setup MSI-X or MSI interrupts if available. */
551	if ((rc = sfxge_intr_setup_msix(sc)) == 0)
552		device_printf(dev, "Using MSI-X interrupts\n");
553	else if ((rc = sfxge_intr_setup_msi(sc)) == 0)
554		device_printf(dev, "Using MSI interrupts\n");
555	else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) {
556		device_printf(dev, "Using fixed interrupts\n");
557	} else {
558		device_printf(dev, "Couldn't setup interrupts\n");
559		return (ENOMEM);
560	}
561
562	/* Set up DMA for interrupts. */
563	if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0)
564		return (ENOMEM);
565
566	intr->state = SFXGE_INTR_INITIALIZED;
567
568	return (0);
569}
570