1/**************************************************************************
2
3Copyright (c) 2007-2009, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_main.c 237263 2012-06-19 07:34:13Z np $");
32
33#include "opt_inet.h"
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/kernel.h>
38#include <sys/bus.h>
39#include <sys/module.h>
40#include <sys/pciio.h>
41#include <sys/conf.h>
42#include <machine/bus.h>
43#include <machine/resource.h>
44#include <sys/bus_dma.h>
45#include <sys/ktr.h>
46#include <sys/rman.h>
47#include <sys/ioccom.h>
48#include <sys/mbuf.h>
49#include <sys/linker.h>
50#include <sys/firmware.h>
51#include <sys/socket.h>
52#include <sys/sockio.h>
53#include <sys/smp.h>
54#include <sys/sysctl.h>
55#include <sys/syslog.h>
56#include <sys/queue.h>
57#include <sys/taskqueue.h>
58#include <sys/proc.h>
59
60#include <net/bpf.h>
61#include <net/ethernet.h>
62#include <net/if.h>
63#include <net/if_arp.h>
64#include <net/if_dl.h>
65#include <net/if_media.h>
66#include <net/if_types.h>
67#include <net/if_vlan_var.h>
68
69#include <netinet/in_systm.h>
70#include <netinet/in.h>
71#include <netinet/if_ether.h>
72#include <netinet/ip.h>
73#include <netinet/ip.h>
74#include <netinet/tcp.h>
75#include <netinet/udp.h>
76
77#include <dev/pci/pcireg.h>
78#include <dev/pci/pcivar.h>
79#include <dev/pci/pci_private.h>
80
81#include <cxgb_include.h>
82
83#ifdef PRIV_SUPPORTED
84#include <sys/priv.h>
85#endif
86
87static int cxgb_setup_interrupts(adapter_t *);
88static void cxgb_teardown_interrupts(adapter_t *);
89static void cxgb_init(void *);
90static int cxgb_init_locked(struct port_info *);
91static int cxgb_uninit_locked(struct port_info *);
92static int cxgb_uninit_synchronized(struct port_info *);
93static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
94static int cxgb_media_change(struct ifnet *);
95static int cxgb_ifm_type(int);
96static void cxgb_build_medialist(struct port_info *);
97static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
98static int setup_sge_qsets(adapter_t *);
99static void cxgb_async_intr(void *);
100static void cxgb_tick_handler(void *, int);
101static void cxgb_tick(void *);
102static void link_check_callout(void *);
103static void check_link_status(void *, int);
104static void setup_rss(adapter_t *sc);
105static int alloc_filters(struct adapter *);
106static int setup_hw_filters(struct adapter *);
107static int set_filter(struct adapter *, int, const struct filter_info *);
108static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
109 unsigned int, u64, u64);
110static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
111 unsigned int, u64, u64);
112#ifdef TCP_OFFLOAD
113static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
114#endif
115
116/* Attachment glue for the PCI controller end of the device. Each port of
117 * the device is attached separately, as defined later.
118 */
119static int cxgb_controller_probe(device_t);
120static int cxgb_controller_attach(device_t);
121static int cxgb_controller_detach(device_t);
122static void cxgb_free(struct adapter *);
123static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
124 unsigned int end);
125static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
126static int cxgb_get_regs_len(void);
127static void touch_bars(device_t dev);
128static void cxgb_update_mac_settings(struct port_info *p);
129#ifdef TCP_OFFLOAD
130static int toe_capability(struct port_info *, int);
131#endif
132
133static device_method_t cxgb_controller_methods[] = {
134 DEVMETHOD(device_probe, cxgb_controller_probe),
135 DEVMETHOD(device_attach, cxgb_controller_attach),
136 DEVMETHOD(device_detach, cxgb_controller_detach),
137
138 DEVMETHOD_END
139};
140
141static driver_t cxgb_controller_driver = {
142 "cxgbc",
143 cxgb_controller_methods,
144 sizeof(struct adapter)
145};
146
147static int cxgbc_mod_event(module_t, int, void *);
148static devclass_t cxgb_controller_devclass;
149DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
150 cxgbc_mod_event, 0);
151MODULE_VERSION(cxgbc, 1);
152
153/*
154 * Attachment glue for the ports. Attachment is done directly to the
155 * controller device.
156 */
157static int cxgb_port_probe(device_t);
158static int cxgb_port_attach(device_t);
159static int cxgb_port_detach(device_t);
160
161static device_method_t cxgb_port_methods[] = {
162 DEVMETHOD(device_probe, cxgb_port_probe),
163 DEVMETHOD(device_attach, cxgb_port_attach),
164 DEVMETHOD(device_detach, cxgb_port_detach),
165 { 0, 0 }
166};
167
168static driver_t cxgb_port_driver = {
169 "cxgb",
170 cxgb_port_methods,
171 0
172};
173
174static d_ioctl_t cxgb_extension_ioctl;
175static d_open_t cxgb_extension_open;
176static d_close_t cxgb_extension_close;
177
178static struct cdevsw cxgb_cdevsw = {
179 .d_version = D_VERSION,
180 .d_flags = 0,
181 .d_open = cxgb_extension_open,
182 .d_close = cxgb_extension_close,
183 .d_ioctl = cxgb_extension_ioctl,
184 .d_name = "cxgb",
185};
186
187static devclass_t cxgb_port_devclass;
188DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
189MODULE_VERSION(cxgb, 1);
190
191static struct mtx t3_list_lock;
192static SLIST_HEAD(, adapter) t3_list;
193#ifdef TCP_OFFLOAD
194static struct mtx t3_uld_list_lock;
195static SLIST_HEAD(, uld_info) t3_uld_list;
196#endif
197
198/*
199 * The driver uses the best interrupt scheme available on a platform in the
200 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
201 * of these schemes the driver may consider as follows:
202 *
203 * msi = 2: choose from among all three options
204 * msi = 1 : only consider MSI and pin interrupts
205 * msi = 0: force pin interrupts
206 */
207static int msi_allowed = 2;
208
209TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
210SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
211SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
212 "MSI-X, MSI, INTx selector");
213
214/*
215 * The driver uses an auto-queue algorithm by default.
216 * To disable it and force a single queue-set per port, use multiq = 0
217 */
218static int multiq = 1;
219TUNABLE_INT("hw.cxgb.multiq", &multiq);
220SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
221 "use min(ncpus/ports, 8) queue-sets per port");
222
223/*
224 * By default the driver will not update the firmware unless
225 * it was compiled against a newer version
226 *
227 */
228static int force_fw_update = 0;
229TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update);
230SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
231 "update firmware even if up to date");
232
233int cxgb_use_16k_clusters = -1;
234TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
235SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
236 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
237
238static int nfilters = -1;
239TUNABLE_INT("hw.cxgb.nfilters", &nfilters);
240SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
241 &nfilters, 0, "max number of entries in the filter table");
242
243enum {
244 MAX_TXQ_ENTRIES = 16384,
245 MAX_CTRL_TXQ_ENTRIES = 1024,
246 MAX_RSPQ_ENTRIES = 16384,
247 MAX_RX_BUFFERS = 16384,
248 MAX_RX_JUMBO_BUFFERS = 16384,
249 MIN_TXQ_ENTRIES = 4,
250 MIN_CTRL_TXQ_ENTRIES = 4,
251 MIN_RSPQ_ENTRIES = 32,
252 MIN_FL_ENTRIES = 32,
253 MIN_FL_JUMBO_ENTRIES = 32
254};
255
256struct filter_info {
257 u32 sip;
258 u32 sip_mask;
259 u32 dip;
260 u16 sport;
261 u16 dport;
262 u32 vlan:12;
263 u32 vlan_prio:3;
264 u32 mac_hit:1;
265 u32 mac_idx:4;
266 u32 mac_vld:1;
267 u32 pkt_type:2;
268 u32 report_filter_id:1;
269 u32 pass:1;
270 u32 rss:1;
271 u32 qset:3;
272 u32 locked:1;
273 u32 valid:1;
274};
275
276enum { FILTER_NO_VLAN_PRI = 7 };
277
278#define EEPROM_MAGIC 0x38E2F10C
279
280#define PORT_MASK ((1 << MAX_NPORTS) - 1)
281
282/* Table for probing the cards. The desc field isn't actually used */
283struct cxgb_ident {
284 uint16_t vendor;
285 uint16_t device;
286 int index;
287 char *desc;
288} cxgb_identifiers[] = {
289 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
290 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
291 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
292 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
293 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
294 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
295 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
296 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
297 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
298 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
299 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
300 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
301 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
302 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
303 {0, 0, 0, NULL}
304};
305
306static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
307
308
309static __inline char
310t3rev2char(struct adapter *adapter)
311{
312 char rev = 'z';
313
314 switch(adapter->params.rev) {
315 case T3_REV_A:
316 rev = 'a';
317 break;
318 case T3_REV_B:
319 case T3_REV_B2:
320 rev = 'b';
321 break;
322 case T3_REV_C:
323 rev = 'c';
324 break;
325 }
326 return rev;
327}
328
329static struct cxgb_ident *
330cxgb_get_ident(device_t dev)
331{
332 struct cxgb_ident *id;
333
334 for (id = cxgb_identifiers; id->desc != NULL; id++) {
335 if ((id->vendor == pci_get_vendor(dev)) &&
336 (id->device == pci_get_device(dev))) {
337 return (id);
338 }
339 }
340 return (NULL);
341}
342
343static const struct adapter_info *
344cxgb_get_adapter_info(device_t dev)
345{
346 struct cxgb_ident *id;
347 const struct adapter_info *ai;
348
349 id = cxgb_get_ident(dev);
350 if (id == NULL)
351 return (NULL);
352
353 ai = t3_get_adapter_info(id->index);
354
355 return (ai);
356}
357
358static int
359cxgb_controller_probe(device_t dev)
360{
361 const struct adapter_info *ai;
362 char *ports, buf[80];
363 int nports;
364
365 ai = cxgb_get_adapter_info(dev);
366 if (ai == NULL)
367 return (ENXIO);
368
369 nports = ai->nports0 + ai->nports1;
370 if (nports == 1)
371 ports = "port";
372 else
373 ports = "ports";
374
375 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
376 device_set_desc_copy(dev, buf);
377 return (BUS_PROBE_DEFAULT);
378}
379
380#define FW_FNAME "cxgb_t3fw"
381#define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
382#define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
383
384static int
385upgrade_fw(adapter_t *sc)
386{
387 const struct firmware *fw;
388 int status;
389 u32 vers;
390
391 if ((fw = firmware_get(FW_FNAME)) == NULL) {
392 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
393 return (ENOENT);
394 } else
395 device_printf(sc->dev, "installing firmware on card\n");
396 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
397
398 if (status != 0) {
399 device_printf(sc->dev, "failed to install firmware: %d\n",
400 status);
401 } else {
402 t3_get_fw_version(sc, &vers);
403 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
404 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
405 G_FW_VERSION_MICRO(vers));
406 }
407
408 firmware_put(fw, FIRMWARE_UNLOAD);
409
410 return (status);
411}
412
413/*
414 * The cxgb_controller_attach function is responsible for the initial
415 * bringup of the device. Its responsibilities include:
416 *
417 * 1. Determine if the device supports MSI or MSI-X.
418 * 2. Allocate bus resources so that we can access the Base Address Register
419 * 3. Create and initialize mutexes for the controller and its control
420 * logic such as SGE and MDIO.
421 * 4. Call hardware specific setup routine for the adapter as a whole.
422 * 5. Allocate the BAR for doing MSI-X.
423 * 6. Setup the line interrupt iff MSI-X is not supported.
424 * 7. Create the driver's taskq.
425 * 8. Start one task queue service thread.
426 * 9. Check if the firmware and SRAM are up-to-date. They will be
427 * auto-updated later (before FULL_INIT_DONE), if required.
428 * 10. Create a child device for each MAC (port)
429 * 11. Initialize T3 private state.
430 * 12. Trigger the LED
431 * 13. Setup offload iff supported.
432 * 14. Reset/restart the tick callout.
433 * 15. Attach sysctls
434 *
435 * NOTE: Any modification or deviation from this list MUST be reflected in
436 * the above comment. Failure to do so will result in problems on various
437 * error conditions including link flapping.
438 */
439static int
440cxgb_controller_attach(device_t dev)
441{
442 device_t child;
443 const struct adapter_info *ai;
444 struct adapter *sc;
445 int i, error = 0;
446 uint32_t vers;
447 int port_qsets = 1;
448 int msi_needed, reg;
449 char buf[80];
450
451 sc = device_get_softc(dev);
452 sc->dev = dev;
453 sc->msi_count = 0;
454 ai = cxgb_get_adapter_info(dev);
455
456 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
457 device_get_unit(dev));
458 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
459
460 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
461 device_get_unit(dev));
462 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
463 device_get_unit(dev));
464 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
465 device_get_unit(dev));
466
467 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
468 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
469 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
470
471 mtx_lock(&t3_list_lock);
472 SLIST_INSERT_HEAD(&t3_list, sc, link);
473 mtx_unlock(&t3_list_lock);
474
475 /* find the PCIe link width and set max read request to 4KB*/
476 if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
477 uint16_t lnk;
478
479 lnk = pci_read_config(dev, reg + PCIR_EXPRESS_LINK_STA, 2);
480 sc->link_width = (lnk & PCIM_LINK_STA_WIDTH) >> 4;
481 if (sc->link_width < 8 &&
482 (ai->caps & SUPPORTED_10000baseT_Full)) {
483 device_printf(sc->dev,
484 "PCIe x%d Link, expect reduced performance\n",
485 sc->link_width);
486 }
487
488 pci_set_max_read_req(dev, 4096);
489 }
490
491 touch_bars(dev);
492 pci_enable_busmaster(dev);
493 /*
494 * Allocate the registers and make them available to the driver.
495 * The registers that we care about for NIC mode are in BAR 0
496 */
497 sc->regs_rid = PCIR_BAR(0);
498 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
499 &sc->regs_rid, RF_ACTIVE)) == NULL) {
500 device_printf(dev, "Cannot allocate BAR region 0\n");
501 error = ENXIO;
502 goto out;
503 }
504
505 sc->bt = rman_get_bustag(sc->regs_res);
506 sc->bh = rman_get_bushandle(sc->regs_res);
507 sc->mmio_len = rman_get_size(sc->regs_res);
508
509 for (i = 0; i < MAX_NPORTS; i++)
510 sc->port[i].adapter = sc;
511
512 if (t3_prep_adapter(sc, ai, 1) < 0) {
513 printf("prep adapter failed\n");
514 error = ENODEV;
515 goto out;
516 }
517
518 sc->udbs_rid = PCIR_BAR(2);
519 sc->udbs_res = NULL;
520 if (is_offload(sc) &&
521 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
522 &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
523 device_printf(dev, "Cannot allocate BAR region 1\n");
524 error = ENXIO;
525 goto out;
526 }
527
528 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
529 * enough messages for the queue sets. If that fails, try falling
530 * back to MSI. If that fails, then try falling back to the legacy
531 * interrupt pin model.
532 */
533 sc->msix_regs_rid = 0x20;
534 if ((msi_allowed >= 2) &&
535 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
536 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
537
538 if (multiq)
539 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
540 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
541
542 if (pci_msix_count(dev) == 0 ||
543 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
544 sc->msi_count != msi_needed) {
545 device_printf(dev, "alloc msix failed - "
546 "msi_count=%d, msi_needed=%d, err=%d; "
547 "will try MSI\n", sc->msi_count,
548 msi_needed, error);
549 sc->msi_count = 0;
550 port_qsets = 1;
551 pci_release_msi(dev);
552 bus_release_resource(dev, SYS_RES_MEMORY,
553 sc->msix_regs_rid, sc->msix_regs_res);
554 sc->msix_regs_res = NULL;
555 } else {
556 sc->flags |= USING_MSIX;
557 sc->cxgb_intr = cxgb_async_intr;
558 device_printf(dev,
559 "using MSI-X interrupts (%u vectors)\n",
560 sc->msi_count);
561 }
562 }
563
564 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
565 sc->msi_count = 1;
566 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
567 device_printf(dev, "alloc msi failed - "
568 "err=%d; will try INTx\n", error);
569 sc->msi_count = 0;
570 port_qsets = 1;
571 pci_release_msi(dev);
572 } else {
573 sc->flags |= USING_MSI;
574 sc->cxgb_intr = t3_intr_msi;
575 device_printf(dev, "using MSI interrupts\n");
576 }
577 }
578 if (sc->msi_count == 0) {
579 device_printf(dev, "using line interrupts\n");
580 sc->cxgb_intr = t3b_intr;
581 }
582
583 /* Create a private taskqueue thread for handling driver events */
584 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
585 taskqueue_thread_enqueue, &sc->tq);
586 if (sc->tq == NULL) {
587 device_printf(dev, "failed to allocate controller task queue\n");
588 goto out;
589 }
590
591 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
592 device_get_nameunit(dev));
593 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
594
595
596 /* Create a periodic callout for checking adapter status */
597 callout_init(&sc->cxgb_tick_ch, TRUE);
598
599 if (t3_check_fw_version(sc) < 0 || force_fw_update) {
600 /*
601 * Warn user that a firmware update will be attempted in init.
602 */
603 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
604 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
605 sc->flags &= ~FW_UPTODATE;
606 } else {
607 sc->flags |= FW_UPTODATE;
608 }
609
610 if (t3_check_tpsram_version(sc) < 0) {
611 /*
612 * Warn user that a firmware update will be attempted in init.
613 */
614 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
615 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
616 sc->flags &= ~TPS_UPTODATE;
617 } else {
618 sc->flags |= TPS_UPTODATE;
619 }
620
621 /*
622 * Create a child device for each MAC. The ethernet attachment
623 * will be done in these children.
624 */
625 for (i = 0; i < (sc)->params.nports; i++) {
626 struct port_info *pi;
627
628 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
629 device_printf(dev, "failed to add child port\n");
630 error = EINVAL;
631 goto out;
632 }
633 pi = &sc->port[i];
634 pi->adapter = sc;
635 pi->nqsets = port_qsets;
636 pi->first_qset = i*port_qsets;
637 pi->port_id = i;
638 pi->tx_chan = i >= ai->nports0;
639 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
640 sc->rxpkt_map[pi->txpkt_intf] = i;
641 sc->port[i].tx_chan = i >= ai->nports0;
642 sc->portdev[i] = child;
643 device_set_softc(child, pi);
644 }
645 if ((error = bus_generic_attach(dev)) != 0)
646 goto out;
647
648 /* initialize sge private state */
649 t3_sge_init_adapter(sc);
650
651 t3_led_ready(sc);
652
653 error = t3_get_fw_version(sc, &vers);
654 if (error)
655 goto out;
656
657 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
658 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
659 G_FW_VERSION_MICRO(vers));
660
661 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
662 ai->desc, is_offload(sc) ? "R" : "",
663 sc->params.vpd.ec, sc->params.vpd.sn);
664 device_set_desc_copy(dev, buf);
665
666 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
667 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
668 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
669
670 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
671 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
672 t3_add_attach_sysctls(sc);
673
674#ifdef TCP_OFFLOAD
675 for (i = 0; i < NUM_CPL_HANDLERS; i++)
676 sc->cpl_handler[i] = cpl_not_handled;
677#endif
678out:
679 if (error)
680 cxgb_free(sc);
681
682 return (error);
683}
684
685/*
686 * The cxgb_controller_detach routine is called with the device is
687 * unloaded from the system.
688 */
689
690static int
691cxgb_controller_detach(device_t dev)
692{
693 struct adapter *sc;
694
695 sc = device_get_softc(dev);
696
697 cxgb_free(sc);
698
699 return (0);
700}
701
702/*
703 * The cxgb_free() is called by the cxgb_controller_detach() routine
704 * to tear down the structures that were built up in
705 * cxgb_controller_attach(), and should be the final piece of work
706 * done when fully unloading the driver.
707 *
708 *
709 * 1. Shutting down the threads started by the cxgb_controller_attach()
710 * routine.
711 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
712 * 3. Detaching all of the port devices created during the
713 * cxgb_controller_attach() routine.
714 * 4. Removing the device children created via cxgb_controller_attach().
715 * 5. Releasing PCI resources associated with the device.
716 * 6. Turning off the offload support, iff it was turned on.
717 * 7. Destroying the mutexes created in cxgb_controller_attach().
718 *
719 */
720static void
721cxgb_free(struct adapter *sc)
722{
723 int i, nqsets = 0;
724
725 ADAPTER_LOCK(sc);
726 sc->flags |= CXGB_SHUTDOWN;
727 ADAPTER_UNLOCK(sc);
728
729 /*
730 * Make sure all child devices are gone.
731 */
732 bus_generic_detach(sc->dev);
733 for (i = 0; i < (sc)->params.nports; i++) {
734 if (sc->portdev[i] &&
735 device_delete_child(sc->dev, sc->portdev[i]) != 0)
736 device_printf(sc->dev, "failed to delete child port\n");
737 nqsets += sc->port[i].nqsets;
738 }
739
740 /*
741 * At this point, it is as if cxgb_port_detach has run on all ports, and
742 * cxgb_down has run on the adapter. All interrupts have been silenced,
743 * all open devices have been closed.
744 */
745 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
746 __func__, sc->open_device_map));
747 for (i = 0; i < sc->params.nports; i++) {
748 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
749 __func__, i));
750 }
751
752 /*
753 * Finish off the adapter's callouts.
754 */
755 callout_drain(&sc->cxgb_tick_ch);
756 callout_drain(&sc->sge_timer_ch);
757
758 /*
759 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
760 * sysctls are cleaned up by the kernel linker.
761 */
762 if (sc->flags & FULL_INIT_DONE) {
763 t3_free_sge_resources(sc, nqsets);
764 sc->flags &= ~FULL_INIT_DONE;
765 }
766
767 /*
768 * Release all interrupt resources.
769 */
770 cxgb_teardown_interrupts(sc);
771 if (sc->flags & (USING_MSI | USING_MSIX)) {
772 device_printf(sc->dev, "releasing msi message(s)\n");
773 pci_release_msi(sc->dev);
774 } else {
775 device_printf(sc->dev, "no msi message to release\n");
776 }
777
778 if (sc->msix_regs_res != NULL) {
779 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
780 sc->msix_regs_res);
781 }
782
783 /*
784 * Free the adapter's taskqueue.
785 */
786 if (sc->tq != NULL) {
787 taskqueue_free(sc->tq);
788 sc->tq = NULL;
789 }
790
791 free(sc->filters, M_DEVBUF);
792 t3_sge_free(sc);
793
794 if (sc->udbs_res != NULL)
795 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
796 sc->udbs_res);
797
798 if (sc->regs_res != NULL)
799 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
800 sc->regs_res);
801
802 MTX_DESTROY(&sc->mdio_lock);
803 MTX_DESTROY(&sc->sge.reg_lock);
804 MTX_DESTROY(&sc->elmer_lock);
805 mtx_lock(&t3_list_lock);
806 SLIST_REMOVE(&t3_list, sc, adapter, link);
807 mtx_unlock(&t3_list_lock);
808 ADAPTER_LOCK_DEINIT(sc);
809}
810
811/**
812 * setup_sge_qsets - configure SGE Tx/Rx/response queues
813 * @sc: the controller softc
814 *
815 * Determines how many sets of SGE queues to use and initializes them.
816 * We support multiple queue sets per port if we have MSI-X, otherwise
817 * just one queue set per port.
818 */
819static int
820setup_sge_qsets(adapter_t *sc)
821{
822 int i, j, err, irq_idx = 0, qset_idx = 0;
823 u_int ntxq = SGE_TXQ_PER_SET;
824
825 if ((err = t3_sge_alloc(sc)) != 0) {
826 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
827 return (err);
828 }
829
830 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
831 irq_idx = -1;
832
833 for (i = 0; i < (sc)->params.nports; i++) {
834 struct port_info *pi = &sc->port[i];
835
836 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
837 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
838 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
839 &sc->params.sge.qset[qset_idx], ntxq, pi);
840 if (err) {
841 t3_free_sge_resources(sc, qset_idx);
842 device_printf(sc->dev,
843 "t3_sge_alloc_qset failed with %d\n", err);
844 return (err);
845 }
846 }
847 }
848
849 return (0);
850}
851
852static void
853cxgb_teardown_interrupts(adapter_t *sc)
854{
855 int i;
856
857 for (i = 0; i < SGE_QSETS; i++) {
858 if (sc->msix_intr_tag[i] == NULL) {
859
860 /* Should have been setup fully or not at all */
861 KASSERT(sc->msix_irq_res[i] == NULL &&
862 sc->msix_irq_rid[i] == 0,
863 ("%s: half-done interrupt (%d).", __func__, i));
864
865 continue;
866 }
867
868 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
869 sc->msix_intr_tag[i]);
870 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
871 sc->msix_irq_res[i]);
872
873 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
874 sc->msix_irq_rid[i] = 0;
875 }
876
877 if (sc->intr_tag) {
878 KASSERT(sc->irq_res != NULL,
879 ("%s: half-done interrupt.", __func__));
880
881 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
882 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
883 sc->irq_res);
884
885 sc->irq_res = sc->intr_tag = NULL;
886 sc->irq_rid = 0;
887 }
888}
889
890static int
891cxgb_setup_interrupts(adapter_t *sc)
892{
893 struct resource *res;
894 void *tag;
895 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
896
897 sc->irq_rid = intr_flag ? 1 : 0;
898 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
899 RF_SHAREABLE | RF_ACTIVE);
900 if (sc->irq_res == NULL) {
901 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
902 intr_flag, sc->irq_rid);
903 err = EINVAL;
904 sc->irq_rid = 0;
905 } else {
906 err = bus_setup_intr(sc->dev, sc->irq_res,
907 INTR_MPSAFE | INTR_TYPE_NET, NULL,
908 sc->cxgb_intr, sc, &sc->intr_tag);
909
910 if (err) {
911 device_printf(sc->dev,
912 "Cannot set up interrupt (%x, %u, %d)\n",
913 intr_flag, sc->irq_rid, err);
914 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
915 sc->irq_res);
916 sc->irq_res = sc->intr_tag = NULL;
917 sc->irq_rid = 0;
918 }
919 }
920
921 /* That's all for INTx or MSI */
922 if (!(intr_flag & USING_MSIX) || err)
923 return (err);
924
925 for (i = 0; i < sc->msi_count - 1; i++) {
926 rid = i + 2;
927 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
928 RF_SHAREABLE | RF_ACTIVE);
929 if (res == NULL) {
930 device_printf(sc->dev, "Cannot allocate interrupt "
931 "for message %d\n", rid);
932 err = EINVAL;
933 break;
934 }
935
936 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
937 NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
938 if (err) {
939 device_printf(sc->dev, "Cannot set up interrupt "
940 "for message %d (%d)\n", rid, err);
941 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
942 break;
943 }
944
945 sc->msix_irq_rid[i] = rid;
946 sc->msix_irq_res[i] = res;
947 sc->msix_intr_tag[i] = tag;
948 }
949
950 if (err)
951 cxgb_teardown_interrupts(sc);
952
953 return (err);
954}
955
956
957static int
958cxgb_port_probe(device_t dev)
959{
960 struct port_info *p;
961 char buf[80];
962 const char *desc;
963
964 p = device_get_softc(dev);
965 desc = p->phy.desc;
966 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
967 device_set_desc_copy(dev, buf);
968 return (0);
969}
970
971
972static int
973cxgb_makedev(struct port_info *pi)
974{
975
976 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
977 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
978
979 if (pi->port_cdev == NULL)
980 return (ENOMEM);
981
982 pi->port_cdev->si_drv1 = (void *)pi;
983
984 return (0);
985}
986
987#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
988 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
989 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE)
990#define CXGB_CAP_ENABLE CXGB_CAP
991
992static int
993cxgb_port_attach(device_t dev)
994{
995 struct port_info *p;
996 struct ifnet *ifp;
997 int err;
998 struct adapter *sc;
999
1000 p = device_get_softc(dev);
1001 sc = p->adapter;
1002 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1003 device_get_unit(device_get_parent(dev)), p->port_id);
1004 PORT_LOCK_INIT(p, p->lockbuf);
1005
1006 callout_init(&p->link_check_ch, CALLOUT_MPSAFE);
1007 TASK_INIT(&p->link_check_task, 0, check_link_status, p);
1008
1009 /* Allocate an ifnet object and set it up */
1010 ifp = p->ifp = if_alloc(IFT_ETHER);
1011 if (ifp == NULL) {
1012 device_printf(dev, "Cannot allocate ifnet\n");
1013 return (ENOMEM);
1014 }
1015
1016 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1017 ifp->if_init = cxgb_init;
1018 ifp->if_softc = p;
1019 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1020 ifp->if_ioctl = cxgb_ioctl;
1021 ifp->if_transmit = cxgb_transmit;
1022 ifp->if_qflush = cxgb_qflush;
1023
1024 ifp->if_capabilities = CXGB_CAP;
1025#ifdef TCP_OFFLOAD
1026 if (is_offload(sc))
1027 ifp->if_capabilities |= IFCAP_TOE4;
1028#endif
1029 ifp->if_capenable = CXGB_CAP_ENABLE;
1030 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO;
1031
1032 /*
1033 * Disable TSO on 4-port - it isn't supported by the firmware.
1034 */
1035 if (sc->params.nports > 2) {
1036 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1037 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1038 ifp->if_hwassist &= ~CSUM_TSO;
1039 }
1040
1041 ether_ifattach(ifp, p->hw_addr);
1042
1043#ifdef DEFAULT_JUMBO
1044 if (sc->params.nports <= 2)
1045 ifp->if_mtu = ETHERMTU_JUMBO;
1046#endif
1047 if ((err = cxgb_makedev(p)) != 0) {
1048 printf("makedev failed %d\n", err);
1049 return (err);
1050 }
1051
1052 /* Create a list of media supported by this port */
1053 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1054 cxgb_media_status);
1055 cxgb_build_medialist(p);
1056
1057 t3_sge_init_port(p);
1058
1059 return (err);
1060}
1061
1062/*
1063 * cxgb_port_detach() is called via the device_detach methods when
1064 * cxgb_free() calls the bus_generic_detach. It is responsible for
1065 * removing the device from the view of the kernel, i.e. from all
1066 * interfaces lists etc. This routine is only called when the driver is
1067 * being unloaded, not when the link goes down.
1068 */
1069static int
1070cxgb_port_detach(device_t dev)
1071{
1072 struct port_info *p;
1073 struct adapter *sc;
1074 int i;
1075
1076 p = device_get_softc(dev);
1077 sc = p->adapter;
1078
1079 /* Tell cxgb_ioctl and if_init that the port is going away */
1080 ADAPTER_LOCK(sc);
1081 SET_DOOMED(p);
1082 wakeup(&sc->flags);
1083 while (IS_BUSY(sc))
1084 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1085 SET_BUSY(sc);
1086 ADAPTER_UNLOCK(sc);
1087
1088 if (p->port_cdev != NULL)
1089 destroy_dev(p->port_cdev);
1090
1091 cxgb_uninit_synchronized(p);
1092 ether_ifdetach(p->ifp);
1093
1094 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1095 struct sge_qset *qs = &sc->sge.qs[i];
1096 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1097
1098 callout_drain(&txq->txq_watchdog);
1099 callout_drain(&txq->txq_timer);
1100 }
1101
1102 PORT_LOCK_DEINIT(p);
1103 if_free(p->ifp);
1104 p->ifp = NULL;
1105
1106 ADAPTER_LOCK(sc);
1107 CLR_BUSY(sc);
1108 wakeup_one(&sc->flags);
1109 ADAPTER_UNLOCK(sc);
1110 return (0);
1111}
1112
1113void
1114t3_fatal_err(struct adapter *sc)
1115{
1116 u_int fw_status[4];
1117
1118 if (sc->flags & FULL_INIT_DONE) {
1119 t3_sge_stop(sc);
1120 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1121 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1122 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1123 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1124 t3_intr_disable(sc);
1125 }
1126 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1127 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1128 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1129 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1130}
1131
1132int
1133t3_os_find_pci_capability(adapter_t *sc, int cap)
1134{
1135 device_t dev;
1136 struct pci_devinfo *dinfo;
1137 pcicfgregs *cfg;
1138 uint32_t status;
1139 uint8_t ptr;
1140
1141 dev = sc->dev;
1142 dinfo = device_get_ivars(dev);
1143 cfg = &dinfo->cfg;
1144
1145 status = pci_read_config(dev, PCIR_STATUS, 2);
1146 if (!(status & PCIM_STATUS_CAPPRESENT))
1147 return (0);
1148
1149 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1150 case 0:
1151 case 1:
1152 ptr = PCIR_CAP_PTR;
1153 break;
1154 case 2:
1155 ptr = PCIR_CAP_PTR_2;
1156 break;
1157 default:
1158 return (0);
1159 break;
1160 }
1161 ptr = pci_read_config(dev, ptr, 1);
1162
1163 while (ptr != 0) {
1164 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1165 return (ptr);
1166 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1167 }
1168
1169 return (0);
1170}
1171
1172int
1173t3_os_pci_save_state(struct adapter *sc)
1174{
1175 device_t dev;
1176 struct pci_devinfo *dinfo;
1177
1178 dev = sc->dev;
1179 dinfo = device_get_ivars(dev);
1180
1181 pci_cfg_save(dev, dinfo, 0);
1182 return (0);
1183}
1184
1185int
1186t3_os_pci_restore_state(struct adapter *sc)
1187{
1188 device_t dev;
1189 struct pci_devinfo *dinfo;
1190
1191 dev = sc->dev;
1192 dinfo = device_get_ivars(dev);
1193
1194 pci_cfg_restore(dev, dinfo);
1195 return (0);
1196}
1197
1198/**
1199 * t3_os_link_changed - handle link status changes
1200 * @sc: the adapter associated with the link change
1201 * @port_id: the port index whose link status has changed
1202 * @link_status: the new status of the link
1203 * @speed: the new speed setting
1204 * @duplex: the new duplex setting
1205 * @fc: the new flow-control setting
1206 *
1207 * This is the OS-dependent handler for link status changes. The OS
1208 * neutral handler takes care of most of the processing for these events,
1209 * then calls this handler for any OS-specific processing.
1210 */
1211void
1212t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1213 int duplex, int fc, int mac_was_reset)
1214{
1215 struct port_info *pi = &adapter->port[port_id];
1216 struct ifnet *ifp = pi->ifp;
1217
1218 /* no race with detach, so ifp should always be good */
1219 KASSERT(ifp, ("%s: if detached.", __func__));
1220
1221 /* Reapply mac settings if they were lost due to a reset */
1222 if (mac_was_reset) {
1223 PORT_LOCK(pi);
1224 cxgb_update_mac_settings(pi);
1225 PORT_UNLOCK(pi);
1226 }
1227
1228 if (link_status) {
1229 ifp->if_baudrate = IF_Mbps(speed);
1230 if_link_state_change(ifp, LINK_STATE_UP);
1231 } else
1232 if_link_state_change(ifp, LINK_STATE_DOWN);
1233}
1234
1235/**
1236 * t3_os_phymod_changed - handle PHY module changes
1237 * @phy: the PHY reporting the module change
1238 * @mod_type: new module type
1239 *
1240 * This is the OS-dependent handler for PHY module changes. It is
1241 * invoked when a PHY module is removed or inserted for any OS-specific
1242 * processing.
1243 */
1244void t3_os_phymod_changed(struct adapter *adap, int port_id)
1245{
1246 static const char *mod_str[] = {
1247 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
1248 };
1249 struct port_info *pi = &adap->port[port_id];
1250 int mod = pi->phy.modtype;
1251
1252 if (mod != pi->media.ifm_cur->ifm_data)
1253 cxgb_build_medialist(pi);
1254
1255 if (mod == phy_modtype_none)
1256 if_printf(pi->ifp, "PHY module unplugged\n");
1257 else {
1258 KASSERT(mod < ARRAY_SIZE(mod_str),
1259 ("invalid PHY module type %d", mod));
1260 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1261 }
1262}
1263
1264void
1265t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1266{
1267
1268 /*
1269 * The ifnet might not be allocated before this gets called,
1270 * as this is called early on in attach by t3_prep_adapter
1271 * save the address off in the port structure
1272 */
1273 if (cxgb_debug)
1274 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1275 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1276}
1277
1278/*
1279 * Programs the XGMAC based on the settings in the ifnet. These settings
1280 * include MTU, MAC address, mcast addresses, etc.
1281 */
1282static void
1283cxgb_update_mac_settings(struct port_info *p)
1284{
1285 struct ifnet *ifp = p->ifp;
1286 struct t3_rx_mode rm;
1287 struct cmac *mac = &p->mac;
1288 int mtu, hwtagging;
1289
1290 PORT_LOCK_ASSERT_OWNED(p);
1291
1292 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1293
1294 mtu = ifp->if_mtu;
1295 if (ifp->if_capenable & IFCAP_VLAN_MTU)
1296 mtu += ETHER_VLAN_ENCAP_LEN;
1297
1298 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1299
1300 t3_mac_set_mtu(mac, mtu);
1301 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1302 t3_mac_set_address(mac, 0, p->hw_addr);
1303 t3_init_rx_mode(&rm, p);
1304 t3_mac_set_rx_mode(mac, &rm);
1305}
1306
1307
1308static int
1309await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1310 unsigned long n)
1311{
1312 int attempts = 5;
1313
1314 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1315 if (!--attempts)
1316 return (ETIMEDOUT);
1317 t3_os_sleep(10);
1318 }
1319 return 0;
1320}
1321
1322static int
1323init_tp_parity(struct adapter *adap)
1324{
1325 int i;
1326 struct mbuf *m;
1327 struct cpl_set_tcb_field *greq;
1328 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1329
1330 t3_tp_set_offload_mode(adap, 1);
1331
1332 for (i = 0; i < 16; i++) {
1333 struct cpl_smt_write_req *req;
1334
1335 m = m_gethdr(M_WAITOK, MT_DATA);
1336 req = mtod(m, struct cpl_smt_write_req *);
1337 m->m_len = m->m_pkthdr.len = sizeof(*req);
1338 memset(req, 0, sizeof(*req));
1339 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1340 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1341 req->iff = i;
1342 t3_mgmt_tx(adap, m);
1343 }
1344
1345 for (i = 0; i < 2048; i++) {
1346 struct cpl_l2t_write_req *req;
1347
1348 m = m_gethdr(M_WAITOK, MT_DATA);
1349 req = mtod(m, struct cpl_l2t_write_req *);
1350 m->m_len = m->m_pkthdr.len = sizeof(*req);
1351 memset(req, 0, sizeof(*req));
1352 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1353 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1354 req->params = htonl(V_L2T_W_IDX(i));
1355 t3_mgmt_tx(adap, m);
1356 }
1357
1358 for (i = 0; i < 2048; i++) {
1359 struct cpl_rte_write_req *req;
1360
1361 m = m_gethdr(M_WAITOK, MT_DATA);
1362 req = mtod(m, struct cpl_rte_write_req *);
1363 m->m_len = m->m_pkthdr.len = sizeof(*req);
1364 memset(req, 0, sizeof(*req));
1365 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1366 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1367 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1368 t3_mgmt_tx(adap, m);
1369 }
1370
1371 m = m_gethdr(M_WAITOK, MT_DATA);
1372 greq = mtod(m, struct cpl_set_tcb_field *);
1373 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1374 memset(greq, 0, sizeof(*greq));
1375 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1376 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1377 greq->mask = htobe64(1);
1378 t3_mgmt_tx(adap, m);
1379
1380 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1381 t3_tp_set_offload_mode(adap, 0);
1382 return (i);
1383}
1384
1385/**
1386 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1387 * @adap: the adapter
1388 *
1389 * Sets up RSS to distribute packets to multiple receive queues. We
1390 * configure the RSS CPU lookup table to distribute to the number of HW
1391 * receive queues, and the response queue lookup table to narrow that
1392 * down to the response queues actually configured for each port.
1393 * We always configure the RSS mapping for two ports since the mapping
1394 * table has plenty of entries.
1395 */
1396static void
1397setup_rss(adapter_t *adap)
1398{
1399 int i;
1400 u_int nq[2];
1401 uint8_t cpus[SGE_QSETS + 1];
1402 uint16_t rspq_map[RSS_TABLE_SIZE];
1403
1404 for (i = 0; i < SGE_QSETS; ++i)
1405 cpus[i] = i;
1406 cpus[SGE_QSETS] = 0xff;
1407
1408 nq[0] = nq[1] = 0;
1409 for_each_port(adap, i) {
1410 const struct port_info *pi = adap2pinfo(adap, i);
1411
1412 nq[pi->tx_chan] += pi->nqsets;
1413 }
1414 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1415 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1416 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1417 }
1418
1419 /* Calculate the reverse RSS map table */
1420 for (i = 0; i < SGE_QSETS; ++i)
1421 adap->rrss_map[i] = 0xff;
1422 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1423 if (adap->rrss_map[rspq_map[i]] == 0xff)
1424 adap->rrss_map[rspq_map[i]] = i;
1425
1426 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1427 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1428 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1429 cpus, rspq_map);
1430
1431}
1432static void
1433send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1434 int hi, int port)
1435{
1436 struct mbuf *m;
1437 struct mngt_pktsched_wr *req;
1438
1439 m = m_gethdr(M_DONTWAIT, MT_DATA);
1440 if (m) {
1441 req = mtod(m, struct mngt_pktsched_wr *);
1442 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1443 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1444 req->sched = sched;
1445 req->idx = qidx;
1446 req->min = lo;
1447 req->max = hi;
1448 req->binding = port;
1449 m->m_len = m->m_pkthdr.len = sizeof(*req);
1450 t3_mgmt_tx(adap, m);
1451 }
1452}
1453
1454static void
1455bind_qsets(adapter_t *sc)
1456{
1457 int i, j;
1458
1459 for (i = 0; i < (sc)->params.nports; ++i) {
1460 const struct port_info *pi = adap2pinfo(sc, i);
1461
1462 for (j = 0; j < pi->nqsets; ++j) {
1463 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1464 -1, pi->tx_chan);
1465
1466 }
1467 }
1468}
1469
1470static void
1471update_tpeeprom(struct adapter *adap)
1472{
1473 const struct firmware *tpeeprom;
1474
1475 uint32_t version;
1476 unsigned int major, minor;
1477 int ret, len;
1478 char rev, name[32];
1479
1480 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1481
1482 major = G_TP_VERSION_MAJOR(version);
1483 minor = G_TP_VERSION_MINOR(version);
1484 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1485 return;
1486
1487 rev = t3rev2char(adap);
1488 snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1489
1490 tpeeprom = firmware_get(name);
1491 if (tpeeprom == NULL) {
1492 device_printf(adap->dev,
1493 "could not load TP EEPROM: unable to load %s\n",
1494 name);
1495 return;
1496 }
1497
1498 len = tpeeprom->datasize - 4;
1499
1500 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1501 if (ret)
1502 goto release_tpeeprom;
1503
1504 if (len != TP_SRAM_LEN) {
1505 device_printf(adap->dev,
1506 "%s length is wrong len=%d expected=%d\n", name,
1507 len, TP_SRAM_LEN);
1508 return;
1509 }
1510
1511 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1512 TP_SRAM_OFFSET);
1513
1514 if (!ret) {
1515 device_printf(adap->dev,
1516 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1517 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1518 } else
1519 device_printf(adap->dev,
1520 "Protocol SRAM image update in EEPROM failed\n");
1521
1522release_tpeeprom:
1523 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1524
1525 return;
1526}
1527
1528static int
1529update_tpsram(struct adapter *adap)
1530{
1531 const struct firmware *tpsram;
1532 int ret;
1533 char rev, name[32];
1534
1535 rev = t3rev2char(adap);
1536 snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1537
1538 update_tpeeprom(adap);
1539
1540 tpsram = firmware_get(name);
1541 if (tpsram == NULL){
1542 device_printf(adap->dev, "could not load TP SRAM\n");
1543 return (EINVAL);
1544 } else
1545 device_printf(adap->dev, "updating TP SRAM\n");
1546
1547 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1548 if (ret)
1549 goto release_tpsram;
1550
1551 ret = t3_set_proto_sram(adap, tpsram->data);
1552 if (ret)
1553 device_printf(adap->dev, "loading protocol SRAM failed\n");
1554
1555release_tpsram:
1556 firmware_put(tpsram, FIRMWARE_UNLOAD);
1557
1558 return ret;
1559}
1560
1561/**
1562 * cxgb_up - enable the adapter
1563 * @adap: adapter being enabled
1564 *
1565 * Called when the first port is enabled, this function performs the
1566 * actions necessary to make an adapter operational, such as completing
1567 * the initialization of HW modules, and enabling interrupts.
1568 */
1569static int
1570cxgb_up(struct adapter *sc)
1571{
1572 int err = 0;
1573 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
1574
1575 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1576 __func__, sc->open_device_map));
1577
1578 if ((sc->flags & FULL_INIT_DONE) == 0) {
1579
1580 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1581
1582 if ((sc->flags & FW_UPTODATE) == 0)
1583 if ((err = upgrade_fw(sc)))
1584 goto out;
1585
1586 if ((sc->flags & TPS_UPTODATE) == 0)
1587 if ((err = update_tpsram(sc)))
1588 goto out;
1589
1590 if (is_offload(sc) && nfilters != 0) {
1591 sc->params.mc5.nservers = 0;
1592
1593 if (nfilters < 0)
1594 sc->params.mc5.nfilters = mxf;
1595 else
1596 sc->params.mc5.nfilters = min(nfilters, mxf);
1597 }
1598
1599 err = t3_init_hw(sc, 0);
1600 if (err)
1601 goto out;
1602
1603 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1604 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1605
1606 err = setup_sge_qsets(sc);
1607 if (err)
1608 goto out;
1609
1610 alloc_filters(sc);
1611 setup_rss(sc);
1612
1613 t3_intr_clear(sc);
1614 err = cxgb_setup_interrupts(sc);
1615 if (err)
1616 goto out;
1617
1618 t3_add_configured_sysctls(sc);
1619 sc->flags |= FULL_INIT_DONE;
1620 }
1621
1622 t3_intr_clear(sc);
1623 t3_sge_start(sc);
1624 t3_intr_enable(sc);
1625
1626 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1627 is_offload(sc) && init_tp_parity(sc) == 0)
1628 sc->flags |= TP_PARITY_INIT;
1629
1630 if (sc->flags & TP_PARITY_INIT) {
1631 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1632 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1633 }
1634
1635 if (!(sc->flags & QUEUES_BOUND)) {
1636 bind_qsets(sc);
1637 setup_hw_filters(sc);
1638 sc->flags |= QUEUES_BOUND;
1639 }
1640
1641 t3_sge_reset_adapter(sc);
1642out:
1643 return (err);
1644}
1645
1646/*
1647 * Called when the last open device is closed. Does NOT undo all of cxgb_up's
1648 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
1649 * during controller_detach, not here.
1650 */
1651static void
1652cxgb_down(struct adapter *sc)
1653{
1654 t3_sge_stop(sc);
1655 t3_intr_disable(sc);
1656}
1657
1658/*
1659 * if_init for cxgb ports.
1660 */
1661static void
1662cxgb_init(void *arg)
1663{
1664 struct port_info *p = arg;
1665 struct adapter *sc = p->adapter;
1666
1667 ADAPTER_LOCK(sc);
1668 cxgb_init_locked(p); /* releases adapter lock */
1669 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1670}
1671
1672static int
1673cxgb_init_locked(struct port_info *p)
1674{
1675 struct adapter *sc = p->adapter;
1676 struct ifnet *ifp = p->ifp;
1677 struct cmac *mac = &p->mac;
1678 int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
1679
1680 ADAPTER_LOCK_ASSERT_OWNED(sc);
1681
1682 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1683 gave_up_lock = 1;
1684 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1685 rc = EINTR;
1686 goto done;
1687 }
1688 }
1689 if (IS_DOOMED(p)) {
1690 rc = ENXIO;
1691 goto done;
1692 }
1693 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1694
1695 /*
1696 * The code that runs during one-time adapter initialization can sleep
1697 * so it's important not to hold any locks across it.
1698 */
1699 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1700
1701 if (may_sleep) {
1702 SET_BUSY(sc);
1703 gave_up_lock = 1;
1704 ADAPTER_UNLOCK(sc);
1705 }
1706
1707 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
1708 goto done;
1709
1710 PORT_LOCK(p);
1711 if (isset(&sc->open_device_map, p->port_id) &&
1712 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1713 PORT_UNLOCK(p);
1714 goto done;
1715 }
1716 t3_port_intr_enable(sc, p->port_id);
1717 if (!mac->multiport)
1718 t3_mac_init(mac);
1719 cxgb_update_mac_settings(p);
1720 t3_link_start(&p->phy, mac, &p->link_config);
1721 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1722 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1723 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1724 PORT_UNLOCK(p);
1725
1726 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1727 struct sge_qset *qs = &sc->sge.qs[i];
1728 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1729
1730 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1731 txq->txq_watchdog.c_cpu);
1732 }
1733
1734 /* all ok */
1735 setbit(&sc->open_device_map, p->port_id);
1736 callout_reset(&p->link_check_ch,
1737 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
1738 link_check_callout, p);
1739
1740done:
1741 if (may_sleep) {
1742 ADAPTER_LOCK(sc);
1743 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1744 CLR_BUSY(sc);
1745 }
1746 if (gave_up_lock)
1747 wakeup_one(&sc->flags);
1748 ADAPTER_UNLOCK(sc);
1749 return (rc);
1750}
1751
1752static int
1753cxgb_uninit_locked(struct port_info *p)
1754{
1755 struct adapter *sc = p->adapter;
1756 int rc;
1757
1758 ADAPTER_LOCK_ASSERT_OWNED(sc);
1759
1760 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1761 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1762 rc = EINTR;
1763 goto done;
1764 }
1765 }
1766 if (IS_DOOMED(p)) {
1767 rc = ENXIO;
1768 goto done;
1769 }
1770 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1771 SET_BUSY(sc);
1772 ADAPTER_UNLOCK(sc);
1773
1774 rc = cxgb_uninit_synchronized(p);
1775
1776 ADAPTER_LOCK(sc);
1777 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1778 CLR_BUSY(sc);
1779 wakeup_one(&sc->flags);
1780done:
1781 ADAPTER_UNLOCK(sc);
1782 return (rc);
1783}
1784
1785/*
1786 * Called on "ifconfig down", and from port_detach
1787 */
1788static int
1789cxgb_uninit_synchronized(struct port_info *pi)
1790{
1791 struct adapter *sc = pi->adapter;
1792 struct ifnet *ifp = pi->ifp;
1793
1794 /*
1795 * taskqueue_drain may cause a deadlock if the adapter lock is held.
1796 */
1797 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1798
1799 /*
1800 * Clear this port's bit from the open device map, and then drain all
1801 * the tasks that can access/manipulate this port's port_info or ifp.
1802 * We disable this port's interrupts here and so the slow/ext
1803 * interrupt tasks won't be enqueued. The tick task will continue to
1804 * be enqueued every second but the runs after this drain will not see
1805 * this port in the open device map.
1806 *
1807 * A well behaved task must take open_device_map into account and ignore
1808 * ports that are not open.
1809 */
1810 clrbit(&sc->open_device_map, pi->port_id);
1811 t3_port_intr_disable(sc, pi->port_id);
1812 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1813 taskqueue_drain(sc->tq, &sc->tick_task);
1814
1815 callout_drain(&pi->link_check_ch);
1816 taskqueue_drain(sc->tq, &pi->link_check_task);
1817
1818 PORT_LOCK(pi);
1819 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1820
1821 /* disable pause frames */
1822 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1823
1824 /* Reset RX FIFO HWM */
1825 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
1826 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1827
1828 DELAY(100 * 1000);
1829
1830 /* Wait for TXFIFO empty */
1831 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1832 F_TXFIFO_EMPTY, 1, 20, 5);
1833
1834 DELAY(100 * 1000);
1835 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1836
1837 pi->phy.ops->power_down(&pi->phy, 1);
1838
1839 PORT_UNLOCK(pi);
1840
1841 pi->link_config.link_ok = 0;
1842 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1843
1844 if (sc->open_device_map == 0)
1845 cxgb_down(pi->adapter);
1846
1847 return (0);
1848}
1849
1850/*
1851 * Mark lro enabled or disabled in all qsets for this port
1852 */
1853static int
1854cxgb_set_lro(struct port_info *p, int enabled)
1855{
1856 int i;
1857 struct adapter *adp = p->adapter;
1858 struct sge_qset *q;
1859
1860 for (i = 0; i < p->nqsets; i++) {
1861 q = &adp->sge.qs[p->first_qset + i];
1862 q->lro.enabled = (enabled != 0);
1863 }
1864 return (0);
1865}
1866
1867static int
1868cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1869{
1870 struct port_info *p = ifp->if_softc;
1871 struct adapter *sc = p->adapter;
1872 struct ifreq *ifr = (struct ifreq *)data;
1873 int flags, error = 0, mtu;
1874 uint32_t mask;
1875
1876 switch (command) {
1877 case SIOCSIFMTU:
1878 ADAPTER_LOCK(sc);
1879 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1880 if (error) {
1881fail:
1882 ADAPTER_UNLOCK(sc);
1883 return (error);
1884 }
1885
1886 mtu = ifr->ifr_mtu;
1887 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
1888 error = EINVAL;
1889 } else {
1890 ifp->if_mtu = mtu;
1891 PORT_LOCK(p);
1892 cxgb_update_mac_settings(p);
1893 PORT_UNLOCK(p);
1894 }
1895 ADAPTER_UNLOCK(sc);
1896 break;
1897 case SIOCSIFFLAGS:
1898 ADAPTER_LOCK(sc);
1899 if (IS_DOOMED(p)) {
1900 error = ENXIO;
1901 goto fail;
1902 }
1903 if (ifp->if_flags & IFF_UP) {
1904 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1905 flags = p->if_flags;
1906 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1907 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
1908 if (IS_BUSY(sc)) {
1909 error = EBUSY;
1910 goto fail;
1911 }
1912 PORT_LOCK(p);
1913 cxgb_update_mac_settings(p);
1914 PORT_UNLOCK(p);
1915 }
1916 ADAPTER_UNLOCK(sc);
1917 } else
1918 error = cxgb_init_locked(p);
1919 p->if_flags = ifp->if_flags;
1920 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1921 error = cxgb_uninit_locked(p);
1922 else
1923 ADAPTER_UNLOCK(sc);
1924
1925 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1926 break;
1927 case SIOCADDMULTI:
1928 case SIOCDELMULTI:
1929 ADAPTER_LOCK(sc);
1930 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1931 if (error)
1932 goto fail;
1933
1934 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1935 PORT_LOCK(p);
1936 cxgb_update_mac_settings(p);
1937 PORT_UNLOCK(p);
1938 }
1939 ADAPTER_UNLOCK(sc);
1940
1941 break;
1942 case SIOCSIFCAP:
1943 ADAPTER_LOCK(sc);
1944 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1945 if (error)
1946 goto fail;
1947
1948 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1949 if (mask & IFCAP_TXCSUM) {
1950 ifp->if_capenable ^= IFCAP_TXCSUM;
1951 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1952
1953 if (IFCAP_TSO & ifp->if_capenable &&
1954 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1955 ifp->if_capenable &= ~IFCAP_TSO;
1956 ifp->if_hwassist &= ~CSUM_TSO;
1957 if_printf(ifp,
1958 "tso disabled due to -txcsum.\n");
1959 }
1960 }
1961 if (mask & IFCAP_RXCSUM)
1962 ifp->if_capenable ^= IFCAP_RXCSUM;
1963 if (mask & IFCAP_TSO) {
1964 ifp->if_capenable ^= IFCAP_TSO;
1965
1966 if (IFCAP_TSO & ifp->if_capenable) {
1967 if (IFCAP_TXCSUM & ifp->if_capenable)
1968 ifp->if_hwassist |= CSUM_TSO;
1969 else {
1970 ifp->if_capenable &= ~IFCAP_TSO;
1971 ifp->if_hwassist &= ~CSUM_TSO;
1972 if_printf(ifp,
1973 "enable txcsum first.\n");
1974 error = EAGAIN;
1975 }
1976 } else
1977 ifp->if_hwassist &= ~CSUM_TSO;
1978 }
1979 if (mask & IFCAP_LRO) {
1980 ifp->if_capenable ^= IFCAP_LRO;
1981
1982 /* Safe to do this even if cxgb_up not called yet */
1983 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
1984 }
1985#ifdef TCP_OFFLOAD
1986 if (mask & IFCAP_TOE4) {
1987 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
1988
1989 error = toe_capability(p, enable);
1990 if (error == 0)
1991 ifp->if_capenable ^= mask;
1992 }
1993#endif
1994 if (mask & IFCAP_VLAN_HWTAGGING) {
1995 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1996 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1997 PORT_LOCK(p);
1998 cxgb_update_mac_settings(p);
1999 PORT_UNLOCK(p);
2000 }
2001 }
2002 if (mask & IFCAP_VLAN_MTU) {
2003 ifp->if_capenable ^= IFCAP_VLAN_MTU;
2004 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2005 PORT_LOCK(p);
2006 cxgb_update_mac_settings(p);
2007 PORT_UNLOCK(p);
2008 }
2009 }
2010 if (mask & IFCAP_VLAN_HWTSO)
2011 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
2012 if (mask & IFCAP_VLAN_HWCSUM)
2013 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2014
2015#ifdef VLAN_CAPABILITIES
2016 VLAN_CAPABILITIES(ifp);
2017#endif
2018 ADAPTER_UNLOCK(sc);
2019 break;
2020 case SIOCSIFMEDIA:
2021 case SIOCGIFMEDIA:
2022 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2023 break;
2024 default:
2025 error = ether_ioctl(ifp, command, data);
2026 }
2027
2028 return (error);
2029}
2030
2031static int
2032cxgb_media_change(struct ifnet *ifp)
2033{
2034 return (EOPNOTSUPP);
2035}
2036
2037/*
2038 * Translates phy->modtype to the correct Ethernet media subtype.
2039 */
2040static int
2041cxgb_ifm_type(int mod)
2042{
2043 switch (mod) {
2044 case phy_modtype_sr:
2045 return (IFM_10G_SR);
2046 case phy_modtype_lr:
2047 return (IFM_10G_LR);
2048 case phy_modtype_lrm:
2049 return (IFM_10G_LRM);
2050 case phy_modtype_twinax:
2051 return (IFM_10G_TWINAX);
2052 case phy_modtype_twinax_long:
2053 return (IFM_10G_TWINAX_LONG);
2054 case phy_modtype_none:
2055 return (IFM_NONE);
2056 case phy_modtype_unknown:
2057 return (IFM_UNKNOWN);
2058 }
2059
2060 KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2061 return (IFM_UNKNOWN);
2062}
2063
2064/*
2065 * Rebuilds the ifmedia list for this port, and sets the current media.
2066 */
2067static void
2068cxgb_build_medialist(struct port_info *p)
2069{
2070 struct cphy *phy = &p->phy;
2071 struct ifmedia *media = &p->media;
2072 int mod = phy->modtype;
2073 int m = IFM_ETHER | IFM_FDX;
2074
2075 PORT_LOCK(p);
2076
2077 ifmedia_removeall(media);
2078 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2079 /* Copper (RJ45) */
2080
2081 if (phy->caps & SUPPORTED_10000baseT_Full)
2082 ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2083
2084 if (phy->caps & SUPPORTED_1000baseT_Full)
2085 ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2086
2087 if (phy->caps & SUPPORTED_100baseT_Full)
2088 ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2089
2090 if (phy->caps & SUPPORTED_10baseT_Full)
2091 ifmedia_add(media, m | IFM_10_T, mod, NULL);
2092
2093 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2094 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2095
2096 } else if (phy->caps & SUPPORTED_TP) {
2097 /* Copper (CX4) */
2098
2099 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2100 ("%s: unexpected cap 0x%x", __func__, phy->caps));
2101
2102 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2103 ifmedia_set(media, m | IFM_10G_CX4);
2104
2105 } else if (phy->caps & SUPPORTED_FIBRE &&
2106 phy->caps & SUPPORTED_10000baseT_Full) {
2107 /* 10G optical (but includes SFP+ twinax) */
2108
2109 m |= cxgb_ifm_type(mod);
2110 if (IFM_SUBTYPE(m) == IFM_NONE)
2111 m &= ~IFM_FDX;
2112
2113 ifmedia_add(media, m, mod, NULL);
2114 ifmedia_set(media, m);
2115
2116 } else if (phy->caps & SUPPORTED_FIBRE &&
2117 phy->caps & SUPPORTED_1000baseT_Full) {
2118 /* 1G optical */
2119
2120 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2121 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2122 ifmedia_set(media, m | IFM_1000_SX);
2123
2124 } else {
2125 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2126 phy->caps));
2127 }
2128
2129 PORT_UNLOCK(p);
2130}
2131
2132static void
2133cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2134{
2135 struct port_info *p = ifp->if_softc;
2136 struct ifmedia_entry *cur = p->media.ifm_cur;
2137 int speed = p->link_config.speed;
2138
2139 if (cur->ifm_data != p->phy.modtype) {
2140 cxgb_build_medialist(p);
2141 cur = p->media.ifm_cur;
2142 }
2143
2144 ifmr->ifm_status = IFM_AVALID;
2145 if (!p->link_config.link_ok)
2146 return;
2147
2148 ifmr->ifm_status |= IFM_ACTIVE;
2149
2150 /*
2151 * active and current will differ iff current media is autoselect. That
2152 * can happen only for copper RJ45.
2153 */
2154 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2155 return;
2156 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2157 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2158
2159 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2160 if (speed == SPEED_10000)
2161 ifmr->ifm_active |= IFM_10G_T;
2162 else if (speed == SPEED_1000)
2163 ifmr->ifm_active |= IFM_1000_T;
2164 else if (speed == SPEED_100)
2165 ifmr->ifm_active |= IFM_100_TX;
2166 else if (speed == SPEED_10)
2167 ifmr->ifm_active |= IFM_10_T;
2168 else
2169 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2170 speed));
2171}
2172
2173static void
2174cxgb_async_intr(void *data)
2175{
2176 adapter_t *sc = data;
2177
2178 t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
2179 (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
2180 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2181}
2182
2183static void
2184link_check_callout(void *arg)
2185{
2186 struct port_info *pi = arg;
2187 struct adapter *sc = pi->adapter;
2188
2189 if (!isset(&sc->open_device_map, pi->port_id))
2190 return;
2191
2192 taskqueue_enqueue(sc->tq, &pi->link_check_task);
2193}
2194
2195static void
2196check_link_status(void *arg, int pending)
2197{
2198 struct port_info *pi = arg;
2199 struct adapter *sc = pi->adapter;
2200
2201 if (!isset(&sc->open_device_map, pi->port_id))
2202 return;
2203
2204 t3_link_changed(sc, pi->port_id);
2205
2206 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ))
2207 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
2208}
2209
2210void
2211t3_os_link_intr(struct port_info *pi)
2212{
2213 /*
2214 * Schedule a link check in the near future. If the link is flapping
2215 * rapidly we'll keep resetting the callout and delaying the check until
2216 * things stabilize a bit.
2217 */
2218 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
2219}
2220
2221static void
2222check_t3b2_mac(struct adapter *sc)
2223{
2224 int i;
2225
2226 if (sc->flags & CXGB_SHUTDOWN)
2227 return;
2228
2229 for_each_port(sc, i) {
2230 struct port_info *p = &sc->port[i];
2231 int status;
2232#ifdef INVARIANTS
2233 struct ifnet *ifp = p->ifp;
2234#endif
2235
2236 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2237 !p->link_config.link_ok)
2238 continue;
2239
2240 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2241 ("%s: state mismatch (drv_flags %x, device_map %x)",
2242 __func__, ifp->if_drv_flags, sc->open_device_map));
2243
2244 PORT_LOCK(p);
2245 status = t3b2_mac_watchdog_task(&p->mac);
2246 if (status == 1)
2247 p->mac.stats.num_toggled++;
2248 else if (status == 2) {
2249 struct cmac *mac = &p->mac;
2250
2251 cxgb_update_mac_settings(p);
2252 t3_link_start(&p->phy, mac, &p->link_config);
2253 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2254 t3_port_intr_enable(sc, p->port_id);
2255 p->mac.stats.num_resets++;
2256 }
2257 PORT_UNLOCK(p);
2258 }
2259}
2260
2261static void
2262cxgb_tick(void *arg)
2263{
2264 adapter_t *sc = (adapter_t *)arg;
2265
2266 if (sc->flags & CXGB_SHUTDOWN)
2267 return;
2268
2269 taskqueue_enqueue(sc->tq, &sc->tick_task);
2270 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2271}
2272
2273static void
2274cxgb_tick_handler(void *arg, int count)
2275{
2276 adapter_t *sc = (adapter_t *)arg;
2277 const struct adapter_params *p = &sc->params;
2278 int i;
2279 uint32_t cause, reset;
2280
2281 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2282 return;
2283
2284 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2285 check_t3b2_mac(sc);
2286
2287 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
2288 if (cause) {
2289 struct sge_qset *qs = &sc->sge.qs[0];
2290 uint32_t mask, v;
2291
2292 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
2293
2294 mask = 1;
2295 for (i = 0; i < SGE_QSETS; i++) {
2296 if (v & mask)
2297 qs[i].rspq.starved++;
2298 mask <<= 1;
2299 }
2300
2301 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
2302
2303 for (i = 0; i < SGE_QSETS * 2; i++) {
2304 if (v & mask) {
2305 qs[i / 2].fl[i % 2].empty++;
2306 }
2307 mask <<= 1;
2308 }
2309
2310 /* clear */
2311 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
2312 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
2313 }
2314
2315 for (i = 0; i < sc->params.nports; i++) {
2316 struct port_info *pi = &sc->port[i];
2317 struct ifnet *ifp = pi->ifp;
2318 struct cmac *mac = &pi->mac;
2319 struct mac_stats *mstats = &mac->stats;
2320 int drops, j;
2321
2322 if (!isset(&sc->open_device_map, pi->port_id))
2323 continue;
2324
2325 PORT_LOCK(pi);
2326 t3_mac_update_stats(mac);
2327 PORT_UNLOCK(pi);
2328
2329 ifp->if_opackets = mstats->tx_frames;
2330 ifp->if_ipackets = mstats->rx_frames;
2331 ifp->if_obytes = mstats->tx_octets;
2332 ifp->if_ibytes = mstats->rx_octets;
2333 ifp->if_omcasts = mstats->tx_mcast_frames;
2334 ifp->if_imcasts = mstats->rx_mcast_frames;
2335 ifp->if_collisions = mstats->tx_total_collisions;
2336 ifp->if_iqdrops = mstats->rx_cong_drops;
2337
2338 drops = 0;
2339 for (j = pi->first_qset; j < pi->first_qset + pi->nqsets; j++)
2340 drops += sc->sge.qs[j].txq[TXQ_ETH].txq_mr->br_drops;
2341 ifp->if_snd.ifq_drops = drops;
2342
2343 ifp->if_oerrors =
2344 mstats->tx_excess_collisions +
2345 mstats->tx_underrun +
2346 mstats->tx_len_errs +
2347 mstats->tx_mac_internal_errs +
2348 mstats->tx_excess_deferral +
2349 mstats->tx_fcs_errs;
2350 ifp->if_ierrors =
2351 mstats->rx_jabber +
2352 mstats->rx_data_errs +
2353 mstats->rx_sequence_errs +
2354 mstats->rx_runt +
2355 mstats->rx_too_long +
2356 mstats->rx_mac_internal_errs +
2357 mstats->rx_short +
2358 mstats->rx_fcs_errs;
2359
2360 if (mac->multiport)
2361 continue;
2362
2363 /* Count rx fifo overflows, once per second */
2364 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2365 reset = 0;
2366 if (cause & F_RXFIFO_OVERFLOW) {
2367 mac->stats.rx_fifo_ovfl++;
2368 reset |= F_RXFIFO_OVERFLOW;
2369 }
2370 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2371 }
2372}
2373
2374static void
2375touch_bars(device_t dev)
2376{
2377 /*
2378 * Don't enable yet
2379 */
2380#if !defined(__LP64__) && 0
2381 u32 v;
2382
2383 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2384 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2385 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2386 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2387 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2388 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2389#endif
2390}
2391
2392static int
2393set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2394{
2395 uint8_t *buf;
2396 int err = 0;
2397 u32 aligned_offset, aligned_len, *p;
2398 struct adapter *adapter = pi->adapter;
2399
2400
2401 aligned_offset = offset & ~3;
2402 aligned_len = (len + (offset & 3) + 3) & ~3;
2403
2404 if (aligned_offset != offset || aligned_len != len) {
2405 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2406 if (!buf)
2407 return (ENOMEM);
2408 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2409 if (!err && aligned_len > 4)
2410 err = t3_seeprom_read(adapter,
2411 aligned_offset + aligned_len - 4,
2412 (u32 *)&buf[aligned_len - 4]);
2413 if (err)
2414 goto out;
2415 memcpy(buf + (offset & 3), data, len);
2416 } else
2417 buf = (uint8_t *)(uintptr_t)data;
2418
2419 err = t3_seeprom_wp(adapter, 0);
2420 if (err)
2421 goto out;
2422
2423 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2424 err = t3_seeprom_write(adapter, aligned_offset, *p);
2425 aligned_offset += 4;
2426 }
2427
2428 if (!err)
2429 err = t3_seeprom_wp(adapter, 1);
2430out:
2431 if (buf != data)
2432 free(buf, M_DEVBUF);
2433 return err;
2434}
2435
2436
2437static int
2438in_range(int val, int lo, int hi)
2439{
2440 return val < 0 || (val <= hi && val >= lo);
2441}
2442
2443static int
2444cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2445{
2446 return (0);
2447}
2448
2449static int
2450cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2451{
2452 return (0);
2453}
2454
2455static int
2456cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2457 int fflag, struct thread *td)
2458{
2459 int mmd, error = 0;
2460 struct port_info *pi = dev->si_drv1;
2461 adapter_t *sc = pi->adapter;
2462
2463#ifdef PRIV_SUPPORTED
2464 if (priv_check(td, PRIV_DRIVER)) {
2465 if (cxgb_debug)
2466 printf("user does not have access to privileged ioctls\n");
2467 return (EPERM);
2468 }
2469#else
2470 if (suser(td)) {
2471 if (cxgb_debug)
2472 printf("user does not have access to privileged ioctls\n");
2473 return (EPERM);
2474 }
2475#endif
2476
2477 switch (cmd) {
2478 case CHELSIO_GET_MIIREG: {
2479 uint32_t val;
2480 struct cphy *phy = &pi->phy;
2481 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2482
2483 if (!phy->mdio_read)
2484 return (EOPNOTSUPP);
2485 if (is_10G(sc)) {
2486 mmd = mid->phy_id >> 8;
2487 if (!mmd)
2488 mmd = MDIO_DEV_PCS;
2489 else if (mmd > MDIO_DEV_VEND2)
2490 return (EINVAL);
2491
2492 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2493 mid->reg_num, &val);
2494 } else
2495 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2496 mid->reg_num & 0x1f, &val);
2497 if (error == 0)
2498 mid->val_out = val;
2499 break;
2500 }
2501 case CHELSIO_SET_MIIREG: {
2502 struct cphy *phy = &pi->phy;
2503 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2504
2505 if (!phy->mdio_write)
2506 return (EOPNOTSUPP);
2507 if (is_10G(sc)) {
2508 mmd = mid->phy_id >> 8;
2509 if (!mmd)
2510 mmd = MDIO_DEV_PCS;
2511 else if (mmd > MDIO_DEV_VEND2)
2512 return (EINVAL);
2513
2514 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2515 mmd, mid->reg_num, mid->val_in);
2516 } else
2517 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2518 mid->reg_num & 0x1f,
2519 mid->val_in);
2520 break;
2521 }
2522 case CHELSIO_SETREG: {
2523 struct ch_reg *edata = (struct ch_reg *)data;
2524 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2525 return (EFAULT);
2526 t3_write_reg(sc, edata->addr, edata->val);
2527 break;
2528 }
2529 case CHELSIO_GETREG: {
2530 struct ch_reg *edata = (struct ch_reg *)data;
2531 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2532 return (EFAULT);
2533 edata->val = t3_read_reg(sc, edata->addr);
2534 break;
2535 }
2536 case CHELSIO_GET_SGE_CONTEXT: {
2537 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2538 mtx_lock_spin(&sc->sge.reg_lock);
2539 switch (ecntxt->cntxt_type) {
2540 case CNTXT_TYPE_EGRESS:
2541 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2542 ecntxt->data);
2543 break;
2544 case CNTXT_TYPE_FL:
2545 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2546 ecntxt->data);
2547 break;
2548 case CNTXT_TYPE_RSP:
2549 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2550 ecntxt->data);
2551 break;
2552 case CNTXT_TYPE_CQ:
2553 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2554 ecntxt->data);
2555 break;
2556 default:
2557 error = EINVAL;
2558 break;
2559 }
2560 mtx_unlock_spin(&sc->sge.reg_lock);
2561 break;
2562 }
2563 case CHELSIO_GET_SGE_DESC: {
2564 struct ch_desc *edesc = (struct ch_desc *)data;
2565 int ret;
2566 if (edesc->queue_num >= SGE_QSETS * 6)
2567 return (EINVAL);
2568 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2569 edesc->queue_num % 6, edesc->idx, edesc->data);
2570 if (ret < 0)
2571 return (EINVAL);
2572 edesc->size = ret;
2573 break;
2574 }
2575 case CHELSIO_GET_QSET_PARAMS: {
2576 struct qset_params *q;
2577 struct ch_qset_params *t = (struct ch_qset_params *)data;
2578 int q1 = pi->first_qset;
2579 int nqsets = pi->nqsets;
2580 int i;
2581
2582 if (t->qset_idx >= nqsets)
2583 return EINVAL;
2584
2585 i = q1 + t->qset_idx;
2586 q = &sc->params.sge.qset[i];
2587 t->rspq_size = q->rspq_size;
2588 t->txq_size[0] = q->txq_size[0];
2589 t->txq_size[1] = q->txq_size[1];
2590 t->txq_size[2] = q->txq_size[2];
2591 t->fl_size[0] = q->fl_size;
2592 t->fl_size[1] = q->jumbo_size;
2593 t->polling = q->polling;
2594 t->lro = q->lro;
2595 t->intr_lat = q->coalesce_usecs;
2596 t->cong_thres = q->cong_thres;
2597 t->qnum = i;
2598
2599 if ((sc->flags & FULL_INIT_DONE) == 0)
2600 t->vector = 0;
2601 else if (sc->flags & USING_MSIX)
2602 t->vector = rman_get_start(sc->msix_irq_res[i]);
2603 else
2604 t->vector = rman_get_start(sc->irq_res);
2605
2606 break;
2607 }
2608 case CHELSIO_GET_QSET_NUM: {
2609 struct ch_reg *edata = (struct ch_reg *)data;
2610 edata->val = pi->nqsets;
2611 break;
2612 }
2613 case CHELSIO_LOAD_FW: {
2614 uint8_t *fw_data;
2615 uint32_t vers;
2616 struct ch_mem_range *t = (struct ch_mem_range *)data;
2617
2618 /*
2619 * You're allowed to load a firmware only before FULL_INIT_DONE
2620 *
2621 * FW_UPTODATE is also set so the rest of the initialization
2622 * will not overwrite what was loaded here. This gives you the
2623 * flexibility to load any firmware (and maybe shoot yourself in
2624 * the foot).
2625 */
2626
2627 ADAPTER_LOCK(sc);
2628 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2629 ADAPTER_UNLOCK(sc);
2630 return (EBUSY);
2631 }
2632
2633 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2634 if (!fw_data)
2635 error = ENOMEM;
2636 else
2637 error = copyin(t->buf, fw_data, t->len);
2638
2639 if (!error)
2640 error = -t3_load_fw(sc, fw_data, t->len);
2641
2642 if (t3_get_fw_version(sc, &vers) == 0) {
2643 snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2644 "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2645 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2646 }
2647
2648 if (!error)
2649 sc->flags |= FW_UPTODATE;
2650
2651 free(fw_data, M_DEVBUF);
2652 ADAPTER_UNLOCK(sc);
2653 break;
2654 }
2655 case CHELSIO_LOAD_BOOT: {
2656 uint8_t *boot_data;
2657 struct ch_mem_range *t = (struct ch_mem_range *)data;
2658
2659 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2660 if (!boot_data)
2661 return ENOMEM;
2662
2663 error = copyin(t->buf, boot_data, t->len);
2664 if (!error)
2665 error = -t3_load_boot(sc, boot_data, t->len);
2666
2667 free(boot_data, M_DEVBUF);
2668 break;
2669 }
2670 case CHELSIO_GET_PM: {
2671 struct ch_pm *m = (struct ch_pm *)data;
2672 struct tp_params *p = &sc->params.tp;
2673
2674 if (!is_offload(sc))
2675 return (EOPNOTSUPP);
2676
2677 m->tx_pg_sz = p->tx_pg_size;
2678 m->tx_num_pg = p->tx_num_pgs;
2679 m->rx_pg_sz = p->rx_pg_size;
2680 m->rx_num_pg = p->rx_num_pgs;
2681 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2682
2683 break;
2684 }
2685 case CHELSIO_SET_PM: {
2686 struct ch_pm *m = (struct ch_pm *)data;
2687 struct tp_params *p = &sc->params.tp;
2688
2689 if (!is_offload(sc))
2690 return (EOPNOTSUPP);
2691 if (sc->flags & FULL_INIT_DONE)
2692 return (EBUSY);
2693
2694 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2695 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2696 return (EINVAL); /* not power of 2 */
2697 if (!(m->rx_pg_sz & 0x14000))
2698 return (EINVAL); /* not 16KB or 64KB */
2699 if (!(m->tx_pg_sz & 0x1554000))
2700 return (EINVAL);
2701 if (m->tx_num_pg == -1)
2702 m->tx_num_pg = p->tx_num_pgs;
2703 if (m->rx_num_pg == -1)
2704 m->rx_num_pg = p->rx_num_pgs;
2705 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2706 return (EINVAL);
2707 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2708 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2709 return (EINVAL);
2710
2711 p->rx_pg_size = m->rx_pg_sz;
2712 p->tx_pg_size = m->tx_pg_sz;
2713 p->rx_num_pgs = m->rx_num_pg;
2714 p->tx_num_pgs = m->tx_num_pg;
2715 break;
2716 }
2717 case CHELSIO_SETMTUTAB: {
2718 struct ch_mtus *m = (struct ch_mtus *)data;
2719 int i;
2720
2721 if (!is_offload(sc))
2722 return (EOPNOTSUPP);
2723 if (offload_running(sc))
2724 return (EBUSY);
2725 if (m->nmtus != NMTUS)
2726 return (EINVAL);
2727 if (m->mtus[0] < 81) /* accommodate SACK */
2728 return (EINVAL);
2729
2730 /*
2731 * MTUs must be in ascending order
2732 */
2733 for (i = 1; i < NMTUS; ++i)
2734 if (m->mtus[i] < m->mtus[i - 1])
2735 return (EINVAL);
2736
2737 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2738 break;
2739 }
2740 case CHELSIO_GETMTUTAB: {
2741 struct ch_mtus *m = (struct ch_mtus *)data;
2742
2743 if (!is_offload(sc))
2744 return (EOPNOTSUPP);
2745
2746 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2747 m->nmtus = NMTUS;
2748 break;
2749 }
2750 case CHELSIO_GET_MEM: {
2751 struct ch_mem_range *t = (struct ch_mem_range *)data;
2752 struct mc7 *mem;
2753 uint8_t *useraddr;
2754 u64 buf[32];
2755
2756 /*
2757 * Use these to avoid modifying len/addr in the return
2758 * struct
2759 */
2760 uint32_t len = t->len, addr = t->addr;
2761
2762 if (!is_offload(sc))
2763 return (EOPNOTSUPP);
2764 if (!(sc->flags & FULL_INIT_DONE))
2765 return (EIO); /* need the memory controllers */
2766 if ((addr & 0x7) || (len & 0x7))
2767 return (EINVAL);
2768 if (t->mem_id == MEM_CM)
2769 mem = &sc->cm;
2770 else if (t->mem_id == MEM_PMRX)
2771 mem = &sc->pmrx;
2772 else if (t->mem_id == MEM_PMTX)
2773 mem = &sc->pmtx;
2774 else
2775 return (EINVAL);
2776
2777 /*
2778 * Version scheme:
2779 * bits 0..9: chip version
2780 * bits 10..15: chip revision
2781 */
2782 t->version = 3 | (sc->params.rev << 10);
2783
2784 /*
2785 * Read 256 bytes at a time as len can be large and we don't
2786 * want to use huge intermediate buffers.
2787 */
2788 useraddr = (uint8_t *)t->buf;
2789 while (len) {
2790 unsigned int chunk = min(len, sizeof(buf));
2791
2792 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2793 if (error)
2794 return (-error);
2795 if (copyout(buf, useraddr, chunk))
2796 return (EFAULT);
2797 useraddr += chunk;
2798 addr += chunk;
2799 len -= chunk;
2800 }
2801 break;
2802 }
2803 case CHELSIO_READ_TCAM_WORD: {
2804 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2805
2806 if (!is_offload(sc))
2807 return (EOPNOTSUPP);
2808 if (!(sc->flags & FULL_INIT_DONE))
2809 return (EIO); /* need MC5 */
2810 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2811 break;
2812 }
2813 case CHELSIO_SET_TRACE_FILTER: {
2814 struct ch_trace *t = (struct ch_trace *)data;
2815 const struct trace_params *tp;
2816
2817 tp = (const struct trace_params *)&t->sip;
2818 if (t->config_tx)
2819 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2820 t->trace_tx);
2821 if (t->config_rx)
2822 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2823 t->trace_rx);
2824 break;
2825 }
2826 case CHELSIO_SET_PKTSCHED: {
2827 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2828 if (sc->open_device_map == 0)
2829 return (EAGAIN);
2830 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2831 p->binding);
2832 break;
2833 }
2834 case CHELSIO_IFCONF_GETREGS: {
2835 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2836 int reglen = cxgb_get_regs_len();
2837 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2838 if (buf == NULL) {
2839 return (ENOMEM);
2840 }
2841 if (regs->len > reglen)
2842 regs->len = reglen;
2843 else if (regs->len < reglen)
2844 error = ENOBUFS;
2845
2846 if (!error) {
2847 cxgb_get_regs(sc, regs, buf);
2848 error = copyout(buf, regs->data, reglen);
2849 }
2850 free(buf, M_DEVBUF);
2851
2852 break;
2853 }
2854 case CHELSIO_SET_HW_SCHED: {
2855 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2856 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2857
2858 if ((sc->flags & FULL_INIT_DONE) == 0)
2859 return (EAGAIN); /* need TP to be initialized */
2860 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2861 !in_range(t->channel, 0, 1) ||
2862 !in_range(t->kbps, 0, 10000000) ||
2863 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2864 !in_range(t->flow_ipg, 0,
2865 dack_ticks_to_usec(sc, 0x7ff)))
2866 return (EINVAL);
2867
2868 if (t->kbps >= 0) {
2869 error = t3_config_sched(sc, t->kbps, t->sched);
2870 if (error < 0)
2871 return (-error);
2872 }
2873 if (t->class_ipg >= 0)
2874 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2875 if (t->flow_ipg >= 0) {
2876 t->flow_ipg *= 1000; /* us -> ns */
2877 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2878 }
2879 if (t->mode >= 0) {
2880 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2881
2882 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2883 bit, t->mode ? bit : 0);
2884 }
2885 if (t->channel >= 0)
2886 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2887 1 << t->sched, t->channel << t->sched);
2888 break;
2889 }
2890 case CHELSIO_GET_EEPROM: {
2891 int i;
2892 struct ch_eeprom *e = (struct ch_eeprom *)data;
2893 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2894
2895 if (buf == NULL) {
2896 return (ENOMEM);
2897 }
2898 e->magic = EEPROM_MAGIC;
2899 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2900 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2901
2902 if (!error)
2903 error = copyout(buf + e->offset, e->data, e->len);
2904
2905 free(buf, M_DEVBUF);
2906 break;
2907 }
2908 case CHELSIO_CLEAR_STATS: {
2909 if (!(sc->flags & FULL_INIT_DONE))
2910 return EAGAIN;
2911
2912 PORT_LOCK(pi);
2913 t3_mac_update_stats(&pi->mac);
2914 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
2915 PORT_UNLOCK(pi);
2916 break;
2917 }
2918 case CHELSIO_GET_UP_LA: {
2919 struct ch_up_la *la = (struct ch_up_la *)data;
2920 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
2921 if (buf == NULL) {
2922 return (ENOMEM);
2923 }
2924 if (la->bufsize < LA_BUFSIZE)
2925 error = ENOBUFS;
2926
2927 if (!error)
2928 error = -t3_get_up_la(sc, &la->stopped, &la->idx,
2929 &la->bufsize, buf);
2930 if (!error)
2931 error = copyout(buf, la->data, la->bufsize);
2932
2933 free(buf, M_DEVBUF);
2934 break;
2935 }
2936 case CHELSIO_GET_UP_IOQS: {
2937 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
2938 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
2939 uint32_t *v;
2940
2941 if (buf == NULL) {
2942 return (ENOMEM);
2943 }
2944 if (ioqs->bufsize < IOQS_BUFSIZE)
2945 error = ENOBUFS;
2946
2947 if (!error)
2948 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
2949
2950 if (!error) {
2951 v = (uint32_t *)buf;
2952
2953 ioqs->ioq_rx_enable = *v++;
2954 ioqs->ioq_tx_enable = *v++;
2955 ioqs->ioq_rx_status = *v++;
2956 ioqs->ioq_tx_status = *v++;
2957
2958 error = copyout(v, ioqs->data, ioqs->bufsize);
2959 }
2960
2961 free(buf, M_DEVBUF);
2962 break;
2963 }
2964 case CHELSIO_SET_FILTER: {
2965 struct ch_filter *f = (struct ch_filter *)data;;
2966 struct filter_info *p;
2967 unsigned int nfilters = sc->params.mc5.nfilters;
2968
2969 if (!is_offload(sc))
2970 return (EOPNOTSUPP); /* No TCAM */
2971 if (!(sc->flags & FULL_INIT_DONE))
2972 return (EAGAIN); /* mc5 not setup yet */
2973 if (nfilters == 0)
2974 return (EBUSY); /* TOE will use TCAM */
2975
2976 /* sanity checks */
2977 if (f->filter_id >= nfilters ||
2978 (f->val.dip && f->mask.dip != 0xffffffff) ||
2979 (f->val.sport && f->mask.sport != 0xffff) ||
2980 (f->val.dport && f->mask.dport != 0xffff) ||
2981 (f->val.vlan && f->mask.vlan != 0xfff) ||
2982 (f->val.vlan_prio &&
2983 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
2984 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
2985 f->qset >= SGE_QSETS ||
2986 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
2987 return (EINVAL);
2988
2989 /* Was allocated with M_WAITOK */
2990 KASSERT(sc->filters, ("filter table NULL\n"));
2991
2992 p = &sc->filters[f->filter_id];
2993 if (p->locked)
2994 return (EPERM);
2995
2996 bzero(p, sizeof(*p));
2997 p->sip = f->val.sip;
2998 p->sip_mask = f->mask.sip;
2999 p->dip = f->val.dip;
3000 p->sport = f->val.sport;
3001 p->dport = f->val.dport;
3002 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
3003 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
3004 FILTER_NO_VLAN_PRI;
3005 p->mac_hit = f->mac_hit;
3006 p->mac_vld = f->mac_addr_idx != 0xffff;
3007 p->mac_idx = f->mac_addr_idx;
3008 p->pkt_type = f->proto;
3009 p->report_filter_id = f->want_filter_id;
3010 p->pass = f->pass;
3011 p->rss = f->rss;
3012 p->qset = f->qset;
3013
3014 error = set_filter(sc, f->filter_id, p);
3015 if (error == 0)
3016 p->valid = 1;
3017 break;
3018 }
3019 case CHELSIO_DEL_FILTER: {
3020 struct ch_filter *f = (struct ch_filter *)data;
3021 struct filter_info *p;
3022 unsigned int nfilters = sc->params.mc5.nfilters;
3023
3024 if (!is_offload(sc))
3025 return (EOPNOTSUPP);
3026 if (!(sc->flags & FULL_INIT_DONE))
3027 return (EAGAIN);
3028 if (nfilters == 0 || sc->filters == NULL)
3029 return (EINVAL);
3030 if (f->filter_id >= nfilters)
3031 return (EINVAL);
3032
3033 p = &sc->filters[f->filter_id];
3034 if (p->locked)
3035 return (EPERM);
3036 if (!p->valid)
3037 return (EFAULT); /* Read "Bad address" as "Bad index" */
3038
3039 bzero(p, sizeof(*p));
3040 p->sip = p->sip_mask = 0xffffffff;
3041 p->vlan = 0xfff;
3042 p->vlan_prio = FILTER_NO_VLAN_PRI;
3043 p->pkt_type = 1;
3044 error = set_filter(sc, f->filter_id, p);
3045 break;
3046 }
3047 case CHELSIO_GET_FILTER: {
3048 struct ch_filter *f = (struct ch_filter *)data;
3049 struct filter_info *p;
3050 unsigned int i, nfilters = sc->params.mc5.nfilters;
3051
3052 if (!is_offload(sc))
3053 return (EOPNOTSUPP);
3054 if (!(sc->flags & FULL_INIT_DONE))
3055 return (EAGAIN);
3056 if (nfilters == 0 || sc->filters == NULL)
3057 return (EINVAL);
3058
3059 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
3060 for (; i < nfilters; i++) {
3061 p = &sc->filters[i];
3062 if (!p->valid)
3063 continue;
3064
3065 bzero(f, sizeof(*f));
3066
3067 f->filter_id = i;
3068 f->val.sip = p->sip;
3069 f->mask.sip = p->sip_mask;
3070 f->val.dip = p->dip;
3071 f->mask.dip = p->dip ? 0xffffffff : 0;
3072 f->val.sport = p->sport;
3073 f->mask.sport = p->sport ? 0xffff : 0;
3074 f->val.dport = p->dport;
3075 f->mask.dport = p->dport ? 0xffff : 0;
3076 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
3077 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
3078 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3079 0 : p->vlan_prio;
3080 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3081 0 : FILTER_NO_VLAN_PRI;
3082 f->mac_hit = p->mac_hit;
3083 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
3084 f->proto = p->pkt_type;
3085 f->want_filter_id = p->report_filter_id;
3086 f->pass = p->pass;
3087 f->rss = p->rss;
3088 f->qset = p->qset;
3089
3090 break;
3091 }
3092
3093 if (i == nfilters)
3094 f->filter_id = 0xffffffff;
3095 break;
3096 }
3097 default:
3098 return (EOPNOTSUPP);
3099 break;
3100 }
3101
3102 return (error);
3103}
3104
3105static __inline void
3106reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3107 unsigned int end)
3108{
3109 uint32_t *p = (uint32_t *)(buf + start);
3110
3111 for ( ; start <= end; start += sizeof(uint32_t))
3112 *p++ = t3_read_reg(ap, start);
3113}
3114
3115#define T3_REGMAP_SIZE (3 * 1024)
3116static int
3117cxgb_get_regs_len(void)
3118{
3119 return T3_REGMAP_SIZE;
3120}
3121
3122static void
3123cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3124{
3125
3126 /*
3127 * Version scheme:
3128 * bits 0..9: chip version
3129 * bits 10..15: chip revision
3130 * bit 31: set for PCIe cards
3131 */
3132 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3133
3134 /*
3135 * We skip the MAC statistics registers because they are clear-on-read.
3136 * Also reading multi-register stats would need to synchronize with the
3137 * periodic mac stats accumulation. Hard to justify the complexity.
3138 */
3139 memset(buf, 0, cxgb_get_regs_len());
3140 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3141 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3142 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3143 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3144 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3145 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3146 XGM_REG(A_XGM_SERDES_STAT3, 1));
3147 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3148 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3149}
3150
3151static int
3152alloc_filters(struct adapter *sc)
3153{
3154 struct filter_info *p;
3155 unsigned int nfilters = sc->params.mc5.nfilters;
3156
3157 if (nfilters == 0)
3158 return (0);
3159
3160 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
3161 sc->filters = p;
3162
3163 p = &sc->filters[nfilters - 1];
3164 p->vlan = 0xfff;
3165 p->vlan_prio = FILTER_NO_VLAN_PRI;
3166 p->pass = p->rss = p->valid = p->locked = 1;
3167
3168 return (0);
3169}
3170
3171static int
3172setup_hw_filters(struct adapter *sc)
3173{
3174 int i, rc;
3175 unsigned int nfilters = sc->params.mc5.nfilters;
3176
3177 if (!sc->filters)
3178 return (0);
3179
3180 t3_enable_filters(sc);
3181
3182 for (i = rc = 0; i < nfilters && !rc; i++) {
3183 if (sc->filters[i].locked)
3184 rc = set_filter(sc, i, &sc->filters[i]);
3185 }
3186
3187 return (rc);
3188}
3189
3190static int
3191set_filter(struct adapter *sc, int id, const struct filter_info *f)
3192{
3193 int len;
3194 struct mbuf *m;
3195 struct ulp_txpkt *txpkt;
3196 struct work_request_hdr *wr;
3197 struct cpl_pass_open_req *oreq;
3198 struct cpl_set_tcb_field *sreq;
3199
3200 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
3201 KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
3202
3203 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
3204 sc->params.mc5.nfilters;
3205
3206 m = m_gethdr(M_WAITOK, MT_DATA);
3207 m->m_len = m->m_pkthdr.len = len;
3208 bzero(mtod(m, char *), len);
3209
3210 wr = mtod(m, struct work_request_hdr *);
3211 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
3212
3213 oreq = (struct cpl_pass_open_req *)(wr + 1);
3214 txpkt = (struct ulp_txpkt *)oreq;
3215 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3216 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
3217 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
3218 oreq->local_port = htons(f->dport);
3219 oreq->peer_port = htons(f->sport);
3220 oreq->local_ip = htonl(f->dip);
3221 oreq->peer_ip = htonl(f->sip);
3222 oreq->peer_netmask = htonl(f->sip_mask);
3223 oreq->opt0h = 0;
3224 oreq->opt0l = htonl(F_NO_OFFLOAD);
3225 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
3226 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
3227 V_VLAN_PRI(f->vlan_prio >> 1) |
3228 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
3229 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
3230 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
3231
3232 sreq = (struct cpl_set_tcb_field *)(oreq + 1);
3233 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
3234 (f->report_filter_id << 15) | (1 << 23) |
3235 ((u64)f->pass << 35) | ((u64)!f->rss << 36));
3236 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
3237 t3_mgmt_tx(sc, m);
3238
3239 if (f->pass && !f->rss) {
3240 len = sizeof(*sreq);
3241 m = m_gethdr(M_WAITOK, MT_DATA);
3242 m->m_len = m->m_pkthdr.len = len;
3243 bzero(mtod(m, char *), len);
3244 sreq = mtod(m, struct cpl_set_tcb_field *);
3245 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3246 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
3247 (u64)sc->rrss_map[f->qset] << 19);
3248 t3_mgmt_tx(sc, m);
3249 }
3250 return 0;
3251}
3252
3253static inline void
3254mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
3255 unsigned int word, u64 mask, u64 val)
3256{
3257 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3258 req->reply = V_NO_REPLY(1);
3259 req->cpu_idx = 0;
3260 req->word = htons(word);
3261 req->mask = htobe64(mask);
3262 req->val = htobe64(val);
3263}
3264
3265static inline void
3266set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3267 unsigned int word, u64 mask, u64 val)
3268{
3269 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3270
3271 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3272 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3273 mk_set_tcb_field(req, tid, word, mask, val);
3274}
3275
3276void
3277t3_iterate(void (*func)(struct adapter *, void *), void *arg)
3278{
3279 struct adapter *sc;
3280
3281 mtx_lock(&t3_list_lock);
3282 SLIST_FOREACH(sc, &t3_list, link) {
3283 /*
3284 * func should not make any assumptions about what state sc is
3285 * in - the only guarantee is that sc->sc_lock is a valid lock.
3286 */
3287 func(sc, arg);
3288 }
3289 mtx_unlock(&t3_list_lock);
3290}
3291
3292#ifdef TCP_OFFLOAD
3293static int
3294toe_capability(struct port_info *pi, int enable)
3295{
3296 int rc;
3297 struct adapter *sc = pi->adapter;
3298
3299 ADAPTER_LOCK_ASSERT_OWNED(sc);
3300
3301 if (!is_offload(sc))
3302 return (ENODEV);
3303
3304 if (enable) {
3305 if (!(sc->flags & FULL_INIT_DONE)) {
3306 log(LOG_WARNING,
3307 "You must enable a cxgb interface first\n");
3308 return (EAGAIN);
3309 }
3310
3311 if (isset(&sc->offload_map, pi->port_id))
3312 return (0);
3313
3314 if (!(sc->flags & TOM_INIT_DONE)) {
3315 rc = t3_activate_uld(sc, ULD_TOM);
3316 if (rc == EAGAIN) {
3317 log(LOG_WARNING,
3318 "You must kldload t3_tom.ko before trying "
3319 "to enable TOE on a cxgb interface.\n");
3320 }
3321 if (rc != 0)
3322 return (rc);
3323 KASSERT(sc->tom_softc != NULL,
3324 ("%s: TOM activated but softc NULL", __func__));
3325 KASSERT(sc->flags & TOM_INIT_DONE,
3326 ("%s: TOM activated but flag not set", __func__));
3327 }
3328
3329 setbit(&sc->offload_map, pi->port_id);
3330
3331 /*
3332 * XXX: Temporary code to allow iWARP to be enabled when TOE is
3333 * enabled on any port. Need to figure out how to enable,
3334 * disable, load, and unload iWARP cleanly.
3335 */
3336 if (!isset(&sc->offload_map, MAX_NPORTS) &&
3337 t3_activate_uld(sc, ULD_IWARP) == 0)
3338 setbit(&sc->offload_map, MAX_NPORTS);
3339 } else {
3340 if (!isset(&sc->offload_map, pi->port_id))
3341 return (0);
3342
3343 KASSERT(sc->flags & TOM_INIT_DONE,
3344 ("%s: TOM never initialized?", __func__));
3345 clrbit(&sc->offload_map, pi->port_id);
3346 }
3347
3348 return (0);
3349}
3350
3351/*
3352 * Add an upper layer driver to the global list.
3353 */
3354int
3355t3_register_uld(struct uld_info *ui)
3356{
3357 int rc = 0;
3358 struct uld_info *u;
3359
3360 mtx_lock(&t3_uld_list_lock);
3361 SLIST_FOREACH(u, &t3_uld_list, link) {
3362 if (u->uld_id == ui->uld_id) {
3363 rc = EEXIST;
3364 goto done;
3365 }
3366 }
3367
3368 SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
3369 ui->refcount = 0;
3370done:
3371 mtx_unlock(&t3_uld_list_lock);
3372 return (rc);
3373}
3374
3375int
3376t3_unregister_uld(struct uld_info *ui)
3377{
3378 int rc = EINVAL;
3379 struct uld_info *u;
3380
3381 mtx_lock(&t3_uld_list_lock);
3382
3383 SLIST_FOREACH(u, &t3_uld_list, link) {
3384 if (u == ui) {
3385 if (ui->refcount > 0) {
3386 rc = EBUSY;
3387 goto done;
3388 }
3389
3390 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
3391 rc = 0;
3392 goto done;
3393 }
3394 }
3395done:
3396 mtx_unlock(&t3_uld_list_lock);
3397 return (rc);
3398}
3399
3400int
3401t3_activate_uld(struct adapter *sc, int id)
3402{
3403 int rc = EAGAIN;
3404 struct uld_info *ui;
3405
3406 mtx_lock(&t3_uld_list_lock);
3407
3408 SLIST_FOREACH(ui, &t3_uld_list, link) {
3409 if (ui->uld_id == id) {
3410 rc = ui->activate(sc);
3411 if (rc == 0)
3412 ui->refcount++;
3413 goto done;
3414 }
3415 }
3416done:
3417 mtx_unlock(&t3_uld_list_lock);
3418
3419 return (rc);
3420}
3421
3422int
3423t3_deactivate_uld(struct adapter *sc, int id)
3424{
3425 int rc = EINVAL;
3426 struct uld_info *ui;
3427
3428 mtx_lock(&t3_uld_list_lock);
3429
3430 SLIST_FOREACH(ui, &t3_uld_list, link) {
3431 if (ui->uld_id == id) {
3432 rc = ui->deactivate(sc);
3433 if (rc == 0)
3434 ui->refcount--;
3435 goto done;
3436 }
3437 }
3438done:
3439 mtx_unlock(&t3_uld_list_lock);
3440
3441 return (rc);
3442}
3443
3444static int
3445cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
3446 struct mbuf *m)
3447{
3448 m_freem(m);
3449 return (EDOOFUS);
3450}
3451
3452int
3453t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3454{
3455 uintptr_t *loc, new;
3456
3457 if (opcode >= NUM_CPL_HANDLERS)
3458 return (EINVAL);
3459
3460 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3461 loc = (uintptr_t *) &sc->cpl_handler[opcode];
3462 atomic_store_rel_ptr(loc, new);
3463
3464 return (0);
3465}
3466#endif
3467
3468static int
3469cxgbc_mod_event(module_t mod, int cmd, void *arg)
3470{
3471 int rc = 0;
3472
3473 switch (cmd) {
3474 case MOD_LOAD:
3475 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
3476 SLIST_INIT(&t3_list);
3477#ifdef TCP_OFFLOAD
3478 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
3479 SLIST_INIT(&t3_uld_list);
3480#endif
3481 break;
3482
3483 case MOD_UNLOAD:
3484#ifdef TCP_OFFLOAD
3485 mtx_lock(&t3_uld_list_lock);
3486 if (!SLIST_EMPTY(&t3_uld_list)) {
3487 rc = EBUSY;
3488 mtx_unlock(&t3_uld_list_lock);
3489 break;
3490 }
3491 mtx_unlock(&t3_uld_list_lock);
3492 mtx_destroy(&t3_uld_list_lock);
3493#endif
3494 mtx_lock(&t3_list_lock);
3495 if (!SLIST_EMPTY(&t3_list)) {
3496 rc = EBUSY;
3497 mtx_unlock(&t3_list_lock);
3498 break;
3499 }
3500 mtx_unlock(&t3_list_lock);
3501 mtx_destroy(&t3_list_lock);
3502 break;
3503 }
3504
3505 return (rc);
3506}