1/* 2 * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved. 3 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#include <linux/pci.h> 35#include <linux/netdevice.h> 36#include <linux/slab.h> 37#include <linux/vmalloc.h> 38 39#include "ipath_kernel.h" 40#include "ipath_common.h" 41 42/* 43 * min buffers we want to have per port, after driver 44 */ 45#define IPATH_MIN_USER_PORT_BUFCNT 7 46 47/* 48 * Number of ports we are configured to use (to allow for more pio 49 * buffers per port, etc.) Zero means use chip value. 50 */ 51static ushort ipath_cfgports; 52 53module_param_named(cfgports, ipath_cfgports, ushort, S_IRUGO); 54MODULE_PARM_DESC(cfgports, "Set max number of ports to use"); 55 56/* 57 * Number of buffers reserved for driver (verbs and layered drivers.) 58 * Initialized based on number of PIO buffers if not set via module interface. 59 * The problem with this is that it's global, but we'll use different 60 * numbers for different chip types. 61 */ 62static ushort ipath_kpiobufs; 63 64static int ipath_set_kpiobufs(const char *val, struct kernel_param *kp); 65 66module_param_call(kpiobufs, ipath_set_kpiobufs, param_get_ushort, 67 &ipath_kpiobufs, S_IWUSR | S_IRUGO); 68MODULE_PARM_DESC(kpiobufs, "Set number of PIO buffers for driver"); 69 70/** 71 * create_port0_egr - allocate the eager TID buffers 72 * @dd: the infinipath device 73 * 74 * This code is now quite different for user and kernel, because 75 * the kernel uses skb's, for the accelerated network performance. 76 * This is the kernel (port0) version. 77 * 78 * Allocate the eager TID buffers and program them into infinipath. 79 * We use the network layer alloc_skb() allocator to allocate the 80 * memory, and either use the buffers as is for things like verbs 81 * packets, or pass the buffers up to the ipath layered driver and 82 * thence the network layer, replacing them as we do so (see 83 * ipath_rcv_layer()). 84 */ 85static int create_port0_egr(struct ipath_devdata *dd) 86{ 87 unsigned e, egrcnt; 88 struct ipath_skbinfo *skbinfo; 89 int ret; 90 91 egrcnt = dd->ipath_p0_rcvegrcnt; 92 93 skbinfo = vmalloc(sizeof(*dd->ipath_port0_skbinfo) * egrcnt); 94 if (skbinfo == NULL) { 95 ipath_dev_err(dd, "allocation error for eager TID " 96 "skb array\n"); 97 ret = -ENOMEM; 98 goto bail; 99 } 100 for (e = 0; e < egrcnt; e++) { 101 /* 102 * This is a bit tricky in that we allocate extra 103 * space for 2 bytes of the 14 byte ethernet header. 104 * These two bytes are passed in the ipath header so 105 * the rest of the data is word aligned. We allocate 106 * 4 bytes so that the data buffer stays word aligned. 107 * See ipath_kreceive() for more details. 108 */ 109 skbinfo[e].skb = ipath_alloc_skb(dd, GFP_KERNEL); 110 if (!skbinfo[e].skb) { 111 ipath_dev_err(dd, "SKB allocation error for " 112 "eager TID %u\n", e); 113 while (e != 0) 114 dev_kfree_skb(skbinfo[--e].skb); 115 vfree(skbinfo); 116 ret = -ENOMEM; 117 goto bail; 118 } 119 } 120 /* 121 * After loop above, so we can test non-NULL to see if ready 122 * to use at receive, etc. 123 */ 124 dd->ipath_port0_skbinfo = skbinfo; 125 126 for (e = 0; e < egrcnt; e++) { 127 dd->ipath_port0_skbinfo[e].phys = 128 ipath_map_single(dd->pcidev, 129 dd->ipath_port0_skbinfo[e].skb->data, 130 dd->ipath_ibmaxlen, PCI_DMA_FROMDEVICE); 131 dd->ipath_f_put_tid(dd, e + (u64 __iomem *) 132 ((char __iomem *) dd->ipath_kregbase + 133 dd->ipath_rcvegrbase), 134 RCVHQ_RCV_TYPE_EAGER, 135 dd->ipath_port0_skbinfo[e].phys); 136 } 137 138 ret = 0; 139 140bail: 141 return ret; 142} 143 144static int bringup_link(struct ipath_devdata *dd) 145{ 146 u64 val, ibc; 147 int ret = 0; 148 149 /* hold IBC in reset */ 150 dd->ipath_control &= ~INFINIPATH_C_LINKENABLE; 151 ipath_write_kreg(dd, dd->ipath_kregs->kr_control, 152 dd->ipath_control); 153 154 /* 155 * set initial max size pkt IBC will send, including ICRC; it's the 156 * PIO buffer size in dwords, less 1; also see ipath_set_mtu() 157 */ 158 val = (dd->ipath_ibmaxlen >> 2) + 1; 159 ibc = val << dd->ibcc_mpl_shift; 160 161 /* flowcontrolwatermark is in units of KBytes */ 162 ibc |= 0x5ULL << INFINIPATH_IBCC_FLOWCTRLWATERMARK_SHIFT; 163 /* 164 * How often flowctrl sent. More or less in usecs; balance against 165 * watermark value, so that in theory senders always get a flow 166 * control update in time to not let the IB link go idle. 167 */ 168 ibc |= 0x3ULL << INFINIPATH_IBCC_FLOWCTRLPERIOD_SHIFT; 169 /* max error tolerance */ 170 ibc |= 0xfULL << INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT; 171 /* use "real" buffer space for */ 172 ibc |= 4ULL << INFINIPATH_IBCC_CREDITSCALE_SHIFT; 173 /* IB credit flow control. */ 174 ibc |= 0xfULL << INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT; 175 /* initially come up waiting for TS1, without sending anything. */ 176 dd->ipath_ibcctrl = ibc; 177 /* 178 * Want to start out with both LINKCMD and LINKINITCMD in NOP 179 * (0 and 0). Don't put linkinitcmd in ipath_ibcctrl, want that 180 * to stay a NOP. Flag that we are disabled, for the (unlikely) 181 * case that some recovery path is trying to bring the link up 182 * before we are ready. 183 */ 184 ibc |= INFINIPATH_IBCC_LINKINITCMD_DISABLE << 185 INFINIPATH_IBCC_LINKINITCMD_SHIFT; 186 dd->ipath_flags |= IPATH_IB_LINK_DISABLED; 187 ipath_cdbg(VERBOSE, "Writing 0x%llx to ibcctrl\n", 188 (unsigned long long) ibc); 189 ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl, ibc); 190 191 // be sure chip saw it 192 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 193 194 ret = dd->ipath_f_bringup_serdes(dd); 195 196 if (ret) 197 dev_info(&dd->pcidev->dev, "Could not initialize SerDes, " 198 "not usable\n"); 199 else { 200 /* enable IBC */ 201 dd->ipath_control |= INFINIPATH_C_LINKENABLE; 202 ipath_write_kreg(dd, dd->ipath_kregs->kr_control, 203 dd->ipath_control); 204 } 205 206 return ret; 207} 208 209static struct ipath_portdata *create_portdata0(struct ipath_devdata *dd) 210{ 211 struct ipath_portdata *pd = NULL; 212 213 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 214 if (pd) { 215 pd->port_dd = dd; 216 pd->port_cnt = 1; 217 /* The port 0 pkey table is used by the layer interface. */ 218 pd->port_pkeys[0] = IPATH_DEFAULT_P_KEY; 219 pd->port_seq_cnt = 1; 220 } 221 return pd; 222} 223 224static int init_chip_first(struct ipath_devdata *dd) 225{ 226 struct ipath_portdata *pd; 227 int ret = 0; 228 u64 val; 229 230 spin_lock_init(&dd->ipath_kernel_tid_lock); 231 spin_lock_init(&dd->ipath_user_tid_lock); 232 spin_lock_init(&dd->ipath_sendctrl_lock); 233 spin_lock_init(&dd->ipath_uctxt_lock); 234 spin_lock_init(&dd->ipath_sdma_lock); 235 spin_lock_init(&dd->ipath_gpio_lock); 236 spin_lock_init(&dd->ipath_eep_st_lock); 237 spin_lock_init(&dd->ipath_sdepb_lock); 238 mutex_init(&dd->ipath_eep_lock); 239 240 /* 241 * skip cfgports stuff because we are not allocating memory, 242 * and we don't want problems if the portcnt changed due to 243 * cfgports. We do still check and report a difference, if 244 * not same (should be impossible). 245 */ 246 dd->ipath_f_config_ports(dd, ipath_cfgports); 247 if (!ipath_cfgports) 248 dd->ipath_cfgports = dd->ipath_portcnt; 249 else if (ipath_cfgports <= dd->ipath_portcnt) { 250 dd->ipath_cfgports = ipath_cfgports; 251 ipath_dbg("Configured to use %u ports out of %u in chip\n", 252 dd->ipath_cfgports, ipath_read_kreg32(dd, 253 dd->ipath_kregs->kr_portcnt)); 254 } else { 255 dd->ipath_cfgports = dd->ipath_portcnt; 256 ipath_dbg("Tried to configured to use %u ports; chip " 257 "only supports %u\n", ipath_cfgports, 258 ipath_read_kreg32(dd, 259 dd->ipath_kregs->kr_portcnt)); 260 } 261 /* 262 * Allocate full portcnt array, rather than just cfgports, because 263 * cleanup iterates across all possible ports. 264 */ 265 dd->ipath_pd = kzalloc(sizeof(*dd->ipath_pd) * dd->ipath_portcnt, 266 GFP_KERNEL); 267 268 if (!dd->ipath_pd) { 269 ipath_dev_err(dd, "Unable to allocate portdata array, " 270 "failing\n"); 271 ret = -ENOMEM; 272 goto done; 273 } 274 275 pd = create_portdata0(dd); 276 if (!pd) { 277 ipath_dev_err(dd, "Unable to allocate portdata for port " 278 "0, failing\n"); 279 ret = -ENOMEM; 280 goto done; 281 } 282 dd->ipath_pd[0] = pd; 283 284 dd->ipath_rcvtidcnt = 285 ipath_read_kreg32(dd, dd->ipath_kregs->kr_rcvtidcnt); 286 dd->ipath_rcvtidbase = 287 ipath_read_kreg32(dd, dd->ipath_kregs->kr_rcvtidbase); 288 dd->ipath_rcvegrcnt = 289 ipath_read_kreg32(dd, dd->ipath_kregs->kr_rcvegrcnt); 290 dd->ipath_rcvegrbase = 291 ipath_read_kreg32(dd, dd->ipath_kregs->kr_rcvegrbase); 292 dd->ipath_palign = 293 ipath_read_kreg32(dd, dd->ipath_kregs->kr_pagealign); 294 dd->ipath_piobufbase = 295 ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendpiobufbase); 296 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendpiosize); 297 dd->ipath_piosize2k = val & ~0U; 298 dd->ipath_piosize4k = val >> 32; 299 if (dd->ipath_piosize4k == 0 && ipath_mtu4096) 300 ipath_mtu4096 = 0; /* 4KB not supported by this chip */ 301 dd->ipath_ibmtu = ipath_mtu4096 ? 4096 : 2048; 302 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendpiobufcnt); 303 dd->ipath_piobcnt2k = val & ~0U; 304 dd->ipath_piobcnt4k = val >> 32; 305 dd->ipath_pio2kbase = 306 (u32 __iomem *) (((char __iomem *) dd->ipath_kregbase) + 307 (dd->ipath_piobufbase & 0xffffffff)); 308 if (dd->ipath_piobcnt4k) { 309 dd->ipath_pio4kbase = (u32 __iomem *) 310 (((char __iomem *) dd->ipath_kregbase) + 311 (dd->ipath_piobufbase >> 32)); 312 /* 313 * 4K buffers take 2 pages; we use roundup just to be 314 * paranoid; we calculate it once here, rather than on 315 * ever buf allocate 316 */ 317 dd->ipath_4kalign = ALIGN(dd->ipath_piosize4k, 318 dd->ipath_palign); 319 ipath_dbg("%u 2k(%x) piobufs @ %p, %u 4k(%x) @ %p " 320 "(%x aligned)\n", 321 dd->ipath_piobcnt2k, dd->ipath_piosize2k, 322 dd->ipath_pio2kbase, dd->ipath_piobcnt4k, 323 dd->ipath_piosize4k, dd->ipath_pio4kbase, 324 dd->ipath_4kalign); 325 } 326 else ipath_dbg("%u 2k piobufs @ %p\n", 327 dd->ipath_piobcnt2k, dd->ipath_pio2kbase); 328 329done: 330 return ret; 331} 332 333/** 334 * init_chip_reset - re-initialize after a reset, or enable 335 * @dd: the infinipath device 336 * 337 * sanity check at least some of the values after reset, and 338 * ensure no receive or transmit (explictly, in case reset 339 * failed 340 */ 341static int init_chip_reset(struct ipath_devdata *dd) 342{ 343 u32 rtmp; 344 int i; 345 unsigned long flags; 346 347 /* 348 * ensure chip does no sends or receives, tail updates, or 349 * pioavail updates while we re-initialize 350 */ 351 dd->ipath_rcvctrl &= ~(1ULL << dd->ipath_r_tailupd_shift); 352 for (i = 0; i < dd->ipath_portcnt; i++) { 353 clear_bit(dd->ipath_r_portenable_shift + i, 354 &dd->ipath_rcvctrl); 355 clear_bit(dd->ipath_r_intravail_shift + i, 356 &dd->ipath_rcvctrl); 357 } 358 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl, 359 dd->ipath_rcvctrl); 360 361 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); 362 dd->ipath_sendctrl = 0U; /* no sdma, etc */ 363 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl); 364 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 365 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); 366 367 ipath_write_kreg(dd, dd->ipath_kregs->kr_control, 0ULL); 368 369 rtmp = ipath_read_kreg32(dd, dd->ipath_kregs->kr_rcvtidcnt); 370 if (rtmp != dd->ipath_rcvtidcnt) 371 dev_info(&dd->pcidev->dev, "tidcnt was %u before " 372 "reset, now %u, using original\n", 373 dd->ipath_rcvtidcnt, rtmp); 374 rtmp = ipath_read_kreg32(dd, dd->ipath_kregs->kr_rcvtidbase); 375 if (rtmp != dd->ipath_rcvtidbase) 376 dev_info(&dd->pcidev->dev, "tidbase was %u before " 377 "reset, now %u, using original\n", 378 dd->ipath_rcvtidbase, rtmp); 379 rtmp = ipath_read_kreg32(dd, dd->ipath_kregs->kr_rcvegrcnt); 380 if (rtmp != dd->ipath_rcvegrcnt) 381 dev_info(&dd->pcidev->dev, "egrcnt was %u before " 382 "reset, now %u, using original\n", 383 dd->ipath_rcvegrcnt, rtmp); 384 rtmp = ipath_read_kreg32(dd, dd->ipath_kregs->kr_rcvegrbase); 385 if (rtmp != dd->ipath_rcvegrbase) 386 dev_info(&dd->pcidev->dev, "egrbase was %u before " 387 "reset, now %u, using original\n", 388 dd->ipath_rcvegrbase, rtmp); 389 390 return 0; 391} 392 393static int init_pioavailregs(struct ipath_devdata *dd) 394{ 395 int ret; 396 397 dd->ipath_pioavailregs_dma = dma_alloc_coherent( 398 &dd->pcidev->dev, PAGE_SIZE, &dd->ipath_pioavailregs_phys, 399 GFP_KERNEL); 400 if (!dd->ipath_pioavailregs_dma) { 401 ipath_dev_err(dd, "failed to allocate PIOavail reg area " 402 "in memory\n"); 403 ret = -ENOMEM; 404 goto done; 405 } 406 407 /* 408 * we really want L2 cache aligned, but for current CPUs of 409 * interest, they are the same. 410 */ 411 dd->ipath_statusp = (u64 *) 412 ((char *)dd->ipath_pioavailregs_dma + 413 ((2 * L1_CACHE_BYTES + 414 dd->ipath_pioavregs * sizeof(u64)) & ~L1_CACHE_BYTES)); 415 /* copy the current value now that it's really allocated */ 416 *dd->ipath_statusp = dd->_ipath_status; 417 /* 418 * setup buffer to hold freeze msg, accessible to apps, 419 * following statusp 420 */ 421 dd->ipath_freezemsg = (char *)&dd->ipath_statusp[1]; 422 /* and its length */ 423 dd->ipath_freezelen = L1_CACHE_BYTES - sizeof(dd->ipath_statusp[0]); 424 425 ret = 0; 426 427done: 428 return ret; 429} 430 431/** 432 * init_shadow_tids - allocate the shadow TID array 433 * @dd: the infinipath device 434 * 435 * allocate the shadow TID array, so we can ipath_munlock previous 436 * entries. It may make more sense to move the pageshadow to the 437 * port data structure, so we only allocate memory for ports actually 438 * in use, since we at 8k per port, now. 439 */ 440static void init_shadow_tids(struct ipath_devdata *dd) 441{ 442 struct page **pages; 443 dma_addr_t *addrs; 444 445 pages = vmalloc(dd->ipath_cfgports * dd->ipath_rcvtidcnt * 446 sizeof(struct page *)); 447 if (!pages) { 448 ipath_dev_err(dd, "failed to allocate shadow page * " 449 "array, no expected sends!\n"); 450 dd->ipath_pageshadow = NULL; 451 return; 452 } 453 454 addrs = vmalloc(dd->ipath_cfgports * dd->ipath_rcvtidcnt * 455 sizeof(dma_addr_t)); 456 if (!addrs) { 457 ipath_dev_err(dd, "failed to allocate shadow dma handle " 458 "array, no expected sends!\n"); 459 vfree(pages); 460 dd->ipath_pageshadow = NULL; 461 return; 462 } 463 464 memset(pages, 0, dd->ipath_cfgports * dd->ipath_rcvtidcnt * 465 sizeof(struct page *)); 466 467 dd->ipath_pageshadow = pages; 468 dd->ipath_physshadow = addrs; 469} 470 471static void enable_chip(struct ipath_devdata *dd, int reinit) 472{ 473 u32 val; 474 u64 rcvmask; 475 unsigned long flags; 476 int i; 477 478 if (!reinit) 479 init_waitqueue_head(&ipath_state_wait); 480 481 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl, 482 dd->ipath_rcvctrl); 483 484 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); 485 /* Enable PIO send, and update of PIOavail regs to memory. */ 486 dd->ipath_sendctrl = INFINIPATH_S_PIOENABLE | 487 INFINIPATH_S_PIOBUFAVAILUPD; 488 489 /* 490 * Set the PIO avail update threshold to host memory 491 * on chips that support it. 492 */ 493 if (dd->ipath_pioupd_thresh) 494 dd->ipath_sendctrl |= dd->ipath_pioupd_thresh 495 << INFINIPATH_S_UPDTHRESH_SHIFT; 496 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl); 497 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 498 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); 499 500 /* 501 * Enable kernel ports' receive and receive interrupt. 502 * Other ports done as user opens and inits them. 503 */ 504 rcvmask = 1ULL; 505 dd->ipath_rcvctrl |= (rcvmask << dd->ipath_r_portenable_shift) | 506 (rcvmask << dd->ipath_r_intravail_shift); 507 if (!(dd->ipath_flags & IPATH_NODMA_RTAIL)) 508 dd->ipath_rcvctrl |= (1ULL << dd->ipath_r_tailupd_shift); 509 510 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl, 511 dd->ipath_rcvctrl); 512 513 /* 514 * now ready for use. this should be cleared whenever we 515 * detect a reset, or initiate one. 516 */ 517 dd->ipath_flags |= IPATH_INITTED; 518 519 /* 520 * Init our shadow copies of head from tail values, 521 * and write head values to match. 522 */ 523 val = ipath_read_ureg32(dd, ur_rcvegrindextail, 0); 524 ipath_write_ureg(dd, ur_rcvegrindexhead, val, 0); 525 526 /* Initialize so we interrupt on next packet received */ 527 ipath_write_ureg(dd, ur_rcvhdrhead, 528 dd->ipath_rhdrhead_intr_off | 529 dd->ipath_pd[0]->port_head, 0); 530 531 /* 532 * by now pioavail updates to memory should have occurred, so 533 * copy them into our working/shadow registers; this is in 534 * case something went wrong with abort, but mostly to get the 535 * initial values of the generation bit correct. 536 */ 537 for (i = 0; i < dd->ipath_pioavregs; i++) { 538 __le64 pioavail; 539 540 /* 541 * Chip Errata bug 6641; even and odd qwords>3 are swapped. 542 */ 543 if (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS)) 544 pioavail = dd->ipath_pioavailregs_dma[i ^ 1]; 545 else 546 pioavail = dd->ipath_pioavailregs_dma[i]; 547 /* 548 * don't need to worry about ipath_pioavailkernel here 549 * because we will call ipath_chg_pioavailkernel() later 550 * in initialization, to busy out buffers as needed 551 */ 552 dd->ipath_pioavailshadow[i] = le64_to_cpu(pioavail); 553 } 554 /* can get counters, stats, etc. */ 555 dd->ipath_flags |= IPATH_PRESENT; 556} 557 558static int init_housekeeping(struct ipath_devdata *dd, int reinit) 559{ 560 char boardn[40]; 561 int ret = 0; 562 563 /* 564 * have to clear shadow copies of registers at init that are 565 * not otherwise set here, or all kinds of bizarre things 566 * happen with driver on chip reset 567 */ 568 dd->ipath_rcvhdrsize = 0; 569 570 /* 571 * Don't clear ipath_flags as 8bit mode was set before 572 * entering this func. However, we do set the linkstate to 573 * unknown, so we can watch for a transition. 574 * PRESENT is set because we want register reads to work, 575 * and the kernel infrastructure saw it in config space; 576 * We clear it if we have failures. 577 */ 578 dd->ipath_flags |= IPATH_LINKUNK | IPATH_PRESENT; 579 dd->ipath_flags &= ~(IPATH_LINKACTIVE | IPATH_LINKARMED | 580 IPATH_LINKDOWN | IPATH_LINKINIT); 581 582 ipath_cdbg(VERBOSE, "Try to read spc chip revision\n"); 583 dd->ipath_revision = 584 ipath_read_kreg64(dd, dd->ipath_kregs->kr_revision); 585 586 /* 587 * set up fundamental info we need to use the chip; we assume 588 * if the revision reg and these regs are OK, we don't need to 589 * special case the rest 590 */ 591 dd->ipath_sregbase = 592 ipath_read_kreg32(dd, dd->ipath_kregs->kr_sendregbase); 593 dd->ipath_cregbase = 594 ipath_read_kreg32(dd, dd->ipath_kregs->kr_counterregbase); 595 dd->ipath_uregbase = 596 ipath_read_kreg32(dd, dd->ipath_kregs->kr_userregbase); 597 ipath_cdbg(VERBOSE, "ipath_kregbase %p, sendbase %x usrbase %x, " 598 "cntrbase %x\n", dd->ipath_kregbase, dd->ipath_sregbase, 599 dd->ipath_uregbase, dd->ipath_cregbase); 600 if ((dd->ipath_revision & 0xffffffff) == 0xffffffff 601 || (dd->ipath_sregbase & 0xffffffff) == 0xffffffff 602 || (dd->ipath_cregbase & 0xffffffff) == 0xffffffff 603 || (dd->ipath_uregbase & 0xffffffff) == 0xffffffff) { 604 ipath_dev_err(dd, "Register read failures from chip, " 605 "giving up initialization\n"); 606 dd->ipath_flags &= ~IPATH_PRESENT; 607 ret = -ENODEV; 608 goto done; 609 } 610 611 612 /* clear diagctrl register, in case diags were running and crashed */ 613 ipath_write_kreg (dd, dd->ipath_kregs->kr_hwdiagctrl, 0); 614 615 /* clear the initial reset flag, in case first driver load */ 616 ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear, 617 INFINIPATH_E_RESET); 618 619 ipath_cdbg(VERBOSE, "Revision %llx (PCI %x)\n", 620 (unsigned long long) dd->ipath_revision, 621 dd->ipath_pcirev); 622 623 if (((dd->ipath_revision >> INFINIPATH_R_SOFTWARE_SHIFT) & 624 INFINIPATH_R_SOFTWARE_MASK) != IPATH_CHIP_SWVERSION) { 625 ipath_dev_err(dd, "Driver only handles version %d, " 626 "chip swversion is %d (%llx), failng\n", 627 IPATH_CHIP_SWVERSION, 628 (int)(dd->ipath_revision >> 629 INFINIPATH_R_SOFTWARE_SHIFT) & 630 INFINIPATH_R_SOFTWARE_MASK, 631 (unsigned long long) dd->ipath_revision); 632 ret = -ENOSYS; 633 goto done; 634 } 635 dd->ipath_majrev = (u8) ((dd->ipath_revision >> 636 INFINIPATH_R_CHIPREVMAJOR_SHIFT) & 637 INFINIPATH_R_CHIPREVMAJOR_MASK); 638 dd->ipath_minrev = (u8) ((dd->ipath_revision >> 639 INFINIPATH_R_CHIPREVMINOR_SHIFT) & 640 INFINIPATH_R_CHIPREVMINOR_MASK); 641 dd->ipath_boardrev = (u8) ((dd->ipath_revision >> 642 INFINIPATH_R_BOARDID_SHIFT) & 643 INFINIPATH_R_BOARDID_MASK); 644 645 ret = dd->ipath_f_get_boardname(dd, boardn, sizeof boardn); 646 647 snprintf(dd->ipath_boardversion, sizeof(dd->ipath_boardversion), 648 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, PCI %u, " 649 "SW Compat %u\n", 650 IPATH_CHIP_VERS_MAJ, IPATH_CHIP_VERS_MIN, boardn, 651 (unsigned)(dd->ipath_revision >> INFINIPATH_R_ARCH_SHIFT) & 652 INFINIPATH_R_ARCH_MASK, 653 dd->ipath_majrev, dd->ipath_minrev, dd->ipath_pcirev, 654 (unsigned)(dd->ipath_revision >> 655 INFINIPATH_R_SOFTWARE_SHIFT) & 656 INFINIPATH_R_SOFTWARE_MASK); 657 658 ipath_dbg("%s", dd->ipath_boardversion); 659 660 if (ret) 661 goto done; 662 663 if (reinit) 664 ret = init_chip_reset(dd); 665 else 666 ret = init_chip_first(dd); 667 668done: 669 return ret; 670} 671 672static void verify_interrupt(unsigned long opaque) 673{ 674 struct ipath_devdata *dd = (struct ipath_devdata *) opaque; 675 676 if (!dd) 677 return; /* being torn down */ 678 679 /* 680 * If we don't have any interrupts, let the user know and 681 * don't bother checking again. 682 */ 683 if (dd->ipath_int_counter == 0) { 684 if (!dd->ipath_f_intr_fallback(dd)) 685 dev_err(&dd->pcidev->dev, "No interrupts detected, " 686 "not usable.\n"); 687 else /* re-arm the timer to see if fallback works */ 688 mod_timer(&dd->ipath_intrchk_timer, jiffies + HZ/2); 689 } else 690 ipath_cdbg(VERBOSE, "%u interrupts at timer check\n", 691 dd->ipath_int_counter); 692} 693 694/** 695 * ipath_init_chip - do the actual initialization sequence on the chip 696 * @dd: the infinipath device 697 * @reinit: reinitializing, so don't allocate new memory 698 * 699 * Do the actual initialization sequence on the chip. This is done 700 * both from the init routine called from the PCI infrastructure, and 701 * when we reset the chip, or detect that it was reset internally, 702 * or it's administratively re-enabled. 703 * 704 * Memory allocation here and in called routines is only done in 705 * the first case (reinit == 0). We have to be careful, because even 706 * without memory allocation, we need to re-write all the chip registers 707 * TIDs, etc. after the reset or enable has completed. 708 */ 709int ipath_init_chip(struct ipath_devdata *dd, int reinit) 710{ 711 int ret = 0; 712 u32 kpiobufs, defkbufs; 713 u32 piobufs, uports; 714 u64 val; 715 struct ipath_portdata *pd; 716 gfp_t gfp_flags = GFP_USER | __GFP_COMP; 717 718 ret = init_housekeeping(dd, reinit); 719 if (ret) 720 goto done; 721 722 /* 723 * we ignore most issues after reporting them, but have to specially 724 * handle hardware-disabled chips. 725 */ 726 if (ret == 2) { 727 /* unique error, known to ipath_init_one */ 728 ret = -EPERM; 729 goto done; 730 } 731 732 /* 733 * We could bump this to allow for full rcvegrcnt + rcvtidcnt, 734 * but then it no longer nicely fits power of two, and since 735 * we now use routines that backend onto __get_free_pages, the 736 * rest would be wasted. 737 */ 738 dd->ipath_rcvhdrcnt = max(dd->ipath_p0_rcvegrcnt, dd->ipath_rcvegrcnt); 739 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvhdrcnt, 740 dd->ipath_rcvhdrcnt); 741 742 /* 743 * Set up the shadow copies of the piobufavail registers, 744 * which we compare against the chip registers for now, and 745 * the in memory DMA'ed copies of the registers. This has to 746 * be done early, before we calculate lastport, etc. 747 */ 748 piobufs = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k; 749 /* 750 * calc number of pioavail registers, and save it; we have 2 751 * bits per buffer. 752 */ 753 dd->ipath_pioavregs = ALIGN(piobufs, sizeof(u64) * BITS_PER_BYTE / 2) 754 / (sizeof(u64) * BITS_PER_BYTE / 2); 755 uports = dd->ipath_cfgports ? dd->ipath_cfgports - 1 : 0; 756 if (piobufs > 144) 757 defkbufs = 32 + dd->ipath_pioreserved; 758 else 759 defkbufs = 16 + dd->ipath_pioreserved; 760 761 if (ipath_kpiobufs && (ipath_kpiobufs + 762 (uports * IPATH_MIN_USER_PORT_BUFCNT)) > piobufs) { 763 int i = (int) piobufs - 764 (int) (uports * IPATH_MIN_USER_PORT_BUFCNT); 765 if (i < 1) 766 i = 1; 767 dev_info(&dd->pcidev->dev, "Allocating %d PIO bufs of " 768 "%d for kernel leaves too few for %d user ports " 769 "(%d each); using %u\n", ipath_kpiobufs, 770 piobufs, uports, IPATH_MIN_USER_PORT_BUFCNT, i); 771 /* 772 * shouldn't change ipath_kpiobufs, because could be 773 * different for different devices... 774 */ 775 kpiobufs = i; 776 } else if (ipath_kpiobufs) 777 kpiobufs = ipath_kpiobufs; 778 else 779 kpiobufs = defkbufs; 780 dd->ipath_lastport_piobuf = piobufs - kpiobufs; 781 dd->ipath_pbufsport = 782 uports ? dd->ipath_lastport_piobuf / uports : 0; 783 /* if not an even divisor, some user ports get extra buffers */ 784 dd->ipath_ports_extrabuf = dd->ipath_lastport_piobuf - 785 (dd->ipath_pbufsport * uports); 786 if (dd->ipath_ports_extrabuf) 787 ipath_dbg("%u pbufs/port leaves some unused, add 1 buffer to " 788 "ports <= %u\n", dd->ipath_pbufsport, 789 dd->ipath_ports_extrabuf); 790 dd->ipath_lastpioindex = 0; 791 dd->ipath_lastpioindexl = dd->ipath_piobcnt2k; 792 /* ipath_pioavailshadow initialized earlier */ 793 ipath_cdbg(VERBOSE, "%d PIO bufs for kernel out of %d total %u " 794 "each for %u user ports\n", kpiobufs, 795 piobufs, dd->ipath_pbufsport, uports); 796 ret = dd->ipath_f_early_init(dd); 797 if (ret) { 798 ipath_dev_err(dd, "Early initialization failure\n"); 799 goto done; 800 } 801 802 /* 803 * Early_init sets rcvhdrentsize and rcvhdrsize, so this must be 804 * done after early_init. 805 */ 806 dd->ipath_hdrqlast = 807 dd->ipath_rcvhdrentsize * (dd->ipath_rcvhdrcnt - 1); 808 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvhdrentsize, 809 dd->ipath_rcvhdrentsize); 810 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvhdrsize, 811 dd->ipath_rcvhdrsize); 812 813 if (!reinit) { 814 ret = init_pioavailregs(dd); 815 init_shadow_tids(dd); 816 if (ret) 817 goto done; 818 } 819 820 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendpioavailaddr, 821 dd->ipath_pioavailregs_phys); 822 823 /* 824 * this is to detect s/w errors, which the h/w works around by 825 * ignoring the low 6 bits of address, if it wasn't aligned. 826 */ 827 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendpioavailaddr); 828 if (val != dd->ipath_pioavailregs_phys) { 829 ipath_dev_err(dd, "Catastrophic software error, " 830 "SendPIOAvailAddr written as %lx, " 831 "read back as %llx\n", 832 (unsigned long) dd->ipath_pioavailregs_phys, 833 (unsigned long long) val); 834 ret = -EINVAL; 835 goto done; 836 } 837 838 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvbthqp, IPATH_KD_QP); 839 840 /* 841 * make sure we are not in freeze, and PIO send enabled, so 842 * writes to pbc happen 843 */ 844 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrmask, 0ULL); 845 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear, 846 ~0ULL&~INFINIPATH_HWE_MEMBISTFAILED); 847 ipath_write_kreg(dd, dd->ipath_kregs->kr_control, 0ULL); 848 849 /* 850 * before error clears, since we expect serdes pll errors during 851 * this, the first time after reset 852 */ 853 if (bringup_link(dd)) { 854 dev_info(&dd->pcidev->dev, "Failed to bringup IB link\n"); 855 ret = -ENETDOWN; 856 goto done; 857 } 858 859 /* 860 * clear any "expected" hwerrs from reset and/or initialization 861 * clear any that aren't enabled (at least this once), and then 862 * set the enable mask 863 */ 864 dd->ipath_f_init_hwerrors(dd); 865 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear, 866 ~0ULL&~INFINIPATH_HWE_MEMBISTFAILED); 867 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrmask, 868 dd->ipath_hwerrmask); 869 870 /* clear all */ 871 ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear, -1LL); 872 /* enable errors that are masked, at least this first time. */ 873 ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask, 874 ~dd->ipath_maskederrs); 875 dd->ipath_maskederrs = 0; /* don't re-enable ignored in timer */ 876 dd->ipath_errormask = 877 ipath_read_kreg64(dd, dd->ipath_kregs->kr_errormask); 878 /* clear any interrupts up to this point (ints still not enabled) */ 879 ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, -1LL); 880 881 dd->ipath_f_tidtemplate(dd); 882 883 /* 884 * Set up the port 0 (kernel) rcvhdr q and egr TIDs. If doing 885 * re-init, the simplest way to handle this is to free 886 * existing, and re-allocate. 887 * Need to re-create rest of port 0 portdata as well. 888 */ 889 pd = dd->ipath_pd[0]; 890 if (reinit) { 891 struct ipath_portdata *npd; 892 893 /* 894 * Alloc and init new ipath_portdata for port0, 895 * Then free old pd. Could lead to fragmentation, but also 896 * makes later support for hot-swap easier. 897 */ 898 npd = create_portdata0(dd); 899 if (npd) { 900 ipath_free_pddata(dd, pd); 901 dd->ipath_pd[0] = npd; 902 pd = npd; 903 } else { 904 ipath_dev_err(dd, "Unable to allocate portdata" 905 " for port 0, failing\n"); 906 ret = -ENOMEM; 907 goto done; 908 } 909 } 910 ret = ipath_create_rcvhdrq(dd, pd); 911 if (!ret) 912 ret = create_port0_egr(dd); 913 if (ret) { 914 ipath_dev_err(dd, "failed to allocate kernel port's " 915 "rcvhdrq and/or egr bufs\n"); 916 goto done; 917 } 918 else 919 enable_chip(dd, reinit); 920 921 /* after enable_chip, so pioavailshadow setup */ 922 ipath_chg_pioavailkernel(dd, 0, piobufs, 1); 923 924 /* 925 * Cancel any possible active sends from early driver load. 926 * Follows early_init because some chips have to initialize 927 * PIO buffers in early_init to avoid false parity errors. 928 * After enable and ipath_chg_pioavailkernel so we can safely 929 * enable pioavail updates and PIOENABLE; packets are now 930 * ready to go out. 931 */ 932 ipath_cancel_sends(dd, 1); 933 934 if (!reinit) { 935 /* 936 * Used when we close a port, for DMA already in flight 937 * at close. 938 */ 939 dd->ipath_dummy_hdrq = dma_alloc_coherent( 940 &dd->pcidev->dev, dd->ipath_pd[0]->port_rcvhdrq_size, 941 &dd->ipath_dummy_hdrq_phys, 942 gfp_flags); 943 if (!dd->ipath_dummy_hdrq) { 944 dev_info(&dd->pcidev->dev, 945 "Couldn't allocate 0x%lx bytes for dummy hdrq\n", 946 dd->ipath_pd[0]->port_rcvhdrq_size); 947 /* fallback to just 0'ing */ 948 dd->ipath_dummy_hdrq_phys = 0UL; 949 } 950 } 951 952 /* 953 * cause retrigger of pending interrupts ignored during init, 954 * even if we had errors 955 */ 956 ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, 0ULL); 957 958 if (!dd->ipath_stats_timer_active) { 959 /* 960 * first init, or after an admin disable/enable 961 * set up stats retrieval timer, even if we had errors 962 * in last portion of setup 963 */ 964 init_timer(&dd->ipath_stats_timer); 965 dd->ipath_stats_timer.function = ipath_get_faststats; 966 dd->ipath_stats_timer.data = (unsigned long) dd; 967 /* every 5 seconds; */ 968 dd->ipath_stats_timer.expires = jiffies + 5 * HZ; 969 /* takes ~16 seconds to overflow at full IB 4x bandwdith */ 970 add_timer(&dd->ipath_stats_timer); 971 dd->ipath_stats_timer_active = 1; 972 } 973 974 /* Set up SendDMA if chip supports it */ 975 if (dd->ipath_flags & IPATH_HAS_SEND_DMA) 976 ret = setup_sdma(dd); 977 978 /* Set up HoL state */ 979 init_timer(&dd->ipath_hol_timer); 980 dd->ipath_hol_timer.function = ipath_hol_event; 981 dd->ipath_hol_timer.data = (unsigned long)dd; 982 dd->ipath_hol_state = IPATH_HOL_UP; 983 984done: 985 if (!ret) { 986 *dd->ipath_statusp |= IPATH_STATUS_CHIP_PRESENT; 987 if (!dd->ipath_f_intrsetup(dd)) { 988 /* now we can enable all interrupts from the chip */ 989 ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 990 -1LL); 991 /* force re-interrupt of any pending interrupts. */ 992 ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, 993 0ULL); 994 /* chip is usable; mark it as initialized */ 995 *dd->ipath_statusp |= IPATH_STATUS_INITTED; 996 997 /* 998 * setup to verify we get an interrupt, and fallback 999 * to an alternate if necessary and possible 1000 */ 1001 if (!reinit) { 1002 init_timer(&dd->ipath_intrchk_timer); 1003 dd->ipath_intrchk_timer.function = 1004 verify_interrupt; 1005 dd->ipath_intrchk_timer.data = 1006 (unsigned long) dd; 1007 } 1008 dd->ipath_intrchk_timer.expires = jiffies + HZ/2; 1009 add_timer(&dd->ipath_intrchk_timer); 1010 } else 1011 ipath_dev_err(dd, "No interrupts enabled, couldn't " 1012 "setup interrupt address\n"); 1013 1014 if (dd->ipath_cfgports > ipath_stats.sps_nports) 1015 /* 1016 * sps_nports is a global, so, we set it to 1017 * the highest number of ports of any of the 1018 * chips we find; we never decrement it, at 1019 * least for now. Since this might have changed 1020 * over disable/enable or prior to reset, always 1021 * do the check and potentially adjust. 1022 */ 1023 ipath_stats.sps_nports = dd->ipath_cfgports; 1024 } else 1025 ipath_dbg("Failed (%d) to initialize chip\n", ret); 1026 1027 /* if ret is non-zero, we probably should do some cleanup 1028 here... */ 1029 return ret; 1030} 1031 1032static int ipath_set_kpiobufs(const char *str, struct kernel_param *kp) 1033{ 1034 struct ipath_devdata *dd; 1035 unsigned long flags; 1036 unsigned short val; 1037 int ret; 1038 1039 ret = ipath_parse_ushort(str, &val); 1040 1041 spin_lock_irqsave(&ipath_devs_lock, flags); 1042 1043 if (ret < 0) 1044 goto bail; 1045 1046 if (val == 0) { 1047 ret = -EINVAL; 1048 goto bail; 1049 } 1050 1051 list_for_each_entry(dd, &ipath_dev_list, ipath_list) { 1052 if (dd->ipath_kregbase) 1053 continue; 1054 if (val > (dd->ipath_piobcnt2k + dd->ipath_piobcnt4k - 1055 (dd->ipath_cfgports * 1056 IPATH_MIN_USER_PORT_BUFCNT))) 1057 { 1058 ipath_dev_err( 1059 dd, 1060 "Allocating %d PIO bufs for kernel leaves " 1061 "too few for %d user ports (%d each)\n", 1062 val, dd->ipath_cfgports - 1, 1063 IPATH_MIN_USER_PORT_BUFCNT); 1064 ret = -EINVAL; 1065 goto bail; 1066 } 1067 dd->ipath_lastport_piobuf = 1068 dd->ipath_piobcnt2k + dd->ipath_piobcnt4k - val; 1069 } 1070 1071 ipath_kpiobufs = val; 1072 ret = 0; 1073bail: 1074 spin_unlock_irqrestore(&ipath_devs_lock, flags); 1075 1076 return ret; 1077} 1078