1/* 2 * Copyright (c) 2008, 2009, 2010 QLogic Corporation. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/spinlock.h> 34#include <linux/pci.h> 35#include <linux/io.h> 36#include <linux/delay.h> 37#include <linux/netdevice.h> 38#include <linux/vmalloc.h> 39#include <linux/moduleparam.h> 40 41#include "qib.h" 42 43static unsigned qib_hol_timeout_ms = 3000; 44module_param_named(hol_timeout_ms, qib_hol_timeout_ms, uint, S_IRUGO); 45MODULE_PARM_DESC(hol_timeout_ms, 46 "duration of user app suspension after link failure"); 47 48unsigned qib_sdma_fetch_arb = 1; 49module_param_named(fetch_arb, qib_sdma_fetch_arb, uint, S_IRUGO); 50MODULE_PARM_DESC(fetch_arb, "IBA7220: change SDMA descriptor arbitration"); 51 52/** 53 * qib_disarm_piobufs - cancel a range of PIO buffers 54 * @dd: the qlogic_ib device 55 * @first: the first PIO buffer to cancel 56 * @cnt: the number of PIO buffers to cancel 57 * 58 * Cancel a range of PIO buffers. Used at user process close, 59 * in case it died while writing to a PIO buffer. 60 */ 61void qib_disarm_piobufs(struct qib_devdata *dd, unsigned first, unsigned cnt) 62{ 63 unsigned long flags; 64 unsigned i; 65 unsigned last; 66 67 last = first + cnt; 68 spin_lock_irqsave(&dd->pioavail_lock, flags); 69 for (i = first; i < last; i++) { 70 __clear_bit(i, dd->pio_need_disarm); 71 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(i)); 72 } 73 spin_unlock_irqrestore(&dd->pioavail_lock, flags); 74} 75 76/* 77 * This is called by a user process when it sees the DISARM_BUFS event 78 * bit is set. 79 */ 80int qib_disarm_piobufs_ifneeded(struct qib_ctxtdata *rcd) 81{ 82 struct qib_devdata *dd = rcd->dd; 83 unsigned i; 84 unsigned last; 85 86 last = rcd->pio_base + rcd->piocnt; 87 /* 88 * Don't need uctxt_lock here, since user has called in to us. 89 * Clear at start in case more interrupts set bits while we 90 * are disarming 91 */ 92 if (rcd->user_event_mask) { 93 /* 94 * subctxt_cnt is 0 if not shared, so do base 95 * separately, first, then remaining subctxt, if any 96 */ 97 clear_bit(_QIB_EVENT_DISARM_BUFS_BIT, &rcd->user_event_mask[0]); 98 for (i = 1; i < rcd->subctxt_cnt; i++) 99 clear_bit(_QIB_EVENT_DISARM_BUFS_BIT, 100 &rcd->user_event_mask[i]); 101 } 102 spin_lock_irq(&dd->pioavail_lock); 103 for (i = rcd->pio_base; i < last; i++) { 104 if (__test_and_clear_bit(i, dd->pio_need_disarm)) 105 dd->f_sendctrl(rcd->ppd, QIB_SENDCTRL_DISARM_BUF(i)); 106 } 107 spin_unlock_irq(&dd->pioavail_lock); 108 return 0; 109} 110 111static struct qib_pportdata *is_sdma_buf(struct qib_devdata *dd, unsigned i) 112{ 113 struct qib_pportdata *ppd; 114 unsigned pidx; 115 116 for (pidx = 0; pidx < dd->num_pports; pidx++) { 117 ppd = dd->pport + pidx; 118 if (i >= ppd->sdma_state.first_sendbuf && 119 i < ppd->sdma_state.last_sendbuf) 120 return ppd; 121 } 122 return NULL; 123} 124 125/* 126 * Return true if send buffer is being used by a user context. 127 * Sets _QIB_EVENT_DISARM_BUFS_BIT in user_event_mask as a side effect 128 */ 129static int find_ctxt(struct qib_devdata *dd, unsigned bufn) 130{ 131 struct qib_ctxtdata *rcd; 132 unsigned ctxt; 133 int ret = 0; 134 135 spin_lock(&dd->uctxt_lock); 136 for (ctxt = dd->first_user_ctxt; ctxt < dd->cfgctxts; ctxt++) { 137 rcd = dd->rcd[ctxt]; 138 if (!rcd || bufn < rcd->pio_base || 139 bufn >= rcd->pio_base + rcd->piocnt) 140 continue; 141 if (rcd->user_event_mask) { 142 int i; 143 /* 144 * subctxt_cnt is 0 if not shared, so do base 145 * separately, first, then remaining subctxt, if any 146 */ 147 set_bit(_QIB_EVENT_DISARM_BUFS_BIT, 148 &rcd->user_event_mask[0]); 149 for (i = 1; i < rcd->subctxt_cnt; i++) 150 set_bit(_QIB_EVENT_DISARM_BUFS_BIT, 151 &rcd->user_event_mask[i]); 152 } 153 ret = 1; 154 break; 155 } 156 spin_unlock(&dd->uctxt_lock); 157 158 return ret; 159} 160 161/* 162 * Disarm a set of send buffers. If the buffer might be actively being 163 * written to, mark the buffer to be disarmed later when it is not being 164 * written to. 165 * 166 * This should only be called from the IRQ error handler. 167 */ 168void qib_disarm_piobufs_set(struct qib_devdata *dd, unsigned long *mask, 169 unsigned cnt) 170{ 171 struct qib_pportdata *ppd, *pppd[QIB_MAX_IB_PORTS]; 172 unsigned i; 173 unsigned long flags; 174 175 for (i = 0; i < dd->num_pports; i++) 176 pppd[i] = NULL; 177 178 for (i = 0; i < cnt; i++) { 179 if (!test_bit(i, mask)) 180 continue; 181 /* 182 * If the buffer is owned by the DMA hardware, 183 * reset the DMA engine. 184 */ 185 ppd = is_sdma_buf(dd, i); 186 if (ppd) { 187 pppd[ppd->port] = ppd; 188 continue; 189 } 190 /* 191 * If the kernel is writing the buffer or the buffer is 192 * owned by a user process, we can't clear it yet. 193 */ 194 spin_lock_irqsave(&dd->pioavail_lock, flags); 195 if (test_bit(i, dd->pio_writing) || 196 (!test_bit(i << 1, dd->pioavailkernel) && 197 find_ctxt(dd, i))) { 198 __set_bit(i, dd->pio_need_disarm); 199 } else { 200 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(i)); 201 } 202 spin_unlock_irqrestore(&dd->pioavail_lock, flags); 203 } 204 205 /* do cancel_sends once per port that had sdma piobufs in error */ 206 for (i = 0; i < dd->num_pports; i++) 207 if (pppd[i]) 208 qib_cancel_sends(pppd[i]); 209} 210 211/** 212 * update_send_bufs - update shadow copy of the PIO availability map 213 * @dd: the qlogic_ib device 214 * 215 * called whenever our local copy indicates we have run out of send buffers 216 */ 217static void update_send_bufs(struct qib_devdata *dd) 218{ 219 unsigned long flags; 220 unsigned i; 221 const unsigned piobregs = dd->pioavregs; 222 223 /* 224 * If the generation (check) bits have changed, then we update the 225 * busy bit for the corresponding PIO buffer. This algorithm will 226 * modify positions to the value they already have in some cases 227 * (i.e., no change), but it's faster than changing only the bits 228 * that have changed. 229 * 230 * We would like to do this atomicly, to avoid spinlocks in the 231 * critical send path, but that's not really possible, given the 232 * type of changes, and that this routine could be called on 233 * multiple cpu's simultaneously, so we lock in this routine only, 234 * to avoid conflicting updates; all we change is the shadow, and 235 * it's a single 64 bit memory location, so by definition the update 236 * is atomic in terms of what other cpu's can see in testing the 237 * bits. The spin_lock overhead isn't too bad, since it only 238 * happens when all buffers are in use, so only cpu overhead, not 239 * latency or bandwidth is affected. 240 */ 241 if (!dd->pioavailregs_dma) 242 return; 243 spin_lock_irqsave(&dd->pioavail_lock, flags); 244 for (i = 0; i < piobregs; i++) { 245 u64 pchbusy, pchg, piov, pnew; 246 247 piov = le64_to_cpu(dd->pioavailregs_dma[i]); 248 pchg = dd->pioavailkernel[i] & 249 ~(dd->pioavailshadow[i] ^ piov); 250 pchbusy = pchg << QLOGIC_IB_SENDPIOAVAIL_BUSY_SHIFT; 251 if (pchg && (pchbusy & dd->pioavailshadow[i])) { 252 pnew = dd->pioavailshadow[i] & ~pchbusy; 253 pnew |= piov & pchbusy; 254 dd->pioavailshadow[i] = pnew; 255 } 256 } 257 spin_unlock_irqrestore(&dd->pioavail_lock, flags); 258} 259 260/* 261 * Debugging code and stats updates if no pio buffers available. 262 */ 263static noinline void no_send_bufs(struct qib_devdata *dd) 264{ 265 dd->upd_pio_shadow = 1; 266 267 /* not atomic, but if we lose a stat count in a while, that's OK */ 268 qib_stats.sps_nopiobufs++; 269} 270 271/* 272 * Common code for normal driver send buffer allocation, and reserved 273 * allocation. 274 * 275 * Do appropriate marking as busy, etc. 276 * Returns buffer pointer if one is found, otherwise NULL. 277 */ 278u32 __iomem *qib_getsendbuf_range(struct qib_devdata *dd, u32 *pbufnum, 279 u32 first, u32 last) 280{ 281 unsigned i, j, updated = 0; 282 unsigned nbufs; 283 unsigned long flags; 284 unsigned long *shadow = dd->pioavailshadow; 285 u32 __iomem *buf; 286 287 if (!(dd->flags & QIB_PRESENT)) 288 return NULL; 289 290 nbufs = last - first + 1; /* number in range to check */ 291 if (dd->upd_pio_shadow) { 292update_shadow: 293 /* 294 * Minor optimization. If we had no buffers on last call, 295 * start out by doing the update; continue and do scan even 296 * if no buffers were updated, to be paranoid. 297 */ 298 update_send_bufs(dd); 299 updated++; 300 } 301 i = first; 302 /* 303 * While test_and_set_bit() is atomic, we do that and then the 304 * change_bit(), and the pair is not. See if this is the cause 305 * of the remaining armlaunch errors. 306 */ 307 spin_lock_irqsave(&dd->pioavail_lock, flags); 308 if (dd->last_pio >= first && dd->last_pio <= last) 309 i = dd->last_pio + 1; 310 if (!first) 311 /* adjust to min possible */ 312 nbufs = last - dd->min_kernel_pio + 1; 313 for (j = 0; j < nbufs; j++, i++) { 314 if (i > last) 315 i = !first ? dd->min_kernel_pio : first; 316 if (__test_and_set_bit((2 * i) + 1, shadow)) 317 continue; 318 /* flip generation bit */ 319 __change_bit(2 * i, shadow); 320 /* remember that the buffer can be written to now */ 321 __set_bit(i, dd->pio_writing); 322 if (!first && first != last) /* first == last on VL15, avoid */ 323 dd->last_pio = i; 324 break; 325 } 326 spin_unlock_irqrestore(&dd->pioavail_lock, flags); 327 328 if (j == nbufs) { 329 if (!updated) 330 /* 331 * First time through; shadow exhausted, but may be 332 * buffers available, try an update and then rescan. 333 */ 334 goto update_shadow; 335 no_send_bufs(dd); 336 buf = NULL; 337 } else { 338 if (i < dd->piobcnt2k) 339 buf = (u32 __iomem *)(dd->pio2kbase + 340 i * dd->palign); 341 else if (i < dd->piobcnt2k + dd->piobcnt4k || !dd->piovl15base) 342 buf = (u32 __iomem *)(dd->pio4kbase + 343 (i - dd->piobcnt2k) * dd->align4k); 344 else 345 buf = (u32 __iomem *)(dd->piovl15base + 346 (i - (dd->piobcnt2k + dd->piobcnt4k)) * 347 dd->align4k); 348 if (pbufnum) 349 *pbufnum = i; 350 dd->upd_pio_shadow = 0; 351 } 352 353 return buf; 354} 355 356/* 357 * Record that the caller is finished writing to the buffer so we don't 358 * disarm it while it is being written and disarm it now if needed. 359 */ 360void qib_sendbuf_done(struct qib_devdata *dd, unsigned n) 361{ 362 unsigned long flags; 363 364 spin_lock_irqsave(&dd->pioavail_lock, flags); 365 __clear_bit(n, dd->pio_writing); 366 if (__test_and_clear_bit(n, dd->pio_need_disarm)) 367 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(n)); 368 spin_unlock_irqrestore(&dd->pioavail_lock, flags); 369} 370 371/** 372 * qib_chg_pioavailkernel - change which send buffers are available for kernel 373 * @dd: the qlogic_ib device 374 * @start: the starting send buffer number 375 * @len: the number of send buffers 376 * @avail: true if the buffers are available for kernel use, false otherwise 377 * @rcd: the context pointer 378 */ 379void qib_chg_pioavailkernel(struct qib_devdata *dd, unsigned start, 380 unsigned len, u32 avail, struct qib_ctxtdata *rcd) 381{ 382 unsigned long flags; 383 unsigned end; 384 unsigned ostart = start; 385 386 /* There are two bits per send buffer (busy and generation) */ 387 start *= 2; 388 end = start + len * 2; 389 390 spin_lock_irqsave(&dd->pioavail_lock, flags); 391 /* Set or clear the busy bit in the shadow. */ 392 while (start < end) { 393 if (avail) { 394 unsigned long dma; 395 int i; 396 397 /* 398 * The BUSY bit will never be set, because we disarm 399 * the user buffers before we hand them back to the 400 * kernel. We do have to make sure the generation 401 * bit is set correctly in shadow, since it could 402 * have changed many times while allocated to user. 403 * We can't use the bitmap functions on the full 404 * dma array because it is always little-endian, so 405 * we have to flip to host-order first. 406 * BITS_PER_LONG is slightly wrong, since it's 407 * always 64 bits per register in chip... 408 * We only work on 64 bit kernels, so that's OK. 409 */ 410 i = start / BITS_PER_LONG; 411 __clear_bit(QLOGIC_IB_SENDPIOAVAIL_BUSY_SHIFT + start, 412 dd->pioavailshadow); 413 dma = (unsigned long) 414 le64_to_cpu(dd->pioavailregs_dma[i]); 415 if (test_bit((QLOGIC_IB_SENDPIOAVAIL_CHECK_SHIFT + 416 start) % BITS_PER_LONG, &dma)) 417 __set_bit(QLOGIC_IB_SENDPIOAVAIL_CHECK_SHIFT + 418 start, dd->pioavailshadow); 419 else 420 __clear_bit(QLOGIC_IB_SENDPIOAVAIL_CHECK_SHIFT 421 + start, dd->pioavailshadow); 422 __set_bit(start, dd->pioavailkernel); 423 if ((start >> 1) < dd->min_kernel_pio) 424 dd->min_kernel_pio = start >> 1; 425 } else { 426 __set_bit(start + QLOGIC_IB_SENDPIOAVAIL_BUSY_SHIFT, 427 dd->pioavailshadow); 428 __clear_bit(start, dd->pioavailkernel); 429 if ((start >> 1) > dd->min_kernel_pio) 430 dd->min_kernel_pio = start >> 1; 431 } 432 start += 2; 433 } 434 435 if (dd->min_kernel_pio > 0 && dd->last_pio < dd->min_kernel_pio - 1) 436 dd->last_pio = dd->min_kernel_pio - 1; 437 spin_unlock_irqrestore(&dd->pioavail_lock, flags); 438 439 dd->f_txchk_change(dd, ostart, len, avail, rcd); 440} 441 442/* 443 * Flush all sends that might be in the ready to send state, as well as any 444 * that are in the process of being sent. Used whenever we need to be 445 * sure the send side is idle. Cleans up all buffer state by canceling 446 * all pio buffers, and issuing an abort, which cleans up anything in the 447 * launch fifo. The cancel is superfluous on some chip versions, but 448 * it's safer to always do it. 449 * PIOAvail bits are updated by the chip as if a normal send had happened. 450 */ 451void qib_cancel_sends(struct qib_pportdata *ppd) 452{ 453 struct qib_devdata *dd = ppd->dd; 454 struct qib_ctxtdata *rcd; 455 unsigned long flags; 456 unsigned ctxt; 457 unsigned i; 458 unsigned last; 459 460 /* 461 * Tell PSM to disarm buffers again before trying to reuse them. 462 * We need to be sure the rcd doesn't change out from under us 463 * while we do so. We hold the two locks sequentially. We might 464 * needlessly set some need_disarm bits as a result, if the 465 * context is closed after we release the uctxt_lock, but that's 466 * fairly benign, and safer than nesting the locks. 467 */ 468 for (ctxt = dd->first_user_ctxt; ctxt < dd->cfgctxts; ctxt++) { 469 spin_lock_irqsave(&dd->uctxt_lock, flags); 470 rcd = dd->rcd[ctxt]; 471 if (rcd && rcd->ppd == ppd) { 472 last = rcd->pio_base + rcd->piocnt; 473 if (rcd->user_event_mask) { 474 /* 475 * subctxt_cnt is 0 if not shared, so do base 476 * separately, first, then remaining subctxt, 477 * if any 478 */ 479 set_bit(_QIB_EVENT_DISARM_BUFS_BIT, 480 &rcd->user_event_mask[0]); 481 for (i = 1; i < rcd->subctxt_cnt; i++) 482 set_bit(_QIB_EVENT_DISARM_BUFS_BIT, 483 &rcd->user_event_mask[i]); 484 } 485 i = rcd->pio_base; 486 spin_unlock_irqrestore(&dd->uctxt_lock, flags); 487 spin_lock_irqsave(&dd->pioavail_lock, flags); 488 for (; i < last; i++) 489 __set_bit(i, dd->pio_need_disarm); 490 spin_unlock_irqrestore(&dd->pioavail_lock, flags); 491 } else 492 spin_unlock_irqrestore(&dd->uctxt_lock, flags); 493 } 494 495 if (!(dd->flags & QIB_HAS_SEND_DMA)) 496 dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_ALL | 497 QIB_SENDCTRL_FLUSH); 498} 499 500/* 501 * Force an update of in-memory copy of the pioavail registers, when 502 * needed for any of a variety of reasons. 503 * If already off, this routine is a nop, on the assumption that the 504 * caller (or set of callers) will "do the right thing". 505 * This is a per-device operation, so just the first port. 506 */ 507void qib_force_pio_avail_update(struct qib_devdata *dd) 508{ 509 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_AVAIL_BLIP); 510} 511 512void qib_hol_down(struct qib_pportdata *ppd) 513{ 514 /* 515 * Cancel sends when the link goes DOWN so that we aren't doing it 516 * at INIT when we might be trying to send SMI packets. 517 */ 518 if (!(ppd->lflags & QIBL_IB_AUTONEG_INPROG)) 519 qib_cancel_sends(ppd); 520} 521 522/* 523 * Link is at INIT. 524 * We start the HoL timer so we can detect stuck packets blocking SMP replies. 525 * Timer may already be running, so use mod_timer, not add_timer. 526 */ 527void qib_hol_init(struct qib_pportdata *ppd) 528{ 529 if (ppd->hol_state != QIB_HOL_INIT) { 530 ppd->hol_state = QIB_HOL_INIT; 531 mod_timer(&ppd->hol_timer, 532 jiffies + msecs_to_jiffies(qib_hol_timeout_ms)); 533 } 534} 535 536/* 537 * Link is up, continue any user processes, and ensure timer 538 * is a nop, if running. Let timer keep running, if set; it 539 * will nop when it sees the link is up. 540 */ 541void qib_hol_up(struct qib_pportdata *ppd) 542{ 543 ppd->hol_state = QIB_HOL_UP; 544} 545 546/* 547 * This is only called via the timer. 548 */ 549void qib_hol_event(struct timer_list *t) 550{ 551 struct qib_pportdata *ppd = from_timer(ppd, t, hol_timer); 552 553 /* If hardware error, etc, skip. */ 554 if (!(ppd->dd->flags & QIB_INITTED)) 555 return; 556 557 if (ppd->hol_state != QIB_HOL_UP) { 558 /* 559 * Try to flush sends in case a stuck packet is blocking 560 * SMP replies. 561 */ 562 qib_hol_down(ppd); 563 mod_timer(&ppd->hol_timer, 564 jiffies + msecs_to_jiffies(qib_hol_timeout_ms)); 565 } 566} 567