1/* 2 * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation. 3 * All rights reserved. 4 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#include <linux/pci.h> 36#include <linux/poll.h> 37#include <linux/cdev.h> 38#include <linux/swap.h> 39#include <linux/vmalloc.h> 40#include <linux/highmem.h> 41#include <linux/io.h> 42#include <linux/uio.h> 43#include <linux/jiffies.h> 44#include <asm/pgtable.h> 45#include <linux/delay.h> 46 47#include "qib.h" 48#include "qib_common.h" 49#include "qib_user_sdma.h" 50 51static int qib_open(struct inode *, struct file *); 52static int qib_close(struct inode *, struct file *); 53static ssize_t qib_write(struct file *, const char __user *, size_t, loff_t *); 54static ssize_t qib_aio_write(struct kiocb *, const struct iovec *, 55 unsigned long, loff_t); 56static unsigned int qib_poll(struct file *, struct poll_table_struct *); 57static int qib_mmapf(struct file *, struct vm_area_struct *); 58 59static const struct file_operations qib_file_ops = { 60 .owner = THIS_MODULE, 61 .write = qib_write, 62 .aio_write = qib_aio_write, 63 .open = qib_open, 64 .release = qib_close, 65 .poll = qib_poll, 66 .mmap = qib_mmapf 67}; 68 69/* 70 * Convert kernel virtual addresses to physical addresses so they don't 71 * potentially conflict with the chip addresses used as mmap offsets. 72 * It doesn't really matter what mmap offset we use as long as we can 73 * interpret it correctly. 74 */ 75static u64 cvt_kvaddr(void *p) 76{ 77 struct page *page; 78 u64 paddr = 0; 79 80 page = vmalloc_to_page(p); 81 if (page) 82 paddr = page_to_pfn(page) << PAGE_SHIFT; 83 84 return paddr; 85} 86 87static int qib_get_base_info(struct file *fp, void __user *ubase, 88 size_t ubase_size) 89{ 90 struct qib_ctxtdata *rcd = ctxt_fp(fp); 91 int ret = 0; 92 struct qib_base_info *kinfo = NULL; 93 struct qib_devdata *dd = rcd->dd; 94 struct qib_pportdata *ppd = rcd->ppd; 95 unsigned subctxt_cnt; 96 int shared, master; 97 size_t sz; 98 99 subctxt_cnt = rcd->subctxt_cnt; 100 if (!subctxt_cnt) { 101 shared = 0; 102 master = 0; 103 subctxt_cnt = 1; 104 } else { 105 shared = 1; 106 master = !subctxt_fp(fp); 107 } 108 109 sz = sizeof(*kinfo); 110 /* If context sharing is not requested, allow the old size structure */ 111 if (!shared) 112 sz -= 7 * sizeof(u64); 113 if (ubase_size < sz) { 114 ret = -EINVAL; 115 goto bail; 116 } 117 118 kinfo = kzalloc(sizeof(*kinfo), GFP_KERNEL); 119 if (kinfo == NULL) { 120 ret = -ENOMEM; 121 goto bail; 122 } 123 124 ret = dd->f_get_base_info(rcd, kinfo); 125 if (ret < 0) 126 goto bail; 127 128 kinfo->spi_rcvhdr_cnt = dd->rcvhdrcnt; 129 kinfo->spi_rcvhdrent_size = dd->rcvhdrentsize; 130 kinfo->spi_tidegrcnt = rcd->rcvegrcnt; 131 kinfo->spi_rcv_egrbufsize = dd->rcvegrbufsize; 132 /* 133 * have to mmap whole thing 134 */ 135 kinfo->spi_rcv_egrbuftotlen = 136 rcd->rcvegrbuf_chunks * rcd->rcvegrbuf_size; 137 kinfo->spi_rcv_egrperchunk = rcd->rcvegrbufs_perchunk; 138 kinfo->spi_rcv_egrchunksize = kinfo->spi_rcv_egrbuftotlen / 139 rcd->rcvegrbuf_chunks; 140 kinfo->spi_tidcnt = dd->rcvtidcnt / subctxt_cnt; 141 if (master) 142 kinfo->spi_tidcnt += dd->rcvtidcnt % subctxt_cnt; 143 /* 144 * for this use, may be cfgctxts summed over all chips that 145 * are are configured and present 146 */ 147 kinfo->spi_nctxts = dd->cfgctxts; 148 /* unit (chip/board) our context is on */ 149 kinfo->spi_unit = dd->unit; 150 kinfo->spi_port = ppd->port; 151 /* for now, only a single page */ 152 kinfo->spi_tid_maxsize = PAGE_SIZE; 153 154 /* 155 * Doing this per context, and based on the skip value, etc. This has 156 * to be the actual buffer size, since the protocol code treats it 157 * as an array. 158 * 159 * These have to be set to user addresses in the user code via mmap. 160 * These values are used on return to user code for the mmap target 161 * addresses only. For 32 bit, same 44 bit address problem, so use 162 * the physical address, not virtual. Before 2.6.11, using the 163 * page_address() macro worked, but in 2.6.11, even that returns the 164 * full 64 bit address (upper bits all 1's). So far, using the 165 * physical addresses (or chip offsets, for chip mapping) works, but 166 * no doubt some future kernel release will change that, and we'll be 167 * on to yet another method of dealing with this. 168 * Normally only one of rcvhdr_tailaddr or rhf_offset is useful 169 * since the chips with non-zero rhf_offset don't normally 170 * enable tail register updates to host memory, but for testing, 171 * both can be enabled and used. 172 */ 173 kinfo->spi_rcvhdr_base = (u64) rcd->rcvhdrq_phys; 174 kinfo->spi_rcvhdr_tailaddr = (u64) rcd->rcvhdrqtailaddr_phys; 175 kinfo->spi_rhf_offset = dd->rhf_offset; 176 kinfo->spi_rcv_egrbufs = (u64) rcd->rcvegr_phys; 177 kinfo->spi_pioavailaddr = (u64) dd->pioavailregs_phys; 178 /* setup per-unit (not port) status area for user programs */ 179 kinfo->spi_status = (u64) kinfo->spi_pioavailaddr + 180 (char *) ppd->statusp - 181 (char *) dd->pioavailregs_dma; 182 kinfo->spi_uregbase = (u64) dd->uregbase + dd->ureg_align * rcd->ctxt; 183 if (!shared) { 184 kinfo->spi_piocnt = rcd->piocnt; 185 kinfo->spi_piobufbase = (u64) rcd->piobufs; 186 kinfo->spi_sendbuf_status = cvt_kvaddr(rcd->user_event_mask); 187 } else if (master) { 188 kinfo->spi_piocnt = (rcd->piocnt / subctxt_cnt) + 189 (rcd->piocnt % subctxt_cnt); 190 /* Master's PIO buffers are after all the slave's */ 191 kinfo->spi_piobufbase = (u64) rcd->piobufs + 192 dd->palign * 193 (rcd->piocnt - kinfo->spi_piocnt); 194 } else { 195 unsigned slave = subctxt_fp(fp) - 1; 196 197 kinfo->spi_piocnt = rcd->piocnt / subctxt_cnt; 198 kinfo->spi_piobufbase = (u64) rcd->piobufs + 199 dd->palign * kinfo->spi_piocnt * slave; 200 } 201 202 if (shared) { 203 kinfo->spi_sendbuf_status = 204 cvt_kvaddr(&rcd->user_event_mask[subctxt_fp(fp)]); 205 /* only spi_subctxt_* fields should be set in this block! */ 206 kinfo->spi_subctxt_uregbase = cvt_kvaddr(rcd->subctxt_uregbase); 207 208 kinfo->spi_subctxt_rcvegrbuf = 209 cvt_kvaddr(rcd->subctxt_rcvegrbuf); 210 kinfo->spi_subctxt_rcvhdr_base = 211 cvt_kvaddr(rcd->subctxt_rcvhdr_base); 212 } 213 214 /* 215 * All user buffers are 2KB buffers. If we ever support 216 * giving 4KB buffers to user processes, this will need some 217 * work. Can't use piobufbase directly, because it has 218 * both 2K and 4K buffer base values. 219 */ 220 kinfo->spi_pioindex = (kinfo->spi_piobufbase - dd->pio2k_bufbase) / 221 dd->palign; 222 kinfo->spi_pioalign = dd->palign; 223 kinfo->spi_qpair = QIB_KD_QP; 224 /* 225 * user mode PIO buffers are always 2KB, even when 4KB can 226 * be received, and sent via the kernel; this is ibmaxlen 227 * for 2K MTU. 228 */ 229 kinfo->spi_piosize = dd->piosize2k - 2 * sizeof(u32); 230 kinfo->spi_mtu = ppd->ibmaxlen; /* maxlen, not ibmtu */ 231 kinfo->spi_ctxt = rcd->ctxt; 232 kinfo->spi_subctxt = subctxt_fp(fp); 233 kinfo->spi_sw_version = QIB_KERN_SWVERSION; 234 kinfo->spi_sw_version |= 1U << 31; /* QLogic-built, not kernel.org */ 235 kinfo->spi_hw_version = dd->revision; 236 237 if (master) 238 kinfo->spi_runtime_flags |= QIB_RUNTIME_MASTER; 239 240 sz = (ubase_size < sizeof(*kinfo)) ? ubase_size : sizeof(*kinfo); 241 if (copy_to_user(ubase, kinfo, sz)) 242 ret = -EFAULT; 243bail: 244 kfree(kinfo); 245 return ret; 246} 247 248/** 249 * qib_tid_update - update a context TID 250 * @rcd: the context 251 * @fp: the qib device file 252 * @ti: the TID information 253 * 254 * The new implementation as of Oct 2004 is that the driver assigns 255 * the tid and returns it to the caller. To reduce search time, we 256 * keep a cursor for each context, walking the shadow tid array to find 257 * one that's not in use. 258 * 259 * For now, if we can't allocate the full list, we fail, although 260 * in the long run, we'll allocate as many as we can, and the 261 * caller will deal with that by trying the remaining pages later. 262 * That means that when we fail, we have to mark the tids as not in 263 * use again, in our shadow copy. 264 * 265 * It's up to the caller to free the tids when they are done. 266 * We'll unlock the pages as they free them. 267 * 268 * Also, right now we are locking one page at a time, but since 269 * the intended use of this routine is for a single group of 270 * virtually contiguous pages, that should change to improve 271 * performance. 272 */ 273static int qib_tid_update(struct qib_ctxtdata *rcd, struct file *fp, 274 const struct qib_tid_info *ti) 275{ 276 int ret = 0, ntids; 277 u32 tid, ctxttid, cnt, i, tidcnt, tidoff; 278 u16 *tidlist; 279 struct qib_devdata *dd = rcd->dd; 280 u64 physaddr; 281 unsigned long vaddr; 282 u64 __iomem *tidbase; 283 unsigned long tidmap[8]; 284 struct page **pagep = NULL; 285 unsigned subctxt = subctxt_fp(fp); 286 287 if (!dd->pageshadow) { 288 ret = -ENOMEM; 289 goto done; 290 } 291 292 cnt = ti->tidcnt; 293 if (!cnt) { 294 ret = -EFAULT; 295 goto done; 296 } 297 ctxttid = rcd->ctxt * dd->rcvtidcnt; 298 if (!rcd->subctxt_cnt) { 299 tidcnt = dd->rcvtidcnt; 300 tid = rcd->tidcursor; 301 tidoff = 0; 302 } else if (!subctxt) { 303 tidcnt = (dd->rcvtidcnt / rcd->subctxt_cnt) + 304 (dd->rcvtidcnt % rcd->subctxt_cnt); 305 tidoff = dd->rcvtidcnt - tidcnt; 306 ctxttid += tidoff; 307 tid = tidcursor_fp(fp); 308 } else { 309 tidcnt = dd->rcvtidcnt / rcd->subctxt_cnt; 310 tidoff = tidcnt * (subctxt - 1); 311 ctxttid += tidoff; 312 tid = tidcursor_fp(fp); 313 } 314 if (cnt > tidcnt) { 315 /* make sure it all fits in tid_pg_list */ 316 qib_devinfo(dd->pcidev, "Process tried to allocate %u " 317 "TIDs, only trying max (%u)\n", cnt, tidcnt); 318 cnt = tidcnt; 319 } 320 pagep = (struct page **) rcd->tid_pg_list; 321 tidlist = (u16 *) &pagep[dd->rcvtidcnt]; 322 pagep += tidoff; 323 tidlist += tidoff; 324 325 memset(tidmap, 0, sizeof(tidmap)); 326 /* before decrement; chip actual # */ 327 ntids = tidcnt; 328 tidbase = (u64 __iomem *) (((char __iomem *) dd->kregbase) + 329 dd->rcvtidbase + 330 ctxttid * sizeof(*tidbase)); 331 332 /* virtual address of first page in transfer */ 333 vaddr = ti->tidvaddr; 334 if (!access_ok(VERIFY_WRITE, (void __user *) vaddr, 335 cnt * PAGE_SIZE)) { 336 ret = -EFAULT; 337 goto done; 338 } 339 ret = qib_get_user_pages(vaddr, cnt, pagep); 340 if (ret) { 341 /* 342 * if (ret == -EBUSY) 343 * We can't continue because the pagep array won't be 344 * initialized. This should never happen, 345 * unless perhaps the user has mpin'ed the pages 346 * themselves. 347 */ 348 qib_devinfo(dd->pcidev, 349 "Failed to lock addr %p, %u pages: " 350 "errno %d\n", (void *) vaddr, cnt, -ret); 351 goto done; 352 } 353 for (i = 0; i < cnt; i++, vaddr += PAGE_SIZE) { 354 for (; ntids--; tid++) { 355 if (tid == tidcnt) 356 tid = 0; 357 if (!dd->pageshadow[ctxttid + tid]) 358 break; 359 } 360 if (ntids < 0) { 361 /* 362 * Oops, wrapped all the way through their TIDs, 363 * and didn't have enough free; see comments at 364 * start of routine 365 */ 366 i--; /* last tidlist[i] not filled in */ 367 ret = -ENOMEM; 368 break; 369 } 370 tidlist[i] = tid + tidoff; 371 /* we "know" system pages and TID pages are same size */ 372 dd->pageshadow[ctxttid + tid] = pagep[i]; 373 dd->physshadow[ctxttid + tid] = 374 qib_map_page(dd->pcidev, pagep[i], 0, PAGE_SIZE, 375 PCI_DMA_FROMDEVICE); 376 /* 377 * don't need atomic or it's overhead 378 */ 379 __set_bit(tid, tidmap); 380 physaddr = dd->physshadow[ctxttid + tid]; 381 /* PERFORMANCE: below should almost certainly be cached */ 382 dd->f_put_tid(dd, &tidbase[tid], 383 RCVHQ_RCV_TYPE_EXPECTED, physaddr); 384 /* 385 * don't check this tid in qib_ctxtshadow, since we 386 * just filled it in; start with the next one. 387 */ 388 tid++; 389 } 390 391 if (ret) { 392 u32 limit; 393cleanup: 394 /* jump here if copy out of updated info failed... */ 395 /* same code that's in qib_free_tid() */ 396 limit = sizeof(tidmap) * BITS_PER_BYTE; 397 if (limit > tidcnt) 398 /* just in case size changes in future */ 399 limit = tidcnt; 400 tid = find_first_bit((const unsigned long *)tidmap, limit); 401 for (; tid < limit; tid++) { 402 if (!test_bit(tid, tidmap)) 403 continue; 404 if (dd->pageshadow[ctxttid + tid]) { 405 dma_addr_t phys; 406 407 phys = dd->physshadow[ctxttid + tid]; 408 dd->physshadow[ctxttid + tid] = dd->tidinvalid; 409 /* PERFORMANCE: below should almost certainly 410 * be cached 411 */ 412 dd->f_put_tid(dd, &tidbase[tid], 413 RCVHQ_RCV_TYPE_EXPECTED, 414 dd->tidinvalid); 415 pci_unmap_page(dd->pcidev, phys, PAGE_SIZE, 416 PCI_DMA_FROMDEVICE); 417 dd->pageshadow[ctxttid + tid] = NULL; 418 } 419 } 420 qib_release_user_pages(pagep, cnt); 421 } else { 422 /* 423 * Copy the updated array, with qib_tid's filled in, back 424 * to user. Since we did the copy in already, this "should 425 * never fail" If it does, we have to clean up... 426 */ 427 if (copy_to_user((void __user *) 428 (unsigned long) ti->tidlist, 429 tidlist, cnt * sizeof(*tidlist))) { 430 ret = -EFAULT; 431 goto cleanup; 432 } 433 if (copy_to_user((void __user *) (unsigned long) ti->tidmap, 434 tidmap, sizeof tidmap)) { 435 ret = -EFAULT; 436 goto cleanup; 437 } 438 if (tid == tidcnt) 439 tid = 0; 440 if (!rcd->subctxt_cnt) 441 rcd->tidcursor = tid; 442 else 443 tidcursor_fp(fp) = tid; 444 } 445 446done: 447 return ret; 448} 449 450/** 451 * qib_tid_free - free a context TID 452 * @rcd: the context 453 * @subctxt: the subcontext 454 * @ti: the TID info 455 * 456 * right now we are unlocking one page at a time, but since 457 * the intended use of this routine is for a single group of 458 * virtually contiguous pages, that should change to improve 459 * performance. We check that the TID is in range for this context 460 * but otherwise don't check validity; if user has an error and 461 * frees the wrong tid, it's only their own data that can thereby 462 * be corrupted. We do check that the TID was in use, for sanity 463 * We always use our idea of the saved address, not the address that 464 * they pass in to us. 465 */ 466static int qib_tid_free(struct qib_ctxtdata *rcd, unsigned subctxt, 467 const struct qib_tid_info *ti) 468{ 469 int ret = 0; 470 u32 tid, ctxttid, cnt, limit, tidcnt; 471 struct qib_devdata *dd = rcd->dd; 472 u64 __iomem *tidbase; 473 unsigned long tidmap[8]; 474 475 if (!dd->pageshadow) { 476 ret = -ENOMEM; 477 goto done; 478 } 479 480 if (copy_from_user(tidmap, (void __user *)(unsigned long)ti->tidmap, 481 sizeof tidmap)) { 482 ret = -EFAULT; 483 goto done; 484 } 485 486 ctxttid = rcd->ctxt * dd->rcvtidcnt; 487 if (!rcd->subctxt_cnt) 488 tidcnt = dd->rcvtidcnt; 489 else if (!subctxt) { 490 tidcnt = (dd->rcvtidcnt / rcd->subctxt_cnt) + 491 (dd->rcvtidcnt % rcd->subctxt_cnt); 492 ctxttid += dd->rcvtidcnt - tidcnt; 493 } else { 494 tidcnt = dd->rcvtidcnt / rcd->subctxt_cnt; 495 ctxttid += tidcnt * (subctxt - 1); 496 } 497 tidbase = (u64 __iomem *) ((char __iomem *)(dd->kregbase) + 498 dd->rcvtidbase + 499 ctxttid * sizeof(*tidbase)); 500 501 limit = sizeof(tidmap) * BITS_PER_BYTE; 502 if (limit > tidcnt) 503 /* just in case size changes in future */ 504 limit = tidcnt; 505 tid = find_first_bit(tidmap, limit); 506 for (cnt = 0; tid < limit; tid++) { 507 /* 508 * small optimization; if we detect a run of 3 or so without 509 * any set, use find_first_bit again. That's mainly to 510 * accelerate the case where we wrapped, so we have some at 511 * the beginning, and some at the end, and a big gap 512 * in the middle. 513 */ 514 if (!test_bit(tid, tidmap)) 515 continue; 516 cnt++; 517 if (dd->pageshadow[ctxttid + tid]) { 518 struct page *p; 519 dma_addr_t phys; 520 521 p = dd->pageshadow[ctxttid + tid]; 522 dd->pageshadow[ctxttid + tid] = NULL; 523 phys = dd->physshadow[ctxttid + tid]; 524 dd->physshadow[ctxttid + tid] = dd->tidinvalid; 525 /* PERFORMANCE: below should almost certainly be 526 * cached 527 */ 528 dd->f_put_tid(dd, &tidbase[tid], 529 RCVHQ_RCV_TYPE_EXPECTED, dd->tidinvalid); 530 pci_unmap_page(dd->pcidev, phys, PAGE_SIZE, 531 PCI_DMA_FROMDEVICE); 532 qib_release_user_pages(&p, 1); 533 } 534 } 535done: 536 return ret; 537} 538 539/** 540 * qib_set_part_key - set a partition key 541 * @rcd: the context 542 * @key: the key 543 * 544 * We can have up to 4 active at a time (other than the default, which is 545 * always allowed). This is somewhat tricky, since multiple contexts may set 546 * the same key, so we reference count them, and clean up at exit. All 4 547 * partition keys are packed into a single qlogic_ib register. It's an 548 * error for a process to set the same pkey multiple times. We provide no 549 * mechanism to de-allocate a pkey at this time, we may eventually need to 550 * do that. I've used the atomic operations, and no locking, and only make 551 * a single pass through what's available. This should be more than 552 * adequate for some time. I'll think about spinlocks or the like if and as 553 * it's necessary. 554 */ 555static int qib_set_part_key(struct qib_ctxtdata *rcd, u16 key) 556{ 557 struct qib_pportdata *ppd = rcd->ppd; 558 int i, any = 0, pidx = -1; 559 u16 lkey = key & 0x7FFF; 560 int ret; 561 562 if (lkey == (QIB_DEFAULT_P_KEY & 0x7FFF)) { 563 /* nothing to do; this key always valid */ 564 ret = 0; 565 goto bail; 566 } 567 568 if (!lkey) { 569 ret = -EINVAL; 570 goto bail; 571 } 572 573 /* 574 * Set the full membership bit, because it has to be 575 * set in the register or the packet, and it seems 576 * cleaner to set in the register than to force all 577 * callers to set it. 578 */ 579 key |= 0x8000; 580 581 for (i = 0; i < ARRAY_SIZE(rcd->pkeys); i++) { 582 if (!rcd->pkeys[i] && pidx == -1) 583 pidx = i; 584 if (rcd->pkeys[i] == key) { 585 ret = -EEXIST; 586 goto bail; 587 } 588 } 589 if (pidx == -1) { 590 ret = -EBUSY; 591 goto bail; 592 } 593 for (any = i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) { 594 if (!ppd->pkeys[i]) { 595 any++; 596 continue; 597 } 598 if (ppd->pkeys[i] == key) { 599 atomic_t *pkrefs = &ppd->pkeyrefs[i]; 600 601 if (atomic_inc_return(pkrefs) > 1) { 602 rcd->pkeys[pidx] = key; 603 ret = 0; 604 goto bail; 605 } else { 606 /* 607 * lost race, decrement count, catch below 608 */ 609 atomic_dec(pkrefs); 610 any++; 611 } 612 } 613 if ((ppd->pkeys[i] & 0x7FFF) == lkey) { 614 /* 615 * It makes no sense to have both the limited and 616 * full membership PKEY set at the same time since 617 * the unlimited one will disable the limited one. 618 */ 619 ret = -EEXIST; 620 goto bail; 621 } 622 } 623 if (!any) { 624 ret = -EBUSY; 625 goto bail; 626 } 627 for (any = i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) { 628 if (!ppd->pkeys[i] && 629 atomic_inc_return(&ppd->pkeyrefs[i]) == 1) { 630 rcd->pkeys[pidx] = key; 631 ppd->pkeys[i] = key; 632 (void) ppd->dd->f_set_ib_cfg(ppd, QIB_IB_CFG_PKEYS, 0); 633 ret = 0; 634 goto bail; 635 } 636 } 637 ret = -EBUSY; 638 639bail: 640 return ret; 641} 642 643/** 644 * qib_manage_rcvq - manage a context's receive queue 645 * @rcd: the context 646 * @subctxt: the subcontext 647 * @start_stop: action to carry out 648 * 649 * start_stop == 0 disables receive on the context, for use in queue 650 * overflow conditions. start_stop==1 re-enables, to be used to 651 * re-init the software copy of the head register 652 */ 653static int qib_manage_rcvq(struct qib_ctxtdata *rcd, unsigned subctxt, 654 int start_stop) 655{ 656 struct qib_devdata *dd = rcd->dd; 657 unsigned int rcvctrl_op; 658 659 if (subctxt) 660 goto bail; 661 /* atomically clear receive enable ctxt. */ 662 if (start_stop) { 663 /* 664 * On enable, force in-memory copy of the tail register to 665 * 0, so that protocol code doesn't have to worry about 666 * whether or not the chip has yet updated the in-memory 667 * copy or not on return from the system call. The chip 668 * always resets it's tail register back to 0 on a 669 * transition from disabled to enabled. 670 */ 671 if (rcd->rcvhdrtail_kvaddr) 672 qib_clear_rcvhdrtail(rcd); 673 rcvctrl_op = QIB_RCVCTRL_CTXT_ENB; 674 } else 675 rcvctrl_op = QIB_RCVCTRL_CTXT_DIS; 676 dd->f_rcvctrl(rcd->ppd, rcvctrl_op, rcd->ctxt); 677 /* always; new head should be equal to new tail; see above */ 678bail: 679 return 0; 680} 681 682static void qib_clean_part_key(struct qib_ctxtdata *rcd, 683 struct qib_devdata *dd) 684{ 685 int i, j, pchanged = 0; 686 u64 oldpkey; 687 struct qib_pportdata *ppd = rcd->ppd; 688 689 /* for debugging only */ 690 oldpkey = (u64) ppd->pkeys[0] | 691 ((u64) ppd->pkeys[1] << 16) | 692 ((u64) ppd->pkeys[2] << 32) | 693 ((u64) ppd->pkeys[3] << 48); 694 695 for (i = 0; i < ARRAY_SIZE(rcd->pkeys); i++) { 696 if (!rcd->pkeys[i]) 697 continue; 698 for (j = 0; j < ARRAY_SIZE(ppd->pkeys); j++) { 699 /* check for match independent of the global bit */ 700 if ((ppd->pkeys[j] & 0x7fff) != 701 (rcd->pkeys[i] & 0x7fff)) 702 continue; 703 if (atomic_dec_and_test(&ppd->pkeyrefs[j])) { 704 ppd->pkeys[j] = 0; 705 pchanged++; 706 } 707 break; 708 } 709 rcd->pkeys[i] = 0; 710 } 711 if (pchanged) 712 (void) ppd->dd->f_set_ib_cfg(ppd, QIB_IB_CFG_PKEYS, 0); 713} 714 715/* common code for the mappings on dma_alloc_coherent mem */ 716static int qib_mmap_mem(struct vm_area_struct *vma, struct qib_ctxtdata *rcd, 717 unsigned len, void *kvaddr, u32 write_ok, char *what) 718{ 719 struct qib_devdata *dd = rcd->dd; 720 unsigned long pfn; 721 int ret; 722 723 if ((vma->vm_end - vma->vm_start) > len) { 724 qib_devinfo(dd->pcidev, 725 "FAIL on %s: len %lx > %x\n", what, 726 vma->vm_end - vma->vm_start, len); 727 ret = -EFAULT; 728 goto bail; 729 } 730 731 /* 732 * shared context user code requires rcvhdrq mapped r/w, others 733 * only allowed readonly mapping. 734 */ 735 if (!write_ok) { 736 if (vma->vm_flags & VM_WRITE) { 737 qib_devinfo(dd->pcidev, 738 "%s must be mapped readonly\n", what); 739 ret = -EPERM; 740 goto bail; 741 } 742 743 /* don't allow them to later change with mprotect */ 744 vma->vm_flags &= ~VM_MAYWRITE; 745 } 746 747 pfn = virt_to_phys(kvaddr) >> PAGE_SHIFT; 748 ret = remap_pfn_range(vma, vma->vm_start, pfn, 749 len, vma->vm_page_prot); 750 if (ret) 751 qib_devinfo(dd->pcidev, "%s ctxt%u mmap of %lx, %x " 752 "bytes failed: %d\n", what, rcd->ctxt, 753 pfn, len, ret); 754bail: 755 return ret; 756} 757 758static int mmap_ureg(struct vm_area_struct *vma, struct qib_devdata *dd, 759 u64 ureg) 760{ 761 unsigned long phys; 762 unsigned long sz; 763 int ret; 764 765 /* 766 * This is real hardware, so use io_remap. This is the mechanism 767 * for the user process to update the head registers for their ctxt 768 * in the chip. 769 */ 770 sz = dd->flags & QIB_HAS_HDRSUPP ? 2 * PAGE_SIZE : PAGE_SIZE; 771 if ((vma->vm_end - vma->vm_start) > sz) { 772 qib_devinfo(dd->pcidev, "FAIL mmap userreg: reqlen " 773 "%lx > PAGE\n", vma->vm_end - vma->vm_start); 774 ret = -EFAULT; 775 } else { 776 phys = dd->physaddr + ureg; 777 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 778 779 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND; 780 ret = io_remap_pfn_range(vma, vma->vm_start, 781 phys >> PAGE_SHIFT, 782 vma->vm_end - vma->vm_start, 783 vma->vm_page_prot); 784 } 785 return ret; 786} 787 788static int mmap_piobufs(struct vm_area_struct *vma, 789 struct qib_devdata *dd, 790 struct qib_ctxtdata *rcd, 791 unsigned piobufs, unsigned piocnt) 792{ 793 unsigned long phys; 794 int ret; 795 796 /* 797 * When we map the PIO buffers in the chip, we want to map them as 798 * writeonly, no read possible; unfortunately, x86 doesn't allow 799 * for this in hardware, but we still prevent users from asking 800 * for it. 801 */ 802 if ((vma->vm_end - vma->vm_start) > (piocnt * dd->palign)) { 803 qib_devinfo(dd->pcidev, "FAIL mmap piobufs: " 804 "reqlen %lx > PAGE\n", 805 vma->vm_end - vma->vm_start); 806 ret = -EINVAL; 807 goto bail; 808 } 809 810 phys = dd->physaddr + piobufs; 811 812#if defined(__powerpc__) 813 /* There isn't a generic way to specify writethrough mappings */ 814 pgprot_val(vma->vm_page_prot) |= _PAGE_NO_CACHE; 815 pgprot_val(vma->vm_page_prot) |= _PAGE_WRITETHRU; 816 pgprot_val(vma->vm_page_prot) &= ~_PAGE_GUARDED; 817#endif 818 819 /* 820 * don't allow them to later change to readable with mprotect (for when 821 * not initially mapped readable, as is normally the case) 822 */ 823 vma->vm_flags &= ~VM_MAYREAD; 824 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND; 825 826 if (qib_wc_pat) 827 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 828 829 ret = io_remap_pfn_range(vma, vma->vm_start, phys >> PAGE_SHIFT, 830 vma->vm_end - vma->vm_start, 831 vma->vm_page_prot); 832bail: 833 return ret; 834} 835 836static int mmap_rcvegrbufs(struct vm_area_struct *vma, 837 struct qib_ctxtdata *rcd) 838{ 839 struct qib_devdata *dd = rcd->dd; 840 unsigned long start, size; 841 size_t total_size, i; 842 unsigned long pfn; 843 int ret; 844 845 size = rcd->rcvegrbuf_size; 846 total_size = rcd->rcvegrbuf_chunks * size; 847 if ((vma->vm_end - vma->vm_start) > total_size) { 848 qib_devinfo(dd->pcidev, "FAIL on egr bufs: " 849 "reqlen %lx > actual %lx\n", 850 vma->vm_end - vma->vm_start, 851 (unsigned long) total_size); 852 ret = -EINVAL; 853 goto bail; 854 } 855 856 if (vma->vm_flags & VM_WRITE) { 857 qib_devinfo(dd->pcidev, "Can't map eager buffers as " 858 "writable (flags=%lx)\n", vma->vm_flags); 859 ret = -EPERM; 860 goto bail; 861 } 862 /* don't allow them to later change to writeable with mprotect */ 863 vma->vm_flags &= ~VM_MAYWRITE; 864 865 start = vma->vm_start; 866 867 for (i = 0; i < rcd->rcvegrbuf_chunks; i++, start += size) { 868 pfn = virt_to_phys(rcd->rcvegrbuf[i]) >> PAGE_SHIFT; 869 ret = remap_pfn_range(vma, start, pfn, size, 870 vma->vm_page_prot); 871 if (ret < 0) 872 goto bail; 873 } 874 ret = 0; 875 876bail: 877 return ret; 878} 879 880/* 881 * qib_file_vma_fault - handle a VMA page fault. 882 */ 883static int qib_file_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 884{ 885 struct page *page; 886 887 page = vmalloc_to_page((void *)(vmf->pgoff << PAGE_SHIFT)); 888 if (!page) 889 return VM_FAULT_SIGBUS; 890 891 get_page(page); 892 vmf->page = page; 893 894 return 0; 895} 896 897static struct vm_operations_struct qib_file_vm_ops = { 898 .fault = qib_file_vma_fault, 899}; 900 901static int mmap_kvaddr(struct vm_area_struct *vma, u64 pgaddr, 902 struct qib_ctxtdata *rcd, unsigned subctxt) 903{ 904 struct qib_devdata *dd = rcd->dd; 905 unsigned subctxt_cnt; 906 unsigned long len; 907 void *addr; 908 size_t size; 909 int ret = 0; 910 911 subctxt_cnt = rcd->subctxt_cnt; 912 size = rcd->rcvegrbuf_chunks * rcd->rcvegrbuf_size; 913 914 /* 915 * Each process has all the subctxt uregbase, rcvhdrq, and 916 * rcvegrbufs mmapped - as an array for all the processes, 917 * and also separately for this process. 918 */ 919 if (pgaddr == cvt_kvaddr(rcd->subctxt_uregbase)) { 920 addr = rcd->subctxt_uregbase; 921 size = PAGE_SIZE * subctxt_cnt; 922 } else if (pgaddr == cvt_kvaddr(rcd->subctxt_rcvhdr_base)) { 923 addr = rcd->subctxt_rcvhdr_base; 924 size = rcd->rcvhdrq_size * subctxt_cnt; 925 } else if (pgaddr == cvt_kvaddr(rcd->subctxt_rcvegrbuf)) { 926 addr = rcd->subctxt_rcvegrbuf; 927 size *= subctxt_cnt; 928 } else if (pgaddr == cvt_kvaddr(rcd->subctxt_uregbase + 929 PAGE_SIZE * subctxt)) { 930 addr = rcd->subctxt_uregbase + PAGE_SIZE * subctxt; 931 size = PAGE_SIZE; 932 } else if (pgaddr == cvt_kvaddr(rcd->subctxt_rcvhdr_base + 933 rcd->rcvhdrq_size * subctxt)) { 934 addr = rcd->subctxt_rcvhdr_base + 935 rcd->rcvhdrq_size * subctxt; 936 size = rcd->rcvhdrq_size; 937 } else if (pgaddr == cvt_kvaddr(&rcd->user_event_mask[subctxt])) { 938 addr = rcd->user_event_mask; 939 size = PAGE_SIZE; 940 } else if (pgaddr == cvt_kvaddr(rcd->subctxt_rcvegrbuf + 941 size * subctxt)) { 942 addr = rcd->subctxt_rcvegrbuf + size * subctxt; 943 /* rcvegrbufs are read-only on the slave */ 944 if (vma->vm_flags & VM_WRITE) { 945 qib_devinfo(dd->pcidev, 946 "Can't map eager buffers as " 947 "writable (flags=%lx)\n", vma->vm_flags); 948 ret = -EPERM; 949 goto bail; 950 } 951 /* 952 * Don't allow permission to later change to writeable 953 * with mprotect. 954 */ 955 vma->vm_flags &= ~VM_MAYWRITE; 956 } else 957 goto bail; 958 len = vma->vm_end - vma->vm_start; 959 if (len > size) { 960 ret = -EINVAL; 961 goto bail; 962 } 963 964 vma->vm_pgoff = (unsigned long) addr >> PAGE_SHIFT; 965 vma->vm_ops = &qib_file_vm_ops; 966 vma->vm_flags |= VM_RESERVED | VM_DONTEXPAND; 967 ret = 1; 968 969bail: 970 return ret; 971} 972 973/** 974 * qib_mmapf - mmap various structures into user space 975 * @fp: the file pointer 976 * @vma: the VM area 977 * 978 * We use this to have a shared buffer between the kernel and the user code 979 * for the rcvhdr queue, egr buffers, and the per-context user regs and pio 980 * buffers in the chip. We have the open and close entries so we can bump 981 * the ref count and keep the driver from being unloaded while still mapped. 982 */ 983static int qib_mmapf(struct file *fp, struct vm_area_struct *vma) 984{ 985 struct qib_ctxtdata *rcd; 986 struct qib_devdata *dd; 987 u64 pgaddr, ureg; 988 unsigned piobufs, piocnt; 989 int ret, match = 1; 990 991 rcd = ctxt_fp(fp); 992 if (!rcd || !(vma->vm_flags & VM_SHARED)) { 993 ret = -EINVAL; 994 goto bail; 995 } 996 dd = rcd->dd; 997 998 /* 999 * This is the qib_do_user_init() code, mapping the shared buffers 1000 * and per-context user registers into the user process. The address 1001 * referred to by vm_pgoff is the file offset passed via mmap(). 1002 * For shared contexts, this is the kernel vmalloc() address of the 1003 * pages to share with the master. 1004 * For non-shared or master ctxts, this is a physical address. 1005 * We only do one mmap for each space mapped. 1006 */ 1007 pgaddr = vma->vm_pgoff << PAGE_SHIFT; 1008 1009 /* 1010 * Check for 0 in case one of the allocations failed, but user 1011 * called mmap anyway. 1012 */ 1013 if (!pgaddr) { 1014 ret = -EINVAL; 1015 goto bail; 1016 } 1017 1018 /* 1019 * Physical addresses must fit in 40 bits for our hardware. 1020 * Check for kernel virtual addresses first, anything else must 1021 * match a HW or memory address. 1022 */ 1023 ret = mmap_kvaddr(vma, pgaddr, rcd, subctxt_fp(fp)); 1024 if (ret) { 1025 if (ret > 0) 1026 ret = 0; 1027 goto bail; 1028 } 1029 1030 ureg = dd->uregbase + dd->ureg_align * rcd->ctxt; 1031 if (!rcd->subctxt_cnt) { 1032 /* ctxt is not shared */ 1033 piocnt = rcd->piocnt; 1034 piobufs = rcd->piobufs; 1035 } else if (!subctxt_fp(fp)) { 1036 /* caller is the master */ 1037 piocnt = (rcd->piocnt / rcd->subctxt_cnt) + 1038 (rcd->piocnt % rcd->subctxt_cnt); 1039 piobufs = rcd->piobufs + 1040 dd->palign * (rcd->piocnt - piocnt); 1041 } else { 1042 unsigned slave = subctxt_fp(fp) - 1; 1043 1044 /* caller is a slave */ 1045 piocnt = rcd->piocnt / rcd->subctxt_cnt; 1046 piobufs = rcd->piobufs + dd->palign * piocnt * slave; 1047 } 1048 1049 if (pgaddr == ureg) 1050 ret = mmap_ureg(vma, dd, ureg); 1051 else if (pgaddr == piobufs) 1052 ret = mmap_piobufs(vma, dd, rcd, piobufs, piocnt); 1053 else if (pgaddr == dd->pioavailregs_phys) 1054 /* in-memory copy of pioavail registers */ 1055 ret = qib_mmap_mem(vma, rcd, PAGE_SIZE, 1056 (void *) dd->pioavailregs_dma, 0, 1057 "pioavail registers"); 1058 else if (pgaddr == rcd->rcvegr_phys) 1059 ret = mmap_rcvegrbufs(vma, rcd); 1060 else if (pgaddr == (u64) rcd->rcvhdrq_phys) 1061 /* 1062 * The rcvhdrq itself; multiple pages, contiguous 1063 * from an i/o perspective. Shared contexts need 1064 * to map r/w, so we allow writing. 1065 */ 1066 ret = qib_mmap_mem(vma, rcd, rcd->rcvhdrq_size, 1067 rcd->rcvhdrq, 1, "rcvhdrq"); 1068 else if (pgaddr == (u64) rcd->rcvhdrqtailaddr_phys) 1069 /* in-memory copy of rcvhdrq tail register */ 1070 ret = qib_mmap_mem(vma, rcd, PAGE_SIZE, 1071 rcd->rcvhdrtail_kvaddr, 0, 1072 "rcvhdrq tail"); 1073 else 1074 match = 0; 1075 if (!match) 1076 ret = -EINVAL; 1077 1078 vma->vm_private_data = NULL; 1079 1080 if (ret < 0) 1081 qib_devinfo(dd->pcidev, 1082 "mmap Failure %d: off %llx len %lx\n", 1083 -ret, (unsigned long long)pgaddr, 1084 vma->vm_end - vma->vm_start); 1085bail: 1086 return ret; 1087} 1088 1089static unsigned int qib_poll_urgent(struct qib_ctxtdata *rcd, 1090 struct file *fp, 1091 struct poll_table_struct *pt) 1092{ 1093 struct qib_devdata *dd = rcd->dd; 1094 unsigned pollflag; 1095 1096 poll_wait(fp, &rcd->wait, pt); 1097 1098 spin_lock_irq(&dd->uctxt_lock); 1099 if (rcd->urgent != rcd->urgent_poll) { 1100 pollflag = POLLIN | POLLRDNORM; 1101 rcd->urgent_poll = rcd->urgent; 1102 } else { 1103 pollflag = 0; 1104 set_bit(QIB_CTXT_WAITING_URG, &rcd->flag); 1105 } 1106 spin_unlock_irq(&dd->uctxt_lock); 1107 1108 return pollflag; 1109} 1110 1111static unsigned int qib_poll_next(struct qib_ctxtdata *rcd, 1112 struct file *fp, 1113 struct poll_table_struct *pt) 1114{ 1115 struct qib_devdata *dd = rcd->dd; 1116 unsigned pollflag; 1117 1118 poll_wait(fp, &rcd->wait, pt); 1119 1120 spin_lock_irq(&dd->uctxt_lock); 1121 if (dd->f_hdrqempty(rcd)) { 1122 set_bit(QIB_CTXT_WAITING_RCV, &rcd->flag); 1123 dd->f_rcvctrl(rcd->ppd, QIB_RCVCTRL_INTRAVAIL_ENB, rcd->ctxt); 1124 pollflag = 0; 1125 } else 1126 pollflag = POLLIN | POLLRDNORM; 1127 spin_unlock_irq(&dd->uctxt_lock); 1128 1129 return pollflag; 1130} 1131 1132static unsigned int qib_poll(struct file *fp, struct poll_table_struct *pt) 1133{ 1134 struct qib_ctxtdata *rcd; 1135 unsigned pollflag; 1136 1137 rcd = ctxt_fp(fp); 1138 if (!rcd) 1139 pollflag = POLLERR; 1140 else if (rcd->poll_type == QIB_POLL_TYPE_URGENT) 1141 pollflag = qib_poll_urgent(rcd, fp, pt); 1142 else if (rcd->poll_type == QIB_POLL_TYPE_ANYRCV) 1143 pollflag = qib_poll_next(rcd, fp, pt); 1144 else /* invalid */ 1145 pollflag = POLLERR; 1146 1147 return pollflag; 1148} 1149 1150/* 1151 * Check that userland and driver are compatible for subcontexts. 1152 */ 1153static int qib_compatible_subctxts(int user_swmajor, int user_swminor) 1154{ 1155 /* this code is written long-hand for clarity */ 1156 if (QIB_USER_SWMAJOR != user_swmajor) { 1157 /* no promise of compatibility if major mismatch */ 1158 return 0; 1159 } 1160 if (QIB_USER_SWMAJOR == 1) { 1161 switch (QIB_USER_SWMINOR) { 1162 case 0: 1163 case 1: 1164 case 2: 1165 /* no subctxt implementation so cannot be compatible */ 1166 return 0; 1167 case 3: 1168 /* 3 is only compatible with itself */ 1169 return user_swminor == 3; 1170 default: 1171 /* >= 4 are compatible (or are expected to be) */ 1172 return user_swminor >= 4; 1173 } 1174 } 1175 /* make no promises yet for future major versions */ 1176 return 0; 1177} 1178 1179static int init_subctxts(struct qib_devdata *dd, 1180 struct qib_ctxtdata *rcd, 1181 const struct qib_user_info *uinfo) 1182{ 1183 int ret = 0; 1184 unsigned num_subctxts; 1185 size_t size; 1186 1187 /* 1188 * If the user is requesting zero subctxts, 1189 * skip the subctxt allocation. 1190 */ 1191 if (uinfo->spu_subctxt_cnt <= 0) 1192 goto bail; 1193 num_subctxts = uinfo->spu_subctxt_cnt; 1194 1195 /* Check for subctxt compatibility */ 1196 if (!qib_compatible_subctxts(uinfo->spu_userversion >> 16, 1197 uinfo->spu_userversion & 0xffff)) { 1198 qib_devinfo(dd->pcidev, 1199 "Mismatched user version (%d.%d) and driver " 1200 "version (%d.%d) while context sharing. Ensure " 1201 "that driver and library are from the same " 1202 "release.\n", 1203 (int) (uinfo->spu_userversion >> 16), 1204 (int) (uinfo->spu_userversion & 0xffff), 1205 QIB_USER_SWMAJOR, QIB_USER_SWMINOR); 1206 goto bail; 1207 } 1208 if (num_subctxts > QLOGIC_IB_MAX_SUBCTXT) { 1209 ret = -EINVAL; 1210 goto bail; 1211 } 1212 1213 rcd->subctxt_uregbase = vmalloc_user(PAGE_SIZE * num_subctxts); 1214 if (!rcd->subctxt_uregbase) { 1215 ret = -ENOMEM; 1216 goto bail; 1217 } 1218 /* Note: rcd->rcvhdrq_size isn't initialized yet. */ 1219 size = ALIGN(dd->rcvhdrcnt * dd->rcvhdrentsize * 1220 sizeof(u32), PAGE_SIZE) * num_subctxts; 1221 rcd->subctxt_rcvhdr_base = vmalloc_user(size); 1222 if (!rcd->subctxt_rcvhdr_base) { 1223 ret = -ENOMEM; 1224 goto bail_ureg; 1225 } 1226 1227 rcd->subctxt_rcvegrbuf = vmalloc_user(rcd->rcvegrbuf_chunks * 1228 rcd->rcvegrbuf_size * 1229 num_subctxts); 1230 if (!rcd->subctxt_rcvegrbuf) { 1231 ret = -ENOMEM; 1232 goto bail_rhdr; 1233 } 1234 1235 rcd->subctxt_cnt = uinfo->spu_subctxt_cnt; 1236 rcd->subctxt_id = uinfo->spu_subctxt_id; 1237 rcd->active_slaves = 1; 1238 rcd->redirect_seq_cnt = 1; 1239 set_bit(QIB_CTXT_MASTER_UNINIT, &rcd->flag); 1240 goto bail; 1241 1242bail_rhdr: 1243 vfree(rcd->subctxt_rcvhdr_base); 1244bail_ureg: 1245 vfree(rcd->subctxt_uregbase); 1246 rcd->subctxt_uregbase = NULL; 1247bail: 1248 return ret; 1249} 1250 1251static int setup_ctxt(struct qib_pportdata *ppd, int ctxt, 1252 struct file *fp, const struct qib_user_info *uinfo) 1253{ 1254 struct qib_devdata *dd = ppd->dd; 1255 struct qib_ctxtdata *rcd; 1256 void *ptmp = NULL; 1257 int ret; 1258 1259 rcd = qib_create_ctxtdata(ppd, ctxt); 1260 1261 /* 1262 * Allocate memory for use in qib_tid_update() at open to 1263 * reduce cost of expected send setup per message segment 1264 */ 1265 if (rcd) 1266 ptmp = kmalloc(dd->rcvtidcnt * sizeof(u16) + 1267 dd->rcvtidcnt * sizeof(struct page **), 1268 GFP_KERNEL); 1269 1270 if (!rcd || !ptmp) { 1271 qib_dev_err(dd, "Unable to allocate ctxtdata " 1272 "memory, failing open\n"); 1273 ret = -ENOMEM; 1274 goto bailerr; 1275 } 1276 rcd->userversion = uinfo->spu_userversion; 1277 ret = init_subctxts(dd, rcd, uinfo); 1278 if (ret) 1279 goto bailerr; 1280 rcd->tid_pg_list = ptmp; 1281 rcd->pid = current->pid; 1282 init_waitqueue_head(&dd->rcd[ctxt]->wait); 1283 strlcpy(rcd->comm, current->comm, sizeof(rcd->comm)); 1284 ctxt_fp(fp) = rcd; 1285 qib_stats.sps_ctxts++; 1286 ret = 0; 1287 goto bail; 1288 1289bailerr: 1290 dd->rcd[ctxt] = NULL; 1291 kfree(rcd); 1292 kfree(ptmp); 1293bail: 1294 return ret; 1295} 1296 1297static inline int usable(struct qib_pportdata *ppd) 1298{ 1299 struct qib_devdata *dd = ppd->dd; 1300 1301 return dd && (dd->flags & QIB_PRESENT) && dd->kregbase && ppd->lid && 1302 (ppd->lflags & QIBL_LINKACTIVE); 1303} 1304 1305/* 1306 * Select a context on the given device, either using a requested port 1307 * or the port based on the context number. 1308 */ 1309static int choose_port_ctxt(struct file *fp, struct qib_devdata *dd, u32 port, 1310 const struct qib_user_info *uinfo) 1311{ 1312 struct qib_pportdata *ppd = NULL; 1313 int ret, ctxt; 1314 1315 if (port) { 1316 if (!usable(dd->pport + port - 1)) { 1317 ret = -ENETDOWN; 1318 goto done; 1319 } else 1320 ppd = dd->pport + port - 1; 1321 } 1322 for (ctxt = dd->first_user_ctxt; ctxt < dd->cfgctxts && dd->rcd[ctxt]; 1323 ctxt++) 1324 ; 1325 if (ctxt == dd->cfgctxts) { 1326 ret = -EBUSY; 1327 goto done; 1328 } 1329 if (!ppd) { 1330 u32 pidx = ctxt % dd->num_pports; 1331 if (usable(dd->pport + pidx)) 1332 ppd = dd->pport + pidx; 1333 else { 1334 for (pidx = 0; pidx < dd->num_pports && !ppd; 1335 pidx++) 1336 if (usable(dd->pport + pidx)) 1337 ppd = dd->pport + pidx; 1338 } 1339 } 1340 ret = ppd ? setup_ctxt(ppd, ctxt, fp, uinfo) : -ENETDOWN; 1341done: 1342 return ret; 1343} 1344 1345static int find_free_ctxt(int unit, struct file *fp, 1346 const struct qib_user_info *uinfo) 1347{ 1348 struct qib_devdata *dd = qib_lookup(unit); 1349 int ret; 1350 1351 if (!dd || (uinfo->spu_port && uinfo->spu_port > dd->num_pports)) 1352 ret = -ENODEV; 1353 else 1354 ret = choose_port_ctxt(fp, dd, uinfo->spu_port, uinfo); 1355 1356 return ret; 1357} 1358 1359static int get_a_ctxt(struct file *fp, const struct qib_user_info *uinfo, 1360 unsigned alg) 1361{ 1362 struct qib_devdata *udd = NULL; 1363 int ret = 0, devmax, npresent, nup, ndev, dusable = 0, i; 1364 u32 port = uinfo->spu_port, ctxt; 1365 1366 devmax = qib_count_units(&npresent, &nup); 1367 if (!npresent) { 1368 ret = -ENXIO; 1369 goto done; 1370 } 1371 if (nup == 0) { 1372 ret = -ENETDOWN; 1373 goto done; 1374 } 1375 1376 if (alg == QIB_PORT_ALG_ACROSS) { 1377 unsigned inuse = ~0U; 1378 /* find device (with ACTIVE ports) with fewest ctxts in use */ 1379 for (ndev = 0; ndev < devmax; ndev++) { 1380 struct qib_devdata *dd = qib_lookup(ndev); 1381 unsigned cused = 0, cfree = 0; 1382 if (!dd) 1383 continue; 1384 if (port && port <= dd->num_pports && 1385 usable(dd->pport + port - 1)) 1386 dusable = 1; 1387 else 1388 for (i = 0; i < dd->num_pports; i++) 1389 if (usable(dd->pport + i)) 1390 dusable++; 1391 if (!dusable) 1392 continue; 1393 for (ctxt = dd->first_user_ctxt; ctxt < dd->cfgctxts; 1394 ctxt++) 1395 if (dd->rcd[ctxt]) 1396 cused++; 1397 else 1398 cfree++; 1399 if (cfree && cused < inuse) { 1400 udd = dd; 1401 inuse = cused; 1402 } 1403 } 1404 if (udd) { 1405 ret = choose_port_ctxt(fp, udd, port, uinfo); 1406 goto done; 1407 } 1408 } else { 1409 for (ndev = 0; ndev < devmax; ndev++) { 1410 struct qib_devdata *dd = qib_lookup(ndev); 1411 if (dd) { 1412 ret = choose_port_ctxt(fp, dd, port, uinfo); 1413 if (!ret) 1414 goto done; 1415 if (ret == -EBUSY) 1416 dusable++; 1417 } 1418 } 1419 } 1420 ret = dusable ? -EBUSY : -ENETDOWN; 1421 1422done: 1423 return ret; 1424} 1425 1426static int find_shared_ctxt(struct file *fp, 1427 const struct qib_user_info *uinfo) 1428{ 1429 int devmax, ndev, i; 1430 int ret = 0; 1431 1432 devmax = qib_count_units(NULL, NULL); 1433 1434 for (ndev = 0; ndev < devmax; ndev++) { 1435 struct qib_devdata *dd = qib_lookup(ndev); 1436 1437 /* device portion of usable() */ 1438 if (!(dd && (dd->flags & QIB_PRESENT) && dd->kregbase)) 1439 continue; 1440 for (i = dd->first_user_ctxt; i < dd->cfgctxts; i++) { 1441 struct qib_ctxtdata *rcd = dd->rcd[i]; 1442 1443 /* Skip ctxts which are not yet open */ 1444 if (!rcd || !rcd->cnt) 1445 continue; 1446 /* Skip ctxt if it doesn't match the requested one */ 1447 if (rcd->subctxt_id != uinfo->spu_subctxt_id) 1448 continue; 1449 /* Verify the sharing process matches the master */ 1450 if (rcd->subctxt_cnt != uinfo->spu_subctxt_cnt || 1451 rcd->userversion != uinfo->spu_userversion || 1452 rcd->cnt >= rcd->subctxt_cnt) { 1453 ret = -EINVAL; 1454 goto done; 1455 } 1456 ctxt_fp(fp) = rcd; 1457 subctxt_fp(fp) = rcd->cnt++; 1458 rcd->subpid[subctxt_fp(fp)] = current->pid; 1459 tidcursor_fp(fp) = 0; 1460 rcd->active_slaves |= 1 << subctxt_fp(fp); 1461 ret = 1; 1462 goto done; 1463 } 1464 } 1465 1466done: 1467 return ret; 1468} 1469 1470static int qib_open(struct inode *in, struct file *fp) 1471{ 1472 /* The real work is performed later in qib_assign_ctxt() */ 1473 fp->private_data = kzalloc(sizeof(struct qib_filedata), GFP_KERNEL); 1474 if (fp->private_data) /* no cpu affinity by default */ 1475 ((struct qib_filedata *)fp->private_data)->rec_cpu_num = -1; 1476 return fp->private_data ? 0 : -ENOMEM; 1477} 1478 1479/* 1480 * Get ctxt early, so can set affinity prior to memory allocation. 1481 */ 1482static int qib_assign_ctxt(struct file *fp, const struct qib_user_info *uinfo) 1483{ 1484 int ret; 1485 int i_minor; 1486 unsigned swmajor, swminor, alg = QIB_PORT_ALG_ACROSS; 1487 1488 /* Check to be sure we haven't already initialized this file */ 1489 if (ctxt_fp(fp)) { 1490 ret = -EINVAL; 1491 goto done; 1492 } 1493 1494 /* for now, if major version is different, bail */ 1495 swmajor = uinfo->spu_userversion >> 16; 1496 if (swmajor != QIB_USER_SWMAJOR) { 1497 ret = -ENODEV; 1498 goto done; 1499 } 1500 1501 swminor = uinfo->spu_userversion & 0xffff; 1502 1503 if (swminor >= 11 && uinfo->spu_port_alg < QIB_PORT_ALG_COUNT) 1504 alg = uinfo->spu_port_alg; 1505 1506 mutex_lock(&qib_mutex); 1507 1508 if (qib_compatible_subctxts(swmajor, swminor) && 1509 uinfo->spu_subctxt_cnt) { 1510 ret = find_shared_ctxt(fp, uinfo); 1511 if (ret) { 1512 if (ret > 0) 1513 ret = 0; 1514 goto done_chk_sdma; 1515 } 1516 } 1517 1518 i_minor = iminor(fp->f_dentry->d_inode) - QIB_USER_MINOR_BASE; 1519 if (i_minor) 1520 ret = find_free_ctxt(i_minor - 1, fp, uinfo); 1521 else 1522 ret = get_a_ctxt(fp, uinfo, alg); 1523 1524done_chk_sdma: 1525 if (!ret) { 1526 struct qib_filedata *fd = fp->private_data; 1527 const struct qib_ctxtdata *rcd = fd->rcd; 1528 const struct qib_devdata *dd = rcd->dd; 1529 1530 if (dd->flags & QIB_HAS_SEND_DMA) { 1531 fd->pq = qib_user_sdma_queue_create(&dd->pcidev->dev, 1532 dd->unit, 1533 rcd->ctxt, 1534 fd->subctxt); 1535 if (!fd->pq) 1536 ret = -ENOMEM; 1537 } 1538 1539 /* 1540 * If process has NOT already set it's affinity, select and 1541 * reserve a processor for it, as a rendevous for all 1542 * users of the driver. If they don't actually later 1543 * set affinity to this cpu, or set it to some other cpu, 1544 * it just means that sooner or later we don't recommend 1545 * a cpu, and let the scheduler do it's best. 1546 */ 1547 if (!ret && cpus_weight(current->cpus_allowed) >= 1548 qib_cpulist_count) { 1549 int cpu; 1550 cpu = find_first_zero_bit(qib_cpulist, 1551 qib_cpulist_count); 1552 if (cpu != qib_cpulist_count) { 1553 __set_bit(cpu, qib_cpulist); 1554 fd->rec_cpu_num = cpu; 1555 } 1556 } else if (cpus_weight(current->cpus_allowed) == 1 && 1557 test_bit(first_cpu(current->cpus_allowed), 1558 qib_cpulist)) 1559 qib_devinfo(dd->pcidev, "%s PID %u affinity " 1560 "set to cpu %d; already allocated\n", 1561 current->comm, current->pid, 1562 first_cpu(current->cpus_allowed)); 1563 } 1564 1565 mutex_unlock(&qib_mutex); 1566 1567done: 1568 return ret; 1569} 1570 1571 1572static int qib_do_user_init(struct file *fp, 1573 const struct qib_user_info *uinfo) 1574{ 1575 int ret; 1576 struct qib_ctxtdata *rcd = ctxt_fp(fp); 1577 struct qib_devdata *dd; 1578 unsigned uctxt; 1579 1580 /* Subctxts don't need to initialize anything since master did it. */ 1581 if (subctxt_fp(fp)) { 1582 ret = wait_event_interruptible(rcd->wait, 1583 !test_bit(QIB_CTXT_MASTER_UNINIT, &rcd->flag)); 1584 goto bail; 1585 } 1586 1587 dd = rcd->dd; 1588 1589 /* some ctxts may get extra buffers, calculate that here */ 1590 uctxt = rcd->ctxt - dd->first_user_ctxt; 1591 if (uctxt < dd->ctxts_extrabuf) { 1592 rcd->piocnt = dd->pbufsctxt + 1; 1593 rcd->pio_base = rcd->piocnt * uctxt; 1594 } else { 1595 rcd->piocnt = dd->pbufsctxt; 1596 rcd->pio_base = rcd->piocnt * uctxt + 1597 dd->ctxts_extrabuf; 1598 } 1599 1600 /* 1601 * All user buffers are 2KB buffers. If we ever support 1602 * giving 4KB buffers to user processes, this will need some 1603 * work. Can't use piobufbase directly, because it has 1604 * both 2K and 4K buffer base values. So check and handle. 1605 */ 1606 if ((rcd->pio_base + rcd->piocnt) > dd->piobcnt2k) { 1607 if (rcd->pio_base >= dd->piobcnt2k) { 1608 qib_dev_err(dd, 1609 "%u:ctxt%u: no 2KB buffers available\n", 1610 dd->unit, rcd->ctxt); 1611 ret = -ENOBUFS; 1612 goto bail; 1613 } 1614 rcd->piocnt = dd->piobcnt2k - rcd->pio_base; 1615 qib_dev_err(dd, "Ctxt%u: would use 4KB bufs, using %u\n", 1616 rcd->ctxt, rcd->piocnt); 1617 } 1618 1619 rcd->piobufs = dd->pio2k_bufbase + rcd->pio_base * dd->palign; 1620 qib_chg_pioavailkernel(dd, rcd->pio_base, rcd->piocnt, 1621 TXCHK_CHG_TYPE_USER, rcd); 1622 /* 1623 * try to ensure that processes start up with consistent avail update 1624 * for their own range, at least. If system very quiet, it might 1625 * have the in-memory copy out of date at startup for this range of 1626 * buffers, when a context gets re-used. Do after the chg_pioavail 1627 * and before the rest of setup, so it's "almost certain" the dma 1628 * will have occurred (can't 100% guarantee, but should be many 1629 * decimals of 9s, with this ordering), given how much else happens 1630 * after this. 1631 */ 1632 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_AVAIL_BLIP); 1633 1634 /* 1635 * Now allocate the rcvhdr Q and eager TIDs; skip the TID 1636 * array for time being. If rcd->ctxt > chip-supported, 1637 * we need to do extra stuff here to handle by handling overflow 1638 * through ctxt 0, someday 1639 */ 1640 ret = qib_create_rcvhdrq(dd, rcd); 1641 if (!ret) 1642 ret = qib_setup_eagerbufs(rcd); 1643 if (ret) 1644 goto bail_pio; 1645 1646 rcd->tidcursor = 0; /* start at beginning after open */ 1647 1648 /* initialize poll variables... */ 1649 rcd->urgent = 0; 1650 rcd->urgent_poll = 0; 1651 1652 /* 1653 * Now enable the ctxt for receive. 1654 * For chips that are set to DMA the tail register to memory 1655 * when they change (and when the update bit transitions from 1656 * 0 to 1. So for those chips, we turn it off and then back on. 1657 * This will (very briefly) affect any other open ctxts, but the 1658 * duration is very short, and therefore isn't an issue. We 1659 * explictly set the in-memory tail copy to 0 beforehand, so we 1660 * don't have to wait to be sure the DMA update has happened 1661 * (chip resets head/tail to 0 on transition to enable). 1662 */ 1663 if (rcd->rcvhdrtail_kvaddr) 1664 qib_clear_rcvhdrtail(rcd); 1665 1666 dd->f_rcvctrl(rcd->ppd, QIB_RCVCTRL_CTXT_ENB | QIB_RCVCTRL_TIDFLOW_ENB, 1667 rcd->ctxt); 1668 1669 /* Notify any waiting slaves */ 1670 if (rcd->subctxt_cnt) { 1671 clear_bit(QIB_CTXT_MASTER_UNINIT, &rcd->flag); 1672 wake_up(&rcd->wait); 1673 } 1674 return 0; 1675 1676bail_pio: 1677 qib_chg_pioavailkernel(dd, rcd->pio_base, rcd->piocnt, 1678 TXCHK_CHG_TYPE_KERN, rcd); 1679bail: 1680 return ret; 1681} 1682 1683/** 1684 * unlock_exptid - unlock any expected TID entries context still had in use 1685 * @rcd: ctxt 1686 * 1687 * We don't actually update the chip here, because we do a bulk update 1688 * below, using f_clear_tids. 1689 */ 1690static void unlock_expected_tids(struct qib_ctxtdata *rcd) 1691{ 1692 struct qib_devdata *dd = rcd->dd; 1693 int ctxt_tidbase = rcd->ctxt * dd->rcvtidcnt; 1694 int i, cnt = 0, maxtid = ctxt_tidbase + dd->rcvtidcnt; 1695 1696 for (i = ctxt_tidbase; i < maxtid; i++) { 1697 struct page *p = dd->pageshadow[i]; 1698 dma_addr_t phys; 1699 1700 if (!p) 1701 continue; 1702 1703 phys = dd->physshadow[i]; 1704 dd->physshadow[i] = dd->tidinvalid; 1705 dd->pageshadow[i] = NULL; 1706 pci_unmap_page(dd->pcidev, phys, PAGE_SIZE, 1707 PCI_DMA_FROMDEVICE); 1708 qib_release_user_pages(&p, 1); 1709 cnt++; 1710 } 1711} 1712 1713static int qib_close(struct inode *in, struct file *fp) 1714{ 1715 int ret = 0; 1716 struct qib_filedata *fd; 1717 struct qib_ctxtdata *rcd; 1718 struct qib_devdata *dd; 1719 unsigned long flags; 1720 unsigned ctxt; 1721 pid_t pid; 1722 1723 mutex_lock(&qib_mutex); 1724 1725 fd = (struct qib_filedata *) fp->private_data; 1726 fp->private_data = NULL; 1727 rcd = fd->rcd; 1728 if (!rcd) { 1729 mutex_unlock(&qib_mutex); 1730 goto bail; 1731 } 1732 1733 dd = rcd->dd; 1734 1735 /* ensure all pio buffer writes in progress are flushed */ 1736 qib_flush_wc(); 1737 1738 /* drain user sdma queue */ 1739 if (fd->pq) { 1740 qib_user_sdma_queue_drain(rcd->ppd, fd->pq); 1741 qib_user_sdma_queue_destroy(fd->pq); 1742 } 1743 1744 if (fd->rec_cpu_num != -1) 1745 __clear_bit(fd->rec_cpu_num, qib_cpulist); 1746 1747 if (--rcd->cnt) { 1748 rcd->active_slaves &= ~(1 << fd->subctxt); 1749 rcd->subpid[fd->subctxt] = 0; 1750 mutex_unlock(&qib_mutex); 1751 goto bail; 1752 } 1753 1754 /* early; no interrupt users after this */ 1755 spin_lock_irqsave(&dd->uctxt_lock, flags); 1756 ctxt = rcd->ctxt; 1757 dd->rcd[ctxt] = NULL; 1758 pid = rcd->pid; 1759 rcd->pid = 0; 1760 spin_unlock_irqrestore(&dd->uctxt_lock, flags); 1761 1762 if (rcd->rcvwait_to || rcd->piowait_to || 1763 rcd->rcvnowait || rcd->pionowait) { 1764 rcd->rcvwait_to = 0; 1765 rcd->piowait_to = 0; 1766 rcd->rcvnowait = 0; 1767 rcd->pionowait = 0; 1768 } 1769 if (rcd->flag) 1770 rcd->flag = 0; 1771 1772 if (dd->kregbase) { 1773 /* atomically clear receive enable ctxt and intr avail. */ 1774 dd->f_rcvctrl(rcd->ppd, QIB_RCVCTRL_CTXT_DIS | 1775 QIB_RCVCTRL_INTRAVAIL_DIS, ctxt); 1776 1777 /* clean up the pkeys for this ctxt user */ 1778 qib_clean_part_key(rcd, dd); 1779 qib_disarm_piobufs(dd, rcd->pio_base, rcd->piocnt); 1780 qib_chg_pioavailkernel(dd, rcd->pio_base, 1781 rcd->piocnt, TXCHK_CHG_TYPE_KERN, NULL); 1782 1783 dd->f_clear_tids(dd, rcd); 1784 1785 if (dd->pageshadow) 1786 unlock_expected_tids(rcd); 1787 qib_stats.sps_ctxts--; 1788 } 1789 1790 mutex_unlock(&qib_mutex); 1791 qib_free_ctxtdata(dd, rcd); /* after releasing the mutex */ 1792 1793bail: 1794 kfree(fd); 1795 return ret; 1796} 1797 1798static int qib_ctxt_info(struct file *fp, struct qib_ctxt_info __user *uinfo) 1799{ 1800 struct qib_ctxt_info info; 1801 int ret; 1802 size_t sz; 1803 struct qib_ctxtdata *rcd = ctxt_fp(fp); 1804 struct qib_filedata *fd; 1805 1806 fd = (struct qib_filedata *) fp->private_data; 1807 1808 info.num_active = qib_count_active_units(); 1809 info.unit = rcd->dd->unit; 1810 info.port = rcd->ppd->port; 1811 info.ctxt = rcd->ctxt; 1812 info.subctxt = subctxt_fp(fp); 1813 /* Number of user ctxts available for this device. */ 1814 info.num_ctxts = rcd->dd->cfgctxts - rcd->dd->first_user_ctxt; 1815 info.num_subctxts = rcd->subctxt_cnt; 1816 info.rec_cpu = fd->rec_cpu_num; 1817 sz = sizeof(info); 1818 1819 if (copy_to_user(uinfo, &info, sz)) { 1820 ret = -EFAULT; 1821 goto bail; 1822 } 1823 ret = 0; 1824 1825bail: 1826 return ret; 1827} 1828 1829static int qib_sdma_get_inflight(struct qib_user_sdma_queue *pq, 1830 u32 __user *inflightp) 1831{ 1832 const u32 val = qib_user_sdma_inflight_counter(pq); 1833 1834 if (put_user(val, inflightp)) 1835 return -EFAULT; 1836 1837 return 0; 1838} 1839 1840static int qib_sdma_get_complete(struct qib_pportdata *ppd, 1841 struct qib_user_sdma_queue *pq, 1842 u32 __user *completep) 1843{ 1844 u32 val; 1845 int err; 1846 1847 if (!pq) 1848 return -EINVAL; 1849 1850 err = qib_user_sdma_make_progress(ppd, pq); 1851 if (err < 0) 1852 return err; 1853 1854 val = qib_user_sdma_complete_counter(pq); 1855 if (put_user(val, completep)) 1856 return -EFAULT; 1857 1858 return 0; 1859} 1860 1861static int disarm_req_delay(struct qib_ctxtdata *rcd) 1862{ 1863 int ret = 0; 1864 1865 if (!usable(rcd->ppd)) { 1866 int i; 1867 /* 1868 * if link is down, or otherwise not usable, delay 1869 * the caller up to 30 seconds, so we don't thrash 1870 * in trying to get the chip back to ACTIVE, and 1871 * set flag so they make the call again. 1872 */ 1873 if (rcd->user_event_mask) { 1874 /* 1875 * subctxt_cnt is 0 if not shared, so do base 1876 * separately, first, then remaining subctxt, if any 1877 */ 1878 set_bit(_QIB_EVENT_DISARM_BUFS_BIT, 1879 &rcd->user_event_mask[0]); 1880 for (i = 1; i < rcd->subctxt_cnt; i++) 1881 set_bit(_QIB_EVENT_DISARM_BUFS_BIT, 1882 &rcd->user_event_mask[i]); 1883 } 1884 for (i = 0; !usable(rcd->ppd) && i < 300; i++) 1885 msleep(100); 1886 ret = -ENETDOWN; 1887 } 1888 return ret; 1889} 1890 1891/* 1892 * Find all user contexts in use, and set the specified bit in their 1893 * event mask. 1894 * See also find_ctxt() for a similar use, that is specific to send buffers. 1895 */ 1896int qib_set_uevent_bits(struct qib_pportdata *ppd, const int evtbit) 1897{ 1898 struct qib_ctxtdata *rcd; 1899 unsigned ctxt; 1900 int ret = 0; 1901 1902 spin_lock(&ppd->dd->uctxt_lock); 1903 for (ctxt = ppd->dd->first_user_ctxt; ctxt < ppd->dd->cfgctxts; 1904 ctxt++) { 1905 rcd = ppd->dd->rcd[ctxt]; 1906 if (!rcd) 1907 continue; 1908 if (rcd->user_event_mask) { 1909 int i; 1910 /* 1911 * subctxt_cnt is 0 if not shared, so do base 1912 * separately, first, then remaining subctxt, if any 1913 */ 1914 set_bit(evtbit, &rcd->user_event_mask[0]); 1915 for (i = 1; i < rcd->subctxt_cnt; i++) 1916 set_bit(evtbit, &rcd->user_event_mask[i]); 1917 } 1918 ret = 1; 1919 break; 1920 } 1921 spin_unlock(&ppd->dd->uctxt_lock); 1922 1923 return ret; 1924} 1925 1926/* 1927 * clear the event notifier events for this context. 1928 * For the DISARM_BUFS case, we also take action (this obsoletes 1929 * the older QIB_CMD_DISARM_BUFS, but we keep it for backwards 1930 * compatibility. 1931 * Other bits don't currently require actions, just atomically clear. 1932 * User process then performs actions appropriate to bit having been 1933 * set, if desired, and checks again in future. 1934 */ 1935static int qib_user_event_ack(struct qib_ctxtdata *rcd, int subctxt, 1936 unsigned long events) 1937{ 1938 int ret = 0, i; 1939 1940 for (i = 0; i <= _QIB_MAX_EVENT_BIT; i++) { 1941 if (!test_bit(i, &events)) 1942 continue; 1943 if (i == _QIB_EVENT_DISARM_BUFS_BIT) { 1944 (void)qib_disarm_piobufs_ifneeded(rcd); 1945 ret = disarm_req_delay(rcd); 1946 } else 1947 clear_bit(i, &rcd->user_event_mask[subctxt]); 1948 } 1949 return ret; 1950} 1951 1952static ssize_t qib_write(struct file *fp, const char __user *data, 1953 size_t count, loff_t *off) 1954{ 1955 const struct qib_cmd __user *ucmd; 1956 struct qib_ctxtdata *rcd; 1957 const void __user *src; 1958 size_t consumed, copy = 0; 1959 struct qib_cmd cmd; 1960 ssize_t ret = 0; 1961 void *dest; 1962 1963 if (count < sizeof(cmd.type)) { 1964 ret = -EINVAL; 1965 goto bail; 1966 } 1967 1968 ucmd = (const struct qib_cmd __user *) data; 1969 1970 if (copy_from_user(&cmd.type, &ucmd->type, sizeof(cmd.type))) { 1971 ret = -EFAULT; 1972 goto bail; 1973 } 1974 1975 consumed = sizeof(cmd.type); 1976 1977 switch (cmd.type) { 1978 case QIB_CMD_ASSIGN_CTXT: 1979 case QIB_CMD_USER_INIT: 1980 copy = sizeof(cmd.cmd.user_info); 1981 dest = &cmd.cmd.user_info; 1982 src = &ucmd->cmd.user_info; 1983 break; 1984 1985 case QIB_CMD_RECV_CTRL: 1986 copy = sizeof(cmd.cmd.recv_ctrl); 1987 dest = &cmd.cmd.recv_ctrl; 1988 src = &ucmd->cmd.recv_ctrl; 1989 break; 1990 1991 case QIB_CMD_CTXT_INFO: 1992 copy = sizeof(cmd.cmd.ctxt_info); 1993 dest = &cmd.cmd.ctxt_info; 1994 src = &ucmd->cmd.ctxt_info; 1995 break; 1996 1997 case QIB_CMD_TID_UPDATE: 1998 case QIB_CMD_TID_FREE: 1999 copy = sizeof(cmd.cmd.tid_info); 2000 dest = &cmd.cmd.tid_info; 2001 src = &ucmd->cmd.tid_info; 2002 break; 2003 2004 case QIB_CMD_SET_PART_KEY: 2005 copy = sizeof(cmd.cmd.part_key); 2006 dest = &cmd.cmd.part_key; 2007 src = &ucmd->cmd.part_key; 2008 break; 2009 2010 case QIB_CMD_DISARM_BUFS: 2011 case QIB_CMD_PIOAVAILUPD: /* force an update of PIOAvail reg */ 2012 copy = 0; 2013 src = NULL; 2014 dest = NULL; 2015 break; 2016 2017 case QIB_CMD_POLL_TYPE: 2018 copy = sizeof(cmd.cmd.poll_type); 2019 dest = &cmd.cmd.poll_type; 2020 src = &ucmd->cmd.poll_type; 2021 break; 2022 2023 case QIB_CMD_ARMLAUNCH_CTRL: 2024 copy = sizeof(cmd.cmd.armlaunch_ctrl); 2025 dest = &cmd.cmd.armlaunch_ctrl; 2026 src = &ucmd->cmd.armlaunch_ctrl; 2027 break; 2028 2029 case QIB_CMD_SDMA_INFLIGHT: 2030 copy = sizeof(cmd.cmd.sdma_inflight); 2031 dest = &cmd.cmd.sdma_inflight; 2032 src = &ucmd->cmd.sdma_inflight; 2033 break; 2034 2035 case QIB_CMD_SDMA_COMPLETE: 2036 copy = sizeof(cmd.cmd.sdma_complete); 2037 dest = &cmd.cmd.sdma_complete; 2038 src = &ucmd->cmd.sdma_complete; 2039 break; 2040 2041 case QIB_CMD_ACK_EVENT: 2042 copy = sizeof(cmd.cmd.event_mask); 2043 dest = &cmd.cmd.event_mask; 2044 src = &ucmd->cmd.event_mask; 2045 break; 2046 2047 default: 2048 ret = -EINVAL; 2049 goto bail; 2050 } 2051 2052 if (copy) { 2053 if ((count - consumed) < copy) { 2054 ret = -EINVAL; 2055 goto bail; 2056 } 2057 if (copy_from_user(dest, src, copy)) { 2058 ret = -EFAULT; 2059 goto bail; 2060 } 2061 consumed += copy; 2062 } 2063 2064 rcd = ctxt_fp(fp); 2065 if (!rcd && cmd.type != QIB_CMD_ASSIGN_CTXT) { 2066 ret = -EINVAL; 2067 goto bail; 2068 } 2069 2070 switch (cmd.type) { 2071 case QIB_CMD_ASSIGN_CTXT: 2072 ret = qib_assign_ctxt(fp, &cmd.cmd.user_info); 2073 if (ret) 2074 goto bail; 2075 break; 2076 2077 case QIB_CMD_USER_INIT: 2078 ret = qib_do_user_init(fp, &cmd.cmd.user_info); 2079 if (ret) 2080 goto bail; 2081 ret = qib_get_base_info(fp, (void __user *) (unsigned long) 2082 cmd.cmd.user_info.spu_base_info, 2083 cmd.cmd.user_info.spu_base_info_size); 2084 break; 2085 2086 case QIB_CMD_RECV_CTRL: 2087 ret = qib_manage_rcvq(rcd, subctxt_fp(fp), cmd.cmd.recv_ctrl); 2088 break; 2089 2090 case QIB_CMD_CTXT_INFO: 2091 ret = qib_ctxt_info(fp, (struct qib_ctxt_info __user *) 2092 (unsigned long) cmd.cmd.ctxt_info); 2093 break; 2094 2095 case QIB_CMD_TID_UPDATE: 2096 ret = qib_tid_update(rcd, fp, &cmd.cmd.tid_info); 2097 break; 2098 2099 case QIB_CMD_TID_FREE: 2100 ret = qib_tid_free(rcd, subctxt_fp(fp), &cmd.cmd.tid_info); 2101 break; 2102 2103 case QIB_CMD_SET_PART_KEY: 2104 ret = qib_set_part_key(rcd, cmd.cmd.part_key); 2105 break; 2106 2107 case QIB_CMD_DISARM_BUFS: 2108 (void)qib_disarm_piobufs_ifneeded(rcd); 2109 ret = disarm_req_delay(rcd); 2110 break; 2111 2112 case QIB_CMD_PIOAVAILUPD: 2113 qib_force_pio_avail_update(rcd->dd); 2114 break; 2115 2116 case QIB_CMD_POLL_TYPE: 2117 rcd->poll_type = cmd.cmd.poll_type; 2118 break; 2119 2120 case QIB_CMD_ARMLAUNCH_CTRL: 2121 rcd->dd->f_set_armlaunch(rcd->dd, cmd.cmd.armlaunch_ctrl); 2122 break; 2123 2124 case QIB_CMD_SDMA_INFLIGHT: 2125 ret = qib_sdma_get_inflight(user_sdma_queue_fp(fp), 2126 (u32 __user *) (unsigned long) 2127 cmd.cmd.sdma_inflight); 2128 break; 2129 2130 case QIB_CMD_SDMA_COMPLETE: 2131 ret = qib_sdma_get_complete(rcd->ppd, 2132 user_sdma_queue_fp(fp), 2133 (u32 __user *) (unsigned long) 2134 cmd.cmd.sdma_complete); 2135 break; 2136 2137 case QIB_CMD_ACK_EVENT: 2138 ret = qib_user_event_ack(rcd, subctxt_fp(fp), 2139 cmd.cmd.event_mask); 2140 break; 2141 } 2142 2143 if (ret >= 0) 2144 ret = consumed; 2145 2146bail: 2147 return ret; 2148} 2149 2150static ssize_t qib_aio_write(struct kiocb *iocb, const struct iovec *iov, 2151 unsigned long dim, loff_t off) 2152{ 2153 struct qib_filedata *fp = iocb->ki_filp->private_data; 2154 struct qib_ctxtdata *rcd = ctxt_fp(iocb->ki_filp); 2155 struct qib_user_sdma_queue *pq = fp->pq; 2156 2157 if (!dim || !pq) 2158 return -EINVAL; 2159 2160 return qib_user_sdma_writev(rcd, pq, iov, dim); 2161} 2162 2163static struct class *qib_class; 2164static dev_t qib_dev; 2165 2166int qib_cdev_init(int minor, const char *name, 2167 const struct file_operations *fops, 2168 struct cdev **cdevp, struct device **devp) 2169{ 2170 const dev_t dev = MKDEV(MAJOR(qib_dev), minor); 2171 struct cdev *cdev; 2172 struct device *device = NULL; 2173 int ret; 2174 2175 cdev = cdev_alloc(); 2176 if (!cdev) { 2177 printk(KERN_ERR QIB_DRV_NAME 2178 ": Could not allocate cdev for minor %d, %s\n", 2179 minor, name); 2180 ret = -ENOMEM; 2181 goto done; 2182 } 2183 2184 cdev->owner = THIS_MODULE; 2185 cdev->ops = fops; 2186 kobject_set_name(&cdev->kobj, name); 2187 2188 ret = cdev_add(cdev, dev, 1); 2189 if (ret < 0) { 2190 printk(KERN_ERR QIB_DRV_NAME 2191 ": Could not add cdev for minor %d, %s (err %d)\n", 2192 minor, name, -ret); 2193 goto err_cdev; 2194 } 2195 2196 device = device_create(qib_class, NULL, dev, NULL, name); 2197 if (!IS_ERR(device)) 2198 goto done; 2199 ret = PTR_ERR(device); 2200 device = NULL; 2201 printk(KERN_ERR QIB_DRV_NAME ": Could not create " 2202 "device for minor %d, %s (err %d)\n", 2203 minor, name, -ret); 2204err_cdev: 2205 cdev_del(cdev); 2206 cdev = NULL; 2207done: 2208 *cdevp = cdev; 2209 *devp = device; 2210 return ret; 2211} 2212 2213void qib_cdev_cleanup(struct cdev **cdevp, struct device **devp) 2214{ 2215 struct device *device = *devp; 2216 2217 if (device) { 2218 device_unregister(device); 2219 *devp = NULL; 2220 } 2221 2222 if (*cdevp) { 2223 cdev_del(*cdevp); 2224 *cdevp = NULL; 2225 } 2226} 2227 2228static struct cdev *wildcard_cdev; 2229static struct device *wildcard_device; 2230 2231int __init qib_dev_init(void) 2232{ 2233 int ret; 2234 2235 ret = alloc_chrdev_region(&qib_dev, 0, QIB_NMINORS, QIB_DRV_NAME); 2236 if (ret < 0) { 2237 printk(KERN_ERR QIB_DRV_NAME ": Could not allocate " 2238 "chrdev region (err %d)\n", -ret); 2239 goto done; 2240 } 2241 2242 qib_class = class_create(THIS_MODULE, "ipath"); 2243 if (IS_ERR(qib_class)) { 2244 ret = PTR_ERR(qib_class); 2245 printk(KERN_ERR QIB_DRV_NAME ": Could not create " 2246 "device class (err %d)\n", -ret); 2247 unregister_chrdev_region(qib_dev, QIB_NMINORS); 2248 } 2249 2250done: 2251 return ret; 2252} 2253 2254void qib_dev_cleanup(void) 2255{ 2256 if (qib_class) { 2257 class_destroy(qib_class); 2258 qib_class = NULL; 2259 } 2260 2261 unregister_chrdev_region(qib_dev, QIB_NMINORS); 2262} 2263 2264static atomic_t user_count = ATOMIC_INIT(0); 2265 2266static void qib_user_remove(struct qib_devdata *dd) 2267{ 2268 if (atomic_dec_return(&user_count) == 0) 2269 qib_cdev_cleanup(&wildcard_cdev, &wildcard_device); 2270 2271 qib_cdev_cleanup(&dd->user_cdev, &dd->user_device); 2272} 2273 2274static int qib_user_add(struct qib_devdata *dd) 2275{ 2276 char name[10]; 2277 int ret; 2278 2279 if (atomic_inc_return(&user_count) == 1) { 2280 ret = qib_cdev_init(0, "ipath", &qib_file_ops, 2281 &wildcard_cdev, &wildcard_device); 2282 if (ret) 2283 goto done; 2284 } 2285 2286 snprintf(name, sizeof(name), "ipath%d", dd->unit); 2287 ret = qib_cdev_init(dd->unit + 1, name, &qib_file_ops, 2288 &dd->user_cdev, &dd->user_device); 2289 if (ret) 2290 qib_user_remove(dd); 2291done: 2292 return ret; 2293} 2294 2295/* 2296 * Create per-unit files in /dev 2297 */ 2298int qib_device_create(struct qib_devdata *dd) 2299{ 2300 int r, ret; 2301 2302 r = qib_user_add(dd); 2303 ret = qib_diag_add(dd); 2304 if (r && !ret) 2305 ret = r; 2306 return ret; 2307} 2308 2309/* 2310 * Remove per-unit files in /dev 2311 * void, core kernel returns no errors for this stuff 2312 */ 2313void qib_device_remove(struct qib_devdata *dd) 2314{ 2315 qib_user_remove(dd); 2316 qib_diag_remove(dd); 2317} 2318