1/*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice unmodified, this list of conditions, and the following 13 * disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30#include <sys/param.h> 31#include <sys/systm.h> 32#include <sys/malloc.h> 33#include <sys/kernel.h> 34#include <sys/sysctl.h> 35#include <sys/proc.h> 36#include <sys/sglist.h> 37#include <sys/sleepqueue.h> 38#include <sys/lock.h> 39#include <sys/mutex.h> 40#include <sys/bus.h> 41#include <sys/fcntl.h> 42#include <sys/file.h> 43#include <sys/filio.h> 44#include <sys/rwlock.h> 45 46#include <vm/vm.h> 47#include <vm/pmap.h> 48 49#include <machine/stdarg.h> 50#include <machine/pmap.h> 51 52#include <linux/kobject.h> 53#include <linux/device.h> 54#include <linux/slab.h> 55#include <linux/module.h> 56#include <linux/cdev.h> 57#include <linux/file.h> 58#include <linux/sysfs.h> 59#include <linux/mm.h> 60#include <linux/io.h> 61#include <linux/vmalloc.h> 62#include <linux/timer.h> 63#include <linux/netdevice.h> 64 65#include <vm/vm_pager.h> 66 67#include <linux/workqueue.h> 68 69MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat"); 70 71#include <linux/rbtree.h> 72/* Undo Linux compat changes. */ 73#undef RB_ROOT 74#undef file 75#undef cdev 76#define RB_ROOT(head) (head)->rbh_root 77 78struct kobject class_root; 79struct device linux_rootdev; 80struct class miscclass; 81struct list_head pci_drivers; 82struct list_head pci_devices; 83struct net init_net; 84spinlock_t pci_lock; 85 86unsigned long linux_timer_hz_mask; 87 88int 89panic_cmp(struct rb_node *one, struct rb_node *two) 90{ 91 panic("no cmp"); 92} 93 94RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); 95 96int 97kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args) 98{ 99 va_list tmp_va; 100 int len; 101 char *old; 102 char *name; 103 char dummy; 104 105 old = kobj->name; 106 107 if (old && fmt == NULL) 108 return (0); 109 110 /* compute length of string */ 111 va_copy(tmp_va, args); 112 len = vsnprintf(&dummy, 0, fmt, tmp_va); 113 va_end(tmp_va); 114 115 /* account for zero termination */ 116 len++; 117 118 /* check for error */ 119 if (len < 1) 120 return (-EINVAL); 121 122 /* allocate memory for string */ 123 name = kzalloc(len, GFP_KERNEL); 124 if (name == NULL) 125 return (-ENOMEM); 126 vsnprintf(name, len, fmt, args); 127 kobj->name = name; 128 129 /* free old string */ 130 kfree(old); 131 132 /* filter new string */ 133 for (; *name != '\0'; name++) 134 if (*name == '/') 135 *name = '!'; 136 return (0); 137} 138 139int 140kobject_set_name(struct kobject *kobj, const char *fmt, ...) 141{ 142 va_list args; 143 int error; 144 145 va_start(args, fmt); 146 error = kobject_set_name_vargs(kobj, fmt, args); 147 va_end(args); 148 149 return (error); 150} 151 152static inline int 153kobject_add_complete(struct kobject *kobj, struct kobject *parent) 154{ 155 struct kobj_type *t; 156 int error; 157 158 kobj->parent = kobject_get(parent); 159 error = sysfs_create_dir(kobj); 160 if (error == 0 && kobj->ktype && kobj->ktype->default_attrs) { 161 struct attribute **attr; 162 t = kobj->ktype; 163 164 for (attr = t->default_attrs; *attr != NULL; attr++) { 165 error = sysfs_create_file(kobj, *attr); 166 if (error) 167 break; 168 } 169 if (error) 170 sysfs_remove_dir(kobj); 171 172 } 173 return (error); 174} 175 176int 177kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...) 178{ 179 va_list args; 180 int error; 181 182 va_start(args, fmt); 183 error = kobject_set_name_vargs(kobj, fmt, args); 184 va_end(args); 185 if (error) 186 return (error); 187 188 return kobject_add_complete(kobj, parent); 189} 190 191void 192kobject_release(struct kref *kref) 193{ 194 struct kobject *kobj; 195 char *name; 196 197 kobj = container_of(kref, struct kobject, kref); 198 sysfs_remove_dir(kobj); 199 if (kobj->parent) 200 kobject_put(kobj->parent); 201 kobj->parent = NULL; 202 name = kobj->name; 203 if (kobj->ktype && kobj->ktype->release) 204 kobj->ktype->release(kobj); 205 kfree(name); 206} 207 208static void 209kobject_kfree(struct kobject *kobj) 210{ 211 kfree(kobj); 212} 213 214static void 215kobject_kfree_name(struct kobject *kobj) 216{ 217 if (kobj) { 218 kfree(kobj->name); 219 } 220} 221 222struct kobj_type kfree_type = { .release = kobject_kfree }; 223 224static void 225dev_release(struct device *dev) 226{ 227 pr_debug("dev_release: %s\n", dev_name(dev)); 228 kfree(dev); 229} 230 231struct device * 232device_create(struct class *class, struct device *parent, dev_t devt, 233 void *drvdata, const char *fmt, ...) 234{ 235 struct device *dev; 236 va_list args; 237 238 dev = kzalloc(sizeof(*dev), M_WAITOK); 239 dev->parent = parent; 240 dev->class = class; 241 dev->devt = devt; 242 dev->driver_data = drvdata; 243 dev->release = dev_release; 244 va_start(args, fmt); 245 kobject_set_name_vargs(&dev->kobj, fmt, args); 246 va_end(args); 247 device_register(dev); 248 249 return (dev); 250} 251 252int 253kobject_init_and_add(struct kobject *kobj, struct kobj_type *ktype, 254 struct kobject *parent, const char *fmt, ...) 255{ 256 va_list args; 257 int error; 258 259 kobject_init(kobj, ktype); 260 kobj->ktype = ktype; 261 kobj->parent = parent; 262 kobj->name = NULL; 263 264 va_start(args, fmt); 265 error = kobject_set_name_vargs(kobj, fmt, args); 266 va_end(args); 267 if (error) 268 return (error); 269 return kobject_add_complete(kobj, parent); 270} 271 272static void 273linux_file_dtor(void *cdp) 274{ 275 struct linux_file *filp; 276 277 filp = cdp; 278 filp->f_op->release(filp->f_vnode, filp); 279 vdrop(filp->f_vnode); 280 kfree(filp); 281} 282 283static int 284linux_dev_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 285{ 286 struct linux_cdev *ldev; 287 struct linux_file *filp; 288 struct file *file; 289 int error; 290 291 file = curthread->td_fpop; 292 ldev = dev->si_drv1; 293 if (ldev == NULL) 294 return (ENODEV); 295 filp = kzalloc(sizeof(*filp), GFP_KERNEL); 296 filp->f_dentry = &filp->f_dentry_store; 297 filp->f_op = ldev->ops; 298 filp->f_flags = file->f_flag; 299 vhold(file->f_vnode); 300 filp->f_vnode = file->f_vnode; 301 if (filp->f_op->open) { 302 error = -filp->f_op->open(file->f_vnode, filp); 303 if (error) { 304 kfree(filp); 305 return (error); 306 } 307 } 308 error = devfs_set_cdevpriv(filp, linux_file_dtor); 309 if (error) { 310 filp->f_op->release(file->f_vnode, filp); 311 kfree(filp); 312 return (error); 313 } 314 315 return 0; 316} 317 318static int 319linux_dev_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 320{ 321 struct linux_cdev *ldev; 322 struct linux_file *filp; 323 struct file *file; 324 int error; 325 326 file = curthread->td_fpop; 327 ldev = dev->si_drv1; 328 if (ldev == NULL) 329 return (0); 330 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 331 return (error); 332 filp->f_flags = file->f_flag; 333 devfs_clear_cdevpriv(); 334 335 336 return (0); 337} 338 339static int 340linux_dev_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, 341 struct thread *td) 342{ 343 struct linux_cdev *ldev; 344 struct linux_file *filp; 345 struct file *file; 346 int error; 347 348 file = curthread->td_fpop; 349 ldev = dev->si_drv1; 350 if (ldev == NULL) 351 return (0); 352 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 353 return (error); 354 filp->f_flags = file->f_flag; 355 /* 356 * Linux does not have a generic ioctl copyin/copyout layer. All 357 * linux ioctls must be converted to void ioctls which pass a 358 * pointer to the address of the data. We want the actual user 359 * address so we dereference here. 360 */ 361 data = *(void **)data; 362 if (filp->f_op->unlocked_ioctl) 363 error = -filp->f_op->unlocked_ioctl(filp, cmd, (u_long)data); 364 else 365 error = ENOTTY; 366 367 return (error); 368} 369 370static int 371linux_dev_read(struct cdev *dev, struct uio *uio, int ioflag) 372{ 373 struct linux_cdev *ldev; 374 struct linux_file *filp; 375 struct file *file; 376 ssize_t bytes; 377 int error; 378 379 file = curthread->td_fpop; 380 ldev = dev->si_drv1; 381 if (ldev == NULL) 382 return (0); 383 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 384 return (error); 385 filp->f_flags = file->f_flag; 386 if (uio->uio_iovcnt != 1) 387 panic("linux_dev_read: uio %p iovcnt %d", 388 uio, uio->uio_iovcnt); 389 if (filp->f_op->read) { 390 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, 391 uio->uio_iov->iov_len, &uio->uio_offset); 392 if (bytes >= 0) { 393 uio->uio_iov->iov_base += bytes; 394 uio->uio_iov->iov_len -= bytes; 395 uio->uio_resid -= bytes; 396 } else 397 error = -bytes; 398 } else 399 error = ENXIO; 400 401 return (error); 402} 403 404static int 405linux_dev_write(struct cdev *dev, struct uio *uio, int ioflag) 406{ 407 struct linux_cdev *ldev; 408 struct linux_file *filp; 409 struct file *file; 410 ssize_t bytes; 411 int error; 412 413 file = curthread->td_fpop; 414 ldev = dev->si_drv1; 415 if (ldev == NULL) 416 return (0); 417 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 418 return (error); 419 filp->f_flags = file->f_flag; 420 if (uio->uio_iovcnt != 1) 421 panic("linux_dev_write: uio %p iovcnt %d", 422 uio, uio->uio_iovcnt); 423 if (filp->f_op->write) { 424 bytes = filp->f_op->write(filp, uio->uio_iov->iov_base, 425 uio->uio_iov->iov_len, &uio->uio_offset); 426 if (bytes >= 0) { 427 uio->uio_iov->iov_base += bytes; 428 uio->uio_iov->iov_len -= bytes; 429 uio->uio_resid -= bytes; 430 } else 431 error = -bytes; 432 } else 433 error = ENXIO; 434 435 return (error); 436} 437 438static int 439linux_dev_poll(struct cdev *dev, int events, struct thread *td) 440{ 441 struct linux_cdev *ldev; 442 struct linux_file *filp; 443 struct file *file; 444 int revents; 445 int error; 446 447 file = curthread->td_fpop; 448 ldev = dev->si_drv1; 449 if (ldev == NULL) 450 return (0); 451 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 452 return (error); 453 filp->f_flags = file->f_flag; 454 if (filp->f_op->poll) 455 revents = filp->f_op->poll(filp, NULL) & events; 456 else 457 revents = 0; 458 459 return (revents); 460} 461 462static int 463linux_dev_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 464 vm_size_t size, struct vm_object **object, int nprot) 465{ 466 struct linux_cdev *ldev; 467 struct linux_file *filp; 468 struct file *file; 469 struct vm_area_struct vma; 470 int error; 471 472 file = curthread->td_fpop; 473 ldev = dev->si_drv1; 474 if (ldev == NULL) 475 return (ENODEV); 476 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 477 return (error); 478 filp->f_flags = file->f_flag; 479 vma.vm_start = 0; 480 vma.vm_end = size; 481 vma.vm_pgoff = *offset / PAGE_SIZE; 482 vma.vm_pfn = 0; 483 vma.vm_page_prot = VM_MEMATTR_DEFAULT; 484 if (filp->f_op->mmap) { 485 error = -filp->f_op->mmap(filp, &vma); 486 if (error == 0) { 487 struct sglist *sg; 488 489 sg = sglist_alloc(1, M_WAITOK); 490 sglist_append_phys(sg, 491 (vm_paddr_t)vma.vm_pfn << PAGE_SHIFT, vma.vm_len); 492 *object = vm_pager_allocate(OBJT_SG, sg, vma.vm_len, 493 nprot, 0, curthread->td_ucred); 494 if (*object == NULL) { 495 sglist_free(sg); 496 return (EINVAL); 497 } 498 *offset = 0; 499 if (vma.vm_page_prot != VM_MEMATTR_DEFAULT) { 500 VM_OBJECT_WLOCK(*object); 501 vm_object_set_memattr(*object, 502 vma.vm_page_prot); 503 VM_OBJECT_WUNLOCK(*object); 504 } 505 } 506 } else 507 error = ENODEV; 508 509 return (error); 510} 511 512struct cdevsw linuxcdevsw = { 513 .d_version = D_VERSION, 514 .d_flags = D_TRACKCLOSE, 515 .d_open = linux_dev_open, 516 .d_close = linux_dev_close, 517 .d_read = linux_dev_read, 518 .d_write = linux_dev_write, 519 .d_ioctl = linux_dev_ioctl, 520 .d_mmap_single = linux_dev_mmap_single, 521 .d_poll = linux_dev_poll, 522}; 523 524static int 525linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred, 526 int flags, struct thread *td) 527{ 528 struct linux_file *filp; 529 ssize_t bytes; 530 int error; 531 532 error = 0; 533 filp = (struct linux_file *)file->f_data; 534 filp->f_flags = file->f_flag; 535 if (uio->uio_iovcnt != 1) 536 panic("linux_file_read: uio %p iovcnt %d", 537 uio, uio->uio_iovcnt); 538 if (filp->f_op->read) { 539 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, 540 uio->uio_iov->iov_len, &uio->uio_offset); 541 if (bytes >= 0) { 542 uio->uio_iov->iov_base += bytes; 543 uio->uio_iov->iov_len -= bytes; 544 uio->uio_resid -= bytes; 545 } else 546 error = -bytes; 547 } else 548 error = ENXIO; 549 550 return (error); 551} 552 553static int 554linux_file_poll(struct file *file, int events, struct ucred *active_cred, 555 struct thread *td) 556{ 557 struct linux_file *filp; 558 int revents; 559 560 filp = (struct linux_file *)file->f_data; 561 filp->f_flags = file->f_flag; 562 if (filp->f_op->poll) 563 revents = filp->f_op->poll(filp, NULL) & events; 564 else 565 revents = 0; 566 567 return (0); 568} 569 570static int 571linux_file_close(struct file *file, struct thread *td) 572{ 573 struct linux_file *filp; 574 int error; 575 576 filp = (struct linux_file *)file->f_data; 577 filp->f_flags = file->f_flag; 578 error = -filp->f_op->release(NULL, filp); 579 funsetown(&filp->f_sigio); 580 kfree(filp); 581 582 return (error); 583} 584 585static int 586linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred, 587 struct thread *td) 588{ 589 struct linux_file *filp; 590 int error; 591 592 filp = (struct linux_file *)fp->f_data; 593 filp->f_flags = fp->f_flag; 594 error = 0; 595 596 switch (cmd) { 597 case FIONBIO: 598 break; 599 case FIOASYNC: 600 if (filp->f_op->fasync == NULL) 601 break; 602 error = filp->f_op->fasync(0, filp, fp->f_flag & FASYNC); 603 break; 604 case FIOSETOWN: 605 error = fsetown(*(int *)data, &filp->f_sigio); 606 if (error == 0) 607 error = filp->f_op->fasync(0, filp, 608 fp->f_flag & FASYNC); 609 break; 610 case FIOGETOWN: 611 *(int *)data = fgetown(&filp->f_sigio); 612 break; 613 default: 614 error = ENOTTY; 615 break; 616 } 617 return (error); 618} 619 620struct fileops linuxfileops = { 621 .fo_read = linux_file_read, 622 .fo_poll = linux_file_poll, 623 .fo_close = linux_file_close, 624 .fo_ioctl = linux_file_ioctl, 625 .fo_chmod = invfo_chmod, 626 .fo_chown = invfo_chown, 627 .fo_sendfile = invfo_sendfile, 628}; 629 630/* 631 * Hash of vmmap addresses. This is infrequently accessed and does not 632 * need to be particularly large. This is done because we must store the 633 * caller's idea of the map size to properly unmap. 634 */ 635struct vmmap { 636 LIST_ENTRY(vmmap) vm_next; 637 void *vm_addr; 638 unsigned long vm_size; 639}; 640 641struct vmmaphd { 642 struct vmmap *lh_first; 643}; 644#define VMMAP_HASH_SIZE 64 645#define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1) 646#define VM_HASH(addr) ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK 647static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE]; 648static struct mtx vmmaplock; 649 650static void 651vmmap_add(void *addr, unsigned long size) 652{ 653 struct vmmap *vmmap; 654 655 vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL); 656 mtx_lock(&vmmaplock); 657 vmmap->vm_size = size; 658 vmmap->vm_addr = addr; 659 LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next); 660 mtx_unlock(&vmmaplock); 661} 662 663static struct vmmap * 664vmmap_remove(void *addr) 665{ 666 struct vmmap *vmmap; 667 668 mtx_lock(&vmmaplock); 669 LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next) 670 if (vmmap->vm_addr == addr) 671 break; 672 if (vmmap) 673 LIST_REMOVE(vmmap, vm_next); 674 mtx_unlock(&vmmaplock); 675 676 return (vmmap); 677} 678 679void * 680_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr) 681{ 682 void *addr; 683 684 addr = pmap_mapdev_attr(phys_addr, size, attr); 685 if (addr == NULL) 686 return (NULL); 687 vmmap_add(addr, size); 688 689 return (addr); 690} 691 692void 693iounmap(void *addr) 694{ 695 struct vmmap *vmmap; 696 697 vmmap = vmmap_remove(addr); 698 if (vmmap == NULL) 699 return; 700 pmap_unmapdev((vm_offset_t)addr, vmmap->vm_size); 701 kfree(vmmap); 702} 703 704 705void * 706vmap(struct page **pages, unsigned int count, unsigned long flags, int prot) 707{ 708 vm_offset_t off; 709 size_t size; 710 711 size = count * PAGE_SIZE; 712 off = kva_alloc(size); 713 if (off == 0) 714 return (NULL); 715 vmmap_add((void *)off, size); 716 pmap_qenter(off, pages, count); 717 718 return ((void *)off); 719} 720 721void 722vunmap(void *addr) 723{ 724 struct vmmap *vmmap; 725 726 vmmap = vmmap_remove(addr); 727 if (vmmap == NULL) 728 return; 729 pmap_qremove((vm_offset_t)addr, vmmap->vm_size / PAGE_SIZE); 730 kva_free((vm_offset_t)addr, vmmap->vm_size); 731 kfree(vmmap); 732} 733 734char * 735kvasprintf(gfp_t gfp, const char *fmt, va_list ap) 736{ 737 unsigned int len; 738 char *p; 739 va_list aq; 740 741 va_copy(aq, ap); 742 len = vsnprintf(NULL, 0, fmt, aq); 743 va_end(aq); 744 745 p = kmalloc(len + 1, gfp); 746 if (p != NULL) 747 vsnprintf(p, len + 1, fmt, ap); 748 749 return (p); 750} 751 752char * 753kasprintf(gfp_t gfp, const char *fmt, ...) 754{ 755 va_list ap; 756 char *p; 757 758 va_start(ap, fmt); 759 p = kvasprintf(gfp, fmt, ap); 760 va_end(ap); 761 762 return (p); 763} 764 765static int 766linux_timer_jiffies_until(unsigned long expires) 767{ 768 int delta = expires - jiffies; 769 /* guard against already expired values */ 770 if (delta < 1) 771 delta = 1; 772 return (delta); 773} 774 775static void 776linux_timer_callback_wrapper(void *context) 777{ 778 struct timer_list *timer; 779 780 timer = context; 781 timer->function(timer->data); 782} 783 784void 785mod_timer(struct timer_list *timer, unsigned long expires) 786{ 787 788 timer->expires = expires; 789 callout_reset(&timer->timer_callout, 790 linux_timer_jiffies_until(expires), 791 &linux_timer_callback_wrapper, timer); 792} 793 794void 795add_timer(struct timer_list *timer) 796{ 797 798 callout_reset(&timer->timer_callout, 799 linux_timer_jiffies_until(timer->expires), 800 &linux_timer_callback_wrapper, timer); 801} 802 803static void 804linux_timer_init(void *arg) 805{ 806 807 /* 808 * Compute an internal HZ value which can divide 2**32 to 809 * avoid timer rounding problems when the tick value wraps 810 * around 2**32: 811 */ 812 linux_timer_hz_mask = 1; 813 while (linux_timer_hz_mask < (unsigned long)hz) 814 linux_timer_hz_mask *= 2; 815 linux_timer_hz_mask--; 816} 817SYSINIT(linux_timer, SI_SUB_DRIVERS, SI_ORDER_FIRST, linux_timer_init, NULL); 818 819void 820linux_complete_common(struct completion *c, int all) 821{ 822 int wakeup_swapper; 823 824 sleepq_lock(c); 825 c->done++; 826 if (all) 827 wakeup_swapper = sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); 828 else 829 wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); 830 sleepq_release(c); 831 if (wakeup_swapper) 832 kick_proc0(); 833} 834 835/* 836 * Indefinite wait for done != 0 with or without signals. 837 */ 838long 839linux_wait_for_common(struct completion *c, int flags) 840{ 841 842 if (flags != 0) 843 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 844 else 845 flags = SLEEPQ_SLEEP; 846 for (;;) { 847 sleepq_lock(c); 848 if (c->done) 849 break; 850 sleepq_add(c, NULL, "completion", flags, 0); 851 if (flags & SLEEPQ_INTERRUPTIBLE) { 852 if (sleepq_wait_sig(c, 0) != 0) 853 return (-ERESTARTSYS); 854 } else 855 sleepq_wait(c, 0); 856 } 857 c->done--; 858 sleepq_release(c); 859 860 return (0); 861} 862 863/* 864 * Time limited wait for done != 0 with or without signals. 865 */ 866long 867linux_wait_for_timeout_common(struct completion *c, long timeout, int flags) 868{ 869 long end = jiffies + timeout; 870 871 if (flags != 0) 872 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 873 else 874 flags = SLEEPQ_SLEEP; 875 for (;;) { 876 int ret; 877 878 sleepq_lock(c); 879 if (c->done) 880 break; 881 sleepq_add(c, NULL, "completion", flags, 0); 882 sleepq_set_timeout(c, linux_timer_jiffies_until(end)); 883 if (flags & SLEEPQ_INTERRUPTIBLE) 884 ret = sleepq_timedwait_sig(c, 0); 885 else 886 ret = sleepq_timedwait(c, 0); 887 if (ret != 0) { 888 /* check for timeout or signal */ 889 if (ret == EWOULDBLOCK) 890 return (0); 891 else 892 return (-ERESTARTSYS); 893 } 894 } 895 c->done--; 896 sleepq_release(c); 897 898 /* return how many jiffies are left */ 899 return (linux_timer_jiffies_until(end)); 900} 901 902int 903linux_try_wait_for_completion(struct completion *c) 904{ 905 int isdone; 906 907 isdone = 1; 908 sleepq_lock(c); 909 if (c->done) 910 c->done--; 911 else 912 isdone = 0; 913 sleepq_release(c); 914 return (isdone); 915} 916 917int 918linux_completion_done(struct completion *c) 919{ 920 int isdone; 921 922 isdone = 1; 923 sleepq_lock(c); 924 if (c->done == 0) 925 isdone = 0; 926 sleepq_release(c); 927 return (isdone); 928} 929 930void 931linux_delayed_work_fn(void *arg) 932{ 933 struct delayed_work *work; 934 935 work = arg; 936 taskqueue_enqueue(work->work.taskqueue, &work->work.work_task); 937} 938 939void 940linux_work_fn(void *context, int pending) 941{ 942 struct work_struct *work; 943 944 work = context; 945 work->fn(work); 946} 947 948void 949linux_flush_fn(void *context, int pending) 950{ 951} 952 953struct workqueue_struct * 954linux_create_workqueue_common(const char *name, int cpus) 955{ 956 struct workqueue_struct *wq; 957 958 wq = kmalloc(sizeof(*wq), M_WAITOK); 959 wq->taskqueue = taskqueue_create(name, M_WAITOK, 960 taskqueue_thread_enqueue, &wq->taskqueue); 961 atomic_set(&wq->draining, 0); 962 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); 963 964 return (wq); 965} 966 967void 968destroy_workqueue(struct workqueue_struct *wq) 969{ 970 taskqueue_free(wq->taskqueue); 971 kfree(wq); 972} 973 974static void 975linux_compat_init(void *arg) 976{ 977 struct sysctl_oid *rootoid; 978 int i; 979 980 rootoid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(), 981 OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); 982 kobject_init(&class_root, &class_ktype); 983 kobject_set_name(&class_root, "class"); 984 class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), 985 OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class"); 986 kobject_init(&linux_rootdev.kobj, &dev_ktype); 987 kobject_set_name(&linux_rootdev.kobj, "device"); 988 linux_rootdev.kobj.oidp = SYSCTL_ADD_NODE(NULL, 989 SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", CTLFLAG_RD, NULL, 990 "device"); 991 linux_rootdev.bsddev = root_bus; 992 miscclass.name = "misc"; 993 class_register(&miscclass); 994 INIT_LIST_HEAD(&pci_drivers); 995 INIT_LIST_HEAD(&pci_devices); 996 spin_lock_init(&pci_lock); 997 mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF); 998 for (i = 0; i < VMMAP_HASH_SIZE; i++) 999 LIST_INIT(&vmmaphead[i]); 1000} 1001 1002SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); 1003 1004static void 1005linux_compat_uninit(void *arg) 1006{ 1007 kobject_kfree_name(&class_root); 1008 kobject_kfree_name(&linux_rootdev.kobj); 1009 kobject_kfree_name(&miscclass.kobj); 1010} 1011SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL); 1012