sys_pipe.c revision 118764
1/* 2 * Copyright (c) 1996 John S. Dyson 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice immediately at the beginning of the file, without modification, 10 * this list of conditions, and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. Absolutely no warranty of function or purpose is made by the author 15 * John S. Dyson. 16 * 4. Modifications may be freely made to this file if the above conditions 17 * are met. 18 */ 19 20/* 21 * This file contains a high-performance replacement for the socket-based 22 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 23 * all features of sockets, but does do everything that pipes normally 24 * do. 25 */ 26 27/* 28 * This code has two modes of operation, a small write mode and a large 29 * write mode. The small write mode acts like conventional pipes with 30 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 31 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 32 * and PIPE_SIZE in size, it is fully mapped and wired into the kernel, and 33 * the receiving process can copy it directly from the pages in the sending 34 * process. 35 * 36 * If the sending process receives a signal, it is possible that it will 37 * go away, and certainly its address space can change, because control 38 * is returned back to the user-mode side. In that case, the pipe code 39 * arranges to copy the buffer supplied by the user process, to a pageable 40 * kernel buffer, and the receiving process will grab the data from the 41 * pageable kernel buffer. Since signals don't happen all that often, 42 * the copy operation is normally eliminated. 43 * 44 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 45 * happen for small transfers so that the system will not spend all of 46 * its time context switching. 47 * 48 * In order to limit the resource use of pipes, two sysctls exist: 49 * 50 * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable 51 * address space available to us in pipe_map. Whenever the amount in use 52 * exceeds half of this value, all new pipes will be created with size 53 * SMALL_PIPE_SIZE, rather than PIPE_SIZE. Big pipe creation will be limited 54 * as well. This value is loader tunable only. 55 * 56 * kern.ipc.maxpipekvawired - This value limits the amount of memory that may 57 * be wired in order to facilitate direct copies using page flipping. 58 * Whenever this value is exceeded, pipes will fall back to using regular 59 * copies. This value is sysctl controllable at all times. 60 * 61 * These values are autotuned in subr_param.c. 62 * 63 * Memory usage may be monitored through the sysctls 64 * kern.ipc.pipes, kern.ipc.pipekva and kern.ipc.pipekvawired. 65 * 66 */ 67 68#include <sys/cdefs.h> 69__FBSDID("$FreeBSD: head/sys/kern/sys_pipe.c 118764 2003-08-11 05:51:51Z silby $"); 70 71#include "opt_mac.h" 72 73#include <sys/param.h> 74#include <sys/systm.h> 75#include <sys/fcntl.h> 76#include <sys/file.h> 77#include <sys/filedesc.h> 78#include <sys/filio.h> 79#include <sys/kernel.h> 80#include <sys/lock.h> 81#include <sys/mac.h> 82#include <sys/mutex.h> 83#include <sys/ttycom.h> 84#include <sys/stat.h> 85#include <sys/malloc.h> 86#include <sys/poll.h> 87#include <sys/selinfo.h> 88#include <sys/signalvar.h> 89#include <sys/sysctl.h> 90#include <sys/sysproto.h> 91#include <sys/pipe.h> 92#include <sys/proc.h> 93#include <sys/vnode.h> 94#include <sys/uio.h> 95#include <sys/event.h> 96 97#include <vm/vm.h> 98#include <vm/vm_param.h> 99#include <vm/vm_object.h> 100#include <vm/vm_kern.h> 101#include <vm/vm_extern.h> 102#include <vm/pmap.h> 103#include <vm/vm_map.h> 104#include <vm/vm_page.h> 105#include <vm/uma.h> 106 107/* 108 * Use this define if you want to disable *fancy* VM things. Expect an 109 * approx 30% decrease in transfer rate. This could be useful for 110 * NetBSD or OpenBSD. 111 */ 112/* #define PIPE_NODIRECT */ 113 114/* 115 * interfaces to the outside world 116 */ 117static fo_rdwr_t pipe_read; 118static fo_rdwr_t pipe_write; 119static fo_ioctl_t pipe_ioctl; 120static fo_poll_t pipe_poll; 121static fo_kqfilter_t pipe_kqfilter; 122static fo_stat_t pipe_stat; 123static fo_close_t pipe_close; 124 125static struct fileops pipeops = { 126 .fo_read = pipe_read, 127 .fo_write = pipe_write, 128 .fo_ioctl = pipe_ioctl, 129 .fo_poll = pipe_poll, 130 .fo_kqfilter = pipe_kqfilter, 131 .fo_stat = pipe_stat, 132 .fo_close = pipe_close, 133 .fo_flags = DFLAG_PASSABLE 134}; 135 136static void filt_pipedetach(struct knote *kn); 137static int filt_piperead(struct knote *kn, long hint); 138static int filt_pipewrite(struct knote *kn, long hint); 139 140static struct filterops pipe_rfiltops = 141 { 1, NULL, filt_pipedetach, filt_piperead }; 142static struct filterops pipe_wfiltops = 143 { 1, NULL, filt_pipedetach, filt_pipewrite }; 144 145#define PIPE_GET_GIANT(pipe) \ 146 do { \ 147 KASSERT(((pipe)->pipe_state & PIPE_LOCKFL) != 0, \ 148 ("%s:%d PIPE_GET_GIANT: line pipe not locked", \ 149 __FILE__, __LINE__)); \ 150 PIPE_UNLOCK(pipe); \ 151 mtx_lock(&Giant); \ 152 } while (0) 153 154#define PIPE_DROP_GIANT(pipe) \ 155 do { \ 156 mtx_unlock(&Giant); \ 157 PIPE_LOCK(pipe); \ 158 } while (0) 159 160/* 161 * Default pipe buffer size(s), this can be kind-of large now because pipe 162 * space is pageable. The pipe code will try to maintain locality of 163 * reference for performance reasons, so small amounts of outstanding I/O 164 * will not wipe the cache. 165 */ 166#define MINPIPESIZE (PIPE_SIZE/3) 167#define MAXPIPESIZE (2*PIPE_SIZE/3) 168 169/* 170 * Limit the number of "big" pipes 171 */ 172#define LIMITBIGPIPES 32 173static int nbigpipe; 174 175static int amountpipes; 176static int amountpipekva; 177static int amountpipekvawired; 178 179SYSCTL_DECL(_kern_ipc); 180 181SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RD, 182 &maxpipekva, 0, "Pipe KVA limit"); 183SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekvawired, CTLFLAG_RW, 184 &maxpipekvawired, 0, "Pipe KVA wired limit"); 185SYSCTL_INT(_kern_ipc, OID_AUTO, pipes, CTLFLAG_RD, 186 &amountpipes, 0, "Current # of pipes"); 187SYSCTL_INT(_kern_ipc, OID_AUTO, bigpipes, CTLFLAG_RD, 188 &nbigpipe, 0, "Current # of big pipes"); 189SYSCTL_INT(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 190 &amountpipekva, 0, "Pipe KVA usage"); 191SYSCTL_INT(_kern_ipc, OID_AUTO, pipekvawired, CTLFLAG_RD, 192 &amountpipekvawired, 0, "Pipe wired KVA usage"); 193 194static void pipeinit(void *dummy __unused); 195static void pipeclose(struct pipe *cpipe); 196static void pipe_free_kmem(struct pipe *cpipe); 197static int pipe_create(struct pipe **cpipep); 198static __inline int pipelock(struct pipe *cpipe, int catch); 199static __inline void pipeunlock(struct pipe *cpipe); 200static __inline void pipeselwakeup(struct pipe *cpipe); 201#ifndef PIPE_NODIRECT 202static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 203static void pipe_destroy_write_buffer(struct pipe *wpipe); 204static int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 205static void pipe_clone_write_buffer(struct pipe *wpipe); 206#endif 207static int pipespace(struct pipe *cpipe, int size); 208 209static uma_zone_t pipe_zone; 210 211SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 212 213static void 214pipeinit(void *dummy __unused) 215{ 216 pipe_zone = uma_zcreate("PIPE", sizeof(struct pipe), NULL, 217 NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 218} 219 220/* 221 * The pipe system call for the DTYPE_PIPE type of pipes 222 */ 223 224/* ARGSUSED */ 225int 226pipe(td, uap) 227 struct thread *td; 228 struct pipe_args /* { 229 int dummy; 230 } */ *uap; 231{ 232 struct filedesc *fdp = td->td_proc->p_fd; 233 struct file *rf, *wf; 234 struct pipe *rpipe, *wpipe; 235 struct mtx *pmtx; 236 int fd, error; 237 238 KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 239 240 pmtx = malloc(sizeof(*pmtx), M_TEMP, M_WAITOK | M_ZERO); 241 242 rpipe = wpipe = NULL; 243 if (pipe_create(&rpipe) || pipe_create(&wpipe)) { 244 pipeclose(rpipe); 245 pipeclose(wpipe); 246 free(pmtx, M_TEMP); 247 return (ENFILE); 248 } 249 250 rpipe->pipe_state |= PIPE_DIRECTOK; 251 wpipe->pipe_state |= PIPE_DIRECTOK; 252 253 error = falloc(td, &rf, &fd); 254 if (error) { 255 pipeclose(rpipe); 256 pipeclose(wpipe); 257 free(pmtx, M_TEMP); 258 return (error); 259 } 260 fhold(rf); 261 td->td_retval[0] = fd; 262 263 /* 264 * Warning: once we've gotten past allocation of the fd for the 265 * read-side, we can only drop the read side via fdrop() in order 266 * to avoid races against processes which manage to dup() the read 267 * side while we are blocked trying to allocate the write side. 268 */ 269 FILE_LOCK(rf); 270 rf->f_flag = FREAD | FWRITE; 271 rf->f_type = DTYPE_PIPE; 272 rf->f_data = rpipe; 273 rf->f_ops = &pipeops; 274 FILE_UNLOCK(rf); 275 error = falloc(td, &wf, &fd); 276 if (error) { 277 FILEDESC_LOCK(fdp); 278 if (fdp->fd_ofiles[td->td_retval[0]] == rf) { 279 fdp->fd_ofiles[td->td_retval[0]] = NULL; 280 FILEDESC_UNLOCK(fdp); 281 fdrop(rf, td); 282 } else 283 FILEDESC_UNLOCK(fdp); 284 fdrop(rf, td); 285 /* rpipe has been closed by fdrop(). */ 286 pipeclose(wpipe); 287 free(pmtx, M_TEMP); 288 return (error); 289 } 290 FILE_LOCK(wf); 291 wf->f_flag = FREAD | FWRITE; 292 wf->f_type = DTYPE_PIPE; 293 wf->f_data = wpipe; 294 wf->f_ops = &pipeops; 295 FILE_UNLOCK(wf); 296 td->td_retval[1] = fd; 297 rpipe->pipe_peer = wpipe; 298 wpipe->pipe_peer = rpipe; 299#ifdef MAC 300 /* 301 * struct pipe represents a pipe endpoint. The MAC label is shared 302 * between the connected endpoints. As a result mac_init_pipe() and 303 * mac_create_pipe() should only be called on one of the endpoints 304 * after they have been connected. 305 */ 306 mac_init_pipe(rpipe); 307 mac_create_pipe(td->td_ucred, rpipe); 308#endif 309 mtx_init(pmtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 310 rpipe->pipe_mtxp = wpipe->pipe_mtxp = pmtx; 311 fdrop(rf, td); 312 313 return (0); 314} 315 316/* 317 * Allocate kva for pipe circular buffer, the space is pageable 318 * This routine will 'realloc' the size of a pipe safely, if it fails 319 * it will retain the old buffer. 320 * If it fails it will return ENOMEM. 321 */ 322static int 323pipespace(cpipe, size) 324 struct pipe *cpipe; 325 int size; 326{ 327 struct vm_object *object; 328 caddr_t buffer; 329 int npages, error; 330 static int curfail = 0; 331 static struct timeval lastfail; 332 333 KASSERT(cpipe->pipe_mtxp == NULL || !mtx_owned(PIPE_MTX(cpipe)), 334 ("pipespace: pipe mutex locked")); 335 336 size = round_page(size); 337 npages = size / PAGE_SIZE; 338 /* 339 * Create an object, I don't like the idea of paging to/from 340 * kernel_object. 341 * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. 342 */ 343 object = vm_object_allocate(OBJT_DEFAULT, npages); 344 buffer = (caddr_t) vm_map_min(pipe_map); 345 346 /* 347 * Insert the object into the kernel map, and allocate kva for it. 348 * The map entry is, by default, pageable. 349 * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. 350 */ 351 error = vm_map_find(pipe_map, object, 0, 352 (vm_offset_t *) &buffer, size, 1, 353 VM_PROT_ALL, VM_PROT_ALL, 0); 354 355 if (error != KERN_SUCCESS) { 356 vm_object_deallocate(object); 357 if (ppsratecheck(&lastfail, &curfail, 1)) 358 printf("kern.maxpipekva exceeded, please see tuning(7).\n"); 359 return (ENOMEM); 360 } 361 362 /* free old resources if we're resizing */ 363 pipe_free_kmem(cpipe); 364 cpipe->pipe_buffer.object = object; 365 cpipe->pipe_buffer.buffer = buffer; 366 cpipe->pipe_buffer.size = size; 367 cpipe->pipe_buffer.in = 0; 368 cpipe->pipe_buffer.out = 0; 369 cpipe->pipe_buffer.cnt = 0; 370 atomic_add_int(&amountpipes, 1); 371 atomic_add_int(&amountpipekva, cpipe->pipe_buffer.size); 372 return (0); 373} 374 375/* 376 * initialize and allocate VM and memory for pipe 377 */ 378static int 379pipe_create(cpipep) 380 struct pipe **cpipep; 381{ 382 struct pipe *cpipe; 383 int error; 384 385 *cpipep = uma_zalloc(pipe_zone, M_WAITOK); 386 if (*cpipep == NULL) 387 return (ENOMEM); 388 389 cpipe = *cpipep; 390 391 /* so pipespace()->pipe_free_kmem() doesn't follow junk pointer */ 392 cpipe->pipe_buffer.object = NULL; 393 /* 394 * protect so pipeclose() doesn't follow a junk pointer 395 * if pipespace() fails. 396 */ 397 bzero(&cpipe->pipe_sel, sizeof(cpipe->pipe_sel)); 398 cpipe->pipe_state = 0; 399 cpipe->pipe_peer = NULL; 400 cpipe->pipe_busy = 0; 401 402#ifndef PIPE_NODIRECT 403 /* 404 * pipe data structure initializations to support direct pipe I/O 405 */ 406 cpipe->pipe_map.cnt = 0; 407 cpipe->pipe_map.kva = 0; 408 cpipe->pipe_map.pos = 0; 409 cpipe->pipe_map.npages = 0; 410 /* cpipe->pipe_map.ms[] = invalid */ 411#endif 412 413 cpipe->pipe_mtxp = NULL; /* avoid pipespace assertion */ 414 /* 415 * Reduce to 1/4th pipe size if we're over our global max. 416 */ 417 if (amountpipekva > maxpipekva / 2) 418 error = pipespace(cpipe, SMALL_PIPE_SIZE); 419 else 420 error = pipespace(cpipe, PIPE_SIZE); 421 if (error) 422 return (error); 423 424 vfs_timestamp(&cpipe->pipe_ctime); 425 cpipe->pipe_atime = cpipe->pipe_ctime; 426 cpipe->pipe_mtime = cpipe->pipe_ctime; 427 428 return (0); 429} 430 431 432/* 433 * lock a pipe for I/O, blocking other access 434 */ 435static __inline int 436pipelock(cpipe, catch) 437 struct pipe *cpipe; 438 int catch; 439{ 440 int error; 441 442 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 443 while (cpipe->pipe_state & PIPE_LOCKFL) { 444 cpipe->pipe_state |= PIPE_LWANT; 445 error = msleep(cpipe, PIPE_MTX(cpipe), 446 catch ? (PRIBIO | PCATCH) : PRIBIO, 447 "pipelk", 0); 448 if (error != 0) 449 return (error); 450 } 451 cpipe->pipe_state |= PIPE_LOCKFL; 452 return (0); 453} 454 455/* 456 * unlock a pipe I/O lock 457 */ 458static __inline void 459pipeunlock(cpipe) 460 struct pipe *cpipe; 461{ 462 463 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 464 cpipe->pipe_state &= ~PIPE_LOCKFL; 465 if (cpipe->pipe_state & PIPE_LWANT) { 466 cpipe->pipe_state &= ~PIPE_LWANT; 467 wakeup(cpipe); 468 } 469} 470 471static __inline void 472pipeselwakeup(cpipe) 473 struct pipe *cpipe; 474{ 475 476 if (cpipe->pipe_state & PIPE_SEL) { 477 cpipe->pipe_state &= ~PIPE_SEL; 478 selwakeup(&cpipe->pipe_sel); 479 } 480 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 481 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 482 KNOTE(&cpipe->pipe_sel.si_note, 0); 483} 484 485/* ARGSUSED */ 486static int 487pipe_read(fp, uio, active_cred, flags, td) 488 struct file *fp; 489 struct uio *uio; 490 struct ucred *active_cred; 491 struct thread *td; 492 int flags; 493{ 494 struct pipe *rpipe = fp->f_data; 495 int error; 496 int nread = 0; 497 u_int size; 498 499 PIPE_LOCK(rpipe); 500 ++rpipe->pipe_busy; 501 error = pipelock(rpipe, 1); 502 if (error) 503 goto unlocked_error; 504 505#ifdef MAC 506 error = mac_check_pipe_read(active_cred, rpipe); 507 if (error) 508 goto locked_error; 509#endif 510 511 while (uio->uio_resid) { 512 /* 513 * normal pipe buffer receive 514 */ 515 if (rpipe->pipe_buffer.cnt > 0) { 516 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 517 if (size > rpipe->pipe_buffer.cnt) 518 size = rpipe->pipe_buffer.cnt; 519 if (size > (u_int) uio->uio_resid) 520 size = (u_int) uio->uio_resid; 521 522 PIPE_UNLOCK(rpipe); 523 error = uiomove( 524 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 525 size, uio); 526 PIPE_LOCK(rpipe); 527 if (error) 528 break; 529 530 rpipe->pipe_buffer.out += size; 531 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 532 rpipe->pipe_buffer.out = 0; 533 534 rpipe->pipe_buffer.cnt -= size; 535 536 /* 537 * If there is no more to read in the pipe, reset 538 * its pointers to the beginning. This improves 539 * cache hit stats. 540 */ 541 if (rpipe->pipe_buffer.cnt == 0) { 542 rpipe->pipe_buffer.in = 0; 543 rpipe->pipe_buffer.out = 0; 544 } 545 nread += size; 546#ifndef PIPE_NODIRECT 547 /* 548 * Direct copy, bypassing a kernel buffer. 549 */ 550 } else if ((size = rpipe->pipe_map.cnt) && 551 (rpipe->pipe_state & PIPE_DIRECTW)) { 552 caddr_t va; 553 if (size > (u_int) uio->uio_resid) 554 size = (u_int) uio->uio_resid; 555 556 va = (caddr_t) rpipe->pipe_map.kva + 557 rpipe->pipe_map.pos; 558 PIPE_UNLOCK(rpipe); 559 error = uiomove(va, size, uio); 560 PIPE_LOCK(rpipe); 561 if (error) 562 break; 563 nread += size; 564 rpipe->pipe_map.pos += size; 565 rpipe->pipe_map.cnt -= size; 566 if (rpipe->pipe_map.cnt == 0) { 567 rpipe->pipe_state &= ~PIPE_DIRECTW; 568 wakeup(rpipe); 569 } 570#endif 571 } else { 572 /* 573 * detect EOF condition 574 * read returns 0 on EOF, no need to set error 575 */ 576 if (rpipe->pipe_state & PIPE_EOF) 577 break; 578 579 /* 580 * If the "write-side" has been blocked, wake it up now. 581 */ 582 if (rpipe->pipe_state & PIPE_WANTW) { 583 rpipe->pipe_state &= ~PIPE_WANTW; 584 wakeup(rpipe); 585 } 586 587 /* 588 * Break if some data was read. 589 */ 590 if (nread > 0) 591 break; 592 593 /* 594 * Unlock the pipe buffer for our remaining processing. 595 * We will either break out with an error or we will 596 * sleep and relock to loop. 597 */ 598 pipeunlock(rpipe); 599 600 /* 601 * Handle non-blocking mode operation or 602 * wait for more data. 603 */ 604 if (fp->f_flag & FNONBLOCK) { 605 error = EAGAIN; 606 } else { 607 rpipe->pipe_state |= PIPE_WANTR; 608 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 609 PRIBIO | PCATCH, 610 "piperd", 0)) == 0) 611 error = pipelock(rpipe, 1); 612 } 613 if (error) 614 goto unlocked_error; 615 } 616 } 617#ifdef MAC 618locked_error: 619#endif 620 pipeunlock(rpipe); 621 622 /* XXX: should probably do this before getting any locks. */ 623 if (error == 0) 624 vfs_timestamp(&rpipe->pipe_atime); 625unlocked_error: 626 --rpipe->pipe_busy; 627 628 /* 629 * PIPE_WANT processing only makes sense if pipe_busy is 0. 630 */ 631 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 632 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 633 wakeup(rpipe); 634 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 635 /* 636 * Handle write blocking hysteresis. 637 */ 638 if (rpipe->pipe_state & PIPE_WANTW) { 639 rpipe->pipe_state &= ~PIPE_WANTW; 640 wakeup(rpipe); 641 } 642 } 643 644 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 645 pipeselwakeup(rpipe); 646 647 PIPE_UNLOCK(rpipe); 648 return (error); 649} 650 651#ifndef PIPE_NODIRECT 652/* 653 * Map the sending processes' buffer into kernel space and wire it. 654 * This is similar to a physical write operation. 655 */ 656static int 657pipe_build_write_buffer(wpipe, uio) 658 struct pipe *wpipe; 659 struct uio *uio; 660{ 661 u_int size; 662 int i; 663 vm_offset_t addr, endaddr; 664 vm_paddr_t paddr; 665 666 GIANT_REQUIRED; 667 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 668 669 size = (u_int) uio->uio_iov->iov_len; 670 if (size > wpipe->pipe_buffer.size) 671 size = wpipe->pipe_buffer.size; 672 673 endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size); 674 addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base); 675 for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) { 676 vm_page_t m; 677 678 /* 679 * vm_fault_quick() can sleep. Consequently, 680 * vm_page_lock_queue() and vm_page_unlock_queue() 681 * should not be performed outside of this loop. 682 */ 683 if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0 || 684 (paddr = pmap_extract(vmspace_pmap(curproc->p_vmspace), 685 addr)) == 0) { 686 int j; 687 688 vm_page_lock_queues(); 689 for (j = 0; j < i; j++) { 690 vm_page_unhold(wpipe->pipe_map.ms[j]); 691 } 692 vm_page_unlock_queues(); 693 return (EFAULT); 694 } 695 696 m = PHYS_TO_VM_PAGE(paddr); 697 vm_page_lock_queues(); 698 vm_page_hold(m); 699 vm_page_unlock_queues(); 700 wpipe->pipe_map.ms[i] = m; 701 } 702 703/* 704 * set up the control block 705 */ 706 wpipe->pipe_map.npages = i; 707 wpipe->pipe_map.pos = 708 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 709 wpipe->pipe_map.cnt = size; 710 711/* 712 * and map the buffer 713 */ 714 if (wpipe->pipe_map.kva == 0) { 715 /* 716 * We need to allocate space for an extra page because the 717 * address range might (will) span pages at times. 718 */ 719 wpipe->pipe_map.kva = kmem_alloc_nofault(kernel_map, 720 wpipe->pipe_buffer.size + PAGE_SIZE); 721 atomic_add_int(&amountpipekvawired, 722 wpipe->pipe_buffer.size + PAGE_SIZE); 723 } 724 pmap_qenter(wpipe->pipe_map.kva, wpipe->pipe_map.ms, 725 wpipe->pipe_map.npages); 726 727/* 728 * and update the uio data 729 */ 730 731 uio->uio_iov->iov_len -= size; 732 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 733 if (uio->uio_iov->iov_len == 0) 734 uio->uio_iov++; 735 uio->uio_resid -= size; 736 uio->uio_offset += size; 737 return (0); 738} 739 740/* 741 * unmap and unwire the process buffer 742 */ 743static void 744pipe_destroy_write_buffer(wpipe) 745 struct pipe *wpipe; 746{ 747 int i; 748 749 GIANT_REQUIRED; 750 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 751 752 if (wpipe->pipe_map.kva) { 753 pmap_qremove(wpipe->pipe_map.kva, wpipe->pipe_map.npages); 754 755 if (amountpipekvawired > maxpipekvawired / 2) { 756 /* Conserve address space */ 757 vm_offset_t kva = wpipe->pipe_map.kva; 758 wpipe->pipe_map.kva = 0; 759 kmem_free(kernel_map, kva, 760 wpipe->pipe_buffer.size + PAGE_SIZE); 761 atomic_subtract_int(&amountpipekvawired, 762 wpipe->pipe_buffer.size + PAGE_SIZE); 763 } 764 } 765 vm_page_lock_queues(); 766 for (i = 0; i < wpipe->pipe_map.npages; i++) { 767 vm_page_unhold(wpipe->pipe_map.ms[i]); 768 } 769 vm_page_unlock_queues(); 770 wpipe->pipe_map.npages = 0; 771} 772 773/* 774 * In the case of a signal, the writing process might go away. This 775 * code copies the data into the circular buffer so that the source 776 * pages can be freed without loss of data. 777 */ 778static void 779pipe_clone_write_buffer(wpipe) 780 struct pipe *wpipe; 781{ 782 int size; 783 int pos; 784 785 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 786 size = wpipe->pipe_map.cnt; 787 pos = wpipe->pipe_map.pos; 788 789 wpipe->pipe_buffer.in = size; 790 wpipe->pipe_buffer.out = 0; 791 wpipe->pipe_buffer.cnt = size; 792 wpipe->pipe_state &= ~PIPE_DIRECTW; 793 794 PIPE_GET_GIANT(wpipe); 795 bcopy((caddr_t) wpipe->pipe_map.kva + pos, 796 wpipe->pipe_buffer.buffer, size); 797 pipe_destroy_write_buffer(wpipe); 798 PIPE_DROP_GIANT(wpipe); 799} 800 801/* 802 * This implements the pipe buffer write mechanism. Note that only 803 * a direct write OR a normal pipe write can be pending at any given time. 804 * If there are any characters in the pipe buffer, the direct write will 805 * be deferred until the receiving process grabs all of the bytes from 806 * the pipe buffer. Then the direct mapping write is set-up. 807 */ 808static int 809pipe_direct_write(wpipe, uio) 810 struct pipe *wpipe; 811 struct uio *uio; 812{ 813 int error; 814 815retry: 816 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 817 while (wpipe->pipe_state & PIPE_DIRECTW) { 818 if (wpipe->pipe_state & PIPE_WANTR) { 819 wpipe->pipe_state &= ~PIPE_WANTR; 820 wakeup(wpipe); 821 } 822 wpipe->pipe_state |= PIPE_WANTW; 823 error = msleep(wpipe, PIPE_MTX(wpipe), 824 PRIBIO | PCATCH, "pipdww", 0); 825 if (error) 826 goto error1; 827 if (wpipe->pipe_state & PIPE_EOF) { 828 error = EPIPE; 829 goto error1; 830 } 831 } 832 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 833 if (wpipe->pipe_buffer.cnt > 0) { 834 if (wpipe->pipe_state & PIPE_WANTR) { 835 wpipe->pipe_state &= ~PIPE_WANTR; 836 wakeup(wpipe); 837 } 838 839 wpipe->pipe_state |= PIPE_WANTW; 840 error = msleep(wpipe, PIPE_MTX(wpipe), 841 PRIBIO | PCATCH, "pipdwc", 0); 842 if (error) 843 goto error1; 844 if (wpipe->pipe_state & PIPE_EOF) { 845 error = EPIPE; 846 goto error1; 847 } 848 goto retry; 849 } 850 851 wpipe->pipe_state |= PIPE_DIRECTW; 852 853 pipelock(wpipe, 0); 854 PIPE_GET_GIANT(wpipe); 855 error = pipe_build_write_buffer(wpipe, uio); 856 PIPE_DROP_GIANT(wpipe); 857 pipeunlock(wpipe); 858 if (error) { 859 wpipe->pipe_state &= ~PIPE_DIRECTW; 860 goto error1; 861 } 862 863 error = 0; 864 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 865 if (wpipe->pipe_state & PIPE_EOF) { 866 pipelock(wpipe, 0); 867 PIPE_GET_GIANT(wpipe); 868 pipe_destroy_write_buffer(wpipe); 869 PIPE_DROP_GIANT(wpipe); 870 pipeselwakeup(wpipe); 871 pipeunlock(wpipe); 872 error = EPIPE; 873 goto error1; 874 } 875 if (wpipe->pipe_state & PIPE_WANTR) { 876 wpipe->pipe_state &= ~PIPE_WANTR; 877 wakeup(wpipe); 878 } 879 pipeselwakeup(wpipe); 880 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 881 "pipdwt", 0); 882 } 883 884 pipelock(wpipe,0); 885 if (wpipe->pipe_state & PIPE_DIRECTW) { 886 /* 887 * this bit of trickery substitutes a kernel buffer for 888 * the process that might be going away. 889 */ 890 pipe_clone_write_buffer(wpipe); 891 } else { 892 PIPE_GET_GIANT(wpipe); 893 pipe_destroy_write_buffer(wpipe); 894 PIPE_DROP_GIANT(wpipe); 895 } 896 pipeunlock(wpipe); 897 return (error); 898 899error1: 900 wakeup(wpipe); 901 return (error); 902} 903#endif 904 905static int 906pipe_write(fp, uio, active_cred, flags, td) 907 struct file *fp; 908 struct uio *uio; 909 struct ucred *active_cred; 910 struct thread *td; 911 int flags; 912{ 913 int error = 0; 914 int orig_resid; 915 struct pipe *wpipe, *rpipe; 916 917 rpipe = fp->f_data; 918 wpipe = rpipe->pipe_peer; 919 920 PIPE_LOCK(rpipe); 921 /* 922 * detect loss of pipe read side, issue SIGPIPE if lost. 923 */ 924 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 925 PIPE_UNLOCK(rpipe); 926 return (EPIPE); 927 } 928#ifdef MAC 929 error = mac_check_pipe_write(active_cred, wpipe); 930 if (error) { 931 PIPE_UNLOCK(rpipe); 932 return (error); 933 } 934#endif 935 ++wpipe->pipe_busy; 936 937 /* 938 * If it is advantageous to resize the pipe buffer, do 939 * so. 940 */ 941 if ((uio->uio_resid > PIPE_SIZE) && 942 (amountpipekva < maxpipekva / 2) && 943 (nbigpipe < LIMITBIGPIPES) && 944 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 945 (wpipe->pipe_buffer.size <= PIPE_SIZE) && 946 (wpipe->pipe_buffer.cnt == 0)) { 947 948 if ((error = pipelock(wpipe, 1)) == 0) { 949 PIPE_GET_GIANT(wpipe); 950 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 951 atomic_add_int(&nbigpipe, 1); 952 PIPE_DROP_GIANT(wpipe); 953 pipeunlock(wpipe); 954 } 955 } 956 957 /* 958 * If an early error occured unbusy and return, waking up any pending 959 * readers. 960 */ 961 if (error) { 962 --wpipe->pipe_busy; 963 if ((wpipe->pipe_busy == 0) && 964 (wpipe->pipe_state & PIPE_WANT)) { 965 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 966 wakeup(wpipe); 967 } 968 PIPE_UNLOCK(rpipe); 969 return(error); 970 } 971 972 orig_resid = uio->uio_resid; 973 974 while (uio->uio_resid) { 975 int space; 976 977#ifndef PIPE_NODIRECT 978 /* 979 * If the transfer is large, we can gain performance if 980 * we do process-to-process copies directly. 981 * If the write is non-blocking, we don't use the 982 * direct write mechanism. 983 * 984 * The direct write mechanism will detect the reader going 985 * away on us. 986 */ 987 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 988 (fp->f_flag & FNONBLOCK) == 0 && 989 amountpipekvawired + uio->uio_resid < maxpipekvawired) { 990 error = pipe_direct_write(wpipe, uio); 991 if (error) 992 break; 993 continue; 994 } 995#endif 996 997 /* 998 * Pipe buffered writes cannot be coincidental with 999 * direct writes. We wait until the currently executing 1000 * direct write is completed before we start filling the 1001 * pipe buffer. We break out if a signal occurs or the 1002 * reader goes away. 1003 */ 1004 retrywrite: 1005 while (wpipe->pipe_state & PIPE_DIRECTW) { 1006 if (wpipe->pipe_state & PIPE_WANTR) { 1007 wpipe->pipe_state &= ~PIPE_WANTR; 1008 wakeup(wpipe); 1009 } 1010 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1011 "pipbww", 0); 1012 if (wpipe->pipe_state & PIPE_EOF) 1013 break; 1014 if (error) 1015 break; 1016 } 1017 if (wpipe->pipe_state & PIPE_EOF) { 1018 error = EPIPE; 1019 break; 1020 } 1021 1022 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1023 1024 /* Writes of size <= PIPE_BUF must be atomic. */ 1025 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1026 space = 0; 1027 1028 if (space > 0) { 1029 if ((error = pipelock(wpipe,1)) == 0) { 1030 int size; /* Transfer size */ 1031 int segsize; /* first segment to transfer */ 1032 1033 /* 1034 * It is possible for a direct write to 1035 * slip in on us... handle it here... 1036 */ 1037 if (wpipe->pipe_state & PIPE_DIRECTW) { 1038 pipeunlock(wpipe); 1039 goto retrywrite; 1040 } 1041 /* 1042 * If a process blocked in uiomove, our 1043 * value for space might be bad. 1044 * 1045 * XXX will we be ok if the reader has gone 1046 * away here? 1047 */ 1048 if (space > wpipe->pipe_buffer.size - 1049 wpipe->pipe_buffer.cnt) { 1050 pipeunlock(wpipe); 1051 goto retrywrite; 1052 } 1053 1054 /* 1055 * Transfer size is minimum of uio transfer 1056 * and free space in pipe buffer. 1057 */ 1058 if (space > uio->uio_resid) 1059 size = uio->uio_resid; 1060 else 1061 size = space; 1062 /* 1063 * First segment to transfer is minimum of 1064 * transfer size and contiguous space in 1065 * pipe buffer. If first segment to transfer 1066 * is less than the transfer size, we've got 1067 * a wraparound in the buffer. 1068 */ 1069 segsize = wpipe->pipe_buffer.size - 1070 wpipe->pipe_buffer.in; 1071 if (segsize > size) 1072 segsize = size; 1073 1074 /* Transfer first segment */ 1075 1076 PIPE_UNLOCK(rpipe); 1077 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1078 segsize, uio); 1079 PIPE_LOCK(rpipe); 1080 1081 if (error == 0 && segsize < size) { 1082 /* 1083 * Transfer remaining part now, to 1084 * support atomic writes. Wraparound 1085 * happened. 1086 */ 1087 if (wpipe->pipe_buffer.in + segsize != 1088 wpipe->pipe_buffer.size) 1089 panic("Expected pipe buffer " 1090 "wraparound disappeared"); 1091 1092 PIPE_UNLOCK(rpipe); 1093 error = uiomove( 1094 &wpipe->pipe_buffer.buffer[0], 1095 size - segsize, uio); 1096 PIPE_LOCK(rpipe); 1097 } 1098 if (error == 0) { 1099 wpipe->pipe_buffer.in += size; 1100 if (wpipe->pipe_buffer.in >= 1101 wpipe->pipe_buffer.size) { 1102 if (wpipe->pipe_buffer.in != 1103 size - segsize + 1104 wpipe->pipe_buffer.size) 1105 panic("Expected " 1106 "wraparound bad"); 1107 wpipe->pipe_buffer.in = size - 1108 segsize; 1109 } 1110 1111 wpipe->pipe_buffer.cnt += size; 1112 if (wpipe->pipe_buffer.cnt > 1113 wpipe->pipe_buffer.size) 1114 panic("Pipe buffer overflow"); 1115 1116 } 1117 pipeunlock(wpipe); 1118 } 1119 if (error) 1120 break; 1121 1122 } else { 1123 /* 1124 * If the "read-side" has been blocked, wake it up now. 1125 */ 1126 if (wpipe->pipe_state & PIPE_WANTR) { 1127 wpipe->pipe_state &= ~PIPE_WANTR; 1128 wakeup(wpipe); 1129 } 1130 1131 /* 1132 * don't block on non-blocking I/O 1133 */ 1134 if (fp->f_flag & FNONBLOCK) { 1135 error = EAGAIN; 1136 break; 1137 } 1138 1139 /* 1140 * We have no more space and have something to offer, 1141 * wake up select/poll. 1142 */ 1143 pipeselwakeup(wpipe); 1144 1145 wpipe->pipe_state |= PIPE_WANTW; 1146 error = msleep(wpipe, PIPE_MTX(rpipe), 1147 PRIBIO | PCATCH, "pipewr", 0); 1148 if (error != 0) 1149 break; 1150 /* 1151 * If read side wants to go away, we just issue a signal 1152 * to ourselves. 1153 */ 1154 if (wpipe->pipe_state & PIPE_EOF) { 1155 error = EPIPE; 1156 break; 1157 } 1158 } 1159 } 1160 1161 --wpipe->pipe_busy; 1162 1163 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1164 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1165 wakeup(wpipe); 1166 } else if (wpipe->pipe_buffer.cnt > 0) { 1167 /* 1168 * If we have put any characters in the buffer, we wake up 1169 * the reader. 1170 */ 1171 if (wpipe->pipe_state & PIPE_WANTR) { 1172 wpipe->pipe_state &= ~PIPE_WANTR; 1173 wakeup(wpipe); 1174 } 1175 } 1176 1177 /* 1178 * Don't return EPIPE if I/O was successful 1179 */ 1180 if ((wpipe->pipe_buffer.cnt == 0) && 1181 (uio->uio_resid == 0) && 1182 (error == EPIPE)) { 1183 error = 0; 1184 } 1185 1186 if (error == 0) 1187 vfs_timestamp(&wpipe->pipe_mtime); 1188 1189 /* 1190 * We have something to offer, 1191 * wake up select/poll. 1192 */ 1193 if (wpipe->pipe_buffer.cnt) 1194 pipeselwakeup(wpipe); 1195 1196 PIPE_UNLOCK(rpipe); 1197 return (error); 1198} 1199 1200/* 1201 * we implement a very minimal set of ioctls for compatibility with sockets. 1202 */ 1203static int 1204pipe_ioctl(fp, cmd, data, active_cred, td) 1205 struct file *fp; 1206 u_long cmd; 1207 void *data; 1208 struct ucred *active_cred; 1209 struct thread *td; 1210{ 1211 struct pipe *mpipe = fp->f_data; 1212#ifdef MAC 1213 int error; 1214#endif 1215 1216 PIPE_LOCK(mpipe); 1217 1218#ifdef MAC 1219 error = mac_check_pipe_ioctl(active_cred, mpipe, cmd, data); 1220 if (error) 1221 return (error); 1222#endif 1223 1224 switch (cmd) { 1225 1226 case FIONBIO: 1227 PIPE_UNLOCK(mpipe); 1228 return (0); 1229 1230 case FIOASYNC: 1231 if (*(int *)data) { 1232 mpipe->pipe_state |= PIPE_ASYNC; 1233 } else { 1234 mpipe->pipe_state &= ~PIPE_ASYNC; 1235 } 1236 PIPE_UNLOCK(mpipe); 1237 return (0); 1238 1239 case FIONREAD: 1240 if (mpipe->pipe_state & PIPE_DIRECTW) 1241 *(int *)data = mpipe->pipe_map.cnt; 1242 else 1243 *(int *)data = mpipe->pipe_buffer.cnt; 1244 PIPE_UNLOCK(mpipe); 1245 return (0); 1246 1247 case FIOSETOWN: 1248 PIPE_UNLOCK(mpipe); 1249 return (fsetown(*(int *)data, &mpipe->pipe_sigio)); 1250 1251 case FIOGETOWN: 1252 PIPE_UNLOCK(mpipe); 1253 *(int *)data = fgetown(&mpipe->pipe_sigio); 1254 return (0); 1255 1256 /* This is deprecated, FIOSETOWN should be used instead. */ 1257 case TIOCSPGRP: 1258 PIPE_UNLOCK(mpipe); 1259 return (fsetown(-(*(int *)data), &mpipe->pipe_sigio)); 1260 1261 /* This is deprecated, FIOGETOWN should be used instead. */ 1262 case TIOCGPGRP: 1263 PIPE_UNLOCK(mpipe); 1264 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1265 return (0); 1266 1267 } 1268 PIPE_UNLOCK(mpipe); 1269 return (ENOTTY); 1270} 1271 1272static int 1273pipe_poll(fp, events, active_cred, td) 1274 struct file *fp; 1275 int events; 1276 struct ucred *active_cred; 1277 struct thread *td; 1278{ 1279 struct pipe *rpipe = fp->f_data; 1280 struct pipe *wpipe; 1281 int revents = 0; 1282#ifdef MAC 1283 int error; 1284#endif 1285 1286 wpipe = rpipe->pipe_peer; 1287 PIPE_LOCK(rpipe); 1288#ifdef MAC 1289 error = mac_check_pipe_poll(active_cred, rpipe); 1290 if (error) 1291 goto locked_error; 1292#endif 1293 if (events & (POLLIN | POLLRDNORM)) 1294 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1295 (rpipe->pipe_buffer.cnt > 0) || 1296 (rpipe->pipe_state & PIPE_EOF)) 1297 revents |= events & (POLLIN | POLLRDNORM); 1298 1299 if (events & (POLLOUT | POLLWRNORM)) 1300 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) || 1301 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1302 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1303 revents |= events & (POLLOUT | POLLWRNORM); 1304 1305 if ((rpipe->pipe_state & PIPE_EOF) || 1306 (wpipe == NULL) || 1307 (wpipe->pipe_state & PIPE_EOF)) 1308 revents |= POLLHUP; 1309 1310 if (revents == 0) { 1311 if (events & (POLLIN | POLLRDNORM)) { 1312 selrecord(td, &rpipe->pipe_sel); 1313 rpipe->pipe_state |= PIPE_SEL; 1314 } 1315 1316 if (events & (POLLOUT | POLLWRNORM)) { 1317 selrecord(td, &wpipe->pipe_sel); 1318 wpipe->pipe_state |= PIPE_SEL; 1319 } 1320 } 1321#ifdef MAC 1322locked_error: 1323#endif 1324 PIPE_UNLOCK(rpipe); 1325 1326 return (revents); 1327} 1328 1329/* 1330 * We shouldn't need locks here as we're doing a read and this should 1331 * be a natural race. 1332 */ 1333static int 1334pipe_stat(fp, ub, active_cred, td) 1335 struct file *fp; 1336 struct stat *ub; 1337 struct ucred *active_cred; 1338 struct thread *td; 1339{ 1340 struct pipe *pipe = fp->f_data; 1341#ifdef MAC 1342 int error; 1343 1344 PIPE_LOCK(pipe); 1345 error = mac_check_pipe_stat(active_cred, pipe); 1346 PIPE_UNLOCK(pipe); 1347 if (error) 1348 return (error); 1349#endif 1350 bzero(ub, sizeof(*ub)); 1351 ub->st_mode = S_IFIFO; 1352 ub->st_blksize = pipe->pipe_buffer.size; 1353 ub->st_size = pipe->pipe_buffer.cnt; 1354 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1355 ub->st_atimespec = pipe->pipe_atime; 1356 ub->st_mtimespec = pipe->pipe_mtime; 1357 ub->st_ctimespec = pipe->pipe_ctime; 1358 ub->st_uid = fp->f_cred->cr_uid; 1359 ub->st_gid = fp->f_cred->cr_gid; 1360 /* 1361 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1362 * XXX (st_dev, st_ino) should be unique. 1363 */ 1364 return (0); 1365} 1366 1367/* ARGSUSED */ 1368static int 1369pipe_close(fp, td) 1370 struct file *fp; 1371 struct thread *td; 1372{ 1373 struct pipe *cpipe = fp->f_data; 1374 1375 fp->f_ops = &badfileops; 1376 fp->f_data = NULL; 1377 funsetown(&cpipe->pipe_sigio); 1378 pipeclose(cpipe); 1379 return (0); 1380} 1381 1382static void 1383pipe_free_kmem(cpipe) 1384 struct pipe *cpipe; 1385{ 1386 1387 KASSERT(cpipe->pipe_mtxp == NULL || !mtx_owned(PIPE_MTX(cpipe)), 1388 ("pipespace: pipe mutex locked")); 1389 1390 if (cpipe->pipe_buffer.buffer != NULL) { 1391 if (cpipe->pipe_buffer.size > PIPE_SIZE) 1392 atomic_subtract_int(&nbigpipe, 1); 1393 atomic_subtract_int(&amountpipekva, cpipe->pipe_buffer.size); 1394 atomic_subtract_int(&amountpipes, 1); 1395 vm_map_remove(pipe_map, 1396 (vm_offset_t)cpipe->pipe_buffer.buffer, 1397 (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size); 1398 cpipe->pipe_buffer.buffer = NULL; 1399 } 1400#ifndef PIPE_NODIRECT 1401 if (cpipe->pipe_map.kva != 0) { 1402 atomic_subtract_int(&amountpipekvawired, 1403 cpipe->pipe_buffer.size + PAGE_SIZE); 1404 kmem_free(kernel_map, 1405 cpipe->pipe_map.kva, 1406 cpipe->pipe_buffer.size + PAGE_SIZE); 1407 cpipe->pipe_map.cnt = 0; 1408 cpipe->pipe_map.kva = 0; 1409 cpipe->pipe_map.pos = 0; 1410 cpipe->pipe_map.npages = 0; 1411 } 1412#endif 1413} 1414 1415/* 1416 * shutdown the pipe 1417 */ 1418static void 1419pipeclose(cpipe) 1420 struct pipe *cpipe; 1421{ 1422 struct pipe *ppipe; 1423 int hadpeer; 1424 1425 if (cpipe == NULL) 1426 return; 1427 1428 hadpeer = 0; 1429 1430 /* partially created pipes won't have a valid mutex. */ 1431 if (PIPE_MTX(cpipe) != NULL) 1432 PIPE_LOCK(cpipe); 1433 1434 pipeselwakeup(cpipe); 1435 1436 /* 1437 * If the other side is blocked, wake it up saying that 1438 * we want to close it down. 1439 */ 1440 while (cpipe->pipe_busy) { 1441 wakeup(cpipe); 1442 cpipe->pipe_state |= PIPE_WANT | PIPE_EOF; 1443 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1444 } 1445 1446#ifdef MAC 1447 if (cpipe->pipe_label != NULL && cpipe->pipe_peer == NULL) 1448 mac_destroy_pipe(cpipe); 1449#endif 1450 1451 /* 1452 * Disconnect from peer 1453 */ 1454 if ((ppipe = cpipe->pipe_peer) != NULL) { 1455 hadpeer++; 1456 pipeselwakeup(ppipe); 1457 1458 ppipe->pipe_state |= PIPE_EOF; 1459 wakeup(ppipe); 1460 KNOTE(&ppipe->pipe_sel.si_note, 0); 1461 ppipe->pipe_peer = NULL; 1462 } 1463 /* 1464 * free resources 1465 */ 1466 if (PIPE_MTX(cpipe) != NULL) { 1467 PIPE_UNLOCK(cpipe); 1468 if (!hadpeer) { 1469 mtx_destroy(PIPE_MTX(cpipe)); 1470 free(PIPE_MTX(cpipe), M_TEMP); 1471 } 1472 } 1473 pipe_free_kmem(cpipe); 1474 uma_zfree(pipe_zone, cpipe); 1475} 1476 1477/*ARGSUSED*/ 1478static int 1479pipe_kqfilter(struct file *fp, struct knote *kn) 1480{ 1481 struct pipe *cpipe; 1482 1483 cpipe = kn->kn_fp->f_data; 1484 switch (kn->kn_filter) { 1485 case EVFILT_READ: 1486 kn->kn_fop = &pipe_rfiltops; 1487 break; 1488 case EVFILT_WRITE: 1489 kn->kn_fop = &pipe_wfiltops; 1490 cpipe = cpipe->pipe_peer; 1491 if (cpipe == NULL) 1492 /* other end of pipe has been closed */ 1493 return (EBADF); 1494 break; 1495 default: 1496 return (1); 1497 } 1498 kn->kn_hook = cpipe; 1499 1500 PIPE_LOCK(cpipe); 1501 SLIST_INSERT_HEAD(&cpipe->pipe_sel.si_note, kn, kn_selnext); 1502 PIPE_UNLOCK(cpipe); 1503 return (0); 1504} 1505 1506static void 1507filt_pipedetach(struct knote *kn) 1508{ 1509 struct pipe *cpipe = (struct pipe *)kn->kn_hook; 1510 1511 PIPE_LOCK(cpipe); 1512 SLIST_REMOVE(&cpipe->pipe_sel.si_note, kn, knote, kn_selnext); 1513 PIPE_UNLOCK(cpipe); 1514} 1515 1516/*ARGSUSED*/ 1517static int 1518filt_piperead(struct knote *kn, long hint) 1519{ 1520 struct pipe *rpipe = kn->kn_fp->f_data; 1521 struct pipe *wpipe = rpipe->pipe_peer; 1522 1523 PIPE_LOCK(rpipe); 1524 kn->kn_data = rpipe->pipe_buffer.cnt; 1525 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1526 kn->kn_data = rpipe->pipe_map.cnt; 1527 1528 if ((rpipe->pipe_state & PIPE_EOF) || 1529 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1530 kn->kn_flags |= EV_EOF; 1531 PIPE_UNLOCK(rpipe); 1532 return (1); 1533 } 1534 PIPE_UNLOCK(rpipe); 1535 return (kn->kn_data > 0); 1536} 1537 1538/*ARGSUSED*/ 1539static int 1540filt_pipewrite(struct knote *kn, long hint) 1541{ 1542 struct pipe *rpipe = kn->kn_fp->f_data; 1543 struct pipe *wpipe = rpipe->pipe_peer; 1544 1545 PIPE_LOCK(rpipe); 1546 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1547 kn->kn_data = 0; 1548 kn->kn_flags |= EV_EOF; 1549 PIPE_UNLOCK(rpipe); 1550 return (1); 1551 } 1552 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1553 if (wpipe->pipe_state & PIPE_DIRECTW) 1554 kn->kn_data = 0; 1555 1556 PIPE_UNLOCK(rpipe); 1557 return (kn->kn_data >= PIPE_BUF); 1558} 1559