sys_pipe.c revision 125367
1/* 2 * Copyright (c) 1996 John S. Dyson 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice immediately at the beginning of the file, without modification, 10 * this list of conditions, and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. Absolutely no warranty of function or purpose is made by the author 15 * John S. Dyson. 16 * 4. Modifications may be freely made to this file if the above conditions 17 * are met. 18 */ 19 20/* 21 * This file contains a high-performance replacement for the socket-based 22 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 23 * all features of sockets, but does do everything that pipes normally 24 * do. 25 */ 26 27/* 28 * This code has two modes of operation, a small write mode and a large 29 * write mode. The small write mode acts like conventional pipes with 30 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 31 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 32 * and PIPE_SIZE in size, it is fully mapped and wired into the kernel, and 33 * the receiving process can copy it directly from the pages in the sending 34 * process. 35 * 36 * If the sending process receives a signal, it is possible that it will 37 * go away, and certainly its address space can change, because control 38 * is returned back to the user-mode side. In that case, the pipe code 39 * arranges to copy the buffer supplied by the user process, to a pageable 40 * kernel buffer, and the receiving process will grab the data from the 41 * pageable kernel buffer. Since signals don't happen all that often, 42 * the copy operation is normally eliminated. 43 * 44 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 45 * happen for small transfers so that the system will not spend all of 46 * its time context switching. 47 * 48 * In order to limit the resource use of pipes, two sysctls exist: 49 * 50 * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable 51 * address space available to us in pipe_map. Whenever the amount in use 52 * exceeds half of this value, all new pipes will be created with size 53 * SMALL_PIPE_SIZE, rather than PIPE_SIZE. Big pipe creation will be limited 54 * as well. This value is loader tunable only. 55 * 56 * kern.ipc.maxpipekvawired - This value limits the amount of memory that may 57 * be wired in order to facilitate direct copies using page flipping. 58 * Whenever this value is exceeded, pipes will fall back to using regular 59 * copies. This value is sysctl controllable at all times. 60 * 61 * These values are autotuned in subr_param.c. 62 * 63 * Memory usage may be monitored through the sysctls 64 * kern.ipc.pipes, kern.ipc.pipekva and kern.ipc.pipekvawired. 65 * 66 */ 67 68#include <sys/cdefs.h> 69__FBSDID("$FreeBSD: head/sys/kern/sys_pipe.c 125367 2004-02-03 04:55:24Z rwatson $"); 70 71#include "opt_mac.h" 72 73#include <sys/param.h> 74#include <sys/systm.h> 75#include <sys/fcntl.h> 76#include <sys/file.h> 77#include <sys/filedesc.h> 78#include <sys/filio.h> 79#include <sys/kernel.h> 80#include <sys/lock.h> 81#include <sys/mac.h> 82#include <sys/mutex.h> 83#include <sys/ttycom.h> 84#include <sys/stat.h> 85#include <sys/malloc.h> 86#include <sys/poll.h> 87#include <sys/selinfo.h> 88#include <sys/signalvar.h> 89#include <sys/sysctl.h> 90#include <sys/sysproto.h> 91#include <sys/pipe.h> 92#include <sys/proc.h> 93#include <sys/vnode.h> 94#include <sys/uio.h> 95#include <sys/event.h> 96 97#include <vm/vm.h> 98#include <vm/vm_param.h> 99#include <vm/vm_object.h> 100#include <vm/vm_kern.h> 101#include <vm/vm_extern.h> 102#include <vm/pmap.h> 103#include <vm/vm_map.h> 104#include <vm/vm_page.h> 105#include <vm/uma.h> 106 107/* 108 * Use this define if you want to disable *fancy* VM things. Expect an 109 * approx 30% decrease in transfer rate. This could be useful for 110 * NetBSD or OpenBSD. 111 */ 112/* #define PIPE_NODIRECT */ 113 114/* 115 * interfaces to the outside world 116 */ 117static fo_rdwr_t pipe_read; 118static fo_rdwr_t pipe_write; 119static fo_ioctl_t pipe_ioctl; 120static fo_poll_t pipe_poll; 121static fo_kqfilter_t pipe_kqfilter; 122static fo_stat_t pipe_stat; 123static fo_close_t pipe_close; 124 125static struct fileops pipeops = { 126 .fo_read = pipe_read, 127 .fo_write = pipe_write, 128 .fo_ioctl = pipe_ioctl, 129 .fo_poll = pipe_poll, 130 .fo_kqfilter = pipe_kqfilter, 131 .fo_stat = pipe_stat, 132 .fo_close = pipe_close, 133 .fo_flags = DFLAG_PASSABLE 134}; 135 136static void filt_pipedetach(struct knote *kn); 137static int filt_piperead(struct knote *kn, long hint); 138static int filt_pipewrite(struct knote *kn, long hint); 139 140static struct filterops pipe_rfiltops = 141 { 1, NULL, filt_pipedetach, filt_piperead }; 142static struct filterops pipe_wfiltops = 143 { 1, NULL, filt_pipedetach, filt_pipewrite }; 144 145/* 146 * Default pipe buffer size(s), this can be kind-of large now because pipe 147 * space is pageable. The pipe code will try to maintain locality of 148 * reference for performance reasons, so small amounts of outstanding I/O 149 * will not wipe the cache. 150 */ 151#define MINPIPESIZE (PIPE_SIZE/3) 152#define MAXPIPESIZE (2*PIPE_SIZE/3) 153 154/* 155 * Limit the number of "big" pipes 156 */ 157#define LIMITBIGPIPES 32 158static int nbigpipe; 159 160static int amountpipes; 161static int amountpipekva; 162static int amountpipekvawired; 163 164SYSCTL_DECL(_kern_ipc); 165 166SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN, 167 &maxpipekva, 0, "Pipe KVA limit"); 168SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekvawired, CTLFLAG_RW, 169 &maxpipekvawired, 0, "Pipe KVA wired limit"); 170SYSCTL_INT(_kern_ipc, OID_AUTO, pipes, CTLFLAG_RD, 171 &amountpipes, 0, "Current # of pipes"); 172SYSCTL_INT(_kern_ipc, OID_AUTO, bigpipes, CTLFLAG_RD, 173 &nbigpipe, 0, "Current # of big pipes"); 174SYSCTL_INT(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 175 &amountpipekva, 0, "Pipe KVA usage"); 176SYSCTL_INT(_kern_ipc, OID_AUTO, pipekvawired, CTLFLAG_RD, 177 &amountpipekvawired, 0, "Pipe wired KVA usage"); 178 179static void pipeinit(void *dummy __unused); 180static void pipeclose(struct pipe *cpipe); 181static void pipe_free_kmem(struct pipe *cpipe); 182static int pipe_create(struct pipe *pipe); 183static __inline int pipelock(struct pipe *cpipe, int catch); 184static __inline void pipeunlock(struct pipe *cpipe); 185static __inline void pipeselwakeup(struct pipe *cpipe); 186#ifndef PIPE_NODIRECT 187static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 188static void pipe_destroy_write_buffer(struct pipe *wpipe); 189static int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 190static void pipe_clone_write_buffer(struct pipe *wpipe); 191#endif 192static int pipespace(struct pipe *cpipe, int size); 193 194static void pipe_zone_ctor(void *mem, int size, void *arg); 195static void pipe_zone_dtor(void *mem, int size, void *arg); 196static void pipe_zone_init(void *mem, int size); 197static void pipe_zone_fini(void *mem, int size); 198 199static uma_zone_t pipe_zone; 200 201SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 202 203static void 204pipeinit(void *dummy __unused) 205{ 206 207 pipe_zone = uma_zcreate("PIPE", sizeof(struct pipepair), 208 pipe_zone_ctor, pipe_zone_dtor, pipe_zone_init, pipe_zone_fini, 209 UMA_ALIGN_PTR, 0); 210 KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 211} 212 213static void 214pipe_zone_ctor(void *mem, int size, void *arg) 215{ 216 struct pipepair *pp; 217 struct pipe *rpipe, *wpipe; 218 219 KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size")); 220 221 pp = (struct pipepair *)mem; 222 223 /* 224 * We zero both pipe endpoints to make sure all the kmem pointers 225 * are NULL, flag fields are zero'd, etc. We timestamp both 226 * endpoints with the same time. 227 */ 228 rpipe = &pp->pp_rpipe; 229 bzero(rpipe, sizeof(*rpipe)); 230 vfs_timestamp(&rpipe->pipe_ctime); 231 rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime; 232 233 wpipe = &pp->pp_wpipe; 234 bzero(wpipe, sizeof(*wpipe)); 235 wpipe->pipe_ctime = rpipe->pipe_ctime; 236 wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime; 237 238 rpipe->pipe_peer = wpipe; 239 rpipe->pipe_pair = pp; 240 wpipe->pipe_peer = rpipe; 241 wpipe->pipe_pair = pp; 242 243 /* 244 * Mark both endpoints as present; they will later get free'd 245 * one at a time. When both are free'd, then the whole pair 246 * is released. 247 */ 248 rpipe->pipe_present = 1; 249 wpipe->pipe_present = 1; 250 251 /* 252 * Eventually, the MAC Framework may initialize the label 253 * in ctor or init, but for now we do it elswhere to avoid 254 * blocking in ctor or init. 255 */ 256 pp->pp_label = NULL; 257 258 atomic_add_int(&amountpipes, 2); 259} 260 261static void 262pipe_zone_dtor(void *mem, int size, void *arg) 263{ 264 struct pipepair *pp; 265 266 KASSERT(size == sizeof(*pp), ("pipe_zone_dtor: wrong size")); 267 268 pp = (struct pipepair *)mem; 269 270 atomic_subtract_int(&amountpipes, 2); 271} 272 273static void 274pipe_zone_init(void *mem, int size) 275{ 276 struct pipepair *pp; 277 278 KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size")); 279 280 pp = (struct pipepair *)mem; 281 282 mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 283} 284 285static void 286pipe_zone_fini(void *mem, int size) 287{ 288 struct pipepair *pp; 289 290 KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size")); 291 292 pp = (struct pipepair *)mem; 293 294 mtx_destroy(&pp->pp_mtx); 295} 296 297/* 298 * The pipe system call for the DTYPE_PIPE type of pipes. If we fail, 299 * let the zone pick up the pieces via pipeclose(). 300 */ 301 302/* ARGSUSED */ 303int 304pipe(td, uap) 305 struct thread *td; 306 struct pipe_args /* { 307 int dummy; 308 } */ *uap; 309{ 310 struct filedesc *fdp = td->td_proc->p_fd; 311 struct file *rf, *wf; 312 struct pipepair *pp; 313 struct pipe *rpipe, *wpipe; 314 int fd, error; 315 316 pp = uma_zalloc(pipe_zone, M_WAITOK); 317#ifdef MAC 318 /* 319 * struct pipe represents a pipe endpoint. The MAC label is shared 320 * between the connected endpoints. As a result mac_init_pipe() and 321 * mac_create_pipe() should only be called on one of the endpoints 322 * after they have been connected. 323 */ 324 mac_init_pipe(pp); 325 mac_create_pipe(td->td_ucred, pp); 326#endif 327 rpipe = &pp->pp_rpipe; 328 wpipe = &pp->pp_wpipe; 329 330 if (pipe_create(rpipe) || pipe_create(wpipe)) { 331 pipeclose(rpipe); 332 pipeclose(wpipe); 333 return (ENFILE); 334 } 335 336 rpipe->pipe_state |= PIPE_DIRECTOK; 337 wpipe->pipe_state |= PIPE_DIRECTOK; 338 339 error = falloc(td, &rf, &fd); 340 if (error) { 341 pipeclose(rpipe); 342 pipeclose(wpipe); 343 return (error); 344 } 345 /* An extra reference on `rf' has been held for us by falloc(). */ 346 td->td_retval[0] = fd; 347 348 /* 349 * Warning: once we've gotten past allocation of the fd for the 350 * read-side, we can only drop the read side via fdrop() in order 351 * to avoid races against processes which manage to dup() the read 352 * side while we are blocked trying to allocate the write side. 353 */ 354 FILE_LOCK(rf); 355 rf->f_flag = FREAD | FWRITE; 356 rf->f_type = DTYPE_PIPE; 357 rf->f_data = rpipe; 358 rf->f_ops = &pipeops; 359 FILE_UNLOCK(rf); 360 error = falloc(td, &wf, &fd); 361 if (error) { 362 FILEDESC_LOCK(fdp); 363 if (fdp->fd_ofiles[td->td_retval[0]] == rf) { 364 fdp->fd_ofiles[td->td_retval[0]] = NULL; 365 fdunused(fdp, td->td_retval[0]); 366 FILEDESC_UNLOCK(fdp); 367 fdrop(rf, td); 368 } else { 369 FILEDESC_UNLOCK(fdp); 370 } 371 fdrop(rf, td); 372 /* rpipe has been closed by fdrop(). */ 373 pipeclose(wpipe); 374 return (error); 375 } 376 /* An extra reference on `wf' has been held for us by falloc(). */ 377 FILE_LOCK(wf); 378 wf->f_flag = FREAD | FWRITE; 379 wf->f_type = DTYPE_PIPE; 380 wf->f_data = wpipe; 381 wf->f_ops = &pipeops; 382 FILE_UNLOCK(wf); 383 fdrop(wf, td); 384 td->td_retval[1] = fd; 385 fdrop(rf, td); 386 387 return (0); 388} 389 390/* 391 * Allocate kva for pipe circular buffer, the space is pageable 392 * This routine will 'realloc' the size of a pipe safely, if it fails 393 * it will retain the old buffer. 394 * If it fails it will return ENOMEM. 395 */ 396static int 397pipespace(cpipe, size) 398 struct pipe *cpipe; 399 int size; 400{ 401 caddr_t buffer; 402 int error; 403 static int curfail = 0; 404 static struct timeval lastfail; 405 406 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked")); 407 408 size = round_page(size); 409 /* 410 * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. 411 */ 412 buffer = (caddr_t) vm_map_min(pipe_map); 413 414 /* 415 * The map entry is, by default, pageable. 416 * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. 417 */ 418 error = vm_map_find(pipe_map, NULL, 0, 419 (vm_offset_t *) &buffer, size, 1, 420 VM_PROT_ALL, VM_PROT_ALL, 0); 421 if (error != KERN_SUCCESS) { 422 if (ppsratecheck(&lastfail, &curfail, 1)) 423 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n"); 424 return (ENOMEM); 425 } 426 427 /* free old resources if we're resizing */ 428 pipe_free_kmem(cpipe); 429 cpipe->pipe_buffer.buffer = buffer; 430 cpipe->pipe_buffer.size = size; 431 cpipe->pipe_buffer.in = 0; 432 cpipe->pipe_buffer.out = 0; 433 cpipe->pipe_buffer.cnt = 0; 434 atomic_add_int(&amountpipekva, cpipe->pipe_buffer.size); 435 return (0); 436} 437 438/* 439 * Initialize and allocate VM and memory for pipe. The structure 440 * will start out zero'd from the ctor, so we just manage the kmem. 441 */ 442static int 443pipe_create(pipe) 444 struct pipe *pipe; 445{ 446 int error; 447 448 /* 449 * Reduce to 1/4th pipe size if we're over our global max. 450 */ 451 if (amountpipekva > maxpipekva / 2) 452 error = pipespace(pipe, SMALL_PIPE_SIZE); 453 else 454 error = pipespace(pipe, PIPE_SIZE); 455 if (error) 456 return (error); 457 458 return (0); 459} 460 461/* 462 * lock a pipe for I/O, blocking other access 463 */ 464static __inline int 465pipelock(cpipe, catch) 466 struct pipe *cpipe; 467 int catch; 468{ 469 int error; 470 471 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 472 while (cpipe->pipe_state & PIPE_LOCKFL) { 473 cpipe->pipe_state |= PIPE_LWANT; 474 error = msleep(cpipe, PIPE_MTX(cpipe), 475 catch ? (PRIBIO | PCATCH) : PRIBIO, 476 "pipelk", 0); 477 if (error != 0) 478 return (error); 479 } 480 cpipe->pipe_state |= PIPE_LOCKFL; 481 return (0); 482} 483 484/* 485 * unlock a pipe I/O lock 486 */ 487static __inline void 488pipeunlock(cpipe) 489 struct pipe *cpipe; 490{ 491 492 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 493 cpipe->pipe_state &= ~PIPE_LOCKFL; 494 if (cpipe->pipe_state & PIPE_LWANT) { 495 cpipe->pipe_state &= ~PIPE_LWANT; 496 wakeup(cpipe); 497 } 498} 499 500static __inline void 501pipeselwakeup(cpipe) 502 struct pipe *cpipe; 503{ 504 505 if (cpipe->pipe_state & PIPE_SEL) { 506 cpipe->pipe_state &= ~PIPE_SEL; 507 selwakeuppri(&cpipe->pipe_sel, PSOCK); 508 } 509 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 510 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 511 KNOTE(&cpipe->pipe_sel.si_note, 0); 512} 513 514/* ARGSUSED */ 515static int 516pipe_read(fp, uio, active_cred, flags, td) 517 struct file *fp; 518 struct uio *uio; 519 struct ucred *active_cred; 520 struct thread *td; 521 int flags; 522{ 523 struct pipe *rpipe = fp->f_data; 524 int error; 525 int nread = 0; 526 u_int size; 527 528 PIPE_LOCK(rpipe); 529 ++rpipe->pipe_busy; 530 error = pipelock(rpipe, 1); 531 if (error) 532 goto unlocked_error; 533 534#ifdef MAC 535 error = mac_check_pipe_read(active_cred, rpipe->pipe_pair); 536 if (error) 537 goto locked_error; 538#endif 539 540 while (uio->uio_resid) { 541 /* 542 * normal pipe buffer receive 543 */ 544 if (rpipe->pipe_buffer.cnt > 0) { 545 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 546 if (size > rpipe->pipe_buffer.cnt) 547 size = rpipe->pipe_buffer.cnt; 548 if (size > (u_int) uio->uio_resid) 549 size = (u_int) uio->uio_resid; 550 551 PIPE_UNLOCK(rpipe); 552 error = uiomove( 553 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 554 size, uio); 555 PIPE_LOCK(rpipe); 556 if (error) 557 break; 558 559 rpipe->pipe_buffer.out += size; 560 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 561 rpipe->pipe_buffer.out = 0; 562 563 rpipe->pipe_buffer.cnt -= size; 564 565 /* 566 * If there is no more to read in the pipe, reset 567 * its pointers to the beginning. This improves 568 * cache hit stats. 569 */ 570 if (rpipe->pipe_buffer.cnt == 0) { 571 rpipe->pipe_buffer.in = 0; 572 rpipe->pipe_buffer.out = 0; 573 } 574 nread += size; 575#ifndef PIPE_NODIRECT 576 /* 577 * Direct copy, bypassing a kernel buffer. 578 */ 579 } else if ((size = rpipe->pipe_map.cnt) && 580 (rpipe->pipe_state & PIPE_DIRECTW)) { 581 caddr_t va; 582 if (size > (u_int) uio->uio_resid) 583 size = (u_int) uio->uio_resid; 584 585 va = (caddr_t) rpipe->pipe_map.kva + 586 rpipe->pipe_map.pos; 587 PIPE_UNLOCK(rpipe); 588 error = uiomove(va, size, uio); 589 PIPE_LOCK(rpipe); 590 if (error) 591 break; 592 nread += size; 593 rpipe->pipe_map.pos += size; 594 rpipe->pipe_map.cnt -= size; 595 if (rpipe->pipe_map.cnt == 0) { 596 rpipe->pipe_state &= ~PIPE_DIRECTW; 597 wakeup(rpipe); 598 } 599#endif 600 } else { 601 /* 602 * detect EOF condition 603 * read returns 0 on EOF, no need to set error 604 */ 605 if (rpipe->pipe_state & PIPE_EOF) 606 break; 607 608 /* 609 * If the "write-side" has been blocked, wake it up now. 610 */ 611 if (rpipe->pipe_state & PIPE_WANTW) { 612 rpipe->pipe_state &= ~PIPE_WANTW; 613 wakeup(rpipe); 614 } 615 616 /* 617 * Break if some data was read. 618 */ 619 if (nread > 0) 620 break; 621 622 /* 623 * Unlock the pipe buffer for our remaining processing. 624 * We will either break out with an error or we will 625 * sleep and relock to loop. 626 */ 627 pipeunlock(rpipe); 628 629 /* 630 * Handle non-blocking mode operation or 631 * wait for more data. 632 */ 633 if (fp->f_flag & FNONBLOCK) { 634 error = EAGAIN; 635 } else { 636 rpipe->pipe_state |= PIPE_WANTR; 637 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 638 PRIBIO | PCATCH, 639 "piperd", 0)) == 0) 640 error = pipelock(rpipe, 1); 641 } 642 if (error) 643 goto unlocked_error; 644 } 645 } 646#ifdef MAC 647locked_error: 648#endif 649 pipeunlock(rpipe); 650 651 /* XXX: should probably do this before getting any locks. */ 652 if (error == 0) 653 vfs_timestamp(&rpipe->pipe_atime); 654unlocked_error: 655 --rpipe->pipe_busy; 656 657 /* 658 * PIPE_WANT processing only makes sense if pipe_busy is 0. 659 */ 660 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 661 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 662 wakeup(rpipe); 663 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 664 /* 665 * Handle write blocking hysteresis. 666 */ 667 if (rpipe->pipe_state & PIPE_WANTW) { 668 rpipe->pipe_state &= ~PIPE_WANTW; 669 wakeup(rpipe); 670 } 671 } 672 673 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 674 pipeselwakeup(rpipe); 675 676 PIPE_UNLOCK(rpipe); 677 return (error); 678} 679 680#ifndef PIPE_NODIRECT 681/* 682 * Map the sending processes' buffer into kernel space and wire it. 683 * This is similar to a physical write operation. 684 */ 685static int 686pipe_build_write_buffer(wpipe, uio) 687 struct pipe *wpipe; 688 struct uio *uio; 689{ 690 pmap_t pmap; 691 u_int size; 692 int i, j; 693 vm_offset_t addr, endaddr; 694 695 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 696 697 size = (u_int) uio->uio_iov->iov_len; 698 if (size > wpipe->pipe_buffer.size) 699 size = wpipe->pipe_buffer.size; 700 701 pmap = vmspace_pmap(curproc->p_vmspace); 702 endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size); 703 addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base); 704 for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) { 705 /* 706 * vm_fault_quick() can sleep. Consequently, 707 * vm_page_lock_queue() and vm_page_unlock_queue() 708 * should not be performed outside of this loop. 709 */ 710 race: 711 if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0) { 712 vm_page_lock_queues(); 713 for (j = 0; j < i; j++) 714 vm_page_unhold(wpipe->pipe_map.ms[j]); 715 vm_page_unlock_queues(); 716 return (EFAULT); 717 } 718 wpipe->pipe_map.ms[i] = pmap_extract_and_hold(pmap, addr, 719 VM_PROT_READ); 720 if (wpipe->pipe_map.ms[i] == NULL) 721 goto race; 722 } 723 724/* 725 * set up the control block 726 */ 727 wpipe->pipe_map.npages = i; 728 wpipe->pipe_map.pos = 729 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 730 wpipe->pipe_map.cnt = size; 731 732/* 733 * and map the buffer 734 */ 735 if (wpipe->pipe_map.kva == 0) { 736 /* 737 * We need to allocate space for an extra page because the 738 * address range might (will) span pages at times. 739 */ 740 wpipe->pipe_map.kva = kmem_alloc_nofault(kernel_map, 741 wpipe->pipe_buffer.size + PAGE_SIZE); 742 atomic_add_int(&amountpipekvawired, 743 wpipe->pipe_buffer.size + PAGE_SIZE); 744 } 745 pmap_qenter(wpipe->pipe_map.kva, wpipe->pipe_map.ms, 746 wpipe->pipe_map.npages); 747 748/* 749 * and update the uio data 750 */ 751 752 uio->uio_iov->iov_len -= size; 753 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 754 if (uio->uio_iov->iov_len == 0) 755 uio->uio_iov++; 756 uio->uio_resid -= size; 757 uio->uio_offset += size; 758 return (0); 759} 760 761/* 762 * unmap and unwire the process buffer 763 */ 764static void 765pipe_destroy_write_buffer(wpipe) 766 struct pipe *wpipe; 767{ 768 int i; 769 770 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 771 if (wpipe->pipe_map.kva) { 772 pmap_qremove(wpipe->pipe_map.kva, wpipe->pipe_map.npages); 773 774 if (amountpipekvawired > maxpipekvawired / 2) { 775 /* Conserve address space */ 776 vm_offset_t kva = wpipe->pipe_map.kva; 777 wpipe->pipe_map.kva = 0; 778 kmem_free(kernel_map, kva, 779 wpipe->pipe_buffer.size + PAGE_SIZE); 780 atomic_subtract_int(&amountpipekvawired, 781 wpipe->pipe_buffer.size + PAGE_SIZE); 782 } 783 } 784 vm_page_lock_queues(); 785 for (i = 0; i < wpipe->pipe_map.npages; i++) { 786 vm_page_unhold(wpipe->pipe_map.ms[i]); 787 } 788 vm_page_unlock_queues(); 789 wpipe->pipe_map.npages = 0; 790} 791 792/* 793 * In the case of a signal, the writing process might go away. This 794 * code copies the data into the circular buffer so that the source 795 * pages can be freed without loss of data. 796 */ 797static void 798pipe_clone_write_buffer(wpipe) 799 struct pipe *wpipe; 800{ 801 int size; 802 int pos; 803 804 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 805 size = wpipe->pipe_map.cnt; 806 pos = wpipe->pipe_map.pos; 807 808 wpipe->pipe_buffer.in = size; 809 wpipe->pipe_buffer.out = 0; 810 wpipe->pipe_buffer.cnt = size; 811 wpipe->pipe_state &= ~PIPE_DIRECTW; 812 813 PIPE_UNLOCK(wpipe); 814 bcopy((caddr_t) wpipe->pipe_map.kva + pos, 815 wpipe->pipe_buffer.buffer, size); 816 pipe_destroy_write_buffer(wpipe); 817 PIPE_LOCK(wpipe); 818} 819 820/* 821 * This implements the pipe buffer write mechanism. Note that only 822 * a direct write OR a normal pipe write can be pending at any given time. 823 * If there are any characters in the pipe buffer, the direct write will 824 * be deferred until the receiving process grabs all of the bytes from 825 * the pipe buffer. Then the direct mapping write is set-up. 826 */ 827static int 828pipe_direct_write(wpipe, uio) 829 struct pipe *wpipe; 830 struct uio *uio; 831{ 832 int error; 833 834retry: 835 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 836 while (wpipe->pipe_state & PIPE_DIRECTW) { 837 if (wpipe->pipe_state & PIPE_WANTR) { 838 wpipe->pipe_state &= ~PIPE_WANTR; 839 wakeup(wpipe); 840 } 841 wpipe->pipe_state |= PIPE_WANTW; 842 error = msleep(wpipe, PIPE_MTX(wpipe), 843 PRIBIO | PCATCH, "pipdww", 0); 844 if (error) 845 goto error1; 846 if (wpipe->pipe_state & PIPE_EOF) { 847 error = EPIPE; 848 goto error1; 849 } 850 } 851 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 852 if (wpipe->pipe_buffer.cnt > 0) { 853 if (wpipe->pipe_state & PIPE_WANTR) { 854 wpipe->pipe_state &= ~PIPE_WANTR; 855 wakeup(wpipe); 856 } 857 858 wpipe->pipe_state |= PIPE_WANTW; 859 error = msleep(wpipe, PIPE_MTX(wpipe), 860 PRIBIO | PCATCH, "pipdwc", 0); 861 if (error) 862 goto error1; 863 if (wpipe->pipe_state & PIPE_EOF) { 864 error = EPIPE; 865 goto error1; 866 } 867 goto retry; 868 } 869 870 wpipe->pipe_state |= PIPE_DIRECTW; 871 872 pipelock(wpipe, 0); 873 PIPE_UNLOCK(wpipe); 874 error = pipe_build_write_buffer(wpipe, uio); 875 PIPE_LOCK(wpipe); 876 pipeunlock(wpipe); 877 if (error) { 878 wpipe->pipe_state &= ~PIPE_DIRECTW; 879 goto error1; 880 } 881 882 error = 0; 883 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 884 if (wpipe->pipe_state & PIPE_EOF) { 885 pipelock(wpipe, 0); 886 PIPE_UNLOCK(wpipe); 887 pipe_destroy_write_buffer(wpipe); 888 PIPE_LOCK(wpipe); 889 pipeselwakeup(wpipe); 890 pipeunlock(wpipe); 891 error = EPIPE; 892 goto error1; 893 } 894 if (wpipe->pipe_state & PIPE_WANTR) { 895 wpipe->pipe_state &= ~PIPE_WANTR; 896 wakeup(wpipe); 897 } 898 pipeselwakeup(wpipe); 899 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 900 "pipdwt", 0); 901 } 902 903 pipelock(wpipe,0); 904 if (wpipe->pipe_state & PIPE_DIRECTW) { 905 /* 906 * this bit of trickery substitutes a kernel buffer for 907 * the process that might be going away. 908 */ 909 pipe_clone_write_buffer(wpipe); 910 } else { 911 PIPE_UNLOCK(wpipe); 912 pipe_destroy_write_buffer(wpipe); 913 PIPE_LOCK(wpipe); 914 } 915 pipeunlock(wpipe); 916 return (error); 917 918error1: 919 wakeup(wpipe); 920 return (error); 921} 922#endif 923 924static int 925pipe_write(fp, uio, active_cred, flags, td) 926 struct file *fp; 927 struct uio *uio; 928 struct ucred *active_cred; 929 struct thread *td; 930 int flags; 931{ 932 int error = 0; 933 int orig_resid; 934 struct pipe *wpipe, *rpipe; 935 936 rpipe = fp->f_data; 937 wpipe = rpipe->pipe_peer; 938 939 PIPE_LOCK(rpipe); 940 /* 941 * detect loss of pipe read side, issue SIGPIPE if lost. 942 */ 943 if ((!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 944 PIPE_UNLOCK(rpipe); 945 return (EPIPE); 946 } 947#ifdef MAC 948 error = mac_check_pipe_write(active_cred, wpipe->pipe_pair); 949 if (error) { 950 PIPE_UNLOCK(rpipe); 951 return (error); 952 } 953#endif 954 ++wpipe->pipe_busy; 955 956 /* 957 * If it is advantageous to resize the pipe buffer, do 958 * so. 959 */ 960 if ((uio->uio_resid > PIPE_SIZE) && 961 (amountpipekva < maxpipekva / 2) && 962 (nbigpipe < LIMITBIGPIPES) && 963 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 964 (wpipe->pipe_buffer.size <= PIPE_SIZE) && 965 (wpipe->pipe_buffer.cnt == 0)) { 966 967 if ((error = pipelock(wpipe, 1)) == 0) { 968 PIPE_UNLOCK(wpipe); 969 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 970 atomic_add_int(&nbigpipe, 1); 971 PIPE_LOCK(wpipe); 972 pipeunlock(wpipe); 973 } 974 } 975 976 /* 977 * If an early error occured unbusy and return, waking up any pending 978 * readers. 979 */ 980 if (error) { 981 --wpipe->pipe_busy; 982 if ((wpipe->pipe_busy == 0) && 983 (wpipe->pipe_state & PIPE_WANT)) { 984 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 985 wakeup(wpipe); 986 } 987 PIPE_UNLOCK(rpipe); 988 return(error); 989 } 990 991 orig_resid = uio->uio_resid; 992 993 while (uio->uio_resid) { 994 int space; 995 996#ifndef PIPE_NODIRECT 997 /* 998 * If the transfer is large, we can gain performance if 999 * we do process-to-process copies directly. 1000 * If the write is non-blocking, we don't use the 1001 * direct write mechanism. 1002 * 1003 * The direct write mechanism will detect the reader going 1004 * away on us. 1005 */ 1006 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 1007 (fp->f_flag & FNONBLOCK) == 0 && 1008 amountpipekvawired + uio->uio_resid < maxpipekvawired) { 1009 error = pipe_direct_write(wpipe, uio); 1010 if (error) 1011 break; 1012 continue; 1013 } 1014#endif 1015 1016 /* 1017 * Pipe buffered writes cannot be coincidental with 1018 * direct writes. We wait until the currently executing 1019 * direct write is completed before we start filling the 1020 * pipe buffer. We break out if a signal occurs or the 1021 * reader goes away. 1022 */ 1023 retrywrite: 1024 while (wpipe->pipe_state & PIPE_DIRECTW) { 1025 if (wpipe->pipe_state & PIPE_WANTR) { 1026 wpipe->pipe_state &= ~PIPE_WANTR; 1027 wakeup(wpipe); 1028 } 1029 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1030 "pipbww", 0); 1031 if (wpipe->pipe_state & PIPE_EOF) 1032 break; 1033 if (error) 1034 break; 1035 } 1036 if (wpipe->pipe_state & PIPE_EOF) { 1037 error = EPIPE; 1038 break; 1039 } 1040 1041 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1042 1043 /* Writes of size <= PIPE_BUF must be atomic. */ 1044 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1045 space = 0; 1046 1047 if (space > 0) { 1048 if ((error = pipelock(wpipe,1)) == 0) { 1049 int size; /* Transfer size */ 1050 int segsize; /* first segment to transfer */ 1051 1052 /* 1053 * It is possible for a direct write to 1054 * slip in on us... handle it here... 1055 */ 1056 if (wpipe->pipe_state & PIPE_DIRECTW) { 1057 pipeunlock(wpipe); 1058 goto retrywrite; 1059 } 1060 /* 1061 * If a process blocked in uiomove, our 1062 * value for space might be bad. 1063 * 1064 * XXX will we be ok if the reader has gone 1065 * away here? 1066 */ 1067 if (space > wpipe->pipe_buffer.size - 1068 wpipe->pipe_buffer.cnt) { 1069 pipeunlock(wpipe); 1070 goto retrywrite; 1071 } 1072 1073 /* 1074 * Transfer size is minimum of uio transfer 1075 * and free space in pipe buffer. 1076 */ 1077 if (space > uio->uio_resid) 1078 size = uio->uio_resid; 1079 else 1080 size = space; 1081 /* 1082 * First segment to transfer is minimum of 1083 * transfer size and contiguous space in 1084 * pipe buffer. If first segment to transfer 1085 * is less than the transfer size, we've got 1086 * a wraparound in the buffer. 1087 */ 1088 segsize = wpipe->pipe_buffer.size - 1089 wpipe->pipe_buffer.in; 1090 if (segsize > size) 1091 segsize = size; 1092 1093 /* Transfer first segment */ 1094 1095 PIPE_UNLOCK(rpipe); 1096 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1097 segsize, uio); 1098 PIPE_LOCK(rpipe); 1099 1100 if (error == 0 && segsize < size) { 1101 /* 1102 * Transfer remaining part now, to 1103 * support atomic writes. Wraparound 1104 * happened. 1105 */ 1106 if (wpipe->pipe_buffer.in + segsize != 1107 wpipe->pipe_buffer.size) 1108 panic("Expected pipe buffer " 1109 "wraparound disappeared"); 1110 1111 PIPE_UNLOCK(rpipe); 1112 error = uiomove( 1113 &wpipe->pipe_buffer.buffer[0], 1114 size - segsize, uio); 1115 PIPE_LOCK(rpipe); 1116 } 1117 if (error == 0) { 1118 wpipe->pipe_buffer.in += size; 1119 if (wpipe->pipe_buffer.in >= 1120 wpipe->pipe_buffer.size) { 1121 if (wpipe->pipe_buffer.in != 1122 size - segsize + 1123 wpipe->pipe_buffer.size) 1124 panic("Expected " 1125 "wraparound bad"); 1126 wpipe->pipe_buffer.in = size - 1127 segsize; 1128 } 1129 1130 wpipe->pipe_buffer.cnt += size; 1131 if (wpipe->pipe_buffer.cnt > 1132 wpipe->pipe_buffer.size) 1133 panic("Pipe buffer overflow"); 1134 1135 } 1136 pipeunlock(wpipe); 1137 } 1138 if (error) 1139 break; 1140 1141 } else { 1142 /* 1143 * If the "read-side" has been blocked, wake it up now. 1144 */ 1145 if (wpipe->pipe_state & PIPE_WANTR) { 1146 wpipe->pipe_state &= ~PIPE_WANTR; 1147 wakeup(wpipe); 1148 } 1149 1150 /* 1151 * don't block on non-blocking I/O 1152 */ 1153 if (fp->f_flag & FNONBLOCK) { 1154 error = EAGAIN; 1155 break; 1156 } 1157 1158 /* 1159 * We have no more space and have something to offer, 1160 * wake up select/poll. 1161 */ 1162 pipeselwakeup(wpipe); 1163 1164 wpipe->pipe_state |= PIPE_WANTW; 1165 error = msleep(wpipe, PIPE_MTX(rpipe), 1166 PRIBIO | PCATCH, "pipewr", 0); 1167 if (error != 0) 1168 break; 1169 /* 1170 * If read side wants to go away, we just issue a signal 1171 * to ourselves. 1172 */ 1173 if (wpipe->pipe_state & PIPE_EOF) { 1174 error = EPIPE; 1175 break; 1176 } 1177 } 1178 } 1179 1180 --wpipe->pipe_busy; 1181 1182 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1183 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1184 wakeup(wpipe); 1185 } else if (wpipe->pipe_buffer.cnt > 0) { 1186 /* 1187 * If we have put any characters in the buffer, we wake up 1188 * the reader. 1189 */ 1190 if (wpipe->pipe_state & PIPE_WANTR) { 1191 wpipe->pipe_state &= ~PIPE_WANTR; 1192 wakeup(wpipe); 1193 } 1194 } 1195 1196 /* 1197 * Don't return EPIPE if I/O was successful 1198 */ 1199 if ((wpipe->pipe_buffer.cnt == 0) && 1200 (uio->uio_resid == 0) && 1201 (error == EPIPE)) { 1202 error = 0; 1203 } 1204 1205 if (error == 0) 1206 vfs_timestamp(&wpipe->pipe_mtime); 1207 1208 /* 1209 * We have something to offer, 1210 * wake up select/poll. 1211 */ 1212 if (wpipe->pipe_buffer.cnt) 1213 pipeselwakeup(wpipe); 1214 1215 PIPE_UNLOCK(rpipe); 1216 return (error); 1217} 1218 1219/* 1220 * we implement a very minimal set of ioctls for compatibility with sockets. 1221 */ 1222static int 1223pipe_ioctl(fp, cmd, data, active_cred, td) 1224 struct file *fp; 1225 u_long cmd; 1226 void *data; 1227 struct ucred *active_cred; 1228 struct thread *td; 1229{ 1230 struct pipe *mpipe = fp->f_data; 1231#ifdef MAC 1232 int error; 1233#endif 1234 1235 PIPE_LOCK(mpipe); 1236 1237#ifdef MAC 1238 error = mac_check_pipe_ioctl(active_cred, mpipe->pipe_pair, cmd, data); 1239 if (error) { 1240 PIPE_UNLOCK(mpipe); 1241 return (error); 1242 } 1243#endif 1244 1245 switch (cmd) { 1246 1247 case FIONBIO: 1248 PIPE_UNLOCK(mpipe); 1249 return (0); 1250 1251 case FIOASYNC: 1252 if (*(int *)data) { 1253 mpipe->pipe_state |= PIPE_ASYNC; 1254 } else { 1255 mpipe->pipe_state &= ~PIPE_ASYNC; 1256 } 1257 PIPE_UNLOCK(mpipe); 1258 return (0); 1259 1260 case FIONREAD: 1261 if (mpipe->pipe_state & PIPE_DIRECTW) 1262 *(int *)data = mpipe->pipe_map.cnt; 1263 else 1264 *(int *)data = mpipe->pipe_buffer.cnt; 1265 PIPE_UNLOCK(mpipe); 1266 return (0); 1267 1268 case FIOSETOWN: 1269 PIPE_UNLOCK(mpipe); 1270 return (fsetown(*(int *)data, &mpipe->pipe_sigio)); 1271 1272 case FIOGETOWN: 1273 PIPE_UNLOCK(mpipe); 1274 *(int *)data = fgetown(&mpipe->pipe_sigio); 1275 return (0); 1276 1277 /* This is deprecated, FIOSETOWN should be used instead. */ 1278 case TIOCSPGRP: 1279 PIPE_UNLOCK(mpipe); 1280 return (fsetown(-(*(int *)data), &mpipe->pipe_sigio)); 1281 1282 /* This is deprecated, FIOGETOWN should be used instead. */ 1283 case TIOCGPGRP: 1284 PIPE_UNLOCK(mpipe); 1285 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1286 return (0); 1287 1288 } 1289 PIPE_UNLOCK(mpipe); 1290 return (ENOTTY); 1291} 1292 1293static int 1294pipe_poll(fp, events, active_cred, td) 1295 struct file *fp; 1296 int events; 1297 struct ucred *active_cred; 1298 struct thread *td; 1299{ 1300 struct pipe *rpipe = fp->f_data; 1301 struct pipe *wpipe; 1302 int revents = 0; 1303#ifdef MAC 1304 int error; 1305#endif 1306 1307 wpipe = rpipe->pipe_peer; 1308 PIPE_LOCK(rpipe); 1309#ifdef MAC 1310 error = mac_check_pipe_poll(active_cred, rpipe->pipe_pair); 1311 if (error) 1312 goto locked_error; 1313#endif 1314 if (events & (POLLIN | POLLRDNORM)) 1315 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1316 (rpipe->pipe_buffer.cnt > 0) || 1317 (rpipe->pipe_state & PIPE_EOF)) 1318 revents |= events & (POLLIN | POLLRDNORM); 1319 1320 if (events & (POLLOUT | POLLWRNORM)) 1321 if (!wpipe->pipe_present || (wpipe->pipe_state & PIPE_EOF) || 1322 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1323 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1324 revents |= events & (POLLOUT | POLLWRNORM); 1325 1326 if ((rpipe->pipe_state & PIPE_EOF) || 1327 (!wpipe->pipe_present) || 1328 (wpipe->pipe_state & PIPE_EOF)) 1329 revents |= POLLHUP; 1330 1331 if (revents == 0) { 1332 if (events & (POLLIN | POLLRDNORM)) { 1333 selrecord(td, &rpipe->pipe_sel); 1334 rpipe->pipe_state |= PIPE_SEL; 1335 } 1336 1337 if (events & (POLLOUT | POLLWRNORM)) { 1338 selrecord(td, &wpipe->pipe_sel); 1339 wpipe->pipe_state |= PIPE_SEL; 1340 } 1341 } 1342#ifdef MAC 1343locked_error: 1344#endif 1345 PIPE_UNLOCK(rpipe); 1346 1347 return (revents); 1348} 1349 1350/* 1351 * We shouldn't need locks here as we're doing a read and this should 1352 * be a natural race. 1353 */ 1354static int 1355pipe_stat(fp, ub, active_cred, td) 1356 struct file *fp; 1357 struct stat *ub; 1358 struct ucred *active_cred; 1359 struct thread *td; 1360{ 1361 struct pipe *pipe = fp->f_data; 1362#ifdef MAC 1363 int error; 1364 1365 PIPE_LOCK(pipe); 1366 error = mac_check_pipe_stat(active_cred, pipe->pipe_pair); 1367 PIPE_UNLOCK(pipe); 1368 if (error) 1369 return (error); 1370#endif 1371 bzero(ub, sizeof(*ub)); 1372 ub->st_mode = S_IFIFO; 1373 ub->st_blksize = pipe->pipe_buffer.size; 1374 ub->st_size = pipe->pipe_buffer.cnt; 1375 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1376 ub->st_atimespec = pipe->pipe_atime; 1377 ub->st_mtimespec = pipe->pipe_mtime; 1378 ub->st_ctimespec = pipe->pipe_ctime; 1379 ub->st_uid = fp->f_cred->cr_uid; 1380 ub->st_gid = fp->f_cred->cr_gid; 1381 /* 1382 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1383 * XXX (st_dev, st_ino) should be unique. 1384 */ 1385 return (0); 1386} 1387 1388/* ARGSUSED */ 1389static int 1390pipe_close(fp, td) 1391 struct file *fp; 1392 struct thread *td; 1393{ 1394 struct pipe *cpipe = fp->f_data; 1395 1396 fp->f_ops = &badfileops; 1397 fp->f_data = NULL; 1398 funsetown(&cpipe->pipe_sigio); 1399 pipeclose(cpipe); 1400 return (0); 1401} 1402 1403static void 1404pipe_free_kmem(cpipe) 1405 struct pipe *cpipe; 1406{ 1407 1408 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), 1409 ("pipe_free_kmem: pipe mutex locked")); 1410 1411 if (cpipe->pipe_buffer.buffer != NULL) { 1412 if (cpipe->pipe_buffer.size > PIPE_SIZE) 1413 atomic_subtract_int(&nbigpipe, 1); 1414 atomic_subtract_int(&amountpipekva, cpipe->pipe_buffer.size); 1415 vm_map_remove(pipe_map, 1416 (vm_offset_t)cpipe->pipe_buffer.buffer, 1417 (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size); 1418 cpipe->pipe_buffer.buffer = NULL; 1419 } 1420#ifndef PIPE_NODIRECT 1421 if (cpipe->pipe_map.kva != 0) { 1422 atomic_subtract_int(&amountpipekvawired, 1423 cpipe->pipe_buffer.size + PAGE_SIZE); 1424 kmem_free(kernel_map, 1425 cpipe->pipe_map.kva, 1426 cpipe->pipe_buffer.size + PAGE_SIZE); 1427 cpipe->pipe_map.cnt = 0; 1428 cpipe->pipe_map.kva = 0; 1429 cpipe->pipe_map.pos = 0; 1430 cpipe->pipe_map.npages = 0; 1431 } 1432#endif 1433} 1434 1435/* 1436 * shutdown the pipe 1437 */ 1438static void 1439pipeclose(cpipe) 1440 struct pipe *cpipe; 1441{ 1442 struct pipepair *pp; 1443 struct pipe *ppipe; 1444 int hadpeer; 1445 1446 KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL")); 1447 1448 hadpeer = 0; 1449 PIPE_LOCK(cpipe); 1450 pp = cpipe->pipe_pair; 1451 1452 pipeselwakeup(cpipe); 1453 1454 /* 1455 * If the other side is blocked, wake it up saying that 1456 * we want to close it down. 1457 */ 1458 while (cpipe->pipe_busy) { 1459 wakeup(cpipe); 1460 cpipe->pipe_state |= PIPE_WANT | PIPE_EOF; 1461 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1462 } 1463 1464 1465 /* 1466 * Disconnect from peer, if any. 1467 */ 1468 ppipe = cpipe->pipe_peer; 1469 if (ppipe->pipe_present != 0) { 1470 hadpeer++; 1471 pipeselwakeup(ppipe); 1472 1473 ppipe->pipe_state |= PIPE_EOF; 1474 wakeup(ppipe); 1475 KNOTE(&ppipe->pipe_sel.si_note, 0); 1476 } 1477 1478 /* 1479 * Mark this endpoint as free. Release kmem resources. We 1480 * don't mark this endpoint as unused until we've finished 1481 * doing that, or the pipe might disappear out from under 1482 * us. 1483 */ 1484 PIPE_UNLOCK(cpipe); 1485 pipe_free_kmem(cpipe); 1486 PIPE_LOCK(cpipe); 1487 cpipe->pipe_present = 0; 1488 1489 /* 1490 * If both endpoints are now closed, release the memory for the 1491 * pipe pair. If not, unlock. 1492 */ 1493 if (ppipe->pipe_present == 0) { 1494 PIPE_UNLOCK(cpipe); 1495#ifdef MAC 1496 mac_destroy_pipe(pp); 1497#endif 1498 uma_zfree(pipe_zone, cpipe->pipe_pair); 1499 } else 1500 PIPE_UNLOCK(cpipe); 1501} 1502 1503/*ARGSUSED*/ 1504static int 1505pipe_kqfilter(struct file *fp, struct knote *kn) 1506{ 1507 struct pipe *cpipe; 1508 1509 cpipe = kn->kn_fp->f_data; 1510 switch (kn->kn_filter) { 1511 case EVFILT_READ: 1512 kn->kn_fop = &pipe_rfiltops; 1513 break; 1514 case EVFILT_WRITE: 1515 kn->kn_fop = &pipe_wfiltops; 1516 cpipe = cpipe->pipe_peer; 1517 if (!cpipe->pipe_present) 1518 /* other end of pipe has been closed */ 1519 return (EPIPE); 1520 break; 1521 default: 1522 return (1); 1523 } 1524 1525 PIPE_LOCK(cpipe); 1526 SLIST_INSERT_HEAD(&cpipe->pipe_sel.si_note, kn, kn_selnext); 1527 PIPE_UNLOCK(cpipe); 1528 return (0); 1529} 1530 1531static void 1532filt_pipedetach(struct knote *kn) 1533{ 1534 struct pipe *cpipe = (struct pipe *)kn->kn_fp->f_data; 1535 1536 if (kn->kn_filter == EVFILT_WRITE) { 1537 if (cpipe->pipe_peer == NULL) 1538 return; 1539 cpipe = cpipe->pipe_peer; 1540 } 1541 1542 PIPE_LOCK(cpipe); 1543 SLIST_REMOVE(&cpipe->pipe_sel.si_note, kn, knote, kn_selnext); 1544 PIPE_UNLOCK(cpipe); 1545} 1546 1547/*ARGSUSED*/ 1548static int 1549filt_piperead(struct knote *kn, long hint) 1550{ 1551 struct pipe *rpipe = kn->kn_fp->f_data; 1552 struct pipe *wpipe = rpipe->pipe_peer; 1553 1554 PIPE_LOCK(rpipe); 1555 kn->kn_data = rpipe->pipe_buffer.cnt; 1556 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1557 kn->kn_data = rpipe->pipe_map.cnt; 1558 1559 if ((rpipe->pipe_state & PIPE_EOF) || 1560 (!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 1561 kn->kn_flags |= EV_EOF; 1562 PIPE_UNLOCK(rpipe); 1563 return (1); 1564 } 1565 PIPE_UNLOCK(rpipe); 1566 return (kn->kn_data > 0); 1567} 1568 1569/*ARGSUSED*/ 1570static int 1571filt_pipewrite(struct knote *kn, long hint) 1572{ 1573 struct pipe *rpipe = kn->kn_fp->f_data; 1574 struct pipe *wpipe = rpipe->pipe_peer; 1575 1576 PIPE_LOCK(rpipe); 1577 if ((!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 1578 kn->kn_data = 0; 1579 kn->kn_flags |= EV_EOF; 1580 PIPE_UNLOCK(rpipe); 1581 return (1); 1582 } 1583 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1584 if (wpipe->pipe_state & PIPE_DIRECTW) 1585 kn->kn_data = 0; 1586 1587 PIPE_UNLOCK(rpipe); 1588 return (kn->kn_data >= PIPE_BUF); 1589} 1590