sys_pipe.c revision 232641
1/*- 2 * Copyright (c) 1996 John S. Dyson 3 * Copyright (c) 2012 Giovanni Trematerra 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice immediately at the beginning of the file, without modification, 11 * this list of conditions, and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Absolutely no warranty of function or purpose is made by the author 16 * John S. Dyson. 17 * 4. Modifications may be freely made to this file if the above conditions 18 * are met. 19 */ 20 21/* 22 * This file contains a high-performance replacement for the socket-based 23 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 24 * all features of sockets, but does do everything that pipes normally 25 * do. 26 */ 27 28/* 29 * This code has two modes of operation, a small write mode and a large 30 * write mode. The small write mode acts like conventional pipes with 31 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 32 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 33 * and PIPE_SIZE in size, the sending process pins the underlying pages in 34 * memory, and the receiving process copies directly from these pinned pages 35 * in the sending process. 36 * 37 * If the sending process receives a signal, it is possible that it will 38 * go away, and certainly its address space can change, because control 39 * is returned back to the user-mode side. In that case, the pipe code 40 * arranges to copy the buffer supplied by the user process, to a pageable 41 * kernel buffer, and the receiving process will grab the data from the 42 * pageable kernel buffer. Since signals don't happen all that often, 43 * the copy operation is normally eliminated. 44 * 45 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 46 * happen for small transfers so that the system will not spend all of 47 * its time context switching. 48 * 49 * In order to limit the resource use of pipes, two sysctls exist: 50 * 51 * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable 52 * address space available to us in pipe_map. This value is normally 53 * autotuned, but may also be loader tuned. 54 * 55 * kern.ipc.pipekva - This read-only sysctl tracks the current amount of 56 * memory in use by pipes. 57 * 58 * Based on how large pipekva is relative to maxpipekva, the following 59 * will happen: 60 * 61 * 0% - 50%: 62 * New pipes are given 16K of memory backing, pipes may dynamically 63 * grow to as large as 64K where needed. 64 * 50% - 75%: 65 * New pipes are given 4K (or PAGE_SIZE) of memory backing, 66 * existing pipes may NOT grow. 67 * 75% - 100%: 68 * New pipes are given 4K (or PAGE_SIZE) of memory backing, 69 * existing pipes will be shrunk down to 4K whenever possible. 70 * 71 * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0. If 72 * that is set, the only resize that will occur is the 0 -> SMALL_PIPE_SIZE 73 * resize which MUST occur for reverse-direction pipes when they are 74 * first used. 75 * 76 * Additional information about the current state of pipes may be obtained 77 * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail, 78 * and kern.ipc.piperesizefail. 79 * 80 * Locking rules: There are two locks present here: A mutex, used via 81 * PIPE_LOCK, and a flag, used via pipelock(). All locking is done via 82 * the flag, as mutexes can not persist over uiomove. The mutex 83 * exists only to guard access to the flag, and is not in itself a 84 * locking mechanism. Also note that there is only a single mutex for 85 * both directions of a pipe. 86 * 87 * As pipelock() may have to sleep before it can acquire the flag, it 88 * is important to reread all data after a call to pipelock(); everything 89 * in the structure may have changed. 90 */ 91 92#include <sys/cdefs.h> 93__FBSDID("$FreeBSD: head/sys/kern/sys_pipe.c 232641 2012-03-07 07:31:50Z kib $"); 94 95#include <sys/param.h> 96#include <sys/systm.h> 97#include <sys/conf.h> 98#include <sys/fcntl.h> 99#include <sys/file.h> 100#include <sys/filedesc.h> 101#include <sys/filio.h> 102#include <sys/kernel.h> 103#include <sys/lock.h> 104#include <sys/mutex.h> 105#include <sys/ttycom.h> 106#include <sys/stat.h> 107#include <sys/malloc.h> 108#include <sys/poll.h> 109#include <sys/selinfo.h> 110#include <sys/signalvar.h> 111#include <sys/syscallsubr.h> 112#include <sys/sysctl.h> 113#include <sys/sysproto.h> 114#include <sys/pipe.h> 115#include <sys/proc.h> 116#include <sys/vnode.h> 117#include <sys/uio.h> 118#include <sys/event.h> 119 120#include <security/mac/mac_framework.h> 121 122#include <vm/vm.h> 123#include <vm/vm_param.h> 124#include <vm/vm_object.h> 125#include <vm/vm_kern.h> 126#include <vm/vm_extern.h> 127#include <vm/pmap.h> 128#include <vm/vm_map.h> 129#include <vm/vm_page.h> 130#include <vm/uma.h> 131 132#include <fs/fifofs/fifo.h> 133 134/* 135 * Use this define if you want to disable *fancy* VM things. Expect an 136 * approx 30% decrease in transfer rate. This could be useful for 137 * NetBSD or OpenBSD. 138 */ 139/* #define PIPE_NODIRECT */ 140 141#define PIPE_PEER(pipe) \ 142 (((pipe)->pipe_state & PIPE_NAMED) ? (pipe) : ((pipe)->pipe_peer)) 143 144/* 145 * interfaces to the outside world 146 */ 147static fo_rdwr_t pipe_read; 148static fo_rdwr_t pipe_write; 149static fo_truncate_t pipe_truncate; 150static fo_ioctl_t pipe_ioctl; 151static fo_poll_t pipe_poll; 152static fo_kqfilter_t pipe_kqfilter; 153static fo_stat_t pipe_stat; 154static fo_close_t pipe_close; 155static fo_chmod_t pipe_chmod; 156static fo_chown_t pipe_chown; 157 158struct fileops pipeops = { 159 .fo_read = pipe_read, 160 .fo_write = pipe_write, 161 .fo_truncate = pipe_truncate, 162 .fo_ioctl = pipe_ioctl, 163 .fo_poll = pipe_poll, 164 .fo_kqfilter = pipe_kqfilter, 165 .fo_stat = pipe_stat, 166 .fo_close = pipe_close, 167 .fo_chmod = pipe_chmod, 168 .fo_chown = pipe_chown, 169 .fo_flags = DFLAG_PASSABLE 170}; 171 172static void filt_pipedetach(struct knote *kn); 173static void filt_pipedetach_notsup(struct knote *kn); 174static int filt_pipenotsup(struct knote *kn, long hint); 175static int filt_piperead(struct knote *kn, long hint); 176static int filt_pipewrite(struct knote *kn, long hint); 177 178static struct filterops pipe_nfiltops = { 179 .f_isfd = 1, 180 .f_detach = filt_pipedetach_notsup, 181 .f_event = filt_pipenotsup 182}; 183static struct filterops pipe_rfiltops = { 184 .f_isfd = 1, 185 .f_detach = filt_pipedetach, 186 .f_event = filt_piperead 187}; 188static struct filterops pipe_wfiltops = { 189 .f_isfd = 1, 190 .f_detach = filt_pipedetach, 191 .f_event = filt_pipewrite 192}; 193 194/* 195 * Default pipe buffer size(s), this can be kind-of large now because pipe 196 * space is pageable. The pipe code will try to maintain locality of 197 * reference for performance reasons, so small amounts of outstanding I/O 198 * will not wipe the cache. 199 */ 200#define MINPIPESIZE (PIPE_SIZE/3) 201#define MAXPIPESIZE (2*PIPE_SIZE/3) 202 203static long amountpipekva; 204static int pipefragretry; 205static int pipeallocfail; 206static int piperesizefail; 207static int piperesizeallowed = 1; 208 209SYSCTL_LONG(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN, 210 &maxpipekva, 0, "Pipe KVA limit"); 211SYSCTL_LONG(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 212 &amountpipekva, 0, "Pipe KVA usage"); 213SYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD, 214 &pipefragretry, 0, "Pipe allocation retries due to fragmentation"); 215SYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD, 216 &pipeallocfail, 0, "Pipe allocation failures"); 217SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD, 218 &piperesizefail, 0, "Pipe resize failures"); 219SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW, 220 &piperesizeallowed, 0, "Pipe resizing allowed"); 221 222static void pipeinit(void *dummy __unused); 223static void pipeclose(struct pipe *cpipe); 224static void pipe_free_kmem(struct pipe *cpipe); 225static int pipe_create(struct pipe *pipe, int backing); 226static int pipe_paircreate(struct thread *td, struct pipepair **p_pp); 227static __inline int pipelock(struct pipe *cpipe, int catch); 228static __inline void pipeunlock(struct pipe *cpipe); 229static __inline void pipeselwakeup(struct pipe *cpipe); 230#ifndef PIPE_NODIRECT 231static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 232static void pipe_destroy_write_buffer(struct pipe *wpipe); 233static int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 234static void pipe_clone_write_buffer(struct pipe *wpipe); 235#endif 236static int pipespace(struct pipe *cpipe, int size); 237static int pipespace_new(struct pipe *cpipe, int size); 238 239static int pipe_zone_ctor(void *mem, int size, void *arg, int flags); 240static int pipe_zone_init(void *mem, int size, int flags); 241static void pipe_zone_fini(void *mem, int size); 242 243static uma_zone_t pipe_zone; 244static struct unrhdr *pipeino_unr; 245static dev_t pipedev_ino; 246 247SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 248 249static void 250pipeinit(void *dummy __unused) 251{ 252 253 pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair), 254 pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini, 255 UMA_ALIGN_PTR, 0); 256 KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 257 pipeino_unr = new_unrhdr(1, INT32_MAX, NULL); 258 KASSERT(pipeino_unr != NULL, ("pipe fake inodes not initialized")); 259 pipedev_ino = devfs_alloc_cdp_inode(); 260 KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized")); 261} 262 263static int 264pipe_zone_ctor(void *mem, int size, void *arg, int flags) 265{ 266 struct pipepair *pp; 267 struct pipe *rpipe, *wpipe; 268 269 KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size")); 270 271 pp = (struct pipepair *)mem; 272 273 /* 274 * We zero both pipe endpoints to make sure all the kmem pointers 275 * are NULL, flag fields are zero'd, etc. We timestamp both 276 * endpoints with the same time. 277 */ 278 rpipe = &pp->pp_rpipe; 279 bzero(rpipe, sizeof(*rpipe)); 280 vfs_timestamp(&rpipe->pipe_ctime); 281 rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime; 282 283 wpipe = &pp->pp_wpipe; 284 bzero(wpipe, sizeof(*wpipe)); 285 wpipe->pipe_ctime = rpipe->pipe_ctime; 286 wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime; 287 288 rpipe->pipe_peer = wpipe; 289 rpipe->pipe_pair = pp; 290 wpipe->pipe_peer = rpipe; 291 wpipe->pipe_pair = pp; 292 293 /* 294 * Mark both endpoints as present; they will later get free'd 295 * one at a time. When both are free'd, then the whole pair 296 * is released. 297 */ 298 rpipe->pipe_present = PIPE_ACTIVE; 299 wpipe->pipe_present = PIPE_ACTIVE; 300 301 /* 302 * Eventually, the MAC Framework may initialize the label 303 * in ctor or init, but for now we do it elswhere to avoid 304 * blocking in ctor or init. 305 */ 306 pp->pp_label = NULL; 307 308 return (0); 309} 310 311static int 312pipe_zone_init(void *mem, int size, int flags) 313{ 314 struct pipepair *pp; 315 316 KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size")); 317 318 pp = (struct pipepair *)mem; 319 320 mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 321 return (0); 322} 323 324static void 325pipe_zone_fini(void *mem, int size) 326{ 327 struct pipepair *pp; 328 329 KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size")); 330 331 pp = (struct pipepair *)mem; 332 333 mtx_destroy(&pp->pp_mtx); 334} 335 336static int 337pipe_paircreate(struct thread *td, struct pipepair **p_pp) 338{ 339 struct pipepair *pp; 340 struct pipe *rpipe, *wpipe; 341 int error; 342 343 *p_pp = pp = uma_zalloc(pipe_zone, M_WAITOK); 344#ifdef MAC 345 /* 346 * The MAC label is shared between the connected endpoints. As a 347 * result mac_pipe_init() and mac_pipe_create() are called once 348 * for the pair, and not on the endpoints. 349 */ 350 mac_pipe_init(pp); 351 mac_pipe_create(td->td_ucred, pp); 352#endif 353 rpipe = &pp->pp_rpipe; 354 wpipe = &pp->pp_wpipe; 355 356 knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe)); 357 knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe)); 358 359 /* Only the forward direction pipe is backed by default */ 360 if ((error = pipe_create(rpipe, 1)) != 0 || 361 (error = pipe_create(wpipe, 0)) != 0) { 362 pipeclose(rpipe); 363 pipeclose(wpipe); 364 return (error); 365 } 366 367 rpipe->pipe_state |= PIPE_DIRECTOK; 368 wpipe->pipe_state |= PIPE_DIRECTOK; 369 return (0); 370} 371 372int 373pipe_named_ctor(struct pipe **ppipe, struct thread *td) 374{ 375 struct pipepair *pp; 376 int error; 377 378 error = pipe_paircreate(td, &pp); 379 if (error != 0) 380 return (error); 381 pp->pp_rpipe.pipe_state |= PIPE_NAMED; 382 *ppipe = &pp->pp_rpipe; 383 return (0); 384} 385 386void 387pipe_dtor(struct pipe *dpipe) 388{ 389 ino_t ino; 390 391 ino = dpipe->pipe_ino; 392 funsetown(&dpipe->pipe_sigio); 393 pipeclose(dpipe); 394 if (dpipe->pipe_state & PIPE_NAMED) { 395 dpipe = dpipe->pipe_peer; 396 funsetown(&dpipe->pipe_sigio); 397 pipeclose(dpipe); 398 } 399 if (ino != 0 && ino != (ino_t)-1) 400 free_unr(pipeino_unr, ino); 401} 402 403/* 404 * The pipe system call for the DTYPE_PIPE type of pipes. If we fail, let 405 * the zone pick up the pieces via pipeclose(). 406 */ 407int 408kern_pipe(struct thread *td, int fildes[2]) 409{ 410 struct filedesc *fdp; 411 struct file *rf, *wf; 412 struct pipe *rpipe, *wpipe; 413 struct pipepair *pp; 414 int fd, error; 415 416 fdp = td->td_proc->p_fd; 417 error = pipe_paircreate(td, &pp); 418 if (error != 0) 419 return (error); 420 rpipe = &pp->pp_rpipe; 421 wpipe = &pp->pp_wpipe; 422 error = falloc(td, &rf, &fd, 0); 423 if (error) { 424 pipeclose(rpipe); 425 pipeclose(wpipe); 426 return (error); 427 } 428 /* An extra reference on `rf' has been held for us by falloc(). */ 429 fildes[0] = fd; 430 431 /* 432 * Warning: once we've gotten past allocation of the fd for the 433 * read-side, we can only drop the read side via fdrop() in order 434 * to avoid races against processes which manage to dup() the read 435 * side while we are blocked trying to allocate the write side. 436 */ 437 finit(rf, FREAD | FWRITE, DTYPE_PIPE, rpipe, &pipeops); 438 error = falloc(td, &wf, &fd, 0); 439 if (error) { 440 fdclose(fdp, rf, fildes[0], td); 441 fdrop(rf, td); 442 /* rpipe has been closed by fdrop(). */ 443 pipeclose(wpipe); 444 return (error); 445 } 446 /* An extra reference on `wf' has been held for us by falloc(). */ 447 finit(wf, FREAD | FWRITE, DTYPE_PIPE, wpipe, &pipeops); 448 fdrop(wf, td); 449 fildes[1] = fd; 450 fdrop(rf, td); 451 452 return (0); 453} 454 455/* ARGSUSED */ 456int 457sys_pipe(struct thread *td, struct pipe_args *uap) 458{ 459 int error; 460 int fildes[2]; 461 462 error = kern_pipe(td, fildes); 463 if (error) 464 return (error); 465 466 td->td_retval[0] = fildes[0]; 467 td->td_retval[1] = fildes[1]; 468 469 return (0); 470} 471 472/* 473 * Allocate kva for pipe circular buffer, the space is pageable 474 * This routine will 'realloc' the size of a pipe safely, if it fails 475 * it will retain the old buffer. 476 * If it fails it will return ENOMEM. 477 */ 478static int 479pipespace_new(cpipe, size) 480 struct pipe *cpipe; 481 int size; 482{ 483 caddr_t buffer; 484 int error, cnt, firstseg; 485 static int curfail = 0; 486 static struct timeval lastfail; 487 488 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked")); 489 KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW), 490 ("pipespace: resize of direct writes not allowed")); 491retry: 492 cnt = cpipe->pipe_buffer.cnt; 493 if (cnt > size) 494 size = cnt; 495 496 size = round_page(size); 497 buffer = (caddr_t) vm_map_min(pipe_map); 498 499 error = vm_map_find(pipe_map, NULL, 0, 500 (vm_offset_t *) &buffer, size, 1, 501 VM_PROT_ALL, VM_PROT_ALL, 0); 502 if (error != KERN_SUCCESS) { 503 if ((cpipe->pipe_buffer.buffer == NULL) && 504 (size > SMALL_PIPE_SIZE)) { 505 size = SMALL_PIPE_SIZE; 506 pipefragretry++; 507 goto retry; 508 } 509 if (cpipe->pipe_buffer.buffer == NULL) { 510 pipeallocfail++; 511 if (ppsratecheck(&lastfail, &curfail, 1)) 512 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n"); 513 } else { 514 piperesizefail++; 515 } 516 return (ENOMEM); 517 } 518 519 /* copy data, then free old resources if we're resizing */ 520 if (cnt > 0) { 521 if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) { 522 firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out; 523 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 524 buffer, firstseg); 525 if ((cnt - firstseg) > 0) 526 bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg], 527 cpipe->pipe_buffer.in); 528 } else { 529 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 530 buffer, cnt); 531 } 532 } 533 pipe_free_kmem(cpipe); 534 cpipe->pipe_buffer.buffer = buffer; 535 cpipe->pipe_buffer.size = size; 536 cpipe->pipe_buffer.in = cnt; 537 cpipe->pipe_buffer.out = 0; 538 cpipe->pipe_buffer.cnt = cnt; 539 atomic_add_long(&amountpipekva, cpipe->pipe_buffer.size); 540 return (0); 541} 542 543/* 544 * Wrapper for pipespace_new() that performs locking assertions. 545 */ 546static int 547pipespace(cpipe, size) 548 struct pipe *cpipe; 549 int size; 550{ 551 552 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 553 ("Unlocked pipe passed to pipespace")); 554 return (pipespace_new(cpipe, size)); 555} 556 557/* 558 * lock a pipe for I/O, blocking other access 559 */ 560static __inline int 561pipelock(cpipe, catch) 562 struct pipe *cpipe; 563 int catch; 564{ 565 int error; 566 567 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 568 while (cpipe->pipe_state & PIPE_LOCKFL) { 569 cpipe->pipe_state |= PIPE_LWANT; 570 error = msleep(cpipe, PIPE_MTX(cpipe), 571 catch ? (PRIBIO | PCATCH) : PRIBIO, 572 "pipelk", 0); 573 if (error != 0) 574 return (error); 575 } 576 cpipe->pipe_state |= PIPE_LOCKFL; 577 return (0); 578} 579 580/* 581 * unlock a pipe I/O lock 582 */ 583static __inline void 584pipeunlock(cpipe) 585 struct pipe *cpipe; 586{ 587 588 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 589 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 590 ("Unlocked pipe passed to pipeunlock")); 591 cpipe->pipe_state &= ~PIPE_LOCKFL; 592 if (cpipe->pipe_state & PIPE_LWANT) { 593 cpipe->pipe_state &= ~PIPE_LWANT; 594 wakeup(cpipe); 595 } 596} 597 598static __inline void 599pipeselwakeup(cpipe) 600 struct pipe *cpipe; 601{ 602 603 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 604 if (cpipe->pipe_state & PIPE_SEL) { 605 selwakeuppri(&cpipe->pipe_sel, PSOCK); 606 if (!SEL_WAITING(&cpipe->pipe_sel)) 607 cpipe->pipe_state &= ~PIPE_SEL; 608 } 609 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 610 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 611 KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0); 612} 613 614/* 615 * Initialize and allocate VM and memory for pipe. The structure 616 * will start out zero'd from the ctor, so we just manage the kmem. 617 */ 618static int 619pipe_create(pipe, backing) 620 struct pipe *pipe; 621 int backing; 622{ 623 int error; 624 625 if (backing) { 626 if (amountpipekva > maxpipekva / 2) 627 error = pipespace_new(pipe, SMALL_PIPE_SIZE); 628 else 629 error = pipespace_new(pipe, PIPE_SIZE); 630 } else { 631 /* If we're not backing this pipe, no need to do anything. */ 632 error = 0; 633 } 634 pipe->pipe_ino = -1; 635 return (error); 636} 637 638/* ARGSUSED */ 639static int 640pipe_read(fp, uio, active_cred, flags, td) 641 struct file *fp; 642 struct uio *uio; 643 struct ucred *active_cred; 644 struct thread *td; 645 int flags; 646{ 647 struct pipe *rpipe; 648 int error; 649 int nread = 0; 650 int size; 651 652 rpipe = fp->f_data; 653 PIPE_LOCK(rpipe); 654 ++rpipe->pipe_busy; 655 error = pipelock(rpipe, 1); 656 if (error) 657 goto unlocked_error; 658 659#ifdef MAC 660 error = mac_pipe_check_read(active_cred, rpipe->pipe_pair); 661 if (error) 662 goto locked_error; 663#endif 664 if (amountpipekva > (3 * maxpipekva) / 4) { 665 if (!(rpipe->pipe_state & PIPE_DIRECTW) && 666 (rpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 667 (rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 668 (piperesizeallowed == 1)) { 669 PIPE_UNLOCK(rpipe); 670 pipespace(rpipe, SMALL_PIPE_SIZE); 671 PIPE_LOCK(rpipe); 672 } 673 } 674 675 while (uio->uio_resid) { 676 /* 677 * normal pipe buffer receive 678 */ 679 if (rpipe->pipe_buffer.cnt > 0) { 680 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 681 if (size > rpipe->pipe_buffer.cnt) 682 size = rpipe->pipe_buffer.cnt; 683 if (size > uio->uio_resid) 684 size = uio->uio_resid; 685 686 PIPE_UNLOCK(rpipe); 687 error = uiomove( 688 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 689 size, uio); 690 PIPE_LOCK(rpipe); 691 if (error) 692 break; 693 694 rpipe->pipe_buffer.out += size; 695 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 696 rpipe->pipe_buffer.out = 0; 697 698 rpipe->pipe_buffer.cnt -= size; 699 700 /* 701 * If there is no more to read in the pipe, reset 702 * its pointers to the beginning. This improves 703 * cache hit stats. 704 */ 705 if (rpipe->pipe_buffer.cnt == 0) { 706 rpipe->pipe_buffer.in = 0; 707 rpipe->pipe_buffer.out = 0; 708 } 709 nread += size; 710#ifndef PIPE_NODIRECT 711 /* 712 * Direct copy, bypassing a kernel buffer. 713 */ 714 } else if ((size = rpipe->pipe_map.cnt) && 715 (rpipe->pipe_state & PIPE_DIRECTW)) { 716 if (size > uio->uio_resid) 717 size = (u_int) uio->uio_resid; 718 719 PIPE_UNLOCK(rpipe); 720 error = uiomove_fromphys(rpipe->pipe_map.ms, 721 rpipe->pipe_map.pos, size, uio); 722 PIPE_LOCK(rpipe); 723 if (error) 724 break; 725 nread += size; 726 rpipe->pipe_map.pos += size; 727 rpipe->pipe_map.cnt -= size; 728 if (rpipe->pipe_map.cnt == 0) { 729 rpipe->pipe_state &= ~PIPE_DIRECTW; 730 wakeup(rpipe); 731 } 732#endif 733 } else { 734 /* 735 * detect EOF condition 736 * read returns 0 on EOF, no need to set error 737 */ 738 if (rpipe->pipe_state & PIPE_EOF) 739 break; 740 741 /* 742 * If the "write-side" has been blocked, wake it up now. 743 */ 744 if (rpipe->pipe_state & PIPE_WANTW) { 745 rpipe->pipe_state &= ~PIPE_WANTW; 746 wakeup(rpipe); 747 } 748 749 /* 750 * Break if some data was read. 751 */ 752 if (nread > 0) 753 break; 754 755 /* 756 * Unlock the pipe buffer for our remaining processing. 757 * We will either break out with an error or we will 758 * sleep and relock to loop. 759 */ 760 pipeunlock(rpipe); 761 762 /* 763 * Handle non-blocking mode operation or 764 * wait for more data. 765 */ 766 if (fp->f_flag & FNONBLOCK) { 767 error = EAGAIN; 768 } else { 769 rpipe->pipe_state |= PIPE_WANTR; 770 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 771 PRIBIO | PCATCH, 772 "piperd", 0)) == 0) 773 error = pipelock(rpipe, 1); 774 } 775 if (error) 776 goto unlocked_error; 777 } 778 } 779#ifdef MAC 780locked_error: 781#endif 782 pipeunlock(rpipe); 783 784 /* XXX: should probably do this before getting any locks. */ 785 if (error == 0) 786 vfs_timestamp(&rpipe->pipe_atime); 787unlocked_error: 788 --rpipe->pipe_busy; 789 790 /* 791 * PIPE_WANT processing only makes sense if pipe_busy is 0. 792 */ 793 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 794 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 795 wakeup(rpipe); 796 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 797 /* 798 * Handle write blocking hysteresis. 799 */ 800 if (rpipe->pipe_state & PIPE_WANTW) { 801 rpipe->pipe_state &= ~PIPE_WANTW; 802 wakeup(rpipe); 803 } 804 } 805 806 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 807 pipeselwakeup(rpipe); 808 809 PIPE_UNLOCK(rpipe); 810 return (error); 811} 812 813#ifndef PIPE_NODIRECT 814/* 815 * Map the sending processes' buffer into kernel space and wire it. 816 * This is similar to a physical write operation. 817 */ 818static int 819pipe_build_write_buffer(wpipe, uio) 820 struct pipe *wpipe; 821 struct uio *uio; 822{ 823 u_int size; 824 int i; 825 826 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 827 KASSERT(wpipe->pipe_state & PIPE_DIRECTW, 828 ("Clone attempt on non-direct write pipe!")); 829 830 if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size) 831 size = wpipe->pipe_buffer.size; 832 else 833 size = uio->uio_iov->iov_len; 834 835 if ((i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 836 (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ, 837 wpipe->pipe_map.ms, PIPENPAGES)) < 0) 838 return (EFAULT); 839 840/* 841 * set up the control block 842 */ 843 wpipe->pipe_map.npages = i; 844 wpipe->pipe_map.pos = 845 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 846 wpipe->pipe_map.cnt = size; 847 848/* 849 * and update the uio data 850 */ 851 852 uio->uio_iov->iov_len -= size; 853 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 854 if (uio->uio_iov->iov_len == 0) 855 uio->uio_iov++; 856 uio->uio_resid -= size; 857 uio->uio_offset += size; 858 return (0); 859} 860 861/* 862 * unmap and unwire the process buffer 863 */ 864static void 865pipe_destroy_write_buffer(wpipe) 866 struct pipe *wpipe; 867{ 868 869 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 870 vm_page_unhold_pages(wpipe->pipe_map.ms, wpipe->pipe_map.npages); 871 wpipe->pipe_map.npages = 0; 872} 873 874/* 875 * In the case of a signal, the writing process might go away. This 876 * code copies the data into the circular buffer so that the source 877 * pages can be freed without loss of data. 878 */ 879static void 880pipe_clone_write_buffer(wpipe) 881 struct pipe *wpipe; 882{ 883 struct uio uio; 884 struct iovec iov; 885 int size; 886 int pos; 887 888 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 889 size = wpipe->pipe_map.cnt; 890 pos = wpipe->pipe_map.pos; 891 892 wpipe->pipe_buffer.in = size; 893 wpipe->pipe_buffer.out = 0; 894 wpipe->pipe_buffer.cnt = size; 895 wpipe->pipe_state &= ~PIPE_DIRECTW; 896 897 PIPE_UNLOCK(wpipe); 898 iov.iov_base = wpipe->pipe_buffer.buffer; 899 iov.iov_len = size; 900 uio.uio_iov = &iov; 901 uio.uio_iovcnt = 1; 902 uio.uio_offset = 0; 903 uio.uio_resid = size; 904 uio.uio_segflg = UIO_SYSSPACE; 905 uio.uio_rw = UIO_READ; 906 uio.uio_td = curthread; 907 uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio); 908 PIPE_LOCK(wpipe); 909 pipe_destroy_write_buffer(wpipe); 910} 911 912/* 913 * This implements the pipe buffer write mechanism. Note that only 914 * a direct write OR a normal pipe write can be pending at any given time. 915 * If there are any characters in the pipe buffer, the direct write will 916 * be deferred until the receiving process grabs all of the bytes from 917 * the pipe buffer. Then the direct mapping write is set-up. 918 */ 919static int 920pipe_direct_write(wpipe, uio) 921 struct pipe *wpipe; 922 struct uio *uio; 923{ 924 int error; 925 926retry: 927 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 928 error = pipelock(wpipe, 1); 929 if (wpipe->pipe_state & PIPE_EOF) 930 error = EPIPE; 931 if (error) { 932 pipeunlock(wpipe); 933 goto error1; 934 } 935 while (wpipe->pipe_state & PIPE_DIRECTW) { 936 if (wpipe->pipe_state & PIPE_WANTR) { 937 wpipe->pipe_state &= ~PIPE_WANTR; 938 wakeup(wpipe); 939 } 940 pipeselwakeup(wpipe); 941 wpipe->pipe_state |= PIPE_WANTW; 942 pipeunlock(wpipe); 943 error = msleep(wpipe, PIPE_MTX(wpipe), 944 PRIBIO | PCATCH, "pipdww", 0); 945 if (error) 946 goto error1; 947 else 948 goto retry; 949 } 950 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 951 if (wpipe->pipe_buffer.cnt > 0) { 952 if (wpipe->pipe_state & PIPE_WANTR) { 953 wpipe->pipe_state &= ~PIPE_WANTR; 954 wakeup(wpipe); 955 } 956 pipeselwakeup(wpipe); 957 wpipe->pipe_state |= PIPE_WANTW; 958 pipeunlock(wpipe); 959 error = msleep(wpipe, PIPE_MTX(wpipe), 960 PRIBIO | PCATCH, "pipdwc", 0); 961 if (error) 962 goto error1; 963 else 964 goto retry; 965 } 966 967 wpipe->pipe_state |= PIPE_DIRECTW; 968 969 PIPE_UNLOCK(wpipe); 970 error = pipe_build_write_buffer(wpipe, uio); 971 PIPE_LOCK(wpipe); 972 if (error) { 973 wpipe->pipe_state &= ~PIPE_DIRECTW; 974 pipeunlock(wpipe); 975 goto error1; 976 } 977 978 error = 0; 979 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 980 if (wpipe->pipe_state & PIPE_EOF) { 981 pipe_destroy_write_buffer(wpipe); 982 pipeselwakeup(wpipe); 983 pipeunlock(wpipe); 984 error = EPIPE; 985 goto error1; 986 } 987 if (wpipe->pipe_state & PIPE_WANTR) { 988 wpipe->pipe_state &= ~PIPE_WANTR; 989 wakeup(wpipe); 990 } 991 pipeselwakeup(wpipe); 992 pipeunlock(wpipe); 993 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 994 "pipdwt", 0); 995 pipelock(wpipe, 0); 996 } 997 998 if (wpipe->pipe_state & PIPE_EOF) 999 error = EPIPE; 1000 if (wpipe->pipe_state & PIPE_DIRECTW) { 1001 /* 1002 * this bit of trickery substitutes a kernel buffer for 1003 * the process that might be going away. 1004 */ 1005 pipe_clone_write_buffer(wpipe); 1006 } else { 1007 pipe_destroy_write_buffer(wpipe); 1008 } 1009 pipeunlock(wpipe); 1010 return (error); 1011 1012error1: 1013 wakeup(wpipe); 1014 return (error); 1015} 1016#endif 1017 1018static int 1019pipe_write(fp, uio, active_cred, flags, td) 1020 struct file *fp; 1021 struct uio *uio; 1022 struct ucred *active_cred; 1023 struct thread *td; 1024 int flags; 1025{ 1026 int error = 0; 1027 int desiredsize; 1028 ssize_t orig_resid; 1029 struct pipe *wpipe, *rpipe; 1030 1031 rpipe = fp->f_data; 1032 wpipe = PIPE_PEER(rpipe); 1033 PIPE_LOCK(rpipe); 1034 error = pipelock(wpipe, 1); 1035 if (error) { 1036 PIPE_UNLOCK(rpipe); 1037 return (error); 1038 } 1039 /* 1040 * detect loss of pipe read side, issue SIGPIPE if lost. 1041 */ 1042 if (wpipe->pipe_present != PIPE_ACTIVE || 1043 (wpipe->pipe_state & PIPE_EOF)) { 1044 pipeunlock(wpipe); 1045 PIPE_UNLOCK(rpipe); 1046 return (EPIPE); 1047 } 1048#ifdef MAC 1049 error = mac_pipe_check_write(active_cred, wpipe->pipe_pair); 1050 if (error) { 1051 pipeunlock(wpipe); 1052 PIPE_UNLOCK(rpipe); 1053 return (error); 1054 } 1055#endif 1056 ++wpipe->pipe_busy; 1057 1058 /* Choose a larger size if it's advantageous */ 1059 desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size); 1060 while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) { 1061 if (piperesizeallowed != 1) 1062 break; 1063 if (amountpipekva > maxpipekva / 2) 1064 break; 1065 if (desiredsize == BIG_PIPE_SIZE) 1066 break; 1067 desiredsize = desiredsize * 2; 1068 } 1069 1070 /* Choose a smaller size if we're in a OOM situation */ 1071 if ((amountpipekva > (3 * maxpipekva) / 4) && 1072 (wpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 1073 (wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 1074 (piperesizeallowed == 1)) 1075 desiredsize = SMALL_PIPE_SIZE; 1076 1077 /* Resize if the above determined that a new size was necessary */ 1078 if ((desiredsize != wpipe->pipe_buffer.size) && 1079 ((wpipe->pipe_state & PIPE_DIRECTW) == 0)) { 1080 PIPE_UNLOCK(wpipe); 1081 pipespace(wpipe, desiredsize); 1082 PIPE_LOCK(wpipe); 1083 } 1084 if (wpipe->pipe_buffer.size == 0) { 1085 /* 1086 * This can only happen for reverse direction use of pipes 1087 * in a complete OOM situation. 1088 */ 1089 error = ENOMEM; 1090 --wpipe->pipe_busy; 1091 pipeunlock(wpipe); 1092 PIPE_UNLOCK(wpipe); 1093 return (error); 1094 } 1095 1096 pipeunlock(wpipe); 1097 1098 orig_resid = uio->uio_resid; 1099 1100 while (uio->uio_resid) { 1101 int space; 1102 1103 pipelock(wpipe, 0); 1104 if (wpipe->pipe_state & PIPE_EOF) { 1105 pipeunlock(wpipe); 1106 error = EPIPE; 1107 break; 1108 } 1109#ifndef PIPE_NODIRECT 1110 /* 1111 * If the transfer is large, we can gain performance if 1112 * we do process-to-process copies directly. 1113 * If the write is non-blocking, we don't use the 1114 * direct write mechanism. 1115 * 1116 * The direct write mechanism will detect the reader going 1117 * away on us. 1118 */ 1119 if (uio->uio_segflg == UIO_USERSPACE && 1120 uio->uio_iov->iov_len >= PIPE_MINDIRECT && 1121 wpipe->pipe_buffer.size >= PIPE_MINDIRECT && 1122 (fp->f_flag & FNONBLOCK) == 0) { 1123 pipeunlock(wpipe); 1124 error = pipe_direct_write(wpipe, uio); 1125 if (error) 1126 break; 1127 continue; 1128 } 1129#endif 1130 1131 /* 1132 * Pipe buffered writes cannot be coincidental with 1133 * direct writes. We wait until the currently executing 1134 * direct write is completed before we start filling the 1135 * pipe buffer. We break out if a signal occurs or the 1136 * reader goes away. 1137 */ 1138 if (wpipe->pipe_state & PIPE_DIRECTW) { 1139 if (wpipe->pipe_state & PIPE_WANTR) { 1140 wpipe->pipe_state &= ~PIPE_WANTR; 1141 wakeup(wpipe); 1142 } 1143 pipeselwakeup(wpipe); 1144 wpipe->pipe_state |= PIPE_WANTW; 1145 pipeunlock(wpipe); 1146 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1147 "pipbww", 0); 1148 if (error) 1149 break; 1150 else 1151 continue; 1152 } 1153 1154 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1155 1156 /* Writes of size <= PIPE_BUF must be atomic. */ 1157 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1158 space = 0; 1159 1160 if (space > 0) { 1161 int size; /* Transfer size */ 1162 int segsize; /* first segment to transfer */ 1163 1164 /* 1165 * Transfer size is minimum of uio transfer 1166 * and free space in pipe buffer. 1167 */ 1168 if (space > uio->uio_resid) 1169 size = uio->uio_resid; 1170 else 1171 size = space; 1172 /* 1173 * First segment to transfer is minimum of 1174 * transfer size and contiguous space in 1175 * pipe buffer. If first segment to transfer 1176 * is less than the transfer size, we've got 1177 * a wraparound in the buffer. 1178 */ 1179 segsize = wpipe->pipe_buffer.size - 1180 wpipe->pipe_buffer.in; 1181 if (segsize > size) 1182 segsize = size; 1183 1184 /* Transfer first segment */ 1185 1186 PIPE_UNLOCK(rpipe); 1187 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1188 segsize, uio); 1189 PIPE_LOCK(rpipe); 1190 1191 if (error == 0 && segsize < size) { 1192 KASSERT(wpipe->pipe_buffer.in + segsize == 1193 wpipe->pipe_buffer.size, 1194 ("Pipe buffer wraparound disappeared")); 1195 /* 1196 * Transfer remaining part now, to 1197 * support atomic writes. Wraparound 1198 * happened. 1199 */ 1200 1201 PIPE_UNLOCK(rpipe); 1202 error = uiomove( 1203 &wpipe->pipe_buffer.buffer[0], 1204 size - segsize, uio); 1205 PIPE_LOCK(rpipe); 1206 } 1207 if (error == 0) { 1208 wpipe->pipe_buffer.in += size; 1209 if (wpipe->pipe_buffer.in >= 1210 wpipe->pipe_buffer.size) { 1211 KASSERT(wpipe->pipe_buffer.in == 1212 size - segsize + 1213 wpipe->pipe_buffer.size, 1214 ("Expected wraparound bad")); 1215 wpipe->pipe_buffer.in = size - segsize; 1216 } 1217 1218 wpipe->pipe_buffer.cnt += size; 1219 KASSERT(wpipe->pipe_buffer.cnt <= 1220 wpipe->pipe_buffer.size, 1221 ("Pipe buffer overflow")); 1222 } 1223 pipeunlock(wpipe); 1224 if (error != 0) 1225 break; 1226 } else { 1227 /* 1228 * If the "read-side" has been blocked, wake it up now. 1229 */ 1230 if (wpipe->pipe_state & PIPE_WANTR) { 1231 wpipe->pipe_state &= ~PIPE_WANTR; 1232 wakeup(wpipe); 1233 } 1234 1235 /* 1236 * don't block on non-blocking I/O 1237 */ 1238 if (fp->f_flag & FNONBLOCK) { 1239 error = EAGAIN; 1240 pipeunlock(wpipe); 1241 break; 1242 } 1243 1244 /* 1245 * We have no more space and have something to offer, 1246 * wake up select/poll. 1247 */ 1248 pipeselwakeup(wpipe); 1249 1250 wpipe->pipe_state |= PIPE_WANTW; 1251 pipeunlock(wpipe); 1252 error = msleep(wpipe, PIPE_MTX(rpipe), 1253 PRIBIO | PCATCH, "pipewr", 0); 1254 if (error != 0) 1255 break; 1256 } 1257 } 1258 1259 pipelock(wpipe, 0); 1260 --wpipe->pipe_busy; 1261 1262 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1263 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1264 wakeup(wpipe); 1265 } else if (wpipe->pipe_buffer.cnt > 0) { 1266 /* 1267 * If we have put any characters in the buffer, we wake up 1268 * the reader. 1269 */ 1270 if (wpipe->pipe_state & PIPE_WANTR) { 1271 wpipe->pipe_state &= ~PIPE_WANTR; 1272 wakeup(wpipe); 1273 } 1274 } 1275 1276 /* 1277 * Don't return EPIPE if I/O was successful 1278 */ 1279 if ((wpipe->pipe_buffer.cnt == 0) && 1280 (uio->uio_resid == 0) && 1281 (error == EPIPE)) { 1282 error = 0; 1283 } 1284 1285 if (error == 0) 1286 vfs_timestamp(&wpipe->pipe_mtime); 1287 1288 /* 1289 * We have something to offer, 1290 * wake up select/poll. 1291 */ 1292 if (wpipe->pipe_buffer.cnt) 1293 pipeselwakeup(wpipe); 1294 1295 pipeunlock(wpipe); 1296 PIPE_UNLOCK(rpipe); 1297 return (error); 1298} 1299 1300/* ARGSUSED */ 1301static int 1302pipe_truncate(fp, length, active_cred, td) 1303 struct file *fp; 1304 off_t length; 1305 struct ucred *active_cred; 1306 struct thread *td; 1307{ 1308 1309 /* For named pipes call the vnode operation. */ 1310 if (fp->f_vnode != NULL) 1311 return (vnops.fo_truncate(fp, length, active_cred, td)); 1312 return (EINVAL); 1313} 1314 1315/* 1316 * we implement a very minimal set of ioctls for compatibility with sockets. 1317 */ 1318static int 1319pipe_ioctl(fp, cmd, data, active_cred, td) 1320 struct file *fp; 1321 u_long cmd; 1322 void *data; 1323 struct ucred *active_cred; 1324 struct thread *td; 1325{ 1326 struct pipe *mpipe = fp->f_data; 1327 int error; 1328 1329 PIPE_LOCK(mpipe); 1330 1331#ifdef MAC 1332 error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data); 1333 if (error) { 1334 PIPE_UNLOCK(mpipe); 1335 return (error); 1336 } 1337#endif 1338 1339 error = 0; 1340 switch (cmd) { 1341 1342 case FIONBIO: 1343 break; 1344 1345 case FIOASYNC: 1346 if (*(int *)data) { 1347 mpipe->pipe_state |= PIPE_ASYNC; 1348 } else { 1349 mpipe->pipe_state &= ~PIPE_ASYNC; 1350 } 1351 break; 1352 1353 case FIONREAD: 1354 if (!(fp->f_flag & FREAD)) { 1355 *(int *)data = 0; 1356 PIPE_UNLOCK(mpipe); 1357 return (0); 1358 } 1359 if (mpipe->pipe_state & PIPE_DIRECTW) 1360 *(int *)data = mpipe->pipe_map.cnt; 1361 else 1362 *(int *)data = mpipe->pipe_buffer.cnt; 1363 break; 1364 1365 case FIOSETOWN: 1366 PIPE_UNLOCK(mpipe); 1367 error = fsetown(*(int *)data, &mpipe->pipe_sigio); 1368 goto out_unlocked; 1369 1370 case FIOGETOWN: 1371 *(int *)data = fgetown(&mpipe->pipe_sigio); 1372 break; 1373 1374 /* This is deprecated, FIOSETOWN should be used instead. */ 1375 case TIOCSPGRP: 1376 PIPE_UNLOCK(mpipe); 1377 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio); 1378 goto out_unlocked; 1379 1380 /* This is deprecated, FIOGETOWN should be used instead. */ 1381 case TIOCGPGRP: 1382 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1383 break; 1384 1385 default: 1386 error = ENOTTY; 1387 break; 1388 } 1389 PIPE_UNLOCK(mpipe); 1390out_unlocked: 1391 return (error); 1392} 1393 1394static int 1395pipe_poll(fp, events, active_cred, td) 1396 struct file *fp; 1397 int events; 1398 struct ucred *active_cred; 1399 struct thread *td; 1400{ 1401 struct pipe *rpipe; 1402 struct pipe *wpipe; 1403 int levents, revents; 1404#ifdef MAC 1405 int error; 1406#endif 1407 1408 revents = 0; 1409 rpipe = fp->f_data; 1410 wpipe = PIPE_PEER(rpipe); 1411 PIPE_LOCK(rpipe); 1412#ifdef MAC 1413 error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair); 1414 if (error) 1415 goto locked_error; 1416#endif 1417 if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) 1418 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1419 (rpipe->pipe_buffer.cnt > 0)) 1420 revents |= events & (POLLIN | POLLRDNORM); 1421 1422 if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) 1423 if (wpipe->pipe_present != PIPE_ACTIVE || 1424 (wpipe->pipe_state & PIPE_EOF) || 1425 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1426 ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF || 1427 wpipe->pipe_buffer.size == 0))) 1428 revents |= events & (POLLOUT | POLLWRNORM); 1429 1430 levents = events & 1431 (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | POLLRDBAND); 1432 if (rpipe->pipe_state & PIPE_NAMED && fp->f_flag & FREAD && levents && 1433 rpipe->pipe_state & PIPE_SAMEWGEN) 1434 events |= POLLINIGNEOF; 1435 1436 if ((events & POLLINIGNEOF) == 0) { 1437 if (rpipe->pipe_state & PIPE_EOF) { 1438 revents |= (events & (POLLIN | POLLRDNORM)); 1439 if (wpipe->pipe_present != PIPE_ACTIVE || 1440 (wpipe->pipe_state & PIPE_EOF)) 1441 revents |= POLLHUP; 1442 } 1443 } 1444 1445 if (revents == 0) { 1446 if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) { 1447 selrecord(td, &rpipe->pipe_sel); 1448 if (SEL_WAITING(&rpipe->pipe_sel)) 1449 rpipe->pipe_state |= PIPE_SEL; 1450 } 1451 1452 if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) { 1453 selrecord(td, &wpipe->pipe_sel); 1454 if (SEL_WAITING(&wpipe->pipe_sel)) 1455 wpipe->pipe_state |= PIPE_SEL; 1456 } 1457 } 1458#ifdef MAC 1459locked_error: 1460#endif 1461 PIPE_UNLOCK(rpipe); 1462 1463 return (revents); 1464} 1465 1466/* 1467 * We shouldn't need locks here as we're doing a read and this should 1468 * be a natural race. 1469 */ 1470static int 1471pipe_stat(fp, ub, active_cred, td) 1472 struct file *fp; 1473 struct stat *ub; 1474 struct ucred *active_cred; 1475 struct thread *td; 1476{ 1477 struct pipe *pipe; 1478 int new_unr; 1479#ifdef MAC 1480 int error; 1481#endif 1482 1483 pipe = fp->f_data; 1484 PIPE_LOCK(pipe); 1485#ifdef MAC 1486 error = mac_pipe_check_stat(active_cred, pipe->pipe_pair); 1487 if (error) { 1488 PIPE_UNLOCK(pipe); 1489 return (error); 1490 } 1491#endif 1492 1493 /* For named pipes ask the underlying filesystem. */ 1494 if (pipe->pipe_state & PIPE_NAMED) { 1495 PIPE_UNLOCK(pipe); 1496 return (vnops.fo_stat(fp, ub, active_cred, td)); 1497 } 1498 1499 /* 1500 * Lazily allocate an inode number for the pipe. Most pipe 1501 * users do not call fstat(2) on the pipe, which means that 1502 * postponing the inode allocation until it is must be 1503 * returned to userland is useful. If alloc_unr failed, 1504 * assign st_ino zero instead of returning an error. 1505 * Special pipe_ino values: 1506 * -1 - not yet initialized; 1507 * 0 - alloc_unr failed, return 0 as st_ino forever. 1508 */ 1509 if (pipe->pipe_ino == (ino_t)-1) { 1510 new_unr = alloc_unr(pipeino_unr); 1511 if (new_unr != -1) 1512 pipe->pipe_ino = new_unr; 1513 else 1514 pipe->pipe_ino = 0; 1515 } 1516 PIPE_UNLOCK(pipe); 1517 1518 bzero(ub, sizeof(*ub)); 1519 ub->st_mode = S_IFIFO; 1520 ub->st_blksize = PAGE_SIZE; 1521 if (pipe->pipe_state & PIPE_DIRECTW) 1522 ub->st_size = pipe->pipe_map.cnt; 1523 else 1524 ub->st_size = pipe->pipe_buffer.cnt; 1525 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1526 ub->st_atim = pipe->pipe_atime; 1527 ub->st_mtim = pipe->pipe_mtime; 1528 ub->st_ctim = pipe->pipe_ctime; 1529 ub->st_uid = fp->f_cred->cr_uid; 1530 ub->st_gid = fp->f_cred->cr_gid; 1531 ub->st_dev = pipedev_ino; 1532 ub->st_ino = pipe->pipe_ino; 1533 /* 1534 * Left as 0: st_nlink, st_rdev, st_flags, st_gen. 1535 */ 1536 return (0); 1537} 1538 1539/* ARGSUSED */ 1540static int 1541pipe_close(fp, td) 1542 struct file *fp; 1543 struct thread *td; 1544{ 1545 1546 if (fp->f_vnode != NULL) 1547 return vnops.fo_close(fp, td); 1548 fp->f_ops = &badfileops; 1549 pipe_dtor(fp->f_data); 1550 fp->f_data = NULL; 1551 return (0); 1552} 1553 1554static int 1555pipe_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td) 1556{ 1557 struct pipe *cpipe; 1558 int error; 1559 1560 cpipe = fp->f_data; 1561 if (cpipe->pipe_state & PIPE_NAMED) 1562 error = vn_chmod(fp, mode, active_cred, td); 1563 else 1564 error = invfo_chmod(fp, mode, active_cred, td); 1565 return (error); 1566} 1567 1568static int 1569pipe_chown(fp, uid, gid, active_cred, td) 1570 struct file *fp; 1571 uid_t uid; 1572 gid_t gid; 1573 struct ucred *active_cred; 1574 struct thread *td; 1575{ 1576 struct pipe *cpipe; 1577 int error; 1578 1579 cpipe = fp->f_data; 1580 if (cpipe->pipe_state & PIPE_NAMED) 1581 error = vn_chown(fp, uid, gid, active_cred, td); 1582 else 1583 error = invfo_chown(fp, uid, gid, active_cred, td); 1584 return (error); 1585} 1586 1587static void 1588pipe_free_kmem(cpipe) 1589 struct pipe *cpipe; 1590{ 1591 1592 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), 1593 ("pipe_free_kmem: pipe mutex locked")); 1594 1595 if (cpipe->pipe_buffer.buffer != NULL) { 1596 atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size); 1597 vm_map_remove(pipe_map, 1598 (vm_offset_t)cpipe->pipe_buffer.buffer, 1599 (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size); 1600 cpipe->pipe_buffer.buffer = NULL; 1601 } 1602#ifndef PIPE_NODIRECT 1603 { 1604 cpipe->pipe_map.cnt = 0; 1605 cpipe->pipe_map.pos = 0; 1606 cpipe->pipe_map.npages = 0; 1607 } 1608#endif 1609} 1610 1611/* 1612 * shutdown the pipe 1613 */ 1614static void 1615pipeclose(cpipe) 1616 struct pipe *cpipe; 1617{ 1618 struct pipepair *pp; 1619 struct pipe *ppipe; 1620 1621 KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL")); 1622 1623 PIPE_LOCK(cpipe); 1624 pipelock(cpipe, 0); 1625 pp = cpipe->pipe_pair; 1626 1627 pipeselwakeup(cpipe); 1628 1629 /* 1630 * If the other side is blocked, wake it up saying that 1631 * we want to close it down. 1632 */ 1633 cpipe->pipe_state |= PIPE_EOF; 1634 while (cpipe->pipe_busy) { 1635 wakeup(cpipe); 1636 cpipe->pipe_state |= PIPE_WANT; 1637 pipeunlock(cpipe); 1638 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1639 pipelock(cpipe, 0); 1640 } 1641 1642 1643 /* 1644 * Disconnect from peer, if any. 1645 */ 1646 ppipe = cpipe->pipe_peer; 1647 if (ppipe->pipe_present == PIPE_ACTIVE) { 1648 pipeselwakeup(ppipe); 1649 1650 ppipe->pipe_state |= PIPE_EOF; 1651 wakeup(ppipe); 1652 KNOTE_LOCKED(&ppipe->pipe_sel.si_note, 0); 1653 } 1654 1655 /* 1656 * Mark this endpoint as free. Release kmem resources. We 1657 * don't mark this endpoint as unused until we've finished 1658 * doing that, or the pipe might disappear out from under 1659 * us. 1660 */ 1661 PIPE_UNLOCK(cpipe); 1662 pipe_free_kmem(cpipe); 1663 PIPE_LOCK(cpipe); 1664 cpipe->pipe_present = PIPE_CLOSING; 1665 pipeunlock(cpipe); 1666 1667 /* 1668 * knlist_clear() may sleep dropping the PIPE_MTX. Set the 1669 * PIPE_FINALIZED, that allows other end to free the 1670 * pipe_pair, only after the knotes are completely dismantled. 1671 */ 1672 knlist_clear(&cpipe->pipe_sel.si_note, 1); 1673 cpipe->pipe_present = PIPE_FINALIZED; 1674 seldrain(&cpipe->pipe_sel); 1675 knlist_destroy(&cpipe->pipe_sel.si_note); 1676 1677 /* 1678 * If both endpoints are now closed, release the memory for the 1679 * pipe pair. If not, unlock. 1680 */ 1681 if (ppipe->pipe_present == PIPE_FINALIZED) { 1682 PIPE_UNLOCK(cpipe); 1683#ifdef MAC 1684 mac_pipe_destroy(pp); 1685#endif 1686 uma_zfree(pipe_zone, cpipe->pipe_pair); 1687 } else 1688 PIPE_UNLOCK(cpipe); 1689} 1690 1691/*ARGSUSED*/ 1692static int 1693pipe_kqfilter(struct file *fp, struct knote *kn) 1694{ 1695 struct pipe *cpipe; 1696 1697 /* 1698 * If a filter is requested that is not supported by this file 1699 * descriptor, don't return an error, but also don't ever generate an 1700 * event. 1701 */ 1702 if ((kn->kn_filter == EVFILT_READ) && !(fp->f_flag & FREAD)) { 1703 kn->kn_fop = &pipe_nfiltops; 1704 return (0); 1705 } 1706 if ((kn->kn_filter == EVFILT_WRITE) && !(fp->f_flag & FWRITE)) { 1707 kn->kn_fop = &pipe_nfiltops; 1708 return (0); 1709 } 1710 cpipe = fp->f_data; 1711 PIPE_LOCK(cpipe); 1712 switch (kn->kn_filter) { 1713 case EVFILT_READ: 1714 kn->kn_fop = &pipe_rfiltops; 1715 break; 1716 case EVFILT_WRITE: 1717 kn->kn_fop = &pipe_wfiltops; 1718 if (cpipe->pipe_peer->pipe_present != PIPE_ACTIVE) { 1719 /* other end of pipe has been closed */ 1720 PIPE_UNLOCK(cpipe); 1721 return (EPIPE); 1722 } 1723 cpipe = PIPE_PEER(cpipe); 1724 break; 1725 default: 1726 PIPE_UNLOCK(cpipe); 1727 return (EINVAL); 1728 } 1729 1730 kn->kn_hook = cpipe; 1731 knlist_add(&cpipe->pipe_sel.si_note, kn, 1); 1732 PIPE_UNLOCK(cpipe); 1733 return (0); 1734} 1735 1736static void 1737filt_pipedetach(struct knote *kn) 1738{ 1739 struct pipe *cpipe = kn->kn_hook; 1740 1741 PIPE_LOCK(cpipe); 1742 knlist_remove(&cpipe->pipe_sel.si_note, kn, 1); 1743 PIPE_UNLOCK(cpipe); 1744} 1745 1746/*ARGSUSED*/ 1747static int 1748filt_piperead(struct knote *kn, long hint) 1749{ 1750 struct pipe *rpipe = kn->kn_hook; 1751 struct pipe *wpipe = rpipe->pipe_peer; 1752 int ret; 1753 1754 PIPE_LOCK(rpipe); 1755 kn->kn_data = rpipe->pipe_buffer.cnt; 1756 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1757 kn->kn_data = rpipe->pipe_map.cnt; 1758 1759 if ((rpipe->pipe_state & PIPE_EOF) || 1760 wpipe->pipe_present != PIPE_ACTIVE || 1761 (wpipe->pipe_state & PIPE_EOF)) { 1762 kn->kn_flags |= EV_EOF; 1763 PIPE_UNLOCK(rpipe); 1764 return (1); 1765 } 1766 ret = kn->kn_data > 0; 1767 PIPE_UNLOCK(rpipe); 1768 return ret; 1769} 1770 1771/*ARGSUSED*/ 1772static int 1773filt_pipewrite(struct knote *kn, long hint) 1774{ 1775 struct pipe *wpipe; 1776 1777 wpipe = kn->kn_hook; 1778 PIPE_LOCK(wpipe); 1779 if (wpipe->pipe_present != PIPE_ACTIVE || 1780 (wpipe->pipe_state & PIPE_EOF)) { 1781 kn->kn_data = 0; 1782 kn->kn_flags |= EV_EOF; 1783 PIPE_UNLOCK(wpipe); 1784 return (1); 1785 } 1786 kn->kn_data = (wpipe->pipe_buffer.size > 0) ? 1787 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) : PIPE_BUF; 1788 if (wpipe->pipe_state & PIPE_DIRECTW) 1789 kn->kn_data = 0; 1790 1791 PIPE_UNLOCK(wpipe); 1792 return (kn->kn_data >= PIPE_BUF); 1793} 1794 1795static void 1796filt_pipedetach_notsup(struct knote *kn) 1797{ 1798 1799} 1800 1801static int 1802filt_pipenotsup(struct knote *kn, long hint) 1803{ 1804 1805 return (0); 1806} 1807