1/* $NetBSD$ */ 2 3/*- 4 * Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 */ 50 51/* 52 * This file contains a high-performance replacement for the socket-based 53 * pipes scheme originally used. It does not support all features of 54 * sockets, but does do everything that pipes normally do. 55 * 56 * This code has two modes of operation, a small write mode and a large 57 * write mode. The small write mode acts like conventional pipes with 58 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 59 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 60 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 61 * using the UVM page loan facility from where the receiving process can copy 62 * the data directly from the pages in the sending process. 63 * 64 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 65 * happen for small transfers so that the system will not spend all of 66 * its time context switching. PIPE_SIZE is constrained by the 67 * amount of kernel virtual memory. 68 */ 69 70#include <sys/cdefs.h> 71__KERNEL_RCSID(0, "$NetBSD$"); 72 73#include <sys/param.h> 74#include <sys/systm.h> 75#include <sys/proc.h> 76#include <sys/fcntl.h> 77#include <sys/file.h> 78#include <sys/filedesc.h> 79#include <sys/filio.h> 80#include <sys/kernel.h> 81#include <sys/ttycom.h> 82#include <sys/stat.h> 83#include <sys/poll.h> 84#include <sys/signalvar.h> 85#include <sys/vnode.h> 86#include <sys/uio.h> 87#include <sys/select.h> 88#include <sys/mount.h> 89#include <sys/syscallargs.h> 90#include <sys/sysctl.h> 91#include <sys/kauth.h> 92#include <sys/atomic.h> 93#include <sys/pipe.h> 94 95#include <uvm/uvm_extern.h> 96 97/* 98 * Use this to disable direct I/O and decrease the code size: 99 * #define PIPE_NODIRECT 100 */ 101 102/* XXX Disabled for now; rare hangs switching between direct/buffered */ 103#define PIPE_NODIRECT 104 105static int pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int); 106static int pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int); 107static int pipe_close(file_t *); 108static int pipe_poll(file_t *, int); 109static int pipe_kqfilter(file_t *, struct knote *); 110static int pipe_stat(file_t *, struct stat *); 111static int pipe_ioctl(file_t *, u_long, void *); 112static void pipe_restart(file_t *); 113 114static const struct fileops pipeops = { 115 .fo_read = pipe_read, 116 .fo_write = pipe_write, 117 .fo_ioctl = pipe_ioctl, 118 .fo_fcntl = fnullop_fcntl, 119 .fo_poll = pipe_poll, 120 .fo_stat = pipe_stat, 121 .fo_close = pipe_close, 122 .fo_kqfilter = pipe_kqfilter, 123 .fo_restart = pipe_restart, 124}; 125 126/* 127 * Default pipe buffer size(s), this can be kind-of large now because pipe 128 * space is pageable. The pipe code will try to maintain locality of 129 * reference for performance reasons, so small amounts of outstanding I/O 130 * will not wipe the cache. 131 */ 132#define MINPIPESIZE (PIPE_SIZE / 3) 133#define MAXPIPESIZE (2 * PIPE_SIZE / 3) 134 135/* 136 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 137 * is there so that on large systems, we don't exhaust it. 138 */ 139#define MAXPIPEKVA (8 * 1024 * 1024) 140static u_int maxpipekva = MAXPIPEKVA; 141 142/* 143 * Limit for direct transfers, we cannot, of course limit 144 * the amount of kva for pipes in general though. 145 */ 146#define LIMITPIPEKVA (16 * 1024 * 1024) 147static u_int limitpipekva = LIMITPIPEKVA; 148 149/* 150 * Limit the number of "big" pipes 151 */ 152#define LIMITBIGPIPES 32 153static u_int maxbigpipes = LIMITBIGPIPES; 154static u_int nbigpipe = 0; 155 156/* 157 * Amount of KVA consumed by pipe buffers. 158 */ 159static u_int amountpipekva = 0; 160 161static void pipeclose(struct pipe *); 162static void pipe_free_kmem(struct pipe *); 163static int pipe_create(struct pipe **, pool_cache_t); 164static int pipelock(struct pipe *, int); 165static inline void pipeunlock(struct pipe *); 166static void pipeselwakeup(struct pipe *, struct pipe *, int); 167#ifndef PIPE_NODIRECT 168static int pipe_direct_write(file_t *, struct pipe *, struct uio *); 169#endif 170static int pipespace(struct pipe *, int); 171static int pipe_ctor(void *, void *, int); 172static void pipe_dtor(void *, void *); 173 174#ifndef PIPE_NODIRECT 175static int pipe_loan_alloc(struct pipe *, int); 176static void pipe_loan_free(struct pipe *); 177#endif /* PIPE_NODIRECT */ 178 179static pool_cache_t pipe_wr_cache; 180static pool_cache_t pipe_rd_cache; 181 182void 183pipe_init(void) 184{ 185 186 /* Writer side is not automatically allocated KVA. */ 187 pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr", 188 NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL); 189 KASSERT(pipe_wr_cache != NULL); 190 191 /* Reader side gets preallocated KVA. */ 192 pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd", 193 NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1); 194 KASSERT(pipe_rd_cache != NULL); 195} 196 197static int 198pipe_ctor(void *arg, void *obj, int flags) 199{ 200 struct pipe *pipe; 201 vaddr_t va; 202 203 pipe = obj; 204 205 memset(pipe, 0, sizeof(struct pipe)); 206 if (arg != NULL) { 207 /* Preallocate space. */ 208 va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, 209 UVM_KMF_PAGEABLE | UVM_KMF_WAITVA); 210 KASSERT(va != 0); 211 pipe->pipe_kmem = va; 212 atomic_add_int(&amountpipekva, PIPE_SIZE); 213 } 214 cv_init(&pipe->pipe_rcv, "pipe_rd"); 215 cv_init(&pipe->pipe_wcv, "pipe_wr"); 216 cv_init(&pipe->pipe_draincv, "pipe_drn"); 217 cv_init(&pipe->pipe_lkcv, "pipe_lk"); 218 selinit(&pipe->pipe_sel); 219 pipe->pipe_state = PIPE_SIGNALR; 220 221 return 0; 222} 223 224static void 225pipe_dtor(void *arg, void *obj) 226{ 227 struct pipe *pipe; 228 229 pipe = obj; 230 231 cv_destroy(&pipe->pipe_rcv); 232 cv_destroy(&pipe->pipe_wcv); 233 cv_destroy(&pipe->pipe_draincv); 234 cv_destroy(&pipe->pipe_lkcv); 235 seldestroy(&pipe->pipe_sel); 236 if (pipe->pipe_kmem != 0) { 237 uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE, 238 UVM_KMF_PAGEABLE); 239 atomic_add_int(&amountpipekva, -PIPE_SIZE); 240 } 241} 242 243/* 244 * The pipe system call for the DTYPE_PIPE type of pipes 245 */ 246int 247pipe1(struct lwp *l, register_t *retval, int flags) 248{ 249 struct pipe *rpipe, *wpipe; 250 file_t *rf, *wf; 251 int fd, error; 252 proc_t *p; 253 254 if (flags & ~(O_CLOEXEC|O_NONBLOCK|O_NOSIGPIPE)) 255 return EINVAL; 256 p = curproc; 257 rpipe = wpipe = NULL; 258 if ((error = pipe_create(&rpipe, pipe_rd_cache)) || 259 (error = pipe_create(&wpipe, pipe_wr_cache))) { 260 goto free2; 261 } 262 rpipe->pipe_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 263 wpipe->pipe_lock = rpipe->pipe_lock; 264 mutex_obj_hold(wpipe->pipe_lock); 265 266 error = fd_allocfile(&rf, &fd); 267 if (error) 268 goto free2; 269 retval[0] = fd; 270 271 error = fd_allocfile(&wf, &fd); 272 if (error) 273 goto free3; 274 retval[1] = fd; 275 276 rf->f_flag = FREAD | flags; 277 rf->f_type = DTYPE_PIPE; 278 rf->f_data = (void *)rpipe; 279 rf->f_ops = &pipeops; 280 fd_set_exclose(l, (int)retval[0], (flags & O_CLOEXEC) != 0); 281 282 wf->f_flag = FWRITE | flags; 283 wf->f_type = DTYPE_PIPE; 284 wf->f_data = (void *)wpipe; 285 wf->f_ops = &pipeops; 286 fd_set_exclose(l, (int)retval[1], (flags & O_CLOEXEC) != 0); 287 288 rpipe->pipe_peer = wpipe; 289 wpipe->pipe_peer = rpipe; 290 291 fd_affix(p, rf, (int)retval[0]); 292 fd_affix(p, wf, (int)retval[1]); 293 return (0); 294free3: 295 fd_abort(p, rf, (int)retval[0]); 296free2: 297 pipeclose(wpipe); 298 pipeclose(rpipe); 299 300 return (error); 301} 302 303/* 304 * Allocate kva for pipe circular buffer, the space is pageable 305 * This routine will 'realloc' the size of a pipe safely, if it fails 306 * it will retain the old buffer. 307 * If it fails it will return ENOMEM. 308 */ 309static int 310pipespace(struct pipe *pipe, int size) 311{ 312 void *buffer; 313 314 /* 315 * Allocate pageable virtual address space. Physical memory is 316 * allocated on demand. 317 */ 318 if (size == PIPE_SIZE && pipe->pipe_kmem != 0) { 319 buffer = (void *)pipe->pipe_kmem; 320 } else { 321 buffer = (void *)uvm_km_alloc(kernel_map, round_page(size), 322 0, UVM_KMF_PAGEABLE); 323 if (buffer == NULL) 324 return (ENOMEM); 325 atomic_add_int(&amountpipekva, size); 326 } 327 328 /* free old resources if we're resizing */ 329 pipe_free_kmem(pipe); 330 pipe->pipe_buffer.buffer = buffer; 331 pipe->pipe_buffer.size = size; 332 pipe->pipe_buffer.in = 0; 333 pipe->pipe_buffer.out = 0; 334 pipe->pipe_buffer.cnt = 0; 335 return (0); 336} 337 338/* 339 * Initialize and allocate VM and memory for pipe. 340 */ 341static int 342pipe_create(struct pipe **pipep, pool_cache_t cache) 343{ 344 struct pipe *pipe; 345 int error; 346 347 pipe = pool_cache_get(cache, PR_WAITOK); 348 KASSERT(pipe != NULL); 349 *pipep = pipe; 350 error = 0; 351 getnanotime(&pipe->pipe_btime); 352 pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime; 353 pipe->pipe_lock = NULL; 354 if (cache == pipe_rd_cache) { 355 error = pipespace(pipe, PIPE_SIZE); 356 } else { 357 pipe->pipe_buffer.buffer = NULL; 358 pipe->pipe_buffer.size = 0; 359 pipe->pipe_buffer.in = 0; 360 pipe->pipe_buffer.out = 0; 361 pipe->pipe_buffer.cnt = 0; 362 } 363 return error; 364} 365 366/* 367 * Lock a pipe for I/O, blocking other access 368 * Called with pipe spin lock held. 369 */ 370static int 371pipelock(struct pipe *pipe, int catch) 372{ 373 int error; 374 375 KASSERT(mutex_owned(pipe->pipe_lock)); 376 377 while (pipe->pipe_state & PIPE_LOCKFL) { 378 pipe->pipe_state |= PIPE_LWANT; 379 if (catch) { 380 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 381 if (error != 0) 382 return error; 383 } else 384 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 385 } 386 387 pipe->pipe_state |= PIPE_LOCKFL; 388 389 return 0; 390} 391 392/* 393 * unlock a pipe I/O lock 394 */ 395static inline void 396pipeunlock(struct pipe *pipe) 397{ 398 399 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 400 401 pipe->pipe_state &= ~PIPE_LOCKFL; 402 if (pipe->pipe_state & PIPE_LWANT) { 403 pipe->pipe_state &= ~PIPE_LWANT; 404 cv_broadcast(&pipe->pipe_lkcv); 405 } 406} 407 408/* 409 * Select/poll wakup. This also sends SIGIO to peer connected to 410 * 'sigpipe' side of pipe. 411 */ 412static void 413pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 414{ 415 int band; 416 417 switch (code) { 418 case POLL_IN: 419 band = POLLIN|POLLRDNORM; 420 break; 421 case POLL_OUT: 422 band = POLLOUT|POLLWRNORM; 423 break; 424 case POLL_HUP: 425 band = POLLHUP; 426 break; 427 case POLL_ERR: 428 band = POLLERR; 429 break; 430 default: 431 band = 0; 432#ifdef DIAGNOSTIC 433 printf("bad siginfo code %d in pipe notification.\n", code); 434#endif 435 break; 436 } 437 438 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 439 440 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 441 return; 442 443 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 444} 445 446static int 447pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 448 int flags) 449{ 450 struct pipe *rpipe = (struct pipe *) fp->f_data; 451 struct pipebuf *bp = &rpipe->pipe_buffer; 452 kmutex_t *lock = rpipe->pipe_lock; 453 int error; 454 size_t nread = 0; 455 size_t size; 456 size_t ocnt; 457 unsigned int wakeup_state = 0; 458 459 mutex_enter(lock); 460 ++rpipe->pipe_busy; 461 ocnt = bp->cnt; 462 463again: 464 error = pipelock(rpipe, 1); 465 if (error) 466 goto unlocked_error; 467 468 while (uio->uio_resid) { 469 /* 470 * Normal pipe buffer receive. 471 */ 472 if (bp->cnt > 0) { 473 size = bp->size - bp->out; 474 if (size > bp->cnt) 475 size = bp->cnt; 476 if (size > uio->uio_resid) 477 size = uio->uio_resid; 478 479 mutex_exit(lock); 480 error = uiomove((char *)bp->buffer + bp->out, size, uio); 481 mutex_enter(lock); 482 if (error) 483 break; 484 485 bp->out += size; 486 if (bp->out >= bp->size) 487 bp->out = 0; 488 489 bp->cnt -= size; 490 491 /* 492 * If there is no more to read in the pipe, reset 493 * its pointers to the beginning. This improves 494 * cache hit stats. 495 */ 496 if (bp->cnt == 0) { 497 bp->in = 0; 498 bp->out = 0; 499 } 500 nread += size; 501 continue; 502 } 503 504#ifndef PIPE_NODIRECT 505 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 506 /* 507 * Direct copy, bypassing a kernel buffer. 508 */ 509 void *va; 510 u_int gen; 511 512 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 513 514 size = rpipe->pipe_map.cnt; 515 if (size > uio->uio_resid) 516 size = uio->uio_resid; 517 518 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 519 gen = rpipe->pipe_map.egen; 520 mutex_exit(lock); 521 522 /* 523 * Consume emap and read the data from loaned pages. 524 */ 525 uvm_emap_consume(gen); 526 error = uiomove(va, size, uio); 527 528 mutex_enter(lock); 529 if (error) 530 break; 531 nread += size; 532 rpipe->pipe_map.pos += size; 533 rpipe->pipe_map.cnt -= size; 534 if (rpipe->pipe_map.cnt == 0) { 535 rpipe->pipe_state &= ~PIPE_DIRECTR; 536 cv_broadcast(&rpipe->pipe_wcv); 537 } 538 continue; 539 } 540#endif 541 /* 542 * Break if some data was read. 543 */ 544 if (nread > 0) 545 break; 546 547 /* 548 * Detect EOF condition. 549 * Read returns 0 on EOF, no need to set error. 550 */ 551 if (rpipe->pipe_state & PIPE_EOF) 552 break; 553 554 /* 555 * Don't block on non-blocking I/O. 556 */ 557 if (fp->f_flag & FNONBLOCK) { 558 error = EAGAIN; 559 break; 560 } 561 562 /* 563 * Unlock the pipe buffer for our remaining processing. 564 * We will either break out with an error or we will 565 * sleep and relock to loop. 566 */ 567 pipeunlock(rpipe); 568 569 /* 570 * Re-check to see if more direct writes are pending. 571 */ 572 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 573 goto again; 574 575#if 1 /* XXX (dsl) I'm sure these aren't needed here ... */ 576 /* 577 * We want to read more, wake up select/poll. 578 */ 579 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 580 581 /* 582 * If the "write-side" is blocked, wake it up now. 583 */ 584 cv_broadcast(&rpipe->pipe_wcv); 585#endif 586 587 if (wakeup_state & PIPE_RESTART) { 588 error = ERESTART; 589 goto unlocked_error; 590 } 591 592 /* Now wait until the pipe is filled */ 593 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 594 if (error != 0) 595 goto unlocked_error; 596 wakeup_state = rpipe->pipe_state; 597 goto again; 598 } 599 600 if (error == 0) 601 getnanotime(&rpipe->pipe_atime); 602 pipeunlock(rpipe); 603 604unlocked_error: 605 --rpipe->pipe_busy; 606 if (rpipe->pipe_busy == 0) { 607 rpipe->pipe_state &= ~PIPE_RESTART; 608 cv_broadcast(&rpipe->pipe_draincv); 609 } 610 if (bp->cnt < MINPIPESIZE) { 611 cv_broadcast(&rpipe->pipe_wcv); 612 } 613 614 /* 615 * If anything was read off the buffer, signal to the writer it's 616 * possible to write more data. Also send signal if we are here for the 617 * first time after last write. 618 */ 619 if ((bp->size - bp->cnt) >= PIPE_BUF 620 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 621 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 622 rpipe->pipe_state &= ~PIPE_SIGNALR; 623 } 624 625 mutex_exit(lock); 626 return (error); 627} 628 629#ifndef PIPE_NODIRECT 630/* 631 * Allocate structure for loan transfer. 632 */ 633static int 634pipe_loan_alloc(struct pipe *wpipe, int npages) 635{ 636 vsize_t len; 637 638 len = (vsize_t)npages << PAGE_SHIFT; 639 atomic_add_int(&amountpipekva, len); 640 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 641 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 642 if (wpipe->pipe_map.kva == 0) { 643 atomic_add_int(&amountpipekva, -len); 644 return (ENOMEM); 645 } 646 647 wpipe->pipe_map.npages = npages; 648 wpipe->pipe_map.pgs = kmem_alloc(npages * sizeof(struct vm_page *), 649 KM_SLEEP); 650 return (0); 651} 652 653/* 654 * Free resources allocated for loan transfer. 655 */ 656static void 657pipe_loan_free(struct pipe *wpipe) 658{ 659 vsize_t len; 660 661 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 662 uvm_emap_remove(wpipe->pipe_map.kva, len); /* XXX */ 663 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 664 wpipe->pipe_map.kva = 0; 665 atomic_add_int(&amountpipekva, -len); 666 kmem_free(wpipe->pipe_map.pgs, 667 wpipe->pipe_map.npages * sizeof(struct vm_page *)); 668 wpipe->pipe_map.pgs = NULL; 669} 670 671/* 672 * NetBSD direct write, using uvm_loan() mechanism. 673 * This implements the pipe buffer write mechanism. Note that only 674 * a direct write OR a normal pipe write can be pending at any given time. 675 * If there are any characters in the pipe buffer, the direct write will 676 * be deferred until the receiving process grabs all of the bytes from 677 * the pipe buffer. Then the direct mapping write is set-up. 678 * 679 * Called with the long-term pipe lock held. 680 */ 681static int 682pipe_direct_write(file_t *fp, struct pipe *wpipe, struct uio *uio) 683{ 684 struct vm_page **pgs; 685 vaddr_t bbase, base, bend; 686 vsize_t blen, bcnt; 687 int error, npages; 688 voff_t bpos; 689 kmutex_t *lock = wpipe->pipe_lock; 690 691 KASSERT(mutex_owned(wpipe->pipe_lock)); 692 KASSERT(wpipe->pipe_map.cnt == 0); 693 694 mutex_exit(lock); 695 696 /* 697 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 698 * not aligned to PAGE_SIZE. 699 */ 700 bbase = (vaddr_t)uio->uio_iov->iov_base; 701 base = trunc_page(bbase); 702 bend = round_page(bbase + uio->uio_iov->iov_len); 703 blen = bend - base; 704 bpos = bbase - base; 705 706 if (blen > PIPE_DIRECT_CHUNK) { 707 blen = PIPE_DIRECT_CHUNK; 708 bend = base + blen; 709 bcnt = PIPE_DIRECT_CHUNK - bpos; 710 } else { 711 bcnt = uio->uio_iov->iov_len; 712 } 713 npages = blen >> PAGE_SHIFT; 714 715 /* 716 * Free the old kva if we need more pages than we have 717 * allocated. 718 */ 719 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 720 pipe_loan_free(wpipe); 721 722 /* Allocate new kva. */ 723 if (wpipe->pipe_map.kva == 0) { 724 error = pipe_loan_alloc(wpipe, npages); 725 if (error) { 726 mutex_enter(lock); 727 return (error); 728 } 729 } 730 731 /* Loan the write buffer memory from writer process */ 732 pgs = wpipe->pipe_map.pgs; 733 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 734 pgs, UVM_LOAN_TOPAGE); 735 if (error) { 736 pipe_loan_free(wpipe); 737 mutex_enter(lock); 738 return (ENOMEM); /* so that caller fallback to ordinary write */ 739 } 740 741 /* Enter the loaned pages to KVA, produce new emap generation number. */ 742 uvm_emap_enter(wpipe->pipe_map.kva, pgs, npages); 743 wpipe->pipe_map.egen = uvm_emap_produce(); 744 745 /* Now we can put the pipe in direct write mode */ 746 wpipe->pipe_map.pos = bpos; 747 wpipe->pipe_map.cnt = bcnt; 748 749 /* 750 * But before we can let someone do a direct read, we 751 * have to wait until the pipe is drained. Release the 752 * pipe lock while we wait. 753 */ 754 mutex_enter(lock); 755 wpipe->pipe_state |= PIPE_DIRECTW; 756 pipeunlock(wpipe); 757 758 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 759 cv_broadcast(&wpipe->pipe_rcv); 760 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 761 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 762 error = EPIPE; 763 } 764 765 /* Pipe is drained; next read will off the direct buffer */ 766 wpipe->pipe_state |= PIPE_DIRECTR; 767 768 /* Wait until the reader is done */ 769 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 770 cv_broadcast(&wpipe->pipe_rcv); 771 pipeselwakeup(wpipe, wpipe, POLL_IN); 772 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 773 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 774 error = EPIPE; 775 } 776 777 /* Take pipe out of direct write mode */ 778 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 779 780 /* Acquire the pipe lock and cleanup */ 781 (void)pipelock(wpipe, 0); 782 mutex_exit(lock); 783 784 if (pgs != NULL) { 785 /* XXX: uvm_emap_remove */ 786 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 787 } 788 if (error || amountpipekva > maxpipekva) 789 pipe_loan_free(wpipe); 790 791 mutex_enter(lock); 792 if (error) { 793 pipeselwakeup(wpipe, wpipe, POLL_ERR); 794 795 /* 796 * If nothing was read from what we offered, return error 797 * straight on. Otherwise update uio resid first. Caller 798 * will deal with the error condition, returning short 799 * write, error, or restarting the write(2) as appropriate. 800 */ 801 if (wpipe->pipe_map.cnt == bcnt) { 802 wpipe->pipe_map.cnt = 0; 803 cv_broadcast(&wpipe->pipe_wcv); 804 return (error); 805 } 806 807 bcnt -= wpipe->pipe_map.cnt; 808 } 809 810 uio->uio_resid -= bcnt; 811 /* uio_offset not updated, not set/used for write(2) */ 812 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 813 uio->uio_iov->iov_len -= bcnt; 814 if (uio->uio_iov->iov_len == 0) { 815 uio->uio_iov++; 816 uio->uio_iovcnt--; 817 } 818 819 wpipe->pipe_map.cnt = 0; 820 return (error); 821} 822#endif /* !PIPE_NODIRECT */ 823 824static int 825pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 826 int flags) 827{ 828 struct pipe *wpipe, *rpipe; 829 struct pipebuf *bp; 830 kmutex_t *lock; 831 int error; 832 unsigned int wakeup_state = 0; 833 834 /* We want to write to our peer */ 835 rpipe = (struct pipe *) fp->f_data; 836 lock = rpipe->pipe_lock; 837 error = 0; 838 839 mutex_enter(lock); 840 wpipe = rpipe->pipe_peer; 841 842 /* 843 * Detect loss of pipe read side, issue SIGPIPE if lost. 844 */ 845 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 846 mutex_exit(lock); 847 return EPIPE; 848 } 849 ++wpipe->pipe_busy; 850 851 /* Aquire the long-term pipe lock */ 852 if ((error = pipelock(wpipe, 1)) != 0) { 853 --wpipe->pipe_busy; 854 if (wpipe->pipe_busy == 0) { 855 wpipe->pipe_state &= ~PIPE_RESTART; 856 cv_broadcast(&wpipe->pipe_draincv); 857 } 858 mutex_exit(lock); 859 return (error); 860 } 861 862 bp = &wpipe->pipe_buffer; 863 864 /* 865 * If it is advantageous to resize the pipe buffer, do so. 866 */ 867 if ((uio->uio_resid > PIPE_SIZE) && 868 (nbigpipe < maxbigpipes) && 869#ifndef PIPE_NODIRECT 870 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 871#endif 872 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 873 874 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 875 atomic_inc_uint(&nbigpipe); 876 } 877 878 while (uio->uio_resid) { 879 size_t space; 880 881#ifndef PIPE_NODIRECT 882 /* 883 * Pipe buffered writes cannot be coincidental with 884 * direct writes. Also, only one direct write can be 885 * in progress at any one time. We wait until the currently 886 * executing direct write is completed before continuing. 887 * 888 * We break out if a signal occurs or the reader goes away. 889 */ 890 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 891 cv_broadcast(&wpipe->pipe_rcv); 892 pipeunlock(wpipe); 893 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 894 (void)pipelock(wpipe, 0); 895 if (wpipe->pipe_state & PIPE_EOF) 896 error = EPIPE; 897 } 898 if (error) 899 break; 900 901 /* 902 * If the transfer is large, we can gain performance if 903 * we do process-to-process copies directly. 904 * If the write is non-blocking, we don't use the 905 * direct write mechanism. 906 * 907 * The direct write mechanism will detect the reader going 908 * away on us. 909 */ 910 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 911 (fp->f_flag & FNONBLOCK) == 0 && 912 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 913 error = pipe_direct_write(fp, wpipe, uio); 914 915 /* 916 * Break out if error occurred, unless it's ENOMEM. 917 * ENOMEM means we failed to allocate some resources 918 * for direct write, so we just fallback to ordinary 919 * write. If the direct write was successful, 920 * process rest of data via ordinary write. 921 */ 922 if (error == 0) 923 continue; 924 925 if (error != ENOMEM) 926 break; 927 } 928#endif /* PIPE_NODIRECT */ 929 930 space = bp->size - bp->cnt; 931 932 /* Writes of size <= PIPE_BUF must be atomic. */ 933 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 934 space = 0; 935 936 if (space > 0) { 937 int size; /* Transfer size */ 938 int segsize; /* first segment to transfer */ 939 940 /* 941 * Transfer size is minimum of uio transfer 942 * and free space in pipe buffer. 943 */ 944 if (space > uio->uio_resid) 945 size = uio->uio_resid; 946 else 947 size = space; 948 /* 949 * First segment to transfer is minimum of 950 * transfer size and contiguous space in 951 * pipe buffer. If first segment to transfer 952 * is less than the transfer size, we've got 953 * a wraparound in the buffer. 954 */ 955 segsize = bp->size - bp->in; 956 if (segsize > size) 957 segsize = size; 958 959 /* Transfer first segment */ 960 mutex_exit(lock); 961 error = uiomove((char *)bp->buffer + bp->in, segsize, 962 uio); 963 964 if (error == 0 && segsize < size) { 965 /* 966 * Transfer remaining part now, to 967 * support atomic writes. Wraparound 968 * happened. 969 */ 970 KASSERT(bp->in + segsize == bp->size); 971 error = uiomove(bp->buffer, 972 size - segsize, uio); 973 } 974 mutex_enter(lock); 975 if (error) 976 break; 977 978 bp->in += size; 979 if (bp->in >= bp->size) { 980 KASSERT(bp->in == size - segsize + bp->size); 981 bp->in = size - segsize; 982 } 983 984 bp->cnt += size; 985 KASSERT(bp->cnt <= bp->size); 986 wakeup_state = 0; 987 } else { 988 /* 989 * If the "read-side" has been blocked, wake it up now. 990 */ 991 cv_broadcast(&wpipe->pipe_rcv); 992 993 /* 994 * Don't block on non-blocking I/O. 995 */ 996 if (fp->f_flag & FNONBLOCK) { 997 error = EAGAIN; 998 break; 999 } 1000 1001 /* 1002 * We have no more space and have something to offer, 1003 * wake up select/poll. 1004 */ 1005 if (bp->cnt) 1006 pipeselwakeup(wpipe, wpipe, POLL_IN); 1007 1008 if (wakeup_state & PIPE_RESTART) { 1009 error = ERESTART; 1010 break; 1011 } 1012 1013 pipeunlock(wpipe); 1014 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 1015 (void)pipelock(wpipe, 0); 1016 if (error != 0) 1017 break; 1018 /* 1019 * If read side wants to go away, we just issue a signal 1020 * to ourselves. 1021 */ 1022 if (wpipe->pipe_state & PIPE_EOF) { 1023 error = EPIPE; 1024 break; 1025 } 1026 wakeup_state = wpipe->pipe_state; 1027 } 1028 } 1029 1030 --wpipe->pipe_busy; 1031 if (wpipe->pipe_busy == 0) { 1032 wpipe->pipe_state &= ~PIPE_RESTART; 1033 cv_broadcast(&wpipe->pipe_draincv); 1034 } 1035 if (bp->cnt > 0) { 1036 cv_broadcast(&wpipe->pipe_rcv); 1037 } 1038 1039 /* 1040 * Don't return EPIPE if I/O was successful 1041 */ 1042 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1043 error = 0; 1044 1045 if (error == 0) 1046 getnanotime(&wpipe->pipe_mtime); 1047 1048 /* 1049 * We have something to offer, wake up select/poll. 1050 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1051 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1052 */ 1053 if (bp->cnt) 1054 pipeselwakeup(wpipe, wpipe, POLL_IN); 1055 1056 /* 1057 * Arrange for next read(2) to do a signal. 1058 */ 1059 wpipe->pipe_state |= PIPE_SIGNALR; 1060 1061 pipeunlock(wpipe); 1062 mutex_exit(lock); 1063 return (error); 1064} 1065 1066/* 1067 * We implement a very minimal set of ioctls for compatibility with sockets. 1068 */ 1069int 1070pipe_ioctl(file_t *fp, u_long cmd, void *data) 1071{ 1072 struct pipe *pipe = fp->f_data; 1073 kmutex_t *lock = pipe->pipe_lock; 1074 1075 switch (cmd) { 1076 1077 case FIONBIO: 1078 return (0); 1079 1080 case FIOASYNC: 1081 mutex_enter(lock); 1082 if (*(int *)data) { 1083 pipe->pipe_state |= PIPE_ASYNC; 1084 } else { 1085 pipe->pipe_state &= ~PIPE_ASYNC; 1086 } 1087 mutex_exit(lock); 1088 return (0); 1089 1090 case FIONREAD: 1091 mutex_enter(lock); 1092#ifndef PIPE_NODIRECT 1093 if (pipe->pipe_state & PIPE_DIRECTW) 1094 *(int *)data = pipe->pipe_map.cnt; 1095 else 1096#endif 1097 *(int *)data = pipe->pipe_buffer.cnt; 1098 mutex_exit(lock); 1099 return (0); 1100 1101 case FIONWRITE: 1102 /* Look at other side */ 1103 pipe = pipe->pipe_peer; 1104 mutex_enter(lock); 1105#ifndef PIPE_NODIRECT 1106 if (pipe->pipe_state & PIPE_DIRECTW) 1107 *(int *)data = pipe->pipe_map.cnt; 1108 else 1109#endif 1110 *(int *)data = pipe->pipe_buffer.cnt; 1111 mutex_exit(lock); 1112 return (0); 1113 1114 case FIONSPACE: 1115 /* Look at other side */ 1116 pipe = pipe->pipe_peer; 1117 mutex_enter(lock); 1118#ifndef PIPE_NODIRECT 1119 /* 1120 * If we're in direct-mode, we don't really have a 1121 * send queue, and any other write will block. Thus 1122 * zero seems like the best answer. 1123 */ 1124 if (pipe->pipe_state & PIPE_DIRECTW) 1125 *(int *)data = 0; 1126 else 1127#endif 1128 *(int *)data = pipe->pipe_buffer.size - 1129 pipe->pipe_buffer.cnt; 1130 mutex_exit(lock); 1131 return (0); 1132 1133 case TIOCSPGRP: 1134 case FIOSETOWN: 1135 return fsetown(&pipe->pipe_pgid, cmd, data); 1136 1137 case TIOCGPGRP: 1138 case FIOGETOWN: 1139 return fgetown(pipe->pipe_pgid, cmd, data); 1140 1141 } 1142 return (EPASSTHROUGH); 1143} 1144 1145int 1146pipe_poll(file_t *fp, int events) 1147{ 1148 struct pipe *rpipe = fp->f_data; 1149 struct pipe *wpipe; 1150 int eof = 0; 1151 int revents = 0; 1152 1153 mutex_enter(rpipe->pipe_lock); 1154 wpipe = rpipe->pipe_peer; 1155 1156 if (events & (POLLIN | POLLRDNORM)) 1157 if ((rpipe->pipe_buffer.cnt > 0) || 1158#ifndef PIPE_NODIRECT 1159 (rpipe->pipe_state & PIPE_DIRECTR) || 1160#endif 1161 (rpipe->pipe_state & PIPE_EOF)) 1162 revents |= events & (POLLIN | POLLRDNORM); 1163 1164 eof |= (rpipe->pipe_state & PIPE_EOF); 1165 1166 if (wpipe == NULL) 1167 revents |= events & (POLLOUT | POLLWRNORM); 1168 else { 1169 if (events & (POLLOUT | POLLWRNORM)) 1170 if ((wpipe->pipe_state & PIPE_EOF) || ( 1171#ifndef PIPE_NODIRECT 1172 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1173#endif 1174 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1175 revents |= events & (POLLOUT | POLLWRNORM); 1176 1177 eof |= (wpipe->pipe_state & PIPE_EOF); 1178 } 1179 1180 if (wpipe == NULL || eof) 1181 revents |= POLLHUP; 1182 1183 if (revents == 0) { 1184 if (events & (POLLIN | POLLRDNORM)) 1185 selrecord(curlwp, &rpipe->pipe_sel); 1186 1187 if (events & (POLLOUT | POLLWRNORM)) 1188 selrecord(curlwp, &wpipe->pipe_sel); 1189 } 1190 mutex_exit(rpipe->pipe_lock); 1191 1192 return (revents); 1193} 1194 1195static int 1196pipe_stat(file_t *fp, struct stat *ub) 1197{ 1198 struct pipe *pipe = fp->f_data; 1199 1200 mutex_enter(pipe->pipe_lock); 1201 memset(ub, 0, sizeof(*ub)); 1202 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1203 ub->st_blksize = pipe->pipe_buffer.size; 1204 if (ub->st_blksize == 0 && pipe->pipe_peer) 1205 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1206 ub->st_size = pipe->pipe_buffer.cnt; 1207 ub->st_blocks = (ub->st_size) ? 1 : 0; 1208 ub->st_atimespec = pipe->pipe_atime; 1209 ub->st_mtimespec = pipe->pipe_mtime; 1210 ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime; 1211 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1212 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1213 1214 /* 1215 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1216 * XXX (st_dev, st_ino) should be unique. 1217 */ 1218 mutex_exit(pipe->pipe_lock); 1219 return 0; 1220} 1221 1222static int 1223pipe_close(file_t *fp) 1224{ 1225 struct pipe *pipe = fp->f_data; 1226 1227 fp->f_data = NULL; 1228 pipeclose(pipe); 1229 return (0); 1230} 1231 1232static void 1233pipe_restart(file_t *fp) 1234{ 1235 struct pipe *pipe = fp->f_data; 1236 1237 /* 1238 * Unblock blocked reads/writes in order to allow close() to complete. 1239 * System calls return ERESTART so that the fd is revalidated. 1240 * (Partial writes return the transfer length.) 1241 */ 1242 mutex_enter(pipe->pipe_lock); 1243 pipe->pipe_state |= PIPE_RESTART; 1244 /* Wakeup both cvs, maybe we only need one, but maybe there are some 1245 * other paths where wakeup is needed, and it saves deciding which! */ 1246 cv_broadcast(&pipe->pipe_rcv); 1247 cv_broadcast(&pipe->pipe_wcv); 1248 mutex_exit(pipe->pipe_lock); 1249} 1250 1251static void 1252pipe_free_kmem(struct pipe *pipe) 1253{ 1254 1255 if (pipe->pipe_buffer.buffer != NULL) { 1256 if (pipe->pipe_buffer.size > PIPE_SIZE) { 1257 atomic_dec_uint(&nbigpipe); 1258 } 1259 if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) { 1260 uvm_km_free(kernel_map, 1261 (vaddr_t)pipe->pipe_buffer.buffer, 1262 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1263 atomic_add_int(&amountpipekva, 1264 -pipe->pipe_buffer.size); 1265 } 1266 pipe->pipe_buffer.buffer = NULL; 1267 } 1268#ifndef PIPE_NODIRECT 1269 if (pipe->pipe_map.kva != 0) { 1270 pipe_loan_free(pipe); 1271 pipe->pipe_map.cnt = 0; 1272 pipe->pipe_map.kva = 0; 1273 pipe->pipe_map.pos = 0; 1274 pipe->pipe_map.npages = 0; 1275 } 1276#endif /* !PIPE_NODIRECT */ 1277} 1278 1279/* 1280 * Shutdown the pipe. 1281 */ 1282static void 1283pipeclose(struct pipe *pipe) 1284{ 1285 kmutex_t *lock; 1286 struct pipe *ppipe; 1287 1288 if (pipe == NULL) 1289 return; 1290 1291 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1292 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1293 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1294 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1295 1296 lock = pipe->pipe_lock; 1297 if (lock == NULL) 1298 /* Must have failed during create */ 1299 goto free_resources; 1300 1301 mutex_enter(lock); 1302 pipeselwakeup(pipe, pipe, POLL_HUP); 1303 1304 /* 1305 * If the other side is blocked, wake it up saying that 1306 * we want to close it down. 1307 */ 1308 pipe->pipe_state |= PIPE_EOF; 1309 if (pipe->pipe_busy) { 1310 while (pipe->pipe_busy) { 1311 cv_broadcast(&pipe->pipe_wcv); 1312 cv_wait_sig(&pipe->pipe_draincv, lock); 1313 } 1314 } 1315 1316 /* 1317 * Disconnect from peer. 1318 */ 1319 if ((ppipe = pipe->pipe_peer) != NULL) { 1320 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1321 ppipe->pipe_state |= PIPE_EOF; 1322 cv_broadcast(&ppipe->pipe_rcv); 1323 ppipe->pipe_peer = NULL; 1324 } 1325 1326 /* 1327 * Any knote objects still left in the list are 1328 * the one attached by peer. Since no one will 1329 * traverse this list, we just clear it. 1330 */ 1331 SLIST_INIT(&pipe->pipe_sel.sel_klist); 1332 1333 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1334 mutex_exit(lock); 1335 mutex_obj_free(lock); 1336 1337 /* 1338 * Free resources. 1339 */ 1340 free_resources: 1341 pipe->pipe_pgid = 0; 1342 pipe->pipe_state = PIPE_SIGNALR; 1343 pipe_free_kmem(pipe); 1344 if (pipe->pipe_kmem != 0) { 1345 pool_cache_put(pipe_rd_cache, pipe); 1346 } else { 1347 pool_cache_put(pipe_wr_cache, pipe); 1348 } 1349} 1350 1351static void 1352filt_pipedetach(struct knote *kn) 1353{ 1354 struct pipe *pipe; 1355 kmutex_t *lock; 1356 1357 pipe = ((file_t *)kn->kn_obj)->f_data; 1358 lock = pipe->pipe_lock; 1359 1360 mutex_enter(lock); 1361 1362 switch(kn->kn_filter) { 1363 case EVFILT_WRITE: 1364 /* Need the peer structure, not our own. */ 1365 pipe = pipe->pipe_peer; 1366 1367 /* If reader end already closed, just return. */ 1368 if (pipe == NULL) { 1369 mutex_exit(lock); 1370 return; 1371 } 1372 1373 break; 1374 default: 1375 /* Nothing to do. */ 1376 break; 1377 } 1378 1379 KASSERT(kn->kn_hook == pipe); 1380 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1381 mutex_exit(lock); 1382} 1383 1384static int 1385filt_piperead(struct knote *kn, long hint) 1386{ 1387 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1388 struct pipe *wpipe; 1389 1390 if ((hint & NOTE_SUBMIT) == 0) { 1391 mutex_enter(rpipe->pipe_lock); 1392 } 1393 wpipe = rpipe->pipe_peer; 1394 kn->kn_data = rpipe->pipe_buffer.cnt; 1395 1396 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1397 kn->kn_data = rpipe->pipe_map.cnt; 1398 1399 if ((rpipe->pipe_state & PIPE_EOF) || 1400 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1401 kn->kn_flags |= EV_EOF; 1402 if ((hint & NOTE_SUBMIT) == 0) { 1403 mutex_exit(rpipe->pipe_lock); 1404 } 1405 return (1); 1406 } 1407 1408 if ((hint & NOTE_SUBMIT) == 0) { 1409 mutex_exit(rpipe->pipe_lock); 1410 } 1411 return (kn->kn_data > 0); 1412} 1413 1414static int 1415filt_pipewrite(struct knote *kn, long hint) 1416{ 1417 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1418 struct pipe *wpipe; 1419 1420 if ((hint & NOTE_SUBMIT) == 0) { 1421 mutex_enter(rpipe->pipe_lock); 1422 } 1423 wpipe = rpipe->pipe_peer; 1424 1425 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1426 kn->kn_data = 0; 1427 kn->kn_flags |= EV_EOF; 1428 if ((hint & NOTE_SUBMIT) == 0) { 1429 mutex_exit(rpipe->pipe_lock); 1430 } 1431 return (1); 1432 } 1433 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1434 if (wpipe->pipe_state & PIPE_DIRECTW) 1435 kn->kn_data = 0; 1436 1437 if ((hint & NOTE_SUBMIT) == 0) { 1438 mutex_exit(rpipe->pipe_lock); 1439 } 1440 return (kn->kn_data >= PIPE_BUF); 1441} 1442 1443static const struct filterops pipe_rfiltops = 1444 { 1, NULL, filt_pipedetach, filt_piperead }; 1445static const struct filterops pipe_wfiltops = 1446 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1447 1448static int 1449pipe_kqfilter(file_t *fp, struct knote *kn) 1450{ 1451 struct pipe *pipe; 1452 kmutex_t *lock; 1453 1454 pipe = ((file_t *)kn->kn_obj)->f_data; 1455 lock = pipe->pipe_lock; 1456 1457 mutex_enter(lock); 1458 1459 switch (kn->kn_filter) { 1460 case EVFILT_READ: 1461 kn->kn_fop = &pipe_rfiltops; 1462 break; 1463 case EVFILT_WRITE: 1464 kn->kn_fop = &pipe_wfiltops; 1465 pipe = pipe->pipe_peer; 1466 if (pipe == NULL) { 1467 /* Other end of pipe has been closed. */ 1468 mutex_exit(lock); 1469 return (EBADF); 1470 } 1471 break; 1472 default: 1473 mutex_exit(lock); 1474 return (EINVAL); 1475 } 1476 1477 kn->kn_hook = pipe; 1478 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1479 mutex_exit(lock); 1480 1481 return (0); 1482} 1483 1484/* 1485 * Handle pipe sysctls. 1486 */ 1487SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1488{ 1489 1490 sysctl_createv(clog, 0, NULL, NULL, 1491 CTLFLAG_PERMANENT, 1492 CTLTYPE_NODE, "kern", NULL, 1493 NULL, 0, NULL, 0, 1494 CTL_KERN, CTL_EOL); 1495 sysctl_createv(clog, 0, NULL, NULL, 1496 CTLFLAG_PERMANENT, 1497 CTLTYPE_NODE, "pipe", 1498 SYSCTL_DESCR("Pipe settings"), 1499 NULL, 0, NULL, 0, 1500 CTL_KERN, KERN_PIPE, CTL_EOL); 1501 1502 sysctl_createv(clog, 0, NULL, NULL, 1503 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1504 CTLTYPE_INT, "maxkvasz", 1505 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1506 "used for pipes"), 1507 NULL, 0, &maxpipekva, 0, 1508 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1509 sysctl_createv(clog, 0, NULL, NULL, 1510 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1511 CTLTYPE_INT, "maxloankvasz", 1512 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1513 NULL, 0, &limitpipekva, 0, 1514 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1515 sysctl_createv(clog, 0, NULL, NULL, 1516 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1517 CTLTYPE_INT, "maxbigpipes", 1518 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1519 NULL, 0, &maxbigpipes, 0, 1520 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1521 sysctl_createv(clog, 0, NULL, NULL, 1522 CTLFLAG_PERMANENT, 1523 CTLTYPE_INT, "nbigpipes", 1524 SYSCTL_DESCR("Number of \"big\" pipes"), 1525 NULL, 0, &nbigpipe, 0, 1526 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1527 sysctl_createv(clog, 0, NULL, NULL, 1528 CTLFLAG_PERMANENT, 1529 CTLTYPE_INT, "kvasize", 1530 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1531 "buffers"), 1532 NULL, 0, &amountpipekva, 0, 1533 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1534} 1535