sys_pipe.c revision 118230
1/* 2 * Copyright (c) 1996 John S. Dyson 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice immediately at the beginning of the file, without modification, 10 * this list of conditions, and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. Absolutely no warranty of function or purpose is made by the author 15 * John S. Dyson. 16 * 4. Modifications may be freely made to this file if the above conditions 17 * are met. 18 */ 19 20/* 21 * This file contains a high-performance replacement for the socket-based 22 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 23 * all features of sockets, but does do everything that pipes normally 24 * do. 25 */ 26 27/* 28 * This code has two modes of operation, a small write mode and a large 29 * write mode. The small write mode acts like conventional pipes with 30 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 31 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 32 * and PIPE_SIZE in size, it is fully mapped and wired into the kernel, and 33 * the receiving process can copy it directly from the pages in the sending 34 * process. 35 * 36 * If the sending process receives a signal, it is possible that it will 37 * go away, and certainly its address space can change, because control 38 * is returned back to the user-mode side. In that case, the pipe code 39 * arranges to copy the buffer supplied by the user process, to a pageable 40 * kernel buffer, and the receiving process will grab the data from the 41 * pageable kernel buffer. Since signals don't happen all that often, 42 * the copy operation is normally eliminated. 43 * 44 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 45 * happen for small transfers so that the system will not spend all of 46 * its time context switching. PIPE_SIZE is constrained by the 47 * amount of kernel virtual memory. 48 * 49 * In order to limit the resource use of pipes, three sysctls exist: 50 * 51 * kern.ipc.maxpipes - A limit on the total number of pipes in the system. 52 * Note that since pipes are bidirectional, the effective value is this 53 * number divided by two. 54 * 55 * kern.ipc.maxpipekva - This value limits the amount of pageable memory that 56 * can be used by pipes. Whenever the amount in use exceeds this value, 57 * all new pipes will be SMALL_PIPE_SIZE in size, rather than PIPE_SIZE. 58 * Big pipe creation will be limited as well. 59 * 60 * kern.ipc.maxpipekvawired - This value limits the amount of memory that may 61 * be wired in order to facilitate direct copies using page flipping. 62 * Whenever this value is exceeded, pipes will fall back to using regular 63 * copies. 64 * 65 * These values are autotuned in subr_param.c. 66 * 67 * Memory usage may be monitored through the sysctls 68 * kern.ipc.pipes, kern.ipc.pipekva and kern.ipc.pipekvawired. 69 * 70 */ 71 72#include <sys/cdefs.h> 73__FBSDID("$FreeBSD: head/sys/kern/sys_pipe.c 118230 2003-07-30 22:50:37Z pb $"); 74 75#include "opt_mac.h" 76 77#include <sys/param.h> 78#include <sys/systm.h> 79#include <sys/fcntl.h> 80#include <sys/file.h> 81#include <sys/filedesc.h> 82#include <sys/filio.h> 83#include <sys/kernel.h> 84#include <sys/lock.h> 85#include <sys/mac.h> 86#include <sys/mutex.h> 87#include <sys/ttycom.h> 88#include <sys/stat.h> 89#include <sys/malloc.h> 90#include <sys/poll.h> 91#include <sys/selinfo.h> 92#include <sys/signalvar.h> 93#include <sys/sysctl.h> 94#include <sys/sysproto.h> 95#include <sys/pipe.h> 96#include <sys/proc.h> 97#include <sys/vnode.h> 98#include <sys/uio.h> 99#include <sys/event.h> 100 101#include <vm/vm.h> 102#include <vm/vm_param.h> 103#include <vm/vm_object.h> 104#include <vm/vm_kern.h> 105#include <vm/vm_extern.h> 106#include <vm/pmap.h> 107#include <vm/vm_map.h> 108#include <vm/vm_page.h> 109#include <vm/uma.h> 110 111/* 112 * Use this define if you want to disable *fancy* VM things. Expect an 113 * approx 30% decrease in transfer rate. This could be useful for 114 * NetBSD or OpenBSD. 115 */ 116/* #define PIPE_NODIRECT */ 117 118/* 119 * interfaces to the outside world 120 */ 121static fo_rdwr_t pipe_read; 122static fo_rdwr_t pipe_write; 123static fo_ioctl_t pipe_ioctl; 124static fo_poll_t pipe_poll; 125static fo_kqfilter_t pipe_kqfilter; 126static fo_stat_t pipe_stat; 127static fo_close_t pipe_close; 128 129static struct fileops pipeops = { 130 .fo_read = pipe_read, 131 .fo_write = pipe_write, 132 .fo_ioctl = pipe_ioctl, 133 .fo_poll = pipe_poll, 134 .fo_kqfilter = pipe_kqfilter, 135 .fo_stat = pipe_stat, 136 .fo_close = pipe_close, 137 .fo_flags = DFLAG_PASSABLE 138}; 139 140static void filt_pipedetach(struct knote *kn); 141static int filt_piperead(struct knote *kn, long hint); 142static int filt_pipewrite(struct knote *kn, long hint); 143 144static struct filterops pipe_rfiltops = 145 { 1, NULL, filt_pipedetach, filt_piperead }; 146static struct filterops pipe_wfiltops = 147 { 1, NULL, filt_pipedetach, filt_pipewrite }; 148 149#define PIPE_GET_GIANT(pipe) \ 150 do { \ 151 KASSERT(((pipe)->pipe_state & PIPE_LOCKFL) != 0, \ 152 ("%s:%d PIPE_GET_GIANT: line pipe not locked", \ 153 __FILE__, __LINE__)); \ 154 PIPE_UNLOCK(pipe); \ 155 mtx_lock(&Giant); \ 156 } while (0) 157 158#define PIPE_DROP_GIANT(pipe) \ 159 do { \ 160 mtx_unlock(&Giant); \ 161 PIPE_LOCK(pipe); \ 162 } while (0) 163 164/* 165 * Default pipe buffer size(s), this can be kind-of large now because pipe 166 * space is pageable. The pipe code will try to maintain locality of 167 * reference for performance reasons, so small amounts of outstanding I/O 168 * will not wipe the cache. 169 */ 170#define MINPIPESIZE (PIPE_SIZE/3) 171#define MAXPIPESIZE (2*PIPE_SIZE/3) 172 173/* 174 * Limit the number of "big" pipes 175 */ 176#define LIMITBIGPIPES 32 177static int nbigpipe; 178 179static int amountpipes; 180static int amountpipekva; 181static int amountpipekvawired; 182 183SYSCTL_DECL(_kern_ipc); 184 185SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipes, CTLFLAG_RW, 186 &maxpipes, 0, "Max # of pipes"); 187SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RW, 188 &maxpipekva, 0, "Pipe KVA limit"); 189SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekvawired, CTLFLAG_RW, 190 &maxpipekvawired, 0, "Pipe KVA wired limit"); 191SYSCTL_INT(_kern_ipc, OID_AUTO, pipes, CTLFLAG_RD, 192 &amountpipes, 0, "Current # of pipes"); 193SYSCTL_INT(_kern_ipc, OID_AUTO, bigpipes, CTLFLAG_RD, 194 &nbigpipe, 0, "Current # of big pipes"); 195SYSCTL_INT(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 196 &amountpipekva, 0, "Pipe KVA usage"); 197SYSCTL_INT(_kern_ipc, OID_AUTO, pipekvawired, CTLFLAG_RD, 198 &amountpipekvawired, 0, "Pipe wired KVA usage"); 199 200static void pipeinit(void *dummy __unused); 201static void pipeclose(struct pipe *cpipe); 202static void pipe_free_kmem(struct pipe *cpipe); 203static int pipe_create(struct pipe **cpipep); 204static __inline int pipelock(struct pipe *cpipe, int catch); 205static __inline void pipeunlock(struct pipe *cpipe); 206static __inline void pipeselwakeup(struct pipe *cpipe); 207#ifndef PIPE_NODIRECT 208static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 209static void pipe_destroy_write_buffer(struct pipe *wpipe); 210static int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 211static void pipe_clone_write_buffer(struct pipe *wpipe); 212#endif 213static int pipespace(struct pipe *cpipe, int size); 214 215static uma_zone_t pipe_zone; 216 217SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 218 219static void 220pipeinit(void *dummy __unused) 221{ 222 pipe_zone = uma_zcreate("PIPE", sizeof(struct pipe), NULL, 223 NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 224} 225 226/* 227 * The pipe system call for the DTYPE_PIPE type of pipes 228 */ 229 230/* ARGSUSED */ 231int 232pipe(td, uap) 233 struct thread *td; 234 struct pipe_args /* { 235 int dummy; 236 } */ *uap; 237{ 238 struct filedesc *fdp = td->td_proc->p_fd; 239 struct file *rf, *wf; 240 struct pipe *rpipe, *wpipe; 241 struct mtx *pmtx; 242 int fd, error; 243 244 KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 245 246 pmtx = malloc(sizeof(*pmtx), M_TEMP, M_WAITOK | M_ZERO); 247 248 rpipe = wpipe = NULL; 249 if (pipe_create(&rpipe) || pipe_create(&wpipe)) { 250 pipeclose(rpipe); 251 pipeclose(wpipe); 252 free(pmtx, M_TEMP); 253 return (ENFILE); 254 } 255 256 rpipe->pipe_state |= PIPE_DIRECTOK; 257 wpipe->pipe_state |= PIPE_DIRECTOK; 258 259 error = falloc(td, &rf, &fd); 260 if (error) { 261 pipeclose(rpipe); 262 pipeclose(wpipe); 263 free(pmtx, M_TEMP); 264 return (error); 265 } 266 fhold(rf); 267 td->td_retval[0] = fd; 268 269 /* 270 * Warning: once we've gotten past allocation of the fd for the 271 * read-side, we can only drop the read side via fdrop() in order 272 * to avoid races against processes which manage to dup() the read 273 * side while we are blocked trying to allocate the write side. 274 */ 275 FILE_LOCK(rf); 276 rf->f_flag = FREAD | FWRITE; 277 rf->f_type = DTYPE_PIPE; 278 rf->f_data = rpipe; 279 rf->f_ops = &pipeops; 280 FILE_UNLOCK(rf); 281 error = falloc(td, &wf, &fd); 282 if (error) { 283 FILEDESC_LOCK(fdp); 284 if (fdp->fd_ofiles[td->td_retval[0]] == rf) { 285 fdp->fd_ofiles[td->td_retval[0]] = NULL; 286 FILEDESC_UNLOCK(fdp); 287 fdrop(rf, td); 288 } else 289 FILEDESC_UNLOCK(fdp); 290 fdrop(rf, td); 291 /* rpipe has been closed by fdrop(). */ 292 pipeclose(wpipe); 293 free(pmtx, M_TEMP); 294 return (error); 295 } 296 FILE_LOCK(wf); 297 wf->f_flag = FREAD | FWRITE; 298 wf->f_type = DTYPE_PIPE; 299 wf->f_data = wpipe; 300 wf->f_ops = &pipeops; 301 FILE_UNLOCK(wf); 302 td->td_retval[1] = fd; 303 rpipe->pipe_peer = wpipe; 304 wpipe->pipe_peer = rpipe; 305#ifdef MAC 306 /* 307 * struct pipe represents a pipe endpoint. The MAC label is shared 308 * between the connected endpoints. As a result mac_init_pipe() and 309 * mac_create_pipe() should only be called on one of the endpoints 310 * after they have been connected. 311 */ 312 mac_init_pipe(rpipe); 313 mac_create_pipe(td->td_ucred, rpipe); 314#endif 315 mtx_init(pmtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 316 rpipe->pipe_mtxp = wpipe->pipe_mtxp = pmtx; 317 fdrop(rf, td); 318 319 return (0); 320} 321 322/* 323 * Allocate kva for pipe circular buffer, the space is pageable 324 * This routine will 'realloc' the size of a pipe safely, if it fails 325 * it will retain the old buffer. 326 * If it fails it will return ENOMEM. 327 */ 328static int 329pipespace(cpipe, size) 330 struct pipe *cpipe; 331 int size; 332{ 333 struct vm_object *object; 334 caddr_t buffer; 335 int npages, error; 336 static int curfail = 0; 337 static struct timeval lastfail; 338 339 GIANT_REQUIRED; 340 KASSERT(cpipe->pipe_mtxp == NULL || !mtx_owned(PIPE_MTX(cpipe)), 341 ("pipespace: pipe mutex locked")); 342 343 if (amountpipes > maxpipes) { 344 if (ppsratecheck(&lastfail, &curfail, 1)) 345 printf("kern.maxpipes exceeded, please see tuning(7).\n"); 346 return (ENOMEM); 347 } 348 349 npages = round_page(size)/PAGE_SIZE; 350 /* 351 * Create an object, I don't like the idea of paging to/from 352 * kernel_object. 353 * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. 354 */ 355 object = vm_object_allocate(OBJT_DEFAULT, npages); 356 buffer = (caddr_t) vm_map_min(kernel_map); 357 358 /* 359 * Insert the object into the kernel map, and allocate kva for it. 360 * The map entry is, by default, pageable. 361 * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. 362 */ 363 error = vm_map_find(kernel_map, object, 0, 364 (vm_offset_t *) &buffer, size, 1, 365 VM_PROT_ALL, VM_PROT_ALL, 0); 366 367 if (error != KERN_SUCCESS) { 368 vm_object_deallocate(object); 369 return (ENOMEM); 370 } 371 372 /* free old resources if we're resizing */ 373 pipe_free_kmem(cpipe); 374 cpipe->pipe_buffer.object = object; 375 cpipe->pipe_buffer.buffer = buffer; 376 cpipe->pipe_buffer.size = size; 377 cpipe->pipe_buffer.in = 0; 378 cpipe->pipe_buffer.out = 0; 379 cpipe->pipe_buffer.cnt = 0; 380 atomic_add_int(&amountpipes, 1); 381 atomic_add_int(&amountpipekva, cpipe->pipe_buffer.size); 382 return (0); 383} 384 385/* 386 * initialize and allocate VM and memory for pipe 387 */ 388static int 389pipe_create(cpipep) 390 struct pipe **cpipep; 391{ 392 struct pipe *cpipe; 393 int error; 394 395 *cpipep = uma_zalloc(pipe_zone, M_WAITOK); 396 if (*cpipep == NULL) 397 return (ENOMEM); 398 399 cpipe = *cpipep; 400 401 /* so pipespace()->pipe_free_kmem() doesn't follow junk pointer */ 402 cpipe->pipe_buffer.object = NULL; 403#ifndef PIPE_NODIRECT 404 cpipe->pipe_map.kva = 0; 405#endif 406 /* 407 * protect so pipeclose() doesn't follow a junk pointer 408 * if pipespace() fails. 409 */ 410 bzero(&cpipe->pipe_sel, sizeof(cpipe->pipe_sel)); 411 cpipe->pipe_state = 0; 412 cpipe->pipe_peer = NULL; 413 cpipe->pipe_busy = 0; 414 415#ifndef PIPE_NODIRECT 416 /* 417 * pipe data structure initializations to support direct pipe I/O 418 */ 419 cpipe->pipe_map.cnt = 0; 420 cpipe->pipe_map.kva = 0; 421 cpipe->pipe_map.pos = 0; 422 cpipe->pipe_map.npages = 0; 423 /* cpipe->pipe_map.ms[] = invalid */ 424#endif 425 426 cpipe->pipe_mtxp = NULL; /* avoid pipespace assertion */ 427 /* 428 * Reduce to 1/4th pipe size if we're over our global max. 429 */ 430 if (amountpipekva > maxpipekva) 431 error = pipespace(cpipe, SMALL_PIPE_SIZE); 432 else 433 error = pipespace(cpipe, PIPE_SIZE); 434 if (error) 435 return (error); 436 437 vfs_timestamp(&cpipe->pipe_ctime); 438 cpipe->pipe_atime = cpipe->pipe_ctime; 439 cpipe->pipe_mtime = cpipe->pipe_ctime; 440 441 return (0); 442} 443 444 445/* 446 * lock a pipe for I/O, blocking other access 447 */ 448static __inline int 449pipelock(cpipe, catch) 450 struct pipe *cpipe; 451 int catch; 452{ 453 int error; 454 455 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 456 while (cpipe->pipe_state & PIPE_LOCKFL) { 457 cpipe->pipe_state |= PIPE_LWANT; 458 error = msleep(cpipe, PIPE_MTX(cpipe), 459 catch ? (PRIBIO | PCATCH) : PRIBIO, 460 "pipelk", 0); 461 if (error != 0) 462 return (error); 463 } 464 cpipe->pipe_state |= PIPE_LOCKFL; 465 return (0); 466} 467 468/* 469 * unlock a pipe I/O lock 470 */ 471static __inline void 472pipeunlock(cpipe) 473 struct pipe *cpipe; 474{ 475 476 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 477 cpipe->pipe_state &= ~PIPE_LOCKFL; 478 if (cpipe->pipe_state & PIPE_LWANT) { 479 cpipe->pipe_state &= ~PIPE_LWANT; 480 wakeup(cpipe); 481 } 482} 483 484static __inline void 485pipeselwakeup(cpipe) 486 struct pipe *cpipe; 487{ 488 489 if (cpipe->pipe_state & PIPE_SEL) { 490 cpipe->pipe_state &= ~PIPE_SEL; 491 selwakeup(&cpipe->pipe_sel); 492 } 493 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 494 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 495 KNOTE(&cpipe->pipe_sel.si_note, 0); 496} 497 498/* ARGSUSED */ 499static int 500pipe_read(fp, uio, active_cred, flags, td) 501 struct file *fp; 502 struct uio *uio; 503 struct ucred *active_cred; 504 struct thread *td; 505 int flags; 506{ 507 struct pipe *rpipe = fp->f_data; 508 int error; 509 int nread = 0; 510 u_int size; 511 512 PIPE_LOCK(rpipe); 513 ++rpipe->pipe_busy; 514 error = pipelock(rpipe, 1); 515 if (error) 516 goto unlocked_error; 517 518#ifdef MAC 519 error = mac_check_pipe_read(active_cred, rpipe); 520 if (error) 521 goto locked_error; 522#endif 523 524 while (uio->uio_resid) { 525 /* 526 * normal pipe buffer receive 527 */ 528 if (rpipe->pipe_buffer.cnt > 0) { 529 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 530 if (size > rpipe->pipe_buffer.cnt) 531 size = rpipe->pipe_buffer.cnt; 532 if (size > (u_int) uio->uio_resid) 533 size = (u_int) uio->uio_resid; 534 535 PIPE_UNLOCK(rpipe); 536 error = uiomove( 537 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 538 size, uio); 539 PIPE_LOCK(rpipe); 540 if (error) 541 break; 542 543 rpipe->pipe_buffer.out += size; 544 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 545 rpipe->pipe_buffer.out = 0; 546 547 rpipe->pipe_buffer.cnt -= size; 548 549 /* 550 * If there is no more to read in the pipe, reset 551 * its pointers to the beginning. This improves 552 * cache hit stats. 553 */ 554 if (rpipe->pipe_buffer.cnt == 0) { 555 rpipe->pipe_buffer.in = 0; 556 rpipe->pipe_buffer.out = 0; 557 } 558 nread += size; 559#ifndef PIPE_NODIRECT 560 /* 561 * Direct copy, bypassing a kernel buffer. 562 */ 563 } else if ((size = rpipe->pipe_map.cnt) && 564 (rpipe->pipe_state & PIPE_DIRECTW)) { 565 caddr_t va; 566 if (size > (u_int) uio->uio_resid) 567 size = (u_int) uio->uio_resid; 568 569 va = (caddr_t) rpipe->pipe_map.kva + 570 rpipe->pipe_map.pos; 571 PIPE_UNLOCK(rpipe); 572 error = uiomove(va, size, uio); 573 PIPE_LOCK(rpipe); 574 if (error) 575 break; 576 nread += size; 577 rpipe->pipe_map.pos += size; 578 rpipe->pipe_map.cnt -= size; 579 if (rpipe->pipe_map.cnt == 0) { 580 rpipe->pipe_state &= ~PIPE_DIRECTW; 581 wakeup(rpipe); 582 } 583#endif 584 } else { 585 /* 586 * detect EOF condition 587 * read returns 0 on EOF, no need to set error 588 */ 589 if (rpipe->pipe_state & PIPE_EOF) 590 break; 591 592 /* 593 * If the "write-side" has been blocked, wake it up now. 594 */ 595 if (rpipe->pipe_state & PIPE_WANTW) { 596 rpipe->pipe_state &= ~PIPE_WANTW; 597 wakeup(rpipe); 598 } 599 600 /* 601 * Break if some data was read. 602 */ 603 if (nread > 0) 604 break; 605 606 /* 607 * Unlock the pipe buffer for our remaining processing. 608 * We will either break out with an error or we will 609 * sleep and relock to loop. 610 */ 611 pipeunlock(rpipe); 612 613 /* 614 * Handle non-blocking mode operation or 615 * wait for more data. 616 */ 617 if (fp->f_flag & FNONBLOCK) { 618 error = EAGAIN; 619 } else { 620 rpipe->pipe_state |= PIPE_WANTR; 621 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 622 PRIBIO | PCATCH, 623 "piperd", 0)) == 0) 624 error = pipelock(rpipe, 1); 625 } 626 if (error) 627 goto unlocked_error; 628 } 629 } 630#ifdef MAC 631locked_error: 632#endif 633 pipeunlock(rpipe); 634 635 /* XXX: should probably do this before getting any locks. */ 636 if (error == 0) 637 vfs_timestamp(&rpipe->pipe_atime); 638unlocked_error: 639 --rpipe->pipe_busy; 640 641 /* 642 * PIPE_WANT processing only makes sense if pipe_busy is 0. 643 */ 644 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 645 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 646 wakeup(rpipe); 647 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 648 /* 649 * Handle write blocking hysteresis. 650 */ 651 if (rpipe->pipe_state & PIPE_WANTW) { 652 rpipe->pipe_state &= ~PIPE_WANTW; 653 wakeup(rpipe); 654 } 655 } 656 657 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 658 pipeselwakeup(rpipe); 659 660 PIPE_UNLOCK(rpipe); 661 return (error); 662} 663 664#ifndef PIPE_NODIRECT 665/* 666 * Map the sending processes' buffer into kernel space and wire it. 667 * This is similar to a physical write operation. 668 */ 669static int 670pipe_build_write_buffer(wpipe, uio) 671 struct pipe *wpipe; 672 struct uio *uio; 673{ 674 u_int size; 675 int i; 676 vm_offset_t addr, endaddr; 677 vm_paddr_t paddr; 678 679 GIANT_REQUIRED; 680 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 681 682 size = (u_int) uio->uio_iov->iov_len; 683 if (size > wpipe->pipe_buffer.size) 684 size = wpipe->pipe_buffer.size; 685 686 endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size); 687 addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base); 688 for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) { 689 vm_page_t m; 690 691 /* 692 * vm_fault_quick() can sleep. Consequently, 693 * vm_page_lock_queue() and vm_page_unlock_queue() 694 * should not be performed outside of this loop. 695 */ 696 if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0 || 697 (paddr = pmap_extract(vmspace_pmap(curproc->p_vmspace), 698 addr)) == 0) { 699 int j; 700 701 vm_page_lock_queues(); 702 for (j = 0; j < i; j++) { 703 vm_page_unwire(wpipe->pipe_map.ms[j], 1); 704 atomic_subtract_int(&amountpipekvawired, 705 PAGE_SIZE); 706 } 707 vm_page_unlock_queues(); 708 return (EFAULT); 709 } 710 711 m = PHYS_TO_VM_PAGE(paddr); 712 vm_page_lock_queues(); 713 vm_page_wire(m); 714 atomic_add_int(&amountpipekvawired, PAGE_SIZE); 715 vm_page_unlock_queues(); 716 wpipe->pipe_map.ms[i] = m; 717 } 718 719/* 720 * set up the control block 721 */ 722 wpipe->pipe_map.npages = i; 723 wpipe->pipe_map.pos = 724 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 725 wpipe->pipe_map.cnt = size; 726 727/* 728 * and map the buffer 729 */ 730 if (wpipe->pipe_map.kva == 0) { 731 /* 732 * We need to allocate space for an extra page because the 733 * address range might (will) span pages at times. 734 */ 735 wpipe->pipe_map.kva = kmem_alloc_nofault(kernel_map, 736 wpipe->pipe_buffer.size + PAGE_SIZE); 737 atomic_add_int(&amountpipekva, 738 wpipe->pipe_buffer.size + PAGE_SIZE); 739 } 740 pmap_qenter(wpipe->pipe_map.kva, wpipe->pipe_map.ms, 741 wpipe->pipe_map.npages); 742 743/* 744 * and update the uio data 745 */ 746 747 uio->uio_iov->iov_len -= size; 748 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 749 if (uio->uio_iov->iov_len == 0) 750 uio->uio_iov++; 751 uio->uio_resid -= size; 752 uio->uio_offset += size; 753 return (0); 754} 755 756/* 757 * unmap and unwire the process buffer 758 */ 759static void 760pipe_destroy_write_buffer(wpipe) 761 struct pipe *wpipe; 762{ 763 int i; 764 765 GIANT_REQUIRED; 766 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 767 768 if (wpipe->pipe_map.kva) { 769 pmap_qremove(wpipe->pipe_map.kva, wpipe->pipe_map.npages); 770 771 if (amountpipekva > maxpipekva) { 772 vm_offset_t kva = wpipe->pipe_map.kva; 773 wpipe->pipe_map.kva = 0; 774 kmem_free(kernel_map, kva, 775 wpipe->pipe_buffer.size + PAGE_SIZE); 776 atomic_subtract_int(&amountpipekva, 777 wpipe->pipe_buffer.size + PAGE_SIZE); 778 } 779 } 780 vm_page_lock_queues(); 781 for (i = 0; i < wpipe->pipe_map.npages; i++) { 782 vm_page_unwire(wpipe->pipe_map.ms[i], 1); 783 atomic_subtract_int(&amountpipekvawired, PAGE_SIZE); 784 } 785 vm_page_unlock_queues(); 786 wpipe->pipe_map.npages = 0; 787} 788 789/* 790 * In the case of a signal, the writing process might go away. This 791 * code copies the data into the circular buffer so that the source 792 * pages can be freed without loss of data. 793 */ 794static void 795pipe_clone_write_buffer(wpipe) 796 struct pipe *wpipe; 797{ 798 int size; 799 int pos; 800 801 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 802 size = wpipe->pipe_map.cnt; 803 pos = wpipe->pipe_map.pos; 804 805 wpipe->pipe_buffer.in = size; 806 wpipe->pipe_buffer.out = 0; 807 wpipe->pipe_buffer.cnt = size; 808 wpipe->pipe_state &= ~PIPE_DIRECTW; 809 810 PIPE_GET_GIANT(wpipe); 811 bcopy((caddr_t) wpipe->pipe_map.kva + pos, 812 wpipe->pipe_buffer.buffer, size); 813 pipe_destroy_write_buffer(wpipe); 814 PIPE_DROP_GIANT(wpipe); 815} 816 817/* 818 * This implements the pipe buffer write mechanism. Note that only 819 * a direct write OR a normal pipe write can be pending at any given time. 820 * If there are any characters in the pipe buffer, the direct write will 821 * be deferred until the receiving process grabs all of the bytes from 822 * the pipe buffer. Then the direct mapping write is set-up. 823 */ 824static int 825pipe_direct_write(wpipe, uio) 826 struct pipe *wpipe; 827 struct uio *uio; 828{ 829 int error; 830 831retry: 832 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 833 while (wpipe->pipe_state & PIPE_DIRECTW) { 834 if (wpipe->pipe_state & PIPE_WANTR) { 835 wpipe->pipe_state &= ~PIPE_WANTR; 836 wakeup(wpipe); 837 } 838 wpipe->pipe_state |= PIPE_WANTW; 839 error = msleep(wpipe, PIPE_MTX(wpipe), 840 PRIBIO | PCATCH, "pipdww", 0); 841 if (error) 842 goto error1; 843 if (wpipe->pipe_state & PIPE_EOF) { 844 error = EPIPE; 845 goto error1; 846 } 847 } 848 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 849 if (wpipe->pipe_buffer.cnt > 0) { 850 if (wpipe->pipe_state & PIPE_WANTR) { 851 wpipe->pipe_state &= ~PIPE_WANTR; 852 wakeup(wpipe); 853 } 854 855 wpipe->pipe_state |= PIPE_WANTW; 856 error = msleep(wpipe, PIPE_MTX(wpipe), 857 PRIBIO | PCATCH, "pipdwc", 0); 858 if (error) 859 goto error1; 860 if (wpipe->pipe_state & PIPE_EOF) { 861 error = EPIPE; 862 goto error1; 863 } 864 goto retry; 865 } 866 867 wpipe->pipe_state |= PIPE_DIRECTW; 868 869 pipelock(wpipe, 0); 870 PIPE_GET_GIANT(wpipe); 871 error = pipe_build_write_buffer(wpipe, uio); 872 PIPE_DROP_GIANT(wpipe); 873 pipeunlock(wpipe); 874 if (error) { 875 wpipe->pipe_state &= ~PIPE_DIRECTW; 876 goto error1; 877 } 878 879 error = 0; 880 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 881 if (wpipe->pipe_state & PIPE_EOF) { 882 pipelock(wpipe, 0); 883 PIPE_GET_GIANT(wpipe); 884 pipe_destroy_write_buffer(wpipe); 885 PIPE_DROP_GIANT(wpipe); 886 pipeselwakeup(wpipe); 887 pipeunlock(wpipe); 888 error = EPIPE; 889 goto error1; 890 } 891 if (wpipe->pipe_state & PIPE_WANTR) { 892 wpipe->pipe_state &= ~PIPE_WANTR; 893 wakeup(wpipe); 894 } 895 pipeselwakeup(wpipe); 896 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 897 "pipdwt", 0); 898 } 899 900 pipelock(wpipe,0); 901 if (wpipe->pipe_state & PIPE_DIRECTW) { 902 /* 903 * this bit of trickery substitutes a kernel buffer for 904 * the process that might be going away. 905 */ 906 pipe_clone_write_buffer(wpipe); 907 } else { 908 PIPE_GET_GIANT(wpipe); 909 pipe_destroy_write_buffer(wpipe); 910 PIPE_DROP_GIANT(wpipe); 911 } 912 pipeunlock(wpipe); 913 return (error); 914 915error1: 916 wakeup(wpipe); 917 return (error); 918} 919#endif 920 921static int 922pipe_write(fp, uio, active_cred, flags, td) 923 struct file *fp; 924 struct uio *uio; 925 struct ucred *active_cred; 926 struct thread *td; 927 int flags; 928{ 929 int error = 0; 930 int orig_resid; 931 struct pipe *wpipe, *rpipe; 932 933 rpipe = fp->f_data; 934 wpipe = rpipe->pipe_peer; 935 936 PIPE_LOCK(rpipe); 937 /* 938 * detect loss of pipe read side, issue SIGPIPE if lost. 939 */ 940 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 941 PIPE_UNLOCK(rpipe); 942 return (EPIPE); 943 } 944#ifdef MAC 945 error = mac_check_pipe_write(active_cred, wpipe); 946 if (error) { 947 PIPE_UNLOCK(rpipe); 948 return (error); 949 } 950#endif 951 ++wpipe->pipe_busy; 952 953 /* 954 * If it is advantageous to resize the pipe buffer, do 955 * so. 956 */ 957 if ((uio->uio_resid > PIPE_SIZE) && 958 (amountpipekva < maxpipekva) && 959 (nbigpipe < LIMITBIGPIPES) && 960 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 961 (wpipe->pipe_buffer.size <= PIPE_SIZE) && 962 (wpipe->pipe_buffer.cnt == 0)) { 963 964 if ((error = pipelock(wpipe, 1)) == 0) { 965 PIPE_GET_GIANT(wpipe); 966 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 967 atomic_add_int(&nbigpipe, 1); 968 PIPE_DROP_GIANT(wpipe); 969 pipeunlock(wpipe); 970 } 971 } 972 973 /* 974 * If an early error occured unbusy and return, waking up any pending 975 * readers. 976 */ 977 if (error) { 978 --wpipe->pipe_busy; 979 if ((wpipe->pipe_busy == 0) && 980 (wpipe->pipe_state & PIPE_WANT)) { 981 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 982 wakeup(wpipe); 983 } 984 PIPE_UNLOCK(rpipe); 985 return(error); 986 } 987 988 orig_resid = uio->uio_resid; 989 990 while (uio->uio_resid) { 991 int space; 992 993#ifndef PIPE_NODIRECT 994 /* 995 * If the transfer is large, we can gain performance if 996 * we do process-to-process copies directly. 997 * If the write is non-blocking, we don't use the 998 * direct write mechanism. 999 * 1000 * The direct write mechanism will detect the reader going 1001 * away on us. 1002 */ 1003 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 1004 (fp->f_flag & FNONBLOCK) == 0 && 1005 amountpipekvawired < maxpipekvawired) { 1006 error = pipe_direct_write(wpipe, uio); 1007 if (error) 1008 break; 1009 continue; 1010 } 1011#endif 1012 1013 /* 1014 * Pipe buffered writes cannot be coincidental with 1015 * direct writes. We wait until the currently executing 1016 * direct write is completed before we start filling the 1017 * pipe buffer. We break out if a signal occurs or the 1018 * reader goes away. 1019 */ 1020 retrywrite: 1021 while (wpipe->pipe_state & PIPE_DIRECTW) { 1022 if (wpipe->pipe_state & PIPE_WANTR) { 1023 wpipe->pipe_state &= ~PIPE_WANTR; 1024 wakeup(wpipe); 1025 } 1026 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1027 "pipbww", 0); 1028 if (wpipe->pipe_state & PIPE_EOF) 1029 break; 1030 if (error) 1031 break; 1032 } 1033 if (wpipe->pipe_state & PIPE_EOF) { 1034 error = EPIPE; 1035 break; 1036 } 1037 1038 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1039 1040 /* Writes of size <= PIPE_BUF must be atomic. */ 1041 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1042 space = 0; 1043 1044 if (space > 0) { 1045 if ((error = pipelock(wpipe,1)) == 0) { 1046 int size; /* Transfer size */ 1047 int segsize; /* first segment to transfer */ 1048 1049 /* 1050 * It is possible for a direct write to 1051 * slip in on us... handle it here... 1052 */ 1053 if (wpipe->pipe_state & PIPE_DIRECTW) { 1054 pipeunlock(wpipe); 1055 goto retrywrite; 1056 } 1057 /* 1058 * If a process blocked in uiomove, our 1059 * value for space might be bad. 1060 * 1061 * XXX will we be ok if the reader has gone 1062 * away here? 1063 */ 1064 if (space > wpipe->pipe_buffer.size - 1065 wpipe->pipe_buffer.cnt) { 1066 pipeunlock(wpipe); 1067 goto retrywrite; 1068 } 1069 1070 /* 1071 * Transfer size is minimum of uio transfer 1072 * and free space in pipe buffer. 1073 */ 1074 if (space > uio->uio_resid) 1075 size = uio->uio_resid; 1076 else 1077 size = space; 1078 /* 1079 * First segment to transfer is minimum of 1080 * transfer size and contiguous space in 1081 * pipe buffer. If first segment to transfer 1082 * is less than the transfer size, we've got 1083 * a wraparound in the buffer. 1084 */ 1085 segsize = wpipe->pipe_buffer.size - 1086 wpipe->pipe_buffer.in; 1087 if (segsize > size) 1088 segsize = size; 1089 1090 /* Transfer first segment */ 1091 1092 PIPE_UNLOCK(rpipe); 1093 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1094 segsize, uio); 1095 PIPE_LOCK(rpipe); 1096 1097 if (error == 0 && segsize < size) { 1098 /* 1099 * Transfer remaining part now, to 1100 * support atomic writes. Wraparound 1101 * happened. 1102 */ 1103 if (wpipe->pipe_buffer.in + segsize != 1104 wpipe->pipe_buffer.size) 1105 panic("Expected pipe buffer " 1106 "wraparound disappeared"); 1107 1108 PIPE_UNLOCK(rpipe); 1109 error = uiomove( 1110 &wpipe->pipe_buffer.buffer[0], 1111 size - segsize, uio); 1112 PIPE_LOCK(rpipe); 1113 } 1114 if (error == 0) { 1115 wpipe->pipe_buffer.in += size; 1116 if (wpipe->pipe_buffer.in >= 1117 wpipe->pipe_buffer.size) { 1118 if (wpipe->pipe_buffer.in != 1119 size - segsize + 1120 wpipe->pipe_buffer.size) 1121 panic("Expected " 1122 "wraparound bad"); 1123 wpipe->pipe_buffer.in = size - 1124 segsize; 1125 } 1126 1127 wpipe->pipe_buffer.cnt += size; 1128 if (wpipe->pipe_buffer.cnt > 1129 wpipe->pipe_buffer.size) 1130 panic("Pipe buffer overflow"); 1131 1132 } 1133 pipeunlock(wpipe); 1134 } 1135 if (error) 1136 break; 1137 1138 } else { 1139 /* 1140 * If the "read-side" has been blocked, wake it up now. 1141 */ 1142 if (wpipe->pipe_state & PIPE_WANTR) { 1143 wpipe->pipe_state &= ~PIPE_WANTR; 1144 wakeup(wpipe); 1145 } 1146 1147 /* 1148 * don't block on non-blocking I/O 1149 */ 1150 if (fp->f_flag & FNONBLOCK) { 1151 error = EAGAIN; 1152 break; 1153 } 1154 1155 /* 1156 * We have no more space and have something to offer, 1157 * wake up select/poll. 1158 */ 1159 pipeselwakeup(wpipe); 1160 1161 wpipe->pipe_state |= PIPE_WANTW; 1162 error = msleep(wpipe, PIPE_MTX(rpipe), 1163 PRIBIO | PCATCH, "pipewr", 0); 1164 if (error != 0) 1165 break; 1166 /* 1167 * If read side wants to go away, we just issue a signal 1168 * to ourselves. 1169 */ 1170 if (wpipe->pipe_state & PIPE_EOF) { 1171 error = EPIPE; 1172 break; 1173 } 1174 } 1175 } 1176 1177 --wpipe->pipe_busy; 1178 1179 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1180 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1181 wakeup(wpipe); 1182 } else if (wpipe->pipe_buffer.cnt > 0) { 1183 /* 1184 * If we have put any characters in the buffer, we wake up 1185 * the reader. 1186 */ 1187 if (wpipe->pipe_state & PIPE_WANTR) { 1188 wpipe->pipe_state &= ~PIPE_WANTR; 1189 wakeup(wpipe); 1190 } 1191 } 1192 1193 /* 1194 * Don't return EPIPE if I/O was successful 1195 */ 1196 if ((wpipe->pipe_buffer.cnt == 0) && 1197 (uio->uio_resid == 0) && 1198 (error == EPIPE)) { 1199 error = 0; 1200 } 1201 1202 if (error == 0) 1203 vfs_timestamp(&wpipe->pipe_mtime); 1204 1205 /* 1206 * We have something to offer, 1207 * wake up select/poll. 1208 */ 1209 if (wpipe->pipe_buffer.cnt) 1210 pipeselwakeup(wpipe); 1211 1212 PIPE_UNLOCK(rpipe); 1213 return (error); 1214} 1215 1216/* 1217 * we implement a very minimal set of ioctls for compatibility with sockets. 1218 */ 1219static int 1220pipe_ioctl(fp, cmd, data, active_cred, td) 1221 struct file *fp; 1222 u_long cmd; 1223 void *data; 1224 struct ucred *active_cred; 1225 struct thread *td; 1226{ 1227 struct pipe *mpipe = fp->f_data; 1228#ifdef MAC 1229 int error; 1230#endif 1231 1232 PIPE_LOCK(mpipe); 1233 1234#ifdef MAC 1235 error = mac_check_pipe_ioctl(active_cred, mpipe, cmd, data); 1236 if (error) 1237 return (error); 1238#endif 1239 1240 switch (cmd) { 1241 1242 case FIONBIO: 1243 PIPE_UNLOCK(mpipe); 1244 return (0); 1245 1246 case FIOASYNC: 1247 if (*(int *)data) { 1248 mpipe->pipe_state |= PIPE_ASYNC; 1249 } else { 1250 mpipe->pipe_state &= ~PIPE_ASYNC; 1251 } 1252 PIPE_UNLOCK(mpipe); 1253 return (0); 1254 1255 case FIONREAD: 1256 if (mpipe->pipe_state & PIPE_DIRECTW) 1257 *(int *)data = mpipe->pipe_map.cnt; 1258 else 1259 *(int *)data = mpipe->pipe_buffer.cnt; 1260 PIPE_UNLOCK(mpipe); 1261 return (0); 1262 1263 case FIOSETOWN: 1264 PIPE_UNLOCK(mpipe); 1265 return (fsetown(*(int *)data, &mpipe->pipe_sigio)); 1266 1267 case FIOGETOWN: 1268 PIPE_UNLOCK(mpipe); 1269 *(int *)data = fgetown(&mpipe->pipe_sigio); 1270 return (0); 1271 1272 /* This is deprecated, FIOSETOWN should be used instead. */ 1273 case TIOCSPGRP: 1274 PIPE_UNLOCK(mpipe); 1275 return (fsetown(-(*(int *)data), &mpipe->pipe_sigio)); 1276 1277 /* This is deprecated, FIOGETOWN should be used instead. */ 1278 case TIOCGPGRP: 1279 PIPE_UNLOCK(mpipe); 1280 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1281 return (0); 1282 1283 } 1284 PIPE_UNLOCK(mpipe); 1285 return (ENOTTY); 1286} 1287 1288static int 1289pipe_poll(fp, events, active_cred, td) 1290 struct file *fp; 1291 int events; 1292 struct ucred *active_cred; 1293 struct thread *td; 1294{ 1295 struct pipe *rpipe = fp->f_data; 1296 struct pipe *wpipe; 1297 int revents = 0; 1298#ifdef MAC 1299 int error; 1300#endif 1301 1302 wpipe = rpipe->pipe_peer; 1303 PIPE_LOCK(rpipe); 1304#ifdef MAC 1305 error = mac_check_pipe_poll(active_cred, rpipe); 1306 if (error) 1307 goto locked_error; 1308#endif 1309 if (events & (POLLIN | POLLRDNORM)) 1310 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1311 (rpipe->pipe_buffer.cnt > 0) || 1312 (rpipe->pipe_state & PIPE_EOF)) 1313 revents |= events & (POLLIN | POLLRDNORM); 1314 1315 if (events & (POLLOUT | POLLWRNORM)) 1316 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) || 1317 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1318 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1319 revents |= events & (POLLOUT | POLLWRNORM); 1320 1321 if ((rpipe->pipe_state & PIPE_EOF) || 1322 (wpipe == NULL) || 1323 (wpipe->pipe_state & PIPE_EOF)) 1324 revents |= POLLHUP; 1325 1326 if (revents == 0) { 1327 if (events & (POLLIN | POLLRDNORM)) { 1328 selrecord(td, &rpipe->pipe_sel); 1329 rpipe->pipe_state |= PIPE_SEL; 1330 } 1331 1332 if (events & (POLLOUT | POLLWRNORM)) { 1333 selrecord(td, &wpipe->pipe_sel); 1334 wpipe->pipe_state |= PIPE_SEL; 1335 } 1336 } 1337#ifdef MAC 1338locked_error: 1339#endif 1340 PIPE_UNLOCK(rpipe); 1341 1342 return (revents); 1343} 1344 1345/* 1346 * We shouldn't need locks here as we're doing a read and this should 1347 * be a natural race. 1348 */ 1349static int 1350pipe_stat(fp, ub, active_cred, td) 1351 struct file *fp; 1352 struct stat *ub; 1353 struct ucred *active_cred; 1354 struct thread *td; 1355{ 1356 struct pipe *pipe = fp->f_data; 1357#ifdef MAC 1358 int error; 1359 1360 PIPE_LOCK(pipe); 1361 error = mac_check_pipe_stat(active_cred, pipe); 1362 PIPE_UNLOCK(pipe); 1363 if (error) 1364 return (error); 1365#endif 1366 bzero(ub, sizeof(*ub)); 1367 ub->st_mode = S_IFIFO; 1368 ub->st_blksize = pipe->pipe_buffer.size; 1369 ub->st_size = pipe->pipe_buffer.cnt; 1370 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1371 ub->st_atimespec = pipe->pipe_atime; 1372 ub->st_mtimespec = pipe->pipe_mtime; 1373 ub->st_ctimespec = pipe->pipe_ctime; 1374 ub->st_uid = fp->f_cred->cr_uid; 1375 ub->st_gid = fp->f_cred->cr_gid; 1376 /* 1377 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1378 * XXX (st_dev, st_ino) should be unique. 1379 */ 1380 return (0); 1381} 1382 1383/* ARGSUSED */ 1384static int 1385pipe_close(fp, td) 1386 struct file *fp; 1387 struct thread *td; 1388{ 1389 struct pipe *cpipe = fp->f_data; 1390 1391 fp->f_ops = &badfileops; 1392 fp->f_data = NULL; 1393 funsetown(&cpipe->pipe_sigio); 1394 pipeclose(cpipe); 1395 return (0); 1396} 1397 1398static void 1399pipe_free_kmem(cpipe) 1400 struct pipe *cpipe; 1401{ 1402 1403 GIANT_REQUIRED; 1404 KASSERT(cpipe->pipe_mtxp == NULL || !mtx_owned(PIPE_MTX(cpipe)), 1405 ("pipespace: pipe mutex locked")); 1406 1407 if (cpipe->pipe_buffer.buffer != NULL) { 1408 if (cpipe->pipe_buffer.size > PIPE_SIZE) 1409 atomic_subtract_int(&nbigpipe, 1); 1410 atomic_subtract_int(&amountpipekva, cpipe->pipe_buffer.size); 1411 atomic_subtract_int(&amountpipes, 1); 1412 kmem_free(kernel_map, 1413 (vm_offset_t)cpipe->pipe_buffer.buffer, 1414 cpipe->pipe_buffer.size); 1415 cpipe->pipe_buffer.buffer = NULL; 1416 } 1417#ifndef PIPE_NODIRECT 1418 if (cpipe->pipe_map.kva != 0) { 1419 atomic_subtract_int(&amountpipekva, 1420 cpipe->pipe_buffer.size + PAGE_SIZE); 1421 kmem_free(kernel_map, 1422 cpipe->pipe_map.kva, 1423 cpipe->pipe_buffer.size + PAGE_SIZE); 1424 cpipe->pipe_map.cnt = 0; 1425 cpipe->pipe_map.kva = 0; 1426 cpipe->pipe_map.pos = 0; 1427 cpipe->pipe_map.npages = 0; 1428 } 1429#endif 1430} 1431 1432/* 1433 * shutdown the pipe 1434 */ 1435static void 1436pipeclose(cpipe) 1437 struct pipe *cpipe; 1438{ 1439 struct pipe *ppipe; 1440 int hadpeer; 1441 1442 if (cpipe == NULL) 1443 return; 1444 1445 hadpeer = 0; 1446 1447 /* partially created pipes won't have a valid mutex. */ 1448 if (PIPE_MTX(cpipe) != NULL) 1449 PIPE_LOCK(cpipe); 1450 1451 pipeselwakeup(cpipe); 1452 1453 /* 1454 * If the other side is blocked, wake it up saying that 1455 * we want to close it down. 1456 */ 1457 while (cpipe->pipe_busy) { 1458 wakeup(cpipe); 1459 cpipe->pipe_state |= PIPE_WANT | PIPE_EOF; 1460 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1461 } 1462 1463#ifdef MAC 1464 if (cpipe->pipe_label != NULL && cpipe->pipe_peer == NULL) 1465 mac_destroy_pipe(cpipe); 1466#endif 1467 1468 /* 1469 * Disconnect from peer 1470 */ 1471 if ((ppipe = cpipe->pipe_peer) != NULL) { 1472 hadpeer++; 1473 pipeselwakeup(ppipe); 1474 1475 ppipe->pipe_state |= PIPE_EOF; 1476 wakeup(ppipe); 1477 KNOTE(&ppipe->pipe_sel.si_note, 0); 1478 ppipe->pipe_peer = NULL; 1479 } 1480 /* 1481 * free resources 1482 */ 1483 if (PIPE_MTX(cpipe) != NULL) { 1484 PIPE_UNLOCK(cpipe); 1485 if (!hadpeer) { 1486 mtx_destroy(PIPE_MTX(cpipe)); 1487 free(PIPE_MTX(cpipe), M_TEMP); 1488 } 1489 } 1490 mtx_lock(&Giant); 1491 pipe_free_kmem(cpipe); 1492 uma_zfree(pipe_zone, cpipe); 1493 mtx_unlock(&Giant); 1494} 1495 1496/*ARGSUSED*/ 1497static int 1498pipe_kqfilter(struct file *fp, struct knote *kn) 1499{ 1500 struct pipe *cpipe; 1501 1502 cpipe = kn->kn_fp->f_data; 1503 switch (kn->kn_filter) { 1504 case EVFILT_READ: 1505 kn->kn_fop = &pipe_rfiltops; 1506 break; 1507 case EVFILT_WRITE: 1508 kn->kn_fop = &pipe_wfiltops; 1509 cpipe = cpipe->pipe_peer; 1510 if (cpipe == NULL) 1511 /* other end of pipe has been closed */ 1512 return (EBADF); 1513 break; 1514 default: 1515 return (1); 1516 } 1517 kn->kn_hook = cpipe; 1518 1519 PIPE_LOCK(cpipe); 1520 SLIST_INSERT_HEAD(&cpipe->pipe_sel.si_note, kn, kn_selnext); 1521 PIPE_UNLOCK(cpipe); 1522 return (0); 1523} 1524 1525static void 1526filt_pipedetach(struct knote *kn) 1527{ 1528 struct pipe *cpipe = (struct pipe *)kn->kn_hook; 1529 1530 PIPE_LOCK(cpipe); 1531 SLIST_REMOVE(&cpipe->pipe_sel.si_note, kn, knote, kn_selnext); 1532 PIPE_UNLOCK(cpipe); 1533} 1534 1535/*ARGSUSED*/ 1536static int 1537filt_piperead(struct knote *kn, long hint) 1538{ 1539 struct pipe *rpipe = kn->kn_fp->f_data; 1540 struct pipe *wpipe = rpipe->pipe_peer; 1541 1542 PIPE_LOCK(rpipe); 1543 kn->kn_data = rpipe->pipe_buffer.cnt; 1544 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1545 kn->kn_data = rpipe->pipe_map.cnt; 1546 1547 if ((rpipe->pipe_state & PIPE_EOF) || 1548 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1549 kn->kn_flags |= EV_EOF; 1550 PIPE_UNLOCK(rpipe); 1551 return (1); 1552 } 1553 PIPE_UNLOCK(rpipe); 1554 return (kn->kn_data > 0); 1555} 1556 1557/*ARGSUSED*/ 1558static int 1559filt_pipewrite(struct knote *kn, long hint) 1560{ 1561 struct pipe *rpipe = kn->kn_fp->f_data; 1562 struct pipe *wpipe = rpipe->pipe_peer; 1563 1564 PIPE_LOCK(rpipe); 1565 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1566 kn->kn_data = 0; 1567 kn->kn_flags |= EV_EOF; 1568 PIPE_UNLOCK(rpipe); 1569 return (1); 1570 } 1571 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1572 if (wpipe->pipe_state & PIPE_DIRECTW) 1573 kn->kn_data = 0; 1574 1575 PIPE_UNLOCK(rpipe); 1576 return (kn->kn_data >= PIPE_BUF); 1577} 1578