sys_pipe.c revision 118572
1171095Ssam/* 2171095Ssam * Copyright (c) 1996 John S. Dyson 3171095Ssam * All rights reserved. 4171095Ssam * 5171095Ssam * Redistribution and use in source and binary forms, with or without 6171095Ssam * modification, are permitted provided that the following conditions 7171095Ssam * are met: 8171095Ssam * 1. Redistributions of source code must retain the above copyright 9171095Ssam * notice immediately at the beginning of the file, without modification, 10171095Ssam * this list of conditions, and the following disclaimer. 11171095Ssam * 2. Redistributions in binary form must reproduce the above copyright 12171095Ssam * notice, this list of conditions and the following disclaimer in the 13171095Ssam * documentation and/or other materials provided with the distribution. 14171095Ssam * 3. Absolutely no warranty of function or purpose is made by the author 15171095Ssam * John S. Dyson. 16171095Ssam * 4. Modifications may be freely made to this file if the above conditions 17171095Ssam * are met. 18171095Ssam */ 19171095Ssam 20171095Ssam/* 21171095Ssam * This file contains a high-performance replacement for the socket-based 22171095Ssam * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 23171095Ssam * all features of sockets, but does do everything that pipes normally 24171095Ssam * do. 25171095Ssam */ 26171095Ssam 27171095Ssam/* 28171095Ssam * This code has two modes of operation, a small write mode and a large 29171095Ssam * write mode. The small write mode acts like conventional pipes with 30171095Ssam * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 31171095Ssam * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 32171095Ssam * and PIPE_SIZE in size, it is fully mapped and wired into the kernel, and 33171095Ssam * the receiving process can copy it directly from the pages in the sending 34171095Ssam * process. 35171095Ssam * 36171095Ssam * If the sending process receives a signal, it is possible that it will 37171095Ssam * go away, and certainly its address space can change, because control 38171095Ssam * is returned back to the user-mode side. In that case, the pipe code 39171095Ssam * arranges to copy the buffer supplied by the user process, to a pageable 40171095Ssam * kernel buffer, and the receiving process will grab the data from the 41171095Ssam * pageable kernel buffer. Since signals don't happen all that often, 42173139Srwatson * the copy operation is normally eliminated. 43171095Ssam * 44171095Ssam * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 45171095Ssam * happen for small transfers so that the system will not spend all of 46171095Ssam * its time context switching. PIPE_SIZE is constrained by the 47171095Ssam * amount of kernel virtual memory. 48171095Ssam * 49171095Ssam * In order to limit the resource use of pipes, three sysctls exist: 50171095Ssam * 51171095Ssam * kern.ipc.maxpipes - A limit on the total number of pipes in the system. 52171095Ssam * Note that since pipes are bidirectional, the effective value is this 53171095Ssam * number divided by two. 54171095Ssam * 55171095Ssam * kern.ipc.maxpipekva - This value limits the amount of pageable memory that 56171095Ssam * can be used by pipes. Whenever the amount in use exceeds this value, 57171095Ssam * all new pipes will be SMALL_PIPE_SIZE in size, rather than PIPE_SIZE. 58171095Ssam * Big pipe creation will be limited as well. 59171095Ssam * 60171095Ssam * kern.ipc.maxpipekvawired - This value limits the amount of memory that may 61171095Ssam * be wired in order to facilitate direct copies using page flipping. 62171095Ssam * Whenever this value is exceeded, pipes will fall back to using regular 63171095Ssam * copies. 64171095Ssam * 65171095Ssam * These values are autotuned in subr_param.c. 66171095Ssam * 67171095Ssam * Memory usage may be monitored through the sysctls 68171095Ssam * kern.ipc.pipes, kern.ipc.pipekva and kern.ipc.pipekvawired. 69171095Ssam * 70171095Ssam */ 71171095Ssam 72171095Ssam#include <sys/cdefs.h> 73171095Ssam__FBSDID("$FreeBSD: head/sys/kern/sys_pipe.c 118572 2003-08-07 04:32:40Z alc $"); 74171095Ssam 75171095Ssam#include "opt_mac.h" 76171095Ssam 77171095Ssam#include <sys/param.h> 78171095Ssam#include <sys/systm.h> 79171095Ssam#include <sys/fcntl.h> 80171095Ssam#include <sys/file.h> 81171095Ssam#include <sys/filedesc.h> 82171095Ssam#include <sys/filio.h> 83171095Ssam#include <sys/kernel.h> 84171095Ssam#include <sys/lock.h> 85171095Ssam#include <sys/mac.h> 86171095Ssam#include <sys/mutex.h> 87171095Ssam#include <sys/ttycom.h> 88171095Ssam#include <sys/stat.h> 89171095Ssam#include <sys/malloc.h> 90171095Ssam#include <sys/poll.h> 91171095Ssam#include <sys/selinfo.h> 92171095Ssam#include <sys/signalvar.h> 93171095Ssam#include <sys/sysctl.h> 94171095Ssam#include <sys/sysproto.h> 95171095Ssam#include <sys/pipe.h> 96171095Ssam#include <sys/proc.h> 97171095Ssam#include <sys/vnode.h> 98171095Ssam#include <sys/uio.h> 99171095Ssam#include <sys/event.h> 100171095Ssam 101171095Ssam#include <vm/vm.h> 102171095Ssam#include <vm/vm_param.h> 103171095Ssam#include <vm/vm_object.h> 104171095Ssam#include <vm/vm_kern.h> 105171095Ssam#include <vm/vm_extern.h> 106171095Ssam#include <vm/pmap.h> 107173139Srwatson#include <vm/vm_map.h> 108171095Ssam#include <vm/vm_page.h> 109171095Ssam#include <vm/uma.h> 110171095Ssam 111171095Ssam/* 112171095Ssam * Use this define if you want to disable *fancy* VM things. Expect an 113171095Ssam * approx 30% decrease in transfer rate. This could be useful for 114171095Ssam * NetBSD or OpenBSD. 115171095Ssam */ 116171095Ssam/* #define PIPE_NODIRECT */ 117171095Ssam 118171095Ssam/* 119171095Ssam * interfaces to the outside world 120171095Ssam */ 121171095Ssamstatic fo_rdwr_t pipe_read; 122171095Ssamstatic fo_rdwr_t pipe_write; 123171095Ssamstatic fo_ioctl_t pipe_ioctl; 124171095Ssamstatic fo_poll_t pipe_poll; 125171095Ssamstatic fo_kqfilter_t pipe_kqfilter; 126171095Ssamstatic fo_stat_t pipe_stat; 127171095Ssamstatic fo_close_t pipe_close; 128171095Ssam 129171095Ssamstatic struct fileops pipeops = { 130171095Ssam .fo_read = pipe_read, 131171095Ssam .fo_write = pipe_write, 132171095Ssam .fo_ioctl = pipe_ioctl, 133171095Ssam .fo_poll = pipe_poll, 134171095Ssam .fo_kqfilter = pipe_kqfilter, 135171095Ssam .fo_stat = pipe_stat, 136171095Ssam .fo_close = pipe_close, 137171095Ssam .fo_flags = DFLAG_PASSABLE 138171095Ssam}; 139171095Ssam 140171095Ssamstatic void filt_pipedetach(struct knote *kn); 141171095Ssamstatic int filt_piperead(struct knote *kn, long hint); 142171095Ssamstatic int filt_pipewrite(struct knote *kn, long hint); 143171095Ssam 144171095Ssamstatic struct filterops pipe_rfiltops = 145171095Ssam { 1, NULL, filt_pipedetach, filt_piperead }; 146171095Ssamstatic struct filterops pipe_wfiltops = 147171095Ssam { 1, NULL, filt_pipedetach, filt_pipewrite }; 148171095Ssam 149171095Ssam#define PIPE_GET_GIANT(pipe) \ 150171095Ssam do { \ 151171095Ssam KASSERT(((pipe)->pipe_state & PIPE_LOCKFL) != 0, \ 152171095Ssam ("%s:%d PIPE_GET_GIANT: line pipe not locked", \ 153171095Ssam __FILE__, __LINE__)); \ 154171095Ssam PIPE_UNLOCK(pipe); \ 155171095Ssam mtx_lock(&Giant); \ 156171095Ssam } while (0) 157171095Ssam 158171095Ssam#define PIPE_DROP_GIANT(pipe) \ 159171095Ssam do { \ 160171095Ssam mtx_unlock(&Giant); \ 161171095Ssam PIPE_LOCK(pipe); \ 162171095Ssam } while (0) 163171095Ssam 164171095Ssam/* 165171095Ssam * Default pipe buffer size(s), this can be kind-of large now because pipe 166171095Ssam * space is pageable. The pipe code will try to maintain locality of 167171095Ssam * reference for performance reasons, so small amounts of outstanding I/O 168171095Ssam * will not wipe the cache. 169171095Ssam */ 170171095Ssam#define MINPIPESIZE (PIPE_SIZE/3) 171171095Ssam#define MAXPIPESIZE (2*PIPE_SIZE/3) 172171095Ssam 173171095Ssam/* 174171095Ssam * Limit the number of "big" pipes 175171095Ssam */ 176171095Ssam#define LIMITBIGPIPES 32 177171095Ssamstatic int nbigpipe; 178171095Ssam 179171095Ssamstatic int amountpipes; 180171095Ssamstatic int amountpipekva; 181171095Ssamstatic int amountpipekvawired; 182171095Ssam 183171095SsamSYSCTL_DECL(_kern_ipc); 184171095Ssam 185171095SsamSYSCTL_INT(_kern_ipc, OID_AUTO, maxpipes, CTLFLAG_RW, 186171095Ssam &maxpipes, 0, "Max # of pipes"); 187171095SsamSYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RW, 188171095Ssam &maxpipekva, 0, "Pipe KVA limit"); 189171095SsamSYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekvawired, CTLFLAG_RW, 190171095Ssam &maxpipekvawired, 0, "Pipe KVA wired limit"); 191171095SsamSYSCTL_INT(_kern_ipc, OID_AUTO, pipes, CTLFLAG_RD, 192171095Ssam &amountpipes, 0, "Current # of pipes"); 193171095SsamSYSCTL_INT(_kern_ipc, OID_AUTO, bigpipes, CTLFLAG_RD, 194171095Ssam &nbigpipe, 0, "Current # of big pipes"); 195171095SsamSYSCTL_INT(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 196171095Ssam &amountpipekva, 0, "Pipe KVA usage"); 197171095SsamSYSCTL_INT(_kern_ipc, OID_AUTO, pipekvawired, CTLFLAG_RD, 198171095Ssam &amountpipekvawired, 0, "Pipe wired KVA usage"); 199171095Ssam 200171095Ssamstatic void pipeinit(void *dummy __unused); 201171095Ssamstatic void pipeclose(struct pipe *cpipe); 202171095Ssamstatic void pipe_free_kmem(struct pipe *cpipe); 203171095Ssamstatic int pipe_create(struct pipe **cpipep); 204171095Ssamstatic __inline int pipelock(struct pipe *cpipe, int catch); 205171095Ssamstatic __inline void pipeunlock(struct pipe *cpipe); 206171095Ssamstatic __inline void pipeselwakeup(struct pipe *cpipe); 207171095Ssam#ifndef PIPE_NODIRECT 208171095Ssamstatic int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 209171095Ssamstatic void pipe_destroy_write_buffer(struct pipe *wpipe); 210171095Ssamstatic int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 211173139Srwatsonstatic void pipe_clone_write_buffer(struct pipe *wpipe); 212173139Srwatson#endif 213173139Srwatsonstatic int pipespace(struct pipe *cpipe, int size); 214173139Srwatson 215173139Srwatsonstatic uma_zone_t pipe_zone; 216171095Ssam 217173139SrwatsonSYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 218173139Srwatson 219173139Srwatsonstatic void 220171095Ssampipeinit(void *dummy __unused) 221171095Ssam{ 222171095Ssam pipe_zone = uma_zcreate("PIPE", sizeof(struct pipe), NULL, 223171095Ssam NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 224171095Ssam} 225171095Ssam 226171095Ssam/* 227171095Ssam * The pipe system call for the DTYPE_PIPE type of pipes 228171095Ssam */ 229171095Ssam 230171095Ssam/* ARGSUSED */ 231171095Ssamint 232171095Ssampipe(td, uap) 233173139Srwatson struct thread *td; 234173139Srwatson struct pipe_args /* { 235173139Srwatson int dummy; 236173139Srwatson } */ *uap; 237173139Srwatson{ 238173139Srwatson struct filedesc *fdp = td->td_proc->p_fd; 239173139Srwatson struct file *rf, *wf; 240173139Srwatson struct pipe *rpipe, *wpipe; 241173139Srwatson struct mtx *pmtx; 242173139Srwatson int fd, error; 243173139Srwatson 244173139Srwatson KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 245173139Srwatson 246171095Ssam pmtx = malloc(sizeof(*pmtx), M_TEMP, M_WAITOK | M_ZERO); 247171095Ssam 248171095Ssam rpipe = wpipe = NULL; 249171095Ssam if (pipe_create(&rpipe) || pipe_create(&wpipe)) { 250171095Ssam pipeclose(rpipe); 251171095Ssam pipeclose(wpipe); 252171095Ssam free(pmtx, M_TEMP); 253171095Ssam return (ENFILE); 254173139Srwatson } 255171095Ssam 256171095Ssam rpipe->pipe_state |= PIPE_DIRECTOK; 257173139Srwatson wpipe->pipe_state |= PIPE_DIRECTOK; 258173139Srwatson 259171095Ssam error = falloc(td, &rf, &fd); 260171095Ssam if (error) { 261171095Ssam pipeclose(rpipe); 262171095Ssam pipeclose(wpipe); 263171095Ssam free(pmtx, M_TEMP); 264171095Ssam return (error); 265173139Srwatson } 266171095Ssam fhold(rf); 267171095Ssam td->td_retval[0] = fd; 268171095Ssam 269171095Ssam /* 270171095Ssam * Warning: once we've gotten past allocation of the fd for the 271171095Ssam * read-side, we can only drop the read side via fdrop() in order 272171095Ssam * to avoid races against processes which manage to dup() the read 273171095Ssam * side while we are blocked trying to allocate the write side. 274171095Ssam */ 275171095Ssam FILE_LOCK(rf); 276171095Ssam rf->f_flag = FREAD | FWRITE; 277171095Ssam rf->f_type = DTYPE_PIPE; 278173139Srwatson rf->f_data = rpipe; 279171095Ssam rf->f_ops = &pipeops; 280171095Ssam FILE_UNLOCK(rf); 281171095Ssam error = falloc(td, &wf, &fd); 282171095Ssam if (error) { 283171095Ssam FILEDESC_LOCK(fdp); 284171095Ssam if (fdp->fd_ofiles[td->td_retval[0]] == rf) { 285171095Ssam fdp->fd_ofiles[td->td_retval[0]] = NULL; 286171095Ssam FILEDESC_UNLOCK(fdp); 287171095Ssam fdrop(rf, td); 288171095Ssam } else 289171095Ssam FILEDESC_UNLOCK(fdp); 290171095Ssam fdrop(rf, td); 291173139Srwatson /* rpipe has been closed by fdrop(). */ 292171095Ssam pipeclose(wpipe); 293171095Ssam free(pmtx, M_TEMP); 294171095Ssam return (error); 295171095Ssam } 296171095Ssam FILE_LOCK(wf); 297171095Ssam wf->f_flag = FREAD | FWRITE; 298173139Srwatson wf->f_type = DTYPE_PIPE; 299171095Ssam wf->f_data = wpipe; 300171095Ssam wf->f_ops = &pipeops; 301171095Ssam FILE_UNLOCK(wf); 302171095Ssam td->td_retval[1] = fd; 303171095Ssam rpipe->pipe_peer = wpipe; 304171095Ssam wpipe->pipe_peer = rpipe; 305171095Ssam#ifdef MAC 306171095Ssam /* 307171095Ssam * struct pipe represents a pipe endpoint. The MAC label is shared 308171095Ssam * between the connected endpoints. As a result mac_init_pipe() and 309171095Ssam * mac_create_pipe() should only be called on one of the endpoints 310171095Ssam * after they have been connected. 311171095Ssam */ 312171095Ssam mac_init_pipe(rpipe); 313171095Ssam mac_create_pipe(td->td_ucred, rpipe); 314171095Ssam#endif 315 mtx_init(pmtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 316 rpipe->pipe_mtxp = wpipe->pipe_mtxp = pmtx; 317 fdrop(rf, td); 318 319 return (0); 320} 321 322/* 323 * Allocate kva for pipe circular buffer, the space is pageable 324 * This routine will 'realloc' the size of a pipe safely, if it fails 325 * it will retain the old buffer. 326 * If it fails it will return ENOMEM. 327 */ 328static int 329pipespace(cpipe, size) 330 struct pipe *cpipe; 331 int size; 332{ 333 struct vm_object *object; 334 caddr_t buffer; 335 int npages, error; 336 static int curfail = 0; 337 static struct timeval lastfail; 338 339 GIANT_REQUIRED; 340 KASSERT(cpipe->pipe_mtxp == NULL || !mtx_owned(PIPE_MTX(cpipe)), 341 ("pipespace: pipe mutex locked")); 342 343 if (amountpipes > maxpipes) { 344 if (ppsratecheck(&lastfail, &curfail, 1)) 345 printf("kern.maxpipes exceeded, please see tuning(7).\n"); 346 return (ENOMEM); 347 } 348 349 npages = round_page(size)/PAGE_SIZE; 350 /* 351 * Create an object, I don't like the idea of paging to/from 352 * kernel_object. 353 * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. 354 */ 355 object = vm_object_allocate(OBJT_DEFAULT, npages); 356 buffer = (caddr_t) vm_map_min(kernel_map); 357 358 /* 359 * Insert the object into the kernel map, and allocate kva for it. 360 * The map entry is, by default, pageable. 361 * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. 362 */ 363 error = vm_map_find(kernel_map, object, 0, 364 (vm_offset_t *) &buffer, size, 1, 365 VM_PROT_ALL, VM_PROT_ALL, 0); 366 367 if (error != KERN_SUCCESS) { 368 vm_object_deallocate(object); 369 return (ENOMEM); 370 } 371 372 /* free old resources if we're resizing */ 373 pipe_free_kmem(cpipe); 374 cpipe->pipe_buffer.object = object; 375 cpipe->pipe_buffer.buffer = buffer; 376 cpipe->pipe_buffer.size = size; 377 cpipe->pipe_buffer.in = 0; 378 cpipe->pipe_buffer.out = 0; 379 cpipe->pipe_buffer.cnt = 0; 380 atomic_add_int(&amountpipes, 1); 381 atomic_add_int(&amountpipekva, cpipe->pipe_buffer.size); 382 return (0); 383} 384 385/* 386 * initialize and allocate VM and memory for pipe 387 */ 388static int 389pipe_create(cpipep) 390 struct pipe **cpipep; 391{ 392 struct pipe *cpipe; 393 int error; 394 395 *cpipep = uma_zalloc(pipe_zone, M_WAITOK); 396 if (*cpipep == NULL) 397 return (ENOMEM); 398 399 cpipe = *cpipep; 400 401 /* so pipespace()->pipe_free_kmem() doesn't follow junk pointer */ 402 cpipe->pipe_buffer.object = NULL; 403#ifndef PIPE_NODIRECT 404 cpipe->pipe_map.kva = 0; 405#endif 406 /* 407 * protect so pipeclose() doesn't follow a junk pointer 408 * if pipespace() fails. 409 */ 410 bzero(&cpipe->pipe_sel, sizeof(cpipe->pipe_sel)); 411 cpipe->pipe_state = 0; 412 cpipe->pipe_peer = NULL; 413 cpipe->pipe_busy = 0; 414 415#ifndef PIPE_NODIRECT 416 /* 417 * pipe data structure initializations to support direct pipe I/O 418 */ 419 cpipe->pipe_map.cnt = 0; 420 cpipe->pipe_map.kva = 0; 421 cpipe->pipe_map.pos = 0; 422 cpipe->pipe_map.npages = 0; 423 /* cpipe->pipe_map.ms[] = invalid */ 424#endif 425 426 cpipe->pipe_mtxp = NULL; /* avoid pipespace assertion */ 427 /* 428 * Reduce to 1/4th pipe size if we're over our global max. 429 */ 430 if (amountpipekva > maxpipekva) 431 error = pipespace(cpipe, SMALL_PIPE_SIZE); 432 else 433 error = pipespace(cpipe, PIPE_SIZE); 434 if (error) 435 return (error); 436 437 vfs_timestamp(&cpipe->pipe_ctime); 438 cpipe->pipe_atime = cpipe->pipe_ctime; 439 cpipe->pipe_mtime = cpipe->pipe_ctime; 440 441 return (0); 442} 443 444 445/* 446 * lock a pipe for I/O, blocking other access 447 */ 448static __inline int 449pipelock(cpipe, catch) 450 struct pipe *cpipe; 451 int catch; 452{ 453 int error; 454 455 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 456 while (cpipe->pipe_state & PIPE_LOCKFL) { 457 cpipe->pipe_state |= PIPE_LWANT; 458 error = msleep(cpipe, PIPE_MTX(cpipe), 459 catch ? (PRIBIO | PCATCH) : PRIBIO, 460 "pipelk", 0); 461 if (error != 0) 462 return (error); 463 } 464 cpipe->pipe_state |= PIPE_LOCKFL; 465 return (0); 466} 467 468/* 469 * unlock a pipe I/O lock 470 */ 471static __inline void 472pipeunlock(cpipe) 473 struct pipe *cpipe; 474{ 475 476 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 477 cpipe->pipe_state &= ~PIPE_LOCKFL; 478 if (cpipe->pipe_state & PIPE_LWANT) { 479 cpipe->pipe_state &= ~PIPE_LWANT; 480 wakeup(cpipe); 481 } 482} 483 484static __inline void 485pipeselwakeup(cpipe) 486 struct pipe *cpipe; 487{ 488 489 if (cpipe->pipe_state & PIPE_SEL) { 490 cpipe->pipe_state &= ~PIPE_SEL; 491 selwakeup(&cpipe->pipe_sel); 492 } 493 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 494 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 495 KNOTE(&cpipe->pipe_sel.si_note, 0); 496} 497 498/* ARGSUSED */ 499static int 500pipe_read(fp, uio, active_cred, flags, td) 501 struct file *fp; 502 struct uio *uio; 503 struct ucred *active_cred; 504 struct thread *td; 505 int flags; 506{ 507 struct pipe *rpipe = fp->f_data; 508 int error; 509 int nread = 0; 510 u_int size; 511 512 PIPE_LOCK(rpipe); 513 ++rpipe->pipe_busy; 514 error = pipelock(rpipe, 1); 515 if (error) 516 goto unlocked_error; 517 518#ifdef MAC 519 error = mac_check_pipe_read(active_cred, rpipe); 520 if (error) 521 goto locked_error; 522#endif 523 524 while (uio->uio_resid) { 525 /* 526 * normal pipe buffer receive 527 */ 528 if (rpipe->pipe_buffer.cnt > 0) { 529 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 530 if (size > rpipe->pipe_buffer.cnt) 531 size = rpipe->pipe_buffer.cnt; 532 if (size > (u_int) uio->uio_resid) 533 size = (u_int) uio->uio_resid; 534 535 PIPE_UNLOCK(rpipe); 536 error = uiomove( 537 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 538 size, uio); 539 PIPE_LOCK(rpipe); 540 if (error) 541 break; 542 543 rpipe->pipe_buffer.out += size; 544 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 545 rpipe->pipe_buffer.out = 0; 546 547 rpipe->pipe_buffer.cnt -= size; 548 549 /* 550 * If there is no more to read in the pipe, reset 551 * its pointers to the beginning. This improves 552 * cache hit stats. 553 */ 554 if (rpipe->pipe_buffer.cnt == 0) { 555 rpipe->pipe_buffer.in = 0; 556 rpipe->pipe_buffer.out = 0; 557 } 558 nread += size; 559#ifndef PIPE_NODIRECT 560 /* 561 * Direct copy, bypassing a kernel buffer. 562 */ 563 } else if ((size = rpipe->pipe_map.cnt) && 564 (rpipe->pipe_state & PIPE_DIRECTW)) { 565 caddr_t va; 566 if (size > (u_int) uio->uio_resid) 567 size = (u_int) uio->uio_resid; 568 569 va = (caddr_t) rpipe->pipe_map.kva + 570 rpipe->pipe_map.pos; 571 PIPE_UNLOCK(rpipe); 572 error = uiomove(va, size, uio); 573 PIPE_LOCK(rpipe); 574 if (error) 575 break; 576 nread += size; 577 rpipe->pipe_map.pos += size; 578 rpipe->pipe_map.cnt -= size; 579 if (rpipe->pipe_map.cnt == 0) { 580 rpipe->pipe_state &= ~PIPE_DIRECTW; 581 wakeup(rpipe); 582 } 583#endif 584 } else { 585 /* 586 * detect EOF condition 587 * read returns 0 on EOF, no need to set error 588 */ 589 if (rpipe->pipe_state & PIPE_EOF) 590 break; 591 592 /* 593 * If the "write-side" has been blocked, wake it up now. 594 */ 595 if (rpipe->pipe_state & PIPE_WANTW) { 596 rpipe->pipe_state &= ~PIPE_WANTW; 597 wakeup(rpipe); 598 } 599 600 /* 601 * Break if some data was read. 602 */ 603 if (nread > 0) 604 break; 605 606 /* 607 * Unlock the pipe buffer for our remaining processing. 608 * We will either break out with an error or we will 609 * sleep and relock to loop. 610 */ 611 pipeunlock(rpipe); 612 613 /* 614 * Handle non-blocking mode operation or 615 * wait for more data. 616 */ 617 if (fp->f_flag & FNONBLOCK) { 618 error = EAGAIN; 619 } else { 620 rpipe->pipe_state |= PIPE_WANTR; 621 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 622 PRIBIO | PCATCH, 623 "piperd", 0)) == 0) 624 error = pipelock(rpipe, 1); 625 } 626 if (error) 627 goto unlocked_error; 628 } 629 } 630#ifdef MAC 631locked_error: 632#endif 633 pipeunlock(rpipe); 634 635 /* XXX: should probably do this before getting any locks. */ 636 if (error == 0) 637 vfs_timestamp(&rpipe->pipe_atime); 638unlocked_error: 639 --rpipe->pipe_busy; 640 641 /* 642 * PIPE_WANT processing only makes sense if pipe_busy is 0. 643 */ 644 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 645 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 646 wakeup(rpipe); 647 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 648 /* 649 * Handle write blocking hysteresis. 650 */ 651 if (rpipe->pipe_state & PIPE_WANTW) { 652 rpipe->pipe_state &= ~PIPE_WANTW; 653 wakeup(rpipe); 654 } 655 } 656 657 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 658 pipeselwakeup(rpipe); 659 660 PIPE_UNLOCK(rpipe); 661 return (error); 662} 663 664#ifndef PIPE_NODIRECT 665/* 666 * Map the sending processes' buffer into kernel space and wire it. 667 * This is similar to a physical write operation. 668 */ 669static int 670pipe_build_write_buffer(wpipe, uio) 671 struct pipe *wpipe; 672 struct uio *uio; 673{ 674 u_int size; 675 int i; 676 vm_offset_t addr, endaddr; 677 vm_paddr_t paddr; 678 679 GIANT_REQUIRED; 680 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 681 682 size = (u_int) uio->uio_iov->iov_len; 683 if (size > wpipe->pipe_buffer.size) 684 size = wpipe->pipe_buffer.size; 685 686 endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size); 687 addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base); 688 for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) { 689 vm_page_t m; 690 691 /* 692 * vm_fault_quick() can sleep. Consequently, 693 * vm_page_lock_queue() and vm_page_unlock_queue() 694 * should not be performed outside of this loop. 695 */ 696 if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0 || 697 (paddr = pmap_extract(vmspace_pmap(curproc->p_vmspace), 698 addr)) == 0) { 699 int j; 700 701 vm_page_lock_queues(); 702 for (j = 0; j < i; j++) { 703 vm_page_unwire(wpipe->pipe_map.ms[j], 1); 704 atomic_subtract_int(&amountpipekvawired, 705 PAGE_SIZE); 706 } 707 vm_page_unlock_queues(); 708 return (EFAULT); 709 } 710 711 m = PHYS_TO_VM_PAGE(paddr); 712 vm_page_lock_queues(); 713 vm_page_wire(m); 714 atomic_add_int(&amountpipekvawired, PAGE_SIZE); 715 vm_page_unlock_queues(); 716 wpipe->pipe_map.ms[i] = m; 717 } 718 719/* 720 * set up the control block 721 */ 722 wpipe->pipe_map.npages = i; 723 wpipe->pipe_map.pos = 724 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 725 wpipe->pipe_map.cnt = size; 726 727/* 728 * and map the buffer 729 */ 730 if (wpipe->pipe_map.kva == 0) { 731 /* 732 * We need to allocate space for an extra page because the 733 * address range might (will) span pages at times. 734 */ 735 wpipe->pipe_map.kva = kmem_alloc_nofault(kernel_map, 736 wpipe->pipe_buffer.size + PAGE_SIZE); 737 atomic_add_int(&amountpipekva, 738 wpipe->pipe_buffer.size + PAGE_SIZE); 739 } 740 pmap_qenter(wpipe->pipe_map.kva, wpipe->pipe_map.ms, 741 wpipe->pipe_map.npages); 742 743/* 744 * and update the uio data 745 */ 746 747 uio->uio_iov->iov_len -= size; 748 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 749 if (uio->uio_iov->iov_len == 0) 750 uio->uio_iov++; 751 uio->uio_resid -= size; 752 uio->uio_offset += size; 753 return (0); 754} 755 756/* 757 * unmap and unwire the process buffer 758 */ 759static void 760pipe_destroy_write_buffer(wpipe) 761 struct pipe *wpipe; 762{ 763 int i; 764 765 GIANT_REQUIRED; 766 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 767 768 if (wpipe->pipe_map.kva) { 769 pmap_qremove(wpipe->pipe_map.kva, wpipe->pipe_map.npages); 770 771 if (amountpipekva > maxpipekva) { 772 vm_offset_t kva = wpipe->pipe_map.kva; 773 wpipe->pipe_map.kva = 0; 774 kmem_free(kernel_map, kva, 775 wpipe->pipe_buffer.size + PAGE_SIZE); 776 atomic_subtract_int(&amountpipekva, 777 wpipe->pipe_buffer.size + PAGE_SIZE); 778 } 779 } 780 vm_page_lock_queues(); 781 for (i = 0; i < wpipe->pipe_map.npages; i++) { 782 vm_page_unwire(wpipe->pipe_map.ms[i], 1); 783 atomic_subtract_int(&amountpipekvawired, PAGE_SIZE); 784 } 785 vm_page_unlock_queues(); 786 wpipe->pipe_map.npages = 0; 787} 788 789/* 790 * In the case of a signal, the writing process might go away. This 791 * code copies the data into the circular buffer so that the source 792 * pages can be freed without loss of data. 793 */ 794static void 795pipe_clone_write_buffer(wpipe) 796 struct pipe *wpipe; 797{ 798 int size; 799 int pos; 800 801 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 802 size = wpipe->pipe_map.cnt; 803 pos = wpipe->pipe_map.pos; 804 805 wpipe->pipe_buffer.in = size; 806 wpipe->pipe_buffer.out = 0; 807 wpipe->pipe_buffer.cnt = size; 808 wpipe->pipe_state &= ~PIPE_DIRECTW; 809 810 PIPE_GET_GIANT(wpipe); 811 bcopy((caddr_t) wpipe->pipe_map.kva + pos, 812 wpipe->pipe_buffer.buffer, size); 813 pipe_destroy_write_buffer(wpipe); 814 PIPE_DROP_GIANT(wpipe); 815} 816 817/* 818 * This implements the pipe buffer write mechanism. Note that only 819 * a direct write OR a normal pipe write can be pending at any given time. 820 * If there are any characters in the pipe buffer, the direct write will 821 * be deferred until the receiving process grabs all of the bytes from 822 * the pipe buffer. Then the direct mapping write is set-up. 823 */ 824static int 825pipe_direct_write(wpipe, uio) 826 struct pipe *wpipe; 827 struct uio *uio; 828{ 829 int error; 830 831retry: 832 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 833 while (wpipe->pipe_state & PIPE_DIRECTW) { 834 if (wpipe->pipe_state & PIPE_WANTR) { 835 wpipe->pipe_state &= ~PIPE_WANTR; 836 wakeup(wpipe); 837 } 838 wpipe->pipe_state |= PIPE_WANTW; 839 error = msleep(wpipe, PIPE_MTX(wpipe), 840 PRIBIO | PCATCH, "pipdww", 0); 841 if (error) 842 goto error1; 843 if (wpipe->pipe_state & PIPE_EOF) { 844 error = EPIPE; 845 goto error1; 846 } 847 } 848 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 849 if (wpipe->pipe_buffer.cnt > 0) { 850 if (wpipe->pipe_state & PIPE_WANTR) { 851 wpipe->pipe_state &= ~PIPE_WANTR; 852 wakeup(wpipe); 853 } 854 855 wpipe->pipe_state |= PIPE_WANTW; 856 error = msleep(wpipe, PIPE_MTX(wpipe), 857 PRIBIO | PCATCH, "pipdwc", 0); 858 if (error) 859 goto error1; 860 if (wpipe->pipe_state & PIPE_EOF) { 861 error = EPIPE; 862 goto error1; 863 } 864 goto retry; 865 } 866 867 wpipe->pipe_state |= PIPE_DIRECTW; 868 869 pipelock(wpipe, 0); 870 PIPE_GET_GIANT(wpipe); 871 error = pipe_build_write_buffer(wpipe, uio); 872 PIPE_DROP_GIANT(wpipe); 873 pipeunlock(wpipe); 874 if (error) { 875 wpipe->pipe_state &= ~PIPE_DIRECTW; 876 goto error1; 877 } 878 879 error = 0; 880 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 881 if (wpipe->pipe_state & PIPE_EOF) { 882 pipelock(wpipe, 0); 883 PIPE_GET_GIANT(wpipe); 884 pipe_destroy_write_buffer(wpipe); 885 PIPE_DROP_GIANT(wpipe); 886 pipeselwakeup(wpipe); 887 pipeunlock(wpipe); 888 error = EPIPE; 889 goto error1; 890 } 891 if (wpipe->pipe_state & PIPE_WANTR) { 892 wpipe->pipe_state &= ~PIPE_WANTR; 893 wakeup(wpipe); 894 } 895 pipeselwakeup(wpipe); 896 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 897 "pipdwt", 0); 898 } 899 900 pipelock(wpipe,0); 901 if (wpipe->pipe_state & PIPE_DIRECTW) { 902 /* 903 * this bit of trickery substitutes a kernel buffer for 904 * the process that might be going away. 905 */ 906 pipe_clone_write_buffer(wpipe); 907 } else { 908 PIPE_GET_GIANT(wpipe); 909 pipe_destroy_write_buffer(wpipe); 910 PIPE_DROP_GIANT(wpipe); 911 } 912 pipeunlock(wpipe); 913 return (error); 914 915error1: 916 wakeup(wpipe); 917 return (error); 918} 919#endif 920 921static int 922pipe_write(fp, uio, active_cred, flags, td) 923 struct file *fp; 924 struct uio *uio; 925 struct ucred *active_cred; 926 struct thread *td; 927 int flags; 928{ 929 int error = 0; 930 int orig_resid; 931 struct pipe *wpipe, *rpipe; 932 933 rpipe = fp->f_data; 934 wpipe = rpipe->pipe_peer; 935 936 PIPE_LOCK(rpipe); 937 /* 938 * detect loss of pipe read side, issue SIGPIPE if lost. 939 */ 940 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 941 PIPE_UNLOCK(rpipe); 942 return (EPIPE); 943 } 944#ifdef MAC 945 error = mac_check_pipe_write(active_cred, wpipe); 946 if (error) { 947 PIPE_UNLOCK(rpipe); 948 return (error); 949 } 950#endif 951 ++wpipe->pipe_busy; 952 953 /* 954 * If it is advantageous to resize the pipe buffer, do 955 * so. 956 */ 957 if ((uio->uio_resid > PIPE_SIZE) && 958 (amountpipekva < maxpipekva) && 959 (nbigpipe < LIMITBIGPIPES) && 960 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 961 (wpipe->pipe_buffer.size <= PIPE_SIZE) && 962 (wpipe->pipe_buffer.cnt == 0)) { 963 964 if ((error = pipelock(wpipe, 1)) == 0) { 965 PIPE_GET_GIANT(wpipe); 966 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 967 atomic_add_int(&nbigpipe, 1); 968 PIPE_DROP_GIANT(wpipe); 969 pipeunlock(wpipe); 970 } 971 } 972 973 /* 974 * If an early error occured unbusy and return, waking up any pending 975 * readers. 976 */ 977 if (error) { 978 --wpipe->pipe_busy; 979 if ((wpipe->pipe_busy == 0) && 980 (wpipe->pipe_state & PIPE_WANT)) { 981 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 982 wakeup(wpipe); 983 } 984 PIPE_UNLOCK(rpipe); 985 return(error); 986 } 987 988 orig_resid = uio->uio_resid; 989 990 while (uio->uio_resid) { 991 int space; 992 993#ifndef PIPE_NODIRECT 994 /* 995 * If the transfer is large, we can gain performance if 996 * we do process-to-process copies directly. 997 * If the write is non-blocking, we don't use the 998 * direct write mechanism. 999 * 1000 * The direct write mechanism will detect the reader going 1001 * away on us. 1002 */ 1003 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 1004 (fp->f_flag & FNONBLOCK) == 0 && 1005 amountpipekvawired < maxpipekvawired) { 1006 error = pipe_direct_write(wpipe, uio); 1007 if (error) 1008 break; 1009 continue; 1010 } 1011#endif 1012 1013 /* 1014 * Pipe buffered writes cannot be coincidental with 1015 * direct writes. We wait until the currently executing 1016 * direct write is completed before we start filling the 1017 * pipe buffer. We break out if a signal occurs or the 1018 * reader goes away. 1019 */ 1020 retrywrite: 1021 while (wpipe->pipe_state & PIPE_DIRECTW) { 1022 if (wpipe->pipe_state & PIPE_WANTR) { 1023 wpipe->pipe_state &= ~PIPE_WANTR; 1024 wakeup(wpipe); 1025 } 1026 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1027 "pipbww", 0); 1028 if (wpipe->pipe_state & PIPE_EOF) 1029 break; 1030 if (error) 1031 break; 1032 } 1033 if (wpipe->pipe_state & PIPE_EOF) { 1034 error = EPIPE; 1035 break; 1036 } 1037 1038 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1039 1040 /* Writes of size <= PIPE_BUF must be atomic. */ 1041 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1042 space = 0; 1043 1044 if (space > 0) { 1045 if ((error = pipelock(wpipe,1)) == 0) { 1046 int size; /* Transfer size */ 1047 int segsize; /* first segment to transfer */ 1048 1049 /* 1050 * It is possible for a direct write to 1051 * slip in on us... handle it here... 1052 */ 1053 if (wpipe->pipe_state & PIPE_DIRECTW) { 1054 pipeunlock(wpipe); 1055 goto retrywrite; 1056 } 1057 /* 1058 * If a process blocked in uiomove, our 1059 * value for space might be bad. 1060 * 1061 * XXX will we be ok if the reader has gone 1062 * away here? 1063 */ 1064 if (space > wpipe->pipe_buffer.size - 1065 wpipe->pipe_buffer.cnt) { 1066 pipeunlock(wpipe); 1067 goto retrywrite; 1068 } 1069 1070 /* 1071 * Transfer size is minimum of uio transfer 1072 * and free space in pipe buffer. 1073 */ 1074 if (space > uio->uio_resid) 1075 size = uio->uio_resid; 1076 else 1077 size = space; 1078 /* 1079 * First segment to transfer is minimum of 1080 * transfer size and contiguous space in 1081 * pipe buffer. If first segment to transfer 1082 * is less than the transfer size, we've got 1083 * a wraparound in the buffer. 1084 */ 1085 segsize = wpipe->pipe_buffer.size - 1086 wpipe->pipe_buffer.in; 1087 if (segsize > size) 1088 segsize = size; 1089 1090 /* Transfer first segment */ 1091 1092 PIPE_UNLOCK(rpipe); 1093 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1094 segsize, uio); 1095 PIPE_LOCK(rpipe); 1096 1097 if (error == 0 && segsize < size) { 1098 /* 1099 * Transfer remaining part now, to 1100 * support atomic writes. Wraparound 1101 * happened. 1102 */ 1103 if (wpipe->pipe_buffer.in + segsize != 1104 wpipe->pipe_buffer.size) 1105 panic("Expected pipe buffer " 1106 "wraparound disappeared"); 1107 1108 PIPE_UNLOCK(rpipe); 1109 error = uiomove( 1110 &wpipe->pipe_buffer.buffer[0], 1111 size - segsize, uio); 1112 PIPE_LOCK(rpipe); 1113 } 1114 if (error == 0) { 1115 wpipe->pipe_buffer.in += size; 1116 if (wpipe->pipe_buffer.in >= 1117 wpipe->pipe_buffer.size) { 1118 if (wpipe->pipe_buffer.in != 1119 size - segsize + 1120 wpipe->pipe_buffer.size) 1121 panic("Expected " 1122 "wraparound bad"); 1123 wpipe->pipe_buffer.in = size - 1124 segsize; 1125 } 1126 1127 wpipe->pipe_buffer.cnt += size; 1128 if (wpipe->pipe_buffer.cnt > 1129 wpipe->pipe_buffer.size) 1130 panic("Pipe buffer overflow"); 1131 1132 } 1133 pipeunlock(wpipe); 1134 } 1135 if (error) 1136 break; 1137 1138 } else { 1139 /* 1140 * If the "read-side" has been blocked, wake it up now. 1141 */ 1142 if (wpipe->pipe_state & PIPE_WANTR) { 1143 wpipe->pipe_state &= ~PIPE_WANTR; 1144 wakeup(wpipe); 1145 } 1146 1147 /* 1148 * don't block on non-blocking I/O 1149 */ 1150 if (fp->f_flag & FNONBLOCK) { 1151 error = EAGAIN; 1152 break; 1153 } 1154 1155 /* 1156 * We have no more space and have something to offer, 1157 * wake up select/poll. 1158 */ 1159 pipeselwakeup(wpipe); 1160 1161 wpipe->pipe_state |= PIPE_WANTW; 1162 error = msleep(wpipe, PIPE_MTX(rpipe), 1163 PRIBIO | PCATCH, "pipewr", 0); 1164 if (error != 0) 1165 break; 1166 /* 1167 * If read side wants to go away, we just issue a signal 1168 * to ourselves. 1169 */ 1170 if (wpipe->pipe_state & PIPE_EOF) { 1171 error = EPIPE; 1172 break; 1173 } 1174 } 1175 } 1176 1177 --wpipe->pipe_busy; 1178 1179 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1180 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1181 wakeup(wpipe); 1182 } else if (wpipe->pipe_buffer.cnt > 0) { 1183 /* 1184 * If we have put any characters in the buffer, we wake up 1185 * the reader. 1186 */ 1187 if (wpipe->pipe_state & PIPE_WANTR) { 1188 wpipe->pipe_state &= ~PIPE_WANTR; 1189 wakeup(wpipe); 1190 } 1191 } 1192 1193 /* 1194 * Don't return EPIPE if I/O was successful 1195 */ 1196 if ((wpipe->pipe_buffer.cnt == 0) && 1197 (uio->uio_resid == 0) && 1198 (error == EPIPE)) { 1199 error = 0; 1200 } 1201 1202 if (error == 0) 1203 vfs_timestamp(&wpipe->pipe_mtime); 1204 1205 /* 1206 * We have something to offer, 1207 * wake up select/poll. 1208 */ 1209 if (wpipe->pipe_buffer.cnt) 1210 pipeselwakeup(wpipe); 1211 1212 PIPE_UNLOCK(rpipe); 1213 return (error); 1214} 1215 1216/* 1217 * we implement a very minimal set of ioctls for compatibility with sockets. 1218 */ 1219static int 1220pipe_ioctl(fp, cmd, data, active_cred, td) 1221 struct file *fp; 1222 u_long cmd; 1223 void *data; 1224 struct ucred *active_cred; 1225 struct thread *td; 1226{ 1227 struct pipe *mpipe = fp->f_data; 1228#ifdef MAC 1229 int error; 1230#endif 1231 1232 PIPE_LOCK(mpipe); 1233 1234#ifdef MAC 1235 error = mac_check_pipe_ioctl(active_cred, mpipe, cmd, data); 1236 if (error) 1237 return (error); 1238#endif 1239 1240 switch (cmd) { 1241 1242 case FIONBIO: 1243 PIPE_UNLOCK(mpipe); 1244 return (0); 1245 1246 case FIOASYNC: 1247 if (*(int *)data) { 1248 mpipe->pipe_state |= PIPE_ASYNC; 1249 } else { 1250 mpipe->pipe_state &= ~PIPE_ASYNC; 1251 } 1252 PIPE_UNLOCK(mpipe); 1253 return (0); 1254 1255 case FIONREAD: 1256 if (mpipe->pipe_state & PIPE_DIRECTW) 1257 *(int *)data = mpipe->pipe_map.cnt; 1258 else 1259 *(int *)data = mpipe->pipe_buffer.cnt; 1260 PIPE_UNLOCK(mpipe); 1261 return (0); 1262 1263 case FIOSETOWN: 1264 PIPE_UNLOCK(mpipe); 1265 return (fsetown(*(int *)data, &mpipe->pipe_sigio)); 1266 1267 case FIOGETOWN: 1268 PIPE_UNLOCK(mpipe); 1269 *(int *)data = fgetown(&mpipe->pipe_sigio); 1270 return (0); 1271 1272 /* This is deprecated, FIOSETOWN should be used instead. */ 1273 case TIOCSPGRP: 1274 PIPE_UNLOCK(mpipe); 1275 return (fsetown(-(*(int *)data), &mpipe->pipe_sigio)); 1276 1277 /* This is deprecated, FIOGETOWN should be used instead. */ 1278 case TIOCGPGRP: 1279 PIPE_UNLOCK(mpipe); 1280 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1281 return (0); 1282 1283 } 1284 PIPE_UNLOCK(mpipe); 1285 return (ENOTTY); 1286} 1287 1288static int 1289pipe_poll(fp, events, active_cred, td) 1290 struct file *fp; 1291 int events; 1292 struct ucred *active_cred; 1293 struct thread *td; 1294{ 1295 struct pipe *rpipe = fp->f_data; 1296 struct pipe *wpipe; 1297 int revents = 0; 1298#ifdef MAC 1299 int error; 1300#endif 1301 1302 wpipe = rpipe->pipe_peer; 1303 PIPE_LOCK(rpipe); 1304#ifdef MAC 1305 error = mac_check_pipe_poll(active_cred, rpipe); 1306 if (error) 1307 goto locked_error; 1308#endif 1309 if (events & (POLLIN | POLLRDNORM)) 1310 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1311 (rpipe->pipe_buffer.cnt > 0) || 1312 (rpipe->pipe_state & PIPE_EOF)) 1313 revents |= events & (POLLIN | POLLRDNORM); 1314 1315 if (events & (POLLOUT | POLLWRNORM)) 1316 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) || 1317 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1318 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1319 revents |= events & (POLLOUT | POLLWRNORM); 1320 1321 if ((rpipe->pipe_state & PIPE_EOF) || 1322 (wpipe == NULL) || 1323 (wpipe->pipe_state & PIPE_EOF)) 1324 revents |= POLLHUP; 1325 1326 if (revents == 0) { 1327 if (events & (POLLIN | POLLRDNORM)) { 1328 selrecord(td, &rpipe->pipe_sel); 1329 rpipe->pipe_state |= PIPE_SEL; 1330 } 1331 1332 if (events & (POLLOUT | POLLWRNORM)) { 1333 selrecord(td, &wpipe->pipe_sel); 1334 wpipe->pipe_state |= PIPE_SEL; 1335 } 1336 } 1337#ifdef MAC 1338locked_error: 1339#endif 1340 PIPE_UNLOCK(rpipe); 1341 1342 return (revents); 1343} 1344 1345/* 1346 * We shouldn't need locks here as we're doing a read and this should 1347 * be a natural race. 1348 */ 1349static int 1350pipe_stat(fp, ub, active_cred, td) 1351 struct file *fp; 1352 struct stat *ub; 1353 struct ucred *active_cred; 1354 struct thread *td; 1355{ 1356 struct pipe *pipe = fp->f_data; 1357#ifdef MAC 1358 int error; 1359 1360 PIPE_LOCK(pipe); 1361 error = mac_check_pipe_stat(active_cred, pipe); 1362 PIPE_UNLOCK(pipe); 1363 if (error) 1364 return (error); 1365#endif 1366 bzero(ub, sizeof(*ub)); 1367 ub->st_mode = S_IFIFO; 1368 ub->st_blksize = pipe->pipe_buffer.size; 1369 ub->st_size = pipe->pipe_buffer.cnt; 1370 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1371 ub->st_atimespec = pipe->pipe_atime; 1372 ub->st_mtimespec = pipe->pipe_mtime; 1373 ub->st_ctimespec = pipe->pipe_ctime; 1374 ub->st_uid = fp->f_cred->cr_uid; 1375 ub->st_gid = fp->f_cred->cr_gid; 1376 /* 1377 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1378 * XXX (st_dev, st_ino) should be unique. 1379 */ 1380 return (0); 1381} 1382 1383/* ARGSUSED */ 1384static int 1385pipe_close(fp, td) 1386 struct file *fp; 1387 struct thread *td; 1388{ 1389 struct pipe *cpipe = fp->f_data; 1390 1391 fp->f_ops = &badfileops; 1392 fp->f_data = NULL; 1393 funsetown(&cpipe->pipe_sigio); 1394 pipeclose(cpipe); 1395 return (0); 1396} 1397 1398static void 1399pipe_free_kmem(cpipe) 1400 struct pipe *cpipe; 1401{ 1402 1403 KASSERT(cpipe->pipe_mtxp == NULL || !mtx_owned(PIPE_MTX(cpipe)), 1404 ("pipespace: pipe mutex locked")); 1405 1406 if (cpipe->pipe_buffer.buffer != NULL) { 1407 if (cpipe->pipe_buffer.size > PIPE_SIZE) 1408 atomic_subtract_int(&nbigpipe, 1); 1409 atomic_subtract_int(&amountpipekva, cpipe->pipe_buffer.size); 1410 atomic_subtract_int(&amountpipes, 1); 1411 kmem_free(kernel_map, 1412 (vm_offset_t)cpipe->pipe_buffer.buffer, 1413 cpipe->pipe_buffer.size); 1414 cpipe->pipe_buffer.buffer = NULL; 1415 } 1416#ifndef PIPE_NODIRECT 1417 if (cpipe->pipe_map.kva != 0) { 1418 atomic_subtract_int(&amountpipekva, 1419 cpipe->pipe_buffer.size + PAGE_SIZE); 1420 kmem_free(kernel_map, 1421 cpipe->pipe_map.kva, 1422 cpipe->pipe_buffer.size + PAGE_SIZE); 1423 cpipe->pipe_map.cnt = 0; 1424 cpipe->pipe_map.kva = 0; 1425 cpipe->pipe_map.pos = 0; 1426 cpipe->pipe_map.npages = 0; 1427 } 1428#endif 1429} 1430 1431/* 1432 * shutdown the pipe 1433 */ 1434static void 1435pipeclose(cpipe) 1436 struct pipe *cpipe; 1437{ 1438 struct pipe *ppipe; 1439 int hadpeer; 1440 1441 if (cpipe == NULL) 1442 return; 1443 1444 hadpeer = 0; 1445 1446 /* partially created pipes won't have a valid mutex. */ 1447 if (PIPE_MTX(cpipe) != NULL) 1448 PIPE_LOCK(cpipe); 1449 1450 pipeselwakeup(cpipe); 1451 1452 /* 1453 * If the other side is blocked, wake it up saying that 1454 * we want to close it down. 1455 */ 1456 while (cpipe->pipe_busy) { 1457 wakeup(cpipe); 1458 cpipe->pipe_state |= PIPE_WANT | PIPE_EOF; 1459 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1460 } 1461 1462#ifdef MAC 1463 if (cpipe->pipe_label != NULL && cpipe->pipe_peer == NULL) 1464 mac_destroy_pipe(cpipe); 1465#endif 1466 1467 /* 1468 * Disconnect from peer 1469 */ 1470 if ((ppipe = cpipe->pipe_peer) != NULL) { 1471 hadpeer++; 1472 pipeselwakeup(ppipe); 1473 1474 ppipe->pipe_state |= PIPE_EOF; 1475 wakeup(ppipe); 1476 KNOTE(&ppipe->pipe_sel.si_note, 0); 1477 ppipe->pipe_peer = NULL; 1478 } 1479 /* 1480 * free resources 1481 */ 1482 if (PIPE_MTX(cpipe) != NULL) { 1483 PIPE_UNLOCK(cpipe); 1484 if (!hadpeer) { 1485 mtx_destroy(PIPE_MTX(cpipe)); 1486 free(PIPE_MTX(cpipe), M_TEMP); 1487 } 1488 } 1489 pipe_free_kmem(cpipe); 1490 uma_zfree(pipe_zone, cpipe); 1491} 1492 1493/*ARGSUSED*/ 1494static int 1495pipe_kqfilter(struct file *fp, struct knote *kn) 1496{ 1497 struct pipe *cpipe; 1498 1499 cpipe = kn->kn_fp->f_data; 1500 switch (kn->kn_filter) { 1501 case EVFILT_READ: 1502 kn->kn_fop = &pipe_rfiltops; 1503 break; 1504 case EVFILT_WRITE: 1505 kn->kn_fop = &pipe_wfiltops; 1506 cpipe = cpipe->pipe_peer; 1507 if (cpipe == NULL) 1508 /* other end of pipe has been closed */ 1509 return (EBADF); 1510 break; 1511 default: 1512 return (1); 1513 } 1514 kn->kn_hook = cpipe; 1515 1516 PIPE_LOCK(cpipe); 1517 SLIST_INSERT_HEAD(&cpipe->pipe_sel.si_note, kn, kn_selnext); 1518 PIPE_UNLOCK(cpipe); 1519 return (0); 1520} 1521 1522static void 1523filt_pipedetach(struct knote *kn) 1524{ 1525 struct pipe *cpipe = (struct pipe *)kn->kn_hook; 1526 1527 PIPE_LOCK(cpipe); 1528 SLIST_REMOVE(&cpipe->pipe_sel.si_note, kn, knote, kn_selnext); 1529 PIPE_UNLOCK(cpipe); 1530} 1531 1532/*ARGSUSED*/ 1533static int 1534filt_piperead(struct knote *kn, long hint) 1535{ 1536 struct pipe *rpipe = kn->kn_fp->f_data; 1537 struct pipe *wpipe = rpipe->pipe_peer; 1538 1539 PIPE_LOCK(rpipe); 1540 kn->kn_data = rpipe->pipe_buffer.cnt; 1541 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1542 kn->kn_data = rpipe->pipe_map.cnt; 1543 1544 if ((rpipe->pipe_state & PIPE_EOF) || 1545 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1546 kn->kn_flags |= EV_EOF; 1547 PIPE_UNLOCK(rpipe); 1548 return (1); 1549 } 1550 PIPE_UNLOCK(rpipe); 1551 return (kn->kn_data > 0); 1552} 1553 1554/*ARGSUSED*/ 1555static int 1556filt_pipewrite(struct knote *kn, long hint) 1557{ 1558 struct pipe *rpipe = kn->kn_fp->f_data; 1559 struct pipe *wpipe = rpipe->pipe_peer; 1560 1561 PIPE_LOCK(rpipe); 1562 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1563 kn->kn_data = 0; 1564 kn->kn_flags |= EV_EOF; 1565 PIPE_UNLOCK(rpipe); 1566 return (1); 1567 } 1568 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1569 if (wpipe->pipe_state & PIPE_DIRECTW) 1570 kn->kn_data = 0; 1571 1572 PIPE_UNLOCK(rpipe); 1573 return (kn->kn_data >= PIPE_BUF); 1574} 1575