sys_pipe.c revision 179242
125184Sjkh/*- 225184Sjkh * Copyright (c) 1996 John S. Dyson 348687Speter * All rights reserved. 425184Sjkh * 525184Sjkh * Redistribution and use in source and binary forms, with or without 625184Sjkh * modification, are permitted provided that the following conditions 725184Sjkh * are met: 825184Sjkh * 1. Redistributions of source code must retain the above copyright 925184Sjkh * notice immediately at the beginning of the file, without modification, 1025184Sjkh * this list of conditions, and the following disclaimer. 1125184Sjkh * 2. Redistributions in binary form must reproduce the above copyright 1225184Sjkh * notice, this list of conditions and the following disclaimer in the 1325184Sjkh * documentation and/or other materials provided with the distribution. 1425184Sjkh * 3. Absolutely no warranty of function or purpose is made by the author 1525184Sjkh * John S. Dyson. 1625184Sjkh * 4. Modifications may be freely made to this file if the above conditions 1725184Sjkh * are met. 1825184Sjkh */ 1925184Sjkh 2025184Sjkh/* 2125184Sjkh * This file contains a high-performance replacement for the socket-based 2225184Sjkh * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 2325184Sjkh * all features of sockets, but does do everything that pipes normally 2425184Sjkh * do. 2525184Sjkh */ 2625184Sjkh 2725184Sjkh/* 2840006Sphk * This code has two modes of operation, a small write mode and a large 2940006Sphk * write mode. The small write mode acts like conventional pipes with 3040006Sphk * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 3140006Sphk * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 3240006Sphk * and PIPE_SIZE in size, it is fully mapped and wired into the kernel, and 3340006Sphk * the receiving process can copy it directly from the pages in the sending 3442621Shm * process. 3542621Shm * 3642621Shm * If the sending process receives a signal, it is possible that it will 3742621Shm * go away, and certainly its address space can change, because control 3842621Shm * is returned back to the user-mode side. In that case, the pipe code 3942627Sjoerg * arranges to copy the buffer supplied by the user process, to a pageable 4042627Sjoerg * kernel buffer, and the receiving process will grab the data from the 4142627Sjoerg * pageable kernel buffer. Since signals don't happen all that often, 4242627Sjoerg * the copy operation is normally eliminated. 4342627Sjoerg * 4442627Sjoerg * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 4542627Sjoerg * happen for small transfers so that the system will not spend all of 4642627Sjoerg * its time context switching. 4742627Sjoerg * 4842627Sjoerg * In order to limit the resource use of pipes, two sysctls exist: 4942627Sjoerg * 5042627Sjoerg * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable 5142627Sjoerg * address space available to us in pipe_map. This value is normally 5242627Sjoerg * autotuned, but may also be loader tuned. 5325184Sjkh * 5448687Speter * kern.ipc.pipekva - This read-only sysctl tracks the current amount of 5548687Speter * memory in use by pipes. 5648687Speter * 5748687Speter * Based on how large pipekva is relative to maxpipekva, the following 5848662Speter * will happen: 5925184Sjkh * 6033682Sbrian * 0% - 50%: 6148662Speter * New pipes are given 16K of memory backing, pipes may dynamically 6225184Sjkh * grow to as large as 64K where needed. 6325184Sjkh * 50% - 75%: 6425184Sjkh * New pipes are given 4K (or PAGE_SIZE) of memory backing, 6525184Sjkh * existing pipes may NOT grow. 6625184Sjkh * 75% - 100%: 6748662Speter * New pipes are given 4K (or PAGE_SIZE) of memory backing, 6825184Sjkh * existing pipes will be shrunk down to 4K whenever possible. 6925184Sjkh * 7025184Sjkh * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0. If 7125184Sjkh * that is set, the only resize that will occur is the 0 -> SMALL_PIPE_SIZE 7225184Sjkh * resize which MUST occur for reverse-direction pipes when they are 7325184Sjkh * first used. 7425184Sjkh * 7525184Sjkh * Additional information about the current state of pipes may be obtained 7648662Speter * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail, 7725184Sjkh * and kern.ipc.piperesizefail. 7825184Sjkh * 7925184Sjkh * Locking rules: There are two locks present here: A mutex, used via 8025184Sjkh * PIPE_LOCK, and a flag, used via pipelock(). All locking is done via 8125184Sjkh * the flag, as mutexes can not persist over uiomove. The mutex 8225184Sjkh * exists only to guard access to the flag, and is not in itself a 8325184Sjkh * locking mechanism. Also note that there is only a single mutex for 8425184Sjkh * both directions of a pipe. 8525184Sjkh * 8648662Speter * As pipelock() may have to sleep before it can acquire the flag, it 8725184Sjkh * is important to reread all data after a call to pipelock(); everything 8848662Speter * in the structure may have changed. 8948662Speter */ 9048662Speter 9148662Speter#include <sys/cdefs.h> 9225184Sjkh__FBSDID("$FreeBSD: head/sys/kern/sys_pipe.c 179242 2008-05-23 11:09:50Z kib $"); 9329300Sdanny 9429300Sdanny#include "opt_mac.h" 9529300Sdanny 9629300Sdanny#include <sys/param.h> 9732382Salex#include <sys/systm.h> 9832382Salex#include <sys/fcntl.h> 9932382Salex#include <sys/file.h> 10029300Sdanny#include <sys/filedesc.h> 10129300Sdanny#include <sys/filio.h> 10229300Sdanny#include <sys/kernel.h> 10329300Sdanny#include <sys/lock.h> 10441077Speter#include <sys/mutex.h> 10529300Sdanny#include <sys/ttycom.h> 10629300Sdanny#include <sys/stat.h> 10729300Sdanny#include <sys/malloc.h> 10829300Sdanny#include <sys/poll.h> 10929300Sdanny#include <sys/selinfo.h> 11029300Sdanny#include <sys/signalvar.h> 11129300Sdanny#include <sys/sysctl.h> 11229300Sdanny#include <sys/sysproto.h> 11329300Sdanny#include <sys/pipe.h> 11445542Sdes#include <sys/proc.h> 11545542Sdes#include <sys/vnode.h> 11645542Sdes#include <sys/uio.h> 11745542Sdes#include <sys/event.h> 11845542Sdes 11945622Sbrian#include <security/mac/mac_framework.h> 12044992Sbrian 12144992Sbrian#include <vm/vm.h> 12244992Sbrian#include <vm/vm_param.h> 12344992Sbrian#include <vm/vm_object.h> 12444992Sbrian#include <vm/vm_kern.h> 12544992Sbrian#include <vm/vm_extern.h> 12644992Sbrian#include <vm/pmap.h> 12744992Sbrian#include <vm/vm_map.h> 12844992Sbrian#include <vm/vm_page.h> 12944992Sbrian#include <vm/uma.h> 13044992Sbrian 13144992Sbrian/* 13229300Sdanny * Use this define if you want to disable *fancy* VM things. Expect an 13333337Salex * approx 30% decrease in transfer rate. This could be useful for 13433337Salex * NetBSD or OpenBSD. 13533149Salex */ 13633149Salex/* #define PIPE_NODIRECT */ 13733149Salex 13833149Salex/* 13929300Sdanny * interfaces to the outside world 14025184Sjkh */ 14125184Sjkhstatic fo_rdwr_t pipe_read; 14240006Sphkstatic fo_rdwr_t pipe_write; 14340006Sphkstatic fo_truncate_t pipe_truncate; 14440006Sphkstatic fo_ioctl_t pipe_ioctl; 14540006Sphkstatic fo_poll_t pipe_poll; 14640006Sphkstatic fo_kqfilter_t pipe_kqfilter; 14729300Sdannystatic fo_stat_t pipe_stat; 14829300Sdannystatic fo_close_t pipe_close; 14925184Sjkh 15025184Sjkhstatic struct fileops pipeops = { 15125184Sjkh .fo_read = pipe_read, 15225184Sjkh .fo_write = pipe_write, 15325184Sjkh .fo_truncate = pipe_truncate, 15425184Sjkh .fo_ioctl = pipe_ioctl, 15525184Sjkh .fo_poll = pipe_poll, 15625184Sjkh .fo_kqfilter = pipe_kqfilter, 15725184Sjkh .fo_stat = pipe_stat, 15825184Sjkh .fo_close = pipe_close, 15925184Sjkh .fo_flags = DFLAG_PASSABLE 16025184Sjkh}; 16125184Sjkh 16225184Sjkhstatic void filt_pipedetach(struct knote *kn); 16327218Spststatic int filt_piperead(struct knote *kn, long hint); 16427218Spststatic int filt_pipewrite(struct knote *kn, long hint); 16547755Sbde 16627218Spststatic struct filterops pipe_rfiltops = 16727218Spst { 1, NULL, filt_pipedetach, filt_piperead }; 16845096Simpstatic struct filterops pipe_wfiltops = 16945096Simp { 1, NULL, filt_pipedetach, filt_pipewrite }; 17047755Sbde 17147755Sbde/* 17245096Simp * Default pipe buffer size(s), this can be kind-of large now because pipe 17345096Simp * space is pageable. The pipe code will try to maintain locality of 17439267Sjkoshy * reference for performance reasons, so small amounts of outstanding I/O 17539267Sjkoshy * will not wipe the cache. 17647755Sbde */ 17739267Sjkoshy#define MINPIPESIZE (PIPE_SIZE/3) 17839267Sjkoshy#define MAXPIPESIZE (2*PIPE_SIZE/3) 17925184Sjkh 18025365Sjkhstatic int amountpipekva; 18147755Sbdestatic int pipefragretry; 18225184Sjkhstatic int pipeallocfail; 18325184Sjkhstatic int piperesizefail; 18433439Sguidostatic int piperesizeallowed = 1; 18533439Sguido 18647755SbdeSYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN, 18733439Sguido &maxpipekva, 0, "Pipe KVA limit"); 18833439SguidoSYSCTL_INT(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 18933439Sguido &amountpipekva, 0, "Pipe KVA usage"); 19033439SguidoSYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD, 19147755Sbde &pipefragretry, 0, "Pipe allocation retries due to fragmentation"); 19233439SguidoSYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD, 19333439Sguido &pipeallocfail, 0, "Pipe allocation failures"); 19447752SphkSYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD, 19547752Sphk &piperesizefail, 0, "Pipe resize failures"); 19647755SbdeSYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW, 19747752Sphk &piperesizeallowed, 0, "Pipe resizing allowed"); 19847752Sphk 19925184Sjkhstatic void pipeinit(void *dummy __unused); 20025365Sjkhstatic void pipeclose(struct pipe *cpipe); 20147755Sbdestatic void pipe_free_kmem(struct pipe *cpipe); 20225184Sjkhstatic int pipe_create(struct pipe *pipe, int backing); 20325184Sjkhstatic __inline int pipelock(struct pipe *cpipe, int catch); 20436174Sjkhstatic __inline void pipeunlock(struct pipe *cpipe); 20547755Sbdestatic __inline void pipeselwakeup(struct pipe *cpipe); 20647755Sbde#ifndef PIPE_NODIRECT 20736174Sjkhstatic int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 20836174Sjkhstatic void pipe_destroy_write_buffer(struct pipe *wpipe); 20936174Sjkhstatic int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 21036174Sjkhstatic void pipe_clone_write_buffer(struct pipe *wpipe); 21136174Sjkh#endif 21236174Sjkhstatic int pipespace(struct pipe *cpipe, int size); 21336174Sjkhstatic int pipespace_new(struct pipe *cpipe, int size); 21436174Sjkh 21525184Sjkhstatic int pipe_zone_ctor(void *mem, int size, void *arg, int flags); 21636174Sjkhstatic int pipe_zone_init(void *mem, int size, int flags); 21725184Sjkhstatic void pipe_zone_fini(void *mem, int size); 21825184Sjkh 21925765Sjkhstatic uma_zone_t pipe_zone; 22036174Sjkh 22136174SjkhSYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 22225765Sjkh 22336174Sjkhstatic void 22434395Sjkhpipeinit(void *dummy __unused) 22534395Sjkh{ 22634395Sjkh 22725184Sjkh pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair), 22825184Sjkh pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini, 22925184Sjkh UMA_ALIGN_PTR, 0); 23025184Sjkh KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 23125184Sjkh} 23225184Sjkh 23325184Sjkhstatic int 23432949Swollmanpipe_zone_ctor(void *mem, int size, void *arg, int flags) 23525184Sjkh{ 23625184Sjkh struct pipepair *pp; 23731472Sobrien struct pipe *rpipe, *wpipe; 23835787Sandreas 23931472Sobrien KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size")); 24025184Sjkh 24131472Sobrien pp = (struct pipepair *)mem; 24235787Sandreas 24325184Sjkh /* 24425184Sjkh * We zero both pipe endpoints to make sure all the kmem pointers 24525184Sjkh * are NULL, flag fields are zero'd, etc. We timestamp both 24625184Sjkh * endpoints with the same time. 24725184Sjkh */ 24825184Sjkh rpipe = &pp->pp_rpipe; 24925184Sjkh bzero(rpipe, sizeof(*rpipe)); 25044668Sjfitz vfs_timestamp(&rpipe->pipe_ctime); 25125184Sjkh rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime; 25225184Sjkh 25325184Sjkh wpipe = &pp->pp_wpipe; 25425184Sjkh bzero(wpipe, sizeof(*wpipe)); 25525184Sjkh wpipe->pipe_ctime = rpipe->pipe_ctime; 25625184Sjkh wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime; 25725184Sjkh 25825184Sjkh rpipe->pipe_peer = wpipe; 25925184Sjkh rpipe->pipe_pair = pp; 26025184Sjkh wpipe->pipe_peer = rpipe; 26125184Sjkh wpipe->pipe_pair = pp; 26225184Sjkh 26325184Sjkh /* 26425184Sjkh * Mark both endpoints as present; they will later get free'd 26525184Sjkh * one at a time. When both are free'd, then the whole pair 26625184Sjkh * is released. 26725184Sjkh */ 26825184Sjkh rpipe->pipe_present = 1; 26925184Sjkh wpipe->pipe_present = 1; 27025184Sjkh 27125184Sjkh /* 27225184Sjkh * Eventually, the MAC Framework may initialize the label 27325184Sjkh * in ctor or init, but for now we do it elswhere to avoid 27425184Sjkh * blocking in ctor or init. 27535149Smarkm */ 27635149Smarkm pp->pp_label = NULL; 27735149Smarkm 27835149Smarkm return (0); 27935149Smarkm} 28035149Smarkm 28135149Smarkmstatic int 28235149Smarkmpipe_zone_init(void *mem, int size, int flags) 28335149Smarkm{ 28440006Sphk struct pipepair *pp; 28540006Sphk 28640006Sphk KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size")); 28740006Sphk 28840006Sphk pp = (struct pipepair *)mem; 28925184Sjkh 29025184Sjkh mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 29125184Sjkh return (0); 29225184Sjkh} 29325184Sjkh 29425184Sjkhstatic void 29525184Sjkhpipe_zone_fini(void *mem, int size) 29625184Sjkh{ 29725184Sjkh struct pipepair *pp; 29825184Sjkh 29925184Sjkh KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size")); 30025184Sjkh 30125184Sjkh pp = (struct pipepair *)mem; 30225184Sjkh 30347755Sbde mtx_destroy(&pp->pp_mtx); 30447755Sbde} 30525184Sjkh 30625916Sjkh/* 30725184Sjkh * The pipe system call for the DTYPE_PIPE type of pipes. If we fail, let 30825184Sjkh * the zone pick up the pieces via pipeclose(). 30925184Sjkh */ 31025184Sjkh/* ARGSUSED */ 31125184Sjkhint 31225184Sjkhpipe(td, uap) 31325184Sjkh struct thread *td; 31425184Sjkh struct pipe_args /* { 31525184Sjkh int dummy; 31625916Sjkh } */ *uap; 31741371Sjkoshy{ 31847755Sbde struct filedesc *fdp = td->td_proc->p_fd; 31941371Sjkoshy struct file *rf, *wf; 32047755Sbde struct pipepair *pp; 32141185Smsmith struct pipe *rpipe, *wpipe; 32225184Sjkh int fd, error; 32325184Sjkh 32425184Sjkh pp = uma_zalloc(pipe_zone, M_WAITOK); 32525184Sjkh#ifdef MAC 32635459Sphk /* 32739380Scracauer * The MAC label is shared between the connected endpoints. As a 32835459Sphk * result mac_pipe_init() and mac_pipe_create() are called once 32947838Sbrian * for the pair, and not on the endpoints. 33047838Sbrian */ 33147838Sbrian mac_pipe_init(pp); 33247838Sbrian mac_pipe_create(td->td_ucred, pp); 33347838Sbrian#endif 33447838Sbrian rpipe = &pp->pp_rpipe; 33525184Sjkh wpipe = &pp->pp_wpipe; 33625184Sjkh 33725184Sjkh knlist_init(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe), NULL, NULL, 33842270Sjkh NULL); 33925184Sjkh knlist_init(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe), NULL, NULL, 34025184Sjkh NULL); 34125184Sjkh 34225184Sjkh /* Only the forward direction pipe is backed by default */ 34331033Ssef if ((error = pipe_create(rpipe, 1)) != 0 || 34431033Ssef (error = pipe_create(wpipe, 0)) != 0) { 34531033Ssef pipeclose(rpipe); 34631033Ssef pipeclose(wpipe); 34731033Ssef return (error); 34831033Ssef } 34938316Sphk 35031033Ssef rpipe->pipe_state |= PIPE_DIRECTOK; 35131033Ssef wpipe->pipe_state |= PIPE_DIRECTOK; 35238316Sphk 35331033Ssef error = falloc(td, &rf, &fd); 35431033Ssef if (error) { 35525184Sjkh pipeclose(rpipe); 35625184Sjkh pipeclose(wpipe); 35725184Sjkh return (error); 35825184Sjkh } 35925184Sjkh /* An extra reference on `rf' has been held for us by falloc(). */ 360 td->td_retval[0] = fd; 361 362 /* 363 * Warning: once we've gotten past allocation of the fd for the 364 * read-side, we can only drop the read side via fdrop() in order 365 * to avoid races against processes which manage to dup() the read 366 * side while we are blocked trying to allocate the write side. 367 */ 368 finit(rf, FREAD | FWRITE, DTYPE_PIPE, rpipe, &pipeops); 369 error = falloc(td, &wf, &fd); 370 if (error) { 371 fdclose(fdp, rf, td->td_retval[0], td); 372 fdrop(rf, td); 373 /* rpipe has been closed by fdrop(). */ 374 pipeclose(wpipe); 375 return (error); 376 } 377 /* An extra reference on `wf' has been held for us by falloc(). */ 378 finit(wf, FREAD | FWRITE, DTYPE_PIPE, wpipe, &pipeops); 379 fdrop(wf, td); 380 td->td_retval[1] = fd; 381 fdrop(rf, td); 382 383 return (0); 384} 385 386/* 387 * Allocate kva for pipe circular buffer, the space is pageable 388 * This routine will 'realloc' the size of a pipe safely, if it fails 389 * it will retain the old buffer. 390 * If it fails it will return ENOMEM. 391 */ 392static int 393pipespace_new(cpipe, size) 394 struct pipe *cpipe; 395 int size; 396{ 397 caddr_t buffer; 398 int error, cnt, firstseg; 399 static int curfail = 0; 400 static struct timeval lastfail; 401 402 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked")); 403 KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW), 404 ("pipespace: resize of direct writes not allowed")); 405retry: 406 cnt = cpipe->pipe_buffer.cnt; 407 if (cnt > size) 408 size = cnt; 409 410 size = round_page(size); 411 buffer = (caddr_t) vm_map_min(pipe_map); 412 413 error = vm_map_find(pipe_map, NULL, 0, 414 (vm_offset_t *) &buffer, size, 1, 415 VM_PROT_ALL, VM_PROT_ALL, 0); 416 if (error != KERN_SUCCESS) { 417 if ((cpipe->pipe_buffer.buffer == NULL) && 418 (size > SMALL_PIPE_SIZE)) { 419 size = SMALL_PIPE_SIZE; 420 pipefragretry++; 421 goto retry; 422 } 423 if (cpipe->pipe_buffer.buffer == NULL) { 424 pipeallocfail++; 425 if (ppsratecheck(&lastfail, &curfail, 1)) 426 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n"); 427 } else { 428 piperesizefail++; 429 } 430 return (ENOMEM); 431 } 432 433 /* copy data, then free old resources if we're resizing */ 434 if (cnt > 0) { 435 if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) { 436 firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out; 437 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 438 buffer, firstseg); 439 if ((cnt - firstseg) > 0) 440 bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg], 441 cpipe->pipe_buffer.in); 442 } else { 443 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 444 buffer, cnt); 445 } 446 } 447 pipe_free_kmem(cpipe); 448 cpipe->pipe_buffer.buffer = buffer; 449 cpipe->pipe_buffer.size = size; 450 cpipe->pipe_buffer.in = cnt; 451 cpipe->pipe_buffer.out = 0; 452 cpipe->pipe_buffer.cnt = cnt; 453 atomic_add_int(&amountpipekva, cpipe->pipe_buffer.size); 454 return (0); 455} 456 457/* 458 * Wrapper for pipespace_new() that performs locking assertions. 459 */ 460static int 461pipespace(cpipe, size) 462 struct pipe *cpipe; 463 int size; 464{ 465 466 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 467 ("Unlocked pipe passed to pipespace")); 468 return (pipespace_new(cpipe, size)); 469} 470 471/* 472 * lock a pipe for I/O, blocking other access 473 */ 474static __inline int 475pipelock(cpipe, catch) 476 struct pipe *cpipe; 477 int catch; 478{ 479 int error; 480 481 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 482 while (cpipe->pipe_state & PIPE_LOCKFL) { 483 cpipe->pipe_state |= PIPE_LWANT; 484 error = msleep(cpipe, PIPE_MTX(cpipe), 485 catch ? (PRIBIO | PCATCH) : PRIBIO, 486 "pipelk", 0); 487 if (error != 0) 488 return (error); 489 } 490 cpipe->pipe_state |= PIPE_LOCKFL; 491 return (0); 492} 493 494/* 495 * unlock a pipe I/O lock 496 */ 497static __inline void 498pipeunlock(cpipe) 499 struct pipe *cpipe; 500{ 501 502 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 503 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 504 ("Unlocked pipe passed to pipeunlock")); 505 cpipe->pipe_state &= ~PIPE_LOCKFL; 506 if (cpipe->pipe_state & PIPE_LWANT) { 507 cpipe->pipe_state &= ~PIPE_LWANT; 508 wakeup(cpipe); 509 } 510} 511 512static __inline void 513pipeselwakeup(cpipe) 514 struct pipe *cpipe; 515{ 516 517 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 518 if (cpipe->pipe_state & PIPE_SEL) { 519 selwakeuppri(&cpipe->pipe_sel, PSOCK); 520 if (!SEL_WAITING(&cpipe->pipe_sel)) 521 cpipe->pipe_state &= ~PIPE_SEL; 522 } 523 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 524 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 525 KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0); 526} 527 528/* 529 * Initialize and allocate VM and memory for pipe. The structure 530 * will start out zero'd from the ctor, so we just manage the kmem. 531 */ 532static int 533pipe_create(pipe, backing) 534 struct pipe *pipe; 535 int backing; 536{ 537 int error; 538 539 if (backing) { 540 if (amountpipekva > maxpipekva / 2) 541 error = pipespace_new(pipe, SMALL_PIPE_SIZE); 542 else 543 error = pipespace_new(pipe, PIPE_SIZE); 544 } else { 545 /* If we're not backing this pipe, no need to do anything. */ 546 error = 0; 547 } 548 return (error); 549} 550 551/* ARGSUSED */ 552static int 553pipe_read(fp, uio, active_cred, flags, td) 554 struct file *fp; 555 struct uio *uio; 556 struct ucred *active_cred; 557 struct thread *td; 558 int flags; 559{ 560 struct pipe *rpipe = fp->f_data; 561 int error; 562 int nread = 0; 563 u_int size; 564 565 PIPE_LOCK(rpipe); 566 ++rpipe->pipe_busy; 567 error = pipelock(rpipe, 1); 568 if (error) 569 goto unlocked_error; 570 571#ifdef MAC 572 error = mac_pipe_check_read(active_cred, rpipe->pipe_pair); 573 if (error) 574 goto locked_error; 575#endif 576 if (amountpipekva > (3 * maxpipekva) / 4) { 577 if (!(rpipe->pipe_state & PIPE_DIRECTW) && 578 (rpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 579 (rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 580 (piperesizeallowed == 1)) { 581 PIPE_UNLOCK(rpipe); 582 pipespace(rpipe, SMALL_PIPE_SIZE); 583 PIPE_LOCK(rpipe); 584 } 585 } 586 587 while (uio->uio_resid) { 588 /* 589 * normal pipe buffer receive 590 */ 591 if (rpipe->pipe_buffer.cnt > 0) { 592 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 593 if (size > rpipe->pipe_buffer.cnt) 594 size = rpipe->pipe_buffer.cnt; 595 if (size > (u_int) uio->uio_resid) 596 size = (u_int) uio->uio_resid; 597 598 PIPE_UNLOCK(rpipe); 599 error = uiomove( 600 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 601 size, uio); 602 PIPE_LOCK(rpipe); 603 if (error) 604 break; 605 606 rpipe->pipe_buffer.out += size; 607 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 608 rpipe->pipe_buffer.out = 0; 609 610 rpipe->pipe_buffer.cnt -= size; 611 612 /* 613 * If there is no more to read in the pipe, reset 614 * its pointers to the beginning. This improves 615 * cache hit stats. 616 */ 617 if (rpipe->pipe_buffer.cnt == 0) { 618 rpipe->pipe_buffer.in = 0; 619 rpipe->pipe_buffer.out = 0; 620 } 621 nread += size; 622#ifndef PIPE_NODIRECT 623 /* 624 * Direct copy, bypassing a kernel buffer. 625 */ 626 } else if ((size = rpipe->pipe_map.cnt) && 627 (rpipe->pipe_state & PIPE_DIRECTW)) { 628 if (size > (u_int) uio->uio_resid) 629 size = (u_int) uio->uio_resid; 630 631 PIPE_UNLOCK(rpipe); 632 error = uiomove_fromphys(rpipe->pipe_map.ms, 633 rpipe->pipe_map.pos, size, uio); 634 PIPE_LOCK(rpipe); 635 if (error) 636 break; 637 nread += size; 638 rpipe->pipe_map.pos += size; 639 rpipe->pipe_map.cnt -= size; 640 if (rpipe->pipe_map.cnt == 0) { 641 rpipe->pipe_state &= ~PIPE_DIRECTW; 642 wakeup(rpipe); 643 } 644#endif 645 } else { 646 /* 647 * detect EOF condition 648 * read returns 0 on EOF, no need to set error 649 */ 650 if (rpipe->pipe_state & PIPE_EOF) 651 break; 652 653 /* 654 * If the "write-side" has been blocked, wake it up now. 655 */ 656 if (rpipe->pipe_state & PIPE_WANTW) { 657 rpipe->pipe_state &= ~PIPE_WANTW; 658 wakeup(rpipe); 659 } 660 661 /* 662 * Break if some data was read. 663 */ 664 if (nread > 0) 665 break; 666 667 /* 668 * Unlock the pipe buffer for our remaining processing. 669 * We will either break out with an error or we will 670 * sleep and relock to loop. 671 */ 672 pipeunlock(rpipe); 673 674 /* 675 * Handle non-blocking mode operation or 676 * wait for more data. 677 */ 678 if (fp->f_flag & FNONBLOCK) { 679 error = EAGAIN; 680 } else { 681 rpipe->pipe_state |= PIPE_WANTR; 682 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 683 PRIBIO | PCATCH, 684 "piperd", 0)) == 0) 685 error = pipelock(rpipe, 1); 686 } 687 if (error) 688 goto unlocked_error; 689 } 690 } 691#ifdef MAC 692locked_error: 693#endif 694 pipeunlock(rpipe); 695 696 /* XXX: should probably do this before getting any locks. */ 697 if (error == 0) 698 vfs_timestamp(&rpipe->pipe_atime); 699unlocked_error: 700 --rpipe->pipe_busy; 701 702 /* 703 * PIPE_WANT processing only makes sense if pipe_busy is 0. 704 */ 705 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 706 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 707 wakeup(rpipe); 708 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 709 /* 710 * Handle write blocking hysteresis. 711 */ 712 if (rpipe->pipe_state & PIPE_WANTW) { 713 rpipe->pipe_state &= ~PIPE_WANTW; 714 wakeup(rpipe); 715 } 716 } 717 718 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 719 pipeselwakeup(rpipe); 720 721 PIPE_UNLOCK(rpipe); 722 return (error); 723} 724 725#ifndef PIPE_NODIRECT 726/* 727 * Map the sending processes' buffer into kernel space and wire it. 728 * This is similar to a physical write operation. 729 */ 730static int 731pipe_build_write_buffer(wpipe, uio) 732 struct pipe *wpipe; 733 struct uio *uio; 734{ 735 pmap_t pmap; 736 u_int size; 737 int i, j; 738 vm_offset_t addr, endaddr; 739 740 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 741 KASSERT(wpipe->pipe_state & PIPE_DIRECTW, 742 ("Clone attempt on non-direct write pipe!")); 743 744 size = (u_int) uio->uio_iov->iov_len; 745 if (size > wpipe->pipe_buffer.size) 746 size = wpipe->pipe_buffer.size; 747 748 pmap = vmspace_pmap(curproc->p_vmspace); 749 endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size); 750 addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base); 751 for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) { 752 /* 753 * vm_fault_quick() can sleep. Consequently, 754 * vm_page_lock_queue() and vm_page_unlock_queue() 755 * should not be performed outside of this loop. 756 */ 757 race: 758 if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0) { 759 vm_page_lock_queues(); 760 for (j = 0; j < i; j++) 761 vm_page_unhold(wpipe->pipe_map.ms[j]); 762 vm_page_unlock_queues(); 763 return (EFAULT); 764 } 765 wpipe->pipe_map.ms[i] = pmap_extract_and_hold(pmap, addr, 766 VM_PROT_READ); 767 if (wpipe->pipe_map.ms[i] == NULL) 768 goto race; 769 } 770 771/* 772 * set up the control block 773 */ 774 wpipe->pipe_map.npages = i; 775 wpipe->pipe_map.pos = 776 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 777 wpipe->pipe_map.cnt = size; 778 779/* 780 * and update the uio data 781 */ 782 783 uio->uio_iov->iov_len -= size; 784 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 785 if (uio->uio_iov->iov_len == 0) 786 uio->uio_iov++; 787 uio->uio_resid -= size; 788 uio->uio_offset += size; 789 return (0); 790} 791 792/* 793 * unmap and unwire the process buffer 794 */ 795static void 796pipe_destroy_write_buffer(wpipe) 797 struct pipe *wpipe; 798{ 799 int i; 800 801 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 802 vm_page_lock_queues(); 803 for (i = 0; i < wpipe->pipe_map.npages; i++) { 804 vm_page_unhold(wpipe->pipe_map.ms[i]); 805 } 806 vm_page_unlock_queues(); 807 wpipe->pipe_map.npages = 0; 808} 809 810/* 811 * In the case of a signal, the writing process might go away. This 812 * code copies the data into the circular buffer so that the source 813 * pages can be freed without loss of data. 814 */ 815static void 816pipe_clone_write_buffer(wpipe) 817 struct pipe *wpipe; 818{ 819 struct uio uio; 820 struct iovec iov; 821 int size; 822 int pos; 823 824 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 825 size = wpipe->pipe_map.cnt; 826 pos = wpipe->pipe_map.pos; 827 828 wpipe->pipe_buffer.in = size; 829 wpipe->pipe_buffer.out = 0; 830 wpipe->pipe_buffer.cnt = size; 831 wpipe->pipe_state &= ~PIPE_DIRECTW; 832 833 PIPE_UNLOCK(wpipe); 834 iov.iov_base = wpipe->pipe_buffer.buffer; 835 iov.iov_len = size; 836 uio.uio_iov = &iov; 837 uio.uio_iovcnt = 1; 838 uio.uio_offset = 0; 839 uio.uio_resid = size; 840 uio.uio_segflg = UIO_SYSSPACE; 841 uio.uio_rw = UIO_READ; 842 uio.uio_td = curthread; 843 uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio); 844 PIPE_LOCK(wpipe); 845 pipe_destroy_write_buffer(wpipe); 846} 847 848/* 849 * This implements the pipe buffer write mechanism. Note that only 850 * a direct write OR a normal pipe write can be pending at any given time. 851 * If there are any characters in the pipe buffer, the direct write will 852 * be deferred until the receiving process grabs all of the bytes from 853 * the pipe buffer. Then the direct mapping write is set-up. 854 */ 855static int 856pipe_direct_write(wpipe, uio) 857 struct pipe *wpipe; 858 struct uio *uio; 859{ 860 int error; 861 862retry: 863 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 864 error = pipelock(wpipe, 1); 865 if (wpipe->pipe_state & PIPE_EOF) 866 error = EPIPE; 867 if (error) { 868 pipeunlock(wpipe); 869 goto error1; 870 } 871 while (wpipe->pipe_state & PIPE_DIRECTW) { 872 if (wpipe->pipe_state & PIPE_WANTR) { 873 wpipe->pipe_state &= ~PIPE_WANTR; 874 wakeup(wpipe); 875 } 876 pipeselwakeup(wpipe); 877 wpipe->pipe_state |= PIPE_WANTW; 878 pipeunlock(wpipe); 879 error = msleep(wpipe, PIPE_MTX(wpipe), 880 PRIBIO | PCATCH, "pipdww", 0); 881 if (error) 882 goto error1; 883 else 884 goto retry; 885 } 886 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 887 if (wpipe->pipe_buffer.cnt > 0) { 888 if (wpipe->pipe_state & PIPE_WANTR) { 889 wpipe->pipe_state &= ~PIPE_WANTR; 890 wakeup(wpipe); 891 } 892 pipeselwakeup(wpipe); 893 wpipe->pipe_state |= PIPE_WANTW; 894 pipeunlock(wpipe); 895 error = msleep(wpipe, PIPE_MTX(wpipe), 896 PRIBIO | PCATCH, "pipdwc", 0); 897 if (error) 898 goto error1; 899 else 900 goto retry; 901 } 902 903 wpipe->pipe_state |= PIPE_DIRECTW; 904 905 PIPE_UNLOCK(wpipe); 906 error = pipe_build_write_buffer(wpipe, uio); 907 PIPE_LOCK(wpipe); 908 if (error) { 909 wpipe->pipe_state &= ~PIPE_DIRECTW; 910 pipeunlock(wpipe); 911 goto error1; 912 } 913 914 error = 0; 915 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 916 if (wpipe->pipe_state & PIPE_EOF) { 917 pipe_destroy_write_buffer(wpipe); 918 pipeselwakeup(wpipe); 919 pipeunlock(wpipe); 920 error = EPIPE; 921 goto error1; 922 } 923 if (wpipe->pipe_state & PIPE_WANTR) { 924 wpipe->pipe_state &= ~PIPE_WANTR; 925 wakeup(wpipe); 926 } 927 pipeselwakeup(wpipe); 928 pipeunlock(wpipe); 929 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 930 "pipdwt", 0); 931 pipelock(wpipe, 0); 932 } 933 934 if (wpipe->pipe_state & PIPE_EOF) 935 error = EPIPE; 936 if (wpipe->pipe_state & PIPE_DIRECTW) { 937 /* 938 * this bit of trickery substitutes a kernel buffer for 939 * the process that might be going away. 940 */ 941 pipe_clone_write_buffer(wpipe); 942 } else { 943 pipe_destroy_write_buffer(wpipe); 944 } 945 pipeunlock(wpipe); 946 return (error); 947 948error1: 949 wakeup(wpipe); 950 return (error); 951} 952#endif 953 954static int 955pipe_write(fp, uio, active_cred, flags, td) 956 struct file *fp; 957 struct uio *uio; 958 struct ucred *active_cred; 959 struct thread *td; 960 int flags; 961{ 962 int error = 0; 963 int desiredsize, orig_resid; 964 struct pipe *wpipe, *rpipe; 965 966 rpipe = fp->f_data; 967 wpipe = rpipe->pipe_peer; 968 969 PIPE_LOCK(rpipe); 970 error = pipelock(wpipe, 1); 971 if (error) { 972 PIPE_UNLOCK(rpipe); 973 return (error); 974 } 975 /* 976 * detect loss of pipe read side, issue SIGPIPE if lost. 977 */ 978 if ((!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 979 pipeunlock(wpipe); 980 PIPE_UNLOCK(rpipe); 981 return (EPIPE); 982 } 983#ifdef MAC 984 error = mac_pipe_check_write(active_cred, wpipe->pipe_pair); 985 if (error) { 986 pipeunlock(wpipe); 987 PIPE_UNLOCK(rpipe); 988 return (error); 989 } 990#endif 991 ++wpipe->pipe_busy; 992 993 /* Choose a larger size if it's advantageous */ 994 desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size); 995 while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) { 996 if (piperesizeallowed != 1) 997 break; 998 if (amountpipekva > maxpipekva / 2) 999 break; 1000 if (desiredsize == BIG_PIPE_SIZE) 1001 break; 1002 desiredsize = desiredsize * 2; 1003 } 1004 1005 /* Choose a smaller size if we're in a OOM situation */ 1006 if ((amountpipekva > (3 * maxpipekva) / 4) && 1007 (wpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 1008 (wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 1009 (piperesizeallowed == 1)) 1010 desiredsize = SMALL_PIPE_SIZE; 1011 1012 /* Resize if the above determined that a new size was necessary */ 1013 if ((desiredsize != wpipe->pipe_buffer.size) && 1014 ((wpipe->pipe_state & PIPE_DIRECTW) == 0)) { 1015 PIPE_UNLOCK(wpipe); 1016 pipespace(wpipe, desiredsize); 1017 PIPE_LOCK(wpipe); 1018 } 1019 if (wpipe->pipe_buffer.size == 0) { 1020 /* 1021 * This can only happen for reverse direction use of pipes 1022 * in a complete OOM situation. 1023 */ 1024 error = ENOMEM; 1025 --wpipe->pipe_busy; 1026 pipeunlock(wpipe); 1027 PIPE_UNLOCK(wpipe); 1028 return (error); 1029 } 1030 1031 pipeunlock(wpipe); 1032 1033 orig_resid = uio->uio_resid; 1034 1035 while (uio->uio_resid) { 1036 int space; 1037 1038 pipelock(wpipe, 0); 1039 if (wpipe->pipe_state & PIPE_EOF) { 1040 pipeunlock(wpipe); 1041 error = EPIPE; 1042 break; 1043 } 1044#ifndef PIPE_NODIRECT 1045 /* 1046 * If the transfer is large, we can gain performance if 1047 * we do process-to-process copies directly. 1048 * If the write is non-blocking, we don't use the 1049 * direct write mechanism. 1050 * 1051 * The direct write mechanism will detect the reader going 1052 * away on us. 1053 */ 1054 if (uio->uio_segflg == UIO_USERSPACE && 1055 uio->uio_iov->iov_len >= PIPE_MINDIRECT && 1056 wpipe->pipe_buffer.size >= PIPE_MINDIRECT && 1057 (fp->f_flag & FNONBLOCK) == 0) { 1058 pipeunlock(wpipe); 1059 error = pipe_direct_write(wpipe, uio); 1060 if (error) 1061 break; 1062 continue; 1063 } 1064#endif 1065 1066 /* 1067 * Pipe buffered writes cannot be coincidental with 1068 * direct writes. We wait until the currently executing 1069 * direct write is completed before we start filling the 1070 * pipe buffer. We break out if a signal occurs or the 1071 * reader goes away. 1072 */ 1073 if (wpipe->pipe_state & PIPE_DIRECTW) { 1074 if (wpipe->pipe_state & PIPE_WANTR) { 1075 wpipe->pipe_state &= ~PIPE_WANTR; 1076 wakeup(wpipe); 1077 } 1078 pipeselwakeup(wpipe); 1079 wpipe->pipe_state |= PIPE_WANTW; 1080 pipeunlock(wpipe); 1081 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1082 "pipbww", 0); 1083 if (error) 1084 break; 1085 else 1086 continue; 1087 } 1088 1089 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1090 1091 /* Writes of size <= PIPE_BUF must be atomic. */ 1092 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1093 space = 0; 1094 1095 if (space > 0) { 1096 int size; /* Transfer size */ 1097 int segsize; /* first segment to transfer */ 1098 1099 /* 1100 * Transfer size is minimum of uio transfer 1101 * and free space in pipe buffer. 1102 */ 1103 if (space > uio->uio_resid) 1104 size = uio->uio_resid; 1105 else 1106 size = space; 1107 /* 1108 * First segment to transfer is minimum of 1109 * transfer size and contiguous space in 1110 * pipe buffer. If first segment to transfer 1111 * is less than the transfer size, we've got 1112 * a wraparound in the buffer. 1113 */ 1114 segsize = wpipe->pipe_buffer.size - 1115 wpipe->pipe_buffer.in; 1116 if (segsize > size) 1117 segsize = size; 1118 1119 /* Transfer first segment */ 1120 1121 PIPE_UNLOCK(rpipe); 1122 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1123 segsize, uio); 1124 PIPE_LOCK(rpipe); 1125 1126 if (error == 0 && segsize < size) { 1127 KASSERT(wpipe->pipe_buffer.in + segsize == 1128 wpipe->pipe_buffer.size, 1129 ("Pipe buffer wraparound disappeared")); 1130 /* 1131 * Transfer remaining part now, to 1132 * support atomic writes. Wraparound 1133 * happened. 1134 */ 1135 1136 PIPE_UNLOCK(rpipe); 1137 error = uiomove( 1138 &wpipe->pipe_buffer.buffer[0], 1139 size - segsize, uio); 1140 PIPE_LOCK(rpipe); 1141 } 1142 if (error == 0) { 1143 wpipe->pipe_buffer.in += size; 1144 if (wpipe->pipe_buffer.in >= 1145 wpipe->pipe_buffer.size) { 1146 KASSERT(wpipe->pipe_buffer.in == 1147 size - segsize + 1148 wpipe->pipe_buffer.size, 1149 ("Expected wraparound bad")); 1150 wpipe->pipe_buffer.in = size - segsize; 1151 } 1152 1153 wpipe->pipe_buffer.cnt += size; 1154 KASSERT(wpipe->pipe_buffer.cnt <= 1155 wpipe->pipe_buffer.size, 1156 ("Pipe buffer overflow")); 1157 } 1158 pipeunlock(wpipe); 1159 if (error != 0) 1160 break; 1161 } else { 1162 /* 1163 * If the "read-side" has been blocked, wake it up now. 1164 */ 1165 if (wpipe->pipe_state & PIPE_WANTR) { 1166 wpipe->pipe_state &= ~PIPE_WANTR; 1167 wakeup(wpipe); 1168 } 1169 1170 /* 1171 * don't block on non-blocking I/O 1172 */ 1173 if (fp->f_flag & FNONBLOCK) { 1174 error = EAGAIN; 1175 pipeunlock(wpipe); 1176 break; 1177 } 1178 1179 /* 1180 * We have no more space and have something to offer, 1181 * wake up select/poll. 1182 */ 1183 pipeselwakeup(wpipe); 1184 1185 wpipe->pipe_state |= PIPE_WANTW; 1186 pipeunlock(wpipe); 1187 error = msleep(wpipe, PIPE_MTX(rpipe), 1188 PRIBIO | PCATCH, "pipewr", 0); 1189 if (error != 0) 1190 break; 1191 } 1192 } 1193 1194 pipelock(wpipe, 0); 1195 --wpipe->pipe_busy; 1196 1197 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1198 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1199 wakeup(wpipe); 1200 } else if (wpipe->pipe_buffer.cnt > 0) { 1201 /* 1202 * If we have put any characters in the buffer, we wake up 1203 * the reader. 1204 */ 1205 if (wpipe->pipe_state & PIPE_WANTR) { 1206 wpipe->pipe_state &= ~PIPE_WANTR; 1207 wakeup(wpipe); 1208 } 1209 } 1210 1211 /* 1212 * Don't return EPIPE if I/O was successful 1213 */ 1214 if ((wpipe->pipe_buffer.cnt == 0) && 1215 (uio->uio_resid == 0) && 1216 (error == EPIPE)) { 1217 error = 0; 1218 } 1219 1220 if (error == 0) 1221 vfs_timestamp(&wpipe->pipe_mtime); 1222 1223 /* 1224 * We have something to offer, 1225 * wake up select/poll. 1226 */ 1227 if (wpipe->pipe_buffer.cnt) 1228 pipeselwakeup(wpipe); 1229 1230 pipeunlock(wpipe); 1231 PIPE_UNLOCK(rpipe); 1232 return (error); 1233} 1234 1235/* ARGSUSED */ 1236static int 1237pipe_truncate(fp, length, active_cred, td) 1238 struct file *fp; 1239 off_t length; 1240 struct ucred *active_cred; 1241 struct thread *td; 1242{ 1243 1244 return (EINVAL); 1245} 1246 1247/* 1248 * we implement a very minimal set of ioctls for compatibility with sockets. 1249 */ 1250static int 1251pipe_ioctl(fp, cmd, data, active_cred, td) 1252 struct file *fp; 1253 u_long cmd; 1254 void *data; 1255 struct ucred *active_cred; 1256 struct thread *td; 1257{ 1258 struct pipe *mpipe = fp->f_data; 1259 int error; 1260 1261 PIPE_LOCK(mpipe); 1262 1263#ifdef MAC 1264 error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data); 1265 if (error) { 1266 PIPE_UNLOCK(mpipe); 1267 return (error); 1268 } 1269#endif 1270 1271 error = 0; 1272 switch (cmd) { 1273 1274 case FIONBIO: 1275 break; 1276 1277 case FIOASYNC: 1278 if (*(int *)data) { 1279 mpipe->pipe_state |= PIPE_ASYNC; 1280 } else { 1281 mpipe->pipe_state &= ~PIPE_ASYNC; 1282 } 1283 break; 1284 1285 case FIONREAD: 1286 if (mpipe->pipe_state & PIPE_DIRECTW) 1287 *(int *)data = mpipe->pipe_map.cnt; 1288 else 1289 *(int *)data = mpipe->pipe_buffer.cnt; 1290 break; 1291 1292 case FIOSETOWN: 1293 PIPE_UNLOCK(mpipe); 1294 error = fsetown(*(int *)data, &mpipe->pipe_sigio); 1295 goto out_unlocked; 1296 1297 case FIOGETOWN: 1298 *(int *)data = fgetown(&mpipe->pipe_sigio); 1299 break; 1300 1301 /* This is deprecated, FIOSETOWN should be used instead. */ 1302 case TIOCSPGRP: 1303 PIPE_UNLOCK(mpipe); 1304 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio); 1305 goto out_unlocked; 1306 1307 /* This is deprecated, FIOGETOWN should be used instead. */ 1308 case TIOCGPGRP: 1309 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1310 break; 1311 1312 default: 1313 error = ENOTTY; 1314 break; 1315 } 1316 PIPE_UNLOCK(mpipe); 1317out_unlocked: 1318 return (error); 1319} 1320 1321static int 1322pipe_poll(fp, events, active_cred, td) 1323 struct file *fp; 1324 int events; 1325 struct ucred *active_cred; 1326 struct thread *td; 1327{ 1328 struct pipe *rpipe = fp->f_data; 1329 struct pipe *wpipe; 1330 int revents = 0; 1331#ifdef MAC 1332 int error; 1333#endif 1334 1335 wpipe = rpipe->pipe_peer; 1336 PIPE_LOCK(rpipe); 1337#ifdef MAC 1338 error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair); 1339 if (error) 1340 goto locked_error; 1341#endif 1342 if (events & (POLLIN | POLLRDNORM)) 1343 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1344 (rpipe->pipe_buffer.cnt > 0) || 1345 (rpipe->pipe_state & PIPE_EOF)) 1346 revents |= events & (POLLIN | POLLRDNORM); 1347 1348 if (events & (POLLOUT | POLLWRNORM)) 1349 if (!wpipe->pipe_present || (wpipe->pipe_state & PIPE_EOF) || 1350 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1351 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1352 revents |= events & (POLLOUT | POLLWRNORM); 1353 1354 if ((rpipe->pipe_state & PIPE_EOF) || 1355 (!wpipe->pipe_present) || 1356 (wpipe->pipe_state & PIPE_EOF)) 1357 revents |= POLLHUP; 1358 1359 if (revents == 0) { 1360 if (events & (POLLIN | POLLRDNORM)) { 1361 selrecord(td, &rpipe->pipe_sel); 1362 if (SEL_WAITING(&rpipe->pipe_sel)) 1363 rpipe->pipe_state |= PIPE_SEL; 1364 } 1365 1366 if (events & (POLLOUT | POLLWRNORM)) { 1367 selrecord(td, &wpipe->pipe_sel); 1368 if (SEL_WAITING(&wpipe->pipe_sel)) 1369 wpipe->pipe_state |= PIPE_SEL; 1370 } 1371 } 1372#ifdef MAC 1373locked_error: 1374#endif 1375 PIPE_UNLOCK(rpipe); 1376 1377 return (revents); 1378} 1379 1380/* 1381 * We shouldn't need locks here as we're doing a read and this should 1382 * be a natural race. 1383 */ 1384static int 1385pipe_stat(fp, ub, active_cred, td) 1386 struct file *fp; 1387 struct stat *ub; 1388 struct ucred *active_cred; 1389 struct thread *td; 1390{ 1391 struct pipe *pipe = fp->f_data; 1392#ifdef MAC 1393 int error; 1394 1395 PIPE_LOCK(pipe); 1396 error = mac_pipe_check_stat(active_cred, pipe->pipe_pair); 1397 PIPE_UNLOCK(pipe); 1398 if (error) 1399 return (error); 1400#endif 1401 bzero(ub, sizeof(*ub)); 1402 ub->st_mode = S_IFIFO; 1403 ub->st_blksize = PAGE_SIZE; 1404 if (pipe->pipe_state & PIPE_DIRECTW) 1405 ub->st_size = pipe->pipe_map.cnt; 1406 else 1407 ub->st_size = pipe->pipe_buffer.cnt; 1408 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1409 ub->st_atimespec = pipe->pipe_atime; 1410 ub->st_mtimespec = pipe->pipe_mtime; 1411 ub->st_ctimespec = pipe->pipe_ctime; 1412 ub->st_uid = fp->f_cred->cr_uid; 1413 ub->st_gid = fp->f_cred->cr_gid; 1414 /* 1415 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1416 * XXX (st_dev, st_ino) should be unique. 1417 */ 1418 return (0); 1419} 1420 1421/* ARGSUSED */ 1422static int 1423pipe_close(fp, td) 1424 struct file *fp; 1425 struct thread *td; 1426{ 1427 struct pipe *cpipe = fp->f_data; 1428 1429 fp->f_ops = &badfileops; 1430 fp->f_data = NULL; 1431 funsetown(&cpipe->pipe_sigio); 1432 pipeclose(cpipe); 1433 return (0); 1434} 1435 1436static void 1437pipe_free_kmem(cpipe) 1438 struct pipe *cpipe; 1439{ 1440 1441 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), 1442 ("pipe_free_kmem: pipe mutex locked")); 1443 1444 if (cpipe->pipe_buffer.buffer != NULL) { 1445 atomic_subtract_int(&amountpipekva, cpipe->pipe_buffer.size); 1446 vm_map_remove(pipe_map, 1447 (vm_offset_t)cpipe->pipe_buffer.buffer, 1448 (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size); 1449 cpipe->pipe_buffer.buffer = NULL; 1450 } 1451#ifndef PIPE_NODIRECT 1452 { 1453 cpipe->pipe_map.cnt = 0; 1454 cpipe->pipe_map.pos = 0; 1455 cpipe->pipe_map.npages = 0; 1456 } 1457#endif 1458} 1459 1460/* 1461 * shutdown the pipe 1462 */ 1463static void 1464pipeclose(cpipe) 1465 struct pipe *cpipe; 1466{ 1467 struct pipepair *pp; 1468 struct pipe *ppipe; 1469 1470 KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL")); 1471 1472 PIPE_LOCK(cpipe); 1473 pipelock(cpipe, 0); 1474 pp = cpipe->pipe_pair; 1475 1476 pipeselwakeup(cpipe); 1477 1478 /* 1479 * If the other side is blocked, wake it up saying that 1480 * we want to close it down. 1481 */ 1482 cpipe->pipe_state |= PIPE_EOF; 1483 while (cpipe->pipe_busy) { 1484 wakeup(cpipe); 1485 cpipe->pipe_state |= PIPE_WANT; 1486 pipeunlock(cpipe); 1487 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1488 pipelock(cpipe, 0); 1489 } 1490 1491 1492 /* 1493 * Disconnect from peer, if any. 1494 */ 1495 ppipe = cpipe->pipe_peer; 1496 if (ppipe->pipe_present != 0) { 1497 pipeselwakeup(ppipe); 1498 1499 ppipe->pipe_state |= PIPE_EOF; 1500 wakeup(ppipe); 1501 KNOTE_LOCKED(&ppipe->pipe_sel.si_note, 0); 1502 } 1503 1504 /* 1505 * Mark this endpoint as free. Release kmem resources. We 1506 * don't mark this endpoint as unused until we've finished 1507 * doing that, or the pipe might disappear out from under 1508 * us. 1509 */ 1510 PIPE_UNLOCK(cpipe); 1511 pipe_free_kmem(cpipe); 1512 PIPE_LOCK(cpipe); 1513 cpipe->pipe_present = 0; 1514 pipeunlock(cpipe); 1515 knlist_clear(&cpipe->pipe_sel.si_note, 1); 1516 knlist_destroy(&cpipe->pipe_sel.si_note); 1517 1518 /* 1519 * If both endpoints are now closed, release the memory for the 1520 * pipe pair. If not, unlock. 1521 */ 1522 if (ppipe->pipe_present == 0) { 1523 PIPE_UNLOCK(cpipe); 1524#ifdef MAC 1525 mac_pipe_destroy(pp); 1526#endif 1527 uma_zfree(pipe_zone, cpipe->pipe_pair); 1528 } else 1529 PIPE_UNLOCK(cpipe); 1530} 1531 1532/*ARGSUSED*/ 1533static int 1534pipe_kqfilter(struct file *fp, struct knote *kn) 1535{ 1536 struct pipe *cpipe; 1537 1538 cpipe = kn->kn_fp->f_data; 1539 PIPE_LOCK(cpipe); 1540 switch (kn->kn_filter) { 1541 case EVFILT_READ: 1542 kn->kn_fop = &pipe_rfiltops; 1543 break; 1544 case EVFILT_WRITE: 1545 kn->kn_fop = &pipe_wfiltops; 1546 if (!cpipe->pipe_peer->pipe_present) { 1547 /* other end of pipe has been closed */ 1548 PIPE_UNLOCK(cpipe); 1549 return (EPIPE); 1550 } 1551 cpipe = cpipe->pipe_peer; 1552 break; 1553 default: 1554 PIPE_UNLOCK(cpipe); 1555 return (EINVAL); 1556 } 1557 1558 knlist_add(&cpipe->pipe_sel.si_note, kn, 1); 1559 PIPE_UNLOCK(cpipe); 1560 return (0); 1561} 1562 1563static void 1564filt_pipedetach(struct knote *kn) 1565{ 1566 struct pipe *cpipe = (struct pipe *)kn->kn_fp->f_data; 1567 1568 PIPE_LOCK(cpipe); 1569 if (kn->kn_filter == EVFILT_WRITE) 1570 cpipe = cpipe->pipe_peer; 1571 knlist_remove(&cpipe->pipe_sel.si_note, kn, 1); 1572 PIPE_UNLOCK(cpipe); 1573} 1574 1575/*ARGSUSED*/ 1576static int 1577filt_piperead(struct knote *kn, long hint) 1578{ 1579 struct pipe *rpipe = kn->kn_fp->f_data; 1580 struct pipe *wpipe = rpipe->pipe_peer; 1581 int ret; 1582 1583 PIPE_LOCK(rpipe); 1584 kn->kn_data = rpipe->pipe_buffer.cnt; 1585 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1586 kn->kn_data = rpipe->pipe_map.cnt; 1587 1588 if ((rpipe->pipe_state & PIPE_EOF) || 1589 (!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 1590 kn->kn_flags |= EV_EOF; 1591 PIPE_UNLOCK(rpipe); 1592 return (1); 1593 } 1594 ret = kn->kn_data > 0; 1595 PIPE_UNLOCK(rpipe); 1596 return ret; 1597} 1598 1599/*ARGSUSED*/ 1600static int 1601filt_pipewrite(struct knote *kn, long hint) 1602{ 1603 struct pipe *rpipe = kn->kn_fp->f_data; 1604 struct pipe *wpipe = rpipe->pipe_peer; 1605 1606 PIPE_LOCK(rpipe); 1607 if ((!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 1608 kn->kn_data = 0; 1609 kn->kn_flags |= EV_EOF; 1610 PIPE_UNLOCK(rpipe); 1611 return (1); 1612 } 1613 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1614 if (wpipe->pipe_state & PIPE_DIRECTW) 1615 kn->kn_data = 0; 1616 1617 PIPE_UNLOCK(rpipe); 1618 return (kn->kn_data >= PIPE_BUF); 1619} 1620