sys_pipe.c revision 232183
1139823Simp/*- 21541Srgrimes * Copyright (c) 1996 John S. Dyson 31541Srgrimes * Copyright (c) 2012 Giovanni Trematerra 41541Srgrimes * All rights reserved. 51541Srgrimes * 61541Srgrimes * Redistribution and use in source and binary forms, with or without 71541Srgrimes * modification, are permitted provided that the following conditions 81541Srgrimes * are met: 91541Srgrimes * 1. Redistributions of source code must retain the above copyright 101541Srgrimes * notice immediately at the beginning of the file, without modification, 111541Srgrimes * this list of conditions, and the following disclaimer. 121541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 131541Srgrimes * notice, this list of conditions and the following disclaimer in the 141541Srgrimes * documentation and/or other materials provided with the distribution. 151541Srgrimes * 3. Absolutely no warranty of function or purpose is made by the author 161541Srgrimes * John S. Dyson. 171541Srgrimes * 4. Modifications may be freely made to this file if the above conditions 181541Srgrimes * are met. 191541Srgrimes */ 201541Srgrimes 211541Srgrimes/* 221541Srgrimes * This file contains a high-performance replacement for the socket-based 231541Srgrimes * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 241541Srgrimes * all features of sockets, but does do everything that pipes normally 251541Srgrimes * do. 261541Srgrimes */ 271541Srgrimes 281541Srgrimes/* 2975175Syar * This code has two modes of operation, a small write mode and a large 3050477Speter * write mode. The small write mode acts like conventional pipes with 3175176Syar * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 321541Srgrimes * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 331541Srgrimes * and PIPE_SIZE in size, the sending process pins the underlying pages in 342168Spaul * memory, and the receiving process copies directly from these pinned pages 352168Spaul * in the sending process. 362168Spaul * 371541Srgrimes * If the sending process receives a signal, it is possible that it will 381541Srgrimes * go away, and certainly its address space can change, because control 3975175Syar * is returned back to the user-mode side. In that case, the pipe code 4063954Sasmodai * arranges to copy the buffer supplied by the user process, to a pageable 4163954Sasmodai * kernel buffer, and the receiving process will grab the data from the 42146702Sbrooks * pageable kernel buffer. Since signals don't happen all that often, 431541Srgrimes * the copy operation is normally eliminated. 441541Srgrimes * 451541Srgrimes * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 461541Srgrimes * happen for small transfers so that the system will not spend all of 471541Srgrimes * its time context switching. 481541Srgrimes * 491541Srgrimes * In order to limit the resource use of pipes, two sysctls exist: 5075175Syar * 5175175Syar * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable 521541Srgrimes * address space available to us in pipe_map. This value is normally 531541Srgrimes * autotuned, but may also be loader tuned. 541541Srgrimes * 551541Srgrimes * kern.ipc.pipekva - This read-only sysctl tracks the current amount of 561541Srgrimes * memory in use by pipes. 576699Sdg * 581541Srgrimes * Based on how large pipekva is relative to maxpipekva, the following 591541Srgrimes * will happen: 601541Srgrimes * 611541Srgrimes * 0% - 50%: 621541Srgrimes * New pipes are given 16K of memory backing, pipes may dynamically 631541Srgrimes * grow to as large as 64K where needed. 641541Srgrimes * 50% - 75%: 651541Srgrimes * New pipes are given 4K (or PAGE_SIZE) of memory backing, 661541Srgrimes * existing pipes may NOT grow. 671541Srgrimes * 75% - 100%: 681541Srgrimes * New pipes are given 4K (or PAGE_SIZE) of memory backing, 691541Srgrimes * existing pipes will be shrunk down to 4K whenever possible. 701541Srgrimes * 711541Srgrimes * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0. If 721541Srgrimes * that is set, the only resize that will occur is the 0 -> SMALL_PIPE_SIZE 731541Srgrimes * resize which MUST occur for reverse-direction pipes when they are 741541Srgrimes * first used. 751541Srgrimes * 761541Srgrimes * Additional information about the current state of pipes may be obtained 771541Srgrimes * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail, 781541Srgrimes * and kern.ipc.piperesizefail. 791541Srgrimes * 801541Srgrimes * Locking rules: There are two locks present here: A mutex, used via 811541Srgrimes * PIPE_LOCK, and a flag, used via pipelock(). All locking is done via 821541Srgrimes * the flag, as mutexes can not persist over uiomove. The mutex 831541Srgrimes * exists only to guard access to the flag, and is not in itself a 841541Srgrimes * locking mechanism. Also note that there is only a single mutex for 851541Srgrimes * both directions of a pipe. 861541Srgrimes * 871541Srgrimes * As pipelock() may have to sleep before it can acquire the flag, it 881541Srgrimes * is important to reread all data after a call to pipelock(); everything 891541Srgrimes * in the structure may have changed. 901541Srgrimes */ 911541Srgrimes 921541Srgrimes#include <sys/cdefs.h> 931541Srgrimes__FBSDID("$FreeBSD: head/sys/kern/sys_pipe.c 232183 2012-02-26 15:14:29Z jilles $"); 941541Srgrimes 951541Srgrimes#include <sys/param.h> 961541Srgrimes#include <sys/systm.h> 971541Srgrimes#include <sys/conf.h> 981541Srgrimes#include <sys/fcntl.h> 9975176Syar#include <sys/file.h> 10075176Syar#include <sys/filedesc.h> 10175176Syar#include <sys/filio.h> 10275176Syar#include <sys/kernel.h> 10375176Syar#include <sys/lock.h> 10475176Syar#include <sys/mutex.h> 10575176Syar#include <sys/ttycom.h> 10675176Syar#include <sys/stat.h> 10775176Syar#include <sys/malloc.h> 10875176Syar#include <sys/poll.h> 10975176Syar#include <sys/selinfo.h> 11075176Syar#include <sys/signalvar.h> 11175176Syar#include <sys/syscallsubr.h> 11275176Syar#include <sys/sysctl.h> 11375176Syar#include <sys/sysproto.h> 11475176Syar#include <sys/pipe.h> 11575176Syar#include <sys/proc.h> 11675176Syar#include <sys/vnode.h> 11775176Syar#include <sys/uio.h> 11875176Syar#include <sys/event.h> 11975176Syar 12075176Syar#include <security/mac/mac_framework.h> 12175176Syar 12275176Syar#include <vm/vm.h> 12375176Syar#include <vm/vm_param.h> 12475176Syar#include <vm/vm_object.h> 12575176Syar#include <vm/vm_kern.h> 12675176Syar#include <vm/vm_extern.h> 12775176Syar#include <vm/pmap.h> 12875176Syar#include <vm/vm_map.h> 12975176Syar#include <vm/vm_page.h> 13075176Syar#include <vm/uma.h> 13175176Syar 13275176Syar#include <fs/fifofs/fifo.h> 13375176Syar 13475176Syar/* 13575176Syar * Use this define if you want to disable *fancy* VM things. Expect an 13675176Syar * approx 30% decrease in transfer rate. This could be useful for 13775176Syar * NetBSD or OpenBSD. 13875176Syar */ 13975176Syar/* #define PIPE_NODIRECT */ 14075176Syar 14175176Syar#define PIPE_PEER(pipe) \ 14275176Syar (((pipe)->pipe_state & PIPE_NAMED) ? (pipe) : ((pipe)->pipe_peer)) 14375176Syar 14475176Syar/* 14575176Syar * interfaces to the outside world 14675176Syar */ 14775176Syarstatic fo_rdwr_t pipe_read; 14875176Syarstatic fo_rdwr_t pipe_write; 14975176Syarstatic fo_truncate_t pipe_truncate; 15075176Syarstatic fo_ioctl_t pipe_ioctl; 15175176Syarstatic fo_poll_t pipe_poll; 15275176Syarstatic fo_kqfilter_t pipe_kqfilter; 15375176Syarstatic fo_stat_t pipe_stat; 15475176Syarstatic fo_close_t pipe_close; 15575176Syarstatic fo_chmod_t pipe_chmod; 15675176Syarstatic fo_chown_t pipe_chown; 15775176Syar 15875176Syarstruct fileops pipeops = { 15975176Syar .fo_read = pipe_read, 16075176Syar .fo_write = pipe_write, 16175176Syar .fo_truncate = pipe_truncate, 16275176Syar .fo_ioctl = pipe_ioctl, 16375176Syar .fo_poll = pipe_poll, 16475176Syar .fo_kqfilter = pipe_kqfilter, 16575176Syar .fo_stat = pipe_stat, 16675176Syar .fo_close = pipe_close, 16775176Syar .fo_chmod = pipe_chmod, 16875176Syar .fo_chown = pipe_chown, 16975176Syar .fo_flags = DFLAG_PASSABLE 17075176Syar}; 17175176Syar 17275176Syarstatic void filt_pipedetach(struct knote *kn); 17375176Syarstatic void filt_pipedetach_notsup(struct knote *kn); 17475176Syarstatic int filt_pipenotsup(struct knote *kn, long hint); 17575176Syarstatic int filt_piperead(struct knote *kn, long hint); 17675176Syarstatic int filt_pipewrite(struct knote *kn, long hint); 17775176Syar 17875176Syarstatic struct filterops pipe_nfiltops = { 17975176Syar .f_isfd = 1, 18075176Syar .f_detach = filt_pipedetach_notsup, 18175176Syar .f_event = filt_pipenotsup 18275176Syar}; 18375176Syarstatic struct filterops pipe_rfiltops = { 18475176Syar .f_isfd = 1, 18575176Syar .f_detach = filt_pipedetach, 18675176Syar .f_event = filt_piperead 18775176Syar}; 18875176Syarstatic struct filterops pipe_wfiltops = { 18975176Syar .f_isfd = 1, 19075176Syar .f_detach = filt_pipedetach, 19175176Syar .f_event = filt_pipewrite 19275176Syar}; 19375176Syar 19475176Syar/* 19575176Syar * Default pipe buffer size(s), this can be kind-of large now because pipe 19675176Syar * space is pageable. The pipe code will try to maintain locality of 19775176Syar * reference for performance reasons, so small amounts of outstanding I/O 19875176Syar * will not wipe the cache. 19975176Syar */ 20075176Syar#define MINPIPESIZE (PIPE_SIZE/3) 20175176Syar#define MAXPIPESIZE (2*PIPE_SIZE/3) 20275176Syar 20375176Syarstatic long amountpipekva; 20475176Syarstatic int pipefragretry; 20575176Syarstatic int pipeallocfail; 20675176Syarstatic int piperesizefail; 20775176Syarstatic int piperesizeallowed = 1; 20875176Syar 20975176SyarSYSCTL_LONG(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN, 21075176Syar &maxpipekva, 0, "Pipe KVA limit"); 21175176SyarSYSCTL_LONG(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 21275176Syar &amountpipekva, 0, "Pipe KVA usage"); 21375176SyarSYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD, 21475176Syar &pipefragretry, 0, "Pipe allocation retries due to fragmentation"); 21575176SyarSYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD, 21675176Syar &pipeallocfail, 0, "Pipe allocation failures"); 21775176SyarSYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD, 21875176Syar &piperesizefail, 0, "Pipe resize failures"); 21975176SyarSYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW, 22075176Syar &piperesizeallowed, 0, "Pipe resizing allowed"); 22175176Syar 22275176Syarstatic void pipeinit(void *dummy __unused); 22375176Syarstatic void pipeclose(struct pipe *cpipe); 22475176Syarstatic void pipe_free_kmem(struct pipe *cpipe); 22575176Syarstatic int pipe_create(struct pipe *pipe, int backing); 22675176Syarstatic int pipe_paircreate(struct thread *td, struct pipepair **p_pp); 22775176Syarstatic __inline int pipelock(struct pipe *cpipe, int catch); 22875176Syarstatic __inline void pipeunlock(struct pipe *cpipe); 22975176Syarstatic __inline void pipeselwakeup(struct pipe *cpipe); 23075176Syar#ifndef PIPE_NODIRECT 23175176Syarstatic int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 23275176Syarstatic void pipe_destroy_write_buffer(struct pipe *wpipe); 23375176Syarstatic int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 23475177Syarstatic void pipe_clone_write_buffer(struct pipe *wpipe); 23575177Syar#endif 23675177Syarstatic int pipespace(struct pipe *cpipe, int size); 23775177Syarstatic int pipespace_new(struct pipe *cpipe, int size); 23875177Syar 23975177Syarstatic int pipe_zone_ctor(void *mem, int size, void *arg, int flags); 24075177Syarstatic int pipe_zone_init(void *mem, int size, int flags); 241219819Sjeffstatic void pipe_zone_fini(void *mem, int size); 242146986Sthompsa 2432168Spaulstatic uma_zone_t pipe_zone; 244121574Sumestatic struct unrhdr *pipeino_unr; 245121574Sumestatic dev_t pipedev_ino; 24663954Sasmodai 24763954SasmodaiSYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 24863954Sasmodai 24963954Sasmodaistatic void 250159965Sthompsapipeinit(void *dummy __unused) 251126263Smlaier{ 252126263Smlaier 253170664Srwatson pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair), 25475175Syar pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini, 255 UMA_ALIGN_PTR, 0); 256 KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 257 pipeino_unr = new_unrhdr(1, INT32_MAX, NULL); 258 KASSERT(pipeino_unr != NULL, ("pipe fake inodes not initialized")); 259 pipedev_ino = devfs_alloc_cdp_inode(); 260 KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized")); 261} 262 263static int 264pipe_zone_ctor(void *mem, int size, void *arg, int flags) 265{ 266 struct pipepair *pp; 267 struct pipe *rpipe, *wpipe; 268 269 KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size")); 270 271 pp = (struct pipepair *)mem; 272 273 /* 274 * We zero both pipe endpoints to make sure all the kmem pointers 275 * are NULL, flag fields are zero'd, etc. We timestamp both 276 * endpoints with the same time. 277 */ 278 rpipe = &pp->pp_rpipe; 279 bzero(rpipe, sizeof(*rpipe)); 280 vfs_timestamp(&rpipe->pipe_ctime); 281 rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime; 282 283 wpipe = &pp->pp_wpipe; 284 bzero(wpipe, sizeof(*wpipe)); 285 wpipe->pipe_ctime = rpipe->pipe_ctime; 286 wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime; 287 288 rpipe->pipe_peer = wpipe; 289 rpipe->pipe_pair = pp; 290 wpipe->pipe_peer = rpipe; 291 wpipe->pipe_pair = pp; 292 293 /* 294 * Mark both endpoints as present; they will later get free'd 295 * one at a time. When both are free'd, then the whole pair 296 * is released. 297 */ 298 rpipe->pipe_present = PIPE_ACTIVE; 299 wpipe->pipe_present = PIPE_ACTIVE; 300 301 /* 302 * Eventually, the MAC Framework may initialize the label 303 * in ctor or init, but for now we do it elswhere to avoid 304 * blocking in ctor or init. 305 */ 306 pp->pp_label = NULL; 307 308 return (0); 309} 310 311static int 312pipe_zone_init(void *mem, int size, int flags) 313{ 314 struct pipepair *pp; 315 316 KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size")); 317 318 pp = (struct pipepair *)mem; 319 320 mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 321 return (0); 322} 323 324static void 325pipe_zone_fini(void *mem, int size) 326{ 327 struct pipepair *pp; 328 329 KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size")); 330 331 pp = (struct pipepair *)mem; 332 333 mtx_destroy(&pp->pp_mtx); 334} 335 336static int 337pipe_paircreate(struct thread *td, struct pipepair **p_pp) 338{ 339 struct pipepair *pp; 340 struct pipe *rpipe, *wpipe; 341 int error; 342 343 *p_pp = pp = uma_zalloc(pipe_zone, M_WAITOK); 344#ifdef MAC 345 /* 346 * The MAC label is shared between the connected endpoints. As a 347 * result mac_pipe_init() and mac_pipe_create() are called once 348 * for the pair, and not on the endpoints. 349 */ 350 mac_pipe_init(pp); 351 mac_pipe_create(td->td_ucred, pp); 352#endif 353 rpipe = &pp->pp_rpipe; 354 wpipe = &pp->pp_wpipe; 355 356 knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe)); 357 knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe)); 358 359 /* Only the forward direction pipe is backed by default */ 360 if ((error = pipe_create(rpipe, 1)) != 0 || 361 (error = pipe_create(wpipe, 0)) != 0) { 362 pipeclose(rpipe); 363 pipeclose(wpipe); 364 return (error); 365 } 366 367 rpipe->pipe_state |= PIPE_DIRECTOK; 368 wpipe->pipe_state |= PIPE_DIRECTOK; 369 return (0); 370} 371 372int 373pipe_named_ctor(struct pipe **ppipe, struct thread *td) 374{ 375 struct pipepair *pp; 376 int error; 377 378 error = pipe_paircreate(td, &pp); 379 if (error != 0) 380 return (error); 381 pp->pp_rpipe.pipe_state |= PIPE_NAMED; 382 *ppipe = &pp->pp_rpipe; 383 return (0); 384} 385 386void 387pipe_dtor(struct pipe *dpipe) 388{ 389 ino_t ino; 390 391 ino = dpipe->pipe_ino; 392 funsetown(&dpipe->pipe_sigio); 393 pipeclose(dpipe); 394 if (dpipe->pipe_state & PIPE_NAMED) { 395 dpipe = dpipe->pipe_peer; 396 funsetown(&dpipe->pipe_sigio); 397 pipeclose(dpipe); 398 } 399 if (ino != 0 && ino != (ino_t)-1) 400 free_unr(pipeino_unr, ino); 401} 402 403/* 404 * The pipe system call for the DTYPE_PIPE type of pipes. If we fail, let 405 * the zone pick up the pieces via pipeclose(). 406 */ 407int 408kern_pipe(struct thread *td, int fildes[2]) 409{ 410 struct filedesc *fdp; 411 struct file *rf, *wf; 412 struct pipe *rpipe, *wpipe; 413 struct pipepair *pp; 414 int fd, error; 415 416 fdp = td->td_proc->p_fd; 417 error = pipe_paircreate(td, &pp); 418 if (error != 0) 419 return (error); 420 rpipe = &pp->pp_rpipe; 421 wpipe = &pp->pp_wpipe; 422 error = falloc(td, &rf, &fd, 0); 423 if (error) { 424 pipeclose(rpipe); 425 pipeclose(wpipe); 426 return (error); 427 } 428 /* An extra reference on `rf' has been held for us by falloc(). */ 429 fildes[0] = fd; 430 431 /* 432 * Warning: once we've gotten past allocation of the fd for the 433 * read-side, we can only drop the read side via fdrop() in order 434 * to avoid races against processes which manage to dup() the read 435 * side while we are blocked trying to allocate the write side. 436 */ 437 finit(rf, FREAD | FWRITE, DTYPE_PIPE, rpipe, &pipeops); 438 error = falloc(td, &wf, &fd, 0); 439 if (error) { 440 fdclose(fdp, rf, fildes[0], td); 441 fdrop(rf, td); 442 /* rpipe has been closed by fdrop(). */ 443 pipeclose(wpipe); 444 return (error); 445 } 446 /* An extra reference on `wf' has been held for us by falloc(). */ 447 finit(wf, FREAD | FWRITE, DTYPE_PIPE, wpipe, &pipeops); 448 fdrop(wf, td); 449 fildes[1] = fd; 450 fdrop(rf, td); 451 452 return (0); 453} 454 455/* ARGSUSED */ 456int 457sys_pipe(struct thread *td, struct pipe_args *uap) 458{ 459 int error; 460 int fildes[2]; 461 462 error = kern_pipe(td, fildes); 463 if (error) 464 return (error); 465 466 td->td_retval[0] = fildes[0]; 467 td->td_retval[1] = fildes[1]; 468 469 return (0); 470} 471 472/* 473 * Allocate kva for pipe circular buffer, the space is pageable 474 * This routine will 'realloc' the size of a pipe safely, if it fails 475 * it will retain the old buffer. 476 * If it fails it will return ENOMEM. 477 */ 478static int 479pipespace_new(cpipe, size) 480 struct pipe *cpipe; 481 int size; 482{ 483 caddr_t buffer; 484 int error, cnt, firstseg; 485 static int curfail = 0; 486 static struct timeval lastfail; 487 488 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked")); 489 KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW), 490 ("pipespace: resize of direct writes not allowed")); 491retry: 492 cnt = cpipe->pipe_buffer.cnt; 493 if (cnt > size) 494 size = cnt; 495 496 size = round_page(size); 497 buffer = (caddr_t) vm_map_min(pipe_map); 498 499 error = vm_map_find(pipe_map, NULL, 0, 500 (vm_offset_t *) &buffer, size, 1, 501 VM_PROT_ALL, VM_PROT_ALL, 0); 502 if (error != KERN_SUCCESS) { 503 if ((cpipe->pipe_buffer.buffer == NULL) && 504 (size > SMALL_PIPE_SIZE)) { 505 size = SMALL_PIPE_SIZE; 506 pipefragretry++; 507 goto retry; 508 } 509 if (cpipe->pipe_buffer.buffer == NULL) { 510 pipeallocfail++; 511 if (ppsratecheck(&lastfail, &curfail, 1)) 512 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n"); 513 } else { 514 piperesizefail++; 515 } 516 return (ENOMEM); 517 } 518 519 /* copy data, then free old resources if we're resizing */ 520 if (cnt > 0) { 521 if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) { 522 firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out; 523 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 524 buffer, firstseg); 525 if ((cnt - firstseg) > 0) 526 bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg], 527 cpipe->pipe_buffer.in); 528 } else { 529 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 530 buffer, cnt); 531 } 532 } 533 pipe_free_kmem(cpipe); 534 cpipe->pipe_buffer.buffer = buffer; 535 cpipe->pipe_buffer.size = size; 536 cpipe->pipe_buffer.in = cnt; 537 cpipe->pipe_buffer.out = 0; 538 cpipe->pipe_buffer.cnt = cnt; 539 atomic_add_long(&amountpipekva, cpipe->pipe_buffer.size); 540 return (0); 541} 542 543/* 544 * Wrapper for pipespace_new() that performs locking assertions. 545 */ 546static int 547pipespace(cpipe, size) 548 struct pipe *cpipe; 549 int size; 550{ 551 552 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 553 ("Unlocked pipe passed to pipespace")); 554 return (pipespace_new(cpipe, size)); 555} 556 557/* 558 * lock a pipe for I/O, blocking other access 559 */ 560static __inline int 561pipelock(cpipe, catch) 562 struct pipe *cpipe; 563 int catch; 564{ 565 int error; 566 567 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 568 while (cpipe->pipe_state & PIPE_LOCKFL) { 569 cpipe->pipe_state |= PIPE_LWANT; 570 error = msleep(cpipe, PIPE_MTX(cpipe), 571 catch ? (PRIBIO | PCATCH) : PRIBIO, 572 "pipelk", 0); 573 if (error != 0) 574 return (error); 575 } 576 cpipe->pipe_state |= PIPE_LOCKFL; 577 return (0); 578} 579 580/* 581 * unlock a pipe I/O lock 582 */ 583static __inline void 584pipeunlock(cpipe) 585 struct pipe *cpipe; 586{ 587 588 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 589 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 590 ("Unlocked pipe passed to pipeunlock")); 591 cpipe->pipe_state &= ~PIPE_LOCKFL; 592 if (cpipe->pipe_state & PIPE_LWANT) { 593 cpipe->pipe_state &= ~PIPE_LWANT; 594 wakeup(cpipe); 595 } 596} 597 598static __inline void 599pipeselwakeup(cpipe) 600 struct pipe *cpipe; 601{ 602 603 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 604 if (cpipe->pipe_state & PIPE_SEL) { 605 selwakeuppri(&cpipe->pipe_sel, PSOCK); 606 if (!SEL_WAITING(&cpipe->pipe_sel)) 607 cpipe->pipe_state &= ~PIPE_SEL; 608 } 609 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 610 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 611 KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0); 612} 613 614/* 615 * Initialize and allocate VM and memory for pipe. The structure 616 * will start out zero'd from the ctor, so we just manage the kmem. 617 */ 618static int 619pipe_create(pipe, backing) 620 struct pipe *pipe; 621 int backing; 622{ 623 int error; 624 625 if (backing) { 626 if (amountpipekva > maxpipekva / 2) 627 error = pipespace_new(pipe, SMALL_PIPE_SIZE); 628 else 629 error = pipespace_new(pipe, PIPE_SIZE); 630 } else { 631 /* If we're not backing this pipe, no need to do anything. */ 632 error = 0; 633 } 634 pipe->pipe_ino = -1; 635 return (error); 636} 637 638/* ARGSUSED */ 639static int 640pipe_read(fp, uio, active_cred, flags, td) 641 struct file *fp; 642 struct uio *uio; 643 struct ucred *active_cred; 644 struct thread *td; 645 int flags; 646{ 647 struct pipe *rpipe; 648 int error; 649 int nread = 0; 650 u_int size; 651 652 rpipe = fp->f_data; 653 PIPE_LOCK(rpipe); 654 ++rpipe->pipe_busy; 655 error = pipelock(rpipe, 1); 656 if (error) 657 goto unlocked_error; 658 659#ifdef MAC 660 error = mac_pipe_check_read(active_cred, rpipe->pipe_pair); 661 if (error) 662 goto locked_error; 663#endif 664 if (amountpipekva > (3 * maxpipekva) / 4) { 665 if (!(rpipe->pipe_state & PIPE_DIRECTW) && 666 (rpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 667 (rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 668 (piperesizeallowed == 1)) { 669 PIPE_UNLOCK(rpipe); 670 pipespace(rpipe, SMALL_PIPE_SIZE); 671 PIPE_LOCK(rpipe); 672 } 673 } 674 675 while (uio->uio_resid) { 676 /* 677 * normal pipe buffer receive 678 */ 679 if (rpipe->pipe_buffer.cnt > 0) { 680 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 681 if (size > rpipe->pipe_buffer.cnt) 682 size = rpipe->pipe_buffer.cnt; 683 if (size > uio->uio_resid) 684 size = (u_int) uio->uio_resid; 685 686 PIPE_UNLOCK(rpipe); 687 error = uiomove( 688 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 689 size, uio); 690 PIPE_LOCK(rpipe); 691 if (error) 692 break; 693 694 rpipe->pipe_buffer.out += size; 695 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 696 rpipe->pipe_buffer.out = 0; 697 698 rpipe->pipe_buffer.cnt -= size; 699 700 /* 701 * If there is no more to read in the pipe, reset 702 * its pointers to the beginning. This improves 703 * cache hit stats. 704 */ 705 if (rpipe->pipe_buffer.cnt == 0) { 706 rpipe->pipe_buffer.in = 0; 707 rpipe->pipe_buffer.out = 0; 708 } 709 nread += size; 710#ifndef PIPE_NODIRECT 711 /* 712 * Direct copy, bypassing a kernel buffer. 713 */ 714 } else if ((size = rpipe->pipe_map.cnt) && 715 (rpipe->pipe_state & PIPE_DIRECTW)) { 716 if (size > uio->uio_resid) 717 size = (u_int) uio->uio_resid; 718 719 PIPE_UNLOCK(rpipe); 720 error = uiomove_fromphys(rpipe->pipe_map.ms, 721 rpipe->pipe_map.pos, size, uio); 722 PIPE_LOCK(rpipe); 723 if (error) 724 break; 725 nread += size; 726 rpipe->pipe_map.pos += size; 727 rpipe->pipe_map.cnt -= size; 728 if (rpipe->pipe_map.cnt == 0) { 729 rpipe->pipe_state &= ~PIPE_DIRECTW; 730 wakeup(rpipe); 731 } 732#endif 733 } else { 734 /* 735 * detect EOF condition 736 * read returns 0 on EOF, no need to set error 737 */ 738 if (rpipe->pipe_state & PIPE_EOF) 739 break; 740 741 /* 742 * If the "write-side" has been blocked, wake it up now. 743 */ 744 if (rpipe->pipe_state & PIPE_WANTW) { 745 rpipe->pipe_state &= ~PIPE_WANTW; 746 wakeup(rpipe); 747 } 748 749 /* 750 * Break if some data was read. 751 */ 752 if (nread > 0) 753 break; 754 755 /* 756 * Unlock the pipe buffer for our remaining processing. 757 * We will either break out with an error or we will 758 * sleep and relock to loop. 759 */ 760 pipeunlock(rpipe); 761 762 /* 763 * Handle non-blocking mode operation or 764 * wait for more data. 765 */ 766 if (fp->f_flag & FNONBLOCK) { 767 error = EAGAIN; 768 } else { 769 rpipe->pipe_state |= PIPE_WANTR; 770 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 771 PRIBIO | PCATCH, 772 "piperd", 0)) == 0) 773 error = pipelock(rpipe, 1); 774 } 775 if (error) 776 goto unlocked_error; 777 } 778 } 779#ifdef MAC 780locked_error: 781#endif 782 pipeunlock(rpipe); 783 784 /* XXX: should probably do this before getting any locks. */ 785 if (error == 0) 786 vfs_timestamp(&rpipe->pipe_atime); 787unlocked_error: 788 --rpipe->pipe_busy; 789 790 /* 791 * PIPE_WANT processing only makes sense if pipe_busy is 0. 792 */ 793 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 794 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 795 wakeup(rpipe); 796 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 797 /* 798 * Handle write blocking hysteresis. 799 */ 800 if (rpipe->pipe_state & PIPE_WANTW) { 801 rpipe->pipe_state &= ~PIPE_WANTW; 802 wakeup(rpipe); 803 } 804 } 805 806 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 807 pipeselwakeup(rpipe); 808 809 PIPE_UNLOCK(rpipe); 810 return (error); 811} 812 813#ifndef PIPE_NODIRECT 814/* 815 * Map the sending processes' buffer into kernel space and wire it. 816 * This is similar to a physical write operation. 817 */ 818static int 819pipe_build_write_buffer(wpipe, uio) 820 struct pipe *wpipe; 821 struct uio *uio; 822{ 823 u_int size; 824 int i; 825 826 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 827 KASSERT(wpipe->pipe_state & PIPE_DIRECTW, 828 ("Clone attempt on non-direct write pipe!")); 829 830 if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size) 831 size = wpipe->pipe_buffer.size; 832 else 833 size = uio->uio_iov->iov_len; 834 835 if ((i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 836 (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ, 837 wpipe->pipe_map.ms, PIPENPAGES)) < 0) 838 return (EFAULT); 839 840/* 841 * set up the control block 842 */ 843 wpipe->pipe_map.npages = i; 844 wpipe->pipe_map.pos = 845 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 846 wpipe->pipe_map.cnt = size; 847 848/* 849 * and update the uio data 850 */ 851 852 uio->uio_iov->iov_len -= size; 853 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 854 if (uio->uio_iov->iov_len == 0) 855 uio->uio_iov++; 856 uio->uio_resid -= size; 857 uio->uio_offset += size; 858 return (0); 859} 860 861/* 862 * unmap and unwire the process buffer 863 */ 864static void 865pipe_destroy_write_buffer(wpipe) 866 struct pipe *wpipe; 867{ 868 869 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 870 vm_page_unhold_pages(wpipe->pipe_map.ms, wpipe->pipe_map.npages); 871 wpipe->pipe_map.npages = 0; 872} 873 874/* 875 * In the case of a signal, the writing process might go away. This 876 * code copies the data into the circular buffer so that the source 877 * pages can be freed without loss of data. 878 */ 879static void 880pipe_clone_write_buffer(wpipe) 881 struct pipe *wpipe; 882{ 883 struct uio uio; 884 struct iovec iov; 885 int size; 886 int pos; 887 888 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 889 size = wpipe->pipe_map.cnt; 890 pos = wpipe->pipe_map.pos; 891 892 wpipe->pipe_buffer.in = size; 893 wpipe->pipe_buffer.out = 0; 894 wpipe->pipe_buffer.cnt = size; 895 wpipe->pipe_state &= ~PIPE_DIRECTW; 896 897 PIPE_UNLOCK(wpipe); 898 iov.iov_base = wpipe->pipe_buffer.buffer; 899 iov.iov_len = size; 900 uio.uio_iov = &iov; 901 uio.uio_iovcnt = 1; 902 uio.uio_offset = 0; 903 uio.uio_resid = size; 904 uio.uio_segflg = UIO_SYSSPACE; 905 uio.uio_rw = UIO_READ; 906 uio.uio_td = curthread; 907 uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio); 908 PIPE_LOCK(wpipe); 909 pipe_destroy_write_buffer(wpipe); 910} 911 912/* 913 * This implements the pipe buffer write mechanism. Note that only 914 * a direct write OR a normal pipe write can be pending at any given time. 915 * If there are any characters in the pipe buffer, the direct write will 916 * be deferred until the receiving process grabs all of the bytes from 917 * the pipe buffer. Then the direct mapping write is set-up. 918 */ 919static int 920pipe_direct_write(wpipe, uio) 921 struct pipe *wpipe; 922 struct uio *uio; 923{ 924 int error; 925 926retry: 927 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 928 error = pipelock(wpipe, 1); 929 if (wpipe->pipe_state & PIPE_EOF) 930 error = EPIPE; 931 if (error) { 932 pipeunlock(wpipe); 933 goto error1; 934 } 935 while (wpipe->pipe_state & PIPE_DIRECTW) { 936 if (wpipe->pipe_state & PIPE_WANTR) { 937 wpipe->pipe_state &= ~PIPE_WANTR; 938 wakeup(wpipe); 939 } 940 pipeselwakeup(wpipe); 941 wpipe->pipe_state |= PIPE_WANTW; 942 pipeunlock(wpipe); 943 error = msleep(wpipe, PIPE_MTX(wpipe), 944 PRIBIO | PCATCH, "pipdww", 0); 945 if (error) 946 goto error1; 947 else 948 goto retry; 949 } 950 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 951 if (wpipe->pipe_buffer.cnt > 0) { 952 if (wpipe->pipe_state & PIPE_WANTR) { 953 wpipe->pipe_state &= ~PIPE_WANTR; 954 wakeup(wpipe); 955 } 956 pipeselwakeup(wpipe); 957 wpipe->pipe_state |= PIPE_WANTW; 958 pipeunlock(wpipe); 959 error = msleep(wpipe, PIPE_MTX(wpipe), 960 PRIBIO | PCATCH, "pipdwc", 0); 961 if (error) 962 goto error1; 963 else 964 goto retry; 965 } 966 967 wpipe->pipe_state |= PIPE_DIRECTW; 968 969 PIPE_UNLOCK(wpipe); 970 error = pipe_build_write_buffer(wpipe, uio); 971 PIPE_LOCK(wpipe); 972 if (error) { 973 wpipe->pipe_state &= ~PIPE_DIRECTW; 974 pipeunlock(wpipe); 975 goto error1; 976 } 977 978 error = 0; 979 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 980 if (wpipe->pipe_state & PIPE_EOF) { 981 pipe_destroy_write_buffer(wpipe); 982 pipeselwakeup(wpipe); 983 pipeunlock(wpipe); 984 error = EPIPE; 985 goto error1; 986 } 987 if (wpipe->pipe_state & PIPE_WANTR) { 988 wpipe->pipe_state &= ~PIPE_WANTR; 989 wakeup(wpipe); 990 } 991 pipeselwakeup(wpipe); 992 pipeunlock(wpipe); 993 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 994 "pipdwt", 0); 995 pipelock(wpipe, 0); 996 } 997 998 if (wpipe->pipe_state & PIPE_EOF) 999 error = EPIPE; 1000 if (wpipe->pipe_state & PIPE_DIRECTW) { 1001 /* 1002 * this bit of trickery substitutes a kernel buffer for 1003 * the process that might be going away. 1004 */ 1005 pipe_clone_write_buffer(wpipe); 1006 } else { 1007 pipe_destroy_write_buffer(wpipe); 1008 } 1009 pipeunlock(wpipe); 1010 return (error); 1011 1012error1: 1013 wakeup(wpipe); 1014 return (error); 1015} 1016#endif 1017 1018static int 1019pipe_write(fp, uio, active_cred, flags, td) 1020 struct file *fp; 1021 struct uio *uio; 1022 struct ucred *active_cred; 1023 struct thread *td; 1024 int flags; 1025{ 1026 int error; 1027 size_t desiredsize, orig_resid; 1028 struct pipe *wpipe, *rpipe; 1029 1030 rpipe = fp->f_data; 1031 wpipe = PIPE_PEER(rpipe); 1032 PIPE_LOCK(rpipe); 1033 error = pipelock(wpipe, 1); 1034 if (error) { 1035 PIPE_UNLOCK(rpipe); 1036 return (error); 1037 } 1038 /* 1039 * detect loss of pipe read side, issue SIGPIPE if lost. 1040 */ 1041 if (wpipe->pipe_present != PIPE_ACTIVE || 1042 (wpipe->pipe_state & PIPE_EOF)) { 1043 pipeunlock(wpipe); 1044 PIPE_UNLOCK(rpipe); 1045 return (EPIPE); 1046 } 1047#ifdef MAC 1048 error = mac_pipe_check_write(active_cred, wpipe->pipe_pair); 1049 if (error) { 1050 pipeunlock(wpipe); 1051 PIPE_UNLOCK(rpipe); 1052 return (error); 1053 } 1054#endif 1055 ++wpipe->pipe_busy; 1056 1057 /* Choose a larger size if it's advantageous */ 1058 desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size); 1059 while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) { 1060 if (piperesizeallowed != 1) 1061 break; 1062 if (amountpipekva > maxpipekva / 2) 1063 break; 1064 if (desiredsize == BIG_PIPE_SIZE) 1065 break; 1066 desiredsize = desiredsize * 2; 1067 } 1068 1069 /* Choose a smaller size if we're in a OOM situation */ 1070 if ((amountpipekva > (3 * maxpipekva) / 4) && 1071 (wpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 1072 (wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 1073 (piperesizeallowed == 1)) 1074 desiredsize = SMALL_PIPE_SIZE; 1075 1076 /* Resize if the above determined that a new size was necessary */ 1077 if ((desiredsize != wpipe->pipe_buffer.size) && 1078 ((wpipe->pipe_state & PIPE_DIRECTW) == 0)) { 1079 PIPE_UNLOCK(wpipe); 1080 pipespace(wpipe, desiredsize); 1081 PIPE_LOCK(wpipe); 1082 } 1083 if (wpipe->pipe_buffer.size == 0) { 1084 /* 1085 * This can only happen for reverse direction use of pipes 1086 * in a complete OOM situation. 1087 */ 1088 error = ENOMEM; 1089 --wpipe->pipe_busy; 1090 pipeunlock(wpipe); 1091 PIPE_UNLOCK(wpipe); 1092 return (error); 1093 } 1094 1095 pipeunlock(wpipe); 1096 1097 orig_resid = uio->uio_resid; 1098 1099 while (uio->uio_resid) { 1100 int space; 1101 1102 pipelock(wpipe, 0); 1103 if (wpipe->pipe_state & PIPE_EOF) { 1104 pipeunlock(wpipe); 1105 error = EPIPE; 1106 break; 1107 } 1108#ifndef PIPE_NODIRECT 1109 /* 1110 * If the transfer is large, we can gain performance if 1111 * we do process-to-process copies directly. 1112 * If the write is non-blocking, we don't use the 1113 * direct write mechanism. 1114 * 1115 * The direct write mechanism will detect the reader going 1116 * away on us. 1117 */ 1118 if (uio->uio_segflg == UIO_USERSPACE && 1119 uio->uio_iov->iov_len >= PIPE_MINDIRECT && 1120 wpipe->pipe_buffer.size >= PIPE_MINDIRECT && 1121 (fp->f_flag & FNONBLOCK) == 0) { 1122 pipeunlock(wpipe); 1123 error = pipe_direct_write(wpipe, uio); 1124 if (error) 1125 break; 1126 continue; 1127 } 1128#endif 1129 1130 /* 1131 * Pipe buffered writes cannot be coincidental with 1132 * direct writes. We wait until the currently executing 1133 * direct write is completed before we start filling the 1134 * pipe buffer. We break out if a signal occurs or the 1135 * reader goes away. 1136 */ 1137 if (wpipe->pipe_state & PIPE_DIRECTW) { 1138 if (wpipe->pipe_state & PIPE_WANTR) { 1139 wpipe->pipe_state &= ~PIPE_WANTR; 1140 wakeup(wpipe); 1141 } 1142 pipeselwakeup(wpipe); 1143 wpipe->pipe_state |= PIPE_WANTW; 1144 pipeunlock(wpipe); 1145 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1146 "pipbww", 0); 1147 if (error) 1148 break; 1149 else 1150 continue; 1151 } 1152 1153 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1154 1155 /* Writes of size <= PIPE_BUF must be atomic. */ 1156 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1157 space = 0; 1158 1159 if (space > 0) { 1160 int size; /* Transfer size */ 1161 int segsize; /* first segment to transfer */ 1162 1163 /* 1164 * Transfer size is minimum of uio transfer 1165 * and free space in pipe buffer. 1166 */ 1167 if (space > uio->uio_resid) 1168 size = uio->uio_resid; 1169 else 1170 size = space; 1171 /* 1172 * First segment to transfer is minimum of 1173 * transfer size and contiguous space in 1174 * pipe buffer. If first segment to transfer 1175 * is less than the transfer size, we've got 1176 * a wraparound in the buffer. 1177 */ 1178 segsize = wpipe->pipe_buffer.size - 1179 wpipe->pipe_buffer.in; 1180 if (segsize > size) 1181 segsize = size; 1182 1183 /* Transfer first segment */ 1184 1185 PIPE_UNLOCK(rpipe); 1186 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1187 segsize, uio); 1188 PIPE_LOCK(rpipe); 1189 1190 if (error == 0 && segsize < size) { 1191 KASSERT(wpipe->pipe_buffer.in + segsize == 1192 wpipe->pipe_buffer.size, 1193 ("Pipe buffer wraparound disappeared")); 1194 /* 1195 * Transfer remaining part now, to 1196 * support atomic writes. Wraparound 1197 * happened. 1198 */ 1199 1200 PIPE_UNLOCK(rpipe); 1201 error = uiomove( 1202 &wpipe->pipe_buffer.buffer[0], 1203 size - segsize, uio); 1204 PIPE_LOCK(rpipe); 1205 } 1206 if (error == 0) { 1207 wpipe->pipe_buffer.in += size; 1208 if (wpipe->pipe_buffer.in >= 1209 wpipe->pipe_buffer.size) { 1210 KASSERT(wpipe->pipe_buffer.in == 1211 size - segsize + 1212 wpipe->pipe_buffer.size, 1213 ("Expected wraparound bad")); 1214 wpipe->pipe_buffer.in = size - segsize; 1215 } 1216 1217 wpipe->pipe_buffer.cnt += size; 1218 KASSERT(wpipe->pipe_buffer.cnt <= 1219 wpipe->pipe_buffer.size, 1220 ("Pipe buffer overflow")); 1221 } 1222 pipeunlock(wpipe); 1223 if (error != 0) 1224 break; 1225 } else { 1226 /* 1227 * If the "read-side" has been blocked, wake it up now. 1228 */ 1229 if (wpipe->pipe_state & PIPE_WANTR) { 1230 wpipe->pipe_state &= ~PIPE_WANTR; 1231 wakeup(wpipe); 1232 } 1233 1234 /* 1235 * don't block on non-blocking I/O 1236 */ 1237 if (fp->f_flag & FNONBLOCK) { 1238 error = EAGAIN; 1239 pipeunlock(wpipe); 1240 break; 1241 } 1242 1243 /* 1244 * We have no more space and have something to offer, 1245 * wake up select/poll. 1246 */ 1247 pipeselwakeup(wpipe); 1248 1249 wpipe->pipe_state |= PIPE_WANTW; 1250 pipeunlock(wpipe); 1251 error = msleep(wpipe, PIPE_MTX(rpipe), 1252 PRIBIO | PCATCH, "pipewr", 0); 1253 if (error != 0) 1254 break; 1255 } 1256 } 1257 1258 pipelock(wpipe, 0); 1259 --wpipe->pipe_busy; 1260 1261 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1262 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1263 wakeup(wpipe); 1264 } else if (wpipe->pipe_buffer.cnt > 0) { 1265 /* 1266 * If we have put any characters in the buffer, we wake up 1267 * the reader. 1268 */ 1269 if (wpipe->pipe_state & PIPE_WANTR) { 1270 wpipe->pipe_state &= ~PIPE_WANTR; 1271 wakeup(wpipe); 1272 } 1273 } 1274 1275 /* 1276 * Don't return EPIPE if I/O was successful 1277 */ 1278 if ((wpipe->pipe_buffer.cnt == 0) && 1279 (uio->uio_resid == 0) && 1280 (error == EPIPE)) { 1281 error = 0; 1282 } 1283 1284 if (error == 0) 1285 vfs_timestamp(&wpipe->pipe_mtime); 1286 1287 /* 1288 * We have something to offer, 1289 * wake up select/poll. 1290 */ 1291 if (wpipe->pipe_buffer.cnt) 1292 pipeselwakeup(wpipe); 1293 1294 pipeunlock(wpipe); 1295 PIPE_UNLOCK(rpipe); 1296 return (error); 1297} 1298 1299/* ARGSUSED */ 1300static int 1301pipe_truncate(fp, length, active_cred, td) 1302 struct file *fp; 1303 off_t length; 1304 struct ucred *active_cred; 1305 struct thread *td; 1306{ 1307 1308 /* For named pipes call the vnode operation. */ 1309 if (fp->f_vnode != NULL) 1310 return (vnops.fo_truncate(fp, length, active_cred, td)); 1311 return (EINVAL); 1312} 1313 1314/* 1315 * we implement a very minimal set of ioctls for compatibility with sockets. 1316 */ 1317static int 1318pipe_ioctl(fp, cmd, data, active_cred, td) 1319 struct file *fp; 1320 u_long cmd; 1321 void *data; 1322 struct ucred *active_cred; 1323 struct thread *td; 1324{ 1325 struct pipe *mpipe = fp->f_data; 1326 int error; 1327 1328 PIPE_LOCK(mpipe); 1329 1330#ifdef MAC 1331 error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data); 1332 if (error) { 1333 PIPE_UNLOCK(mpipe); 1334 return (error); 1335 } 1336#endif 1337 1338 error = 0; 1339 switch (cmd) { 1340 1341 case FIONBIO: 1342 break; 1343 1344 case FIOASYNC: 1345 if (*(int *)data) { 1346 mpipe->pipe_state |= PIPE_ASYNC; 1347 } else { 1348 mpipe->pipe_state &= ~PIPE_ASYNC; 1349 } 1350 break; 1351 1352 case FIONREAD: 1353 if (!(fp->f_flag & FREAD)) { 1354 *(int *)data = 0; 1355 PIPE_UNLOCK(mpipe); 1356 return (0); 1357 } 1358 if (mpipe->pipe_state & PIPE_DIRECTW) 1359 *(int *)data = mpipe->pipe_map.cnt; 1360 else 1361 *(int *)data = mpipe->pipe_buffer.cnt; 1362 break; 1363 1364 case FIOSETOWN: 1365 PIPE_UNLOCK(mpipe); 1366 error = fsetown(*(int *)data, &mpipe->pipe_sigio); 1367 goto out_unlocked; 1368 1369 case FIOGETOWN: 1370 *(int *)data = fgetown(&mpipe->pipe_sigio); 1371 break; 1372 1373 /* This is deprecated, FIOSETOWN should be used instead. */ 1374 case TIOCSPGRP: 1375 PIPE_UNLOCK(mpipe); 1376 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio); 1377 goto out_unlocked; 1378 1379 /* This is deprecated, FIOGETOWN should be used instead. */ 1380 case TIOCGPGRP: 1381 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1382 break; 1383 1384 default: 1385 error = ENOTTY; 1386 break; 1387 } 1388 PIPE_UNLOCK(mpipe); 1389out_unlocked: 1390 return (error); 1391} 1392 1393static int 1394pipe_poll(fp, events, active_cred, td) 1395 struct file *fp; 1396 int events; 1397 struct ucred *active_cred; 1398 struct thread *td; 1399{ 1400 struct pipe *rpipe; 1401 struct pipe *wpipe; 1402 int levents, revents; 1403#ifdef MAC 1404 int error; 1405#endif 1406 1407 revents = 0; 1408 rpipe = fp->f_data; 1409 wpipe = PIPE_PEER(rpipe); 1410 PIPE_LOCK(rpipe); 1411#ifdef MAC 1412 error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair); 1413 if (error) 1414 goto locked_error; 1415#endif 1416 if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) 1417 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1418 (rpipe->pipe_buffer.cnt > 0)) 1419 revents |= events & (POLLIN | POLLRDNORM); 1420 1421 if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) 1422 if (wpipe->pipe_present != PIPE_ACTIVE || 1423 (wpipe->pipe_state & PIPE_EOF) || 1424 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1425 ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF || 1426 wpipe->pipe_buffer.size == 0))) 1427 revents |= events & (POLLOUT | POLLWRNORM); 1428 1429 levents = events & 1430 (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | POLLRDBAND); 1431 if (rpipe->pipe_state & PIPE_NAMED && fp->f_flag & FREAD && levents && 1432 fifo_iseof(fp)) 1433 events |= POLLINIGNEOF; 1434 1435 if ((events & POLLINIGNEOF) == 0) { 1436 if (rpipe->pipe_state & PIPE_EOF) { 1437 revents |= (events & (POLLIN | POLLRDNORM)); 1438 if (wpipe->pipe_present != PIPE_ACTIVE || 1439 (wpipe->pipe_state & PIPE_EOF)) 1440 revents |= POLLHUP; 1441 } 1442 } 1443 1444 if (revents == 0) { 1445 if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) { 1446 selrecord(td, &rpipe->pipe_sel); 1447 if (SEL_WAITING(&rpipe->pipe_sel)) 1448 rpipe->pipe_state |= PIPE_SEL; 1449 } 1450 1451 if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) { 1452 selrecord(td, &wpipe->pipe_sel); 1453 if (SEL_WAITING(&wpipe->pipe_sel)) 1454 wpipe->pipe_state |= PIPE_SEL; 1455 } 1456 } 1457#ifdef MAC 1458locked_error: 1459#endif 1460 PIPE_UNLOCK(rpipe); 1461 1462 return (revents); 1463} 1464 1465/* 1466 * We shouldn't need locks here as we're doing a read and this should 1467 * be a natural race. 1468 */ 1469static int 1470pipe_stat(fp, ub, active_cred, td) 1471 struct file *fp; 1472 struct stat *ub; 1473 struct ucred *active_cred; 1474 struct thread *td; 1475{ 1476 struct pipe *pipe; 1477 int new_unr; 1478#ifdef MAC 1479 int error; 1480#endif 1481 1482 pipe = fp->f_data; 1483 PIPE_LOCK(pipe); 1484#ifdef MAC 1485 error = mac_pipe_check_stat(active_cred, pipe->pipe_pair); 1486 if (error) { 1487 PIPE_UNLOCK(pipe); 1488 return (error); 1489 } 1490#endif 1491 1492 /* For named pipes ask the underlying filesystem. */ 1493 if (pipe->pipe_state & PIPE_NAMED) { 1494 PIPE_UNLOCK(pipe); 1495 return (vnops.fo_stat(fp, ub, active_cred, td)); 1496 } 1497 1498 /* 1499 * Lazily allocate an inode number for the pipe. Most pipe 1500 * users do not call fstat(2) on the pipe, which means that 1501 * postponing the inode allocation until it is must be 1502 * returned to userland is useful. If alloc_unr failed, 1503 * assign st_ino zero instead of returning an error. 1504 * Special pipe_ino values: 1505 * -1 - not yet initialized; 1506 * 0 - alloc_unr failed, return 0 as st_ino forever. 1507 */ 1508 if (pipe->pipe_ino == (ino_t)-1) { 1509 new_unr = alloc_unr(pipeino_unr); 1510 if (new_unr != -1) 1511 pipe->pipe_ino = new_unr; 1512 else 1513 pipe->pipe_ino = 0; 1514 } 1515 PIPE_UNLOCK(pipe); 1516 1517 bzero(ub, sizeof(*ub)); 1518 ub->st_mode = S_IFIFO; 1519 ub->st_blksize = PAGE_SIZE; 1520 if (pipe->pipe_state & PIPE_DIRECTW) 1521 ub->st_size = pipe->pipe_map.cnt; 1522 else 1523 ub->st_size = pipe->pipe_buffer.cnt; 1524 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1525 ub->st_atim = pipe->pipe_atime; 1526 ub->st_mtim = pipe->pipe_mtime; 1527 ub->st_ctim = pipe->pipe_ctime; 1528 ub->st_uid = fp->f_cred->cr_uid; 1529 ub->st_gid = fp->f_cred->cr_gid; 1530 ub->st_dev = pipedev_ino; 1531 ub->st_ino = pipe->pipe_ino; 1532 /* 1533 * Left as 0: st_nlink, st_rdev, st_flags, st_gen. 1534 */ 1535 return (0); 1536} 1537 1538/* ARGSUSED */ 1539static int 1540pipe_close(fp, td) 1541 struct file *fp; 1542 struct thread *td; 1543{ 1544 1545 if (fp->f_vnode != NULL) 1546 return vnops.fo_close(fp, td); 1547 fp->f_ops = &badfileops; 1548 pipe_dtor(fp->f_data); 1549 fp->f_data = NULL; 1550 return (0); 1551} 1552 1553static int 1554pipe_chmod(fp, mode, active_cred, td) 1555 struct file *fp; 1556 mode_t mode; 1557 struct ucred *active_cred; 1558 struct thread *td; 1559{ 1560 struct pipe *cpipe; 1561 int error; 1562 1563 cpipe = fp->f_data; 1564 if (cpipe->pipe_state & PIPE_NAMED) 1565 error = vn_chmod(fp, mode, active_cred, td); 1566 else 1567 error = invfo_chmod(fp, mode, active_cred, td); 1568 return (error); 1569} 1570 1571static int 1572pipe_chown(fp, uid, gid, active_cred, td) 1573 struct file *fp; 1574 uid_t uid; 1575 gid_t gid; 1576 struct ucred *active_cred; 1577 struct thread *td; 1578{ 1579 struct pipe *cpipe; 1580 int error; 1581 1582 cpipe = fp->f_data; 1583 if (cpipe->pipe_state & PIPE_NAMED) 1584 error = vn_chown(fp, uid, gid, active_cred, td); 1585 else 1586 error = invfo_chown(fp, uid, gid, active_cred, td); 1587 return (error); 1588} 1589 1590static void 1591pipe_free_kmem(cpipe) 1592 struct pipe *cpipe; 1593{ 1594 1595 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), 1596 ("pipe_free_kmem: pipe mutex locked")); 1597 1598 if (cpipe->pipe_buffer.buffer != NULL) { 1599 atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size); 1600 vm_map_remove(pipe_map, 1601 (vm_offset_t)cpipe->pipe_buffer.buffer, 1602 (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size); 1603 cpipe->pipe_buffer.buffer = NULL; 1604 } 1605#ifndef PIPE_NODIRECT 1606 { 1607 cpipe->pipe_map.cnt = 0; 1608 cpipe->pipe_map.pos = 0; 1609 cpipe->pipe_map.npages = 0; 1610 } 1611#endif 1612} 1613 1614/* 1615 * shutdown the pipe 1616 */ 1617static void 1618pipeclose(cpipe) 1619 struct pipe *cpipe; 1620{ 1621 struct pipepair *pp; 1622 struct pipe *ppipe; 1623 1624 KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL")); 1625 1626 PIPE_LOCK(cpipe); 1627 pipelock(cpipe, 0); 1628 pp = cpipe->pipe_pair; 1629 1630 pipeselwakeup(cpipe); 1631 1632 /* 1633 * If the other side is blocked, wake it up saying that 1634 * we want to close it down. 1635 */ 1636 cpipe->pipe_state |= PIPE_EOF; 1637 while (cpipe->pipe_busy) { 1638 wakeup(cpipe); 1639 cpipe->pipe_state |= PIPE_WANT; 1640 pipeunlock(cpipe); 1641 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1642 pipelock(cpipe, 0); 1643 } 1644 1645 1646 /* 1647 * Disconnect from peer, if any. 1648 */ 1649 ppipe = cpipe->pipe_peer; 1650 if (ppipe->pipe_present == PIPE_ACTIVE) { 1651 pipeselwakeup(ppipe); 1652 1653 ppipe->pipe_state |= PIPE_EOF; 1654 wakeup(ppipe); 1655 KNOTE_LOCKED(&ppipe->pipe_sel.si_note, 0); 1656 } 1657 1658 /* 1659 * Mark this endpoint as free. Release kmem resources. We 1660 * don't mark this endpoint as unused until we've finished 1661 * doing that, or the pipe might disappear out from under 1662 * us. 1663 */ 1664 PIPE_UNLOCK(cpipe); 1665 pipe_free_kmem(cpipe); 1666 PIPE_LOCK(cpipe); 1667 cpipe->pipe_present = PIPE_CLOSING; 1668 pipeunlock(cpipe); 1669 1670 /* 1671 * knlist_clear() may sleep dropping the PIPE_MTX. Set the 1672 * PIPE_FINALIZED, that allows other end to free the 1673 * pipe_pair, only after the knotes are completely dismantled. 1674 */ 1675 knlist_clear(&cpipe->pipe_sel.si_note, 1); 1676 cpipe->pipe_present = PIPE_FINALIZED; 1677 seldrain(&cpipe->pipe_sel); 1678 knlist_destroy(&cpipe->pipe_sel.si_note); 1679 1680 /* 1681 * If both endpoints are now closed, release the memory for the 1682 * pipe pair. If not, unlock. 1683 */ 1684 if (ppipe->pipe_present == PIPE_FINALIZED) { 1685 PIPE_UNLOCK(cpipe); 1686#ifdef MAC 1687 mac_pipe_destroy(pp); 1688#endif 1689 uma_zfree(pipe_zone, cpipe->pipe_pair); 1690 } else 1691 PIPE_UNLOCK(cpipe); 1692} 1693 1694/*ARGSUSED*/ 1695static int 1696pipe_kqfilter(struct file *fp, struct knote *kn) 1697{ 1698 struct pipe *cpipe; 1699 1700 /* 1701 * If a filter is requested that is not supported by this file 1702 * descriptor, don't return an error, but also don't ever generate an 1703 * event. 1704 */ 1705 if ((kn->kn_filter == EVFILT_READ) && !(fp->f_flag & FREAD)) { 1706 kn->kn_fop = &pipe_nfiltops; 1707 return (0); 1708 } 1709 if ((kn->kn_filter == EVFILT_WRITE) && !(fp->f_flag & FWRITE)) { 1710 kn->kn_fop = &pipe_nfiltops; 1711 return (0); 1712 } 1713 cpipe = fp->f_data; 1714 PIPE_LOCK(cpipe); 1715 switch (kn->kn_filter) { 1716 case EVFILT_READ: 1717 kn->kn_fop = &pipe_rfiltops; 1718 break; 1719 case EVFILT_WRITE: 1720 kn->kn_fop = &pipe_wfiltops; 1721 if (cpipe->pipe_peer->pipe_present != PIPE_ACTIVE) { 1722 /* other end of pipe has been closed */ 1723 PIPE_UNLOCK(cpipe); 1724 return (EPIPE); 1725 } 1726 cpipe = PIPE_PEER(cpipe); 1727 break; 1728 default: 1729 PIPE_UNLOCK(cpipe); 1730 return (EINVAL); 1731 } 1732 1733 kn->kn_hook = cpipe; 1734 knlist_add(&cpipe->pipe_sel.si_note, kn, 1); 1735 PIPE_UNLOCK(cpipe); 1736 return (0); 1737} 1738 1739static void 1740filt_pipedetach(struct knote *kn) 1741{ 1742 struct pipe *cpipe = kn->kn_hook; 1743 1744 PIPE_LOCK(cpipe); 1745 knlist_remove(&cpipe->pipe_sel.si_note, kn, 1); 1746 PIPE_UNLOCK(cpipe); 1747} 1748 1749/*ARGSUSED*/ 1750static int 1751filt_piperead(struct knote *kn, long hint) 1752{ 1753 struct pipe *rpipe = kn->kn_hook; 1754 struct pipe *wpipe = rpipe->pipe_peer; 1755 int ret; 1756 1757 PIPE_LOCK(rpipe); 1758 kn->kn_data = rpipe->pipe_buffer.cnt; 1759 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1760 kn->kn_data = rpipe->pipe_map.cnt; 1761 1762 if ((rpipe->pipe_state & PIPE_EOF) || 1763 wpipe->pipe_present != PIPE_ACTIVE || 1764 (wpipe->pipe_state & PIPE_EOF)) { 1765 kn->kn_flags |= EV_EOF; 1766 PIPE_UNLOCK(rpipe); 1767 return (1); 1768 } 1769 ret = kn->kn_data > 0; 1770 PIPE_UNLOCK(rpipe); 1771 return ret; 1772} 1773 1774/*ARGSUSED*/ 1775static int 1776filt_pipewrite(struct knote *kn, long hint) 1777{ 1778 struct pipe *wpipe; 1779 1780 wpipe = kn->kn_hook; 1781 PIPE_LOCK(wpipe); 1782 if (wpipe->pipe_present != PIPE_ACTIVE || 1783 (wpipe->pipe_state & PIPE_EOF)) { 1784 kn->kn_data = 0; 1785 kn->kn_flags |= EV_EOF; 1786 PIPE_UNLOCK(wpipe); 1787 return (1); 1788 } 1789 kn->kn_data = (wpipe->pipe_buffer.size > 0) ? 1790 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) : PIPE_BUF; 1791 if (wpipe->pipe_state & PIPE_DIRECTW) 1792 kn->kn_data = 0; 1793 1794 PIPE_UNLOCK(wpipe); 1795 return (kn->kn_data >= PIPE_BUF); 1796} 1797 1798static void 1799filt_pipedetach_notsup(struct knote *kn) 1800{ 1801 1802} 1803 1804static int 1805filt_pipenotsup(struct knote *kn, long hint) 1806{ 1807 1808 return (0); 1809} 1810