#include <sys/param.h>
#include <sys/systm.h>
#include <sys/filedesc.h>
#include <sys/kernel.h>
#include <sys/vnode.h>
#include <sys/proc_internal.h>
#include <sys/kauth.h>
#include <sys/file_internal.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/fcntl.h>
#include <sys/malloc.h>
#include <sys/syslog.h>
#include <sys/unistd.h>
#include <sys/resourcevar.h>
#include <sys/aio_kern.h>
#include <sys/signalvar.h>
#include <sys/pipe.h>
#include <sys/sysproto.h>
#include <bsm/audit_kernel.h>
#include <sys/kdebug.h>
#include <kern/zalloc.h>
#include <vm/vm_kern.h>
#include <libkern/OSAtomic.h>
#define f_flag f_fglob->fg_flag
#define f_type f_fglob->fg_type
#define f_msgcount f_fglob->fg_msgcount
#define f_cred f_fglob->fg_cred
#define f_ops f_fglob->fg_ops
#define f_offset f_fglob->fg_offset
#define f_data f_fglob->fg_data
#define PIPE_NODIRECT 1
#ifndef PIPE_NODIRECT
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_page.h>
#include <vm/uma.h>
#endif
static int pipe_read(struct fileproc *fp, struct uio *uio,
kauth_cred_t cred, int flags, struct proc *p);
static int pipe_write(struct fileproc *fp, struct uio *uio,
kauth_cred_t cred, int flags, struct proc *p);
static int pipe_close(struct fileglob *fg, struct proc *p);
static int pipe_select(struct fileproc *fp, int which, void * wql, struct proc *p);
static int pipe_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p);
static int pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p);
struct fileops pipeops =
{ pipe_read,
pipe_write,
pipe_ioctl,
pipe_select,
pipe_close,
pipe_kqfilter,
0 };
static void filt_pipedetach(struct knote *kn);
static int filt_piperead(struct knote *kn, long hint);
static int filt_pipewrite(struct knote *kn, long hint);
static struct filterops pipe_rfiltops =
{ 1, NULL, filt_pipedetach, filt_piperead };
static struct filterops pipe_wfiltops =
{ 1, NULL, filt_pipedetach, filt_pipewrite };
#define MINPIPESIZE (PIPE_SIZE/3)
#define LIMITBIGPIPES 32
static int nbigpipe;
static int amountpipes;
static int amountpipekva;
#ifndef PIPE_NODIRECT
static int amountpipekvawired;
#endif
int maxpipekva = 1024 * 1024 * 16;
#if PIPE_SYSCTLS
SYSCTL_DECL(_kern_ipc);
SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RD,
&maxpipekva, 0, "Pipe KVA limit");
SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekvawired, CTLFLAG_RW,
&maxpipekvawired, 0, "Pipe KVA wired limit");
SYSCTL_INT(_kern_ipc, OID_AUTO, pipes, CTLFLAG_RD,
&amountpipes, 0, "Current # of pipes");
SYSCTL_INT(_kern_ipc, OID_AUTO, bigpipes, CTLFLAG_RD,
&nbigpipe, 0, "Current # of big pipes");
SYSCTL_INT(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD,
&amountpipekva, 0, "Pipe KVA usage");
SYSCTL_INT(_kern_ipc, OID_AUTO, pipekvawired, CTLFLAG_RD,
&amountpipekvawired, 0, "Pipe wired KVA usage");
#endif
void pipeinit(void *dummy __unused);
static void pipeclose(struct pipe *cpipe);
static void pipe_free_kmem(struct pipe *cpipe);
static int pipe_create(struct pipe **cpipep);
static void pipeselwakeup(struct pipe *cpipe, struct pipe *spipe);
static __inline int pipelock(struct pipe *cpipe, int catch);
static __inline void pipeunlock(struct pipe *cpipe);
#ifndef PIPE_NODIRECT
static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
static void pipe_destroy_write_buffer(struct pipe *wpipe);
static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
static void pipe_clone_write_buffer(struct pipe *wpipe);
#endif
extern int postpipeevent(struct pipe *, int);
extern void evpipefree(struct pipe *cpipe);
static int pipespace(struct pipe *cpipe, int size);
static lck_grp_t *pipe_mtx_grp;
static lck_attr_t *pipe_mtx_attr;
static lck_grp_attr_t *pipe_mtx_grp_attr;
static zone_t pipe_zone;
SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL);
void
pipeinit(void *dummy __unused)
{
pipe_zone = (zone_t)zinit(sizeof(struct pipe), 8192 * sizeof(struct pipe), 4096, "pipe zone");
pipe_mtx_grp_attr = lck_grp_attr_alloc_init();
pipe_mtx_grp = lck_grp_alloc_init("pipe", pipe_mtx_grp_attr);
pipe_mtx_attr = lck_attr_alloc_init();
}
int
pipe(struct proc *p, __unused struct pipe_args *uap, register_t *retval)
{
struct fileproc *rf, *wf;
struct pipe *rpipe, *wpipe;
lck_mtx_t *pmtx;
int fd, error;
if ((pmtx = lck_mtx_alloc_init(pipe_mtx_grp, pipe_mtx_attr)) == NULL)
return (ENOMEM);
rpipe = wpipe = NULL;
if (pipe_create(&rpipe) || pipe_create(&wpipe)) {
error = ENFILE;
goto freepipes;
}
if (amountpipekva > maxpipekva / 2)
error = pipespace(rpipe, SMALL_PIPE_SIZE);
else
error = pipespace(rpipe, PIPE_SIZE);
if (error)
goto freepipes;
#ifndef PIPE_NODIRECT
rpipe->pipe_state |= PIPE_DIRECTOK;
wpipe->pipe_state |= PIPE_DIRECTOK;
#endif
TAILQ_INIT(&rpipe->pipe_evlist);
TAILQ_INIT(&wpipe->pipe_evlist);
error = falloc(p, &rf, &fd);
if (error) {
goto freepipes;
}
retval[0] = fd;
rf->f_flag = FREAD;
rf->f_type = DTYPE_PIPE;
rf->f_data = (caddr_t)rpipe;
rf->f_ops = &pipeops;
error = falloc(p, &wf, &fd);
if (error) {
fp_free(p, retval[0], rf);
goto freepipes;
}
wf->f_flag = FWRITE;
wf->f_type = DTYPE_PIPE;
wf->f_data = (caddr_t)wpipe;
wf->f_ops = &pipeops;
rpipe->pipe_peer = wpipe;
wpipe->pipe_peer = rpipe;
rpipe->pipe_mtxp = wpipe->pipe_mtxp = pmtx;
retval[1] = fd;
#ifdef MAC
mac_init_pipe(rpipe);
mac_create_pipe(td->td_ucred, rpipe);
#endif
proc_fdlock(p);
procfdtbl_releasefd(p, retval[0], NULL);
procfdtbl_releasefd(p, retval[1], NULL);
fp_drop(p, retval[0], rf, 1);
fp_drop(p, retval[1], wf, 1);
proc_fdunlock(p);
return (0);
freepipes:
pipeclose(rpipe);
pipeclose(wpipe);
lck_mtx_free(pmtx, pipe_mtx_grp);
return (error);
}
int
pipe_stat(struct pipe *cpipe, struct stat *ub)
{
#ifdef MAC
int error;
#endif
struct timeval now;
if (cpipe == NULL)
return (EBADF);
#ifdef MAC
PIPE_LOCK(cpipe);
error = mac_check_pipe_stat(active_cred, cpipe);
PIPE_UNLOCK(cpipe);
if (error)
return (error);
#endif
if (cpipe->pipe_buffer.buffer == 0) {
cpipe = cpipe->pipe_peer;
if (cpipe == NULL)
return (EBADF);
}
bzero(ub, sizeof(*ub));
ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
ub->st_blksize = cpipe->pipe_buffer.size;
ub->st_size = cpipe->pipe_buffer.cnt;
ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize;
ub->st_nlink = 1;
ub->st_uid = kauth_getuid();
ub->st_gid = kauth_getgid();
microtime(&now);
ub->st_atimespec.tv_sec = now.tv_sec;
ub->st_atimespec.tv_nsec = now.tv_usec * 1000;
ub->st_mtimespec.tv_sec = now.tv_sec;
ub->st_mtimespec.tv_nsec = now.tv_usec * 1000;
ub->st_ctimespec.tv_sec = now.tv_sec;
ub->st_ctimespec.tv_nsec = now.tv_usec * 1000;
return (0);
}
static int
pipespace(struct pipe *cpipe, int size)
{
vm_offset_t buffer;
size = round_page(size);
if (kmem_alloc(kernel_map, &buffer, size) != KERN_SUCCESS)
return(ENOMEM);
pipe_free_kmem(cpipe);
cpipe->pipe_buffer.buffer = (caddr_t)buffer;
cpipe->pipe_buffer.size = size;
cpipe->pipe_buffer.in = 0;
cpipe->pipe_buffer.out = 0;
cpipe->pipe_buffer.cnt = 0;
OSAddAtomic(1, (SInt32 *)&amountpipes);
OSAddAtomic(cpipe->pipe_buffer.size, (SInt32 *)&amountpipekva);
return (0);
}
static int
pipe_create(struct pipe **cpipep)
{
struct pipe *cpipe;
cpipe = (struct pipe *)zalloc(pipe_zone);
if ((*cpipep = cpipe) == NULL)
return (ENOMEM);
bzero(cpipe, sizeof *cpipe);
return (0);
}
static __inline int
pipelock(cpipe, catch)
struct pipe *cpipe;
int catch;
{
int error;
while (cpipe->pipe_state & PIPE_LOCKFL) {
cpipe->pipe_state |= PIPE_LWANT;
error = msleep(cpipe, PIPE_MTX(cpipe), catch ? (PRIBIO | PCATCH) : PRIBIO,
"pipelk", 0);
if (error != 0)
return (error);
}
cpipe->pipe_state |= PIPE_LOCKFL;
return (0);
}
static __inline void
pipeunlock(cpipe)
struct pipe *cpipe;
{
cpipe->pipe_state &= ~PIPE_LOCKFL;
if (cpipe->pipe_state & PIPE_LWANT) {
cpipe->pipe_state &= ~PIPE_LWANT;
wakeup(cpipe);
}
}
static void
pipeselwakeup(cpipe, spipe)
struct pipe *cpipe;
struct pipe *spipe;
{
if (cpipe->pipe_state & PIPE_SEL) {
cpipe->pipe_state &= ~PIPE_SEL;
selwakeup(&cpipe->pipe_sel);
}
if (cpipe->pipe_state & PIPE_KNOTE)
KNOTE(&cpipe->pipe_sel.si_note, 1);
postpipeevent(cpipe, EV_RWBYTES);
if (spipe && (spipe->pipe_state & PIPE_ASYNC) && spipe->pipe_pgid) {
struct proc *p;
if (spipe->pipe_pgid < 0)
gsignal(-spipe->pipe_pgid, SIGIO);
else if ((p = pfind(spipe->pipe_pgid)) != (struct proc *)0)
psignal(p, SIGIO);
}
}
static int
pipe_read(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cred, __unused int flags, __unused struct proc *p)
{
struct pipe *rpipe = (struct pipe *)fp->f_data;
int error;
int nread = 0;
u_int size;
PIPE_LOCK(rpipe);
++rpipe->pipe_busy;
error = pipelock(rpipe, 1);
if (error)
goto unlocked_error;
#ifdef MAC
error = mac_check_pipe_read(active_cred, rpipe);
if (error)
goto locked_error;
#endif
while (uio_resid(uio)) {
if (rpipe->pipe_buffer.cnt > 0) {
size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;
if (size > rpipe->pipe_buffer.cnt)
size = rpipe->pipe_buffer.cnt;
if (size > (u_int) uio_resid(uio))
size = (u_int) uio_resid(uio);
PIPE_UNLOCK(rpipe);
error = uiomove(
&rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],
size, uio);
PIPE_LOCK(rpipe);
if (error)
break;
rpipe->pipe_buffer.out += size;
if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)
rpipe->pipe_buffer.out = 0;
rpipe->pipe_buffer.cnt -= size;
if (rpipe->pipe_buffer.cnt == 0) {
rpipe->pipe_buffer.in = 0;
rpipe->pipe_buffer.out = 0;
}
nread += size;
#ifndef PIPE_NODIRECT
} else if ((size = rpipe->pipe_map.cnt) &&
(rpipe->pipe_state & PIPE_DIRECTW)) {
caddr_t va;
if (size > (u_int) uio_resid(uio))
size = (u_int) uio_resid(uio);
va = (caddr_t) rpipe->pipe_map.kva +
rpipe->pipe_map.pos;
PIPE_UNLOCK(rpipe);
error = uiomove(va, size, uio);
PIPE_LOCK(rpipe);
if (error)
break;
nread += size;
rpipe->pipe_map.pos += size;
rpipe->pipe_map.cnt -= size;
if (rpipe->pipe_map.cnt == 0) {
rpipe->pipe_state &= ~PIPE_DIRECTW;
wakeup(rpipe);
}
#endif
} else {
if (rpipe->pipe_state & PIPE_EOF)
break;
if (rpipe->pipe_state & PIPE_WANTW) {
rpipe->pipe_state &= ~PIPE_WANTW;
wakeup(rpipe);
}
if (nread > 0)
break;
pipeunlock(rpipe);
if (fp->f_flag & FNONBLOCK) {
error = EAGAIN;
} else {
rpipe->pipe_state |= PIPE_WANTR;
error = msleep(rpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, "piperd", 0);
if (error == 0)
error = pipelock(rpipe, 1);
}
if (error)
goto unlocked_error;
}
}
#ifdef MAC
locked_error:
#endif
pipeunlock(rpipe);
unlocked_error:
--rpipe->pipe_busy;
if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {
rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);
wakeup(rpipe);
} else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {
if (rpipe->pipe_state & PIPE_WANTW) {
rpipe->pipe_state &= ~PIPE_WANTW;
wakeup(rpipe);
}
}
if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF)
pipeselwakeup(rpipe, rpipe->pipe_peer);
PIPE_UNLOCK(rpipe);
return (error);
}
#ifndef PIPE_NODIRECT
static int
pipe_build_write_buffer(wpipe, uio)
struct pipe *wpipe;
struct uio *uio;
{
pmap_t pmap;
u_int size;
int i, j;
vm_offset_t addr, endaddr;
size = (u_int) uio->uio_iov->iov_len;
if (size > wpipe->pipe_buffer.size)
size = wpipe->pipe_buffer.size;
pmap = vmspace_pmap(curproc->p_vmspace);
endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size);
addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base);
for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) {
race:
if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0) {
vm_page_lock_queues();
for (j = 0; j < i; j++)
vm_page_unhold(wpipe->pipe_map.ms[j]);
vm_page_unlock_queues();
return (EFAULT);
}
wpipe->pipe_map.ms[i] = pmap_extract_and_hold(pmap, addr,
VM_PROT_READ);
if (wpipe->pipe_map.ms[i] == NULL)
goto race;
}
wpipe->pipe_map.npages = i;
wpipe->pipe_map.pos =
((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK;
wpipe->pipe_map.cnt = size;
if (wpipe->pipe_map.kva == 0) {
wpipe->pipe_map.kva = kmem_alloc_nofault(kernel_map,
wpipe->pipe_buffer.size + PAGE_SIZE);
atomic_add_int(&amountpipekvawired,
wpipe->pipe_buffer.size + PAGE_SIZE);
}
pmap_qenter(wpipe->pipe_map.kva, wpipe->pipe_map.ms,
wpipe->pipe_map.npages);
uio->uio_iov->iov_len -= size;
uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size;
if (uio->uio_iov->iov_len == 0)
uio->uio_iov++;
uio_setresid(uio, (uio_resid(uio) - size));
uio->uio_offset += size;
return (0);
}
static void
pipe_destroy_write_buffer(wpipe)
struct pipe *wpipe;
{
int i;
if (wpipe->pipe_map.kva) {
pmap_qremove(wpipe->pipe_map.kva, wpipe->pipe_map.npages);
if (amountpipekvawired > maxpipekvawired / 2) {
vm_offset_t kva = wpipe->pipe_map.kva;
wpipe->pipe_map.kva = 0;
kmem_free(kernel_map, kva,
wpipe->pipe_buffer.size + PAGE_SIZE);
atomic_subtract_int(&amountpipekvawired,
wpipe->pipe_buffer.size + PAGE_SIZE);
}
}
vm_page_lock_queues();
for (i = 0; i < wpipe->pipe_map.npages; i++) {
vm_page_unhold(wpipe->pipe_map.ms[i]);
}
vm_page_unlock_queues();
wpipe->pipe_map.npages = 0;
}
static void
pipe_clone_write_buffer(wpipe)
struct pipe *wpipe;
{
int size;
int pos;
size = wpipe->pipe_map.cnt;
pos = wpipe->pipe_map.pos;
wpipe->pipe_buffer.in = size;
wpipe->pipe_buffer.out = 0;
wpipe->pipe_buffer.cnt = size;
wpipe->pipe_state &= ~PIPE_DIRECTW;
PIPE_UNLOCK(wpipe);
bcopy((caddr_t) wpipe->pipe_map.kva + pos,
wpipe->pipe_buffer.buffer, size);
pipe_destroy_write_buffer(wpipe);
PIPE_LOCK(wpipe);
}
static int
pipe_direct_write(wpipe, uio)
struct pipe *wpipe;
struct uio *uio;
{
int error;
retry:
while (wpipe->pipe_state & PIPE_DIRECTW) {
if (wpipe->pipe_state & PIPE_WANTR) {
wpipe->pipe_state &= ~PIPE_WANTR;
wakeup(wpipe);
}
wpipe->pipe_state |= PIPE_WANTW;
error = msleep(wpipe, PIPE_MTX(wpipe),
PRIBIO | PCATCH, "pipdww", 0);
if (error)
goto error1;
if (wpipe->pipe_state & PIPE_EOF) {
error = EPIPE;
goto error1;
}
}
wpipe->pipe_map.cnt = 0;
if (wpipe->pipe_buffer.cnt > 0) {
if (wpipe->pipe_state & PIPE_WANTR) {
wpipe->pipe_state &= ~PIPE_WANTR;
wakeup(wpipe);
}
wpipe->pipe_state |= PIPE_WANTW;
error = msleep(wpipe, PIPE_MTX(wpipe),
PRIBIO | PCATCH, "pipdwc", 0);
if (error)
goto error1;
if (wpipe->pipe_state & PIPE_EOF) {
error = EPIPE;
goto error1;
}
goto retry;
}
wpipe->pipe_state |= PIPE_DIRECTW;
pipelock(wpipe, 0);
PIPE_UNLOCK(wpipe);
error = pipe_build_write_buffer(wpipe, uio);
PIPE_LOCK(wpipe);
pipeunlock(wpipe);
if (error) {
wpipe->pipe_state &= ~PIPE_DIRECTW;
goto error1;
}
error = 0;
while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) {
if (wpipe->pipe_state & PIPE_EOF) {
pipelock(wpipe, 0);
PIPE_UNLOCK(wpipe);
pipe_destroy_write_buffer(wpipe);
PIPE_LOCK(wpipe);
pipeselwakeup(wpipe, wpipe);
pipeunlock(wpipe);
error = EPIPE;
goto error1;
}
if (wpipe->pipe_state & PIPE_WANTR) {
wpipe->pipe_state &= ~PIPE_WANTR;
wakeup(wpipe);
}
pipeselwakeup(wpipe, wpipe);
error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,
"pipdwt", 0);
}
pipelock(wpipe,0);
if (wpipe->pipe_state & PIPE_DIRECTW) {
pipe_clone_write_buffer(wpipe);
} else {
PIPE_UNLOCK(wpipe);
pipe_destroy_write_buffer(wpipe);
PIPE_LOCK(wpipe);
}
pipeunlock(wpipe);
return (error);
error1:
wakeup(wpipe);
return (error);
}
#endif
static int
pipe_write(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cred, __unused int flags, __unused struct proc *p)
{
int error = 0;
int orig_resid;
int pipe_size;
struct pipe *wpipe, *rpipe;
rpipe = (struct pipe *)fp->f_data;
PIPE_LOCK(rpipe);
wpipe = rpipe->pipe_peer;
if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF)) {
PIPE_UNLOCK(rpipe);
return (EPIPE);
}
#ifdef MAC
error = mac_check_pipe_write(active_cred, wpipe);
if (error) {
PIPE_UNLOCK(rpipe);
return (error);
}
#endif
++wpipe->pipe_busy;
pipe_size = 0;
if (wpipe->pipe_buffer.buffer == 0) {
if (amountpipekva > maxpipekva / 2)
pipe_size = SMALL_PIPE_SIZE;
else
pipe_size = PIPE_SIZE;
}
if ((uio_resid(uio) > PIPE_SIZE) &&
(wpipe->pipe_buffer.size <= PIPE_SIZE) &&
(amountpipekva < maxpipekva / 2) &&
(nbigpipe < LIMITBIGPIPES) &&
#ifndef PIPE_NODIRECT
(wpipe->pipe_state & PIPE_DIRECTW) == 0 &&
#endif
(wpipe->pipe_buffer.cnt == 0)) {
pipe_size = BIG_PIPE_SIZE;
}
if (pipe_size) {
if ((error = pipelock(wpipe, 1)) == 0) {
PIPE_UNLOCK(wpipe);
if (pipespace(wpipe, pipe_size) == 0)
OSAddAtomic(1, (SInt32 *)&nbigpipe);
PIPE_LOCK(wpipe);
pipeunlock(wpipe);
if (wpipe->pipe_buffer.buffer == 0) {
error = ENOMEM;
}
}
if (error) {
--wpipe->pipe_busy;
if ((wpipe->pipe_busy == 0) &&
(wpipe->pipe_state & PIPE_WANT)) {
wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
wakeup(wpipe);
}
PIPE_UNLOCK(rpipe);
return(error);
}
}
orig_resid = uio_resid(uio);
while (uio_resid(uio)) {
int space;
#ifndef PIPE_NODIRECT
if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) &&
(fp->f_flag & FNONBLOCK) == 0 &&
amountpipekvawired + uio->uio_resid < maxpipekvawired) {
error = pipe_direct_write(wpipe, uio);
if (error)
break;
continue;
}
retrywrite:
while (wpipe->pipe_state & PIPE_DIRECTW) {
if (wpipe->pipe_state & PIPE_WANTR) {
wpipe->pipe_state &= ~PIPE_WANTR;
wakeup(wpipe);
}
error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, "pipbww", 0);
if (wpipe->pipe_state & PIPE_EOF)
break;
if (error)
break;
}
#else
retrywrite:
#endif
space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
if ((space < uio_resid(uio)) && (orig_resid <= PIPE_BUF))
space = 0;
if (space > 0) {
if ((error = pipelock(wpipe,1)) == 0) {
int size;
int segsize;
if (wpipe->pipe_state & PIPE_EOF) {
pipeunlock(wpipe);
error = EPIPE;
break;
}
#ifndef PIPE_NODIRECT
if (wpipe->pipe_state & PIPE_DIRECTW) {
pipeunlock(wpipe);
goto retrywrite;
}
#endif
if (space > (int)(wpipe->pipe_buffer.size -
wpipe->pipe_buffer.cnt)) {
pipeunlock(wpipe);
goto retrywrite;
}
if (space > uio_resid(uio))
size = uio_resid(uio);
else
size = space;
segsize = wpipe->pipe_buffer.size -
wpipe->pipe_buffer.in;
if (segsize > size)
segsize = size;
PIPE_UNLOCK(rpipe);
error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],
segsize, uio);
PIPE_LOCK(rpipe);
if (error == 0 && segsize < size) {
if (wpipe->pipe_buffer.in + segsize !=
wpipe->pipe_buffer.size)
panic("Expected pipe buffer "
"wraparound disappeared");
PIPE_UNLOCK(rpipe);
error = uiomove(
&wpipe->pipe_buffer.buffer[0],
size - segsize, uio);
PIPE_LOCK(rpipe);
}
if (error == 0) {
wpipe->pipe_buffer.in += size;
if (wpipe->pipe_buffer.in >=
wpipe->pipe_buffer.size) {
if (wpipe->pipe_buffer.in !=
size - segsize +
wpipe->pipe_buffer.size)
panic("Expected "
"wraparound bad");
wpipe->pipe_buffer.in = size -
segsize;
}
wpipe->pipe_buffer.cnt += size;
if (wpipe->pipe_buffer.cnt >
wpipe->pipe_buffer.size)
panic("Pipe buffer overflow");
}
pipeunlock(wpipe);
}
if (error)
break;
} else {
if (wpipe->pipe_state & PIPE_WANTR) {
wpipe->pipe_state &= ~PIPE_WANTR;
wakeup(wpipe);
}
if (fp->f_flag & FNONBLOCK) {
error = EAGAIN;
break;
}
pipeselwakeup(wpipe, wpipe);
wpipe->pipe_state |= PIPE_WANTW;
error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, "pipewr", 0);
if (error != 0)
break;
if (wpipe->pipe_state & PIPE_EOF) {
error = EPIPE;
break;
}
}
}
--wpipe->pipe_busy;
if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {
wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
wakeup(wpipe);
}
if (wpipe->pipe_buffer.cnt > 0) {
if (wpipe->pipe_state & PIPE_WANTR) {
wpipe->pipe_state &= ~PIPE_WANTR;
wakeup(wpipe);
}
pipeselwakeup(wpipe, wpipe);
}
PIPE_UNLOCK(rpipe);
return (error);
}
static int
pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, __unused struct proc *p)
{
struct pipe *mpipe = (struct pipe *)fp->f_data;
#ifdef MAC
int error;
#endif
PIPE_LOCK(mpipe);
#ifdef MAC
error = mac_check_pipe_ioctl(active_cred, mpipe, cmd, data);
if (error) {
PIPE_UNLOCK(mpipe);
return (error);
}
#endif
switch (cmd) {
case FIONBIO:
PIPE_UNLOCK(mpipe);
return (0);
case FIOASYNC:
if (*(int *)data) {
mpipe->pipe_state |= PIPE_ASYNC;
} else {
mpipe->pipe_state &= ~PIPE_ASYNC;
}
PIPE_UNLOCK(mpipe);
return (0);
case FIONREAD:
#ifndef PIPE_NODIRECT
if (mpipe->pipe_state & PIPE_DIRECTW)
*(int *)data = mpipe->pipe_map.cnt;
else
#endif
*(int *)data = mpipe->pipe_buffer.cnt;
PIPE_UNLOCK(mpipe);
return (0);
case TIOCSPGRP:
mpipe->pipe_pgid = *(int *)data;
PIPE_UNLOCK(mpipe);
return (0);
case TIOCGPGRP:
*(int *)data = mpipe->pipe_pgid;
PIPE_UNLOCK(mpipe);
return (0);
}
PIPE_UNLOCK(mpipe);
return (ENOTTY);
}
static int
pipe_select(struct fileproc *fp, int which, void *wql, struct proc *p)
{
struct pipe *rpipe = (struct pipe *)fp->f_data;
struct pipe *wpipe;
int retnum = 0;
if (rpipe == NULL || rpipe == (struct pipe *)-1)
return (retnum);
PIPE_LOCK(rpipe);
wpipe = rpipe->pipe_peer;
switch (which) {
case FREAD:
if ((rpipe->pipe_state & PIPE_DIRECTW) ||
(rpipe->pipe_buffer.cnt > 0) ||
(rpipe->pipe_state & PIPE_EOF)) {
retnum = 1;
} else {
rpipe->pipe_state |= PIPE_SEL;
selrecord(p, &rpipe->pipe_sel, wql);
}
break;
case FWRITE:
if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) ||
(((wpipe->pipe_state & PIPE_DIRECTW) == 0) &&
(wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) {
retnum = 1;
} else {
wpipe->pipe_state |= PIPE_SEL;
selrecord(p, &wpipe->pipe_sel, wql);
}
break;
case 0:
rpipe->pipe_state |= PIPE_SEL;
selrecord(p, &rpipe->pipe_sel, wql);
break;
}
PIPE_UNLOCK(rpipe);
return (retnum);
}
static int
pipe_close(struct fileglob *fg, __unused struct proc *p)
{
struct pipe *cpipe;
proc_fdlock(p);
cpipe = (struct pipe *)fg->fg_data;
fg->fg_data = NULL;
proc_fdunlock(p);
if (cpipe)
pipeclose(cpipe);
return (0);
}
static void
pipe_free_kmem(struct pipe *cpipe)
{
if (cpipe->pipe_buffer.buffer != NULL) {
if (cpipe->pipe_buffer.size > PIPE_SIZE)
OSAddAtomic(-1, (SInt32 *)&nbigpipe);
OSAddAtomic(cpipe->pipe_buffer.size, (SInt32 *)&amountpipekva);
OSAddAtomic(-1, (SInt32 *)&amountpipes);
kmem_free(kernel_map, (vm_offset_t)cpipe->pipe_buffer.buffer,
cpipe->pipe_buffer.size);
cpipe->pipe_buffer.buffer = NULL;
}
#ifndef PIPE_NODIRECT
if (cpipe->pipe_map.kva != 0) {
atomic_subtract_int(&amountpipekvawired,
cpipe->pipe_buffer.size + PAGE_SIZE);
kmem_free(kernel_map,
cpipe->pipe_map.kva,
cpipe->pipe_buffer.size + PAGE_SIZE);
cpipe->pipe_map.cnt = 0;
cpipe->pipe_map.kva = 0;
cpipe->pipe_map.pos = 0;
cpipe->pipe_map.npages = 0;
}
#endif
}
static void
pipeclose(struct pipe *cpipe)
{
struct pipe *ppipe;
if (cpipe == NULL)
return;
if (PIPE_MTX(cpipe) != NULL)
PIPE_LOCK(cpipe);
pipeselwakeup(cpipe, cpipe);
while (cpipe->pipe_busy) {
cpipe->pipe_state |= PIPE_WANT | PIPE_EOF;
wakeup(cpipe);
msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0);
}
#ifdef MAC
if (cpipe->pipe_label != NULL && cpipe->pipe_peer == NULL)
mac_destroy_pipe(cpipe);
#endif
if ((ppipe = cpipe->pipe_peer) != NULL) {
ppipe->pipe_state |= PIPE_EOF;
pipeselwakeup(ppipe, ppipe);
wakeup(ppipe);
if (cpipe->pipe_state & PIPE_KNOTE)
KNOTE(&ppipe->pipe_sel.si_note, 1);
postpipeevent(ppipe, EV_RCLOSED);
ppipe->pipe_peer = NULL;
}
evpipefree(cpipe);
if (PIPE_MTX(cpipe) != NULL) {
if (ppipe != NULL) {
PIPE_UNLOCK(cpipe);
} else {
lck_mtx_free(PIPE_MTX(cpipe), pipe_mtx_grp);
}
}
pipe_free_kmem(cpipe);
zfree(pipe_zone, cpipe);
}
static int
pipe_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused struct proc *p)
{
struct pipe *cpipe;
cpipe = (struct pipe *)kn->kn_fp->f_data;
PIPE_LOCK(cpipe);
switch (kn->kn_filter) {
case EVFILT_READ:
kn->kn_fop = &pipe_rfiltops;
break;
case EVFILT_WRITE:
kn->kn_fop = &pipe_wfiltops;
if (cpipe->pipe_peer == NULL) {
PIPE_UNLOCK(cpipe);
return (EPIPE);
}
cpipe = cpipe->pipe_peer;
break;
default:
PIPE_UNLOCK(cpipe);
return (1);
}
if (KNOTE_ATTACH(&cpipe->pipe_sel.si_note, kn))
cpipe->pipe_state |= PIPE_KNOTE;
PIPE_UNLOCK(cpipe);
return (0);
}
static void
filt_pipedetach(struct knote *kn)
{
struct pipe *cpipe = (struct pipe *)kn->kn_fp->f_data;
PIPE_LOCK(cpipe);
if (kn->kn_filter == EVFILT_WRITE) {
if (cpipe->pipe_peer == NULL) {
PIPE_UNLOCK(cpipe);
return;
}
cpipe = cpipe->pipe_peer;
}
if (cpipe->pipe_state & PIPE_KNOTE) {
if (KNOTE_DETACH(&cpipe->pipe_sel.si_note, kn))
cpipe->pipe_state &= ~PIPE_KNOTE;
}
PIPE_UNLOCK(cpipe);
}
static int
filt_piperead(struct knote *kn, long hint)
{
struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data;
struct pipe *wpipe;
int retval;
if (hint == 0)
PIPE_LOCK(rpipe);
wpipe = rpipe->pipe_peer;
kn->kn_data = rpipe->pipe_buffer.cnt;
#ifndef PIPE_NODIRECT
if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW))
kn->kn_data = rpipe->pipe_map.cnt;
#endif
if ((rpipe->pipe_state & PIPE_EOF) ||
(wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
kn->kn_flags |= EV_EOF;
retval = 1;
} else
retval = (kn->kn_sfflags & NOTE_LOWAT) ?
(kn->kn_data >= kn->kn_sdata) : (kn->kn_data > 0);
if (hint == 0)
PIPE_UNLOCK(rpipe);
return (retval);
}
static int
filt_pipewrite(struct knote *kn, long hint)
{
struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data;
struct pipe *wpipe;
if (hint == 0)
PIPE_LOCK(rpipe);
wpipe = rpipe->pipe_peer;
if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
kn->kn_data = 0;
kn->kn_flags |= EV_EOF;
if (hint == 0)
PIPE_UNLOCK(rpipe);
return (1);
}
kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
#ifndef PIPE_NODIRECT
if (wpipe->pipe_state & PIPE_DIRECTW)
kn->kn_data = 0;
#endif
if (hint == 0)
PIPE_UNLOCK(rpipe);
return (kn->kn_data >= ((kn->kn_sfflags & NOTE_LOWAT) ?
kn->kn_sdata : PIPE_BUF));
}