#include <mach/boolean.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/user.h>
#include <sys/proc.h>
#include <sys/kauth.h>
#include <sys/buf.h>
#include <sys/uio.h>
#include <sys/vnode_internal.h>
#include <sys/namei.h>
#include <sys/mount_internal.h>
#include <sys/ubc_internal.h>
#include <sys/lock.h>
#include <mach/mach_types.h>
#include <mach/memory_object_types.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>
#include <libkern/libkern.h>
#include <vm/vnode_pager.h>
#include <vm/vm_pageout.h>
#include <kern/assert.h>
#include <sys/kdebug.h>
#include <machine/spl.h>
#include <nfs/rpcv2.h>
#include <nfs/nfsproto.h>
#include <nfs/nfs.h>
#include <vm/vm_protos.h>
unsigned int vp_pagein=0;
unsigned int vp_pgodirty=0;
unsigned int vp_pgoclean=0;
unsigned int dp_pgouts=0;
unsigned int dp_pgins=0;
vm_object_offset_t
vnode_pager_get_filesize(struct vnode *vp)
{
return (vm_object_offset_t) ubc_getsize(vp);
}
pager_return_t
vnode_pageout(struct vnode *vp,
upl_t upl,
vm_offset_t upl_offset,
vm_object_offset_t f_offset,
vm_size_t size,
int flags,
int *errorp)
{
struct proc *p = current_proc();
int result = PAGER_SUCCESS;
int error = 0;
int error_ret = 0;
daddr64_t blkno;
int isize;
int pg_index;
int base_index;
int offset;
upl_page_info_t *pl;
struct vfs_context context;
context.vc_proc = p;
context.vc_ucred = kauth_cred_get();
isize = (int)size;
if (isize <= 0) {
result = PAGER_ERROR;
error_ret = EINVAL;
goto out;
}
UBCINFOCHECK("vnode_pageout", vp);
if (UBCINVALID(vp)) {
result = PAGER_ERROR;
error_ret = EINVAL;
if (upl && !(flags & UPL_NOCOMMIT))
ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
goto out;
}
if ( !(flags & UPL_VNODE_PAGER)) {
dp_pgouts++;
KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
size, 1, 0, 0, 0);
if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
(size_t)size, flags, &context)) )
result = PAGER_ERROR;
KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
size, 1, 0, 0, 0);
goto out;
}
pl = ubc_upl_pageinfo(upl);
if (ubc_getsize(vp) == 0) {
for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) {
#if NFSCLIENT
if (vp->v_tag == VT_NFS)
error = nfs_buf_page_inval(vp, (off_t)f_offset);
else
#endif
{
blkno = ubc_offtoblk(vp, (off_t)f_offset);
error = buf_invalblkno(vp, blkno, 0);
}
if (error) {
if ( !(flags & UPL_NOCOMMIT))
ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
if (error_ret == 0)
error_ret = error;
result = PAGER_ERROR;
} else if ( !(flags & UPL_NOCOMMIT)) {
ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
}
f_offset += PAGE_SIZE;
}
goto out;
}
base_index = upl_offset / PAGE_SIZE;
for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
if (upl_page_present(pl, --pg_index))
break;
if (pg_index == base_index) {
if ( !(flags & UPL_NOCOMMIT))
ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
goto out;
}
}
isize = (pg_index + 1) * PAGE_SIZE;
offset = upl_offset;
pg_index = base_index;
while (isize) {
int xsize;
int num_of_pages;
if ( !upl_page_present(pl, pg_index)) {
offset += PAGE_SIZE;
isize -= PAGE_SIZE;
pg_index++;
continue;
}
if ( !upl_dirty_page(pl, pg_index)) {
vp_pgoclean++;
#if NFSCLIENT
if (vp->v_tag == VT_NFS)
error = nfs_buf_page_inval(vp, (off_t)(f_offset + offset));
else
#endif
{
blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset));
error = buf_invalblkno(vp, blkno, 0);
}
if (error) {
if ( !(flags & UPL_NOCOMMIT))
ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
if (error_ret == 0)
error_ret = error;
result = PAGER_ERROR;
} else if ( !(flags & UPL_NOCOMMIT)) {
ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
}
offset += PAGE_SIZE;
isize -= PAGE_SIZE;
pg_index++;
continue;
}
vp_pgodirty++;
num_of_pages = 1;
xsize = isize - PAGE_SIZE;
while (xsize) {
if ( !upl_dirty_page(pl, pg_index + num_of_pages))
break;
num_of_pages++;
xsize -= PAGE_SIZE;
}
xsize = num_of_pages * PAGE_SIZE;
KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
xsize, (int)(f_offset + offset), 0, 0, 0);
if ( (error = VNOP_PAGEOUT(vp, upl, (vm_offset_t)offset,
(off_t)(f_offset + offset), xsize,
flags, &context)) ) {
if (error_ret == 0)
error_ret = error;
result = PAGER_ERROR;
}
KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
xsize, 0, 0, 0, 0);
offset += xsize;
isize -= xsize;
pg_index += num_of_pages;
}
out:
if (errorp)
*errorp = error_ret;
return (result);
}
void IOSleep(int);
pager_return_t
vnode_pagein(
struct vnode *vp,
upl_t upl,
vm_offset_t upl_offset,
vm_object_offset_t f_offset,
vm_size_t size,
int flags,
int *errorp)
{
struct proc *p = current_proc();
struct uthread *ut;
upl_page_info_t *pl;
int result = PAGER_SUCCESS;
int error = 0;
int pages_in_upl;
int start_pg;
int last_pg;
int first_pg;
int xsize;
int abort_needed = 1;
UBCINFOCHECK("vnode_pagein", vp);
if (UBCINVALID(vp)) {
result = PAGER_ERROR;
error = PAGER_ERROR;
if (upl && !(flags & UPL_NOCOMMIT)) {
ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
}
goto out;
}
if (upl == (upl_t)NULL) {
if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) {
result = PAGER_ERROR;
error = PAGER_ERROR;
goto out;
}
ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
if (upl == (upl_t)NULL) {
result = PAGER_ABSENT;
error = PAGER_ABSENT;
goto out;
}
upl_offset = 0;
flags &= ~UPL_NOCOMMIT;
vp_pagein++;
} else {
pl = ubc_upl_pageinfo(upl);
dp_pgins++;
}
pages_in_upl = size / PAGE_SIZE;
first_pg = upl_offset / PAGE_SIZE;
for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
if (upl_page_present(pl, last_pg))
break;
}
pages_in_upl = last_pg + 1;
for (last_pg = first_pg; last_pg < pages_in_upl;) {
for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
if (upl_page_present(pl, last_pg))
break;
}
if (last_pg == pages_in_upl)
break;
for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
break;
}
if (last_pg > start_pg) {
xsize = (last_pg - start_pg) * PAGE_SIZE;
if (!(flags & UPL_NOCOMMIT))
ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
abort_needed = 0;
}
if (last_pg == pages_in_upl)
break;
if (!upl_page_present(pl, last_pg))
continue;
for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
break;
}
if (last_pg > start_pg) {
int xoff;
struct vfs_context context;
context.vc_proc = p;
context.vc_ucred = kauth_cred_get();
xsize = (last_pg - start_pg) * PAGE_SIZE;
xoff = start_pg * PAGE_SIZE;
if ( (error = VNOP_PAGEIN(vp, upl, (vm_offset_t) xoff,
(off_t)f_offset + xoff,
xsize, flags, &context)) ) {
result = PAGER_ERROR;
error = PAGER_ERROR;
}
abort_needed = 0;
}
}
if (!(flags & UPL_NOCOMMIT) && abort_needed)
ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
out:
if (errorp)
*errorp = result;
ut = get_bsdthread_info(current_thread());
if (ut->uu_lowpri_delay) {
IOSleep(ut->uu_lowpri_delay);
ut->uu_lowpri_delay = 0;
}
return (error);
}
void
vnode_pager_shutdown(void)
{
int i;
vnode_t vp;
for(i = 0; i < MAX_BACKING_STORE; i++) {
vp = (vnode_t)(bs_port_table[i]).vp;
if (vp) {
(bs_port_table[i]).vp = 0;
vnode_rele(vp);
}
}
}
void *
upl_get_internal_page_list(upl_t upl)
{
return(UPL_GET_INTERNAL_PAGE_LIST(upl));
}