#define ZALLOC_METADATA 1
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/buf.h>
#include <sys/vnode.h>
#include <sys/mount.h>
#include <sys/trace.h>
#include <sys/malloc.h>
#include <sys/resourcevar.h>
#include <miscfs/specfs/specdev.h>
#include <sys/ubc.h>
#include <vm/vm_pageout.h>
#if DIAGNOSTIC
#include <kern/assert.h>
#endif
#include <kern/task.h>
#include <kern/zalloc.h>
#include <sys/kdebug.h>
extern void bufqinc(int q);
extern void bufqdec(int q);
extern void bufq_balance_thread_init();
extern void reassignbuf(struct buf *, struct vnode *);
static struct buf *getnewbuf(int slpflag, int slptimeo, int *queue);
extern int niobuf;
int blaundrycnt;
#if TRACE
struct proc *traceproc;
int tracewhich, tracebuf[TRCSIZ];
u_int tracex;
char traceflags[TR_NFLAGS];
#endif
#define BUFHASH(dvp, lbn) \
(&bufhashtbl[((long)(dvp) / sizeof(*(dvp)) + (int)(lbn)) & bufhash])
LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
u_long bufhash;
struct bufstats bufstats;
#if 0
#define binshash(bp, dp) LIST_INSERT_HEAD(dp, bp, b_hash)
#define bremhash(bp) LIST_REMOVE(bp, b_hash)
#endif
TAILQ_HEAD(ioqueue, buf) iobufqueue;
TAILQ_HEAD(bqueues, buf) bufqueues[BQUEUES];
int needbuffer;
int need_iobuffer;
#define binsheadfree(bp, dp, whichq) do { \
TAILQ_INSERT_HEAD(dp, bp, b_freelist); \
bufqinc((whichq)); \
(bp)->b_whichq = whichq; \
(bp)->b_timestamp = time.tv_sec; \
} while (0)
#define binstailfree(bp, dp, whichq) do { \
TAILQ_INSERT_TAIL(dp, bp, b_freelist); \
bufqinc((whichq)); \
(bp)->b_whichq = whichq; \
(bp)->b_timestamp = time.tv_sec; \
} while (0)
#define BHASHENTCHECK(bp) \
if ((bp)->b_hash.le_prev != (struct buf **)0xdeadbeef) \
panic("%x: b_hash.le_prev is not deadbeef", (bp));
#define BLISTNONE(bp) \
(bp)->b_hash.le_next = (struct buf *)0; \
(bp)->b_hash.le_prev = (struct buf **)0xdeadbeef;
simple_lock_data_t bufhashlist_slock;
#define LRU_IS_STALE 120
#define AGE_IS_STALE 60
#define META_IS_STALE 180
int lru_is_stale = LRU_IS_STALE;
int age_is_stale = AGE_IS_STALE;
int meta_is_stale = META_IS_STALE;
#if 1
void
blistenterhead(struct bufhashhdr * head, struct buf * bp)
{
if ((bp->b_hash.le_next = (head)->lh_first) != NULL)
(head)->lh_first->b_hash.le_prev = &(bp)->b_hash.le_next;
(head)->lh_first = bp;
bp->b_hash.le_prev = &(head)->lh_first;
if (bp->b_hash.le_prev == (struct buf **)0xdeadbeef)
panic("blistenterhead: le_prev is deadbeef");
}
#endif
#if 1
void
binshash(struct buf *bp, struct bufhashhdr *dp)
{
int s;
struct buf *nbp;
simple_lock(&bufhashlist_slock);
#if 0
if(incore(bp->b_vp, bp->b_lblkno)) {
panic("adding to queue already existing element");
}
#endif
BHASHENTCHECK(bp);
nbp = dp->lh_first;
for(; nbp != NULL; nbp = nbp->b_hash.le_next) {
if(nbp == bp)
panic("buf already in hashlist");
}
#if 0
LIST_INSERT_HEAD(dp, bp, b_hash);
#else
blistenterhead(dp, bp);
#endif
simple_unlock(&bufhashlist_slock);
}
void
bremhash(struct buf *bp)
{
int s;
simple_lock(&bufhashlist_slock);
if (bp->b_hash.le_prev == (struct buf **)0xdeadbeef)
panic("bremhash le_prev is deadbeef");
if (bp->b_hash.le_next == bp)
panic("bremhash: next points to self");
if (bp->b_hash.le_next != NULL)
bp->b_hash.le_next->b_hash.le_prev = bp->b_hash.le_prev;
*bp->b_hash.le_prev = (bp)->b_hash.le_next;
simple_unlock(&bufhashlist_slock);
}
#endif
void
bremfree(bp)
struct buf *bp;
{
struct bqueues *dp = NULL;
int whichq = -1;
if (bp->b_freelist.tqe_next == NULL) {
for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
if (dp->tqh_last == &bp->b_freelist.tqe_next)
break;
if (dp == &bufqueues[BQUEUES])
panic("bremfree: lost tail");
}
TAILQ_REMOVE(dp, bp, b_freelist);
whichq = bp->b_whichq;
bufqdec(whichq);
bp->b_whichq = -1;
bp->b_timestamp = 0;
}
static __inline__ void
bufhdrinit(struct buf *bp)
{
bzero((char *)bp, sizeof *bp);
bp->b_dev = NODEV;
bp->b_rcred = NOCRED;
bp->b_wcred = NOCRED;
bp->b_vnbufs.le_next = NOLIST;
bp->b_flags = B_INVAL;
return;
}
void
bufinit()
{
register struct buf *bp;
register struct bqueues *dp;
register int i;
int metabuf;
long whichq;
static void bufzoneinit();
static void bcleanbuf_thread_init();
for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
TAILQ_INIT(dp);
bufhashtbl = hashinit(nbuf, M_CACHE, &bufhash);
simple_lock_init(&bufhashlist_slock );
metabuf = nbuf/8;
for (i = 0; i < nbuf; i++) {
bp = &buf[i];
bufhdrinit(bp);
if (--metabuf)
whichq = BQ_META;
else
whichq = BQ_EMPTY;
BLISTNONE(bp);
dp = &bufqueues[whichq];
binsheadfree(bp, dp, whichq);
binshash(bp, &invalhash);
}
for (; i < nbuf + niobuf; i++) {
bp = &buf[i];
bufhdrinit(bp);
binsheadfree(bp, &iobufqueue, -1);
}
printf("using %d buffer headers and %d cluster IO buffer headers\n",
nbuf, niobuf);
bufzoneinit();
bcleanbuf_thread_init();
#if 0
bufq_balance_thread_init();
#endif
}
struct buf *
bio_doread(vp, blkno, size, cred, async, queuetype)
struct vnode *vp;
daddr_t blkno;
int size;
struct ucred *cred;
int async;
int queuetype;
{
register struct buf *bp;
struct proc *p = current_proc();
bp = getblk(vp, blkno, size, 0, 0, queuetype);
if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) {
SET(bp->b_flags, B_READ | async);
if (cred != NOCRED && bp->b_rcred == NOCRED) {
bp->b_rcred = crdup(cred);
}
VOP_STRATEGY(bp);
trace(TR_BREADMISS, pack(vp, size), blkno);
if (p && p->p_stats)
p->p_stats->p_ru.ru_inblock++;
} else if (async) {
brelse(bp);
}
trace(TR_BREADHIT, pack(vp, size), blkno);
return (bp);
}
int
bread(vp, blkno, size, cred, bpp)
struct vnode *vp;
daddr_t blkno;
int size;
struct ucred *cred;
struct buf **bpp;
{
register struct buf *bp;
bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_READ);
return (biowait(bp));
}
int
meta_bread(vp, blkno, size, cred, bpp)
struct vnode *vp;
daddr_t blkno;
int size;
struct ucred *cred;
struct buf **bpp;
{
register struct buf *bp;
bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_META);
return (biowait(bp));
}
int
breadn(vp, blkno, size, rablks, rasizes, nrablks, cred, bpp)
struct vnode *vp;
daddr_t blkno; int size;
daddr_t rablks[]; int rasizes[];
int nrablks;
struct ucred *cred;
struct buf **bpp;
{
register struct buf *bp;
int i;
bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_READ);
for (i = 0; i < nrablks; i++) {
if (incore(vp, rablks[i]))
continue;
(void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC, BLK_READ);
}
return (biowait(bp));
}
int
breada(vp, blkno, size, rablkno, rabsize, cred, bpp)
struct vnode *vp;
daddr_t blkno; int size;
daddr_t rablkno; int rabsize;
struct ucred *cred;
struct buf **bpp;
{
return (breadn(vp, blkno, size, &rablkno, &rabsize, 1, cred, bpp));
}
int
bwrite(bp)
struct buf *bp;
{
int rv, sync, wasdelayed;
struct proc *p = current_proc();
upl_t upl;
upl_page_info_t *pl;
void * object;
kern_return_t kret;
struct vnode *vp = bp->b_vp;
sync = !ISSET(bp->b_flags, B_ASYNC);
wasdelayed = ISSET(bp->b_flags, B_DELWRI);
CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI));
if (!sync) {
if (wasdelayed)
reassignbuf(bp, vp);
else
if (p && p->p_stats)
p->p_stats->p_ru.ru_oublock++;
}
trace(TR_BWRITE, pack(vp, bp->b_bcount), bp->b_lblkno);
SET(bp->b_flags, B_WRITEINPROG);
vp->v_numoutput++;
VOP_STRATEGY(bp);
if (sync) {
rv = biowait(bp);
if (wasdelayed)
reassignbuf(bp, vp);
else
if (p && p->p_stats)
p->p_stats->p_ru.ru_oublock++;
brelse(bp);
return (rv);
} else {
return (0);
}
}
int
vn_bwrite(ap)
struct vop_bwrite_args *ap;
{
return (bwrite(ap->a_bp));
}
void
bdwrite(bp)
struct buf *bp;
{
struct proc *p = current_proc();
kern_return_t kret;
upl_t upl;
upl_page_info_t *pl;
if (!ISSET(bp->b_flags, B_DELWRI)) {
SET(bp->b_flags, B_DELWRI);
if (p && p->p_stats)
p->p_stats->p_ru.ru_oublock++;
reassignbuf(bp, bp->b_vp);
}
if (ISSET(bp->b_flags, B_TAPE)) {
VOP_BWRITE(bp);
return;
}
SET(bp->b_flags, B_DONE);
brelse(bp);
}
void
bawrite(bp)
struct buf *bp;
{
SET(bp->b_flags, B_ASYNC);
VOP_BWRITE(bp);
}
void
brelse(bp)
struct buf *bp;
{
struct bqueues *bufq;
int s;
long whichq;
KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 388)) | DBG_FUNC_START,
bp->b_lblkno * PAGE_SIZE, bp, bp->b_data, bp->b_flags, 0);
trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno);
if (!ISSET(bp->b_flags, B_META)
&& UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) {
kern_return_t kret;
upl_t upl;
int upl_flags;
if ( !ISSET(bp->b_flags, B_PAGELIST)) {
if ( !ISSET(bp->b_flags, B_INVAL)) {
kret = ubc_create_upl(bp->b_vp,
ubc_blktooff(bp->b_vp, bp->b_lblkno),
bp->b_bufsize,
&upl,
NULL,
UPL_PRECIOUS);
if (kret != KERN_SUCCESS)
panic("brelse: Failed to get pagelists");
#ifdef UBC_DEBUG
upl_ubc_alias_set(upl, bp, 5);
#endif
} else
upl = (upl_t) 0;
} else {
upl = bp->b_pagelist;
kret = ubc_upl_unmap(upl);
if (kret != KERN_SUCCESS)
panic("kernel_upl_unmap failed");
bp->b_data = 0;
}
if (upl) {
if (bp->b_flags & (B_ERROR | B_INVAL)) {
if (bp->b_flags & (B_READ | B_INVAL))
upl_flags = UPL_ABORT_DUMP_PAGES;
else
upl_flags = 0;
ubc_upl_abort(upl, upl_flags);
} else {
if (ISSET(bp->b_flags, (B_DELWRI | B_WASDIRTY)))
upl_flags = UPL_COMMIT_SET_DIRTY ;
else
upl_flags = UPL_COMMIT_CLEAR_DIRTY ;
ubc_upl_commit_range(upl, 0, bp->b_bufsize, upl_flags |
UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
}
s = splbio();
CLR(bp->b_flags, B_PAGELIST);
bp->b_pagelist = 0;
splx(s);
}
} else {
if(ISSET(bp->b_flags, B_PAGELIST))
panic("brelse: pagelist set for non VREG; vp=%x", bp->b_vp);
}
if (needbuffer) {
needbuffer = 0;
wakeup(&needbuffer);
}
if (ISSET(bp->b_flags, B_WANTED)) {
CLR(bp->b_flags, B_WANTED);
wakeup(bp);
}
s = splbio();
if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR))
CLR(bp->b_flags, B_ERROR);
if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR)))
SET(bp->b_flags, B_INVAL);
if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL)) {
if (bp->b_vp)
brelvp(bp);
CLR(bp->b_flags, B_DELWRI);
if (bp->b_bufsize <= 0)
whichq = BQ_EMPTY;
else
whichq = BQ_AGE;
bufq = &bufqueues[whichq];
binsheadfree(bp, bufq, whichq);
} else {
if (ISSET(bp->b_flags, B_LOCKED))
whichq = BQ_LOCKED;
else if (ISSET(bp->b_flags, B_META))
whichq = BQ_META;
else if (ISSET(bp->b_flags, B_AGE))
whichq = BQ_AGE;
else
whichq = BQ_LRU;
bufq = &bufqueues[whichq];
binstailfree(bp, bufq, whichq);
}
CLR(bp->b_flags, (B_AGE | B_ASYNC | B_BUSY | B_NOCACHE));
splx(s);
KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 388)) | DBG_FUNC_END,
bp, bp->b_data, bp->b_flags, 0, 0);
}
struct buf *
incore(vp, blkno)
struct vnode *vp;
daddr_t blkno;
{
struct buf *bp;
int bufseen = 0;
bp = BUFHASH(vp, blkno)->lh_first;
for (; bp != NULL; bp = bp->b_hash.le_next, bufseen++) {
if (bp->b_lblkno == blkno && bp->b_vp == vp &&
!ISSET(bp->b_flags, B_INVAL))
return (bp);
if(bufseen >= nbuf)
panic("walked more than nbuf in incore");
}
return (0);
}
struct buf *
getblk(vp, blkno, size, slpflag, slptimeo, operation)
register struct vnode *vp;
daddr_t blkno;
int size, slpflag, slptimeo, operation;
{
struct buf *bp;
int s, err;
upl_t upl;
upl_page_info_t *pl;
kern_return_t kret;
int error=0;
int pagedirty = 0;
KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 386)) | DBG_FUNC_START,
blkno * PAGE_SIZE, size, operation, 0, 0);
start:
s = splbio();
if (bp = incore(vp, blkno)) {
if (ISSET(bp->b_flags, B_BUSY)) {
switch (operation) {
case BLK_READ:
case BLK_WRITE:
case BLK_META:
SET(bp->b_flags, B_WANTED);
bufstats.bufs_busyincore++;
err = tsleep(bp, slpflag | (PRIBIO + 1), "getblk",
slptimeo);
splx(s);
if (err && ((slpflag & PCATCH) ||
((err == EWOULDBLOCK) && slptimeo)))
return (NULL);
goto start;
break;
case BLK_PAGEIN:
panic("getblk: pagein for incore busy buffer");
splx(s);
break;
case BLK_PAGEOUT:
panic("getblk: pageout for incore busy buffer");
splx(s);
break;
default:
panic("getblk: %d unknown operation 1", operation);
break;
}
} else {
SET(bp->b_flags, (B_BUSY | B_CACHE));
bremfree(bp);
bufstats.bufs_incore++;
splx(s);
allocbuf(bp, size);
if (ISSET(bp->b_flags, B_PAGELIST))
panic("pagelist buffer is not busy");
switch (operation) {
case BLK_READ:
case BLK_WRITE:
if (UBCISVALID(bp->b_vp) && bp->b_bufsize) {
kret = ubc_create_upl(vp,
ubc_blktooff(vp, bp->b_lblkno),
bp->b_bufsize,
&upl,
&pl,
UPL_PRECIOUS);
if (kret != KERN_SUCCESS)
panic("Failed to get pagelists");
SET(bp->b_flags, B_PAGELIST);
bp->b_pagelist = upl;
if ( !upl_valid_page(pl, 0))
panic("getblk: incore buffer without valid page");
if (upl_dirty_page(pl, 0))
SET(bp->b_flags, B_WASDIRTY);
else
CLR(bp->b_flags, B_WASDIRTY);
kret = ubc_upl_map(upl, (vm_address_t *)&(bp->b_data));
if (kret != KERN_SUCCESS) {
panic("getblk: ubc_upl_map() failed with (%d)",
kret);
}
if (bp->b_data == 0) panic("ubc_upl_map mapped 0");
}
break;
case BLK_META:
if(bp->b_data == 0)
panic("bp->b_data null incore buf=%x", bp);
break;
case BLK_PAGEIN:
case BLK_PAGEOUT:
panic("getblk: paging operation 1");
break;
default:
panic("getblk: %d unknown operation 2", operation);
break;
}
}
} else {
int queue = BQ_EMPTY;
splx(s);
if ((operation == BLK_META) || (UBCINVALID(vp)) ||
!(UBCINFOEXISTS(vp))) {
operation = BLK_META;
}
if ((bp = getnewbuf(slpflag, slptimeo, &queue)) == NULL)
goto start;
if (incore(vp, blkno)) {
SET(bp->b_flags, B_INVAL);
binshash(bp, &invalhash);
brelse(bp);
goto start;
}
if (operation == BLK_META) {
SET(bp->b_flags, B_META);
queue = BQ_META;
}
binshash(bp, BUFHASH(vp, blkno));
allocbuf(bp, size);
switch (operation) {
case BLK_META:
#if !ZALLOC_METADATA
if (bp->b_data)
panic("bp->b_data is not nul; %x",bp);
kret = kmem_alloc(kernel_map,
&bp->b_data, bp->b_bufsize);
if (kret != KERN_SUCCESS)
panic("getblk: kmem_alloc() returned %d", kret);
#endif
if(bp->b_data == 0)
panic("bp->b_data is null %x",bp);
bp->b_blkno = bp->b_lblkno = blkno;
s = splbio();
bgetvp(vp, bp);
bufstats.bufs_miss++;
splx(s);
if (bp->b_data == 0)
panic("b_data is 0: 2");
CLR(bp->b_flags, B_WANTED);
wakeup(bp);
break;
case BLK_READ:
case BLK_WRITE:
if (ISSET(bp->b_flags, B_PAGELIST))
panic("B_PAGELIST in bp=%x",bp);
kret = ubc_create_upl(vp,
ubc_blktooff(vp, blkno),
bp->b_bufsize,
&upl,
&pl,
UPL_PRECIOUS);
if (kret != KERN_SUCCESS)
panic("Failed to get pagelists");
#ifdef UBC_DEBUG
upl_ubc_alias_set(upl, bp, 4);
#endif
bp->b_blkno = bp->b_lblkno = blkno;
bp->b_pagelist = upl;
SET(bp->b_flags, B_PAGELIST);
if (upl_valid_page(pl, 0)) {
SET(bp->b_flags, B_CACHE | B_DONE);
bufstats.bufs_vmhits++;
pagedirty = upl_dirty_page(pl, 0);
if (pagedirty)
SET(bp->b_flags, B_WASDIRTY);
if (vp->v_tag == VT_NFS) {
off_t f_offset;
int valid_size;
bp->b_validoff = 0;
bp->b_dirtyoff = 0;
f_offset = ubc_blktooff(vp, blkno);
if (f_offset > vp->v_ubcinfo->ui_size) {
CLR(bp->b_flags, (B_CACHE|B_DONE|B_WASDIRTY));
bp->b_validend = 0;
bp->b_dirtyend = 0;
} else {
valid_size = min(((unsigned int)(vp->v_ubcinfo->ui_size - f_offset)), PAGE_SIZE);
bp->b_validend = valid_size;
if (pagedirty)
bp->b_dirtyend = valid_size;
else
bp->b_dirtyend = 0;
KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 386)) | DBG_FUNC_NONE,
bp->b_validend, bp->b_dirtyend,
(int)vp->v_ubcinfo->ui_size, 0, 0);
}
} else {
bp->b_validoff = 0;
bp->b_dirtyoff = 0;
if (pagedirty) {
bp->b_validend = bp->b_bcount;
bp->b_dirtyend = bp->b_bcount;
} else {
bp->b_validend = bp->b_bcount;
bp->b_dirtyend = 0;
}
}
if (error = VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL)) {
panic("VOP_BMAP failed in getblk");
bp->b_error = error;
SET(bp->b_flags, (B_ERROR | B_INVAL));
CLR(bp->b_flags, B_DONE);
brelse(bp);
return (0);
}
} else {
bufstats.bufs_miss++;
}
kret = ubc_upl_map(upl, (vm_address_t *)&(bp->b_data));
if (kret != KERN_SUCCESS) {
panic("getblk: ubc_upl_map() "
"failed with (%d)", kret);
}
if (bp->b_data == 0) panic("kernel_upl_map mapped 0");
s = splbio();
bgetvp(vp, bp);
splx(s);
break;
case BLK_PAGEIN:
case BLK_PAGEOUT:
panic("getblk: paging operation 2");
break;
default:
panic("getblk: %d unknown operation 3", operation);
break;
}
}
if (bp->b_data == NULL)
panic("getblk: bp->b_addr is null");
if (bp->b_bufsize & 0xfff) {
#if ZALLOC_METADATA
if (ISSET(bp->b_flags, B_META) && (bp->b_bufsize & 0x1ff))
#endif
panic("getblk: bp->b_bufsize = %d", bp->b_bufsize);
}
KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 386)) | DBG_FUNC_END,
bp, bp->b_data, bp->b_flags, 3, 0);
return (bp);
}
struct buf *
geteblk(size)
int size;
{
struct buf *bp;
int queue = BQ_EMPTY;
#if !ZALLOC_METADATA
kern_return_t kret;
vm_size_t desired_size = roundup(size, CLBYTES);
if (desired_size > MAXBSIZE)
panic("geteblk: buffer larger than MAXBSIZE requested");
#endif
while ((bp = getnewbuf(0, 0, &queue)) == 0)
;
#if ZALLOC_METADATA
SET(bp->b_flags, (B_META|B_INVAL));
#else
SET(bp->b_flags, B_INVAL);
#endif
#if DIAGNOSTIC
assert(queue == BQ_EMPTY);
#endif
#if !ZALLOC_METADATA
kret = kmem_alloc_aligned(kernel_map, &bp->b_data, desired_size);
if (kret != KERN_SUCCESS)
panic("geteblk: kmem_alloc_aligned returned %d", kret);
#endif
binshash(bp, &invalhash);
allocbuf(bp, size);
bufstats.bufs_eblk++;
return (bp);
}
#if ZALLOC_METADATA
#define MINMETA 512
#define MAXMETA 4096
struct meta_zone_entry {
zone_t mz_zone;
vm_size_t mz_size;
vm_size_t mz_max;
char *mz_name;
};
struct meta_zone_entry meta_zones[] = {
{NULL, (MINMETA * 1), 128 * (MINMETA * 1), "buf.512" },
{NULL, (MINMETA * 2), 64 * (MINMETA * 2), "buf.1024" },
{NULL, (MINMETA * 3), 16 * (MINMETA * 3), "buf.1536" },
{NULL, (MINMETA * 4), 16 * (MINMETA * 4), "buf.2048" },
{NULL, (MINMETA * 5), 16 * (MINMETA * 5), "buf.2560" },
{NULL, (MINMETA * 6), 16 * (MINMETA * 6), "buf.3072" },
{NULL, (MINMETA * 7), 16 * (MINMETA * 7), "buf.3584" },
{NULL, (MINMETA * 8), 512 * (MINMETA * 8), "buf.4096" },
{NULL, 0, 0, "" }
};
#endif
zone_t buf_hdr_zone;
int buf_hdr_count;
static void
bufzoneinit(void)
{
#if ZALLOC_METADATA
int i;
for (i = 0; meta_zones[i].mz_size != 0; i++) {
meta_zones[i].mz_zone =
zinit(meta_zones[i].mz_size,
meta_zones[i].mz_max,
PAGE_SIZE,
meta_zones[i].mz_name);
}
#endif
buf_hdr_zone = zinit(sizeof(struct buf), 32, PAGE_SIZE, "buf headers");
}
#if ZALLOC_METADATA
static zone_t
getbufzone(size_t size)
{
int i;
if (size % 512)
panic("getbufzone: incorect size = %d", size);
i = (size / 512) - 1;
return (meta_zones[i].mz_zone);
}
#endif
int
allocbuf(bp, size)
struct buf *bp;
int size;
{
vm_size_t desired_size;
desired_size = roundup(size, CLBYTES);
if(desired_size < PAGE_SIZE)
desired_size = PAGE_SIZE;
if (desired_size > MAXBSIZE)
panic("allocbuf: buffer larger than MAXBSIZE requested");
#if ZALLOC_METADATA
if (ISSET(bp->b_flags, B_META)) {
kern_return_t kret;
zone_t zprev, z;
size_t nsize = roundup(size, MINMETA);
if (bp->b_data) {
vm_offset_t elem = (vm_offset_t)bp->b_data;
if (ISSET(bp->b_flags, B_ZALLOC))
if (bp->b_bufsize <= MAXMETA) {
if (bp->b_bufsize < nsize) {
desired_size = nsize;
zprev = getbufzone(bp->b_bufsize);
z = getbufzone(nsize);
bp->b_data = (caddr_t)zalloc(z);
if(bp->b_data == 0)
panic("allocbuf: zalloc() returned NULL");
bcopy(elem, bp->b_data, bp->b_bufsize);
zfree(zprev, elem);
} else {
desired_size = bp->b_bufsize;
}
} else
panic("allocbuf: B_ZALLOC set incorrectly");
else
if (bp->b_bufsize < desired_size) {
kret = kmem_alloc(kernel_map, &bp->b_data, desired_size);
if (kret != KERN_SUCCESS)
panic("allocbuf: kmem_alloc() returned %d", kret);
if(bp->b_data == 0)
panic("allocbuf: null b_data");
bcopy(elem, bp->b_data, bp->b_bufsize);
kmem_free(kernel_map, elem, bp->b_bufsize);
} else {
desired_size = bp->b_bufsize;
}
} else {
if (nsize <= MAXMETA) {
desired_size = nsize;
z = getbufzone(nsize);
bp->b_data = (caddr_t)zalloc(z);
if(bp->b_data == 0)
panic("allocbuf: zalloc() returned NULL 2");
SET(bp->b_flags, B_ZALLOC);
} else {
kret = kmem_alloc(kernel_map, &bp->b_data, desired_size);
if (kret != KERN_SUCCESS)
panic("allocbuf: kmem_alloc() 2 returned %d", kret);
if(bp->b_data == 0)
panic("allocbuf: null b_data 2");
}
}
}
if (ISSET(bp->b_flags, B_META) && (bp->b_data == 0))
panic("allocbuf: bp->b_data is NULL");
#endif
bp->b_bufsize = desired_size;
bp->b_bcount = size;
}
static struct buf *
getnewbuf(slpflag, slptimeo, queue)
int slpflag, slptimeo;
int *queue;
{
register struct buf *bp;
register struct buf *lru_bp;
register struct buf *age_bp;
register struct buf *meta_bp;
register int age_time, lru_time, bp_time, meta_time;
int s;
struct ucred *cred;
int req = *queue;
start:
s = splbio();
if ((*queue > BQUEUES) || (*queue < 0)
|| (*queue == BQ_LAUNDRY) || (*queue == BQ_LOCKED))
*queue = BQ_EMPTY;
if (*queue != BQUEUES) {
bp = bufqueues[*queue].tqh_first;
if (bp)
goto found;
}
age_bp = bufqueues[BQ_AGE].tqh_first;
lru_bp = bufqueues[BQ_LRU].tqh_first;
meta_bp = bufqueues[BQ_META].tqh_first;
if (!age_bp && !lru_bp && !meta_bp) {
bp = bufqueues[BQ_EMPTY].tqh_first;
if (bp) {
*queue = BQ_EMPTY;
goto found;
}
bp = (struct buf *)zalloc(buf_hdr_zone);
if (bp) {
bufhdrinit(bp);
BLISTNONE(bp);
binshash(bp, &invalhash);
SET(bp->b_flags, B_HDRALLOC);
*queue = BQ_EMPTY;
binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY);
buf_hdr_count++;
goto found;
}
printf("getnewbuf: No useful buffers");
needbuffer = 1;
bufstats.bufs_sleeps++;
tsleep(&needbuffer, slpflag|(PRIBIO+1), "getnewbuf", slptimeo);
splx(s);
return (0);
}
bp = NULL;
*queue = -1;
if (!age_bp) {
bp = lru_bp;
*queue = BQ_LRU;
} else if (!lru_bp) {
bp = age_bp;
*queue = BQ_AGE;
} else {
age_time = time.tv_sec - age_bp->b_timestamp;
lru_time = time.tv_sec - lru_bp->b_timestamp;
if ((age_time < 0) || (lru_time < 0)) {
bp = age_bp;
*queue = BQ_AGE;
} else {
if ((lru_time >= lru_is_stale) && (age_time < age_is_stale)) {
bp = lru_bp;
*queue = BQ_LRU;
} else {
bp = age_bp;
*queue = BQ_AGE;
}
}
}
if (!bp) {
bp = meta_bp;
*queue = BQ_META;
} else if (meta_bp) {
bp_time = time.tv_sec - bp->b_timestamp;
meta_time = time.tv_sec - meta_bp->b_timestamp;
if (!(bp_time < 0) && !(meta_time < 0)) {
int bp_is_stale;
bp_is_stale = (*queue == BQ_LRU) ?
lru_is_stale : age_is_stale;
if ((meta_time >= meta_is_stale) &&
(bp_time < bp_is_stale)) {
bp = meta_bp;
*queue = BQ_META;
}
}
}
if (bp == NULL)
panic("getnewbuf: null bp");
found:
if (bp->b_hash.le_prev == (struct buf **)0xdeadbeef)
panic("getnewbuf: le_prev is deadbeef");
if(ISSET(bp->b_flags, B_BUSY))
panic("getnewbuf reusing BUSY buf");
if (bcleanbuf(bp)) {
splx(s);
*queue = req;
goto start;
}
splx(s);
return (bp);
}
#include <mach/mach_types.h>
#include <mach/memory_object_types.h>
int
bcleanbuf(struct buf *bp)
{
int s;
struct ucred *cred;
s = splbio();
bremfree(bp);
SET(bp->b_flags, B_BUSY);
if (bp->b_hash.le_prev == (struct buf **)0xdeadbeef)
panic("bcleanbuf: le_prev is deadbeef");
if (ISSET(bp->b_flags, B_DELWRI)) {
splx(s);
binstailfree(bp, &bufqueues[BQ_LAUNDRY], BQ_LAUNDRY);
blaundrycnt++;
wakeup(&blaundrycnt);
return (1);
}
if (bp->b_vp)
brelvp(bp);
bremhash(bp);
BLISTNONE(bp);
splx(s);
if (ISSET(bp->b_flags, B_META)) {
#if ZALLOC_METADATA
vm_offset_t elem = (vm_offset_t)bp->b_data;
if (elem == 0)
panic("bcleanbuf: NULL bp->b_data B_META buffer");
if (ISSET(bp->b_flags, B_ZALLOC)) {
if (bp->b_bufsize <= MAXMETA) {
zone_t z;
z = getbufzone(bp->b_bufsize);
bp->b_data = (caddr_t)0xdeadbeef;
zfree(z, elem);
CLR(bp->b_flags, B_ZALLOC);
} else
panic("bcleanbuf: B_ZALLOC set incorrectly");
} else {
bp->b_data = (caddr_t)0xdeadbeef;
kmem_free(kernel_map, elem, bp->b_bufsize);
}
#else
if (bp->b_data == 0)
panic("bcleanbuf: bp->b_data == NULL for B_META buffer");
kmem_free(kernel_map, bp->b_data, bp->b_bufsize);
#endif
}
trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno);
s = splbio();
bp->b_bufsize = 0;
bp->b_data = 0;
bp->b_flags = B_BUSY;
bp->b_dev = NODEV;
bp->b_blkno = bp->b_lblkno = 0;
bp->b_iodone = 0;
bp->b_error = 0;
bp->b_resid = 0;
bp->b_bcount = 0;
bp->b_dirtyoff = bp->b_dirtyend = 0;
bp->b_validoff = bp->b_validend = 0;
cred = bp->b_rcred;
if (cred != NOCRED) {
bp->b_rcred = NOCRED;
crfree(cred);
}
cred = bp->b_wcred;
if (cred != NOCRED) {
bp->b_wcred = NOCRED;
crfree(cred);
}
splx(s);
return (0);
}
int
biowait(bp)
struct buf *bp;
{
upl_t upl;
upl_page_info_t *pl;
int s;
kern_return_t kret;
s = splbio();
while (!ISSET(bp->b_flags, B_DONE))
tsleep(bp, PRIBIO + 1, "biowait", 0);
splx(s);
if (ISSET(bp->b_flags, B_EINTR)) {
CLR(bp->b_flags, B_EINTR);
return (EINTR);
} else if (ISSET(bp->b_flags, B_ERROR))
return (bp->b_error ? bp->b_error : EIO);
else
return (0);
}
void
biodone(bp)
struct buf *bp;
{
boolean_t funnel_state;
int s;
funnel_state = thread_funnel_set(kernel_flock, TRUE);
KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 387)) | DBG_FUNC_START,
bp, bp->b_data, bp->b_flags, 0, 0);
if (ISSET(bp->b_flags, B_DONE))
panic("biodone already");
SET(bp->b_flags, B_DONE);
CLR(bp->b_flags, B_WASDIRTY);
if (!ISSET(bp->b_flags, B_READ) && !ISSET(bp->b_flags, B_RAW))
vwakeup(bp);
if (ISSET(bp->b_flags, B_CALL)) {
CLR(bp->b_flags, B_CALL);
(*bp->b_iodone)(bp);
} else if (ISSET(bp->b_flags, B_ASYNC))
brelse(bp);
else {
CLR(bp->b_flags, B_WANTED);
wakeup(bp);
}
KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 387)) | DBG_FUNC_END,
bp, bp->b_data, bp->b_flags, 0, 0);
thread_funnel_set(kernel_flock, funnel_state);
}
int
count_lock_queue()
{
register struct buf *bp;
register int n = 0;
for (bp = bufqueues[BQ_LOCKED].tqh_first; bp;
bp = bp->b_freelist.tqe_next)
n++;
return (n);
}
int
count_busy_buffers()
{
register struct buf *bp;
register int nbusy = 0;
for (bp = &buf[nbuf]; --bp >= buf; )
if ((bp->b_flags & (B_BUSY|B_INVAL)) == B_BUSY)
nbusy++;
return (nbusy);
}
#if 1
void
vfs_bufstats()
{
int s, i, j, count;
register struct buf *bp;
register struct bqueues *dp;
int counts[MAXBSIZE/CLBYTES+1];
static char *bname[BQUEUES] =
{ "LOCKED", "LRU", "AGE", "EMPTY", "META", "LAUNDRY" };
for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
count = 0;
for (j = 0; j <= MAXBSIZE/CLBYTES; j++)
counts[j] = 0;
s = splbio();
for (bp = dp->tqh_first; bp; bp = bp->b_freelist.tqe_next) {
counts[bp->b_bufsize/CLBYTES]++;
count++;
}
splx(s);
printf("%s: total-%d", bname[i], count);
for (j = 0; j <= MAXBSIZE/CLBYTES; j++)
if (counts[j] != 0)
printf(", %d-%d", j * CLBYTES, counts[j]);
printf("\n");
}
}
#endif
#define NRESERVEDIOBUFS 16
struct buf *
alloc_io_buf(vp, priv)
struct vnode *vp;
int priv;
{
register struct buf *bp;
int s;
s = splbio();
while (niobuf - NRESERVEDIOBUFS < bufstats.bufs_iobufinuse && !priv) {
need_iobuffer = 1;
bufstats.bufs_iobufsleeps++;
(void) tsleep(&need_iobuffer, (PRIBIO+1), "alloc_io_buf", 0);
}
while ((bp = iobufqueue.tqh_first) == NULL) {
need_iobuffer = 1;
bufstats.bufs_iobufsleeps++;
(void) tsleep(&need_iobuffer, (PRIBIO+1), "alloc_io_buf1", 0);
}
TAILQ_REMOVE(&iobufqueue, bp, b_freelist);
bp->b_timestamp = 0;
bp->b_flags = B_BUSY;
bp->b_blkno = bp->b_lblkno = 0;
bp->b_iodone = 0;
bp->b_error = 0;
bp->b_resid = 0;
bp->b_bcount = 0;
bp->b_bufsize = 0;
bp->b_vp = vp;
if (vp->v_type == VBLK || vp->v_type == VCHR)
bp->b_dev = vp->v_rdev;
else
bp->b_dev = NODEV;
bufstats.bufs_iobufinuse++;
if (bufstats.bufs_iobufinuse > bufstats.bufs_iobufmax)
bufstats.bufs_iobufmax = bufstats.bufs_iobufinuse;
splx(s);
return (bp);
}
void
free_io_buf(bp)
struct buf *bp;
{
int s;
s = splbio();
bp->b_vp = NULL;
bp->b_flags = B_INVAL;
binsheadfree(bp, &iobufqueue, -1);
if (need_iobuffer) {
need_iobuffer = 0;
wakeup(&need_iobuffer);
}
bufstats.bufs_iobufinuse--;
splx(s);
}
typedef long long blsize_t;
blsize_t MAXNBUF;
blsize_t nbufh;
blsize_t nbuflow;
blsize_t nbufhigh;
blsize_t nbuftarget;
struct bufqlim {
blsize_t bl_nlow;
blsize_t bl_num;
blsize_t bl_nlhigh;
blsize_t bl_target;
long bl_stale;
} bufqlim[BQUEUES];
long bufqscanwait = 0;
extern void bufqscan_thread();
extern int balancebufq(int q);
extern int btrimempty(int n);
extern int initbufqscan(void);
extern int nextbufq(int q);
extern void buqlimprt(int all);
void
bufq_balance_thread_init()
{
if (bufqscanwait++ == 0) {
int i;
MAXNBUF = (mem_size / PAGE_SIZE);
nbufh = nbuf;
nbuflow = min(nbufh, 100);
nbufhigh = min(MAXNBUF, max(nbufh, 2048));
nbuftarget = (mem_size >> 5) / PAGE_SIZE;
nbuftarget = max(nbuflow, nbuftarget);
nbuftarget = min(nbufhigh, nbuftarget);
bufqlim[BQ_LOCKED].bl_nlow = 0;
bufqlim[BQ_LOCKED].bl_nlhigh = 32;
bufqlim[BQ_LOCKED].bl_target = 0;
bufqlim[BQ_LOCKED].bl_stale = 30;
bufqlim[BQ_LRU].bl_nlow = 0;
bufqlim[BQ_LRU].bl_nlhigh = nbufhigh/4;
bufqlim[BQ_LRU].bl_target = nbuftarget/4;
bufqlim[BQ_LRU].bl_stale = LRU_IS_STALE;
bufqlim[BQ_AGE].bl_nlow = 0;
bufqlim[BQ_AGE].bl_nlhigh = nbufhigh/4;
bufqlim[BQ_AGE].bl_target = nbuftarget/4;
bufqlim[BQ_AGE].bl_stale = AGE_IS_STALE;
bufqlim[BQ_EMPTY].bl_nlow = 0;
bufqlim[BQ_EMPTY].bl_nlhigh = nbufhigh/4;
bufqlim[BQ_EMPTY].bl_target = nbuftarget/4;
bufqlim[BQ_EMPTY].bl_stale = 600000;
bufqlim[BQ_META].bl_nlow = 0;
bufqlim[BQ_META].bl_nlhigh = nbufhigh/4;
bufqlim[BQ_META].bl_target = nbuftarget/4;
bufqlim[BQ_META].bl_stale = META_IS_STALE;
bufqlim[BQ_LOCKED].bl_nlow = 0;
bufqlim[BQ_LOCKED].bl_nlhigh = 32;
bufqlim[BQ_LOCKED].bl_target = 0;
bufqlim[BQ_LOCKED].bl_stale = 30;
buqlimprt(1);
}
kernel_thread(kernel_task, bufqscan_thread);
}
void
bufqscan_thread()
{
boolean_t funnel_state;
int moretodo = 0;
funnel_state = thread_funnel_set(kernel_flock, TRUE);
for(;;) {
do {
int q;
for (q = initbufqscan(); q; ) {
moretodo |= balancebufq(q);
q = nextbufq(q);
}
} while (moretodo);
#if 1 || DIAGNOSTIC
vfs_bufstats();
buqlimprt(0);
#endif
(void)tsleep((void *)&bufqscanwait, PRIBIO, "bufqscanwait", 60 * hz);
moretodo = 0;
}
(void) thread_funnel_set(kernel_flock, FALSE);
}
int
initbufqscan()
{
return (BQ_AGE);
}
int
nextbufq(int q)
{
int order[] = { BQ_AGE, BQ_LRU, BQ_META, BQ_EMPTY, 0 };
q++;
q %= sizeof(order);
return (order[q]);
}
int
balancebufq(int q)
{
int moretodo = 0;
int s = splbio();
int n;
if ((q < 0) || (q >= BQUEUES))
goto out;
if ((q == BQ_LOCKED) || (q == BQ_LAUNDRY))
goto out;
n = (bufqlim[q].bl_num - bufqlim[q].bl_target);
if (n < 0)
goto out;
if ( n > 8 ) {
n >>= 3;
}
if (q == BQ_EMPTY) {
moretodo |= btrimempty(n);
goto out;
}
for (; n > 0; n--) {
struct buf *bp = bufqueues[q].tqh_first;
if (!bp)
break;
if ((time.tv_sec - bp->b_timestamp) > bufqlim[q].bl_stale) {
if (bcleanbuf(bp)) {
moretodo = 1;
} else {
SET(bp->b_flags, B_INVAL);
brelse(bp);
}
} else
break;
}
out:
splx(s);
return (moretodo);
}
int
btrimempty(int n)
{
return (0);
}
void
bufqinc(int q)
{
if ((q < 0) || (q >= BQUEUES))
return;
bufqlim[q].bl_num++;
return;
}
void
bufqdec(int q)
{
if ((q < 0) || (q >= BQUEUES))
return;
bufqlim[q].bl_num--;
return;
}
void
buqlimprt(int all)
{
int i;
static char *bname[BQUEUES] =
{ "LOCKED", "LRU", "AGE", "EMPTY", "META", "LAUNDRY" };
if (all)
for (i = 0; i < BQUEUES; i++) {
printf("%s : ", bname[i]);
printf("min = %d, ", (long)bufqlim[i].bl_nlow);
printf("cur = %d, ", (long)bufqlim[i].bl_num);
printf("max = %d, ", (long)bufqlim[i].bl_nlhigh);
printf("target = %d, ", (long)bufqlim[i].bl_target);
printf("stale after %d seconds\n", bufqlim[i].bl_stale);
}
else
for (i = 0; i < BQUEUES; i++) {
printf("%s : ", bname[i]);
printf("cur = %d, ", (long)bufqlim[i].bl_num);
}
}
static void
bcleanbuf_thread_init()
{
static void bcleanbuf_thread();
kernel_thread(kernel_task, bcleanbuf_thread);
}
static void
bcleanbuf_thread()
{
boolean_t funnel_state;
struct buf *bp;
funnel_state = thread_funnel_set(kernel_flock, TRUE);
doit:
while (blaundrycnt == 0)
(void)tsleep((void *)&blaundrycnt, PRIBIO, "blaundry", 60 * hz);
bp = TAILQ_FIRST(&bufqueues[BQ_LAUNDRY]);
bremfree(bp);
blaundrycnt--;
bawrite(bp);
goto doit;
(void) thread_funnel_set(kernel_flock, funnel_state);
}