static int BC_minimum_mem_size = 256;
#define BC_READAHEAD_WAIT_DEFAULT 10
#define BC_READAHEAD_WAIT_CDROM 45
static int BC_wait_for_readahead = BC_READAHEAD_WAIT_DEFAULT;
static int BC_chit_check_start = 500;
static int BC_chit_low_level = 50;
static int BC_cache_timeout = 120;
#define BC_MAX_READ (64 * 1024)
#define DBG_BOOTCACHE 6
#define DBG_BC_CUT 1
#ifdef DEBUG
# define MACH_DEBUG
# define debug(fmt, args...) printf("**** %s: " fmt "\n", __FUNCTION__ , ##args)
extern void Debugger(char *);
#else
# define debug(fmt, args...)
#endif
#define message(fmt, args...) printf("BootCache: " fmt "\n" , ##args)
#ifdef EXTRA_DEBUG
# define xdebug(fmt, args...) printf("+++ %s: " fmt "\n", __FUNCTION__ , ##args)
#else
# define xdebug(fmt, args...)
#endif
#include <sys/param.h>
#include <sys/proc.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/kdebug.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/ubc.h>
#include <ipc/ipc_types.h>
#include <mach/host_priv.h>
#include <mach/kmod.h>
#include <mach/vm_map.h>
#include <mach/vm_param.h>
#include <vm/vm_kern.h>
#include <libkern/libkern.h>
#include <mach/mach_types.h>
#include <kern/assert.h>
#include <kern/host.h>
#include <kern/thread_call.h>
#include <IOKit/storage/IOMediaBSDClient.h>
#include <pexpert/pexpert.h>
#include "BootCache.h"
#ifdef STATIC_PLAYLIST
# include "playlist.h"
#endif
struct BC_cache_extent {
u_int64_t ce_offset;
u_int64_t ce_length;
caddr_t ce_data;
int ce_flags;
#define CE_ABORTED (1 << 8)
};
struct BC_history_cluster {
struct BC_history_cluster *hc_link;
int hc_entries;
struct BC_history_entry hc_data[0];
};
#ifdef READ_HISTORY_BUFFER
struct BC_read_history_entry {
struct BC_cache_extent *rh_extent;
u_int64_t rh_blkno;
u_int64_t rh_bcount;
int rh_result;
# define BC_RHISTORY_OK 0
# define BC_RHISTORY_FAIL 1
};
#endif
#ifdef STATIC_HISTORY
# define BC_HISTORY_ALLOC 128000
# define BC_HISTORY_ENTRIES \
(((BC_HISTORY_ALLOC - sizeof(struct BC_history_cluster)) / \
sizeof(struct BC_history_entry)) - 1)
# define BC_HISTORY_MAXCLUSTERS 1
#else
# define BC_HISTORY_ALLOC 16000
# define BC_HISTORY_ENTRIES \
(((BC_HISTORY_ALLOC - sizeof(struct BC_history_cluster)) / \
sizeof(struct BC_history_entry)) - 1)
# define BC_HISTORY_MAXCLUSTERS ((BC_MAXENTRIES / BC_HISTORY_ENTRIES) + 1)
#endif
struct BC_cache_control {
dev_t c_dev;
struct vnode *c_devvp;
struct bdevsw *c_devsw;
strategy_fcn_t *c_strategy;
u_int64_t c_blocksize;
u_int64_t c_devsize;
int c_flags;
#define BC_FLAG_SHUTDOWN (1<<0)
#define BC_FLAG_CACHEACTIVE (1<<1)
#define BC_FLAG_HTRUNCATED (1<<2)
#define BC_FLAG_IOBUSY (1<<3)
#define BC_FLAG_STARTED (1<<4)
int c_strategycalls;
int c_bypasscalls;
int c_batch;
int c_batch_count;
struct BC_cache_extent *c_extents;
int c_extent_count;
struct BC_cache_extent *c_extent_tail;
caddr_t c_buffer;
int c_buffer_pages;
int c_buffer_blocks;
u_int32_t *c_blockmap;
u_int32_t *c_pagemap;
u_int32_t *c_iopagemap;
struct BC_history_cluster *c_history;
u_int64_t c_history_bytes;
struct BC_statistics c_stats;
mach_port_t c_map_port;
vm_map_t c_map;
vm_address_t c_mapbase;
vm_size_t c_mapsize;
mach_port_t c_entry_port;
#ifdef READ_HISTORY_BUFFER
int c_rhistory_idx;
struct BC_read_history_entry *c_rhistory;
#endif
};
#define CB_PAGEBLOCKS(c) (PAGE_SIZE / (c)->c_blocksize)
#define CB_BLOCK_TO_PAGE(c, block) ((block) / CB_PAGEBLOCKS(c))
#define CB_PAGE_TO_BLOCK(c, page) ((page) * CB_PAGEBLOCKS(c))
#define CB_BLOCK_TO_BYTE(c, block) ((block) * (c)->c_blocksize)
#define CB_BYTE_TO_BLOCK(c, byte) ((byte) / (c)->c_blocksize)
#define CB_MAPFIELDBITS 32
#define CB_MAPFIELDBYTES 4
#define CB_MAPIDX(x) ((x) / CB_MAPFIELDBITS)
#define CB_MAPBIT(x) (1 << ((x) % CB_MAPFIELDBITS))
#define CB_BLOCK_PRESENT(c, block) \
((c)->c_blockmap[CB_MAPIDX(block)] & CB_MAPBIT(block))
#define CB_MARK_BLOCK_PRESENT(c, block) \
((c)->c_blockmap[CB_MAPIDX(block)] |= CB_MAPBIT(block))
#define CB_MARK_BLOCK_VACANT(c, block) \
((c)->c_blockmap[CB_MAPIDX(block)] &= ~CB_MAPBIT(block))
#define CB_PTR_TO_BLOCK(c, ptr) \
((vm_offset_t)((caddr_t)(ptr) - (c)->c_buffer) / (c)->c_blocksize)
#define CB_BLOCK_TO_PTR(c, block) \
((c)->c_buffer + ((block) * (c)->c_blocksize))
#define CB_PAGE_PRESENT(c, page) ((c)->c_pagemap[CB_MAPIDX(page)] & CB_MAPBIT(page))
#define CB_MARK_PAGE_PRESENT(c, page) ((c)->c_pagemap[CB_MAPIDX(page)] |= CB_MAPBIT(page))
#define CB_MARK_PAGE_VACANT(c, page) ((c)->c_pagemap[CB_MAPIDX(page)] &= ~CB_MAPBIT(page))
#define CB_IOPAGE_BUSY(c, page) ((c)->c_iopagemap[CB_MAPIDX(page)] & CB_MAPBIT(page))
#define CB_MARK_IOPAGE_BUSY(c, page) ((c)->c_iopagemap[CB_MAPIDX(page)] |= CB_MAPBIT(page))
#define CB_MARK_IOPAGE_UNBUSY(c, page) ((c)->c_iopagemap[CB_MAPIDX(page)] &= ~CB_MAPBIT(page))
#define CB_PTR_TO_PAGE(c, ptr) (((ptr) - (c)->c_buffer) / PAGE_SIZE)
#define CB_PAGE_TO_PTR(c, page) ((c)->c_buffer + ((page) * PAGE_SIZE))
#define CB_PAGE_VACANT(c, page) \
(!((c)->c_blockmap[CB_MAPIDX(CB_PAGE_TO_BLOCK((c), (page)))] & \
(((1 << CB_PAGEBLOCKS(c)) - 1) << (CB_PAGE_TO_BLOCK((c), (page)) % \
CB_MAPFIELDBITS))))
#define _FREE_ZERO(p, c) \
do { \
if (p != NULL) { \
_FREE(p, c); \
p = NULL; \
} \
} while (0);
static struct buf *BC_private_bp = NULL;
static struct BC_cache_control BC_cache_softc;
static struct BC_cache_control *BC_cache = &BC_cache_softc;
extern kmod_info_t kmod_info;
static struct BC_playlist_header *BC_preloaded_playlist;
static int BC_preloaded_playlist_size;
static struct BC_cache_extent *BC_find_extent(u_int64_t offset, u_int64_t length, int contained);
static int BC_discard_blocks(struct BC_cache_extent *ce, u_int64_t offset, u_int64_t length);
static int BC_blocks_present(int base, int nblk);
static void BC_reader_thread(void *param0, wait_result_t param1);
static void BC_strategy_bypass(struct buf *bp);
static void BC_strategy(struct buf *bp);
static void BC_handle_write(struct buf *bp);
static int BC_terminate_readahead(void);
static int BC_terminate_cache(void);
static int BC_terminate_history(void);
static int BC_copyin_playlist(size_t length, void *uptr);
static int BC_init_cache(size_t length, caddr_t uptr, u_int64_t blocksize);
static void BC_timeout_cache(void *);
static void BC_add_history(u_int64_t offset, u_int64_t length, int flags);
static size_t BC_size_history(void);
static int BC_copyout_history(void *uptr);
static void BC_discard_history(void);
static void BC_auto_start(void);
static int BC_sysctl SYSCTL_HANDLER_ARGS;
SYSCTL_PROC(_kern, OID_AUTO, BootCache, CTLFLAG_RW,
0, 0, BC_sysctl,
"S,BC_command",
"BootCache command interface");
extern int netboot_root(void);
extern void (* mountroot_post_hook)(void);
static int BC_alloc_pagebuffer(size_t size);
static void BC_free_pagebuffer();
static void BC_free_page(int page);
extern vm_map_t convert_port_entry_to_map(ipc_port_t port);
extern vm_object_t convert_port_entry_to_object(ipc_port_t port);
extern void ipc_port_release_send(ipc_port_t port);
extern kern_return_t memory_object_page_op(
memory_object_control_t control,
memory_object_offset_t offset,
int ops,
vm_offset_t *phys_entry,
int *flags);
#ifdef __ppc__
extern void mapping_prealloc(unsigned int);
extern void mapping_relpre(void);
#endif
#define MACH_KERNEL 1
#include <mach-o/loader.h>
extern void *getsectdatafromheader(struct mach_header *, char *, char *, int *);
static struct BC_cache_extent *
BC_find_extent(u_int64_t offset, u_int64_t length, int contained)
{
struct BC_cache_extent *p, *base, *last;
size_t lim;
assert(length > 0);
base = BC_cache->c_extents;
last = base + BC_cache->c_extent_count - 1;
if (!(BC_cache->c_flags & BC_FLAG_CACHEACTIVE) || (base == NULL))
return(NULL);
if (((offset + length) <= base->ce_offset) ||
(offset >= (last->ce_offset + last->ce_length)))
return(NULL);
for (lim = BC_cache->c_extent_count; lim != 0; lim >>= 1) {
p = base + (lim >> 1);
if ((offset < (p->ce_offset + p->ce_length)) &&
((offset + length) > p->ce_offset)) {
if ((contained != 0) &&
((offset < p->ce_offset) ||
((offset + length) > (p->ce_offset + p->ce_length))))
return(NULL);
return(p);
}
if (offset > p->ce_offset) {
base = p + 1;
lim--;
}
}
return(NULL);
}
static int
BC_discard_blocks(struct BC_cache_extent *ce, u_int64_t offset, u_int64_t length)
{
u_int64_t estart, eend, dstart, dend;
int base, i, page, discarded, reallyfree, nblk;
assert(length > 0);
dstart = offset;
dend = offset + length;
assert(dend > dstart);
estart = ce->ce_offset;
eend = ce->ce_offset + ce->ce_length;
assert(eend > estart);
if (dend <= estart)
return(0);
if (eend <= dstart)
return(0);
if (dstart < estart)
dstart = estart;
if (dend > eend)
dend = eend;
xdebug("discard %lu-%lu within %lu-%lu",
(unsigned long)dstart, (unsigned long)dend, (unsigned long)estart, (unsigned long)eend);
nblk = CB_BYTE_TO_BLOCK(BC_cache, dend - dstart);
reallyfree = 1;
if (BC_cache->c_batch < (ce->ce_flags & CE_BATCH_MASK) ||
((BC_cache->c_batch == (ce->ce_flags & CE_BATCH_MASK)) &&
(BC_cache->c_extent_tail <= ce))) {
reallyfree = 0;
}
base = CB_PTR_TO_BLOCK(BC_cache, ce->ce_data + (dstart - ce->ce_offset));
assert(base >= 0);
discarded = 0;
for (i = 0; i < nblk; i++) {
assert((base + i) < BC_cache->c_buffer_blocks);
if (CB_BLOCK_PRESENT(BC_cache, base + i)) {
CB_MARK_BLOCK_VACANT(BC_cache, base + i);
discarded++;
if (!reallyfree)
continue;
page = CB_BLOCK_TO_PAGE(BC_cache, base + i);
assert(CB_PAGE_PRESENT(BC_cache, page));
if ((CB_PAGE_PRESENT(BC_cache, page)) && CB_PAGE_VACANT(BC_cache, page)) {
CB_MARK_PAGE_VACANT(BC_cache, page);
BC_free_page(page);
}
}
}
return(discarded);
}
static int
BC_blocks_present(int base, int nblk)
{
int blk;
for (blk = 0; blk < nblk; blk++) {
assert((base + blk) < BC_cache->c_buffer_blocks);
if (!CB_BLOCK_PRESENT(BC_cache, base + blk)) {
BC_cache->c_stats.ss_hit_blkmissing++;
return(0);
}
}
return(1);
}
static void
BC_reader_thread(void *param0, wait_result_t param1)
{
struct BC_cache_extent *ce = NULL;
boolean_t funnel_state;
struct buf *bp;
u_int64_t bytesdone;
int count;
int i, x;
int bcount;
uintptr_t bptr;
uintptr_t offset_in_map;
kern_return_t kret;
funnel_state = thread_funnel_set(kernel_flock, TRUE);
BC_cache->c_flags |= BC_FLAG_IOBUSY;
BC_private_bp = buf_alloc(BC_cache->c_devvp);
bp = BC_private_bp;
debug("reader thread started");
while (BC_cache->c_batch <= BC_cache->c_batch_count) {
debug("starting batch %d", BC_cache->c_batch);
for (ce = BC_cache->c_extents;
ce < (BC_cache->c_extents + BC_cache->c_extent_count);
ce++) {
if ((ce->ce_flags & CE_BATCH_MASK) != BC_cache->c_batch)
continue;
buf_setcount(bp, 0);
BC_cache->c_extent_tail = ce;
for (;;) {
if (BC_cache->c_flags & BC_FLAG_SHUTDOWN)
goto out;
if (buf_count(bp) != 0) {
daddr64_t blkno;
blkno = buf_blkno(bp) + CB_BYTE_TO_BLOCK(BC_cache, buf_count(bp));
buf_setblkno(bp, blkno);
bytesdone = CB_BLOCK_TO_BYTE(BC_cache, blkno) - ce->ce_offset;
buf_setcount(bp, MIN(ce->ce_length - bytesdone, BC_MAX_READ));
buf_setdataptr(bp, (uintptr_t)(ce->ce_data + bytesdone));
} else {
buf_setblkno(bp, (daddr64_t)CB_BYTE_TO_BLOCK(BC_cache, ce->ce_offset));
buf_setcount(bp, MIN(ce->ce_length, BC_MAX_READ));
buf_setdataptr(bp, (uintptr_t)(ce->ce_data));
}
bcount = buf_count(bp);
bptr = buf_dataptr(bp);
for (i = 0; i < bcount; ) {
CB_MARK_IOPAGE_BUSY(BC_cache, CB_PTR_TO_PAGE(BC_cache, (caddr_t)bptr));
x = 4096 - ((int)bptr & 4095);
bptr += x;
bcount -= x;
}
buf_setresid(bp, buf_count(bp));
buf_reset(bp, B_READ);
offset_in_map = (uintptr_t)(((unsigned char *)buf_dataptr(bp) - (unsigned char *)BC_cache->c_buffer) +
(unsigned char *)BC_cache->c_mapbase);
kret = vm_map_wire(BC_cache->c_map, (vm_map_offset_t)trunc_page(offset_in_map),
(vm_map_offset_t)round_page(offset_in_map+buf_count(bp)),
VM_PROT_READ | VM_PROT_WRITE, FALSE);
KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_DKRW, DKIO_READ) | DBG_FUNC_NONE,
(unsigned int)bp, buf_device(bp), (int)buf_blkno(bp), buf_count(bp), 0);
BC_cache->c_stats.ss_initiated_reads++;
BC_cache->c_strategy(bp);
buf_biowait(bp);
if (kret == KERN_SUCCESS) {
kret = vm_map_unwire(BC_cache->c_map, (vm_map_offset_t)trunc_page(offset_in_map),
(vm_map_offset_t)round_page(offset_in_map+buf_count(bp)), FALSE);
if (kret != KERN_SUCCESS)
panic("BootCache: vm_map_unwire returned %d\n", kret);
}
bcount = buf_count(bp);
bptr = buf_dataptr(bp);
for (i = 0; i < bcount; ) {
CB_MARK_IOPAGE_UNBUSY(BC_cache, CB_PTR_TO_PAGE(BC_cache, (caddr_t)bptr));
x = 4096 - ((int)bptr & 4095);
bptr += x;
bcount -= x;
}
wakeup(&BC_cache->c_iopagemap);
if (buf_error(bp) || (buf_resid(bp) != 0)) {
debug("read error: extent %d %lu/%lu "
"(error buf %ld/%ld flags %08x resid %d)",
ce - BC_cache->c_extents,
(unsigned long)ce->ce_offset, (unsigned long)ce->ce_length,
(long)buf_blkno(bp), (long)buf_count(bp),
buf_flags(bp), buf_resid(bp));
count = BC_discard_blocks(ce, CB_BLOCK_TO_BYTE(BC_cache, buf_blkno(bp)),
buf_count(bp));
debug("read error: discarded %d blocks", count);
BC_cache->c_stats.ss_read_errors++;
BC_cache->c_stats.ss_error_discards += count;
}
#ifdef READ_HISTORY_BUFFER
if (BC_cache->c_rhistory_idx < READ_HISTORY_BUFFER) {
BC_cache->c_rhistory[BC_cache->c_rhistory_idx].rh_extent = ce;
BC_cache->c_rhistory[BC_cache->c_rhistory_idx].rh_blkno = buf_blkno(bp);
BC_cache->c_rhistory[BC_cache->c_rhistory_idx].rh_bcount = buf_count(bp);
if (buf_error(bp) || (buf_resid(bp) != 0)) {
BC_cache->c_rhistory[BC_cache->c_rhistory_idx].rh_result = BC_RHISTORY_FAIL;
BC_cache->c_rhistory_idx++;
} else {
# ifdef READ_HISTORY_ALL
BC_cache->c_rhistory[BC_cache->c_rhistory_idx].rh_result = BC_RHISTORY_OK;
BC_cache->c_rhistory_idx++;
# endif
}
}
#endif
if (((CB_BLOCK_TO_BYTE(BC_cache, buf_blkno(bp)) - ce->ce_offset) + buf_count(bp)) >= ce->ce_length)
break;
}
BC_cache->c_stats.ss_read_blocks += CB_BYTE_TO_BLOCK(BC_cache, ce->ce_length);
wakeup(ce);
}
debug("batch %d done", BC_cache->c_batch);
BC_cache->c_extent_tail = BC_cache->c_extents;
BC_cache->c_batch++;
microtime(&BC_cache->c_stats.ss_batch_time[MIN(BC_cache->c_batch, STAT_BATCHMAX)]);
ce = NULL;
}
out:
if (ce != NULL) {
struct BC_cache_extent *tmp = BC_cache->c_extents;
while ((tmp - BC_cache->c_extents) < BC_cache->c_extent_count) {
if (BC_cache->c_batch < (tmp->ce_flags & CE_BATCH_MASK)) {
tmp->ce_flags |= CE_ABORTED;
}
if ((tmp >= ce) && (BC_cache->c_batch == (tmp->ce_flags & CE_BATCH_MASK))) {
tmp->ce_flags |= CE_ABORTED;
}
wakeup(tmp);
tmp++;
}
}
microtime(&BC_cache->c_stats.ss_batch_time[STAT_BATCHMAX]);
BC_cache->c_flags &= ~BC_FLAG_IOBUSY;
wakeup(&BC_cache->c_flags);
debug("reader thread done in %d sec",
(int) BC_cache->c_stats.ss_batch_time[STAT_BATCHMAX].tv_sec -
(int) BC_cache->c_stats.ss_batch_time[0].tv_sec);
BC_private_bp = NULL;
buf_free(bp);
(void) thread_funnel_set(kernel_flock, funnel_state);
}
static void
BC_strategy_bypass(struct buf *bp)
{
int isread;
assert(bp != NULL);
BC_cache->c_bypasscalls++;
if (buf_flags(bp) & B_READ) {
BC_add_history(CB_BLOCK_TO_BYTE(BC_cache, buf_blkno(bp)), buf_count(bp), BC_HE_MISS);
isread = 1;
} else {
isread = 0;
}
if (BC_cache->c_flags & BC_FLAG_CACHEACTIVE) {
BC_cache->c_stats.ss_strategy_bypassed++;
if (BC_cache->c_flags & BC_FLAG_IOBUSY)
BC_cache->c_stats.ss_strategy_bypass_active++;
}
if (isread &&
(BC_cache->c_extents != NULL) &&
(BC_cache->c_flags & BC_FLAG_CACHEACTIVE)) {
if (!(BC_cache->c_flags & BC_FLAG_IOBUSY) &&
(BC_cache->c_stats.ss_extent_lookups > BC_chit_check_start) &&
(((BC_cache->c_stats.ss_extent_hits * 100) /
BC_cache->c_stats.ss_extent_lookups) < BC_chit_low_level)) {
debug("hit rate below threshold (%d hits on %d lookups)",
BC_cache->c_stats.ss_extent_hits,
BC_cache->c_stats.ss_extent_lookups);
if (BC_terminate_readahead()) {
message("could not stop readahead on bad cache hitrate");
} else {
if (BC_terminate_cache()) {
message("could not terminate cache on bad hitrate");
}
}
}
}
KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_BOOTCACHE, DBG_BC_CUT),
(unsigned int)bp, 0, 0, 0, 0);
BC_cache->c_strategy(bp);
BC_cache->c_bypasscalls--;
}
static void
BC_strategy(struct buf *bp)
{
struct BC_cache_extent *ce;
boolean_t funnel_state;
int base, nblk;
caddr_t p, s;
int retry;
struct timeval blocktime, now, elapsed;
daddr64_t blkno;
int bcount;
caddr_t vaddr;
assert(bp != NULL);
funnel_state = thread_funnel_set(kernel_flock, TRUE);
blkno = buf_blkno(bp);
bcount = buf_count(bp);
if (buf_device(bp) != BC_cache->c_dev) {
BC_cache->c_strategy(bp);
(void) thread_funnel_set(kernel_flock, funnel_state);
return;
}
if (!(BC_cache->c_flags & BC_FLAG_CACHEACTIVE)) {
BC_strategy_bypass(bp);
(void) thread_funnel_set(kernel_flock, funnel_state);
return;
}
BC_cache->c_strategycalls++;
BC_cache->c_stats.ss_strategy_calls++;
if ( !(buf_flags(bp) & B_READ)) {
BC_handle_write(bp);
BC_add_history(CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount, BC_HE_WRITE);
BC_cache->c_stats.ss_strategy_nonread++;
goto bypass;
}
BC_cache->c_stats.ss_requested_blocks += CB_BYTE_TO_BLOCK(BC_cache, bcount);
BC_cache->c_stats.ss_extent_lookups++;
if ((ce = BC_find_extent(CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount, 1)) == NULL)
goto bypass;
BC_cache->c_stats.ss_extent_hits++;
blocktime.tv_sec = 0;
for (retry = 0; ; retry++) {
if (!(BC_cache->c_flags & BC_FLAG_IOBUSY))
break;
if ((ce->ce_flags & CE_BATCH_MASK) < BC_cache->c_batch) {
break;
}
if(((ce->ce_flags & CE_BATCH_MASK) == BC_cache->c_batch) &&
ce < BC_cache->c_extent_tail) {
break;
}
if (ce->ce_flags & CE_ABORTED)
break;
if (retry > (BC_wait_for_readahead * 10)) {
debug("timed out waiting for extent to be read (need %d, tail %d)",
ce - BC_cache->c_extents, BC_cache->c_extent_tail - BC_cache->c_extents);
goto bypass;
}
if (blocktime.tv_sec == 0) {
microtime(&blocktime);
BC_cache->c_stats.ss_strategy_blocked++;
}
tsleep(ce, PRIBIO, "BC_strategy", hz / 10);
}
if (blocktime.tv_sec != 0) {
microtime(&now);
timersub(&now, &blocktime, &elapsed);
timeradd(&elapsed, &BC_cache->c_stats.ss_wait_time, &BC_cache->c_stats.ss_wait_time);
}
if (ce->ce_flags & CE_ABORTED)
goto bypass;
base = CB_PTR_TO_BLOCK(BC_cache, ce->ce_data) +
(blkno - CB_BYTE_TO_BLOCK(BC_cache, ce->ce_offset));
nblk = CB_BYTE_TO_BLOCK(BC_cache, bcount);
assert(base >= 0);
if (!BC_blocks_present(base, nblk))
goto bypass;
if ((BC_cache->c_flags & BC_FLAG_IOBUSY))
BC_cache->c_stats.ss_strategy_duringio++;
#ifdef EMULATE_ONLY
BC_add_history(CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount, BC_HE_HIT);
BC_cache->c_stats.ss_hit_blocks +=
BC_discard_blocks(ce, CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount);
BC_cache->c_strategy(bp);
#else
if (buf_map(bp, &vaddr)) {
goto bypass;
}
p = vaddr;
if (!BC_blocks_present(base, nblk)) {
buf_unmap(bp);
BC_cache->c_stats.ss_strategy_stolen++;
goto bypass;
}
s = CB_BLOCK_TO_PTR(BC_cache, base);
bcopy(s, p, bcount);
buf_setresid(bp, 0);
buf_unmap(bp);
buf_biodone(bp);
BC_cache->c_stats.ss_hit_blocks +=
BC_discard_blocks(ce, CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount);
BC_add_history(CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount, BC_HE_HIT);
#endif
BC_cache->c_strategycalls--;
(void) thread_funnel_set(kernel_flock, funnel_state);
return;
bypass:
BC_cache->c_strategycalls--;
BC_strategy_bypass(bp);
(void) thread_funnel_set(kernel_flock, funnel_state);
return;
}
static void
BC_handle_write(struct buf *bp)
{
struct BC_cache_extent *ce, *p;
int count;
daddr64_t blkno;
int bcount;
assert(bp != NULL);
blkno = buf_blkno(bp);
bcount = buf_count(bp);
if ((ce = BC_find_extent(CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount, 0)) == NULL)
return;
count = BC_discard_blocks(ce, CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount);
BC_cache->c_stats.ss_write_discards += count;
assert(count != 0);
p = ce - 1;
while (p >= BC_cache->c_extents) {
count = BC_discard_blocks(p, CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount);
if (count == 0)
break;
BC_cache->c_stats.ss_write_discards += count;
p--;
}
p = ce + 1;
while (p < (BC_cache->c_extents + BC_cache->c_extent_count)) {
count = BC_discard_blocks(p, CB_BLOCK_TO_BYTE(BC_cache, blkno), bcount);
if (count == 0)
break;
BC_cache->c_stats.ss_write_discards += count;
p++;
}
}
static int
BC_terminate_readahead(void)
{
int error;
if (BC_cache->c_flags & BC_FLAG_IOBUSY) {
debug("terminating active readahead");
BC_cache->c_flags |= BC_FLAG_SHUTDOWN;
error = tsleep(&BC_cache->c_flags, PRIBIO, "BC_terminate_readahead", hz * 10);
if (error == EWOULDBLOCK) {
debug("timed out waiting for I/O to stop");
if (!(BC_cache->c_flags & BC_FLAG_IOBUSY)) {
debug("but I/O has stopped!");
} else {
debug("doing extent %d of %d %lu/%lu @ %p",
BC_cache->c_extent_tail - BC_cache->c_extents,
BC_cache->c_extent_count,
(unsigned long)BC_cache->c_extent_tail->ce_offset,
(unsigned long)BC_cache->c_extent_tail->ce_length,
BC_cache->c_extent_tail->ce_data);
debug("current buf:");
if (BC_private_bp) {
debug(" blkno %qd bcount %d resid %d flags 0x%x dataptr %p",
buf_blkno(BC_private_bp),
buf_count(BC_private_bp),
buf_resid(BC_private_bp),
buf_flags(BC_private_bp),
(void *) buf_dataptr(BC_private_bp));
} else
debug("NULL pointer");
}
#ifdef DEBUG
Debugger("I/O Kit wedged on us");
#endif
return(EBUSY);
}
}
return(0);
}
static int
BC_terminate_cache(void)
{
int retry, i;
if (BC_cache->c_flags & BC_FLAG_IOBUSY) {
debug("cannot terminate cache while readahead is in progress");
return(EBUSY);
}
if (!(BC_cache->c_flags & BC_FLAG_CACHEACTIVE)) {
debug("cannot terminate cache when not active");
return(ENXIO);
}
BC_cache->c_flags &= ~BC_FLAG_CACHEACTIVE;
debug("terminating cache...");
retry = 0;
while (BC_cache->c_strategycalls > 0) {
tsleep(BC_cache, PRIBIO, "BC_terminate_cache", hz / 10);
if (retry++ > 50) {
debug("could not terminate cache, timed out with %d caller%s in BC_strategy",
BC_cache->c_strategycalls,
BC_cache->c_strategycalls == 1 ? "" : "s");
BC_cache->c_flags |= BC_FLAG_CACHEACTIVE;
return(EBUSY);
}
}
BC_cache->c_stats.ss_spurious_blocks = 0;
for (i = 0; i < BC_cache->c_buffer_blocks; i++)
if (CB_BLOCK_PRESENT(BC_cache, i))
BC_cache->c_stats.ss_spurious_blocks++;
BC_cache->c_stats.ss_spurious_pages = 0;
for (i = 0; i < BC_cache->c_buffer_pages; i++)
if (CB_PAGE_PRESENT(BC_cache, i)) {
BC_cache->c_stats.ss_spurious_pages++;
BC_free_page(i);
}
BC_free_pagebuffer();
_FREE_ZERO(BC_cache->c_extents, M_TEMP);
_FREE_ZERO(BC_cache->c_pagemap, M_TEMP);
_FREE_ZERO(BC_cache->c_iopagemap, M_TEMP);
_FREE_ZERO(BC_cache->c_blockmap, M_TEMP);
return(0);
}
static int
BC_terminate_history(void)
{
int retry;
debug("terminating history collection...");
if ((BC_cache->c_devsw != NULL) &&
(BC_cache->c_strategy != NULL))
BC_cache->c_devsw->d_strategy = BC_cache->c_strategy;
retry = 0;
while (BC_cache->c_bypasscalls > 0) {
tsleep(BC_cache, PRIBIO, "BC_terminate_history", hz / 10);
if (retry++ > 50) {
debug("could not terminate history, timed out with %d caller%s in BC_strategy_bypass",
BC_cache->c_bypasscalls,
BC_cache->c_bypasscalls == 1 ? "" : "s");
return(EBUSY);
}
}
microtime(&BC_cache->c_stats.ss_cache_stop);
return(0);
}
static int
BC_copyin_playlist(size_t length, void *uptr)
{
struct BC_playlist_entry *pce;
struct BC_cache_extent *ce;
int error, idx;
caddr_t p;
u_int64_t size;
int entries;
int actual;
error = 0;
pce = NULL;
entries = length / sizeof(*pce);
BC_cache->c_extents = _MALLOC(entries * sizeof(struct BC_cache_extent),
M_TEMP, M_WAITOK);
if (BC_cache->c_extents == NULL) {
message("can't allocate memory for extents");
error = ENOMEM;
goto out;
}
BC_cache->c_extent_count = entries;
BC_cache->c_stats.ss_total_extents = entries;
ce = BC_cache->c_extents;
size = 0;
if (BC_preloaded_playlist) {
debug("using static playlist with %d entries", entries);
pce = (struct BC_playlist_entry *)uptr;
for (idx = 0; idx < entries; idx++) {
ce[idx].ce_offset = pce[idx].pce_offset;
ce[idx].ce_length = pce[idx].pce_length;
ce[idx].ce_flags = pce[idx].pce_batch & CE_BATCH_MASK;
#ifdef IGNORE_BATCH
ce[idx].ce_flags &= ~CE_BATCH_MASK;
#endif
ce[idx].ce_data = NULL;
size += pce[idx].pce_length;
if (ce[idx].ce_flags > BC_cache->c_batch_count) {
BC_cache->c_batch_count = ce[idx].ce_flags;
}
}
} else {
debug("using supplied playlist with %d entries", entries);
pce = _MALLOC(BC_PLC_CHUNK * sizeof(struct BC_playlist_entry),
M_TEMP, M_WAITOK);
if (pce == NULL) {
message("can't allocate unpack buffer");
goto out;
}
while (entries > 0) {
actual = min(entries, BC_PLC_CHUNK);
if ((error = copyin(CAST_USER_ADDR_T(uptr), pce,
actual * sizeof(struct BC_playlist_entry))) != 0) {
message("copyin from %p to %p failed", uptr, pce);
_FREE(pce, M_TEMP);
goto out;
}
for (idx = 0; idx < actual; idx++) {
ce[idx].ce_offset = pce[idx].pce_offset;
ce[idx].ce_length = pce[idx].pce_length;
ce[idx].ce_flags = pce[idx].pce_batch & CE_BATCH_MASK;
#ifdef IGNORE_BATCH
ce[idx].ce_flags &= ~CE_BATCH_MASK;
#endif
ce[idx].ce_data = NULL;
size += pce[idx].pce_length;
if (ce[idx].ce_flags > BC_cache->c_batch_count) {
BC_cache->c_batch_count = ce[idx].ce_flags;
}
}
entries -= actual;
uptr = (struct BC_playlist_entry *)uptr + actual;
ce += actual;
}
_FREE(pce, M_TEMP);
}
size = roundup(size, PAGE_SIZE * CB_MAPFIELDBITS);
if (size > (mem_size / 2)) {
message("cache size (%lu bytes) too large for physical memory (%u bytes)",
(unsigned long)size, mem_size);
_FREE_ZERO(BC_cache->c_extents, M_TEMP);
BC_cache->c_extents = NULL;
BC_cache->c_extent_count = 0;
BC_cache->c_stats.ss_total_extents = 0;
return(0);
}
BC_cache->c_buffer_blocks = CB_BYTE_TO_BLOCK(BC_cache, size);
BC_cache->c_buffer_pages = CB_BLOCK_TO_PAGE(BC_cache, BC_cache->c_buffer_blocks);
if ((BC_cache->c_blockmap =
_MALLOC(BC_cache->c_buffer_blocks / (CB_MAPFIELDBITS / CB_MAPFIELDBYTES),
M_TEMP, M_WAITOK)) == NULL) {
message("can't allocate %d bytes for blockmap",
BC_cache->c_buffer_blocks / CB_MAPFIELDBYTES);
error = ENOMEM;
goto out;
}
if ((BC_cache->c_pagemap =
_MALLOC(BC_cache->c_buffer_pages / (CB_MAPFIELDBITS / CB_MAPFIELDBYTES),
M_TEMP, M_WAITOK)) == NULL) {
message("can't allocate %d bytes for pagemap",
BC_cache->c_buffer_pages / CB_MAPFIELDBYTES);
error = ENOMEM;
goto out;
}
if ((BC_cache->c_iopagemap =
_MALLOC(BC_cache->c_buffer_pages / (CB_MAPFIELDBITS / CB_MAPFIELDBYTES),
M_TEMP, M_WAITOK)) == NULL) {
message("can't allocate %d bytes for iopagemap",
BC_cache->c_buffer_pages / CB_MAPFIELDBYTES);
error = ENOMEM;
goto out;
}
bzero(BC_cache->c_iopagemap, BC_cache->c_buffer_pages / (CB_MAPFIELDBITS / CB_MAPFIELDBYTES));
if (BC_alloc_pagebuffer(BC_cache->c_buffer_pages * PAGE_SIZE) != 0) {
message("can't allocate %d bytes for cache buffer",
BC_cache->c_buffer_pages * PAGE_SIZE);
error = ENOMEM;
goto out;
}
for (idx = 0; idx < BC_cache->c_buffer_pages; idx++)
CB_MARK_PAGE_PRESENT(BC_cache, idx);
for (idx = 0; idx < BC_cache->c_buffer_blocks; idx++)
CB_MARK_BLOCK_PRESENT(BC_cache, idx);
p = BC_cache->c_buffer;
for (idx = 0; idx < BC_cache->c_extent_count; idx++) {
(BC_cache->c_extents + idx)->ce_data = p;
p += BC_cache->c_extents[idx].ce_length;
}
out:
if (error != 0) {
debug("cache setup failed, aborting");
if (BC_cache->c_buffer != NULL)
BC_free_pagebuffer();
_FREE_ZERO(BC_cache->c_blockmap, M_TEMP);
_FREE_ZERO(BC_cache->c_pagemap, M_TEMP);
_FREE_ZERO(BC_cache->c_iopagemap, M_TEMP);
_FREE_ZERO(BC_cache->c_extents, M_TEMP);
BC_cache->c_extent_count = 0;
}
return(error);
}
static int
BC_init_cache(size_t length, caddr_t uptr, u_int64_t blocksize)
{
u_int32_t blksize;
u_int64_t blkcount;
int error;
unsigned int boot_arg;
boolean_t funnel_state;
thread_t rthread;
error = 0;
if (BC_cache->c_flags != 0)
return(EBUSY);
bzero(BC_cache, sizeof(*BC_cache));
if ((mem_size / (1024 * 1024)) < BC_minimum_mem_size) {
debug("insufficient physical memory (%dMB < %dMB required)",
(int)(mem_size / (1024 * 1024)), BC_minimum_mem_size);
return(ENOMEM);
}
if (netboot_root() &&
(!PE_parse_boot_arg("BootCacheOverride", &boot_arg) ||
(boot_arg != 1)))
return(ENXIO);
BC_cache->c_dev = rootdev;
BC_cache->c_devvp = rootvp;
BC_cache->c_devsw = &bdevsw[major(rootdev)];
assert(BC_cache->c_devsw != NULL);
BC_cache->c_strategy = BC_cache->c_devsw->d_strategy;
if (BC_cache->c_strategy == NULL)
return(ENXIO);
BC_cache->c_devsw->d_ioctl(BC_cache->c_dev,
#warning SHOULD USE DKIOCGETBLOCKSIZE64
DKIOCGETBLOCKSIZE,
(caddr_t)&blksize,
0,
NULL);
if (blksize == 0) {
message("can't determine device block size, defaulting to 512 bytes");
blksize = 512;
}
BC_cache->c_devsw->d_ioctl(BC_cache->c_dev,
DKIOCGETBLOCKCOUNT,
(caddr_t)&blkcount,
0,
NULL);
if (blkcount == 0) {
message("can't determine device size, not checking");
}
BC_cache->c_blocksize = blksize;
BC_cache->c_devsize = BC_cache->c_blocksize * blkcount;
BC_cache->c_stats.ss_blocksize = BC_cache->c_blocksize;
debug("blocksize %lu bytes, filesystem %lu bytes",
(unsigned long)BC_cache->c_blocksize, (unsigned long)BC_cache->c_devsize);
if (length > 0) {
if (BC_cache->c_blocksize > blocksize) {
message("playlist blocksize %lu not compatible with root device's %lu",
(unsigned long)blocksize, (unsigned long)BC_cache->c_blocksize);
return(EINVAL);
}
if ((error = BC_copyin_playlist(length, uptr)) != 0) {
message("can't load playlist");
return(error);
}
} else {
debug("no playlist, recording only");
}
microtime(&BC_cache->c_stats.ss_batch_time[0]);
#ifndef NO_HISTORY
# ifdef STATIC_HISTORY
if ((BC_cache->c_history = _MALLOC(BC_HISTORY_ALLOC, M_TEMP, M_WAITOK)) == NULL) {
message("can't allocate %d bytes for static history buffer", BC_HISTORY_ALLOC);
return(ENOMEM);
}
bzero(BC_cache->c_history, BC_HISTORY_ALLOC);
BC_cache->c_stats.ss_history_clusters = 1;
# endif
#endif
#ifdef READ_HISTORY_BUFFER
if ((BC_cache->c_rhistory = _MALLOC(READ_HISTORY_BUFFER * sizeof(struct BC_read_history_entry),
M_TEMP, M_WAITOK)) == NULL) {
message("can't allocate %d bytes for read history buffer",
READ_HISTORY_BUFFER * sizeof(struct BC_read_history_entry));
return(ENOMEM);
}
bzero(BC_cache->c_rhistory, READ_HISTORY_BUFFER * sizeof(struct BC_read_history_entry));
#endif
funnel_state = thread_funnel_set(kernel_flock, TRUE);
#ifdef EMULATE_ONLY
BC_cache->c_extent_tail = BC_cache->c_extents + BC_cache->c_extent_count;
debug("emulated complete cache fill");
#else
if (BC_cache->c_extents != NULL) {
debug("starting readahead");
BC_cache->c_flags |= BC_FLAG_IOBUSY;
BC_cache->c_batch = 0;
kernel_thread_start(BC_reader_thread, NULL, &rthread);
thread_deallocate(rthread);
}
#endif
BC_cache->c_devsw->d_strategy = BC_strategy;
BC_cache->c_flags |= BC_FLAG_CACHEACTIVE;
(void) thread_funnel_set(kernel_flock, funnel_state);
return(0);
}
static void
BC_timeout_cache(void *junk)
{
boolean_t funnel_state;
funnel_state = thread_funnel_set(kernel_flock, TRUE);
BC_terminate_readahead();
BC_terminate_cache();
BC_terminate_history();
(void) thread_funnel_set(kernel_flock, funnel_state);
}
static void
BC_add_history(u_int64_t offset, u_int64_t length, int flags)
{
struct BC_history_cluster *hc;
struct BC_history_entry *he;
kern_return_t kret;
if (BC_cache->c_flags & BC_FLAG_HTRUNCATED)
return;
if ((BC_cache->c_history_bytes + length) > (mem_size / 2))
return;
BC_cache->c_history_bytes += length;
#ifdef NO_HISTORY
BC_cache->c_flags |= BC_FLAG_HTRUNCATED;
debug("history disabled, truncating");
return;
#endif
hc = BC_cache->c_history;
if ((hc == NULL) || (hc->hc_entries >= BC_HISTORY_ENTRIES)) {
BC_cache->c_stats.ss_history_clusters++;
if (BC_cache->c_stats.ss_history_clusters >= BC_HISTORY_MAXCLUSTERS) {
message("too many history clusters (%d, limit %ld)",
BC_cache->c_stats.ss_history_clusters,
(long)BC_HISTORY_MAXCLUSTERS);
BC_cache->c_flags |= BC_FLAG_HTRUNCATED;
BC_cache->c_stats.ss_history_clusters--;
return;
}
kret = kmem_alloc(kernel_map, (vm_offset_t *)&hc, BC_HISTORY_ALLOC);
if (kret != KERN_SUCCESS) {
message("could not allocate %d bytes for history cluster",
BC_HISTORY_ALLOC);
BC_cache->c_flags |= BC_FLAG_HTRUNCATED;
return;
}
hc->hc_entries = 0;
hc->hc_link = BC_cache->c_history;
BC_cache->c_history = hc;
}
he = &(hc->hc_data[hc->hc_entries]);
assert(he >= &hc->hc_data[0]);
assert(he < &hc->hc_data[BC_HISTORY_ENTRIES]);
he->he_offset = offset;
he->he_length = length;
he->he_flags = flags;
hc->hc_entries++;
}
static size_t
BC_size_history(void)
{
struct BC_history_cluster *hc;
int nentries;
nentries = 0;
for (hc = BC_cache->c_history; hc != NULL; hc = hc->hc_link)
nentries += hc->hc_entries;
return(nentries * sizeof(struct BC_history_entry));
}
static int
BC_copyout_history(void *uptr)
{
struct BC_history_cluster *hc, *ohc;
int error, nentries;
assert(uptr != NULL);
nentries = 0;
for (hc = BC_cache->c_history; hc != NULL; hc = hc->hc_link)
nentries += hc->hc_entries;
ohc = NULL;
for (;;) {
hc = BC_cache->c_history;
while (hc->hc_link != ohc)
hc = hc->hc_link;
ohc = hc;
if ((error = copyout(hc->hc_data, CAST_USER_ADDR_T(uptr),
hc->hc_entries * sizeof(struct BC_history_entry))) != 0)
return(error);
uptr = (caddr_t)uptr + hc->hc_entries * sizeof(struct BC_history_entry);
if (hc == BC_cache->c_history)
break;
}
return(0);
}
static void
BC_discard_history(void)
{
struct BC_history_cluster *hc;
while (BC_cache->c_history != NULL) {
hc = BC_cache->c_history;
BC_cache->c_history = hc->hc_link;
kmem_free(kernel_map, (vm_offset_t)hc, BC_HISTORY_ALLOC);
}
}
static void
BC_auto_start(void)
{
int error;
#ifdef STATIC_PLAYLIST
error = BC_init_cache(sizeof(BC_data), (caddr_t)&BC_data[0], BC_playlist_blocksize);
#else
error = BC_init_cache(BC_preloaded_playlist->ph_entries * sizeof(struct BC_playlist_entry),
(caddr_t)BC_preloaded_playlist + sizeof(struct BC_playlist_header),
BC_preloaded_playlist->ph_blocksize);
#endif
if (error != 0)
printf("BootCache autostart failed: %d\n", error);
BC_cache->c_flags |= BC_FLAG_STARTED;
}
static int
BC_sysctl SYSCTL_HANDLER_ARGS
{
struct BC_command bc;
boolean_t funnel_state;
int error;
funnel_state = thread_funnel_set(kernel_flock, TRUE);
if ((error = SYSCTL_IN(req, &bc, sizeof(bc))) != 0) {
debug("couldn't get command");
(void) thread_funnel_set(kernel_flock, funnel_state);
return(error);
}
if (bc.bc_magic != BC_MAGIC) {
debug("bad command magic");
(void) thread_funnel_set(kernel_flock, funnel_state);
return(EINVAL);
}
switch (bc.bc_opcode) {
#ifndef STATIC_PLAYLIST
case BC_OP_START:
if (BC_preloaded_playlist) {
error = EINVAL;
break;
}
debug("BC_OP_START(%ld)", bc.bc_length);
error = BC_init_cache(bc.bc_length, bc.bc_data, (u_int64_t)bc.bc_param);
if (error != 0) {
message("cache init failed");
} else {
timeout(BC_timeout_cache, NULL, hz * BC_cache_timeout);
BC_cache->c_flags |= BC_FLAG_STARTED;
}
break;
#endif
case BC_OP_STOP:
debug("BC_OP_STOP");
untimeout(BC_timeout_cache, NULL);
BC_terminate_readahead();
BC_terminate_cache();
BC_terminate_history();
if (BC_cache->c_flags & BC_FLAG_HTRUNCATED) {
bc.bc_length = 0;
} else {
bc.bc_length = BC_size_history();
}
if ((error = SYSCTL_OUT(req, &bc, sizeof(bc))) != 0)
debug("could not return history size");
break;
case BC_OP_HISTORY:
debug("BC_OP_HISTORY");
if (bc.bc_data != NULL) {
if (bc.bc_length < BC_size_history()) {
debug("supplied history buffer too small");
error = ENOMEM;
break;
}
if ((error = BC_copyout_history(bc.bc_data)) != 0)
debug("could not copy out history");
}
BC_discard_history();
BC_cache->c_flags = 0;
break;
case BC_OP_STATS:
debug("BC_OP_STATS");
if (bc.bc_length != sizeof(BC_cache->c_stats)) {
debug("stats structure wrong size");
error = ENOMEM;
} else {
BC_cache->c_stats.ss_cache_flags = BC_cache->c_flags;
if ((error = copyout(&BC_cache->c_stats, CAST_USER_ADDR_T(bc.bc_data), bc.bc_length)) != 0)
debug("could not copy out statistics");
}
break;
case BC_OP_TAG:
debug("BC_OP_TAG");
BC_add_history((u_int64_t)bc.bc_param, 0, BC_HE_TAG);
break;
default:
error = EINVAL;
}
(void) thread_funnel_set(kernel_flock, funnel_state);
return(error);
}
static int
BC_alloc_pagebuffer(size_t size)
{
kern_return_t kret;
vm_object_offset_t s;
int i;
BC_cache->c_mapsize = size;
kret = vm_region_object_create(kernel_map,
BC_cache->c_mapsize,
&BC_cache->c_map_port);
if (kret != KERN_SUCCESS) {
debug("vm_region_object_create failed - %d", kret);
return(ENOMEM);
}
BC_cache->c_map = convert_port_entry_to_map(BC_cache->c_map_port);
BC_cache->c_mapbase = 0;
kret = vm_allocate(BC_cache->c_map,
&BC_cache->c_mapbase,
BC_cache->c_mapsize,
FALSE);
if (kret != KERN_SUCCESS) {
debug("vm_allocate failed - %d", kret);
return(ENOMEM);
}
s = (vm_object_offset_t)size;
kret = mach_make_memory_entry_64(
BC_cache->c_map,
&s,
0,
VM_PROT_READ | VM_PROT_WRITE,
&BC_cache->c_entry_port,
NULL);
if ((kret != KERN_SUCCESS) || (s != size)) {
debug("mach_make_memory_entry failed - %d", kret);
return(ENOMEM);
}
kret = vm_protect(BC_cache->c_map,
BC_cache->c_mapbase,
BC_cache->c_mapsize,
TRUE,
VM_PROT_READ | VM_PROT_WRITE);
if ((kret != KERN_SUCCESS) || (s != size)) {
debug("vm_protect failed - %d", kret);
return(ENOMEM);
}
BC_cache->c_buffer = NULL;
kret = vm_map(kernel_map,
(vm_offset_t *)&BC_cache->c_buffer,
BC_cache->c_mapsize,
0,
1,
BC_cache->c_map_port,
BC_cache->c_mapbase,
FALSE,
VM_PROT_READ | VM_PROT_WRITE,
VM_PROT_READ | VM_PROT_WRITE,
VM_INHERIT_NONE);
if (kret != KERN_SUCCESS) {
debug("vm_map failed - %d", kret);
return(ENOMEM);
}
#ifdef WIRE_BUFFER
kret = vm_wire(host_priv_self(),
kernel_map,
BC_cache->c_buffer,
BC_cache->c_mapsize,
VM_PROT_READ | VM_PROT_WRITE);
if ((kret != KERN_SUCCESS) || (s != size)) {
debug("vm_wire failed - %d", kret);
return(ENOMEM);
}
#endif
#ifdef __ppc__
mapping_prealloc(BC_cache->c_mapsize);
#endif
for (i = 0; i < BC_cache->c_mapsize; i += PAGE_SIZE)
*(char *)(BC_cache->c_buffer + i) = 0;
#ifdef __ppc__
mapping_relpre();
#endif
return(0);
}
static void
BC_free_pagebuffer(void)
{
if (BC_cache->c_buffer != NULL) {
vm_deallocate(kernel_map,
(vm_address_t)BC_cache->c_buffer,
BC_cache->c_mapsize);
BC_cache->c_buffer = NULL;
BC_cache->c_mapbase = 0;
BC_cache->c_mapsize = 0;
}
if (BC_cache->c_map != 0) {
vm_map_deallocate(BC_cache->c_map);
BC_cache->c_map = 0;
}
if (BC_cache->c_entry_port != 0) {
ipc_port_release_send(BC_cache->c_entry_port);
BC_cache->c_entry_port = 0;
}
if (BC_cache->c_map_port != 0) {
ipc_port_release_send(BC_cache->c_map_port);
BC_cache->c_map_port = 0;
}
}
static void
BC_free_page(int page)
{
while (CB_IOPAGE_BUSY(BC_cache, page))
tsleep(&BC_cache->c_iopagemap, PRIBIO, "BC_free_page", hz / 10);
#ifdef WIRE_BUFFER
vm_wire(host_priv_self(),
BC_cache->c_map,
BC_cache->c_mapbase + (page * PAGE_SIZE),
PAGE_SIZE,
VM_PROT_NONE);
#endif
vm_deallocate(BC_cache->c_map,
BC_cache->c_mapbase + (page * PAGE_SIZE),
PAGE_SIZE);
mach_memory_entry_page_op(
BC_cache->c_entry_port,
(page * PAGE_SIZE),
UPL_POP_DUMP,
NULL,
NULL);
}
void
BC_start(void)
{
#ifndef STATIC_PLAYLIST
struct mach_header *mh;
int xsize;
sysctl_register_oid(&sysctl__kern_BootCache);
if (kmod_info.info_version != KMOD_INFO_VERSION) {
message("incompatible kmod_info versions");
return;
}
mh = (struct mach_header *)kmod_info.address;
BC_preloaded_playlist = getsectdatafromheader(mh, "BootCache", "playlist",
&BC_preloaded_playlist_size);
debug("preload section: %d @ %p",
BC_preloaded_playlist_size, BC_preloaded_playlist);
if (BC_preloaded_playlist != NULL) {
if (BC_preloaded_playlist_size < sizeof(struct BC_playlist_header)) {
message("preloaded playlist too small");
return;
}
if (BC_preloaded_playlist->ph_magic != PH_MAGIC) {
message("preloaded playlist has invalid magic (%x, expected %x)",
BC_preloaded_playlist->ph_magic, PH_MAGIC);
return;
}
xsize = sizeof(struct BC_playlist_header) +
(BC_preloaded_playlist->ph_entries * sizeof(struct BC_playlist_entry));
if (xsize > BC_preloaded_playlist_size) {
message("preloaded playlist too small (%d, expected at least %d)",
BC_preloaded_playlist_size, xsize);
return;
}
BC_wait_for_readahead = BC_READAHEAD_WAIT_CDROM;
mountroot_post_hook = BC_auto_start;
debug("waiting for root mount...");
} else {
BC_preloaded_playlist = NULL;
debug("waiting for playlist...");
}
#else
sysctl_register_oid(&sysctl__kern_BootCache);
mountroot_post_hook = BC_auto_start;
debug("waiting for root mount...");
#endif
}
int
BC_stop(void)
{
boolean_t funnel_state;
int error;
error = 1;
funnel_state = thread_funnel_set(kernel_flock, TRUE);
debug("preparing to unload...");
untimeout(BC_timeout_cache, NULL);
if (BC_terminate_readahead())
goto out;
BC_terminate_cache();
BC_terminate_history();
BC_discard_history();
BC_cache->c_flags = 0;
sysctl_unregister_oid(&sysctl__kern_BootCache);
error = 0;
out:
(void) thread_funnel_set(kernel_flock, funnel_state);
return(error);
}