#if !HFS_COMPRESSION
char register_decmpfs_decompressor;
char unregister_decmpfs_decompressor;
#else
#include <sys/kernel.h>
#include <sys/vnode_internal.h>
#include <sys/file_internal.h>
#include <sys/stat.h>
#include <sys/fcntl.h>
#include <sys/xattr.h>
#include <sys/namei.h>
#include <sys/user.h>
#include <sys/mount_internal.h>
#include <sys/ubc.h>
#include <sys/decmpfs.h>
#include <sys/uio_internal.h>
#include <libkern/OSByteOrder.h>
#pragma mark --- debugging ---
#define COMPRESSION_DEBUG 0
#define COMPRESSION_DEBUG_VERBOSE 0
#define MALLOC_DEBUG 0
static const char *
baseName(const char *path)
{
if (!path)
return NULL;
const char *ret = path;
int i;
for (i = 0; path[i] != 0; i++) {
if (path[i] == '/')
ret = &path[i + 1];
}
return ret;
}
#define ErrorLog(x, args...) printf("%s:%d:%s: " x, baseName(__FILE__), __LINE__, __FUNCTION__, ## args)
#if COMPRESSION_DEBUG
#define DebugLog ErrorLog
#else
#define DebugLog(x...) do { } while(0)
#endif
#if COMPRESSION_DEBUG_VERBOSE
#define VerboseLog ErrorLog
#else
#define VerboseLog(x...) do { } while(0)
#endif
#if MALLOC_DEBUG
static SInt32 totalAlloc;
typedef struct {
uint32_t allocSz;
uint32_t magic;
const char *file;
int line;
} allocated;
static void *
_malloc(uint32_t sz, __unused int type, __unused int flags, const char *file, int line)
{
uint32_t allocSz = sz + 2 * sizeof(allocated);
allocated *alloc = NULL;
MALLOC(alloc, allocated *, allocSz, type, flags);
if (!alloc) {
ErrorLog("malloc failed\n");
return NULL;
}
char *ret = (char*)&alloc[1];
allocated *alloc2 = (allocated*)(ret + sz);
alloc->allocSz = allocSz;
alloc->magic = 0xdadadada;
alloc->file = file;
alloc->line = line;
*alloc2 = *alloc;
int s = OSAddAtomic(sz, &totalAlloc);
ErrorLog("malloc(%d) -> %p, total allocations %d\n", sz, ret, s + sz);
return ret;
}
static void
_free(char *ret, __unused int type, const char *file, int line)
{
if (!ret) {
ErrorLog("freeing null\n");
return;
}
allocated *alloc = (allocated*)ret;
alloc--;
uint32_t sz = alloc->allocSz - 2 * sizeof(allocated);
allocated *alloc2 = (allocated*)(ret + sz);
if (alloc->magic != 0xdadadada) {
panic("freeing bad pointer");
}
if (memcmp(alloc, alloc2, sizeof(*alloc)) != 0) {
panic("clobbered data");
}
memset(ret, 0xce, sz);
alloc2->file = file;
alloc2->line = line;
FREE(alloc, type);
int s = OSAddAtomic(-sz, &totalAlloc);
ErrorLog("free(%p,%d) -> total allocations %d\n", ret, sz, s - sz);
}
#undef MALLOC
#undef FREE
#define MALLOC(space, cast, size, type, flags) (space) = (cast)_malloc(size, type, flags, __FILE__, __LINE__)
#define FREE(addr, type) _free((void *)addr, type, __FILE__, __LINE__)
#endif
#pragma mark --- globals ---
static lck_grp_t *decmpfs_lockgrp;
static decmpfs_registration * decompressors[CMP_MAX];
static lck_rw_t * decompressorsLock;
static int decompress_channel;
static lck_mtx_t *decompress_channel_mtx;
vfs_context_t decmpfs_ctx;
#pragma mark --- decmp_get_func ---
#define offsetof_func(func) ((uintptr_t)(&(((decmpfs_registration*)NULL)->func)))
static void *
_func_from_offset(uint32_t type, uintptr_t offset)
{
decmpfs_registration *reg = decompressors[type];
char *regChar = (char*)reg;
char *func = ®Char[offset];
void **funcPtr = (void**)func;
switch (reg->decmpfs_registration) {
case DECMPFS_REGISTRATION_VERSION_V1:
if (offset > offsetof_func(free_data))
return NULL;
break;
case DECMPFS_REGISTRATION_VERSION_V3:
if (offset > offsetof_func(get_flags))
return NULL;
break;
default:
return NULL;
}
return funcPtr[0];
}
extern void IOServicePublishResource( const char * property, boolean_t value );
extern boolean_t IOServiceWaitForMatchingResource( const char * property, uint64_t timeout );
extern boolean_t IOCatalogueMatchingDriversPresent( const char * property );
static void *
_decmp_get_func(uint32_t type, uintptr_t offset)
{
if (type >= CMP_MAX)
return NULL;
if (decompressors[type] != NULL) {
return _func_from_offset(type, offset);
}
char providesName[80];
snprintf(providesName, sizeof(providesName), "com.apple.AppleFSCompression.providesType%u", type);
if (IOCatalogueMatchingDriversPresent(providesName)) {
char resourceName[80];
uint64_t delay = 10000000ULL; snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", type);
printf("waiting for %s\n", resourceName);
while(decompressors[type] == NULL) {
lck_rw_unlock_shared(decompressorsLock); if (IOServiceWaitForMatchingResource(resourceName, delay)) {
lck_rw_lock_shared(decompressorsLock);
break;
}
if (!IOCatalogueMatchingDriversPresent(providesName)) {
printf("the kext with %s is no longer present\n", providesName);
lck_rw_lock_shared(decompressorsLock);
break;
}
printf("still waiting for %s\n", resourceName);
delay *= 2;
lck_rw_lock_shared(decompressorsLock);
}
if (decompressors[type] == NULL) {
ErrorLog("we found %s, but the type still isn't registered\n", providesName);
return NULL;
}
return _func_from_offset(type, offset);
}
ErrorLog("tried to access a compressed file of unregistered type %d\n", type);
return NULL;
}
#define decmp_get_func(type, func) ((typeof(((decmpfs_registration*)NULL)->func))_decmp_get_func(type, offsetof_func(func)))
#pragma mark --- utilities ---
#if COMPRESSION_DEBUG
static char*
vnpath(vnode_t vp, char *path, int len)
{
int origlen = len;
path[0] = 0;
vn_getpath(vp, path, &len);
path[origlen - 1] = 0;
return path;
}
static int
vnsize(vnode_t vp, uint64_t *size)
{
struct vnode_attr va;
VATTR_INIT(&va);
VATTR_WANTED(&va, va_data_size);
int error = vnode_getattr(vp, &va, decmpfs_ctx);
if (error != 0) {
ErrorLog("vnode_getattr err %d\n", error);
return error;
}
*size = va.va_data_size;
return 0;
}
#endif
#pragma mark --- cnode routines ---
void
decmpfs_cnode_init(decmpfs_cnode *cp)
{
memset(cp, 0, sizeof(*cp));
lck_rw_init(&cp->compressed_data_lock, decmpfs_lockgrp, NULL);
}
void
decmpfs_cnode_destroy(decmpfs_cnode *cp)
{
lck_rw_destroy(&cp->compressed_data_lock, decmpfs_lockgrp);
}
boolean_t
decmpfs_trylock_compressed_data(decmpfs_cnode *cp, int exclusive)
{
void *thread = current_thread();
boolean_t retval = FALSE;
if (cp->lockowner == thread) {
cp->lockcount++;
retval = TRUE;
} else if (exclusive) {
if ((retval = lck_rw_try_lock_exclusive(&cp->compressed_data_lock))) {
cp->lockowner = thread;
cp->lockcount = 1;
}
} else {
if ((retval = lck_rw_try_lock_shared(&cp->compressed_data_lock))) {
cp->lockowner = (void *)-1;
}
}
return retval;
}
void
decmpfs_lock_compressed_data(decmpfs_cnode *cp, int exclusive)
{
void *thread = current_thread();
if (cp->lockowner == thread) {
cp->lockcount++;
} else if (exclusive) {
lck_rw_lock_exclusive(&cp->compressed_data_lock);
cp->lockowner = thread;
cp->lockcount = 1;
} else {
lck_rw_lock_shared(&cp->compressed_data_lock);
cp->lockowner = (void *)-1;
}
}
void
decmpfs_unlock_compressed_data(decmpfs_cnode *cp, __unused int exclusive)
{
void *thread = current_thread();
if (cp->lockowner == thread) {
if ((--cp->lockcount) > 0) {
return;
}
cp->lockowner = NULL;
}
lck_rw_done(&cp->compressed_data_lock);
}
uint32_t
decmpfs_cnode_get_vnode_state(decmpfs_cnode *cp)
{
return cp->cmp_state;
}
void
decmpfs_cnode_set_vnode_state(decmpfs_cnode *cp, uint32_t state, int skiplock)
{
if (!skiplock) decmpfs_lock_compressed_data(cp, 1);
cp->cmp_state = state;
if (state == FILE_TYPE_UNKNOWN) {
cp->cmp_type = 0;
}
if (!skiplock) decmpfs_unlock_compressed_data(cp, 1);
}
static void
decmpfs_cnode_set_vnode_cmp_type(decmpfs_cnode *cp, uint32_t cmp_type, int skiplock)
{
if (!skiplock) decmpfs_lock_compressed_data(cp, 1);
cp->cmp_type = cmp_type;
if (!skiplock) decmpfs_unlock_compressed_data(cp, 1);
}
static void
decmpfs_cnode_set_vnode_minimal_xattr(decmpfs_cnode *cp, int minimal_xattr, int skiplock)
{
if (!skiplock) decmpfs_lock_compressed_data(cp, 1);
cp->cmp_minimal_xattr = minimal_xattr;
if (!skiplock) decmpfs_unlock_compressed_data(cp, 1);
}
uint64_t
decmpfs_cnode_get_vnode_cached_size(decmpfs_cnode *cp)
{
return cp->uncompressed_size;
}
static void
decmpfs_cnode_set_vnode_cached_size(decmpfs_cnode *cp, uint64_t size)
{
while(1) {
uint64_t old = cp->uncompressed_size;
if (OSCompareAndSwap64(old, size, (UInt64*)&cp->uncompressed_size)) {
return;
} else {
}
}
}
static uint64_t
decmpfs_cnode_get_decompression_flags(decmpfs_cnode *cp)
{
return cp->decompression_flags;
}
static void
decmpfs_cnode_set_decompression_flags(decmpfs_cnode *cp, uint64_t flags)
{
while(1) {
uint64_t old = cp->decompression_flags;
if (OSCompareAndSwap64(old, flags, (UInt64*)&cp->decompression_flags)) {
return;
} else {
}
}
}
#pragma mark --- decmpfs state routines ---
static int
decmpfs_fetch_compressed_header(vnode_t vp, decmpfs_cnode *cp, decmpfs_header **hdrOut, int returnInvalid)
{
size_t read_size = 0;
size_t attr_size = 0;
uio_t attr_uio = NULL;
int err = 0;
char *data = NULL;
decmpfs_header *hdr = NULL;
char uio_buf[ UIO_SIZEOF(1) ];
if ((cp != NULL) &&
(cp->cmp_type != 0) &&
(cp->cmp_minimal_xattr != 0)) {
MALLOC(data, char *, sizeof(decmpfs_header), M_TEMP, M_WAITOK);
if (!data) {
err = ENOMEM;
goto out;
}
hdr = (decmpfs_header*)data;
hdr->attr_size = sizeof(decmpfs_disk_header);
hdr->compression_magic = DECMPFS_MAGIC;
hdr->compression_type = cp->cmp_type;
hdr->uncompressed_size = decmpfs_cnode_get_vnode_cached_size(cp);
} else {
err = vn_getxattr(vp, DECMPFS_XATTR_NAME, NULL, &attr_size, XATTR_NOSECURITY, decmpfs_ctx);
if (err != 0)
goto out;
if (attr_size < sizeof(decmpfs_disk_header) || attr_size > MAX_DECMPFS_XATTR_SIZE) {
err = EINVAL;
goto out;
}
MALLOC(data, char *, attr_size + sizeof(hdr->attr_size), M_TEMP, M_WAITOK);
if (!data) {
err = ENOMEM;
goto out;
}
attr_uio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, &uio_buf[0], sizeof(uio_buf));
uio_addiov(attr_uio, CAST_USER_ADDR_T(data + sizeof(hdr->attr_size)), attr_size);
err = vn_getxattr(vp, DECMPFS_XATTR_NAME, attr_uio, &read_size, XATTR_NOSECURITY, decmpfs_ctx);
if (err != 0)
goto out;
if (read_size != attr_size) {
err = EINVAL;
goto out;
}
hdr = (decmpfs_header*)data;
hdr->attr_size = attr_size;
hdr->compression_magic = OSSwapLittleToHostInt32(hdr->compression_magic);
hdr->compression_type = OSSwapLittleToHostInt32(hdr->compression_type);
hdr->uncompressed_size = OSSwapLittleToHostInt64(hdr->uncompressed_size);
}
if (hdr->compression_magic != DECMPFS_MAGIC) {
ErrorLog("invalid compression_magic 0x%08x, should be 0x%08x\n", hdr->compression_magic, DECMPFS_MAGIC);
err = EINVAL;
goto out;
}
if (hdr->compression_type >= CMP_MAX) {
if (returnInvalid) {
err = ERANGE;
} else {
ErrorLog("compression_type %d out of range\n", hdr->compression_type);
err = EINVAL;
}
goto out;
}
out:
if (err && (err != ERANGE)) {
DebugLog("err %d\n", err);
if (data) FREE(data, M_TEMP);
*hdrOut = NULL;
} else {
*hdrOut = hdr;
}
return err;
}
static int
decmpfs_fast_get_state(decmpfs_cnode *cp)
{
int cmp_state = decmpfs_cnode_get_vnode_state(cp);
switch(cmp_state) {
case FILE_IS_NOT_COMPRESSED:
case FILE_IS_COMPRESSED:
case FILE_IS_CONVERTING:
return cmp_state;
case FILE_TYPE_UNKNOWN:
ErrorLog("decmpfs_fast_get_state called on unknown file\n");
return FILE_IS_NOT_COMPRESSED;
default:
ErrorLog("unknown cmp_state %d\n", cmp_state);
return FILE_IS_NOT_COMPRESSED;
}
}
static int
decmpfs_fast_file_is_compressed(decmpfs_cnode *cp)
{
int cmp_state = decmpfs_cnode_get_vnode_state(cp);
switch(cmp_state) {
case FILE_IS_NOT_COMPRESSED:
return 0;
case FILE_IS_COMPRESSED:
case FILE_IS_CONVERTING:
return 1;
case FILE_TYPE_UNKNOWN:
ErrorLog("decmpfs_fast_get_state called on unknown file\n");
return 0;
default:
ErrorLog("unknown cmp_state %d\n", cmp_state);
return 0;
}
}
errno_t
decmpfs_validate_compressed_file(vnode_t vp, decmpfs_cnode *cp)
{
decmpfs_header *hdr = NULL;
errno_t err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
if (err) {
if (decmpfs_fast_get_state(cp) == FILE_IS_NOT_COMPRESSED) {
err = 0;
}
goto out;
}
lck_rw_lock_shared(decompressorsLock);
decmpfs_validate_compressed_file_func validate = decmp_get_func(hdr->compression_type, validate);
if (validate) {
err = validate(vp, decmpfs_ctx, hdr);
} else if (decmp_get_func(hdr->compression_type, fetch) == NULL) {
err = EIO;
} else {
err = 0;
}
lck_rw_unlock_shared(decompressorsLock);
out:
if (hdr) FREE(hdr, M_TEMP);
#if COMPRESSION_DEBUG
if (err) {
DebugLog("decmpfs_validate_compressed_file ret %d, vp->v_flag %d\n", err, vp->v_flag);
}
#endif
return err;
}
int
decmpfs_file_is_compressed(vnode_t vp, decmpfs_cnode *cp)
{
int ret = 0;
int error = 0;
uint32_t cmp_state;
struct vnode_attr va_fetch;
decmpfs_header *hdr = NULL;
mount_t mp = NULL;
int cnode_locked = 0;
int saveInvalid = 0; uint64_t decompression_flags = 0;
if (vnode_isnamedstream(vp)) {
return 0;
}
cmp_state = decmpfs_cnode_get_vnode_state(cp);
switch(cmp_state) {
case FILE_IS_NOT_COMPRESSED:
return 0;
case FILE_IS_COMPRESSED:
return 1;
case FILE_IS_CONVERTING:
return 1;
case FILE_TYPE_UNKNOWN:
break;
default:
ErrorLog("unknown cmp_state %d\n", cmp_state);
return 0;
}
if (!vnode_isreg(vp)) {
ret = FILE_IS_NOT_COMPRESSED;
goto done;
}
mp = vnode_mount(vp);
if (mp == NULL) {
ret = FILE_IS_NOT_COMPRESSED;
goto done;
}
if ((mp->mnt_flag & MNT_LOCAL) == 0) {
ret = FILE_IS_NOT_COMPRESSED;
goto done;
}
decmpfs_lock_compressed_data(cp, 1);
cnode_locked = 1;
VATTR_INIT(&va_fetch);
VATTR_WANTED(&va_fetch, va_flags);
error = vnode_getattr(vp, &va_fetch, decmpfs_ctx);
if (error) {
ret = FILE_IS_NOT_COMPRESSED;
goto done;
}
if (va_fetch.va_flags & UF_COMPRESSED) {
error = decmpfs_fetch_compressed_header(vp, cp, &hdr, 1);
if ((hdr != NULL) && (error == ERANGE)) {
saveInvalid = 1;
}
if (error) {
ret = FILE_IS_NOT_COMPRESSED;
goto done;
}
ret = FILE_IS_COMPRESSED;
goto done;
}
ret = FILE_IS_NOT_COMPRESSED;
done:
if (((ret == FILE_IS_COMPRESSED) || saveInvalid) && hdr) {
if (!cnode_locked) {
decmpfs_lock_compressed_data(cp, 1);
cnode_locked = 1;
}
decmpfs_cnode_set_vnode_cached_size(cp, hdr->uncompressed_size);
decmpfs_cnode_set_vnode_state(cp, ret, 1);
decmpfs_cnode_set_vnode_cmp_type(cp, hdr->compression_type, 1);
if (hdr->attr_size == sizeof(decmpfs_disk_header)) {
decmpfs_cnode_set_vnode_minimal_xattr(cp, 1, 1);
}
if (ret == FILE_IS_COMPRESSED) {
ubc_setsize(vp, hdr->uncompressed_size);
lck_rw_lock_shared(decompressorsLock);
decmpfs_get_decompression_flags_func get_flags = decmp_get_func(hdr->compression_type, get_flags);
if (get_flags) {
decompression_flags = get_flags(vp, decmpfs_ctx, hdr);
}
lck_rw_unlock_shared(decompressorsLock);
decmpfs_cnode_set_decompression_flags(cp, decompression_flags);
}
} else {
decmpfs_cnode_set_vnode_state(cp, ret, cnode_locked);
}
if (cnode_locked) decmpfs_unlock_compressed_data(cp, 1);
if (hdr) FREE(hdr, M_TEMP);
switch(ret) {
case FILE_IS_NOT_COMPRESSED:
return 0;
case FILE_IS_COMPRESSED:
case FILE_IS_CONVERTING:
return 1;
default:
ErrorLog("unknown ret %d\n", ret);
return 0;
}
}
int
decmpfs_update_attributes(vnode_t vp, struct vnode_attr *vap)
{
int error = 0;
if (VATTR_IS_ACTIVE(vap, va_flags)) {
if (vap->va_flags & UF_COMPRESSED) {
struct vnode_attr va_fetch;
int old_flags = 0;
VATTR_INIT(&va_fetch);
VATTR_WANTED(&va_fetch, va_flags);
error = vnode_getattr(vp, &va_fetch, decmpfs_ctx);
if (error)
return error;
old_flags = va_fetch.va_flags;
if (!(old_flags & UF_COMPRESSED)) {
if (VATTR_IS_ACTIVE(vap, va_data_size)) {
vap->va_flags &= ~UF_COMPRESSED;
return 0;
}
decmpfs_header *hdr = NULL;
error = decmpfs_fetch_compressed_header(vp, NULL, &hdr, 1);
if (error == 0) {
VATTR_SET_ACTIVE(vap, va_data_size);
vap->va_data_size = 0;
} else if (error == ERANGE) {
} else {
vap->va_flags &= ~UF_COMPRESSED;
}
if (hdr) FREE(hdr, M_TEMP);
}
}
}
return 0;
}
static int
wait_for_decompress(decmpfs_cnode *cp)
{
int state;
lck_mtx_lock(decompress_channel_mtx);
do {
state = decmpfs_fast_get_state(cp);
if (state != FILE_IS_CONVERTING) {
lck_mtx_unlock(decompress_channel_mtx);
return state;
}
msleep((caddr_t)&decompress_channel, decompress_channel_mtx, PINOD, "wait_for_decompress", NULL);
} while(1);
}
#pragma mark --- decmpfs hide query routines ---
int
decmpfs_hides_rsrc(vfs_context_t ctx, decmpfs_cnode *cp)
{
if (ctx == decmpfs_ctx)
return 0;
if (!decmpfs_fast_file_is_compressed(cp))
return 0;
return 1;
}
int
decmpfs_hides_xattr(vfs_context_t ctx, decmpfs_cnode *cp, const char *xattr)
{
if (ctx == decmpfs_ctx)
return 0;
if (strncmp(xattr, XATTR_RESOURCEFORK_NAME, 22) == 0)
return decmpfs_hides_rsrc(ctx, cp);
if (!decmpfs_fast_file_is_compressed(cp))
return 0;
if (strncmp(xattr, DECMPFS_XATTR_NAME, 11) == 0)
return 1;
return 0;
}
#pragma mark --- registration/validation routines ---
static inline int registration_valid(decmpfs_registration *registration)
{
return registration && ((registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V1) || (registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V3));
}
errno_t
register_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration)
{
errno_t ret = 0;
int locked = 0;
char resourceName[80];
if ((compression_type >= CMP_MAX) || !registration_valid(registration)) {
ret = EINVAL;
goto out;
}
lck_rw_lock_exclusive(decompressorsLock); locked = 1;
if (decompressors[compression_type] != NULL) {
ret = EEXIST;
goto out;
}
decompressors[compression_type] = registration;
snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type);
IOServicePublishResource(resourceName, TRUE);
out:
if (locked) lck_rw_unlock_exclusive(decompressorsLock);
return ret;
}
errno_t
unregister_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration)
{
errno_t ret = 0;
int locked = 0;
char resourceName[80];
if ((compression_type >= CMP_MAX) || !registration_valid(registration)) {
ret = EINVAL;
goto out;
}
lck_rw_lock_exclusive(decompressorsLock); locked = 1;
if (decompressors[compression_type] != registration) {
ret = EEXIST;
goto out;
}
decompressors[compression_type] = NULL;
snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type);
IOServicePublishResource(resourceName, FALSE);
out:
if (locked) lck_rw_unlock_exclusive(decompressorsLock);
return ret;
}
static int
compression_type_valid(decmpfs_header *hdr)
{
int ret = 0;
lck_rw_lock_shared(decompressorsLock);
if (decmp_get_func(hdr->compression_type, fetch) != NULL) {
ret = 1;
}
lck_rw_unlock_shared(decompressorsLock);
return ret;
}
#pragma mark --- compression/decompression routines ---
static int
decmpfs_fetch_uncompressed_data(vnode_t vp, decmpfs_cnode *cp, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read)
{
int err = 0;
*bytes_read = 0;
if ((uint64_t)offset >= hdr->uncompressed_size) {
err = 0;
goto out;
}
if (offset < 0) {
err = EINVAL;
goto out;
}
if ((uint64_t)(offset + size) > hdr->uncompressed_size) {
size = hdr->uncompressed_size - offset;
}
if (size == 0) {
err = 0;
goto out;
}
lck_rw_lock_shared(decompressorsLock);
decmpfs_fetch_uncompressed_data_func fetch = decmp_get_func(hdr->compression_type, fetch);
if (fetch) {
err = fetch(vp, decmpfs_ctx, hdr, offset, size, nvec, vec, bytes_read);
lck_rw_unlock_shared(decompressorsLock);
if (err == 0) {
uint64_t decompression_flags = decmpfs_cnode_get_decompression_flags(cp);
if (decompression_flags & DECMPFS_FLAGS_FORCE_FLUSH_ON_DECOMPRESS) {
#if !defined(__i386__) && !defined(__x86_64__)
int i;
for (i = 0; i < nvec; i++) {
flush_dcache64((addr64_t)(uintptr_t)vec[i].buf, vec[i].size, FALSE);
}
#endif
}
}
} else {
err = ENOTSUP;
lck_rw_unlock_shared(decompressorsLock);
}
out:
return err;
}
static kern_return_t
commit_upl(upl_t upl, upl_offset_t pl_offset, size_t uplSize, int flags, int abort)
{
kern_return_t kr = 0;
if (abort) {
VerboseLog("aborting upl, flags 0x%08x\n", flags);
kr = ubc_upl_abort_range(upl, pl_offset, uplSize, flags);
if (kr != KERN_SUCCESS)
ErrorLog("ubc_upl_commit_range error %d\n", (int)kr);
} else {
VerboseLog("committing upl, flags 0x%08x\n", flags | UPL_COMMIT_CLEAR_DIRTY);
kr = ubc_upl_commit_range(upl, pl_offset, uplSize, flags | UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_WRITTEN_BY_KERNEL);
if (kr != KERN_SUCCESS)
ErrorLog("ubc_upl_commit_range error %d\n", (int)kr);
}
return kr;
}
errno_t
decmpfs_pagein_compressed(struct vnop_pagein_args *ap, int *is_compressed, decmpfs_cnode *cp)
{
int err = 0;
struct vnode *vp = ap->a_vp;
upl_t pl = ap->a_pl;
upl_offset_t pl_offset = ap->a_pl_offset;
off_t f_offset = ap->a_f_offset;
size_t size = ap->a_size;
int flags = ap->a_flags;
off_t uplPos = 0;
user_ssize_t uplSize = 0;
void *data = NULL;
decmpfs_header *hdr = NULL;
int abort_pagein = 0;
uint64_t cachedSize = 0;
int cmpdata_locked = 0;
if(!decmpfs_trylock_compressed_data(cp, 0)) {
return EAGAIN;
}
cmpdata_locked = 1;
if (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)) {
DebugLog("pagein: unknown flags 0x%08x\n", (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)));
}
err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
if (err != 0) {
goto out;
}
cachedSize = hdr->uncompressed_size;
if (!compression_type_valid(hdr)) {
err = ENOTSUP;
goto out;
}
kern_return_t kr = ubc_upl_map(pl, (vm_offset_t*)&data);
if ((kr != KERN_SUCCESS) || (data == NULL)) {
goto out;
}
uplPos = f_offset;
uplSize = size;
if ((uint64_t)uplPos + uplSize > cachedSize) {
uplSize = cachedSize - uplPos;
}
decmpfs_vector vec;
decompress:
vec.buf = (char*)data + pl_offset;
vec.size = size;
uint64_t did_read = 0;
if (decmpfs_fast_get_state(cp) == FILE_IS_CONVERTING) {
ErrorLog("unexpected pagein during decompress\n");
err = 0;
did_read = 0;
} else {
err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, uplPos, uplSize, 1, &vec, &did_read);
}
if (err) {
DebugLog("decmpfs_fetch_uncompressed_data err %d\n", err);
int cmp_state = decmpfs_fast_get_state(cp);
if (cmp_state == FILE_IS_CONVERTING) {
DebugLog("cmp_state == FILE_IS_CONVERTING\n");
cmp_state = wait_for_decompress(cp);
if (cmp_state == FILE_IS_COMPRESSED) {
DebugLog("cmp_state == FILE_IS_COMPRESSED\n");
goto decompress;
}
}
if (cmp_state == FILE_IS_NOT_COMPRESSED) {
DebugLog("cmp_state == FILE_IS_NOT_COMPRESSED\n");
abort_pagein = 1;
*is_compressed = 0;
}
}
uint64_t total_size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
if (did_read < total_size) {
memset((char*)vec.buf + did_read, 0, total_size - did_read);
}
kr = ubc_upl_unmap(pl); data = NULL;
if (kr != KERN_SUCCESS)
ErrorLog("ubc_upl_unmap error %d\n", (int)kr);
else {
if (!abort_pagein) {
kr = commit_upl(pl, pl_offset, total_size, UPL_COMMIT_FREE_ON_EMPTY, 0);
}
}
out:
if (data) ubc_upl_unmap(pl);
if (hdr) FREE(hdr, M_TEMP);
if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 0);
if (err)
ErrorLog("err %d\n", err);
return err;
}
errno_t
decmpfs_read_compressed(struct vnop_read_args *ap, int *is_compressed, decmpfs_cnode *cp)
{
uio_t uio = ap->a_uio;
vnode_t vp = ap->a_vp;
int err = 0;
int countInt = 0;
off_t uplPos = 0;
user_ssize_t uplSize = 0;
user_ssize_t uplRemaining = 0;
off_t curUplPos = 0;
user_ssize_t curUplSize = 0;
kern_return_t kr = KERN_SUCCESS;
int abort_read = 0;
void *data = NULL;
uint64_t did_read = 0;
upl_t upl = NULL;
upl_page_info_t *pli = NULL;
decmpfs_header *hdr = NULL;
uint64_t cachedSize = 0;
off_t uioPos = 0;
user_ssize_t uioRemaining = 0;
int cmpdata_locked = 0;
decmpfs_lock_compressed_data(cp, 0); cmpdata_locked = 1;
uplPos = uio_offset(uio);
uplSize = uio_resid(uio);
VerboseLog("uplPos %lld uplSize %lld\n", uplPos, uplSize);
cachedSize = decmpfs_cnode_get_vnode_cached_size(cp);
if ((uint64_t)uplPos + uplSize > cachedSize) {
uplSize = cachedSize - uplPos;
}
countInt = (uplSize > INT_MAX) ? INT_MAX : uplSize;
err = cluster_copy_ubc_data(vp, uio, &countInt, 0);
if (err != 0)
goto out;
uioPos = uio_offset(uio);
uioRemaining = uio_resid(uio);
if ((uint64_t)uioPos + uioRemaining > cachedSize) {
uioRemaining = cachedSize - uioPos;
}
if (uioRemaining <= 0) {
goto out;
}
err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
if (err != 0) {
goto out;
}
if (!compression_type_valid(hdr)) {
err = ENOTSUP;
goto out;
}
uplPos = uioPos;
uplSize = uioRemaining;
#if COMPRESSION_DEBUG
char path[PATH_MAX];
DebugLog("%s: uplPos %lld uplSize %lld\n", vnpath(vp, path, sizeof(path)), (uint64_t)uplPos, (uint64_t)uplSize);
#endif
lck_rw_lock_shared(decompressorsLock);
decmpfs_adjust_fetch_region_func adjust_fetch = decmp_get_func(hdr->compression_type, adjust_fetch);
if (adjust_fetch) {
adjust_fetch(vp, decmpfs_ctx, hdr, &uplPos, &uplSize);
VerboseLog("adjusted uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize);
}
lck_rw_unlock_shared(decompressorsLock);
if ((uint64_t)uplPos + uplSize > cachedSize) {
uplSize = cachedSize - uplPos;
}
if (uplSize <= 0) {
goto out;
}
if (uplPos & (PAGE_SIZE - 1)) {
uplSize += (uplPos & (PAGE_SIZE - 1));
uplPos &= ~(PAGE_SIZE - 1);
}
uplSize = (uplSize + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
VerboseLog("new uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize);
uplRemaining = uplSize;
curUplPos = uplPos;
curUplSize = 0;
while(uplRemaining > 0) {
curUplPos += curUplSize;
curUplSize = uplRemaining;
if (curUplSize > MAX_UPL_SIZE * PAGE_SIZE) {
curUplSize = MAX_UPL_SIZE * PAGE_SIZE;
}
kr = ubc_create_upl(vp, curUplPos, curUplSize, &upl, &pli, UPL_SET_LITE);
if (kr != KERN_SUCCESS) {
ErrorLog("ubc_create_upl error %d\n", (int)kr);
err = EINVAL;
goto out;
}
VerboseLog("curUplPos %lld curUplSize %lld\n", (uint64_t)curUplPos, (uint64_t)curUplSize);
kr = ubc_upl_map(upl, (vm_offset_t*)&data);
if (kr != KERN_SUCCESS) {
ErrorLog("ubc_upl_map error %d\n", (int)kr);
err = EINVAL;
goto out;
}
if (!data) {
ErrorLog("ubc_upl_map mapped null\n");
err = EINVAL;
goto out;
}
decmpfs_vector vec;
decompress:
vec = (decmpfs_vector){ .buf = data, .size = curUplSize };
err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, curUplPos, curUplSize, 1, &vec, &did_read);
if (err) {
ErrorLog("decmpfs_fetch_uncompressed_data err %d\n", err);
int cmp_state = decmpfs_fast_get_state(cp);
if (cmp_state == FILE_IS_CONVERTING) {
ErrorLog("cmp_state == FILE_IS_CONVERTING\n");
cmp_state = wait_for_decompress(cp);
if (cmp_state == FILE_IS_COMPRESSED) {
ErrorLog("cmp_state == FILE_IS_COMPRESSED\n");
goto decompress;
}
}
if (cmp_state == FILE_IS_NOT_COMPRESSED) {
ErrorLog("cmp_state == FILE_IS_NOT_COMPRESSED\n");
abort_read = 1;
*is_compressed = 0;
}
kr = KERN_FAILURE;
did_read = 0;
}
memset((char*)data + did_read, 0, curUplSize - did_read);
kr = ubc_upl_unmap(upl);
if (kr == KERN_SUCCESS) {
if (abort_read) {
kr = commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1);
} else {
VerboseLog("uioPos %lld uioRemaining %lld\n", (uint64_t)uioPos, (uint64_t)uioRemaining);
if (uioRemaining) {
off_t uplOff = uioPos - curUplPos;
if (uplOff < 0) {
ErrorLog("uplOff %lld should never be negative\n", (int64_t)uplOff);
err = EINVAL;
} else {
off_t count = curUplPos + curUplSize - uioPos;
if (count < 0) {
} else {
if (count > uioRemaining)
count = uioRemaining;
int io_resid = count;
err = cluster_copy_upl_data(uio, upl, uplOff, &io_resid);
int copied = count - io_resid;
VerboseLog("uplOff %lld count %lld copied %lld\n", (uint64_t)uplOff, (uint64_t)count, (uint64_t)copied);
if (err) {
ErrorLog("cluster_copy_upl_data err %d\n", err);
}
uioPos += copied;
uioRemaining -= copied;
}
}
}
kr = commit_upl(upl, 0, curUplSize, UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE, 0);
if (err) {
goto out;
}
}
} else {
ErrorLog("ubc_upl_unmap error %d\n", (int)kr);
}
uplRemaining -= curUplSize;
}
out:
if (hdr) FREE(hdr, M_TEMP);
if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 0);
if (err) {
ErrorLog("err %d\n", err);
return err;
}
#if COMPRESSION_DEBUG
uplSize = uio_resid(uio);
if (uplSize)
VerboseLog("still %lld bytes to copy\n", uplSize);
#endif
return 0;
}
int
decmpfs_free_compressed_data(vnode_t vp, decmpfs_cnode *cp)
{
decmpfs_header *hdr = NULL;
int err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
if (err) {
ErrorLog("decmpfs_fetch_compressed_header err %d\n", err);
} else {
lck_rw_lock_shared(decompressorsLock);
decmpfs_free_compressed_data_func free_data = decmp_get_func(hdr->compression_type, free_data);
if (free_data) {
err = free_data(vp, decmpfs_ctx, hdr);
} else {
err = 0;
}
lck_rw_unlock_shared(decompressorsLock);
if (err != 0) {
ErrorLog("decompressor err %d\n", err);
}
}
err = vn_removexattr(vp, DECMPFS_XATTR_NAME, 0, decmpfs_ctx);
if (err != 0) {
goto out;
}
out:
if (hdr) FREE(hdr, M_TEMP);
return err;
}
#pragma mark --- file conversion routines ---
static int
unset_compressed_flag(vnode_t vp)
{
int err = 0;
struct vnode_attr va;
int new_bsdflags = 0;
VATTR_INIT(&va);
VATTR_WANTED(&va, va_flags);
err = vnode_getattr(vp, &va, decmpfs_ctx);
if (err != 0) {
ErrorLog("vnode_getattr err %d\n", err);
} else {
new_bsdflags = va.va_flags & ~UF_COMPRESSED;
VATTR_INIT(&va);
VATTR_SET(&va, va_flags, new_bsdflags);
err = vnode_setattr(vp, &va, decmpfs_ctx);
if (err != 0) {
ErrorLog("vnode_setattr err %d\n", err);
}
}
return err;
}
int
decmpfs_decompress_file(vnode_t vp, decmpfs_cnode *cp, off_t toSize, int truncate_okay, int skiplock)
{
int err = 0;
char *data = NULL;
uio_t uio_w = 0;
off_t offset = 0;
uint32_t old_state = 0;
uint32_t new_state = 0;
int update_file_state = 0;
int allocSize = 0;
decmpfs_header *hdr = NULL;
int cmpdata_locked = 0;
off_t remaining = 0;
uint64_t uncompressed_size = 0;
if (!skiplock) {
decmpfs_lock_compressed_data(cp, 1); cmpdata_locked = 1;
}
decompress:
old_state = decmpfs_fast_get_state(cp);
switch(old_state) {
case FILE_IS_NOT_COMPRESSED:
{
err = 0;
goto out;
}
case FILE_TYPE_UNKNOWN:
{
(void)decmpfs_file_is_compressed(vp, cp);
goto decompress;
}
case FILE_IS_COMPRESSED:
{
break;
}
default:
{
err = EINVAL;
goto out;
}
}
err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
if (err != 0) {
goto out;
}
uncompressed_size = hdr->uncompressed_size;
if (toSize == -1)
toSize = hdr->uncompressed_size;
if (toSize == 0) {
goto nodecmp;
} else if ((uint64_t)toSize > hdr->uncompressed_size) {
toSize = hdr->uncompressed_size;
}
allocSize = MIN(64*1024, toSize);
MALLOC(data, char *, allocSize, M_TEMP, M_WAITOK);
if (!data) {
err = ENOMEM;
goto out;
}
uio_w = uio_create(1, 0LL, UIO_SYSSPACE, UIO_WRITE);
if (!uio_w) {
err = ENOMEM;
goto out;
}
uio_w->uio_flags |= UIO_FLAGS_IS_COMPRESSED_FILE;
remaining = toSize;
ubc_setsize(vp, 0);
decmpfs_cnode_set_vnode_state(cp, FILE_IS_CONVERTING, 1);
while(remaining > 0) {
uint64_t bytes_read = 0;
decmpfs_vector vec = { .buf = data, .size = MIN(allocSize, remaining) };
err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset, vec.size, 1, &vec, &bytes_read);
if (err != 0) {
ErrorLog("decmpfs_fetch_uncompressed_data err %d\n", err);
goto out;
}
if (bytes_read == 0) {
break;
}
uio_reset(uio_w, offset, UIO_SYSSPACE, UIO_WRITE);
err = uio_addiov(uio_w, CAST_USER_ADDR_T(data), bytes_read);
if (err != 0) {
ErrorLog("uio_addiov err %d\n", err);
err = ENOMEM;
goto out;
}
err = VNOP_WRITE(vp, uio_w, 0, decmpfs_ctx);
if (err != 0) {
ErrorLog("VNOP_WRITE err %d\n", err);
break;
}
offset += bytes_read;
remaining -= bytes_read;
}
if (err == 0) {
if (offset != toSize) {
ErrorLog("file decompressed to %lld instead of %lld\n", offset, toSize);
err = EINVAL;
goto out;
}
}
if (err == 0) {
err = VNOP_FSYNC(vp, MNT_WAIT, decmpfs_ctx);
if (err != 0) {
ErrorLog("VNOP_FSYNC err %d\n", err);
goto out;
}
}
if (err != 0) {
ErrorLog("aborting decompress, err %d\n", err);
if (truncate_okay) {
int error = vnode_setsize(vp, 0, 0, decmpfs_ctx);
ErrorLog("vnode_setsize err %d\n", error);
}
goto out;
}
nodecmp:
unset_compressed_flag(vp);
err = decmpfs_free_compressed_data(vp, cp);
if (err != 0) {
ErrorLog("decmpfs_free_compressed_data err %d\n", err);
}
err = 0;
update_file_state = 1;
new_state = FILE_IS_NOT_COMPRESSED;
#if COMPRESSION_DEBUG
{
uint64_t filesize = 0;
vnsize(vp, &filesize);
DebugLog("new file size %lld\n", filesize);
}
#endif
out:
if (hdr) FREE(hdr, M_TEMP);
if (data) FREE(data, M_TEMP);
if (uio_w) uio_free(uio_w);
if (err != 0) {
update_file_state = 1;
new_state = FILE_TYPE_UNKNOWN;
if (uncompressed_size) {
ubc_setsize(vp, 0);
ubc_setsize(vp, uncompressed_size);
}
}
if (update_file_state) {
lck_mtx_lock(decompress_channel_mtx);
decmpfs_cnode_set_vnode_state(cp, new_state, 1);
wakeup((caddr_t)&decompress_channel);
lck_mtx_unlock(decompress_channel_mtx);
}
if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 1);
return err;
}
#pragma mark --- Type1 compressor ---
static int
decmpfs_validate_compressed_file_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr)
{
int err = 0;
if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) {
err = EINVAL;
goto out;
}
out:
return err;
}
static int
decmpfs_fetch_uncompressed_data_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read)
{
int err = 0;
int i;
user_ssize_t remaining;
if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) {
err = EINVAL;
goto out;
}
#if COMPRESSION_DEBUG
static int dummy = 0; char path[PATH_MAX];
DebugLog("%s: %d memcpy %lld at %lld\n", vnpath(vp, path, sizeof(path)), dummy++, size, (uint64_t)offset);
#endif
remaining = size;
for (i = 0; (i < nvec) && (remaining > 0); i++) {
user_ssize_t curCopy = vec[i].size;
if (curCopy > remaining)
curCopy = remaining;
memcpy(vec[i].buf, hdr->attr_bytes + offset, curCopy);
offset += curCopy;
remaining -= curCopy;
}
if ((bytes_read) && (err == 0))
*bytes_read = (size - remaining);
out:
return err;
}
static decmpfs_registration Type1Reg =
{
.decmpfs_registration = DECMPFS_REGISTRATION_VERSION,
.validate = decmpfs_validate_compressed_file_Type1,
.adjust_fetch = NULL,
.fetch = decmpfs_fetch_uncompressed_data_Type1,
.free_data = NULL,
.get_flags = NULL
};
#pragma mark --- decmpfs initialization ---
void decmpfs_init()
{
static int done = 0;
if (done) return;
decmpfs_ctx = vfs_context_create(vfs_context_kernel());
lck_grp_attr_t *attr = lck_grp_attr_alloc_init();
decmpfs_lockgrp = lck_grp_alloc_init("VFSCOMP", attr);
decompressorsLock = lck_rw_alloc_init(decmpfs_lockgrp, NULL);
decompress_channel_mtx = lck_mtx_alloc_init(decmpfs_lockgrp, NULL);
register_decmpfs_decompressor(CMP_Type1, &Type1Reg);
done = 1;
}
#endif