#if CONFIG_PROTECT
#include "hfs_key_roll.h"
#include <sys/cprotect.h>
#include <sys/ubc.h>
#include <sys/fcntl.h>
#include <sys/xattr.h>
#include <kern/sched_prim.h>
#include <vm/vm_pageout.h>
#include <pexpert/pexpert.h>
#include "hfs.h"
#include "hfs_extents.h"
#include "hfs_kdebug.h"
#define PTR_ADD(type, base, offset) (type)((uintptr_t)(base) + (offset))
#define HFS_KEY_ROLL_MAX_CHUNK_BYTES 2 * 1024 * 1024
static inline void log_error(const char *func, unsigned line, errno_t err)
{
printf("%s:%u error: %d\n", func, line, err);
}
#define LOG_ERROR(err) log_error(__func__, __LINE__, err)
#define CHECK(x, var, goto_label) \
do { \
var = (x); \
if (var) { \
if (var != ENOSPC) \
LOG_ERROR(var); \
goto goto_label; \
} \
} while (0)
#define min(a, b) \
({ typeof (a) _a = (a); typeof (b) _b = (b); _a < _b ? _a : _b; })
typedef struct {
void *object;
int flags;
} hfs_lock_ctx_t;
#define DECL_HFS_LOCK_CTX(var, unlocker) \
hfs_lock_ctx_t var __attribute__((cleanup(unlocker))) = {}
static inline bool hfs_is_locked(hfs_lock_ctx_t *lock_ctx)
{
return lock_ctx->object != NULL;
}
static inline int hfs_lock_flags(hfs_lock_ctx_t *lock_ctx)
{
return lock_ctx->flags;
}
static inline void hfs_lock_cp(cnode_t *cp,
enum hfs_locktype lock_type,
hfs_lock_ctx_t *lock_ctx)
{
hfs_lock_always(cp, lock_type);
lock_ctx->object = cp;
}
static inline void hfs_unlock_cp(hfs_lock_ctx_t *lock_ctx)
{
if (lock_ctx->object) {
hfs_unlock(lock_ctx->object);
lock_ctx->object = NULL;
}
}
static inline void hfs_lock_trunc(cnode_t *cp,
enum hfs_locktype lock_type,
hfs_lock_ctx_t *lock_ctx)
{
hfs_lock_truncate(cp, lock_type, 0);
lock_ctx->object = cp;
}
static inline void hfs_unlock_trunc(hfs_lock_ctx_t *lock_ctx)
{
if (lock_ctx->object) {
hfs_unlock_truncate(lock_ctx->object, 0);
lock_ctx->object = NULL;
}
}
static inline void hfs_lock_sys(struct hfsmount *hfsmp, int flags,
enum hfs_locktype lock_type,
hfs_lock_ctx_t *lock_ctx)
{
lock_ctx->flags |= hfs_systemfile_lock(hfsmp, flags, lock_type);
lock_ctx->object = hfsmp;
}
static inline void hfs_unlock_sys(hfs_lock_ctx_t *lock_ctx)
{
if (lock_ctx->object) {
hfs_systemfile_unlock(lock_ctx->object, lock_ctx->flags);
lock_ctx->object = NULL;
lock_ctx->flags = 0;
}
}
#if DEBUG
static const uint32_t ckr_magic1 = 0x7b726b63;
static const uint32_t ckr_magic2 = 0x726b637d;
#endif
hfs_cp_key_roll_ctx_t *hfs_key_roll_ctx_alloc(const hfs_cp_key_roll_ctx_t *old,
uint16_t pers_key_len,
uint16_t cached_key_len,
cp_key_pair_t **pcpkp)
{
hfs_cp_key_roll_ctx_t *ckr;
size_t size = (sizeof(*ckr) - sizeof(cp_key_pair_t)
+ cpkp_size(pers_key_len, cached_key_len));
#if DEBUG
size += 4;
#endif
ckr = hfs_mallocz(size);
#if DEBUG
ckr->ckr_magic1 = ckr_magic1;
*PTR_ADD(uint32_t *, ckr, size - 4) = ckr_magic2;
#endif
cpkp_init(&ckr->ckr_keys, pers_key_len, cached_key_len);
if (old) {
if (old->ckr_busy)
panic("hfs_key_roll_ctx_alloc: old context busy!");
ckr->ckr_off_rsrc = old->ckr_off_rsrc;
ckr->ckr_preferred_next_block = old->ckr_preferred_next_block;
cpx_copy(cpkp_cpx(&old->ckr_keys), cpkp_cpx(&ckr->ckr_keys));
}
*pcpkp = &ckr->ckr_keys;
return ckr;
}
static void hfs_key_roll_ctx_free(hfs_cp_key_roll_ctx_t *ckr)
{
size_t size = (sizeof(*ckr) - sizeof(cp_key_pair_t)
+ cpkp_sizex(&ckr->ckr_keys));
#if DEBUG
size += 4;
#endif
hfs_free(ckr, size);
}
void hfs_release_key_roll_ctx(hfsmount_t *hfsmp, cprotect_t cpr)
{
hfs_cp_key_roll_ctx_t *ckr = cpr->cp_key_roll_ctx;
if (!ckr)
return;
cpkp_flush(&ckr->ckr_keys);
if (ckr->ckr_tentative_reservation) {
int lockf = hfs_systemfile_lock(hfsmp, SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
hfs_free_tentative(hfsmp, &ckr->ckr_tentative_reservation);
hfs_systemfile_unlock(hfsmp, lockf);
}
cpr->cp_key_roll_ctx = NULL;
cnode_t *cp = cpr->cp_backing_cnode;
if (cp)
wakeup(&cp->c_cpentry);
#if DEBUG
hfs_assert(ckr->ckr_magic1 == ckr_magic1);
hfs_assert(*PTR_ADD(uint32_t *, ckr, sizeof(*ckr) - sizeof(cp_key_pair_t)
+ cpkp_sizex(&ckr->ckr_keys) == ckr_magic2));
#endif
hfs_key_roll_ctx_free(ckr);
}
static void hfs_key_roll_status(cnode_t *cp, hfs_key_roll_args_t *args)
{
hfsmount_t *hfsmp = VTOHFS(cp->c_vp);
cprotect_t const cpr = cp->c_cpentry;
filefork_t *dfork = cp->c_datafork;
if (!cpr || !dfork) {
args->key_class_generation = 0;
args->key_revision = 0;
args->key_os_version = 0;
args->done = -1;
args->total = 0;
return;
}
uint32_t total_blocks = cp->c_blocks - dfork->ff_unallocblocks;
if (cp->c_rsrcfork)
total_blocks -= cp->c_rsrcfork->ff_unallocblocks;
args->total = hfs_blk_to_bytes(total_blocks, hfsmp->blockSize);
args->key_class_generation = cp_get_crypto_generation(cpr->cp_pclass);
args->key_revision = cpr->cp_key_revision;
args->key_os_version = cpr->cp_key_os_version;
hfs_cp_key_roll_ctx_t *ckr = cpr->cp_key_roll_ctx;
if (!ckr)
args->done = -1;
else {
args->done = off_rsrc_get_off(ckr->ckr_off_rsrc);
if (off_rsrc_is_rsrc(ckr->ckr_off_rsrc)) {
args->done += hfs_blk_to_bytes(ff_allocblocks(dfork),
hfsmp->blockSize);
}
}
}
errno_t hfs_key_roll_op(vfs_context_t vfs_ctx, vnode_t vp,
hfs_key_roll_args_t *args)
{
errno_t ret;
cnode_t * const cp = VTOC(vp);
cprotect_t const cpr = cp->c_cpentry;
hfs_cp_key_roll_ctx_t *ckr = NULL;
DECL_HFS_LOCK_CTX(cp_lock, hfs_unlock_cp);
DECL_HFS_LOCK_CTX(trunc_lock, hfs_unlock_trunc);
KDBG(HFSDBG_KEY_ROLL | DBG_FUNC_START, kdebug_vnode(vp), args->operation);
if (args->api_version != HFS_KR_API_VERSION_1) {
ret = ENOTSUP;
goto exit;
}
if (args->operation != HFS_KR_OP_STATUS) {
ret = cp_handle_vnop(vp, CP_WRITE_ACCESS, 0);
if (ret)
goto exit;
}
switch (args->operation) {
case HFS_KR_OP_START:
if (!cpr) {
ret = ENOTSUP;
goto exit;
}
hfs_lock_trunc(cp, HFS_EXCLUSIVE_LOCK, &trunc_lock);
hfs_lock_cp(cp, HFS_EXCLUSIVE_LOCK, &cp_lock);
ckr = cpr->cp_key_roll_ctx;
if (ckr)
break;
if (ISSET(args->flags, HFS_KR_MATCH_KEY_CLASS_GENERATION)
&& (args->key_class_generation
!= cp_get_crypto_generation(cpr->cp_pclass))) {
break;
}
if (ISSET(args->flags, HFS_KR_MATCH_KEY_REVISION)
&& args->key_revision != cpr->cp_key_revision) {
break;
}
if (ISSET(args->flags, HFS_KR_MATCH_KEY_OS_VERSION)
&& args->key_os_version != cpr->cp_key_os_version) {
break;
}
if (cp->c_cpentry->cp_raw_open_count > 0) {
ret = EBUSY;
goto exit;
}
ret = hfs_key_roll_start(cp);
if (ret)
goto exit;
break;
case HFS_KR_OP_STATUS:
break;
case HFS_KR_OP_STEP:
CHECK(hfs_key_roll_step(vfs_ctx, vp, INT64_MAX), ret, exit);
break;
case HFS_KR_OP_SET_INFO:
if (!ISSET(PE_i_can_has_kernel_configuration(), kPEICanHasDiagnosticAPI)) {
ret = EPERM;
goto exit;
}
if (!cpr) {
ret = ENOTSUP;
goto exit;
}
hfs_lock_cp(cp, HFS_EXCLUSIVE_LOCK, &cp_lock);
cpr->cp_key_revision = args->key_revision;
cpr->cp_key_os_version = args->key_os_version;
ret = cp_setxattr(cp, cpr, VTOHFS(vp), 0, 0);
if (ret)
goto exit;
break;
default:
ret = EINVAL;
goto exit;
}
if (!hfs_is_locked(&cp_lock))
hfs_lock_cp(cp, HFS_SHARED_LOCK, &cp_lock);
hfs_key_roll_status(cp, args);
ret = 0;
exit:
hfs_unlock_cp(&cp_lock);
hfs_unlock_trunc(&trunc_lock);
KDBG(HFSDBG_KEY_ROLL | DBG_FUNC_END, ret, ret ? 0 : args->done);
return ret;
}
errno_t hfs_key_roll_start(cnode_t *cp)
{
errno_t ret;
hfs_assert(!cp->c_cpentry->cp_key_roll_ctx);
if (CP_CLASS(cp->c_cpentry->cp_pclass) == PROTECTION_CLASS_F)
return ENOTSUP;
hfsmount_t *hfsmp = VTOHFS(cp->c_vp);
if (ISSET(hfsmp->hfs_flags, HFS_READ_ONLY))
return EROFS;
if (!hfsmp->jnl || !S_ISREG(cp->c_mode))
return ENOTSUP;
cprotect_t cpr = cp->c_cpentry;
cp_key_class_t key_class = cpr->cp_pclass;
hfs_unlock(cp);
hfs_cp_key_roll_ctx_t *ckr;
int attempts = 0;
cp_key_revision_t rev = cp_next_key_revision(cp->c_cpentry->cp_key_revision);
for (;;) {
ret = cp_new(&key_class, hfsmp, cp, cp->c_mode, CP_KEYWRAP_DIFFCLASS, rev,
(cp_new_alloc_fn)hfs_key_roll_ctx_alloc, (void **)&ckr);
if (ret) {
hfs_lock_always(cp, HFS_EXCLUSIVE_LOCK);
goto exit;
}
if (key_class == cpr->cp_pclass) {
hfs_lock_always(cp, HFS_EXCLUSIVE_LOCK);
break;
}
vnode_waitforwrites(cp->c_vp, 0, 0, 0, "hfs_key_roll_start");
if (cp->c_rsrc_vp)
vnode_waitforwrites(cp->c_rsrc_vp, 0, 0, 0, "hfs_key_roll_start");
hfs_lock_always(cp, HFS_EXCLUSIVE_LOCK);
cp_key_class_t key_class2 = key_class;
cprotect_t new_entry;
ret = cp_rewrap(cp, hfsmp, &key_class2, &cpr->cp_keys, cpr,
(cp_new_alloc_fn)cp_entry_alloc, (void **)&new_entry);
if (ret) {
hfs_key_roll_ctx_free(ckr);
goto exit;
}
if (key_class2 == key_class) {
cp_replace_entry(hfsmp, cp, new_entry);
cpr = new_entry;
cpr->cp_pclass = key_class;
break;
}
hfs_key_roll_ctx_free(ckr);
cp_entry_destroy(hfsmp, new_entry);
if (++attempts > 3) {
ret = EPERM;
goto exit;
}
}
cpr->cp_key_roll_ctx = ckr;
cpr->cp_key_revision = rev;
cpr->cp_key_os_version = cp_os_version();
return 0;
exit:
wakeup(&cp->c_cpentry);
return ret;
}
errno_t hfs_key_roll_up_to(vfs_context_t vfs_ctx, vnode_t vp, off_rsrc_t up_to)
{
cnode_t * const cp = VTOC(vp);
for (;;) {
hfs_lock_always(cp, HFS_SHARED_LOCK);
cprotect_t cpr = cp->c_cpentry;
if (!cpr)
break;
hfs_cp_key_roll_ctx_t *ckr = cpr->cp_key_roll_ctx;
if (!ckr || ckr->ckr_off_rsrc >= up_to)
break;
hfs_unlock(cp);
errno_t ret = hfs_key_roll_step(vfs_ctx, vp, up_to);
if (ret)
return ret;
}
hfs_unlock(cp);
return 0;
}
errno_t hfs_key_roll_check(cnode_t *cp, bool have_excl_trunc_lock)
{
hfs_assert(cp->c_lockowner == current_thread());
errno_t ret;
cprotect_t cpr;
hfs_cp_key_roll_ctx_t *ckr = NULL;
hfsmount_t *hfsmp = VTOHFS(cp->c_vp);
bool downgrade_trunc_lock = false;
off_rsrc_t orig_off_rsrc;
again:
cpr = cp->c_cpentry;
if (!cpr) {
ret = 0;
goto exit;
}
ckr = cpr->cp_key_roll_ctx;
if (!ckr) {
ret = 0;
goto exit;
}
hfs_assert(have_excl_trunc_lock || ckr->ckr_busy);
if (!cp->c_datafork) {
ret = 0;
goto exit;
}
orig_off_rsrc = ckr->ckr_off_rsrc;
off_rsrc_t new_off_rsrc = orig_off_rsrc;
if (orig_off_rsrc == INT64_MAX) {
} else if (off_rsrc_is_rsrc(orig_off_rsrc)) {
off_t size;
if (!cp->c_rsrcfork) {
size = hfs_blk_to_bytes(cp->c_blocks - cp->c_datafork->ff_blocks,
hfsmp->blockSize);
} else {
size = min(cp->c_rsrcfork->ff_size,
hfs_blk_to_bytes(ff_allocblocks(cp->c_rsrcfork),
hfsmp->blockSize));
}
if (off_rsrc_get_off(orig_off_rsrc) >= size)
new_off_rsrc = INT64_MAX;
} else {
off_t size = min(cp->c_datafork->ff_size,
hfs_blk_to_bytes(ff_allocblocks(cp->c_datafork),
hfsmp->blockSize));
if (off_rsrc_get_off(orig_off_rsrc) >= size) {
new_off_rsrc = hfs_has_rsrc(cp) ? off_rsrc_make(0, true) : INT64_MAX;
}
}
if (new_off_rsrc == INT64_MAX) {
if (!have_excl_trunc_lock) {
ckr->ckr_busy = false;
hfs_unlock(cp);
if (!hfs_truncate_lock_upgrade(cp))
hfs_lock_truncate(cp, HFS_EXCLUSIVE_LOCK, 0);
have_excl_trunc_lock = true;
downgrade_trunc_lock = true;
hfs_lock_always(cp, HFS_EXCLUSIVE_LOCK);
goto again;
}
vnode_waitforwrites(cp->c_vp, 0, 0, 0, "hfs_key_roll_check_size");
if (cp->c_rsrc_vp) {
vnode_waitforwrites(cp->c_rsrc_vp, 0, 0,
0, "hfs_key_roll_check_size");
}
if (!cpkp_can_copy(&ckr->ckr_keys, &cpr->cp_keys)) {
cprotect_t new_cpr = cp_entry_alloc(cpr,
ckr->ckr_keys.cpkp_max_pers_key_len,
CP_MAX_CACHEBUFLEN, NULL);
cpkp_copy(&ckr->ckr_keys, &new_cpr->cp_keys);
cp_replace_entry(hfsmp, cp, new_cpr);
cpr = new_cpr;
} else {
cpkp_copy(&ckr->ckr_keys, &cpr->cp_keys);
hfs_release_key_roll_ctx(hfsmp, cpr);
}
ckr = NULL;
}
if (new_off_rsrc != orig_off_rsrc) {
if (ckr)
ckr->ckr_off_rsrc = new_off_rsrc;
ret = cp_setxattr(cp, cpr, hfsmp, 0, XATTR_REPLACE);
if (ret) {
if (ckr)
ckr->ckr_off_rsrc = orig_off_rsrc;
goto exit;
}
}
ret = 0;
exit:
if (downgrade_trunc_lock) {
if (ckr)
ckr->ckr_busy = true;
hfs_truncate_lock_downgrade(cp);
}
return ret;
}
static errno_t kr_upl_create(vnode_t vp, off_t offset, int len,
upl_offset_t *start_upl_offset,
upl_t *upl, upl_page_info_t **pl)
{
const int rounding = offset & PAGE_MASK;
*start_upl_offset = rounding;
offset -= rounding;
len += rounding;
off_t max = VTOF(vp)->ff_size - offset;
if (len > max)
len = max;
len = round_page_32(len);
return mach_to_bsd_errno(ubc_create_upl(vp, offset, len, upl, pl,
UPL_CLEAN_IN_PLACE | UPL_SET_LITE));
}
static errno_t kr_page_out(vnode_t vp, upl_t upl, upl_offset_t upl_offset,
off_t fork_offset, int len)
{
const int rounding = upl_offset & PAGE_MASK;
upl_offset -= rounding;
fork_offset -= rounding;
len += rounding;
const off_t fork_size = VTOF(vp)->ff_size;
if (fork_offset + len > fork_size)
len = fork_size - fork_offset;
len = round_page_32(len);
return cluster_pageout(vp, upl, upl_offset, fork_offset, len,
fork_size, UPL_IOSYNC | UPL_NOCOMMIT);
}
static void kr_upl_commit(upl_t upl, upl_offset_t upl_offset, int len, bool last)
{
if (!upl)
return;
const int rounding = upl_offset & PAGE_MASK;
upl_offset -= rounding;
len += rounding;
if (last)
len = upl_get_size(upl) - upl_offset;
else
len = trunc_page_32(len);
__unused errno_t err
= ubc_upl_commit_range(upl, upl_offset, len,
UPL_COMMIT_FREE_ON_EMPTY
| UPL_COMMIT_SPECULATE);
hfs_assert(!err);
}
errno_t hfs_key_roll_step(__unused vfs_context_t vfs_ctx, vnode_t vp, off_rsrc_t limit)
{
int ret = EIO;
cnode_t * const cp = VTOC(vp);
cprotect_t cpr = NULL;
hfsmount_t * const hfsmp = VTOHFS(vp);
upl_t upl = NULL;
bool transaction_open = false;
bool need_put = false;
bool marked_busy = false;
hfs_cp_key_roll_ctx_t *ckr = NULL;
filefork_t *pfork = NULL;
off_rsrc_t roll_back_off_rsrc = 0;
int ext_count = 0;
const int max_extents = 8; enum {
ROLL_BACK_EXTENTS_MASK = 0x00ff,
ROLL_BACK_XATTR = 0x0100,
ROLL_BACK_OFFSET = 0x0200,
};
int roll_back = 0;
#if 0
printf ("{ hfs_key_roll_step\n");
#endif
if (ISSET(hfsmp->hfs_flags, HFS_READ_ONLY))
return EROFS;
HFSPlusExtentDescriptor extents[max_extents];
struct rl_entry *reservations[max_extents];
DECL_HFS_LOCK_CTX(trunc_lock, hfs_unlock_trunc);
DECL_HFS_LOCK_CTX(cp_lock, hfs_unlock_cp);
DECL_HFS_LOCK_CTX(sys_lock, hfs_unlock_sys);
for (;;) {
hfs_lock_trunc(cp, HFS_SHARED_LOCK, &trunc_lock);
hfs_lock_cp(cp, HFS_EXCLUSIVE_LOCK, &cp_lock);
cpr = cp->c_cpentry;
if (!cpr) {
ret = 0;
goto exit;
}
ckr = cpr->cp_key_roll_ctx;
if (!ckr) {
ret = 0;
goto exit;
}
if (!ckr->ckr_busy)
break;
assert_wait(&cp->c_cpentry, THREAD_ABORTSAFE);
hfs_unlock_cp(&cp_lock);
hfs_unlock_trunc(&trunc_lock);
if (msleep(NULL, NULL, PINOD | PCATCH,
"hfs_key_roll", NULL) == EINTR) {
ret = EINTR;
goto exit;
}
}
ckr->ckr_busy = true;
marked_busy = true;
CHECK(hfs_key_roll_check(cp, false), ret, exit);
cpr = cp->c_cpentry;
ckr = cpr->cp_key_roll_ctx;
if (!ckr) {
ret = 0;
goto exit;
}
if (ckr->ckr_off_rsrc >= limit) {
ret = 0;
goto exit;
}
if (!hfs_freeblks(hfsmp, true)) {
ret = ENOSPC;
goto exit;
}
if (off_rsrc_is_rsrc(ckr->ckr_off_rsrc)) {
if (!VNODE_IS_RSRC(vp)) {
vnode_t rvp;
CHECK(hfs_vgetrsrc(hfsmp, vp, &rvp), ret, exit);
need_put = true;
vp = rvp;
}
pfork = cp->c_rsrcfork;
} else {
if (VNODE_IS_RSRC(vp)) {
CHECK(vnode_get(cp->c_vp), ret, exit);
vp = cp->c_vp;
need_put = true;
}
pfork = cp->c_datafork;
}
hfs_unlock_cp(&cp_lock);
const uint32_t fork_blocks = min(howmany(pfork->ff_size,
hfsmp->blockSize),
ff_allocblocks(pfork));
off_t off = off_rsrc_get_off(ckr->ckr_off_rsrc);
hfs_assert(!(off % hfsmp->blockSize));
uint32_t block = off / hfsmp->blockSize;
uint32_t rem_fork_blocks;
if (fork_blocks < block)
rem_fork_blocks = 0;
else
rem_fork_blocks = fork_blocks - block;
uint32_t chunk_blocks = min(rem_fork_blocks,
HFS_KEY_ROLL_MAX_CHUNK_BYTES / hfsmp->blockSize);
off_t chunk_bytes = chunk_blocks * hfsmp->blockSize;
upl_offset_t upl_offset = 0;
if (chunk_bytes) {
if (!ckr->ckr_preferred_next_block && off) {
hfs_ext_iter_t *iter;
iter = hfs_malloc(sizeof(*iter));
hfs_lock_sys(hfsmp, SFL_EXTENTS, HFS_EXCLUSIVE_LOCK, &sys_lock);
if (!hfs_ext_find(vp, off - 1, iter)) {
ckr->ckr_preferred_next_block = (iter->group[iter->ndx].startBlock
+ off / hfsmp->blockSize
- iter->file_block);
}
hfs_unlock_sys(&sys_lock);
hfs_free(iter, sizeof(*iter));
}
cl_direct_read_lock_t *lck = cluster_lock_direct_read(vp, LCK_RW_TYPE_EXCLUSIVE);
upl_page_info_t *pl;
ret = kr_upl_create(vp, off, chunk_bytes, &upl_offset, &upl, &pl);
cluster_unlock_direct_read(lck);
if (ret) {
LOG_ERROR(ret);
goto exit;
}
int page_count = upl_get_size(upl) >> PAGE_SHIFT;
int page_ndx = 0;
for (;;) {
while (page_ndx < page_count && upl_valid_page(pl, page_ndx))
++page_ndx;
if (page_ndx >= page_count)
break;
const int page_group_start = page_ndx;
do {
++page_ndx;
} while (page_ndx < page_count && !upl_valid_page(pl, page_ndx));
const upl_offset_t start = page_group_start << PAGE_SHIFT;
CHECK(cluster_pagein(vp, upl, start,
off - upl_offset + start,
(page_ndx - page_group_start) << PAGE_SHIFT,
pfork->ff_size,
UPL_IOSYNC | UPL_NOCOMMIT), ret, exit);
}
}
bool tried_hard = false;
for (;;) {
off_t bytes_rolled = 0;
roll_back_off_rsrc = ckr->ckr_off_rsrc;
ext_count = 0;
while (chunk_bytes && ext_count < max_extents) {
CHECK(hfs_start_transaction(hfsmp), ret, exit);
transaction_open = true;
hfs_lock_sys(hfsmp, SFL_BITMAP, HFS_EXCLUSIVE_LOCK, &sys_lock);
HFSPlusExtentDescriptor *ext = &extents[ext_count];
if (!tried_hard
&& (!ckr->ckr_tentative_reservation
|| !rl_len(ckr->ckr_tentative_reservation))) {
hfs_free_tentative(hfsmp, &ckr->ckr_tentative_reservation);
tried_hard = true;
HFSPlusExtentDescriptor extent = {
.startBlock = ckr->ckr_preferred_next_block,
.blockCount = 1, };
hfs_alloc_extra_args_t args = {
.max_blocks = rem_fork_blocks,
.reservation_out = &ckr->ckr_tentative_reservation,
.alignment = PAGE_SIZE / hfsmp->blockSize,
.alignment_offset = (off + bytes_rolled) / hfsmp->blockSize,
};
ret = hfs_block_alloc(hfsmp, &extent,
HFS_ALLOC_TENTATIVE | HFS_ALLOC_TRY_HARD,
&args);
if (ret == ENOSPC && ext_count) {
ext->blockCount = 0;
goto roll_what_we_have;
} else if (ret) {
if (ret != ENOSPC)
LOG_ERROR(ret);
goto exit;
}
}
ext->startBlock = ckr->ckr_preferred_next_block;
ext->blockCount = 1;
hfs_alloc_extra_args_t args = {
.max_blocks = chunk_blocks,
.reservation_in = &ckr->ckr_tentative_reservation,
.reservation_out = &reservations[ext_count],
.alignment = PAGE_SIZE / hfsmp->blockSize,
.alignment_offset = (off + bytes_rolled) / hfsmp->blockSize,
};
ret = hfs_block_alloc(hfsmp, ext,
(HFS_ALLOC_USE_TENTATIVE
| HFS_ALLOC_LOCKED), &args);
if (ret == ENOSPC && ext_count) {
ext->blockCount = 0;
} else if (ret) {
if (ret != ENOSPC)
LOG_ERROR(ret);
goto exit;
}
roll_what_we_have:
hfs_unlock_sys(&sys_lock);
transaction_open = false;
CHECK(hfs_end_transaction(hfsmp), ret, exit);
if (!ext->blockCount)
break;
const off_t ext_bytes = hfs_blk_to_bytes(ext->blockCount,
hfsmp->blockSize);
hfs_lock_cp(cp, HFS_EXCLUSIVE_LOCK, &cp_lock);
ckr->ckr_off_rsrc += ext_bytes;
roll_back |= ROLL_BACK_OFFSET;
ckr->ckr_roll_extent = *ext;
hfs_unlock_cp(&cp_lock);
CHECK(kr_page_out(vp, upl, upl_offset + bytes_rolled,
off + bytes_rolled,
ext_bytes), ret, exit);
chunk_bytes -= ext_bytes;
chunk_blocks -= ext->blockCount;
rem_fork_blocks -= ext->blockCount;
ckr->ckr_preferred_next_block += ext->blockCount;
bytes_rolled += ext_bytes;
++ext_count;
}
if (bytes_rolled)
CHECK(hfs_flush(hfsmp, HFS_FLUSH_BARRIER), ret, exit);
hfs_lock_cp(cp, HFS_EXCLUSIVE_LOCK, &cp_lock);
CHECK(hfs_start_transaction(hfsmp), ret, exit);
transaction_open = true;
hfs_lock_sys(hfsmp, SFL_BITMAP, HFS_EXCLUSIVE_LOCK, &sys_lock);
hfs_alloc_extra_args_t args = {};
for (int i = 0; i < ext_count; ++i) {
args.reservation_in = &reservations[i];
CHECK(hfs_block_alloc(hfsmp, &extents[i],
HFS_ALLOC_COMMIT, &args), ret, exit);
roll_back |= 1 << i;
}
hfs_unlock_sys(&sys_lock);
HFSPlusExtentRecord cat_extents;
if (!chunk_bytes) {
off_t fork_end = min(pfork->ff_size,
hfs_blk_to_bytes(ff_allocblocks(pfork), hfsmp->blockSize));
if (off + bytes_rolled >= fork_end) {
if (!off_rsrc_is_rsrc(ckr->ckr_off_rsrc)
&& hfs_has_rsrc(cp)) {
ckr->ckr_off_rsrc = off_rsrc_make(0, true);
} else {
ckr->ckr_off_rsrc = INT64_MAX;
}
}
}
roll_back |= ROLL_BACK_XATTR;
CHECK(cp_setxattr(cp, cpr, hfsmp, 0, XATTR_REPLACE), ret, exit);
hfs_lock_sys(hfsmp, SFL_EXTENTS | SFL_CATALOG, HFS_EXCLUSIVE_LOCK, &sys_lock);
CHECK(hfs_ext_replace(hfsmp, vp, off / hfsmp->blockSize,
extents, ext_count, cat_extents), ret, exit);
hfs_unlock_sys(&sys_lock);
transaction_open = false;
roll_back = 0;
CHECK(hfs_end_transaction(hfsmp), ret, exit);
if (cat_extents[0].blockCount)
memcpy(pfork->ff_data.cf_extents, cat_extents, sizeof(cat_extents));
ckr->ckr_roll_extent = (HFSPlusExtentDescriptor){ 0, 0 };
hfs_unlock_cp(&cp_lock);
kr_upl_commit(upl, upl_offset, bytes_rolled, !chunk_bytes);
if (!chunk_bytes) {
break;
}
upl_offset += bytes_rolled;
off += bytes_rolled;
}
upl = NULL;
hfs_lock_cp(cp, HFS_EXCLUSIVE_LOCK, &cp_lock);
hfs_key_roll_check(cp, false);
ret = 0;
exit:
cpr = cp->c_cpentry;
ckr = cpr->cp_key_roll_ctx;
if (roll_back & ROLL_BACK_EXTENTS_MASK) {
if (!ISSET(hfs_lock_flags(&sys_lock), SFL_BITMAP)) {
hfs_lock_sys(hfsmp, SFL_BITMAP, HFS_EXCLUSIVE_LOCK,
&sys_lock);
}
for (int i = 0; i < ext_count; ++i) {
if (!ISSET(roll_back, 1 << i))
continue;
if (BlockDeallocate(hfsmp, extents[i].startBlock,
extents[i].blockCount, 0)) {
hfs_mark_inconsistent(hfsmp, HFS_ROLLBACK_FAILED);
}
}
}
hfs_unlock_sys(&sys_lock);
if (roll_back & ROLL_BACK_XATTR) {
hfs_assert(hfs_is_locked(&cp_lock));
if (cp_setxattr(cp, cpr, hfsmp, 0, XATTR_REPLACE))
hfs_mark_inconsistent(hfsmp, HFS_ROLLBACK_FAILED);
}
if (transaction_open)
hfs_end_transaction(hfsmp);
if (roll_back & ROLL_BACK_OFFSET) {
if (!hfs_is_locked(&cp_lock))
hfs_lock_cp(cp, HFS_EXCLUSIVE_LOCK, &cp_lock);
ckr->ckr_off_rsrc = roll_back_off_rsrc;
ckr->ckr_roll_extent = (HFSPlusExtentDescriptor){ 0, 0 };
}
if (marked_busy && ckr) {
if (!hfs_is_locked(&cp_lock))
hfs_lock_cp(cp, HFS_EXCLUSIVE_LOCK, &cp_lock);
ckr->ckr_busy = false;
wakeup(&cp->c_cpentry);
}
hfs_unlock_cp(&cp_lock);
hfs_unlock_trunc(&trunc_lock);
if (upl)
ubc_upl_abort(upl, UPL_ABORT_FREE_ON_EMPTY);
if (ext_count && reservations[ext_count - 1]) {
hfs_lock_sys(hfsmp, SFL_BITMAP, HFS_EXCLUSIVE_LOCK, &sys_lock);
for (int i = 0; i < ext_count; ++i)
hfs_free_locked(hfsmp, &reservations[i]);
hfs_unlock_sys(&sys_lock);
}
if (need_put)
vnode_put(vp);
if (ret == ESTALE) {
hfs_mark_inconsistent(hfsmp, HFS_INCONSISTENCY_DETECTED);
ret = HFS_EINCONSISTENT;
}
#if 0
printf ("hfs_key_roll_step }\n");
#endif
return ret;
}
bool hfs_is_key_rolling(cnode_t *cp)
{
return (cp->c_cpentry && cp->c_cpentry->cp_key_roll_ctx
&& cp->c_cpentry->cp_key_roll_ctx->ckr_off_rsrc != INT64_MAX);
}
#endif // CONFIG_PROTECT