voucher.c   [plain text]

/*
 * Copyright (c) 2013-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_APACHE_LICENSE_HEADER_START@
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @APPLE_APACHE_LICENSE_HEADER_END@
 */

#include "internal.h"

#if !defined(VOUCHER_EXPORT_PERSONA_SPI)
#if TARGET_OS_IPHONE
#define VOUCHER_EXPORT_PERSONA_SPI 1
#else
#define VOUCHER_EXPORT_PERSONA_SPI 0
#endif
#endif

#ifndef PERSONA_ID_NONE
#define PERSONA_ID_NONE ((uid_t)-1)
#endif

#if !DISPATCH_VARIANT_DYLD_STUB

#if VOUCHER_USE_MACH_VOUCHER
#if !HAVE_PTHREAD_WORKQUEUE_QOS
#error Unsupported configuration, workqueue QoS support is required
#endif
#include <mach/mach_voucher.h>
#include <sys/proc_info.h>

#define MACH_ACTIVITY_ID_RANGE_SIZE 16
#define MACH_ACTIVITY_ID_MASK ((1ULL << FIREHOSE_ACTIVITY_ID_FLAGS_SHIFT) - 1)
#define FIREHOSE_ACTIVITY_ID_MAKE(aid, flags) \
		FIREHOSE_ACTIVITY_ID_MERGE_FLAGS((aid) & MACH_ACTIVITY_ID_MASK, flags)

static volatile uint64_t _voucher_aid_next;

#pragma mark -
#pragma mark voucher_t

OS_OBJECT_CLASS_DECL(voucher, object);
#if !USE_OBJC
OS_OBJECT_VTABLE_INSTANCE(voucher,
		(void (*)(_os_object_t))_voucher_xref_dispose,
		(void (*)(_os_object_t))_voucher_dispose);
#endif // USE_OBJC
#define VOUCHER_CLASS OS_OBJECT_VTABLE(voucher)

static inline voucher_t
_voucher_alloc(mach_voucher_attr_recipe_size_t extra)
{
	voucher_t voucher;
	size_t voucher_size = sizeof(voucher_s) + extra;
	voucher = (voucher_t)_os_object_alloc_realized(VOUCHER_CLASS, voucher_size);
#if VOUCHER_ENABLE_RECIPE_OBJECTS
	voucher->v_recipe_extra_size = extra;
	voucher->v_recipe_extra_offset = voucher_size - extra;
#else
	dispatch_assert(!extra);
#endif
	_dispatch_voucher_debug("alloc", voucher);
	return voucher;
}

#if VOUCHER_ENABLE_RECIPE_OBJECTS
voucher_t
voucher_create(voucher_recipe_t recipe)
{
	// TODO: capture current activities or current kvoucher ?
	mach_voucher_attr_recipe_size_t extra = recipe ? recipe->vr_size : 0;
	voucher_t voucher = _voucher_alloc(extra);
	if (extra) {
		memcpy(_voucher_extra_recipes(voucher), recipe->vr_data, extra);
	}
	return voucher;
}
#endif

DISPATCH_ALWAYS_INLINE
static inline voucher_t
_voucher_clone(const voucher_t ov, voucher_fields_t ignore_fields)
{
	mach_voucher_attr_recipe_size_t extra = 0;
	voucher_t v;

	if (ov && !(ignore_fields & VOUCHER_FIELD_EXTRA)) {
		extra = _voucher_extra_size(ov);
	}
	v = _voucher_alloc(extra);
	if (ov) {
		voucher_fields_t fields = ~ignore_fields;
		if ((fields & VOUCHER_FIELD_KVOUCHER) && ov->v_kvoucher) {
			voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov;
			v->v_kvbase = _voucher_retain(kvb);
			v->v_kvoucher = kvb->v_kvoucher;
			v->v_kv_has_importance = kvb->v_kv_has_importance;
		}
		if (fields & VOUCHER_FIELD_PRIORITY) {
			v->v_priority = ov->v_priority;
		}
		if (fields & VOUCHER_FIELD_ACTIVITY) {
			v->v_activity = ov->v_activity;
			v->v_activity_creator = ov->v_activity_creator;
			v->v_parent_activity = ov->v_parent_activity;
		}
		if ((fields & VOUCHER_FIELD_EXTRA) && extra) {
			memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov),extra);
		}
	}
	return v;
}

voucher_t
voucher_adopt(voucher_t voucher)
{
	if (voucher == VOUCHER_CURRENT) {
		return _voucher_copy();
	}
	return _voucher_adopt(voucher);
}

voucher_t
voucher_copy(void)
{
	return _voucher_copy();
}

voucher_t
voucher_copy_without_importance(void)
{
	return _voucher_copy_without_importance();
}

voucher_t
voucher_retain(voucher_t voucher)
{
	return _voucher_retain(voucher);
}

void
voucher_release(voucher_t voucher)
{
	return _voucher_release(voucher);
}

void
_voucher_thread_cleanup(void *voucher)
{
	// when a thread exits and has a voucher left, the kernel
	// will get rid of the voucher kernel object that is set on the thread,
	// we only need to release the voucher_t object.
	_voucher_release(voucher);
}

DISPATCH_CACHELINE_ALIGN
static TAILQ_HEAD(, voucher_s) _vouchers[VL_HASH_SIZE];
#define _vouchers_head(kv) (&_vouchers[VL_HASH((kv))])
static dispatch_unfair_lock_s _vouchers_lock;
#define _vouchers_lock_lock() _dispatch_unfair_lock_lock(&_vouchers_lock)
#define _vouchers_lock_unlock() _dispatch_unfair_lock_unlock(&_vouchers_lock)

static voucher_t
_voucher_find_and_retain(mach_voucher_t kv)
{
	voucher_t v;
	if (!kv) return NULL;
	_vouchers_lock_lock();
	TAILQ_FOREACH(v, _vouchers_head(kv), v_list) {
		if (v->v_ipc_kvoucher == kv) {
			int xref_cnt = os_atomic_inc2o(v, os_obj_xref_cnt, relaxed);
			_dispatch_voucher_debug("retain  -> %d", v, xref_cnt + 1);
			if (slowpath(xref_cnt < 0)) {
				_dispatch_voucher_debug("over-release", v);
				_OS_OBJECT_CLIENT_CRASH("Voucher over-release");
			}
			if (xref_cnt == 0) {
				// resurrection: raced with _voucher_remove
				(void)os_atomic_inc2o(v, os_obj_ref_cnt, relaxed);
			}
			break;
		}
	}
	_vouchers_lock_unlock();
	return v;
}

static void
_voucher_insert(voucher_t v)
{
	mach_voucher_t kv = v->v_ipc_kvoucher;
	if (!kv) return;
	_vouchers_lock_lock();
	if (slowpath(_TAILQ_IS_ENQUEUED(v, v_list))) {
		_dispatch_voucher_debug("corruption", v);
		DISPATCH_CLIENT_CRASH(v->v_list.tqe_prev, "Voucher corruption");
	}
	TAILQ_INSERT_TAIL(_vouchers_head(kv), v, v_list);
	_vouchers_lock_unlock();
}

static void
_voucher_remove(voucher_t v)
{
	mach_voucher_t kv = v->v_ipc_kvoucher;
	if (!_TAILQ_IS_ENQUEUED(v, v_list)) return;
	_vouchers_lock_lock();
	if (slowpath(!kv)) {
		_dispatch_voucher_debug("corruption", v);
		DISPATCH_CLIENT_CRASH(0, "Voucher corruption");
	}
	// check for resurrection race with _voucher_find_and_retain
	if (os_atomic_load2o(v, os_obj_xref_cnt, ordered) < 0 &&
			_TAILQ_IS_ENQUEUED(v, v_list)) {
		TAILQ_REMOVE(_vouchers_head(kv), v, v_list);
		_TAILQ_MARK_NOT_ENQUEUED(v, v_list);
		v->v_list.tqe_next = (void*)~0ull;
	}
	_vouchers_lock_unlock();
}

void
_voucher_dealloc_mach_voucher(mach_voucher_t kv)
{
	_dispatch_kvoucher_debug("dealloc", kv);
	_dispatch_voucher_debug_machport(kv);
	kern_return_t kr = mach_voucher_deallocate(kv);
	DISPATCH_VERIFY_MIG(kr);
	(void)dispatch_assume_zero(kr);
}

static inline kern_return_t
_voucher_create_mach_voucher(const mach_voucher_attr_recipe_data_t *recipes,
		size_t recipes_size, mach_voucher_t *kvp)
{
	kern_return_t kr;
	mach_port_t mhp = _dispatch_get_mach_host_port();
	mach_voucher_t kv = MACH_VOUCHER_NULL;
	mach_voucher_attr_raw_recipe_array_t kvr;
	mach_voucher_attr_recipe_size_t kvr_size;
	kvr = (mach_voucher_attr_raw_recipe_array_t)recipes;
	kvr_size = (mach_voucher_attr_recipe_size_t)recipes_size;
	kr = host_create_mach_voucher(mhp, kvr, kvr_size, &kv);
	DISPATCH_VERIFY_MIG(kr);
	if (!kr) {
		_dispatch_kvoucher_debug("create", kv);
		_dispatch_voucher_debug_machport(kv);
	}
	*kvp = kv;
	return kr;
}

void
_voucher_task_mach_voucher_init(void* ctxt DISPATCH_UNUSED)
{
	kern_return_t kr;
	mach_voucher_t kv = MACH_VOUCHER_NULL;
#if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
	static const mach_voucher_attr_recipe_data_t task_create_recipe = {
		.key = MACH_VOUCHER_ATTR_KEY_BANK,
		.command = MACH_VOUCHER_ATTR_BANK_CREATE,
	};
	kr = _voucher_create_mach_voucher(&task_create_recipe,
			sizeof(task_create_recipe), &kv);
	if (slowpath(kr)) {
		DISPATCH_CLIENT_CRASH(kr, "Could not create task mach voucher");
	}
	_voucher_default_task_mach_voucher = kv;
#endif
	_voucher_task_mach_voucher = kv;
}

void
voucher_replace_default_voucher(void)
{
	(void)_voucher_get_task_mach_voucher(); // initalize task mach voucher
	mach_voucher_t kv, tkv = MACH_VOUCHER_NULL;
	voucher_t v = _voucher_get();
	if (v && v->v_kvoucher) {
		kern_return_t kr;
		kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
		const mach_voucher_attr_recipe_data_t task_copy_recipe = {
			.key = MACH_VOUCHER_ATTR_KEY_BANK,
			.command = MACH_VOUCHER_ATTR_COPY,
			.previous_voucher = kv,
		};
		kr = _voucher_create_mach_voucher(&task_copy_recipe,
				sizeof(task_copy_recipe), &tkv);
		if (dispatch_assume_zero(kr)) {
			tkv = MACH_VOUCHER_NULL;
		}
	}
	if (!tkv) tkv = _voucher_default_task_mach_voucher;
	kv = os_atomic_xchg(&_voucher_task_mach_voucher, tkv, relaxed);
	if (kv && kv != _voucher_default_task_mach_voucher) {
		_voucher_dealloc_mach_voucher(kv);
	}
	_dispatch_voucher_debug("kvoucher[0x%08x] replace default voucher", v, tkv);
}

#define _voucher_mach_recipe_size(payload_size) \
	(sizeof(mach_voucher_attr_recipe_data_t) + (payload_size))

#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
#define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
		_voucher_mach_recipe_size(0) + \
		_voucher_mach_recipe_size(sizeof(ipc_pthread_priority_value_t)) + \
		_voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
		_voucher_extra_size(v)))
#else
#define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
		_voucher_mach_recipe_size(0) + \
		_voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
		_voucher_extra_size(v)))
#endif

DISPATCH_ALWAYS_INLINE
static inline mach_voucher_attr_recipe_size_t
_voucher_mach_recipe_init(mach_voucher_attr_recipe_t mvar_buf, voucher_s *v,
		mach_voucher_t kvb, pthread_priority_t pp)
{
	mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(v);
	mach_voucher_attr_recipe_size_t size = 0;

	// normalize to just the QoS class and 0 relative priority
	pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
	if (pp) pp |= _PTHREAD_PRIORITY_PRIORITY_MASK;

	*mvar_buf++ = (mach_voucher_attr_recipe_data_t){
		.key = MACH_VOUCHER_ATTR_KEY_ALL,
		.command = MACH_VOUCHER_ATTR_COPY,
		.previous_voucher = kvb,
	};
	size += _voucher_mach_recipe_size(0);

#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
	if (pp) {
		ipc_pthread_priority_value_t value = (ipc_pthread_priority_value_t)pp;
		*mvar_buf++ = (mach_voucher_attr_recipe_data_t){
			.key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
			.command = MACH_VOUCHER_ATTR_PTHPRIORITY_CREATE,
			.content_size = sizeof(value),
		};
		mvar_buf = _dispatch_memappend(mvar_buf, &value);
		size += _voucher_mach_recipe_size(sizeof(value));
	}
#endif // VOUCHER_USE_MACH_VOUCHER_PRIORITY

	if ((v && v->v_activity) || pp) {
		_voucher_mach_udata_s *udata_buf;
		unsigned udata_size = 0;

		if (v && v->v_activity) {
			udata_size = offsetof(_voucher_mach_udata_s, _vmu_after_activity);
		} else {
			udata_size = offsetof(_voucher_mach_udata_s, _vmu_after_priority);
		}
		*mvar_buf = (mach_voucher_attr_recipe_data_t){
			.key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
			.command = MACH_VOUCHER_ATTR_USER_DATA_STORE,
			.content_size = udata_size,
		};
		udata_buf = (_voucher_mach_udata_s *)(mvar_buf->content);

		if (v && v->v_activity) {
			*udata_buf = (_voucher_mach_udata_s){
				.vmu_magic = VOUCHER_MAGIC_V3,
				.vmu_priority = (_voucher_priority_t)pp,
				.vmu_activity = v->v_activity,
				.vmu_activity_pid = v->v_activity_creator,
				.vmu_parent_activity = v->v_parent_activity,
			};
		} else {
			*udata_buf = (_voucher_mach_udata_s){
				.vmu_magic = VOUCHER_MAGIC_V3,
				.vmu_priority = (_voucher_priority_t)pp,
			};
		}

		mvar_buf = (mach_voucher_attr_recipe_t)(mvar_buf->content + udata_size);
		size += _voucher_mach_recipe_size(udata_size);
	}

	if (extra) {
		memcpy(mvar_buf, _voucher_extra_recipes(v), extra);
		size += extra;
	}
	return size;
}

mach_voucher_t
_voucher_get_mach_voucher(voucher_t voucher)
{
	if (!voucher) return MACH_VOUCHER_NULL;
	if (voucher->v_ipc_kvoucher) return voucher->v_ipc_kvoucher;
	mach_voucher_t kvb = voucher->v_kvoucher;
	if (!kvb) kvb = _voucher_get_task_mach_voucher();
	if (!voucher->v_activity && !voucher->v_priority &&
			!_voucher_extra_size(voucher)) {
		return kvb;
	}

	mach_voucher_attr_recipe_t mvar = _voucher_mach_recipe_alloca(voucher);
	mach_voucher_attr_recipe_size_t size;
	mach_voucher_t kv, kvo;
	kern_return_t kr;

	size = _voucher_mach_recipe_init(mvar, voucher, kvb, voucher->v_priority);
	kr = _voucher_create_mach_voucher(mvar, size, &kv);
	if (dispatch_assume_zero(kr) || !kv){
		return MACH_VOUCHER_NULL;
	}
	if (!os_atomic_cmpxchgv2o(voucher, v_ipc_kvoucher, MACH_VOUCHER_NULL,
			kv, &kvo, relaxed)) {
		_voucher_dealloc_mach_voucher(kv);
		kv = kvo;
	} else {
		if (kv == voucher->v_kvoucher) {
			// if v_kvoucher == v_ipc_kvoucher we keep only one reference
			_voucher_dealloc_mach_voucher(kv);
		}
		_voucher_insert(voucher);
		_dispatch_voucher_debug("kvoucher[0x%08x] create", voucher, kv);
	}
	return kv;
}

mach_voucher_t
_voucher_create_mach_voucher_with_priority(voucher_t voucher,
		pthread_priority_t priority)
{
	if (priority == _voucher_get_priority(voucher)) {
		return MACH_VOUCHER_NULL; // caller will use _voucher_get_mach_voucher
	}
	kern_return_t kr;
	mach_voucher_t kv, kvb = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
	if (!kvb) kvb = _voucher_get_task_mach_voucher();

	mach_voucher_attr_recipe_t mvar = _voucher_mach_recipe_alloca(voucher);
	mach_voucher_attr_recipe_size_t size;

	size = _voucher_mach_recipe_init(mvar, voucher, kvb, priority);
	kr = _voucher_create_mach_voucher(mvar, size, &kv);
	if (dispatch_assume_zero(kr) || !kv){
		return MACH_VOUCHER_NULL;
	}
	_dispatch_kvoucher_debug("create with priority from voucher[%p]", kv,
			voucher);
	return kv;
}

static voucher_t
_voucher_create_with_mach_voucher(mach_voucher_t kv, mach_msg_bits_t msgh_bits)
{
	if (!kv) return NULL;
	kern_return_t kr;
	mach_voucher_attr_recipe_t vr;
	size_t vr_size;
	mach_voucher_attr_recipe_size_t kvr_size = 0;
	mach_voucher_attr_content_size_t udata_sz = 0;
	_voucher_mach_udata_s *udata = NULL;
#if !VOUCHER_USE_BANK_AUTOREDEEM
	mach_voucher_t rkv;
	const mach_voucher_attr_recipe_data_t redeem_recipe[] = {
		[0] = {
			.key = MACH_VOUCHER_ATTR_KEY_ALL,
			.command = MACH_VOUCHER_ATTR_COPY,
			.previous_voucher = kv,
		},
		[1] = {
			.key = MACH_VOUCHER_ATTR_KEY_BANK,
			.command = MACH_VOUCHER_ATTR_REDEEM,
		},
	};
	kr = _voucher_create_mach_voucher(redeem_recipe, sizeof(redeem_recipe),
			&rkv);
	if (!dispatch_assume_zero(kr)) {
		_voucher_dealloc_mach_voucher(kv);
		_dispatch_kvoucher_debug("redeemed from 0x%08x", rkv, kv);
		kv = rkv;
	} else {
		_dispatch_voucher_debug_machport(kv);
	}
#endif
	voucher_t v = _voucher_find_and_retain(kv);
	if (v) {
		_dispatch_voucher_debug("kvoucher[0x%08x] found", v, kv);
		_voucher_dealloc_mach_voucher(kv);
		return v;
	}
	vr_size = sizeof(*vr) + sizeof(_voucher_mach_udata_s);
	vr = alloca(vr_size);
	if (kv) {
		kvr_size = (mach_voucher_attr_recipe_size_t)vr_size;
		kr = mach_voucher_extract_attr_recipe(kv,
				MACH_VOUCHER_ATTR_KEY_USER_DATA, (void*)vr, &kvr_size);
		DISPATCH_VERIFY_MIG(kr);
		if (!dispatch_assume_zero(kr) && kvr_size >= sizeof(*vr)) {
			udata_sz = vr->content_size;
			udata = (_voucher_mach_udata_s*)vr->content;
			dispatch_assume(udata_sz >= sizeof(_voucher_magic_t));
		}
	}
	vr = NULL;

	v = _voucher_alloc(0);
	v->v_ipc_kvoucher = v->v_kvoucher = kv;
	v->v_kv_has_importance = !!(msgh_bits & MACH_MSGH_BITS_RAISEIMP);

	if (udata_sz >= offsetof(_voucher_mach_udata_s,_vmu_after_priority)){
		if (udata->vmu_magic == VOUCHER_MAGIC_V3) {
			v->v_priority = udata->vmu_priority;
		}
	}
	bool remove_kv_userdata = false;
	if (udata_sz >= offsetof(_voucher_mach_udata_s, _vmu_after_activity)) {
#if !RDAR_25050791
		remove_kv_userdata = true;
#endif
		if (udata->vmu_magic == VOUCHER_MAGIC_V3 && udata->vmu_activity) {
			v->v_activity = udata->vmu_activity;
			v->v_activity_creator = udata->vmu_activity_pid;
			v->v_parent_activity = udata->vmu_parent_activity;
		}
	}

	if (remove_kv_userdata) {
		mach_voucher_t nkv = MACH_VOUCHER_NULL;
		const mach_voucher_attr_recipe_data_t remove_userdata_recipe[] = {
			[0] = {
				.key = MACH_VOUCHER_ATTR_KEY_ALL,
				.command = MACH_VOUCHER_ATTR_COPY,
				.previous_voucher = kv,
			},
			[1] = {
				.key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
				.command = MACH_VOUCHER_ATTR_REMOVE,
			},
#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
			[2] = {
				.key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
				.command = MACH_VOUCHER_ATTR_REMOVE,
			},
#endif
		};
		mach_voucher_attr_recipe_size_t size = sizeof(remove_userdata_recipe);

		kr = _voucher_create_mach_voucher(remove_userdata_recipe, size, &nkv);
		if (!dispatch_assume_zero(kr)) {
			_dispatch_voucher_debug("kvoucher[0x%08x] udata removal "
					"(created 0x%08x)", v, kv, nkv);
			v->v_ipc_kvoucher = MACH_VOUCHER_NULL;
			v->v_kvoucher = nkv;
			v->v_kvbase = _voucher_find_and_retain(nkv);
			if (v->v_kvbase) {
				_voucher_dealloc_mach_voucher(nkv); // borrow base reference
			}
			_voucher_dealloc_mach_voucher(kv);
			kv = nkv;
		} else {
			_dispatch_voucher_debug_machport(kv);
		}
	}

	_voucher_insert(v);
	_dispatch_voucher_debug("kvoucher[0x%08x] create", v, kv);
	return v;
}

voucher_t
_voucher_create_with_priority_and_mach_voucher(voucher_t ov,
		pthread_priority_t priority, mach_voucher_t kv)
{
	if (priority == _voucher_get_priority(ov)) {
		if (kv) _voucher_dealloc_mach_voucher(kv);
		return ov ? _voucher_retain(ov) : NULL;
	}
	voucher_t v = _voucher_find_and_retain(kv);
	voucher_fields_t ignore_fields = VOUCHER_FIELD_PRIORITY;

	if (v) {
		_dispatch_voucher_debug("kvoucher[0x%08x] find", v, kv);
		_voucher_dealloc_mach_voucher(kv);
		return v;
	}

	if (kv) ignore_fields |= VOUCHER_FIELD_KVOUCHER;
	v = _voucher_clone(ov, ignore_fields);
	if (priority) {
		v->v_priority = (_voucher_priority_t)priority;
	}
	if (kv) {
		v->v_ipc_kvoucher = v->v_kvoucher = kv;
		_voucher_insert(v);
		_dispatch_voucher_debug("kvoucher[0x%08x] create with priority from "
				"voucher[%p]", v, kv, ov);
		_dispatch_voucher_debug_machport(kv);
	}
	return v;
}

voucher_t
_voucher_create_without_importance(voucher_t ov)
{
	// Nothing to do unless the old voucher has a kernel voucher. If it
	// doesn't, it can't have any importance, now or in the future.
	if (!ov) return NULL;
	if (!ov->v_kvoucher || !ov->v_kv_has_importance) return _voucher_retain(ov);
	kern_return_t kr;
	mach_voucher_t kv, okv;
	// Copy kernel voucher, removing importance.
	okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
	const mach_voucher_attr_recipe_data_t importance_remove_recipe[] = {
		[0] = {
			.key = MACH_VOUCHER_ATTR_KEY_ALL,
			.command = MACH_VOUCHER_ATTR_COPY,
			.previous_voucher = okv,
		},
		[1] = {
			.key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
			.command = MACH_VOUCHER_ATTR_REMOVE,
		},
	};
	kr = _voucher_create_mach_voucher(importance_remove_recipe,
			sizeof(importance_remove_recipe), &kv);
	if (dispatch_assume_zero(kr) || !kv){
		if (ov->v_ipc_kvoucher) return NULL;
		kv = MACH_VOUCHER_NULL;
	}
	if (kv == okv) {
		_voucher_dealloc_mach_voucher(kv);
		return _voucher_retain(ov);
	}
	voucher_t v = _voucher_find_and_retain(kv);
	if (v && ov->v_ipc_kvoucher) {
		_dispatch_voucher_debug("kvoucher[0x%08x] find without importance "
				"from voucher[%p]", v, kv, ov);
		_voucher_dealloc_mach_voucher(kv);
		return v;
	}
	voucher_t kvbase = v;
	voucher_fields_t ignore_fields = VOUCHER_FIELD_KVOUCHER;
	v = _voucher_clone(ov, ignore_fields);
	v->v_kvoucher = kv;
	if (ov->v_ipc_kvoucher) {
		v->v_ipc_kvoucher = kv;
		_voucher_insert(v);
	} else if (kvbase) {
		v->v_kvbase = kvbase;
		_voucher_dealloc_mach_voucher(kv); // borrow base reference
	}
	if (!kvbase) {
		_dispatch_voucher_debug("kvoucher[0x%08x] create without importance "
				"from voucher[%p]", v, kv, ov);
	}
	return v;
}

voucher_t
_voucher_create_accounting_voucher(voucher_t ov)
{
	// Nothing to do unless the old voucher has a kernel voucher. If it does
	// doesn't, it can't have any accounting attributes.
	if (!ov || !ov->v_kvoucher) return NULL;
	kern_return_t kr = KERN_SUCCESS;
	mach_voucher_t okv, kv = MACH_VOUCHER_NULL;
	okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
	const mach_voucher_attr_recipe_data_t accounting_copy_recipe = {
		.key = MACH_VOUCHER_ATTR_KEY_BANK,
		.command = MACH_VOUCHER_ATTR_COPY,
		.previous_voucher = okv,
	};
	kr = _voucher_create_mach_voucher(&accounting_copy_recipe,
			sizeof(accounting_copy_recipe), &kv);
	if (dispatch_assume_zero(kr) || !kv){
		return NULL;
	}
	voucher_t v = _voucher_find_and_retain(kv);
	if (v) {
		_dispatch_voucher_debug("kvoucher[0x%08x] find accounting voucher "
				"from voucher[%p]", v, kv, ov);
		_voucher_dealloc_mach_voucher(kv);
		return v;
	}
	v = _voucher_alloc(0);
	v->v_ipc_kvoucher = v->v_kvoucher = kv;
	if (kv == okv) {
		v->v_kvbase = _voucher_retain(ov);
		_voucher_dealloc_mach_voucher(kv); // borrow base reference
	}
	_voucher_insert(v);
	_dispatch_voucher_debug("kvoucher[0x%08x] create accounting voucher "
			"from voucher[%p]", v, kv, ov);
	return v;
}

voucher_t
voucher_create_with_mach_msg(mach_msg_header_t *msg)
{
	mach_msg_bits_t msgh_bits;
	mach_voucher_t kv = _voucher_mach_msg_get(msg, &msgh_bits);
	return _voucher_create_with_mach_voucher(kv, msgh_bits);
}

void
voucher_decrement_importance_count4CF(voucher_t v)
{
	if (!v || !v->v_kvoucher || !v->v_kv_has_importance) return;
	kern_return_t kr;
	mach_voucher_t kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
	uint32_t dec = 1;
	mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&dec;
	mach_voucher_attr_content_size_t kvc_in_size = sizeof(dec);
	mach_voucher_attr_content_t kvc_out = NULL;
	mach_voucher_attr_content_size_t kvc_out_size = 0;
#if DISPATCH_DEBUG
	uint32_t count = UINT32_MAX;
	kvc_out = (mach_voucher_attr_content_t)&count;
	kvc_out_size = sizeof(count);
#endif
	kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
			MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL, kvc_in, kvc_in_size,
			kvc_out, &kvc_out_size);
	DISPATCH_VERIFY_MIG(kr);
	if (kr == KERN_INVALID_TASK) return; // non-denap receiver rdar://25643185
#if DISPATCH_DEBUG
	_dispatch_voucher_debug("kvoucher[0x%08x] decrement importance count to %u:"
			" %s - 0x%x", v, kv, count, mach_error_string(kr), kr);
#endif
	if (slowpath(dispatch_assume_zero(kr) == KERN_FAILURE)) {
		DISPATCH_CLIENT_CRASH(kr, "Voucher importance count underflow");
	}
}

#if VOUCHER_ENABLE_GET_MACH_VOUCHER
mach_voucher_t
voucher_get_mach_voucher(voucher_t voucher)
{
	return _voucher_get_mach_voucher(voucher);
}
#endif

void
_voucher_xref_dispose(voucher_t voucher)
{
	_dispatch_voucher_debug("xref_dispose", voucher);
	_voucher_remove(voucher);
	return _os_object_release_internal_inline((_os_object_t)voucher);
}

void
_voucher_dispose(voucher_t voucher)
{
	_dispatch_voucher_debug("dispose", voucher);
	if (slowpath(_TAILQ_IS_ENQUEUED(voucher, v_list))) {
		_dispatch_voucher_debug("corruption", voucher);
		DISPATCH_CLIENT_CRASH(voucher->v_list.tqe_prev, "Voucher corruption");
	}
	voucher->v_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
	if (voucher->v_ipc_kvoucher) {
		if (voucher->v_ipc_kvoucher != voucher->v_kvoucher) {
			_voucher_dealloc_mach_voucher(voucher->v_ipc_kvoucher);
		}
		voucher->v_ipc_kvoucher = MACH_VOUCHER_NULL;
	}
	if (voucher->v_kvoucher) {
		if (!voucher->v_kvbase) {
			_voucher_dealloc_mach_voucher(voucher->v_kvoucher);
		}
		voucher->v_kvoucher = MACH_VOUCHER_NULL;
	}
	if (voucher->v_kvbase) {
		_voucher_release(voucher->v_kvbase);
		voucher->v_kvbase = NULL;
	}
	voucher->v_activity = 0;
	voucher->v_activity_creator = 0;
	voucher->v_parent_activity = 0;
	voucher->v_priority = 0;
#if VOUCHER_ENABLE_RECIPE_OBJECTS
	voucher->v_recipe_extra_size = 0;
	voucher->v_recipe_extra_offset = 0;
#endif
	return _os_object_dealloc((_os_object_t)voucher);
}

static void
_voucher_activity_debug_channel_barrier_nop(void *ctxt DISPATCH_UNUSED)
{
}

void
_voucher_activity_debug_channel_init(void)
{
	dispatch_mach_handler_function_t handler = NULL;

	if (_voucher_libtrace_hooks && _voucher_libtrace_hooks->vah_version >= 2) {
		handler = _voucher_libtrace_hooks->vah_debug_channel_handler;
	}

	if (!handler) return;

	dispatch_mach_t dm;
	mach_port_t dbgp;
	kern_return_t kr;

	kr = task_get_debug_control_port(mach_task_self(), &dbgp);
	DISPATCH_VERIFY_MIG(kr);
	if (kr) {
		DISPATCH_CLIENT_CRASH(kr, "Couldn't get debug control port");
	}
	if (dbgp) {
		dm = dispatch_mach_create_f("com.apple.debug-channel",
				DISPATCH_TARGET_QUEUE_DEFAULT, NULL, handler);
		dispatch_mach_connect(dm, dbgp, MACH_PORT_NULL, NULL);
		// will force the DISPATCH_MACH_CONNECTED event
		dispatch_mach_send_barrier_f(dm, NULL,
				_voucher_activity_debug_channel_barrier_nop);
		_voucher_activity_debug_channel = dm;
	}
}

void
_voucher_atfork_child(void)
{
	_dispatch_thread_setspecific(dispatch_voucher_key, NULL);
	_voucher_task_mach_voucher_pred = 0;
	_voucher_task_mach_voucher = MACH_VOUCHER_NULL;
#if !VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
	_voucher_default_task_mach_voucher = MACH_PORT_NULL;
#endif
	_voucher_aid_next = 0;
	_firehose_task_buffer_pred = 0;
	_firehose_task_buffer = NULL; // firehose buffer is VM_INHERIT_NONE
}

#if VOUCHER_EXPORT_PERSONA_SPI
#if VOUCHER_USE_PERSONA
static kern_return_t
_voucher_get_current_persona_token(struct persona_token *token)
{
	kern_return_t kr = KERN_FAILURE;
	voucher_t v = _voucher_get();

	if (v && v->v_kvoucher) {
		mach_voucher_t kv = v->v_ipc_kvoucher ?: v->v_kvoucher;
		mach_voucher_attr_content_t kvc_in = NULL;
		mach_voucher_attr_content_size_t kvc_in_size = 0;
		mach_voucher_attr_content_t kvc_out =
			(mach_voucher_attr_content_t)token;
		mach_voucher_attr_content_size_t kvc_out_size = sizeof(*token);

		kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_BANK,
				BANK_PERSONA_TOKEN, kvc_in, kvc_in_size,
				kvc_out, &kvc_out_size);
		if (kr != KERN_NOT_SUPPORTED
				// Voucher doesn't have a PERSONA_TOKEN
				&& kr != KERN_INVALID_VALUE
				// Kernel doesn't understand BANK_PERSONA_TOKEN
				&& kr != KERN_INVALID_ARGUMENT) {
			(void)dispatch_assume_zero(kr);
		}
	}
	return kr;
}
#endif

uid_t
voucher_get_current_persona(void)
{
	uid_t persona_id = PERSONA_ID_NONE;

#if VOUCHER_USE_PERSONA
	struct persona_token token;
	int err;

	if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
		return token.originator.persona_id;
	}

	// falling back to the process persona if there is no adopted voucher
	if (kpersona_get(&persona_id) < 0) {
		err = errno;
		if (err != ESRCH) {
			(void)dispatch_assume_zero(err);
		}
	}
#endif
	return persona_id;
}

int
voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
{
#if VOUCHER_USE_PERSONA
	struct persona_token token;
	if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
		*persona_info = token.originator;
		return 0;
	}
#else
	(void)persona_info;
#endif
	return -1;
}

int
voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
{
#if VOUCHER_USE_PERSONA
	struct persona_token token;
	if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
		*persona_info = token.proximate;
		return 0;
	}
#else
	(void)persona_info;
#endif
	return -1;
}
#endif

#pragma mark -
#pragma mark _voucher_init

boolean_t
voucher_mach_msg_set(mach_msg_header_t *msg)
{
	return _voucher_mach_msg_set(msg, _voucher_get());
}

void
voucher_mach_msg_clear(mach_msg_header_t *msg)
{
	(void)_voucher_mach_msg_clear(msg, false);
}

voucher_mach_msg_state_t
voucher_mach_msg_adopt(mach_msg_header_t *msg)
{
	mach_msg_bits_t msgh_bits;
	mach_voucher_t kv = _voucher_mach_msg_get(msg, &msgh_bits);
	if (!kv) return VOUCHER_MACH_MSG_STATE_UNCHANGED;
	voucher_t v = _voucher_create_with_mach_voucher(kv, msgh_bits);
	return (voucher_mach_msg_state_t)_voucher_adopt(v);
}

void
voucher_mach_msg_revert(voucher_mach_msg_state_t state)
{
	if (state == VOUCHER_MACH_MSG_STATE_UNCHANGED) return;
	_voucher_replace((voucher_t)state);
}

#if DISPATCH_USE_LIBKERNEL_VOUCHER_INIT
#include <_libkernel_init.h>

static const struct _libkernel_voucher_functions _voucher_libkernel_functions =
{
	.version = 1,
	.voucher_mach_msg_set = voucher_mach_msg_set,
	.voucher_mach_msg_clear = voucher_mach_msg_clear,
	.voucher_mach_msg_adopt = voucher_mach_msg_adopt,
	.voucher_mach_msg_revert = voucher_mach_msg_revert,
};

static void
_voucher_libkernel_init(void)
{
	kern_return_t kr = __libkernel_voucher_init(&_voucher_libkernel_functions);
	dispatch_assert(!kr);
}
#else
#define _voucher_libkernel_init()
#endif

void
voucher_activity_initialize_4libtrace(voucher_activity_hooks_t hooks)
{
	if (!os_atomic_cmpxchg(&_voucher_libtrace_hooks, NULL,
			hooks, relaxed)) {
		DISPATCH_CLIENT_CRASH(_voucher_libtrace_hooks,
				"voucher_activity_initialize_4libtrace called twice");
	}
}

void
_voucher_init(void)
{
	_voucher_libkernel_init();
	unsigned int i;
	for (i = 0; i < VL_HASH_SIZE; i++) {
		TAILQ_INIT(&_vouchers[i]);
	}
}

#pragma mark -
#pragma mark voucher_activity_t

DISPATCH_NOINLINE
static uint64_t
_voucher_activity_id_allocate_slow(uint64_t aid)
{
	kern_return_t kr;
	uint64_t next;

	kr = mach_generate_activity_id(mach_task_self(), 1, &next);
	if (unlikely(kr)) {
		DISPATCH_CLIENT_CRASH(kr, "Could not generate an activity ID");
	}
	next *= MACH_ACTIVITY_ID_RANGE_SIZE;
	next &= MACH_ACTIVITY_ID_MASK;
	if (unlikely(next == 0)) {
		next++;
	}

	if (unlikely(aid == 0)) {
		if (os_atomic_cmpxchg(&_voucher_aid_next, 0, next + 1, relaxed)) {
			return next;
		}
	}
	return os_atomic_xchg(&_voucher_aid_next, next, relaxed);
}

DISPATCH_ALWAYS_INLINE
static firehose_activity_id_t
_voucher_activity_id_allocate(firehose_activity_flags_t flags)
{
	uint64_t aid, next;
	os_atomic_rmw_loop(&_voucher_aid_next, aid, next, relaxed, {
		next = aid + 1;
		if (aid == 0 || next % MACH_ACTIVITY_ID_RANGE_SIZE == 0) {
			os_atomic_rmw_loop_give_up({
				aid = _voucher_activity_id_allocate_slow(aid);
				break;
			});
		}
	});
	return FIREHOSE_ACTIVITY_ID_MAKE(aid, flags);
}

#define _voucher_activity_tracepoint_reserve(stamp, stream, pub, priv, privbuf) \
		firehose_buffer_tracepoint_reserve(_firehose_task_buffer, stamp, \
				stream, pub, priv, privbuf)

#define _voucher_activity_tracepoint_flush(ft, ftid) \
		firehose_buffer_tracepoint_flush(_firehose_task_buffer, ft, ftid)

DISPATCH_NOINLINE
static void
_firehose_task_buffer_init(void *ctx OS_UNUSED)
{
	mach_port_t logd_port;

	/* Query the uniquepid of the current process */
	struct proc_uniqidentifierinfo p_uniqinfo = { };
	int info_size = 0;

	info_size = proc_pidinfo(getpid(), PROC_PIDUNIQIDENTIFIERINFO, 1,
			&p_uniqinfo, PROC_PIDUNIQIDENTIFIERINFO_SIZE);
	if (slowpath(info_size != PROC_PIDUNIQIDENTIFIERINFO_SIZE)) {
		DISPATCH_INTERNAL_CRASH(info_size, "Unable to get the unique pid");
	}
	_voucher_unique_pid = p_uniqinfo.p_uniqueid;


	if (!fastpath(_voucher_libtrace_hooks)) {
		if (0) { // <rdar://problem/23393959>
			DISPATCH_CLIENT_CRASH(0,
					"Activity subsystem isn't initialized yet");
		}
		return;
	}
	logd_port = _voucher_libtrace_hooks->vah_get_logd_port();
	if (logd_port) {
		unsigned long flags = 0;
#if DISPATCH_USE_MEMORYPRESSURE_SOURCE
		if (_dispatch_memory_warn) {
			flags |= FIREHOSE_BUFFER_BANK_FLAG_LOW_MEMORY;
		}
#endif
		// firehose_buffer_create always consumes the send-right
		_firehose_task_buffer = firehose_buffer_create(logd_port,
				_voucher_unique_pid, flags);
	}
}

DISPATCH_ALWAYS_INLINE
static inline bool
_voucher_activity_disabled(void)
{
	dispatch_once_f(&_firehose_task_buffer_pred,
			NULL, _firehose_task_buffer_init);

	firehose_buffer_t fb = _firehose_task_buffer;
	if (fastpath(fb)) {
		return slowpath(fb->fb_header.fbh_sendp == MACH_PORT_DEAD);
	}
	return true;
}

void*
voucher_activity_get_metadata_buffer(size_t *length)
{
	if (_voucher_activity_disabled()) {
		*length = 0;
		return NULL;
	}

	firehose_buffer_header_t fbh = &_firehose_task_buffer->fb_header;

	*length = FIREHOSE_BUFFER_LIBTRACE_HEADER_SIZE;
	return (void *)((uintptr_t)(fbh + 1) - *length);
}

voucher_t
voucher_activity_create(firehose_tracepoint_id_t trace_id,
		voucher_t base, firehose_activity_flags_t flags, uint64_t location)
{
	return voucher_activity_create_with_location(&trace_id, base, flags, location);
}

voucher_t
voucher_activity_create_with_location(firehose_tracepoint_id_t *trace_id,
		voucher_t base, firehose_activity_flags_t flags, uint64_t location)
{
	firehose_activity_id_t va_id = 0, current_id = 0, parent_id = 0;
	firehose_tracepoint_id_u ftid = { .ftid_value = *trace_id };
	uint16_t pubsize = sizeof(va_id) + sizeof(location);
	uint64_t creator_id = 0;
	voucher_t ov = _voucher_get();
	voucher_t v;

	if (base == VOUCHER_CURRENT) {
		base = ov;
	}
	if (_voucher_activity_disabled()) {
		*trace_id = 0;
		return base ? _voucher_retain(base) : VOUCHER_NULL;
	}

	FIREHOSE_TRACE_ID_CLEAR_FLAG(ftid, base, has_unique_pid);
	if (ov && (current_id = ov->v_activity)) {
		FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
		pubsize += sizeof(firehose_activity_id_t);
		if ((creator_id = ov->v_activity_creator)) {
			FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_unique_pid);
			pubsize += sizeof(uint64_t);
		}
	}
	if (base != VOUCHER_NULL) {
		parent_id = base->v_activity;
	}

	if (parent_id) {
		FIREHOSE_TRACE_ID_SET_FLAG(ftid, activity, has_other_aid);
		pubsize += sizeof(firehose_activity_id_t);
		flags |= FIREHOSE_ACTIVITY_ID_FLAGS(parent_id);
	}

	if (firehose_precise_timestamps_enabled()) {
		flags |= firehose_activity_flags_precise_timestamp;
	}
	voucher_fields_t ignore_fields = VOUCHER_FIELD_ACTIVITY;
	v = _voucher_clone(base, ignore_fields);
	v->v_activity = va_id = _voucher_activity_id_allocate(flags);
	v->v_activity_creator = _voucher_unique_pid;
	v->v_parent_activity = parent_id;

	static const firehose_stream_t streams[2] = {
		firehose_stream_metadata,
		firehose_stream_persist,
	};
	firehose_tracepoint_t ft;
	uint64_t stamp = firehose_tracepoint_time(flags);

	for (size_t i = 0; i < countof(streams); i++) {
		ft = _voucher_activity_tracepoint_reserve(stamp, streams[i], pubsize,
				0, NULL);
		if (!fastpath(ft)) continue;

		uint8_t *pubptr = ft->ft_data;
		if (current_id) {
			pubptr = _dispatch_memappend(pubptr, &current_id);
		}
		if (creator_id) {
			pubptr = _dispatch_memappend(pubptr, &creator_id);
		}
		if (parent_id) {
			pubptr = _dispatch_memappend(pubptr, &parent_id);
		}
		pubptr = _dispatch_memappend(pubptr, &va_id);
		pubptr = _dispatch_memappend(pubptr, &location);
		_voucher_activity_tracepoint_flush(ft, ftid);
	}
	*trace_id = ftid.ftid_value;
	return v;
}

void
_voucher_activity_swap(firehose_activity_id_t old_id,
		firehose_activity_id_t new_id)
{
	if (_voucher_activity_disabled()) return;

	firehose_tracepoint_id_u ftid = { .ftid = {
		._namespace = firehose_tracepoint_namespace_activity,
		._type = _firehose_tracepoint_type_activity_swap,
	} };
	uint16_t pubsize = 0;

	if (old_id) {
		FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
		pubsize += sizeof(firehose_activity_id_t);
	}
	if (new_id) {
		FIREHOSE_TRACE_ID_SET_FLAG(ftid, activity, has_other_aid);
		pubsize += sizeof(firehose_activity_id_t);
	}

	firehose_stream_t stream = firehose_stream_metadata;
	firehose_tracepoint_t ft;
	firehose_activity_flags_t flags = FIREHOSE_ACTIVITY_ID_FLAGS(old_id) |
			FIREHOSE_ACTIVITY_ID_FLAGS(new_id);
	uint64_t stamp = firehose_tracepoint_time(flags);

	_dispatch_voucher_ktrace_activity_adopt(new_id);

	ft = _voucher_activity_tracepoint_reserve(stamp, stream, pubsize, 0, NULL);
	if (!fastpath(ft)) return;
	uint8_t *pubptr = ft->ft_data;
	if (old_id) pubptr = _dispatch_memappend(pubptr, &old_id);
	if (new_id) pubptr = _dispatch_memappend(pubptr, &new_id);
	_voucher_activity_tracepoint_flush(ft, ftid);
}

firehose_activity_id_t
voucher_get_activity_id_and_creator(voucher_t v, uint64_t *creator_pid,
		firehose_activity_id_t *parent_id)
{
	if (v == VOUCHER_CURRENT) {
		v = _voucher_get();
	}
	if (v == VOUCHER_NULL) {
		if (creator_pid) *creator_pid = 0;
		if (parent_id) *parent_id = FIREHOSE_ACTIVITY_ID_NULL;
		return FIREHOSE_ACTIVITY_ID_NULL;
	}
	if (creator_pid) *creator_pid = v->v_activity_creator;
	if (parent_id) *parent_id = v->v_parent_activity;
	return v->v_activity;
}

firehose_activity_id_t
voucher_get_activity_id(voucher_t v, firehose_activity_id_t *parent_id)
{
	return voucher_get_activity_id_and_creator(v, NULL, parent_id);
}

void
voucher_activity_flush(firehose_stream_t stream)
{
	if (_voucher_activity_disabled()) return;
	firehose_buffer_stream_flush(_firehose_task_buffer, stream);
}

DISPATCH_ALWAYS_INLINE
static inline firehose_tracepoint_id_t
_voucher_activity_trace(firehose_stream_t stream,
		firehose_tracepoint_id_u ftid, uint64_t stamp,
		const void *pubdata, size_t publen,
		const void *privdata, size_t privlen)
{
	const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
	const size_t _firehose_chunk_payload_size =
			sizeof(((struct firehose_buffer_chunk_s *)0)->fbc_data);

	if (_voucher_activity_disabled()) return 0;

	firehose_tracepoint_t ft;
	firehose_activity_id_t va_id = 0;
	firehose_buffer_chunk_t fbc;
	uint8_t *privptr, *pubptr;
	size_t pubsize = publen;
	voucher_t ov = _voucher_get();
	uint64_t creator_pid;

	if ((va_id = _voucher_get_activity_id(ov, &creator_pid))) {
		FIREHOSE_TRACE_ID_SET_FLAG(ftid, base, has_current_aid);
		pubsize += sizeof(va_id);
	}
	if (FIREHOSE_TRACE_ID_HAS_FLAG(ftid, base, has_unique_pid)) {
		if (creator_pid) {
			pubsize += sizeof(creator_pid);
		} else {
			FIREHOSE_TRACE_ID_CLEAR_FLAG(ftid, base, has_unique_pid);
		}
	} else {
		creator_pid = 0;
	}

	if (privlen) {
		FIREHOSE_TRACE_ID_SET_FLAG(ftid, log, has_private_data);
		pubsize += sizeof(struct firehose_buffer_range_s);
	}

	if (slowpath(ft_size + pubsize + privlen > _firehose_chunk_payload_size)) {
		DISPATCH_CLIENT_CRASH(ft_size + pubsize + privlen, "Log is too large");
	}

	ft = _voucher_activity_tracepoint_reserve(stamp, stream, (uint16_t)pubsize,
				(uint16_t)privlen, &privptr);
	if (!fastpath(ft)) return 0;
	pubptr = ft->ft_data;
	if (va_id) {
		pubptr = _dispatch_memappend(pubptr, &va_id);
	}
	if (creator_pid) {
		pubptr = _dispatch_memappend(pubptr, &creator_pid);
	}
	if (privlen) {
		fbc = firehose_buffer_chunk_for_address(ft);
		struct firehose_buffer_range_s range = {
			.fbr_offset = (uint16_t)(privptr - fbc->fbc_start),
			.fbr_length = (uint16_t)privlen,
		};
		pubptr = _dispatch_memappend(pubptr, &range);
		_dispatch_mempcpy(privptr, privdata, privlen);
	}
	_dispatch_mempcpy(pubptr, pubdata, publen);
	_voucher_activity_tracepoint_flush(ft, ftid);
	return ftid.ftid_value;
}

firehose_tracepoint_id_t
voucher_activity_trace(firehose_stream_t stream,
		firehose_tracepoint_id_t trace_id, uint64_t timestamp,
		const void *pubdata, size_t publen)
{
	firehose_tracepoint_id_u ftid = { .ftid_value = trace_id };
	return _voucher_activity_trace(stream, ftid, timestamp, pubdata, publen,
			NULL, 0);
}

firehose_tracepoint_id_t
voucher_activity_trace_with_private_strings(firehose_stream_t stream,
		firehose_tracepoint_id_t trace_id, uint64_t timestamp,
		const void *pubdata, size_t publen,
		const void *privdata, size_t privlen)
{
	firehose_tracepoint_id_u ftid = { .ftid_value = trace_id };
	return _voucher_activity_trace(stream, ftid, timestamp,
			pubdata, publen, privdata, privlen);
}

#pragma mark -
#pragma mark _voucher_debug

size_t
_voucher_debug(voucher_t v, char* buf, size_t bufsiz)
{
	size_t offset = 0;
	#define bufprintf(...) \
			offset += dsnprintf(&buf[offset], bufsiz - offset, ##__VA_ARGS__)
	bufprintf("voucher[%p] = { xrefcnt = 0x%x, refcnt = 0x%x", v,
			v->os_obj_xref_cnt + 1, v->os_obj_ref_cnt + 1);

	if (v->v_kvbase) {
		bufprintf(", base voucher %p", v->v_kvbase);
	}
	if (v->v_kvoucher) {
		bufprintf(", kvoucher%s 0x%x", v->v_kvoucher == v->v_ipc_kvoucher ?
				" & ipc kvoucher" : "", v->v_kvoucher);
	}
	if (v->v_ipc_kvoucher && v->v_ipc_kvoucher != v->v_kvoucher) {
		bufprintf(", ipc kvoucher 0x%x", v->v_ipc_kvoucher);
	}
	if (v->v_priority) {
		bufprintf(", QOS 0x%x", v->v_priority);
	}
	if (v->v_activity) {
		bufprintf(", activity 0x%llx (pid: 0x%16llx, parent 0x%llx)",
				v->v_activity, v->v_activity_creator, v->v_parent_activity);
	}
	bufprintf(" }");
	return offset;
}

#else // VOUCHER_USE_MACH_VOUCHER

#pragma mark -
#pragma mark Simulator / vouchers disabled

#if VOUCHER_ENABLE_RECIPE_OBJECTS
voucher_t
voucher_create(voucher_recipe_t recipe)
{
	(void)recipe;
	return NULL;
}
#endif

voucher_t
voucher_adopt(voucher_t voucher)
{
	return voucher;
}

voucher_t
voucher_copy(void)
{
	return NULL;
}

voucher_t
voucher_copy_without_importance(void)
{
	return NULL;
}

voucher_t
voucher_retain(voucher_t voucher)
{
	return voucher;
}

void
voucher_release(voucher_t voucher)
{
	(void)voucher;
}

void
voucher_replace_default_voucher(void)
{
}

void
voucher_decrement_importance_count4CF(voucher_t v)
{
	(void)v;
}

void
_voucher_thread_cleanup(void *voucher)
{
	(void)voucher;
}

void
_voucher_dealloc_mach_voucher(mach_voucher_t kv)
{
	(void)kv;
}

mach_voucher_t
_voucher_get_mach_voucher(voucher_t voucher)
{
	(void)voucher;
	return MACH_VOUCHER_NULL;
}

mach_voucher_t
_voucher_create_mach_voucher_with_priority(voucher_t voucher,
		pthread_priority_t priority)
{
	(void)voucher; (void)priority;
	return MACH_VOUCHER_NULL;
}

voucher_t
_voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
		pthread_priority_t priority, mach_voucher_t kv)
{
	(void)voucher; (void)priority; (void)kv;
	return NULL;
}

voucher_t
_voucher_create_accounting_voucher(voucher_t voucher)
{
	(void)voucher;
	return NULL;
}

voucher_t
voucher_create_with_mach_msg(mach_msg_header_t *msg)
{
	(void)msg;
	return NULL;
}

#if VOUCHER_ENABLE_GET_MACH_VOUCHER
mach_voucher_t
voucher_get_mach_voucher(voucher_t voucher)
{
	(void)voucher;
	return 0;
}
#endif

void
_voucher_xref_dispose(voucher_t voucher)
{
	(void)voucher;
}

void
_voucher_dispose(voucher_t voucher)
{
	(void)voucher;
}

#if VOUCHER_EXPORT_PERSONA_SPI
uid_t
voucher_get_current_persona(void)
{
	return PERSONA_ID_NONE;
}

int
voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
{
	(void)persona_info;
	return -1;
}

int
voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
{
	(void)persona_info;
	return -1;
}
#endif

void
_voucher_activity_debug_channel_init(void)
{
}

void
_voucher_atfork_child(void)
{
}

void
_voucher_init(void)
{
}

void*
voucher_activity_get_metadata_buffer(size_t *length)
{
    *length = 0;
    return NULL;
}

voucher_t
voucher_activity_create(firehose_tracepoint_id_t trace_id,
		voucher_t base, firehose_activity_flags_t flags, uint64_t location)
{
	(void)trace_id; (void)base; (void)flags; (void)location;
	return NULL;
}

voucher_t
voucher_activity_create_with_location(firehose_tracepoint_id_t *trace_id,
		voucher_t base, firehose_activity_flags_t flags, uint64_t location)
{
	(void)trace_id; (void)base; (void)flags; (void)location;
	return NULL;
}

firehose_activity_id_t
voucher_get_activity_id(voucher_t voucher, firehose_activity_id_t *parent_id)
{
	(void)voucher; (void)parent_id;
	return 0;
}

firehose_activity_id_t
voucher_get_activity_id_and_creator(voucher_t voucher, uint64_t *creator_pid,
		firehose_activity_id_t *parent_id)
{
	if (creator_pid) *creator_pid = 0;
	(void)voucher; (void)parent_id;
	return 0;
}

firehose_tracepoint_id_t
voucher_activity_trace(firehose_stream_t stream,
		firehose_tracepoint_id_t trace_id, uint64_t timestamp,
		const void *pubdata, size_t publen)
{
	(void)stream; (void)trace_id; (void)timestamp;
	(void)pubdata; (void)publen;
	return 0;
}

firehose_tracepoint_id_t
voucher_activity_trace_with_private_strings(firehose_stream_t stream,
		firehose_tracepoint_id_t trace_id, uint64_t timestamp,
		const void *pubdata, size_t publen,
		const void *privdata, size_t privlen)
{
	(void)stream; (void)trace_id; (void)timestamp;
	(void)pubdata; (void)publen; (void)privdata; (void)privlen;
	return 0;
}

void
voucher_activity_flush(firehose_stream_t stream)
{
	(void)stream;
}

void
voucher_activity_initialize_4libtrace(voucher_activity_hooks_t hooks)
{
	(void)hooks;
}

size_t
_voucher_debug(voucher_t v, char* buf, size_t bufsiz)
{
	(void)v; (void)buf; (void)bufsiz;
	return 0;
}

#endif // VOUCHER_USE_MACH_VOUCHER

#else // DISPATCH_VARIANT_DYLD_STUB

firehose_activity_id_t
voucher_get_activity_id_4dyld(void)
{
#if VOUCHER_USE_MACH_VOUCHER
	return _voucher_get_activity_id(_voucher_get(), NULL);
#else
	return 0;
#endif
}

#endif // DISPATCH_VARIANT_DYLD_STUB