ipc_importance.c   [plain text]

/*
 * Copyright (c) 2013 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach/mach_types.h>
#include <mach/notify.h>
#include <ipc/ipc_types.h>
#include <ipc/ipc_importance.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_voucher.h>
#include <kern/ipc_kobject.h>
#include <kern/ipc_tt.h>
#include <kern/mach_param.h>
#include <kern/misc_protos.h>
#include <kern/kalloc.h>
#include <kern/zalloc.h>
#include <kern/queue.h>
#include <kern/task.h>

#include <sys/kdebug.h>

#include <mach/mach_voucher_attr_control.h>
#include <mach/machine/sdt.h>

extern int	proc_pid(void *);
extern int	proc_selfpid(void);
extern uint64_t proc_uniqueid(void *p);
extern char	*proc_name_address(void *p);

/*
 * Globals for delayed boost drop processing.
 */
static queue_head_t ipc_importance_delayed_drop_queue;
static thread_call_t ipc_importance_delayed_drop_call;
static uint64_t ipc_importance_delayed_drop_timestamp;
static boolean_t ipc_importance_delayed_drop_call_requested = FALSE;

#define DENAP_DROP_TARGET (1000 * NSEC_PER_MSEC) /* optimum denap delay */
#define DENAP_DROP_SKEW    (100 * NSEC_PER_MSEC) /* request skew for wakeup */
#define DENAP_DROP_LEEWAY  (2 * DENAP_DROP_SKEW)  /* specified wakeup leeway */

#define DENAP_DROP_DELAY (DENAP_DROP_TARGET + DENAP_DROP_SKEW)
#define DENAP_DROP_FLAGS (THREAD_CALL_DELAY_SYS_NORMAL | THREAD_CALL_DELAY_LEEWAY)

/*
 * Importance Voucher Attribute Manager
 */

static lck_spin_t ipc_importance_lock_data;	/* single lock for now */


#define ipc_importance_lock_init() \
	lck_spin_init(&ipc_importance_lock_data, &ipc_lck_grp, &ipc_lck_attr)
#define ipc_importance_lock_destroy() \
	lck_spin_destroy(&ipc_importance_lock_data, &ipc_lck_grp)
#define	ipc_importance_lock() \
	lck_spin_lock(&ipc_importance_lock_data)
#define	ipc_importance_lock_try() \
	lck_spin_try_lock(&ipc_importance_lock_data)
#define	ipc_importance_unlock() \
	lck_spin_unlock(&ipc_importance_lock_data)
#define ipc_importance_sleep(elem) lck_spin_sleep(&ipc_importance_lock_data,	\
					LCK_SLEEP_DEFAULT,			\
					(event_t)(elem),			\
					THREAD_UNINT)
#define ipc_importance_wakeup(elem) thread_wakeup((event_t)(elem))

#if IIE_REF_DEBUG
#define incr_ref_counter(x) (hw_atomic_add(&(x), 1))

static inline
uint32_t ipc_importance_reference_internal(ipc_importance_elem_t elem) 
{
	incr_ref_counter(elem->iie_refs_added);
	return (hw_atomic_add(&elem->iie_bits, 1) & IIE_REFS_MASK);
}

static inline
uint32_t ipc_importance_release_internal(ipc_importance_elem_t elem)
{
	incr_ref_counter(elem->iie_refs_dropped);
	return (hw_atomic_sub(&elem->iie_bits, 1) & IIE_REFS_MASK);
}

static inline 
uint32_t ipc_importance_task_reference_internal(ipc_importance_task_t task_imp)
{
  uint32_t out;
  out = ipc_importance_reference_internal(&task_imp->iit_elem);
  incr_ref_counter(task_imp->iit_elem.iie_task_refs_added);
  return out;
}

static inline
uint32_t ipc_importance_task_release_internal(ipc_importance_task_t task_imp)
{
  uint32_t out;

  assert(1 < IIT_REFS(task_imp));
  incr_ref_counter(task_imp->iit_elem.iie_task_refs_dropped);
  out = ipc_importance_release_internal(&task_imp->iit_elem);
  return out;
}

static inline
void ipc_importance_counter_init(ipc_importance_elem_t elem)
{
  
	elem->iie_refs_added = 0;
	elem->iie_refs_dropped = 0;
	elem->iie_kmsg_refs_added = 0;
	elem->iie_kmsg_refs_inherited = 0;
	elem->iie_kmsg_refs_coalesced = 0;
	elem->iie_kmsg_refs_dropped = 0;
	elem->iie_task_refs_added = 0;
	elem->iie_task_refs_added_inherit_from = 0;
	elem->iie_task_refs_added_transition = 0;
	elem->iie_task_refs_self_added = 0;
	elem->iie_task_refs_inherited = 0;
	elem->iie_task_refs_coalesced = 0;
	elem->iie_task_refs_dropped = 0;
}
#else
#define incr_ref_counter(x)
#endif

#if DEVELOPMENT || DEBUG
static queue_head_t global_iit_alloc_queue;
#endif

/* TODO: remove this varibale when interactive daemon audit is complete */
boolean_t ipc_importance_interactive_receiver = FALSE;

static zone_t ipc_importance_task_zone;
static zone_t ipc_importance_inherit_zone;

static ipc_voucher_attr_control_t ipc_importance_control;

/*
 *	Routine:	ipc_importance_kmsg_link
 *	Purpose:
 *		Link the kmsg onto the appropriate propagation chain.
 *		If the element is a task importance, we link directly
 *		on its propagation chain. Otherwise, we link onto the
 *		destination task of the inherit.
 *	Conditions:
 *		Importance lock held.
 *		Caller is donating an importance elem reference to the kmsg.
 */
static void
ipc_importance_kmsg_link(
	ipc_kmsg_t		kmsg,
	ipc_importance_elem_t	elem)
{
	ipc_importance_elem_t link_elem;

	assert(IIE_NULL == kmsg->ikm_importance);

	link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
		(ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task :
		elem;

	queue_enter(&link_elem->iie_kmsgs, kmsg, ipc_kmsg_t, ikm_inheritance);
	kmsg->ikm_importance = elem;
}

/*
 *	Routine:	ipc_importance_kmsg_unlink
 *	Purpose:
 *		Unlink the kmsg from its current propagation chain.
 *		If the element is a task importance, we unlink directly
 *		from its propagation chain. Otherwise, we unlink from the
 *		destination task of the inherit.
 *	Returns:
 *		The reference to the importance element it was linked on.
 *	Conditions:
 *		Importance lock held.
 *		Caller is responsible for dropping reference on returned elem.
 */
static ipc_importance_elem_t
ipc_importance_kmsg_unlink(
	ipc_kmsg_t		kmsg)
{
	ipc_importance_elem_t elem = kmsg->ikm_importance;

	if (IIE_NULL != elem) {
		ipc_importance_elem_t unlink_elem;

		unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
			(ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : 
			elem;

		queue_remove(&unlink_elem->iie_kmsgs, kmsg, ipc_kmsg_t, ikm_inheritance);
		kmsg->ikm_importance = IIE_NULL;
	}
	return elem;
}

/*
 *	Routine:	ipc_importance_inherit_link
 *	Purpose:
 *		Link the inherit onto the appropriate propagation chain.
 *		If the element is a task importance, we link directly
 *		on its propagation chain. Otherwise, we link onto the
 *		destination task of the inherit.
 *	Conditions:
 *		Importance lock held.
 *		Caller is donating an elem importance reference to the inherit.
 */
static void
ipc_importance_inherit_link(
	ipc_importance_inherit_t inherit,
	ipc_importance_elem_t elem)
{
	ipc_importance_elem_t link_elem;

	assert(IIE_NULL == inherit->iii_from_elem);
	link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
		(ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task :
		elem;

	queue_enter(&link_elem->iie_inherits, inherit, 
		    ipc_importance_inherit_t, iii_inheritance);
	inherit->iii_from_elem = elem;
}

/*
 *	Routine:	ipc_importance_inherit_unlink
 *	Purpose:
 *		Unlink the inherit from its current propagation chain.
 *		If the element is a task importance, we unlink directly
 *		from its propagation chain. Otherwise, we unlink from the
 *		destination task of the inherit.
 *	Returns:
 *		The reference to the importance element it was linked on.
 *	Conditions:
 *		Importance lock held.
 *		Caller is responsible for dropping reference on returned elem.
 */
static ipc_importance_elem_t
ipc_importance_inherit_unlink(
	ipc_importance_inherit_t inherit)
{
	ipc_importance_elem_t elem = inherit->iii_from_elem;

	if (IIE_NULL != elem) {
		ipc_importance_elem_t unlink_elem;

		unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
			(ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : 
			elem;

		queue_remove(&unlink_elem->iie_inherits, inherit, 
			     ipc_importance_inherit_t, iii_inheritance);
		inherit->iii_from_elem = IIE_NULL;
	}
	return elem;
}

/*
 *	Routine:	ipc_importance_reference
 *	Purpose:
 *		Add a reference to the importance element.
 *	Conditions:
 *		Caller must hold a reference on the element.
 */
void
ipc_importance_reference(ipc_importance_elem_t elem)
{
	assert(0 < IIE_REFS(elem));
	ipc_importance_reference_internal(elem);
}

/*
 *	Routine:	ipc_importance_release_locked
 *	Purpose:
 *		Release a reference on an importance attribute value,
 *		unlinking and deallocating the attribute if the last reference.
 *	Conditions:
 *		Entered with importance lock held, leaves with it unlocked.
 */
static void
ipc_importance_release_locked(ipc_importance_elem_t elem)
{
	assert(0 < IIE_REFS(elem));

	if (0 < ipc_importance_release_internal(elem)) {

#if DEVELOPMENT || DEBUG
		ipc_importance_inherit_t temp_inherit;
		ipc_importance_task_t link_task;
		ipc_kmsg_t temp_kmsg;
		uint32_t expected = 0;

		if (0 < elem->iie_made)
	  		expected++;

		link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ?
			((ipc_importance_inherit_t)elem)->iii_to_task : 
			(ipc_importance_task_t)elem;

		queue_iterate(&link_task->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance)
			if (temp_kmsg->ikm_importance == elem)
				expected++;
		queue_iterate(&link_task->iit_inherits, temp_inherit,
			      ipc_importance_inherit_t, iii_inheritance)
			if (temp_inherit->iii_from_elem == elem)
				expected++;

		if (IIE_REFS(elem) < expected)
			panic("ipc_importance_release_locked (%p)", elem);
#endif
		ipc_importance_unlock();
		return;
	}

	/* last ref */
	/* can't get to no refs if we contribute to something else's importance */
	assert(queue_empty(&elem->iie_kmsgs));
	assert(queue_empty(&elem->iie_inherits));

	switch (IIE_TYPE(elem)) {

	/* just a "from" task reference to drop */
	case IIE_TYPE_TASK:
	{
		ipc_importance_task_t task_elem;

		task_elem = (ipc_importance_task_t)elem;
		assert(TASK_NULL == task_elem->iit_task);

#if DEVELOPMENT || DEBUG
		queue_remove(&global_iit_alloc_queue, task_elem, ipc_importance_task_t, iit_allocation);
#endif

		ipc_importance_unlock();

		zfree(ipc_importance_task_zone, task_elem);
		break;
	}

	/* dropping an inherit element */
	case IIE_TYPE_INHERIT:
	{
		ipc_importance_inherit_t inherit;
		ipc_importance_elem_t from_elem;
		ipc_importance_task_t to_task;


		inherit = (ipc_importance_inherit_t)elem;
		to_task = inherit->iii_to_task;
		assert(IIT_NULL != to_task);
		assert(!inherit->iii_donating);

		/* unlink and release the inherit */
		assert(ipc_importance_task_is_any_receiver_type(to_task));
		from_elem = ipc_importance_inherit_unlink(inherit);
		assert(IIE_NULL != from_elem);
		ipc_importance_release_locked(from_elem);
		/* unlocked on return */

		ipc_importance_task_release(to_task);

		zfree(ipc_importance_inherit_zone, inherit);
		break;
	}
	}
}

/*
 *	Routine:	ipc_importance_release
 *	Purpose:
 *		Release a reference on an importance attribute value,
 *		unlinking and deallocating the attribute if the last reference.
 *	Conditions:
 *		nothing locked on entrance, nothing locked on exit.
 *		May block.
 */
void
ipc_importance_release(ipc_importance_elem_t elem)
{
	if (IIE_NULL == elem)
		return;

	ipc_importance_lock();
	ipc_importance_release_locked(elem);
	/* unlocked */
}

/*
 *	Routine:	ipc_importance_task_reference


 *	Purpose:
 *		Retain a reference on a task importance attribute value.
 *	Conditions:
 *		nothing locked on entrance, nothing locked on exit.
 *		caller holds a reference already.
 */
void
ipc_importance_task_reference(ipc_importance_task_t task_elem)
{
	if (IIT_NULL == task_elem)
		return;
#if IIE_REF_DEBUG
	incr_ref_counter(task_elem->iit_elem.iie_task_refs_added);
#endif
	ipc_importance_reference(&task_elem->iit_elem);
}

/*
 *	Routine:	ipc_importance_task_release
 *	Purpose:
 *		Release a reference on a task importance attribute value,
 *		unlinking and deallocating the attribute if the last reference.
 *	Conditions:
 *		nothing locked on entrance, nothing locked on exit.
 *		May block.
 */
void
ipc_importance_task_release(ipc_importance_task_t task_elem)
{
	if (IIT_NULL == task_elem)
		return;

	ipc_importance_lock();
#if IIE_REF_DEBUG
	incr_ref_counter(task_elem->iit_elem.iie_task_refs_dropped);
#endif
	ipc_importance_release_locked(&task_elem->iit_elem);
	/* unlocked */
}

/*
 *	Routine:	ipc_importance_task_release_locked
 *	Purpose:
 *		Release a reference on a task importance attribute value,
 *		unlinking and deallocating the attribute if the last reference.
 *	Conditions:
 *		importance lock held on entry, nothing locked on exit.
 *		May block.
 */
static void
ipc_importance_task_release_locked(ipc_importance_task_t task_elem)
{
	if (IIT_NULL == task_elem) {
		ipc_importance_unlock();
		return;
	}
#if IIE_REF_DEBUG
	incr_ref_counter(task_elem->iit_elem.iie_task_refs_dropped);
#endif
	ipc_importance_release_locked(&task_elem->iit_elem);
	/* unlocked */
}

/*
 * Routines for importance donation/inheritance/boosting
 */


/*
 * External importance assertions are managed by the process in userspace
 * Internal importance assertions are the responsibility of the kernel
 * Assertions are changed from internal to external via task_importance_externalize_assertion
 */

/*
 *	Routine:	ipc_importance_task_check_transition
 *	Purpose:
 *		Increase or decrement the internal task importance counter of the
 *		specified task and determine if propagation and a task policy
 *		update is required.
 *
 *		If it is already enqueued for a policy update, steal it from that queue
 *		(as we are reversing that update before it happens).
 *
 *	Conditions:
 *		Called with the importance lock held.
 *		It is the caller's responsibility to perform the propagation of the
 *		transition and/or policy changes by checking the return value.
 */
static boolean_t
ipc_importance_task_check_transition(
	ipc_importance_task_t task_imp,
	iit_update_type_t type,
	uint32_t delta)
{

	task_t target_task = task_imp->iit_task;
	boolean_t boost = (IIT_UPDATE_HOLD == type);
	boolean_t before_boosted, after_boosted;

	if (!ipc_importance_task_is_any_receiver_type(task_imp))
		return FALSE;

#if IMPORTANCE_DEBUG
	int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1;

	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_START,
					  proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0);
#endif

	/* snapshot the effective boosting status before making any changes */
	before_boosted = (task_imp->iit_assertcnt > 0);

	/* Adjust the assertcnt appropriately */
	if (boost) {
		task_imp->iit_assertcnt += delta;
#if IMPORTANCE_DEBUG
        DTRACE_BOOST6(send_boost, task_t, target_task, int, target_pid,
                      task_t, current_task(), int, proc_selfpid(), int, delta, int, task_imp->iit_assertcnt);
#endif
	} else {
	  	// assert(delta <= task_imp->iit_assertcnt);
		if (delta > task_imp->iit_assertcnt - IIT_EXTERN(task_imp)) {
			/* TODO: Turn this back into a panic <rdar://problem/12592649> */
			if (target_task != TASK_NULL) {
				printf("Over-release of kernel-internal importance assertions for pid %d (%s), "
				       "dropping %d assertion(s) but task only has %d remaining (%d external).\n",
				       audit_token_pid_from_task(target_task),
				       (target_task->bsd_info == NULL) ? "" : proc_name_address(target_task->bsd_info),
				       delta,
				       task_imp->iit_assertcnt,
				       IIT_EXTERN(task_imp));
			}
			task_imp->iit_assertcnt = IIT_EXTERN(task_imp);
		} else {
			task_imp->iit_assertcnt -= delta;
		}
#if IMPORTANCE_DEBUG
		// This convers both legacy and voucher-based importance.
		DTRACE_BOOST4(drop_boost, task_t, target_task, int, target_pid, int, delta, int, task_imp->iit_assertcnt);
#endif
	}

#if IMPORTANCE_DEBUG
	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_END,
				  proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0);
#endif

	/* did the change result in an effective donor status change? */
	after_boosted = (task_imp->iit_assertcnt > 0);

	if (after_boosted != before_boosted) {

		/* 
		 * If the task importance is already on an update queue, we just reversed the need for a
		 * pending policy update.  If the queue is any other than the delayed-drop-queue, pull it 
		 * off that queue and release the reference it got going onto the update queue.  If it is
		 * the delayed-drop-queue we leave it in place in case it comes back into the drop state
		 * before its time delay is up.
		 *
		 * We still need to propagate the change downstream to reverse the assertcnt effects,
		 * but we no longer need to update this task's boost policy state.
		 *
		 * Otherwise, mark it as needing a policy update.
		 */
		assert(0 == task_imp->iit_updatepolicy);
		if (NULL != task_imp->iit_updateq) {
			if (&ipc_importance_delayed_drop_queue != task_imp->iit_updateq) {
				queue_remove(task_imp->iit_updateq, task_imp, ipc_importance_task_t, iit_updates);
				task_imp->iit_updateq = NULL;
				ipc_importance_task_release_internal(task_imp); /* can't be last ref */
			}
		} else {
			task_imp->iit_updatepolicy = 1;
		}
		return TRUE;
	}

	return FALSE;
}


/*
 *	Routine:	ipc_importance_task_propagate_helper
 *	Purpose:
 *		Increase or decrement the internal task importance counter of all
 *		importance tasks inheriting from the specified one.  If this causes
 *		that importance task to change state, add it to the list of tasks
 *		to do a policy update against.
 *	Conditions:
 *		Called with the importance lock held.
 *		It is the caller's responsibility to iterate down the generated list
 *		and propagate any subsequent assertion changes from there.
 */
static void
ipc_importance_task_propagate_helper(
	ipc_importance_task_t task_imp,
	iit_update_type_t type,
	queue_t propagation)
{
	ipc_importance_task_t temp_task_imp;

	/* 
	 * iterate the downstream kmsgs, adjust their boosts, 
	 * and capture the next task to adjust for each message
	 */

	ipc_kmsg_t temp_kmsg;

	queue_iterate(&task_imp->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) {
		mach_msg_header_t *hdr = temp_kmsg->ikm_header;
		mach_port_delta_t delta;
		ipc_port_t port;

		/* toggle the kmsg importance bit as a barrier to parallel adjusts */
		if (IIT_UPDATE_HOLD == type) {
			if (MACH_MSGH_BITS_RAISED_IMPORTANCE(hdr->msgh_bits)) {
				continue;
			}

			/* mark the message as now carrying importance */
			hdr->msgh_bits |= MACH_MSGH_BITS_RAISEIMP;
			delta = 1;
		} else {
			if (!MACH_MSGH_BITS_RAISED_IMPORTANCE(hdr->msgh_bits)) {
				continue;
			}

			/* clear the message as now carrying importance */
			hdr->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
			delta = -1;
		}

		/* determine the task importance to adjust as result (if any) */
		port = (ipc_port_t) hdr->msgh_remote_port;
		assert(IP_VALID(port));
		ip_lock(port);
		temp_task_imp = IIT_NULL;
		if (!ipc_port_importance_delta_internal(port, &delta, &temp_task_imp)) {
			ip_unlock(port);
		}

		/* no task importance to adjust associated with the port? */
		if (IIT_NULL == temp_task_imp) {
			continue;
		}

		/* hold a reference on temp_task_imp */

		/* Adjust the task assertions and determine if an edge was crossed */
		if (ipc_importance_task_check_transition(temp_task_imp, type, 1)) {
			incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition);
			queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props);
			/* reference donated */
		} else {
			ipc_importance_task_release_internal(temp_task_imp);
		}
	}

	/*
	 * iterate the downstream importance inherits
	 * and capture the next task importance to boost for each
	 */
	ipc_importance_inherit_t temp_inherit;

	queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) {
		uint32_t assertcnt = III_EXTERN(temp_inherit); 

		temp_task_imp = temp_inherit->iii_to_task;
		assert(IIT_NULL != temp_task_imp);

		if (IIT_UPDATE_HOLD == type) {
			/* if no undropped externcnts in the inherit, nothing to do */
			if (0 == assertcnt) {
				assert(temp_inherit->iii_donating == FALSE);
				continue;
			}

			/* nothing to do if the inherit is already donating (forced donation) */
			if (temp_inherit->iii_donating) {
				continue;
			}

			/* mark it donating and contribute to the task externcnts */
			temp_inherit->iii_donating = TRUE;
			temp_task_imp->iit_externcnt += temp_inherit->iii_externcnt;
			temp_task_imp->iit_externdrop += temp_inherit->iii_externdrop;

		} else {
			/* if no contributing assertions, move on */
			if (0 == assertcnt) {
				assert(temp_inherit->iii_donating == FALSE);
				continue;
			}

			/* nothing to do if the inherit is not donating */
			if (!temp_inherit->iii_donating) { 
				continue;
			}

			/* mark it no longer donating */
			temp_inherit->iii_donating = FALSE;

			/* remove the contribution the inherit made to the to-task */
			assert(IIT_EXTERN(temp_task_imp) >= III_EXTERN(temp_inherit));
			assert(temp_task_imp->iit_externcnt >= temp_inherit->iii_externcnt);
			assert(temp_task_imp->iit_externdrop >= temp_inherit->iii_externdrop);
			temp_task_imp->iit_externcnt -= temp_inherit->iii_externcnt;
			temp_task_imp->iit_externdrop -= temp_inherit->iii_externdrop;

		}

		/* Adjust the task assertions and determine if an edge was crossed */
		assert(ipc_importance_task_is_any_receiver_type(temp_task_imp));
		if (ipc_importance_task_check_transition(temp_task_imp, type, assertcnt)) {
			ipc_importance_task_reference(temp_task_imp);
			incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition);
			queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props);
		} 
	}
}

/*
 *	Routine:	ipc_importance_task_process_updates
 *	Purpose:
 *	        Process the queue of task importances and apply the policy
 *		update called for.  Only process tasks in the queue with an
 *		update timestamp less than the supplied max.
 *	Conditions:
 *		Called and returns with importance locked.
 *		May drop importance lock and block temporarily.
 */
static void
ipc_importance_task_process_updates(
	queue_t   supplied_queue,
	boolean_t boost,
	uint64_t  max_timestamp)
{
	ipc_importance_task_t task_imp;
	queue_head_t second_chance;
	queue_t queue = supplied_queue;

	/*
	 * This queue will hold the task's we couldn't trylock on first pass.
	 * By using a second (private) queue, we guarantee all tasks that get
	 * entered on this queue have a timestamp under the maximum.
	 */
	queue_init(&second_chance);

	/* process any resulting policy updates */
 retry:
	while(!queue_empty(queue)) {
		task_t target_task;
		struct task_pend_token pend_token = {};

		task_imp = (ipc_importance_task_t)queue_first(queue);
		assert(0 == task_imp->iit_updatepolicy);
		assert(queue == task_imp->iit_updateq);

		/* if timestamp is too big, we're done */
		if (task_imp->iit_updatetime > max_timestamp) {
			break;
		}

		/* we were given a reference on each task in the queue */

		/* remove it from the supplied queue */
		queue_remove(queue, task_imp, ipc_importance_task_t, iit_updates);
		task_imp->iit_updateq = NULL;

		target_task = task_imp->iit_task;

		/* Is it well on the way to exiting? */
		if (TASK_NULL == target_task) {
			ipc_importance_task_release_locked(task_imp);
			/* importance unlocked */
			ipc_importance_lock();
			continue;
		}

		/* Has the update been reversed on the hysteresis queue? */
		if (0 < task_imp->iit_assertcnt &&
		    queue == &ipc_importance_delayed_drop_queue) {
			ipc_importance_task_release_locked(task_imp);
			/* importance unlocked */
			ipc_importance_lock();
			continue;
		}

		/* 
		 * Can we get the task lock out-of-order? 
		 * If not, stick this back on the second-chance queue.
		 */
		if (!task_lock_try(target_task)) {
			boolean_t should_wait_lock = (queue == &second_chance);
			task_imp->iit_updateq = &second_chance;
		
			/*
			 * If we're already processing second-chances on
			 * tasks, keep this task on the front of the queue.
			 * We will wait for the task lock before coming
			 * back and trying again, and we have a better
			 * chance of re-acquiring the lock if we come back
			 * to it right away.
			 */
			if (should_wait_lock){
				task_reference(target_task);
				queue_enter_first(&second_chance, task_imp,
						  ipc_importance_task_t, iit_updates);
			} else {
				queue_enter(&second_chance, task_imp, 
					    ipc_importance_task_t, iit_updates);
			}
			ipc_importance_unlock();
			
			if (should_wait_lock) {
				task_lock(target_task);
				task_unlock(target_task);
				task_deallocate(target_task);
			}

			ipc_importance_lock();
			continue;
		}

		/* is it going away? */
		if (!target_task->active) {
			task_unlock(target_task);
			ipc_importance_task_release_locked(task_imp);
			/* importance unlocked */
			ipc_importance_lock();
			continue;
		}

		/* take a task reference for while we don't have the importance lock */
		task_reference(target_task);

		/* count the transition */
		if (boost)
			task_imp->iit_transitions++;

		ipc_importance_unlock();

		/* apply the policy adjust to the target task (while it is still locked) */
		task_update_boost_locked(target_task, boost, &pend_token);

		/* complete the policy update with the task unlocked */
		ipc_importance_task_release(task_imp);
		task_unlock(target_task);
		task_policy_update_complete_unlocked(target_task, THREAD_NULL, &pend_token);
		task_deallocate(target_task);

		ipc_importance_lock();
	}

	/* If there are tasks we couldn't update the first time, try again */
	if (!queue_empty(&second_chance)) {
		queue = &second_chance;
		goto retry;
	}
}


/*
 *	Routine:	ipc_importance_task_delayed_drop_scan
 *	Purpose:
 *	        The thread call routine to scan the delayed drop queue,
 *		requesting all updates with a deadline up to the last target
 *		for the thread-call (which is DENAP_DROP_SKEW beyond the first
 *		thread's optimum delay).
 *		update to drop its boost.
 *	Conditions:
 *		Nothing locked
 */
static void
ipc_importance_task_delayed_drop_scan(
	__unused void *arg1,
	__unused void *arg2) 
{
	ipc_importance_lock();

	/* process all queued task drops with timestamps up to TARGET(first)+SKEW */
	ipc_importance_task_process_updates(&ipc_importance_delayed_drop_queue, 
					    FALSE,
					    ipc_importance_delayed_drop_timestamp);

	/* importance lock may have been temporarily dropped */

	/* If there are any entries left in the queue, re-arm the call here */
	if (!queue_empty(&ipc_importance_delayed_drop_queue)) {
		ipc_importance_task_t task_imp;
		uint64_t deadline;
		uint64_t leeway;

		task_imp = (ipc_importance_task_t)queue_first(&ipc_importance_delayed_drop_queue);

		nanoseconds_to_absolutetime(DENAP_DROP_DELAY, &deadline);
		deadline += task_imp->iit_updatetime;
		ipc_importance_delayed_drop_timestamp = deadline;

		nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY, &leeway);

		thread_call_enter_delayed_with_leeway(
					ipc_importance_delayed_drop_call,
					NULL,
					deadline,
					leeway,
					DENAP_DROP_FLAGS);
	} else {
		ipc_importance_delayed_drop_call_requested = FALSE;
	}
	ipc_importance_unlock();
}

/*
 *	Routine:	ipc_importance_task_delayed_drop
 *	Purpose:
 *		Queue the specified task importance for delayed policy
 *		update to drop its boost.
 *	Conditions:
 *		Called with the importance lock held.
 */
static void
ipc_importance_task_delayed_drop(ipc_importance_task_t task_imp)
{
	uint64_t timestamp = mach_absolute_time(); /* no mach_approximate_time() in kernel */

	assert(ipc_importance_delayed_drop_call != NULL);

	/*
	 * If still on an update queue from a previous change,
	 * remove it first (and use that reference).  Otherwise, take
	 * a new reference for the delay drop update queue.
	 */
	if (NULL != task_imp->iit_updateq) {
		queue_remove(task_imp->iit_updateq, task_imp, 
			     ipc_importance_task_t, iit_updates);
	} else {
		ipc_importance_task_reference_internal(task_imp);
	}

	task_imp->iit_updateq = &ipc_importance_delayed_drop_queue;
	task_imp->iit_updatetime = timestamp;

	queue_enter(&ipc_importance_delayed_drop_queue, task_imp, 
		    ipc_importance_task_t, iit_updates);

	/* request the delayed thread-call if not already requested */
	if (!ipc_importance_delayed_drop_call_requested) {
		uint64_t deadline;
		uint64_t leeway;

		nanoseconds_to_absolutetime(DENAP_DROP_DELAY, &deadline);
		deadline += task_imp->iit_updatetime;
		ipc_importance_delayed_drop_timestamp = deadline;

		nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY, &leeway);

		ipc_importance_delayed_drop_call_requested = TRUE;
		thread_call_enter_delayed_with_leeway(
					ipc_importance_delayed_drop_call,
					NULL,
					deadline,
					leeway,
					DENAP_DROP_FLAGS);
	}
}


/*
 *	Routine:	ipc_importance_task_propagate_assertion_locked
 *	Purpose:
 *		Propagate the importance transition type to every item
 *		If this causes a boost to be applied, determine if that
 *		boost should propagate downstream.
 *	Conditions:
 *		Called with the importance lock held.
 */
static void
ipc_importance_task_propagate_assertion_locked(
	ipc_importance_task_t task_imp,
	iit_update_type_t type,
	boolean_t update_task_imp)
{
	boolean_t boost = (IIT_UPDATE_HOLD == type);
	ipc_importance_task_t temp_task_imp;
	queue_head_t propagate;
	queue_head_t updates;

	queue_init(&updates);
	queue_init(&propagate);

	/*
	 * If we're going to update the policy for the provided task,
	 * enqueue it on the propagate queue itself.  Otherwise, only
	 * enqueue downstream things.
	 */
	if (update_task_imp) {
		queue_enter(&propagate, task_imp, ipc_importance_task_t, iit_props);
	} else {
		ipc_importance_task_propagate_helper(task_imp, type, &propagate);
	}

	/*
	 * for each item on the propagation list, propagate any change downstream,
	 * adding new tasks to propagate further if they transistioned as well.
	 */
	while (!queue_empty(&propagate)) {
		boolean_t need_update;

		queue_remove_first(&propagate, temp_task_imp, ipc_importance_task_t, iit_props);
		assert(IIT_NULL != temp_task_imp);

		/* only propagate for receivers not already marked as a donor */
		if (!ipc_importance_task_is_marked_donor(temp_task_imp) &&
		    ipc_importance_task_is_marked_receiver(temp_task_imp)) {
			ipc_importance_task_propagate_helper(temp_task_imp, type, &propagate);
		}

		/* if we have a policy update to apply, enqueue a reference for later processing */
		need_update = (0 != temp_task_imp->iit_updatepolicy);
		temp_task_imp->iit_updatepolicy = 0;
		if (need_update && TASK_NULL != temp_task_imp->iit_task) {
			if (NULL == temp_task_imp->iit_updateq) {
				temp_task_imp->iit_updatetime = 0;
				temp_task_imp->iit_updateq = &updates;
				ipc_importance_task_reference_internal(temp_task_imp);
				if (boost) {
					queue_enter(&updates, temp_task_imp,
						    ipc_importance_task_t, iit_updates);
				} else {
					queue_enter_first(&updates, temp_task_imp,
							  ipc_importance_task_t, iit_updates);
				}
			} else {
				/* Must already be on the AppNap hysteresis queue */
				assert(&ipc_importance_delayed_drop_queue);
				assert(ipc_importance_task_is_marked_denap_receiver(temp_task_imp));
			}	
		}
	}

	/* apply updates to task (may drop importance lock) */
	if (!queue_empty(&updates)) {
		ipc_importance_task_process_updates(&updates, boost, 0);
	}
}

/*
 *	Routine:	ipc_importance_task_hold_internal_assertion_locked
 *	Purpose:
 *		Increment the assertion count on the task importance.
 *		If this results in a boost state change in that task,
 *		prepare to update task policy for this task AND, if
 *		if not just waking out of App Nap, all down-stream
 *		tasks that have a similar transition through inheriting
 *		this update.
 *	Conditions:
 *		importance locked on entry and exit.
 *		May temporarily drop importance lock and block.
 */
static kern_return_t
ipc_importance_task_hold_internal_assertion_locked(ipc_importance_task_t task_imp, uint32_t count)
{
	if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_HOLD, count)) {
		ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_HOLD, TRUE);
	}
	return KERN_SUCCESS;
}

/*
 *	Routine:	ipc_importance_task_drop_internal_assertion_locked
 *	Purpose:
 *		Decrement the assertion count on the task importance.
 *		If this results in a boost state change in that task,
 *		prepare to update task policy for this task AND, if
 *		if not just waking out of App Nap, all down-stream
 *		tasks that have a similar transition through inheriting
 *		this update.
 *	Conditions:
 *		importance locked on entry and exit.
 *		May temporarily drop importance lock and block.
 */
static kern_return_t
ipc_importance_task_drop_internal_assertion_locked(ipc_importance_task_t task_imp, uint32_t count)
{
	if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_DROP, count)) {
		ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, TRUE);
	}
	return KERN_SUCCESS;
}

/*
 *      Routine:        ipc_importance_task_hold_internal_assertion
 *      Purpose:
 *              Increment the assertion count on the task importance.
 *              If this results in a 0->1 change in that count,
 *              prepare to update task policy for this task AND
 *              (potentially) all down-stream tasks that have a
 *		similar transition through inheriting this update.
 *      Conditions:
 *              Nothing locked
 *              May block after dropping importance lock.
 */
int
ipc_importance_task_hold_internal_assertion(ipc_importance_task_t task_imp, uint32_t count)
{
	int ret = KERN_SUCCESS;

	if (ipc_importance_task_is_any_receiver_type(task_imp)) {
		ipc_importance_lock();
		ret = ipc_importance_task_hold_internal_assertion_locked(task_imp, count);
		ipc_importance_unlock();
	}
	return ret;
}

/*
 *	Routine:	ipc_importance_task_drop_internal_assertion
 *	Purpose:
 *		Decrement the assertion count on the task importance.
 *		If this results in a X->0 change in that count,
 *		prepare to update task policy for this task AND
 *		all down-stream tasks that have a similar transition
 *		through inheriting this drop update.
 *	Conditions:
 *		Nothing locked on entry.
 *		May block after dropping importance lock.
 */
kern_return_t
ipc_importance_task_drop_internal_assertion(ipc_importance_task_t task_imp, uint32_t count)
{
	kern_return_t ret = KERN_SUCCESS;

	if (ipc_importance_task_is_any_receiver_type(task_imp)) {
		ipc_importance_lock();
		ret = ipc_importance_task_drop_internal_assertion_locked(task_imp, count);
		ipc_importance_unlock();
	}
	return ret;
}

/*
 *      Routine:        ipc_importance_task_hold_file_lock_assertion
 *      Purpose:
 *              Increment the file lock assertion count on the task importance.
 *              If this results in a 0->1 change in that count,
 *              prepare to update task policy for this task AND
 *              (potentially) all down-stream tasks that have a
 *		similar transition through inheriting this update.
 *      Conditions:
 *              Nothing locked
 *              May block after dropping importance lock.
 */
kern_return_t
ipc_importance_task_hold_file_lock_assertion(ipc_importance_task_t task_imp, uint32_t count)
{
	kern_return_t ret = KERN_SUCCESS;

	if (ipc_importance_task_is_any_receiver_type(task_imp)) {
		ipc_importance_lock();
		ret = ipc_importance_task_hold_internal_assertion_locked(task_imp, count);
		if (KERN_SUCCESS == ret) {
			task_imp->iit_filelocks += count;
		}
		ipc_importance_unlock();
	}
	return ret;
}

/*
 *	Routine:	ipc_importance_task_drop_file_lock_assertion
 *	Purpose:
 *		Decrement the assertion count on the task importance.
 *		If this results in a X->0 change in that count,
 *		prepare to update task policy for this task AND
 *		all down-stream tasks that have a similar transition
 *		through inheriting this drop update.
 *	Conditions:
 *		Nothing locked on entry.
 *		May block after dropping importance lock.
 */
kern_return_t
ipc_importance_task_drop_file_lock_assertion(ipc_importance_task_t task_imp, uint32_t count)
{
	kern_return_t ret = KERN_SUCCESS;

	if (ipc_importance_task_is_any_receiver_type(task_imp)) {
		ipc_importance_lock();
		if (count <= task_imp->iit_filelocks) {
			task_imp->iit_filelocks -= count;
			ret = ipc_importance_task_drop_internal_assertion_locked(task_imp, count);
		} else {
			ret = KERN_INVALID_ARGUMENT;
		}
		ipc_importance_unlock();
	}
	return ret;
}

/*
 *	Routine:	ipc_importance_task_hold_legacy_external_assertion
 *	Purpose:
 *		Increment the external assertion count on the task importance.
 *		This cannot result in an 0->1 transition, as the caller must
 *		already hold an external boost.
 *	Conditions:
 *		Nothing locked on entry.
 *		May block after dropping importance lock.
 *		A queue of task importance structures is returned
 *		by ipc_importance_task_hold_assertion_locked(). Each
 *		needs to be updated (outside the importance lock hold).
 */
kern_return_t
ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_imp, uint32_t count)
{
	task_t target_task;
	uint32_t target_assertcnt;
	uint32_t target_externcnt;
	uint32_t target_legacycnt;

	kern_return_t ret;

	ipc_importance_lock();
	target_task = task_imp->iit_task;

#if IMPORTANCE_DEBUG
	int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1;

	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START,
		proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
#endif

	if (IIT_LEGACY_EXTERN(task_imp) == 0) {
		/* Only allowed to take a new boost assertion when holding an external boost */
		/* save data for diagnostic printf below */
		target_assertcnt = task_imp->iit_assertcnt;
		target_externcnt = IIT_EXTERN(task_imp);
		target_legacycnt = IIT_LEGACY_EXTERN(task_imp);
		ret = KERN_FAILURE;
		count = 0;
	} else {
		assert(ipc_importance_task_is_any_receiver_type(task_imp));
		assert(0 < task_imp->iit_assertcnt);
		assert(0 < IIT_EXTERN(task_imp));
		task_imp->iit_assertcnt += count;
		task_imp->iit_externcnt += count;
		task_imp->iit_legacy_externcnt += count;
		ret = KERN_SUCCESS;
	}
	ipc_importance_unlock();

#if IMPORTANCE_DEBUG
	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END,
				  proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
        // This covers the legacy case where a task takes an extra boost.
	DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, proc_selfpid(), int, count, int, task_imp->iit_assertcnt);
#endif

	if (KERN_FAILURE == ret && target_task != TASK_NULL) {
		printf("BUG in process %s[%d]: "
		       "attempt to acquire an additional legacy external boost assertion without holding an existing legacy external assertion. "
		       "(%d total, %d external, %d legacy-external)\n",
		       proc_name_address(target_task->bsd_info), audit_token_pid_from_task(target_task),
		       target_assertcnt, target_externcnt, target_legacycnt);
	}

	return(ret);
}

/*
 *	Routine:	ipc_importance_task_drop_legacy_external_assertion
 *	Purpose:
 *		Drop the legacy external assertion count on the task and
 *		reflect that change to total external assertion count and
 *		then onto the internal importance count.
 *
 *		If this results in a X->0 change in the internal,
 *		count, prepare to update task policy for this task AND
 *		all down-stream tasks that have a similar transition
 *		through inheriting this update.
 *	Conditions:
 *		Nothing locked on entry.
 */
kern_return_t
ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_imp, uint32_t count)
{
	int ret = KERN_SUCCESS;
	task_t target_task;
	uint32_t target_assertcnt;
	uint32_t target_externcnt;
	uint32_t target_legacycnt;

	if (count > 1) {
		return KERN_INVALID_ARGUMENT;
	}
		
	ipc_importance_lock();
	target_task = task_imp->iit_task;

#if IMPORTANCE_DEBUG
	int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1;

	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START,
		proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
#endif

	if (count > IIT_LEGACY_EXTERN(task_imp)) {
		/* Process over-released its boost count - save data for diagnostic printf */
		/* TODO: If count > 1, we should clear out as many external assertions as there are left. */
		target_assertcnt = task_imp->iit_assertcnt;
		target_externcnt = IIT_EXTERN(task_imp);
		target_legacycnt = IIT_LEGACY_EXTERN(task_imp);
		ret = KERN_FAILURE;
	} else {
		/* 
		 * decrement legacy external count from the top level and reflect
		 * into internal for this and all subsequent updates.
		 */
		assert(ipc_importance_task_is_any_receiver_type(task_imp));
		assert(IIT_EXTERN(task_imp) >= count);

		task_imp->iit_legacy_externdrop += count;
		task_imp->iit_externdrop += count;

		/* reset extern counters (if appropriate) */
		if (IIT_LEGACY_EXTERN(task_imp) == 0) {
			if (IIT_EXTERN(task_imp) != 0) {
				task_imp->iit_externcnt -= task_imp->iit_legacy_externcnt;
				task_imp->iit_externdrop -= task_imp->iit_legacy_externdrop;
			} else {
				task_imp->iit_externcnt = 0;
				task_imp->iit_externdrop = 0;
			}
			task_imp->iit_legacy_externcnt = 0;
			task_imp->iit_legacy_externdrop = 0;
		}
		
		/* reflect the drop to the internal assertion count (and effect any importance change) */
		if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_DROP, count)) {
			ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, TRUE);
		}
		ret = KERN_SUCCESS;
	}

#if IMPORTANCE_DEBUG
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END,
					  proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
#endif

	ipc_importance_unlock();

	/* delayed printf for user-supplied data failures */
	if (KERN_FAILURE == ret && TASK_NULL != target_task) {
		printf("BUG in process %s[%d]: over-released legacy external boost assertions (%d total, %d external, %d legacy-external)\n",
		       proc_name_address(target_task->bsd_info), audit_token_pid_from_task(target_task),
		       target_assertcnt, target_externcnt, target_legacycnt);
	}

	return(ret);
}



/* Transfer an assertion to legacy userspace responsibility */
static kern_return_t
ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, uint32_t count, __unused int sender_pid)
{
	task_t target_task;

	assert(IIT_NULL != task_imp);
	target_task = task_imp->iit_task;

	if (TASK_NULL == target_task ||
	    !ipc_importance_task_is_any_receiver_type(task_imp)) {
		return KERN_FAILURE;
	}

#if IMPORTANCE_DEBUG
	int target_pid = audit_token_pid_from_task(target_task);

	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_START,
	        proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0);
#endif

	ipc_importance_lock();
	/* assert(task_imp->iit_assertcnt >= IIT_EXTERN(task_imp) + count); */
	assert(IIT_EXTERN(task_imp) >= IIT_LEGACY_EXTERN(task_imp));
	task_imp->iit_legacy_externcnt += count;
	task_imp->iit_externcnt += count;
	ipc_importance_unlock();

#if IMPORTANCE_DEBUG
	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_END,
				  proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0);
    // This is the legacy boosting path
	DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, sender_pid, int, count, int, IIT_LEGACY_EXTERN(task_imp));
#endif /* IMPORTANCE_DEBUG */

	return(KERN_SUCCESS);
}

/*
 *	Routine:	ipc_importance_task_update_live_donor
 *	Purpose:
 *		Read the live donor status and update the live_donor bit/propagate the change in importance.
 *	Conditions:
 *		Nothing locked on entrance, nothing locked on exit.
 *
 *		TODO: Need tracepoints around this function...
 */
void
ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp)
{
	uint32_t task_live_donor;
	boolean_t before_donor;
	boolean_t after_donor;
	task_t target_task;
  
	assert(task_imp != NULL);

	/*
	 * Nothing to do if the task is not marked as expecting
	 * live donor updates.
	 */
	if (!ipc_importance_task_is_marked_live_donor(task_imp)) {
		return;
	}

	ipc_importance_lock();

	/* If the task got disconnected on the way here, no use (or ability) adjusting live donor status */
	target_task = task_imp->iit_task;
	if (TASK_NULL == target_task) {
		ipc_importance_unlock();
		return;
	}
	before_donor = ipc_importance_task_is_marked_donor(task_imp);

	/* snapshot task live donor status - may change, but another call will accompany the change */
	task_live_donor = target_task->effective_policy.t_live_donor;

#if IMPORTANCE_DEBUG
	int target_pid = audit_token_pid_from_task(target_task);

	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	                          (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_START,
	                          target_pid, task_imp->iit_donor, task_live_donor, before_donor, 0);
#endif	                          

	/* update the task importance live donor status based on the task's value */
	task_imp->iit_donor = task_live_donor;

	after_donor = ipc_importance_task_is_marked_donor(task_imp);

	/* Has the effectiveness of being a donor changed as a result of this update? */
	if (before_donor != after_donor) {
		iit_update_type_t type;

		/* propagate assertions without updating the current task policy (already handled) */
		if (0 == before_donor) {
			task_imp->iit_transitions++;
			type = IIT_UPDATE_HOLD;
		} else {
			type = IIT_UPDATE_DROP;
		}
		ipc_importance_task_propagate_assertion_locked(task_imp, type, FALSE);
	}

#if IMPORTANCE_DEBUG
	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	                          (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_END,
	                          target_pid, task_imp->iit_donor, task_live_donor, after_donor, 0);
#endif

	ipc_importance_unlock();
}


/*
 *	Routine:	ipc_importance_task_mark_donor
 *	Purpose:
 *		Set the task importance donor flag.
 *	Conditions:
 *		Nothing locked on entrance, nothing locked on exit.
 *
 *		This is only called while the task is being constructed,
 *		so no need to update task policy or propagate downstream.
 */
void
ipc_importance_task_mark_donor(ipc_importance_task_t task_imp, boolean_t donating)
{
	assert(task_imp != NULL);

	ipc_importance_lock();

	int old_donor = task_imp->iit_donor;

	task_imp->iit_donor = (donating ? 1 : 0);

	if (task_imp->iit_donor > 0 && old_donor == 0)
		task_imp->iit_transitions++;

	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	                          (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_INIT_DONOR_STATE)) | DBG_FUNC_NONE,
	                          audit_token_pid_from_task(task_imp->iit_task), donating,
	                          old_donor, task_imp->iit_donor, 0);
	
	ipc_importance_unlock();
}

/*
 *	Routine:	ipc_importance_task_marked_donor
 *	Purpose:
 *		Query the donor flag for the given task importance.
 *	Conditions:
 *		May be called without taking the importance lock.
 *		In that case, donor status can change so you must
 *		check only once for each donation event.
 */
boolean_t
ipc_importance_task_is_marked_donor(ipc_importance_task_t task_imp)
{
	if (IIT_NULL == task_imp) {
  		return FALSE;
	}
	return (0 != task_imp->iit_donor);
}

/*
 *	Routine:	ipc_importance_task_mark_live_donor
 *	Purpose:
 *		Indicate that the task is eligible for live donor updates.
 *	Conditions:
 *		Nothing locked on entrance, nothing locked on exit.
 *
 *		This is only called while the task is being constructed.
 */
void
ipc_importance_task_mark_live_donor(ipc_importance_task_t task_imp, boolean_t live_donating)
{
	assert(task_imp != NULL);

	ipc_importance_lock();
	task_imp->iit_live_donor = (live_donating ? 1 : 0);
	ipc_importance_unlock();
}

/*
 *	Routine:	ipc_importance_task_marked_live_donor
 *	Purpose:
 *		Query the live donor and donor flags for the given task importance.
 *	Conditions:
 *		May be called without taking the importance lock.
 *		In that case, donor status can change so you must
 *		check only once for each donation event.
 */
boolean_t
ipc_importance_task_is_marked_live_donor(ipc_importance_task_t task_imp)
{
	if (IIT_NULL == task_imp) {
  		return FALSE;
	}
	return (0 != task_imp->iit_live_donor);
}

/*
 *	Routine:	ipc_importance_task_is_donor
 *	Purpose:
 *		Query the full donor status for the given task importance.
 *	Conditions:
 *		May be called without taking the importance lock.
 *		In that case, donor status can change so you must
 *		check only once for each donation event.
 */
boolean_t
ipc_importance_task_is_donor(ipc_importance_task_t task_imp)
{
	if (IIT_NULL == task_imp) {
  		return FALSE;
	}
	return (ipc_importance_task_is_marked_donor(task_imp) ||
		(ipc_importance_task_is_marked_receiver(task_imp) &&
		 task_imp->iit_assertcnt > 0));
}

/*
 *	Routine:	ipc_importance_task_is_never_donor
 *	Purpose:
 *		Query if a given task can ever donate importance.
 *	Conditions:
 *		May be called without taking the importance lock.
 *		Condition is permanent for a give task.		
 */
boolean_t
ipc_importance_task_is_never_donor(ipc_importance_task_t task_imp)
{
	if (IIT_NULL == task_imp) {
  		return FALSE;
	}
	return (!ipc_importance_task_is_marked_donor(task_imp) &&
		!ipc_importance_task_is_marked_live_donor(task_imp) &&
		!ipc_importance_task_is_marked_receiver(task_imp));
}

/*
 *	Routine:	ipc_importance_task_mark_receiver
 *	Purpose:
 *		Update the task importance receiver flag.
 *	Conditions:
 *		Nothing locked on entrance, nothing locked on exit.
 *		This can only be invoked before the task is discoverable,
 *		so no worries about atomicity(?)
 */
void
ipc_importance_task_mark_receiver(ipc_importance_task_t task_imp, boolean_t receiving)
{
	assert(task_imp != NULL);	

	ipc_importance_lock();
	if (receiving) {
		assert(task_imp->iit_assertcnt == 0);
		assert(task_imp->iit_externcnt == 0);
		assert(task_imp->iit_externdrop == 0);
		assert(task_imp->iit_denap == 0);
		task_imp->iit_receiver = 1;  /* task can receive importance boost */
	} else if (task_imp->iit_receiver) {
		assert(task_imp->iit_denap == 0);
		if (task_imp->iit_assertcnt != 0 || IIT_EXTERN(task_imp) != 0) {
			panic("disabling imp_receiver on task with pending importance boosts!");
		}
		task_imp->iit_receiver = 0;
	}
	ipc_importance_unlock();
}


/*
 *	Routine:	ipc_importance_task_marked_receiver
 *	Purpose:
 *		Query the receiver flag for the given task importance.
 *	Conditions:
 *		May be called without taking the importance lock as
 *		the importance flag can never change after task init.
 */
boolean_t
ipc_importance_task_is_marked_receiver(ipc_importance_task_t task_imp)
{
	return (IIT_NULL != task_imp && 0 != task_imp->iit_receiver);
}


/*
 *	Routine:	ipc_importance_task_mark_denap_receiver
 *	Purpose:
 *		Update the task importance de-nap receiver flag.
 *	Conditions:
 *		Nothing locked on entrance, nothing locked on exit.
 *		This can only be invoked before the task is discoverable,
 *		so no worries about atomicity(?)
 */
void
ipc_importance_task_mark_denap_receiver(ipc_importance_task_t task_imp, boolean_t denap)
{
	assert(task_imp != NULL);	

	ipc_importance_lock();
	if (denap) {
		assert(task_imp->iit_assertcnt == 0);
		assert(task_imp->iit_externcnt == 0);
		assert(task_imp->iit_receiver == 0);
		task_imp->iit_denap = 1;  /* task can receive de-nap boost */
	} else if (task_imp->iit_denap) {
		assert(task_imp->iit_receiver == 0);
		if (0 < task_imp->iit_assertcnt || 0 < IIT_EXTERN(task_imp)) {
			panic("disabling de-nap on task with pending de-nap boosts!");
		}
		task_imp->iit_denap = 0;
	}
	ipc_importance_unlock();
}


/*
 *	Routine:	ipc_importance_task_marked_denap_receiver
 *	Purpose:
 *		Query the de-nap receiver flag for the given task importance.
 *	Conditions:
 *		May be called without taking the importance lock as
 *		the de-nap flag can never change after task init.
 */
boolean_t
ipc_importance_task_is_marked_denap_receiver(ipc_importance_task_t task_imp)
{
	return (IIT_NULL != task_imp && 0 != task_imp->iit_denap);
}

/*
 *	Routine:	ipc_importance_task_is_denap_receiver
 *	Purpose:
 *		Query the full de-nap receiver status for the given task importance.
 *		For now, that is simply whether the receiver flag is set.
 *	Conditions:
 *		May be called without taking the importance lock as
 *		the de-nap receiver flag can never change after task init.
 */
boolean_t
ipc_importance_task_is_denap_receiver(ipc_importance_task_t task_imp)
{
	return (ipc_importance_task_is_marked_denap_receiver(task_imp));
}

/*
 *	Routine:	ipc_importance_task_is_any_receiver_type
 *	Purpose:
 *		Query if the task is marked to receive boosts - either
 *		importance or denap.
 *	Conditions:
 *		May be called without taking the importance lock as both
 *		the importance and de-nap receiver flags can never change
 *		after task init.
 */
boolean_t
ipc_importance_task_is_any_receiver_type(ipc_importance_task_t task_imp)
{
	return (ipc_importance_task_is_marked_receiver(task_imp) ||
		ipc_importance_task_is_marked_denap_receiver(task_imp));
}

#if 0 /* currently unused */

/*
 *	Routine:	ipc_importance_inherit_reference
 *	Purpose:
 *		Add a reference to the inherit importance element.
 *	Conditions:
 *		Caller most hold a reference on the inherit element.
 */
static inline void
ipc_importance_inherit_reference(ipc_importance_inherit_t inherit)
{
	ipc_importance_reference(&inherit->iii_elem);
}
#endif /* currently unused */

/*
 *	Routine:	ipc_importance_inherit_release_locked
 *	Purpose:
 *		Release a reference on an inherit importance attribute value,
 *		unlinking and deallocating the attribute if the last reference.
 *	Conditions:
 *		Entered with importance lock held, leaves with it unlocked.
 */
static inline void
ipc_importance_inherit_release_locked(ipc_importance_inherit_t inherit)
{
	ipc_importance_release_locked(&inherit->iii_elem);
}

#if 0 /* currently unused */
/*
 *	Routine:	ipc_importance_inherit_release
 *	Purpose:
 *		Release a reference on an inherit importance attribute value,
 *		unlinking and deallocating the attribute if the last reference.
 *	Conditions:
 *		nothing locked on entrance, nothing locked on exit.
 *		May block.
 */
void
ipc_importance_inherit_release(ipc_importance_inherit_t inherit)
{
	if (III_NULL != inherit)
		ipc_importance_release(&inherit->iii_elem);
}
#endif /* 0 currently unused */
	
/*
 *	Routine:	ipc_importance_for_task
 *	Purpose:
 *		Create a reference for the specified task's base importance
 *		element.  If the base importance element doesn't exist, make it and
 *		bind it to the active task.  If the task is inactive, there isn't
 *		any need to return a new reference.
 *	Conditions:
 *		If made is true, a "made" reference is returned (for donating to
 *		the voucher system).  Otherwise	an internal reference is returned.
 *
 *		Nothing locked on entry.  May block.
 */
ipc_importance_task_t
ipc_importance_for_task(task_t task, boolean_t made)
{
	ipc_importance_task_t task_elem;
	boolean_t first_pass = TRUE;

	assert(TASK_NULL != task);

 retry:
	/* No use returning anything for inactive task */
	if (!task->active)
		return IIT_NULL;

	ipc_importance_lock();
	task_elem = task->task_imp_base;
	if (IIT_NULL != task_elem) {
		/* Add a made reference (borrowing active task ref to do it) */
		if (made) {
			if (0 == task_elem->iit_made++) {
				assert(IIT_REFS_MAX > IIT_REFS(task_elem));
				ipc_importance_task_reference_internal(task_elem);
			}
		} else {
			assert(IIT_REFS_MAX > IIT_REFS(task_elem));
			ipc_importance_task_reference_internal(task_elem);
		}
		ipc_importance_unlock();
		return task_elem;
	}
	ipc_importance_unlock();

	if (!first_pass)
		return IIT_NULL;
	first_pass = FALSE;

	/* Need to make one - may race with others (be prepared to drop) */
	task_elem = (ipc_importance_task_t)zalloc(ipc_importance_task_zone);
	if (IIT_NULL == task_elem)
		goto retry;

	task_elem->iit_bits = IIE_TYPE_TASK | 2; /* one for task, one for return/made */
	task_elem->iit_made = (made) ? 1 : 0;
	task_elem->iit_task = task; /* take actual ref when we're sure */
	task_elem->iit_updateq = NULL;
	task_elem->iit_receiver = 0;
	task_elem->iit_denap = 0;
	task_elem->iit_donor = 0;
	task_elem->iit_live_donor = 0;
	task_elem->iit_updatepolicy = 0;
	task_elem->iit_reserved = 0;
	task_elem->iit_filelocks = 0;
	task_elem->iit_updatetime = 0;
	task_elem->iit_transitions = 0;
	task_elem->iit_assertcnt = 0;
	task_elem->iit_externcnt = 0;
	task_elem->iit_externdrop = 0;
	task_elem->iit_legacy_externcnt = 0;
	task_elem->iit_legacy_externdrop = 0;
#if IIE_REF_DEBUG
	ipc_importance_counter_init(&task_elem->iit_elem);
#endif
	queue_init(&task_elem->iit_kmsgs);
	queue_init(&task_elem->iit_inherits);

	ipc_importance_lock();
	if (!task->active) {
		ipc_importance_unlock();
		zfree(ipc_importance_task_zone, task_elem);
		return IIT_NULL;
	}

	/* did we lose the race? */
	if (IIT_NULL != task->task_imp_base) {
		ipc_importance_unlock();
		zfree(ipc_importance_task_zone, task_elem);
		goto retry;
	}

	/* we won the race */
	task->task_imp_base = task_elem;
	task_reference(task);
#if DEVELOPMENT || DEBUG
	queue_enter(&global_iit_alloc_queue, task_elem, ipc_importance_task_t, iit_allocation);
	task_importance_update_owner_info(task);
#endif
	ipc_importance_unlock();

	return task_elem;
}

#if DEVELOPMENT || DEBUG
void task_importance_update_owner_info(task_t task) {

	if (task != TASK_NULL && task->task_imp_base != IIT_NULL) {
		ipc_importance_task_t task_elem = task->task_imp_base;

		task_elem->iit_bsd_pid = audit_token_pid_from_task(task);
		if (task->bsd_info) {
			strncpy(&task_elem->iit_procname[0], proc_name_address(task->bsd_info), 16);
			task_elem->iit_procname[16] = '\0';
		} else {
			strncpy(&task_elem->iit_procname[0], "unknown", 16);
		}
	}
}
#endif

/*
 *	Routine:	ipc_importance_reset_locked
 *	Purpose:
 *		Reset a task's IPC importance (the task is going away or exec'ing)
 *
 *		Remove the donor bit and legacy externalized assertions from the
 *		current task importance and see if that wipes out downstream donations.
 *	Conditions:
 *		importance lock held.
 */

static void
ipc_importance_reset_locked(ipc_importance_task_t task_imp, boolean_t donor)
{
	boolean_t before_donor, after_donor;

	/* remove the donor bit, live-donor bit and externalized boosts */
	before_donor = ipc_importance_task_is_donor(task_imp);
	if (donor) {
		task_imp->iit_donor = 0;
	}
	assert(IIT_LEGACY_EXTERN(task_imp) <= IIT_EXTERN(task_imp));
	assert(task_imp->iit_legacy_externcnt <= task_imp->iit_externcnt);
	assert(task_imp->iit_legacy_externdrop <= task_imp->iit_externdrop);
	task_imp->iit_externcnt -= task_imp->iit_legacy_externcnt;
	task_imp->iit_externdrop -= task_imp->iit_legacy_externdrop;

	/* assert(IIT_LEGACY_EXTERN(task_imp) <= task_imp->iit_assertcnt); */
	if (IIT_LEGACY_EXTERN(task_imp) < task_imp->iit_assertcnt) {
		task_imp->iit_assertcnt -= IIT_LEGACY_EXTERN(task_imp);
	} else {
		assert(IIT_LEGACY_EXTERN(task_imp) == task_imp->iit_assertcnt);
		task_imp->iit_assertcnt = 0;
	}
	task_imp->iit_legacy_externcnt = 0;
	task_imp->iit_legacy_externdrop = 0;
	after_donor = ipc_importance_task_is_donor(task_imp);

#if DEVELOPMENT || DEBUG
	if (task_imp->iit_assertcnt > 0 && task_imp->iit_live_donor) {
		printf("Live donor task %s[%d] still has %d importance assertions after reset\n", 
		       task_imp->iit_procname, task_imp->iit_bsd_pid, task_imp->iit_assertcnt);
	}
#endif

	/* propagate a downstream drop if there was a change in donor status */
	if (after_donor != before_donor) {
		ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, FALSE);
	}
}

/*
 *	Routine:	ipc_importance_reset
 *	Purpose:
 *		Reset a task's IPC importance
 *
 *		The task is being reset, although staying around. Arrange to have the
 *		external state of the task reset from the importance.
 *	Conditions:
 *		importance lock not held.
 */

void
ipc_importance_reset(ipc_importance_task_t task_imp, boolean_t donor)
{
	if (IIT_NULL == task_imp) {
		return;
	}
	ipc_importance_lock();
	ipc_importance_reset_locked(task_imp, donor);
	ipc_importance_unlock();
}

/*
 *	Routine:	ipc_importance_disconnect_task
 *	Purpose:
 *		Disconnect a task from its importance.
 *
 *		Clear the task pointer from the importance and drop the
 *		reference the task held on the importance object.  Before
 *		doing that, reset the effects the current task holds on
 *		the importance and see if that wipes out downstream donations.
 *
 *		We allow the upstream boosts to continue to affect downstream
 *		even though the local task is being effectively pulled from
 *		the chain.
 *	Conditions:
 *		Nothing locked.
 */
void
ipc_importance_disconnect_task(task_t task)
{
	ipc_importance_task_t task_imp;

	task_lock(task);
	ipc_importance_lock();
	task_imp = task->task_imp_base;

	/* did somebody beat us to it? */
	if (IIT_NULL == task_imp) {
		ipc_importance_unlock();
		task_unlock(task);
		return;
	}

	/* disconnect the task from this importance */
	assert(task_imp->iit_task == task);
	task_imp->iit_task = TASK_NULL;
	task->task_imp_base = IIT_NULL;
	task_unlock(task);
	
	/* reset the effects the current task hold on the importance */
	ipc_importance_reset_locked(task_imp, TRUE);

	ipc_importance_task_release_locked(task_imp);
	/* importance unlocked */

	/* deallocate the task now that the importance is unlocked */
	task_deallocate(task);
}

/*
 *	Routine:	ipc_importance_send
 *	Purpose:
 *		Post the importance voucher attribute [if sent] or a static
 *		importance boost depending upon options and conditions.
 *	Conditions:
 *		Destination port locked on entry and exit, may be dropped during the call.
 *	Returns:
 *		A boolean identifying if the port lock was tempoarily dropped.
 */
boolean_t
ipc_importance_send(
	ipc_kmsg_t		kmsg,
	mach_msg_option_t	option)
{
	ipc_port_t port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
	boolean_t port_lock_dropped = FALSE;
	ipc_importance_elem_t elem;
	task_t task;
	ipc_importance_task_t task_imp;
	kern_return_t kr;


	assert(IP_VALID(port));

	/* If no donation to be made, return quickly */
	if ((port->ip_impdonation == 0) ||
	    (option & MACH_SEND_NOIMPORTANCE) != 0) {
		return port_lock_dropped;
	}

	task = current_task();

	/* If forced sending a static boost, go update the port */
	if ((option & MACH_SEND_IMPORTANCE) != 0) {
		kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP;
		goto portupdate;
	}

	task_imp = task->task_imp_base;
	assert(IIT_NULL != task_imp);

	/* If the sender can never donate importance, nothing to do */
	if (ipc_importance_task_is_never_donor(task_imp)) {
		return port_lock_dropped;
	}

	elem = IIE_NULL;

	/* If importance receiver and passing a voucher, look for importance in there */
	if (IP_VALID(kmsg->ikm_voucher) &&
	    ipc_importance_task_is_marked_receiver(task_imp)) {
		mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED];
		mach_voucher_attr_value_handle_array_size_t val_count;
		ipc_voucher_t voucher;
			
		assert(ip_kotype(kmsg->ikm_voucher) == IKOT_VOUCHER);
		voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;

		/* check to see if the voucher has an importance attribute */
		val_count = MACH_VOUCHER_ATTR_VALUE_MAX_NESTED;
		kr = mach_voucher_attr_control_get_values(ipc_importance_control, voucher,
							  vals, &val_count);
		assert(KERN_SUCCESS == kr);

		/*
		 * Only use importance associated with our task (either directly
		 * or through an inherit that donates to our task).
		 */
		if (0 < val_count) {
			ipc_importance_elem_t check_elem;

			check_elem = (ipc_importance_elem_t)vals[0];
			assert(IIE_NULL != check_elem);
			if (IIE_TYPE_INHERIT == IIE_TYPE(check_elem)) {
				ipc_importance_inherit_t inherit;
				inherit = (ipc_importance_inherit_t) check_elem;
				if (inherit->iii_to_task == task_imp) {
					elem = check_elem;
				}
			} else if (check_elem == (ipc_importance_elem_t)task_imp) {
				elem = check_elem;
			}
		}
	}

	/* If we haven't found an importance attribute to send yet, use the task's */
	if (IIE_NULL == elem) {
		elem = (ipc_importance_elem_t)task_imp;
	}

	/* take a reference for the message to hold */
	ipc_importance_reference_internal(elem);

	/* acquire the importance lock while trying to hang on to port lock */
	if (!ipc_importance_lock_try()) {
		port_lock_dropped = TRUE;
		ip_unlock(port);
		ipc_importance_lock();
	}

	/* link kmsg onto the donor element propagation chain */
	ipc_importance_kmsg_link(kmsg, elem);
	/* elem reference transfered to kmsg */

	incr_ref_counter(elem->iie_kmsg_refs_added);

	/* If the sender isn't currently a donor, no need to apply boost */
	if (!ipc_importance_task_is_donor(task_imp)) {
		ipc_importance_unlock();

		/* re-acquire port lock, if needed */
		if (TRUE == port_lock_dropped)
			ip_lock(port);

		return port_lock_dropped;
	}

	/* Mark the fact that we are (currently) donating through this message */
	kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP;

	/*
	 * If we need to relock the port, do it with the importance still locked.
	 * This assures we get to add the importance boost through the port to
	 * the task BEFORE anyone else can attempt to undo that operation because
	 * the sender lost donor status.
	 */
	if (TRUE == port_lock_dropped) {
		ip_lock(port);
	}
	ipc_importance_unlock();

 portupdate:
				
#if IMPORTANCE_DEBUG
	if (kdebug_enable) {
		mach_msg_max_trailer_t *dbgtrailer = (mach_msg_max_trailer_t *)
		        	((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size));
		unsigned int sender_pid = dbgtrailer->msgh_audit.val[5];
		mach_msg_id_t imp_msgh_id = kmsg->ikm_header->msgh_id;
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_START,
		                           audit_token_pid_from_task(task), sender_pid, imp_msgh_id, 0, 0);
	}
#endif /* IMPORTANCE_DEBUG */

	/* adjust port boost count (with port locked) */
	if (TRUE == ipc_port_importance_delta(port, 1)) {
		port_lock_dropped = TRUE;
		ip_lock(port);
	}
	return port_lock_dropped;
}
	
/*
 *	Routine:	ipc_importance_inherit_from
 *	Purpose:
 *		Create a "made" reference for an importance attribute representing
 *		an inheritance between the sender of a message (if linked) and the
 *		current task importance.  If the message is not linked, a static
 *		boost may be created, based on the boost state of the message.
 *
 *		Any transfer from kmsg linkage to inherit linkage must be atomic.
 *
 *		If the task is inactive, there isn't any need to return a new reference.
 *	Conditions:
 *		Nothing locked on entry.  May block.
 */
static ipc_importance_inherit_t
ipc_importance_inherit_from(ipc_kmsg_t kmsg)
{
	ipc_importance_task_t	task_imp = IIT_NULL;
	ipc_importance_elem_t 	from_elem = kmsg->ikm_importance;
	ipc_importance_elem_t 	elem;
	task_t	task_self = current_task();

	ipc_port_t port = kmsg->ikm_header->msgh_remote_port;
	ipc_importance_inherit_t inherit = III_NULL;
	ipc_importance_inherit_t alloc = III_NULL;
	ipc_importance_inherit_t temp_inherit;
	boolean_t cleared_self_donation = FALSE;
	boolean_t donating;
	uint32_t depth = 1;

	/* The kmsg must have an importance donor or static boost to proceed */
	if (IIE_NULL == kmsg->ikm_importance &&
	    !MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
		return III_NULL;
	}

	/* 
	 * No need to set up an inherit linkage if the dest isn't a receiver
	 * of one type or the other.
	 */
	if (!ipc_importance_task_is_any_receiver_type(task_self->task_imp_base)) {
		ipc_importance_lock();
		goto out_locked;
	}

	/* Grab a reference on the importance of the destination */
	task_imp = ipc_importance_for_task(task_self, FALSE);

	ipc_importance_lock();

	if (IIT_NULL == task_imp) {
		goto out_locked;
	}

	incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_inherit_from);

	/* If message is already associated with an inherit... */
	if (IIE_TYPE_INHERIT == IIE_TYPE(from_elem)) {
		ipc_importance_inherit_t from_inherit = (ipc_importance_inherit_t)from_elem;

		/* already targeting our task? - just use it */
		if (from_inherit->iii_to_task == task_imp) {
			/* clear self-donation if not also present in inherit */
			if (!from_inherit->iii_donating &&
			    MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
				kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
				cleared_self_donation = TRUE;
			}
			inherit = from_inherit;
				
		} else if (III_DEPTH_MAX == III_DEPTH(from_inherit)) {
			ipc_importance_task_t to_task;
			ipc_importance_elem_t unlinked_from;

			/*
			 * Chain too long. Switch to looking
			 * directly at the from_inherit's to-task
			 * as our source of importance. 
			 */
			to_task = from_inherit->iii_to_task;
			ipc_importance_task_reference(to_task);
			from_elem = (ipc_importance_elem_t)to_task;
			depth = III_DEPTH_RESET | 1;

			/* Fixup the kmsg linkage to reflect change */
			unlinked_from = ipc_importance_kmsg_unlink(kmsg);
			assert(unlinked_from == (ipc_importance_elem_t)from_inherit);
			ipc_importance_kmsg_link(kmsg, from_elem);
			ipc_importance_inherit_release_locked(from_inherit);
			/* importance unlocked */
			ipc_importance_lock();

		} else {
			/* inheriting from an inherit */
			depth = from_inherit->iii_depth + 1;
		}
	} 

	/* 
	 * Don't allow a task to inherit from itself (would keep it permanently
	 * boosted even if all other donors to the task went away).
	 */

	if (from_elem == (ipc_importance_elem_t)task_imp) {
		goto out_locked;
	}

	/* 
	 * But if the message isn't associated with any linked source, it is
	 * intended to be permanently boosting (static boost from kernel).
	 * In that case DO let the process permanently boost itself.
	 */
	if (IIE_NULL == from_elem) {
		assert(MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits));
		ipc_importance_task_reference_internal(task_imp);
		from_elem = (ipc_importance_elem_t)task_imp;
	}
	
	/* 
	 * Now that we have the from_elem figured out,
	 * check to see if we already have an inherit for this pairing
	 */
	while (III_NULL == inherit) {
		queue_iterate(&from_elem->iie_inherits, temp_inherit,
			      ipc_importance_inherit_t, iii_inheritance) {
			if (temp_inherit->iii_to_task == task_imp &&
			    temp_inherit->iii_depth == depth) {
				inherit = temp_inherit;
				break;
			}
		}

		/* Do we have to allocate a new inherit */
		if (III_NULL == inherit) {
			if (III_NULL != alloc) {
				break;
			}

			/* allocate space */
			ipc_importance_unlock();
			alloc = (ipc_importance_inherit_t)
				zalloc(ipc_importance_inherit_zone);
			ipc_importance_lock();
		}
	}

	/* snapshot the donating status while we have importance locked */
	donating = MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits);

	if (III_NULL != inherit) {
		/* We found one, piggyback on that */
		assert(0 < III_REFS(inherit));
		assert(0 < IIE_REFS(inherit->iii_from_elem));
		assert(inherit->iii_externcnt >= inherit->iii_made);

		/* add in a made reference */
		if (0 == inherit->iii_made++) {
			assert(III_REFS_MAX > III_REFS(inherit));
			ipc_importance_inherit_reference_internal(inherit);
		}

		/* Reflect the inherit's change of status into the task boosts */
		if (0 == III_EXTERN(inherit)) {
			assert(!inherit->iii_donating);
			inherit->iii_donating = donating;
			if (donating) {
				task_imp->iit_externcnt += inherit->iii_externcnt;
				task_imp->iit_externdrop += inherit->iii_externdrop;
			}
		} else {
			assert(donating == inherit->iii_donating);
		}

		/* add in a external reference for this use of the inherit */
		inherit->iii_externcnt++;
		if (donating) {
			task_imp->iit_externcnt++;
		}
	} else {
		/* initialize the previously allocated space */
		inherit = alloc;
		inherit->iii_bits = IIE_TYPE_INHERIT | 1;
		inherit->iii_made = 1;
		inherit->iii_externcnt = 1;
		inherit->iii_externdrop = 0;
		inherit->iii_depth = depth;
		inherit->iii_to_task = task_imp;
		inherit->iii_from_elem = IIE_NULL;
		queue_init(&inherit->iii_kmsgs);
		queue_init(&inherit->iii_inherits);

		/* If donating, reflect that in the task externcnt */
		if (donating) {
			inherit->iii_donating = TRUE;
			task_imp->iit_externcnt++;
		} else {
			inherit->iii_donating = FALSE;
		}

		/*
		 * Chain our new inherit on the element it inherits from.
		 * The new inherit takes our reference on from_elem.
		 */
		ipc_importance_inherit_link(inherit, from_elem);

#if IIE_REF_DEBUG
		ipc_importance_counter_init(&inherit->iii_elem);
		from_elem->iie_kmsg_refs_inherited++;
		task_imp->iit_elem.iie_task_refs_inherited++;
#endif
	}

 out_locked:
	/* 
	 * for those paths that came straight here: snapshot the donating status
	 * (this should match previous snapshot for other paths).
	 */
	donating = MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits);

	/* unlink the kmsg inheritance (if any) */
	elem = ipc_importance_kmsg_unlink(kmsg);
	assert(elem == from_elem);

	/* If we didn't create a new inherit, we have some resources to release */
	if (III_NULL == inherit || inherit != alloc) {
		if (IIE_NULL != from_elem) {
			if (III_NULL != inherit) {
				incr_ref_counter(from_elem->iie_kmsg_refs_coalesced);
			} else {
				incr_ref_counter(from_elem->iie_kmsg_refs_dropped);
			}
			ipc_importance_release_locked(from_elem);
			/* importance unlocked */
		} else {
			ipc_importance_unlock();
		}

		if (IIT_NULL != task_imp) {
			if (III_NULL != inherit) {
				incr_ref_counter(task_imp->iit_elem.iie_task_refs_coalesced);
			}
			ipc_importance_task_release(task_imp);
		}

		if (III_NULL != alloc)
			zfree(ipc_importance_inherit_zone, alloc);
	} else {
		/* from_elem and task_imp references transferred to new inherit */
		ipc_importance_unlock();
	}

	/* decrement port boost count */ 
	if (donating) {
		ip_lock(port);
		if (III_NULL != inherit) {
			/* task assertions transferred to inherit, just adjust port count */
			ipc_port_impcount_delta(port, -1, IP_NULL);
			ip_unlock(port);
		}  else	{
			/* drop importance from port and destination task */
			if (ipc_port_importance_delta(port, -1) == FALSE) {
				ip_unlock(port);
			}
		}
	} else if (cleared_self_donation) {
		ip_lock(port);
		/* drop cleared donation from port and destination task */
		if (ipc_port_importance_delta(port, -1) == FALSE) {
			ip_unlock(port);
		}
	}

	if (III_NULL != inherit) {
		/* have an associated importance attr, even if currently not donating */
		kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP;
	} else {
		/* we won't have an importance attribute associated with our message */
		kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
	}

	return inherit;
}

/*
 *	Routine:	ipc_importance_receive
 *	Purpose:
 *		Process importance attributes in a received message.
 *
 *		If an importance voucher attribute was sent, transform
 *		that into an attribute value reflecting the inheritance
 *		from the sender to the receiver.
 *
 *		If a static boost is received (or the voucher isn't on
 *		a voucher-based boost), export a static boost.
 *	Conditions:
 *		Nothing locked.
 */
void
ipc_importance_receive(
	ipc_kmsg_t 		kmsg,
	mach_msg_option_t	option)
{
	unsigned int sender_pid = ((mach_msg_max_trailer_t *)
				   ((vm_offset_t)kmsg->ikm_header + 
				    round_msg(kmsg->ikm_header->msgh_size)))->msgh_audit.val[5];
	task_t task_self = current_task();
	int impresult = -1;
	
	/* convert to a voucher with an inherit importance attribute? */
	if ((option & MACH_RCV_VOUCHER) != 0) {
		uint8_t recipes[2 * sizeof(ipc_voucher_attr_recipe_data_t) + 
				sizeof(mach_voucher_attr_value_handle_t)];
		ipc_voucher_attr_raw_recipe_array_size_t recipe_size = 0;
		ipc_voucher_attr_recipe_t recipe = (ipc_voucher_attr_recipe_t)recipes;
			ipc_voucher_t recv_voucher;
		mach_voucher_attr_value_handle_t handle;
		ipc_importance_inherit_t inherit;
		kern_return_t kr;

		/* set up recipe to copy the old voucher */
		if (IP_VALID(kmsg->ikm_voucher)) {
			ipc_voucher_t sent_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;

			recipe->key = MACH_VOUCHER_ATTR_KEY_ALL;
			recipe->command = MACH_VOUCHER_ATTR_COPY;
			recipe->previous_voucher = sent_voucher;
			recipe->content_size = 0;
			recipe_size += sizeof(*recipe);
		}

		/*
		 * create an inheritance attribute from the kmsg (may be NULL)
		 * transferring any boosts from the kmsg linkage through the
		 * port directly to the new inheritance object.
		 */
		inherit = ipc_importance_inherit_from(kmsg);
		handle = (mach_voucher_attr_value_handle_t)inherit;

		assert(IIE_NULL == kmsg->ikm_importance);

		/* replace the importance attribute with the handle we created */
		/*  our made reference on the inhert is donated to the voucher */
		recipe = (ipc_voucher_attr_recipe_t)&recipes[recipe_size];
		recipe->key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE;
		recipe->command = MACH_VOUCHER_ATTR_SET_VALUE_HANDLE;
		recipe->previous_voucher = IPC_VOUCHER_NULL;
		recipe->content_size = sizeof(mach_voucher_attr_value_handle_t);
		*(mach_voucher_attr_value_handle_t *)(void *)recipe->content = handle;
		recipe_size += sizeof(*recipe) + sizeof(mach_voucher_attr_value_handle_t);

		kr = ipc_voucher_attr_control_create_mach_voucher(ipc_importance_control,
								  recipes,
								  recipe_size,
								  &recv_voucher);
		assert(KERN_SUCCESS == kr);

		/* swap the voucher port (and set voucher bits in case it didn't already exist) */
		kmsg->ikm_header->msgh_bits |= (MACH_MSG_TYPE_MOVE_SEND << 16);
		ipc_port_release_send(kmsg->ikm_voucher);
		kmsg->ikm_voucher = convert_voucher_to_port(recv_voucher);
		if (III_NULL != inherit)
			impresult = 2;

	} else { /* Don't want a voucher */

		/* got linked importance? have to drop */
		if (IIE_NULL != kmsg->ikm_importance) {
			ipc_importance_elem_t elem;

			ipc_importance_lock();
			elem = ipc_importance_kmsg_unlink(kmsg);
#if IIE_REF_DEBUG
			elem->iie_kmsg_refs_dropped++;
#endif
			ipc_importance_release_locked(elem);
			/* importance unlocked */
		}

		/* With kmsg unlinked, can safely examine message importance attribute. */
		if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
			ipc_importance_task_t task_imp = task_self->task_imp_base;
			ipc_port_t port = kmsg->ikm_header->msgh_remote_port;

			/* defensive deduction for release builds lacking the assert */
			ip_lock(port);
			ipc_port_impcount_delta(port, -1, IP_NULL);
			ip_unlock(port);

			/* will user accept legacy responsibility for the importance boost */
			if (KERN_SUCCESS == ipc_importance_task_externalize_legacy_assertion(task_imp, 1, sender_pid)) {
				impresult = 1;
			} else {
				/* The importance boost never applied to task (clear the bit) */
				kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
				impresult = 0;
			}
		}
	}

#if IMPORTANCE_DEBUG
	if (-1 < impresult)
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_DELV)) | DBG_FUNC_NONE,
				sender_pid, audit_token_pid_from_task(task_self),
				kmsg->ikm_header->msgh_id, impresult, 0);
	if (impresult == 2){
		/*
		 * This probe only covers new voucher-based path.  Legacy importance
		 * will trigger the probe in ipc_importance_task_externalize_assertion() 
		 * above and have impresult==1 here.
		 */
		DTRACE_BOOST5(receive_boost, task_t, task_self, int, audit_token_pid_from_task(task_self), int, sender_pid, int, 1, int, task_self->task_imp_base->iit_assertcnt);
    }
#endif /* IMPORTANCE_DEBUG */
}

/*
 *	Routine:	ipc_importance_unreceive
 *	Purpose:
 *		Undo receive of importance attributes in a message.
 *
 *	Conditions:
 *		Nothing locked.
 */
void
ipc_importance_unreceive(
	ipc_kmsg_t 		kmsg,
	mach_msg_option_t	__unused option)
{
	/* importance should already be in the voucher and out of the kmsg */
	assert(IIE_NULL == kmsg->ikm_importance);

	/* See if there is a legacy boost to be dropped from receiver */
	if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
		ipc_importance_task_t task_imp;

		kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
		task_imp = current_task()->task_imp_base;
		if (!IP_VALID(kmsg->ikm_voucher) && IIT_NULL != task_imp) {
			ipc_importance_task_drop_legacy_external_assertion(task_imp, 1);
		}
		/*
		 * ipc_kmsg_copyout_dest() will consume the voucher
		 * and any contained importance.
		 */
	}
}

/*
 *	Routine:	ipc_importance_clean
 *	Purpose:
 *		Clean up importance state in a kmsg that is being cleaned.
 *		Unlink the importance chain if one was set up, and drop
 *		the reference this kmsg held on the donor.  Then check to
 *		if importance was carried to the port, and remove that if
 *		needed.
 *	Conditions:
 *		Nothing locked.
 */
void
ipc_importance_clean(
	ipc_kmsg_t		kmsg)
{
	ipc_port_t		port;

	/* Is the kmsg still linked? If so, remove that first */
	if (IIE_NULL != kmsg->ikm_importance) {
		ipc_importance_elem_t	elem;

		ipc_importance_lock();
		elem = ipc_importance_kmsg_unlink(kmsg);
		assert(IIE_NULL != elem);
		ipc_importance_release_locked(elem);
		/* importance unlocked */
	}

	/* See if there is a legacy importance boost to be dropped from port */
	if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) {
		kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP;
		port = kmsg->ikm_header->msgh_remote_port;
		if (IP_VALID(port)) {
			ip_lock(port);
			/* inactive ports already had their importance boosts dropped */
			if (!ip_active(port) || 
			    ipc_port_importance_delta(port, -1) == FALSE) {
				ip_unlock(port);
			}
		}
	}
}

void
ipc_importance_assert_clean(__assert_only ipc_kmsg_t kmsg)
{
  assert(IIE_NULL == kmsg->ikm_importance);
  assert(!MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits));
}

/*
 * IPC Importance Attribute Manager definition
 */

static kern_return_t
ipc_importance_release_value(
	ipc_voucher_attr_manager_t		manager,
	mach_voucher_attr_key_t			key,
	mach_voucher_attr_value_handle_t	value,
	mach_voucher_attr_value_reference_t	sync);

static kern_return_t
ipc_importance_get_value(
	ipc_voucher_attr_manager_t			manager,
	mach_voucher_attr_key_t				key,
	mach_voucher_attr_recipe_command_t		command,
	mach_voucher_attr_value_handle_array_t		prev_values,
	mach_voucher_attr_value_handle_array_size_t	prev_value_count,
	mach_voucher_attr_content_t			content,
	mach_voucher_attr_content_size_t		content_size,
	mach_voucher_attr_value_handle_t		*out_value,
	ipc_voucher_t					*out_value_voucher);

static kern_return_t
ipc_importance_extract_content(
	ipc_voucher_attr_manager_t			manager,
	mach_voucher_attr_key_t				key,
	mach_voucher_attr_value_handle_array_t		values,
	mach_voucher_attr_value_handle_array_size_t	value_count,
	mach_voucher_attr_recipe_command_t		*out_command,
	mach_voucher_attr_content_t			out_content,
	mach_voucher_attr_content_size_t		*in_out_content_size);

static kern_return_t
ipc_importance_command(
	ipc_voucher_attr_manager_t			manager,
	mach_voucher_attr_key_t				key,
	mach_voucher_attr_value_handle_array_t		values,
	mach_msg_type_number_t				value_count,
	mach_voucher_attr_command_t			command,
	mach_voucher_attr_content_t			in_content,
	mach_voucher_attr_content_size_t		in_content_size,
	mach_voucher_attr_content_t			out_content,
	mach_voucher_attr_content_size_t		*out_content_size);

static void
ipc_importance_manager_release(
	ipc_voucher_attr_manager_t		manager);

struct ipc_voucher_attr_manager ipc_importance_manager = {
	.ivam_release_value =	ipc_importance_release_value,
	.ivam_get_value =	ipc_importance_get_value,
	.ivam_extract_content =	ipc_importance_extract_content,
	.ivam_command = 	ipc_importance_command,
	.ivam_release =		ipc_importance_manager_release,
};

#define IMPORTANCE_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_IMPORTANCE == (key))
#define IMPORTANCE_ASSERT_MANAGER(manager) assert(&ipc_importance_manager == (manager))

/*
 *	Routine:	ipc_importance_release_value [Voucher Attribute Manager Interface]
 *	Purpose:
 *		Release what the voucher system believes is the last "made" reference
 *		on an importance attribute value handle.  The sync parameter is used to
 *		avoid races with new made references concurrently being returned to the
 *		voucher system in other threads.
 *	Conditions:
 *		Nothing locked on entry.  May block.
 */
static kern_return_t
ipc_importance_release_value(
	ipc_voucher_attr_manager_t		__assert_only manager,
	mach_voucher_attr_key_t			__assert_only key,
	mach_voucher_attr_value_handle_t	value,
	mach_voucher_attr_value_reference_t	sync)
{
	ipc_importance_elem_t elem;

	IMPORTANCE_ASSERT_MANAGER(manager);
	IMPORTANCE_ASSERT_KEY(key);
	assert(0 < sync);

	elem = (ipc_importance_elem_t)value;

	ipc_importance_lock();

	/* Any oustanding made refs? */
	if (sync != elem->iie_made) {
		assert(sync < elem->iie_made);
		ipc_importance_unlock();
		return KERN_FAILURE;
	}

	/* clear made */ 
	elem->iie_made = 0;

	/* 
	 * If there are pending external boosts represented by this attribute, 
	 * drop them from the apropriate task 
	 */
	if (IIE_TYPE_INHERIT == IIE_TYPE(elem)) {
		ipc_importance_inherit_t inherit = (ipc_importance_inherit_t)elem;

		assert(inherit->iii_externcnt >= inherit->iii_externdrop);

		if (inherit->iii_donating) {
			ipc_importance_task_t imp_task = inherit->iii_to_task;
			uint32_t assertcnt = III_EXTERN(inherit);

			assert(ipc_importance_task_is_any_receiver_type(imp_task));
			assert(imp_task->iit_externcnt >= inherit->iii_externcnt);
			assert(imp_task->iit_externdrop >= inherit->iii_externdrop);
			imp_task->iit_externcnt -= inherit->iii_externcnt;
			imp_task->iit_externdrop -= inherit->iii_externdrop;
			inherit->iii_externcnt = 0;
			inherit->iii_externdrop = 0;
			inherit->iii_donating = FALSE;

			/* adjust the internal assertions - and propagate if needed */
			if (ipc_importance_task_check_transition(imp_task, IIT_UPDATE_DROP, assertcnt)) {
				ipc_importance_task_propagate_assertion_locked(imp_task, IIT_UPDATE_DROP, TRUE);
			}
		} else {
			inherit->iii_externcnt = 0;
			inherit->iii_externdrop = 0;
		}
	} 

	/* drop the made reference on elem */
	ipc_importance_release_locked(elem);
	/* returns unlocked */

	return KERN_SUCCESS;
}


/*
 *	Routine:	ipc_importance_get_value [Voucher Attribute Manager Interface]
 *	Purpose:
 *		Convert command and content data into a reference on a [potentially new]
 *		attribute value.  The importance attribute manager will only allow the
 *		caller to get a value for the current task's importance, or to redeem
 *		an importance attribute from an existing voucher.
 *	Conditions:
 *		Nothing locked on entry.  May block.
 */
static kern_return_t
ipc_importance_get_value(
	ipc_voucher_attr_manager_t			__assert_only manager,
	mach_voucher_attr_key_t				__assert_only key,
	mach_voucher_attr_recipe_command_t		command,
	mach_voucher_attr_value_handle_array_t		prev_values,
	mach_voucher_attr_value_handle_array_size_t	prev_value_count,
	mach_voucher_attr_content_t			__unused content,
	mach_voucher_attr_content_size_t		content_size,
	mach_voucher_attr_value_handle_t		*out_value,
	ipc_voucher_t					*out_value_voucher)
{
	ipc_importance_elem_t elem;
	task_t self;

	IMPORTANCE_ASSERT_MANAGER(manager);
	IMPORTANCE_ASSERT_KEY(key);

	if (0 != content_size)
		return KERN_INVALID_ARGUMENT;

	/* never an out voucher */

	switch (command) {

	case MACH_VOUCHER_ATTR_REDEEM:

		/* redeem of previous values is the value */
		if (0 < prev_value_count) {
			elem = (ipc_importance_elem_t)prev_values[0];
			assert(IIE_NULL != elem);

			ipc_importance_lock();
			assert(0 < elem->iie_made);
			elem->iie_made++;
			ipc_importance_unlock();

			*out_value = prev_values[0];
			return KERN_SUCCESS;
		}

		/* redeem of default is default */
		*out_value = 0;
		*out_value_voucher = IPC_VOUCHER_NULL;
		return KERN_SUCCESS;

	case MACH_VOUCHER_ATTR_IMPORTANCE_SELF:
		self = current_task();

		elem = (ipc_importance_elem_t)ipc_importance_for_task(self, TRUE);
		/* made reference added (or IIE_NULL which isn't referenced) */

		*out_value = (mach_voucher_attr_value_handle_t)elem;
		*out_value_voucher = IPC_VOUCHER_NULL;
		return KERN_SUCCESS;
		
	default:
		/* 
		 * every other command is unknown
		 *
		 * Specifically, there is no mechanism provided to construct an
		 * importance attribute for a task/process from just a pid or
		 * task port.  It has to be copied (or redeemed) from a previous
		 * voucher that has it.
		 */
		return KERN_INVALID_ARGUMENT;
	}
}

/*
 *	Routine:	ipc_importance_extract_content [Voucher Attribute Manager Interface]
 *	Purpose:
 *		Extract meaning from the attribute value present in a voucher.  While
 *		the real goal is to provide commands and data that can reproduce the
 *		voucher's value "out of thin air", this isn't possible with importance
 *		attribute values.  Instead, return debug info to help track down dependencies.
 *	Conditions:
 *		Nothing locked on entry.  May block.
 */
static kern_return_t
ipc_importance_extract_content(
	ipc_voucher_attr_manager_t			__assert_only manager,
	mach_voucher_attr_key_t				__assert_only key,
	mach_voucher_attr_value_handle_array_t		values,
	mach_voucher_attr_value_handle_array_size_t	value_count,
	mach_voucher_attr_recipe_command_t		*out_command,
	mach_voucher_attr_content_t			out_content,
	mach_voucher_attr_content_size_t		*in_out_content_size)
{
	mach_voucher_attr_content_size_t size = 0;
	ipc_importance_elem_t elem;
	unsigned int i;

	IMPORTANCE_ASSERT_MANAGER(manager);
	IMPORTANCE_ASSERT_KEY(key);

	/* the first non-default value provides the data */
	for (i = 0; i < value_count ; i++) {
		elem = (ipc_importance_elem_t)values[i];
		if (IIE_NULL == elem)
			continue;

		snprintf((char *)out_content, *in_out_content_size, "Importance for pid ");
		size = (mach_voucher_attr_content_size_t)strlen((char *)out_content);

                for(;;) {
			ipc_importance_inherit_t inherit = III_NULL;
			ipc_importance_task_t task_imp;
			task_t task;
			int task_pid;

			if (IIE_TYPE_TASK == IIE_TYPE(elem)) {
				task_imp = (ipc_importance_task_t)elem;
				task = task_imp->iit_task;
				task_pid = (TASK_NULL != task) ?
				           audit_token_pid_from_task(task) : -1;
				snprintf((char *)out_content + size, *in_out_content_size - size, "%d", task_pid);
			} else {
				inherit = (ipc_importance_inherit_t)elem;
				task_imp = inherit->iii_to_task;
				task = task_imp->iit_task;
				task_pid = (TASK_NULL != task) ?
				           audit_token_pid_from_task(task) : -1;
				snprintf((char *)out_content + size, *in_out_content_size - size, 
					 "%d (%d of %d boosts) %s from pid ", task_pid,
					 III_EXTERN(inherit), inherit->iii_externcnt, 
					 (inherit->iii_donating) ? "donated" : "linked");
			}
			
			size = (mach_voucher_attr_content_size_t)strlen((char *)out_content);

			if (III_NULL == inherit)
				break;

			elem = inherit->iii_from_elem;
                }
                size++; /* account for NULL */
	}
	*out_command = MACH_VOUCHER_ATTR_NOOP; /* cannot be used to regenerate value */
	*in_out_content_size = size;
	return KERN_SUCCESS;
}

/*
 *	Routine:	ipc_importance_command [Voucher Attribute Manager Interface]
 *	Purpose:
 *		Run commands against the importance attribute value found in a voucher.
 *		No such commands are currently supported.
 *	Conditions:
 *		Nothing locked on entry.  May block.
 */
static kern_return_t
ipc_importance_command(
	ipc_voucher_attr_manager_t		__assert_only manager,
	mach_voucher_attr_key_t			__assert_only key,
	mach_voucher_attr_value_handle_array_t	values,
	mach_msg_type_number_t			value_count,
	mach_voucher_attr_command_t		command,
	mach_voucher_attr_content_t		in_content,
	mach_voucher_attr_content_size_t	in_content_size,
	mach_voucher_attr_content_t		out_content,
	mach_voucher_attr_content_size_t	*out_content_size)
{
	ipc_importance_inherit_t inherit;
	ipc_importance_task_t to_task;
	uint32_t refs, *outrefsp;
	mach_msg_type_number_t i;
	uint32_t externcnt;

	IMPORTANCE_ASSERT_MANAGER(manager);
	IMPORTANCE_ASSERT_KEY(key);

	if (in_content_size != sizeof(refs) ||
	    (*out_content_size != 0 && *out_content_size != sizeof(refs))) {
		return KERN_INVALID_ARGUMENT;
	}
	refs = *(uint32_t *)(void *)in_content;
	outrefsp = (*out_content_size != 0) ? (uint32_t *)(void *)out_content : NULL;

	if (MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL != command) {
		return KERN_NOT_SUPPORTED;
	}

	/* the first non-default value of the apropos type provides the data */
	inherit = III_NULL;
	for (i = 0; i < value_count; i++) {
		ipc_importance_elem_t elem = (ipc_importance_elem_t)values[i];

		if (IIE_NULL != elem && IIE_TYPE_INHERIT == IIE_TYPE(elem)) {
			inherit = (ipc_importance_inherit_t)elem;
			break;
		}
	}
	if (III_NULL == inherit) {
		return KERN_INVALID_ARGUMENT;
	}
		
	ipc_importance_lock();

	if (0 == refs) {
		if (NULL != outrefsp) {
			*outrefsp = III_EXTERN(inherit);
		}
		ipc_importance_unlock();
		return KERN_SUCCESS;
	}

	/* Enough external references left to drop? */
	if (III_EXTERN(inherit) < refs) {
		ipc_importance_unlock();
		return KERN_FAILURE;
	}

	to_task = inherit->iii_to_task;
	assert(ipc_importance_task_is_any_receiver_type(to_task));

	/* re-base external and internal counters at the inherit and the to-task (if apropos) */
	if (inherit->iii_donating) {
		assert(IIT_EXTERN(to_task) >= III_EXTERN(inherit));
		assert(to_task->iit_externcnt >= inherit->iii_externcnt);
		assert(to_task->iit_externdrop >= inherit->iii_externdrop);
		inherit->iii_externdrop += refs;
		to_task->iit_externdrop += refs;
		externcnt = III_EXTERN(inherit);
		if (0 == externcnt) {
			inherit->iii_donating = FALSE;
			to_task->iit_externcnt -= inherit->iii_externcnt;
			to_task->iit_externdrop -= inherit->iii_externdrop;


			/* Start AppNap delay hysteresis - even if not the last boost for the task. */
			if (ipc_importance_delayed_drop_call != NULL &&
			    ipc_importance_task_is_marked_denap_receiver(to_task)) {
				ipc_importance_task_delayed_drop(to_task);
			} 

			/* drop task assertions associated with the dropped boosts */
			if (ipc_importance_task_check_transition(to_task, IIT_UPDATE_DROP, refs)) {
				ipc_importance_task_propagate_assertion_locked(to_task, IIT_UPDATE_DROP, TRUE);
				/* may have dropped and retaken importance lock */
			}
		} else {
			/* assert(to_task->iit_assertcnt >= refs + externcnt); */
			/* defensive deduction in case of assertcnt underflow */
			if (to_task->iit_assertcnt > refs + externcnt) {
				to_task->iit_assertcnt -= refs;
			} else {
				to_task->iit_assertcnt = externcnt;
			}
		}
	} else {
		inherit->iii_externdrop += refs;
		externcnt = III_EXTERN(inherit);
	}

	/* capture result (if requested) */
	if (NULL != outrefsp) {
		*outrefsp = externcnt;
	}

	ipc_importance_unlock();
	return KERN_SUCCESS;
}

/*
 *	Routine:	ipc_importance_manager_release [Voucher Attribute Manager Interface]
 *	Purpose:
 *		Release the Voucher system's reference on the IPC importance attribute
 *		manager.
 *	Conditions:
 *		As this can only occur after the manager drops the Attribute control
 *		reference granted back at registration time, and that reference is never
 *		dropped, this should never be called.
 */
static void
ipc_importance_manager_release(
	ipc_voucher_attr_manager_t		__assert_only manager)
{
	IMPORTANCE_ASSERT_MANAGER(manager);
	panic("Voucher importance manager released");
}

/*
 *	Routine:	ipc_importance_init
 *	Purpose:
 *		Initialize the  IPC importance manager.
 *	Conditions:
 *		Zones and Vouchers are already initialized.
 */
void
ipc_importance_init(void)
{
	natural_t ipc_importance_max = (task_max + thread_max) * 2;
	char temp_buf[26];
	kern_return_t kr;

	if (PE_parse_boot_argn("imp_interactive_receiver", temp_buf, sizeof(temp_buf))) {
		ipc_importance_interactive_receiver = TRUE;
	}

	ipc_importance_task_zone = zinit(sizeof(struct ipc_importance_task),
					 ipc_importance_max * sizeof(struct ipc_importance_task),
					 sizeof(struct ipc_importance_task),
					 "ipc task importance");
	zone_change(ipc_importance_task_zone, Z_NOENCRYPT, TRUE);

	ipc_importance_inherit_zone = zinit(sizeof(struct ipc_importance_inherit),
					    ipc_importance_max * sizeof(struct ipc_importance_inherit),
					    sizeof(struct ipc_importance_inherit),
					    "ipc importance inherit");
	zone_change(ipc_importance_inherit_zone, Z_NOENCRYPT, TRUE);


#if DEVELOPMENT || DEBUG 
	queue_init(&global_iit_alloc_queue);
#endif

	/* initialize global locking */
	ipc_importance_lock_init();

	kr = ipc_register_well_known_mach_voucher_attr_manager(&ipc_importance_manager,
						(mach_voucher_attr_value_handle_t)0,
						MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
						&ipc_importance_control);
	if (KERN_SUCCESS != kr)
		printf("Voucher importance manager register returned %d", kr);
}

/*
 *	Routine:	ipc_importance_thread_call_init
 *	Purpose:
 *		Initialize the IPC importance code dependent upon
 *		thread-call support being available.
 *	Conditions:
 *		Thread-call mechanism is already initialized.
 */
void
ipc_importance_thread_call_init(void)
{
	/* initialize delayed drop queue and thread-call */
	queue_init(&ipc_importance_delayed_drop_queue);
	ipc_importance_delayed_drop_call = 
		thread_call_allocate(ipc_importance_task_delayed_drop_scan, NULL);
	if (NULL == ipc_importance_delayed_drop_call) {
		panic("ipc_importance_init");
	}
}

/*
 * Routing: task_importance_list_pids
 * Purpose: list pids where task in donating importance.
 * Conditions: To be called only from kdp stackshot code.
 *             Will panic the system otherwise.
 */
extern int
task_importance_list_pids(task_t task, int flags, int *pid_list, unsigned int max_count)
{
	if (lck_spin_is_acquired(&ipc_importance_lock_data) ||
	      max_count < 1 ||
	      task->task_imp_base == IIT_NULL ||
	      pid_list == NULL ||
	      flags != TASK_IMP_LIST_DONATING_PIDS) {
		return 0;
	}
	unsigned int pidcount = 0;
	task_t temp_task;
	ipc_importance_task_t task_imp = task->task_imp_base;
	ipc_kmsg_t temp_kmsg;
	ipc_importance_inherit_t temp_inherit;
	ipc_importance_elem_t elem;
	int target_pid;

	queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) {
		/* check space in buffer */
		if (pidcount >= max_count) 
			break;
		target_pid = -1;

		if (temp_inherit->iii_donating) {

#if DEVELOPMENT || DEBUG
			target_pid = temp_inherit->iii_to_task->iit_bsd_pid;
#else
			temp_task = temp_inherit->iii_to_task->iit_task;
			if (temp_task != TASK_NULL) {
				target_pid = audit_token_pid_from_task(temp_task);
			}
#endif
		}

		if (target_pid != -1) {
			pid_list[pidcount++] = target_pid;
		}

	}

	queue_iterate(&task_imp->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) {
		if (pidcount >= max_count)
			break;
		target_pid = -1;
		elem = temp_kmsg->ikm_importance;
		temp_task = TASK_NULL;

		if (elem == IIE_NULL) {
			continue;
		}

		if (!(temp_kmsg->ikm_header && MACH_MSGH_BITS_RAISED_IMPORTANCE(temp_kmsg->ikm_header->msgh_bits))) {
			continue;
		}

		if (IIE_TYPE_TASK == IIE_TYPE(elem) && 
			(((ipc_importance_task_t)elem)->iit_task != TASK_NULL)) {
			target_pid = audit_token_pid_from_task(((ipc_importance_task_t)elem)->iit_task);
		} else {
			temp_inherit = (ipc_importance_inherit_t)elem;
#if DEVELOPMENT || DEBUG
			target_pid = temp_inherit->iii_to_task->iit_bsd_pid;
#else
			temp_task = temp_inherit->iii_to_task->iit_task;
			if (temp_task != TASK_NULL) {
				target_pid = audit_token_pid_from_task(temp_task);
			}
#endif
		}

		if (target_pid != -1) {
			pid_list[pidcount++] = target_pid;
		}
	}

	return pidcount;
}