thread_group.c   [plain text]

/*
 * Copyright (c) 2016-2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach/mach_types.h>
#include <kern/kern_types.h>
#include <kern/processor.h>
#include <kern/thread.h>
#include <kern/thread_group.h>
#include <kern/zalloc.h>
#include <kern/task.h>
#include <kern/machine.h>
#include <kern/coalition.h>
#include <sys/errno.h>
#include <kern/queue.h>
#include <kern/locks.h>
#include <kern/thread_group.h>
#include <kern/sched_clutch.h>

#if CONFIG_THREAD_GROUPS

#define CACHELINE_SIZE (1 << MMU_CLINE)

struct thread_group {
	uint64_t                tg_id;
	char                    tg_name[THREAD_GROUP_MAXNAME];
	struct os_refcnt        tg_refcount;
	uint32_t                tg_flags;
	cluster_type_t          tg_recommendation;
	queue_chain_t           tg_queue_chain;
#if CONFIG_SCHED_CLUTCH
	struct sched_clutch     tg_sched_clutch;
#endif /* CONFIG_SCHED_CLUTCH */
	// 16 bytes of padding here
	uint8_t                 tg_machine_data[] __attribute__((aligned(CACHELINE_SIZE)));
} __attribute__((aligned(8)));

static SECURITY_READ_ONLY_LATE(zone_t) tg_zone;
static uint32_t tg_count;
static queue_head_t tg_queue;
static LCK_GRP_DECLARE(tg_lck_grp, "thread_group");
static LCK_MTX_DECLARE(tg_lock, &tg_lck_grp);
static LCK_SPIN_DECLARE(tg_flags_update_lock, &tg_lck_grp);

static uint64_t tg_next_id = 0;
static uint32_t tg_size;
static uint32_t tg_machine_data_size;
static struct thread_group *tg_system;
static struct thread_group *tg_background;
static struct thread_group *tg_adaptive;
static struct thread_group *tg_vm;
static struct thread_group *tg_io_storage;
static struct thread_group *tg_perf_controller;
int tg_set_by_bankvoucher;

static bool thread_group_retain_try(struct thread_group *tg);

/*
 * Initialize thread groups at boot
 */
void
thread_group_init(void)
{
	// Get thread group structure extension from EDT or boot-args (which can override EDT)
	if (!PE_parse_boot_argn("kern.thread_group_extra_bytes", &tg_machine_data_size, sizeof(tg_machine_data_size))) {
		if (!PE_get_default("kern.thread_group_extra_bytes", &tg_machine_data_size, sizeof(tg_machine_data_size))) {
			tg_machine_data_size = 8;
		}
	}

	// Check if thread group can be set by voucher adoption from EDT or boot-args (which can override EDT)
	if (!PE_parse_boot_argn("kern.thread_group_set_by_bankvoucher", &tg_set_by_bankvoucher, sizeof(tg_set_by_bankvoucher))) {
		if (!PE_get_default("kern.thread_group_set_by_bankvoucher", &tg_set_by_bankvoucher, sizeof(tg_set_by_bankvoucher))) {
			tg_set_by_bankvoucher = 1;
		}
	}

	tg_size = sizeof(struct thread_group) + tg_machine_data_size;
	if (tg_size % CACHELINE_SIZE) {
		tg_size += CACHELINE_SIZE - (tg_size % CACHELINE_SIZE);
	}
	tg_machine_data_size = tg_size - sizeof(struct thread_group);
	// printf("tg_size=%d(%lu+%d)\n", tg_size, sizeof(struct thread_group), tg_machine_data_size);
	assert(offsetof(struct thread_group, tg_machine_data) % CACHELINE_SIZE == 0);
	tg_zone = zone_create("thread_groups", tg_size, ZC_NOENCRYPT | ZC_ALIGNMENT_REQUIRED);

	queue_head_init(tg_queue);
	tg_system = thread_group_create_and_retain();
	thread_group_set_name(tg_system, "system");
	tg_background = thread_group_create_and_retain();
	thread_group_set_name(tg_background, "background");
	tg_adaptive = thread_group_create_and_retain();
	thread_group_set_name(tg_adaptive, "adaptive");
	tg_vm = thread_group_create_and_retain();
	thread_group_set_name(tg_vm, "VM");
	tg_io_storage = thread_group_create_and_retain();
	thread_group_set_name(tg_io_storage, "io storage");
	tg_perf_controller = thread_group_create_and_retain();
	thread_group_set_name(tg_perf_controller, "perf_controller");

	/*
	 * If CLPC is disabled, it would recommend SMP for all thread groups.
	 * In that mode, the scheduler would like to restrict the kernel thread
	 * groups to the E-cluster while all other thread groups are run on the
	 * P-cluster. To identify the kernel thread groups, mark them with a
	 * special flag THREAD_GROUP_FLAGS_SMP_RESTRICT which is looked at by
	 * recommended_pset_type().
	 */
	tg_system->tg_flags |= THREAD_GROUP_FLAGS_SMP_RESTRICT;
	tg_vm->tg_flags |= THREAD_GROUP_FLAGS_SMP_RESTRICT;
	tg_io_storage->tg_flags |= THREAD_GROUP_FLAGS_SMP_RESTRICT;
	tg_perf_controller->tg_flags |= THREAD_GROUP_FLAGS_SMP_RESTRICT;
}

#if CONFIG_SCHED_CLUTCH
/*
 * sched_clutch_for_thread
 *
 * The routine provides a back linkage from the thread to the
 * sched_clutch it belongs to. This relationship is based on the
 * thread group membership of the thread. Since that membership is
 * changed from the thread context with the thread lock held, this
 * linkage should be looked at only with the thread lock held or
 * when the thread cannot be running (for eg. the thread is in the
 * runq and being removed as part of thread_select().
 */
sched_clutch_t
sched_clutch_for_thread(thread_t thread)
{
	assert(thread->thread_group != NULL);
	return &(thread->thread_group->tg_sched_clutch);
}

sched_clutch_t
sched_clutch_for_thread_group(struct thread_group *thread_group)
{
	return &(thread_group->tg_sched_clutch);
}

/*
 * Translate the TG flags to a priority boost for the sched_clutch.
 * This priority boost will apply to the entire clutch represented
 * by the thread group.
 */
static void
sched_clutch_update_tg_flags(sched_clutch_t clutch, uint8_t flags)
{
	sched_clutch_tg_priority_t sc_tg_pri = 0;
	if (flags & THREAD_GROUP_FLAGS_UI_APP) {
		sc_tg_pri = SCHED_CLUTCH_TG_PRI_HIGH;
	} else if (flags & THREAD_GROUP_FLAGS_EFFICIENT) {
		sc_tg_pri = SCHED_CLUTCH_TG_PRI_LOW;
	} else {
		sc_tg_pri = SCHED_CLUTCH_TG_PRI_MED;
	}
	os_atomic_store(&clutch->sc_tg_priority, sc_tg_pri, relaxed);
}

#endif /* CONFIG_SCHED_CLUTCH */

/*
 * Use a spinlock to protect all thread group flag updates.
 * The lock should not have heavy contention since these flag updates should
 * be infrequent. If this lock has contention issues, it should be changed to
 * a per thread-group lock.
 *
 * The lock protects the flags field in the thread_group structure. It is also
 * held while doing callouts to CLPC to reflect these flag changes.
 */

void
thread_group_flags_update_lock(void)
{
	lck_spin_lock_grp(&tg_flags_update_lock, &tg_lck_grp);
}

void
thread_group_flags_update_unlock(void)
{
	lck_spin_unlock(&tg_flags_update_lock);
}

/*
 * Inform platform code about already existing thread groups
 * or ask it to free state for all thread groups
 */
void
thread_group_resync(boolean_t create)
{
	struct thread_group *tg;

	lck_mtx_lock(&tg_lock);
	qe_foreach_element(tg, &tg_queue, tg_queue_chain) {
		if (create) {
			machine_thread_group_init(tg);
		} else {
			machine_thread_group_deinit(tg);
		}
	}
	lck_mtx_unlock(&tg_lock);
}

/*
 * Create new thread group and add new reference to it.
 */
struct thread_group *
thread_group_create_and_retain(void)
{
	struct thread_group *tg;

	tg = (struct thread_group *)zalloc(tg_zone);
	if (tg == NULL) {
		panic("thread group zone over commit");
	}
	assert((uintptr_t)tg % CACHELINE_SIZE == 0);
	bzero(tg, sizeof(struct thread_group));

#if CONFIG_SCHED_CLUTCH
	/*
	 * The clutch scheduler maintains a bunch of runqs per thread group. For
	 * each thread group it maintains a sched_clutch structure. The lifetime
	 * of that structure is tied directly to the lifetime of the thread group.
	 */
	sched_clutch_init_with_thread_group(&(tg->tg_sched_clutch), tg);

	/*
	 * Since the thread group flags are used to determine any priority promotions
	 * for the threads in the thread group, initialize them to 0.
	 */
	sched_clutch_update_tg_flags(&(tg->tg_sched_clutch), 0);

#endif /* CONFIG_SCHED_CLUTCH */

	lck_mtx_lock(&tg_lock);
	tg->tg_id = tg_next_id++;
	tg->tg_recommendation = CLUSTER_TYPE_SMP; // no recommendation yet
	os_ref_init(&tg->tg_refcount, NULL);
	tg_count++;
	enqueue_tail(&tg_queue, &tg->tg_queue_chain);
	lck_mtx_unlock(&tg_lock);

	// call machine layer init before this thread group becomes visible
	machine_thread_group_init(tg);

	KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NEW), tg->tg_id);

	return tg;
}

/*
 * Point newly created thread to its home thread group
 */
void
thread_group_init_thread(thread_t t, task_t task)
{
	struct thread_group *tg = task_coalition_get_thread_group(task);
	t->thread_group = tg;
	KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_SET),
	    THREAD_GROUP_INVALID, tg->tg_id, (uintptr_t)thread_tid(t));
}

/*
 * Set thread group name
 */
void
thread_group_set_name(__unused struct thread_group *tg, __unused const char *name)
{
	if (name == NULL) {
		return;
	}
	if (!thread_group_retain_try(tg)) {
		return;
	}
	if (tg->tg_name[0] == '\0') {
		strncpy(&tg->tg_name[0], name, THREAD_GROUP_MAXNAME);
#if defined(__LP64__)
		KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NAME),
		    tg->tg_id,
		    *(uint64_t*)(void*)&tg->tg_name[0],
		    *(uint64_t*)(void*)&tg->tg_name[sizeof(uint64_t)]
		    );
#else /* defined(__LP64__) */
		KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NAME),
		    tg->tg_id,
		    *(uint32_t*)(void*)&tg->tg_name[0],
		    *(uint32_t*)(void*)&tg->tg_name[sizeof(uint32_t)]
		    );
#endif /* defined(__LP64__) */
	}
	thread_group_release(tg);
}

void
thread_group_set_flags(struct thread_group *tg, uint64_t flags)
{
	thread_group_flags_update_lock();
	thread_group_set_flags_locked(tg, flags);
	thread_group_flags_update_unlock();
}

void
thread_group_clear_flags(struct thread_group *tg, uint64_t flags)
{
	thread_group_flags_update_lock();
	thread_group_clear_flags_locked(tg, flags);
	thread_group_flags_update_unlock();
}

/*
 * Set thread group flags and perform related actions.
 * The tg_flags_update_lock should be held.
 * Currently supported flags are:
 * - THREAD_GROUP_FLAGS_EFFICIENT
 * - THREAD_GROUP_FLAGS_UI_APP
 */

void
thread_group_set_flags_locked(struct thread_group *tg, uint64_t flags)
{
	if ((flags & THREAD_GROUP_FLAGS_VALID) != flags) {
		panic("thread_group_set_flags: Invalid flags %llu", flags);
	}

	if ((tg->tg_flags & flags) == flags) {
		return;
	}

	__kdebug_only uint64_t old_flags = tg->tg_flags;
	tg->tg_flags |= flags;
	machine_thread_group_flags_update(tg, tg->tg_flags);
#if CONFIG_SCHED_CLUTCH
	sched_clutch_update_tg_flags(&(tg->tg_sched_clutch), tg->tg_flags);
#endif /* CONFIG_SCHED_CLUTCH */
	KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_FLAGS),
	    tg->tg_id, tg->tg_flags, old_flags);
}

/*
 * Clear thread group flags and perform related actions
 * The tg_flags_update_lock should be held.
 * Currently supported flags are:
 * - THREAD_GROUP_FLAGS_EFFICIENT
 * - THREAD_GROUP_FLAGS_UI_APP
 */

void
thread_group_clear_flags_locked(struct thread_group *tg, uint64_t flags)
{
	if ((flags & THREAD_GROUP_FLAGS_VALID) != flags) {
		panic("thread_group_clear_flags: Invalid flags %llu", flags);
	}

	if ((tg->tg_flags & flags) == 0) {
		return;
	}

	__kdebug_only uint64_t old_flags = tg->tg_flags;
	tg->tg_flags &= ~flags;
#if CONFIG_SCHED_CLUTCH
	sched_clutch_update_tg_flags(&(tg->tg_sched_clutch), tg->tg_flags);
#endif /* CONFIG_SCHED_CLUTCH */
	machine_thread_group_flags_update(tg, tg->tg_flags);
	KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_FLAGS),
	    tg->tg_id, tg->tg_flags, old_flags);
}



/*
 * Find thread group with specified name and put new reference to it.
 */
struct thread_group *
thread_group_find_by_name_and_retain(char *name)
{
	struct thread_group *result = NULL;

	if (name == NULL) {
		return NULL;
	}

	if (strncmp("system", name, THREAD_GROUP_MAXNAME) == 0) {
		return thread_group_retain(tg_system);
	} else if (strncmp("background", name, THREAD_GROUP_MAXNAME) == 0) {
		return thread_group_retain(tg_background);
	} else if (strncmp("adaptive", name, THREAD_GROUP_MAXNAME) == 0) {
		return thread_group_retain(tg_adaptive);
	} else if (strncmp("perf_controller", name, THREAD_GROUP_MAXNAME) == 0) {
		return thread_group_retain(tg_perf_controller);
	}

	struct thread_group *tg;
	lck_mtx_lock(&tg_lock);
	qe_foreach_element(tg, &tg_queue, tg_queue_chain) {
		if (strncmp(tg->tg_name, name, THREAD_GROUP_MAXNAME) == 0 &&
		    thread_group_retain_try(tg)) {
			result = tg;
			break;
		}
	}
	lck_mtx_unlock(&tg_lock);
	return result;
}

/*
 * Find thread group with specified ID and add new reference to it.
 */
struct thread_group *
thread_group_find_by_id_and_retain(uint64_t id)
{
	struct thread_group *tg = NULL;
	struct thread_group *result = NULL;

	switch (id) {
	case THREAD_GROUP_SYSTEM:
		result = tg_system;
		thread_group_retain(tg_system);
		break;
	case THREAD_GROUP_BACKGROUND:
		result = tg_background;
		thread_group_retain(tg_background);
		break;
	case THREAD_GROUP_ADAPTIVE:
		result = tg_adaptive;
		thread_group_retain(tg_adaptive);
		break;
	case THREAD_GROUP_VM:
		result = tg_vm;
		thread_group_retain(tg_vm);
		break;
	case THREAD_GROUP_IO_STORAGE:
		result = tg_io_storage;
		thread_group_retain(tg_io_storage);
		break;
	case THREAD_GROUP_PERF_CONTROLLER:
		result = tg_perf_controller;
		thread_group_retain(tg_perf_controller);
		break;
	default:
		lck_mtx_lock(&tg_lock);
		qe_foreach_element(tg, &tg_queue, tg_queue_chain) {
			if (tg->tg_id == id && thread_group_retain_try(tg)) {
				result = tg;
				break;
			}
		}
		lck_mtx_unlock(&tg_lock);
	}
	return result;
}

/*
 * Add new reference to specified thread group
 */
struct thread_group *
thread_group_retain(struct thread_group *tg)
{
	os_ref_retain(&tg->tg_refcount);
	return tg;
}

/*
 * Similar to thread_group_retain, but fails for thread groups with a
 * zero reference count. Returns true if retained successfully.
 */
static bool
thread_group_retain_try(struct thread_group *tg)
{
	return os_ref_retain_try(&tg->tg_refcount);
}

/*
 * Drop a reference to specified thread group
 */
void
thread_group_release(struct thread_group *tg)
{
	if (os_ref_release(&tg->tg_refcount) == 0) {
		lck_mtx_lock(&tg_lock);
		tg_count--;
		remqueue(&tg->tg_queue_chain);
		lck_mtx_unlock(&tg_lock);
		static_assert(THREAD_GROUP_MAXNAME >= (sizeof(uint64_t) * 2), "thread group name is too short");
		static_assert(__alignof(struct thread_group) >= __alignof(uint64_t), "thread group name is not 8 bytes aligned");
#if defined(__LP64__)
		KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NAME_FREE),
		    tg->tg_id,
		    *(uint64_t*)(void*)&tg->tg_name[0],
		    *(uint64_t*)(void*)&tg->tg_name[sizeof(uint64_t)]
		    );
#else /* defined(__LP64__) */
		KDBG(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_NAME_FREE),
		    tg->tg_id,
		    *(uint32_t*)(void*)&tg->tg_name[0],
		    *(uint32_t*)(void*)&tg->tg_name[sizeof(uint32_t)]
		    );
#endif /* defined(__LP64__) */
		KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_FREE), tg->tg_id);
#if CONFIG_SCHED_CLUTCH
		sched_clutch_destroy(&(tg->tg_sched_clutch));
#endif /* CONFIG_SCHED_CLUTCH */
		machine_thread_group_deinit(tg);
		zfree(tg_zone, tg);
	}
}

/*
 * Get thread's current thread group
 */
inline struct thread_group *
thread_group_get(thread_t t)
{
	return t->thread_group;
}

struct thread_group *
thread_group_get_home_group(thread_t t)
{
	return task_coalition_get_thread_group(t->task);
}

#if CONFIG_SCHED_AUTO_JOIN

/*
 * thread_set_thread_group_auto_join()
 *
 * Sets the thread group of a thread based on auto-join rules.
 *
 * Preconditions:
 * - Thread must not be part of a runq (freshly made runnable threads or terminating only)
 * - Thread must be locked by the caller already
 */
static void
thread_set_thread_group_auto_join(thread_t t, struct thread_group *tg, __unused struct thread_group *old_tg)
{
	assert(t->runq == PROCESSOR_NULL);
	t->thread_group = tg;

	/*
	 * If the thread group is being changed for the current thread, callout to
	 * CLPC to update the thread's information at that layer. This makes sure CLPC
	 * has consistent state when the current thread is going off-core.
	 */
	if (t == current_thread()) {
		uint64_t ctime = mach_approximate_time();
		uint64_t arg1, arg2;
		machine_thread_going_on_core(t, thread_get_urgency(t, &arg1, &arg2), 0, 0, ctime);
		machine_switch_perfcontrol_state_update(THREAD_GROUP_UPDATE, ctime, PERFCONTROL_CALLOUT_WAKE_UNSAFE, t);
	}
}

#endif /* CONFIG_SCHED_AUTO_JOIN */

/*
 * thread_set_thread_group_explicit()
 *
 * Sets the thread group of a thread based on default non auto-join rules.
 *
 * Preconditions:
 * - Thread must be the current thread
 * - Caller must not have the thread locked
 * - Interrupts must be disabled
 */
static void
thread_set_thread_group_explicit(thread_t t, struct thread_group *tg, __unused struct thread_group *old_tg)
{
	assert(t == current_thread());
	/*
	 * In the clutch scheduler world, the runq membership of the thread
	 * is based on its thread group membership and its scheduling bucket.
	 * In order to synchronize with the priority (and therefore bucket)
	 * getting updated concurrently, it is important to perform the
	 * thread group change also under the thread lock.
	 */
	thread_lock(t);
	t->thread_group = tg;

#if CONFIG_SCHED_CLUTCH
	sched_clutch_t old_clutch = (old_tg) ? &(old_tg->tg_sched_clutch) : NULL;
	sched_clutch_t new_clutch = (tg) ? &(tg->tg_sched_clutch) : NULL;
	if (SCHED_CLUTCH_THREAD_ELIGIBLE(t)) {
		sched_clutch_thread_clutch_update(t, old_clutch, new_clutch);
	}
#endif /* CONFIG_SCHED_CLUTCH */

	thread_unlock(t);

	uint64_t ctime = mach_approximate_time();
	uint64_t arg1, arg2;
	machine_thread_going_on_core(t, thread_get_urgency(t, &arg1, &arg2), 0, 0, ctime);
	machine_switch_perfcontrol_state_update(THREAD_GROUP_UPDATE, ctime, 0, t);
}

/*
 * thread_set_thread_group()
 *
 * Overrides the current home thread group with an override group. However,
 * an adopted work interval overrides the override. Does not take a reference
 * on the group, so caller must guarantee group lifetime lasts as long as the
 * group is set.
 *
 * The thread group is set according to a hierarchy:
 *
 * 1) work interval specified group (explicit API)
 * 2) Auto-join thread group (wakeup tracking for special work intervals)
 * 3) bank voucher carried group (implicitly set)
 * 4) coalition default thread group (ambient)
 */
static void
thread_set_thread_group(thread_t t, struct thread_group *tg, bool auto_join)
{
	struct thread_group *home_tg = thread_group_get_home_group(t);
	struct thread_group *old_tg = NULL;

	if (tg == NULL) {
		/* when removing an override, revert to home group */
		tg = home_tg;
	}

	spl_t s = splsched();

	old_tg = t->thread_group;

	if (old_tg != tg) {
		KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_THREAD_GROUP, MACH_THREAD_GROUP_SET),
		    t->thread_group ? t->thread_group->tg_id : 0,
		    tg->tg_id, (uintptr_t)thread_tid(t), home_tg->tg_id);

		/*
		 * Based on whether this is a change due to auto-join, the join does
		 * different things and has different expectations.
		 */
		if (auto_join) {
#if CONFIG_SCHED_AUTO_JOIN
			/*
			 * set thread group with auto-join rules. This has the
			 * implicit assumption that the thread lock is already held.
			 * Also this could happen to any thread (current or thread
			 * being context switched).
			 */
			thread_set_thread_group_auto_join(t, tg, old_tg);
#else /* CONFIG_SCHED_AUTO_JOIN */
			panic("Auto-Join unsupported on this platform");
#endif /* CONFIG_SCHED_AUTO_JOIN */
		} else {
			/*
			 * set thread group with the explicit join rules. This has
			 * the implicit assumption that the thread is not locked. Also
			 * this would be done only to the current thread.
			 */
			thread_set_thread_group_explicit(t, tg, old_tg);
		}
	}

	splx(s);
}

void
thread_group_set_bank(thread_t t, struct thread_group *tg)
{
	/* work interval group overrides any bank override group */
	if (t->th_work_interval) {
		return;
	}

	/* boot arg disables groups in bank */
	if (tg_set_by_bankvoucher == FALSE) {
		return;
	}

	thread_set_thread_group(t, tg, false);
}

/*
 * thread_set_work_interval_thread_group()
 *
 * Sets the thread's group to the work interval thread group.
 * If auto_join == true, thread group is being overriden through scheduler
 * auto-join policies.
 *
 * Preconditions for auto-join case:
 * - t is not current_thread and t should be locked.
 * - t should not be running on a remote core; thread context switching is a valid state for this.
 */
void
thread_set_work_interval_thread_group(thread_t t, struct thread_group *tg, bool auto_join)
{
	if (tg == NULL) {
		/*
		 * when removing a work interval override, fall back
		 * to the current voucher override.
		 *
		 * In the auto_join case, the thread is already locked by the caller so
		 * its unsafe to get the thread group from the current voucher (since
		 * that might require taking task lock and ivac lock). However, the
		 * auto-join policy does not allow threads to switch thread groups based
		 * on voucher overrides.
		 *
		 * For the normal case, lookup the thread group from the currently adopted
		 * voucher and use that as the fallback tg.
		 */

		if (auto_join == false) {
			tg = thread_get_current_voucher_thread_group(t);
		}
	}

	thread_set_thread_group(t, tg, auto_join);
}

inline cluster_type_t
thread_group_recommendation(struct thread_group *tg)
{
	if (tg == NULL) {
		return CLUSTER_TYPE_SMP;
	} else {
		return tg->tg_recommendation;
	}
}

inline uint64_t
thread_group_get_id(struct thread_group *tg)
{
	return tg->tg_id;
}

uint32_t
thread_group_count(void)
{
	return tg_count;
}

/*
 * Can only be called while tg cannot be destroyed
 */
inline const char*
thread_group_get_name(struct thread_group *tg)
{
	return tg->tg_name;
}

inline void *
thread_group_get_machine_data(struct thread_group *tg)
{
	return &tg->tg_machine_data;
}

inline uint32_t
thread_group_machine_data_size(void)
{
	return tg_machine_data_size;
}

kern_return_t
thread_group_iterate_stackshot(thread_group_iterate_fn_t callout, void *arg)
{
	struct thread_group *tg;
	int i = 0;
	qe_foreach_element(tg, &tg_queue, tg_queue_chain) {
		if (tg == NULL || !ml_validate_nofault((vm_offset_t)tg, sizeof(struct thread_group))) {
			return KERN_FAILURE;
		}
		callout(arg, i, tg);
		i++;
	}
	return KERN_SUCCESS;
}

void
thread_group_join_io_storage(void)
{
	struct thread_group *tg = thread_group_find_by_id_and_retain(THREAD_GROUP_IO_STORAGE);
	assert(tg != NULL);
	thread_set_thread_group(current_thread(), tg, false);
}

void
thread_group_join_perf_controller(void)
{
	struct thread_group *tg = thread_group_find_by_id_and_retain(THREAD_GROUP_PERF_CONTROLLER);
	assert(tg != NULL);
	thread_set_thread_group(current_thread(), tg, false);
}

void
thread_group_vm_add(void)
{
	assert(tg_vm != NULL);
	thread_set_thread_group(current_thread(), thread_group_find_by_id_and_retain(THREAD_GROUP_VM), false);
}

uint64_t
kdp_thread_group_get_flags(struct thread_group *tg)
{
	return tg->tg_flags;
}

/*
 * Returns whether the thread group is restricted to the E-cluster when CLPC is
 * turned off.
 */
boolean_t
thread_group_smp_restricted(struct thread_group *tg)
{
	if (tg->tg_flags & THREAD_GROUP_FLAGS_SMP_RESTRICT) {
		return true;
	} else {
		return false;
	}
}

void
thread_group_update_recommendation(struct thread_group *tg, cluster_type_t new_recommendation)
{
	/*
	 * Since the tg->tg_recommendation field is read by CPUs trying to determine
	 * where a thread/thread group needs to be placed, it is important to use
	 * atomic operations to update the recommendation.
	 */
	os_atomic_store(&tg->tg_recommendation, new_recommendation, relaxed);
}

#if CONFIG_SCHED_EDGE

int sched_edge_restrict_ut = 1;
int sched_edge_restrict_bg = 1;

void
sched_perfcontrol_thread_group_recommend(__unused void *machine_data, __unused cluster_type_t new_recommendation)
{
	struct thread_group *tg = (struct thread_group *)((uintptr_t)machine_data - offsetof(struct thread_group, tg_machine_data));
	/*
	 * CLUSTER_TYPE_SMP was used for some debugging support when CLPC dynamic control was turned off.
	 * In more recent implementations, CLPC simply recommends "P-spill" when dynamic control is turned off. So it should
	 * never be recommending CLUSTER_TYPE_SMP for thread groups.
	 */
	assert(new_recommendation != CLUSTER_TYPE_SMP);
	/*
	 * The Edge scheduler expects preferred cluster recommendations for each QoS level within a TG. Until the new CLPC
	 * routine is being called, fake out the call from the old CLPC interface.
	 */
	uint32_t tg_bucket_preferred_cluster[TH_BUCKET_SCHED_MAX] = {0};
	/*
	 * For all buckets higher than UT, apply the recommendation to the thread group bucket
	 */
	for (sched_bucket_t bucket = TH_BUCKET_FIXPRI; bucket < TH_BUCKET_SHARE_UT; bucket++) {
		tg_bucket_preferred_cluster[bucket] = (new_recommendation == pset_type_for_id(0)) ? 0 : 1;
	}
	/* For UT & BG QoS, set the recommendation only if they havent been restricted via sysctls */
	if (!sched_edge_restrict_ut) {
		tg_bucket_preferred_cluster[TH_BUCKET_SHARE_UT] = (new_recommendation == pset_type_for_id(0)) ? 0 : 1;
	}
	if (!sched_edge_restrict_bg) {
		tg_bucket_preferred_cluster[TH_BUCKET_SHARE_BG] = (new_recommendation == pset_type_for_id(0)) ? 0 : 1;
	}
	sched_perfcontrol_preferred_cluster_options_t options = 0;
	if (new_recommendation == CLUSTER_TYPE_P) {
		options |= SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNING;
	}
	sched_edge_tg_preferred_cluster_change(tg, tg_bucket_preferred_cluster, options);
}

void
sched_perfcontrol_edge_matrix_get(sched_clutch_edge *edge_matrix, bool *edge_request_bitmap, uint64_t flags, uint64_t matrix_order)
{
	sched_edge_matrix_get(edge_matrix, edge_request_bitmap, flags, matrix_order);
}

void
sched_perfcontrol_edge_matrix_set(sched_clutch_edge *edge_matrix, bool *edge_changes_bitmap, uint64_t flags, uint64_t matrix_order)
{
	sched_edge_matrix_set(edge_matrix, edge_changes_bitmap, flags, matrix_order);
}

void
sched_perfcontrol_thread_group_preferred_clusters_set(void *machine_data, uint32_t tg_preferred_cluster,
    uint32_t overrides[PERFCONTROL_CLASS_MAX], sched_perfcontrol_preferred_cluster_options_t options)
{
	struct thread_group *tg = (struct thread_group *)((uintptr_t)machine_data - offsetof(struct thread_group, tg_machine_data));
	uint32_t tg_bucket_preferred_cluster[TH_BUCKET_SCHED_MAX] = {
		[TH_BUCKET_FIXPRI]   = (overrides[PERFCONTROL_CLASS_ABOVEUI] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_ABOVEUI] : tg_preferred_cluster,
		[TH_BUCKET_SHARE_FG] = (overrides[PERFCONTROL_CLASS_UI] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_UI] : tg_preferred_cluster,
		[TH_BUCKET_SHARE_IN] = (overrides[PERFCONTROL_CLASS_UI] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_UI] : tg_preferred_cluster,
		[TH_BUCKET_SHARE_DF] = (overrides[PERFCONTROL_CLASS_NONUI] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_NONUI] : tg_preferred_cluster,
		[TH_BUCKET_SHARE_UT] = (overrides[PERFCONTROL_CLASS_UTILITY] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_UTILITY] : tg_preferred_cluster,
		[TH_BUCKET_SHARE_BG] = (overrides[PERFCONTROL_CLASS_BACKGROUND] != SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE) ? overrides[PERFCONTROL_CLASS_BACKGROUND] : tg_preferred_cluster,
	};
	sched_edge_tg_preferred_cluster_change(tg, tg_bucket_preferred_cluster, options);
}

#else /* CONFIG_SCHED_EDGE */

void
sched_perfcontrol_thread_group_recommend(__unused void *machine_data, __unused cluster_type_t new_recommendation)
{
	struct thread_group *tg = (struct thread_group *)((uintptr_t)machine_data - offsetof(struct thread_group, tg_machine_data));
	SCHED(thread_group_recommendation_change)(tg, new_recommendation);
}

void
sched_perfcontrol_edge_matrix_get(__unused sched_clutch_edge *edge_matrix, __unused bool *edge_request_bitmap, __unused uint64_t flags, __unused uint64_t matrix_order)
{
}

void
sched_perfcontrol_edge_matrix_set(__unused sched_clutch_edge *edge_matrix, __unused bool *edge_changes_bitmap, __unused uint64_t flags, __unused uint64_t matrix_order)
{
}

void
sched_perfcontrol_thread_group_preferred_clusters_set(__unused void *machine_data, __unused uint32_t tg_preferred_cluster,
    __unused uint32_t overrides[PERFCONTROL_CLASS_MAX], __unused sched_perfcontrol_preferred_cluster_options_t options)
{
}

#endif /* CONFIG_SCHED_EDGE */

#endif /* CONFIG_THREAD_GROUPS */