#include <kern/kern_types.h>
#include <mach/mach_types.h>
#include <mach/boolean.h>
#include <kern/coalition.h>
#include <kern/host.h>
#include <kern/ledger.h>
#include <kern/kalloc.h>
#include <kern/mach_param.h>
#include <kern/task.h>
#include <kern/zalloc.h>
#include <libkern/OSAtomic.h>
#include <mach/coalition_notification_server.h>
#include <mach/host_priv.h>
#include <mach/host_special_ports.h>
#include <sys/errno.h>
extern ledger_template_t task_ledger_template;
#define CONFIG_COALITION_MAX CONFIG_TASK_MAX
#define COALITION_CHUNK TASK_CHUNK
int unrestrict_coalition_syscalls;
lck_attr_t coalitions_lck_attr;
lck_grp_t coalitions_lck_grp;
lck_grp_attr_t coalitions_lck_grp_attr;
decl_lck_mtx_data(static,coalitions_list_lock);
static uint64_t coalition_count;
static uint64_t coalition_next_id = 1;
static queue_head_t coalitions;
coalition_t default_coalition;
zone_t coalition_zone;
struct coalition {
uint64_t id;
ledger_t ledger;
uint64_t bytesread;
uint64_t byteswritten;
uint64_t gpu_time;
uint64_t last_became_nonempty_time;
uint64_t time_nonempty;
uint64_t task_count;
uint64_t dead_task_count;
queue_head_t tasks;
queue_chain_t coalitions;
decl_lck_mtx_data(,lock)
uint32_t ref_count;
uint32_t active_count;
unsigned int privileged : 1;
unsigned int termrequested : 1;
unsigned int terminated : 1;
unsigned int reaped : 1;
unsigned int notified : 1;
};
#define coalition_lock(c) do{ lck_mtx_lock(&c->lock); }while(0)
#define coalition_unlock(c) do{ lck_mtx_unlock(&c->lock); }while(0)
static void
coalition_notify_user(uint64_t id, uint32_t flags)
{
mach_port_t user_port;
kern_return_t kr;
kr = host_get_coalition_port(host_priv_self(), &user_port);
if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
return;
}
coalition_notification(user_port, id, flags);
}
static coalition_t
coalition_find_by_id_internal(uint64_t coal_id)
{
if (coal_id == 0) {
return COALITION_NULL;
}
lck_mtx_assert(&coalitions_list_lock, LCK_MTX_ASSERT_OWNED);
coalition_t coal;
queue_iterate(&coalitions, coal, coalition_t, coalitions) {
if (coal->id == coal_id) {
return coal;
}
}
return COALITION_NULL;
}
kern_return_t
coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_usage *cru_out)
{
kern_return_t kr;
ledger_amount_t credit, debit;
ledger_t sum_ledger = ledger_instantiate(task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES);
if (sum_ledger == LEDGER_NULL) {
return KERN_RESOURCE_SHORTAGE;
}
coalition_lock(coal);
ledger_rollup(sum_ledger, coal->ledger);
uint64_t bytesread = coal->bytesread;
uint64_t byteswritten = coal->byteswritten;
uint64_t gpu_time = coal->gpu_time;
task_t task;
queue_iterate(&coal->tasks, task, task_t, coalition_tasks) {
ledger_rollup(sum_ledger, task->ledger);
bytesread += task->task_io_stats->disk_reads.size;
byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
gpu_time += task_gpu_utilisation(task);
}
cru_out->tasks_started = coal->task_count;
cru_out->tasks_exited = coal->dead_task_count;
uint64_t time_nonempty = coal->time_nonempty;
uint64_t last_became_nonempty_time = coal->last_became_nonempty_time;
coalition_unlock(coal);
kr = ledger_get_entries(sum_ledger, task_ledgers.cpu_time,
&credit, &debit);
if (kr != KERN_SUCCESS) {
credit = 0;
}
cru_out->cpu_time = credit;
kr = ledger_get_entries(sum_ledger, task_ledgers.interrupt_wakeups,
&credit, &debit);
if (kr != KERN_SUCCESS) {
credit = 0;
}
cru_out->interrupt_wakeups = credit;
kr = ledger_get_entries(sum_ledger, task_ledgers.platform_idle_wakeups,
&credit, &debit);
if (kr != KERN_SUCCESS) {
credit = 0;
}
cru_out->platform_idle_wakeups = credit;
cru_out->bytesread = bytesread;
cru_out->byteswritten = byteswritten;
cru_out->gpu_time = gpu_time;
ledger_dereference(sum_ledger);
sum_ledger = LEDGER_NULL;
if (last_became_nonempty_time) {
time_nonempty += mach_absolute_time() - last_became_nonempty_time;
}
absolutetime_to_nanoseconds(time_nonempty, &cru_out->time_nonempty);
return KERN_SUCCESS;
}
kern_return_t
coalition_create_internal(coalition_t *out, boolean_t privileged)
{
struct coalition *new_coal = (struct coalition *)zalloc(coalition_zone);
if (new_coal == COALITION_NULL) {
return KERN_RESOURCE_SHORTAGE;
}
bzero(new_coal, sizeof(*new_coal));
new_coal->ledger = ledger_instantiate(task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES);
if (new_coal->ledger == NULL) {
zfree(coalition_zone, new_coal);
return KERN_RESOURCE_SHORTAGE;
}
new_coal->ref_count = 2;
new_coal->privileged = privileged ? TRUE : FALSE;
lck_mtx_init(&new_coal->lock, &coalitions_lck_grp, &coalitions_lck_attr);
queue_init(&new_coal->tasks);
lck_mtx_lock(&coalitions_list_lock);
new_coal->id = coalition_next_id++;
coalition_count++;
queue_enter(&coalitions, new_coal, coalition_t, coalitions);
lck_mtx_unlock(&coalitions_list_lock);
#if COALITION_DEBUG
printf("%s: new coal id %llu\n", __func__, new_coal->id);
#endif
*out = new_coal;
return KERN_SUCCESS;
}
void
coalition_release(coalition_t coal)
{
boolean_t do_dealloc = FALSE;
coalition_lock(coal);
coal->ref_count--;
if (coal->ref_count == 0) {
do_dealloc = TRUE;
}
#if COALITION_DEBUG
uint32_t rc = coal->ref_count;
#endif
coalition_unlock(coal);
#if COALITION_DEBUG
printf("%s: coal %llu ref_count-- -> %u%s\n", __func__, coal->id, rc,
do_dealloc ? ", will deallocate now" : "");
#endif
if (do_dealloc) {
assert(coal->termrequested);
assert(coal->terminated);
assert(coal->active_count == 0);
assert(coal->reaped);
ledger_dereference(coal->ledger);
lck_mtx_destroy(&coal->lock, &coalitions_lck_grp);
zfree(coalition_zone, coal);
}
}
coalition_t
coalition_find_by_id(uint64_t cid)
{
if (cid == 0) {
return COALITION_NULL;
}
lck_mtx_lock(&coalitions_list_lock);
coalition_t coal = coalition_find_by_id_internal(cid);
if (coal == COALITION_NULL) {
lck_mtx_unlock(&coalitions_list_lock);
return COALITION_NULL;
}
coalition_lock(coal);
if (coal->reaped) {
coalition_unlock(coal);
lck_mtx_unlock(&coalitions_list_lock);
return COALITION_NULL;
}
if (coal->ref_count == 0) {
panic("resurrecting coalition %p id %llu, active_count = %u\n",
coal, coal->id, coal->active_count);
}
coal->ref_count++;
#if COALITION_DEBUG
uint32_t rc = coal->ref_count;
#endif
coalition_unlock(coal);
lck_mtx_unlock(&coalitions_list_lock);
#if COALITION_DEBUG
printf("%s: coal %llu ref_count++ -> %u\n", __func__, coal->id, rc);
#endif
return coal;
}
coalition_t
coalition_find_and_activate_by_id(uint64_t cid)
{
if (cid == 0) {
return COALITION_NULL;
}
lck_mtx_lock(&coalitions_list_lock);
coalition_t coal = coalition_find_by_id_internal(cid);
if (coal == COALITION_NULL) {
lck_mtx_unlock(&coalitions_list_lock);
return COALITION_NULL;
}
coalition_lock(coal);
if (coal->reaped || coal->terminated) {
coalition_unlock(coal);
lck_mtx_unlock(&coalitions_list_lock);
return COALITION_NULL;
}
if (coal->ref_count == 0) {
panic("resurrecting coalition %p id %llu, active_count = %u\n",
coal, coal->id, coal->active_count);
}
coal->ref_count++;
coal->active_count++;
#if COALITION_DEBUG
uint32_t rc = coal->ref_count;
uint32_t ac = coal->active_count;
#endif
coalition_unlock(coal);
lck_mtx_unlock(&coalitions_list_lock);
#if COALITION_DEBUG
printf("%s: coal %llu ref_count++ -> %u, active_count++ -> %u\n",
__func__, coal->id, rc, ac);
#endif
return coal;
}
uint64_t
coalition_id(coalition_t coal)
{
return coal->id;
}
uint64_t
task_coalition_id(task_t task)
{
return task->coalition->id;
}
boolean_t
coalition_is_privileged(coalition_t coal)
{
return coal->privileged || unrestrict_coalition_syscalls;
}
boolean_t
task_is_in_privileged_coalition(task_t task)
{
return task->coalition->privileged || unrestrict_coalition_syscalls;
}
ledger_t
coalition_get_ledger(coalition_t coal)
{
return coal->ledger;
}
kern_return_t
coalition_extend_active(coalition_t coal)
{
coalition_lock(coal);
if (coal->reaped) {
panic("cannot make a reaped coalition active again");
}
if (coal->terminated) {
coalition_unlock(coal);
return KERN_TERMINATED;
}
assert(coal->active_count > 0);
coal->active_count++;
coalition_unlock(coal);
return KERN_SUCCESS;
}
void
coalition_remove_active(coalition_t coal)
{
coalition_lock(coal);
assert(!coal->reaped);
assert(coal->active_count > 0);
coal->active_count--;
boolean_t do_notify = FALSE;
uint64_t notify_id = 0;
uint32_t notify_flags = 0;
if (coal->termrequested && coal->active_count == 0) {
assert(!coal->terminated);
coal->terminated = TRUE;
assert(!coal->notified);
coal->notified = TRUE;
do_notify = TRUE;
notify_id = coal->id;
notify_flags = 0;
}
coalition_unlock(coal);
if (do_notify) {
coalition_notify_user(notify_id, notify_flags);
}
}
kern_return_t
coalition_default_adopt_task(task_t task)
{
kern_return_t kr;
kr = coalition_adopt_task(default_coalition, task);
if (kr != KERN_SUCCESS) {
panic("failed to adopt task %p into default coalition: %d", task, kr);
}
return kr;
}
kern_return_t
coalition_adopt_task(coalition_t coal, task_t task)
{
if (task->coalition) {
return KERN_ALREADY_IN_SET;
}
coalition_lock(coal);
if (coal->reaped || coal->terminated) {
coalition_unlock(coal);
return KERN_TERMINATED;
}
coal->active_count++;
coal->ref_count++;
task->coalition = coal;
queue_enter(&coal->tasks, task, task_t, coalition_tasks);
coal->task_count++;
if(coal->task_count < coal->dead_task_count) {
panic("%s: coalition %p id %llu task_count < dead_task_count", __func__, coal, coal->id);
}
if (coal->task_count - coal->dead_task_count == 1) {
coal->last_became_nonempty_time = mach_absolute_time();
}
#if COALITION_DEBUG
uint32_t rc = coal->ref_count;
#endif
coalition_unlock(coal);
#if COALITION_DEBUG
if (rc) {
printf("%s: coal %llu ref_count++ -> %u\n", __func__, coal->id, rc);
}
#endif
return KERN_SUCCESS;
}
kern_return_t
coalition_remove_task(task_t task)
{
coalition_t coal = task->coalition;
assert(coal);
coalition_lock(coal);
queue_remove(&coal->tasks, task, task_t, coalition_tasks);
coal->dead_task_count++;
if(coal->task_count < coal->dead_task_count) {
panic("%s: coalition %p id %llu task_count < dead_task_count", __func__, coal, coal->id);
}
if (coal->task_count - coal->dead_task_count == 0) {
uint64_t last_time_nonempty = mach_absolute_time() - coal->last_became_nonempty_time;
coal->last_became_nonempty_time = 0;
coal->time_nonempty += last_time_nonempty;
}
ledger_rollup(coal->ledger, task->ledger);
coal->bytesread += task->task_io_stats->disk_reads.size;
coal->byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
coal->gpu_time += task_gpu_utilisation(task);
coalition_unlock(coal);
coalition_remove_active(coal);
return KERN_SUCCESS;
}
kern_return_t
coalition_request_terminate_internal(coalition_t coal)
{
if (coal == default_coalition) {
return KERN_DEFAULT_SET;
}
coalition_lock(coal);
if (coal->reaped) {
coalition_unlock(coal);
return KERN_INVALID_NAME;
}
if (coal->terminated || coal->termrequested) {
coalition_unlock(coal);
return KERN_TERMINATED;
}
coal->termrequested = TRUE;
boolean_t do_notify = FALSE;
uint64_t note_id = 0;
uint32_t note_flags = 0;
if (coal->active_count == 0) {
assert(!coal->terminated);
coal->terminated = TRUE;
assert(!coal->notified);
coal->notified = TRUE;
do_notify = TRUE;
note_id = coal->id;
note_flags = 0;
}
coalition_unlock(coal);
if (do_notify) {
coalition_notify_user(note_id, note_flags);
}
return KERN_SUCCESS;
}
kern_return_t
coalition_reap_internal(coalition_t coal)
{
if (coal == default_coalition) {
return KERN_DEFAULT_SET;
}
coalition_lock(coal);
if (coal->reaped) {
coalition_unlock(coal);
return KERN_TERMINATED;
}
if (!coal->terminated) {
coalition_unlock(coal);
return KERN_FAILURE;
}
assert(coal->termrequested);
if (coal->active_count > 0) {
coalition_unlock(coal);
return KERN_FAILURE;
}
coal->reaped = TRUE;
assert(coal->ref_count > 2);
coalition_unlock(coal);
lck_mtx_lock(&coalitions_list_lock);
coalition_count--;
queue_remove(&coalitions, coal, coalition_t, coalitions);
lck_mtx_unlock(&coalitions_list_lock);
coalition_release(coal);
coalition_release(coal);
return KERN_SUCCESS;
}
void
coalition_init(void)
{
coalition_zone = zinit(
sizeof(struct coalition),
CONFIG_COALITION_MAX * sizeof(struct coalition),
COALITION_CHUNK * sizeof(struct coalition),
"coalitions");
zone_change(coalition_zone, Z_NOENCRYPT, TRUE);
queue_init(&coalitions);
if (!PE_parse_boot_argn("unrestrict_coalition_syscalls", &unrestrict_coalition_syscalls,
sizeof (unrestrict_coalition_syscalls))) {
unrestrict_coalition_syscalls = 0;
}
lck_grp_attr_setdefault(&coalitions_lck_grp_attr);
lck_grp_init(&coalitions_lck_grp, "coalition", &coalitions_lck_grp_attr);
lck_attr_setdefault(&coalitions_lck_attr);
lck_mtx_init(&coalitions_list_lock, &coalitions_lck_grp, &coalitions_lck_attr);
init_task_ledgers();
kern_return_t kr = coalition_create_internal(&default_coalition, TRUE);
if (kr != KERN_SUCCESS) {
panic("%s: could not create default coalition: %d", __func__, kr);
}
}