#include <kern/kern_types.h>
#include <kern/ledger.h>
#include <kern/kalloc.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/coalition.h>
#include <kern/processor.h>
#include <kern/machine.h>
#include <kern/queue.h>
#include <kern/policy_internal.h>
#include <sys/errno.h>
#include <libkern/OSAtomic.h>
#include <mach/mach_types.h>
#include <os/overflow.h>
#include <vm/pmap.h>
#define LF_ENTRY_ACTIVE 0x0001
#define LF_WAKE_NEEDED 0x0100
#define LF_WAKE_INPROGRESS 0x0200
#define LF_REFILL_SCHEDULED 0x0400
#define LF_REFILL_INPROGRESS 0x0800
#define LF_CALLED_BACK 0x1000
#define LF_WARNED 0x2000
#define LF_TRACKING_MAX 0x4000
#define LF_PANIC_ON_NEGATIVE 0x8000
#define LF_TRACK_CREDIT_ONLY 0x10000
#define ENTRY_VALID(l, e) \
(((l) != NULL) && ((e) >= 0) && ((e) < (l)->l_size) && \
(((l)->l_entries[e].le_flags & LF_ENTRY_ACTIVE) == LF_ENTRY_ACTIVE))
#define ASSERT(a) assert(a)
#ifdef LEDGER_DEBUG
int ledger_debug = 0;
#define lprintf(a) if (ledger_debug) { \
printf("%lld ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \
printf a ; \
}
#else
#define lprintf(a)
#endif
struct ledger_callback {
ledger_callback_t lc_func;
const void *lc_param0;
const void *lc_param1;
};
struct entry_template {
char et_key[LEDGER_NAME_MAX];
char et_group[LEDGER_NAME_MAX];
char et_units[LEDGER_NAME_MAX];
uint32_t et_flags;
struct ledger_callback *et_callback;
};
LCK_GRP_DECLARE(ledger_lck_grp, "ledger");
os_refgrp_decl(static, ledger_refgrp, "ledger", NULL);
struct ledger_template {
const char *lt_name;
int lt_refs;
int lt_cnt;
int lt_table_size;
volatile uint32_t lt_inuse;
lck_mtx_t lt_lock;
zone_t lt_zone;
bool lt_initialized;
struct entry_template *lt_entries;
};
#define template_lock(template) lck_mtx_lock(&(template)->lt_lock)
#define template_unlock(template) lck_mtx_unlock(&(template)->lt_lock)
#define TEMPLATE_INUSE(s, t) { \
s = splsched(); \
while (OSCompareAndSwap(0, 1, &((t)->lt_inuse))) \
; \
}
#define TEMPLATE_IDLE(s, t) { \
(t)->lt_inuse = 0; \
splx(s); \
}
static int ledger_cnt = 0;
static uint32_t ledger_check_needblock(ledger_t l, uint64_t now);
static kern_return_t ledger_perform_blocking(ledger_t l);
static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
static void ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
int entry, struct ledger_entry *le);
#if 0
static void
debug_callback(const void *p0, __unused const void *p1)
{
printf("ledger: resource exhausted [%s] for task %p\n",
(const char *)p0, p1);
}
#endif
static uint64_t
abstime_to_nsecs(uint64_t abstime)
{
uint64_t nsecs;
absolutetime_to_nanoseconds(abstime, &nsecs);
return nsecs;
}
static uint64_t
nsecs_to_abstime(uint64_t nsecs)
{
uint64_t abstime;
nanoseconds_to_absolutetime(nsecs, &abstime);
return abstime;
}
ledger_template_t
ledger_template_create(const char *name)
{
ledger_template_t template;
template = (ledger_template_t)kalloc(sizeof(*template));
if (template == NULL) {
return NULL;
}
template->lt_name = name;
template->lt_refs = 1;
template->lt_cnt = 0;
template->lt_table_size = 1;
template->lt_inuse = 0;
template->lt_zone = NULL;
lck_mtx_init(&template->lt_lock, &ledger_lck_grp, LCK_ATTR_NULL);
template->lt_entries = (struct entry_template *)
kalloc(sizeof(struct entry_template) * template->lt_table_size);
if (template->lt_entries == NULL) {
kfree(template, sizeof(*template));
template = NULL;
}
return template;
}
ledger_template_t
ledger_template_copy(ledger_template_t template, const char *name)
{
struct entry_template * new_entries = NULL;
ledger_template_t new_template = ledger_template_create(name);
if (new_template == NULL) {
return new_template;
}
template_lock(template);
assert(template->lt_initialized);
new_entries = (struct entry_template *)
kalloc(sizeof(struct entry_template) * template->lt_table_size);
if (new_entries) {
bcopy(template->lt_entries, new_entries, sizeof(struct entry_template) * template->lt_table_size);
kfree(new_template->lt_entries, sizeof(struct entry_template) * new_template->lt_table_size);
new_template->lt_entries = new_entries;
new_template->lt_table_size = template->lt_table_size;
new_template->lt_cnt = template->lt_cnt;
} else {
ledger_template_dereference(new_template);
new_template = NULL;
}
template_unlock(template);
return new_template;
}
void
ledger_template_dereference(ledger_template_t template)
{
template_lock(template);
template->lt_refs--;
template_unlock(template);
if (template->lt_refs == 0) {
kfree(template->lt_entries, sizeof(struct entry_template) * template->lt_table_size);
lck_mtx_destroy(&template->lt_lock, &ledger_lck_grp);
kfree(template, sizeof(*template));
}
}
int
ledger_entry_add(ledger_template_t template, const char *key,
const char *group, const char *units)
{
int idx;
struct entry_template *et;
if ((key == NULL) || (strlen(key) >= LEDGER_NAME_MAX) || (template->lt_zone != NULL)) {
return -1;
}
template_lock(template);
if (template->lt_cnt == template->lt_table_size) {
struct entry_template *new_entries, *old_entries;
int old_cnt, old_sz, new_sz = 0;
spl_t s;
old_cnt = template->lt_table_size;
old_sz = old_cnt * (int)(sizeof(struct entry_template));
if (os_mul_overflow(old_sz, 2, &new_sz)) {
template_unlock(template);
return -1;
}
new_entries = kalloc(new_sz);
if (new_entries == NULL) {
template_unlock(template);
return -1;
}
memcpy(new_entries, template->lt_entries, old_sz);
memset(((char *)new_entries) + old_sz, 0, old_sz);
template->lt_table_size = old_cnt * 2;
old_entries = template->lt_entries;
TEMPLATE_INUSE(s, template);
template->lt_entries = new_entries;
TEMPLATE_IDLE(s, template);
kfree(old_entries, old_sz);
}
et = &template->lt_entries[template->lt_cnt];
strlcpy(et->et_key, key, LEDGER_NAME_MAX);
strlcpy(et->et_group, group, LEDGER_NAME_MAX);
strlcpy(et->et_units, units, LEDGER_NAME_MAX);
et->et_flags = LF_ENTRY_ACTIVE;
et->et_callback = NULL;
idx = template->lt_cnt++;
template_unlock(template);
return idx;
}
kern_return_t
ledger_entry_setactive(ledger_t ledger, int entry)
{
struct ledger_entry *le;
if ((ledger == NULL) || (entry < 0) || (entry >= ledger->l_size)) {
return KERN_INVALID_ARGUMENT;
}
le = &ledger->l_entries[entry];
if ((le->le_flags & LF_ENTRY_ACTIVE) == 0) {
flag_set(&le->le_flags, LF_ENTRY_ACTIVE);
}
return KERN_SUCCESS;
}
int
ledger_key_lookup(ledger_template_t template, const char *key)
{
int idx;
template_lock(template);
for (idx = 0; idx < template->lt_cnt; idx++) {
if (template->lt_entries != NULL &&
(strcmp(key, template->lt_entries[idx].et_key) == 0)) {
break;
}
}
if (idx >= template->lt_cnt) {
idx = -1;
}
template_unlock(template);
return idx;
}
void
ledger_template_complete(ledger_template_t template)
{
size_t ledger_size;
ledger_size = sizeof(struct ledger) + (template->lt_cnt * sizeof(struct ledger_entry));
template->lt_zone = zone_create(template->lt_name, ledger_size, ZC_NONE);
template->lt_initialized = true;
}
void
ledger_template_complete_secure_alloc(ledger_template_t template)
{
size_t ledger_size;
ledger_size = sizeof(struct ledger) + (template->lt_cnt * sizeof(struct ledger_entry));
pmap_ledger_alloc_init(ledger_size);
template->lt_initialized = true;
}
ledger_t
ledger_instantiate(ledger_template_t template, int entry_type)
{
ledger_t ledger;
size_t cnt;
int i;
template_lock(template);
template->lt_refs++;
cnt = template->lt_cnt;
template_unlock(template);
if (template->lt_zone) {
ledger = (ledger_t)zalloc(template->lt_zone);
} else {
ledger = pmap_ledger_alloc();
}
if (ledger == NULL) {
ledger_template_dereference(template);
return LEDGER_NULL;
}
ledger->l_template = template;
ledger->l_id = ledger_cnt++;
os_ref_init(&ledger->l_refs, &ledger_refgrp);
ledger->l_size = (int32_t)cnt;
template_lock(template);
assert(ledger->l_size <= template->lt_cnt);
for (i = 0; i < ledger->l_size; i++) {
struct ledger_entry *le = &ledger->l_entries[i];
struct entry_template *et = &template->lt_entries[i];
le->le_flags = et->et_flags;
if (entry_type == LEDGER_CREATE_INACTIVE_ENTRIES) {
flag_clear(&le->le_flags, LF_ENTRY_ACTIVE);
}
if (et->et_callback != NULL) {
flag_set(&le->le_flags, LEDGER_ACTION_CALLBACK);
}
le->le_credit = 0;
le->le_debit = 0;
le->le_limit = LEDGER_LIMIT_INFINITY;
le->le_warn_percent = LEDGER_PERCENT_NONE;
le->_le.le_refill.le_refill_period = 0;
le->_le.le_refill.le_last_refill = 0;
}
template_unlock(template);
return ledger;
}
static uint32_t
flag_set(volatile uint32_t *flags, uint32_t bit)
{
return OSBitOrAtomic(bit, flags);
}
static uint32_t
flag_clear(volatile uint32_t *flags, uint32_t bit)
{
return OSBitAndAtomic(~bit, flags);
}
void
ledger_reference(ledger_t ledger)
{
if (!LEDGER_VALID(ledger)) {
return;
}
os_ref_retain(&ledger->l_refs);
}
void
ledger_dereference(ledger_t ledger)
{
if (!LEDGER_VALID(ledger)) {
return;
}
if (os_ref_release(&ledger->l_refs) == 0) {
if (ledger->l_template->lt_zone) {
zfree(ledger->l_template->lt_zone, ledger);
} else {
pmap_ledger_free(ledger);
}
}
}
static inline int
warn_level_exceeded(struct ledger_entry *le)
{
ledger_amount_t balance;
if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
assert(le->le_debit == 0);
} else {
assert((le->le_credit >= 0) && (le->le_debit >= 0));
}
balance = le->le_credit - le->le_debit;
if (le->le_warn_percent != LEDGER_PERCENT_NONE &&
((balance > (le->le_limit * le->le_warn_percent) >> 16))) {
return 1;
}
return 0;
}
static inline int
limit_exceeded(struct ledger_entry *le)
{
ledger_amount_t balance;
if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
assert(le->le_debit == 0);
} else {
assert((le->le_credit >= 0) && (le->le_debit >= 0));
}
balance = le->le_credit - le->le_debit;
if ((le->le_limit <= 0) && (balance < le->le_limit)) {
return 1;
}
if ((le->le_limit > 0) && (balance > le->le_limit)) {
return 1;
}
return 0;
}
static inline struct ledger_callback *
entry_get_callback(ledger_t ledger, int entry)
{
struct ledger_callback *callback;
spl_t s;
TEMPLATE_INUSE(s, ledger->l_template);
callback = ledger->l_template->lt_entries[entry].et_callback;
TEMPLATE_IDLE(s, ledger->l_template);
return callback;
}
static inline void
ledger_limit_entry_wakeup(struct ledger_entry *le)
{
uint32_t flags;
if (!limit_exceeded(le)) {
flags = flag_clear(&le->le_flags, LF_CALLED_BACK);
while (le->le_flags & LF_WAKE_NEEDED) {
flag_clear(&le->le_flags, LF_WAKE_NEEDED);
thread_wakeup((event_t)le);
}
}
}
static void
ledger_refill(uint64_t now, ledger_t ledger, int entry)
{
uint64_t elapsed, period, periods;
struct ledger_entry *le;
ledger_amount_t balance, due;
assert(entry >= 0 && entry < ledger->l_size);
le = &ledger->l_entries[entry];
assert(le->le_limit != LEDGER_LIMIT_INFINITY);
if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
assert(le->le_debit == 0);
return;
}
if (flag_set(&le->le_flags, LF_REFILL_INPROGRESS) & LF_REFILL_INPROGRESS) {
return;
}
if (now <= le->_le.le_refill.le_last_refill) {
flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
return;
}
period = le->_le.le_refill.le_refill_period;
elapsed = now - le->_le.le_refill.le_last_refill;
if ((period == 0) || (elapsed < period)) {
flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
return;
}
periods = 0;
while ((periods < 2) && (elapsed > 0)) {
periods++;
elapsed -= period;
}
if (elapsed > 0) {
periods = (now - le->_le.le_refill.le_last_refill) / period;
}
balance = le->le_credit - le->le_debit;
due = periods * le->le_limit;
if (balance - due < 0) {
due = balance;
}
if (due < 0 && (le->le_flags & LF_PANIC_ON_NEGATIVE)) {
assertf(due >= 0, "now=%llu, ledger=%p, entry=%d, balance=%lld, due=%lld", now, ledger, entry, balance, due);
} else {
OSAddAtomic64(due, &le->le_debit);
assert(le->le_debit >= 0);
}
if (balance == due) {
le->_le.le_refill.le_last_refill = now;
} else {
le->_le.le_refill.le_last_refill += (le->_le.le_refill.le_refill_period * periods);
}
flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
lprintf(("Refill %lld %lld->%lld\n", periods, balance, balance - due));
if (!limit_exceeded(le)) {
ledger_limit_entry_wakeup(le);
}
}
void
ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
int entry, struct ledger_entry *le)
{
if (le->le_flags & LF_TRACKING_MAX) {
ledger_amount_t balance = le->le_credit - le->le_debit;
if (balance > le->_le._le_max.le_lifetime_max) {
le->_le._le_max.le_lifetime_max = balance;
}
#if CONFIG_LEDGER_INTERVAL_MAX
if (balance > le->_le._le_max.le_interval_max) {
le->_le._le_max.le_interval_max = balance;
}
#endif
}
if (le->le_flags & LF_REFILL_SCHEDULED) {
assert(!(le->le_flags & LF_TRACKING_MAX));
uint64_t now = mach_absolute_time();
if ((now - le->_le.le_refill.le_last_refill) > le->_le.le_refill.le_refill_period) {
ledger_refill(now, ledger, entry);
}
}
if (limit_exceeded(le)) {
if ((le->le_flags & LEDGER_ACTION_BLOCK) ||
(!(le->le_flags & LF_CALLED_BACK) &&
entry_get_callback(ledger, entry))) {
act_set_astledger_async(thread);
}
} else {
if (le->le_flags & LF_WAKE_NEEDED) {
ledger_limit_entry_wakeup(le);
}
if (le->le_flags & LEDGER_ACTION_CALLBACK) {
if (warn_level_exceeded(le)) {
if ((le->le_flags & LF_WARNED) == 0) {
act_set_astledger_async(thread);
}
} else {
if (le->le_flags & LF_WARNED) {
act_set_astledger_async(thread);
}
}
}
}
if ((le->le_flags & LF_PANIC_ON_NEGATIVE) &&
(le->le_credit < le->le_debit)) {
panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:%lld balance:%lld\n",
ledger, entry, le,
le->le_credit,
le->le_debit,
le->le_credit - le->le_debit);
}
}
void
ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry)
{
struct ledger_entry *le;
assert(entry > 0 && entry <= ledger->l_size);
le = &ledger->l_entries[entry];
ledger_entry_check_new_balance(thread, ledger, entry, le);
}
kern_return_t
ledger_credit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
{
ledger_amount_t old, new;
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry) || (amount < 0)) {
return KERN_INVALID_VALUE;
}
if (amount == 0) {
return KERN_SUCCESS;
}
le = &ledger->l_entries[entry];
old = OSAddAtomic64(amount, &le->le_credit);
new = old + amount;
lprintf(("%p Credit %lld->%lld\n", thread, old, new));
if (thread) {
ledger_entry_check_new_balance(thread, ledger, entry, le);
}
return KERN_SUCCESS;
}
kern_return_t
ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
{
return ledger_credit_thread(current_thread(), ledger, entry, amount);
}
kern_return_t
ledger_credit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
{
return ledger_credit_thread(NULL, ledger, entry, amount);
}
kern_return_t
ledger_rollup(ledger_t to_ledger, ledger_t from_ledger)
{
int i;
assert(to_ledger->l_template->lt_cnt == from_ledger->l_template->lt_cnt);
for (i = 0; i < to_ledger->l_size; i++) {
ledger_rollup_entry(to_ledger, from_ledger, i);
}
return KERN_SUCCESS;
}
kern_return_t
ledger_rollup_entry(ledger_t to_ledger, ledger_t from_ledger, int entry)
{
struct ledger_entry *from_le, *to_le;
assert(to_ledger->l_template->lt_cnt == from_ledger->l_template->lt_cnt);
if (ENTRY_VALID(from_ledger, entry) && ENTRY_VALID(to_ledger, entry)) {
from_le = &from_ledger->l_entries[entry];
to_le = &to_ledger->l_entries[entry];
OSAddAtomic64(from_le->le_credit, &to_le->le_credit);
OSAddAtomic64(from_le->le_debit, &to_le->le_debit);
}
return KERN_SUCCESS;
}
kern_return_t
ledger_zero_balance(ledger_t ledger, int entry)
{
struct ledger_entry *le;
ledger_amount_t debit, credit;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
le = &ledger->l_entries[entry];
top:
debit = le->le_debit;
credit = le->le_credit;
if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
assert(le->le_debit == 0);
if (!OSCompareAndSwap64(credit, 0, &le->le_credit)) {
goto top;
}
lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, 0));
} else if (credit > debit) {
if (!OSCompareAndSwap64(debit, credit, &le->le_debit)) {
goto top;
}
lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_debit, le->le_credit));
} else if (credit < debit) {
if (!OSCompareAndSwap64(credit, debit, &le->le_credit)) {
goto top;
}
lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, le->le_debit));
}
return KERN_SUCCESS;
}
kern_return_t
ledger_get_limit(ledger_t ledger, int entry, ledger_amount_t *limit)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
le = &ledger->l_entries[entry];
*limit = le->le_limit;
lprintf(("ledger_get_limit: %lld\n", *limit));
return KERN_SUCCESS;
}
kern_return_t
ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit,
uint8_t warn_level_percentage)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
lprintf(("ledger_set_limit: %lld\n", limit));
le = &ledger->l_entries[entry];
if (limit == LEDGER_LIMIT_INFINITY) {
ledger_disable_refill(ledger, entry);
}
le->le_limit = limit;
if (le->le_flags & LF_REFILL_SCHEDULED) {
assert(!(le->le_flags & LF_TRACKING_MAX));
le->_le.le_refill.le_last_refill = 0;
}
flag_clear(&le->le_flags, LF_CALLED_BACK);
flag_clear(&le->le_flags, LF_WARNED);
ledger_limit_entry_wakeup(le);
if (warn_level_percentage != 0) {
assert(warn_level_percentage <= 100);
assert(limit > 0);
assert(limit != LEDGER_LIMIT_INFINITY);
le->le_warn_percent = warn_level_percentage * (1u << 16) / 100;
} else {
le->le_warn_percent = LEDGER_PERCENT_NONE;
}
return KERN_SUCCESS;
}
#if CONFIG_LEDGER_INTERVAL_MAX
kern_return_t
ledger_get_interval_max(ledger_t ledger, int entry,
ledger_amount_t *max_interval_balance, int reset)
{
struct ledger_entry *le;
le = &ledger->l_entries[entry];
if (!ENTRY_VALID(ledger, entry) || !(le->le_flags & LF_TRACKING_MAX)) {
return KERN_INVALID_VALUE;
}
*max_interval_balance = le->_le._le_max.le_interval_max;
lprintf(("ledger_get_interval_max: %lld%s\n", *max_interval_balance,
(reset) ? " --> 0" : ""));
if (reset) {
le->_le._le_max.le_interval_max = 0;
}
return KERN_SUCCESS;
}
#endif
kern_return_t
ledger_get_lifetime_max(ledger_t ledger, int entry,
ledger_amount_t *max_lifetime_balance)
{
struct ledger_entry *le;
le = &ledger->l_entries[entry];
if (!ENTRY_VALID(ledger, entry) || !(le->le_flags & LF_TRACKING_MAX)) {
return KERN_INVALID_VALUE;
}
*max_lifetime_balance = le->_le._le_max.le_lifetime_max;
lprintf(("ledger_get_lifetime_max: %lld\n", *max_lifetime_balance));
return KERN_SUCCESS;
}
kern_return_t
ledger_track_maximum(ledger_template_t template, int entry,
__unused int period_in_secs)
{
template_lock(template);
if ((entry < 0) || (entry >= template->lt_cnt)) {
template_unlock(template);
return KERN_INVALID_VALUE;
}
if (template->lt_entries[entry].et_flags & LF_REFILL_SCHEDULED) {
return KERN_INVALID_VALUE;
}
template->lt_entries[entry].et_flags |= LF_TRACKING_MAX;
template_unlock(template);
return KERN_SUCCESS;
}
kern_return_t
ledger_panic_on_negative(ledger_template_t template, int entry)
{
template_lock(template);
if ((entry < 0) || (entry >= template->lt_cnt)) {
template_unlock(template);
return KERN_INVALID_VALUE;
}
template->lt_entries[entry].et_flags |= LF_PANIC_ON_NEGATIVE;
template_unlock(template);
return KERN_SUCCESS;
}
kern_return_t
ledger_track_credit_only(ledger_template_t template, int entry)
{
template_lock(template);
if ((entry < 0) || (entry >= template->lt_cnt)) {
template_unlock(template);
return KERN_INVALID_VALUE;
}
template->lt_entries[entry].et_flags |= LF_TRACK_CREDIT_ONLY;
template_unlock(template);
return KERN_SUCCESS;
}
kern_return_t
ledger_set_callback(ledger_template_t template, int entry,
ledger_callback_t func, const void *param0, const void *param1)
{
struct entry_template *et;
struct ledger_callback *old_cb, *new_cb;
if ((entry < 0) || (entry >= template->lt_cnt)) {
return KERN_INVALID_VALUE;
}
if (func) {
new_cb = (struct ledger_callback *)kalloc(sizeof(*new_cb));
new_cb->lc_func = func;
new_cb->lc_param0 = param0;
new_cb->lc_param1 = param1;
} else {
new_cb = NULL;
}
template_lock(template);
et = &template->lt_entries[entry];
old_cb = et->et_callback;
et->et_callback = new_cb;
template_unlock(template);
if (old_cb) {
kfree(old_cb, sizeof(*old_cb));
}
return KERN_SUCCESS;
}
kern_return_t
ledger_disable_callback(ledger_t ledger, int entry)
{
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
ledger->l_entries[entry].le_warn_percent = LEDGER_PERCENT_NONE;
flag_clear(&ledger->l_entries[entry].le_flags, LEDGER_ACTION_CALLBACK);
return KERN_SUCCESS;
}
kern_return_t
ledger_enable_callback(ledger_t ledger, int entry)
{
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
assert(entry_get_callback(ledger, entry) != NULL);
flag_set(&ledger->l_entries[entry].le_flags, LEDGER_ACTION_CALLBACK);
return KERN_SUCCESS;
}
kern_return_t
ledger_get_period(ledger_t ledger, int entry, uint64_t *period)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
le = &ledger->l_entries[entry];
*period = abstime_to_nsecs(le->_le.le_refill.le_refill_period);
lprintf(("ledger_get_period: %llx\n", *period));
return KERN_SUCCESS;
}
kern_return_t
ledger_set_period(ledger_t ledger, int entry, uint64_t period)
{
struct ledger_entry *le;
lprintf(("ledger_set_period: %llx\n", period));
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
le = &ledger->l_entries[entry];
assert(le->le_limit != LEDGER_LIMIT_INFINITY);
if (le->le_flags & LF_TRACKING_MAX) {
return KERN_INVALID_VALUE;
}
le->_le.le_refill.le_refill_period = nsecs_to_abstime(period);
le->_le.le_refill.le_last_refill = mach_absolute_time();
ledger_zero_balance(ledger, entry);
flag_set(&le->le_flags, LF_REFILL_SCHEDULED);
return KERN_SUCCESS;
}
kern_return_t
ledger_disable_refill(ledger_t ledger, int entry)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
le = &ledger->l_entries[entry];
flag_clear(&le->le_flags, LF_REFILL_SCHEDULED);
return KERN_SUCCESS;
}
kern_return_t
ledger_get_actions(ledger_t ledger, int entry, int *actions)
{
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
*actions = ledger->l_entries[entry].le_flags & LEDGER_ACTION_MASK;
lprintf(("ledger_get_actions: %#x\n", *actions));
return KERN_SUCCESS;
}
kern_return_t
ledger_set_action(ledger_t ledger, int entry, int action)
{
lprintf(("ledger_set_action: %#x\n", action));
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_VALUE;
}
flag_set(&ledger->l_entries[entry].le_flags, action);
return KERN_SUCCESS;
}
kern_return_t
ledger_debit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
{
struct ledger_entry *le;
ledger_amount_t old, new;
if (!ENTRY_VALID(ledger, entry) || (amount < 0)) {
return KERN_INVALID_ARGUMENT;
}
if (amount == 0) {
return KERN_SUCCESS;
}
le = &ledger->l_entries[entry];
if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
assert(le->le_debit == 0);
old = OSAddAtomic64(-amount, &le->le_credit);
new = old - amount;
} else {
old = OSAddAtomic64(amount, &le->le_debit);
new = old + amount;
}
lprintf(("%p Debit %lld->%lld\n", thread, old, new));
if (thread) {
ledger_entry_check_new_balance(thread, ledger, entry, le);
}
return KERN_SUCCESS;
}
kern_return_t
ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
{
return ledger_debit_thread(current_thread(), ledger, entry, amount);
}
kern_return_t
ledger_debit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
{
return ledger_debit_thread(NULL, ledger, entry, amount);
}
void
ledger_ast(thread_t thread)
{
struct ledger *l = thread->t_ledger;
struct ledger *thl;
struct ledger *coalition_ledger;
uint32_t block;
uint64_t now;
uint8_t task_flags;
uint8_t task_percentage;
uint64_t task_interval;
kern_return_t ret;
task_t task = thread->task;
lprintf(("Ledger AST for %p\n", thread));
ASSERT(task != NULL);
ASSERT(thread == current_thread());
top:
task_lock(task);
task_flags = task->rusage_cpu_flags;
task_percentage = task->rusage_cpu_perthr_percentage;
task_interval = task->rusage_cpu_perthr_interval;
task_unlock(task);
if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) != 0) &&
((thread->options & TH_OPT_PRVT_CPULIMIT) == 0)) {
uint8_t percentage;
uint64_t interval;
int action;
thread_get_cpulimit(&action, &percentage, &interval);
if (((thread->options & TH_OPT_PROC_CPULIMIT) == 0) ||
(interval != task_interval) || (percentage != task_percentage)) {
thread_set_cpulimit(THREAD_CPULIMIT_EXCEPTION, task_percentage, task_interval);
assert((thread->options & TH_OPT_PROC_CPULIMIT) != 0);
}
} else if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) &&
(thread->options & TH_OPT_PROC_CPULIMIT)) {
assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
thread_set_cpulimit(THREAD_CPULIMIT_DISABLE, 0, 0);
assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
}
if ((l == NULL) || !task->active || task->halting || !thread->active) {
return;
}
block = 0;
now = mach_absolute_time();
thl = thread->t_threadledger;
if (LEDGER_VALID(thl)) {
block |= ledger_check_needblock(thl, now);
}
block |= ledger_check_needblock(l, now);
coalition_ledger = coalition_ledger_get_from_task(task);
if (LEDGER_VALID(coalition_ledger)) {
block |= ledger_check_needblock(coalition_ledger, now);
}
ledger_dereference(coalition_ledger);
if (block) {
if (LEDGER_VALID(thl)) {
ret = ledger_perform_blocking(thl);
if (ret != KERN_SUCCESS) {
goto top;
}
}
ret = ledger_perform_blocking(l);
if (ret != KERN_SUCCESS) {
goto top;
}
}
}
static uint32_t
ledger_check_needblock(ledger_t l, uint64_t now)
{
int i;
uint32_t flags, block = 0;
struct ledger_entry *le;
struct ledger_callback *lc;
for (i = 0; i < l->l_size; i++) {
le = &l->l_entries[i];
lc = entry_get_callback(l, i);
if (limit_exceeded(le) == FALSE) {
if (le->le_flags & LEDGER_ACTION_CALLBACK) {
assert(lc != NULL);
if (warn_level_exceeded(le)) {
flags = flag_set(&le->le_flags, LF_WARNED);
if ((flags & LF_WARNED) == 0) {
lc->lc_func(LEDGER_WARNING_ROSE_ABOVE, lc->lc_param0, lc->lc_param1);
}
} else {
flags = flag_clear(&le->le_flags, LF_WARNED);
if (flags & LF_WARNED) {
lc->lc_func(LEDGER_WARNING_DIPPED_BELOW, lc->lc_param0, lc->lc_param1);
}
}
}
continue;
}
if (le->le_flags & LF_REFILL_SCHEDULED) {
assert(!(le->le_flags & LF_TRACKING_MAX));
if ((le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period) <= now) {
ledger_refill(now, l, i);
if (limit_exceeded(le) == FALSE) {
continue;
}
}
}
if (le->le_flags & LEDGER_ACTION_BLOCK) {
block = 1;
}
if ((le->le_flags & LEDGER_ACTION_CALLBACK) == 0) {
continue;
}
assert(lc != NULL);
flags = flag_set(&le->le_flags, LF_CALLED_BACK);
if (flags & LF_CALLED_BACK) {
continue;
}
lc->lc_func(FALSE, lc->lc_param0, lc->lc_param1);
}
return block;
}
static kern_return_t
ledger_perform_blocking(ledger_t l)
{
int i;
kern_return_t ret;
struct ledger_entry *le;
for (i = 0; i < l->l_size; i++) {
le = &l->l_entries[i];
if ((!limit_exceeded(le)) ||
((le->le_flags & LEDGER_ACTION_BLOCK) == 0)) {
continue;
}
assert(!(le->le_flags & LF_TRACKING_MAX));
ret = assert_wait_deadline(le, THREAD_INTERRUPTIBLE,
le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period);
if (ret != THREAD_WAITING) {
return KERN_SUCCESS;
}
flag_set(&le->le_flags, LF_WAKE_NEEDED);
ret = thread_block_reason(THREAD_CONTINUE_NULL, NULL,
AST_LEDGER);
if (ret != THREAD_AWAKENED) {
return KERN_SUCCESS;
}
return KERN_FAILURE;
}
return KERN_SUCCESS;
}
kern_return_t
ledger_get_entries(ledger_t ledger, int entry, ledger_amount_t *credit,
ledger_amount_t *debit)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_ARGUMENT;
}
le = &ledger->l_entries[entry];
*credit = le->le_credit;
*debit = le->le_debit;
return KERN_SUCCESS;
}
kern_return_t
ledger_reset_callback_state(ledger_t ledger, int entry)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_ARGUMENT;
}
le = &ledger->l_entries[entry];
flag_clear(&le->le_flags, LF_CALLED_BACK);
return KERN_SUCCESS;
}
kern_return_t
ledger_disable_panic_on_negative(ledger_t ledger, int entry)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_ARGUMENT;
}
le = &ledger->l_entries[entry];
flag_clear(&le->le_flags, LF_PANIC_ON_NEGATIVE);
return KERN_SUCCESS;
}
kern_return_t
ledger_get_panic_on_negative(ledger_t ledger, int entry, int *panic_on_negative)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_ARGUMENT;
}
le = &ledger->l_entries[entry];
if (le->le_flags & LF_PANIC_ON_NEGATIVE) {
*panic_on_negative = TRUE;
} else {
*panic_on_negative = FALSE;
}
return KERN_SUCCESS;
}
kern_return_t
ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
{
struct ledger_entry *le;
if (!ENTRY_VALID(ledger, entry)) {
return KERN_INVALID_ARGUMENT;
}
le = &ledger->l_entries[entry];
if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
assert(le->le_debit == 0);
} else {
assert((le->le_credit >= 0) && (le->le_debit >= 0));
}
*balance = le->le_credit - le->le_debit;
return KERN_SUCCESS;
}
int
ledger_template_info(void **buf, int *len)
{
struct ledger_template_info *lti;
struct entry_template *et;
int i;
ledger_t l;
l = current_task()->ledger;
if ((*len < 0) || (l == NULL)) {
return EINVAL;
}
if (*len > l->l_size) {
*len = l->l_size;
}
lti = kheap_alloc(KHEAP_DATA_BUFFERS,
(*len) * sizeof(struct ledger_template_info), Z_WAITOK);
if (lti == NULL) {
return ENOMEM;
}
*buf = lti;
template_lock(l->l_template);
et = l->l_template->lt_entries;
for (i = 0; i < *len; i++) {
memset(lti, 0, sizeof(*lti));
strlcpy(lti->lti_name, et->et_key, LEDGER_NAME_MAX);
strlcpy(lti->lti_group, et->et_group, LEDGER_NAME_MAX);
strlcpy(lti->lti_units, et->et_units, LEDGER_NAME_MAX);
et++;
lti++;
}
template_unlock(l->l_template);
return 0;
}
static void
ledger_fill_entry_info(struct ledger_entry *le,
struct ledger_entry_info *lei,
uint64_t now)
{
assert(le != NULL);
assert(lei != NULL);
memset(lei, 0, sizeof(*lei));
lei->lei_limit = le->le_limit;
lei->lei_credit = le->le_credit;
lei->lei_debit = le->le_debit;
lei->lei_balance = lei->lei_credit - lei->lei_debit;
lei->lei_refill_period = (le->le_flags & LF_REFILL_SCHEDULED) ?
abstime_to_nsecs(le->_le.le_refill.le_refill_period) : 0;
lei->lei_last_refill = abstime_to_nsecs(now - le->_le.le_refill.le_last_refill);
}
int
ledger_get_task_entry_info_multiple(task_t task, void **buf, int *len)
{
struct ledger_entry_info *lei;
struct ledger_entry *le;
uint64_t now = mach_absolute_time();
int i;
ledger_t l;
if ((*len < 0) || ((l = task->ledger) == NULL)) {
return EINVAL;
}
if (*len > l->l_size) {
*len = l->l_size;
}
lei = kheap_alloc(KHEAP_DATA_BUFFERS,
(*len) * sizeof(struct ledger_entry_info), Z_WAITOK);
if (lei == NULL) {
return ENOMEM;
}
*buf = lei;
le = l->l_entries;
for (i = 0; i < *len; i++) {
ledger_fill_entry_info(le, lei, now);
le++;
lei++;
}
return 0;
}
void
ledger_get_entry_info(ledger_t ledger,
int entry,
struct ledger_entry_info *lei)
{
uint64_t now = mach_absolute_time();
assert(ledger != NULL);
assert(lei != NULL);
if (entry >= 0 && entry < ledger->l_size) {
struct ledger_entry *le = &ledger->l_entries[entry];
ledger_fill_entry_info(le, lei, now);
}
}
int
ledger_info(task_t task, struct ledger_info *info)
{
ledger_t l;
if ((l = task->ledger) == NULL) {
return ENOENT;
}
memset(info, 0, sizeof(*info));
strlcpy(info->li_name, l->l_template->lt_name, LEDGER_NAME_MAX);
info->li_id = l->l_id;
info->li_entries = l->l_size;
return 0;
}
#ifdef LEDGER_DEBUG
int
ledger_limit(task_t task, struct ledger_limit_args *args)
{
ledger_t l;
int64_t limit;
int idx;
if ((l = task->ledger) == NULL) {
return EINVAL;
}
idx = ledger_key_lookup(l->l_template, args->lla_name);
if ((idx < 0) || (idx >= l->l_size)) {
return EINVAL;
}
if (idx == task_ledgers.cpu_time) {
int64_t nsecs;
if (args->lla_refill_period) {
args->lla_refill_period *= 1000000;
nsecs = args->lla_limit *
(args->lla_refill_period / 100);
lprintf(("CPU limited to %lld nsecs per second\n",
nsecs));
} else {
nsecs = args->lla_limit;
lprintf(("CPU limited to %lld nsecs\n", nsecs));
}
limit = nsecs_to_abstime(nsecs);
} else {
limit = args->lla_limit;
lprintf(("%s limited to %lld\n", args->lla_name, limit));
}
if (args->lla_refill_period > 0) {
ledger_set_period(l, idx, args->lla_refill_period);
}
ledger_set_limit(l, idx, limit);
flag_set(&l->l_entries[idx].le_flags, LEDGER_ACTION_BLOCK);
return 0;
}
#endif