#include <atm/atm_internal.h>
#include <mach/mach_types.h>
#include <mach/kern_return.h>
#include <ipc/ipc_port.h>
#include <mach/mach_vm.h>
#include <mach/vm_map.h>
#include <vm/vm_map.h>
#include <atm/atm_notification.h>
#include <mach/host_priv.h>
#include <mach/host_special_ports.h>
#include <kern/host.h>
#include <kern/kalloc.h>
#define MAX_ATM_VALUES (2 * 4096)
#define MAX_TRACE_BUFFER_SIZE (0x40000000)
#define MAX_MAILBOX_SIZE (8 * 4096)
#define ATM_VALUE_TO_HANDLE(x) (CAST_DOWN(atm_voucher_id_t, (x)))
#define HANDLE_TO_ATM_VALUE(x) (CAST_DOWN(atm_value_t, (x)))
#define ATM_MAX_HASH_TABLE_SIZE (256)
#define AID_HASH_MASK (0xFF)
#define AID_TO_HASH(x) ((x) & (AID_HASH_MASK))
#define ATM_LIST_DEAD_MAX 15
#define AID_ARRAY_COUNT_MAX (256)
struct atm_value_hash atm_value_hash_table[ATM_MAX_HASH_TABLE_SIZE];
extern int maxproc;
boolean_t disable_atm = FALSE;
#if DEVELOPMENT || DEBUG
queue_head_t atm_descriptors_list;
queue_head_t atm_values_list;
#endif
ipc_voucher_attr_control_t voucher_attr_control;
static zone_t atm_value_zone, atm_descriptors_zone, atm_link_objects_zone;
static aid_t get_aid();
static atm_value_t atm_value_alloc_init();
static void atm_value_dealloc(atm_value_t atm_value);
static void atm_hash_table_init();
static void atm_value_hash_table_insert(atm_value_t new_atm_value);
static void atm_value_hash_table_delete(atm_value_t atm_value);
static atm_value_t get_atm_value_from_aid(aid_t aid);
static void atm_value_get_ref(atm_value_t atm_value);
static kern_return_t atm_listener_insert(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset);
static void atm_listener_delete_all(atm_value_t atm_value);
static atm_task_descriptor_t atm_task_descriptor_alloc_init(mach_port_t trace_buffer,uint64_t buffer_size, void *mailbox_addr, uint64_t mailbox_array_size, __assert_only task_t task);
static void atm_descriptor_get_reference(atm_task_descriptor_t task_descriptor);
static void atm_task_descriptor_dealloc(atm_task_descriptor_t task_descriptor);
static mach_atm_subaid_t atm_get_min_sub_aid(atm_value_t atm_value);
static void
atm_get_min_sub_aid_array(aid_t *aid_array, mach_atm_subaid_t *subaid_array, uint32_t count) __unused;
static kern_return_t atm_value_unregister(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset);
static kern_return_t atm_listener_delete(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset);
static void atm_link_get_reference(atm_link_object_t link_object);
static void atm_link_dealloc(atm_link_object_t link_object);
kern_return_t atm_invoke_collection(atm_value_t atm_value, uint64_t sub_activity_id, uint32_t flags);
kern_return_t atm_send_user_notification(aid_t aid, uint64_t subaid, mach_port_t *buffers_array, uint64_t *sizes_array, mach_msg_type_number_t count, uint32_t flags);
kern_return_t
atm_release_value(
ipc_voucher_attr_manager_t __assert_only manager,
mach_voucher_attr_key_t __assert_only key,
mach_voucher_attr_value_handle_t value,
mach_voucher_attr_value_reference_t sync);
kern_return_t
atm_get_value(
ipc_voucher_attr_manager_t __assert_only manager,
mach_voucher_attr_key_t __assert_only key,
mach_voucher_attr_recipe_command_t command,
mach_voucher_attr_value_handle_array_t prev_values,
mach_msg_type_number_t __assert_only prev_value_count,
mach_voucher_attr_content_t recipe,
mach_voucher_attr_content_size_t recipe_size,
mach_voucher_attr_value_handle_t *out_value,
ipc_voucher_t *out_value_voucher);
kern_return_t
atm_extract_content(
ipc_voucher_attr_manager_t __assert_only manager,
mach_voucher_attr_key_t __assert_only key,
mach_voucher_attr_value_handle_array_t values,
mach_msg_type_number_t value_count,
mach_voucher_attr_recipe_command_t *out_command,
mach_voucher_attr_content_t out_recipe,
mach_voucher_attr_content_size_t *in_out_recipe_size);
kern_return_t
atm_command(
ipc_voucher_attr_manager_t __assert_only manager,
mach_voucher_attr_key_t __assert_only key,
mach_voucher_attr_value_handle_array_t values,
mach_msg_type_number_t value_count,
mach_voucher_attr_command_t command,
mach_voucher_attr_content_t in_content,
mach_voucher_attr_content_size_t in_content_size,
mach_voucher_attr_content_t out_content,
mach_voucher_attr_content_size_t *in_out_content_size);
void
atm_release(ipc_voucher_attr_manager_t __assert_only manager);
struct ipc_voucher_attr_manager atm_manager = {
.ivam_release_value = atm_release_value,
.ivam_get_value = atm_get_value,
.ivam_extract_content = atm_extract_content,
.ivam_command = atm_command,
.ivam_release = atm_release,
};
#if DEVELOPMENT || DEBUG
decl_lck_mtx_data(, atm_descriptors_list_lock);
decl_lck_mtx_data(, atm_values_list_lock);
lck_grp_t atm_dev_lock_grp;
lck_attr_t atm_dev_lock_attr;
lck_grp_attr_t atm_dev_lock_grp_attr;
#endif
extern vm_map_t kernel_map;
aid_t global_aid;
lck_grp_t atm_lock_grp;
lck_attr_t atm_lock_attr;
lck_grp_attr_t atm_lock_grp_attr;
void
atm_init()
{
kern_return_t kr = KERN_SUCCESS;
char temp_buf[20];
if ((PE_get_default("kern.disable_atm", temp_buf, sizeof(temp_buf))) ||
(PE_parse_boot_argn("-disable_atm", temp_buf, sizeof(temp_buf)))) {
disable_atm = TRUE;
}
atm_value_zone = zinit(sizeof(struct atm_value),
MAX_ATM_VALUES * sizeof(struct atm_value),
sizeof(struct atm_value),
"atm_values");
atm_descriptors_zone = zinit(sizeof(struct atm_task_descriptor),
MAX_ATM_VALUES * sizeof(struct atm_task_descriptor),
sizeof(struct atm_task_descriptor),
"atm_task_descriptors");
atm_link_objects_zone = zinit(sizeof(struct atm_link_object),
MAX_ATM_VALUES * sizeof(struct atm_link_object),
sizeof(struct atm_link_object),
"atm_link_objects");
lck_grp_attr_setdefault(&atm_lock_grp_attr);
lck_grp_init(&atm_lock_grp, "atm_lock", &atm_lock_grp_attr);
lck_attr_setdefault(&atm_lock_attr);
global_aid = 1;
atm_hash_table_init();
#if DEVELOPMENT || DEBUG
lck_grp_attr_setdefault(&atm_dev_lock_grp_attr);
lck_grp_init(&atm_dev_lock_grp, "atm_dev_lock", &atm_dev_lock_grp_attr);
lck_attr_setdefault(&atm_dev_lock_attr);
lck_mtx_init(&atm_descriptors_list_lock, &atm_dev_lock_grp, &atm_dev_lock_attr);
lck_mtx_init(&atm_values_list_lock, &atm_dev_lock_grp, &atm_dev_lock_attr);
queue_init(&atm_descriptors_list);
queue_init(&atm_values_list);
#endif
kr = ipc_register_well_known_mach_voucher_attr_manager(
&atm_manager,
0,
MACH_VOUCHER_ATTR_KEY_ATM,
&voucher_attr_control);
if (kr != KERN_SUCCESS )
panic("ATM subsystem initialization failed");
kprintf("ATM subsystem is initialized\n");
return ;
}
kern_return_t
atm_release_value(
ipc_voucher_attr_manager_t __assert_only manager,
mach_voucher_attr_key_t __assert_only key,
mach_voucher_attr_value_handle_t value,
mach_voucher_attr_value_reference_t sync)
{
atm_value_t atm_value = ATM_VALUE_NULL;
assert(MACH_VOUCHER_ATTR_KEY_ATM == key);
assert(manager == &atm_manager);
atm_value = HANDLE_TO_ATM_VALUE(value);
if (atm_value == VAM_DEFAULT_VALUE) {
return KERN_SUCCESS;
}
if (atm_value->sync != sync) {
return KERN_FAILURE;
}
atm_value_hash_table_delete(atm_value);
atm_value_dealloc(atm_value);
return KERN_SUCCESS;
}
kern_return_t
atm_get_value(
ipc_voucher_attr_manager_t __assert_only manager,
mach_voucher_attr_key_t __assert_only key,
mach_voucher_attr_recipe_command_t command,
mach_voucher_attr_value_handle_array_t prev_values,
mach_msg_type_number_t __assert_only prev_value_count,
mach_voucher_attr_content_t __unused recipe,
mach_voucher_attr_content_size_t __unused recipe_size,
mach_voucher_attr_value_handle_t *out_value,
ipc_voucher_t *out_value_voucher)
{
atm_value_t atm_value = ATM_VALUE_NULL;
mach_voucher_attr_value_handle_t atm_handle;
atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL;
task_t task;
mailbox_offset_t mailbox_offset;
natural_t i;
kern_return_t kr = KERN_SUCCESS;
assert(MACH_VOUCHER_ATTR_KEY_ATM == key);
assert(manager == &atm_manager);
*out_value_voucher = IPC_VOUCHER_NULL;
if (disable_atm)
return KERN_NOT_SUPPORTED;
switch (command) {
case MACH_VOUCHER_ATTR_ATM_REGISTER:
for (i = 0; i < prev_value_count; i++) {
atm_handle = prev_values[i];
atm_value = HANDLE_TO_ATM_VALUE(atm_handle);
if (atm_value == VAM_DEFAULT_VALUE)
continue;
task = current_task();
task_descriptor = task->atm_context;
if (task_descriptor != ATM_TASK_DESCRIPTOR_NULL) {
if (recipe_size != sizeof(mailbox_offset_t)) {
kr = KERN_INVALID_ARGUMENT;
break;
}
memcpy(&mailbox_offset, recipe, sizeof(mailbox_offset_t));
if (mailbox_offset > task_descriptor->mailbox_array_size) {
kr = KERN_INVALID_ARGUMENT;
break;
}
kr = atm_listener_insert(atm_value, task_descriptor, mailbox_offset);
if (kr != KERN_SUCCESS) {
break;
}
}
lck_mtx_lock(&atm_value->listener_lock);
atm_value->sync++;
lck_mtx_unlock(&atm_value->listener_lock);
*out_value = atm_handle;
return kr;
}
*out_value = ATM_VALUE_TO_HANDLE(VAM_DEFAULT_VALUE);
break;
case MACH_VOUCHER_ATTR_ATM_CREATE:
atm_value = atm_value_alloc_init();
atm_value_hash_table_insert(atm_value);
if (atm_value == ATM_VALUE_NULL) {
return KERN_RESOURCE_SHORTAGE;
}
*out_value = ATM_VALUE_TO_HANDLE(atm_value);
break;
case MACH_VOUCHER_ATTR_ATM_NULL:
default:
kr = KERN_INVALID_ARGUMENT;
break;
}
return kr;
}
kern_return_t
atm_extract_content(
ipc_voucher_attr_manager_t __assert_only manager,
mach_voucher_attr_key_t __assert_only key,
mach_voucher_attr_value_handle_array_t values,
mach_msg_type_number_t value_count,
mach_voucher_attr_recipe_command_t *out_command,
mach_voucher_attr_content_t out_recipe,
mach_voucher_attr_content_size_t *in_out_recipe_size)
{
atm_value_t atm_value;
mach_voucher_attr_value_handle_t atm_handle;
natural_t i;
assert(MACH_VOUCHER_ATTR_KEY_ATM == key);
assert(manager == &atm_manager);
for (i = 0; i < value_count; i++) {
atm_handle = values[i];
atm_value = HANDLE_TO_ATM_VALUE(atm_handle);
if (atm_value == VAM_DEFAULT_VALUE)
continue;
if (( sizeof(aid_t)) > *in_out_recipe_size) {
*in_out_recipe_size = 0;
return KERN_NO_SPACE;
}
memcpy(&out_recipe[0], &atm_value->aid, sizeof(aid_t));
*out_command = MACH_VOUCHER_ATTR_ATM_NULL;
*in_out_recipe_size = sizeof(aid_t);
return KERN_SUCCESS;
}
*in_out_recipe_size = 0;
return KERN_SUCCESS;
}
kern_return_t
atm_command(
ipc_voucher_attr_manager_t __assert_only manager,
mach_voucher_attr_key_t __assert_only key,
mach_voucher_attr_value_handle_array_t values,
mach_msg_type_number_t value_count,
mach_voucher_attr_command_t command,
mach_voucher_attr_content_t in_content,
mach_voucher_attr_content_size_t in_content_size,
mach_voucher_attr_content_t out_content,
mach_voucher_attr_content_size_t *out_content_size)
{
assert(MACH_VOUCHER_ATTR_KEY_ATM == key);
assert(manager == &atm_manager);
atm_value_t atm_value = ATM_VALUE_NULL;
natural_t i = 0;
aid_t *aid_array = NULL;
mach_atm_subaid_t *subaid_array = NULL;
uint32_t aid_array_count = 0;
atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL;
task_t task;
uint32_t collection_flags = ATM_ACTION_LOGFAIL;
kern_return_t kr = KERN_SUCCESS;
switch (command) {
case ATM_ACTION_COLLECT:
collection_flags = ATM_ACTION_COLLECT;
case ATM_ACTION_LOGFAIL: {
mach_atm_subaid_t sub_aid = 0;
for (i = 0; i < value_count; i++) {
atm_value = HANDLE_TO_ATM_VALUE(values[i]);
if (atm_value != VAM_DEFAULT_VALUE)
break;
}
if (atm_value == NULL) {
return KERN_FAILURE;
}
if (in_content == NULL || in_content_size < sizeof(mach_atm_subaid_t) ){
return KERN_INVALID_ARGUMENT;
}
sub_aid = *(mach_atm_subaid_t *)(void *)in_content;
*out_content_size = 0;
kr = atm_invoke_collection(atm_value, sub_aid, collection_flags);
break;
}
case ATM_FIND_MIN_SUB_AID:
if ((in_content_size/sizeof(aid_t)) > (*out_content_size/sizeof(mach_atm_subaid_t)))
return KERN_FAILURE;
aid_array_count = in_content_size / sizeof(aid_t);
if (aid_array_count > AID_ARRAY_COUNT_MAX)
return KERN_FAILURE;
aid_array = (aid_t *) kalloc(aid_array_count * sizeof(aid_t));
if (aid_array == NULL)
return KERN_NO_SPACE;
subaid_array = (mach_atm_subaid_t *) kalloc(aid_array_count * sizeof(mach_atm_subaid_t));
if (subaid_array == NULL) {
kfree(aid_array, aid_array_count * sizeof(aid_t));
return KERN_NO_SPACE;
}
memcpy(aid_array, in_content, aid_array_count * sizeof(aid_t));
atm_get_min_sub_aid_array(aid_array, subaid_array, aid_array_count);
memcpy(out_content, subaid_array, aid_array_count * sizeof(mach_atm_subaid_t));
*out_content_size = aid_array_count * sizeof(mach_atm_subaid_t);
kfree(aid_array, aid_array_count * sizeof(aid_t));
kfree(subaid_array, aid_array_count * sizeof(mach_atm_subaid_t));
kr = KERN_SUCCESS;
break;
case ATM_ACTION_UNREGISTER:
for (i = 0; i < value_count; i++) {
atm_value = HANDLE_TO_ATM_VALUE(values[i]);
if (atm_value != VAM_DEFAULT_VALUE)
break;
}
if (atm_value == NULL) {
return KERN_FAILURE;
}
if (in_content == NULL || in_content_size != sizeof(mailbox_offset_t)){
return KERN_INVALID_ARGUMENT;
}
mailbox_offset_t mailbox_offset;
memcpy(&mailbox_offset, in_content, sizeof(mailbox_offset_t));
task = current_task();
task_descriptor = task->atm_context;
kr = atm_value_unregister(atm_value, task_descriptor, mailbox_offset);
break;
default:
kr = KERN_INVALID_ARGUMENT;
break;
}
return kr;
}
void
atm_release(
ipc_voucher_attr_manager_t __assert_only manager)
{
assert(manager == &atm_manager);
}
kern_return_t
atm_invoke_collection(
atm_value_t atm_value,
subaid_t sub_activity_id,
uint32_t flags)
{
aid_t aid = atm_value->aid;
kern_return_t kr = KERN_SUCCESS;
uint32_t array_count = 0, i = 0, requestor_index = 0;
uint64_t *sizes_array = NULL;
atm_link_object_t link_object = NULL;
mach_port_t *mem_array = NULL;
boolean_t need_swap_first = FALSE;
atm_task_descriptor_t requesting_descriptor = current_task()->atm_context;
lck_mtx_lock(&atm_value->listener_lock);
array_count = atm_value->listener_count;
lck_mtx_unlock(&atm_value->listener_lock);
if (array_count == 0){
return KERN_SUCCESS;
}
mem_array = kalloc(sizeof(mach_port_t) * array_count);
if (mem_array == NULL){
return KERN_NO_SPACE;
}
sizes_array = kalloc(sizeof(uint64_t) * array_count);
if (sizes_array == NULL){
kfree(mem_array, sizeof(mach_port_t) * array_count);
return KERN_NO_SPACE;
}
lck_mtx_lock(&atm_value->listener_lock);
queue_iterate(&atm_value->listeners, link_object, atm_link_object_t, listeners_element) {
if (i >= array_count){
break;
}
if (!need_swap_first && requesting_descriptor == link_object->descriptor){
assert(requesting_descriptor != NULL);
requestor_index = i;
need_swap_first = TRUE;
}
sizes_array[i] = link_object->descriptor->trace_buffer_size;
mem_array[i] = ipc_port_copy_send(link_object->descriptor->trace_buffer);
if (!IPC_PORT_VALID(mem_array[i])){
mem_array[i] = NULL;
}
i++;
}
lck_mtx_unlock(&atm_value->listener_lock);
if (need_swap_first && requestor_index != 0){
assert(requestor_index < array_count);
mach_port_t tmp_port = mem_array[0];
uint64_t tmp_size = sizes_array[0];
mem_array[0] = mem_array[requestor_index];
sizes_array[0] = sizes_array[requestor_index];
mem_array[requestor_index] = tmp_port;
sizes_array[requestor_index] = tmp_size;
}
if (i > 0) {
kr = atm_send_user_notification(aid, sub_activity_id, mem_array, sizes_array, i, flags);
}
kfree(mem_array, sizeof(mach_port_t) * array_count);
kfree(sizes_array, sizeof(uint64_t) * array_count);
return kr;
}
kern_return_t
atm_send_user_notification(
aid_t aid,
subaid_t subaid,
mach_port_t *buffers_array,
uint64_t *sizes_array,
mach_msg_type_number_t count,
uint32_t flags)
{
mach_port_t user_port;
int error;
error = host_get_atm_notification_port(host_priv_self(), &user_port);
if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
return KERN_FAILURE;
}
return atm_collect_trace_info(user_port, aid, subaid, flags, buffers_array, count, sizes_array, count);
}
kern_return_t
atm_send_proc_inspect_notification(
task_t task,
int32_t traced_pid,
uint64_t traced_uniqueid)
{
mach_port_t user_port = MACH_PORT_NULL;
mach_port_t memory_port = MACH_PORT_NULL;
atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL;
uint64_t buffer_size = 0;
int error;
if (!task)
return KERN_INVALID_TASK;
task_lock(task);
if (task->atm_context){
task_descriptor = task->atm_context;
atm_descriptor_get_reference(task_descriptor);
}
task_unlock(task);
if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL){
return KERN_FAILURE;
}
memory_port = ipc_port_copy_send(task_descriptor->trace_buffer);
buffer_size = task_descriptor->trace_buffer_size;
atm_task_descriptor_dealloc(task_descriptor);
error = host_get_atm_notification_port(host_priv_self(), &user_port);
if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
ipc_port_release_send(memory_port);
return KERN_FAILURE;
}
return atm_inspect_process_buffer(user_port, traced_pid, traced_uniqueid, buffer_size, memory_port);
}
static atm_value_t
atm_value_alloc_init()
{
atm_value_t new_atm_value = ATM_VALUE_NULL;
new_atm_value = (atm_value_t) zalloc(atm_value_zone);
if (new_atm_value == ATM_VALUE_NULL)
panic("Ran out of ATM values structure.\n\n");
new_atm_value->aid = get_aid();
queue_init(&new_atm_value->listeners);
new_atm_value->sync = 1;
new_atm_value->listener_count = 0;
new_atm_value->reference_count = 1;
lck_mtx_init(&new_atm_value->listener_lock, &atm_lock_grp, &atm_lock_attr);
#if DEVELOPMENT || DEBUG
lck_mtx_lock(&atm_values_list_lock);
queue_enter(&atm_values_list, new_atm_value, atm_value_t, value_elt);
lck_mtx_unlock(&atm_values_list_lock);
#endif
return new_atm_value;
}
static aid_t
get_aid()
{
aid_t aid;
aid = (aid_t)OSIncrementAtomic64((SInt64 *)&global_aid);
return aid;
}
static void
atm_value_dealloc(atm_value_t atm_value)
{
lck_mtx_lock(&atm_value->listener_lock);
atm_value->reference_count--;
assert(atm_value->reference_count >= 0);
if (atm_value->reference_count > 0) {
lck_mtx_unlock(&atm_value->listener_lock);
return;
}
lck_mtx_unlock(&atm_value->listener_lock);
atm_listener_delete_all(atm_value);
lck_mtx_destroy(&atm_value->listener_lock, &atm_lock_grp);
#if DEVELOPMENT || DEBUG
lck_mtx_lock(&atm_values_list_lock);
queue_remove(&atm_values_list, atm_value, atm_value_t, value_elt);
lck_mtx_unlock(&atm_values_list_lock);
#endif
zfree(atm_value_zone, atm_value);
return;
}
static void
atm_hash_table_init()
{
int i;
for (i = 0; i < ATM_MAX_HASH_TABLE_SIZE; i++) {
queue_init(&atm_value_hash_table[i].hash_list);
lck_mtx_init(&atm_value_hash_table[i].hash_list_lock, &atm_lock_grp, &atm_lock_attr);
}
}
static void
atm_value_hash_table_insert(atm_value_t new_atm_value)
{
int hash_index;
atm_value_hash_t hash_list_head;
aid_t aid = new_atm_value->aid;
hash_index = AID_TO_HASH(aid);
hash_list_head = &atm_value_hash_table[hash_index];
lck_mtx_lock(&hash_list_head->hash_list_lock);
queue_enter(&hash_list_head->hash_list, new_atm_value, atm_value_t, vid_hash_elt);
lck_mtx_unlock(&hash_list_head->hash_list_lock);
}
static void
atm_value_hash_table_delete(atm_value_t atm_value)
{
int hash_index;
atm_value_hash_t hash_list_head;
aid_t aid = atm_value->aid;
hash_index = AID_TO_HASH(aid);
hash_list_head = &atm_value_hash_table[hash_index];
lck_mtx_lock(&hash_list_head->hash_list_lock);
queue_remove(&hash_list_head->hash_list, atm_value, atm_value_t, vid_hash_elt);
lck_mtx_unlock(&hash_list_head->hash_list_lock);
}
static atm_value_t
get_atm_value_from_aid(aid_t aid)
{
int hash_index;
atm_value_hash_t hash_list_head;
atm_value_t next;
hash_index = AID_TO_HASH(aid);
hash_list_head = &atm_value_hash_table[hash_index];
lck_mtx_lock(&hash_list_head->hash_list_lock);
queue_iterate(&hash_list_head->hash_list, next, atm_value_t, vid_hash_elt) {
if (next->aid == aid) {
atm_value_get_ref(next);
lck_mtx_unlock(&hash_list_head->hash_list_lock);
return (next);
}
}
lck_mtx_unlock(&hash_list_head->hash_list_lock);
return ATM_VALUE_NULL;
}
static void
atm_value_get_ref(atm_value_t atm_value)
{
lck_mtx_lock(&atm_value->listener_lock);
atm_value->reference_count++;
lck_mtx_unlock(&atm_value->listener_lock);
}
static kern_return_t
atm_listener_insert(
atm_value_t atm_value,
atm_task_descriptor_t task_descriptor,
mailbox_offset_t mailbox_offset)
{
atm_link_object_t new_link_object;
atm_link_object_t next;
void *mailbox = (void *)((char *)task_descriptor->mailbox_kernel_addr + mailbox_offset);
new_link_object = (atm_link_object_t) zalloc(atm_link_objects_zone);
new_link_object->descriptor = task_descriptor;
new_link_object->reference_count = 1;
new_link_object->flags = 0;
new_link_object->mailbox = mailbox;
atm_descriptor_get_reference(task_descriptor);
lck_mtx_lock(&atm_value->listener_lock);
queue_iterate(&atm_value->listeners, next, atm_link_object_t, listeners_element) {
if (next->descriptor == task_descriptor) {
next->mailbox = mailbox;
lck_mtx_unlock(&atm_value->listener_lock);
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_GETVALUE_INFO, (ATM_VALUE_REPLACED))) | DBG_FUNC_NONE,
atm_value, atm_value->aid, mailbox_offset, 0, 0);
atm_task_descriptor_dealloc(task_descriptor);
zfree(atm_link_objects_zone, new_link_object);
return KERN_SUCCESS;
}
}
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_GETVALUE_INFO, (ATM_VALUE_ADDED))) | DBG_FUNC_NONE,
atm_value, atm_value->aid, mailbox_offset, 0, 0);
queue_enter(&atm_value->listeners, new_link_object, atm_link_object_t, listeners_element);
atm_value->listener_count++;
lck_mtx_unlock(&atm_value->listener_lock);
return KERN_SUCCESS;
}
static void
atm_listener_delete_all(atm_value_t atm_value)
{
atm_link_object_t next;
while(!queue_empty(&atm_value->listeners)) {
queue_remove_first(&atm_value->listeners, next, atm_link_object_t, listeners_element);
atm_link_dealloc(next);
}
}
static kern_return_t
atm_listener_delete(
atm_value_t atm_value,
atm_task_descriptor_t task_descriptor,
mailbox_offset_t mailbox_offset)
{
queue_head_t free_listeners;
atm_link_object_t next, elem;
void *mailbox = (void *)((char *)task_descriptor->mailbox_kernel_addr + mailbox_offset);
kern_return_t kr = KERN_FAILURE;
queue_init(&free_listeners);
lck_mtx_lock(&atm_value->listener_lock);
next = (atm_link_object_t)(void *) queue_first(&atm_value->listeners);
while (!queue_end(&atm_value->listeners, (queue_entry_t)next)) {
elem = next;
next = (atm_link_object_t)(void *) queue_next(&next->listeners_element);
if (elem->descriptor == task_descriptor) {
if (elem->mailbox == mailbox) {
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_UNREGISTER_INFO,
(ATM_VALUE_UNREGISTERED))) | DBG_FUNC_NONE,
atm_value, atm_value->aid, mailbox_offset, 0, 0);
queue_remove(&atm_value->listeners, elem, atm_link_object_t, listeners_element);
queue_enter(&free_listeners, elem, atm_link_object_t, listeners_element);
atm_value->listener_count--;
kr = KERN_SUCCESS;
break;
} else {
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_UNREGISTER_INFO,
(ATM_VALUE_DIFF_MAILBOX))) | DBG_FUNC_NONE,
atm_value, atm_value->aid, 0, 0, 0);
kr = KERN_INVALID_VALUE;
break;
}
}
}
lck_mtx_unlock(&atm_value->listener_lock);
while(!queue_empty(&free_listeners)) {
queue_remove_first(&free_listeners, next, atm_link_object_t, listeners_element);
atm_link_dealloc(next);
}
return kr;
}
static atm_task_descriptor_t
atm_task_descriptor_alloc_init(
mach_port_t trace_buffer,
uint64_t buffer_size,
void * mailbox_addr,
uint64_t mailbox_array_size,
task_t __assert_only task)
{
atm_task_descriptor_t new_task_descriptor;
new_task_descriptor = (atm_task_descriptor_t) zalloc(atm_descriptors_zone);
new_task_descriptor->trace_buffer = trace_buffer;
new_task_descriptor->trace_buffer_size = buffer_size;
new_task_descriptor->mailbox_array_size = mailbox_array_size;
new_task_descriptor->mailbox_kernel_addr = mailbox_addr;
new_task_descriptor->reference_count = 1;
new_task_descriptor->flags = 0;
lck_mtx_init(&new_task_descriptor->lock, &atm_lock_grp, &atm_lock_attr);
#if DEVELOPMENT || DEBUG
new_task_descriptor->task = task;
lck_mtx_lock(&atm_descriptors_list_lock);
queue_enter(&atm_descriptors_list, new_task_descriptor, atm_task_descriptor_t, descriptor_elt);
lck_mtx_unlock(&atm_descriptors_list_lock);
#endif
return new_task_descriptor;
}
static void
atm_descriptor_get_reference(atm_task_descriptor_t task_descriptor)
{
lck_mtx_lock(&task_descriptor->lock);
task_descriptor->reference_count++;
lck_mtx_unlock(&task_descriptor->lock);
}
static void
atm_task_descriptor_dealloc(atm_task_descriptor_t task_descriptor)
{
lck_mtx_lock(&task_descriptor->lock);
task_descriptor->reference_count--;
assert(task_descriptor->reference_count >= 0);
if (task_descriptor->reference_count > 0) {
lck_mtx_unlock(&task_descriptor->lock);
return;
}
#if DEVELOPMENT || DEBUG
lck_mtx_lock(&atm_descriptors_list_lock);
queue_remove(&atm_descriptors_list, task_descriptor, atm_task_descriptor_t, descriptor_elt);
lck_mtx_unlock(&atm_descriptors_list_lock);
#endif
mach_vm_deallocate(kernel_map, (mach_vm_address_t)task_descriptor->mailbox_kernel_addr,
task_descriptor->mailbox_array_size);
task_descriptor->mailbox_kernel_addr = NULL;
task_descriptor->mailbox_array_size = 0;
ipc_port_release_send(task_descriptor->trace_buffer);
lck_mtx_unlock(&task_descriptor->lock);
lck_mtx_destroy(&task_descriptor->lock, &atm_lock_grp);
zfree(atm_descriptors_zone, task_descriptor);
return;
}
static void
atm_link_get_reference(atm_link_object_t link_object)
{
atm_link_object_reference_internal(link_object);
}
static void
atm_link_dealloc(atm_link_object_t link_object)
{
if (0 < atm_link_object_release_internal(link_object)) {
return;
}
assert(link_object->reference_count == 0);
atm_task_descriptor_dealloc(link_object->descriptor);
zfree(atm_link_objects_zone, link_object);
}
kern_return_t
atm_register_trace_memory(
task_t task,
uint64_t trace_buffer_address,
uint64_t buffer_size,
uint64_t mailbox_array_size)
{
atm_task_descriptor_t task_descriptor;
mach_port_t trace_buffer = MACH_PORT_NULL;
mach_vm_offset_t mailbox_kernel_ptr = 0;
kern_return_t kr = KERN_SUCCESS;
if (disable_atm)
return KERN_NOT_SUPPORTED;
if (task != current_task())
return KERN_INVALID_ARGUMENT;
if (task->atm_context != NULL
|| (void *)trace_buffer_address == NULL
|| buffer_size == 0
|| (buffer_size & PAGE_MASK) != 0
|| buffer_size > MAX_TRACE_BUFFER_SIZE
|| mailbox_array_size == 0
|| mailbox_array_size >= buffer_size
|| mailbox_array_size > MAX_MAILBOX_SIZE
|| mailbox_array_size & PAGE_MIN_MASK) {
return KERN_INVALID_ARGUMENT;
}
vm_map_t map = current_map();
memory_object_size_t mo_size = (memory_object_size_t) buffer_size;
kr = mach_make_memory_entry_64(map,
&mo_size,
(mach_vm_offset_t)trace_buffer_address,
VM_PROT_READ,
&trace_buffer,
NULL);
if (kr != KERN_SUCCESS)
return kr;
kr = mach_vm_map(kernel_map,
&mailbox_kernel_ptr,
mailbox_array_size,
0,
VM_FLAGS_ANYWHERE,
trace_buffer,
0,
FALSE,
VM_PROT_READ,
VM_PROT_READ,
VM_INHERIT_NONE
);
if (kr != KERN_SUCCESS){
ipc_port_release_send(trace_buffer);
return kr;
}
task_descriptor = atm_task_descriptor_alloc_init(trace_buffer, buffer_size, (void *)mailbox_kernel_ptr, mailbox_array_size, task);
if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL) {
ipc_port_release_send(trace_buffer);
mach_vm_deallocate(kernel_map, (mach_vm_address_t)mailbox_kernel_ptr, mailbox_array_size);
return KERN_NO_SPACE;
}
task_lock(task);
if (task->atm_context == NULL) {
task->atm_context = task_descriptor;
kr = KERN_SUCCESS;
} else {
kr = KERN_FAILURE;
}
task_unlock(task);
if (kr != KERN_SUCCESS) {
atm_task_descriptor_dealloc(task_descriptor);
}
return KERN_SUCCESS;
}
static void
atm_get_min_sub_aid_array(
aid_t *aid_array,
mach_atm_subaid_t *subaid_array,
uint32_t count)
{
atm_value_t atm_value;
uint32_t i;
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_CALLED))) | DBG_FUNC_START,
0, 0, 0, 0, 0);
for (i = 0; i < count; i++) {
atm_value = get_atm_value_from_aid(aid_array[i]);
if (atm_value == ATM_VALUE_NULL) {
subaid_array[i] = ATM_SUBAID32_MAX;
continue;
}
subaid_array[i] = atm_get_min_sub_aid(atm_value);
atm_value_dealloc(atm_value);
}
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_CALLED))) | DBG_FUNC_END,
count, 0, 0, 0, 0);
}
static mach_atm_subaid_t
atm_get_min_sub_aid(atm_value_t atm_value)
{
int32_t i = 0, j, freed_count = 0, dead_but_not_freed = 0;
int32_t listener_count;
atm_subaid32_t min_subaid = ATM_SUBAID32_MAX, subaid, max_subaid;
atm_link_object_t *link_object_array = NULL;
atm_link_object_t next, elem;
queue_head_t free_listeners;
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_LINK_LIST))) | DBG_FUNC_START,
0, 0, 0, 0, 0);
lck_mtx_lock(&atm_value->listener_lock);
listener_count = atm_value->listener_count;
lck_mtx_unlock(&atm_value->listener_lock);
link_object_array = (atm_link_object_t *) kalloc(sizeof(atm_link_object_t) * listener_count);
if (link_object_array == NULL) {
return 0;
}
lck_mtx_lock(&atm_value->listener_lock);
queue_iterate(&atm_value->listeners, next, atm_link_object_t, listeners_element) {
if (i >= listener_count)
break;
atm_link_get_reference(next);
link_object_array[i] = (atm_link_object_t)next;
i++;
}
lck_mtx_unlock(&atm_value->listener_lock);
j = i;
for (i = 0; i < j; i++) {
if (link_object_array[i]->descriptor->flags == ATM_TASK_DEAD)
continue;
subaid = *((atm_subaid32_t *)link_object_array[i]->mailbox);
if (subaid < min_subaid)
min_subaid = subaid;
}
for (i = j - 1; i >= 0; i--) {
if (link_object_array[i]->descriptor->flags == ATM_TASK_DEAD) {
if (dead_but_not_freed > ATM_LIST_DEAD_MAX) {
link_object_array[i]->flags = ATM_LINK_REMOVE;
freed_count++;
} else {
max_subaid = *(((atm_subaid32_t *)link_object_array[i]->mailbox) + 1);
if (max_subaid < min_subaid) {
link_object_array[i]->flags = ATM_LINK_REMOVE;
freed_count++;
} else {
dead_but_not_freed++;
}
}
}
atm_link_dealloc(link_object_array[i]);
link_object_array[i] = NULL;
}
assert((j - (freed_count + dead_but_not_freed)) <= maxproc);
kfree(link_object_array, (sizeof(atm_link_object_t) * listener_count));
lck_mtx_lock(&atm_value->listener_lock);
queue_init(&free_listeners);
next = (atm_link_object_t)(void *) queue_first(&atm_value->listeners);
while (!queue_end(&atm_value->listeners, (queue_entry_t)next)) {
elem = next;
next = (atm_link_object_t)(void *) queue_next(&next->listeners_element);
if (elem->flags == ATM_LINK_REMOVE) {
queue_remove(&atm_value->listeners, elem, atm_link_object_t, listeners_element);
queue_enter(&free_listeners, elem, atm_link_object_t, listeners_element);
atm_value->listener_count--;
}
}
lck_mtx_unlock(&atm_value->listener_lock);
while(!queue_empty(&free_listeners)) {
queue_remove_first(&free_listeners, next, atm_link_object_t, listeners_element);
atm_link_dealloc(next);
}
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_LINK_LIST))) | DBG_FUNC_END,
j, freed_count, dead_but_not_freed, 0, 0);
return CAST_DOWN(mach_atm_subaid_t, min_subaid);
}
static kern_return_t
atm_value_unregister(
atm_value_t atm_value,
atm_task_descriptor_t task_descriptor,
mailbox_offset_t mailbox_offset)
{
kern_return_t kr;
if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL)
return KERN_INVALID_ARGUMENT;
if (mailbox_offset > task_descriptor->mailbox_array_size)
return KERN_INVALID_ARGUMENT;
kr = atm_listener_delete(atm_value, task_descriptor, mailbox_offset);
return kr;
}
void
atm_task_descriptor_destroy(atm_task_descriptor_t task_descriptor)
{
lck_mtx_lock(&task_descriptor->lock);
task_descriptor->flags = ATM_TASK_DEAD;
lck_mtx_unlock(&task_descriptor->lock);
atm_task_descriptor_dealloc(task_descriptor);
}