#include <mach/mach_types.h>
#include <mach/kern_return.h>
#include <mach/port.h>
#include <mach/message.h>
#include <mach/mig_errors.h>
#include <mach/mach_traps.h>
#include <kern/kern_types.h>
#include <kern/assert.h>
#include <kern/counters.h>
#include <kern/cpu_number.h>
#include <kern/ipc_kobject.h>
#include <kern/ipc_mig.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/lock.h>
#include <kern/sched_prim.h>
#include <kern/exception.h>
#include <kern/misc_protos.h>
#include <kern/kalloc.h>
#include <kern/processor.h>
#include <kern/syscall_subr.h>
#include <vm/vm_map.h>
#include <ipc/ipc_types.h>
#include <ipc/ipc_kmsg.h>
#include <ipc/ipc_mqueue.h>
#include <ipc/ipc_object.h>
#include <ipc/ipc_notify.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_pset.h>
#include <ipc/ipc_space.h>
#include <ipc/ipc_entry.h>
#include <machine/machine_routines.h>
#include <security/mac_mach_internal.h>
#include <sys/kdebug.h>
#ifndef offsetof
#define offsetof(type, member) ((size_t)(&((type *)0)->member))
#endif
mach_msg_return_t mach_msg_send(
mach_msg_header_t *msg,
mach_msg_option_t option,
mach_msg_size_t send_size,
mach_msg_timeout_t send_timeout,
mach_port_name_t notify);
mach_msg_return_t mach_msg_receive(
mach_msg_header_t *msg,
mach_msg_option_t option,
mach_msg_size_t rcv_size,
mach_port_name_t rcv_name,
mach_msg_timeout_t rcv_timeout,
void (*continuation)(mach_msg_return_t),
mach_msg_size_t slist_size);
mach_msg_return_t mach_msg_receive_results(void);
mach_msg_return_t msg_receive_error(
ipc_kmsg_t kmsg,
mach_vm_address_t msg_addr,
mach_msg_option_t option,
mach_port_seqno_t seqno,
ipc_space_t space);
security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE;
audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE;
mach_msg_format_0_trailer_t trailer_template = {
MACH_MSG_TRAILER_FORMAT_0,
MACH_MSG_TRAILER_MINIMUM_SIZE,
0,
KERNEL_SECURITY_TOKEN_VALUE
};
mach_msg_return_t
mach_msg_send(
mach_msg_header_t *msg,
mach_msg_option_t option,
mach_msg_size_t send_size,
mach_msg_timeout_t send_timeout,
mach_port_name_t notify)
{
ipc_space_t space = current_space();
vm_map_t map = current_map();
ipc_kmsg_t kmsg;
mach_msg_return_t mr;
mach_msg_size_t msg_and_trailer_size;
mach_msg_max_trailer_t *trailer;
if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3))
return MACH_SEND_MSG_TOO_SMALL;
if (send_size > MACH_MSG_SIZE_MAX - MAX_TRAILER_SIZE)
return MACH_SEND_TOO_LARGE;
msg_and_trailer_size = send_size + MAX_TRAILER_SIZE;
kmsg = ipc_kmsg_alloc(msg_and_trailer_size);
if (kmsg == IKM_NULL)
return MACH_SEND_NO_BUFFER;
(void) memcpy((void *) kmsg->ikm_header, (const void *) msg, send_size);
kmsg->ikm_header->msgh_size = send_size;
trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + send_size);
trailer->msgh_sender = current_thread()->task->sec_token;
trailer->msgh_audit = current_thread()->task->audit_token;
trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
if (option & MACH_SEND_CANCEL) {
if (notify == MACH_PORT_NULL)
mr = MACH_SEND_INVALID_NOTIFY;
else
mr = ipc_kmsg_copyin(kmsg, space, map, notify);
} else
mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
return mr;
}
mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, send_timeout);
if (mr != MACH_MSG_SUCCESS) {
mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
(void) memcpy((void *) msg, (const void *) kmsg->ikm_header,
kmsg->ikm_header->msgh_size);
ipc_kmsg_free(kmsg);
}
return mr;
}
mach_msg_return_t
mach_msg_receive_results(void)
{
thread_t self = current_thread();
ipc_space_t space = current_space();
vm_map_t map = current_map();
ipc_object_t object = self->ith_object;
mach_msg_return_t mr = self->ith_state;
mach_vm_address_t msg_addr = self->ith_msg_addr;
mach_msg_option_t option = self->ith_option;
ipc_kmsg_t kmsg = self->ith_kmsg;
mach_port_seqno_t seqno = self->ith_seqno;
mach_msg_max_trailer_t *trailer;
ipc_object_release(object);
if (mr != MACH_MSG_SUCCESS) {
if (mr == MACH_RCV_TOO_LARGE ) {
if (option & MACH_RCV_LARGE) {
if (copyout((char *) &self->ith_msize,
msg_addr + offsetof(mach_msg_header_t, msgh_size),
sizeof(mach_msg_size_t)))
mr = MACH_RCV_INVALID_DATA;
goto out;
}
if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
== MACH_RCV_INVALID_DATA)
mr = MACH_RCV_INVALID_DATA;
}
goto out;
}
trailer = (mach_msg_max_trailer_t *)
((vm_offset_t)kmsg->ikm_header +
round_msg(kmsg->ikm_header->msgh_size));
if (option & MACH_RCV_TRAILER_MASK) {
trailer->msgh_seqno = seqno;
trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_AV)) {
#if CONFIG_MACF_MACH
if (kmsg->ikm_sender != NULL &&
IP_VALID(kmsg->ikm_header->msgh_remote_port) &&
mac_port_check_method(kmsg->ikm_sender,
&kmsg->ikm_sender->maclabel,
&((ipc_port_t)kmsg->ikm_header->msgh_remote_port)->ip_label,
kmsg->ikm_header->msgh_id) == 0)
trailer->msgh_ad = 1;
else
#endif
trailer->msgh_ad = 0;
}
if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) {
#if CONFIG_MACF_MACH
if (kmsg->ikm_sender != NULL) {
ipc_labelh_t lh = kmsg->ikm_sender->label;
kern_return_t kr;
ip_lock(lh->lh_port);
lh->lh_port->ip_mscount++;
lh->lh_port->ip_srights++;
ip_reference(lh->lh_port);
ip_unlock(lh->lh_port);
kr = ipc_object_copyout(space, (ipc_object_t)lh->lh_port,
MACH_MSG_TYPE_PORT_SEND, 0,
&trailer->msgh_labels.sender);
if (kr != KERN_SUCCESS) {
ip_lock(lh->lh_port);
ip_release(lh->lh_port);
ip_check_unlock(lh->lh_port);
trailer->msgh_labels.sender = 0;
}
} else {
trailer->msgh_labels.sender = 0;
}
#else
trailer->msgh_labels.sender = 0;
#endif
}
}
if (option & MACH_RCV_OVERWRITE) {
mach_msg_size_t slist_size = self->ith_scatter_list_size;
mach_msg_body_t *slist;
slist = ipc_kmsg_get_scatter(msg_addr, slist_size, kmsg);
mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist);
ipc_kmsg_free_scatter(slist, slist_size);
} else {
mr = ipc_kmsg_copyout(kmsg, space, map,
MACH_PORT_NULL, MACH_MSG_BODY_NULL);
}
if (mr != MACH_MSG_SUCCESS) {
if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
mr = MACH_RCV_INVALID_DATA;
}
else {
if (msg_receive_error(kmsg, msg_addr, option, seqno, space)
== MACH_RCV_INVALID_DATA)
mr = MACH_RCV_INVALID_DATA;
}
goto out;
}
mr = ipc_kmsg_put(msg_addr,
kmsg,
kmsg->ikm_header->msgh_size +
trailer->msgh_trailer_size);
out:
return mr;
}
mach_msg_return_t
mach_msg_receive(
mach_msg_header_t *msg,
mach_msg_option_t option,
mach_msg_size_t rcv_size,
mach_port_name_t rcv_name,
mach_msg_timeout_t rcv_timeout,
void (*continuation)(mach_msg_return_t),
mach_msg_size_t slist_size)
{
thread_t self = current_thread();
ipc_space_t space = current_space();
ipc_object_t object;
ipc_mqueue_t mqueue;
mach_msg_return_t mr;
mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
if (mr != MACH_MSG_SUCCESS) {
return mr;
}
self->ith_msg_addr = CAST_DOWN(mach_vm_address_t, msg);
self->ith_object = object;
self->ith_msize = rcv_size;
self->ith_option = option;
self->ith_scatter_list_size = slist_size;
self->ith_continuation = continuation;
ipc_mqueue_receive(mqueue, option, rcv_size, rcv_timeout, THREAD_ABORTSAFE);
if ((option & MACH_RCV_TIMEOUT) && rcv_timeout == 0)
thread_poll_yield(self);
return mach_msg_receive_results();
}
void
mach_msg_receive_continue(void)
{
thread_t self = current_thread();
(*self->ith_continuation)(mach_msg_receive_results());
}
#define ENABLE_HOTPATH 1
#if ENABLE_HOTPATH
#define HOTPATH_DEBUG 0
#if HOTPATH_DEBUG
#define HOT(expr) expr
unsigned int c_mmot_FIRST = 0;
unsigned int c_mmot_combined_S_R = 0;
unsigned int c_mach_msg_trap_switch_fast = 0;
unsigned int c_mmot_kernel_send = 0;
unsigned int c_mmot_cold_000 = 0;
unsigned int c_mmot_smallsendsize = 0;
unsigned int c_mmot_oddsendsize = 0;
unsigned int c_mmot_bigsendsize = 0;
unsigned int c_mmot_copyinmsg_fail = 0;
unsigned int c_mmot_g_slow_copyin3 = 0;
unsigned int c_mmot_cold_006 = 0;
unsigned int c_mmot_cold_007 = 0;
unsigned int c_mmot_cold_008 = 0;
unsigned int c_mmot_cold_009 = 0;
unsigned int c_mmot_cold_010 = 0;
unsigned int c_mmot_cold_012 = 0;
unsigned int c_mmot_cold_013 = 0;
unsigned int c_mmot_cold_014 = 0;
unsigned int c_mmot_cold_016 = 0;
unsigned int c_mmot_cold_018 = 0;
unsigned int c_mmot_cold_019 = 0;
unsigned int c_mmot_cold_020 = 0;
unsigned int c_mmot_cold_021 = 0;
unsigned int c_mmot_cold_022 = 0;
unsigned int c_mmot_cold_023 = 0;
unsigned int c_mmot_cold_024 = 0;
unsigned int c_mmot_cold_025 = 0;
unsigned int c_mmot_cold_026 = 0;
unsigned int c_mmot_cold_027 = 0;
unsigned int c_mmot_hot_fSR_ok = 0;
unsigned int c_mmot_cold_029 = 0;
unsigned int c_mmot_cold_030 = 0;
unsigned int c_mmot_cold_031 = 0;
unsigned int c_mmot_cold_032 = 0;
unsigned int c_mmot_cold_033 = 0;
unsigned int c_mmot_bad_rcvr = 0;
unsigned int c_mmot_rcvr_swapped = 0;
unsigned int c_mmot_rcvr_locked = 0;
unsigned int c_mmot_rcvr_tswapped = 0;
unsigned int c_mmot_rcvr_freed = 0;
unsigned int c_mmot_g_slow_copyout6 = 0;
unsigned int c_mmot_g_slow_copyout5 = 0;
unsigned int c_mmot_cold_037 = 0;
unsigned int c_mmot_cold_038 = 0;
unsigned int c_mmot_cold_039 = 0;
unsigned int c_mmot_g_slow_copyout4 = 0;
unsigned int c_mmot_g_slow_copyout3 = 0;
unsigned int c_mmot_hot_ok1 = 0;
unsigned int c_mmot_hot_ok2 = 0;
unsigned int c_mmot_hot_ok3 = 0;
unsigned int c_mmot_g_slow_copyout1 = 0;
unsigned int c_mmot_g_slow_copyout2 = 0;
unsigned int c_mmot_getback_fast_copyin = 0;
unsigned int c_mmot_cold_048 = 0;
unsigned int c_mmot_getback_FastSR = 0;
unsigned int c_mmot_cold_050 = 0;
unsigned int c_mmot_cold_051 = 0;
unsigned int c_mmot_cold_052 = 0;
unsigned int c_mmot_cold_053 = 0;
unsigned int c_mmot_fastkernelreply = 0;
unsigned int c_mmot_cold_055 = 0;
unsigned int c_mmot_getback_fast_put = 0;
unsigned int c_mmot_LAST = 0;
void db_mmot_zero_counters(void);
void db_mmot_show_counters(void);
void
db_mmot_zero_counters(void)
{
register unsigned int *ip = &c_mmot_FIRST;
while (ip <= &c_mmot_LAST)
*ip++ = 0;
}
void
db_mmot_show_counters(void)
{
#define xx(str) printf("%s: %d\n", # str, str);
xx(c_mmot_combined_S_R);
xx(c_mach_msg_trap_switch_fast);
xx(c_mmot_kernel_send);
xx(c_mmot_cold_000);
xx(c_mmot_smallsendsize);
xx(c_mmot_oddsendsize);
xx(c_mmot_bigsendsize);
xx(c_mmot_copyinmsg_fail);
xx(c_mmot_g_slow_copyin3);
xx(c_mmot_cold_006);
xx(c_mmot_cold_007);
xx(c_mmot_cold_008);
xx(c_mmot_cold_009);
xx(c_mmot_cold_010);
xx(c_mmot_cold_012);
xx(c_mmot_cold_013);
xx(c_mmot_cold_014);
xx(c_mmot_cold_016);
xx(c_mmot_cold_018);
xx(c_mmot_cold_019);
xx(c_mmot_cold_020);
xx(c_mmot_cold_021);
xx(c_mmot_cold_022);
xx(c_mmot_cold_023);
xx(c_mmot_cold_024);
xx(c_mmot_cold_025);
xx(c_mmot_cold_026);
xx(c_mmot_cold_027);
xx(c_mmot_hot_fSR_ok);
xx(c_mmot_cold_029);
xx(c_mmot_cold_030);
xx(c_mmot_cold_031);
xx(c_mmot_cold_032);
xx(c_mmot_cold_033);
xx(c_mmot_bad_rcvr);
xx(c_mmot_rcvr_swapped);
xx(c_mmot_rcvr_locked);
xx(c_mmot_rcvr_tswapped);
xx(c_mmot_rcvr_freed);
xx(c_mmot_g_slow_copyout6);
xx(c_mmot_g_slow_copyout5);
xx(c_mmot_cold_037);
xx(c_mmot_cold_038);
xx(c_mmot_cold_039);
xx(c_mmot_g_slow_copyout4);
xx(c_mmot_g_slow_copyout3);
xx(c_mmot_g_slow_copyout1);
xx(c_mmot_hot_ok3);
xx(c_mmot_hot_ok2);
xx(c_mmot_hot_ok1);
xx(c_mmot_g_slow_copyout2);
xx(c_mmot_getback_fast_copyin);
xx(c_mmot_cold_048);
xx(c_mmot_getback_FastSR);
xx(c_mmot_cold_050);
xx(c_mmot_cold_051);
xx(c_mmot_cold_052);
xx(c_mmot_cold_053);
xx(c_mmot_fastkernelreply);
xx(c_mmot_cold_055);
xx(c_mmot_getback_fast_put);
#undef xx
}
#else
unsigned int c_mmot_combined_S_R = 0;
unsigned int c_mach_msg_trap_switch_fast = 0;
unsigned int c_mmot_kernel_send = 0;
#define HOT(expr)
#endif
#if CONFIG_MACF_MACH
boolean_t enable_hotpath = FALSE;
#else
boolean_t enable_hotpath = TRUE;
#endif
#endif
mach_msg_return_t
mach_msg_overwrite_trap(
struct mach_msg_overwrite_trap_args *args)
{
mach_vm_address_t msg_addr = args->msg;
mach_msg_option_t option = args->option;
mach_msg_size_t send_size = args->send_size;
mach_msg_size_t rcv_size = args->rcv_size;
mach_port_name_t rcv_name = args->rcv_name;
mach_msg_timeout_t msg_timeout = args->timeout;
mach_port_name_t notify = args->notify;
mach_vm_address_t rcv_msg_addr = args->rcv_msg;
mach_msg_size_t scatter_list_size = 0;
__unused mach_port_seqno_t temp_seqno = 0;
mach_msg_return_t mr = MACH_MSG_SUCCESS;
#if ENABLE_HOTPATH
mach_msg_option_t masked_option =
option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
register mach_msg_header_t *hdr;
if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
thread_t self = current_thread();
mach_msg_format_0_trailer_t *trailer;
ipc_space_t space = self->task->itk_space;
ipc_kmsg_t kmsg;
register ipc_port_t dest_port;
ipc_object_t rcv_object;
ipc_mqueue_t rcv_mqueue;
mach_msg_size_t reply_size;
c_mmot_combined_S_R++;
mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
if (mr != KERN_SUCCESS) {
return mr;
}
hdr = kmsg->ikm_header;
trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
send_size);
switch (hdr->msgh_bits) {
case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
MACH_MSG_TYPE_MAKE_SEND_ONCE): {
register ipc_entry_t table;
register ipc_entry_num_t size;
register ipc_port_t reply_port;
{
register mach_port_index_t index;
register mach_port_gen_t gen;
{
register mach_port_name_t reply_name =
(mach_port_name_t)hdr->msgh_local_port;
if (reply_name != rcv_name) {
HOT(c_mmot_g_slow_copyin3++);
goto slow_copyin;
}
index = MACH_PORT_INDEX(reply_name);
gen = MACH_PORT_GEN(reply_name);
is_read_lock(space);
assert(space->is_active);
size = space->is_table_size;
table = space->is_table;
{
register ipc_entry_t entry;
register ipc_entry_bits_t bits;
if (index < size) {
entry = &table[index];
bits = entry->ie_bits;
if (IE_BITS_GEN(bits) != gen ||
(bits & IE_BITS_COLLISION)) {
entry = IE_NULL;
}
} else {
entry = IE_NULL;
bits = 0;
}
if (entry == IE_NULL) {
entry = ipc_entry_lookup(space, reply_name);
if (entry == IE_NULL) {
HOT(c_mmot_cold_006++);
goto abort_request_copyin;
}
bits = entry->ie_bits;
}
if (! (bits & MACH_PORT_TYPE_RECEIVE)) {
HOT(c_mmot_cold_007++);
goto abort_request_copyin;
}
reply_port = (ipc_port_t) entry->ie_object;
assert(reply_port != IP_NULL);
}
}
}
{
register mach_port_index_t index;
register mach_port_gen_t gen;
{
register mach_port_name_t dest_name =
(mach_port_name_t)hdr->msgh_remote_port;
index = MACH_PORT_INDEX(dest_name);
gen = MACH_PORT_GEN(dest_name);
{
register ipc_entry_t entry;
register ipc_entry_bits_t bits;
if (index < size) {
entry = &table[index];
bits = entry->ie_bits;
if (IE_BITS_GEN(bits) != gen ||
(bits & IE_BITS_COLLISION)) {
entry = IE_NULL;
}
} else {
entry = IE_NULL;
bits = 0;
}
if (entry == IE_NULL) {
entry = ipc_entry_lookup(space, dest_name);
if (entry == IE_NULL) {
HOT(c_mmot_cold_008++);
goto abort_request_copyin;
}
bits = entry->ie_bits;
}
if (! (bits & MACH_PORT_TYPE_SEND)) {
HOT(c_mmot_cold_009++);
goto abort_request_copyin;
}
assert(IE_BITS_UREFS(bits) > 0);
dest_port = (ipc_port_t) entry->ie_object;
assert(dest_port != IP_NULL);
}
}
}
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port)) {
ip_unlock(dest_port);
HOT(c_mmot_cold_010++);
goto abort_request_copyin;
}
is_read_unlock(space);
assert(dest_port->ip_srights > 0);
dest_port->ip_srights++;
ip_reference(dest_port);
assert(ip_active(reply_port));
assert(reply_port->ip_receiver_name ==
(mach_port_name_t)hdr->msgh_local_port);
assert(reply_port->ip_receiver == space);
reply_port->ip_sorights++;
ip_reference(reply_port);
hdr->msgh_bits =
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE);
hdr->msgh_remote_port = dest_port;
hdr->msgh_local_port = reply_port;
if (dest_port->ip_receiver == ipc_space_kernel) {
ip_unlock(reply_port);
assert(ip_active(dest_port));
dest_port->ip_messages.imq_seqno++;
ip_unlock(dest_port);
goto kernel_send;
}
if (imq_full(&dest_port->ip_messages)) {
HOT(c_mmot_cold_013++);
goto abort_request_send_receive;
}
rcv_object = (ipc_object_t) reply_port;
io_reference(rcv_object);
rcv_mqueue = &reply_port->ip_messages;
io_unlock(rcv_object);
HOT(c_mmot_hot_fSR_ok++);
goto fast_send_receive;
abort_request_copyin:
is_read_unlock(space);
goto slow_copyin;
abort_request_send_receive:
ip_unlock(dest_port);
ip_unlock(reply_port);
goto slow_send;
}
case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
register ipc_entry_num_t size;
register ipc_entry_t table;
{
register mach_port_name_t reply_name =
(mach_port_name_t)hdr->msgh_local_port;
if (reply_name != MACH_PORT_NULL) {
HOT(c_mmot_cold_018++);
goto slow_copyin;
}
}
is_write_lock(space);
assert(space->is_active);
size = space->is_table_size;
table = space->is_table;
{
register ipc_entry_t entry;
register mach_port_gen_t gen;
register mach_port_index_t index;
{
register mach_port_name_t dest_name =
(mach_port_name_t)hdr->msgh_remote_port;
index = MACH_PORT_INDEX(dest_name);
gen = MACH_PORT_GEN(dest_name);
}
if (index >= size) {
HOT(c_mmot_cold_019++);
goto abort_reply_dest_copyin;
}
entry = &table[index];
if ((entry->ie_bits & (IE_BITS_GEN_MASK|
IE_BITS_COLLISION|
MACH_PORT_TYPE_SEND_ONCE)) !=
(gen | MACH_PORT_TYPE_SEND_ONCE)) {
HOT(c_mmot_cold_020++);
goto abort_reply_dest_copyin;
}
assert(IE_BITS_TYPE(entry->ie_bits) ==
MACH_PORT_TYPE_SEND_ONCE);
assert(IE_BITS_UREFS(entry->ie_bits) == 1);
if (entry->ie_request != 0) {
HOT(c_mmot_cold_021++);
goto abort_reply_dest_copyin;
}
dest_port = (ipc_port_t) entry->ie_object;
assert(dest_port != IP_NULL);
ip_lock(dest_port);
if (!ip_active(dest_port)) {
ip_unlock(dest_port);
HOT(c_mmot_cold_022++);
goto abort_reply_dest_copyin;
}
assert(dest_port->ip_sorights > 0);
entry->ie_bits = gen;
entry->ie_next = table->ie_next;
table->ie_next = index;
entry->ie_object = IO_NULL;
}
hdr->msgh_bits =
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
0);
hdr->msgh_remote_port = dest_port;
assert(dest_port->ip_receiver != ipc_space_kernel);
{
register ipc_entry_t entry;
register ipc_entry_bits_t bits;
{
register mach_port_index_t index;
register mach_port_gen_t gen;
index = MACH_PORT_INDEX(rcv_name);
gen = MACH_PORT_GEN(rcv_name);
if (index < size) {
entry = &table[index];
bits = entry->ie_bits;
if (IE_BITS_GEN(bits) != gen ||
(bits & IE_BITS_COLLISION)) {
entry = IE_NULL;
}
} else {
entry = IE_NULL;
bits = 0;
}
if (entry == IE_NULL) {
entry = ipc_entry_lookup(space, rcv_name);
if (entry == IE_NULL) {
HOT(c_mmot_cold_024++);
goto abort_reply_rcv_copyin;
}
bits = entry->ie_bits;
}
}
#if 0
if (bits & MACH_PORT_TYPE_PORT_SET) {
register ipc_pset_t rcv_pset;
rcv_pset = (ipc_pset_t) entry->ie_object;
assert(rcv_pset != IPS_NULL);
ips_lock(rcv_pset);
assert(ips_active(rcv_pset));
rcv_object = (ipc_object_t) rcv_pset;
rcv_mqueue = &rcv_pset->ips_messages;
} else
#endif
if (bits & MACH_PORT_TYPE_RECEIVE) {
register ipc_port_t rcv_port;
rcv_port = (ipc_port_t) entry->ie_object;
assert(rcv_port != IP_NULL);
if (!ip_lock_try(rcv_port)) {
HOT(c_mmot_cold_025++);
goto abort_reply_rcv_copyin;
}
assert(ip_active(rcv_port));
if (rcv_port->ip_pset_count != 0) {
ip_unlock(rcv_port);
HOT(c_mmot_cold_026++);
goto abort_reply_rcv_copyin;
}
rcv_object = (ipc_object_t) rcv_port;
rcv_mqueue = &rcv_port->ip_messages;
} else {
HOT(c_mmot_cold_027++);
goto abort_reply_rcv_copyin;
}
}
is_write_unlock(space);
io_reference(rcv_object);
io_unlock(rcv_object);
HOT(c_mmot_hot_fSR_ok++);
goto fast_send_receive;
abort_reply_dest_copyin:
is_write_unlock(space);
HOT(c_mmot_cold_029++);
goto slow_copyin;
abort_reply_rcv_copyin:
ip_unlock(dest_port);
is_write_unlock(space);
HOT(c_mmot_cold_030++);
goto slow_send;
}
default:
HOT(c_mmot_cold_031++);
goto slow_copyin;
}
fast_send_receive:
assert(ip_active(dest_port));
assert(dest_port->ip_receiver != ipc_space_kernel);
assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
{
register ipc_mqueue_t dest_mqueue;
wait_queue_t waitq;
thread_t receiver;
spl_t s;
s = splsched();
dest_mqueue = &dest_port->ip_messages;
waitq = &dest_mqueue->imq_wait_queue;
imq_lock(dest_mqueue);
get_next_receiver:
receiver = wait_queue_wakeup64_identity_locked(waitq,
IPC_MQUEUE_RECEIVE,
THREAD_AWAKENED,
FALSE);
if ( receiver == THREAD_NULL ) {
imq_unlock(dest_mqueue);
splx(s);
ip_unlock(dest_port);
ipc_object_release(rcv_object);
HOT(c_mmot_cold_032++);
goto slow_send;
}
if (receiver->ith_msize <
ipc_kmsg_copyout_size(kmsg, receiver->map) +
REQUESTED_TRAILER_SIZE(receiver->ith_option))
{
receiver->ith_msize = kmsg->ikm_header->msgh_size;
receiver->ith_state = MACH_RCV_TOO_LARGE;
if ((receiver->ith_option & MACH_RCV_LARGE) != 0) {
receiver->ith_kmsg = IKM_NULL;
receiver->ith_seqno = 0;
thread_unlock(receiver);
HOT(c_mmot_bad_rcvr++);
goto get_next_receiver;
}
} else {
receiver->ith_state = MACH_MSG_SUCCESS;
}
c_mach_msg_trap_switch_fast++;
receiver->ith_kmsg = kmsg;
receiver->ith_seqno = dest_mqueue->imq_seqno++;
thread_unlock(receiver);
imq_unlock(dest_mqueue);
ip_unlock(dest_port);
current_task()->messages_sent++;
imq_lock(rcv_mqueue);
if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
imq_unlock(rcv_mqueue);
splx(s);
HOT(c_mmot_cold_033++);
goto slow_receive;
}
thread_lock(self);
self->ith_msg_addr = (rcv_msg_addr) ? rcv_msg_addr : msg_addr;
self->ith_object = rcv_object;
self->ith_msize = rcv_size;
self->ith_option = option;
self->ith_scatter_list_size = scatter_list_size;
self->ith_continuation = thread_syscall_return;
waitq = &rcv_mqueue->imq_wait_queue;
(void)wait_queue_assert_wait64_locked(waitq,
IPC_MQUEUE_RECEIVE,
THREAD_ABORTSAFE, 0,
self);
thread_unlock(self);
imq_unlock(rcv_mqueue);
splx(s);
thread_block(ipc_mqueue_receive_continue);
}
fast_copyout:
reply_size = send_size + trailer->msgh_trailer_size;
if (rcv_size < reply_size) {
HOT(c_mmot_g_slow_copyout6++);
goto slow_copyout;
}
switch (hdr->msgh_bits) {
case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
MACH_MSG_TYPE_PORT_SEND_ONCE): {
ipc_port_t reply_port =
(ipc_port_t) hdr->msgh_local_port;
mach_port_name_t dest_name, reply_name;
if (!IP_VALID(reply_port)) {
HOT(c_mmot_g_slow_copyout5++);
goto slow_copyout;
}
is_write_lock(space);
assert(space->is_active);
ip_lock(dest_port);
if (!ip_active(dest_port) ||
!ip_lock_try(reply_port)) {
HOT(c_mmot_cold_037++);
goto abort_request_copyout;
}
if (!ip_active(reply_port)) {
ip_unlock(reply_port);
HOT(c_mmot_cold_038++);
goto abort_request_copyout;
}
assert(reply_port->ip_sorights > 0);
ip_unlock(reply_port);
{
register ipc_entry_t table;
register ipc_entry_t entry;
register mach_port_index_t index;
table = space->is_table;
index = table->ie_next;
if (index == 0) {
HOT(c_mmot_cold_039++);
goto abort_request_copyout;
}
entry = &table[index];
table->ie_next = entry->ie_next;
entry->ie_request = 0;
{
register mach_port_gen_t gen;
assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
gen = IE_BITS_NEW_GEN(entry->ie_bits);
reply_name = MACH_PORT_MAKE(index, gen);
entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
}
assert(MACH_PORT_VALID(reply_name));
entry->ie_object = (ipc_object_t) reply_port;
is_write_unlock(space);
}
assert(dest_port->ip_srights > 0);
ip_release(dest_port);
if (dest_port->ip_receiver == space)
dest_name = dest_port->ip_receiver_name;
else
dest_name = MACH_PORT_NULL;
if ((--dest_port->ip_srights == 0) &&
(dest_port->ip_nsrequest != IP_NULL)) {
ipc_port_t nsrequest;
mach_port_mscount_t mscount;
nsrequest = dest_port->ip_nsrequest;
mscount = dest_port->ip_mscount;
dest_port->ip_nsrequest = IP_NULL;
ip_unlock(dest_port);
ipc_notify_no_senders(nsrequest, mscount);
} else
ip_unlock(dest_port);
hdr->msgh_bits =
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
MACH_MSG_TYPE_PORT_SEND);
hdr->msgh_remote_port = (mach_port_t)reply_name;
hdr->msgh_local_port = (mach_port_t)dest_name;
HOT(c_mmot_hot_ok1++);
goto fast_put;
abort_request_copyout:
ip_unlock(dest_port);
is_write_unlock(space);
HOT(c_mmot_g_slow_copyout4++);
goto slow_copyout;
}
case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
register mach_port_name_t dest_name;
ip_lock(dest_port);
if (!ip_active(dest_port)) {
ip_unlock(dest_port);
HOT(c_mmot_g_slow_copyout3++);
goto slow_copyout;
}
assert(dest_port->ip_sorights > 0);
if (dest_port->ip_receiver == space) {
ip_release(dest_port);
dest_port->ip_sorights--;
dest_name = dest_port->ip_receiver_name;
ip_unlock(dest_port);
} else {
ip_unlock(dest_port);
ipc_notify_send_once(dest_port);
dest_name = MACH_PORT_NULL;
}
hdr->msgh_bits = MACH_MSGH_BITS(0,
MACH_MSG_TYPE_PORT_SEND_ONCE);
hdr->msgh_remote_port = MACH_PORT_NULL;
hdr->msgh_local_port = (ipc_port_t)dest_name;
HOT(c_mmot_hot_ok2++);
goto fast_put;
}
case MACH_MSGH_BITS_COMPLEX|
MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
register mach_port_name_t dest_name;
ip_lock(dest_port);
if (!ip_active(dest_port)) {
ip_unlock(dest_port);
HOT(c_mmot_g_slow_copyout1++);
goto slow_copyout;
}
assert(dest_port->ip_sorights > 0);
if (dest_port->ip_receiver == space) {
ip_release(dest_port);
dest_port->ip_sorights--;
dest_name = dest_port->ip_receiver_name;
ip_unlock(dest_port);
} else {
ip_unlock(dest_port);
ipc_notify_send_once(dest_port);
dest_name = MACH_PORT_NULL;
}
hdr->msgh_bits =
MACH_MSGH_BITS_COMPLEX |
MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE);
hdr->msgh_remote_port = MACH_PORT_NULL;
hdr->msgh_local_port = (mach_port_t)dest_name;
mr = ipc_kmsg_copyout_body(kmsg, space,
current_map(),
MACH_MSG_BODY_NULL);
if (mr != MACH_MSG_SUCCESS) {
if (ipc_kmsg_put(msg_addr, kmsg,
kmsg->ikm_header->msgh_size +
trailer->msgh_trailer_size) ==
MACH_RCV_INVALID_DATA)
return MACH_RCV_INVALID_DATA;
else
return mr | MACH_RCV_BODY_ERROR;
}
HOT(c_mmot_hot_ok3++);
goto fast_put;
}
default:
HOT(c_mmot_g_slow_copyout2++);
goto slow_copyout;
}
fast_put:
mr = ipc_kmsg_put(rcv_msg_addr ? rcv_msg_addr : msg_addr,
kmsg,
kmsg->ikm_header->msgh_size +
trailer->msgh_trailer_size);
if (mr != MACH_MSG_SUCCESS) {
return MACH_RCV_INVALID_DATA;
}
current_task()->messages_received++;
return mr;
slow_copyin:
{
register mach_port_name_t reply_name =
(mach_port_name_t)hdr->msgh_local_port;
mr = ipc_kmsg_copyin(kmsg, space, current_map(),
MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
return(mr);
}
hdr = kmsg->ikm_header;
send_size = hdr->msgh_size;
if ((reply_name != rcv_name) ||
(hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) {
HOT(c_mmot_cold_048++);
goto slow_send;
}
dest_port = (ipc_port_t) hdr->msgh_remote_port;
assert(IP_VALID(dest_port));
ip_lock(dest_port);
if (!ip_active(dest_port)) {
ip_unlock(dest_port);
goto slow_send;
}
if (dest_port->ip_receiver == ipc_space_kernel) {
dest_port->ip_messages.imq_seqno++;
ip_unlock(dest_port);
goto kernel_send;
}
if (!imq_full(&dest_port->ip_messages) ||
(MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
MACH_MSG_TYPE_PORT_SEND_ONCE))
{
register ipc_port_t reply_port;
reply_port = (ipc_port_t) hdr->msgh_local_port;
if (IP_VALID(reply_port)) {
if (ip_lock_try(reply_port)) {
if (ip_active(reply_port) &&
reply_port->ip_receiver == space &&
reply_port->ip_receiver_name == rcv_name &&
reply_port->ip_pset_count == 0)
{
rcv_object = (ipc_object_t) reply_port;
io_reference(rcv_object);
rcv_mqueue = &reply_port->ip_messages;
io_unlock(rcv_object);
HOT(c_mmot_getback_FastSR++);
goto fast_send_receive;
}
ip_unlock(reply_port);
}
}
}
ip_unlock(dest_port);
HOT(c_mmot_cold_050++);
goto slow_send;
kernel_send:
{
register ipc_port_t reply_port;
spl_t s;
c_mmot_kernel_send++;
current_task()->messages_sent++;
kmsg = ipc_kobject_server(kmsg);
if (kmsg == IKM_NULL) {
HOT(c_mmot_cold_051++);
goto slow_get_rcv_port;
}
hdr = kmsg->ikm_header;
send_size = hdr->msgh_size;
trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
round_msg(send_size));
reply_port = (ipc_port_t) hdr->msgh_remote_port;
ip_lock(reply_port);
if ((!ip_active(reply_port)) ||
(reply_port->ip_receiver != space) ||
(reply_port->ip_receiver_name != rcv_name) ||
(reply_port->ip_pset_count != 0))
{
ip_unlock(reply_port);
ipc_kmsg_send_always(kmsg);
HOT(c_mmot_cold_052++);
goto slow_get_rcv_port;
}
s = splsched();
rcv_mqueue = &reply_port->ip_messages;
imq_lock(rcv_mqueue);
if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) ||
(ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL))
{
imq_unlock(rcv_mqueue);
splx(s);
ip_unlock(reply_port);
ipc_kmsg_send_always(kmsg);
HOT(c_mmot_cold_053++);
goto slow_get_rcv_port;
}
dest_port = reply_port;
temp_seqno = rcv_mqueue->imq_seqno++;
imq_unlock(rcv_mqueue);
splx(s);
ip_check_unlock(reply_port);
if (option & MACH_RCV_TRAILER_MASK) {
trailer->msgh_seqno = temp_seqno;
trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
}
HOT(c_mmot_fastkernelreply++);
goto fast_copyout;
}
slow_send:
mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE,
MACH_MSG_TIMEOUT_NONE);
if (mr != MACH_MSG_SUCCESS) {
mr |= ipc_kmsg_copyout_pseudo(kmsg, space,
current_map(),
MACH_MSG_BODY_NULL);
(void) ipc_kmsg_put(msg_addr, kmsg,
kmsg->ikm_header->msgh_size);
return(mr);
}
slow_get_rcv_port:
mr = ipc_mqueue_copyin(space, rcv_name,
&rcv_mqueue, &rcv_object);
if (mr != MACH_MSG_SUCCESS) {
return(mr);
}
slow_receive:
self->ith_continuation = (void (*)(mach_msg_return_t))0;
ipc_mqueue_receive(rcv_mqueue,
MACH_MSG_OPTION_NONE,
MACH_MSG_SIZE_MAX,
MACH_MSG_TIMEOUT_NONE,
THREAD_ABORTSAFE);
mr = self->ith_state;
temp_seqno = self->ith_seqno;
ipc_object_release(rcv_object);
if (mr != MACH_MSG_SUCCESS) {
return(mr);
}
kmsg = self->ith_kmsg;
hdr = kmsg->ikm_header;
send_size = hdr->msgh_size;
trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
round_msg(send_size));
if (option & MACH_RCV_TRAILER_MASK) {
trailer->msgh_seqno = temp_seqno;
trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
}
dest_port = (ipc_port_t) hdr->msgh_remote_port;
HOT(c_mmot_cold_055++);
goto fast_copyout;
slow_copyout:
reply_size = ipc_kmsg_copyout_size(kmsg, current_map()) +
REQUESTED_TRAILER_SIZE(option);
temp_seqno = trailer->msgh_seqno;
if (rcv_size < reply_size) {
if (msg_receive_error(kmsg, msg_addr, option, temp_seqno,
space) == MACH_RCV_INVALID_DATA) {
mr = MACH_RCV_INVALID_DATA;
return(mr);
}
else {
mr = MACH_RCV_TOO_LARGE;
return(mr);
}
}
mr = ipc_kmsg_copyout(kmsg, space, current_map(),
MACH_PORT_NULL, MACH_MSG_BODY_NULL);
if (mr != MACH_MSG_SUCCESS) {
if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) {
if (ipc_kmsg_put(msg_addr, kmsg, reply_size) ==
MACH_RCV_INVALID_DATA)
mr = MACH_RCV_INVALID_DATA;
}
else {
if (msg_receive_error(kmsg, msg_addr, option,
temp_seqno, space) == MACH_RCV_INVALID_DATA)
mr = MACH_RCV_INVALID_DATA;
}
return(mr);
}
HOT(c_mmot_getback_fast_put++);
goto fast_put;
}
}
#endif
if (option & MACH_SEND_MSG) {
ipc_space_t space = current_space();
vm_map_t map = current_map();
ipc_kmsg_t kmsg;
mr = ipc_kmsg_get(msg_addr, send_size, &kmsg);
if (mr != MACH_MSG_SUCCESS)
return mr;
if (option & MACH_SEND_CANCEL) {
if (notify == MACH_PORT_NULL)
mr = MACH_SEND_INVALID_NOTIFY;
else
mr = ipc_kmsg_copyin(kmsg, space, map, notify);
} else
mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL);
if (mr != MACH_MSG_SUCCESS) {
ipc_kmsg_free(kmsg);
return mr;
}
mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, msg_timeout);
if (mr != MACH_MSG_SUCCESS) {
mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL);
(void) ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size);
return mr;
}
}
if (option & MACH_RCV_MSG) {
thread_t self = current_thread();
ipc_space_t space = current_space();
ipc_object_t object;
ipc_mqueue_t mqueue;
mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object);
if (mr != MACH_MSG_SUCCESS) {
return mr;
}
if (option & MACH_RCV_OVERWRITE)
self->ith_msg_addr = rcv_msg_addr;
else if (rcv_msg_addr != (mach_vm_address_t)0)
self->ith_msg_addr = rcv_msg_addr;
else
self->ith_msg_addr = msg_addr;
self->ith_object = object;
self->ith_msize = rcv_size;
self->ith_option = option;
self->ith_scatter_list_size = scatter_list_size;
self->ith_continuation = thread_syscall_return;
ipc_mqueue_receive(mqueue, option, rcv_size, msg_timeout, THREAD_ABORTSAFE);
if ((option & MACH_RCV_TIMEOUT) && msg_timeout == 0)
thread_poll_yield(self);
return mach_msg_receive_results();
}
return MACH_MSG_SUCCESS;
}
mach_msg_return_t
mach_msg_trap(
struct mach_msg_overwrite_trap_args *args)
{
kern_return_t kr;
args->rcv_msg = (mach_vm_address_t)0;
kr = mach_msg_overwrite_trap(args);
return kr;
}
mach_msg_return_t
msg_receive_error(
ipc_kmsg_t kmsg,
mach_vm_address_t msg_addr,
mach_msg_option_t option,
mach_port_seqno_t seqno,
ipc_space_t space)
{
mach_msg_format_0_trailer_t *trailer;
ipc_kmsg_copyout_dest(kmsg, space);
trailer = (mach_msg_format_0_trailer_t *)
((vm_offset_t)kmsg->ikm_header +
round_msg(sizeof(mach_msg_header_t)));
kmsg->ikm_header->msgh_size = sizeof(mach_msg_header_t);
bcopy( (char *)&trailer_template,
(char *)trailer,
sizeof(trailer_template));
if (option & MACH_RCV_TRAILER_MASK) {
trailer->msgh_seqno = seqno;
trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
}
if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size +
trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA)
return(MACH_RCV_INVALID_DATA);
else
return(MACH_MSG_SUCCESS);
}