#include <kern/assert.h>
#include <mach/mach_types.h>
#include <mach/boolean.h>
#include <mach/vm_param.h>
#include <kern/kern_types.h>
#include <kern/mach_param.h>
#include <kern/thread.h>
#include <kern/task.h>
#include <kern/kern_cdata.h>
#include <kern/kalloc.h>
#include <mach/mach_vm.h>
static kern_return_t kcdata_get_memory_addr_with_flavor(kcdata_descriptor_t data, uint32_t type, uint32_t size, uint64_t flags, mach_vm_address_t *user_addr);
static size_t kcdata_get_memory_size_for_data(uint32_t size);
static kern_return_t kcdata_compress_chunk_with_flags(kcdata_descriptor_t data, uint32_t type, const void *input_data, uint32_t input_size, uint64_t flags);
static kern_return_t kcdata_compress_chunk(kcdata_descriptor_t data, uint32_t type, const void *input_data, uint32_t input_size);
static kern_return_t kcdata_write_compression_stats(kcdata_descriptor_t data);
static kern_return_t kcdata_get_compression_stats(kcdata_descriptor_t data, uint64_t *totalout, uint64_t *totalin);
#define ZLIB_METADATA_SIZE 1440
#define kcdata_debug_printf(...) ;
#pragma pack(push, 4)
struct _uint64_with_description_data {
char desc[KCDATA_DESC_MAXLEN];
uint64_t data;
};
struct _uint32_with_description_data {
char desc[KCDATA_DESC_MAXLEN];
uint32_t data;
};
#pragma pack(pop)
uint32_t
kcdata_estimate_required_buffer_size(uint32_t num_items, uint32_t payload_size)
{
uint32_t max_padding_bytes = 0;
uint32_t max_padding_with_item_description_bytes = 0;
uint32_t estimated_required_buffer_size = 0;
const uint32_t begin_and_end_marker_bytes = 2 * sizeof(struct kcdata_item);
if (os_mul_overflow(num_items, KCDATA_ALIGNMENT_SIZE - 1, &max_padding_bytes)) {
panic("%s: Overflow in required buffer size estimate", __func__);
}
if (os_mul_and_add_overflow(num_items, sizeof(struct kcdata_item), max_padding_bytes, &max_padding_with_item_description_bytes)) {
panic("%s: Overflow in required buffer size estimate", __func__);
}
if (os_add3_overflow(max_padding_with_item_description_bytes, begin_and_end_marker_bytes, payload_size, &estimated_required_buffer_size)) {
panic("%s: Overflow in required buffer size estimate", __func__);
}
return estimated_required_buffer_size;
}
kcdata_descriptor_t
kcdata_memory_alloc_init(mach_vm_address_t buffer_addr_p, unsigned data_type, unsigned size, unsigned flags)
{
kcdata_descriptor_t data = NULL;
mach_vm_address_t user_addr = 0;
uint16_t clamped_flags = (uint16_t) flags;
data = kalloc_flags(sizeof(struct kcdata_descriptor), Z_WAITOK | Z_ZERO);
if (data == NULL) {
return NULL;
}
data->kcd_addr_begin = buffer_addr_p;
data->kcd_addr_end = buffer_addr_p;
data->kcd_flags = (clamped_flags & KCFLAG_USE_COPYOUT) ? clamped_flags : clamped_flags | KCFLAG_USE_MEMCOPY;
data->kcd_length = size;
if (KERN_SUCCESS != kcdata_get_memory_addr(data, data_type, 0, &user_addr)) {
kcdata_memory_destroy(data);
return NULL;
}
return data;
}
kern_return_t
kcdata_memory_static_init(kcdata_descriptor_t data, mach_vm_address_t buffer_addr_p, unsigned data_type, unsigned size, unsigned flags)
{
mach_vm_address_t user_addr = 0;
uint16_t clamped_flags = (uint16_t) flags;
if (data == NULL) {
return KERN_INVALID_ARGUMENT;
}
bzero(data, sizeof(struct kcdata_descriptor));
data->kcd_addr_begin = buffer_addr_p;
data->kcd_addr_end = buffer_addr_p;
data->kcd_flags = (clamped_flags & KCFLAG_USE_COPYOUT) ? clamped_flags : clamped_flags | KCFLAG_USE_MEMCOPY;
data->kcd_length = size;
return kcdata_get_memory_addr(data, data_type, 0, &user_addr);
}
void *
kcdata_memory_get_begin_addr(kcdata_descriptor_t data)
{
if (data == NULL) {
return NULL;
}
return (void *)data->kcd_addr_begin;
}
uint64_t
kcdata_memory_get_used_bytes(kcdata_descriptor_t kcd)
{
assert(kcd != NULL);
return ((uint64_t)kcd->kcd_addr_end - (uint64_t)kcd->kcd_addr_begin) + sizeof(struct kcdata_item);
}
uint64_t
kcdata_memory_get_uncompressed_bytes(kcdata_descriptor_t kcd)
{
kern_return_t kr;
assert(kcd != NULL);
if (kcd->kcd_flags & KCFLAG_USE_COMPRESSION) {
uint64_t totalout, totalin;
kr = kcdata_get_compression_stats(kcd, &totalout, &totalin);
if (kr == KERN_SUCCESS) {
return totalin;
} else {
return 0;
}
} else {
return kcdata_memory_get_used_bytes(kcd);
}
}
kern_return_t
kcdata_memory_destroy(kcdata_descriptor_t data)
{
if (!data) {
return KERN_INVALID_ARGUMENT;
}
kfree(data, sizeof(struct kcdata_descriptor));
return KERN_SUCCESS;
}
static void *
kcdata_compress_zalloc(void *opaque, u_int items, u_int size)
{
void *result;
struct kcdata_compress_descriptor *cd = opaque;
int alloc_size = ~31L & (31 + (items * size));
result = (void *)(cd->kcd_cd_base + cd->kcd_cd_offset);
if ((uintptr_t) result + alloc_size > (uintptr_t) cd->kcd_cd_base + cd->kcd_cd_maxoffset) {
result = Z_NULL;
} else {
cd->kcd_cd_offset += alloc_size;
}
kcdata_debug_printf("%s: %d * %d = %d => %p\n", __func__, items, size, items * size, result);
return result;
}
static void
kcdata_compress_zfree(void *opaque, void *ptr)
{
(void) opaque;
(void) ptr;
kcdata_debug_printf("%s: ptr %p\n", __func__, ptr);
}
static kern_return_t
kcdata_init_compress_state(kcdata_descriptor_t data, void (*memcpy_f)(void *, const void *, size_t), uint64_t type, mach_vm_address_t totalout_addr, mach_vm_address_t totalin_addr)
{
kern_return_t ret = KERN_SUCCESS;
size_t size;
int wbits = 12, memlevel = 3;
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
cd->kcd_cd_memcpy_f = memcpy_f;
cd->kcd_cd_compression_type = type;
cd->kcd_cd_totalout_addr = totalout_addr;
cd->kcd_cd_totalin_addr = totalin_addr;
switch (type) {
case KCDCT_ZLIB:
size = round_page(ZLIB_METADATA_SIZE + zlib_deflate_memory_size(wbits, memlevel));
kcdata_debug_printf("%s: size = %zu kcd_length: %d\n", __func__, size, data->kcd_length);
kcdata_debug_printf("%s: kcd buffer [%p - %p]\n", __func__, (void *) data->kcd_addr_begin, (void *) data->kcd_addr_begin + data->kcd_length);
if (4 * size > data->kcd_length) {
return KERN_INSUFFICIENT_BUFFER_SIZE;
}
cd->kcd_cd_zs.avail_in = 0;
cd->kcd_cd_zs.next_in = NULL;
cd->kcd_cd_zs.avail_out = 0;
cd->kcd_cd_zs.next_out = NULL;
cd->kcd_cd_zs.opaque = cd;
cd->kcd_cd_zs.zalloc = kcdata_compress_zalloc;
cd->kcd_cd_zs.zfree = kcdata_compress_zfree;
cd->kcd_cd_base = (void *) data->kcd_addr_begin + data->kcd_length - size;
data->kcd_length -= size;
cd->kcd_cd_offset = 0;
cd->kcd_cd_maxoffset = size;
cd->kcd_cd_flags = 0;
kcdata_debug_printf("%s: buffer [%p - %p]\n", __func__, cd->kcd_cd_base, cd->kcd_cd_base + size);
if (deflateInit2(&cd->kcd_cd_zs, Z_BEST_SPEED, Z_DEFLATED, wbits, memlevel, Z_DEFAULT_STRATEGY) != Z_OK) {
kcdata_debug_printf("EMERGENCY: deflateInit2 failed!\n");
ret = KERN_INVALID_ARGUMENT;
}
break;
default:
panic("kcdata_init_compress_state: invalid compression type: %d", (int) type);
}
return ret;
}
kern_return_t
kcdata_init_compress(kcdata_descriptor_t data, int hdr_tag, void (*memcpy_f)(void *, const void *, size_t), uint64_t type)
{
kern_return_t kr;
mach_vm_address_t user_addr, totalout_addr, totalin_addr;
struct _uint64_with_description_data save_data;
const uint64_t size_req = sizeof(save_data);
assert(data && (data->kcd_flags & KCFLAG_USE_COMPRESSION) == 0);
bzero(&data->kcd_comp_d, sizeof(struct kcdata_compress_descriptor));
kcdata_add_uint64_with_description(data, type, "kcd_c_type");
bzero(&save_data, size_req);
strlcpy(&(save_data.desc[0]), "kcd_c_totalout", sizeof(save_data.desc));
kr = kcdata_get_memory_addr(data, KCDATA_TYPE_UINT64_DESC, size_req, &totalout_addr);
if (kr != KERN_SUCCESS) {
return kr;
}
memcpy((void *)totalout_addr, &save_data, size_req);
bzero(&save_data, size_req);
strlcpy(&(save_data.desc[0]), "kcd_c_totalin", sizeof(save_data.desc));
kr = kcdata_get_memory_addr(data, KCDATA_TYPE_UINT64_DESC, size_req, &totalin_addr);
if (kr != KERN_SUCCESS) {
return kr;
}
memcpy((void *)totalin_addr, &save_data, size_req);
kcdata_get_memory_addr(data, hdr_tag, 0, &user_addr);
data->kcd_flags |= KCFLAG_USE_COMPRESSION;
kr = kcdata_init_compress_state(data, memcpy_f, type, totalout_addr + offsetof(struct _uint64_with_description_data, data), totalin_addr + offsetof(struct _uint64_with_description_data, data));
if (kr != KERN_SUCCESS) {
kcdata_debug_printf("%s: failed to initialize compression state!\n", __func__);
return kr;
}
return KERN_SUCCESS;
}
static inline
int
kcdata_zlib_translate_kcd_cf_flag(enum kcdata_compression_flush flush)
{
switch (flush) {
case KCDCF_NO_FLUSH: return Z_NO_FLUSH;
case KCDCF_SYNC_FLUSH: return Z_SYNC_FLUSH;
case KCDCF_FINISH: return Z_FINISH;
default: panic("invalid kcdata_zlib_translate_kcd_cf_flag flag");
}
}
static inline
int
kcdata_zlib_translate_kcd_cf_expected_ret(enum kcdata_compression_flush flush)
{
switch (flush) {
case KCDCF_NO_FLUSH:
case KCDCF_SYNC_FLUSH: return Z_OK;
case KCDCF_FINISH: return Z_STREAM_END;
default: panic("invalid kcdata_zlib_translate_kcd_cf_expected_ret flag");
}
}
static kern_return_t
kcdata_do_compress_zlib(kcdata_descriptor_t data, void *inbuffer,
size_t insize, void *outbuffer, size_t outsize, size_t *wrote,
enum kcdata_compression_flush flush)
{
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
z_stream *zs = &cd->kcd_cd_zs;
int expected_ret, ret;
zs->next_out = outbuffer;
zs->avail_out = (unsigned int) outsize;
zs->next_in = inbuffer;
zs->avail_in = (unsigned int) insize;
ret = deflate(zs, kcdata_zlib_translate_kcd_cf_flag(flush));
if (zs->avail_in != 0 || zs->avail_out <= 0) {
return KERN_INSUFFICIENT_BUFFER_SIZE;
}
expected_ret = kcdata_zlib_translate_kcd_cf_expected_ret(flush);
if (ret != expected_ret) {
panic("zlib kcdata compression ret = %d\n", ret);
}
kcdata_debug_printf("%s: %p (%zu) <- %p (%zu); flush: %d; ret = %ld\n",
__func__, outbuffer, outsize, inbuffer, insize, flush, outsize - zs->avail_out);
if (wrote) {
*wrote = outsize - zs->avail_out;
}
return KERN_SUCCESS;
}
static kern_return_t
kcdata_do_compress(kcdata_descriptor_t data, void *inbuffer, size_t insize,
void *outbuffer, size_t outsize, size_t *wrote, enum kcdata_compression_flush flush)
{
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
assert(data->kcd_flags & KCFLAG_USE_COMPRESSION);
kcdata_debug_printf("%s: %p (%zu) <- %p (%zu); flush: %d\n",
__func__, outbuffer, outsize, inbuffer, insize, flush);
if (cd->kcd_cd_flags & KCD_CD_FLAG_IN_MARK || data->kcd_comp_d.kcd_cd_compression_type == KCDCT_NONE) {
assert(cd->kcd_cd_memcpy_f);
if (outsize >= insize) {
cd->kcd_cd_memcpy_f(outbuffer, inbuffer, insize);
if (wrote) {
*wrote = insize;
}
return KERN_SUCCESS;
} else {
return KERN_INSUFFICIENT_BUFFER_SIZE;
}
}
switch (data->kcd_comp_d.kcd_cd_compression_type) {
case KCDCT_ZLIB:
return kcdata_do_compress_zlib(data, inbuffer, insize, outbuffer, outsize, wrote, flush);
default:
panic("invalid compression type 0x%llx in kcdata_do_compress", data->kcd_comp_d.kcd_cd_compression_type);
}
}
static size_t
kcdata_compression_bound_zlib(kcdata_descriptor_t data, size_t size)
{
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
z_stream *zs = &cd->kcd_cd_zs;
return (size_t) deflateBound(zs, (unsigned long) size);
}
static size_t
kcdata_compression_bound(kcdata_descriptor_t data, size_t size)
{
switch (data->kcd_comp_d.kcd_cd_compression_type) {
case KCDCT_ZLIB:
return kcdata_compression_bound_zlib(data, size);
case KCDCT_NONE:
return size;
default:
panic("%s: unknown compression method", __func__);
}
}
kern_return_t
kcdata_compress_chunk_with_flags(kcdata_descriptor_t data, uint32_t type, const void *input_data, uint32_t input_size, uint64_t kcdata_flags)
{
assert(data);
assert((data->kcd_flags & KCFLAG_USE_COMPRESSION));
assert(input_data);
struct kcdata_item info;
char padding_data[16] = {0};
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
size_t wrote = 0;
kern_return_t kr;
kcdata_debug_printf("%s: type: %d input_data: %p (%d) kcdata_flags: 0x%llx\n",
__func__, type, input_data, input_size, kcdata_flags);
size_t total_uncompressed_size = kcdata_compression_bound(data, (size_t) kcdata_get_memory_size_for_data(input_size));
if (total_uncompressed_size > data->kcd_length ||
data->kcd_length - total_uncompressed_size < data->kcd_addr_end - data->kcd_addr_begin) {
kcdata_debug_printf("%s: insufficient buffer size: kcd_length => %d e-b=> %lld our size: %zu\n",
__func__, data->kcd_length, data->kcd_addr_end - data->kcd_addr_begin, total_uncompressed_size);
return KERN_INSUFFICIENT_BUFFER_SIZE;
}
uint32_t padding = kcdata_calc_padding(input_size);
assert(padding < sizeof(padding_data));
void *space_start = (void *) data->kcd_addr_end;
void *space_ptr = space_start;
size_t total_uncompressed_space_remaining = total_uncompressed_size;
bzero(&info, sizeof(info));
info.type = type;
info.size = input_size + padding;
info.flags = kcdata_flags;
enum kcdata_compression_flush flush = (padding || input_size) ? KCDCF_NO_FLUSH :
cd->kcd_cd_flags & KCD_CD_FLAG_FINALIZE ? KCDCF_FINISH :
KCDCF_SYNC_FLUSH;
kr = kcdata_do_compress(data, &info, sizeof(info), space_ptr, total_uncompressed_space_remaining, &wrote, flush);
if (kr != KERN_SUCCESS) {
return kr;
}
kcdata_debug_printf("%s: first wrote = %zu\n", __func__, wrote);
space_ptr += wrote;
total_uncompressed_space_remaining -= wrote;
if (input_size) {
flush = padding ? KCDCF_NO_FLUSH :
cd->kcd_cd_flags & KCD_CD_FLAG_FINALIZE ? KCDCF_FINISH :
KCDCF_SYNC_FLUSH;
kr = kcdata_do_compress(data, (void *) (uintptr_t) input_data, input_size, space_ptr, total_uncompressed_space_remaining, &wrote, flush);
if (kr != KERN_SUCCESS) {
return kr;
}
kcdata_debug_printf("%s: 2nd wrote = %zu\n", __func__, wrote);
space_ptr += wrote;
total_uncompressed_space_remaining -= wrote;
}
if (padding) {
kr = kcdata_do_compress(data, padding_data, padding, space_ptr, total_uncompressed_space_remaining, &wrote,
cd->kcd_cd_flags & KCD_CD_FLAG_FINALIZE ? KCDCF_FINISH : KCDCF_SYNC_FLUSH);
if (kr != KERN_SUCCESS) {
return kr;
}
kcdata_debug_printf("%s: 3rd wrote = %zu\n", __func__, wrote);
if (wrote == 0) {
return KERN_FAILURE;
}
space_ptr += wrote;
total_uncompressed_space_remaining -= wrote;
}
assert((size_t)(space_ptr - space_start) <= total_uncompressed_size);
data->kcd_addr_end = (mach_vm_address_t) (space_start + (total_uncompressed_size - total_uncompressed_space_remaining));
return KERN_SUCCESS;
}
kern_return_t
kcdata_compress_chunk(kcdata_descriptor_t data, uint32_t type, const void *input_data, uint32_t input_size)
{
uint64_t flags = (KCDATA_FLAGS_STRUCT_PADDING_MASK & kcdata_calc_padding(input_size)) | KCDATA_FLAGS_STRUCT_HAS_PADDING;
return kcdata_compress_chunk_with_flags(data, type, input_data, input_size, flags);
}
kern_return_t
kcdata_push_data(kcdata_descriptor_t data, uint32_t type, uint32_t size, const void *input_data)
{
if (data->kcd_flags & KCFLAG_USE_COMPRESSION) {
return kcdata_compress_chunk(data, type, input_data, size);
} else {
kern_return_t ret;
mach_vm_address_t uaddr = 0;
ret = kcdata_get_memory_addr(data, type, size, &uaddr);
if (ret != KERN_SUCCESS) {
return ret;
}
kcdata_memcpy(data, uaddr, input_data, size);
return KERN_SUCCESS;
}
}
kern_return_t
kcdata_push_array(kcdata_descriptor_t data, uint32_t type_of_element, uint32_t size_of_element, uint32_t count, const void *input_data)
{
uint64_t flags = type_of_element;
flags = (flags << 32) | count;
uint32_t total_size = count * size_of_element;
uint32_t pad = kcdata_calc_padding(total_size);
if (data->kcd_flags & KCFLAG_USE_COMPRESSION) {
return kcdata_compress_chunk_with_flags(data, KCDATA_TYPE_ARRAY_PAD0 | pad, input_data, total_size, flags);
} else {
kern_return_t ret;
mach_vm_address_t uaddr = 0;
ret = kcdata_get_memory_addr_with_flavor(data, KCDATA_TYPE_ARRAY_PAD0 | pad, total_size, flags, &uaddr);
if (ret != KERN_SUCCESS) {
return ret;
}
kcdata_memcpy(data, uaddr, input_data, total_size);
return KERN_SUCCESS;
}
}
void
kcdata_compression_window_open(kcdata_descriptor_t data)
{
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
assert((cd->kcd_cd_flags & KCD_CD_FLAG_IN_MARK) == 0);
if (data->kcd_flags & KCFLAG_USE_COMPRESSION) {
cd->kcd_cd_flags |= KCD_CD_FLAG_IN_MARK;
cd->kcd_cd_mark_begin = data->kcd_addr_end;
}
}
kern_return_t
kcdata_compression_window_close(kcdata_descriptor_t data)
{
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
uint64_t total_size, max_size;
void *space_start, *space_ptr;
size_t total_uncompressed_space_remaining, wrote = 0;
kern_return_t kr;
if ((data->kcd_flags & KCFLAG_USE_COMPRESSION) == 0) {
return KERN_SUCCESS;
}
assert(cd->kcd_cd_flags & KCD_CD_FLAG_IN_MARK);
if (data->kcd_addr_end == (mach_vm_address_t) cd->kcd_cd_mark_begin) {
cd->kcd_cd_flags &= ~KCD_CD_FLAG_IN_MARK;
return KERN_SUCCESS;
}
assert(cd->kcd_cd_mark_begin < data->kcd_addr_end);
total_size = data->kcd_addr_end - (uint64_t) cd->kcd_cd_mark_begin;
max_size = (uint64_t) kcdata_compression_bound(data, total_size);
kcdata_debug_printf("%s: total_size = %lld\n", __func__, total_size);
if (max_size > data->kcd_length ||
data->kcd_length - max_size < data->kcd_addr_end - data->kcd_addr_begin) {
kcdata_debug_printf("%s: insufficient buffer size: kcd_length => %d e-b=> %lld our size: %lld\n",
__func__, data->kcd_length, data->kcd_addr_end - data->kcd_addr_begin, max_size);
return KERN_INSUFFICIENT_BUFFER_SIZE;
}
cd->kcd_cd_flags &= ~KCD_CD_FLAG_IN_MARK;
space_start = (void *) data->kcd_addr_end;
space_ptr = space_start;
total_uncompressed_space_remaining = (unsigned int) max_size;
kr = kcdata_do_compress(data, (void *) cd->kcd_cd_mark_begin, total_size, space_ptr,
total_uncompressed_space_remaining, &wrote, KCDCF_SYNC_FLUSH);
if (kr != KERN_SUCCESS) {
return kr;
}
kcdata_debug_printf("%s: first wrote = %zu\n", __func__, wrote);
if (wrote == 0) {
return KERN_FAILURE;
}
space_ptr += wrote;
total_uncompressed_space_remaining -= wrote;
assert((size_t)(space_ptr - space_start) <= max_size);
kcdata_memcpy(data, cd->kcd_cd_mark_begin, space_start, (uint32_t) (max_size - total_uncompressed_space_remaining));
data->kcd_addr_end = cd->kcd_cd_mark_begin + (max_size - total_uncompressed_space_remaining);
return KERN_SUCCESS;
}
static kern_return_t
kcdata_get_compression_stats_zlib(kcdata_descriptor_t data, uint64_t *totalout, uint64_t *totalin)
{
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
z_stream *zs = &cd->kcd_cd_zs;
assert((cd->kcd_cd_flags & KCD_CD_FLAG_IN_MARK) == 0);
*totalout = (uint64_t) zs->total_out;
*totalin = (uint64_t) zs->total_in;
return KERN_SUCCESS;
}
static kern_return_t
kcdata_get_compression_stats(kcdata_descriptor_t data, uint64_t *totalout, uint64_t *totalin)
{
kern_return_t kr;
switch (data->kcd_comp_d.kcd_cd_compression_type) {
case KCDCT_ZLIB:
kr = kcdata_get_compression_stats_zlib(data, totalout, totalin);
break;
case KCDCT_NONE:
kr = KERN_SUCCESS;
break;
default:
panic("invalid compression flag 0x%llx in kcdata_write_compression_stats", (data->kcd_comp_d.kcd_cd_compression_type));
}
return kr;
}
kern_return_t
kcdata_write_compression_stats(kcdata_descriptor_t data)
{
kern_return_t kr;
uint64_t totalout, totalin;
kr = kcdata_get_compression_stats(data, &totalout, &totalin);
if (kr != KERN_SUCCESS) {
return kr;
}
*(uint64_t *)data->kcd_comp_d.kcd_cd_totalout_addr = totalout;
*(uint64_t *)data->kcd_comp_d.kcd_cd_totalin_addr = totalin;
return kr;
}
static kern_return_t
kcdata_finish_compression_zlib(kcdata_descriptor_t data)
{
struct kcdata_compress_descriptor *cd = &data->kcd_comp_d;
z_stream *zs = &cd->kcd_cd_zs;
void* stackshot_end = (char*)data->kcd_addr_begin + kcdata_memory_get_used_bytes(data);
uint32_t zero_fill_size = data->kcd_length - kcdata_memory_get_used_bytes(data);
bzero(stackshot_end, zero_fill_size);
if (deflateEnd(zs) == Z_OK) {
return KERN_SUCCESS;
} else {
return KERN_FAILURE;
}
}
kern_return_t
kcdata_finish_compression(kcdata_descriptor_t data)
{
kcdata_write_compression_stats(data);
switch (data->kcd_comp_d.kcd_cd_compression_type) {
case KCDCT_ZLIB:
data->kcd_length += data->kcd_comp_d.kcd_cd_maxoffset;
return kcdata_finish_compression_zlib(data);
case KCDCT_NONE:
return KERN_SUCCESS;
default:
panic("invalid compression type 0x%llxin kcdata_finish_compression", data->kcd_comp_d.kcd_cd_compression_type);
}
}
void
kcd_finalize_compression(kcdata_descriptor_t data)
{
if (data->kcd_flags & KCFLAG_USE_COMPRESSION) {
data->kcd_comp_d.kcd_cd_flags |= KCD_CD_FLAG_FINALIZE;
}
}
kern_return_t
kcdata_get_memory_addr(kcdata_descriptor_t data, uint32_t type, uint32_t size, mach_vm_address_t * user_addr)
{
uint64_t flags = (KCDATA_FLAGS_STRUCT_PADDING_MASK & kcdata_calc_padding(size)) | KCDATA_FLAGS_STRUCT_HAS_PADDING;
return kcdata_get_memory_addr_with_flavor(data, type, size, flags, user_addr);
}
kern_return_t
kcdata_write_buffer_end(kcdata_descriptor_t data)
{
struct kcdata_item info;
bzero(&info, sizeof(info));
info.type = KCDATA_TYPE_BUFFER_END;
info.size = 0;
return kcdata_memcpy(data, data->kcd_addr_end, &info, sizeof(info));
}
static kern_return_t
kcdata_get_memory_addr_with_flavor(
kcdata_descriptor_t data,
uint32_t type,
uint32_t size,
uint64_t flags,
mach_vm_address_t *user_addr)
{
kern_return_t kr;
struct kcdata_item info;
uint32_t orig_size = size;
uint32_t padding = kcdata_calc_padding(size);
size += padding;
uint32_t total_size = size + sizeof(info);
if (user_addr == NULL || data == NULL || total_size + sizeof(info) < orig_size) {
return KERN_INVALID_ARGUMENT;
}
assert(((data->kcd_flags & KCFLAG_USE_COMPRESSION) && (data->kcd_comp_d.kcd_cd_flags & KCD_CD_FLAG_IN_MARK))
|| ((data->kcd_flags & KCFLAG_USE_COMPRESSION) == 0));
bzero(&info, sizeof(info));
info.type = type;
info.size = size;
info.flags = flags;
if (total_size + sizeof(info) > data->kcd_length ||
data->kcd_length - (total_size + sizeof(info)) < data->kcd_addr_end - data->kcd_addr_begin) {
return KERN_INSUFFICIENT_BUFFER_SIZE;
}
kr = kcdata_memcpy(data, data->kcd_addr_end, &info, sizeof(info));
if (kr) {
return kr;
}
data->kcd_addr_end += sizeof(info);
if (padding) {
kr = kcdata_bzero(data, data->kcd_addr_end + size - padding, padding);
if (kr) {
return kr;
}
}
*user_addr = data->kcd_addr_end;
data->kcd_addr_end += size;
if (!(data->kcd_flags & KCFLAG_NO_AUTO_ENDBUFFER)) {
return kcdata_write_buffer_end(data);
} else {
return KERN_SUCCESS;
}
}
static size_t
kcdata_get_memory_size_for_data(uint32_t size)
{
return size + kcdata_calc_padding(size) + sizeof(struct kcdata_item);
}
kern_return_t
kcdata_get_memory_addr_for_array(
kcdata_descriptor_t data,
uint32_t type_of_element,
uint32_t size_of_element,
uint32_t count,
mach_vm_address_t *user_addr)
{
uint64_t flags = type_of_element;
flags = (flags << 32) | count;
uint32_t total_size = count * size_of_element;
uint32_t pad = kcdata_calc_padding(total_size);
return kcdata_get_memory_addr_with_flavor(data, KCDATA_TYPE_ARRAY_PAD0 | pad, total_size, flags, user_addr);
}
kern_return_t
kcdata_add_container_marker(
kcdata_descriptor_t data,
uint32_t header_type,
uint32_t container_type,
uint64_t identifier)
{
mach_vm_address_t user_addr;
kern_return_t kr;
uint32_t data_size;
assert(header_type == KCDATA_TYPE_CONTAINER_END || header_type == KCDATA_TYPE_CONTAINER_BEGIN);
data_size = (header_type == KCDATA_TYPE_CONTAINER_BEGIN)? sizeof(uint32_t): 0;
if (!(data->kcd_flags & KCFLAG_USE_COMPRESSION)) {
kr = kcdata_get_memory_addr_with_flavor(data, header_type, data_size, identifier, &user_addr);
if (kr != KERN_SUCCESS) {
return kr;
}
if (data_size) {
kr = kcdata_memcpy(data, user_addr, &container_type, data_size);
}
} else {
kr = kcdata_compress_chunk_with_flags(data, header_type, &container_type, data_size, identifier);
}
return kr;
}
kern_return_t
kcdata_undo_add_container_begin(kcdata_descriptor_t data)
{
const mach_vm_address_t padded_payload_size = 16;
data->kcd_addr_end -= sizeof(struct kcdata_item) + padded_payload_size;
if (!(data->kcd_flags & KCFLAG_NO_AUTO_ENDBUFFER)) {
return kcdata_write_buffer_end(data);
} else {
return KERN_SUCCESS;
}
}
kern_return_t
kcdata_memcpy(kcdata_descriptor_t data, mach_vm_address_t dst_addr, const void *src_addr, uint32_t size)
{
if (data->kcd_flags & KCFLAG_USE_COPYOUT) {
if (copyout(src_addr, dst_addr, size)) {
return KERN_NO_ACCESS;
}
} else {
memcpy((void *)dst_addr, src_addr, size);
}
return KERN_SUCCESS;
}
kern_return_t
kcdata_bzero(kcdata_descriptor_t data, mach_vm_address_t dst_addr, uint32_t size)
{
kern_return_t kr = KERN_SUCCESS;
if (data->kcd_flags & KCFLAG_USE_COPYOUT) {
uint8_t zeros[16] = {};
while (size) {
uint32_t block_size = MIN(size, 16);
kr = copyout(&zeros, dst_addr, block_size);
if (kr) {
return KERN_NO_ACCESS;
}
size -= block_size;
}
return KERN_SUCCESS;
} else {
bzero((void*)dst_addr, size);
return KERN_SUCCESS;
}
}
kern_return_t
kcdata_add_type_definition(
kcdata_descriptor_t data,
uint32_t type_id,
char *type_name,
struct kcdata_subtype_descriptor *elements_array_addr,
uint32_t elements_count)
{
kern_return_t kr = KERN_SUCCESS;
struct kcdata_type_definition kc_type_definition;
mach_vm_address_t user_addr;
uint32_t total_size = sizeof(struct kcdata_type_definition);
bzero(&kc_type_definition, sizeof(kc_type_definition));
if (strlen(type_name) >= KCDATA_DESC_MAXLEN) {
return KERN_INVALID_ARGUMENT;
}
strlcpy(&kc_type_definition.kct_name[0], type_name, KCDATA_DESC_MAXLEN);
kc_type_definition.kct_num_elements = elements_count;
kc_type_definition.kct_type_identifier = type_id;
total_size += elements_count * sizeof(struct kcdata_subtype_descriptor);
if (KERN_SUCCESS != (kr = kcdata_get_memory_addr_with_flavor(data, KCDATA_TYPE_TYPEDEFINTION, total_size,
kcdata_calc_padding(total_size), &user_addr))) {
return kr;
}
if (KERN_SUCCESS != (kr = kcdata_memcpy(data, user_addr, (void *)&kc_type_definition, sizeof(struct kcdata_type_definition)))) {
return kr;
}
user_addr += sizeof(struct kcdata_type_definition);
if (KERN_SUCCESS != (kr = kcdata_memcpy(data, user_addr, (void *)elements_array_addr, elements_count * sizeof(struct kcdata_subtype_descriptor)))) {
return kr;
}
return kr;
}
kern_return_t
kcdata_add_uint64_with_description(kcdata_descriptor_t data_desc, uint64_t data, const char * description)
{
if (strlen(description) >= KCDATA_DESC_MAXLEN) {
return KERN_INVALID_ARGUMENT;
}
kern_return_t kr = 0;
mach_vm_address_t user_addr;
struct _uint64_with_description_data save_data;
const uint64_t size_req = sizeof(save_data);
bzero(&save_data, size_req);
strlcpy(&(save_data.desc[0]), description, sizeof(save_data.desc));
save_data.data = data;
if (data_desc->kcd_flags & KCFLAG_USE_COMPRESSION) {
return kcdata_compress_chunk(data_desc, KCDATA_TYPE_UINT64_DESC, &save_data, size_req);
}
kr = kcdata_get_memory_addr(data_desc, KCDATA_TYPE_UINT64_DESC, size_req, &user_addr);
if (kr != KERN_SUCCESS) {
return kr;
}
if (data_desc->kcd_flags & KCFLAG_USE_COPYOUT) {
if (copyout(&save_data, user_addr, size_req)) {
return KERN_NO_ACCESS;
}
} else {
memcpy((void *)user_addr, &save_data, size_req);
}
return KERN_SUCCESS;
}
kern_return_t
kcdata_add_uint32_with_description(
kcdata_descriptor_t data_desc,
uint32_t data,
const char *description)
{
assert(strlen(description) < KCDATA_DESC_MAXLEN);
if (strlen(description) >= KCDATA_DESC_MAXLEN) {
return KERN_INVALID_ARGUMENT;
}
kern_return_t kr = 0;
mach_vm_address_t user_addr;
struct _uint32_with_description_data save_data;
const uint64_t size_req = sizeof(save_data);
bzero(&save_data, size_req);
strlcpy(&(save_data.desc[0]), description, sizeof(save_data.desc));
save_data.data = data;
if (data_desc->kcd_flags & KCFLAG_USE_COMPRESSION) {
return kcdata_compress_chunk(data_desc, KCDATA_TYPE_UINT32_DESC, &save_data, size_req);
}
kr = kcdata_get_memory_addr(data_desc, KCDATA_TYPE_UINT32_DESC, size_req, &user_addr);
if (kr != KERN_SUCCESS) {
return kr;
}
if (data_desc->kcd_flags & KCFLAG_USE_COPYOUT) {
if (copyout(&save_data, user_addr, size_req)) {
return KERN_NO_ACCESS;
}
} else {
memcpy((void *)user_addr, &save_data, size_req);
}
return KERN_SUCCESS;
}