#ifdef __x86_64__
#include <i386/mp.h>
#include <i386/cpu_data.h>
#include <i386/bit_routines.h>
#include <i386/machine_cpu.h>
#include <i386/machine_routines.h>
#include <i386/misc_protos.h>
#include <i386/serial_io.h>
#endif
#include <libkern/OSAtomic.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <console/video_console.h>
#include <console/serial_protos.h>
#include <kern/kalloc.h>
#include <kern/thread.h>
#include <kern/cpu_data.h>
#ifndef MAX_CPU_SLOTS
#define MAX_CPU_SLOTS (MAX_CPUS)
#endif
static struct {
char * buffer;
int len;
int used;
char * write_ptr;
char * read_ptr;
decl_simple_lock_data(, read_lock);
decl_simple_lock_data(, write_lock);
} console_ring;
hw_lock_data_t cnputc_lock;
static volatile uint32_t console_output = 0;
#define CPU_CONS_BUF_SIZE 256
#define CPU_BUF_FREE_HEX 0xf2eec075
#define KERN_CONSOLE_BUF_SIZE vm_map_round_page(CPU_CONS_BUF_SIZE *(MAX_CPU_SLOTS + 1), PAGE_SIZE - 1)
#define KERN_CONSOLE_RING_SIZE (KERN_CONSOLE_BUF_SIZE - (CPU_CONS_BUF_SIZE * MAX_CPU_SLOTS))
#define MAX_INT_DISABLED_FLUSH_SIZE 8
#define MAX_TOTAL_FLUSH_SIZE (MAX(2, MAX_CPU_SLOTS) * CPU_CONS_BUF_SIZE)
typedef struct console_buf {
char * buf_base;
char * buf_end;
char * buf_ptr;
#define CPU_BUFFER_LEN (CPU_CONS_BUF_SIZE - 3 * (sizeof(char *)))
char buf[CPU_BUFFER_LEN];
} console_buf_t;
extern int serial_getc(void);
extern void serial_putc(char);
static void _serial_putc(int, int, int);
struct console_ops cons_ops[] = {
{
.putc = _serial_putc, .getc = _serial_getc,
},
{
.putc = vcputc, .getc = vcgetc,
},
};
uint32_t nconsops = (sizeof cons_ops / sizeof cons_ops[0]);
uint32_t cons_ops_index = VC_CONS_OPS;
static bool console_suspended = false;
static void
console_ring_lock_init(void)
{
simple_lock_init(&console_ring.read_lock, 0);
simple_lock_init(&console_ring.write_lock, 0);
}
void
console_init(void)
{
int ret, i;
uint32_t * p;
if (!OSCompareAndSwap(0, KERN_CONSOLE_RING_SIZE, (UInt32 *)&console_ring.len))
return;
assert(console_ring.len > 0);
ret = kmem_alloc(kernel_map, (vm_offset_t *)&console_ring.buffer, KERN_CONSOLE_BUF_SIZE, VM_KERN_MEMORY_OSFMK);
if (ret != KERN_SUCCESS) {
panic("console_ring_init() failed to allocate ring buffer, error %d\n", ret);
}
for (i = 0; i < MAX_CPU_SLOTS; i++) {
p = (uint32_t *)((uintptr_t)console_ring.buffer + console_ring.len + (i * sizeof(console_buf_t)));
*p = CPU_BUF_FREE_HEX;
}
console_ring.used = 0;
console_ring.read_ptr = console_ring.buffer;
console_ring.write_ptr = console_ring.buffer;
console_ring_lock_init();
hw_lock_init(&cnputc_lock);
}
void *
console_cpu_alloc(__unused boolean_t boot_processor)
{
console_buf_t * cbp;
int i;
uint32_t * p;
console_init();
assert(console_ring.buffer != NULL);
for (i = 0; i < MAX_CPU_SLOTS; i++) {
p = (uint32_t *)((uintptr_t)console_ring.buffer + console_ring.len + (i * sizeof(console_buf_t)));
if (OSCompareAndSwap(CPU_BUF_FREE_HEX, 0, (UInt32 *)p))
break;
}
assert(i < MAX_CPU_SLOTS);
cbp = (console_buf_t *)(uintptr_t)p;
if ((uintptr_t)cbp >= (uintptr_t)console_ring.buffer + KERN_CONSOLE_BUF_SIZE) {
printf("console_cpu_alloc() failed to allocate cpu buffer\n");
return NULL;
}
cbp->buf_base = (char *)&cbp->buf;
cbp->buf_ptr = cbp->buf_base;
cbp->buf_end = cbp->buf_base + CPU_BUFFER_LEN;
return (void *)cbp;
}
void
console_cpu_free(void * buf)
{
assert((uintptr_t)buf > (uintptr_t)console_ring.buffer);
assert((uintptr_t)buf < (uintptr_t)console_ring.buffer + KERN_CONSOLE_BUF_SIZE);
if (buf != NULL)
*(uint32_t *)buf = CPU_BUF_FREE_HEX;
}
static inline int
console_ring_space(void)
{
return console_ring.len - console_ring.used;
}
static boolean_t
console_ring_put(char ch)
{
if (console_ring.used < console_ring.len) {
console_ring.used++;
*console_ring.write_ptr++ = ch;
if (console_ring.write_ptr - console_ring.buffer == console_ring.len)
console_ring.write_ptr = console_ring.buffer;
return TRUE;
} else {
return FALSE;
}
}
static inline boolean_t
cpu_buffer_put(console_buf_t * cbp, char ch)
{
if (ch != '\0' && cbp->buf_ptr < cbp->buf_end) {
*(cbp->buf_ptr++) = ch;
return TRUE;
} else {
return FALSE;
}
}
static inline int
cpu_buffer_size(console_buf_t * cbp)
{
return (int)(cbp->buf_ptr - cbp->buf_base);
}
static inline void
_cnputs(char * c, int size)
{
#ifdef __x86_64__
uint32_t lock_timeout_ticks = UINT32_MAX;
#else
uint32_t lock_timeout_ticks = LockTimeOut;
#endif
mp_disable_preemption();
if (!hw_lock_to(&cnputc_lock, lock_timeout_ticks)) {
hw_lock_data_t _shadow_lock;
memcpy(&_shadow_lock, &cnputc_lock, sizeof(cnputc_lock));
if (debug_mode) {
mp_enable_preemption();
hw_lock_init(&cnputc_lock);
hw_lock_lock(&cnputc_lock);
} else {
panic("Lock acquire timeout in _cnputs() lock=%p, lock owner thread=0x%lx, current_thread: %p\n", &_shadow_lock,
_shadow_lock.lock_data, current_thread());
}
}
while (size-- > 0) {
cons_ops[cons_ops_index].putc(0, 0, *c);
if (*c == '\n')
cons_ops[cons_ops_index].putc(0, 0, '\r');
c++;
}
hw_lock_unlock(&cnputc_lock);
mp_enable_preemption();
}
void
cnputc_unbuffered(char c)
{
_cnputs(&c, 1);
}
void
cnputcusr(char c)
{
boolean_t state;
while (console_output != 0)
;
state = ml_set_interrupts_enabled(FALSE);
_cnputs(&c, 1);
ml_set_interrupts_enabled(state);
}
static void
console_ring_try_empty(void)
{
#ifdef __x86_64__
boolean_t handle_tlb_flushes = (ml_get_interrupts_enabled() == FALSE);
#endif
int nchars_out = 0;
int total_chars_out = 0;
int size_before_wrap = 0;
do {
#ifdef __x86_64__
if (handle_tlb_flushes)
handle_pending_TLB_flushes();
#endif
if (!simple_lock_try(&console_ring.read_lock)) {
delay(1);
return;
}
boolean_t state = ml_set_interrupts_enabled(FALSE);
(void)hw_atomic_add(&console_output, 1);
simple_lock_try_lock_loop(&console_ring.write_lock);
nchars_out = MIN(console_ring.used, MAX_INT_DISABLED_FLUSH_SIZE);
size_before_wrap = (int)((console_ring.buffer + console_ring.len) - console_ring.read_ptr);
if (nchars_out > size_before_wrap)
nchars_out = size_before_wrap;
if (nchars_out > 0) {
_cnputs(console_ring.read_ptr, nchars_out);
console_ring.read_ptr =
console_ring.buffer + ((console_ring.read_ptr - console_ring.buffer + nchars_out) % console_ring.len);
console_ring.used -= nchars_out;
total_chars_out += nchars_out;
}
simple_unlock(&console_ring.write_lock);
(void)hw_atomic_sub(&console_output, 1);
simple_unlock(&console_ring.read_lock);
ml_set_interrupts_enabled(state);
if (debug_mode == 0 && !console_suspended && (total_chars_out >= MAX_TOTAL_FLUSH_SIZE))
break;
} while (nchars_out > 0);
}
void
console_suspend()
{
console_suspended = true;
console_ring_try_empty();
}
void
console_resume()
{
console_suspended = false;
}
void
console_write(char * str, int size)
{
console_init();
int chunk_size = size;
int i = 0;
if (size > console_ring.len)
chunk_size = CPU_CONS_BUF_SIZE;
while (size > 0) {
boolean_t state = ml_set_interrupts_enabled(FALSE);
simple_lock_try_lock_loop(&console_ring.write_lock);
while (chunk_size > console_ring_space()) {
simple_unlock(&console_ring.write_lock);
ml_set_interrupts_enabled(state);
console_ring_try_empty();
state = ml_set_interrupts_enabled(FALSE);
simple_lock_try_lock_loop(&console_ring.write_lock);
}
for (i = 0; i < chunk_size; i++)
console_ring_put(str[i]);
str = &str[i];
size -= chunk_size;
simple_unlock(&console_ring.write_lock);
ml_set_interrupts_enabled(state);
}
console_ring_try_empty();
}
void
cnputc(char c)
{
console_buf_t * cbp;
cpu_data_t * cpu_data_p;
boolean_t state;
boolean_t needs_print = TRUE;
char * cp;
restart:
mp_disable_preemption();
cpu_data_p = current_cpu_datap();
cbp = (console_buf_t *)cpu_data_p->cpu_console_buf;
if (console_suspended || cbp == NULL) {
mp_enable_preemption();
_cnputs(&c, 1);
return;
}
#ifndef __x86_64__
if (cpu_data_p->PAB_active) {
console_ring_lock_init();
}
#endif
state = ml_set_interrupts_enabled(FALSE);
if (needs_print && !cpu_buffer_put(cbp, c)) {
simple_lock_try_lock_loop(&console_ring.write_lock);
if (cpu_buffer_size(cbp) > console_ring_space()) {
simple_unlock(&console_ring.write_lock);
ml_set_interrupts_enabled(state);
mp_enable_preemption();
console_ring_try_empty();
goto restart;
}
for (cp = cbp->buf_base; cp < cbp->buf_ptr; cp++)
console_ring_put(*cp);
cbp->buf_ptr = cbp->buf_base;
simple_unlock(&console_ring.write_lock);
cpu_buffer_put(cbp, c);
}
needs_print = FALSE;
if (c != '\n') {
ml_set_interrupts_enabled(state);
mp_enable_preemption();
return;
}
simple_lock_try_lock_loop(&console_ring.write_lock);
if (cpu_buffer_size(cbp) > console_ring_space()) {
simple_unlock(&console_ring.write_lock);
ml_set_interrupts_enabled(state);
mp_enable_preemption();
console_ring_try_empty();
goto restart;
}
for (cp = cbp->buf_base; cp < cbp->buf_ptr; cp++)
console_ring_put(*cp);
cbp->buf_ptr = cbp->buf_base;
simple_unlock(&console_ring.write_lock);
ml_set_interrupts_enabled(state);
mp_enable_preemption();
console_ring_try_empty();
return;
}
int
_serial_getc(__unused int a, __unused int b, boolean_t wait, __unused boolean_t raw)
{
int c;
do {
c = serial_getc();
} while (wait && c < 0);
return c;
}
static void
_serial_putc(__unused int a, __unused int b, int c)
{
serial_putc(c);
}
int
cngetc(void)
{
return cons_ops[cons_ops_index].getc(0, 0, TRUE, FALSE);
}
int
cnmaygetc(void)
{
return cons_ops[cons_ops_index].getc(0, 0, FALSE, FALSE);
}
int
vcgetc(__unused int l, __unused int u, __unused boolean_t wait, __unused boolean_t raw)
{
char c;
if (0 == (*PE_poll_input)(0, &c))
return c;
else
return 0;
}
void
console_set_serial_ops(struct console_ops * newops)
{
cons_ops[SERIAL_CONS_OPS] = *newops;
}