workqueue_internal.h [plain text]
#ifndef _WORKQUEUE_INTERNAL_H_
#define _WORKQUEUE_INTERNAL_H_
#define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE)
#define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST)
#define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY)
#define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST)
#define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX
#define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared
#define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr
#ifdef KERNEL
#pragma mark wq structs
#define SCHED_CALL_BLOCK 0x1
#define SCHED_CALL_UNBLOCK 0x2
#define WORKQUEUE_HIGH_PRIOQUEUE 0
#define WORKQUEUE_DEFAULT_PRIOQUEUE 1
#define WORKQUEUE_LOW_PRIOQUEUE 2
#define WORKQUEUE_BG_PRIOQUEUE 3
#define WORKQUEUE_CONSTRAINED_FACTOR 5
#if BSD_KERNEL_PRIVATE
#include <kern/mpsc_queue.h>
#include <kern/priority_queue.h>
#include <kern/thread_call.h>
#include <kern/turnstile.h>
#include <mach/kern_return.h>
#include <sys/queue.h>
#include <sys/kernel_types.h>
#define UT_WORKQ_NEW 0x01
#define UT_WORKQ_RUNNING 0x02
#define UT_WORKQ_DYING 0x04
#define UT_WORKQ_OVERCOMMIT 0x08
#define UT_WORKQ_OUTSIDE_QOS 0x10
#define UT_WORKQ_IDLE_CLEANUP 0x20
#define UT_WORKQ_EARLY_BOUND 0x40
#define UT_WORKQ_CPUPERCENT 0x80
typedef union workq_threadreq_param_s {
struct {
uint16_t trp_flags;
uint8_t trp_pri;
uint8_t trp_pol;
uint32_t trp_cpupercent: 8,
trp_refillms: 24;
};
uint64_t trp_value;
} workq_threadreq_param_t;
#define TRP_PRIORITY 0x1
#define TRP_POLICY 0x2
#define TRP_CPUPERCENT 0x4
#define TRP_RELEASED 0x8000
__enum_decl(workq_tr_state_t, uint8_t, {
WORKQ_TR_STATE_IDLE = 0,
WORKQ_TR_STATE_NEW = 1,
WORKQ_TR_STATE_QUEUED = 2,
WORKQ_TR_STATE_CANCELED = 3,
WORKQ_TR_STATE_BINDING = 4,
WORKQ_TR_STATE_BOUND = 5,
});
__options_decl(workq_tr_flags_t, uint8_t, {
WORKQ_TR_FLAG_KEVENT = 0x01,
WORKQ_TR_FLAG_WORKLOOP = 0x02,
WORKQ_TR_FLAG_OVERCOMMIT = 0x04,
WORKQ_TR_FLAG_WL_PARAMS = 0x08,
WORKQ_TR_FLAG_WL_OUTSIDE_QOS = 0x10,
});
typedef struct workq_threadreq_s {
union {
struct priority_queue_entry tr_entry;
thread_t tr_thread;
};
uint16_t tr_count;
workq_tr_flags_t tr_flags;
workq_tr_state_t tr_state;
thread_qos_t tr_qos;
kq_index_t tr_kq_override_index;
kq_index_t tr_kq_qos_index;
bool tr_kq_wakeup;
} workq_threadreq_s, *workq_threadreq_t;
TAILQ_HEAD(threadreq_head, workq_threadreq_s);
#if defined(__LP64__)
typedef unsigned __int128 wq_thactive_t;
#else
typedef uint64_t wq_thactive_t;
#endif
__options_decl(workq_state_flags_t, uint32_t, {
WQ_EXITING = 0x0001,
WQ_PROC_SUSPENDED = 0x0002,
WQ_DEATH_CALL_SCHEDULED = 0x0004,
WQ_DELAYED_CALL_SCHEDULED = 0x0010,
WQ_DELAYED_CALL_PENDED = 0x0020,
WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040,
WQ_IMMEDIATE_CALL_PENDED = 0x0080,
});
TAILQ_HEAD(workq_uthread_head, uthread);
struct workqueue {
thread_call_t wq_delayed_call;
thread_call_t wq_immediate_call;
thread_call_t wq_death_call;
union {
struct turnstile *wq_turnstile;
struct mpsc_queue_chain wq_destroy_link;
};
lck_spin_t wq_lock;
uint64_t wq_thread_call_last_run;
struct os_refcnt wq_refcnt;
workq_state_flags_t _Atomic wq_flags;
uint32_t wq_fulfilled;
uint32_t wq_creations;
uint32_t wq_timer_interval;
uint32_t wq_event_manager_priority;
uint32_t wq_reqcount;
uint16_t wq_thdying_count;
uint16_t wq_threads_scheduled;
uint16_t wq_constrained_threads_scheduled;
uint16_t wq_nthreads;
uint16_t wq_thidlecount;
uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS];
_Atomic wq_thactive_t wq_thactive;
_Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS];
struct proc *wq_proc;
struct uthread *wq_creator;
turnstile_inheritor_t wq_inheritor;
thread_t wq_turnstile_updater; struct workq_uthread_head wq_thrunlist;
struct workq_uthread_head wq_thnewlist;
struct workq_uthread_head wq_thidlelist;
struct priority_queue wq_overcommit_queue;
struct priority_queue wq_constrained_queue;
struct priority_queue wq_special_queue;
workq_threadreq_t wq_event_manager_threadreq;
};
#define WORKQUEUE_MAXTHREADS 512
#define WQ_STALLED_WINDOW_USECS 200
#define WQ_REDUCE_POOL_WINDOW_USECS 5000000
#define WQ_MAX_TIMER_INTERVAL_USECS 50000
#pragma mark definitions
struct workq_threadreq_s;
uint32_t _get_pwq_state_kdp(proc_t p);
void workq_exit(struct proc *p);
void workq_mark_exiting(struct proc *p);
bool workq_is_exiting(struct proc *p);
void workq_thread_set_max_qos(struct proc *p, struct workq_threadreq_s *kqr);
void workq_thread_terminate(struct proc *p, struct uthread *uth);
__options_decl(workq_kern_threadreq_flags_t, uint32_t, {
WORKQ_THREADREQ_NONE = 0x00,
WORKQ_THREADREQ_SET_AST_ON_FAILURE = 0x01,
WORKQ_THREADREQ_ATTEMPT_REBIND = 0x02,
WORKQ_THREADREQ_CAN_CREATE_THREADS = 0x04,
WORKQ_THREADREQ_MAKE_OVERCOMMIT = 0x08,
});
bool workq_kern_threadreq_initiate(struct proc *p, struct workq_threadreq_s *kqr,
struct turnstile *ts, thread_qos_t qos, workq_kern_threadreq_flags_t flags);
void workq_kern_threadreq_modify(struct proc *p, struct workq_threadreq_s *kqr,
thread_qos_t qos, workq_kern_threadreq_flags_t flags);
void workq_kern_threadreq_update_inheritor(struct proc *p, struct workq_threadreq_s *kqr,
thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags);
void workq_kern_threadreq_lock(struct proc *p);
void workq_kern_threadreq_unlock(struct proc *p);
void workq_kern_threadreq_redrive(struct proc *p, workq_kern_threadreq_flags_t flags);
enum workq_set_self_flags {
WORKQ_SET_SELF_QOS_FLAG = 0x01,
WORKQ_SET_SELF_VOUCHER_FLAG = 0x02,
WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x04,
WORKQ_SET_SELF_TIMESHARE_FLAG = 0x08,
WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10,
};
void workq_proc_suspended(struct proc *p);
void workq_proc_resumed(struct proc *p);
#endif // BSD_KERNEL_PRIVATE
void workq_init(void);
#endif // KERNEL
#endif // _WORKQUEUE_INTERNAL_H_