#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/mbuf.h>
#include <sys/sysctl.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/protosw.h>
#include <sys/domain.h>
#include <sys/mcache.h>
#include <sys/queue.h>
#include <kern/locks.h>
#include <kern/cpu_number.h>
#include <mach/boolean.h>
#include <net/route.h>
#include <net/if_var.h>
#include <net/ntstat.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/in_pcb.h>
#if INET6
#include <netinet6/in6_pcb.h>
#endif
#include <netinet/ip_var.h>
#include <netinet/tcp.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_cc.h>
#if INET6
#include <netinet6/tcp6_var.h>
#endif
#include <netinet/tcpip.h>
#if TCPDEBUG
#include <netinet/tcp_debug.h>
#endif
#include <sys/kdebug.h>
#include <mach/sdt.h>
#include <netinet/mptcp_var.h>
#define TIMERENTRY_TO_TP(te) ((struct tcpcb *)((uintptr_t)te - offsetof(struct tcpcb, tentry.le.le_next)))
#define VERIFY_NEXT_LINK(elm,field) do { \
if (LIST_NEXT((elm),field) != NULL && \
LIST_NEXT((elm),field)->field.le_prev != \
&((elm)->field.le_next)) \
panic("Bad link elm %p next->prev != elm", (elm)); \
} while(0)
#define VERIFY_PREV_LINK(elm,field) do { \
if (*(elm)->field.le_prev != (elm)) \
panic("Bad link elm %p prev->next != elm", (elm)); \
} while(0)
#define TCP_SET_TIMER_MODE(mode, i) do { \
if (IS_TIMER_HZ_10MS(i)) \
(mode) |= TCP_TIMERLIST_10MS_MODE; \
else if (IS_TIMER_HZ_100MS(i)) \
(mode) |= TCP_TIMERLIST_100MS_MODE; \
else \
(mode) |= TCP_TIMERLIST_500MS_MODE; \
} while(0)
#define TCP_STRETCHACK_DELAY_THRESHOLD 5
struct tcptimerlist tcp_timer_list;
struct tcptailq tcp_tw_tailq;
static int
sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS
{
#pragma unused(arg1, arg2)
int error, s, tt;
tt = *(int *)oidp->oid_arg1;
s = tt * 1000 / TCP_RETRANSHZ;;
error = sysctl_handle_int(oidp, &s, 0, req);
if (error || !req->newptr)
return (error);
tt = s * TCP_RETRANSHZ / 1000;
if (tt < 1)
return (EINVAL);
*(int *)oidp->oid_arg1 = tt;
return (0);
}
int tcp_keepinit;
SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
&tcp_keepinit, 0, sysctl_msec_to_ticks, "I", "");
int tcp_keepidle;
SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
&tcp_keepidle, 0, sysctl_msec_to_ticks, "I", "");
int tcp_keepintvl;
SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
&tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", "");
int tcp_keepcnt;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
&tcp_keepcnt, 0, "number of times to repeat keepalive");
int tcp_msl;
SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
&tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
u_int32_t tcp_max_persist_timeout = 0;
SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
&tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I",
"Maximum persistence timeout for ZWP");
static int always_keepalive = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, always_keepalive,
CTLFLAG_RW | CTLFLAG_LOCKED,
&always_keepalive , 0, "Assume SO_KEEPALIVE on all TCP connections");
int timer_fastmode_idlemax = TCP_FASTMODE_IDLERUN_MAX;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, timer_fastmode_idlemax,
CTLFLAG_RW | CTLFLAG_LOCKED,
&timer_fastmode_idlemax, 0, "Maximum idle generations in fast mode");
static int tcp_broken_peer_syn_rxmit_thres = 7;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, broken_peer_syn_rxmit_thres,
CTLFLAG_RW | CTLFLAG_LOCKED,
&tcp_broken_peer_syn_rxmit_thres, 0,
"Number of retransmitted SYNs before "
"TCP disables rfc1323 and rfc1644 during the rest of attempts");
static int tcp_broken_peer_syn_rxmit_thres_local = 10;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, broken_peer_syn_rexmit_thres_local,
CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_broken_peer_syn_rxmit_thres_local, 0,
"Number of retransmitted SYNs before disabling RFC 1323 "
"options on local connections");
static int tcp_timer_advanced = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_timer_advanced,
CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_timer_advanced, 0,
"Number of times one of the timers was advanced");
static int tcp_resched_timerlist = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist,
CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0,
"Number of times timer list was rescheduled as part of processing a packet");
int tcp_pmtud_black_hole_detect = 1 ;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_detection,
CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_pmtud_black_hole_detect, 0,
"Path MTU Discovery Black Hole Detection");
int tcp_pmtud_black_hole_mss = 1200 ;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_mss,
CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_pmtud_black_hole_mss, 0,
"Path MTU Discovery Black Hole Detection lowered MSS");
static boolean_t tcp_gc_done = FALSE;
int tcp_maxpersistidle;
int tcp_delack = TCP_RETRANSHZ / 10;
#if MPTCP
int tcp_jack_rxmt = TCP_RETRANSHZ / 2;
#endif
static void tcp_remove_timer(struct tcpcb *tp);
static void tcp_sched_timerlist(uint32_t offset);
static u_int32_t tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *mode);
static void tcp_sched_timers(struct tcpcb *tp);
static inline void tcp_set_lotimer_index(struct tcpcb *);
static void tcp_rexmt_save_state(struct tcpcb *tp);
__private_extern__ void tcp_remove_from_time_wait(struct inpcb *inp);
__private_extern__ void tcp_report_stats(void);
inline int32_t
timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2) {
return (int32_t)((t1 + toff1) - (t2 + toff2));
};
static u_int64_t tcp_last_report_time;
#define TCP_REPORT_STATS_INTERVAL 345600
#define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST)
#define TCP_TIMERLIST_MAX_OFFSET (60 * 60 * TCP_RETRANSHZ)
static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay);
static boolean_t tcp_garbage_collect(struct inpcb *, int);
static void
add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay)
{
struct inpcbinfo *pcbinfo = &tcbinfo;
struct inpcb *inp = tp->t_inpcb;
uint32_t timer;
lck_rw_assert(pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE);
if (!(inp->inp_flags2 & INP2_TIMEWAIT)) {
pcbinfo->ipi_twcount++;
inp->inp_flags2 |= INP2_TIMEWAIT;
LIST_REMOVE(inp, inp_list);
} else {
TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
}
timer = tcp_now + delay;
if (TIMER_IS_ON_LIST(tp))
tcp_remove_timer(tp);
tp->t_timer[TCPT_2MSL] = timer;
TAILQ_INSERT_TAIL(&tcp_tw_tailq, tp, t_twentry);
}
void
add_to_time_wait(struct tcpcb *tp, uint32_t delay)
{
struct inpcbinfo *pcbinfo = &tcbinfo;
if (tp->t_inpcb->inp_socket->so_options & SO_NOWAKEFROMSLEEP)
socket_post_kev_msg_closed(tp->t_inpcb->inp_socket);
if (!lck_rw_try_lock_exclusive(pcbinfo->ipi_lock)) {
tcp_unlock(tp->t_inpcb->inp_socket, 0, 0);
lck_rw_lock_exclusive(pcbinfo->ipi_lock);
tcp_lock(tp->t_inpcb->inp_socket, 0, 0);
}
add_to_time_wait_locked(tp, delay);
lck_rw_done(pcbinfo->ipi_lock);
inpcb_gc_sched(pcbinfo, INPCB_TIMER_LAZY);
}
void
tcp_remove_from_time_wait(struct inpcb *inp)
{
struct tcpcb *tp = intotcpcb(inp);
if (inp->inp_flags2 & INP2_TIMEWAIT)
TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
}
static boolean_t
tcp_garbage_collect(struct inpcb *inp, int istimewait)
{
boolean_t active = FALSE;
struct socket *so;
struct tcpcb *tp;
so = inp->inp_socket;
tp = intotcpcb(inp);
if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx))
return (TRUE);
if (so->so_usecount > 1) {
if (inp->inp_wantcnt == WNT_STOPUSING)
active = TRUE;
lck_mtx_unlock(&inp->inpcb_mtx);
return (active);
}
if (istimewait &&
TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) &&
tp->t_state != TCPS_CLOSED) {
lck_mtx_convert_spin(&inp->inpcb_mtx);
tcp_close(tp);
}
if (so->so_usecount == 1 &&
((istimewait && (so->so_flags & SOF_OVERFLOW)) ||
((tp != NULL) && (tp->t_state == TCPS_CLOSED) &&
(so->so_head != NULL) &&
((so->so_state & (SS_INCOMP|SS_CANTSENDMORE|SS_CANTRCVMORE)) ==
(SS_INCOMP|SS_CANTSENDMORE|SS_CANTRCVMORE))))) {
if (inp->inp_state != INPCB_STATE_DEAD) {
lck_mtx_convert_spin(&inp->inpcb_mtx);
#if INET6
if (SOCK_CHECK_DOM(so, PF_INET6))
in6_pcbdetach(inp);
else
#endif
in_pcbdetach(inp);
}
so->so_usecount--;
if (inp->inp_wantcnt == WNT_STOPUSING)
active = TRUE;
lck_mtx_unlock(&inp->inpcb_mtx);
return (active);
} else if (inp->inp_wantcnt != WNT_STOPUSING) {
lck_mtx_unlock(&inp->inpcb_mtx);
return (FALSE);
}
if (so->so_usecount == 0) {
DTRACE_TCP4(state__change, void, NULL, struct inpcb *, inp,
struct tcpcb *, tp, int32_t, TCPS_CLOSED);
lck_mtx_convert_spin(&inp->inpcb_mtx);
if (TIMER_IS_ON_LIST(tp)) {
tcp_remove_timer(tp);
}
if (inp->inp_state != INPCB_STATE_DEAD) {
#if INET6
if (SOCK_CHECK_DOM(so, PF_INET6))
in6_pcbdetach(inp);
else
#endif
in_pcbdetach(inp);
}
in_pcbdispose(inp);
return (FALSE);
}
lck_mtx_unlock(&inp->inpcb_mtx);
return (TRUE);
}
void
tcp_gc(struct inpcbinfo *ipi)
{
struct inpcb *inp, *nxt;
struct tcpcb *tw_tp, *tw_ntp;
#if TCPDEBUG
int ostate;
#endif
#if KDEBUG
static int tws_checked = 0;
#endif
KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_START, 0, 0, 0, 0, 0);
calculate_tcp_clock();
if (lck_rw_try_lock_exclusive(ipi->ipi_lock) == FALSE) {
if (tcp_gc_done == TRUE) {
tcp_gc_done = FALSE;
KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END,
tws_checked, cur_tw_slot, 0, 0, 0);
atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1);
return;
}
lck_rw_lock_exclusive(ipi->ipi_lock);
}
tcp_gc_done = TRUE;
LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
if (tcp_garbage_collect(inp, 0))
atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1);
}
TAILQ_FOREACH_SAFE(tw_tp, &tcp_tw_tailq, t_twentry, tw_ntp) {
if (tw_tp->t_state == TCPS_CLOSED ||
TSTMP_GEQ(tcp_now, tw_tp->t_timer[TCPT_2MSL])) {
if (tcp_garbage_collect(tw_tp->t_inpcb, 1))
atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, 1);
}
}
atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, ipi->ipi_twcount);
lck_rw_done(ipi->ipi_lock);
if (so_cache_timer())
atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, 1);
KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, tws_checked,
cur_tw_slot, 0, 0, 0);
return;
}
void
tcp_canceltimers(tp)
struct tcpcb *tp;
{
register int i;
tcp_remove_timer(tp);
for (i = 0; i < TCPT_NTIMERS; i++)
tp->t_timer[i] = 0;
tp->tentry.timer_start = tcp_now;
tp->tentry.index = TCPT_NONE;
}
int tcp_syn_backoff[TCP_MAXRXTSHIFT + 1] =
{ 1, 1, 1, 1, 1, 2, 4, 8, 16, 32, 64, 64, 64 };
int tcp_backoff[TCP_MAXRXTSHIFT + 1] =
{ 1, 2, 4, 8, 16, 32, 64, 64, 64, 64, 64, 64, 64 };
static int tcp_totbackoff = 511;
static void tcp_rexmt_save_state(struct tcpcb *tp)
{
u_int32_t fsize;
if (TSTMP_SUPPORTED(tp)) {
fsize = tp->snd_max - tp->snd_una;
tp->snd_ssthresh_prev = max(fsize, tp->snd_ssthresh);
tp->snd_recover_prev = tp->snd_recover;
} else {
tp->snd_cwnd_prev = tp->snd_cwnd;
tp->snd_ssthresh_prev = tp->snd_ssthresh;
tp->snd_recover_prev = tp->snd_recover;
if (IN_FASTRECOVERY(tp))
tp->t_flags |= TF_WASFRECOVERY;
else
tp->t_flags &= ~TF_WASFRECOVERY;
}
tp->t_srtt_prev = (tp->t_srtt >> TCP_RTT_SHIFT) + 2;
tp->t_rttvar_prev = (tp->t_rttvar >> TCP_RTTVAR_SHIFT);
tp->t_flagsext &= ~(TF_RECOMPUTE_RTT);
}
void tcp_pmtud_revert_segment_size(struct tcpcb *tp)
{
int32_t optlen;
VERIFY(tp->t_pmtud_saved_maxopd > 0);
tp->t_flags |= TF_PMTUD;
tp->t_flags &= ~TF_BLACKHOLE;
optlen = tp->t_maxopd - tp->t_maxseg;
tp->t_maxopd = tp->t_pmtud_saved_maxopd;
tp->t_maxseg = tp->t_maxopd - optlen;
if (CC_ALGO(tp)->cwnd_init != NULL)
CC_ALGO(tp)->cwnd_init(tp);
tp->t_pmtud_start_ts = 0;
tcpstat.tcps_pmtudbh_reverted++;
}
struct tcpcb *
tcp_timers(tp, timer)
register struct tcpcb *tp;
int timer;
{
int32_t rexmt, optlen = 0, idle_time = 0;
struct socket *so;
struct tcptemp *t_template;
#if TCPDEBUG
int ostate;
#endif
#if INET6
int isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV4) == 0;
#endif
so = tp->t_inpcb->inp_socket;
idle_time = tcp_now - tp->t_rcvtime;
switch (timer) {
case TCPT_2MSL:
tcp_free_sackholes(tp);
if (tp->t_state != TCPS_TIME_WAIT &&
tp->t_state != TCPS_FIN_WAIT_2 &&
((idle_time > 0) && (idle_time < TCP_CONN_MAXIDLE(tp)))) {
tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp,
(u_int32_t)TCP_CONN_KEEPINTVL(tp));
} else {
tp = tcp_close(tp);
return(tp);
}
break;
case TCPT_REXMT:
if (++tp->t_rxtshift > TCP_MAXRXTSHIFT ||
(tp->t_rxt_conndroptime > 0
&& tp->t_rxtstart > 0 &&
(tcp_now - tp->t_rxtstart) >= tp->t_rxt_conndroptime)
|| ((tp->t_flagsext & TF_RXTFINDROP) != 0 &&
(tp->t_flags & TF_SENTFIN) != 0 &&
tp->t_rxtshift >= 4)) {
if ((tp->t_flagsext & TF_RXTFINDROP) != 0) {
tcpstat.tcps_rxtfindrop++;
} else {
tcpstat.tcps_timeoutdrop++;
}
tp->t_rxtshift = TCP_MAXRXTSHIFT;
postevent(so, 0, EV_TIMEOUT);
soevent(so,
(SO_FILT_HINT_LOCKED|SO_FILT_HINT_TIMEOUT));
tp = tcp_drop(tp, tp->t_softerror ?
tp->t_softerror : ETIMEDOUT);
break;
}
tcpstat.tcps_rexmttimeo++;
if (tp->t_rxtshift == 1 &&
tp->t_state == TCPS_ESTABLISHED) {
tp->t_rxtstart = tcp_now;
tcp_rexmt_save_state(tp);
}
#if MPTCP
if ((tp->t_rxtshift >= mptcp_fail_thresh) &&
(tp->t_state == TCPS_ESTABLISHED) &&
(tp->t_mpflags & TMPF_MPTCP_TRUE)) {
mptcp_act_on_txfail(so);
}
#endif
if (tp->t_adaptive_wtimo > 0 &&
tp->t_rxtshift > tp->t_adaptive_wtimo &&
TCPS_HAVEESTABLISHED(tp->t_state)) {
soevent(so,
(SO_FILT_HINT_LOCKED|
SO_FILT_HINT_ADAPTIVE_WTIMO));
}
if (tp->t_flagsext & TF_SENT_TLPROBE) {
tp->t_flagsext &= ~(TF_SENT_TLPROBE);
tcpstat.tcps_rto_after_pto++;
}
if (tp->t_flagsext & TF_DELAY_RECOVERY) {
tp->t_reorderwin = 0;
tp->t_timer[TCPT_DELAYFR] = 0;
tp->t_flagsext &= ~(TF_DELAY_RECOVERY);
}
if (tp->t_state == TCPS_SYN_SENT) {
rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift];
tp->t_stat.synrxtshift = tp->t_rxtshift;
} else {
rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
}
TCPT_RANGESET(tp->t_rxtcur, rexmt,
tp->t_rttmin, TCPTV_REXMTMAX,
TCP_ADD_REXMTSLOP(tp));
tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
if (INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb))
goto fc_output;
tcp_free_sackholes(tp);
if (tcp_pmtud_black_hole_detect &&
!(tp->t_flagsext & TF_NOBLACKHOLE_DETECTION) &&
(tp->t_state == TCPS_ESTABLISHED)) {
if (((tp->t_flags & (TF_PMTUD|TF_MAXSEGSNT))
== (TF_PMTUD|TF_MAXSEGSNT)) &&
(tp->t_rxtshift == 2)) {
tp->t_flags &= ~TF_PMTUD;
tp->t_flags |= TF_BLACKHOLE;
optlen = tp->t_maxopd - tp->t_maxseg;
tp->t_pmtud_saved_maxopd = tp->t_maxopd;
tp->t_pmtud_start_ts = tcp_now;
if (tp->t_pmtud_start_ts == 0)
tp->t_pmtud_start_ts++;
if (tp->t_maxopd > tcp_pmtud_black_hole_mss) {
tp->t_maxopd = tcp_pmtud_black_hole_mss;
} else {
tp->t_maxopd =
#if INET6
isipv6 ? tcp_v6mssdflt :
#endif
tcp_mssdflt;
}
tp->t_maxseg = tp->t_maxopd - optlen;
if (CC_ALGO(tp)->cwnd_init != NULL)
CC_ALGO(tp)->cwnd_init(tp);
}
else {
if ((tp->t_flags & TF_BLACKHOLE) &&
(tp->t_rxtshift > 4)) {
tcp_pmtud_revert_segment_size(tp);
}
}
}
if (tp->t_state == TCPS_SYN_SENT &&
((!(tp->t_flags & TF_LOCAL) &&
tp->t_rxtshift == tcp_broken_peer_syn_rxmit_thres) ||
((tp->t_flags & TF_LOCAL) &&
tp->t_rxtshift == tcp_broken_peer_syn_rxmit_thres_local)))
tp->t_flags &= ~(TF_REQ_SCALE|TF_REQ_TSTMP|TF_REQ_CC);
if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
#if INET6
if (isipv6)
in6_losing(tp->t_inpcb);
else
#endif
in_losing(tp->t_inpcb);
tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
tp->t_srtt = 0;
}
tp->snd_nxt = tp->snd_una;
tp->snd_recover = tp->snd_max;
tp->t_flags |= TF_ACKNOW;
tp->t_rtttime = 0;
if (!IN_FASTRECOVERY(tp) && tp->t_rxtshift == 1)
tcpstat.tcps_tailloss_rto++;
if (tp->t_rxtshift == 1 &&
CC_ALGO(tp)->after_timeout != NULL)
CC_ALGO(tp)->after_timeout(tp);
EXIT_FASTRECOVERY(tp);
if ((tp->ecn_flags & TE_ECN_ON) == TE_ECN_ON) {
tp->ecn_flags |= TE_SENDCWR;
}
fc_output:
tcp_ccdbg_trace(tp, NULL, TCP_CC_REXMT_TIMEOUT);
(void) tcp_output(tp);
break;
case TCPT_PERSIST:
tcpstat.tcps_persisttimeo++;
if ((tp->t_rxtshift == TCP_MAXRXTSHIFT &&
(idle_time >= tcp_maxpersistidle ||
idle_time >= TCP_REXMTVAL(tp) * tcp_totbackoff)) ||
((tp->t_persist_stop != 0) &&
TSTMP_LEQ(tp->t_persist_stop, tcp_now))) {
tcpstat.tcps_persistdrop++;
postevent(so, 0, EV_TIMEOUT);
soevent(so,
(SO_FILT_HINT_LOCKED|SO_FILT_HINT_TIMEOUT));
tp = tcp_drop(tp, ETIMEDOUT);
break;
}
tcp_setpersist(tp);
tp->t_flagsext |= TF_FORCE;
(void) tcp_output(tp);
tp->t_flagsext &= ~TF_FORCE;
break;
case TCPT_KEEP:
tcpstat.tcps_keeptimeo++;
#if MPTCP
struct mptcb *mp_tp = tp->t_mptcb;
if ((tp->t_mpflags & TMPF_MPTCP_TRUE) &&
(tp->t_state > TCPS_ESTABLISHED)) {
goto dropit;
} else if (mp_tp != NULL) {
if ((mptcp_ok_to_keepalive(mp_tp) == 0))
goto dropit;
}
#endif
if (tp->t_state < TCPS_ESTABLISHED)
goto dropit;
if ((always_keepalive ||
(tp->t_inpcb->inp_socket->so_options & SO_KEEPALIVE) ||
(tp->t_flagsext & TF_DETECT_READSTALL)) &&
(tp->t_state <= TCPS_CLOSING || tp->t_state == TCPS_FIN_WAIT_2)) {
if (idle_time >= TCP_CONN_KEEPIDLE(tp) + TCP_CONN_MAXIDLE(tp))
goto dropit;
tcpstat.tcps_keepprobe++;
t_template = tcp_maketemplate(tp);
if (t_template) {
struct inpcb *inp = tp->t_inpcb;
struct tcp_respond_args tra;
bzero(&tra, sizeof(tra));
tra.nocell = INP_NO_CELLULAR(inp);
tra.noexpensive = INP_NO_EXPENSIVE(inp);
tra.awdl_unrestricted = INP_AWDL_UNRESTRICTED(inp);
if (tp->t_inpcb->inp_flags & INP_BOUND_IF)
tra.ifscope = tp->t_inpcb->inp_boundifp->if_index;
else
tra.ifscope = IFSCOPE_NONE;
tcp_respond(tp, t_template->tt_ipgen,
&t_template->tt_t, (struct mbuf *)NULL,
tp->rcv_nxt, tp->snd_una - 1, 0, &tra);
(void) m_free(dtom(t_template));
if (tp->t_flagsext & TF_DETECT_READSTALL)
tp->t_rtimo_probes++;
}
tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
TCP_CONN_KEEPINTVL(tp));
} else {
tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
TCP_CONN_KEEPIDLE(tp));
}
if (tp->t_flagsext & TF_DETECT_READSTALL) {
if (tp->t_rtimo_probes > tp->t_adaptive_rtimo) {
soevent(so,
(SO_FILT_HINT_LOCKED|
SO_FILT_HINT_ADAPTIVE_RTIMO));
tcp_keepalive_reset(tp);
} else {
tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(
tp, TCP_REXMTVAL(tp));
}
}
break;
case TCPT_DELACK:
if (tcp_delack_enabled && (tp->t_flags & TF_DELACK)) {
tp->t_flags &= ~TF_DELACK;
tp->t_timer[TCPT_DELACK] = 0;
tp->t_flags |= TF_ACKNOW;
if ((tp->t_flags & TF_STRETCHACK)) {
if (tp->t_unacksegs > 1 &&
tp->t_unacksegs < maxseg_unacked)
tp->t_stretchack_delayed++;
if (tp->t_stretchack_delayed >
TCP_STRETCHACK_DELAY_THRESHOLD) {
tp->t_flagsext |= TF_DISABLE_STRETCHACK;
tp->rcv_nostrack_ts = tcp_now;
tcpstat.tcps_nostretchack++;
tp->t_stretchack_delayed = 0;
}
tcp_reset_stretch_ack(tp);
}
CLEAR_IAJ_STATE(tp);
tcpstat.tcps_delack++;
(void) tcp_output(tp);
}
break;
#if MPTCP
case TCPT_JACK_RXMT:
if ((tp->t_state == TCPS_ESTABLISHED) &&
(tp->t_mpflags & TMPF_PREESTABLISHED) &&
(tp->t_mpflags & TMPF_JOINED_FLOW)) {
if (++tp->t_mprxtshift > TCP_MAXRXTSHIFT) {
tcpstat.tcps_timeoutdrop++;
postevent(so, 0, EV_TIMEOUT);
soevent(so,
(SO_FILT_HINT_LOCKED|
SO_FILT_HINT_TIMEOUT));
tp = tcp_drop(tp, tp->t_softerror ?
tp->t_softerror : ETIMEDOUT);
break;
}
tcpstat.tcps_join_rxmts++;
tp->t_flags |= TF_ACKNOW;
(void) tcp_output(tp);
}
break;
#endif
case TCPT_PTO:
{
tcp_seq old_snd_nxt;
int32_t snd_len;
boolean_t rescue_rxt = FALSE;
tp->t_flagsext &= ~(TF_SENT_TLPROBE);
if (tp->t_state != TCPS_ESTABLISHED ||
tp->t_rxtshift > 0 || tp->snd_max == tp->snd_una ||
!SACK_ENABLED(tp) || TAILQ_EMPTY(&tp->snd_holes) ||
(IN_FASTRECOVERY(tp) &&
(SEQ_GEQ(tp->snd_fack, tp->snd_recover) ||
SEQ_GT(tp->snd_nxt, tp->sack_newdata))))
break;
tcpstat.tcps_pto++;
tp->t_rtttime = 0;
if (IN_FASTRECOVERY(tp)) {
old_snd_nxt = tp->snd_nxt;
rescue_rxt = TRUE;
VERIFY(SEQ_GEQ(tp->snd_fack, tp->snd_una));
snd_len = min((tp->snd_recover - tp->snd_fack),
tp->t_maxseg);
tp->snd_nxt = tp->snd_recover - snd_len;
tcpstat.tcps_pto_in_recovery++;
tcp_ccdbg_trace(tp, NULL, TCP_CC_TLP_IN_FASTRECOVERY);
} else {
snd_len = min(so->so_snd.sb_cc, tp->snd_wnd)
- (tp->snd_max - tp->snd_una);
if (snd_len > 0) {
tp->snd_nxt = tp->snd_max;
} else {
snd_len = min((tp->snd_max - tp->snd_una),
tp->t_maxseg);
tp->snd_nxt = tp->snd_max - snd_len;
}
}
tp->t_flagsext |= TF_SENT_TLPROBE;
tp->t_tlpstart = tcp_now;
tp->snd_cwnd += tp->t_maxseg;
(void )tcp_output(tp);
tp->snd_cwnd -= tp->t_maxseg;
tp->t_tlphighrxt = tp->snd_nxt;
if (rescue_rxt)
tp->snd_nxt = old_snd_nxt;
break;
}
case TCPT_DELAYFR:
tp->t_flagsext &= ~TF_DELAY_RECOVERY;
if (IN_FASTRECOVERY(tp) ||
SEQ_GEQ(tp->snd_una, tp->snd_recover) ||
tp->t_rxtshift > 0)
break;
VERIFY(SACK_ENABLED(tp));
if (CC_ALGO(tp)->pre_fr != NULL)
CC_ALGO(tp)->pre_fr(tp);
ENTER_FASTRECOVERY(tp);
if ((tp->ecn_flags & TE_ECN_ON) == TE_ECN_ON)
tp->ecn_flags |= TE_SENDCWR;
tp->t_timer[TCPT_REXMT] = 0;
tcpstat.tcps_sack_recovery_episode++;
tp->sack_newdata = tp->snd_nxt;
tp->snd_cwnd = tp->t_maxseg;
tcp_ccdbg_trace(tp, NULL, TCP_CC_ENTER_FASTRECOVERY);
(void) tcp_output(tp);
break;
dropit:
tcpstat.tcps_keepdrops++;
postevent(so, 0, EV_TIMEOUT);
soevent(so,
(SO_FILT_HINT_LOCKED|SO_FILT_HINT_TIMEOUT));
tp = tcp_drop(tp, ETIMEDOUT);
break;
}
#if TCPDEBUG
if (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)
tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0,
PRU_SLOWTIMO);
#endif
return (tp);
}
void
tcp_remove_timer(struct tcpcb *tp)
{
struct tcptimerlist *listp = &tcp_timer_list;
lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
if (!(TIMER_IS_ON_LIST(tp))) {
return;
}
lck_mtx_lock(listp->mtx);
if (!(TIMER_IS_ON_LIST(tp))) {
lck_mtx_unlock(listp->mtx);
return;
}
if (listp->next_te != NULL && listp->next_te == &tp->tentry)
listp->next_te = LIST_NEXT(&tp->tentry, le);
LIST_REMOVE(&tp->tentry, le);
tp->t_flags &= ~(TF_TIMER_ONLIST);
listp->entries--;
tp->tentry.le.le_next = NULL;
tp->tentry.le.le_prev = NULL;
lck_mtx_unlock(listp->mtx);
}
static boolean_t
need_to_resched_timerlist(u_int32_t runtime, u_int16_t mode)
{
struct tcptimerlist *listp = &tcp_timer_list;
int32_t diff;
if (listp->running)
return (TRUE);
if (!listp->scheduled)
return (TRUE);
diff = timer_diff(listp->runtime, 0, runtime, 0);
if (diff <= 0) {
return (FALSE);
} else {
if (mode & TCP_TIMERLIST_10MS_MODE) {
if (diff <= TCP_TIMER_10MS_QUANTUM)
return (FALSE);
} else if (mode & TCP_TIMERLIST_100MS_MODE) {
if (diff <= TCP_TIMER_100MS_QUANTUM)
return (FALSE);
} else {
if (diff <= TCP_TIMER_500MS_QUANTUM)
return (FALSE);
}
}
return (TRUE);
}
void
tcp_sched_timerlist(uint32_t offset)
{
uint64_t deadline = 0;
struct tcptimerlist *listp = &tcp_timer_list;
lck_mtx_assert(listp->mtx, LCK_MTX_ASSERT_OWNED);
offset = min(offset, TCP_TIMERLIST_MAX_OFFSET);
listp->runtime = tcp_now + offset;
if (listp->runtime == 0) {
listp->runtime++;
offset++;
}
clock_interval_to_deadline(offset, USEC_PER_SEC, &deadline);
thread_call_enter_delayed(listp->call, deadline);
listp->scheduled = TRUE;
}
u_int32_t
tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode) {
struct socket *so;
u_int16_t i = 0, index = TCPT_NONE, lo_index = TCPT_NONE;
u_int32_t timer_val, offset = 0, lo_timer = 0;
int32_t diff;
boolean_t needtorun[TCPT_NTIMERS];
int count = 0;
VERIFY(tp != NULL);
bzero(needtorun, sizeof(needtorun));
*te_mode = 0;
tcp_lock(tp->t_inpcb->inp_socket, 1, 0);
so = tp->t_inpcb->inp_socket;
if (in_pcb_checkstate(tp->t_inpcb, WNT_RELEASE, 1)
== WNT_STOPUSING) {
if (TIMER_IS_ON_LIST(tp)) {
tcp_remove_timer(tp);
}
goto done;
}
if ((index = tp->tentry.index) == TCPT_NONE)
goto done;
timer_val = tp->t_timer[index];
diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0);
if (diff > 0) {
if (tp->tentry.index != TCPT_NONE) {
offset = diff;
*(te_mode) = tp->tentry.mode;
}
goto done;
}
tp->t_timer[index] = 0;
if (timer_val > 0) {
tp = tcp_timers(tp, index);
if (tp == NULL)
goto done;
}
tp->tentry.mode = 0;
for (i = 0; i < TCPT_NTIMERS; ++i) {
if (tp->t_timer[i] != 0) {
diff = timer_diff(tp->tentry.timer_start,
tp->t_timer[i], tcp_now, 0);
if (diff <= 0) {
needtorun[i] = TRUE;
count++;
} else {
tp->t_timer[i] = diff;
needtorun[i] = FALSE;
if (lo_timer == 0 || diff < lo_timer) {
lo_timer = diff;
lo_index = i;
}
TCP_SET_TIMER_MODE(tp->tentry.mode, i);
}
}
}
tp->tentry.timer_start = tcp_now;
tp->tentry.index = lo_index;
VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0);
if (tp->tentry.index != TCPT_NONE) {
tp->tentry.runtime = tp->tentry.timer_start +
tp->t_timer[tp->tentry.index];
if (tp->tentry.runtime == 0)
tp->tentry.runtime++;
}
if (count > 0) {
for (i = 0; i < TCPT_NTIMERS; ++i) {
if (needtorun[i]) {
tp->t_timer[i] = 0;
tp = tcp_timers(tp, i);
if (tp == NULL) {
offset = 0;
*(te_mode) = 0;
goto done;
}
}
}
tcp_set_lotimer_index(tp);
}
if (tp->tentry.index < TCPT_NONE) {
offset = tp->t_timer[tp->tentry.index];
*(te_mode) = tp->tentry.mode;
}
done:
if (tp != NULL && tp->tentry.index == TCPT_NONE) {
tcp_remove_timer(tp);
offset = 0;
}
tcp_unlock(so, 1, 0);
return(offset);
}
void
tcp_run_timerlist(void * arg1, void * arg2) {
#pragma unused(arg1, arg2)
struct tcptimerentry *te, *next_te;
struct tcptimerlist *listp = &tcp_timer_list;
struct tcpcb *tp;
uint32_t next_timer = 0;
u_int16_t te_mode = 0;
u_int16_t list_mode = 0;
uint32_t active_count = 0;
calculate_tcp_clock();
lck_mtx_lock(listp->mtx);
listp->running = TRUE;
LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) {
uint32_t offset = 0;
uint32_t runtime = te->runtime;
if (te->index < TCPT_NONE && TSTMP_GT(runtime, tcp_now)) {
offset = timer_diff(runtime, 0, tcp_now, 0);
if (next_timer == 0 || offset < next_timer) {
next_timer = offset;
}
list_mode |= te->mode;
continue;
}
tp = TIMERENTRY_TO_TP(te);
if (in_pcb_checkstate(tp->t_inpcb, WNT_ACQUIRE, 0)
== WNT_STOPUSING) {
if (TIMER_IS_ON_LIST(tp)) {
tp->t_flags &= ~(TF_TIMER_ONLIST);
LIST_REMOVE(&tp->tentry, le);
listp->entries--;
tp->tentry.le.le_next = NULL;
tp->tentry.le.le_prev = NULL;
}
continue;
}
active_count++;
listp->next_te = next_te;
VERIFY_NEXT_LINK(&tp->tentry, le);
VERIFY_PREV_LINK(&tp->tentry, le);
lck_mtx_unlock(listp->mtx);
offset = tcp_run_conn_timer(tp, &te_mode);
lck_mtx_lock(listp->mtx);
next_te = listp->next_te;
listp->next_te = NULL;
if (offset > 0 && te_mode != 0) {
list_mode |= te_mode;
if (next_timer == 0 || offset < next_timer)
next_timer = offset;
}
}
if (!LIST_EMPTY(&listp->lhead)) {
u_int16_t next_mode = 0;
if ((list_mode & TCP_TIMERLIST_10MS_MODE) ||
(listp->pref_mode & TCP_TIMERLIST_10MS_MODE))
next_mode = TCP_TIMERLIST_10MS_MODE;
else if ((list_mode & TCP_TIMERLIST_100MS_MODE) ||
(listp->pref_mode & TCP_TIMERLIST_100MS_MODE))
next_mode = TCP_TIMERLIST_100MS_MODE;
else
next_mode = TCP_TIMERLIST_500MS_MODE;
if (next_mode != TCP_TIMERLIST_500MS_MODE) {
listp->idleruns = 0;
} else {
if (listp->mode != next_mode &&
listp->idleruns < timer_fastmode_idlemax) {
listp->idleruns++;
next_mode = listp->mode;
next_timer = TCP_TIMER_100MS_QUANTUM;
} else {
listp->idleruns = 0;
}
}
listp->mode = next_mode;
if (listp->pref_offset != 0)
next_timer = min(listp->pref_offset, next_timer);
if (listp->mode == TCP_TIMERLIST_500MS_MODE)
next_timer = max(next_timer,
TCP_TIMER_500MS_QUANTUM);
tcp_sched_timerlist(next_timer);
} else {
tcp_sched_timerlist(TCP_TIMERLIST_MAX_OFFSET);
}
listp->running = FALSE;
listp->pref_mode = 0;
listp->pref_offset = 0;
lck_mtx_unlock(listp->mtx);
}
void
tcp_sched_timers(struct tcpcb *tp)
{
struct tcptimerentry *te = &tp->tentry;
u_int16_t index = te->index;
u_int16_t mode = te->mode;
struct tcptimerlist *listp = &tcp_timer_list;
int32_t offset = 0;
boolean_t list_locked = FALSE;
if (tp->t_inpcb->inp_state == INPCB_STATE_DEAD) {
if (TIMER_IS_ON_LIST(tp)) {
tcp_remove_timer(tp);
}
return;
}
if (index == TCPT_NONE) {
tcp_remove_timer(tp);
return;
}
offset = timer_diff(te->runtime, 0, tcp_now, 0);
if (offset <= 0) {
offset = 1;
tcp_timer_advanced++;
}
if (!TIMER_IS_ON_LIST(tp)) {
if (!list_locked) {
lck_mtx_lock(listp->mtx);
list_locked = TRUE;
}
LIST_INSERT_HEAD(&listp->lhead, te, le);
tp->t_flags |= TF_TIMER_ONLIST;
listp->entries++;
if (listp->entries > listp->maxentries)
listp->maxentries = listp->entries;
if (!listp->scheduled)
goto schedule;
}
if (need_to_resched_timerlist(te->runtime, mode)) {
tcp_resched_timerlist++;
if (!list_locked) {
lck_mtx_lock(listp->mtx);
list_locked = TRUE;
}
VERIFY_NEXT_LINK(te, le);
VERIFY_PREV_LINK(te, le);
if (listp->running) {
listp->pref_mode |= mode;
if (listp->pref_offset == 0 ||
offset < listp->pref_offset) {
listp->pref_offset = offset;
}
} else {
if (listp->scheduled) {
int32_t diff;
diff = timer_diff(listp->runtime, 0,
tcp_now, offset);
if (diff <= 0)
goto done;
else
goto schedule;
} else {
goto schedule;
}
}
}
goto done;
schedule:
if (mode & TCP_TIMERLIST_10MS_MODE) {
listp->mode = TCP_TIMERLIST_10MS_MODE;
listp->idleruns = 0;
offset = min(offset, TCP_TIMER_10MS_QUANTUM);
} else if (mode & TCP_TIMERLIST_100MS_MODE) {
if (listp->mode > TCP_TIMERLIST_100MS_MODE)
listp->mode = TCP_TIMERLIST_100MS_MODE;
listp->idleruns = 0;
offset = min(offset, TCP_TIMER_100MS_QUANTUM);
}
tcp_sched_timerlist(offset);
done:
if (list_locked)
lck_mtx_unlock(listp->mtx);
return;
}
static inline void
tcp_set_lotimer_index(struct tcpcb *tp) {
uint16_t i, lo_index = TCPT_NONE, mode = 0;
uint32_t lo_timer = 0;
for (i = 0; i < TCPT_NTIMERS; ++i) {
if (tp->t_timer[i] != 0) {
TCP_SET_TIMER_MODE(mode, i);
if (lo_timer == 0 || tp->t_timer[i] < lo_timer) {
lo_timer = tp->t_timer[i];
lo_index = i;
}
}
}
tp->tentry.index = lo_index;
tp->tentry.mode = mode;
VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0);
if (tp->tentry.index != TCPT_NONE) {
tp->tentry.runtime = tp->tentry.timer_start
+ tp->t_timer[tp->tentry.index];
if (tp->tentry.runtime == 0)
tp->tentry.runtime++;
}
}
void
tcp_check_timer_state(struct tcpcb *tp) {
lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT)
return;
tcp_set_lotimer_index(tp);
tcp_sched_timers(tp);
return;
}
__private_extern__ void
tcp_report_stats(void)
{
struct nstat_sysinfo_data data;
struct sockaddr_in dst;
struct sockaddr_in6 dst6;
struct rtentry *rt = NULL;
u_int64_t var, uptime;
#define stat data.u.tcp_stats
if (((uptime = net_uptime()) - tcp_last_report_time) <
TCP_REPORT_STATS_INTERVAL)
return;
tcp_last_report_time = uptime;
bzero(&data, sizeof(data));
data.flags = NSTAT_SYSINFO_TCP_STATS;
bzero(&dst, sizeof(dst));
dst.sin_len = sizeof(dst);
dst.sin_family = AF_INET;
lck_mtx_lock(rnh_lock);
rt = rt_lookup(TRUE, (struct sockaddr *)&dst, NULL,
rt_tables[AF_INET], IFSCOPE_NONE);
lck_mtx_unlock(rnh_lock);
if (rt != NULL) {
RT_LOCK(rt);
if (rt_primary_default(rt, rt_key(rt)) &&
rt->rt_stats != NULL) {
stat.ipv4_avgrtt = rt->rt_stats->nstat_avg_rtt;
}
RT_UNLOCK(rt);
rtfree(rt);
rt = NULL;
}
bzero(&dst6, sizeof(dst6));
dst6.sin6_len = sizeof(dst6);
dst6.sin6_family = AF_INET6;
lck_mtx_lock(rnh_lock);
rt = rt_lookup(TRUE,(struct sockaddr *)&dst6, NULL,
rt_tables[AF_INET6], IFSCOPE_NONE);
lck_mtx_unlock(rnh_lock);
if (rt != NULL) {
RT_LOCK(rt);
if (rt_primary_default(rt, rt_key(rt)) &&
rt->rt_stats != NULL) {
stat.ipv6_avgrtt = rt->rt_stats->nstat_avg_rtt;
}
RT_UNLOCK(rt);
rtfree(rt);
rt = NULL;
}
if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_sndrexmitpack > 0) {
var = tcpstat.tcps_sndrexmitpack << 10;
stat.send_plr = (var * 100) / tcpstat.tcps_sndpack;
}
if (tcpstat.tcps_rcvpack > 0 && tcpstat.tcps_recovered_pkts > 0) {
var = tcpstat.tcps_recovered_pkts << 10;
stat.recv_plr = (var * 100) / tcpstat.tcps_rcvpack;
}
if (tcpstat.tcps_sndrexmitpack > 0
&& tcpstat.tcps_tailloss_rto > 0) {
var = tcpstat.tcps_tailloss_rto << 10;
stat.send_tlrto_rate =
(var * 100) / tcpstat.tcps_sndrexmitpack;
}
if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_reordered_pkts > 0) {
var = tcpstat.tcps_reordered_pkts << 10;
stat.send_reorder_rate =
(var * 100) / tcpstat.tcps_sndpack;
}
nstat_sysinfo_send_data(&data);
#undef stat
}