jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
728 Commits
15 Branches
2 Tags
165 MiB
 
 
 
 
 
 
Tree: 51d5790938
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '51d5790938'
${ noResults }
utun2/src/etcp_bbr.c

925 lines
30 KiB
Raw Blame History

#include "etcp_bbr.h"
#include "../lib/u_async.h"
#include <stdlib.h>
static uint64_t bbr_now(const struct bbr* bbr) { return bbr->now_tb ? bbr->now_tb : get_time_tb(); }
static void bbr_start_bw_probe_down(struct bbr* bbr);
static void bbr_start_bw_probe_cruise(struct bbr* bbr);
static void bbr_exit_probe_rtt(struct bbr* bbr);
static const int bbr_pacing_gain[] = {
BBR_UNIT * 5 / 4, /* PROBE_UP: 1.25 */
BBR_UNIT * 3 / 4, /* PROBE_DOWN: 0.75 */
BBR_UNIT, /* CRUISE: 1.00 */
BBR_UNIT, /* REFILL: 1.00 */
};
static uint32_t bbr_max_bw(const struct bbr* bbr)
{
return (uint32_t)(bbr->bw_hi[0] > bbr->bw_hi[1] ? bbr->bw_hi[0] : bbr->bw_hi[1]);
}
static uint32_t bbr_bw(const struct bbr* bbr)
{
uint32_t max_bw = bbr_max_bw(bbr);
return (uint32_t)(max_bw < bbr->bw_lo ? max_bw : bbr->bw_lo);
}
static int bbr_full_bw_reached(const struct bbr* bbr)
{
return bbr->full_bw_reached;
}
static uint16_t bbr_extra_acked(const struct bbr* bbr)
{
return (uint16_t)(bbr->extra_acked[0] > bbr->extra_acked[1] ?
bbr->extra_acked[0] : bbr->extra_acked[1]);
}
static uint64_t bbr_rate_bytes_per_sec(uint64_t rate, int gain, int margin, uint32_t mss)
{
rate *= mss;
rate *= (uint64_t)gain;
rate >>= BBR_SCALE;
rate *= (uint64_t)(1000000 / 100) * (uint64_t)(100 - margin);
rate >>= BW_SCALE;
if (rate < 1) rate = 1;
return rate;
}
static uint32_t bbr_bw_to_pacing_rate(uint32_t bw, int gain, uint32_t mss)
{
uint64_t rate = bbr_rate_bytes_per_sec(bw, gain, 1, mss);
return (uint32_t)rate;
}
static uint32_t bbr_bdp(const struct bbr* bbr, uint32_t bw, int gain, uint32_t mss)
{
uint64_t w;
if (bbr->min_rtt_us == ~0U)
return (uint32_t)bbr->init_cwnd * mss;
w = (uint64_t)bw * bbr->min_rtt_us;
return (uint32_t)((((w * (uint64_t)gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT) * mss;
}
static uint32_t bbr_quantization_budget(struct bbr* bbr, uint32_t cwnd, uint32_t mss)
{
cwnd = (uint32_t)(cwnd > BBR_CWND_MIN_TARGET * mss ? cwnd : BBR_CWND_MIN_TARGET * mss);
if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP)
cwnd += 2;
return cwnd;
}
static uint32_t bbr_inflight(const struct bbr* bbr, uint32_t bw, int gain, uint32_t mss)
{
uint32_t inflight = bbr_bdp(bbr, bw, gain, mss);
return bbr_quantization_budget((struct bbr*)bbr, inflight, mss);
}
static uint32_t bbr_ack_aggregation_cwnd(const struct bbr* bbr, uint32_t mss)
{
uint32_t max_aggr_cwnd, aggr_cwnd = 0;
(void)mss;
max_aggr_cwnd = (uint32_t)((uint64_t)bbr_bw(bbr) * 100000U / BW_UNIT);
aggr_cwnd = (uint32_t)((uint32_t)BBR_EXTRA_ACKED_GAIN * bbr_extra_acked(bbr) >> BBR_SCALE);
aggr_cwnd = (uint32_t)(aggr_cwnd < max_aggr_cwnd ? aggr_cwnd : max_aggr_cwnd);
return aggr_cwnd;
}
static uint32_t bbr_target_inflight(const struct bbr* bbr, uint32_t cwnd, uint32_t mss)
{
uint32_t bdp = bbr_inflight(bbr, bbr_bw(bbr), BBR_UNIT, mss);
return (uint32_t)(bdp < cwnd ? bdp : cwnd);
}
static int bbr_is_probing_bandwidth(const struct bbr* bbr)
{
return (bbr->mode == BBR_STARTUP) ||
(bbr->mode == BBR_PROBE_BW &&
(bbr->cycle_idx == BBR_BW_PROBE_REFILL ||
bbr->cycle_idx == BBR_BW_PROBE_UP));
}
static int bbr_has_elapsed_in_phase(const struct bbr* bbr, uint32_t interval_us)
{
uint64_t now_tb = bbr_now(bbr);
uint64_t expire_tb = bbr->cycle_mstamp_tb + (uint64_t)(interval_us / 100U);
return (int32_t)((uint32_t)expire_tb - (uint32_t)now_tb) <= 0;
}
static uint32_t bbr_update_round_start(struct bbr* bbr, const struct bbr_rate_sample* rs)
{
uint32_t round_delivered = 0;
bbr->round_start = 0;
if (rs->interval_us > 0 && !((int32_t)(rs->prior_delivered - bbr->next_rtt_delivered) < 0)) {
round_delivered = bbr->delivered - bbr->next_rtt_delivered;
bbr->next_rtt_delivered = bbr->delivered;
bbr->round_start = 1;
}
return round_delivered;
}
static uint32_t bbr_calculate_bw_sample(const struct bbr_rate_sample* rs)
{
if (rs->interval_us > 0)
return (uint32_t)(((uint64_t)rs->delivered * BW_UNIT + rs->interval_us - 1) / rs->interval_us);
return 0;
}
static void bbr_update_latest_delivery_signals(struct bbr* bbr,
const struct bbr_rate_sample* rs, uint32_t sample_bw)
{
bbr->loss_round_start = 0;
if (rs->interval_us <= 0 || !rs->acked_sacked)
return;
bbr->bw_latest = (uint32_t)(bbr->bw_latest > sample_bw ? bbr->bw_latest : sample_bw);
bbr->inflight_latest = (uint32_t)(bbr->inflight_latest > (uint32_t)rs->delivered ?
bbr->inflight_latest : (uint32_t)rs->delivered);
if (!((int32_t)(rs->prior_delivered - bbr->loss_round_delivered) < 0)) {
bbr->loss_round_delivered = bbr->delivered;
bbr->loss_round_start = 1;
}
}
static void bbr_advance_latest_delivery_signals(struct bbr* bbr,
const struct bbr_rate_sample* rs, uint32_t sample_bw)
{
if (bbr->loss_round_start) {
bbr->bw_latest = sample_bw;
bbr->inflight_latest = (uint32_t)rs->delivered;
}
}
static void bbr_take_max_bw_sample(struct bbr* bbr, uint32_t bw)
{
bbr->bw_hi[1] = (uint32_t)(bw > bbr->bw_hi[1] ? bw : bbr->bw_hi[1]);
}
static void bbr_init_lower_bounds(struct bbr* bbr, int init_bw, uint32_t cwnd)
{
if (init_bw && bbr->bw_lo == ~0U)
bbr->bw_lo = bbr_max_bw(bbr);
if (bbr->inflight_lo == ~0U)
bbr->inflight_lo = cwnd;
if (bbr->inflight_hi == ~0U)
bbr->inflight_hi = (uint32_t)(cwnd > 4U * (uint32_t)bbr->init_cwnd ? cwnd : 4U * (uint32_t)bbr->init_cwnd);
}
static void bbr_loss_lower_bounds(struct bbr* bbr)
{
uint32_t loss_cut = BBR_UNIT - BBR_BETA;
bbr->bw_lo = (uint32_t)(bbr->bw_latest >
(uint32_t)((uint64_t)bbr->bw_lo * loss_cut >> BBR_SCALE) ?
bbr->bw_latest :
(uint32_t)((uint64_t)bbr->bw_lo * loss_cut >> BBR_SCALE));
bbr->inflight_lo = (uint32_t)(bbr->inflight_latest >
(uint32_t)((uint64_t)bbr->inflight_lo * loss_cut >> BBR_SCALE) ?
bbr->inflight_latest :
(uint32_t)((uint64_t)bbr->inflight_lo * loss_cut >> BBR_SCALE));
}
static void bbr_adapt_lower_bounds(struct bbr* bbr, uint32_t cwnd)
{
if (bbr_is_probing_bandwidth(bbr))
return;
if (bbr->loss_in_round) {
bbr_init_lower_bounds(bbr, 1, cwnd);
bbr_loss_lower_bounds(bbr);
}
bbr->bw_lo = (uint32_t)(bbr->bw_lo > 1U ? bbr->bw_lo : 1U);
}
static void bbr_reset_lower_bounds(struct bbr* bbr)
{
bbr->bw_lo = ~0U;
bbr->inflight_lo = ~0U;
}
static void bbr_reset_congestion_signals(struct bbr* bbr)
{
bbr->loss_in_round = 0;
bbr->loss_in_cycle = 0;
bbr->bw_latest = 0;
bbr->inflight_latest = 0;
}
static int bbr_is_inflight_too_high(const struct bbr* bbr, const struct bbr_rate_sample* rs)
{
uint32_t loss_thresh;
(void)bbr;
if (rs->lost > 0 && rs->tx_in_flight) {
loss_thresh = (uint32_t)((uint64_t)rs->tx_in_flight * BBR_LOSS_THRESH >> BBR_SCALE);
if ((uint32_t)rs->lost > loss_thresh)
return 1;
}
return 0;
}
static void bbr_handle_queue_too_high_in_startup(struct bbr* bbr, uint32_t mss)
{
uint32_t bdp;
(void)mss;
bbr->full_bw_reached = 1;
bdp = bbr_inflight(bbr, bbr_max_bw(bbr), BBR_UNIT, mss);
bbr->inflight_hi = (uint32_t)(bdp > bbr->inflight_latest ? bdp : bbr->inflight_latest);
}
static void bbr_check_loss_too_high_in_startup(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t mss)
{
if (bbr->full_bw_reached)
return;
if (rs->lost > 0 && bbr->loss_events_in_round < 0xf)
bbr->loss_events_in_round++;
if (bbr->loss_round_start &&
bbr->loss_events_in_round >= BBR_FULL_LOSS_CNT &&
bbr_is_inflight_too_high(bbr, rs)) {
bbr_handle_queue_too_high_in_startup(bbr, mss);
return;
}
if (bbr->loss_round_start)
bbr->loss_events_in_round = 0;
}
static void bbr_reset_full_bw(struct bbr* bbr)
{
bbr->full_bw = 0;
bbr->full_bw_cnt = 0;
bbr->full_bw_now = 0;
}
static void bbr_advance_max_bw_filter(struct bbr* bbr)
{
if (!bbr->bw_hi[1])
return;
bbr->bw_hi[0] = bbr->bw_hi[1];
bbr->bw_hi[1] = 0;
}
static void bbr_check_full_bw_reached(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t sample_bw)
{
uint32_t bw_thresh;
if (bbr->full_bw_now || rs->is_app_limited)
return;
bw_thresh = (uint32_t)((uint64_t)bbr->full_bw * BBR_FULL_BW_THRESH >> BBR_SCALE);
if (sample_bw >= bw_thresh) {
bbr_reset_full_bw(bbr);
bbr->full_bw = sample_bw;
return;
}
if (!bbr->round_start)
return;
bbr->full_bw_cnt = (uint8_t)(bbr->full_bw_cnt + 1);
bbr->full_bw_now = (uint8_t)(bbr->full_bw_cnt >= BBR_FULL_BW_CNT ? 1 : 0);
bbr->full_bw_reached |= bbr->full_bw_now;
}
static void bbr_raise_inflight_hi_slope(struct bbr* bbr, uint32_t cwnd)
{
uint32_t growth_this_round = 1U << bbr->bw_probe_up_rounds;
uint32_t cnt;
bbr->bw_probe_up_rounds = (uint8_t)(bbr->bw_probe_up_rounds + 1 < 30 ?
bbr->bw_probe_up_rounds + 1 : 30);
cnt = cwnd / growth_this_round;
cnt = (uint32_t)(cnt > 1U ? cnt : 1U);
bbr->bw_probe_up_cnt = cnt;
}
static void bbr_set_cycle_idx(struct bbr* bbr, int cycle_idx)
{
bbr->cycle_idx = (uint8_t)cycle_idx;
bbr->try_fast_path = 0;
}
static void bbr_start_bw_probe_refill(struct bbr* bbr, uint32_t bw_probe_up_rounds)
{
bbr_reset_lower_bounds(bbr);
bbr->bw_probe_up_rounds = (uint8_t)bw_probe_up_rounds;
bbr->bw_probe_up_acks = 0;
bbr->stopped_risky_probe = 0;
bbr->ack_phase = BBR_ACKS_REFILLING;
bbr->next_rtt_delivered = bbr->delivered;
bbr_set_cycle_idx(bbr, BBR_BW_PROBE_REFILL);
}
static void bbr_start_bw_probe_up(struct bbr* bbr, uint32_t sample_bw, uint32_t cwnd)
{
bbr->ack_phase = BBR_ACKS_PROBE_STARTING;
bbr->next_rtt_delivered = bbr->delivered;
bbr->cycle_mstamp_tb = bbr_now(bbr);
bbr_reset_full_bw(bbr);
bbr->full_bw = sample_bw;
bbr_set_cycle_idx(bbr, BBR_BW_PROBE_UP);
bbr_raise_inflight_hi_slope(bbr, cwnd);
}
static void bbr_save_cwnd(struct bbr* bbr, uint32_t cwnd)
{
if (bbr->mode != BBR_PROBE_RTT)
bbr->prior_cwnd = cwnd;
else
bbr->prior_cwnd = (uint32_t)(bbr->prior_cwnd > cwnd ? bbr->prior_cwnd : cwnd);
}
static void bbr_exit_probe_rtt(struct bbr* bbr)
{
bbr_reset_lower_bounds(bbr);
if (bbr->full_bw_reached) {
bbr->mode = BBR_PROBE_BW;
bbr_start_bw_probe_down(bbr);
bbr_start_bw_probe_cruise(bbr);
} else {
bbr->mode = BBR_STARTUP;
}
}
static uint32_t bbr_probe_rtt_cwnd(const struct bbr* bbr, uint32_t mss)
{
(void)bbr;
return BBR_CWND_MIN_TARGET * mss;
}
static uint32_t bbr_inflight_with_headroom(const struct bbr* bbr)
{
uint32_t headroom;
if (bbr->inflight_hi == ~0U)
return ~0U;
headroom = (uint32_t)((uint64_t)bbr->inflight_hi * BBR_INFLIGHT_HEADROOM >> BBR_SCALE);
headroom = (uint32_t)(headroom > 1U ? headroom : 1U);
return bbr->inflight_hi - headroom;
}
static void bbr_pick_probe_wait(struct bbr* bbr)
{
bbr->rounds_since_probe = (uint8_t)((uint32_t)rand() % BBR_BW_PROBE_RAND_ROUNDS);
bbr->probe_wait_us = BBR_BW_PROBE_BASE_US + (uint32_t)rand() % BBR_BW_PROBE_RAND_US;
}
static void bbr_start_bw_probe_down(struct bbr* bbr)
{
bbr_reset_congestion_signals(bbr);
bbr->bw_probe_up_cnt = ~0U;
bbr_pick_probe_wait(bbr);
bbr->cycle_mstamp_tb = bbr_now(bbr);
bbr->ack_phase = BBR_ACKS_PROBE_STOPPING;
bbr->next_rtt_delivered = bbr->delivered;
bbr_set_cycle_idx(bbr, BBR_BW_PROBE_DOWN);
}
static void bbr_start_bw_probe_cruise(struct bbr* bbr)
{
if (bbr->inflight_lo != ~0U)
bbr->inflight_lo = (uint32_t)(bbr->inflight_lo < bbr->inflight_hi ?
bbr->inflight_lo : bbr->inflight_hi);
bbr_set_cycle_idx(bbr, BBR_BW_PROBE_CRUISE);
}
static void bbr_handle_inflight_too_high(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t cwnd, uint32_t mss)
{
const uint32_t beta = BBR_BETA;
bbr->prev_probe_too_high = 1;
bbr->bw_probe_samples = 0;
if (!rs->is_app_limited) {
uint32_t ti = bbr_target_inflight(bbr, cwnd, mss);
bbr->inflight_hi = (uint32_t)(rs->tx_in_flight >
(uint32_t)((uint64_t)ti * (BBR_UNIT - beta) >> BBR_SCALE) ?
rs->tx_in_flight :
(uint32_t)((uint64_t)ti * (BBR_UNIT - beta) >> BBR_SCALE));
}
if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP)
bbr_start_bw_probe_down(bbr);
}
static void bbr_probe_inflight_hi_upward(struct bbr* bbr, const struct bbr_rate_sample* rs,
int is_cwnd_limited, uint32_t cwnd)
{
uint32_t delta;
if (!is_cwnd_limited || cwnd < bbr->inflight_hi)
return;
bbr->bw_probe_up_acks += rs->acked_sacked;
if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt &&
bbr->bw_probe_up_cnt > 0) {
delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt;
bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt;
bbr->inflight_hi += delta;
bbr->try_fast_path = 0;
}
if (bbr->round_start)
bbr_raise_inflight_hi_slope(bbr, cwnd);
}
static void bbr_adapt_upper_bounds(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t sample_bw, int is_cwnd_limited,
uint32_t cwnd, uint32_t mss, int* decided)
{
(void)sample_bw;
*decided = 0;
if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start)
bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK;
if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) {
bbr->bw_probe_samples = 0;
bbr->ack_phase = BBR_ACKS_INIT;
if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited)
bbr_advance_max_bw_filter(bbr);
if (bbr->mode == BBR_PROBE_BW && bbr->stopped_risky_probe && !bbr->prev_probe_too_high) {
bbr_start_bw_probe_refill(bbr, 0);
*decided = 1;
return;
}
}
if (bbr_is_inflight_too_high(bbr, rs)) {
if (bbr->bw_probe_samples)
bbr_handle_inflight_too_high(bbr, rs, cwnd, mss);
} else {
if (bbr->inflight_hi == ~0U)
return;
if (rs->tx_in_flight > bbr->inflight_hi)
bbr->inflight_hi = rs->tx_in_flight;
if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP)
bbr_probe_inflight_hi_upward(bbr, rs, is_cwnd_limited, cwnd);
}
}
static int bbr_is_reno_coexistence_probe_time(struct bbr* bbr, uint32_t cwnd, uint32_t mss)
{
uint32_t rounds = bbr_target_inflight(bbr, cwnd, mss);
if (rounds > BBR_BW_PROBE_MAX_ROUNDS)
rounds = BBR_BW_PROBE_MAX_ROUNDS;
return bbr->rounds_since_probe >= rounds;
}
static int bbr_check_time_to_probe_bw(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t cwnd, uint32_t mss)
{
(void)rs;
if (bbr_has_elapsed_in_phase(bbr, bbr->probe_wait_us) ||
bbr_is_reno_coexistence_probe_time(bbr, cwnd, mss)) {
bbr_start_bw_probe_refill(bbr, 0);
return 1;
}
return 0;
}
static int bbr_check_time_to_cruise(const struct bbr* bbr, uint32_t inflight,
uint32_t bw, uint32_t mss)
{
if (inflight > bbr_inflight_with_headroom(bbr))
return 0;
return inflight <= bbr_inflight(bbr, bw, BBR_UNIT, mss);
}
static void bbr_update_cycle_phase(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t sample_bw, uint32_t mss,
uint32_t inflight_packets, int is_cwnd_limited,
uint32_t cwnd)
{
int decided = 0;
int is_bw_probe_done;
uint32_t inflight, bw;
if (!bbr->full_bw_reached)
return;
bbr_adapt_upper_bounds(bbr, rs, sample_bw, is_cwnd_limited, cwnd, mss, &decided);
if (decided)
return;
if (bbr->mode != BBR_PROBE_BW)
return;
inflight = inflight_packets;
bw = bbr_max_bw(bbr);
switch (bbr->cycle_idx) {
case BBR_BW_PROBE_CRUISE:
if (bbr_check_time_to_probe_bw(bbr, rs, cwnd, mss))
return;
break;
case BBR_BW_PROBE_REFILL:
if (bbr->round_start) {
bbr->bw_probe_samples = 1;
bbr_start_bw_probe_up(bbr, sample_bw, cwnd);
}
break;
case BBR_BW_PROBE_UP:
is_bw_probe_done = 0;
if (bbr->prev_probe_too_high && inflight >= bbr->inflight_hi) {
bbr->stopped_risky_probe = 1;
is_bw_probe_done = 1;
} else {
if (is_cwnd_limited && cwnd >= bbr->inflight_hi) {
bbr_reset_full_bw(bbr);
bbr->full_bw = sample_bw;
} else if (bbr->full_bw_now) {
is_bw_probe_done = 1;
}
}
if (is_bw_probe_done) {
bbr->prev_probe_too_high = 0;
bbr_start_bw_probe_down(bbr);
}
break;
case BBR_BW_PROBE_DOWN:
if (bbr_check_time_to_probe_bw(bbr, rs, cwnd, mss))
return;
if (bbr_check_time_to_cruise(bbr, inflight, bw, mss))
bbr_start_bw_probe_cruise(bbr);
break;
default:
break;
}
}
static void bbr_check_drain(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t sample_bw, uint32_t mss, uint32_t inflight_packets)
{
(void)rs; (void)sample_bw;
if (bbr->mode == BBR_STARTUP && bbr->full_bw_reached) {
bbr->mode = BBR_DRAIN;
bbr_reset_congestion_signals(bbr);
}
if (bbr->mode == BBR_DRAIN &&
inflight_packets <= bbr_inflight(bbr, bbr_max_bw(bbr), BBR_UNIT, mss)) {
bbr->mode = BBR_PROBE_BW;
bbr_start_bw_probe_down(bbr);
}
}
static void bbr_check_probe_rtt_done(struct bbr* bbr, uint32_t* cwnd)
{
if (!bbr->probe_rtt_done_stamp_tb)
return;
if (!((int32_t)((uint32_t)bbr_now(bbr) - (uint32_t)bbr->probe_rtt_done_stamp_tb) >= 0))
return;
bbr->probe_rtt_min_stamp_tb = bbr_now(bbr);
*cwnd = (uint32_t)(*cwnd > bbr->prior_cwnd ? *cwnd : bbr->prior_cwnd);
bbr_exit_probe_rtt(bbr);
}
static void bbr_update_min_rtt(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t* cwnd, uint32_t mss, uint32_t inflight_packets)
{
uint32_t expire;
int probe_rtt_expired, min_rtt_expired;
(void)mss;
expire = (uint32_t)(bbr->probe_rtt_min_stamp_tb + BBR_PROBE_RTT_WIN_MS * 10U);
probe_rtt_expired = ((int32_t)(expire - (uint32_t)bbr_now(bbr)) < 0);
if (rs->rtt_us > 0 &&
(rs->rtt_us < bbr->probe_rtt_min_us || probe_rtt_expired)) {
bbr->probe_rtt_min_us = rs->rtt_us;
bbr->probe_rtt_min_stamp_tb = bbr_now(bbr);
bbr->has_seen_rtt = 1;
}
if (bbr->probe_rtt_min_us == 0)
bbr->probe_rtt_min_us = ~0U;
expire = (uint32_t)(bbr->min_rtt_stamp_tb + BBR_MIN_RTT_WIN_SEC * 10000U);
min_rtt_expired = ((int32_t)(expire - (uint32_t)bbr_now(bbr)) < 0);
if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || min_rtt_expired) {
bbr->min_rtt_us = bbr->probe_rtt_min_us;
bbr->min_rtt_stamp_tb = bbr_now(bbr);
}
if (BBR_PROBE_RTT_MS > 0 && probe_rtt_expired &&
!bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) {
bbr->mode = BBR_PROBE_RTT;
bbr_save_cwnd(bbr, *cwnd);
bbr->probe_rtt_done_stamp_tb = 0;
bbr->ack_phase = BBR_ACKS_PROBE_STOPPING;
bbr->next_rtt_delivered = bbr->delivered;
}
if (bbr->mode == BBR_PROBE_RTT) {
if (!bbr->probe_rtt_done_stamp_tb &&
inflight_packets <= bbr_probe_rtt_cwnd(bbr, mss)) {
bbr->probe_rtt_done_stamp_tb = (uint32_t)bbr_now(bbr) + BBR_PROBE_RTT_MS * 10U;
bbr->probe_rtt_round_done = 0;
bbr->next_rtt_delivered = bbr->delivered;
} else if (bbr->probe_rtt_done_stamp_tb) {
if (bbr->round_start)
bbr->probe_rtt_round_done = 1;
if (bbr->probe_rtt_round_done)
bbr_check_probe_rtt_done(bbr, cwnd);
}
}
if (rs->delivered > 0)
bbr->idle_restart = 0;
}
static void bbr_reset_startup_mode(struct bbr* bbr)
{
bbr->mode = BBR_STARTUP;
}
static void bbr_update_gains(struct bbr* bbr)
{
switch (bbr->mode) {
case BBR_STARTUP:
bbr->pacing_gain = BBR_STARTUP_PACING_GAIN;
bbr->cwnd_gain = BBR_STARTUP_CWND_GAIN;
break;
case BBR_DRAIN:
bbr->pacing_gain = BBR_DRAIN_GAIN;
bbr->cwnd_gain = BBR_STARTUP_CWND_GAIN;
break;
case BBR_PROBE_BW:
bbr->pacing_gain = bbr_pacing_gain[bbr->cycle_idx];
bbr->cwnd_gain = BBR_CWND_GAIN;
break;
case BBR_PROBE_RTT:
bbr->pacing_gain = BBR_UNIT;
bbr->cwnd_gain = BBR_UNIT;
break;
default:
break;
}
}
static void bbr_update_congestion_signals(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t sample_bw, uint32_t cwnd)
{
uint32_t bw;
if (rs->interval_us <= 0 || !rs->acked_sacked)
return;
bw = sample_bw;
if (!rs->is_app_limited || bw >= bbr_max_bw(bbr))
bbr_take_max_bw_sample(bbr, bw);
bbr->loss_in_round |= (rs->lost > 0);
if (!bbr->loss_round_start)
return;
bbr_adapt_lower_bounds(bbr, cwnd);
bbr->loss_in_round = 0;
}
static void bbr_update_ack_aggregation(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t cwnd, uint32_t mss)
{
uint32_t epoch_us, expected_acked, extra_acked;
uint32_t extra_acked_win_rtts_thresh = BBR_EXTRA_ACKED_WIN_RTTS;
uint64_t now_us;
(void)mss;
if (rs->acked_sacked <= 0 || rs->interval_us <= 0)
return;
if (bbr->round_start) {
bbr->extra_acked_win_rtts = (uint8_t)(bbr->extra_acked_win_rtts + 1 < 0x1F ?
bbr->extra_acked_win_rtts + 1 : 0x1F);
if (!bbr->full_bw_reached)
extra_acked_win_rtts_thresh = 1;
if (bbr->extra_acked_win_rtts >= extra_acked_win_rtts_thresh) {
bbr->extra_acked_win_rtts = 0;
bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? 0 : 1;
bbr->extra_acked[bbr->extra_acked_win_idx] = 0;
}
}
now_us = bbr_now(bbr) * 100ULL;
if (now_us > bbr->ack_epoch_mstamp_tb * 100ULL)
epoch_us = (uint32_t)(now_us - bbr->ack_epoch_mstamp_tb * 100ULL);
else
epoch_us = 0;
expected_acked = (uint32_t)((uint64_t)bbr_bw(bbr) * epoch_us / BW_UNIT);
if (bbr->ack_epoch_acked <= expected_acked ||
(bbr->ack_epoch_acked + rs->acked_sacked >= BBR_ACK_EPOCH_ACKED_RESET_THRESH)) {
bbr->ack_epoch_acked = 0;
bbr->ack_epoch_mstamp_tb = bbr_now(bbr);
expected_acked = 0;
}
bbr->ack_epoch_acked = (uint32_t)(bbr->ack_epoch_acked + rs->acked_sacked < 0xFFFFF ?
bbr->ack_epoch_acked + rs->acked_sacked : 0xFFFFF);
extra_acked = bbr->ack_epoch_acked - expected_acked;
extra_acked = (uint32_t)(extra_acked < cwnd ? extra_acked : cwnd);
if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx])
bbr->extra_acked[bbr->extra_acked_win_idx] = (uint16_t)extra_acked;
}
static void bbr_init_pacing_rate_from_rtt(struct bbr* bbr, uint32_t cwnd,
uint32_t mss, uint32_t* pacing_rate_out)
{
uint64_t bw;
uint32_t rtt_us;
if (bbr->has_seen_rtt && bbr->min_rtt_us != ~0U) {
rtt_us = bbr->min_rtt_us;
if (rtt_us < 1) rtt_us = 1;
} else {
rtt_us = 1000;
}
bw = (uint64_t)cwnd * BW_UNIT / rtt_us;
*pacing_rate_out = (uint32_t)bbr_rate_bytes_per_sec(bw, BBR_STARTUP_PACING_GAIN, 1, mss);
}
static void bbr_set_pacing_rate(struct bbr* bbr, uint32_t bw, int gain,
uint32_t mss, uint32_t* pacing_rate_out)
{
uint32_t rate = bbr_bw_to_pacing_rate(bw, gain, mss);
if (!bbr->has_seen_rtt && bbr->min_rtt_us != ~0U)
bbr_init_pacing_rate_from_rtt(bbr, *pacing_rate_out > 0 ? *pacing_rate_out : 4, mss, pacing_rate_out);
if (bbr->full_bw_reached || rate > *pacing_rate_out)
*pacing_rate_out = rate;
}
static void bbr_set_cwnd(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t acked, uint32_t bw, int gain,
uint32_t mss, uint32_t* cwnd)
{
uint32_t target_cwnd = 0;
uint32_t local_cwnd = *cwnd;
(void)rs;
if (!acked)
goto done;
target_cwnd = bbr_bdp(bbr, bw, gain, mss);
target_cwnd += bbr_ack_aggregation_cwnd(bbr, mss);
target_cwnd = bbr_quantization_budget(bbr, target_cwnd, mss);
bbr->try_fast_path = 0;
if (bbr->full_bw_reached) {
local_cwnd += acked;
if (local_cwnd >= target_cwnd) {
local_cwnd = target_cwnd;
bbr->try_fast_path = 1;
}
} else if (local_cwnd < target_cwnd || local_cwnd < 2 * (uint32_t)bbr->init_cwnd * mss) {
local_cwnd += acked;
} else {
bbr->try_fast_path = 1;
}
local_cwnd = (uint32_t)(local_cwnd > BBR_CWND_MIN_TARGET * mss ?
local_cwnd : BBR_CWND_MIN_TARGET * mss);
done:
*cwnd = local_cwnd;
if (bbr->mode == BBR_PROBE_RTT) {
uint32_t pr_cwnd = bbr_probe_rtt_cwnd(bbr, mss);
*cwnd = (uint32_t)(*cwnd < pr_cwnd ? *cwnd : pr_cwnd);
}
}
static void bbr_bound_cwnd_for_inflight_model(struct bbr* bbr, uint32_t* cwnd, uint32_t mss)
{
uint32_t cap = ~0U;
if (!bbr->initialized)
return;
if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx != BBR_BW_PROBE_CRUISE)
cap = bbr->inflight_hi;
else if (bbr->mode == BBR_PROBE_RTT ||
(bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_CRUISE))
cap = bbr_inflight_with_headroom(bbr);
cap = (uint32_t)(cap > BBR_CWND_MIN_TARGET * mss ? cap : BBR_CWND_MIN_TARGET * mss);
cap = (uint32_t)(cap < bbr->inflight_lo ? cap : bbr->inflight_lo);
*cwnd = (uint32_t)(cap < *cwnd ? cap : *cwnd);
}
static int bbr_run_fast_path(struct bbr* bbr, int* update_model,
const struct bbr_rate_sample* rs, uint32_t sample_bw,
uint32_t mss, uint32_t inflight_packets,
int is_cwnd_limited, uint32_t* cwnd)
{
uint32_t prev_min_rtt_us;
uint8_t prev_mode;
if (bbr->try_fast_path && rs->is_app_limited &&
sample_bw < bbr_max_bw(bbr) && !bbr->loss_in_round) {
prev_mode = bbr->mode;
prev_min_rtt_us = bbr->min_rtt_us;
bbr_check_drain(bbr, rs, sample_bw, mss, inflight_packets);
bbr_update_cycle_phase(bbr, rs, sample_bw, mss, inflight_packets,
is_cwnd_limited, *cwnd);
bbr_update_min_rtt(bbr, rs, cwnd, mss, inflight_packets);
if (bbr->mode == prev_mode &&
bbr->min_rtt_us == prev_min_rtt_us &&
bbr->try_fast_path)
return 1;
*update_model = 0;
}
return 0;
}
void bbr_init(struct bbr* bbr)
{
bbr->initialized = 1;
bbr->init_cwnd = 0;
bbr->prior_cwnd = 0;
bbr->delivered = 0;
bbr->next_rtt_delivered = 0;
bbr->probe_rtt_done_stamp_tb = 0;
bbr->probe_rtt_round_done = 0;
bbr->probe_rtt_min_us = ~0U;
bbr->probe_rtt_min_stamp_tb = bbr_now(bbr);
bbr->min_rtt_us = ~0U;
bbr->min_rtt_stamp_tb = bbr_now(bbr);
bbr->has_seen_rtt = 0;
bbr->round_start = 0;
bbr->idle_restart = 0;
bbr->full_bw_reached = 0;
bbr->full_bw = 0;
bbr->full_bw_cnt = 0;
bbr->full_bw_now = 0;
bbr->cycle_mstamp_tb = 0;
bbr->cycle_idx = 0;
bbr_reset_startup_mode(bbr);
bbr->ack_epoch_mstamp_tb = bbr_now(bbr);
bbr->ack_epoch_acked = 0;
bbr->extra_acked_win_rtts = 0;
bbr->extra_acked_win_idx = 0;
bbr->extra_acked[0] = 0;
bbr->extra_acked[1] = 0;
bbr->try_fast_path = 0;
bbr->loss_round_delivered = 1;
bbr->loss_round_start = 0;
bbr->loss_events_in_round = 0;
bbr_reset_congestion_signals(bbr);
bbr->bw_lo = ~0U;
bbr->bw_hi[0] = 0;
bbr->bw_hi[1] = 0;
bbr->inflight_lo = ~0U;
bbr->inflight_hi = ~0U;
bbr_reset_full_bw(bbr);
bbr->bw_probe_up_cnt = ~0U;
bbr->bw_probe_up_acks = 0;
bbr->bw_probe_up_rounds = 0;
bbr->probe_wait_us = 0;
bbr->stopped_risky_probe = 0;
bbr->ack_phase = BBR_ACKS_INIT;
bbr->rounds_since_probe = 0;
bbr->bw_probe_samples = 0;
bbr->prev_probe_too_high = 0;
}
void bbr_main(struct bbr* bbr, const struct bbr_rate_sample* rs,
uint32_t* cwnd_out, uint32_t* pacing_rate_out,
uint32_t mss, uint32_t inflight_bytes, int is_cwnd_limited)
{
uint32_t cwnd = *cwnd_out;
uint32_t sample_bw, bw;
int update_model = 1;
bbr->delivered += rs->delivered;
bbr_update_round_start(bbr, rs);
if (bbr->round_start)
bbr->rounds_since_probe = (uint8_t)(bbr->rounds_since_probe + 1 < 0xFF ?
bbr->rounds_since_probe + 1 : 0xFF);
sample_bw = bbr_calculate_bw_sample(rs);
bbr_update_latest_delivery_signals(bbr, rs, sample_bw);
if (bbr_run_fast_path(bbr, &update_model, rs, sample_bw, mss,
inflight_bytes, is_cwnd_limited, &cwnd))
goto out;
if (update_model) {
bbr_update_congestion_signals(bbr, rs, sample_bw, cwnd);
bbr_update_ack_aggregation(bbr, rs, cwnd, mss);
bbr_check_loss_too_high_in_startup(bbr, rs, mss);
bbr_check_full_bw_reached(bbr, rs, sample_bw);
bbr_check_drain(bbr, rs, sample_bw, mss, inflight_bytes);
bbr_update_cycle_phase(bbr, rs, sample_bw, mss, inflight_bytes,
is_cwnd_limited, cwnd);
bbr_update_min_rtt(bbr, rs, &cwnd, mss, inflight_bytes);
}
bbr_update_gains(bbr);
bw = bbr_bw(bbr);
bbr_set_pacing_rate(bbr, bw, bbr->pacing_gain, mss, pacing_rate_out);
bbr_set_cwnd(bbr, rs, rs->acked_sacked, bw, bbr->cwnd_gain, mss, &cwnd);
bbr_bound_cwnd_for_inflight_model(bbr, &cwnd, mss);
out:
bbr_advance_latest_delivery_signals(bbr, rs, sample_bw);
bbr->loss_in_cycle |= (rs->lost > 0);
*cwnd_out = cwnd;
}
void bbr_note_loss(struct bbr* bbr)
{
if (!bbr->loss_in_round)
bbr->loss_round_delivered = bbr->delivered;
bbr->loss_in_round = 1;
bbr->loss_in_cycle = 1;
}
void bbr_tx_start(struct bbr* bbr)
{
bbr->idle_restart = 1;
bbr->ack_epoch_mstamp_tb = bbr_now(bbr);
bbr->ack_epoch_acked = 0;
}