jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

etcp_router: fix inflight underflow by splitting rx_acked into tx_acked + last_sent_ack_seq

Browse Source 
rx_acked was used for two conflicting purposes:
1. remote ack of our sends (inflight = tx_seq - rx_acked)
2. our last sent ACK seq (dedup: rx_seq != rx_acked)

When the ack timer fired and set rx_acked = rx_seq, it overwrote
the inflight-tracking value. If rx_seq > tx_seq, the computation
tx_seq - rx_acked underflowed (e.g. 18 - 24 = 0xFFFFFFFA),
permanently blocking router_drain_send_q and causing send_q to
grow indefinitely.

Fix:
- Split rx_acked into tx_acked (remote ack, for inflight) and
  last_sent_ack_seq (our ACK, for dedup)
- Incoming ACK handler only advances tx_acked forward (stale guard)
- Use int32_t cast on all inflight comparisons to handle stale states
- Add etcp_router architecture diagram (doc/etcp_router_arch.md)
This commit is contained in:
Evgeny committed 2026-06-02 22:38:20 +03:00
1 parent 3a2b427287
commit 7f5325ac24
4 files changed
+353 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/etcp_router.c
+20 -15
View File
@@ -78,9 +78,9 @@ static int router_send_one_flags(struct ETCP_ROUTER_CONN* rconn, const uint8_t*
queue_entry_free(entry); queue_dgram_free(entry); if (!seq_flags) rconn->tx_seq--; return -1;
}
rconn->last_dgram_ts = get_current_timestamp();
DEBUG_TRACE(DEBUG_CATEGORY_ETCPROUTE, "router_send: seq=%u → %016llx svc_id=%u len=%zu inflight=%u",
DEBUG_TRACE(DEBUG_CATEGORY_ETCPROUTE, "router_send: seq=%u → %016llx svc_id=%u len=%zu inflight=%d",
seq, (unsigned long long)rconn->remote_node_id, rconn->svc_id, pl_len,
rconn->tx_seq - rconn->rx_acked);
(int32_t)(rconn->tx_seq - rconn->tx_acked));
return etcp_send(conn, entry);
}
@@ -150,7 +150,7 @@ static void router_close_and_notify(struct ETCP_ROUTER_CONN* rconn) {
static void router_drain_send_q(struct ETCP_ROUTER_CONN* rconn) {
while (1) {
if (rconn->tx_seq - rconn->rx_acked >= ROUTER_MAX_INFLIGHT) break;
if ((int32_t)(rconn->tx_seq - rconn->tx_acked) >= ROUTER_MAX_INFLIGHT) break;
struct ll_entry* e = queue_data_get(rconn->send_q);
if (!e) { rconn->send_blocked = 0; break; }
router_send_one(rconn, e->dgram, e->len);
@@ -178,8 +178,8 @@ static void router_enqueue_send(struct ETCP_ROUTER_CONN* rconn, const uint8_t* p
if (!qe->dgram) { queue_entry_free(qe); return; }
qe->len = pl_len;
if (pl_len > 0) memcpy(qe->dgram, payload, pl_len);
DEBUG_DEBUG(DEBUG_CATEGORY_ETCPROUTE, "router_send_q: queued svc_id=%u inflight=%u send_q=%d",
rconn->svc_id, rconn->tx_seq - rconn->rx_acked, queue_entry_count(rconn->send_q));
DEBUG_DEBUG(DEBUG_CATEGORY_ETCPROUTE, "router_send_q: queued svc_id=%u inflight=%d send_q=%d",
rconn->svc_id, (int32_t)(rconn->tx_seq - rconn->tx_acked), queue_entry_count(rconn->send_q));
queue_data_put(rconn->send_q, qe);
rconn->send_blocked = 1;
if (!rconn->send_resume_timer)
@@ -230,11 +230,11 @@ static void router_send_ack(struct ETCP_ROUTER_CONN* rconn) {
static void router_ack_timer_cb(void* arg) {
struct ETCP_ROUTER_CONN* rconn = (struct ETCP_ROUTER_CONN*)arg;
rconn->ack_timer = NULL;
if (rconn->rx_seq != rconn->rx_acked) {
if (rconn->rx_seq != rconn->last_sent_ack_seq) {
router_send_ack(rconn);
rconn->rx_acked = rconn->rx_seq;
rconn->last_sent_ack_seq = rconn->rx_seq;
}
if (rconn->rx_seq != rconn->rx_acked)
if (rconn->rx_seq != rconn->last_sent_ack_seq)
rconn->ack_timer = uasync_set_timeout(rconn->inst->ua,
ROUTER_ACK_INTERVAL_TB, rconn, router_ack_timer_cb, "router_ack");
else
@@ -245,9 +245,9 @@ static void router_ack_timer_cb(void* arg) {
static void router_idle_ack_timer_cb(void* arg) {
struct ETCP_ROUTER_CONN* rconn = (struct ETCP_ROUTER_CONN*)arg;
rconn->idle_ack_timer = NULL;
if (rconn->rx_seq != rconn->rx_acked) {
if (rconn->rx_seq != rconn->last_sent_ack_seq) {
router_send_ack(rconn);
rconn->rx_acked = rconn->rx_seq;
rconn->last_sent_ack_seq = rconn->rx_seq;
}
}
@@ -320,9 +320,13 @@ static void etcp_router_recv_cb(struct ETCP_CONN* conn, struct ll_entry* entry)
}
if (pl_len == 0) {
// ACK-пакет: обновляем rx_acked, drain send_q, выходим
if (rconn) {
rconn->rx_acked = hdr->seq;
if ((int32_t)(hdr->seq - rconn->tx_acked) >= 0) {
rconn->tx_acked = hdr->seq;
} else {
DEBUG_WARN(DEBUG_CATEGORY_ETCPROUTE, "router: stale ACK seq=%u tx_acked=%u from %016llx, ignoring",
hdr->seq, rconn->tx_acked, (unsigned long long)hdr->src_node_id);
}
rconn->last_dgram_ts = get_current_timestamp();
if (rconn->send_blocked) router_drain_send_q(rconn);
}
@@ -497,7 +501,7 @@ int etcp_route_send(struct UTUN_INSTANCE* inst, uint64_t dst_node_id, struct ll_
struct ETCP_ROUTER_CONN* rconn = etcp_router_conn_get(inst, dst_node_id, svc_id);
if (!rconn) { queue_dgram_free(entry); queue_entry_free(entry); return -1; }
if (rconn->tx_seq - rconn->rx_acked >= ROUTER_MAX_INFLIGHT) {
if ((int32_t)(rconn->tx_seq - rconn->tx_acked) >= ROUTER_MAX_INFLIGHT) {
router_enqueue_send(rconn, entry->dgram + 1, payload_len);
queue_dgram_free(entry); queue_entry_free(entry);
return 0;
@@ -550,7 +554,8 @@ struct ETCP_ROUTER_CONN* etcp_router_conn_get(struct UTUN_INSTANCE* inst,
rconn->last_dgram_ts = get_current_timestamp();
rconn->tx_seq = 0;
rconn->rx_seq = 0;
rconn->rx_acked = 0;
rconn->tx_acked = 0;
rconn->last_sent_ack_seq = 0;
rconn->inst = inst;
rconn->ack_timer = NULL;
rconn->idle_ack_timer = NULL;
@@ -573,7 +578,7 @@ int etcp_router_conn_send(struct ETCP_ROUTER_CONN* rconn,
(void*)rconn, (const void*)data, len);
return -1;
}
if (rconn->tx_seq - rconn->rx_acked >= ROUTER_MAX_INFLIGHT) {
if ((int32_t)(rconn->tx_seq - rconn->tx_acked) >= ROUTER_MAX_INFLIGHT) {
router_enqueue_send(rconn, data, len);
return 0;
}
src/etcp_router.h
+2 -1
View File
@@ -32,7 +32,8 @@ struct ETCP_ROUTER_CONN {
uint32_t tx_seq; // следующий seq для отправки
uint32_t rx_seq; // ожидаемый seq для сборки (next expected)
uint32_t rx_acked; // последний отправленный ACK (= rx_seq на момент отправки)
uint32_t tx_acked; // сколько наших пакетов подтвердил remote (для inflight)
uint32_t last_sent_ack_seq; // последний отправленный ACK (= rx_seq на момент отправки)
struct UTUN_INSTANCE* inst;
struct ll_queue* recv_q; // reorder очередь: hash по seq (4 байта, offset 0)