jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1754 Commits
15 Branches
2 Tags
165 MiB
 
 
 
 
 
 
Tree: 1c51df039f
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '1c51df039f'
${ noResults }
utun2/tests/test_etcp_router_faults.c

383 lines
20 KiB
Raw Blame History

/* Deterministic SVC_ROUTE fault injection. Real router/queues/timers, no UDP:
* lower ETCP must not conceal the router's own loss/reorder/restart bugs. */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "etcp.h"
#include "pkt_normalizer.h"
#include "etcp_router.h"
#include "utun_instance.h"
#include "../lib/debug_config.h"
#include "../lib/mem.h"
#define SVC 0x42
static uint8_t svc_base = SVC;
static int drop_wire;
#define COUNT 160
#define REQUIRE(x) do { if (!(x)) { fprintf(stderr, "FAIL %s:%d: %s\n", __func__, __LINE__, #x); return -1; } } while (0)
struct endpoint {
struct UTUN_INSTANCE inst;
struct ETCP_CONN conn, alternate;
struct ll_entry* route;
unsigned sent[2], received[2], notified[2];
};
static struct endpoint ep[2];
static struct UASYNC* ua;
static int failed, drop_data, drop_ack, drop_hello[2], drop_challenge[2], drop_confirm[2], drop_all_ack;
static int reorder, duplicated, dropped_data, dropped_ack, handshake_drops, closed_in_handler;
static unsigned max_wire;
static struct ll_entry *held, *saved_data, *saved_ack, *saved_hello, *saved_confirm;
static void release(struct ll_entry* e) { if (e) { queue_dgram_free(e); queue_entry_free(e); } }
static struct ll_entry* copy_packet(const struct ll_entry* e) {
struct ll_entry* copy = ll_alloc_lldgram(e->len);
if (!copy) abort();
memcpy(copy->dgram, e->dgram, e->len); copy->len = e->len;
return copy;
}
static struct ETCP_ROUTER_CONN* rc(int side, int svc) {
return etcp_router_conn_get(&ep[side].inst, 0, ep[1-side].inst.node_id, svc_base + svc);
}
static void receive(struct ETCP_CONN* conn, struct ll_entry* e) {
int side = conn->instance == &ep[0].inst ? 0 : 1;
unsigned svc = e->dgram[0] - svc_base;
uint64_t src, dst, group;
memcpy(&src, e->dgram + ROUTER_SVC_SRC_OFF, 8); memcpy(&dst, e->dgram + ROUTER_SVC_DST_OFF, 8);
memcpy(&group, e->dgram + ROUTER_SVC_GROUP_OFF, 8);
if (svc >= 2 || src != ep[1-side].inst.node_id || dst != ep[side].inst.node_id || group) {
fprintf(stderr, "FAIL delivery identity side=%d svc=%u\n", side, svc); failed = 1; release(e); return;
}
if (e->len == ROUTER_SVC_HDR_SIZE) { ep[side].notified[svc]++; release(e); return; }
uint32_t number = 0;
if (e->len < ROUTER_SVC_HDR_SIZE + 4) { failed = 1; release(e); return; }
memcpy(&number, e->dgram + ROUTER_SVC_PAYLOAD_OFF, 4);
if (number != ep[side].received[svc]) {
fprintf(stderr, "FAIL delivery sequence side=%d svc=%u got=%u expected=%u\n", side, svc, number, ep[side].received[svc]);
failed = 1;
}
size_t len = 4 + number % 181;
if (e->len != ROUTER_SVC_HDR_SIZE + len) failed = 1;
for (size_t i = 4; i < len; i++) if (e->dgram[ROUTER_SVC_PAYLOAD_OFF + i] != (uint8_t)(number ^ i ^ svc)) failed = 1;
ep[side].received[svc]++;
release(e);
if (closed_in_handler) { closed_in_handler = 0; etcp_router_conn_close(rc(side, svc)); }
}
static void inject(int side, struct ll_entry* e) {
ep[side].inst.api_bindings.callbacks[ETCP_RT_ID_SVC_ROUTE](&ep[side].conn, e);
}
static void wire(int from, struct ll_queue* q) {
if ((unsigned)q->count > max_wire) max_wire = q->count;
struct ll_entry* e = queue_data_get(q);
if (!e) return;
if (e->len < SVC_ROUTE_HDR_SIZE) { release(e); return; }
struct SVC_ROUTE_HDR* h = (struct SVC_ROUTE_HDR*)e->dgram;
if (e->dgram[e->len - 1] != 1) { failed = 1; release(e); return; } /* every source retry starts afresh */
if (drop_wire) { release(e); return; }
int data = e->len > SVC_ROUTE_HDR_SIZE + 9;
if (!data && h->flags == ROUTER_FLAG_START && !saved_hello && from == 0) saved_hello = copy_packet(e);
if (!data && h->flags == (ROUTER_FLAG_START | ROUTER_FLAG_RST) && !saved_confirm && from == 0) saved_confirm = copy_packet(e);
int* drop = NULL;
if (!data && h->flags == ROUTER_FLAG_START) drop = &drop_hello[from];
if (!data && h->flags == ROUTER_FLAG_RST) drop = &drop_challenge[from];
if (!data && h->flags == (ROUTER_FLAG_START | ROUTER_FLAG_RST)) drop = &drop_confirm[from];
if (drop && *drop) { --*drop; handshake_drops++; release(e); return; }
if (data && from == 0 && h->svc_id == svc_base) {
if (!saved_data) saved_data = copy_packet(e);
if (drop_data && h->seq == 2) { drop_data--; dropped_data++; release(e); return; }
if (reorder && h->seq == 3 && !held) { held = e; return; }
if (reorder && h->seq == 4 && held) {
struct ll_entry* dup = copy_packet(e);
inject(1, e); inject(1, dup); inject(1, held); held = NULL; reorder = 0; duplicated++;
return;
}
}
if (!data && h->flags == 0 && from == 1) {
if (!saved_ack) saved_ack = copy_packet(e);
if (drop_all_ack || drop_ack) { if (drop_ack) drop_ack--; dropped_ack++; release(e); return; }
}
inject(1-from, e);
}
static void step(void) {
uasync_poll(ua, 1);
for (int i = 0; i < 2; i++) { wire(i, ep[i].conn.send_input_q); wire(i, ep[i].alternate.send_input_q); }
}
static int setup(void) {
memset(ep, 0, sizeof(ep)); failed = 0; ua = uasync_create(); REQUIRE(ua);
drop_data = drop_ack = drop_all_ack = reorder = duplicated = dropped_data = dropped_ack = handshake_drops = closed_in_handler = 0;
memset(drop_hello, 0, sizeof(drop_hello)); memset(drop_challenge, 0, sizeof(drop_challenge)); memset(drop_confirm, 0, sizeof(drop_confirm));
drop_wire = 0;
max_wire = 0; held = saved_data = saved_ack = saved_hello = saved_confirm = NULL;
for (int i = 0; i < 2; i++) {
ep[i].inst.ua = ua; ep[i].inst.node_id = i + 1;
REQUIRE(etcp_router_init(&ep[i].inst) == 0);
REQUIRE(etcp_router_bind(&ep[i].inst, svc_base, receive) == 0);
REQUIRE(etcp_router_bind(&ep[i].inst, svc_base+1, receive) == 0);
ep[i].conn.instance = ep[i].alternate.instance = &ep[i].inst;
ep[i].conn.peer_node_id = ep[i].alternate.peer_node_id = 2-i;
ep[i].conn.send_input_q = queue_new(ua, 0, 0, 0, "wire");
ep[i].alternate.send_input_q = queue_new(ua, 0, 0, 0, "alternate_wire");
queue_set_waiter_defer(ep[i].conn.send_input_q, 1); queue_set_waiter_defer(ep[i].alternate.send_input_q, 1);
ep[i].inst.connections = queue_new(ua, 16, 0, 8, "connections");
ep[i].route = queue_entry_new(sizeof(struct conn_queue_entry));
struct conn_queue_entry* ce = (struct conn_queue_entry*)ep[i].route->data;
ce->peer_node_id = 2-i; ce->conn = &ep[i].conn;
queue_data_put_with_index(ep[i].inst.connections, ep[i].route);
}
return 0;
}
static void cleanup(void) {
for (int i = 0; i < 2; i++) etcp_router_destroy(&ep[i].inst);
uasync_poll(ua, 0);
for (int i = 0; i < 2; i++) {
struct ll_entry* e;
while ((e = queue_data_get(ep[i].conn.send_input_q))) release(e);
while ((e = queue_data_get(ep[i].alternate.send_input_q))) release(e);
queue_free(ep[i].conn.send_input_q); queue_free(ep[i].alternate.send_input_q);
release(queue_data_get(ep[i].inst.connections)); queue_free(ep[i].inst.connections);
}
release(held); release(saved_data); release(saved_ack); release(saved_hello); release(saved_confirm);
uasync_poll(ua, 0);
if (ua->timer_alloc_count != ua->timer_free_count) {
fprintf(stderr, "FAIL teardown: live timers=%llu\n", (unsigned long long)(ua->timer_alloc_count - ua->timer_free_count));
failed = 1;
}
uasync_destroy(ua, 0);
}
static int send_one(int side, int svc) {
uint32_t n = ep[side].sent[svc]; uint8_t data[185]; size_t len = 4 + n % 181;
memcpy(data, &n, 4);
for (size_t i = 4; i < len; i++) data[i] = n ^ i ^ svc;
if (etcp_router_conn_send(rc(side, svc), data, len, 0) != 0) return -1;
ep[side].sent[svc]++; return 0;
}
static int settled(int side, int svc) {
struct ETCP_ROUTER_CONN* c = rc(side, svc);
return c->tx_acked == c->tx_seq && !c->send_q->count && !c->inflight_q->count;
}
static int wait_settled(int side, int svc) {
uint64_t deadline = get_time_tb() + 40000;
while (!settled(side, svc) && !failed && get_time_tb() < deadline) step();
REQUIRE(!failed); REQUIRE(settled(side, svc));
REQUIRE(ep[1-side].received[svc] == ep[side].sent[svc]);
return 0;
}
static int transfer(void) {
uint64_t deadline = get_time_tb() + 40000;
int complete = 0, backpressure = 0;
while (!complete && !failed && get_time_tb() < deadline) {
complete = 1;
for (int side = 0; side < 2; side++) for (int svc = 0; svc < 2; svc++) {
while (ep[side].sent[svc] < COUNT) if (send_one(side, svc) != 0) { backpressure++; break; }
if (ep[side].sent[svc] < COUNT || !settled(side, svc)) complete = 0;
}
step();
}
REQUIRE(!failed); REQUIRE(complete); REQUIRE(backpressure);
for (int side = 0; side < 2; side++) for (int svc = 0; svc < 2; svc++) {
REQUIRE(ep[side].received[svc] == COUNT); REQUIRE(ep[side].notified[svc] == 0);
}
printf(" delivered=%u backpressure=%d data_drop=%d ack_drop=%d reorder=%d retrans=%u wire_max=%u\n",
4*COUNT, backpressure, dropped_data, dropped_ack, duplicated, rc(0,0)->c_retrans_done, max_wire);
return 0;
}
static int loss_reorder(void) {
REQUIRE(setup() == 0); drop_data = 1; drop_ack = 2; reorder = 1;
REQUIRE(transfer() == 0); REQUIRE(dropped_data == 1 && dropped_ack == 2 && duplicated == 1);
REQUIRE(rc(0,0)->c_retrans_done > 0); REQUIRE(max_wire < 20);
cleanup(); return 0;
}
static int handshake_loss(void) {
REQUIRE(setup() == 0);
for (int i = 0; i < 2; i++) drop_hello[i] = drop_challenge[i] = drop_confirm[i] = 1;
REQUIRE(transfer() == 0); REQUIRE(handshake_drops == 6);
cleanup(); return 0;
}
static int ack_bounds(void) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); REQUIRE(saved_ack);
struct ETCP_ROUTER_CONN* c = rc(0,0); uint64_t progress = c->last_ack_changed_tb; c->no_ack_count = 9;
uint32_t invalid[] = {0, 1, 2, 0x7fffffffU, 0x80000001U};
for (unsigned i = 0; i < sizeof(invalid)/sizeof(invalid[0]); i++) {
struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = invalid[i]; inject(0,e);
REQUIRE(c->tx_acked == 1 && c->no_ack_count == 9 && c->last_ack_changed_tb == progress);
}
REQUIRE(send_one(0,0) == 0); // queued, not transmitted until deferred waiter runs
struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = 2; inject(0,e);
REQUIRE(c->tx_acked == 1); REQUIRE(wait_settled(0,0) == 0);
cleanup(); return 0;
}
static int reinit_retains(void) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
REQUIRE(send_one(0,0) == 0); uasync_poll(ua,0);
struct ETCP_ROUTER_CONN* c = rc(0,0); REQUIRE(c->inflight_q->count == 1);
uint64_t epoch = c->reset_id;
etcp_router_pause_retrans_for_node(&ep[0].inst, ep[1].inst.node_id);
struct ll_entry* e; while ((e = queue_data_get(ep[0].conn.send_input_q))) release(e);
REQUIRE(c->inflight_q->count == 1 && c->retrans_timer && c->reset_id == epoch);
REQUIRE(wait_settled(0,0) == 0); REQUIRE(c->c_retrans_done > 0);
REQUIRE(ep[0].notified[0] == 0 && ep[1].notified[0] == 0);
cleanup(); return 0;
}
static int stale_session(void) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
uint64_t previous = rc(0,0)->reset_id;
etcp_router_conn_restart(&ep[0].inst, 0, ep[1].inst.node_id, SVC);
REQUIRE(rc(0,0)->reset_id != previous);
REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1);
uint64_t epoch = rc(1,0)->peer_reset_id;
inject(1,copy_packet(saved_data)); inject(0,copy_packet(saved_ack));
inject(1,copy_packet(saved_hello)); inject(1,copy_packet(saved_confirm));
struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->flags = ROUTER_FLAG_CLOSE; inject(0,e);
for (int i = 0; i < 50; i++) step();
REQUIRE(rc(1,0)->peer_reset_id == epoch && ep[1].received[0] == 2);
REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1);
REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
cleanup(); return 0;
}
static int wraparound(void) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
rc(0,0)->tx_seq = rc(0,0)->tx_sent = rc(0,0)->tx_acked = UINT32_MAX-2;
rc(1,0)->rx_seq = rc(1,0)->last_sent_ack_seq = UINT32_MAX-2;
for (int i = 0; i < 7; i++) REQUIRE(send_one(0,0) == 0);
REQUIRE(wait_settled(0,0) == 0); REQUIRE(rc(0,0)->tx_acked == 4);
cleanup(); return 0;
}
static int waiter_lifetime(void) {
for (int deferred = 0; deferred < 2; deferred++) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
struct ll_queue* old = ep[0].conn.send_input_q;
if (!deferred) queue_data_put(old, queue_entry_new(0));
REQUIRE(send_one(0,0) == 0);
struct ETCP_ROUTER_CONN* c = rc(0,0);
REQUIRE(c->send_waiter_q == old);
REQUIRE(deferred ? c->send_waiter.call_soon_id != NULL : c->send_waiter.internal != NULL);
((struct conn_queue_entry*)ep[0].route->data)->conn = &ep[0].alternate;
etcp_router_conn_close(c); uasync_poll(ua,0);
struct ll_entry* e; while ((e = queue_data_get(old))) release(e);
REQUIRE(old->waiter_head == NULL);
cleanup();
}
return 0;
}
static int close_from_handler(void) {
REQUIRE(setup() == 0); closed_in_handler = 1; REQUIRE(send_one(0,0) == 0);
uint64_t deadline = get_time_tb()+10000;
while (!ep[1].received[0] && get_time_tb()<deadline) step();
REQUIRE(ep[1].received[0] == 1); REQUIRE(!failed);
cleanup(); return 0;
}
static void producer_closes(struct ll_queue* q, void* arg) {
(void)q;
struct ETCP_ROUTER_CONN* c = arg;
etcp_router_conn_close(c);
}
static int close_from_producer(void) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
struct ETCP_ROUTER_CONN* c = rc(0,0);
for (int i=0; i<ROUTER_MAX_SEND_Q_PACKETS; i++) REQUIRE(send_one(0,0) == 0);
struct queue_waiter_handle waiter = {0};
etcp_router_on_send_ready(&ep[0].inst, 0, ep[1].inst.node_id, svc_base, &waiter, producer_closes, c);
REQUIRE(waiter.internal);
step();
REQUIRE(ep[0].notified[0] == 1 && ep[0].inst.router_conns->count == 0);
REQUIRE(!waiter.internal && !waiter.call_soon_id);
cleanup(); return 0;
}
static int route_switch(void) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
struct ll_queue* old = ep[0].conn.send_input_q;
queue_data_put(old, queue_entry_new(0)); REQUIRE(send_one(0,0) == 0);
REQUIRE(rc(0,0)->send_waiter.internal);
((struct conn_queue_entry*)ep[0].route->data)->conn = &ep[0].alternate;
uint64_t deadline = get_time_tb()+20000;
while (!settled(0,0) && get_time_tb()<deadline) {
uasync_poll(ua,1); wire(0,ep[0].alternate.send_input_q); wire(1,ep[1].conn.send_input_q);
}
REQUIRE(settled(0,0) && ep[1].received[0] == 2); REQUIRE(old->waiter_head == NULL);
cleanup(); return 0;
}
static int no_route_restore(void) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
struct conn_queue_entry* route = (struct conn_queue_entry*)ep[0].route->data;
route->conn = NULL; REQUIRE(send_one(0,0) == 0);
REQUIRE(rc(0,0)->no_route && rc(0,0)->send_q->count == 1);
for (int i=0; i<30; i++) step();
route->conn = &ep[0].alternate; REQUIRE(wait_settled(0,0) == 0);
REQUIRE(ep[0].notified[0] == 0 && ep[1].notified[0] == 0);
cleanup(); return 0;
}
static int timeout_with_duplicate_ack(void) {
REQUIRE(setup() == 0); drop_all_ack = 1; REQUIRE(send_one(0,0) == 0);
uint64_t deadline = get_time_tb()+70000, last_dup = 0;
while (!ep[0].notified[0] && get_time_tb()<deadline) {
if (saved_ack && get_time_tb()-last_dup >= 500) {
struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = 0;
inject(0,e); last_dup = get_time_tb();
}
step();
}
REQUIRE(ep[0].notified[0] == 1); REQUIRE(ep[1].received[0] == 1);
REQUIRE(dropped_ack >= ROUTER_NO_ACK_MAX_RETRANS-1);
REQUIRE(ep[0].inst.router_conns->count == 0);
cleanup(); return 0;
}
static int simultaneous_restart(void) {
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(send_one(1,0) == 0);
REQUIRE(wait_settled(0,0) == 0); REQUIRE(wait_settled(1,0) == 0);
for (int i=0; i<2; i++) etcp_router_conn_restart(&ep[i].inst, 0, ep[1-i].inst.node_id, SVC);
REQUIRE(send_one(0,0) == 0); REQUIRE(send_one(1,0) == 0);
REQUIRE(wait_settled(0,0) == 0); REQUIRE(wait_settled(1,0) == 0);
REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1);
cleanup(); return 0;
}
static int call_ten_second_gap(void) {
svc_base = ETCP_RT_ID_CALL;
REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
uint64_t epoch = rc(0,0)->reset_id;
drop_wire = 1;
REQUIRE(send_one(0,0) == 0);
uint64_t deadline = get_time_tb() + 100000;
while (get_time_tb() < deadline && !failed) step();
REQUIRE(!failed); REQUIRE(ep[0].notified[0] == 0);
REQUIRE(rc(0,0)->reset_id == epoch && rc(0,0)->inflight_q->count == 1);
drop_wire = 0;
REQUIRE(wait_settled(0,0) == 0);
REQUIRE(ep[1].received[0] == 2 && ep[1].notified[0] == 0);
REQUIRE(rc(0,0)->reset_id == epoch && rc(0,0)->c_retrans_done > 0);
cleanup(); svc_base = SVC; return 0;
}
static int size_bounds(void) {
REQUIRE(setup() == 0);
uint8_t* data = u_calloc(1, PKTNORM_MAX_DGRAM_SIZE);
REQUIRE(data);
struct ETCP_ROUTER_CONN* c = rc(0,0);
REQUIRE(etcp_router_conn_send(c, data, 0, 0) == -1);
REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE + 1, 0) == -1);
REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE, ROUTER_FLAG_SIGNED) == -1);
REQUIRE(etcp_router_conn_send(c, data, SIZE_MAX, 0) == -1);
REQUIRE(c->tx_seq == 0 && c->send_q->count == 0);
REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE - ROUTER_PATH_MAX_SIZE + 1, 0) == -1);
REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE - ROUTER_PATH_MAX_SIZE, 0) == 0);
u_free(data); cleanup(); return 0;
}
int main(void) {
debug_config_init(); debug_set_categories(DEBUG_CATEGORY_ALL); debug_set_level(DEBUG_LEVEL_WARN);
const char* log = getenv("ROUTER_TEST_LOG");
if (log) { debug_enable_file_output(log, 1); debug_set_category_level(DEBUG_CATEGORY_ETCPROUTE, DEBUG_LEVEL_DEBUG); }
struct { const char* name; int (*run)(void); } tests[] = {
{"loss/reorder/duplicate and duplex multiplexing", loss_reorder}, {"lost HELLO/challenge/confirm", handshake_loss},
{"ACK bounds and no false progress", ack_bounds}, {"transport reinit retains inflight", reinit_retains},
{"restart rejects old DATA/ACK/CLOSE/HELLO/confirm", stale_session}, {"sequence wraparound", wraparound},
{"waiter lifetime after route switch", waiter_lifetime}, {"service closes during delivery", close_from_handler}, {"route switch while old next hop blocked", route_switch},
{"no route then restore without data loss", no_route_restore}, {"duplicate ACK cannot prevent timeout", timeout_with_duplicate_ack},
{"simultaneous endpoint restart", simultaneous_restart}, {"call: 10s outage retains session and pending audio", call_ten_second_gap}, {"payload size boundaries", size_bounds}, {"producer closes during send queue drain", close_from_producer}
};
for (unsigned i=0; i<sizeof(tests)/sizeof(tests[0]); i++) {
printf("TEST %u: %s\n", i+1, tests[i].name); fflush(stdout);
if (tests[i].run() != 0 || failed) return 1;
puts("PASS");
}
puts("All router fault scenarios passed"); return 0;
}