/* Deterministic SVC_ROUTE fault injection. Real router/queues/timers, no UDP: * lower ETCP must not conceal the router's own loss/reorder/restart bugs. */ #include #include #include #include "etcp.h" #include "pkt_normalizer.h" #include "etcp_router.h" #include "utun_instance.h" #include "../lib/debug_config.h" #include "../lib/mem.h" #define SVC 0x42 #define COUNT 160 #define REQUIRE(x) do { if (!(x)) { fprintf(stderr, "FAIL %s:%d: %s\n", __func__, __LINE__, #x); return -1; } } while (0) struct endpoint { struct UTUN_INSTANCE inst; struct ETCP_CONN conn, alternate; struct ll_entry* route; unsigned sent[2], received[2], notified[2]; }; static struct endpoint ep[2]; static struct UASYNC* ua; static int failed, drop_data, drop_ack, drop_hello[2], drop_challenge[2], drop_confirm[2], drop_all_ack; static int reorder, duplicated, dropped_data, dropped_ack, handshake_drops, closed_in_handler; static unsigned max_wire; static struct ll_entry *held, *saved_data, *saved_ack, *saved_hello, *saved_confirm; static void release(struct ll_entry* e) { if (e) { queue_dgram_free(e); queue_entry_free(e); } } static struct ll_entry* copy_packet(const struct ll_entry* e) { struct ll_entry* copy = ll_alloc_lldgram(e->len); if (!copy) abort(); memcpy(copy->dgram, e->dgram, e->len); copy->len = e->len; return copy; } static struct ETCP_ROUTER_CONN* rc(int side, int svc) { return etcp_router_conn_get(&ep[side].inst, 0, ep[1-side].inst.node_id, SVC + svc); } static void receive(struct ETCP_CONN* conn, struct ll_entry* e) { int side = conn->instance == &ep[0].inst ? 0 : 1; unsigned svc = e->dgram[0] - SVC; uint64_t src, dst, group; memcpy(&src, e->dgram + ROUTER_SVC_SRC_OFF, 8); memcpy(&dst, e->dgram + ROUTER_SVC_DST_OFF, 8); memcpy(&group, e->dgram + ROUTER_SVC_GROUP_OFF, 8); if (svc >= 2 || src != ep[1-side].inst.node_id || dst != ep[side].inst.node_id || group) { fprintf(stderr, "FAIL delivery identity side=%d svc=%u\n", side, svc); failed = 1; release(e); return; } if (e->len == ROUTER_SVC_HDR_SIZE) { ep[side].notified[svc]++; release(e); return; } uint32_t number = 0; if (e->len < ROUTER_SVC_HDR_SIZE + 4) { failed = 1; release(e); return; } memcpy(&number, e->dgram + ROUTER_SVC_PAYLOAD_OFF, 4); if (number != ep[side].received[svc]) { fprintf(stderr, "FAIL delivery sequence side=%d svc=%u got=%u expected=%u\n", side, svc, number, ep[side].received[svc]); failed = 1; } size_t len = 4 + number % 181; if (e->len != ROUTER_SVC_HDR_SIZE + len) failed = 1; for (size_t i = 4; i < len; i++) if (e->dgram[ROUTER_SVC_PAYLOAD_OFF + i] != (uint8_t)(number ^ i ^ svc)) failed = 1; ep[side].received[svc]++; release(e); if (closed_in_handler) { closed_in_handler = 0; etcp_router_conn_close(rc(side, svc)); } } static void inject(int side, struct ll_entry* e) { ep[side].inst.api_bindings.callbacks[ETCP_RT_ID_SVC_ROUTE](&ep[side].conn, e); } static void wire(int from, struct ll_queue* q) { if ((unsigned)q->count > max_wire) max_wire = q->count; struct ll_entry* e = queue_data_get(q); if (!e) return; if (e->len < SVC_ROUTE_HDR_SIZE) { release(e); return; } struct SVC_ROUTE_HDR* h = (struct SVC_ROUTE_HDR*)e->dgram; int data = e->len > SVC_ROUTE_HDR_SIZE; if (!data && h->flags == ROUTER_FLAG_START && !saved_hello && from == 0) saved_hello = copy_packet(e); if (!data && h->flags == (ROUTER_FLAG_START | ROUTER_FLAG_RST) && !saved_confirm && from == 0) saved_confirm = copy_packet(e); int* drop = NULL; if (!data && h->flags == ROUTER_FLAG_START) drop = &drop_hello[from]; if (!data && h->flags == ROUTER_FLAG_RST) drop = &drop_challenge[from]; if (!data && h->flags == (ROUTER_FLAG_START | ROUTER_FLAG_RST)) drop = &drop_confirm[from]; if (drop && *drop) { --*drop; handshake_drops++; release(e); return; } if (data && from == 0 && h->svc_id == SVC) { if (!saved_data) saved_data = copy_packet(e); if (drop_data && h->seq == 2) { drop_data--; dropped_data++; release(e); return; } if (reorder && h->seq == 3 && !held) { held = e; return; } if (reorder && h->seq == 4 && held) { struct ll_entry* dup = copy_packet(e); inject(1, e); inject(1, dup); inject(1, held); held = NULL; reorder = 0; duplicated++; return; } } if (!data && h->flags == 0 && from == 1) { if (!saved_ack) saved_ack = copy_packet(e); if (drop_all_ack || drop_ack) { if (drop_ack) drop_ack--; dropped_ack++; release(e); return; } } inject(1-from, e); } static void step(void) { uasync_poll(ua, 1); for (int i = 0; i < 2; i++) { wire(i, ep[i].conn.send_input_q); wire(i, ep[i].alternate.send_input_q); } } static int setup(void) { memset(ep, 0, sizeof(ep)); failed = 0; ua = uasync_create(); REQUIRE(ua); drop_data = drop_ack = drop_all_ack = reorder = duplicated = dropped_data = dropped_ack = handshake_drops = closed_in_handler = 0; memset(drop_hello, 0, sizeof(drop_hello)); memset(drop_challenge, 0, sizeof(drop_challenge)); memset(drop_confirm, 0, sizeof(drop_confirm)); max_wire = 0; held = saved_data = saved_ack = saved_hello = saved_confirm = NULL; for (int i = 0; i < 2; i++) { ep[i].inst.ua = ua; ep[i].inst.node_id = i + 1; REQUIRE(etcp_router_init(&ep[i].inst) == 0); REQUIRE(etcp_router_bind(&ep[i].inst, SVC, receive) == 0); REQUIRE(etcp_router_bind(&ep[i].inst, SVC+1, receive) == 0); ep[i].conn.instance = ep[i].alternate.instance = &ep[i].inst; ep[i].conn.peer_node_id = ep[i].alternate.peer_node_id = 2-i; ep[i].conn.send_input_q = queue_new(ua, 0, 0, 0, "wire"); ep[i].alternate.send_input_q = queue_new(ua, 0, 0, 0, "alternate_wire"); queue_set_waiter_defer(ep[i].conn.send_input_q, 1); queue_set_waiter_defer(ep[i].alternate.send_input_q, 1); ep[i].inst.connections = queue_new(ua, 16, 0, 8, "connections"); ep[i].route = queue_entry_new(sizeof(struct conn_queue_entry)); struct conn_queue_entry* ce = (struct conn_queue_entry*)ep[i].route->data; ce->peer_node_id = 2-i; ce->conn = &ep[i].conn; queue_data_put_with_index(ep[i].inst.connections, ep[i].route); } return 0; } static void cleanup(void) { for (int i = 0; i < 2; i++) etcp_router_destroy(&ep[i].inst); uasync_poll(ua, 0); for (int i = 0; i < 2; i++) { struct ll_entry* e; while ((e = queue_data_get(ep[i].conn.send_input_q))) release(e); while ((e = queue_data_get(ep[i].alternate.send_input_q))) release(e); queue_free(ep[i].conn.send_input_q); queue_free(ep[i].alternate.send_input_q); release(queue_data_get(ep[i].inst.connections)); queue_free(ep[i].inst.connections); } release(held); release(saved_data); release(saved_ack); release(saved_hello); release(saved_confirm); uasync_poll(ua, 0); if (ua->timer_alloc_count != ua->timer_free_count) { fprintf(stderr, "FAIL teardown: live timers=%llu\n", (unsigned long long)(ua->timer_alloc_count - ua->timer_free_count)); failed = 1; } uasync_destroy(ua, 0); } static int send_one(int side, int svc) { uint32_t n = ep[side].sent[svc]; uint8_t data[185]; size_t len = 4 + n % 181; memcpy(data, &n, 4); for (size_t i = 4; i < len; i++) data[i] = n ^ i ^ svc; if (etcp_router_conn_send(rc(side, svc), data, len, 0) != 0) return -1; ep[side].sent[svc]++; return 0; } static int settled(int side, int svc) { struct ETCP_ROUTER_CONN* c = rc(side, svc); return c->tx_acked == c->tx_seq && !c->send_q->count && !c->inflight_q->count; } static int wait_settled(int side, int svc) { uint64_t deadline = get_time_tb() + 40000; while (!settled(side, svc) && !failed && get_time_tb() < deadline) step(); REQUIRE(!failed); REQUIRE(settled(side, svc)); REQUIRE(ep[1-side].received[svc] == ep[side].sent[svc]); return 0; } static int transfer(void) { uint64_t deadline = get_time_tb() + 40000; int complete = 0, backpressure = 0; while (!complete && !failed && get_time_tb() < deadline) { complete = 1; for (int side = 0; side < 2; side++) for (int svc = 0; svc < 2; svc++) { while (ep[side].sent[svc] < COUNT) if (send_one(side, svc) != 0) { backpressure++; break; } if (ep[side].sent[svc] < COUNT || !settled(side, svc)) complete = 0; } step(); } REQUIRE(!failed); REQUIRE(complete); REQUIRE(backpressure); for (int side = 0; side < 2; side++) for (int svc = 0; svc < 2; svc++) { REQUIRE(ep[side].received[svc] == COUNT); REQUIRE(ep[side].notified[svc] == 0); } printf(" delivered=%u backpressure=%d data_drop=%d ack_drop=%d reorder=%d retrans=%u wire_max=%u\n", 4*COUNT, backpressure, dropped_data, dropped_ack, duplicated, rc(0,0)->c_retrans_done, max_wire); return 0; } static int loss_reorder(void) { REQUIRE(setup() == 0); drop_data = 1; drop_ack = 2; reorder = 1; REQUIRE(transfer() == 0); REQUIRE(dropped_data == 1 && dropped_ack == 2 && duplicated == 1); REQUIRE(rc(0,0)->c_retrans_done > 0); REQUIRE(max_wire < 20); cleanup(); return 0; } static int handshake_loss(void) { REQUIRE(setup() == 0); for (int i = 0; i < 2; i++) drop_hello[i] = drop_challenge[i] = drop_confirm[i] = 1; REQUIRE(transfer() == 0); REQUIRE(handshake_drops == 6); cleanup(); return 0; } static int ack_bounds(void) { REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); REQUIRE(saved_ack); struct ETCP_ROUTER_CONN* c = rc(0,0); uint64_t progress = c->last_ack_changed_tb; c->no_ack_count = 9; uint32_t invalid[] = {0, 1, 2, 0x7fffffffU, 0x80000001U}; for (unsigned i = 0; i < sizeof(invalid)/sizeof(invalid[0]); i++) { struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = invalid[i]; inject(0,e); REQUIRE(c->tx_acked == 1 && c->no_ack_count == 9 && c->last_ack_changed_tb == progress); } REQUIRE(send_one(0,0) == 0); // queued, not transmitted until deferred waiter runs struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = 2; inject(0,e); REQUIRE(c->tx_acked == 1); REQUIRE(wait_settled(0,0) == 0); cleanup(); return 0; } static int reinit_retains(void) { REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); REQUIRE(send_one(0,0) == 0); uasync_poll(ua,0); struct ETCP_ROUTER_CONN* c = rc(0,0); REQUIRE(c->inflight_q->count == 1); uint64_t epoch = c->reset_id; etcp_router_pause_retrans_for_node(&ep[0].inst, ep[1].inst.node_id); struct ll_entry* e; while ((e = queue_data_get(ep[0].conn.send_input_q))) release(e); REQUIRE(c->inflight_q->count == 1 && c->retrans_timer && c->reset_id == epoch); REQUIRE(wait_settled(0,0) == 0); REQUIRE(c->c_retrans_done > 0); REQUIRE(ep[0].notified[0] == 0 && ep[1].notified[0] == 0); cleanup(); return 0; } static int stale_session(void) { REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); uint64_t previous = rc(0,0)->reset_id; etcp_router_conn_restart(&ep[0].inst, 0, ep[1].inst.node_id, SVC); REQUIRE(rc(0,0)->reset_id != previous); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1); uint64_t epoch = rc(1,0)->peer_reset_id; inject(1,copy_packet(saved_data)); inject(0,copy_packet(saved_ack)); inject(1,copy_packet(saved_hello)); inject(1,copy_packet(saved_confirm)); struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->flags = ROUTER_FLAG_CLOSE; inject(0,e); for (int i = 0; i < 50; i++) step(); REQUIRE(rc(1,0)->peer_reset_id == epoch && ep[1].received[0] == 2); REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); cleanup(); return 0; } static int wraparound(void) { REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); rc(0,0)->tx_seq = rc(0,0)->tx_sent = rc(0,0)->tx_acked = UINT32_MAX-2; rc(1,0)->rx_seq = rc(1,0)->last_sent_ack_seq = UINT32_MAX-2; for (int i = 0; i < 7; i++) REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); REQUIRE(rc(0,0)->tx_acked == 4); cleanup(); return 0; } static int waiter_lifetime(void) { for (int deferred = 0; deferred < 2; deferred++) { REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); struct ll_queue* old = ep[0].conn.send_input_q; if (!deferred) queue_data_put(old, queue_entry_new(0)); REQUIRE(send_one(0,0) == 0); struct ETCP_ROUTER_CONN* c = rc(0,0); REQUIRE(c->send_waiter_q == old); REQUIRE(deferred ? c->send_waiter.call_soon_id != NULL : c->send_waiter.internal != NULL); ((struct conn_queue_entry*)ep[0].route->data)->conn = &ep[0].alternate; etcp_router_conn_close(c); uasync_poll(ua,0); struct ll_entry* e; while ((e = queue_data_get(old))) release(e); REQUIRE(old->waiter_head == NULL); cleanup(); } return 0; } static int close_from_handler(void) { REQUIRE(setup() == 0); closed_in_handler = 1; REQUIRE(send_one(0,0) == 0); uint64_t deadline = get_time_tb()+10000; while (!ep[1].received[0] && get_time_tb()count == 0); REQUIRE(!waiter.internal && !waiter.call_soon_id); cleanup(); return 0; } static int route_switch(void) { REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); struct ll_queue* old = ep[0].conn.send_input_q; queue_data_put(old, queue_entry_new(0)); REQUIRE(send_one(0,0) == 0); REQUIRE(rc(0,0)->send_waiter.internal); ((struct conn_queue_entry*)ep[0].route->data)->conn = &ep[0].alternate; uint64_t deadline = get_time_tb()+20000; while (!settled(0,0) && get_time_tb()waiter_head == NULL); cleanup(); return 0; } static int no_route_restore(void) { REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); struct conn_queue_entry* route = (struct conn_queue_entry*)ep[0].route->data; route->conn = NULL; REQUIRE(send_one(0,0) == 0); REQUIRE(rc(0,0)->no_route && rc(0,0)->send_q->count == 1); for (int i=0; i<30; i++) step(); route->conn = &ep[0].alternate; REQUIRE(wait_settled(0,0) == 0); REQUIRE(ep[0].notified[0] == 0 && ep[1].notified[0] == 0); cleanup(); return 0; } static int timeout_with_duplicate_ack(void) { REQUIRE(setup() == 0); drop_all_ack = 1; REQUIRE(send_one(0,0) == 0); uint64_t deadline = get_time_tb()+70000, last_dup = 0; while (!ep[0].notified[0] && get_time_tb()= 500) { struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = 0; inject(0,e); last_dup = get_time_tb(); } step(); } REQUIRE(ep[0].notified[0] == 1); REQUIRE(ep[1].received[0] == 1); REQUIRE(dropped_ack >= ROUTER_NO_ACK_MAX_RETRANS-1); REQUIRE(ep[0].inst.router_conns->count == 0); cleanup(); return 0; } static int simultaneous_restart(void) { REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(send_one(1,0) == 0); REQUIRE(wait_settled(0,0) == 0); REQUIRE(wait_settled(1,0) == 0); for (int i=0; i<2; i++) etcp_router_conn_restart(&ep[i].inst, 0, ep[1-i].inst.node_id, SVC); REQUIRE(send_one(0,0) == 0); REQUIRE(send_one(1,0) == 0); REQUIRE(wait_settled(0,0) == 0); REQUIRE(wait_settled(1,0) == 0); REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1); cleanup(); return 0; } static int size_bounds(void) { REQUIRE(setup() == 0); uint8_t* data = u_calloc(1, PKTNORM_MAX_DGRAM_SIZE); REQUIRE(data); struct ETCP_ROUTER_CONN* c = rc(0,0); REQUIRE(etcp_router_conn_send(c, data, 0, 0) == -1); REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE + 1, 0) == -1); REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE, ROUTER_FLAG_SIGNED) == -1); REQUIRE(etcp_router_conn_send(c, data, SIZE_MAX, 0) == -1); REQUIRE(c->tx_seq == 0 && c->send_q->count == 0); REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE, 0) == 0); u_free(data); cleanup(); return 0; } int main(void) { debug_config_init(); debug_set_categories(DEBUG_CATEGORY_ALL); debug_set_level(DEBUG_LEVEL_WARN); const char* log = getenv("ROUTER_TEST_LOG"); if (log) { debug_enable_file_output(log, 1); debug_set_category_level(DEBUG_CATEGORY_ETCPROUTE, DEBUG_LEVEL_DEBUG); } struct { const char* name; int (*run)(void); } tests[] = { {"loss/reorder/duplicate and duplex multiplexing", loss_reorder}, {"lost HELLO/challenge/confirm", handshake_loss}, {"ACK bounds and no false progress", ack_bounds}, {"transport reinit retains inflight", reinit_retains}, {"restart rejects old DATA/ACK/CLOSE/HELLO/confirm", stale_session}, {"sequence wraparound", wraparound}, {"waiter lifetime after route switch", waiter_lifetime}, {"service closes during delivery", close_from_handler}, {"route switch while old next hop blocked", route_switch}, {"no route then restore without data loss", no_route_restore}, {"duplicate ACK cannot prevent timeout", timeout_with_duplicate_ack}, {"simultaneous endpoint restart", simultaneous_restart}, {"payload size boundaries", size_bounds}, {"producer closes during send queue drain", close_from_producer} }; for (unsigned i=0; i