utun2/tests/test_etcp_router_faults.c

/* Deterministic SVC_ROUTE fault injection. Real router/queues/timers, no UDP:
 * lower ETCP must not conceal the router's own loss/reorder/restart bugs. */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "etcp.h"
#include "pkt_normalizer.h"
#include "etcp_router.h"
#include "utun_instance.h"
#include "../lib/debug_config.h"
#include "../lib/mem.h"

#define SVC 0x42
#define COUNT 160
#define REQUIRE(x) do { if (!(x)) { fprintf(stderr, "FAIL %s:%d: %s\n", __func__, __LINE__, #x); return -1; } } while (0)

struct endpoint {
    struct UTUN_INSTANCE inst;
    struct ETCP_CONN conn, alternate;
    struct ll_entry* route;
    unsigned sent[2], received[2], notified[2];
};
static struct endpoint ep[2];
static struct UASYNC* ua;
static int failed, drop_data, drop_ack, drop_hello[2], drop_challenge[2], drop_confirm[2], drop_all_ack;
static int reorder, duplicated, dropped_data, dropped_ack, handshake_drops, closed_in_handler;
static unsigned max_wire;
static struct ll_entry *held, *saved_data, *saved_ack, *saved_hello, *saved_confirm;

static void release(struct ll_entry* e) { if (e) { queue_dgram_free(e); queue_entry_free(e); } }
static struct ll_entry* copy_packet(const struct ll_entry* e) {
    struct ll_entry* copy = ll_alloc_lldgram(e->len);
    if (!copy) abort();
    memcpy(copy->dgram, e->dgram, e->len); copy->len = e->len;
    return copy;
}
static struct ETCP_ROUTER_CONN* rc(int side, int svc) {
    return etcp_router_conn_get(&ep[side].inst, 0, ep[1-side].inst.node_id, SVC + svc);
}
static void receive(struct ETCP_CONN* conn, struct ll_entry* e) {
    int side = conn->instance == &ep[0].inst ? 0 : 1;
    unsigned svc = e->dgram[0] - SVC;
    uint64_t src, dst, group;
    memcpy(&src, e->dgram + ROUTER_SVC_SRC_OFF, 8); memcpy(&dst, e->dgram + ROUTER_SVC_DST_OFF, 8);
    memcpy(&group, e->dgram + ROUTER_SVC_GROUP_OFF, 8);
    if (svc >= 2 || src != ep[1-side].inst.node_id || dst != ep[side].inst.node_id || group) {
        fprintf(stderr, "FAIL delivery identity side=%d svc=%u\n", side, svc); failed = 1; release(e); return;
    }
    if (e->len == ROUTER_SVC_HDR_SIZE) { ep[side].notified[svc]++; release(e); return; }
    uint32_t number = 0;
    if (e->len < ROUTER_SVC_HDR_SIZE + 4) { failed = 1; release(e); return; }
    memcpy(&number, e->dgram + ROUTER_SVC_PAYLOAD_OFF, 4);
    if (number != ep[side].received[svc]) {
        fprintf(stderr, "FAIL delivery sequence side=%d svc=%u got=%u expected=%u\n", side, svc, number, ep[side].received[svc]);
        failed = 1;
    }
    size_t len = 4 + number % 181;
    if (e->len != ROUTER_SVC_HDR_SIZE + len) failed = 1;
    for (size_t i = 4; i < len; i++) if (e->dgram[ROUTER_SVC_PAYLOAD_OFF + i] != (uint8_t)(number ^ i ^ svc)) failed = 1;
    ep[side].received[svc]++;
    release(e);
    if (closed_in_handler) { closed_in_handler = 0; etcp_router_conn_close(rc(side, svc)); }
}
static void inject(int side, struct ll_entry* e) {
    ep[side].inst.api_bindings.callbacks[ETCP_RT_ID_SVC_ROUTE](&ep[side].conn, e);
}
static void wire(int from, struct ll_queue* q) {
    if ((unsigned)q->count > max_wire) max_wire = q->count;
    struct ll_entry* e = queue_data_get(q);
    if (!e) return;
    if (e->len < SVC_ROUTE_HDR_SIZE) { release(e); return; }
    struct SVC_ROUTE_HDR* h = (struct SVC_ROUTE_HDR*)e->dgram;
    if (e->dgram[e->len - 1] != 1) { failed = 1; release(e); return; } /* every source retry starts afresh */
    int data = e->len > SVC_ROUTE_HDR_SIZE + 9;
    if (!data && h->flags == ROUTER_FLAG_START && !saved_hello && from == 0) saved_hello = copy_packet(e);
    if (!data && h->flags == (ROUTER_FLAG_START | ROUTER_FLAG_RST) && !saved_confirm && from == 0) saved_confirm = copy_packet(e);
    int* drop = NULL;
    if (!data && h->flags == ROUTER_FLAG_START) drop = &drop_hello[from];
    if (!data && h->flags == ROUTER_FLAG_RST) drop = &drop_challenge[from];
    if (!data && h->flags == (ROUTER_FLAG_START | ROUTER_FLAG_RST)) drop = &drop_confirm[from];
    if (drop && *drop) { --*drop; handshake_drops++; release(e); return; }
    if (data && from == 0 && h->svc_id == SVC) {
        if (!saved_data) saved_data = copy_packet(e);
        if (drop_data && h->seq == 2) { drop_data--; dropped_data++; release(e); return; }
        if (reorder && h->seq == 3 && !held) { held = e; return; }
        if (reorder && h->seq == 4 && held) {
            struct ll_entry* dup = copy_packet(e);
            inject(1, e); inject(1, dup); inject(1, held); held = NULL; reorder = 0; duplicated++;
            return;
        }
    }
    if (!data && h->flags == 0 && from == 1) {
        if (!saved_ack) saved_ack = copy_packet(e);
        if (drop_all_ack || drop_ack) { if (drop_ack) drop_ack--; dropped_ack++; release(e); return; }
    }
    inject(1-from, e);
}
static void step(void) {
    uasync_poll(ua, 1);
    for (int i = 0; i < 2; i++) { wire(i, ep[i].conn.send_input_q); wire(i, ep[i].alternate.send_input_q); }
}
static int setup(void) {
    memset(ep, 0, sizeof(ep)); failed = 0; ua = uasync_create(); REQUIRE(ua);
    drop_data = drop_ack = drop_all_ack = reorder = duplicated = dropped_data = dropped_ack = handshake_drops = closed_in_handler = 0;
    memset(drop_hello, 0, sizeof(drop_hello)); memset(drop_challenge, 0, sizeof(drop_challenge)); memset(drop_confirm, 0, sizeof(drop_confirm));
    max_wire = 0; held = saved_data = saved_ack = saved_hello = saved_confirm = NULL;
    for (int i = 0; i < 2; i++) {
        ep[i].inst.ua = ua; ep[i].inst.node_id = i + 1;
        REQUIRE(etcp_router_init(&ep[i].inst) == 0);
        REQUIRE(etcp_router_bind(&ep[i].inst, SVC, receive) == 0);
        REQUIRE(etcp_router_bind(&ep[i].inst, SVC+1, receive) == 0);
        ep[i].conn.instance = ep[i].alternate.instance = &ep[i].inst;
        ep[i].conn.peer_node_id = ep[i].alternate.peer_node_id = 2-i;
        ep[i].conn.send_input_q = queue_new(ua, 0, 0, 0, "wire");
        ep[i].alternate.send_input_q = queue_new(ua, 0, 0, 0, "alternate_wire");
        queue_set_waiter_defer(ep[i].conn.send_input_q, 1); queue_set_waiter_defer(ep[i].alternate.send_input_q, 1);
        ep[i].inst.connections = queue_new(ua, 16, 0, 8, "connections");
        ep[i].route = queue_entry_new(sizeof(struct conn_queue_entry));
        struct conn_queue_entry* ce = (struct conn_queue_entry*)ep[i].route->data;
        ce->peer_node_id = 2-i; ce->conn = &ep[i].conn;
        queue_data_put_with_index(ep[i].inst.connections, ep[i].route);
    }
    return 0;
}
static void cleanup(void) {
    for (int i = 0; i < 2; i++) etcp_router_destroy(&ep[i].inst);
    uasync_poll(ua, 0);
    for (int i = 0; i < 2; i++) {
        struct ll_entry* e;
        while ((e = queue_data_get(ep[i].conn.send_input_q))) release(e);
        while ((e = queue_data_get(ep[i].alternate.send_input_q))) release(e);
        queue_free(ep[i].conn.send_input_q); queue_free(ep[i].alternate.send_input_q);
        release(queue_data_get(ep[i].inst.connections)); queue_free(ep[i].inst.connections);
    }
    release(held); release(saved_data); release(saved_ack); release(saved_hello); release(saved_confirm);
    uasync_poll(ua, 0);
    if (ua->timer_alloc_count != ua->timer_free_count) {
        fprintf(stderr, "FAIL teardown: live timers=%llu\n", (unsigned long long)(ua->timer_alloc_count - ua->timer_free_count));
        failed = 1;
    }
    uasync_destroy(ua, 0);
}
static int send_one(int side, int svc) {
    uint32_t n = ep[side].sent[svc]; uint8_t data[185]; size_t len = 4 + n % 181;
    memcpy(data, &n, 4);
    for (size_t i = 4; i < len; i++) data[i] = n ^ i ^ svc;
    if (etcp_router_conn_send(rc(side, svc), data, len, 0) != 0) return -1;
    ep[side].sent[svc]++; return 0;
}
static int settled(int side, int svc) {
    struct ETCP_ROUTER_CONN* c = rc(side, svc);
    return c->tx_acked == c->tx_seq && !c->send_q->count && !c->inflight_q->count;
}
static int wait_settled(int side, int svc) {
    uint64_t deadline = get_time_tb() + 40000;
    while (!settled(side, svc) && !failed && get_time_tb() < deadline) step();
    REQUIRE(!failed); REQUIRE(settled(side, svc));
    REQUIRE(ep[1-side].received[svc] == ep[side].sent[svc]);
    return 0;
}
static int transfer(void) {
    uint64_t deadline = get_time_tb() + 40000;
    int complete = 0, backpressure = 0;
    while (!complete && !failed && get_time_tb() < deadline) {
        complete = 1;
        for (int side = 0; side < 2; side++) for (int svc = 0; svc < 2; svc++) {
            while (ep[side].sent[svc] < COUNT) if (send_one(side, svc) != 0) { backpressure++; break; }
            if (ep[side].sent[svc] < COUNT || !settled(side, svc)) complete = 0;
        }
        step();
    }
    REQUIRE(!failed); REQUIRE(complete); REQUIRE(backpressure);
    for (int side = 0; side < 2; side++) for (int svc = 0; svc < 2; svc++) {
        REQUIRE(ep[side].received[svc] == COUNT); REQUIRE(ep[side].notified[svc] == 0);
    }
    printf("  delivered=%u backpressure=%d data_drop=%d ack_drop=%d reorder=%d retrans=%u wire_max=%u\n",
           4*COUNT, backpressure, dropped_data, dropped_ack, duplicated, rc(0,0)->c_retrans_done, max_wire);
    return 0;
}
static int loss_reorder(void) {
    REQUIRE(setup() == 0); drop_data = 1; drop_ack = 2; reorder = 1;
    REQUIRE(transfer() == 0); REQUIRE(dropped_data == 1 && dropped_ack == 2 && duplicated == 1);
    REQUIRE(rc(0,0)->c_retrans_done > 0); REQUIRE(max_wire < 20);
    cleanup(); return 0;
}
static int handshake_loss(void) {
    REQUIRE(setup() == 0);
    for (int i = 0; i < 2; i++) drop_hello[i] = drop_challenge[i] = drop_confirm[i] = 1;
    REQUIRE(transfer() == 0); REQUIRE(handshake_drops == 6);
    cleanup(); return 0;
}
static int ack_bounds(void) {
    REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0); REQUIRE(saved_ack);
    struct ETCP_ROUTER_CONN* c = rc(0,0); uint64_t progress = c->last_ack_changed_tb; c->no_ack_count = 9;
    uint32_t invalid[] = {0, 1, 2, 0x7fffffffU, 0x80000001U};
    for (unsigned i = 0; i < sizeof(invalid)/sizeof(invalid[0]); i++) {
        struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = invalid[i]; inject(0,e);
        REQUIRE(c->tx_acked == 1 && c->no_ack_count == 9 && c->last_ack_changed_tb == progress);
    }
    REQUIRE(send_one(0,0) == 0); // queued, not transmitted until deferred waiter runs
    struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = 2; inject(0,e);
    REQUIRE(c->tx_acked == 1); REQUIRE(wait_settled(0,0) == 0);
    cleanup(); return 0;
}
static int reinit_retains(void) {
    REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
    REQUIRE(send_one(0,0) == 0); uasync_poll(ua,0);
    struct ETCP_ROUTER_CONN* c = rc(0,0); REQUIRE(c->inflight_q->count == 1);
    uint64_t epoch = c->reset_id;
    etcp_router_pause_retrans_for_node(&ep[0].inst, ep[1].inst.node_id);
    struct ll_entry* e; while ((e = queue_data_get(ep[0].conn.send_input_q))) release(e);
    REQUIRE(c->inflight_q->count == 1 && c->retrans_timer && c->reset_id == epoch);
    REQUIRE(wait_settled(0,0) == 0); REQUIRE(c->c_retrans_done > 0);
    REQUIRE(ep[0].notified[0] == 0 && ep[1].notified[0] == 0);
    cleanup(); return 0;
}
static int stale_session(void) {
    REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
    uint64_t previous = rc(0,0)->reset_id;
    etcp_router_conn_restart(&ep[0].inst, 0, ep[1].inst.node_id, SVC);
    REQUIRE(rc(0,0)->reset_id != previous);
    REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
    REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1);
    uint64_t epoch = rc(1,0)->peer_reset_id;
    inject(1,copy_packet(saved_data)); inject(0,copy_packet(saved_ack));
    inject(1,copy_packet(saved_hello)); inject(1,copy_packet(saved_confirm));
    struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->flags = ROUTER_FLAG_CLOSE; inject(0,e);
    for (int i = 0; i < 50; i++) step();
    REQUIRE(rc(1,0)->peer_reset_id == epoch && ep[1].received[0] == 2);
    REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1);
    REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
    cleanup(); return 0;
}
static int wraparound(void) {
    REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
    rc(0,0)->tx_seq = rc(0,0)->tx_sent = rc(0,0)->tx_acked = UINT32_MAX-2;
    rc(1,0)->rx_seq = rc(1,0)->last_sent_ack_seq = UINT32_MAX-2;
    for (int i = 0; i < 7; i++) REQUIRE(send_one(0,0) == 0);
    REQUIRE(wait_settled(0,0) == 0); REQUIRE(rc(0,0)->tx_acked == 4);
    cleanup(); return 0;
}
static int waiter_lifetime(void) {
    for (int deferred = 0; deferred < 2; deferred++) {
        REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
        struct ll_queue* old = ep[0].conn.send_input_q;
        if (!deferred) queue_data_put(old, queue_entry_new(0));
        REQUIRE(send_one(0,0) == 0);
        struct ETCP_ROUTER_CONN* c = rc(0,0);
        REQUIRE(c->send_waiter_q == old);
        REQUIRE(deferred ? c->send_waiter.call_soon_id != NULL : c->send_waiter.internal != NULL);
        ((struct conn_queue_entry*)ep[0].route->data)->conn = &ep[0].alternate;
        etcp_router_conn_close(c); uasync_poll(ua,0);
        struct ll_entry* e; while ((e = queue_data_get(old))) release(e);
        REQUIRE(old->waiter_head == NULL);
        cleanup();
    }
    return 0;
}
static int close_from_handler(void) {
    REQUIRE(setup() == 0); closed_in_handler = 1; REQUIRE(send_one(0,0) == 0);
    uint64_t deadline = get_time_tb()+10000;
    while (!ep[1].received[0] && get_time_tb()<deadline) step();
    REQUIRE(ep[1].received[0] == 1); REQUIRE(!failed);
    cleanup(); return 0;
}
static void producer_closes(struct ll_queue* q, void* arg) {
    (void)q;
    struct ETCP_ROUTER_CONN* c = arg;
    etcp_router_conn_close(c);
}
static int close_from_producer(void) {
    REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
    struct ETCP_ROUTER_CONN* c = rc(0,0);
    for (int i=0; i<ROUTER_MAX_SEND_Q_PACKETS; i++) REQUIRE(send_one(0,0) == 0);
    struct queue_waiter_handle waiter = {0};
    etcp_router_on_send_ready(&ep[0].inst, 0, ep[1].inst.node_id, SVC, &waiter, producer_closes, c);
    REQUIRE(waiter.internal);
    step();
    REQUIRE(ep[0].notified[0] == 1 && ep[0].inst.router_conns->count == 0);
    REQUIRE(!waiter.internal && !waiter.call_soon_id);
    cleanup(); return 0;
}
static int route_switch(void) {
    REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
    struct ll_queue* old = ep[0].conn.send_input_q;
    queue_data_put(old, queue_entry_new(0)); REQUIRE(send_one(0,0) == 0);
    REQUIRE(rc(0,0)->send_waiter.internal);
    ((struct conn_queue_entry*)ep[0].route->data)->conn = &ep[0].alternate;
    uint64_t deadline = get_time_tb()+20000;
    while (!settled(0,0) && get_time_tb()<deadline) {
        uasync_poll(ua,1); wire(0,ep[0].alternate.send_input_q); wire(1,ep[1].conn.send_input_q);
    }
    REQUIRE(settled(0,0) && ep[1].received[0] == 2); REQUIRE(old->waiter_head == NULL);
    cleanup(); return 0;
}
static int no_route_restore(void) {
    REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(wait_settled(0,0) == 0);
    struct conn_queue_entry* route = (struct conn_queue_entry*)ep[0].route->data;
    route->conn = NULL; REQUIRE(send_one(0,0) == 0);
    REQUIRE(rc(0,0)->no_route && rc(0,0)->send_q->count == 1);
    for (int i=0; i<30; i++) step();
    route->conn = &ep[0].alternate; REQUIRE(wait_settled(0,0) == 0);
    REQUIRE(ep[0].notified[0] == 0 && ep[1].notified[0] == 0);
    cleanup(); return 0;
}
static int timeout_with_duplicate_ack(void) {
    REQUIRE(setup() == 0); drop_all_ack = 1; REQUIRE(send_one(0,0) == 0);
    uint64_t deadline = get_time_tb()+70000, last_dup = 0;
    while (!ep[0].notified[0] && get_time_tb()<deadline) {
        if (saved_ack && get_time_tb()-last_dup >= 500) {
            struct ll_entry* e = copy_packet(saved_ack); ((struct SVC_ROUTE_HDR*)e->dgram)->seq = 0;
            inject(0,e); last_dup = get_time_tb();
        }
        step();
    }
    REQUIRE(ep[0].notified[0] == 1); REQUIRE(ep[1].received[0] == 1);
    REQUIRE(dropped_ack >= ROUTER_NO_ACK_MAX_RETRANS-1);
    REQUIRE(ep[0].inst.router_conns->count == 0);
    cleanup(); return 0;
}
static int simultaneous_restart(void) {
    REQUIRE(setup() == 0); REQUIRE(send_one(0,0) == 0); REQUIRE(send_one(1,0) == 0);
    REQUIRE(wait_settled(0,0) == 0); REQUIRE(wait_settled(1,0) == 0);
    for (int i=0; i<2; i++) etcp_router_conn_restart(&ep[i].inst, 0, ep[1-i].inst.node_id, SVC);
    REQUIRE(send_one(0,0) == 0); REQUIRE(send_one(1,0) == 0);
    REQUIRE(wait_settled(0,0) == 0); REQUIRE(wait_settled(1,0) == 0);
    REQUIRE(ep[0].notified[0] == 1 && ep[1].notified[0] == 1);
    cleanup(); return 0;
}
static int size_bounds(void) {
    REQUIRE(setup() == 0);
    uint8_t* data = u_calloc(1, PKTNORM_MAX_DGRAM_SIZE);
    REQUIRE(data);
    struct ETCP_ROUTER_CONN* c = rc(0,0);
    REQUIRE(etcp_router_conn_send(c, data, 0, 0) == -1);
    REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE + 1, 0) == -1);
    REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE, ROUTER_FLAG_SIGNED) == -1);
    REQUIRE(etcp_router_conn_send(c, data, SIZE_MAX, 0) == -1);
    REQUIRE(c->tx_seq == 0 && c->send_q->count == 0);
    REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE - ROUTER_PATH_MAX_SIZE + 1, 0) == -1);
    REQUIRE(etcp_router_conn_send(c, data, PKTNORM_MAX_DGRAM_SIZE - SVC_ROUTE_HDR_SIZE - ROUTER_PATH_MAX_SIZE, 0) == 0);
    u_free(data); cleanup(); return 0;
}

int main(void) {
    debug_config_init(); debug_set_categories(DEBUG_CATEGORY_ALL); debug_set_level(DEBUG_LEVEL_WARN);
    const char* log = getenv("ROUTER_TEST_LOG");
    if (log) { debug_enable_file_output(log, 1); debug_set_category_level(DEBUG_CATEGORY_ETCPROUTE, DEBUG_LEVEL_DEBUG); }
    struct { const char* name; int (*run)(void); } tests[] = {
        {"loss/reorder/duplicate and duplex multiplexing", loss_reorder}, {"lost HELLO/challenge/confirm", handshake_loss},
        {"ACK bounds and no false progress", ack_bounds}, {"transport reinit retains inflight", reinit_retains},
        {"restart rejects old DATA/ACK/CLOSE/HELLO/confirm", stale_session}, {"sequence wraparound", wraparound},
        {"waiter lifetime after route switch", waiter_lifetime}, {"service closes during delivery", close_from_handler}, {"route switch while old next hop blocked", route_switch},
        {"no route then restore without data loss", no_route_restore}, {"duplicate ACK cannot prevent timeout", timeout_with_duplicate_ack},
        {"simultaneous endpoint restart", simultaneous_restart}, {"payload size boundaries", size_bounds}, {"producer closes during send queue drain", close_from_producer}
    };
    for (unsigned i=0; i<sizeof(tests)/sizeof(tests[0]); i++) {
        printf("TEST %u: %s\n", i+1, tests[i].name); fflush(stdout);
        if (tests[i].run() != 0 || failed) return 1;
        puts("PASS");
    }
    puts("All router fault scenarios passed"); return 0;
}