Add deterministic normalizer and restart fault injection hooks

tests/Makefile.am

@@ -1034,7 +1034,8 @@ test_stcp_sigpipe_SOURCES = test_stcp_sigpipe.c
 test_stcp_sigpipe_LDADD = $(LIBUTUN) $(COMMON_LIBS)
 test_etcp_windows_LDADD = $(LIBUTUN) $(COMMON_LIBS)
 test_etcp_windows_LDFLAGS = $(test_clock_ldflags) -Wl,--wrap=etcp_encrypt_send -Wl,--wrap=memory_pool_alloc_impl \
-    -Wl,--wrap=bbr_main -Wl,--wrap=uasync_set_timeout -Wl,--wrap=uasync_call_soon
+    -Wl,--wrap=bbr_main -Wl,--wrap=uasync_set_timeout -Wl,--wrap=uasync_call_soon -Wl,--wrap=ll_alloc_lldgram \
+    -Wl,--wrap=u_malloc_impl -Wl,--wrap=u_calloc_impl -Wl,--wrap=u_realloc_impl -Wl,--wrap=random_bytes
 
 test_etcp_router_tcp_SOURCES = test_etcp_router_tcp.c
 test_etcp_router_tcp_LDADD = $(LIBUTUN) $(COMMON_LIBS)

tests/test_etcp_windows.c

@@ -5,6 +5,7 @@
 #include "etcp_api.h"
 #include "etcp_bbr.h"
 #include "etcp_loadbalancer.h"
+#include "etcp_session.h"
 #include "config_parser.h"
 #include "utun_instance.h"
 #include "test_clock.h"
@@ -23,21 +24,31 @@ static struct ETCP_LINK* last_sent_link;
 static unsigned last_sent_request, last_sent_payload;
 static struct bbr_rate_sample last_sample;
 static uint32_t last_mss;
+static unsigned fail_rx_message_at, fail_tx_inflight, fail_normalizer_timer, fail_normalizer_waiter, fail_session_timer_at;
+static unsigned no_alloc, unexpected_alloc, fail_rng, session_sends, received, receive_action;
+static uint8_t received_values[8];
 
 void* __real_uasync_call_soon(struct UASYNC*, void*, timeout_callback_t);
 void* __wrap_uasync_call_soon(struct UASYNC* loop, void* arg, timeout_callback_t callback) {
+    if (fail_normalizer_timer && conn && arg == conn->normalizer) { fail_normalizer_timer--; return NULL; }
+    if (fail_normalizer_waiter && conn && arg == &conn->normalizer->input_waiter_handle) {
+        fail_normalizer_waiter--; return NULL;
+    }
     if (fail_tx_timer && arg == conn) { fail_tx_timer--; return NULL; }
     return __real_uasync_call_soon(loop, arg, callback);
 }
 
 void* __real_uasync_set_timeout(struct UASYNC*, int, void*, timeout_callback_t, const char*);
 void* __wrap_uasync_set_timeout(struct UASYNC* loop, int delay, void* arg, timeout_callback_t callback, const char* name) {
+    if (fail_session_timer_at && !strcmp(name, "etcp_session") && !--fail_session_timer_at) return NULL;
     if (fail_shaper_timer && !strcmp(name, "etcp_shaper")) { fail_shaper_timer--; return NULL; }
     if (fail_tx_timer && !strcmp(name, "etcp_tx")) { fail_tx_timer--; return NULL; }
     return __real_uasync_set_timeout(loop, delay, arg, callback, name);
 }
 
 int __wrap_etcp_encrypt_send(struct ETCP_DGRAM* pkt) {
+    if (pkt->data_len == ETCP_SESSION_CONTROL_SIZE && pkt->data[0] >= ETCP_SESSION_HELLO &&
+        pkt->data[0] <= ETCP_SESSION_CONFIRM) { session_sends++; return pkt->data_len; }
     CHECK(conn->tx_running && pkt->link && pkt->link->initialized && pkt->link->link_status);
     sends++;
     last_sent_link = pkt->link; last_sent_request = pkt->ack_request; last_sent_payload = pkt->payload_len;
@@ -47,6 +58,8 @@ int __wrap_etcp_encrypt_send(struct ETCP_DGRAM* pkt) {
 
 void* __real_memory_pool_alloc_impl(struct memory_pool* pool, const char* location);
 void* __wrap_memory_pool_alloc_impl(struct memory_pool* pool, const char* location) {
+    if (no_alloc) { unexpected_alloc++; return NULL; }
+    if (fail_tx_inflight && conn && pool == conn->inflight_pool) { fail_tx_inflight--; return NULL; }
     if (pool == inst.data_pool && fail_rx_data) { fail_rx_data--; return NULL; }
     if (fail_rx_fragment && conn && pool == conn->io_pool) { fail_rx_fragment--; return NULL; }
     if (pool == inst.ack_pool && fail_ack_entry) { fail_ack_entry--; return NULL; }
@@ -54,6 +67,32 @@ void* __wrap_memory_pool_alloc_impl(struct memory_pool* pool, const char* locati
     return __real_memory_pool_alloc_impl(pool, location);
 }
 
+struct ll_entry* __real_ll_alloc_lldgram(uint16_t);
+struct ll_entry* __wrap_ll_alloc_lldgram(uint16_t len) {
+    if (fail_rx_message_at && !--fail_rx_message_at) return NULL;
+    return __real_ll_alloc_lldgram(len);
+}
+void* __real_u_malloc_impl(uint32_t, const char*);
+void* __wrap_u_malloc_impl(uint32_t size, const char* where) {
+    if (no_alloc) { unexpected_alloc++; return NULL; }
+    return __real_u_malloc_impl(size, where);
+}
+void* __real_u_calloc_impl(uint32_t, uint32_t, const char*);
+void* __wrap_u_calloc_impl(uint32_t count, uint32_t size, const char* where) {
+    if (no_alloc) { unexpected_alloc++; return NULL; }
+    return __real_u_calloc_impl(count, size, where);
+}
+void* __real_u_realloc_impl(void*, uint32_t, const char*);
+void* __wrap_u_realloc_impl(void* ptr, uint32_t size, const char* where) {
+    if (no_alloc) { unexpected_alloc++; return NULL; }
+    return __real_u_realloc_impl(ptr, size, where);
+}
+int __real_random_bytes(uint8_t*, size_t);
+int __wrap_random_bytes(uint8_t* data, size_t len) {
+    if (fail_rng) { fail_rng--; return -1; }
+    return __real_random_bytes(data, len);
+}
+
 void __real_bbr_main(struct bbr*, const struct bbr_rate_sample*, uint32_t*, uint32_t*, uint32_t, uint32_t, int);
 void __wrap_bbr_main(struct bbr* model, const struct bbr_rate_sample* sample, uint32_t* cwnd,
                     uint32_t* pacing, uint32_t mss, uint32_t inflight, int limited) {
@@ -91,6 +130,9 @@ static void setup(void) {
     sends = fail_alloc = fail_send = fail_tx_timer = bbr_calls = saw_cwnd_limited = saw_app_limited = 0;
     fail_shaper_timer = 0;
     fail_rx_data = fail_rx_fragment = fail_ack_entry = 0;
+    fail_rx_message_at = fail_tx_inflight = fail_normalizer_timer = fail_normalizer_waiter = fail_session_timer_at = 0;
+    no_alloc = unexpected_alloc = fail_rng = session_sends = received = receive_action = 0;
+    memset(received_values, 0, sizeof(received_values));
     last_sent_link = NULL; last_sent_request = last_sent_payload = 0;
     ua = uasync_create(); CHECK(ua);
     inst.ua = ua; inst.node_id = 1; inst.config = &cfg; cfg.global.bbr_max_cwnd = INFLIGHT_LIM_MAX;