From b98b87ea12bcf6c8465d85b85ae3efa8dfce08d3 Mon Sep 17 00:00:00 2001 From: evgeny Date: Sat, 24 Oct 2026 10:26:14 +0200 Subject: [PATCH] Explain retry choices between stalled multipath channels --- tests/Makefile.am | 2 +- tests/test_etcp_multipath.c | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 5524a223..94a5d480 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1045,7 +1045,7 @@ test_dummynet_channels_LDFLAGS = $(test_clock_ldflags) test_etcp_multipath_SOURCES = test_etcp_multipath.c $(test_clock_sources) test_etcp_multipath_LDADD = $(LIBUTUN) $(COMMON_LIBS) test_etcp_multipath_LDFLAGS = $(test_clock_ldflags) -Wl,--wrap=etcp_dump_pkt_sections -Wl,--wrap=bbr_main \ - -Wl,--wrap=etcp_tx_complete -Wl,--wrap=queue_data_put_with_index + -Wl,--wrap=etcp_tx_complete -Wl,--wrap=queue_data_put_with_index -Wl,--wrap=etcp_loadbalancer_select_link test_stcp_sigpipe_SOURCES = test_stcp_sigpipe.c test_stcp_sigpipe_LDADD = $(LIBUTUN) $(COMMON_LIBS) test_etcp_windows_LDADD = $(LIBUTUN) $(COMMON_LIBS) diff --git a/tests/test_etcp_multipath.c b/tests/test_etcp_multipath.c index b23c5002..823a611f 100644 --- a/tests/test_etcp_multipath.c +++ b/tests/test_etcp_multipath.c @@ -60,6 +60,7 @@ static uint64_t sample_logged[2][PATHS]; static struct recovery_stats recovery; static uint32_t fault_rto_us[PATHS]; // максимум RTO каждого пути с начала отказа, независимо от измеренной паузы static uint32_t rx_trigger[2]; // DATA, разбор которого сейчас разблокировал выдачу сообщения +static unsigned fault_paths; static FILE* csv; static const char* case_name; static const char* pubkeys[2] = { @@ -188,6 +189,28 @@ int __wrap_queue_data_put_with_index(struct ll_queue* q, struct ll_entry* entry) return __real_queue_data_put_with_index(q, entry); } +struct ETCP_LINK* __real_etcp_loadbalancer_select_link(struct ETCP_CONN*, uint32_t, const struct INFLIGHT_PACKET*); +struct ETCP_LINK* __wrap_etcp_loadbalancer_select_link(struct ETCP_CONN* conn, uint32_t bytes, const struct INFLIGHT_PACKET* retry) { + struct ETCP_LINK* selected = __real_etcp_loadbalancer_select_link(conn, bytes, retry); + if (!measuring || !fault_start || conn != ep[0].conn || !retry || !selected || !retry->last_link) return selected; + int previous = -1, chosen = -1; + for (unsigned p = 0; p < path_count; p++) { + if (ss_get_port(&retry->last_link->remote_addr) == dummynet_get_listen_port(channels[p])) previous = p; + if (ss_get_port(&selected->remote_addr) == dummynet_get_listen_port(channels[p])) chosen = p; + } + if (previous < 0 || chosen < 0 || !(fault_paths & (1u << previous)) || !(fault_paths & (1u << chosen))) return selected; + for (struct ETCP_LINK* healthy = conn->links; healthy; healthy = healthy->next) for (unsigned p = 0; p < path_count; p++) { + if ((fault_paths & (1u << p)) || ss_get_port(&healthy->remote_addr) != dummynet_get_listen_port(channels[p])) continue; + int full = bytes > healthy->inflight_lim_bytes || healthy->inflight_bytes > healthy->inflight_lim_bytes - bytes; + DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "FAULT retry-choice %s seq=%u attempt=%u previous=%d selected=%d elapsed_us=%llu " + "healthy=%u initialized=%u UP=%u pacing_blocked=%d window_full=%d inflight=%u cwnd=%u total_attempts=%u", case_name, + retry->seq, retry->attempt_count, previous, chosen, (unsigned long long)(get_time_us() - fault_start), + p, healthy->initialized, healthy->link_status, healthy->shaper_timer != NULL, full, + healthy->inflight_bytes, healthy->inflight_lim_bytes, conn->active_attempts); + } + return selected; +} + static void receive_message(struct ETCP_CONN* conn, struct ll_entry* entry) { int side = conn->instance == ep[0].inst ? 0 : 1; struct endpoint* dest = &ep[side]; @@ -240,6 +263,7 @@ static void setup(const struct profile* settings, unsigned count, unsigned mask) memset(sample_logged, 0, sizeof(sample_logged)); memset(&recovery, 0, sizeof(recovery)); memset(fault_rto_us, 0, sizeof(fault_rto_us)); memset(rx_trigger, 0, sizeof(rx_trigger)); + fault_paths = 0; path_count = count; enabled_mask = mask; measuring = 0; fault_start = 0; fault_trace_count = 0; memcpy(profiles, settings, count * sizeof(*settings)); ua = uasync_create(); CHECK(ua); @@ -438,7 +462,7 @@ static void stalled(const struct profile* settings, unsigned count, unsigned bro run_for(1, 1, "arm-fault"); } CHECK(pending == broken && fault_seq); - begin_measure(); fault_start = get_time_us(); + begin_measure(); fault_start = get_time_us(); fault_paths = broken; for (struct ETCP_LINK* link = ep[0].conn->links; link; link = link->next) for (unsigned p = 0; p < count; p++) if (ss_get_port(&link->remote_addr) == dummynet_get_listen_port(channels[p])) fault_rto_us[p] = etcp_tx_rto_us(link);