merkle_sync: per-session out_q with backpressure, WAL timer, send_to, recompute returns changed

This commit is contained in:
evgeny committed 2026-08-15 12:36:05 +03:00
1 parent dcdff59c67
commit 00ac80f76c
2 files changed
+135 -61

No files matched your search

+122 -60
View File
@@ -6,13 +6,14 @@
#include "../../lib/debug_config.h"
#include "../../lib/mem.h"
#include "../../lib/u_async.h"
#include "../../lib/ll_queue.h"
#include <string.h>
#include <stdlib.h>
#include <sqlite3.h>
#define MS_ID "merkle_sync"
#define MS_BG_INTERVAL_MS 5000
#define MS_ID "merkle_sync"
#define MS_WAL_INTERVAL_MS 60000
#define MS_PENDING_MAX 128
struct bucket_entry { uint8_t level; uint8_t prefix_bytes; uint64_t prefix; };
@@ -30,6 +31,7 @@ struct ms_pending {
struct ms_session {
struct ms_session* next;
struct merkle_sync* ms;
char ns[64];
uint64_t peer;
uint8_t sess_state;
@@ -37,6 +39,8 @@ struct ms_session {
void* cb_arg;
struct ms_pending pending[MS_PENDING_MAX];
int pending_count;
struct ll_queue* out_q;
struct queue_waiter_handle waiter;
};
struct merkle_sync {
@@ -46,7 +50,7 @@ struct merkle_sync {
void* data_ctx;
struct ms_session* sessions;
uint8_t initialized;
void* bg_timer;
void* wal_timer;
};
/* ── DB access ── */
@@ -121,6 +125,7 @@ static int _recompute_bucket(struct merkle_sync* ms, const char* ns,
"DELETE FROM merkle_tree_hash WHERE namespace=? AND level=? AND prefix64=?", -1, &ds, NULL);
if (ds) { sqlite3_bind_text(ds, 1, ns, -1, SQLITE_STATIC); sqlite3_bind_int(ds, 2, level);
sqlite3_bind_int64(ds, 3, (sqlite3_int64)prefix64); sqlite3_step(ds); sqlite3_finalize(ds); }
return 1;
} else { sqlite3_finalize(cs); }
}
return 0;
@@ -149,15 +154,20 @@ static int _recompute_bucket(struct merkle_sync* ms, const char* ns,
sqlite3_bind_int(is, 5, count);
sqlite3_step(is); sqlite3_finalize(is);
}
return 0;
return 1;
}
void merkle_sync_recompute_path(struct UTUN_INSTANCE* inst, const char* ns, uint64_t key) {
struct merkle_sync* ms = inst->msync;
if (!ms || !ms->initialized) { DEBUG_WARN(DEBUG_CATEGORY_MEMBER_SYNC, "%s: recompute_path called but msync not initialized", MS_ID); return; }
int merkle_sync_recompute_path(struct UTUN_INSTANCE* inst, const char* ns, uint64_t key) {
struct merkle_sync* ms = inst ? inst->msync : NULL;
if (!ms || !ms->initialized) { DEBUG_WARN(DEBUG_CATEGORY_MEMBER_SYNC, "%s: recompute_path called but msync not initialized", MS_ID); return -1; }
DEBUG_TRACE(DEBUG_CATEGORY_MEMBER_SYNC, "%s: ns=%s key=%016llx", MS_ID, ns, (unsigned long long)key);
for (uint8_t level = 1; level <= MT_MAX_LEVEL; level++)
_recompute_bucket(ms, ns, level, merkle_sync_level_prefix(key, level));
int changed = 0;
for (uint8_t level = 1; level <= MT_MAX_LEVEL; level++) {
int r = _recompute_bucket(ms, ns, level, merkle_sync_level_prefix(key, level));
if (r < 0) return -1;
if (r > 0) changed = 1;
}
return changed;
}
const uint8_t* merkle_sync_get_hash(struct UTUN_INSTANCE* inst, const char* ns,
@@ -238,6 +248,76 @@ static struct ETCP_CONN* ms_find_conn_for_node(struct UTUN_INSTANCE* inst, uint6
return NULL;
}
/* ── Per-session out queue (broadcast/push с backpressure) ── */
static void ms_session_outq_free(struct ms_session* s) {
if (!s) return;
if (s->out_q && s->ms && s->ms->inst) {
struct ETCP_CONN* conn = ms_find_conn_for_node(s->ms->inst, s->peer);
if (conn && conn->send_input_q) queue_waiter_cancel(conn->send_input_q, &s->waiter);
}
if (s->out_q) {
struct ll_entry* e;
while ((e = queue_data_get(s->out_q)) != NULL) {
if (e->dgram) u_free(e->dgram);
queue_entry_free(e);
}
queue_free(s->out_q);
s->out_q = NULL;
}
}
/* Инициализировать out-очередь сессии (вызывается при создании сессии). */
static void ms_session_setup(struct merkle_sync* ms, struct ms_session* s) {
s->ms = ms;
s->out_q = queue_new(ms->inst->ua, 0, 0, 0, "ms_out");
if (!s->out_q) DEBUG_ERROR(DEBUG_CATEGORY_MEMBER_SYNC, "%s: out_q queue_new failed", MS_ID);
}
static void ms_outq_drain_cb(struct ll_queue* q, void* arg) {
struct ms_session* s = (struct ms_session*)arg;
struct merkle_sync* ms = s ? s->ms : NULL;
if (!ms || !s->out_q) return;
struct ll_entry* e = queue_data_get(s->out_q);
if (!e) return;
struct ETCP_CONN* conn = ms_find_conn_for_node(ms->inst, s->peer);
if (!conn) {
DEBUG_WARN(DEBUG_CATEGORY_MEMBER_SYNC, "%s: outq drain — no conn for peer=%016llx, drop %u bytes",
MS_ID, (unsigned long long)s->peer, e->len);
if (e->dgram) u_free(e->dgram);
queue_entry_free(e);
} else {
int r = etcp_send(conn, e);
if (r != 0) { if (e->dgram) u_free(e->dgram); queue_entry_free(e); }
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: outq drain peer=%016llx len=%u rc=%d left=%d",
MS_ID, (unsigned long long)s->peer, e->len, r, queue_entry_count(s->out_q));
}
if (queue_entry_count(s->out_q) > 0 && conn && conn->send_input_q)
queue_waiter_wait(conn->send_input_q, &s->waiter, ms_outq_drain_cb, s);
}
/* Собрать [svc_id][payload] и положить в out_q сессии (с backpressure на send_input_q). */
static int ms_outq_enqueue(struct ms_session* s, const uint8_t* payload, size_t plen) {
struct merkle_sync* ms = s ? s->ms : NULL;
if (!ms || !s->out_q) return -1;
struct ETCP_CONN* conn = ms_find_conn_for_node(ms->inst, s->peer);
if (!conn) { DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: outq enqueue — no conn for peer=%016llx", MS_ID, (unsigned long long)s->peer); return -1; }
uint8_t* buf = u_malloc(1 + plen);
if (!buf) return -1;
buf[0] = ms->svc_id;
memcpy(buf + 1, payload, plen);
struct ll_entry* e = queue_entry_new(0);
if (!e) { u_free(buf); return -1; }
e->dgram = buf; e->len = (uint16_t)(1 + plen);
int was_empty = (queue_entry_count(s->out_q) == 0);
if (queue_data_put(s->out_q, e) != 0) { u_free(buf); queue_entry_free(e); return -1; }
if (was_empty && conn->send_input_q)
queue_waiter_wait(conn->send_input_q, &s->waiter, ms_outq_drain_cb, s);
return 0;
}
static int _send_msg(struct merkle_sync* ms, uint64_t peer,
const uint8_t* payload, size_t len) {
if (!ms->inst || len < 1) return -1;
@@ -256,10 +336,10 @@ static int _send_msg(struct merkle_sync* ms, uint64_t peer,
return r;
}
static int _send_broadcast_data(struct merkle_sync* ms, uint64_t peer, const char* ns,
const uint8_t* item_data, size_t item_len) {
static int _broadcast_to_session(struct merkle_sync* ms, struct ms_session* s,
const char* ns, const uint8_t* item_data, size_t item_len) {
uint8_t ch_len = (uint8_t)strlen(ns);
size_t sz = 1 + 1 + ch_len + 1 + 1 + 1 + 2 + item_len;
size_t sz = 1 + ch_len + 1 + 1 + 1 + 1 + 2 + item_len;
uint8_t* buf = u_malloc(sz); if (!buf) return -1;
uint8_t* p = buf;
*p++ = ch_len; memcpy(p, ns, ch_len); p += ch_len;
@@ -269,7 +349,7 @@ static int _send_broadcast_data(struct merkle_sync* ms, uint64_t peer, const cha
*p++ = 1; /* is_data=1 */
uint16_t dlen = (uint16_t)item_len; memcpy(p, &dlen, 2); p += 2;
memcpy(p, item_data, item_len); p += item_len;
int r = _send_msg(ms, peer, buf, (size_t)(p - buf));
int r = ms_outq_enqueue(s, buf, (size_t)(p - buf));
u_free(buf);
return r;
}
@@ -413,6 +493,7 @@ static void _handle_hashes(struct merkle_sync* ms, uint64_t peer, const char* ns
s = u_calloc(1, sizeof(*s)); if (!s) return;
snprintf(s->ns, sizeof(s->ns), "%s", ns); s->peer = peer;
s->next = ms->sessions; ms->sessions = s;
ms_session_setup(ms, s);
} else {
s->pending_count = 0;
}
@@ -624,11 +705,10 @@ void merkle_sync_push_update(struct UTUN_INSTANCE* inst, const char* ns,
uint8_t ch_len = (uint8_t)strlen(ns);
if (ch_len > 63) return;
size_t pkt = 1 + 1 + ch_len + 1 + 8 + 1 + len;
size_t pkt = 1 + ch_len + 1 + 8 + 1 + len;
uint8_t* buf = u_malloc(pkt);
if (!buf) return;
uint8_t* p = buf;
*p++ = ms->svc_id;
*p++ = ch_len; memcpy(p, ns, ch_len); p += ch_len;
*p++ = 0x04; /* MSG_ITEM_UPDATE */
memcpy(p, &key, 8); p += 8;
@@ -637,18 +717,7 @@ void merkle_sync_push_update(struct UTUN_INSTANCE* inst, const char* ns,
for (struct ms_session* s = ms->sessions; s; s = s->next) {
if (s->sess_state != SESS_SYNCED || strcmp(s->ns, ns) != 0) continue;
struct ll_entry* entry = queue_entry_new(0);
if (!entry) continue;
uint8_t* dcopy = u_malloc(pkt);
if (!dcopy) { queue_entry_free(entry); continue; }
memcpy(dcopy, buf, pkt);
entry->dgram = dcopy; entry->len = pkt;
struct ETCP_CONN* conn = ms_find_conn_for_node(inst, s->peer);
if (!conn) { u_free(dcopy); queue_entry_free(entry); continue; }
int r = etcp_send(conn, entry);
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: push_update sent type=%02x to=%016llx rc=%d",
MS_ID, type, (unsigned long long)s->peer, r);
if (r != 0) { u_free(dcopy); queue_entry_free(entry); }
ms_outq_enqueue(s, buf, pkt);
}
u_free(buf);
}
@@ -663,10 +732,19 @@ void merkle_sync_broadcast(struct UTUN_INSTANCE* inst, const char* ns,
MS_ID, ns, (unsigned long long)from_peer, len);
for (struct ms_session* s = ms->sessions; s; s = s->next) {
if (s->peer == from_peer || strcmp(s->ns, ns) != 0) continue;
_send_broadcast_data(ms, s->peer, ns, data, len);
_broadcast_to_session(ms, s, ns, data, len);
}
}
int merkle_sync_send_to(struct UTUN_INSTANCE* inst, const char* ns,
uint64_t peer, const uint8_t* data, size_t len) {
struct merkle_sync* ms = inst ? inst->msync : NULL;
if (!ms || !ms->initialized || !ns || !data || len < 2) return -1;
struct ms_session* s = _session_find(ms, peer, ns);
if (!s) { DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: send_to — no session peer=%016llx ns=%s", MS_ID, (unsigned long long)peer, ns); return -1; }
return _broadcast_to_session(ms, s, ns, data, len);
}
/* ── Background consistency check ── */
int merkle_sync_bg_check(struct UTUN_INSTANCE* inst, const char* ns) {
@@ -699,38 +777,18 @@ int merkle_sync_bg_check(struct UTUN_INSTANCE* inst, const char* ns) {
return 0;
}
static void _bg_timer_cb(void* arg) {
static void _wal_timer_cb(void* arg) {
struct merkle_sync* ms = (struct merkle_sync*)arg;
if (!ms || !ms->initialized || !ms->inst) return;
sqlite3* db = _db(ms->inst); if (!db) return;
sqlite3* db = _db(ms->inst); if (!db) { ms->wal_timer = NULL; return; }
static int tick = 0; tick++;
int pn = 0, ckpt = 0;
sqlite3_wal_checkpoint_v2(db, NULL, SQLITE_CHECKPOINT_PASSIVE, &pn, &ckpt);
if (pn > 0 || ckpt > 0)
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: WAL checkpoint done: %d frames in log, %d checkpointed", MS_ID, pn, ckpt);
sqlite3_stmt* cs = NULL;
sqlite3_prepare_v2(db, "SELECT channel_id FROM channels", -1, &cs, NULL);
if (cs) {
int ch_idx = 0;
while (sqlite3_step(cs) == SQLITE_ROW) {
const char* ch_id = (const char*)sqlite3_column_text(cs, 0);
if (!ch_id) { ch_idx++; continue; }
if ((tick % (ch_idx + 1)) == 0) {
int rc = merkle_sync_bg_check(ms->inst, ch_id);
if (rc >= 0) DEBUG_TRACE(DEBUG_CATEGORY_MEMBER_SYNC, "%s: bg_check ns=%s → %s", MS_ID, ch_id, rc == 0 ? "consistent" : "RECALCULATED");
}
ch_idx++;
}
sqlite3_finalize(cs);
if (tick % 12 == 0) { /* every ~60s: try to checkpoint WAL, keep it under control */
int pn = 0, ckpt = 0;
sqlite3_wal_checkpoint_v2(db, NULL, SQLITE_CHECKPOINT_PASSIVE, &pn, &ckpt);
if (pn > 0 || ckpt > 0)
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: WAL checkpoint done: %d frames in log, %d checkpointed", MS_ID, pn, ckpt);
}
}
ms->bg_timer = uasync_set_timeout(ms->inst->ua, (uint32_t)(MS_BG_INTERVAL_MS * 10),
ms, _bg_timer_cb, "ms_bg");
ms->wal_timer = uasync_set_timeout(ms->inst->ua, (uint32_t)(MS_WAL_INTERVAL_MS * 10),
ms, _wal_timer_cb, "ms_wal");
}
/* ── Public lifecycle ── */
@@ -749,8 +807,8 @@ int merkle_sync_init(struct UTUN_INSTANCE* inst, uint8_t svc_id,
etcp_bind(inst, svc_id, _recv_cb);
ms->bg_timer = uasync_set_timeout(inst->ua, (uint32_t)(MS_BG_INTERVAL_MS * 10),
ms, _bg_timer_cb, "ms_bg");
ms->wal_timer = uasync_set_timeout(inst->ua, (uint32_t)(MS_WAL_INTERVAL_MS * 10),
ms, _wal_timer_cb, "ms_wal");
DEBUG_INFO(DEBUG_CATEGORY_MEMBER_SYNC, "%s: initialized svc=%02x", MS_ID, svc_id);
return 0;
@@ -764,10 +822,11 @@ void merkle_sync_destroy(struct UTUN_INSTANCE* inst) {
etcp_unbind(inst, ms->svc_id);
if (ms->bg_timer) { uasync_cancel_timeout(inst->ua, ms->bg_timer); ms->bg_timer = NULL; }
if (ms->wal_timer) { uasync_cancel_timeout(inst->ua, ms->wal_timer); ms->wal_timer = NULL; }
struct ms_session* s = ms->sessions;
while (s) { struct ms_session* next = s->next;
ms_session_outq_free(s);
u_free(s); s = next; }
u_free(ms);
@@ -790,6 +849,7 @@ int merkle_sync_start(struct UTUN_INSTANCE* inst, uint64_t peer,
s->peer = peer;
s->next = ms->sessions;
ms->sessions = s;
ms_session_setup(ms, s);
} else {
s->pending_count = 0;
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: start OVERWRITE peer=%016llx ns=%s", MS_ID, (unsigned long long)peer, ns);
@@ -809,7 +869,9 @@ void merkle_sync_cancel(struct UTUN_INSTANCE* inst, uint64_t peer, const char* n
while (*p) {
struct ms_session* s = *p;
if (s->peer == peer && strcmp(s->ns, ns) == 0) {
*p = s->next; u_free(s);
*p = s->next;
ms_session_outq_free(s);
u_free(s);
return;
}
p = &(*p)->next;
+13 -1
View File
@@ -222,8 +222,11 @@ void merkle_sync_cancel(struct UTUN_INSTANCE* inst, uint64_t peer, const char* n
* Вызывается после изменения данных: merkle_sync сам не знает,
* когда данные изменились — потребитель должен вызвать явно.
* member_sync делает это внутри put/del.
*
* Возвращает 1 если хеш хотя бы одного бакета изменился (данные реально
* изменились), 0 если всё совпало, -1 при ошибке.
*/
void merkle_sync_recompute_path(struct UTUN_INSTANCE* inst, const char* ns, uint64_t key);
int merkle_sync_recompute_path(struct UTUN_INSTANCE* inst, const char* ns, uint64_t key);
/*
* Отправить лёгкое обновление всем synced-пирам в namespace ns.
@@ -247,6 +250,15 @@ void merkle_sync_push_update(struct UTUN_INSTANCE* inst, const char* ns,
void merkle_sync_broadcast(struct UTUN_INSTANCE* inst, const char* ns,
uint64_t from_peer, const uint8_t* data, size_t len);
/*
* Отправить данные одного мембера конкретному пиру (а не всем).
* Используется для "send back" при обнаружении stale-версии.
* data, len — wire-формат [count:2][key:8][val:4]... как от get_items.
* Возвращает 0 при успехе, -1 если нет сессии/соединения к пиру.
*/
int merkle_sync_send_to(struct UTUN_INSTANCE* inst, const char* ns,
uint64_t peer, const uint8_t* data, size_t len);
/* ── Background consistency check ── */
/*