merkle_sync: per-session out_q with backpressure, WAL timer, send_to, recompute returns changed
This commit is contained in:
1 parent
dcdff59c67
commit
00ac80f76c
2 files changed
+135
-61
No files matched your search
+122
-60
@@ -6,13 +6,14 @@
|
||||
#include "../../lib/debug_config.h"
|
||||
#include "../../lib/mem.h"
|
||||
#include "../../lib/u_async.h"
|
||||
#include "../../lib/ll_queue.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <sqlite3.h>
|
||||
|
||||
#define MS_ID "merkle_sync"
|
||||
#define MS_BG_INTERVAL_MS 5000
|
||||
#define MS_ID "merkle_sync"
|
||||
#define MS_WAL_INTERVAL_MS 60000
|
||||
#define MS_PENDING_MAX 128
|
||||
|
||||
struct bucket_entry { uint8_t level; uint8_t prefix_bytes; uint64_t prefix; };
|
||||
@@ -30,6 +31,7 @@ struct ms_pending {
|
||||
|
||||
struct ms_session {
|
||||
struct ms_session* next;
|
||||
struct merkle_sync* ms;
|
||||
char ns[64];
|
||||
uint64_t peer;
|
||||
uint8_t sess_state;
|
||||
@@ -37,6 +39,8 @@ struct ms_session {
|
||||
void* cb_arg;
|
||||
struct ms_pending pending[MS_PENDING_MAX];
|
||||
int pending_count;
|
||||
struct ll_queue* out_q;
|
||||
struct queue_waiter_handle waiter;
|
||||
};
|
||||
|
||||
struct merkle_sync {
|
||||
@@ -46,7 +50,7 @@ struct merkle_sync {
|
||||
void* data_ctx;
|
||||
struct ms_session* sessions;
|
||||
uint8_t initialized;
|
||||
void* bg_timer;
|
||||
void* wal_timer;
|
||||
};
|
||||
|
||||
/* ── DB access ── */
|
||||
@@ -121,6 +125,7 @@ static int _recompute_bucket(struct merkle_sync* ms, const char* ns,
|
||||
"DELETE FROM merkle_tree_hash WHERE namespace=? AND level=? AND prefix64=?", -1, &ds, NULL);
|
||||
if (ds) { sqlite3_bind_text(ds, 1, ns, -1, SQLITE_STATIC); sqlite3_bind_int(ds, 2, level);
|
||||
sqlite3_bind_int64(ds, 3, (sqlite3_int64)prefix64); sqlite3_step(ds); sqlite3_finalize(ds); }
|
||||
return 1;
|
||||
} else { sqlite3_finalize(cs); }
|
||||
}
|
||||
return 0;
|
||||
@@ -149,15 +154,20 @@ static int _recompute_bucket(struct merkle_sync* ms, const char* ns,
|
||||
sqlite3_bind_int(is, 5, count);
|
||||
sqlite3_step(is); sqlite3_finalize(is);
|
||||
}
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
void merkle_sync_recompute_path(struct UTUN_INSTANCE* inst, const char* ns, uint64_t key) {
|
||||
struct merkle_sync* ms = inst->msync;
|
||||
if (!ms || !ms->initialized) { DEBUG_WARN(DEBUG_CATEGORY_MEMBER_SYNC, "%s: recompute_path called but msync not initialized", MS_ID); return; }
|
||||
int merkle_sync_recompute_path(struct UTUN_INSTANCE* inst, const char* ns, uint64_t key) {
|
||||
struct merkle_sync* ms = inst ? inst->msync : NULL;
|
||||
if (!ms || !ms->initialized) { DEBUG_WARN(DEBUG_CATEGORY_MEMBER_SYNC, "%s: recompute_path called but msync not initialized", MS_ID); return -1; }
|
||||
DEBUG_TRACE(DEBUG_CATEGORY_MEMBER_SYNC, "%s: ns=%s key=%016llx", MS_ID, ns, (unsigned long long)key);
|
||||
for (uint8_t level = 1; level <= MT_MAX_LEVEL; level++)
|
||||
_recompute_bucket(ms, ns, level, merkle_sync_level_prefix(key, level));
|
||||
int changed = 0;
|
||||
for (uint8_t level = 1; level <= MT_MAX_LEVEL; level++) {
|
||||
int r = _recompute_bucket(ms, ns, level, merkle_sync_level_prefix(key, level));
|
||||
if (r < 0) return -1;
|
||||
if (r > 0) changed = 1;
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
const uint8_t* merkle_sync_get_hash(struct UTUN_INSTANCE* inst, const char* ns,
|
||||
@@ -238,6 +248,76 @@ static struct ETCP_CONN* ms_find_conn_for_node(struct UTUN_INSTANCE* inst, uint6
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* ── Per-session out queue (broadcast/push с backpressure) ── */
|
||||
|
||||
static void ms_session_outq_free(struct ms_session* s) {
|
||||
if (!s) return;
|
||||
if (s->out_q && s->ms && s->ms->inst) {
|
||||
struct ETCP_CONN* conn = ms_find_conn_for_node(s->ms->inst, s->peer);
|
||||
if (conn && conn->send_input_q) queue_waiter_cancel(conn->send_input_q, &s->waiter);
|
||||
}
|
||||
if (s->out_q) {
|
||||
struct ll_entry* e;
|
||||
while ((e = queue_data_get(s->out_q)) != NULL) {
|
||||
if (e->dgram) u_free(e->dgram);
|
||||
queue_entry_free(e);
|
||||
}
|
||||
queue_free(s->out_q);
|
||||
s->out_q = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* Инициализировать out-очередь сессии (вызывается при создании сессии). */
|
||||
static void ms_session_setup(struct merkle_sync* ms, struct ms_session* s) {
|
||||
s->ms = ms;
|
||||
s->out_q = queue_new(ms->inst->ua, 0, 0, 0, "ms_out");
|
||||
if (!s->out_q) DEBUG_ERROR(DEBUG_CATEGORY_MEMBER_SYNC, "%s: out_q queue_new failed", MS_ID);
|
||||
}
|
||||
|
||||
static void ms_outq_drain_cb(struct ll_queue* q, void* arg) {
|
||||
struct ms_session* s = (struct ms_session*)arg;
|
||||
struct merkle_sync* ms = s ? s->ms : NULL;
|
||||
if (!ms || !s->out_q) return;
|
||||
struct ll_entry* e = queue_data_get(s->out_q);
|
||||
if (!e) return;
|
||||
struct ETCP_CONN* conn = ms_find_conn_for_node(ms->inst, s->peer);
|
||||
if (!conn) {
|
||||
DEBUG_WARN(DEBUG_CATEGORY_MEMBER_SYNC, "%s: outq drain — no conn for peer=%016llx, drop %u bytes",
|
||||
MS_ID, (unsigned long long)s->peer, e->len);
|
||||
if (e->dgram) u_free(e->dgram);
|
||||
queue_entry_free(e);
|
||||
} else {
|
||||
int r = etcp_send(conn, e);
|
||||
if (r != 0) { if (e->dgram) u_free(e->dgram); queue_entry_free(e); }
|
||||
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: outq drain peer=%016llx len=%u rc=%d left=%d",
|
||||
MS_ID, (unsigned long long)s->peer, e->len, r, queue_entry_count(s->out_q));
|
||||
}
|
||||
if (queue_entry_count(s->out_q) > 0 && conn && conn->send_input_q)
|
||||
queue_waiter_wait(conn->send_input_q, &s->waiter, ms_outq_drain_cb, s);
|
||||
}
|
||||
|
||||
/* Собрать [svc_id][payload] и положить в out_q сессии (с backpressure на send_input_q). */
|
||||
static int ms_outq_enqueue(struct ms_session* s, const uint8_t* payload, size_t plen) {
|
||||
struct merkle_sync* ms = s ? s->ms : NULL;
|
||||
if (!ms || !s->out_q) return -1;
|
||||
struct ETCP_CONN* conn = ms_find_conn_for_node(ms->inst, s->peer);
|
||||
if (!conn) { DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: outq enqueue — no conn for peer=%016llx", MS_ID, (unsigned long long)s->peer); return -1; }
|
||||
|
||||
uint8_t* buf = u_malloc(1 + plen);
|
||||
if (!buf) return -1;
|
||||
buf[0] = ms->svc_id;
|
||||
memcpy(buf + 1, payload, plen);
|
||||
struct ll_entry* e = queue_entry_new(0);
|
||||
if (!e) { u_free(buf); return -1; }
|
||||
e->dgram = buf; e->len = (uint16_t)(1 + plen);
|
||||
|
||||
int was_empty = (queue_entry_count(s->out_q) == 0);
|
||||
if (queue_data_put(s->out_q, e) != 0) { u_free(buf); queue_entry_free(e); return -1; }
|
||||
if (was_empty && conn->send_input_q)
|
||||
queue_waiter_wait(conn->send_input_q, &s->waiter, ms_outq_drain_cb, s);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int _send_msg(struct merkle_sync* ms, uint64_t peer,
|
||||
const uint8_t* payload, size_t len) {
|
||||
if (!ms->inst || len < 1) return -1;
|
||||
@@ -256,10 +336,10 @@ static int _send_msg(struct merkle_sync* ms, uint64_t peer,
|
||||
return r;
|
||||
}
|
||||
|
||||
static int _send_broadcast_data(struct merkle_sync* ms, uint64_t peer, const char* ns,
|
||||
const uint8_t* item_data, size_t item_len) {
|
||||
static int _broadcast_to_session(struct merkle_sync* ms, struct ms_session* s,
|
||||
const char* ns, const uint8_t* item_data, size_t item_len) {
|
||||
uint8_t ch_len = (uint8_t)strlen(ns);
|
||||
size_t sz = 1 + 1 + ch_len + 1 + 1 + 1 + 2 + item_len;
|
||||
size_t sz = 1 + ch_len + 1 + 1 + 1 + 1 + 2 + item_len;
|
||||
uint8_t* buf = u_malloc(sz); if (!buf) return -1;
|
||||
uint8_t* p = buf;
|
||||
*p++ = ch_len; memcpy(p, ns, ch_len); p += ch_len;
|
||||
@@ -269,7 +349,7 @@ static int _send_broadcast_data(struct merkle_sync* ms, uint64_t peer, const cha
|
||||
*p++ = 1; /* is_data=1 */
|
||||
uint16_t dlen = (uint16_t)item_len; memcpy(p, &dlen, 2); p += 2;
|
||||
memcpy(p, item_data, item_len); p += item_len;
|
||||
int r = _send_msg(ms, peer, buf, (size_t)(p - buf));
|
||||
int r = ms_outq_enqueue(s, buf, (size_t)(p - buf));
|
||||
u_free(buf);
|
||||
return r;
|
||||
}
|
||||
@@ -413,6 +493,7 @@ static void _handle_hashes(struct merkle_sync* ms, uint64_t peer, const char* ns
|
||||
s = u_calloc(1, sizeof(*s)); if (!s) return;
|
||||
snprintf(s->ns, sizeof(s->ns), "%s", ns); s->peer = peer;
|
||||
s->next = ms->sessions; ms->sessions = s;
|
||||
ms_session_setup(ms, s);
|
||||
} else {
|
||||
s->pending_count = 0;
|
||||
}
|
||||
@@ -624,11 +705,10 @@ void merkle_sync_push_update(struct UTUN_INSTANCE* inst, const char* ns,
|
||||
|
||||
uint8_t ch_len = (uint8_t)strlen(ns);
|
||||
if (ch_len > 63) return;
|
||||
size_t pkt = 1 + 1 + ch_len + 1 + 8 + 1 + len;
|
||||
size_t pkt = 1 + ch_len + 1 + 8 + 1 + len;
|
||||
uint8_t* buf = u_malloc(pkt);
|
||||
if (!buf) return;
|
||||
uint8_t* p = buf;
|
||||
*p++ = ms->svc_id;
|
||||
*p++ = ch_len; memcpy(p, ns, ch_len); p += ch_len;
|
||||
*p++ = 0x04; /* MSG_ITEM_UPDATE */
|
||||
memcpy(p, &key, 8); p += 8;
|
||||
@@ -637,18 +717,7 @@ void merkle_sync_push_update(struct UTUN_INSTANCE* inst, const char* ns,
|
||||
|
||||
for (struct ms_session* s = ms->sessions; s; s = s->next) {
|
||||
if (s->sess_state != SESS_SYNCED || strcmp(s->ns, ns) != 0) continue;
|
||||
struct ll_entry* entry = queue_entry_new(0);
|
||||
if (!entry) continue;
|
||||
uint8_t* dcopy = u_malloc(pkt);
|
||||
if (!dcopy) { queue_entry_free(entry); continue; }
|
||||
memcpy(dcopy, buf, pkt);
|
||||
entry->dgram = dcopy; entry->len = pkt;
|
||||
struct ETCP_CONN* conn = ms_find_conn_for_node(inst, s->peer);
|
||||
if (!conn) { u_free(dcopy); queue_entry_free(entry); continue; }
|
||||
int r = etcp_send(conn, entry);
|
||||
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: push_update sent type=%02x to=%016llx rc=%d",
|
||||
MS_ID, type, (unsigned long long)s->peer, r);
|
||||
if (r != 0) { u_free(dcopy); queue_entry_free(entry); }
|
||||
ms_outq_enqueue(s, buf, pkt);
|
||||
}
|
||||
u_free(buf);
|
||||
}
|
||||
@@ -663,10 +732,19 @@ void merkle_sync_broadcast(struct UTUN_INSTANCE* inst, const char* ns,
|
||||
MS_ID, ns, (unsigned long long)from_peer, len);
|
||||
for (struct ms_session* s = ms->sessions; s; s = s->next) {
|
||||
if (s->peer == from_peer || strcmp(s->ns, ns) != 0) continue;
|
||||
_send_broadcast_data(ms, s->peer, ns, data, len);
|
||||
_broadcast_to_session(ms, s, ns, data, len);
|
||||
}
|
||||
}
|
||||
|
||||
int merkle_sync_send_to(struct UTUN_INSTANCE* inst, const char* ns,
|
||||
uint64_t peer, const uint8_t* data, size_t len) {
|
||||
struct merkle_sync* ms = inst ? inst->msync : NULL;
|
||||
if (!ms || !ms->initialized || !ns || !data || len < 2) return -1;
|
||||
struct ms_session* s = _session_find(ms, peer, ns);
|
||||
if (!s) { DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: send_to — no session peer=%016llx ns=%s", MS_ID, (unsigned long long)peer, ns); return -1; }
|
||||
return _broadcast_to_session(ms, s, ns, data, len);
|
||||
}
|
||||
|
||||
/* ── Background consistency check ── */
|
||||
|
||||
int merkle_sync_bg_check(struct UTUN_INSTANCE* inst, const char* ns) {
|
||||
@@ -699,38 +777,18 @@ int merkle_sync_bg_check(struct UTUN_INSTANCE* inst, const char* ns) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void _bg_timer_cb(void* arg) {
|
||||
static void _wal_timer_cb(void* arg) {
|
||||
struct merkle_sync* ms = (struct merkle_sync*)arg;
|
||||
if (!ms || !ms->initialized || !ms->inst) return;
|
||||
sqlite3* db = _db(ms->inst); if (!db) return;
|
||||
sqlite3* db = _db(ms->inst); if (!db) { ms->wal_timer = NULL; return; }
|
||||
|
||||
static int tick = 0; tick++;
|
||||
int pn = 0, ckpt = 0;
|
||||
sqlite3_wal_checkpoint_v2(db, NULL, SQLITE_CHECKPOINT_PASSIVE, &pn, &ckpt);
|
||||
if (pn > 0 || ckpt > 0)
|
||||
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: WAL checkpoint done: %d frames in log, %d checkpointed", MS_ID, pn, ckpt);
|
||||
|
||||
sqlite3_stmt* cs = NULL;
|
||||
sqlite3_prepare_v2(db, "SELECT channel_id FROM channels", -1, &cs, NULL);
|
||||
if (cs) {
|
||||
int ch_idx = 0;
|
||||
while (sqlite3_step(cs) == SQLITE_ROW) {
|
||||
const char* ch_id = (const char*)sqlite3_column_text(cs, 0);
|
||||
if (!ch_id) { ch_idx++; continue; }
|
||||
if ((tick % (ch_idx + 1)) == 0) {
|
||||
int rc = merkle_sync_bg_check(ms->inst, ch_id);
|
||||
if (rc >= 0) DEBUG_TRACE(DEBUG_CATEGORY_MEMBER_SYNC, "%s: bg_check ns=%s → %s", MS_ID, ch_id, rc == 0 ? "consistent" : "RECALCULATED");
|
||||
}
|
||||
ch_idx++;
|
||||
}
|
||||
sqlite3_finalize(cs);
|
||||
|
||||
if (tick % 12 == 0) { /* every ~60s: try to checkpoint WAL, keep it under control */
|
||||
int pn = 0, ckpt = 0;
|
||||
sqlite3_wal_checkpoint_v2(db, NULL, SQLITE_CHECKPOINT_PASSIVE, &pn, &ckpt);
|
||||
if (pn > 0 || ckpt > 0)
|
||||
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: WAL checkpoint done: %d frames in log, %d checkpointed", MS_ID, pn, ckpt);
|
||||
}
|
||||
}
|
||||
|
||||
ms->bg_timer = uasync_set_timeout(ms->inst->ua, (uint32_t)(MS_BG_INTERVAL_MS * 10),
|
||||
ms, _bg_timer_cb, "ms_bg");
|
||||
ms->wal_timer = uasync_set_timeout(ms->inst->ua, (uint32_t)(MS_WAL_INTERVAL_MS * 10),
|
||||
ms, _wal_timer_cb, "ms_wal");
|
||||
}
|
||||
|
||||
/* ── Public lifecycle ── */
|
||||
@@ -749,8 +807,8 @@ int merkle_sync_init(struct UTUN_INSTANCE* inst, uint8_t svc_id,
|
||||
|
||||
etcp_bind(inst, svc_id, _recv_cb);
|
||||
|
||||
ms->bg_timer = uasync_set_timeout(inst->ua, (uint32_t)(MS_BG_INTERVAL_MS * 10),
|
||||
ms, _bg_timer_cb, "ms_bg");
|
||||
ms->wal_timer = uasync_set_timeout(inst->ua, (uint32_t)(MS_WAL_INTERVAL_MS * 10),
|
||||
ms, _wal_timer_cb, "ms_wal");
|
||||
|
||||
DEBUG_INFO(DEBUG_CATEGORY_MEMBER_SYNC, "%s: initialized svc=%02x", MS_ID, svc_id);
|
||||
return 0;
|
||||
@@ -764,10 +822,11 @@ void merkle_sync_destroy(struct UTUN_INSTANCE* inst) {
|
||||
|
||||
etcp_unbind(inst, ms->svc_id);
|
||||
|
||||
if (ms->bg_timer) { uasync_cancel_timeout(inst->ua, ms->bg_timer); ms->bg_timer = NULL; }
|
||||
if (ms->wal_timer) { uasync_cancel_timeout(inst->ua, ms->wal_timer); ms->wal_timer = NULL; }
|
||||
|
||||
struct ms_session* s = ms->sessions;
|
||||
while (s) { struct ms_session* next = s->next;
|
||||
ms_session_outq_free(s);
|
||||
u_free(s); s = next; }
|
||||
|
||||
u_free(ms);
|
||||
@@ -790,6 +849,7 @@ int merkle_sync_start(struct UTUN_INSTANCE* inst, uint64_t peer,
|
||||
s->peer = peer;
|
||||
s->next = ms->sessions;
|
||||
ms->sessions = s;
|
||||
ms_session_setup(ms, s);
|
||||
} else {
|
||||
s->pending_count = 0;
|
||||
DEBUG_DEBUG(DEBUG_CATEGORY_MEMBER_SYNC, "%s: start OVERWRITE peer=%016llx ns=%s", MS_ID, (unsigned long long)peer, ns);
|
||||
@@ -809,7 +869,9 @@ void merkle_sync_cancel(struct UTUN_INSTANCE* inst, uint64_t peer, const char* n
|
||||
while (*p) {
|
||||
struct ms_session* s = *p;
|
||||
if (s->peer == peer && strcmp(s->ns, ns) == 0) {
|
||||
*p = s->next; u_free(s);
|
||||
*p = s->next;
|
||||
ms_session_outq_free(s);
|
||||
u_free(s);
|
||||
return;
|
||||
}
|
||||
p = &(*p)->next;
|
||||
|
||||
+13
-1
@@ -222,8 +222,11 @@ void merkle_sync_cancel(struct UTUN_INSTANCE* inst, uint64_t peer, const char* n
|
||||
* Вызывается после изменения данных: merkle_sync сам не знает,
|
||||
* когда данные изменились — потребитель должен вызвать явно.
|
||||
* member_sync делает это внутри put/del.
|
||||
*
|
||||
* Возвращает 1 если хеш хотя бы одного бакета изменился (данные реально
|
||||
* изменились), 0 если всё совпало, -1 при ошибке.
|
||||
*/
|
||||
void merkle_sync_recompute_path(struct UTUN_INSTANCE* inst, const char* ns, uint64_t key);
|
||||
int merkle_sync_recompute_path(struct UTUN_INSTANCE* inst, const char* ns, uint64_t key);
|
||||
|
||||
/*
|
||||
* Отправить лёгкое обновление всем synced-пирам в namespace ns.
|
||||
@@ -247,6 +250,15 @@ void merkle_sync_push_update(struct UTUN_INSTANCE* inst, const char* ns,
|
||||
void merkle_sync_broadcast(struct UTUN_INSTANCE* inst, const char* ns,
|
||||
uint64_t from_peer, const uint8_t* data, size_t len);
|
||||
|
||||
/*
|
||||
* Отправить данные одного мембера конкретному пиру (а не всем).
|
||||
* Используется для "send back" при обнаружении stale-версии.
|
||||
* data, len — wire-формат [count:2][key:8][val:4]... как от get_items.
|
||||
* Возвращает 0 при успехе, -1 если нет сессии/соединения к пиру.
|
||||
*/
|
||||
int merkle_sync_send_to(struct UTUN_INSTANCE* inst, const char* ns,
|
||||
uint64_t peer, const uint8_t* data, size_t len);
|
||||
|
||||
/* ── Background consistency check ── */
|
||||
|
||||
/*
|
||||
|
||||
Reference in new issue
Block a user