jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

dm: прямой обмен сообщениями (DM) + soundtouch + архитектура звонков

Browse Source 
- src/dm/: dm_core/dm_crypto/dm_mailbox — прямой обмен сообщениями (DM)
- lib/soundtouch-master/: vendored SoundTouch (изменение темпа/высоты аудио)
- doc/dm_arch.md, doc/chat_call_arch.md: архитектура DM и звонков
- tests: test_dm.c, test_dm_e2e.c
- chat: рефакторинг chat_* модулей, member_sync, invite_build
- chatgui/chatgui-android: синхронизация с новым API чата
- lib/u_async, debug_config, utun_instance: сопутствующие изменения
This commit is contained in:
evgeny committed 2026-09-15 21:09:17 +03:00
1 parent 761f64cf76
commit 73fe99b33f
123 files changed
+14095 -1270
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/dm/dm_mailbox.c
+353
View File
@@ -0,0 +1,353 @@
/*
* dm_mailbox.c — offline-хранение DM-сообщений на storage-узлах
*
* Storage-узел хранит зашифрованные блобы в таблице dm_mail и отдаёт их
* получателю по PULL, удаляя после ACK. Отправитель кладёт блобы по PUT.
* Содержимое для storage-узла непрозрачно (E2E-шифрование + подпись автора).
*/
#include "dm_mailbox.h"
#include "dm_core.h"
#include "../chat/chat_core.h"
#include "../utun_instance.h"
#include "../ntp_time.h"
#include "../routing_layer/etcp_router.h"
#include "../routing_layer/topo_node_sqlite.h"
#include "../transport_layer/etcp_api.h"
#include "../transport_layer/etcp.h"
#include "../../lib/mem.h"
#include "../../lib/ll_queue.h"
#include "../../lib/debug_config.h"
#include "../../lib/json_flat.h"
#include "../../lib/platform_compat.h"
#include <string.h>
#include <stdlib.h>
#include <sqlite3.h>
#define MB_ID "dm_mailbox"
/* Подкоманды протокола mailbox (первый байт payload после ROUTER_SVC-заголовка). */
#define MB_SUBCMD_PUT 0x01 /* отправитель → storage: [recipient:8][sender:8][msg] */
#define MB_SUBCMD_PULL 0x02 /* получатель → storage: [recipient:8][sender:8][since_seq:8] */
#define MB_SUBCMD_PULL_RESP 0x03 /* storage → получатель: [recipient:8][count:2][msg]* */
#define MB_SUBCMD_ACK 0x04 /* получатель → storage: [recipient:8][sender:8][up_to_seq:8] */
#define MB_TTL_DAYS 7
#define MB_MAX_BATCH 32
/* Per-instance состояние модуля. */
struct dm_mb_state {
struct UTUN_INSTANCE* inst;
sqlite3* db;
int initialized;
dm_mail_deliver_fn deliver_fn;
void* deliver_arg;
};
static struct dm_mb_state* mb_of(struct UTUN_INSTANCE* inst) {
return inst ? inst->dm_mailbox : NULL;
}
void dm_mailbox_set_deliver_cb(struct UTUN_INSTANCE* inst, dm_mail_deliver_fn fn, void* arg) {
struct dm_mb_state* mb = mb_of(inst);
if (!mb) return;
mb->deliver_fn = fn;
mb->deliver_arg = arg;
}
/* Выполнить SQL без результата. Возвращает 0/ошибку. */
static int mb_exec(struct dm_mb_state* mb, const char* sql) {
char* err = NULL;
if (sqlite3_exec(mb->db, sql, NULL, NULL, &err) != SQLITE_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_DM, "%s: sql failed: %s (%s)", MB_ID, sql, err ? err : "?");
if (err) sqlite3_free(err);
return -1;
}
return 0;
}
/* Прочитать seq из канонического тела сообщения (смещение 8). */
static uint64_t mb_msg_seq(const uint8_t* msg) {
uint64_t seq; memcpy(&seq, msg + 8, 8); return seq;
}
/* Создать таблицу dm_mail (на любом узле; фактически используется storage-узлами). */
static void mb_create_tables(struct dm_mb_state* mb) {
mb_exec(mb, "CREATE TABLE IF NOT EXISTS dm_mail ("
" recipient INTEGER NOT NULL,"
" sender INTEGER NOT NULL,"
" seq INTEGER NOT NULL,"
" msg BLOB NOT NULL,"
" ttl INTEGER NOT NULL,"
" PRIMARY KEY (recipient, sender, seq))");
}
/* Отправить payload подкомандой subcmd в dst через etcp_router (mode=0, своё E2E). */
static int mb_route_send(struct dm_mb_state* mb, uint64_t group_id, uint64_t dst, uint8_t subcmd,
const uint8_t* body, size_t body_len) {
struct ll_entry* entry = queue_entry_new(0);
if (!entry) return -1;
entry->dgram = u_malloc(1 + 1 + body_len);
if (!entry->dgram) { queue_entry_free(entry); return -1; }
entry->dgram[0] = ETCP_RT_ID_DM_MAILBOX;
memcpy(entry->dgram + 1, &subcmd, 1);
if (body_len) memcpy(entry->dgram + 1 + 1, body, body_len);
entry->len = (uint16_t)(1 + 1 + body_len);
int rc = etcp_route_send(mb->inst, group_id, dst, entry, 1, 0);
if (rc != 0) { queue_dgram_free(entry); queue_entry_free(entry); return -1; }
return 0;
}
/* ── обработка входящих mailbox-пакетов (etcp_router 0x35) ── */
static void mb_recv_cb(struct ETCP_CONN* conn, struct ll_entry* entry) {
if (!entry || !conn || !entry->dgram || entry->len < ROUTER_SVC_PAYLOAD_OFF + 1) {
if (entry) { queue_dgram_free(entry); queue_entry_free(entry); }
return;
}
struct UTUN_INSTANCE* inst = conn->instance;
struct dm_mb_state* mb = mb_of(inst);
if (!inst || !mb || !mb->db) { queue_dgram_free(entry); queue_entry_free(entry); return; }
const uint8_t* d = entry->dgram;
uint64_t group_id; memcpy(&group_id, d + ROUTER_SVC_GROUP_OFF, 8);
const uint8_t* p = d + ROUTER_SVC_PAYLOAD_OFF;
size_t plen = entry->len - ROUTER_SVC_PAYLOAD_OFF;
uint8_t subcmd = p[0];
if (subcmd == MB_SUBCMD_PUT) {
/* [subcmd][recipient:8][sender:8][msg] — storage: сохранить блоб */
if (plen < 1 + 8 + 8 + 16) { queue_dgram_free(entry); queue_entry_free(entry); return; }
uint64_t recipient; memcpy(&recipient, p + 1, 8);
uint64_t sender; memcpy(&sender, p + 9, 8);
const uint8_t* msg = p + 17;
size_t mlen = plen - 17;
uint64_t seq = mb_msg_seq(msg);
int64_t ttl = (int64_t)ntp_time_get_seconds(inst) + MB_TTL_DAYS * 86400;
sqlite3_stmt* st = NULL;
if (sqlite3_prepare_v2(mb->db,
"INSERT OR IGNORE INTO dm_mail(recipient,sender,seq,msg,ttl) VALUES(?,?,?,?,?)",
-1, &st, NULL) == SQLITE_OK) {
sqlite3_bind_int64(st, 1, (sqlite3_int64)recipient);
sqlite3_bind_int64(st, 2, (sqlite3_int64)sender);
sqlite3_bind_int64(st, 3, (sqlite3_int64)seq);
sqlite3_bind_blob(st, 4, msg, (int)mlen, SQLITE_STATIC);
sqlite3_bind_int64(st, 5, ttl);
int rc = sqlite3_step(st);
DEBUG_INFO(DEBUG_CATEGORY_DM, "%s: PUT recipient=0x%016llx sender=0x%016llx seq=%llu rc=%d",
MB_ID, (unsigned long long)recipient, (unsigned long long)sender,
(unsigned long long)seq, rc);
}
if (st) sqlite3_finalize(st);
} else if (subcmd == MB_SUBCMD_PULL) {
/* [subcmd][recipient:8][sender:8][since_seq:8] — storage: вернуть очередь */
if (plen < 1 + 8 + 8 + 8) { queue_dgram_free(entry); queue_entry_free(entry); return; }
uint64_t recipient; memcpy(&recipient, p + 1, 8);
uint64_t sender; memcpy(&sender, p + 9, 8);
uint64_t since_seq; memcpy(&since_seq, p + 17, 8);
/* прочитать блобы (до MB_MAX_BATCH) в буфер: [recipient:8][sender:8][count:2][msg]* */
size_t buf_cap = 18 + (size_t)MB_MAX_BATCH * 4096;
uint8_t* body = u_malloc(buf_cap);
if (!body) { queue_dgram_free(entry); queue_entry_free(entry); return; }
memcpy(body, &recipient, 8);
memcpy(body + 8, &sender, 8);
uint16_t count = 0;
size_t boff = 18;
sqlite3_stmt* st = NULL;
if (sqlite3_prepare_v2(mb->db,
"SELECT msg, length(msg) FROM dm_mail WHERE recipient=? AND sender=? AND seq>? ORDER BY seq LIMIT ?",
-1, &st, NULL) == SQLITE_OK) {
sqlite3_bind_int64(st, 1, (sqlite3_int64)recipient);
sqlite3_bind_int64(st, 2, (sqlite3_int64)sender);
sqlite3_bind_int64(st, 3, (sqlite3_int64)since_seq);
sqlite3_bind_int(st, 4, MB_MAX_BATCH);
while (sqlite3_step(st) == SQLITE_ROW) {
const uint8_t* msg = (const uint8_t*)sqlite3_column_blob(st, 0);
int mlen = sqlite3_column_int(st, 1);
if (!msg || mlen <= 0) continue;
if (boff + (size_t)mlen > buf_cap) break;
memcpy(body + boff, msg, (size_t)mlen);
boff += (size_t)mlen;
count++;
}
}
if (st) sqlite3_finalize(st);
memcpy(body + 16, &count, 2);
DEBUG_INFO(DEBUG_CATEGORY_DM, "%s: PULL recipient=0x%016llx sender=0x%016llx since=%llu → %u msgs",
MB_ID, (unsigned long long)recipient, (unsigned long long)sender,
(unsigned long long)since_seq, count);
if (count > 0)
mb_route_send(mb, group_id, recipient, MB_SUBCMD_PULL_RESP, body, boff);
u_free(body);
} else if (subcmd == MB_SUBCMD_PULL_RESP) {
/* [subcmd][recipient:8][sender:8][count:2][msg]* — получатель: передать в dm_core + ACK storage */
if (plen < 1 + 8 + 8 + 2) { queue_dgram_free(entry); queue_entry_free(entry); return; }
uint64_t storage; memcpy(&storage, d + ROUTER_SVC_SRC_OFF, 8);
uint64_t recipient; memcpy(&recipient, p + 1, 8);
uint64_t sender; memcpy(&sender, p + 9, 8);
uint16_t count; memcpy(&count, p + 17, 2);
const uint8_t* m = p + 19;
size_t remaining = plen - 19;
uint64_t max_seq = 0;
for (int i = 0; i < count; i++) {
size_t mlen = 0;
if (!dm_msg_len(m, remaining, &mlen)) break;
uint64_t seq = mb_msg_seq(m);
if (seq > max_seq) max_seq = seq;
if (mb->deliver_fn) mb->deliver_fn(m, mlen, mb->deliver_arg);
m += mlen; remaining -= mlen;
}
/* подтвердить storage-узлу приём — удалить доставленное (recipient/sender/up_to_seq) */
if (count > 0 && max_seq > 0) {
uint8_t ack[24];
memcpy(ack, &recipient, 8);
memcpy(ack + 8, &sender, 8);
memcpy(ack + 16, &max_seq, 8);
mb_route_send(mb, group_id, storage, MB_SUBCMD_ACK, ack, sizeof(ack));
DEBUG_INFO(DEBUG_CATEGORY_DM, "%s: PULL_RESP recipient=0x%016llx sender=0x%016llx count=%u → ACK up_to=%llu",
MB_ID, (unsigned long long)recipient, (unsigned long long)sender, count,
(unsigned long long)max_seq);
}
} else if (subcmd == MB_SUBCMD_ACK) {
/* [subcmd][recipient:8][sender:8][up_to_seq:8] — storage: удалить доставленное */
if (plen < 1 + 8 + 8 + 8) { queue_dgram_free(entry); queue_entry_free(entry); return; }
uint64_t recipient; memcpy(&recipient, p + 1, 8);
uint64_t sender; memcpy(&sender, p + 9, 8);
uint64_t up_to_seq; memcpy(&up_to_seq, p + 17, 8);
sqlite3_stmt* st = NULL;
if (sqlite3_prepare_v2(mb->db,
"DELETE FROM dm_mail WHERE recipient=? AND sender=? AND seq<=?",
-1, &st, NULL) == SQLITE_OK) {
sqlite3_bind_int64(st, 1, (sqlite3_int64)recipient);
sqlite3_bind_int64(st, 2, (sqlite3_int64)sender);
sqlite3_bind_int64(st, 3, (sqlite3_int64)up_to_seq);
sqlite3_step(st);
}
if (st) sqlite3_finalize(st);
DEBUG_INFO(DEBUG_CATEGORY_DM, "%s: ACK recipient=0x%016llx sender=0x%016llx up_to=%llu",
MB_ID, (unsigned long long)recipient, (unsigned long long)sender,
(unsigned long long)up_to_seq);
} else {
DEBUG_WARN(DEBUG_CATEGORY_DM, "%s: unknown subcmd=%02x", MB_ID, subcmd);
}
queue_dgram_free(entry); queue_entry_free(entry);
}
/* ── Жизненный цикл ── */
int dm_mailbox_init(struct UTUN_INSTANCE* inst) {
if (!inst) return -1;
if (inst->dm_mailbox && inst->dm_mailbox->initialized) return 0;
struct dm_mb_state* mb = u_calloc(1, sizeof(*mb));
if (!mb) return -1;
inst->dm_mailbox = mb;
mb->inst = inst;
mb->db = chat_core_get_db(inst);
if (!mb->db) { DEBUG_ERROR(DEBUG_CATEGORY_DM, "%s: no chat db", MB_ID); u_free(mb); inst->dm_mailbox = NULL; return -1; }
mb_create_tables(mb);
if (etcp_router_bind(inst, ETCP_RT_ID_DM_MAILBOX, mb_recv_cb) != 0) {
DEBUG_ERROR(DEBUG_CATEGORY_DM, "%s: etcp_router_bind failed", MB_ID);
u_free(mb); inst->dm_mailbox = NULL; return -1;
}
mb->initialized = 1;
DEBUG_INFO(DEBUG_CATEGORY_DM, "%s: initialized", MB_ID);
return 0;
}
void dm_mailbox_destroy(struct UTUN_INSTANCE* inst) {
struct dm_mb_state* mb = mb_of(inst);
if (!inst || !mb || !mb->initialized) return;
etcp_router_unbind(inst, ETCP_RT_ID_DM_MAILBOX);
u_free(mb);
inst->dm_mailbox = NULL;
DEBUG_INFO(DEBUG_CATEGORY_DM, "%s: destroyed", MB_ID);
}
/* ── Публичные операции ── */
/* Собрать storage-узлы (dm_storage=yes) из общих групп без дублей. */
int dm_mailbox_get_storage_nodes(struct UTUN_INSTANCE* inst, uint64_t* out, int max) {
struct dm_mb_state* mb = mb_of(inst);
if (!inst || !out || max <= 0 || !mb || !mb->db) return 0;
uint64_t my_id = inst->node_id;
uint64_t* ch_ids = NULL;
int ch_count = 0;
if (topo_node_sqlite_get_member_channels(mb->db, my_id, &ch_ids, &ch_count) != 0) return 0;
int found = 0;
char buf[256];
for (int c = 0; c < ch_count && found < max; c++) {
char ch[64]; snprintf(ch, sizeof(ch), "%llu", (unsigned long long)ch_ids[c]);
char tbl[80]; snprintf(tbl, sizeof(tbl), "peers_%s", ch);
char sql[256]; snprintf(sql, sizeof(sql),
"SELECT node_id, COALESCE(adm_tags,'') FROM \"%s\"", tbl);
sqlite3_stmt* st = NULL;
if (sqlite3_prepare_v2(mb->db, sql, -1, &st, NULL) != SQLITE_OK) continue;
while (sqlite3_step(st) == SQLITE_ROW && found < max) {
uint64_t nid = (uint64_t)sqlite3_column_int64(st, 0);
if (nid == my_id) continue;
const char* tags = (const char*)sqlite3_column_text(st, 1);
if (tags && json_flat_get(tags, "dm_storage", buf, sizeof(buf)) == 0
&& strcmp(buf, "yes") == 0) {
int dup = 0;
for (int i = 0; i < found; i++) if (out[i] == nid) { dup = 1; break; }
if (!dup) out[found++] = nid;
}
}
sqlite3_finalize(st);
}
if (ch_ids) u_free(ch_ids);
return found;
}
/* Отправитель: разослать зашифрованное сообщение на все storage-узлы. */
int dm_mailbox_put(struct UTUN_INSTANCE* inst, uint64_t recipient, uint64_t sender,
const uint8_t* msg, size_t msg_len) {
struct dm_mb_state* mb = mb_of(inst);
if (!inst || !mb || !msg || msg_len == 0) return -1;
uint64_t nodes[8];
int n = dm_mailbox_get_storage_nodes(inst, nodes, 8);
if (n == 0) {
DEBUG_WARN(DEBUG_CATEGORY_DM, "%s: no storage nodes for offline delivery", MB_ID);
return -1;
}
/* body = [recipient:8][sender:8][msg] */
size_t body_len = 8 + 8 + msg_len;
uint8_t* body = u_malloc(body_len);
if (!body) return -1;
memcpy(body, &recipient, 8);
memcpy(body + 8, &sender, 8);
memcpy(body + 16, msg, msg_len);
int sent = 0;
for (int i = 0; i < n; i++) {
if (mb_route_send(mb, 0, nodes[i], MB_SUBCMD_PUT, body, body_len) == 0) sent++;
}
u_free(body);
DEBUG_INFO(DEBUG_CATEGORY_DM, "%s: put → %d/%d storage nodes", MB_ID, sent, n);
return sent > 0 ? 0 : -1;
}
/* Получатель: запросить почту от отправителя со всех storage-узлов. */
void dm_mailbox_pull(struct UTUN_INSTANCE* inst, uint64_t peer_node_id, uint64_t since_seq) {
struct dm_mb_state* mb = mb_of(inst);
if (!inst || !mb) return;
uint64_t nodes[8];
int n = dm_mailbox_get_storage_nodes(inst, nodes, 8);
if (n == 0) return;
uint8_t body[24];
memcpy(body, &inst->node_id, 8);
memcpy(body + 8, &peer_node_id, 8);
memcpy(body + 16, &since_seq, 8);
for (int i = 0; i < n; i++)
mb_route_send(mb, 0, nodes[i], MB_SUBCMD_PULL, body, sizeof(body));
DEBUG_INFO(DEBUG_CATEGORY_DM, "%s: pull from %d storage nodes (peer=0x%016llx since=%llu)",
MB_ID, n, (unsigned long long)peer_node_id, (unsigned long long)since_seq);
}