Recover attachment indexes from history without losing message events

This commit is contained in:
evgeny committed 2026-10-27 02:18:30 +01:00
1 parent dbfda91e9e
commit c781fb368d
2 files changed
+100 -36

No files matched your search

+1 -1
View File
@@ -99,7 +99,7 @@ struct chat_core_ctx {
void* media_backfill_timer;/* was chat_msg.c g_media_startup_backfill_timer */
int media_backfill_ran; /* was chat_msg.c g_media_startup_backfill_ran */
struct UTUN_INSTANCE* media_backfill_inst; /* was chat_msg.c g_media_startup_backfill_inst */
void* media_channel_backfill_timer; /* debounce per-channel докачки */
void* media_channel_backfill_timer; /* ограниченный batch докачки/восстановления индекса через 2с от первой заявки */
char media_channel_backfill[16][64]; /* pending ch_id */
int media_channel_backfill_pending;
void* attachment_timer;
+99 -35
View File
@@ -29,6 +29,8 @@
#include <openssl/rand.h>
#include <stdlib.h>
static void media_queue_channel_backfill(struct UTUN_INSTANCE* inst, const char* channel);
chat_whisper_trigger_fn g_chat_whisper_trigger = NULL;
/* forward decl */
@@ -772,7 +774,8 @@ static void md_download_done_cb(void* arg, int err) {
struct chat_core_ctx* cc = CC(ctx->inst);
if (!err) {
if (!strcmp(ctx->content_type, ENCRYPTED_ATTACHMENT_TYPE)) {
chat_attachment_downloaded(ctx->inst, ctx->channel_id, ctx->message_id);
if (chat_attachment_downloaded(ctx->inst, ctx->channel_id, ctx->message_id))
media_queue_channel_backfill(ctx->inst, ctx->channel_id);
}
char attrs[1024];
snprintf(attrs, sizeof(attrs), "{\"st\":\"fl\",\"fp\":\"%s\"}", ctx->dest_relpath);
@@ -1016,14 +1019,17 @@ void on_msg_inserted(struct DB_SYNC_INSTANCE* si, const uint8_t message_id[32],
chat_event_post(inst, CHAT_EVT_MSG_UPDATED, evt, 1 + size + 32);
return;
}
if (chat_attachment_observe(inst, ch_id, message_id, author, data, len)) {
DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "encrypted media: invalid manifest ch=%s", ch_id); return;
int attachment_result = chat_attachment_observe(inst, ch_id, message_id, author, data, len);
if (attachment_result) {
DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "encrypted media: manifest indexing deferred/rejected ch=%s result=%d", ch_id, attachment_result);
if (attachment_result == -1) media_queue_channel_backfill(inst, ch_id);
}
chat_feedback_check_message(inst, ch_id, message_id);
uint8_t evt[73]; uint8_t cl = (uint8_t)strlen(ch_id); evt[0] = cl; memcpy(evt + 1, ch_id, cl);
memcpy(evt + 1 + cl, &author, 8);
evt[1 + cl + 8] = initial_sync != 0;
chat_event_post(inst, CHAT_EVT_MSG_RECEIVED, evt, 1 + cl + 9);
if (attachment_result) return; // История уже committed; ошибка производного индекса не отменяет её уведомление.
/* Медиа live-PUSH загружается сразу; записи из Merkle-страниц — через backfill после сверки истории. */
if (initial_sync && data && len > 0 && author != cc->my_node_id) {
@@ -1090,8 +1096,10 @@ void on_msg_inserted(struct DB_SYNC_INSTANCE* si, const uint8_t message_id[32],
/* ─── manual attachment download (called from JNI bridge via uasync post) ─── */
void chat_core_attachment_download(struct UTUN_INSTANCE* inst, const char* channel_id, int64_t msg_id) {
struct chat_core_ctx* cc = CC(inst);
if (!cc || !channel_id || msg_id <= 0) return;
struct chat_core_ctx* cc = inst ? CC(inst) : NULL;
if (!cc || !channel_id || msg_id <= 0 || !sqlite3_get_autocommit(cc->db)) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "manual media download: invalid state/arguments or external transaction"); return;
}
char tbl[80]; msg_table_name(channel_id, tbl, sizeof(tbl));
char sql[256];
snprintf(sql, sizeof(sql), "SELECT data, timestamp, author_signature, node_id FROM \"%s\" WHERE id=?", tbl);
@@ -1119,11 +1127,15 @@ void chat_core_attachment_download(struct UTUN_INSTANCE* inst, const char* chann
}
const char* ds = strstr((const char*)jdata, "\"d\":\"");
if (!ds) { sqlite3_finalize(st); return; }
if (!ds) { DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "manual media download: missing body ch=%s", channel_id); sqlite3_finalize(st); return; }
const char* d_start = ds + 5;
char body[4096]; size_t bi = 0;
while (*d_start && *d_start != '\"' && bi < sizeof(body) - 1) body[bi++] = *d_start++;
body[bi] = '\0';
const char* d_end = strchr(d_start, '"');
if (!d_end || d_end - d_start > 65536) {
DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "manual media download: invalid body length ch=%s", channel_id); sqlite3_finalize(st); return;
}
size_t bi = (size_t)(d_end - d_start); char* body = u_malloc(bi + 1);
if (!body) { DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "manual media download: body allocation failed bytes=%zu", bi); sqlite3_finalize(st); return; }
memcpy(body, d_start, bi); body[bi] = 0;
char content_type[32] = {0};
{
@@ -1140,20 +1152,40 @@ void chat_core_attachment_download(struct UTUN_INSTANCE* inst, const char* chann
extract_base_filename(body, base_filename, sizeof(base_filename));
uint8_t author_sig[64]; memcpy(author_sig, sig_blob, 64);
sqlite3_finalize(st);
uint8_t message[32]; int encrypted_attachment = !strcmp(content_type, ENCRYPTED_ATTACHMENT_TYPE);
if (encrypted_attachment) {
int observed = msg_read_id(cc, channel_id, msg_id, message) ? -1 :
chat_attachment_observe(inst, channel_id, message, (uint64_t)author_node_id, (const char*)jdata, (size_t)jlen);
if (observed) {
DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "manual encrypted download: cannot index manifest ch=%s result=%d", channel_id, observed);
if (observed == -1) media_queue_channel_backfill(inst, channel_id);
sqlite3_finalize(st); u_free(body); return;
}
}
int final_rc = sqlite3_finalize(st);
if (final_rc != SQLITE_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "manual media download: history finalize failed ch=%s rc=%d", channel_id, final_rc); u_free(body); return;
}
if (!strcmp(content_type, ENCRYPTED_ATTACHMENT_TYPE)) {
uint8_t message[32];
if (msg_read_id(cc, channel_id, msg_id, message)) return;
chat_attachment_want(inst, channel_id, message);
if (encrypted_attachment) {
if (chat_attachment_want(inst, channel_id, message)) { u_free(body); return; }
sqlite3_stmt* encrypted = NULL;
if (sqlite3_prepare_v2(cc->db, "SELECT cipher_path FROM attachment_keys WHERE channel=? AND message_id=? AND downloaded=1",
-1, &encrypted, NULL) == SQLITE_OK) {
sqlite3_bind_text(encrypted, 1, channel_id, -1, SQLITE_STATIC); sqlite3_bind_blob(encrypted, 2, message, 32, SQLITE_STATIC);
int present = sqlite3_step(encrypted) == SQLITE_ROW && ma_file_size((const char*)sqlite3_column_text(encrypted, 0)) > 0;
sqlite3_finalize(encrypted);
if (present) { chat_attachment_downloaded(inst, channel_id, message); return; }
} else DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "encrypted media: local file lookup failed: %s", sqlite3_errmsg(cc->db));
int rc = sqlite3_prepare_v2(cc->db, "SELECT cipher_path FROM attachment_keys WHERE channel=? AND message_id=? AND downloaded=1",
-1, &encrypted, NULL);
if (rc == SQLITE_OK) rc = sqlite3_bind_text(encrypted, 1, channel_id, -1, SQLITE_STATIC);
if (rc == SQLITE_OK) rc = sqlite3_bind_blob(encrypted, 2, message, 32, SQLITE_STATIC);
if (rc == SQLITE_OK) rc = sqlite3_step(encrypted);
const char* path = rc == SQLITE_ROW ? (const char*)sqlite3_column_text(encrypted, 0) : NULL;
int present = path && ma_file_size(path) > 0;
int finish = sqlite3_finalize(encrypted);
if ((rc != SQLITE_ROW && rc != SQLITE_DONE) || finish != SQLITE_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "encrypted media: local file lookup failed ch=%s step=%d finalize=%d db=%s",
channel_id, rc, finish, sqlite3_errmsg(cc->db)); u_free(body); return;
}
if (present) {
if (chat_attachment_downloaded(inst, channel_id, message)) media_queue_channel_backfill(inst, channel_id);
u_free(body); return;
}
}
/* сбрасываем пометку «файл есть»: файл мог быть вычищен кешем, а local_attrs ещё "fl".
@@ -1163,6 +1195,7 @@ void chat_core_attachment_download(struct UTUN_INSTANCE* inst, const char* chann
DEBUG_INFO(DEBUG_CATEGORY_GENERAL, "%s: manual download ch=%s id=%lld ct=%s file=%s",
CC_ID, channel_id, (long long)msg_id, content_type, base_filename);
md_start_download(inst, body, bi, channel_id, base_filename, (uint64_t)db_ts, author_sig, (uint64_t)author_node_id, msg_id, content_type);
u_free(body);
}
static void chat_core_attachment_download_trampoline_impl(void* arg) {
@@ -1296,7 +1329,7 @@ static void bf_collect(struct chat_core_ctx* cc, const char* media_base, const c
char sql[256];
snprintf(sql, sizeof(sql),
"SELECT id, timestamp, node_id, data, author_signature, local_attrs FROM \"%s\"", tbl);
"SELECT id, timestamp, node_id, data, author_signature, local_attrs, message_id FROM \"%s\"", tbl);
sqlite3_stmt* ms = NULL;
if (sqlite3_prepare_v2(cc->db, sql, -1, &ms, NULL) != SQLITE_OK) continue;
@@ -1333,6 +1366,21 @@ static void bf_collect(struct chat_core_ctx* cc, const char* media_base, const c
content_type[i] = '\0';
}
int encrypted_attachment = !strcmp(content_type, ENCRYPTED_ATTACHMENT_TYPE);
uint8_t message[32];
if (encrypted_attachment) {
if (sqlite3_column_bytes(ms, 6) != 32 || !sqlite3_column_blob(ms, 6)) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "media backfill: missing encrypted message identity ch=%s", ch_id);
media_index_result_free(&result); u_free(body); continue;
}
memcpy(message, sqlite3_column_blob(ms, 6), 32);
int observed = chat_attachment_observe(cc->inst, ch_id, message, author, jdata, (size_t)sqlite3_column_bytes(ms, 3));
if (observed) {
if (observed == -1) media_queue_channel_backfill(cc->inst, ch_id);
media_index_result_free(&result); u_free(body); continue;
}
}
char base_filename[256];
extract_base_filename(body, base_filename, sizeof(base_filename));
@@ -1346,6 +1394,8 @@ static void bf_collect(struct chat_core_ctx* cc, const char* media_base, const c
snprintf(full_path, sizeof(full_path), "%s/media/%s/%s", media_base, ch_id, final_name);
int64_t actual = ma_file_size(full_path);
if (actual < 0) actual = 0;
if (encrypted_attachment && actual > 0 && chat_attachment_downloaded(cc->inst, ch_id, message))
media_queue_channel_backfill(cc->inst, ch_id);
if (l->count == l->cap) {
int nc = l->cap ? l->cap * 2 : 64;
@@ -1830,26 +1880,40 @@ static void media_startup_backfill_done_cb(struct DB_SYNC_INSTANCE* si, uint64_t
static void media_channel_backfill_timer_cb(void* arg) {
struct UTUN_INSTANCE* inst = (struct UTUN_INSTANCE*)arg;
struct chat_core_ctx* cc = CC(inst);
if (!cc) return;
cc->media_channel_backfill_timer = NULL;
for (int i = 0; i < cc->media_channel_backfill_pending; i++)
chat_media_autodownload_backfill_channel(inst, cc->media_channel_backfill[i]);
char channels[16][64]; int count = cc->media_channel_backfill_pending;
memcpy(channels, cc->media_channel_backfill, (size_t)count * sizeof(channels[0]));
cc->media_channel_backfill_pending = 0;
for (int i = 0; i < count; i++) chat_media_autodownload_backfill_channel(inst, channels[i]);
}
/* Один ограниченный batch; отказ индекса во время scan ставит следующий timer, не меняя текущий batch. */
static void media_queue_channel_backfill(struct UTUN_INSTANCE* inst, const char* ch_id) {
struct chat_core_ctx* cc = CC(inst);
if (!cc || !cc->initialized || !cc->media_backfill_inst) return;
for (int i = 0; i < cc->media_channel_backfill_pending; i++)
if (strcmp(cc->media_channel_backfill[i], ch_id) == 0) goto schedule;
if (cc->media_channel_backfill_pending == (int)(sizeof(cc->media_channel_backfill) / sizeof(cc->media_channel_backfill[0]))) {
DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "media backfill: channel batch full, scheduling complete scan ch=%s", ch_id);
if (!cc->media_backfill_timer) cc->media_backfill_timer = uasync_set_timeout(inst->ua, 20000, inst,
media_startup_backfill_timer_cb, "chat_media_bl");
if (!cc->media_backfill_timer) DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "media backfill: complete scan timer allocation failed");
return;
}
snprintf(cc->media_channel_backfill[cc->media_channel_backfill_pending++], sizeof(cc->media_channel_backfill[0]), "%s", ch_id);
schedule:
if (!cc->media_channel_backfill_timer) cc->media_channel_backfill_timer = uasync_set_timeout(inst->ua, 20000, inst,
media_channel_backfill_timer_cb, "chat_md_chbl");
if (!cc->media_channel_backfill_timer) DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "media backfill: channel timer allocation failed ch=%s", ch_id);
else DEBUG_DEBUG(DEBUG_CATEGORY_MEDIA, "media backfill: channel queued ch=%s pending=%d", ch_id, cc->media_channel_backfill_pending);
}
static void media_channel_backfill_done_cb(struct DB_SYNC_INSTANCE* si, uint64_t peer_node_id, void* arg) {
struct UTUN_INSTANCE* inst = (struct UTUN_INSTANCE*)arg;
struct chat_core_ctx* cc = CC(inst);
(void)peer_node_id;
if (!cc || !si) return;
if (!si) return;
char ch_id[32]; snprintf(ch_id, sizeof(ch_id), "%llu", (unsigned long long)db_sync_instance_group_id(si));
for (int i = 0; i < cc->media_channel_backfill_pending; i++)
if (strcmp(cc->media_channel_backfill[i], ch_id) == 0) return;
if (cc->media_channel_backfill_pending < (int)(sizeof(cc->media_channel_backfill) / sizeof(cc->media_channel_backfill[0])))
snprintf(cc->media_channel_backfill[cc->media_channel_backfill_pending++],
sizeof(cc->media_channel_backfill[0]), "%s", ch_id);
if (cc->media_channel_backfill_timer) uasync_cancel_timeout(inst->ua, cc->media_channel_backfill_timer);
cc->media_channel_backfill_timer = uasync_set_timeout(inst->ua, 20000, inst,
media_channel_backfill_timer_cb, "chat_md_chbl");
media_queue_channel_backfill(arg, ch_id);
}
void chat_media_startup_backfill_init(struct UTUN_INSTANCE* inst) {