Commit attachment retry reservations before network delivery
This commit is contained in:
1 parent
5a6a209436
commit
74fccbc5da
2 files changed
+122
-29
No files matched your search
+72
-23
@@ -37,21 +37,16 @@ struct ak_file {
|
||||
/* Все ошибки SQLite содержат конкретную операцию. */
|
||||
static sqlite3_stmt* ak_prepare(struct UTUN_INSTANCE* inst, const char* sql) {
|
||||
sqlite3_stmt* st = NULL;
|
||||
if (!CC(inst) || sqlite3_prepare_v2(CC(inst)->db, sql, -1, &st, NULL) != SQLITE_OK)
|
||||
if (!inst || !CC(inst) || sqlite3_prepare_v2(CC(inst)->db, sql, -1, &st, NULL) != SQLITE_OK) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "attachment keys: prepare failed sql=%s error=%s", sql,
|
||||
CC(inst) ? sqlite3_errmsg(CC(inst)->db) : "stopped");
|
||||
inst && CC(inst) ? sqlite3_errmsg(CC(inst)->db) : "stopped");
|
||||
int rc = sqlite3_finalize(st);
|
||||
if (rc != SQLITE_OK) DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "attachment keys: prepare cleanup finalize rc=%d", rc);
|
||||
return NULL;
|
||||
}
|
||||
return st;
|
||||
}
|
||||
|
||||
/* Statement всегда освобождается. */
|
||||
static int ak_step(struct UTUN_INSTANCE* inst, sqlite3_stmt* st) {
|
||||
if (!st) return -1;
|
||||
int rc = sqlite3_step(st);
|
||||
if (rc != SQLITE_DONE) DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "attachment keys: write failed rc=%d: %s", rc, sqlite3_errmsg(CC(inst)->db));
|
||||
sqlite3_finalize(st);
|
||||
return rc == SQLITE_DONE ? 0 : -1;
|
||||
}
|
||||
|
||||
/* Управление транзакциями без открытых statement. */
|
||||
static int ak_exec(struct UTUN_INSTANCE* inst, const char* sql) {
|
||||
int rc = sqlite3_exec(CC(inst)->db, sql, NULL, NULL, NULL);
|
||||
@@ -1053,21 +1048,72 @@ fail:
|
||||
ak_rollback(inst, "downloaded", channel); return -1;
|
||||
}
|
||||
|
||||
/* Выбрать команду и зафиксировать следующую попытку до сети; 1 — команда, 0 — нет, -1 — ошибка. */
|
||||
static int ak_next_command(struct UTUN_INSTANCE* inst, char channel[32], uint8_t message[32], uint64_t* peer, int* cmd) {
|
||||
sqlite3* db = CC(inst)->db; sqlite3_stmt* st = NULL;
|
||||
if (!sqlite3_get_autocommit(db)) {
|
||||
DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "attachment keys: outbox postponed inside external transaction"); return -1;
|
||||
}
|
||||
const char* stage = "begin outbox";
|
||||
int rc = sqlite3_exec(db, "BEGIN IMMEDIATE /* attachment outbox */", NULL, NULL, NULL);
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
stage = "prepare outbox";
|
||||
rc = sqlite3_prepare_v2(db, "SELECT channel,message_id,peer,command FROM attachment_key_outbox"
|
||||
" WHERE retry_at<=unixepoch() ORDER BY retry_at,rowid LIMIT 1", -1, &st, NULL);
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
stage = "read outbox"; rc = sqlite3_step(st);
|
||||
if (rc != SQLITE_ROW && rc != SQLITE_DONE) goto fail;
|
||||
int found = rc == SQLITE_ROW;
|
||||
if (found) {
|
||||
const char* name = (const char*)sqlite3_column_text(st, 0);
|
||||
if (!ak_channel(name) || sqlite3_column_bytes(st, 1) != 32 || !sqlite3_column_blob(st, 1)) {
|
||||
stage = "outbox identity"; rc = SQLITE_MISMATCH; goto fail;
|
||||
}
|
||||
snprintf(channel, 32, "%s", name); memcpy(message, sqlite3_column_blob(st, 1), 32);
|
||||
*peer = (uint64_t)sqlite3_column_int64(st, 2); *cmd = sqlite3_column_int(st, 3);
|
||||
if (!*peer || *peer == inst->node_id || (*cmd != AK_REQUEST && *cmd != AK_GRANT && *cmd != AK_DENY)) {
|
||||
stage = "outbox peer/command"; rc = SQLITE_MISMATCH; goto fail;
|
||||
}
|
||||
}
|
||||
stage = "finalize outbox"; rc = sqlite3_finalize(st); st = NULL;
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
if (found) {
|
||||
stage = "prepare retry";
|
||||
rc = sqlite3_prepare_v2(db, "UPDATE attachment_key_outbox SET retry_at=unixepoch()+5 WHERE channel=? AND message_id=? AND peer=?",
|
||||
-1, &st, NULL);
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
stage = "bind channel"; rc = sqlite3_bind_text(st, 1, channel, -1, SQLITE_STATIC);
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
stage = "bind message"; rc = sqlite3_bind_blob(st, 2, message, 32, SQLITE_STATIC);
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
stage = "bind peer"; rc = sqlite3_bind_int64(st, 3, (sqlite3_int64)*peer);
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
stage = "write retry"; rc = sqlite3_step(st);
|
||||
if (rc != SQLITE_DONE) goto fail;
|
||||
int changed = sqlite3_changes(db);
|
||||
stage = "finalize retry"; rc = sqlite3_finalize(st); st = NULL;
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
if (changed != 1) { stage = "row count"; rc = SQLITE_CONSTRAINT; goto fail; }
|
||||
}
|
||||
stage = "commit outbox"; rc = sqlite3_exec(db, "COMMIT /* attachment outbox */", NULL, NULL, NULL);
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
if (found) DEBUG_DEBUG(DEBUG_CATEGORY_MEDIA, "attachment keys: retry committed channel=%s peer=%llu command=%d",
|
||||
channel, (unsigned long long)*peer, *cmd);
|
||||
return found;
|
||||
fail:
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "attachment keys: outbox channel=%s stage=%s rc=%d (%s) db=%s",
|
||||
channel, stage, rc, sqlite3_errstr(rc), sqlite3_errmsg(db));
|
||||
int final_rc = sqlite3_finalize(st);
|
||||
if (final_rc != SQLITE_OK) DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "attachment keys: outbox cleanup finalize rc=%d", final_rc);
|
||||
ak_rollback(inst, "outbox", channel); return -1;
|
||||
}
|
||||
|
||||
/* Один проход ограничен; retry_at даёт всем задачам очередь без голодания. */
|
||||
static void ak_pump(struct UTUN_INSTANCE* inst) {
|
||||
for (unsigned i = 0; i < 16; i++) {
|
||||
sqlite3_stmt* st = ak_prepare(inst, "SELECT channel,message_id,peer,command FROM attachment_key_outbox"
|
||||
" WHERE retry_at<=unixepoch() ORDER BY retry_at,rowid LIMIT 1");
|
||||
if (!st) return;
|
||||
int rc = sqlite3_step(st);
|
||||
if (rc != SQLITE_ROW) { sqlite3_finalize(st); if (rc != SQLITE_DONE) DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "attachment keys: outbox read failed rc=%d", rc); return; }
|
||||
char channel[32]; uint8_t message[32], wire[AK_WIRE];
|
||||
snprintf(channel, sizeof(channel), "%s", sqlite3_column_text(st, 0)); memcpy(message, sqlite3_column_blob(st, 1), 32);
|
||||
uint64_t peer = (uint64_t)sqlite3_column_int64(st, 2); int cmd = sqlite3_column_int(st, 3); sqlite3_finalize(st);
|
||||
st = ak_prepare(inst, "UPDATE attachment_key_outbox SET retry_at=unixepoch()+5 WHERE channel=? AND message_id=? AND peer=?");
|
||||
if (!st) return;
|
||||
sqlite3_bind_text(st, 1, channel, -1, SQLITE_STATIC); sqlite3_bind_blob(st, 2, message, 32, SQLITE_STATIC);
|
||||
sqlite3_bind_int64(st, 3, (sqlite3_int64)peer); if (ak_step(inst, st)) return;
|
||||
char channel[32] = {0}; uint8_t message[32], wire[AK_WIRE]; uint64_t peer = 0; int cmd = 0;
|
||||
int rc = ak_next_command(inst, channel, message, &peer, &cmd);
|
||||
if (rc <= 0) return;
|
||||
if (cmd == AK_GRANT && !ak_authorized(inst, channel, message, peer)) continue;
|
||||
int n = ak_body(inst, channel, message, (uint8_t)cmd, wire);
|
||||
if (n > 0) {
|
||||
@@ -1145,6 +1191,9 @@ fail:
|
||||
static int ak_claim_decryption(struct ak_file* job) {
|
||||
struct UTUN_INSTANCE* inst = job->inst;
|
||||
sqlite3* db = CC(inst)->db; sqlite3_stmt* st = NULL;
|
||||
if (!sqlite3_get_autocommit(db)) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "attachment keys: claim rejected inside external transaction channel=%s", job->channel); return -1;
|
||||
}
|
||||
const char* stage = "begin claim";
|
||||
int rc = sqlite3_exec(db, "BEGIN IMMEDIATE", NULL, NULL, NULL);
|
||||
if (rc != SQLITE_OK) goto fail;
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
#include "../lib/debug_config.h"
|
||||
#include "../lib/mem.h"
|
||||
|
||||
enum operation { NO_OPERATION, IDENTITY, MEMBER, AUTHOR, READ, INSERT, OUTBOX, PENDING, ELIGIBLE };
|
||||
enum operation { NO_OPERATION, IDENTITY, MEMBER, AUTHOR, READ, INSERT, OUTBOX, PENDING, ELIGIBLE, PUMP_READ, PUMP_RETRY };
|
||||
enum failure { NO_FAILURE, BEGIN, PREPARE, BIND, STEP, FINALIZE, CHANGES, COMMIT };
|
||||
static enum failure fault;
|
||||
static enum operation fault_operation, last_operation;
|
||||
@@ -41,7 +41,7 @@ struct fixture {
|
||||
uint64_t peer;
|
||||
struct snapshot expected;
|
||||
int64_t started;
|
||||
int allow_result, route_up, room, send_rc, corrupt, flags, outgoing;
|
||||
int allow_result, route_up, room, send_rc, corrupt, flags, outgoing, pump_only;
|
||||
unsigned acks, events;
|
||||
};
|
||||
static struct fixture* active;
|
||||
@@ -60,8 +60,9 @@ static enum operation operation(const char* query) {
|
||||
"SELECT x25519_pubkey,ed25519_pubkey,signed_by,signature,join_sig,join_ts", "SELECT 1 FROM msg_",
|
||||
"SELECT incoming,state FROM attachment_key_requests WHERE", "INSERT INTO attachment_key_requests(",
|
||||
"INSERT INTO attachment_key_outbox(", "SELECT channel,message_id,crypto_id,author,key,cipher_hash,cipher_path,plain_path",
|
||||
"SELECT 1 FROM attachment_keys a WHERE"};
|
||||
for (enum operation op = IDENTITY; op <= ELIGIBLE; op++)
|
||||
"SELECT 1 FROM attachment_keys a WHERE", "SELECT channel,message_id,peer,command FROM attachment_key_outbox",
|
||||
"UPDATE attachment_key_outbox SET retry_at=unixepoch()+5 WHERE"};
|
||||
for (enum operation op = IDENTITY; op <= PUMP_RETRY; op++)
|
||||
if (!strncmp(query, prefixes[op], strlen(prefixes[op]))) return op;
|
||||
return NO_OPERATION;
|
||||
}
|
||||
@@ -75,8 +76,10 @@ static int take(enum failure point, enum operation op, int index) {
|
||||
extern int __real_sqlite3_exec(sqlite3*, const char*, int (*)(void*, int, char**, char**), void*, char**);
|
||||
int __wrap_sqlite3_exec(sqlite3* db, const char* query, int (*cb)(void*, int, char**, char**), void* arg, char** error) {
|
||||
if (active && db == active->inst->topo_sqlite_db) {
|
||||
if (!strcmp(query, "BEGIN IMMEDIATE") && take(BEGIN, NO_OPERATION, 0)) return SQLITE_BUSY;
|
||||
if (!strcmp(query, "COMMIT") && take(COMMIT, NO_OPERATION, 0)) return SQLITE_BUSY;
|
||||
const char* begin = active->pump_only ? "BEGIN IMMEDIATE /* attachment outbox */" : "BEGIN IMMEDIATE";
|
||||
const char* commit = active->pump_only ? "COMMIT /* attachment outbox */" : "COMMIT";
|
||||
if (!strcmp(query, begin) && take(BEGIN, NO_OPERATION, 0)) return SQLITE_BUSY;
|
||||
if (!strcmp(query, commit) && take(COMMIT, NO_OPERATION, 0)) return SQLITE_BUSY;
|
||||
if (!strcmp(query, "ROLLBACK") && fail_rollback) {
|
||||
fail_rollback = 0; DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "request test: injected rollback failure"); return SQLITE_IOERR;
|
||||
}
|
||||
@@ -507,11 +510,52 @@ static void test_outgoing_restart(void) {
|
||||
DEBUG_INFO(DEBUG_CATEGORY_MEDIA, "PASS outgoing requests: no route/backpressure/send error preserve queue and date across real instance restart");
|
||||
}
|
||||
|
||||
static void pump_reset(struct fixture* f) {
|
||||
outgoing_reset(f, 0); f->pump_only = 1;
|
||||
char query[768];
|
||||
snprintf(query, sizeof(query), "INSERT INTO attachment_key_requests(channel,message_id,peer,incoming,state,created_at)"
|
||||
" SELECT channel,message_id,author,0,0,123 FROM attachment_keys WHERE channel='%s';"
|
||||
" INSERT INTO attachment_key_outbox(channel,message_id,peer,command,retry_at)"
|
||||
" SELECT channel,message_id,author,1,0 FROM attachment_keys WHERE channel='%s'", f->channel, f->channel); sql(f, query);
|
||||
}
|
||||
|
||||
static void test_pump(void) {
|
||||
struct fixture f; setup(&f); unsigned count = 0;
|
||||
const enum operation operations[] = {NO_OPERATION, PUMP_READ, PUMP_RETRY, NO_OPERATION};
|
||||
for (unsigned op = 0; op < sizeof(operations) / sizeof(operations[0]); op++) {
|
||||
int bindings = operations[op] == PUMP_RETRY ? 3 : 0, points = operations[op] ? 3 + bindings + (bindings != 0) : 1;
|
||||
for (int point = 0; point < points; point++) {
|
||||
pump_reset(&f); fault_operation = operations[op]; fault_index = point;
|
||||
fault = !operations[op] ? (op ? COMMIT : BEGIN) : point == 0 ? PREPARE : point <= bindings ? BIND :
|
||||
point == bindings + 1 ? STEP : point == bindings + 2 ? FINALIZE : CHANGES;
|
||||
unsigned previous = fault_hits; outgoing_tick(&f, 0, 0);
|
||||
assert(!fault && fault_hits == previous + 1 && sqlite3_get_autocommit(f.inst->topo_sqlite_db));
|
||||
outgoing_tick(&f, 0, 1); outgoing_tick(&f, 0, 0); count++;
|
||||
}
|
||||
}
|
||||
const char* triggers[] = {
|
||||
"CREATE TEMP TRIGGER reject_retry BEFORE UPDATE ON attachment_key_outbox BEGIN SELECT RAISE(IGNORE); END",
|
||||
"CREATE TEMP TRIGGER reject_retry BEFORE UPDATE ON attachment_key_outbox BEGIN SELECT RAISE(ABORT,'retry'); END"
|
||||
};
|
||||
for (unsigned i = 0; i < 2; i++) {
|
||||
pump_reset(&f); sql(&f, triggers[i]); outgoing_tick(&f, 0, 0);
|
||||
sql(&f, "DROP TRIGGER reject_retry"); outgoing_tick(&f, 0, 1);
|
||||
}
|
||||
pump_reset(&f); unsigned commits = 0; sqlite3_commit_hook(f.inst->topo_sqlite_db, veto, &commits);
|
||||
outgoing_tick(&f, 0, 0); assert(commits == 1); sqlite3_commit_hook(f.inst->topo_sqlite_db, NULL, NULL); outgoing_tick(&f, 0, 1);
|
||||
pump_reset(&f); fault = COMMIT; fault_operation = NO_OPERATION; fail_rollback = 1; outgoing_tick(&f, 0, 0);
|
||||
assert(!fault && !fail_rollback && !f.inst->running && !CC(f.inst)->attachment_timer && !sqlite3_get_autocommit(f.inst->topo_sqlite_db));
|
||||
chat_event_set_handler(f.inst, NULL); active = NULL; utun_instance_destroy(f.inst); equal(&f);
|
||||
assert(sqlite3_close(f.reader) == SQLITE_OK); uasync_destroy(f.ua, 0);
|
||||
DEBUG_INFO(DEBUG_CATEGORY_MEDIA, "PASS outbox:%u SQL faults, ABORT/IGNORE/veto/rollback failure; no send before retry commit, exact queue/date preserved", count);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
debug_config_init(); debug_set_level(DEBUG_LEVEL_ERROR); debug_set_category_level(DEBUG_CATEGORY_MEDIA, DEBUG_LEVEL_INFO);
|
||||
utun_instance_set_tun_init_enabled(0); size_t baseline = u_get_allocated_count();
|
||||
if (argc != 2 || strcmp(argv[1], "--outgoing")) { test_faults(); test_controls(); test_ack_retry(); test_rollback_failure(); }
|
||||
test_outgoing_controls(); test_outgoing_faults(); test_outgoing_restart(); test_outgoing_rollback_failure();
|
||||
test_pump();
|
||||
assert(!fault && !fail_rollback && u_get_allocated_count() == baseline);
|
||||
DEBUG_INFO(DEBUG_CATEGORY_MEDIA, "PASS attachment requests: reader before ACK/events, stable duplicate date, neighboring key and allocations preserved");
|
||||
return 0;
|
||||
|
||||
Reference in new issue
Block a user