jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add signed FEC stream core and source buffering

Browse Source
This commit is contained in:
evgeny committed 2026-10-12 00:01:38 +02:00
1 parent 2c5deafe56
commit ec82d95a03
16 files changed
+1960 -14
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/Makefile.am
+13
View File
@@ -3,8 +3,20 @@ noinst_LIBRARIES = libutun.a
EXTRA_DIST = lwip_tcp/COPYING dnsmasq/COPYING dnsmasq/COPYING-v3
stream_core_files = \
stream/stream_proto.h \
stream/stream_block.h \
stream/stream_block.c \
stream/stream_buffer.h \
stream/stream_buffer.c \
stream/stream_metrics.h \
stream/stream_metrics.c \
stream/stream_source.h \
stream/stream_source.c
# Sources that are always compiled
utun_CORE_SOURCES = \
$(stream_core_files) \
utun.c \
utun_instance.c \
config_parser.c \
@@ -152,6 +164,7 @@ utun_CORE_SOURCES = \
# libutun: all core sources except main()
libutun_a_SOURCES = \
$(stream_core_files) \
utun_instance.c \
config_parser.c \
config_updater.c \
src/stream/stream_block.c
+180
View File
@@ -0,0 +1,180 @@
#include <string.h>
#include <openssl/crypto.h>
#include <openssl/evp.h>
#include "../../lib/debug_config.h"
#include "../../lib/mem.h"
#include "stream_block.h"
struct stream_encoder {
struct stream_descriptor descriptor;
struct rs_codec *rs;
uint8_t private_key[SC_PRIVKEY_SIZE];
uint8_t public_key[SC_PUBKEY_SIZE];
uint8_t *input;
uint8_t *wire;
};
/* Проверить ключ трансляции и параметры до расчёта размеров/выделения памяти. */
int stream_descriptor_validate(const struct stream_descriptor *d) {
if (!d || !d->group_id || !d->source_node_id || !d->source_epoch || !d->n || d->r >= d->n
|| d->block_size <= RS_HEADER_SIZE || d->block_size > RS_MAX_BLOCK_SIZE) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "descriptor invalid: present=%d group=%llx source=%llx epoch=%llx n=%u r=%u size=%u",
d != NULL, d ? (unsigned long long)d->group_id : 0, d ? (unsigned long long)d->source_node_id : 0,
d ? (unsigned long long)d->source_epoch : 0, d ? d->n : 0, d ? d->r : 0, d ? d->block_size : 0);
return STREAM_ERR_ARGUMENT;
}
return STREAM_OK;
}
size_t stream_block_size(const struct stream_descriptor *d) {
return stream_descriptor_validate(d) == STREAM_OK ? d->block_size + STREAM_BLOCK_OVERHEAD : 0;
}
size_t stream_input_size(const struct stream_descriptor *d) {
return stream_descriptor_validate(d) == STREAM_OK ? (size_t)(d->n - d->r) * (d->block_size - RS_HEADER_SIZE) : 0;
}
/* Создать рабочую память RS и получить публичный ключ именно переданного Ed25519 seed. */
int stream_encoder_create(struct stream_encoder **out, const struct stream_descriptor *d, const uint8_t private_key[32]) {
if (!out || *out || !private_key || stream_descriptor_validate(d) != STREAM_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encoder create invalid arguments out=%p key=%p", (void *)out, (const void *)private_key);
return STREAM_ERR_ARGUMENT;
}
struct stream_encoder *encoder = u_calloc(1, sizeof(*encoder));
if (!encoder) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encoder allocation failed source=%llx", (unsigned long long)d->source_node_id);
return STREAM_ERR_MEMORY;
}
encoder->descriptor = *d;
memcpy(encoder->private_key, private_key, sizeof(encoder->private_key));
EVP_PKEY *key = EVP_PKEY_new_raw_private_key(EVP_PKEY_ED25519, NULL, private_key, SC_PRIVKEY_SIZE);
size_t public_length = SC_PUBKEY_SIZE;
int key_ok = key && EVP_PKEY_get_raw_public_key(key, encoder->public_key, &public_length) == 1 && public_length == SC_PUBKEY_SIZE;
EVP_PKEY_free(key);
if (!key_ok) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encoder Ed25519 key initialization failed source=%llx",
(unsigned long long)d->source_node_id);
stream_encoder_destroy(encoder);
return STREAM_ERR_CRYPTO;
}
if (rs_init(&encoder->rs, d->n, d->r, d->block_size) != RS_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encoder RS initialization failed n=%u r=%u size=%u", d->n, d->r, d->block_size);
stream_encoder_destroy(encoder);
return STREAM_ERR_FEC;
}
encoder->input = u_malloc((uint32_t)stream_input_size(d));
encoder->wire = u_malloc((uint32_t)(d->n * stream_block_size(d)));
if (!encoder->input || !encoder->wire) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encoder working memory allocation failed n=%u size=%u", d->n, d->block_size);
stream_encoder_destroy(encoder);
return STREAM_ERR_MEMORY;
}
*out = encoder;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "encoder created group=%llx source=%llx epoch=%llx FEC=%u+%u block=%zu input=%zu",
(unsigned long long)d->group_id, (unsigned long long)d->source_node_id, (unsigned long long)d->source_epoch,
d->n - d->r, d->r, stream_block_size(d), stream_input_size(d));
return STREAM_OK;
}
/* Очистить секретный seed и рабочее содержимое до возврата памяти allocator. */
void stream_encoder_destroy(struct stream_encoder *encoder) {
if (!encoder) return;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "encoder destroy source=%llx epoch=%llx",
(unsigned long long)encoder->descriptor.source_node_id, (unsigned long long)encoder->descriptor.source_epoch);
OPENSSL_cleanse(encoder->private_key, sizeof(encoder->private_key));
if (encoder->input) OPENSSL_cleanse(encoder->input, stream_input_size(&encoder->descriptor));
u_free(encoder->input);
u_free(encoder->wire);
rs_free(encoder->rs);
u_free(encoder);
}
const uint8_t *stream_encoder_public_key(const struct stream_encoder *encoder) {
if (!encoder) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encoder public key: null context"); return NULL; }
return encoder->public_key;
}
/* Сформировать канонический заголовок, RS-кодограммы и подпись каждого блока. */
int stream_encoder_encode(struct stream_encoder *encoder, uint64_t seq, uint16_t timestamp_ms, uint8_t flags,
const uint8_t *input, size_t length, const uint8_t **blocks) {
if (!encoder || !input || !blocks || !length || seq == UINT64_MAX || (flags & ~STREAM_BLOCK_TIME_RESET)
|| length > stream_input_size(&encoder->descriptor)) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encode invalid arguments seq=%llu len=%zu flags=%u", (unsigned long long)seq, length, flags);
return STREAM_ERR_ARGUMENT;
}
const struct stream_descriptor *d = &encoder->descriptor;
size_t wire_size = stream_block_size(d);
uint8_t *rs_blocks[RS_MAX_BLOCKS];
memcpy(encoder->input, input, length);
memset(encoder->input + length, 0, stream_input_size(d) - length);
for (unsigned i = 0; i < d->n; i++) {
uint8_t *wire = encoder->wire + i * wire_size;
memcpy(wire, STREAM_BLOCK_DOMAIN, STREAM_BLOCK_DOMAIN_SIZE);
stream_write_be(wire + STREAM_BLOCK_GROUP_OFF, d->group_id, 8);
stream_write_be(wire + STREAM_BLOCK_SOURCE_OFF, d->source_node_id, 8);
stream_write_be(wire + STREAM_BLOCK_EPOCH_OFF, d->source_epoch, 8);
wire[STREAM_BLOCK_N_OFF] = d->n;
wire[STREAM_BLOCK_R_OFF] = d->r;
stream_write_be(wire + STREAM_BLOCK_SIZE_OFF, d->block_size, 2);
stream_write_be(wire + STREAM_BLOCK_TIMESTAMP_OFF, timestamp_ms, 2);
stream_write_be(wire + STREAM_BLOCK_LENGTH_OFF, length, 4);
wire[STREAM_BLOCK_FLAGS_OFF] = flags;
rs_blocks[i] = wire + STREAM_BLOCK_RS_OFF;
}
if (rs_encode(encoder->rs, seq, encoder->input, rs_blocks) != RS_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encode RS failed seq=%llu", (unsigned long long)seq);
return STREAM_ERR_FEC;
}
for (unsigned i = 0; i < d->n; i++) {
uint8_t *wire = encoder->wire + i * wire_size;
if (sc_ed25519_sign(encoder->private_key, wire, wire_size - SC_SIGN_SIZE, wire + wire_size - SC_SIGN_SIZE) != SC_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "encode signature failed seq=%llu sub=%u", (unsigned long long)seq, i);
return STREAM_ERR_CRYPTO;
}
}
for (unsigned i = 0; i < d->n; i++) blocks[i] = encoder->wire + i * wire_size;
DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "encoded seq=%llu timestamp=%u len=%zu n=%u flags=%u",
(unsigned long long)seq, timestamp_ms, length, d->n, flags);
return STREAM_OK;
}
/* Проверить размеры, область подписи, дескриптор, метаданные и подпись до публикации view. */
int stream_block_verify(const struct stream_descriptor *d, const uint8_t public_key[32], const uint8_t *wire,
size_t length, struct stream_block_view *view) {
if (!public_key || !wire || !view || stream_descriptor_validate(d) != STREAM_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "verify invalid arguments length=%zu", length);
return STREAM_ERR_ARGUMENT;
}
if (length != stream_block_size(d) || memcmp(wire, STREAM_BLOCK_DOMAIN, STREAM_BLOCK_DOMAIN_SIZE)
|| stream_read_be(wire + STREAM_BLOCK_GROUP_OFF, 8) != d->group_id
|| stream_read_be(wire + STREAM_BLOCK_SOURCE_OFF, 8) != d->source_node_id
|| stream_read_be(wire + STREAM_BLOCK_EPOCH_OFF, 8) != d->source_epoch
|| wire[STREAM_BLOCK_N_OFF] != d->n || wire[STREAM_BLOCK_R_OFF] != d->r
|| stream_read_be(wire + STREAM_BLOCK_SIZE_OFF, 2) != d->block_size) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "verify format/descriptor mismatch source=%llx epoch=%llx length=%zu expected=%zu",
(unsigned long long)d->source_node_id, (unsigned long long)d->source_epoch, length, stream_block_size(d));
return STREAM_ERR_FORMAT;
}
struct stream_block_view parsed = {
.seq = stream_read_be(wire + STREAM_BLOCK_SEQ_OFF, 8),
.data_length = (uint32_t)stream_read_be(wire + STREAM_BLOCK_LENGTH_OFF, 4),
.timestamp_ms = (uint16_t)stream_read_be(wire + STREAM_BLOCK_TIMESTAMP_OFF, 2),
.subblock = wire[STREAM_BLOCK_SUB_OFF], .flags = wire[STREAM_BLOCK_FLAGS_OFF], .rs = wire + STREAM_BLOCK_RS_OFF
};
if (parsed.seq == UINT64_MAX || parsed.subblock >= d->n || !parsed.data_length
|| parsed.data_length > stream_input_size(d) || (parsed.flags & ~STREAM_BLOCK_TIME_RESET)) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "verify invalid metadata seq=%llu sub=%u length=%u flags=%u",
(unsigned long long)parsed.seq, parsed.subblock, parsed.data_length, parsed.flags);
return STREAM_ERR_FORMAT;
}
if (sc_ed25519_verify(public_key, wire, length - SC_SIGN_SIZE, wire + length - SC_SIGN_SIZE) != SC_OK) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "verify source signature failed source=%llx epoch=%llx seq=%llu sub=%u",
(unsigned long long)d->source_node_id, (unsigned long long)d->source_epoch,
(unsigned long long)parsed.seq, parsed.subblock);
return STREAM_ERR_SIGNATURE;
}
*view = parsed;
return STREAM_OK;
}
src/stream/stream_block.h
+48
View File
@@ -0,0 +1,48 @@
/* Формирование и проверка подписанных FEC-блоков без сокетов и внешних callbacks.
* Контексты одного uasync-потока; descriptor и ключ источника уже проверены членством группы.
* encode возвращает заимствованные блоки до следующего encode/destroy; ошибочные результаты не публикуют.
* verify не меняет выходной view при отказе. Ключ непосредственного ретранслятора здесь не используется. */
#ifndef STREAM_BLOCK_H
#define STREAM_BLOCK_H
#include "stream_proto.h"
#ifdef __cplusplus
extern "C" {
#endif
struct stream_encoder;
struct stream_block_view {
uint64_t seq;
uint32_t data_length;
uint16_t timestamp_ms;
uint8_t subblock;
uint8_t flags;
const uint8_t *rs; /* Заимствованный диапазон descriptor.block_size байт. */
};
int stream_descriptor_validate(const struct stream_descriptor *descriptor);
size_t stream_block_size(const struct stream_descriptor *descriptor);
size_t stream_input_size(const struct stream_descriptor *descriptor);
/* out заранее NULL; ключ копируется и очищается при destroy. */
int stream_encoder_create(struct stream_encoder **out, const struct stream_descriptor *descriptor,
const uint8_t private_key[SC_PRIVKEY_SIZE]);
void stream_encoder_destroy(struct stream_encoder *encoder);
/* Заимствованный публичный ключ, полученный из private_key этого encoder. */
const uint8_t *stream_encoder_public_key(const struct stream_encoder *encoder);
/* blocks — массив descriptor.n указателей; input не перекрывается с рабочими буферами encoder.
* 0 < length <= stream_input_size; остаток пачки дополняется нулями. seq != UINT64_MAX.
* При ошибке blocks не изменяется; собственные рабочие буферы могут измениться. */
int stream_encoder_encode(struct stream_encoder *encoder, uint64_t seq, uint16_t timestamp_ms, uint8_t flags,
const uint8_t *input, size_t length, const uint8_t **blocks);
int stream_block_verify(const struct stream_descriptor *descriptor, const uint8_t public_key[SC_PUBKEY_SIZE],
const uint8_t *wire, size_t length, struct stream_block_view *view);
#ifdef __cplusplus
}
#endif
#endif
src/stream/stream_buffer.c
+330
View File
@@ -0,0 +1,330 @@
#include <string.h>
#include "../../lib/debug_config.h"
#include "../../lib/mem.h"
#include "stream_buffer.h"
#define STREAM_SLOT_NONE UINT32_MAX
struct stream_cached_slot {
uint32_t next;
uint8_t subblock;
};
struct stream_cached_batch {
struct stream_cached_batch *hash_next;
uint64_t seq;
uint32_t first_slot;
uint32_t data_length;
uint16_t timestamp_ms;
uint16_t count;
uint8_t flags;
};
struct stream_buffer {
struct stream_descriptor descriptor;
uint8_t public_key[SC_PUBKEY_SIZE];
struct rs_codec *rs;
uint8_t *decode_output;
uint8_t *data;
struct stream_cached_slot *slots;
struct stream_cached_batch **hash;
struct stream_cached_batch **heap;
uint64_t latest[RS_MAX_BLOCKS];
uint8_t latest_valid[RS_MAX_BLOCKS];
size_t block_bytes;
uint32_t capacity;
uint32_t hash_size;
uint32_t free_slot;
struct stream_buffer_stats stats;
};
/* Индекс пачки не зависит от порядка поступления; множество дырок не требует линейного поиска. */
static uint32_t buffer_hash(const struct stream_buffer *b, uint64_t seq) {
seq ^= seq >> 33;
seq *= UINT64_C(0xff51afd7ed558ccd);
seq ^= seq >> 33;
return (uint32_t)seq & (b->hash_size - 1);
}
static struct stream_cached_batch *buffer_batch(const struct stream_buffer *b, uint64_t seq) {
for (struct stream_cached_batch *batch = b->hash[buffer_hash(b, seq)]; batch; batch = batch->hash_next)
if (batch->seq == seq) return batch;
return NULL;
}
/* Поиск не более n блоков пачки; пустые субпотоки не занимают отдельные позиции в массиве данных. */
static uint32_t buffer_slot(const struct stream_buffer *b, const struct stream_cached_batch *batch, uint8_t subblock) {
for (uint32_t slot = batch->first_slot; slot != STREAM_SLOT_NONE; slot = b->slots[slot].next)
if (b->slots[slot].subblock == subblock) return slot;
return STREAM_SLOT_NONE;
}
/* Обмен узлов min-heap, упорядоченного по индексу пачки. */
static void buffer_heap_swap(struct stream_buffer *b, uint32_t first, uint32_t second) {
struct stream_cached_batch *batch = b->heap[first];
b->heap[first] = b->heap[second];
b->heap[second] = batch;
}
/* Добавить новую пачку в хеш и heap; память уже выделена, отказов здесь нет. */
static void buffer_insert_batch(struct stream_buffer *b, struct stream_cached_batch *batch) {
uint32_t hash = buffer_hash(b, batch->seq);
batch->hash_next = b->hash[hash];
b->hash[hash] = batch;
uint32_t index = b->stats.batches++;
b->heap[index] = batch;
while (index && b->heap[(index - 1) / 2]->seq > batch->seq) {
buffer_heap_swap(b, index, (index - 1) / 2);
index = (index - 1) / 2;
}
}
/* Удалить целую минимальную пачку; выданные наружу указатели становятся недействительными. */
static void buffer_evict_oldest(struct stream_buffer *b) {
struct stream_cached_batch *batch = b->heap[0];
uint32_t hash = buffer_hash(b, batch->seq);
struct stream_cached_batch **link = &b->hash[hash];
while (*link != batch) link = &(*link)->hash_next;
*link = batch->hash_next;
uint32_t remaining = --b->stats.batches;
if (remaining) {
b->heap[0] = b->heap[remaining];
uint32_t index = 0;
while (index * 2 + 1 < remaining) {
uint32_t child = index * 2 + 1;
if (child + 1 < remaining && b->heap[child + 1]->seq < b->heap[child]->seq) child++;
if (b->heap[index]->seq <= b->heap[child]->seq) break;
buffer_heap_swap(b, index, child);
index = child;
}
}
for (uint32_t slot = batch->first_slot; slot != STREAM_SLOT_NONE;) {
uint32_t next = b->slots[slot].next;
unsigned sub = b->slots[slot].subblock;
if (b->latest_valid[sub] && b->latest[sub] == batch->seq) b->latest_valid[sub] = 0;
b->slots[slot].next = b->free_slot;
b->free_slot = slot;
slot = next;
}
b->stats.expired_before = batch->seq + 1;
b->stats.blocks -= batch->count;
b->stats.stored_bytes -= batch->count * b->block_bytes;
b->stats.evicted_batches++;
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "buffer evicted seq=%llu blocks=%u retained=%u bytes=%zu low=%llu",
(unsigned long long)batch->seq, batch->count, b->stats.blocks, b->stats.stored_bytes,
(unsigned long long)b->stats.expired_before);
u_free(batch);
}
/* Выделить ограниченные массивы данных/индексов; память пачек растёт только по фактическому поступлению. */
int stream_buffer_create(struct stream_buffer **out, const struct stream_descriptor *d, const uint8_t public_key[32], size_t bytes) {
if (!out || *out || !public_key || stream_descriptor_validate(d) != STREAM_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer create invalid arguments out=%p key=%p", (void *)out, (const void *)public_key);
return STREAM_ERR_ARGUMENT;
}
size_t block_bytes = stream_block_size(d);
if (!bytes) bytes = STREAM_BUFFER_BYTES;
if (bytes > STREAM_BUFFER_BYTES || bytes < d->n * block_bytes) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer budget invalid bytes=%zu batch=%zu max=%u", bytes, d->n * block_bytes, STREAM_BUFFER_BYTES);
return STREAM_ERR_ARGUMENT;
}
struct stream_buffer *b = u_calloc(1, sizeof(*b));
if (!b) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer context allocation failed bytes=%zu", bytes); return STREAM_ERR_MEMORY; }
b->descriptor = *d;
memcpy(b->public_key, public_key, SC_PUBKEY_SIZE);
b->block_bytes = block_bytes;
b->capacity = (uint32_t)(bytes / block_bytes);
b->hash_size = 1;
while (b->hash_size < b->capacity) b->hash_size *= 2;
b->stats.capacity_bytes = b->capacity * block_bytes;
b->data = u_malloc((uint32_t)b->stats.capacity_bytes);
b->slots = u_calloc(b->capacity, sizeof(*b->slots));
b->heap = u_calloc(b->capacity, sizeof(*b->heap));
b->hash = u_calloc(b->hash_size, sizeof(*b->hash));
b->decode_output = u_malloc((uint32_t)stream_input_size(d));
if (!b->data || !b->slots || !b->heap || !b->hash || !b->decode_output) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer arrays allocation failed capacity=%u bytes=%zu", b->capacity, b->stats.capacity_bytes);
stream_buffer_destroy(b);
return STREAM_ERR_MEMORY;
}
for (uint32_t i = 0; i < b->capacity; i++) b->slots[i].next = i + 1;
b->slots[b->capacity - 1].next = STREAM_SLOT_NONE;
if (rs_init(&b->rs, d->n, d->r, d->block_size) != RS_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer decode RS initialization failed n=%u r=%u", d->n, d->r);
stream_buffer_destroy(b);
return STREAM_ERR_FEC;
}
*out = b;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "buffer created source=%llx epoch=%llx bytes=%zu block=%zu slots=%u",
(unsigned long long)d->source_node_id, (unsigned long long)d->source_epoch,
b->stats.capacity_bytes, b->block_bytes, b->capacity);
return STREAM_OK;
}
/* Прекращение владельцем всех обращений предшествует destroy; кольцо ничего не знает о чужих NCD handles. */
void stream_buffer_destroy(struct stream_buffer *b) {
if (!b) return;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "buffer destroy source=%llx blocks=%u batches=%u bytes=%zu evicted=%llu duplicates=%llu",
(unsigned long long)b->descriptor.source_node_id, b->stats.blocks, b->stats.batches, b->stats.stored_bytes,
(unsigned long long)b->stats.evicted_batches, (unsigned long long)b->stats.duplicates);
for (uint32_t i = 0; i < b->stats.batches; i++) u_free(b->heap[i]);
rs_free(b->rs);
u_free(b->decode_output);
u_free(b->data);
u_free(b->slots);
u_free(b->heap);
u_free(b->hash);
u_free(b);
}
/* Вставить только подписанный блок; OOM/конфликт проверяются до вытеснения ранее сохранённых данных. */
int stream_buffer_put(struct stream_buffer *b, const uint8_t *wire, size_t length) {
if (!b || !wire) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer put invalid arguments length=%zu", length); return STREAM_ERR_ARGUMENT; }
struct stream_cached_batch *batch = NULL;
if (length == b->block_bytes) {
batch = buffer_batch(b, stream_read_be(wire + STREAM_BLOCK_SEQ_OFF, 8));
uint8_t sub = wire[STREAM_BLOCK_SUB_OFF];
if (batch && sub < b->descriptor.n) {
uint32_t slot = buffer_slot(b, batch, sub);
if (slot != STREAM_SLOT_NONE && !memcmp(b->data + slot * b->block_bytes, wire, length)) {
b->stats.duplicates++;
DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "buffer duplicate seq=%llu sub=%u", (unsigned long long)batch->seq, sub);
return STREAM_DUPLICATE;
}
}
}
struct stream_block_view view;
int result = stream_block_verify(&b->descriptor, b->public_key, wire, length, &view);
if (result != STREAM_OK) return result;
if (view.seq < b->stats.expired_before) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "buffer expired incoming seq=%llu sub=%u low=%llu full=%d",
(unsigned long long)view.seq, view.subblock, (unsigned long long)b->stats.expired_before, b->stats.blocks == b->capacity);
return STREAM_EXPIRED;
}
if (batch && (batch->timestamp_ms != view.timestamp_ms || batch->data_length != view.data_length || batch->flags != view.flags
|| buffer_slot(b, batch, view.subblock) != STREAM_SLOT_NONE)) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "buffer source conflict seq=%llu sub=%u timestamp=%u/%u length=%u/%u flags=%u/%u",
(unsigned long long)view.seq, view.subblock, batch->timestamp_ms, view.timestamp_ms,
batch->data_length, view.data_length, batch->flags, view.flags);
return STREAM_ERR_CONFLICT;
}
if (b->stats.blocks == b->capacity && view.seq <= b->heap[0]->seq) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "buffer full: incoming seq=%llu sub=%u cannot replace newer data oldest=%llu",
(unsigned long long)view.seq, view.subblock, (unsigned long long)b->heap[0]->seq);
return STREAM_EXPIRED;
}
int new_batch = batch == NULL;
if (new_batch) {
batch = u_calloc(1, sizeof(*batch));
if (!batch) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer batch allocation failed seq=%llu", (unsigned long long)view.seq);
return STREAM_ERR_MEMORY;
}
batch->seq = view.seq;
batch->first_slot = STREAM_SLOT_NONE;
batch->timestamp_ms = view.timestamp_ms;
batch->data_length = view.data_length;
batch->flags = view.flags;
}
if (b->stats.blocks == b->capacity) buffer_evict_oldest(b);
if (new_batch) buffer_insert_batch(b, batch);
uint32_t slot = b->free_slot;
b->free_slot = b->slots[slot].next;
b->slots[slot].subblock = view.subblock;
b->slots[slot].next = batch->first_slot;
batch->first_slot = slot;
batch->count++;
memcpy(b->data + slot * b->block_bytes, wire, length);
b->stats.blocks++;
b->stats.stored_bytes += length;
if (view.seq > b->stats.highest_seq) b->stats.highest_seq = view.seq;
if (!b->latest_valid[view.subblock] || view.seq > b->latest[view.subblock]) {
b->latest[view.subblock] = view.seq;
b->latest_valid[view.subblock] = 1;
}
DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "buffer stored seq=%llu sub=%u present=%u/%u bytes=%zu",
(unsigned long long)view.seq, view.subblock, batch->count, b->descriptor.n, b->stats.stored_bytes);
return STREAM_OK;
}
/* Найти точную позицию либо последний блок субпотока, не меняя его историю/срок хранения. */
int stream_buffer_read(struct stream_buffer *b, uint64_t seq, uint8_t subblock,
const uint8_t **wire, size_t *length, uint64_t *actual_seq) {
if (!b || !wire || !length || !actual_seq || subblock >= b->descriptor.n) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer read invalid arguments seq=%llu sub=%u", (unsigned long long)seq, subblock);
return STREAM_ERR_ARGUMENT;
}
if (seq == UINT64_MAX) {
if (!b->latest_valid[subblock]) return STREAM_WAIT;
seq = b->latest[subblock];
}
if (seq < b->stats.expired_before) {
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "buffer read expired seq=%llu low=%llu", (unsigned long long)seq,
(unsigned long long)b->stats.expired_before);
return STREAM_EXPIRED;
}
if (!b->stats.blocks || seq > b->stats.highest_seq) return STREAM_WAIT;
struct stream_cached_batch *batch = buffer_batch(b, seq);
uint32_t slot = batch ? buffer_slot(b, batch, subblock) : STREAM_SLOT_NONE;
if (slot == STREAM_SLOT_NONE) {
DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "buffer read gap seq=%llu sub=%u", (unsigned long long)seq, subblock);
return STREAM_GAP;
}
*wire = b->data + slot * b->block_bytes;
*length = b->block_bytes;
*actual_seq = seq;
return STREAM_OK;
}
/* Границы наличия конкретного субпотока не обещают заполнения всех промежуточных индексов. */
int stream_buffer_range(struct stream_buffer *b, uint8_t subblock, uint64_t *first, uint64_t *last) {
if (!b || !first || !last || subblock >= b->descriptor.n) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer range invalid sub=%u", subblock);
return STREAM_ERR_ARGUMENT;
}
if (!b->latest_valid[subblock]) return STREAM_WAIT;
uint64_t low = b->latest[subblock];
for (uint32_t i = 0; i < b->stats.batches; i++) {
struct stream_cached_batch *batch = b->heap[i];
if (batch->seq < low && buffer_slot(b, batch, subblock) != STREAM_SLOT_NONE) low = batch->seq;
if (low == b->heap[0]->seq) break;
}
*first = low;
*last = b->latest[subblock];
return STREAM_OK;
}
/* RS пишет в собственный scratch; ошибка или недостающие блоки не повреждают выход потребителя. */
int stream_buffer_decode(struct stream_buffer *b, uint64_t seq, uint8_t *output, size_t capacity, size_t *length) {
if (!b || !output || !length || seq == UINT64_MAX) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer decode invalid arguments seq=%llu capacity=%zu", (unsigned long long)seq, capacity);
return STREAM_ERR_ARGUMENT;
}
if (seq < b->stats.expired_before) return STREAM_EXPIRED;
struct stream_cached_batch *batch = buffer_batch(b, seq);
if (!batch) return !b->stats.blocks || seq > b->stats.highest_seq ? STREAM_WAIT : STREAM_GAP;
if (batch->count < b->descriptor.n - b->descriptor.r) return STREAM_WAIT;
if (capacity < batch->data_length) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer decode output too small seq=%llu capacity=%zu needed=%u",
(unsigned long long)seq, capacity, batch->data_length);
return STREAM_ERR_ARGUMENT;
}
const uint8_t *blocks[RS_MAX_BLOCKS] = {0};
for (uint32_t slot = batch->first_slot; slot != STREAM_SLOT_NONE; slot = b->slots[slot].next)
blocks[b->slots[slot].subblock] = b->data + slot * b->block_bytes + STREAM_BLOCK_RS_OFF;
if (rs_decode(b->rs, blocks, b->decode_output) != RS_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer decode RS failed seq=%llu present=%u", (unsigned long long)seq, batch->count);
return STREAM_ERR_FEC;
}
memcpy(output, b->decode_output, batch->data_length);
*length = batch->data_length;
DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "buffer decoded seq=%llu present=%u length=%u", (unsigned long long)seq, batch->count, batch->data_length);
return STREAM_OK;
}
int stream_buffer_get_stats(const struct stream_buffer *b, struct stream_buffer_stats *stats) {
if (!b || !stats) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "buffer stats invalid arguments"); return STREAM_ERR_ARGUMENT; }
*stats = b->stats;
return STREAM_OK;
}
src/stream/stream_buffer.h
+53
View File
@@ -0,0 +1,53 @@
/* Кольцо проверенных блоков одного запуска: общий бюджет данных для всех субпотоков.
* Пустые субпотоки не резервируют блоки: 32 МБ занимают реально сохранённые кодограммы.
* Вытеснение целой старейшей пачки не закрепляет данные для медленного получателя.
* Все вызовы — один uasync-поток. read возвращает память до следующего put/destroy.
* public_key должен происходить из проверенной записи источника в группе. */
#ifndef STREAM_BUFFER_H
#define STREAM_BUFFER_H
#include "stream_block.h"
#ifdef __cplusplus
extern "C" {
#endif
struct stream_buffer;
struct stream_buffer_stats {
size_t capacity_bytes;
size_t stored_bytes;
uint32_t blocks;
uint32_t batches;
uint64_t expired_before;
uint64_t highest_seq;
uint64_t evicted_batches;
uint64_t duplicates;
};
/* out заранее NULL; bytes=0 выбирает 32 МБ, меньший бюджет допускается для стенда.
* Должна помещаться хотя бы одна полная пачка n блоков; верхний предел — STREAM_BUFFER_BYTES. */
int stream_buffer_create(struct stream_buffer **out, const struct stream_descriptor *descriptor,
const uint8_t public_key[SC_PUBKEY_SIZE], size_t bytes);
void stream_buffer_destroy(struct stream_buffer *buffer);
/* Новый блок проверяется до изменения кольца. Повтор неизменного блока: STREAM_DUPLICATE.
* STREAM_EXPIRED не меняет более свежие данные; неверная подпись/конфликт возвращает отрицательный код. */
int stream_buffer_put(struct stream_buffer *buffer, const uint8_t *wire, size_t length);
/* seq=UINT64_MAX читает последний имеющийся блок именно запрошенного субпотока.
* WAIT — блок впереди головы/пустое кольцо; GAP — дыра; EXPIRED — вытеснен.
* При отсутствии блока выходные указатели/номер не меняются. */
int stream_buffer_read(struct stream_buffer *buffer, uint64_t seq, uint8_t subblock,
const uint8_t **wire, size_t *length, uint64_t *actual_seq);
int stream_buffer_range(struct stream_buffer *buffer, uint8_t subblock, uint64_t *first, uint64_t *last);
/* Восстановить пачку из проверенных блоков; достаточно capacity >= действительной длины.
* Выход не меняется при WAIT/GAP/EXPIRED/ошибке; восстановленные отсутствующие блоки в кольцо не добавляются. */
int stream_buffer_decode(struct stream_buffer *buffer, uint64_t seq, uint8_t *output, size_t capacity, size_t *length);
int stream_buffer_get_stats(const struct stream_buffer *buffer, struct stream_buffer_stats *stats);
#ifdef __cplusplus
}
#endif
#endif
src/stream/stream_metrics.c
+136
View File
@@ -0,0 +1,136 @@
#include <string.h>
#include "../../lib/debug_config.h"
#include "stream_metrics.h"
/* Единственная корзина секундного интервала; old данные не переживают оборот окна. */
static struct stream_metric_bucket *metrics_bucket(struct stream_metrics *m, uint64_t now_ms) {
uint64_t second = now_ms / 1000;
struct stream_metric_bucket *bucket = &m->buckets[second % STREAM_METRIC_SECONDS];
if (!bucket->valid || bucket->second != second) {
memset(bucket, 0, sizeof(*bucket));
bucket->second = second;
bucket->valid = 1;
}
return bucket;
}
/* Переполнение вычисленного отставания не может превращать длительный stall в нулевую задержку. */
static uint64_t metrics_add(uint64_t first, uint64_t second) {
return second > UINT64_MAX - first ? UINT64_MAX : first + second;
}
/* Фиксировать одну остановку только после отсутствия нового блока сверх ожидаемого интервала + 500 мс. */
static void metrics_detect_stop(struct stream_metrics *m, uint64_t now_ms) {
if (!m->initialized || m->ended || m->stopped) return;
uint64_t threshold = metrics_add(m->expected_interval_ms, STREAM_LATE_MS);
uint64_t elapsed = now_ms - m->last_receive_ms;
if (elapsed <= threshold) return;
uint64_t started = metrics_add(m->last_receive_ms, metrics_add(threshold, 1));
metrics_bucket(m, started)->stops++;
m->stopped = 1;
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "subflow stopped after seq=%llu elapsed=%llums expected=%llums lag=%llums",
(unsigned long long)m->last_seq, (unsigned long long)elapsed,
(unsigned long long)m->expected_interval_ms, (unsigned long long)m->lag_ms);
}
/* Начальная ожидаемая пауза — source flush 100 мс; после замеров используется фактический темп источника. */
void stream_metrics_init(struct stream_metrics *m) {
if (!m) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "metrics init null context"); return; }
memset(m, 0, sizeof(*m));
m->expected_interval_ms = STREAM_SOURCE_FLUSH_TB / 10;
}
/* Дельты относительно предшествующего блока исключают смещение часов и учитывают отставание по всей цепочке. */
int stream_metrics_observe(struct stream_metrics *m, uint64_t seq, uint16_t timestamp_ms, uint8_t flags,
uint32_t bytes, uint64_t now_ms) {
if (!m || seq == UINT64_MAX || !bytes || (flags & ~STREAM_BLOCK_TIME_RESET)) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "metrics observe invalid seq=%llu bytes=%u flags=%u", (unsigned long long)seq, bytes, flags);
return STREAM_ERR_ARGUMENT;
}
if (m->ended || (m->initialized && now_ms < m->last_receive_ms)) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "metrics state/clock invalid seq=%llu now=%llu previous=%llu ended=%d",
(unsigned long long)seq, (unsigned long long)now_ms, (unsigned long long)m->last_receive_ms, m->ended);
return STREAM_ERR_STATE;
}
if (m->initialized && seq <= m->last_seq) return STREAM_DUPLICATE;
metrics_detect_stop(m, now_ms);
uint64_t receive_delta = m->initialized ? now_ms - m->last_receive_ms : 0;
int rebase = !m->initialized || (flags & STREAM_BLOCK_TIME_RESET) || seq != m->last_seq + 1
|| receive_delta >= STREAM_TIMESTAMP_PERIOD_MS;
uint16_t source_delta = (uint16_t)(timestamp_ms - m->last_timestamp_ms);
struct stream_metric_bucket *bucket = metrics_bucket(m, now_ms);
bucket->blocks++;
if (rebase) {
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "metrics rebase seq=%llu previous=%llu timestamp=%u/%u receive_delta=%llu flags=%u",
(unsigned long long)seq, (unsigned long long)m->last_seq, m->last_timestamp_ms, timestamp_ms,
(unsigned long long)receive_delta, flags);
m->lag_ms = 0;
m->expected_interval_ms = STREAM_SOURCE_FLUSH_TB / 10;
} else {
if (receive_delta >= source_delta) m->lag_ms = metrics_add(m->lag_ms, receive_delta - source_delta);
else m->lag_ms = source_delta - receive_delta >= m->lag_ms ? 0 : m->lag_ms - (source_delta - receive_delta);
bucket->bytes += bytes;
bucket->source_ms += source_delta;
bucket->receive_ms += receive_delta;
bucket->intervals++;
if (bucket->source_ms) {
m->expected_interval_ms = bucket->source_ms / bucket->intervals;
if (!m->expected_interval_ms) m->expected_interval_ms = 1;
}
if (m->lag_ms > STREAM_LATE_MS) bucket->late_blocks++;
DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "metrics seq=%llu source_delta=%u receive_delta=%llu lag=%llu expected=%llu",
(unsigned long long)seq, source_delta, (unsigned long long)receive_delta,
(unsigned long long)m->lag_ms, (unsigned long long)m->expected_interval_ms);
}
m->last_seq = seq;
m->last_timestamp_ms = timestamp_ms;
m->last_receive_ms = now_ms;
m->initialized = 1;
m->stopped = 0;
return STREAM_OK;
}
/* Скорость суммируется по объёму и времени, а не усреднением частных rates разных интервалов. */
int stream_metrics_poll(struct stream_metrics *m, uint64_t now_ms, struct stream_metric_snapshot *snapshot) {
if (!m || !snapshot) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "metrics poll invalid arguments"); return STREAM_ERR_ARGUMENT; }
if (m->initialized && now_ms < m->last_receive_ms) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "metrics poll clock regression now=%llu previous=%llu",
(unsigned long long)now_ms, (unsigned long long)m->last_receive_ms);
return STREAM_ERR_STATE;
}
struct stream_metric_snapshot result = {0};
if (!m->initialized) { *snapshot = result; return STREAM_WAIT; }
metrics_detect_stop(m, now_ms);
result.lag_ms = m->lag_ms;
result.expected_interval_ms = m->expected_interval_ms;
result.stopped = m->stopped;
if (!m->ended && now_ms - m->last_receive_ms > m->expected_interval_ms)
result.lag_ms = metrics_add(result.lag_ms, now_ms - m->last_receive_ms - m->expected_interval_ms);
uint64_t second = now_ms / 1000;
uint64_t bytes = 0, source_ms = 0, receive_ms = 0;
for (unsigned i = 0; i < STREAM_METRIC_SECONDS; i++) {
const struct stream_metric_bucket *bucket = &m->buckets[i];
if (!bucket->valid || bucket->second > second || second - bucket->second >= STREAM_METRIC_SECONDS) continue;
result.blocks += bucket->blocks;
result.late_blocks += bucket->late_blocks;
result.stops += bucket->stops;
bytes += bucket->bytes;
source_ms += bucket->source_ms;
receive_ms += bucket->receive_ms;
}
result.source_rate_valid = source_ms != 0;
result.receive_rate_valid = receive_ms != 0;
if (source_ms) result.source_rate_bytes_sec = bytes * 1000 / source_ms;
if (receive_ms) result.receive_rate_bytes_sec = bytes * 1000 / receive_ms;
*snapshot = result;
return STREAM_OK;
}
/* END останавливает ожидание будущих данных, история качества остаётся доступной. */
void stream_metrics_end(struct stream_metrics *m) {
if (!m) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "metrics end null context"); return; }
m->ended = 1;
m->stopped = 0;
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "metrics ended at seq=%llu", (unsigned long long)m->last_seq);
}
src/stream/stream_metrics.h
+64
View File
@@ -0,0 +1,64 @@
/* Относительное отставание одного субпотока от одного поставщика, без NTP и сокетов.
* now_ms — монотонные часы получателя; timestamp — неизменяемые 16-битные часы источника.
* Первый блок/разрыв устанавливает новую базу, история проблем остаётся за последние 60 секунд.
* observe/poll вызываются одним uasync-потоком. DUPLICATE не меняет ни часы, ни статистику. */
#ifndef STREAM_METRICS_H
#define STREAM_METRICS_H
#include "stream_proto.h"
#ifdef __cplusplus
extern "C" {
#endif
#define STREAM_METRIC_SECONDS 60
struct stream_metric_bucket {
uint64_t second;
uint64_t blocks;
uint64_t late_blocks;
uint64_t stops;
uint64_t bytes;
uint64_t source_ms;
uint64_t receive_ms;
uint64_t intervals;
int valid;
};
struct stream_metrics {
struct stream_metric_bucket buckets[STREAM_METRIC_SECONDS];
uint64_t last_seq;
uint64_t last_receive_ms;
uint64_t lag_ms;
uint64_t expected_interval_ms;
uint16_t last_timestamp_ms;
int initialized;
int stopped;
int ended;
};
struct stream_metric_snapshot {
uint64_t lag_ms;
uint64_t blocks;
uint64_t late_blocks;
uint64_t stops;
uint64_t source_rate_bytes_sec;
uint64_t receive_rate_bytes_sec;
uint64_t expected_interval_ms;
int stopped;
int source_rate_valid;
int receive_rate_valid;
};
void stream_metrics_init(struct stream_metrics *metrics);
int stream_metrics_observe(struct stream_metrics *metrics, uint64_t seq, uint16_t timestamp_ms, uint8_t flags,
uint32_t bytes, uint64_t now_ms);
/* Также обнаруживает полную остановку; episode учитывается в момент превышения порога, не на каждом poll.
* До первого блока возвращает WAIT и обнулённый snapshot; при ошибке snapshot остаётся прежним. */
int stream_metrics_poll(struct stream_metrics *metrics, uint64_t now_ms, struct stream_metric_snapshot *snapshot);
void stream_metrics_end(struct stream_metrics *metrics);
#ifdef __cplusplus
}
#endif
#endif
src/stream/stream_proto.h
+73
View File
@@ -0,0 +1,73 @@
/* Неизменяемый блок источника: все числа big-endian, подпись охватывает тело до signature.
* Данные ETCP-выдачи (handle/сессия/путь) добавляются снаружи и в подпись не входят.
* seq/subblock берутся из заголовка RS; второго независимого номера блока нет. */
#ifndef STREAM_PROTO_H
#define STREAM_PROTO_H
#include <stddef.h>
#include <stdint.h>
#include "../../lib/reed_solomon.h"
#include "../transport_layer/secure_channel.h"
#define STREAM_BLOCK_DOMAIN "stream_block"
#define STREAM_BLOCK_DOMAIN_SIZE 12
#define STREAM_BLOCK_GROUP_OFF 12
#define STREAM_BLOCK_SOURCE_OFF 20
#define STREAM_BLOCK_EPOCH_OFF 28
#define STREAM_BLOCK_N_OFF 36
#define STREAM_BLOCK_R_OFF 37
#define STREAM_BLOCK_SIZE_OFF 38
#define STREAM_BLOCK_TIMESTAMP_OFF 40
#define STREAM_BLOCK_LENGTH_OFF 42
#define STREAM_BLOCK_FLAGS_OFF 46
#define STREAM_BLOCK_RS_OFF 47
#define STREAM_BLOCK_SEQ_OFF STREAM_BLOCK_RS_OFF
#define STREAM_BLOCK_SUB_OFF (STREAM_BLOCK_RS_OFF + 8)
#define STREAM_BLOCK_OVERHEAD (STREAM_BLOCK_RS_OFF + SC_SIGN_SIZE)
#define STREAM_BLOCK_DEFAULT_SIZE 1300
#define STREAM_RS_DEFAULT_SIZE (STREAM_BLOCK_DEFAULT_SIZE - STREAM_BLOCK_OVERHEAD)
#define STREAM_BLOCK_MAX_SIZE (RS_MAX_BLOCK_SIZE + STREAM_BLOCK_OVERHEAD)
#define STREAM_BLOCK_TIME_RESET 0x01
#define STREAM_BUFFER_BYTES (32u * 1024u * 1024u)
#define STREAM_SOURCE_FLUSH_TB 1000 /* 100 мс, timebase 0.1 мс */
#define STREAM_LATE_MS 500
#define STREAM_TIMESTAMP_PERIOD_MS 65536u
struct stream_descriptor {
uint64_t group_id;
uint64_t source_node_id;
uint64_t source_epoch;
uint16_t block_size; /* Полный размер RS-кодограммы, без stream-метаданных/подписи. */
uint8_t n;
uint8_t r;
};
enum stream_result {
STREAM_OK = 0,
STREAM_DUPLICATE = 1,
STREAM_WAIT = 2,
STREAM_GAP = 3,
STREAM_EXPIRED = 4,
STREAM_ERR_ARGUMENT = -1,
STREAM_ERR_MEMORY = -2,
STREAM_ERR_FORMAT = -3,
STREAM_ERR_SIGNATURE = -4,
STREAM_ERR_CONFLICT = -5,
STREAM_ERR_STATE = -6,
STREAM_ERR_CRYPTO = -7,
STREAM_ERR_FEC = -8
};
/* Короткие примитивы без доступа за границы: размер проверяет вызывающий parser. */
static inline uint64_t stream_read_be(const uint8_t *data, unsigned bytes) {
uint64_t result = 0;
for (unsigned i = 0; i < bytes; i++) result = (result << 8) | data[i];
return result;
}
static inline void stream_write_be(uint8_t *data, uint64_t value, unsigned bytes) {
for (unsigned i = 0; i < bytes; i++) data[bytes - i - 1] = (uint8_t)(value >> (8 * i));
}
#endif
src/stream/stream_source.c
+181
View File
@@ -0,0 +1,181 @@
#include <string.h>
#include "../../lib/debug_config.h"
#include "../../lib/mem.h"
#include "../../lib/u_async.h"
#include "stream_source.h"
struct stream_source {
struct UASYNC *ua;
struct stream_descriptor descriptor;
struct stream_encoder *encoder;
struct stream_buffer *buffer;
uint8_t *input;
size_t capacity;
size_t used;
uint64_t next_seq;
uint64_t last_batch_ms;
uint64_t emitted_bytes;
uint64_t report_ms;
void *timer;
stream_source_ready_fn ready;
void *arg;
int busy;
int has_batch;
int failed;
int ended;
};
/* Ошибка не скрывает уже принятый префикс: прекращаем таймер и дальнейшие write. */
static int source_fail(struct stream_source *source, int result) {
source->failed = 1;
if (source->timer) { uasync_cancel_timeout(source->ua, source->timer); source->timer = NULL; }
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source failed source=%llx seq=%llu pending=%zu result=%d",
(unsigned long long)source->descriptor.source_node_id, (unsigned long long)source->next_seq, source->used, result);
return result;
}
/* Сформировать блоки и сохранить их до уведомления потребителей; busy установлен внешней операцией. */
static int source_flush(struct stream_source *source) {
if (!source->used) return STREAM_OK;
if (source->timer) { uasync_cancel_timeout(source->ua, source->timer); source->timer = NULL; }
uint64_t now_ms = get_time_tb() / 10;
if (source->next_seq == UINT64_MAX || (source->has_batch && now_ms < source->last_batch_ms)) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source sequence/clock invalid seq=%llu now=%llu previous=%llu",
(unsigned long long)source->next_seq, (unsigned long long)now_ms, (unsigned long long)source->last_batch_ms);
return source_fail(source, STREAM_ERR_STATE);
}
uint8_t flags = source->has_batch && now_ms - source->last_batch_ms >= STREAM_TIMESTAMP_PERIOD_MS ? STREAM_BLOCK_TIME_RESET : 0;
const uint8_t *blocks[RS_MAX_BLOCKS];
int result = stream_encoder_encode(source->encoder, source->next_seq, (uint16_t)now_ms, flags, source->input, source->used, blocks);
if (result != STREAM_OK) return source_fail(source, result);
for (unsigned i = 0; i < source->descriptor.n; i++) {
result = stream_buffer_put(source->buffer, blocks[i], stream_block_size(&source->descriptor));
if (result != STREAM_OK) return source_fail(source, result < 0 ? result : STREAM_ERR_STATE);
}
uint64_t seq = source->next_seq++;
source->emitted_bytes += source->used;
source->used = 0;
source->has_batch = 1;
source->last_batch_ms = now_ms;
if (!source->report_ms || now_ms - source->report_ms >= 1000) {
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "source progress source=%llx seq=%llu bytes=%llu timestamp=%u flags=%u",
(unsigned long long)source->descriptor.source_node_id, (unsigned long long)seq,
(unsigned long long)source->emitted_bytes, (uint16_t)now_ms, flags);
source->report_ms = now_ms;
}
if (source->ready) source->ready(source, seq, source->arg);
return STREAM_OK;
}
/* Срабатывание единственного таймера неполной пачки; handle больше не принадлежит таймерному heap. */
static void source_flush_timer(void *arg) {
struct stream_source *source = arg;
source->timer = NULL;
source->busy = 1;
int result = source_flush(source);
source->busy = 0;
if (result != STREAM_OK) DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source timed flush failed result=%d", result);
}
/* Создать только реальную локальную публикацию; память удалённых анонсов сюда не попадает. */
int stream_source_create(struct stream_source **out, struct UASYNC *ua, const struct stream_descriptor *d,
const uint8_t private_key[32], stream_source_ready_fn ready, void *arg) {
if (!out || *out || !ua || !private_key || stream_descriptor_validate(d) != STREAM_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source create invalid arguments out=%p ua=%p", (void *)out, (void *)ua);
return STREAM_ERR_ARGUMENT;
}
struct stream_source *source = u_calloc(1, sizeof(*source));
if (!source) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source context allocation failed"); return STREAM_ERR_MEMORY; }
source->ua = ua;
source->descriptor = *d;
source->ready = ready;
source->arg = arg;
source->capacity = stream_input_size(d);
source->input = u_malloc((uint32_t)source->capacity);
int result = source->input ? stream_encoder_create(&source->encoder, d, private_key) : STREAM_ERR_MEMORY;
if (result == STREAM_OK)
result = stream_buffer_create(&source->buffer, d, stream_encoder_public_key(source->encoder), STREAM_BUFFER_BYTES);
if (result != STREAM_OK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source initialization failed source=%llx result=%d", (unsigned long long)d->source_node_id, result);
stream_source_destroy(source);
return result;
}
*out = source;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "source created group=%llx source=%llx epoch=%llx input=%zu flush=100ms",
(unsigned long long)d->group_id, (unsigned long long)d->source_node_id,
(unsigned long long)d->source_epoch, source->capacity);
return STREAM_OK;
}
void stream_source_destroy(struct stream_source *source) {
if (!source) return;
if (source->busy) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source destroy inside callback rejected"); return; }
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "source destroy source=%llx batches=%llu bytes=%llu pending=%zu ended=%d failed=%d",
(unsigned long long)source->descriptor.source_node_id, (unsigned long long)source->next_seq,
(unsigned long long)source->emitted_bytes, source->used, source->ended, source->failed);
if (source->timer) uasync_cancel_timeout(source->ua, source->timer);
stream_buffer_destroy(source->buffer);
stream_encoder_destroy(source->encoder);
u_free(source->input);
u_free(source);
}
size_t stream_source_input_capacity(const struct stream_source *source) {
if (!source) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source capacity null context"); return 0; }
return source->capacity;
}
struct stream_buffer *stream_source_buffer(struct stream_source *source) {
if (!source) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source buffer null context"); return NULL; }
return source->buffer;
}
/* Принимать ограниченную порцию, дополнять старую пачку и заводить таймер только на новый остаток. */
int stream_source_write(struct stream_source *source, const uint8_t *data, size_t length, size_t *accepted) {
if (accepted) *accepted = 0;
if (!source || !accepted || (length && !data) || length > source->capacity) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source write invalid arguments length=%zu", length);
return STREAM_ERR_ARGUMENT;
}
if (source->busy || source->failed || source->ended) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "source write state rejected busy=%d failed=%d ended=%d", source->busy, source->failed, source->ended);
return STREAM_ERR_STATE;
}
if (!length) return STREAM_OK;
source->busy = 1;
int result = STREAM_OK;
while (*accepted < length) {
size_t take = source->capacity - source->used;
if (take > length - *accepted) take = length - *accepted;
memcpy(source->input + source->used, data + *accepted, take);
source->used += take;
*accepted += take;
if (source->used == source->capacity && (result = source_flush(source)) != STREAM_OK) break;
}
if (result == STREAM_OK && source->used && !source->timer) {
source->timer = uasync_set_timeout(source->ua, STREAM_SOURCE_FLUSH_TB, source, source_flush_timer, "stream_source_flush");
if (!source->timer) result = source_fail(source, STREAM_ERR_MEMORY);
}
source->busy = 0;
return result;
}
/* Finish отменяет таймер и публикует остаток сразу; дальнейшие байты требуют нового source_epoch. */
int stream_source_finish(struct stream_source *source) {
if (!source || source->busy || source->failed) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "source finish invalid state present=%d", source != NULL);
return source ? STREAM_ERR_STATE : STREAM_ERR_ARGUMENT;
}
if (source->ended) return STREAM_OK;
source->busy = 1;
int result = source_flush(source);
source->busy = 0;
if (result == STREAM_OK) {
source->ended = 1;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "source finished source=%llx batches=%llu bytes=%llu",
(unsigned long long)source->descriptor.source_node_id, (unsigned long long)source->next_seq,
(unsigned long long)source->emitted_bytes);
}
return result;
}
src/stream/stream_source.h
+36
View File
@@ -0,0 +1,36 @@
/* Источник непрозрачных байтов: одна подписанная FEC-пачка за ограниченный write,
* неполная пачка сбрасывается таймером 100 мс, все блоки сначала сохраняются в кольце.
* Один uasync-поток; источник сам владеет encoder/кольцом/таймером. NCD здесь ещё нет.
* ready вызывается после помещения всей пачки и может читать кольцо; write/finish/destroy
* из ready запрещены. destroy выполняется вне callbacks источника. */
#ifndef STREAM_SOURCE_H
#define STREAM_SOURCE_H
#include "stream_buffer.h"
#ifdef __cplusplus
extern "C" {
#endif
struct UASYNC;
struct stream_source;
typedef void (*stream_source_ready_fn)(struct stream_source *source, uint64_t seq, void *arg);
int stream_source_create(struct stream_source **out, struct UASYNC *ua, const struct stream_descriptor *descriptor,
const uint8_t private_key[SC_PRIVKEY_SIZE], stream_source_ready_fn ready, void *arg);
void stream_source_destroy(struct stream_source *source);
size_t stream_source_input_capacity(const struct stream_source *source);
/* Только заимствованное чтение/decode; вставлять блоки/освобождать это кольцо должен сам источник. */
struct stream_buffer *stream_source_buffer(struct stream_source *source);
/* length <= input_capacity: вызов кодирует не более одной пачки, не захватывая uasync большим внешним буфером.
* accepted получает принятую часть даже при ошибке; нулевой write допустим и не создаёт пустую пачку.
* Ошибка публикации/таймера делает источник failed; его уничтожают, не повторяют принятую часть как новую трансляцию. */
int stream_source_write(struct stream_source *source, const uint8_t *data, size_t length, size_t *accepted);
/* Сбросить остаток и остановить приём; повторный finish допустим, кольцо доступно до destroy. */
int stream_source_finish(struct stream_source *source);
#ifdef __cplusplus
}
#endif
#endif