jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Define FEC protected audiovisual blocks and frame assembly

Browse Source
This commit is contained in:
evgeny committed 2026-10-12 01:45:08 +02:00
1 parent 47caca82a1
commit 1389bac261
2 files changed
+283
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/stream/stream_media.c
+222
View File
@@ -0,0 +1,222 @@
#include <string.h>
#include "../../lib/debug_config.h"
#include "../../lib/mem.h"
#include "stream_media.h"
struct stream_media_receiver {
stream_media_frame_fn callback;
void *arg;
struct stream_media_format format;
struct stream_media_video frame;
uint8_t *data;
uint32_t used;
uint64_t last_video_pts, last_audio_pts, last_frame_id;
int configured, waiting_key, video_seen, audio_seen;
};
/* Проверить весь набор описаний до чтения payload или выделения памяти под кадр. */
static int media_validate(const struct stream_media_block *b, size_t *video_size, size_t *audio_size, uint32_t *first) {
if (!b || b->video_count > STREAM_MEDIA_MAX_ENTRIES || b->audio_count > STREAM_MEDIA_MAX_ENTRIES
|| (!b->video_count && !b->audio_count) || b->format.video_codec != STREAM_MEDIA_H264
|| b->format.audio_codec > STREAM_MEDIA_OPUS || !b->format.width || b->format.width > 4096
|| !b->format.height || b->format.height > 4096 || b->format.rotation > 270 || b->format.rotation % 90) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media format/count invalid"); return STREAM_ERR_FORMAT;
}
*video_size = *audio_size = 0; *first = UINT32_MAX;
for (unsigned i = 0; i < b->video_count; i++) {
const struct stream_media_video *v = &b->video[i];
if (!v->data || !v->frame_id || !v->frame_size || v->frame_size > STREAM_MEDIA_MAX_FRAME || !v->length
|| v->offset >= v->frame_size || v->length > v->frame_size - v->offset || (v->flags & ~STREAM_MEDIA_KEY)
|| v->pts_us > INT64_MAX || v->length > STREAM_MEDIA_MAX_BLOCK - *video_size) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media video invalid entry=%u id=%llu offset=%u length=%u frame=%u flags=%u",
i, (unsigned long long)v->frame_id, v->offset, v->length, v->frame_size, v->flags);
return STREAM_ERR_FORMAT;
}
if (i) {
const struct stream_media_video *p = &b->video[i - 1];
int same = p->frame_id == v->frame_id;
if ((same && (v->offset != p->offset + p->length || v->frame_size != p->frame_size
|| v->pts_us != p->pts_us || v->flags != p->flags))
|| (!same && (v->frame_id <= p->frame_id || v->offset || p->offset + p->length != p->frame_size
|| v->pts_us <= p->pts_us))) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media video order invalid entry=%u", i); return STREAM_ERR_FORMAT;
}
}
if (!v->offset && *first == UINT32_MAX) *first = (uint32_t)*video_size;
*video_size += v->length;
}
for (unsigned i = 0; i < b->audio_count; i++) {
const struct stream_media_audio *a = &b->audio[i];
if (b->format.audio_codec != STREAM_MEDIA_OPUS || !a->data || !a->length || a->length > STREAM_MEDIA_MAX_OPUS
|| a->samples != 960 || a->pts_us > INT64_MAX || (i && a->pts_us <= b->audio[i - 1].pts_us)) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media audio invalid entry=%u length=%u samples=%u", i, a->length, a->samples);
return STREAM_ERR_FORMAT;
}
*audio_size += a->length;
}
return STREAM_OK;
}
/* Канонический header и последовательные payload-области исключают перекрытия фрагментов. */
int stream_media_encode(const struct stream_media_block *b, uint8_t *output, size_t capacity, size_t *length) {
size_t video_size, audio_size; uint32_t first;
if (!output || !length) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media encode missing output"); return STREAM_ERR_ARGUMENT; }
int result = media_validate(b, &video_size, &audio_size, &first);
if (result != STREAM_OK) return result;
size_t header = STREAM_MEDIA_HEADER_SIZE + b->video_count * STREAM_MEDIA_VIDEO_SIZE + b->audio_count * STREAM_MEDIA_AUDIO_SIZE;
size_t total = header + video_size + audio_size;
if (total > capacity || total > STREAM_MEDIA_MAX_BLOCK) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media encode capacity=%zu needed=%zu", capacity, total); return STREAM_ERR_ARGUMENT;
}
memset(output, 0, header);
memcpy(output, "uAV1", 4);
output[4] = b->format.video_codec; output[5] = b->format.audio_codec;
output[6] = (uint8_t)b->video_count; output[7] = (uint8_t)b->audio_count;
stream_write_be(output + 8, b->format.width, 2); stream_write_be(output + 10, b->format.height, 2);
stream_write_be(output + 12, b->format.rotation, 2); stream_write_be(output + 14, header, 2);
stream_write_be(output + 16, video_size, 4); stream_write_be(output + 20, audio_size, 4);
stream_write_be(output + 24, first, 4); stream_write_be(output + 28, 1, 4);
size_t payload = header, entry = STREAM_MEDIA_HEADER_SIZE;
for (unsigned i = 0; i < b->video_count; i++, entry += STREAM_MEDIA_VIDEO_SIZE) {
const struct stream_media_video *v = &b->video[i];
stream_write_be(output + entry, v->frame_id, 8); stream_write_be(output + entry + 8, v->pts_us, 8);
stream_write_be(output + entry + 16, v->frame_size, 4); stream_write_be(output + entry + 20, v->offset, 4);
stream_write_be(output + entry + 24, v->length, 4); output[entry + 28] = v->flags;
memcpy(output + payload, v->data, v->length); payload += v->length;
}
for (unsigned i = 0; i < b->audio_count; i++, entry += STREAM_MEDIA_AUDIO_SIZE) {
const struct stream_media_audio *a = &b->audio[i];
stream_write_be(output + entry, a->pts_us, 8); stream_write_be(output + entry + 8, a->length, 4);
stream_write_be(output + entry + 12, a->samples, 2);
memcpy(output + payload, a->data, a->length); payload += a->length;
}
*length = total;
return STREAM_OK;
}
/* Проверить арифметику областей до вычисления указателей, затем валидировать все описания. */
int stream_media_parse(const uint8_t *data, size_t length, struct stream_media_block *out) {
if (!data || !out || length < STREAM_MEDIA_HEADER_SIZE || length > STREAM_MEDIA_MAX_BLOCK || memcmp(data, "uAV1", 4)) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media parse bad prefix/length=%zu", length); return STREAM_ERR_FORMAT;
}
struct stream_media_block b = {.video_count = data[6], .audio_count = data[7]};
b.format = (struct stream_media_format){(uint16_t)stream_read_be(data + 8, 2), (uint16_t)stream_read_be(data + 10, 2),
(uint16_t)stream_read_be(data + 12, 2), data[4], data[5]};
size_t header = STREAM_MEDIA_HEADER_SIZE + b.video_count * STREAM_MEDIA_VIDEO_SIZE + b.audio_count * STREAM_MEDIA_AUDIO_SIZE;
size_t video_size = (size_t)stream_read_be(data + 16, 4), audio_size = (size_t)stream_read_be(data + 20, 4);
if (b.video_count > STREAM_MEDIA_MAX_ENTRIES || b.audio_count > STREAM_MEDIA_MAX_ENTRIES || header > length
|| stream_read_be(data + 14, 2) != header || video_size > length - header || audio_size != length - header - video_size
|| stream_read_be(data + 28, 4) != 1) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media parse header/regions invalid length=%zu header=%zu video=%zu audio=%zu",
length, header, video_size, audio_size); return STREAM_ERR_FORMAT;
}
size_t entry = STREAM_MEDIA_HEADER_SIZE, payload = header;
for (unsigned i = 0; i < b.video_count; i++, entry += STREAM_MEDIA_VIDEO_SIZE) {
struct stream_media_video *v = &b.video[i];
*v = (struct stream_media_video){.frame_id = stream_read_be(data + entry, 8), .pts_us = stream_read_be(data + entry + 8, 8),
.frame_size = (uint32_t)stream_read_be(data + entry + 16, 4), .offset = (uint32_t)stream_read_be(data + entry + 20, 4),
.length = (uint32_t)stream_read_be(data + entry + 24, 4), .flags = data[entry + 28], .data = data + payload};
if (v->length > header + video_size - payload || data[entry + 29] || data[entry + 30] || data[entry + 31]) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media parse video bounds/reserved entry=%u", i); return STREAM_ERR_FORMAT;
}
payload += v->length;
}
if (payload != header + video_size) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media parse video size mismatch"); return STREAM_ERR_FORMAT;
}
for (unsigned i = 0; i < b.audio_count; i++, entry += STREAM_MEDIA_AUDIO_SIZE) {
struct stream_media_audio *a = &b.audio[i];
*a = (struct stream_media_audio){.pts_us = stream_read_be(data + entry, 8),
.length = (uint32_t)stream_read_be(data + entry + 8, 4), .samples = (uint16_t)stream_read_be(data + entry + 12, 2),
.data = data + payload};
if (a->length > length - payload || data[entry + 14] || data[entry + 15]) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media parse audio bounds/reserved entry=%u", i); return STREAM_ERR_FORMAT;
}
payload += a->length;
}
size_t actual_video, actual_audio; uint32_t first;
int result = media_validate(&b, &actual_video, &actual_audio, &first);
if (result != STREAM_OK) return result;
if (actual_video != video_size || actual_audio != audio_size || stream_read_be(data + 24, 4) != first) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media parse payload/first offset mismatch"); return STREAM_ERR_FORMAT;
}
b.first_frame_offset = first; *out = b;
return STREAM_OK;
}
/* Receiver имеет один ограниченный буфер; ожидание KEY не мешает доставке независимого аудио. */
int stream_media_receiver_create(struct stream_media_receiver **out, stream_media_frame_fn callback, void *arg) {
if (!out || *out || !callback) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media receiver create invalid arguments"); return STREAM_ERR_ARGUMENT; }
struct stream_media_receiver *r = u_calloc(1, sizeof(*r));
if (!r) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media receiver allocation failed"); return STREAM_ERR_MEMORY; }
r->callback = callback; r->arg = arg; r->waiting_key = 1; *out = r;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "media receiver created frame_limit=%u", STREAM_MEDIA_MAX_FRAME);
return STREAM_OK;
}
/* Сбросить недособранный кадр и зависимые P-кадры; PTS-история сохраняется в пределах эпохи. */
void stream_media_receiver_gap(struct stream_media_receiver *r) {
if (!r) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media receiver gap missing context"); return; }
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "media receiver GAP frame=%llu used=%u/%u; waiting for KEY",
(unsigned long long)r->frame.frame_id, r->used, r->frame.frame_size);
u_free(r->data); r->data = NULL; r->used = 0; r->waiting_key = 1;
}
/* Только полный последовательный кадр допускается в декодер; никакого склеивания через GAP. */
int stream_media_receiver_feed(struct stream_media_receiver *r, const uint8_t *data, size_t length) {
if (!r) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media receiver feed missing context"); return STREAM_ERR_ARGUMENT; }
struct stream_media_block b;
int result = stream_media_parse(data, length, &b);
if (result != STREAM_OK) { stream_media_receiver_gap(r); return result; }
if (r->configured && (r->format.width != b.format.width || r->format.height != b.format.height
|| r->format.rotation != b.format.rotation || r->format.audio_codec != b.format.audio_codec)) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media format changed within source epoch");
stream_media_receiver_gap(r); return STREAM_ERR_CONFLICT;
}
r->format = b.format; r->configured = 1;
for (unsigned i = 0; i < b.video_count; i++) {
const struct stream_media_video *v = &b.video[i];
if (!v->offset) {
if (r->data) stream_media_receiver_gap(r);
if (r->video_seen && (v->pts_us <= r->last_video_pts || v->frame_id <= r->last_frame_id)) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "media stale video frame=%llu", (unsigned long long)v->frame_id); continue;
}
if (r->waiting_key && !(v->flags & STREAM_MEDIA_KEY)) continue;
r->data = u_malloc(v->frame_size);
if (!r->data) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media frame allocation failed bytes=%u", v->frame_size); return STREAM_ERR_MEMORY; }
r->frame = *v; r->used = 0;
}
if (!r->data) continue;
if (v->frame_id != r->frame.frame_id || v->pts_us != r->frame.pts_us || v->frame_size != r->frame.frame_size
|| v->flags != r->frame.flags || v->offset != r->used) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "media fragment discontinuity frame=%llu offset=%u expected=%u",
(unsigned long long)v->frame_id, v->offset, r->used);
stream_media_receiver_gap(r); continue;
}
memcpy(r->data + r->used, v->data, v->length); r->used += v->length;
if (r->used == r->frame.frame_size) {
r->frame.data = r->data; r->frame.length = r->used; r->frame.offset = 0;
if (r->waiting_key) DEBUG_INFO(DEBUG_CATEGORY_STREAM, "media decoder resumes KEY frame=%llu pts=%llu bytes=%u",
(unsigned long long)v->frame_id, (unsigned long long)v->pts_us, r->used);
r->waiting_key = 0; r->video_seen = 1; r->last_video_pts = v->pts_us; r->last_frame_id = v->frame_id;
r->callback(r->arg, &r->format, &r->frame, NULL);
u_free(r->data); r->data = NULL; r->used = 0;
}
}
for (unsigned i = 0; i < b.audio_count; i++) {
const struct stream_media_audio *a = &b.audio[i];
if (r->audio_seen && a->pts_us <= r->last_audio_pts) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "media stale audio pts=%llu", (unsigned long long)a->pts_us); continue;
}
r->audio_seen = 1; r->last_audio_pts = a->pts_us; r->callback(r->arg, &r->format, NULL, a);
}
return STREAM_OK;
}
/* Освободить только собственный буфер, не заимствованные данные callback. */
void stream_media_receiver_destroy(struct stream_media_receiver *r) {
if (!r) return;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "media receiver destroy pending=%u waiting_key=%d", r->used, r->waiting_key);
u_free(r->data); u_free(r);
}
src/stream/stream_media.h
+61
View File
@@ -0,0 +1,61 @@
/* Медиаблок внутри одной FEC-пачки: big-endian, uAV1, fixed header(32),
* video descriptors(32 each), audio descriptors(16 each), video bytes, audio bytes.
* video descriptor: frame_id:8, PTS_us:8, frame_size:4, fragment_offset:4, length:4, flags:1, reserved:3.
* audio descriptor: PTS_us:8, length:4, samples:2, reserved:2. Opus: 48kHz mono, 960 samples.
* H.264 Annex B; полный ключевой кадр включает SPS/PPS. Конфигурация фиксирована на source_epoch.
* first_frame_offset относится к video bytes; UINT32_MAX означает отсутствие начала кадра.
* Parser проверяет весь блок до публикации view. Receiver собирает один кадр, после GAP ждёт KEY.
* Все вызовы одного контекста последовательны; callbacks заимствуют данные до возврата, без reentry. */
#ifndef STREAM_MEDIA_H
#define STREAM_MEDIA_H
#include "stream_proto.h"
#ifdef __cplusplus
extern "C" {
#endif
#define STREAM_MEDIA_HEADER_SIZE 32
#define STREAM_MEDIA_VIDEO_SIZE 32
#define STREAM_MEDIA_AUDIO_SIZE 16
#define STREAM_MEDIA_MAX_ENTRIES 16
#define STREAM_MEDIA_MAX_FRAME (2u * 1024u * 1024u)
#define STREAM_MEDIA_MAX_BLOCK 65536u
#define STREAM_MEDIA_MAX_OPUS 1275u
#define STREAM_MEDIA_KEY 1
#define STREAM_MEDIA_H264 1
#define STREAM_MEDIA_OPUS 1
struct stream_media_format { uint16_t width, height, rotation; uint8_t video_codec, audio_codec; };
struct stream_media_video {
uint64_t frame_id, pts_us;
uint32_t frame_size, offset, length;
uint8_t flags;
const uint8_t *data;
};
struct stream_media_audio { uint64_t pts_us; uint32_t length; uint16_t samples; const uint8_t *data; };
struct stream_media_block {
struct stream_media_format format;
uint32_t first_frame_offset;
unsigned video_count, audio_count;
struct stream_media_video video[STREAM_MEDIA_MAX_ENTRIES];
struct stream_media_audio audio[STREAM_MEDIA_MAX_ENTRIES];
};
/* Encode не меняет output/length при отказе; данные не перекрывают output. */
int stream_media_encode(const struct stream_media_block *block, uint8_t *output, size_t capacity, size_t *length);
int stream_media_parse(const uint8_t *data, size_t length, struct stream_media_block *block);
struct stream_media_receiver;
/* video=NULL означает один Opus-пакет; audio=NULL — полный H.264 кадр. */
typedef void (*stream_media_frame_fn)(void *arg, const struct stream_media_format *format,
const struct stream_media_video *video, const struct stream_media_audio *audio);
int stream_media_receiver_create(struct stream_media_receiver **out, stream_media_frame_fn callback, void *arg);
void stream_media_receiver_gap(struct stream_media_receiver *receiver);
int stream_media_receiver_feed(struct stream_media_receiver *receiver, const uint8_t *data, size_t length);
void stream_media_receiver_destroy(struct stream_media_receiver *receiver);
#ifdef __cplusplus
}
#endif
#endif