diff --git a/doc/stream.txt b/doc/stream.txt index 3f9b4bec..afd1ebcc 100644 --- a/doc/stream.txt +++ b/doc/stream.txt @@ -720,7 +720,7 @@ Restart в той же source_epoch сохраняет consumer cursor: bootstra fixed header (32 байта): magic="uAV1":4, video_codec:1, audio_codec:1, video_count:1, audio_count:1, width:2, height:2, rotation:2, header_size:2, video_size:4, audio_size:4, - first_video_frame_offset:4, config_revision=1:4. + first_video_frame_offset:4, config_revision:4 (ненулевой, возрастает при пересоздании видеозахвата). Далее video_count описаний по 32 байта, audio_count описаний по 16 байт, затем video_size видеобайтов и audio_size аудиобайтов. Описания задают последовательные неперекрывающиеся диапазоны внутри соответствующего трека; сумма длин должна совпадать с header. @@ -735,7 +735,9 @@ first_video_frame_offset относительно начала video bytes: UINT таблица описывает каждое. Данные конфигурации не являются отдельным кадром. KEY=bit0 video flags; остальные биты запрещены. frame_id>0, <=16 описаний каждого трека. Один H.264 кадр <=2MiB. Общий формат допускает медиаблок <=64KiB; реальный максимум дополнительно -ограничен входом RS. Конфигурация, разрешение, поворот и набор треков фиксированы на source_epoch. +ограничен входом RS. Аудиокодек фиксирован на source_epoch. Разрешение, поворот и SPS/PPS видео меняются по config_revision. +Новая revision может пропускать номера; старая игнорируется. Изменение metadata без новой revision запрещено. +При переходе receiver удаляет неполный видеокадр и ждёт полный KEY, сохраняя историю PTS/кадров и аудио. video_codec=1: AVC/H.264 Annex B, без B-кадров. Каждый KEY содержит SPS, PPS и IDR в одном логическом кадре; это позволяет позднему viewer настроить декодер после получения полного KEY. @@ -751,8 +753,11 @@ Android: - Один foreground publisher на процесс, один viewer. Каждая операция имеет новый session ID. - StreamService владеет камерой, микрофоном и PARTIAL_WAKE_LOCK; пересоздание Activity не завершает источник. Сервис camera|microphone запускается из видимого приложения после runtime-разрешений; без микрофона — camera. -- Camera2 TEMPLATE_RECORD пишет в Surface AVC encoder. Выбирается совместимое разрешение <=1280x720, - 30fps, AVC baseline, 1–2Mbit/s, I-frame interval=1s. Выход нормализуется в Annex B; +- Camera2 TEMPLATE_RECORD пишет в Surface AVC encoder. Режимы камеры <=1920x1080 пересекаются с возможностями + конкретного AVC baseline Surface encoder. Обычные 30/60fps требуют диапазона AE и подходящей minFrameDuration; + скоростные 120/240fps требуют фиксированного диапазона getHighSpeedVideoFpsRangesFor, скоростной сессии, + createHighSpeedRequestList и setRepeatingBurst. Битрейт зависит от размера/FPS, ограничен 16Mbit/s и codec caps. + I-frame interval=1s. Выход нормализуется в Annex B; partial output собирается в access unit. Codec config передаётся с каждым IDR. - Camera MediaCodec Surface timestamps и AudioTimestamp.TIMEBASE_MONOTONIC приводятся к общему origin System.nanoTime; время callback/получения блока не заменяет capture PTS. @@ -766,13 +771,18 @@ Android: При video pressure отбрасываются целые ожидающие кадры, P-кадры запрещены до нового KEY. Кодировщик получает отказ admission и запрашивает KEY с ограничением частоты 500ms. - RX<=4MiB/256 records. Pressure очищает media, сохраняет control results и сообщает GAP. - Kotlin viewer дополнительно ограничивает video queue 12 кадрами/2MiB, audio queue 16 пакетами. + Kotlin viewer ограничивает video queue 250ms/128 кадрами/2MiB, audio queue 16 пакетами. + За проход подаётся до 8 видео-кадров, выход декодера обслуживается до 16 раз. Нельзя выбросить reference P-кадр и продолжить с зависимыми: переполнение сбрасывает декодер до KEY. Просроченный кадр при нормальной последовательности декодируется без показа. - После prebuffer 150ms AudioTrack clock определяет время показа видео; видео без аудио использует монотонные часы. Короткие audio gaps заполняются Opus PLC, большой разрыв сбрасывает timeline. Локальный preview использует обычную подписку на собственный источник с отключённым воспроизведением - своего микрофона, чтобы исключить акустическую обратную связь. Камера при закрытии preview продолжает работать. + своего микрофона, чтобы исключить акустическую обратную связь. Локальный preview ориентируется по сенсору своей камеры + и текущему положению экрана; ручной поворот трансляции меняет только метку «Верх», показывающую верх картинки зрителей. + Положение камеры хранится per-revision отдельно от wire, чтобы queued кадры не использовали параметры новой камеры. + На SM проверено: при ручных 0/90/180/270 размеры и ориентация превью сохраняются, меняется только положение метки. + Удалённый viewer применяет video rotation источника без локальной метки. Камера при закрытии preview продолжает работать. Viewer закрывается на Activity ON_STOP и создаётся заново на ON_START; publisher от UI не зависит. Видимый player удерживает экран включённым. Preview ограничен оставшимся местом, кнопки управления не перекрывает. - Source list общий для Android/desktop: stream_sources_json в uasync использует stream_member_load @@ -785,7 +795,19 @@ Android: несколько выбираются из списка по имени/ID. Пункт меню «Трансляции» также открывает управление публикацией. Выбранная эпоха проверяется actor до подписки и по фактически открытому handle. Перезапуск источника требует нового выбора. Ошибка viewer kind=6 передаёт stream_close_reason в flags. -- Stop сначала завершает capture producers и encoder EOS (максимум 1s), затем FIN. +- Ручная ориентация доступна до запуска и во время своей трансляции: 0/90/180/270 градусов относительно + обычного положения устройства. Поворот не зависит от текущего положения экрана; SENSOR_ORIENTATION и направление + камеры учитываются при расчёте video rotation. Камера/режим/ориентация хранятся и подтверждаются одной конфигурацией. + При смене камеры выбранная ориентация сохраняется; rollback восстанавливает всю предыдущую конфигурацию. +- Смена камеры/режима/ориентации сохраняет publisher owner, source_epoch, микрофон, origin и frame IDs. + Camera callbacks и codec output отсеиваются по поколению. Новая камера открывается после onClosed предыдущей; + close timeout=3s, KEY/подтверждение режима timeout=8s. + Скоростной режим подтверждается после 3s фактического encode rate >=80% целевой FPS; до этого lastGood не заменяется. + При ошибке — одна попытка восстановления последнего успешного режима. + Новая конфигурация и первый KEY принимаются native bridge атомарно; старая очередь видео удаляется, аудио сохраняется. + Каждая запись JNI pull имеет header 24 bytes: kind:1, flags:1, width:2, height:2, rotation:2, PTS_us:8, length:4, revision:4. + Kotlin и desktop viewer пересоздают только видеодекодер при смене revision; audio clock не сбрасывается. +- Stop сначала закрывает capture producers и encoder, затем FIN. Publisher сохраняет конечный буфер ещё 2s для сетевой выдачи, затем закрывается. Это ограниченный drain, не подтверждение воспроизведения всеми viewers. Abort/core restart очищает очередь сразу. Поздние команды старого session ID не меняют новый запуск. @@ -796,7 +818,21 @@ Android: Kotlin progress — DEBUG раз в секунду; C packet-level диагностика — TRACE по запросу. Проверки: test_stream_media (все пары потерь 6+2, parser/границы/GAP), test_android_stream (живое ядро, preview, verified snapshots, FIN/cancel, concurrent producer/detach и leak count), -Android H264AnnexBTest (Annex B/length-prefixed NAL, SPS/PPS/IDR и усечённые данные). +Android H264AnnexBTest (Annex B/length-prefixed NAL, SPS/PPS/IDR и усечённые данные), +StreamCameraModesTest (точный режим новой камеры, снижение FPS и fallback 720p30), +StreamOrientationTest (четыре положения переднего/заднего сенсора, включая SENSOR_ORIENTATION=0), +StreamPreviewTest (независимое превью, верх передней/задней камеры, экран и camera switch при queued frames). +Ручной поворот без изменения размера также проверяется test_android_stream с непрерывным аудио. +C-тесты проверяют новую revision, потерю первого KEY, stale revision, сохранение аудио и source_epoch. +Проверка 2026-10-12 на SM-A525F / LineageOS 23.2-20260622-NIGHTLY-a52q: +переключение передней/задней камеры, 720p/1080p30, выход/возврат на экран и stop работают. +Ручные 0/90/180/270 градусов проверены на SM до запуска и во время трансляции; viewer сохраняет подписку, +ориентация превью совпадает с камерой, метка «Верх» меняется вместе с ручным video rotation. После смены камеры и повторного открытия экрана угол сохраняется. +Проверен rollback отклонённого 120fps режима: прежние 30fps и ручные 270 градусов восстанавливаются вместе. +Заявленные 720p120/240 не прошли подтверждение скорости: при 720p120 задняя камера даёт около 3.8fps, +передняя — около 30fps. Camera HAL при [120,120] на задней камере возвращает SENSOR_FRAME_DURATION=266666666ns. +Проверка нового режима отвергает такую скорость и восстанавливает прежний режим с уведомлением. +Это наблюдение относится к данной прошивке; metadata Camera2 сами по себе не подтверждают рабочую скорость. Desktop: - Значок трансляций расположен в шапке MessageList. Источники берутся из того же каталога ядра; @@ -815,6 +851,6 @@ Desktop: Закрытие окна снимает только reader, собственная публикация сохраняется. Удаление канала сообщает STREAM_CLOSE_GROUP; остановка ядра снимает media и catalogue owners. - Без FFmpeg окно сообщает, что декодирование недоступно. Нужна desktop-сборка с HAVE_FFMPEG. -Проверки: test_stream_view — настоящее ядро, старые эпохи/cancel, RX pressure, H.264 с поворотом, +Проверки: test_stream_view — настоящее ядро, старые эпохи/cancel, RX pressure, H.264 с поворотом и сменой revision без новой эпохи, Opus/громкость на настоящем null audio backend, закрытие канала и полный баланс tracked allocations. test_stream_service также проверяет общий каталог двух независимых источников одного канала. diff --git a/src/stream/stream_media.c b/src/stream/stream_media.c index 34808a07..ed5a8a58 100644 --- a/src/stream/stream_media.c +++ b/src/stream/stream_media.c @@ -18,7 +18,7 @@ struct stream_media_receiver { /* Проверить весь набор описаний до чтения payload или выделения памяти под кадр. */ static int media_validate(const struct stream_media_block *b, size_t *video_size, size_t *audio_size, uint32_t *first) { if (!b || b->video_count > STREAM_MEDIA_MAX_ENTRIES || b->audio_count > STREAM_MEDIA_MAX_ENTRIES - || (!b->video_count && !b->audio_count) || b->format.video_codec != STREAM_MEDIA_H264 + || !b->format.revision || (!b->video_count && !b->audio_count) || b->format.video_codec != STREAM_MEDIA_H264 || b->format.audio_codec > STREAM_MEDIA_OPUS || !b->format.width || b->format.width > 4096 || !b->format.height || b->format.height > 4096 || b->format.rotation > 270 || b->format.rotation % 90) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media format/count invalid present=%d video=%u audio=%u codecs=%u/%u dimensions=%ux%u rotation=%u", @@ -82,7 +82,7 @@ int stream_media_encode(const struct stream_media_block *b, uint8_t *output, siz stream_write_be(output + 8, b->format.width, 2); stream_write_be(output + 10, b->format.height, 2); stream_write_be(output + 12, b->format.rotation, 2); stream_write_be(output + 14, header, 2); stream_write_be(output + 16, video_size, 4); stream_write_be(output + 20, audio_size, 4); - stream_write_be(output + 24, first, 4); stream_write_be(output + 28, 1, 4); + stream_write_be(output + 24, first, 4); stream_write_be(output + 28, b->format.revision, 4); size_t payload = header, entry = STREAM_MEDIA_HEADER_SIZE; for (unsigned i = 0; i < b->video_count; i++, entry += STREAM_MEDIA_VIDEO_SIZE) { const struct stream_media_video *v = &b->video[i]; @@ -108,12 +108,12 @@ int stream_media_parse(const uint8_t *data, size_t length, struct stream_media_b } struct stream_media_block b = {.video_count = data[6], .audio_count = data[7]}; b.format = (struct stream_media_format){(uint16_t)stream_read_be(data + 8, 2), (uint16_t)stream_read_be(data + 10, 2), - (uint16_t)stream_read_be(data + 12, 2), data[4], data[5]}; + (uint16_t)stream_read_be(data + 12, 2), data[4], data[5], (uint32_t)stream_read_be(data + 28, 4)}; size_t header = STREAM_MEDIA_HEADER_SIZE + b.video_count * STREAM_MEDIA_VIDEO_SIZE + b.audio_count * STREAM_MEDIA_AUDIO_SIZE; size_t video_size = (size_t)stream_read_be(data + 16, 4), audio_size = (size_t)stream_read_be(data + 20, 4); if (b.video_count > STREAM_MEDIA_MAX_ENTRIES || b.audio_count > STREAM_MEDIA_MAX_ENTRIES || header > length || stream_read_be(data + 14, 2) != header || video_size > length - header || audio_size != length - header - video_size - || stream_read_be(data + 28, 4) != 1) { + || !b.format.revision) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media parse header/regions invalid length=%zu header=%zu video=%zu audio=%zu", length, header, video_size, audio_size); return STREAM_ERR_FORMAT; } @@ -175,13 +175,23 @@ int stream_media_receiver_feed(struct stream_media_receiver *r, const uint8_t *d struct stream_media_block b; int result = stream_media_parse(data, length, &b); if (result != STREAM_OK) { stream_media_receiver_gap(r); return result; } - if (r->configured && (r->format.width != b.format.width || r->format.height != b.format.height - || r->format.rotation != b.format.rotation || r->format.audio_codec != b.format.audio_codec)) { - DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media format changed within source epoch dimensions=%ux%u/%ux%u rotation=%u/%u audio=%u/%u", - r->format.width, r->format.height, b.format.width, b.format.height, + if (r->configured && b.format.revision < r->format.revision) { + DEBUG_WARN(DEBUG_CATEGORY_STREAM, "media stale revision=%u current=%u", b.format.revision, r->format.revision); + return STREAM_OK; + } + if (r->configured && (r->format.audio_codec != b.format.audio_codec + || (r->format.revision == b.format.revision && (r->format.width != b.format.width + || r->format.height != b.format.height || r->format.rotation != b.format.rotation)))) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "media conflicting configuration revision=%u/%u dimensions=%ux%u/%ux%u rotation=%u/%u audio=%u/%u", + r->format.revision, b.format.revision, r->format.width, r->format.height, b.format.width, b.format.height, r->format.rotation, b.format.rotation, r->format.audio_codec, b.format.audio_codec); stream_media_receiver_gap(r); return STREAM_ERR_CONFLICT; } + if (r->configured && b.format.revision > r->format.revision) { + DEBUG_INFO(DEBUG_CATEGORY_STREAM, "media video configuration revision=%u->%u dimensions=%ux%u rotation=%u; preserve audio", + r->format.revision, b.format.revision, b.format.width, b.format.height, b.format.rotation); + stream_media_receiver_gap(r); + } r->format = b.format; r->configured = 1; for (unsigned i = 0; i < b.video_count; i++) { const struct stream_media_video *v = &b.video[i]; diff --git a/src/stream/stream_media.h b/src/stream/stream_media.h index b61d26de..7466212c 100644 --- a/src/stream/stream_media.h +++ b/src/stream/stream_media.h @@ -2,7 +2,7 @@ * video descriptors(32 each), audio descriptors(16 each), video bytes, audio bytes. * video descriptor: frame_id:8, PTS_us:8, frame_size:4, fragment_offset:4, length:4, flags:1, reserved:3. * audio descriptor: PTS_us:8, length:4, samples:2, reserved:2. Opus: 48kHz mono, 960 samples. - * H.264 Annex B; полный ключевой кадр включает SPS/PPS. Конфигурация фиксирована на source_epoch. + * H.264 Annex B; полный ключевой кадр включает SPS/PPS. Видео меняется по revision; аудиотрек фиксирован на source_epoch. * first_frame_offset относится к video bytes; UINT32_MAX означает отсутствие начала кадра. * Parser проверяет весь блок до публикации view. Receiver собирает один кадр, после GAP ждёт KEY. * Все вызовы одного контекста последовательны; callbacks заимствуют данные до возврата, без reentry. */ @@ -26,7 +26,7 @@ extern "C" { #define STREAM_MEDIA_H264 1 #define STREAM_MEDIA_OPUS 1 -struct stream_media_format { uint16_t width, height, rotation; uint8_t video_codec, audio_codec; }; +struct stream_media_format { uint16_t width, height, rotation; uint8_t video_codec, audio_codec; uint32_t revision; }; struct stream_media_video { uint64_t frame_id, pts_us; uint32_t frame_size, offset, length; diff --git a/tests/test_android_stream.c b/tests/test_android_stream.c index 49416a27..47dcb8c4 100644 --- a/tests/test_android_stream.c +++ b/tests/test_android_stream.c @@ -13,10 +13,15 @@ #include "../src/utun_instance.h" #include "../tools/chatgui-android/libutun_lite/stream_bridge.h" #include "../src/stream/stream_proto.h" +#include "../src/stream/stream_media.h" #include "test_clock.h" #define CHECK(x) do { if (!(x)) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "Android stream test line=%d: %s", __LINE__, #x); abort(); } } while (0) static uint8_t video[9000]; +static struct stream_media_format current_format = {1280, 720, 90, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS, 1}; +static int feed(uint64_t id, int is_video, uint64_t pts, int key, const uint8_t *data, size_t length) { + return android_stream_feed(id, is_video, pts, key, data, length, is_video ? ¤t_format : NULL); +} static struct UASYNC *ua; static int producer_started, producer_stop; static unsigned source_events; @@ -45,7 +50,7 @@ static void *producer(void *arg) { __atomic_store_n(&producer_started, 1, __ATOMIC_RELEASE); uint64_t pts = 500000; while (!__atomic_load_n(&producer_stop, __ATOMIC_ACQUIRE)) { - int result = android_stream_feed(3, 1, pts++, 1, video, sizeof(video)); + int result = feed(3, 1, pts++, 1, video, sizeof(video)); if (result < 0) break; } return NULL; @@ -80,28 +85,74 @@ int main(void) { u_free(wait_event(2, 4, &size)); CHECK(source_events == 2); /* Подписка не меняет каталог публикаций. */ for (unsigned i = 0; i < sizeof(video); i++) video[i] = (uint8_t)i; - CHECK(android_stream_feed(1, 1, 100000, 0, video, sizeof(video)) == 0); - CHECK(android_stream_feed(1, 1, 100000, 1, video, sizeof(video)) == 1); - CHECK(android_stream_feed(1, 0, 100000, 0, (const uint8_t *)"ops", 3) == 1); + CHECK(feed(1, 1, 100000, 0, video, sizeof(video)) == 0); + CHECK(feed(1, 1, 100000, 1, video, sizeof(video)) == 1); + CHECK(feed(1, 0, 100000, 0, (const uint8_t *)"ops", 3) == 1); uint8_t *output = NULL; int video_seen = 0, audio_seen = 0; for (unsigned i = 0; i < 1000 && (!video_seen || !audio_seen); i++) { output = android_stream_pull(2, &size); if (output && output[0] == 1) { - CHECK(size == sizeof(video) + 20 && stream_read_be(output + 8, 8) == 100000 && !memcmp(output + 20, video, sizeof(video))); + CHECK(size == sizeof(video) + ANDROID_STREAM_HEADER_SIZE && stream_read_be(output + 8, 8) == 100000 && !memcmp(output + ANDROID_STREAM_HEADER_SIZE, video, sizeof(video))); video_seen++; - } else if (output && output[0] == 2) { CHECK(size == 23 && !memcmp(output + 20, "ops", 3)); audio_seen++; } + } else if (output && output[0] == 2) { CHECK(size == 3 + ANDROID_STREAM_HEADER_SIZE && !memcmp(output + ANDROID_STREAM_HEADER_SIZE, "ops", 3)); audio_seen++; } + u_free(output); test_clock_poll(ua, 100); + } + CHECK(video_seen == 1 && audio_seen == 1); + /* Commit новой revision удаляет старое видео, сохраняет аудио/epoch и не требует нового owner. */ + CHECK(feed(1, 1, 200000, 0, video, sizeof(video)) == 1); + CHECK(feed(1, 0, 120000, 0, (const uint8_t *)"ops", 3) == 1); + struct stream_media_format next_format = {1920, 1080, 270, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS, 2}; + CHECK(android_stream_feed(1, 1, 300000, 0, video, sizeof(video), &next_format) == 0); + CHECK(android_stream_feed(1, 1, 300000, 1, video, sizeof(video), &next_format) == 1); + CHECK(feed(1, 1, 350000, 1, video, sizeof(video)) == 0); + next_format.width++; + CHECK(android_stream_feed(1, 1, 400000, 1, video, sizeof(video), &next_format) == -1); + next_format.width--; + video_seen = audio_seen = 0; + for (unsigned i = 0; i < 1000 && (!video_seen || !audio_seen); i++) { + output = android_stream_pull(2, &size); + if (output && output[0] == 1) { + CHECK(stream_read_be(output + 8, 8) == 300000 && stream_read_be(output + 20, 4) == 2); + CHECK(stream_read_be(output + 2, 2) == 1920 && stream_read_be(output + 6, 2) == 270); video_seen++; + } else if (output && output[0] == 2) { CHECK(stream_read_be(output + 8, 8) == 120000); audio_seen++; } + else if (output) CHECK(output[0] != 6); u_free(output); test_clock_poll(ua, 100); } CHECK(video_seen == 1 && audio_seen == 1); + /* Ручной поворот при том же разрешении: новый KEY, непрерывное аудио и тот же viewer. */ + const uint16_t rotations[] = {0, 90, 180, 270}; + for (unsigned turn = 0; turn < 4; turn++) { + next_format.revision++; next_format.rotation = rotations[turn]; + uint64_t video_pts = 400000 + turn * 100000, audio_pts = 140000 + turn * 20000; + CHECK(android_stream_feed(1, 1, video_pts, 0, video, sizeof(video), &next_format) == 0); + CHECK(android_stream_feed(1, 1, video_pts, 1, video, sizeof(video), &next_format) == 1); + CHECK(feed(1, 0, audio_pts, 0, (const uint8_t *)"ops", 3) == 1); + video_seen = audio_seen = 0; + for (unsigned i = 0; i < 1000 && (!video_seen || !audio_seen); i++) { + output = android_stream_pull(2, &size); + if (output && output[0] == 1) { + CHECK(stream_read_be(output + 8, 8) == video_pts && stream_read_be(output + 20, 4) == next_format.revision); + CHECK(stream_read_be(output + 2, 2) == 1920 && stream_read_be(output + 4, 2) == 1080); + CHECK(stream_read_be(output + 6, 2) == rotations[turn] && (output[1] & STREAM_MEDIA_KEY)); video_seen++; + } else if (output && output[0] == 2) { CHECK(stream_read_be(output + 8, 8) == audio_pts); audio_seen++; } + else if (output) CHECK(output[0] != 6); + u_free(output); test_clock_poll(ua, 100); + } + CHECK(video_seen == 1 && audio_seen == 1); + } + struct stream_member_state after_switch; + CHECK(stream_member_load(inst, group, inst->node_id, &after_switch, NULL) == STREAM_OK); + CHECK(after_switch.published && after_switch.source.source_epoch == member.source.source_epoch); android_stream_stop(1); u_free(wait_event(2, 5, &size)); android_stream_stop(2); u_free(wait_event(1, 5, &size)); while ((output = android_stream_pull(2, &size))) u_free(output); CHECK(!strcmp(source_snapshot, "[]")); + current_format.rotation = 0; current_format.audio_codec = 0; CHECK(android_stream_open(3, group, 0, 1280, 720, 0, 0, 0) == 0); u_free(wait_event(3, 4, &size)); pthread_t thread; CHECK(pthread_create(&thread, NULL, producer, NULL) == 0); while (!__atomic_load_n(&producer_started, __ATOMIC_ACQUIRE)) {} android_stream_detach(inst); __atomic_store_n(&producer_stop, 1, __ATOMIC_RELEASE); CHECK(pthread_join(thread, NULL) == 0); - CHECK(android_stream_feed(3, 1, 900000, 1, video, sizeof(video)) == -1); + CHECK(feed(3, 1, 900000, 1, video, sizeof(video)) == -1); while ((output = android_stream_pull(3, &size))) u_free(output); android_stream_attach(inst); CHECK(android_stream_open(4, group, 0, 1280, 720, 0, 0, 0) == 0); android_stream_stop(4); diff --git a/tests/test_stream_media.c b/tests/test_stream_media.c index eda126ce..3a4713f0 100644 --- a/tests/test_stream_media.c +++ b/tests/test_stream_media.c @@ -8,7 +8,7 @@ #include "../src/stream/stream_source.h" #include "../src/stream/stream_media.h" -#define CHECK(x) do { if (!(x)) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "stream media test line=%d: %s", __LINE__, #x); exit(1); } } while (0) +#define CHECK(x) do { if (!(x)) { debug_set_category_level(DEBUG_CATEGORY_STREAM, DEBUG_LEVEL_ERROR); DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "stream media test line=%d: %s", __LINE__, #x); exit(1); } } while (0) static unsigned video_frames, audio_frames, batches; static uint8_t expected_video[9000]; @@ -26,7 +26,7 @@ static void ready(struct stream_source *source, uint64_t seq, void *arg) { static void test_media(void) { struct stream_media_receiver *receiver = NULL; CHECK(stream_media_receiver_create(&receiver, frame, NULL) == STREAM_OK); - struct stream_media_block b = {.format = {1280, 720, 90, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS}, .video_count = 1, .audio_count = 1}; + struct stream_media_block b = {.format = {1280, 720, 90, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS, 1}, .video_count = 1, .audio_count = 1}; b.video[0] = (struct stream_media_video){1, 1000000, sizeof(expected_video), 0, 6000, STREAM_MEDIA_KEY, expected_video}; b.audio[0] = (struct stream_media_audio){1000000, 3, 960, (const uint8_t *)"ops"}; uint8_t wire[7080], saved[7080]; size_t length = 0; @@ -35,7 +35,7 @@ static void test_media(void) { memcpy(saved, wire, length); struct stream_media_block parsed; CHECK(stream_media_parse(wire, length, &parsed) == STREAM_OK && parsed.first_frame_offset == 0); - const unsigned mutations[] = {0, 4, 5, 6, 7, 8, 12, 14, 16, 20, 24, 28, 48, 52, 56, 60, 61, 72, 76, 78}; + const unsigned mutations[] = {0, 4, 5, 6, 7, 8, 12, 14, 16, 20, 24, 48, 52, 56, 60, 61, 72, 76, 78}; for (unsigned i = 0; i < sizeof(mutations) / sizeof(mutations[0]); i++) { memcpy(wire, saved, length); wire[mutations[i]] = 255; struct stream_media_block before; memset(&before, 0xa5, sizeof(before)); parsed = before; @@ -81,7 +81,7 @@ static void test_batches(void) { .block_size = STREAM_RS_DEFAULT_SIZE}; uint8_t key[32] = {1}; struct stream_encoder *encoder = NULL; CHECK(stream_encoder_create(&encoder, &d, key) == STREAM_OK); - struct stream_media_block b = {.format = {1280, 720, 90, STREAM_MEDIA_H264, 0}, .video_count = 1}; + struct stream_media_block b = {.format = {1280, 720, 90, STREAM_MEDIA_H264, 0, 1}, .video_count = 1}; b.video[0] = (struct stream_media_video){1, 1000000, 100, 0, 100, STREAM_MEDIA_KEY, expected_video}; uint8_t input[7080], output[7080]; size_t length = 0; CHECK(stream_media_encode(&b, input, sizeof(input), &length) == STREAM_OK); @@ -114,13 +114,76 @@ static void test_batches(void) { DEBUG_INFO(DEBUG_CATEGORY_STREAM, "PASS batches: atomic boundaries, all FEC losses and mode isolation"); } +static unsigned revised_videos, revised_audios; +static uint32_t received_revision; +static void revised_frame(void *arg, const struct stream_media_format *format, + const struct stream_media_video *video, const struct stream_media_audio *audio) { + (void)arg; + if (video) { revised_videos++; received_revision = format->revision; CHECK(video->flags & STREAM_MEDIA_KEY); } + if (audio) revised_audios++; +} + +/* Revision переживает потерю первого IDR; смена видео не сбрасывает PTS аудио и кадров. */ +static void test_revisions(void) { + struct stream_media_receiver *r = NULL; + CHECK(stream_media_receiver_create(&r, revised_frame, NULL) == STREAM_OK); + struct stream_media_block b = {.format = {1280, 720, 90, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS, 1}, .video_count = 1}; + b.video[0] = (struct stream_media_video){1, 1000000, 12, 0, 6, STREAM_MEDIA_KEY, expected_video}; + uint8_t wire[7080]; size_t length; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_videos == 0); + b.format = (struct stream_media_format){1920, 1080, 270, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS, 2}; + b.video_count = 0; b.audio_count = 1; + b.audio[0] = (struct stream_media_audio){1000000, 3, 960, (const uint8_t *)"ops"}; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_audios == 1); + b.video_count = 1; b.audio_count = 0; + b.video[0] = (struct stream_media_video){2, 1100000, 12, 0, 12, 0, expected_video}; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_videos == 0); + b.video[0] = (struct stream_media_video){3, 1200000, 12, 0, 6, STREAM_MEDIA_KEY, expected_video}; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK); + stream_media_receiver_gap(r); + b.video[0].offset = 6; b.video[0].data = expected_video + 6; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_videos == 0); + b.video[0] = (struct stream_media_video){4, 1300000, 12, 0, 12, STREAM_MEDIA_KEY, expected_video}; + b.audio_count = 1; b.audio[0].pts_us = 1020000; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_videos == 1 && revised_audios == 2 && received_revision == 2); + b.format.revision = 1; b.video[0].frame_id = 5; b.video[0].pts_us = 1400000; + b.audio[0].pts_us = 1040000; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_videos == 1 && revised_audios == 2); + b.format.revision = 4; b.video[0].flags = 0; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_videos == 1 && revised_audios == 3); + b.video[0].frame_id = 6; b.video[0].pts_us = 1500000; b.video[0].flags = STREAM_MEDIA_KEY; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_videos == 2 && received_revision == 4 && revised_audios == 3); + b.format.width++; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_ERR_CONFLICT); + b.format.width--; b.format.revision = 5; b.format.audio_codec = 0; b.audio_count = 0; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_ERR_CONFLICT); + b.format.audio_codec = STREAM_MEDIA_OPUS; b.format.revision = 6; b.video[0].pts_us = 1400000; + CHECK(stream_media_encode(&b, wire, sizeof(wire), &length) == STREAM_OK); + CHECK(stream_media_receiver_feed(r, wire, length) == STREAM_OK && revised_videos == 2); + memset(wire + 28, 0, 4); + struct stream_media_block parsed; + CHECK(stream_media_parse(wire, length, &parsed) == STREAM_ERR_FORMAT); + stream_media_receiver_destroy(r); +} + int main(void) { debug_config_init(); debug_set_level(DEBUG_LEVEL_INFO); /* Усечения проверяются массово; ожидаемые отказы не загромождают диагностический лог. */ debug_set_category_level(DEBUG_CATEGORY_STREAM, DEBUG_LEVEL_DISABLED); size_t before = u_get_allocated_count(); for (unsigned i = 0; i < sizeof(expected_video); i++) expected_video[i] = (uint8_t)(i * 31 + 7); - test_media(); test_batches(); + test_media(); test_batches(); test_revisions(); CHECK(u_get_allocated_count() == before); debug_set_category_level(DEBUG_CATEGORY_STREAM, DEBUG_LEVEL_INFO); DEBUG_INFO(DEBUG_CATEGORY_STREAM, "PASS stream_media: protocol, FEC, frame recovery, boundaries, no leaks"); diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CameraLensManager.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CameraLensManager.kt index 4adf44a4..877d0a9a 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CameraLensManager.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CameraLensManager.kt @@ -4,6 +4,7 @@ import com.utun.chat.R import android.graphics.ImageFormat import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraMetadata +import android.hardware.camera2.CameraManager import android.util.Size import androidx.camera.camera2.interop.Camera2CameraInfo import androidx.camera.core.CameraSelector @@ -48,28 +49,36 @@ object CameraLensManager { val fixedFocus: Boolean ) - fun enumerate(provider: ProcessCameraProvider): List { - val result = mutableListOf() - val backs = mutableListOf() + private data class LensInfo(val id: String, val facing: Int, val eq: Float, val minFocus: Float) - for (info in provider.availableCameraInfos) { - val c2 = Camera2CameraInfo.from(info) - val cameraId = c2.cameraId - val facing = c2.getCameraCharacteristic(CameraCharacteristics.LENS_FACING) - ?: CameraMetadata.LENS_FACING_BACK + fun enumerate(provider: ProcessCameraProvider): List = enumerate(provider.availableCameraInfos.map { info -> + val c2 = Camera2CameraInfo.from(info) + LensInfo(c2.cameraId, c2.getCameraCharacteristic(CameraCharacteristics.LENS_FACING) ?: CameraMetadata.LENS_FACING_BACK, + eqFocal35(c2), c2.getCameraCharacteristic(CameraCharacteristics.LENS_INFO_MINIMUM_FOCUS_DISTANCE) ?: 0f) + }) - if (facing == CameraMetadata.LENS_FACING_FRONT) { - val eq = eqFocal35(c2) - result.add(LensOption(LensType.SELFIE, cameraId, eq)) - continue - } - if (facing != CameraMetadata.LENS_FACING_BACK) continue + fun enumerate(manager: CameraManager): List = enumerate(manager.cameraIdList.mapNotNull { id -> + try { + val c = manager.getCameraCharacteristics(id) + val focal = c.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS)?.firstOrNull() ?: 0f + val sensor = c.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE) + val diagonal = sensor?.let { sqrt(it.width * it.width + it.height * it.height) } ?: 0f + LensInfo(id, c.get(CameraCharacteristics.LENS_FACING) ?: CameraMetadata.LENS_FACING_BACK, + if (diagonal > 0f) focal * 43.27f / diagonal else focal, + c.get(CameraCharacteristics.LENS_INFO_MINIMUM_FOCUS_DISTANCE) ?: 0f) + } catch (e: Exception) { LogManager.addLog("ERROR", "Lens", "camera enumeration id=$id: ${e.message}"); null } + }) - val eq = eqFocal35(c2) - val minFocus = c2.getCameraCharacteristic(CameraCharacteristics.LENS_INFO_MINIMUM_FOCUS_DISTANCE) ?: 0f - val fixedFocus = minFocus == 0f - backs.add(BackLens(cameraId, eq, fixedFocus)) - LogManager.addLog("INFO", "Lens", "back cam id=$cameraId eq=${"%.1f".format(eq)} fixedFocus=$fixedFocus") + private fun enumerate(cameras: List): List { + val result = mutableListOf() + val backs = mutableListOf() + for (info in cameras) { + if (info.facing == CameraMetadata.LENS_FACING_FRONT) { + result.add(LensOption(LensType.SELFIE, info.id, info.eq)); continue + } + if (info.facing != CameraMetadata.LENS_FACING_BACK) continue + backs.add(BackLens(info.id, info.eq, info.minFocus == 0f)) + LogManager.addLog("INFO", "Lens", "back cam id=${info.id} eq=${info.eq} fixedFocus=${info.minFocus == 0f}") } if (backs.isEmpty()) { diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/NativeLib.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/NativeLib.kt index a29cadab..601e96f7 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/NativeLib.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/NativeLib.kt @@ -202,7 +202,9 @@ object NativeLib { rotation: Int = 0, audio: Boolean = true, epoch: Long = 0): Boolean = nativeStreamOpen(id, group, source, width, height, rotation, audio, epoch) fun streamWatch(group: Long): Boolean = nativeStreamWatch(group) - fun streamFeed(id: Long, video: Boolean, pts: Long, key: Boolean, data: ByteArray): Int = nativeStreamFeed(id, video, pts, key, data) + fun streamFeed(id: Long, video: Boolean, pts: Long, key: Boolean, data: ByteArray, revision: Long = 0, + width: Int = 0, height: Int = 0, rotation: Int = 0, audio: Boolean = true): Int = + nativeStreamFeed(id, video, pts, key, data, revision, width, height, rotation, audio) fun streamStop(id: Long) { nativeStreamStop(id) } fun streamPull(id: Long): ByteArray? = nativeStreamPull(id) fun streamOpusCreate(encode: Boolean): Long = nativeStreamOpusCreate(encode) @@ -212,7 +214,8 @@ object NativeLib { private external fun nativeStreamOpen(id: Long, group: Long, source: Long, width: Int, height: Int, rotation: Int, audio: Boolean, epoch: Long): Boolean private external fun nativeStreamWatch(group: Long): Boolean - private external fun nativeStreamFeed(id: Long, video: Boolean, pts: Long, key: Boolean, data: ByteArray): Int + private external fun nativeStreamFeed(id: Long, video: Boolean, pts: Long, key: Boolean, data: ByteArray, + revision: Long, width: Int, height: Int, rotation: Int, audio: Boolean): Int private external fun nativeStreamStop(id: Long) private external fun nativeStreamPull(id: Long): ByteArray? private external fun nativeStreamOpusCreate(encode: Boolean): Long diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCameraModes.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCameraModes.kt new file mode 100644 index 00000000..a81b3d7c --- /dev/null +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCameraModes.kt @@ -0,0 +1,88 @@ +package com.utun.chat.data + +import android.content.Context +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraManager +import android.media.MediaCodecInfo +import android.media.MediaCodecList +import android.media.MediaFormat +import android.os.Build + +/** Режим является пересечением возможностей камеры и конкретного Surface AVC encoder. */ +data class StreamCaptureMode(val camera: String, val width: Int, val height: Int, val fps: Int, + val minimumFps: Int, val highSpeed: Boolean, val codec: String, val bitrate: Int) { + val label get() = "${width}×${height} · $fps fps" +} +data class StreamCamera(val id: String, val lens: LensOption, val modes: List) + +object StreamCameraModes { + fun enumerate(context: Context): List { + val manager = context.getSystemService(CameraManager::class.java) + val codecs = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { codec -> + codec.isEncoder && codec.supportedTypes.any { it.equals(MediaFormat.MIMETYPE_VIDEO_AVC, true) } + } + return CameraLensManager.enumerate(manager).mapNotNull { lens -> + try { + val c = manager.getCameraCharacteristics(lens.cameraId) + val map = c.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) ?: error("Missing stream map") + val output = map.getOutputSizes(android.media.MediaCodec::class.java).orEmpty().toList() + val common = output.filter { it.width <= 1920 && it.height <= 1080 && + (it.width to it.height) in setOf(640 to 480, 1280 to 720, 1920 to 1080) } + val sizes = common.ifEmpty { output.filter { it.width <= 1280 && it.height <= 720 } + .maxByOrNull { it.width * it.height }?.let { listOf(it) }.orEmpty() } + val modes = mutableListOf() + fun add(width: Int, height: Int, fps: Int, lower: Int, highSpeed: Boolean) { + for (codec in codecs) { + try { + if (highSpeed && Build.VERSION.SDK_INT >= 29 && !codec.isHardwareAccelerated) continue + val caps = codec.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC) + val video = caps.videoCapabilities + if (!caps.colorFormats.contains(MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) || + caps.profileLevels.none { it.profile == MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline } || + !video.areSizeAndRateSupported(width, height, fps.toDouble())) continue + val bitrate = (width.toLong() * height * fps / 14).coerceIn(512_000L, 16_000_000L).toInt() + modes.add(StreamCaptureMode(lens.cameraId, width, height, fps, lower, highSpeed, codec.name, + video.bitrateRange.clamp(bitrate))) + LogManager.addLog("INFO", "video", "camera mode camera=${lens.cameraId} ${width}x$height fps=$lower..$fps " + + "highSpeed=$highSpeed codec=${codec.name} bitrate=${modes.last().bitrate} achievable=${video.getAchievableFrameRatesFor(width, height)}") + return + } catch (e: Exception) { LogManager.addLog("WARN", "video", "codec probe ${codec.name}: ${e.message}") } + } + LogManager.addLog("DEBUG", "video", "camera mode rejected: no AVC encoder camera=${lens.cameraId} ${width}x$height fps=$fps") + } + val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() + val normalFps = listOf(30, 60).filter { fps -> rates.any { it.upper == fps } } + .ifEmpty { rates.map { it.upper }.filter { it <= 30 }.maxOrNull()?.let { listOf(it) }.orEmpty() } + for (size in sizes) for (fps in normalFps) { + val duration = map.getOutputMinFrameDuration(android.media.MediaCodec::class.java, size) + if (duration > 0 && duration > 1_000_000_000L / fps + 1) continue + val range = rates.filter { it.upper == fps }.maxByOrNull { it.lower } ?: continue + add(size.width, size.height, fps, range.lower, false) + } + if ((c.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES) ?: intArrayOf()) + .contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_CONSTRAINED_HIGH_SPEED_VIDEO)) { + for (size in map.highSpeedVideoSizes.orEmpty()) { + if (size !in output || size.width > 1920 || size.height > 1080) continue + for (rate in map.getHighSpeedVideoFpsRangesFor(size)) { + if (rate.lower == rate.upper && rate.upper in listOf(120, 240)) + add(size.width, size.height, rate.upper, rate.lower, true) + } + } + } + if (modes.isEmpty()) { LogManager.addLog("WARN", "video", "camera has no compatible live modes id=${lens.cameraId}"); null } + else StreamCamera(lens.cameraId, lens, modes.distinct().sortedWith(compareBy({ it.fps }, { it.width * it.height }))) + } catch (e: Exception) { LogManager.addLog("ERROR", "video", "camera mode enumeration id=${lens.cameraId}: ${e.message}"); null } + }.sortedBy { when (it.lens.type) { LensType.MAIN -> 0; LensType.SELFIE -> 1; else -> 2 } } + } + + /** Смена камеры сохраняет точный режим, затем разрешение с меньшей FPS, затем штатный режим. */ + fun choose(modes: List, previous: StreamCaptureMode? = null): StreamCaptureMode? { + if (previous != null) { + modes.firstOrNull { it.width == previous.width && it.height == previous.height && it.fps == previous.fps }?.let { return it } + modes.filter { it.width == previous.width && it.height == previous.height && it.fps <= previous.fps } + .maxByOrNull { it.fps }?.let { return it } + } + return modes.firstOrNull { it.width == 1280 && it.height == 720 && it.fps == 30 } + ?: modes.filter { !it.highSpeed }.maxWithOrNull(compareBy({ it.fps }, { it.width * it.height })) ?: modes.firstOrNull() + } +} diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureConfig.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureConfig.kt new file mode 100644 index 00000000..e3230b92 --- /dev/null +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureConfig.kt @@ -0,0 +1,11 @@ +package com.utun.chat.data + +/** Ручной поворот относительно обычного положения устройства; фронтальный сенсор учитывается отдельно. */ +enum class StreamOrientation(val degrees: Int) { + UPRIGHT(0), RIGHT(90), UPSIDE_DOWN(180), LEFT(270); + + fun sensorRotation(sensor: Int, front: Boolean): Int = (sensor + (if (front) degrees else -degrees) + 360) % 360 +} + +/** Режим камеры и ориентация подтверждаются и восстанавливаются одним значением. */ +data class StreamCaptureConfig(val mode: StreamCaptureMode, val orientation: StreamOrientation = StreamOrientation.UPRIGHT) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt index 69bd42f9..6cfb692e 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt @@ -3,16 +3,18 @@ package com.utun.chat.data import android.annotation.SuppressLint import android.content.Context import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.CaptureResult +import android.hardware.camera2.TotalCaptureResult import android.media.AudioFormat import android.media.AudioRecord import android.media.AudioTimestamp import android.media.MediaCodec import android.media.MediaCodecInfo -import android.media.MediaCodecList import android.media.MediaFormat import android.media.MediaRecorder import android.os.Bundle @@ -22,13 +24,12 @@ import android.util.Range import android.view.Surface import java.io.ByteArrayOutputStream -/** Сервис владеет камерой/кодировщиком; локальный preview — обычный reader собственного стрима. - * Камера MediaCodec Surface и AudioTimestamp используют CLOCK_MONOTONIC; PTS относительно общего originNs. - * Кодек/камера — один HandlerThread, Opus — отдельный audio thread. UI не владеет ресурсами захвата. */ +/** Camera/codec принадлежат HandlerThread; микрофон и origin сохраняются при смене видео. + * Preview читает тот же источник. Поколение отсеивает callbacks закрываемой камеры/кодека. */ class StreamCaptureEngine( private val context: Context, private val id: Long, private val group: Long, - private val cameraId: String, private val microphone: Boolean, private val displayRotation: Int, - private val failure: (String) -> Unit + private val initialConfig: StreamCaptureConfig, private val microphone: Boolean, private val preview: StreamPreview, + private val ready: (StreamCaptureConfig, Boolean) -> Unit, private val failure: (String) -> Unit ) { private val thread = HandlerThread("stream-camera").apply { start() } private val handler = Handler(thread.looper) @@ -40,9 +41,21 @@ class StreamCaptureEngine( private var record: AudioRecord? = null private var audioThread: Thread? = null @Volatile private var audioRunning = false - private var stopped = false + @Volatile private var stopped = false private var stopping = false - private var stopDone: (() -> Unit)? = null + private var terminalFailure = false + private val stopCallbacks = mutableListOf<() -> Unit>() + private var opening = false + private var openingGeneration = 0L + private var releaseDone: ((Boolean) -> Unit)? = null + private var generation = 0L + private var lastGood: StreamCaptureConfig? = null + private var reverting = false + private var waitingFirst = true + private var confirming = false + private var confirmStartNs = 0L + private var confirmFrames = 0L + private var switchStartedNs = 0L private var originNs = 0L private var csd = byteArrayOf() private val partial = ByteArrayOutputStream() @@ -51,109 +64,198 @@ class StreamCaptureEngine( private var requireKey = true private var lastKeyRequest = 0L private var frames = 0L + private var dropped = 0L + private var encodedFrames = 0L private var bytesSent = 0L private var reportNs = 0L + private var reportFrames = 0L + private var reportBytes = 0L private var lastPts = -1L - fun start() { handler.post { try { configure() } catch (e: Exception) { failed("setup: ${e.message}") } } } + fun start() { + handler.post { + if (stopping || stopped) return@post + try { + val mode = initialConfig.mode + val rotation = cameraPose(mode.camera).rotation(initialConfig.orientation.degrees) + check(NativeLib.streamOpen(id, group, width = mode.width, height = mode.height, + rotation = rotation, audio = microphone)) { "Publisher open rejected" } + originNs = System.nanoTime() + if (microphone) startAudio() + configure(initialConfig) + } catch (e: Exception) { failed("setup: ${e.message}") } + } + } - @SuppressLint("MissingPermission") - private fun configure() { - if (stopping || stopped) return - val characteristics = manager.getCameraCharacteristics(cameraId) - val sizes = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) - ?.getOutputSizes(MediaCodec::class.java)?.toList().orEmpty() - val info = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.firstOrNull { codec -> - codec.isEncoder && codec.supportedTypes.any { it.equals(MediaFormat.MIMETYPE_VIDEO_AVC, true) } && - codec.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC).profileLevels.any { - it.profile == MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline - } && sizes.any { s -> s.width <= 1280 && s.height <= 720 && - codec.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC).videoCapabilities.areSizeAndRateSupported(s.width, s.height, 30.0) } - } ?: error("No AVC baseline camera/encoder combination at 30 fps") - val size = sizes.filter { it.width <= 1280 && it.height <= 720 && - info.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC).videoCapabilities.areSizeAndRateSupported(it.width, it.height, 30.0) } - .maxByOrNull { it.width * it.height } ?: error("No compatible camera size") - val front = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT - val sensor = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 - val rotation = (sensor + if (front) displayRotation else -displayRotation + 360) % 360 - val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, size.width, size.height).apply { - setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) - setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline) - setInteger(MediaFormat.KEY_BIT_RATE, if (size.width * size.height >= 1280 * 720) 2_000_000 else 1_000_000) - setInteger(MediaFormat.KEY_FRAME_RATE, 30); setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) + fun change(config: StreamCaptureConfig) { + handler.post { + if (stopping || stopped || terminalFailure || waitingFirst || confirming || releaseDone != null) { + LogManager.addLog("WARN", "video", "capture change rejected id=$id busy=$waitingFirst stopping=$stopping"); return@post + } + if (config == lastGood) { ready(config, false); return@post } + waitingFirst = true; reverting = false; switchStartedNs = System.nanoTime() + LogManager.addLog("INFO", "video", "switch capture id=$id camera=${config.mode.camera} ${config.mode.label} orientation=${config.orientation.degrees}") + releaseVideo { closed -> if (closed) configure(config) else failed("camera close timed out during switch") } } - check(NativeLib.streamOpen(id, group, width = size.width, height = size.height, rotation = rotation, audio = microphone)) - originNs = System.nanoTime() - encoder = MediaCodec.createByCodecName(info.name) - encoder!!.also { codec -> - codec.setCallback(object : MediaCodec.Callback() { - override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {} - override fun onError(codec: MediaCodec, error: MediaCodec.CodecException) { failed("encoder: ${error.diagnosticInfo}") } - override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { - try { - val parts = listOf("csd-0", "csd-1").mapNotNull { name -> format.getByteBuffer(name)?.duplicate()?.let { b -> - ByteArray(b.remaining()).also { b.get(it) }.let(H264AnnexB::normalize) - } } - csd = parts.fold(byteArrayOf()) { a, b -> a + b } - val types = H264AnnexB.nals(csd).map { it.first }.toSet() - check(7 in types && 8 in types) { "AVC format missing SPS/PPS" } - LogManager.addLog("INFO", "video", "encoder format id=$id csd=${csd.size} name=${codec.name}") - } catch (e: Exception) { failed("encoder format: ${e.message}") } - } - override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) { - if (stopped) return - try { - if (info.size > 0) { - val buffer = codec.getOutputBuffer(index) ?: error("Missing AVC output") - buffer.position(info.offset); buffer.limit(info.offset + info.size) - val bytes = ByteArray(info.size).also { buffer.get(it) } - if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { - val config = H264AnnexB.normalize(bytes) - val types = H264AnnexB.nals(config).map { it.first }.toSet() - if (7 in types && 8 in types) csd = config - } else encoded(bytes, info) + } + + private fun cameraPose(cameraId: String): StreamPreviewCamera { + val c = manager.getCameraCharacteristics(cameraId) + return StreamPreviewCamera(c.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0, + c.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT) + } + + @SuppressLint("MissingPermission") + private fun configure(config: StreamCaptureConfig) { + if (stopping || stopped || terminalFailure) return + val mode = config.mode + val token = ++generation + try { + check(token <= 0xffffffffL) { "Video revision exhausted" } + val cameraPose = cameraPose(mode.camera) + val rotation = cameraPose.rotation(config.orientation.degrees) + preview.register(token, cameraPose) + LogManager.addLog("DEBUG", "video", "preview camera id=$id revision=$token camera=${mode.camera} " + + "sensor=${cameraPose.sensor} front=${cameraPose.front} broadcastRotation=$rotation") + csd = byteArrayOf(); partial.reset(); requireKey = true; lastKeyRequest = 0; waitingFirst = true + confirming = mode.highSpeed; confirmStartNs = 0 + reportNs = System.nanoTime(); reportFrames = encodedFrames; reportBytes = bytesSent + if (switchStartedNs == 0L) switchStartedNs = reportNs + val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, mode.width, mode.height).apply { + setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) + setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline) + setInteger(MediaFormat.KEY_BIT_RATE, mode.bitrate) + setInteger(MediaFormat.KEY_FRAME_RATE, mode.fps); setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) + } + encoder = MediaCodec.createByCodecName(mode.codec) + encoder!!.also { codec -> + codec.setCallback(object : MediaCodec.Callback() { + override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {} + override fun onError(codec: MediaCodec, error: MediaCodec.CodecException) { captureError(token, "encoder: ${error.diagnosticInfo}") } + override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { + if (!current(token)) return + try { + csd = listOf("csd-0", "csd-1").mapNotNull { name -> format.getByteBuffer(name)?.duplicate()?.let { b -> + ByteArray(b.remaining()).also { b.get(it) }.let(H264AnnexB::normalize) + } }.fold(byteArrayOf()) { a, b -> a + b } + check(H264AnnexB.nals(csd).map { it.first }.toSet().containsAll(setOf(7, 8))) { "AVC format missing SPS/PPS" } + LogManager.addLog("INFO", "video", "encoder format id=$id revision=$token codec=${codec.name} csd=${csd.size} format=$format") + } catch (e: Exception) { captureError(token, "encoder format: ${e.message}") } + } + override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) { + if (!current(token)) return + var problem: String? = null + try { + if (info.size > 0) { + val buffer = codec.getOutputBuffer(index) ?: error("Missing AVC output") + buffer.position(info.offset); buffer.limit(info.offset + info.size) + val bytes = ByteArray(info.size).also { buffer.get(it) } + if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { + val codecConfig = H264AnnexB.normalize(bytes) + if (H264AnnexB.nals(codecConfig).map { it.first }.toSet().containsAll(setOf(7, 8))) csd = codecConfig + } else encoded(bytes, info, config, rotation, token) + } + if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) problem = "unexpected encoder EOS" + } catch (e: Exception) { problem = "encoded frame: ${e.message}" } + finally { + try { codec.releaseOutputBuffer(index, false) } + catch (e: Exception) { problem = "encoder output release: ${e.message}" } } - } catch (e: Exception) { failed("encoded frame: ${e.message}") } - finally { if (!stopped) codec.releaseOutputBuffer(index, false) } - if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { - if (stopping) finishStop() else failed("unexpected encoder EOS") + problem?.let { captureError(token, it) } } - } - }, handler) - codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) - input = codec.createInputSurface(); codec.start() - } - if (microphone) startAudio() - manager.openCamera(cameraId, object : CameraDevice.StateCallback() { - override fun onOpened(device: CameraDevice) { - if (stopping || stopped) { device.close(); return } - camera = device - device.createCaptureSession(listOf(input!!), object : CameraCaptureSession.StateCallback() { - override fun onConfigured(configured: CameraCaptureSession) { - if (stopping || stopped) { configured.close(); return } - session = configured + }, handler) + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + input = codec.createInputSurface(); codec.start() + } + opening = true; openingGeneration = token + try { + manager.openCamera(mode.camera, object : CameraDevice.StateCallback() { + override fun onOpened(device: CameraDevice) { + if (openingGeneration == token) { opening = false; camera = device } + if (!current(token)) { device.close(); return } try { - val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { - addTarget(input!!) - val rates = characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() - rates.filter { it.upper == 30 }.minByOrNull { it.upper - it.lower }?.let { - set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(it.lower, it.upper)) + val callback = object : CameraCaptureSession.StateCallback() { + override fun onConfigured(configured: CameraCaptureSession) { + if (!current(token)) { configured.close(); return } + session = configured + try { + val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { + addTarget(input!!) + set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(mode.minimumFps, mode.fps)) + }.build() + var sensorResults = 0L + var sensorReportCount = 0L + var sensorReportNs = System.nanoTime() + val diagnostics = object : CameraCaptureSession.CaptureCallback() { + override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) { + if (!current(token)) return + sensorResults++ + val now = System.nanoTime() + if (now - sensorReportNs >= 1_000_000_000) { + val fps = (sensorResults - sensorReportCount) * 1_000_000_000.0 / (now - sensorReportNs) + LogManager.addLog("DEBUG", "video", "sensor id=$id revision=$token results=$sensorResults callbackFps=${"%.1f".format(fps)} " + + "range=${result.get(CaptureResult.CONTROL_AE_TARGET_FPS_RANGE)} " + + "durationNs=${result.get(CaptureResult.SENSOR_FRAME_DURATION)} " + + "exposureNs=${result.get(CaptureResult.SENSOR_EXPOSURE_TIME)} timestamp=${result.get(CaptureResult.SENSOR_TIMESTAMP)}") + sensorReportNs = now; sensorReportCount = sensorResults + } + } + override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, + failed: android.hardware.camera2.CaptureFailure) { + if (current(token)) LogManager.addLog("WARN", "video", "sensor capture failed id=$id revision=$token " + + "frame=${failed.frameNumber} reason=${failed.reason} captured=${failed.wasImageCaptured()}") + } + } + if (mode.highSpeed) { + val highSpeed = configured as CameraConstrainedHighSpeedCaptureSession + highSpeed.setRepeatingBurst(highSpeed.createHighSpeedRequestList(request), diagnostics, handler) + } else configured.setRepeatingRequest(request, diagnostics, handler) + LogManager.addLog("INFO", "video", "capture started id=$id revision=$token camera=${mode.camera} ${mode.label} " + + "range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} codec=${mode.codec} " + + "bitrate=${mode.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs") + } catch (e: Exception) { captureError(token, "camera request: ${e.message}") } } - }.build() - configured.setRepeatingRequest(request, null, handler) - LogManager.addLog("INFO", "video", "capture started id=$id camera=$cameraId ${size.width}x${size.height} rotation=$rotation " + - "codec=${info.name} audio=$microphone timebase=MONOTONIC origin=$originNs") - } catch (e: Exception) { failed("camera request: ${e.message}") } + override fun onConfigureFailed(session: CameraCaptureSession) { + session.close(); captureError(token, "camera session configuration") + } + } + if (mode.highSpeed) device.createConstrainedHighSpeedCaptureSession(listOf(input!!), callback, handler) + else device.createCaptureSession(listOf(input!!), callback, handler) + } catch (e: Exception) { captureError(token, "camera session: ${e.message}") } + } + override fun onClosed(device: CameraDevice) { + if (camera === device) camera = null + if (releaseDone != null && camera == null && !opening) finishRelease(true) + if (stopped && camera == null && !opening) thread.quitSafely() + } + override fun onDisconnected(device: CameraDevice) { + if (openingGeneration == token) { opening = false; camera = device } + device.close(); captureError(token, "camera disconnected") + } + override fun onError(device: CameraDevice, error: Int) { + if (openingGeneration == token) { opening = false; camera = device } + device.close(); captureError(token, "camera error=$error") } - override fun onConfigureFailed(session: CameraCaptureSession) { session.close(); failed("camera session configuration") } }, handler) - } - override fun onDisconnected(device: CameraDevice) { device.close(); failed("camera disconnected") } - override fun onError(device: CameraDevice, error: Int) { device.close(); failed("camera error=$error") } - }, handler) + } catch (e: Exception) { opening = false; throw e } + handler.postDelayed({ if (current(token) && (waitingFirst || confirming)) captureError(token, "keyframe/capture rate validation timeout") }, 8000) + } catch (e: Exception) { captureError(token, "video setup: ${e.message}") } } - private fun encoded(bytes: ByteArray, info: MediaCodec.BufferInfo) { + private fun current(token: Long) = token == generation && !stopping && !stopped && !terminalFailure && releaseDone == null + + private fun captureError(token: Long, detail: String) { + if (!current(token)) { LogManager.addLog("DEBUG", "video", "stale capture error revision=$token: $detail"); return } + LogManager.addLog("ERROR", "video", "capture error id=$id revision=$token: $detail") + val previous = lastGood + if (previous == null || reverting) { failed(detail); return } + waitingFirst = true; reverting = true + LogManager.addLog("WARN", "video", "restoring camera=${previous.mode.camera} ${previous.mode.label} orientation=${previous.orientation.degrees} id=$id") + releaseVideo { closed -> if (closed) configure(previous) else failed("camera close timed out during restore") } + } + + private fun encoded(bytes: ByteArray, info: MediaCodec.BufferInfo, config: StreamCaptureConfig, rotation: Int, revision: Long) { + val mode = config.mode if (partial.size() == 0) { partialPts = info.presentationTimeUs; partialKey = false } check(info.presentationTimeUs == partialPts) { "Partial AVC frame PTS changed" } check(partial.size() + bytes.size <= H264AnnexB.MAX_FRAME) { "AVC frame exceeds limit" } @@ -161,31 +263,84 @@ class StreamCaptureEngine( if (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME != 0) return val data = H264AnnexB.normalize(partial.toByteArray()); partial.reset() val key = partialKey && H264AnnexB.nals(data).any { it.first == 5 } - if (requireKey && !key) { requestKey(); return } - val output = if (key) H264AnnexB.keyFrame(data, csd) else data val pts = partialPts - originNs / 1000 - check(pts >= 0 && pts > lastPts) { "AVC PTS outside monotonic timeline pts=$pts last=$lastPts" } - lastPts = pts - when (NativeLib.streamFeed(id, true, pts, key, output)) { - 1 -> { requireKey = false; frames++; bytesSent += output.size } - 0 -> { requireKey = true; requestKey() } - else -> failed("stream publisher detached") + check(pts >= 0 && pts > lastPts) { "AVC PTS outside timeline pts=$pts last=$lastPts" } + lastPts = pts; encodedFrames++ + if (requireKey && !key) { dropped++; requestKey() } + else { + val output = if (key) H264AnnexB.keyFrame(data, csd) else data + when (NativeLib.streamFeed(id, true, pts, key, output, revision, mode.width, mode.height, rotation, microphone)) { + 1 -> { + requireKey = false; frames++; bytesSent += output.size + if (waitingFirst) { + check(key); waitingFirst = false + confirmStartNs = System.nanoTime(); confirmFrames = encodedFrames + if (!confirming) confirmed(config, revision, pts) + } + } + 0 -> { dropped++; requireKey = true; requestKey() } + else -> failed("stream publisher detached") + } } val now = System.nanoTime() + if (confirming && confirmStartNs != 0L && now - confirmStartNs >= 3_000_000_000) { + val actualFps = (encodedFrames - confirmFrames) * 1_000_000_000.0 / (now - confirmStartNs) + if (actualFps < mode.fps * 0.8) { + captureError(revision, "high speed rate insufficient: camera=${mode.camera} target=${mode.fps} actual=${"%.1f".format(actualFps)} fps") + return + } + confirmed(config, revision, pts) + } if (now - reportNs >= 1_000_000_000) { - reportNs = now - LogManager.addLog("DEBUG", "video", "capture id=$id frames=$frames bytes=$bytesSent pts=$pts key=$key waitingKey=$requireKey") + val seconds = (now - reportNs) / 1_000_000_000.0 + LogManager.addLog("DEBUG", "video", "capture id=$id revision=$revision targetFps=${mode.fps} " + + "fps=${"%.1f".format((encodedFrames - reportFrames) / seconds)} bitrate=${((bytesSent - reportBytes) * 8 / seconds).toLong()} " + + "frames=$frames dropped=$dropped pts=$pts waitingKey=$requireKey") + reportNs = now; reportFrames = encodedFrames; reportBytes = bytesSent } } + /** Новый скоростной режим подтверждается реальной скоростью; до этого lastGood сохраняется для rollback. */ + private fun confirmed(config: StreamCaptureConfig, revision: Long, pts: Long) { + val mode = config.mode + confirming = false; lastGood = config + LogManager.addLog("INFO", "video", "capture ready id=$id revision=$revision camera=${mode.camera} ${mode.label} " + + "orientation=${config.orientation.degrees} switchMs=${(System.nanoTime() - switchStartedNs) / 1_000_000} restored=$reverting pts=$pts") + ready(config, reverting); reverting = false; switchStartedNs = 0 + } + private fun requestKey() { val now = System.nanoTime() if (now - lastKeyRequest < 500_000_000) return lastKeyRequest = now encoder?.setParameters(Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0) }) - LogManager.addLog("INFO", "video", "requested AVC keyframe id=$id") + LogManager.addLog("INFO", "video", "requested AVC keyframe id=$id revision=$generation") } + /** Новая камера открывается только после onClosed; старые codec callbacks уже отсечены. */ + private fun releaseVideo(done: (Boolean) -> Unit) { + if (releaseDone != null) { releaseDone = done; return } + val token = ++generation + releaseDone = done + try { session?.close() } catch (e: Exception) { LogManager.addLog("WARN", "video", "camera session close id=$id: ${e.message}") } + session = null + try { camera?.close() } catch (e: Exception) { LogManager.addLog("WARN", "video", "camera close id=$id: ${e.message}") } + if (camera == null && !opening) finishRelease(true) + else handler.postDelayed({ + if (releaseDone != null && generation == token) { + LogManager.addLog("ERROR", "video", "camera close timeout id=$id opening=$opening"); finishRelease(false) + } + }, 3000) + } + + private fun finishRelease(closed: Boolean) { + val done = releaseDone ?: return + try { encoder?.stop() } catch (e: Exception) { LogManager.addLog("WARN", "video", "encoder stop id=$id: ${e.message}") } + try { encoder?.release() } catch (e: Exception) { LogManager.addLog("WARN", "video", "encoder release id=$id: ${e.message}") } + encoder = null; input?.release(); input = null; partial.reset(); csd = byteArrayOf(); releaseDone = null + LogManager.addLog("DEBUG", "video", "video resources released id=$id closed=$closed revision=$generation") + done(closed) + } @SuppressLint("MissingPermission") private fun startAudio() { val min = AudioRecord.getMinBufferSize(48000, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) @@ -225,35 +380,33 @@ class StreamCaptureEngine( } private fun failed(detail: String) { - if (stopping || stopped) return + if (stopping || stopped || terminalFailure) return + terminalFailure = true LogManager.addLog("ERROR", "stream", "capture failed id=$id $detail") failure(detail) } - /** Сначала прекращаем producers, затем EOS; completion разрешает native FIN. */ + /** Producers закрываются до native FIN; уже принятые кадры дренирует native publisher. */ fun stop(done: () -> Unit) { - handler.post { + if (stopped) { done(); return } + if (!handler.post({ if (stopped) { done(); return@post } - if (stopping) { LogManager.addLog("DEBUG", "stream", "duplicate capture stop id=$id"); return@post } - stopping = true; stopDone = done; audioRunning = false + stopCallbacks.add(done) + if (stopping) return@post + stopping = true; audioRunning = false try { record?.stop() } catch (e: Exception) { LogManager.addLog("WARN", "stream", "audio stop id=$id: ${e.message}") } audioThread?.join(); audioThread = null - record?.release(); record = null - session?.close(); session = null; camera?.close(); camera = null - try { - if (encoder != null) { encoder!!.signalEndOfInputStream(); handler.postDelayed({ finishStop() }, 1000) } - else finishStop() - } catch (e: Exception) { LogManager.addLog("WARN", "stream", "encoder EOS id=$id: ${e.message}"); finishStop() } + try { record?.release() } catch (e: Exception) { LogManager.addLog("WARN", "stream", "audio release id=$id: ${e.message}") } + record = null + releaseVideo { + stopped = true + LogManager.addLog("INFO", "stream", "capture released id=$id frames=$frames dropped=$dropped bytes=$bytesSent") + stopCallbacks.toList().also { stopCallbacks.clear() }.forEach { it() } + if (camera == null && !opening) thread.quitSafely() + } + })) { + LogManager.addLog("WARN", "stream", "capture handler already closed id=$id; complete stop") + done() } } - - private fun finishStop() { - if (stopped) return - stopped = true - try { encoder?.stop() } catch (e: Exception) { LogManager.addLog("WARN", "video", "encoder stop id=$id: ${e.message}") } - try { encoder?.release() } catch (e: Exception) { LogManager.addLog("WARN", "video", "encoder release id=$id: ${e.message}") } - encoder = null; input?.release(); input = null; partial.reset() - LogManager.addLog("INFO", "stream", "capture released id=$id frames=$frames bytes=$bytesSent") - stopDone?.invoke(); stopDone = null; thread.quitSafely() - } } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamController.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamController.kt index b6bf218b..c59d95bd 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamController.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamController.kt @@ -12,7 +12,8 @@ import kotlinx.coroutines.flow.StateFlow import java.util.concurrent.atomic.AtomicLong enum class StreamPhase { STARTING, LIVE, STOPPING, ENDED, FAILED } -data class StreamState(val id: Long, val group: Long, val phase: StreamPhase, val error: Int? = null) { +data class StreamState(val id: Long, val group: Long, val phase: StreamPhase, val error: Int? = null, + val config: StreamCaptureConfig? = null, val pendingConfig: StreamCaptureConfig? = null, val notice: Int? = null, val preview: StreamPreview = StreamPreview()) { val ongoing get() = phase == StreamPhase.STARTING || phase == StreamPhase.LIVE || phase == StreamPhase.STOPPING } @@ -27,7 +28,7 @@ object StreamController { private set private var service: StreamService? = null - fun start(group: Long, camera: String, audio: Boolean, rotation: Int) { + fun start(group: Long, config: StreamCaptureConfig, audio: Boolean) { check(Looper.myLooper() == Looper.getMainLooper()) if (mutable.value?.ongoing == true) { LogManager.addLog("WARN", "stream", "publisher already owned"); return } val id = nextId() @@ -36,16 +37,36 @@ object StreamController { LogManager.addLog("WARN", "stream", "start rejected: core/call/radio/voice busy group=$group") mutable.value = StreamState(id, group, StreamPhase.FAILED, R.string.stream_resources_busy); return } - capturing = true; mutable.value = StreamState(id, group, StreamPhase.STARTING) + capturing = true; mutable.value = StreamState(id, group, StreamPhase.STARTING, pendingConfig = config) try { ContextCompat.startForegroundService(ChatApplication.instance, Intent(ChatApplication.instance, StreamService::class.java) - .putExtra("id", id).putExtra("group", group).putExtra("camera", camera).putExtra("audio", audio).putExtra("rotation", rotation)) + .putExtra("id", id).putExtra("group", group).putExtra("audio", audio)) } catch (e: Exception) { LogManager.addLog("ERROR", "stream", "service start failed id=$id: ${e.message}") capturing = false; mutable.value = StreamState(id, group, StreamPhase.FAILED, R.string.stream_failed) } } + fun change(config: StreamCaptureConfig) { + check(Looper.myLooper() == Looper.getMainLooper()) + val st = mutable.value ?: return + val owner = service + if (st.phase != StreamPhase.LIVE || st.pendingConfig != null || owner == null) { + LogManager.addLog("WARN", "video", "publisher change rejected id=${st.id} phase=${st.phase} pending=${st.pendingConfig}"); return + } + if (st.config == config) return + mutable.value = st.copy(pendingConfig = config, notice = null) + owner.change(config) + } + + fun captureReady(id: Long, config: StreamCaptureConfig, restored: Boolean) { + main.post { + val st = mutable.value ?: return@post + if (st.id != id || !st.ongoing || st.phase == StreamPhase.STOPPING) return@post + mutable.value = st.copy(config = config, pendingConfig = null, notice = if (restored) R.string.stream_mode_restored else null) + } + } + fun serviceReady(id: Long, owner: StreamService): Boolean { if (mutable.value?.id != id || mutable.value?.phase != StreamPhase.STARTING) return false service = owner; return true @@ -78,7 +99,7 @@ object StreamController { val st = mutable.value ?: return if (st.id != id || !st.ongoing) return capturing = false - mutable.value = st.copy(phase = if (st.error != null) StreamPhase.FAILED else StreamPhase.ENDED) + mutable.value = st.copy(phase = if (st.error != null) StreamPhase.FAILED else StreamPhase.ENDED, pendingConfig = null) LogManager.addLog("INFO", "stream", "publisher released id=$id") } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt index 75f69a54..e19ea012 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt @@ -21,11 +21,13 @@ import kotlin.math.roundToInt * Просроченные P-кадры декодируются без показа, чтобы сохранить references; при переполнении ждём KEY. * viewerLock сериализует stop/open разных экранов до подтверждения освобождения native owner. */ class StreamPlayerEngine(private val group: Long, private val source: Long, private val epoch: Long, - private val muteAudio: Boolean = false) { + private val localPreview: StreamPreview? = null, private val displayDegrees: Int = 0) { companion object { private val viewerLock = ReentrantLock() } - private data class Frame(val pts: Long, val key: Boolean, val width: Int, val height: Int, val rotation: Int, val data: ByteArray) + private data class Frame(val pts: Long, val key: Boolean, val width: Int, val height: Int, val rotation: Int, val top: Int?, val revision: Long, val data: ByteArray) private val mutableAspect = MutableStateFlow(16f / 9f) val aspect: StateFlow = mutableAspect + private val mutableTop = MutableStateFlow(null) + val top: StateFlow = mutableTop private val mutableStatus = MutableStateFlow(R.string.stream_connecting) val status: StateFlow = mutableStatus private val id = StreamController.nextId() @@ -55,6 +57,8 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv val audioTimestamp = AudioTimestamp() val bufferInfo = MediaCodec.BufferInfo() var report = 0L + var videoRevision = 0L + var reportedDecoded = 0L var decoded = 0L var discarded = 0L var lastHead = -1L @@ -127,31 +131,50 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv var prefillUntil = 0L while (running && surface.isValid) { if (!NativeLib.isReady()) error("Core stopped") - repeat(16) { + repeat(64) { val raw = if (!ended) NativeLib.streamPull(id) else null if (raw != null) { - check(raw.size >= 20) + check(raw.size >= 24) val b = ByteBuffer.wrap(raw).order(ByteOrder.BIG_ENDIAN) val kind = b.get().toInt(); val flags = b.get().toInt() and 255; val key = flags and 1 != 0 val width = b.short.toInt() and 65535; val height = b.short.toInt() and 65535 - val rotation = b.short.toInt() and 65535; val pts = b.long; val size = b.int + val broadcastRotation = b.short.toInt() and 65535; val pts = b.long; val size = b.int + val revision = b.int.toLong() and 0xffffffffL check(size == b.remaining()) when (kind) { 1, 2 -> { - val frame = Frame(pts, key, width, height, rotation, ByteArray(size).also { b.get(it) }) + val camera = if (kind == 1) localPreview?.camera(revision) else null + if (kind == 1 && localPreview != null && camera == null) { + LogManager.addLog("WARN", "video", "preview camera unavailable id=$id revision=$revision; wait KEY") + resetVideo(); return@repeat + } + val display = StreamVideoDisplay.forFrame(broadcastRotation, camera, displayDegrees) + val rotation = display.rotation + val frame = Frame(pts, key, width, height, rotation, display.top, revision, ByteArray(size).also { b.get(it) }) if (prefillUntil == 0L) prefillUntil = System.nanoTime() + 150_000_000 if (kind == 1) { - if (videos.size >= 12 || videoBytes + size > H264AnnexB.MAX_FRAME) resetVideo() + if (revision != videoRevision) { + check(revision > videoRevision) { "Video revision regressed" } + LogManager.addLog("INFO", "video", "viewer configuration id=$id revision=$videoRevision->$revision ${width}x$height " + + "rotation=$rotation broadcastRotation=$broadcastRotation preview=${localPreview != null} top=${display.top}") + resetVideo(); videoRevision = revision + } + if (videos.size >= 128 || videoBytes + size > H264AnnexB.MAX_FRAME || + (videos.isNotEmpty() && pts - videos.first.pts > 250000)) { + LogManager.addLog("WARN", "video", "viewer video overflow id=$id frames=${videos.size} bytes=$videoBytes revision=$revision") + resetVideo() + } if (!waitingKey || key) { if (waitingKey) { check(H264AnnexB.nals(frame.data).map { it.first }.toSet().containsAll(setOf(5, 7, 8))) waitingKey = false mutableAspect.value = if (rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width + mutableTop.value = frame.top } videos.addLast(frame); videoBytes += size } } else { - if (muteAudio) return@repeat + if (localPreview != null) return@repeat if (audios.size >= 16) { LogManager.addLog("WARN", "stream", "audio queue overflow id=$id; reset timeline") resetAudio(); resetVideo() @@ -213,7 +236,8 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv } if (videoBase < 0 && videos.isNotEmpty()) { videoBase = videos.first.pts; videoStartNs = now } val clock = currentPts(now) - if (videos.isNotEmpty() && videos.first.pts <= clock + 80000) { + repeat(8) { + if (videos.isEmpty() || videos.first.pts > clock + 80000) return@repeat val frame = videos.first if (decoder == null) { check(frame.key) @@ -240,7 +264,7 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv } } val codec = decoder - if (codec != null) repeat(8) { + if (codec != null) repeat(16) { val index = codec.dequeueOutputBuffer(bufferInfo, 0) if (index >= 0) { val clockPts = currentPts(System.nanoTime()) @@ -256,9 +280,11 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv if (now >= endDrainUntil) { mutableStatus.value = R.string.stream_ended; break } } if (now - report >= 1_000_000_000) { + val seconds = if (report == 0L) 1.0 else (now - report) / 1_000_000_000.0 report = now - LogManager.addLog("DEBUG", "stream", "viewer id=$id decoded=$decoded discarded=$discarded videoQ=${videos.size}/$videoBytes " + + LogManager.addLog("DEBUG", "stream", "viewer id=$id revision=$videoRevision decodeFps=${"%.1f".format((decoded - reportedDecoded) / seconds)} decoded=$decoded discarded=$discarded videoQ=${videos.size}/$videoBytes " + "audioQ=${audios.size} written=$written clock=${currentPts(now)} waitingKey=$waitingKey") + reportedDecoded = decoded } Thread.sleep(5) } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt new file mode 100644 index 00000000..9b98e28c --- /dev/null +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt @@ -0,0 +1,27 @@ +package com.utun.chat.data + +/** Положение сенсора конкретной камеры; ручной поворот относится только к трансляции. */ +data class StreamPreviewCamera(val sensor: Int, val front: Boolean) { + fun rotation(displayDegrees: Int): Int = (sensor + (if (front) displayDegrees else -displayDegrees) + 360) % 360 +} + +data class StreamVideoDisplay(val rotation: Int, val top: Int?) { + companion object { + fun forFrame(broadcastRotation: Int, camera: StreamPreviewCamera?, displayDegrees: Int): StreamVideoDisplay { + if (camera == null) return StreamVideoDisplay(broadcastRotation, null) + val previewRotation = camera.rotation(displayDegrees) + return StreamVideoDisplay(previewRotation, (previewRotation - broadcastRotation + 360) % 360) + } + } +} + +/** Принадлежит одной публикации. Camera thread пишет immutable snapshots; viewer читает по revision своего кадра. */ +class StreamPreview { + @Volatile private var cameras: List> = emptyList() + + internal fun register(revision: Long, camera: StreamPreviewCamera) { + cameras = cameras.filterNot { it.first == revision }.takeLast(31) + (revision to camera) + } + + fun camera(revision: Long): StreamPreviewCamera? = cameras.firstOrNull { it.first == revision }?.second +} diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamService.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamService.kt index e63c5213..4e7369ba 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamService.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamService.kt @@ -54,8 +54,11 @@ class StreamService : Service() { setReferenceCounted(false); acquire() } val group = intent!!.getLongExtra("group", 0) - capture = StreamCaptureEngine(this, id, group, intent.getStringExtra("camera") ?: error("Missing camera"), - audio, intent.getIntExtra("rotation", 0)) { detail -> StreamController.failed(id, detail) }.also { it.start() } + val publisher = StreamController.state.value ?: error("Missing publisher") + val config = publisher.pendingConfig ?: error("Missing capture configuration") + capture = StreamCaptureEngine(this, id, group, config, audio, publisher.preview, + { active, restored -> StreamController.captureReady(id, active, restored) }, + { detail -> StreamController.failed(id, detail) }).also { it.start() } poll = scope.launch { val deadline = System.nanoTime() + 10_000_000_000 var opened = false @@ -80,6 +83,11 @@ class StreamService : Service() { return START_NOT_STICKY } + fun change(config: StreamCaptureConfig) { + if (stopping || capture == null) { LogManager.addLog("WARN", "video", "service capture change rejected id=$id"); return } + capture!!.change(config) + } + /** Producers завершатся до FIN; при уничтожении ядра очередь не ждём. */ fun shutdown(abort: Boolean = false) { aborting = aborting || abort diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt index 5df6a9ae..ac402a89 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt @@ -1,24 +1,30 @@ package com.utun.chat.ui.screens import android.Manifest +import android.app.Activity import android.content.pm.PackageManager import android.graphics.SurfaceTexture -import android.hardware.camera2.CameraCharacteristics -import android.hardware.camera2.CameraManager import android.view.Surface import android.view.TextureView import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.* +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ArrowUpward import androidx.compose.material.icons.automirrored.filled.ArrowBack import androidx.compose.material3.* import androidx.compose.runtime.* +import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.ui.Modifier import androidx.compose.ui.Alignment +import androidx.compose.ui.draw.rotate +import androidx.compose.ui.graphics.Color import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalConfiguration import androidx.compose.ui.platform.LocalLifecycleOwner import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp @@ -31,7 +37,17 @@ import com.utun.chat.data.LogManager import com.utun.chat.data.StreamController import com.utun.chat.data.StreamPhase import com.utun.chat.data.StreamPlayerEngine +import com.utun.chat.data.StreamPreview import com.utun.chat.data.StreamSource +import com.utun.chat.data.StreamCaptureMode +import com.utun.chat.data.StreamCaptureConfig +import com.utun.chat.data.StreamOrientation +import com.utun.chat.data.StreamCamera +import com.utun.chat.data.StreamCameraModes +import com.utun.chat.data.LensType +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext /** Выход с экрана закрывает только reader; foreground publisher останавливает явная кнопка/notification. */ @OptIn(ExperimentalMaterial3Api::class) @@ -44,14 +60,18 @@ fun StreamScreen(channelId: String, channelName: String, sources: List(null) } var audio by remember { mutableStateOf(true) } - var camera by remember { mutableStateOf("") } - val cameras = remember { - val manager = context.getSystemService(CameraManager::class.java) - try { manager.cameraIdList.map { id -> id to (manager.getCameraCharacteristics(id).get(CameraCharacteristics.LENS_FACING) - == CameraCharacteristics.LENS_FACING_FRONT) } } - catch (e: Exception) { LogManager.addLog("ERROR", "stream", "camera enumeration: ${e.message}"); emptyList() } + var mode by remember { mutableStateOf(null) } + var orientation by rememberSaveable { mutableStateOf(StreamOrientation.UPRIGHT) } + val config = mode?.let { StreamCaptureConfig(it, orientation) } + var cameras by remember { mutableStateOf>(emptyList()) } + var loadingModes by remember { mutableStateOf(true) } + LaunchedEffect(Unit) { + try { cameras = withContext(Dispatchers.Default) { StreamCameraModes.enumerate(context) } } + catch (e: CancellationException) { throw e } + catch (e: Exception) { LogManager.addLog("ERROR", "video", "capture modes failed: ${e.message}") } + loadingModes = false + mode = StreamCameraModes.choose((cameras.firstOrNull { it.lens.type == LensType.MAIN } ?: cameras.firstOrNull())?.modes.orEmpty()) } - LaunchedEffect(cameras) { if (camera.isEmpty()) camera = cameras.firstOrNull { !it.second }?.first ?: cameras.firstOrNull()?.first.orEmpty() } var permissionVersion by remember { mutableStateOf(0) } val permissions = rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { permissionVersion++ } val cameraGranted = permissionVersion.let { ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED } @@ -64,6 +84,14 @@ fun StreamScreen(channelId: String, channelName: String, sources: List R.string.stream_live - StreamPhase.STOPPING -> R.string.stream_stopping - else -> R.string.stream_connecting - })) - Button(onClick = StreamController::stop, enabled = local.phase != StreamPhase.STOPPING) { Text(stringResource(R.string.stream_stop)) } } else { - Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { - cameras.forEach { (id, front) -> FilterChip(selected = camera == id, onClick = { camera = id }, - label = { Text(stringResource(if (front) R.string.stream_front_camera else R.string.stream_back_camera) + " $id") }) } - } - Row { Checkbox(checked = audio, onCheckedChange = { audio = it }); Text(stringResource(R.string.microphone), Modifier.padding(top = 12.dp)) } - Button(onClick = { - if (!cameraGranted || (audio && !micGranted)) permissions.launch( - listOf(Manifest.permission.CAMERA).plus(if (audio) listOf(Manifest.permission.RECORD_AUDIO) else emptyList()).toTypedArray()) - else { - val degrees = when ((context as? android.app.Activity)?.windowManager?.defaultDisplay?.rotation) { - Surface.ROTATION_90 -> 90; Surface.ROTATION_180 -> 180; Surface.ROTATION_270 -> 270; else -> 0 + if (local?.ongoing == true) { + CaptureOptions(cameras, local.pendingConfig ?: local.config, local.phase == StreamPhase.LIVE && local.pendingConfig == null, + StreamController::change) + CaptureModeStatus(local.config, local.pendingConfig, local.notice) + Text(stringResource(when (local.phase) { + StreamPhase.LIVE -> R.string.stream_live + StreamPhase.STOPPING -> R.string.stream_stopping + else -> R.string.stream_connecting + })) + Button(onClick = StreamController::stop, enabled = local.phase != StreamPhase.STOPPING) { Text(stringResource(R.string.stream_stop)) } + } else { + CaptureOptions(cameras, config, !loadingModes) { mode = it.mode; orientation = it.orientation } + if (!loadingModes && cameras.isEmpty()) Text(stringResource(R.string.stream_no_modes), color = MaterialTheme.colorScheme.error) + Row { Checkbox(checked = audio, onCheckedChange = { audio = it }); Text(stringResource(R.string.microphone), Modifier.padding(top = 12.dp)) } + Button(onClick = { + if (!cameraGranted || (audio && !micGranted)) permissions.launch( + listOf(Manifest.permission.CAMERA).plus(if (audio) listOf(Manifest.permission.RECORD_AUDIO) else emptyList()).toTypedArray()) + else { + config?.let { StreamController.start(group, it, audio) } } - StreamController.start(group, camera, audio, degrees) + }, enabled = config != null && group != 0L && state?.ongoing != true) { Text(stringResource(R.string.stream_start)) } + } + local?.error?.let { Text(stringResource(it), color = MaterialTheme.colorScheme.error) } + if (sourcesFailed) Text(stringResource(R.string.stream_list_failed), color = MaterialTheme.colorScheme.error) + else if (sources.isEmpty() && local?.ongoing != true) Text(stringResource(R.string.stream_none)) + Column(Modifier.fillMaxWidth().heightIn(max = 160.dp).verticalScroll(rememberScrollState())) { + sources.forEach { source -> + TextButton(onClick = { onSelect(source) }) { Text(stringResource(R.string.stream_watch, source.displayName)) } } - }, enabled = camera.isNotEmpty() && group != 0L && state?.ongoing != true) { Text(stringResource(R.string.stream_start)) } - } - local?.error?.let { Text(stringResource(it), color = MaterialTheme.colorScheme.error) } - if (sourcesFailed) Text(stringResource(R.string.stream_list_failed), color = MaterialTheme.colorScheme.error) - else if (sources.isEmpty() && local?.ongoing != true) Text(stringResource(R.string.stream_none)) - Column(Modifier.fillMaxWidth().heightIn(max = 160.dp).verticalScroll(rememberScrollState())) { - sources.forEach { source -> - TextButton(onClick = { onSelect(source) }) { Text(stringResource(R.string.stream_watch, source.displayName)) } } } } @@ -118,7 +149,50 @@ fun StreamScreen(channelId: String, channelName: String, sources: List, config: StreamCaptureConfig?, enabled: Boolean, + onChange: (StreamCaptureConfig) -> Unit) { + val mode = config?.mode + Row(Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), horizontalArrangement = Arrangement.spacedBy(8.dp)) { + cameras.forEach { camera -> + FilterChip(selected = camera.id == mode?.camera, enabled = enabled, onClick = { + StreamCameraModes.choose(camera.modes, mode)?.let { onChange(StreamCaptureConfig(it, config?.orientation ?: StreamOrientation.UPRIGHT)) } + }, label = { Text("${camera.lens.label} ${camera.lens.mm}".trim()) }) + } + } + val modes = cameras.firstOrNull { it.id == mode?.camera }?.modes.orEmpty() + var expanded by remember { mutableStateOf(false) } + var orientationsExpanded by remember { mutableStateOf(false) } + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Box { + OutlinedButton(onClick = { expanded = true }, enabled = enabled && modes.isNotEmpty()) { Text(mode?.label.orEmpty()) } + DropdownMenu(expanded = expanded && enabled, onDismissRequest = { expanded = false }) { + modes.forEach { option -> DropdownMenuItem(text = { Text(option.label) }, onClick = { + expanded = false; config?.let { onChange(it.copy(mode = option)) } + }) } + } + } + Box { + OutlinedButton(onClick = { orientationsExpanded = true }, enabled = enabled && config != null) { + Text(stringResource(R.string.stream_orientation, config?.orientation?.degrees ?: 0)) + } + DropdownMenu(expanded = orientationsExpanded && enabled, onDismissRequest = { orientationsExpanded = false }) { + StreamOrientation.values().forEach { option -> DropdownMenuItem(text = { Text("${option.degrees}°") }, onClick = { + orientationsExpanded = false; config?.let { onChange(it.copy(orientation = option)) } + }) } + } + } + } +} + +@Composable +private fun LiveStreamView(group: Long, source: Long, epoch: Long, localPreview: StreamPreview?) { val owner = LocalLifecycleOwner.current var visible by remember(owner) { mutableStateOf(owner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)) } DisposableEffect(owner) { @@ -129,36 +203,62 @@ private fun LiveStreamView(group: Long, source: Long, epoch: Long, localPreview: owner.lifecycle.addObserver(observer) onDispose { owner.lifecycle.removeObserver(observer) } } - if (visible) LiveStreamSurface(group, source, epoch, localPreview) + val context = LocalContext.current + val configuration = LocalConfiguration.current + val displayDegrees = if (localPreview != null) remember(configuration) { + ((context as? Activity)?.windowManager?.defaultDisplay?.rotation ?: Surface.ROTATION_0) * 90 + } else 0 + if (visible) key(localPreview, displayDegrees) { LiveStreamSurface(group, source, epoch, localPreview, displayDegrees) } } @Composable -private fun LiveStreamSurface(group: Long, source: Long, epoch: Long, localPreview: Boolean) { - val engine = remember(group, source, epoch, localPreview) { StreamPlayerEngine(group, source, epoch, localPreview) } +private fun LiveStreamSurface(group: Long, source: Long, epoch: Long, localPreview: StreamPreview?, displayDegrees: Int) { + val engine = remember(group, source, epoch, localPreview) { StreamPlayerEngine(group, source, epoch, localPreview, displayDegrees) } val aspect by engine.aspect.collectAsState() + val top by engine.top.collectAsState() val status by engine.status.collectAsState() var surface by remember { mutableStateOf(null) } DisposableEffect(engine) { onDispose { engine.stop() } } Column(Modifier.fillMaxSize()) { Text(stringResource(status)) Box(Modifier.weight(1f).fillMaxWidth(), contentAlignment = Alignment.Center) { - AndroidView(factory = { context -> - TextureView(context).apply { - keepScreenOn = true - surfaceTextureListener = object : TextureView.SurfaceTextureListener { - override fun onSurfaceTextureAvailable(texture: SurfaceTexture, width: Int, height: Int) { - surface = Surface(texture); engine.start(surface!!) - } - override fun onSurfaceTextureSizeChanged(texture: SurfaceTexture, width: Int, height: Int) {} - override fun onSurfaceTextureDestroyed(texture: SurfaceTexture): Boolean { - engine.stop(); surface?.release(); surface = null; return true + Box(Modifier.aspectRatio(aspect, matchHeightConstraintsFirst = true)) { + AndroidView(factory = { context -> + TextureView(context).apply { + keepScreenOn = true + surfaceTextureListener = object : TextureView.SurfaceTextureListener { + override fun onSurfaceTextureAvailable(texture: SurfaceTexture, width: Int, height: Int) { + surface = Surface(texture); engine.start(surface!!) + } + override fun onSurfaceTextureSizeChanged(texture: SurfaceTexture, width: Int, height: Int) {} + override fun onSurfaceTextureDestroyed(texture: SurfaceTexture): Boolean { + engine.stop(); surface?.release(); surface = null; return true + } + override fun onSurfaceTextureUpdated(texture: SurfaceTexture) {} } - override fun onSurfaceTextureUpdated(texture: SurfaceTexture) {} } - } - }, modifier = Modifier.aspectRatio(aspect, matchHeightConstraintsFirst = true), onRelease = { view -> - engine.stop(); view.surfaceTextureListener = null; surface?.release(); surface = null - }) + }, modifier = Modifier.matchParentSize(), onRelease = { view -> + engine.stop(); view.surfaceTextureListener = null; surface?.release(); surface = null + }) + top?.let { StreamTopMarker(it) } + } + } + } +} + +@Composable +private fun BoxScope.StreamTopMarker(direction: Int) { + val alignment = when (direction) { + 90 -> Alignment.CenterEnd + 180 -> Alignment.BottomCenter + 270 -> Alignment.CenterStart + else -> Alignment.TopCenter + } + Surface(modifier = Modifier.align(alignment).padding(8.dp), shape = RoundedCornerShape(6.dp), color = Color.Black.copy(alpha = 0.65f)) { + Row(Modifier.padding(horizontal = 8.dp, vertical = 4.dp), verticalAlignment = Alignment.CenterVertically) { + Icon(Icons.Filled.ArrowUpward, contentDescription = null, tint = Color.White, modifier = Modifier.size(16.dp).rotate(direction.toFloat())) + Spacer(Modifier.width(4.dp)) + Text(stringResource(R.string.stream_top), color = Color.White, style = MaterialTheme.typography.bodySmall) } } } diff --git a/tools/chatgui-android/app/src/main/res/values-ru/strings.xml b/tools/chatgui-android/app/src/main/res/values-ru/strings.xml index 4ae65ff1..822794ce 100644 --- a/tools/chatgui-android/app/src/main/res/values-ru/strings.xml +++ b/tools/chatgui-android/app/src/main/res/values-ru/strings.xml @@ -452,4 +452,9 @@ "Автопередача выключена" "В обоих режимах передаётся до 0,6 с звука перед срабатыванием. Уровень измеряется до компрессора рации. Для включения нужны 40 мс на пороге включения или выше. Передача заканчивается после заданной паузы ниже порога отключения. Измеритель работает и без автопередачи, пока открыта эта вкладка." "Удерживайте для настройки режима и порогов" + Не удалось переключить захват. Восстановлены прежняя камера, режим и ориентация. + Переключение камеры, режима или ориентации… + Нет совместимой камеры или видеокодировщика + Поворот: %1$d° + Верх diff --git a/tools/chatgui-android/app/src/main/res/values/strings.xml b/tools/chatgui-android/app/src/main/res/values/strings.xml index c451e585..baf806d8 100644 --- a/tools/chatgui-android/app/src/main/res/values/strings.xml +++ b/tools/chatgui-android/app/src/main/res/values/strings.xml @@ -451,4 +451,9 @@ "Automatic transmission disabled" "Both modes include 0.6 s of prebuffer. Level is measured before the radio compressor. Start requires 40 ms at or above the start threshold. Transmission ends after the configured silence below the stop threshold. The meter also works with automatic transmission disabled while this tab is visible." "Hold to adjust activation mode and thresholds" + Could not switch capture. The previous camera, mode and orientation have been restored. + Switching camera, video mode or orientation… + No compatible camera or video encoder found + Rotation: %1$d° + Top diff --git a/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamCameraModesTest.kt b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamCameraModesTest.kt new file mode 100644 index 00000000..d589832f --- /dev/null +++ b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamCameraModesTest.kt @@ -0,0 +1,28 @@ +package com.utun.chat.data + +import org.junit.Assert.* +import org.junit.Test + +class StreamCameraModesTest { + private fun mode(camera: String, width: Int, height: Int, fps: Int) = + StreamCaptureMode(camera, width, height, fps, fps, fps >= 120, "codec", 2_000_000) + + @Test fun cameraChangePreservesExactMode() { + val previous = mode("back", 1920, 1080, 120) + val modes = listOf(mode("front", 1280, 720, 30), mode("front", 1920, 1080, 120)) + assertEquals(modes[1], StreamCameraModes.choose(modes, previous)) + } + + @Test fun cameraChangeUsesLowerRateAtSameResolution() { + val previous = mode("back", 1920, 1080, 240) + val modes = listOf(mode("front", 1280, 720, 30), mode("front", 1920, 1080, 30), mode("front", 1920, 1080, 120)) + assertEquals(modes[2], StreamCameraModes.choose(modes, previous)) + } + + @Test fun defaultAndUnsupportedModeUse720p30() { + val modes = listOf(mode("other", 640, 480, 30), mode("other", 1280, 720, 30), mode("other", 1920, 1080, 120)) + assertEquals(modes[1], StreamCameraModes.choose(modes)) + assertEquals(modes[1], StreamCameraModes.choose(modes, mode("back", 1920, 1080, 30))) + assertNull(StreamCameraModes.choose(emptyList())) + } +} diff --git a/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamOrientationTest.kt b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamOrientationTest.kt new file mode 100644 index 00000000..a373f471 --- /dev/null +++ b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamOrientationTest.kt @@ -0,0 +1,21 @@ +package com.utun.chat.data + +import org.junit.Assert.assertEquals +import org.junit.Test + +class StreamOrientationTest { + @Test fun rearCameraRotationFollowsManualOrientation() { + assertEquals(listOf(90, 0, 270, 180), StreamOrientation.values().map { it.sensorRotation(90, false) }) + assertEquals(listOf(270, 180, 90, 0), StreamOrientation.values().map { it.sensorRotation(270, false) }) + } + + @Test fun frontCameraUsesOppositeSensorCompensation() { + assertEquals(listOf(270, 0, 90, 180), StreamOrientation.values().map { it.sensorRotation(270, true) }) + assertEquals(listOf(90, 180, 270, 0), StreamOrientation.values().map { it.sensorRotation(90, true) }) + } + + @Test fun camerasWithZeroSensorRotationKeepAllFourDirections() { + assertEquals(listOf(0, 270, 180, 90), StreamOrientation.values().map { it.sensorRotation(0, false) }) + assertEquals(listOf(0, 90, 180, 270), StreamOrientation.values().map { it.sensorRotation(0, true) }) + } +} diff --git a/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt new file mode 100644 index 00000000..1ebcb9be --- /dev/null +++ b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt @@ -0,0 +1,48 @@ +package com.utun.chat.data + +import org.junit.Assert.* +import org.junit.Test + +class StreamPreviewTest { + @Test fun manualRotationMovesRearTopMarkerWithoutRotatingCameraPreview() { + val camera = StreamPreviewCamera(90, false) + val display = StreamOrientation.values().map { + StreamVideoDisplay.forFrame(it.sensorRotation(90, false), camera, 0) + } + assertEquals(listOf(90, 90, 90, 90), display.map { it.rotation }) + assertEquals(listOf(0, 90, 180, 270), display.map { it.top }) + } + + @Test fun frontCameraTopMarkerUsesItsOwnSensorDirection() { + val camera = StreamPreviewCamera(270, true) + val display = StreamOrientation.values().map { + StreamVideoDisplay.forFrame(it.sensorRotation(270, true), camera, 0) + } + assertEquals(listOf(270, 270, 270, 270), display.map { it.rotation }) + assertEquals(listOf(0, 270, 180, 90), display.map { it.top }) + } + + @Test fun previewFollowsScreenWhileBroadcastKeepsManualOrientation() { + assertEquals(StreamVideoDisplay(0, 270), StreamVideoDisplay.forFrame(90, StreamPreviewCamera(90, false), 90)) + assertEquals(StreamVideoDisplay(0, 90), StreamVideoDisplay.forFrame(270, StreamPreviewCamera(270, true), 90)) + } + + @Test fun remoteViewerKeepsBroadcastRotationWithoutLocalMarker() { + for (rotation in listOf(0, 90, 180, 270)) + assertEquals(StreamVideoDisplay(rotation, null), StreamVideoDisplay.forFrame(rotation, null, 90)) + } + + @Test fun cameraSwitchRetainsPoseForAlreadyQueuedFrames() { + val preview = StreamPreview() + val rear = StreamPreviewCamera(90, false) + val front = StreamPreviewCamera(270, true) + preview.register(1, rear); preview.register(3, front) + assertEquals(rear, preview.camera(1)) + assertEquals(front, preview.camera(3)) + assertNull(preview.camera(2)) + for (revision in 4L..40L) preview.register(revision, rear) + assertNull(preview.camera(1)) + assertEquals(rear, preview.camera(9)) + assertEquals(rear, preview.camera(40)) + } +} diff --git a/tools/chatgui-android/jni_bridge/android_stream_jni.c b/tools/chatgui-android/jni_bridge/android_stream_jni.c index e0a23961..f9a342ec 100644 --- a/tools/chatgui-android/jni_bridge/android_stream_jni.c +++ b/tools/chatgui-android/jni_bridge/android_stream_jni.c @@ -7,6 +7,7 @@ #include "../../../lib/mem.h" #include "../../../lib/opus_codec.h" #include "../libutun_lite/stream_bridge.h" +#include "../../../src/stream/stream_media.h" struct android_stream_opus { opus_codec_encoder_t *encoder; opus_codec_decoder_t *decoder; }; @@ -22,13 +23,17 @@ JNIEXPORT jboolean JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamWatch(J } JNIEXPORT jint JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamFeed( - JNIEnv *env, jobject obj, jlong id, jboolean video, jlong pts, jboolean key, jbyteArray data) { + JNIEnv *env, jobject obj, jlong id, jboolean video, jlong pts, jboolean key, jbyteArray data, + jlong revision, jint width, jint height, jint rotation, jboolean audio) { (void)obj; - if (!data || pts < 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream feed missing data/invalid PTS"); return -1; } + if (!data || pts < 0 || (video && (revision <= 0 || revision > UINT32_MAX || width <= 0 || width > 4096 + || height <= 0 || height > 4096 || rotation < 0 || rotation > 270 || rotation % 90))) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream feed missing data/invalid PTS"); return -1; } jsize length = (*env)->GetArrayLength(env, data); jbyte *bytes = (*env)->GetByteArrayElements(env, data, NULL); if (!bytes) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream feed array access failed"); return -1; } - int result = android_stream_feed((uint64_t)id, video, (uint64_t)pts, key, (const uint8_t *)bytes, (size_t)length); + struct stream_media_format format = {(uint16_t)width, (uint16_t)height, (uint16_t)rotation, + STREAM_MEDIA_H264, audio ? STREAM_MEDIA_OPUS : 0, (uint32_t)revision}; + int result = android_stream_feed((uint64_t)id, video, (uint64_t)pts, key, (const uint8_t *)bytes, (size_t)length, video ? &format : NULL); (*env)->ReleaseByteArrayElements(env, data, bytes, JNI_ABORT); return result; } diff --git a/tools/chatgui-android/libutun_lite/stream_bridge.c b/tools/chatgui-android/libutun_lite/stream_bridge.c index 30c9d5b0..dc9eb07e 100644 --- a/tools/chatgui-android/libutun_lite/stream_bridge.c +++ b/tools/chatgui-android/libutun_lite/stream_bridge.c @@ -91,25 +91,25 @@ static int media_schedule(void) { static void media_output(uint64_t id, int kind, const struct stream_media_format *format, uint64_t pts, int flags, const uint8_t *data, size_t size) { if (!id || size > STREAM_MEDIA_MAX_FRAME) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "Android output invalid id/size=%zu", size); return; } - if (output_queue.count >= RX_RECORDS || size + 20 > RX_LIMIT - output_queue.bytes) { + if (output_queue.count >= RX_RECORDS || size + ANDROID_STREAM_HEADER_SIZE > RX_LIMIT - output_queue.bytes) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "Android stream RX full id=%llu bytes=%zu count=%u", (unsigned long long)id, output_queue.bytes, output_queue.count); viewer.rx_overflow = 1; media_drop_output(); if (kind == 1 || kind == 2) { media_output(id, 3, NULL, 0, 0, NULL, 0); return; } } - struct media_record *record = u_calloc(1, sizeof(*record) + size + 20); + struct media_record *record = u_calloc(1, sizeof(*record) + size + ANDROID_STREAM_HEADER_SIZE); if (!record) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "Android stream RX allocation failed bytes=%zu", size); viewer.rx_overflow = 1; return; } - record->id = id; record->length = size + 20; record->created = get_time_us(); + record->id = id; record->length = size + ANDROID_STREAM_HEADER_SIZE; record->created = get_time_us(); uint8_t *p = record->data; p[0] = (uint8_t)kind; p[1] = (uint8_t)flags; if (format) { stream_write_be(p + 2, format->width, 2); stream_write_be(p + 4, format->height, 2); - stream_write_be(p + 6, format->rotation, 2); + stream_write_be(p + 6, format->rotation, 2); stream_write_be(p + 20, format->revision, 4); } stream_write_be(p + 8, pts, 8); stream_write_be(p + 16, size, 4); - if (size) memcpy(p + 20, data, size); + if (size) memcpy(p + ANDROID_STREAM_HEADER_SIZE, data, size); queue_append(&output_queue, record); } @@ -328,7 +328,7 @@ int android_stream_open(uint64_t id, uint64_t group, uint64_t source, unsigned w } else if (media_schedule() == 0) { *s = (struct android_media_session){.id = id, .group = group, .source = source, .expected_epoch = expected_epoch, .opening = 1, .need_key = 1, - .format = {(uint16_t)width, (uint16_t)height, (uint16_t)rotation, STREAM_MEDIA_H264, audio ? STREAM_MEDIA_OPUS : 0}}; + .format = {(uint16_t)width, (uint16_t)height, (uint16_t)rotation, STREAM_MEDIA_H264, audio ? STREAM_MEDIA_OPUS : 0, 1}}; result = 0; } pthread_mutex_unlock(&media_lock); return result; @@ -344,7 +344,8 @@ int android_stream_watch(uint64_t group) { } /* Перегрузка удаляет целые видео-кадры и блокирует P-кадры до нового KEY. */ -int android_stream_feed(uint64_t id, int video, uint64_t pts, int key, const uint8_t *data, size_t length) { +int android_stream_feed(uint64_t id, int video, uint64_t pts, int key, const uint8_t *data, size_t length, + const struct stream_media_format *format) { if (!data || !length || length > (video ? STREAM_MEDIA_MAX_FRAME : STREAM_MEDIA_MAX_OPUS) || pts > INT64_MAX) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "Android stream feed invalid length=%zu", length); return -1; } @@ -355,23 +356,48 @@ int android_stream_feed(uint64_t id, int video, uint64_t pts, int key, const uin (unsigned long long)id, (unsigned long long)publisher.id, media_inst != NULL, publisher.stopping, publisher.finished); goto done; } + int reconfigure = 0; + if (video) { + if (!format || !format->revision || !format->width || format->width > 4096 || !format->height || format->height > 4096 + || format->rotation > 270 || format->rotation % 90 || format->video_codec != STREAM_MEDIA_H264 + || format->audio_codec != publisher.format.audio_codec) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "Android video feed invalid configuration id=%llu", (unsigned long long)id); goto done; + } + if (format->revision < publisher.format.revision) { + DEBUG_WARN(DEBUG_CATEGORY_STREAM, "Android stale video revision=%u current=%u", format->revision, publisher.format.revision); + result = 0; goto done; + } + reconfigure = format->revision > publisher.format.revision; + if (reconfigure && !key) { DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "Android new revision=%u waits KEY", format->revision); result = 0; goto done; } + if (!reconfigure && (format->width != publisher.format.width || format->height != publisher.format.height + || format->rotation != publisher.format.rotation)) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "Android video configuration changed without revision=%u", format->revision); goto done; + } + } if ((!video && !publisher.format.audio_codec) || (publisher.seen[track] && pts <= publisher.last_pts[track])) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "Android stream feed invalid track/PTS id=%llu pts=%llu", (unsigned long long)id, (unsigned long long)pts); goto done; } - if (video && (video_queue.bytes + audio_queue.bytes + length > TX_LIMIT + if (video && !reconfigure && (video_queue.bytes + audio_queue.bytes + length > TX_LIMIT || video_queue.count + audio_queue.count >= TX_RECORDS)) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "Android TX overload bytes=%zu count=%u; require KEY", video_queue.bytes + audio_queue.bytes, video_queue.count + audio_queue.count); queue_clear(&video_queue); publisher.need_key = 1; } if (video && publisher.need_key && !key) { result = 0; goto done; } - if (video_queue.bytes + audio_queue.bytes + length > TX_LIMIT || video_queue.count + audio_queue.count >= TX_RECORDS) { + if ((reconfigure ? 0 : video_queue.bytes) + audio_queue.bytes + length > TX_LIMIT + || (reconfigure ? 0 : video_queue.count) + audio_queue.count >= TX_RECORDS) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "Android TX admission refused length=%zu video=%d", length, video); result = 0; goto done; } struct media_record *record = u_calloc(1, sizeof(*record) + length); if (!record) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "Android TX allocation failed bytes=%zu", length); goto done; } if (media_schedule() < 0) { u_free(record); goto done; } + if (reconfigure) { + queue_clear(&video_queue); + DEBUG_INFO(DEBUG_CATEGORY_STREAM, "Android video configuration committed id=%llu revision=%u->%u dimensions=%ux%u rotation=%u", + (unsigned long long)id, publisher.format.revision, format->revision, format->width, format->height, format->rotation); + publisher.format = *format; + } record->length = length; record->pts = pts; record->created = get_time_us(); record->key = key ? STREAM_MEDIA_KEY : 0; if (video) { record->frame_id = ++publisher.next_frame; publisher.need_key = 0; } memcpy(record->data, data, length); queue_append(video ? &video_queue : &audio_queue, record); diff --git a/tools/chatgui-android/libutun_lite/stream_bridge.h b/tools/chatgui-android/libutun_lite/stream_bridge.h index d7dc2278..3353c8a2 100644 --- a/tools/chatgui-android/libutun_lite/stream_bridge.h +++ b/tools/chatgui-android/libutun_lite/stream_bridge.h @@ -2,7 +2,7 @@ * UI/кодировщики используют только session_id, никогда не сохраняют inst/ua. * Один publisher и один viewer, TX <=2MB/128 records, RX <=4MB/256 records. * Data принимает полные H.264 Annex B кадры (KEY содержит SPS/PPS/IDR) и Opus 48k mono/20ms. - * pull передаёт владение u_malloc-буфером: kind:1,flags:1,width:2,height:2,rotation:2,PTS_us:8,length:4,data. + * pull передаёт владение u_malloc-буфером: kind:1,flags:1,width:2,height:2,rotation:2,PTS_us:8,length:4,revision:4,data. * kinds: video=1,audio=2,gap=3,opened=4,ended=5,error=6; error.flags = stream_close_reason. Числа big-endian. * Медиаданные передаются через pull, не через общий чатовый event bus. */ #ifndef ANDROID_STREAM_BRIDGE_H @@ -12,6 +12,8 @@ #include struct UTUN_INSTANCE; +struct stream_media_format; +#define ANDROID_STREAM_HEADER_SIZE 24 /* 0 attached, -1 invalid/busy: нельзя передать singleton другому ядру до detach старого. */ int android_stream_attach(struct UTUN_INSTANCE *inst); void android_stream_detach(struct UTUN_INSTANCE *inst); @@ -20,7 +22,9 @@ int android_stream_open(uint64_t id, uint64_t group, uint64_t source, unsigned w unsigned rotation, int audio, uint64_t expected_epoch); /* Наблюдение выбранного канала через CHAT_EVT_STREAM_SOURCES; group=0 снимает его. */ int android_stream_watch(uint64_t group); -int android_stream_feed(uint64_t id, int video, uint64_t pts, int key, const uint8_t *data, size_t length); +/* Видео содержит revision/размер/поворот; новая revision принимается атомарно с KEY. Для аудио format=NULL. */ +int android_stream_feed(uint64_t id, int video, uint64_t pts, int key, const uint8_t *data, size_t length, + const struct stream_media_format *format); /* feed: 1 accepted, 0 dropped (video: запросить KEY), -1 detached/stopped/invalid. */ void android_stream_stop(uint64_t id); uint8_t *android_stream_pull(uint64_t id, size_t *length); diff --git a/tools/chatgui/src/streamplayer_window.cpp b/tools/chatgui/src/streamplayer_window.cpp index 72a45244..9761a428 100644 --- a/tools/chatgui/src/streamplayer_window.cpp +++ b/tools/chatgui/src/streamplayer_window.cpp @@ -179,6 +179,7 @@ void StreamPlayerWindow::decode() { bool waitKey = true, opened = false; int64_t deadline = nowUs() + 10000000, reportAt = nowUs(), expectedAudio = -1; uint64_t decoded = 0, discarded = 0; + uint32_t videoRevision = 0; auto reset = [&] { avcodec_free_context(&decoder); opus_codec_decoder_destroy(opus); opus = nullptr; expectedAudio = -1; waitKey = true; @@ -239,6 +240,13 @@ void StreamPlayerWindow::decode() { } else expectedAudio = static_cast(record.pts) + 20000; continue; } + if (videoRevision != record.format.revision) { + DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop video revision=%u->%u dimensions=%ux%u rotation=%u; preserve audio", + videoRevision, record.format.revision, record.format.width, record.format.height, record.format.rotation); + avcodec_free_context(&decoder); av_frame_unref(frame); waitKey = true; videoRevision = record.format.revision; + { std::lock_guard guard(p.lock); p.videos.clear(); } + deadline = nowUs() + 10000000; + } if (waitKey && !(record.flags & STREAM_MEDIA_KEY)) continue; if (!decoder) { const AVCodec* codec = avcodec_find_decoder(AV_CODEC_ID_H264); diff --git a/tools/chatgui/tests/test_stream_view.cpp b/tools/chatgui/tests/test_stream_view.cpp index e5fa7bb3..a4b57427 100644 --- a/tools/chatgui/tests/test_stream_view.cpp +++ b/tools/chatgui/tests/test_stream_view.cpp @@ -59,6 +59,8 @@ class StreamViewTest : public QObject { uint64_t group = 0, epoch = 0, frameId = 0; QTimer poll; size_t allocations = 0; + uint32_t revision = 1; + uint16_t rotation = 90; bool receive(uint64_t id, StreamRecord& record) { QElapsedTimer elapsed; elapsed.start(); @@ -70,7 +72,7 @@ class StreamViewTest : public QObject { } void publish(const QByteArray& video, const QByteArray& audio = {}) { - stream_media_block media{}; media.format = {160, 90, 90, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS}; + stream_media_block media{}; media.format = {160, 90, rotation, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS, revision}; media.first_frame_offset = 0; media.video_count = 1; media.video[0] = {++frameId, frameId * 100000, static_cast(video.size()), 0, static_cast(video.size()), STREAM_MEDIA_KEY, reinterpret_cast(video.constData())}; @@ -169,6 +171,10 @@ private slots: QSignalSpy shown(&window, &StreamPlayerWindow::framePresented); window.show(); QTimer sender; connect(&sender, &QTimer::timeout, this, [this, video, audio] { publish(video, audio); }); sender.start(100); QTRY_VERIFY_WITH_TIMEOUT(shown.count() > 0, 5000); + int beforeSwitch = shown.count(); revision++; rotation = 270; + QTRY_VERIFY_WITH_TIMEOUT(shown.count() > beforeSwitch + 2, 5000); + stream_info switched{}; QCOMPARE(stream_get_info(publisher, &switched), STREAM_OK); + QCOMPARE(switched.descriptor.source_epoch, epoch); QCOMPARE(switched.owners, 2u); #ifdef TEST_NULL_AUDIO QTRY_VERIFY_WITH_TIMEOUT(nonzeroSamples.load() > 0, 5000); auto* volume = window.findChild(); QVERIFY(volume); volume->setValue(0);