From 0bc2715a309dbdaca871274bd3ff2c33f8e4515c Mon Sep 17 00:00:00 2001 From: evgeny Date: Mon, 19 Oct 2026 17:25:50 +0200 Subject: [PATCH] Preserve Android encoder outputs and throttle raw input before encoding --- tests/test_call_video_playout.c | 4 +-- .../java/com/utun/chat/data/CallVideoCodec.kt | 6 ++++- .../com/utun/chat/data/CallVideoEngine.kt | 26 +++++++++++++++---- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/test_call_video_playout.c b/tests/test_call_video_playout.c index 7c0fb7f9..81ce045e 100644 --- a/tests/test_call_video_playout.c +++ b/tests/test_call_video_playout.c @@ -37,9 +37,9 @@ static void simulate(unsigned pause_ms, int cpu, int64_t quantum, int accelerate if (shown < COUNT && frames[shown].received_us <= now) CHECK(call_video_playout_poll(&p, frames + shown, NULL, now, &audio) == CALL_VIDEO_PLAYOUT_WAIT); } - CHECK(shown == COUNT && accelerated && recovered_lag < 50000); - DEBUG_INFO(DEBUG_CATEGORY_CALL, "PASS sequential recovery pause_ms=%u cpu=%d quantum_us=%lld audio_fast=%d shown=%u/%u lag_us=%lld", + DEBUG_INFO(DEBUG_CATEGORY_CALL, "sequential recovery pause_ms=%u cpu=%d quantum_us=%lld audio_fast=%d shown=%u/%u lag_us=%lld", pause_ms, cpu, (long long)quantum, accelerated_audio, shown, COUNT, (long long)recovered_lag); + CHECK(shown == COUNT && accelerated && recovered_lag < 50000); } static void order_and_clock(void) { diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt index e4afda60..e8b2f41b 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt @@ -24,6 +24,8 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name private var partialFlags = 0 private var preparing = false private var lastOutput = -1L + var pendingFrames = 0 + private set init { try { @@ -69,7 +71,8 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name else for (x in 0 until w) buffer.put(start + x * dst.pixelStride, row[x]) } } - input.close(); codec.queueInputBuffer(index, 0, format.width * format.height * 3 / 2, pts, 0); return true + input.close(); codec.queueInputBuffer(index, 0, format.width * format.height * 3 / 2, pts, 0) + pendingFrames++; return true } /** Вызывается перед очередным IDR; подтверждение ядру — только после проверки этого IDR. */ @@ -115,6 +118,7 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name val output = if (key) type.keyFrame(data, config) else data check(info.presentationTimeUs > lastOutput) { "Encoder reordered PTS ${info.presentationTimeUs}/$lastOutput" } lastOutput = info.presentationTimeUs + check(pendingFrames > 0) { "Encoder output without input" }; pendingFrames-- if (key) { if (preparing) { val actual = checkNotNull(CallVideoNative.inspect(format.codec, output)) { "Invalid encoder bitstream" } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt index e372d3a5..e4c07a9d 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt @@ -73,6 +73,9 @@ class CallVideoEngine(private val context: Context, private val id: Long) { private var encodedFrames = 0 private var rejectedGeneration = 0L private var excluded = emptySet() + private data class Pending(val metadata: LongArray, val bytes: ByteArray) + private val pendingFrames = java.util.ArrayDeque() + private var canEncode = false init { preview.bind { target -> @@ -138,6 +141,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { try { val s = CallVideoNative.status(id) ?: return check(s[5] == 0L) { "Core rejected video format" } + canEncode = s[10] != 0L if (s[6] >= 300000 && s[7] >= s[6]) mutable.value = mutable.value.copy(range = CallVideoBitrateRange((s[6] / 1000).toInt(), (s[7] / 1000).toInt())) if (keySerial != s[0]) { keySerial = s[0]; needKey = true } val current = format @@ -158,7 +162,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val pending = if (confirmingRequest != 0L) confirmingRequest else pendingRequest if (pending != 0L) CallVideoNative.applied(id, format?.generation ?: 0, pending, 0) confirmingRequest = 0; pendingRequest = 0 - closeCamera(); encoder?.close(); encoder = null; active = null; report("Video off") + closeCamera(); encoder?.close(); encoder = null; pendingFrames.clear(); active = null; report("Video off") } } else if (wanted != active && !opening && !closing && (lastKeyRequest < 0 || System.nanoTime() / 1000 - lastKeyRequest >= 1_000_000)) { val peer = CallVideoNative.peer(id) @@ -178,7 +182,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val previousName = previous?.name; val previousRate = previous?.bitrate val range = mutable.value.range val initialRate = (previousRate ?: 1_000_000).coerceIn(range.minimum * 1000, range.maximum * 1000) - previous?.close(); encoder = null + previous?.close(); encoder = null; pendingFrames.clear() for (codec in if (wanted.codec == 0) listOf(2, 1) else listOf(wanted.codec)) { if (codec in excluded) continue val implementation = encoders[codec] ?: continue @@ -230,6 +234,10 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val pts = (image.timestamp + offset) / 1000 // Camera2 уже ограничена 30 fps. Порог 33333 мкс отбрасывал каждый второй кадр при дрожании часов. if (pts <= lastPts) return@setOnImageAvailableListener + drain() + // До восьми принятых входов/готовых выходов: backpressure действует до кодирования. + if (!canEncode || pendingFrames.isNotEmpty() || (encoder?.pendingFrames ?: 0) >= 8) + return@setOnImageAvailableListener // Автоматический IDR кодека не переносит нашу границу: иначе смена bitrate может ждать бесконечно. val key = needKey || confirmingRequest != 0L || lastKeyRequest < 0 || pts - lastKeyRequest >= 1_000_000 if (key && pendingRequest != 0L) { @@ -345,14 +353,22 @@ class CallVideoEngine(private val context: Context, private val id: Long) { mutable.value = mutable.value.copy(bitrate = codec.bitrate) LogManager.addLog("INFO", "call", "video bitrate applied id=$id generation=${f.generation} bitrate=${codec.bitrate} pts=${p.pts}") } - if (needKey && !p.key) continue val metadata = f.array() + longArrayOf(++frame, p.pts, if (p.key) 1 else 0) - needKey = CallVideoNative.send(id, metadata, p.bytes) != 1 + pendingFrames.addLast(Pending(metadata, p.bytes)) + if (p.key) needKey = false + } + while (pendingFrames.isNotEmpty()) { + val p = pendingFrames.first + val sent = CallVideoNative.send(id, p.metadata, p.bytes) + if (sent == 0) break + check(sent == 1) { "Video TX rejected encoded frame ${p.metadata[9]}" } + pendingFrames.removeFirst() } val now = System.nanoTime() if (now - reportAt > 1_000_000_000) { if (reportAt != 0L) LogManager.addLog("DEBUG", "call", "video output id=$id fps=$encodedFrames " + - "bps=${encodedBytes * 8_000_000_000 / (now - reportAt)} target=${codec.bitrate} needKey=$needKey") + "bps=${encodedBytes * 8_000_000_000 / (now - reportAt)} target=${codec.bitrate} needKey=$needKey " + + "pending=${pendingFrames.size} codecPending=${codec.pendingFrames} canEncode=$canEncode") reportAt = now; encodedFrames = 0; encodedBytes = 0 } }