From 47cca28d394f814da4cb5f2494bf45688084c93e Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 14 Oct 2026 23:12:00 +0200 Subject: [PATCH] Probe Android encoders and decoders independently with actual formats --- .../java/com/utun/chat/data/CallVideoCodec.kt | 125 +++++++++++------- .../com/utun/chat/data/CallVideoNative.kt | 1 + .../jni_bridge/android_call_video_jni.c | 5 + 3 files changed, 80 insertions(+), 51 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt index e2289f5c..a09299ab 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt @@ -10,7 +10,9 @@ import android.os.Bundle import java.nio.ByteBuffer /** Codec принадлежит одному worker. YUV input отделён от камеры: замена на IDR не меняет capture PTS. */ -internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, name: String) : AutoCloseable { +internal class CallVideoCodec(requested: CallVideoFormat, val bitrate: Int, val name: String) : AutoCloseable { + var format = requested + private set data class Packet(val pts: Long, val key: Boolean, val bytes: ByteArray) private val type = StreamVideoCodec.fromId(format.codec) private val codec = MediaCodec.createByCodecName(name) @@ -18,12 +20,17 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam private val partial = java.io.ByteArrayOutputStream() private var partialPts = -1L private var partialFlags = 0 + private var preparing = false + private var lastOutput = -1L init { try { val media = MediaFormat.createVideoFormat(type.mime, format.width, format.height).apply { setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) - setInteger(MediaFormat.KEY_PROFILE, if (format.codec == 1) MediaCodecInfo.CodecProfileLevel.AVCProfileConstrainedBaseline + setInteger(MediaFormat.KEY_PROFILE, if (format.codec == 1) { + if (format.profile == 1) MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline + else MediaCodecInfo.CodecProfileLevel.AVCProfileConstrainedBaseline + } else MediaCodecInfo.CodecProfileLevel.HEVCProfileMain) setInteger(MediaFormat.KEY_LEVEL, platformLevel(format.codec, format.level)) setInteger(MediaFormat.KEY_BIT_RATE, bitrate); setInteger(MediaFormat.KEY_FRAME_RATE, format.fps) @@ -92,7 +99,16 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam val data = type.normalize(partial.toByteArray()); partial.reset() val key = partialFlags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0 val output = if (key) type.keyFrame(data, config) else data - if (key) validateKey(output, format) + check(info.presentationTimeUs > lastOutput) { "Encoder reordered PTS ${info.presentationTimeUs}/$lastOutput" } + lastOutput = info.presentationTimeUs + if (key) { + if (preparing) { + val actual = checkNotNull(CallVideoNative.inspect(format.codec, output)) { "Invalid encoder bitstream" } + check(actual[0] == format.width.toLong() && actual[1] == format.height.toLong()) { "Encoder changed geometry" } + format = format.copy(level = actual[2].toInt(), profile = actual[3].toInt()) + } + validateKey(output, format) + } packets += Packet(info.presentationTimeUs, key, output) } finally { codec.releaseOutputBuffer(index, false) } } @@ -102,6 +118,21 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam try { codec.stop() } catch (e: Exception) { LogManager.addLog("WARN", "call", "encoder stop: ${e.message}") } codec.release() } + /** Проверяем фактический режим до публикации; пробные кадры не уходят в звонок. */ + fun prepare() { + preparing = true + try { + var input = 0; var output = 0; var key = false + val deadline = System.nanoTime() + 5_000_000_000 + while (output < 8 && System.nanoTime() < deadline) { + if (input < 8 && submit(null, 1_000_000L + input * 33333, input == 0)) input++ + for (packet in drain()) { if (output == 0) key = packet.key; output++ } + Thread.sleep(2) + } + check(input == 8 && output == 8 && key) { "Encoder probe incomplete inputs=$input outputs=$output IDR=$key" } + LogManager.addLog("INFO", "call", "video encoder verified name=$name actual=$format bitrate=$bitrate") + } finally { preparing = false } + } companion object { fun platformLevel(codec: Int, level: Int): Int = if (codec == 1) when (level) { @@ -113,39 +144,58 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam 93 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel31 else -> error("Unsupported HEVC level $level") } - /** Реальная проба input → KEY → decoder; не объявляем codec только по списку MediaCodecInfo. */ - fun probe(): Map { - val result = mutableMapOf() + data class Encoder(val name: String, val profile: Int, val level: Int) + data class Decoder(val name: String, val profiles: Int) + /** Encode и decode независимы: ошибка собственного encoder не запрещает приём. */ + fun probe(): Map { + val result = mutableMapOf() for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { it.isEncoder }) { if (entry.supportedTypes.none { it == type.mime }) continue try { - val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93) + val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93, + profile = if (type.id == 1) 2 else 1) val caps = entry.getCapabilitiesForType(type.mime) if (caps.videoCapabilities?.areSizeAndRateSupported(1280, 720, 30.0) != true || caps.encoderCapabilities?.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR) != true) continue CallVideoCodec(f, 1_000_000, entry.name).use { encoder -> - val packets = mutableListOf(); var input = 0 - val deadline = System.nanoTime() + 5_000_000_000 - while (packets.size < 4 && System.nanoTime() < deadline) { - if (input < 8 && encoder.submit(null, 1_000_000L + input * 33333, input == 0)) input++ - packets += encoder.drain(); Thread.sleep(2) - } - check(packets.firstOrNull()?.key == true) { "No initial KEY" } - verifyDecode(f, packets) + encoder.prepare() + result[type.id] = Encoder(entry.name, encoder.format.profile, encoder.format.level) } - result[type.id] = entry.name LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name}"); break } catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name}: ${e.message}") } } } return result } - - private fun verifyDecode(f: CallVideoFormat, packets: List) { + fun probeDecoders(): Map { + val result = mutableMapOf() + for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { + for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { !it.isEncoder }) { + if (entry.supportedTypes.none { it == type.mime }) continue + try { + val caps = entry.getCapabilitiesForType(type.mime) + if (caps.videoCapabilities?.areSizeAndRateSupported(1280, 720, 30.0) != true) continue + var profiles = 0 + for (profile in if (type.id == 1) listOf(1, 2) else listOf(1)) { + val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93, profile = profile) + val key = checkNotNull(CallVideoNative.probeKey(type.id, profile)) + validateKey(key, f) + verifyDecode(f, List(4) { Packet(1_000_000L + it * 33333, true, key) }, entry.name) + profiles = profiles or profile + } + result[type.id] = Decoder(entry.name, profiles) + LogManager.addLog("INFO", "call", "video decoder verified codec=${type.id} name=${entry.name} profiles=$profiles") + break + } catch (e: Exception) { LogManager.addLog("WARN", "call", "video decoder probe rejected ${entry.name}: ${e.message}") } + } + } + return result + } + private fun verifyDecode(f: CallVideoFormat, packets: List, name: String) { val type = StreamVideoCodec.fromId(f.codec) val config = type.unpackKey(packets.first().bytes).first - val decoder = MediaCodec.createDecoderByType(type.mime) + val decoder = MediaCodec.createByCodecName(name) try { decoder.configure(type.decoderFormat(f.width, f.height, 0, config), null, null, 0); decoder.start() var sent = 0; var decoded = 0; val info = MediaCodec.BufferInfo(); val deadline = System.nanoTime() + 2_000_000_000 @@ -170,40 +220,13 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam } finally { decoder.release() } } - /** Проверяем профиль и level в SPS, а не принятый configure option. */ + /** Единый C validator используется также ChatGUI. */ fun validateKey(bytes: ByteArray, f: CallVideoFormat) { - val type = StreamVideoCodec.fromId(f.codec); val (config, data) = type.unpackKey(bytes) - val units = H264AnnexB.nals(config + data).map { (_, n) -> n.copyOfRange(if (n[2] == 1.toByte()) 3 else 4, n.size) } - val sps = units.first { if (f.codec == 1) it[0].toInt() and 31 == 7 else (it[0].toInt() ushr 1) and 63 == 33 } - val raw = java.io.ByteArrayOutputStream(); var zeros = 0 - for (b in sps.drop(if (f.codec == 1) 1 else 2)) { - if (zeros >= 2 && b == 3.toByte()) { zeros = 0; continue } - raw.write(b.toInt()); zeros = if (b == 0.toByte()) zeros + 1 else 0 - } - val bits = Bits(raw.toByteArray()) - if (f.codec == 1) { - val profile = bits.read(8); val constraints = bits.read(8); val level = bits.read(8) - check(profile == 66 && constraints and 64 != 0) { "AVC profile=$profile constraints=$constraints level=$level SPS=${sps.joinToString { "%02x".format(it) }}" } - check(level <= f.level); bits.ue(); bits.ue() - when (bits.ue()) { 0 -> bits.ue(); 2 -> Unit; else -> error("Unsupported AVC POC") } - check(bits.ue() <= 1) { "AVC uses more than one reference" } - } else { - bits.read(4); check(bits.read(3) == 0) { "HEVC temporal layers" }; bits.read(1) - bits.read(2); val tier = bits.read(1); val profile = bits.read(5) - check(tier == 0 && profile == 1) { "HEVC profile=$profile tier=$tier SPS=${sps.joinToString { "%02x".format(it) }}" } - repeat(10) { bits.read(8) }; check(bits.read(8) <= f.level) - bits.ue(); check(bits.ue() == 1); bits.ue(); bits.ue() - if (bits.read(1) != 0) repeat(4) { bits.ue() } - check(bits.ue() == 0 && bits.ue() == 0); bits.ue(); bits.read(1) - val dpb = bits.ue(); check(bits.ue() == 0) { "HEVC reordered frames" } - check(dpb <= 1) { "HEVC decoded pictures=${dpb + 1} SPS=${sps.joinToString { "%02x".format(it) }}" } + val s = checkNotNull(CallVideoNative.inspect(f.codec, bytes)) { "Invalid video KEY bitstream" } + check(s[0] == f.width.toLong() && s[1] == f.height.toLong() && s[2] <= f.level + && (s[3] == f.profile.toLong() || (f.codec == 1 && f.profile == 1 && s[3] == 2L))) { + "Video KEY mismatch actual=${s.contentToString()} expected=$f" } } } - private class Bits(val bytes: ByteArray) { - var at = 0 - fun read(n: Int): Int { require(n in 0..24 && at + n <= bytes.size * 8); var v = 0 - repeat(n) { v = (v shl 1) or ((bytes[at / 8].toInt() ushr (7 - at % 8)) and 1); at++ }; return v } - fun ue(): Int { var n = 0; while (read(1) == 0) { n++; require(n <= 20) }; return (1 shl n) - 1 + read(n) } - } } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt index 4bf4f3b9..6418056b 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt @@ -15,6 +15,7 @@ internal object CallVideoNative { external fun requestKey(id: Long) external fun range(id: Long, minimum: Int, maximum: Int): Boolean external fun inspect(codec: Int, bytes: ByteArray): LongArray? + external fun probeKey(codec: Int, profile: Int): ByteArray? external fun supported(local: ByteArray, peer: ByteArray, format: LongArray): Boolean } diff --git a/tools/chatgui-android/jni_bridge/android_call_video_jni.c b/tools/chatgui-android/jni_bridge/android_call_video_jni.c index 22c77821..34422f78 100644 --- a/tools/chatgui-android/jni_bridge/android_call_video_jni.c +++ b/tools/chatgui-android/jni_bridge/android_call_video_jni.c @@ -110,6 +110,11 @@ JNIEXPORT jlongArray JNICALL Java_com_utun_chat_data_CallVideoNative_inspect(JNI jlong values[] = {s.width, s.height, s.level, s.profile, s.dpb}; jlongArray out = (*e)->NewLongArray(e, 5); if (out) (*e)->SetLongArrayRegion(e, out, 0, 5, values); return out; } +JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_CallVideoNative_probeKey(JNIEnv *e, jobject o, jint codec, jint profile) { + (void)o; uint8_t bytes[8192]; int size = call_video_probe_key(codec, profile, bytes, sizeof(bytes)); + if (size < 0) return NULL; + jbyteArray out = (*e)->NewByteArray(e, size); if (out) (*e)->SetByteArrayRegion(e, out, 0, size, (jbyte *)bytes); return out; +} JNIEXPORT jboolean JNICALL Java_com_utun_chat_data_CallVideoNative_supported( JNIEnv *e, jobject o, jbyteArray local, jbyteArray remote, jlongArray array) { (void)o; struct call_video_caps sender, receiver; struct call_video_format f; jlong values[12];