Probe Android encoders and decoders independently with actual formats

tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt

@@ -10,7 +10,9 @@ import android.os.Bundle
 import java.nio.ByteBuffer
 
 /** Codec принадлежит одному worker. YUV input отделён от камеры: замена на IDR не меняет capture PTS. */
-internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, name: String) : AutoCloseable {
+internal class CallVideoCodec(requested: CallVideoFormat, val bitrate: Int, val name: String) : AutoCloseable {
+    var format = requested
+        private set
     data class Packet(val pts: Long, val key: Boolean, val bytes: ByteArray)
     private val type = StreamVideoCodec.fromId(format.codec)
     private val codec = MediaCodec.createByCodecName(name)
@@ -18,12 +20,17 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam
     private val partial = java.io.ByteArrayOutputStream()
     private var partialPts = -1L
     private var partialFlags = 0
+    private var preparing = false
+    private var lastOutput = -1L
 
     init {
         try {
             val media = MediaFormat.createVideoFormat(type.mime, format.width, format.height).apply {
                 setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible)
-                setInteger(MediaFormat.KEY_PROFILE, if (format.codec == 1) MediaCodecInfo.CodecProfileLevel.AVCProfileConstrainedBaseline
+                setInteger(MediaFormat.KEY_PROFILE, if (format.codec == 1) {
+                    if (format.profile == 1) MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline
+                    else MediaCodecInfo.CodecProfileLevel.AVCProfileConstrainedBaseline
+                }
                     else MediaCodecInfo.CodecProfileLevel.HEVCProfileMain)
                 setInteger(MediaFormat.KEY_LEVEL, platformLevel(format.codec, format.level))
                 setInteger(MediaFormat.KEY_BIT_RATE, bitrate); setInteger(MediaFormat.KEY_FRAME_RATE, format.fps)
@@ -92,7 +99,16 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam
                 val data = type.normalize(partial.toByteArray()); partial.reset()
                 val key = partialFlags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0
                 val output = if (key) type.keyFrame(data, config) else data
-                if (key) validateKey(output, format)
+                check(info.presentationTimeUs > lastOutput) { "Encoder reordered PTS ${info.presentationTimeUs}/$lastOutput" }
+                lastOutput = info.presentationTimeUs
+                if (key) {
+                    if (preparing) {
+                        val actual = checkNotNull(CallVideoNative.inspect(format.codec, output)) { "Invalid encoder bitstream" }
+                        check(actual[0] == format.width.toLong() && actual[1] == format.height.toLong()) { "Encoder changed geometry" }
+                        format = format.copy(level = actual[2].toInt(), profile = actual[3].toInt())
+                    }
+                    validateKey(output, format)
+                }
                 packets += Packet(info.presentationTimeUs, key, output)
             } finally { codec.releaseOutputBuffer(index, false) }
         }
@@ -102,6 +118,21 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam
         try { codec.stop() } catch (e: Exception) { LogManager.addLog("WARN", "call", "encoder stop: ${e.message}") }
         codec.release()
     }
+    /** Проверяем фактический режим до публикации; пробные кадры не уходят в звонок. */
+    fun prepare() {
+        preparing = true
+        try {
+            var input = 0; var output = 0; var key = false
+            val deadline = System.nanoTime() + 5_000_000_000
+            while (output < 8 && System.nanoTime() < deadline) {
+                if (input < 8 && submit(null, 1_000_000L + input * 33333, input == 0)) input++
+                for (packet in drain()) { if (output == 0) key = packet.key; output++ }
+                Thread.sleep(2)
+            }
+            check(input == 8 && output == 8 && key) { "Encoder probe incomplete inputs=$input outputs=$output IDR=$key" }
+            LogManager.addLog("INFO", "call", "video encoder verified name=$name actual=$format bitrate=$bitrate")
+        } finally { preparing = false }
+    }
 
     companion object {
         fun platformLevel(codec: Int, level: Int): Int = if (codec == 1) when (level) {
@@ -113,39 +144,58 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam
             93 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel31
             else -> error("Unsupported HEVC level $level")
         }
-        /** Реальная проба input → KEY → decoder; не объявляем codec только по списку MediaCodecInfo. */
-        fun probe(): Map<Int, String> {
-            val result = mutableMapOf<Int, String>()
+        data class Encoder(val name: String, val profile: Int, val level: Int)
+        data class Decoder(val name: String, val profiles: Int)
+        /** Encode и decode независимы: ошибка собственного encoder не запрещает приём. */
+        fun probe(): Map<Int, Encoder> {
+            val result = mutableMapOf<Int, Encoder>()
             for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) {
                 for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { it.isEncoder }) {
                     if (entry.supportedTypes.none { it == type.mime }) continue
                     try {
-                        val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93)
+                        val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93,
+                            profile = if (type.id == 1) 2 else 1)
                         val caps = entry.getCapabilitiesForType(type.mime)
                         if (caps.videoCapabilities?.areSizeAndRateSupported(1280, 720, 30.0) != true
                             || caps.encoderCapabilities?.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR) != true) continue
                         CallVideoCodec(f, 1_000_000, entry.name).use { encoder ->
-                            val packets = mutableListOf<Packet>(); var input = 0
-                            val deadline = System.nanoTime() + 5_000_000_000
-                            while (packets.size < 4 && System.nanoTime() < deadline) {
-                                if (input < 8 && encoder.submit(null, 1_000_000L + input * 33333, input == 0)) input++
-                                packets += encoder.drain(); Thread.sleep(2)
-                            }
-                            check(packets.firstOrNull()?.key == true) { "No initial KEY" }
-                            verifyDecode(f, packets)
+                            encoder.prepare()
+                            result[type.id] = Encoder(entry.name, encoder.format.profile, encoder.format.level)
                         }
-                        result[type.id] = entry.name
                         LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name}"); break
                     } catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name}: ${e.message}") }
                 }
             }
             return result
         }
-
-        private fun verifyDecode(f: CallVideoFormat, packets: List<Packet>) {
+        fun probeDecoders(): Map<Int, Decoder> {
+            val result = mutableMapOf<Int, Decoder>()
+            for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) {
+                for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { !it.isEncoder }) {
+                    if (entry.supportedTypes.none { it == type.mime }) continue
+                    try {
+                        val caps = entry.getCapabilitiesForType(type.mime)
+                        if (caps.videoCapabilities?.areSizeAndRateSupported(1280, 720, 30.0) != true) continue
+                        var profiles = 0
+                        for (profile in if (type.id == 1) listOf(1, 2) else listOf(1)) {
+                            val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93, profile = profile)
+                            val key = checkNotNull(CallVideoNative.probeKey(type.id, profile))
+                            validateKey(key, f)
+                            verifyDecode(f, List(4) { Packet(1_000_000L + it * 33333, true, key) }, entry.name)
+                            profiles = profiles or profile
+                        }
+                        result[type.id] = Decoder(entry.name, profiles)
+                        LogManager.addLog("INFO", "call", "video decoder verified codec=${type.id} name=${entry.name} profiles=$profiles")
+                        break
+                    } catch (e: Exception) { LogManager.addLog("WARN", "call", "video decoder probe rejected ${entry.name}: ${e.message}") }
+                }
+            }
+            return result
+        }
+        private fun verifyDecode(f: CallVideoFormat, packets: List<Packet>, name: String) {
             val type = StreamVideoCodec.fromId(f.codec)
             val config = type.unpackKey(packets.first().bytes).first
-            val decoder = MediaCodec.createDecoderByType(type.mime)
+            val decoder = MediaCodec.createByCodecName(name)
             try {
                 decoder.configure(type.decoderFormat(f.width, f.height, 0, config), null, null, 0); decoder.start()
                 var sent = 0; var decoded = 0; val info = MediaCodec.BufferInfo(); val deadline = System.nanoTime() + 2_000_000_000
@@ -170,40 +220,13 @@ internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, nam
             } finally { decoder.release() }
         }
 
-        /** Проверяем профиль и level в SPS, а не принятый configure option. */
+        /** Единый C validator используется также ChatGUI. */
         fun validateKey(bytes: ByteArray, f: CallVideoFormat) {
-            val type = StreamVideoCodec.fromId(f.codec); val (config, data) = type.unpackKey(bytes)
-            val units = H264AnnexB.nals(config + data).map { (_, n) -> n.copyOfRange(if (n[2] == 1.toByte()) 3 else 4, n.size) }
-            val sps = units.first { if (f.codec == 1) it[0].toInt() and 31 == 7 else (it[0].toInt() ushr 1) and 63 == 33 }
-            val raw = java.io.ByteArrayOutputStream(); var zeros = 0
-            for (b in sps.drop(if (f.codec == 1) 1 else 2)) {
-                if (zeros >= 2 && b == 3.toByte()) { zeros = 0; continue }
-                raw.write(b.toInt()); zeros = if (b == 0.toByte()) zeros + 1 else 0
-            }
-            val bits = Bits(raw.toByteArray())
-            if (f.codec == 1) {
-                val profile = bits.read(8); val constraints = bits.read(8); val level = bits.read(8)
-                check(profile == 66 && constraints and 64 != 0) { "AVC profile=$profile constraints=$constraints level=$level SPS=${sps.joinToString { "%02x".format(it) }}" }
-                check(level <= f.level); bits.ue(); bits.ue()
-                when (bits.ue()) { 0 -> bits.ue(); 2 -> Unit; else -> error("Unsupported AVC POC") }
-                check(bits.ue() <= 1) { "AVC uses more than one reference" }
-            } else {
-                bits.read(4); check(bits.read(3) == 0) { "HEVC temporal layers" }; bits.read(1)
-                bits.read(2); val tier = bits.read(1); val profile = bits.read(5)
-                check(tier == 0 && profile == 1) { "HEVC profile=$profile tier=$tier SPS=${sps.joinToString { "%02x".format(it) }}" }
-                repeat(10) { bits.read(8) }; check(bits.read(8) <= f.level)
-                bits.ue(); check(bits.ue() == 1); bits.ue(); bits.ue()
-                if (bits.read(1) != 0) repeat(4) { bits.ue() }
-                check(bits.ue() == 0 && bits.ue() == 0); bits.ue(); bits.read(1)
-                val dpb = bits.ue(); check(bits.ue() == 0) { "HEVC reordered frames" }
-                check(dpb <= 1) { "HEVC decoded pictures=${dpb + 1} SPS=${sps.joinToString { "%02x".format(it) }}" }
+            val s = checkNotNull(CallVideoNative.inspect(f.codec, bytes)) { "Invalid video KEY bitstream" }
+            check(s[0] == f.width.toLong() && s[1] == f.height.toLong() && s[2] <= f.level
+                && (s[3] == f.profile.toLong() || (f.codec == 1 && f.profile == 1 && s[3] == 2L))) {
+                "Video KEY mismatch actual=${s.contentToString()} expected=$f"
             }
         }
     }
-    private class Bits(val bytes: ByteArray) {
-        var at = 0
-        fun read(n: Int): Int { require(n in 0..24 && at + n <= bytes.size * 8); var v = 0
-            repeat(n) { v = (v shl 1) or ((bytes[at / 8].toInt() ushr (7 - at % 8)) and 1); at++ }; return v }
-        fun ue(): Int { var n = 0; while (read(1) == 0) { n++; require(n <= 20) }; return (1 shl n) - 1 + read(n) }
-    }
 }

tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt

@@ -15,6 +15,7 @@ internal object CallVideoNative {
     external fun requestKey(id: Long)
     external fun range(id: Long, minimum: Int, maximum: Int): Boolean
     external fun inspect(codec: Int, bytes: ByteArray): LongArray?
+    external fun probeKey(codec: Int, profile: Int): ByteArray?
     external fun supported(local: ByteArray, peer: ByteArray, format: LongArray): Boolean
 }
 

tools/chatgui-android/jni_bridge/android_call_video_jni.c

@@ -110,6 +110,11 @@ JNIEXPORT jlongArray JNICALL Java_com_utun_chat_data_CallVideoNative_inspect(JNI
     jlong values[] = {s.width, s.height, s.level, s.profile, s.dpb};
     jlongArray out = (*e)->NewLongArray(e, 5); if (out) (*e)->SetLongArrayRegion(e, out, 0, 5, values); return out;
 }
+JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_CallVideoNative_probeKey(JNIEnv *e, jobject o, jint codec, jint profile) {
+    (void)o; uint8_t bytes[8192]; int size = call_video_probe_key(codec, profile, bytes, sizeof(bytes));
+    if (size < 0) return NULL;
+    jbyteArray out = (*e)->NewByteArray(e, size); if (out) (*e)->SetByteArrayRegion(e, out, 0, size, (jbyte *)bytes); return out;
+}
 JNIEXPORT jboolean JNICALL Java_com_utun_chat_data_CallVideoNative_supported(
     JNIEnv *e, jobject o, jbyteArray local, jbyteArray remote, jlongArray array) {
     (void)o; struct call_video_caps sender, receiver; struct call_video_format f; jlong values[12];