From 4a759115c9eaad615901ed0b34e12fd0801b3f28 Mon Sep 17 00:00:00 2001 From: evgeny Date: Tue, 20 Oct 2026 00:09:56 +0200 Subject: [PATCH] Probe Android video codecs at 30, 50 and 60 fps --- .../java/com/utun/chat/data/CallVideoCodec.kt | 57 ++++++++++++------- .../com/utun/chat/data/CallVideoNative.kt | 2 +- .../jni_bridge/android_call_video_jni.c | 4 +- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt index be8019fc..e1abee9b 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt @@ -183,7 +183,7 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name var input = 0; var output = 0; var key = false val deadline = System.nanoTime() + 5_000_000_000 while (output < 8 && System.nanoTime() < deadline) { - if (input < 8 && submit(null, 1_000_000L + input * 33333, input == 0)) input++ + if (input < 8 && submit(null, 1_000_000L + input * 1_000_000L / format.fps, input == 0)) input++ for (packet in drain()) { if (output == 0) key = packet.key; output++ } Thread.sleep(2) } @@ -196,28 +196,36 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name fun platformLevel(codec: Int, level: Int): Int = if (codec == 1) when (level) { 30 -> MediaCodecInfo.CodecProfileLevel.AVCLevel3 31 -> MediaCodecInfo.CodecProfileLevel.AVCLevel31 + 32 -> MediaCodecInfo.CodecProfileLevel.AVCLevel32 40 -> MediaCodecInfo.CodecProfileLevel.AVCLevel4 + 42 -> MediaCodecInfo.CodecProfileLevel.AVCLevel42 else -> error("Unsupported AVC level $level") } else when (level) { 90 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel3 93 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel31 120 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel4 + 123 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel41 else -> error("Unsupported HEVC level $level") } - data class EncoderMode(val size: Size, val name: String, val profile: Int, val level: Int) + data class EncoderMode(val size: Size, val name: String, val profile: Int, val level: Int, val fps: Int = 30) data class Encoder(val modes: List) { - val sizes get() = modes.map { it.size } + val sizes get() = modes.map { it.size }.distinct() + val fps get() = modes.maxOf { it.fps } + val pixelsPerSecond get() = modes.maxOf { it.size.width * it.size.height * it.fps } val profile get() = modes.fold(0) { bits, mode -> bits or mode.profile } val level get() = modes.maxOf { it.level } } - data class DecoderMode(val size: Size, val name: String, val profiles: Int, val level: Int, val pipeline: Int) { - fun supports(f: CallVideoFormat) = profiles and f.profile != 0 && f.level <= level && + data class DecoderMode(val size: Size, val name: String, val profiles: Int, val level: Int, val pipeline: Int, val fps: Int = 30) { + fun supports(f: CallVideoFormat) = profiles and f.profile != 0 && f.level <= level && f.fps <= fps && + f.width.toLong() * f.height * f.fps <= size.width.toLong() * size.height * fps && maxOf(f.width, f.height) <= maxOf(size.width, size.height) && f.width.toLong() * f.height <= size.width.toLong() * size.height } data class Decoder(val modes: List) { val size get() = modes.maxBy { it.size.width * it.size.height }.size val profiles get() = modes.fold(0) { bits, mode -> bits or mode.profiles } val level get() = modes.maxOf { it.level } + val fps get() = modes.maxOf { it.fps } + val pixelsPerSecond get() = modes.maxOf { it.size.width * it.size.height * it.fps } } /** Аппаратные реализации проверяются первыми; software остаётся резервом после реальной пробы. */ private fun candidates(encoder: Boolean) = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos @@ -241,6 +249,7 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { val modes = mutableListOf() for (size in listOf(Size(640, 480), Size(1280, 720), Size(1920, 1080))) { + for (fps in listOf(30, 50, 60)) { for (entry in candidates(true)) { if (entry.supportedTypes.none { it == type.mime }) continue try { @@ -248,13 +257,14 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name if (caps.encoderCapabilities?.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR) != true) continue var foundProfile = 0; var foundLevel = 0 for ((w, h) in listOf(size.width to size.height, size.height to size.width)) { - check(caps.videoCapabilities?.areSizeAndRateSupported(w, h, 30.0) == true) { "Geometry unsupported: ${w}x$h" } + check(caps.videoCapabilities?.areSizeAndRateSupported(w, h, fps.toDouble()) == true) { "Geometry/rate unsupported: ${w}x$h@$fps" } val fullhd = size.width > 1280 - val f = CallVideoFormat(1, w, h, codec = type.id, - level = if (type.id == 1) { if (fullhd) 40 else 31 } else { if (fullhd) 120 else 93 }, + val f = CallVideoFormat(1, w, h, fps = fps, codec = type.id, + level = if (type.id == 1) { if (fullhd) { if (fps > 30) 42 else 40 } else { if (fps > 30) 32 else 31 } } + else { if (fullhd) { if (fps > 30) 123 else 120 } else { if (fps > 30) 120 else 93 } }, profile = if (type.id == 1) 2 else 1) var verified: CallVideoFormat? = null - for (level in (if (type.id == 2) listOf(f.level, 120) else listOf(f.level)).distinct()) { + for (level in (if (type.id == 2) listOf(f.level, maxOf(f.level, 120)) else listOf(f.level)).distinct()) { try { CallVideoCodec(f.copy(level = level), 1_000_000, entry.name).use { encoder -> encoder.prepare(); verified = encoder.format @@ -268,10 +278,11 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name check(foundProfile == 0 || foundProfile == actual.profile) { "Geometry changes profile/tier" } foundProfile = actual.profile; foundLevel = maxOf(foundLevel, actual.level) } - modes += EncoderMode(size, entry.name, foundProfile, foundLevel) - LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name} size=$size"); break - } catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name} size=$size: ${e.message}") } + modes += EncoderMode(size, entry.name, foundProfile, foundLevel, fps) + LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name} size=$size fps=$fps"); break + } catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name} size=$size fps=$fps: ${e.message}") } } + } } if (modes.isNotEmpty()) result[type.id] = Encoder(modes) } @@ -282,27 +293,28 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { val modes = mutableListOf() for (size in listOf(Size(1280, 720), Size(1920, 1080))) { + for (fps in listOf(30, 50, 60)) { for (entry in candidates(false)) { if (entry.supportedTypes.none { it == type.mime }) continue try { val caps = entry.getCapabilitiesForType(type.mime) - if (caps.videoCapabilities?.areSizeAndRateSupported(size.width, size.height, 30.0) != true) continue + if (caps.videoCapabilities?.areSizeAndRateSupported(size.width, size.height, fps.toDouble()) != true) continue var profiles = 0; var pipeline = 0; var maximumLevel = 0 for (profile in listOf(1, 2)) { try { - val level = if (type.id == 1) { if (size.width > 1280) 40 else 31 } - else if (size.width > 1280 || profile == 2) 120 else 93 + val level = if (type.id == 1) { if (size.width > 1280) { if (fps > 30) 42 else 40 } else { if (fps > 30) 32 else 31 } } + else if (size.width > 1280) { if (fps > 30) 123 else 120 } else if (fps > 30 || profile == 2) 120 else 93 var profilePipeline = 0 for (portrait in listOf(false, true)) { val w = if (portrait) size.height else size.width; val h = if (portrait) size.width else size.height - check(caps.videoCapabilities?.areSizeAndRateSupported(w, h, 30.0) == true) { + check(caps.videoCapabilities?.areSizeAndRateSupported(w, h, fps.toDouble()) == true) { "Decoder geometry unsupported: ${w}x$h" } - val f = CallVideoFormat(1, w, h, codec = type.id, level = level, profile = profile) - val key = checkNotNull(CallVideoNative.probeKey(type.id, profile, w, h)) + val f = CallVideoFormat(1, w, h, fps = fps, codec = type.id, level = level, profile = profile) + val key = checkNotNull(CallVideoNative.probeKey(type.id, profile, w, h, if (fps > 30) 60 else 30)) validateKey(key, f) profilePipeline = maxOf(profilePipeline, - verifyDecode(f, List(8) { Packet(1_000_000L + it * 33333, true, key) }, entry.name, true)) + verifyDecode(f, List(8) { Packet(1_000_000L + it * 1_000_000L / fps, true, key) }, entry.name, true)) } profiles = profiles or profile; maximumLevel = maxOf(maximumLevel, level) pipeline = maxOf(pipeline, profilePipeline) @@ -312,11 +324,12 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name } } check(profiles != 0) { "No verified decoder profiles" } - modes += DecoderMode(size, entry.name, profiles, maximumLevel, pipeline) - LogManager.addLog("INFO", "call", "video decoder verified codec=${type.id} name=${entry.name} size=$size profiles=$profiles pipeline=$pipeline") + modes += DecoderMode(size, entry.name, profiles, maximumLevel, pipeline, fps) + LogManager.addLog("INFO", "call", "video decoder verified codec=${type.id} name=${entry.name} size=$size fps=$fps profiles=$profiles pipeline=$pipeline") break } catch (e: Exception) { LogManager.addLog("WARN", "call", "video decoder probe rejected ${entry.name} size=$size: ${e.message}") } } + } } if (modes.isNotEmpty()) result[type.id] = Decoder(modes) } @@ -331,7 +344,7 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name var sent = 0; var decoded = 0; var pipeline = 0 val info = MediaCodec.BufferInfo(); val started = System.nanoTime(); val deadline = started + 2_000_000_000 while (decoded < packets.size && System.nanoTime() < deadline) { - if (sent < packets.size && (!paced || System.nanoTime() - started >= sent * 33_333_000L)) { + if (sent < packets.size && (!paced || System.nanoTime() - started >= sent * 1_000_000_000L / f.fps)) { val i = decoder.dequeueInputBuffer(0) if (i >= 0) { val p = packets[sent++]; val data = if (p.key) type.unpackKey(p.bytes).second else p.bytes diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt index 8d933b6f..3eae3dd6 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt @@ -26,7 +26,7 @@ internal object CallVideoNative { external fun requestKey(id: Long) external fun range(id: Long, minimum: Int, maximum: Int): Boolean external fun inspect(codec: Int, bytes: ByteArray): LongArray? - external fun probeKey(codec: Int, profile: Int, width: Int, height: Int): ByteArray? + external fun probeKey(codec: Int, profile: Int, width: Int, height: Int, fps: Int = 30): ByteArray? external fun supported(local: ByteArray, peer: ByteArray, format: LongArray): Boolean external fun decoded(id: Long, generation: Long, accepted: Boolean) } diff --git a/tools/chatgui-android/jni_bridge/android_call_video_jni.c b/tools/chatgui-android/jni_bridge/android_call_video_jni.c index 93f0b431..154459d6 100644 --- a/tools/chatgui-android/jni_bridge/android_call_video_jni.c +++ b/tools/chatgui-android/jni_bridge/android_call_video_jni.c @@ -182,8 +182,8 @@ JNIEXPORT jlongArray JNICALL Java_com_utun_chat_data_CallVideoNative_inspect(JNI jlongArray out = (*e)->NewLongArray(e, 5); if (out) (*e)->SetLongArrayRegion(e, out, 0, 5, values); return out; } JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_CallVideoNative_probeKey( - JNIEnv *e, jobject o, jint codec, jint profile, jint width, jint height) { - (void)o; uint8_t bytes[8192]; int size = call_video_probe_key(codec, profile, width, height, bytes, sizeof(bytes)); + JNIEnv *e, jobject o, jint codec, jint profile, jint width, jint height, jint fps) { + (void)o; uint8_t bytes[8192]; int size = call_video_probe_key(codec, profile, width, height, fps, bytes, sizeof(bytes)); if (size < 0) return NULL; jbyteArray out = (*e)->NewByteArray(e, size); if (out) (*e)->SetByteArrayRegion(e, out, 0, size, (jbyte *)bytes); return out; }