From 902e7bb92135ea190ec9c0573806bf614d868018 Mon Sep 17 00:00:00 2001 From: evgeny Date: Mon, 19 Oct 2026 22:45:38 +0200 Subject: [PATCH] Use explicit HEVC Main tier and diagnose fixture rejection --- src/call/call_video_fullhd_fixtures.h | 8 +-- tests/test_call_video.c | 7 ++- .../java/com/utun/chat/data/CallVideoProbe.kt | 16 ++--- .../utun/chat/data/CallVideoRotationProbe.kt | 8 ++- .../java/com/utun/chat/data/CallVideoCodec.kt | 63 ++++++++++++------- .../com/utun/chat/data/CallVideoEngine.kt | 16 +++-- .../com/utun/chat/data/CallVideoNative.kt | 2 +- .../com/utun/chat/data/CallVideoPlayer.kt | 8 ++- .../jni_bridge/android_call_video_jni.c | 4 +- tools/generate_call_video_portrait.py | 2 +- 10 files changed, 82 insertions(+), 52 deletions(-) diff --git a/src/call/call_video_fullhd_fixtures.h b/src/call/call_video_fullhd_fixtures.h index 5da14b5c..3f843ee0 100644 --- a/src/call/call_video_fullhd_fixtures.h +++ b/src/call/call_video_fullhd_fixtures.h @@ -317,8 +317,8 @@ static const uint8_t call_video_fullhd_1920_1080_1[] = { 0xd7, 0x5e, }; static const uint8_t call_video_fullhd_1920_1080_2[] = { - 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x21, 0x60, 0x00, 0x00, 0x03, 0x00, - 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x92, 0x80, 0x90, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x21, + 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, + 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x92, 0x80, 0x90, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xa0, 0x03, 0xc0, 0x80, 0x11, 0x07, 0xcb, 0x96, 0x4a, 0x4a, 0x4c, 0x2f, 0x01, 0x68, 0x08, 0x00, 0x00, 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x40, 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc0, 0x73, 0xc0, 0x89, 0x00, 0x00, 0x00, 0x01, 0x28, 0x01, 0xad, 0xe0, 0xd1, 0x17, @@ -660,8 +660,8 @@ static const uint8_t call_video_fullhd_1080_1920_1[] = { 0x75, 0xd7, 0x5d, 0x75, 0xd7, 0x5d, 0x75, 0xd7, 0x5e, }; static const uint8_t call_video_fullhd_1080_1920_2[] = { - 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x21, 0x60, 0x00, 0x00, 0x03, 0x00, - 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x92, 0x80, 0x90, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x21, + 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, + 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x92, 0x80, 0x90, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xa0, 0x02, 0x20, 0x80, 0x1e, 0x07, 0x2f, 0x96, 0x4a, 0x4a, 0x4c, 0x2f, 0x01, 0x68, 0x08, 0x00, 0x00, 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x40, 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc0, 0x73, 0xc0, 0x89, 0x00, 0x00, 0x00, 0x01, 0x28, 0x01, 0xad, 0xe0, 0xd1, 0x17, diff --git a/tests/test_call_video.c b/tests/test_call_video.c index 2eae456e..7f72e11c 100644 --- a/tests/test_call_video.c +++ b/tests/test_call_video.c @@ -163,7 +163,10 @@ static void bitstreams(void) { for (int fullhd = 0; fullhd <= 1; ++fullhd) for (int portrait = 0; portrait <= 1; portrait++) { unsigned w = fullhd ? 1920 : 1280, h = fullhd ? 1080 : 720; int size = call_video_probe_key(codec, profile, portrait ? h : w, portrait ? w : h, bytes, sizeof(bytes)); CHECK(size > 0); - CHECK(!call_video_inspect_key(codec, bytes, size, &s)); + int inspected = call_video_inspect_key(codec, bytes, size, &s); + if (inspected) DEBUG_ERROR(DEBUG_CATEGORY_CALL, "fixture rejected codec=%d profile=%d size=%ux%u bytes=%d", + codec, profile, portrait ? h : w, portrait ? w : h, size); + CHECK(!inspected); CHECK(s.width == (portrait ? h : w) && s.height == (portrait ? w : h) && s.profile == (unsigned)profile && s.level == (codec == 1 ? (fullhd ? 40u : 31u) : (fullhd ? 120u : 93u))); CHECK(s.dpb >= 3); @@ -232,6 +235,6 @@ static void hevc_pps_flags(void) { puts("video: HEVC PPS flags, independent/dependent IDR segments, output flag and CTB address validation PASS"); } int main(void) { - debug_config_init(); debug_set_level(DEBUG_LEVEL_ERROR); + debug_config_init(); debug_set_level(DEBUG_LEVEL_WARN); capabilities(); portrait_limits(); envelopes(); roundtrip(); rejected(); bitstreams(); hevc_pps_flags(); return 0; } diff --git a/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/CallVideoProbe.kt b/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/CallVideoProbe.kt index 4d2f7be9..dd44c8cd 100644 --- a/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/CallVideoProbe.kt +++ b/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/CallVideoProbe.kt @@ -27,7 +27,7 @@ class CallVideoProbe : Instrumentation() { val actual = checkNotNull(CallVideoNative.inspect(codec, bytes)) val format = CallVideoFormat(1, actual[0].toInt(), actual[1].toInt(), codec = codec, level = actual[2].toInt(), profile = actual[3].toInt()) - val decoder = checkNotNull(decoders[codec]) + val decoder = checkNotNull(decoders[codec]?.modes?.firstOrNull { it.supports(format) }) CallVideoCodec.verifyDecode(format, List(4) { CallVideoCodec.Packet(1_000_000L + it * 33333, true, bytes) }, decoder.name) finish(0, Bundle().apply { putString("remote_decoded", "$name using ${decoder.name} format=$format") }) return @@ -43,13 +43,14 @@ class CallVideoProbe : Instrumentation() { bytes.putShort(size.width.toShort()).putShort(size.height.toShort()).putShort(30) .putInt(size.width * size.height * 30).putInt(4_000_000).put(entry.level.toByte()).put(entry.profile.toByte()) bytes.position(32 + (codec - 1) * 16) - bytes.putShort(1280).putShort(720).putShort(30).putInt(1280 * 720 * 30) - .putInt(4_000_000).put((if (codec == 1) 31 else 93).toByte()).put(decoder.profiles.toByte()) + bytes.putShort(decoder.size.width.toShort()).putShort(decoder.size.height.toShort()).putShort(30) + .putInt(decoder.size.width * decoder.size.height * 30) + .putInt(4_000_000).put(decoder.level.toByte()).put(decoder.profiles.toByte()) val caps = checkNotNull(CallVideoNative.rotatableCaps(bytes.array())) { "Verified CAPS rejected codec=$codec" } for (mode in entry.sizes) for (portrait in listOf(false, true)) { val desired = CallVideoFormat(1, if (portrait) mode.height else mode.width, if (portrait) mode.width else mode.height, codec = codec) - val selected = checkNotNull(CallVideoNative.choose(caps, caps, desired.array())) { "No common mode $desired" } + val selected = CallVideoNative.choose(caps, caps, desired.array()) ?: continue check(CallVideoNative.supported(caps, caps, selected)) { "Chosen mode rejected $desired" } } android.util.Log.i("CallVideoProbe", "verified directed CAPS codec=$codec sizes=${entry.sizes}; both orientations agreed") @@ -61,9 +62,10 @@ class CallVideoProbe : Instrumentation() { } for ((codec, entry) in codecs) { if (selectedCodec != 0 && selectedCodec != codec) continue - val f = CallVideoFormat(1, 640, 480, codec = codec, level = entry.level, profile = entry.profile) - verifyRateChanges(f, entry.name) - CallVideoCodec(f, 1_000_000, entry.name).use { encoder -> + val mode = entry.modes.firstOrNull { it.size.width == 640 && it.size.height == 480 } ?: continue + val f = CallVideoFormat(1, 640, 480, codec = codec, level = mode.level, profile = mode.profile) + verifyRateChanges(f, mode.name) + CallVideoCodec(f, 1_000_000, mode.name).use { encoder -> encoder.prepare(); var key: ByteArray? = null; var input = 0 val deadline = System.nanoTime() + 3_000_000_000 while (key == null && System.nanoTime() < deadline) { diff --git a/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/CallVideoRotationProbe.kt b/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/CallVideoRotationProbe.kt index 84953402..1c347413 100644 --- a/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/CallVideoRotationProbe.kt +++ b/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/CallVideoRotationProbe.kt @@ -20,11 +20,12 @@ internal object CallVideoRotationProbe { for ((codec, entry) in codecs) { if (selected != 0 && selected != codec) continue val decoder = checkNotNull(decoders[codec]) { "No decoder for rotation test codec=$codec" } - for (size in entry.sizes) verify(size, codec, entry, decoder.name) + for (mode in entry.modes) verify(codec, mode, decoder) } } - private fun verify(size: Size, codec: Int, entry: CallVideoCodec.Companion.Encoder, decoder: String) { + private fun verify(codec: Int, entry: CallVideoCodec.Companion.EncoderMode, decoder: CallVideoCodec.Companion.Decoder) { + val size = entry.size ImageReader.newInstance(size.width, size.height, ImageFormat.YUV_420_888, 2).use { reader -> ImageWriter.newInstance(reader.surface, 1).use { writer -> writer.dequeueInputImage().use { source -> fill(source) @@ -52,7 +53,8 @@ internal object CallVideoRotationProbe { Thread.sleep(1) } check(sent == 12 && packets.size == 12 && packets.first().key) { "Incomplete rotated encoder output" } - decode(encoder.format, packets, decoder, quarter) + val implementation = checkNotNull(decoder.modes.firstOrNull { it.supports(encoder.format) }) + decode(encoder.format, packets, implementation.name, quarter) val result = "codec=$codec name=${entry.name} source=$size output=${format.width}x${format.height} " + "rotations=${quarter * 90}/${(quarter + 2) * 90} frames=$sent submitAvgUs=${elapsed / sent} submitMaxUs=$maximum " + "waitAvgUs=${waitUs / sent} copyAvgUs=${copyUs / sent} queueAvgUs=${queueUs / sent}" diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt index 03d641e7..ae4290c3 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt @@ -195,14 +195,29 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name fun platformLevel(codec: Int, level: Int): Int = if (codec == 1) when (level) { 30 -> MediaCodecInfo.CodecProfileLevel.AVCLevel3 31 -> MediaCodecInfo.CodecProfileLevel.AVCLevel31 + 40 -> MediaCodecInfo.CodecProfileLevel.AVCLevel4 else -> error("Unsupported AVC level $level") } else when (level) { 90 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel3 93 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel31 + 120 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel4 else -> error("Unsupported HEVC level $level") } - data class Encoder(val name: String, val profile: Int, val level: Int, val sizes: List) - data class Decoder(val name: String, val profiles: Int, val pipeline: Int = 0) + data class EncoderMode(val size: Size, val name: String, val profile: Int, val level: Int) + data class Encoder(val modes: List) { + val sizes get() = modes.map { it.size } + val profile get() = modes.fold(0) { bits, mode -> bits or mode.profile } + val level get() = modes.maxOf { it.level } + } + data class DecoderMode(val size: Size, val name: String, val profiles: Int, val level: Int, val pipeline: Int) { + fun supports(f: CallVideoFormat) = profiles and f.profile != 0 && f.level <= level && + maxOf(f.width, f.height) <= maxOf(size.width, size.height) && f.width.toLong() * f.height <= size.width.toLong() * size.height + } + data class Decoder(val modes: List) { + val size get() = modes.maxBy { it.size.width * it.size.height }.size + val profiles get() = modes.fold(0) { bits, mode -> bits or mode.profiles } + val level get() = modes.maxOf { it.level } + } /** Аппаратные реализации проверяются первыми; software остаётся резервом после реальной пробы. */ private fun candidates(encoder: Boolean) = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos .filter { it.isEncoder == encoder }.sortedBy { @@ -223,18 +238,19 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name fun probe(): Map { val result = mutableMapOf() for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { + val modes = mutableListOf() + for (size in listOf(Size(640, 480), Size(1280, 720), Size(1920, 1080))) { for (entry in candidates(true)) { if (entry.supportedTypes.none { it == type.mime }) continue try { val caps = entry.getCapabilitiesForType(type.mime) if (caps.encoderCapabilities?.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR) != true) continue - val sizes = mutableListOf(); var profile = 0; var level = 0 - for (size in listOf(Size(640, 480), Size(1280, 720))) { - try { - var foundProfile = profile; var foundLevel = level + var foundProfile = 0; var foundLevel = 0 for ((w, h) in listOf(size.width to size.height, size.height to size.width)) { check(caps.videoCapabilities?.areSizeAndRateSupported(w, h, 30.0) == true) { "Geometry unsupported: ${w}x$h" } - val f = CallVideoFormat(1, w, h, codec = type.id, level = if (type.id == 1) 31 else 93, + val fullhd = size.width > 1280 + val f = CallVideoFormat(1, w, h, codec = type.id, + level = if (type.id == 1) { if (fullhd) 40 else 31 } else { if (fullhd) 120 else 93 }, profile = if (type.id == 1) 2 else 1) CallVideoCodec(f, 1_000_000, entry.name).use { encoder -> encoder.prepare() @@ -242,45 +258,46 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name foundProfile = encoder.format.profile; foundLevel = maxOf(foundLevel, encoder.format.level) } } - profile = foundProfile; level = foundLevel; sizes += size - } catch (e: Exception) { - LogManager.addLog("WARN", "call", "video orientation probe rejected ${entry.name} size=$size: ${e.message}") - } - } - if (sizes.isEmpty()) continue - result[type.id] = Encoder(entry.name, profile, level, sizes) - LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name} sizes=$sizes"); break - } catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name}: ${e.message}") } + modes += EncoderMode(size, entry.name, foundProfile, foundLevel) + LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name} size=$size"); break + } catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name} size=$size: ${e.message}") } } + } + if (modes.isNotEmpty()) result[type.id] = Encoder(modes) } return result } fun probeDecoders(): Map { val result = mutableMapOf() for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { + val modes = mutableListOf() + for (size in listOf(Size(1280, 720), Size(1920, 1080))) { for (entry in candidates(false)) { if (entry.supportedTypes.none { it == type.mime }) continue try { val caps = entry.getCapabilitiesForType(type.mime) - if (caps.videoCapabilities?.areSizeAndRateSupported(1280, 720, 30.0) != true) continue + if (caps.videoCapabilities?.areSizeAndRateSupported(size.width, size.height, 30.0) != true) continue var profiles = 0; var pipeline = 0 + val level = if (type.id == 1) { if (size.width > 1280) 40 else 31 } else { if (size.width > 1280) 120 else 93 } for (profile in if (type.id == 1) listOf(1, 2) else listOf(1)) { for (portrait in listOf(false, true)) { - val w = if (portrait) 720 else 1280; val h = if (portrait) 1280 else 720 + val w = if (portrait) size.height else size.width; val h = if (portrait) size.width else size.height check(caps.videoCapabilities?.areSizeAndRateSupported(w, h, 30.0) == true) { "Decoder geometry unsupported: ${w}x$h" } - val f = CallVideoFormat(1, w, h, codec = type.id, level = if (type.id == 1) 31 else 93, profile = profile) - val key = checkNotNull(CallVideoNative.probeKey(type.id, profile, portrait)) + val f = CallVideoFormat(1, w, h, codec = type.id, level = level, profile = profile) + val key = checkNotNull(CallVideoNative.probeKey(type.id, profile, w, h)) validateKey(key, f) pipeline = maxOf(pipeline, verifyDecode(f, List(8) { Packet(1_000_000L + it * 33333, true, key) }, entry.name, true)) } profiles = profiles or profile } - result[type.id] = Decoder(entry.name, profiles, pipeline) - LogManager.addLog("INFO", "call", "video decoder verified codec=${type.id} name=${entry.name} profiles=$profiles pipeline=$pipeline") + modes += DecoderMode(size, entry.name, profiles, level, pipeline) + LogManager.addLog("INFO", "call", "video decoder verified codec=${type.id} name=${entry.name} size=$size profiles=$profiles pipeline=$pipeline") break - } catch (e: Exception) { LogManager.addLog("WARN", "call", "video decoder probe rejected ${entry.name}: ${e.message}") } + } catch (e: Exception) { LogManager.addLog("WARN", "call", "video decoder probe rejected ${entry.name} size=$size: ${e.message}") } } + } + if (modes.isNotEmpty()) result[type.id] = Decoder(modes) } return result } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt index c53477c1..468bcb59 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt @@ -137,8 +137,13 @@ class CallVideoEngine(private val context: Context, private val id: Long) { if (stopped) return@post val cameras = CameraLensManager.enumerate(manager).mapNotNull { lens -> val c = manager.getCameraCharacteristics(lens.cameraId) - val sizes = c.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)?.getOutputSizes(ImageFormat.YUV_420_888) - .orEmpty().filter { it == Size(640, 480) || it == Size(1280, 720) } + val map = c.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) + val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() + val sizes = map?.getOutputSizes(ImageFormat.YUV_420_888).orEmpty().filter { + (it == Size(640, 480) || it == Size(1280, 720) || it == Size(1920, 1080)) && + rates.any { range -> range.upper == 30 } && + map!!.getOutputMinFrameDuration(ImageFormat.YUV_420_888, it) <= 33_366_700L + } val supported = sizes.filter { size -> encoders.values.any { size in it.sizes } } if (supported.isEmpty()) null else CallVideoCamera(lens.cameraId, lens.label, supported, c.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0, @@ -147,9 +152,9 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val bytes = ByteBuffer.wrap(caps).order(ByteOrder.BIG_ENDIAN) repeat(2) { direction -> for (codec in 1..2) { val profile = if (direction == 0) encoders[codec]?.profile else decoders[codec]?.profiles - val level = if (direction == 0) encoders[codec]?.level else if (codec == 1) 31 else 93 + val level = if (direction == 0) encoders[codec]?.level else decoders[codec]?.level if (profile != null && level != null) { - val size = if (direction == 0) encoders.getValue(codec).sizes.maxBy { it.width * it.height } else Size(1280, 720) + val size = if (direction == 0) encoders.getValue(codec).sizes.maxBy { it.width * it.height } else decoders.getValue(codec).size bytes.putShort(size.width.toShort()).putShort(size.height.toShort()).putShort(30).putInt(size.width * size.height * 30) .putInt(4_000_000).put(level.toByte()).put(profile.toByte()) } @@ -257,8 +262,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val height = if (turn % 180 == 0) settings.height else settings.width for (codec in if (settings.codec == 0) listOf(2, 1) else listOf(settings.codec)) { if (codec in excluded) continue - val implementation = encoders[codec] ?: continue - if (Size(settings.width, settings.height) !in implementation.sizes) continue + val implementation = encoders[codec]?.modes?.firstOrNull { it.size == Size(settings.width, settings.height) } ?: continue val desired = CallVideoFormat(++generation, width, height, codec = codec) val selected = CallVideoNative.choose(caps, peer, desired.array()) ?: continue var candidate: CallVideoCodec? = null diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt index 804a54d5..8d933b6f 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoNative.kt @@ -26,7 +26,7 @@ internal object CallVideoNative { external fun requestKey(id: Long) external fun range(id: Long, minimum: Int, maximum: Int): Boolean external fun inspect(codec: Int, bytes: ByteArray): LongArray? - external fun probeKey(codec: Int, profile: Int, portrait: Boolean): ByteArray? + external fun probeKey(codec: Int, profile: Int, width: Int, height: Int): ByteArray? external fun supported(local: ByteArray, peer: ByteArray, format: LongArray): Boolean external fun decoded(id: Long, generation: Long, accepted: Boolean) } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt index 12fcc019..64199848 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt @@ -101,7 +101,8 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin val first = if (awaitingPts < 0) pictures.peekFirst() else null val localQueued = inputs.size + pictures.size + (if (pending != null) 1 else 0) + (if (awaitingPts >= 0) 1 else 0) val decision = CallVideoNative.playoutStep(playout, first?.timing?.pts ?: -1, first?.timing?.received ?: 0, - now / 1000, first?.timing?.interval ?: 0, localQueued, 1 + (verified[format?.codec]?.pipeline ?: 0)) + now / 1000, first?.timing?.interval ?: 0, localQueued, + 1 + (format?.let { f -> verified[f.codec]?.modes?.firstOrNull { it.supports(f) }?.pipeline } ?: 0)) if (decision > 0) wakeUs = now / 1000 + decision else if (first != null) { val render = decision == 0L @@ -140,8 +141,9 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin if (!key) { pending = null; CallVideoNative.requestKey(id); continue } CallVideoCodec.validateKey(data, next) val type = StreamVideoCodec.fromId(next.codec); val config = type.unpackKey(data).first - val implementation = checkNotNull(verified[next.codec]) { "No verified decoder" } - check(implementation.profiles and next.profile != 0) { "Unsupported decoder profile" } + val implementation = checkNotNull(verified[next.codec]?.modes?.firstOrNull { it.supports(next) }) { + "No verified decoder for $next" + } decoder = MediaCodec.createByCodecName(implementation.name) decoder!!.configure(CallVideoCodec.decoderFormat(next, decoder!!, config), surface, null, 0) decoder!!.start(); format = next; lastProgress = System.nanoTime() diff --git a/tools/chatgui-android/jni_bridge/android_call_video_jni.c b/tools/chatgui-android/jni_bridge/android_call_video_jni.c index 1e6f6c25..93f0b431 100644 --- a/tools/chatgui-android/jni_bridge/android_call_video_jni.c +++ b/tools/chatgui-android/jni_bridge/android_call_video_jni.c @@ -182,8 +182,8 @@ JNIEXPORT jlongArray JNICALL Java_com_utun_chat_data_CallVideoNative_inspect(JNI jlongArray out = (*e)->NewLongArray(e, 5); if (out) (*e)->SetLongArrayRegion(e, out, 0, 5, values); return out; } JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_CallVideoNative_probeKey( - JNIEnv *e, jobject o, jint codec, jint profile, jboolean portrait) { - (void)o; uint8_t bytes[8192]; int size = call_video_probe_key(codec, profile, portrait, bytes, sizeof(bytes)); + JNIEnv *e, jobject o, jint codec, jint profile, jint width, jint height) { + (void)o; uint8_t bytes[8192]; int size = call_video_probe_key(codec, profile, width, height, bytes, sizeof(bytes)); if (size < 0) return NULL; jbyteArray out = (*e)->NewByteArray(e, size); if (out) (*e)->SetByteArrayRegion(e, out, 0, size, (jbyte *)bytes); return out; } diff --git a/tools/generate_call_video_portrait.py b/tools/generate_call_video_portrait.py index fdacc04a..e5a99558 100644 --- a/tools/generate_call_video_portrait.py +++ b/tools/generate_call_video_portrait.py @@ -19,7 +19,7 @@ for width, height, codec in ([(w, h, c) for w, h in ((1920, 1080), (1080, 1920)) if codec == 1: command += ["-profile:v", "baseline", "-level:v", level, "-x264-params", "ref=3:slices=1:threads=1"] else: - command += ["-profile:v", "main", "-x265-params", f"level-idc={level}:ref=3:bframes=0:pools=none:frame-threads=1:log-level=error"] + command += ["-profile:v", "main", "-x265-params", f"level-idc={level}:high-tier=0:ref=3:bframes=0:pools=none:frame-threads=1:log-level=error"] stream = subprocess.check_output(command + ["-f", "h264" if codec == 1 else "hevc", "pipe:1"]) config, picture = bytearray(), bytearray() for nal in re.split(b"\x00\x00\x00?\x01", stream):