diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt index cb2f9f2c..b33fdabf 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt @@ -181,12 +181,28 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name else -> error("Unsupported HEVC level $level") } data class Encoder(val name: String, val profile: Int, val level: Int) - data class Decoder(val name: String, val profiles: Int) + data class Decoder(val name: String, val profiles: Int, val pipeline: Int = 0) + /** Аппаратные реализации проверяются первыми; software остаётся резервом после реальной пробы. */ + private fun candidates(encoder: Boolean) = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos + .filter { it.isEncoder == encoder }.sortedBy { + if (Build.VERSION.SDK_INT >= 29) { if (it.isHardwareAccelerated) 0 else 1 } + else if (it.name.startsWith("OMX.google.") || it.name.startsWith("c2.android.")) 1 else 0 + } + /** Тот же opt-in low latency, что у viewer стримов; применяется и в пробе, и в live. */ + fun decoderFormat(f: CallVideoFormat, decoder: MediaCodec, config: ByteArray): MediaFormat { + val type = StreamVideoCodec.fromId(f.codec) + val media = type.decoderFormat(f.width, f.height, f.rotation, config) + val lowLatency = Build.VERSION.SDK_INT >= 30 && decoder.codecInfo.getCapabilitiesForType(type.mime) + .isFeatureSupported(MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency) + if (lowLatency) media.setInteger(MediaFormat.KEY_LOW_LATENCY, 1) + LogManager.addLog("INFO", "call", "video decoder configure name=${decoder.name} lowLatency=$lowLatency format=$f") + return media + } /** Encode и decode независимы: ошибка собственного encoder не запрещает приём. */ fun probe(): Map { val result = mutableMapOf() for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { - for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { it.isEncoder }) { + for (entry in candidates(true)) { if (entry.supportedTypes.none { it == type.mime }) continue try { val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93, @@ -207,36 +223,38 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name fun probeDecoders(): Map { val result = mutableMapOf() for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { - for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { !it.isEncoder }) { + for (entry in candidates(false)) { if (entry.supportedTypes.none { it == type.mime }) continue try { val caps = entry.getCapabilitiesForType(type.mime) if (caps.videoCapabilities?.areSizeAndRateSupported(1280, 720, 30.0) != true) continue - var profiles = 0 + var profiles = 0; var pipeline = 0 for (profile in if (type.id == 1) listOf(1, 2) else listOf(1)) { val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93, profile = profile) val key = checkNotNull(CallVideoNative.probeKey(type.id, profile)) validateKey(key, f) - verifyDecode(f, List(4) { Packet(1_000_000L + it * 33333, true, key) }, entry.name) + pipeline = maxOf(pipeline, verifyDecode(f, List(8) { Packet(1_000_000L + it * 33333, true, key) }, + entry.name, true)) profiles = profiles or profile } - result[type.id] = Decoder(entry.name, profiles) - LogManager.addLog("INFO", "call", "video decoder verified codec=${type.id} name=${entry.name} profiles=$profiles") + result[type.id] = Decoder(entry.name, profiles, pipeline) + LogManager.addLog("INFO", "call", "video decoder verified codec=${type.id} name=${entry.name} profiles=$profiles pipeline=$pipeline") break } catch (e: Exception) { LogManager.addLog("WARN", "call", "video decoder probe rejected ${entry.name}: ${e.message}") } } } return result } - internal fun verifyDecode(f: CallVideoFormat, packets: List, name: String) { + internal fun verifyDecode(f: CallVideoFormat, packets: List, name: String, paced: Boolean = false): Int { val type = StreamVideoCodec.fromId(f.codec) val config = type.unpackKey(packets.first().bytes).first val decoder = MediaCodec.createByCodecName(name) try { - decoder.configure(type.decoderFormat(f.width, f.height, 0, config), null, null, 0); decoder.start() - var sent = 0; var decoded = 0; val info = MediaCodec.BufferInfo(); val deadline = System.nanoTime() + 2_000_000_000 + decoder.configure(decoderFormat(f.copy(rotation = 0), decoder, config), null, null, 0); decoder.start() + var sent = 0; var decoded = 0; var pipeline = 0 + val info = MediaCodec.BufferInfo(); val started = System.nanoTime(); val deadline = started + 2_000_000_000 while (decoded < packets.size && System.nanoTime() < deadline) { - if (sent < packets.size) { + if (sent < packets.size && (!paced || System.nanoTime() - started >= sent * 33_333_000L)) { val i = decoder.dequeueInputBuffer(0) if (i >= 0) { val p = packets[sent++]; val data = if (p.key) type.unpackKey(p.bytes).second else p.bytes @@ -245,6 +263,7 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name } val out = decoder.dequeueOutputBuffer(info, 1000) if (out >= 0) { + if (decoded == 0) pipeline = sent - 1 check(decoded < packets.size && info.presentationTimeUs == packets[decoded].pts) { "Decoder reordered PTS" } val actual = decoder.outputFormat check(actual.getInteger(MediaFormat.KEY_WIDTH) >= f.width && actual.getInteger(MediaFormat.KEY_HEIGHT) >= f.height) @@ -252,7 +271,7 @@ internal class CallVideoCodec(requested: CallVideoFormat, bitrate: Int, val name } } check(decoded == packets.size) { "Decoder did not return all probe frames $decoded/${packets.size}" } - decoder.stop() + decoder.stop(); return pipeline } finally { decoder.release() } } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt index d8e7dd35..12fcc019 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt @@ -101,7 +101,7 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin val first = if (awaitingPts < 0) pictures.peekFirst() else null val localQueued = inputs.size + pictures.size + (if (pending != null) 1 else 0) + (if (awaitingPts >= 0) 1 else 0) val decision = CallVideoNative.playoutStep(playout, first?.timing?.pts ?: -1, first?.timing?.received ?: 0, - now / 1000, first?.timing?.interval ?: 0, localQueued, 1) + now / 1000, first?.timing?.interval ?: 0, localQueued, 1 + (verified[format?.codec]?.pipeline ?: 0)) if (decision > 0) wakeUs = now / 1000 + decision else if (first != null) { val render = decision == 0L @@ -143,7 +143,7 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin val implementation = checkNotNull(verified[next.codec]) { "No verified decoder" } check(implementation.profiles and next.profile != 0) { "Unsupported decoder profile" } decoder = MediaCodec.createByCodecName(implementation.name) - decoder!!.configure(type.decoderFormat(next.width, next.height, next.rotation, config), surface, null, 0) + decoder!!.configure(CallVideoCodec.decoderFormat(next, decoder!!, config), surface, null, 0) decoder!!.start(); format = next; lastProgress = System.nanoTime() LogManager.addLog("INFO", "call", "video decoder started call=$id codec=${implementation.name} format=$next") }