From 5dbc5bcf59e1b41f45913133ee04e23698415676 Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 14 Oct 2026 22:42:16 +0200 Subject: [PATCH] Probe Android call encoders through real YUV encode and decode --- .../java/com/utun/chat/data/CallVideoCodec.kt | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt new file mode 100644 index 00000000..f461cafe --- /dev/null +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt @@ -0,0 +1,207 @@ +package com.utun.chat.data + +import android.media.Image +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaCodecList +import android.media.MediaFormat +import android.os.Build +import android.os.Bundle +import java.nio.ByteBuffer + +/** Codec принадлежит одному worker. YUV input отделён от камеры: замена на IDR не меняет capture PTS. */ +internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, name: String) : AutoCloseable { + data class Packet(val pts: Long, val key: Boolean, val bytes: ByteArray) + private val type = StreamVideoCodec.fromId(format.codec) + private val codec = MediaCodec.createByCodecName(name) + private var config = byteArrayOf() + private val partial = java.io.ByteArrayOutputStream() + private var partialPts = -1L + private var partialFlags = 0 + + init { + try { + val media = MediaFormat.createVideoFormat(type.mime, format.width, format.height).apply { + setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) + setInteger(MediaFormat.KEY_PROFILE, if (format.codec == 1) MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline + else MediaCodecInfo.CodecProfileLevel.HEVCProfileMain) + setInteger(MediaFormat.KEY_LEVEL, platformLevel(format.codec, format.level)) + setInteger(MediaFormat.KEY_BIT_RATE, bitrate); setInteger(MediaFormat.KEY_FRAME_RATE, format.fps) + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) + setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR) + if (Build.VERSION.SDK_INT >= 29) setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0) + setInteger(MediaFormat.KEY_LATENCY, 0) + } + codec.configure(media, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); codec.start() + LogManager.addLog("INFO", "call", "video encoder open name=$name format=$format bitrate=$bitrate actual=${codec.inputFormat}") + } catch (e: Exception) { codec.release(); throw e } + } + + fun submit(image: Image?, pts: Long, key: Boolean): Boolean { + val index = codec.dequeueInputBuffer(0) + if (index < 0) return false + if (key) codec.setParameters(Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0) }) + val input = checkNotNull(codec.getInputImage(index)) { "Encoder lacks flexible YUV input" } + check(input.width >= format.width && input.height >= format.height) + for (plane in 0..2) { + val w = format.width shr (if (plane == 0) 0 else 1) + val h = format.height shr (if (plane == 0) 0 else 1) + val dst = input.planes[plane]; val src = image?.planes?.get(plane) + val row = ByteArray(w) + for (y in 0 until h) { + if (src == null) row.fill(if (plane == 0) 64 else 128.toByte()) + else { + val buffer = src.buffer.duplicate(); val start = buffer.position() + y * src.rowStride + if (src.pixelStride == 1) { buffer.position(start); buffer.get(row) } + else for (x in 0 until w) row[x] = buffer.get(start + x * src.pixelStride) + } + val buffer = dst.buffer.duplicate(); val start = buffer.position() + y * dst.rowStride + if (dst.pixelStride == 1) { buffer.position(start); buffer.put(row) } + else for (x in 0 until w) buffer.put(start + x * dst.pixelStride, row[x]) + } + } + input.close(); codec.queueInputBuffer(index, 0, format.width * format.height * 3 / 2, pts, 0); return true + } + + fun drain(): List { + val packets = mutableListOf(); val info = MediaCodec.BufferInfo() + repeat(16) { + val index = codec.dequeueOutputBuffer(info, 0) + if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + val f = codec.outputFormat + config = listOf("csd-0", "csd-1").mapNotNull { f.getByteBuffer(it)?.duplicate()?.let { b -> + ByteArray(b.remaining()).also(b::get).let(type::normalize) + } }.fold(byteArrayOf()) { a, b -> a + b } + check(config.size <= StreamVideoCodec.MAX_CONFIG) + LogManager.addLog("INFO", "call", "video encoder output format=$f config=${config.size}") + return@repeat + } + if (index < 0) return packets + try { + if (info.size == 0) return@repeat + val b = checkNotNull(codec.getOutputBuffer(index)).duplicate() + b.position(info.offset); b.limit(info.offset + info.size) + val bytes = ByteArray(info.size).also(b::get) + if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { + config = type.mergeConfig(config, bytes); return@repeat + } + if (partial.size() == 0) { partialPts = info.presentationTimeUs; partialFlags = 0 } + check(partialPts == info.presentationTimeUs && partial.size() + bytes.size <= H264AnnexB.MAX_FRAME) + partial.write(bytes); partialFlags = partialFlags or info.flags + if (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME != 0) return@repeat + val data = type.normalize(partial.toByteArray()); partial.reset() + val key = partialFlags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0 + val output = if (key) type.keyFrame(data, config) else data + if (key) validateKey(output, format) + packets += Packet(info.presentationTimeUs, key, output) + } finally { codec.releaseOutputBuffer(index, false) } + } + return packets + } + override fun close() { + try { codec.stop() } catch (e: Exception) { LogManager.addLog("WARN", "call", "encoder stop: ${e.message}") } + codec.release() + } + + companion object { + fun platformLevel(codec: Int, level: Int): Int = if (codec == 1) when (level) { + 30 -> MediaCodecInfo.CodecProfileLevel.AVCLevel3 + 31 -> MediaCodecInfo.CodecProfileLevel.AVCLevel31 + else -> error("Unsupported AVC level $level") + } else when (level) { + 90 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel3 + 93 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel31 + else -> error("Unsupported HEVC level $level") + } + /** Реальная проба input → KEY → decoder; не объявляем codec только по списку MediaCodecInfo. */ + fun probe(): Map { + val result = mutableMapOf() + for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) { + for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { it.isEncoder }) { + if (entry.supportedTypes.none { it == type.mime }) continue + try { + val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93) + val caps = entry.getCapabilitiesForType(type.mime) + if (!caps.videoCapabilities.areSizeAndRateSupported(1280, 720, 30.0) + || !caps.encoderCapabilities.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR)) continue + CallVideoCodec(f, 1_000_000, entry.name).use { encoder -> + val packets = mutableListOf(); var input = 0 + val deadline = System.nanoTime() + 2_000_000_000 + while (packets.size < 4 && System.nanoTime() < deadline) { + if (input < 8 && encoder.submit(null, 1_000_000L + input * 33333, input == 0)) input++ + packets += encoder.drain(); Thread.sleep(2) + } + check(packets.firstOrNull()?.key == true) { "No initial KEY" } + verifyDecode(f, packets) + } + result[type.id] = entry.name + LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name}"); break + } catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name}: ${e.message}") } + } + } + return result + } + + private fun verifyDecode(f: CallVideoFormat, packets: List) { + val type = StreamVideoCodec.fromId(f.codec) + val config = type.unpackKey(packets.first().bytes).first + val decoder = MediaCodec.createDecoderByType(type.mime) + try { + decoder.configure(type.decoderFormat(f.width, f.height, 0, config), null, null, 0); decoder.start() + var sent = 0; var decoded = 0; val info = MediaCodec.BufferInfo(); val deadline = System.nanoTime() + 2_000_000_000 + while (decoded < packets.size && System.nanoTime() < deadline) { + if (sent < packets.size) { + val i = decoder.dequeueInputBuffer(0) + if (i >= 0) { + val p = packets[sent++]; val data = if (p.key) type.unpackKey(p.bytes).second else p.bytes + decoder.getInputBuffer(i)!!.put(data); decoder.queueInputBuffer(i, 0, data.size, p.pts, 0) + } + } + val out = decoder.dequeueOutputBuffer(info, 1000) + if (out >= 0) { + check(decoded < packets.size && info.presentationTimeUs == packets[decoded].pts) { "Decoder reordered PTS" } + val actual = decoder.outputFormat + check(actual.getInteger(MediaFormat.KEY_WIDTH) >= f.width && actual.getInteger(MediaFormat.KEY_HEIGHT) >= f.height) + decoded++; decoder.releaseOutputBuffer(out, false) + } + } + check(decoded == packets.size) { "Decoder did not return all probe frames $decoded/${packets.size}" } + decoder.stop() + } finally { decoder.release() } + } + + /** Проверяем профиль и level в SPS, а не принятый configure option. */ + fun validateKey(bytes: ByteArray, f: CallVideoFormat) { + val type = StreamVideoCodec.fromId(f.codec); val (config, data) = type.unpackKey(bytes) + val units = H264AnnexB.nals(config + data).map { (_, n) -> n.copyOfRange(if (n[2] == 1.toByte()) 3 else 4, n.size) } + val sps = units.first { if (f.codec == 1) it[0].toInt() and 31 == 7 else (it[0].toInt() ushr 1) and 63 == 33 } + val raw = java.io.ByteArrayOutputStream(); var zeros = 0 + for (b in sps.drop(if (f.codec == 1) 1 else 2)) { + if (zeros >= 2 && b == 3.toByte()) { zeros = 0; continue } + raw.write(b.toInt()); zeros = if (b == 0.toByte()) zeros + 1 else 0 + } + val bits = Bits(raw.toByteArray()) + if (f.codec == 1) { + check(bits.read(8) == 66 && bits.read(8) and 64 != 0) { "AVC is not Constrained Baseline" } + check(bits.read(8) <= f.level); bits.ue(); bits.ue() + when (bits.ue()) { 0 -> bits.ue(); 2 -> Unit; else -> error("Unsupported AVC POC") } + check(bits.ue() <= 1) { "AVC uses more than one reference" } + } else { + bits.read(4); check(bits.read(3) == 0) { "HEVC temporal layers" }; bits.read(1) + bits.read(2); check(bits.read(1) == 0 && bits.read(5) == 1) { "HEVC is not Main/Main tier" } + repeat(10) { bits.read(8) }; check(bits.read(8) <= f.level) + bits.ue(); check(bits.ue() == 1); bits.ue(); bits.ue() + if (bits.read(1) != 0) repeat(4) { bits.ue() } + check(bits.ue() == 0 && bits.ue() == 0); bits.ue(); bits.read(1) + val dpb = bits.ue(); check(bits.ue() == 0) { "HEVC reordered frames" } + check(dpb <= 1) { "HEVC requires more than two decoded pictures" } + } + } + } + private class Bits(val bytes: ByteArray) { + var at = 0 + fun read(n: Int): Int { require(n in 0..24 && at + n <= bytes.size * 8); var v = 0 + repeat(n) { v = (v shl 1) or ((bytes[at / 8].toInt() ushr (7 - at % 8)) and 1); at++ }; return v } + fun ue(): Int { var n = 0; while (read(1) == 0) { n++; require(n <= 20) }; return (1 shl n) - 1 + read(n) } + } +}