jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Probe Android call encoders through real YUV encode and decode

Browse Source
This commit is contained in:
evgeny committed 2026-10-14 22:42:16 +02:00
1 parent c986221cb6
commit 5dbc5bcf59
1 file changed
+207
Download Patch File Download Diff File Expand all files Collapse all files
tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt
+207
View File
@@ -0,0 +1,207 @@
package com.utun.chat.data
import android.media.Image
import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaCodecList
import android.media.MediaFormat
import android.os.Build
import android.os.Bundle
import java.nio.ByteBuffer
/** Codec принадлежит одному worker. YUV input отделён от камеры: замена на IDR не меняет capture PTS. */
internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, name: String) : AutoCloseable {
data class Packet(val pts: Long, val key: Boolean, val bytes: ByteArray)
private val type = StreamVideoCodec.fromId(format.codec)
private val codec = MediaCodec.createByCodecName(name)
private var config = byteArrayOf()
private val partial = java.io.ByteArrayOutputStream()
private var partialPts = -1L
private var partialFlags = 0
init {
try {
val media = MediaFormat.createVideoFormat(type.mime, format.width, format.height).apply {
setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible)
setInteger(MediaFormat.KEY_PROFILE, if (format.codec == 1) MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline
else MediaCodecInfo.CodecProfileLevel.HEVCProfileMain)
setInteger(MediaFormat.KEY_LEVEL, platformLevel(format.codec, format.level))
setInteger(MediaFormat.KEY_BIT_RATE, bitrate); setInteger(MediaFormat.KEY_FRAME_RATE, format.fps)
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR)
if (Build.VERSION.SDK_INT >= 29) setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0)
setInteger(MediaFormat.KEY_LATENCY, 0)
}
codec.configure(media, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); codec.start()
LogManager.addLog("INFO", "call", "video encoder open name=$name format=$format bitrate=$bitrate actual=${codec.inputFormat}")
} catch (e: Exception) { codec.release(); throw e }
}
fun submit(image: Image?, pts: Long, key: Boolean): Boolean {
val index = codec.dequeueInputBuffer(0)
if (index < 0) return false
if (key) codec.setParameters(Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0) })
val input = checkNotNull(codec.getInputImage(index)) { "Encoder lacks flexible YUV input" }
check(input.width >= format.width && input.height >= format.height)
for (plane in 0..2) {
val w = format.width shr (if (plane == 0) 0 else 1)
val h = format.height shr (if (plane == 0) 0 else 1)
val dst = input.planes[plane]; val src = image?.planes?.get(plane)
val row = ByteArray(w)
for (y in 0 until h) {
if (src == null) row.fill(if (plane == 0) 64 else 128.toByte())
else {
val buffer = src.buffer.duplicate(); val start = buffer.position() + y * src.rowStride
if (src.pixelStride == 1) { buffer.position(start); buffer.get(row) }
else for (x in 0 until w) row[x] = buffer.get(start + x * src.pixelStride)
}
val buffer = dst.buffer.duplicate(); val start = buffer.position() + y * dst.rowStride
if (dst.pixelStride == 1) { buffer.position(start); buffer.put(row) }
else for (x in 0 until w) buffer.put(start + x * dst.pixelStride, row[x])
}
}
input.close(); codec.queueInputBuffer(index, 0, format.width * format.height * 3 / 2, pts, 0); return true
}
fun drain(): List<Packet> {
val packets = mutableListOf<Packet>(); val info = MediaCodec.BufferInfo()
repeat(16) {
val index = codec.dequeueOutputBuffer(info, 0)
if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
val f = codec.outputFormat
config = listOf("csd-0", "csd-1").mapNotNull { f.getByteBuffer(it)?.duplicate()?.let { b ->
ByteArray(b.remaining()).also(b::get).let(type::normalize)
} }.fold(byteArrayOf()) { a, b -> a + b }
check(config.size <= StreamVideoCodec.MAX_CONFIG)
LogManager.addLog("INFO", "call", "video encoder output format=$f config=${config.size}")
return@repeat
}
if (index < 0) return packets
try {
if (info.size == 0) return@repeat
val b = checkNotNull(codec.getOutputBuffer(index)).duplicate()
b.position(info.offset); b.limit(info.offset + info.size)
val bytes = ByteArray(info.size).also(b::get)
if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
config = type.mergeConfig(config, bytes); return@repeat
}
if (partial.size() == 0) { partialPts = info.presentationTimeUs; partialFlags = 0 }
check(partialPts == info.presentationTimeUs && partial.size() + bytes.size <= H264AnnexB.MAX_FRAME)
partial.write(bytes); partialFlags = partialFlags or info.flags
if (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME != 0) return@repeat
val data = type.normalize(partial.toByteArray()); partial.reset()
val key = partialFlags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0
val output = if (key) type.keyFrame(data, config) else data
if (key) validateKey(output, format)
packets += Packet(info.presentationTimeUs, key, output)
} finally { codec.releaseOutputBuffer(index, false) }
}
return packets
}
override fun close() {
try { codec.stop() } catch (e: Exception) { LogManager.addLog("WARN", "call", "encoder stop: ${e.message}") }
codec.release()
}
companion object {
fun platformLevel(codec: Int, level: Int): Int = if (codec == 1) when (level) {
30 -> MediaCodecInfo.CodecProfileLevel.AVCLevel3
31 -> MediaCodecInfo.CodecProfileLevel.AVCLevel31
else -> error("Unsupported AVC level $level")
} else when (level) {
90 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel3
93 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel31
else -> error("Unsupported HEVC level $level")
}
/** Реальная проба input → KEY → decoder; не объявляем codec только по списку MediaCodecInfo. */
fun probe(): Map<Int, String> {
val result = mutableMapOf<Int, String>()
for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) {
for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { it.isEncoder }) {
if (entry.supportedTypes.none { it == type.mime }) continue
try {
val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93)
val caps = entry.getCapabilitiesForType(type.mime)
if (!caps.videoCapabilities.areSizeAndRateSupported(1280, 720, 30.0)
|| !caps.encoderCapabilities.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR)) continue
CallVideoCodec(f, 1_000_000, entry.name).use { encoder ->
val packets = mutableListOf<Packet>(); var input = 0
val deadline = System.nanoTime() + 2_000_000_000
while (packets.size < 4 && System.nanoTime() < deadline) {
if (input < 8 && encoder.submit(null, 1_000_000L + input * 33333, input == 0)) input++
packets += encoder.drain(); Thread.sleep(2)
}
check(packets.firstOrNull()?.key == true) { "No initial KEY" }
verifyDecode(f, packets)
}
result[type.id] = entry.name
LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name}"); break
} catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name}: ${e.message}") }
}
}
return result
}
private fun verifyDecode(f: CallVideoFormat, packets: List<Packet>) {
val type = StreamVideoCodec.fromId(f.codec)
val config = type.unpackKey(packets.first().bytes).first
val decoder = MediaCodec.createDecoderByType(type.mime)
try {
decoder.configure(type.decoderFormat(f.width, f.height, 0, config), null, null, 0); decoder.start()
var sent = 0; var decoded = 0; val info = MediaCodec.BufferInfo(); val deadline = System.nanoTime() + 2_000_000_000
while (decoded < packets.size && System.nanoTime() < deadline) {
if (sent < packets.size) {
val i = decoder.dequeueInputBuffer(0)
if (i >= 0) {
val p = packets[sent++]; val data = if (p.key) type.unpackKey(p.bytes).second else p.bytes
decoder.getInputBuffer(i)!!.put(data); decoder.queueInputBuffer(i, 0, data.size, p.pts, 0)
}
}
val out = decoder.dequeueOutputBuffer(info, 1000)
if (out >= 0) {
check(decoded < packets.size && info.presentationTimeUs == packets[decoded].pts) { "Decoder reordered PTS" }
val actual = decoder.outputFormat
check(actual.getInteger(MediaFormat.KEY_WIDTH) >= f.width && actual.getInteger(MediaFormat.KEY_HEIGHT) >= f.height)
decoded++; decoder.releaseOutputBuffer(out, false)
}
}
check(decoded == packets.size) { "Decoder did not return all probe frames $decoded/${packets.size}" }
decoder.stop()
} finally { decoder.release() }
}
/** Проверяем профиль и level в SPS, а не принятый configure option. */
fun validateKey(bytes: ByteArray, f: CallVideoFormat) {
val type = StreamVideoCodec.fromId(f.codec); val (config, data) = type.unpackKey(bytes)
val units = H264AnnexB.nals(config + data).map { (_, n) -> n.copyOfRange(if (n[2] == 1.toByte()) 3 else 4, n.size) }
val sps = units.first { if (f.codec == 1) it[0].toInt() and 31 == 7 else (it[0].toInt() ushr 1) and 63 == 33 }
val raw = java.io.ByteArrayOutputStream(); var zeros = 0
for (b in sps.drop(if (f.codec == 1) 1 else 2)) {
if (zeros >= 2 && b == 3.toByte()) { zeros = 0; continue }
raw.write(b.toInt()); zeros = if (b == 0.toByte()) zeros + 1 else 0
}
val bits = Bits(raw.toByteArray())
if (f.codec == 1) {
check(bits.read(8) == 66 && bits.read(8) and 64 != 0) { "AVC is not Constrained Baseline" }
check(bits.read(8) <= f.level); bits.ue(); bits.ue()
when (bits.ue()) { 0 -> bits.ue(); 2 -> Unit; else -> error("Unsupported AVC POC") }
check(bits.ue() <= 1) { "AVC uses more than one reference" }
} else {
bits.read(4); check(bits.read(3) == 0) { "HEVC temporal layers" }; bits.read(1)
bits.read(2); check(bits.read(1) == 0 && bits.read(5) == 1) { "HEVC is not Main/Main tier" }
repeat(10) { bits.read(8) }; check(bits.read(8) <= f.level)
bits.ue(); check(bits.ue() == 1); bits.ue(); bits.ue()
if (bits.read(1) != 0) repeat(4) { bits.ue() }
check(bits.ue() == 0 && bits.ue() == 0); bits.ue(); bits.read(1)
val dpb = bits.ue(); check(bits.ue() == 0) { "HEVC reordered frames" }
check(dpb <= 1) { "HEVC requires more than two decoded pictures" }
}
}
}
private class Bits(val bytes: ByteArray) {
var at = 0
fun read(n: Int): Int { require(n in 0..24 && at + n <= bytes.size * 8); var v = 0
repeat(n) { v = (v shl 1) or ((bytes[at / 8].toInt() ushr (7 - at % 8)) and 1); at++ }; return v }
fun ue(): Int { var n = 0; while (read(1) == 0) { n++; require(n <= 20) }; return (1 shl n) - 1 + read(n) }
}
}